[flutter_tools] join flutter specific with home cache (#105343)

packages/flutter_tools/lib/src/dart/pub.dart

@@ -35,6 +35,68 @@ const int _kPubExitCodeUnavailable = 69;
 
 typedef MessageFilter = String? Function(String message);
 
+/// globalCachePath is the directory in which the content of the localCachePath will be moved in
+void joinCaches({
+  required FileSystem fileSystem,
+  required Directory globalCacheDirectory,
+  required Directory dependencyDirectory,
+}) {
+  for (final FileSystemEntity entity in dependencyDirectory.listSync()) {
+    final String newPath = fileSystem.path.join(globalCacheDirectory.path, entity.basename);
+    if (entity is File) {
+      if (!fileSystem.file(newPath).existsSync()) {
+        entity.copySync(newPath);
+      }
+    } else if (entity is Directory) {
+      if (!globalCacheDirectory.childDirectory(entity.basename).existsSync()) {
+        final Directory newDirectory = globalCacheDirectory.childDirectory(entity.basename);
+        newDirectory.createSync();
+        joinCaches(
+          fileSystem: fileSystem,
+          globalCacheDirectory: newDirectory,
+          dependencyDirectory: entity,
+        );
+      }
+    }
+  }
+}
+
+Directory createDependencyDirectory(Directory pubGlobalDirectory, String dependencyName) {
+  final Directory newDirectory = pubGlobalDirectory.childDirectory(dependencyName);
+  newDirectory.createSync();
+  return newDirectory;
+}
+
+bool tryDelete(Directory directory, Logger logger) {
+  try {
+    if (directory.existsSync()) {
+      directory.deleteSync(recursive: true);
+    }
+  } on FileSystemException {
+    logger.printWarning('Failed to delete directory at: ${directory.path}');
+    return false;
+  }
+  return true;
+}
+
+/// When local cache (flutter_root/.pub-cache) and global cache (HOME/.pub-cache) are present a
+/// merge needs to be done leaving only the global
+///
+/// Valid pubCache should look like this ./localCachePath/.pub-cache/hosted/pub.dartlang.org
+bool needsToJoinCache({
+  required FileSystem fileSystem,
+  required String localCachePath,
+  required Directory? globalDirectory,
+}) {
+  if (globalDirectory == null) {
+    return false;
+  }
+  final Directory localDirectory = fileSystem.directory(localCachePath);
+
+  return globalDirectory.childDirectory('hosted').childDirectory('pub.dartlang.org').existsSync() &&
+    localDirectory.childDirectory('hosted').childDirectory('pub.dartlang.org').existsSync();
+}
+
 /// Represents Flutter-specific data that is added to the `PUB_ENVIRONMENT`
 /// environment variable and allows understanding the type of requests made to
 /// the package site on Flutter's behalf.
@@ -494,21 +556,88 @@ class _DefaultPub implements Pub {
     return values.join(':');
   }
 
-  String? _getRootPubCacheIfAvailable() {
+  /// There are 3 ways to get the pub cache location
+  ///
+  /// 1) Provide the _kPubCacheEnvironmentKey.
+  /// 2) There is a local cache (in the Flutter SDK) but not a global one (in the user's home directory).
+  /// 3) If both local and global are available then merge the local into global and return the global.
+  String? _getPubCacheIfAvailable() {
     if (_platform.environment.containsKey(_kPubCacheEnvironmentKey)) {
       return _platform.environment[_kPubCacheEnvironmentKey];
     }
 
-    final String cachePath = _fileSystem.path.join(Cache.flutterRoot!, '.pub-cache');
+    final String localCachePath = _fileSystem.path.join(Cache.flutterRoot!, '.pub-cache');
-    if (_fileSystem.directory(cachePath).existsSync()) {
+    final Directory? globalDirectory;
-      _logger.printTrace('Using $cachePath for the pub cache.');
+    if (_platform.isWindows) {
-      return cachePath;
+      globalDirectory = _getWindowsGlobalDirectory;
+    }
+    else {
+      if (_platform.environment['HOME'] == null) {
+        globalDirectory = null;
+      } else {
+        final String homeDirectoryPath = _platform.environment['HOME']!;
+        globalDirectory = _fileSystem.directory(_fileSystem.path.join(homeDirectoryPath, '.pub-cache'));
+      }
     }
 
+    if (needsToJoinCache(
+      fileSystem: _fileSystem,
+      localCachePath: localCachePath,
+      globalDirectory: globalDirectory,
+    )) {
+      final Directory localDirectoryPub = _fileSystem.directory(
+        _fileSystem.path.join(localCachePath, 'hosted', 'pub.dartlang.org')
+      );
+      final Directory globalDirectoryPub = _fileSystem.directory(
+        _fileSystem.path.join(globalDirectory!.path, 'hosted', 'pub.dartlang.org')
+      );
+      for (final FileSystemEntity entity in localDirectoryPub.listSync()) {
+        if (entity is Directory && !globalDirectoryPub.childDirectory(entity.basename).existsSync()){
+          try {
+            final Directory newDirectory = createDependencyDirectory(globalDirectoryPub, entity.basename);
+            joinCaches(
+              fileSystem: _fileSystem,
+              globalCacheDirectory: newDirectory,
+              dependencyDirectory: entity,
+            );
+          } on FileSystemException {
+            if (!tryDelete(globalDirectoryPub.childDirectory(entity.basename), _logger)) {
+              _logger.printWarning('The join of pub-caches failed');
+              _logger.printStatus('Running "dart pub cache repair"');
+              _processManager.runSync(<String>['dart', 'pub', 'cache', 'repair']);
+            }
+          }
+        }
+      }
+      tryDelete(_fileSystem.directory(localCachePath), _logger);
+      return globalDirectory.path;
+    } else if (globalDirectory != null && globalDirectory.existsSync()) {
+      return globalDirectory.path;
+    } else if (_fileSystem.directory(localCachePath).existsSync()) {
+      return localCachePath;
+    }
     // Use pub's default location by returning null.
     return null;
   }
 
+  Directory? get _getWindowsGlobalDirectory {
+    // %LOCALAPPDATA% is preferred as the cache location over %APPDATA%, because the latter is synchronised between
+    // devices when the user roams between them, whereas the former is not.
+    // The default cache dir used to be in %APPDATA%, so to avoid breaking old installs,
+    // we use the old dir in %APPDATA% if it exists. Else, we use the new default location
+    // in %LOCALAPPDATA%.
+    for (final String envVariable in <String>['APPDATA', 'LOCALAPPDATA']) {
+      if (_platform.environment[envVariable] != null) {
+        final String homePath = _platform.environment[envVariable]!;
+        final Directory globalDirectory = _fileSystem.directory(_fileSystem.path.join(homePath, 'Pub', 'Cache'));
+        if (globalDirectory.existsSync()) {
+          return globalDirectory;
+        }
+      }
+    }
+    return null;
+  }
+
   /// The full environment used when running pub.
   ///
   /// [context] provides extra information to package server requests to
@@ -518,7 +647,7 @@ class _DefaultPub implements Pub {
       'FLUTTER_ROOT': flutterRootOverride ?? Cache.flutterRoot!,
       _kPubEnvironmentKey: await _getPubEnvironmentValue(context),
     };
-    final String? pubCache = _getRootPubCacheIfAvailable();
+    final String? pubCache = _getPubCacheIfAvailable();
     if (pubCache != null) {
       environment[_kPubCacheEnvironmentKey] = pubCache;
     }

packages/flutter_tools/test/general.shard/base/pub_join_caches_test.dart

@@ -0,0 +1,97 @@
+// Copyright 2014 The Flutter Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+import 'package:file/file.dart';
+import 'package:file/memory.dart';
+import 'package:flutter_tools/src/base/file_system.dart';
+import 'package:flutter_tools/src/dart/pub.dart';
+
+import '../../src/common.dart';
+
+void main() {
+  testWithoutContext('join two folders', () async {
+    final MemoryFileSystem fileSystem = MemoryFileSystem();
+    final Directory target = fileSystem.currentDirectory.childDirectory('target');
+    final Directory extra = fileSystem.currentDirectory.childDirectory('extra');
+    target.createSync();
+    target.childFile('first.file').createSync();
+    target.childDirectory('dir').createSync();
+
+    extra.createSync();
+    extra.childFile('second.file').writeAsBytesSync(<int>[0]);
+    extra.childDirectory('dir').createSync();
+    extra.childDirectory('dir').childFile('third.file').writeAsBytesSync(<int>[0]);
+    extra.childDirectory('dir_2').createSync();
+    extra.childDirectory('dir_2').childFile('fourth.file').writeAsBytesSync(<int>[0]);
+    extra.childDirectory('dir_3').createSync();
+    extra.childDirectory('dir_3').childFile('fifth.file').writeAsBytesSync(<int>[0]);
+    joinCaches(
+      fileSystem: fileSystem,
+      globalCacheDirectory: target,
+      dependencyDirectory: extra,
+    );
+
+    expect(target.childFile('second.file').existsSync(), true);
+    expect(target.childDirectory('dir').childFile('third.file').existsSync(), false);
+    expect(target.childDirectory('dir_2').childFile('fourth.file').existsSync(), true);
+    expect(target.childDirectory('dir_3').childFile('fifth.file').existsSync(), true);
+    expect(extra.childDirectory('dir').childFile('third.file').existsSync(), true);
+  });
+
+  group('needsToJoinCache()', (){
+    testWithoutContext('make join', () async {
+      final MemoryFileSystem fileSystem = MemoryFileSystem();
+      final Directory local = fileSystem.currentDirectory.childDirectory('local');
+      final Directory global = fileSystem.currentDirectory.childDirectory('global');
+
+      for (final Directory directory in <Directory>[local, global]) {
+        directory.createSync();
+        directory.childDirectory('hosted').createSync();
+        directory.childDirectory('hosted').childDirectory('pub.dartlang.org').createSync();
+      }
+      final bool pass = needsToJoinCache(
+        fileSystem: fileSystem,
+        localCachePath: local.path,
+        globalDirectory: global,
+      );
+      expect(pass, true);
+    });
+
+    testWithoutContext('detects when global pub-cache does not have a pub.dartlang.org dir', () async {
+      final MemoryFileSystem fileSystem = MemoryFileSystem();
+      final Directory local = fileSystem.currentDirectory.childDirectory('local');
+      final Directory global = fileSystem.currentDirectory.childDirectory('global');
+      local.createSync();
+      global.createSync();
+      local.childDirectory('hosted').createSync();
+      local.childDirectory('hosted').childDirectory('pub.dartlang.org').createSync();
+
+      expect(
+        needsToJoinCache(
+          fileSystem: fileSystem,
+          localCachePath: local.path,
+          globalDirectory: global
+        ),
+        false
+      );
+    });
+    testWithoutContext("don't join global directory null", () async {
+      final MemoryFileSystem fileSystem = MemoryFileSystem();
+      final Directory local = fileSystem.currentDirectory.childDirectory('local');
+      const Directory? global = null;
+      local.createSync();
+      local.childDirectory('hosted').createSync();
+      local.childDirectory('hosted').childDirectory('pub.dartlang.org').createSync();
+
+      expect(
+        needsToJoinCache(
+          fileSystem: fileSystem,
+          localCachePath: local.path,
+          globalDirectory: global
+        ),
+        false
+      );
+    });
+  });
+}

packages/flutter_tools/test/general.shard/dart/pub_get_test.dart

@@ -733,13 +733,12 @@ last line of pub output: "err3"
     expect(processManager, hasNoRemainingExpectations);
   });
 
-  testWithoutContext('pub cache in root is used', () async {
+  testWithoutContext('pub cache in flutter root is ignored', () async {
     String? error;
     final FileSystem fileSystem = MemoryFileSystem.test();
-    final Directory pubCache = fileSystem.directory(Cache.flutterRoot).childDirectory('.pub-cache')..createSync();
     final FakeProcessManager processManager = FakeProcessManager.list(<FakeCommand>[
-      FakeCommand(
+      const FakeCommand(
-        command: const <String>[
+        command: <String>[
           'bin/cache/dart-sdk/bin/dart',
           '__deprecated_pub',
           '--verbosity=warning',
@@ -750,7 +749,6 @@ last line of pub output: "err3"
         environment: <String, String>{
           'FLUTTER_ROOT': '',
           'PUB_ENVIRONMENT': 'flutter_cli:flutter_tests',
-          'PUB_CACHE': pubCache.path,
         },
       ),
     ]);
@@ -776,6 +774,77 @@ last line of pub output: "err3"
     });
   });
 
+  testWithoutContext('pub cache local is merge to global', () async {
+    String? error;
+    final FileSystem fileSystem = MemoryFileSystem.test();
+    final Directory local = fileSystem.currentDirectory.childDirectory('.pub-cache');
+    final Directory global = fileSystem.currentDirectory.childDirectory('/global');
+    global.createSync();
+    for (final Directory dir in <Directory>[global.childDirectory('.pub-cache'), local]) {
+      dir.createSync();
+      dir.childDirectory('hosted').createSync();
+      dir.childDirectory('hosted').childDirectory('pub.dartlang.org').createSync();
+    }
+
+    final Directory globalHosted = global.childDirectory('.pub-cache').childDirectory('hosted').childDirectory('pub.dartlang.org');
+    globalHosted.childFile('first.file').createSync();
+    globalHosted.childDirectory('dir').createSync();
+
+    final Directory localHosted = local.childDirectory('hosted').childDirectory('pub.dartlang.org');
+    localHosted.childFile('second.file').writeAsBytesSync(<int>[0]);
+    localHosted.childDirectory('dir').createSync();
+    localHosted.childDirectory('dir').childFile('third.file').writeAsBytesSync(<int>[0]);
+    localHosted.childDirectory('dir_2').createSync();
+    localHosted.childDirectory('dir_2').childFile('fourth.file').writeAsBytesSync(<int>[0]);
+
+    final FakeProcessManager processManager = FakeProcessManager.list(<FakeCommand>[
+      const FakeCommand(
+        command: <String>[
+          'bin/cache/dart-sdk/bin/dart',
+          '__deprecated_pub',
+          '--verbosity=warning',
+          'get',
+          '--no-precompile',
+        ],
+        exitCode: 69,
+        environment: <String, String>{
+          'FLUTTER_ROOT': '',
+          'PUB_CACHE': '/global/.pub-cache',
+          'PUB_ENVIRONMENT': 'flutter_cli:flutter_tests',
+        },
+      ),
+    ]);
+
+    final Platform platform = FakePlatform(
+      environment: <String, String>{'HOME': '/global'}
+    );
+    final Pub pub = Pub(
+      platform: platform,
+      usage: TestUsage(),
+      fileSystem: fileSystem,
+      logger: BufferLogger.test(),
+      processManager: processManager,
+      botDetector: const BotDetectorAlwaysNo(),
+    );
+
+    FakeAsync().run((FakeAsync time) {
+      pub.get(context: PubContext.flutterTests).then((void value) {
+        error = 'test completed unexpectedly';
+      }, onError: (dynamic thrownError) {
+        error = thrownError.toString();
+      });
+      time.elapse(const Duration(milliseconds: 500));
+      expect(error, isNull);
+      expect(processManager, hasNoRemainingExpectations);
+      expect(local.existsSync(), false);
+      expect(globalHosted.childFile('second.file').existsSync(), false);
+      expect(
+          globalHosted.childDirectory('dir').childFile('third.file').existsSync(), false
+      ); // do not copy dependencies that are already downloaded
+      expect(globalHosted.childDirectory('dir_2').childFile('fourth.file').existsSync(), true);
+    });
+  });
+
   testWithoutContext('pub cache in environment is used', () async {
     final FileSystem fileSystem = MemoryFileSystem.test();
     fileSystem.directory('custom/pub-cache/path').createSync(recursive: true);