From 72d6bcc3f7555c5a242f46cb09484f6c12e86f39 Mon Sep 17 00:00:00 2001 From: Greg Spencer Date: Tue, 5 Dec 2017 12:02:14 -0800 Subject: [PATCH] Use .pub-cache from Flutter root, if it exists. (#13248) The purpose of this PR is to make it so that when the user runs 'flutter', if they have a .pub-cache directory in their flutter root, we use that instead of the default location for the pub cache. Otherwise, it should act as before. The eventual goal is to support a pre-populated flutter .zip/.tar.gz file that has everything the developer needs in one bundle. In order for that to actually work, we need to have the pub cache be self-contained, and not in the user's home dir. Another advantage of this is that if you have multiple flutter repos that you're switching between, then the versions in the pub cache will remain static when you switch between them. --- .gitignore | 1 + bin/flutter | 63 ++++++++------- bin/flutter.bat | 4 + dev/bots/docs.sh | 13 +++- dev/bots/test.dart | 7 +- dev/tools/dartdoc.dart | 25 ++++-- packages/flutter_tools/lib/src/dart/pub.dart | 33 +++++++- .../flutter_tools/test/dart/pub_get_test.dart | 78 ++++++++++++++++++- 8 files changed, 181 insertions(+), 43 deletions(-) diff --git a/.gitignore b/.gitignore index 749c86dc6f..75f33e5b02 100644 --- a/.gitignore +++ b/.gitignore @@ -32,6 +32,7 @@ **/doc/api/ .flutter-plugins .packages +.pub-cache/ .pub/ build/ flutter_*.png diff --git a/bin/flutter b/bin/flutter index ee4131c002..587711b2c0 100755 --- a/bin/flutter +++ b/bin/flutter @@ -16,7 +16,7 @@ set -e function follow_links() { cd -P "${1%/*}" - file="$PWD/${1##*/}" + local file="$PWD/${1##*/}" while [ -h "$file" ]; do # On Mac OS, readlink -f doesn't work. cd -P "${file%/*}" @@ -33,6 +33,39 @@ function path_uri() { echo "$1" | sed -E -e "s,^/+,/," } +function upgrade_flutter () { + if hash flock 2>/dev/null; then + flock 3 # ensures that we don't simultaneously update Dart in multiple parallel instances + # some platforms (e.g. Mac) don't have flock or any reliable alternative + fi + + local revision=`(cd "$FLUTTER_ROOT"; git rev-parse HEAD)` + if [ ! -f "$SNAPSHOT_PATH" ] || [ ! -s "$STAMP_PATH" ] || [ `cat "$STAMP_PATH"` != "$revision" ] || [ "$FLUTTER_TOOLS_DIR/pubspec.yaml" -nt "$FLUTTER_TOOLS_DIR/pubspec.lock" ]; then + mkdir -p "$FLUTTER_ROOT/bin/cache" + touch "$FLUTTER_ROOT/bin/cache/.dartignore" + "$FLUTTER_ROOT/bin/internal/update_dart_sdk.sh" + + echo Building flutter tool... + if [ "$TRAVIS" == "true" ] || [ "$BOT" == "true" ] || [ "$CONTINUOUS_INTEGRATION" == "true" ] || [ "$CHROME_HEADLESS" == "1" ] || [ "$APPVEYOR" == "true" ] || [ "$CI" == "true" ]; then + PUB_ENVIRONMENT="$PUB_ENVIRONMENT:flutter_bot" + fi + export PUB_ENVIRONMENT="$PUB_ENVIRONMENT:flutter_install" + + if [ -d "$FLUTTER_ROOT/.pub-cache" ]; then + export PUB_CACHE="${PUB_CACHE:-"$FLUTTER_ROOT/.pub-cache"}" + fi + + while : ; do + cd "$FLUTTER_TOOLS_DIR" + "$PUB" upgrade --verbosity=error --no-packages-dir && break + echo Error: Unable to 'pub upgrade' flutter tool. Retrying in five seconds... + sleep 5 + done + "$DART" --snapshot="$SNAPSHOT_PATH" --packages="$FLUTTER_TOOLS_DIR/.packages" "$SCRIPT_PATH" + echo "$revision" > "$STAMP_PATH" + fi +} + PROG_NAME="$(path_uri "$(follow_links "$BASH_SOURCE")")" BIN_DIR="$(cd "${PROG_NAME%/*}" ; pwd -P)" export FLUTTER_ROOT="$(cd "${BIN_DIR}/.." ; pwd -P)" @@ -70,33 +103,7 @@ FLUTTER_TOOL_ARGS="--assert-initializer $FLUTTER_TOOL_ARGS" # FLUTTER_TOOL_ARGS="--checked $FLUTTER_TOOL_ARGS" # FLUTTER_TOOL_ARGS="$FLUTTER_TOOL_ARGS --observe=65432" -( - if hash flock 2>/dev/null; then - flock 3 # ensures that we don't simultaneously update Dart in multiple parallel instances - # some platforms (e.g. Mac) don't have flock or any reliable alternative - fi - REVISION=`(cd "$FLUTTER_ROOT"; git rev-parse HEAD)` - if [ ! -f "$SNAPSHOT_PATH" ] || [ ! -s "$STAMP_PATH" ] || [ `cat "$STAMP_PATH"` != "$REVISION" ] || [ "$FLUTTER_TOOLS_DIR/pubspec.yaml" -nt "$FLUTTER_TOOLS_DIR/pubspec.lock" ]; then - mkdir -p "$FLUTTER_ROOT/bin/cache" - touch "$FLUTTER_ROOT/bin/cache/.dartignore" - "$FLUTTER_ROOT/bin/internal/update_dart_sdk.sh" - - echo Building flutter tool... - LOCAL_PUB_ENV="$PUB_ENVIRONMENT" - if [ "$TRAVIS" == "true" ] || [ "$BOT" == "true" ] || [ "$CONTINUOUS_INTEGRATION" == "true" ] || [ "$CHROME_HEADLESS" == "1" ] || [ "$APPVEYOR" == "true" ] || [ "$CI" == "true" ]; then - LOCAL_PUB_ENV="$LOCAL_PUB_ENV:flutter_bot" - fi - LOCAL_PUB_ENV="$LOCAL_PUB_ENV:flutter_install" - while : ; do - cd "$FLUTTER_TOOLS_DIR" - PUB_ENVIRONMENT="$LOCAL_PUB_ENV" "$PUB" upgrade --verbosity=error --no-packages-dir && break - echo Error: Unable to 'pub upgrade' flutter tool. Retrying in five seconds... - sleep 5 - done - "$DART" --snapshot="$SNAPSHOT_PATH" --packages="$FLUTTER_TOOLS_DIR/.packages" "$SCRIPT_PATH" - echo $REVISION > "$STAMP_PATH" - fi -) 3< "$PROG_NAME" +(upgrade_flutter) 3< "$PROG_NAME" set +e "$DART" $FLUTTER_TOOL_ARGS "$SNAPSHOT_PATH" "$@" diff --git a/bin/flutter.bat b/bin/flutter.bat index 5318ccf834..ce317cdfec 100644 --- a/bin/flutter.bat +++ b/bin/flutter.bat @@ -24,6 +24,7 @@ SET script_path=%flutter_tools_dir%\bin\flutter_tools.dart SET dart_sdk_path=%cache_dir%\dart-sdk SET dart_stamp_path=%cache_dir%\dart-sdk.stamp SET dart_version_path=%FLUTTER_ROOT%\bin\internal\dart-sdk.version +SET pub_cache_path=%FLUTTER_ROOT%\.pub-cache SET dart=%dart_sdk_path%\bin\dart.exe SET pub=%dart_sdk_path%\bin\pub.bat @@ -106,6 +107,9 @@ GOTO :after_subroutine SET PUB_ENVIRONMENT=%PUB_ENVIRONMENT%:flutter_bot :not_on_bot SET PUB_ENVIRONMENT=%PUB_ENVIRONMENT%:flutter_install + IF "%PUB_CACHE%" == "" ( + IF EXIST "%pub_cache_path%" SET PUB_CACHE=%pub_cache_path% + ) :retry_pub_upgrade CALL "%pub%" upgrade --verbosity=error --no-packages-dir IF "%ERRORLEVEL%" NEQ "0" ( diff --git a/dev/bots/docs.sh b/dev/bots/docs.sh index 4d9aaabce1..fe07298f95 100755 --- a/dev/bots/docs.sh +++ b/dev/bots/docs.sh @@ -3,20 +3,29 @@ set -e # If you want to run this script locally, make sure you run it from # the root of the flutter repository. +export FLUTTER_ROOT="$PWD" # This is called from travis_upload.sh on Travis. # Make sure dart is installed bin/flutter --version +# If the pub cache directory exists in the root, then use that. +FLUTTER_PUB_CACHE="$FLUTTER_ROOT/.pub-cache" +if [ -d "$FLUTTER_PUB_CACHE" ]; then + # This has to be exported, because pub interprets setting it + # to the empty string in the same way as setting it to ".". + export PUB_CACHE="${PUB_CACHE:-"$FLUTTER_PUB_CACHE"}" +fi + # Install dartdoc. bin/cache/dart-sdk/bin/pub global activate dartdoc 0.14.1 # This script generates a unified doc set, and creates # a custom index.html, placing everything into dev/docs/doc. (cd dev/tools; ../../bin/cache/dart-sdk/bin/pub get) -FLUTTER_ROOT=$PWD bin/cache/dart-sdk/bin/dart dev/tools/dartdoc.dart -FLUTTER_ROOT=$PWD bin/cache/dart-sdk/bin/dart dev/tools/java_and_objc_doc.dart +bin/cache/dart-sdk/bin/dart dev/tools/dartdoc.dart +bin/cache/dart-sdk/bin/dart dev/tools/java_and_objc_doc.dart # Ensure google webmaster tools can verify our site. cp dev/docs/google2ed1af765c529f57.html dev/docs/doc diff --git a/dev/bots/test.dart b/dev/bots/test.dart index 05135fb783..dbfc795edc 100644 --- a/dev/bots/test.dart +++ b/dev/bots/test.dart @@ -14,6 +14,7 @@ final String flutterRoot = path.dirname(path.dirname(path.dirname(path.fromUri(P final String flutter = path.join(flutterRoot, 'bin', Platform.isWindows ? 'flutter.bat' : 'flutter'); final String dart = path.join(flutterRoot, 'bin', 'cache', 'dart-sdk', 'bin', Platform.isWindows ? 'dart.exe' : 'dart'); final String pub = path.join(flutterRoot, 'bin', 'cache', 'dart-sdk', 'bin', Platform.isWindows ? 'pub.bat' : 'pub'); +final String pubCache = path.join(flutterRoot, '.pub-cache'); final String flutterTestArgs = Platform.environment['FLUTTER_TEST_ARGS']; final bool hasColor = stdout.supportsAnsiEscapes; @@ -204,8 +205,12 @@ Future _pubRunTest( final List args = ['run', 'test', '-j1', '-rexpanded']; if (testPath != null) args.add(testPath); + final Map pubEnvironment = {'DART_VM_OPTIONS': '--assert-initializer'}; + if (new Directory(pubCache).existsSync()) { + pubEnvironment['PUB_CACHE'] = pubCache; + } return _runCommand(pub, args, workingDirectory: workingDirectory, - environment: {'DART_VM_OPTIONS': '--assert-initializer'}); + environment: pubEnvironment); } class EvalResult { diff --git a/dev/tools/dartdoc.dart b/dev/tools/dartdoc.dart index 4bb53d0559..36816c794c 100644 --- a/dev/tools/dartdoc.dart +++ b/dev/tools/dartdoc.dart @@ -65,14 +65,25 @@ dependencies: } new File('dev/docs/lib/temp_doc.dart').writeAsStringSync(contents.toString()); + final String flutterRoot = Directory.current.path; + final Map pubEnvironment = { + 'FLUTTER_ROOT': flutterRoot, + }; + + // If there's a .pub-cache dir in the flutter root, use that. + final String pubCachePath = '$flutterRoot/.pub-cache'; + if (new Directory(pubCachePath).existsSync()) { + pubEnvironment['PUB_CACHE'] = pubCachePath; + } + + final String pubExecutable = '$flutterRoot/bin/cache/dart-sdk/bin/pub'; + // Run pub. Process process = await Process.start( - '../../bin/cache/dart-sdk/bin/pub', + pubExecutable, ['get'], workingDirectory: 'dev/docs', - environment: { - 'FLUTTER_ROOT': Directory.current.path, - }, + environment: pubEnvironment, ); printStream(process.stdout, prefix: 'pub:stdout: '); printStream(process.stderr, prefix: 'pub:stderr: '); @@ -84,9 +95,10 @@ dependencies: // Verify which version of dartdoc we're using. final ProcessResult result = Process.runSync( - '../../bin/cache/dart-sdk/bin/pub', + pubExecutable, ['global', 'run', 'dartdoc', '--version'], workingDirectory: 'dev/docs', + environment: pubEnvironment, ); print('\n${result.stdout}'); @@ -113,9 +125,10 @@ dependencies: } process = await Process.start( - '../../bin/cache/dart-sdk/bin/pub', + pubExecutable, args, workingDirectory: 'dev/docs', + environment: pubEnvironment, ); printStream(process.stdout, prefix: 'dartdoc:stdout: ', filter: kVerbose ? const [] : [ diff --git a/packages/flutter_tools/lib/src/dart/pub.dart b/packages/flutter_tools/lib/src/dart/pub.dart index b1376b5884..2ec4dc3f3f 100644 --- a/packages/flutter_tools/lib/src/dart/pub.dart +++ b/packages/flutter_tools/lib/src/dart/pub.dart @@ -148,16 +148,26 @@ List _pubCommand(List arguments) { /// /// [context] provides extra information to package server requests to /// understand usage. It must match the regular expression `[a-z][a-z_]*[a-z]`. -Map _createPubEnvironment(String context) => { - 'FLUTTER_ROOT': Cache.flutterRoot, - _pubEnvironmentKey: _getPubEnvironmentValue(context), -}; +Map _createPubEnvironment(String context) { + final Map environment = { + 'FLUTTER_ROOT': Cache.flutterRoot, + _pubEnvironmentKey: _getPubEnvironmentValue(context), + }; + final String pubCache = _getRootPubCacheIfAvailable(); + if (pubCache != null) { + environment[_pubCacheEnvironmentKey] = pubCache; + } + return environment; +} final RegExp _analyzerWarning = new RegExp(r'^! \w+ [^ ]+ from path \.\./\.\./bin/cache/dart-sdk/lib/\w+$'); /// The console environment key used by the pub tool. const String _pubEnvironmentKey = 'PUB_ENVIRONMENT'; +/// The console environment key used by the pub tool to find the cache directory. +const String _pubCacheEnvironmentKey = 'PUB_CACHE'; + final RegExp _validContext = new RegExp('[a-z][a-z_]*[a-z]'); /// Returns the environment value that should be used when running pub. @@ -189,6 +199,21 @@ String _getPubEnvironmentValue(String pubContext) { return values.join(':'); } +String _getRootPubCacheIfAvailable() { + if (platform.environment.containsKey(_pubCacheEnvironmentKey)) { + return platform.environment[_pubCacheEnvironmentKey]; + } + + final String cachePath = fs.path.join(Cache.flutterRoot, '.pub-cache'); + if (fs.directory(cachePath).existsSync()) { + printTrace('Using $cachePath for the pub cache.'); + return cachePath; + } + + // Use pub's default location by returning null. + return null; +} + String _filterOverrideWarnings(String message) { // This function filters out these three messages: // Warning: You are using these overridden dependencies: diff --git a/packages/flutter_tools/test/dart/pub_get_test.dart b/packages/flutter_tools/test/dart/pub_get_test.dart index 7756ab4210..6cbbb47251 100644 --- a/packages/flutter_tools/test/dart/pub_get_test.dart +++ b/packages/flutter_tools/test/dart/pub_get_test.dart @@ -8,7 +8,9 @@ import 'package:file/file.dart'; import 'package:file/memory.dart'; import 'package:flutter_tools/src/base/context.dart'; import 'package:flutter_tools/src/base/io.dart'; +import 'package:flutter_tools/src/base/platform.dart'; import 'package:flutter_tools/src/dart/pub.dart'; + import 'package:mockito/mockito.dart'; import 'package:process/process.dart'; import 'package:quiver/testing/async.dart'; @@ -26,8 +28,8 @@ void main() { expect(processMock.lastPubEnvironmment, isNull); pubGet(context: 'flutter_tests', checkLastModified: false).then((Null value) { error = 'test completed unexpectedly'; - }, onError: (dynamic error) { - error = 'test failed unexpectedly'; + }, onError: (dynamic thrownError) { + error = 'test failed unexpectedly: $thrownError'; }); expect(testLogger.statusText, ''); time.elapse(const Duration(milliseconds: 500)); @@ -36,6 +38,7 @@ void main() { 'pub get failed (69) -- attempting retry 1 in 1 second...\n' ); expect(processMock.lastPubEnvironmment, contains('flutter_cli:ctx_flutter_tests')); + expect(processMock.lastPubCache, isNull); time.elapse(const Duration(milliseconds: 500)); expect(testLogger.statusText, 'Running "flutter packages get" in /...\n' @@ -80,6 +83,55 @@ void main() { ProcessManager: () => new MockProcessManager(69), FileSystem: () => new MockFileSystem(), }); + + testUsingContext('pub cache in root is used', () async { + String error; + + final MockProcessManager processMock = context.getVariable(ProcessManager); + + new FakeAsync().run((FakeAsync time) { + MockDirectory.findCache = true; + expect(processMock.lastPubEnvironmment, isNull); + expect(processMock.lastPubCache, isNull); + pubGet(context: 'flutter_tests', checkLastModified: false).then((Null value) { + error = 'test completed unexpectedly'; + }, onError: (dynamic thrownError) { + error = 'test failed unexpectedly: $thrownError'; + }); + time.elapse(const Duration(milliseconds: 500)); + expect(processMock.lastPubCache, endsWith('flutter/.pub-cache')); + expect(error, isNull); + }); + }, overrides: { + ProcessManager: () => new MockProcessManager(69), + FileSystem: () => new MockFileSystem(), + }); + + testUsingContext('pub cache in environment is used', () async { + String error; + + final MockProcessManager processMock = context.getVariable(ProcessManager); + + new FakeAsync().run((FakeAsync time) { + MockDirectory.findCache = false; + expect(processMock.lastPubEnvironmment, isNull); + expect(processMock.lastPubCache, isNull); + pubGet(context: 'flutter_tests', checkLastModified: false).then((Null value) { + error = 'test completed unexpectedly'; + }, onError: (dynamic thrownError) { + error = 'test failed unexpectedly: $thrownError'; + }); + time.elapse(const Duration(milliseconds: 500)); + expect(processMock.lastPubCache, equals('path/to/pub-cache')); + expect(error, isNull); + }); + }, overrides: { + ProcessManager: () => new MockProcessManager(69), + FileSystem: () => new MockFileSystem(), + Platform: () => new FakePlatform( + environment: {'PUB_CACHE': 'path/to/pub-cache'}, + ), + }); } typedef void StartCallback(List command); @@ -90,6 +142,7 @@ class MockProcessManager implements ProcessManager { final int fakeExitCode; String lastPubEnvironmment; + String lastPubCache; @override Future start( @@ -101,6 +154,7 @@ class MockProcessManager implements ProcessManager { ProcessStartMode mode: ProcessStartMode.NORMAL, }) { lastPubEnvironmment = environment['PUB_ENVIRONMENT']; + lastPubCache = environment['PUB_CACHE']; return new Future.value(new MockProcess(fakeExitCode)); } @@ -159,6 +213,11 @@ class MockFileSystem extends MemoryFileSystem { File file(dynamic path) { return new MockFile(); } + + @override + Directory directory(dynamic path) { + return new MockDirectory(path); + } } class MockFile implements File { @@ -177,4 +236,19 @@ class MockFile implements File { dynamic noSuchMethod(Invocation invocation) => null; } +class MockDirectory implements Directory { + static bool findCache = false; + + MockDirectory(this.path); + + @override + final String path; + + @override + bool existsSync() => findCache && path.endsWith('flutter/.pub-cache'); + + @override + dynamic noSuchMethod(Invocation invocation) => null; +} + class MockRandomAccessFile extends Mock implements RandomAccessFile {}