Revert "Clean up test infrastructure (#41880)" (#42982)

This reverts commit 1781d5c9bbb4a1b408e40bd40e433c3541bb68fc.

dev/README.md

@@ -2,6 +2,3 @@ This directory contains tools and resources that the Flutter team uses
 during development of the framework. The tools in this directory
 should not be necessary for developing Flutter applications, though of
 course they may be interesting if you are curious.
-
-The tests in this directory are run in the `framework_tests_misc-*`
-shards.

dev/bots/cirrus_setup.sh

@@ -1,19 +1,31 @@
 #!/bin/bash
 set -e
 
-# This script is only meant to be run by the Cirrus CI system, not locally.
-# It must be run from the root of the Flutter repo.
-
 function error() {
   echo "$@" 1>&2
 }
 
+# This script is only meant to be run by the Cirrus CI system, not locally.
+# It must be run from the root of the Flutter repo.
+
 function accept_android_licenses() {
   yes "y" | flutter doctor --android-licenses > /dev/null 2>&1
 }
 
 echo "Flutter SDK directory is: $PWD"
 
+# Run flutter to download dependencies and precompile things, and to disable
+# analytics on the bots.
+echo "Downloading build dependencies and pre-compiling Flutter snapshot"
+./bin/flutter config --no-analytics
+
+# Run doctor, to print it to the log for debugging purposes.
+./bin/flutter doctor -v
+
 # Accept licenses.
 echo "Accepting Android licenses."
 accept_android_licenses || (error "Accepting Android licenses failed." && false)
+
+# Run pub get in all the repo packages.
+echo "Updating packages for Flutter."
+./bin/flutter update-packages

dev/bots/deploy_gallery.sh

@@ -24,90 +24,88 @@ set -x
 
 cd "$FLUTTER_ROOT"
 
-version="$(<version)"
+if [[ "$SHARD" = "deploy_gallery" ]]; then
-if [[ "$OS" == "linux" ]]; then
+  version="$(<version)"
-  echo "Building Flutter Gallery $version for Android..."
+  if [[ "$OS" == "linux" ]]; then
-  # ANDROID_SDK_ROOT must be set in the env.
+    echo "Building Flutter Gallery $version for Android..."
-  (
+
-    cd examples/flutter_gallery
+    # ANDROID_SDK_ROOT must be set in the env.
-    flutter build apk --release -t lib/main_publish.dart
-  )
-  echo "Android Flutter Gallery built"
-  if [[ -z "$CIRRUS_PR" && "$CIRRUS_BRANCH" == "dev" && "$version" != *"pre"* ]]; then
-    echo "Deploying Flutter Gallery $version to Play Store..."
-    set +x # Don't echo back the below.
-    if [ -n "$ANDROID_GALLERY_UPLOAD_KEY" ]; then
-      echo "$ANDROID_GALLERY_UPLOAD_KEY" | base64 --decode > /root/.android/debug.keystore
-    fi
-    set -x
     (
-      cd examples/flutter_gallery/android
+      cd examples/flutter_gallery
-      fastlane deploy_play_store
+      flutter build apk --release -t lib/main_publish.dart
     )
-  else
+    echo "Android Flutter Gallery built"
-    echo "(Not deploying; Flutter Gallery is only deployed to Play store for tagged dev branch commits.)"
+    if [[ -z "$CIRRUS_PR" && "$CIRRUS_BRANCH" == "dev" && "$version" != *"pre"* ]]; then
-  fi
+      echo "Deploying Flutter Gallery $version to Play Store..."
-elif [[ "$OS" == "darwin" ]]; then
+      set +x # Don't echo back the below.
-  echo "Building Flutter Gallery $version for iOS..."
+      if [ -n "$ANDROID_GALLERY_UPLOAD_KEY" ]; then
-  (
+        echo "$ANDROID_GALLERY_UPLOAD_KEY" | base64 --decode > /root/.android/debug.keystore
-    cd examples/flutter_gallery
+      fi
-    flutter build ios --release --no-codesign -t lib/main_publish.dart
+      set -x
-
-    # flutter build ios will run CocoaPods script. Check generated locations.
-    if [[ ! -d "ios/Pods" ]]; then
-      echo "Error: pod install failed to setup plugins"
-      exit 1
-    fi
-
-    if [[ ! -d "ios/.symlinks/plugins" ]]; then
-      echo "Error: pod install failed to setup plugin symlinks"
-      exit 1
-    fi
-
-    if [[ -d "ios/.symlinks/flutter" ]]; then
-      echo "Error: pod install created flutter symlink"
-      exit 1
-    fi
-
-    if [[ ! -d "build/ios/iphoneos/Runner.app/Frameworks/App.framework/flutter_assets" ]]; then
-      echo "Error: flutter_assets not assembled"
-      exit 1
-    fi
-
-    if [[
-      -d "build/ios/iphoneos/Runner.app/Frameworks/App.framework/flutter_assets/isolate_snapshot_data" ||
-      -d "build/ios/iphoneos/Runner.app/Frameworks/App.framework/flutter_assets/kernel_blob.bin" ||
-      -d "build/ios/iphoneos/Runner.app/Frameworks/App.framework/flutter_assets/vm_snapshot_data"
-     ]]; then
-      echo "Error: compiled debug version of app with --release flag"
-      exit 1
-    fi
-  )
-  echo "iOS Flutter Gallery built"
-  if [[ -z "$CIRRUS_PR" ]]; then
-    if [[ "$CIRRUS_BRANCH" == "dev" && "$version" != *"pre"* ]]; then
-      echo "Archiving with distribution profile and deploying to TestFlight..."
       (
-        cd examples/flutter_gallery/ios
+        cd examples/flutter_gallery/android
-        export DELIVER_ITMSTRANSPORTER_ADDITIONAL_UPLOAD_PARAMETERS="-t DAV"
+        fastlane deploy_play_store
-        fastlane build_and_deploy_testflight upload:true
       )
     else
-      # On iOS the signing can break as well, so we verify this regularly (not just
+      echo "Not deployed: Flutter Gallery is only deployed to the Play Store on merged and tagged dev branch commits"
-      # on tagged dev branch commits). We can only do this post-merge, though, because
+    fi
-      # the secrets aren't available on PRs.
+  elif [[ "$OS" == "darwin" ]]; then
-      echo "Testing archiving with distribution profile..."
+    echo "Building Flutter Gallery $version for iOS..."
-      (
+    (
-        cd examples/flutter_gallery/ios
+      cd examples/flutter_gallery
-        fastlane build_and_deploy_testflight
+      flutter build ios --release --no-codesign -t lib/main_publish.dart
-      )
+
-      echo "(Not deploying; Flutter Gallery is only deployed to TestFlight for tagged dev branch commits.)"
+      # flutter build ios will run CocoaPods script. Check generated locations.
+      if [[ ! -d "ios/Pods" ]]; then
+        echo "Error: pod install failed to setup plugins"
+        exit 1
+      fi
+
+      if [[ ! -d "ios/.symlinks/plugins" ]]; then
+        echo "Error: pod install failed to setup plugin symlinks"
+        exit 1
+      fi
+
+      if [[ -d "ios/.symlinks/flutter" ]]; then
+        echo "Error: pod install created flutter symlink"
+        exit 1
+      fi
+
+      if [[ ! -d "build/ios/iphoneos/Runner.app/Frameworks/App.framework/flutter_assets" ]]; then
+        echo "Error: flutter_assets not assembled"
+        exit 1
+      fi
+
+      if [[ 
+        -d "build/ios/iphoneos/Runner.app/Frameworks/App.framework/flutter_assets/isolate_snapshot_data" ||
+        -d "build/ios/iphoneos/Runner.app/Frameworks/App.framework/flutter_assets/kernel_blob.bin" ||
+        -d "build/ios/iphoneos/Runner.app/Frameworks/App.framework/flutter_assets/vm_snapshot_data"
+       ]]; then
+        echo "Error: compiled debug version of app with --release flag"
+        exit 1
+      fi
+    )
+    echo "iOS Flutter Gallery built"
+    if [[ -z "$CIRRUS_PR" ]]; then
+      if [[ "$CIRRUS_BRANCH" == "dev" && "$version" != *"pre"* ]]; then
+        echo "Archiving with distribution profile and deploying to TestFlight..."
+        (
+          cd examples/flutter_gallery/ios
+          export DELIVER_ITMSTRANSPORTER_ADDITIONAL_UPLOAD_PARAMETERS="-t DAV"
+          fastlane build_and_deploy_testflight upload:true
+        )
+      else
+        echo "Archiving with distribution profile..."
+        (
+          cd examples/flutter_gallery/ios
+          fastlane build_and_deploy_testflight
+        )
+        echo "Archive is only deployed to TestFlight on tagged dev branch commits"
+      fi
+    else
+      echo "Not deployed: Flutter Gallery is only deployed to TestFlight on merged and tagged dev branch commits"
     fi
-  else
-    echo "(Not archiving or deploying; Flutter Gallery archiving is only tested post-commit.)"
   fi
 else
-  echo "Unknown OS: $OS"
+  echo "Doing nothing: not on the 'deploy_gallery' SHARD."
-  echo "Aborted."
-  exit 1
 fi

dev/bots/run_command.dart

@@ -95,6 +95,7 @@ Future<void> runCommand(String executable, List<String> arguments, {
   OutputMode outputMode = OutputMode.print,
   CapturedOutput output,
   bool skip = false,
+  bool expectFlaky = false,
   bool Function(String) removeLine,
 }) async {
   assert((outputMode == OutputMode.capture) == (output != null),
@@ -144,6 +145,10 @@ Future<void> runCommand(String executable, List<String> arguments, {
     output.stderr = _flattenToString(await savedStderr);
   }
 
+  // If the test is flaky we don't care about the actual exit.
+  if (expectFlaky)
+    return;
+
   if ((exitCode == 0) == expectNonZeroExit || (expectedExitCode != null && exitCode != expectedExitCode)) {
     if (failureMessage != null) {
       print(failureMessage);

dev/bots/test/bot_test.dart

@@ -1,13 +0,0 @@
-// Copyright 2017 The Chromium Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file.
-
-import 'dart:io';
-
-import 'common.dart';
-
-void main() {
-  test('BOT variable is set on bots', () {
-    expect(Platform.environment['BOT'], 'true');
-  });
-}

dev/bots/test/sdk_directory_has_space_test.dart

@@ -1,21 +0,0 @@
-// Copyright 2017 The Chromium Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file.
-
-import 'dart:io';
-
-import 'package:path/path.dart' as path;
-
-import 'common.dart';
-
-void main() {
-  test('We are in a directory with a space in it', () async {
-    // The Flutter SDK should be in a directory with a space in it, to make sure
-    // our tools support that.
-    final String expectedName = Platform.environment['FLUTTER_SDK_PATH_WITH_SPACE'];
-    expect(expectedName, 'flutter sdk');
-    expect(expectedName, contains(' '));
-    final List<String> parts = path.split(Directory.current.absolute.path);
-    expect(parts.reversed.take(3), <String>['bots', 'dev', expectedName]);
-  });
-}

dev/bots/test/test_test.dart

@@ -29,11 +29,7 @@ void main() {
       ];
       for (String version in valid_versions) {
         when(file.readAsString()).thenAnswer((Invocation invocation) => Future<String>.value(version));
-        expect(
+        expect(await verifyVersion(version, file), isTrue, reason: '$version is invalid');
-          await verifyVersion(file),
-          isNull,
-          reason: '$version is valid but verifyVersionFile said it was bad',
-        );
       }
     });
 
@@ -45,15 +41,10 @@ void main() {
         '1.2.3-pre',
         '1.2.3-pre.1+hotfix.1',
         '  1.2.3',
-        '1.2.3-hotfix.1',
       ];
       for (String version in invalid_versions) {
         when(file.readAsString()).thenAnswer((Invocation invocation) => Future<String>.value(version));
-        expect(
+        expect(await verifyVersion(version, file), isFalse);
-          await verifyVersion(file),
-          'The version logic generated an invalid version string: "$version".',
-          reason: '$version is invalid but verifyVersionFile said it was fine',
-        );
       }
     });
   });

packages/flutter_tools/dart_test.yaml

@@ -1,9 +1,5 @@
-# package:test configuration
+tags:
-# https://github.com/dart-lang/test/blob/master/pkgs/test/doc/configuration.md
+  "no_coverage":
-
+  "create":
-# Some of our tests take an absurdly long time to run, and on some
+  "integration":
-# hosts they can take even longer due to the host suddenly being
+ 
-# overloaded. For this reason, we set the test timeout to
-# significantly more than it would be by default, and we never set the
-# timeouts in the tests themselves.
-timeout: 15m

packages/flutter_tools/test/commands.shard/hermetic/ide_config_test.dart

@@ -135,7 +135,7 @@ void main() {
       return _updateIdeConfig(
         expectedContents: expectedContents,
       );
-    });
+    }, timeout: const Timeout.factor(2.0));
 
     testUsingContext('creates non-existent files', () async {
       final Map<String, String> templateManifest = _getManifest(
@@ -155,7 +155,7 @@ void main() {
       return _updateIdeConfig(
         expectedContents: expectedContents,
       );
-    });
+    }, timeout: const Timeout.factor(2.0));
 
     testUsingContext('overwrites existing files with --overwrite', () async {
       final Map<String, String> templateManifest = _getManifest(
@@ -181,7 +181,7 @@ void main() {
         args: <String>['--overwrite'],
         expectedContents: expectedContents,
       );
-    });
+    }, timeout: const Timeout.factor(2.0));
 
     testUsingContext('only adds new templates without --overwrite', () async {
       final Map<String, String> templateManifest = _getManifest(
@@ -212,7 +212,7 @@ void main() {
         args: <String>['--update-templates'],
         expectedContents: expectedContents,
       );
-    });
+    }, timeout: const Timeout.factor(2.0));
 
     testUsingContext('update all templates with --overwrite', () async {
       final Map<String, String> templateManifest = _getManifest(
@@ -239,7 +239,7 @@ void main() {
         args: <String>['--update-templates', '--overwrite'],
         expectedContents: expectedContents,
       );
-    });
+    }, timeout: const Timeout.factor(2.0));
 
     testUsingContext('removes deleted imls with --overwrite', () async {
       final Map<String, String> templateManifest = _getManifest(
@@ -275,7 +275,7 @@ void main() {
         args: <String>['--update-templates', '--overwrite'],
         expectedContents: expectedContents,
       );
-    });
+    }, timeout: const Timeout.factor(2.0));
 
     testUsingContext('removes deleted imls with --overwrite, including empty parent dirs', () async {
       final Map<String, String> templateManifest = _getManifest(
@@ -316,7 +316,7 @@ void main() {
         args: <String>['--update-templates', '--overwrite'],
         expectedContents: expectedContents,
       );
-    });
+    }, timeout: const Timeout.factor(2.0));
 
   });
 }

packages/flutter_tools/test/commands.shard/permeable/analyze_once_test.dart

@@ -16,6 +16,9 @@ import 'package:flutter_tools/src/runner/flutter_command.dart';
 import '../../src/common.dart';
 import '../../src/context.dart';
 
+/// Test case timeout for tests involving project analysis.
+const Timeout allowForSlowAnalyzeTests = Timeout.factor(5.0);
+
 final Generator _kNoColorTerminalPlatform = () => FakePlatform.fromPlatform(const LocalPlatform())..stdoutSupportsAnsi = false;
 final Map<Type, Generator> noColorTerminalOverride = <Type, Generator>{
   Platform: _kNoColorTerminalPlatform,
@@ -51,7 +54,7 @@ void main() {
         ],
       );
       expect(libMain.existsSync(), isTrue);
-    }, overrides: <Type, Generator>{
+    }, timeout: allowForRemotePubInvocation, overrides: <Type, Generator>{
       Pub: () => const Pub(),
     });
 
@@ -62,7 +65,7 @@ void main() {
         arguments: <String>['analyze'],
         statusTextContains: <String>['No issues found!'],
       );
-    }, overrides: <Type, Generator>{
+    }, timeout: allowForSlowAnalyzeTests, overrides: <Type, Generator>{
       Pub: () => const Pub(),
     });
 
@@ -77,7 +80,6 @@ void main() {
     }, overrides: <Type, Generator>{
       Pub: () => const Pub(),
     });
-
     // Analyze in the current directory - no arguments
     testUsingContext('working directory with errors', () async {
       // Break the code to produce the "The parameter 'onPressed' is required" hint
@@ -108,7 +110,7 @@ void main() {
         exitMessageContains: '2 issues found.',
         toolExit: true,
       );
-    }, overrides: <Type, Generator>{
+    }, timeout: allowForSlowAnalyzeTests, overrides: <Type, Generator>{
       Pub: () => const Pub(),
       ...noColorTerminalOverride,
     });
@@ -138,7 +140,7 @@ void main() {
         exitMessageContains: '3 issues found.',
         toolExit: true,
       );
-    }, overrides: <Type, Generator>{
+    }, timeout: allowForSlowAnalyzeTests, overrides: <Type, Generator>{
       Pub: () => const Pub(),
       ...noColorTerminalOverride
     });

packages/flutter_tools/test/commands.shard/permeable/build_bundle_test.dart

@@ -73,7 +73,7 @@ void main() {
 
     expect(await command.usageValues,
         containsPair(CustomDimensions.commandBuildBundleIsModule, 'true'));
-  });
+  }, timeout: allowForCreateFlutterProject);
 
   testUsingContext('bundle getUsage indicate that project is not a module', () async {
     final String projectPath = await createProject(tempDir,
@@ -83,7 +83,7 @@ void main() {
 
     expect(await command.usageValues,
         containsPair(CustomDimensions.commandBuildBundleIsModule, 'false'));
-  });
+  }, timeout: allowForCreateFlutterProject);
 
   testUsingContext('bundle getUsage indicate the target platform', () async {
     final String projectPath = await createProject(tempDir,
@@ -93,7 +93,7 @@ void main() {
 
     expect(await command.usageValues,
         containsPair(CustomDimensions.commandBuildBundleTargetPlatform, 'android-arm'));
-  });
+  }, timeout: allowForCreateFlutterProject);
 
   testUsingContext('bundle fails to build for Windows if feature is disabled', () async {
     fs.file('lib/main.dart').createSync(recursive: true);

packages/flutter_tools/test/commands.shard/permeable/create_test.dart

@@ -78,7 +78,7 @@ void main() {
       ],
     );
     return _runFlutterTest(projectDir);
-  }, overrides: <Type, Generator>{
+  }, timeout: allowForRemotePubInvocation, overrides: <Type, Generator>{
     Pub: () => const Pub(),
   });
 
@@ -96,7 +96,7 @@ void main() {
         'ios/Runner/GeneratedPluginRegistrant.h',
       ],
     );
-  }, overrides: <Type, Generator>{
+  }, timeout: allowForRemotePubInvocation, overrides: <Type, Generator>{
     Pub: () => const Pub(),
   });
 
@@ -118,9 +118,11 @@ void main() {
       'ios/',
     ]);
     return _runFlutterTest(projectDir);
-  }, overrides: <Type, Generator>{
+  },
-    Pub: () => const Pub(),
+    timeout: allowForRemotePubInvocation, overrides: <Type, Generator>{
-  });
+      Pub: () => const Pub(),
+    },
+  );
 
   testUsingContext('cannot create a project if non-empty non-project directory exists with .metadata', () async {
     await projectDir.absolute.childDirectory('blag').create(recursive: true);
@@ -136,7 +138,7 @@ void main() {
           '.ios/',
         ]),
       throwsToolExit(message: 'Sorry, unable to detect the type of project to recreate'));
-  }, overrides: <Type, Generator>{
+  }, timeout: allowForRemotePubInvocation, overrides: <Type, Generator>{
     Pub: () => const Pub(),
     ...noColorTerminalOverride,
   });
@@ -162,7 +164,7 @@ void main() {
         '.ios/',
       ],
     );
-  }, overrides: <Type, Generator>{
+  }, timeout: allowForRemotePubInvocation, overrides: <Type, Generator>{
     Pub: () => const Pub(),
   });
 
@@ -187,7 +189,7 @@ void main() {
         '.ios/',
       ],
     );
-  }, overrides: <Type, Generator>{
+  }, timeout: allowForRemotePubInvocation, overrides: <Type, Generator>{
     Pub: () => const Pub(),
   });
 
@@ -212,7 +214,7 @@ void main() {
         'lib/flutter_project.dart',
       ],
     );
-  }, overrides: <Type, Generator>{
+  }, timeout: allowForRemotePubInvocation, overrides: <Type, Generator>{
     Pub: () => const Pub(),
   });
 
@@ -243,7 +245,7 @@ void main() {
         'test/widget_test.dart',
       ],
     );
-  }, overrides: <Type, Generator>{
+  }, timeout: allowForRemotePubInvocation, overrides: <Type, Generator>{
     Pub: () => const Pub(),
   });
 
@@ -265,7 +267,7 @@ void main() {
         'ios/Runner/main.m',
       ],
     );
-  }, overrides: <Type, Generator>{
+  }, timeout: allowForCreateFlutterProject, overrides: <Type, Generator>{
     Pub: () => const Pub(),
   });
 
@@ -295,7 +297,7 @@ void main() {
       ],
     );
     return _runFlutterTest(projectDir);
-  }, overrides: <Type, Generator>{
+  }, timeout: allowForRemotePubInvocation, overrides: <Type, Generator>{
     Pub: () => const Pub(),
   });
 
@@ -317,7 +319,7 @@ void main() {
       ],
     );
     return _runFlutterTest(projectDir.childDirectory('example'));
-  }, overrides: <Type, Generator>{
+  }, timeout: allowForRemotePubInvocation, overrides: <Type, Generator>{
     Pub: () => const Pub(),
   });
 
@@ -344,7 +346,7 @@ void main() {
         'example/ios/Runner/main.m',
       ],
     );
-  });
+  }, timeout: allowForCreateFlutterProject);
 
   testUsingContext('plugin project with custom org', () async {
     return _createProject(
@@ -364,7 +366,7 @@ void main() {
         'example/android/app/src/main/java/com/example/flutter_project_example/MainActivity.java',
       ],
     );
-  });
+  }, timeout: allowForCreateFlutterProject);
 
   testUsingContext('plugin project with valid custom project name', () async {
     return _createProject(
@@ -384,7 +386,7 @@ void main() {
         'example/android/app/src/main/java/com/example/flutter_project_example/MainActivity.java',
       ],
     );
-  });
+  }, timeout: allowForCreateFlutterProject);
 
   testUsingContext('plugin project with invalid custom project name', () async {
     expect(
@@ -394,7 +396,7 @@ void main() {
       ),
       throwsToolExit(message: '"xyz.xyz" is not a valid Dart package name.'),
     );
-  });
+  }, timeout: allowForCreateFlutterProject);
 
   testUsingContext('legacy app project with-driver-test', () async {
     return _createAndAnalyzeProject(
@@ -402,7 +404,7 @@ void main() {
       <String>['--with-driver-test', '--template=app'],
       <String>['lib/main.dart'],
     );
-  }, overrides: <Type, Generator>{
+  }, timeout: allowForRemotePubInvocation, overrides: <Type, Generator>{
     Pub: () => const Pub(),
   });
 
@@ -437,7 +439,7 @@ void main() {
       'android/',
       'ios/',
     ]);
-  }, overrides: <Type, Generator>{
+  }, timeout: allowForRemotePubInvocation, overrides: <Type, Generator>{
     Pub: () => const Pub(),
   });
 
@@ -461,7 +463,7 @@ void main() {
     final String actualContents = await fs.file(projectDir.path + '/android/gradle.properties').readAsString();
 
     expect(actualContents.contains('useAndroidX'), true);
-  });
+  }, timeout: allowForCreateFlutterProject);
 
   testUsingContext('non androidx app project', () async {
     Cache.flutterRoot = '../..';
@@ -482,7 +484,7 @@ void main() {
     final String actualContents = await fs.file(projectDir.path + '/android/gradle.properties').readAsString();
 
     expect(actualContents.contains('useAndroidX'), false);
-  });
+  }, timeout: allowForCreateFlutterProject);
 
   testUsingContext('androidx is used by default in a module project', () async {
     Cache.flutterRoot = '../..';
@@ -499,7 +501,7 @@ void main() {
       project.usesAndroidX,
       true,
     );
-  });
+  }, timeout: allowForCreateFlutterProject);
 
   testUsingContext('non androidx module', () async {
     Cache.flutterRoot = '../..';
@@ -516,7 +518,7 @@ void main() {
       project.usesAndroidX,
       false,
     );
-  });
+  }, timeout: allowForCreateFlutterProject);
 
   testUsingContext('androidx is used by default in a plugin project', () async {
     Cache.flutterRoot = '../..';
@@ -537,7 +539,7 @@ void main() {
     final String actualContents = await fs.file(projectDir.path + '/android/gradle.properties').readAsString();
 
     expect(actualContents.contains('useAndroidX'), true);
-  });
+  }, timeout: allowForCreateFlutterProject);
 
   testUsingContext('non androidx plugin project', () async {
     Cache.flutterRoot = '../..';
@@ -558,7 +560,7 @@ void main() {
     final String actualContents = await fs.file(projectDir.path + '/android/gradle.properties').readAsString();
 
     expect(actualContents.contains('useAndroidX'), false);
-  });
+  }, timeout: allowForCreateFlutterProject);
 
   testUsingContext('app supports macOS if requested', () async {
     Cache.flutterRoot = '../..';
@@ -571,7 +573,7 @@ void main() {
     await runner.run(<String>['create', '--no-pub', '--macos', projectDir.path]);
 
     expect(projectDir.childDirectory('macos').childDirectory('Runner.xcworkspace').existsSync(), true);
-  });
+  }, timeout: allowForCreateFlutterProject);
 
   testUsingContext('app does not include macOS by default', () async {
     Cache.flutterRoot = '../..';
@@ -584,7 +586,7 @@ void main() {
     await runner.run(<String>['create', '--no-pub', projectDir.path]);
 
     expect(projectDir.childDirectory('macos').childDirectory('Runner.xcworkspace').existsSync(), false);
-  });
+  }, timeout: allowForCreateFlutterProject);
 
   testUsingContext('plugin supports macOS if requested', () async {
     Cache.flutterRoot = '../..';
@@ -597,7 +599,7 @@ void main() {
     await runner.run(<String>['create', '--no-pub', '--template=plugin', '--macos', projectDir.path]);
 
     expect(projectDir.childDirectory('macos').childFile('flutter_project.podspec').existsSync(), true);
-  });
+  }, timeout: allowForCreateFlutterProject);
 
   testUsingContext('plugin does not include macOS by default', () async {
     Cache.flutterRoot = '../..';
@@ -610,7 +612,7 @@ void main() {
     await runner.run(<String>['create', '--no-pub', '--template=plugin', projectDir.path]);
 
     expect(projectDir.childDirectory('macos').childFile('flutter_project.podspec').existsSync(), false);
-  });
+  }, timeout: allowForCreateFlutterProject);
 
   testUsingContext('has correct content and formatting with module template', () async {
     Cache.flutterRoot = '../..';
@@ -706,7 +708,7 @@ void main() {
   }, overrides: <Type, Generator>{
     FlutterVersion: () => mockFlutterVersion,
     Platform: _kNoColorTerminalPlatform,
-  });
+  }, timeout: allowForCreateFlutterProject);
 
   testUsingContext('has correct content and formatting with app template', () async {
     Cache.flutterRoot = '../..';
@@ -777,7 +779,7 @@ void main() {
   }, overrides: <Type, Generator>{
     FlutterVersion: () => mockFlutterVersion,
     Platform: _kNoColorTerminalPlatform,
-  });
+  }, timeout: allowForCreateFlutterProject);
 
   testUsingContext('has correct application id for android and bundle id for ios', () async {
     Cache.flutterRoot = '../..';
@@ -825,7 +827,7 @@ void main() {
   }, overrides: <Type, Generator>{
     FlutterVersion: () => mockFlutterVersion,
     Platform: _kNoColorTerminalPlatform,
-  });
+  }, timeout: allowForCreateFlutterProject);
 
   testUsingContext('can re-gen default template over existing project', () async {
     Cache.flutterRoot = '../..';
@@ -839,7 +841,7 @@ void main() {
 
     final String metadata = fs.file(fs.path.join(projectDir.path, '.metadata')).readAsStringSync();
     expect(metadata, contains('project_type: app\n'));
-  });
+  }, timeout: allowForCreateFlutterProject);
 
   testUsingContext('can re-gen default template over existing app project with no metadta and detect the type', () async {
     Cache.flutterRoot = '../..';
@@ -856,7 +858,7 @@ void main() {
 
     final String metadata = fs.file(fs.path.join(projectDir.path, '.metadata')).readAsStringSync();
     expect(metadata, contains('project_type: app\n'));
-  });
+  }, timeout: allowForCreateFlutterProject);
 
   testUsingContext('can re-gen app template over existing app project and detect the type', () async {
     Cache.flutterRoot = '../..';
@@ -870,7 +872,7 @@ void main() {
 
     final String metadata = fs.file(fs.path.join(projectDir.path, '.metadata')).readAsStringSync();
     expect(metadata, contains('project_type: app\n'));
-  });
+  }, timeout: allowForCreateFlutterProject);
 
   testUsingContext('can re-gen template over existing module project and detect the type', () async {
     Cache.flutterRoot = '../..';
@@ -884,7 +886,7 @@ void main() {
 
     final String metadata = fs.file(fs.path.join(projectDir.path, '.metadata')).readAsStringSync();
     expect(metadata, contains('project_type: module\n'));
-  });
+  }, timeout: allowForCreateFlutterProject);
 
   testUsingContext('can re-gen default template over existing plugin project and detect the type', () async {
     Cache.flutterRoot = '../..';
@@ -898,7 +900,7 @@ void main() {
 
     final String metadata = fs.file(fs.path.join(projectDir.path, '.metadata')).readAsStringSync();
     expect(metadata, contains('project_type: plugin'));
-  });
+  }, timeout: allowForCreateFlutterProject);
 
   testUsingContext('can re-gen default template over existing package project and detect the type', () async {
     Cache.flutterRoot = '../..';
@@ -912,7 +914,7 @@ void main() {
 
     final String metadata = fs.file(fs.path.join(projectDir.path, '.metadata')).readAsStringSync();
     expect(metadata, contains('project_type: package'));
-  });
+  }, timeout: allowForCreateFlutterProject);
 
   testUsingContext('can re-gen module .android/ folder, reusing custom org', () async {
     await _createProject(
@@ -928,7 +930,7 @@ void main() {
         '.android/app/src/main/java/com/bar/foo/flutter_project/host/MainActivity.java',
       ],
     );
-  }, overrides: <Type, Generator>{
+  }, timeout: allowForRemotePubInvocation, overrides: <Type, Generator>{
     Pub: () => const Pub(),
   });
 
@@ -945,7 +947,7 @@ void main() {
       await project.ios.productBundleIdentifier,
       'com.bar.foo.flutterProject',
     );
-  }, overrides: <Type, Generator>{
+  }, timeout: allowForRemotePubInvocation, overrides: <Type, Generator>{
     Pub: () => const Pub(),
   });
 
@@ -972,7 +974,7 @@ void main() {
         'android/app/src/main/java/com/example/flutter_project/MainActivity.java',
       ],
     );
-  });
+  }, timeout: allowForCreateFlutterProject);
 
   testUsingContext('can re-gen app ios/ folder, reusing custom org', () async {
     await _createProject(
@@ -987,7 +989,7 @@ void main() {
       await project.ios.productBundleIdentifier,
       'com.bar.foo.flutterProject',
     );
-  });
+  }, timeout: allowForCreateFlutterProject);
 
   testUsingContext('can re-gen plugin ios/ and example/ folders, reusing custom org', () async {
     await _createProject(
@@ -1020,7 +1022,7 @@ void main() {
       await project.example.ios.productBundleIdentifier,
       'com.bar.foo.flutterProjectExample',
     );
-  });
+  }, timeout: allowForCreateFlutterProject);
 
   testUsingContext('fails to re-gen without specified org when org is ambiguous', () async {
     await _createProject(
@@ -1038,7 +1040,7 @@ void main() {
       () => _createProject(projectDir, <String>[], <String>[]),
       throwsToolExit(message: 'Ambiguous organization'),
     );
-  });
+  }, timeout: allowForCreateFlutterProject);
 
   // Verify that we help the user correct an option ordering issue
   testUsingContext('produces sensible error message', () async {
@@ -1100,7 +1102,7 @@ void main() {
         'ios/Runner/GeneratedPluginRegistrant.h',
       ],
     );
-  }, overrides: <Type, Generator>{
+  }, timeout: allowForRemotePubInvocation, overrides: <Type, Generator>{
     Pub: () => const Pub(),
   });
 
@@ -1126,6 +1128,7 @@ void main() {
       expect(loggingProcessManager.commands.first, contains(matches(r'dart-sdk[\\/]bin[\\/]pub')));
       expect(loggingProcessManager.commands.first, contains('--offline'));
     },
+    timeout: allowForCreateFlutterProject,
     overrides: <Type, Generator>{
       ProcessManager: () => loggingProcessManager,
       Pub: () => const Pub(),
@@ -1144,6 +1147,7 @@ void main() {
       expect(loggingProcessManager.commands.first, contains(matches(r'dart-sdk[\\/]bin[\\/]pub')));
       expect(loggingProcessManager.commands.first, isNot(contains('--offline')));
     },
+    timeout: allowForCreateFlutterProject,
     overrides: <Type, Generator>{
       ProcessManager: () => loggingProcessManager,
       Pub: () => const Pub(),
@@ -1164,7 +1168,7 @@ void main() {
     );
     expect(projectDir.childDirectory('lib').childFile('main.dart').readAsStringSync(),
       contains('void main() {}'));
-  }, overrides: <Type, Generator>{
+  }, timeout: allowForRemotePubInvocation, overrides: <Type, Generator>{
     HttpClientFactory: () => () => MockHttpClient(200, result: 'void main() {}'),
   });
 

packages/flutter_tools/test/commands.shard/permeable/packages_test.dart

@@ -207,7 +207,7 @@ void main() {
 
       expectDependenciesResolved(projectPath);
       expectZeroPluginsInjected(projectPath);
-    }, overrides: <Type, Generator>{
+    }, timeout: allowForRemotePubInvocation, overrides: <Type, Generator>{
       Pub: () => const Pub(),
     });
 
@@ -220,7 +220,7 @@ void main() {
 
       expectDependenciesResolved(projectPath);
       expectZeroPluginsInjected(projectPath);
-    }, overrides: <Type, Generator>{
+    }, timeout: allowForCreateFlutterProject, overrides: <Type, Generator>{
       Pub: () => const Pub(),
     });
 
@@ -234,7 +234,7 @@ void main() {
 
       expect(await getCommand.usageValues,
              containsPair(CustomDimensions.commandPackagesNumberPlugins, '0'));
-    }, overrides: <Type, Generator>{
+    }, timeout: allowForCreateFlutterProject, overrides: <Type, Generator>{
       Pub: () => const Pub(),
     });
 
@@ -248,7 +248,7 @@ void main() {
 
       expect(await getCommand.usageValues,
              containsPair(CustomDimensions.commandPackagesProjectModule, 'false'));
-    }, overrides: <Type, Generator>{
+    }, timeout: allowForCreateFlutterProject, overrides: <Type, Generator>{
       Pub: () => const Pub(),
     });
 
@@ -262,7 +262,7 @@ void main() {
 
       expect(await getCommand.usageValues,
              containsPair(CustomDimensions.commandPackagesProjectModule, 'true'));
-    }, overrides: <Type, Generator>{
+    }, timeout: allowForCreateFlutterProject, overrides: <Type, Generator>{
       Pub: () => const Pub(),
     });
 
@@ -275,7 +275,7 @@ void main() {
 
       expectDependenciesResolved(projectPath);
       expectZeroPluginsInjected(projectPath);
-    }, overrides: <Type, Generator>{
+    }, timeout: allowForRemotePubInvocation, overrides: <Type, Generator>{
       Pub: () => const Pub(),
     });
 
@@ -288,7 +288,7 @@ void main() {
 
       expectDependenciesResolved(projectPath);
       expectModulePluginInjected(projectPath);
-    }, overrides: <Type, Generator>{
+    }, timeout: allowForRemotePubInvocation, overrides: <Type, Generator>{
       Pub: () => const Pub(),
     });
 
@@ -309,7 +309,7 @@ void main() {
 
       expectDependenciesResolved(exampleProjectPath);
       expectPluginInjected(exampleProjectPath);
-    }, overrides: <Type, Generator>{
+    }, timeout: allowForRemotePubInvocation, overrides: <Type, Generator>{
       Pub: () => const Pub(),
     });
   });

packages/flutter_tools/test/general.shard/commands/build_aar_test.dart

@@ -57,7 +57,7 @@ void main() {
 
     }, overrides: <Type, Generator>{
       AndroidBuilder: () => FakeAndroidBuilder(),
-    });
+    }, timeout: allowForCreateFlutterProject);
 
     testUsingContext('indicate that project is a plugin', () async {
       final String projectPath = await createProject(tempDir,
@@ -69,7 +69,7 @@ void main() {
 
     }, overrides: <Type, Generator>{
       AndroidBuilder: () => FakeAndroidBuilder(),
-    });
+    }, timeout: allowForCreateFlutterProject);
 
     testUsingContext('indicate the target platform', () async {
       final String projectPath = await createProject(tempDir,
@@ -82,7 +82,7 @@ void main() {
 
     }, overrides: <Type, Generator>{
       AndroidBuilder: () => FakeAndroidBuilder(),
-    });
+    }, timeout: allowForCreateFlutterProject);
   });
 
   group('Gradle', () {

packages/flutter_tools/test/general.shard/commands/build_apk_test.dart

@@ -48,7 +48,7 @@ void main() {
 
     }, overrides: <Type, Generator>{
       AndroidBuilder: () => FakeAndroidBuilder(),
-    });
+    }, timeout: allowForCreateFlutterProject);
 
     testUsingContext('split per abi', () async {
       final String projectPath = await createProject(tempDir,
@@ -65,7 +65,7 @@ void main() {
 
     }, overrides: <Type, Generator>{
       AndroidBuilder: () => FakeAndroidBuilder(),
-    });
+    }, timeout: allowForCreateFlutterProject);
 
     testUsingContext('build type', () async {
       final String projectPath = await createProject(tempDir,
@@ -92,7 +92,7 @@ void main() {
 
     }, overrides: <Type, Generator>{
       AndroidBuilder: () => FakeAndroidBuilder(),
-    });
+    }, timeout: allowForCreateFlutterProject);
   });
 
   group('Gradle', () {
@@ -223,7 +223,8 @@ flutter:
       FlutterProjectFactory: () => FakeFlutterProjectFactory(tempDir),
       GradleUtils: () => GradleUtils(),
       ProcessManager: () => mockProcessManager,
-    });
+    },
+    timeout: allowForCreateFlutterProject);
 
     testUsingContext('shrinking is disabled when --no-shrink is passed', () async {
       final String projectPath = await createProject(tempDir,
@@ -254,7 +255,8 @@ flutter:
       FlutterProjectFactory: () => FakeFlutterProjectFactory(tempDir),
       GradleUtils: () => GradleUtils(),
       ProcessManager: () => mockProcessManager,
-    });
+    },
+    timeout: allowForCreateFlutterProject);
 
     testUsingContext('guides the user when the shrinker fails', () async {
       final String projectPath = await createProject(tempDir,
@@ -310,7 +312,8 @@ flutter:
       FlutterProjectFactory: () => FakeFlutterProjectFactory(tempDir),
       ProcessManager: () => mockProcessManager,
       Usage: () => mockUsage,
-    });
+    },
+    timeout: allowForCreateFlutterProject);
 
     testUsingContext('reports when the app isn\'t using AndroidX', () async {
       final String projectPath = await createProject(tempDir,
@@ -363,7 +366,8 @@ flutter:
       FlutterProjectFactory: () => FakeFlutterProjectFactory(tempDir),
       ProcessManager: () => mockProcessManager,
       Usage: () => mockUsage,
-    });
+    },
+    timeout: allowForCreateFlutterProject);
 
     testUsingContext('reports when the app is using AndroidX', () async {
       final String projectPath = await createProject(tempDir,
@@ -418,7 +422,8 @@ flutter:
       FlutterProjectFactory: () => FakeFlutterProjectFactory(tempDir),
       ProcessManager: () => mockProcessManager,
       Usage: () => mockUsage,
-    });
+    },
+    timeout: allowForCreateFlutterProject);
   });
 }
 

packages/flutter_tools/test/general.shard/commands/build_appbundle_test.dart

@@ -48,7 +48,7 @@ void main() {
 
     }, overrides: <Type, Generator>{
       AndroidBuilder: () => FakeAndroidBuilder(),
-    });
+    }, timeout: allowForCreateFlutterProject);
 
     testUsingContext('build type', () async {
       final String projectPath = await createProject(tempDir,
@@ -75,7 +75,7 @@ void main() {
 
     }, overrides: <Type, Generator>{
       AndroidBuilder: () => FakeAndroidBuilder(),
-    });
+    }, timeout: allowForCreateFlutterProject);
   });
 
   group('Gradle', () {
@@ -212,7 +212,8 @@ flutter:
       FlutterProjectFactory: () => FakeFlutterProjectFactory(tempDir),
       GradleUtils: () => GradleUtils(),
       ProcessManager: () => mockProcessManager,
-    });
+    },
+    timeout: allowForCreateFlutterProject);
 
     testUsingContext('shrinking is disabled when --no-shrink is passed', () async {
       final String projectPath = await createProject(
@@ -245,7 +246,8 @@ flutter:
       FlutterProjectFactory: () => FakeFlutterProjectFactory(tempDir),
       GradleUtils: () => GradleUtils(),
       ProcessManager: () => mockProcessManager,
-    });
+    },
+    timeout: allowForCreateFlutterProject);
 
     testUsingContext('guides the user when the shrinker fails', () async {
       final String projectPath = await createProject(tempDir,
@@ -301,7 +303,8 @@ flutter:
       FlutterProjectFactory: () => FakeFlutterProjectFactory(tempDir),
       ProcessManager: () => mockProcessManager,
       Usage: () => mockUsage,
-    });
+    },
+    timeout: allowForCreateFlutterProject);
 
     testUsingContext('reports when the app isn\'t using AndroidX', () async {
       final String projectPath = await createProject(tempDir,
@@ -354,7 +357,8 @@ flutter:
       FlutterProjectFactory: () => FakeFlutterProjectFactory(tempDir),
       ProcessManager: () => mockProcessManager,
       Usage: () => mockUsage,
-    });
+    },
+    timeout: allowForCreateFlutterProject);
 
     testUsingContext('reports when the app is using AndroidX', () async {
       final String projectPath = await createProject(tempDir,
@@ -409,7 +413,8 @@ flutter:
       FlutterProjectFactory: () => FakeFlutterProjectFactory(tempDir),
       ProcessManager: () => mockProcessManager,
       Usage: () => mockUsage,
-    });
+    },
+    timeout: allowForCreateFlutterProject);
   });
 }
 

packages/flutter_tools/test/integration.shard/daemon_mode_test.dart

@@ -2,9 +2,13 @@
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
+// Integration tests which invoke flutter instead of unit testing the code
+// will not produce meaningful coverage information - we can measure coverage
+// from the isolate running the test, but not from the isolate started via
+// the command line process.
+@Tags(<String>['no_coverage'])
 import 'dart:async';
 import 'dart:convert';
-import 'dart:io';
 
 import 'package:file/file.dart';
 import 'package:flutter_tools/src/base/file_system.dart';
@@ -17,58 +21,79 @@ import 'test_driver.dart';
 import 'test_utils.dart';
 
 void main() {
-  test('device.getDevices', () async {
+  group('daemon_mode', () {
-    final Directory tempDir = createResolvedTempDirectorySync('daemon_mode_test.');
+    Directory tempDir;
-
     final BasicProject _project = BasicProject();
-    await _project.setUpIn(tempDir);
+    Process process;
 
-    final String flutterBin = fs.path.join(getFlutterRoot(), 'bin', 'flutter');
+    setUp(() async {
+      tempDir = createResolvedTempDirectorySync('daemon_mode_test.');
+      await _project.setUpIn(tempDir);
+    });
 
-    const ProcessManager processManager = LocalProcessManager();
+    tearDown(() async {
-    final Process process = await processManager.start(
+      tryToDelete(tempDir);
-      <String>[flutterBin, '--show-test-device', 'daemon'],
+      process?.kill();
-      workingDirectory: tempDir.path,
+    });
-    );
 
-    final StreamController<String> stdout = StreamController<String>.broadcast();
+    test('device.getDevices', () async {
-    transformToLines(process.stdout).listen((String line) => stdout.add(line));
+      final String flutterBin =
-    final Stream<Map<String, dynamic>> stream = stdout
+          fs.path.join(getFlutterRoot(), 'bin', 'flutter');
-      .stream
-      .map<Map<String, dynamic>>(parseFlutterResponse)
-      .where((Map<String, dynamic> value) => value != null);
 
-    Map<String, dynamic> response = await stream.first;
+      const ProcessManager processManager = LocalProcessManager();
-    expect(response['event'], 'daemon.connected');
+      process = await processManager.start(
+          <String>[flutterBin, '--show-test-device', 'daemon'],
+          workingDirectory: tempDir.path);
 
-    // start listening for devices
+      final StreamController<String> stdout =
-    process.stdin.writeln('[${jsonEncode(<String, dynamic>{
+          StreamController<String>.broadcast();
-      'id': 1,
-      'method': 'device.enable',
-    })}]');
-    response = await stream.first;
-    expect(response['id'], 1);
-    expect(response['error'], isNull);
 
-    // [{"event":"device.added","params":{"id":"flutter-tester","name":
+      transformToLines(process.stdout)
-    //   "Flutter test device","platform":"flutter-tester","emulator":false}}]
+          .listen((String line) => stdout.add(line));
-    response = await stream.first;
-    expect(response['event'], 'device.added');
 
-    // get the list of all devices
+      final Stream<Map<String, dynamic>> stream =
-    process.stdin.writeln('[${jsonEncode(<String, dynamic>{
+        stdout.stream.where((String line) {
-      'id': 2,
+          final Map<String, dynamic> response = parseFlutterResponse(line);
-      'method': 'device.getDevices',
+          // ignore 'Starting device daemon...'
-    })}]');
+          if (response == null) {
-    response = await stream.first;
+            return false;
-    expect(response['id'], 2);
+          }
-    expect(response['error'], isNull);
+          // TODO(devoncarew): Remove this after #25440 lands.
+          if (response['event'] == 'daemon.showMessage') {
+            return false;
+          }
+          return true;
+        }).map(parseFlutterResponse);
 
-    final dynamic result = response['result'];
+      Map<String, dynamic> response = await stream.first;
-    expect(result, isList);
+      expect(response['event'], 'daemon.connected');
-    expect(result, isNotEmpty);
 
-    tryToDelete(tempDir);
+      // start listening for devices
-    process.kill();
+      process.stdin.writeln('[${jsonEncode(<String, dynamic>{
-  });
+        'id': 1,
+        'method': 'device.enable',
+      })}]');
+      response = await stream.first;
+      expect(response['id'], 1);
+      expect(response['error'], isNull);
+
+      // [{"event":"device.added","params":{"id":"flutter-tester","name":
+      //   "Flutter test device","platform":"flutter-tester","emulator":false}}]
+      response = await stream.first;
+      expect(response['event'], 'device.added');
+
+      // get the list of all devices
+      process.stdin.writeln('[${jsonEncode(<String, dynamic>{
+        'id': 2,
+        'method': 'device.getDevices',
+      })}]');
+      response = await stream.first;
+      expect(response['id'], 2);
+      expect(response['error'], isNull);
+
+      final dynamic result = response['result'];
+      expect(result, isList);
+      expect(result, isNotEmpty);
+    });
+  }, timeout: const Timeout.factor(10), tags: <String>['integration']); // This test uses the `flutter` tool, which could be blocked behind the startup lock for a long time.
 }

packages/flutter_tools/test/integration.shard/debugger_stepping_test.dart

@@ -2,8 +2,11 @@
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
-import 'dart:io';
+// Integration tests which invoke flutter instead of unit testing the code
-
+// will not produce meaningful coverage information - we can measure coverage
+// from the isolate running the test, but not from the isolate started via
+// the command line process.
+@Tags(<String>['no_coverage'])
 import 'package:file/file.dart';
 import 'package:flutter_tools/src/base/file_system.dart';
 
@@ -13,37 +16,43 @@ import 'test_driver.dart';
 import 'test_utils.dart';
 
 void main() {
-  test('can step over statements', () async {
+  group('debugger', () {
-    final Directory tempDir = createResolvedTempDirectorySync('debugger_stepping_test.');
+    Directory tempDir;
-
     final SteppingProject _project = SteppingProject();
-    await _project.setUpIn(tempDir);
+    FlutterRunTestDriver _flutter;
 
-    final FlutterRunTestDriver _flutter = FlutterRunTestDriver(tempDir);
+    setUp(() async {
+      tempDir = createResolvedTempDirectorySync('debugger_stepping_test.');
+      await _project.setUpIn(tempDir);
+      _flutter = FlutterRunTestDriver(tempDir);
+    });
 
-    await _flutter.run(withDebugger: true, startPaused: true);
+    tearDown(() async {
-    await _flutter.addBreakpoint(_project.breakpointUri, _project.breakpointLine);
+      await _flutter.stop();
-    await _flutter.resume();
+      tryToDelete(tempDir);
-    await _flutter.waitForPause(); // Now we should be on the breakpoint.
+    });
 
-    expect((await _flutter.getSourceLocation()).line, equals(_project.breakpointLine));
+    test('can step over statements', () async {
+      await _flutter.run(withDebugger: true, startPaused: true);
+      await _flutter.addBreakpoint(_project.breakpointUri, _project.breakpointLine);
+      await _flutter.resume();
+      await _flutter.waitForPause(); // Now we should be on the breakpoint.
 
-    // Issue 5 steps, ensuring that we end up on the annotated lines each time.
+      expect((await _flutter.getSourceLocation()).line, equals(_project.breakpointLine));
-    for (int i = 1; i <= _project.numberOfSteps; i += 1) {
-      await _flutter.stepOverOrOverAsyncSuspension();
-      final SourcePosition location = await _flutter.getSourceLocation();
-      final int actualLine = location.line;
 
-      // Get the line we're expected to stop at by searching for the comment
+      // Issue 5 steps, ensuring that we end up on the annotated lines each time.
-      // within the source code.
+      for (int i = 1; i <= _project.numberOfSteps; i += 1) {
-      final int expectedLine = _project.lineForStep(i);
+        await _flutter.stepOverOrOverAsyncSuspension();
+        final SourcePosition location = await _flutter.getSourceLocation();
+        final int actualLine = location.line;
 
-      expect(actualLine, equals(expectedLine),
+        // Get the line we're expected to stop at by searching for the comment
-        reason: 'After $i steps, debugger should stop at $expectedLine but stopped at $actualLine'
+        // within the source code.
-      );
+        final int expectedLine = _project.lineForStep(i);
-    }
 
-    await _flutter.stop();
+        expect(actualLine, equals(expectedLine),
-    tryToDelete(tempDir);
+          reason: 'After $i steps, debugger should stop at $expectedLine but stopped at $actualLine');
-  });
+      }
+    });
+  }, timeout: const Timeout.factor(10), tags: <String>['integration']); // The DevFS sync takes a really long time, so these tests can be slow.
 }

packages/flutter_tools/test/integration.shard/expression_evaluation_test.dart

@@ -2,8 +2,12 @@
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
+// Integration tests which invoke flutter instead of unit testing the code
+// will not produce meaningful coverage information - we can measure coverage
+// from the isolate running the test, but not from the isolate started via
+// the command line process.
+@Tags(<String>['no_coverage'])
 import 'dart:async';
-import 'dart:io';
 
 import 'package:file/file.dart';
 import 'package:flutter_tools/src/base/file_system.dart';
@@ -16,133 +20,120 @@ import 'test_data/tests_project.dart';
 import 'test_driver.dart';
 import 'test_utils.dart';
 
-void batch1() {
+void main() {
-  final BasicProject _project = BasicProject();
+  group('flutter run expression evaluation', () {
-  Directory tempDir;
+    Directory tempDir;
-  FlutterRunTestDriver _flutter;
+    final BasicProject _project = BasicProject();
+    FlutterRunTestDriver _flutter;
 
-  Future<void> initProject() async {
+    setUp(() async {
-    tempDir = createResolvedTempDirectorySync('run_expression_eval_test.');
+      tempDir = createResolvedTempDirectorySync('run_expression_eval_test.');
-    await _project.setUpIn(tempDir);
+      await _project.setUpIn(tempDir);
-    _flutter = FlutterRunTestDriver(tempDir);
+      _flutter = FlutterRunTestDriver(tempDir);
-  }
+    });
 
-  Future<void> cleanProject() async {
+    tearDown(() async {
-    await _flutter.stop();
+      await _flutter.stop();
-    tryToDelete(tempDir);
+      tryToDelete(tempDir);
-  }
+    });
 
-  Future<void> breakInBuildMethod(FlutterTestDriver flutter) async {
+    Future<void> breakInBuildMethod(FlutterTestDriver flutter) async {
-    await _flutter.breakAt(
+      await _flutter.breakAt(
-      _project.buildMethodBreakpointUri,
+        _project.buildMethodBreakpointUri,
-      _project.buildMethodBreakpointLine,
+        _project.buildMethodBreakpointLine,
-    );
+      );
-  }
+    }
 
-  Future<void> breakInTopLevelFunction(FlutterTestDriver flutter) async {
+    Future<void> breakInTopLevelFunction(FlutterTestDriver flutter) async {
-    await _flutter.breakAt(
+      await _flutter.breakAt(
-      _project.topLevelFunctionBreakpointUri,
+        _project.topLevelFunctionBreakpointUri,
-      _project.topLevelFunctionBreakpointLine,
+        _project.topLevelFunctionBreakpointLine,
-    );
+      );
-  }
+    }
 
-  test('flutter run expression evaluation - can evaluate trivial expressions in top level function', () async {
+    test('can evaluate trivial expressions in top level function', () async {
-    await initProject();
+      await _flutter.run(withDebugger: true);
-    await _flutter.run(withDebugger: true);
+      await breakInTopLevelFunction(_flutter);
-    await breakInTopLevelFunction(_flutter);
+      await evaluateTrivialExpressions(_flutter);
-    await evaluateTrivialExpressions(_flutter);
+    });
-    await cleanProject();
-  });
 
-  test('flutter run expression evaluation - can evaluate trivial expressions in build method', () async {
+    test('can evaluate trivial expressions in build method', () async {
-    await initProject();
+      await _flutter.run(withDebugger: true);
-    await _flutter.run(withDebugger: true);
+      await breakInBuildMethod(_flutter);
-    await breakInBuildMethod(_flutter);
+      await evaluateTrivialExpressions(_flutter);
-    await evaluateTrivialExpressions(_flutter);
+    });
-    await cleanProject();
-  });
 
-  test('flutter run expression evaluation - can evaluate complex expressions in top level function', () async {
+    test('can evaluate complex expressions in top level function', () async {
-    await initProject();
+      await _flutter.run(withDebugger: true);
-    await _flutter.run(withDebugger: true);
+      await breakInTopLevelFunction(_flutter);
-    await breakInTopLevelFunction(_flutter);
+      await evaluateComplexExpressions(_flutter);
-    await evaluateComplexExpressions(_flutter);
+    });
-    await cleanProject();
-  });
 
-  test('flutter run expression evaluation - can evaluate complex expressions in build method', () async {
+    test('can evaluate complex expressions in build method', () async {
-    await initProject();
+      await _flutter.run(withDebugger: true);
-    await _flutter.run(withDebugger: true);
+      await breakInBuildMethod(_flutter);
-    await breakInBuildMethod(_flutter);
+      await evaluateComplexExpressions(_flutter);
-    await evaluateComplexExpressions(_flutter);
+    });
-    await cleanProject();
-  });
 
-  test('flutter run expression evaluation - can evaluate expressions returning complex objects in top level function', () async {
+    test('can evaluate expressions returning complex objects in top level function', () async {
-    await initProject();
+      await _flutter.run(withDebugger: true);
-    await _flutter.run(withDebugger: true);
+      await breakInTopLevelFunction(_flutter);
-    await breakInTopLevelFunction(_flutter);
+      await evaluateComplexReturningExpressions(_flutter);
-    await evaluateComplexReturningExpressions(_flutter);
+    });
-    await cleanProject();
-  });
 
-  test('flutter run expression evaluation - can evaluate expressions returning complex objects in build method', () async {
+    test('can evaluate expressions returning complex objects in build method', () async {
-    await initProject();
+      await _flutter.run(withDebugger: true);
-    await _flutter.run(withDebugger: true);
+      await breakInBuildMethod(_flutter);
-    await breakInBuildMethod(_flutter);
+      await evaluateComplexReturningExpressions(_flutter);
-    await evaluateComplexReturningExpressions(_flutter);
+    });
-    await cleanProject();
+  }, timeout: const Timeout.factor(10), tags: <String>['integration']); // The DevFS sync takes a really long time, so these tests can be slow.
-  });
-}
 
-void batch2() {
+  group('flutter test expression evaluation', () {
-  final TestsProject _project = TestsProject();
+    Directory tempDir;
-  Directory tempDir;
+    final TestsProject _project = TestsProject();
-  FlutterTestTestDriver _flutter;
+    FlutterTestTestDriver _flutter;
 
-  Future<void> initProject() async {
+    setUp(() async {
-    tempDir = createResolvedTempDirectorySync('test_expression_eval_test.');
+      tempDir = createResolvedTempDirectorySync('test_expression_eval_test.');
-    await _project.setUpIn(tempDir);
+      await _project.setUpIn(tempDir);
-    _flutter = FlutterTestTestDriver(tempDir);
+      _flutter = FlutterTestTestDriver(tempDir);
-  }
+    });
 
-  Future<void> cleanProject() async {
+    tearDown(() async {
-    await _flutter?.quit();
+      await _flutter.quit();
-    tryToDelete(tempDir);
+      tryToDelete(tempDir);
-  }
+    });
 
-  test('flutter test expression evaluation - can evaluate trivial expressions in a test', () async {
+    test('can evaluate trivial expressions in a test', () async {
-    await initProject();
+      await _flutter.test(
-    await _flutter.test(
+        withDebugger: true,
-      withDebugger: true,
+        beforeStart: () => _flutter.addBreakpoint(_project.breakpointUri, _project.breakpointLine),
-      beforeStart: () => _flutter.addBreakpoint(_project.breakpointUri, _project.breakpointLine),
+      );
-    );
+      await _flutter.waitForPause();
-    await _flutter.waitForPause();
+      await evaluateTrivialExpressions(_flutter);
-    await evaluateTrivialExpressions(_flutter);
+      await _flutter.resume();
-    await cleanProject();
+    });
-  });
 
-  test('flutter test expression evaluation - can evaluate complex expressions in a test', () async {
+    test('can evaluate complex expressions in a test', () async {
-    await initProject();
+      await _flutter.test(
-    await _flutter.test(
+        withDebugger: true,
-      withDebugger: true,
+        beforeStart: () => _flutter.addBreakpoint(_project.breakpointUri, _project.breakpointLine),
-      beforeStart: () => _flutter.addBreakpoint(_project.breakpointUri, _project.breakpointLine),
+      );
-    );
+      await _flutter.waitForPause();
-    await _flutter.waitForPause();
+      await evaluateComplexExpressions(_flutter);
-    await evaluateComplexExpressions(_flutter);
+      await _flutter.resume();
-    await cleanProject();
+    });
-  });
 
-  test('flutter test expression evaluation - can evaluate expressions returning complex objects in a test', () async {
+    test('can evaluate expressions returning complex objects in a test', () async {
-    await initProject();
+      await _flutter.test(
-    await _flutter.test(
+        withDebugger: true,
-      withDebugger: true,
+        beforeStart: () => _flutter.addBreakpoint(_project.breakpointUri, _project.breakpointLine),
-      beforeStart: () => _flutter.addBreakpoint(_project.breakpointUri, _project.breakpointLine),
+      );
-    );
+      await _flutter.waitForPause();
-    await _flutter.waitForPause();
+      await evaluateComplexReturningExpressions(_flutter);
-    await evaluateComplexReturningExpressions(_flutter);
+      await _flutter.resume();
-    await cleanProject();
+    });
-  });
+  }, timeout: const Timeout.factor(10), tags: <String>['integration']); // The DevFS sync takes a really long time, so these tests can be slow.
 }
 
 Future<void> evaluateTrivialExpressions(FlutterTestDriver flutter) async {
@@ -173,8 +164,3 @@ Future<void> evaluateComplexReturningExpressions(FlutterTestDriver flutter) asyn
   final InstanceRef res = await flutter.evaluate(resp.id, r'"$year-$month-$day"');
   expect(res.valueAsString, equals('${now.year}-${now.month}-${now.day}'));
 }
-
-void main() {
-  batch1();
-  batch2();
-}

packages/flutter_tools/test/integration.shard/flutter_attach_test.dart

@@ -2,8 +2,11 @@
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
-import 'dart:io';
+// Integration tests which invoke flutter instead of unit testing the code
-
+// will not produce meaningful coverage information - we can measure coverage
+// from the isolate running the test, but not from the isolate started via
+// the command line process.
+@Tags(<String>['no_coverage'])
 import 'package:file/file.dart';
 import 'package:flutter_tools/src/base/file_system.dart';
 
@@ -30,36 +33,35 @@ void main() {
     tryToDelete(tempDir);
   });
 
-  test('writes pid-file', () async {
+  group('attached process', () {
-    final File pidFile = tempDir.childFile('test.pid');
+    test('writes pid-file', () async {
-    await _flutterRun.run(withDebugger: true);
+      final File pidFile = tempDir.childFile('test.pid');
-    await _flutterAttach.attach(
+      await _flutterRun.run(withDebugger: true);
-      _flutterRun.vmServicePort,
+      await _flutterAttach.attach(
-      pidFile: pidFile,
+        _flutterRun.vmServicePort,
-    );
+        pidFile: pidFile,
-    expect(pidFile.existsSync(), isTrue);
+      );
-  });
+      expect(pidFile.existsSync(), isTrue);
-
+    });
-  test('can hot reload', () async {
+    test('can hot reload', () async {
-    await _flutterRun.run(withDebugger: true);
+      await _flutterRun.run(withDebugger: true);
-    await _flutterAttach.attach(_flutterRun.vmServicePort);
+      await _flutterAttach.attach(_flutterRun.vmServicePort);
-    await _flutterAttach.hotReload();
+      await _flutterAttach.hotReload();
-  });
+    });
-
+    test('can detach, reattach, hot reload', () async {
-  test('can detach, reattach, hot reload', () async {
+      await _flutterRun.run(withDebugger: true);
-    await _flutterRun.run(withDebugger: true);
+      await _flutterAttach.attach(_flutterRun.vmServicePort);
-    await _flutterAttach.attach(_flutterRun.vmServicePort);
+      await _flutterAttach.detach();
-    await _flutterAttach.detach();
+      await _flutterAttach.attach(_flutterRun.vmServicePort);
-    await _flutterAttach.attach(_flutterRun.vmServicePort);
+      await _flutterAttach.hotReload();
-    await _flutterAttach.hotReload();
+    });
-  });
+    test('killing process behaves the same as detach ', () async {
-
+      await _flutterRun.run(withDebugger: true);
-  test('killing process behaves the same as detach ', () async {
+      await _flutterAttach.attach(_flutterRun.vmServicePort);
-    await _flutterRun.run(withDebugger: true);
+      await _flutterAttach.quit();
-    await _flutterAttach.attach(_flutterRun.vmServicePort);
+      _flutterAttach = FlutterRunTestDriver(tempDir, logPrefix: 'ATTACH-2');
-    await _flutterAttach.quit();
+      await _flutterAttach.attach(_flutterRun.vmServicePort);
-    _flutterAttach = FlutterRunTestDriver(tempDir, logPrefix: 'ATTACH-2');
+      await _flutterAttach.hotReload();
-    await _flutterAttach.attach(_flutterRun.vmServicePort);
+    });
-    await _flutterAttach.hotReload();
+  }, timeout: const Timeout.factor(10), tags: <String>['integration']); // The DevFS sync takes a really long time, so these tests can be slow.
-  });
 }

packages/flutter_tools/test/integration.shard/flutter_run_test.dart

@@ -2,6 +2,11 @@
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
+// Integration tests which invoke flutter instead of unit testing the code
+// will not produce meaningful coverage information - we can measure coverage
+// from the isolate running the test, but not from the isolate started via
+// the command line process.
+@Tags(<String>['no_coverage'])
 import 'package:file/file.dart';
 import 'package:flutter_tools/src/base/file_system.dart';
 import 'package:flutter_tools/src/base/io.dart';
@@ -13,45 +18,47 @@ import 'test_driver.dart';
 import 'test_utils.dart';
 
 void main() {
-  Directory tempDir;
+  group('flutter_run', () {
-  final BasicProject _project = BasicProject();
+    Directory tempDir;
-  FlutterRunTestDriver _flutter;
+    final BasicProject _project = BasicProject();
+    FlutterRunTestDriver _flutter;
 
-  setUp(() async {
+    setUp(() async {
-    tempDir = createResolvedTempDirectorySync('run_test.');
+      tempDir = createResolvedTempDirectorySync('run_test.');
-    await _project.setUpIn(tempDir);
+      await _project.setUpIn(tempDir);
-    _flutter = FlutterRunTestDriver(tempDir);
+      _flutter = FlutterRunTestDriver(tempDir);
-  });
+    });
 
-  tearDown(() async {
+    tearDown(() async {
-    await _flutter.stop();
+      await _flutter.stop();
-    tryToDelete(tempDir);
+      tryToDelete(tempDir);
-  });
+    });
 
-  test('flutter run reports an error if an invalid device is supplied', () async {
+    test('reports an error if an invalid device is supplied', () async {
-    // This test forces flutter to check for all possible devices to catch issues
+      // This test forces flutter to check for all possible devices to catch issues
-    // like https://github.com/flutter/flutter/issues/21418 which were skipped
+      // like https://github.com/flutter/flutter/issues/21418 which were skipped
-    // over because other integration tests run using flutter-tester which short-cuts
+      // over because other integration tests run using flutter-tester which short-cuts
-    // some of the checks for devices.
+      // some of the checks for devices.
-    final String flutterBin = fs.path.join(getFlutterRoot(), 'bin', 'flutter');
+      final String flutterBin = fs.path.join(getFlutterRoot(), 'bin', 'flutter');
 
-    const ProcessManager _processManager = LocalProcessManager();
+      const ProcessManager _processManager = LocalProcessManager();
-    final ProcessResult _proc = await _processManager.run(
+      final ProcessResult _proc = await _processManager.run(
-      <String>[flutterBin, 'run', '-d', 'invalid-device-id'],
+        <String>[flutterBin, 'run', '-d', 'invalid-device-id'],
-      workingDirectory: tempDir.path,
+        workingDirectory: tempDir.path,
-    );
+      );
 
-    expect(_proc.stdout, isNot(contains('flutter has exited unexpectedly')));
+      expect(_proc.stdout, isNot(contains('flutter has exited unexpectedly')));
-    expect(_proc.stderr, isNot(contains('flutter has exited unexpectedly')));
+      expect(_proc.stderr, isNot(contains('flutter has exited unexpectedly')));
-    if (!_proc.stderr.toString().contains('Unable to locate a development')
+      if (!_proc.stderr.toString().contains('Unable to locate a development')
-        && !_proc.stdout.toString().contains('No devices found with name or id matching')) {
+          && !_proc.stdout.toString().contains('No devices found with name or id matching')) {
-      fail("'flutter run -d invalid-device-id' did not produce the expected error");
+        fail("'flutter run -d invalid-device-id' did not produce the expected error");
-    }
+      }
-  });
+    });
 
-  test('flutter run writes pid-file', () async {
+    test('writes pid-file', () async {
-    final File pidFile = tempDir.childFile('test.pid');
+      final File pidFile = tempDir.childFile('test.pid');
-    await _flutter.run(pidFile: pidFile);
+      await _flutter.run(pidFile: pidFile);
-    expect(pidFile.existsSync(), isTrue);
+      expect(pidFile.existsSync(), isTrue);
-  });
+    });
+  }, timeout: const Timeout.factor(10), tags: <String>['integration']); // The DevFS sync takes a really long time, so these tests can be slow.
 }

packages/flutter_tools/test/integration.shard/hot_reload_test.dart

@@ -2,6 +2,11 @@
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
+// Integration tests which invoke flutter instead of unit testing the code
+// will not produce meaningful coverage information - we can measure coverage
+// from the isolate running the test, but not from the isolate started via
+// the command line process.
+@Tags(<String>['no_coverage'])
 import 'dart:async';
 
 import 'package:file/file.dart';
@@ -15,137 +20,139 @@ import 'test_driver.dart';
 import 'test_utils.dart';
 
 void main() {
-  Directory tempDir;
+  group('hot reload tests', () {
-  final HotReloadProject _project = HotReloadProject();
+    Directory tempDir;
-  FlutterRunTestDriver _flutter;
+    final HotReloadProject _project = HotReloadProject();
+    FlutterRunTestDriver _flutter;
 
-  setUp(() async {
+    setUp(() async {
-    tempDir = createResolvedTempDirectorySync('hot_reload_test.');
+      tempDir = createResolvedTempDirectorySync('hot_reload_test.');
-    await _project.setUpIn(tempDir);
+      await _project.setUpIn(tempDir);
-    _flutter = FlutterRunTestDriver(tempDir);
+      _flutter = FlutterRunTestDriver(tempDir);
-  });
+    });
 
-  tearDown(() async {
+    tearDown(() async {
-    await _flutter?.stop();
+      await _flutter?.stop();
-    tryToDelete(tempDir);
+      tryToDelete(tempDir);
-  });
+    });
 
-  test('hot reload works without error', () async {
+    test('hot reload works without error', () async {
-    await _flutter.run();
+      await _flutter.run();
-    await _flutter.hotReload();
-  });
-
-  test('newly added code executes during hot reload', () async {
-    await _flutter.run();
-    _project.uncommentHotReloadPrint();
-    final StringBuffer stdout = StringBuffer();
-    final StreamSubscription<String> subscription = _flutter.stdout.listen(stdout.writeln);
-    try {
       await _flutter.hotReload();
-      expect(stdout.toString(), contains('(((((RELOAD WORKED)))))'));
+    });
-    } finally {
+
+    test('newly added code executes during hot reload', () async {
+      await _flutter.run();
+      _project.uncommentHotReloadPrint();
+      final StringBuffer stdout = StringBuffer();
+      final StreamSubscription<String> subscription = _flutter.stdout.listen(stdout.writeln);
+      try {
+        await _flutter.hotReload();
+        expect(stdout.toString(), contains('(((((RELOAD WORKED)))))'));
+      } finally {
+        await subscription.cancel();
+      }
+    });
+
+    test('hot restart works without error', () async {
+      await _flutter.run();
+      await _flutter.hotRestart();
+    });
+
+    test('breakpoints are hit after hot reload', () async {
+      Isolate isolate;
+      await _flutter.run(withDebugger: true, startPaused: true);
+      final Completer<void> sawTick1 = Completer<void>();
+      final Completer<void> sawTick3 = Completer<void>();
+      final Completer<void> sawDebuggerPausedMessage = Completer<void>();
+      final StreamSubscription<String> subscription = _flutter.stdout.listen(
+        (String line) {
+          if (line.contains('((((TICK 1))))')) {
+            expect(sawTick1.isCompleted, isFalse);
+            sawTick1.complete();
+          }
+          if (line.contains('((((TICK 3))))')) {
+            expect(sawTick3.isCompleted, isFalse);
+            sawTick3.complete();
+          }
+          if (line.contains('The application is paused in the debugger on a breakpoint.')) {
+            expect(sawDebuggerPausedMessage.isCompleted, isFalse);
+            sawDebuggerPausedMessage.complete();
+          }
+        },
+      );
+      await _flutter.resume(); // we start paused so we can set up our TICK 1 listener before the app starts
+      unawaited(sawTick1.future.timeout(
+        const Duration(seconds: 5),
+        onTimeout: () { print('The test app is taking longer than expected to print its synchronization line...'); },
+      ));
+      await sawTick1.future; // after this, app is in steady state
+      await _flutter.addBreakpoint(
+        _project.scheduledBreakpointUri,
+        _project.scheduledBreakpointLine,
+      );
+      await _flutter.hotReload(); // reload triggers code which eventually hits the breakpoint
+      isolate = await _flutter.waitForPause();
+      expect(isolate.pauseEvent.kind, equals(EventKind.kPauseBreakpoint));
+      await _flutter.resume();
+      await _flutter.addBreakpoint(
+        _project.buildBreakpointUri,
+        _project.buildBreakpointLine,
+      );
+      bool reloaded = false;
+      final Future<void> reloadFuture = _flutter.hotReload().then((void value) { reloaded = true; });
+      await sawTick3.future; // this should happen before it pauses
+      isolate = await _flutter.waitForPause();
+      expect(isolate.pauseEvent.kind, equals(EventKind.kPauseBreakpoint));
+      await sawDebuggerPausedMessage.future;
+      expect(reloaded, isFalse);
+      await _flutter.resume();
+      await reloadFuture;
+      expect(reloaded, isTrue);
+      reloaded = false;
       await subscription.cancel();
-    }
+    });
-  });
 
-  test('hot restart works without error', () async {
+    test('hot reload doesn\'t reassemble if paused', () async {
-    await _flutter.run();
+      await _flutter.run(withDebugger: true);
-    await _flutter.hotRestart();
+      final Completer<void> sawTick2 = Completer<void>();
-  });
+      final Completer<void> sawTick3 = Completer<void>();
-
+      final Completer<void> sawDebuggerPausedMessage1 = Completer<void>();
-  test('breakpoints are hit after hot reload', () async {
+      final Completer<void> sawDebuggerPausedMessage2 = Completer<void>();
-    Isolate isolate;
+      final StreamSubscription<String> subscription = _flutter.stdout.listen(
-    await _flutter.run(withDebugger: true, startPaused: true);
+        (String line) {
-    final Completer<void> sawTick1 = Completer<void>();
+          if (line.contains('((((TICK 2))))')) {
-    final Completer<void> sawTick3 = Completer<void>();
+            expect(sawTick2.isCompleted, isFalse);
-    final Completer<void> sawDebuggerPausedMessage = Completer<void>();
+            sawTick2.complete();
-    final StreamSubscription<String> subscription = _flutter.stdout.listen(
+          }
-      (String line) {
+          if (line.contains('The application is paused in the debugger on a breakpoint.')) {
-        if (line.contains('((((TICK 1))))')) {
+            expect(sawDebuggerPausedMessage1.isCompleted, isFalse);
-          expect(sawTick1.isCompleted, isFalse);
+            sawDebuggerPausedMessage1.complete();
-          sawTick1.complete();
+          }
-        }
+          if (line.contains('The application is paused in the debugger on a breakpoint; interface might not update.')) {
-        if (line.contains('((((TICK 3))))')) {
+            expect(sawDebuggerPausedMessage2.isCompleted, isFalse);
-          expect(sawTick3.isCompleted, isFalse);
+            sawDebuggerPausedMessage2.complete();
-          sawTick3.complete();
+          }
-        }
+        },
-        if (line.contains('The application is paused in the debugger on a breakpoint.')) {
+      );
-          expect(sawDebuggerPausedMessage.isCompleted, isFalse);
+      await _flutter.addBreakpoint(
-          sawDebuggerPausedMessage.complete();
+        _project.buildBreakpointUri,
-        }
+        _project.buildBreakpointLine,
-      },
+      );
-    );
+      bool reloaded = false;
-    await _flutter.resume(); // we start paused so we can set up our TICK 1 listener before the app starts
+      final Future<void> reloadFuture = _flutter.hotReload().then((void value) { reloaded = true; });
-    unawaited(sawTick1.future.timeout(
+      await sawTick2.future; // this should happen before it pauses
-      const Duration(seconds: 5),
+      final Isolate isolate = await _flutter.waitForPause();
-      onTimeout: () { print('The test app is taking longer than expected to print its synchronization line...'); },
+      expect(isolate.pauseEvent.kind, equals(EventKind.kPauseBreakpoint));
-    ));
+      expect(reloaded, isFalse);
-    await sawTick1.future; // after this, app is in steady state
+      await sawDebuggerPausedMessage1.future; // this is the one where it say "uh, you broke into the debugger while reloading"
-    await _flutter.addBreakpoint(
+      await reloadFuture; // this is the one where it times out because you're in the debugger
-      _project.scheduledBreakpointUri,
+      expect(reloaded, isTrue);
-      _project.scheduledBreakpointLine,
+      await _flutter.hotReload(); // now we're already paused
-    );
+      expect(sawTick3.isCompleted, isFalse);
-    await _flutter.hotReload(); // reload triggers code which eventually hits the breakpoint
+      await sawDebuggerPausedMessage2.future; // so we just get told that nothing is going to happen
-    isolate = await _flutter.waitForPause();
+      await _flutter.resume();
-    expect(isolate.pauseEvent.kind, equals(EventKind.kPauseBreakpoint));
+      await subscription.cancel();
-    await _flutter.resume();
+    });
-    await _flutter.addBreakpoint(
+  }, timeout: const Timeout.factor(10), tags: <String>['integration']); // The DevFS sync takes a really long time, so these tests can be slow.
-      _project.buildBreakpointUri,
-      _project.buildBreakpointLine,
-    );
-    bool reloaded = false;
-    final Future<void> reloadFuture = _flutter.hotReload().then((void value) { reloaded = true; });
-    await sawTick3.future; // this should happen before it pauses
-    isolate = await _flutter.waitForPause();
-    expect(isolate.pauseEvent.kind, equals(EventKind.kPauseBreakpoint));
-    await sawDebuggerPausedMessage.future;
-    expect(reloaded, isFalse);
-    await _flutter.resume();
-    await reloadFuture;
-    expect(reloaded, isTrue);
-    reloaded = false;
-    await subscription.cancel();
-  });
-
-  test('hot reload doesn\'t reassemble if paused', () async {
-    await _flutter.run(withDebugger: true);
-    final Completer<void> sawTick2 = Completer<void>();
-    final Completer<void> sawTick3 = Completer<void>();
-    final Completer<void> sawDebuggerPausedMessage1 = Completer<void>();
-    final Completer<void> sawDebuggerPausedMessage2 = Completer<void>();
-    final StreamSubscription<String> subscription = _flutter.stdout.listen(
-      (String line) {
-        if (line.contains('((((TICK 2))))')) {
-          expect(sawTick2.isCompleted, isFalse);
-          sawTick2.complete();
-        }
-        if (line.contains('The application is paused in the debugger on a breakpoint.')) {
-          expect(sawDebuggerPausedMessage1.isCompleted, isFalse);
-          sawDebuggerPausedMessage1.complete();
-        }
-        if (line.contains('The application is paused in the debugger on a breakpoint; interface might not update.')) {
-          expect(sawDebuggerPausedMessage2.isCompleted, isFalse);
-          sawDebuggerPausedMessage2.complete();
-        }
-      },
-    );
-    await _flutter.addBreakpoint(
-      _project.buildBreakpointUri,
-      _project.buildBreakpointLine,
-    );
-    bool reloaded = false;
-    final Future<void> reloadFuture = _flutter.hotReload().then((void value) { reloaded = true; });
-    await sawTick2.future; // this should happen before it pauses
-    final Isolate isolate = await _flutter.waitForPause();
-    expect(isolate.pauseEvent.kind, equals(EventKind.kPauseBreakpoint));
-    expect(reloaded, isFalse);
-    await sawDebuggerPausedMessage1.future; // this is the one where it say "uh, you broke into the debugger while reloading"
-    await reloadFuture; // this is the one where it times out because you're in the debugger
-    expect(reloaded, isTrue);
-    await _flutter.hotReload(); // now we're already paused
-    expect(sawTick3.isCompleted, isFalse);
-    await sawDebuggerPausedMessage2.future; // so we just get told that nothing is going to happen
-    await _flutter.resume();
-    await subscription.cancel();
-  });
 }

packages/flutter_tools/test/integration.shard/lifetime_test.dart

@@ -2,6 +2,11 @@
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
+// Integration tests which invoke flutter instead of unit testing the code
+// will not produce meaningful coverage information - we can measure coverage
+// from the isolate running the test, but not from the isolate started via
+// the command line process.
+@Tags(<String>['no_coverage'])
 import 'dart:async';
 
 import 'package:file/file.dart';
@@ -18,30 +23,32 @@ import 'test_utils.dart';
 const Duration requiredLifespan = Duration(seconds: 5);
 
 void main() {
-  final BasicProject _project = BasicProject();
+  group('flutter run', () {
-  FlutterRunTestDriver _flutter;
+    final BasicProject _project = BasicProject();
-  Directory tempDir;
+    FlutterRunTestDriver _flutter;
+    Directory tempDir;
 
-  setUp(() async {
+    setUp(() async {
-    tempDir = createResolvedTempDirectorySync('lifetime_test.');
+      tempDir = createResolvedTempDirectorySync('lifetime_test.');
-    await _project.setUpIn(tempDir);
+      await _project.setUpIn(tempDir);
-    _flutter = FlutterRunTestDriver(tempDir);
+      _flutter = FlutterRunTestDriver(tempDir);
-  });
+    });
 
-  tearDown(() async {
+    tearDown(() async {
-    await _flutter.stop();
+      await _flutter.stop();
-    tryToDelete(tempDir);
+      tryToDelete(tempDir);
-  });
+    });
 
-  test('flutter run does not terminate when a debugger is attached', () async {
+    test('does not terminate when a debugger is attached', () async {
-    await _flutter.run(withDebugger: true);
+      await _flutter.run(withDebugger: true);
-    await Future<void>.delayed(requiredLifespan);
+      await Future<void>.delayed(requiredLifespan);
-    expect(_flutter.hasExited, equals(false));
+      expect(_flutter.hasExited, equals(false));
-  });
+    });
 
-  test('fluter run does not terminate when a debugger is attached and pause-on-exceptions', () async {
+    test('does not terminate when a debugger is attached and pause-on-exceptions', () async {
-    await _flutter.run(withDebugger: true, pauseOnExceptions: true);
+      await _flutter.run(withDebugger: true, pauseOnExceptions: true);
-    await Future<void>.delayed(requiredLifespan);
+      await Future<void>.delayed(requiredLifespan);
-    expect(_flutter.hasExited, equals(false));
+      expect(_flutter.hasExited, equals(false));
-  });
+    });
+  }, timeout: const Timeout.factor(10), tags: <String>['integration']); // The DevFS sync takes a really long time, so these tests can be slow.
 }

packages/flutter_tools/test/src/common.dart

@@ -127,6 +127,13 @@ Future<String> createProject(Directory temp, { List<String> arguments }) async {
   return projectPath;
 }
 
+/// Test case timeout for tests involving remote calls to `pub get` or similar.
+const Timeout allowForRemotePubInvocation = Timeout.factor(10.0);
+
+/// Test case timeout for tests involving creating a Flutter project with
+/// `--no-pub`. Use [allowForRemotePubInvocation] when creation involves `pub`.
+const Timeout allowForCreateFlutterProject = Timeout.factor(3.0);
+
 Future<void> expectToolExitLater(Future<dynamic> future, Matcher messageMatcher) async {
   try {
     await future;

packages/flutter_tools/test/src/context.dart

@@ -48,6 +48,7 @@ typedef ContextInitializer = void Function(AppContext testContext);
 void testUsingContext(
   String description,
   dynamic testMethod(), {
+  Timeout timeout,
   Map<Type, Generator> overrides = const <Type, Generator>{},
   bool initializeFlutterRoot = true,
   String testOn,
@@ -136,7 +137,8 @@ void testUsingContext(
         },
       );
     });
-  }, testOn: testOn, skip: skip);
+  }, timeout: timeout ?? const Timeout(Duration(seconds: 60)),
+      testOn: testOn, skip: skip);
 }
 
 void _printBufferedErrors(AppContext testContext) {