[integration_test] Add a run method for proper reporting of test results (#70075)

packages/integration_test/README.md

@@ -18,9 +18,9 @@ assertions.
 import 'package:flutter_test/flutter_test.dart';
 import 'package:integration_test/integration_test.dart';
 
-void main() {
+void main() => run(_testMain);
-  IntegrationTestWidgetsFlutterBinding.ensureInitialized();
 
+void _testMain() {
   testWidgets("failing test example", (WidgetTester tester) async {
     expect(2 + 2, equals(5));
   });

packages/integration_test/example/integration_test/_example_test_web.dart

@@ -12,12 +12,10 @@
 import 'dart:html' as html;
 import 'package:flutter/material.dart';
 import 'package:flutter_test/flutter_test.dart';
-import 'package:integration_test/integration_test.dart';
 
 import 'package:integration_test_example/main.dart' as app;
 
 void main() {
-  IntegrationTestWidgetsFlutterBinding.ensureInitialized();
   testWidgets('verify text', (WidgetTester tester) async {
     // Build our app and trigger a frame.
     app.main();

packages/integration_test/example/integration_test/_extended_test_io.dart

@@ -12,13 +12,10 @@
 import 'dart:io' show Platform;
 import 'package:flutter/material.dart';
 import 'package:flutter_test/flutter_test.dart';
-import 'package:integration_test/integration_test.dart';
 
 import 'package:integration_test_example/main.dart' as app;
 
 void main() {
-  IntegrationTestWidgetsFlutterBinding.ensureInitialized();
-
   testWidgets('verify text', (WidgetTester tester) async {
     // Build our app and trigger a frame.
     app.main();

packages/integration_test/example/integration_test/example_test.dart

@@ -14,7 +14,4 @@ import 'package:integration_test/integration_test.dart';
 import '_example_test_io.dart' if (dart.library.html) '_example_test_web.dart'
     as tests;
 
-void main() {
+void main() => run(tests.main);
-  IntegrationTestWidgetsFlutterBinding.ensureInitialized();
-  tests.main();
-}

packages/integration_test/example/integration_test/extended_test.dart

@@ -17,7 +17,4 @@ import 'package:integration_test/integration_test.dart';
 import '_extended_test_io.dart' if (dart.library.html) '_extended_test_web.dart'
     as tests;
 
-void main() {
+void main() => run(tests.main);
-  IntegrationTestWidgetsFlutterBinding.ensureInitialized();
-  tests.main();
-}

packages/integration_test/lib/common.dart

@@ -106,21 +106,48 @@ class Response {
   }
 }
 
-/// Representing a failure includes the method name and the failure details.
+/// Represents the result of running a test.
-class Failure {
+class TestResult {
-  /// Constructor requiring all fields during initialization.
+  TestResult._(this.methodName);
-  Failure(this.methodName, this.details);
 
   /// The name of the test method which failed.
   final String methodName;
+}
 
-  /// The details of the failure such as stack trace.
+/// Represents successful execution of a test.
-  final String details;
+class Success extends TestResult {
+  /// Constructor requiring all fields during initialization.
+  Success(String methodName) : super._(methodName);
+}
+
+/// Represents a test failure.
+class Failure extends TestResult {
+  /// Constructor requiring all fields during initialization.
+  ///
+  /// If [error] is passed, [errors] will be ignored.
+  Failure(String methodName, String details, {
+    Object error,
+    List<AsyncError> errors,
+  }) :
+    errors = error != null
+      ? <AsyncError>[AsyncError(error, StackTrace.fromString(details))]
+      : errors ?? <AsyncError>[],
+    super._(methodName);
+
+  /// Errors that were thrown during the test.
+  final List<AsyncError> errors;
+
+  /// The first error that was thrown during the test.
+  Object get error => errors.isEmpty ? null : errors.first.error;
+
+  /// The details of the first failure such as stack trace.
+  String get details => errors.isEmpty ? null : errors.first.stackTrace.toString();
 
   /// Serializes the object to JSON.
   String toJson() {
     return json.encode(<String, String>{
       'methodName': methodName,
+      'error': error.toString(),
       'details': details,
     });
   }

packages/integration_test/lib/integration_test.dart

@@ -6,70 +6,157 @@ import 'dart:async';
 import 'dart:developer' as developer;
 import 'dart:ui';
 
-import 'package:flutter/rendering.dart';
-import 'package:flutter_test/flutter_test.dart';
 import 'package:flutter/foundation.dart';
+import 'package:flutter/rendering.dart';
 import 'package:flutter/services.dart';
 import 'package:flutter/widgets.dart';
+import 'package:flutter_test/flutter_test.dart';
+// ignore: implementation_imports
+import 'package:test_core/src/direct_run.dart';
+// ignore: implementation_imports
+import 'package:test_core/src/runner/engine.dart';
 import 'package:vm_service/vm_service.dart' as vm;
 import 'package:vm_service/vm_service_io.dart' as vm_io;
 
-import '_callback_io.dart' if (dart.library.html) '_callback_web.dart' as driver_actions;
+import '_callback_io.dart' if (dart.library.html) '_callback_web.dart'
+    as driver_actions;
 import '_extension_io.dart' if (dart.library.html) '_extension_web.dart';
 import 'common.dart';
+import 'src/constants.dart';
+import 'src/reporter.dart';
 
-const String _success = 'success';
+/// Toggles the legacy reporting mechansim where results are only collected
+/// for [testWidgets].
+///
+/// If [run] is called, this will be disabled.
+bool _isUsingLegacyReporting = true;
+
+/// Executes a block that contains tests.
+///
+/// Example Usage:
+/// ```
+/// import 'package:flutter_test/flutter_test.dart';
+/// import 'package:integration_test/integration_test.dart';
+///
+/// void main() => run(_testMain);
+///
+/// void _testMain() {
+///   test('A test', () {
+///     expect(true, true);
+///   });
+/// }
+/// ```
+///
+/// If not explicitly passed, the default [reporter] will send results over the
+/// platform channel to native.
+Future<void> run(
+  FutureOr<void> Function() testMain, {
+  Reporter reporter = const _ReporterImpl(),
+}) async {
+  _isUsingLegacyReporting = false;
+  final IntegrationTestWidgetsFlutterBinding binding = IntegrationTestWidgetsFlutterBinding.ensureInitialized() as IntegrationTestWidgetsFlutterBinding;
+
+  // Pipe detailed exceptions within [testWidgets] to `package:test`.
+  reportTestException = (FlutterErrorDetails details, String testDescription) {
+    registerException('Test $testDescription failed: $details');
+  };
+
+  final Completer<List<TestResult>> resultsCompleter = Completer<List<TestResult>>();
+
+  await directRunTests(
+    testMain,
+    reporterFactory: (Engine engine) => ResultReporter(engine, resultsCompleter),
+  );
+
+  final List<TestResult> results = await resultsCompleter.future;
+
+  binding._updateTestResultState(<String, TestResult>{
+    for (final TestResult result in results)
+      result.methodName: result,
+  });
+  await reporter.report(results);
+}
+
+/// Abstract interface for a result reporter.
+abstract class Reporter {
+  /// Reports test results.
+  ///
+  /// This method will be called at the end of [run] with the [results] of
+  /// running the test suite.
+  Future<void> report(List<TestResult> results);
+}
+
+/// Default implementation of the reporter that sends results over to the
+/// platform side.
+class _ReporterImpl implements Reporter {
+  const _ReporterImpl();
+
+  @override
+  Future<void> report(
+    List<TestResult> results,
+  ) async {
+    try {
+      await IntegrationTestWidgetsFlutterBinding._channel.invokeMethod<void>(
+        'allTestsFinished',
+        <String, dynamic>{
+          'results': <String, String>{
+            for (final TestResult result in results)
+              result.methodName: result is Failure
+                  ? _formatFailureForPlatform(result)
+                  : success
+          }
+        },
+      );
+    } on MissingPluginException {
+      print('Warning: integration_test test plugin was not detected.');
+    }
+  }
+}
+
+String _formatFailureForPlatform(Failure failure) => '${failure.error} ${failure.details}';
 
 /// A subclass of [LiveTestWidgetsFlutterBinding] that reports tests results
 /// on a channel to adapt them to native instrumentation test format.
-class IntegrationTestWidgetsFlutterBinding extends LiveTestWidgetsFlutterBinding implements IntegrationTestResults {
+class IntegrationTestWidgetsFlutterBinding extends LiveTestWidgetsFlutterBinding
-  /// Sets up a listener to report that the tests are finished when everything is
+    implements IntegrationTestResults {
-  /// torn down.
+  /// If [run] is not used, sets up a listener to report that the tests are
+  /// finished when everything is torn down.
+  ///
+  /// This functionality is deprecated – clients are expected to use [run] to
+  /// execute their tests instead.
   IntegrationTestWidgetsFlutterBinding() {
-    // TODO(jackson): Report test results as they arrive
+    if (!_isUsingLegacyReporting) {
+      // TODO(jiahaog): Point users to use the CLI https://github.com/flutter/flutter/issues/66264.
+      print('Using the legacy test result reporter, which will not catch all '
+          'errors thrown in declared tests. Consider wrapping tests with '
+          'https://api.flutter.dev/flutter/integration_test/run.html instead.');
+      return;
+    }
+
     tearDownAll(() async {
-      try {
+      _updateTestResultState(results);
-        // For web integration tests we are not using the
+      await const _ReporterImpl().report(results.values.toList());
-        // `plugins.flutter.io/integration_test`. Mark the tests as complete
-        // before invoking the channel.
-        if (kIsWeb) {
-          if (!_allTestsPassed.isCompleted) {
-            _allTestsPassed.complete(true);
-          }
-        }
-        callbackManager.cleanup();
-        await _channel.invokeMethod<void>(
-          'allTestsFinished',
-          <String, dynamic>{
-            'results': results.map((String name, Object result) {
-              if (result is Failure) {
-                return MapEntry<String, dynamic>(name, result.details);
-              }
-              return MapEntry<String, Object>(name, result);
-            })
-          },
-        );
-      } on MissingPluginException {
-        print('Warning: integration_test test plugin was not detected.');
-      }
-      if (!_allTestsPassed.isCompleted) {
-        _allTestsPassed.complete(true);
-      }
     });
 
-    // TODO(jackson): Report the results individually instead of all at once
-    // See https://github.com/flutter/flutter/issues/38985
     final TestExceptionReporter oldTestExceptionReporter = reportTestException;
-    reportTestException =
+    reportTestException = (FlutterErrorDetails details, String testDescription) {
-        (FlutterErrorDetails details, String testDescription) {
+      results[testDescription] = Failure(
-      results[testDescription] = Failure(testDescription, details.toString());
+        testDescription,
-      if (!_allTestsPassed.isCompleted) {
+        details.toString(),
-        _allTestsPassed.complete(false);
+        error: details.exception,
-      }
+      );
       oldTestExceptionReporter(details, testDescription);
     };
   }
 
+  void _updateTestResultState(Map<String, TestResult> results) {
+    this.results = results;
+    print('Test execution completed: $results');
+
+    _allTestsPassed.complete(!results.values.any((TestResult result) => result is Failure));
+    callbackManager.cleanup();
+  }
+
   @override
   bool get overrideHttpClient => false;
 
@@ -131,11 +218,8 @@ class IntegrationTestWidgetsFlutterBinding extends LiveTestWidgetsFlutterBinding
       MethodChannel('plugins.flutter.io/integration_test');
 
   /// Test results that will be populated after the tests have completed.
-  ///
-  /// Keys are the test descriptions, and values are either [_success] or
-  /// a [Failure].
   @visibleForTesting
-  Map<String, Object> results = <String, Object>{};
+  Map<String, TestResult> results = <String, TestResult>{};
 
   List<Failure> get _failures => results.values.whereType<Failure>().toList();
 
@@ -191,7 +275,7 @@ class IntegrationTestWidgetsFlutterBinding extends LiveTestWidgetsFlutterBinding
       description: description,
       timeout: timeout,
     );
-    results[description] ??= _success;
+    results[description] ??= Success(description);
   }
 
   vm.VmService _vmService;

packages/integration_test/lib/src/constants.dart

@@ -0,0 +1,6 @@
+// Copyright 2014 The Flutter Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+/// Represents a successful test.
+const String success = 'success';

packages/integration_test/lib/src/reporter.dart

@@ -0,0 +1,55 @@
+// Copyright 2014 The Flutter Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+import 'dart:async';
+
+// ignore: implementation_imports
+import 'package:test_api/src/backend/live_test.dart';
+// ignore: implementation_imports
+import 'package:test_core/src/runner/engine.dart';
+// ignore: implementation_imports
+import 'package:test_core/src/runner/reporter.dart';
+
+import '../common.dart';
+import 'constants.dart';
+
+/// A reporter that plugs into [directRunTests] from `package:test_core`.
+class ResultReporter implements Reporter {
+  /// When the [_engine] has completed execution of tests, [_resultsCompleter]
+  /// will be completed with the test results.
+  ResultReporter(this._engine, this._resultsCompleter) {
+    _subscriptions.add(_engine.success.asStream().listen(_onDone));
+  }
+  final Engine _engine;
+  final Completer<List<TestResult>> _resultsCompleter;
+
+  final Set<StreamSubscription<Object>> _subscriptions = <StreamSubscription<Object>>{};
+
+  void _onDone(bool _) {
+    _cancel();
+    final List<TestResult> results = <TestResult>[
+      for (final LiveTest liveTest in _engine.liveTests)
+        liveTest.state.result.name == success
+            ? Success(liveTest.test.name)
+            : Failure(
+                liveTest.test.name,
+                null,
+                errors: liveTest.errors,
+              )
+    ];
+    _resultsCompleter.complete(results);
+  }
+
+  void _cancel() {
+    for (final StreamSubscription<Object> subscription in _subscriptions) {
+      subscription.cancel();
+    }
+    _subscriptions.clear();
+  }
+
+  @override
+  void pause() {}
+  @override
+  void resume() {}
+}

packages/integration_test/pubspec.yaml

@@ -15,22 +15,40 @@ dependencies:
   flutter_test:
     sdk: flutter
   path: 1.8.0-nullsafety.3
+  test_core: 0.3.12-nullsafety.9
   vm_service: 5.2.0
 
+  _fe_analyzer_shared: 7.0.0 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
+  analyzer: 0.39.17 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
   archive: 2.0.13 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
   args: 1.6.0 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
   async: 2.5.0-nullsafety.3 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
   boolean_selector: 2.1.0-nullsafety.3 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
   characters: 1.1.0-nullsafety.5 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
   charcode: 1.2.0-nullsafety.3 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
+  cli_util: 0.2.0 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
   clock: 1.1.0-nullsafety.3 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
   collection: 1.15.0-nullsafety.5 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
   convert: 2.1.1 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
+  coverage: 0.14.1 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
   crypto: 2.1.5 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
+  csslib: 0.16.2 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
   fake_async: 1.2.0-nullsafety.3 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
   file: 6.0.0-nullsafety.4 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
+  glob: 1.2.0 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
+  html: 0.14.0+4 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
+  io: 0.3.4 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
+  js: 0.6.3-nullsafety.3 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
+  logging: 0.11.4 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
   matcher: 0.12.10-nullsafety.3 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
   meta: 1.3.0-nullsafety.6 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
+  node_interop: 1.2.0 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
+  node_io: 1.1.1 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
+  package_config: 1.9.3 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
+  pool: 1.5.0-nullsafety.3 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
+  pub_semver: 1.4.4 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
+  source_map_stack_trace: 2.1.0-nullsafety.4 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
+  source_maps: 0.10.10-nullsafety.3 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
   source_span: 1.8.0-nullsafety.4 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
   stack_trace: 1.10.0-nullsafety.6 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
   stream_channel: 2.1.0-nullsafety.3 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
@@ -40,7 +58,9 @@ dependencies:
   test_api: 0.2.19-nullsafety.6 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
   typed_data: 1.3.0-nullsafety.5 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
   vector_math: 2.1.0-nullsafety.5 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
+  watcher: 0.9.7+15 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
   webdriver: 2.1.2 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
+  yaml: 2.2.1 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
 
 dev_dependencies:
   pedantic: 1.10.0-nullsafety.3
@@ -55,4 +75,4 @@ flutter:
       ios:
         pluginClass: IntegrationTestPlugin
 
-# PUBSPEC CHECKSUM: f9bc
+# PUBSPEC CHECKSUM: a5dd

packages/integration_test/test/binding_fail_test.dart

@@ -1,79 +0,0 @@
-// Copyright 2014 The Flutter Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file.
-
-import 'dart:async';
-import 'dart:convert';
-import 'dart:io';
-
-import 'package:flutter_test/flutter_test.dart';
-import 'package:path/path.dart' as path;
-
-final String bat = Platform.isWindows ? '.bat' : '';
-final String _flutterBin = path.join(Directory.current.parent.parent.parent.path, 'bin', 'flutter$bat');
-const String _integrationResultsPrefix =
-    'IntegrationTestWidgetsFlutterBinding test results:';
-const String _failureExcerpt = r'Expected: <false>\n  Actual: <true>';
-
-Future<void> main() async {
-  group('Integration binding result', () {
-    test('when multiple tests pass', () async {
-      final Map<String, dynamic> results = await _runTest(path.join('test', 'data', 'pass_test_script.dart'));
-
-      expect(
-          results,
-          equals(<String, dynamic>{
-            'passing test 1': 'success',
-            'passing test 2': 'success',
-          }));
-    });
-
-    test('when multiple tests fail', () async {
-      final Map<String, dynamic> results = await _runTest(path.join('test', 'data', 'fail_test_script.dart'));
-
-      expect(results, hasLength(2));
-      expect(results, containsPair('failing test 1', contains(_failureExcerpt)));
-      expect(results, containsPair('failing test 2', contains(_failureExcerpt)));
-    });
-
-    test('when one test passes, then another fails', () async {
-      final Map<String, dynamic> results = await _runTest(path.join('test', 'data', 'pass_then_fail_test_script.dart'));
-
-      expect(results, hasLength(2));
-      expect(results, containsPair('passing test', equals('success')));
-      expect(results, containsPair('failing test', contains(_failureExcerpt)));
-    });
-  });
-}
-
-/// Runs a test script and returns the [IntegrationTestWidgetsFlutterBinding.result].
-///
-/// [scriptPath] is relative to the package root.
-Future<Map<String, dynamic>> _runTest(String scriptPath) async {
-  final Process process =
-      await Process.start(_flutterBin, <String>['test', '--machine', scriptPath]);
-
-  /// In the test [tearDownAll] block, the test results are encoded into JSON and
-  /// are printed with the [_integrationResultsPrefix] prefix.
-  ///
-  /// See the following for the test event spec which we parse the printed lines
-  /// out of: https://github.com/dart-lang/test/blob/master/pkgs/test/doc/json_reporter.md
-  final String testResults = (await process.stdout
-          .transform(utf8.decoder)
-          .expand((String text) => text.split('\n'))
-          .map((String line) {
-            try {
-              return jsonDecode(line) as Map<String, dynamic>;
-            } on FormatException {
-              // Only interested in test events which are JSON.
-            }
-          })
-          .where((Map<String, dynamic> testEvent) =>
-              testEvent != null && testEvent['type'] == 'print')
-          .map((Map<String, dynamic> printEvent) => printEvent['message'] as String)
-          .firstWhere((String message) =>
-              message.startsWith(_integrationResultsPrefix)))
-      .replaceAll(_integrationResultsPrefix, '');
-
-  return jsonDecode(testResults) as Map<String, dynamic>;
-}

packages/integration_test/test/reporter/data/fail_test_script.dart

@@ -2,23 +2,24 @@
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
-import 'dart:convert';
-
 import 'package:integration_test/integration_test.dart';
 import 'package:flutter_test/flutter_test.dart';
 
-Future<void> main() async {
+import '../utils.dart';
+
+void main() {
   final IntegrationTestWidgetsFlutterBinding binding = IntegrationTestWidgetsFlutterBinding.ensureInitialized() as IntegrationTestWidgetsFlutterBinding;
 
-  testWidgets('failing test 1', (WidgetTester tester) async {
+  testWidgets('Failing test 1', (WidgetTester tester) async {
-    expect(true, false);
+    expect(false, true);
   });
 
-  testWidgets('failing test 2', (WidgetTester tester) async {
+  testWidgets('Failing test 2', (WidgetTester tester) async {
-    expect(true, false);
+    expect(false, true);
   });
 
   tearDownAll(() {
-    print('IntegrationTestWidgetsFlutterBinding test results: ${jsonEncode(binding.results)}');
+    print(
+        'IntegrationTestWidgetsFlutterBinding test results: ${testResultsToJson(binding.results)}');
   });
 }

packages/integration_test/test/reporter/data/pass_test_script.dart

@@ -2,24 +2,24 @@
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
-import 'dart:convert';
-
 import 'package:integration_test/integration_test.dart';
 import 'package:flutter_test/flutter_test.dart';
 
-Future<void> main() async {
+import '../utils.dart';
+
+void main() {
   final IntegrationTestWidgetsFlutterBinding binding = IntegrationTestWidgetsFlutterBinding.ensureInitialized() as IntegrationTestWidgetsFlutterBinding;
 
-  testWidgets('passing test 1', (WidgetTester tester) async {
+  testWidgets('Passing test 1', (WidgetTester tester) async {
     expect(true, true);
   });
 
-  testWidgets('passing test 2', (WidgetTester tester) async {
+  testWidgets('Passing test 2', (WidgetTester tester) async {
     expect(true, true);
   });
 
   tearDownAll(() {
     print(
-        'IntegrationTestWidgetsFlutterBinding test results: ${jsonEncode(binding.results)}');
+        'IntegrationTestWidgetsFlutterBinding test results: ${testResultsToJson(binding.results)}');
   });
 }

packages/integration_test/test/reporter/data/pass_then_fail_test_script.dart

@@ -2,24 +2,24 @@
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
-import 'dart:convert';
-
 import 'package:integration_test/integration_test.dart';
 import 'package:flutter_test/flutter_test.dart';
 
-Future<void> main() async {
+import '../utils.dart';
+
+void main() {
   final IntegrationTestWidgetsFlutterBinding binding = IntegrationTestWidgetsFlutterBinding.ensureInitialized() as IntegrationTestWidgetsFlutterBinding;
 
-  testWidgets('passing test', (WidgetTester tester) async {
+  testWidgets('Passing test', (WidgetTester tester) async {
     expect(true, true);
   });
 
-  testWidgets('failing test', (WidgetTester tester) async {
+  testWidgets('Failing test', (WidgetTester tester) async {
-    expect(true, false);
+    expect(false, true);
   });
 
   tearDownAll(() {
     print(
-        'IntegrationTestWidgetsFlutterBinding test results: ${jsonEncode(binding.results)}');
+        'IntegrationTestWidgetsFlutterBinding test results: ${testResultsToJson(binding.results)}');
   });
 }

packages/integration_test/test/reporter/legacy_reporter_test.dart

@@ -0,0 +1,77 @@
+// Copyright 2014 The Flutter Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+import 'dart:async';
+import 'dart:convert';
+import 'dart:io';
+
+import 'package:flutter_test/flutter_test.dart';
+import 'package:integration_test/src/constants.dart';
+
+import 'utils.dart';
+
+// Assumes that the flutter command is in `$PATH`.
+const String _flutterBin = 'flutter';
+const String _integrationResultsPrefix = 'IntegrationTestWidgetsFlutterBinding test results:';
+
+Future<void> main() async {
+  test('When multiple tests pass', () async {
+    final Map<String, dynamic> results = await _runTest('test/reporter/data/pass_test_script.dart');
+
+    expect(results, hasLength(2));
+    expect(results, containsPair('Passing test 1', _isSuccess));
+    expect(results, containsPair('Passing test 2', _isSuccess));
+  });
+
+  test('When multiple tests fail', () async {
+    final Map<String, dynamic> results = await _runTest('test/reporter/data/fail_test_script.dart');
+
+    expect(results, hasLength(2));
+    expect(results, containsPair('Failing test 1', _isSerializedFailure));
+    expect(results, containsPair('Failing test 2', _isSerializedFailure));
+  });
+
+  test('When one test passes, then another fails', () async {
+    final Map<String, dynamic> results = await _runTest('test/reporter/data/pass_then_fail_test_script.dart');
+
+    expect(results, hasLength(2));
+    expect(results, containsPair('Passing test', _isSuccess));
+    expect(results, containsPair('Failing test', _isSerializedFailure));
+  });
+}
+
+/// Runs a test script and returns the [IntegrationTestWidgetsFlutterBinding.result].
+///
+/// [scriptPath] is relative to the package root.
+Future<Map<String, dynamic>> _runTest(String scriptPath) async {
+  final Process process = await Process.start(_flutterBin, <String>['test', '--machine', scriptPath]);
+
+  /// In the test [tearDownAll] block, the test results are encoded into JSON and
+  /// are printed with the [_integrationResultsPrefix] prefix.
+  ///
+  /// See the following for the test event spec which we parse the printed lines
+  /// out of: https://github.com/dart-lang/test/blob/master/pkgs/test/doc/json_reporter.md
+  final String testResults = (await process.stdout
+          .transform(utf8.decoder)
+          .expand((String text) => text.split('\n'))
+          .map<dynamic>((String line) {
+            try {
+              return jsonDecode(line);
+            } on FormatException {
+              // Only interested in test events which are JSON.
+            }
+          })
+          .where((dynamic testEvent) =>
+              testEvent != null && testEvent['type'] == 'print')
+          .map((dynamic printEvent) => printEvent['message'] as String)
+          .firstWhere((String message) =>
+              message.startsWith(_integrationResultsPrefix)))
+      .replaceAll(_integrationResultsPrefix, '');
+
+  return jsonDecode(testResults) as Map<String, dynamic>;
+}
+
+bool _isSuccess(Object object) => object == success;
+
+bool _isSerializedFailure(dynamic object) => object.toString().contains(failureExcerpt);

packages/integration_test/test/reporter/multiple_tests_fail_test.dart

@@ -0,0 +1,29 @@
+// Copyright 2014 The Flutter Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+import 'package:flutter_test/flutter_test.dart';
+import 'package:integration_test/common.dart';
+
+import 'utils.dart';
+
+void main() {
+  test('When multiple tests fail', () async {
+    final List<TestResult> results = await runAndCollectResults(_testMain);
+
+    expect(results, <dynamic>[
+      isFailure('Failing testWidgets()'),
+      isFailure('Failing test()')
+    ]);
+  });
+}
+
+void _testMain() {
+  testWidgets('Failing testWidgets()', (WidgetTester tester) async {
+    expect(false, true);
+  });
+
+  test('Failing test()', () {
+    expect(false, true);
+  });
+}

packages/integration_test/test/reporter/multiple_tests_pass_test.dart

@@ -0,0 +1,29 @@
+// Copyright 2014 The Flutter Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+import 'package:flutter_test/flutter_test.dart';
+import 'package:integration_test/common.dart';
+
+import 'utils.dart';
+
+void main() {
+  test('When multiple tests pass', () async {
+    final List<TestResult> results = await runAndCollectResults(_testMain);
+
+    expect(results, <dynamic>[
+      isSuccess('Passing testWidgets()'),
+      isSuccess('Passing test()')
+    ]);
+  });
+}
+
+void _testMain() {
+  testWidgets('Passing testWidgets()', (WidgetTester tester) async {
+    expect(true, true);
+  });
+
+  test('Passing test()', () {
+    expect(true, true);
+  });
+}

packages/integration_test/test/reporter/pass_then_fail_test.dart

@@ -0,0 +1,29 @@
+// Copyright 2014 The Flutter Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+import 'package:flutter_test/flutter_test.dart';
+import 'package:integration_test/common.dart';
+
+import 'utils.dart';
+
+void main() {
+  test('when one test passes, then another fails', () async {
+    final List<TestResult> results = await runAndCollectResults(_testMain);
+
+    expect(results, <dynamic>[
+      isSuccess('Passing test'),
+      isFailure('Failing test')
+    ]);
+  });
+}
+
+void _testMain() {
+  testWidgets('Passing test', (WidgetTester tester) async {
+    expect(true, true);
+  });
+
+  testWidgets('Failing test', (WidgetTester tester) async {
+    expect(false, true);
+  });
+}

packages/integration_test/test/reporter/utils.dart

@@ -0,0 +1,44 @@
+// Copyright 2014 The Flutter Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+import 'dart:async';
+import 'dart:convert';
+
+import 'package:flutter_test/flutter_test.dart';
+import 'package:integration_test/common.dart';
+import 'package:integration_test/integration_test.dart';
+import 'package:integration_test/src/constants.dart';
+
+const String failureExcerpt = 'Expected: <true>';
+
+dynamic isSuccess(String methodName) => isA<Success>()
+  .having((Success s) => s.methodName, 'methodName', methodName);
+
+dynamic isFailure(String methodName) => isA<Failure>()
+  .having((Failure e) => e.methodName, 'methodName', methodName)
+  .having((Failure e) => e.error.toString(), 'error', contains(failureExcerpt));
+
+
+Future<List<TestResult>> runAndCollectResults(
+  FutureOr<void> Function() testMain,
+) async {
+  final _TestReporter reporter = _TestReporter();
+  await run(testMain, reporter: reporter);
+  return reporter.results;
+}
+
+class _TestReporter implements Reporter {
+  final Completer<List<TestResult>> _resultsCompleter = Completer<List<TestResult>>();
+  Future<List<TestResult>> get results => _resultsCompleter.future;
+
+  @override
+  Future<void> report(List<TestResult> results) async => _resultsCompleter.complete(results);
+}
+
+String testResultsToJson(Map<String, TestResult> results) {
+  return jsonEncode(<String, Object>{
+    for (TestResult result in results.values)
+      result.methodName: result is Failure ? result : success
+  });
+}