[flutter_tools] measure driver success and failure (#69226)

Currently we only have a test that measures if flutter drive can fail. Unfortunately it has been failing for multiple years based on the fact that the error is due to a missing Gradle config.

Add a smoke success test case, and update both tests to use the test API and the tester-device.

dev/bots/test.dart

@@ -239,15 +239,23 @@ Future<void> _runSmokeTests() async {
         expectFailure: true,
         printOutput: false,
       ),
-      runCommand(flutter,
-        <String>['drive', '--use-existing-app', '-t', path.join('test_driver', 'failure.dart')],
-        workingDirectory: path.join(flutterRoot, 'packages', 'flutter_driver'),
-        expectNonZeroExit: true,
-        outputMode: OutputMode.capture,
-      ),
     ],
   );
 
+  // The flutter-tester device cannot be run concurrently in the same project directory.
+  await runCommand(flutter,
+    <String>['drive', '--show-test-device', '-d', 'flutter-tester', '-t', path.join('test_driver', 'success.dart')],
+    workingDirectory: path.join(flutterRoot, 'packages', 'flutter_driver'),
+    expectNonZeroExit: false,
+    outputMode: OutputMode.capture,
+  );
+  await runCommand(flutter,
+    <String>['drive', '--show-test-device', '-d', 'flutter-tester', '-t', path.join('test_driver', 'failure.dart')],
+    workingDirectory: path.join(flutterRoot, 'packages', 'flutter_driver'),
+    expectNonZeroExit: true,
+    outputMode: OutputMode.capture,
+  );
+
   // Verify that we correctly generated the version file.
   final String versionError = await verifyVersion(File(path.join(flutterRoot, 'version')));
   if (versionError != null)

packages/_flutter_web_build_script/pubspec.yaml

@@ -63,7 +63,7 @@ dependencies:
   path: 1.8.0-nullsafety.2 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
   pedantic: 1.10.0-nullsafety.2 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
   pool: 1.5.0-nullsafety.2 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
-  protobuf: 1.0.1 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
+  protobuf: 1.1.0 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
   pub_semver: 1.4.4 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
   pubspec_parse: 0.1.5 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
   quiver: 2.1.4+1 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
@@ -93,4 +93,4 @@ dartdoc:
   # Exclude this package from the hosted API docs.
   nodoc: true
 
-# PUBSPEC CHECKSUM: 92e6
+# PUBSPEC CHECKSUM: 96e6

packages/flutter_driver/pubspec.yaml

@@ -48,5 +48,37 @@ dependencies:
 
 dev_dependencies:
   quiver: 2.1.4+1
+  test: 1.16.0-nullsafety.7
 
-# PUBSPEC CHECKSUM: 7eaf
+  _fe_analyzer_shared: 7.0.0 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
+  analyzer: 0.39.17 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
+  cli_util: 0.2.0 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
+  coverage: 0.14.1 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
+  csslib: 0.16.2 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
+  glob: 1.2.0 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
+  html: 0.14.0+4 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
+  http_multi_server: 2.2.0 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
+  http_parser: 3.1.4 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
+  io: 0.3.4 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
+  js: 0.6.3-nullsafety.2 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
+  logging: 0.11.4 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
+  mime: 0.9.7 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
+  node_interop: 1.1.1 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
+  node_io: 1.1.1 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
+  node_preamble: 1.4.12 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
+  package_config: 1.9.3 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
+  pedantic: 1.10.0-nullsafety.2 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
+  pool: 1.5.0-nullsafety.2 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
+  pub_semver: 1.4.4 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
+  shelf: 0.7.5 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
+  shelf_packages_handler: 2.0.0 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
+  shelf_static: 0.2.8 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
+  shelf_web_socket: 0.2.3 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
+  source_map_stack_trace: 2.1.0-nullsafety.3 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
+  source_maps: 0.10.10-nullsafety.2 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
+  test_core: 0.3.12-nullsafety.7 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
+  watcher: 0.9.7+15 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
+  webkit_inspection_protocol: 0.7.3 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
+  yaml: 2.2.1 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
+
+# PUBSPEC CHECKSUM: 9e1b

packages/flutter_driver/test_driver/failure.dart

@@ -4,6 +4,9 @@
 
 // @dart = 2.8
 
+import 'package:flutter_driver/driver_extension.dart';
+
 void main() {
   // Dummy. Only needed because driver needs an entry point.
+  enableFlutterDriverExtension();
 }

packages/flutter_driver/test_driver/failure_test.dart

@@ -4,8 +4,12 @@
 
 // @dart = 2.8
 
+import '../test/common.dart';
+
 void main() {
   // Intentionally fail the test. We want to see driver return a non-zero exit
   // code when this happens.
-  throw 'Failed';
+  test('it fails a test', () {
+    expect(true, isFalse);
+  });
 }

packages/flutter_driver/test_driver/success.dart

@@ -0,0 +1,12 @@
+// Copyright 2014 The Flutter Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+// @dart = 2.8
+
+import 'package:flutter_driver/driver_extension.dart';
+
+void main() {
+  // Dummy. Only needed because driver needs an entry point.
+  enableFlutterDriverExtension();
+}

packages/flutter_driver/test_driver/success_test.dart

@@ -0,0 +1,14 @@
+// Copyright 2014 The Flutter Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+// @dart = 2.8
+
+import '../test/common.dart';
+
+void main() {
+  // Tooling requires a single trivial test
+  test('it runs a test', () {
+    expect(true, true);
+  });
+}

packages/flutter_tools/pubspec.yaml

@@ -90,7 +90,7 @@ dependencies:
   source_map_stack_trace: 2.1.0-nullsafety.3 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
   source_maps: 0.10.10-nullsafety.2 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
   source_span: 1.8.0-nullsafety.3 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
-  sse: 3.5.0 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
+  sse: 3.6.0 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
   stream_transform: 1.2.0 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
   string_scanner: 1.1.0-nullsafety.2 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
   sync_http: 0.2.0 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
@@ -113,4 +113,4 @@ dartdoc:
   # Exclude this package from the hosted API docs.
   nodoc: true
 
-# PUBSPEC CHECKSUM: b417
+# PUBSPEC CHECKSUM: 5918