Migrate customer_testing to sharded tests. (#138659)

This will allow Dart Team to run customer tests as part of monorepo and will be a step forward to remove ad_hoc tests.

Bug: https://github.com/dart-lang/sdk/issues/51042
Bug: https://github.com/flutter/flutter/issues/115476

.ci.yaml

@@ -376,13 +376,12 @@ targets:
   - name: Linux customer_testing
     enabled_branches:
       - master
-    recipe: flutter/flutter
+    recipe: flutter/flutter_drone
     timeout: 60
     properties:
+      shard: customer_testing
       tags: >
-        ["framework", "hostonly", "linux"]
+        ["framework", "hostonly", "shard", "linux"]
-      validation: customer_testing
-      validation_name: Customer testing
 
   - name: Linux docs_publish
     recipe: flutter/flutter
@@ -3099,13 +3098,12 @@ targets:
   - name: Mac customer_testing
     enabled_branches:
       - master
-    recipe: flutter/flutter
+    recipe: flutter/flutter_drone
     timeout: 60
     properties:
-      validation: customer_testing
+      shard: customer_testing
-      validation_name: Customer testing
       tags: >
-        ["framework", "hostonly", "mac"]
+        ["framework", "hostonly", "shard", "mac"]
 
   - name: Mac dart_plugin_registry_test
     recipe: devicelab/devicelab_drone
@@ -4691,13 +4689,12 @@ targets:
   - name: Windows customer_testing
     enabled_branches:
       - master
-    recipe: flutter/flutter
+    recipe: flutter/flutter_drone
     timeout: 60
     properties:
-      validation: customer_testing
+      shard: customer_testing
-      validation_name: Customer testing
       tags: >
-        ["framework", "hostonly", "windows"]
+        ["framework", "hostonly", "shard", "windows"]
 
   - name: Windows framework_tests_libraries
     recipe: flutter/flutter_drone

TESTOWNERS

@@ -297,8 +297,6 @@
 ## Host only framework tests
 # Linux analyze
 /dev/bots/analyze.dart @HansMuller @flutter/framework
-# Linux/Mac/Windows customer_testing
-/dev/customer_testing/run_tests.dart @HansMuller @flutter/framework
 # Linux docs
 # Linux docs_test
 # Linux docs_publish
@@ -326,10 +324,12 @@
 # build_tests @eliasyishak @flutter/tool
 # ci_yaml flutter roller @caseyhillers @flutter/infra
 # coverage @godofredoc @flutter/infra
+# customer_testing @HansMuller @flutter/framework
 # flutter_packaging @godofredoc @flutter/infra
 # flutter_plugins @stuartmorgan @flutter/plugin
 # framework_tests @HansMuller @flutter/framework
 # fuchsia_precache @christopherfujino @flutter/tool
+# realm_checker @jacksongardner @flutter/tool
 # skp_generator @Hixie
 # test_ownership @keyonghan
 # tool_host_cross_arch_tests @andrewkolos @flutter/tool
@@ -341,4 +341,3 @@
 # web_long_running_tests @yjbanov @flutter/web
 # web_tests @yjbanov @flutter/web
 # web_tool_tests @eliasyishak @flutter/tool
-# realm_checker @jacksongardner @flutter/tool

dev/bots/run_command.dart

@@ -89,11 +89,12 @@ Future<Command> startCommand(String executable, List<String> arguments, {
   bool Function(String)? removeLine,
   void Function(String, io.Process)? outputListener,
 }) async {
-  final String commandDescription = '${path.relative(executable, from: workingDirectory)} ${arguments.join(' ')}';
   final String relativeWorkingDir = path.relative(workingDirectory ?? io.Directory.current.path);
+  final String commandDescription = '${path.relative(executable, from: workingDirectory)} ${arguments.join(' ')}';
   print('RUNNING: cd $cyan$relativeWorkingDir$reset; $green$commandDescription$reset');
 
   final Stopwatch time = Stopwatch()..start();
+  print('workingDirectory: $workingDirectory, executable: $executable, arguments: $arguments');
   final io.Process process = await io.Process.start(executable, arguments,
     workingDirectory: workingDirectory,
     environment: environment,
@@ -160,7 +161,7 @@ Future<CommandResult> runCommand(String executable, List<String> arguments, {
   void Function(String, io.Process)? outputListener,
 }) async {
   final String commandDescription = '${path.relative(executable, from: workingDirectory)} ${arguments.join(' ')}';
-  final String relativeWorkingDir = path.relative(workingDirectory ?? io.Directory.current.path);
+  final String relativeWorkingDir = workingDirectory ?? path.relative(io.Directory.current.path);
 
   final Command command = await startCommand(executable, arguments,
     workingDirectory: workingDirectory,

dev/bots/test.dart

@@ -263,6 +263,7 @@ Future<void> main(List<String> args) async {
       'flutter_plugins': _runFlutterPackagesTests,
       'skp_generator': _runSkpGeneratorTests,
       'realm_checker': _runRealmCheckerTest,
+      'customer_testing': _runCustomerTesting,
       kTestHarnessShardName: _runTestHarnessTests, // Used for testing this script; also run as part of SHARD=framework_tests, SUBSHARD=misc.
     });
   } catch (error, stackTrace) {
@@ -1547,6 +1548,46 @@ Future<void> _runFlutterPackagesTests() async {
   });
 }
 
+// Runs customer_testing.
+Future<void> _runCustomerTesting() async {
+  printProgress('${green}Running customer testing$reset');
+  await runCommand(
+    'git',
+    <String>[
+      'fetch',
+      'origin',
+      'master',
+    ],
+    workingDirectory: flutterRoot,
+  );
+  await runCommand(
+    'git',
+    <String>[
+      'checkout',
+      'master',
+    ],
+    workingDirectory: flutterRoot,
+  );
+  final Map<String, String> env = Platform.environment;
+  final String? revision = env['REVISION'];
+  if (revision != null) {
+    await runCommand(
+      'git',
+      <String>[
+        'checkout',
+        revision,
+      ],
+      workingDirectory: flutterRoot,
+    );
+  }
+  final String winScript = path.join(flutterRoot, 'dev', 'customer_testing', 'ci.bat');
+  await runCommand(
+    Platform.isWindows? winScript: './ci.sh',
+    <String>[],
+    workingDirectory: path.join(flutterRoot, 'dev', 'customer_testing'),
+  );
+}
+
 /// Runs the skp_generator from the flutter/tests repo.
 ///
 /// See also the customer_tests shard.

dev/customer_testing/ci.bat

@@ -19,7 +19,7 @@ ECHO.
 ECHO Finding correct version of customer tests...
 CMD /S /C "IF EXIST "..\..\bin\cache\pkg\tests\" RMDIR /S /Q ..\..\bin\cache\pkg\tests"
 git clone https://github.com/flutter/tests.git ..\..\bin\cache\pkg\tests
-FOR /F "usebackq tokens=*" %%a IN (`dart --enable-asserts ..\tools\bin\find_commit.dart ..\..\bin\cache\pkg\tests`) DO git -C ..\..\bin\cache\pkg\tests checkout %%a
+FOR /F "usebackq tokens=*" %%a IN (`dart --enable-asserts ..\tools\bin\find_commit.dart . master ..\..\bin\cache\pkg\tests main`) DO git -C ..\..\bin\cache\pkg\tests checkout %%a
 
 ECHO.
 ECHO Running tests...

dev/customer_testing/ci.sh

@@ -31,7 +31,7 @@ dart pub get
 # shard, it should still pass, even if we rolled one of the tests.)
 rm -rf ../../bin/cache/pkg/tests
 git clone https://github.com/flutter/tests.git ../../bin/cache/pkg/tests
-git -C ../../bin/cache/pkg/tests checkout `dart --enable-asserts ../tools/bin/find_commit.dart ../../bin/cache/pkg/tests`
+git -C ../../bin/cache/pkg/tests checkout `dart --enable-asserts ../tools/bin/find_commit.dart . master ../../bin/cache/pkg/tests main`
 
 # Finally, run the tests.
 dart --enable-asserts run_tests.dart --verbose --skip-on-fetch-failure --skip-template ../../bin/cache/pkg/tests/registry/*.test

dev/tools/bin/find_commit.dart

@@ -68,24 +68,24 @@ String git(String workingDirectory, List<String> arguments, {bool allowFailure =
 }
 
 void main(List<String> arguments) {
-  if (arguments.isEmpty || arguments.length > 2 || arguments.contains('--help') || arguments.contains('-h')) {
+  if (arguments.isEmpty || arguments.length != 4 || arguments.contains('--help') || arguments.contains('-h')) {
     print(
-      'Usage: dart find_commit.dart [<path-to-primary-repo>] <path-to-secondary-repo>\n'
+      'Usage: dart find_commit.dart <path-to-primary-repo> <primary-trunk> <path-to-secondary-repo> <secondary-branch>\n'
       'This script will find the commit in the secondary repo that was contemporary\n'
       'when the commit in the primary repo was created. If that commit is on a\n'
-      "branch, then the date of the branch's last merge is used instead.\n"
+      "branch, then the date of the branch's last merge is used instead."
-      'If <path-to-primary-repo> is omitted, the current directory is used for the\n'
-      'primary repo.'
     );
   } else {
-    final String primaryRepo = arguments.length == 1 ? '.' : arguments.first;
+    final String primaryRepo = arguments.first;
-    final String secondaryRepo = arguments.last;
+    final String primaryTrunk = arguments[1];
+    final String secondaryRepo = arguments[2];
+    final String secondaryBranch = arguments.last;
     print(findCommit(
       primaryRepoDirectory: primaryRepo,
       primaryBranch: git(primaryRepo, <String>['rev-parse', '--abbrev-ref', 'HEAD']).trim(),
-      primaryTrunk: 'master',
+      primaryTrunk: primaryTrunk,
       secondaryRepoDirectory: secondaryRepo,
-      secondaryBranch: 'master',
+      secondaryBranch: secondaryBranch,
     ).trim());
   }
 }