Add support for setup steps in flutter customer tests (#135726)

This PR adds support for adding `setup` steps that must be run before the `update` steps in Flutter customer tests.

```
setup=./generate_my_mocks.sh
update=.
test= ...
```

This PR also cleans up the `CustomerTest` class in the spirit of yak-shaving and making the code more robust.

dev/customer_testing/lib/customer_test.dart

@@ -4,6 +4,7 @@
 
 import 'dart:io';
 
+import 'package:collection/collection.dart';
 import 'package:meta/meta.dart';
 
 @immutable
@@ -12,58 +13,50 @@ class CustomerTest {
     final String errorPrefix = 'Could not parse: ${testFile.path}\n';
     final List<String> contacts = <String>[];
     final List<String> fetch = <String>[];
+    final List<String> setup = <String>[];
     final List<Directory> update = <Directory>[];
     final List<String> test = <String>[];
     int? iterations;
     bool hasTests = false;
     for (final String line in testFile.readAsLinesSync().map((String line) => line.trim())) {
-      if (line.isEmpty) {
+      if (line.isEmpty || line.startsWith('#')) {
-        // blank line
+        // Blank line or comment.
-      } else if (line.startsWith('#')) {
+        continue;
-        // comment
+      }
-      } else if (line.startsWith('contact=')) {
+
-        contacts.add(line.substring(8));
+      final bool isUnknownDirective = _TestDirective.values.firstWhereOrNull((_TestDirective d) => line.startsWith(d.name)) == null;
-      } else if (line.startsWith('fetch=')) {
+      if (isUnknownDirective) {
-        fetch.add(line.substring(6));
-      } else if (line.startsWith('update=')) {
-        update.add(Directory(line.substring(7)));
-      } else if (line.startsWith('iterations=')) {
-        if (iterations != null) {
-          throw const FormatException('Cannot specify "iterations" directive multiple times.');
-        }
-        iterations = int.parse(line.substring(11));
-        if (iterations < 1) {
-          throw const FormatException('The "iterations" directive must have a positive integer value.');
-        }
-      } else if (line.startsWith('test=')) {
-        hasTests = true;
-        test.add(line.substring(5));
-      } else if (line.startsWith('test.windows=')) {
-        hasTests = true;
-        if (Platform.isWindows) {
-          test.add(line.substring(13));
-        }
-      } else if (line.startsWith('test.macos=')) {
-        hasTests = true;
-        if (Platform.isMacOS) {
-          test.add(line.substring(11));
-        }
-      } else if (line.startsWith('test.linux=')) {
-        hasTests = true;
-        if (Platform.isLinux) {
-          test.add(line.substring(11));
-        }
-      } else if (line.startsWith('test.posix=')) {
-        hasTests = true;
-        if (Platform.isLinux || Platform.isMacOS) {
-          test.add(line.substring(11));
-        }
-      } else {
         throw FormatException('${errorPrefix}Unexpected directive:\n$line');
       }
+
+      _maybeAddTestConfig(line, directive: _TestDirective.contact, directiveValues: contacts);
+      _maybeAddTestConfig(line, directive: _TestDirective.fetch, directiveValues: fetch);
+      _maybeAddTestConfig(line, directive: _TestDirective.setup, directiveValues: setup, platformAgnostic: false);
+
+      final String updatePrefix = _directive(_TestDirective.update);
+      if (line.startsWith(updatePrefix)) {
+        update.add(Directory(line.substring(updatePrefix.length)));
+      }
+
+      final String iterationsPrefix = _directive(_TestDirective.iterations);
+      if (line.startsWith(iterationsPrefix)) {
+        if (iterations != null) {
+          throw FormatException('Cannot specify "${_TestDirective.iterations.name}" directive multiple times.');
+        }
+        iterations = int.parse(line.substring(iterationsPrefix.length));
+        if (iterations < 1) {
+          throw FormatException('The "${_TestDirective.iterations.name}" directive must have a positive integer value.');
+        }
+      }
+
+      if (line.startsWith(_directive(_TestDirective.test)) || line.startsWith('${_TestDirective.test.name}.')) {
+        hasTests = true;
+      }
+      _maybeAddTestConfig(line, directive: _TestDirective.test, directiveValues: test, platformAgnostic: false);
     }
+
     if (contacts.isEmpty) {
-      throw FormatException('${errorPrefix}No contacts specified. At least one contact e-mail address must be specified.');
+      throw FormatException('${errorPrefix}No "${_TestDirective.contact.name}" directives specified. At least one contact e-mail address must be specified.');
     }
     for (final String email in contacts) {
       if (!email.contains(_email) || email.endsWith('@example.com')) {
@@ -71,33 +64,41 @@ class CustomerTest {
       }
     }
     if (fetch.isEmpty) {
-      throw FormatException('${errorPrefix}No "fetch" directives specified. Two lines are expected: "git clone https://github.com/USERNAME/REPOSITORY.git tests" and "git -C tests checkout HASH".');
+      throw FormatException('${errorPrefix}No "${_TestDirective.fetch.name}" directives specified. Two lines are expected: "git clone https://github.com/USERNAME/REPOSITORY.git tests" and "git -C tests checkout HASH".');
     }
     if (fetch.length < 2) {
-      throw FormatException('${errorPrefix}Only one "fetch" directive specified. Two lines are expected: "git clone https://github.com/USERNAME/REPOSITORY.git tests" and "git -C tests checkout HASH".');
+      throw FormatException('${errorPrefix}Only one "${_TestDirective.fetch.name}" directive specified. Two lines are expected: "git clone https://github.com/USERNAME/REPOSITORY.git tests" and "git -C tests checkout HASH".');
     }
     if (!fetch[0].contains(_fetch1)) {
-      throw FormatException('${errorPrefix}First "fetch" directive does not match expected pattern (expected "git clone https://github.com/USERNAME/REPOSITORY.git tests").');
+      throw FormatException('${errorPrefix}First "${_TestDirective.fetch.name}" directive does not match expected pattern (expected "git clone https://github.com/USERNAME/REPOSITORY.git tests").');
     }
     if (!fetch[1].contains(_fetch2)) {
-      throw FormatException('${errorPrefix}Second "fetch" directive does not match expected pattern (expected "git -C tests checkout HASH").');
+      throw FormatException('${errorPrefix}Second "${_TestDirective.fetch.name}" directive does not match expected pattern (expected "git -C tests checkout HASH").');
     }
     if (update.isEmpty) {
-      throw FormatException('${errorPrefix}No "update" directives specified. At least one directory must be specified. (It can be "." to just upgrade the root of the repository.)');
+      throw FormatException('${errorPrefix}No "${_TestDirective.update.name}" directives specified. At least one directory must be specified. (It can be "." to just upgrade the root of the repository.)');
     }
     if (!hasTests) {
-      throw FormatException('${errorPrefix}No "test" directives specified. At least one command must be specified to run tests.');
+      throw FormatException('${errorPrefix}No "${_TestDirective.test.name}" directives specified. At least one command must be specified to run tests.');
     }
     return CustomerTest._(
       List<String>.unmodifiable(contacts),
       List<String>.unmodifiable(fetch),
+      List<String>.unmodifiable(setup),
       List<Directory>.unmodifiable(update),
       List<String>.unmodifiable(test),
       iterations,
     );
   }
 
-  const CustomerTest._(this.contacts, this.fetch, this.update, this.tests, this.iterations);
+  const CustomerTest._(
+    this.contacts,
+    this.fetch,
+    this.setup,
+    this.update,
+    this.tests,
+    this.iterations,
+  );
 
   // (e-mail regexp from HTML standard)
   static final RegExp _email = RegExp(r"^[a-zA-Z0-9.!#$%&'*+\/=?^_`{|}~-]+@[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?(?:\.[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?)*$");
@@ -106,7 +107,60 @@ class CustomerTest {
 
   final List<String> contacts;
   final List<String> fetch;
+  final List<String> setup;
   final List<Directory> update;
   final List<String> tests;
   final int? iterations;
+
+  static void _maybeAddTestConfig(
+    String line, {
+    required _TestDirective directive,
+    required List<String> directiveValues,
+    bool platformAgnostic = true,
+  }) {
+    final List<_PlatformType> platforms = platformAgnostic
+        ? <_PlatformType>[_PlatformType.all]
+        : _PlatformType.values;
+    for (final _PlatformType platform in platforms) {
+      final String directiveName = _directive(directive, platform: platform);
+      if (line.startsWith(directiveName) && platform.conditionMet) {
+        directiveValues.add(line.substring(directiveName.length));
+      }
+    }
+  }
+
+  static String _directive(
+    _TestDirective directive, {
+    _PlatformType platform = _PlatformType.all,
+  }) {
+    return switch (platform) {
+      _PlatformType.all => '${directive.name}=',
+      _ => '${directive.name}.${platform.name}=',
+    };
+  }
+}
+
+enum _PlatformType {
+  all,
+  windows,
+  macos,
+  linux,
+  posix;
+
+  bool get conditionMet => switch (this) {
+        _PlatformType.all => true,
+        _PlatformType.windows => Platform.isWindows,
+        _PlatformType.macos => Platform.isMacOS,
+        _PlatformType.linux => Platform.isLinux,
+        _PlatformType.posix => Platform.isLinux || Platform.isMacOS,
+      };
+}
+
+enum _TestDirective {
+  contact,
+  fetch,
+  setup,
+  update,
+  test,
+  iterations,
 }

dev/customer_testing/lib/runner.dart

@@ -108,6 +108,21 @@ Future<bool> runTests({
       }
       if (success) {
         final Directory customerRepo = Directory(path.join(checkout.path, 'tests'));
+        for (final String setupCommand in instructions.setup) {
+          if (verbose) {
+            print('Running setup command: $setupCommand');
+          }
+          success = await shell(
+            setupCommand,
+            customerRepo,
+            verbose: verbose,
+            failedCallback: printHeader,
+          );
+          if (!success) {
+            failure('Setup command failed: $setupCommand');
+            break;
+          }
+        }
         for (final Directory updateDirectory in instructions.update) {
           final Directory resolvedUpdateDirectory = Directory(path.join(customerRepo.path, updateDirectory.path));
           if (verbose) {

dev/customer_testing/test/customer_test_test.dart

@@ -16,6 +16,11 @@ void main() {
 contact=abc@gmail.com
 fetch=git clone https://github.com/flutter/cocoon.git tests
 fetch=git -C tests checkout abc123
+setup=flutter --version
+setup.windows=flutter doctor
+setup.posix=flutter -h
+setup.linux=flutter analyze -h
+setup.macos=flutter build -h
 update=.
 # Runs flutter analyze, flutter test, and builds web platform
 test.posix=./test_utilities/bin/flutter_test_runner.sh app_flutter
@@ -30,7 +35,11 @@ test.windows=.\test_utilities\bin\flutter_test_runner.bat repo_dashboard
         test.fetch,
         containsAllInOrder(
             <String>['git clone https://github.com/flutter/cocoon.git tests', 'git -C tests checkout abc123']));
+    expect(test.setup.first, 'flutter --version');
     if (Platform.isLinux || Platform.isMacOS) {
+      expect(test.setup.length, 3);
+      expect(test.setup[1], 'flutter -h');
+      expect(test.setup[2], Platform.isLinux ? 'flutter analyze -h' : 'flutter build -h');
       expect(
         test.tests,
         containsAllInOrder(<String>[
@@ -39,6 +48,8 @@ test.windows=.\test_utilities\bin\flutter_test_runner.bat repo_dashboard
         ]),
       );
     } else if (Platform.isWindows) {
+      expect(test.setup.length, 2);
+      expect(test.setup[1], 'flutter doctor');
       expect(test.tests, containsAllInOrder(<String>['.\test_utilities\bin\flutter_test_runner.bat repo_dashboard']));
     }
   });