From fe7299d318431d9a6aeb9fab0832e7f20bcf2d1a Mon Sep 17 00:00:00 2001 From: godofredoc Date: Mon, 27 Nov 2023 16:41:53 -0800 Subject: [PATCH] Migrate customer_testing to sharded tests. (#138659) This will allow Dart Team to run customer tests as part of monorepo and will be a step forward to remove ad_hoc tests. Bug: https://github.com/dart-lang/sdk/issues/51042 Bug: https://github.com/flutter/flutter/issues/115476 --- .ci.yaml | 21 ++++++++--------- TESTOWNERS | 5 ++--- dev/bots/run_command.dart | 5 +++-- dev/bots/test.dart | 41 ++++++++++++++++++++++++++++++++++ dev/customer_testing/ci.bat | 2 +- dev/customer_testing/ci.sh | 2 +- dev/tools/bin/find_commit.dart | 18 +++++++-------- 7 files changed, 66 insertions(+), 28 deletions(-) diff --git a/.ci.yaml b/.ci.yaml index 40335bc967..6534fa6519 100644 --- a/.ci.yaml +++ b/.ci.yaml @@ -376,13 +376,12 @@ targets: - name: Linux customer_testing enabled_branches: - master - recipe: flutter/flutter + recipe: flutter/flutter_drone timeout: 60 properties: + shard: customer_testing tags: > - ["framework", "hostonly", "linux"] - validation: customer_testing - validation_name: Customer testing + ["framework", "hostonly", "shard", "linux"] - name: Linux docs_publish recipe: flutter/flutter @@ -3099,13 +3098,12 @@ targets: - name: Mac customer_testing enabled_branches: - master - recipe: flutter/flutter + recipe: flutter/flutter_drone timeout: 60 properties: - validation: customer_testing - validation_name: Customer testing + shard: customer_testing tags: > - ["framework", "hostonly", "mac"] + ["framework", "hostonly", "shard", "mac"] - name: Mac dart_plugin_registry_test recipe: devicelab/devicelab_drone @@ -4691,13 +4689,12 @@ targets: - name: Windows customer_testing enabled_branches: - master - recipe: flutter/flutter + recipe: flutter/flutter_drone timeout: 60 properties: - validation: customer_testing - validation_name: Customer testing + shard: customer_testing tags: > - ["framework", "hostonly", "windows"] + ["framework", "hostonly", "shard", "windows"] - name: Windows framework_tests_libraries recipe: flutter/flutter_drone diff --git a/TESTOWNERS b/TESTOWNERS index 2e66e0a51c..deb1ebf399 100644 --- a/TESTOWNERS +++ b/TESTOWNERS @@ -297,8 +297,6 @@ ## Host only framework tests # Linux analyze /dev/bots/analyze.dart @HansMuller @flutter/framework -# Linux/Mac/Windows customer_testing -/dev/customer_testing/run_tests.dart @HansMuller @flutter/framework # Linux docs # Linux docs_test # Linux docs_publish @@ -326,10 +324,12 @@ # build_tests @eliasyishak @flutter/tool # ci_yaml flutter roller @caseyhillers @flutter/infra # coverage @godofredoc @flutter/infra +# customer_testing @HansMuller @flutter/framework # flutter_packaging @godofredoc @flutter/infra # flutter_plugins @stuartmorgan @flutter/plugin # framework_tests @HansMuller @flutter/framework # fuchsia_precache @christopherfujino @flutter/tool +# realm_checker @jacksongardner @flutter/tool # skp_generator @Hixie # test_ownership @keyonghan # tool_host_cross_arch_tests @andrewkolos @flutter/tool @@ -341,4 +341,3 @@ # web_long_running_tests @yjbanov @flutter/web # web_tests @yjbanov @flutter/web # web_tool_tests @eliasyishak @flutter/tool -# realm_checker @jacksongardner @flutter/tool diff --git a/dev/bots/run_command.dart b/dev/bots/run_command.dart index f3af0f16cf..e32b0f8ba1 100644 --- a/dev/bots/run_command.dart +++ b/dev/bots/run_command.dart @@ -89,11 +89,12 @@ Future startCommand(String executable, List arguments, { bool Function(String)? removeLine, void Function(String, io.Process)? outputListener, }) async { - final String commandDescription = '${path.relative(executable, from: workingDirectory)} ${arguments.join(' ')}'; final String relativeWorkingDir = path.relative(workingDirectory ?? io.Directory.current.path); + final String commandDescription = '${path.relative(executable, from: workingDirectory)} ${arguments.join(' ')}'; print('RUNNING: cd $cyan$relativeWorkingDir$reset; $green$commandDescription$reset'); final Stopwatch time = Stopwatch()..start(); + print('workingDirectory: $workingDirectory, executable: $executable, arguments: $arguments'); final io.Process process = await io.Process.start(executable, arguments, workingDirectory: workingDirectory, environment: environment, @@ -160,7 +161,7 @@ Future runCommand(String executable, List arguments, { void Function(String, io.Process)? outputListener, }) async { final String commandDescription = '${path.relative(executable, from: workingDirectory)} ${arguments.join(' ')}'; - final String relativeWorkingDir = path.relative(workingDirectory ?? io.Directory.current.path); + final String relativeWorkingDir = workingDirectory ?? path.relative(io.Directory.current.path); final Command command = await startCommand(executable, arguments, workingDirectory: workingDirectory, diff --git a/dev/bots/test.dart b/dev/bots/test.dart index b13d851540..06f473e29e 100644 --- a/dev/bots/test.dart +++ b/dev/bots/test.dart @@ -263,6 +263,7 @@ Future main(List args) async { 'flutter_plugins': _runFlutterPackagesTests, 'skp_generator': _runSkpGeneratorTests, 'realm_checker': _runRealmCheckerTest, + 'customer_testing': _runCustomerTesting, kTestHarnessShardName: _runTestHarnessTests, // Used for testing this script; also run as part of SHARD=framework_tests, SUBSHARD=misc. }); } catch (error, stackTrace) { @@ -1547,6 +1548,46 @@ Future _runFlutterPackagesTests() async { }); } +// Runs customer_testing. +Future _runCustomerTesting() async { + printProgress('${green}Running customer testing$reset'); + await runCommand( + 'git', + [ + 'fetch', + 'origin', + 'master', + ], + workingDirectory: flutterRoot, + ); + await runCommand( + 'git', + [ + 'checkout', + 'master', + ], + workingDirectory: flutterRoot, + ); + final Map env = Platform.environment; + final String? revision = env['REVISION']; + if (revision != null) { + await runCommand( + 'git', + [ + 'checkout', + revision, + ], + workingDirectory: flutterRoot, + ); + } + final String winScript = path.join(flutterRoot, 'dev', 'customer_testing', 'ci.bat'); + await runCommand( + Platform.isWindows? winScript: './ci.sh', + [], + workingDirectory: path.join(flutterRoot, 'dev', 'customer_testing'), + ); +} + /// Runs the skp_generator from the flutter/tests repo. /// /// See also the customer_tests shard. diff --git a/dev/customer_testing/ci.bat b/dev/customer_testing/ci.bat index 3221f9a0c8..8afdafbe61 100644 --- a/dev/customer_testing/ci.bat +++ b/dev/customer_testing/ci.bat @@ -19,7 +19,7 @@ ECHO. ECHO Finding correct version of customer tests... CMD /S /C "IF EXIST "..\..\bin\cache\pkg\tests\" RMDIR /S /Q ..\..\bin\cache\pkg\tests" git clone https://github.com/flutter/tests.git ..\..\bin\cache\pkg\tests -FOR /F "usebackq tokens=*" %%a IN (`dart --enable-asserts ..\tools\bin\find_commit.dart ..\..\bin\cache\pkg\tests`) DO git -C ..\..\bin\cache\pkg\tests checkout %%a +FOR /F "usebackq tokens=*" %%a IN (`dart --enable-asserts ..\tools\bin\find_commit.dart . master ..\..\bin\cache\pkg\tests main`) DO git -C ..\..\bin\cache\pkg\tests checkout %%a ECHO. ECHO Running tests... diff --git a/dev/customer_testing/ci.sh b/dev/customer_testing/ci.sh index dba10d3156..48b42a8353 100755 --- a/dev/customer_testing/ci.sh +++ b/dev/customer_testing/ci.sh @@ -31,7 +31,7 @@ dart pub get # shard, it should still pass, even if we rolled one of the tests.) rm -rf ../../bin/cache/pkg/tests git clone https://github.com/flutter/tests.git ../../bin/cache/pkg/tests -git -C ../../bin/cache/pkg/tests checkout `dart --enable-asserts ../tools/bin/find_commit.dart ../../bin/cache/pkg/tests` +git -C ../../bin/cache/pkg/tests checkout `dart --enable-asserts ../tools/bin/find_commit.dart . master ../../bin/cache/pkg/tests main` # Finally, run the tests. dart --enable-asserts run_tests.dart --verbose --skip-on-fetch-failure --skip-template ../../bin/cache/pkg/tests/registry/*.test diff --git a/dev/tools/bin/find_commit.dart b/dev/tools/bin/find_commit.dart index 6cc43cde7f..33289f90f3 100644 --- a/dev/tools/bin/find_commit.dart +++ b/dev/tools/bin/find_commit.dart @@ -68,24 +68,24 @@ String git(String workingDirectory, List arguments, {bool allowFailure = } void main(List arguments) { - if (arguments.isEmpty || arguments.length > 2 || arguments.contains('--help') || arguments.contains('-h')) { + if (arguments.isEmpty || arguments.length != 4 || arguments.contains('--help') || arguments.contains('-h')) { print( - 'Usage: dart find_commit.dart [] \n' + 'Usage: dart find_commit.dart \n' 'This script will find the commit in the secondary repo that was contemporary\n' 'when the commit in the primary repo was created. If that commit is on a\n' - "branch, then the date of the branch's last merge is used instead.\n" - 'If is omitted, the current directory is used for the\n' - 'primary repo.' + "branch, then the date of the branch's last merge is used instead." ); } else { - final String primaryRepo = arguments.length == 1 ? '.' : arguments.first; - final String secondaryRepo = arguments.last; + final String primaryRepo = arguments.first; + final String primaryTrunk = arguments[1]; + final String secondaryRepo = arguments[2]; + final String secondaryBranch = arguments.last; print(findCommit( primaryRepoDirectory: primaryRepo, primaryBranch: git(primaryRepo, ['rev-parse', '--abbrev-ref', 'HEAD']).trim(), - primaryTrunk: 'master', + primaryTrunk: primaryTrunk, secondaryRepoDirectory: secondaryRepo, - secondaryBranch: 'master', + secondaryBranch: secondaryBranch, ).trim()); } }