diff --git a/dev/customer_testing/README.md b/dev/customer_testing/README.md new file mode 100644 index 0000000000..0b21722a2f --- /dev/null +++ b/dev/customer_testing/README.md @@ -0,0 +1,16 @@ +# customer_testing + +This tool checks out at the commit SHA +specified in [`tests.version`](tests.version), and runs the tests registered to +verify that end-user apps and libraries are working at the current tip-of-tree +of Flutter. + +To (locally) test a specific SHA, use `ci.dart`: + +```sh +cd dev/customer_testing +dart ci.dart [sha] +``` + +Or, to update the SHA for our CI, edit and send a PR for +[`tests.version`](tests.version). diff --git a/dev/customer_testing/ci.bat b/dev/customer_testing/ci.bat index 56cf7db139..1536db554a 100644 --- a/dev/customer_testing/ci.bat +++ b/dev/customer_testing/ci.bat @@ -8,20 +8,16 @@ REM This should match the ci.sh file in this directory. REM This is called from the LUCI recipes: REM https://github.com/flutter/flutter/blob/main/dev/bots/suite_runners/run_customer_testing_tests.dart +REM This script does not assume that "flutter update-packages" has been +REM run, to allow CIs to save time by skipping that steps since it's +REM largely not needed to run the flutter/tests tests. +REM +REM However, we do need to update this directory. +SETLOCAL +cd /d %~dp0 ECHO. ECHO Updating pub packages... CALL dart pub get -CD ..\tools -CALL dart pub get -CD ..\customer_testing -ECHO. -ECHO Finding correct version of customer tests... -CMD /S /C "IF EXIST "..\..\bin\cache\pkg\tests\" RMDIR /S /Q ..\..\bin\cache\pkg\tests" -git clone https://github.com/flutter/tests.git ..\..\bin\cache\pkg\tests -FOR /F "usebackq tokens=*" %%a IN (`dart --enable-asserts ..\tools\bin\find_commit.dart . master ..\..\bin\cache\pkg\tests main`) DO git -C ..\..\bin\cache\pkg\tests checkout %%a - -ECHO. -ECHO Running tests... -CD ..\..\bin\cache\pkg\tests -CALL dart --enable-asserts ..\..\..\..\dev\customer_testing\run_tests.dart --verbose --skip-on-fetch-failure --skip-template registry/*.test +REM Run the cross-platform script. +CALL ..\..\bin\dart.bat ci.dart diff --git a/dev/customer_testing/ci.dart b/dev/customer_testing/ci.dart new file mode 100644 index 0000000000..024e706749 --- /dev/null +++ b/dev/customer_testing/ci.dart @@ -0,0 +1,91 @@ +// Copyright 2014 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'dart:io' as io; + +import 'package:path/path.dart' as p; + +/// To run this script, either: +/// +/// ```sh +/// cd dev/customer_testing +/// dart ci.dart [sha] +/// ``` +/// +/// Or: +/// +/// ```sh +/// ./dev/customer_testing/ci.sh +/// ./dev/customer_testing/ci.bat +/// ``` +void main(List args) async { + final String sha; + if (args.isEmpty) { + sha = io.File('tests.version').readAsStringSync().trim(); + } else if (args.length == 1) { + sha = args.first; + } else { + io.stderr.writeln('Usage: dart ci.dart [sha]'); + io.exitCode = 1; + return; + } + + final String flutterRootPath = p.canonicalize('../../'); + final io.Directory testsCacheDir = io.Directory( + p.join(flutterRootPath, 'bin', 'cache', 'pkg', 'tests'), + ); + + if (testsCacheDir.existsSync()) { + io.stderr.writeln('Cleaning up existing repo: ${testsCacheDir.path}'); + testsCacheDir.deleteSync(recursive: true); + } + + io.stderr.writeln('Cloning flutter/tests'); + final io.Process clone = await io.Process.start('git', [ + 'clone', + '--depth', + '1', + 'https://github.com/flutter/tests.git', + testsCacheDir.path, + ], mode: io.ProcessStartMode.inheritStdio); + if ((await clone.exitCode) != 0) { + io.exitCode = 1; + return; + } + + io.stderr.writeln('Fetching/checking out $sha'); + final io.Process fetch = await io.Process.start( + 'git', + ['fetch', 'origin', sha], + mode: io.ProcessStartMode.inheritStdio, + workingDirectory: testsCacheDir.path, + ); + if ((await fetch.exitCode) != 0) { + io.exitCode = 1; + return; + } + final io.Process checkout = await io.Process.start( + 'git', + ['checkout', sha], + mode: io.ProcessStartMode.inheritStdio, + workingDirectory: testsCacheDir.path, + ); + if ((await checkout.exitCode) != 0) { + io.exitCode = 1; + return; + } + + io.stderr.writeln('Running tests...'); + final io.Process test = await io.Process.start('dart', [ + '--enable-asserts', + 'run_tests.dart', + '--skip-on-fetch-failure', + '--skip-template', + p.posix.joinAll([...p.split(testsCacheDir.path), 'registry', '*.test']), + ], mode: io.ProcessStartMode.inheritStdio); + if ((await test.exitCode) != 0) { + io.exitCode = 1; + return; + } +} diff --git a/dev/customer_testing/ci.sh b/dev/customer_testing/ci.sh index 5b6b28c601..c9c8d4a7f0 100755 --- a/dev/customer_testing/ci.sh +++ b/dev/customer_testing/ci.sh @@ -8,30 +8,28 @@ # This is called from the LUCI recipes: # https://github.com/flutter/flutter/blob/main/dev/bots/suite_runners/run_customer_testing_tests.dart -set -ex +set -e + +function script_location() { + local script_location="${BASH_SOURCE[0]}" + # Resolve symlinks + while [[ -h "$script_location" ]]; do + DIR="$(cd -P "$( dirname "$script_location")" >/dev/null && pwd)" + script_location="$(readlink "$script_location")" + [[ "$script_location" != /* ]] && script_location="$DIR/$script_location" + done + cd -P "$(dirname "$script_location")" >/dev/null && pwd +} + +# So that users can run this script from anywhere and it will work as expected. +cd "$(script_location)" # This script does not assume that "flutter update-packages" has been # run, to allow CIs to save time by skipping that steps since it's # largely not needed to run the flutter/tests tests. # -# However, we do need to update this directory and the tools directory. +# However, we do need to update this directory. dart pub get -(cd ../tools; dart pub get) # used for find_commit.dart below -# Next we need to update the flutter/tests checkout. -# -# We use find_commit.dart so that we pull the version of flutter/tests -# that was contemporary when the branch we are on was created. That -# way, we can still run the tests on long-lived branches without being -# affected by breaking changes on trunk causing changes to the tests -# that wouldn't work on the long-lived branch. -# -# (This also prevents trunk from suddenly failing when tests are -# revved on flutter/tests -- if you rerun a passing customer_tests -# shard, it should still pass, even if we rolled one of the tests.) -rm -rf ../../bin/cache/pkg/tests -git clone https://github.com/flutter/tests.git ../../bin/cache/pkg/tests -git -C ../../bin/cache/pkg/tests checkout `dart --enable-asserts ../tools/bin/find_commit.dart . master ../../bin/cache/pkg/tests main` - -# Finally, run the tests. -dart --enable-asserts run_tests.dart --skip-on-fetch-failure --skip-template ../../bin/cache/pkg/tests/registry/*.test +# Run the cross-platform script. +../../bin/dart run ci.dart diff --git a/dev/customer_testing/run_tests.dart b/dev/customer_testing/run_tests.dart index 1509324ab8..509e908b7e 100644 --- a/dev/customer_testing/run_tests.dart +++ b/dev/customer_testing/run_tests.dart @@ -76,6 +76,10 @@ Future run(List arguments) async { .where((File file) => !skipTemplate || path.basename(file.path) != 'template.test') .toList(); + if (files.isEmpty && parsedArguments.rest.isNotEmpty) { + print('No files resolved from glob(s): ${parsedArguments.rest}'); + } + if (help || repeat == null || files.isEmpty || diff --git a/dev/customer_testing/tests.version b/dev/customer_testing/tests.version new file mode 100644 index 0000000000..0cdbc13e6e --- /dev/null +++ b/dev/customer_testing/tests.version @@ -0,0 +1 @@ +b4cc097211814cfb0e1f08affb4574cf7de6b1cd diff --git a/dev/tools/bin/find_commit.dart b/dev/tools/bin/find_commit.dart deleted file mode 100644 index 9c79d4bf06..0000000000 --- a/dev/tools/bin/find_commit.dart +++ /dev/null @@ -1,130 +0,0 @@ -// Copyright 2014 The Flutter Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -// This script looks at the current commit and branch of the git repository in -// which it was run, and finds the contemporary commit in the master branch of -// another git repository, whose path is provided on the command line. The -// contemporary commit is the one that was public at the time of the last commit -// on the master branch before the current commit's branch was created. - -import 'dart:io'; - -const bool debugLogging = false; - -void log(String message) { - if (debugLogging) { - print(message); - } -} - -const String _commitTimestampFormat = '--format=%cI'; -DateTime _parseTimestamp(String line) => DateTime.parse(line.trim()); -int _countLines(String output) => - output.trim().split('/n').where((String line) => line.isNotEmpty).length; - -String findCommit({ - required String primaryRepoDirectory, - required String primaryBranch, - required String primaryTrunk, - required String secondaryRepoDirectory, - required String secondaryBranch, -}) { - final DateTime anchor; - if (primaryBranch == primaryTrunk) { - log('on $primaryTrunk, using last commit time'); - anchor = _parseTimestamp( - git(primaryRepoDirectory, [ - 'log', - _commitTimestampFormat, - '--max-count=1', - primaryBranch, - '--', - ]), - ); - } else { - final String mergeBase = - git(primaryRepoDirectory, [ - 'merge-base', - primaryBranch, - primaryTrunk, - ], allowFailure: true).trim(); - if (mergeBase.isEmpty) { - throw StateError( - 'Branch $primaryBranch does not seem to have a common history with trunk $primaryTrunk.', - ); - } - anchor = _parseTimestamp( - git(primaryRepoDirectory, [ - 'log', - _commitTimestampFormat, - '--max-count=1', - mergeBase, - '--', - ]), - ); - if (debugLogging) { - final int missingTrunkCommits = _countLines( - git(primaryRepoDirectory, ['rev-list', primaryTrunk, '^$primaryBranch', '--']), - ); - final int extraCommits = _countLines( - git(primaryRepoDirectory, ['rev-list', primaryBranch, '^$primaryTrunk', '--']), - ); - if (missingTrunkCommits == 0 && extraCommits == 0) { - log('$primaryBranch is even with $primaryTrunk at $mergeBase'); - } else { - log( - '$primaryBranch branched from $primaryTrunk $missingTrunkCommits commits ago, trunk has advanced by $extraCommits commits since then.', - ); - } - } - } - return git(secondaryRepoDirectory, [ - 'log', - '--format=%H', - '--until=${anchor.toIso8601String()}', - '--max-count=1', - secondaryBranch, - '--', - ]); -} - -String git(String workingDirectory, List arguments, {bool allowFailure = false}) { - final ProcessResult result = Process.runSync( - 'git', - arguments, - workingDirectory: workingDirectory, - ); - if (!allowFailure && result.exitCode != 0 || '${result.stderr}'.isNotEmpty) { - throw ProcessException('git', arguments, '${result.stdout}${result.stderr}', result.exitCode); - } - return '${result.stdout}'; -} - -void main(List arguments) { - if (arguments.isEmpty || - arguments.length != 4 || - arguments.contains('--help') || - arguments.contains('-h')) { - print( - 'Usage: dart find_commit.dart \n' - 'This script will find the commit in the secondary repo that was contemporary\n' - 'when the commit in the primary repo was created. If that commit is on a\n' - "branch, then the date of the branch's last merge is used instead.", - ); - } else { - final String primaryRepo = arguments.first; - final String primaryTrunk = arguments[1]; - final String secondaryRepo = arguments[2]; - final String secondaryBranch = arguments.last; - print( - findCommit( - primaryRepoDirectory: primaryRepo, - primaryBranch: git(primaryRepo, ['rev-parse', '--abbrev-ref', 'HEAD']).trim(), - primaryTrunk: primaryTrunk, - secondaryRepoDirectory: secondaryRepo, - secondaryBranch: secondaryBranch, - ).trim(), - ); - } -}