firka/flutter
firka/flutter
5
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Pin customer_testing to the SHA specified in tests.version (#162048)

Browse Source 
Closes https://github.com/flutter/flutter/issues/162041.

I also converted the `.sh` and `.bat` files (largely) to Dart, mostly
because I don't know enough Windows XP command prompt in order to make
the change safely. They should still run the same from the CI
bootstrapping, but most of the logic is now Dart code.

Will send a separate PR to update the `flutter/tests` PR template.

/cc @johnmccutchan @Piinks
This commit is contained in:
Matan Lurey 2025-01-25 14:15:28 -08:00 committed by GitHub
parent a3f0704f25
commit a16de734bd
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
7 changed files with 139 additions and 163 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
dev/customer_testing/README.md Normal file
View File

@ -0,0 +1,16 @@
# customer_testing
This tool checks out <https://github.com/flutter/tests> at the commit SHA
specified in [`tests.version`](tests.version), and runs the tests registered to
verify that end-user apps and libraries are working at the current tip-of-tree
of Flutter.
To (locally) test a specific SHA, use `ci.dart`:
```sh
cd dev/customer_testing
dart ci.dart [sha]
```
Or, to update the SHA for our CI, edit and send a PR for
[`tests.version`](tests.version).

22
dev/customer_testing/ci.bat
View File

@ -8,20 +8,16 @@ REM This should match the ci.sh file in this directory.
REM This is called from the LUCI recipes:
REM https://github.com/flutter/flutter/blob/main/dev/bots/suite_runners/run_customer_testing_tests.dart
REM This script does not assume that "flutter update-packages" has been
REM run, to allow CIs to save time by skipping that steps since it's
REM largely not needed to run the flutter/tests tests.
REM
REM However, we do need to update this directory.
SETLOCAL
cd /d %~dp0
ECHO.
ECHO Updating pub packages...
CALL dart pub get
CD ..\tools
CALL dart pub get
CD ..\customer_testing
ECHO.
ECHO Finding correct version of customer tests...
CMD /S /C "IF EXIST "..\..\bin\cache\pkg\tests\" RMDIR /S /Q ..\..\bin\cache\pkg\tests"
git clone https://github.com/flutter/tests.git ..\..\bin\cache\pkg\tests
FOR /F "usebackq tokens=*" %%a IN (`dart --enable-asserts ..\tools\bin\find_commit.dart . master ..\..\bin\cache\pkg\tests main`) DO git -C ..\..\bin\cache\pkg\tests checkout %%a
ECHO.
ECHO Running tests...
CD ..\..\bin\cache\pkg\tests
CALL dart --enable-asserts ..\..\..\..\dev\customer_testing\run_tests.dart --verbose --skip-on-fetch-failure --skip-template registry/*.test
REM Run the cross-platform script.
CALL ..\..\bin\dart.bat ci.dart

91
dev/customer_testing/ci.dart Normal file
View File

@ -0,0 +1,91 @@
// Copyright 2014 The Flutter Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
import 'dart:io' as io;
import 'package:path/path.dart' as p;
/// To run this script, either:
///
/// ```sh
/// cd dev/customer_testing
/// dart ci.dart [sha]
/// ```
///
/// Or:
///
/// ```sh
/// ./dev/customer_testing/ci.sh
/// ./dev/customer_testing/ci.bat
/// ```
void main(List<String> args) async {
final String sha;
if (args.isEmpty) {
sha = io.File('tests.version').readAsStringSync().trim();
} else if (args.length == 1) {
sha = args.first;
} else {
io.stderr.writeln('Usage: dart ci.dart [sha]');
io.exitCode = 1;
return;
}
final String flutterRootPath = p.canonicalize('../../');
final io.Directory testsCacheDir = io.Directory(
p.join(flutterRootPath, 'bin', 'cache', 'pkg', 'tests'),
);
if (testsCacheDir.existsSync()) {
io.stderr.writeln('Cleaning up existing repo: ${testsCacheDir.path}');
testsCacheDir.deleteSync(recursive: true);
}
io.stderr.writeln('Cloning flutter/tests');
final io.Process clone = await io.Process.start('git', <String>[
'clone',
'--depth',
'1',
'https://github.com/flutter/tests.git',
testsCacheDir.path,
], mode: io.ProcessStartMode.inheritStdio);
if ((await clone.exitCode) != 0) {
io.exitCode = 1;
return;
}
io.stderr.writeln('Fetching/checking out $sha');
final io.Process fetch = await io.Process.start(
'git',
<String>['fetch', 'origin', sha],
mode: io.ProcessStartMode.inheritStdio,
workingDirectory: testsCacheDir.path,
);
if ((await fetch.exitCode) != 0) {
io.exitCode = 1;
return;
}
final io.Process checkout = await io.Process.start(
'git',
<String>['checkout', sha],
mode: io.ProcessStartMode.inheritStdio,
workingDirectory: testsCacheDir.path,
);
if ((await checkout.exitCode) != 0) {
io.exitCode = 1;
return;
}
io.stderr.writeln('Running tests...');
final io.Process test = await io.Process.start('dart', <String>[
'--enable-asserts',
'run_tests.dart',
'--skip-on-fetch-failure',
'--skip-template',
p.posix.joinAll(<String>[...p.split(testsCacheDir.path), 'registry', '*.test']),
], mode: io.ProcessStartMode.inheritStdio);
if ((await test.exitCode) != 0) {
io.exitCode = 1;
return;
}
}

38
dev/customer_testing/ci.sh
View File

@ -8,30 +8,28 @@
# This is called from the LUCI recipes:
# https://github.com/flutter/flutter/blob/main/dev/bots/suite_runners/run_customer_testing_tests.dart
set -ex
set -e
function script_location() {
local script_location="${BASH_SOURCE[0]}"
# Resolve symlinks
while [[ -h "$script_location" ]]; do
DIR="$(cd -P "$( dirname "$script_location")" >/dev/null && pwd)"
script_location="$(readlink "$script_location")"
[[ "$script_location" != /* ]] && script_location="$DIR/$script_location"
done
cd -P "$(dirname "$script_location")" >/dev/null && pwd
}
# So that users can run this script from anywhere and it will work as expected.
cd "$(script_location)"
# This script does not assume that "flutter update-packages" has been
# run, to allow CIs to save time by skipping that steps since it's
# largely not needed to run the flutter/tests tests.
#
# However, we do need to update this directory and the tools directory.
# However, we do need to update this directory.
dart pub get
(cd ../tools; dart pub get) # used for find_commit.dart below
# Next we need to update the flutter/tests checkout.
#
# We use find_commit.dart so that we pull the version of flutter/tests
# that was contemporary when the branch we are on was created. That
# way, we can still run the tests on long-lived branches without being
# affected by breaking changes on trunk causing changes to the tests
# that wouldn't work on the long-lived branch.
#
# (This also prevents trunk from suddenly failing when tests are
# revved on flutter/tests -- if you rerun a passing customer_tests
# shard, it should still pass, even if we rolled one of the tests.)
rm -rf ../../bin/cache/pkg/tests
git clone https://github.com/flutter/tests.git ../../bin/cache/pkg/tests
git -C ../../bin/cache/pkg/tests checkout `dart --enable-asserts ../tools/bin/find_commit.dart . master ../../bin/cache/pkg/tests main`
# Finally, run the tests.
dart --enable-asserts run_tests.dart --skip-on-fetch-failure --skip-template ../../bin/cache/pkg/tests/registry/*.test
# Run the cross-platform script.
../../bin/dart run ci.dart

4
dev/customer_testing/run_tests.dart
View File

@ -76,6 +76,10 @@ Future<bool> run(List<String> arguments) async {
.where((File file) => !skipTemplate || path.basename(file.path) != 'template.test')
.toList();
if (files.isEmpty && parsedArguments.rest.isNotEmpty) {
print('No files resolved from glob(s): ${parsedArguments.rest}');
}
if (help ||
repeat == null ||
files.isEmpty ||

1
dev/customer_testing/tests.version Normal file
View File

@ -0,0 +1 @@
b4cc097211814cfb0e1f08affb4574cf7de6b1cd

130
dev/tools/bin/find_commit.dart
View File

@ -1,130 +0,0 @@
// Copyright 2014 The Flutter Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
// This script looks at the current commit and branch of the git repository in
// which it was run, and finds the contemporary commit in the master branch of
// another git repository, whose path is provided on the command line. The
// contemporary commit is the one that was public at the time of the last commit
// on the master branch before the current commit's branch was created.
import 'dart:io';
const bool debugLogging = false;
void log(String message) {
if (debugLogging) {
print(message);
}
}
const String _commitTimestampFormat = '--format=%cI';
DateTime _parseTimestamp(String line) => DateTime.parse(line.trim());
int _countLines(String output) =>
output.trim().split('/n').where((String line) => line.isNotEmpty).length;
String findCommit({
required String primaryRepoDirectory,
required String primaryBranch,
required String primaryTrunk,
required String secondaryRepoDirectory,
required String secondaryBranch,
}) {
final DateTime anchor;
if (primaryBranch == primaryTrunk) {
log('on $primaryTrunk, using last commit time');
anchor = _parseTimestamp(
git(primaryRepoDirectory, <String>[
'log',
_commitTimestampFormat,
'--max-count=1',
primaryBranch,
'--',
]),
);
} else {
final String mergeBase =
git(primaryRepoDirectory, <String>[
'merge-base',
primaryBranch,
primaryTrunk,
], allowFailure: true).trim();
if (mergeBase.isEmpty) {
throw StateError(
'Branch $primaryBranch does not seem to have a common history with trunk $primaryTrunk.',
);
}
anchor = _parseTimestamp(
git(primaryRepoDirectory, <String>[
'log',
_commitTimestampFormat,
'--max-count=1',
mergeBase,
'--',
]),
);
if (debugLogging) {
final int missingTrunkCommits = _countLines(
git(primaryRepoDirectory, <String>['rev-list', primaryTrunk, '^$primaryBranch', '--']),
);
final int extraCommits = _countLines(
git(primaryRepoDirectory, <String>['rev-list', primaryBranch, '^$primaryTrunk', '--']),
);
if (missingTrunkCommits == 0 && extraCommits == 0) {
log('$primaryBranch is even with $primaryTrunk at $mergeBase');
} else {
log(
'$primaryBranch branched from $primaryTrunk $missingTrunkCommits commits ago, trunk has advanced by $extraCommits commits since then.',
);
}
}
}
return git(secondaryRepoDirectory, <String>[
'log',
'--format=%H',
'--until=${anchor.toIso8601String()}',
'--max-count=1',
secondaryBranch,
'--',
]);
}
String git(String workingDirectory, List<String> arguments, {bool allowFailure = false}) {
final ProcessResult result = Process.runSync(
'git',
arguments,
workingDirectory: workingDirectory,
);
if (!allowFailure && result.exitCode != 0 || '${result.stderr}'.isNotEmpty) {
throw ProcessException('git', arguments, '${result.stdout}${result.stderr}', result.exitCode);
}
return '${result.stdout}';
}
void main(List<String> arguments) {
if (arguments.isEmpty ||
arguments.length != 4 ||
arguments.contains('--help') ||
arguments.contains('-h')) {
print(
'Usage: dart find_commit.dart <path-to-primary-repo> <primary-trunk> <path-to-secondary-repo> <secondary-branch>\n'
'This script will find the commit in the secondary repo that was contemporary\n'
'when the commit in the primary repo was created. If that commit is on a\n'
"branch, then the date of the branch's last merge is used instead.",
);
} else {
final String primaryRepo = arguments.first;
final String primaryTrunk = arguments[1];
final String secondaryRepo = arguments[2];
final String secondaryBranch = arguments.last;
print(
findCommit(
primaryRepoDirectory: primaryRepo,
primaryBranch: git(primaryRepo, <String>['rev-parse', '--abbrev-ref', 'HEAD']).trim(),
primaryTrunk: primaryTrunk,
secondaryRepoDirectory: secondaryRepo,
secondaryBranch: secondaryBranch,
).trim(),
);
}
}