firka/flutter
firka/flutter
5
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Create web tests suite & runner_utils (#146592)

Browse Source 
Move from `test.dart` into new `runner_utils.dart` file:
- shard calculation related methods + `shuffleSeed` logic
- env variable flag calculation
- runner wrapper methods `runDartTest` and `_runFromList`

Create web tests suite class containing:
- web long running tests
- runWebHtmlUnitTests
- runWebCanvasKitUnitTests
- runWebSkwasmUnitTests
- the associate helper methods with the above
This commit is contained in:
Jesse 2024-04-17 16:44:08 -04:00 committed by GitHub
parent 808081e68c
commit e660a93c63
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
6 changed files with 1162 additions and 1141 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
dev/bots/suite_runners/run_flutter_packages_tests.dart
View File

@ -9,7 +9,6 @@ import 'package:file/local.dart';
import 'package:path/path.dart' as path;
import '../run_command.dart';
import '../test.dart';
import '../utils.dart';
/// Executes the test suite for the flutter/packages repo.

586
dev/bots/suite_runners/run_web_long_running_tests.dart
View File

@ -1,586 +0,0 @@
// Copyright 2014 The Flutter Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
import 'dart:convert';
import 'dart:io' show File, HttpClient, HttpClientRequest, HttpClientResponse, Process, RawSocket, SocketDirection, SocketException;
import 'dart:math' as math;
import 'package:path/path.dart' as path;
import '../browser.dart';
import '../run_command.dart';
import '../service_worker_test.dart';
import '../test.dart';
import '../utils.dart';
const List<String> _kAllBuildModes = <String>['debug', 'profile', 'release'];
/// Coarse-grained integration tests running on the Web.
Future<void> webLongRunningTestsRunner(String flutterRoot) async {
final String engineVersionFile = path.join(flutterRoot, 'bin', 'internal', 'engine.version');
final String engineRealmFile = path.join(flutterRoot, 'bin', 'internal', 'engine.realm');
final String engineVersion = File(engineVersionFile).readAsStringSync().trim();
final String engineRealm = File(engineRealmFile).readAsStringSync().trim();
if (engineRealm.isNotEmpty) {
return;
}
final List<ShardRunner> tests = <ShardRunner>[
for (final String buildMode in _kAllBuildModes) ...<ShardRunner>[
() => _runFlutterDriverWebTest(
testAppDirectory: path.join('packages', 'integration_test', 'example'),
target: path.join('test_driver', 'failure.dart'),
buildMode: buildMode,
renderer: 'canvaskit',
wasm: false,
// This test intentionally fails and prints stack traces in the browser
// logs. To avoid confusion, silence browser output.
silenceBrowserOutput: true,
),
() => _runFlutterDriverWebTest(
testAppDirectory: path.join('packages', 'integration_test', 'example'),
target: path.join('integration_test', 'example_test.dart'),
driver: path.join('test_driver', 'integration_test.dart'),
buildMode: buildMode,
renderer: 'canvaskit',
wasm: false,
expectWriteResponseFile: true,
expectResponseFileContent: 'null',
),
() => _runFlutterDriverWebTest(
testAppDirectory: path.join('packages', 'integration_test', 'example'),
target: path.join('integration_test', 'example_test.dart'),
driver: path.join('test_driver', 'integration_test.dart'),
buildMode: buildMode,
renderer: 'skwasm',
wasm: true,
expectWriteResponseFile: true,
expectResponseFileContent: 'null',
),
() => _runFlutterDriverWebTest(
testAppDirectory: path.join('packages', 'integration_test', 'example'),
target: path.join('integration_test', 'extended_test.dart'),
driver: path.join('test_driver', 'extended_integration_test.dart'),
buildMode: buildMode,
renderer: 'canvaskit',
wasm: false,
expectWriteResponseFile: true,
expectResponseFileContent: '''
{
"screenshots": [
{
"screenshotName": "platform_name",
"bytes": []
},
{
"screenshotName": "platform_name_2",
"bytes": []
}
]
}''',
),
],
// This test doesn't do anything interesting w.r.t. rendering, so we don't run the full build mode x renderer matrix.
() => _runWebE2eTest('platform_messages_integration', buildMode: 'debug', renderer: 'canvaskit'),
() => _runWebE2eTest('platform_messages_integration', buildMode: 'profile', renderer: 'html'),
() => _runWebE2eTest('platform_messages_integration', buildMode: 'release', renderer: 'html'),
// This test doesn't do anything interesting w.r.t. rendering, so we don't run the full build mode x renderer matrix.
() => _runWebE2eTest('profile_diagnostics_integration', buildMode: 'debug', renderer: 'html'),
() => _runWebE2eTest('profile_diagnostics_integration', buildMode: 'profile', renderer: 'canvaskit'),
() => _runWebE2eTest('profile_diagnostics_integration', buildMode: 'release', renderer: 'html'),
// This test is only known to work in debug mode.
() => _runWebE2eTest('scroll_wheel_integration', buildMode: 'debug', renderer: 'html'),
// This test doesn't do anything interesting w.r.t. rendering, so we don't run the full build mode x renderer matrix.
// These tests have been extremely flaky, so we are temporarily disabling them until we figure out how to make them more robust.
// See https://github.com/flutter/flutter/issues/143834
// () => _runWebE2eTest('text_editing_integration', buildMode: 'debug', renderer: 'canvaskit'),
// () => _runWebE2eTest('text_editing_integration', buildMode: 'profile', renderer: 'html'),
// () => _runWebE2eTest('text_editing_integration', buildMode: 'release', renderer: 'html'),
// This test doesn't do anything interesting w.r.t. rendering, so we don't run the full build mode x renderer matrix.
() => _runWebE2eTest('url_strategy_integration', buildMode: 'debug', renderer: 'html'),
() => _runWebE2eTest('url_strategy_integration', buildMode: 'profile', renderer: 'canvaskit'),
() => _runWebE2eTest('url_strategy_integration', buildMode: 'release', renderer: 'html'),
// This test doesn't do anything interesting w.r.t. rendering, so we don't run the full build mode x renderer matrix.
() => _runWebE2eTest('capabilities_integration_canvaskit', buildMode: 'debug', renderer: 'auto'),
() => _runWebE2eTest('capabilities_integration_canvaskit', buildMode: 'profile', renderer: 'canvaskit'),
() => _runWebE2eTest('capabilities_integration_html', buildMode: 'release', renderer: 'html'),
() => _runWebE2eTest('capabilities_integration_skwasm', buildMode: 'release', renderer: 'skwasm', wasm: true),
// This test doesn't do anything interesting w.r.t. rendering, so we don't run the full build mode x renderer matrix.
// CacheWidth and CacheHeight are only currently supported in CanvasKit mode, so we don't run the test in HTML mode.
() => _runWebE2eTest('cache_width_cache_height_integration', buildMode: 'debug', renderer: 'auto'),
() => _runWebE2eTest('cache_width_cache_height_integration', buildMode: 'profile', renderer: 'canvaskit'),
() => _runWebTreeshakeTest(),
() => _runFlutterDriverWebTest(
testAppDirectory: path.join(flutterRoot, 'examples', 'hello_world'),
target: 'test_driver/smoke_web_engine.dart',
buildMode: 'profile',
renderer: 'auto',
wasm: false,
),
() => _runGalleryE2eWebTest('debug'),
() => _runGalleryE2eWebTest('debug', canvasKit: true),
() => _runGalleryE2eWebTest('profile'),
() => _runGalleryE2eWebTest('profile', canvasKit: true),
() => _runGalleryE2eWebTest('release'),
() => _runGalleryE2eWebTest('release', canvasKit: true),
() => runWebServiceWorkerTest(headless: true, testType: ServiceWorkerTestType.withoutFlutterJs),
() => runWebServiceWorkerTest(headless: true, testType: ServiceWorkerTestType.withFlutterJs),
() => runWebServiceWorkerTest(headless: true, testType: ServiceWorkerTestType.withFlutterJsShort),
() => runWebServiceWorkerTest(headless: true, testType: ServiceWorkerTestType.withFlutterJsEntrypointLoadedEvent),
() => runWebServiceWorkerTest(headless: true, testType: ServiceWorkerTestType.withFlutterJsTrustedTypesOn),
() => runWebServiceWorkerTest(headless: true, testType: ServiceWorkerTestType.withFlutterJsNonceOn),
() => runWebServiceWorkerTestWithCachingResources(headless: true, testType: ServiceWorkerTestType.withoutFlutterJs),
() => runWebServiceWorkerTestWithCachingResources(headless: true, testType: ServiceWorkerTestType.withFlutterJs),
() => runWebServiceWorkerTestWithCachingResources(headless: true, testType: ServiceWorkerTestType.withFlutterJsShort),
() => runWebServiceWorkerTestWithCachingResources(headless: true, testType: ServiceWorkerTestType.withFlutterJsEntrypointLoadedEvent),
() => runWebServiceWorkerTestWithCachingResources(headless: true, testType: ServiceWorkerTestType.withFlutterJsTrustedTypesOn),
() => runWebServiceWorkerTestWithGeneratedEntrypoint(headless: true),
() => runWebServiceWorkerTestWithBlockedServiceWorkers(headless: true),
() => runWebServiceWorkerTestWithCustomServiceWorkerVersion(headless: true),
() => _runWebStackTraceTest('profile', 'lib/stack_trace.dart'),
() => _runWebStackTraceTest('release', 'lib/stack_trace.dart'),
() => _runWebStackTraceTest('profile', 'lib/framework_stack_trace.dart'),
() => _runWebStackTraceTest('release', 'lib/framework_stack_trace.dart'),
() => _runWebDebugTest('lib/stack_trace.dart'),
() => _runWebDebugTest('lib/framework_stack_trace.dart'),
() => _runWebDebugTest('lib/web_directory_loading.dart'),
() => _runWebDebugTest('lib/web_resources_cdn_test.dart',
additionalArguments: <String>[
'--dart-define=TEST_FLUTTER_ENGINE_VERSION=$engineVersion',
]),
() => _runWebDebugTest('test/test.dart'),
() => _runWebDebugTest('lib/null_safe_main.dart'),
() => _runWebDebugTest('lib/web_define_loading.dart',
additionalArguments: <String>[
'--dart-define=test.valueA=Example,A',
'--dart-define=test.valueB=Value',
]
),
() => _runWebReleaseTest('lib/web_define_loading.dart',
additionalArguments: <String>[
'--dart-define=test.valueA=Example,A',
'--dart-define=test.valueB=Value',
]
),
() => _runWebDebugTest('lib/sound_mode.dart'),
() => _runWebReleaseTest('lib/sound_mode.dart'),
() => runFlutterWebTest(
'html',
path.join(flutterRoot, 'packages', 'integration_test'),
<String>['test/web_extension_test.dart'],
false,
),
() => runFlutterWebTest(
'canvaskit',
path.join(flutterRoot, 'packages', 'integration_test'),
<String>['test/web_extension_test.dart'],
false,
),
() => runFlutterWebTest(
'skwasm',
path.join(flutterRoot, 'packages', 'integration_test'),
<String>['test/web_extension_test.dart'],
true,
),
];
// Shuffling mixes fast tests with slow tests so shards take roughly the same
// amount of time to run.
tests.shuffle(math.Random(0));
await _ensureChromeDriverIsRunning();
await runShardRunnerIndexOfTotalSubshard(tests);
await _stopChromeDriver();
}
/// Runs one of the `dev/integration_tests/web_e2e_tests` tests.
Future<void> _runWebE2eTest(
String name, {
required String buildMode,
required String renderer,
bool wasm = false,
}) async {
await _runFlutterDriverWebTest(
target: path.join('test_driver', '$name.dart'),
buildMode: buildMode,
renderer: renderer,
testAppDirectory: path.join(flutterRoot, 'dev', 'integration_tests', 'web_e2e_tests'),
wasm: wasm,
);
}
Future<void> _runFlutterDriverWebTest({
required String target,
required String buildMode,
required String renderer,
required String testAppDirectory,
required bool wasm,
String? driver,
bool expectFailure = false,
bool silenceBrowserOutput = false,
bool expectWriteResponseFile = false,
String expectResponseFileContent = '',
}) async {
printProgress('${green}Running integration tests $target in $buildMode mode.$reset');
await runCommand(
flutter,
<String>[ 'clean' ],
workingDirectory: testAppDirectory,
);
final String responseFile =
path.join(testAppDirectory, 'build', 'integration_response_data.json');
if (File(responseFile).existsSync()) {
File(responseFile).deleteSync();
}
await runCommand(
flutter,
<String>[
...flutterTestArgs,
'drive',
if (driver != null) '--driver=$driver',
'--target=$target',
'--browser-name=chrome',
'-d',
'web-server',
'--$buildMode',
'--web-renderer=$renderer',
if (wasm) '--wasm',
],
expectNonZeroExit: expectFailure,
workingDirectory: testAppDirectory,
environment: <String, String>{
'FLUTTER_WEB': 'true',
},
removeLine: (String line) {
if (!silenceBrowserOutput) {
return false;
}
if (line.trim().startsWith('[INFO]')) {
return true;
}
return false;
},
);
if (expectWriteResponseFile) {
if (!File(responseFile).existsSync()) {
foundError(<String>[
'$bold${red}Command did not write the response file but expected response file written.$reset',
]);
} else {
final String response = File(responseFile).readAsStringSync();
if (response != expectResponseFileContent) {
foundError(<String>[
'$bold${red}Command write the response file with $response but expected response file with $expectResponseFileContent.$reset',
]);
}
}
}
}
// Compiles a sample web app and checks that its JS doesn't contain certain
// debug code that we expect to be tree shaken out.
//
// The app is compiled in `--profile` mode to prevent the compiler from
// minifying the symbols.
Future<void> _runWebTreeshakeTest() async {
final String testAppDirectory = path.join(flutterRoot, 'dev', 'integration_tests', 'web_e2e_tests');
final String target = path.join('lib', 'treeshaking_main.dart');
await runCommand(
flutter,
<String>[ 'clean' ],
workingDirectory: testAppDirectory,
);
await runCommand(
flutter,
<String>[
'build',
'web',
'--target=$target',
'--profile',
],
workingDirectory: testAppDirectory,
environment: <String, String>{
'FLUTTER_WEB': 'true',
},
);
final File mainDartJs = File(path.join(testAppDirectory, 'build', 'web', 'main.dart.js'));
final String javaScript = mainDartJs.readAsStringSync();
// Check that we're not looking at minified JS. Otherwise this test would result in false positive.
expect(javaScript.contains('RootElement'), true);
const String word = 'debugFillProperties';
int count = 0;
int pos = javaScript.indexOf(word);
final int contentLength = javaScript.length;
while (pos != -1) {
count += 1;
pos += word.length;
if (pos >= contentLength || count > 100) {
break;
}
pos = javaScript.indexOf(word, pos);
}
// The following are classes from `timeline.dart` that should be treeshaken
// off unless the app (typically a benchmark) uses methods that need them.
expect(javaScript.contains('AggregatedTimedBlock'), false);
expect(javaScript.contains('AggregatedTimings'), false);
expect(javaScript.contains('_BlockBuffer'), false);
expect(javaScript.contains('_StringListChain'), false);
expect(javaScript.contains('_Float64ListChain'), false);
const int kMaxExpectedDebugFillProperties = 11;
if (count > kMaxExpectedDebugFillProperties) {
throw Exception(
'Too many occurrences of "$word" in compiled JavaScript.\n'
'Expected no more than $kMaxExpectedDebugFillProperties, but found $count.'
);
}
}
/// Exercises the old gallery in a browser for a long period of time, looking
/// for memory leaks and dangling pointers.
///
/// This is not a performance test.
///
/// If [canvasKit] is set to true, runs the test in CanvasKit mode.
///
/// The test is written using `package:integration_test` (despite the "e2e" in
/// the name, which is there for historic reasons).
Future<void> _runGalleryE2eWebTest(String buildMode, { bool canvasKit = false }) async {
printProgress('${green}Running flutter_gallery integration test in --$buildMode using ${canvasKit ? 'CanvasKit' : 'HTML'} renderer.$reset');
final String testAppDirectory = path.join(flutterRoot, 'dev', 'integration_tests', 'flutter_gallery');
await runCommand(
flutter,
<String>[ 'clean' ],
workingDirectory: testAppDirectory,
);
await runCommand(
flutter,
<String>[
...flutterTestArgs,
'drive',
if (canvasKit)
'--dart-define=FLUTTER_WEB_USE_SKIA=true',
if (!canvasKit)
'--dart-define=FLUTTER_WEB_USE_SKIA=false',
if (!canvasKit)
'--dart-define=FLUTTER_WEB_AUTO_DETECT=false',
'--driver=test_driver/transitions_perf_e2e_test.dart',
'--target=test_driver/transitions_perf_e2e.dart',
'--browser-name=chrome',
'-d',
'web-server',
'--$buildMode',
],
workingDirectory: testAppDirectory,
environment: <String, String>{
'FLUTTER_WEB': 'true',
},
);
}
Future<void> _runWebStackTraceTest(String buildMode, String entrypoint) async {
final String testAppDirectory = path.join(flutterRoot, 'dev', 'integration_tests', 'web');
final String appBuildDirectory = path.join(testAppDirectory, 'build', 'web');
// Build the app.
await runCommand(
flutter,
<String>[ 'clean' ],
workingDirectory: testAppDirectory,
);
await runCommand(
flutter,
<String>[
'build',
'web',
'--$buildMode',
'-t',
entrypoint,
],
workingDirectory: testAppDirectory,
environment: <String, String>{
'FLUTTER_WEB': 'true',
},
);
// Run the app.
final int serverPort = await findAvailablePortAndPossiblyCauseFlakyTests();
final int browserDebugPort = await findAvailablePortAndPossiblyCauseFlakyTests();
final String result = await evalTestAppInChrome(
appUrl: 'http://localhost:$serverPort/index.html',
appDirectory: appBuildDirectory,
serverPort: serverPort,
browserDebugPort: browserDebugPort,
);
if (!result.contains('--- TEST SUCCEEDED ---')) {
foundError(<String>[
result,
'${red}Web stack trace integration test failed.$reset',
]);
}
}
/// Debug mode is special because `flutter build web` doesn't build in debug mode.
///
/// Instead, we use `flutter run --debug` and sniff out the standard output.
Future<void> _runWebDebugTest(String target, {
List<String> additionalArguments = const<String>[],
}) async {
final String testAppDirectory = path.join(flutterRoot, 'dev', 'integration_tests', 'web');
bool success = false;
final Map<String, String> environment = <String, String>{
'FLUTTER_WEB': 'true',
};
adjustEnvironmentToEnableFlutterAsserts(environment);
final CommandResult result = await runCommand(
flutter,
<String>[
'run',
'--debug',
'-d',
'chrome',
'--web-run-headless',
'--dart-define=FLUTTER_WEB_USE_SKIA=false',
'--dart-define=FLUTTER_WEB_AUTO_DETECT=false',
...additionalArguments,
'-t',
target,
],
outputMode: OutputMode.capture,
outputListener: (String line, Process process) {
if (line.contains('--- TEST SUCCEEDED ---')) {
success = true;
}
if (success || line.contains('--- TEST FAILED ---')) {
process.stdin.add('q'.codeUnits);
}
},
workingDirectory: testAppDirectory,
environment: environment,
);
if (!success) {
foundError(<String>[
result.flattenedStdout!,
result.flattenedStderr!,
'${red}Web stack trace integration test failed.$reset',
]);
}
}
/// Run a web integration test in release mode.
Future<void> _runWebReleaseTest(String target, {
List<String> additionalArguments = const<String>[],
}) async {
final String testAppDirectory = path.join(flutterRoot, 'dev', 'integration_tests', 'web');
final String appBuildDirectory = path.join(testAppDirectory, 'build', 'web');
// Build the app.
await runCommand(
flutter,
<String>[ 'clean' ],
workingDirectory: testAppDirectory,
);
await runCommand(
flutter,
<String>[
...flutterTestArgs,
'build',
'web',
'--release',
...additionalArguments,
'-t',
target,
],
workingDirectory: testAppDirectory,
environment: <String, String>{
'FLUTTER_WEB': 'true',
},
);
// Run the app.
final int serverPort = await findAvailablePortAndPossiblyCauseFlakyTests();
final int browserDebugPort = await findAvailablePortAndPossiblyCauseFlakyTests();
final String result = await evalTestAppInChrome(
appUrl: 'http://localhost:$serverPort/index.html',
appDirectory: appBuildDirectory,
serverPort: serverPort,
browserDebugPort: browserDebugPort,
);
if (!result.contains('--- TEST SUCCEEDED ---')) {
foundError(<String>[
result,
'${red}Web release mode test failed.$reset',
]);
}
}
// The `chromedriver` process created by this test.
//
// If an existing chromedriver is already available on port 4444, the existing
// process is reused and this variable remains null.
Command? _chromeDriver;
Future<bool> _isChromeDriverRunning() async {
try {
final RawSocket socket = await RawSocket.connect('localhost', 4444);
socket.shutdown(SocketDirection.both);
await socket.close();
return true;
} on SocketException {
return false;
}
}
Future<void> _stopChromeDriver() async {
if (_chromeDriver == null) {
return;
}
print('Stopping chromedriver');
_chromeDriver!.process.kill();
}
Future<void> _ensureChromeDriverIsRunning() async {
// If we cannot connect to ChromeDriver, assume it is not running. Launch it.
if (!await _isChromeDriverRunning()) {
printProgress('Starting chromedriver');
// Assume chromedriver is in the PATH.
_chromeDriver = await startCommand(
// TODO(ianh): this is the only remaining consumer of startCommand other than runCommand
// and it doesn't use most of startCommand's features; we could simplify this a lot by
// inlining the relevant parts of startCommand here.
'chromedriver',
<String>['--port=4444'],
);
while (!await _isChromeDriverRunning()) {
await Future<void>.delayed(const Duration(milliseconds: 100));
print('Waiting for chromedriver to start up.');
}
}
final HttpClient client = HttpClient();
final Uri chromeDriverUrl = Uri.parse('http://localhost:4444/status');
final HttpClientRequest request = await client.getUrl(chromeDriverUrl);
final HttpClientResponse response = await request.close();
final Map<String, dynamic> webDriverStatus = json.decode(await response.transform(utf8.decoder).join()) as Map<String, dynamic>;
client.close();
final bool webDriverReady = (webDriverStatus['value'] as Map<String, dynamic>)['ready'] as bool;
if (!webDriverReady) {
throw Exception('WebDriver not available.');
}
}

760
dev/bots/suite_runners/run_web_tests.dart Normal file
View File

@ -0,0 +1,760 @@
// Copyright 2014 The Flutter Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
import 'dart:convert';
import 'dart:io' show Directory, File, FileSystemEntity, HttpClient, HttpClientRequest, HttpClientResponse, Platform, Process, RawSocket, SocketDirection, SocketException;
import 'dart:math' as math;
import 'package:file/local.dart';
import 'package:path/path.dart' as path;
import '../browser.dart';
import '../run_command.dart';
import '../service_worker_test.dart';
import '../utils.dart';
typedef ShardRunner = Future<void> Function();
class WebTestsSuite {
WebTestsSuite(this.flutterRoot, this.flutterTestArgs);
/// Tests that we don't run on Web.
///
/// In general avoid adding new tests here. If a test cannot run on the web
/// because it fails at runtime, such as when a piece of functionality is not
/// implemented or not implementable on the web, prefer using `skip` in the
/// test code. Only add tests here that cannot be skipped using `skip`. For
/// example:
///
/// * Test code cannot be compiled because it uses Dart VM-specific
/// functionality. In this case `skip` doesn't help because the code cannot
/// reach the point where it can even run the skipping logic.
/// * Migrations. It is OK to put tests here that need to be temporarily
/// disabled in certain modes because of some migration or initial bringup.
///
/// The key in the map is the renderer type that the list applies to. The value
/// is the list of tests known to fail for that renderer.
//
// TODO(yjbanov): we're getting rid of this as part of https://github.com/flutter/flutter/projects/60
static const Map<String, List<String>> kWebTestFileKnownFailures = <String, List<String>>{
'html': <String>[
// These tests are not compilable on the web due to dependencies on
// VM-specific functionality.
'test/services/message_codecs_vm_test.dart',
'test/examples/sector_layout_test.dart',
],
'canvaskit': <String>[
// These tests are not compilable on the web due to dependencies on
// VM-specific functionality.
'test/services/message_codecs_vm_test.dart',
'test/examples/sector_layout_test.dart',
// These tests are broken and need to be fixed.
// TODO(yjbanov): https://github.com/flutter/flutter/issues/71604
'test/material/text_field_test.dart',
'test/widgets/performance_overlay_test.dart',
'test/widgets/html_element_view_test.dart',
'test/cupertino/scaffold_test.dart',
'test/rendering/platform_view_test.dart',
],
'skwasm': <String>[
// These tests are not compilable on the web due to dependencies on
// VM-specific functionality.
'test/services/message_codecs_vm_test.dart',
'test/examples/sector_layout_test.dart',
// These tests are broken and need to be fixed.
// TODO(jacksongardner): https://github.com/flutter/flutter/issues/71604
'test/material/text_field_test.dart',
'test/widgets/performance_overlay_test.dart',
],
};
/// The number of Cirrus jobs that run Web tests in parallel.
///
/// The default is 8 shards. Typically .cirrus.yml would define the
/// WEB_SHARD_COUNT environment variable rather than relying on the default.
///
/// WARNING: if you change this number, also change .cirrus.yml
/// and make sure it runs _all_ shards.
///
/// The last shard also runs the Web plugin tests.
int get webShardCount => Platform.environment.containsKey('WEB_SHARD_COUNT')
? int.parse(Platform.environment['WEB_SHARD_COUNT']!)
: 8;
static const List<String> _kAllBuildModes = <String>['debug', 'profile', 'release'];
final String flutterRoot;
final List<String> flutterTestArgs;
/// Coarse-grained integration tests running on the Web.
Future<void> webLongRunningTestsRunner() async {
final String engineVersionFile = path.join(flutterRoot, 'bin', 'internal', 'engine.version');
final String engineRealmFile = path.join(flutterRoot, 'bin', 'internal', 'engine.realm');
final String engineVersion = File(engineVersionFile).readAsStringSync().trim();
final String engineRealm = File(engineRealmFile).readAsStringSync().trim();
if (engineRealm.isNotEmpty) {
return;
}
final List<ShardRunner> tests = <ShardRunner>[
for (final String buildMode in _kAllBuildModes) ...<ShardRunner>[
() => _runFlutterDriverWebTest(
testAppDirectory: path.join('packages', 'integration_test', 'example'),
target: path.join('test_driver', 'failure.dart'),
buildMode: buildMode,
renderer: 'canvaskit',
// This test intentionally fails and prints stack traces in the browser
// logs. To avoid confusion, silence browser output.
silenceBrowserOutput: true,
),
() => _runFlutterDriverWebTest(
testAppDirectory: path.join('packages', 'integration_test', 'example'),
target: path.join('integration_test', 'example_test.dart'),
driver: path.join('test_driver', 'integration_test.dart'),
buildMode: buildMode,
renderer: 'canvaskit',
expectWriteResponseFile: true,
expectResponseFileContent: 'null',
),
() => _runFlutterDriverWebTest(
testAppDirectory: path.join('packages', 'integration_test', 'example'),
target: path.join('integration_test', 'extended_test.dart'),
driver: path.join('test_driver', 'extended_integration_test.dart'),
buildMode: buildMode,
renderer: 'canvaskit',
expectWriteResponseFile: true,
expectResponseFileContent: '''
{
"screenshots": [
{
"screenshotName": "platform_name",
"bytes": []
},
{
"screenshotName": "platform_name_2",
"bytes": []
}
]
}''',
),
],
// This test doesn't do anything interesting w.r.t. rendering, so we don't run the full build mode x renderer matrix.
() => _runWebE2eTest('platform_messages_integration', buildMode: 'debug', renderer: 'canvaskit'),
() => _runWebE2eTest('platform_messages_integration', buildMode: 'profile', renderer: 'html'),
() => _runWebE2eTest('platform_messages_integration', buildMode: 'release', renderer: 'html'),
// This test doesn't do anything interesting w.r.t. rendering, so we don't run the full build mode x renderer matrix.
() => _runWebE2eTest('profile_diagnostics_integration', buildMode: 'debug', renderer: 'html'),
() => _runWebE2eTest('profile_diagnostics_integration', buildMode: 'profile', renderer: 'canvaskit'),
() => _runWebE2eTest('profile_diagnostics_integration', buildMode: 'release', renderer: 'html'),
// This test is only known to work in debug mode.
() => _runWebE2eTest('scroll_wheel_integration', buildMode: 'debug', renderer: 'html'),
// This test doesn't do anything interesting w.r.t. rendering, so we don't run the full build mode x renderer matrix.
// These tests have been extremely flaky, so we are temporarily disabling them until we figure out how to make them more robust.
// See https://github.com/flutter/flutter/issues/143834
// () => _runWebE2eTest('text_editing_integration', buildMode: 'debug', renderer: 'canvaskit'),
// () => _runWebE2eTest('text_editing_integration', buildMode: 'profile', renderer: 'html'),
// () => _runWebE2eTest('text_editing_integration', buildMode: 'release', renderer: 'html'),
// This test doesn't do anything interesting w.r.t. rendering, so we don't run the full build mode x renderer matrix.
() => _runWebE2eTest('url_strategy_integration', buildMode: 'debug', renderer: 'html'),
() => _runWebE2eTest('url_strategy_integration', buildMode: 'profile', renderer: 'canvaskit'),
() => _runWebE2eTest('url_strategy_integration', buildMode: 'release', renderer: 'html'),
// This test doesn't do anything interesting w.r.t. rendering, so we don't run the full build mode x renderer matrix.
() => _runWebE2eTest('capabilities_integration_canvaskit', buildMode: 'debug', renderer: 'auto'),
() => _runWebE2eTest('capabilities_integration_canvaskit', buildMode: 'profile', renderer: 'canvaskit'),
() => _runWebE2eTest('capabilities_integration_html', buildMode: 'release', renderer: 'html'),
// This test doesn't do anything interesting w.r.t. rendering, so we don't run the full build mode x renderer matrix.
// CacheWidth and CacheHeight are only currently supported in CanvasKit mode, so we don't run the test in HTML mode.
() => _runWebE2eTest('cache_width_cache_height_integration', buildMode: 'debug', renderer: 'auto'),
() => _runWebE2eTest('cache_width_cache_height_integration', buildMode: 'profile', renderer: 'canvaskit'),
() => _runWebTreeshakeTest(),
() => _runFlutterDriverWebTest(
testAppDirectory: path.join(flutterRoot, 'examples', 'hello_world'),
target: 'test_driver/smoke_web_engine.dart',
buildMode: 'profile',
renderer: 'auto',
),
() => _runGalleryE2eWebTest('debug'),
() => _runGalleryE2eWebTest('debug', canvasKit: true),
() => _runGalleryE2eWebTest('profile'),
() => _runGalleryE2eWebTest('profile', canvasKit: true),
() => _runGalleryE2eWebTest('release'),
() => _runGalleryE2eWebTest('release', canvasKit: true),
() => runWebServiceWorkerTest(headless: true, testType: ServiceWorkerTestType.withoutFlutterJs),
() => runWebServiceWorkerTest(headless: true, testType: ServiceWorkerTestType.withFlutterJs),
() => runWebServiceWorkerTest(headless: true, testType: ServiceWorkerTestType.withFlutterJsShort),
() => runWebServiceWorkerTest(headless: true, testType: ServiceWorkerTestType.withFlutterJsEntrypointLoadedEvent),
() => runWebServiceWorkerTest(headless: true, testType: ServiceWorkerTestType.withFlutterJsTrustedTypesOn),
() => runWebServiceWorkerTest(headless: true, testType: ServiceWorkerTestType.withFlutterJsNonceOn),
() => runWebServiceWorkerTestWithCachingResources(headless: true, testType: ServiceWorkerTestType.withoutFlutterJs),
() => runWebServiceWorkerTestWithCachingResources(headless: true, testType: ServiceWorkerTestType.withFlutterJs),
() => runWebServiceWorkerTestWithCachingResources(headless: true, testType: ServiceWorkerTestType.withFlutterJsShort),
() => runWebServiceWorkerTestWithCachingResources(headless: true, testType: ServiceWorkerTestType.withFlutterJsEntrypointLoadedEvent),
() => runWebServiceWorkerTestWithCachingResources(headless: true, testType: ServiceWorkerTestType.withFlutterJsTrustedTypesOn),
() => runWebServiceWorkerTestWithGeneratedEntrypoint(headless: true),
() => runWebServiceWorkerTestWithBlockedServiceWorkers(headless: true),
() => runWebServiceWorkerTestWithCustomServiceWorkerVersion(headless: true),
() => _runWebStackTraceTest('profile', 'lib/stack_trace.dart'),
() => _runWebStackTraceTest('release', 'lib/stack_trace.dart'),
() => _runWebStackTraceTest('profile', 'lib/framework_stack_trace.dart'),
() => _runWebStackTraceTest('release', 'lib/framework_stack_trace.dart'),
() => _runWebDebugTest('lib/stack_trace.dart'),
() => _runWebDebugTest('lib/framework_stack_trace.dart'),
() => _runWebDebugTest('lib/web_directory_loading.dart'),
() => _runWebDebugTest('lib/web_resources_cdn_test.dart',
additionalArguments: <String>[
'--dart-define=TEST_FLUTTER_ENGINE_VERSION=$engineVersion',
]),
() => _runWebDebugTest('test/test.dart'),
() => _runWebDebugTest('lib/null_safe_main.dart'),
() => _runWebDebugTest('lib/web_define_loading.dart',
additionalArguments: <String>[
'--dart-define=test.valueA=Example,A',
'--dart-define=test.valueB=Value',
]
),
() => _runWebReleaseTest('lib/web_define_loading.dart',
additionalArguments: <String>[
'--dart-define=test.valueA=Example,A',
'--dart-define=test.valueB=Value',
]
),
() => _runWebDebugTest('lib/sound_mode.dart'),
() => _runWebReleaseTest('lib/sound_mode.dart'),
() => _runFlutterWebTest(
'html',
path.join(flutterRoot, 'packages', 'integration_test'),
<String>['test/web_extension_test.dart'],
false,
),
() => _runFlutterWebTest(
'canvaskit',
path.join(flutterRoot, 'packages', 'integration_test'),
<String>['test/web_extension_test.dart'],
false,
),
() => _runFlutterWebTest(
'skwasm',
path.join(flutterRoot, 'packages', 'integration_test'),
<String>['test/web_extension_test.dart'],
true,
),
];
// Shuffling mixes fast tests with slow tests so shards take roughly the same
// amount of time to run.
tests.shuffle(math.Random(0));
await _ensureChromeDriverIsRunning();
await runShardRunnerIndexOfTotalSubshard(tests);
await _stopChromeDriver();
}
Future<void> runWebHtmlUnitTests() {
return _runWebUnitTests('html', false);
}
Future<void> runWebCanvasKitUnitTests() {
return _runWebUnitTests('canvaskit', false);
}
Future<void> runWebSkwasmUnitTests() {
return _runWebUnitTests('skwasm', true);
}
/// Runs one of the `dev/integration_tests/web_e2e_tests` tests.
Future<void> _runWebE2eTest(
String name, {
required String buildMode,
required String renderer,
}) async {
await _runFlutterDriverWebTest(
target: path.join('test_driver', '$name.dart'),
buildMode: buildMode,
renderer: renderer,
testAppDirectory: path.join(flutterRoot, 'dev', 'integration_tests', 'web_e2e_tests'),
);
}
Future<void> _runFlutterDriverWebTest({
required String target,
required String buildMode,
required String renderer,
required String testAppDirectory,
String? driver,
bool expectFailure = false,
bool silenceBrowserOutput = false,
bool expectWriteResponseFile = false,
String expectResponseFileContent = '',
}) async {
printProgress('${green}Running integration tests $target in $buildMode mode.$reset');
await runCommand(
flutter,
<String>[ 'clean' ],
workingDirectory: testAppDirectory,
);
final String responseFile =
path.join(testAppDirectory, 'build', 'integration_response_data.json');
if (File(responseFile).existsSync()) {
File(responseFile).deleteSync();
}
await runCommand(
flutter,
<String>[
...flutterTestArgs,
'drive',
if (driver != null) '--driver=$driver',
'--target=$target',
'--browser-name=chrome',
'-d',
'web-server',
'--$buildMode',
'--web-renderer=$renderer',
],
expectNonZeroExit: expectFailure,
workingDirectory: testAppDirectory,
environment: <String, String>{
'FLUTTER_WEB': 'true',
},
removeLine: (String line) {
if (!silenceBrowserOutput) {
return false;
}
if (line.trim().startsWith('[INFO]')) {
return true;
}
return false;
},
);
if (expectWriteResponseFile) {
if (!File(responseFile).existsSync()) {
foundError(<String>[
'$bold${red}Command did not write the response file but expected response file written.$reset',
]);
} else {
final String response = File(responseFile).readAsStringSync();
if (response != expectResponseFileContent) {
foundError(<String>[
'$bold${red}Command write the response file with $response but expected response file with $expectResponseFileContent.$reset',
]);
}
}
}
}
// Compiles a sample web app and checks that its JS doesn't contain certain
// debug code that we expect to be tree shaken out.
//
// The app is compiled in `--profile` mode to prevent the compiler from
// minifying the symbols.
Future<void> _runWebTreeshakeTest() async {
final String testAppDirectory = path.join(flutterRoot, 'dev', 'integration_tests', 'web_e2e_tests');
final String target = path.join('lib', 'treeshaking_main.dart');
await runCommand(
flutter,
<String>[ 'clean' ],
workingDirectory: testAppDirectory,
);
await runCommand(
flutter,
<String>[
'build',
'web',
'--target=$target',
'--profile',
],
workingDirectory: testAppDirectory,
environment: <String, String>{
'FLUTTER_WEB': 'true',
},
);
final File mainDartJs = File(path.join(testAppDirectory, 'build', 'web', 'main.dart.js'));
final String javaScript = mainDartJs.readAsStringSync();
// Check that we're not looking at minified JS. Otherwise this test would result in false positive.
expect(javaScript.contains('RootElement'), true);
const String word = 'debugFillProperties';
int count = 0;
int pos = javaScript.indexOf(word);
final int contentLength = javaScript.length;
while (pos != -1) {
count += 1;
pos += word.length;
if (pos >= contentLength || count > 100) {
break;
}
pos = javaScript.indexOf(word, pos);
}
// The following are classes from `timeline.dart` that should be treeshaken
// off unless the app (typically a benchmark) uses methods that need them.
expect(javaScript.contains('AggregatedTimedBlock'), false);
expect(javaScript.contains('AggregatedTimings'), false);
expect(javaScript.contains('_BlockBuffer'), false);
expect(javaScript.contains('_StringListChain'), false);
expect(javaScript.contains('_Float64ListChain'), false);
const int kMaxExpectedDebugFillProperties = 11;
if (count > kMaxExpectedDebugFillProperties) {
throw Exception(
'Too many occurrences of "$word" in compiled JavaScript.\n'
'Expected no more than $kMaxExpectedDebugFillProperties, but found $count.'
);
}
}
/// Exercises the old gallery in a browser for a long period of time, looking
/// for memory leaks and dangling pointers.
///
/// This is not a performance test.
///
/// If [canvasKit] is set to true, runs the test in CanvasKit mode.
///
/// The test is written using `package:integration_test` (despite the "e2e" in
/// the name, which is there for historic reasons).
Future<void> _runGalleryE2eWebTest(String buildMode, { bool canvasKit = false }) async {
printProgress('${green}Running flutter_gallery integration test in --$buildMode using ${canvasKit ? 'CanvasKit' : 'HTML'} renderer.$reset');
final String testAppDirectory = path.join(flutterRoot, 'dev', 'integration_tests', 'flutter_gallery');
await runCommand(
flutter,
<String>[ 'clean' ],
workingDirectory: testAppDirectory,
);
await runCommand(
flutter,
<String>[
...flutterTestArgs,
'drive',
if (canvasKit)
'--dart-define=FLUTTER_WEB_USE_SKIA=true',
if (!canvasKit)
'--dart-define=FLUTTER_WEB_USE_SKIA=false',
if (!canvasKit)
'--dart-define=FLUTTER_WEB_AUTO_DETECT=false',
'--driver=test_driver/transitions_perf_e2e_test.dart',
'--target=test_driver/transitions_perf_e2e.dart',
'--browser-name=chrome',
'-d',
'web-server',
'--$buildMode',
],
workingDirectory: testAppDirectory,
environment: <String, String>{
'FLUTTER_WEB': 'true',
},
);
}
Future<void> _runWebStackTraceTest(String buildMode, String entrypoint) async {
final String testAppDirectory = path.join(flutterRoot, 'dev', 'integration_tests', 'web');
final String appBuildDirectory = path.join(testAppDirectory, 'build', 'web');
// Build the app.
await runCommand(
flutter,
<String>[ 'clean' ],
workingDirectory: testAppDirectory,
);
await runCommand(
flutter,
<String>[
'build',
'web',
'--$buildMode',
'-t',
entrypoint,
],
workingDirectory: testAppDirectory,
environment: <String, String>{
'FLUTTER_WEB': 'true',
},
);
// Run the app.
final int serverPort = await findAvailablePortAndPossiblyCauseFlakyTests();
final int browserDebugPort = await findAvailablePortAndPossiblyCauseFlakyTests();
final String result = await evalTestAppInChrome(
appUrl: 'http://localhost:$serverPort/index.html',
appDirectory: appBuildDirectory,
serverPort: serverPort,
browserDebugPort: browserDebugPort,
);
if (!result.contains('--- TEST SUCCEEDED ---')) {
foundError(<String>[
result,
'${red}Web stack trace integration test failed.$reset',
]);
}
}
/// Debug mode is special because `flutter build web` doesn't build in debug mode.
///
/// Instead, we use `flutter run --debug` and sniff out the standard output.
Future<void> _runWebDebugTest(String target, {
List<String> additionalArguments = const<String>[],
}) async {
final String testAppDirectory = path.join(flutterRoot, 'dev', 'integration_tests', 'web');
bool success = false;
final Map<String, String> environment = <String, String>{
'FLUTTER_WEB': 'true',
};
adjustEnvironmentToEnableFlutterAsserts(environment);
final CommandResult result = await runCommand(
flutter,
<String>[
'run',
'--debug',
'-d',
'chrome',
'--web-run-headless',
'--dart-define=FLUTTER_WEB_USE_SKIA=false',
'--dart-define=FLUTTER_WEB_AUTO_DETECT=false',
...additionalArguments,
'-t',
target,
],
outputMode: OutputMode.capture,
outputListener: (String line, Process process) {
if (line.contains('--- TEST SUCCEEDED ---')) {
success = true;
}
if (success || line.contains('--- TEST FAILED ---')) {
process.stdin.add('q'.codeUnits);
}
},
workingDirectory: testAppDirectory,
environment: environment,
);
if (!success) {
foundError(<String>[
result.flattenedStdout!,
result.flattenedStderr!,
'${red}Web stack trace integration test failed.$reset',
]);
}
}
/// Run a web integration test in release mode.
Future<void> _runWebReleaseTest(String target, {
List<String> additionalArguments = const<String>[],
}) async {
final String testAppDirectory = path.join(flutterRoot, 'dev', 'integration_tests', 'web');
final String appBuildDirectory = path.join(testAppDirectory, 'build', 'web');
// Build the app.
await runCommand(
flutter,
<String>[ 'clean' ],
workingDirectory: testAppDirectory,
);
await runCommand(
flutter,
<String>[
...flutterTestArgs,
'build',
'web',
'--release',
...additionalArguments,
'-t',
target,
],
workingDirectory: testAppDirectory,
environment: <String, String>{
'FLUTTER_WEB': 'true',
},
);
// Run the app.
final int serverPort = await findAvailablePortAndPossiblyCauseFlakyTests();
final int browserDebugPort = await findAvailablePortAndPossiblyCauseFlakyTests();
final String result = await evalTestAppInChrome(
appUrl: 'http://localhost:$serverPort/index.html',
appDirectory: appBuildDirectory,
serverPort: serverPort,
browserDebugPort: browserDebugPort,
);
if (!result.contains('--- TEST SUCCEEDED ---')) {
foundError(<String>[
result,
'${red}Web release mode test failed.$reset',
]);
}
}
Future<void> _runWebUnitTests(String webRenderer, bool useWasm) async {
final Map<String, ShardRunner> subshards = <String, ShardRunner>{};
final Directory flutterPackageDirectory = Directory(path.join(flutterRoot, 'packages', 'flutter'));
final Directory flutterPackageTestDirectory = Directory(path.join(flutterPackageDirectory.path, 'test'));
final List<String> allTests = flutterPackageTestDirectory
.listSync()
.whereType<Directory>()
.expand((Directory directory) => directory
.listSync(recursive: true)
.where((FileSystemEntity entity) => entity.path.endsWith('_test.dart'))
)
.whereType<File>()
.map<String>((File file) => path.relative(file.path, from: flutterPackageDirectory.path))
.where((String filePath) => !kWebTestFileKnownFailures[webRenderer]!.contains(path.split(filePath).join('/')))
.toList()
// Finally we shuffle the list because we want the average cost per file to be uniformly
// distributed. If the list is not sorted then different shards and batches may have
// very different characteristics.
// We use a constant seed for repeatability.
..shuffle(math.Random(0));
assert(webShardCount >= 1);
final int testsPerShard = (allTests.length / webShardCount).ceil();
assert(testsPerShard * webShardCount >= allTests.length);
// This for loop computes all but the last shard.
for (int index = 0; index < webShardCount - 1; index += 1) {
subshards['$index'] = () => _runFlutterWebTest(
webRenderer,
flutterPackageDirectory.path,
allTests.sublist(
index * testsPerShard,
(index + 1) * testsPerShard,
),
useWasm,
);
}
// The last shard also runs the flutter_web_plugins tests.
//
// We make sure the last shard ends in _last so it's easier to catch mismatches
// between `.cirrus.yml` and `test.dart`.
subshards['${webShardCount - 1}_last'] = () async {
await _runFlutterWebTest(
webRenderer,
flutterPackageDirectory.path,
allTests.sublist(
(webShardCount - 1) * testsPerShard,
allTests.length,
),
useWasm,
);
await _runFlutterWebTest(
webRenderer,
path.join(flutterRoot, 'packages', 'flutter_web_plugins'),
<String>['test'],
useWasm,
);
await _runFlutterWebTest(
webRenderer,
path.join(flutterRoot, 'packages', 'flutter_driver'),
<String>[path.join('test', 'src', 'web_tests', 'web_extension_test.dart')],
useWasm,
);
};
await selectSubshard(subshards);
}
Future<void> _runFlutterWebTest(
String webRenderer,
String workingDirectory,
List<String> tests,
bool useWasm,
) async {
const LocalFileSystem fileSystem = LocalFileSystem();
final String suffix = DateTime.now().microsecondsSinceEpoch.toString();
final File metricFile = fileSystem.systemTempDirectory.childFile('metrics_$suffix.json');
await runCommand(
flutter,
<String>[
'test',
'--reporter=expanded',
'--file-reporter=json:${metricFile.path}',
'-v',
'--platform=chrome',
if (useWasm) '--wasm',
'--web-renderer=$webRenderer',
'--dart-define=DART_HHH_BOT=$runningInDartHHHBot',
...flutterTestArgs,
...tests,
],
workingDirectory: workingDirectory,
environment: <String, String>{
'FLUTTER_WEB': 'true',
},
);
// metriciFile is a transitional file that needs to be deleted once it is parsed.
// TODO(godofredoc): Ensure metricFile is parsed and aggregated before deleting.
// https://github.com/flutter/flutter/issues/146003
metricFile.deleteSync();
}
// The `chromedriver` process created by this test.
//
// If an existing chromedriver is already available on port 4444, the existing
// process is reused and this variable remains null.
Command? _chromeDriver;
Future<bool> _isChromeDriverRunning() async {
try {
final RawSocket socket = await RawSocket.connect('localhost', 4444);
socket.shutdown(SocketDirection.both);
await socket.close();
return true;
} on SocketException {
return false;
}
}
Future<void> _stopChromeDriver() async {
if (_chromeDriver == null) {
return;
}
print('Stopping chromedriver');
_chromeDriver!.process.kill();
}
Future<void> _ensureChromeDriverIsRunning() async {
// If we cannot connect to ChromeDriver, assume it is not running. Launch it.
if (!await _isChromeDriverRunning()) {
printProgress('Starting chromedriver');
// Assume chromedriver is in the PATH.
_chromeDriver = await startCommand(
// TODO(ianh): this is the only remaining consumer of startCommand other than runCommand
// and it doesn't use most of startCommand's features; we could simplify this a lot by
// inlining the relevant parts of startCommand here.
'chromedriver',
<String>['--port=4444'],
);
while (!await _isChromeDriverRunning()) {
await Future<void>.delayed(const Duration(milliseconds: 100));
print('Waiting for chromedriver to start up.');
}
}
final HttpClient client = HttpClient();
final Uri chromeDriverUrl = Uri.parse('http://localhost:4444/status');
final HttpClientRequest request = await client.getUrl(chromeDriverUrl);
final HttpClientResponse response = await request.close();
final Map<String, dynamic> webDriverStatus = json.decode(await response.transform(utf8.decoder).join()) as Map<String, dynamic>;
client.close();
final bool webDriverReady = (webDriverStatus['value'] as Map<String, dynamic>)['ready'] as bool;
if (!webDriverReady) {
throw Exception('WebDriver not available.');
}
}
}

610
dev/bots/test.dart
View File

@ -56,8 +56,6 @@ import 'dart:math' as math;
import 'dart:typed_data';
import 'package:archive/archive.dart';
import 'package:file/file.dart' as fs;
import 'package:file/local.dart';
import 'package:path/path.dart' as path;
import 'run_command.dart';
@ -70,28 +68,11 @@ import 'suite_runners/run_fuchsia_precache.dart';
import 'suite_runners/run_realm_checker_tests.dart';
import 'suite_runners/run_skp_generator_tests.dart';
import 'suite_runners/run_verify_binaries_codesigned_tests.dart';
import 'suite_runners/run_web_long_running_tests.dart';
import 'tool_subsharding.dart';
import 'suite_runners/run_web_tests.dart';
import 'utils.dart';
typedef ShardRunner = Future<void> Function();
/// A function used to validate the output of a test.
///
/// If the output matches expectations, the function shall return null.
///
/// If the output does not match expectations, the function shall return an
/// appropriate error message.
typedef OutputChecker = String? Function(CommandResult);
final String exe = Platform.isWindows ? '.exe' : '';
final String bat = Platform.isWindows ? '.bat' : '';
final String flutterRoot = path.dirname(path.dirname(path.dirname(path.fromUri(Platform.script))));
final String flutter = path.join(flutterRoot, 'bin', 'flutter$bat');
final String dart = path.join(flutterRoot, 'bin', 'cache', 'dart-sdk', 'bin', 'dart$exe');
final String pubCache = path.join(flutterRoot, '.pub-cache');
final String engineVersionFile = path.join(flutterRoot, 'bin', 'internal', 'engine.version');
String get platformFolderName {
if (Platform.isWindows) {
return 'windows-x64';
@ -106,101 +87,13 @@ String get platformFolderName {
}
final String flutterTester = path.join(flutterRoot, 'bin', 'cache', 'artifacts', 'engine', platformFolderName, 'flutter_tester$exe');
/// The arguments to pass to `flutter test` (typically the local engine
/// configuration) -- prefilled with the arguments passed to test.dart.
final List<String> flutterTestArgs = <String>[];
/// Environment variables to override the local engine when running `pub test`,
/// if such flags are provided to `test.dart`.
final Map<String,String> localEngineEnv = <String, String>{};
const String kShardKey = 'SHARD';
const String kSubshardKey = 'SUBSHARD';
/// The number of Cirrus jobs that run Web tests in parallel.
///
/// The default is 8 shards. Typically .cirrus.yml would define the
/// WEB_SHARD_COUNT environment variable rather than relying on the default.
///
/// WARNING: if you change this number, also change .cirrus.yml
/// and make sure it runs _all_ shards.
///
/// The last shard also runs the Web plugin tests.
int get webShardCount => Platform.environment.containsKey('WEB_SHARD_COUNT')
? int.parse(Platform.environment['WEB_SHARD_COUNT']!)
: 8;
/// Tests that we don't run on Web.
///
/// In general avoid adding new tests here. If a test cannot run on the web
/// because it fails at runtime, such as when a piece of functionality is not
/// implemented or not implementable on the web, prefer using `skip` in the
/// test code. Only add tests here that cannot be skipped using `skip`. For
/// example:
///
/// * Test code cannot be compiled because it uses Dart VM-specific
/// functionality. In this case `skip` doesn't help because the code cannot
/// reach the point where it can even run the skipping logic.
/// * Migrations. It is OK to put tests here that need to be temporarily
/// disabled in certain modes because of some migration or initial bringup.
///
/// The key in the map is the renderer type that the list applies to. The value
/// is the list of tests known to fail for that renderer.
//
// TODO(yjbanov): we're getting rid of this as part of https://github.com/flutter/flutter/projects/60
const Map<String, List<String>> kWebTestFileKnownFailures = <String, List<String>>{
'html': <String>[
// These tests are not compilable on the web due to dependencies on
// VM-specific functionality.
'test/services/message_codecs_vm_test.dart',
'test/examples/sector_layout_test.dart',
],
'canvaskit': <String>[
// These tests are not compilable on the web due to dependencies on
// VM-specific functionality.
'test/services/message_codecs_vm_test.dart',
'test/examples/sector_layout_test.dart',
// These tests are broken and need to be fixed.
// TODO(yjbanov): https://github.com/flutter/flutter/issues/71604
'test/material/text_field_test.dart',
'test/widgets/performance_overlay_test.dart',
'test/widgets/html_element_view_test.dart',
'test/cupertino/scaffold_test.dart',
'test/rendering/platform_view_test.dart',
],
'skwasm': <String>[
// These tests are not compilable on the web due to dependencies on
// VM-specific functionality.
'test/services/message_codecs_vm_test.dart',
'test/examples/sector_layout_test.dart',
// These tests are broken and need to be fixed.
// TODO(jacksongardner): https://github.com/flutter/flutter/issues/71604
'test/material/text_field_test.dart',
'test/widgets/performance_overlay_test.dart',
],
};
const String kTestHarnessShardName = 'test_harness_tests';
// The seed used to shuffle tests. If not passed with
// --test-randomize-ordering-seed=<seed> on the command line, it will be set the
// first time it is accessed. Pass zero to turn off shuffling.
String? _shuffleSeed;
String get shuffleSeed {
if (_shuffleSeed == null) {
// Change the seed at 7am, UTC.
final DateTime seedTime = DateTime.now().toUtc().subtract(const Duration(hours: 7));
// Generates YYYYMMDD as the seed, so that testing continues to fail for a
// day after the seed changes, and on other days the seed can be used to
// replicate failures.
_shuffleSeed = '${seedTime.year * 10000 + seedTime.month * 100 + seedTime.day}';
}
return _shuffleSeed!;
}
final bool _isRandomizationOff = bool.tryParse(Platform.environment['TEST_RANDOMIZATION_OFF'] ?? '') ?? false;
const String CIRRUS_TASK_NAME = 'CIRRUS_TASK_NAME';
/// When you call this, you can pass additional arguments to pass custom
/// arguments to flutter test. For example, you might want to call this
@ -226,7 +119,7 @@ Future<void> main(List<String> args) async {
localEngineEnv['FLUTTER_LOCAL_ENGINE_SRC_PATH'] = arg.substring('--local-engine-src-path='.length);
flutterTestArgs.add(arg);
} else if (arg.startsWith('--test-randomize-ordering-seed=')) {
_shuffleSeed = arg.substring('--test-randomize-ordering-seed='.length);
shuffleSeed = arg.substring('--test-randomize-ordering-seed='.length);
} else if (arg.startsWith('--verbose')) {
print = (Object? message) {
system.print(message);
@ -242,6 +135,7 @@ Future<void> main(List<String> args) async {
if (Platform.environment.containsKey(CIRRUS_TASK_NAME)) {
printProgress('Running task: ${Platform.environment[CIRRUS_TASK_NAME]}');
}
final WebTestsSuite webTestsSuite = WebTestsSuite(flutterRoot, flutterTestArgs);
await selectShard(<String, ShardRunner>{
'add_to_app_life_cycle_tests': () => addToAppLifeCycleRunner(flutterRoot),
'build_tests': _runBuildTests,
@ -254,13 +148,13 @@ Future<void> main(List<String> args) async {
'android_preview_tool_integration_tests': _runAndroidPreviewIntegrationToolTests,
'tool_host_cross_arch_tests': _runToolHostCrossArchTests,
// All the unit/widget tests run using `flutter test --platform=chrome --web-renderer=html`
'web_tests': _runWebHtmlUnitTests,
'web_tests': webTestsSuite.runWebHtmlUnitTests,
// All the unit/widget tests run using `flutter test --platform=chrome --web-renderer=canvaskit`
'web_canvaskit_tests': _runWebCanvasKitUnitTests,
'web_canvaskit_tests': webTestsSuite.runWebCanvasKitUnitTests,
// All the unit/widget tests run using `flutter test --platform=chrome --wasm --web-renderer=skwasm`
'web_skwasm_tests': _runWebSkwasmUnitTests,
'web_skwasm_tests': webTestsSuite.runWebSkwasmUnitTests,
// All web integration tests
'web_long_running_tests': () => webLongRunningTestsRunner(flutterRoot),
'web_long_running_tests': webTestsSuite.webLongRunningTestsRunner,
'flutter_plugins': () => flutterPackagesRunner(flutterRoot),
'skp_generator': skpGeneratorTestsRunner,
'realm_checker': () => realmCheckerTestRunner(flutterRoot),
@ -287,19 +181,15 @@ Future<void> main(List<String> args) async {
reportSuccessAndExit('${bold}Test successful.$reset');
}
final String _luciBotId = Platform.environment['SWARMING_BOT_ID'] ?? '';
final bool _runningInDartHHHBot =
_luciBotId.startsWith('luci-dart-') || _luciBotId.startsWith('dart-tests-');
/// Verify the Flutter Engine is the revision in
/// bin/cache/internal/engine.version.
Future<void> _validateEngineHash() async {
if (_runningInDartHHHBot) {
if (runningInDartHHHBot) {
// The Dart HHH bots intentionally modify the local artifact cache
// and then use this script to run Flutter's test suites.
// Because the artifacts have been changed, this particular test will return
// a false positive and should be skipped.
print('${yellow}Skipping Flutter Engine Version Validation for swarming bot $_luciBotId.');
print('${yellow}Skipping Flutter Engine Version Validation for swarming bot $luciBotId.');
return;
}
final String expectedVersion = File(engineVersionFile).readAsStringSync().trim();
@ -341,18 +231,18 @@ Future<void> _runTestHarnessTests() async {
// on 20-core machines. However, we have a race condition, so for now...
// Race condition issue: https://github.com/flutter/flutter/issues/90026
final List<ShardRunner> tests = <ShardRunner>[
() => _runFlutterTest(
() => runFlutterTest(
automatedTests,
script: path.join('test_smoke_test', 'pass_test.dart'),
printOutput: false,
),
() => _runFlutterTest(
() => runFlutterTest(
automatedTests,
script: path.join('test_smoke_test', 'fail_test.dart'),
expectFailure: true,
printOutput: false,
),
() => _runFlutterTest(
() => runFlutterTest(
automatedTests,
script: path.join('test_smoke_test', 'pending_timer_fail_test.dart'),
expectFailure: true,
@ -365,7 +255,7 @@ Future<void> _runTestHarnessTests() async {
'stderr:\n${result.flattenedStderr}';
},
),
() => _runFlutterTest(
() => runFlutterTest(
automatedTests,
script: path.join('test_smoke_test', 'fail_test_on_exception_after_test.dart'),
expectFailure: true,
@ -383,31 +273,31 @@ Future<void> _runTestHarnessTests() async {
'Actual stderr:\n${result.flattenedStderr}';
},
),
() => _runFlutterTest(
() => runFlutterTest(
automatedTests,
script: path.join('test_smoke_test', 'crash1_test.dart'),
expectFailure: true,
printOutput: false,
),
() => _runFlutterTest(
() => runFlutterTest(
automatedTests,
script: path.join('test_smoke_test', 'crash2_test.dart'),
expectFailure: true,
printOutput: false,
),
() => _runFlutterTest(
() => runFlutterTest(
automatedTests,
script: path.join('test_smoke_test', 'syntax_error_test.broken_dart'),
expectFailure: true,
printOutput: false,
),
() => _runFlutterTest(
() => runFlutterTest(
automatedTests,
script: path.join('test_smoke_test', 'missing_import_test.broken_dart'),
expectFailure: true,
printOutput: false,
),
() => _runFlutterTest(
() => runFlutterTest(
automatedTests,
script: path.join('test_smoke_test', 'disallow_error_reporter_modification_test.dart'),
expectFailure: true,
@ -420,7 +310,7 @@ Future<void> _runTestHarnessTests() async {
// Run all tests unless sharding is explicitly specified.
final String? shardName = Platform.environment[kShardKey];
if (shardName == kTestHarnessShardName) {
testsToRun = _selectIndexOfTotalSubshard<ShardRunner>(tests);
testsToRun = selectIndexOfTotalSubshard<ShardRunner>(tests);
} else {
testsToRun = tests;
}
@ -438,7 +328,7 @@ Future<void> _runTestHarnessTests() async {
final String _toolsPath = path.join(flutterRoot, 'packages', 'flutter_tools');
Future<void> _runGeneralToolTests() async {
await _runDartTest(
await runDartTest(
_toolsPath,
testPaths: <String>[path.join('test', 'general.shard')],
enableFlutterToolAsserts: false,
@ -451,7 +341,7 @@ Future<void> _runGeneralToolTests() async {
}
Future<void> _runCommandsToolTests() async {
await _runDartTest(
await runDartTest(
_toolsPath,
forceSingleCore: true,
testPaths: <String>[path.join('test', 'commands.shard')],
@ -467,16 +357,16 @@ Future<void> _runWebToolTests() async {
allTests.add(file.path);
}
}
await _runDartTest(
await runDartTest(
_toolsPath,
forceSingleCore: true,
testPaths: _selectIndexOfTotalSubshard<String>(allTests),
testPaths: selectIndexOfTotalSubshard<String>(allTests),
includeLocalEngineEnv: true,
);
}
Future<void> _runToolHostCrossArchTests() {
return _runDartTest(
return runDartTest(
_toolsPath,
// These are integration tests
forceSingleCore: true,
@ -490,10 +380,10 @@ Future<void> _runIntegrationToolTests() async {
.map<String>((FileSystemEntity entry) => path.relative(entry.path, from: _toolsPath))
.where((String testPath) => path.basename(testPath).endsWith('_test.dart')).toList();
await _runDartTest(
await runDartTest(
_toolsPath,
forceSingleCore: true,
testPaths: _selectIndexOfTotalSubshard<String>(allTests),
testPaths: selectIndexOfTotalSubshard<String>(allTests),
collectMetrics: true,
);
}
@ -504,10 +394,10 @@ Future<void> _runAndroidPreviewIntegrationToolTests() async {
.map<String>((FileSystemEntity entry) => path.relative(entry.path, from: _toolsPath))
.where((String testPath) => path.basename(testPath).endsWith('_test.dart')).toList();
await _runDartTest(
await runDartTest(
_toolsPath,
forceSingleCore: true,
testPaths: _selectIndexOfTotalSubshard<String>(allTests),
testPaths: selectIndexOfTotalSubshard<String>(allTests),
collectMetrics: true,
);
}
@ -815,7 +705,7 @@ Future<void> _runFrameworkTests() async {
Future<void> runWidgets() async {
printProgress('${green}Running packages/flutter tests $reset for ${cyan}test/widgets/$reset');
for (final String trackWidgetCreationOption in trackWidgetCreationAlternatives) {
await _runFlutterTest(
await runFlutterTest(
path.join(flutterRoot, 'packages', 'flutter'),
options: <String>[trackWidgetCreationOption],
tests: <String>[ path.join('test', 'widgets') + path.separator ],
@ -823,20 +713,20 @@ Future<void> _runFrameworkTests() async {
}
// Try compiling code outside of the packages/flutter directory with and without --track-widget-creation
for (final String trackWidgetCreationOption in trackWidgetCreationAlternatives) {
await _runFlutterTest(
await runFlutterTest(
path.join(flutterRoot, 'dev', 'integration_tests', 'flutter_gallery'),
options: <String>[trackWidgetCreationOption],
fatalWarnings: false, // until we've migrated video_player
);
}
// Run release mode tests (see packages/flutter/test_release/README.md)
await _runFlutterTest(
await runFlutterTest(
path.join(flutterRoot, 'packages', 'flutter'),
options: <String>['--dart-define=dart.vm.product=true'],
tests: <String>['test_release${path.separator}'],
);
// Run profile mode tests (see packages/flutter/test_profile/README.md)
await _runFlutterTest(
await runFlutterTest(
path.join(flutterRoot, 'packages', 'flutter'),
options: <String>['--dart-define=dart.vm.product=false', '--dart-define=dart.vm.profile=true'],
tests: <String>['test_profile${path.separator}'],
@ -845,7 +735,7 @@ Future<void> _runFrameworkTests() async {
Future<void> runImpeller() async {
printProgress('${green}Running packages/flutter tests $reset in Impeller$reset');
await _runFlutterTest(
await runFlutterTest(
path.join(flutterRoot, 'packages', 'flutter'),
options: <String>['--enable-impeller'],
);
@ -861,7 +751,7 @@ Future<void> _runFrameworkTests() async {
.toList();
printProgress('${green}Running packages/flutter tests$reset for $cyan${tests.join(", ")}$reset');
for (final String trackWidgetCreationOption in trackWidgetCreationAlternatives) {
await _runFlutterTest(
await runFlutterTest(
path.join(flutterRoot, 'packages', 'flutter'),
options: <String>[trackWidgetCreationOption],
tests: tests,
@ -884,7 +774,7 @@ Future<void> _runFrameworkTests() async {
if (entity is! Directory || !Directory(path.join(entity.path, 'test')).existsSync()) {
continue;
}
await _runFlutterTest(entity.path);
await runFlutterTest(entity.path);
}
}
@ -892,7 +782,7 @@ Future<void> _runFrameworkTests() async {
final String tracingDirectory = path.join(flutterRoot, 'dev', 'tracing_tests');
// run the tests for debug mode
await _runFlutterTest(tracingDirectory, options: <String>['--enable-vmservice']);
await runFlutterTest(tracingDirectory, options: <String>['--enable-vmservice']);
Future<List<String>> verifyTracingAppBuild({
required String modeArgument,
@ -1022,24 +912,24 @@ Future<void> _runFrameworkTests() async {
printProgress('${green}Running package tests$reset for directories other than packages/flutter');
await _runTestHarnessTests();
await runExampleTests();
await _runFlutterTest(
await runFlutterTest(
path.join(flutterRoot, 'dev', 'a11y_assessments'),
tests: <String>[ 'test' ],
);
await _runDartTest(path.join(flutterRoot, 'dev', 'bots'));
await _runDartTest(path.join(flutterRoot, 'dev', 'devicelab'), ensurePrecompiledTool: false); // See https://github.com/flutter/flutter/issues/86209
await _runDartTest(path.join(flutterRoot, 'dev', 'conductor', 'core'), forceSingleCore: true);
await runDartTest(path.join(flutterRoot, 'dev', 'bots'));
await runDartTest(path.join(flutterRoot, 'dev', 'devicelab'), ensurePrecompiledTool: false); // See https://github.com/flutter/flutter/issues/86209
await runDartTest(path.join(flutterRoot, 'dev', 'conductor', 'core'), forceSingleCore: true);
// TODO(gspencergoog): Remove the exception for fatalWarnings once https://github.com/flutter/flutter/issues/113782 has landed.
await _runFlutterTest(path.join(flutterRoot, 'dev', 'integration_tests', 'android_semantics_testing'), fatalWarnings: false);
await _runFlutterTest(path.join(flutterRoot, 'dev', 'integration_tests', 'ui'));
await _runFlutterTest(path.join(flutterRoot, 'dev', 'manual_tests'));
await _runFlutterTest(path.join(flutterRoot, 'dev', 'tools'));
await _runFlutterTest(path.join(flutterRoot, 'dev', 'tools', 'vitool'));
await _runFlutterTest(path.join(flutterRoot, 'dev', 'tools', 'gen_defaults'));
await _runFlutterTest(path.join(flutterRoot, 'dev', 'tools', 'gen_keycodes'));
await _runFlutterTest(path.join(flutterRoot, 'dev', 'benchmarks', 'test_apps', 'stocks'));
await _runFlutterTest(path.join(flutterRoot, 'packages', 'flutter_driver'), tests: <String>[path.join('test', 'src', 'real_tests')]);
await _runFlutterTest(path.join(flutterRoot, 'packages', 'integration_test'), options: <String>[
await runFlutterTest(path.join(flutterRoot, 'dev', 'integration_tests', 'android_semantics_testing'), fatalWarnings: false);
await runFlutterTest(path.join(flutterRoot, 'dev', 'integration_tests', 'ui'));
await runFlutterTest(path.join(flutterRoot, 'dev', 'manual_tests'));
await runFlutterTest(path.join(flutterRoot, 'dev', 'tools'));
await runFlutterTest(path.join(flutterRoot, 'dev', 'tools', 'vitool'));
await runFlutterTest(path.join(flutterRoot, 'dev', 'tools', 'gen_defaults'));
await runFlutterTest(path.join(flutterRoot, 'dev', 'tools', 'gen_keycodes'));
await runFlutterTest(path.join(flutterRoot, 'dev', 'benchmarks', 'test_apps', 'stocks'));
await runFlutterTest(path.join(flutterRoot, 'packages', 'flutter_driver'), tests: <String>[path.join('test', 'src', 'real_tests')]);
await runFlutterTest(path.join(flutterRoot, 'packages', 'integration_test'), options: <String>[
'--enable-vmservice',
// Web-specific tests depend on Chromium, so they run as part of the web_long_running_tests shard.
'--exclude-tags=web',
@ -1061,18 +951,18 @@ Future<void> _runFrameworkTests() async {
],
workingDirectory: path.join(flutterRoot, 'packages', 'integration_test', 'example', 'android'),
);
await _runFlutterTest(path.join(flutterRoot, 'packages', 'flutter_goldens'));
await _runFlutterTest(path.join(flutterRoot, 'packages', 'flutter_localizations'));
await _runFlutterTest(path.join(flutterRoot, 'packages', 'flutter_test'));
await _runFlutterTest(path.join(flutterRoot, 'packages', 'fuchsia_remote_debug_protocol'));
await _runFlutterTest(path.join(flutterRoot, 'dev', 'integration_tests', 'non_nullable'));
await runFlutterTest(path.join(flutterRoot, 'packages', 'flutter_goldens'));
await runFlutterTest(path.join(flutterRoot, 'packages', 'flutter_localizations'));
await runFlutterTest(path.join(flutterRoot, 'packages', 'flutter_test'));
await runFlutterTest(path.join(flutterRoot, 'packages', 'fuchsia_remote_debug_protocol'));
await runFlutterTest(path.join(flutterRoot, 'dev', 'integration_tests', 'non_nullable'));
const String httpClientWarning =
'Warning: At least one test in this suite creates an HttpClient. When running a test suite that uses\n'
'TestWidgetsFlutterBinding, all HTTP requests will return status code 400, and no network request\n'
'will actually be made. Any test expecting a real network connection and status code will fail.\n'
'To test code that needs an HttpClient, provide your own HttpClient implementation to the code under\n'
'test, so that your test can consistently provide a testable response to the code under test.';
await _runFlutterTest(
await runFlutterTest(
path.join(flutterRoot, 'packages', 'flutter_test'),
script: path.join('test', 'bindings_test_failure.dart'),
expectFailure: true,
@ -1109,7 +999,7 @@ Future<void> _runFrameworkCoverage() async {
return;
}
coverageFile.deleteSync();
await _runFlutterTest(path.join(flutterRoot, 'packages', 'flutter'),
await runFlutterTest(path.join(flutterRoot, 'packages', 'flutter'),
options: const <String>['--coverage'],
);
if (!coverageFile.existsSync()) {
@ -1122,308 +1012,6 @@ Future<void> _runFrameworkCoverage() async {
}
}
Future<void> _runWebHtmlUnitTests() {
return _runWebUnitTests('html', false);
}
Future<void> _runWebCanvasKitUnitTests() {
return _runWebUnitTests('canvaskit', false);
}
Future<void> _runWebSkwasmUnitTests() {
return _runWebUnitTests('skwasm', true);
}
Future<void> _runWebUnitTests(String webRenderer, bool useWasm) async {
final Map<String, ShardRunner> subshards = <String, ShardRunner>{};
final Directory flutterPackageDirectory = Directory(path.join(flutterRoot, 'packages', 'flutter'));
final Directory flutterPackageTestDirectory = Directory(path.join(flutterPackageDirectory.path, 'test'));
final List<String> allTests = flutterPackageTestDirectory
.listSync()
.whereType<Directory>()
.expand((Directory directory) => directory
.listSync(recursive: true)
.where((FileSystemEntity entity) => entity.path.endsWith('_test.dart'))
)
.whereType<File>()
.map<String>((File file) => path.relative(file.path, from: flutterPackageDirectory.path))
.where((String filePath) => !kWebTestFileKnownFailures[webRenderer]!.contains(path.split(filePath).join('/')))
.toList()
// Finally we shuffle the list because we want the average cost per file to be uniformly
// distributed. If the list is not sorted then different shards and batches may have
// very different characteristics.
// We use a constant seed for repeatability.
..shuffle(math.Random(0));
assert(webShardCount >= 1);
final int testsPerShard = (allTests.length / webShardCount).ceil();
assert(testsPerShard * webShardCount >= allTests.length);
// This for loop computes all but the last shard.
for (int index = 0; index < webShardCount - 1; index += 1) {
subshards['$index'] = () => runFlutterWebTest(
webRenderer,
flutterPackageDirectory.path,
allTests.sublist(
index * testsPerShard,
(index + 1) * testsPerShard,
),
useWasm,
);
}
// The last shard also runs the flutter_web_plugins tests.
//
// We make sure the last shard ends in _last so it's easier to catch mismatches
// between `.cirrus.yml` and `test.dart`.
subshards['${webShardCount - 1}_last'] = () async {
await runFlutterWebTest(
webRenderer,
flutterPackageDirectory.path,
allTests.sublist(
(webShardCount - 1) * testsPerShard,
allTests.length,
),
useWasm,
);
await runFlutterWebTest(
webRenderer,
path.join(flutterRoot, 'packages', 'flutter_web_plugins'),
<String>['test'],
useWasm,
);
await runFlutterWebTest(
webRenderer,
path.join(flutterRoot, 'packages', 'flutter_driver'),
<String>[path.join('test', 'src', 'web_tests', 'web_extension_test.dart')],
useWasm,
);
};
await selectSubshard(subshards);
}
Future<void> runFlutterWebTest(
String webRenderer,
String workingDirectory,
List<String> tests,
bool useWasm,
) async {
const LocalFileSystem fileSystem = LocalFileSystem();
final String suffix = DateTime.now().microsecondsSinceEpoch.toString();
final File metricFile = fileSystem.systemTempDirectory.childFile('metrics_$suffix.json');
await runCommand(
flutter,
<String>[
'test',
'--reporter=expanded',
'--file-reporter=json:${metricFile.path}',
'-v',
'--platform=chrome',
if (useWasm) '--wasm',
'--web-renderer=$webRenderer',
'--dart-define=DART_HHH_BOT=$_runningInDartHHHBot',
...flutterTestArgs,
...tests,
],
workingDirectory: workingDirectory,
environment: <String, String>{
'FLUTTER_WEB': 'true',
},
);
// metriciFile is a transitional file that needs to be deleted once it is parsed.
// TODO(godofredoc): Ensure metricFile is parsed and aggregated before deleting.
// https://github.com/flutter/flutter/issues/146003
metricFile.deleteSync();
}
// TODO(sigmund): includeLocalEngineEnv should default to true. Currently we
// only enable it on flutter-web test because some test suites do not work
// properly when overriding the local engine (for example, because some platform
// dependent targets are only built on some engines).
// See https://github.com/flutter/flutter/issues/72368
Future<void> _runDartTest(String workingDirectory, {
List<String>? testPaths,
bool enableFlutterToolAsserts = true,
bool useBuildRunner = false,
String? coverage,
bool forceSingleCore = false,
Duration? perTestTimeout,
bool includeLocalEngineEnv = false,
bool ensurePrecompiledTool = true,
bool shuffleTests = true,
bool collectMetrics = false,
}) async {
int? cpus;
final String? cpuVariable = Platform.environment['CPU']; // CPU is set in cirrus.yml
if (cpuVariable != null) {
cpus = int.tryParse(cpuVariable, radix: 10);
if (cpus == null) {
foundError(<String>[
'${red}The CPU environment variable, if set, must be set to the integer number of available cores.$reset',
'Actual value: "$cpuVariable"',
]);
return;
}
} else {
cpus = 2; // Don't default to 1, otherwise we won't catch race conditions.
}
// Integration tests that depend on external processes like chrome
// can get stuck if there are multiple instances running at once.
if (forceSingleCore) {
cpus = 1;
}
const LocalFileSystem fileSystem = LocalFileSystem();
final String suffix = DateTime.now().microsecondsSinceEpoch.toString();
final File metricFile = fileSystem.systemTempDirectory.childFile('metrics_$suffix.json');
final List<String> args = <String>[
'run',
'test',
'--reporter=expanded',
'--file-reporter=json:${metricFile.path}',
if (shuffleTests) '--test-randomize-ordering-seed=$shuffleSeed',
'-j$cpus',
if (!hasColor)
'--no-color',
if (coverage != null)
'--coverage=$coverage',
if (perTestTimeout != null)
'--timeout=${perTestTimeout.inMilliseconds}ms',
if (testPaths != null)
for (final String testPath in testPaths)
testPath,
];
final Map<String, String> environment = <String, String>{
'FLUTTER_ROOT': flutterRoot,
if (includeLocalEngineEnv)
...localEngineEnv,
if (Directory(pubCache).existsSync())
'PUB_CACHE': pubCache,
};
if (enableFlutterToolAsserts) {
adjustEnvironmentToEnableFlutterAsserts(environment);
}
if (ensurePrecompiledTool) {
// We rerun the `flutter` tool here just to make sure that it is compiled
// before tests run, because the tests might time out if they have to rebuild
// the tool themselves.
await runCommand(flutter, <String>['--version'], environment: environment);
}
await runCommand(
dart,
args,
workingDirectory: workingDirectory,
environment: environment,
removeLine: useBuildRunner ? (String line) => line.startsWith('[INFO]') : null,
);
final TestFileReporterResults test = TestFileReporterResults.fromFile(metricFile); // --file-reporter name
final File info = fileSystem.file(path.join(flutterRoot, 'error.log'));
info.writeAsStringSync(json.encode(test.errors));
if (collectMetrics) {
try {
final List<String> testList = <String>[
for (final TestSpecs testSpecs in test.allTestSpecs.values) testSpecs.toJson(),
];
if (testList.isNotEmpty) {
final String testJson = json.encode(testList);
final File testResults = fileSystem.file(
path.join(flutterRoot, 'test_results.json'));
testResults.writeAsStringSync(testJson);
}
} on fs.FileSystemException catch (e) {
print('Failed to generate metrics: $e');
}
}
// metriciFile is a transitional file that needs to be deleted once it is parsed.
// TODO(godofredoc): Ensure metricFile is parsed and aggregated before deleting.
// https://github.com/flutter/flutter/issues/146003
metricFile.deleteSync();
}
Future<void> _runFlutterTest(String workingDirectory, {
String? script,
bool expectFailure = false,
bool printOutput = true,
OutputChecker? outputChecker,
List<String> options = const <String>[],
Map<String, String>? environment,
List<String> tests = const <String>[],
bool shuffleTests = true,
bool fatalWarnings = true,
}) async {
assert(!printOutput || outputChecker == null, 'Output either can be printed or checked but not both');
final List<String> tags = <String>[];
// Recipe-configured reduced test shards will only execute tests with the
// appropriate tag.
if (Platform.environment['REDUCED_TEST_SET'] == 'True') {
tags.addAll(<String>['-t', 'reduced-test-set']);
}
const LocalFileSystem fileSystem = LocalFileSystem();
final String suffix = DateTime.now().microsecondsSinceEpoch.toString();
final File metricFile = fileSystem.systemTempDirectory.childFile('metrics_$suffix.json');
final List<String> args = <String>[
'test',
'--reporter=expanded',
'--file-reporter=json:${metricFile.path}',
if (shuffleTests && !_isRandomizationOff) '--test-randomize-ordering-seed=$shuffleSeed',
if (fatalWarnings) '--fatal-warnings',
...options,
...tags,
...flutterTestArgs,
];
if (script != null) {
final String fullScriptPath = path.join(workingDirectory, script);
if (!FileSystemEntity.isFileSync(fullScriptPath)) {
foundError(<String>[
'${red}Could not find test$reset: $green$fullScriptPath$reset',
'Working directory: $cyan$workingDirectory$reset',
'Script: $green$script$reset',
if (!printOutput)
'This is one of the tests that does not normally print output.',
]);
return;
}
args.add(script);
}
args.addAll(tests);
final OutputMode outputMode = outputChecker == null && printOutput
? OutputMode.print
: OutputMode.capture;
final CommandResult result = await runCommand(
flutter,
args,
workingDirectory: workingDirectory,
expectNonZeroExit: expectFailure,
outputMode: outputMode,
environment: environment,
);
// metriciFile is a transitional file that needs to be deleted once it is parsed.
// TODO(godofredoc): Ensure metricFile is parsed and aggregated before deleting.
// https://github.com/flutter/flutter/issues/146003
metricFile.deleteSync();
if (outputChecker != null) {
final String? message = outputChecker(result);
if (message != null) {
foundError(<String>[message]);
}
}
}
/// This will force the next run of the Flutter tool (if it uses the provided
/// environment) to have asserts enabled, by setting an environment variable.
@ -1457,85 +1045,3 @@ Future<String?> verifyVersion(File file) async {
}
return null;
}
/// Parse (one-)index/total-named subshards from environment variable SUBSHARD
/// and equally distribute [tests] between them.
/// Subshard format is "{index}_{total number of shards}".
/// The scheduler can change the number of total shards without needing an additional
/// commit in this repository.
///
/// Examples:
/// 1_3
/// 2_3
/// 3_3
List<T> _selectIndexOfTotalSubshard<T>(List<T> tests, {String subshardKey = kSubshardKey}) {
// Example: "1_3" means the first (one-indexed) shard of three total shards.
final String? subshardName = Platform.environment[subshardKey];
if (subshardName == null) {
print('$kSubshardKey environment variable is missing, skipping sharding');
return tests;
}
printProgress('$bold$subshardKey=$subshardName$reset');
final RegExp pattern = RegExp(r'^(\d+)_(\d+)$');
final Match? match = pattern.firstMatch(subshardName);
if (match == null || match.groupCount != 2) {
foundError(<String>[
'${red}Invalid subshard name "$subshardName". Expected format "[int]_[int]" ex. "1_3"',
]);
throw Exception('Invalid subshard name: $subshardName');
}
// One-indexed.
final int index = int.parse(match.group(1)!);
final int total = int.parse(match.group(2)!);
if (index > total) {
foundError(<String>[
'${red}Invalid subshard name "$subshardName". Index number must be greater or equal to total.',
]);
return <T>[];
}
final int testsPerShard = (tests.length / total).ceil();
final int start = (index - 1) * testsPerShard;
final int end = math.min(index * testsPerShard, tests.length);
print('Selecting subshard $index of $total (tests ${start + 1}-$end of ${tests.length})');
return tests.sublist(start, end);
}
Future<void> runShardRunnerIndexOfTotalSubshard(List<ShardRunner> tests) async {
final List<ShardRunner> sublist = _selectIndexOfTotalSubshard<ShardRunner>(tests);
for (final ShardRunner test in sublist) {
await test();
}
}
Future<void> selectShard(Map<String, ShardRunner> shards) => _runFromList(shards, kShardKey, 'shard', 0);
Future<void> selectSubshard(Map<String, ShardRunner> subshards) => _runFromList(subshards, kSubshardKey, 'subshard', 1);
const String CIRRUS_TASK_NAME = 'CIRRUS_TASK_NAME';
Future<void> _runFromList(Map<String, ShardRunner> items, String key, String name, int positionInTaskName) async {
String? item = Platform.environment[key];
if (item == null && Platform.environment.containsKey(CIRRUS_TASK_NAME)) {
final List<String> parts = Platform.environment[CIRRUS_TASK_NAME]!.split('-');
assert(positionInTaskName < parts.length);
item = parts[positionInTaskName];
}
if (item == null) {
for (final String currentItem in items.keys) {
printProgress('$bold$key=$currentItem$reset');
await items[currentItem]!();
}
} else {
printProgress('$bold$key=$item$reset');
if (!items.containsKey(item)) {
foundError(<String>[
'${red}Invalid $name: $item$reset',
'The available ${name}s are: ${items.keys.join(", ")}',
]);
return;
}
await items[item]!();
}
}

1
dev/bots/test/test_test.dart
View File

@ -9,6 +9,7 @@ import 'package:file/memory.dart';
import 'package:path/path.dart' as path;
import 'package:process/process.dart';
import '../analyze.dart';
import '../suite_runners/run_flutter_packages_tests.dart';
import '../test.dart';
import 'common.dart';

345
dev/bots/utils.dart
View File

@ -3,6 +3,7 @@
// found in the LICENSE file.
import 'dart:async';
import 'dart:convert';
import 'dart:core' hide print;
import 'dart:io' as system show exit;
import 'dart:io' hide exit;
@ -10,15 +11,31 @@ import 'dart:math' as math;
import 'package:analyzer/dart/analysis/results.dart';
import 'package:analyzer/dart/ast/ast.dart';
import 'package:file/file.dart' as fs;
import 'package:file/local.dart';
import 'package:meta/meta.dart';
import 'package:path/path.dart' as path;
import 'run_command.dart';
import 'tool_subsharding.dart';
typedef ShardRunner = Future<void> Function();
/// A function used to validate the output of a test.
///
/// If the output matches expectations, the function shall return null.
///
/// If the output does not match expectations, the function shall return an
/// appropriate error message.
typedef OutputChecker = String? Function(CommandResult);
const Duration _quietTimeout = Duration(minutes: 10); // how long the output should be hidden between calls to printProgress before just being verbose
// If running from LUCI set to False.
final bool isLuci = Platform.environment['LUCI_CI'] == 'True';
final bool hasColor = stdout.supportsAnsiEscapes && !isLuci;
final bool _isRandomizationOff = bool.tryParse(Platform.environment['TEST_RANDOMIZATION_OFF'] ?? '') ?? false;
final String bold = hasColor ? '\x1B[1m' : ''; // shard titles
final String red = hasColor ? '\x1B[31m' : ''; // errors
@ -30,6 +47,30 @@ final String gray = hasColor ? '\x1B[30m' : ''; // subtle decorative items (usua
final String white = hasColor ? '\x1B[37m' : ''; // last log line (usually renders as light gray)
final String reset = hasColor ? '\x1B[0m' : '';
final String exe = Platform.isWindows ? '.exe' : '';
final String bat = Platform.isWindows ? '.bat' : '';
final String flutterRoot = path.dirname(path.dirname(path.dirname(path.fromUri(Platform.script))));
final String flutter = path.join(flutterRoot, 'bin', 'flutter$bat');
final String dart = path.join(flutterRoot, 'bin', 'cache', 'dart-sdk', 'bin', 'dart$exe');
final String pubCache = path.join(flutterRoot, '.pub-cache');
final String engineVersionFile = path.join(flutterRoot, 'bin', 'internal', 'engine.version');
final String luciBotId = Platform.environment['SWARMING_BOT_ID'] ?? '';
final bool runningInDartHHHBot =
luciBotId.startsWith('luci-dart-') || luciBotId.startsWith('dart-tests-');
const String kShardKey = 'SHARD';
const String kSubshardKey = 'SUBSHARD';
const String CIRRUS_TASK_NAME = 'CIRRUS_TASK_NAME';
/// Environment variables to override the local engine when running `pub test`,
/// if such flags are provided to `test.dart`.
final Map<String,String> localEngineEnv = <String, String>{};
/// The arguments to pass to `flutter test` (typically the local engine
/// configuration) -- prefilled with the arguments passed to test.dart.
final List<String> flutterTestArgs = <String>[];
const int kESC = 0x1B;
const int kOpenSquareBracket = 0x5B;
const int kCSIParameterRangeStart = 0x30;
@ -39,7 +80,6 @@ const int kCSIIntermediateRangeEnd = 0x2F;
const int kCSIFinalRangeStart = 0x40;
const int kCSIFinalRangeEnd = 0x7E;
String get redLine {
if (hasColor) {
return '$red${'━' * stdout.terminalColumns}$reset';
@ -260,3 +300,304 @@ Future<bool> _isPortAvailable(int port) async {
String locationInFile(ResolvedUnitResult unit, AstNode node, String workingDirectory) {
return '${path.relative(path.relative(unit.path, from: workingDirectory))}:${unit.lineInfo.getLocation(node.offset).lineNumber}';
}
// The seed used to shuffle tests. If not passed with
// --test-randomize-ordering-seed=<seed> on the command line, it will be set the
// first time it is accessed. Pass zero to turn off shuffling.
String? _shuffleSeed;
set shuffleSeed(String? newSeed) {
_shuffleSeed = newSeed;
}
String get shuffleSeed {
if (_shuffleSeed != null) {
return _shuffleSeed!;
}
// Attempt to load from the command-line argument
final String? seedArg = Platform.environment['--test-randomize-ordering-seed'];
if (seedArg != null) {
return seedArg;
}
// Fallback to the original time-based seed generation
final DateTime seedTime = DateTime.now().toUtc().subtract(const Duration(hours: 7));
_shuffleSeed = '${seedTime.year * 10000 + seedTime.month * 100 + seedTime.day}';
return _shuffleSeed!;
}
// TODO(sigmund): includeLocalEngineEnv should default to true. Currently we
// only enable it on flutter-web test because some test suites do not work
// properly when overriding the local engine (for example, because some platform
// dependent targets are only built on some engines).
// See https://github.com/flutter/flutter/issues/72368
Future<void> runDartTest(String workingDirectory, {
List<String>? testPaths,
bool enableFlutterToolAsserts = true,
bool useBuildRunner = false,
String? coverage,
bool forceSingleCore = false,
Duration? perTestTimeout,
bool includeLocalEngineEnv = false,
bool ensurePrecompiledTool = true,
bool shuffleTests = true,
bool collectMetrics = false,
}) async {
int? cpus;
final String? cpuVariable = Platform.environment['CPU']; // CPU is set in cirrus.yml
if (cpuVariable != null) {
cpus = int.tryParse(cpuVariable, radix: 10);
if (cpus == null) {
foundError(<String>[
'${red}The CPU environment variable, if set, must be set to the integer number of available cores.$reset',
'Actual value: "$cpuVariable"',
]);
return;
}
} else {
cpus = 2; // Don't default to 1, otherwise we won't catch race conditions.
}
// Integration tests that depend on external processes like chrome
// can get stuck if there are multiple instances running at once.
if (forceSingleCore) {
cpus = 1;
}
const LocalFileSystem fileSystem = LocalFileSystem();
final String suffix = DateTime.now().microsecondsSinceEpoch.toString();
final File metricFile = fileSystem.systemTempDirectory.childFile('metrics_$suffix.json');
final List<String> args = <String>[
'run',
'test',
'--reporter=expanded',
'--file-reporter=json:${metricFile.path}',
if (shuffleTests) '--test-randomize-ordering-seed=$shuffleSeed',
'-j$cpus',
if (!hasColor)
'--no-color',
if (coverage != null)
'--coverage=$coverage',
if (perTestTimeout != null)
'--timeout=${perTestTimeout.inMilliseconds}ms',
if (testPaths != null)
for (final String testPath in testPaths)
testPath,
];
final Map<String, String> environment = <String, String>{
'FLUTTER_ROOT': flutterRoot,
if (includeLocalEngineEnv)
...localEngineEnv,
if (Directory(pubCache).existsSync())
'PUB_CACHE': pubCache,
};
if (enableFlutterToolAsserts) {
adjustEnvironmentToEnableFlutterAsserts(environment);
}
if (ensurePrecompiledTool) {
// We rerun the `flutter` tool here just to make sure that it is compiled
// before tests run, because the tests might time out if they have to rebuild
// the tool themselves.
await runCommand(flutter, <String>['--version'], environment: environment);
}
await runCommand(
dart,
args,
workingDirectory: workingDirectory,
environment: environment,
removeLine: useBuildRunner ? (String line) => line.startsWith('[INFO]') : null,
);
final TestFileReporterResults test = TestFileReporterResults.fromFile(metricFile); // --file-reporter name
final File info = fileSystem.file(path.join(flutterRoot, 'error.log'));
info.writeAsStringSync(json.encode(test.errors));
if (collectMetrics) {
try {
final List<String> testList = <String>[];
final Map<int, TestSpecs> allTestSpecs = test.allTestSpecs;
for (final TestSpecs testSpecs in allTestSpecs.values) {
testList.add(testSpecs.toJson());
}
if (testList.isNotEmpty) {
final String testJson = json.encode(testList);
final File testResults = fileSystem.file(
path.join(flutterRoot, 'test_results.json'));
testResults.writeAsStringSync(testJson);
}
} on fs.FileSystemException catch (e) {
print('Failed to generate metrics: $e');
}
}
// metriciFile is a transitional file that needs to be deleted once it is parsed.
// TODO(godofredoc): Ensure metricFile is parsed and aggregated before deleting.
// https://github.com/flutter/flutter/issues/146003
metricFile.deleteSync();
}
Future<void> runFlutterTest(String workingDirectory, {
String? script,
bool expectFailure = false,
bool printOutput = true,
OutputChecker? outputChecker,
List<String> options = const <String>[],
Map<String, String>? environment,
List<String> tests = const <String>[],
bool shuffleTests = true,
bool fatalWarnings = true,
}) async {
assert(!printOutput || outputChecker == null, 'Output either can be printed or checked but not both');
final List<String> tags = <String>[];
// Recipe-configured reduced test shards will only execute tests with the
// appropriate tag.
if (Platform.environment['REDUCED_TEST_SET'] == 'True') {
tags.addAll(<String>['-t', 'reduced-test-set']);
}
const LocalFileSystem fileSystem = LocalFileSystem();
final String suffix = DateTime.now().microsecondsSinceEpoch.toString();
final File metricFile = fileSystem.systemTempDirectory.childFile('metrics_$suffix.json');
final List<String> args = <String>[
'test',
'--reporter=expanded',
'--file-reporter=json:${metricFile.path}',
if (shuffleTests && !_isRandomizationOff) '--test-randomize-ordering-seed=$shuffleSeed',
if (fatalWarnings) '--fatal-warnings',
...options,
...tags,
...flutterTestArgs,
];
if (script != null) {
final String fullScriptPath = path.join(workingDirectory, script);
if (!FileSystemEntity.isFileSync(fullScriptPath)) {
foundError(<String>[
'${red}Could not find test$reset: $green$fullScriptPath$reset',
'Working directory: $cyan$workingDirectory$reset',
'Script: $green$script$reset',
if (!printOutput)
'This is one of the tests that does not normally print output.',
]);
return;
}
args.add(script);
}
args.addAll(tests);
final OutputMode outputMode = outputChecker == null && printOutput
? OutputMode.print
: OutputMode.capture;
final CommandResult result = await runCommand(
flutter,
args,
workingDirectory: workingDirectory,
expectNonZeroExit: expectFailure,
outputMode: outputMode,
environment: environment,
);
// metriciFile is a transitional file that needs to be deleted once it is parsed.
// TODO(godofredoc): Ensure metricFile is parsed and aggregated before deleting.
// https://github.com/flutter/flutter/issues/146003
metricFile.deleteSync();
if (outputChecker != null) {
final String? message = outputChecker(result);
if (message != null) {
foundError(<String>[message]);
}
}
}
/// This will force the next run of the Flutter tool (if it uses the provided
/// environment) to have asserts enabled, by setting an environment variable.
void adjustEnvironmentToEnableFlutterAsserts(Map<String, String> environment) {
// If an existing env variable exists append to it, but only if
// it doesn't appear to already include enable-asserts.
String toolsArgs = Platform.environment['FLUTTER_TOOL_ARGS'] ?? '';
if (!toolsArgs.contains('--enable-asserts')) {
toolsArgs += ' --enable-asserts';
}
environment['FLUTTER_TOOL_ARGS'] = toolsArgs.trim();
}
Future<void> selectShard(Map<String, ShardRunner> shards) => _runFromList(shards, kShardKey, 'shard', 0);
Future<void> selectSubshard(Map<String, ShardRunner> subshards) => _runFromList(subshards, kSubshardKey, 'subshard', 1);
Future<void> runShardRunnerIndexOfTotalSubshard(List<ShardRunner> tests) async {
final List<ShardRunner> sublist = selectIndexOfTotalSubshard<ShardRunner>(tests);
for (final ShardRunner test in sublist) {
await test();
}
}
/// Parse (one-)index/total-named subshards from environment variable SUBSHARD
/// and equally distribute [tests] between them.
/// Subshard format is "{index}_{total number of shards}".
/// The scheduler can change the number of total shards without needing an additional
/// commit in this repository.
///
/// Examples:
/// 1_3
/// 2_3
/// 3_3
List<T> selectIndexOfTotalSubshard<T>(List<T> tests, {String subshardKey = kSubshardKey}) {
// Example: "1_3" means the first (one-indexed) shard of three total shards.
final String? subshardName = Platform.environment[subshardKey];
if (subshardName == null) {
print('$kSubshardKey environment variable is missing, skipping sharding');
return tests;
}
printProgress('$bold$subshardKey=$subshardName$reset');
final RegExp pattern = RegExp(r'^(\d+)_(\d+)$');
final Match? match = pattern.firstMatch(subshardName);
if (match == null || match.groupCount != 2) {
foundError(<String>[
'${red}Invalid subshard name "$subshardName". Expected format "[int]_[int]" ex. "1_3"',
]);
throw Exception('Invalid subshard name: $subshardName');
}
// One-indexed.
final int index = int.parse(match.group(1)!);
final int total = int.parse(match.group(2)!);
if (index > total) {
foundError(<String>[
'${red}Invalid subshard name "$subshardName". Index number must be greater or equal to total.',
]);
return <T>[];
}
final int testsPerShard = (tests.length / total).ceil();
final int start = (index - 1) * testsPerShard;
final int end = math.min(index * testsPerShard, tests.length);
print('Selecting subshard $index of $total (tests ${start + 1}-$end of ${tests.length})');
return tests.sublist(start, end);
}
Future<void> _runFromList(Map<String, ShardRunner> items, String key, String name, int positionInTaskName) async {
String? item = Platform.environment[key];
if (item == null && Platform.environment.containsKey(CIRRUS_TASK_NAME)) {
final List<String> parts = Platform.environment[CIRRUS_TASK_NAME]!.split('-');
assert(positionInTaskName < parts.length);
item = parts[positionInTaskName];
}
if (item == null) {
for (final String currentItem in items.keys) {
printProgress('$bold$key=$currentItem$reset');
await items[currentItem]!();
}
} else {
printProgress('$bold$key=$item$reset');
if (!items.containsKey(item)) {
foundError(<String>[
'${red}Invalid $name: $item$reset',
'The available ${name}s are: ${items.keys.join(", ")}',
]);
return;
}
await items[item]!();
}
}