firka/flutter
firka/flutter
5
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

consolidate all web integration tests under the same shard (#82307)

Browse Source 
* consolidate all web integration tests under the same shard
This commit is contained in:
Yegor 2021-05-12 15:01:11 -07:00 committed by GitHub
parent 13a72114b2
commit 0420d56335
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 311 additions and 282 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
.cirrus.yml
View File

@ -140,16 +140,6 @@ task:
- (cd dev/bots; pub get)
- dart --enable-asserts ./dev/bots/test.dart
- name: web_integration_tests
only_if: "changesInclude('.cirrus.yml', 'dev/**', 'packages/flutter/**', 'packages/flutter_test/**', 'packages/flutter_tools/**', 'packages/flutter_web_plugins/**', 'bin/**') && $CIRRUS_PR != ''"
environment:
# As of October 2019, the Web shards needed more than 6G of RAM.
CPU: 2
MEMORY: 8G
CHROME_NO_SANDBOX: true
script:
- dart --enable-asserts ./dev/bots/test.dart
- name: docs-linux # linux-only
environment:
CPU: 4

25
dev/bots/service_worker_test.dart
View File

@ -12,6 +12,7 @@ import 'package:shelf/shelf.dart';
import 'browser.dart';
import 'run_command.dart';
import 'test/common.dart';
import 'utils.dart';
final String _bat = Platform.isWindows ? '.bat' : '';
final String _flutterRoot = path.dirname(path.dirname(path.dirname(path.fromUri(Platform.script))));
@ -53,10 +54,22 @@ Future<void> _rebuildApp({ @required int version }) async {
);
}
/// A drop-in replacement for `package:test` expect that can run outside the
/// test zone.
void expect(Object actual, Object expected) {
final Matcher matcher = wrapMatcher(expected);
final Map<Object, Object> matchState = <Object, Object>{};
if (matcher.matches(actual, matchState)) {
return;
}
final StringDescription mismatchDescription = StringDescription();
matcher.describeMismatch(actual, mismatchDescription, matchState, true);
throw TestFailure(mismatchDescription.toString());
}
Future<void> runWebServiceWorkerTest({
@required bool headless,
}) async {
test('flutter_service_worker.js', () async {
await _rebuildApp(version: 1);
final Map<String, int> requestedPathCounts = <String, int>{};
@ -85,10 +98,16 @@ Future<void> runWebServiceWorkerTest({
Future<void> startAppServer({
@required String cacheControl,
}) async {
final int serverPort = await findAvailablePort();
final int browserDebugPort = await findAvailablePort();
server = await AppServer.start(
headless: headless,
cacheControl: cacheControl,
appUrl: 'http://localhost:8080/index.html',
// TODO(yjbanov): use a better port disambiguation strategy than trying
// to guess what ports other tests use.
appUrl: 'http://localhost:$serverPort/index.html',
serverPort: serverPort,
browserDebugPort: browserDebugPort,
appDirectory: _appBuildDirectory,
additionalRequestHandlers: <Handler>[
(Request request) {
@ -262,6 +281,4 @@ Future<void> runWebServiceWorkerTest({
await _setAppVersion(1);
await server?.stop();
}
// This is a long test. The default 30 seconds is not enough.
}, timeout: const Timeout(Duration(minutes: 10)));
}

85
dev/bots/test.dart
View File

@ -108,8 +108,9 @@ Future<void> main(List<String> args) async {
'tool_tests': _runToolTests,
'web_tool_tests': _runToolTests,
'tool_integration_tests': _runIntegrationToolTests,
// All the unit/widget tests run using `flutter test --platform=chrome`
'web_tests': _runWebUnitTests,
'web_integration_tests': _runWebIntegrationTests,
// All web integration tests
'web_long_running_tests': _runWebLongRunningTests,
'flutter_plugins': _runFlutterPluginsTests,
'skp_generator': _runSkpGeneratorTests,
@ -850,6 +851,34 @@ Future<void> _runWebLongRunningTests() async {
() => _runGalleryE2eWebTest('release'),
() => _runGalleryE2eWebTest('release', canvasKit: true),
() => runWebServiceWorkerTest(headless: true),
() => _runWebStackTraceTest('profile', 'lib/stack_trace.dart'),
() => _runWebStackTraceTest('release', 'lib/stack_trace.dart'),
() => _runWebStackTraceTest('profile', 'lib/framework_stack_trace.dart'),
() => _runWebStackTraceTest('release', 'lib/framework_stack_trace.dart'),
() => _runWebDebugTest('lib/stack_trace.dart'),
() => _runWebDebugTest('lib/framework_stack_trace.dart'),
() => _runWebDebugTest('lib/web_directory_loading.dart'),
() => _runWebDebugTest('test/test.dart'),
() => _runWebDebugTest('lib/null_assert_main.dart', enableNullSafety: true),
() => _runWebDebugTest('lib/null_safe_main.dart', enableNullSafety: true),
() => _runWebDebugTest('lib/web_define_loading.dart',
additionalArguments: <String>[
'--dart-define=test.valueA=Example,A',
'--dart-define=test.valueB=Value',
]
),
() => _runWebReleaseTest('lib/web_define_loading.dart',
additionalArguments: <String>[
'--dart-define=test.valueA=Example,A',
'--dart-define=test.valueB=Value',
]
),
() => _runWebDebugTest('lib/sound_mode.dart', additionalArguments: <String>[
'--sound-null-safety',
]),
() => _runWebReleaseTest('lib/sound_mode.dart', additionalArguments: <String>[
'--sound-null-safety',
]),
];
await _ensureChromeDriverIsRunning();
await _runShardRunnerIndexOfTotalSubshard(tests);
@ -1069,48 +1098,6 @@ Future<void> _runGalleryE2eWebTest(String buildMode, { bool canvasKit = false })
print('${green}Integration test passed.$reset');
}
/// !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
/// !!! WARNING WARNING WARNING WARNING WARNING WARNING!!!
/// !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
///
/// Do not put any more tests here. This shard is not properly subsharded.
/// Adding more tests here will linearly increase the runtime of the shard
/// making the overall Flutter CI build longer. Consider adding tests to
/// [_runWebLongRunningTests] instead (increasing subshard count if necessary).
///
// TODO(yjbanov): increase subshard count in _runWebLongRunningTests and retire
// this shard.
Future<void> _runWebIntegrationTests() async {
await _runWebStackTraceTest('profile', 'lib/stack_trace.dart');
await _runWebStackTraceTest('release', 'lib/stack_trace.dart');
await _runWebStackTraceTest('profile', 'lib/framework_stack_trace.dart');
await _runWebStackTraceTest('release', 'lib/framework_stack_trace.dart');
await _runWebDebugTest('lib/stack_trace.dart');
await _runWebDebugTest('lib/framework_stack_trace.dart');
await _runWebDebugTest('lib/web_directory_loading.dart');
await _runWebDebugTest('test/test.dart');
await _runWebDebugTest('lib/null_assert_main.dart', enableNullSafety: true);
await _runWebDebugTest('lib/null_safe_main.dart', enableNullSafety: true);
await _runWebDebugTest('lib/web_define_loading.dart',
additionalArguments: <String>[
'--dart-define=test.valueA=Example,A',
'--dart-define=test.valueB=Value',
]
);
await _runWebReleaseTest('lib/web_define_loading.dart',
additionalArguments: <String>[
'--dart-define=test.valueA=Example,A',
'--dart-define=test.valueB=Value',
]
);
await _runWebDebugTest('lib/sound_mode.dart', additionalArguments: <String>[
'--sound-null-safety',
]);
await _runWebReleaseTest('lib/sound_mode.dart', additionalArguments: <String>[
'--sound-null-safety',
]);
}
Future<void> _runWebStackTraceTest(String buildMode, String entrypoint) async {
final String testAppDirectory = path.join(flutterRoot, 'dev', 'integration_tests', 'web');
final String appBuildDirectory = path.join(testAppDirectory, 'build', 'web');
@ -1137,9 +1124,13 @@ Future<void> _runWebStackTraceTest(String buildMode, String entrypoint) async {
);
// Run the app.
final int serverPort = await findAvailablePort();
final int browserDebugPort = await findAvailablePort();
final String result = await evalTestAppInChrome(
appUrl: 'http://localhost:8080/index.html',
appUrl: 'http://localhost:$serverPort/index.html',
appDirectory: appBuildDirectory,
serverPort: serverPort,
browserDebugPort: browserDebugPort,
);
if (result.contains('--- TEST SUCCEEDED ---')) {
@ -1181,9 +1172,13 @@ Future<void> _runWebReleaseTest(String target, {
);
// Run the app.
final int serverPort = await findAvailablePort();
final int browserDebugPort = await findAvailablePort();
final String result = await evalTestAppInChrome(
appUrl: 'http://localhost:8080/index.html',
appUrl: 'http://localhost:$serverPort/index.html',
appDirectory: appBuildDirectory,
serverPort: serverPort,
browserDebugPort: browserDebugPort,
);
if (result.contains('--- TEST SUCCEEDED ---')) {

21
dev/bots/utils.dart
View File

@ -70,3 +70,24 @@ String prettyPrintDuration(Duration duration) {
void printProgress(String action, String workingDir, String command) {
print('$clock $action: cd $cyan$workingDir$reset; $green$command$reset');
}
int _portCounter = 8080;
/// Finds the next available local port.
Future<int> findAvailablePort() async {
while (!await _isPortAvailable(_portCounter)) {
_portCounter += 1;
}
return _portCounter++;
}
Future<bool> _isPortAvailable(int port) async {
try {
final RawSocket socket = await RawSocket.connect('localhost', port);
socket.shutdown(SocketDirection.both);
await socket.close();
return false;
} on SocketException {
return true;
}
}

22
dev/prod_builders.json
View File

@ -523,27 +523,33 @@
"flaky": false
},
{
"name": "Linux web_integration_tests",
"name": "Linux web_long_running_tests_1_5",
"repo": "flutter",
"task_name": "linux_web_integration_tests",
"task_name": "linux_web_long_running_tests_1_5",
"flaky": false
},
{
"name": "Linux web_long_running_tests_1_3",
"name": "Linux web_long_running_tests_2_5",
"repo": "flutter",
"task_name": "linux_web_long_running_tests_1_3",
"task_name": "linux_web_long_running_tests_2_5",
"flaky": false
},
{
"name": "Linux web_long_running_tests_2_3",
"name": "Linux web_long_running_tests_3_5",
"repo": "flutter",
"task_name": "linux_web_long_running_tests_2_3",
"task_name": "linux_web_long_running_tests_3_5",
"flaky": false
},
{
"name": "Linux web_long_running_tests_3_3",
"name": "Linux web_long_running_tests_4_5",
"repo": "flutter",
"task_name": "linux_web_long_running_tests_3_3",
"task_name": "linux_web_long_running_tests_4_5",
"flaky": false
},
{
"name": "Linux web_long_running_tests_5_5",
"repo": "flutter",
"task_name": "linux_web_long_running_tests_5_5",
"flaky": false
},
{

40
dev/try_builders.json
View File

@ -305,37 +305,37 @@
"run_if": ["dev/", "packages/", "bin/"]
},
{
"name": "Linux web_integration_tests",
"name": "Linux web_long_running_tests_1_5",
"repo": "flutter",
"task_name": "linux_web_integration_tests",
"enabled": true,
"run_if": [
"dev/",
"packages/flutter/",
"packages/flutter_test/",
"packages/flutter_tools/",
"packages/flutter_web_plugins/",
"bin/"
]
},
{
"name": "Linux web_long_running_tests_1_3",
"repo": "flutter",
"task_name": "web_long_running_tests_1_3",
"task_name": "web_long_running_tests_1_5",
"enabled": true,
"run_if": ["dev/", "packages/", "bin/"]
},
{
"name": "Linux web_long_running_tests_2_3",
"name": "Linux web_long_running_tests_2_5",
"repo": "flutter",
"task_name": "web_long_running_tests_2_3",
"task_name": "web_long_running_tests_2_5",
"enabled": true,
"run_if": ["dev/", "packages/", "bin/"]
},
{
"name": "Linux web_long_running_tests_3_3",
"name": "Linux web_long_running_tests_3_5",
"repo": "flutter",
"task_name": "web_long_running_tests_1_3",
"task_name": "web_long_running_tests_3_5",
"enabled": true,
"run_if": ["dev/", "packages/", "bin/"]
},
{
"name": "Linux web_long_running_tests_4_5",
"repo": "flutter",
"task_name": "web_long_running_tests_4_5",
"enabled": true,
"run_if": ["dev/", "packages/", "bin/"]
},
{
"name": "Linux web_long_running_tests_5_5",
"repo": "flutter",
"task_name": "web_long_running_tests_5_5",
"enabled": true,
"run_if": ["dev/", "packages/", "bin/"]
},