From 0420d56335d11900555c3b3f9360bd387da6aef3 Mon Sep 17 00:00:00 2001 From: Yegor Date: Wed, 12 May 2021 15:01:11 -0700 Subject: [PATCH] consolidate all web integration tests under the same shard (#82307) * consolidate all web integration tests under the same shard --- .cirrus.yml | 10 - dev/bots/service_worker_test.dart | 415 ++++++++++++++++-------------- dev/bots/test.dart | 85 +++--- dev/bots/utils.dart | 21 ++ dev/prod_builders.json | 22 +- dev/try_builders.json | 40 +-- 6 files changed, 311 insertions(+), 282 deletions(-) diff --git a/.cirrus.yml b/.cirrus.yml index b11c708d2e..4cd648e849 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -140,16 +140,6 @@ task: - (cd dev/bots; pub get) - dart --enable-asserts ./dev/bots/test.dart - - name: web_integration_tests - only_if: "changesInclude('.cirrus.yml', 'dev/**', 'packages/flutter/**', 'packages/flutter_test/**', 'packages/flutter_tools/**', 'packages/flutter_web_plugins/**', 'bin/**') && $CIRRUS_PR != ''" - environment: - # As of October 2019, the Web shards needed more than 6G of RAM. - CPU: 2 - MEMORY: 8G - CHROME_NO_SANDBOX: true - script: - - dart --enable-asserts ./dev/bots/test.dart - - name: docs-linux # linux-only environment: CPU: 4 diff --git a/dev/bots/service_worker_test.dart b/dev/bots/service_worker_test.dart index 6d93f30e42..c5d7295052 100644 --- a/dev/bots/service_worker_test.dart +++ b/dev/bots/service_worker_test.dart @@ -12,6 +12,7 @@ import 'package:shelf/shelf.dart'; import 'browser.dart'; import 'run_command.dart'; import 'test/common.dart'; +import 'utils.dart'; final String _bat = Platform.isWindows ? '.bat' : ''; final String _flutterRoot = path.dirname(path.dirname(path.dirname(path.fromUri(Platform.script)))); @@ -53,215 +54,231 @@ Future _rebuildApp({ @required int version }) async { ); } +/// A drop-in replacement for `package:test` expect that can run outside the +/// test zone. +void expect(Object actual, Object expected) { + final Matcher matcher = wrapMatcher(expected); + final Map matchState = {}; + if (matcher.matches(actual, matchState)) { + return; + } + final StringDescription mismatchDescription = StringDescription(); + matcher.describeMismatch(actual, mismatchDescription, matchState, true); + throw TestFailure(mismatchDescription.toString()); +} + Future runWebServiceWorkerTest({ @required bool headless, }) async { - test('flutter_service_worker.js', () async { - await _rebuildApp(version: 1); + await _rebuildApp(version: 1); - final Map requestedPathCounts = {}; - void expectRequestCounts(Map expectedCounts) { - expect(requestedPathCounts, expectedCounts); - requestedPathCounts.clear(); - } + final Map requestedPathCounts = {}; + void expectRequestCounts(Map expectedCounts) { + expect(requestedPathCounts, expectedCounts); + requestedPathCounts.clear(); + } - AppServer server; - Future waitForAppToLoad(Map waitForCounts) async { - print('Waiting for app to load $waitForCounts'); - await Future.any(>[ - () async { - while (!waitForCounts.entries.every((MapEntry entry) => (requestedPathCounts[entry.key] ?? 0) >= entry.value)) { - await Future.delayed(const Duration(milliseconds: 100)); + AppServer server; + Future waitForAppToLoad(Map waitForCounts) async { + print('Waiting for app to load $waitForCounts'); + await Future.any(>[ + () async { + while (!waitForCounts.entries.every((MapEntry entry) => (requestedPathCounts[entry.key] ?? 0) >= entry.value)) { + await Future.delayed(const Duration(milliseconds: 100)); + } + }(), + server.onChromeError.then((String error) { + throw Exception('Chrome error: $error'); + }), + ]); + } + + String reportedVersion; + + Future startAppServer({ + @required String cacheControl, + }) async { + final int serverPort = await findAvailablePort(); + final int browserDebugPort = await findAvailablePort(); + server = await AppServer.start( + headless: headless, + cacheControl: cacheControl, + // TODO(yjbanov): use a better port disambiguation strategy than trying + // to guess what ports other tests use. + appUrl: 'http://localhost:$serverPort/index.html', + serverPort: serverPort, + browserDebugPort: browserDebugPort, + appDirectory: _appBuildDirectory, + additionalRequestHandlers: [ + (Request request) { + final String requestedPath = request.url.path; + requestedPathCounts.putIfAbsent(requestedPath, () => 0); + requestedPathCounts[requestedPath] += 1; + if (requestedPath == 'CLOSE') { + reportedVersion = request.url.queryParameters['version']; + return Response.ok('OK'); } - }(), - server.onChromeError.then((String error) { - throw Exception('Chrome error: $error'); - }), - ]); - } + return Response.notFound(''); + }, + ], + ); + } - String reportedVersion; + try { + ////////////////////////////////////////////////////// + // Caching server + ////////////////////////////////////////////////////// + print('With cache: test first page load'); + await startAppServer(cacheControl: 'max-age=3600'); + await waitForAppToLoad({ + 'CLOSE': 1, + 'flutter_service_worker.js': 1, + }); - Future startAppServer({ - @required String cacheControl, - }) async { - server = await AppServer.start( - headless: headless, - cacheControl: cacheControl, - appUrl: 'http://localhost:8080/index.html', - appDirectory: _appBuildDirectory, - additionalRequestHandlers: [ - (Request request) { - final String requestedPath = request.url.path; - requestedPathCounts.putIfAbsent(requestedPath, () => 0); - requestedPathCounts[requestedPath] += 1; - if (requestedPath == 'CLOSE') { - reportedVersion = request.url.queryParameters['version']; - return Response.ok('OK'); - } - return Response.notFound(''); - }, - ], - ); - } - - try { - ////////////////////////////////////////////////////// - // Caching server - ////////////////////////////////////////////////////// - print('With cache: test first page load'); - await startAppServer(cacheControl: 'max-age=3600'); - await waitForAppToLoad({ - 'CLOSE': 1, - 'flutter_service_worker.js': 1, - }); - - expectRequestCounts({ - '': 1, - // Even though the server is caching index.html is downloaded twice, - // once by the initial page load, and once by the service worker. - // Other resources are loaded once only by the service worker. - 'index.html': 2, - 'main.dart.js': 1, - 'flutter_service_worker.js': 1, - 'assets/FontManifest.json': 1, - 'assets/NOTICES': 1, - 'assets/AssetManifest.json': 1, - 'CLOSE': 1, - // In headless mode Chrome does not load 'manifest.json' and 'favicon.ico'. - if (!headless) - ...{ - 'manifest.json': 1, - 'favicon.ico': 1, - } - }); - expect(reportedVersion, '1'); - reportedVersion = null; - - print('With cache: test page reload'); - await server.chrome.reloadPage(); - await waitForAppToLoad({ - 'CLOSE': 1, - 'flutter_service_worker.js': 1, - }); - - expectRequestCounts({ - 'flutter_service_worker.js': 1, - 'CLOSE': 1, - }); - expect(reportedVersion, '1'); - reportedVersion = null; - - print('With cache: test page reload after rebuild'); - await _rebuildApp(version: 2); - - // Since we're caching, we need to ignore cache when reloading the page. - await server.chrome.reloadPage(ignoreCache: true); - await waitForAppToLoad({ - 'CLOSE': 1, - 'flutter_service_worker.js': 2, - }); - expectRequestCounts({ - 'index.html': 2, - 'flutter_service_worker.js': 2, - '': 1, - 'main.dart.js': 1, - 'assets/NOTICES': 1, - 'assets/AssetManifest.json': 1, - 'assets/FontManifest.json': 1, - 'CLOSE': 1, - if (!headless) - 'favicon.ico': 1, - }); - - expect(reportedVersion, '2'); - reportedVersion = null; - await server.stop(); - - - ////////////////////////////////////////////////////// - // Non-caching server - ////////////////////////////////////////////////////// - print('No cache: test first page load'); - await _rebuildApp(version: 3); - await startAppServer(cacheControl: 'max-age=0'); - await waitForAppToLoad({ - 'CLOSE': 1, - 'flutter_service_worker.js': 1, - }); - - expectRequestCounts({ - '': 1, - 'index.html': 2, - // We still download some resources multiple times if the server is non-caching. - 'main.dart.js': 2, - 'assets/FontManifest.json': 2, - 'flutter_service_worker.js': 1, - 'assets/NOTICES': 1, - 'assets/AssetManifest.json': 1, - 'CLOSE': 1, - // In headless mode Chrome does not load 'manifest.json' and 'favicon.ico'. - if (!headless) - ...{ - 'manifest.json': 1, - 'favicon.ico': 1, - } - }); - - expect(reportedVersion, '3'); - reportedVersion = null; - - print('No cache: test page reload'); - await server.chrome.reloadPage(); - await waitForAppToLoad({ - 'CLOSE': 1, - 'flutter_service_worker.js': 1, - }); - - expectRequestCounts({ - 'flutter_service_worker.js': 1, - 'CLOSE': 1, - if (!headless) + expectRequestCounts({ + '': 1, + // Even though the server is caching index.html is downloaded twice, + // once by the initial page load, and once by the service worker. + // Other resources are loaded once only by the service worker. + 'index.html': 2, + 'main.dart.js': 1, + 'flutter_service_worker.js': 1, + 'assets/FontManifest.json': 1, + 'assets/NOTICES': 1, + 'assets/AssetManifest.json': 1, + 'CLOSE': 1, + // In headless mode Chrome does not load 'manifest.json' and 'favicon.ico'. + if (!headless) + ...{ 'manifest.json': 1, - }); - expect(reportedVersion, '3'); - reportedVersion = null; + 'favicon.ico': 1, + } + }); + expect(reportedVersion, '1'); + reportedVersion = null; - print('No cache: test page reload after rebuild'); - await _rebuildApp(version: 4); + print('With cache: test page reload'); + await server.chrome.reloadPage(); + await waitForAppToLoad({ + 'CLOSE': 1, + 'flutter_service_worker.js': 1, + }); - // TODO(yjbanov): when running Chrome with DevTools protocol, for some - // reason a hard refresh is still required. This works without a hard - // refresh when running Chrome manually as normal. At the time of writing - // this test I wasn't able to figure out what's wrong with the way we run - // Chrome from tests. - await server.chrome.reloadPage(ignoreCache: true); - await waitForAppToLoad({ - 'CLOSE': 1, - 'flutter_service_worker.js': 1, - }); - expectRequestCounts({ - '': 1, - 'index.html': 2, - 'flutter_service_worker.js': 2, - 'main.dart.js': 2, - 'assets/NOTICES': 1, - 'assets/AssetManifest.json': 1, - 'assets/FontManifest.json': 2, - 'CLOSE': 1, - if (!headless) - ...{ - 'manifest.json': 1, - 'favicon.ico': 1, - } - }); + expectRequestCounts({ + 'flutter_service_worker.js': 1, + 'CLOSE': 1, + }); + expect(reportedVersion, '1'); + reportedVersion = null; - expect(reportedVersion, '4'); - reportedVersion = null; - } finally { - await _setAppVersion(1); - await server?.stop(); - } - // This is a long test. The default 30 seconds is not enough. - }, timeout: const Timeout(Duration(minutes: 10))); + print('With cache: test page reload after rebuild'); + await _rebuildApp(version: 2); + + // Since we're caching, we need to ignore cache when reloading the page. + await server.chrome.reloadPage(ignoreCache: true); + await waitForAppToLoad({ + 'CLOSE': 1, + 'flutter_service_worker.js': 2, + }); + expectRequestCounts({ + 'index.html': 2, + 'flutter_service_worker.js': 2, + '': 1, + 'main.dart.js': 1, + 'assets/NOTICES': 1, + 'assets/AssetManifest.json': 1, + 'assets/FontManifest.json': 1, + 'CLOSE': 1, + if (!headless) + 'favicon.ico': 1, + }); + + expect(reportedVersion, '2'); + reportedVersion = null; + await server.stop(); + + + ////////////////////////////////////////////////////// + // Non-caching server + ////////////////////////////////////////////////////// + print('No cache: test first page load'); + await _rebuildApp(version: 3); + await startAppServer(cacheControl: 'max-age=0'); + await waitForAppToLoad({ + 'CLOSE': 1, + 'flutter_service_worker.js': 1, + }); + + expectRequestCounts({ + '': 1, + 'index.html': 2, + // We still download some resources multiple times if the server is non-caching. + 'main.dart.js': 2, + 'assets/FontManifest.json': 2, + 'flutter_service_worker.js': 1, + 'assets/NOTICES': 1, + 'assets/AssetManifest.json': 1, + 'CLOSE': 1, + // In headless mode Chrome does not load 'manifest.json' and 'favicon.ico'. + if (!headless) + ...{ + 'manifest.json': 1, + 'favicon.ico': 1, + } + }); + + expect(reportedVersion, '3'); + reportedVersion = null; + + print('No cache: test page reload'); + await server.chrome.reloadPage(); + await waitForAppToLoad({ + 'CLOSE': 1, + 'flutter_service_worker.js': 1, + }); + + expectRequestCounts({ + 'flutter_service_worker.js': 1, + 'CLOSE': 1, + if (!headless) + 'manifest.json': 1, + }); + expect(reportedVersion, '3'); + reportedVersion = null; + + print('No cache: test page reload after rebuild'); + await _rebuildApp(version: 4); + + // TODO(yjbanov): when running Chrome with DevTools protocol, for some + // reason a hard refresh is still required. This works without a hard + // refresh when running Chrome manually as normal. At the time of writing + // this test I wasn't able to figure out what's wrong with the way we run + // Chrome from tests. + await server.chrome.reloadPage(ignoreCache: true); + await waitForAppToLoad({ + 'CLOSE': 1, + 'flutter_service_worker.js': 1, + }); + expectRequestCounts({ + '': 1, + 'index.html': 2, + 'flutter_service_worker.js': 2, + 'main.dart.js': 2, + 'assets/NOTICES': 1, + 'assets/AssetManifest.json': 1, + 'assets/FontManifest.json': 2, + 'CLOSE': 1, + if (!headless) + ...{ + 'manifest.json': 1, + 'favicon.ico': 1, + } + }); + + expect(reportedVersion, '4'); + reportedVersion = null; + } finally { + await _setAppVersion(1); + await server?.stop(); + } } diff --git a/dev/bots/test.dart b/dev/bots/test.dart index 585e909850..adc4a47102 100644 --- a/dev/bots/test.dart +++ b/dev/bots/test.dart @@ -108,8 +108,9 @@ Future main(List args) async { 'tool_tests': _runToolTests, 'web_tool_tests': _runToolTests, 'tool_integration_tests': _runIntegrationToolTests, + // All the unit/widget tests run using `flutter test --platform=chrome` 'web_tests': _runWebUnitTests, - 'web_integration_tests': _runWebIntegrationTests, + // All web integration tests 'web_long_running_tests': _runWebLongRunningTests, 'flutter_plugins': _runFlutterPluginsTests, 'skp_generator': _runSkpGeneratorTests, @@ -850,6 +851,34 @@ Future _runWebLongRunningTests() async { () => _runGalleryE2eWebTest('release'), () => _runGalleryE2eWebTest('release', canvasKit: true), () => runWebServiceWorkerTest(headless: true), + () => _runWebStackTraceTest('profile', 'lib/stack_trace.dart'), + () => _runWebStackTraceTest('release', 'lib/stack_trace.dart'), + () => _runWebStackTraceTest('profile', 'lib/framework_stack_trace.dart'), + () => _runWebStackTraceTest('release', 'lib/framework_stack_trace.dart'), + () => _runWebDebugTest('lib/stack_trace.dart'), + () => _runWebDebugTest('lib/framework_stack_trace.dart'), + () => _runWebDebugTest('lib/web_directory_loading.dart'), + () => _runWebDebugTest('test/test.dart'), + () => _runWebDebugTest('lib/null_assert_main.dart', enableNullSafety: true), + () => _runWebDebugTest('lib/null_safe_main.dart', enableNullSafety: true), + () => _runWebDebugTest('lib/web_define_loading.dart', + additionalArguments: [ + '--dart-define=test.valueA=Example,A', + '--dart-define=test.valueB=Value', + ] + ), + () => _runWebReleaseTest('lib/web_define_loading.dart', + additionalArguments: [ + '--dart-define=test.valueA=Example,A', + '--dart-define=test.valueB=Value', + ] + ), + () => _runWebDebugTest('lib/sound_mode.dart', additionalArguments: [ + '--sound-null-safety', + ]), + () => _runWebReleaseTest('lib/sound_mode.dart', additionalArguments: [ + '--sound-null-safety', + ]), ]; await _ensureChromeDriverIsRunning(); await _runShardRunnerIndexOfTotalSubshard(tests); @@ -1069,48 +1098,6 @@ Future _runGalleryE2eWebTest(String buildMode, { bool canvasKit = false }) print('${green}Integration test passed.$reset'); } -/// !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! -/// !!! WARNING WARNING WARNING WARNING WARNING WARNING!!! -/// !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! -/// -/// Do not put any more tests here. This shard is not properly subsharded. -/// Adding more tests here will linearly increase the runtime of the shard -/// making the overall Flutter CI build longer. Consider adding tests to -/// [_runWebLongRunningTests] instead (increasing subshard count if necessary). -/// -// TODO(yjbanov): increase subshard count in _runWebLongRunningTests and retire -// this shard. -Future _runWebIntegrationTests() async { - await _runWebStackTraceTest('profile', 'lib/stack_trace.dart'); - await _runWebStackTraceTest('release', 'lib/stack_trace.dart'); - await _runWebStackTraceTest('profile', 'lib/framework_stack_trace.dart'); - await _runWebStackTraceTest('release', 'lib/framework_stack_trace.dart'); - await _runWebDebugTest('lib/stack_trace.dart'); - await _runWebDebugTest('lib/framework_stack_trace.dart'); - await _runWebDebugTest('lib/web_directory_loading.dart'); - await _runWebDebugTest('test/test.dart'); - await _runWebDebugTest('lib/null_assert_main.dart', enableNullSafety: true); - await _runWebDebugTest('lib/null_safe_main.dart', enableNullSafety: true); - await _runWebDebugTest('lib/web_define_loading.dart', - additionalArguments: [ - '--dart-define=test.valueA=Example,A', - '--dart-define=test.valueB=Value', - ] - ); - await _runWebReleaseTest('lib/web_define_loading.dart', - additionalArguments: [ - '--dart-define=test.valueA=Example,A', - '--dart-define=test.valueB=Value', - ] - ); - await _runWebDebugTest('lib/sound_mode.dart', additionalArguments: [ - '--sound-null-safety', - ]); - await _runWebReleaseTest('lib/sound_mode.dart', additionalArguments: [ - '--sound-null-safety', - ]); -} - Future _runWebStackTraceTest(String buildMode, String entrypoint) async { final String testAppDirectory = path.join(flutterRoot, 'dev', 'integration_tests', 'web'); final String appBuildDirectory = path.join(testAppDirectory, 'build', 'web'); @@ -1137,9 +1124,13 @@ Future _runWebStackTraceTest(String buildMode, String entrypoint) async { ); // Run the app. + final int serverPort = await findAvailablePort(); + final int browserDebugPort = await findAvailablePort(); final String result = await evalTestAppInChrome( - appUrl: 'http://localhost:8080/index.html', + appUrl: 'http://localhost:$serverPort/index.html', appDirectory: appBuildDirectory, + serverPort: serverPort, + browserDebugPort: browserDebugPort, ); if (result.contains('--- TEST SUCCEEDED ---')) { @@ -1181,9 +1172,13 @@ Future _runWebReleaseTest(String target, { ); // Run the app. + final int serverPort = await findAvailablePort(); + final int browserDebugPort = await findAvailablePort(); final String result = await evalTestAppInChrome( - appUrl: 'http://localhost:8080/index.html', + appUrl: 'http://localhost:$serverPort/index.html', appDirectory: appBuildDirectory, + serverPort: serverPort, + browserDebugPort: browserDebugPort, ); if (result.contains('--- TEST SUCCEEDED ---')) { diff --git a/dev/bots/utils.dart b/dev/bots/utils.dart index 34e105e3f3..0311fc94c7 100644 --- a/dev/bots/utils.dart +++ b/dev/bots/utils.dart @@ -70,3 +70,24 @@ String prettyPrintDuration(Duration duration) { void printProgress(String action, String workingDir, String command) { print('$clock $action: cd $cyan$workingDir$reset; $green$command$reset'); } + +int _portCounter = 8080; + +/// Finds the next available local port. +Future findAvailablePort() async { + while (!await _isPortAvailable(_portCounter)) { + _portCounter += 1; + } + return _portCounter++; +} + +Future _isPortAvailable(int port) async { + try { + final RawSocket socket = await RawSocket.connect('localhost', port); + socket.shutdown(SocketDirection.both); + await socket.close(); + return false; + } on SocketException { + return true; + } +} diff --git a/dev/prod_builders.json b/dev/prod_builders.json index 79d91f4e3f..ca62885d56 100644 --- a/dev/prod_builders.json +++ b/dev/prod_builders.json @@ -523,27 +523,33 @@ "flaky": false }, { - "name": "Linux web_integration_tests", + "name": "Linux web_long_running_tests_1_5", "repo": "flutter", - "task_name": "linux_web_integration_tests", + "task_name": "linux_web_long_running_tests_1_5", "flaky": false }, { - "name": "Linux web_long_running_tests_1_3", + "name": "Linux web_long_running_tests_2_5", "repo": "flutter", - "task_name": "linux_web_long_running_tests_1_3", + "task_name": "linux_web_long_running_tests_2_5", "flaky": false }, { - "name": "Linux web_long_running_tests_2_3", + "name": "Linux web_long_running_tests_3_5", "repo": "flutter", - "task_name": "linux_web_long_running_tests_2_3", + "task_name": "linux_web_long_running_tests_3_5", "flaky": false }, { - "name": "Linux web_long_running_tests_3_3", + "name": "Linux web_long_running_tests_4_5", "repo": "flutter", - "task_name": "linux_web_long_running_tests_3_3", + "task_name": "linux_web_long_running_tests_4_5", + "flaky": false + }, + { + "name": "Linux web_long_running_tests_5_5", + "repo": "flutter", + "task_name": "linux_web_long_running_tests_5_5", "flaky": false }, { diff --git a/dev/try_builders.json b/dev/try_builders.json index 681874e7b2..be5e4eb64a 100644 --- a/dev/try_builders.json +++ b/dev/try_builders.json @@ -305,37 +305,37 @@ "run_if": ["dev/", "packages/", "bin/"] }, { - "name": "Linux web_integration_tests", + "name": "Linux web_long_running_tests_1_5", "repo": "flutter", - "task_name": "linux_web_integration_tests", - "enabled": true, - "run_if": [ - "dev/", - "packages/flutter/", - "packages/flutter_test/", - "packages/flutter_tools/", - "packages/flutter_web_plugins/", - "bin/" - ] - }, - { - "name": "Linux web_long_running_tests_1_3", - "repo": "flutter", - "task_name": "web_long_running_tests_1_3", + "task_name": "web_long_running_tests_1_5", "enabled": true, "run_if": ["dev/", "packages/", "bin/"] }, { - "name": "Linux web_long_running_tests_2_3", + "name": "Linux web_long_running_tests_2_5", "repo": "flutter", - "task_name": "web_long_running_tests_2_3", + "task_name": "web_long_running_tests_2_5", "enabled": true, "run_if": ["dev/", "packages/", "bin/"] }, { - "name": "Linux web_long_running_tests_3_3", + "name": "Linux web_long_running_tests_3_5", "repo": "flutter", - "task_name": "web_long_running_tests_1_3", + "task_name": "web_long_running_tests_3_5", + "enabled": true, + "run_if": ["dev/", "packages/", "bin/"] + }, + { + "name": "Linux web_long_running_tests_4_5", + "repo": "flutter", + "task_name": "web_long_running_tests_4_5", + "enabled": true, + "run_if": ["dev/", "packages/", "bin/"] + }, + { + "name": "Linux web_long_running_tests_5_5", + "repo": "flutter", + "task_name": "web_long_running_tests_5_5", "enabled": true, "run_if": ["dev/", "packages/", "bin/"] },