From 2fa03438de85485d6aa5c3558d17c09a7af6fd28 Mon Sep 17 00:00:00 2001 From: Yegor Date: Thu, 29 Oct 2020 14:23:02 -0700 Subject: [PATCH] add web_long_running_tests shard containing long-running web tests (#67324) --- dev/bots/run_command.dart | 6 +- dev/bots/test.dart | 125 ++++++++++++++++++ .../test_driver/run_demos.dart | 11 +- .../test_driver/transitions_perf_e2e.dart | 2 - .../lib/src/driver/web_driver.dart | 6 +- .../lib/src/drive/web_driver_service.dart | 5 +- .../drive/web_driver_service_test.dart | 10 +- 7 files changed, 154 insertions(+), 11 deletions(-) diff --git a/dev/bots/run_command.dart b/dev/bots/run_command.dart index bc04450713..5a5aaeb35e 100644 --- a/dev/bots/run_command.dart +++ b/dev/bots/run_command.dart @@ -128,10 +128,8 @@ Future startCommand(String executable, List arguments, { .transform(const Utf8Encoder()); switch (outputMode) { case OutputMode.print: - await Future.wait(>[ - io.stdout.addStream(stdoutSource), - io.stderr.addStream(process.stderr), - ]); + stdoutSource.listen(io.stdout.add); + process.stderr.listen(io.stderr.add); break; case OutputMode.capture: savedStdout = stdoutSource.toList(); diff --git a/dev/bots/test.dart b/dev/bots/test.dart index e05c6fc335..5d9ae8efc9 100644 --- a/dev/bots/test.dart +++ b/dev/bots/test.dart @@ -75,6 +75,11 @@ int get webShardCount => Platform.environment.containsKey('WEB_SHARD_COUNT') ? int.parse(Platform.environment['WEB_SHARD_COUNT']) : 8; +/// The number of shards the long-running Web tests are split into. +/// +/// WARNING: this number must match the shard count in LUCI configs. +const int kWebLongRunningTestShardCount = 3; + /// Tests that we don't run on Web for various reasons. // // TODO(yjbanov): we're getting rid of this as part of https://github.com/flutter/flutter/projects/60 @@ -122,6 +127,7 @@ Future main(List args) async { 'tool_tests': _runToolTests, 'web_tests': _runWebUnitTests, 'web_integration_tests': _runWebIntegrationTests, + 'web_long_running_tests': _runWebLongRunningTests, }); } on ExitException catch (error) { error.apply(); @@ -813,6 +819,125 @@ Future _runWebUnitTests() async { await selectSubshard(subshards); } +/// Coarse-grained integration tests running on the Web. +/// +/// These tests are sharded into [kWebLongRunningTestShardCount] shards. +Future _runWebLongRunningTests() async { + final List tests = [ + () => _runGalleryE2eWebTest('debug'), + () => _runGalleryE2eWebTest('debug', canvasKit: true), + () => _runGalleryE2eWebTest('profile'), + () => _runGalleryE2eWebTest('profile', canvasKit: true), + () => _runGalleryE2eWebTest('release'), + () => _runGalleryE2eWebTest('release', canvasKit: true), + ].map(_withChromeDriver).toList(); + await _selectIndexedSubshard(tests, kWebLongRunningTestShardCount); +} + +// The `chromedriver` process created by this test. +// +// If an existing chromedriver is already available on port 4444, the existing +// process is reused and this variable remains null. +Command _chromeDriver; + +/// Creates a shard runner that runs the given [originalRunner] with ChromeDriver +/// enabled. +ShardRunner _withChromeDriver(ShardRunner originalRunner) { + return () async { + try { + await _ensureChromeDriverIsRunning(); + await originalRunner(); + } finally { + await _stopChromeDriver(); + } + }; +} + +Future _isChromeDriverRunning() async { + try { + (await Socket.connect('localhost', 4444)).destroy(); + return true; + } on SocketException { + return false; + } +} + +Future _ensureChromeDriverIsRunning() async { + // If we cannot connect to ChromeDriver, assume it is not running. Launch it. + if (!await _isChromeDriverRunning()) { + print('Starting chromedriver'); + // Assume chromedriver is in the PATH. + _chromeDriver = await startCommand( + 'chromedriver', + ['--port=4444'], + ); + while (!await _isChromeDriverRunning()) { + await Future.delayed(const Duration(milliseconds: 100)); + print('Waiting for chromedriver to start up.'); + } + } + + final HttpClient client = HttpClient(); + final Uri chromeDriverUrl = Uri.parse('http://localhost:4444/status'); + final HttpClientRequest request = await client.getUrl(chromeDriverUrl); + final HttpClientResponse response = await request.close(); + final Map webDriverStatus = json.decode(await response.transform(utf8.decoder).join('')) as Map; + client.close(); + final bool webDriverReady = webDriverStatus['value']['ready'] as bool; + if (!webDriverReady) { + throw Exception('WebDriver not available.'); + } +} + +Future _stopChromeDriver() async { + if (_chromeDriver == null) { + return; + } + _chromeDriver.process.kill(); + while (await _isChromeDriverRunning()) { + await Future.delayed(const Duration(milliseconds: 100)); + print('Waiting for chromedriver to stop.'); + } +} + +/// Exercises the old gallery in a browser for a long period of time, looking +/// for memory leaks and dangling pointers. +/// +/// This is not a performance test. +/// +/// If [canvasKit] is set to true, runs the test in CanvasKit mode. +/// +/// The test is written using `package:integration_test` (despite the "e2e" in +/// the name, which is there for historic reasons). +Future _runGalleryE2eWebTest(String buildMode, { bool canvasKit = false }) async { + print('${green}Running flutter_gallery integration test in --$buildMode using ${canvasKit ? 'CanvasKit' : 'HTML'} renderer.$reset'); + final String testAppDirectory = path.join(flutterRoot, 'dev', 'integration_tests', 'flutter_gallery'); + await runCommand( + flutter, + [ 'clean' ], + workingDirectory: testAppDirectory, + ); + await runCommand( + flutter, + [ + 'drive', + if (canvasKit) + '--dart-define=FLUTTER_WEB_USE_SKIA=true', + '--driver=test_driver/transitions_perf_e2e_test.dart', + '--target=test_driver/transitions_perf_e2e.dart', + '--browser-name=chrome', + '-d', + 'web-server', + '--$buildMode', + ], + workingDirectory: testAppDirectory, + environment: { + 'FLUTTER_WEB': 'true', + }, + ); + print('${green}Integration test passed.$reset'); +} + Future _runWebIntegrationTests() async { await _runWebStackTraceTest('profile', 'lib/stack_trace.dart'); await _runWebStackTraceTest('release', 'lib/stack_trace.dart'); diff --git a/dev/integration_tests/flutter_gallery/test_driver/run_demos.dart b/dev/integration_tests/flutter_gallery/test_driver/run_demos.dart index 09c29fe351..2df9fab7ec 100644 --- a/dev/integration_tests/flutter_gallery/test_driver/run_demos.dart +++ b/dev/integration_tests/flutter_gallery/test_driver/run_demos.dart @@ -5,12 +5,21 @@ import 'dart:ui'; import 'package:flutter/cupertino.dart'; +import 'package:flutter/foundation.dart'; import 'package:flutter/widgets.dart'; import 'package:flutter_test/flutter_test.dart'; import 'package:flutter_gallery/demo_lists.dart'; -const List kSkippedDemos = []; +/// The demos we don't run as part of the integraiton test. +/// +/// Demo names are formatted as 'DEMO_NAME@DEMO_CATEGORY' (see +/// `demo_lists.dart` for more examples). +final List kSkippedDemos = [ + // The CI uses Chromium, which lacks the video codecs to run this demo. + if (kIsWeb) + 'Video@Media', +]; /// Scrolls each demo menu item into view, launches it, then returns to the /// home screen twice. diff --git a/dev/integration_tests/flutter_gallery/test_driver/transitions_perf_e2e.dart b/dev/integration_tests/flutter_gallery/test_driver/transitions_perf_e2e.dart index 69e9be5c04..d1f6c4f070 100644 --- a/dev/integration_tests/flutter_gallery/test_driver/transitions_perf_e2e.dart +++ b/dev/integration_tests/flutter_gallery/test_driver/transitions_perf_e2e.dart @@ -14,8 +14,6 @@ import 'package:flutter_gallery/demo_lists.dart'; import 'run_demos.dart'; -const List kSkippedDemos = []; - // All of the gallery demos, identified as "title@category". // // These names are reported by the test app, see _handleMessages() diff --git a/packages/flutter_driver/lib/src/driver/web_driver.dart b/packages/flutter_driver/lib/src/driver/web_driver.dart index 2126b75e3e..7c5facd267 100644 --- a/packages/flutter_driver/lib/src/driver/web_driver.dart +++ b/packages/flutter_driver/lib/src/driver/web_driver.dart @@ -187,7 +187,11 @@ class WebFlutterDriver extends FlutterDriver { class FlutterWebConnection { /// Creates a FlutterWebConnection with WebDriver /// and whether the WebDriver supports timeline action. - FlutterWebConnection(this._driver, this.supportsTimelineAction); + FlutterWebConnection(this._driver, this.supportsTimelineAction) { + _driver.logs.get(async_io.LogType.browser).listen((async_io.LogEntry entry) { + print('[${entry.level}]: ${entry.message}'); + }); + } final async_io.WebDriver _driver; diff --git a/packages/flutter_tools/lib/src/drive/web_driver_service.dart b/packages/flutter_tools/lib/src/drive/web_driver_service.dart index 21a6a9b9b4..481d071554 100644 --- a/packages/flutter_tools/lib/src/drive/web_driver_service.dart +++ b/packages/flutter_tools/lib/src/drive/web_driver_service.dart @@ -186,7 +186,10 @@ Map getDesiredCapabilities(Browser browser, bool headless, [Str return { 'acceptInsecureCerts': true, 'browserName': 'chrome', - 'goog:loggingPrefs': { async_io.LogType.performance: 'ALL'}, + 'goog:loggingPrefs': { + async_io.LogType.browser: 'INFO', + async_io.LogType.performance: 'ALL', + }, 'chromeOptions': { if (chromeBinary != null) 'binary': chromeBinary, diff --git a/packages/flutter_tools/test/general.shard/drive/web_driver_service_test.dart b/packages/flutter_tools/test/general.shard/drive/web_driver_service_test.dart index 9b85ab1565..91f2c776fb 100644 --- a/packages/flutter_tools/test/general.shard/drive/web_driver_service_test.dart +++ b/packages/flutter_tools/test/general.shard/drive/web_driver_service_test.dart @@ -12,7 +12,10 @@ void main() { final Map expected = { 'acceptInsecureCerts': true, 'browserName': 'chrome', - 'goog:loggingPrefs': { sync_io.LogType.performance: 'ALL'}, + 'goog:loggingPrefs': { + sync_io.LogType.browser: 'INFO', + sync_io.LogType.performance: 'ALL', + }, 'chromeOptions': { 'w3c': false, 'args': [ @@ -44,7 +47,10 @@ void main() { final Map expected = { 'acceptInsecureCerts': true, 'browserName': 'chrome', - 'goog:loggingPrefs': { sync_io.LogType.performance: 'ALL'}, + 'goog:loggingPrefs': { + sync_io.LogType.browser: 'INFO', + sync_io.LogType.performance: 'ALL', + }, 'chromeOptions': { 'binary': chromeBinary, 'w3c': false,