firka/flutter
firka/flutter
5
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

consolidate all web integration tests under the same shard (#82307)

Browse Source 
* consolidate all web integration tests under the same shard
This commit is contained in:
Yegor 2021-05-12 15:01:11 -07:00 committed by GitHub
parent 13a72114b2
commit 0420d56335
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 311 additions and 282 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
.cirrus.yml
View File

@ -140,16 +140,6 @@ task:
- (cd dev/bots; pub get)
- dart --enable-asserts ./dev/bots/test.dart
- name: web_integration_tests
only_if: "changesInclude('.cirrus.yml', 'dev/**', 'packages/flutter/**', 'packages/flutter_test/**', 'packages/flutter_tools/**', 'packages/flutter_web_plugins/**', 'bin/**') && $CIRRUS_PR != ''"
environment:
# As of October 2019, the Web shards needed more than 6G of RAM.
CPU: 2
MEMORY: 8G
CHROME_NO_SANDBOX: true
script:
- dart --enable-asserts ./dev/bots/test.dart
- name: docs-linux # linux-only
environment:
CPU: 4

415
dev/bots/service_worker_test.dart
View File

@ -12,6 +12,7 @@ import 'package:shelf/shelf.dart';
import 'browser.dart';
import 'run_command.dart';
import 'test/common.dart';
import 'utils.dart';
final String _bat = Platform.isWindows ? '.bat' : '';
final String _flutterRoot = path.dirname(path.dirname(path.dirname(path.fromUri(Platform.script))));
@ -53,215 +54,231 @@ Future<void> _rebuildApp({ @required int version }) async {
);
}
/// A drop-in replacement for `package:test` expect that can run outside the
/// test zone.
void expect(Object actual, Object expected) {
final Matcher matcher = wrapMatcher(expected);
final Map<Object, Object> matchState = <Object, Object>{};
if (matcher.matches(actual, matchState)) {
return;
}
final StringDescription mismatchDescription = StringDescription();
matcher.describeMismatch(actual, mismatchDescription, matchState, true);
throw TestFailure(mismatchDescription.toString());
}
Future<void> runWebServiceWorkerTest({
@required bool headless,
}) async {
test('flutter_service_worker.js', () async {
await _rebuildApp(version: 1);
await _rebuildApp(version: 1);
final Map<String, int> requestedPathCounts = <String, int>{};
void expectRequestCounts(Map<String, int> expectedCounts) {
expect(requestedPathCounts, expectedCounts);
requestedPathCounts.clear();
}
final Map<String, int> requestedPathCounts = <String, int>{};
void expectRequestCounts(Map<String, int> expectedCounts) {
expect(requestedPathCounts, expectedCounts);
requestedPathCounts.clear();
}
AppServer server;
Future<void> waitForAppToLoad(Map<String, int> waitForCounts) async {
print('Waiting for app to load $waitForCounts');
await Future.any(<Future<void>>[
() async {
while (!waitForCounts.entries.every((MapEntry<String, int> entry) => (requestedPathCounts[entry.key] ?? 0) >= entry.value)) {
await Future<void>.delayed(const Duration(milliseconds: 100));
AppServer server;
Future<void> waitForAppToLoad(Map<String, int> waitForCounts) async {
print('Waiting for app to load $waitForCounts');
await Future.any(<Future<void>>[
() async {
while (!waitForCounts.entries.every((MapEntry<String, int> entry) => (requestedPathCounts[entry.key] ?? 0) >= entry.value)) {
await Future<void>.delayed(const Duration(milliseconds: 100));
}
}(),
server.onChromeError.then((String error) {
throw Exception('Chrome error: $error');
}),
]);
}
String reportedVersion;
Future<void> startAppServer({
@required String cacheControl,
}) async {
final int serverPort = await findAvailablePort();
final int browserDebugPort = await findAvailablePort();
server = await AppServer.start(
headless: headless,
cacheControl: cacheControl,
// TODO(yjbanov): use a better port disambiguation strategy than trying
// to guess what ports other tests use.
appUrl: 'http://localhost:$serverPort/index.html',
serverPort: serverPort,
browserDebugPort: browserDebugPort,
appDirectory: _appBuildDirectory,
additionalRequestHandlers: <Handler>[
(Request request) {
final String requestedPath = request.url.path;
requestedPathCounts.putIfAbsent(requestedPath, () => 0);
requestedPathCounts[requestedPath] += 1;
if (requestedPath == 'CLOSE') {
reportedVersion = request.url.queryParameters['version'];
return Response.ok('OK');
}
}(),
server.onChromeError.then((String error) {
throw Exception('Chrome error: $error');
}),
]);
}
return Response.notFound('');
},
],
);
}
String reportedVersion;
try {
//////////////////////////////////////////////////////
// Caching server
//////////////////////////////////////////////////////
print('With cache: test first page load');
await startAppServer(cacheControl: 'max-age=3600');
await waitForAppToLoad(<String, int>{
'CLOSE': 1,
'flutter_service_worker.js': 1,
});
Future<void> startAppServer({
@required String cacheControl,
}) async {
server = await AppServer.start(
headless: headless,
cacheControl: cacheControl,
appUrl: 'http://localhost:8080/index.html',
appDirectory: _appBuildDirectory,
additionalRequestHandlers: <Handler>[
(Request request) {
final String requestedPath = request.url.path;
requestedPathCounts.putIfAbsent(requestedPath, () => 0);
requestedPathCounts[requestedPath] += 1;
if (requestedPath == 'CLOSE') {
reportedVersion = request.url.queryParameters['version'];
return Response.ok('OK');
}
return Response.notFound('');
},
],
);
}
try {
//////////////////////////////////////////////////////
// Caching server
//////////////////////////////////////////////////////
print('With cache: test first page load');
await startAppServer(cacheControl: 'max-age=3600');
await waitForAppToLoad(<String, int>{
'CLOSE': 1,
'flutter_service_worker.js': 1,
});
expectRequestCounts(<String, int>{
'': 1,
// Even though the server is caching index.html is downloaded twice,
// once by the initial page load, and once by the service worker.
// Other resources are loaded once only by the service worker.
'index.html': 2,
'main.dart.js': 1,
'flutter_service_worker.js': 1,
'assets/FontManifest.json': 1,
'assets/NOTICES': 1,
'assets/AssetManifest.json': 1,
'CLOSE': 1,
// In headless mode Chrome does not load 'manifest.json' and 'favicon.ico'.
if (!headless)
...<String, int>{
'manifest.json': 1,
'favicon.ico': 1,
}
});
expect(reportedVersion, '1');
reportedVersion = null;
print('With cache: test page reload');
await server.chrome.reloadPage();
await waitForAppToLoad(<String, int>{
'CLOSE': 1,
'flutter_service_worker.js': 1,
});
expectRequestCounts(<String, int>{
'flutter_service_worker.js': 1,
'CLOSE': 1,
});
expect(reportedVersion, '1');
reportedVersion = null;
print('With cache: test page reload after rebuild');
await _rebuildApp(version: 2);
// Since we're caching, we need to ignore cache when reloading the page.
await server.chrome.reloadPage(ignoreCache: true);
await waitForAppToLoad(<String, int>{
'CLOSE': 1,
'flutter_service_worker.js': 2,
});
expectRequestCounts(<String, int>{
'index.html': 2,
'flutter_service_worker.js': 2,
'': 1,
'main.dart.js': 1,
'assets/NOTICES': 1,
'assets/AssetManifest.json': 1,
'assets/FontManifest.json': 1,
'CLOSE': 1,
if (!headless)
'favicon.ico': 1,
});
expect(reportedVersion, '2');
reportedVersion = null;
await server.stop();
//////////////////////////////////////////////////////
// Non-caching server
//////////////////////////////////////////////////////
print('No cache: test first page load');
await _rebuildApp(version: 3);
await startAppServer(cacheControl: 'max-age=0');
await waitForAppToLoad(<String, int>{
'CLOSE': 1,
'flutter_service_worker.js': 1,
});
expectRequestCounts(<String, int>{
'': 1,
'index.html': 2,
// We still download some resources multiple times if the server is non-caching.
'main.dart.js': 2,
'assets/FontManifest.json': 2,
'flutter_service_worker.js': 1,
'assets/NOTICES': 1,
'assets/AssetManifest.json': 1,
'CLOSE': 1,
// In headless mode Chrome does not load 'manifest.json' and 'favicon.ico'.
if (!headless)
...<String, int>{
'manifest.json': 1,
'favicon.ico': 1,
}
});
expect(reportedVersion, '3');
reportedVersion = null;
print('No cache: test page reload');
await server.chrome.reloadPage();
await waitForAppToLoad(<String, int>{
'CLOSE': 1,
'flutter_service_worker.js': 1,
});
expectRequestCounts(<String, int>{
'flutter_service_worker.js': 1,
'CLOSE': 1,
if (!headless)
expectRequestCounts(<String, int>{
'': 1,
// Even though the server is caching index.html is downloaded twice,
// once by the initial page load, and once by the service worker.
// Other resources are loaded once only by the service worker.
'index.html': 2,
'main.dart.js': 1,
'flutter_service_worker.js': 1,
'assets/FontManifest.json': 1,
'assets/NOTICES': 1,
'assets/AssetManifest.json': 1,
'CLOSE': 1,
// In headless mode Chrome does not load 'manifest.json' and 'favicon.ico'.
if (!headless)
...<String, int>{
'manifest.json': 1,
});
expect(reportedVersion, '3');
reportedVersion = null;
'favicon.ico': 1,
}
});
expect(reportedVersion, '1');
reportedVersion = null;
print('No cache: test page reload after rebuild');
await _rebuildApp(version: 4);
print('With cache: test page reload');
await server.chrome.reloadPage();
await waitForAppToLoad(<String, int>{
'CLOSE': 1,
'flutter_service_worker.js': 1,
});
// TODO(yjbanov): when running Chrome with DevTools protocol, for some
// reason a hard refresh is still required. This works without a hard
// refresh when running Chrome manually as normal. At the time of writing
// this test I wasn't able to figure out what's wrong with the way we run
// Chrome from tests.
await server.chrome.reloadPage(ignoreCache: true);
await waitForAppToLoad(<String, int>{
'CLOSE': 1,
'flutter_service_worker.js': 1,
});
expectRequestCounts(<String, int>{
'': 1,
'index.html': 2,
'flutter_service_worker.js': 2,
'main.dart.js': 2,
'assets/NOTICES': 1,
'assets/AssetManifest.json': 1,
'assets/FontManifest.json': 2,
'CLOSE': 1,
if (!headless)
...<String, int>{
'manifest.json': 1,
'favicon.ico': 1,
}
});
expectRequestCounts(<String, int>{
'flutter_service_worker.js': 1,
'CLOSE': 1,
});
expect(reportedVersion, '1');
reportedVersion = null;
expect(reportedVersion, '4');
reportedVersion = null;
} finally {
await _setAppVersion(1);
await server?.stop();
}
// This is a long test. The default 30 seconds is not enough.
}, timeout: const Timeout(Duration(minutes: 10)));
print('With cache: test page reload after rebuild');
await _rebuildApp(version: 2);
// Since we're caching, we need to ignore cache when reloading the page.
await server.chrome.reloadPage(ignoreCache: true);
await waitForAppToLoad(<String, int>{
'CLOSE': 1,
'flutter_service_worker.js': 2,
});
expectRequestCounts(<String, int>{
'index.html': 2,
'flutter_service_worker.js': 2,
'': 1,
'main.dart.js': 1,
'assets/NOTICES': 1,
'assets/AssetManifest.json': 1,
'assets/FontManifest.json': 1,
'CLOSE': 1,
if (!headless)
'favicon.ico': 1,
});
expect(reportedVersion, '2');
reportedVersion = null;
await server.stop();
//////////////////////////////////////////////////////
// Non-caching server
//////////////////////////////////////////////////////
print('No cache: test first page load');
await _rebuildApp(version: 3);
await startAppServer(cacheControl: 'max-age=0');
await waitForAppToLoad(<String, int>{
'CLOSE': 1,
'flutter_service_worker.js': 1,
});
expectRequestCounts(<String, int>{
'': 1,
'index.html': 2,
// We still download some resources multiple times if the server is non-caching.
'main.dart.js': 2,
'assets/FontManifest.json': 2,
'flutter_service_worker.js': 1,
'assets/NOTICES': 1,
'assets/AssetManifest.json': 1,
'CLOSE': 1,
// In headless mode Chrome does not load 'manifest.json' and 'favicon.ico'.
if (!headless)
...<String, int>{
'manifest.json': 1,
'favicon.ico': 1,
}
});
expect(reportedVersion, '3');
reportedVersion = null;
print('No cache: test page reload');
await server.chrome.reloadPage();
await waitForAppToLoad(<String, int>{
'CLOSE': 1,
'flutter_service_worker.js': 1,
});
expectRequestCounts(<String, int>{
'flutter_service_worker.js': 1,
'CLOSE': 1,
if (!headless)
'manifest.json': 1,
});
expect(reportedVersion, '3');
reportedVersion = null;
print('No cache: test page reload after rebuild');
await _rebuildApp(version: 4);
// TODO(yjbanov): when running Chrome with DevTools protocol, for some
// reason a hard refresh is still required. This works without a hard
// refresh when running Chrome manually as normal. At the time of writing
// this test I wasn't able to figure out what's wrong with the way we run
// Chrome from tests.
await server.chrome.reloadPage(ignoreCache: true);
await waitForAppToLoad(<String, int>{
'CLOSE': 1,
'flutter_service_worker.js': 1,
});
expectRequestCounts(<String, int>{
'': 1,
'index.html': 2,
'flutter_service_worker.js': 2,
'main.dart.js': 2,
'assets/NOTICES': 1,
'assets/AssetManifest.json': 1,
'assets/FontManifest.json': 2,
'CLOSE': 1,
if (!headless)
...<String, int>{
'manifest.json': 1,
'favicon.ico': 1,
}
});
expect(reportedVersion, '4');
reportedVersion = null;
} finally {
await _setAppVersion(1);
await server?.stop();
}
}

85
dev/bots/test.dart
View File

@ -108,8 +108,9 @@ Future<void> main(List<String> args) async {
'tool_tests': _runToolTests,
'web_tool_tests': _runToolTests,
'tool_integration_tests': _runIntegrationToolTests,
// All the unit/widget tests run using `flutter test --platform=chrome`
'web_tests': _runWebUnitTests,
'web_integration_tests': _runWebIntegrationTests,
// All web integration tests
'web_long_running_tests': _runWebLongRunningTests,
'flutter_plugins': _runFlutterPluginsTests,
'skp_generator': _runSkpGeneratorTests,
@ -850,6 +851,34 @@ Future<void> _runWebLongRunningTests() async {
() => _runGalleryE2eWebTest('release'),
() => _runGalleryE2eWebTest('release', canvasKit: true),
() => runWebServiceWorkerTest(headless: true),
() => _runWebStackTraceTest('profile', 'lib/stack_trace.dart'),
() => _runWebStackTraceTest('release', 'lib/stack_trace.dart'),
() => _runWebStackTraceTest('profile', 'lib/framework_stack_trace.dart'),
() => _runWebStackTraceTest('release', 'lib/framework_stack_trace.dart'),
() => _runWebDebugTest('lib/stack_trace.dart'),
() => _runWebDebugTest('lib/framework_stack_trace.dart'),
() => _runWebDebugTest('lib/web_directory_loading.dart'),
() => _runWebDebugTest('test/test.dart'),
() => _runWebDebugTest('lib/null_assert_main.dart', enableNullSafety: true),
() => _runWebDebugTest('lib/null_safe_main.dart', enableNullSafety: true),
() => _runWebDebugTest('lib/web_define_loading.dart',
additionalArguments: <String>[
'--dart-define=test.valueA=Example,A',
'--dart-define=test.valueB=Value',
]
),
() => _runWebReleaseTest('lib/web_define_loading.dart',
additionalArguments: <String>[
'--dart-define=test.valueA=Example,A',
'--dart-define=test.valueB=Value',
]
),
() => _runWebDebugTest('lib/sound_mode.dart', additionalArguments: <String>[
'--sound-null-safety',
]),
() => _runWebReleaseTest('lib/sound_mode.dart', additionalArguments: <String>[
'--sound-null-safety',
]),
];
await _ensureChromeDriverIsRunning();
await _runShardRunnerIndexOfTotalSubshard(tests);
@ -1069,48 +1098,6 @@ Future<void> _runGalleryE2eWebTest(String buildMode, { bool canvasKit = false })
print('${green}Integration test passed.$reset');
}
/// !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
/// !!! WARNING WARNING WARNING WARNING WARNING WARNING!!!
/// !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
///
/// Do not put any more tests here. This shard is not properly subsharded.
/// Adding more tests here will linearly increase the runtime of the shard
/// making the overall Flutter CI build longer. Consider adding tests to
/// [_runWebLongRunningTests] instead (increasing subshard count if necessary).
///
// TODO(yjbanov): increase subshard count in _runWebLongRunningTests and retire
// this shard.
Future<void> _runWebIntegrationTests() async {
await _runWebStackTraceTest('profile', 'lib/stack_trace.dart');
await _runWebStackTraceTest('release', 'lib/stack_trace.dart');
await _runWebStackTraceTest('profile', 'lib/framework_stack_trace.dart');
await _runWebStackTraceTest('release', 'lib/framework_stack_trace.dart');
await _runWebDebugTest('lib/stack_trace.dart');
await _runWebDebugTest('lib/framework_stack_trace.dart');
await _runWebDebugTest('lib/web_directory_loading.dart');
await _runWebDebugTest('test/test.dart');
await _runWebDebugTest('lib/null_assert_main.dart', enableNullSafety: true);
await _runWebDebugTest('lib/null_safe_main.dart', enableNullSafety: true);
await _runWebDebugTest('lib/web_define_loading.dart',
additionalArguments: <String>[
'--dart-define=test.valueA=Example,A',
'--dart-define=test.valueB=Value',
]
);
await _runWebReleaseTest('lib/web_define_loading.dart',
additionalArguments: <String>[
'--dart-define=test.valueA=Example,A',
'--dart-define=test.valueB=Value',
]
);
await _runWebDebugTest('lib/sound_mode.dart', additionalArguments: <String>[
'--sound-null-safety',
]);
await _runWebReleaseTest('lib/sound_mode.dart', additionalArguments: <String>[
'--sound-null-safety',
]);
}
Future<void> _runWebStackTraceTest(String buildMode, String entrypoint) async {
final String testAppDirectory = path.join(flutterRoot, 'dev', 'integration_tests', 'web');
final String appBuildDirectory = path.join(testAppDirectory, 'build', 'web');
@ -1137,9 +1124,13 @@ Future<void> _runWebStackTraceTest(String buildMode, String entrypoint) async {
);
// Run the app.
final int serverPort = await findAvailablePort();
final int browserDebugPort = await findAvailablePort();
final String result = await evalTestAppInChrome(
appUrl: 'http://localhost:8080/index.html',
appUrl: 'http://localhost:$serverPort/index.html',
appDirectory: appBuildDirectory,
serverPort: serverPort,
browserDebugPort: browserDebugPort,
);
if (result.contains('--- TEST SUCCEEDED ---')) {
@ -1181,9 +1172,13 @@ Future<void> _runWebReleaseTest(String target, {
);
// Run the app.
final int serverPort = await findAvailablePort();
final int browserDebugPort = await findAvailablePort();
final String result = await evalTestAppInChrome(
appUrl: 'http://localhost:8080/index.html',
appUrl: 'http://localhost:$serverPort/index.html',
appDirectory: appBuildDirectory,
serverPort: serverPort,
browserDebugPort: browserDebugPort,
);
if (result.contains('--- TEST SUCCEEDED ---')) {

21
dev/bots/utils.dart
View File

@ -70,3 +70,24 @@ String prettyPrintDuration(Duration duration) {
void printProgress(String action, String workingDir, String command) {
print('$clock $action: cd $cyan$workingDir$reset; $green$command$reset');
}
int _portCounter = 8080;
/// Finds the next available local port.
Future<int> findAvailablePort() async {
while (!await _isPortAvailable(_portCounter)) {
_portCounter += 1;
}
return _portCounter++;
}
Future<bool> _isPortAvailable(int port) async {
try {
final RawSocket socket = await RawSocket.connect('localhost', port);
socket.shutdown(SocketDirection.both);
await socket.close();
return false;
} on SocketException {
return true;
}
}

22
dev/prod_builders.json
View File

@ -523,27 +523,33 @@
"flaky": false
},
{
"name": "Linux web_integration_tests",
"name": "Linux web_long_running_tests_1_5",
"repo": "flutter",
"task_name": "linux_web_integration_tests",
"task_name": "linux_web_long_running_tests_1_5",
"flaky": false
},
{
"name": "Linux web_long_running_tests_1_3",
"name": "Linux web_long_running_tests_2_5",
"repo": "flutter",
"task_name": "linux_web_long_running_tests_1_3",
"task_name": "linux_web_long_running_tests_2_5",
"flaky": false
},
{
"name": "Linux web_long_running_tests_2_3",
"name": "Linux web_long_running_tests_3_5",
"repo": "flutter",
"task_name": "linux_web_long_running_tests_2_3",
"task_name": "linux_web_long_running_tests_3_5",
"flaky": false
},
{
"name": "Linux web_long_running_tests_3_3",
"name": "Linux web_long_running_tests_4_5",
"repo": "flutter",
"task_name": "linux_web_long_running_tests_3_3",
"task_name": "linux_web_long_running_tests_4_5",
"flaky": false
},
{
"name": "Linux web_long_running_tests_5_5",
"repo": "flutter",
"task_name": "linux_web_long_running_tests_5_5",
"flaky": false
},
{

40
dev/try_builders.json
View File

@ -305,37 +305,37 @@
"run_if": ["dev/", "packages/", "bin/"]
},
{
"name": "Linux web_integration_tests",
"name": "Linux web_long_running_tests_1_5",
"repo": "flutter",
"task_name": "linux_web_integration_tests",
"enabled": true,
"run_if": [
"dev/",
"packages/flutter/",
"packages/flutter_test/",
"packages/flutter_tools/",
"packages/flutter_web_plugins/",
"bin/"
]
},
{
"name": "Linux web_long_running_tests_1_3",
"repo": "flutter",
"task_name": "web_long_running_tests_1_3",
"task_name": "web_long_running_tests_1_5",
"enabled": true,
"run_if": ["dev/", "packages/", "bin/"]
},
{
"name": "Linux web_long_running_tests_2_3",
"name": "Linux web_long_running_tests_2_5",
"repo": "flutter",
"task_name": "web_long_running_tests_2_3",
"task_name": "web_long_running_tests_2_5",
"enabled": true,
"run_if": ["dev/", "packages/", "bin/"]
},
{
"name": "Linux web_long_running_tests_3_3",
"name": "Linux web_long_running_tests_3_5",
"repo": "flutter",
"task_name": "web_long_running_tests_1_3",
"task_name": "web_long_running_tests_3_5",
"enabled": true,
"run_if": ["dev/", "packages/", "bin/"]
},
{
"name": "Linux web_long_running_tests_4_5",
"repo": "flutter",
"task_name": "web_long_running_tests_4_5",
"enabled": true,
"run_if": ["dev/", "packages/", "bin/"]
},
{
"name": "Linux web_long_running_tests_5_5",
"repo": "flutter",
"task_name": "web_long_running_tests_5_5",
"enabled": true,
"run_if": ["dev/", "packages/", "bin/"]
},