diff --git a/.cirrus.yml b/.cirrus.yml index ba8185ce28..3cd9ad8752 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -25,7 +25,7 @@ task: setup_script: ./dev/bots/cirrus_setup.sh matrix: - name: docs - skip: "!changesInclude('packages/flutter/**', 'packages/flutter_test/**', 'packages/flutter_drive/**', 'packages/flutter_localizations/**', 'packages/flutter_goldens/**', 'bin/internal/**') && $CIRRUS_BRANCH != 'master'" + skip: "!changesInclude('dev/**', 'packages/flutter/**', 'packages/flutter_test/**', 'packages/flutter_drive/**', 'packages/flutter_localizations/**', 'packages/flutter_goldens/**', 'bin/internal/**') && $CIRRUS_BRANCH != 'master'" env: SHARD: docs # For uploading master docs to Firebase master branch staging site @@ -67,7 +67,7 @@ task: cpu: 4 memory: 12G - name: tests_widgets-linux - skip: "!changesInclude('packages/flutter/**', 'packages/flutter_test/**', 'packages/flutter_tools/lib/src/test/**', 'bin/internal/**') && $CIRRUS_BRANCH != 'master' && $CIRRUS_BRANCH != 'stable' && $CIRRUS_BRANCH != 'beta' && $CIRRUS_BRANCH != 'dev'" + skip: "!changesInclude('dev/**', 'packages/flutter/**', 'packages/flutter_test/**', 'packages/flutter_tools/lib/src/test/**', 'bin/internal/**') && $CIRRUS_BRANCH != 'master' && $CIRRUS_BRANCH != 'stable' && $CIRRUS_BRANCH != 'beta' && $CIRRUS_BRANCH != 'dev'" env: GCLOUD_SERVICE_ACCOUNT_KEY: ENCRYPTED[f12abe60f5045d619ef4c79b83dd1e0722a0b0b13dbea95fbe334e2db7fffbcd841a5a92da8824848b539a19afe0c9fb] SHARD: tests @@ -81,7 +81,7 @@ task: cpu: 4 memory: 12G - name: tests_framework_other-linux - skip: "!changesInclude('packages/flutter/**', 'packages/flutter_test/**', 'packages/flutter_tools/lib/src/test/**', 'bin/internal/**') && $CIRRUS_BRANCH != 'master' && $CIRRUS_BRANCH != 'stable' && $CIRRUS_BRANCH != 'beta' && $CIRRUS_BRANCH != 'dev'" + skip: "!changesInclude('dev/**', 'packages/flutter/**', 'packages/flutter_test/**', 'packages/flutter_tools/lib/src/test/**', 'bin/internal/**') && $CIRRUS_BRANCH != 'master' && $CIRRUS_BRANCH != 'stable' && $CIRRUS_BRANCH != 'beta' && $CIRRUS_BRANCH != 'dev'" env: GCLOUD_SERVICE_ACCOUNT_KEY: ENCRYPTED[f12abe60f5045d619ef4c79b83dd1e0722a0b0b13dbea95fbe334e2db7fffbcd841a5a92da8824848b539a19afe0c9fb] SHARD: tests @@ -95,7 +95,7 @@ task: cpu: 4 memory: 12G - name: tests_extras-linux - skip: "!changesInclude('packages/flutter/**', 'packages/flutter_test/**', 'packages/flutter_tools/lib/src/test/**', 'bin/internal/**', 'dev/**') && $CIRRUS_BRANCH != 'master' && $CIRRUS_BRANCH != 'stable' && $CIRRUS_BRANCH != 'beta' && $CIRRUS_BRANCH != 'dev'" + skip: "!changesInclude('dev/**', 'packages/flutter/**', 'packages/flutter_test/**', 'packages/flutter_tools/lib/src/test/**', 'bin/internal/**', 'dev/**') && $CIRRUS_BRANCH != 'master' && $CIRRUS_BRANCH != 'stable' && $CIRRUS_BRANCH != 'beta' && $CIRRUS_BRANCH != 'dev'" env: GCLOUD_SERVICE_ACCOUNT_KEY: ENCRYPTED[f12abe60f5045d619ef4c79b83dd1e0722a0b0b13dbea95fbe334e2db7fffbcd841a5a92da8824848b539a19afe0c9fb] SHARD: tests @@ -106,23 +106,23 @@ task: cpu: 4 memory: 12G # all of the tests except the ones in test/integration and test/commands/create_test for packages/flutter_tools - - name: tool_tests-linux - skip: "!changesInclude('packages/flutter_tools/**', 'bin/internal/**') && $CIRRUS_BRANCH != 'master' && $CIRRUS_BRANCH != 'stable' && $CIRRUS_BRANCH != 'beta' && $CIRRUS_BRANCH != 'dev'" + - name: tool_tests_general-linux + skip: "!changesInclude('dev/**', 'packages/flutter_tools/**', 'bin/internal/**') && $CIRRUS_BRANCH != 'master' && $CIRRUS_BRANCH != 'stable' && $CIRRUS_BRANCH != 'beta' && $CIRRUS_BRANCH != 'dev'" env: GCLOUD_SERVICE_ACCOUNT_KEY: ENCRYPTED[f12abe60f5045d619ef4c79b83dd1e0722a0b0b13dbea95fbe334e2db7fffbcd841a5a92da8824848b539a19afe0c9fb] SHARD: tool_tests - SUBSHARD: tool + SUBSHARD: general test_script: - dart --enable-asserts ./dev/bots/test.dart container: cpu: 4 memory: 12G - - name: tool_tests_create-linux - skip: "!changesInclude('packages/flutter_tools/**', 'bin/internal/**') && $CIRRUS_BRANCH != 'master' && $CIRRUS_BRANCH != 'stable' && $CIRRUS_BRANCH != 'beta' && $CIRRUS_BRANCH != 'dev'" + - name: tool_tests_commands-linux + skip: "!changesInclude('dev/**', 'packages/flutter_tools/**', 'bin/internal/**') && $CIRRUS_BRANCH != 'master' && $CIRRUS_BRANCH != 'stable' && $CIRRUS_BRANCH != 'beta' && $CIRRUS_BRANCH != 'dev'" env: GCLOUD_SERVICE_ACCOUNT_KEY: ENCRYPTED[f12abe60f5045d619ef4c79b83dd1e0722a0b0b13dbea95fbe334e2db7fffbcd841a5a92da8824848b539a19afe0c9fb] SHARD: tool_tests - SUBSHARD: create + SUBSHARD: commands test_script: - dart --enable-asserts ./dev/bots/test.dart container: @@ -130,7 +130,7 @@ task: memory: 12G # all of the tests in test/integration for packages/flutter_tools - name: tool_tests_integration-linux - skip: "!changesInclude('packages/flutter_tools/**', 'bin/internal/**') && $CIRRUS_BRANCH != 'master' && $CIRRUS_BRANCH != 'stable' && $CIRRUS_BRANCH != 'beta' && $CIRRUS_BRANCH != 'dev'" + skip: "!changesInclude('dev/**', 'packages/flutter_tools/**', 'bin/internal/**') && $CIRRUS_BRANCH != 'master' && $CIRRUS_BRANCH != 'stable' && $CIRRUS_BRANCH != 'beta' && $CIRRUS_BRANCH != 'dev'" env: GCLOUD_SERVICE_ACCOUNT_KEY: ENCRYPTED[f12abe60f5045d619ef4c79b83dd1e0722a0b0b13dbea95fbe334e2db7fffbcd841a5a92da8824848b539a19afe0c9fb] SHARD: tool_tests @@ -141,7 +141,7 @@ task: cpu: 4 memory: 12G - name: tool_coverage-linux - skip: "!changesInclude('packages/flutter_tools/**/*.dart') && $CIRRUS_BRANCH != 'master' && $CIRRUS_BRANCH != 'stable' && $CIRRUS_BRANCH != 'beta' && $CIRRUS_BRANCH != 'dev'" + skip: "!changesInclude('dev/**', 'packages/flutter_tools/**/*.dart') && $CIRRUS_BRANCH != 'master' && $CIRRUS_BRANCH != 'stable' && $CIRRUS_BRANCH != 'beta' && $CIRRUS_BRANCH != 'dev'" env: GCLOUD_SERVICE_ACCOUNT_KEY: ENCRYPTED[f12abe60f5045d619ef4c79b83dd1e0722a0b0b13dbea95fbe334e2db7fffbcd841a5a92da8824848b539a19afe0c9fb] CODECOV_TOKEN: ENCRYPTED[7c76a7f8c9264f3b7f3fd63fcf186f93c62c4dfe43ec288861c2f506d456681032b89efe7b7a139c82156350ca2c752c] @@ -276,23 +276,22 @@ task: - dart --enable-asserts dev\bots\test.dart matrix: # all of the tests except test/integration and test/commands/create_test for packages/flutter_tools - - name: tool_tests-windows - skip: "!changesInclude('packages/flutter_tools/**', 'bin/internal/**') && $CIRRUS_BRANCH != 'master' && $CIRRUS_BRANCH != 'stable' && $CIRRUS_BRANCH != 'beta' && $CIRRUS_BRANCH != 'dev'" + - name: tool_tests_general-windows + skip: "!changesInclude('dev/**', 'packages/flutter_tools/**', 'bin/internal/**') && $CIRRUS_BRANCH != 'master' && $CIRRUS_BRANCH != 'stable' && $CIRRUS_BRANCH != 'beta' && $CIRRUS_BRANCH != 'dev'" env: GCLOUD_SERVICE_ACCOUNT_KEY: ENCRYPTED[f12abe60f5045d619ef4c79b83dd1e0722a0b0b13dbea95fbe334e2db7fffbcd841a5a92da8824848b539a19afe0c9fb] SHARD: tool_tests - SUBSHARD: tool - SHARD_INDEX: 1 + SUBSHARD: general # all of the tests in test/commands/create_test - - name: tool_tests_create-windows - skip: "!changesInclude('packages/flutter_tools/**', 'bin/internal/**') && $CIRRUS_BRANCH != 'master' && $CIRRUS_BRANCH != 'stable' && $CIRRUS_BRANCH != 'beta' && $CIRRUS_BRANCH != 'dev'" + - name: tool_tests_commands-windows + skip: "!changesInclude('dev/**', 'packages/flutter_tools/**', 'bin/internal/**') && $CIRRUS_BRANCH != 'master' && $CIRRUS_BRANCH != 'stable' && $CIRRUS_BRANCH != 'beta' && $CIRRUS_BRANCH != 'dev'" env: GCLOUD_SERVICE_ACCOUNT_KEY: ENCRYPTED[f12abe60f5045d619ef4c79b83dd1e0722a0b0b13dbea95fbe334e2db7fffbcd841a5a92da8824848b539a19afe0c9fb] SHARD: tool_tests - SUBSHARD: create + SUBSHARD: commands # all of the tests in test/integration for packages/flutter_tools - name: tool_tests_integration-windows - skip: "!changesInclude('packages/flutter_tools/**', 'bin/internal/**') && $CIRRUS_BRANCH != 'master' && $CIRRUS_BRANCH != 'stable' && $CIRRUS_BRANCH != 'beta' && $CIRRUS_BRANCH != 'dev'" + skip: "!changesInclude('dev/**', 'packages/flutter_tools/**', 'bin/internal/**') && $CIRRUS_BRANCH != 'master' && $CIRRUS_BRANCH != 'stable' && $CIRRUS_BRANCH != 'beta' && $CIRRUS_BRANCH != 'dev'" env: GCLOUD_SERVICE_ACCOUNT_KEY: ENCRYPTED[f12abe60f5045d619ef4c79b83dd1e0722a0b0b13dbea95fbe334e2db7fffbcd841a5a92da8824848b539a19afe0c9fb] SHARD: tool_tests @@ -333,7 +332,7 @@ task: - cd dev\bots - pub run test - name: tests_widgets-windows - skip: "!changesInclude('packages/flutter/**', 'packages/flutter_test/**', 'packages/flutter_tools/lib/src/test/**', 'bin/internal/**') && $CIRRUS_BRANCH != 'master' && $CIRRUS_BRANCH != 'stable' && $CIRRUS_BRANCH != 'beta' && $CIRRUS_BRANCH != 'dev'" + skip: "!changesInclude('dev/**', 'packages/flutter/**', 'packages/flutter_test/**', 'packages/flutter_tools/lib/src/test/**', 'bin/internal/**') && $CIRRUS_BRANCH != 'master' && $CIRRUS_BRANCH != 'stable' && $CIRRUS_BRANCH != 'beta' && $CIRRUS_BRANCH != 'dev'" env: GCLOUD_SERVICE_ACCOUNT_KEY: ENCRYPTED[f12abe60f5045d619ef4c79b83dd1e0722a0b0b13dbea95fbe334e2db7fffbcd841a5a92da8824848b539a19afe0c9fb] SHARD: tests @@ -344,7 +343,7 @@ task: test_all_script: - dart --enable-asserts dev\bots\test.dart - name: tests_framework_other-windows - skip: "!changesInclude('packages/flutter/**', 'packages/flutter_test/**', 'packages/flutter_tools/lib/src/test/**', 'bin/internal/**') && $CIRRUS_BRANCH != 'master' && $CIRRUS_BRANCH != 'stable' && $CIRRUS_BRANCH != 'beta' && $CIRRUS_BRANCH != 'dev'" + skip: "!changesInclude('dev/**', 'packages/flutter/**', 'packages/flutter_test/**', 'packages/flutter_tools/lib/src/test/**', 'bin/internal/**') && $CIRRUS_BRANCH != 'master' && $CIRRUS_BRANCH != 'stable' && $CIRRUS_BRANCH != 'beta' && $CIRRUS_BRANCH != 'dev'" env: GCLOUD_SERVICE_ACCOUNT_KEY: ENCRYPTED[f12abe60f5045d619ef4c79b83dd1e0722a0b0b13dbea95fbe334e2db7fffbcd841a5a92da8824848b539a19afe0c9fb] SHARD: tests @@ -355,7 +354,7 @@ task: test_all_script: - dart --enable-asserts dev\bots\test.dart - name: tests_extras-windows - skip: "!changesInclude('packages/flutter/**', 'packages/flutter_test/**', 'packages/flutter_tools/lib/src/test/**', 'bin/internal/**', 'dev/**') && $CIRRUS_BRANCH != 'master' && $CIRRUS_BRANCH != 'stable' && $CIRRUS_BRANCH != 'beta' && $CIRRUS_BRANCH != 'dev'" + skip: "!changesInclude('dev/**', 'packages/flutter/**', 'packages/flutter_test/**', 'packages/flutter_tools/lib/src/test/**', 'bin/internal/**', 'dev/**') && $CIRRUS_BRANCH != 'master' && $CIRRUS_BRANCH != 'stable' && $CIRRUS_BRANCH != 'beta' && $CIRRUS_BRANCH != 'dev'" env: GCLOUD_SERVICE_ACCOUNT_KEY: ENCRYPTED[f12abe60f5045d619ef4c79b83dd1e0722a0b0b13dbea95fbe334e2db7fffbcd841a5a92da8824848b539a19afe0c9fb] SHARD: tests @@ -461,20 +460,19 @@ task: - bin/cache/dart-sdk/bin/dart --enable-asserts dev/bots/test.dart matrix: # all of the tests except test/integration and test/commands/create_test for packages/flutter_tools - - name: tool_tests-macos - skip: "!changesInclude('packages/flutter_tools/**', 'bin/internal/**') && $CIRRUS_BRANCH != 'master' && $CIRRUS_BRANCH != 'stable' && $CIRRUS_BRANCH != 'beta' && $CIRRUS_BRANCH != 'dev'" + - name: tool_tests_general-macos + skip: "!changesInclude('dev/**', 'packages/flutter_tools/**', 'bin/internal/**') && $CIRRUS_BRANCH != 'master' && $CIRRUS_BRANCH != 'stable' && $CIRRUS_BRANCH != 'beta' && $CIRRUS_BRANCH != 'dev'" env: GCLOUD_SERVICE_ACCOUNT_KEY: ENCRYPTED[f12abe60f5045d619ef4c79b83dd1e0722a0b0b13dbea95fbe334e2db7fffbcd841a5a92da8824848b539a19afe0c9fb] SHARD: tool_tests - SUBSHARD: tool - SHARD_INDEX: 1 + SUBSHARD: general # all of the tests in test/commands/create_test - - name: tool_tests_create-macos - skip: "!changesInclude('packages/flutter_tools/**', 'bin/internal/**') && $CIRRUS_BRANCH != 'master' && $CIRRUS_BRANCH != 'stable' && $CIRRUS_BRANCH != 'beta' && $CIRRUS_BRANCH != 'dev'" + - name: tool_tests_commands-macos + skip: "!changesInclude('dev/**', 'packages/flutter_tools/**', 'bin/internal/**') && $CIRRUS_BRANCH != 'master' && $CIRRUS_BRANCH != 'stable' && $CIRRUS_BRANCH != 'beta' && $CIRRUS_BRANCH != 'dev'" env: GCLOUD_SERVICE_ACCOUNT_KEY: ENCRYPTED[f12abe60f5045d619ef4c79b83dd1e0722a0b0b13dbea95fbe334e2db7fffbcd841a5a92da8824848b539a19afe0c9fb] SHARD: tool_tests - SUBSHARD: create + SUBSHARD: commands # all of the tests in test/integration for packages/flutter_tools - name: tool_tests_integration-macos only_if: $CIRRUS_BRANCH == 'master' @@ -573,8 +571,8 @@ docker_builder: - tests_widgets-linux - tests_framework_other-linux - tests_extras-linux - - tool_tests-linux - - tool_tests_create-linux + - tool_tests_general-linux + - tool_tests_commands-linux - tool_tests_integration-linux - build_tests-linux - integration_tests-linux diff --git a/bin/flutter.bat b/bin/flutter.bat index b53961c2b5..66a6fa26ca 100644 --- a/bin/flutter.bat +++ b/bin/flutter.bat @@ -60,7 +60,7 @@ IF NOT EXIST "%cache_dir%" ( REM To debug the tool, you can uncomment the following lines to enable checked mode and set an observatory port: -REM SET FLUTTER_TOOL_ARGS="--checked %FLUTTER_TOOL_ARGS%" +REM SET FLUTTER_TOOL_ARGS="--enable-asserts %FLUTTER_TOOL_ARGS%" REM SET FLUTTER_TOOL_ARGS="%FLUTTER_TOOL_ARGS% --observe=65432" :acquire_lock diff --git a/dev/bots/test.dart b/dev/bots/test.dart index 2beb892369..9375e80ea1 100644 --- a/dev/bots/test.dart +++ b/dev/bots/test.dart @@ -33,8 +33,7 @@ final String toolRoot = path.join(flutterRoot, 'packages', 'flutter_tools'); final List flutterTestArgs = []; final bool useFlutterTestFormatter = Platform.environment['FLUTTER_TEST_FORMATTER'] == 'true'; - -final bool noUseBuildRunner = Platform.environment['FLUTTER_TEST_NO_BUILD_RUNNER'] == 'true'; +final bool canUseBuildRunner = Platform.environment['FLUTTER_TEST_NO_BUILD_RUNNER'] != 'true'; const Map _kShards = { 'tests': _runTests, @@ -186,84 +185,51 @@ Future _getBigqueryApi() async { } } -// Partition tool tests into two groups, see explanation on `_runToolCoverage`. -List> _partitionToolTests() { - final String toolTestDir = path.join(toolRoot, 'test'); - final List pending = [ - for (FileSystemEntity entity in Directory(toolTestDir).listSync(recursive: true)) - if (entity is File && entity.path.endsWith('_test.dart')) - path.relative(entity.path, from: toolRoot), - ]; - - // Shuffle the tests to avoid giving an expensive test directory like - // integration to a single run of tests. - pending.shuffle(); - final int aboutHalf = pending.length ~/ 2; - final List groupA = pending.take(aboutHalf).toList(); - final List groupB = pending.skip(aboutHalf).toList(); - return >[groupA, groupB]; -} - -// Tools tests run with coverage enabled have much higher memory usage than -// our current CI infrastructure can support. We partition the tests into -// two sets and run them in separate invocations of dart to reduce peak memory -// usage. codecov.io automatically handles merging different coverage files -// together, so producing separate files is OK. -// -// See: https://github.com/flutter/flutter/issues/35025 Future _runToolCoverage() async { - final List> tests = _partitionToolTests(); - // Precompile tests to speed up subsequent runs. - await runCommand( + await runCommand( // Precompile tests to speed up subsequent runs. pub, ['run', 'build_runner', 'build'], workingDirectory: toolRoot, ); - - // The name of this subshard has to match the --file path provided at - // the end of this test script in `.cirrus.yml`. - const List subshards = ['A', 'B']; - for (int i = 0; i < tests.length; i++) { - final List testGroup = tests[i]; - await runCommand( - dart, - [path.join('tool', 'tool_coverage.dart'), '--', ...testGroup], - workingDirectory: toolRoot, - environment: { - 'FLUTTER_ROOT': flutterRoot, - 'SUBSHARD': subshards[i], - }, - ); - } + await runCommand( + dart, + [path.join('tool', 'tool_coverage.dart')], + workingDirectory: toolRoot, + environment: { + 'FLUTTER_ROOT': flutterRoot, + } + ); } Future _runToolTests() async { final bq.BigqueryApi bigqueryApi = await _getBigqueryApi(); await _runSmokeTests(); - // The flutter_tool will currently be snapshotted without asserts. We need - // to force it to be regenerated with them enabled. - if (!Platform.isWindows) { - File(path.join(flutterRoot, 'bin', 'cache', 'flutter_tools.snapshot')).deleteSync(); - File(path.join(flutterRoot, 'bin', 'cache', 'flutter_tools.stamp')).deleteSync(); - } - // reduce overhead of build_runner in the create case. - if (noUseBuildRunner || Platform.environment['SUBSHARD'] == 'create') { - await _pubRunTest( - path.join(flutterRoot, 'packages', 'flutter_tools'), - tableData: bigqueryApi?.tabledata, - enableFlutterToolAsserts: !Platform.isWindows, - ); - } else { - await _buildRunnerTest( - path.join(flutterRoot, 'packages', 'flutter_tools'), - flutterRoot, - tableData: bigqueryApi?.tabledata, - enableFlutterToolAsserts: !Platform.isWindows, - ); - } + const String kDotShard = '.shard'; + const String kTest = 'test'; + final String toolsPath = path.join(flutterRoot, 'packages', 'flutter_tools'); - print('${bold}DONE: All tests successful.$reset'); + final Map subshards = Map.fromIterable( + Directory(path.join(toolsPath, kTest)) + .listSync() + .map((FileSystemEntity entry) => entry.path) + .where((String name) => name.endsWith(kDotShard)) + .map((String name) => path.basenameWithoutExtension(name)), + // The `dynamic` on the next line is because Map.fromIterable isn't generic. + value: (dynamic subshard) => () async { + await _pubRunTest( + toolsPath, + testPath: path.join(kTest, '$subshard$kDotShard'), + useBuildRunner: canUseBuildRunner, + tableData: bigqueryApi?.tabledata, + // TODO(ianh): The integration tests fail to start on Windows if asserts are enabled. + // See https://github.com/flutter/flutter/issues/36476 + enableFlutterToolAsserts: !(subshard == 'integration' && Platform.isWindows), + ); + }, + ); + + await selectSubshard(subshards); } /// Verifies that AOT, APK, and IPA (if on macOS) builds the @@ -503,53 +469,45 @@ Future _runCoverage() async { print('This file should have been generated by the `flutter test --coverage` script, but was not.'); exit(1); } - print('${bold}DONE: Coverage collection successful.$reset'); } -Future _buildRunnerTest( - String workingDirectory, - String flutterRoot, { +Future _pubRunTest(String workingDirectory, { String testPath, - bool enableFlutterToolAsserts = false, + bool enableFlutterToolAsserts = true, + bool useBuildRunner = false, bq.TabledataResourceApi tableData, }) async { - final List args = [ - 'run', - 'build_runner', - 'test', - '--', - if (useFlutterTestFormatter) '-rjson' else '-rcompact', - '-j1', - if (!hasColor) '--no-color', - if (testPath != null) testPath, - ]; + final List args = ['run', '--verbose']; + if (useBuildRunner) { + args.addAll(['build_runner', 'test', '--']); + } else { + args.add('test'); + } + args.add(useFlutterTestFormatter ? '-rjson' : '-rcompact'); + args.add('-j1'); // TODO(ianh): Scale based on CPUs. + if (!hasColor) + args.add('--no-color'); + if (testPath != null) + args.add(testPath); final Map pubEnvironment = { 'FLUTTER_ROOT': flutterRoot, - if (Directory(pubCache).existsSync()) 'PUB_CACHE': pubCache, }; + if (Directory(pubCache).existsSync()) { + pubEnvironment['PUB_CACHE'] = pubCache; + } if (enableFlutterToolAsserts) { // If an existing env variable exists append to it, but only if // it doesn't appear to already include enable-asserts. String toolsArgs = Platform.environment['FLUTTER_TOOL_ARGS'] ?? ''; if (!toolsArgs.contains('--enable-asserts')) - toolsArgs += ' --enable-asserts'; + toolsArgs += ' --enable-asserts'; pubEnvironment['FLUTTER_TOOL_ARGS'] = toolsArgs.trim(); + // The flutter_tool will originally have been snapshotted without asserts. + // We need to force it to be regenerated with them enabled. + deleteFile(path.join(flutterRoot, 'bin', 'cache', 'flutter_tools.snapshot')); + deleteFile(path.join(flutterRoot, 'bin', 'cache', 'flutter_tools.stamp')); } - - final String subShard = Platform.environment['SUBSHARD']; - switch (subShard) { - case 'integration': - args.addAll(['--tags', 'integration']); - break; - case 'create': - args.addAll(['--tags', 'create']); - break; - case 'tool': - args.addAll(['-x', 'integration', '-x', 'create']); - break; - } - if (useFlutterTestFormatter) { final FlutterCompactFormatter formatter = FlutterCompactFormatter(); final Stream testOutput = runAndGetStdout( @@ -564,68 +522,19 @@ Future _buildRunnerTest( await runCommand( pub, args, - workingDirectory:workingDirectory, - environment:pubEnvironment, - removeLine: (String line) => line.contains('[INFO]'), + workingDirectory: workingDirectory, + environment: pubEnvironment, + removeLine: useBuildRunner ? (String line) => line.startsWith('[INFO]') : null, ); } } -Future _pubRunTest( - String workingDirectory, { - String testPath, - bool enableFlutterToolAsserts = false, - bq.TabledataResourceApi tableData, -}) async { - final List args = [ - 'run', - 'test', - if (useFlutterTestFormatter) '-rjson' else '-rcompact', - '-j1', - if (!hasColor) '--no-color', - if (testPath != null) testPath, - ]; - final Map pubEnvironment = { - if (Directory(pubCache).existsSync()) 'PUB_CACHE': pubCache, - }; - if (enableFlutterToolAsserts) { - // If an existing env variable exists append to it, but only if - // it doesn't appear to already include enable-asserts. - String toolsArgs = Platform.environment['FLUTTER_TOOL_ARGS'] ?? ''; - if (!toolsArgs.contains('--enable-asserts')) - toolsArgs += ' --enable-asserts'; - pubEnvironment['FLUTTER_TOOL_ARGS'] = toolsArgs.trim(); - } - - final String subShard = Platform.environment['SUBSHARD']; - switch (subShard) { - case 'integration': - args.addAll(['--tags', 'integration']); - break; - case 'tool': - args.addAll(['--exclude-tags', 'integration']); - break; - case 'create': - args.addAll([path.join('test', 'general.shard', 'commands', 'create_test.dart')]); - break; - } - - if (useFlutterTestFormatter) { - final FlutterCompactFormatter formatter = FlutterCompactFormatter(); - final Stream testOutput = runAndGetStdout( - pub, - args, - workingDirectory: workingDirectory, - beforeExit: formatter.finish, - ); - await _processTestOutput(formatter, testOutput, tableData); - } else { - await runCommand( - pub, - args, - workingDirectory: workingDirectory, - ); - } +void deleteFile(String path) { + // There's a race condition here but in theory we're not racing anyone + // while this script runs, so should be ok. + final File file = File(path); + if (file.existsSync()) + file.deleteSync(); } enum CiProviders { @@ -1039,3 +948,25 @@ Future _androidGradleTests(String subShard) async { await _runDevicelabTest('module_host_with_custom_build_test', env: env); } } + +Future selectShard(Map shards) => _runFromList(shards, 'SHARD', 'shard'); +Future selectSubshard(Map subshards) => _runFromList(subshards, 'SUBSHARD', 'subshard'); + +Future _runFromList(Map items, String key, String name) async { + final String item = Platform.environment[key]; + if (item != null) { + if (!items.containsKey(item)) { + print('${red}Invalid $name: $item$reset'); + print('The available ${name}s are: ${items.keys.join(", ")}'); + exit(1); + } + print('$bold$key=$item$reset'); + await items[item](); + } else { + for (String currentItem in items.keys) { + print('$bold$key=$currentItem$reset'); + await items[currentItem](); + print(''); + } + } +} diff --git a/packages/flutter_tools/test/commands.shard/README.md b/packages/flutter_tools/test/commands.shard/README.md new file mode 100644 index 0000000000..9564f0f484 --- /dev/null +++ b/packages/flutter_tools/test/commands.shard/README.md @@ -0,0 +1,10 @@ +This directory contains tests for specific `flutter` commands. + +Tests that are self-contained unit tests should go in `hermetic/`. + +Tests that are more end-to-end, e.g. that involve actually running +subprocesses, should go in `permeable/`. + +The `../../tool/coverage_tool.dart` script (which is used to collect +coverage for the tool) runs only the tests in the `hermetic` directory +when collecting coverage. diff --git a/packages/flutter_tools/test/general.shard/commands/analyze_continuously_test.dart b/packages/flutter_tools/test/commands.shard/hermetic/analyze_continuously_test.dart similarity index 100% rename from packages/flutter_tools/test/general.shard/commands/analyze_continuously_test.dart rename to packages/flutter_tools/test/commands.shard/hermetic/analyze_continuously_test.dart diff --git a/packages/flutter_tools/test/general.shard/commands/analyze_test.dart b/packages/flutter_tools/test/commands.shard/hermetic/analyze_test.dart similarity index 100% rename from packages/flutter_tools/test/general.shard/commands/analyze_test.dart rename to packages/flutter_tools/test/commands.shard/hermetic/analyze_test.dart diff --git a/packages/flutter_tools/test/general.shard/commands/assemble_test.dart b/packages/flutter_tools/test/commands.shard/hermetic/assemble_test.dart similarity index 100% rename from packages/flutter_tools/test/general.shard/commands/assemble_test.dart rename to packages/flutter_tools/test/commands.shard/hermetic/assemble_test.dart diff --git a/packages/flutter_tools/test/general.shard/commands/attach_test.dart b/packages/flutter_tools/test/commands.shard/hermetic/attach_test.dart similarity index 100% rename from packages/flutter_tools/test/general.shard/commands/attach_test.dart rename to packages/flutter_tools/test/commands.shard/hermetic/attach_test.dart diff --git a/packages/flutter_tools/test/general.shard/commands/build_fuchsia_test.dart b/packages/flutter_tools/test/commands.shard/hermetic/build_fuchsia_test.dart similarity index 100% rename from packages/flutter_tools/test/general.shard/commands/build_fuchsia_test.dart rename to packages/flutter_tools/test/commands.shard/hermetic/build_fuchsia_test.dart diff --git a/packages/flutter_tools/test/general.shard/commands/build_linux_test.dart b/packages/flutter_tools/test/commands.shard/hermetic/build_linux_test.dart similarity index 100% rename from packages/flutter_tools/test/general.shard/commands/build_linux_test.dart rename to packages/flutter_tools/test/commands.shard/hermetic/build_linux_test.dart diff --git a/packages/flutter_tools/test/general.shard/commands/build_macos_test.dart b/packages/flutter_tools/test/commands.shard/hermetic/build_macos_test.dart similarity index 100% rename from packages/flutter_tools/test/general.shard/commands/build_macos_test.dart rename to packages/flutter_tools/test/commands.shard/hermetic/build_macos_test.dart diff --git a/packages/flutter_tools/test/general.shard/commands/build_web_test.dart b/packages/flutter_tools/test/commands.shard/hermetic/build_web_test.dart similarity index 100% rename from packages/flutter_tools/test/general.shard/commands/build_web_test.dart rename to packages/flutter_tools/test/commands.shard/hermetic/build_web_test.dart diff --git a/packages/flutter_tools/test/general.shard/commands/build_windows_test.dart b/packages/flutter_tools/test/commands.shard/hermetic/build_windows_test.dart similarity index 100% rename from packages/flutter_tools/test/general.shard/commands/build_windows_test.dart rename to packages/flutter_tools/test/commands.shard/hermetic/build_windows_test.dart diff --git a/packages/flutter_tools/test/general.shard/commands/clean_test.dart b/packages/flutter_tools/test/commands.shard/hermetic/clean_test.dart similarity index 100% rename from packages/flutter_tools/test/general.shard/commands/clean_test.dart rename to packages/flutter_tools/test/commands.shard/hermetic/clean_test.dart diff --git a/packages/flutter_tools/test/general.shard/commands/config_test.dart b/packages/flutter_tools/test/commands.shard/hermetic/config_test.dart similarity index 100% rename from packages/flutter_tools/test/general.shard/commands/config_test.dart rename to packages/flutter_tools/test/commands.shard/hermetic/config_test.dart diff --git a/packages/flutter_tools/test/general.shard/commands/create_usage_test.dart b/packages/flutter_tools/test/commands.shard/hermetic/create_usage_test.dart similarity index 76% rename from packages/flutter_tools/test/general.shard/commands/create_usage_test.dart rename to packages/flutter_tools/test/commands.shard/hermetic/create_usage_test.dart index c95e606491..d7c63ec275 100644 --- a/packages/flutter_tools/test/general.shard/commands/create_usage_test.dart +++ b/packages/flutter_tools/test/commands.shard/hermetic/create_usage_test.dart @@ -29,7 +29,7 @@ void main() { fs.path.join('flutter', 'packages', 'flutter_test', 'pubspec.yaml'), fs.path.join('flutter', 'bin', 'cache', 'artifacts', 'gradle_wrapper', 'wrapper'), fs.path.join('usr', 'local', 'bin', 'adb'), - fs.path.join('Android', 'platform-tools', 'foo'), + fs.path.join('Android', 'platform-tools', 'adb.exe'), ]; for (String path in paths) { fs.file(path).createSync(recursive: true); @@ -43,25 +43,17 @@ void main() { final CreateCommand command = CreateCommand(); final CommandRunner runner = createTestCommandRunner(command); - await runner.run([ - 'create', '--flutter-root=flutter', '--no-pub', '--template=module', 'testy']); - expect(await command.usageValues, - containsPair(CustomDimensions.commandCreateProjectType, 'module')); + await runner.run(['create', '--flutter-root=flutter', '--no-pub', '--template=module', 'testy']); + expect(await command.usageValues, containsPair(CustomDimensions.commandCreateProjectType, 'module')); - await runner.run([ - 'create', '--flutter-root=flutter', '--no-pub', '--template=app', 'testy']); - expect(await command.usageValues, - containsPair(CustomDimensions.commandCreateProjectType, 'app')); + await runner.run(['create', '--flutter-root=flutter', '--no-pub', '--template=app', 'testy']); + expect(await command.usageValues, containsPair(CustomDimensions.commandCreateProjectType, 'app')); - await runner.run([ - 'create', '--flutter-root=flutter', '--no-pub', '--template=package', 'testy']); - expect(await command.usageValues, - containsPair(CustomDimensions.commandCreateProjectType, 'package')); + await runner.run(['create', '--flutter-root=flutter', '--no-pub', '--template=package', 'testy']); + expect(await command.usageValues, containsPair(CustomDimensions.commandCreateProjectType, 'package')); - await runner.run([ - 'create', '--flutter-root=flutter', '--no-pub', '--template=plugin', 'testy']); - expect(await command.usageValues, - containsPair(CustomDimensions.commandCreateProjectType, 'plugin')); + await runner.run(['create', '--flutter-root=flutter', '--no-pub', '--template=plugin', 'testy']); + expect(await command.usageValues, containsPair(CustomDimensions.commandCreateProjectType, 'plugin')); })); test('set iOS host language type as usage value', () => testbed.run(() async { diff --git a/packages/flutter_tools/test/general.shard/commands/daemon_test.dart b/packages/flutter_tools/test/commands.shard/hermetic/daemon_test.dart similarity index 98% rename from packages/flutter_tools/test/general.shard/commands/daemon_test.dart rename to packages/flutter_tools/test/commands.shard/hermetic/daemon_test.dart index f779433ef3..c9eadf15fc 100644 --- a/packages/flutter_tools/test/general.shard/commands/daemon_test.dart +++ b/packages/flutter_tools/test/commands.shard/hermetic/daemon_test.dart @@ -139,10 +139,10 @@ void main() { final StreamController> commands = StreamController>(); final StreamController> responses = StreamController>(); daemon = Daemon( - commands.stream, - responses.add, - daemonCommand: command, - notifyingLogger: notifyingLogger, + commands.stream, + responses.add, + daemonCommand: command, + notifyingLogger: notifyingLogger, ); commands.add({ diff --git a/packages/flutter_tools/test/general.shard/commands/devices_test.dart b/packages/flutter_tools/test/commands.shard/hermetic/devices_test.dart similarity index 100% rename from packages/flutter_tools/test/general.shard/commands/devices_test.dart rename to packages/flutter_tools/test/commands.shard/hermetic/devices_test.dart diff --git a/packages/flutter_tools/test/general.shard/commands/doctor_test.dart b/packages/flutter_tools/test/commands.shard/hermetic/doctor_test.dart similarity index 100% rename from packages/flutter_tools/test/general.shard/commands/doctor_test.dart rename to packages/flutter_tools/test/commands.shard/hermetic/doctor_test.dart diff --git a/packages/flutter_tools/test/general.shard/commands/drive_test.dart b/packages/flutter_tools/test/commands.shard/hermetic/drive_test.dart similarity index 100% rename from packages/flutter_tools/test/general.shard/commands/drive_test.dart rename to packages/flutter_tools/test/commands.shard/hermetic/drive_test.dart diff --git a/packages/flutter_tools/test/general.shard/commands/ide_config_test.dart b/packages/flutter_tools/test/commands.shard/hermetic/ide_config_test.dart similarity index 98% rename from packages/flutter_tools/test/general.shard/commands/ide_config_test.dart rename to packages/flutter_tools/test/commands.shard/hermetic/ide_config_test.dart index dd6defa554..abad5430c3 100644 --- a/packages/flutter_tools/test/general.shard/commands/ide_config_test.dart +++ b/packages/flutter_tools/test/commands.shard/hermetic/ide_config_test.dart @@ -46,11 +46,9 @@ void main() { fs.path.join(basePath, '.idea'): 'dir', fs.path.join(basePath, '.idea', 'modules.xml$suffix'): 'modules $marker', fs.path.join(basePath, '.idea', 'vcs.xml$suffix'): 'vcs $marker', - fs.path.join(basePath, '.idea', '.name$suffix'): - 'codeStyleSettings $marker', + fs.path.join(basePath, '.idea', '.name$suffix'): 'codeStyleSettings $marker', fs.path.join(basePath, '.idea', 'runConfigurations'): 'dir', - fs.path.join(basePath, '.idea', 'runConfigurations', 'hello_world.xml$suffix'): - 'hello_world $marker', + fs.path.join(basePath, '.idea', 'runConfigurations', 'hello_world.xml$suffix'): 'hello_world $marker', fs.path.join(basePath, 'flutter.iml$suffix'): 'flutter $marker', fs.path.join(basePath, 'packages', 'new', 'deep.iml$suffix'): 'deep $marker', }; diff --git a/packages/flutter_tools/test/general.shard/commands/install_test.dart b/packages/flutter_tools/test/commands.shard/hermetic/install_test.dart similarity index 100% rename from packages/flutter_tools/test/general.shard/commands/install_test.dart rename to packages/flutter_tools/test/commands.shard/hermetic/install_test.dart diff --git a/packages/flutter_tools/test/general.shard/commands/precache_test.dart b/packages/flutter_tools/test/commands.shard/hermetic/precache_test.dart similarity index 100% rename from packages/flutter_tools/test/general.shard/commands/precache_test.dart rename to packages/flutter_tools/test/commands.shard/hermetic/precache_test.dart diff --git a/packages/flutter_tools/test/general.shard/commands/run_test.dart b/packages/flutter_tools/test/commands.shard/hermetic/run_test.dart similarity index 100% rename from packages/flutter_tools/test/general.shard/commands/run_test.dart rename to packages/flutter_tools/test/commands.shard/hermetic/run_test.dart diff --git a/packages/flutter_tools/test/general.shard/commands/shell_completion_test.dart b/packages/flutter_tools/test/commands.shard/hermetic/shell_completion_test.dart similarity index 100% rename from packages/flutter_tools/test/general.shard/commands/shell_completion_test.dart rename to packages/flutter_tools/test/commands.shard/hermetic/shell_completion_test.dart diff --git a/packages/flutter_tools/test/general.shard/commands/version_test.dart b/packages/flutter_tools/test/commands.shard/hermetic/version_test.dart similarity index 100% rename from packages/flutter_tools/test/general.shard/commands/version_test.dart rename to packages/flutter_tools/test/commands.shard/hermetic/version_test.dart diff --git a/packages/flutter_tools/test/general.shard/commands/analyze_once_test.dart b/packages/flutter_tools/test/commands.shard/permeable/analyze_once_test.dart similarity index 100% rename from packages/flutter_tools/test/general.shard/commands/analyze_once_test.dart rename to packages/flutter_tools/test/commands.shard/permeable/analyze_once_test.dart diff --git a/packages/flutter_tools/test/general.shard/commands/build_bundle_test.dart b/packages/flutter_tools/test/commands.shard/permeable/build_bundle_test.dart similarity index 100% rename from packages/flutter_tools/test/general.shard/commands/build_bundle_test.dart rename to packages/flutter_tools/test/commands.shard/permeable/build_bundle_test.dart diff --git a/packages/flutter_tools/test/general.shard/commands/create_test.dart b/packages/flutter_tools/test/commands.shard/permeable/create_test.dart similarity index 99% rename from packages/flutter_tools/test/general.shard/commands/create_test.dart rename to packages/flutter_tools/test/commands.shard/permeable/create_test.dart index 1cb66504dc..895565bc82 100644 --- a/packages/flutter_tools/test/general.shard/commands/create_test.dart +++ b/packages/flutter_tools/test/commands.shard/permeable/create_test.dart @@ -2,8 +2,6 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. -// This test performs too poorly to run with coverage enabled. -@Tags(['create', 'no_coverage']) import 'dart:async'; import 'dart:convert'; import 'dart:typed_data'; @@ -25,7 +23,6 @@ import 'package:process/process.dart'; import '../../src/common.dart'; import '../../src/context.dart'; - const String frameworkRevision = '12345678'; const String frameworkChannel = 'omega'; final Generator _kNoColorTerminalPlatform = () => FakePlatform.fromPlatform(const LocalPlatform())..stdoutSupportsAnsi = false; @@ -116,7 +113,9 @@ void main() { 'ios/', ]); return _runFlutterTest(projectDir); - }, timeout: allowForRemotePubInvocation); + }, + timeout: allowForRemotePubInvocation, + ); testUsingContext('cannot create a project if non-empty non-project directory exists with .metadata', () async { await projectDir.absolute.childDirectory('blag').create(recursive: true); diff --git a/packages/flutter_tools/test/general.shard/commands/format_test.dart b/packages/flutter_tools/test/commands.shard/permeable/format_test.dart similarity index 100% rename from packages/flutter_tools/test/general.shard/commands/format_test.dart rename to packages/flutter_tools/test/commands.shard/permeable/format_test.dart diff --git a/packages/flutter_tools/test/general.shard/commands/packages_test.dart b/packages/flutter_tools/test/commands.shard/permeable/packages_test.dart similarity index 100% rename from packages/flutter_tools/test/general.shard/commands/packages_test.dart rename to packages/flutter_tools/test/commands.shard/permeable/packages_test.dart diff --git a/packages/flutter_tools/test/general.shard/commands/test_test.dart b/packages/flutter_tools/test/commands.shard/permeable/test_test.dart similarity index 100% rename from packages/flutter_tools/test/general.shard/commands/test_test.dart rename to packages/flutter_tools/test/commands.shard/permeable/test_test.dart diff --git a/packages/flutter_tools/test/general.shard/commands/upgrade_test.dart b/packages/flutter_tools/test/commands.shard/permeable/upgrade_test.dart similarity index 100% rename from packages/flutter_tools/test/general.shard/commands/upgrade_test.dart rename to packages/flutter_tools/test/commands.shard/permeable/upgrade_test.dart diff --git a/packages/flutter_tools/tool/tool_coverage.dart b/packages/flutter_tools/tool/tool_coverage.dart index 3504e3680a..6529e748f2 100644 --- a/packages/flutter_tools/tool/tool_coverage.dart +++ b/packages/flutter_tools/tool/tool_coverage.dart @@ -39,7 +39,18 @@ Future main(List arguments) async { [Runtime.vm], () => vmPlatform, ); - await test.main(['-x', 'no_coverage', '--no-color', '-r', 'compact', '-j', '1', ...arguments]); + if (arguments.isEmpty) { + arguments = [ + path.join('test', 'general.shard'), + path.join('test', 'commands.shard', 'hermetic'), + ]; + } + await test.main([ + '--no-color', + '-r', 'compact', + '-j', '1', + ...arguments + ]); exit(exitCode); }); } @@ -141,8 +152,7 @@ class VMPlatform extends PlatformPlugin { final String result = await coverageCollector.finalizeCoverage( formatter: formatter, ); - final String prefix = Platform.environment['SUBSHARD'] ?? ''; - final String outputLcovPath = path.join('coverage', '$prefix.lcov.info'); + final String outputLcovPath = path.join('coverage', 'lcov.info'); File(outputLcovPath) ..createSync(recursive: true) ..writeAsStringSync(result);