From 738ce43d97218e0d2d4c4d5f5ecbeec39f0dbef1 Mon Sep 17 00:00:00 2001 From: Greg Spencer Date: Mon, 26 Jul 2021 23:40:49 -0700 Subject: [PATCH] Randomize tests, exclude tests that fail with randomization. (#86793) * Randomize tests, exclude tests that fail with randomization. * Disable some more tool tests --- dev/bots/run_command.dart | 13 ++++++-- dev/bots/test.dart | 32 +++++++++++++++++-- packages/flutter/dart_test.yaml | 5 +++ .../animation/animation_controller_test.dart | 6 ++++ .../test/cupertino/action_sheet_test.dart | 6 ++++ .../test/cupertino/date_picker_test.dart | 6 ++++ .../cupertino/nav_bar_transition_test.dart | 6 ++++ .../test/cupertino/text_field_test.dart | 6 ++++ .../foundation/service_extensions_test.dart | 6 ++++ .../flutter/test/gestures/debug_test.dart | 6 ++++ .../test/gestures/long_press_test.dart | 6 ++++ packages/flutter/test/gestures/tap_test.dart | 6 ++++ packages/flutter/test/material/app_test.dart | 6 ++++ .../flutter/test/material/dropdown_test.dart | 6 ++++ .../material/paginated_data_table_test.dart | 6 ++++ .../material/progress_indicator_test.dart | 6 ++++ .../flutter/test/material/scaffold_test.dart | 6 ++++ .../flutter/test/material/scrollbar_test.dart | 6 ++++ .../test/material/text_field_test.dart | 6 ++++ .../test/painting/image_stream_test.dart | 6 ++++ .../flutter/test/physics/newton_test.dart | 6 ++++ .../flutter/test/rendering/editable_test.dart | 6 ++++ .../flutter/test/rendering/viewport_test.dart | 6 ++++ .../test/scheduler/scheduler_test.dart | 6 ++++ .../flutter/test/scheduler/ticker_test.dart | 6 ++++ .../test/services/platform_channel_test.dart | 6 ++++ .../test/widgets/app_overrides_test.dart | 6 ++++ .../flutter/test/widgets/binding_test.dart | 6 ++++ packages/flutter/test/widgets/clip_test.dart | 6 ++++ packages/flutter/test/widgets/debug_test.dart | 6 ++++ .../test/widgets/dismissible_test.dart | 6 ++++ .../flutter/test/widgets/draggable_test.dart | 6 ++++ .../test/widgets/platform_view_test.dart | 6 ++++ .../flutter/test/widgets/routes_test.dart | 6 ++++ .../scroll_aware_image_provider_test.dart | 6 ++++ .../test/widgets/widget_inspector_test.dart | 15 ++++++--- packages/flutter_driver/dart_test.yaml | 5 +++ .../test/src/real_tests/extension_test.dart | 6 ++++ .../src/real_tests/flutter_driver_test.dart | 6 ++++ packages/flutter_goldens/dart_test.yaml | 5 +++ .../test/flutter_goldens_test.dart | 6 ++++ packages/flutter_test/dart_test.yaml | 5 +++ packages/flutter_test/test/bindings_test.dart | 6 ++++ .../test/test_text_input_test.dart | 6 ++++ packages/flutter_test/test/window_test.dart | 6 ++++ packages/flutter_tools/dart_test.yaml | 6 ++++ .../hermetic/build_ios_framework_test.dart | 6 ++++ .../commands.shard/hermetic/daemon_test.dart | 6 ++++ .../commands.shard/hermetic/doctor_test.dart | 6 ++++ .../commands.shard/hermetic/run_test.dart | 6 ++++ .../permeable/build_bundle_test.dart | 6 ++++ .../permeable/packages_test.dart | 6 ++++ .../targets/dart_plugin_registrant_test.dart | 6 ++++ .../test/general.shard/channel_test.dart | 6 ++++ .../dart/language_version_test.dart | 6 ++++ .../break_on_framework_exceptions_test.dart | 6 ++++ .../overall_experience_test.dart | 6 ++++ .../test/integration.shard/test_test.dart | 6 ++++ 58 files changed, 377 insertions(+), 9 deletions(-) create mode 100644 packages/flutter/dart_test.yaml create mode 100644 packages/flutter_driver/dart_test.yaml create mode 100644 packages/flutter_goldens/dart_test.yaml create mode 100644 packages/flutter_test/dart_test.yaml diff --git a/dev/bots/run_command.dart b/dev/bots/run_command.dart index 13a97f2073..f443198958 100644 --- a/dev/bots/run_command.dart +++ b/dev/bots/run_command.dart @@ -113,7 +113,8 @@ Future startCommand(String executable, List arguments, { environment: environment, ); - Future>>? savedStdout, savedStderr; + Future>> savedStdout = Future>>.value(>[]); + Future>> savedStderr = Future>>.value(>[]); final Stream> stdoutSource = process.stdout .transform(const Utf8Decoder()) .transform(const LineSplitter()) @@ -128,8 +129,14 @@ Future startCommand(String executable, List arguments, { .transform(const Utf8Encoder()); switch (outputMode) { case OutputMode.print: - stdoutSource.listen(io.stdout.add); - process.stderr.listen(io.stdout.add); + stdoutSource.listen((List output) { + io.stdout.add(output); + savedStdout.then((List> list) => list.add(output)); + }); + process.stderr.listen((List output) { + io.stdout.add(output); + savedStdout.then((List> list) => list.add(output)); + }); break; case OutputMode.capture: savedStdout = stdoutSource.toList(); diff --git a/dev/bots/test.dart b/dev/bots/test.dart index 04a6afa232..64045be329 100644 --- a/dev/bots/test.dart +++ b/dev/bots/test.dart @@ -85,6 +85,22 @@ const List kWebTestFileKnownFailures = [ const String kSmokeTestShardName = 'smoke_tests'; const List _kAllBuildModes = ['debug', 'profile', 'release']; +// The seed used to shuffle tests. If not passed with +// --test-randomize-ordering-seed= on the command line, it will be set the +// first time it is accessed. Pass zero to turn off shuffling. +String? _shuffleSeed; +String get shuffleSeed { + if (_shuffleSeed == null) { + // Change the seed at 7am, UTC. + final DateTime seedTime = DateTime.now().toUtc().subtract(const Duration(hours: 7)); + // Generates YYYYMMDD as the seed, so that testing continues to fail for a + // day after the seed changes, and on other days the seed can be used to + // replicate failures. + _shuffleSeed = '${seedTime.year * 10000 + seedTime.month * 100 + seedTime.day}'; + } + return _shuffleSeed!; +} + /// When you call this, you can pass additional arguments to pass custom /// arguments to flutter test. For example, you might want to call this /// script with the parameter --local-engine=host_debug_unopt to @@ -99,12 +115,20 @@ Future main(List args) async { print('$clock STARTING ANALYSIS'); try { flutterTestArgs.addAll(args); + final Set removeArgs = {}; for (final String arg in args) { - if (arg.startsWith('--local-engine=')) + if (arg.startsWith('--local-engine=')) { localEngineEnv['FLUTTER_LOCAL_ENGINE'] = arg.substring('--local-engine='.length); - if (arg.startsWith('--local-engine-src-path=')) + } + if (arg.startsWith('--local-engine-src-path=')) { localEngineEnv['FLUTTER_LOCAL_ENGINE_SRC_PATH'] = arg.substring('--local-engine-src-path='.length); + } + if (arg.startsWith('--test-randomize-ordering-seed=')) { + _shuffleSeed = arg.substring('--test-randomize-ordering-seed='.length); + removeArgs.add(arg); + } } + flutterTestArgs.removeWhere((String arg) => removeArgs.contains(arg)); if (Platform.environment.containsKey(CIRRUS_TASK_NAME)) print('Running task: ${Platform.environment[CIRRUS_TASK_NAME]}'); print('═' * 80); @@ -1437,6 +1461,7 @@ Future _pubRunTest(String workingDirectory, { Duration? perTestTimeout, bool includeLocalEngineEnv = false, bool ensurePrecompiledTool = true, + bool shuffleTests = true, }) async { int? cpus; final String? cpuVariable = Platform.environment['CPU']; // CPU is set in cirrus.yml @@ -1459,6 +1484,7 @@ Future _pubRunTest(String workingDirectory, { final List args = [ 'run', 'test', + if (shuffleTests) '--test-randomize-ordering-seed=$shuffleSeed', if (useFlutterTestFormatter) '-rjson' else @@ -1532,11 +1558,13 @@ Future _runFlutterTest(String workingDirectory, { List options = const [], Map? environment, List tests = const [], + bool shuffleTests = true, }) async { assert(!printOutput || outputChecker == null, 'Output either can be printed or checked but not both'); final List args = [ 'test', + if (shuffleTests) '--test-randomize-ordering-seed=$shuffleSeed', ...options, ...flutterTestArgs, ]; diff --git a/packages/flutter/dart_test.yaml b/packages/flutter/dart_test.yaml new file mode 100644 index 0000000000..fb3b8a21d3 --- /dev/null +++ b/packages/flutter/dart_test.yaml @@ -0,0 +1,5 @@ +tags: + # This tag tells the test framework to not shuffle the test order according to + # the --test-randomize-ordering-seed for the suites that have this tag. + no-shuffle: + allow_test_randomization: false diff --git a/packages/flutter/test/animation/animation_controller_test.dart b/packages/flutter/test/animation/animation_controller_test.dart index 2aff62d3a8..3584941212 100644 --- a/packages/flutter/test/animation/animation_controller_test.dart +++ b/packages/flutter/test/animation/animation_controller_test.dart @@ -2,6 +2,12 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. +// TODO(gspencergoog): Remove this tag once this test's state leaks/test +// dependencies have been fixed. +// https://github.com/flutter/flutter/issues/85160 +// Fails with "flutter test --test-randomize-ordering-seed=123" +@Tags(['no-shuffle']) + import 'dart:ui' as ui; import 'package:flutter/foundation.dart'; diff --git a/packages/flutter/test/cupertino/action_sheet_test.dart b/packages/flutter/test/cupertino/action_sheet_test.dart index 1f46897bc6..1859a1b809 100644 --- a/packages/flutter/test/cupertino/action_sheet_test.dart +++ b/packages/flutter/test/cupertino/action_sheet_test.dart @@ -2,6 +2,12 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. +// TODO(gspencergoog): Remove this tag once this test's state leaks/test +// dependencies have been fixed. +// https://github.com/flutter/flutter/issues/85160 +// Fails with "flutter test --test-randomize-ordering-seed=123" +@Tags(['no-shuffle']) + import 'package:flutter/cupertino.dart'; import 'package:flutter/material.dart'; diff --git a/packages/flutter/test/cupertino/date_picker_test.dart b/packages/flutter/test/cupertino/date_picker_test.dart index 2bd8541847..38c910958a 100644 --- a/packages/flutter/test/cupertino/date_picker_test.dart +++ b/packages/flutter/test/cupertino/date_picker_test.dart @@ -2,6 +2,12 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. +// TODO(gspencergoog): Remove this tag once this test's state leaks/test +// dependencies have been fixed. +// https://github.com/flutter/flutter/issues/85160 +// Fails with "flutter test --test-randomize-ordering-seed=123" +@Tags(['no-shuffle']) + import 'dart:ui'; import 'package:flutter/cupertino.dart'; diff --git a/packages/flutter/test/cupertino/nav_bar_transition_test.dart b/packages/flutter/test/cupertino/nav_bar_transition_test.dart index d9a4ba8d96..7dbe487354 100644 --- a/packages/flutter/test/cupertino/nav_bar_transition_test.dart +++ b/packages/flutter/test/cupertino/nav_bar_transition_test.dart @@ -3,6 +3,12 @@ // found in the LICENSE file. @TestOn('!chrome') +// TODO(gspencergoog): Remove this tag once this test's state leaks/test +// dependencies have been fixed. +// https://github.com/flutter/flutter/issues/85160 +// Fails with "flutter test --test-randomize-ordering-seed=456" +@Tags(['no-shuffle']) + import 'package:flutter/cupertino.dart'; import 'package:flutter/rendering.dart'; import 'package:flutter_test/flutter_test.dart'; diff --git a/packages/flutter/test/cupertino/text_field_test.dart b/packages/flutter/test/cupertino/text_field_test.dart index a69aa7e04b..ed2c875ad8 100644 --- a/packages/flutter/test/cupertino/text_field_test.dart +++ b/packages/flutter/test/cupertino/text_field_test.dart @@ -2,6 +2,12 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. +// TODO(gspencergoog): Remove this tag once this test's state leaks/test +// dependencies have been fixed. +// https://github.com/flutter/flutter/issues/85160 +// Fails with "flutter test --test-randomize-ordering-seed=123" +@Tags(['no-shuffle']) + import 'dart:ui' as ui show BoxHeightStyle, BoxWidthStyle, Color; import 'package:flutter/cupertino.dart'; diff --git a/packages/flutter/test/foundation/service_extensions_test.dart b/packages/flutter/test/foundation/service_extensions_test.dart index e2f592b368..2916000532 100644 --- a/packages/flutter/test/foundation/service_extensions_test.dart +++ b/packages/flutter/test/foundation/service_extensions_test.dart @@ -2,6 +2,12 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. +// TODO(gspencergoog): Remove this tag once this test's state leaks/test +// dependencies have been fixed. +// https://github.com/flutter/flutter/issues/85160 +// Fails with "flutter test --test-randomize-ordering-seed=123" +@Tags(['no-shuffle']) + import 'dart:async'; import 'dart:convert'; import 'dart:ui' as ui; diff --git a/packages/flutter/test/gestures/debug_test.dart b/packages/flutter/test/gestures/debug_test.dart index b2b24bc04d..9aa34e1470 100644 --- a/packages/flutter/test/gestures/debug_test.dart +++ b/packages/flutter/test/gestures/debug_test.dart @@ -2,6 +2,12 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. +// TODO(gspencergoog): Remove this tag once this test's state leaks/test +// dependencies have been fixed. +// https://github.com/flutter/flutter/issues/85160 +// Fails with "flutter test --test-randomize-ordering-seed=123" +@Tags(['no-shuffle']) + import 'package:flutter/foundation.dart'; import 'package:flutter/gestures.dart'; import 'package:flutter_test/flutter_test.dart'; diff --git a/packages/flutter/test/gestures/long_press_test.dart b/packages/flutter/test/gestures/long_press_test.dart index a07de9471f..e3f3880025 100644 --- a/packages/flutter/test/gestures/long_press_test.dart +++ b/packages/flutter/test/gestures/long_press_test.dart @@ -2,6 +2,12 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. +// TODO(gspencergoog): Remove this tag once this test's state leaks/test +// dependencies have been fixed. +// https://github.com/flutter/flutter/issues/85160 +// Fails with "flutter test --test-randomize-ordering-seed=123" +@Tags(['no-shuffle']) + import 'package:flutter/gestures.dart'; import 'package:flutter_test/flutter_test.dart'; diff --git a/packages/flutter/test/gestures/tap_test.dart b/packages/flutter/test/gestures/tap_test.dart index 2266b5478d..9779004b58 100644 --- a/packages/flutter/test/gestures/tap_test.dart +++ b/packages/flutter/test/gestures/tap_test.dart @@ -2,6 +2,12 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. +// TODO(gspencergoog): Remove this tag once this test's state leaks/test +// dependencies have been fixed. +// https://github.com/flutter/flutter/issues/85160 +// Fails with "flutter test --test-randomize-ordering-seed=1000" +@Tags(['no-shuffle']) + import 'package:flutter/foundation.dart'; import 'package:flutter/gestures.dart'; import 'package:flutter_test/flutter_test.dart'; diff --git a/packages/flutter/test/material/app_test.dart b/packages/flutter/test/material/app_test.dart index 55bd0fce88..6828cdb9c4 100644 --- a/packages/flutter/test/material/app_test.dart +++ b/packages/flutter/test/material/app_test.dart @@ -2,6 +2,12 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. +// TODO(gspencergoog): Remove this tag once this test's state leaks/test +// dependencies have been fixed. +// https://github.com/flutter/flutter/issues/85160 +// Fails with "flutter test --test-randomize-ordering-seed=123" +@Tags(['no-shuffle']) + import 'package:flutter/cupertino.dart'; import 'package:flutter/foundation.dart'; import 'package:flutter/material.dart'; diff --git a/packages/flutter/test/material/dropdown_test.dart b/packages/flutter/test/material/dropdown_test.dart index 491ae4cb11..f5c30a8fbf 100644 --- a/packages/flutter/test/material/dropdown_test.dart +++ b/packages/flutter/test/material/dropdown_test.dart @@ -2,6 +2,12 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. +// TODO(gspencergoog): Remove this tag once this test's state leaks/test +// dependencies have been fixed. +// https://github.com/flutter/flutter/issues/85160 +// Fails with "flutter test --test-randomize-ordering-seed=1408669812" +@Tags(['no-shuffle']) + import 'dart:math' as math; import 'dart:ui' show window; diff --git a/packages/flutter/test/material/paginated_data_table_test.dart b/packages/flutter/test/material/paginated_data_table_test.dart index 5e0c60eb61..23b5a8f826 100644 --- a/packages/flutter/test/material/paginated_data_table_test.dart +++ b/packages/flutter/test/material/paginated_data_table_test.dart @@ -2,6 +2,12 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. +// TODO(gspencergoog): Remove this tag once this test's state leaks/test +// dependencies have been fixed. +// https://github.com/flutter/flutter/issues/85160 +// Fails with "flutter test --test-randomize-ordering-seed=1000" +@Tags(['no-shuffle']) + import 'package:flutter/gestures.dart' show DragStartBehavior; import 'package:flutter/material.dart'; import 'package:flutter/rendering.dart'; diff --git a/packages/flutter/test/material/progress_indicator_test.dart b/packages/flutter/test/material/progress_indicator_test.dart index 9f7a8ccea2..f018449212 100644 --- a/packages/flutter/test/material/progress_indicator_test.dart +++ b/packages/flutter/test/material/progress_indicator_test.dart @@ -2,6 +2,12 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. +// TODO(gspencergoog): Remove this tag once this test's state leaks/test +// dependencies have been fixed. +// https://github.com/flutter/flutter/issues/85160 +// Fails with "flutter test --test-randomize-ordering-seed=456" +@Tags(['no-shuffle']) + import 'package:flutter/cupertino.dart'; import 'package:flutter/material.dart'; import 'package:flutter/rendering.dart'; diff --git a/packages/flutter/test/material/scaffold_test.dart b/packages/flutter/test/material/scaffold_test.dart index 8f4a55fcd5..5650d2a737 100644 --- a/packages/flutter/test/material/scaffold_test.dart +++ b/packages/flutter/test/material/scaffold_test.dart @@ -2,6 +2,12 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. +// TODO(gspencergoog): Remove this tag once this test's state leaks/test +// dependencies have been fixed. +// https://github.com/flutter/flutter/issues/85160 +// Fails with "flutter test --test-randomize-ordering-seed=123" +@Tags(['no-shuffle']) + import 'package:flutter/foundation.dart'; import 'package:flutter/gestures.dart' show DragStartBehavior; import 'package:flutter/material.dart'; diff --git a/packages/flutter/test/material/scrollbar_test.dart b/packages/flutter/test/material/scrollbar_test.dart index 97b7facd96..3bdfd1261e 100644 --- a/packages/flutter/test/material/scrollbar_test.dart +++ b/packages/flutter/test/material/scrollbar_test.dart @@ -2,6 +2,12 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. +// TODO(gspencergoog): Remove this tag once this test's state leaks/test +// dependencies have been fixed. +// https://github.com/flutter/flutter/issues/85160 +// Fails with "flutter test --test-randomize-ordering-seed=382757700" +@Tags(['no-shuffle']) + import 'dart:ui' as ui; import 'package:flutter/cupertino.dart'; diff --git a/packages/flutter/test/material/text_field_test.dart b/packages/flutter/test/material/text_field_test.dart index d4f9cb489d..8bd5ca7238 100644 --- a/packages/flutter/test/material/text_field_test.dart +++ b/packages/flutter/test/material/text_field_test.dart @@ -2,6 +2,12 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. +// TODO(gspencergoog): Remove this tag once this test's state leaks/test +// dependencies have been fixed. +// https://github.com/flutter/flutter/issues/85160 +// Fails with "flutter test --test-randomize-ordering-seed=3890307731" +@Tags(['no-shuffle']) + import 'dart:math' as math; import 'dart:ui' as ui show window, BoxHeightStyle, BoxWidthStyle, WindowPadding; diff --git a/packages/flutter/test/painting/image_stream_test.dart b/packages/flutter/test/painting/image_stream_test.dart index 274fd32a00..264e81ea48 100644 --- a/packages/flutter/test/painting/image_stream_test.dart +++ b/packages/flutter/test/painting/image_stream_test.dart @@ -2,6 +2,12 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. +// TODO(gspencergoog): Remove this tag once this test's state leaks/test +// dependencies have been fixed. +// https://github.com/flutter/flutter/issues/85160 +// Fails with "flutter test --test-randomize-ordering-seed=456" +@Tags(['no-shuffle']) + import 'dart:async'; import 'dart:typed_data'; import 'dart:ui'; diff --git a/packages/flutter/test/physics/newton_test.dart b/packages/flutter/test/physics/newton_test.dart index 34556213e4..e70d7363b0 100644 --- a/packages/flutter/test/physics/newton_test.dart +++ b/packages/flutter/test/physics/newton_test.dart @@ -2,6 +2,12 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. +// TODO(gspencergoog): Remove this tag once this test's state leaks/test +// dependencies have been fixed. +// https://github.com/flutter/flutter/issues/85160 +// Fails with "flutter test --test-randomize-ordering-seed=123" +@Tags(['no-shuffle']) + import 'package:flutter/foundation.dart'; import 'package:flutter/physics.dart'; import 'package:flutter/widgets.dart'; diff --git a/packages/flutter/test/rendering/editable_test.dart b/packages/flutter/test/rendering/editable_test.dart index 4f0719f1d9..d68bcae49c 100644 --- a/packages/flutter/test/rendering/editable_test.dart +++ b/packages/flutter/test/rendering/editable_test.dart @@ -2,6 +2,12 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. +// TODO(gspencergoog): Remove this tag once this test's state leaks/test +// dependencies have been fixed. +// https://github.com/flutter/flutter/issues/85160 +// Fails with "flutter test --test-randomize-ordering-seed=20210704" +@Tags(['no-shuffle']) + import 'dart:io' show Platform; import 'package:flutter/foundation.dart'; diff --git a/packages/flutter/test/rendering/viewport_test.dart b/packages/flutter/test/rendering/viewport_test.dart index 14a1b2c5be..4520de3b41 100644 --- a/packages/flutter/test/rendering/viewport_test.dart +++ b/packages/flutter/test/rendering/viewport_test.dart @@ -7,6 +7,12 @@ // initialize a binding, which rendering_tester will attempt to re-initialize // (or vice versa). +// TODO(gspencergoog): Remove this tag once this test's state leaks/test +// dependencies have been fixed. +// https://github.com/flutter/flutter/issues/85160 +// Fails with "flutter test --test-randomize-ordering-seed=123" +@Tags(['no-shuffle']) + import 'package:flutter/foundation.dart'; import 'package:flutter/rendering.dart'; import 'package:flutter/widgets.dart'; diff --git a/packages/flutter/test/scheduler/scheduler_test.dart b/packages/flutter/test/scheduler/scheduler_test.dart index ec1b905453..914c6c0e9b 100644 --- a/packages/flutter/test/scheduler/scheduler_test.dart +++ b/packages/flutter/test/scheduler/scheduler_test.dart @@ -2,6 +2,12 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. +// TODO(gspencergoog): Remove this tag once this test's state leaks/test +// dependencies have been fixed. +// https://github.com/flutter/flutter/issues/85160 +// Fails with "flutter test --test-randomize-ordering-seed=123" +@Tags(['no-shuffle']) + import 'dart:async'; import 'dart:ui' show window; diff --git a/packages/flutter/test/scheduler/ticker_test.dart b/packages/flutter/test/scheduler/ticker_test.dart index 6c8dedc5bc..7e9dd1ac18 100644 --- a/packages/flutter/test/scheduler/ticker_test.dart +++ b/packages/flutter/test/scheduler/ticker_test.dart @@ -2,6 +2,12 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. +// TODO(gspencergoog): Remove this tag once this test's state leaks/test +// dependencies have been fixed. +// https://github.com/flutter/flutter/issues/85160 +// Fails with "flutter test --test-randomize-ordering-seed=4281596210" +@Tags(['no-shuffle']) + import 'package:flutter/foundation.dart'; import 'package:flutter/scheduler.dart'; import 'package:flutter/services.dart'; diff --git a/packages/flutter/test/services/platform_channel_test.dart b/packages/flutter/test/services/platform_channel_test.dart index a97fa9bb1f..98ec187f4f 100644 --- a/packages/flutter/test/services/platform_channel_test.dart +++ b/packages/flutter/test/services/platform_channel_test.dart @@ -2,6 +2,12 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. +// TODO(gspencergoog): Remove this tag once this test's state leaks/test +// dependencies have been fixed. +// https://github.com/flutter/flutter/issues/85160 +// Fails with "flutter test --test-randomize-ordering-seed=456" +@Tags(['no-shuffle']) + import 'package:flutter/services.dart'; import 'package:flutter_test/flutter_test.dart'; diff --git a/packages/flutter/test/widgets/app_overrides_test.dart b/packages/flutter/test/widgets/app_overrides_test.dart index 9599ca2516..7eae8e98fb 100644 --- a/packages/flutter/test/widgets/app_overrides_test.dart +++ b/packages/flutter/test/widgets/app_overrides_test.dart @@ -2,6 +2,12 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. +// TODO(gspencergoog): Remove this tag once this test's state leaks/test +// dependencies have been fixed. +// https://github.com/flutter/flutter/issues/85160 +// Fails with "flutter test --test-randomize-ordering-seed=123" +@Tags(['no-shuffle']) + import 'package:flutter/widgets.dart'; import 'package:flutter_test/flutter_test.dart'; diff --git a/packages/flutter/test/widgets/binding_test.dart b/packages/flutter/test/widgets/binding_test.dart index db64af8249..9099ced5f0 100644 --- a/packages/flutter/test/widgets/binding_test.dart +++ b/packages/flutter/test/widgets/binding_test.dart @@ -2,6 +2,12 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. +// TODO(gspencergoog): Remove this tag once this test's state leaks/test +// dependencies have been fixed. +// https://github.com/flutter/flutter/issues/85160 +// Fails with "flutter test --test-randomize-ordering-seed=382757700" +@Tags(['no-shuffle']) + import 'package:flutter/foundation.dart'; import 'package:flutter/services.dart'; import 'package:flutter/widgets.dart'; diff --git a/packages/flutter/test/widgets/clip_test.dart b/packages/flutter/test/widgets/clip_test.dart index 0d1f306d15..4086efd0b7 100644 --- a/packages/flutter/test/widgets/clip_test.dart +++ b/packages/flutter/test/widgets/clip_test.dart @@ -2,6 +2,12 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. +// TODO(gspencergoog): Remove this tag once this test's state leaks/test +// dependencies have been fixed. +// https://github.com/flutter/flutter/issues/85160 +// Fails with "flutter test --test-randomize-ordering-seed=456" +@Tags(['no-shuffle']) + import 'package:flutter/material.dart'; import 'package:flutter/rendering.dart'; import 'package:flutter_test/flutter_test.dart'; diff --git a/packages/flutter/test/widgets/debug_test.dart b/packages/flutter/test/widgets/debug_test.dart index 888b0799f0..127996f591 100644 --- a/packages/flutter/test/widgets/debug_test.dart +++ b/packages/flutter/test/widgets/debug_test.dart @@ -2,6 +2,12 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. +// TODO(gspencergoog): Remove this tag once this test's state leaks/test +// dependencies have been fixed. +// https://github.com/flutter/flutter/issues/85160 +// Fails with "flutter test --test-randomize-ordering-seed=123" +@Tags(['no-shuffle']) + import 'package:flutter/foundation.dart'; import 'package:flutter/widgets.dart'; import 'package:flutter_test/flutter_test.dart'; diff --git a/packages/flutter/test/widgets/dismissible_test.dart b/packages/flutter/test/widgets/dismissible_test.dart index e8b5582449..30b255dc07 100644 --- a/packages/flutter/test/widgets/dismissible_test.dart +++ b/packages/flutter/test/widgets/dismissible_test.dart @@ -2,6 +2,12 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. +// TODO(gspencergoog): Remove this tag once this test's state leaks/test +// dependencies have been fixed. +// https://github.com/flutter/flutter/issues/85160 +// Fails with "flutter test --test-randomize-ordering-seed=456" +@Tags(['no-shuffle']) + import 'package:flutter/gestures.dart' show DragStartBehavior; import 'package:flutter/widgets.dart'; import 'package:flutter_test/flutter_test.dart'; diff --git a/packages/flutter/test/widgets/draggable_test.dart b/packages/flutter/test/widgets/draggable_test.dart index ef50d705c2..63df639624 100644 --- a/packages/flutter/test/widgets/draggable_test.dart +++ b/packages/flutter/test/widgets/draggable_test.dart @@ -2,6 +2,12 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. +// TODO(gspencergoog): Remove this tag once this test's state leaks/test +// dependencies have been fixed. +// https://github.com/flutter/flutter/issues/85160 +// Fails with "flutter test --test-randomize-ordering-seed=123" +@Tags(['no-shuffle']) + import 'package:flutter/gestures.dart'; import 'package:flutter/material.dart'; import 'package:flutter/semantics.dart'; diff --git a/packages/flutter/test/widgets/platform_view_test.dart b/packages/flutter/test/widgets/platform_view_test.dart index d3cfbce40d..4c30efa2de 100644 --- a/packages/flutter/test/widgets/platform_view_test.dart +++ b/packages/flutter/test/widgets/platform_view_test.dart @@ -3,6 +3,12 @@ // found in the LICENSE file. @TestOn('!chrome') +// TODO(gspencergoog): Remove this tag once this test's state leaks/test +// dependencies have been fixed. +// https://github.com/flutter/flutter/issues/85160 +// Fails with "flutter test --test-randomize-ordering-seed=4281596210" +@Tags(['no-shuffle']) + import 'dart:async'; import 'dart:typed_data'; import 'dart:ui'; diff --git a/packages/flutter/test/widgets/routes_test.dart b/packages/flutter/test/widgets/routes_test.dart index d74dc4f7e9..9699948b81 100644 --- a/packages/flutter/test/widgets/routes_test.dart +++ b/packages/flutter/test/widgets/routes_test.dart @@ -2,6 +2,12 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. +// TODO(gspencergoog): Remove this tag once this test's state leaks/test +// dependencies have been fixed. +// https://github.com/flutter/flutter/issues/85160 +// Fails with "flutter test --test-randomize-ordering-seed=123" +@Tags(['no-shuffle']) + import 'dart:collection'; import 'package:flutter/material.dart'; diff --git a/packages/flutter/test/widgets/scroll_aware_image_provider_test.dart b/packages/flutter/test/widgets/scroll_aware_image_provider_test.dart index 954d1e84b4..e3bfa02b27 100644 --- a/packages/flutter/test/widgets/scroll_aware_image_provider_test.dart +++ b/packages/flutter/test/widgets/scroll_aware_image_provider_test.dart @@ -2,6 +2,12 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. +// TODO(gspencergoog): Remove this tag once this test's state leaks/test +// dependencies have been fixed. +// https://github.com/flutter/flutter/issues/85160 +// Fails with "flutter test --test-randomize-ordering-seed=123" +@Tags(['no-shuffle']) + import 'dart:ui' as ui show Image; import 'package:flutter/widgets.dart'; diff --git a/packages/flutter/test/widgets/widget_inspector_test.dart b/packages/flutter/test/widgets/widget_inspector_test.dart index 1e167ac980..d4e21e475f 100644 --- a/packages/flutter/test/widgets/widget_inspector_test.dart +++ b/packages/flutter/test/widgets/widget_inspector_test.dart @@ -3,6 +3,13 @@ // found in the LICENSE file. @TestOn('!chrome') + +// TODO(gspencergoog): Remove this tag once this test's state leaks/test +// dependencies have been fixed. +// https://github.com/flutter/flutter/issues/85160 +// Fails with "flutter test --test-randomize-ordering-seed=456" +@Tags(['no-shuffle']) + import 'dart:async'; import 'dart:convert'; import 'dart:math'; @@ -1992,7 +1999,7 @@ class _TestWidgetInspectorService extends TestWidgetInspectorService { _CreationLocation location = knownLocations[id]!; expect(location.file, equals(file)); // ClockText widget. - expect(location.line, equals(53)); + expect(location.line, equals(60)); expect(location.column, equals(9)); expect(location.name, equals('ClockText')); expect(count, equals(1)); @@ -2002,7 +2009,7 @@ class _TestWidgetInspectorService extends TestWidgetInspectorService { location = knownLocations[id]!; expect(location.file, equals(file)); // Text widget in _ClockTextState build method. - expect(location.line, equals(91)); + expect(location.line, equals(98)); expect(location.column, equals(12)); expect(location.name, equals('Text')); expect(count, equals(1)); @@ -2029,7 +2036,7 @@ class _TestWidgetInspectorService extends TestWidgetInspectorService { location = knownLocations[id]!; expect(location.file, equals(file)); // ClockText widget. - expect(location.line, equals(53)); + expect(location.line, equals(60)); expect(location.column, equals(9)); expect(location.name, equals('ClockText')); expect(count, equals(3)); // 3 clock widget instances rebuilt. @@ -2039,7 +2046,7 @@ class _TestWidgetInspectorService extends TestWidgetInspectorService { location = knownLocations[id]!; expect(location.file, equals(file)); // Text widget in _ClockTextState build method. - expect(location.line, equals(91)); + expect(location.line, equals(98)); expect(location.column, equals(12)); expect(location.name, equals('Text')); expect(count, equals(3)); // 3 clock widget instances rebuilt. diff --git a/packages/flutter_driver/dart_test.yaml b/packages/flutter_driver/dart_test.yaml new file mode 100644 index 0000000000..fb3b8a21d3 --- /dev/null +++ b/packages/flutter_driver/dart_test.yaml @@ -0,0 +1,5 @@ +tags: + # This tag tells the test framework to not shuffle the test order according to + # the --test-randomize-ordering-seed for the suites that have this tag. + no-shuffle: + allow_test_randomization: false diff --git a/packages/flutter_driver/test/src/real_tests/extension_test.dart b/packages/flutter_driver/test/src/real_tests/extension_test.dart index 966c3141c7..b3d7a83bb4 100644 --- a/packages/flutter_driver/test/src/real_tests/extension_test.dart +++ b/packages/flutter_driver/test/src/real_tests/extension_test.dart @@ -2,6 +2,12 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. +// TODO(gspencergoog): Remove this tag once this test's state leaks/test +// dependencies have been fixed. +// https://github.com/flutter/flutter/issues/85160 +// Fails with "flutter test --test-randomize-ordering-seed=20210721" +@Tags(['no-shuffle']) + import 'package:flutter/foundation.dart'; import 'package:flutter/material.dart'; import 'package:flutter/rendering.dart'; diff --git a/packages/flutter_driver/test/src/real_tests/flutter_driver_test.dart b/packages/flutter_driver/test/src/real_tests/flutter_driver_test.dart index 3125efa339..7dd1e3a8b1 100644 --- a/packages/flutter_driver/test/src/real_tests/flutter_driver_test.dart +++ b/packages/flutter_driver/test/src/real_tests/flutter_driver_test.dart @@ -2,6 +2,12 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. +// TODO(gspencergoog): Remove this tag once this test's state leaks/test +// dependencies have been fixed. +// https://github.com/flutter/flutter/issues/85160 +// Fails with "flutter test --test-randomize-ordering-seed=20210721" +@Tags(['no-shuffle']) + import 'dart:async'; import 'dart:convert'; import 'dart:io'; diff --git a/packages/flutter_goldens/dart_test.yaml b/packages/flutter_goldens/dart_test.yaml new file mode 100644 index 0000000000..fb3b8a21d3 --- /dev/null +++ b/packages/flutter_goldens/dart_test.yaml @@ -0,0 +1,5 @@ +tags: + # This tag tells the test framework to not shuffle the test order according to + # the --test-randomize-ordering-seed for the suites that have this tag. + no-shuffle: + allow_test_randomization: false diff --git a/packages/flutter_goldens/test/flutter_goldens_test.dart b/packages/flutter_goldens/test/flutter_goldens_test.dart index f81fa37223..74468da9b0 100644 --- a/packages/flutter_goldens/test/flutter_goldens_test.dart +++ b/packages/flutter_goldens/test/flutter_goldens_test.dart @@ -2,6 +2,12 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. +// TODO(gspencergoog): Remove this tag once this test's state leaks/test +// dependencies have been fixed. +// https://github.com/flutter/flutter/issues/85160 +// Fails with "flutter test --test-randomize-ordering-seed=123" +@Tags(['no-shuffle']) + import 'dart:async'; import 'dart:convert'; import 'dart:core'; diff --git a/packages/flutter_test/dart_test.yaml b/packages/flutter_test/dart_test.yaml new file mode 100644 index 0000000000..fb3b8a21d3 --- /dev/null +++ b/packages/flutter_test/dart_test.yaml @@ -0,0 +1,5 @@ +tags: + # This tag tells the test framework to not shuffle the test order according to + # the --test-randomize-ordering-seed for the suites that have this tag. + no-shuffle: + allow_test_randomization: false diff --git a/packages/flutter_test/test/bindings_test.dart b/packages/flutter_test/test/bindings_test.dart index bc98524c79..c5a4b923db 100644 --- a/packages/flutter_test/test/bindings_test.dart +++ b/packages/flutter_test/test/bindings_test.dart @@ -2,6 +2,12 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. +// TODO(gspencergoog): Remove this tag once this test's state leaks/test +// dependencies have been fixed. +// https://github.com/flutter/flutter/issues/85160 +// Fails with "flutter test --test-randomize-ordering-seed=20210721" +@Tags(['no-shuffle']) + import 'dart:io'; import 'package:flutter/widgets.dart'; diff --git a/packages/flutter_test/test/test_text_input_test.dart b/packages/flutter_test/test/test_text_input_test.dart index f47f71033c..20e2d68322 100644 --- a/packages/flutter_test/test/test_text_input_test.dart +++ b/packages/flutter_test/test/test_text_input_test.dart @@ -2,6 +2,12 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. +// TODO(gspencergoog): Remove this tag once this test's state leaks/test +// dependencies have been fixed. +// https://github.com/flutter/flutter/issues/85160 +// Fails with "flutter test --test-randomize-ordering-seed=20210721" +@Tags(['no-shuffle']) + import 'package:flutter/material.dart'; import 'package:flutter/services.dart'; import 'package:flutter_test/flutter_test.dart'; diff --git a/packages/flutter_test/test/window_test.dart b/packages/flutter_test/test/window_test.dart index 56a283c440..872297e5e9 100644 --- a/packages/flutter_test/test/window_test.dart +++ b/packages/flutter_test/test/window_test.dart @@ -2,6 +2,12 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. +// TODO(gspencergoog): Remove this tag once this test's state leaks/test +// dependencies have been fixed. +// https://github.com/flutter/flutter/issues/85160 +// Fails with "flutter test --test-randomize-ordering-seed=123" +@Tags(['no-shuffle']) + import 'dart:ui' as ui show window; import 'dart:ui' show Size, Locale, WindowPadding, AccessibilityFeatures, Brightness; diff --git a/packages/flutter_tools/dart_test.yaml b/packages/flutter_tools/dart_test.yaml index 789a255466..807c2988ee 100644 --- a/packages/flutter_tools/dart_test.yaml +++ b/packages/flutter_tools/dart_test.yaml @@ -7,3 +7,9 @@ # significantly more than it would be by default, and we never set the # timeouts in the tests themselves. timeout: 15m + +tags: + # This tag tells the test framework to not shuffle the test order according to + # the --test-randomize-ordering-seed for the suites that have this tag. + no-shuffle: + allow_test_randomization: false diff --git a/packages/flutter_tools/test/commands.shard/hermetic/build_ios_framework_test.dart b/packages/flutter_tools/test/commands.shard/hermetic/build_ios_framework_test.dart index 0532604fb5..d731bb3fda 100644 --- a/packages/flutter_tools/test/commands.shard/hermetic/build_ios_framework_test.dart +++ b/packages/flutter_tools/test/commands.shard/hermetic/build_ios_framework_test.dart @@ -4,6 +4,12 @@ // @dart = 2.8 +// TODO(gspencergoog): Remove this tag once this test's state leaks/test +// dependencies have been fixed. +// https://github.com/flutter/flutter/issues/85160 +// Fails with "flutter test --test-randomize-ordering-seed=1000" +@Tags(['no-shuffle']) + import 'package:file/memory.dart'; import 'package:flutter_tools/src/base/file_system.dart'; import 'package:flutter_tools/src/base/platform.dart'; diff --git a/packages/flutter_tools/test/commands.shard/hermetic/daemon_test.dart b/packages/flutter_tools/test/commands.shard/hermetic/daemon_test.dart index 028cba970e..85ac67ca65 100644 --- a/packages/flutter_tools/test/commands.shard/hermetic/daemon_test.dart +++ b/packages/flutter_tools/test/commands.shard/hermetic/daemon_test.dart @@ -4,6 +4,12 @@ // @dart = 2.8 +// TODO(gspencergoog): Remove this tag once this test's state leaks/test +// dependencies have been fixed. +// https://github.com/flutter/flutter/issues/85160 +// Fails with "flutter test --test-randomize-ordering-seed=1000" +@Tags(['no-shuffle']) + import 'dart:async'; import 'package:fake_async/fake_async.dart'; diff --git a/packages/flutter_tools/test/commands.shard/hermetic/doctor_test.dart b/packages/flutter_tools/test/commands.shard/hermetic/doctor_test.dart index edad331c6c..ec4ce073e8 100644 --- a/packages/flutter_tools/test/commands.shard/hermetic/doctor_test.dart +++ b/packages/flutter_tools/test/commands.shard/hermetic/doctor_test.dart @@ -4,6 +4,12 @@ // @dart = 2.8 +// TODO(gspencergoog): Remove this tag once this test's state leaks/test +// dependencies have been fixed. +// https://github.com/flutter/flutter/issues/85160 +// Fails with "flutter test --test-randomize-ordering-seed=20210723" +@Tags(['no-shuffle']) + import 'dart:async'; import 'package:args/command_runner.dart'; diff --git a/packages/flutter_tools/test/commands.shard/hermetic/run_test.dart b/packages/flutter_tools/test/commands.shard/hermetic/run_test.dart index ff17e1f937..46d19d53f8 100644 --- a/packages/flutter_tools/test/commands.shard/hermetic/run_test.dart +++ b/packages/flutter_tools/test/commands.shard/hermetic/run_test.dart @@ -4,6 +4,12 @@ // @dart = 2.8 +// TODO(gspencergoog): Remove this tag once this test's state leaks/test +// dependencies have been fixed. +// https://github.com/flutter/flutter/issues/85160 +// Fails with "flutter test --test-randomize-ordering-seed=1000" +@Tags(['no-shuffle']) + import 'dart:async'; import 'package:file/file.dart'; diff --git a/packages/flutter_tools/test/commands.shard/permeable/build_bundle_test.dart b/packages/flutter_tools/test/commands.shard/permeable/build_bundle_test.dart index e5d90df27f..c725e32aba 100644 --- a/packages/flutter_tools/test/commands.shard/permeable/build_bundle_test.dart +++ b/packages/flutter_tools/test/commands.shard/permeable/build_bundle_test.dart @@ -4,6 +4,12 @@ // @dart = 2.8 +// TODO(gspencergoog): Remove this tag once this test's state leaks/test +// dependencies have been fixed. +// https://github.com/flutter/flutter/issues/85160 +// Fails with "flutter test --test-randomize-ordering-seed=20210722" +@Tags(['no-shuffle']) + import 'package:args/command_runner.dart'; import 'package:file/memory.dart'; import 'package:flutter_tools/src/base/file_system.dart'; diff --git a/packages/flutter_tools/test/commands.shard/permeable/packages_test.dart b/packages/flutter_tools/test/commands.shard/permeable/packages_test.dart index 99ad383d2f..dad2a43970 100644 --- a/packages/flutter_tools/test/commands.shard/permeable/packages_test.dart +++ b/packages/flutter_tools/test/commands.shard/permeable/packages_test.dart @@ -4,6 +4,12 @@ // @dart = 2.8 +// TODO(gspencergoog): Remove this tag once this test's state leaks/test +// dependencies have been fixed. +// https://github.com/flutter/flutter/issues/85160 +// Fails with "flutter test --test-randomize-ordering-seed=1000" +@Tags(['no-shuffle']) + import 'dart:async'; import 'dart:convert'; diff --git a/packages/flutter_tools/test/general.shard/build_system/targets/dart_plugin_registrant_test.dart b/packages/flutter_tools/test/general.shard/build_system/targets/dart_plugin_registrant_test.dart index 89f3253743..4806d2b6e3 100644 --- a/packages/flutter_tools/test/general.shard/build_system/targets/dart_plugin_registrant_test.dart +++ b/packages/flutter_tools/test/general.shard/build_system/targets/dart_plugin_registrant_test.dart @@ -4,6 +4,12 @@ // @dart = 2.8 +// TODO(gspencergoog): Remove this tag once this test's state leaks/test +// dependencies have been fixed. +// https://github.com/flutter/flutter/issues/85160 +// Fails with "flutter test --test-randomize-ordering-seed=20210722" +@Tags(['no-shuffle']) + import 'package:file/memory.dart'; import 'package:flutter_tools/src/base/file_system.dart'; import 'package:flutter_tools/src/base/logger.dart'; diff --git a/packages/flutter_tools/test/general.shard/channel_test.dart b/packages/flutter_tools/test/general.shard/channel_test.dart index 33e1886f4c..d5cdbedd25 100644 --- a/packages/flutter_tools/test/general.shard/channel_test.dart +++ b/packages/flutter_tools/test/general.shard/channel_test.dart @@ -4,6 +4,12 @@ // @dart = 2.8 +// TODO(gspencergoog): Remove this tag once this test's state leaks/test +// dependencies have been fixed. +// https://github.com/flutter/flutter/issues/85160 +// Fails with "flutter test --test-randomize-ordering-seed=20210723" +@Tags(['no-shuffle']) + import 'package:args/command_runner.dart'; import 'package:file/memory.dart'; import 'package:flutter_tools/src/base/file_system.dart'; diff --git a/packages/flutter_tools/test/general.shard/dart/language_version_test.dart b/packages/flutter_tools/test/general.shard/dart/language_version_test.dart index 0d5f25a984..c974e821dd 100644 --- a/packages/flutter_tools/test/general.shard/dart/language_version_test.dart +++ b/packages/flutter_tools/test/general.shard/dart/language_version_test.dart @@ -2,6 +2,12 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. +// TODO(gspencergoog): Remove this tag once this test's state leaks/test +// dependencies have been fixed. +// https://github.com/flutter/flutter/issues/85160 +// Fails with "flutter test --test-randomize-ordering-seed=456" +@Tags(['no-shuffle']) + import 'package:file/memory.dart'; import 'package:flutter_tools/src/base/file_system.dart'; import 'package:flutter_tools/src/dart/language_version.dart'; diff --git a/packages/flutter_tools/test/integration.shard/break_on_framework_exceptions_test.dart b/packages/flutter_tools/test/integration.shard/break_on_framework_exceptions_test.dart index bd6a6fc6ea..1ad607c47b 100644 --- a/packages/flutter_tools/test/integration.shard/break_on_framework_exceptions_test.dart +++ b/packages/flutter_tools/test/integration.shard/break_on_framework_exceptions_test.dart @@ -4,6 +4,12 @@ // @dart = 2.8 +// TODO(gspencergoog): Remove this tag once this test's state leaks/test +// dependencies have been fixed. +// https://github.com/flutter/flutter/issues/85160 +// Fails with "flutter test --test-randomize-ordering-seed=1000" +@Tags(['no-shuffle']) + import 'package:file/file.dart'; import '../src/common.dart'; diff --git a/packages/flutter_tools/test/integration.shard/overall_experience_test.dart b/packages/flutter_tools/test/integration.shard/overall_experience_test.dart index 3f0778f7de..5c2a9434d8 100644 --- a/packages/flutter_tools/test/integration.shard/overall_experience_test.dart +++ b/packages/flutter_tools/test/integration.shard/overall_experience_test.dart @@ -25,6 +25,12 @@ // @dart = 2.8 // This file is ready to transition, just uncomment /*?*/, /*!*/, and /*late*/. +// TODO(gspencergoog): Remove this tag once this test's state leaks/test +// dependencies have been fixed. +// https://github.com/flutter/flutter/issues/85160 +// Fails with "flutter test --test-randomize-ordering-seed=1000" +@Tags(['no-shuffle']) + import 'dart:async'; import 'dart:convert'; import 'dart:io'; diff --git a/packages/flutter_tools/test/integration.shard/test_test.dart b/packages/flutter_tools/test/integration.shard/test_test.dart index 0251128210..bb2aa3751f 100644 --- a/packages/flutter_tools/test/integration.shard/test_test.dart +++ b/packages/flutter_tools/test/integration.shard/test_test.dart @@ -4,6 +4,12 @@ // @dart = 2.8 +// TODO(gspencergoog): Remove this tag once this test's state leaks/test +// dependencies have been fixed. +// https://github.com/flutter/flutter/issues/85160 +// Fails with "flutter test --test-randomize-ordering-seed=1000" +@Tags(['no-shuffle']) + import 'dart:async'; import 'dart:convert';