firka/flutter
firka/flutter
5
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Run flutter tests through mini test engine when run directly (flutter run -t test_file) (#24930)

Browse Source
This commit is contained in:
Jonah Williams 2018-12-12 12:51:16 -08:00 committed by GitHub
parent f198d66332
commit f30029bae1
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 619 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
dev/automated_tests/flutter_run_test/flutter_run_test.dart Normal file
View File

@ -0,0 +1,19 @@
// Copyright 2018 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
import 'package:flutter_test/flutter_test.dart';
void main() {
group('example', () {
test('passed', () {
expect(true, true);
});
test('failed', () {
expect(true, false);
});
test('skipped', () {
expect(true, false);
}, skip: true);
});
}

67
dev/devicelab/bin/tasks/flutter_run_test.dart Normal file
View File

@ -0,0 +1,67 @@
// Copyright 2018 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
import 'dart:async';
import 'dart:io';
import 'package:path/path.dart' as path;
import 'package:flutter_devicelab/framework/adb.dart';
import 'package:flutter_devicelab/framework/framework.dart';
import 'package:flutter_devicelab/framework/utils.dart';
final Directory flutterGalleryDir = dir(path.join(flutterDirectory.path, 'examples/hello_world'));
final File runTestSource = File(path.join(
flutterDirectory.path, 'dev', 'automated_tests', 'flutter_run_test', 'flutter_run_test.dart',
));
const Pattern passedMessageMatch = '+0: example passed';
const Pattern failedMessageMatch = '+1: example failed [E]';
const Pattern skippedMessageMatch = '+1 -1: example skipped';
const Pattern finishedMessageMatch = '+1 ~1 -1: Some tests failed.';
Future<void> main() async {
deviceOperatingSystem = DeviceOperatingSystem.android;
await task(createFlutterRunTask);
}
// verifies that the messages above are printed as a test script is executed.
Future<TaskResult> createFlutterRunTask() async {
bool passedTest = false;
bool failedTest = false;
bool skippedTest = false;
bool finishedMessage = false;
final Device device = await devices.workingDevice;
await device.unlock();
final List<String> options = <String>[
'-t', runTestSource.absolute.path, '-d', device.deviceId,
];
setLocalEngineOptionIfNecessary(options);
await inDirectory<void>(flutterGalleryDir, () async {
startProcess(
path.join(flutterDirectory.path, 'bin', 'flutter'),
<String>['run']..addAll(options),
environment: null
);
final Completer<void> finished = Completer<void>();
final StreamSubscription<void> subscription = device.logcat.listen((String line) {
// tests execute in order.
if (line.contains(passedMessageMatch)) {
passedTest = true;
} else if (line.contains(failedMessageMatch)) {
failedTest = true;
} else if (line.contains(skippedMessageMatch)) {
skippedTest = true;
} else if (line.contains(finishedMessageMatch)) {
finishedMessage = true;
finished.complete();
}
});
await finished.future.timeout(const Duration(minutes: 1));
subscription.cancel();
});
return passedTest && failedTest && skippedTest && finishedMessage
? TaskResult.success(<String, dynamic>{})
: TaskResult.failure('Test did not execute as expected.');
}

7
dev/devicelab/manifest.yaml
View File

@ -283,6 +283,13 @@ tasks:
stage: devicelab
required_agent_capabilities: ["linux/android"]
flutter_run_test:
description: >
Tests the `flutter run -t` command with a testfile.
stage: devicelab
required_agent_capabilities: ["linux/android"]
flaky: true
named_isolates_test:
description: >
Tests naming and attaching to specific isolates.

1
packages/flutter_test/lib/flutter_test.dart
View File

@ -56,6 +56,7 @@ export 'src/matchers.dart';
export 'src/nonconst.dart';
export 'src/stack_manipulation.dart';
export 'src/test_async_utils.dart';
export 'src/test_compat.dart';
export 'src/test_exception_reporter.dart';
export 'src/test_pointer.dart';
export 'src/test_text_input.dart';

514
packages/flutter_test/lib/src/test_compat.dart Normal file
View File

@ -0,0 +1,514 @@
// Copyright 2018 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
import 'dart:async';
import 'package:test_api/src/backend/declarer.dart'; // ignore: implementation_imports
import 'package:test_api/src/frontend/timeout.dart'; // ignore: implementation_imports
import 'package:test_api/src/backend/group.dart'; // ignore: implementation_imports
import 'package:test_api/src/backend/group_entry.dart'; // ignore: implementation_imports
import 'package:test_api/src/backend/test.dart'; // ignore: implementation_imports
import 'package:test_api/src/backend/suite.dart'; // ignore: implementation_imports
import 'package:test_api/src/backend/live_test.dart'; // ignore: implementation_imports
import 'package:test_api/src/backend/suite_platform.dart'; // ignore: implementation_imports
import 'package:test_api/src/backend/runtime.dart'; // ignore: implementation_imports
import 'package:test_api/src/backend/message.dart'; // ignore: implementation_imports
import 'package:test_api/src/backend/invoker.dart'; // ignore: implementation_imports
import 'package:test_api/src/backend/state.dart'; // ignore: implementation_imports
import 'package:test_api/test_api.dart';
Declarer _localDeclarer;
Declarer get _declarer {
final Declarer declarer = Zone.current[#test.declarer];
if (declarer != null) {
return declarer;
}
// If no declarer is defined, this test is being run via `flutter run -t test_file.dart`.
if (_localDeclarer == null) {
_localDeclarer = Declarer();
Future<void>(() {
Invoker.guard<Future<void>>(() async {
final _Reporter reporter = _Reporter(color: false); // disable color when run directly.
final Group group = _declarer.build();
final Suite suite = Suite(group, SuitePlatform(Runtime.vm));
await _runGroup(suite, group, <Group>[], reporter);
reporter._onDone();
});
});
}
return _localDeclarer;
}
Future<void> _runGroup(Suite suiteConfig, Group group, List<Group> parents, _Reporter reporter) async {
parents.add(group);
try {
final bool skipGroup = group.metadata.skip;
bool setUpAllSucceeded = true;
if (!skipGroup && group.setUpAll != null) {
final LiveTest liveTest = group.setUpAll.load(suiteConfig, groups: parents);
await _runLiveTest(suiteConfig, liveTest, reporter, countSuccess: false);
setUpAllSucceeded = liveTest.state.result.isPassing;
}
if (setUpAllSucceeded) {
for (GroupEntry entry in group.entries) {
if (entry is Group) {
await _runGroup(suiteConfig, entry, parents, reporter);
} else if (entry.metadata.skip) {
await _runSkippedTest(suiteConfig, entry, parents, reporter);
} else {
final Test test = entry;
await _runLiveTest(suiteConfig, test.load(suiteConfig, groups: parents), reporter);
}
}
}
// Even if we're closed or setUpAll failed, we want to run all the
// teardowns to ensure that any state is properly cleaned up.
if (!skipGroup && group.tearDownAll != null) {
final LiveTest liveTest = group.tearDownAll.load(suiteConfig, groups: parents);
await _runLiveTest(suiteConfig, liveTest, reporter, countSuccess: false);
}
} finally {
parents.remove(group);
}
}
Future<void> _runLiveTest(Suite suiteConfig, LiveTest liveTest, _Reporter reporter, {bool countSuccess = true}) async {
reporter._onTestStarted(liveTest);
// Schedule a microtask to ensure that [onTestStarted] fires before the
// first [LiveTest.onStateChange] event.
await Future<void>.microtask(liveTest.run);
// Once the test finishes, use await null to do a coarse-grained event
// loop pump to avoid starving non-microtask events.
await null;
final bool isSuccess = liveTest.state.result.isPassing;
if (isSuccess) {
reporter.passed.add(liveTest);
} else {
reporter.failed.add(liveTest);
}
}
Future<void> _runSkippedTest(Suite suiteConfig, Test test, List<Group> parents, _Reporter reporter) async {
final LocalTest skipped = LocalTest(test.name, test.metadata, () {}, trace: test.trace);
if (skipped.metadata.skipReason != null) {
print('Skip: ${skipped.metadata.skipReason}');
}
final LiveTest liveTest = skipped.load(suiteConfig);
reporter._onTestStarted(liveTest);
reporter.skipped.add(skipped);
}
// TODO(nweiz): This and other top-level functions should throw exceptions if
// they're called after the declarer has finished declaring.
/// Creates a new test case with the given description (converted to a string)
/// and body.
///
/// The description will be added to the descriptions of any surrounding
/// [group]s. If [testOn] is passed, it's parsed as a [platform selector][]; the
/// test will only be run on matching platforms.
///
/// [platform selector]: https://github.com/dart-lang/test/tree/master/pkgs/test#platform-selectors
///
/// If [timeout] is passed, it's used to modify or replace the default timeout
/// of 30 seconds. Timeout modifications take precedence in suite-group-test
/// order, so [timeout] will also modify any timeouts set on the group or suite.
///
/// If [skip] is a String or `true`, the test is skipped. If it's a String, it
/// should explain why the test is skipped; this reason will be printed instead
/// of running the test.
///
/// If [tags] is passed, it declares user-defined tags that are applied to the
/// test. These tags can be used to select or skip the test on the command line,
/// or to do bulk test configuration. All tags should be declared in the
/// [package configuration file][configuring tags]. The parameter can be an
/// [Iterable] of tag names, or a [String] representing a single tag.
///
/// If [retry] is passed, the test will be retried the provided number of times
/// before being marked as a failure.
///
/// [configuring tags]: https://github.com/dart-lang/test/blob/master/doc/package_config.md#configuring-tags
///
/// [onPlatform] allows tests to be configured on a platform-by-platform
/// basis. It's a map from strings that are parsed as [PlatformSelector]s to
/// annotation classes: [Timeout], [Skip], or lists of those. These
/// annotations apply only on the given platforms. For example:
///
/// test('potentially slow test', () {
/// // ...
/// }, onPlatform: {
/// // This test is especially slow on Windows.
/// 'windows': new Timeout.factor(2),
/// 'browser': [
/// new Skip('TODO: add browser support'),
/// // This will be slow on browsers once it works on them.
/// new Timeout.factor(2)
/// ]
/// });
///
/// If multiple platforms match, the annotations apply in order as through
/// they were in nested groups.
///
/// If the `solo` flag is `true`, only tests and groups marked as
/// 'solo' will be be run. This only restricts tests *within this test
/// suite*—tests in other suites will run as normal. We recommend that users
/// avoid this flag if possible and instead use the test runner flag `-n` to
/// filter tests by name.
void test(Object description, Function body, {
String testOn,
Timeout timeout,
dynamic skip,
dynamic tags,
Map<String, dynamic> onPlatform,
int retry,
}) {
_declarer.test(
description.toString(), body,
testOn: testOn,
timeout: timeout,
skip: skip,
onPlatform: onPlatform,
tags: tags,
retry: retry,
);
}
/// Creates a group of tests.
///
/// A group's description (converted to a string) is included in the descriptions
/// of any tests or sub-groups it contains. [setUp] and [tearDown] are also scoped
/// to the containing group.
///
/// If [testOn] is passed, it's parsed as a [platform selector][]; the test will
/// only be run on matching platforms.
///
/// [platform selector]: https://github.com/dart-lang/test/tree/master/pkgs/test#platform-selectors
///
/// If [timeout] is passed, it's used to modify or replace the default timeout
/// of 30 seconds. Timeout modifications take precedence in suite-group-test
/// order, so [timeout] will also modify any timeouts set on the suite, and will
/// be modified by any timeouts set on individual tests.
///
/// If [skip] is a String or `true`, the group is skipped. If it's a String, it
/// should explain why the group is skipped; this reason will be printed instead
/// of running the group's tests.
///
/// If [tags] is passed, it declares user-defined tags that are applied to the
/// test. These tags can be used to select or skip the test on the command line,
/// or to do bulk test configuration. All tags should be declared in the
/// [package configuration file][configuring tags]. The parameter can be an
/// [Iterable] of tag names, or a [String] representing a single tag.
///
/// [configuring tags]: https://github.com/dart-lang/test/blob/master/doc/package_config.md#configuring-tags
///
/// [onPlatform] allows groups to be configured on a platform-by-platform
/// basis. It's a map from strings that are parsed as [PlatformSelector]s to
/// annotation classes: [Timeout], [Skip], or lists of those. These
/// annotations apply only on the given platforms. For example:
///
/// group('potentially slow tests', () {
/// // ...
/// }, onPlatform: {
/// // These tests are especially slow on Windows.
/// 'windows': new Timeout.factor(2),
/// 'browser': [
/// new Skip('TODO: add browser support'),
/// // They'll be slow on browsers once it works on them.
/// new Timeout.factor(2)
/// ]
/// });
///
/// If multiple platforms match, the annotations apply in order as through
/// they were in nested groups.
///
/// If the `solo` flag is `true`, only tests and groups marked as
/// 'solo' will be be run. This only restricts tests *within this test
/// suite*—tests in other suites will run as normal. We recommend that users
/// avoid this flag if possible, and instead use the test runner flag `-n` to
/// filter tests by name.
void group(Object description, Function body) {
_declarer.group(description.toString(), body);
}
/// Registers a function to be run before tests.
///
/// This function will be called before each test is run. [callback] may be
/// asynchronous; if so, it must return a [Future].
///
/// If this is called within a test group, it applies only to tests in that
/// group. [callback] will be run after any set-up callbacks in parent groups or
/// at the top level.
///
/// Each callback at the top level or in a given group will be run in the order
/// they were declared.
void setUp(Function body) {
_declarer.setUp(body);
}
/// Registers a function to be run after tests.
///
/// This function will be called after each test is run. [callback] may be
/// asynchronous; if so, it must return a [Future].
///
/// If this is called within a test group, it applies only to tests in that
/// group. [callback] will be run before any tear-down callbacks in parent
/// groups or at the top level.
///
/// Each callback at the top level or in a given group will be run in the
/// reverse of the order they were declared.
///
/// See also [addTearDown], which adds tear-downs to a running test.
void tearDown(Function body) {
_declarer.tearDown(body);
}
/// Registers a function to be run once before all tests.
///
/// [callback] may be asynchronous; if so, it must return a [Future].
///
/// If this is called within a test group, [callback] will run before all tests
/// in that group. It will be run after any [setUpAll] callbacks in parent
/// groups or at the top level. It won't be run if none of the tests in the
/// group are run.
///
/// **Note**: This function makes it very easy to accidentally introduce hidden
/// dependencies between tests that should be isolated. In general, you should
/// prefer [setUp], and only use [setUpAll] if the callback is prohibitively
/// slow.
void setUpAll(Function body) {
_declarer.setUpAll(body);
}
/// Registers a function to be run once after all tests.
///
/// If this is called within a test group, [callback] will run after all tests
/// in that group. It will be run before any [tearDownAll] callbacks in parent
/// groups or at the top level. It won't be run if none of the tests in the
/// group are run.
///
/// **Note**: This function makes it very easy to accidentally introduce hidden
/// dependencies between tests that should be isolated. In general, you should
/// prefer [tearDown], and only use [tearDownAll] if the callback is
/// prohibitively slow.
void tearDownAll(Function body) {
_declarer.tearDownAll(body);
}
/// A reporter that prints each test on its own line.
///
/// This is currently used in place of [CompactReporter] by `lib/test.dart`,
/// which can't transitively import `dart:io` but still needs access to a runner
/// so that test files can be run directly. This means that until issue 6943 is
/// fixed, this must not import `dart:io`.
class _Reporter {
_Reporter({bool color = true, bool printPath = true})
: _printPath = printPath,
_green = color ? '\u001b[32m' : '',
_red = color ? '\u001b[31m' : '',
_yellow = color ? '\u001b[33m' : '',
_bold = color ? '\u001b[1m' : '',
_noColor = color ? '\u001b[0m' : '';
final List<LiveTest> passed = <LiveTest>[];
final List<LiveTest> failed = <LiveTest>[];
final List<Test> skipped = <Test>[];
/// The terminal escape for green text, or the empty string if this is Windows
/// or not outputting to a terminal.
final String _green;
/// The terminal escape for red text, or the empty string if this is Windows
/// or not outputting to a terminal.
final String _red;
/// The terminal escape for yellow text, or the empty string if this is
/// Windows or not outputting to a terminal.
final String _yellow;
/// The terminal escape for bold text, or the empty string if this is
/// Windows or not outputting to a terminal.
final String _bold;
/// The terminal escape for removing test coloring, or the empty string if
/// this is Windows or not outputting to a terminal.
final String _noColor;
/// Whether the path to each test's suite should be printed.
final bool _printPath;
/// A stopwatch that tracks the duration of the full run.
final Stopwatch _stopwatch = Stopwatch();
/// The size of `_engine.passed` last time a progress notification was
/// printed.
int _lastProgressPassed;
/// The size of `_engine.skipped` last time a progress notification was
/// printed.
int _lastProgressSkipped;
/// The size of `_engine.failed` last time a progress notification was
/// printed.
int _lastProgressFailed;
/// The message printed for the last progress notification.
String _lastProgressMessage;
/// The suffix added to the last progress notification.
String _lastProgressSuffix;
/// The set of all subscriptions to various streams.
final Set<StreamSubscription<void>> _subscriptions = Set<StreamSubscription<void>>();
/// A callback called when the engine begins running [liveTest].
void _onTestStarted(LiveTest liveTest) {
if (!_stopwatch.isRunning) {
_stopwatch.start();
}
_progressLine(_description(liveTest));
_subscriptions.add(liveTest.onStateChange.listen((State state) => _onStateChange(liveTest, state)));
_subscriptions.add(liveTest.onError.listen((AsyncError error) => _onError(liveTest, error.error, error.stackTrace)));
_subscriptions.add(liveTest.onMessage.listen((Message message) {
_progressLine(_description(liveTest));
String text = message.text;
if (message.type == MessageType.skip) {
text = ' $_yellow$text$_noColor';
}
print(text);
}));
}
/// A callback called when [liveTest]'s state becomes [state].
void _onStateChange(LiveTest liveTest, State state) {
if (state.status != Status.complete) {
return;
}
}
/// A callback called when [liveTest] throws [error].
void _onError(LiveTest liveTest, Object error, StackTrace stackTrace) {
if (liveTest.state.status != Status.complete) {
return;
}
_progressLine(_description(liveTest), suffix: ' $_bold$_red[E]$_noColor');
print(_indent(error.toString()));
print(_indent('$stackTrace'));
}
/// A callback called when the engine is finished running tests.
///
/// [success] will be `true` if all tests passed, `false` if some tests
/// failed, and `null` if the engine was closed prematurely.
void _onDone() {
final bool success = failed.isEmpty;
if (success == null) {
return;
}
if (!success) {
_progressLine('Some tests failed.', color: _red);
} else if (passed.isEmpty) {
_progressLine('All tests skipped.');
} else {
_progressLine('All tests passed!');
}
}
/// Prints a line representing the current state of the tests.
///
/// [message] goes after the progress report. If [color] is passed, it's used
/// as the color for [message]. If [suffix] is passed, it's added to the end
/// of [message].
void _progressLine(String message, {String color, String suffix}) {
// Print nothing if nothing has changed since the last progress line.
if (passed.length == _lastProgressPassed &&
skipped.length == _lastProgressSkipped &&
failed.length == _lastProgressFailed &&
message == _lastProgressMessage &&
// Don't re-print just because a suffix was removed.
(suffix == null || suffix == _lastProgressSuffix)) {
return;
}
_lastProgressPassed = passed.length;
_lastProgressSkipped = skipped.length;
_lastProgressFailed = failed.length;
_lastProgressMessage = message;
_lastProgressSuffix = suffix;
if (suffix != null) {
message += suffix;
}
color ??= '';
final Duration duration = _stopwatch.elapsed;
final StringBuffer buffer = StringBuffer();
// \r moves back to the beginning of the current line.
buffer.write('${_timeString(duration)} ');
buffer.write(_green);
buffer.write('+');
buffer.write(passed.length);
buffer.write(_noColor);
if (skipped.isNotEmpty) {
buffer.write(_yellow);
buffer.write(' ~');
buffer.write(skipped.length);
buffer.write(_noColor);
}
if (failed.isNotEmpty) {
buffer.write(_red);
buffer.write(' -');
buffer.write(failed.length);
buffer.write(_noColor);
}
buffer.write(': ');
buffer.write(color);
buffer.write(message);
buffer.write(_noColor);
print(buffer.toString());
}
/// Returns a representation of [duration] as `MM:SS`.
String _timeString(Duration duration) {
final String minutes = duration.inMinutes.toString().padLeft(2, '0');
final String seconds = (duration.inSeconds % 60).toString().padLeft(2, '0');
return '$minutes:$seconds';
}
/// Returns a description of [liveTest].
///
/// This differs from the test's own description in that it may also include
/// the suite's name.
String _description(LiveTest liveTest) {
String name = liveTest.test.name;
if (_printPath && liveTest.suite.path != null) {
name = '${liveTest.suite.path}: $name';
}
return name;
}
}
String _indent(String string, {int size, String first}) {
size ??= first == null ? 2 : first.length;
return _prefixLines(string, ' ' * size, first: first);
}
String _prefixLines(String text, String prefix, {String first, String last, String single}) {
first ??= prefix;
last ??= prefix;
single ??= first ?? last ?? prefix;
final List<String> lines = text.split('\n');
if (lines.length == 1) {
return '$single$text';
}
final StringBuffer buffer = StringBuffer('$first${lines.first}\n');
// Write out all but the first and last lines with [prefix].
for (String line in lines.skip(1).take(lines.length - 2)) {
buffer.writeln('$prefix$line');
}
buffer.write('$last${lines.last}');
return buffer.toString();
}

12
packages/flutter_test/lib/src/widget_tester.dart
View File

@ -19,12 +19,22 @@ import 'controller.dart';
import 'finders.dart';
import 'matchers.dart';
import 'test_async_utils.dart';
import 'test_compat.dart';
import 'test_text_input.dart';
/// Keep users from needing multiple imports to test semantics.
export 'package:flutter/rendering.dart' show SemanticsHandle;
/// Hide these imports so that they do not conflict with our own implementations in
/// test_compat.dart. This handles setting up a declarer when one is not defined, which
/// can happen when a test is executed via flutter_run.
export 'package:test_api/test_api.dart' hide
test,
group,
setUpAll,
tearDownAll,
setUp,
tearDown,
expect, // we have our own wrapper below
TypeMatcher, // matcher's TypeMatcher conflicts with the one in the Flutter framework
isInstanceOf; // we have our own wrapper in matchers.dart
@ -63,7 +73,7 @@ void testWidgets(String description, WidgetTesterCallback callback, {
final TestWidgetsFlutterBinding binding = TestWidgetsFlutterBinding.ensureInitialized();
final WidgetTester tester = WidgetTester._(binding);
timeout ??= binding.defaultTestTimeout;
test_package.test(
test(
description,
() {
tester._recordNumberOfSemanticsHandles();