move CI tests into the main repo (#5758)
This commit is contained in:
Yegor
GitHub
parent
4ec5144427
commit
1ba1562293
@ -35,6 +35,7 @@ fi
|
||||
(cd packages/flutter_test; flutter test)
|
||||
(cd packages/flutter_tools; dart -c test/all.dart)
|
||||
|
||||
(cd dev/devicelab; dart -c test/all.dart)
|
||||
(cd dev/manual_tests; flutter test)
|
||||
(cd examples/hello_world; flutter test)
|
||||
(cd examples/layers; flutter test)
|
||||
@ -55,4 +56,4 @@ if [ -n "$COVERAGE_FLAG" ]; then
|
||||
fi
|
||||
|
||||
# generate the API docs, upload them
|
||||
dev/bots/docs.sh
|
||||
dev/bots/docs.sh
|
||||
|
||||
9
dev/devicelab/.gitignore
vendored
Normal file
9
dev/devicelab/.gitignore
vendored
Normal file
@ -0,0 +1,9 @@
|
||||
.DS_Store
|
||||
.buildlog
|
||||
.idea
|
||||
.packages
|
||||
.pub/
|
||||
build/
|
||||
packages
|
||||
pubspec.lock
|
||||
.atom/
|
||||
96
dev/devicelab/README.md
Normal file
96
dev/devicelab/README.md
Normal file
@ -0,0 +1,96 @@
|
||||
# Flutter devicelab
|
||||
|
||||
This package contains the test framework and tests that run on physical devices.
|
||||
More generally the tests are referred to as "tasks" in the API, but since we
|
||||
primarily use it for testing, this document refers to them as "tests".
|
||||
|
||||
# Writing tests
|
||||
|
||||
A test is a simple Dart program that lives under `bin/tests` and uses
|
||||
`package:flutter_devicelab/framework/framework.dart` to define and run a _task_.
|
||||
|
||||
Example:
|
||||
|
||||
```dart
|
||||
import 'dart:async';
|
||||
|
||||
import 'package:flutter_devicelab/framework/framework.dart';
|
||||
|
||||
Future<Null> main() async {
|
||||
await task(() async {
|
||||
... do something interesting ...
|
||||
|
||||
// Aggregate results into a JSONable Map structure.
|
||||
Map<String, dynamic> testResults = ...;
|
||||
|
||||
// Report success.
|
||||
return new TaskResult.success(testResults);
|
||||
|
||||
// Or you can also report a failure.
|
||||
return new TaskResult.failure('Something went wrong!');
|
||||
});
|
||||
}
|
||||
```
|
||||
|
||||
Only one `task` is permitted per program. However, that task can run any number
|
||||
of tests internally. A task has a name. It succeeds and fails independently of
|
||||
other tasks, and is reported to the dashboard independently of other tasks.
|
||||
|
||||
A task runs in its own standalone Dart VM and reports results via Dart VM
|
||||
service protocol. This ensures that tasks do not interfere with each other and
|
||||
lets the CI system time out and clean up tasks that get stuck.
|
||||
|
||||
# Adding tests to the CI environment
|
||||
|
||||
The `manifest.yaml` file describes a subset of tests we run in the CI. To add
|
||||
your test edit `manifest.yaml` and add the following in the "tasks" dictionary:
|
||||
|
||||
```
|
||||
{NAME_OF_TEST}:
|
||||
description: {DESCRIPTION}
|
||||
stage: {STAGE}
|
||||
required_agent_capabilities: {CAPABILITIES}
|
||||
```
|
||||
|
||||
Where:
|
||||
|
||||
- `{NAME_OF_TEST}` is the name of your test that also matches the name of the
|
||||
file in `bin/tests` without the `.dart` extension.
|
||||
- `{DESCRIPTION}` is the plain English description of your test that helps
|
||||
others understand what this test is testing.
|
||||
- `{STAGE}` is `devicelab` if you want to run on Android, or `devicelab_ios` if
|
||||
you want to run on iOS.
|
||||
- `{CAPABILITIES}` is an array that lists the capabilities required of
|
||||
the test agent (the computer that runs the test) to run your test. Available
|
||||
capabilities are: `has-android-device`, `has-ios-device`.
|
||||
|
||||
# Running tests locally
|
||||
|
||||
Do make sure your tests pass locally before deploying to the CI environment.
|
||||
Below is a handful of commands that run tests in a fashion very close to how the
|
||||
CI environment runs them. These commands are also useful when you need to
|
||||
reproduce a CI test failure locally.
|
||||
|
||||
To run a test use option `-t` (`--task`):
|
||||
|
||||
```sh
|
||||
dart bin/run.dart -t {NAME_OF_TEST}
|
||||
```
|
||||
|
||||
To run multiple tests repeat option `-t` (`--task`) multiple times:
|
||||
|
||||
```sh
|
||||
dart bin/run.dart -t test1 -t test2 -t test3
|
||||
```
|
||||
|
||||
To run all tests defined in `manifest.yaml` use option `-a` (`--all`):
|
||||
|
||||
```sh
|
||||
dart bin/run.dart -a
|
||||
```
|
||||
|
||||
To run tests from a specific stage use option `-s` (`--stage`):
|
||||
|
||||
```sh
|
||||
dart bin/run.dart -s {NAME_OF_STAGE}
|
||||
```
|
||||
101
dev/devicelab/bin/run.dart
Normal file
101
dev/devicelab/bin/run.dart
Normal file
@ -0,0 +1,101 @@
|
||||
// Copyright 2016 The Chromium Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file.
|
||||
|
||||
import 'dart:async';
|
||||
import 'dart:convert';
|
||||
import 'dart:io';
|
||||
|
||||
import 'package:args/args.dart';
|
||||
|
||||
import 'package:flutter_devicelab/framework/manifest.dart';
|
||||
import 'package:flutter_devicelab/framework/runner.dart';
|
||||
import 'package:flutter_devicelab/framework/utils.dart';
|
||||
|
||||
/// Runs tasks.
|
||||
///
|
||||
/// The tasks are chosen depending on the command-line options
|
||||
/// (see [_argParser]).
|
||||
Future<Null> main(List<String> rawArgs) async {
|
||||
ArgResults args;
|
||||
try {
|
||||
args = _argParser.parse(rawArgs);
|
||||
} on FormatException catch(error) {
|
||||
stderr.writeln('${error.message}\n');
|
||||
stderr.writeln('Usage:\n');
|
||||
stderr.writeln(_argParser.usage);
|
||||
exitCode = 1;
|
||||
return null;
|
||||
}
|
||||
|
||||
List<String> taskNames = <String>[];
|
||||
if (args.wasParsed('task')) {
|
||||
taskNames.addAll(args['task']);
|
||||
} else if (args.wasParsed('stage')) {
|
||||
String stageName = args['stage'];
|
||||
List<ManifestTask> tasks = loadTaskManifest().tasks;
|
||||
for (ManifestTask task in tasks) {
|
||||
if (task.stage == stageName)
|
||||
taskNames.add(task.name);
|
||||
}
|
||||
} else if (args.wasParsed('all')) {
|
||||
List<ManifestTask> tasks = loadTaskManifest().tasks;
|
||||
for (ManifestTask task in tasks) {
|
||||
taskNames.add(task.name);
|
||||
}
|
||||
}
|
||||
|
||||
if (taskNames.isEmpty) {
|
||||
stderr.writeln('Failed to find tasks to run based on supplied options.');
|
||||
exitCode = 1;
|
||||
return null;
|
||||
}
|
||||
|
||||
for (String taskName in taskNames) {
|
||||
section('Running task "$taskName"');
|
||||
Map<String, dynamic> result = await runTask(taskName);
|
||||
|
||||
if (!result['success'])
|
||||
exitCode = 1;
|
||||
|
||||
print('Task result:');
|
||||
print(new JsonEncoder.withIndent(' ').convert(result));
|
||||
section('Finished task "$taskName"');
|
||||
}
|
||||
}
|
||||
|
||||
/// Command-line options for the `run.dart` command.
|
||||
final ArgParser _argParser = new ArgParser()
|
||||
..addOption(
|
||||
'task',
|
||||
abbr: 't',
|
||||
allowMultiple: true,
|
||||
splitCommas: true,
|
||||
help: 'Name of the task to run. This option may be repeated to '
|
||||
'specify multiple tasks. A task selected by name does not have to be '
|
||||
'defined in manifest.yaml. It only needs a Dart executable in bin/tasks.',
|
||||
)
|
||||
..addOption(
|
||||
'stage',
|
||||
abbr: 's',
|
||||
help: 'Name of the stage. Runs all tasks for that stage. '
|
||||
'The tasks and their stages are read from manifest.yaml.',
|
||||
)
|
||||
..addOption(
|
||||
'all',
|
||||
abbr: 'a',
|
||||
help: 'Runs all tasks defined in manifest.yaml.',
|
||||
)
|
||||
..addOption(
|
||||
'test',
|
||||
hide: true,
|
||||
allowMultiple: true,
|
||||
splitCommas: true,
|
||||
callback: (List<String> value) {
|
||||
if (value.isNotEmpty) {
|
||||
throw new FormatException(
|
||||
'Invalid option --test. Did you mean --task (-t)?',
|
||||
);
|
||||
}
|
||||
},
|
||||
);
|
||||
20
dev/devicelab/bin/tasks/analyzer_cli__analysis_time.dart
Normal file
20
dev/devicelab/bin/tasks/analyzer_cli__analysis_time.dart
Normal file
@ -0,0 +1,20 @@
|
||||
// Copyright 2016 The Chromium Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file.
|
||||
|
||||
import 'dart:async';
|
||||
|
||||
import 'package:flutter_devicelab/tasks/analysis.dart';
|
||||
import 'package:flutter_devicelab/framework/framework.dart';
|
||||
import 'package:flutter_devicelab/framework/utils.dart';
|
||||
|
||||
Future<Null> main() async {
|
||||
String revision = await getCurrentFlutterRepoCommit();
|
||||
DateTime revisionTimestamp = await getFlutterRepoCommitTimestamp(revision);
|
||||
String dartSdkVersion = await getDartVersion();
|
||||
await task(createAnalyzerCliTest(
|
||||
sdk: dartSdkVersion,
|
||||
commit: revision,
|
||||
timestamp: revisionTimestamp,
|
||||
));
|
||||
}
|
||||
20
dev/devicelab/bin/tasks/analyzer_server__analysis_time.dart
Normal file
20
dev/devicelab/bin/tasks/analyzer_server__analysis_time.dart
Normal file
@ -0,0 +1,20 @@
|
||||
// Copyright 2016 The Chromium Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file.
|
||||
|
||||
import 'dart:async';
|
||||
|
||||
import 'package:flutter_devicelab/tasks/analysis.dart';
|
||||
import 'package:flutter_devicelab/framework/framework.dart';
|
||||
import 'package:flutter_devicelab/framework/utils.dart';
|
||||
|
||||
Future<Null> main() async {
|
||||
String revision = await getCurrentFlutterRepoCommit();
|
||||
DateTime revisionTimestamp = await getFlutterRepoCommitTimestamp(revision);
|
||||
String dartSdkVersion = await getDartVersion();
|
||||
await task(createAnalyzerServerTest(
|
||||
sdk: dartSdkVersion,
|
||||
commit: revision,
|
||||
timestamp: revisionTimestamp,
|
||||
));
|
||||
}
|
||||
12
dev/devicelab/bin/tasks/basic_material_app__size.dart
Normal file
12
dev/devicelab/bin/tasks/basic_material_app__size.dart
Normal file
@ -0,0 +1,12 @@
|
||||
// Copyright 2016 The Chromium Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file.
|
||||
|
||||
import 'dart:async';
|
||||
|
||||
import 'package:flutter_devicelab/tasks/size_tests.dart';
|
||||
import 'package:flutter_devicelab/framework/framework.dart';
|
||||
|
||||
Future<Null> main() async {
|
||||
await task(createBasicMaterialAppSizeTest());
|
||||
}
|
||||
12
dev/devicelab/bin/tasks/complex_layout__build.dart
Normal file
12
dev/devicelab/bin/tasks/complex_layout__build.dart
Normal file
@ -0,0 +1,12 @@
|
||||
// Copyright 2016 The Chromium Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file.
|
||||
|
||||
import 'dart:async';
|
||||
|
||||
import 'package:flutter_devicelab/tasks/perf_tests.dart';
|
||||
import 'package:flutter_devicelab/framework/framework.dart';
|
||||
|
||||
Future<Null> main() async {
|
||||
await task(createComplexLayoutBuildTest());
|
||||
}
|
||||
12
dev/devicelab/bin/tasks/complex_layout__start_up.dart
Normal file
12
dev/devicelab/bin/tasks/complex_layout__start_up.dart
Normal file
@ -0,0 +1,12 @@
|
||||
// Copyright 2016 The Chromium Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file.
|
||||
|
||||
import 'dart:async';
|
||||
|
||||
import 'package:flutter_devicelab/tasks/perf_tests.dart';
|
||||
import 'package:flutter_devicelab/framework/framework.dart';
|
||||
|
||||
Future<Null> main() async {
|
||||
await task(createComplexLayoutStartupTest(ios: false));
|
||||
}
|
||||
12
dev/devicelab/bin/tasks/complex_layout_ios__start_up.dart
Normal file
12
dev/devicelab/bin/tasks/complex_layout_ios__start_up.dart
Normal file
@ -0,0 +1,12 @@
|
||||
// Copyright 2016 The Chromium Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file.
|
||||
|
||||
import 'dart:async';
|
||||
|
||||
import 'package:flutter_devicelab/tasks/perf_tests.dart';
|
||||
import 'package:flutter_devicelab/framework/framework.dart';
|
||||
|
||||
Future<Null> main() async {
|
||||
await task(createComplexLayoutStartupTest(ios: true));
|
||||
}
|
||||
@ -0,0 +1,12 @@
|
||||
// Copyright 2016 The Chromium Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file.
|
||||
|
||||
import 'dart:async';
|
||||
|
||||
import 'package:flutter_devicelab/tasks/perf_tests.dart';
|
||||
import 'package:flutter_devicelab/framework/framework.dart';
|
||||
|
||||
Future<Null> main() async {
|
||||
await task(createComplexLayoutScrollPerfTest(ios: false));
|
||||
}
|
||||
@ -0,0 +1,12 @@
|
||||
// Copyright 2016 The Chromium Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file.
|
||||
|
||||
import 'dart:async';
|
||||
|
||||
import 'package:flutter_devicelab/tasks/perf_tests.dart';
|
||||
import 'package:flutter_devicelab/framework/framework.dart';
|
||||
|
||||
Future<Null> main() async {
|
||||
await task(createComplexLayoutScrollPerfTest(ios: true));
|
||||
}
|
||||
12
dev/devicelab/bin/tasks/flutter_gallery__build.dart
Normal file
12
dev/devicelab/bin/tasks/flutter_gallery__build.dart
Normal file
@ -0,0 +1,12 @@
|
||||
// Copyright 2016 The Chromium Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file.
|
||||
|
||||
import 'dart:async';
|
||||
|
||||
import 'package:flutter_devicelab/tasks/perf_tests.dart';
|
||||
import 'package:flutter_devicelab/framework/framework.dart';
|
||||
|
||||
Future<Null> main() async {
|
||||
await task(createFlutterGalleryBuildTest());
|
||||
}
|
||||
12
dev/devicelab/bin/tasks/flutter_gallery__start_up.dart
Normal file
12
dev/devicelab/bin/tasks/flutter_gallery__start_up.dart
Normal file
@ -0,0 +1,12 @@
|
||||
// Copyright 2016 The Chromium Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file.
|
||||
|
||||
import 'dart:async';
|
||||
|
||||
import 'package:flutter_devicelab/tasks/perf_tests.dart';
|
||||
import 'package:flutter_devicelab/framework/framework.dart';
|
||||
|
||||
Future<Null> main() async {
|
||||
await task(createFlutterGalleryStartupTest(ios: false));
|
||||
}
|
||||
@ -0,0 +1,12 @@
|
||||
// Copyright 2016 The Chromium Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file.
|
||||
|
||||
import 'dart:async';
|
||||
|
||||
import 'package:flutter_devicelab/tasks/gallery.dart';
|
||||
import 'package:flutter_devicelab/framework/framework.dart';
|
||||
|
||||
Future<Null> main() async {
|
||||
await task(createGalleryTransitionTest(ios: false));
|
||||
}
|
||||
12
dev/devicelab/bin/tasks/flutter_gallery_ios__start_up.dart
Normal file
12
dev/devicelab/bin/tasks/flutter_gallery_ios__start_up.dart
Normal file
@ -0,0 +1,12 @@
|
||||
// Copyright 2016 The Chromium Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file.
|
||||
|
||||
import 'dart:async';
|
||||
|
||||
import 'package:flutter_devicelab/tasks/perf_tests.dart';
|
||||
import 'package:flutter_devicelab/framework/framework.dart';
|
||||
|
||||
Future<Null> main() async {
|
||||
await task(createFlutterGalleryStartupTest(ios: true));
|
||||
}
|
||||
@ -0,0 +1,12 @@
|
||||
// Copyright 2016 The Chromium Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file.
|
||||
|
||||
import 'dart:async';
|
||||
|
||||
import 'package:flutter_devicelab/tasks/gallery.dart';
|
||||
import 'package:flutter_devicelab/framework/framework.dart';
|
||||
|
||||
Future<Null> main() async {
|
||||
await task(createGalleryTransitionTest(ios: true));
|
||||
}
|
||||
18
dev/devicelab/bin/tasks/mega_gallery__refresh_time.dart
Normal file
18
dev/devicelab/bin/tasks/mega_gallery__refresh_time.dart
Normal file
@ -0,0 +1,18 @@
|
||||
// Copyright 2016 The Chromium Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file.
|
||||
|
||||
import 'dart:async';
|
||||
|
||||
import 'package:flutter_devicelab/tasks/refresh.dart';
|
||||
import 'package:flutter_devicelab/framework/framework.dart';
|
||||
import 'package:flutter_devicelab/framework/utils.dart';
|
||||
|
||||
Future<Null> main() async {
|
||||
String revision = await getCurrentFlutterRepoCommit();
|
||||
DateTime revisionTimestamp = await getFlutterRepoCommitTimestamp(revision);
|
||||
await task(createRefreshTest(
|
||||
commit: revision,
|
||||
timestamp: revisionTimestamp,
|
||||
));
|
||||
}
|
||||
14
dev/devicelab/bin/tasks/smoke_test_failure.dart
Normal file
14
dev/devicelab/bin/tasks/smoke_test_failure.dart
Normal file
@ -0,0 +1,14 @@
|
||||
// Copyright 2016 The Chromium Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file.
|
||||
|
||||
import 'dart:async';
|
||||
|
||||
import 'package:flutter_devicelab/framework/framework.dart';
|
||||
|
||||
/// Smoke test of a task that fails by returning an unsuccessful response.
|
||||
Future<Null> main() async {
|
||||
await task(() async {
|
||||
return new TaskResult.failure('Failed');
|
||||
});
|
||||
}
|
||||
13
dev/devicelab/bin/tasks/smoke_test_setup_failure.dart
Normal file
13
dev/devicelab/bin/tasks/smoke_test_setup_failure.dart
Normal file
@ -0,0 +1,13 @@
|
||||
// Copyright 2016 The Chromium Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file.
|
||||
|
||||
import 'dart:async';
|
||||
|
||||
/// Creates a situation when the test framework was not properly initialized.
|
||||
///
|
||||
/// By not calling `task()` the VM service extension is not registered and
|
||||
/// therefore will not accept requests to run tasks. When the runner attempts to
|
||||
/// connect and run the test it will receive a "method not found" error from the
|
||||
/// VM service, will likely retry and finally time out.
|
||||
Future<Null> main() async {}
|
||||
14
dev/devicelab/bin/tasks/smoke_test_success.dart
Normal file
14
dev/devicelab/bin/tasks/smoke_test_success.dart
Normal file
@ -0,0 +1,14 @@
|
||||
// Copyright 2016 The Chromium Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file.
|
||||
|
||||
import 'dart:async';
|
||||
|
||||
import 'package:flutter_devicelab/framework/framework.dart';
|
||||
|
||||
/// Smoke test of a successful task.
|
||||
Future<Null> main() async {
|
||||
await task(() async {
|
||||
return new TaskResult.success(<String, dynamic>{});
|
||||
});
|
||||
}
|
||||
14
dev/devicelab/bin/tasks/smoke_test_throws.dart
Normal file
14
dev/devicelab/bin/tasks/smoke_test_throws.dart
Normal file
@ -0,0 +1,14 @@
|
||||
// Copyright 2016 The Chromium Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file.
|
||||
|
||||
import 'dart:async';
|
||||
|
||||
import 'package:flutter_devicelab/framework/framework.dart';
|
||||
|
||||
/// Smoke test of a task that fails with an exception.
|
||||
Future<Null> main() async {
|
||||
await task(() async {
|
||||
throw 'failed';
|
||||
});
|
||||
}
|
||||
202
dev/devicelab/lib/framework/adb.dart
Normal file
202
dev/devicelab/lib/framework/adb.dart
Normal file
@ -0,0 +1,202 @@
|
||||
// Copyright 2016 The Chromium Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file.
|
||||
|
||||
import 'dart:async';
|
||||
import 'dart:io';
|
||||
import 'dart:math' as math;
|
||||
|
||||
import 'package:path/path.dart' as path;
|
||||
|
||||
import 'utils.dart';
|
||||
|
||||
typedef Future<Adb> AdbGetter();
|
||||
|
||||
/// Get an instance of [Adb].
|
||||
///
|
||||
/// See [realAdbGetter] for signature. This can be overwritten for testing.
|
||||
AdbGetter adb = realAdbGetter;
|
||||
|
||||
Adb _currentDevice;
|
||||
|
||||
/// Picks a random Android device out of connected devices and sets it as
|
||||
/// [_currentDevice].
|
||||
Future<Null> pickNextDevice() async {
|
||||
List<Adb> allDevices =
|
||||
(await Adb.deviceIds).map((String id) => new Adb(deviceId: id)).toList();
|
||||
|
||||
if (allDevices.length == 0) throw 'No Android devices detected';
|
||||
|
||||
// TODO(yjbanov): filter out and warn about those with low battery level
|
||||
_currentDevice = allDevices[new math.Random().nextInt(allDevices.length)];
|
||||
}
|
||||
|
||||
Future<Adb> realAdbGetter() async {
|
||||
if (_currentDevice == null) await pickNextDevice();
|
||||
return _currentDevice;
|
||||
}
|
||||
|
||||
/// Gets the ID of an unlocked device, unlocking it if necessary.
|
||||
// TODO(yjbanov): abstract away iOS from Android.
|
||||
Future<String> getUnlockedDeviceId({ bool ios: false }) async {
|
||||
if (ios) {
|
||||
// We currently do not have a way to lock/unlock iOS devices, or even to
|
||||
// pick one out of many. So we pick the first random iPhone and assume it's
|
||||
// already unlocked. For now we'll just keep them at minimum screen
|
||||
// brightness so they don't drain battery too fast.
|
||||
List<String> iosDeviceIds =
|
||||
grep('UniqueDeviceID', from: await eval('ideviceinfo', <String>[]))
|
||||
.map((String line) => line.split(' ').last)
|
||||
.toList();
|
||||
|
||||
if (iosDeviceIds.isEmpty) throw 'No connected iOS devices found.';
|
||||
|
||||
return iosDeviceIds.first;
|
||||
}
|
||||
|
||||
Adb device = await adb();
|
||||
await device.unlock();
|
||||
return device.deviceId;
|
||||
}
|
||||
|
||||
/// Android Debug Bridge (`adb`) client that exposes a subset of functions
|
||||
/// relevant to on-device testing.
|
||||
class Adb {
|
||||
Adb({ this.deviceId });
|
||||
|
||||
final String deviceId;
|
||||
|
||||
// Parses information about a device. Example:
|
||||
//
|
||||
// 015d172c98400a03 device usb:340787200X product:nakasi model:Nexus_7 device:grouper
|
||||
static final RegExp _kDeviceRegex = new RegExp(r'^(\S+)\s+(\S+)(.*)');
|
||||
|
||||
/// Reports connection health for every device.
|
||||
static Future<Map<String, HealthCheckResult>> checkDevices() async {
|
||||
Map<String, HealthCheckResult> results = <String, HealthCheckResult>{};
|
||||
for (String deviceId in await deviceIds) {
|
||||
try {
|
||||
Adb device = new Adb(deviceId: deviceId);
|
||||
// Just a smoke test that we can read wakefulness state
|
||||
// TODO(yjbanov): also check battery level
|
||||
await device._getWakefulness();
|
||||
results['android-device-$deviceId'] = new HealthCheckResult.success();
|
||||
} catch (e, s) {
|
||||
results['android-device-$deviceId'] = new HealthCheckResult.error(e, s);
|
||||
}
|
||||
}
|
||||
return results;
|
||||
}
|
||||
|
||||
/// Kills the `adb` server causing it to start a new instance upon next
|
||||
/// command.
|
||||
///
|
||||
/// Restarting `adb` helps with keeping device connections alive. When `adb`
|
||||
/// runs non-stop for too long it loses connections to devices.
|
||||
static Future<Null> restart() async {
|
||||
await exec(adbPath, <String>['kill-server'], canFail: false);
|
||||
}
|
||||
|
||||
/// List of device IDs visible to `adb`.
|
||||
static Future<List<String>> get deviceIds async {
|
||||
List<String> output =
|
||||
(await eval(adbPath, <String>['devices', '-l'], canFail: false))
|
||||
.trim()
|
||||
.split('\n');
|
||||
List<String> results = <String>[];
|
||||
for (String line in output) {
|
||||
// Skip lines like: * daemon started successfully *
|
||||
if (line.startsWith('* daemon ')) continue;
|
||||
|
||||
if (line.startsWith('List of devices')) continue;
|
||||
|
||||
if (_kDeviceRegex.hasMatch(line)) {
|
||||
Match match = _kDeviceRegex.firstMatch(line);
|
||||
|
||||
String deviceID = match[1];
|
||||
String deviceState = match[2];
|
||||
|
||||
if (!const <String>['unauthorized', 'offline'].contains(deviceState)) {
|
||||
results.add(deviceID);
|
||||
}
|
||||
} else {
|
||||
throw 'Failed to parse device from adb output: $line';
|
||||
}
|
||||
}
|
||||
|
||||
return results;
|
||||
}
|
||||
|
||||
/// Whether the device is awake.
|
||||
Future<bool> isAwake() async {
|
||||
return await _getWakefulness() == 'Awake';
|
||||
}
|
||||
|
||||
/// Whether the device is asleep.
|
||||
Future<bool> isAsleep() async {
|
||||
return await _getWakefulness() == 'Asleep';
|
||||
}
|
||||
|
||||
/// Wake up the device if it is not awake using [togglePower].
|
||||
Future<Null> wakeUp() async {
|
||||
if (!(await isAwake())) await togglePower();
|
||||
}
|
||||
|
||||
/// Send the device to sleep mode if it is not asleep using [togglePower].
|
||||
Future<Null> sendToSleep() async {
|
||||
if (!(await isAsleep())) await togglePower();
|
||||
}
|
||||
|
||||
/// Sends `KEYCODE_POWER` (26), which causes the device to toggle its mode
|
||||
/// between awake and asleep.
|
||||
Future<Null> togglePower() async {
|
||||
await shellExec('input', const <String>['keyevent', '26']);
|
||||
}
|
||||
|
||||
/// Unlocks the device by sending `KEYCODE_MENU` (82).
|
||||
///
|
||||
/// This only works when the device doesn't have a secure unlock pattern.
|
||||
Future<Null> unlock() async {
|
||||
await wakeUp();
|
||||
await shellExec('input', const <String>['keyevent', '82']);
|
||||
}
|
||||
|
||||
/// Retrieves device's wakefulness state.
|
||||
///
|
||||
/// See: https://android.googlesource.com/platform/frameworks/base/+/master/core/java/android/os/PowerManagerInternal.java
|
||||
Future<String> _getWakefulness() async {
|
||||
String powerInfo = await shellEval('dumpsys', <String>['power']);
|
||||
String wakefulness =
|
||||
grep('mWakefulness=', from: powerInfo).single.split('=')[1].trim();
|
||||
return wakefulness;
|
||||
}
|
||||
|
||||
/// Executes [command] on `adb shell` and returns its exit code.
|
||||
Future<Null> shellExec(String command, List<String> arguments,
|
||||
{ Map<String, String> env }) async {
|
||||
await exec(adbPath, <String>['shell', command]..addAll(arguments),
|
||||
env: env, canFail: false);
|
||||
}
|
||||
|
||||
/// Executes [command] on `adb shell` and returns its standard output as a [String].
|
||||
Future<String> shellEval(String command, List<String> arguments,
|
||||
{ Map<String, String> env }) {
|
||||
return eval(adbPath, <String>['shell', command]..addAll(arguments),
|
||||
env: env, canFail: false);
|
||||
}
|
||||
}
|
||||
|
||||
/// Path to the `adb` executable.
|
||||
String get adbPath {
|
||||
String androidHome = Platform.environment['ANDROID_HOME'];
|
||||
|
||||
if (androidHome == null)
|
||||
throw 'ANDROID_HOME environment variable missing. This variable must '
|
||||
'point to the Android SDK directory containing platform-tools.';
|
||||
|
||||
File adbPath = file(path.join(androidHome, 'platform-tools/adb'));
|
||||
|
||||
if (!adbPath.existsSync()) throw 'adb not found at: $adbPath';
|
||||
|
||||
return adbPath.absolute.path;
|
||||
}
|
||||
62
dev/devicelab/lib/framework/benchmarks.dart
Normal file
62
dev/devicelab/lib/framework/benchmarks.dart
Normal file
@ -0,0 +1,62 @@
|
||||
// Copyright (c) 2016 The Chromium Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file.
|
||||
|
||||
import 'dart:async';
|
||||
|
||||
import 'framework.dart';
|
||||
|
||||
/// A benchmark harness used to run a benchmark multiple times and report the
|
||||
/// best result.
|
||||
abstract class Benchmark {
|
||||
Benchmark(this.name);
|
||||
|
||||
final String name;
|
||||
|
||||
TaskResult bestResult;
|
||||
|
||||
Future<Null> init() => new Future<Null>.value();
|
||||
|
||||
Future<num> run();
|
||||
TaskResult get lastResult;
|
||||
|
||||
@override
|
||||
String toString() => name;
|
||||
}
|
||||
|
||||
/// Runs a [benchmark] [iterations] times and reports the best result.
|
||||
///
|
||||
/// Use [warmUpBenchmark] to discard cold performance results.
|
||||
Future<num> runBenchmark(Benchmark benchmark, {
|
||||
int iterations: 1,
|
||||
bool warmUpBenchmark: false
|
||||
}) async {
|
||||
await benchmark.init();
|
||||
|
||||
List<num> allRuns = <num>[];
|
||||
|
||||
num minValue;
|
||||
|
||||
if (warmUpBenchmark)
|
||||
await benchmark.run();
|
||||
|
||||
while (iterations > 0) {
|
||||
iterations--;
|
||||
|
||||
print('');
|
||||
|
||||
try {
|
||||
num result = await benchmark.run();
|
||||
allRuns.add(result);
|
||||
|
||||
if (minValue == null || result < minValue) {
|
||||
benchmark.bestResult = benchmark.lastResult;
|
||||
minValue = result;
|
||||
}
|
||||
} catch (error) {
|
||||
print('benchmark failed with error: $error');
|
||||
}
|
||||
}
|
||||
|
||||
return minValue;
|
||||
}
|
||||
217
dev/devicelab/lib/framework/framework.dart
Normal file
217
dev/devicelab/lib/framework/framework.dart
Normal file
@ -0,0 +1,217 @@
|
||||
// Copyright 2016 The Chromium Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file.
|
||||
|
||||
import 'dart:async';
|
||||
import 'dart:convert';
|
||||
import 'dart:developer';
|
||||
import 'dart:io';
|
||||
import 'dart:isolate';
|
||||
|
||||
import 'package:logging/logging.dart';
|
||||
|
||||
import 'utils.dart';
|
||||
|
||||
/// Maximum amount of time a single task is allowed to take to run.
|
||||
///
|
||||
/// If exceeded the task is considered to have failed.
|
||||
const Duration taskTimeout = const Duration(minutes: 10);
|
||||
|
||||
/// Represents a unit of work performed in the CI environment that can
|
||||
/// succeed, fail and be retried independently of others.
|
||||
typedef Future<TaskResult> TaskFunction();
|
||||
|
||||
bool _isTaskRegistered = false;
|
||||
|
||||
/// Registers a [task] to run, returns the result when it is complete.
|
||||
///
|
||||
/// Note, the task does not run immediately but waits for the request via the
|
||||
/// VM service protocol to run it.
|
||||
///
|
||||
/// It is ok for a [task] to perform many things. However, only one task can be
|
||||
/// registered per Dart VM.
|
||||
Future<TaskResult> task(TaskFunction task) {
|
||||
if (_isTaskRegistered)
|
||||
throw new StateError('A task is already registered');
|
||||
|
||||
_isTaskRegistered = true;
|
||||
|
||||
// TODO: allow overriding logging.
|
||||
Logger.root.level = Level.ALL;
|
||||
Logger.root.onRecord.listen((LogRecord rec) {
|
||||
print('${rec.level.name}: ${rec.time}: ${rec.message}');
|
||||
});
|
||||
|
||||
_TaskRunner runner = new _TaskRunner(task);
|
||||
runner.keepVmAliveUntilTaskRunRequested();
|
||||
return runner.whenDone;
|
||||
}
|
||||
|
||||
class _TaskRunner {
|
||||
static final Logger logger = new Logger('TaskRunner');
|
||||
|
||||
final TaskFunction task;
|
||||
|
||||
// TODO: workaround for https://github.com/dart-lang/sdk/issues/23797
|
||||
RawReceivePort _keepAlivePort;
|
||||
Timer _startTaskTimeout;
|
||||
bool _taskStarted = false;
|
||||
|
||||
final Completer<TaskResult> _completer = new Completer<TaskResult>();
|
||||
|
||||
_TaskRunner(this.task) {
|
||||
registerExtension('ext.cocoonRunTask',
|
||||
(String method, Map<String, String> parameters) async {
|
||||
TaskResult result = await run();
|
||||
return new ServiceExtensionResponse.result(JSON.encode(result.toJson()));
|
||||
});
|
||||
registerExtension('ext.cocoonRunnerReady',
|
||||
(String method, Map<String, String> parameters) async {
|
||||
return new ServiceExtensionResponse.result('"ready"');
|
||||
});
|
||||
}
|
||||
|
||||
/// Signals that this task runner finished running the task.
|
||||
Future<TaskResult> get whenDone => _completer.future;
|
||||
|
||||
Future<TaskResult> run() async {
|
||||
try {
|
||||
_taskStarted = true;
|
||||
TaskResult result = await _performTask().timeout(taskTimeout);
|
||||
_completer.complete(result);
|
||||
return result;
|
||||
} on TimeoutException catch (_) {
|
||||
return new TaskResult.failure('Task timed out after $taskTimeout');
|
||||
} finally {
|
||||
forceQuitRunningProcesses();
|
||||
_closeKeepAlivePort();
|
||||
}
|
||||
}
|
||||
|
||||
/// Causes the Dart VM to stay alive until a request to run the task is
|
||||
/// received via the VM service protocol.
|
||||
void keepVmAliveUntilTaskRunRequested() {
|
||||
if (_taskStarted)
|
||||
throw new StateError('Task already started.');
|
||||
|
||||
// Merely creating this port object will cause the VM to stay alive and keep
|
||||
// the VM service server running until the port is disposed of.
|
||||
_keepAlivePort = new RawReceivePort();
|
||||
|
||||
// Timeout if nothing bothers to connect and ask us to run the task.
|
||||
const Duration taskStartTimeout = const Duration(seconds: 10);
|
||||
_startTaskTimeout = new Timer(taskStartTimeout, () {
|
||||
if (!_taskStarted) {
|
||||
logger.severe('Task did not start in $taskStartTimeout.');
|
||||
_closeKeepAlivePort();
|
||||
exitCode = 1;
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
/// Disables the keep-alive port, allowing the VM to exit.
|
||||
void _closeKeepAlivePort() {
|
||||
_startTaskTimeout?.cancel();
|
||||
_keepAlivePort?.close();
|
||||
}
|
||||
|
||||
Future<TaskResult> _performTask() async {
|
||||
try {
|
||||
return await task();
|
||||
} catch (taskError, taskErrorStack) {
|
||||
String message = 'Task failed: $taskError';
|
||||
if (taskErrorStack != null) {
|
||||
message += '\n\n$taskErrorStack';
|
||||
}
|
||||
return new TaskResult.failure(message);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// A result of running a single task.
|
||||
class TaskResult {
|
||||
/// Constructs a successful result.
|
||||
TaskResult.success(this.data, {this.benchmarkScoreKeys: const <String>[]})
|
||||
: this.succeeded = true,
|
||||
this.message = 'success' {
|
||||
const JsonEncoder prettyJson = const JsonEncoder.withIndent(' ');
|
||||
if (benchmarkScoreKeys != null) {
|
||||
for (String key in benchmarkScoreKeys) {
|
||||
if (!data.containsKey(key)) {
|
||||
throw 'Invalid Golem score key "$key". It does not exist in task '
|
||||
'result data ${prettyJson.convert(data)}';
|
||||
} else if (data[key] is! num) {
|
||||
throw 'Invalid Golem score for key "$key". It is expected to be a num '
|
||||
'but was ${data[key].runtimeType}: ${prettyJson.convert(data[key])}';
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Constructs a successful result using JSON data stored in a file.
|
||||
factory TaskResult.successFromFile(File file,
|
||||
{List<String> benchmarkScoreKeys}) {
|
||||
return new TaskResult.success(JSON.decode(file.readAsStringSync()),
|
||||
benchmarkScoreKeys: benchmarkScoreKeys);
|
||||
}
|
||||
|
||||
/// Constructs an unsuccessful result.
|
||||
TaskResult.failure(this.message)
|
||||
: this.succeeded = false,
|
||||
this.data = null,
|
||||
this.benchmarkScoreKeys = const <String>[];
|
||||
|
||||
/// Whether the task succeeded.
|
||||
final bool succeeded;
|
||||
|
||||
/// Task-specific JSON data
|
||||
final Map<String, dynamic> data;
|
||||
|
||||
/// Keys in [data] that store scores that will be submitted to Golem.
|
||||
///
|
||||
/// Each key is also part of a benchmark's name tracked by Golem.
|
||||
/// A benchmark name is computed by combining [Task.name] with a key
|
||||
/// separated by a dot. For example, if a task's name is
|
||||
/// `"complex_layout__start_up"` and score key is
|
||||
/// `"engineEnterTimestampMicros"`, the score will be submitted to Golem under
|
||||
/// `"complex_layout__start_up.engineEnterTimestampMicros"`.
|
||||
///
|
||||
/// This convention reduces the amount of configuration that needs to be done
|
||||
/// to submit benchmark scores to Golem.
|
||||
final List<String> benchmarkScoreKeys;
|
||||
|
||||
/// Whether the task failed.
|
||||
bool get failed => !succeeded;
|
||||
|
||||
/// Explains the result in a human-readable format.
|
||||
final String message;
|
||||
|
||||
/// Serializes this task result to JSON format.
|
||||
///
|
||||
/// The JSON format is as follows:
|
||||
///
|
||||
/// {
|
||||
/// "success": true|false,
|
||||
/// "data": arbitrary JSON data valid only for successful results,
|
||||
/// "benchmarkScoreKeys": [
|
||||
/// contains keys into "data" that represent benchmarks scores, which
|
||||
/// can be uploaded, for example. to golem, valid only for successful
|
||||
/// results
|
||||
/// ],
|
||||
/// "reason": failure reason string valid only for unsuccessful results
|
||||
/// }
|
||||
Map<String, dynamic> toJson() {
|
||||
Map<String, dynamic> json = <String, dynamic>{
|
||||
'success': succeeded,
|
||||
};
|
||||
|
||||
if (succeeded) {
|
||||
json['data'] = data;
|
||||
json['benchmarkScoreKeys'] = benchmarkScoreKeys;
|
||||
} else {
|
||||
json['reason'] = message;
|
||||
}
|
||||
|
||||
return json;
|
||||
}
|
||||
}
|
||||
127
dev/devicelab/lib/framework/manifest.dart
Normal file
127
dev/devicelab/lib/framework/manifest.dart
Normal file
@ -0,0 +1,127 @@
|
||||
// Copyright 2016 The Chromium Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file.
|
||||
|
||||
import 'package:meta/meta.dart';
|
||||
import 'package:yaml/yaml.dart';
|
||||
|
||||
import 'utils.dart';
|
||||
|
||||
/// Loads manifest data from `manifest.yaml` file or from [yaml], if present.
|
||||
Manifest loadTaskManifest([ String yaml ]) {
|
||||
dynamic manifestYaml = yaml == null
|
||||
? loadYaml(file('manifest.yaml').readAsStringSync())
|
||||
: loadYamlNode(yaml);
|
||||
|
||||
_checkType(manifestYaml is Map, manifestYaml, 'Manifest', 'dictionary');
|
||||
return _validateAndParseManifest(manifestYaml);
|
||||
}
|
||||
|
||||
/// Contains CI task information.
|
||||
class Manifest {
|
||||
Manifest._(this.tasks);
|
||||
|
||||
/// CI tasks.
|
||||
final List<ManifestTask> tasks;
|
||||
}
|
||||
|
||||
/// A CI task.
|
||||
class ManifestTask {
|
||||
ManifestTask._({
|
||||
@required this.name,
|
||||
@required this.description,
|
||||
@required this.stage,
|
||||
@required this.requiredAgentCapabilities,
|
||||
}) {
|
||||
String taskName = 'task "$name"';
|
||||
_checkIsNotBlank(name, 'Task name', taskName);
|
||||
_checkIsNotBlank(description, 'Task description', taskName);
|
||||
_checkIsNotBlank(stage, 'Task stage', taskName);
|
||||
_checkIsNotBlank(requiredAgentCapabilities, 'requiredAgentCapabilities', taskName);
|
||||
}
|
||||
|
||||
/// Task name as it appears on the dashboard.
|
||||
final String name;
|
||||
|
||||
/// A human-readable description of the task.
|
||||
final String description;
|
||||
|
||||
/// The stage this task should run in.
|
||||
final String stage;
|
||||
|
||||
/// Capabilities required of the build agent to be able to perform this task.
|
||||
final List<String> requiredAgentCapabilities;
|
||||
}
|
||||
|
||||
/// Thrown when the manifest YAML is not valid.
|
||||
class ManifestError extends Error {
|
||||
ManifestError(this.message);
|
||||
|
||||
final String message;
|
||||
|
||||
@override
|
||||
String toString() => '$ManifestError: $message';
|
||||
}
|
||||
|
||||
// There's no good YAML validator, at least not for Dart, so we validate
|
||||
// manually. It's not too much code and produces good error messages.
|
||||
Manifest _validateAndParseManifest(Map<String, dynamic> manifestYaml) {
|
||||
_checkKeys(manifestYaml, 'manifest', const <String>['tasks']);
|
||||
return new Manifest._(_validateAndParseTasks(manifestYaml['tasks']));
|
||||
}
|
||||
|
||||
List<ManifestTask> _validateAndParseTasks(dynamic tasksYaml) {
|
||||
_checkType(tasksYaml is Map, tasksYaml, 'Value of "tasks"', 'dictionary');
|
||||
return tasksYaml.keys.map((dynamic taskName) => _validateAndParseTask(taskName, tasksYaml[taskName])).toList();
|
||||
}
|
||||
|
||||
ManifestTask _validateAndParseTask(dynamic taskName, dynamic taskYaml) {
|
||||
_checkType(taskName is String, taskName, 'Task name', 'string');
|
||||
_checkType(taskYaml is Map, taskYaml, 'Value of task "$taskName"', 'dictionary');
|
||||
_checkKeys(taskYaml, 'Value of task "$taskName"', const <String>[
|
||||
'description',
|
||||
'stage',
|
||||
'required_agent_capabilities',
|
||||
]);
|
||||
|
||||
List<String> capabilities = _validateAndParseCapabilities(taskName, taskYaml['required_agent_capabilities']);
|
||||
return new ManifestTask._(
|
||||
name: taskName,
|
||||
description: taskYaml['description'],
|
||||
stage: taskYaml['stage'],
|
||||
requiredAgentCapabilities: capabilities,
|
||||
);
|
||||
}
|
||||
|
||||
List<String> _validateAndParseCapabilities(String taskName, dynamic capabilitiesYaml) {
|
||||
_checkType(capabilitiesYaml is List, capabilitiesYaml, 'required_agent_capabilities', 'list');
|
||||
for (int i = 0; i < capabilitiesYaml.length; i++) {
|
||||
dynamic capability = capabilitiesYaml[i];
|
||||
_checkType(capability is String, capability, 'required_agent_capabilities[$i]', 'string');
|
||||
}
|
||||
return capabilitiesYaml;
|
||||
}
|
||||
|
||||
void _checkType(bool isValid, dynamic value, String variableName, String typeName) {
|
||||
if (!isValid) {
|
||||
throw new ManifestError(
|
||||
'$variableName must be a $typeName but was ${value.runtimeType}: $value',
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
void _checkIsNotBlank(dynamic value, String variableName, String ownerName) {
|
||||
if (value == null || value.isEmpty) {
|
||||
throw new ManifestError('$variableName must not be empty in $ownerName.');
|
||||
}
|
||||
}
|
||||
|
||||
void _checkKeys(Map<String, dynamic> map, String variableName, List<String> allowedKeys) {
|
||||
for (String key in map.keys) {
|
||||
if (!allowedKeys.contains(key)) {
|
||||
throw new ManifestError(
|
||||
'Unrecognized property "$key" in $variableName. '
|
||||
'Allowed properties: ${allowedKeys.join(', ')}');
|
||||
}
|
||||
}
|
||||
}
|
||||
129
dev/devicelab/lib/framework/runner.dart
Normal file
129
dev/devicelab/lib/framework/runner.dart
Normal file
@ -0,0 +1,129 @@
|
||||
// Copyright 2016 The Chromium Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file.
|
||||
|
||||
import 'dart:async';
|
||||
import 'dart:convert';
|
||||
import 'dart:io';
|
||||
|
||||
import 'package:vm_service_client/vm_service_client.dart';
|
||||
|
||||
import 'package:flutter_devicelab/framework/utils.dart';
|
||||
|
||||
/// Slightly longer than task timeout that gives the task runner a chance to
|
||||
/// clean-up before forcefully quitting it.
|
||||
const Duration taskTimeoutWithGracePeriod = const Duration(minutes: 11);
|
||||
|
||||
/// Runs a task in a separate Dart VM and collects the result using the VM
|
||||
/// service protocol.
|
||||
///
|
||||
/// [taskName] is the name of the task. The corresponding task executable is
|
||||
/// expected to be found under `bin/tasks`.
|
||||
Future<Map<String, dynamic>> runTask(String taskName) async {
|
||||
String taskExecutable = 'bin/tasks/$taskName.dart';
|
||||
|
||||
if (!file(taskExecutable).existsSync())
|
||||
throw 'Executable Dart file not found: $taskExecutable';
|
||||
|
||||
int vmServicePort = await _findAvailablePort();
|
||||
Process runner = await startProcess(dartBin, <String>[
|
||||
'--enable-vm-service=$vmServicePort',
|
||||
'--no-pause-isolates-on-exit',
|
||||
taskExecutable,
|
||||
]);
|
||||
|
||||
bool runnerFinished = false;
|
||||
|
||||
runner.exitCode.then((_) {
|
||||
runnerFinished = true;
|
||||
});
|
||||
|
||||
StreamSubscription<String> stdoutSub = runner.stdout
|
||||
.transform(new Utf8Decoder())
|
||||
.transform(new LineSplitter())
|
||||
.listen((String line) {
|
||||
stdout.writeln('[$taskName] [STDOUT] $line');
|
||||
});
|
||||
|
||||
StreamSubscription<String> stderrSub = runner.stderr
|
||||
.transform(new Utf8Decoder())
|
||||
.transform(new LineSplitter())
|
||||
.listen((String line) {
|
||||
stderr.writeln('[$taskName] [STDERR] $line');
|
||||
});
|
||||
|
||||
String waitingFor = 'connection';
|
||||
try {
|
||||
VMIsolate isolate = await _connectToRunnerIsolate(vmServicePort);
|
||||
waitingFor = 'task completion';
|
||||
Map<String, dynamic> taskResult =
|
||||
await isolate.invokeExtension('ext.cocoonRunTask').timeout(taskTimeoutWithGracePeriod);
|
||||
waitingFor = 'task process to exit';
|
||||
await runner.exitCode.timeout(const Duration(seconds: 1));
|
||||
return taskResult;
|
||||
} on TimeoutException catch (timeout) {
|
||||
runner.kill(ProcessSignal.SIGINT);
|
||||
return <String, dynamic>{
|
||||
'success': false,
|
||||
'reason': 'Timeout waiting for $waitingFor: ${timeout.message}',
|
||||
};
|
||||
} finally {
|
||||
if (!runnerFinished)
|
||||
runner.kill(ProcessSignal.SIGKILL);
|
||||
await stdoutSub.cancel();
|
||||
await stderrSub.cancel();
|
||||
}
|
||||
}
|
||||
|
||||
Future<VMIsolate> _connectToRunnerIsolate(int vmServicePort) async {
|
||||
String url = 'ws://localhost:$vmServicePort/ws';
|
||||
DateTime started = new DateTime.now();
|
||||
|
||||
// TODO(yjbanov): due to lack of imagination at the moment the handshake with
|
||||
// the task process is very rudimentary and requires this small
|
||||
// delay to let the task process open up the VM service port.
|
||||
// Otherwise we almost always hit the non-ready case first and
|
||||
// wait a whole 1 second, which is annoying.
|
||||
await new Future<Null>.delayed(const Duration(milliseconds: 100));
|
||||
|
||||
while (true) {
|
||||
try {
|
||||
// Make sure VM server is up by successfully opening and closing a socket.
|
||||
await (await WebSocket.connect(url)).close();
|
||||
|
||||
// Look up the isolate.
|
||||
VMServiceClient client = new VMServiceClient.connect(url);
|
||||
VM vm = await client.getVM();
|
||||
VMIsolate isolate = vm.isolates.single;
|
||||
String response = await isolate.invokeExtension('ext.cocoonRunnerReady');
|
||||
if (response != 'ready') throw 'not ready yet';
|
||||
return isolate;
|
||||
} catch (error) {
|
||||
const Duration connectionTimeout = const Duration(seconds: 2);
|
||||
if (new DateTime.now().difference(started) > connectionTimeout) {
|
||||
throw new TimeoutException(
|
||||
'Failed to connect to the task runner process',
|
||||
connectionTimeout,
|
||||
);
|
||||
}
|
||||
print('VM service not ready yet: $error');
|
||||
const Duration pauseBetweenRetries = const Duration(milliseconds: 200);
|
||||
print('Will retry in $pauseBetweenRetries.');
|
||||
await new Future<Null>.delayed(pauseBetweenRetries);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Future<int> _findAvailablePort() async {
|
||||
int port = 20000;
|
||||
while (true) {
|
||||
try {
|
||||
ServerSocket socket =
|
||||
await ServerSocket.bind(InternetAddress.LOOPBACK_IP_V4, port);
|
||||
await socket.close();
|
||||
return port;
|
||||
} catch (_) {
|
||||
port++;
|
||||
}
|
||||
}
|
||||
}
|
||||
412
dev/devicelab/lib/framework/utils.dart
Normal file
412
dev/devicelab/lib/framework/utils.dart
Normal file
@ -0,0 +1,412 @@
|
||||
// Copyright (c) 2016 The Chromium Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file.
|
||||
|
||||
import 'dart:async';
|
||||
import 'dart:convert';
|
||||
import 'dart:io';
|
||||
|
||||
import 'package:meta/meta.dart';
|
||||
import 'package:path/path.dart' as path;
|
||||
import 'package:stack_trace/stack_trace.dart';
|
||||
|
||||
/// Virtual current working directory, which affect functions, such as [exec].
|
||||
String cwd = Directory.current.path;
|
||||
|
||||
List<ProcessInfo> _runningProcesses = <ProcessInfo>[];
|
||||
|
||||
class ProcessInfo {
|
||||
ProcessInfo(this.command, this.process);
|
||||
|
||||
final DateTime startTime = new DateTime.now();
|
||||
final String command;
|
||||
final Process process;
|
||||
|
||||
@override
|
||||
String toString() {
|
||||
return '''
|
||||
command : $command
|
||||
started : $startTime
|
||||
pid : ${process.pid}
|
||||
'''
|
||||
.trim();
|
||||
}
|
||||
}
|
||||
|
||||
/// Result of a health check for a specific parameter.
|
||||
class HealthCheckResult {
|
||||
HealthCheckResult.success([this.details]) : succeeded = true;
|
||||
HealthCheckResult.failure(this.details) : succeeded = false;
|
||||
HealthCheckResult.error(dynamic error, dynamic stackTrace)
|
||||
: succeeded = false,
|
||||
details = 'ERROR: $error${'\n$stackTrace' ?? ''}';
|
||||
|
||||
final bool succeeded;
|
||||
final String details;
|
||||
|
||||
@override
|
||||
String toString() {
|
||||
StringBuffer buf = new StringBuffer(succeeded ? 'succeeded' : 'failed');
|
||||
if (details != null && details.trim().isNotEmpty) {
|
||||
buf.writeln();
|
||||
// Indent details by 4 spaces
|
||||
for (String line in details.trim().split('\n')) {
|
||||
buf.writeln(' $line');
|
||||
}
|
||||
}
|
||||
return '$buf';
|
||||
}
|
||||
}
|
||||
|
||||
class BuildFailedError extends Error {
|
||||
BuildFailedError(this.message);
|
||||
|
||||
final String message;
|
||||
|
||||
@override
|
||||
String toString() => message;
|
||||
}
|
||||
|
||||
void fail(String message) {
|
||||
throw new BuildFailedError(message);
|
||||
}
|
||||
|
||||
void rm(FileSystemEntity entity) {
|
||||
if (entity.existsSync())
|
||||
entity.deleteSync();
|
||||
}
|
||||
|
||||
/// Remove recursively.
|
||||
void rmTree(FileSystemEntity entity) {
|
||||
if (entity.existsSync())
|
||||
entity.deleteSync(recursive: true);
|
||||
}
|
||||
|
||||
List<FileSystemEntity> ls(Directory directory) => directory.listSync();
|
||||
|
||||
Directory dir(String path) => new Directory(path);
|
||||
|
||||
File file(String path) => new File(path);
|
||||
|
||||
void copy(File sourceFile, Directory targetDirectory, {String name}) {
|
||||
File target = file(
|
||||
path.join(targetDirectory.path, name ?? path.basename(sourceFile.path)));
|
||||
target.writeAsBytesSync(sourceFile.readAsBytesSync());
|
||||
}
|
||||
|
||||
FileSystemEntity move(FileSystemEntity whatToMove,
|
||||
{Directory to, String name}) {
|
||||
return whatToMove
|
||||
.renameSync(path.join(to.path, name ?? path.basename(whatToMove.path)));
|
||||
}
|
||||
|
||||
/// Equivalent of `mkdir directory`.
|
||||
void mkdir(Directory directory) {
|
||||
directory.createSync();
|
||||
}
|
||||
|
||||
/// Equivalent of `mkdir -p directory`.
|
||||
void mkdirs(Directory directory) {
|
||||
directory.createSync(recursive: true);
|
||||
}
|
||||
|
||||
bool exists(FileSystemEntity entity) => entity.existsSync();
|
||||
|
||||
void section(String title) {
|
||||
print('\n••• $title •••');
|
||||
}
|
||||
|
||||
Future<String> getDartVersion() async {
|
||||
// The Dart VM returns the version text to stderr.
|
||||
ProcessResult result = Process.runSync(dartBin, <String>['--version']);
|
||||
String version = result.stderr.trim();
|
||||
|
||||
// Convert:
|
||||
// Dart VM version: 1.17.0-dev.2.0 (Tue May 3 12:14:52 2016) on "macos_x64"
|
||||
// to:
|
||||
// 1.17.0-dev.2.0
|
||||
if (version.indexOf('(') != -1)
|
||||
version = version.substring(0, version.indexOf('(')).trim();
|
||||
if (version.indexOf(':') != -1)
|
||||
version = version.substring(version.indexOf(':') + 1).trim();
|
||||
|
||||
return version.replaceAll('"', "'");
|
||||
}
|
||||
|
||||
Future<String> getCurrentFlutterRepoCommit() {
|
||||
if (!dir('${flutterDirectory.path}/.git').existsSync()) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return inDirectory(flutterDirectory, () {
|
||||
return eval('git', <String>['rev-parse', 'HEAD']);
|
||||
});
|
||||
}
|
||||
|
||||
Future<DateTime> getFlutterRepoCommitTimestamp(String commit) {
|
||||
// git show -s --format=%at 4b546df7f0b3858aaaa56c4079e5be1ba91fbb65
|
||||
return inDirectory(flutterDirectory, () async {
|
||||
String unixTimestamp = await eval('git', <String>[
|
||||
'show',
|
||||
'-s',
|
||||
'--format=%at',
|
||||
commit,
|
||||
]);
|
||||
int secondsSinceEpoch = int.parse(unixTimestamp);
|
||||
return new DateTime.fromMillisecondsSinceEpoch(secondsSinceEpoch * 1000);
|
||||
});
|
||||
}
|
||||
|
||||
Future<Process> startProcess(String executable, List<String> arguments,
|
||||
{Map<String, String> env}) async {
|
||||
String command = '$executable ${arguments?.join(" ") ?? ""}';
|
||||
print('Executing: $command');
|
||||
Process proc = await Process.start(executable, arguments,
|
||||
environment: env, workingDirectory: cwd);
|
||||
ProcessInfo procInfo = new ProcessInfo(command, proc);
|
||||
_runningProcesses.add(procInfo);
|
||||
|
||||
proc.exitCode.then((_) {
|
||||
_runningProcesses.remove(procInfo);
|
||||
});
|
||||
|
||||
return proc;
|
||||
}
|
||||
|
||||
Future<Null> forceQuitRunningProcesses() async {
|
||||
if (_runningProcesses.isEmpty)
|
||||
return;
|
||||
|
||||
// Give normally quitting processes a chance to report their exit code.
|
||||
await new Future<Null>.delayed(const Duration(seconds: 1));
|
||||
|
||||
// Whatever's left, kill it.
|
||||
for (ProcessInfo p in _runningProcesses) {
|
||||
print('Force quitting process:\n$p');
|
||||
if (!p.process.kill()) {
|
||||
print('Failed to force quit process');
|
||||
}
|
||||
}
|
||||
_runningProcesses.clear();
|
||||
}
|
||||
|
||||
/// Executes a command and returns its exit code.
|
||||
Future<int> exec(String executable, List<String> arguments,
|
||||
{Map<String, String> env, bool canFail: false}) async {
|
||||
Process proc = await startProcess(executable, arguments, env: env);
|
||||
|
||||
proc.stdout
|
||||
.transform(UTF8.decoder)
|
||||
.transform(const LineSplitter())
|
||||
.listen(print);
|
||||
proc.stderr
|
||||
.transform(UTF8.decoder)
|
||||
.transform(const LineSplitter())
|
||||
.listen(stderr.writeln);
|
||||
|
||||
int exitCode = await proc.exitCode;
|
||||
|
||||
if (exitCode != 0 && !canFail)
|
||||
fail('Executable failed with exit code $exitCode.');
|
||||
|
||||
return exitCode;
|
||||
}
|
||||
|
||||
/// Executes a command and returns its standard output as a String.
|
||||
///
|
||||
/// Standard error is redirected to the current process' standard error stream.
|
||||
Future<String> eval(String executable, List<String> arguments,
|
||||
{Map<String, String> env, bool canFail: false}) async {
|
||||
Process proc = await startProcess(executable, arguments, env: env);
|
||||
proc.stderr.listen((List<int> data) {
|
||||
stderr.add(data);
|
||||
});
|
||||
String output = await UTF8.decodeStream(proc.stdout);
|
||||
int exitCode = await proc.exitCode;
|
||||
|
||||
if (exitCode != 0 && !canFail)
|
||||
fail('Executable failed with exit code $exitCode.');
|
||||
|
||||
return output.trimRight();
|
||||
}
|
||||
|
||||
Future<int> flutter(String command,
|
||||
{List<String> options: const <String>[], bool canFail: false}) {
|
||||
List<String> args = <String>[command]..addAll(options);
|
||||
return exec(path.join(flutterDirectory.path, 'bin', 'flutter'), args,
|
||||
canFail: canFail);
|
||||
}
|
||||
|
||||
String get dartBin =>
|
||||
path.join(flutterDirectory.path, 'bin', 'cache', 'dart-sdk', 'bin', 'dart');
|
||||
|
||||
Future<int> dart(List<String> args) => exec(dartBin, args);
|
||||
|
||||
Future<int> pub(String command) {
|
||||
return exec(
|
||||
path.join(flutterDirectory.path, 'bin', 'cache', 'dart-sdk', 'bin', 'pub'),
|
||||
<String>[command]);
|
||||
}
|
||||
|
||||
Future<dynamic> inDirectory(dynamic directory, Future<dynamic> action()) async {
|
||||
String previousCwd = cwd;
|
||||
try {
|
||||
cd(directory);
|
||||
return await action();
|
||||
} finally {
|
||||
cd(previousCwd);
|
||||
}
|
||||
}
|
||||
|
||||
void cd(dynamic directory) {
|
||||
Directory d;
|
||||
if (directory is String) {
|
||||
cwd = directory;
|
||||
d = dir(directory);
|
||||
} else if (directory is Directory) {
|
||||
cwd = directory.path;
|
||||
d = directory;
|
||||
} else {
|
||||
throw 'Unsupported type ${directory.runtimeType} of $directory';
|
||||
}
|
||||
|
||||
if (!d.existsSync())
|
||||
throw 'Cannot cd into directory that does not exist: $directory';
|
||||
}
|
||||
|
||||
Directory get flutterDirectory => dir('../..').absolute;
|
||||
|
||||
String requireEnvVar(String name) {
|
||||
String value = Platform.environment[name];
|
||||
|
||||
if (value == null) fail('$name environment variable is missing. Quitting.');
|
||||
|
||||
return value;
|
||||
}
|
||||
|
||||
dynamic/*=T*/ requireConfigProperty/*<T>*/(
|
||||
Map<String, dynamic/*<T>*/ > map, String propertyName) {
|
||||
if (!map.containsKey(propertyName))
|
||||
fail('Configuration property not found: $propertyName');
|
||||
|
||||
return map[propertyName];
|
||||
}
|
||||
|
||||
String jsonEncode(dynamic data) {
|
||||
return new JsonEncoder.withIndent(' ').convert(data) + '\n';
|
||||
}
|
||||
|
||||
Future<Null> getFlutter(String revision) async {
|
||||
section('Get Flutter!');
|
||||
|
||||
if (exists(flutterDirectory)) {
|
||||
rmTree(flutterDirectory);
|
||||
}
|
||||
|
||||
await inDirectory(flutterDirectory.parent, () async {
|
||||
await exec('git', <String>['clone', 'https://github.com/flutter/flutter.git']);
|
||||
});
|
||||
|
||||
await inDirectory(flutterDirectory, () async {
|
||||
await exec('git', <String>['checkout', revision]);
|
||||
});
|
||||
|
||||
await flutter('config', options: <String>['--no-analytics']);
|
||||
|
||||
section('flutter doctor');
|
||||
await flutter('doctor');
|
||||
|
||||
section('flutter update-packages');
|
||||
await flutter('update-packages');
|
||||
}
|
||||
|
||||
void checkNotNull(Object o1,
|
||||
[Object o2 = 1,
|
||||
Object o3 = 1,
|
||||
Object o4 = 1,
|
||||
Object o5 = 1,
|
||||
Object o6 = 1,
|
||||
Object o7 = 1,
|
||||
Object o8 = 1,
|
||||
Object o9 = 1,
|
||||
Object o10 = 1]) {
|
||||
if (o1 == null)
|
||||
throw 'o1 is null';
|
||||
if (o2 == null)
|
||||
throw 'o2 is null';
|
||||
if (o3 == null)
|
||||
throw 'o3 is null';
|
||||
if (o4 == null)
|
||||
throw 'o4 is null';
|
||||
if (o5 == null)
|
||||
throw 'o5 is null';
|
||||
if (o6 == null)
|
||||
throw 'o6 is null';
|
||||
if (o7 == null)
|
||||
throw 'o7 is null';
|
||||
if (o8 == null)
|
||||
throw 'o8 is null';
|
||||
if (o9 == null)
|
||||
throw 'o9 is null';
|
||||
if (o10 == null)
|
||||
throw 'o10 is null';
|
||||
}
|
||||
|
||||
/// Add benchmark values to a JSON results file.
|
||||
///
|
||||
/// If the file contains information about how long the benchmark took to run
|
||||
/// (a `time` field), then return that info.
|
||||
// TODO(yjbanov): move this data to __metadata__
|
||||
num addBuildInfo(File jsonFile,
|
||||
{num expected, String sdk, String commit, DateTime timestamp}) {
|
||||
Map<String, dynamic> json;
|
||||
|
||||
if (jsonFile.existsSync())
|
||||
json = JSON.decode(jsonFile.readAsStringSync());
|
||||
else
|
||||
json = <String, dynamic>{};
|
||||
|
||||
if (expected != null)
|
||||
json['expected'] = expected;
|
||||
if (sdk != null)
|
||||
json['sdk'] = sdk;
|
||||
if (commit != null)
|
||||
json['commit'] = commit;
|
||||
if (timestamp != null)
|
||||
json['timestamp'] = timestamp.millisecondsSinceEpoch;
|
||||
|
||||
jsonFile.writeAsStringSync(jsonEncode(json));
|
||||
|
||||
// Return the elapsed time of the benchmark (if any).
|
||||
return json['time'];
|
||||
}
|
||||
|
||||
/// Splits [from] into lines and selects those that contain [pattern].
|
||||
Iterable<String> grep(Pattern pattern, {@required String from}) {
|
||||
return from.split('\n').where((String line) {
|
||||
return line.contains(pattern);
|
||||
});
|
||||
}
|
||||
|
||||
/// Captures asynchronous stack traces thrown by [callback].
|
||||
///
|
||||
/// This is a convenience wrapper around [Chain] optimized for use with
|
||||
/// `async`/`await`.
|
||||
///
|
||||
/// Example:
|
||||
///
|
||||
/// try {
|
||||
/// await captureAsyncStacks(() { /* async things */ });
|
||||
/// } catch (error, chain) {
|
||||
///
|
||||
/// }
|
||||
Future<Null> runAndCaptureAsyncStacks(Future<Null> callback()) {
|
||||
Completer<Null> completer = new Completer<Null>();
|
||||
Chain.capture(() async {
|
||||
await callback();
|
||||
completer.complete();
|
||||
}, onError: (dynamic error, Chain chain) async {
|
||||
completer.completeError(error, chain);
|
||||
});
|
||||
return completer.future;
|
||||
}
|
||||
116
dev/devicelab/lib/tasks/analysis.dart
Normal file
116
dev/devicelab/lib/tasks/analysis.dart
Normal file
@ -0,0 +1,116 @@
|
||||
// Copyright (c) 2016 The Chromium Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file.
|
||||
|
||||
import 'dart:async';
|
||||
import 'dart:io';
|
||||
|
||||
import 'package:meta/meta.dart';
|
||||
import 'package:path/path.dart' as path;
|
||||
|
||||
import '../framework/benchmarks.dart';
|
||||
import '../framework/framework.dart';
|
||||
import '../framework/utils.dart';
|
||||
|
||||
TaskFunction createAnalyzerCliTest({
|
||||
@required String sdk,
|
||||
@required String commit,
|
||||
@required DateTime timestamp,
|
||||
}) {
|
||||
return new AnalyzerCliTask(sdk, commit, timestamp);
|
||||
}
|
||||
|
||||
TaskFunction createAnalyzerServerTest({
|
||||
@required String sdk,
|
||||
@required String commit,
|
||||
@required DateTime timestamp,
|
||||
}) {
|
||||
return new AnalyzerServerTask(sdk, commit, timestamp);
|
||||
}
|
||||
|
||||
abstract class AnalyzerTask {
|
||||
Benchmark benchmark;
|
||||
|
||||
Future<TaskResult> call() async {
|
||||
section(benchmark.name);
|
||||
await runBenchmark(benchmark, iterations: 3, warmUpBenchmark: true);
|
||||
return benchmark.bestResult;
|
||||
}
|
||||
}
|
||||
|
||||
class AnalyzerCliTask extends AnalyzerTask {
|
||||
AnalyzerCliTask(String sdk, String commit, DateTime timestamp) {
|
||||
this.benchmark = new FlutterAnalyzeBenchmark(sdk, commit, timestamp);
|
||||
}
|
||||
}
|
||||
|
||||
class AnalyzerServerTask extends AnalyzerTask {
|
||||
AnalyzerServerTask(String sdk, String commit, DateTime timestamp) {
|
||||
this.benchmark = new FlutterAnalyzeAppBenchmark(sdk, commit, timestamp);
|
||||
}
|
||||
}
|
||||
|
||||
class FlutterAnalyzeBenchmark extends Benchmark {
|
||||
FlutterAnalyzeBenchmark(this.sdk, this.commit, this.timestamp)
|
||||
: super('flutter analyze --flutter-repo');
|
||||
|
||||
final String sdk;
|
||||
final String commit;
|
||||
final DateTime timestamp;
|
||||
|
||||
File get benchmarkFile =>
|
||||
file(path.join(flutterDirectory.path, 'analysis_benchmark.json'));
|
||||
|
||||
@override
|
||||
TaskResult get lastResult => new TaskResult.successFromFile(benchmarkFile);
|
||||
|
||||
@override
|
||||
Future<num> run() async {
|
||||
rm(benchmarkFile);
|
||||
await inDirectory(flutterDirectory, () async {
|
||||
await flutter('analyze', options: <String>[
|
||||
'--flutter-repo',
|
||||
'--benchmark',
|
||||
]);
|
||||
});
|
||||
return addBuildInfo(benchmarkFile,
|
||||
timestamp: timestamp, expected: 25.0, sdk: sdk, commit: commit);
|
||||
}
|
||||
}
|
||||
|
||||
class FlutterAnalyzeAppBenchmark extends Benchmark {
|
||||
FlutterAnalyzeAppBenchmark(this.sdk, this.commit, this.timestamp)
|
||||
: super('analysis server mega_gallery');
|
||||
|
||||
final String sdk;
|
||||
final String commit;
|
||||
final DateTime timestamp;
|
||||
|
||||
@override
|
||||
TaskResult get lastResult => new TaskResult.successFromFile(benchmarkFile);
|
||||
|
||||
Directory get megaDir => dir(
|
||||
path.join(flutterDirectory.path, 'dev/benchmarks/mega_gallery'));
|
||||
File get benchmarkFile =>
|
||||
file(path.join(megaDir.path, 'analysis_benchmark.json'));
|
||||
|
||||
@override
|
||||
Future<Null> init() {
|
||||
return inDirectory(flutterDirectory, () async {
|
||||
await dart(<String>['dev/tools/mega_gallery.dart']);
|
||||
});
|
||||
}
|
||||
|
||||
@override
|
||||
Future<num> run() async {
|
||||
rm(benchmarkFile);
|
||||
await inDirectory(megaDir, () async {
|
||||
await flutter('analyze', options: <String>[
|
||||
'--watch',
|
||||
'--benchmark',
|
||||
]);
|
||||
});
|
||||
return addBuildInfo(benchmarkFile,
|
||||
timestamp: timestamp, expected: 10.0, sdk: sdk, commit: commit);
|
||||
}
|
||||
}
|
||||
58
dev/devicelab/lib/tasks/gallery.dart
Normal file
58
dev/devicelab/lib/tasks/gallery.dart
Normal file
@ -0,0 +1,58 @@
|
||||
// Copyright 2016 The Chromium Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file.
|
||||
|
||||
import 'dart:async';
|
||||
import 'dart:convert';
|
||||
import 'dart:io';
|
||||
|
||||
import 'package:meta/meta.dart';
|
||||
|
||||
import '../framework/adb.dart';
|
||||
import '../framework/framework.dart';
|
||||
import '../framework/utils.dart';
|
||||
|
||||
TaskFunction createGalleryTransitionTest({ @required bool ios: false }) {
|
||||
return new GalleryTransitionTest(ios: ios);
|
||||
}
|
||||
|
||||
class GalleryTransitionTest {
|
||||
GalleryTransitionTest({ this.ios });
|
||||
|
||||
final bool ios;
|
||||
|
||||
Future<TaskResult> call() async {
|
||||
String deviceId = await getUnlockedDeviceId(ios: ios);
|
||||
Directory galleryDirectory =
|
||||
dir('${flutterDirectory.path}/examples/flutter_gallery');
|
||||
await inDirectory(galleryDirectory, () async {
|
||||
await pub('get');
|
||||
|
||||
if (ios) {
|
||||
// This causes an Xcode project to be created.
|
||||
await flutter('build', options: <String>['ios', '--profile']);
|
||||
}
|
||||
|
||||
await flutter('drive', options: <String>[
|
||||
'--profile',
|
||||
'--trace-startup',
|
||||
'-t',
|
||||
'test_driver/transitions_perf.dart',
|
||||
'-d',
|
||||
deviceId,
|
||||
]);
|
||||
});
|
||||
|
||||
// Route paths contains slashes, which Firebase doesn't accept in keys, so we
|
||||
// remove them.
|
||||
Map<String, dynamic> original = JSON.decode(file(
|
||||
'${galleryDirectory.path}/build/transition_durations.timeline.json')
|
||||
.readAsStringSync());
|
||||
Map<String, dynamic> clean = new Map<String, dynamic>.fromIterable(
|
||||
original.keys,
|
||||
key: (String key) => key.replaceAll('/', ''),
|
||||
value: (String key) => original[key]);
|
||||
|
||||
return new TaskResult.success(clean);
|
||||
}
|
||||
}
|
||||
164
dev/devicelab/lib/tasks/perf_tests.dart
Normal file
164
dev/devicelab/lib/tasks/perf_tests.dart
Normal file
@ -0,0 +1,164 @@
|
||||
// Copyright (c) 2016 The Chromium Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file.
|
||||
|
||||
import 'dart:async';
|
||||
import 'dart:convert' show JSON;
|
||||
|
||||
import 'package:meta/meta.dart';
|
||||
|
||||
import '../framework/adb.dart';
|
||||
import '../framework/framework.dart';
|
||||
import '../framework/utils.dart';
|
||||
|
||||
TaskFunction createComplexLayoutScrollPerfTest({ @required bool ios: false }) {
|
||||
return new PerfTest(
|
||||
'${flutterDirectory.path}/dev/benchmarks/complex_layout',
|
||||
'test_driver/scroll_perf.dart',
|
||||
'complex_layout_scroll_perf',
|
||||
ios: ios
|
||||
);
|
||||
}
|
||||
|
||||
TaskFunction createFlutterGalleryStartupTest({ bool ios: false }) {
|
||||
return new StartupTest(
|
||||
'${flutterDirectory.path}/examples/flutter_gallery',
|
||||
ios: ios
|
||||
);
|
||||
}
|
||||
|
||||
TaskFunction createComplexLayoutStartupTest({ bool ios: false }) {
|
||||
return new StartupTest(
|
||||
'${flutterDirectory.path}/dev/benchmarks/complex_layout',
|
||||
ios: ios
|
||||
);
|
||||
}
|
||||
|
||||
TaskFunction createFlutterGalleryBuildTest() {
|
||||
return new BuildTest('${flutterDirectory.path}/examples/flutter_gallery');
|
||||
}
|
||||
|
||||
TaskFunction createComplexLayoutBuildTest() {
|
||||
return new BuildTest('${flutterDirectory.path}/dev/benchmarks/complex_layout');
|
||||
}
|
||||
|
||||
/// Measure application startup performance.
|
||||
class StartupTest {
|
||||
static const Duration _startupTimeout = const Duration(minutes: 2);
|
||||
|
||||
StartupTest(this.testDirectory, { this.ios });
|
||||
|
||||
final String testDirectory;
|
||||
final bool ios;
|
||||
|
||||
Future<TaskResult> call() async {
|
||||
return await inDirectory(testDirectory, () async {
|
||||
String deviceId = await getUnlockedDeviceId(ios: ios);
|
||||
await pub('get');
|
||||
|
||||
if (ios) {
|
||||
// This causes an Xcode project to be created.
|
||||
await flutter('build', options: <String>['ios', '--profile']);
|
||||
}
|
||||
|
||||
await flutter('run', options: <String>[
|
||||
'--profile',
|
||||
'--trace-startup',
|
||||
'-d',
|
||||
deviceId,
|
||||
]).timeout(_startupTimeout);
|
||||
Map<String, dynamic> data = JSON.decode(file('$testDirectory/build/start_up_info.json').readAsStringSync());
|
||||
return new TaskResult.success(data, benchmarkScoreKeys: <String>[
|
||||
'engineEnterTimestampMicros',
|
||||
'timeToFirstFrameMicros',
|
||||
]);
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
/// Measures application runtime performance, specifically per-frame
|
||||
/// performance.
|
||||
class PerfTest {
|
||||
|
||||
PerfTest(this.testDirectory, this.testTarget, this.timelineFileName, { this.ios });
|
||||
|
||||
final String testDirectory;
|
||||
final String testTarget;
|
||||
final String timelineFileName;
|
||||
final bool ios;
|
||||
|
||||
Future<TaskResult> call() {
|
||||
return inDirectory(testDirectory, () async {
|
||||
String deviceId = await getUnlockedDeviceId(ios: ios);
|
||||
await pub('get');
|
||||
|
||||
if (ios) {
|
||||
// This causes an Xcode project to be created.
|
||||
await flutter('build', options: <String>['ios', '--profile']);
|
||||
}
|
||||
|
||||
await flutter('drive', options: <String>[
|
||||
'-v',
|
||||
'--profile',
|
||||
'--trace-startup', // Enables "endless" timeline event buffering.
|
||||
'-t',
|
||||
testTarget,
|
||||
'-d',
|
||||
deviceId,
|
||||
]);
|
||||
Map<String, dynamic> data = JSON.decode(file('$testDirectory/build/$timelineFileName.timeline_summary.json').readAsStringSync());
|
||||
return new TaskResult.success(data, benchmarkScoreKeys: <String>[
|
||||
'average_frame_build_time_millis',
|
||||
'worst_frame_build_time_millis',
|
||||
'missed_frame_build_budget_count',
|
||||
]);
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
class BuildTest {
|
||||
|
||||
BuildTest(this.testDirectory);
|
||||
|
||||
final String testDirectory;
|
||||
|
||||
Future<TaskResult> call() async {
|
||||
return await inDirectory(testDirectory, () async {
|
||||
Adb device = await adb();
|
||||
await device.unlock();
|
||||
await pub('get');
|
||||
|
||||
Stopwatch watch = new Stopwatch()..start();
|
||||
await flutter('build', options: <String>[
|
||||
'aot',
|
||||
'--profile',
|
||||
'--no-pub',
|
||||
'--target-platform', 'android-arm' // Generate blobs instead of assembly.
|
||||
]);
|
||||
watch.stop();
|
||||
|
||||
int vmisolateSize = file("$testDirectory/build/aot/snapshot_aot_vmisolate").lengthSync();
|
||||
int isolateSize = file("$testDirectory/build/aot/snapshot_aot_isolate").lengthSync();
|
||||
int instructionsSize = file("$testDirectory/build/aot/snapshot_aot_instr").lengthSync();
|
||||
int rodataSize = file("$testDirectory/build/aot/snapshot_aot_rodata").lengthSync();
|
||||
int totalSize = vmisolateSize + isolateSize + instructionsSize + rodataSize;
|
||||
|
||||
Map<String, dynamic> data = <String, dynamic>{
|
||||
'aot_snapshot_build_millis': watch.elapsedMilliseconds,
|
||||
'aot_snapshot_size_vmisolate': vmisolateSize,
|
||||
'aot_snapshot_size_isolate': isolateSize,
|
||||
'aot_snapshot_size_instructions': instructionsSize,
|
||||
'aot_snapshot_size_rodata': rodataSize,
|
||||
'aot_snapshot_size_total': totalSize,
|
||||
};
|
||||
return new TaskResult.success(data, benchmarkScoreKeys: <String>[
|
||||
'aot_snapshot_build_millis',
|
||||
'aot_snapshot_size_vmisolate',
|
||||
'aot_snapshot_size_isolate',
|
||||
'aot_snapshot_size_instructions',
|
||||
'aot_snapshot_size_rodata',
|
||||
'aot_snapshot_size_total',
|
||||
]);
|
||||
});
|
||||
}
|
||||
}
|
||||
75
dev/devicelab/lib/tasks/refresh.dart
Normal file
75
dev/devicelab/lib/tasks/refresh.dart
Normal file
@ -0,0 +1,75 @@
|
||||
// Copyright (c) 2016 The Chromium Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file.
|
||||
|
||||
import 'dart:async';
|
||||
import 'dart:io';
|
||||
|
||||
import 'package:path/path.dart' as path;
|
||||
|
||||
import '../framework/adb.dart';
|
||||
import '../framework/benchmarks.dart';
|
||||
import '../framework/framework.dart';
|
||||
import '../framework/utils.dart';
|
||||
|
||||
TaskFunction createRefreshTest({ String commit, DateTime timestamp }) =>
|
||||
new EditRefreshTask(commit, timestamp);
|
||||
|
||||
class EditRefreshTask {
|
||||
EditRefreshTask(this.commit, this.timestamp) {
|
||||
assert(commit != null);
|
||||
assert(timestamp != null);
|
||||
}
|
||||
|
||||
final String commit;
|
||||
final DateTime timestamp;
|
||||
|
||||
Future<TaskResult> call() async {
|
||||
Adb device = await adb();
|
||||
await device.unlock();
|
||||
Benchmark benchmark = new EditRefreshBenchmark(commit, timestamp);
|
||||
section(benchmark.name);
|
||||
await runBenchmark(benchmark, iterations: 3, warmUpBenchmark: true);
|
||||
return benchmark.bestResult;
|
||||
}
|
||||
}
|
||||
|
||||
class EditRefreshBenchmark extends Benchmark {
|
||||
EditRefreshBenchmark(this.commit, this.timestamp) : super('edit refresh');
|
||||
|
||||
final String commit;
|
||||
final DateTime timestamp;
|
||||
|
||||
Directory get megaDir => dir(
|
||||
path.join(flutterDirectory.path, 'dev/benchmarks/mega_gallery'));
|
||||
File get benchmarkFile =>
|
||||
file(path.join(megaDir.path, 'refresh_benchmark.json'));
|
||||
|
||||
@override
|
||||
TaskResult get lastResult => new TaskResult.successFromFile(benchmarkFile);
|
||||
|
||||
@override
|
||||
Future<Null> init() {
|
||||
return inDirectory(flutterDirectory, () async {
|
||||
await dart(<String>['dev/tools/mega_gallery.dart']);
|
||||
});
|
||||
}
|
||||
|
||||
@override
|
||||
Future<num> run() async {
|
||||
Adb device = await adb();
|
||||
rm(benchmarkFile);
|
||||
int exitCode = await inDirectory(megaDir, () async {
|
||||
return await flutter('run',
|
||||
options: <String>['-d', device.deviceId, '--benchmark'],
|
||||
canFail: true);
|
||||
});
|
||||
if (exitCode != 0) return new Future<num>.error(exitCode);
|
||||
return addBuildInfo(
|
||||
benchmarkFile,
|
||||
timestamp: timestamp,
|
||||
expected: 200,
|
||||
commit: commit,
|
||||
);
|
||||
}
|
||||
}
|
||||
38
dev/devicelab/lib/tasks/size_tests.dart
Normal file
38
dev/devicelab/lib/tasks/size_tests.dart
Normal file
@ -0,0 +1,38 @@
|
||||
// Copyright 2016 The Chromium Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file.
|
||||
|
||||
import 'dart:io';
|
||||
|
||||
import '../framework/framework.dart';
|
||||
import '../framework/utils.dart';
|
||||
|
||||
TaskFunction createBasicMaterialAppSizeTest() {
|
||||
return () async {
|
||||
const String sampleAppName = 'sample_flutter_app';
|
||||
Directory sampleDir = dir('${Directory.systemTemp.path}/$sampleAppName');
|
||||
|
||||
if (await sampleDir.exists())
|
||||
rmTree(sampleDir);
|
||||
|
||||
int apkSizeInBytes;
|
||||
|
||||
await inDirectory(Directory.systemTemp, () async {
|
||||
await flutter('create', options: <String>[sampleAppName]);
|
||||
|
||||
if (!(await sampleDir.exists()))
|
||||
throw 'Failed to create sample Flutter app in ${sampleDir.path}';
|
||||
|
||||
await inDirectory(sampleDir, () async {
|
||||
await pub('get');
|
||||
await flutter('build', options: <String>['clean']);
|
||||
await flutter('build', options: <String>['apk', '--release']);
|
||||
apkSizeInBytes = await file('${sampleDir.path}/build/app.apk').length();
|
||||
});
|
||||
});
|
||||
|
||||
return new TaskResult.success(
|
||||
<String, dynamic>{'release_size_in_bytes': apkSizeInBytes},
|
||||
benchmarkScoreKeys: <String>['release_size_in_bytes']);
|
||||
};
|
||||
}
|
||||
133
dev/devicelab/manifest.yaml
Normal file
133
dev/devicelab/manifest.yaml
Normal file
@ -0,0 +1,133 @@
|
||||
# Describes the tasks we run in the continuous integration (CI) environment.
|
||||
#
|
||||
# Cocoon[1] uses this file to generate a checklist of tasks to be performed for
|
||||
# every master commit.
|
||||
#
|
||||
# [1] github.com/flutter/cocoon
|
||||
|
||||
# CI tasks.
|
||||
#
|
||||
# Each key in this dictionary is the unique name of a task, which also
|
||||
# corresponds to a file in the "bin/" directory that the task runner will run.
|
||||
#
|
||||
# Due to historic reasons that may go away at some point, the suffix of the task
|
||||
# name is significant. It is used by the dashboard to pick the right HTML
|
||||
# template to display the results in a card. If you use a known name suffix also
|
||||
# make sure that your task outputs data in the expected format for that card.
|
||||
#
|
||||
# Known suffixes:
|
||||
#
|
||||
# __analysis_time:
|
||||
# Analyzer performance benchmarks.
|
||||
# __refresh_time:
|
||||
# Edit refresh cycle benchmarks.
|
||||
# __start_up:
|
||||
# Application startup speed benchmarks.
|
||||
# __timeline_summary:
|
||||
# Per-frame timings and missed/average/total counts.
|
||||
# __transition_perf:
|
||||
# Flutter Gallery app transitions benchmark.
|
||||
# __size:
|
||||
# Application size benchmarks.
|
||||
|
||||
tasks:
|
||||
# Deviceless tests
|
||||
|
||||
# TODO: make these not require "has-android-device"; it is only there to
|
||||
# ensure we have the Android SDK.
|
||||
|
||||
flutter_gallery__build:
|
||||
description: >
|
||||
Collects various performance metrics from AOT builds of the Flutter
|
||||
Gallery.
|
||||
stage: devicelab
|
||||
required_agent_capabilities: ["has-android-device"]
|
||||
|
||||
complex_layout__build:
|
||||
description: >
|
||||
Collects various performance metrics from AOT builds of the Complex
|
||||
Layout sample app.
|
||||
stage: devicelab
|
||||
required_agent_capabilities: ["has-android-device"]
|
||||
|
||||
basic_material_app__size:
|
||||
description: >
|
||||
Measures the APK/IPA sizes of a basic material app.
|
||||
stage: devicelab
|
||||
required_agent_capabilities: ["has-android-device"]
|
||||
|
||||
analyzer_cli__analysis_time:
|
||||
description: >
|
||||
Measures the speed of analyzing Flutter itself in batch mode.
|
||||
stage: devicelab
|
||||
required_agent_capabilities: ["has-android-device"]
|
||||
|
||||
analyzer_server__analysis_time:
|
||||
description: >
|
||||
Measures the speed of analyzing Flutter itself in server mode.
|
||||
stage: devicelab
|
||||
required_agent_capabilities: ["has-android-device"]
|
||||
|
||||
|
||||
# Android on-device tests
|
||||
|
||||
complex_layout_scroll_perf__timeline_summary:
|
||||
description: >
|
||||
Measures the runtime performance of the Complex Layout sample app on
|
||||
Android.
|
||||
stage: devicelab
|
||||
required_agent_capabilities: ["has-android-device"]
|
||||
|
||||
flutter_gallery__start_up:
|
||||
description: >
|
||||
Measures the startup time of the Flutter Gallery app on Android.
|
||||
stage: devicelab
|
||||
required_agent_capabilities: ["has-android-device"]
|
||||
|
||||
complex_layout__start_up:
|
||||
description: >
|
||||
Measures the startup time of the Complex Layout sample app on Android.
|
||||
stage: devicelab
|
||||
required_agent_capabilities: ["has-android-device"]
|
||||
|
||||
flutter_gallery__transition_perf:
|
||||
description: >
|
||||
Measures the performance of screen transitions in Flutter Gallery on
|
||||
Android.
|
||||
stage: devicelab
|
||||
required_agent_capabilities: ["has-android-device"]
|
||||
|
||||
mega_gallery__refresh_time:
|
||||
description: >
|
||||
Measures AOT snapshot rebuild performance on a generated large app.
|
||||
stage: devicelab
|
||||
required_agent_capabilities: ["has-android-device"]
|
||||
|
||||
|
||||
# iOS on-device tests
|
||||
|
||||
complex_layout_scroll_perf_ios__timeline_summary:
|
||||
description: >
|
||||
Measures the runtime performance of the Complex Layout sample app on
|
||||
iOS.
|
||||
stage: devicelab_ios
|
||||
required_agent_capabilities: ["has-ios-device"]
|
||||
|
||||
flutter_gallery_ios__start_up:
|
||||
stage: devicelab_ios
|
||||
required_agent_capabilities: ["has-ios-device"]
|
||||
description: >
|
||||
Measures the startup time of the Flutter Gallery app on iOS.
|
||||
|
||||
complex_layout_ios__start_up:
|
||||
description: >
|
||||
Measures the startup time of the Complex Layout sample app on iOS.
|
||||
stage: devicelab_ios
|
||||
required_agent_capabilities: ["has-ios-device"]
|
||||
|
||||
flutter_gallery_ios__transition_perf:
|
||||
stage: devicelab_ios
|
||||
required_agent_capabilities: ["has-ios-device"]
|
||||
description: >
|
||||
Measures the performance of screen transitions in Flutter Gallery on
|
||||
iOS.
|
||||
22
dev/devicelab/pubspec.yaml
Normal file
22
dev/devicelab/pubspec.yaml
Normal file
@ -0,0 +1,22 @@
|
||||
name: flutter_devicelab
|
||||
version: 0.0.1
|
||||
author: Flutter Authors <flutter-dev@googlegroups.com>
|
||||
description: Flutter continuous integration performance and correctness tests.
|
||||
homepage: https://github.com/flutter/flutter
|
||||
|
||||
environment:
|
||||
sdk: '>=1.12.0 <2.0.0'
|
||||
|
||||
dependencies:
|
||||
args: ^0.13.4
|
||||
meta: ^1.0.3
|
||||
path: ^1.3.0
|
||||
stack_trace: ^1.4.0
|
||||
vm_service_client: '^0.2.0'
|
||||
|
||||
# See packages/flutter_test/pubspec.yaml for why we're pinning this version
|
||||
analyzer: 0.28.2-alpha.0
|
||||
|
||||
dev_dependencies:
|
||||
# See packages/flutter_test/pubspec.yaml for why we're pinning this version
|
||||
test: 0.12.15+4
|
||||
190
dev/devicelab/test/adb_test.dart
Normal file
190
dev/devicelab/test/adb_test.dart
Normal file
@ -0,0 +1,190 @@
|
||||
// Copyright 2016 The Chromium Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file.
|
||||
|
||||
import 'dart:async';
|
||||
|
||||
import 'package:test/test.dart';
|
||||
import 'package:collection/collection.dart';
|
||||
|
||||
import 'package:flutter_devicelab/framework/adb.dart';
|
||||
|
||||
void main() {
|
||||
group('adb', () {
|
||||
Adb device;
|
||||
|
||||
setUp(() {
|
||||
FakeAdb.resetLog();
|
||||
adb = null;
|
||||
device = new FakeAdb();
|
||||
});
|
||||
|
||||
tearDown(() {
|
||||
adb = realAdbGetter;
|
||||
});
|
||||
|
||||
group('isAwake/isAsleep', () {
|
||||
test('reads Awake', () async {
|
||||
FakeAdb.pretendAwake();
|
||||
expect(await device.isAwake(), isTrue);
|
||||
expect(await device.isAsleep(), isFalse);
|
||||
});
|
||||
|
||||
test('reads Asleep', () async {
|
||||
FakeAdb.pretendAsleep();
|
||||
expect(await device.isAwake(), isFalse);
|
||||
expect(await device.isAsleep(), isTrue);
|
||||
});
|
||||
});
|
||||
|
||||
group('togglePower', () {
|
||||
test('sends power event', () async {
|
||||
await device.togglePower();
|
||||
expectLog(<CommandArgs>[
|
||||
cmd(command: 'input', arguments: <String>['keyevent', '26']),
|
||||
]);
|
||||
});
|
||||
});
|
||||
|
||||
group('wakeUp', () {
|
||||
test('when awake', () async {
|
||||
FakeAdb.pretendAwake();
|
||||
await device.wakeUp();
|
||||
expectLog(<CommandArgs>[
|
||||
cmd(command: 'dumpsys', arguments: <String>['power']),
|
||||
]);
|
||||
});
|
||||
|
||||
test('when asleep', () async {
|
||||
FakeAdb.pretendAsleep();
|
||||
await device.wakeUp();
|
||||
expectLog(<CommandArgs>[
|
||||
cmd(command: 'dumpsys', arguments: <String>['power']),
|
||||
cmd(command: 'input', arguments: <String>['keyevent', '26']),
|
||||
]);
|
||||
});
|
||||
});
|
||||
|
||||
group('sendToSleep', () {
|
||||
test('when asleep', () async {
|
||||
FakeAdb.pretendAsleep();
|
||||
await device.sendToSleep();
|
||||
expectLog(<CommandArgs>[
|
||||
cmd(command: 'dumpsys', arguments: <String>['power']),
|
||||
]);
|
||||
});
|
||||
|
||||
test('when awake', () async {
|
||||
FakeAdb.pretendAwake();
|
||||
await device.sendToSleep();
|
||||
expectLog(<CommandArgs>[
|
||||
cmd(command: 'dumpsys', arguments: <String>['power']),
|
||||
cmd(command: 'input', arguments: <String>['keyevent', '26']),
|
||||
]);
|
||||
});
|
||||
});
|
||||
|
||||
group('unlock', () {
|
||||
test('sends unlock event', () async {
|
||||
FakeAdb.pretendAwake();
|
||||
await device.unlock();
|
||||
expectLog(<CommandArgs>[
|
||||
cmd(command: 'dumpsys', arguments: <String>['power']),
|
||||
cmd(command: 'input', arguments: <String>['keyevent', '82']),
|
||||
]);
|
||||
});
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
void expectLog(List<CommandArgs> log) {
|
||||
expect(FakeAdb.commandLog, log);
|
||||
}
|
||||
|
||||
CommandArgs cmd({ String command, List<String> arguments, Map<String, String> env }) => new CommandArgs(
|
||||
command: command,
|
||||
arguments: arguments,
|
||||
env: env
|
||||
);
|
||||
|
||||
typedef dynamic ExitErrorFactory();
|
||||
|
||||
class CommandArgs {
|
||||
CommandArgs({ this.command, this.arguments, this.env });
|
||||
|
||||
final String command;
|
||||
final List<String> arguments;
|
||||
final Map<String, String> env;
|
||||
|
||||
@override
|
||||
String toString() => 'CommandArgs(command: $command, arguments: $arguments, env: $env)';
|
||||
|
||||
@override
|
||||
bool operator==(Object other) {
|
||||
if (other.runtimeType != CommandArgs)
|
||||
return false;
|
||||
|
||||
CommandArgs otherCmd = other;
|
||||
return otherCmd.command == this.command &&
|
||||
const ListEquality<String>().equals(otherCmd.arguments, this.arguments) &&
|
||||
const MapEquality<String, String>().equals(otherCmd.env, this.env);
|
||||
}
|
||||
|
||||
@override
|
||||
int get hashCode => 17 * (17 * command.hashCode + _hashArguments) + _hashEnv;
|
||||
|
||||
int get _hashArguments => arguments != null
|
||||
? const ListEquality<String>().hash(arguments)
|
||||
: null.hashCode;
|
||||
|
||||
int get _hashEnv => env != null
|
||||
? const MapEquality<String, String>().hash(env)
|
||||
: null.hashCode;
|
||||
}
|
||||
|
||||
class FakeAdb extends Adb {
|
||||
FakeAdb({ String deviceId: null }) : super(deviceId: deviceId);
|
||||
|
||||
static String output = '';
|
||||
static ExitErrorFactory exitErrorFactory = () => null;
|
||||
|
||||
static List<CommandArgs> commandLog = <CommandArgs>[];
|
||||
|
||||
static void resetLog() {
|
||||
commandLog.clear();
|
||||
}
|
||||
|
||||
static void pretendAwake() {
|
||||
output = '''
|
||||
mWakefulness=Awake
|
||||
''';
|
||||
}
|
||||
|
||||
static void pretendAsleep() {
|
||||
output = '''
|
||||
mWakefulness=Asleep
|
||||
''';
|
||||
}
|
||||
|
||||
@override
|
||||
Future<String> shellEval(String command, List<String> arguments, {Map<String, String> env}) async {
|
||||
commandLog.add(new CommandArgs(
|
||||
command: command,
|
||||
arguments: arguments,
|
||||
env: env
|
||||
));
|
||||
return output;
|
||||
}
|
||||
|
||||
@override
|
||||
Future<Null> shellExec(String command, List<String> arguments, {Map<String, String> env}) async {
|
||||
commandLog.add(new CommandArgs(
|
||||
command: command,
|
||||
arguments: arguments,
|
||||
env: env
|
||||
));
|
||||
dynamic exitError = exitErrorFactory();
|
||||
if (exitError != null)
|
||||
throw exitError;
|
||||
}
|
||||
}
|
||||
13
dev/devicelab/test/all.dart
Normal file
13
dev/devicelab/test/all.dart
Normal file
@ -0,0 +1,13 @@
|
||||
// Copyright 2016 The Chromium Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file.
|
||||
|
||||
import 'adb_test.dart' as adb_test;
|
||||
import 'manifest_test.dart' as manifest_test;
|
||||
import 'run_test.dart' as run_test;
|
||||
|
||||
void main() {
|
||||
adb_test.main();
|
||||
manifest_test.main();
|
||||
run_test.main();
|
||||
}
|
||||
143
dev/devicelab/test/manifest_test.dart
Normal file
143
dev/devicelab/test/manifest_test.dart
Normal file
@ -0,0 +1,143 @@
|
||||
// Copyright 2016 The Chromium Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file.
|
||||
|
||||
import 'package:test/test.dart';
|
||||
|
||||
import 'package:flutter_devicelab/framework/manifest.dart';
|
||||
|
||||
void main() {
|
||||
group('production manifest', () {
|
||||
test('must be valid', () {
|
||||
Manifest manifest = loadTaskManifest();
|
||||
expect(manifest.tasks, isNotEmpty);
|
||||
|
||||
ManifestTask task = manifest.tasks.firstWhere((ManifestTask task) => task.name == 'flutter_gallery__start_up');
|
||||
expect(task.description, 'Measures the startup time of the Flutter Gallery app on Android.\n');
|
||||
expect(task.stage, 'devicelab');
|
||||
expect(task.requiredAgentCapabilities, <String>['has-android-device']);
|
||||
});
|
||||
});
|
||||
|
||||
group('manifest parser', () {
|
||||
void testManifestError(
|
||||
String testDescription,
|
||||
String errorMessage,
|
||||
String yaml,
|
||||
) {
|
||||
test(testDescription, () {
|
||||
try {
|
||||
loadTaskManifest(yaml);
|
||||
} on ManifestError catch(error) {
|
||||
expect(error.message, errorMessage);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
testManifestError(
|
||||
'invalid top-level type',
|
||||
'Manifest must be a dictionary but was YamlScalar: null',
|
||||
'',
|
||||
);
|
||||
|
||||
testManifestError(
|
||||
'invalid top-level key',
|
||||
'Unrecognized property "bad" in manifest. Allowed properties: tasks',
|
||||
'''
|
||||
bad:
|
||||
key: yes
|
||||
''',
|
||||
);
|
||||
|
||||
testManifestError(
|
||||
'invalid tasks list type',
|
||||
'Value of "tasks" must be a dictionary but was YamlList: [a, b]',
|
||||
'''
|
||||
tasks:
|
||||
- a
|
||||
- b
|
||||
'''
|
||||
);
|
||||
|
||||
testManifestError(
|
||||
'invalid task name type',
|
||||
'Task name must be a string but was int: 1',
|
||||
'''
|
||||
tasks:
|
||||
1: 2
|
||||
'''
|
||||
);
|
||||
|
||||
testManifestError(
|
||||
'invalid task type',
|
||||
'Value of task "foo" must be a dictionary but was int: 2',
|
||||
'''
|
||||
tasks:
|
||||
foo: 2
|
||||
'''
|
||||
);
|
||||
|
||||
testManifestError(
|
||||
'invalid task property',
|
||||
'Unrecognized property "bar" in Value of task "foo". Allowed properties: description, stage, required_agent_capabilities',
|
||||
'''
|
||||
tasks:
|
||||
foo:
|
||||
bar: 2
|
||||
'''
|
||||
);
|
||||
|
||||
testManifestError(
|
||||
'invalid required_agent_capabilities type',
|
||||
'required_agent_capabilities must be a list but was int: 1',
|
||||
'''
|
||||
tasks:
|
||||
foo:
|
||||
required_agent_capabilities: 1
|
||||
'''
|
||||
);
|
||||
|
||||
testManifestError(
|
||||
'invalid required_agent_capabilities element type',
|
||||
'required_agent_capabilities[0] must be a string but was int: 1',
|
||||
'''
|
||||
tasks:
|
||||
foo:
|
||||
required_agent_capabilities: [1]
|
||||
'''
|
||||
);
|
||||
|
||||
testManifestError(
|
||||
'missing description',
|
||||
'Task description must not be empty in task "foo".',
|
||||
'''
|
||||
tasks:
|
||||
foo:
|
||||
required_agent_capabilities: ["a"]
|
||||
'''
|
||||
);
|
||||
|
||||
testManifestError(
|
||||
'missing stage',
|
||||
'Task stage must not be empty in task "foo".',
|
||||
'''
|
||||
tasks:
|
||||
foo:
|
||||
description: b
|
||||
required_agent_capabilities: ["a"]
|
||||
'''
|
||||
);
|
||||
|
||||
testManifestError(
|
||||
'missing stage',
|
||||
'requiredAgentCapabilities must not be empty in task "foo".',
|
||||
'''
|
||||
tasks:
|
||||
foo:
|
||||
description: b
|
||||
stage: c
|
||||
required_agent_capabilities: []
|
||||
'''
|
||||
);
|
||||
});
|
||||
}
|
||||
50
dev/devicelab/test/run_test.dart
Normal file
50
dev/devicelab/test/run_test.dart
Normal file
@ -0,0 +1,50 @@
|
||||
// Copyright 2016 The Chromium Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file.
|
||||
|
||||
import 'dart:async';
|
||||
import 'dart:io';
|
||||
|
||||
import 'package:test/test.dart';
|
||||
|
||||
void main() {
|
||||
group('run.dart script', () {
|
||||
Future<int> runScript(List<String> testNames) async {
|
||||
List<String> options = <String>['bin/run.dart'];
|
||||
for (String testName in testNames) {
|
||||
options..addAll(<String>['-t', testName]);
|
||||
}
|
||||
Process scriptProcess = await Process.start(
|
||||
'../../bin/cache/dart-sdk/bin/dart',
|
||||
options,
|
||||
);
|
||||
return scriptProcess.exitCode;
|
||||
}
|
||||
|
||||
test('Exits with code 0 when succeeds', () async {
|
||||
expect(await runScript(<String>['smoke_test_success']), 0);
|
||||
});
|
||||
|
||||
test('Exits with code 1 when task throws', () async {
|
||||
expect(await runScript(<String>['smoke_test_throws']), 1);
|
||||
});
|
||||
|
||||
test('Exits with code 1 when fails', () async {
|
||||
expect(await runScript(<String>['smoke_test_failure']), 1);
|
||||
});
|
||||
|
||||
test('Exits with code 1 when fails to connect', () async {
|
||||
expect(await runScript(<String>['smoke_test_setup_failure']), 1);
|
||||
});
|
||||
|
||||
test('Exits with code 1 when results are mixed', () async {
|
||||
expect(
|
||||
await runScript(<String>[
|
||||
'smoke_test_failure',
|
||||
'smoke_test_success',
|
||||
]),
|
||||
1,
|
||||
);
|
||||
});
|
||||
});
|
||||
}
|
||||
18
dev/devicelab/test/utils_test.dart
Normal file
18
dev/devicelab/test/utils_test.dart
Normal file
@ -0,0 +1,18 @@
|
||||
// Copyright (c) 2016 The Chromium Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file.
|
||||
|
||||
import 'package:test/test.dart';
|
||||
import 'package:flutter_devicelab/framework/utils.dart';
|
||||
|
||||
void main() {
|
||||
group('grep', () {
|
||||
test('greps lines', () {
|
||||
expect(grep('b', from: 'ab\ncd\nba'), <String>['ab', 'ba']);
|
||||
});
|
||||
|
||||
test('understands RegExp', () {
|
||||
expect(grep(new RegExp('^b'), from: 'ab\nba'), <String>['ba']);
|
||||
});
|
||||
});
|
||||
}
|
||||
@ -7,7 +7,7 @@ homepage: http://flutter.io
|
||||
dependencies:
|
||||
collection: '>=1.9.1 <2.0.0'
|
||||
intl: '>=0.14.0 <0.15.0'
|
||||
meta: ^1.0.2
|
||||
meta: ^1.0.3
|
||||
vector_math: '>=2.0.3 <3.0.0'
|
||||
|
||||
sky_engine:
|
||||
|
||||
@ -22,7 +22,7 @@ dependencies:
|
||||
json_schema: 1.0.3
|
||||
|
||||
linter: ^0.1.21
|
||||
meta: ^1.0.0
|
||||
meta: ^1.0.3
|
||||
mustache4dart: ^1.0.0
|
||||
package_config: '>=0.1.5 <2.0.0'
|
||||
path: ^1.3.0
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user