diff --git a/dev/bots/test.sh b/dev/bots/test.sh index d62b3164cc..50dc4d083b 100755 --- a/dev/bots/test.sh +++ b/dev/bots/test.sh @@ -35,6 +35,7 @@ fi (cd packages/flutter_test; flutter test) (cd packages/flutter_tools; dart -c test/all.dart) +(cd dev/devicelab; dart -c test/all.dart) (cd dev/manual_tests; flutter test) (cd examples/hello_world; flutter test) (cd examples/layers; flutter test) @@ -55,4 +56,4 @@ if [ -n "$COVERAGE_FLAG" ]; then fi # generate the API docs, upload them -dev/bots/docs.sh \ No newline at end of file +dev/bots/docs.sh diff --git a/dev/devicelab/.gitignore b/dev/devicelab/.gitignore new file mode 100644 index 0000000000..00154e92d3 --- /dev/null +++ b/dev/devicelab/.gitignore @@ -0,0 +1,9 @@ +.DS_Store +.buildlog +.idea +.packages +.pub/ +build/ +packages +pubspec.lock +.atom/ diff --git a/dev/devicelab/README.md b/dev/devicelab/README.md new file mode 100644 index 0000000000..8ded2af2b4 --- /dev/null +++ b/dev/devicelab/README.md @@ -0,0 +1,96 @@ +# Flutter devicelab + +This package contains the test framework and tests that run on physical devices. +More generally the tests are referred to as "tasks" in the API, but since we +primarily use it for testing, this document refers to them as "tests". + +# Writing tests + +A test is a simple Dart program that lives under `bin/tests` and uses +`package:flutter_devicelab/framework/framework.dart` to define and run a _task_. + +Example: + +```dart +import 'dart:async'; + +import 'package:flutter_devicelab/framework/framework.dart'; + +Future main() async { + await task(() async { + ... do something interesting ... + + // Aggregate results into a JSONable Map structure. + Map testResults = ...; + + // Report success. + return new TaskResult.success(testResults); + + // Or you can also report a failure. + return new TaskResult.failure('Something went wrong!'); + }); +} +``` + +Only one `task` is permitted per program. However, that task can run any number +of tests internally. A task has a name. It succeeds and fails independently of +other tasks, and is reported to the dashboard independently of other tasks. + +A task runs in its own standalone Dart VM and reports results via Dart VM +service protocol. This ensures that tasks do not interfere with each other and +lets the CI system time out and clean up tasks that get stuck. + +# Adding tests to the CI environment + +The `manifest.yaml` file describes a subset of tests we run in the CI. To add +your test edit `manifest.yaml` and add the following in the "tasks" dictionary: + +``` + {NAME_OF_TEST}: + description: {DESCRIPTION} + stage: {STAGE} + required_agent_capabilities: {CAPABILITIES} +``` + +Where: + + - `{NAME_OF_TEST}` is the name of your test that also matches the name of the + file in `bin/tests` without the `.dart` extension. + - `{DESCRIPTION}` is the plain English description of your test that helps + others understand what this test is testing. + - `{STAGE}` is `devicelab` if you want to run on Android, or `devicelab_ios` if + you want to run on iOS. + - `{CAPABILITIES}` is an array that lists the capabilities required of + the test agent (the computer that runs the test) to run your test. Available + capabilities are: `has-android-device`, `has-ios-device`. + +# Running tests locally + +Do make sure your tests pass locally before deploying to the CI environment. +Below is a handful of commands that run tests in a fashion very close to how the +CI environment runs them. These commands are also useful when you need to +reproduce a CI test failure locally. + +To run a test use option `-t` (`--task`): + +```sh +dart bin/run.dart -t {NAME_OF_TEST} +``` + +To run multiple tests repeat option `-t` (`--task`) multiple times: + +```sh +dart bin/run.dart -t test1 -t test2 -t test3 +``` + +To run all tests defined in `manifest.yaml` use option `-a` (`--all`): + +```sh +dart bin/run.dart -a +``` + +To run tests from a specific stage use option `-s` (`--stage`): + +```sh +dart bin/run.dart -s {NAME_OF_STAGE} +``` diff --git a/dev/devicelab/bin/run.dart b/dev/devicelab/bin/run.dart new file mode 100644 index 0000000000..c386349556 --- /dev/null +++ b/dev/devicelab/bin/run.dart @@ -0,0 +1,101 @@ +// Copyright 2016 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'dart:async'; +import 'dart:convert'; +import 'dart:io'; + +import 'package:args/args.dart'; + +import 'package:flutter_devicelab/framework/manifest.dart'; +import 'package:flutter_devicelab/framework/runner.dart'; +import 'package:flutter_devicelab/framework/utils.dart'; + +/// Runs tasks. +/// +/// The tasks are chosen depending on the command-line options +/// (see [_argParser]). +Future main(List rawArgs) async { + ArgResults args; + try { + args = _argParser.parse(rawArgs); + } on FormatException catch(error) { + stderr.writeln('${error.message}\n'); + stderr.writeln('Usage:\n'); + stderr.writeln(_argParser.usage); + exitCode = 1; + return null; + } + + List taskNames = []; + if (args.wasParsed('task')) { + taskNames.addAll(args['task']); + } else if (args.wasParsed('stage')) { + String stageName = args['stage']; + List tasks = loadTaskManifest().tasks; + for (ManifestTask task in tasks) { + if (task.stage == stageName) + taskNames.add(task.name); + } + } else if (args.wasParsed('all')) { + List tasks = loadTaskManifest().tasks; + for (ManifestTask task in tasks) { + taskNames.add(task.name); + } + } + + if (taskNames.isEmpty) { + stderr.writeln('Failed to find tasks to run based on supplied options.'); + exitCode = 1; + return null; + } + + for (String taskName in taskNames) { + section('Running task "$taskName"'); + Map result = await runTask(taskName); + + if (!result['success']) + exitCode = 1; + + print('Task result:'); + print(new JsonEncoder.withIndent(' ').convert(result)); + section('Finished task "$taskName"'); + } +} + +/// Command-line options for the `run.dart` command. +final ArgParser _argParser = new ArgParser() + ..addOption( + 'task', + abbr: 't', + allowMultiple: true, + splitCommas: true, + help: 'Name of the task to run. This option may be repeated to ' + 'specify multiple tasks. A task selected by name does not have to be ' + 'defined in manifest.yaml. It only needs a Dart executable in bin/tasks.', + ) + ..addOption( + 'stage', + abbr: 's', + help: 'Name of the stage. Runs all tasks for that stage. ' + 'The tasks and their stages are read from manifest.yaml.', + ) + ..addOption( + 'all', + abbr: 'a', + help: 'Runs all tasks defined in manifest.yaml.', + ) + ..addOption( + 'test', + hide: true, + allowMultiple: true, + splitCommas: true, + callback: (List value) { + if (value.isNotEmpty) { + throw new FormatException( + 'Invalid option --test. Did you mean --task (-t)?', + ); + } + }, + ); diff --git a/dev/devicelab/bin/tasks/analyzer_cli__analysis_time.dart b/dev/devicelab/bin/tasks/analyzer_cli__analysis_time.dart new file mode 100644 index 0000000000..7b0ebcad18 --- /dev/null +++ b/dev/devicelab/bin/tasks/analyzer_cli__analysis_time.dart @@ -0,0 +1,20 @@ +// Copyright 2016 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'dart:async'; + +import 'package:flutter_devicelab/tasks/analysis.dart'; +import 'package:flutter_devicelab/framework/framework.dart'; +import 'package:flutter_devicelab/framework/utils.dart'; + +Future main() async { + String revision = await getCurrentFlutterRepoCommit(); + DateTime revisionTimestamp = await getFlutterRepoCommitTimestamp(revision); + String dartSdkVersion = await getDartVersion(); + await task(createAnalyzerCliTest( + sdk: dartSdkVersion, + commit: revision, + timestamp: revisionTimestamp, + )); +} diff --git a/dev/devicelab/bin/tasks/analyzer_server__analysis_time.dart b/dev/devicelab/bin/tasks/analyzer_server__analysis_time.dart new file mode 100644 index 0000000000..c02b5375c4 --- /dev/null +++ b/dev/devicelab/bin/tasks/analyzer_server__analysis_time.dart @@ -0,0 +1,20 @@ +// Copyright 2016 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'dart:async'; + +import 'package:flutter_devicelab/tasks/analysis.dart'; +import 'package:flutter_devicelab/framework/framework.dart'; +import 'package:flutter_devicelab/framework/utils.dart'; + +Future main() async { + String revision = await getCurrentFlutterRepoCommit(); + DateTime revisionTimestamp = await getFlutterRepoCommitTimestamp(revision); + String dartSdkVersion = await getDartVersion(); + await task(createAnalyzerServerTest( + sdk: dartSdkVersion, + commit: revision, + timestamp: revisionTimestamp, + )); +} diff --git a/dev/devicelab/bin/tasks/basic_material_app__size.dart b/dev/devicelab/bin/tasks/basic_material_app__size.dart new file mode 100644 index 0000000000..c0202c9544 --- /dev/null +++ b/dev/devicelab/bin/tasks/basic_material_app__size.dart @@ -0,0 +1,12 @@ +// Copyright 2016 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'dart:async'; + +import 'package:flutter_devicelab/tasks/size_tests.dart'; +import 'package:flutter_devicelab/framework/framework.dart'; + +Future main() async { + await task(createBasicMaterialAppSizeTest()); +} diff --git a/dev/devicelab/bin/tasks/complex_layout__build.dart b/dev/devicelab/bin/tasks/complex_layout__build.dart new file mode 100644 index 0000000000..7fd0f9eb23 --- /dev/null +++ b/dev/devicelab/bin/tasks/complex_layout__build.dart @@ -0,0 +1,12 @@ +// Copyright 2016 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'dart:async'; + +import 'package:flutter_devicelab/tasks/perf_tests.dart'; +import 'package:flutter_devicelab/framework/framework.dart'; + +Future main() async { + await task(createComplexLayoutBuildTest()); +} diff --git a/dev/devicelab/bin/tasks/complex_layout__start_up.dart b/dev/devicelab/bin/tasks/complex_layout__start_up.dart new file mode 100644 index 0000000000..f2e5f9e4fe --- /dev/null +++ b/dev/devicelab/bin/tasks/complex_layout__start_up.dart @@ -0,0 +1,12 @@ +// Copyright 2016 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'dart:async'; + +import 'package:flutter_devicelab/tasks/perf_tests.dart'; +import 'package:flutter_devicelab/framework/framework.dart'; + +Future main() async { + await task(createComplexLayoutStartupTest(ios: false)); +} diff --git a/dev/devicelab/bin/tasks/complex_layout_ios__start_up.dart b/dev/devicelab/bin/tasks/complex_layout_ios__start_up.dart new file mode 100644 index 0000000000..cac132d815 --- /dev/null +++ b/dev/devicelab/bin/tasks/complex_layout_ios__start_up.dart @@ -0,0 +1,12 @@ +// Copyright 2016 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'dart:async'; + +import 'package:flutter_devicelab/tasks/perf_tests.dart'; +import 'package:flutter_devicelab/framework/framework.dart'; + +Future main() async { + await task(createComplexLayoutStartupTest(ios: true)); +} diff --git a/dev/devicelab/bin/tasks/complex_layout_scroll_perf__timeline_summary.dart b/dev/devicelab/bin/tasks/complex_layout_scroll_perf__timeline_summary.dart new file mode 100644 index 0000000000..a9f4d0c308 --- /dev/null +++ b/dev/devicelab/bin/tasks/complex_layout_scroll_perf__timeline_summary.dart @@ -0,0 +1,12 @@ +// Copyright 2016 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'dart:async'; + +import 'package:flutter_devicelab/tasks/perf_tests.dart'; +import 'package:flutter_devicelab/framework/framework.dart'; + +Future main() async { + await task(createComplexLayoutScrollPerfTest(ios: false)); +} diff --git a/dev/devicelab/bin/tasks/complex_layout_scroll_perf_ios__timeline_summary.dart b/dev/devicelab/bin/tasks/complex_layout_scroll_perf_ios__timeline_summary.dart new file mode 100644 index 0000000000..5cc6fa13ee --- /dev/null +++ b/dev/devicelab/bin/tasks/complex_layout_scroll_perf_ios__timeline_summary.dart @@ -0,0 +1,12 @@ +// Copyright 2016 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'dart:async'; + +import 'package:flutter_devicelab/tasks/perf_tests.dart'; +import 'package:flutter_devicelab/framework/framework.dart'; + +Future main() async { + await task(createComplexLayoutScrollPerfTest(ios: true)); +} diff --git a/dev/devicelab/bin/tasks/flutter_gallery__build.dart b/dev/devicelab/bin/tasks/flutter_gallery__build.dart new file mode 100644 index 0000000000..0d743db4ac --- /dev/null +++ b/dev/devicelab/bin/tasks/flutter_gallery__build.dart @@ -0,0 +1,12 @@ +// Copyright 2016 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'dart:async'; + +import 'package:flutter_devicelab/tasks/perf_tests.dart'; +import 'package:flutter_devicelab/framework/framework.dart'; + +Future main() async { + await task(createFlutterGalleryBuildTest()); +} diff --git a/dev/devicelab/bin/tasks/flutter_gallery__start_up.dart b/dev/devicelab/bin/tasks/flutter_gallery__start_up.dart new file mode 100644 index 0000000000..c6024e4028 --- /dev/null +++ b/dev/devicelab/bin/tasks/flutter_gallery__start_up.dart @@ -0,0 +1,12 @@ +// Copyright 2016 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'dart:async'; + +import 'package:flutter_devicelab/tasks/perf_tests.dart'; +import 'package:flutter_devicelab/framework/framework.dart'; + +Future main() async { + await task(createFlutterGalleryStartupTest(ios: false)); +} diff --git a/dev/devicelab/bin/tasks/flutter_gallery__transition_perf.dart b/dev/devicelab/bin/tasks/flutter_gallery__transition_perf.dart new file mode 100644 index 0000000000..d541ff9915 --- /dev/null +++ b/dev/devicelab/bin/tasks/flutter_gallery__transition_perf.dart @@ -0,0 +1,12 @@ +// Copyright 2016 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'dart:async'; + +import 'package:flutter_devicelab/tasks/gallery.dart'; +import 'package:flutter_devicelab/framework/framework.dart'; + +Future main() async { + await task(createGalleryTransitionTest(ios: false)); +} diff --git a/dev/devicelab/bin/tasks/flutter_gallery_ios__start_up.dart b/dev/devicelab/bin/tasks/flutter_gallery_ios__start_up.dart new file mode 100644 index 0000000000..24042a0149 --- /dev/null +++ b/dev/devicelab/bin/tasks/flutter_gallery_ios__start_up.dart @@ -0,0 +1,12 @@ +// Copyright 2016 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'dart:async'; + +import 'package:flutter_devicelab/tasks/perf_tests.dart'; +import 'package:flutter_devicelab/framework/framework.dart'; + +Future main() async { + await task(createFlutterGalleryStartupTest(ios: true)); +} diff --git a/dev/devicelab/bin/tasks/flutter_gallery_ios__transition_perf.dart b/dev/devicelab/bin/tasks/flutter_gallery_ios__transition_perf.dart new file mode 100644 index 0000000000..1745cb04b8 --- /dev/null +++ b/dev/devicelab/bin/tasks/flutter_gallery_ios__transition_perf.dart @@ -0,0 +1,12 @@ +// Copyright 2016 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'dart:async'; + +import 'package:flutter_devicelab/tasks/gallery.dart'; +import 'package:flutter_devicelab/framework/framework.dart'; + +Future main() async { + await task(createGalleryTransitionTest(ios: true)); +} diff --git a/dev/devicelab/bin/tasks/mega_gallery__refresh_time.dart b/dev/devicelab/bin/tasks/mega_gallery__refresh_time.dart new file mode 100644 index 0000000000..1bb289b7aa --- /dev/null +++ b/dev/devicelab/bin/tasks/mega_gallery__refresh_time.dart @@ -0,0 +1,18 @@ +// Copyright 2016 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'dart:async'; + +import 'package:flutter_devicelab/tasks/refresh.dart'; +import 'package:flutter_devicelab/framework/framework.dart'; +import 'package:flutter_devicelab/framework/utils.dart'; + +Future main() async { + String revision = await getCurrentFlutterRepoCommit(); + DateTime revisionTimestamp = await getFlutterRepoCommitTimestamp(revision); + await task(createRefreshTest( + commit: revision, + timestamp: revisionTimestamp, + )); +} diff --git a/dev/devicelab/bin/tasks/smoke_test_failure.dart b/dev/devicelab/bin/tasks/smoke_test_failure.dart new file mode 100644 index 0000000000..523429a22f --- /dev/null +++ b/dev/devicelab/bin/tasks/smoke_test_failure.dart @@ -0,0 +1,14 @@ +// Copyright 2016 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'dart:async'; + +import 'package:flutter_devicelab/framework/framework.dart'; + +/// Smoke test of a task that fails by returning an unsuccessful response. +Future main() async { + await task(() async { + return new TaskResult.failure('Failed'); + }); +} diff --git a/dev/devicelab/bin/tasks/smoke_test_setup_failure.dart b/dev/devicelab/bin/tasks/smoke_test_setup_failure.dart new file mode 100644 index 0000000000..7724d8b0ad --- /dev/null +++ b/dev/devicelab/bin/tasks/smoke_test_setup_failure.dart @@ -0,0 +1,13 @@ +// Copyright 2016 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'dart:async'; + +/// Creates a situation when the test framework was not properly initialized. +/// +/// By not calling `task()` the VM service extension is not registered and +/// therefore will not accept requests to run tasks. When the runner attempts to +/// connect and run the test it will receive a "method not found" error from the +/// VM service, will likely retry and finally time out. +Future main() async {} diff --git a/dev/devicelab/bin/tasks/smoke_test_success.dart b/dev/devicelab/bin/tasks/smoke_test_success.dart new file mode 100644 index 0000000000..5602cbb758 --- /dev/null +++ b/dev/devicelab/bin/tasks/smoke_test_success.dart @@ -0,0 +1,14 @@ +// Copyright 2016 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'dart:async'; + +import 'package:flutter_devicelab/framework/framework.dart'; + +/// Smoke test of a successful task. +Future main() async { + await task(() async { + return new TaskResult.success({}); + }); +} diff --git a/dev/devicelab/bin/tasks/smoke_test_throws.dart b/dev/devicelab/bin/tasks/smoke_test_throws.dart new file mode 100644 index 0000000000..681f7c8579 --- /dev/null +++ b/dev/devicelab/bin/tasks/smoke_test_throws.dart @@ -0,0 +1,14 @@ +// Copyright 2016 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'dart:async'; + +import 'package:flutter_devicelab/framework/framework.dart'; + +/// Smoke test of a task that fails with an exception. +Future main() async { + await task(() async { + throw 'failed'; + }); +} diff --git a/dev/devicelab/lib/framework/adb.dart b/dev/devicelab/lib/framework/adb.dart new file mode 100644 index 0000000000..ed2c48b82a --- /dev/null +++ b/dev/devicelab/lib/framework/adb.dart @@ -0,0 +1,202 @@ +// Copyright 2016 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'dart:async'; +import 'dart:io'; +import 'dart:math' as math; + +import 'package:path/path.dart' as path; + +import 'utils.dart'; + +typedef Future AdbGetter(); + +/// Get an instance of [Adb]. +/// +/// See [realAdbGetter] for signature. This can be overwritten for testing. +AdbGetter adb = realAdbGetter; + +Adb _currentDevice; + +/// Picks a random Android device out of connected devices and sets it as +/// [_currentDevice]. +Future pickNextDevice() async { + List allDevices = + (await Adb.deviceIds).map((String id) => new Adb(deviceId: id)).toList(); + + if (allDevices.length == 0) throw 'No Android devices detected'; + + // TODO(yjbanov): filter out and warn about those with low battery level + _currentDevice = allDevices[new math.Random().nextInt(allDevices.length)]; +} + +Future realAdbGetter() async { + if (_currentDevice == null) await pickNextDevice(); + return _currentDevice; +} + +/// Gets the ID of an unlocked device, unlocking it if necessary. +// TODO(yjbanov): abstract away iOS from Android. +Future getUnlockedDeviceId({ bool ios: false }) async { + if (ios) { + // We currently do not have a way to lock/unlock iOS devices, or even to + // pick one out of many. So we pick the first random iPhone and assume it's + // already unlocked. For now we'll just keep them at minimum screen + // brightness so they don't drain battery too fast. + List iosDeviceIds = + grep('UniqueDeviceID', from: await eval('ideviceinfo', [])) + .map((String line) => line.split(' ').last) + .toList(); + + if (iosDeviceIds.isEmpty) throw 'No connected iOS devices found.'; + + return iosDeviceIds.first; + } + + Adb device = await adb(); + await device.unlock(); + return device.deviceId; +} + +/// Android Debug Bridge (`adb`) client that exposes a subset of functions +/// relevant to on-device testing. +class Adb { + Adb({ this.deviceId }); + + final String deviceId; + + // Parses information about a device. Example: + // + // 015d172c98400a03 device usb:340787200X product:nakasi model:Nexus_7 device:grouper + static final RegExp _kDeviceRegex = new RegExp(r'^(\S+)\s+(\S+)(.*)'); + + /// Reports connection health for every device. + static Future> checkDevices() async { + Map results = {}; + for (String deviceId in await deviceIds) { + try { + Adb device = new Adb(deviceId: deviceId); + // Just a smoke test that we can read wakefulness state + // TODO(yjbanov): also check battery level + await device._getWakefulness(); + results['android-device-$deviceId'] = new HealthCheckResult.success(); + } catch (e, s) { + results['android-device-$deviceId'] = new HealthCheckResult.error(e, s); + } + } + return results; + } + + /// Kills the `adb` server causing it to start a new instance upon next + /// command. + /// + /// Restarting `adb` helps with keeping device connections alive. When `adb` + /// runs non-stop for too long it loses connections to devices. + static Future restart() async { + await exec(adbPath, ['kill-server'], canFail: false); + } + + /// List of device IDs visible to `adb`. + static Future> get deviceIds async { + List output = + (await eval(adbPath, ['devices', '-l'], canFail: false)) + .trim() + .split('\n'); + List results = []; + for (String line in output) { + // Skip lines like: * daemon started successfully * + if (line.startsWith('* daemon ')) continue; + + if (line.startsWith('List of devices')) continue; + + if (_kDeviceRegex.hasMatch(line)) { + Match match = _kDeviceRegex.firstMatch(line); + + String deviceID = match[1]; + String deviceState = match[2]; + + if (!const ['unauthorized', 'offline'].contains(deviceState)) { + results.add(deviceID); + } + } else { + throw 'Failed to parse device from adb output: $line'; + } + } + + return results; + } + + /// Whether the device is awake. + Future isAwake() async { + return await _getWakefulness() == 'Awake'; + } + + /// Whether the device is asleep. + Future isAsleep() async { + return await _getWakefulness() == 'Asleep'; + } + + /// Wake up the device if it is not awake using [togglePower]. + Future wakeUp() async { + if (!(await isAwake())) await togglePower(); + } + + /// Send the device to sleep mode if it is not asleep using [togglePower]. + Future sendToSleep() async { + if (!(await isAsleep())) await togglePower(); + } + + /// Sends `KEYCODE_POWER` (26), which causes the device to toggle its mode + /// between awake and asleep. + Future togglePower() async { + await shellExec('input', const ['keyevent', '26']); + } + + /// Unlocks the device by sending `KEYCODE_MENU` (82). + /// + /// This only works when the device doesn't have a secure unlock pattern. + Future unlock() async { + await wakeUp(); + await shellExec('input', const ['keyevent', '82']); + } + + /// Retrieves device's wakefulness state. + /// + /// See: https://android.googlesource.com/platform/frameworks/base/+/master/core/java/android/os/PowerManagerInternal.java + Future _getWakefulness() async { + String powerInfo = await shellEval('dumpsys', ['power']); + String wakefulness = + grep('mWakefulness=', from: powerInfo).single.split('=')[1].trim(); + return wakefulness; + } + + /// Executes [command] on `adb shell` and returns its exit code. + Future shellExec(String command, List arguments, + { Map env }) async { + await exec(adbPath, ['shell', command]..addAll(arguments), + env: env, canFail: false); + } + + /// Executes [command] on `adb shell` and returns its standard output as a [String]. + Future shellEval(String command, List arguments, + { Map env }) { + return eval(adbPath, ['shell', command]..addAll(arguments), + env: env, canFail: false); + } +} + +/// Path to the `adb` executable. +String get adbPath { + String androidHome = Platform.environment['ANDROID_HOME']; + + if (androidHome == null) + throw 'ANDROID_HOME environment variable missing. This variable must ' + 'point to the Android SDK directory containing platform-tools.'; + + File adbPath = file(path.join(androidHome, 'platform-tools/adb')); + + if (!adbPath.existsSync()) throw 'adb not found at: $adbPath'; + + return adbPath.absolute.path; +} diff --git a/dev/devicelab/lib/framework/benchmarks.dart b/dev/devicelab/lib/framework/benchmarks.dart new file mode 100644 index 0000000000..7eccabe83e --- /dev/null +++ b/dev/devicelab/lib/framework/benchmarks.dart @@ -0,0 +1,62 @@ +// Copyright (c) 2016 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'dart:async'; + +import 'framework.dart'; + +/// A benchmark harness used to run a benchmark multiple times and report the +/// best result. +abstract class Benchmark { + Benchmark(this.name); + + final String name; + + TaskResult bestResult; + + Future init() => new Future.value(); + + Future run(); + TaskResult get lastResult; + + @override + String toString() => name; +} + +/// Runs a [benchmark] [iterations] times and reports the best result. +/// +/// Use [warmUpBenchmark] to discard cold performance results. +Future runBenchmark(Benchmark benchmark, { + int iterations: 1, + bool warmUpBenchmark: false +}) async { + await benchmark.init(); + + List allRuns = []; + + num minValue; + + if (warmUpBenchmark) + await benchmark.run(); + + while (iterations > 0) { + iterations--; + + print(''); + + try { + num result = await benchmark.run(); + allRuns.add(result); + + if (minValue == null || result < minValue) { + benchmark.bestResult = benchmark.lastResult; + minValue = result; + } + } catch (error) { + print('benchmark failed with error: $error'); + } + } + + return minValue; +} diff --git a/dev/devicelab/lib/framework/framework.dart b/dev/devicelab/lib/framework/framework.dart new file mode 100644 index 0000000000..4ac2ffb70e --- /dev/null +++ b/dev/devicelab/lib/framework/framework.dart @@ -0,0 +1,217 @@ +// Copyright 2016 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'dart:async'; +import 'dart:convert'; +import 'dart:developer'; +import 'dart:io'; +import 'dart:isolate'; + +import 'package:logging/logging.dart'; + +import 'utils.dart'; + +/// Maximum amount of time a single task is allowed to take to run. +/// +/// If exceeded the task is considered to have failed. +const Duration taskTimeout = const Duration(minutes: 10); + +/// Represents a unit of work performed in the CI environment that can +/// succeed, fail and be retried independently of others. +typedef Future TaskFunction(); + +bool _isTaskRegistered = false; + +/// Registers a [task] to run, returns the result when it is complete. +/// +/// Note, the task does not run immediately but waits for the request via the +/// VM service protocol to run it. +/// +/// It is ok for a [task] to perform many things. However, only one task can be +/// registered per Dart VM. +Future task(TaskFunction task) { + if (_isTaskRegistered) + throw new StateError('A task is already registered'); + + _isTaskRegistered = true; + + // TODO: allow overriding logging. + Logger.root.level = Level.ALL; + Logger.root.onRecord.listen((LogRecord rec) { + print('${rec.level.name}: ${rec.time}: ${rec.message}'); + }); + + _TaskRunner runner = new _TaskRunner(task); + runner.keepVmAliveUntilTaskRunRequested(); + return runner.whenDone; +} + +class _TaskRunner { + static final Logger logger = new Logger('TaskRunner'); + + final TaskFunction task; + + // TODO: workaround for https://github.com/dart-lang/sdk/issues/23797 + RawReceivePort _keepAlivePort; + Timer _startTaskTimeout; + bool _taskStarted = false; + + final Completer _completer = new Completer(); + + _TaskRunner(this.task) { + registerExtension('ext.cocoonRunTask', + (String method, Map parameters) async { + TaskResult result = await run(); + return new ServiceExtensionResponse.result(JSON.encode(result.toJson())); + }); + registerExtension('ext.cocoonRunnerReady', + (String method, Map parameters) async { + return new ServiceExtensionResponse.result('"ready"'); + }); + } + + /// Signals that this task runner finished running the task. + Future get whenDone => _completer.future; + + Future run() async { + try { + _taskStarted = true; + TaskResult result = await _performTask().timeout(taskTimeout); + _completer.complete(result); + return result; + } on TimeoutException catch (_) { + return new TaskResult.failure('Task timed out after $taskTimeout'); + } finally { + forceQuitRunningProcesses(); + _closeKeepAlivePort(); + } + } + + /// Causes the Dart VM to stay alive until a request to run the task is + /// received via the VM service protocol. + void keepVmAliveUntilTaskRunRequested() { + if (_taskStarted) + throw new StateError('Task already started.'); + + // Merely creating this port object will cause the VM to stay alive and keep + // the VM service server running until the port is disposed of. + _keepAlivePort = new RawReceivePort(); + + // Timeout if nothing bothers to connect and ask us to run the task. + const Duration taskStartTimeout = const Duration(seconds: 10); + _startTaskTimeout = new Timer(taskStartTimeout, () { + if (!_taskStarted) { + logger.severe('Task did not start in $taskStartTimeout.'); + _closeKeepAlivePort(); + exitCode = 1; + } + }); + } + + /// Disables the keep-alive port, allowing the VM to exit. + void _closeKeepAlivePort() { + _startTaskTimeout?.cancel(); + _keepAlivePort?.close(); + } + + Future _performTask() async { + try { + return await task(); + } catch (taskError, taskErrorStack) { + String message = 'Task failed: $taskError'; + if (taskErrorStack != null) { + message += '\n\n$taskErrorStack'; + } + return new TaskResult.failure(message); + } + } +} + +/// A result of running a single task. +class TaskResult { + /// Constructs a successful result. + TaskResult.success(this.data, {this.benchmarkScoreKeys: const []}) + : this.succeeded = true, + this.message = 'success' { + const JsonEncoder prettyJson = const JsonEncoder.withIndent(' '); + if (benchmarkScoreKeys != null) { + for (String key in benchmarkScoreKeys) { + if (!data.containsKey(key)) { + throw 'Invalid Golem score key "$key". It does not exist in task ' + 'result data ${prettyJson.convert(data)}'; + } else if (data[key] is! num) { + throw 'Invalid Golem score for key "$key". It is expected to be a num ' + 'but was ${data[key].runtimeType}: ${prettyJson.convert(data[key])}'; + } + } + } + } + + /// Constructs a successful result using JSON data stored in a file. + factory TaskResult.successFromFile(File file, + {List benchmarkScoreKeys}) { + return new TaskResult.success(JSON.decode(file.readAsStringSync()), + benchmarkScoreKeys: benchmarkScoreKeys); + } + + /// Constructs an unsuccessful result. + TaskResult.failure(this.message) + : this.succeeded = false, + this.data = null, + this.benchmarkScoreKeys = const []; + + /// Whether the task succeeded. + final bool succeeded; + + /// Task-specific JSON data + final Map data; + + /// Keys in [data] that store scores that will be submitted to Golem. + /// + /// Each key is also part of a benchmark's name tracked by Golem. + /// A benchmark name is computed by combining [Task.name] with a key + /// separated by a dot. For example, if a task's name is + /// `"complex_layout__start_up"` and score key is + /// `"engineEnterTimestampMicros"`, the score will be submitted to Golem under + /// `"complex_layout__start_up.engineEnterTimestampMicros"`. + /// + /// This convention reduces the amount of configuration that needs to be done + /// to submit benchmark scores to Golem. + final List benchmarkScoreKeys; + + /// Whether the task failed. + bool get failed => !succeeded; + + /// Explains the result in a human-readable format. + final String message; + + /// Serializes this task result to JSON format. + /// + /// The JSON format is as follows: + /// + /// { + /// "success": true|false, + /// "data": arbitrary JSON data valid only for successful results, + /// "benchmarkScoreKeys": [ + /// contains keys into "data" that represent benchmarks scores, which + /// can be uploaded, for example. to golem, valid only for successful + /// results + /// ], + /// "reason": failure reason string valid only for unsuccessful results + /// } + Map toJson() { + Map json = { + 'success': succeeded, + }; + + if (succeeded) { + json['data'] = data; + json['benchmarkScoreKeys'] = benchmarkScoreKeys; + } else { + json['reason'] = message; + } + + return json; + } +} diff --git a/dev/devicelab/lib/framework/manifest.dart b/dev/devicelab/lib/framework/manifest.dart new file mode 100644 index 0000000000..2bc97062c9 --- /dev/null +++ b/dev/devicelab/lib/framework/manifest.dart @@ -0,0 +1,127 @@ +// Copyright 2016 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'package:meta/meta.dart'; +import 'package:yaml/yaml.dart'; + +import 'utils.dart'; + +/// Loads manifest data from `manifest.yaml` file or from [yaml], if present. +Manifest loadTaskManifest([ String yaml ]) { + dynamic manifestYaml = yaml == null + ? loadYaml(file('manifest.yaml').readAsStringSync()) + : loadYamlNode(yaml); + + _checkType(manifestYaml is Map, manifestYaml, 'Manifest', 'dictionary'); + return _validateAndParseManifest(manifestYaml); +} + +/// Contains CI task information. +class Manifest { + Manifest._(this.tasks); + + /// CI tasks. + final List tasks; +} + +/// A CI task. +class ManifestTask { + ManifestTask._({ + @required this.name, + @required this.description, + @required this.stage, + @required this.requiredAgentCapabilities, + }) { + String taskName = 'task "$name"'; + _checkIsNotBlank(name, 'Task name', taskName); + _checkIsNotBlank(description, 'Task description', taskName); + _checkIsNotBlank(stage, 'Task stage', taskName); + _checkIsNotBlank(requiredAgentCapabilities, 'requiredAgentCapabilities', taskName); + } + + /// Task name as it appears on the dashboard. + final String name; + + /// A human-readable description of the task. + final String description; + + /// The stage this task should run in. + final String stage; + + /// Capabilities required of the build agent to be able to perform this task. + final List requiredAgentCapabilities; +} + +/// Thrown when the manifest YAML is not valid. +class ManifestError extends Error { + ManifestError(this.message); + + final String message; + + @override + String toString() => '$ManifestError: $message'; +} + +// There's no good YAML validator, at least not for Dart, so we validate +// manually. It's not too much code and produces good error messages. +Manifest _validateAndParseManifest(Map manifestYaml) { + _checkKeys(manifestYaml, 'manifest', const ['tasks']); + return new Manifest._(_validateAndParseTasks(manifestYaml['tasks'])); +} + +List _validateAndParseTasks(dynamic tasksYaml) { + _checkType(tasksYaml is Map, tasksYaml, 'Value of "tasks"', 'dictionary'); + return tasksYaml.keys.map((dynamic taskName) => _validateAndParseTask(taskName, tasksYaml[taskName])).toList(); +} + +ManifestTask _validateAndParseTask(dynamic taskName, dynamic taskYaml) { + _checkType(taskName is String, taskName, 'Task name', 'string'); + _checkType(taskYaml is Map, taskYaml, 'Value of task "$taskName"', 'dictionary'); + _checkKeys(taskYaml, 'Value of task "$taskName"', const [ + 'description', + 'stage', + 'required_agent_capabilities', + ]); + + List capabilities = _validateAndParseCapabilities(taskName, taskYaml['required_agent_capabilities']); + return new ManifestTask._( + name: taskName, + description: taskYaml['description'], + stage: taskYaml['stage'], + requiredAgentCapabilities: capabilities, + ); +} + +List _validateAndParseCapabilities(String taskName, dynamic capabilitiesYaml) { + _checkType(capabilitiesYaml is List, capabilitiesYaml, 'required_agent_capabilities', 'list'); + for (int i = 0; i < capabilitiesYaml.length; i++) { + dynamic capability = capabilitiesYaml[i]; + _checkType(capability is String, capability, 'required_agent_capabilities[$i]', 'string'); + } + return capabilitiesYaml; +} + +void _checkType(bool isValid, dynamic value, String variableName, String typeName) { + if (!isValid) { + throw new ManifestError( + '$variableName must be a $typeName but was ${value.runtimeType}: $value', + ); + } +} + +void _checkIsNotBlank(dynamic value, String variableName, String ownerName) { + if (value == null || value.isEmpty) { + throw new ManifestError('$variableName must not be empty in $ownerName.'); + } +} + +void _checkKeys(Map map, String variableName, List allowedKeys) { + for (String key in map.keys) { + if (!allowedKeys.contains(key)) { + throw new ManifestError( + 'Unrecognized property "$key" in $variableName. ' + 'Allowed properties: ${allowedKeys.join(', ')}'); + } + } +} diff --git a/dev/devicelab/lib/framework/runner.dart b/dev/devicelab/lib/framework/runner.dart new file mode 100644 index 0000000000..54fac0d7c9 --- /dev/null +++ b/dev/devicelab/lib/framework/runner.dart @@ -0,0 +1,129 @@ +// Copyright 2016 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'dart:async'; +import 'dart:convert'; +import 'dart:io'; + +import 'package:vm_service_client/vm_service_client.dart'; + +import 'package:flutter_devicelab/framework/utils.dart'; + +/// Slightly longer than task timeout that gives the task runner a chance to +/// clean-up before forcefully quitting it. +const Duration taskTimeoutWithGracePeriod = const Duration(minutes: 11); + +/// Runs a task in a separate Dart VM and collects the result using the VM +/// service protocol. +/// +/// [taskName] is the name of the task. The corresponding task executable is +/// expected to be found under `bin/tasks`. +Future> runTask(String taskName) async { + String taskExecutable = 'bin/tasks/$taskName.dart'; + + if (!file(taskExecutable).existsSync()) + throw 'Executable Dart file not found: $taskExecutable'; + + int vmServicePort = await _findAvailablePort(); + Process runner = await startProcess(dartBin, [ + '--enable-vm-service=$vmServicePort', + '--no-pause-isolates-on-exit', + taskExecutable, + ]); + + bool runnerFinished = false; + + runner.exitCode.then((_) { + runnerFinished = true; + }); + + StreamSubscription stdoutSub = runner.stdout + .transform(new Utf8Decoder()) + .transform(new LineSplitter()) + .listen((String line) { + stdout.writeln('[$taskName] [STDOUT] $line'); + }); + + StreamSubscription stderrSub = runner.stderr + .transform(new Utf8Decoder()) + .transform(new LineSplitter()) + .listen((String line) { + stderr.writeln('[$taskName] [STDERR] $line'); + }); + + String waitingFor = 'connection'; + try { + VMIsolate isolate = await _connectToRunnerIsolate(vmServicePort); + waitingFor = 'task completion'; + Map taskResult = + await isolate.invokeExtension('ext.cocoonRunTask').timeout(taskTimeoutWithGracePeriod); + waitingFor = 'task process to exit'; + await runner.exitCode.timeout(const Duration(seconds: 1)); + return taskResult; + } on TimeoutException catch (timeout) { + runner.kill(ProcessSignal.SIGINT); + return { + 'success': false, + 'reason': 'Timeout waiting for $waitingFor: ${timeout.message}', + }; + } finally { + if (!runnerFinished) + runner.kill(ProcessSignal.SIGKILL); + await stdoutSub.cancel(); + await stderrSub.cancel(); + } +} + +Future _connectToRunnerIsolate(int vmServicePort) async { + String url = 'ws://localhost:$vmServicePort/ws'; + DateTime started = new DateTime.now(); + + // TODO(yjbanov): due to lack of imagination at the moment the handshake with + // the task process is very rudimentary and requires this small + // delay to let the task process open up the VM service port. + // Otherwise we almost always hit the non-ready case first and + // wait a whole 1 second, which is annoying. + await new Future.delayed(const Duration(milliseconds: 100)); + + while (true) { + try { + // Make sure VM server is up by successfully opening and closing a socket. + await (await WebSocket.connect(url)).close(); + + // Look up the isolate. + VMServiceClient client = new VMServiceClient.connect(url); + VM vm = await client.getVM(); + VMIsolate isolate = vm.isolates.single; + String response = await isolate.invokeExtension('ext.cocoonRunnerReady'); + if (response != 'ready') throw 'not ready yet'; + return isolate; + } catch (error) { + const Duration connectionTimeout = const Duration(seconds: 2); + if (new DateTime.now().difference(started) > connectionTimeout) { + throw new TimeoutException( + 'Failed to connect to the task runner process', + connectionTimeout, + ); + } + print('VM service not ready yet: $error'); + const Duration pauseBetweenRetries = const Duration(milliseconds: 200); + print('Will retry in $pauseBetweenRetries.'); + await new Future.delayed(pauseBetweenRetries); + } + } +} + +Future _findAvailablePort() async { + int port = 20000; + while (true) { + try { + ServerSocket socket = + await ServerSocket.bind(InternetAddress.LOOPBACK_IP_V4, port); + await socket.close(); + return port; + } catch (_) { + port++; + } + } +} diff --git a/dev/devicelab/lib/framework/utils.dart b/dev/devicelab/lib/framework/utils.dart new file mode 100644 index 0000000000..5586d22621 --- /dev/null +++ b/dev/devicelab/lib/framework/utils.dart @@ -0,0 +1,412 @@ +// Copyright (c) 2016 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'dart:async'; +import 'dart:convert'; +import 'dart:io'; + +import 'package:meta/meta.dart'; +import 'package:path/path.dart' as path; +import 'package:stack_trace/stack_trace.dart'; + +/// Virtual current working directory, which affect functions, such as [exec]. +String cwd = Directory.current.path; + +List _runningProcesses = []; + +class ProcessInfo { + ProcessInfo(this.command, this.process); + + final DateTime startTime = new DateTime.now(); + final String command; + final Process process; + + @override + String toString() { + return ''' + command : $command + started : $startTime + pid : ${process.pid} +''' + .trim(); + } +} + +/// Result of a health check for a specific parameter. +class HealthCheckResult { + HealthCheckResult.success([this.details]) : succeeded = true; + HealthCheckResult.failure(this.details) : succeeded = false; + HealthCheckResult.error(dynamic error, dynamic stackTrace) + : succeeded = false, + details = 'ERROR: $error${'\n$stackTrace' ?? ''}'; + + final bool succeeded; + final String details; + + @override + String toString() { + StringBuffer buf = new StringBuffer(succeeded ? 'succeeded' : 'failed'); + if (details != null && details.trim().isNotEmpty) { + buf.writeln(); + // Indent details by 4 spaces + for (String line in details.trim().split('\n')) { + buf.writeln(' $line'); + } + } + return '$buf'; + } +} + +class BuildFailedError extends Error { + BuildFailedError(this.message); + + final String message; + + @override + String toString() => message; +} + +void fail(String message) { + throw new BuildFailedError(message); +} + +void rm(FileSystemEntity entity) { + if (entity.existsSync()) + entity.deleteSync(); +} + +/// Remove recursively. +void rmTree(FileSystemEntity entity) { + if (entity.existsSync()) + entity.deleteSync(recursive: true); +} + +List ls(Directory directory) => directory.listSync(); + +Directory dir(String path) => new Directory(path); + +File file(String path) => new File(path); + +void copy(File sourceFile, Directory targetDirectory, {String name}) { + File target = file( + path.join(targetDirectory.path, name ?? path.basename(sourceFile.path))); + target.writeAsBytesSync(sourceFile.readAsBytesSync()); +} + +FileSystemEntity move(FileSystemEntity whatToMove, + {Directory to, String name}) { + return whatToMove + .renameSync(path.join(to.path, name ?? path.basename(whatToMove.path))); +} + +/// Equivalent of `mkdir directory`. +void mkdir(Directory directory) { + directory.createSync(); +} + +/// Equivalent of `mkdir -p directory`. +void mkdirs(Directory directory) { + directory.createSync(recursive: true); +} + +bool exists(FileSystemEntity entity) => entity.existsSync(); + +void section(String title) { + print('\n••• $title •••'); +} + +Future getDartVersion() async { + // The Dart VM returns the version text to stderr. + ProcessResult result = Process.runSync(dartBin, ['--version']); + String version = result.stderr.trim(); + + // Convert: + // Dart VM version: 1.17.0-dev.2.0 (Tue May 3 12:14:52 2016) on "macos_x64" + // to: + // 1.17.0-dev.2.0 + if (version.indexOf('(') != -1) + version = version.substring(0, version.indexOf('(')).trim(); + if (version.indexOf(':') != -1) + version = version.substring(version.indexOf(':') + 1).trim(); + + return version.replaceAll('"', "'"); +} + +Future getCurrentFlutterRepoCommit() { + if (!dir('${flutterDirectory.path}/.git').existsSync()) { + return null; + } + + return inDirectory(flutterDirectory, () { + return eval('git', ['rev-parse', 'HEAD']); + }); +} + +Future getFlutterRepoCommitTimestamp(String commit) { + // git show -s --format=%at 4b546df7f0b3858aaaa56c4079e5be1ba91fbb65 + return inDirectory(flutterDirectory, () async { + String unixTimestamp = await eval('git', [ + 'show', + '-s', + '--format=%at', + commit, + ]); + int secondsSinceEpoch = int.parse(unixTimestamp); + return new DateTime.fromMillisecondsSinceEpoch(secondsSinceEpoch * 1000); + }); +} + +Future startProcess(String executable, List arguments, + {Map env}) async { + String command = '$executable ${arguments?.join(" ") ?? ""}'; + print('Executing: $command'); + Process proc = await Process.start(executable, arguments, + environment: env, workingDirectory: cwd); + ProcessInfo procInfo = new ProcessInfo(command, proc); + _runningProcesses.add(procInfo); + + proc.exitCode.then((_) { + _runningProcesses.remove(procInfo); + }); + + return proc; +} + +Future forceQuitRunningProcesses() async { + if (_runningProcesses.isEmpty) + return; + + // Give normally quitting processes a chance to report their exit code. + await new Future.delayed(const Duration(seconds: 1)); + + // Whatever's left, kill it. + for (ProcessInfo p in _runningProcesses) { + print('Force quitting process:\n$p'); + if (!p.process.kill()) { + print('Failed to force quit process'); + } + } + _runningProcesses.clear(); +} + +/// Executes a command and returns its exit code. +Future exec(String executable, List arguments, + {Map env, bool canFail: false}) async { + Process proc = await startProcess(executable, arguments, env: env); + + proc.stdout + .transform(UTF8.decoder) + .transform(const LineSplitter()) + .listen(print); + proc.stderr + .transform(UTF8.decoder) + .transform(const LineSplitter()) + .listen(stderr.writeln); + + int exitCode = await proc.exitCode; + + if (exitCode != 0 && !canFail) + fail('Executable failed with exit code $exitCode.'); + + return exitCode; +} + +/// Executes a command and returns its standard output as a String. +/// +/// Standard error is redirected to the current process' standard error stream. +Future eval(String executable, List arguments, + {Map env, bool canFail: false}) async { + Process proc = await startProcess(executable, arguments, env: env); + proc.stderr.listen((List data) { + stderr.add(data); + }); + String output = await UTF8.decodeStream(proc.stdout); + int exitCode = await proc.exitCode; + + if (exitCode != 0 && !canFail) + fail('Executable failed with exit code $exitCode.'); + + return output.trimRight(); +} + +Future flutter(String command, + {List options: const [], bool canFail: false}) { + List args = [command]..addAll(options); + return exec(path.join(flutterDirectory.path, 'bin', 'flutter'), args, + canFail: canFail); +} + +String get dartBin => + path.join(flutterDirectory.path, 'bin', 'cache', 'dart-sdk', 'bin', 'dart'); + +Future dart(List args) => exec(dartBin, args); + +Future pub(String command) { + return exec( + path.join(flutterDirectory.path, 'bin', 'cache', 'dart-sdk', 'bin', 'pub'), + [command]); +} + +Future inDirectory(dynamic directory, Future action()) async { + String previousCwd = cwd; + try { + cd(directory); + return await action(); + } finally { + cd(previousCwd); + } +} + +void cd(dynamic directory) { + Directory d; + if (directory is String) { + cwd = directory; + d = dir(directory); + } else if (directory is Directory) { + cwd = directory.path; + d = directory; + } else { + throw 'Unsupported type ${directory.runtimeType} of $directory'; + } + + if (!d.existsSync()) + throw 'Cannot cd into directory that does not exist: $directory'; +} + +Directory get flutterDirectory => dir('../..').absolute; + +String requireEnvVar(String name) { + String value = Platform.environment[name]; + + if (value == null) fail('$name environment variable is missing. Quitting.'); + + return value; +} + +dynamic/*=T*/ requireConfigProperty/**/( + Map*/ > map, String propertyName) { + if (!map.containsKey(propertyName)) + fail('Configuration property not found: $propertyName'); + + return map[propertyName]; +} + +String jsonEncode(dynamic data) { + return new JsonEncoder.withIndent(' ').convert(data) + '\n'; +} + +Future getFlutter(String revision) async { + section('Get Flutter!'); + + if (exists(flutterDirectory)) { + rmTree(flutterDirectory); + } + + await inDirectory(flutterDirectory.parent, () async { + await exec('git', ['clone', 'https://github.com/flutter/flutter.git']); + }); + + await inDirectory(flutterDirectory, () async { + await exec('git', ['checkout', revision]); + }); + + await flutter('config', options: ['--no-analytics']); + + section('flutter doctor'); + await flutter('doctor'); + + section('flutter update-packages'); + await flutter('update-packages'); +} + +void checkNotNull(Object o1, + [Object o2 = 1, + Object o3 = 1, + Object o4 = 1, + Object o5 = 1, + Object o6 = 1, + Object o7 = 1, + Object o8 = 1, + Object o9 = 1, + Object o10 = 1]) { + if (o1 == null) + throw 'o1 is null'; + if (o2 == null) + throw 'o2 is null'; + if (o3 == null) + throw 'o3 is null'; + if (o4 == null) + throw 'o4 is null'; + if (o5 == null) + throw 'o5 is null'; + if (o6 == null) + throw 'o6 is null'; + if (o7 == null) + throw 'o7 is null'; + if (o8 == null) + throw 'o8 is null'; + if (o9 == null) + throw 'o9 is null'; + if (o10 == null) + throw 'o10 is null'; +} + +/// Add benchmark values to a JSON results file. +/// +/// If the file contains information about how long the benchmark took to run +/// (a `time` field), then return that info. +// TODO(yjbanov): move this data to __metadata__ +num addBuildInfo(File jsonFile, + {num expected, String sdk, String commit, DateTime timestamp}) { + Map json; + + if (jsonFile.existsSync()) + json = JSON.decode(jsonFile.readAsStringSync()); + else + json = {}; + + if (expected != null) + json['expected'] = expected; + if (sdk != null) + json['sdk'] = sdk; + if (commit != null) + json['commit'] = commit; + if (timestamp != null) + json['timestamp'] = timestamp.millisecondsSinceEpoch; + + jsonFile.writeAsStringSync(jsonEncode(json)); + + // Return the elapsed time of the benchmark (if any). + return json['time']; +} + +/// Splits [from] into lines and selects those that contain [pattern]. +Iterable grep(Pattern pattern, {@required String from}) { + return from.split('\n').where((String line) { + return line.contains(pattern); + }); +} + +/// Captures asynchronous stack traces thrown by [callback]. +/// +/// This is a convenience wrapper around [Chain] optimized for use with +/// `async`/`await`. +/// +/// Example: +/// +/// try { +/// await captureAsyncStacks(() { /* async things */ }); +/// } catch (error, chain) { +/// +/// } +Future runAndCaptureAsyncStacks(Future callback()) { + Completer completer = new Completer(); + Chain.capture(() async { + await callback(); + completer.complete(); + }, onError: (dynamic error, Chain chain) async { + completer.completeError(error, chain); + }); + return completer.future; +} diff --git a/dev/devicelab/lib/tasks/analysis.dart b/dev/devicelab/lib/tasks/analysis.dart new file mode 100644 index 0000000000..46036d956f --- /dev/null +++ b/dev/devicelab/lib/tasks/analysis.dart @@ -0,0 +1,116 @@ +// Copyright (c) 2016 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'dart:async'; +import 'dart:io'; + +import 'package:meta/meta.dart'; +import 'package:path/path.dart' as path; + +import '../framework/benchmarks.dart'; +import '../framework/framework.dart'; +import '../framework/utils.dart'; + +TaskFunction createAnalyzerCliTest({ + @required String sdk, + @required String commit, + @required DateTime timestamp, +}) { + return new AnalyzerCliTask(sdk, commit, timestamp); +} + +TaskFunction createAnalyzerServerTest({ + @required String sdk, + @required String commit, + @required DateTime timestamp, +}) { + return new AnalyzerServerTask(sdk, commit, timestamp); +} + +abstract class AnalyzerTask { + Benchmark benchmark; + + Future call() async { + section(benchmark.name); + await runBenchmark(benchmark, iterations: 3, warmUpBenchmark: true); + return benchmark.bestResult; + } +} + +class AnalyzerCliTask extends AnalyzerTask { + AnalyzerCliTask(String sdk, String commit, DateTime timestamp) { + this.benchmark = new FlutterAnalyzeBenchmark(sdk, commit, timestamp); + } +} + +class AnalyzerServerTask extends AnalyzerTask { + AnalyzerServerTask(String sdk, String commit, DateTime timestamp) { + this.benchmark = new FlutterAnalyzeAppBenchmark(sdk, commit, timestamp); + } +} + +class FlutterAnalyzeBenchmark extends Benchmark { + FlutterAnalyzeBenchmark(this.sdk, this.commit, this.timestamp) + : super('flutter analyze --flutter-repo'); + + final String sdk; + final String commit; + final DateTime timestamp; + + File get benchmarkFile => + file(path.join(flutterDirectory.path, 'analysis_benchmark.json')); + + @override + TaskResult get lastResult => new TaskResult.successFromFile(benchmarkFile); + + @override + Future run() async { + rm(benchmarkFile); + await inDirectory(flutterDirectory, () async { + await flutter('analyze', options: [ + '--flutter-repo', + '--benchmark', + ]); + }); + return addBuildInfo(benchmarkFile, + timestamp: timestamp, expected: 25.0, sdk: sdk, commit: commit); + } +} + +class FlutterAnalyzeAppBenchmark extends Benchmark { + FlutterAnalyzeAppBenchmark(this.sdk, this.commit, this.timestamp) + : super('analysis server mega_gallery'); + + final String sdk; + final String commit; + final DateTime timestamp; + + @override + TaskResult get lastResult => new TaskResult.successFromFile(benchmarkFile); + + Directory get megaDir => dir( + path.join(flutterDirectory.path, 'dev/benchmarks/mega_gallery')); + File get benchmarkFile => + file(path.join(megaDir.path, 'analysis_benchmark.json')); + + @override + Future init() { + return inDirectory(flutterDirectory, () async { + await dart(['dev/tools/mega_gallery.dart']); + }); + } + + @override + Future run() async { + rm(benchmarkFile); + await inDirectory(megaDir, () async { + await flutter('analyze', options: [ + '--watch', + '--benchmark', + ]); + }); + return addBuildInfo(benchmarkFile, + timestamp: timestamp, expected: 10.0, sdk: sdk, commit: commit); + } +} diff --git a/dev/devicelab/lib/tasks/gallery.dart b/dev/devicelab/lib/tasks/gallery.dart new file mode 100644 index 0000000000..873ff1a008 --- /dev/null +++ b/dev/devicelab/lib/tasks/gallery.dart @@ -0,0 +1,58 @@ +// Copyright 2016 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'dart:async'; +import 'dart:convert'; +import 'dart:io'; + +import 'package:meta/meta.dart'; + +import '../framework/adb.dart'; +import '../framework/framework.dart'; +import '../framework/utils.dart'; + +TaskFunction createGalleryTransitionTest({ @required bool ios: false }) { + return new GalleryTransitionTest(ios: ios); +} + +class GalleryTransitionTest { + GalleryTransitionTest({ this.ios }); + + final bool ios; + + Future call() async { + String deviceId = await getUnlockedDeviceId(ios: ios); + Directory galleryDirectory = + dir('${flutterDirectory.path}/examples/flutter_gallery'); + await inDirectory(galleryDirectory, () async { + await pub('get'); + + if (ios) { + // This causes an Xcode project to be created. + await flutter('build', options: ['ios', '--profile']); + } + + await flutter('drive', options: [ + '--profile', + '--trace-startup', + '-t', + 'test_driver/transitions_perf.dart', + '-d', + deviceId, + ]); + }); + + // Route paths contains slashes, which Firebase doesn't accept in keys, so we + // remove them. + Map original = JSON.decode(file( + '${galleryDirectory.path}/build/transition_durations.timeline.json') + .readAsStringSync()); + Map clean = new Map.fromIterable( + original.keys, + key: (String key) => key.replaceAll('/', ''), + value: (String key) => original[key]); + + return new TaskResult.success(clean); + } +} diff --git a/dev/devicelab/lib/tasks/perf_tests.dart b/dev/devicelab/lib/tasks/perf_tests.dart new file mode 100644 index 0000000000..681558430c --- /dev/null +++ b/dev/devicelab/lib/tasks/perf_tests.dart @@ -0,0 +1,164 @@ +// Copyright (c) 2016 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'dart:async'; +import 'dart:convert' show JSON; + +import 'package:meta/meta.dart'; + +import '../framework/adb.dart'; +import '../framework/framework.dart'; +import '../framework/utils.dart'; + +TaskFunction createComplexLayoutScrollPerfTest({ @required bool ios: false }) { + return new PerfTest( + '${flutterDirectory.path}/dev/benchmarks/complex_layout', + 'test_driver/scroll_perf.dart', + 'complex_layout_scroll_perf', + ios: ios + ); +} + +TaskFunction createFlutterGalleryStartupTest({ bool ios: false }) { + return new StartupTest( + '${flutterDirectory.path}/examples/flutter_gallery', + ios: ios + ); +} + +TaskFunction createComplexLayoutStartupTest({ bool ios: false }) { + return new StartupTest( + '${flutterDirectory.path}/dev/benchmarks/complex_layout', + ios: ios + ); +} + +TaskFunction createFlutterGalleryBuildTest() { + return new BuildTest('${flutterDirectory.path}/examples/flutter_gallery'); +} + +TaskFunction createComplexLayoutBuildTest() { + return new BuildTest('${flutterDirectory.path}/dev/benchmarks/complex_layout'); +} + +/// Measure application startup performance. +class StartupTest { + static const Duration _startupTimeout = const Duration(minutes: 2); + + StartupTest(this.testDirectory, { this.ios }); + + final String testDirectory; + final bool ios; + + Future call() async { + return await inDirectory(testDirectory, () async { + String deviceId = await getUnlockedDeviceId(ios: ios); + await pub('get'); + + if (ios) { + // This causes an Xcode project to be created. + await flutter('build', options: ['ios', '--profile']); + } + + await flutter('run', options: [ + '--profile', + '--trace-startup', + '-d', + deviceId, + ]).timeout(_startupTimeout); + Map data = JSON.decode(file('$testDirectory/build/start_up_info.json').readAsStringSync()); + return new TaskResult.success(data, benchmarkScoreKeys: [ + 'engineEnterTimestampMicros', + 'timeToFirstFrameMicros', + ]); + }); + } +} + +/// Measures application runtime performance, specifically per-frame +/// performance. +class PerfTest { + + PerfTest(this.testDirectory, this.testTarget, this.timelineFileName, { this.ios }); + + final String testDirectory; + final String testTarget; + final String timelineFileName; + final bool ios; + + Future call() { + return inDirectory(testDirectory, () async { + String deviceId = await getUnlockedDeviceId(ios: ios); + await pub('get'); + + if (ios) { + // This causes an Xcode project to be created. + await flutter('build', options: ['ios', '--profile']); + } + + await flutter('drive', options: [ + '-v', + '--profile', + '--trace-startup', // Enables "endless" timeline event buffering. + '-t', + testTarget, + '-d', + deviceId, + ]); + Map data = JSON.decode(file('$testDirectory/build/$timelineFileName.timeline_summary.json').readAsStringSync()); + return new TaskResult.success(data, benchmarkScoreKeys: [ + 'average_frame_build_time_millis', + 'worst_frame_build_time_millis', + 'missed_frame_build_budget_count', + ]); + }); + } +} + +class BuildTest { + + BuildTest(this.testDirectory); + + final String testDirectory; + + Future call() async { + return await inDirectory(testDirectory, () async { + Adb device = await adb(); + await device.unlock(); + await pub('get'); + + Stopwatch watch = new Stopwatch()..start(); + await flutter('build', options: [ + 'aot', + '--profile', + '--no-pub', + '--target-platform', 'android-arm' // Generate blobs instead of assembly. + ]); + watch.stop(); + + int vmisolateSize = file("$testDirectory/build/aot/snapshot_aot_vmisolate").lengthSync(); + int isolateSize = file("$testDirectory/build/aot/snapshot_aot_isolate").lengthSync(); + int instructionsSize = file("$testDirectory/build/aot/snapshot_aot_instr").lengthSync(); + int rodataSize = file("$testDirectory/build/aot/snapshot_aot_rodata").lengthSync(); + int totalSize = vmisolateSize + isolateSize + instructionsSize + rodataSize; + + Map data = { + 'aot_snapshot_build_millis': watch.elapsedMilliseconds, + 'aot_snapshot_size_vmisolate': vmisolateSize, + 'aot_snapshot_size_isolate': isolateSize, + 'aot_snapshot_size_instructions': instructionsSize, + 'aot_snapshot_size_rodata': rodataSize, + 'aot_snapshot_size_total': totalSize, + }; + return new TaskResult.success(data, benchmarkScoreKeys: [ + 'aot_snapshot_build_millis', + 'aot_snapshot_size_vmisolate', + 'aot_snapshot_size_isolate', + 'aot_snapshot_size_instructions', + 'aot_snapshot_size_rodata', + 'aot_snapshot_size_total', + ]); + }); + } +} diff --git a/dev/devicelab/lib/tasks/refresh.dart b/dev/devicelab/lib/tasks/refresh.dart new file mode 100644 index 0000000000..5bea990905 --- /dev/null +++ b/dev/devicelab/lib/tasks/refresh.dart @@ -0,0 +1,75 @@ +// Copyright (c) 2016 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'dart:async'; +import 'dart:io'; + +import 'package:path/path.dart' as path; + +import '../framework/adb.dart'; +import '../framework/benchmarks.dart'; +import '../framework/framework.dart'; +import '../framework/utils.dart'; + +TaskFunction createRefreshTest({ String commit, DateTime timestamp }) => + new EditRefreshTask(commit, timestamp); + +class EditRefreshTask { + EditRefreshTask(this.commit, this.timestamp) { + assert(commit != null); + assert(timestamp != null); + } + + final String commit; + final DateTime timestamp; + + Future call() async { + Adb device = await adb(); + await device.unlock(); + Benchmark benchmark = new EditRefreshBenchmark(commit, timestamp); + section(benchmark.name); + await runBenchmark(benchmark, iterations: 3, warmUpBenchmark: true); + return benchmark.bestResult; + } +} + +class EditRefreshBenchmark extends Benchmark { + EditRefreshBenchmark(this.commit, this.timestamp) : super('edit refresh'); + + final String commit; + final DateTime timestamp; + + Directory get megaDir => dir( + path.join(flutterDirectory.path, 'dev/benchmarks/mega_gallery')); + File get benchmarkFile => + file(path.join(megaDir.path, 'refresh_benchmark.json')); + + @override + TaskResult get lastResult => new TaskResult.successFromFile(benchmarkFile); + + @override + Future init() { + return inDirectory(flutterDirectory, () async { + await dart(['dev/tools/mega_gallery.dart']); + }); + } + + @override + Future run() async { + Adb device = await adb(); + rm(benchmarkFile); + int exitCode = await inDirectory(megaDir, () async { + return await flutter('run', + options: ['-d', device.deviceId, '--benchmark'], + canFail: true); + }); + if (exitCode != 0) return new Future.error(exitCode); + return addBuildInfo( + benchmarkFile, + timestamp: timestamp, + expected: 200, + commit: commit, + ); + } +} diff --git a/dev/devicelab/lib/tasks/size_tests.dart b/dev/devicelab/lib/tasks/size_tests.dart new file mode 100644 index 0000000000..215162abd9 --- /dev/null +++ b/dev/devicelab/lib/tasks/size_tests.dart @@ -0,0 +1,38 @@ +// Copyright 2016 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'dart:io'; + +import '../framework/framework.dart'; +import '../framework/utils.dart'; + +TaskFunction createBasicMaterialAppSizeTest() { + return () async { + const String sampleAppName = 'sample_flutter_app'; + Directory sampleDir = dir('${Directory.systemTemp.path}/$sampleAppName'); + + if (await sampleDir.exists()) + rmTree(sampleDir); + + int apkSizeInBytes; + + await inDirectory(Directory.systemTemp, () async { + await flutter('create', options: [sampleAppName]); + + if (!(await sampleDir.exists())) + throw 'Failed to create sample Flutter app in ${sampleDir.path}'; + + await inDirectory(sampleDir, () async { + await pub('get'); + await flutter('build', options: ['clean']); + await flutter('build', options: ['apk', '--release']); + apkSizeInBytes = await file('${sampleDir.path}/build/app.apk').length(); + }); + }); + + return new TaskResult.success( + {'release_size_in_bytes': apkSizeInBytes}, + benchmarkScoreKeys: ['release_size_in_bytes']); + }; +} diff --git a/dev/devicelab/manifest.yaml b/dev/devicelab/manifest.yaml new file mode 100644 index 0000000000..21afb54417 --- /dev/null +++ b/dev/devicelab/manifest.yaml @@ -0,0 +1,133 @@ +# Describes the tasks we run in the continuous integration (CI) environment. +# +# Cocoon[1] uses this file to generate a checklist of tasks to be performed for +# every master commit. +# +# [1] github.com/flutter/cocoon + +# CI tasks. +# +# Each key in this dictionary is the unique name of a task, which also +# corresponds to a file in the "bin/" directory that the task runner will run. +# +# Due to historic reasons that may go away at some point, the suffix of the task +# name is significant. It is used by the dashboard to pick the right HTML +# template to display the results in a card. If you use a known name suffix also +# make sure that your task outputs data in the expected format for that card. +# +# Known suffixes: +# +# __analysis_time: +# Analyzer performance benchmarks. +# __refresh_time: +# Edit refresh cycle benchmarks. +# __start_up: +# Application startup speed benchmarks. +# __timeline_summary: +# Per-frame timings and missed/average/total counts. +# __transition_perf: +# Flutter Gallery app transitions benchmark. +# __size: +# Application size benchmarks. + +tasks: + # Deviceless tests + + # TODO: make these not require "has-android-device"; it is only there to + # ensure we have the Android SDK. + + flutter_gallery__build: + description: > + Collects various performance metrics from AOT builds of the Flutter + Gallery. + stage: devicelab + required_agent_capabilities: ["has-android-device"] + + complex_layout__build: + description: > + Collects various performance metrics from AOT builds of the Complex + Layout sample app. + stage: devicelab + required_agent_capabilities: ["has-android-device"] + + basic_material_app__size: + description: > + Measures the APK/IPA sizes of a basic material app. + stage: devicelab + required_agent_capabilities: ["has-android-device"] + + analyzer_cli__analysis_time: + description: > + Measures the speed of analyzing Flutter itself in batch mode. + stage: devicelab + required_agent_capabilities: ["has-android-device"] + + analyzer_server__analysis_time: + description: > + Measures the speed of analyzing Flutter itself in server mode. + stage: devicelab + required_agent_capabilities: ["has-android-device"] + + + # Android on-device tests + + complex_layout_scroll_perf__timeline_summary: + description: > + Measures the runtime performance of the Complex Layout sample app on + Android. + stage: devicelab + required_agent_capabilities: ["has-android-device"] + + flutter_gallery__start_up: + description: > + Measures the startup time of the Flutter Gallery app on Android. + stage: devicelab + required_agent_capabilities: ["has-android-device"] + + complex_layout__start_up: + description: > + Measures the startup time of the Complex Layout sample app on Android. + stage: devicelab + required_agent_capabilities: ["has-android-device"] + + flutter_gallery__transition_perf: + description: > + Measures the performance of screen transitions in Flutter Gallery on + Android. + stage: devicelab + required_agent_capabilities: ["has-android-device"] + + mega_gallery__refresh_time: + description: > + Measures AOT snapshot rebuild performance on a generated large app. + stage: devicelab + required_agent_capabilities: ["has-android-device"] + + + # iOS on-device tests + + complex_layout_scroll_perf_ios__timeline_summary: + description: > + Measures the runtime performance of the Complex Layout sample app on + iOS. + stage: devicelab_ios + required_agent_capabilities: ["has-ios-device"] + + flutter_gallery_ios__start_up: + stage: devicelab_ios + required_agent_capabilities: ["has-ios-device"] + description: > + Measures the startup time of the Flutter Gallery app on iOS. + + complex_layout_ios__start_up: + description: > + Measures the startup time of the Complex Layout sample app on iOS. + stage: devicelab_ios + required_agent_capabilities: ["has-ios-device"] + + flutter_gallery_ios__transition_perf: + stage: devicelab_ios + required_agent_capabilities: ["has-ios-device"] + description: > + Measures the performance of screen transitions in Flutter Gallery on + iOS. diff --git a/dev/devicelab/pubspec.yaml b/dev/devicelab/pubspec.yaml new file mode 100644 index 0000000000..bab13d3919 --- /dev/null +++ b/dev/devicelab/pubspec.yaml @@ -0,0 +1,22 @@ +name: flutter_devicelab +version: 0.0.1 +author: Flutter Authors +description: Flutter continuous integration performance and correctness tests. +homepage: https://github.com/flutter/flutter + +environment: + sdk: '>=1.12.0 <2.0.0' + +dependencies: + args: ^0.13.4 + meta: ^1.0.3 + path: ^1.3.0 + stack_trace: ^1.4.0 + vm_service_client: '^0.2.0' + + # See packages/flutter_test/pubspec.yaml for why we're pinning this version + analyzer: 0.28.2-alpha.0 + +dev_dependencies: + # See packages/flutter_test/pubspec.yaml for why we're pinning this version + test: 0.12.15+4 diff --git a/dev/devicelab/test/adb_test.dart b/dev/devicelab/test/adb_test.dart new file mode 100644 index 0000000000..8eb4a2bc48 --- /dev/null +++ b/dev/devicelab/test/adb_test.dart @@ -0,0 +1,190 @@ +// Copyright 2016 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'dart:async'; + +import 'package:test/test.dart'; +import 'package:collection/collection.dart'; + +import 'package:flutter_devicelab/framework/adb.dart'; + +void main() { + group('adb', () { + Adb device; + + setUp(() { + FakeAdb.resetLog(); + adb = null; + device = new FakeAdb(); + }); + + tearDown(() { + adb = realAdbGetter; + }); + + group('isAwake/isAsleep', () { + test('reads Awake', () async { + FakeAdb.pretendAwake(); + expect(await device.isAwake(), isTrue); + expect(await device.isAsleep(), isFalse); + }); + + test('reads Asleep', () async { + FakeAdb.pretendAsleep(); + expect(await device.isAwake(), isFalse); + expect(await device.isAsleep(), isTrue); + }); + }); + + group('togglePower', () { + test('sends power event', () async { + await device.togglePower(); + expectLog([ + cmd(command: 'input', arguments: ['keyevent', '26']), + ]); + }); + }); + + group('wakeUp', () { + test('when awake', () async { + FakeAdb.pretendAwake(); + await device.wakeUp(); + expectLog([ + cmd(command: 'dumpsys', arguments: ['power']), + ]); + }); + + test('when asleep', () async { + FakeAdb.pretendAsleep(); + await device.wakeUp(); + expectLog([ + cmd(command: 'dumpsys', arguments: ['power']), + cmd(command: 'input', arguments: ['keyevent', '26']), + ]); + }); + }); + + group('sendToSleep', () { + test('when asleep', () async { + FakeAdb.pretendAsleep(); + await device.sendToSleep(); + expectLog([ + cmd(command: 'dumpsys', arguments: ['power']), + ]); + }); + + test('when awake', () async { + FakeAdb.pretendAwake(); + await device.sendToSleep(); + expectLog([ + cmd(command: 'dumpsys', arguments: ['power']), + cmd(command: 'input', arguments: ['keyevent', '26']), + ]); + }); + }); + + group('unlock', () { + test('sends unlock event', () async { + FakeAdb.pretendAwake(); + await device.unlock(); + expectLog([ + cmd(command: 'dumpsys', arguments: ['power']), + cmd(command: 'input', arguments: ['keyevent', '82']), + ]); + }); + }); + }); +} + +void expectLog(List log) { + expect(FakeAdb.commandLog, log); +} + +CommandArgs cmd({ String command, List arguments, Map env }) => new CommandArgs( + command: command, + arguments: arguments, + env: env +); + +typedef dynamic ExitErrorFactory(); + +class CommandArgs { + CommandArgs({ this.command, this.arguments, this.env }); + + final String command; + final List arguments; + final Map env; + + @override + String toString() => 'CommandArgs(command: $command, arguments: $arguments, env: $env)'; + + @override + bool operator==(Object other) { + if (other.runtimeType != CommandArgs) + return false; + + CommandArgs otherCmd = other; + return otherCmd.command == this.command && + const ListEquality().equals(otherCmd.arguments, this.arguments) && + const MapEquality().equals(otherCmd.env, this.env); + } + + @override + int get hashCode => 17 * (17 * command.hashCode + _hashArguments) + _hashEnv; + + int get _hashArguments => arguments != null + ? const ListEquality().hash(arguments) + : null.hashCode; + + int get _hashEnv => env != null + ? const MapEquality().hash(env) + : null.hashCode; +} + +class FakeAdb extends Adb { + FakeAdb({ String deviceId: null }) : super(deviceId: deviceId); + + static String output = ''; + static ExitErrorFactory exitErrorFactory = () => null; + + static List commandLog = []; + + static void resetLog() { + commandLog.clear(); + } + + static void pretendAwake() { + output = ''' + mWakefulness=Awake + '''; + } + + static void pretendAsleep() { + output = ''' + mWakefulness=Asleep + '''; + } + + @override + Future shellEval(String command, List arguments, {Map env}) async { + commandLog.add(new CommandArgs( + command: command, + arguments: arguments, + env: env + )); + return output; + } + + @override + Future shellExec(String command, List arguments, {Map env}) async { + commandLog.add(new CommandArgs( + command: command, + arguments: arguments, + env: env + )); + dynamic exitError = exitErrorFactory(); + if (exitError != null) + throw exitError; + } +} diff --git a/dev/devicelab/test/all.dart b/dev/devicelab/test/all.dart new file mode 100644 index 0000000000..a6bc062e8a --- /dev/null +++ b/dev/devicelab/test/all.dart @@ -0,0 +1,13 @@ +// Copyright 2016 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'adb_test.dart' as adb_test; +import 'manifest_test.dart' as manifest_test; +import 'run_test.dart' as run_test; + +void main() { + adb_test.main(); + manifest_test.main(); + run_test.main(); +} diff --git a/dev/devicelab/test/manifest_test.dart b/dev/devicelab/test/manifest_test.dart new file mode 100644 index 0000000000..28eb0d99a1 --- /dev/null +++ b/dev/devicelab/test/manifest_test.dart @@ -0,0 +1,143 @@ +// Copyright 2016 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'package:test/test.dart'; + +import 'package:flutter_devicelab/framework/manifest.dart'; + +void main() { + group('production manifest', () { + test('must be valid', () { + Manifest manifest = loadTaskManifest(); + expect(manifest.tasks, isNotEmpty); + + ManifestTask task = manifest.tasks.firstWhere((ManifestTask task) => task.name == 'flutter_gallery__start_up'); + expect(task.description, 'Measures the startup time of the Flutter Gallery app on Android.\n'); + expect(task.stage, 'devicelab'); + expect(task.requiredAgentCapabilities, ['has-android-device']); + }); + }); + + group('manifest parser', () { + void testManifestError( + String testDescription, + String errorMessage, + String yaml, + ) { + test(testDescription, () { + try { + loadTaskManifest(yaml); + } on ManifestError catch(error) { + expect(error.message, errorMessage); + } + }); + } + + testManifestError( + 'invalid top-level type', + 'Manifest must be a dictionary but was YamlScalar: null', + '', + ); + + testManifestError( + 'invalid top-level key', + 'Unrecognized property "bad" in manifest. Allowed properties: tasks', + ''' + bad: + key: yes + ''', + ); + + testManifestError( + 'invalid tasks list type', + 'Value of "tasks" must be a dictionary but was YamlList: [a, b]', + ''' + tasks: + - a + - b + ''' + ); + + testManifestError( + 'invalid task name type', + 'Task name must be a string but was int: 1', + ''' + tasks: + 1: 2 + ''' + ); + + testManifestError( + 'invalid task type', + 'Value of task "foo" must be a dictionary but was int: 2', + ''' + tasks: + foo: 2 + ''' + ); + + testManifestError( + 'invalid task property', + 'Unrecognized property "bar" in Value of task "foo". Allowed properties: description, stage, required_agent_capabilities', + ''' + tasks: + foo: + bar: 2 + ''' + ); + + testManifestError( + 'invalid required_agent_capabilities type', + 'required_agent_capabilities must be a list but was int: 1', + ''' + tasks: + foo: + required_agent_capabilities: 1 + ''' + ); + + testManifestError( + 'invalid required_agent_capabilities element type', + 'required_agent_capabilities[0] must be a string but was int: 1', + ''' + tasks: + foo: + required_agent_capabilities: [1] + ''' + ); + + testManifestError( + 'missing description', + 'Task description must not be empty in task "foo".', + ''' + tasks: + foo: + required_agent_capabilities: ["a"] + ''' + ); + + testManifestError( + 'missing stage', + 'Task stage must not be empty in task "foo".', + ''' + tasks: + foo: + description: b + required_agent_capabilities: ["a"] + ''' + ); + + testManifestError( + 'missing stage', + 'requiredAgentCapabilities must not be empty in task "foo".', + ''' + tasks: + foo: + description: b + stage: c + required_agent_capabilities: [] + ''' + ); + }); +} diff --git a/dev/devicelab/test/run_test.dart b/dev/devicelab/test/run_test.dart new file mode 100644 index 0000000000..382f32a000 --- /dev/null +++ b/dev/devicelab/test/run_test.dart @@ -0,0 +1,50 @@ +// Copyright 2016 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'dart:async'; +import 'dart:io'; + +import 'package:test/test.dart'; + +void main() { + group('run.dart script', () { + Future runScript(List testNames) async { + List options = ['bin/run.dart']; + for (String testName in testNames) { + options..addAll(['-t', testName]); + } + Process scriptProcess = await Process.start( + '../../bin/cache/dart-sdk/bin/dart', + options, + ); + return scriptProcess.exitCode; + } + + test('Exits with code 0 when succeeds', () async { + expect(await runScript(['smoke_test_success']), 0); + }); + + test('Exits with code 1 when task throws', () async { + expect(await runScript(['smoke_test_throws']), 1); + }); + + test('Exits with code 1 when fails', () async { + expect(await runScript(['smoke_test_failure']), 1); + }); + + test('Exits with code 1 when fails to connect', () async { + expect(await runScript(['smoke_test_setup_failure']), 1); + }); + + test('Exits with code 1 when results are mixed', () async { + expect( + await runScript([ + 'smoke_test_failure', + 'smoke_test_success', + ]), + 1, + ); + }); + }); +} diff --git a/dev/devicelab/test/utils_test.dart b/dev/devicelab/test/utils_test.dart new file mode 100644 index 0000000000..03bb51e16f --- /dev/null +++ b/dev/devicelab/test/utils_test.dart @@ -0,0 +1,18 @@ +// Copyright (c) 2016 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'package:test/test.dart'; +import 'package:flutter_devicelab/framework/utils.dart'; + +void main() { + group('grep', () { + test('greps lines', () { + expect(grep('b', from: 'ab\ncd\nba'), ['ab', 'ba']); + }); + + test('understands RegExp', () { + expect(grep(new RegExp('^b'), from: 'ab\nba'), ['ba']); + }); + }); +} diff --git a/packages/flutter/pubspec.yaml b/packages/flutter/pubspec.yaml index 4a18ee7e20..c996c60567 100644 --- a/packages/flutter/pubspec.yaml +++ b/packages/flutter/pubspec.yaml @@ -7,7 +7,7 @@ homepage: http://flutter.io dependencies: collection: '>=1.9.1 <2.0.0' intl: '>=0.14.0 <0.15.0' - meta: ^1.0.2 + meta: ^1.0.3 vector_math: '>=2.0.3 <3.0.0' sky_engine: diff --git a/packages/flutter_tools/pubspec.yaml b/packages/flutter_tools/pubspec.yaml index d2f75dfcf8..41b927d7fd 100644 --- a/packages/flutter_tools/pubspec.yaml +++ b/packages/flutter_tools/pubspec.yaml @@ -22,7 +22,7 @@ dependencies: json_schema: 1.0.3 linter: ^0.1.21 - meta: ^1.0.0 + meta: ^1.0.3 mustache4dart: ^1.0.0 package_config: '>=0.1.5 <2.0.0' path: ^1.3.0