add microbenchmarks to devicelab (fixes #7081) (#7396)

dev/benchmarks/microbenchmarks/lib/common.dart

@@ -0,0 +1,79 @@
+// Copyright 2017 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+import 'dart:convert' show JsonEncoder;
+
+import 'package:meta/meta.dart';
+
+/// This class knows how to format benchmark results for machine and human
+/// consumption.
+///
+
+/// Example:
+///
+///     BenchmarkResultPrinter printer = new BenchmarkResultPrinter();
+///     printer.add(
+///       description: 'Average frame time',
+///       value: averageFrameTime,
+///       unit: 'ms',
+///       name: 'average_frame_time',
+///     );
+///     printer.printToStdout();
+///
+class BenchmarkResultPrinter {
+
+  final List<_BenchmarkResult> _results = <_BenchmarkResult>[];
+
+  /// Adds a benchmark result to the list of results.
+  ///
+  /// [description] is a human-readable description of the result. [value] is a
+  /// result value. [unit] is the unit of measurement, such as "ms", "km", "h".
+  /// [name] is a computer-readable name of the result used as a key in the JSON
+  /// serialization of the results.
+  void addResult({ @required String description, @required double value, @required String unit, @required String name }) {
+    _results.add(new _BenchmarkResult(description, value, unit, name));
+  }
+
+  /// Prints the results added via [addResult] to standard output, once as JSON
+  /// for computer consumption and once formatted as plain text for humans.
+  void printToStdout() {
+    // IMPORTANT: keep these values in sync with dev/devicelab/bin/tasks/microbenchmarks.dart
+    print('================ RESULTS ================');
+    print(_printJson());
+    print('================ FORMATTED ==============');
+    print(_printPlainText());
+  }
+
+  String _printJson() {
+    const JsonEncoder encoder = const JsonEncoder.withIndent('  ');
+    return encoder.convert(new Map<String, double>.fromIterable(_results,
+      key: (_BenchmarkResult result) => result.name,
+      value: (_BenchmarkResult result) => result.value,
+    ));
+  }
+
+  String _printPlainText() {
+    StringBuffer buf = new StringBuffer();
+    for (_BenchmarkResult result in _results) {
+      buf.writeln('${result.description}: ${result.value.toStringAsFixed(1)} ${result.unit}');
+    }
+    return buf.toString();
+  }
+}
+
+class _BenchmarkResult {
+  _BenchmarkResult(this.description, this.value, this.unit, this.name);
+
+  /// Human-readable description of the result, e.g. "Average frame time".
+  final String description;
+
+  /// Result value that in agreement with [unit].
+  final double value;
+
+  /// Unit of measurement that is in agreement with [value].
+  final String unit;
+
+  /// Computer-readable name of the result.
+  final String name;
+}

dev/benchmarks/microbenchmarks/lib/gestures/velocity_tracker_bench.dart

@@ -6,6 +6,8 @@ import 'dart:io';
 import 'package:flutter/gestures.dart';
 import 'data/velocity_tracker_data.dart';
 
+import '../common.dart';
+
 const int _kNumIters = 10000;
 
 void main() {
@@ -22,6 +24,14 @@ void main() {
     }
   }
   watch.stop();
-  print('Velocity tracker: ${(watch.elapsedMicroseconds / _kNumIters).toStringAsFixed(1)}µs per iteration');
+
+  BenchmarkResultPrinter printer = new BenchmarkResultPrinter();
+  printer.addResult(
+    description: 'Velocity tracker',
+    value: watch.elapsedMicroseconds / _kNumIters,
+    unit: 'µs per iteration',
+    name: 'velocity_tracker_iteration',
+  );
+  printer.printToStdout();
   exit(0);
 }

dev/benchmarks/microbenchmarks/lib/stocks/animation_bench.dart

@@ -11,6 +11,8 @@ import 'package:flutter/scheduler.dart';
 import 'package:stocks/main.dart' as stocks;
 import 'package:stocks/stock_data.dart' as stock_data;
 
+import '../common.dart';
+
 const Duration kBenchmarkTime = const Duration(seconds: 15);
 
 class BenchmarkingBinding extends LiveTestWidgetsFlutterBinding {
@@ -73,10 +75,32 @@ Future<Null> main() async {
     }
   });
 
-  print('Stock animation (ran for ${(wallClockWatch.elapsedMicroseconds / (1000 * 1000)).toStringAsFixed(1)}s):');
+  BenchmarkResultPrinter printer = new BenchmarkResultPrinter();
-  print('  Opening first frame average time: ${(totalOpenFrameElapsedMicroseconds / (totalOpenIterationCount)).toStringAsFixed(1)}µs per frame ($totalOpenIterationCount frames)');
+  printer.addResult(
-  print('  Closing first frame average time: ${(totalCloseFrameElapsedMicroseconds / (totalCloseIterationCount)).toStringAsFixed(1)}µs per frame ($totalCloseIterationCount frames)');
+    description: 'Stock animation',
-  print('  Subsequent frames average time: ${(totalSubsequentFramesElapsedMicroseconds / (totalSubsequentFramesIterationCount)).toStringAsFixed(1)}µs per frame ($totalSubsequentFramesIterationCount frames)');
+    value: wallClockWatch.elapsedMicroseconds / (1000 * 1000),
+    unit: 's',
+    name: 'stock_animation_total_run_time',
+  );
+  printer.addResult(
+    description: '  Opening first frame average time',
+    value: totalOpenFrameElapsedMicroseconds / totalOpenIterationCount,
+    unit: 'µs per frame ($totalOpenIterationCount frames)',
+    name: 'stock_animation_open_first_frame_average',
+  );
+  printer.addResult(
+    description: '  Closing first frame average time',
+    value: totalCloseFrameElapsedMicroseconds / totalCloseIterationCount,
+    unit: 'µs per frame ($totalCloseIterationCount frames)',
+    name: 'stock_animation_close_first_frame_average',
+  );
+  printer.addResult(
+    description: '  Subsequent frames average time',
+    value: totalSubsequentFramesElapsedMicroseconds / totalSubsequentFramesIterationCount,
+    unit: 'µs per frame ($totalSubsequentFramesIterationCount frames)',
+    name: 'stock_animation_subsequent_frame_average',
+  );
+  printer.printToStdout();
 
   exit(0);
 }

dev/benchmarks/microbenchmarks/lib/stocks/build_bench.dart

@@ -6,6 +6,8 @@ import 'package:flutter/rendering.dart';
 import 'package:stocks/main.dart' as stocks;
 import 'package:stocks/stock_data.dart' as stock_data;
 
+import '../common.dart';
+
 const Duration kBenchmarkTime = const Duration(seconds: 15);
 
 Future<Null> main() async {
@@ -36,6 +38,13 @@ Future<Null> main() async {
     watch.stop();
   });
 
-  print('Stock build: ${(watch.elapsedMicroseconds / iterations).toStringAsFixed(1)}µs per iteration');
+  BenchmarkResultPrinter printer = new BenchmarkResultPrinter();
+  printer.addResult(
+    description: 'Stock build',
+    value: watch.elapsedMicroseconds / iterations,
+    unit: 'µs per iteration',
+    name: 'stock_build_iteration',
+  );
+  printer.printToStdout();
   exit(0);
 }

dev/benchmarks/microbenchmarks/lib/stocks/layout_bench.dart

@@ -6,10 +6,10 @@ import 'package:flutter/rendering.dart';
 import 'package:stocks/main.dart' as stocks;
 import 'package:stocks/stock_data.dart' as stock_data;
 
+import '../common.dart';
+
 const Duration kBenchmarkTime = const Duration(seconds: 15);
 
-
-
 Future<Null> main() async {
   stock_data.StockDataFetcher.actuallyFetchData = false;
 
@@ -37,6 +37,13 @@ Future<Null> main() async {
     watch.stop();
   });
 
-  print('Stock layout: ${(watch.elapsedMicroseconds / iterations).toStringAsFixed(1)}µs per iteration');
+  BenchmarkResultPrinter printer = new BenchmarkResultPrinter();
+  printer.addResult(
+    description: 'Stock layout',
+    value: watch.elapsedMicroseconds / iterations,
+    unit: 'µs per iteration',
+    name: 'stock_layout_iteration',
+  );
+  printer.printToStdout();
   exit(0);
 }

dev/benchmarks/microbenchmarks/pubspec.yaml

@@ -1,6 +1,7 @@
 name: microbenchmarks
 description: A new flutter project.
 dependencies:
+  meta: ^1.0.3
   flutter:
     sdk: flutter
   flutter_test:

dev/devicelab/bin/tasks/microbenchmarks.dart

@@ -0,0 +1,90 @@
+// Copyright 2017 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+import 'dart:async';
+import 'dart:convert';
+import 'dart:io';
+
+import 'package:path/path.dart' as path;
+
+import 'package:flutter_devicelab/framework/adb.dart';
+import 'package:flutter_devicelab/framework/framework.dart';
+import 'package:flutter_devicelab/framework/utils.dart';
+
+/// Runs benchmarks in `dev/benchmarks/microbenchmarks` in the device lab and
+/// reports results to the dashboard.
+Future<Null> main() async {
+  await task(() async {
+    Device device = await devices.workingDevice;
+    await device.unlock();
+
+    Future<Map<String, double>> _runMicrobench(String benchmarkPath) async {
+      print('Running $benchmarkPath');
+      Directory appDir = dir(path.join(flutterDirectory.path, 'dev/benchmarks/microbenchmarks'));
+      Process flutterProcess = await inDirectory(appDir, () async {
+        return await _startFlutter(
+          options: <String>[
+            '--release',
+            '-d',
+            device.deviceId,
+            benchmarkPath,
+          ],
+          canFail: false,
+        );
+      });
+
+      return await _readJsonResults(flutterProcess);
+    }
+
+    Map<String, double> allResults = <String, double>{};
+    allResults.addAll(await _runMicrobench('lib/stocks/layout_bench.dart'));
+    allResults.addAll(await _runMicrobench('lib/stocks/build_bench.dart'));
+    allResults.addAll(await _runMicrobench('lib/stocks/animation_bench.dart'));
+    allResults.addAll(await _runMicrobench('lib/gestures/velocity_tracker_bench.dart'));
+
+    return new TaskResult.success(allResults, benchmarkScoreKeys: allResults.keys.toList());
+  });
+}
+
+Future<Process> _startFlutter({String command = 'run', List<String> options: const <String>[], bool canFail: false, Map<String, String> env}) {
+  List<String> args = <String>['run']..addAll(options);
+  return startProcess(path.join(flutterDirectory.path, 'bin', 'flutter'), args, env: env);
+}
+
+Future<Map<String, double>> _readJsonResults(Process process) {
+  // IMPORTANT: keep these values in sync with dev/benchmarks/microbenchmarks/lib/common.dart
+  const String jsonStart = '================ RESULTS ================';
+  const String jsonEnd = '================ FORMATTED ==============';
+  bool jsonStarted = false;
+  StringBuffer jsonBuf = new StringBuffer();
+  Completer<Map<String, double>> completer = new Completer<Map<String, double>>();
+  StreamSubscription<String> stdoutSub;
+
+  int prefixLength = 0;
+  stdoutSub = process.stdout
+      .transform(const Utf8Decoder())
+      .transform(const LineSplitter())
+      .listen((String line) {
+    print(line);
+
+    if (line.contains(jsonStart)) {
+      jsonStarted = true;
+      prefixLength = line.indexOf(jsonStart);
+      return;
+    }
+
+    if (line.contains(jsonEnd)) {
+      jsonStarted = false;
+      stdoutSub.cancel();
+      process.kill(ProcessSignal.SIGINT);  // flutter run doesn't quit automatically
+      completer.complete(JSON.decode(jsonBuf.toString()));
+      return;
+    }
+
+    if (jsonStarted)
+      jsonBuf.writeln(line.substring(prefixLength));
+  });
+
+  return completer.future;
+}

dev/devicelab/manifest.yaml

@@ -127,6 +127,12 @@ tasks:
     stage: devicelab
     required_agent_capabilities: ["has-android-device"]
 
+  microbenchmarks:
+    description: >
+      Runs benchmarks from dev/benchmarks/microbenchmarks.
+    stage: devicelab
+    required_agent_capabilities: ["has-android-device"]
+
 
   # iOS on-device tests