[devicelab] Add results path flag to test runner (#72765)

dev/devicelab/bin/run.dart

@@ -42,6 +42,9 @@ String luciBuilder;
 /// Whether to exit on first test failure.
 bool exitOnFirstTestFailure;
 
+/// Path to write test results to.
+String resultsPath;
+
 /// File containing a service account token.
 ///
 /// If passed, the test run results will be uploaded to Flutter infrastructure.
@@ -65,6 +68,16 @@ Future<void> main(List<String> rawArgs) async {
     return;
   }
 
+  deviceId = args['device-id'] as String;
+  exitOnFirstTestFailure = args['exit'] as bool;
+  gitBranch = args['git-branch'] as String;
+  localEngine = args['local-engine'] as String;
+  localEngineSrcPath = args['local-engine-src-path'] as String;
+  luciBuilder = args['luci-builder'] as String;
+  resultsPath = args['results-file'] as String;
+  serviceAccountTokenFile = args['service-account-token-file'] as String;
+  silent = args['silent'] as bool;
+
   if (!args.wasParsed('task')) {
     if (args.wasParsed('stage') || args.wasParsed('all')) {
       addTasks(
@@ -89,15 +102,6 @@ Future<void> main(List<String> rawArgs) async {
     return;
   }
 
-  deviceId = args['device-id'] as String;
-  exitOnFirstTestFailure = args['exit'] as bool;
-  gitBranch = args['git-branch'] as String;
-  localEngine = args['local-engine'] as String;
-  localEngineSrcPath = args['local-engine-src-path'] as String;
-  luciBuilder = args['luci-builder'] as String;
-  serviceAccountTokenFile = args['service-account-token-file'] as String;
-  silent = args['silent'] as bool;
-
   if (args.wasParsed('ab')) {
     await _runABTest();
   } else {
@@ -120,8 +124,17 @@ Future<void> _runTasks() async {
     print(const JsonEncoder.withIndent('  ').convert(result));
     section('Finished task "$taskName"');
 
-    if (serviceAccountTokenFile != null) {
+    if (resultsPath != null) {
+      final Cocoon cocoon = Cocoon();
+      await cocoon.writeTaskResultToFile(
+        builderName: luciBuilder,
+        gitBranch: gitBranch,
+        result: result,
+        resultsPath: resultsPath,
+      );
+    } else if (serviceAccountTokenFile != null) {
       final Cocoon cocoon = Cocoon(serviceAccountTokenPath: serviceAccountTokenFile);
+
       /// Cocoon references LUCI tasks by the [luciBuilder] instead of [taskName].
       await cocoon.sendTaskResult(builderName: luciBuilder, result: result, gitBranch: gitBranch);
     }
@@ -224,7 +237,7 @@ File _uniqueFile(String filenameTemplate) {
   File file = File(parts[0] + parts[1]);
   int i = 1;
   while (file.existsSync()) {
-    file = File(parts[0]+i.toString()+parts[1]);
+    file = File(parts[0] + i.toString() + parts[1]);
     i++;
   }
   return file;
@@ -355,10 +368,7 @@ final ArgParser _argParser = ArgParser()
           'locally. Defaults to \$FLUTTER_ENGINE if set, or tries to guess at\n'
           'the location based on the value of the --flutter-root option.',
   )
-  ..addOption(
+  ..addOption('luci-builder', help: '[Flutter infrastructure] Name of the LUCI builder being run on.')
-    'luci-builder',
-    help: '[Flutter infrastructure] Name of the LUCI builder being run on.'
-  )
   ..addFlag(
     'match-host-platform',
     defaultsTo: true,
@@ -367,6 +377,11 @@ final ArgParser _argParser = ArgParser()
           'on a windows host). Each test publishes its '
           '`required_agent_capabilities`\nin the `manifest.yaml` file.',
   )
+  ..addOption(
+    'results-file',
+    help: '[Flutter infrastructure] File path for test results. If passed with\n'
+          'task, will write test results to the file.'
+  )
   ..addOption(
     'service-account-token-file',
     help: '[Flutter infrastructure] Authentication for uploading results.',

dev/devicelab/bin/test_runner.dart

@@ -0,0 +1,21 @@
+// Copyright 2014 The Flutter Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+import 'dart:io';
+
+import 'package:args/command_runner.dart';
+import 'package:flutter_devicelab/command/upload_metrics.dart';
+
+final CommandRunner<void> runner =
+    CommandRunner<void>('devicelab_runner', 'DeviceLab test runner for recording performance metrics on applications')
+      ..addCommand(UploadMetricsCommand());
+
+Future<void> main(List<String> rawArgs) async {
+  runner.run(rawArgs).catchError((dynamic error) {
+    stderr.writeln('$error\n');
+    stderr.writeln('Usage:\n');
+    stderr.writeln(runner.usage);
+    exit(64); // Exit code 64 indicates a usage error.
+  });
+}

dev/devicelab/lib/command/upload_metrics.dart

@@ -0,0 +1,32 @@
+// Copyright 2014 The Flutter Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+import 'package:args/command_runner.dart';
+
+import '../framework/cocoon.dart';
+
+class UploadMetricsCommand extends Command<void> {
+  UploadMetricsCommand() {
+    argParser.addOption('results-file', help: 'Test results JSON to upload to Cocoon.');
+    argParser.addOption(
+      'service-account-token-file',
+      help: 'Authentication token for uploading results.',
+    );
+  }
+
+  @override
+  String get name => 'upload-metrics';
+
+  @override
+  String get description => '[Flutter infrastructure] Upload metrics data to Cocoon';
+
+  @override
+  Future<void> run() async {
+    final String resultsPath = argResults['results-file'] as String;
+    final String serviceAccountTokenFile = argResults['service-account-token-file'] as String;
+
+    final Cocoon cocoon = Cocoon(serviceAccountTokenPath: serviceAccountTokenFile);
+    return cocoon.sendResultsPath(resultsPath);
+  }
+}

dev/devicelab/lib/framework/cocoon.dart

@@ -34,9 +34,9 @@ class Cocoon {
   Cocoon({
     String serviceAccountTokenPath,
     @visibleForTesting Client httpClient,
-    @visibleForTesting FileSystem filesystem,
+    @visibleForTesting this.fs = const LocalFileSystem(),
     @visibleForTesting this.processRunSync = Process.runSync,
-  }) : _httpClient = AuthenticatedCocoonClient(serviceAccountTokenPath, httpClient: httpClient, filesystem: filesystem);
+  }) : _httpClient = AuthenticatedCocoonClient(serviceAccountTokenPath, httpClient: httpClient, filesystem: fs);
 
   /// Client to make http requests to Cocoon.
   final AuthenticatedCocoonClient _httpClient;
@@ -46,6 +46,9 @@ class Cocoon {
   /// Url used to send results to.
   static const String baseCocoonApiUrl = 'https://flutter-dashboard.appspot.com/api';
 
+  /// Underlying [FileSystem] to use.
+  final FileSystem fs;
+
   static final Logger logger = Logger('CocoonClient');
 
   String get commitSha => _commitSha ?? _readCommitSha();
@@ -61,8 +64,25 @@ class Cocoon {
     return _commitSha = result.stdout as String;
   }
 
+  /// Upload the JSON results in [resultsPath] to Cocoon.
+  ///
+  /// Flutter infrastructure's workflow is:
+  /// 1. Run DeviceLab test, writing results to a known path
+  /// 2. Request service account token from luci auth (valid for at least 3 minutes)
+  /// 3. Upload results from (1) to Cocooon
+  Future<void> sendResultsPath(String resultsPath) async {
+    final File resultFile = fs.file(resultsPath);
+    final Map<String, dynamic> resultsJson = json.decode(await resultFile.readAsString()) as Map<String, dynamic>;
+    await _sendUpdateTaskRequest(resultsJson);
+  }
+
   /// Send [TaskResult] to Cocoon.
-  Future<void> sendTaskResult({@required String builderName, @required TaskResult result, @required String gitBranch}) async {
+  // TODO(chillers): Remove when sendResultsPath is used in prod. https://github.com/flutter/flutter/issues/72457
+  Future<void> sendTaskResult({
+    @required String builderName,
+    @required TaskResult result,
+    @required String gitBranch,
+  }) async {
     assert(builderName != null);
     assert(gitBranch != null);
     assert(result != null);
@@ -73,7 +93,45 @@ class Cocoon {
       print('${rec.level.name}: ${rec.time}: ${rec.message}');
     });
 
-    final Map<String, dynamic> status = <String, dynamic>{
+    final Map<String, dynamic> updateRequest = _constructUpdateRequest(
+      gitBranch: gitBranch,
+      builderName: builderName,
+      result: result,
+    );
+    await _sendUpdateTaskRequest(updateRequest);
+  }
+
+  /// Write the given parameters into an update task request and store the JSON in [resultsPath].
+  Future<void> writeTaskResultToFile({
+    @required String builderName,
+    @required String gitBranch,
+    @required TaskResult result,
+    @required String resultsPath,
+  }) async {
+    assert(builderName != null);
+    assert(gitBranch != null);
+    assert(result != null);
+    assert(resultsPath != null);
+
+    final Map<String, dynamic> updateRequest = _constructUpdateRequest(
+      gitBranch: gitBranch,
+      builderName: builderName,
+      result: result,
+    );
+    final File resultFile = fs.file(resultsPath);
+    if (resultFile.existsSync()) {
+      resultFile.deleteSync();
+    }
+    resultFile.createSync();
+    resultFile.writeAsStringSync(json.encode(updateRequest));
+  }
+
+  Map<String, dynamic> _constructUpdateRequest({
+    @required String builderName,
+    @required TaskResult result,
+    @required String gitBranch,
+  }) {
+    final Map<String, dynamic> updateRequest = <String, dynamic>{
       'CommitBranch': gitBranch,
       'CommitSha': commitSha,
       'BuilderName': builderName,
@@ -81,7 +139,7 @@ class Cocoon {
     };
 
     // Make a copy of result data because we may alter it for validation below.
-    status['ResultData'] = result.data;
+    updateRequest['ResultData'] = result.data;
 
     final List<String> validScoreKeys = <String>[];
     if (result.benchmarkScoreKeys != null) {
@@ -95,9 +153,13 @@ class Cocoon {
         }
       }
     }
-    status['BenchmarkScoreKeys'] = validScoreKeys;
+    updateRequest['BenchmarkScoreKeys'] = validScoreKeys;
 
-    final Map<String, dynamic> response = await _sendCocoonRequest('update-task-status', status);
+    return updateRequest;
+  }
+
+  Future<void> _sendUpdateTaskRequest(Map<String, dynamic> postBody) async {
+    final Map<String, dynamic> response = await _sendCocoonRequest('update-task-status', postBody);
     if (response['Name'] != null) {
       logger.info('Updated Cocoon with results from this task');
     } else {

dev/devicelab/test/cocoon_test.dart

@@ -48,7 +48,7 @@ void main() {
       _processResult = ProcessResult(1, 0, commitSha, '');
       cocoon = Cocoon(
         serviceAccountTokenPath: serviceAccountTokenPath,
-        filesystem: fs,
+        fs: fs,
         httpClient: mockClient,
         processRunSync: runSyncStub,
       );
@@ -60,7 +60,7 @@ void main() {
       _processResult = ProcessResult(1, 1, '', '');
       cocoon = Cocoon(
         serviceAccountTokenPath: serviceAccountTokenPath,
-        filesystem: fs,
+        fs: fs,
         httpClient: mockClient,
         processRunSync: runSyncStub,
       );
@@ -68,12 +68,69 @@ void main() {
       expect(() => cocoon.commitSha, throwsA(isA<CocoonException>()));
     });
 
+    test('writes expected update task json', () async {
+      _processResult = ProcessResult(1, 0, commitSha, '');
+      final TaskResult result = TaskResult.fromJson(<String, dynamic>{
+        'success': true,
+        'data': <String, dynamic>{
+          'i': 0,
+          'j': 0,
+          'not_a_metric': 'something',
+        },
+        'benchmarkScoreKeys': <String>['i', 'j'],
+      });
+
+      cocoon = Cocoon(
+        fs: fs,
+        processRunSync: runSyncStub,
+      );
+
+      const String resultsPath = 'results.json';
+      await cocoon.writeTaskResultToFile(
+        builderName: 'builderAbc',
+        gitBranch: 'master',
+        result: result,
+        resultsPath: resultsPath,
+      );
+
+      final String resultJson = fs.file(resultsPath).readAsStringSync();
+      const String expectedJson = '{'
+          '"CommitBranch":"master",'
+          '"CommitSha":"$commitSha",'
+          '"BuilderName":"builderAbc",'
+          '"NewStatus":"Succeeded",'
+          '"ResultData":{"i":0.0,"j":0.0,"not_a_metric":"something"},'
+          '"BenchmarkScoreKeys":["i","j"]}';
+      expect(resultJson, expectedJson);
+    });
+
+    test('uploads expected update task payload from results file', () async {
+      _processResult = ProcessResult(1, 0, commitSha, '');
+      cocoon = Cocoon(
+        fs: fs,
+        httpClient: mockClient,
+        processRunSync: runSyncStub,
+        serviceAccountTokenPath: serviceAccountTokenPath,
+      );
+
+      const String resultsPath = 'results.json';
+      const String updateTaskJson = '{'
+          '"CommitBranch":"master",'
+          '"CommitSha":"$commitSha",'
+          '"BuilderName":"builderAbc",'
+          '"NewStatus":"Succeeded",'
+          '"ResultData":{"i":0.0,"j":0.0,"not_a_metric":"something"},'
+          '"BenchmarkScoreKeys":["i","j"]}';
+      fs.file(resultsPath).writeAsStringSync(updateTaskJson);
+      await cocoon.sendResultsPath(resultsPath);
+    });
+
     test('sends expected request from successful task', () async {
       mockClient = MockClient((Request request) async => Response('{}', 200));
 
       cocoon = Cocoon(
         serviceAccountTokenPath: serviceAccountTokenPath,
-        filesystem: fs,
+        fs: fs,
         httpClient: mockClient,
       );
 
@@ -87,12 +144,13 @@ void main() {
 
       cocoon = Cocoon(
         serviceAccountTokenPath: serviceAccountTokenPath,
-        filesystem: fs,
+        fs: fs,
         httpClient: mockClient,
       );
 
       final TaskResult result = TaskResult.success(<String, dynamic>{});
-      expect(() => cocoon.sendTaskResult(builderName: 'builderAbc', gitBranch: 'branchAbc', result: result), throwsA(isA<ClientException>()));
+      expect(() => cocoon.sendTaskResult(builderName: 'builderAbc', gitBranch: 'branchAbc', result: result),
+          throwsA(isA<ClientException>()));
     });
 
     test('null git branch throws error', () async {
@@ -100,12 +158,13 @@ void main() {
 
       cocoon = Cocoon(
         serviceAccountTokenPath: serviceAccountTokenPath,
-        filesystem: fs,
+        fs: fs,
         httpClient: mockClient,
       );
 
       final TaskResult result = TaskResult.success(<String, dynamic>{});
-      expect(() => cocoon.sendTaskResult(builderName: 'builderAbc', gitBranch: null, result: result), throwsA(isA<AssertionError>()));
+      expect(() => cocoon.sendTaskResult(builderName: 'builderAbc', gitBranch: null, result: result),
+          throwsA(isA<AssertionError>()));
     });
   });