firka/flutter
firka/flutter
5
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

[devicelab] Add results path flag to test runner (#72765)

Browse Source
This commit is contained in:
Casey Hillers 2020-12-22 16:45:38 -08:00 committed by GitHub
parent dafc13f11d
commit 849784e262
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 218 additions and 29 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

45
dev/devicelab/bin/run.dart
View File

@ -42,6 +42,9 @@ String luciBuilder;
/// Whether to exit on first test failure.
bool exitOnFirstTestFailure;
/// Path to write test results to.
String resultsPath;
/// File containing a service account token.
///
/// If passed, the test run results will be uploaded to Flutter infrastructure.
@ -65,6 +68,16 @@ Future<void> main(List<String> rawArgs) async {
return;
}
deviceId = args['device-id'] as String;
exitOnFirstTestFailure = args['exit'] as bool;
gitBranch = args['git-branch'] as String;
localEngine = args['local-engine'] as String;
localEngineSrcPath = args['local-engine-src-path'] as String;
luciBuilder = args['luci-builder'] as String;
resultsPath = args['results-file'] as String;
serviceAccountTokenFile = args['service-account-token-file'] as String;
silent = args['silent'] as bool;
if (!args.wasParsed('task')) {
if (args.wasParsed('stage') || args.wasParsed('all')) {
addTasks(
@ -89,15 +102,6 @@ Future<void> main(List<String> rawArgs) async {
return;
}
deviceId = args['device-id'] as String;
exitOnFirstTestFailure = args['exit'] as bool;
gitBranch = args['git-branch'] as String;
localEngine = args['local-engine'] as String;
localEngineSrcPath = args['local-engine-src-path'] as String;
luciBuilder = args['luci-builder'] as String;
serviceAccountTokenFile = args['service-account-token-file'] as String;
silent = args['silent'] as bool;
if (args.wasParsed('ab')) {
await _runABTest();
} else {
@ -120,8 +124,17 @@ Future<void> _runTasks() async {
print(const JsonEncoder.withIndent(' ').convert(result));
section('Finished task "$taskName"');
if (serviceAccountTokenFile != null) {
if (resultsPath != null) {
final Cocoon cocoon = Cocoon();
await cocoon.writeTaskResultToFile(
builderName: luciBuilder,
gitBranch: gitBranch,
result: result,
resultsPath: resultsPath,
);
} else if (serviceAccountTokenFile != null) {
final Cocoon cocoon = Cocoon(serviceAccountTokenPath: serviceAccountTokenFile);
/// Cocoon references LUCI tasks by the [luciBuilder] instead of [taskName].
await cocoon.sendTaskResult(builderName: luciBuilder, result: result, gitBranch: gitBranch);
}
@ -224,7 +237,7 @@ File _uniqueFile(String filenameTemplate) {
File file = File(parts[0] + parts[1]);
int i = 1;
while (file.existsSync()) {
file = File(parts[0]+i.toString()+parts[1]);
file = File(parts[0] + i.toString() + parts[1]);
i++;
}
return file;
@ -355,10 +368,7 @@ final ArgParser _argParser = ArgParser()
'locally. Defaults to \$FLUTTER_ENGINE if set, or tries to guess at\n'
'the location based on the value of the --flutter-root option.',
)
..addOption(
'luci-builder',
help: '[Flutter infrastructure] Name of the LUCI builder being run on.'
)
..addOption('luci-builder', help: '[Flutter infrastructure] Name of the LUCI builder being run on.')
..addFlag(
'match-host-platform',
defaultsTo: true,
@ -367,6 +377,11 @@ final ArgParser _argParser = ArgParser()
'on a windows host). Each test publishes its '
'`required_agent_capabilities`\nin the `manifest.yaml` file.',
)
..addOption(
'results-file',
help: '[Flutter infrastructure] File path for test results. If passed with\n'
'task, will write test results to the file.'
)
..addOption(
'service-account-token-file',
help: '[Flutter infrastructure] Authentication for uploading results.',

21
dev/devicelab/bin/test_runner.dart Normal file
View File

@ -0,0 +1,21 @@
// Copyright 2014 The Flutter Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
import 'dart:io';
import 'package:args/command_runner.dart';
import 'package:flutter_devicelab/command/upload_metrics.dart';
final CommandRunner<void> runner =
CommandRunner<void>('devicelab_runner', 'DeviceLab test runner for recording performance metrics on applications')
..addCommand(UploadMetricsCommand());
Future<void> main(List<String> rawArgs) async {
runner.run(rawArgs).catchError((dynamic error) {
stderr.writeln('$error\n');
stderr.writeln('Usage:\n');
stderr.writeln(runner.usage);
exit(64); // Exit code 64 indicates a usage error.
});
}

32
dev/devicelab/lib/command/upload_metrics.dart Normal file
View File

@ -0,0 +1,32 @@
// Copyright 2014 The Flutter Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
import 'package:args/command_runner.dart';
import '../framework/cocoon.dart';
class UploadMetricsCommand extends Command<void> {
UploadMetricsCommand() {
argParser.addOption('results-file', help: 'Test results JSON to upload to Cocoon.');
argParser.addOption(
'service-account-token-file',
help: 'Authentication token for uploading results.',
);
}
@override
String get name => 'upload-metrics';
@override
String get description => '[Flutter infrastructure] Upload metrics data to Cocoon';
@override
Future<void> run() async {
final String resultsPath = argResults['results-file'] as String;
final String serviceAccountTokenFile = argResults['service-account-token-file'] as String;
final Cocoon cocoon = Cocoon(serviceAccountTokenPath: serviceAccountTokenFile);
return cocoon.sendResultsPath(resultsPath);
}
}

76
dev/devicelab/lib/framework/cocoon.dart
View File

@ -34,9 +34,9 @@ class Cocoon {
Cocoon({
String serviceAccountTokenPath,
@visibleForTesting Client httpClient,
@visibleForTesting FileSystem filesystem,
@visibleForTesting this.fs = const LocalFileSystem(),
@visibleForTesting this.processRunSync = Process.runSync,
}) : _httpClient = AuthenticatedCocoonClient(serviceAccountTokenPath, httpClient: httpClient, filesystem: filesystem);
}) : _httpClient = AuthenticatedCocoonClient(serviceAccountTokenPath, httpClient: httpClient, filesystem: fs);
/// Client to make http requests to Cocoon.
final AuthenticatedCocoonClient _httpClient;
@ -46,6 +46,9 @@ class Cocoon {
/// Url used to send results to.
static const String baseCocoonApiUrl = 'https://flutter-dashboard.appspot.com/api';
/// Underlying [FileSystem] to use.
final FileSystem fs;
static final Logger logger = Logger('CocoonClient');
String get commitSha => _commitSha ?? _readCommitSha();
@ -61,8 +64,25 @@ class Cocoon {
return _commitSha = result.stdout as String;
}
/// Upload the JSON results in [resultsPath] to Cocoon.
///
/// Flutter infrastructure's workflow is:
/// 1. Run DeviceLab test, writing results to a known path
/// 2. Request service account token from luci auth (valid for at least 3 minutes)
/// 3. Upload results from (1) to Cocooon
Future<void> sendResultsPath(String resultsPath) async {
final File resultFile = fs.file(resultsPath);
final Map<String, dynamic> resultsJson = json.decode(await resultFile.readAsString()) as Map<String, dynamic>;
await _sendUpdateTaskRequest(resultsJson);
}
/// Send [TaskResult] to Cocoon.
Future<void> sendTaskResult({@required String builderName, @required TaskResult result, @required String gitBranch}) async {
// TODO(chillers): Remove when sendResultsPath is used in prod. https://github.com/flutter/flutter/issues/72457
Future<void> sendTaskResult({
@required String builderName,
@required TaskResult result,
@required String gitBranch,
}) async {
assert(builderName != null);
assert(gitBranch != null);
assert(result != null);
@ -73,7 +93,45 @@ class Cocoon {
print('${rec.level.name}: ${rec.time}: ${rec.message}');
});
final Map<String, dynamic> status = <String, dynamic>{
final Map<String, dynamic> updateRequest = _constructUpdateRequest(
gitBranch: gitBranch,
builderName: builderName,
result: result,
);
await _sendUpdateTaskRequest(updateRequest);
}
/// Write the given parameters into an update task request and store the JSON in [resultsPath].
Future<void> writeTaskResultToFile({
@required String builderName,
@required String gitBranch,
@required TaskResult result,
@required String resultsPath,
}) async {
assert(builderName != null);
assert(gitBranch != null);
assert(result != null);
assert(resultsPath != null);
final Map<String, dynamic> updateRequest = _constructUpdateRequest(
gitBranch: gitBranch,
builderName: builderName,
result: result,
);
final File resultFile = fs.file(resultsPath);
if (resultFile.existsSync()) {
resultFile.deleteSync();
}
resultFile.createSync();
resultFile.writeAsStringSync(json.encode(updateRequest));
}
Map<String, dynamic> _constructUpdateRequest({
@required String builderName,
@required TaskResult result,
@required String gitBranch,
}) {
final Map<String, dynamic> updateRequest = <String, dynamic>{
'CommitBranch': gitBranch,
'CommitSha': commitSha,
'BuilderName': builderName,
@ -81,7 +139,7 @@ class Cocoon {
};
// Make a copy of result data because we may alter it for validation below.
status['ResultData'] = result.data;
updateRequest['ResultData'] = result.data;
final List<String> validScoreKeys = <String>[];
if (result.benchmarkScoreKeys != null) {
@ -95,9 +153,13 @@ class Cocoon {
}
}
}
status['BenchmarkScoreKeys'] = validScoreKeys;
updateRequest['BenchmarkScoreKeys'] = validScoreKeys;
final Map<String, dynamic> response = await _sendCocoonRequest('update-task-status', status);
return updateRequest;
}
Future<void> _sendUpdateTaskRequest(Map<String, dynamic> postBody) async {
final Map<String, dynamic> response = await _sendCocoonRequest('update-task-status', postBody);
if (response['Name'] != null) {
logger.info('Updated Cocoon with results from this task');
} else {

73
dev/devicelab/test/cocoon_test.dart
View File

@ -48,7 +48,7 @@ void main() {
_processResult = ProcessResult(1, 0, commitSha, '');
cocoon = Cocoon(
serviceAccountTokenPath: serviceAccountTokenPath,
filesystem: fs,
fs: fs,
httpClient: mockClient,
processRunSync: runSyncStub,
);
@ -60,7 +60,7 @@ void main() {
_processResult = ProcessResult(1, 1, '', '');
cocoon = Cocoon(
serviceAccountTokenPath: serviceAccountTokenPath,
filesystem: fs,
fs: fs,
httpClient: mockClient,
processRunSync: runSyncStub,
);
@ -68,12 +68,69 @@ void main() {
expect(() => cocoon.commitSha, throwsA(isA<CocoonException>()));
});
test('writes expected update task json', () async {
_processResult = ProcessResult(1, 0, commitSha, '');
final TaskResult result = TaskResult.fromJson(<String, dynamic>{
'success': true,
'data': <String, dynamic>{
'i': 0,
'j': 0,
'not_a_metric': 'something',
},
'benchmarkScoreKeys': <String>['i', 'j'],
});
cocoon = Cocoon(
fs: fs,
processRunSync: runSyncStub,
);
const String resultsPath = 'results.json';
await cocoon.writeTaskResultToFile(
builderName: 'builderAbc',
gitBranch: 'master',
result: result,
resultsPath: resultsPath,
);
final String resultJson = fs.file(resultsPath).readAsStringSync();
const String expectedJson = '{'
'"CommitBranch":"master",'
'"CommitSha":"$commitSha",'
'"BuilderName":"builderAbc",'
'"NewStatus":"Succeeded",'
'"ResultData":{"i":0.0,"j":0.0,"not_a_metric":"something"},'
'"BenchmarkScoreKeys":["i","j"]}';
expect(resultJson, expectedJson);
});
test('uploads expected update task payload from results file', () async {
_processResult = ProcessResult(1, 0, commitSha, '');
cocoon = Cocoon(
fs: fs,
httpClient: mockClient,
processRunSync: runSyncStub,
serviceAccountTokenPath: serviceAccountTokenPath,
);
const String resultsPath = 'results.json';
const String updateTaskJson = '{'
'"CommitBranch":"master",'
'"CommitSha":"$commitSha",'
'"BuilderName":"builderAbc",'
'"NewStatus":"Succeeded",'
'"ResultData":{"i":0.0,"j":0.0,"not_a_metric":"something"},'
'"BenchmarkScoreKeys":["i","j"]}';
fs.file(resultsPath).writeAsStringSync(updateTaskJson);
await cocoon.sendResultsPath(resultsPath);
});
test('sends expected request from successful task', () async {
mockClient = MockClient((Request request) async => Response('{}', 200));
cocoon = Cocoon(
serviceAccountTokenPath: serviceAccountTokenPath,
filesystem: fs,
fs: fs,
httpClient: mockClient,
);
@ -87,12 +144,13 @@ void main() {
cocoon = Cocoon(
serviceAccountTokenPath: serviceAccountTokenPath,
filesystem: fs,
fs: fs,
httpClient: mockClient,
);
final TaskResult result = TaskResult.success(<String, dynamic>{});
expect(() => cocoon.sendTaskResult(builderName: 'builderAbc', gitBranch: 'branchAbc', result: result), throwsA(isA<ClientException>()));
expect(() => cocoon.sendTaskResult(builderName: 'builderAbc', gitBranch: 'branchAbc', result: result),
throwsA(isA<ClientException>()));
});
test('null git branch throws error', () async {
@ -100,12 +158,13 @@ void main() {
cocoon = Cocoon(
serviceAccountTokenPath: serviceAccountTokenPath,
filesystem: fs,
fs: fs,
httpClient: mockClient,
);
final TaskResult result = TaskResult.success(<String, dynamic>{});
expect(() => cocoon.sendTaskResult(builderName: 'builderAbc', gitBranch: null, result: result), throwsA(isA<AssertionError>()));
expect(() => cocoon.sendTaskResult(builderName: 'builderAbc', gitBranch: null, result: result),
throwsA(isA<AssertionError>()));
});
});