firka/flutter
firka/flutter
5
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Extend test runner command to update test flaky status (#86513)

Browse Source
This commit is contained in:
keyonghan 2021-07-19 09:46:03 -07:00 committed by GitHub
parent 1995da2c0c
commit 3aeb794298
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 90 additions and 100 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
dev/devicelab/bin/test_runner.dart
View File

@ -9,13 +9,13 @@ import 'dart:io';
import 'package:args/command_runner.dart';
import 'package:flutter_devicelab/command/test.dart';
import 'package:flutter_devicelab/command/upload_metrics.dart';
import 'package:flutter_devicelab/command/upload_results.dart';
import 'package:flutter_devicelab/common.dart';
final CommandRunner<void> runner =
CommandRunner<void>('devicelab_runner', 'DeviceLab test runner for recording performance metrics on applications')
CommandRunner<void>('devicelab_runner', 'DeviceLab test runner for recording test results')
..addCommand(TestCommand())
..addCommand(UploadMetricsCommand());
..addCommand(UploadResultsCommand());
Future<void> main(List<String> rawArgs) async {
unawaited(runner.run(rawArgs).catchError((dynamic error) {

32
dev/devicelab/lib/command/upload_metrics.dart
View File

@ -1,32 +0,0 @@
// Copyright 2014 The Flutter Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
import 'package:args/command_runner.dart';
import '../framework/cocoon.dart';
class UploadMetricsCommand extends Command<void> {
UploadMetricsCommand() {
argParser.addOption('results-file', help: 'Test results JSON to upload to Cocoon.');
argParser.addOption(
'service-account-token-file',
help: 'Authentication token for uploading results.',
);
}
@override
String get name => 'upload-metrics';
@override
String get description => '[Flutter infrastructure] Upload metrics data to Cocoon';
@override
Future<void> run() async {
final String resultsPath = argResults!['results-file'] as String;
final String? serviceAccountTokenFile = argResults!['service-account-token-file'] as String?;
final Cocoon cocoon = Cocoon(serviceAccountTokenPath: serviceAccountTokenFile);
return cocoon.sendResultsPath(resultsPath);
}
}

50
dev/devicelab/lib/command/upload_results.dart Normal file
View File

@ -0,0 +1,50 @@
// Copyright 2014 The Flutter Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
import 'package:args/command_runner.dart';
import '../framework/cocoon.dart';
class UploadResultsCommand extends Command<void> {
UploadResultsCommand() {
argParser.addOption('results-file', help: 'Test results JSON to upload to Cocoon.');
argParser.addOption(
'service-account-token-file',
help: 'Authentication token for uploading results.',
);
argParser.addOption('test-flaky', help: 'Flag to show whether the test is flaky');
argParser.addOption(
'git-branch',
help: '[Flutter infrastructure] Git branch of the current commit. LUCI\n'
'checkouts run in detached HEAD state, so the branch must be passed.',
);
argParser.addOption('luci-builder', help: '[Flutter infrastructure] Name of the LUCI builder being run on.');
argParser.addOption('test-status', help: 'Test status: Succeeded|Failed');
}
@override
String get name => 'upload-metrics';
@override
String get description => '[Flutter infrastructure] Upload results data to Cocoon';
@override
Future<void> run() async {
final String? resultsPath = argResults!['results-file'] as String?;
final String? serviceAccountTokenFile = argResults!['service-account-token-file'] as String?;
final bool? isTestFlaky = argResults!['test-flaky'] as bool?;
final String? gitBranch = argResults!['git-branch'] as String?;
final String? builderName = argResults!['luci-builder'] as String?;
final String? testStatus = argResults!['test-status'] as String?;
final Cocoon cocoon = Cocoon(serviceAccountTokenPath: serviceAccountTokenFile);
return cocoon.sendResultsPath(
resultsPath: resultsPath,
isTestFlaky: isTestFlaky,
gitBranch: gitBranch,
builderName: builderName,
testStatus: testStatus,
);
}
}

49
dev/devicelab/lib/framework/cocoon.dart
View File

@ -77,35 +77,28 @@ class Cocoon {
/// 1. Run DeviceLab test, writing results to a known path
/// 2. Request service account token from luci auth (valid for at least 3 minutes)
/// 3. Upload results from (1) to Cocoon
Future<void> sendResultsPath(String resultsPath) async {
final File resultFile = fs.file(resultsPath);
final Map<String, dynamic> resultsJson = json.decode(await resultFile.readAsString()) as Map<String, dynamic>;
await _sendUpdateTaskRequest(resultsJson);
}
/// Send [TaskResult] to Cocoon.
// TODO(chillers): Remove when sendResultsPath is used in prod. https://github.com/flutter/flutter/issues/72457
Future<void> sendTaskResult({
required String builderName,
required TaskResult result,
required String gitBranch,
///
/// The `resultsPath` is not available for all tests. When it doesn't show up, we
/// need to append `CommitBranch`, `CommitSha`, and `BuilderName`.
Future<void> sendResultsPath({
String? resultsPath,
bool? isTestFlaky,
String? gitBranch,
String? builderName,
String? testStatus,
}) async {
assert(builderName != null);
assert(gitBranch != null);
assert(result != null);
// Skip logging on test runs
Logger.root.level = Level.ALL;
Logger.root.onRecord.listen((LogRecord rec) {
print('${rec.level.name}: ${rec.time}: ${rec.message}');
});
final Map<String, dynamic> updateRequest = _constructUpdateRequest(
gitBranch: gitBranch,
builderName: builderName,
result: result,
);
await _sendUpdateTaskRequest(updateRequest);
Map<String, dynamic> resultsJson = <String, dynamic>{};
if (resultsPath != null) {
final File resultFile = fs.file(resultsPath);
resultsJson = json.decode(await resultFile.readAsString()) as Map<String, dynamic>;
} else {
resultsJson['CommitBranch'] = gitBranch;
resultsJson['CommitSha'] = commitSha;
resultsJson['BuilderName'] = builderName;
resultsJson['NewStatus'] = testStatus;
}
resultsJson['TestFlaky'] = isTestFlaky ?? false;
await _sendUpdateTaskRequest(resultsJson);
}
/// Write the given parameters into an update task request and store the JSON in [resultsPath].

53
dev/devicelab/test/cocoon_test.dart
View File

@ -107,14 +107,15 @@ void main() {
test('uploads metrics sends expected post body', () async {
_processResult = ProcessResult(1, 0, commitSha, '');
const String uploadMetricsRequestWithSpaces = '{"CommitBranch":"master","CommitSha":"a4952838bf288a81d8ea11edfd4b4cd649fa94cc","BuilderName":"builder a b c","NewStatus":"Succeeded","ResultData":{},"BenchmarkScoreKeys":[]}';
const String uploadMetricsRequestWithSpaces =
'{"CommitBranch":"master","CommitSha":"a4952838bf288a81d8ea11edfd4b4cd649fa94cc","BuilderName":"builder a b c","NewStatus":"Succeeded","ResultData":{},"BenchmarkScoreKeys":[],"TestFlaky":false}';
final MockClient client = MockClient((Request request) async {
if (request.body == uploadMetricsRequestWithSpaces) {
return Response('{}', 200);
}
return Response('Expected: $uploadMetricsRequestWithSpaces\nReceived: ${request.body}', 500);
});
});
cocoon = Cocoon(
fs: fs,
httpClient: client,
@ -127,12 +128,12 @@ void main() {
const String updateTaskJson = '{'
'"CommitBranch":"master",'
'"CommitSha":"$commitSha",'
'"BuilderName":"builder a b c",' //ignore: missing_whitespace_between_adjacent_strings
'"BuilderName":"builder a b c",' //ignore: missing_whitespace_between_adjacent_strings
'"NewStatus":"Succeeded",'
'"ResultData":{},'
'"BenchmarkScoreKeys":[]}';
fs.file(resultsPath).writeAsStringSync(updateTaskJson);
await cocoon.sendResultsPath(resultsPath);
await cocoon.sendResultsPath(resultsPath: resultsPath);
});
test('uploads expected update task payload from results file', () async {
@ -154,22 +155,7 @@ void main() {
'"ResultData":{"i":0.0,"j":0.0,"not_a_metric":"something"},'
'"BenchmarkScoreKeys":["i","j"]}';
fs.file(resultsPath).writeAsStringSync(updateTaskJson);
await cocoon.sendResultsPath(resultsPath);
});
test('sends expected request from successful task', () async {
mockClient = MockClient((Request request) async => Response('{}', 200));
cocoon = Cocoon(
serviceAccountTokenPath: serviceAccountTokenPath,
fs: fs,
httpClient: mockClient,
requestRetryLimit: 0,
);
final TaskResult result = TaskResult.success(<String, dynamic>{});
// This should not throw an error.
await cocoon.sendTaskResult(builderName: 'builderAbc', gitBranch: 'branchAbc', result: result);
await cocoon.sendResultsPath(resultsPath: resultsPath);
});
test('throws client exception on non-200 responses', () async {
@ -182,25 +168,18 @@ void main() {
requestRetryLimit: 0,
);
final TaskResult result = TaskResult.success(<String, dynamic>{});
expect(() => cocoon.sendTaskResult(builderName: 'builderAbc', gitBranch: 'branchAbc', result: result),
const String resultsPath = 'results.json';
const String updateTaskJson = '{'
'"CommitBranch":"master",'
'"CommitSha":"$commitSha",'
'"BuilderName":"builderAbc",'
'"NewStatus":"Succeeded",'
'"ResultData":{"i":0.0,"j":0.0,"not_a_metric":"something"},'
'"BenchmarkScoreKeys":["i","j"]}';
fs.file(resultsPath).writeAsStringSync(updateTaskJson);
expect(() => cocoon.sendResultsPath(resultsPath: resultsPath),
throwsA(isA<ClientException>()));
});
test('null git branch throws error', () async {
mockClient = MockClient((Request request) async => Response('', 500));
cocoon = Cocoon(
serviceAccountTokenPath: serviceAccountTokenPath,
fs: fs,
httpClient: mockClient,
requestRetryLimit: 0,
);
final TaskResult result = TaskResult.success(<String, dynamic>{});
expect(() => cocoon.sendTaskResult(builderName: 'builderAbc', gitBranch: null, result: result),
throwsA(isA<AssertionError>()));
});
});
group('AuthenticatedCocoonClient', () {