firka/flutter
firka/flutter
5
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Re-land: "Remove Cocoon from dev/devicelab, keeping Skia perf stats upload" (#165755)

Browse Source 
🚫 **Merge after**
https://flutter-review.googlesource.com/c/recipes/+/64300.

---

This reverts commit 0a2d9f56585824689d26a751f3326bf22865ea11.
This commit is contained in:
Matan Lurey 2025-03-22 14:06:17 -07:00 committed by GitHub
parent 0a2d9f5658
commit 5840236ae5
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
8 changed files with 247 additions and 770 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
dev/devicelab/bin/test_runner.dart
View File

@ -8,12 +8,12 @@ import 'dart:io';
import 'package:args/command_runner.dart';
import 'package:flutter_devicelab/command/test.dart';
import 'package:flutter_devicelab/command/upload_results.dart';
import 'package:flutter_devicelab/command/upload_metrics.dart';
final CommandRunner<void> runner =
CommandRunner<void>('devicelab_runner', 'DeviceLab test runner for recording test results')
..addCommand(TestCommand())
..addCommand(UploadResultsCommand());
..addCommand(UploadMetricsCommand());
Future<void> main(List<String> rawArgs) async {
unawaited(

42
dev/devicelab/lib/command/upload_metrics.dart Normal file
View File

@ -0,0 +1,42 @@
// Copyright 2014 The Flutter Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
import 'package:args/command_runner.dart';
import '../framework/metrics_center.dart';
class UploadMetricsCommand extends Command<void> {
UploadMetricsCommand() {
argParser.addOption('results-file', help: 'Test results JSON to upload to Cocoon.');
argParser.addOption('commit-time', help: 'Commit time in UNIX timestamp');
argParser.addOption(
'task-name',
help: '[Flutter infrastructure] Name of the task being run on.',
);
argParser.addOption(
'benchmark-tags',
help: '[Flutter infrastructure] Benchmark tags to surface on Skia Perf',
);
}
@override
String get name => 'upload-metrics';
@override
String get description => '[Flutter infrastructure] Upload results data to Cocoon/Skia Perf';
@override
Future<void> run() async {
final String? resultsPath = argResults!['results-file'] as String?;
final String? commitTime = argResults!['commit-time'] as String?;
final String? taskName = argResults!['task-name'] as String?;
final String? benchmarkTags = argResults!['benchmark-tags'] as String?;
// Upload metrics to skia perf from test runner when `resultsPath` is specified.
if (resultsPath != null) {
await uploadToSkiaPerf(resultsPath, commitTime, taskName, benchmarkTags);
print('Successfully uploaded metrics to skia perf');
}
}
}

84
dev/devicelab/lib/command/upload_results.dart
View File

@ -1,84 +0,0 @@
// Copyright 2014 The Flutter Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
import 'package:args/command_runner.dart';
import '../framework/metrics_center.dart';
class UploadResultsCommand extends Command<void> {
UploadResultsCommand() {
argParser.addOption('results-file', help: 'Test results JSON to upload to Cocoon.');
argParser.addOption(
'service-account-token-file',
help: 'Authentication token for uploading results.',
);
argParser.addOption(
'test-flaky',
help: 'Flag to show whether the test is flaky: "True" or "False"',
);
argParser.addOption(
'git-branch',
help:
'[Flutter infrastructure] Git branch of the current commit. LUCI\n'
'checkouts run in detached HEAD state, so the branch must be passed.',
);
argParser.addOption(
'luci-builder',
help: '[Flutter infrastructure] Name of the LUCI builder being run on.',
);
argParser.addOption(
'task-name',
help: '[Flutter infrastructure] Name of the task being run on.',
);
argParser.addOption(
'benchmark-tags',
help: '[Flutter infrastructure] Benchmark tags to surface on Skia Perf',
);
argParser.addOption('test-status', help: 'Test status: Succeeded|Failed');
argParser.addOption('commit-time', help: 'Commit time in UNIX timestamp');
argParser.addOption(
'builder-bucket',
help: '[Flutter infrastructure] Luci builder bucket the test is running in.',
);
}
@override
String get name => 'upload-metrics';
@override
String get description => '[Flutter infrastructure] Upload results data to Cocoon/Skia Perf';
@override
Future<void> run() async {
final String? resultsPath = argResults!['results-file'] as String?;
// final String? serviceAccountTokenFile = argResults!['service-account-token-file'] as String?;
// final String? testFlakyStatus = argResults!['test-flaky'] as String?;
final String? gitBranch = argResults!['git-branch'] as String?;
final String? builderName = argResults!['luci-builder'] as String?;
final String? testStatus = argResults!['test-status'] as String?;
final String? commitTime = argResults!['commit-time'] as String?;
final String? taskName = argResults!['task-name'] as String?;
final String? benchmarkTags = argResults!['benchmark-tags'] as String?;
final String? builderBucket = argResults!['builder-bucket'] as String?;
// Upload metrics to skia perf from test runner when `resultsPath` is specified.
if (resultsPath != null) {
await uploadToSkiaPerf(resultsPath, commitTime, taskName, benchmarkTags);
print('Successfully uploaded metrics to skia perf');
}
print(
'Intentionally skipping /api/update-task-status ($gitBranch/$builderName/$testStatus/$builderBucket) because yjbanov@ said so',
);
// final Cocoon cocoon = Cocoon(serviceAccountTokenPath: serviceAccountTokenFile);
// return cocoon.sendTaskStatus(
// resultsPath: resultsPath,
// isTestFlaky: testFlakyStatus == 'True',
// gitBranch: gitBranch,
// builderName: builderName,
// testStatus: testStatus,
// builderBucket: builderBucket,
// );
}
}

265
dev/devicelab/lib/framework/cocoon.dart
View File

@ -1,265 +0,0 @@
// Copyright 2014 The Flutter Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
import 'dart:async';
import 'dart:convert' show Encoding, json;
import 'dart:io';
import 'package:file/file.dart';
import 'package:file/local.dart';
import 'package:http/http.dart';
import 'package:logging/logging.dart';
import 'package:meta/meta.dart';
import 'task_result.dart';
import 'utils.dart';
typedef ProcessRunSync =
ProcessResult Function(
String,
List<String>, {
Map<String, String>? environment,
bool includeParentEnvironment,
bool runInShell,
Encoding? stderrEncoding,
Encoding? stdoutEncoding,
String? workingDirectory,
});
/// Class for test runner to interact with Flutter's infrastructure service, Cocoon.
///
/// Cocoon assigns bots to run these devicelab tasks on real devices.
/// To retrieve these results, the test runner needs to send results back so the database can be updated.
class Cocoon {
Cocoon({
String? serviceAccountTokenPath,
@visibleForTesting Client? httpClient,
@visibleForTesting this.fs = const LocalFileSystem(),
@visibleForTesting this.processRunSync = Process.runSync,
@visibleForTesting this.requestRetryLimit = 5,
@visibleForTesting this.requestTimeoutLimit = 30,
}) : _httpClient = AuthenticatedCocoonClient(
serviceAccountTokenPath,
httpClient: httpClient,
filesystem: fs,
);
/// Client to make http requests to Cocoon.
final AuthenticatedCocoonClient _httpClient;
final ProcessRunSync processRunSync;
/// Url used to send results to.
static const String baseCocoonApiUrl = 'https://flutter-dashboard.appspot.com/api';
/// Threshold to auto retry a failed test.
static const int retryNumber = 2;
/// Underlying [FileSystem] to use.
final FileSystem fs;
static final Logger logger = Logger('CocoonClient');
@visibleForTesting
final int requestRetryLimit;
@visibleForTesting
final int requestTimeoutLimit;
String get commitSha => _commitSha ?? _readCommitSha();
String? _commitSha;
/// Parse the local repo for the current running commit.
String _readCommitSha() {
final ProcessResult result = processRunSync('git', <String>['rev-parse', 'HEAD']);
if (result.exitCode != 0) {
throw CocoonException(result.stderr as String);
}
return _commitSha = result.stdout as String;
}
/// Update test status to Cocoon.
///
/// Flutter infrastructure's workflow is:
/// 1. Run DeviceLab test
/// 2. Request service account token from luci auth (valid for at least 3 minutes)
/// 3. Update test status from (1) to Cocoon
///
/// The `resultsPath` is not available for all tests. When it doesn't show up, we
/// need to append `CommitBranch`, `CommitSha`, and `BuilderName`.
Future<void> sendTaskStatus({
String? resultsPath,
bool? isTestFlaky,
String? gitBranch,
String? builderName,
String? testStatus,
String? builderBucket,
}) async {
Map<String, dynamic> resultsJson = <String, dynamic>{};
if (resultsPath != null) {
final File resultFile = fs.file(resultsPath);
resultsJson = json.decode(await resultFile.readAsString()) as Map<String, dynamic>;
} else {
resultsJson['CommitBranch'] = gitBranch;
resultsJson['CommitSha'] = commitSha;
resultsJson['BuilderName'] = builderName;
resultsJson['NewStatus'] = testStatus;
}
resultsJson['TestFlaky'] = isTestFlaky ?? false;
if (_shouldUpdateCocoon(resultsJson, builderBucket ?? 'prod')) {
await retry(
() async =>
_sendUpdateTaskRequest(resultsJson).timeout(Duration(seconds: requestTimeoutLimit)),
retryIf:
(Exception e) => e is SocketException || e is TimeoutException || e is ClientException,
maxAttempts: requestRetryLimit,
);
}
}
/// Only post-submit tests on `master` are allowed to update in cocoon.
bool _shouldUpdateCocoon(Map<String, dynamic> resultJson, String builderBucket) {
const List<String> supportedBranches = <String>['master'];
return supportedBranches.contains(resultJson['CommitBranch']) && builderBucket == 'prod';
}
/// Write the given parameters into an update task request and store the JSON in [resultsPath].
Future<void> writeTaskResultToFile({
String? builderName,
String? gitBranch,
required TaskResult result,
required String resultsPath,
}) async {
final Map<String, dynamic> updateRequest = _constructUpdateRequest(
gitBranch: gitBranch,
builderName: builderName,
result: result,
);
final File resultFile = fs.file(resultsPath);
if (resultFile.existsSync()) {
resultFile.deleteSync();
}
logger.fine('Writing results: ${json.encode(updateRequest)}');
resultFile.createSync();
resultFile.writeAsStringSync(json.encode(updateRequest));
}
Map<String, dynamic> _constructUpdateRequest({
String? builderName,
required TaskResult result,
String? gitBranch,
}) {
final Map<String, dynamic> updateRequest = <String, dynamic>{
'CommitBranch': gitBranch,
'CommitSha': commitSha,
'BuilderName': builderName,
'NewStatus': result.succeeded ? 'Succeeded' : 'Failed',
};
logger.fine('Update request: $updateRequest');
// Make a copy of result data because we may alter it for validation below.
updateRequest['ResultData'] = result.data;
final List<String> validScoreKeys = <String>[];
if (result.benchmarkScoreKeys != null) {
for (final String scoreKey in result.benchmarkScoreKeys!) {
final Object score = result.data![scoreKey] as Object;
if (score is num) {
// Convert all metrics to double, which provide plenty of precision
// without having to add support for multiple numeric types in Cocoon.
result.data![scoreKey] = score.toDouble();
validScoreKeys.add(scoreKey);
}
}
}
updateRequest['BenchmarkScoreKeys'] = validScoreKeys;
return updateRequest;
}
Future<void> _sendUpdateTaskRequest(Map<String, dynamic> postBody) async {
logger.info('Attempting to send update task request to Cocoon.');
final Map<String, dynamic> response = await _sendCocoonRequest('update-task-status', postBody);
if (response['Name'] != null) {
logger.info('Updated Cocoon with results from this task');
} else {
logger.info(response);
logger.severe('Failed to updated Cocoon with results from this task');
}
}
/// Make an API request to Cocoon.
Future<Map<String, dynamic>> _sendCocoonRequest(String apiPath, [dynamic jsonData]) async {
final Uri url = Uri.parse('$baseCocoonApiUrl/$apiPath');
/// Retry requests to Cocoon as sometimes there are issues with the servers, such
/// as version changes to the backend, datastore issues, or latency issues.
final Response response = await retry(
() => _httpClient.post(url, body: json.encode(jsonData)),
retryIf:
(Exception e) => e is SocketException || e is TimeoutException || e is ClientException,
maxAttempts: requestRetryLimit,
);
return json.decode(response.body) as Map<String, dynamic>;
}
}
/// [HttpClient] for sending authenticated requests to Cocoon.
class AuthenticatedCocoonClient extends BaseClient {
AuthenticatedCocoonClient(
this._serviceAccountTokenPath, {
@visibleForTesting Client? httpClient,
@visibleForTesting FileSystem? filesystem,
}) : _delegate = httpClient ?? Client(),
_fs = filesystem ?? const LocalFileSystem();
/// Authentication token to have the ability to upload and record test results.
///
/// This is intended to only be passed on automated runs on LUCI post-submit.
final String? _serviceAccountTokenPath;
/// Underlying [HttpClient] to send requests to.
final Client _delegate;
/// Underlying [FileSystem] to use.
final FileSystem _fs;
/// Value contained in the service account token file that can be used in http requests.
String get serviceAccountToken => _serviceAccountToken ?? _readServiceAccountTokenFile();
String? _serviceAccountToken;
/// Get [serviceAccountToken] from the given service account file.
String _readServiceAccountTokenFile() {
return _serviceAccountToken = _fs.file(_serviceAccountTokenPath).readAsStringSync().trim();
}
@override
Future<StreamedResponse> send(BaseRequest request) async {
request.headers['Service-Account-Token'] = serviceAccountToken;
final StreamedResponse response = await _delegate.send(request);
if (response.statusCode != 200) {
throw ClientException(
'AuthenticatedClientError:\n'
' URI: ${request.url}\n'
' HTTP Status: ${response.statusCode}\n'
' Response body:\n'
'${(await Response.fromStream(response)).body}',
request.url,
);
}
return response;
}
}
class CocoonException implements Exception {
CocoonException(this.message);
/// The message to show to the issuer to explain the error.
final String message;
@override
String toString() => 'CocoonException: $message';
}

119
dev/devicelab/lib/framework/metrics_result_writer.dart Normal file
View File

@ -0,0 +1,119 @@
// Copyright 2014 The Flutter Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
import 'dart:async';
import 'dart:convert' show Encoding, json;
import 'dart:io';
import 'package:file/file.dart';
import 'package:file/local.dart';
import 'package:logging/logging.dart';
import 'package:meta/meta.dart';
import 'task_result.dart';
/// Class for test runner to write device-lab metrics results for Skia Perf.
interface class MetricsResultWriter {
MetricsResultWriter({
@visibleForTesting this.fs = const LocalFileSystem(),
@visibleForTesting this.processRunSync = Process.runSync,
});
final ProcessResult Function(
String,
List<String>, {
Map<String, String>? environment,
bool includeParentEnvironment,
bool runInShell,
Encoding? stderrEncoding,
Encoding? stdoutEncoding,
String? workingDirectory,
})
processRunSync;
/// Threshold to auto retry a failed test.
static const int retryNumber = 2;
/// Underlying [FileSystem] to use.
final FileSystem fs;
static final Logger logger = Logger('CocoonClient');
String get commitSha => _commitSha ?? _readCommitSha();
String? _commitSha;
/// Parse the local repo for the current running commit.
String _readCommitSha() {
final ProcessResult result = processRunSync('git', <String>['rev-parse', 'HEAD']);
if (result.exitCode != 0) {
throw CocoonException(result.stderr as String);
}
return _commitSha = result.stdout as String;
}
/// Write the given parameters into an update task request and store the JSON in [resultsPath].
Future<void> writeTaskResultToFile({
String? builderName,
String? gitBranch,
required TaskResult result,
required String resultsPath,
}) async {
final Map<String, dynamic> updateRequest = _constructUpdateRequest(
gitBranch: gitBranch,
builderName: builderName,
result: result,
);
final File resultFile = fs.file(resultsPath);
if (resultFile.existsSync()) {
resultFile.deleteSync();
}
logger.fine('Writing results: ${json.encode(updateRequest)}');
resultFile.createSync();
resultFile.writeAsStringSync(json.encode(updateRequest));
}
Map<String, dynamic> _constructUpdateRequest({
String? builderName,
required TaskResult result,
String? gitBranch,
}) {
final Map<String, dynamic> updateRequest = <String, dynamic>{
'CommitBranch': gitBranch,
'CommitSha': commitSha,
'BuilderName': builderName,
'NewStatus': result.succeeded ? 'Succeeded' : 'Failed',
};
logger.fine('Update request: $updateRequest');
// Make a copy of result data because we may alter it for validation below.
updateRequest['ResultData'] = result.data;
final List<String> validScoreKeys = <String>[];
if (result.benchmarkScoreKeys != null) {
for (final String scoreKey in result.benchmarkScoreKeys!) {
final Object score = result.data![scoreKey] as Object;
if (score is num) {
// Convert all metrics to double, which provide plenty of precision
// without having to add support for multiple numeric types in Cocoon.
result.data![scoreKey] = score.toDouble();
validScoreKeys.add(scoreKey);
}
}
}
updateRequest['BenchmarkScoreKeys'] = validScoreKeys;
return updateRequest;
}
}
class CocoonException implements Exception {
CocoonException(this.message);
/// The message to show to the issuer to explain the error.
final String message;
@override
String toString() => 'CocoonException: $message';
}

6
dev/devicelab/lib/framework/runner.dart
View File

@ -10,8 +10,8 @@ import 'package:meta/meta.dart';
import 'package:vm_service/vm_service.dart';
import 'package:vm_service/vm_service_io.dart';
import 'cocoon.dart';
import 'devices.dart';
import 'metrics_result_writer.dart';
import 'task_result.dart';
import 'utils.dart';
@ -49,7 +49,7 @@ Future<void> runTasks(
for (final String taskName in taskNames) {
TaskResult result = TaskResult.success(null);
int failureCount = 0;
while (failureCount <= Cocoon.retryNumber) {
while (failureCount <= MetricsResultWriter.retryNumber) {
result = await rerunTask(
taskName,
deviceId: deviceId,
@ -137,7 +137,7 @@ Future<TaskResult> rerunTask(
section('Finished task "$taskName"');
if (resultsPath != null) {
final Cocoon cocoon = Cocoon();
final MetricsResultWriter cocoon = MetricsResultWriter();
await cocoon.writeTaskResultToFile(
builderName: luciBuilder,
gitBranch: gitBranch,

416
dev/devicelab/test/cocoon_test.dart
View File

@ -1,416 +0,0 @@
// Copyright 2014 The Flutter Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
import 'dart:async';
import 'dart:convert';
import 'dart:io';
import 'package:file/file.dart';
import 'package:file/memory.dart';
import 'package:flutter_devicelab/framework/cocoon.dart';
import 'package:flutter_devicelab/framework/task_result.dart';
import 'package:http/http.dart';
import 'package:http/testing.dart';
import 'common.dart';
void main() {
late ProcessResult processResult;
ProcessResult runSyncStub(
String executable,
List<String> args, {
Map<String, String>? environment,
bool includeParentEnvironment = true,
bool runInShell = false,
Encoding? stderrEncoding,
Encoding? stdoutEncoding,
String? workingDirectory,
}) => processResult;
// Expected test values.
const String commitSha = 'a4952838bf288a81d8ea11edfd4b4cd649fa94cc';
const String serviceAccountTokenPath = 'test_account_file';
const String serviceAccountToken = 'test_token';
group('Cocoon', () {
late Client mockClient;
late Cocoon cocoon;
late FileSystem fs;
setUp(() {
fs = MemoryFileSystem();
mockClient = MockClient((Request request) async => Response('{}', 200));
final File serviceAccountFile = fs.file(serviceAccountTokenPath)..createSync();
serviceAccountFile.writeAsStringSync(serviceAccountToken);
});
test('returns expected commit sha', () {
processResult = ProcessResult(1, 0, commitSha, '');
cocoon = Cocoon(
serviceAccountTokenPath: serviceAccountTokenPath,
fs: fs,
httpClient: mockClient,
processRunSync: runSyncStub,
);
expect(cocoon.commitSha, commitSha);
});
test('throws exception on git cli errors', () {
processResult = ProcessResult(1, 1, '', '');
cocoon = Cocoon(
serviceAccountTokenPath: serviceAccountTokenPath,
fs: fs,
httpClient: mockClient,
processRunSync: runSyncStub,
);
expect(() => cocoon.commitSha, throwsA(isA<CocoonException>()));
});
test('writes expected update task json', () async {
processResult = ProcessResult(1, 0, commitSha, '');
final TaskResult result = TaskResult.fromJson(<String, dynamic>{
'success': true,
'data': <String, dynamic>{'i': 0, 'j': 0, 'not_a_metric': 'something'},
'benchmarkScoreKeys': <String>['i', 'j'],
});
cocoon = Cocoon(fs: fs, processRunSync: runSyncStub);
const String resultsPath = 'results.json';
await cocoon.writeTaskResultToFile(
builderName: 'builderAbc',
gitBranch: 'master',
result: result,
resultsPath: resultsPath,
);
final String resultJson = fs.file(resultsPath).readAsStringSync();
const String expectedJson =
'{'
'"CommitBranch":"master",'
'"CommitSha":"$commitSha",'
'"BuilderName":"builderAbc",'
'"NewStatus":"Succeeded",'
'"ResultData":{"i":0.0,"j":0.0,"not_a_metric":"something"},'
'"BenchmarkScoreKeys":["i","j"]}';
expect(resultJson, expectedJson);
});
test('uploads metrics sends expected post body', () async {
processResult = ProcessResult(1, 0, commitSha, '');
const String uploadMetricsRequestWithSpaces =
'{"CommitBranch":"master","CommitSha":"a4952838bf288a81d8ea11edfd4b4cd649fa94cc","BuilderName":"builder a b c","NewStatus":"Succeeded","ResultData":{},"BenchmarkScoreKeys":[],"TestFlaky":false}';
final MockClient client = MockClient((Request request) async {
if (request.body == uploadMetricsRequestWithSpaces) {
return Response('{}', 200);
}
return Response(
'Expected: $uploadMetricsRequestWithSpaces\nReceived: ${request.body}',
500,
);
});
cocoon = Cocoon(
fs: fs,
httpClient: client,
processRunSync: runSyncStub,
serviceAccountTokenPath: serviceAccountTokenPath,
requestRetryLimit: 0,
);
const String resultsPath = 'results.json';
const String updateTaskJson =
'{'
'"CommitBranch":"master",'
'"CommitSha":"$commitSha",'
'"BuilderName":"builder a b c",' //ignore: missing_whitespace_between_adjacent_strings
'"NewStatus":"Succeeded",'
'"ResultData":{},'
'"BenchmarkScoreKeys":[]}';
fs.file(resultsPath).writeAsStringSync(updateTaskJson);
await cocoon.sendTaskStatus(resultsPath: resultsPath);
});
test('uploads expected update task payload from results file', () async {
processResult = ProcessResult(1, 0, commitSha, '');
cocoon = Cocoon(
fs: fs,
httpClient: mockClient,
processRunSync: runSyncStub,
serviceAccountTokenPath: serviceAccountTokenPath,
requestRetryLimit: 0,
);
const String resultsPath = 'results.json';
const String updateTaskJson =
'{'
'"CommitBranch":"master",'
'"CommitSha":"$commitSha",'
'"BuilderName":"builderAbc",'
'"NewStatus":"Succeeded",'
'"ResultData":{"i":0.0,"j":0.0,"not_a_metric":"something"},'
'"BenchmarkScoreKeys":["i","j"]}';
fs.file(resultsPath).writeAsStringSync(updateTaskJson);
await cocoon.sendTaskStatus(resultsPath: resultsPath);
});
test('Verify retries for task result upload', () async {
int requestCount = 0;
mockClient = MockClient((Request request) async {
requestCount++;
if (requestCount == 1) {
return Response('{}', 500);
} else {
return Response('{}', 200);
}
});
processResult = ProcessResult(1, 0, commitSha, '');
cocoon = Cocoon(
fs: fs,
httpClient: mockClient,
processRunSync: runSyncStub,
serviceAccountTokenPath: serviceAccountTokenPath,
requestRetryLimit: 3,
);
const String resultsPath = 'results.json';
const String updateTaskJson =
'{'
'"CommitBranch":"master",'
'"CommitSha":"$commitSha",'
'"BuilderName":"builderAbc",'
'"NewStatus":"Succeeded",'
'"ResultData":{"i":0.0,"j":0.0,"not_a_metric":"something"},'
'"BenchmarkScoreKeys":["i","j"]}';
fs.file(resultsPath).writeAsStringSync(updateTaskJson);
await cocoon.sendTaskStatus(resultsPath: resultsPath);
});
test('Verify timeout and retry for task result upload', () async {
int requestCount = 0;
const int timeoutValue = 2;
mockClient = MockClient((Request request) async {
requestCount++;
if (requestCount == 1) {
await Future<void>.delayed(const Duration(seconds: timeoutValue + 2));
throw Exception('Should not reach this, because timeout should trigger');
} else {
return Response('{}', 200);
}
});
processResult = ProcessResult(1, 0, commitSha, '');
cocoon = Cocoon(
fs: fs,
httpClient: mockClient,
processRunSync: runSyncStub,
serviceAccountTokenPath: serviceAccountTokenPath,
requestRetryLimit: 2,
requestTimeoutLimit: timeoutValue,
);
const String resultsPath = 'results.json';
const String updateTaskJson =
'{'
'"CommitBranch":"master",'
'"CommitSha":"$commitSha",'
'"BuilderName":"builderAbc",'
'"NewStatus":"Succeeded",'
'"ResultData":{"i":0.0,"j":0.0,"not_a_metric":"something"},'
'"BenchmarkScoreKeys":["i","j"]}';
fs.file(resultsPath).writeAsStringSync(updateTaskJson);
await cocoon.sendTaskStatus(resultsPath: resultsPath);
});
test('Verify timeout does not trigger for result upload', () async {
int requestCount = 0;
const int timeoutValue = 2;
mockClient = MockClient((Request request) async {
requestCount++;
if (requestCount == 1) {
await Future<void>.delayed(const Duration(seconds: timeoutValue - 1));
return Response('{}', 200);
} else {
throw Exception('This iteration should not be reached, since timeout should not happen.');
}
});
processResult = ProcessResult(1, 0, commitSha, '');
cocoon = Cocoon(
fs: fs,
httpClient: mockClient,
processRunSync: runSyncStub,
serviceAccountTokenPath: serviceAccountTokenPath,
requestRetryLimit: 2,
requestTimeoutLimit: timeoutValue,
);
const String resultsPath = 'results.json';
const String updateTaskJson =
'{'
'"CommitBranch":"master",'
'"CommitSha":"$commitSha",'
'"BuilderName":"builderAbc",'
'"NewStatus":"Succeeded",'
'"ResultData":{"i":0.0,"j":0.0,"not_a_metric":"something"},'
'"BenchmarkScoreKeys":["i","j"]}';
fs.file(resultsPath).writeAsStringSync(updateTaskJson);
await cocoon.sendTaskStatus(resultsPath: resultsPath);
});
test('Verify failure without retries for task result upload', () async {
int requestCount = 0;
mockClient = MockClient((Request request) async {
requestCount++;
if (requestCount == 1) {
return Response('{}', 500);
} else {
return Response('{}', 200);
}
});
processResult = ProcessResult(1, 0, commitSha, '');
cocoon = Cocoon(
fs: fs,
httpClient: mockClient,
processRunSync: runSyncStub,
serviceAccountTokenPath: serviceAccountTokenPath,
requestRetryLimit: 0,
);
const String resultsPath = 'results.json';
const String updateTaskJson =
'{'
'"CommitBranch":"master",'
'"CommitSha":"$commitSha",'
'"BuilderName":"builderAbc",'
'"NewStatus":"Succeeded",'
'"ResultData":{"i":0.0,"j":0.0,"not_a_metric":"something"},'
'"BenchmarkScoreKeys":["i","j"]}';
fs.file(resultsPath).writeAsStringSync(updateTaskJson);
expect(
() => cocoon.sendTaskStatus(resultsPath: resultsPath),
throwsA(isA<ClientException>()),
);
});
test('throws client exception on non-200 responses', () async {
mockClient = MockClient((Request request) async => Response('', 500));
cocoon = Cocoon(
serviceAccountTokenPath: serviceAccountTokenPath,
fs: fs,
httpClient: mockClient,
requestRetryLimit: 0,
);
const String resultsPath = 'results.json';
const String updateTaskJson =
'{'
'"CommitBranch":"master",'
'"CommitSha":"$commitSha",'
'"BuilderName":"builderAbc",'
'"NewStatus":"Succeeded",'
'"ResultData":{"i":0.0,"j":0.0,"not_a_metric":"something"},'
'"BenchmarkScoreKeys":["i","j"]}';
fs.file(resultsPath).writeAsStringSync(updateTaskJson);
expect(
() => cocoon.sendTaskStatus(resultsPath: resultsPath),
throwsA(isA<ClientException>()),
);
});
test('does not upload results on non-supported branches', () async {
// Any network failure would cause the upload to fail
mockClient = MockClient((Request request) async => Response('', 500));
cocoon = Cocoon(
serviceAccountTokenPath: serviceAccountTokenPath,
fs: fs,
httpClient: mockClient,
requestRetryLimit: 0,
);
const String resultsPath = 'results.json';
const String updateTaskJson =
'{'
'"CommitBranch":"stable",'
'"CommitSha":"$commitSha",'
'"BuilderName":"builderAbc",'
'"NewStatus":"Succeeded",'
'"ResultData":{"i":0.0,"j":0.0,"not_a_metric":"something"},'
'"BenchmarkScoreKeys":["i","j"]}';
fs.file(resultsPath).writeAsStringSync(updateTaskJson);
// This will fail if it decided to upload results
await cocoon.sendTaskStatus(resultsPath: resultsPath);
});
test('does not update for staging test', () async {
// Any network failure would cause the upload to fail
mockClient = MockClient((Request request) async => Response('', 500));
cocoon = Cocoon(
serviceAccountTokenPath: serviceAccountTokenPath,
fs: fs,
httpClient: mockClient,
requestRetryLimit: 0,
);
const String resultsPath = 'results.json';
const String updateTaskJson =
'{'
'"CommitBranch":"master",'
'"CommitSha":"$commitSha",'
'"BuilderName":"builderAbc",'
'"NewStatus":"Succeeded",'
'"ResultData":{"i":0.0,"j":0.0,"not_a_metric":"something"},'
'"BenchmarkScoreKeys":["i","j"]}';
fs.file(resultsPath).writeAsStringSync(updateTaskJson);
// This will fail if it decided to upload results
await cocoon.sendTaskStatus(resultsPath: resultsPath, builderBucket: 'staging');
});
});
group('AuthenticatedCocoonClient', () {
late FileSystem fs;
setUp(() {
fs = MemoryFileSystem();
final File serviceAccountFile = fs.file(serviceAccountTokenPath)..createSync();
serviceAccountFile.writeAsStringSync(serviceAccountToken);
});
test('reads token from service account file', () {
final AuthenticatedCocoonClient client = AuthenticatedCocoonClient(
serviceAccountTokenPath,
filesystem: fs,
);
expect(client.serviceAccountToken, serviceAccountToken);
});
test('reads token from service account file with whitespace', () {
final File serviceAccountFile = fs.file(serviceAccountTokenPath)..createSync();
serviceAccountFile.writeAsStringSync('$serviceAccountToken \n');
final AuthenticatedCocoonClient client = AuthenticatedCocoonClient(
serviceAccountTokenPath,
filesystem: fs,
);
expect(client.serviceAccountToken, serviceAccountToken);
});
test('throws error when service account file not found', () {
final AuthenticatedCocoonClient client = AuthenticatedCocoonClient(
'idontexist',
filesystem: fs,
);
expect(() => client.serviceAccountToken, throwsA(isA<FileSystemException>()));
});
});
}

81
dev/devicelab/test/metrics_result_writer_test.dart Normal file
View File

@ -0,0 +1,81 @@
// Copyright 2014 The Flutter Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
import 'dart:convert';
import 'dart:io';
import 'package:file/file.dart';
import 'package:file/memory.dart';
import 'package:flutter_devicelab/framework/metrics_result_writer.dart';
import 'package:flutter_devicelab/framework/task_result.dart';
import 'common.dart';
void main() {
late ProcessResult processResult;
ProcessResult runSyncStub(
String executable,
List<String> args, {
Map<String, String>? environment,
bool includeParentEnvironment = true,
bool runInShell = false,
Encoding? stderrEncoding,
Encoding? stdoutEncoding,
String? workingDirectory,
}) => processResult;
// Expected test values.
const String commitSha = 'a4952838bf288a81d8ea11edfd4b4cd649fa94cc';
late MetricsResultWriter writer;
late FileSystem fs;
setUp(() {
fs = MemoryFileSystem();
});
test('returns expected commit sha', () {
processResult = ProcessResult(1, 0, commitSha, '');
writer = MetricsResultWriter(fs: fs, processRunSync: runSyncStub);
expect(writer.commitSha, commitSha);
});
test('throws exception on git cli errors', () {
processResult = ProcessResult(1, 1, '', '');
writer = MetricsResultWriter(fs: fs, processRunSync: runSyncStub);
expect(() => writer.commitSha, throwsA(isA<CocoonException>()));
});
test('writes expected update task json', () async {
processResult = ProcessResult(1, 0, commitSha, '');
final TaskResult result = TaskResult.fromJson(<String, dynamic>{
'success': true,
'data': <String, dynamic>{'i': 0, 'j': 0, 'not_a_metric': 'something'},
'benchmarkScoreKeys': <String>['i', 'j'],
});
writer = MetricsResultWriter(fs: fs, processRunSync: runSyncStub);
const String resultsPath = 'results.json';
await writer.writeTaskResultToFile(
builderName: 'builderAbc',
gitBranch: 'master',
result: result,
resultsPath: resultsPath,
);
final String resultJson = fs.file(resultsPath).readAsStringSync();
const String expectedJson =
'{'
'"CommitBranch":"master",'
'"CommitSha":"$commitSha",'
'"BuilderName":"builderAbc",'
'"NewStatus":"Succeeded",'
'"ResultData":{"i":0.0,"j":0.0,"not_a_metric":"something"},'
'"BenchmarkScoreKeys":["i","j"]}';
expect(resultJson, expectedJson);
});
}