diff --git a/dev/devicelab/bin/test_runner.dart b/dev/devicelab/bin/test_runner.dart index 7d10c7e4c7..d3f4ed4f6a 100644 --- a/dev/devicelab/bin/test_runner.dart +++ b/dev/devicelab/bin/test_runner.dart @@ -8,12 +8,12 @@ import 'dart:io'; import 'package:args/command_runner.dart'; import 'package:flutter_devicelab/command/test.dart'; -import 'package:flutter_devicelab/command/upload_results.dart'; +import 'package:flutter_devicelab/command/upload_metrics.dart'; final CommandRunner runner = CommandRunner('devicelab_runner', 'DeviceLab test runner for recording test results') ..addCommand(TestCommand()) - ..addCommand(UploadResultsCommand()); + ..addCommand(UploadMetricsCommand()); Future main(List rawArgs) async { unawaited( diff --git a/dev/devicelab/lib/command/upload_metrics.dart b/dev/devicelab/lib/command/upload_metrics.dart new file mode 100644 index 0000000000..3ba8522377 --- /dev/null +++ b/dev/devicelab/lib/command/upload_metrics.dart @@ -0,0 +1,42 @@ +// Copyright 2014 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'package:args/command_runner.dart'; + +import '../framework/metrics_center.dart'; + +class UploadMetricsCommand extends Command { + UploadMetricsCommand() { + argParser.addOption('results-file', help: 'Test results JSON to upload to Cocoon.'); + argParser.addOption('commit-time', help: 'Commit time in UNIX timestamp'); + argParser.addOption( + 'task-name', + help: '[Flutter infrastructure] Name of the task being run on.', + ); + argParser.addOption( + 'benchmark-tags', + help: '[Flutter infrastructure] Benchmark tags to surface on Skia Perf', + ); + } + + @override + String get name => 'upload-metrics'; + + @override + String get description => '[Flutter infrastructure] Upload results data to Cocoon/Skia Perf'; + + @override + Future run() async { + final String? resultsPath = argResults!['results-file'] as String?; + final String? commitTime = argResults!['commit-time'] as String?; + final String? taskName = argResults!['task-name'] as String?; + final String? benchmarkTags = argResults!['benchmark-tags'] as String?; + + // Upload metrics to skia perf from test runner when `resultsPath` is specified. + if (resultsPath != null) { + await uploadToSkiaPerf(resultsPath, commitTime, taskName, benchmarkTags); + print('Successfully uploaded metrics to skia perf'); + } + } +} diff --git a/dev/devicelab/lib/command/upload_results.dart b/dev/devicelab/lib/command/upload_results.dart deleted file mode 100644 index 9c15645285..0000000000 --- a/dev/devicelab/lib/command/upload_results.dart +++ /dev/null @@ -1,84 +0,0 @@ -// Copyright 2014 The Flutter Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -import 'package:args/command_runner.dart'; - -import '../framework/metrics_center.dart'; - -class UploadResultsCommand extends Command { - UploadResultsCommand() { - argParser.addOption('results-file', help: 'Test results JSON to upload to Cocoon.'); - argParser.addOption( - 'service-account-token-file', - help: 'Authentication token for uploading results.', - ); - argParser.addOption( - 'test-flaky', - help: 'Flag to show whether the test is flaky: "True" or "False"', - ); - argParser.addOption( - 'git-branch', - help: - '[Flutter infrastructure] Git branch of the current commit. LUCI\n' - 'checkouts run in detached HEAD state, so the branch must be passed.', - ); - argParser.addOption( - 'luci-builder', - help: '[Flutter infrastructure] Name of the LUCI builder being run on.', - ); - argParser.addOption( - 'task-name', - help: '[Flutter infrastructure] Name of the task being run on.', - ); - argParser.addOption( - 'benchmark-tags', - help: '[Flutter infrastructure] Benchmark tags to surface on Skia Perf', - ); - argParser.addOption('test-status', help: 'Test status: Succeeded|Failed'); - argParser.addOption('commit-time', help: 'Commit time in UNIX timestamp'); - argParser.addOption( - 'builder-bucket', - help: '[Flutter infrastructure] Luci builder bucket the test is running in.', - ); - } - - @override - String get name => 'upload-metrics'; - - @override - String get description => '[Flutter infrastructure] Upload results data to Cocoon/Skia Perf'; - - @override - Future run() async { - final String? resultsPath = argResults!['results-file'] as String?; - // final String? serviceAccountTokenFile = argResults!['service-account-token-file'] as String?; - // final String? testFlakyStatus = argResults!['test-flaky'] as String?; - final String? gitBranch = argResults!['git-branch'] as String?; - final String? builderName = argResults!['luci-builder'] as String?; - final String? testStatus = argResults!['test-status'] as String?; - final String? commitTime = argResults!['commit-time'] as String?; - final String? taskName = argResults!['task-name'] as String?; - final String? benchmarkTags = argResults!['benchmark-tags'] as String?; - final String? builderBucket = argResults!['builder-bucket'] as String?; - - // Upload metrics to skia perf from test runner when `resultsPath` is specified. - if (resultsPath != null) { - await uploadToSkiaPerf(resultsPath, commitTime, taskName, benchmarkTags); - print('Successfully uploaded metrics to skia perf'); - } - - print( - 'Intentionally skipping /api/update-task-status ($gitBranch/$builderName/$testStatus/$builderBucket) because yjbanov@ said so', - ); - // final Cocoon cocoon = Cocoon(serviceAccountTokenPath: serviceAccountTokenFile); - // return cocoon.sendTaskStatus( - // resultsPath: resultsPath, - // isTestFlaky: testFlakyStatus == 'True', - // gitBranch: gitBranch, - // builderName: builderName, - // testStatus: testStatus, - // builderBucket: builderBucket, - // ); - } -} diff --git a/dev/devicelab/lib/framework/cocoon.dart b/dev/devicelab/lib/framework/cocoon.dart deleted file mode 100644 index 0ad00662c8..0000000000 --- a/dev/devicelab/lib/framework/cocoon.dart +++ /dev/null @@ -1,265 +0,0 @@ -// Copyright 2014 The Flutter Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -import 'dart:async'; -import 'dart:convert' show Encoding, json; -import 'dart:io'; - -import 'package:file/file.dart'; -import 'package:file/local.dart'; -import 'package:http/http.dart'; -import 'package:logging/logging.dart'; -import 'package:meta/meta.dart'; - -import 'task_result.dart'; -import 'utils.dart'; - -typedef ProcessRunSync = - ProcessResult Function( - String, - List, { - Map? environment, - bool includeParentEnvironment, - bool runInShell, - Encoding? stderrEncoding, - Encoding? stdoutEncoding, - String? workingDirectory, - }); - -/// Class for test runner to interact with Flutter's infrastructure service, Cocoon. -/// -/// Cocoon assigns bots to run these devicelab tasks on real devices. -/// To retrieve these results, the test runner needs to send results back so the database can be updated. -class Cocoon { - Cocoon({ - String? serviceAccountTokenPath, - @visibleForTesting Client? httpClient, - @visibleForTesting this.fs = const LocalFileSystem(), - @visibleForTesting this.processRunSync = Process.runSync, - @visibleForTesting this.requestRetryLimit = 5, - @visibleForTesting this.requestTimeoutLimit = 30, - }) : _httpClient = AuthenticatedCocoonClient( - serviceAccountTokenPath, - httpClient: httpClient, - filesystem: fs, - ); - - /// Client to make http requests to Cocoon. - final AuthenticatedCocoonClient _httpClient; - - final ProcessRunSync processRunSync; - - /// Url used to send results to. - static const String baseCocoonApiUrl = 'https://flutter-dashboard.appspot.com/api'; - - /// Threshold to auto retry a failed test. - static const int retryNumber = 2; - - /// Underlying [FileSystem] to use. - final FileSystem fs; - - static final Logger logger = Logger('CocoonClient'); - - @visibleForTesting - final int requestRetryLimit; - - @visibleForTesting - final int requestTimeoutLimit; - - String get commitSha => _commitSha ?? _readCommitSha(); - String? _commitSha; - - /// Parse the local repo for the current running commit. - String _readCommitSha() { - final ProcessResult result = processRunSync('git', ['rev-parse', 'HEAD']); - if (result.exitCode != 0) { - throw CocoonException(result.stderr as String); - } - - return _commitSha = result.stdout as String; - } - - /// Update test status to Cocoon. - /// - /// Flutter infrastructure's workflow is: - /// 1. Run DeviceLab test - /// 2. Request service account token from luci auth (valid for at least 3 minutes) - /// 3. Update test status from (1) to Cocoon - /// - /// The `resultsPath` is not available for all tests. When it doesn't show up, we - /// need to append `CommitBranch`, `CommitSha`, and `BuilderName`. - Future sendTaskStatus({ - String? resultsPath, - bool? isTestFlaky, - String? gitBranch, - String? builderName, - String? testStatus, - String? builderBucket, - }) async { - Map resultsJson = {}; - if (resultsPath != null) { - final File resultFile = fs.file(resultsPath); - resultsJson = json.decode(await resultFile.readAsString()) as Map; - } else { - resultsJson['CommitBranch'] = gitBranch; - resultsJson['CommitSha'] = commitSha; - resultsJson['BuilderName'] = builderName; - resultsJson['NewStatus'] = testStatus; - } - resultsJson['TestFlaky'] = isTestFlaky ?? false; - if (_shouldUpdateCocoon(resultsJson, builderBucket ?? 'prod')) { - await retry( - () async => - _sendUpdateTaskRequest(resultsJson).timeout(Duration(seconds: requestTimeoutLimit)), - retryIf: - (Exception e) => e is SocketException || e is TimeoutException || e is ClientException, - maxAttempts: requestRetryLimit, - ); - } - } - - /// Only post-submit tests on `master` are allowed to update in cocoon. - bool _shouldUpdateCocoon(Map resultJson, String builderBucket) { - const List supportedBranches = ['master']; - return supportedBranches.contains(resultJson['CommitBranch']) && builderBucket == 'prod'; - } - - /// Write the given parameters into an update task request and store the JSON in [resultsPath]. - Future writeTaskResultToFile({ - String? builderName, - String? gitBranch, - required TaskResult result, - required String resultsPath, - }) async { - final Map updateRequest = _constructUpdateRequest( - gitBranch: gitBranch, - builderName: builderName, - result: result, - ); - final File resultFile = fs.file(resultsPath); - if (resultFile.existsSync()) { - resultFile.deleteSync(); - } - logger.fine('Writing results: ${json.encode(updateRequest)}'); - resultFile.createSync(); - resultFile.writeAsStringSync(json.encode(updateRequest)); - } - - Map _constructUpdateRequest({ - String? builderName, - required TaskResult result, - String? gitBranch, - }) { - final Map updateRequest = { - 'CommitBranch': gitBranch, - 'CommitSha': commitSha, - 'BuilderName': builderName, - 'NewStatus': result.succeeded ? 'Succeeded' : 'Failed', - }; - logger.fine('Update request: $updateRequest'); - - // Make a copy of result data because we may alter it for validation below. - updateRequest['ResultData'] = result.data; - - final List validScoreKeys = []; - if (result.benchmarkScoreKeys != null) { - for (final String scoreKey in result.benchmarkScoreKeys!) { - final Object score = result.data![scoreKey] as Object; - if (score is num) { - // Convert all metrics to double, which provide plenty of precision - // without having to add support for multiple numeric types in Cocoon. - result.data![scoreKey] = score.toDouble(); - validScoreKeys.add(scoreKey); - } - } - } - updateRequest['BenchmarkScoreKeys'] = validScoreKeys; - - return updateRequest; - } - - Future _sendUpdateTaskRequest(Map postBody) async { - logger.info('Attempting to send update task request to Cocoon.'); - final Map response = await _sendCocoonRequest('update-task-status', postBody); - if (response['Name'] != null) { - logger.info('Updated Cocoon with results from this task'); - } else { - logger.info(response); - logger.severe('Failed to updated Cocoon with results from this task'); - } - } - - /// Make an API request to Cocoon. - Future> _sendCocoonRequest(String apiPath, [dynamic jsonData]) async { - final Uri url = Uri.parse('$baseCocoonApiUrl/$apiPath'); - - /// Retry requests to Cocoon as sometimes there are issues with the servers, such - /// as version changes to the backend, datastore issues, or latency issues. - final Response response = await retry( - () => _httpClient.post(url, body: json.encode(jsonData)), - retryIf: - (Exception e) => e is SocketException || e is TimeoutException || e is ClientException, - maxAttempts: requestRetryLimit, - ); - return json.decode(response.body) as Map; - } -} - -/// [HttpClient] for sending authenticated requests to Cocoon. -class AuthenticatedCocoonClient extends BaseClient { - AuthenticatedCocoonClient( - this._serviceAccountTokenPath, { - @visibleForTesting Client? httpClient, - @visibleForTesting FileSystem? filesystem, - }) : _delegate = httpClient ?? Client(), - _fs = filesystem ?? const LocalFileSystem(); - - /// Authentication token to have the ability to upload and record test results. - /// - /// This is intended to only be passed on automated runs on LUCI post-submit. - final String? _serviceAccountTokenPath; - - /// Underlying [HttpClient] to send requests to. - final Client _delegate; - - /// Underlying [FileSystem] to use. - final FileSystem _fs; - - /// Value contained in the service account token file that can be used in http requests. - String get serviceAccountToken => _serviceAccountToken ?? _readServiceAccountTokenFile(); - String? _serviceAccountToken; - - /// Get [serviceAccountToken] from the given service account file. - String _readServiceAccountTokenFile() { - return _serviceAccountToken = _fs.file(_serviceAccountTokenPath).readAsStringSync().trim(); - } - - @override - Future send(BaseRequest request) async { - request.headers['Service-Account-Token'] = serviceAccountToken; - final StreamedResponse response = await _delegate.send(request); - - if (response.statusCode != 200) { - throw ClientException( - 'AuthenticatedClientError:\n' - ' URI: ${request.url}\n' - ' HTTP Status: ${response.statusCode}\n' - ' Response body:\n' - '${(await Response.fromStream(response)).body}', - request.url, - ); - } - return response; - } -} - -class CocoonException implements Exception { - CocoonException(this.message); - - /// The message to show to the issuer to explain the error. - final String message; - - @override - String toString() => 'CocoonException: $message'; -} diff --git a/dev/devicelab/lib/framework/runner.dart b/dev/devicelab/lib/framework/runner.dart index 13dd86825b..7f151c6901 100644 --- a/dev/devicelab/lib/framework/runner.dart +++ b/dev/devicelab/lib/framework/runner.dart @@ -10,11 +10,12 @@ import 'package:meta/meta.dart'; import 'package:vm_service/vm_service.dart'; import 'package:vm_service/vm_service_io.dart'; -import 'cocoon.dart'; import 'devices.dart'; import 'task_result.dart'; import 'utils.dart'; +const int retryNumber = 2; + /// Run a list of tasks. /// /// For each task, an auto rerun will be triggered when task fails. @@ -49,7 +50,7 @@ Future runTasks( for (final String taskName in taskNames) { TaskResult result = TaskResult.success(null); int failureCount = 0; - while (failureCount <= Cocoon.retryNumber) { + while (failureCount <= retryNumber) { result = await rerunTask( taskName, deviceId: deviceId, @@ -135,16 +136,6 @@ Future rerunTask( print('Task result:'); print(const JsonEncoder.withIndent(' ').convert(result)); section('Finished task "$taskName"'); - - if (resultsPath != null) { - final Cocoon cocoon = Cocoon(); - await cocoon.writeTaskResultToFile( - builderName: luciBuilder, - gitBranch: gitBranch, - result: result, - resultsPath: resultsPath, - ); - } return result; } diff --git a/dev/devicelab/test/cocoon_test.dart b/dev/devicelab/test/cocoon_test.dart deleted file mode 100644 index e0c5b1c9db..0000000000 --- a/dev/devicelab/test/cocoon_test.dart +++ /dev/null @@ -1,416 +0,0 @@ -// Copyright 2014 The Flutter Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -import 'dart:async'; -import 'dart:convert'; -import 'dart:io'; - -import 'package:file/file.dart'; -import 'package:file/memory.dart'; -import 'package:flutter_devicelab/framework/cocoon.dart'; -import 'package:flutter_devicelab/framework/task_result.dart'; -import 'package:http/http.dart'; -import 'package:http/testing.dart'; - -import 'common.dart'; - -void main() { - late ProcessResult processResult; - ProcessResult runSyncStub( - String executable, - List args, { - Map? environment, - bool includeParentEnvironment = true, - bool runInShell = false, - Encoding? stderrEncoding, - Encoding? stdoutEncoding, - String? workingDirectory, - }) => processResult; - - // Expected test values. - const String commitSha = 'a4952838bf288a81d8ea11edfd4b4cd649fa94cc'; - const String serviceAccountTokenPath = 'test_account_file'; - const String serviceAccountToken = 'test_token'; - - group('Cocoon', () { - late Client mockClient; - late Cocoon cocoon; - late FileSystem fs; - - setUp(() { - fs = MemoryFileSystem(); - mockClient = MockClient((Request request) async => Response('{}', 200)); - - final File serviceAccountFile = fs.file(serviceAccountTokenPath)..createSync(); - serviceAccountFile.writeAsStringSync(serviceAccountToken); - }); - - test('returns expected commit sha', () { - processResult = ProcessResult(1, 0, commitSha, ''); - cocoon = Cocoon( - serviceAccountTokenPath: serviceAccountTokenPath, - fs: fs, - httpClient: mockClient, - processRunSync: runSyncStub, - ); - - expect(cocoon.commitSha, commitSha); - }); - - test('throws exception on git cli errors', () { - processResult = ProcessResult(1, 1, '', ''); - cocoon = Cocoon( - serviceAccountTokenPath: serviceAccountTokenPath, - fs: fs, - httpClient: mockClient, - processRunSync: runSyncStub, - ); - - expect(() => cocoon.commitSha, throwsA(isA())); - }); - - test('writes expected update task json', () async { - processResult = ProcessResult(1, 0, commitSha, ''); - final TaskResult result = TaskResult.fromJson({ - 'success': true, - 'data': {'i': 0, 'j': 0, 'not_a_metric': 'something'}, - 'benchmarkScoreKeys': ['i', 'j'], - }); - - cocoon = Cocoon(fs: fs, processRunSync: runSyncStub); - - const String resultsPath = 'results.json'; - await cocoon.writeTaskResultToFile( - builderName: 'builderAbc', - gitBranch: 'master', - result: result, - resultsPath: resultsPath, - ); - - final String resultJson = fs.file(resultsPath).readAsStringSync(); - const String expectedJson = - '{' - '"CommitBranch":"master",' - '"CommitSha":"$commitSha",' - '"BuilderName":"builderAbc",' - '"NewStatus":"Succeeded",' - '"ResultData":{"i":0.0,"j":0.0,"not_a_metric":"something"},' - '"BenchmarkScoreKeys":["i","j"]}'; - expect(resultJson, expectedJson); - }); - - test('uploads metrics sends expected post body', () async { - processResult = ProcessResult(1, 0, commitSha, ''); - const String uploadMetricsRequestWithSpaces = - '{"CommitBranch":"master","CommitSha":"a4952838bf288a81d8ea11edfd4b4cd649fa94cc","BuilderName":"builder a b c","NewStatus":"Succeeded","ResultData":{},"BenchmarkScoreKeys":[],"TestFlaky":false}'; - final MockClient client = MockClient((Request request) async { - if (request.body == uploadMetricsRequestWithSpaces) { - return Response('{}', 200); - } - - return Response( - 'Expected: $uploadMetricsRequestWithSpaces\nReceived: ${request.body}', - 500, - ); - }); - cocoon = Cocoon( - fs: fs, - httpClient: client, - processRunSync: runSyncStub, - serviceAccountTokenPath: serviceAccountTokenPath, - requestRetryLimit: 0, - ); - - const String resultsPath = 'results.json'; - const String updateTaskJson = - '{' - '"CommitBranch":"master",' - '"CommitSha":"$commitSha",' - '"BuilderName":"builder a b c",' //ignore: missing_whitespace_between_adjacent_strings - '"NewStatus":"Succeeded",' - '"ResultData":{},' - '"BenchmarkScoreKeys":[]}'; - fs.file(resultsPath).writeAsStringSync(updateTaskJson); - await cocoon.sendTaskStatus(resultsPath: resultsPath); - }); - - test('uploads expected update task payload from results file', () async { - processResult = ProcessResult(1, 0, commitSha, ''); - cocoon = Cocoon( - fs: fs, - httpClient: mockClient, - processRunSync: runSyncStub, - serviceAccountTokenPath: serviceAccountTokenPath, - requestRetryLimit: 0, - ); - - const String resultsPath = 'results.json'; - const String updateTaskJson = - '{' - '"CommitBranch":"master",' - '"CommitSha":"$commitSha",' - '"BuilderName":"builderAbc",' - '"NewStatus":"Succeeded",' - '"ResultData":{"i":0.0,"j":0.0,"not_a_metric":"something"},' - '"BenchmarkScoreKeys":["i","j"]}'; - fs.file(resultsPath).writeAsStringSync(updateTaskJson); - await cocoon.sendTaskStatus(resultsPath: resultsPath); - }); - - test('Verify retries for task result upload', () async { - int requestCount = 0; - mockClient = MockClient((Request request) async { - requestCount++; - if (requestCount == 1) { - return Response('{}', 500); - } else { - return Response('{}', 200); - } - }); - - processResult = ProcessResult(1, 0, commitSha, ''); - cocoon = Cocoon( - fs: fs, - httpClient: mockClient, - processRunSync: runSyncStub, - serviceAccountTokenPath: serviceAccountTokenPath, - requestRetryLimit: 3, - ); - - const String resultsPath = 'results.json'; - const String updateTaskJson = - '{' - '"CommitBranch":"master",' - '"CommitSha":"$commitSha",' - '"BuilderName":"builderAbc",' - '"NewStatus":"Succeeded",' - '"ResultData":{"i":0.0,"j":0.0,"not_a_metric":"something"},' - '"BenchmarkScoreKeys":["i","j"]}'; - fs.file(resultsPath).writeAsStringSync(updateTaskJson); - await cocoon.sendTaskStatus(resultsPath: resultsPath); - }); - - test('Verify timeout and retry for task result upload', () async { - int requestCount = 0; - const int timeoutValue = 2; - mockClient = MockClient((Request request) async { - requestCount++; - if (requestCount == 1) { - await Future.delayed(const Duration(seconds: timeoutValue + 2)); - throw Exception('Should not reach this, because timeout should trigger'); - } else { - return Response('{}', 200); - } - }); - - processResult = ProcessResult(1, 0, commitSha, ''); - cocoon = Cocoon( - fs: fs, - httpClient: mockClient, - processRunSync: runSyncStub, - serviceAccountTokenPath: serviceAccountTokenPath, - requestRetryLimit: 2, - requestTimeoutLimit: timeoutValue, - ); - - const String resultsPath = 'results.json'; - const String updateTaskJson = - '{' - '"CommitBranch":"master",' - '"CommitSha":"$commitSha",' - '"BuilderName":"builderAbc",' - '"NewStatus":"Succeeded",' - '"ResultData":{"i":0.0,"j":0.0,"not_a_metric":"something"},' - '"BenchmarkScoreKeys":["i","j"]}'; - fs.file(resultsPath).writeAsStringSync(updateTaskJson); - await cocoon.sendTaskStatus(resultsPath: resultsPath); - }); - - test('Verify timeout does not trigger for result upload', () async { - int requestCount = 0; - const int timeoutValue = 2; - mockClient = MockClient((Request request) async { - requestCount++; - if (requestCount == 1) { - await Future.delayed(const Duration(seconds: timeoutValue - 1)); - return Response('{}', 200); - } else { - throw Exception('This iteration should not be reached, since timeout should not happen.'); - } - }); - - processResult = ProcessResult(1, 0, commitSha, ''); - cocoon = Cocoon( - fs: fs, - httpClient: mockClient, - processRunSync: runSyncStub, - serviceAccountTokenPath: serviceAccountTokenPath, - requestRetryLimit: 2, - requestTimeoutLimit: timeoutValue, - ); - - const String resultsPath = 'results.json'; - const String updateTaskJson = - '{' - '"CommitBranch":"master",' - '"CommitSha":"$commitSha",' - '"BuilderName":"builderAbc",' - '"NewStatus":"Succeeded",' - '"ResultData":{"i":0.0,"j":0.0,"not_a_metric":"something"},' - '"BenchmarkScoreKeys":["i","j"]}'; - fs.file(resultsPath).writeAsStringSync(updateTaskJson); - await cocoon.sendTaskStatus(resultsPath: resultsPath); - }); - - test('Verify failure without retries for task result upload', () async { - int requestCount = 0; - mockClient = MockClient((Request request) async { - requestCount++; - if (requestCount == 1) { - return Response('{}', 500); - } else { - return Response('{}', 200); - } - }); - - processResult = ProcessResult(1, 0, commitSha, ''); - cocoon = Cocoon( - fs: fs, - httpClient: mockClient, - processRunSync: runSyncStub, - serviceAccountTokenPath: serviceAccountTokenPath, - requestRetryLimit: 0, - ); - - const String resultsPath = 'results.json'; - const String updateTaskJson = - '{' - '"CommitBranch":"master",' - '"CommitSha":"$commitSha",' - '"BuilderName":"builderAbc",' - '"NewStatus":"Succeeded",' - '"ResultData":{"i":0.0,"j":0.0,"not_a_metric":"something"},' - '"BenchmarkScoreKeys":["i","j"]}'; - fs.file(resultsPath).writeAsStringSync(updateTaskJson); - expect( - () => cocoon.sendTaskStatus(resultsPath: resultsPath), - throwsA(isA()), - ); - }); - - test('throws client exception on non-200 responses', () async { - mockClient = MockClient((Request request) async => Response('', 500)); - - cocoon = Cocoon( - serviceAccountTokenPath: serviceAccountTokenPath, - fs: fs, - httpClient: mockClient, - requestRetryLimit: 0, - ); - - const String resultsPath = 'results.json'; - const String updateTaskJson = - '{' - '"CommitBranch":"master",' - '"CommitSha":"$commitSha",' - '"BuilderName":"builderAbc",' - '"NewStatus":"Succeeded",' - '"ResultData":{"i":0.0,"j":0.0,"not_a_metric":"something"},' - '"BenchmarkScoreKeys":["i","j"]}'; - fs.file(resultsPath).writeAsStringSync(updateTaskJson); - expect( - () => cocoon.sendTaskStatus(resultsPath: resultsPath), - throwsA(isA()), - ); - }); - - test('does not upload results on non-supported branches', () async { - // Any network failure would cause the upload to fail - mockClient = MockClient((Request request) async => Response('', 500)); - - cocoon = Cocoon( - serviceAccountTokenPath: serviceAccountTokenPath, - fs: fs, - httpClient: mockClient, - requestRetryLimit: 0, - ); - - const String resultsPath = 'results.json'; - const String updateTaskJson = - '{' - '"CommitBranch":"stable",' - '"CommitSha":"$commitSha",' - '"BuilderName":"builderAbc",' - '"NewStatus":"Succeeded",' - '"ResultData":{"i":0.0,"j":0.0,"not_a_metric":"something"},' - '"BenchmarkScoreKeys":["i","j"]}'; - fs.file(resultsPath).writeAsStringSync(updateTaskJson); - - // This will fail if it decided to upload results - await cocoon.sendTaskStatus(resultsPath: resultsPath); - }); - - test('does not update for staging test', () async { - // Any network failure would cause the upload to fail - mockClient = MockClient((Request request) async => Response('', 500)); - - cocoon = Cocoon( - serviceAccountTokenPath: serviceAccountTokenPath, - fs: fs, - httpClient: mockClient, - requestRetryLimit: 0, - ); - - const String resultsPath = 'results.json'; - const String updateTaskJson = - '{' - '"CommitBranch":"master",' - '"CommitSha":"$commitSha",' - '"BuilderName":"builderAbc",' - '"NewStatus":"Succeeded",' - '"ResultData":{"i":0.0,"j":0.0,"not_a_metric":"something"},' - '"BenchmarkScoreKeys":["i","j"]}'; - fs.file(resultsPath).writeAsStringSync(updateTaskJson); - - // This will fail if it decided to upload results - await cocoon.sendTaskStatus(resultsPath: resultsPath, builderBucket: 'staging'); - }); - }); - - group('AuthenticatedCocoonClient', () { - late FileSystem fs; - - setUp(() { - fs = MemoryFileSystem(); - final File serviceAccountFile = fs.file(serviceAccountTokenPath)..createSync(); - serviceAccountFile.writeAsStringSync(serviceAccountToken); - }); - - test('reads token from service account file', () { - final AuthenticatedCocoonClient client = AuthenticatedCocoonClient( - serviceAccountTokenPath, - filesystem: fs, - ); - expect(client.serviceAccountToken, serviceAccountToken); - }); - - test('reads token from service account file with whitespace', () { - final File serviceAccountFile = fs.file(serviceAccountTokenPath)..createSync(); - serviceAccountFile.writeAsStringSync('$serviceAccountToken \n'); - final AuthenticatedCocoonClient client = AuthenticatedCocoonClient( - serviceAccountTokenPath, - filesystem: fs, - ); - expect(client.serviceAccountToken, serviceAccountToken); - }); - - test('throws error when service account file not found', () { - final AuthenticatedCocoonClient client = AuthenticatedCocoonClient( - 'idontexist', - filesystem: fs, - ); - expect(() => client.serviceAccountToken, throwsA(isA())); - }); - }); -}