diff --git a/dev/devicelab/lib/command/upload_results.dart b/dev/devicelab/lib/command/upload_results.dart index 7bc2d95af3..df7f06fd7d 100644 --- a/dev/devicelab/lib/command/upload_results.dart +++ b/dev/devicelab/lib/command/upload_results.dart @@ -5,6 +5,7 @@ import 'package:args/command_runner.dart'; import '../framework/cocoon.dart'; +import '../framework/metrics_center.dart'; class UploadResultsCommand extends Command { UploadResultsCommand() { @@ -21,6 +22,7 @@ class UploadResultsCommand extends Command { ); argParser.addOption('luci-builder', help: '[Flutter infrastructure] Name of the LUCI builder being run on.'); argParser.addOption('test-status', help: 'Test status: Succeeded|Failed'); + argParser.addOption('commit-time', help: 'Commit time in UNIX timestamp'); } @override @@ -37,6 +39,20 @@ class UploadResultsCommand extends Command { final String? gitBranch = argResults!['git-branch'] as String?; final String? builderName = argResults!['luci-builder'] as String?; final String? testStatus = argResults!['test-status'] as String?; + final String? commitTime = argResults!['commit-time'] as String?; + + // Upload metrics to metrics_center from test runner when `commitTime` is specified. This + // is mainly for testing purpose. + // The upload step will be skipped from cocoon once this is validated. + // TODO(keyong): remove try block to block test when this is validated to work https://github.com/flutter/flutter/issues/88484 + try { + if (commitTime != null) { + await uploadToMetricsCenter(resultsPath, commitTime); + print('Successfully uploaded metrics to metrics center'); + } + } on Exception catch (e, stacktrace) { + print('Uploading metrics failure: $e\n\n$stacktrace'); + } final Cocoon cocoon = Cocoon(serviceAccountTokenPath: serviceAccountTokenFile); return cocoon.sendResultsPath( diff --git a/dev/devicelab/lib/framework/metrics_center.dart b/dev/devicelab/lib/framework/metrics_center.dart new file mode 100644 index 0000000000..b96cbe2db3 --- /dev/null +++ b/dev/devicelab/lib/framework/metrics_center.dart @@ -0,0 +1,94 @@ +// Copyright 2014 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'dart:convert'; +import 'dart:io'; + +import 'package:metrics_center/metrics_center.dart'; + +/// Authenticate and connect to gcloud storage. +/// +/// It supports both token and credential authentications. +Future connectFlutterDestination() async { + const String kTokenPath = 'TOKEN_PATH'; + const String kGcpProject = 'GCP_PROJECT'; + final Map env = Platform.environment; + final bool isTesting = env['IS_TESTING'] == 'true'; + if (env.containsKey(kTokenPath) && env.containsKey(kGcpProject)) { + return FlutterDestination.makeFromAccessToken( + File(env[kTokenPath]!).readAsStringSync(), + env[kGcpProject]!, + isTesting: isTesting, + ); + } + return FlutterDestination.makeFromCredentialsJson( + jsonDecode(env['BENCHMARK_GCP_CREDENTIALS']!) as Map, + isTesting: isTesting, + ); +} + +/// Parse results into Metric Points. +/// +/// An example of `resultsJson`: +/// { +/// "CommitBranch": "master", +/// "CommitSha": "abc", +/// "BuilderName": "test", +/// "ResultData": { +/// "average_frame_build_time_millis": 0.4550425531914895, +/// "90th_percentile_frame_build_time_millis": 0.473 +/// }, +/// "BenchmarkScoreKeys": [ +/// "average_frame_build_time_millis", +/// "90th_percentile_frame_build_time_millis" +/// ] +/// } +List parse(Map resultsJson) { + final List scoreKeys = (resultsJson['BenchmarkScoreKeys'] as List).cast(); + final Map resultData = resultsJson['ResultData'] as Map; + final String gitBranch = (resultsJson['CommitBranch'] as String).trim(); + final String gitSha = (resultsJson['CommitSha'] as String).trim(); + final String builderName = resultsJson['BuilderName'] as String; + final List metricPoints = []; + for (final String scoreKey in scoreKeys) { + metricPoints.add( + MetricPoint( + (resultData[scoreKey] as num).toDouble(), + { + kGithubRepoKey: kFlutterFrameworkRepo, + kGitRevisionKey: gitSha, + 'branch': gitBranch, + kNameKey: builderName, + kSubResultKey: scoreKey, + }, + ), + ); + } + return metricPoints; +} + +/// Upload test metrics to metrics center. +Future uploadToMetricsCenter(String? resultsPath, String? commitTime) async { + int commitTimeSinceEpoch; + if (resultsPath == null) { + return; + } + if (commitTime != null) { + commitTimeSinceEpoch = 1000 * int.parse(commitTime); + } else { + commitTimeSinceEpoch = DateTime.now().millisecondsSinceEpoch; + } + final File resultFile = File(resultsPath); + Map resultsJson = {}; + resultsJson = json.decode(await resultFile.readAsString()) as Map; + final List metricPoints = parse(resultsJson); + final FlutterDestination metricsDestination = await connectFlutterDestination(); + await metricsDestination.update( + metricPoints, + DateTime.fromMillisecondsSinceEpoch( + commitTimeSinceEpoch, + isUtc: true, + ), + ); +} diff --git a/dev/devicelab/pubspec.yaml b/dev/devicelab/pubspec.yaml index 9ff00f2113..5194c32b08 100644 --- a/dev/devicelab/pubspec.yaml +++ b/dev/devicelab/pubspec.yaml @@ -11,6 +11,7 @@ dependencies: file: 6.1.2 http: 0.13.3 meta: 1.7.0 + metrics_center: 1.0.0 path: 1.8.0 platform: 3.0.2 process: 4.2.3 @@ -19,11 +20,16 @@ dependencies: logging: 1.0.1 + _discoveryapis_commons: 1.0.1 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade" async: 2.8.2 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade" charcode: 1.3.1 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade" checked_yaml: 2.0.1 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade" collection: 1.15.0 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade" crypto: 3.0.1 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade" + equatable: 2.0.3 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade" + gcloud: 0.8.2 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade" + googleapis: 3.0.0 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade" + googleapis_auth: 1.1.0 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade" http_parser: 4.0.0 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade" json_annotation: 4.1.0 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade" pedantic: 1.11.1 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade" @@ -67,4 +73,4 @@ dev_dependencies: web_socket_channel: 2.1.0 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade" webkit_inspection_protocol: 1.0.0 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade" -# PUBSPEC CHECKSUM: 1094 +# PUBSPEC CHECKSUM: 3624 diff --git a/dev/devicelab/test/metrics_center_test.dart b/dev/devicelab/test/metrics_center_test.dart new file mode 100644 index 0000000000..470a350b54 --- /dev/null +++ b/dev/devicelab/test/metrics_center_test.dart @@ -0,0 +1,32 @@ +// Copyright 2014 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'package:flutter_devicelab/framework/metrics_center.dart'; +import 'package:metrics_center/metrics_center.dart'; + +import 'common.dart'; + +void main() { + group('Parse', () { + test('succeeds', () { + final Map results = { + 'CommitBranch': 'master', + 'CommitSha': 'abc', + 'BuilderName': 'test', + 'ResultData': { + 'average_frame_build_time_millis': 0.4550425531914895, + '90th_percentile_frame_build_time_millis': 0.473 + }, + 'BenchmarkScoreKeys': [ + 'average_frame_build_time_millis', + '90th_percentile_frame_build_time_millis' + ] + }; + final List metricPoints = parse(results); + + expect(metricPoints[0].value, equals(0.4550425531914895)); + expect(metricPoints[1].value, equals(0.473)); + }); + }); +}