diff --git a/dev/devicelab/lib/command/upload_results.dart b/dev/devicelab/lib/command/upload_results.dart index 57593b33a8..de60882c4b 100644 --- a/dev/devicelab/lib/command/upload_results.dart +++ b/dev/devicelab/lib/command/upload_results.dart @@ -22,6 +22,7 @@ class UploadResultsCommand extends Command { ); argParser.addOption('luci-builder', help: '[Flutter infrastructure] Name of the LUCI builder being run on.'); argParser.addOption('task-name', help: '[Flutter infrastructure] Name of the task being run on.'); + argParser.addOption('benchmark-tags', help: '[Flutter infrastructure] Benchmark tags to surface on Skia Perf'); argParser.addOption('test-status', help: 'Test status: Succeeded|Failed'); argParser.addOption('commit-time', help: 'Commit time in UNIX timestamp'); argParser.addOption('builder-bucket', help: '[Flutter infrastructure] Luci builder bucket the test is running in.'); @@ -31,7 +32,7 @@ class UploadResultsCommand extends Command { String get name => 'upload-metrics'; @override - String get description => '[Flutter infrastructure] Upload results data to Cocoon'; + String get description => '[Flutter infrastructure] Upload results data to Cocoon/Skia Perf'; @override Future run() async { @@ -43,11 +44,12 @@ class UploadResultsCommand extends Command { final String? testStatus = argResults!['test-status'] as String?; final String? commitTime = argResults!['commit-time'] as String?; final String? taskName = argResults!['task-name'] as String?; + final String? benchmarkTags = argResults!['benchmark-tags'] as String?; final String? builderBucket = argResults!['builder-bucket'] as String?; // Upload metrics to skia perf from test runner when `resultsPath` is specified. if (resultsPath != null) { - await uploadToSkiaPerf(resultsPath, commitTime, taskName); + await uploadToSkiaPerf(resultsPath, commitTime, taskName, benchmarkTags); print('Successfully uploaded metrics to skia perf'); } diff --git a/dev/devicelab/lib/framework/metrics_center.dart b/dev/devicelab/lib/framework/metrics_center.dart index 7627fd73dc..03a9965780 100644 --- a/dev/devicelab/lib/framework/metrics_center.dart +++ b/dev/devicelab/lib/framework/metrics_center.dart @@ -5,6 +5,7 @@ import 'dart:convert'; import 'dart:io'; +import 'package:collection/collection.dart'; import 'package:metrics_center/metrics_center.dart'; /// Authenticate and connect to gcloud storage. @@ -28,7 +29,7 @@ Future connectFlutterDestination() async { ); } -/// Parse results into Metric Points. +/// Parse results and append additional benchmark tags into Metric Points. /// /// An example of `resultsJson`: /// { @@ -44,8 +45,18 @@ Future connectFlutterDestination() async { /// "90th_percentile_frame_build_time_millis" /// ] /// } -List parse(Map resultsJson) { +/// +/// An example of `benchmarkTags`: +/// { +/// "arch": "intel", +/// "device_type": "Moto G Play", +/// "device_version": "android-25", +/// "host_type": "linux", +/// "host_version": "debian-10.11" +/// } +List parse(Map resultsJson, Map benchmarkTags) { print('Results to upload to skia perf: $resultsJson'); + print('Benchmark tags to upload to skia perf: $benchmarkTags'); final List scoreKeys = (resultsJson['BenchmarkScoreKeys'] as List?)?.cast() ?? const []; final Map resultData = @@ -55,16 +66,20 @@ List parse(Map resultsJson) { final String builderName = (resultsJson['BuilderName'] as String).trim(); final List metricPoints = []; for (final String scoreKey in scoreKeys) { + Map tags = { + kGithubRepoKey: kFlutterFrameworkRepo, + kGitRevisionKey: gitSha, + 'branch': gitBranch, + kNameKey: builderName, + kSubResultKey: scoreKey, + }; + // Append additional benchmark tags, which will surface in Skia Perf dashboards. + tags = mergeMaps( + tags, benchmarkTags.map((String key, dynamic value) => MapEntry(key, value.toString()))); metricPoints.add( MetricPoint( (resultData[scoreKey] as num).toDouble(), - { - kGithubRepoKey: kFlutterFrameworkRepo, - kGitRevisionKey: gitSha, - 'branch': gitBranch, - kNameKey: builderName, - kSubResultKey: scoreKey, - }, + tags, ), ); } @@ -102,7 +117,7 @@ Future upload( /// 1. Run DeviceLab test, writing results to a known path /// 2. Request service account token from luci auth (valid for at least 3 minutes) /// 3. Upload results from (1) to skia perf. -Future uploadToSkiaPerf(String? resultsPath, String? commitTime, String? taskName) async { +Future uploadToSkiaPerf(String? resultsPath, String? commitTime, String? taskName, String? benchmarkTags) async { int commitTimeSinceEpoch; if (resultsPath == null) { return; @@ -112,10 +127,11 @@ Future uploadToSkiaPerf(String? resultsPath, String? commitTime, String? t } else { commitTimeSinceEpoch = DateTime.now().millisecondsSinceEpoch; } + final Map benchmarkTagsMap = jsonDecode(benchmarkTags ?? '{}') as Map; final File resultFile = File(resultsPath); Map resultsJson = {}; resultsJson = json.decode(await resultFile.readAsString()) as Map; - final List metricPoints = parse(resultsJson); + final List metricPoints = parse(resultsJson, benchmarkTagsMap); final FlutterDestination metricsDestination = await connectFlutterDestination(); await upload(metricsDestination, metricPoints, commitTimeSinceEpoch, taskName); } diff --git a/dev/devicelab/test/metrics_center_test.dart b/dev/devicelab/test/metrics_center_test.dart index b256d941cc..f9bebee5fc 100644 --- a/dev/devicelab/test/metrics_center_test.dart +++ b/dev/devicelab/test/metrics_center_test.dart @@ -24,7 +24,7 @@ class FakeFlutterDestination implements FlutterDestination { void main() { group('Parse', () { - test('succeeds', () { + test('without additional benchmark tags', () { final Map results = { 'CommitBranch': 'master', 'CommitSha': 'abc', @@ -38,12 +38,41 @@ void main() { '90th_percentile_frame_build_time_millis', ], }; - final List metricPoints = parse(results); + final List metricPoints = parse(results, {}); expect(metricPoints[0].value, equals(0.4550425531914895)); expect(metricPoints[1].value, equals(0.473)); }); + test('with additional benchmark tags', () { + final Map results = { + 'CommitBranch': 'master', + 'CommitSha': 'abc', + 'BuilderName': 'test', + 'ResultData': { + 'average_frame_build_time_millis': 0.4550425531914895, + '90th_percentile_frame_build_time_millis': 0.473, + }, + 'BenchmarkScoreKeys': [ + 'average_frame_build_time_millis', + '90th_percentile_frame_build_time_millis', + ], + }; + final Map tags = { + 'arch': 'intel', + 'device_type': 'Moto G Play', + 'device_version': 'android-25', + 'host_type': 'linux', + 'host_version': 'debian-10.11' + }; + final List metricPoints = parse(results, tags); + + expect(metricPoints[0].value, equals(0.4550425531914895)); + expect(metricPoints[0].tags.keys.contains('arch'), isTrue); + expect(metricPoints[1].value, equals(0.473)); + expect(metricPoints[1].tags.keys.contains('device_type'), isTrue); + }); + test('succeeds - null ResultData', () { final Map results = { 'CommitBranch': 'master', @@ -52,7 +81,7 @@ void main() { 'ResultData': null, 'BenchmarkScoreKeys': null, }; - final List metricPoints = parse(results); + final List metricPoints = parse(results, {}); expect(metricPoints.length, 0); }); @@ -73,7 +102,7 @@ void main() { '90th_percentile_frame_build_time_millis', ], }; - final List metricPoints = parse(results); + final List metricPoints = parse(results, {}); final FakeFlutterDestination flutterDestination = FakeFlutterDestination(); String? taskName; const int commitTimeSinceEpoch = 1629220312; @@ -97,7 +126,7 @@ void main() { '90th_percentile_frame_build_time_millis', ], }; - final List metricPoints = parse(results); + final List metricPoints = parse(results, {}); final FakeFlutterDestination flutterDestination = FakeFlutterDestination(); const String taskName = 'test'; const int commitTimeSinceEpoch = 1629220312;