firka/flutter
firka/flutter
5
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Add devicelab benchmark tags support (#92141)

Browse Source
This commit is contained in:
keyonghan 2021-10-20 15:03:06 -07:00 committed by GitHub
parent 305a855f6d
commit 7de92e2653
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 65 additions and 18 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
dev/devicelab/lib/command/upload_results.dart
View File

@ -22,6 +22,7 @@ class UploadResultsCommand extends Command<void> {
);
argParser.addOption('luci-builder', help: '[Flutter infrastructure] Name of the LUCI builder being run on.');
argParser.addOption('task-name', help: '[Flutter infrastructure] Name of the task being run on.');
argParser.addOption('benchmark-tags', help: '[Flutter infrastructure] Benchmark tags to surface on Skia Perf');
argParser.addOption('test-status', help: 'Test status: Succeeded|Failed');
argParser.addOption('commit-time', help: 'Commit time in UNIX timestamp');
argParser.addOption('builder-bucket', help: '[Flutter infrastructure] Luci builder bucket the test is running in.');
@ -31,7 +32,7 @@ class UploadResultsCommand extends Command<void> {
String get name => 'upload-metrics';
@override
String get description => '[Flutter infrastructure] Upload results data to Cocoon';
String get description => '[Flutter infrastructure] Upload results data to Cocoon/Skia Perf';
@override
Future<void> run() async {
@ -43,11 +44,12 @@ class UploadResultsCommand extends Command<void> {
final String? testStatus = argResults!['test-status'] as String?;
final String? commitTime = argResults!['commit-time'] as String?;
final String? taskName = argResults!['task-name'] as String?;
final String? benchmarkTags = argResults!['benchmark-tags'] as String?;
final String? builderBucket = argResults!['builder-bucket'] as String?;
// Upload metrics to skia perf from test runner when `resultsPath` is specified.
if (resultsPath != null) {
await uploadToSkiaPerf(resultsPath, commitTime, taskName);
await uploadToSkiaPerf(resultsPath, commitTime, taskName, benchmarkTags);
print('Successfully uploaded metrics to skia perf');
}

38
dev/devicelab/lib/framework/metrics_center.dart
View File

@ -5,6 +5,7 @@
import 'dart:convert';
import 'dart:io';
import 'package:collection/collection.dart';
import 'package:metrics_center/metrics_center.dart';
/// Authenticate and connect to gcloud storage.
@ -28,7 +29,7 @@ Future<FlutterDestination> connectFlutterDestination() async {
);
}
/// Parse results into Metric Points.
/// Parse results and append additional benchmark tags into Metric Points.
///
/// An example of `resultsJson`:
/// {
@ -44,8 +45,18 @@ Future<FlutterDestination> connectFlutterDestination() async {
/// "90th_percentile_frame_build_time_millis"
/// ]
/// }
List<MetricPoint> parse(Map<String, dynamic> resultsJson) {
///
/// An example of `benchmarkTags`:
/// {
/// "arch": "intel",
/// "device_type": "Moto G Play",
/// "device_version": "android-25",
/// "host_type": "linux",
/// "host_version": "debian-10.11"
/// }
List<MetricPoint> parse(Map<String, dynamic> resultsJson, Map<String, dynamic> benchmarkTags) {
print('Results to upload to skia perf: $resultsJson');
print('Benchmark tags to upload to skia perf: $benchmarkTags');
final List<String> scoreKeys =
(resultsJson['BenchmarkScoreKeys'] as List<dynamic>?)?.cast<String>() ?? const <String>[];
final Map<String, dynamic> resultData =
@ -55,16 +66,20 @@ List<MetricPoint> parse(Map<String, dynamic> resultsJson) {
final String builderName = (resultsJson['BuilderName'] as String).trim();
final List<MetricPoint> metricPoints = <MetricPoint>[];
for (final String scoreKey in scoreKeys) {
Map<String, String> tags = <String, String>{
kGithubRepoKey: kFlutterFrameworkRepo,
kGitRevisionKey: gitSha,
'branch': gitBranch,
kNameKey: builderName,
kSubResultKey: scoreKey,
};
// Append additional benchmark tags, which will surface in Skia Perf dashboards.
tags = mergeMaps<String, String>(
tags, benchmarkTags.map((String key, dynamic value) => MapEntry<String, String>(key, value.toString())));
metricPoints.add(
MetricPoint(
(resultData[scoreKey] as num).toDouble(),
<String, String>{
kGithubRepoKey: kFlutterFrameworkRepo,
kGitRevisionKey: gitSha,
'branch': gitBranch,
kNameKey: builderName,
kSubResultKey: scoreKey,
},
tags,
),
);
}
@ -102,7 +117,7 @@ Future<void> upload(
/// 1. Run DeviceLab test, writing results to a known path
/// 2. Request service account token from luci auth (valid for at least 3 minutes)
/// 3. Upload results from (1) to skia perf.
Future<void> uploadToSkiaPerf(String? resultsPath, String? commitTime, String? taskName) async {
Future<void> uploadToSkiaPerf(String? resultsPath, String? commitTime, String? taskName, String? benchmarkTags) async {
int commitTimeSinceEpoch;
if (resultsPath == null) {
return;
@ -112,10 +127,11 @@ Future<void> uploadToSkiaPerf(String? resultsPath, String? commitTime, String? t
} else {
commitTimeSinceEpoch = DateTime.now().millisecondsSinceEpoch;
}
final Map<String, dynamic> benchmarkTagsMap = jsonDecode(benchmarkTags ?? '{}') as Map<String, dynamic>;
final File resultFile = File(resultsPath);
Map<String, dynamic> resultsJson = <String, dynamic>{};
resultsJson = json.decode(await resultFile.readAsString()) as Map<String, dynamic>;
final List<MetricPoint> metricPoints = parse(resultsJson);
final List<MetricPoint> metricPoints = parse(resultsJson, benchmarkTagsMap);
final FlutterDestination metricsDestination = await connectFlutterDestination();
await upload(metricsDestination, metricPoints, commitTimeSinceEpoch, taskName);
}

39
dev/devicelab/test/metrics_center_test.dart
View File

@ -24,7 +24,7 @@ class FakeFlutterDestination implements FlutterDestination {
void main() {
group('Parse', () {
test('succeeds', () {
test('without additional benchmark tags', () {
final Map<String, dynamic> results = <String, dynamic>{
'CommitBranch': 'master',
'CommitSha': 'abc',
@ -38,12 +38,41 @@ void main() {
'90th_percentile_frame_build_time_millis',
],
};
final List<MetricPoint> metricPoints = parse(results);
final List<MetricPoint> metricPoints = parse(results, <String, String>{});
expect(metricPoints[0].value, equals(0.4550425531914895));
expect(metricPoints[1].value, equals(0.473));
});
test('with additional benchmark tags', () {
final Map<String, dynamic> results = <String, dynamic>{
'CommitBranch': 'master',
'CommitSha': 'abc',
'BuilderName': 'test',
'ResultData': <String, dynamic>{
'average_frame_build_time_millis': 0.4550425531914895,
'90th_percentile_frame_build_time_millis': 0.473,
},
'BenchmarkScoreKeys': <String>[
'average_frame_build_time_millis',
'90th_percentile_frame_build_time_millis',
],
};
final Map<String, dynamic> tags = <String, dynamic>{
'arch': 'intel',
'device_type': 'Moto G Play',
'device_version': 'android-25',
'host_type': 'linux',
'host_version': 'debian-10.11'
};
final List<MetricPoint> metricPoints = parse(results, tags);
expect(metricPoints[0].value, equals(0.4550425531914895));
expect(metricPoints[0].tags.keys.contains('arch'), isTrue);
expect(metricPoints[1].value, equals(0.473));
expect(metricPoints[1].tags.keys.contains('device_type'), isTrue);
});
test('succeeds - null ResultData', () {
final Map<String, dynamic> results = <String, dynamic>{
'CommitBranch': 'master',
@ -52,7 +81,7 @@ void main() {
'ResultData': null,
'BenchmarkScoreKeys': null,
};
final List<MetricPoint> metricPoints = parse(results);
final List<MetricPoint> metricPoints = parse(results, <String, String>{});
expect(metricPoints.length, 0);
});
@ -73,7 +102,7 @@ void main() {
'90th_percentile_frame_build_time_millis',
],
};
final List<MetricPoint> metricPoints = parse(results);
final List<MetricPoint> metricPoints = parse(results, <String, String>{});
final FakeFlutterDestination flutterDestination = FakeFlutterDestination();
String? taskName;
const int commitTimeSinceEpoch = 1629220312;
@ -97,7 +126,7 @@ void main() {
'90th_percentile_frame_build_time_millis',
],
};
final List<MetricPoint> metricPoints = parse(results);
final List<MetricPoint> metricPoints = parse(results, <String, String>{});
final FakeFlutterDestination flutterDestination = FakeFlutterDestination();
const String taskName = 'test';
const int commitTimeSinceEpoch = 1629220312;