firka/flutter
firka/flutter
5
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Add extra benchmark metrics to test name in addition to builder name (#92530)

Browse Source
This commit is contained in:
keyonghan 2021-10-27 15:33:06 -07:00 committed by GitHub
parent 53de889996
commit 30d3866ab9
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 47 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

21
dev/devicelab/lib/framework/metrics_center.dart
View File

@ -54,7 +54,7 @@ Future<FlutterDestination> connectFlutterDestination() async {
/// "host_type": "linux",
/// "host_version": "debian-10.11"
/// }
List<MetricPoint> parse(Map<String, dynamic> resultsJson, Map<String, dynamic> benchmarkTags) {
List<MetricPoint> parse(Map<String, dynamic> resultsJson, Map<String, dynamic> benchmarkTags, String taskName) {
print('Results to upload to skia perf: $resultsJson');
print('Benchmark tags to upload to skia perf: $benchmarkTags');
final List<String> scoreKeys =
@ -82,6 +82,18 @@ List<MetricPoint> parse(Map<String, dynamic> resultsJson, Map<String, dynamic> b
tags,
),
);
// Add an extra entry under test name. This way we have duplicate metrics
// under both builder name and test name. Once we have enough data and update
// existing alerts to point to test name, we can deprecate builder name ones.
// https://github.com/flutter/flutter/issues/74522#issuecomment-942575581
tags[kNameKey] = taskName;
metricPoints.add(
MetricPoint(
(resultData[scoreKey] as num).toDouble(),
tags,
),
);
}
return metricPoints;
}
@ -99,7 +111,7 @@ Future<void> upload(
FlutterDestination metricsDestination,
List<MetricPoint> metricPoints,
int commitTimeSinceEpoch,
String? taskName,
String taskName,
) async {
await metricsDestination.update(
metricPoints,
@ -107,7 +119,7 @@ Future<void> upload(
commitTimeSinceEpoch,
isUtc: true,
),
taskName ?? 'default',
taskName,
);
}
@ -127,11 +139,12 @@ Future<void> uploadToSkiaPerf(String? resultsPath, String? commitTime, String? t
} else {
commitTimeSinceEpoch = DateTime.now().millisecondsSinceEpoch;
}
taskName = taskName ?? 'default';
final Map<String, dynamic> benchmarkTagsMap = jsonDecode(benchmarkTags ?? '{}') as Map<String, dynamic>;
final File resultFile = File(resultsPath);
Map<String, dynamic> resultsJson = <String, dynamic>{};
resultsJson = json.decode(await resultFile.readAsString()) as Map<String, dynamic>;
final List<MetricPoint> metricPoints = parse(resultsJson, benchmarkTagsMap);
final List<MetricPoint> metricPoints = parse(resultsJson, benchmarkTagsMap, taskName);
final FlutterDestination metricsDestination = await connectFlutterDestination();
await upload(metricsDestination, metricPoints, commitTimeSinceEpoch, taskName);
}

39
dev/devicelab/test/metrics_center_test.dart
View File

@ -24,6 +24,27 @@ class FakeFlutterDestination implements FlutterDestination {
void main() {
group('Parse', () {
test('duplicate entries for both builder name and test name', () {
final Map<String, dynamic> results = <String, dynamic>{
'CommitBranch': 'master',
'CommitSha': 'abc',
'BuilderName': 'Linux test',
'ResultData': <String, dynamic>{
'average_frame_build_time_millis': 0.4550425531914895,
},
'BenchmarkScoreKeys': <String>[
'average_frame_build_time_millis',
],
};
final List<MetricPoint> metricPoints = parse(results, <String, String>{}, 'test');
expect(metricPoints.length, 2);
expect(metricPoints[0].value, equals(0.4550425531914895));
expect(metricPoints[0].tags[kNameKey], 'Linux test');
expect(metricPoints[1].value, equals(0.4550425531914895));
expect(metricPoints[1].tags[kNameKey], 'test');
});
test('without additional benchmark tags', () {
final Map<String, dynamic> results = <String, dynamic>{
'CommitBranch': 'master',
@ -38,10 +59,10 @@ void main() {
'90th_percentile_frame_build_time_millis',
],
};
final List<MetricPoint> metricPoints = parse(results, <String, String>{});
final List<MetricPoint> metricPoints = parse(results, <String, String>{}, 'task abc');
expect(metricPoints[0].value, equals(0.4550425531914895));
expect(metricPoints[1].value, equals(0.473));
expect(metricPoints[2].value, equals(0.473));
});
test('with additional benchmark tags', () {
@ -65,12 +86,12 @@ void main() {
'host_type': 'linux',
'host_version': 'debian-10.11'
};
final List<MetricPoint> metricPoints = parse(results, tags);
final List<MetricPoint> metricPoints = parse(results, tags, 'task abc');
expect(metricPoints[0].value, equals(0.4550425531914895));
expect(metricPoints[0].tags.keys.contains('arch'), isTrue);
expect(metricPoints[1].value, equals(0.473));
expect(metricPoints[1].tags.keys.contains('device_type'), isTrue);
expect(metricPoints[2].value, equals(0.473));
expect(metricPoints[2].tags.keys.contains('device_type'), isTrue);
});
test('succeeds - null ResultData', () {
@ -81,7 +102,7 @@ void main() {
'ResultData': null,
'BenchmarkScoreKeys': null,
};
final List<MetricPoint> metricPoints = parse(results, <String, String>{});
final List<MetricPoint> metricPoints = parse(results, <String, String>{}, 'tetask abcst');
expect(metricPoints.length, 0);
});
@ -102,9 +123,9 @@ void main() {
'90th_percentile_frame_build_time_millis',
],
};
final List<MetricPoint> metricPoints = parse(results, <String, String>{});
final List<MetricPoint> metricPoints = parse(results, <String, String>{}, 'task abc');
final FakeFlutterDestination flutterDestination = FakeFlutterDestination();
String? taskName;
const String taskName = 'default';
const int commitTimeSinceEpoch = 1629220312;
await upload(flutterDestination, metricPoints, commitTimeSinceEpoch, taskName);
@ -126,7 +147,7 @@ void main() {
'90th_percentile_frame_build_time_millis',
],
};
final List<MetricPoint> metricPoints = parse(results, <String, String>{});
final List<MetricPoint> metricPoints = parse(results, <String, String>{}, 'task abc');
final FakeFlutterDestination flutterDestination = FakeFlutterDestination();
const String taskName = 'test';
const int commitTimeSinceEpoch = 1629220312;