firka/flutter
firka/flutter
5
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Upload devicelab test metrics from test runner (#88447)

Browse Source
This commit is contained in:
keyonghan 2021-08-20 12:57:03 -07:00 committed by GitHub
parent 9f9aa46b7d
commit 208d312ff0
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 149 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
dev/devicelab/lib/command/upload_results.dart
View File

@ -5,6 +5,7 @@
import 'package:args/command_runner.dart';
import '../framework/cocoon.dart';
import '../framework/metrics_center.dart';
class UploadResultsCommand extends Command<void> {
UploadResultsCommand() {
@ -21,6 +22,7 @@ class UploadResultsCommand extends Command<void> {
);
argParser.addOption('luci-builder', help: '[Flutter infrastructure] Name of the LUCI builder being run on.');
argParser.addOption('test-status', help: 'Test status: Succeeded|Failed');
argParser.addOption('commit-time', help: 'Commit time in UNIX timestamp');
}
@override
@ -37,6 +39,20 @@ class UploadResultsCommand extends Command<void> {
final String? gitBranch = argResults!['git-branch'] as String?;
final String? builderName = argResults!['luci-builder'] as String?;
final String? testStatus = argResults!['test-status'] as String?;
final String? commitTime = argResults!['commit-time'] as String?;
// Upload metrics to metrics_center from test runner when `commitTime` is specified. This
// is mainly for testing purpose.
// The upload step will be skipped from cocoon once this is validated.
// TODO(keyong): remove try block to block test when this is validated to work https://github.com/flutter/flutter/issues/88484
try {
if (commitTime != null) {
await uploadToMetricsCenter(resultsPath, commitTime);
print('Successfully uploaded metrics to metrics center');
}
} on Exception catch (e, stacktrace) {
print('Uploading metrics failure: $e\n\n$stacktrace');
}
final Cocoon cocoon = Cocoon(serviceAccountTokenPath: serviceAccountTokenFile);
return cocoon.sendResultsPath(

94
dev/devicelab/lib/framework/metrics_center.dart Normal file
View File

@ -0,0 +1,94 @@
// Copyright 2014 The Flutter Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
import 'dart:convert';
import 'dart:io';
import 'package:metrics_center/metrics_center.dart';
/// Authenticate and connect to gcloud storage.
///
/// It supports both token and credential authentications.
Future<FlutterDestination> connectFlutterDestination() async {
const String kTokenPath = 'TOKEN_PATH';
const String kGcpProject = 'GCP_PROJECT';
final Map<String, String> env = Platform.environment;
final bool isTesting = env['IS_TESTING'] == 'true';
if (env.containsKey(kTokenPath) && env.containsKey(kGcpProject)) {
return FlutterDestination.makeFromAccessToken(
File(env[kTokenPath]!).readAsStringSync(),
env[kGcpProject]!,
isTesting: isTesting,
);
}
return FlutterDestination.makeFromCredentialsJson(
jsonDecode(env['BENCHMARK_GCP_CREDENTIALS']!) as Map<String, dynamic>,
isTesting: isTesting,
);
}
/// Parse results into Metric Points.
///
/// An example of `resultsJson`:
/// {
/// "CommitBranch": "master",
/// "CommitSha": "abc",
/// "BuilderName": "test",
/// "ResultData": {
/// "average_frame_build_time_millis": 0.4550425531914895,
/// "90th_percentile_frame_build_time_millis": 0.473
/// },
/// "BenchmarkScoreKeys": [
/// "average_frame_build_time_millis",
/// "90th_percentile_frame_build_time_millis"
/// ]
/// }
List<MetricPoint> parse(Map<String, dynamic> resultsJson) {
final List<String> scoreKeys = (resultsJson['BenchmarkScoreKeys'] as List<dynamic>).cast<String>();
final Map<String, dynamic> resultData = resultsJson['ResultData'] as Map<String, dynamic>;
final String gitBranch = (resultsJson['CommitBranch'] as String).trim();
final String gitSha = (resultsJson['CommitSha'] as String).trim();
final String builderName = resultsJson['BuilderName'] as String;
final List<MetricPoint> metricPoints = <MetricPoint>[];
for (final String scoreKey in scoreKeys) {
metricPoints.add(
MetricPoint(
(resultData[scoreKey] as num).toDouble(),
<String, String>{
kGithubRepoKey: kFlutterFrameworkRepo,
kGitRevisionKey: gitSha,
'branch': gitBranch,
kNameKey: builderName,
kSubResultKey: scoreKey,
},
),
);
}
return metricPoints;
}
/// Upload test metrics to metrics center.
Future<void> uploadToMetricsCenter(String? resultsPath, String? commitTime) async {
int commitTimeSinceEpoch;
if (resultsPath == null) {
return;
}
if (commitTime != null) {
commitTimeSinceEpoch = 1000 * int.parse(commitTime);
} else {
commitTimeSinceEpoch = DateTime.now().millisecondsSinceEpoch;
}
final File resultFile = File(resultsPath);
Map<String, dynamic> resultsJson = <String, dynamic>{};
resultsJson = json.decode(await resultFile.readAsString()) as Map<String, dynamic>;
final List<MetricPoint> metricPoints = parse(resultsJson);
final FlutterDestination metricsDestination = await connectFlutterDestination();
await metricsDestination.update(
metricPoints,
DateTime.fromMillisecondsSinceEpoch(
commitTimeSinceEpoch,
isUtc: true,
),
);
}

8
dev/devicelab/pubspec.yaml
View File

@ -11,6 +11,7 @@ dependencies:
file: 6.1.2
http: 0.13.3
meta: 1.7.0
metrics_center: 1.0.0
path: 1.8.0
platform: 3.0.2
process: 4.2.3
@ -19,11 +20,16 @@ dependencies:
logging: 1.0.1
_discoveryapis_commons: 1.0.1 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
async: 2.8.2 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
charcode: 1.3.1 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
checked_yaml: 2.0.1 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
collection: 1.15.0 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
crypto: 3.0.1 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
equatable: 2.0.3 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
gcloud: 0.8.2 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
googleapis: 3.0.0 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
googleapis_auth: 1.1.0 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
http_parser: 4.0.0 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
json_annotation: 4.1.0 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
pedantic: 1.11.1 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
@ -67,4 +73,4 @@ dev_dependencies:
web_socket_channel: 2.1.0 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
webkit_inspection_protocol: 1.0.0 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
# PUBSPEC CHECKSUM: 1094
# PUBSPEC CHECKSUM: 3624

32
dev/devicelab/test/metrics_center_test.dart Normal file
View File

@ -0,0 +1,32 @@
// Copyright 2014 The Flutter Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
import 'package:flutter_devicelab/framework/metrics_center.dart';
import 'package:metrics_center/metrics_center.dart';
import 'common.dart';
void main() {
group('Parse', () {
test('succeeds', () {
final Map<String, dynamic> results = <String, dynamic>{
'CommitBranch': 'master',
'CommitSha': 'abc',
'BuilderName': 'test',
'ResultData': <String, dynamic>{
'average_frame_build_time_millis': 0.4550425531914895,
'90th_percentile_frame_build_time_millis': 0.473
},
'BenchmarkScoreKeys': <String>[
'average_frame_build_time_millis',
'90th_percentile_frame_build_time_millis'
]
};
final List<MetricPoint> metricPoints = parse(results);
expect(metricPoints[0].value, equals(0.4550425531914895));
expect(metricPoints[1].value, equals(0.473));
});
});
}