From ada6b7e57e17c99b55e8f1da6d606dd130bbe824 Mon Sep 17 00:00:00 2001 From: Yuqian Li Date: Wed, 4 Nov 2020 17:13:41 -0800 Subject: [PATCH] Initial migration of metrics_center (#69629) For the ease of code reviews, this only includes minimal code from MetricPoint, GoogleBenchmarksParser, and their unit tests. See go/flutter-metrics-center-migration for the overall plan. --- .../metrics_center/lib/google_benchmark.dart | 69 +++++++++++++++++++ .../metrics_center/lib/src/common.dart | 54 +++++++++++++++ dev/benchmarks/metrics_center/pubspec.yaml | 68 ++++++++++++++++++ .../metrics_center/test/common.dart | 27 ++++++++ .../test/example_google_benchmark.json | 32 +++++++++ .../test/google_benchmark_test.dart | 38 ++++++++++ .../metrics_center/test/utility.dart | 10 +++ dev/bots/test.dart | 1 + 8 files changed, 299 insertions(+) create mode 100644 dev/benchmarks/metrics_center/lib/google_benchmark.dart create mode 100644 dev/benchmarks/metrics_center/lib/src/common.dart create mode 100644 dev/benchmarks/metrics_center/pubspec.yaml create mode 100644 dev/benchmarks/metrics_center/test/common.dart create mode 100644 dev/benchmarks/metrics_center/test/example_google_benchmark.json create mode 100644 dev/benchmarks/metrics_center/test/google_benchmark_test.dart create mode 100644 dev/benchmarks/metrics_center/test/utility.dart diff --git a/dev/benchmarks/metrics_center/lib/google_benchmark.dart b/dev/benchmarks/metrics_center/lib/google_benchmark.dart new file mode 100644 index 0000000000..701b1e0d67 --- /dev/null +++ b/dev/benchmarks/metrics_center/lib/google_benchmark.dart @@ -0,0 +1,69 @@ +// Copyright 2014 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'dart:convert'; +import 'dart:io'; + +import 'package:metrics_center/src/common.dart'; + +const String _kTimeUnitKey = 'time_unit'; + +const List _kNonNumericalValueSubResults = [ + kNameKey, + _kTimeUnitKey, + 'iterations', + 'big_o', +]; + +/// Parse the json result of https://github.com/google/benchmark. +class GoogleBenchmarkParser { + /// Given a Google benchmark json output, parse its content into a list of [MetricPoint]. + static Future> parse(String jsonFileName) async { + final Map jsonResult = + jsonDecode(File(jsonFileName).readAsStringSync()) + as Map; + + final Map rawContext = jsonResult['context'] as Map; + final Map context = rawContext.map( + (String k, dynamic v) => MapEntry(k, v.toString()), + ); + final List points = []; + for (final Map item in jsonResult['benchmarks']) { + _parseAnItem(item, points, context); + } + return points; + } +} + +void _parseAnItem( + Map item, + List points, + Map context, +) { + final String name = item[kNameKey] as String; + final Map timeUnitMap = { + kUnitKey: item[_kTimeUnitKey] as String + }; + for (final String subResult in item.keys) { + if (!_kNonNumericalValueSubResults.contains(subResult)) { + num rawValue; + try { + rawValue = item[subResult] as num; + } catch (e) { + print('$subResult: ${item[subResult]} (${item[subResult].runtimeType}) is not a number'); + rethrow; + } + + final double value = rawValue is int ? rawValue.toDouble() : rawValue as double; + points.add( + MetricPoint( + value, + {kNameKey: name, kSubResultKey: subResult} + ..addAll(context) + ..addAll(subResult.endsWith('time') ? timeUnitMap : {}), + ), + ); + } + } +} diff --git a/dev/benchmarks/metrics_center/lib/src/common.dart b/dev/benchmarks/metrics_center/lib/src/common.dart new file mode 100644 index 0000000000..d1efd07f96 --- /dev/null +++ b/dev/benchmarks/metrics_center/lib/src/common.dart @@ -0,0 +1,54 @@ +// Copyright 2014 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'dart:collection'; +import 'dart:convert'; + +import 'package:crypto/crypto.dart'; +import 'package:equatable/equatable.dart'; + +/// Common format of a metric data point. +class MetricPoint extends Equatable { + MetricPoint( + this.value, + Map tags, + ) : _tags = SplayTreeMap.from(tags); + + /// Can store integer values. + final double value; + + /// Test name, unit, timestamp, configs, git revision, ..., in sorted order. + UnmodifiableMapView get tags => + UnmodifiableMapView(_tags); + + /// Unique identifier for updating existing data point. + /// + /// We shouldn't have to worry about hash collisions until we have about + /// 2^128 points. + /// + /// This id should stay constant even if the [tags.keys] are reordered. + /// (Because we are using an ordered SplayTreeMap to generate the id.) + String get id => sha256.convert(utf8.encode('$_tags')).toString(); + + @override + String toString() { + return 'MetricPoint(value=$value, tags=$_tags)'; + } + + final SplayTreeMap _tags; + + @override + List get props => [value, tags]; +} + +/// Some common tag keys +const String kGithubRepoKey = 'gitRepo'; +const String kGitRevisionKey = 'gitRevision'; +const String kUnitKey = 'unit'; +const String kNameKey = 'name'; +const String kSubResultKey = 'subResult'; + +/// Known github repo +const String kFlutterFrameworkRepo = 'flutter/flutter'; +const String kFlutterEngineRepo = 'flutter/engine'; diff --git a/dev/benchmarks/metrics_center/pubspec.yaml b/dev/benchmarks/metrics_center/pubspec.yaml new file mode 100644 index 0000000000..3bfca0642c --- /dev/null +++ b/dev/benchmarks/metrics_center/pubspec.yaml @@ -0,0 +1,68 @@ +name: metrics_center + +dependencies: + args: 1.6.0 + crypto: 2.1.5 + gcloud: 0.7.3 + googleapis: 0.56.1 + googleapis_auth: 0.2.12 + github: 7.0.3 + equatable: 1.2.5 + mockito: 4.1.1 + + _discoveryapis_commons: 0.2.0 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade" + async: 2.5.0-nullsafety.2 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade" + boolean_selector: 2.1.0-nullsafety.2 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade" + charcode: 1.2.0-nullsafety.2 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade" + collection: 1.15.0-nullsafety.4 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade" + convert: 2.1.1 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade" + http: 0.12.2 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade" + http_parser: 3.1.4 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade" + json_annotation: 3.1.0 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade" + matcher: 0.12.10-nullsafety.2 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade" + meta: 1.3.0-nullsafety.5 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade" + path: 1.8.0-nullsafety.2 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade" + source_span: 1.8.0-nullsafety.3 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade" + stack_trace: 1.10.0-nullsafety.5 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade" + stream_channel: 2.1.0-nullsafety.2 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade" + string_scanner: 1.1.0-nullsafety.2 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade" + term_glyph: 1.2.0-nullsafety.2 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade" + test_api: 0.2.19-nullsafety.4 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade" + typed_data: 1.3.0-nullsafety.4 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade" + +dev_dependencies: + test: 1.16.0-nullsafety.7 + pedantic: 1.10.0-nullsafety.2 + + _fe_analyzer_shared: 7.0.0 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade" + analyzer: 0.39.17 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade" + cli_util: 0.2.0 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade" + coverage: 0.14.1 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade" + csslib: 0.16.2 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade" + glob: 1.2.0 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade" + html: 0.14.0+4 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade" + http_multi_server: 2.2.0 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade" + io: 0.3.4 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade" + js: 0.6.3-nullsafety.2 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade" + logging: 0.11.4 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade" + mime: 0.9.7 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade" + node_interop: 1.2.0 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade" + node_io: 1.1.1 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade" + node_preamble: 1.4.12 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade" + package_config: 1.9.3 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade" + pool: 1.5.0-nullsafety.2 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade" + pub_semver: 1.4.4 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade" + shelf: 0.7.5 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade" + shelf_packages_handler: 2.0.0 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade" + shelf_static: 0.2.8 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade" + shelf_web_socket: 0.2.3 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade" + source_map_stack_trace: 2.1.0-nullsafety.3 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade" + source_maps: 0.10.10-nullsafety.2 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade" + test_core: 0.3.12-nullsafety.7 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade" + vm_service: 5.2.0 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade" + watcher: 0.9.7+15 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade" + web_socket_channel: 1.1.0 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade" + webkit_inspection_protocol: 0.7.3 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade" + yaml: 2.2.1 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade" + +# PUBSPEC CHECKSUM: 0dd0 diff --git a/dev/benchmarks/metrics_center/test/common.dart b/dev/benchmarks/metrics_center/test/common.dart new file mode 100644 index 0000000000..6ca543fdc0 --- /dev/null +++ b/dev/benchmarks/metrics_center/test/common.dart @@ -0,0 +1,27 @@ +// Copyright 2014 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'dart:io'; + +import 'package:test/test.dart' hide TypeMatcher, isInstanceOf; +import 'package:test/test.dart' as test_package show TypeMatcher; + +export 'package:test/test.dart' hide TypeMatcher, isInstanceOf; + +// Defines a 'package:test' shim. +// TODO(ianh): Remove this file once https://github.com/dart-lang/matcher/issues/98 is fixed + +/// A matcher that compares the type of the actual value to the type argument T. +test_package.TypeMatcher isInstanceOf() => isA(); + +void tryToDelete(Directory directory) { + // This should not be necessary, but it turns out that + // on Windows it's common for deletions to fail due to + // bogus (we think) "access denied" errors. + try { + directory.deleteSync(recursive: true); + } on FileSystemException catch (error) { + print('Failed to delete ${directory.path}: $error'); + } +} diff --git a/dev/benchmarks/metrics_center/test/example_google_benchmark.json b/dev/benchmarks/metrics_center/test/example_google_benchmark.json new file mode 100644 index 0000000000..212ce9cf37 --- /dev/null +++ b/dev/benchmarks/metrics_center/test/example_google_benchmark.json @@ -0,0 +1,32 @@ +{ + "context": { + "date": "2019-12-17 15:14:14", + "num_cpus": 56, + "mhz_per_cpu": 2594, + "cpu_scaling_enabled": true, + "library_build_type": "release" + }, + "benchmarks": [ + { + "name": "BM_PaintRecordInit", + "iterations": 6749079, + "real_time": 101, + "cpu_time": 101, + "time_unit": "ns" + }, + { + "name": "BM_ParagraphShortLayout", + "iterations": 151761, + "real_time": 4460, + "cpu_time": 4460, + "time_unit": "ns" + }, + { + "name": "BM_ParagraphStylesBigO_BigO", + "cpu_coefficient": 6548, + "real_coefficient": 6548, + "big_o": "N", + "time_unit": "ns" + } + ] +} diff --git a/dev/benchmarks/metrics_center/test/google_benchmark_test.dart b/dev/benchmarks/metrics_center/test/google_benchmark_test.dart new file mode 100644 index 0000000000..629b5a0867 --- /dev/null +++ b/dev/benchmarks/metrics_center/test/google_benchmark_test.dart @@ -0,0 +1,38 @@ +// Copyright 2014 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'package:metrics_center/src/common.dart'; +import 'package:metrics_center/google_benchmark.dart'; + +import 'common.dart'; +import 'utility.dart'; + +void main() { + test('GoogleBenchmarkParser parses example json.', () async { + final List points = + await GoogleBenchmarkParser.parse('test/example_google_benchmark.json'); + expect(points.length, 6); + expectSetMatch( + points.map((MetricPoint p) => p.value), + [101, 101, 4460, 4460, 6548, 6548], + ); + expectSetMatch( + points.map((MetricPoint p) => p.tags[kSubResultKey]), + [ + 'cpu_time', + 'real_time', + 'cpu_coefficient', + 'real_coefficient', + ], + ); + expectSetMatch( + points.map((MetricPoint p) => p.tags[kNameKey]), + [ + 'BM_PaintRecordInit', + 'BM_ParagraphShortLayout', + 'BM_ParagraphStylesBigO_BigO', + ], + ); + }); +} diff --git a/dev/benchmarks/metrics_center/test/utility.dart b/dev/benchmarks/metrics_center/test/utility.dart new file mode 100644 index 0000000000..b0030d6c2e --- /dev/null +++ b/dev/benchmarks/metrics_center/test/utility.dart @@ -0,0 +1,10 @@ +// Copyright 2014 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'common.dart'; + +// This will be used in many of our unit tests. +void expectSetMatch(Iterable actual, Iterable expected) { + expect(Set.from(actual), equals(Set.from(expected))); +} diff --git a/dev/bots/test.dart b/dev/bots/test.dart index d88f0180b4..bfea3d7d0e 100644 --- a/dev/bots/test.dart +++ b/dev/bots/test.dart @@ -671,6 +671,7 @@ Future _runFrameworkTests() async { await _pubRunTest(path.join(flutterRoot, 'dev', 'devicelab'), tableData: bigqueryApi?.tabledata); await _pubRunTest(path.join(flutterRoot, 'dev', 'snippets'), tableData: bigqueryApi?.tabledata); await _pubRunTest(path.join(flutterRoot, 'dev', 'tools'), tableData: bigqueryApi?.tabledata); + await _pubRunTest(path.join(flutterRoot, 'dev', 'benchmarks', 'metrics_center'), tableData: bigqueryApi?.tabledata); await _runFlutterTest(path.join(flutterRoot, 'dev', 'integration_tests', 'android_semantics_testing'), tableData: bigqueryApi?.tabledata); await _runFlutterTest(path.join(flutterRoot, 'dev', 'manual_tests'), tableData: bigqueryApi?.tabledata); await _runFlutterTest(path.join(flutterRoot, 'dev', 'tools', 'vitool'), tableData: bigqueryApi?.tabledata);