From 304b9c668efd36c158664f285ee8ff38c48bfa3f Mon Sep 17 00:00:00 2001 From: Dan Field Date: Fri, 14 May 2021 19:59:24 -0700 Subject: [PATCH] Reland GC tracking benchmarks (#82069) --- .../test_driver/animated_image_test.dart | 2 +- .../bin/tasks/animated_image_gc_perf.dart | 5 ++- dev/devicelab/lib/tasks/gallery.dart | 1 + dev/devicelab/lib/tasks/perf_tests.dart | 4 ++ .../lib/src/frame_timing_summarizer.dart | 18 +++++++- .../lib/integration_test.dart | 45 ++++++++++++++++--- 6 files changed, 67 insertions(+), 8 deletions(-) diff --git a/dev/benchmarks/macrobenchmarks/test_driver/animated_image_test.dart b/dev/benchmarks/macrobenchmarks/test_driver/animated_image_test.dart index f0e62af443..3e1d85b505 100644 --- a/dev/benchmarks/macrobenchmarks/test_driver/animated_image_test.dart +++ b/dev/benchmarks/macrobenchmarks/test_driver/animated_image_test.dart @@ -6,7 +6,7 @@ import 'package:flutter_driver/flutter_driver.dart'; import 'package:test/test.dart' hide TypeMatcher, isInstanceOf; Future main() async { - const String fileName = 'large_image_changer'; + const String fileName = 'animated_image'; test('Animate for 250 frames', () async { final FlutterDriver driver = await FlutterDriver.connect(); diff --git a/dev/devicelab/bin/tasks/animated_image_gc_perf.dart b/dev/devicelab/bin/tasks/animated_image_gc_perf.dart index 7371f6905f..23f68d3097 100644 --- a/dev/devicelab/bin/tasks/animated_image_gc_perf.dart +++ b/dev/devicelab/bin/tasks/animated_image_gc_perf.dart @@ -9,8 +9,11 @@ import 'package:flutter_devicelab/tasks/perf_tests.dart'; Future main() async { deviceOperatingSystem = DeviceOperatingSystem.android; - await task(DevToolsMemoryTest( + await task(PerfTest( '${flutterDirectory.path}/dev/benchmarks/macrobenchmarks', 'test_driver/animated_image.dart', + 'animated_image', + measureCpuGpu: true, + measureMemory: true, ).run); } diff --git a/dev/devicelab/lib/tasks/gallery.dart b/dev/devicelab/lib/tasks/gallery.dart index 29927e10bd..b7f5cf9103 100644 --- a/dev/devicelab/lib/tasks/gallery.dart +++ b/dev/devicelab/lib/tasks/gallery.dart @@ -87,6 +87,7 @@ class GalleryTransitionTest { : '${testFile}_test'); section('DRIVE START'); await flutter('drive', options: [ + '--no-dds', '--profile', if (needFullTimeline) '--trace-startup', diff --git a/dev/devicelab/lib/tasks/perf_tests.dart b/dev/devicelab/lib/tasks/perf_tests.dart index baee1f98dd..3ec9c5ed60 100644 --- a/dev/devicelab/lib/tasks/perf_tests.dart +++ b/dev/devicelab/lib/tasks/perf_tests.dart @@ -705,6 +705,7 @@ class PerfTest { final String deviceId = device.deviceId; await flutter('drive', options: [ + '--no-dds', // TODO(dnfield): consider removing when https://github.com/flutter/flutter/issues/81707 is fixed '--no-android-gradle-daemon', '-v', '--verbose-system-logs', @@ -777,6 +778,8 @@ const List _kCommonScoreKeys = [ 'worst_frame_rasterizer_time_millis', '90th_percentile_frame_rasterizer_time_millis', '99th_percentile_frame_rasterizer_time_millis', + 'new_gen_gc_count', + 'old_gen_gc_count', ]; class PerfTestWithSkSL extends PerfTest { @@ -868,6 +871,7 @@ class PerfTestWithSkSL extends PerfTest { _flutterPath, [ 'run', + '--no-dds', if (deviceOperatingSystem == DeviceOperatingSystem.ios) ...[ '--device-timeout', '5', diff --git a/packages/flutter_test/lib/src/frame_timing_summarizer.dart b/packages/flutter_test/lib/src/frame_timing_summarizer.dart index e43bce2938..6573e5f47f 100644 --- a/packages/flutter_test/lib/src/frame_timing_summarizer.dart +++ b/packages/flutter_test/lib/src/frame_timing_summarizer.dart @@ -17,7 +17,11 @@ class FrameTimingSummarizer { /// Summarize `data` to frame build time and frame rasterizer time statistics. /// /// See [TimelineSummary.summaryJson] for detail. - factory FrameTimingSummarizer(List data) { + factory FrameTimingSummarizer( + List data, { + int? newGenGCCount, + int? oldGenGCCount, + }) { assert(data != null); assert(data.isNotEmpty); final List frameBuildTime = List.unmodifiable( @@ -58,6 +62,8 @@ class FrameTimingSummarizer { p90VsyncOverhead: _findPercentile(vsyncOverheadSorted, 0.90), p99VsyncOverhead: _findPercentile(vsyncOverheadSorted, 0.99), worstVsyncOverhead: vsyncOverheadSorted.last, + newGenGCCount: newGenGCCount ?? -1, + oldGenGCCount: oldGenGCCount ?? -1, ); } @@ -79,6 +85,8 @@ class FrameTimingSummarizer { required this.p90VsyncOverhead, required this.p99VsyncOverhead, required this.worstVsyncOverhead, + required this.newGenGCCount, + required this.oldGenGCCount, }); /// List of frame build time in microseconds @@ -133,6 +141,12 @@ class FrameTimingSummarizer { /// The largest value of [vsyncOverhead] in milliseconds. final Duration worstVsyncOverhead; + /// The number of new generation GCs. + final int newGenGCCount; + + /// The number of old generation GCs. + final int oldGenGCCount; + /// Convert the summary result to a json object. /// /// See [TimelineSummary.summaryJson] for detail. @@ -162,6 +176,8 @@ class FrameTimingSummarizer { 'frame_rasterizer_times': frameRasterizerTime .map((Duration datum) => datum.inMicroseconds) .toList(), + 'new_gen_gc_count': newGenGCCount, + 'old_gen_gc_count': oldGenGCCount, }; } diff --git a/packages/integration_test/lib/integration_test.dart b/packages/integration_test/lib/integration_test.dart index 1e39d35d3d..8f5fea6c82 100644 --- a/packages/integration_test/lib/integration_test.dart +++ b/packages/integration_test/lib/integration_test.dart @@ -228,8 +228,7 @@ https://flutter.dev/docs/testing/integration-tests#testing-on-firebase-test-lab _vmService = vmService; } if (_vmService == null) { - final developer.ServiceProtocolInfo info = - await developer.Service.getInfo(); + final developer.ServiceProtocolInfo info = await developer.Service.getInfo(); assert(info.serverUri != null); _vmService = await vm_io.vmServiceConnectUri( 'ws://localhost:${info.serverUri!.port}${info.serverUri!.path}ws', @@ -302,6 +301,29 @@ https://flutter.dev/docs/testing/integration-tests#testing-on-firebase-test-lab reportData![reportKey] = timeline.toJson(); } + Future<_GarbageCollectionInfo> _runAndGetGCInfo(Future Function() action) async { + if (kIsWeb) { + await action(); + return const _GarbageCollectionInfo(); + } + + final vm.Timeline timeline = await traceTimeline( + action, + streams: ['GC'], + ); + + final int oldGenGCCount = timeline.traceEvents!.where((vm.TimelineEvent event) { + return event.json!['cat'] == 'GC' && event.json!['name'] == 'CollectOldGeneration'; + }).length; + final int newGenGCCount = timeline.traceEvents!.where((vm.TimelineEvent event) { + return event.json!['cat'] == 'GC' && event.json!['name'] == 'CollectNewGeneration'; + }).length; + return _GarbageCollectionInfo( + oldCount: oldGenGCCount, + newCount: newGenGCCount, + ); + } + /// Watches the [FrameTiming] during `action` and report it to the binding /// with key `reportKey`. /// @@ -340,11 +362,16 @@ https://flutter.dev/docs/testing/integration-tests#testing-on-firebase-test-lab await Future.delayed(const Duration(seconds: 2)); // flush old FrameTimings final TimingsCallback watcher = frameTimings.addAll; addTimingsCallback(watcher); - await action(); + final _GarbageCollectionInfo gcInfo = await _runAndGetGCInfo(action); + await delayForFrameTimings(); // make sure all FrameTimings are reported removeTimingsCallback(watcher); - final FrameTimingSummarizer frameTimes = - FrameTimingSummarizer(frameTimings); + + final FrameTimingSummarizer frameTimes = FrameTimingSummarizer( + frameTimings, + newGenGCCount: gcInfo.newCount, + oldGenGCCount: gcInfo.oldCount, + ); reportData ??= {}; reportData![reportKey] = frameTimes.summary; } @@ -381,3 +408,11 @@ https://flutter.dev/docs/testing/integration-tests#testing-on-firebase-test-lab // `LiveTestWidgetsFlutterBinding` https://github.com/flutter/flutter/issues/81534 } } + +@immutable +class _GarbageCollectionInfo { + const _GarbageCollectionInfo({this.oldCount = -1, this.newCount = -1}); + + final int oldCount; + final int newCount; +}