diff --git a/dev/devicelab/bin/tasks/flutter_gallery_sksl_warmup__transition_perf_e2e.dart b/dev/devicelab/bin/tasks/flutter_gallery_sksl_warmup__transition_perf_e2e.dart new file mode 100644 index 0000000000..3127957ab6 --- /dev/null +++ b/dev/devicelab/bin/tasks/flutter_gallery_sksl_warmup__transition_perf_e2e.dart @@ -0,0 +1,14 @@ +// Copyright 2014 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'dart:async'; + +import 'package:flutter_devicelab/tasks/perf_tests.dart'; +import 'package:flutter_devicelab/framework/adb.dart'; +import 'package:flutter_devicelab/framework/framework.dart'; + +Future main() async { + deviceOperatingSystem = DeviceOperatingSystem.android; + await task(createFlutterGalleryTransitionsPerfSkSLWarmupE2ETest()); +} diff --git a/dev/devicelab/bin/tasks/flutter_gallery_sksl_warmup__transition_perf_e2e_ios32.dart b/dev/devicelab/bin/tasks/flutter_gallery_sksl_warmup__transition_perf_e2e_ios32.dart new file mode 100644 index 0000000000..30b9466fba --- /dev/null +++ b/dev/devicelab/bin/tasks/flutter_gallery_sksl_warmup__transition_perf_e2e_ios32.dart @@ -0,0 +1,14 @@ +// Copyright 2014 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'dart:async'; + +import 'package:flutter_devicelab/tasks/perf_tests.dart'; +import 'package:flutter_devicelab/framework/adb.dart'; +import 'package:flutter_devicelab/framework/framework.dart'; + +Future main() async { + deviceOperatingSystem = DeviceOperatingSystem.ios; + await task(createFlutterGalleryTransitionsPerfSkSLWarmupE2ETest()); +} diff --git a/dev/devicelab/lib/tasks/perf_tests.dart b/dev/devicelab/lib/tasks/perf_tests.dart index 2e3f14cc81..b02975c2b1 100644 --- a/dev/devicelab/lib/tasks/perf_tests.dart +++ b/dev/devicelab/lib/tasks/perf_tests.dart @@ -77,7 +77,7 @@ TaskFunction createCullOpacityPerfTest() { } TaskFunction createCullOpacityPerfE2ETest() { - return E2EPerfTest( + return PerfTest.e2e( '${flutterDirectory.path}/dev/benchmarks/macrobenchmarks', 'test/cull_opacity_perf_e2e.dart', ).run; @@ -109,6 +109,14 @@ TaskFunction createFlutterGalleryTransitionsPerfSkSLWarmupTest() { ).run; } +TaskFunction createFlutterGalleryTransitionsPerfSkSLWarmupE2ETest() { + return PerfTestWithSkSL.e2e( + '${flutterDirectory.path}/dev/integration_tests/flutter_gallery', + 'test_driver/transitions_perf_e2e.dart', + testDriver: 'test_driver/transitions_perf_e2e_test.dart', + ).run; +} + TaskFunction createBackdropFilterPerfTest({bool measureCpuGpu = false}) { return PerfTest( '${flutterDirectory.path}/dev/benchmarks/macrobenchmarks', @@ -162,7 +170,7 @@ TaskFunction createPictureCachePerfTest() { } TaskFunction createPictureCachePerfE2ETest() { - return E2EPerfTest( + return PerfTest.e2e( '${flutterDirectory.path}/dev/benchmarks/macrobenchmarks', 'test/picture_cache_perf_e2e.dart', ).run; @@ -284,7 +292,7 @@ TaskFunction createsMultiWidgetConstructPerfTest() { } TaskFunction createsMultiWidgetConstructPerfE2ETest() { - return E2EPerfTest( + return PerfTest.e2e( '${flutterDirectory.path}/dev/benchmarks/macrobenchmarks', 'test/multi_widget_construction_perf_e2e.dart', ).run; @@ -380,7 +388,19 @@ class PerfTest { this.needsFullTimeline = true, this.benchmarkScoreKeys, this.dartDefine = '', - }); + String resultFilename, + }): _resultFilename = resultFilename; + + const PerfTest.e2e( + this.testDirectory, + this.testTarget, { + this.measureCpuGpu = false, + this.testDriver = 'test_driver/e2e_test.dart', + this.needsFullTimeline = false, + this.benchmarkScoreKeys = _kCommonScoreKeys, + this.dartDefine = '', + String resultFilename = 'e2e_perf_summary', + }) : saveTraceFile = false, timelineFileName = null, _resultFilename = resultFilename; /// The directory where the app under test is defined. final String testDirectory; @@ -388,8 +408,9 @@ class PerfTest { final String testTarget; // The prefix name of the filename such as `.timeline_summary.json`. final String timelineFileName; - String get resultFilename => '$timelineFileName.timeline_summary'; String get traceFilename => '$timelineFileName.timeline'; + String get resultFilename => _resultFilename ?? '$timelineFileName.timeline_summary'; + final String _resultFilename; /// The test file to run on the host. final String testDriver; /// Whether to collect CPU and GPU metrics. @@ -480,14 +501,7 @@ class PerfTest { data, detailFiles: detailFiles.isNotEmpty ? detailFiles : null, benchmarkScoreKeys: benchmarkScoreKeys ?? [ - 'average_frame_build_time_millis', - 'worst_frame_build_time_millis', - '90th_percentile_frame_build_time_millis', - '99th_percentile_frame_build_time_millis', - 'average_frame_rasterizer_time_millis', - 'worst_frame_rasterizer_time_millis', - '90th_percentile_frame_rasterizer_time_millis', - '99th_percentile_frame_rasterizer_time_millis', + ..._kCommonScoreKeys, 'average_vsync_transitions_missed', '90th_percentile_vsync_transitions_missed', '99th_percentile_vsync_transitions_missed', @@ -499,34 +513,16 @@ class PerfTest { } } -class E2EPerfTest extends PerfTest { - const E2EPerfTest( - String testDirectory, - String testTarget, { - String summaryFilename, - List benchmarkScoreKeys, - } - ) : super( - testDirectory, - testTarget, - summaryFilename, - testDriver: 'test_driver/e2e_test.dart', - needsFullTimeline: false, - benchmarkScoreKeys: benchmarkScoreKeys ?? const [ - 'average_frame_build_time_millis', - 'worst_frame_build_time_millis', - '90th_percentile_frame_build_time_millis', - '99th_percentile_frame_build_time_millis', - 'average_frame_rasterizer_time_millis', - 'worst_frame_rasterizer_time_millis', - '90th_percentile_frame_rasterizer_time_millis', - '99th_percentile_frame_rasterizer_time_millis', - ], - ); - - @override - String get resultFilename => timelineFileName ?? 'e2e_perf_summary'; -} +const List _kCommonScoreKeys = [ + 'average_frame_build_time_millis', + 'worst_frame_build_time_millis', + '90th_percentile_frame_build_time_millis', + '99th_percentile_frame_build_time_millis', + 'average_frame_rasterizer_time_millis', + 'worst_frame_rasterizer_time_millis', + '90th_percentile_frame_rasterizer_time_millis', + '99th_percentile_frame_rasterizer_time_millis', +]; class PerfTestWithSkSL extends PerfTest { PerfTestWithSkSL( @@ -535,12 +531,30 @@ class PerfTestWithSkSL extends PerfTest { String timelineFileName, { bool measureCpuGpu = false, String testDriver, + bool needsFullTimeline = true, + List benchmarkScoreKeys, }) : super( testDirectory, testTarget, timelineFileName, measureCpuGpu: measureCpuGpu, testDriver: testDriver, + needsFullTimeline: needsFullTimeline, + benchmarkScoreKeys: benchmarkScoreKeys, + ); + + + PerfTestWithSkSL.e2e( + String testDirectory, + String testTarget, { + String testDriver = 'test_driver/e2e_test.dart', + String resultFilename = 'e2e_perf_summary', + }) : super.e2e( + testDirectory, + testTarget, + testDriver: testDriver, + needsFullTimeline: false, + resultFilename: resultFilename, ); @override diff --git a/dev/devicelab/manifest.yaml b/dev/devicelab/manifest.yaml index 7d3b448b1c..7adbddcdb7 100644 --- a/dev/devicelab/manifest.yaml +++ b/dev/devicelab/manifest.yaml @@ -216,6 +216,14 @@ tasks: required_agent_capabilities: ["mac/ios32"] flaky: true + flutter_gallery_sksl_warmup__transition_perf_e2e_ios32: + description: > + Measures the runtime performance of Flutter gallery transitions on iPhone4s + with SkSL shader warm-up with e2e. + stage: devicelab_ios + required_agent_capabilities: ["mac/ios32"] + flaky: true + backdrop_filter_perf__timeline_summary: description: > Measures the runtime performance of backdrop filter blurs on Android. @@ -796,6 +804,14 @@ tasks: stage: devicelab required_agent_capabilities: ["linux/android"] + flutter_gallery_sksl_warmup__transition_perf_e2e: + description: > + Measures the runtime performance of Flutter gallery transitions on Android + with SkSL shader warm-up with e2e. + stage: devicelab + required_agent_capabilities: ["linux/android"] + flaky: true + flutter_gallery__transition_perf_with_semantics: description: > Measures the delta in performance of screen transitions without and diff --git a/dev/integration_tests/flutter_gallery/.gitignore b/dev/integration_tests/flutter_gallery/.gitignore index d3e68fd01e..a61bb46ed6 100644 --- a/dev/integration_tests/flutter_gallery/.gitignore +++ b/dev/integration_tests/flutter_gallery/.gitignore @@ -1 +1,3 @@ lib/generated_plugin_registrant.dart +vmservice.out +*.sksl.json