From f7bd03202f0788bf3c7aee748b38ba5e5eff8761 Mon Sep 17 00:00:00 2001 From: Jonah Williams Date: Mon, 14 Aug 2023 11:47:08 -0700 Subject: [PATCH] [devicelab] boot up benchmarks. (#132148) Enable Impeller benchmarks for drawAtlas/drawVertices on iOS/Metal, Android/GLES, and Android/Vulkan. Enable impeller tessellation benchmarks on iOS/Metal and Android/Vulkan - not GLES as this is measuring backend agnostic performance. --- .ci.yaml | 107 ++++++++++++++++++ TESTOWNERS | 10 ++ .../draw_atlas_perf__timeline_summary.dart | 14 +++ ...draw_atlas_perf_ios__timeline_summary.dart | 14 +++ ...atlas_perf_opengles__timeline_summary.dart | 14 +++ .../draw_vertices_perf__timeline_summary.dart | 14 +++ ...w_vertices_perf_ios__timeline_summary.dart | 14 +++ ...tices_perf_opengles__timeline_summary.dart | 14 +++ ...h_tessellation_perf__timeline_summary.dart | 14 +++ ...ssellation_perf_ios__timeline_summary.dart | 14 +++ ...h_tessellation_perf__timeline_summary.dart | 14 +++ ...ssellation_perf_ios__timeline_summary.dart | 14 +++ dev/devicelab/lib/tasks/perf_tests.dart | 50 ++++++++ 13 files changed, 307 insertions(+) create mode 100644 dev/devicelab/bin/tasks/draw_atlas_perf__timeline_summary.dart create mode 100644 dev/devicelab/bin/tasks/draw_atlas_perf_ios__timeline_summary.dart create mode 100644 dev/devicelab/bin/tasks/draw_atlas_perf_opengles__timeline_summary.dart create mode 100644 dev/devicelab/bin/tasks/draw_vertices_perf__timeline_summary.dart create mode 100644 dev/devicelab/bin/tasks/draw_vertices_perf_ios__timeline_summary.dart create mode 100644 dev/devicelab/bin/tasks/draw_vertices_perf_opengles__timeline_summary.dart create mode 100644 dev/devicelab/bin/tasks/dynamic_path_tessellation_perf__timeline_summary.dart create mode 100644 dev/devicelab/bin/tasks/dynamic_path_tessellation_perf_ios__timeline_summary.dart create mode 100644 dev/devicelab/bin/tasks/static_path_tessellation_perf__timeline_summary.dart create mode 100644 dev/devicelab/bin/tasks/static_path_tessellation_perf_ios__timeline_summary.dart diff --git a/.ci.yaml b/.ci.yaml index d016251983..c2a797edde 100644 --- a/.ci.yaml +++ b/.ci.yaml @@ -1664,6 +1664,51 @@ targets: ["devicelab", "android", "linux", "samsung", "s10"] task_name: backdrop_filter_perf__timeline_summary + + - name: Linux_samsung_s10 draw_atlas_perf_opengles__timeline_summary + recipe: devicelab/devicelab_drone + presubmit: false + bringup: true + timeout: 60 + properties: + ignore_flakiness: "true" + tags: > + ["devicelab", "android", "linux", "samsung", "s10"] + task_name: draw_atlas_perf_opengles__timeline_summary + + - name: Linux_samsung_s10 draw_atlas_perf__timeline_summary + recipe: devicelab/devicelab_drone + presubmit: false + bringup: true + timeout: 60 + properties: + ignore_flakiness: "true" + tags: > + ["devicelab", "android", "linux", "samsung", "s10"] + task_name: draw_atlas_perf__timeline_summary + + - name: Linux_samsung_s10 dynamic_path_tessellation_perf__timeline_summary + recipe: devicelab/devicelab_drone + presubmit: false + bringup: true + timeout: 60 + properties: + ignore_flakiness: "true" + tags: > + ["devicelab", "android", "linux", "samsung", "s10"] + task_name: dynamic_path_tessellation_perf__timeline_summary + + - name: Linux_samsung_s10 static_path_tessellation_perf__timeline_summary + recipe: devicelab/devicelab_drone + presubmit: false + bringup: true + timeout: 60 + properties: + ignore_flakiness: "true" + tags: > + ["devicelab", "android", "linux", "samsung", "s10"] + task_name: static_path_tessellation_perf__timeline_summary + - name: Linux_android basic_material_app_android__compile recipe: devicelab/devicelab_drone presubmit: false @@ -2656,6 +2701,68 @@ targets: ["devicelab", "android", "linux", "samsung", "s10"] task_name: animated_blur_backdrop_filter_perf_opengles__timeline_summary + - name: Linux_samsung_s10 draw_vertices_perf_opengles__timeline_summary + recipe: devicelab/devicelab_drone + presubmit: false + bringup: true + timeout: 60 + properties: + ignore_flakiness: "true" + tags: > + ["devicelab", "android", "linux", "samsung", "s10"] + task_name: draw_vertices_perf_opengles__timeline_summary + + - name: Linux_samsung_s10 draw_vertices_perf__timeline_summary + recipe: devicelab/devicelab_drone + presubmit: false + bringup: true + timeout: 60 + properties: + ignore_flakiness: "true" + tags: > + ["devicelab", "android", "linux", "samsung", "s10"] + task_name: draw_vertices_perf__timeline_summary + + - name: Mac_ios draw_vertices_perf_ios__timeline_summary + recipe: devicelab/devicelab_drone + presubmit: false + timeout: 60 + bringup: true + properties: + tags: > + ["devicelab", "ios", "mac"] + task_name: draw_vertices_perf_ios__timeline_summary + + - name: Mac_ios draw_atlas_perf_ios__timeline_summary + recipe: devicelab/devicelab_drone + presubmit: false + timeout: 60 + bringup: true + properties: + tags: > + ["devicelab", "ios", "mac"] + task_name: draw_atlas_perf_ios__timeline_summary + + - name: Mac_ios static_path_tessellation_perf_ios__timeline_summary + recipe: devicelab/devicelab_drone + presubmit: false + timeout: 60 + bringup: true + properties: + tags: > + ["devicelab", "ios", "mac"] + task_name: static_path_tessellation_perf_ios__timeline_summary + + - name: Mac_ios dynamic_path_tessellation_perf_ios__timeline_summary + recipe: devicelab/devicelab_drone + presubmit: false + timeout: 60 + bringup: true + properties: + tags: > + ["devicelab", "ios", "mac"] + task_name: dynamic_path_tessellation_perf_ios__timeline_summary + - name: Staging_build_linux analyze presubmit: false bringup: true diff --git a/TESTOWNERS b/TESTOWNERS index dfd325dd47..ed1c757708 100644 --- a/TESTOWNERS +++ b/TESTOWNERS @@ -97,6 +97,12 @@ /dev/devicelab/bin/tasks/animated_blur_backdrop_filter_perf__timeline_summary.dart @jonahwilliams @flutter/engine /dev/devicelab/bin/tasks/animated_blur_backdrop_filter_perf_opengles__timeline_summary.dart @gaaclarke @flutter/engine /dev/devicelab/bin/tasks/slider_perf_android.dart @tahatesser @flutter/framework +/dev/devicelab/bin/tasks/draw_vertices_perf_opengles__timeline_summary.dart @jonahwilliams @flutter/engine +/dev/devicelab/bin/tasks/draw_atlas_perf_opengles__timeline_summary.dart @jonahwilliams @flutter/engine +/dev/devicelab/bin/tasks/draw_vertices_perf__timeline_summary.dart @jonahwilliams @flutter/engine +/dev/devicelab/bin/tasks/draw_atlas_perf__timeline_summary.dart @jonahwilliams @flutter/engine +/dev/devicelab/bin/tasks/static_path_tessellation_perf__timeline_summary.dart @jonahwilliams @flutter/engine +/dev/devicelab/bin/tasks/dynamic_path_tessellation_perf__timeline_summary.dart @jonahwilliams @flutter/engine ## Windows Android DeviceLab tests /dev/devicelab/bin/tasks/basic_material_app_win__compile.dart @zanderso @flutter/tool @@ -208,6 +214,10 @@ /dev/devicelab/bin/tasks/tiles_scroll_perf_ios__timeline_summary.dart @cyanglaz @flutter/engine /dev/devicelab/bin/tasks/animated_blur_backdrop_filter_perf_ios__timeline_summary.dart @jonahwilliams @flutter/engine /dev/devicelab/bin/tasks/draw_points_perf_ios__timeline_summary.dart @jonahwilliams @flutter/engine +/dev/devicelab/bin/tasks/draw_vertices_perf_ios__timeline_summary.dart @jonahwilliams @flutter/engine +/dev/devicelab/bin/tasks/draw_atlas_perf_ios__timeline_summary.dart @jonahwilliams @flutter/engine +/dev/devicelab/bin/tasks/static_path_tessellation_perf_ios__timeline_summary.dart @jonahwilliams @flutter/engine +/dev/devicelab/bin/tasks/dynamic_path_tessellation_perf_ios__timeline_summary.dart @jonahwilliams @flutter/engine ## Host only DeviceLab tests /dev/devicelab/bin/tasks/animated_complex_opacity_perf_macos__e2e_summary.dart @cbracken @flutter/desktop diff --git a/dev/devicelab/bin/tasks/draw_atlas_perf__timeline_summary.dart b/dev/devicelab/bin/tasks/draw_atlas_perf__timeline_summary.dart new file mode 100644 index 0000000000..9386d64d3b --- /dev/null +++ b/dev/devicelab/bin/tasks/draw_atlas_perf__timeline_summary.dart @@ -0,0 +1,14 @@ +// Copyright 2014 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'dart:async'; + +import 'package:flutter_devicelab/framework/devices.dart'; +import 'package:flutter_devicelab/framework/framework.dart'; +import 'package:flutter_devicelab/tasks/perf_tests.dart'; + +Future main() async { + deviceOperatingSystem = DeviceOperatingSystem.android; + await task(createDrawAtlasPerfTest()); +} diff --git a/dev/devicelab/bin/tasks/draw_atlas_perf_ios__timeline_summary.dart b/dev/devicelab/bin/tasks/draw_atlas_perf_ios__timeline_summary.dart new file mode 100644 index 0000000000..374e5105a8 --- /dev/null +++ b/dev/devicelab/bin/tasks/draw_atlas_perf_ios__timeline_summary.dart @@ -0,0 +1,14 @@ +// Copyright 2014 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'dart:async'; + +import 'package:flutter_devicelab/framework/devices.dart'; +import 'package:flutter_devicelab/framework/framework.dart'; +import 'package:flutter_devicelab/tasks/perf_tests.dart'; + +Future main() async { + deviceOperatingSystem = DeviceOperatingSystem.ios; + await task(createDrawAtlasPerfTest()); +} diff --git a/dev/devicelab/bin/tasks/draw_atlas_perf_opengles__timeline_summary.dart b/dev/devicelab/bin/tasks/draw_atlas_perf_opengles__timeline_summary.dart new file mode 100644 index 0000000000..bbdb935fd6 --- /dev/null +++ b/dev/devicelab/bin/tasks/draw_atlas_perf_opengles__timeline_summary.dart @@ -0,0 +1,14 @@ +// Copyright 2014 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'dart:async'; + +import 'package:flutter_devicelab/framework/devices.dart'; +import 'package:flutter_devicelab/framework/framework.dart'; +import 'package:flutter_devicelab/tasks/perf_tests.dart'; + +Future main() async { + deviceOperatingSystem = DeviceOperatingSystem.android; + await task(createDrawAtlasPerfTest(forceOpenGLES: true)); +} diff --git a/dev/devicelab/bin/tasks/draw_vertices_perf__timeline_summary.dart b/dev/devicelab/bin/tasks/draw_vertices_perf__timeline_summary.dart new file mode 100644 index 0000000000..635e4296d2 --- /dev/null +++ b/dev/devicelab/bin/tasks/draw_vertices_perf__timeline_summary.dart @@ -0,0 +1,14 @@ +// Copyright 2014 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'dart:async'; + +import 'package:flutter_devicelab/framework/devices.dart'; +import 'package:flutter_devicelab/framework/framework.dart'; +import 'package:flutter_devicelab/tasks/perf_tests.dart'; + +Future main() async { + deviceOperatingSystem = DeviceOperatingSystem.android; + await task(createDrawVerticesPerfTest()); +} diff --git a/dev/devicelab/bin/tasks/draw_vertices_perf_ios__timeline_summary.dart b/dev/devicelab/bin/tasks/draw_vertices_perf_ios__timeline_summary.dart new file mode 100644 index 0000000000..96c02f1409 --- /dev/null +++ b/dev/devicelab/bin/tasks/draw_vertices_perf_ios__timeline_summary.dart @@ -0,0 +1,14 @@ +// Copyright 2014 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'dart:async'; + +import 'package:flutter_devicelab/framework/devices.dart'; +import 'package:flutter_devicelab/framework/framework.dart'; +import 'package:flutter_devicelab/tasks/perf_tests.dart'; + +Future main() async { + deviceOperatingSystem = DeviceOperatingSystem.ios; + await task(createDrawVerticesPerfTest()); +} diff --git a/dev/devicelab/bin/tasks/draw_vertices_perf_opengles__timeline_summary.dart b/dev/devicelab/bin/tasks/draw_vertices_perf_opengles__timeline_summary.dart new file mode 100644 index 0000000000..d1163ce2ba --- /dev/null +++ b/dev/devicelab/bin/tasks/draw_vertices_perf_opengles__timeline_summary.dart @@ -0,0 +1,14 @@ +// Copyright 2014 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'dart:async'; + +import 'package:flutter_devicelab/framework/devices.dart'; +import 'package:flutter_devicelab/framework/framework.dart'; +import 'package:flutter_devicelab/tasks/perf_tests.dart'; + +Future main() async { + deviceOperatingSystem = DeviceOperatingSystem.android; + await task(createDrawVerticesPerfTest(forceOpenGLES: true)); +} diff --git a/dev/devicelab/bin/tasks/dynamic_path_tessellation_perf__timeline_summary.dart b/dev/devicelab/bin/tasks/dynamic_path_tessellation_perf__timeline_summary.dart new file mode 100644 index 0000000000..47ae2d1ff2 --- /dev/null +++ b/dev/devicelab/bin/tasks/dynamic_path_tessellation_perf__timeline_summary.dart @@ -0,0 +1,14 @@ +// Copyright 2014 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'dart:async'; + +import 'package:flutter_devicelab/framework/devices.dart'; +import 'package:flutter_devicelab/framework/framework.dart'; +import 'package:flutter_devicelab/tasks/perf_tests.dart'; + +Future main() async { + deviceOperatingSystem = DeviceOperatingSystem.android; + await task(createPathTessellationDynamicPerfTest()); +} diff --git a/dev/devicelab/bin/tasks/dynamic_path_tessellation_perf_ios__timeline_summary.dart b/dev/devicelab/bin/tasks/dynamic_path_tessellation_perf_ios__timeline_summary.dart new file mode 100644 index 0000000000..79509a3b36 --- /dev/null +++ b/dev/devicelab/bin/tasks/dynamic_path_tessellation_perf_ios__timeline_summary.dart @@ -0,0 +1,14 @@ +// Copyright 2014 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'dart:async'; + +import 'package:flutter_devicelab/framework/devices.dart'; +import 'package:flutter_devicelab/framework/framework.dart'; +import 'package:flutter_devicelab/tasks/perf_tests.dart'; + +Future main() async { + deviceOperatingSystem = DeviceOperatingSystem.ios; + await task(createPathTessellationDynamicPerfTest()); +} diff --git a/dev/devicelab/bin/tasks/static_path_tessellation_perf__timeline_summary.dart b/dev/devicelab/bin/tasks/static_path_tessellation_perf__timeline_summary.dart new file mode 100644 index 0000000000..10285bf9e0 --- /dev/null +++ b/dev/devicelab/bin/tasks/static_path_tessellation_perf__timeline_summary.dart @@ -0,0 +1,14 @@ +// Copyright 2014 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'dart:async'; + +import 'package:flutter_devicelab/framework/devices.dart'; +import 'package:flutter_devicelab/framework/framework.dart'; +import 'package:flutter_devicelab/tasks/perf_tests.dart'; + +Future main() async { + deviceOperatingSystem = DeviceOperatingSystem.android; + await task(createPathTessellationStaticPerfTest()); +} diff --git a/dev/devicelab/bin/tasks/static_path_tessellation_perf_ios__timeline_summary.dart b/dev/devicelab/bin/tasks/static_path_tessellation_perf_ios__timeline_summary.dart new file mode 100644 index 0000000000..5278dfbb2d --- /dev/null +++ b/dev/devicelab/bin/tasks/static_path_tessellation_perf_ios__timeline_summary.dart @@ -0,0 +1,14 @@ +// Copyright 2014 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'dart:async'; + +import 'package:flutter_devicelab/framework/devices.dart'; +import 'package:flutter_devicelab/framework/framework.dart'; +import 'package:flutter_devicelab/tasks/perf_tests.dart'; + +Future main() async { + deviceOperatingSystem = DeviceOperatingSystem.ios; + await task(createPathTessellationStaticPerfTest()); +} diff --git a/dev/devicelab/lib/tasks/perf_tests.dart b/dev/devicelab/lib/tasks/perf_tests.dart index 989b5dcc00..0b890f84d8 100644 --- a/dev/devicelab/lib/tasks/perf_tests.dart +++ b/dev/devicelab/lib/tasks/perf_tests.dart @@ -672,6 +672,56 @@ TaskFunction createDrawPointsPerfTest({ ).run; } +TaskFunction createDrawAtlasPerfTest({ + bool? forceOpenGLES, +}) { + return PerfTest( + '${flutterDirectory.path}/dev/benchmarks/macrobenchmarks', + 'test_driver/run_app.dart', + 'draw_atlas_perf', + enableImpeller: true, + testDriver: 'test_driver/draw_atlas_perf.dart', + saveTraceFile: true, + forceOpenGLES: forceOpenGLES, + ).run; +} + +TaskFunction createDrawVerticesPerfTest({ + bool? forceOpenGLES, +}) { + return PerfTest( + '${flutterDirectory.path}/dev/benchmarks/macrobenchmarks', + 'test_driver/run_app.dart', + 'draw_vertices_perf', + enableImpeller: true, + testDriver: 'test_driver/draw_vertices_perf_test.dart', + saveTraceFile: true, + forceOpenGLES: forceOpenGLES, + ).run; +} + +TaskFunction createPathTessellationStaticPerfTest() { + return PerfTest( + '${flutterDirectory.path}/dev/benchmarks/macrobenchmarks', + 'test_driver/run_app.dart', + 'tessellation_perf_static', + enableImpeller: true, + testDriver: 'test_driver/path_tessellation_static_perf_test.dart', + saveTraceFile: true, + ).run; +} + +TaskFunction createPathTessellationDynamicPerfTest() { + return PerfTest( + '${flutterDirectory.path}/dev/benchmarks/macrobenchmarks', + 'test_driver/run_app.dart', + 'tessellation_perf_dynamic', + enableImpeller: true, + testDriver: 'test_driver/path_tessellation_dynamic_perf_test.dart', + saveTraceFile: true, + ).run; +} + TaskFunction createAnimatedComplexOpacityPerfE2ETest({ bool? enableImpeller, }) {