From 27cacca01117f4788d4ecdfecbce9b37f85138c4 Mon Sep 17 00:00:00 2001 From: Jonah Williams Date: Thu, 15 Feb 2024 14:32:38 -0800 Subject: [PATCH] [devicelab] migrate new gallery benchmarks to local copy. (#143545) This is the next step in archiving flutter/gallery - move devicelab tests to use the local copy. --- .../flutter_gallery_v2_chrome_run_test.dart | 19 +++++--------- .../flutter_gallery_v2_web_compile_test.dart | 24 +++++++---------- .../bin/tasks/new_gallery__crane_perf.dart | 26 +++++-------------- .../tasks/new_gallery__transition_perf.dart | 26 +++++-------------- ...new_gallery_impeller__transition_perf.dart | 13 +--------- ...ry_impeller_old_zoom__transition_perf.dart | 13 +--------- .../new_gallery_ios__transition_perf.dart | 13 +--------- ...ry_opengles_impeller__transition_perf.dart | 13 +--------- ...new_gallery_skia_ios__transition_perf.dart | 13 +--------- dev/devicelab/lib/framework/utils.dart | 16 ------------ dev/devicelab/lib/tasks/new_gallery.dart | 21 +++------------ dev/devicelab/lib/tasks/perf_tests.dart | 15 +++++++++++ dev/devicelab/lib/versions/gallery.dart | 6 ----- 13 files changed, 52 insertions(+), 166 deletions(-) delete mode 100644 dev/devicelab/lib/versions/gallery.dart diff --git a/dev/devicelab/bin/tasks/flutter_gallery_v2_chrome_run_test.dart b/dev/devicelab/bin/tasks/flutter_gallery_v2_chrome_run_test.dart index 501e5e0665..4f7fdb2890 100644 --- a/dev/devicelab/bin/tasks/flutter_gallery_v2_chrome_run_test.dart +++ b/dev/devicelab/bin/tasks/flutter_gallery_v2_chrome_run_test.dart @@ -9,8 +9,6 @@ import 'dart:io'; import 'package:flutter_devicelab/framework/framework.dart'; import 'package:flutter_devicelab/framework/task_result.dart'; import 'package:flutter_devicelab/framework/utils.dart'; -import 'package:flutter_devicelab/versions/gallery.dart' show galleryVersion; -import 'package:path/path.dart' as path; Future main() async { await task(const NewGalleryChromeRunTest().run); @@ -34,14 +32,13 @@ class NewGalleryChromeRunTest { /// Runs the test. Future run() async { - final Directory galleryParentDir = - Directory.systemTemp.createTempSync('flutter_gallery_v2_chrome_run.'); - final Directory galleryDir = - Directory(path.join(galleryParentDir.path, 'gallery')); - - await getNewGallery(galleryVersion, galleryDir); - - final TaskResult result = await inDirectory(galleryDir, () async { + final TaskResult result = await inDirectory('${flutterDirectory.path}/dev/integration_tests/new_gallery/', () async { + await flutter('create', options: [ + '--platforms', + 'web,android,ios', + '--no-overwrite', + '.' + ]); await flutter('doctor'); await flutter('packages', options: ['get']); @@ -105,8 +102,6 @@ class NewGalleryChromeRunTest { } }); - rmTree(galleryParentDir); - return result; } } diff --git a/dev/devicelab/bin/tasks/flutter_gallery_v2_web_compile_test.dart b/dev/devicelab/bin/tasks/flutter_gallery_v2_web_compile_test.dart index 1356645d0e..0adc68e56f 100644 --- a/dev/devicelab/bin/tasks/flutter_gallery_v2_web_compile_test.dart +++ b/dev/devicelab/bin/tasks/flutter_gallery_v2_web_compile_test.dart @@ -2,14 +2,10 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. -import 'dart:io'; - import 'package:flutter_devicelab/framework/framework.dart'; import 'package:flutter_devicelab/framework/task_result.dart'; import 'package:flutter_devicelab/framework/utils.dart'; import 'package:flutter_devicelab/tasks/perf_tests.dart' show WebCompileTest; -import 'package:flutter_devicelab/versions/gallery.dart' show galleryVersion; -import 'package:path/path.dart' as path; Future main() async { await task(const NewGalleryWebCompileTest().run); @@ -24,28 +20,26 @@ class NewGalleryWebCompileTest { /// Runs the test. Future run() async { - final Directory galleryParentDir = - Directory.systemTemp.createTempSync('flutter_gallery_v2_web_compile.'); - final Directory galleryDir = - Directory(path.join(galleryParentDir.path, 'gallery')); - - await getNewGallery(galleryVersion, galleryDir); - final Map metrics = await inDirectory>( - galleryDir, + '${flutterDirectory.path}/dev/integration_tests/new_gallery/', () async { await flutter('doctor'); + await flutter('create', options: [ + '--platforms', + 'web,android,ios', + '--no-overwrite', + '.' + ]); + return WebCompileTest.runSingleBuildTest( - directory: galleryDir.path, + directory: '${flutterDirectory.path}/dev/integration_tests/new_gallery/', metric: metricKeyPrefix, measureBuildTime: true, ); }, ); - rmTree(galleryParentDir); - return TaskResult.success(metrics, benchmarkScoreKeys: metrics.keys.toList()); } } diff --git a/dev/devicelab/bin/tasks/new_gallery__crane_perf.dart b/dev/devicelab/bin/tasks/new_gallery__crane_perf.dart index 92f7c84720..3a64591c1b 100644 --- a/dev/devicelab/bin/tasks/new_gallery__crane_perf.dart +++ b/dev/devicelab/bin/tasks/new_gallery__crane_perf.dart @@ -2,31 +2,17 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. -import 'dart:io'; - import 'package:flutter_devicelab/framework/devices.dart'; import 'package:flutter_devicelab/framework/framework.dart'; -import 'package:flutter_devicelab/framework/utils.dart'; import 'package:flutter_devicelab/tasks/new_gallery.dart'; -import 'package:path/path.dart' as path; Future main() async { deviceOperatingSystem = DeviceOperatingSystem.android; - final Directory galleryParentDir = - Directory.systemTemp.createTempSync('flutter_new_gallery_test.'); - final Directory galleryDir = - Directory(path.join(galleryParentDir.path, 'gallery')); - - try { - await task( - NewGalleryPerfTest( - galleryDir, - timelineFileName: 'transitions-crane', - dartDefine: 'onlyCrane=true', - ).run, - ); - } finally { - rmTree(galleryParentDir); - } + await task( + NewGalleryPerfTest( + timelineFileName: 'transitions-crane', + dartDefine: 'onlyCrane=true', + ).run, + ); } diff --git a/dev/devicelab/bin/tasks/new_gallery__transition_perf.dart b/dev/devicelab/bin/tasks/new_gallery__transition_perf.dart index dfac78c5b7..aa5c0a8a48 100644 --- a/dev/devicelab/bin/tasks/new_gallery__transition_perf.dart +++ b/dev/devicelab/bin/tasks/new_gallery__transition_perf.dart @@ -2,30 +2,18 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. -import 'dart:io'; - import 'package:flutter_devicelab/framework/devices.dart'; import 'package:flutter_devicelab/framework/framework.dart'; -import 'package:flutter_devicelab/framework/utils.dart'; import 'package:flutter_devicelab/tasks/new_gallery.dart'; -import 'package:path/path.dart' as path; Future main() async { deviceOperatingSystem = DeviceOperatingSystem.android; - final Directory galleryParentDir = Directory.systemTemp.createTempSync('flutter_new_gallery_test.'); - final Directory galleryDir = Directory(path.join(galleryParentDir.path, 'gallery')); - - try { - await task( - NewGalleryPerfTest( - galleryDir, - // time out after 20 minutes allowing the tool to take a screenshot to debug - // https://github.com/flutter/flutter/issues/114025. - timeoutSeconds: 20 * 60, - ).run, - ); - } finally { - rmTree(galleryParentDir); - } + await task( + NewGalleryPerfTest( + // time out after 20 minutes allowing the tool to take a screenshot to debug + // https://github.com/flutter/flutter/issues/114025. + timeoutSeconds: 20 * 60, + ).run, + ); } diff --git a/dev/devicelab/bin/tasks/new_gallery_impeller__transition_perf.dart b/dev/devicelab/bin/tasks/new_gallery_impeller__transition_perf.dart index 3ddb1555a3..1bd1c417a6 100644 --- a/dev/devicelab/bin/tasks/new_gallery_impeller__transition_perf.dart +++ b/dev/devicelab/bin/tasks/new_gallery_impeller__transition_perf.dart @@ -2,23 +2,12 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. -import 'dart:io'; - import 'package:flutter_devicelab/framework/devices.dart'; import 'package:flutter_devicelab/framework/framework.dart'; -import 'package:flutter_devicelab/framework/utils.dart'; import 'package:flutter_devicelab/tasks/new_gallery.dart'; -import 'package:path/path.dart' as path; Future main() async { deviceOperatingSystem = DeviceOperatingSystem.android; - final Directory galleryParentDir = Directory.systemTemp.createTempSync('flutter_new_gallery_test.'); - final Directory galleryDir = Directory(path.join(galleryParentDir.path, 'gallery')); - - try { - await task(NewGalleryPerfTest(galleryDir, enableImpeller: true).run); - } finally { - rmTree(galleryParentDir); - } + await task(NewGalleryPerfTest(enableImpeller: true).run); } diff --git a/dev/devicelab/bin/tasks/new_gallery_impeller_old_zoom__transition_perf.dart b/dev/devicelab/bin/tasks/new_gallery_impeller_old_zoom__transition_perf.dart index 5b2552fea4..fbe0cc78c8 100644 --- a/dev/devicelab/bin/tasks/new_gallery_impeller_old_zoom__transition_perf.dart +++ b/dev/devicelab/bin/tasks/new_gallery_impeller_old_zoom__transition_perf.dart @@ -2,23 +2,12 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. -import 'dart:io'; - import 'package:flutter_devicelab/framework/devices.dart'; import 'package:flutter_devicelab/framework/framework.dart'; -import 'package:flutter_devicelab/framework/utils.dart'; import 'package:flutter_devicelab/tasks/new_gallery.dart'; -import 'package:path/path.dart' as path; Future main() async { deviceOperatingSystem = DeviceOperatingSystem.android; - final Directory galleryParentDir = Directory.systemTemp.createTempSync('flutter_new_gallery_test.'); - final Directory galleryDir = Directory(path.join(galleryParentDir.path, 'gallery')); - - try { - await task(NewGalleryPerfTest(galleryDir, enableImpeller: true, dartDefine: 'flutter.benchmarks.force_disable_snapshot=true').run); - } finally { - rmTree(galleryParentDir); - } + await task(NewGalleryPerfTest(enableImpeller: true, dartDefine: 'flutter.benchmarks.force_disable_snapshot=true').run); } diff --git a/dev/devicelab/bin/tasks/new_gallery_ios__transition_perf.dart b/dev/devicelab/bin/tasks/new_gallery_ios__transition_perf.dart index aab083fedd..fbf0203936 100644 --- a/dev/devicelab/bin/tasks/new_gallery_ios__transition_perf.dart +++ b/dev/devicelab/bin/tasks/new_gallery_ios__transition_perf.dart @@ -2,23 +2,12 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. -import 'dart:io'; - import 'package:flutter_devicelab/framework/devices.dart'; import 'package:flutter_devicelab/framework/framework.dart'; -import 'package:flutter_devicelab/framework/utils.dart'; import 'package:flutter_devicelab/tasks/new_gallery.dart'; -import 'package:path/path.dart' as path; Future main() async { deviceOperatingSystem = DeviceOperatingSystem.ios; - final Directory galleryParentDir = Directory.systemTemp.createTempSync('flutter_new_gallery_test.'); - final Directory galleryDir = Directory(path.join(galleryParentDir.path, 'gallery')); - - try { - await task(NewGalleryPerfTest(galleryDir).run); - } finally { - rmTree(galleryParentDir); - } + await task(NewGalleryPerfTest().run); } diff --git a/dev/devicelab/bin/tasks/new_gallery_opengles_impeller__transition_perf.dart b/dev/devicelab/bin/tasks/new_gallery_opengles_impeller__transition_perf.dart index 2a33ce72a8..d2a213cbfa 100644 --- a/dev/devicelab/bin/tasks/new_gallery_opengles_impeller__transition_perf.dart +++ b/dev/devicelab/bin/tasks/new_gallery_opengles_impeller__transition_perf.dart @@ -2,23 +2,12 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. -import 'dart:io'; - import 'package:flutter_devicelab/framework/devices.dart'; import 'package:flutter_devicelab/framework/framework.dart'; -import 'package:flutter_devicelab/framework/utils.dart'; import 'package:flutter_devicelab/tasks/new_gallery.dart'; -import 'package:path/path.dart' as path; Future main() async { deviceOperatingSystem = DeviceOperatingSystem.android; - final Directory galleryParentDir = Directory.systemTemp.createTempSync('flutter_new_gallery_test.'); - final Directory galleryDir = Directory(path.join(galleryParentDir.path, 'gallery')); - - try { - await task(NewGalleryPerfTest(galleryDir, enableImpeller: true, forceOpenGLES: true).run); - } finally { - rmTree(galleryParentDir); - } + await task(NewGalleryPerfTest(enableImpeller: true, forceOpenGLES: true).run); } diff --git a/dev/devicelab/bin/tasks/new_gallery_skia_ios__transition_perf.dart b/dev/devicelab/bin/tasks/new_gallery_skia_ios__transition_perf.dart index ab0e23750c..4fd4ddf2ed 100644 --- a/dev/devicelab/bin/tasks/new_gallery_skia_ios__transition_perf.dart +++ b/dev/devicelab/bin/tasks/new_gallery_skia_ios__transition_perf.dart @@ -2,23 +2,12 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. -import 'dart:io'; - import 'package:flutter_devicelab/framework/devices.dart'; import 'package:flutter_devicelab/framework/framework.dart'; -import 'package:flutter_devicelab/framework/utils.dart'; import 'package:flutter_devicelab/tasks/new_gallery.dart'; -import 'package:path/path.dart' as path; Future main() async { deviceOperatingSystem = DeviceOperatingSystem.ios; - final Directory galleryParentDir = Directory.systemTemp.createTempSync('flutter_new_gallery_test.'); - final Directory galleryDir = Directory(path.join(galleryParentDir.path, 'gallery')); - - try { - await task(NewGalleryPerfTest(galleryDir, enableImpeller: false).run); - } finally { - rmTree(galleryParentDir); - } + await task(NewGalleryPerfTest(enableImpeller: false).run); } diff --git a/dev/devicelab/lib/framework/utils.dart b/dev/devicelab/lib/framework/utils.dart index 9d19e6863c..95f119484d 100644 --- a/dev/devicelab/lib/framework/utils.dart +++ b/dev/devicelab/lib/framework/utils.dart @@ -706,22 +706,6 @@ String jsonEncode(dynamic data) { return '$jsonValue\n'; } -Future getNewGallery(String revision, Directory galleryDir) async { - section('Get New Flutter Gallery!'); - - if (exists(galleryDir)) { - galleryDir.deleteSync(recursive: true); - } - - await inDirectory(galleryDir.parent, () async { - await exec('git', ['clone', 'https://github.com/flutter/gallery.git']); - }); - - await inDirectory(galleryDir, () async { - await exec('git', ['checkout', revision]); - }); -} - /// Splits [from] into lines and selects those that contain [pattern]. Iterable grep(Pattern pattern, {required String from}) { return from.split('\n').where((String line) { diff --git a/dev/devicelab/lib/tasks/new_gallery.dart b/dev/devicelab/lib/tasks/new_gallery.dart index 6f8b998b4b..87f23b05db 100644 --- a/dev/devicelab/lib/tasks/new_gallery.dart +++ b/dev/devicelab/lib/tasks/new_gallery.dart @@ -2,36 +2,21 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. -import 'dart:io'; - -import '../framework/task_result.dart'; import '../framework/utils.dart'; -import '../versions/gallery.dart' show galleryVersion; import 'perf_tests.dart'; class NewGalleryPerfTest extends PerfTest { - NewGalleryPerfTest( - this.galleryDir, { + NewGalleryPerfTest({ String timelineFileName = 'transitions', String dartDefine = '', super.enableImpeller, super.timeoutSeconds, super.forceOpenGLES, }) : super( - galleryDir.path, + '${flutterDirectory.path}/dev/integration_tests/new_gallery', 'test_driver/transitions_perf.dart', timelineFileName, dartDefine: dartDefine, + createPlatforms: ['android', 'ios', 'web'] ); - - @override - Future run() async { - // Manually roll the new gallery version for now. If the new gallery repo - // turns out to be updated frequently in the future, we can set up an auto - // roller to update this version. - await getNewGallery(galleryVersion, galleryDir); - return super.run(); - } - - final Directory galleryDir; } diff --git a/dev/devicelab/lib/tasks/perf_tests.dart b/dev/devicelab/lib/tasks/perf_tests.dart index ad4561b9da..cf3c039734 100644 --- a/dev/devicelab/lib/tasks/perf_tests.dart +++ b/dev/devicelab/lib/tasks/perf_tests.dart @@ -1185,6 +1185,7 @@ class PerfTest { this.enableImpeller, this.forceOpenGLES, this.disablePartialRepaint = false, + this.createPlatforms = const [], }): _resultFilename = resultFilename; const PerfTest.e2e( @@ -1204,6 +1205,7 @@ class PerfTest { this.enableImpeller, this.forceOpenGLES, this.disablePartialRepaint = false, + this.createPlatforms = const [], }) : saveTraceFile = false, timelineFileName = null, _resultFilename = resultFilename; /// The directory where the app under test is defined. @@ -1274,6 +1276,10 @@ class PerfTest { /// Additional flags for `--dart-define` to control the test final String dartDefine; + /// Additional platforms to create with `flutter create` before running + /// the test. + final List createPlatforms; + Future run() { return internalRun(); } @@ -1295,6 +1301,15 @@ class PerfTest { final String? localEngineHost = localEngineHostFromEnv; final String? localEngineSrcPath = localEngineSrcPathFromEnv; + if (createPlatforms.isNotEmpty) { + await flutter('create', options: [ + '--platforms', + createPlatforms.join(','), + '--no-overwrite', + '.' + ]); + } + bool changedPlist = false; bool changedManifest = false; diff --git a/dev/devicelab/lib/versions/gallery.dart b/dev/devicelab/lib/versions/gallery.dart deleted file mode 100644 index c339c671d2..0000000000 --- a/dev/devicelab/lib/versions/gallery.dart +++ /dev/null @@ -1,6 +0,0 @@ -// Copyright 2014 The Flutter Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -/// The pinned version of flutter gallery, used for devicelab tests. -const String galleryVersion = 'fa031bfe9d131010e7a56ee5d343f9f85b367d64';