From 30f4fecfeb7357ebd977e263d4f6d568809e9d6c Mon Sep 17 00:00:00 2001 From: Matan Lurey Date: Wed, 22 Jan 2025 13:06:10 -0800 Subject: [PATCH] Add piping and bringup nodes for `_vulkan` and `_opengles` suites. (#162020) Towards https://github.com/flutter/flutter/issues/161333. This adds the initial `.ci.yaml` configuration (marked `bringup: true`), without changing the existing shard to avoid the problems we ran into last time when I renamed the shard atomically, as well as piping for providing context in which the test will run (Vulkan or OpenGLES), which can't be introspected in the _running_ app. In a follow-up PR I'll: - Edit `AndroidManifest.xml` before the suite, setting `ImpellerBackend` appropriately - Move the bringup nodes to presubmit - Delete the existing (legacy) node --- .ci.yaml | 26 ++++++++++++++ TESTOWNERS | 4 ++- .../run_android_engine_tests.dart | 35 +++++++++++-------- dev/bots/test.dart | 8 ++++- .../test_driver/_luci_skia_gold_prelude.dart | 9 +++++ ...urface_producer_smiley_face_main_test.dart | 2 +- ...surface_texture_smiley_face_main_test.dart | 2 +- ...ter_rendered_blue_rectangle_main_test.dart | 2 +- ...d_composition_platform_view_main_test.dart | 2 +- ...d_composition_platform_view_main_test.dart | 2 +- ...rtual_display_platform_view_main_test.dart | 2 +- ...tform_view_tap_color_change_main_test.dart | 2 +- 12 files changed, 73 insertions(+), 23 deletions(-) diff --git a/.ci.yaml b/.ci.yaml index 549478d858..48138aa81f 100644 --- a/.ci.yaml +++ b/.ci.yaml @@ -1523,6 +1523,32 @@ targets: - engine/** - DEPS + - name: Linux_android_emu_34 android_engine_vulkan_tests + recipe: flutter/flutter_drone + bringup: true + timeout: 60 + properties: + shard: android_engine_vulkan_tests + tags: > + ["framework", "hostonly", "shard", "linux"] + dependencies: >- + [ + {"dependency": "goldctl", "version": "git_revision:2387d6fff449587eecbb7e45b2692ca0710b63b9"} + ] + + - name: Linux_android_emu_34 android_engine_opengles_tests + recipe: flutter/flutter_drone + bringup: true + timeout: 60 + properties: + shard: android_engine_opengles_tests + tags: > + ["framework", "hostonly", "shard", "linux"] + dependencies: >- + [ + {"dependency": "goldctl", "version": "git_revision:2387d6fff449587eecbb7e45b2692ca0710b63b9"} + ] + - name: Linux_android_emu flutter_driver_android_test recipe: flutter/flutter_drone bringup: true diff --git a/TESTOWNERS b/TESTOWNERS index 72a8a9355e..14564e840c 100644 --- a/TESTOWNERS +++ b/TESTOWNERS @@ -323,12 +323,14 @@ # https://github.com/flutter/flutter/issues/82068 # # analyze @Piinks @flutter/framework +# android_engine_vulkan_tests @matanlurey @jonahwilliams +# android_engine_opengles_tests @matanlurey @jonahwilliams # build_tests @andrewkolos @flutter/tool # ci_yaml flutter roller @keyonghan @flutter/infra # coverage @goderbauer @flutter/infra # customer_testing @Piinks @flutter/framework # docs @Piinks @flutter/framework -# flutter_driver_android_test @matanlurey @johnmccutchan +# flutter_driver_android_test @matanlurey @jonahwilliams # flutter_packaging @christopherfujino @flutter/infra # flutter_plugins @stuartmorgan @flutter/plugin # framework_tests @Piinks @flutter/framework diff --git a/dev/bots/suite_runners/run_android_engine_tests.dart b/dev/bots/suite_runners/run_android_engine_tests.dart index ec0ea12700..770160c90a 100644 --- a/dev/bots/suite_runners/run_android_engine_tests.dart +++ b/dev/bots/suite_runners/run_android_engine_tests.dart @@ -18,35 +18,42 @@ import '../utils.dart'; /// /// ```sh /// # Generate a baseline of local golden files. -/// SHARD=android_engine_tests UPDATE_GOLDENS=1 bin/cache/dart-sdk/bin/dart dev/bots/test.dart +/// SHARD=android_engine_vulkan_tests UPDATE_GOLDENS=1 bin/cache/dart-sdk/bin/dart dev/bots/test.dart /// ``` /// /// 4. Then, re-run the command against the baseline images: /// /// ```sh -/// SHARD=android_engine_tests bin/cache/dart-sdk/bin/dart dev/bots/test.dart +/// SHARD=android_engine_vulkan_tests bin/cache/dart-sdk/bin/dart dev/bots/test.dart /// ``` /// /// If you are trying to debug a commit, you will want to run step (3) first, /// then apply the commit (or flag), and then run step (4). If you are trying /// to determine flakiness in the *same* state, or want better debugging, see /// `dev/integration_tests/android_engine_test/README.md`. -Future runAndroidEngineTests() async { +Future runAndroidEngineTests({required ImpellerBackend impellerBackend}) async { print('Running Flutter Driver Android tests...'); final String androidEngineTestPath = path.join('dev', 'integration_tests', 'android_engine_test'); final List mains = Glob('$androidEngineTestPath/lib/**_main.dart').listSync(); for (final FileSystemEntity file in mains) { - await runCommand('flutter', [ - 'drive', - path.relative(file.path, from: androidEngineTestPath), - // There are no reason to enable development flags for this test. - // Disable them to work around flakiness issues, and in general just - // make less things start up unnecessarily. - '--no-dds', - '--no-enable-dart-profiling', - '--test-arguments=test', - '--test-arguments=--reporter=expanded', - ], workingDirectory: androidEngineTestPath); + await runCommand( + 'flutter', + [ + 'drive', + path.relative(file.path, from: androidEngineTestPath), + // There are no reason to enable development flags for this test. + // Disable them to work around flakiness issues, and in general just + // make less things start up unnecessarily. + '--no-dds', + '--no-enable-dart-profiling', + '--test-arguments=test', + '--test-arguments=--reporter=expanded', + ], + workingDirectory: androidEngineTestPath, + environment: {'ANDROID_ENGINE_TEST_GOLDEN_VARIANT': impellerBackend.name}, + ); } } + +enum ImpellerBackend { vulkan, opengles } diff --git a/dev/bots/test.dart b/dev/bots/test.dart index e715acaccb..0845740085 100644 --- a/dev/bots/test.dart +++ b/dev/bots/test.dart @@ -137,7 +137,13 @@ Future main(List args) async { 'web_skwasm_tests': webTestsSuite.runWebSkwasmUnitTests, // All web integration tests 'web_long_running_tests': webTestsSuite.webLongRunningTestsRunner, - 'android_engine_tests': runAndroidEngineTests, + // TODO(matanlurey): Remove once a post-submit runs with the new shards. + // (Part of https://github.com/flutter/flutter/issues/161333) + 'android_engine_tests': () => runAndroidEngineTests(impellerBackend: ImpellerBackend.vulkan), + 'android_engine_vulkan_tests': + () => runAndroidEngineTests(impellerBackend: ImpellerBackend.vulkan), + 'android_engine_opengles_tests': + () => runAndroidEngineTests(impellerBackend: ImpellerBackend.opengles), 'flutter_plugins': flutterPackagesRunner, 'skp_generator': skpGeneratorTestsRunner, 'customer_testing': customerTestingRunner, diff --git a/dev/integration_tests/android_engine_test/test_driver/_luci_skia_gold_prelude.dart b/dev/integration_tests/android_engine_test/test_driver/_luci_skia_gold_prelude.dart index 2485438509..4fff84c030 100644 --- a/dev/integration_tests/android_engine_test/test_driver/_luci_skia_gold_prelude.dart +++ b/dev/integration_tests/android_engine_test/test_driver/_luci_skia_gold_prelude.dart @@ -6,3 +6,12 @@ import 'dart:io' as io; /// Whether the current environment is LUCI. bool get isLuci => io.Platform.environment['LUCI_CI'] == 'True'; + +/// What golden suffix this test should use for different engine configurations. +String get goldenVariant { + final String? variant = io.Platform.environment['ANDROID_ENGINE_TEST_GOLDEN_VARIANT']; + if (variant == null) { + return ''; + } + return '.$variant'; +} diff --git a/dev/integration_tests/android_engine_test/test_driver/external_texture/surface_producer_smiley_face_main_test.dart b/dev/integration_tests/android_engine_test/test_driver/external_texture/surface_producer_smiley_face_main_test.dart index 3f51dd184f..9876258e87 100644 --- a/dev/integration_tests/android_engine_test/test_driver/external_texture/surface_producer_smiley_face_main_test.dart +++ b/dev/integration_tests/android_engine_test/test_driver/external_texture/surface_producer_smiley_face_main_test.dart @@ -29,7 +29,7 @@ void main() async { setUpAll(() async { if (isLuci) { - await enableSkiaGoldComparator(namePrefix: 'android_engine_test'); + await enableSkiaGoldComparator(namePrefix: 'android_engine_test$goldenVariant'); } flutterDriver = await FlutterDriver.connect(); nativeDriver = await AndroidNativeDriver.connect(flutterDriver); diff --git a/dev/integration_tests/android_engine_test/test_driver/external_texture/surface_texture_smiley_face_main_test.dart b/dev/integration_tests/android_engine_test/test_driver/external_texture/surface_texture_smiley_face_main_test.dart index 97dc1a8ab9..62434e2ead 100644 --- a/dev/integration_tests/android_engine_test/test_driver/external_texture/surface_texture_smiley_face_main_test.dart +++ b/dev/integration_tests/android_engine_test/test_driver/external_texture/surface_texture_smiley_face_main_test.dart @@ -28,7 +28,7 @@ void main() async { setUpAll(() async { if (isLuci) { - await enableSkiaGoldComparator(namePrefix: 'android_engine_test'); + await enableSkiaGoldComparator(namePrefix: 'android_engine_test$goldenVariant'); } flutterDriver = await FlutterDriver.connect(); nativeDriver = await AndroidNativeDriver.connect(flutterDriver); diff --git a/dev/integration_tests/android_engine_test/test_driver/flutter_rendered_blue_rectangle_main_test.dart b/dev/integration_tests/android_engine_test/test_driver/flutter_rendered_blue_rectangle_main_test.dart index 94607fde2d..31bbb21b1c 100644 --- a/dev/integration_tests/android_engine_test/test_driver/flutter_rendered_blue_rectangle_main_test.dart +++ b/dev/integration_tests/android_engine_test/test_driver/flutter_rendered_blue_rectangle_main_test.dart @@ -28,7 +28,7 @@ void main() async { setUpAll(() async { if (isLuci) { - await enableSkiaGoldComparator(namePrefix: 'android_engine_test'); + await enableSkiaGoldComparator(namePrefix: 'android_engine_test$goldenVariant'); } flutterDriver = await FlutterDriver.connect(); nativeDriver = await AndroidNativeDriver.connect(flutterDriver); diff --git a/dev/integration_tests/android_engine_test/test_driver/platform_view/hybrid_composition_platform_view_main_test.dart b/dev/integration_tests/android_engine_test/test_driver/platform_view/hybrid_composition_platform_view_main_test.dart index 63705b6ca5..c66443d8a3 100644 --- a/dev/integration_tests/android_engine_test/test_driver/platform_view/hybrid_composition_platform_view_main_test.dart +++ b/dev/integration_tests/android_engine_test/test_driver/platform_view/hybrid_composition_platform_view_main_test.dart @@ -30,7 +30,7 @@ void main() async { setUpAll(() async { if (isLuci) { - await enableSkiaGoldComparator(namePrefix: 'android_engine_test'); + await enableSkiaGoldComparator(namePrefix: 'android_engine_test$goldenVariant'); } flutterDriver = await FlutterDriver.connect(); nativeDriver = await AndroidNativeDriver.connect(flutterDriver); diff --git a/dev/integration_tests/android_engine_test/test_driver/platform_view/texture_layer_hybrid_composition_platform_view_main_test.dart b/dev/integration_tests/android_engine_test/test_driver/platform_view/texture_layer_hybrid_composition_platform_view_main_test.dart index 9e5a67cc6d..68d8d4cc49 100644 --- a/dev/integration_tests/android_engine_test/test_driver/platform_view/texture_layer_hybrid_composition_platform_view_main_test.dart +++ b/dev/integration_tests/android_engine_test/test_driver/platform_view/texture_layer_hybrid_composition_platform_view_main_test.dart @@ -30,7 +30,7 @@ void main() async { setUpAll(() async { if (isLuci) { - await enableSkiaGoldComparator(namePrefix: 'android_engine_test'); + await enableSkiaGoldComparator(namePrefix: 'android_engine_test$goldenVariant'); } flutterDriver = await FlutterDriver.connect(); nativeDriver = await AndroidNativeDriver.connect(flutterDriver); diff --git a/dev/integration_tests/android_engine_test/test_driver/platform_view/virtual_display_platform_view_main_test.dart b/dev/integration_tests/android_engine_test/test_driver/platform_view/virtual_display_platform_view_main_test.dart index 26ea7bcb17..a987990a54 100644 --- a/dev/integration_tests/android_engine_test/test_driver/platform_view/virtual_display_platform_view_main_test.dart +++ b/dev/integration_tests/android_engine_test/test_driver/platform_view/virtual_display_platform_view_main_test.dart @@ -30,7 +30,7 @@ void main() async { setUpAll(() async { if (isLuci) { - await enableSkiaGoldComparator(namePrefix: 'android_engine_test'); + await enableSkiaGoldComparator(namePrefix: 'android_engine_test$goldenVariant'); } flutterDriver = await FlutterDriver.connect(); nativeDriver = await AndroidNativeDriver.connect(flutterDriver); diff --git a/dev/integration_tests/android_engine_test/test_driver/platform_view_tap_color_change_main_test.dart b/dev/integration_tests/android_engine_test/test_driver/platform_view_tap_color_change_main_test.dart index 8506f8619e..a0cc2339b8 100644 --- a/dev/integration_tests/android_engine_test/test_driver/platform_view_tap_color_change_main_test.dart +++ b/dev/integration_tests/android_engine_test/test_driver/platform_view_tap_color_change_main_test.dart @@ -28,7 +28,7 @@ void main() async { setUpAll(() async { if (isLuci) { - await enableSkiaGoldComparator(namePrefix: 'android_engine_test'); + await enableSkiaGoldComparator(namePrefix: 'android_engine_test$goldenVariant'); } flutterDriver = await FlutterDriver.connect(); nativeDriver = await AndroidNativeDriver.connect(flutterDriver);