diff --git a/.ci.yaml b/.ci.yaml index 69b6387ff8..633b750b7d 100644 --- a/.ci.yaml +++ b/.ci.yaml @@ -1528,7 +1528,7 @@ targets: bringup: true timeout: 60 properties: - shard: flutter_driver_android + shard: android_engine_tests tags: > ["framework", "hostonly", "shard", "linux"] dependencies: >- @@ -1541,7 +1541,7 @@ targets: recipe: flutter/flutter_drone timeout: 60 properties: - shard: flutter_driver_android + shard: android_engine_tests tags: > ["framework", "hostonly", "shard", "linux"] dependencies: >- diff --git a/dev/bots/suite_runners/run_flutter_driver_android_tests.dart b/dev/bots/suite_runners/run_android_engine_tests.dart similarity index 68% rename from dev/bots/suite_runners/run_flutter_driver_android_tests.dart rename to dev/bots/suite_runners/run_android_engine_tests.dart index 78107d5cc5..ec0ea12700 100644 --- a/dev/bots/suite_runners/run_flutter_driver_android_tests.dart +++ b/dev/bots/suite_runners/run_android_engine_tests.dart @@ -17,13 +17,21 @@ import '../utils.dart'; /// 3. Run the following command from the root of the Flutter repository: /// /// ```sh -/// SHARD=flutter_driver_android bin/cache/dart-sdk/bin/dart dev/bots/test.dart +/// # Generate a baseline of local golden files. +/// SHARD=android_engine_tests UPDATE_GOLDENS=1 bin/cache/dart-sdk/bin/dart dev/bots/test.dart /// ``` /// -/// For debugging, you need to instead run and launch these tests -/// individually _in_ the `dev/integration_tests/android_engine_test` directory. -/// Comparisons against goldens cant happen locally. -Future runFlutterDriverAndroidTests() async { +/// 4. Then, re-run the command against the baseline images: +/// +/// ```sh +/// SHARD=android_engine_tests bin/cache/dart-sdk/bin/dart dev/bots/test.dart +/// ``` +/// +/// If you are trying to debug a commit, you will want to run step (3) first, +/// then apply the commit (or flag), and then run step (4). If you are trying +/// to determine flakiness in the *same* state, or want better debugging, see +/// `dev/integration_tests/android_engine_test/README.md`. +Future runAndroidEngineTests() async { print('Running Flutter Driver Android tests...'); final String androidEngineTestPath = path.join('dev', 'integration_tests', 'android_engine_test'); diff --git a/dev/bots/test.dart b/dev/bots/test.dart index 1d20a0e0a2..792816f4bb 100644 --- a/dev/bots/test.dart +++ b/dev/bots/test.dart @@ -52,11 +52,11 @@ import 'package:path/path.dart' as path; import 'run_command.dart'; import 'suite_runners/run_add_to_app_life_cycle_tests.dart'; import 'suite_runners/run_analyze_tests.dart'; +import 'suite_runners/run_android_engine_tests.dart'; import 'suite_runners/run_android_java11_integration_tool_tests.dart'; import 'suite_runners/run_android_preview_integration_tool_tests.dart'; import 'suite_runners/run_customer_testing_tests.dart'; import 'suite_runners/run_docs_tests.dart'; -import 'suite_runners/run_flutter_driver_android_tests.dart'; import 'suite_runners/run_flutter_packages_tests.dart'; import 'suite_runners/run_framework_coverage_tests.dart'; import 'suite_runners/run_framework_tests.dart'; @@ -139,7 +139,7 @@ Future main(List args) async { 'web_skwasm_tests': webTestsSuite.runWebSkwasmUnitTests, // All web integration tests 'web_long_running_tests': webTestsSuite.webLongRunningTestsRunner, - 'flutter_driver_android': runFlutterDriverAndroidTests, + 'android_engine_tests': runAndroidEngineTests, 'flutter_plugins': flutterPackagesRunner, 'skp_generator': skpGeneratorTestsRunner, 'customer_testing': customerTestingRunner, diff --git a/dev/integration_tests/android_engine_test/README.md b/dev/integration_tests/android_engine_test/README.md index 5e02cd956b..70793738ba 100644 --- a/dev/integration_tests/android_engine_test/README.md +++ b/dev/integration_tests/android_engine_test/README.md @@ -5,10 +5,35 @@ This directory contains a sample app and tests that demonstrate how to use the Android devices or emulators, interact with and capture screenshots of the app, and compare the screenshots against golden images. +> [!CAUTION] +> This test suite is a _very_ end-to-end suite that is testing a combination of +> the graphics backend, the Android embedder, the Flutter framework, and Flutter +> tools, and only useful when the documentation and naming stays up to date and +> is clearly actionable. +> +> Please take extra care when updating the test suite to also update the REAMDE. + +## How it runs on CI (LUCI) + +See [`dev/bots/suite_runners/run_android_engine_tests.dart`](../../bots/suite_runners/run_android_engine_tests.dart), but tl;dr: + +```sh +# TIP: If golden-files do not exist locally, this command will fail locally. +SHARD=android_engine_tests bin/cache/dart-sdk/bin/dart dev/bots/test.dart +``` + ## Running the apps and tests Each `lib/{prefix}_main.dart` file is a standalone Flutter app that you can run -on an Android device or emulator: +on an Android device or emulator. + +- [`flutter_rendered_blue_rectangle`](#flutter_rendered_blue_rectangle) +- [`external_texture/surface_producer_smiley_face`](#external_texturesurface_producer_smiley_face) +- [`external_texture/surface_texture_smiley_face`](#external_texturesurface_texture_smiley_face) +- [`platform_view/hybrid_composition_platform_view`](#platform_viewhybrid_composition_platform_view) +- [`platform_view/texture_layer_hybrid_composition_platform_view`](#platform_viewtexture_layer_hybrid_composition_platform_view) +- [`platform_view/virtual_display_platform_view`](#platform_viewvirtual_display_platform_view) +- [`platform_view_tap_color_change`](#platform_view_tap_color_change) ### `flutter_rendered_blue_rectangle` @@ -25,13 +50,97 @@ $ flutter run lib/flutter_rendered_blue_rectangle_main.dart $ flutter drive lib/flutter_rendered_blue_rectangle_main.dart ``` -Files of significance: +### `external_texture/surface_producer_smiley_face` -- [Entrypoint](lib/flutter_rendered_blue_rectangle_main.dart) -- [Test](test_driver/flutter_rendered_blue_rectangle_main_test.dart) +This app displays a full screen rectangular deformed smiley face with a yellow +background. It tests the [`SurfaceProducer`](https://api.flutter.dev/javadoc/io/flutter/view/TextureRegistry.SurfaceProducer.html) API end-to-end, including historic regression cases around +backgrounding the app, trimming memory, and resuming the app. -## Debugging tips +```sh +# Run the app +$ flutter run lib/external_texture/surface_producer_smiley_face_main.dart -- Use `flutter drive --keep-app-running` to keep the app running after the test. -- USe `flutter run` followed by `flutter drive --use-existing-app` for faster - test iterations. +# Run the test +$ flutter drive lib/external_texture/surface_producer_smiley_face_main.dart +``` + +### `external_texture/surface_texture_smiley_face` + +This app displays a full screen rectangular deformed smiley face with a yellow +background. It tests the [`SurfaceTexture`](https://api.flutter.dev/javadoc/io/flutter/view/TextureRegistry.SurfaceTexture.html) API end-to-end. + +```sh +# Run the app +$ flutter run lib/external_texture/surface_texture_smiley_face_main.dart + +# Run the test +$ flutter drive lib/external_texture/surface_texture_smiley_face_main.dart +``` + +### `platform_view/hybrid_composition_platform_view` + +This app displays a blue orange gradient, the app is backgrounded, and then +resumed. It tests the [Hybrid Composition](../../../docs/platforms/android/Android-Platform-Views.md#hybrid-composition) implementation. + +```sh +# Run the app +$ flutter run lib/platform_view/hybrid_composition_platform_view_main.dart + +# Run the test +$ flutter drive lib/platform_view/hybrid_composition_platform_view_main.dart +``` + +### `platform_view/texture_layer_hybrid_composition_platform_view` + +This app displays a blue orange gradient, the app is backgrounded, and then +resumed. It tests the [Texture Layer Hybrid Composition](../../../docs/platforms/android/Android-Platform-Views.md#texture-layer-hybrid-composition) implementation. + +```sh +# Run the app +$ flutter run lib/platform_view/texture_layer_hybrid_composition_platform_view_main.dart + +# Run the test +$ flutter drive lib/platform_view/texture_layer_hybrid_composition_platform_view_main.dart +``` + +### `platform_view/virtual_display_platform_view` + +This app displays a blue orange gradient, the app is backgrounded, and then +resumed. It tests the [Virtual Display](../../../docs/platforms/android/Android-Platform-Views.md#virtual-display) implementation. + +```sh +# Run the app +$ flutter run lib/platform_view/virtual_display_platform_view_main.dart + +# Run the test +$ flutter drive lib/platform_view/virtual_display_platform_view_main.dart +``` + +### `platform_view_tap_color_change` + +This app displays a blue rectangle, using platform views, which upon +being tapped (natively, not by Flutter), changes from blue to red. + +```sh +# Run the app +$ flutter run lib/platform_view_tap_color_change_main.dart + +# Run the test +$ flutter drive lib/platform_view_tap_color_change_main_test.dart +``` + +## Deflaking + +Use `tool/deflake.dart ` to, in 1-command: + +- Build an APK. +- Establish a baseline set of golden-files locally. +- Run N tests (by default, 10) in the same state, asserting the same output. + +For example: + +```sh +dart tool/deflake.dart lib/flutter_rendered_blue_rectangle_main.dart +``` + +For more options, see `dart tool/deflake.dart --help`. diff --git a/dev/integration_tests/android_engine_test/lib/external_texture_smiley_face_main.dart b/dev/integration_tests/android_engine_test/lib/external_texture/surface_producer_smiley_face_main.dart similarity index 97% rename from dev/integration_tests/android_engine_test/lib/external_texture_smiley_face_main.dart rename to dev/integration_tests/android_engine_test/lib/external_texture/surface_producer_smiley_face_main.dart index b1b5bc9e4b..be288a8c1a 100644 --- a/dev/integration_tests/android_engine_test/lib/external_texture_smiley_face_main.dart +++ b/dev/integration_tests/android_engine_test/lib/external_texture/surface_producer_smiley_face_main.dart @@ -9,7 +9,7 @@ import 'package:flutter/material.dart'; import 'package:flutter/services.dart'; import 'package:flutter_driver/driver_extension.dart'; -import 'src/allow_list_devices.dart'; +import '../src/allow_list_devices.dart'; const MethodChannel _channel = MethodChannel('smiley_face_texture'); Future _fetchTexture(int width, int height) async { diff --git a/dev/integration_tests/android_engine_test/lib/external_texture_other_face_main.dart b/dev/integration_tests/android_engine_test/lib/external_texture/surface_texture_smiley_face_main.dart similarity index 97% rename from dev/integration_tests/android_engine_test/lib/external_texture_other_face_main.dart rename to dev/integration_tests/android_engine_test/lib/external_texture/surface_texture_smiley_face_main.dart index 40073f09ca..c18fe7eb67 100644 --- a/dev/integration_tests/android_engine_test/lib/external_texture_other_face_main.dart +++ b/dev/integration_tests/android_engine_test/lib/external_texture/surface_texture_smiley_face_main.dart @@ -9,7 +9,7 @@ import 'package:flutter/material.dart'; import 'package:flutter/services.dart'; import 'package:flutter_driver/driver_extension.dart'; -import 'src/allow_list_devices.dart'; +import '../src/allow_list_devices.dart'; const MethodChannel _channel = MethodChannel('other_face_texture'); Future _fetchTexture(int width, int height) async { diff --git a/dev/integration_tests/android_engine_test/lib/platform_view/hybrid_composition_platform_view_main.dart b/dev/integration_tests/android_engine_test/lib/platform_view/hybrid_composition_platform_view_main.dart new file mode 100644 index 0000000000..fc54890ef9 --- /dev/null +++ b/dev/integration_tests/android_engine_test/lib/platform_view/hybrid_composition_platform_view_main.dart @@ -0,0 +1,64 @@ +// Copyright 2014 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'package:android_driver_extensions/extension.dart'; +import 'package:flutter/foundation.dart'; +import 'package:flutter/gestures.dart'; +import 'package:flutter/material.dart'; +import 'package:flutter/rendering.dart'; +import 'package:flutter/services.dart'; +import 'package:flutter_driver/driver_extension.dart'; + +import '../src/allow_list_devices.dart'; + +void main() async { + ensureAndroidDevice(); + enableFlutterDriverExtension(commands: [nativeDriverCommands]); + + // Run on full screen. + await SystemChrome.setEnabledSystemUIMode(SystemUiMode.immersive); + runApp(const MainApp()); +} + +final class MainApp extends StatelessWidget { + const MainApp({super.key}); + + @override + Widget build(BuildContext context) { + return const MaterialApp( + debugShowCheckedModeBanner: false, + home: _HybridCompositionAndroidPlatformView(viewType: 'blue_orange_gradient_platform_view'), + ); + } +} + +final class _HybridCompositionAndroidPlatformView extends StatelessWidget { + const _HybridCompositionAndroidPlatformView({required this.viewType}); + + final String viewType; + + @override + Widget build(BuildContext context) { + return PlatformViewLink( + viewType: viewType, + surfaceFactory: (BuildContext context, PlatformViewController controller) { + return AndroidViewSurface( + controller: controller as AndroidViewController, + gestureRecognizers: const >{}, + hitTestBehavior: PlatformViewHitTestBehavior.opaque, + ); + }, + onCreatePlatformView: (PlatformViewCreationParams params) { + return PlatformViewsService.initExpensiveAndroidView( + id: params.id, + viewType: viewType, + layoutDirection: TextDirection.ltr, + creationParamsCodec: const StandardMessageCodec(), + ) + ..addOnPlatformViewCreatedListener(params.onPlatformViewCreated) + ..create(); + }, + ); + } +} diff --git a/dev/integration_tests/android_engine_test/test_driver/external_texture_smiley_face_main_test.dart b/dev/integration_tests/android_engine_test/test_driver/external_texture/surface_producer_smiley_face_main_test.dart similarity index 69% rename from dev/integration_tests/android_engine_test/test_driver/external_texture_smiley_face_main_test.dart rename to dev/integration_tests/android_engine_test/test_driver/external_texture/surface_producer_smiley_face_main_test.dart index 22560218b3..3f51dd184f 100644 --- a/dev/integration_tests/android_engine_test/test_driver/external_texture_smiley_face_main_test.dart +++ b/dev/integration_tests/android_engine_test/test_driver/external_texture/surface_producer_smiley_face_main_test.dart @@ -7,19 +7,29 @@ import 'package:android_driver_extensions/skia_gold.dart'; import 'package:flutter_driver/flutter_driver.dart'; import 'package:test/test.dart'; -import '_luci_skia_gold_prelude.dart'; +import '../_luci_skia_gold_prelude.dart'; +/// For local debugging, a (local) golden-file is required as a baseline: +/// +/// ```sh +/// # Checkout HEAD, i.e. *before* changes you want to test. +/// UPDATE_GOLDENS=1 flutter drive lib/external_texture/surface_producer_smiley_face_main.dart +/// +/// # Make your changes. +/// +/// # Run the test against baseline. +/// flutter drive lib/external_texture/surface_producer_smiley_face_main.dart +/// ``` +/// +/// For a convenient way to deflake a test, see `tool/deflake.dart`. void main() async { - // To test the golden file generation locally, comment out the following line. - // autoUpdateGoldenFiles = true; - const String appName = 'com.example.android_engine_test'; late final FlutterDriver flutterDriver; late final NativeDriver nativeDriver; setUpAll(() async { if (isLuci) { - await enableSkiaGoldComparator(); + await enableSkiaGoldComparator(namePrefix: 'android_engine_test'); } flutterDriver = await FlutterDriver.connect(); nativeDriver = await AndroidNativeDriver.connect(flutterDriver); @@ -32,8 +42,6 @@ void main() async { }); test('should screenshot and match an external smiley face texture', () async { - await flutterDriver.waitFor(find.byType('Texture')); - // On Android: Background the app, trim memory, and restore the app. if (nativeDriver case final AndroidNativeDriver nativeDriver) { print('Backgrounding the app, trimming memory, and resuming the app.'); @@ -48,7 +56,7 @@ void main() async { await expectLater( nativeDriver.screenshot(), - matchesGoldenFile('external_texture_smiley_face.android.png'), + matchesGoldenFile('external_texture_surface_producer_smiley_face.png'), ); }, timeout: Timeout.none); } diff --git a/dev/integration_tests/android_engine_test/test_driver/external_texture_other_face_main_test.dart b/dev/integration_tests/android_engine_test/test_driver/external_texture/surface_texture_smiley_face_main_test.dart similarity index 59% rename from dev/integration_tests/android_engine_test/test_driver/external_texture_other_face_main_test.dart rename to dev/integration_tests/android_engine_test/test_driver/external_texture/surface_texture_smiley_face_main_test.dart index 072d0713da..97dc1a8ab9 100644 --- a/dev/integration_tests/android_engine_test/test_driver/external_texture_other_face_main_test.dart +++ b/dev/integration_tests/android_engine_test/test_driver/external_texture/surface_texture_smiley_face_main_test.dart @@ -7,17 +7,28 @@ import 'package:android_driver_extensions/skia_gold.dart'; import 'package:flutter_driver/flutter_driver.dart'; import 'package:test/test.dart'; -import '_luci_skia_gold_prelude.dart'; +import '../_luci_skia_gold_prelude.dart'; +/// For local debugging, a (local) golden-file is required as a baseline: +/// +/// ```sh +/// # Checkout HEAD, i.e. *before* changes you want to test. +/// UPDATE_GOLDENS=1 flutter drive lib/external_texture/surface_texture_smiley_face_main.dart +/// +/// # Make your changes. +/// +/// # Run the test against baseline. +/// flutter drive lib/external_texture/surface_texture_smiley_face_main.dart +/// ``` +/// +/// For a convenient way to deflake a test, see `tool/deflake.dart`. void main() async { - // To test the golden file generation locally, comment out the following line. - // autoUpdateGoldenFiles = true; late final FlutterDriver flutterDriver; late final NativeDriver nativeDriver; setUpAll(() async { if (isLuci) { - await enableSkiaGoldComparator(); + await enableSkiaGoldComparator(namePrefix: 'android_engine_test'); } flutterDriver = await FlutterDriver.connect(); nativeDriver = await AndroidNativeDriver.connect(flutterDriver); @@ -30,11 +41,9 @@ void main() async { }); test('should screenshot and match a smiley face texture using the trampoline', () async { - await flutterDriver.waitFor(find.byType('Texture')); - await expectLater( nativeDriver.screenshot(), - matchesGoldenFile('external_texture_other_face.android.png'), + matchesGoldenFile('external_texture_surface_texture_smiley_face.png'), ); }, timeout: Timeout.none); } diff --git a/dev/integration_tests/android_engine_test/test_driver/flutter_rendered_blue_rectangle_main_test.dart b/dev/integration_tests/android_engine_test/test_driver/flutter_rendered_blue_rectangle_main_test.dart index c1095488ce..94607fde2d 100644 --- a/dev/integration_tests/android_engine_test/test_driver/flutter_rendered_blue_rectangle_main_test.dart +++ b/dev/integration_tests/android_engine_test/test_driver/flutter_rendered_blue_rectangle_main_test.dart @@ -9,16 +9,26 @@ import 'package:test/test.dart'; import '_luci_skia_gold_prelude.dart'; +/// For local debugging, a (local) golden-file is required as a baseline: +/// +/// ```sh +/// # Checkout HEAD, i.e. *before* changes you want to test. +/// UPDATE_GOLDENS=1 flutter drive lib/flutter_rendered_blue_rectangle_main.dart +/// +/// # Make your changes. +/// +/// # Run the test against baseline. +/// flutter drive lib/flutter_rendered_blue_rectangle_main.dart +/// ``` +/// +/// For a convenient way to deflake a test, see `tool/deflake.dart`. void main() async { - // To test the golden file generation locally, comment out the following line. - // autoUpdateGoldenFiles = true; - late final FlutterDriver flutterDriver; late final NativeDriver nativeDriver; setUpAll(() async { if (isLuci) { - await enableSkiaGoldComparator(); + await enableSkiaGoldComparator(namePrefix: 'android_engine_test'); } flutterDriver = await FlutterDriver.connect(); nativeDriver = await AndroidNativeDriver.connect(flutterDriver); @@ -31,10 +41,9 @@ void main() async { }); test('should screenshot and match a full-screen blue rectangle', () async { - await flutterDriver.waitFor(find.byType('DecoratedBox')); await expectLater( nativeDriver.screenshot(), - matchesGoldenFile('fluttered_rendered_blue_rectangle.android.png'), + matchesGoldenFile('fluttered_rendered_blue_rectangle.png'), ); }, timeout: Timeout.none); } diff --git a/dev/integration_tests/android_engine_test/test_driver/platform_view/hybrid_composition_platform_view_main_test.dart b/dev/integration_tests/android_engine_test/test_driver/platform_view/hybrid_composition_platform_view_main_test.dart new file mode 100644 index 0000000000..63705b6ca5 --- /dev/null +++ b/dev/integration_tests/android_engine_test/test_driver/platform_view/hybrid_composition_platform_view_main_test.dart @@ -0,0 +1,66 @@ +// Copyright 2014 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'package:android_driver_extensions/native_driver.dart'; +import 'package:android_driver_extensions/skia_gold.dart'; +import 'package:flutter_driver/flutter_driver.dart'; +import 'package:test/test.dart'; + +import '../_luci_skia_gold_prelude.dart'; + +/// For local debugging, a (local) golden-file is required as a baseline: +/// +/// ```sh +/// # Checkout HEAD, i.e. *before* changes you want to test. +/// UPDATE_GOLDENS=1 flutter drive lib/platform_view/hybrid_compoisition_platform_view_main.dart +/// +/// # Make your changes. +/// +/// # Run the test against baseline. +/// flutter drive lib/platform_view/hybrid_compoisition_platform_view_main.dart +/// ``` +/// +/// For a convenient way to deflake a test, see `tool/deflake.dart`. +void main() async { + const String goldenPrefix = 'hybrid_composition_platform_view'; + + late final FlutterDriver flutterDriver; + late final NativeDriver nativeDriver; + + setUpAll(() async { + if (isLuci) { + await enableSkiaGoldComparator(namePrefix: 'android_engine_test'); + } + flutterDriver = await FlutterDriver.connect(); + nativeDriver = await AndroidNativeDriver.connect(flutterDriver); + await nativeDriver.configureForScreenshotTesting(); + await flutterDriver.waitUntilFirstFrameRasterized(); + }); + + tearDownAll(() async { + await nativeDriver.close(); + await flutterDriver.close(); + }); + + test('should screenshot and match a blue -> orange gradient', () async { + await expectLater( + nativeDriver.screenshot(), + matchesGoldenFile('$goldenPrefix.blue_orange_gradient_portrait.png'), + ); + }, timeout: Timeout.none); + + test('should rotate landscape and screenshot the gradient', () async { + await nativeDriver.rotateToLandscape(); + await expectLater( + nativeDriver.screenshot(), + matchesGoldenFile('$goldenPrefix.blue_orange_gradient_landscape_rotated.png'), + ); + + await nativeDriver.rotateResetDefault(); + await expectLater( + nativeDriver.screenshot(), + matchesGoldenFile('$goldenPrefix.blue_orange_gradient_portait_rotated_back.png'), + ); + }, timeout: Timeout.none); +} diff --git a/dev/integration_tests/android_engine_test/test_driver/platform_view/texture_layer_hybrid_composition_platform_view_main_test.dart b/dev/integration_tests/android_engine_test/test_driver/platform_view/texture_layer_hybrid_composition_platform_view_main_test.dart index feb47acd08..9e5a67cc6d 100644 --- a/dev/integration_tests/android_engine_test/test_driver/platform_view/texture_layer_hybrid_composition_platform_view_main_test.dart +++ b/dev/integration_tests/android_engine_test/test_driver/platform_view/texture_layer_hybrid_composition_platform_view_main_test.dart @@ -9,10 +9,20 @@ import 'package:test/test.dart'; import '../_luci_skia_gold_prelude.dart'; +/// For local debugging, a (local) golden-file is required as a baseline: +/// +/// ```sh +/// # Checkout HEAD, i.e. *before* changes you want to test. +/// UPDATE_GOLDENS=1 flutter drive lib/platform_view/texture_layer_hybrid_composition_platform_view_main.dart +/// +/// # Make your changes. +/// +/// # Run the test against baseline. +/// flutter drive lib/platform_view/texture_layer_hybrid_composition_platform_view_main.dart +/// ``` +/// +/// For a convenient way to deflake a test, see `tool/deflake.dart`. void main() async { - // To test the golden file generation locally, comment out the following line. - // autoUpdateGoldenFiles = true; - const String goldenPrefix = 'texture_layer_hybrid_composition_platform_view'; late final FlutterDriver flutterDriver; @@ -20,7 +30,7 @@ void main() async { setUpAll(() async { if (isLuci) { - await enableSkiaGoldComparator(); + await enableSkiaGoldComparator(namePrefix: 'android_engine_test'); } flutterDriver = await FlutterDriver.connect(); nativeDriver = await AndroidNativeDriver.connect(flutterDriver); @@ -40,25 +50,23 @@ void main() async { }); test('should screenshot and match a blue -> orange gradient', () async { - await flutterDriver.waitFor(find.byType('AndroidView')); await expectLater( nativeDriver.screenshot(), - matchesGoldenFile('$goldenPrefix.blue_orange_gradient_portrait.android.png'), + matchesGoldenFile('$goldenPrefix.blue_orange_gradient_portrait.png'), ); }, timeout: Timeout.none); test('should rotate landscape and screenshot the gradient', () async { - await flutterDriver.waitFor(find.byType('AndroidView')); await nativeDriver.rotateToLandscape(); await expectLater( nativeDriver.screenshot(), - matchesGoldenFile('$goldenPrefix.blue_orange_gradient_landscape_rotated.android.png'), + matchesGoldenFile('$goldenPrefix.blue_orange_gradient_landscape_rotated.png'), ); await nativeDriver.rotateResetDefault(); await expectLater( nativeDriver.screenshot(), - matchesGoldenFile('$goldenPrefix.blue_orange_gradient_portait_rotated_back.android.png'), + matchesGoldenFile('$goldenPrefix.blue_orange_gradient_portait_rotated_back.png'), ); }, timeout: Timeout.none); } diff --git a/dev/integration_tests/android_engine_test/test_driver/platform_view/virtual_display_platform_view_main_test.dart b/dev/integration_tests/android_engine_test/test_driver/platform_view/virtual_display_platform_view_main_test.dart index f6b6a26ac1..26ea7bcb17 100644 --- a/dev/integration_tests/android_engine_test/test_driver/platform_view/virtual_display_platform_view_main_test.dart +++ b/dev/integration_tests/android_engine_test/test_driver/platform_view/virtual_display_platform_view_main_test.dart @@ -9,10 +9,20 @@ import 'package:test/test.dart'; import '../_luci_skia_gold_prelude.dart'; +/// For local debugging, a (local) golden-file is required as a baseline: +/// +/// ```sh +/// # Checkout HEAD, i.e. *before* changes you want to test. +/// UPDATE_GOLDENS=1 flutter drive lib/platform_view/virtual_display_platform_view_main.dart +/// +/// # Make your changes. +/// +/// # Run the test against baseline. +/// flutter drive lib/platform_view/virtual_display_platform_view_main.dart +/// ``` +/// +/// For a convenient way to deflake a test, see `tool/deflake.dart`. void main() async { - // To test the golden file generation locally, comment out the following line. - // autoUpdateGoldenFiles = true; - const String goldenPrefix = 'virtual_display_platform_view'; late final FlutterDriver flutterDriver; @@ -20,7 +30,7 @@ void main() async { setUpAll(() async { if (isLuci) { - await enableSkiaGoldComparator(); + await enableSkiaGoldComparator(namePrefix: 'android_engine_test'); } flutterDriver = await FlutterDriver.connect(); nativeDriver = await AndroidNativeDriver.connect(flutterDriver); @@ -40,25 +50,23 @@ void main() async { }); test('should screenshot and match a blue -> orange gradient', () async { - await flutterDriver.waitFor(find.byType('AndroidView')); await expectLater( nativeDriver.screenshot(), - matchesGoldenFile('$goldenPrefix.blue_orange_gradient_portrait.android.png'), + matchesGoldenFile('$goldenPrefix.blue_orange_gradient_portrait.png'), ); }, timeout: Timeout.none); test('should rotate landscape and screenshot the gradient', () async { - await flutterDriver.waitFor(find.byType('AndroidView')); await nativeDriver.rotateToLandscape(); await expectLater( nativeDriver.screenshot(), - matchesGoldenFile('$goldenPrefix.blue_orange_gradient_landscape_rotated.android.png'), + matchesGoldenFile('$goldenPrefix.blue_orange_gradient_landscape_rotated.png'), ); await nativeDriver.rotateResetDefault(); await expectLater( nativeDriver.screenshot(), - matchesGoldenFile('$goldenPrefix.blue_orange_gradient_portait_rotated_back.android.png'), + matchesGoldenFile('$goldenPrefix.blue_orange_gradient_portait_rotated_back.png'), ); }, timeout: Timeout.none); } diff --git a/dev/integration_tests/android_engine_test/test_driver/platform_view_tap_color_change_main_test.dart b/dev/integration_tests/android_engine_test/test_driver/platform_view_tap_color_change_main_test.dart index 5bda56244e..8506f8619e 100644 --- a/dev/integration_tests/android_engine_test/test_driver/platform_view_tap_color_change_main_test.dart +++ b/dev/integration_tests/android_engine_test/test_driver/platform_view_tap_color_change_main_test.dart @@ -9,16 +9,26 @@ import 'package:test/test.dart'; import '_luci_skia_gold_prelude.dart'; +/// For local debugging, a (local) golden-file is required as a baseline: +/// +/// ```sh +/// # Checkout HEAD, i.e. *before* changes you want to test. +/// UPDATE_GOLDENS=1 flutter drive lib/platform_view_tap_color_change_main.dart +/// +/// # Make your changes. +/// +/// # Run the test against baseline. +/// flutter drive lib/platform_view_tap_color_change_main.dart +/// ``` +/// +/// For a convenient way to deflake a test, see `tool/deflake.dart`. void main() async { - // To test the golden file generation locally, comment out the following line. - // autoUpdateGoldenFiles = true; - late final FlutterDriver flutterDriver; late final NativeDriver nativeDriver; setUpAll(() async { if (isLuci) { - await enableSkiaGoldComparator(); + await enableSkiaGoldComparator(namePrefix: 'android_engine_test'); } flutterDriver = await FlutterDriver.connect(); nativeDriver = await AndroidNativeDriver.connect(flutterDriver); @@ -31,16 +41,15 @@ void main() async { }); test('should screenshot a rectangle that becomes blue after a tap', () async { - await flutterDriver.waitFor(find.byType('AndroidView')); await expectLater( nativeDriver.screenshot(), - matchesGoldenFile('platform_view_tap_color_change_initial.android.png'), + matchesGoldenFile('platform_view_tap_color_change_initial.png'), ); await nativeDriver.tap(const ByNativeAccessibilityLabel('Change color')); await expectLater( nativeDriver.screenshot(), - matchesGoldenFile('platform_view_tap_color_change_tapped.android.png'), + matchesGoldenFile('platform_view_tap_color_change_tapped.png'), ); }, timeout: Timeout.none); } diff --git a/dev/integration_tests/android_engine_test/tool/deflake.dart b/dev/integration_tests/android_engine_test/tool/deflake.dart new file mode 100644 index 0000000000..4c31f6055c --- /dev/null +++ b/dev/integration_tests/android_engine_test/tool/deflake.dart @@ -0,0 +1,163 @@ +// Copyright 2014 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'dart:convert'; +import 'dart:io' as io; + +import 'package:args/args.dart'; +import 'package:path/path.dart' as p; + +final ArgParser _argParser = + ArgParser() + ..addFlag('help', abbr: 'h', help: 'Display usage information.', negatable: false) + ..addFlag('verbose', abbr: 'v', help: 'Show noisy output while running', negatable: false) + ..addFlag( + 'generate-initial-golden', + help: + 'Whether an initial run (not part of "runs") should generate the ' + 'base golden file. If false, it is assumed the golden file wasl already generated.', + defaultsTo: true, + ) + ..addFlag( + 'build-app-once', + help: + 'Whether to use flutter build and --use-application-binary instead of rebuilding every iteration.', + defaultsTo: true, + ) + ..addOption('runs', abbr: 'n', help: 'How many times to run the test.', defaultsTo: '10'); + +/// Builds, establishes a baseline, and runs a golden-file test N number of times. +/// +/// Example use: +/// ```sh +/// dart ./tool/deflake.dart lib/external_texture/surface_texture_smiley_face_main.dart +/// ``` +/// +/// By default it will: +/// - Build the app once (and reuse the APK); +/// - Generate a baseline (local) golden-file, overwriting your local file system; +/// - Run N (by default, 10) subsequent tests, asserting the generated golden exactly matches. +/// +/// For advanced usage, see `dart ./tool/deflake.dart --help`. +void main(List args) async { + final ArgResults argResults = _argParser.parse(args); + if (argResults.flag('help')) { + return _printUsage(); + } + + final List testFiles = argResults.rest; + if (testFiles.length != 1) { + io.stderr.writeln('Exactly one test-file must be specified'); + _printUsage(); + io.exitCode = 1; + return; + } + + final io.File testFile = io.File(testFiles.single); + if (!testFile.existsSync()) { + io.stderr.writeln('Not a file: ${testFile.path}'); + _printUsage(); + io.exitCode = 1; + return; + } + + final bool generateInitialGolden = argResults.flag('generate-initial-golden'); + final bool buildAppOnce = argResults.flag('build-app-once'); + final bool verbose = argResults.flag('verbose'); + final int runs; + { + final String rawRuns = argResults.option('runs')!; + final int? parsedRuns = int.tryParse(rawRuns); + if (parsedRuns == null || parsedRuns < 1) { + io.stderr.writeln('--runs must be a positive number: "$rawRuns".'); + io.exitCode = 1; + return; + } + runs = parsedRuns; + } + + final List driverArgs; + if (buildAppOnce) { + io.stderr.writeln('Building initial app with "flutter build apk --debug...'); + final io.Process proccess = await io.Process.start('flutter', [ + 'build', + 'apk', + '--debug', + testFile.path, + ], mode: verbose ? io.ProcessStartMode.inheritStdio : io.ProcessStartMode.normal); + if (await proccess.exitCode case final int exitCode when exitCode != 0) { + io.stderr.writeln('Failed to build (exit code = $exitCode).'); + io.stderr.writeln(_collectStdOut(proccess)); + io.exitCode = 1; + return; + } + + // Strictly speaking, it would be better to parse stdout for: + // "✓ Built build/app/outputs/flutter-apk/app-debug.apk" + // + // ... _or_ specify the expected out ourselves and rely on that. + driverArgs = [ + 'drive', + '--use-application-binary', + p.join('build', 'app', 'outputs', 'flutter-apk', 'app-debug.apk'), + testFile.path, + ]; + } else { + // I can't imagine wanting to do this, but here is the option anyway! + driverArgs = ['drive', testFile.path]; + } + + Future runDriverTest({Map? environment}) async { + final io.Process proccess = await io.Process.start( + 'flutter', + driverArgs, + mode: verbose ? io.ProcessStartMode.inheritStdio : io.ProcessStartMode.normal, + environment: environment, + ); + if (await proccess.exitCode case final int exitCode when exitCode != 0) { + io.stderr.writeln('Failed to build (exit code = $exitCode).'); + io.stderr.writeln(_collectStdOut(proccess)); + return false; + } + return true; + } + + // Do an initial baseline run. + if (generateInitialGolden) { + io.stderr.writeln('Generating a baseline set of golden-files...'); + await runDriverTest(environment: {'UPDATE_GOLDENS': '1'}); + } + + // Now run. + int totalFailed = 0; + for (int i = 0; i < runs; i++) { + io.stderr.writeln('RUN ${i + 1} of $runs'); + final bool result = await runDriverTest(); + if (!result) { + totalFailed++; + io.stderr.writeln('FAIL'); + } else { + io.stderr.writeln('PASS'); + } + } + + io.stderr.writeln('PASSED: ${runs - totalFailed} / $runs'); + if (totalFailed != 0) { + io.exitCode = 1; + } +} + +void _printUsage() { + io.stdout.writeln('Usage: dart tool/deflake.dart lib/.dart'); + io.stdout.writeln(_argParser.usage); +} + +Future _collectStdOut(io.Process process) async { + final StringBuffer buffer = StringBuffer(); + buffer.writeln('stdout:'); + buffer.writeln(await utf8.decodeStream(process.stdout)); + buffer.writeln('stderr:'); + buffer.writeln(await utf8.decodeStream(process.stderr)); + return buffer.toString(); +} diff --git a/dev/tools/android_driver_extensions/lib/skia_gold.dart b/dev/tools/android_driver_extensions/lib/skia_gold.dart index e2f065f75a..aa1c49d8be 100644 --- a/dev/tools/android_driver_extensions/lib/skia_gold.dart +++ b/dev/tools/android_driver_extensions/lib/skia_gold.dart @@ -46,6 +46,12 @@ Future enableSkiaGoldComparator({String? namePrefix}) async { 'Set it to use Skia Gold.', ); } + if (namePrefix != null) { + assert( + !namePrefix.endsWith('.'), + 'The namePrefix automatically has a suffix of ".", so remove the last character from "$namePrefix".', + ); + } final io.Directory tmpDir = io.Directory.systemTemp.createTempSync('android_driver_test'); final bool isPresubmit = io.Platform.environment.containsKey(_kGoldctlPresubmitKey); io.stderr.writeln( @@ -126,12 +132,6 @@ final class _GoldenFileComparator extends GoldenFileComparator { 'Golden files in the Flutter framework must end with the file extension ' '.png.', ); - return Uri.parse( - [ - if (namePrefix != null) namePrefix!, - baseDir.pathSegments[baseDir.pathSegments.length - 2], - golden.toString(), - ].join('.'), - ); + return Uri.parse([if (namePrefix != null) namePrefix!, golden.toString()].join('.')); } } diff --git a/dev/tools/android_driver_extensions/lib/src/goldens.dart b/dev/tools/android_driver_extensions/lib/src/goldens.dart index 85e477473d..180a78d7e0 100644 --- a/dev/tools/android_driver_extensions/lib/src/goldens.dart +++ b/dev/tools/android_driver_extensions/lib/src/goldens.dart @@ -16,7 +16,16 @@ part of '../native_driver.dart'; /// /// When this is `true`, [matchesGoldenFile] will always report a successful /// match, because the bytes being tested implicitly become the new golden. -bool autoUpdateGoldenFiles = false; +/// +/// Defaults to `true` if the environment variable `UPDATE_GOLDENS` is either +/// `true` or `1` (case insensitive). +bool autoUpdateGoldenFiles = () { + final String? updateGoldens = io.Platform.environment['UPDATE_GOLDENS']; + return switch (updateGoldens?.toLowerCase()) { + '1' || 'true' => true, + _ => false, + }; +}(); /// Compares pixels against those of a golden image file. /// @@ -94,7 +103,17 @@ final class NaiveLocalFileComparator extends GoldenFileComparator { try { goldenBytes = await goldenFile.readAsBytes(); } on io.PathNotFoundException { - throw TestFailure('Golden file not found: ${goldenFile.path}'); + throw TestFailure( + 'Golden file not found: ${path.relative(goldenFile.path)}.\n' + '\n' + 'For local development, you must establish a local baseline image before ' + 'running tests, otherwise the test will always fail. Use UPDATE_GOLDENS=1 ' + 'when running "flutter drive" to establish a baseline, and then subequent ' + '"flutter drive" instances will be tested against that (local) golden.\n' + '\n' + 'See the documentation at dev/tools/android_engine_test/README.md for ' + 'details.', + ); } if (goldenBytes.length != imageBytes.length) {