diff --git a/packages/flutter_tools/bin/fuchsia_tester.dart b/packages/flutter_tools/bin/fuchsia_tester.dart index 697a66591a..7c4ebc74ce 100644 --- a/packages/flutter_tools/bin/fuchsia_tester.dart +++ b/packages/flutter_tools/bin/fuchsia_tester.dart @@ -151,19 +151,19 @@ Future run(List args) async { } // TODO(dnfield): This should be injected. + final BuildInfo buildInfo = BuildInfo( + BuildMode.debug, + '', + treeShakeIcons: false, + packageConfigPath: globals.fs.path.normalize( + globals.fs.path.absolute(argResults[_kOptionPackages] as String), + ), + ); exitCode = await const FlutterTestRunner().runTests( const TestWrapper(), tests.keys.map(Uri.file).toList(), - debuggingOptions: DebuggingOptions.enabled( - BuildInfo( - BuildMode.debug, - '', - treeShakeIcons: false, - packageConfigPath: globals.fs.path.normalize( - globals.fs.path.absolute(argResults[_kOptionPackages] as String), - ), - ), - ), + debuggingOptions: DebuggingOptions.enabled(buildInfo), + buildInfo: buildInfo, watcher: collector, enableVmService: collector != null, precompiledDillFiles: tests, diff --git a/packages/flutter_tools/lib/src/device.dart b/packages/flutter_tools/lib/src/device.dart index e62c7dbac2..0a5052a831 100644 --- a/packages/flutter_tools/lib/src/device.dart +++ b/packages/flutter_tools/lib/src/device.dart @@ -1008,7 +1008,7 @@ class DebuggingOptions { startPaused = false, dartFlags = '', disableServiceAuthCodes = false, - enableDds = true, + enableDds = false, cacheStartupProfile = false, enableSoftwareRendering = false, skiaDeterministicRendering = false, diff --git a/packages/flutter_tools/lib/src/test/flutter_platform.dart b/packages/flutter_tools/lib/src/test/flutter_platform.dart index d415b42f34..f4b069770e 100644 --- a/packages/flutter_tools/lib/src/test/flutter_platform.dart +++ b/packages/flutter_tools/lib/src/test/flutter_platform.dart @@ -3,16 +3,20 @@ // found in the LICENSE file. import 'dart:async'; +import 'dart:typed_data'; import 'package:meta/meta.dart'; import 'package:package_config/package_config.dart'; +import 'package:process/process.dart'; import 'package:stream_channel/stream_channel.dart'; import 'package:test_core/src/platform.dart'; // ignore: implementation_imports +import 'package:vm_service/vm_service.dart'; import '../base/async_guard.dart'; import '../base/common.dart'; import '../base/file_system.dart'; import '../base/io.dart'; +import '../base/logger.dart'; import '../base/process.dart'; import '../build_info.dart'; import '../cache.dart'; @@ -32,6 +36,7 @@ import 'integration_test_device.dart'; import 'test_compiler.dart'; import 'test_config.dart'; import 'test_device.dart'; +import 'test_golden_comparator.dart'; import 'test_time_recorder.dart'; import 'watcher.dart'; @@ -53,6 +58,10 @@ FlutterPlatform installHook({ TestWrapper testWrapper = const TestWrapper(), required String shellPath, required DebuggingOptions debuggingOptions, + required BuildInfo buildInfo, + required FileSystem fileSystem, + required Logger logger, + required ProcessManager processManager, TestWatcher? watcher, bool enableVmService = false, bool machine = false, @@ -69,7 +78,6 @@ FlutterPlatform installHook({ String? integrationTestUserIdentifier, TestTimeRecorder? testTimeRecorder, TestCompilerNativeAssetsBuilder? nativeAssetsBuilder, - BuildInfo? buildInfo, }) { assert( enableVmService || @@ -101,6 +109,9 @@ FlutterPlatform installHook({ testTimeRecorder: testTimeRecorder, nativeAssetsBuilder: nativeAssetsBuilder, buildInfo: buildInfo, + fileSystem: fileSystem, + logger: logger, + processManager: processManager, ); platformPluginRegistration(platform); return platform; @@ -289,6 +300,10 @@ class FlutterPlatform extends PlatformPlugin { FlutterPlatform({ required this.shellPath, required this.debuggingOptions, + required this.buildInfo, + required this.logger, + required FileSystem fileSystem, + required ProcessManager processManager, this.watcher, this.enableVmService, this.machine, @@ -304,9 +319,19 @@ class FlutterPlatform extends PlatformPlugin { this.integrationTestUserIdentifier, this.testTimeRecorder, this.nativeAssetsBuilder, - this.buildInfo, this.shutdownHooks, - }); + }) { + _testGoldenComparator = TestGoldenComparator( + flutterTesterBinPath: shellPath, + compilerFactory: + () => + compiler ?? + TestCompiler(buildInfo, flutterProject, testTimeRecorder: testTimeRecorder), + fileSystem: fileSystem, + logger: logger, + processManager: processManager, + ); + } final String shellPath; final DebuggingOptions debuggingOptions; @@ -323,7 +348,8 @@ class FlutterPlatform extends PlatformPlugin { final String? icudtlPath; final TestTimeRecorder? testTimeRecorder; final TestCompilerNativeAssetsBuilder? nativeAssetsBuilder; - final BuildInfo? buildInfo; + final BuildInfo buildInfo; + final Logger logger; final ShutdownHooks? shutdownHooks; /// The device to run the test on for Integration Tests. @@ -332,6 +358,7 @@ class FlutterPlatform extends PlatformPlugin { /// Tester; otherwise it will run as a Integration Test on this device. final Device? integrationTestDevice; bool get _isIntegrationTest => integrationTestDevice != null; + late final TestGoldenComparator _testGoldenComparator; final String? integrationTestUserIdentifier; @@ -375,7 +402,9 @@ class FlutterPlatform extends PlatformPlugin { return controller.suite; } - StreamChannel loadChannel(String path, SuitePlatform platform) { + /// Used as an implementation detail for [load]ing a test suite. + @visibleForTesting + StreamChannel loadChannel(String path, SuitePlatform platform) { if (_testCount > 0) { // Fail if there will be a port conflict. if (debuggingOptions.hostVmServicePort != null) { @@ -481,15 +510,64 @@ class FlutterPlatform extends PlatformPlugin { ); } - void _handleStartedDevice(Uri? uri, int testCount) { + void _handleStartedDevice({required Uri? uri, required int testCount, required String testPath}) { if (uri != null) { globals.printTrace('test $testCount: VM Service uri is available at $uri'); + if (_isIntegrationTest) { + _listenToVmServiceForGoldens(uri: uri, testPath: testPath); + } } else { globals.printTrace('test $testCount: VM Service uri is not available'); } watcher?.handleStartedDevice(uri); } + static const String _kEventName = 'integration_test.VmServiceProxyGoldenFileComparator'; + static const String _kExtension = 'ext.$_kEventName'; + + Future _listenToVmServiceForGoldens({required Uri uri, required String testPath}) async { + final Uri goldensBaseUri = Uri.parse(testPath); + final FlutterVmService vmService = await connectToVmService(uri, logger: logger); + final IsolateRef testAppIsolate = await vmService.findExtensionIsolate(_kExtension); + await vmService.service.streamListen(_kEventName); + vmService.service.onEvent(_kEventName).listen((Event e) async { + if (!const ['compare', 'update'].contains(e.extensionKind)) { + throw StateError('Unexpected command: "${e.extensionKind}".'); + } + + final Map? data = e.extensionData?.data; + if (data == null) { + throw StateError('Expected VM service data, but got null.'); + } + final int id = data['id']! as int; + final Uri relativePath = Uri.parse(data['path']! as String); + final Uint8List bytes = base64.decode(data['bytes']! as String); + + final Map args; + if (e.extensionKind == 'update') { + switch (await _testGoldenComparator.update(goldensBaseUri, bytes, relativePath)) { + case TestGoldenUpdateDone(): + args = {'result': true}; + case TestGoldenUpdateError(error: final String error): + args = {'error': error}; + } + } else { + switch (await _testGoldenComparator.compare(goldensBaseUri, bytes, relativePath)) { + case TestGoldenComparisonDone(matched: final bool matched): + args = {'result': matched}; + case TestGoldenComparisonError(error: final String error): + args = {'error': error}; + } + } + + await vmService.callMethodWrapper( + _kExtension, + isolateId: testAppIsolate.id, + args: {'id': id, ...args}, + ); + }); + } + Future<_AsyncError?> _startTest( String testPath, StreamChannel testHarnessChannel, @@ -625,7 +703,11 @@ class FlutterPlatform extends PlatformPlugin { // This future may depend on [_handleStartedDevice] having been called remoteChannelCompleter.future, testDevice.vmServiceUri.then((Uri? processVmServiceUri) { - _handleStartedDevice(processVmServiceUri, ourTestCount); + _handleStartedDevice( + uri: processVmServiceUri, + testCount: ourTestCount, + testPath: testPath, + ); }), ], // If [remoteChannelCompleter.future] errors, we may never get the @@ -727,8 +809,7 @@ class FlutterPlatform extends PlatformPlugin { testUrl: testUrl, testConfigFile: findTestConfigFile(globals.fs.file(testUrl), globals.logger), // This MUST be a file URI. - packageConfigUri: - buildInfo != null ? globals.fs.path.toUri(buildInfo!.packageConfigPath) : null, + packageConfigUri: globals.fs.path.toUri(buildInfo.packageConfigPath), host: host!, updateGoldens: updateGoldens!, flutterTestDep: packageConfig['flutter_test'] != null, diff --git a/packages/flutter_tools/lib/src/test/runner.dart b/packages/flutter_tools/lib/src/test/runner.dart index 6254b884e8..fdd93d3c4f 100644 --- a/packages/flutter_tools/lib/src/test/runner.dart +++ b/packages/flutter_tools/lib/src/test/runner.dart @@ -63,7 +63,7 @@ abstract class FlutterTestRunner { String? integrationTestUserIdentifier, TestTimeRecorder? testTimeRecorder, TestCompilerNativeAssetsBuilder? nativeAssetsBuilder, - BuildInfo? buildInfo, + required BuildInfo buildInfo, }); /// Runs tests using the experimental strategy of spawning each test in a @@ -130,7 +130,7 @@ class _FlutterTestRunnerImpl implements FlutterTestRunner { String? integrationTestUserIdentifier, TestTimeRecorder? testTimeRecorder, TestCompilerNativeAssetsBuilder? nativeAssetsBuilder, - BuildInfo? buildInfo, + required BuildInfo buildInfo, }) async { // Configure package:test to use the Flutter engine for child processes. final String flutterTesterBinPath = globals.artifacts!.getArtifactPath(Artifact.flutterTester); @@ -236,6 +236,9 @@ class _FlutterTestRunnerImpl implements FlutterTestRunner { testTimeRecorder: testTimeRecorder, nativeAssetsBuilder: nativeAssetsBuilder, buildInfo: buildInfo, + fileSystem: globals.fs, + logger: globals.logger, + processManager: globals.processManager, ); try { diff --git a/packages/flutter_tools/test/general.shard/flutter_platform_test.dart b/packages/flutter_tools/test/general.shard/flutter_platform_test.dart index 9396d2642c..cd8b93ebea 100644 --- a/packages/flutter_tools/test/general.shard/flutter_platform_test.dart +++ b/packages/flutter_tools/test/general.shard/flutter_platform_test.dart @@ -2,8 +2,12 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. +import 'dart:async'; +import 'dart:io' as io; + import 'package:file/memory.dart'; import 'package:flutter_tools/src/application_package.dart'; +import 'package:flutter_tools/src/artifacts.dart'; import 'package:flutter_tools/src/base/file_system.dart'; import 'package:flutter_tools/src/base/io.dart'; import 'package:flutter_tools/src/base/logger.dart'; @@ -14,8 +18,12 @@ import 'package:flutter_tools/src/flutter_manifest.dart'; import 'package:flutter_tools/src/globals.dart' as globals; import 'package:flutter_tools/src/project.dart'; import 'package:flutter_tools/src/test/flutter_platform.dart'; +import 'package:flutter_tools/src/test/test_compiler.dart'; +import 'package:flutter_tools/src/vmservice.dart'; +import 'package:stream_channel/stream_channel.dart'; import 'package:test/fake.dart'; import 'package:test_core/backend.dart'; +import 'package:vm_service/src/vm_service.dart'; import '../src/common.dart'; import '../src/context.dart'; @@ -44,6 +52,10 @@ void main() { shellPath: '/', debuggingOptions: DebuggingOptions.enabled(BuildInfo.debug, hostVmServicePort: 1234), enableVmService: false, + buildInfo: BuildInfo.debug, + fileSystem: fileSystem, + processManager: FakeProcessManager.empty(), + logger: BufferLogger.test(), ); flutterPlatform.loadChannel('test1.dart', fakeSuitePlatform); @@ -67,6 +79,10 @@ void main() { shellPath: '/', precompiledDillPath: 'example.dill', enableVmService: false, + buildInfo: BuildInfo.debug, + fileSystem: fileSystem, + processManager: FakeProcessManager.empty(), + logger: BufferLogger.test(), ); flutterPlatform.loadChannel('test1.dart', fakeSuitePlatform); @@ -93,6 +109,10 @@ void main() { flutterProject: _FakeFlutterProject(), host: InternetAddress.anyIPv4, updateGoldens: false, + buildInfo: BuildInfo.debug, + fileSystem: fileSystem, + processManager: FakeProcessManager.empty(), + logger: BufferLogger.test(), ); await expectLater( @@ -133,6 +153,10 @@ void main() { host: InternetAddress.anyIPv4, updateGoldens: false, shutdownHooks: shutdownHooks, + buildInfo: BuildInfo.debug, + fileSystem: fileSystem, + processManager: FakeProcessManager.empty(), + logger: BufferLogger.test(), ); await expectLater( @@ -156,6 +180,10 @@ void main() { () => installHook( shellPath: 'abc', debuggingOptions: DebuggingOptions.enabled(BuildInfo.debug, startPaused: true), + buildInfo: BuildInfo.debug, + fileSystem: fileSystem, + processManager: FakeProcessManager.empty(), + logger: BufferLogger.test(), ), throwsAssertionError, ); @@ -168,6 +196,10 @@ void main() { startPaused: true, hostVmServicePort: 123, ), + buildInfo: BuildInfo.debug, + fileSystem: fileSystem, + processManager: FakeProcessManager.empty(), + logger: BufferLogger.test(), ), throwsAssertionError, ); @@ -193,6 +225,10 @@ void main() { platformPluginRegistration: (FlutterPlatform platform) { capturedPlatform = platform; }, + buildInfo: BuildInfo.debug, + fileSystem: fileSystem, + processManager: FakeProcessManager.empty(), + logger: BufferLogger.test(), ); expect(identical(capturedPlatform, flutterPlatform), equals(true)); @@ -247,6 +283,254 @@ void main() { }); }); }); + + group('proxies goldenFileComparator using the VM service driver', () { + late SuitePlatform fakeSuitePlatform; + late MemoryFileSystem fileSystem; + late Artifacts artifacts; + late FlutterProject flutterProject; + late FakeProcessManager processManager; + late BufferLogger logger; + late TestCompiler testCompiler; + late _FakeFlutterVmService flutterVmService; + late Completer testCompleter; + + setUp(() { + fakeSuitePlatform = SuitePlatform(Runtime.vm); + fileSystem = MemoryFileSystem.test(); + fileSystem.file('.dart_tool/package_config.json') + ..createSync(recursive: true) + ..writeAsStringSync('{"configVersion":2,"packages":[]}'); + artifacts = Artifacts.test(fileSystem: fileSystem); + + flutterProject = FlutterProject.fromDirectoryTest(fileSystem.systemTempDirectory); + testCompleter = Completer(); + processManager = FakeProcessManager.empty(); + logger = BufferLogger.test(); + testCompiler = _FakeTestCompiler(); + flutterVmService = _FakeFlutterVmService(); + }); + + tearDown(() { + printOnFailure(logger.errorText); + }); + + void addFlutterTesterDeviceExpectation() { + processManager.addCommand( + FakeCommand( + command: const [ + 'flutter_tester', + '--disable-vm-service', + '--enable-checked-mode', + '--verify-entry-points', + '--enable-software-rendering', + '--skia-deterministic-rendering', + '--enable-dart-profiling', + '--non-interactive', + '--use-test-fonts', + '--disable-asset-fonts', + '--packages=.dart_tool/package_config.json', + '', + ], + exitCode: -9, + completer: testCompleter, + ), + ); + } + + testUsingContext( + 'should not listen in a non-integration test', + () async { + addFlutterTesterDeviceExpectation(); + + const Device? notAnIntegrationTest = null; + final FlutterPlatform flutterPlatform = FlutterPlatform( + debuggingOptions: DebuggingOptions.disabled(BuildInfo.debug), + shellPath: 'flutter_tester', + enableVmService: false, + // ignore: avoid_redundant_argument_values + integrationTestDevice: notAnIntegrationTest, + flutterProject: flutterProject, + host: InternetAddress.anyIPv4, + updateGoldens: false, + buildInfo: BuildInfo.debug, + fileSystem: fileSystem, + processManager: processManager, + logger: BufferLogger.test(), + ); + flutterPlatform.compiler = testCompiler; + + // Simulate the test immediately completing. + testCompleter.complete(); + + final StreamChannel channel = flutterPlatform.loadChannel( + 'test1.dart', + fakeSuitePlatform, + ); + + // Without draining, the sink will never complete. + unawaited(channel.stream.drain()); + + await expectLater(channel.sink.done, completes); + }, + overrides: { + FileSystem: () => fileSystem, + ProcessManager: () => processManager, + Logger: () => logger, + VMServiceConnector: () => (_) => throw UnimplementedError(), + }, + ); + + // This is not a complete test of all the possible cases supported by the + // golden-file integration, which is a complex multi-process implementation + // that lives across multiple packages and files. + // + // Instead, this is a unit-test based smoke test that the overall flow works + // as expected to give a quicker turn-around signal that either the test + // should be updated, or the process was broken; run an integration_test on + // an Android or iOS device or emulator/simulator that takes screenshots + // and compares them with matchesGoldenFile for a full e2e-test of the + // entire workflow. + testUsingContext( + 'should listen in an integration test', + () async { + processManager.addCommand( + const FakeCommand( + command: [ + 'flutter_tester', + '--disable-vm-service', + '--non-interactive', + '--packages=.dart_tool/package_config.json', + '', + ], + stdout: '{"success": true}\n', + ), + ); + addFlutterTesterDeviceExpectation(); + + final FlutterPlatform flutterPlatform = FlutterPlatform( + debuggingOptions: DebuggingOptions.disabled(BuildInfo.debug), + shellPath: 'flutter_tester', + enableVmService: false, + flutterProject: flutterProject, + integrationTestDevice: _WorkingDevice(), + host: InternetAddress.anyIPv4, + updateGoldens: false, + buildInfo: BuildInfo.debug, + fileSystem: fileSystem, + processManager: processManager, + logger: BufferLogger.test(), + ); + flutterPlatform.compiler = testCompiler; + + final StreamChannel channel = flutterPlatform.loadChannel( + 'test1.dart', + fakeSuitePlatform, + ); + + // Responds to update events. + flutterVmService.service.onExtensionEventController.add( + Event( + extensionData: ExtensionData.parse({ + 'id': 1, + 'path': 'foo', + 'bytes': '', + }), + extensionKind: 'update', + ), + ); + + // Wait for tiny async tasks to complete. + await pumpEventQueue(); + await flutterVmService.service.onExtensionEventController.close(); + + final (String event, String? isolateId, Map? data) = + flutterVmService.callMethodWrapperInvocation!; + expect(event, 'ext.integration_test.VmServiceProxyGoldenFileComparator'); + expect(isolateId, null); + expect(data, {'id': 1, 'result': true}); + + // Without draining, the sink will never complete. + unawaited(channel.stream.drain()); + + // Allow the test to finish. + testCompleter.complete(); + + await expectLater(channel.sink.done, completes); + }, + overrides: { + FileSystem: () => fileSystem, + ProcessManager: () => processManager, + Logger: () => logger, + VMServiceConnector: + () => + ( + Uri httpUri, { + ReloadSources? reloadSources, + Restart? restart, + CompileExpression? compileExpression, + GetSkSLMethod? getSkSLMethod, + FlutterProject? flutterProject, + PrintStructuredErrorLogMethod? printStructuredErrorLogMethod, + io.CompressionOptions? compression, + Device? device, + Logger? logger, + }) async => flutterVmService, + ApplicationPackageFactory: _FakeApplicationPackageFactory.new, + Artifacts: () => artifacts, + }, + ); + }); +} + +class _FakeFlutterVmService extends Fake implements FlutterVmService { + @override + Future findExtensionIsolate(String extensionName) async { + return IsolateRef(); + } + + @override + final _FakeVmService service = _FakeVmService(); + + (String, String?, Map?)? callMethodWrapperInvocation; + + @override + Future callMethodWrapper( + String method, { + String? isolateId, + Map? args, + }) async { + callMethodWrapperInvocation = (method, isolateId, args); + return Response(); + } +} + +class _FakeVmService extends Fake implements VmService { + String? lastStreamListenId; + + @override + Future streamListen(String streamId) async { + lastStreamListenId = streamId; + return Success(); + } + + final StreamController onExtensionEventController = StreamController(); + + @override + Stream get onExtensionEvent => const Stream.empty(); + + @override + Stream onEvent(String streamId) { + return onExtensionEventController.stream; + } + + @override + Future get onDone => onExtensionEventController.done; +} + +class _FakeTestCompiler extends Fake implements TestCompiler { + @override + Future compile(Uri mainDart) async => ''; } class _UnstartableDevice extends Fake implements Device { diff --git a/packages/integration_test/example/integration_test/matches_golden_test.dart b/packages/integration_test/example/integration_test/matches_golden_test.dart new file mode 100644 index 0000000000..72f0399da3 --- /dev/null +++ b/packages/integration_test/example/integration_test/matches_golden_test.dart @@ -0,0 +1,43 @@ +// Copyright 2014 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +@Tags(['reduced-test-set']) +library; + +import 'package:flutter/material.dart'; +import 'package:flutter_test/flutter_test.dart'; +import 'package:integration_test/integration_test.dart'; + +import 'package:integration_test_example/main.dart' as app; + +/// To run: +/// +/// ```sh +/// # Be in this directory +/// cd dev/packages/integration_test/example +/// +/// flutter test integration_test/matches_golden_test.dart +/// ``` +/// +/// To run on a particular device, see `flutter -d`. +void main() { + IntegrationTestWidgetsFlutterBinding.ensureInitialized(); + + // TODO(matanlurey): Make this automatic as part of the bootstrap. + VmServiceProxyGoldenFileComparator.useIfRunningOnDevice(); + + testWidgets('can use matchesGoldenFile with integration_test', (WidgetTester tester) async { + // Build our app and trigger a frame. + app.main(); + + // TODO(matanlurey): Is this necessary? + await tester.pumpAndSettle(); + + // Take a screenshot. + await expectLater( + find.byType(MaterialApp), + matchesGoldenFile('integration_test_matches_golden_file.png'), + ); + }); +} diff --git a/packages/integration_test/lib/integration_test.dart b/packages/integration_test/lib/integration_test.dart index af8a88b9a0..ea5e0e7210 100644 --- a/packages/integration_test/lib/integration_test.dart +++ b/packages/integration_test/lib/integration_test.dart @@ -24,6 +24,8 @@ import 'src/callback.dart' as driver_actions; import 'src/channel.dart'; import 'src/extension.dart'; +export 'src/vm_service_golden_client.dart'; + const String _success = 'success'; /// Whether results should be reported to the native side over the method diff --git a/packages/integration_test/lib/src/vm_service_golden_client.dart b/packages/integration_test/lib/src/vm_service_golden_client.dart new file mode 100644 index 0000000000..3c73f9c10d --- /dev/null +++ b/packages/integration_test/lib/src/vm_service_golden_client.dart @@ -0,0 +1,226 @@ +// Copyright 2014 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +// Examples can assume: +// import 'dart:developer' as dev; + +import 'dart:async'; +import 'dart:convert'; +import 'dart:developer' as dev; +import 'dart:io' as io; + +import 'package:flutter/foundation.dart'; +import 'package:flutter_test/flutter_test.dart'; +import 'package:meta/meta.dart'; + +/// Compares image pixels against a golden image file on the host system. +/// +/// This comparator will send a request, using the VM service protocol, to a +/// host script (i.e. the _driver_ script, running in a Dart VM on the host +/// desktop OS), which will then forward the comparison to a concrete +/// [GoldenFileComparator]. +/// +/// To use, run [useIfRunningOnDevice] in the `main()` of a test file or similar: +/// +/// ```dart +/// import 'package:integration_test/integration_test.dart'; +/// +/// void main() { +/// VmServiceProxyGoldenFileComparator.useIfRunningOnDevice(); +/// +/// // Actual tests and such below. +/// } +/// ``` +/// +/// When either [compare] or [update] is called, the following event is sent +/// with [dev.postEvent]: +/// +/// ```dart +/// dev.postEvent('compare' /* or 'update' */, { +/// 'id': 1001, // a valid unique integer, often incrementing; +/// 'path': 'path/to/image.png', // golden key created by matchesGoldenFile; +/// 'bytes': '...base64encoded', // base64 encoded bytes representing the current image. +/// }, stream: 'integration_test.VmServiceProxyGoldenFileComparator'); +/// ``` +/// +/// The comparator expects a response at the service extension +/// `ext.integration_test.VmServiceProxyGoldenFileComparator` that is either +/// of the following formats: +/// +/// ```dart +/// { +/// 'error': 'Description of why the operation failed' +/// } +/// ``` +/// +/// or: +/// +/// ```dart +/// { +/// 'result': true /* or possibly false, in the case of 'compare' calls */ +/// } +/// ``` +/// +/// See also: +/// +/// * [matchesGoldenFile], the function that invokes the comparator. +@experimental +final class VmServiceProxyGoldenFileComparator extends GoldenFileComparator { + VmServiceProxyGoldenFileComparator._() : _postEvent = dev.postEvent { + dev.registerExtension(_kServiceName, (_, Map parameters) { + return handleEvent(parameters); + }); + } + + /// Creates an instance of [VmServiceProxyGoldenFileComparator] for internal testing. + /// + /// @nodoc + @visibleForTesting + VmServiceProxyGoldenFileComparator.forTesting(this._postEvent); + + static bool get _isRunningOnHost { + if (kIsWeb) { + return false; + } + return !io.Platform.isAndroid && !io.Platform.isIOS; + } + + static void _assertNotRunningOnFuchsia() { + if (!kIsWeb && io.Platform.isFuchsia) { + throw UnsupportedError('Golden testing with integration_test does not support Fuchsia.'); + } + } + + /// Conditionally sets [goldenFileComparator] to [VmServiceProxyGoldenFileComparator]. + /// + /// If running on a non-mobile non-web platform (i.e. desktop), this method has no effect. + static void useIfRunningOnDevice() { + if (_isRunningOnHost) { + return; + } + _assertNotRunningOnFuchsia(); + goldenFileComparator = _kInstance; + } + + static final GoldenFileComparator _kInstance = VmServiceProxyGoldenFileComparator._(); + static const String _kServiceName = 'ext.$_kEventName'; + static const String _kEventName = 'integration_test.VmServiceProxyGoldenFileComparator'; + final void Function(String, Map, {String stream}) _postEvent; + + /// Handles the received method and parameters as an incoming event. + /// + /// Each event is treated as if it were received by the Dart developer + /// extension protocol; this method is public only to be able to write unit + /// tests that do not have to bring up and use a VM service. + /// + /// @nodoc + @visibleForTesting + Future handleEvent(Map parameters) async { + // Treat the method as the ID number of the pending request. + final String? methodIdString = parameters['id']; + if (methodIdString == null) { + return dev.ServiceExtensionResponse.error( + dev.ServiceExtensionResponse.extensionError, + 'Required parameter "id" not present in response.', + ); + } + final int? methodId = int.tryParse(methodIdString); + if (methodId == null) { + return dev.ServiceExtensionResponse.error( + dev.ServiceExtensionResponse.extensionError, + 'Required parameter "id" not a valid integer: "$methodIdString".', + ); + } + final Completer<_Result>? completer = _pendingRequests[methodId]; + if (completer == null) { + return dev.ServiceExtensionResponse.error( + dev.ServiceExtensionResponse.extensionError, + 'No pending request with method ID "$methodIdString".', + ); + } + assert(!completer.isCompleted, 'Can never occur, as the completer should be removed'); + final String? error = parameters['error']; + if (error != null) { + completer.complete(_Failure(error)); + return dev.ServiceExtensionResponse.result('{}'); + } + final String? result = parameters['result']; + if (result == null) { + return dev.ServiceExtensionResponse.error( + dev.ServiceExtensionResponse.invalidParams, + 'Required parameter "result" not present in response.', + ); + } + if (bool.tryParse(result) case final bool result) { + completer.complete(_Success(result)); + return dev.ServiceExtensionResponse.result('{}'); + } else { + return dev.ServiceExtensionResponse.error( + dev.ServiceExtensionResponse.invalidParams, + 'Required parameter "result" not a valid boolean: "$result".', + ); + } + } + + int _nextId = 0; + final Map> _pendingRequests = >{}; + + Future<_Result> _postAndWait( + Uint8List imageBytes, + Uri golden, { + required String operation, + }) async { + final int nextId = ++_nextId; + assert(!_pendingRequests.containsKey(nextId)); + + final Completer<_Result> completer = Completer<_Result>(); + _postEvent(operation, { + 'id': nextId, + 'path': '$golden', + 'bytes': base64.encode(imageBytes), + }, stream: _kEventName); + + _pendingRequests[nextId] = completer; + completer.future.whenComplete(() { + _pendingRequests.remove(nextId); + }); + return completer.future; + } + + @override + Future compare(Uint8List imageBytes, Uri golden) async { + return switch (await _postAndWait(imageBytes, golden, operation: 'compare')) { + _Success(:final bool result) => result, + _Failure(:final String error) => Future.error(error), + }; + } + + @override + Future update(Uri golden, Uint8List imageBytes) async { + final _Result result = await _postAndWait(imageBytes, golden, operation: 'update'); + if (result is _Failure) { + return Future.error(result.error); + } + } +} + +// These wrapper classes allow us to use a Completer to indicate both a failed +// response and a successful response, without making a call of completeError +// within handleEvent, which is difficult or impossible to use correctly because +// of the semantics of error zones. +// +// Of course, this is a private implementation detail, others are welcome to try +// an alternative approach that might simplify the code above, but it's probably +// not worth it. +sealed class _Result {} + +final class _Success implements _Result { + _Success(this.result); + final bool result; +} + +final class _Failure implements _Result { + _Failure(this.error); + final String error; +} diff --git a/packages/integration_test/test/vm_service_golden_client_test.dart b/packages/integration_test/test/vm_service_golden_client_test.dart new file mode 100644 index 0000000000..0dc5453f14 --- /dev/null +++ b/packages/integration_test/test/vm_service_golden_client_test.dart @@ -0,0 +1,297 @@ +// Copyright 2014 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'dart:convert'; +import 'dart:developer' as dev; +import 'dart:typed_data'; + +import 'package:flutter_test/flutter_test.dart'; +import 'package:integration_test/integration_test.dart' as integration_test; + +void main() { + setUp(() { + // Ensure that we reset to a throwing comparator by default. + goldenFileComparator = const _NullGoldenFileComparator(); + }); + + group('useIfRunningOnDevice', () { + test('is skipped on the web', () { + integration_test.VmServiceProxyGoldenFileComparator.useIfRunningOnDevice(); + expect(goldenFileComparator, isInstanceOf<_NullGoldenFileComparator>()); + }, testOn: 'js'); + + test('is skipped on desktop platforms', () { + integration_test.VmServiceProxyGoldenFileComparator.useIfRunningOnDevice(); + expect(goldenFileComparator, isInstanceOf<_NullGoldenFileComparator>()); + }, testOn: 'windows || mac-os || linux'); + + test('is set on mobile platforms', () { + integration_test.VmServiceProxyGoldenFileComparator.useIfRunningOnDevice(); + expect( + goldenFileComparator, + isInstanceOf(), + ); + }, testOn: 'ios || android'); + }); + + group('handleEvent', () { + late integration_test.VmServiceProxyGoldenFileComparator goldenFileComparator; + + setUp(() { + goldenFileComparator = integration_test.VmServiceProxyGoldenFileComparator.forTesting( + (String operation, Map params, {String stream = ''}) {}, + ); + }); + + test('"id" must be provided', () async { + final dev.ServiceExtensionResponse response = await goldenFileComparator.handleEvent( + {'result': 'true'}, + ); + expect(response.errorDetail, contains('Required parameter "id" not present in response')); + }); + + test('"id" must be an integer', () async { + final dev.ServiceExtensionResponse response = await goldenFileComparator.handleEvent( + {'id': 'not-an-integer', 'result': 'true'}, + ); + expect( + response.errorDetail, + stringContainsInOrder([ + 'Required parameter "id" not a valid integer', + 'not-an-integer', + ]), + ); + }); + + test('"id" must match a pending request (never occurred)', () async { + final dev.ServiceExtensionResponse response = await goldenFileComparator.handleEvent( + {'id': '12345', 'result': 'true'}, + ); + expect( + response.errorDetail, + stringContainsInOrder(['No pending request with method ID', '12345']), + ); + }); + + test('"id" must match a pending request (already occurred)', () async { + // This is based on an implementation detail of knowing how IDs are generated. + const int nextId = 1; + goldenFileComparator.update(Uri(path: 'some-file'), Uint8List(0)); + + dev.ServiceExtensionResponse response; + + response = await goldenFileComparator.handleEvent({ + 'id': '$nextId', + 'result': 'true', + }); + expect(response.errorDetail, isNull); + + response = await goldenFileComparator.handleEvent({ + 'id': '$nextId', + 'result': 'true', + }); + expect( + response.errorDetail, + stringContainsInOrder(['No pending request with method ID', '1']), + ); + }); + + test('requests that contain "error" completes it as an error', () async { + // This is based on an implementation detail of knowing how IDs are generated. + const int nextId = 1; + + expect( + goldenFileComparator.compare(Uint8List(0), Uri(path: 'some-file')), + throwsA(contains('We did a bad')), + ); + + final dev.ServiceExtensionResponse response = await goldenFileComparator.handleEvent( + {'id': '$nextId', 'error': 'We did a bad'}, + ); + expect(response.errorDetail, isNull); + expect(response.result, '{}'); + }); + + test('requests that do not contain "error" return an empty response', () async { + // This is based on an implementation detail of knowing how IDs are generated. + const int nextId = 1; + goldenFileComparator.update(Uri(path: 'some-file'), Uint8List(0)); + + final dev.ServiceExtensionResponse response = await goldenFileComparator.handleEvent( + {'id': '$nextId', 'result': 'true'}, + ); + expect(response.errorDetail, isNull); + expect(response.result, '{}'); + }); + + test('"result" must be provided if "error" is omitted', () async { + // This is based on an implementation detail of knowing how IDs are generated. + const int nextId = 1; + goldenFileComparator.update(Uri(path: 'some-file'), Uint8List(0)); + + final dev.ServiceExtensionResponse response = await goldenFileComparator.handleEvent( + {'id': '$nextId'}, + ); + expect(response.errorDetail, contains('Required parameter "result" not present in response')); + }); + + test('"result" must be a boolean', () async { + // This is based on an implementation detail of knowing how IDs are generated. + const int nextId = 1; + goldenFileComparator.update(Uri(path: 'some-file'), Uint8List(0)); + + final dev.ServiceExtensionResponse response = await goldenFileComparator.handleEvent( + {'id': '$nextId', 'result': 'not-a-boolean'}, + ); + expect( + response.errorDetail, + stringContainsInOrder([ + 'Required parameter "result" not a valid boolean', + 'not-a-boolean', + ]), + ); + }); + + group('compare', () { + late integration_test.VmServiceProxyGoldenFileComparator goldenFileComparator; + late List<(String, Map)> postedEvents; + + setUp(() { + postedEvents = <(String, Map)>[]; + goldenFileComparator = integration_test.VmServiceProxyGoldenFileComparator.forTesting(( + String operation, + Map params, { + String stream = '', + }) { + postedEvents.add((operation, params)); + }); + }); + + test('posts an event and returns true', () async { + // This is based on an implementation detail of knowing how IDs are generated. + const int nextId = 1; + + final Uint8List bytes = Uint8List.fromList([1, 2, 3, 4, 5]); + expect(goldenFileComparator.compare(bytes, Uri(path: 'golden-path')), completion(true)); + + await goldenFileComparator.handleEvent({'id': '$nextId', 'result': 'true'}); + + final (String event, Map params) = postedEvents.single; + expect(event, 'compare'); + expect(params, { + 'id': nextId, + 'path': 'golden-path', + 'bytes': base64.encode(bytes), + }); + }); + + test('posts an event and returns false', () async { + // This is based on an implementation detail of knowing how IDs are generated. + const int nextId = 1; + + final Uint8List bytes = Uint8List.fromList([1, 2, 3, 4, 5]); + expect(goldenFileComparator.compare(bytes, Uri(path: 'golden-path')), completion(false)); + + await goldenFileComparator.handleEvent({ + 'id': '$nextId', + 'result': 'false', + }); + + final (String event, Map params) = postedEvents.single; + expect(event, 'compare'); + expect(params, { + 'id': nextId, + 'path': 'golden-path', + 'bytes': base64.encode(bytes), + }); + }); + + test('posts an event and returns an error', () async { + // This is based on an implementation detail of knowing how IDs are generated. + const int nextId = 1; + + final Uint8List bytes = Uint8List.fromList([1, 2, 3, 4, 5]); + expect( + goldenFileComparator.compare(bytes, Uri(path: 'golden-path')), + throwsA(contains('We did a bad')), + ); + + await goldenFileComparator.handleEvent({ + 'id': '$nextId', + 'error': 'We did a bad', + }); + + final (String event, Map params) = postedEvents.single; + expect(event, 'compare'); + expect(params, { + 'id': nextId, + 'path': 'golden-path', + 'bytes': base64.encode(bytes), + }); + }); + }); + + group('update', () { + late integration_test.VmServiceProxyGoldenFileComparator goldenFileComparator; + late List<(String, Map)> postedEvents; + + setUp(() { + postedEvents = <(String, Map)>[]; + goldenFileComparator = integration_test.VmServiceProxyGoldenFileComparator.forTesting(( + String operation, + Map params, { + String stream = '', + }) { + postedEvents.add((operation, params)); + }); + }); + + test('posts an event and returns', () async { + // This is based on an implementation detail of knowing how IDs are generated. + const int nextId = 1; + + final Uint8List bytes = Uint8List.fromList([1, 2, 3, 4, 5]); + expect(goldenFileComparator.update(Uri(path: 'golden-path'), bytes), completes); + + await goldenFileComparator.handleEvent({'id': '$nextId', 'result': 'true'}); + + final (String event, Map params) = postedEvents.single; + expect(event, 'update'); + expect(params, { + 'id': nextId, + 'path': 'golden-path', + 'bytes': base64.encode(bytes), + }); + }); + + test('posts an event and returns an error', () async { + // This is based on an implementation detail of knowing how IDs are generated. + const int nextId = 1; + + final Uint8List bytes = Uint8List.fromList([1, 2, 3, 4, 5]); + expect( + goldenFileComparator.update(Uri(path: 'golden-path'), bytes), + throwsA(contains('We did a bad')), + ); + + await goldenFileComparator.handleEvent({ + 'id': '$nextId', + 'error': 'We did a bad', + }); + + final (String event, Map params) = postedEvents.single; + expect(event, 'update'); + expect(params, { + 'id': nextId, + 'path': 'golden-path', + 'bytes': base64.encode(bytes), + }); + }); + }); + }); +} + +final class _NullGoldenFileComparator with Fake implements GoldenFileComparator { + const _NullGoldenFileComparator(); +}