diff --git a/packages/flutter_goldens/test/flutter_goldens_test.dart b/packages/flutter_goldens/test/flutter_goldens_test.dart index bdcdd79830..e3ab17fdc7 100644 --- a/packages/flutter_goldens/test/flutter_goldens_test.dart +++ b/packages/flutter_goldens/test/flutter_goldens_test.dart @@ -62,6 +62,89 @@ void main() { ); }); + test('web HTML test', () async { + platform = FakePlatform( + environment: { + 'GOLDCTL': 'goldctl', + 'FLUTTER_ROOT': _kFlutterRoot, + 'FLUTTER_TEST_BROWSER': 'Chrome', + 'FLUTTER_WEB_RENDERER': 'html', + }, + operatingSystem: 'macos' + ); + skiaClient = SkiaGoldClient( + workDirectory, + fs: fs, + process: process, + platform: platform, + httpClient: fakeHttpClient, + ); + + final File goldenFile = fs.file('/workDirectory/temp/golden_file_test.png') + ..createSync(recursive: true); + + const RunInvocation goldctlInvocation = RunInvocation( + [ + 'goldctl', + 'imgtest', 'add', + '--work-dir', '/workDirectory/temp', + '--test-name', 'golden_file_test', + '--png-file', '/workDirectory/temp/golden_file_test.png', + '--passfail', + '--add-test-optional-key', 'image_matching_algorithm:fuzzy', + '--add-test-optional-key', 'fuzzy_max_different_pixels:20', + '--add-test-optional-key', 'fuzzy_pixel_delta_threshold:4', + ], + null, + ); + process.processResults[goldctlInvocation] = ProcessResult(123, 0, '', ''); + + expect( + await skiaClient.imgtestAdd('golden_file_test.png', goldenFile), + isTrue, + ); + }); + + test('web CanvasKit test', () async { + platform = FakePlatform( + environment: { + 'GOLDCTL': 'goldctl', + 'FLUTTER_ROOT': _kFlutterRoot, + 'FLUTTER_TEST_BROWSER': 'Chrome', + 'FLUTTER_WEB_RENDERER': 'canvaskit', + }, + operatingSystem: 'macos' + ); + skiaClient = SkiaGoldClient( + workDirectory, + fs: fs, + process: process, + platform: platform, + httpClient: fakeHttpClient, + ); + + final File goldenFile = fs.file('/workDirectory/temp/golden_file_test.png') + ..createSync(recursive: true); + + const RunInvocation goldctlInvocation = RunInvocation( + [ + 'goldctl', + 'imgtest', 'add', + '--work-dir', '/workDirectory/temp', + '--test-name', 'golden_file_test', + '--png-file', '/workDirectory/temp/golden_file_test.png', + '--passfail', + ], + null, + ); + process.processResults[goldctlInvocation] = ProcessResult(123, 0, '', ''); + + expect( + await skiaClient.imgtestAdd('golden_file_test.png', goldenFile), + isTrue, + ); + }); + test('auth performs minimal work if already authorized', () async { final File authFile = fs.file('/workDirectory/temp/auth_opt.json') ..createSync(recursive: true); diff --git a/packages/flutter_goldens_client/lib/skia_client.dart b/packages/flutter_goldens_client/lib/skia_client.dart index d79f220a15..1539911829 100644 --- a/packages/flutter_goldens_client/lib/skia_client.dart +++ b/packages/flutter_goldens_client/lib/skia_client.dart @@ -192,6 +192,7 @@ class SkiaGoldClient { '--test-name', cleanTestName(testName), '--png-file', goldenFile.path, '--passfail', + ..._getPixelMatchingArguments(), ]; final io.ProcessResult result = await process.run(imgtestCommand); @@ -303,6 +304,7 @@ class SkiaGoldClient { .path, '--test-name', cleanTestName(testName), '--png-file', goldenFile.path, + ..._getPixelMatchingArguments(), ]; final io.ProcessResult result = await process.run(imgtestCommand); @@ -323,6 +325,51 @@ class SkiaGoldClient { } } + // Constructs arguments for `goldctl` for controlling how pixels are compared. + // + // For AOT and CanvasKit exact pixel matching is used. For the HTML renderer + // on the web a fuzzy matching algorithm is used that allows very small deltas + // because Chromium cannot exactly reproduce the same golden on all computers. + // It seems to depend on the hardware/OS/driver combination. However, those + // differences are very small (typically not noticeable to human eye). + List _getPixelMatchingArguments() { + // Only use fuzzy pixel matching in the HTML renderer. + if (!_isBrowserTest || _isBrowserCanvasKitTest) { + return const []; + } + + // The algorithm to be used when matching images. The available options are: + // - "fuzzy": Allows for customizing the thresholds of pixel differences. + // - "sobel": Same as "fuzzy" but performs edge detection before performing + // a fuzzy match. + const String algorithm = 'fuzzy'; + + // The number of pixels in this image that are allowed to differ from the + // baseline. + // + // The chosen number - 20 - is arbitrary. Even for a small golden file, say + // 50 x 50, it would be less than 1% of the total number of pixels. This + // number should not grow too much. If it's growing, it is probably due to a + // larger issue that needs to be addressed at the infra level. + const int maxDifferentPixels = 20; + + // The maximum acceptable difference per pixel. + // + // Uses the Manhattan distance using the RGBA color components as + // coordinates. The chosen number - 4 - is arbitrary. It's small enough to + // both not be noticeable and not trigger test flakes due to sub-pixel + // golden deltas. This number should not grow too much. If it's growing, it + // is probably due to a larger issue that needs to be addressed at the infra + // level. + const int pixelDeltaThreshold = 4; + + return [ + '--add-test-optional-key', 'image_matching_algorithm:$algorithm', + '--add-test-optional-key', 'fuzzy_max_different_pixels:$maxDifferentPixels', + '--add-test-optional-key', 'fuzzy_pixel_delta_threshold:$pixelDeltaThreshold', + ]; + } + /// Returns the latest positive digest for the given test known to Flutter /// Gold at head. Future getExpectationForTest(String testName) async { @@ -405,10 +452,10 @@ class SkiaGoldClient { 'Platform' : platform.operatingSystem, 'CI' : 'luci', }; - if (platform.environment[_kTestBrowserKey] != null) { - keys['Browser'] = platform.environment[_kTestBrowserKey]; + if (_isBrowserTest) { + keys['Browser'] = _browserKey; keys['Platform'] = '${keys['Platform']}-browser'; - if (platform.environment[_kWebRendererKey] == 'canvaskit') { + if (_isBrowserCanvasKitTest) { keys['WebRenderer'] = 'canvaskit'; } } @@ -451,14 +498,27 @@ class SkiaGoldClient { ]; } + bool get _isBrowserTest { + return platform.environment[_kTestBrowserKey] != null; + } + + bool get _isBrowserCanvasKitTest { + return _isBrowserTest && platform.environment[_kWebRendererKey] == 'canvaskit'; + } + + String get _browserKey { + assert(_isBrowserTest); + return platform.environment[_kTestBrowserKey]!; + } + /// Returns a trace id based on the current testing environment to lookup /// the latest positive digest on Flutter Gold with a hex-encoded md5 hash of /// the image keys. String getTraceID(String testName) { final Map keys = { - if (platform.environment[_kTestBrowserKey] != null) - 'Browser' : platform.environment[_kTestBrowserKey], - if (platform.environment[_kTestBrowserKey] != null && platform.environment[_kWebRendererKey] == 'canvaskit') + if (_isBrowserTest) + 'Browser' : _browserKey, + if (_isBrowserCanvasKitTest) 'WebRenderer' : 'canvaskit', 'CI' : 'luci', 'Platform' : platform.operatingSystem,