diff --git a/dev/devicelab/bin/tasks/module_test.dart b/dev/devicelab/bin/tasks/module_test.dart index a4d7219956..bfe24991d1 100644 --- a/dev/devicelab/bin/tasks/module_test.dart +++ b/dev/devicelab/bin/tasks/module_test.dart @@ -2,8 +2,11 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. +import 'dart:convert'; import 'dart:io'; +import 'dart:typed_data'; +import 'package:archive/archive.dart'; import 'package:flutter_devicelab/framework/apk_utils.dart'; import 'package:flutter_devicelab/framework/framework.dart'; import 'package:flutter_devicelab/framework/task_result.dart'; @@ -314,6 +317,24 @@ Future main() async { 'lib/armeabi-v7a/libflutter.so', ], await getFilesInApk(releaseHostApk)); + section('Check the NOTICE file is correct'); + + await inDirectory(hostApp, () async { + final File apkFile = File(releaseHostApk); + final Archive apk = ZipDecoder().decodeBytes(apkFile.readAsBytesSync()); + // Shouldn't be missing since we already checked it exists above. + final ArchiveFile noticesFile = apk.findFile('assets/flutter_assets/NOTICES.Z'); + + final Uint8List licenseData = noticesFile.content as Uint8List; + if (licenseData == null) { + return TaskResult.failure('Invalid license file.'); + } + final String licenseString = utf8.decode(gzip.decode(licenseData)); + if (!licenseString.contains('skia') || !licenseString.contains('Flutter Authors')) { + return TaskResult.failure('License content missing.'); + } + }); + section('Check release AndroidManifest.xml'); final String androidManifestRelease = await getAndroidManifest(debugHostApk); diff --git a/dev/devicelab/bin/tasks/module_test_ios.dart b/dev/devicelab/bin/tasks/module_test_ios.dart index f16de2a1d2..749f6e790c 100644 --- a/dev/devicelab/bin/tasks/module_test_ios.dart +++ b/dev/devicelab/bin/tasks/module_test_ios.dart @@ -2,7 +2,9 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. +import 'dart:convert'; import 'dart:io'; +import 'dart:typed_data'; import 'package:flutter_devicelab/framework/framework.dart'; import 'package:flutter_devicelab/framework/ios.dart'; @@ -212,6 +214,8 @@ Future main() async { final File objectiveCAnalyticsOutputFile = File(path.join(tempDir.path, 'analytics-objc.log')); final Directory objectiveCBuildDirectory = Directory(path.join(tempDir.path, 'build-objc')); + + section('Build iOS Objective-C host app'); await inDirectory(objectiveCHostApp, () async { await exec( 'pod', @@ -268,6 +272,28 @@ Future main() async { 'isolate_snapshot_data', )); + section('Check the NOTICE file is correct'); + + final String licenseFilePath = path.join( + objectiveCBuildDirectory.path, + 'Host.app', + 'Frameworks', + 'App.framework', + 'flutter_assets', + 'NOTICES.Z', + ); + checkFileExists(licenseFilePath); + + await inDirectory(objectiveCBuildDirectory, () async { + final Uint8List licenseData = File(licenseFilePath).readAsBytesSync(); + final String licenseString = utf8.decode(gzip.decode(licenseData)); + if (!licenseString.contains('skia') || !licenseString.contains('Flutter Authors')) { + return TaskResult.failure('License content missing'); + } + }); + + section('Check that the host build sends the correct analytics'); + final String objectiveCAnalyticsOutput = objectiveCAnalyticsOutputFile.readAsStringSync(); if (!objectiveCAnalyticsOutput.contains('cd24: ios') || !objectiveCAnalyticsOutput.contains('cd25: true') diff --git a/dev/devicelab/lib/framework/apk_utils.dart b/dev/devicelab/lib/framework/apk_utils.dart index 4c349043f8..45b22da807 100644 --- a/dev/devicelab/lib/framework/apk_utils.dart +++ b/dev/devicelab/lib/framework/apk_utils.dart @@ -13,7 +13,7 @@ final String platformLineSep = Platform.isWindows ? '\r\n' : '\n'; final List flutterAssets = [ 'assets/flutter_assets/AssetManifest.json', - 'assets/flutter_assets/NOTICES', + 'assets/flutter_assets/NOTICES.Z', 'assets/flutter_assets/fonts/MaterialIcons-Regular.otf', 'assets/flutter_assets/packages/cupertino_icons/assets/CupertinoIcons.ttf', ]; diff --git a/dev/devicelab/lib/tasks/perf_tests.dart b/dev/devicelab/lib/tasks/perf_tests.dart index 5467766afb..16d3911c69 100644 --- a/dev/devicelab/lib/tasks/perf_tests.dart +++ b/dev/devicelab/lib/tasks/perf_tests.dart @@ -1070,7 +1070,7 @@ class CompileTest { final _UnzipListEntry libflutter = fileToMetadata['lib/armeabi-v7a/libflutter.so']; final _UnzipListEntry libapp = fileToMetadata['lib/armeabi-v7a/libapp.so']; - final _UnzipListEntry license = fileToMetadata['assets/flutter_assets/NOTICES']; + final _UnzipListEntry license = fileToMetadata['assets/flutter_assets/NOTICES.Z']; return { 'libflutter_uncompressed_bytes': libflutter.uncompressedSize, diff --git a/dev/integration_tests/flutter_gallery/test/example_code_parser_test.dart b/dev/integration_tests/flutter_gallery/test/example_code_parser_test.dart index ef4ec9c5b6..b3ad8d74ae 100644 --- a/dev/integration_tests/flutter_gallery/test/example_code_parser_test.dart +++ b/dev/integration_tests/flutter_gallery/test/example_code_parser_test.dart @@ -46,7 +46,7 @@ class TestAssetBundle extends AssetBundle { } @override - Future loadString(String key, { bool cache = true }) async { + Future loadString(String key, { bool cache = true, bool unzip = false }) async { if (key == 'lib/gallery/example_code.dart') return testCodeFile; return ''; diff --git a/packages/flutter/lib/src/services/asset_bundle.dart b/packages/flutter/lib/src/services/asset_bundle.dart index 1a81fdd469..1820be29e5 100644 --- a/packages/flutter/lib/src/services/asset_bundle.dart +++ b/packages/flutter/lib/src/services/asset_bundle.dart @@ -64,7 +64,16 @@ abstract class AssetBundle { /// caller is going to be doing its own caching. (It might not be cached if /// it's set to true either, that depends on the asset bundle /// implementation.) - Future loadString(String key, { bool cache = true }) async { + /// + /// If the `unzip` argument is set to true, it would first unzip file at the + /// specified location before retrieving the string content. + Future loadString( + String key, + { + bool cache = true, + bool unzip = false, + } + ) async { final ByteData data = await load(key); // Note: data has a non-nullable type, but might be null when running with // weak checking, so we need to null check it anyway (and ignore the warning @@ -73,15 +82,26 @@ abstract class AssetBundle { throw FlutterError('Unable to load asset: $key'); // ignore: dead_code // 50 KB of data should take 2-3 ms to parse on a Moto G4, and about 400 μs // on a Pixel 4. - if (data.lengthInBytes < 50 * 1024) { - return utf8.decode(data.buffer.asUint8List()); + if (data.lengthInBytes < 50 * 1024 && !unzip) { + return _utf8Decode(data); } + // For strings larger than 50 KB, run the computation in an isolate to // avoid causing main thread jank. - return compute(_utf8decode, data, debugLabel: 'UTF8 decode for "$key"'); + return compute( + unzip ? _utf8ZipDecode : _utf8Decode, + data, + debugLabel: '${unzip ? "Unzip and ": ""}UTF8 decode for "$key"', + ); } - static String _utf8decode(ByteData data) { + static String _utf8ZipDecode(ByteData data) { + List bytes = data.buffer.asUint8List(); + bytes = gzip.decode(bytes); + return utf8.decode(bytes); + } + + static String _utf8Decode(ByteData data) { return utf8.decode(data.buffer.asUint8List()); } @@ -163,10 +183,10 @@ abstract class CachingAssetBundle extends AssetBundle { final Map> _structuredDataCache = >{}; @override - Future loadString(String key, { bool cache = true }) { + Future loadString(String key, { bool cache = true, bool unzip = false }) { if (cache) - return _stringCache.putIfAbsent(key, () => super.loadString(key)); - return super.loadString(key); + return _stringCache.putIfAbsent(key, () => super.loadString(key, unzip: unzip)); + return super.loadString(key, unzip: unzip); } /// Retrieve a string from the asset bundle, parse it with the given function, diff --git a/packages/flutter/lib/src/services/binding.dart b/packages/flutter/lib/src/services/binding.dart index 8992ba60e2..a4942970bd 100644 --- a/packages/flutter/lib/src/services/binding.dart +++ b/packages/flutter/lib/src/services/binding.dart @@ -104,7 +104,19 @@ mixin ServicesBinding on BindingBase, SchedulerBinding { // TODO(ianh): Remove this complexity once these bugs are fixed. final Completer rawLicenses = Completer(); scheduleTask(() async { - rawLicenses.complete(await rootBundle.loadString('NOTICES', cache: false)); + rawLicenses.complete( + await rootBundle.loadString( + // NOTICES for web isn't compressed since we don't have access to + // dart:io on the client side and it's already compressed between + // the server and client. + // + // The compressed version doesn't have a more common .gz extension + // because gradle for Android non-transparently manipulates .gz files. + kIsWeb ? 'NOTICES' : 'NOTICES.Z', + cache: false, + unzip: !kIsWeb, + ) + ); }, Priority.animation); await rawLicenses.future; final Completer> parsedLicenses = Completer>(); diff --git a/packages/flutter/test/services/asset_bundle_test.dart b/packages/flutter/test/services/asset_bundle_test.dart index c3517de2ce..b13ad178ec 100644 --- a/packages/flutter/test/services/asset_bundle_test.dart +++ b/packages/flutter/test/services/asset_bundle_test.dart @@ -3,6 +3,7 @@ // found in the LICENSE file. import 'dart:convert'; +import 'dart:io'; import 'dart:typed_data'; import 'package:flutter/foundation.dart'; @@ -15,10 +16,13 @@ class TestAssetBundle extends CachingAssetBundle { @override Future load(String key) async { + loadCallCount[key] = loadCallCount[key] ?? 0 + 1; if (key == 'AssetManifest.json') return ByteData.view(Uint8List.fromList(const Utf8Encoder().convert('{"one": ["one"]}')).buffer); - loadCallCount[key] = loadCallCount[key] ?? 0 + 1; + if (key == 'NOTICES.Z') + return ByteData.view(Uint8List.fromList(gzip.encode(utf8.encode('All your base are belong to us'))).buffer); + if (key == 'one') return ByteData(1)..setInt8(0, 49); throw FlutterError('key not found'); @@ -48,6 +52,26 @@ void main() { expect(loadException, isFlutterError); }); + test('Test loading zipped strings', () async { + final TestAssetBundle bundle = TestAssetBundle(); + + String assetString = await bundle.loadString('NOTICES.Z', unzip: true); + expect(assetString, equals('All your base are belong to us')); + + expect(bundle.loadCallCount['NOTICES.Z'], 1); + + assetString = await bundle.loadString('NOTICES.Z', unzip: true); + expect(assetString, equals('All your base are belong to us')); + + // Should have been cached and shouldn't retrieve and decode another time. + expect(bundle.loadCallCount['NOTICES.Z'], 1); + }, onPlatform: { + 'browser': const Skip( + 'Skip the NOTICES unzipping test because NOTICES are' + 'not zipped for the web' + ), + }); + test('AssetImage.obtainKey succeeds with ImageConfiguration.empty', () async { // This is a regression test for https://github.com/flutter/flutter/issues/12392 final AssetImage assetImage = AssetImage('one', bundle: TestAssetBundle()); diff --git a/packages/flutter/test/services/binding_test.dart b/packages/flutter/test/services/binding_test.dart index 513eb3828e..92ddc9fb05 100644 --- a/packages/flutter/test/services/binding_test.dart +++ b/packages/flutter/test/services/binding_test.dart @@ -2,6 +2,8 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. +import 'dart:convert'; +import 'dart:io'; import 'dart:typed_data'; import 'package:flutter/foundation.dart'; @@ -44,7 +46,10 @@ class TestBinding extends BindingBase with SchedulerBinding, ServicesBinding { BinaryMessenger createBinaryMessenger() { return super.createBinaryMessenger() ..setMockMessageHandler('flutter/assets', (ByteData? message) async { - if (const StringCodec().decodeMessage(message) == 'NOTICES') { + if (const StringCodec().decodeMessage(message) == 'NOTICES.Z' && !kIsWeb) { + return Uint8List.fromList(gzip.encode(utf8.encode(licenses))).buffer.asByteData(); + } + if (const StringCodec().decodeMessage(message) == 'NOTICES' && kIsWeb) { return const StringCodec().encodeMessage(licenses); } return null; diff --git a/packages/flutter/test/widgets/image_resolution_test.dart b/packages/flutter/test/widgets/image_resolution_test.dart index c46b5eed09..f4bec3347e 100644 --- a/packages/flutter/test/widgets/image_resolution_test.dart +++ b/packages/flutter/test/widgets/image_resolution_test.dart @@ -66,7 +66,7 @@ class TestAssetBundle extends CachingAssetBundle { } @override - Future loadString(String key, { bool cache = true }) { + Future loadString(String key, { bool cache = true, bool unzip = false }) { if (key == 'AssetManifest.json') return SynchronousFuture(manifest); return SynchronousFuture(''); diff --git a/packages/flutter_tools/bin/fuchsia_asset_builder.dart b/packages/flutter_tools/bin/fuchsia_asset_builder.dart index 033df4df0d..a242f9b63d 100644 --- a/packages/flutter_tools/bin/fuchsia_asset_builder.dart +++ b/packages/flutter_tools/bin/fuchsia_asset_builder.dart @@ -7,6 +7,7 @@ import 'package:flutter_tools/src/asset.dart' hide defaultManifestPath; import 'package:flutter_tools/src/base/context.dart'; import 'package:flutter_tools/src/base/file_system.dart' as libfs; import 'package:flutter_tools/src/base/io.dart'; +import 'package:flutter_tools/src/build_info.dart'; import 'package:flutter_tools/src/cache.dart'; import 'package:flutter_tools/src/context_runner.dart'; import 'package:flutter_tools/src/devfs.dart'; @@ -59,6 +60,7 @@ Future run(List args) async { manifestPath: argResults[_kOptionManifest] as String ?? defaultManifestPath, assetDirPath: assetDir, packagesPath: argResults[_kOptionPackages] as String, + targetPlatform: TargetPlatform.fuchsia_arm64 // This is not arch specific. ); if (assets == null) { diff --git a/packages/flutter_tools/lib/src/asset.dart b/packages/flutter_tools/lib/src/asset.dart index 5a9f4d46cb..83d051d56b 100644 --- a/packages/flutter_tools/lib/src/asset.dart +++ b/packages/flutter_tools/lib/src/asset.dart @@ -7,6 +7,7 @@ import 'package:package_config/package_config.dart'; import 'base/context.dart'; import 'base/file_system.dart'; +import 'base/io.dart'; import 'base/logger.dart'; import 'base/platform.dart'; import 'build_info.dart'; @@ -72,6 +73,7 @@ abstract class AssetBundle { String manifestPath = defaultManifestPath, String assetDirPath, @required String packagesPath, + TargetPlatform targetPlatform, }); } @@ -122,6 +124,13 @@ class ManifestAssetBundle implements AssetBundle { static const String _kAssetManifestJson = 'AssetManifest.json'; static const String _kNoticeFile = 'NOTICES'; + // Comically, this can't be name with the more common .gz file extension + // because when it's part of an AAR and brought into another APK via gradle, + // gradle individually traverses all the files of the AAR and unzips .gz + // files (b/37117906). A less common .Z extension still describes how the + // file is formatted if users want to manually inspect the application + // bundle and is recognized by default file handlers on OS such as macOS.˚ + static const String _kNoticeZippedFile = 'NOTICES.Z'; @override bool wasBuiltOnce() => _lastBuildTimestamp != null; @@ -160,6 +169,7 @@ class ManifestAssetBundle implements AssetBundle { String manifestPath = defaultManifestPath, String assetDirPath, @required String packagesPath, + TargetPlatform targetPlatform, }) async { assetDirPath ??= getAssetBuildDirectory(); FlutterProject flutterProject; @@ -320,7 +330,6 @@ class ManifestAssetBundle implements AssetBundle { final DevFSStringContent assetManifest = _createAssetManifest(assetVariants); final DevFSStringContent fontManifest = DevFSStringContent(json.encode(fonts)); final LicenseResult licenseResult = _licenseCollector.obtainLicenses(packageConfig); - final DevFSStringContent licenses = DevFSStringContent(licenseResult.combinedLicenses); additionalDependencies = licenseResult.dependencies; if (wildcardDirectories.isNotEmpty) { @@ -338,7 +347,27 @@ class ManifestAssetBundle implements AssetBundle { _setIfChanged(_kAssetManifestJson, assetManifest); _setIfChanged(kFontManifestJson, fontManifest); - _setIfChanged(_kNoticeFile, licenses); + if (targetPlatform == TargetPlatform.web_javascript) { + // Don't compress the NOTICES file on web since the client doesn't have + // dart:io to decompress it. + _setIfChanged(_kNoticeFile, DevFSStringContent(licenseResult.combinedLicenses)); + } else { + final List licenseBytes = utf8.encode(licenseResult.combinedLicenses); + if (entries[_kNoticeZippedFile] == null || + gzip.decode((entries[_kNoticeZippedFile] as DevFSByteContent).bytes) + != licenseBytes) { + entries[_kNoticeZippedFile] = DevFSByteContent( + ZLibEncoder( + // A zlib dictionary is a hinting string sequence with the most + // likely string occurrences at the end. This ends up just being + // common English words with domain specific words like copyright. + dictionary: utf8.encode('copyrightsoftwaretothisinandorofthe'), + gzip: true, + level: 9, + ).convert(licenseBytes) + ); + } + } return 0; } diff --git a/packages/flutter_tools/lib/src/base/io.dart b/packages/flutter_tools/lib/src/base/io.dart index cd60b5c7ce..246daf70f0 100644 --- a/packages/flutter_tools/lib/src/base/io.dart +++ b/packages/flutter_tools/lib/src/base/io.dart @@ -102,7 +102,8 @@ export 'dart:io' systemEncoding, WebSocket, WebSocketException, - WebSocketTransformer; + WebSocketTransformer, + ZLibEncoder; /// Exits the process with the given [exitCode]. typedef ExitFunction = void Function(int exitCode); diff --git a/packages/flutter_tools/lib/src/build_system/targets/assets.dart b/packages/flutter_tools/lib/src/build_system/targets/assets.dart index 5eb1790412..481074c436 100644 --- a/packages/flutter_tools/lib/src/build_system/targets/assets.dart +++ b/packages/flutter_tools/lib/src/build_system/targets/assets.dart @@ -54,6 +54,7 @@ Future copyAssets(Environment environment, Directory outputDirectory, { manifestPath: pubspecFile.path, packagesPath: environment.projectDir.childFile('.packages').path, assetDirPath: null, + targetPlatform: targetPlatform, ); if (resultCode != 0) { throw Exception('Failed to bundle asset files.'); diff --git a/packages/flutter_tools/lib/src/bundle.dart b/packages/flutter_tools/lib/src/bundle.dart index 1eb23f2333..d080bb260f 100644 --- a/packages/flutter_tools/lib/src/bundle.dart +++ b/packages/flutter_tools/lib/src/bundle.dart @@ -192,6 +192,7 @@ Future buildAssets({ String manifestPath, String assetDirPath, @required String packagesPath, + TargetPlatform targetPlatform, }) async { assetDirPath ??= getAssetBuildDirectory(); packagesPath ??= globals.fs.path.absolute(packagesPath); @@ -202,6 +203,7 @@ Future buildAssets({ manifestPath: manifestPath, assetDirPath: assetDirPath, packagesPath: packagesPath, + targetPlatform: targetPlatform, ); if (result != 0) { return null; diff --git a/packages/flutter_tools/lib/src/isolated/resident_web_runner.dart b/packages/flutter_tools/lib/src/isolated/resident_web_runner.dart index c320c3e7cf..973354337b 100644 --- a/packages/flutter_tools/lib/src/isolated/resident_web_runner.dart +++ b/packages/flutter_tools/lib/src/isolated/resident_web_runner.dart @@ -705,7 +705,10 @@ class _ResidentWebRunner extends ResidentWebRunner { final bool rebuildBundle = assetBundle.needsBuild(); if (rebuildBundle) { globals.printTrace('Updating assets'); - final int result = await assetBundle.build(packagesPath: debuggingOptions.buildInfo.packagesPath); + final int result = await assetBundle.build( + packagesPath: debuggingOptions.buildInfo.packagesPath, + targetPlatform: TargetPlatform.web_javascript, + ); if (result != 0) { return UpdateFSReport(success: false); } diff --git a/packages/flutter_tools/test/general.shard/build_system/targets/assets_test.dart b/packages/flutter_tools/test/general.shard/build_system/targets/assets_test.dart index ae7cd2d4e9..d2276c6c59 100644 --- a/packages/flutter_tools/test/general.shard/build_system/targets/assets_test.dart +++ b/packages/flutter_tools/test/general.shard/build_system/targets/assets_test.dart @@ -88,7 +88,7 @@ flutter: expect(fileSystem.file('${environment.buildDir.path}/flutter_assets/AssetManifest.json'), exists); expect(fileSystem.file('${environment.buildDir.path}/flutter_assets/FontManifest.json'), exists); - expect(fileSystem.file('${environment.buildDir.path}/flutter_assets/NOTICES'), exists); + expect(fileSystem.file('${environment.buildDir.path}/flutter_assets/NOTICES.Z'), exists); // See https://github.com/flutter/flutter/issues/35293 expect(fileSystem.file('${environment.buildDir.path}/flutter_assets/assets/foo/bar.png'), exists); // See https://github.com/flutter/flutter/issues/46163