let NOTICES be double gzip wrapped to reduce on-disk installed space (#71899)

dev/devicelab/bin/tasks/module_test.dart

@@ -2,8 +2,11 @@
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
+import 'dart:convert';
 import 'dart:io';
+import 'dart:typed_data';
 
+import 'package:archive/archive.dart';
 import 'package:flutter_devicelab/framework/apk_utils.dart';
 import 'package:flutter_devicelab/framework/framework.dart';
 import 'package:flutter_devicelab/framework/task_result.dart';
@@ -314,6 +317,24 @@ Future<void> main() async {
         'lib/armeabi-v7a/libflutter.so',
       ], await getFilesInApk(releaseHostApk));
 
+      section('Check the NOTICE file is correct');
+
+      await inDirectory(hostApp, () async {
+        final File apkFile = File(releaseHostApk);
+        final Archive apk = ZipDecoder().decodeBytes(apkFile.readAsBytesSync());
+        // Shouldn't be missing since we already checked it exists above.
+        final ArchiveFile noticesFile = apk.findFile('assets/flutter_assets/NOTICES.Z');
+
+        final Uint8List licenseData = noticesFile.content as Uint8List;
+        if (licenseData == null) {
+          return TaskResult.failure('Invalid license file.');
+        }
+        final String licenseString = utf8.decode(gzip.decode(licenseData));
+        if (!licenseString.contains('skia') || !licenseString.contains('Flutter Authors')) {
+          return TaskResult.failure('License content missing.');
+        }
+      });
+
       section('Check release AndroidManifest.xml');
 
       final String androidManifestRelease = await getAndroidManifest(debugHostApk);

dev/devicelab/bin/tasks/module_test_ios.dart

@@ -2,7 +2,9 @@
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
+import 'dart:convert';
 import 'dart:io';
+import 'dart:typed_data';
 
 import 'package:flutter_devicelab/framework/framework.dart';
 import 'package:flutter_devicelab/framework/ios.dart';
@@ -212,6 +214,8 @@ Future<void> main() async {
 
       final File objectiveCAnalyticsOutputFile = File(path.join(tempDir.path, 'analytics-objc.log'));
       final Directory objectiveCBuildDirectory = Directory(path.join(tempDir.path, 'build-objc'));
+
+      section('Build iOS Objective-C host app');
       await inDirectory(objectiveCHostApp, () async {
         await exec(
           'pod',
@@ -268,6 +272,28 @@ Future<void> main() async {
         'isolate_snapshot_data',
       ));
 
+      section('Check the NOTICE file is correct');
+
+      final String licenseFilePath = path.join(
+        objectiveCBuildDirectory.path,
+        'Host.app',
+        'Frameworks',
+        'App.framework',
+        'flutter_assets',
+        'NOTICES.Z',
+      );
+      checkFileExists(licenseFilePath);
+
+      await inDirectory(objectiveCBuildDirectory, () async {
+        final Uint8List licenseData = File(licenseFilePath).readAsBytesSync();
+        final String licenseString = utf8.decode(gzip.decode(licenseData));
+        if (!licenseString.contains('skia') || !licenseString.contains('Flutter Authors')) {
+          return TaskResult.failure('License content missing');
+        }
+      });
+
+      section('Check that the host build sends the correct analytics');
+
       final String objectiveCAnalyticsOutput = objectiveCAnalyticsOutputFile.readAsStringSync();
       if (!objectiveCAnalyticsOutput.contains('cd24: ios')
           || !objectiveCAnalyticsOutput.contains('cd25: true')

dev/devicelab/lib/framework/apk_utils.dart

@@ -13,7 +13,7 @@ final String platformLineSep = Platform.isWindows ? '\r\n' : '\n';
 
 final List<String> flutterAssets = <String>[
   'assets/flutter_assets/AssetManifest.json',
-  'assets/flutter_assets/NOTICES',
+  'assets/flutter_assets/NOTICES.Z',
   'assets/flutter_assets/fonts/MaterialIcons-Regular.otf',
   'assets/flutter_assets/packages/cupertino_icons/assets/CupertinoIcons.ttf',
 ];

dev/devicelab/lib/tasks/perf_tests.dart

@@ -1070,7 +1070,7 @@ class CompileTest {
 
     final _UnzipListEntry libflutter = fileToMetadata['lib/armeabi-v7a/libflutter.so'];
     final _UnzipListEntry libapp = fileToMetadata['lib/armeabi-v7a/libapp.so'];
-    final _UnzipListEntry license = fileToMetadata['assets/flutter_assets/NOTICES'];
+    final _UnzipListEntry license = fileToMetadata['assets/flutter_assets/NOTICES.Z'];
 
     return <String, dynamic>{
       'libflutter_uncompressed_bytes': libflutter.uncompressedSize,

dev/integration_tests/flutter_gallery/test/example_code_parser_test.dart

@@ -46,7 +46,7 @@ class TestAssetBundle extends AssetBundle {
   }
 
   @override
-  Future<String> loadString(String key, { bool cache = true }) async {
+  Future<String> loadString(String key, { bool cache = true, bool unzip = false }) async {
     if (key == 'lib/gallery/example_code.dart')
       return testCodeFile;
     return '';

packages/flutter/lib/src/services/asset_bundle.dart

@@ -64,7 +64,16 @@ abstract class AssetBundle {
   /// caller is going to be doing its own caching. (It might not be cached if
   /// it's set to true either, that depends on the asset bundle
   /// implementation.)
-  Future<String> loadString(String key, { bool cache = true }) async {
+  ///
+  /// If the `unzip` argument is set to true, it would first unzip file at the
+  /// specified location before retrieving the string content.
+  Future<String> loadString(
+    String key,
+    {
+      bool cache = true,
+      bool unzip = false,
+    }
+  ) async {
     final ByteData data = await load(key);
     // Note: data has a non-nullable type, but might be null when running with
     // weak checking, so we need to null check it anyway (and ignore the warning
@@ -73,15 +82,26 @@ abstract class AssetBundle {
       throw FlutterError('Unable to load asset: $key'); // ignore: dead_code
     // 50 KB of data should take 2-3 ms to parse on a Moto G4, and about 400 μs
     // on a Pixel 4.
-    if (data.lengthInBytes < 50 * 1024) {
-      return utf8.decode(data.buffer.asUint8List());
-    }
-    // For strings larger than 50 KB, run the computation in an isolate to
-    // avoid causing main thread jank.
-    return compute(_utf8decode, data, debugLabel: 'UTF8 decode for "$key"');
+    if (data.lengthInBytes < 50 * 1024 && !unzip) {
+      return _utf8Decode(data);
     }
 
-  static String _utf8decode(ByteData data) {
+    // For strings larger than 50 KB, run the computation in an isolate to
+    // avoid causing main thread jank.
+    return compute(
+      unzip ? _utf8ZipDecode : _utf8Decode,
+      data,
+      debugLabel: '${unzip ? "Unzip and ": ""}UTF8 decode for "$key"',
+    );
+  }
+
+  static String _utf8ZipDecode(ByteData data) {
+    List<int> bytes = data.buffer.asUint8List();
+    bytes = gzip.decode(bytes);
+    return utf8.decode(bytes);
+  }
+
+  static String _utf8Decode(ByteData data) {
     return utf8.decode(data.buffer.asUint8List());
   }
 
@@ -163,10 +183,10 @@ abstract class CachingAssetBundle extends AssetBundle {
   final Map<String, Future<dynamic>> _structuredDataCache = <String, Future<dynamic>>{};
 
   @override
-  Future<String> loadString(String key, { bool cache = true }) {
+  Future<String> loadString(String key, { bool cache = true, bool unzip = false }) {
     if (cache)
-      return _stringCache.putIfAbsent(key, () => super.loadString(key));
-    return super.loadString(key);
+      return _stringCache.putIfAbsent(key, () => super.loadString(key, unzip: unzip));
+    return super.loadString(key, unzip: unzip);
   }
 
   /// Retrieve a string from the asset bundle, parse it with the given function,

packages/flutter/lib/src/services/binding.dart

@@ -104,7 +104,19 @@ mixin ServicesBinding on BindingBase, SchedulerBinding {
     // TODO(ianh): Remove this complexity once these bugs are fixed.
     final Completer<String> rawLicenses = Completer<String>();
     scheduleTask(() async {
-      rawLicenses.complete(await rootBundle.loadString('NOTICES', cache: false));
+      rawLicenses.complete(
+        await rootBundle.loadString(
+          // NOTICES for web isn't compressed since we don't have access to
+          // dart:io on the client side and it's already compressed between
+          // the server and client.
+          //
+          // The compressed version doesn't have a more common .gz extension
+          // because gradle for Android non-transparently manipulates .gz files.
+          kIsWeb ? 'NOTICES' : 'NOTICES.Z',
+          cache: false,
+          unzip: !kIsWeb,
+        )
+      );
     }, Priority.animation);
     await rawLicenses.future;
     final Completer<List<LicenseEntry>> parsedLicenses = Completer<List<LicenseEntry>>();

packages/flutter/test/services/asset_bundle_test.dart

@@ -3,6 +3,7 @@
 // found in the LICENSE file.
 
 import 'dart:convert';
+import 'dart:io';
 import 'dart:typed_data';
 
 import 'package:flutter/foundation.dart';
@@ -15,10 +16,13 @@ class TestAssetBundle extends CachingAssetBundle {
 
   @override
   Future<ByteData> load(String key) async {
+    loadCallCount[key] = loadCallCount[key] ?? 0 + 1;
     if (key == 'AssetManifest.json')
       return ByteData.view(Uint8List.fromList(const Utf8Encoder().convert('{"one": ["one"]}')).buffer);
 
-    loadCallCount[key] = loadCallCount[key] ?? 0 + 1;
+    if (key == 'NOTICES.Z')
+      return ByteData.view(Uint8List.fromList(gzip.encode(utf8.encode('All your base are belong to us'))).buffer);
+
     if (key == 'one')
       return ByteData(1)..setInt8(0, 49);
     throw FlutterError('key not found');
@@ -48,6 +52,26 @@ void main() {
     expect(loadException, isFlutterError);
   });
 
+  test('Test loading zipped strings', () async {
+    final TestAssetBundle bundle = TestAssetBundle();
+
+    String assetString = await bundle.loadString('NOTICES.Z', unzip: true);
+    expect(assetString, equals('All your base are belong to us'));
+
+    expect(bundle.loadCallCount['NOTICES.Z'], 1);
+
+    assetString = await bundle.loadString('NOTICES.Z', unzip: true);
+    expect(assetString, equals('All your base are belong to us'));
+
+    // Should have been cached and shouldn't retrieve and decode another time.
+    expect(bundle.loadCallCount['NOTICES.Z'], 1);
+  }, onPlatform: <String, dynamic>{
+    'browser': const Skip(
+      'Skip the NOTICES unzipping test because NOTICES are'
+      'not zipped for the web'
+    ),
+  });
+
   test('AssetImage.obtainKey succeeds with ImageConfiguration.empty', () async {
     // This is a regression test for https://github.com/flutter/flutter/issues/12392
     final AssetImage assetImage = AssetImage('one', bundle: TestAssetBundle());

packages/flutter/test/services/binding_test.dart

@@ -2,6 +2,8 @@
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
+import 'dart:convert';
+import 'dart:io';
 import 'dart:typed_data';
 
 import 'package:flutter/foundation.dart';
@@ -44,7 +46,10 @@ class TestBinding extends BindingBase with SchedulerBinding, ServicesBinding {
   BinaryMessenger createBinaryMessenger() {
     return super.createBinaryMessenger()
       ..setMockMessageHandler('flutter/assets', (ByteData? message) async {
-        if (const StringCodec().decodeMessage(message) == 'NOTICES') {
+        if (const StringCodec().decodeMessage(message) == 'NOTICES.Z' && !kIsWeb) {
+          return Uint8List.fromList(gzip.encode(utf8.encode(licenses))).buffer.asByteData();
+        }
+        if (const StringCodec().decodeMessage(message) == 'NOTICES' && kIsWeb) {
           return const StringCodec().encodeMessage(licenses);
         }
         return null;

packages/flutter/test/widgets/image_resolution_test.dart

@@ -66,7 +66,7 @@ class TestAssetBundle extends CachingAssetBundle {
   }
 
   @override
-  Future<String> loadString(String key, { bool cache = true }) {
+  Future<String> loadString(String key, { bool cache = true, bool unzip = false }) {
     if (key == 'AssetManifest.json')
       return SynchronousFuture<String>(manifest);
     return SynchronousFuture<String>('');

packages/flutter_tools/bin/fuchsia_asset_builder.dart

@@ -7,6 +7,7 @@ import 'package:flutter_tools/src/asset.dart' hide defaultManifestPath;
 import 'package:flutter_tools/src/base/context.dart';
 import 'package:flutter_tools/src/base/file_system.dart' as libfs;
 import 'package:flutter_tools/src/base/io.dart';
+import 'package:flutter_tools/src/build_info.dart';
 import 'package:flutter_tools/src/cache.dart';
 import 'package:flutter_tools/src/context_runner.dart';
 import 'package:flutter_tools/src/devfs.dart';
@@ -59,6 +60,7 @@ Future<void> run(List<String> args) async {
     manifestPath: argResults[_kOptionManifest] as String ?? defaultManifestPath,
     assetDirPath: assetDir,
     packagesPath: argResults[_kOptionPackages] as String,
+    targetPlatform: TargetPlatform.fuchsia_arm64 // This is not arch specific.
   );
 
   if (assets == null) {

packages/flutter_tools/lib/src/asset.dart

@@ -7,6 +7,7 @@ import 'package:package_config/package_config.dart';
 
 import 'base/context.dart';
 import 'base/file_system.dart';
+import 'base/io.dart';
 import 'base/logger.dart';
 import 'base/platform.dart';
 import 'build_info.dart';
@@ -72,6 +73,7 @@ abstract class AssetBundle {
     String manifestPath = defaultManifestPath,
     String assetDirPath,
     @required String packagesPath,
+    TargetPlatform targetPlatform,
   });
 }
 
@@ -122,6 +124,13 @@ class ManifestAssetBundle implements AssetBundle {
 
   static const String _kAssetManifestJson = 'AssetManifest.json';
   static const String _kNoticeFile = 'NOTICES';
+  // Comically, this can't be name with the more common .gz file extension
+  // because when it's part of an AAR and brought into another APK via gradle,
+  // gradle individually traverses all the files of the AAR and unzips .gz
+  // files (b/37117906). A less common .Z extension still describes how the
+  // file is formatted if users want to manually inspect the application
+  // bundle and is recognized by default file handlers on OS such as macOS.˚
+  static const String _kNoticeZippedFile = 'NOTICES.Z';
 
   @override
   bool wasBuiltOnce() => _lastBuildTimestamp != null;
@@ -160,6 +169,7 @@ class ManifestAssetBundle implements AssetBundle {
     String manifestPath = defaultManifestPath,
     String assetDirPath,
     @required String packagesPath,
+    TargetPlatform targetPlatform,
   }) async {
     assetDirPath ??= getAssetBuildDirectory();
     FlutterProject flutterProject;
@@ -320,7 +330,6 @@ class ManifestAssetBundle implements AssetBundle {
     final DevFSStringContent assetManifest  = _createAssetManifest(assetVariants);
     final DevFSStringContent fontManifest = DevFSStringContent(json.encode(fonts));
     final LicenseResult licenseResult = _licenseCollector.obtainLicenses(packageConfig);
-    final DevFSStringContent licenses = DevFSStringContent(licenseResult.combinedLicenses);
     additionalDependencies = licenseResult.dependencies;
 
     if (wildcardDirectories.isNotEmpty) {
@@ -338,7 +347,27 @@ class ManifestAssetBundle implements AssetBundle {
 
     _setIfChanged(_kAssetManifestJson, assetManifest);
     _setIfChanged(kFontManifestJson, fontManifest);
-    _setIfChanged(_kNoticeFile, licenses);
+    if (targetPlatform == TargetPlatform.web_javascript) {
+      // Don't compress the NOTICES file on web since the client doesn't have
+      // dart:io to decompress it.
+      _setIfChanged(_kNoticeFile, DevFSStringContent(licenseResult.combinedLicenses));
+    } else {
+      final List<int> licenseBytes = utf8.encode(licenseResult.combinedLicenses);
+      if (entries[_kNoticeZippedFile] == null ||
+          gzip.decode((entries[_kNoticeZippedFile] as DevFSByteContent).bytes)
+              != licenseBytes) {
+        entries[_kNoticeZippedFile] = DevFSByteContent(
+          ZLibEncoder(
+            // A zlib dictionary is a hinting string sequence with the most
+            // likely string occurrences at the end. This ends up just being
+            // common English words with domain specific words like copyright.
+            dictionary: utf8.encode('copyrightsoftwaretothisinandorofthe'),
+            gzip: true,
+            level: 9,
+          ).convert(licenseBytes)
+        );
+      }
+    }
     return 0;
   }
 

packages/flutter_tools/lib/src/base/io.dart

@@ -102,7 +102,8 @@ export 'dart:io'
         systemEncoding,
         WebSocket,
         WebSocketException,
-        WebSocketTransformer;
+        WebSocketTransformer,
+        ZLibEncoder;
 
 /// Exits the process with the given [exitCode].
 typedef ExitFunction = void Function(int exitCode);

packages/flutter_tools/lib/src/build_system/targets/assets.dart

@@ -54,6 +54,7 @@ Future<Depfile> copyAssets(Environment environment, Directory outputDirectory, {
     manifestPath: pubspecFile.path,
     packagesPath: environment.projectDir.childFile('.packages').path,
     assetDirPath: null,
+    targetPlatform: targetPlatform,
   );
   if (resultCode != 0) {
     throw Exception('Failed to bundle asset files.');

packages/flutter_tools/lib/src/bundle.dart

@@ -192,6 +192,7 @@ Future<AssetBundle> buildAssets({
   String manifestPath,
   String assetDirPath,
   @required String packagesPath,
+  TargetPlatform targetPlatform,
 }) async {
   assetDirPath ??= getAssetBuildDirectory();
   packagesPath ??= globals.fs.path.absolute(packagesPath);
@@ -202,6 +203,7 @@ Future<AssetBundle> buildAssets({
     manifestPath: manifestPath,
     assetDirPath: assetDirPath,
     packagesPath: packagesPath,
+    targetPlatform: targetPlatform,
   );
   if (result != 0) {
     return null;

packages/flutter_tools/lib/src/isolated/resident_web_runner.dart

@@ -705,7 +705,10 @@ class _ResidentWebRunner extends ResidentWebRunner {
     final bool rebuildBundle = assetBundle.needsBuild();
     if (rebuildBundle) {
       globals.printTrace('Updating assets');
-      final int result = await assetBundle.build(packagesPath: debuggingOptions.buildInfo.packagesPath);
+      final int result = await assetBundle.build(
+        packagesPath: debuggingOptions.buildInfo.packagesPath,
+        targetPlatform: TargetPlatform.web_javascript,
+      );
       if (result != 0) {
         return UpdateFSReport(success: false);
       }

packages/flutter_tools/test/general.shard/build_system/targets/assets_test.dart

@@ -88,7 +88,7 @@ flutter:
 
     expect(fileSystem.file('${environment.buildDir.path}/flutter_assets/AssetManifest.json'), exists);
     expect(fileSystem.file('${environment.buildDir.path}/flutter_assets/FontManifest.json'), exists);
-    expect(fileSystem.file('${environment.buildDir.path}/flutter_assets/NOTICES'), exists);
+    expect(fileSystem.file('${environment.buildDir.path}/flutter_assets/NOTICES.Z'), exists);
     // See https://github.com/flutter/flutter/issues/35293
     expect(fileSystem.file('${environment.buildDir.path}/flutter_assets/assets/foo/bar.png'), exists);
     // See https://github.com/flutter/flutter/issues/46163