[web] Split all 1MB+ fallback fonts (including CJK) (flutter/engine#56388)

By splitting all large fallback fonts (1MB+) into smaller parts, we get faster downloads and fast decoding.

Some fonts are split into 100+ parts, and that's causing `main.dart.js`'s size to grow by ~47KB (Brotli-compressed). The increase in size is due to the extra data we have to store about all the parts of these fonts.

The PR also makes  changes to ensure we don't download the same license file 100 times (once for each part of the split font).

Fixes https://github.com/flutter/flutter/issues/138288
Part of https://github.com/flutter/flutter/issues/153974

DEPS

@@ -925,7 +925,7 @@ deps = {
       'packages': [
         {
           'package': 'flutter/flutter_font_fallbacks',
-          'version': '10da6a95fedad127634500aa854466fe9e3fa760220a2a1c7c20df84073fce76'
+          'version': '8a753fd2150c398a5777a7fdb24fc9d4d5fe8015088c5237b61cf0ff26653fd0'
         }
       ],
       'dep_type': 'cipd',

engine/src/flutter/lib/web_ui/dev/roll_fallback_fonts.dart

@@ -65,6 +65,12 @@ class RollFallbackFontsCommand extends Command<bool>
       ...await _processSplitFallbackFonts(client, splitFallbackFonts),
       ...await _processFallbackFonts(client, apiFallbackFonts),
     ];
+
+    final List<String> failedUrls = await _checkForLicenseAttributions(client, fallbackFontInfo);
+    if (failedUrls.isNotEmpty) {
+      throw ToolExit('Could not find license attribution at:\n - ${failedUrls.join('\n - ')}');
+    }
+
     final List<_Font> fallbackFontData = <_Font>[];
 
     final Map<String, String> charsetForFamily = <String, String>{};
@@ -79,19 +85,11 @@ class RollFallbackFontsCommand extends Command<bool>
       if (fontResponse.statusCode != 200) {
         throw ToolExit('Failed to download font for $family');
       }
-      final String urlString = uri.toString();
+      final String urlSuffix = getUrlSuffix(uri);
-      if (!urlString.startsWith(expectedUrlPrefix)) {
-        throw ToolExit('Unexpected url format received from Google Fonts API: $urlString.');
-      }
-      final String urlSuffix = urlString.substring(expectedUrlPrefix.length);
       final io.File fontFile =
           io.File(path.join(fontDir.path, urlSuffix));
 
       final Uint8List bodyBytes = fontResponse.bodyBytes;
-      if (!await _checkForLicenseAttribution(client, urlSuffix, bodyBytes)) {
-        throw ToolExit(
-            'Expected license attribution not found in file: $urlString');
-      }
       hasher.add(utf8.encode(urlSuffix));
       hasher.add(bodyBytes);
 
@@ -144,12 +142,7 @@ class RollFallbackFontsCommand extends Command<bool>
 
     for (final _Font font in fallbackFontData) {
       final String family = font.info.family;
-      final String urlString = font.info.uri.toString();
+      final String urlSuffix = getUrlSuffix(font.info.uri);
-      if (!urlString.startsWith(expectedUrlPrefix)) {
-        throw ToolExit(
-            'Unexpected url format received from Google Fonts API: $urlString.');
-      }
-      final String urlSuffix = urlString.substring(expectedUrlPrefix.length);
       sb.writeln(" NotoFont('$family', '$urlSuffix'),");
     }
     sb.writeln('];');
@@ -385,7 +378,6 @@ const List<String> apiFallbackFonts = <String>[
   'Noto Sans',
   'Noto Music',
   'Noto Sans Symbols',
-  'Noto Sans Symbols 2',
   'Noto Sans Adlam',
   'Noto Sans Anatolian Hieroglyphs',
   'Noto Sans Arabic',
@@ -407,12 +399,9 @@ const List<String> apiFallbackFonts = <String>[
   'Noto Sans Cham',
   'Noto Sans Cherokee',
   'Noto Sans Coptic',
-  'Noto Sans Cuneiform',
   'Noto Sans Cypriot',
   'Noto Sans Deseret',
   'Noto Sans Devanagari',
-  'Noto Sans Duployan',
-  'Noto Sans Egyptian Hieroglyphs',
   'Noto Sans Elbasan',
   'Noto Sans Elymaic',
   'Noto Sans Ethiopic',
@@ -423,7 +412,6 @@ const List<String> apiFallbackFonts = <String>[
   'Noto Sans Gujarati',
   'Noto Sans Gunjala Gondi',
   'Noto Sans Gurmukhi',
-  'Noto Sans HK',
   'Noto Sans Hanunoo',
   'Noto Sans Hatran',
   'Noto Sans Hebrew',
@@ -431,9 +419,7 @@ const List<String> apiFallbackFonts = <String>[
   'Noto Sans Indic Siyaq Numbers',
   'Noto Sans Inscriptional Pahlavi',
   'Noto Sans Inscriptional Parthian',
-  'Noto Sans JP',
   'Noto Sans Javanese',
-  'Noto Sans KR',
   'Noto Sans Kaithi',
   'Noto Sans Kannada',
   'Noto Sans Kayah Li',
@@ -492,7 +478,6 @@ const List<String> apiFallbackFonts = <String>[
   'Noto Sans Psalter Pahlavi',
   'Noto Sans Rejang',
   'Noto Sans Runic',
-  'Noto Sans SC',
   'Noto Sans Saurashtra',
   'Noto Sans Sharada',
   'Noto Sans Shavian',
@@ -504,7 +489,6 @@ const List<String> apiFallbackFonts = <String>[
   'Noto Sans Sundanese',
   'Noto Sans Syloti Nagri',
   'Noto Sans Syriac',
-  'Noto Sans TC',
   'Noto Sans Tagalog',
   'Noto Sans Tagbanwa',
   'Noto Sans Tai Le',
@@ -531,27 +515,61 @@ const List<String> apiFallbackFonts = <String>[
 /// handling.
 const List<String> splitFallbackFonts = <String>[
   'Noto Color Emoji',
+  'Noto Sans Symbols 2',
+  'Noto Sans Cuneiform',
+  'Noto Sans Duployan',
+  'Noto Sans Egyptian Hieroglyphs',
+  'Noto Sans HK',
+  'Noto Sans JP',
+  'Noto Sans KR',
+  'Noto Sans SC',
+  'Noto Sans TC',
 ];
 
-Future<bool> _checkForLicenseAttribution(
+String getUrlSuffix(Uri fontUri) {
+  final String urlString = fontUri.toString();
+  if (!urlString.startsWith(expectedUrlPrefix)) {
+    throw ToolExit('Unexpected url format received from Google Fonts API: $urlString.');
+  }
+  return urlString.substring(expectedUrlPrefix.length);
+}
+
+/// Checks the license attribution for unique fonts in the [fallbackFontInfo]
+/// list.
+///
+/// Returns a list of URLs that failed to contain the license attribution.
+Future<List<String>> _checkForLicenseAttributions(
   http.Client client,
-  String urlSuffix,
+  List<_FontInfo> fallbackFontInfo,
-  Uint8List fontBytes,
 ) async {
   const String googleFontsUpstream =
       'https://github.com/google/fonts/tree/main/ofl';
   const String attributionString =
       'This Font Software is licensed under the SIL Open Font License, Version 1.1.';
 
-  final String fontPackageName = urlSuffix.split('/').first;
+  final failedUrls = <String>[];
-  final String fontLicenseUrl = '$googleFontsUpstream/$fontPackageName/OFL.txt';
-  final http.Response response = await client.get(Uri.parse(fontLicenseUrl));
-  if (response.statusCode != 200) {
-    throw ToolExit('Failed to download `$fontPackageName` license at $fontLicenseUrl');
-  }
-  final String licenseString = response.body;
 
-  return licenseString.contains(attributionString);
+  final uniqueFontPackageNames = <String>{};
+  for (final (family: _, :uri) in fallbackFontInfo) {
+    final String urlSuffix = getUrlSuffix(uri);
+    final String fontPackageName = urlSuffix.split('/').first;
+    uniqueFontPackageNames.add(fontPackageName);
+  }
+
+  for (final String fontPackageName in uniqueFontPackageNames) {
+    final String fontLicenseUrl = '$googleFontsUpstream/$fontPackageName/OFL.txt';
+    final http.Response response = await client.get(Uri.parse(fontLicenseUrl));
+    if (response.statusCode != 200) {
+      failedUrls.add(fontLicenseUrl);
+      continue;
+    }
+    final String licenseString = response.body;
+    if (!licenseString.contains(attributionString)) {
+      failedUrls.add(fontLicenseUrl);
+    }
+  }
+
+  return failedUrls;
 }
 
 // The basic information of a font: its [family] (name) and [uri].

engine/src/flutter/lib/web_ui/lib/src/engine/font_fallbacks.dart

@@ -4,6 +4,7 @@
 
 import 'dart:async';
 
+import 'package:meta/meta.dart';
 import 'package:ui/src/engine.dart';
 
 abstract class FallbackFontRegistry {
@@ -12,17 +13,18 @@ abstract class FallbackFontRegistry {
   void updateFallbackFontFamilies(List<String> families);
 }
 
+bool _isNotoSansSC(NotoFont font) => font.name.startsWith('Noto Sans SC');
+bool _isNotoSansTC(NotoFont font) => font.name.startsWith('Noto Sans TC');
+bool _isNotoSansHK(NotoFont font) => font.name.startsWith('Noto Sans HK');
+bool _isNotoSansJP(NotoFont font) => font.name.startsWith('Noto Sans JP');
+bool _isNotoSansKR(NotoFont font) => font.name.startsWith('Noto Sans KR');
+
 /// Global static font fallback data.
 class FontFallbackManager {
   factory FontFallbackManager(FallbackFontRegistry registry) =>
       FontFallbackManager._(registry, getFallbackFontList());
 
   FontFallbackManager._(this.registry, this.fallbackFonts) :
-    _notoSansSC = fallbackFonts.singleWhere((NotoFont font) => font.name == 'Noto Sans SC'),
-    _notoSansTC = fallbackFonts.singleWhere((NotoFont font) => font.name == 'Noto Sans TC'),
-    _notoSansHK = fallbackFonts.singleWhere((NotoFont font) => font.name == 'Noto Sans HK'),
-    _notoSansJP = fallbackFonts.singleWhere((NotoFont font) => font.name == 'Noto Sans JP'),
-    _notoSansKR = fallbackFonts.singleWhere((NotoFont font) => font.name == 'Noto Sans KR'),
     _notoSymbols = fallbackFonts.singleWhere((NotoFont font) => font.name == 'Noto Sans Symbols') {
       downloadQueue = FallbackFontDownloadQueue(this);
   }
@@ -39,11 +41,17 @@ class FontFallbackManager {
 
   final List<NotoFont> fallbackFonts;
 
-  final NotoFont _notoSansSC;
+  // By default, we use the system language to determine the user's preferred
-  final NotoFont _notoSansTC;
+  // language. This can be overridden through [debugUserPreferredLanguage] for testing.
-  final NotoFont _notoSansHK;
+  String _language = domWindow.navigator.language;
-  final NotoFont _notoSansJP;
+
-  final NotoFont _notoSansKR;
+  @visibleForTesting
+  String get debugUserPreferredLanguage => _language;
+
+  @visibleForTesting
+  set debugUserPreferredLanguage(String value) {
+    _language = value;
+  }
 
   final NotoFont _notoSymbols;
 
@@ -257,56 +265,49 @@ class FontFallbackManager {
       }
     }
 
-    // If the list of best fonts are all CJK fonts, choose the best one based
+    NotoFont? bestFontForLanguage;
-    // on locale. Otherwise just choose the first font.
     if (bestFonts.length > 1) {
+      // If the list of best fonts are all CJK fonts, choose the best one based
+      // on user preferred language. Otherwise just choose the first font.
       if (bestFonts.every((NotoFont font) =>
-          font == _notoSansSC ||
+          _isNotoSansSC(font) ||
-          font == _notoSansTC ||
+          _isNotoSansTC(font) ||
-          font == _notoSansHK ||
+          _isNotoSansHK(font) ||
-          font == _notoSansJP ||
+          _isNotoSansJP(font) ||
-          font == _notoSansKR)) {
+          _isNotoSansKR(font))) {
-        final String language = domWindow.navigator.language;
+        if (_language == 'zh-Hans' ||
-
+            _language == 'zh-CN' ||
-        if (language == 'zh-Hans' ||
+            _language == 'zh-SG' ||
-            language == 'zh-CN' ||
+            _language == 'zh-MY') {
-            language == 'zh-SG' ||
+          bestFontForLanguage = bestFonts.firstWhereOrNull(_isNotoSansSC);
-            language == 'zh-MY') {
+        } else if (_language == 'zh-Hant' ||
-          if (bestFonts.contains(_notoSansSC)) {
+            _language == 'zh-TW' ||
-            bestFont = _notoSansSC;
+            _language == 'zh-MO') {
-          }
+          bestFontForLanguage = bestFonts.firstWhereOrNull(_isNotoSansTC);
-        } else if (language == 'zh-Hant' ||
+        } else if (_language == 'zh-HK') {
-            language == 'zh-TW' ||
+          bestFontForLanguage = bestFonts.firstWhereOrNull(_isNotoSansHK);
-            language == 'zh-MO') {
+        } else if (_language == 'ja') {
-          if (bestFonts.contains(_notoSansTC)) {
+          bestFontForLanguage = bestFonts.firstWhereOrNull(_isNotoSansJP);
-            bestFont = _notoSansTC;
+        } else if (_language == 'ko') {
-          }
+          bestFontForLanguage = bestFonts.firstWhereOrNull(_isNotoSansKR);
-        } else if (language == 'zh-HK') {
+        } else {
-          if (bestFonts.contains(_notoSansHK)) {
+          // Default to `Noto Sans SC` when the user preferred language is not CJK.
-            bestFont = _notoSansHK;
+          bestFontForLanguage = bestFonts.firstWhereOrNull(_isNotoSansSC);
-          }
-        } else if (language == 'ja') {
-          if (bestFonts.contains(_notoSansJP)) {
-            bestFont = _notoSansJP;
-          }
-        } else if (language == 'ko') {
-          if (bestFonts.contains(_notoSansKR)) {
-            bestFont = _notoSansKR;
-          }
-        } else if (bestFonts.contains(_notoSansSC)) {
-          bestFont = _notoSansSC;
         }
       } else {
         // To be predictable, if there is a tie for best font, choose a font
         // from this list first, then just choose the first font.
         if (bestFonts.contains(_notoSymbols)) {
           bestFont = _notoSymbols;
-        } else if (bestFonts.contains(_notoSansSC)) {
+        } else {
-          bestFont = _notoSansSC;
+          final notoSansSC = bestFonts.firstWhereOrNull(_isNotoSansSC);
+          if (notoSansSC != null) {
+            bestFont = notoSansSC;
+          }
         }
       }
     }
-    return bestFont!;
+    return bestFontForLanguage ?? bestFont!;
   }
 
   late final List<FallbackFontComponent> fontComponents =

engine/src/flutter/lib/web_ui/test/ui/fallback_fonts_golden_test.dart

@@ -229,19 +229,35 @@ void testMain() {
     /// supports split fonts, without hardcoding the shard number (which we
     /// don't own).
     Future<void> checkDownloadedFamilyForCharCode(
-        int charCode, String partialFontFamilyName) async {
+      int charCode,
+      String partialFontFamilyName, {
+      String? userPreferredLanguage,
+    }) async {
+      // downloadedFontFamilies.clear();
+      // renderer.fontCollection.debugResetFallbackFonts();
+
+      final fallbackManager = renderer.fontCollection.fontFallbackManager!;
+      final oldLanguage = fallbackManager.debugUserPreferredLanguage;
+      if (userPreferredLanguage != null) {
+        fallbackManager.debugUserPreferredLanguage = userPreferredLanguage;
+      }
+
       // Try rendering text that requires fallback fonts, initially before the fonts are loaded.
       final ui.ParagraphBuilder pb = ui.ParagraphBuilder(ui.ParagraphStyle());
       pb.addText(String.fromCharCode(charCode));
       pb.build().layout(const ui.ParagraphConstraints(width: 1000));
 
       await renderer.fontCollection.fontFallbackManager!.debugWhenIdle();
+      if (userPreferredLanguage != null) {
+        fallbackManager.debugUserPreferredLanguage = oldLanguage;
+      }
 
       expect(
         downloadedFontFamilies,
         hasLength(1),
         reason:
           'Downloaded more than one font family for character: 0x${charCode.toRadixString(16)}'
+          '${userPreferredLanguage == null ? '' : ' (userPreferredLanguage: $userPreferredLanguage)'}',
       );
       expect(
         downloadedFontFamilies.first,
@@ -256,7 +272,7 @@ void testMain() {
         'can find fonts for two adjacent unmatched code points from different fonts',
         () async {
       await checkDownloadedFamiliesForString('ヽಠ', <String>[
-        'Noto Sans SC',
+        'Noto Sans SC 68',
         'Noto Sans Kannada',
       ]);
     });
@@ -293,6 +309,57 @@ void testMain() {
       await checkDownloadedFamilyForCharCode(0x1f3d5, 'Noto Color Emoji');
     });
 
+    // 0x700b is a CJK Unified Ideograph code point that exists in all of our
+    // CJK fonts.
+
+    // Simplified Chinese
+    test('prioritizes Noto Sans SC for lang=zh', () async {
+      await checkDownloadedFamilyForCharCode(0x700b, 'Noto Sans SC', userPreferredLanguage: 'zh');
+    });
+    test('prioritizes Noto Sans SC for lang=zh-Hans', () async {
+      await checkDownloadedFamilyForCharCode(0x700b, 'Noto Sans SC', userPreferredLanguage: 'zh-Hans');
+    });
+    test('prioritizes Noto Sans SC for lang=zh-CN', () async {
+      await checkDownloadedFamilyForCharCode(0x700b, 'Noto Sans SC', userPreferredLanguage: 'zh-CN');
+    });
+    test('prioritizes Noto Sans SC for lang=zh-SG', () async {
+      await checkDownloadedFamilyForCharCode(0x700b, 'Noto Sans SC', userPreferredLanguage: 'zh-SG');
+    });
+    test('prioritizes Noto Sans SC for lang=zh-MY', () async {
+      await checkDownloadedFamilyForCharCode(0x700b, 'Noto Sans SC', userPreferredLanguage: 'zh-MY');
+    });
+
+    // Simplified Chinese is prioritized when preferred language is non-CJK.
+    test('prioritizes Noto Sans SC for lang=en-US', () async {
+      await checkDownloadedFamilyForCharCode(0x700b, 'Noto Sans SC', userPreferredLanguage: 'en-US');
+    });
+
+    // Traditional Chinese
+    test('prioritizes Noto Sans TC for lang=zh-Hant', () async {
+      await checkDownloadedFamilyForCharCode(0x700b, 'Noto Sans TC', userPreferredLanguage: 'zh-Hant');
+    });
+    test('prioritizes Noto Sans TC for lang=zh-TW', () async {
+      await checkDownloadedFamilyForCharCode(0x700b, 'Noto Sans TC', userPreferredLanguage: 'zh-TW');
+    });
+    test('prioritizes Noto Sans TC for lang=zh-MO', () async {
+      await checkDownloadedFamilyForCharCode(0x700b, 'Noto Sans TC', userPreferredLanguage: 'zh-MO');
+    });
+
+    // Hong Kong
+    test('prioritizes Noto Sans HK for lang=zh-HK', () async {
+      await checkDownloadedFamilyForCharCode(0x700b, 'Noto Sans HK', userPreferredLanguage: 'zh-HK');
+    });
+
+    // Japanese
+    test('prioritizes Noto Sans JP for lang=ja', () async {
+      await checkDownloadedFamilyForCharCode(0x700b, 'Noto Sans JP', userPreferredLanguage: 'ja');
+    });
+
+    // Korean
+    test('prioritizes Noto Sans KR for lang=ko', () async {
+      await checkDownloadedFamilyForCharCode(0x700b, 'Noto Sans KR', userPreferredLanguage: 'ko');
+    });
+
     test('findMinimumFontsForCodePoints for all supported code points',
         () async {
       // Collect all supported code points from all fallback fonts in the Noto
@@ -311,25 +378,20 @@ void testMain() {
 
       expect(
           supportedUniqueCodePoints.length, greaterThan(10000)); // sanity check
-      expect(
+      final allFonts = <String>{
-          testedFonts,
+            ...[for (int i = 0; i <= 11; i++) 'Noto Color Emoji $i'],
-          unorderedEquals(<String>{
+            ...[for (int i = 0; i <= 5; i++) 'Noto Sans Symbols 2 $i'],
-            'Noto Color Emoji 0',
+            ...[for (int i = 0; i <= 2; i++) 'Noto Sans Cuneiform $i'],
-            'Noto Color Emoji 1',
+            ...[for (int i = 0; i <= 2; i++) 'Noto Sans Duployan $i'],
-            'Noto Color Emoji 2',
+            ...[for (int i = 0; i <= 2; i++) 'Noto Sans Egyptian Hieroglyphs $i'],
-            'Noto Color Emoji 3',
+            ...[for (int i = 0; i <= 103; i++) 'Noto Sans HK $i'],
-            'Noto Color Emoji 4',
+            ...[for (int i = 0; i <= 123; i++) 'Noto Sans JP $i'],
-            'Noto Color Emoji 5',
+            ...[for (int i = 0; i <= 117; i++) 'Noto Sans KR $i'],
-            'Noto Color Emoji 6',
+            ...[for (int i = 0; i <= 95; i++) 'Noto Sans SC $i'],
-            'Noto Color Emoji 7',
+            ...[for (int i = 0; i <= 99; i++) 'Noto Sans TC $i'],
-            'Noto Color Emoji 8',
-            'Noto Color Emoji 9',
-            'Noto Color Emoji 10',
-            'Noto Color Emoji 11',
             'Noto Music',
             'Noto Sans',
             'Noto Sans Symbols',
-            'Noto Sans Symbols 2',
             'Noto Sans Adlam',
             'Noto Sans Anatolian Hieroglyphs',
             'Noto Sans Arabic',
@@ -351,12 +413,9 @@ void testMain() {
             'Noto Sans Cham',
             'Noto Sans Cherokee',
             'Noto Sans Coptic',
-            'Noto Sans Cuneiform',
             'Noto Sans Cypriot',
             'Noto Sans Deseret',
             'Noto Sans Devanagari',
-            'Noto Sans Duployan',
-            'Noto Sans Egyptian Hieroglyphs',
             'Noto Sans Elbasan',
             'Noto Sans Elymaic',
             'Noto Sans Ethiopic',
@@ -367,7 +426,6 @@ void testMain() {
             'Noto Sans Gujarati',
             'Noto Sans Gunjala Gondi',
             'Noto Sans Gurmukhi',
-            'Noto Sans HK',
             'Noto Sans Hanunoo',
             'Noto Sans Hatran',
             'Noto Sans Hebrew',
@@ -375,9 +433,7 @@ void testMain() {
             'Noto Sans Indic Siyaq Numbers',
             'Noto Sans Inscriptional Pahlavi',
             'Noto Sans Inscriptional Parthian',
-            'Noto Sans JP',
             'Noto Sans Javanese',
-            'Noto Sans KR',
             'Noto Sans Kaithi',
             'Noto Sans Kannada',
             'Noto Sans Kayah Li',
@@ -436,7 +492,6 @@ void testMain() {
             'Noto Sans Psalter Pahlavi',
             'Noto Sans Rejang',
             'Noto Sans Runic',
-            'Noto Sans SC',
             'Noto Sans Saurashtra',
             'Noto Sans Sharada',
             'Noto Sans Shavian',
@@ -448,7 +503,6 @@ void testMain() {
             'Noto Sans Sundanese',
             'Noto Sans Syloti Nagri',
             'Noto Sans Syriac',
-            'Noto Sans TC',
             'Noto Sans Tagalog',
             'Noto Sans Tagbanwa',
             'Noto Sans Tai Le',
@@ -469,7 +523,14 @@ void testMain() {
             'Noto Sans Yi',
             'Noto Sans Zanabazar Square',
             'Noto Serif Tibetan',
-          }));
+          };
+      expect(
+        testedFonts,
+        unorderedEquals(allFonts),
+        reason: 'Found mismatch in fonts.\n'
+            'Missing fonts: ${allFonts.difference(testedFonts)}\n'
+            'Extra fonts: ${testedFonts.difference(allFonts)}',
+      );
 
       // Construct random paragraphs out of supported code points.
       final math.Random random = math.Random(0);