firka/flutter
firka/flutter
4
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

[web] Split all 1MB+ fallback fonts (including CJK) (flutter/engine#56388)

Browse Source 
By splitting all large fallback fonts (1MB+) into smaller parts, we get faster downloads and fast decoding.

Some fonts are split into 100+ parts, and that's causing `main.dart.js`'s size to grow by ~47KB (Brotli-compressed). The increase in size is due to the extra data we have to store about all the parts of these fonts.

The PR also makes  changes to ensure we don't download the same license file 100 times (once for each part of the split font).

Fixes https://github.com/flutter/flutter/issues/138288
Part of https://github.com/flutter/flutter/issues/153974
This commit is contained in:
Mouad Debbar 2024-11-08 20:44:08 +00:00 committed by GitHub
parent cff2f440f9
commit efe78cfd00
5 changed files with 44018 additions and 23198 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
DEPS
View File

@ -925,7 +925,7 @@ deps = {
'packages': [
{
'package': 'flutter/flutter_font_fallbacks',
'version': '10da6a95fedad127634500aa854466fe9e3fa760220a2a1c7c20df84073fce76'
'version': '8a753fd2150c398a5777a7fdb24fc9d4d5fe8015088c5237b61cf0ff26653fd0'
}
],
'dep_type': 'cipd',

88
engine/src/flutter/lib/web_ui/dev/roll_fallback_fonts.dart
View File

@ -65,6 +65,12 @@ class RollFallbackFontsCommand extends Command<bool>
...await _processSplitFallbackFonts(client, splitFallbackFonts),
...await _processFallbackFonts(client, apiFallbackFonts),
];
final List<String> failedUrls = await _checkForLicenseAttributions(client, fallbackFontInfo);
if (failedUrls.isNotEmpty) {
throw ToolExit('Could not find license attribution at:\n - ${failedUrls.join('\n - ')}');
}
final List<_Font> fallbackFontData = <_Font>[];
final Map<String, String> charsetForFamily = <String, String>{};
@ -79,19 +85,11 @@ class RollFallbackFontsCommand extends Command<bool>
if (fontResponse.statusCode != 200) {
throw ToolExit('Failed to download font for $family');
}
final String urlString = uri.toString();
if (!urlString.startsWith(expectedUrlPrefix)) {
throw ToolExit('Unexpected url format received from Google Fonts API: $urlString.');
}
final String urlSuffix = urlString.substring(expectedUrlPrefix.length);
final String urlSuffix = getUrlSuffix(uri);
final io.File fontFile =
io.File(path.join(fontDir.path, urlSuffix));
final Uint8List bodyBytes = fontResponse.bodyBytes;
if (!await _checkForLicenseAttribution(client, urlSuffix, bodyBytes)) {
throw ToolExit(
'Expected license attribution not found in file: $urlString');
}
hasher.add(utf8.encode(urlSuffix));
hasher.add(bodyBytes);
@ -144,12 +142,7 @@ class RollFallbackFontsCommand extends Command<bool>
for (final _Font font in fallbackFontData) {
final String family = font.info.family;
final String urlString = font.info.uri.toString();
if (!urlString.startsWith(expectedUrlPrefix)) {
throw ToolExit(
'Unexpected url format received from Google Fonts API: $urlString.');
}
final String urlSuffix = urlString.substring(expectedUrlPrefix.length);
final String urlSuffix = getUrlSuffix(font.info.uri);
sb.writeln(" NotoFont('$family', '$urlSuffix'),");
}
sb.writeln('];');
@ -385,7 +378,6 @@ const List<String> apiFallbackFonts = <String>[
'Noto Sans',
'Noto Music',
'Noto Sans Symbols',
'Noto Sans Symbols 2',
'Noto Sans Adlam',
'Noto Sans Anatolian Hieroglyphs',
'Noto Sans Arabic',
@ -407,12 +399,9 @@ const List<String> apiFallbackFonts = <String>[
'Noto Sans Cham',
'Noto Sans Cherokee',
'Noto Sans Coptic',
'Noto Sans Cuneiform',
'Noto Sans Cypriot',
'Noto Sans Deseret',
'Noto Sans Devanagari',
'Noto Sans Duployan',
'Noto Sans Egyptian Hieroglyphs',
'Noto Sans Elbasan',
'Noto Sans Elymaic',
'Noto Sans Ethiopic',
@ -423,7 +412,6 @@ const List<String> apiFallbackFonts = <String>[
'Noto Sans Gujarati',
'Noto Sans Gunjala Gondi',
'Noto Sans Gurmukhi',
'Noto Sans HK',
'Noto Sans Hanunoo',
'Noto Sans Hatran',
'Noto Sans Hebrew',
@ -431,9 +419,7 @@ const List<String> apiFallbackFonts = <String>[
'Noto Sans Indic Siyaq Numbers',
'Noto Sans Inscriptional Pahlavi',
'Noto Sans Inscriptional Parthian',
'Noto Sans JP',
'Noto Sans Javanese',
'Noto Sans KR',
'Noto Sans Kaithi',
'Noto Sans Kannada',
'Noto Sans Kayah Li',
@ -492,7 +478,6 @@ const List<String> apiFallbackFonts = <String>[
'Noto Sans Psalter Pahlavi',
'Noto Sans Rejang',
'Noto Sans Runic',
'Noto Sans SC',
'Noto Sans Saurashtra',
'Noto Sans Sharada',
'Noto Sans Shavian',
@ -504,7 +489,6 @@ const List<String> apiFallbackFonts = <String>[
'Noto Sans Sundanese',
'Noto Sans Syloti Nagri',
'Noto Sans Syriac',
'Noto Sans TC',
'Noto Sans Tagalog',
'Noto Sans Tagbanwa',
'Noto Sans Tai Le',
@ -531,27 +515,61 @@ const List<String> apiFallbackFonts = <String>[
/// handling.
const List<String> splitFallbackFonts = <String>[
'Noto Color Emoji',
'Noto Sans Symbols 2',
'Noto Sans Cuneiform',
'Noto Sans Duployan',
'Noto Sans Egyptian Hieroglyphs',
'Noto Sans HK',
'Noto Sans JP',
'Noto Sans KR',
'Noto Sans SC',
'Noto Sans TC',
];
Future<bool> _checkForLicenseAttribution(
String getUrlSuffix(Uri fontUri) {
final String urlString = fontUri.toString();
if (!urlString.startsWith(expectedUrlPrefix)) {
throw ToolExit('Unexpected url format received from Google Fonts API: $urlString.');
}
return urlString.substring(expectedUrlPrefix.length);
}
/// Checks the license attribution for unique fonts in the [fallbackFontInfo]
/// list.
///
/// Returns a list of URLs that failed to contain the license attribution.
Future<List<String>> _checkForLicenseAttributions(
http.Client client,
String urlSuffix,
Uint8List fontBytes,
List<_FontInfo> fallbackFontInfo,
) async {
const String googleFontsUpstream =
'https://github.com/google/fonts/tree/main/ofl';
const String attributionString =
'This Font Software is licensed under the SIL Open Font License, Version 1.1.';
final String fontPackageName = urlSuffix.split('/').first;
final String fontLicenseUrl = '$googleFontsUpstream/$fontPackageName/OFL.txt';
final http.Response response = await client.get(Uri.parse(fontLicenseUrl));
if (response.statusCode != 200) {
throw ToolExit('Failed to download `$fontPackageName` license at $fontLicenseUrl');
}
final String licenseString = response.body;
final failedUrls = <String>[];
return licenseString.contains(attributionString);
final uniqueFontPackageNames = <String>{};
for (final (family: _, :uri) in fallbackFontInfo) {
final String urlSuffix = getUrlSuffix(uri);
final String fontPackageName = urlSuffix.split('/').first;
uniqueFontPackageNames.add(fontPackageName);
}
for (final String fontPackageName in uniqueFontPackageNames) {
final String fontLicenseUrl = '$googleFontsUpstream/$fontPackageName/OFL.txt';
final http.Response response = await client.get(Uri.parse(fontLicenseUrl));
if (response.statusCode != 200) {
failedUrls.add(fontLicenseUrl);
continue;
}
final String licenseString = response.body;
if (!licenseString.contains(attributionString)) {
failedUrls.add(fontLicenseUrl);
}
}
return failedUrls;
}
// The basic information of a font: its [family] (name) and [uri].

66912
engine/src/flutter/lib/web_ui/lib/src/engine/font_fallback_data.dart
View File

File diff suppressed because one or more lines are too long

99
engine/src/flutter/lib/web_ui/lib/src/engine/font_fallbacks.dart
View File

@ -4,6 +4,7 @@
import 'dart:async';
import 'package:meta/meta.dart';
import 'package:ui/src/engine.dart';
abstract class FallbackFontRegistry {
@ -12,17 +13,18 @@ abstract class FallbackFontRegistry {
void updateFallbackFontFamilies(List<String> families);
}
bool _isNotoSansSC(NotoFont font) => font.name.startsWith('Noto Sans SC');
bool _isNotoSansTC(NotoFont font) => font.name.startsWith('Noto Sans TC');
bool _isNotoSansHK(NotoFont font) => font.name.startsWith('Noto Sans HK');
bool _isNotoSansJP(NotoFont font) => font.name.startsWith('Noto Sans JP');
bool _isNotoSansKR(NotoFont font) => font.name.startsWith('Noto Sans KR');
/// Global static font fallback data.
class FontFallbackManager {
factory FontFallbackManager(FallbackFontRegistry registry) =>
FontFallbackManager._(registry, getFallbackFontList());
FontFallbackManager._(this.registry, this.fallbackFonts) :
_notoSansSC = fallbackFonts.singleWhere((NotoFont font) => font.name == 'Noto Sans SC'),
_notoSansTC = fallbackFonts.singleWhere((NotoFont font) => font.name == 'Noto Sans TC'),
_notoSansHK = fallbackFonts.singleWhere((NotoFont font) => font.name == 'Noto Sans HK'),
_notoSansJP = fallbackFonts.singleWhere((NotoFont font) => font.name == 'Noto Sans JP'),
_notoSansKR = fallbackFonts.singleWhere((NotoFont font) => font.name == 'Noto Sans KR'),
_notoSymbols = fallbackFonts.singleWhere((NotoFont font) => font.name == 'Noto Sans Symbols') {
downloadQueue = FallbackFontDownloadQueue(this);
}
@ -39,11 +41,17 @@ class FontFallbackManager {
final List<NotoFont> fallbackFonts;
final NotoFont _notoSansSC;
final NotoFont _notoSansTC;
final NotoFont _notoSansHK;
final NotoFont _notoSansJP;
final NotoFont _notoSansKR;
// By default, we use the system language to determine the user's preferred
// language. This can be overridden through [debugUserPreferredLanguage] for testing.
String _language = domWindow.navigator.language;
@visibleForTesting
String get debugUserPreferredLanguage => _language;
@visibleForTesting
set debugUserPreferredLanguage(String value) {
_language = value;
}
final NotoFont _notoSymbols;
@ -257,56 +265,49 @@ class FontFallbackManager {
}
}
// If the list of best fonts are all CJK fonts, choose the best one based
// on locale. Otherwise just choose the first font.
NotoFont? bestFontForLanguage;
if (bestFonts.length > 1) {
// If the list of best fonts are all CJK fonts, choose the best one based
// on user preferred language. Otherwise just choose the first font.
if (bestFonts.every((NotoFont font) =>
font == _notoSansSC ||
font == _notoSansTC ||
font == _notoSansHK ||
font == _notoSansJP ||
font == _notoSansKR)) {
final String language = domWindow.navigator.language;
if (language == 'zh-Hans' ||
language == 'zh-CN' ||
language == 'zh-SG' ||
language == 'zh-MY') {
if (bestFonts.contains(_notoSansSC)) {
bestFont = _notoSansSC;
}
} else if (language == 'zh-Hant' ||
language == 'zh-TW' ||
language == 'zh-MO') {
if (bestFonts.contains(_notoSansTC)) {
bestFont = _notoSansTC;
}
} else if (language == 'zh-HK') {
if (bestFonts.contains(_notoSansHK)) {
bestFont = _notoSansHK;
}
} else if (language == 'ja') {
if (bestFonts.contains(_notoSansJP)) {
bestFont = _notoSansJP;
}
} else if (language == 'ko') {
if (bestFonts.contains(_notoSansKR)) {
bestFont = _notoSansKR;
}
} else if (bestFonts.contains(_notoSansSC)) {
bestFont = _notoSansSC;
_isNotoSansSC(font) ||
_isNotoSansTC(font) ||
_isNotoSansHK(font) ||
_isNotoSansJP(font) ||
_isNotoSansKR(font))) {
if (_language == 'zh-Hans' ||
_language == 'zh-CN' ||
_language == 'zh-SG' ||
_language == 'zh-MY') {
bestFontForLanguage = bestFonts.firstWhereOrNull(_isNotoSansSC);
} else if (_language == 'zh-Hant' ||
_language == 'zh-TW' ||
_language == 'zh-MO') {
bestFontForLanguage = bestFonts.firstWhereOrNull(_isNotoSansTC);
} else if (_language == 'zh-HK') {
bestFontForLanguage = bestFonts.firstWhereOrNull(_isNotoSansHK);
} else if (_language == 'ja') {
bestFontForLanguage = bestFonts.firstWhereOrNull(_isNotoSansJP);
} else if (_language == 'ko') {
bestFontForLanguage = bestFonts.firstWhereOrNull(_isNotoSansKR);
} else {
// Default to `Noto Sans SC` when the user preferred language is not CJK.
bestFontForLanguage = bestFonts.firstWhereOrNull(_isNotoSansSC);
}
} else {
// To be predictable, if there is a tie for best font, choose a font
// from this list first, then just choose the first font.
if (bestFonts.contains(_notoSymbols)) {
bestFont = _notoSymbols;
} else if (bestFonts.contains(_notoSansSC)) {
bestFont = _notoSansSC;
} else {
final notoSansSC = bestFonts.firstWhereOrNull(_isNotoSansSC);
if (notoSansSC != null) {
bestFont = notoSansSC;
}
}
}
}
return bestFont!;
return bestFontForLanguage ?? bestFont!;
}
late final List<FallbackFontComponent> fontComponents =

115
engine/src/flutter/lib/web_ui/test/ui/fallback_fonts_golden_test.dart
View File

@ -229,19 +229,35 @@ void testMain() {
/// supports split fonts, without hardcoding the shard number (which we
/// don't own).
Future<void> checkDownloadedFamilyForCharCode(
int charCode, String partialFontFamilyName) async {
int charCode,
String partialFontFamilyName, {
String? userPreferredLanguage,
}) async {
// downloadedFontFamilies.clear();
// renderer.fontCollection.debugResetFallbackFonts();
final fallbackManager = renderer.fontCollection.fontFallbackManager!;
final oldLanguage = fallbackManager.debugUserPreferredLanguage;
if (userPreferredLanguage != null) {
fallbackManager.debugUserPreferredLanguage = userPreferredLanguage;
}
// Try rendering text that requires fallback fonts, initially before the fonts are loaded.
final ui.ParagraphBuilder pb = ui.ParagraphBuilder(ui.ParagraphStyle());
pb.addText(String.fromCharCode(charCode));
pb.build().layout(const ui.ParagraphConstraints(width: 1000));
await renderer.fontCollection.fontFallbackManager!.debugWhenIdle();
if (userPreferredLanguage != null) {
fallbackManager.debugUserPreferredLanguage = oldLanguage;
}
expect(
downloadedFontFamilies,
hasLength(1),
reason:
'Downloaded more than one font family for character: 0x${charCode.toRadixString(16)}'
'${userPreferredLanguage == null ? '' : ' (userPreferredLanguage: $userPreferredLanguage)'}',
);
expect(
downloadedFontFamilies.first,
@ -256,7 +272,7 @@ void testMain() {
'can find fonts for two adjacent unmatched code points from different fonts',
() async {
await checkDownloadedFamiliesForString('ヽಠ', <String>[
'Noto Sans SC',
'Noto Sans SC 68',
'Noto Sans Kannada',
]);
});
@ -293,6 +309,57 @@ void testMain() {
await checkDownloadedFamilyForCharCode(0x1f3d5, 'Noto Color Emoji');
});
// 0x700b is a CJK Unified Ideograph code point that exists in all of our
// CJK fonts.
// Simplified Chinese
test('prioritizes Noto Sans SC for lang=zh', () async {
await checkDownloadedFamilyForCharCode(0x700b, 'Noto Sans SC', userPreferredLanguage: 'zh');
});
test('prioritizes Noto Sans SC for lang=zh-Hans', () async {
await checkDownloadedFamilyForCharCode(0x700b, 'Noto Sans SC', userPreferredLanguage: 'zh-Hans');
});
test('prioritizes Noto Sans SC for lang=zh-CN', () async {
await checkDownloadedFamilyForCharCode(0x700b, 'Noto Sans SC', userPreferredLanguage: 'zh-CN');
});
test('prioritizes Noto Sans SC for lang=zh-SG', () async {
await checkDownloadedFamilyForCharCode(0x700b, 'Noto Sans SC', userPreferredLanguage: 'zh-SG');
});
test('prioritizes Noto Sans SC for lang=zh-MY', () async {
await checkDownloadedFamilyForCharCode(0x700b, 'Noto Sans SC', userPreferredLanguage: 'zh-MY');
});
// Simplified Chinese is prioritized when preferred language is non-CJK.
test('prioritizes Noto Sans SC for lang=en-US', () async {
await checkDownloadedFamilyForCharCode(0x700b, 'Noto Sans SC', userPreferredLanguage: 'en-US');
});
// Traditional Chinese
test('prioritizes Noto Sans TC for lang=zh-Hant', () async {
await checkDownloadedFamilyForCharCode(0x700b, 'Noto Sans TC', userPreferredLanguage: 'zh-Hant');
});
test('prioritizes Noto Sans TC for lang=zh-TW', () async {
await checkDownloadedFamilyForCharCode(0x700b, 'Noto Sans TC', userPreferredLanguage: 'zh-TW');
});
test('prioritizes Noto Sans TC for lang=zh-MO', () async {
await checkDownloadedFamilyForCharCode(0x700b, 'Noto Sans TC', userPreferredLanguage: 'zh-MO');
});
// Hong Kong
test('prioritizes Noto Sans HK for lang=zh-HK', () async {
await checkDownloadedFamilyForCharCode(0x700b, 'Noto Sans HK', userPreferredLanguage: 'zh-HK');
});
// Japanese
test('prioritizes Noto Sans JP for lang=ja', () async {
await checkDownloadedFamilyForCharCode(0x700b, 'Noto Sans JP', userPreferredLanguage: 'ja');
});
// Korean
test('prioritizes Noto Sans KR for lang=ko', () async {
await checkDownloadedFamilyForCharCode(0x700b, 'Noto Sans KR', userPreferredLanguage: 'ko');
});
test('findMinimumFontsForCodePoints for all supported code points',
() async {
// Collect all supported code points from all fallback fonts in the Noto
@ -311,25 +378,20 @@ void testMain() {
expect(
supportedUniqueCodePoints.length, greaterThan(10000)); // sanity check
expect(
testedFonts,
unorderedEquals(<String>{
'Noto Color Emoji 0',
'Noto Color Emoji 1',
'Noto Color Emoji 2',
'Noto Color Emoji 3',
'Noto Color Emoji 4',
'Noto Color Emoji 5',
'Noto Color Emoji 6',
'Noto Color Emoji 7',
'Noto Color Emoji 8',
'Noto Color Emoji 9',
'Noto Color Emoji 10',
'Noto Color Emoji 11',
final allFonts = <String>{
...[for (int i = 0; i <= 11; i++) 'Noto Color Emoji $i'],
...[for (int i = 0; i <= 5; i++) 'Noto Sans Symbols 2 $i'],
...[for (int i = 0; i <= 2; i++) 'Noto Sans Cuneiform $i'],
...[for (int i = 0; i <= 2; i++) 'Noto Sans Duployan $i'],
...[for (int i = 0; i <= 2; i++) 'Noto Sans Egyptian Hieroglyphs $i'],
...[for (int i = 0; i <= 103; i++) 'Noto Sans HK $i'],
...[for (int i = 0; i <= 123; i++) 'Noto Sans JP $i'],
...[for (int i = 0; i <= 117; i++) 'Noto Sans KR $i'],
...[for (int i = 0; i <= 95; i++) 'Noto Sans SC $i'],
...[for (int i = 0; i <= 99; i++) 'Noto Sans TC $i'],
'Noto Music',
'Noto Sans',
'Noto Sans Symbols',
'Noto Sans Symbols 2',
'Noto Sans Adlam',
'Noto Sans Anatolian Hieroglyphs',
'Noto Sans Arabic',
@ -351,12 +413,9 @@ void testMain() {
'Noto Sans Cham',
'Noto Sans Cherokee',
'Noto Sans Coptic',
'Noto Sans Cuneiform',
'Noto Sans Cypriot',
'Noto Sans Deseret',
'Noto Sans Devanagari',
'Noto Sans Duployan',
'Noto Sans Egyptian Hieroglyphs',
'Noto Sans Elbasan',
'Noto Sans Elymaic',
'Noto Sans Ethiopic',
@ -367,7 +426,6 @@ void testMain() {
'Noto Sans Gujarati',
'Noto Sans Gunjala Gondi',
'Noto Sans Gurmukhi',
'Noto Sans HK',
'Noto Sans Hanunoo',
'Noto Sans Hatran',
'Noto Sans Hebrew',
@ -375,9 +433,7 @@ void testMain() {
'Noto Sans Indic Siyaq Numbers',
'Noto Sans Inscriptional Pahlavi',
'Noto Sans Inscriptional Parthian',
'Noto Sans JP',
'Noto Sans Javanese',
'Noto Sans KR',
'Noto Sans Kaithi',
'Noto Sans Kannada',
'Noto Sans Kayah Li',
@ -436,7 +492,6 @@ void testMain() {
'Noto Sans Psalter Pahlavi',
'Noto Sans Rejang',
'Noto Sans Runic',
'Noto Sans SC',
'Noto Sans Saurashtra',
'Noto Sans Sharada',
'Noto Sans Shavian',
@ -448,7 +503,6 @@ void testMain() {
'Noto Sans Sundanese',
'Noto Sans Syloti Nagri',
'Noto Sans Syriac',
'Noto Sans TC',
'Noto Sans Tagalog',
'Noto Sans Tagbanwa',
'Noto Sans Tai Le',
@ -469,7 +523,14 @@ void testMain() {
'Noto Sans Yi',
'Noto Sans Zanabazar Square',
'Noto Serif Tibetan',
}));
};
expect(
testedFonts,
unorderedEquals(allFonts),
reason: 'Found mismatch in fonts.\n'
'Missing fonts: ${allFonts.difference(testedFonts)}\n'
'Extra fonts: ${testedFonts.difference(allFonts)}',
);
// Construct random paragraphs out of supported code points.
final math.Random random = math.Random(0);