diff --git a/packages/flutter_tools/lib/src/commands/generate_localizations.dart b/packages/flutter_tools/lib/src/commands/generate_localizations.dart index f7e320b857..beae3f30a0 100644 --- a/packages/flutter_tools/lib/src/commands/generate_localizations.dart +++ b/packages/flutter_tools/lib/src/commands/generate_localizations.dart @@ -192,6 +192,15 @@ class GenerateLocalizationsCommand extends FlutterCommand { 'format', help: 'When specified, the "dart format" command is run after generating the localization files.' ); + argParser.addFlag( + 'use-escaping', + help: 'Whether or not to use escaping for messages.\n' + '\n' + 'By default, this value is set to false for backwards compatibility. ' + 'Turning this flag on will cause the parser to treat any special characters ' + 'contained within pairs of single quotes as normal strings and treat all ' + 'consecutive pairs of single quotes as a single quote character.', + ); } final FileSystem _fileSystem; @@ -248,6 +257,7 @@ class GenerateLocalizationsCommand extends FlutterCommand { final String? projectPathString = stringArgDeprecated('project-dir'); final bool areResourceAttributesRequired = boolArgDeprecated('required-resource-attributes'); final bool usesNullableGetter = boolArgDeprecated('nullable-getter'); + final bool useEscaping = boolArgDeprecated('use-escaping'); precacheLanguageAndRegionTags(); @@ -269,6 +279,7 @@ class GenerateLocalizationsCommand extends FlutterCommand { areResourceAttributesRequired: areResourceAttributesRequired, untranslatedMessagesFile: untranslatedMessagesFile, usesNullableGetter: usesNullableGetter, + useEscaping: useEscaping, logger: _logger, ) ..loadResources() diff --git a/packages/flutter_tools/lib/src/localizations/gen_l10n.dart b/packages/flutter_tools/lib/src/localizations/gen_l10n.dart index b7913cb320..b5831af3d5 100644 --- a/packages/flutter_tools/lib/src/localizations/gen_l10n.dart +++ b/packages/flutter_tools/lib/src/localizations/gen_l10n.dart @@ -65,6 +65,7 @@ LocalizationsGenerator generateLocalizations({ areResourceAttributesRequired: options.areResourceAttributesRequired, untranslatedMessagesFile: options.untranslatedMessagesFile?.toFilePath(), usesNullableGetter: options.usesNullableGetter, + useEscaping: options.useEscaping, logger: logger, ) ..loadResources() @@ -453,6 +454,7 @@ class LocalizationsGenerator { bool areResourceAttributesRequired = false, String? untranslatedMessagesFile, bool usesNullableGetter = true, + bool useEscaping = false, required Logger logger, }) { final Directory? projectDirectory = projectDirFromPath(fileSystem, projectPathString); @@ -474,6 +476,7 @@ class LocalizationsGenerator { untranslatedMessagesFile: _untranslatedMessagesFileFromPath(fileSystem, untranslatedMessagesFile), inputsAndOutputsListFile: _inputsAndOutputsListFileFromPath(fileSystem, inputsAndOutputsListPath), areResourceAttributesRequired: areResourceAttributesRequired, + useEscaping: useEscaping, logger: logger, ); } @@ -497,6 +500,7 @@ class LocalizationsGenerator { this.untranslatedMessagesFile, this.usesNullableGetter = true, required this.logger, + this.useEscaping = false, }); final FileSystem _fs; @@ -566,6 +570,9 @@ class LocalizationsGenerator { // all of the messages. bool requiresIntlImport = false; + // Whether we want to use escaping for ICU messages. + bool useEscaping = false; + /// The list of all arb path strings in [inputDirectory]. List get arbPathStrings { return _allBundles.bundles.map((AppResourceBundle bundle) => bundle.file.path).toList(); @@ -845,7 +852,7 @@ class LocalizationsGenerator { // files in inputDirectory. Also initialized: supportedLocales. void loadResources() { _allMessages = _templateBundle.resourceIds.map((String id) => Message( - _templateBundle, _allBundles, id, areResourceAttributesRequired, + _templateBundle, _allBundles, id, areResourceAttributesRequired, useEscaping: useEscaping, )); for (final String resourceId in _templateBundle.resourceIds) { if (!_isValidGetterAndMethodName(resourceId)) { diff --git a/packages/flutter_tools/lib/src/localizations/gen_l10n_types.dart b/packages/flutter_tools/lib/src/localizations/gen_l10n_types.dart index a186bae474..ea12fe6a2b 100644 --- a/packages/flutter_tools/lib/src/localizations/gen_l10n_types.dart +++ b/packages/flutter_tools/lib/src/localizations/gen_l10n_types.dart @@ -318,7 +318,8 @@ class Message { AppResourceBundle templateBundle, AppResourceBundleCollection allBundles, this.resourceId, - bool isResourceAttributeRequired + bool isResourceAttributeRequired, + { this.useEscaping = false } ) : assert(templateBundle != null), assert(allBundles != null), assert(resourceId != null && resourceId.isNotEmpty), @@ -334,7 +335,7 @@ class Message { filenames[bundle.locale] = bundle.file.basename; final String? translation = bundle.translationFor(resourceId); messages[bundle.locale] = translation; - parsedMessages[bundle.locale] = translation == null ? null : Parser(resourceId, bundle.file.basename, translation).parse(); + parsedMessages[bundle.locale] = translation == null ? null : Parser(resourceId, bundle.file.basename, translation, useEscaping: useEscaping).parse(); } // Using parsed translations, attempt to infer types of placeholders used by plurals and selects. for (final LocaleInfo locale in parsedMessages.keys) { @@ -400,6 +401,7 @@ class Message { late final Map messages; final Map parsedMessages; final Map placeholders; + final bool useEscaping; bool get placeholdersRequireFormatting => placeholders.values.any((Placeholder p) => p.requiresFormatting); diff --git a/packages/flutter_tools/lib/src/localizations/localizations_utils.dart b/packages/flutter_tools/lib/src/localizations/localizations_utils.dart index b9e02808e0..048686db23 100644 --- a/packages/flutter_tools/lib/src/localizations/localizations_utils.dart +++ b/packages/flutter_tools/lib/src/localizations/localizations_utils.dart @@ -339,6 +339,7 @@ class LocalizationOptions { this.areResourceAttributesRequired = false, this.usesNullableGetter = true, this.format = false, + this.useEscaping = false, }) : assert(useSyntheticPackage != null); /// The `--arb-dir` argument. @@ -410,6 +411,11 @@ class LocalizationOptions { /// /// Whether or not to format the generated files. final bool format; + + /// The `use-escaping` argument. + /// + /// Whether or not the ICU escaping syntax is used. + final bool useEscaping; } /// Parse the localizations configuration options from [file]. @@ -445,6 +451,7 @@ LocalizationOptions parseLocalizationsOptions({ areResourceAttributesRequired: _tryReadBool(yamlNode, 'required-resource-attributes', logger) ?? false, usesNullableGetter: _tryReadBool(yamlNode, 'nullable-getter', logger) ?? true, format: _tryReadBool(yamlNode, 'format', logger) ?? true, + useEscaping: _tryReadBool(yamlNode, 'use-escaping', logger) ?? false, ); } diff --git a/packages/flutter_tools/lib/src/localizations/message_parser.dart b/packages/flutter_tools/lib/src/localizations/message_parser.dart index 1ca0d42eb9..3624f84a8c 100644 --- a/packages/flutter_tools/lib/src/localizations/message_parser.dart +++ b/packages/flutter_tools/lib/src/localizations/message_parser.dart @@ -149,7 +149,10 @@ $indent])'''; } } -RegExp unescapedString = RegExp(r'[^{}]+'); +RegExp escapedString = RegExp(r"'[^']*'"); +RegExp unescapedString = RegExp(r"[^{}']+"); +RegExp normalString = RegExp(r'[^{}]+'); + RegExp brace = RegExp(r'{|}'); RegExp whitespace = RegExp(r'\s+'); @@ -174,11 +177,17 @@ Map matchers = { }; class Parser { - Parser(this.messageId, this.filename, this.messageString); + Parser( + this.messageId, + this.filename, + this.messageString, + { this.useEscaping = false } + ); final String messageId; final String messageString; final String filename; + final bool useEscaping; static String indentForError(int position) { return '${List.filled(position, ' ').join()}^'; @@ -201,20 +210,41 @@ class Parser { while (startIndex < messageString.length) { Match? match; if (isString) { - // TODO(thkim1011): Uncomment this when we add escaping as an option. - // See https://github.com/flutter/flutter/issues/113455. - // match = escapedString.matchAsPrefix(message, startIndex); - // if (match != null) { - // final String string = match.group(0)!; - // tokens.add(Node.string(startIndex, string == "''" ? "'" : string.substring(1, string.length - 1))); - // startIndex = match.end; - // continue; - // } - match = unescapedString.matchAsPrefix(messageString, startIndex); - if (match != null) { - tokens.add(Node.string(startIndex, match.group(0)!)); - startIndex = match.end; - continue; + if (useEscaping) { + // This case is slightly involved. Essentially, wrapping any syntax in + // single quotes escapes the syntax except when there are consecutive pair of single + // quotes. For example, "Hello! 'Flutter''s amazing'. { unescapedPlaceholder }" + // converts the '' in "Flutter's" to a single quote for convenience, since technically, + // we should interpret this as two strings 'Flutter' and 's amazing'. To get around this, + // we also check if the previous character is a ', and if so, add a single quote at the beginning + // of the token. + match = escapedString.matchAsPrefix(messageString, startIndex); + if (match != null) { + final String string = match.group(0)!; + if (string == "''") { + tokens.add(Node.string(startIndex, "'")); + } else if (startIndex > 1 && messageString[startIndex - 1] == "'") { + // Include a single quote in the beginning of the token. + tokens.add(Node.string(startIndex, string.substring(0, string.length - 1))); + } else { + tokens.add(Node.string(startIndex, string.substring(1, string.length - 1))); + } + startIndex = match.end; + continue; + } + match = unescapedString.matchAsPrefix(messageString, startIndex); + if (match != null) { + tokens.add(Node.string(startIndex, match.group(0)!)); + startIndex = match.end; + continue; + } + } else { + match = normalString.matchAsPrefix(messageString, startIndex); + if (match != null) { + tokens.add(Node.string(startIndex, match.group(0)!)); + startIndex = match.end; + continue; + } } match = brace.matchAsPrefix(messageString, startIndex); if (match != null) { diff --git a/packages/flutter_tools/test/general.shard/message_parser_test.dart b/packages/flutter_tools/test/general.shard/message_parser_test.dart index 48825bc452..ddeb60c932 100644 --- a/packages/flutter_tools/test/general.shard/message_parser_test.dart +++ b/packages/flutter_tools/test/general.shard/message_parser_test.dart @@ -187,32 +187,36 @@ void main() { ])); }); - // TODO(thkim1011): Uncomment when implementing escaping. - // See https://github.com/flutter/flutter/issues/113455. - // testWithoutContext('lexer escaping', () { - // final List tokens1 = Parser("''").lexIntoTokens(); - // expect(tokens1, equals([Node.string(0, "'")])); + testWithoutContext('lexer escaping', () { + final List tokens1 = Parser('escaping', 'app_en.arb', "''", useEscaping: true).lexIntoTokens(); + expect(tokens1, equals([Node.string(0, "'")])); - // final List tokens2 = Parser("'hello world { name }'").lexIntoTokens(); - // expect(tokens2, equals([Node.string(0, 'hello world { name }')])); + final List tokens2 = Parser('escaping', 'app_en.arb', "'hello world { name }'", useEscaping: true).lexIntoTokens(); + expect(tokens2, equals([Node.string(0, 'hello world { name }')])); - // final List tokens3 = Parser("'{ escaped string }' { not escaped }").lexIntoTokens(); - // expect(tokens3, equals([ - // Node.string(0, '{ escaped string }'), - // Node.string(20, ' '), - // Node.openBrace(21), - // Node.identifier(23, 'not'), - // Node.identifier(27, 'escaped'), - // Node.closeBrace(35), - // ])); + final List tokens3 = Parser('escaping', 'app_en.arb', "'{ escaped string }' { not escaped }", useEscaping: true).lexIntoTokens(); + expect(tokens3, equals([ + Node.string(0, '{ escaped string }'), + Node.string(20, ' '), + Node.openBrace(21), + Node.identifier(23, 'not'), + Node.identifier(27, 'escaped'), + Node.closeBrace(35), + ])); - // final List tokens4 = Parser("Flutter''s amazing!").lexIntoTokens(); - // expect(tokens4, equals([ - // Node.string(0, 'Flutter'), - // Node.string(7, "'"), - // Node.string(9, 's amazing!'), - // ])); - // }); + final List tokens4 = Parser('escaping', 'app_en.arb', "Flutter''s amazing!", useEscaping: true).lexIntoTokens(); + expect(tokens4, equals([ + Node.string(0, 'Flutter'), + Node.string(7, "'"), + Node.string(9, 's amazing!'), + ])); + + final List tokens5 = Parser('escaping', 'app_en.arb', "'Flutter''s amazing!'", useEscaping: true).lexIntoTokens(); + expect(tokens5, equals([ + Node(ST.string, 0, value: 'Flutter'), + Node(ST.string, 9, value: "'s amazing!"), + ])); + }); testWithoutContext('lexer: lexically correct but syntactically incorrect', () { final List tokens = Parser( @@ -235,22 +239,20 @@ void main() { ])); }); - // TODO(thkim1011): Uncomment when implementing escaping. - // See https://github.com/flutter/flutter/issues/113455. -// testWithoutContext('lexer unmatched single quote', () { -// const String message = "here''s an unmatched single quote: '"; -// const String expectedError = ''' -// ICU Lexing Error: Unmatched single quotes. -// here''s an unmatched single quote: ' -// ^'''; -// expect( -// () => Parser(message).lexIntoTokens(), -// throwsA(isA().having( -// (L10nException e) => e.message, -// 'message', -// contains(expectedError), -// ))); -// }); + testWithoutContext('lexer unmatched single quote', () { + const String message = "here''s an unmatched single quote: '"; + const String expectedError = ''' +ICU Lexing Error: Unmatched single quotes. +[app_en.arb:escaping] here''s an unmatched single quote: ' + ^'''; + expect( + () => Parser('escaping', 'app_en.arb', message, useEscaping: true).lexIntoTokens(), + throwsA(isA().having( + (L10nException e) => e.message, + 'message', + contains(expectedError), + ))); + }); testWithoutContext('lexer unexpected character', () { const String message = '{ * }'; @@ -367,17 +369,22 @@ ICU Lexing Error: Unexpected character. )); }); - // TODO(thkim1011): Uncomment when implementing escaping. - // See https://github.com/flutter/flutter/issues/113455. - // testWithoutContext('parser basic 2', () { - // expect(Parser("Flutter''s amazing!").parse(), equals( - // Node(ST.message, 0, children: [ - // Node(ST.string, 0, value: 'Flutter'), - // Node(ST.string, 7, value: "'"), - // Node(ST.string, 9, value: 's amazing!'), - // ]) - // )); - // }); + testWithoutContext('parser escaping', () { + expect(Parser('escaping', 'app_en.arb', "Flutter''s amazing!", useEscaping: true).parse(), equals( + Node(ST.message, 0, children: [ + Node(ST.string, 0, value: 'Flutter'), + Node(ST.string, 7, value: "'"), + Node(ST.string, 9, value: 's amazing!'), + ]) + )); + + expect(Parser('escaping', 'app_en.arb', "'Flutter''s amazing!'", useEscaping: true).parse(), equals( + Node(ST.message, 0, children: [ + Node(ST.string, 0, value: 'Flutter'), + Node(ST.string, 9, value: "'s amazing!"), + ]) + )); + }); testWithoutContext('parser recursive', () { expect(Parser(