firka/flutter
firka/flutter
5
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

[flutter_tools] refactor shared memory filesystem logic (#70795)

Browse Source
This commit is contained in:
Jonah Williams 2020-11-18 17:29:13 -08:00 committed by GitHub
parent 42c9c22553
commit f8940709b2
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 190 additions and 228 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

149
packages/flutter_tools/lib/src/isolated/devfs_web.dart
View File

@ -24,7 +24,6 @@ import '../base/io.dart';
import '../base/logger.dart';
import '../base/net.dart';
import '../base/platform.dart';
import '../base/utils.dart';
import '../build_info.dart';
import '../bundle.dart';
import '../cache.dart';
@ -36,6 +35,7 @@ import '../globals.dart' as globals;
import '../project.dart';
import '../web/bootstrap.dart';
import '../web/chrome.dart';
import '../web/memory_fs.dart';
/// Web rendering backend mode.
enum WebRendererMode {
@ -148,10 +148,20 @@ class WebAssetServer implements AssetReader {
final String name = moduleName.replaceAll('.lib.js', '');
final String path = moduleName.replaceAll('.js', '');
_modules[name] = path;
_digests[name] = _files[moduleName].hashCode.toString();
_digests[name] = _webMemoryFS.files[moduleName].hashCode.toString();
}
}
@visibleForTesting
List<String> write(
File codeFile,
File manifestFile,
File sourcemapFile,
File metadataFile,
) {
return _webMemoryFS.write(codeFile, manifestFile, sourcemapFile, metadataFile);
}
/// Start the web asset server on a [hostname] and [port].
///
/// If [testMode] is true, do not actually initialize dwds or the shelf static
@ -293,12 +303,9 @@ class WebAssetServer implements AssetReader {
final NullSafetyMode _nullSafetyMode;
final HttpServer _httpServer;
// If holding these in memory is too much overhead, this can be switched to a
// RandomAccessFile and read on demand.
final Map<String, Uint8List> _files = <String, Uint8List>{};
final Map<String, Uint8List> _sourcemaps = <String, Uint8List>{};
final Map<String, Uint8List> _metadataFiles = <String, Uint8List>{};
String _mergedMetadata;
final WebMemoryFS _webMemoryFS = WebMemoryFS();
final PackageConfig _packages;
final InternetAddress internetAddress;
/* late final */ Dwds dwds;
@ -308,13 +315,13 @@ class WebAssetServer implements AssetReader {
HttpHeaders get defaultResponseHeaders => _httpServer.defaultResponseHeaders;
@visibleForTesting
Uint8List getFile(String path) => _files[path];
Uint8List getFile(String path) => _webMemoryFS.files[path];
@visibleForTesting
Uint8List getSourceMap(String path) => _sourcemaps[path];
Uint8List getSourceMap(String path) => _webMemoryFS.sourcemaps[path];
@visibleForTesting
Uint8List getMetadata(String path) => _metadataFiles[path];
Uint8List getMetadata(String path) => _webMemoryFS.metadataFiles[path];
@visibleForTesting
@ -352,7 +359,7 @@ class WebAssetServer implements AssetReader {
// Attempt to look up the file by URI.
final String webServerPath =
requestPath.replaceFirst('.dart.js', '.dart.lib.js');
if (_files.containsKey(requestPath) || _files.containsKey(webServerPath)) {
if (_webMemoryFS.files.containsKey(requestPath) || _webMemoryFS.files.containsKey(webServerPath)) {
final List<int> bytes = getFile(requestPath) ?? getFile(webServerPath);
// Use the underlying buffer hashCode as a revision string. This buffer is
// replaced whenever the frontend_server produces new output files, which
@ -368,7 +375,7 @@ class WebAssetServer implements AssetReader {
}
// If this is a sourcemap file, then it might be in the in-memory cache.
// Attempt to lookup the file by URI.
if (_sourcemaps.containsKey(requestPath)) {
if (_webMemoryFS.sourcemaps.containsKey(requestPath)) {
final List<int> bytes = getSourceMap(requestPath);
final String etag = bytes.hashCode.toString();
if (ifNoneMatch == etag) {
@ -382,7 +389,7 @@ class WebAssetServer implements AssetReader {
// If this is a metadata file, then it might be in the in-memory cache.
// Attempt to lookup the file by URI.
if (_metadataFiles.containsKey(requestPath)) {
if (_webMemoryFS.metadataFiles.containsKey(requestPath)) {
final List<int> bytes = getMetadata(requestPath);
final String etag = bytes.hashCode.toString();
if (ifNoneMatch == etag) {
@ -460,93 +467,7 @@ class WebAssetServer implements AssetReader {
}
void writeBytes(String filePath, Uint8List contents) {
_files[filePath] = contents;
}
/// Update the in-memory asset server with the provided source and manifest files.
///
/// Returns a list of updated modules.
List<String> write(
File codeFile, File manifestFile, File sourcemapFile, File metadataFile) {
final List<String> modules = <String>[];
final Uint8List codeBytes = codeFile.readAsBytesSync();
final Uint8List sourcemapBytes = sourcemapFile.readAsBytesSync();
final Uint8List metadataBytes = metadataFile.readAsBytesSync();
final Map<String, dynamic> manifest =
castStringKeyedMap(json.decode(manifestFile.readAsStringSync()));
for (final String filePath in manifest.keys) {
if (filePath == null) {
globals.printTrace('Invalid manifest file: $filePath');
continue;
}
final Map<String, dynamic> offsets =
castStringKeyedMap(manifest[filePath]);
final List<int> codeOffsets =
(offsets['code'] as List<dynamic>).cast<int>();
final List<int> sourcemapOffsets =
(offsets['sourcemap'] as List<dynamic>).cast<int>();
final List<int> metadataOffsets =
(offsets['metadata'] as List<dynamic>).cast<int>();
if (codeOffsets.length != 2 ||
sourcemapOffsets.length != 2 ||
metadataOffsets.length != 2) {
globals.printTrace('Invalid manifest byte offsets: $offsets');
continue;
}
final int codeStart = codeOffsets[0];
final int codeEnd = codeOffsets[1];
if (codeStart < 0 || codeEnd > codeBytes.lengthInBytes) {
globals.printTrace('Invalid byte index: [$codeStart, $codeEnd]');
continue;
}
final Uint8List byteView = Uint8List.view(
codeBytes.buffer,
codeStart,
codeEnd - codeStart,
);
final String fileName =
filePath.startsWith('/') ? filePath.substring(1) : filePath;
_files[fileName] = byteView;
final int sourcemapStart = sourcemapOffsets[0];
final int sourcemapEnd = sourcemapOffsets[1];
if (sourcemapStart < 0 || sourcemapEnd > sourcemapBytes.lengthInBytes) {
globals
.printTrace('Invalid byte index: [$sourcemapStart, $sourcemapEnd]');
continue;
}
final Uint8List sourcemapView = Uint8List.view(
sourcemapBytes.buffer,
sourcemapStart,
sourcemapEnd - sourcemapStart,
);
final String sourcemapName = '$fileName.map';
_sourcemaps[sourcemapName] = sourcemapView;
final int metadataStart = metadataOffsets[0];
final int metadataEnd = metadataOffsets[1];
if (metadataStart < 0 || metadataEnd > metadataBytes.lengthInBytes) {
globals
.printTrace('Invalid byte index: [$metadataStart, $metadataEnd]');
continue;
}
final Uint8List metadataView = Uint8List.view(
metadataBytes.buffer,
metadataStart,
metadataEnd - metadataStart,
);
final String metadataName = '$fileName.metadata';
_metadataFiles[metadataName] = metadataView;
modules.add(fileName);
}
_mergedMetadata = _metadataFiles.values
.map((Uint8List encoded) => utf8.decode(encoded))
.join('\n');
return modules;
_webMemoryFS.files[filePath] = contents;
}
/// Determines what rendering backed to use.
@ -681,16 +602,16 @@ class WebAssetServer implements AssetReader {
@override
Future<String> sourceMapContents(String serverPath) async {
return utf8.decode(_sourcemaps[serverPath]);
return utf8.decode(_webMemoryFS.sourcemaps[serverPath]);
}
@override
Future<String> metadataContents(String serverPath) async {
if (serverPath == 'main_module.ddc_merged_metadata') {
return _mergedMetadata;
return _webMemoryFS.mergedMetadata;
}
if (_metadataFiles.containsKey(serverPath)) {
return utf8.decode(_metadataFiles[serverPath]);
if (_webMemoryFS.metadataFiles.containsKey(serverPath)) {
return utf8.decode(_webMemoryFS.metadataFiles[serverPath]);
}
return null;
}
@ -942,18 +863,12 @@ class WebDevFS implements DevFS {
File metadataFile;
List<String> modules;
try {
final Directory parentDirectory =
globals.fs.directory(outputDirectoryPath);
codeFile =
parentDirectory.childFile('${compilerOutput.outputFilename}.sources');
manifestFile =
parentDirectory.childFile('${compilerOutput.outputFilename}.json');
sourcemapFile =
parentDirectory.childFile('${compilerOutput.outputFilename}.map');
metadataFile =
parentDirectory.childFile('${compilerOutput.outputFilename}.metadata');
modules =
webAssetServer.write(codeFile, manifestFile, sourcemapFile, metadataFile);
final Directory parentDirectory = globals.fs.directory(outputDirectoryPath);
codeFile = parentDirectory.childFile('${compilerOutput.outputFilename}.sources');
manifestFile = parentDirectory.childFile('${compilerOutput.outputFilename}.json');
sourcemapFile = parentDirectory.childFile('${compilerOutput.outputFilename}.map');
metadataFile = parentDirectory.childFile('${compilerOutput.outputFilename}.metadata');
modules = webAssetServer._webMemoryFS.write(codeFile, manifestFile, sourcemapFile, metadataFile);
} on FileSystemException catch (err) {
throwToolExit('Failed to load recompiled sources:\n$err');
}

96
packages/flutter_tools/lib/src/isolated/web_compilation_delegate.dart
View File

@ -2,20 +2,17 @@
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
import 'dart:typed_data';
import 'package:meta/meta.dart';
import '../artifacts.dart';
import '../base/common.dart';
import '../base/file_system.dart';
import '../base/utils.dart';
import '../build_info.dart';
import '../bundle.dart';
import '../compile.dart';
import '../convert.dart';
import '../globals.dart' as globals;
import '../web/compile.dart';
import '../web/memory_fs.dart';
// TODO(jonahwilliams): this class was kept around to reduce the diff in the migration
// from build_runner to the frontend_server, but should be removed/refactored to be
@ -24,7 +21,7 @@ class BuildRunnerWebCompilationProxy extends WebCompilationProxy {
BuildRunnerWebCompilationProxy();
@override
Future<WebVirtualFS> initialize({
Future<WebMemoryFS> initialize({
@required Directory projectDirectory,
@required String testOutputDir,
@required List<String> testFiles,
@ -105,93 +102,8 @@ class BuildRunnerWebCompilationProxy extends WebCompilationProxy {
final File manifestFile = outputDirectory.childFile('${output.outputFilename}.json');
final File sourcemapFile = outputDirectory.childFile('${output.outputFilename}.map');
final File metadataFile = outputDirectory.childFile('${output.outputFilename}.metadata');
final WebVirtualFS webVirtualFS = WebVirtualFS();
_write(
codeFile,
manifestFile,
sourcemapFile,
metadataFile,
webVirtualFS,
);
return webVirtualFS;
}
void _write(
File codeFile,
File manifestFile,
File sourcemapFile,
File metadataFile,
WebVirtualFS webVirtualFS,
) {
final Uint8List codeBytes = codeFile.readAsBytesSync();
final Uint8List sourcemapBytes = sourcemapFile.readAsBytesSync();
final Uint8List metadataBytes = metadataFile.readAsBytesSync();
final Map<String, dynamic> manifest =
castStringKeyedMap(json.decode(manifestFile.readAsStringSync()));
for (final String filePath in manifest.keys) {
if (filePath == null) {
globals.printTrace('Invalid manifest file: $filePath');
continue;
}
final Map<String, dynamic> offsets =
castStringKeyedMap(manifest[filePath]);
final List<int> codeOffsets =
(offsets['code'] as List<dynamic>).cast<int>();
final List<int> sourcemapOffsets =
(offsets['sourcemap'] as List<dynamic>).cast<int>();
final List<int> metadataOffsets =
(offsets['metadata'] as List<dynamic>).cast<int>();
if (codeOffsets.length != 2 ||
sourcemapOffsets.length != 2 ||
metadataOffsets.length != 2) {
globals.printTrace('Invalid manifest byte offsets: $offsets');
continue;
}
final int codeStart = codeOffsets[0];
final int codeEnd = codeOffsets[1];
if (codeStart < 0 || codeEnd > codeBytes.lengthInBytes) {
globals.printTrace('Invalid byte index: [$codeStart, $codeEnd]');
continue;
}
final Uint8List byteView = Uint8List.view(
codeBytes.buffer,
codeStart,
codeEnd - codeStart,
);
final String fileName =
filePath.startsWith('/') ? filePath.substring(1) : filePath;
webVirtualFS.files[fileName] = byteView;
final int sourcemapStart = sourcemapOffsets[0];
final int sourcemapEnd = sourcemapOffsets[1];
if (sourcemapStart < 0 || sourcemapEnd > sourcemapBytes.lengthInBytes) {
globals.printTrace('Invalid byte index: [$sourcemapStart, $sourcemapEnd]');
continue;
}
final Uint8List sourcemapView = Uint8List.view(
sourcemapBytes.buffer,
sourcemapStart,
sourcemapEnd - sourcemapStart,
);
final String sourcemapName = '$fileName.map';
webVirtualFS.sourcemaps[sourcemapName] = sourcemapView;
final int metadataStart = metadataOffsets[0];
final int metadataEnd = metadataOffsets[1];
if (metadataStart < 0 || metadataEnd > metadataBytes.lengthInBytes) {
globals
.printTrace('Invalid byte index: [$metadataStart, $metadataEnd]');
continue;
}
final Uint8List metadataView = Uint8List.view(
metadataBytes.buffer,
metadataStart,
metadataEnd - metadataStart,
);
final String metadataName = '$fileName.metadata';
webVirtualFS.metadataFiles[metadataName] = metadataView;
}
return WebMemoryFS()
..write(codeFile, manifestFile, sourcemapFile, metadataFile);
}
String _generateEntrypoint(String relativeTestPath, String absolutePath) {

14
packages/flutter_tools/lib/src/test/flutter_web_platform.dart
View File

@ -33,7 +33,7 @@ import '../dart/package_map.dart';
import '../globals.dart' as globals;
import '../project.dart';
import '../web/chrome.dart';
import '../web/compile.dart';
import '../web/memory_fs.dart';
import 'test_compiler.dart';
import 'test_config.dart';
@ -43,7 +43,7 @@ class FlutterWebPlatform extends PlatformPlugin {
String shellPath,
this.updateGoldens,
@required this.buildInfo,
@required this.webVirtualFS,
@required this.webMemoryFS,
}) {
final shelf.Cascade cascade = shelf.Cascade()
.add(_webSocketHandler.handler)
@ -68,7 +68,7 @@ class FlutterWebPlatform extends PlatformPlugin {
);
}
final WebVirtualFS webVirtualFS;
final WebMemoryFS webMemoryFS;
final BuildInfo buildInfo;
static Future<FlutterWebPlatform> start(String root, {
@ -77,7 +77,7 @@ class FlutterWebPlatform extends PlatformPlugin {
bool updateGoldens = false,
bool pauseAfterLoad = false,
@required BuildInfo buildInfo,
@required WebVirtualFS webVirtualFS,
@required WebMemoryFS webMemoryFS,
}) async {
final shelf_io.IOServer server = shelf_io.IOServer(await HttpMultiServer.loopback(0));
return FlutterWebPlatform._(
@ -88,7 +88,7 @@ class FlutterWebPlatform extends PlatformPlugin {
shellPath: shellPath,
updateGoldens: updateGoldens,
buildInfo: buildInfo,
webVirtualFS: webVirtualFS,
webMemoryFS: webMemoryFS,
);
}
@ -188,12 +188,12 @@ class FlutterWebPlatform extends PlatformPlugin {
}
if (request.url.path.endsWith('.dart.js')) {
final String path = request.url.path.split('.dart.js')[0];
return shelf.Response.ok(webVirtualFS.files[path + '.dart.lib.js'], headers: <String, String>{
return shelf.Response.ok(webMemoryFS.files[path + '.dart.lib.js'], headers: <String, String>{
HttpHeaders.contentTypeHeader: 'text/javascript',
});
}
if (request.url.path.endsWith('.lib.js.map')) {
return shelf.Response.ok(webVirtualFS.sourcemaps[request.url.path], headers: <String, String>{
return shelf.Response.ok(webMemoryFS.sourcemaps[request.url.path], headers: <String, String>{
HttpHeaders.contentTypeHeader: 'text/plain',
});
}

5
packages/flutter_tools/lib/src/test/runner.dart
View File

@ -12,6 +12,7 @@ import '../build_info.dart';
import '../globals.dart' as globals;
import '../project.dart';
import '../web/compile.dart';
import '../web/memory_fs.dart';
import 'flutter_platform.dart' as loader;
import 'flutter_web_platform.dart';
import 'test_wrapper.dart';
@ -121,7 +122,7 @@ class _FlutterTestRunnerImpl implements FlutterTestRunner {
.absolute
.uri
.toFilePath();
final WebVirtualFS result = await webCompilationProxy.initialize(
final WebMemoryFS result = await webCompilationProxy.initialize(
projectDirectory: flutterProject.directory,
testOutputDir: tempBuildDir,
testFiles: testFiles,
@ -145,7 +146,7 @@ class _FlutterTestRunnerImpl implements FlutterTestRunner {
flutterProject: flutterProject,
pauseAfterLoad: startPaused,
buildInfo: buildInfo,
webVirtualFS: result,
webMemoryFS: result,
);
},
);

12
packages/flutter_tools/lib/src/web/compile.dart
View File

@ -2,8 +2,6 @@
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
import 'dart:typed_data';
import 'package:meta/meta.dart';
import '../base/common.dart';
@ -20,6 +18,7 @@ import '../globals.dart' as globals;
import '../platform_plugins.dart';
import '../plugins.dart';
import '../project.dart';
import '../web/memory_fs.dart';
/// The [WebCompilationProxy] instance.
WebCompilationProxy get webCompilationProxy => context.get<WebCompilationProxy>();
@ -98,7 +97,7 @@ class WebCompilationProxy {
const WebCompilationProxy();
/// Initialize the web compiler from the `projectDirectory`.
Future<WebVirtualFS> initialize({
Future<WebMemoryFS> initialize({
@required Directory projectDirectory,
@required String testOutputDir,
@required List<String> testFiles,
@ -107,10 +106,3 @@ class WebCompilationProxy {
throw UnimplementedError();
}
}
class WebVirtualFS {
final Map<String, Uint8List> metadataFiles = <String, Uint8List>{};
final Map<String, Uint8List> files = <String, Uint8List>{};
final Map<String, Uint8List> sourcemaps = <String, Uint8List>{};
}

106
packages/flutter_tools/lib/src/web/memory_fs.dart Normal file
View File

@ -0,0 +1,106 @@
// Copyright 2014 The Flutter Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
import 'dart:typed_data';
import '../base/file_system.dart';
import '../base/utils.dart';
import '../convert.dart';
/// A psuedo-filesystem stored in memory.
///
/// To support output to arbitrary multi-root file schemes, the frontend server
/// will output web sources, sourcemaps, and metadata to concatenated single files
/// with an additional manifest file containing the correct offsets.
class WebMemoryFS {
final Map<String, Uint8List> metadataFiles = <String, Uint8List>{};
final Map<String, Uint8List> files = <String, Uint8List>{};
final Map<String, Uint8List> sourcemaps = <String, Uint8List>{};
String get mergedMetadata => _mergedMetadata;
String _mergedMetadata;
/// Update the filesystem with the provided source and manifest files.
///
/// Returns the list of updated files.
List<String> write(
File codeFile,
File manifestFile,
File sourcemapFile,
File metadataFile,
) {
final List<String> modules = <String>[];
final Uint8List codeBytes = codeFile.readAsBytesSync();
final Uint8List sourcemapBytes = sourcemapFile.readAsBytesSync();
final Uint8List metadataBytes = metadataFile.readAsBytesSync();
final Map<String, dynamic> manifest =
castStringKeyedMap(json.decode(manifestFile.readAsStringSync()));
for (final String filePath in manifest.keys) {
if (filePath == null) {
continue;
}
final Map<String, dynamic> offsets =
castStringKeyedMap(manifest[filePath]);
final List<int> codeOffsets =
(offsets['code'] as List<dynamic>).cast<int>();
final List<int> sourcemapOffsets =
(offsets['sourcemap'] as List<dynamic>).cast<int>();
final List<int> metadataOffsets =
(offsets['metadata'] as List<dynamic>).cast<int>();
if (codeOffsets.length != 2 ||
sourcemapOffsets.length != 2 ||
metadataOffsets.length != 2) {
continue;
}
final int codeStart = codeOffsets[0];
final int codeEnd = codeOffsets[1];
if (codeStart < 0 || codeEnd > codeBytes.lengthInBytes) {
continue;
}
final Uint8List byteView = Uint8List.view(
codeBytes.buffer,
codeStart,
codeEnd - codeStart,
);
final String fileName =
filePath.startsWith('/') ? filePath.substring(1) : filePath;
files[fileName] = byteView;
final int sourcemapStart = sourcemapOffsets[0];
final int sourcemapEnd = sourcemapOffsets[1];
if (sourcemapStart < 0 || sourcemapEnd > sourcemapBytes.lengthInBytes) {
continue;
}
final Uint8List sourcemapView = Uint8List.view(
sourcemapBytes.buffer,
sourcemapStart,
sourcemapEnd - sourcemapStart,
);
final String sourcemapName = '$fileName.map';
sourcemaps[sourcemapName] = sourcemapView;
final int metadataStart = metadataOffsets[0];
final int metadataEnd = metadataOffsets[1];
if (metadataStart < 0 || metadataEnd > metadataBytes.lengthInBytes) {
continue;
}
final Uint8List metadataView = Uint8List.view(
metadataBytes.buffer,
metadataStart,
metadataEnd - metadataStart,
);
final String metadataName = '$fileName.metadata';
metadataFiles[metadataName] = metadataView;
modules.add(fileName);
}
_mergedMetadata = metadataFiles.values
.map((Uint8List encoded) => utf8.decode(encoded))
.join('\n');
return modules;
}
}

36
packages/flutter_tools/test/general.shard/web/memory_fs_test.dart Normal file
View File

@ -0,0 +1,36 @@
// Copyright 2014 The Flutter Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
import 'dart:convert';
import 'package:file/memory.dart';
import 'package:flutter_tools/src/base/file_system.dart';
import 'package:flutter_tools/src/web/memory_fs.dart';
import '../../src/common.dart';
void main() {
testWithoutContext('correctly parses source, source map, metadata, manifest files', () {
final MemoryFileSystem fileSystem = MemoryFileSystem();
final File source = fileSystem.file('source')
..writeAsStringSync('main() {}');
final File sourcemap = fileSystem.file('sourcemap')
..writeAsStringSync('{}');
final File metadata = fileSystem.file('metadata')
..writeAsStringSync('{}');
final File manifest = fileSystem.file('manifest')
..writeAsStringSync(json.encode(<String, Object>{'/foo.js': <String, Object>{
'code': <int>[0, source.lengthSync()],
'sourcemap': <int>[0, 2],
'metadata': <int>[0, 2],
}}));
final WebMemoryFS webMemoryFS = WebMemoryFS();
webMemoryFS.write(source, manifest, sourcemap, metadata);
expect(utf8.decode(webMemoryFS.files['foo.js']), 'main() {}');
expect(utf8.decode(webMemoryFS.sourcemaps['foo.js.map']), '{}');
expect(utf8.decode(webMemoryFS.metadataFiles['foo.js.metadata']), '{}');
expect(webMemoryFS.mergedMetadata, '{}');
});
}