Make ProjectFileInvalidator.findInvalidated able to use the async FileStat.stat (#42028)

Empirical measurements indicate on the network file system we use
internally, using `FileStat.stat` on thousands of files is much
faster than using `FileStat.statSync`. (It can be slower for files
on a local SSD, however.)

Add a flag to `ProjectFileInvalidator.findInvalidated` to let it
use `FileStat.stat` instead of `FileStat.statSync` when scanning for
modified files.  This can be enabled by overriding `HotRunnerConfig`.

I considered creating a separate, asynchronous version of
`findInvalidated`, but that led to more code duplication than I
liked, and it would be harder to avoid drift between the versions.

packages/flutter_tools/lib/src/run_hot.dart

@@ -7,6 +7,7 @@ import 'dart:async';
 import 'package:json_rpc_2/error_code.dart' as rpc_error_code;
 import 'package:json_rpc_2/json_rpc_2.dart' as rpc;
 import 'package:meta/meta.dart';
+import 'package:pool/pool.dart';
 
 import 'base/async_guard.dart';
 import 'base/common.dart';
@@ -29,6 +30,10 @@ import 'vmservice.dart';
 class HotRunnerConfig {
   /// Should the hot runner assume that the minimal Dart dependencies do not change?
   bool stableDartDependencies = false;
+
+  /// Whether the hot runner should scan for modified files asynchronously.
+  bool asyncScanning = false;
+
   /// A hook for implementations to perform any necessary initialization prior
   /// to a hot restart. Should return true if the hot restart should continue.
   Future<bool> setupHotRestart() async {
@@ -296,10 +301,11 @@ class HotRunner extends ResidentRunner {
 
     // Picking up first device's compiler as a source of truth - compilers
     // for all devices should be in sync.
-    final List<Uri> invalidatedFiles = ProjectFileInvalidator.findInvalidated(
+    final List<Uri> invalidatedFiles = await ProjectFileInvalidator.findInvalidated(
       lastCompiled: flutterDevices[0].devFS.lastCompiled,
       urisToMonitor: flutterDevices[0].devFS.sources,
       packagesPath: packagesFilePath,
+      asyncScanning: hotRunnerConfig.asyncScanning,
     );
     final UpdateFSReport results = UpdateFSReport(success: true);
     for (FlutterDevice device in flutterDevices) {
@@ -1044,11 +1050,20 @@ class ProjectFileInvalidator {
   static const String _pubCachePathLinuxAndMac = '.pub-cache';
   static const String _pubCachePathWindows = 'Pub/Cache';
 
-  static List<Uri> findInvalidated({
+  // As of writing, Dart supports up to 32 asynchronous I/O threads per
+  // isolate.  We also want to avoid hitting platform limits on open file
+  // handles/descriptors.
+  //
+  // This value was chosen based on empirical tests scanning a set of
+  // ~2000 files.
+  static const int _kMaxPendingStats = 8;
+
+  static Future<List<Uri>> findInvalidated({
     @required DateTime lastCompiled,
     @required List<Uri> urisToMonitor,
     @required String packagesPath,
-  }) {
+    bool asyncScanning = false,
+  }) async {
     assert(urisToMonitor != null);
     assert(packagesPath != null);
 
@@ -1068,17 +1083,36 @@ class ProjectFileInvalidator {
       fs.file(packagesPath).uri,
     ];
     final List<Uri> invalidatedFiles = <Uri>[];
-    for (final Uri uri in urisToScan) {
+
-      final DateTime updatedAt = fs.statSync(
+    if (asyncScanning) {
-        uri.toFilePath(windows: platform.isWindows),
+      final Pool pool = Pool(_kMaxPendingStats);
-      ).modified;
+      final List<Future<void>> waitList = <Future<void>>[];
-      if (updatedAt != null && updatedAt.isAfter(lastCompiled)) {
+      for (final Uri uri in urisToScan) {
-        invalidatedFiles.add(uri);
+        waitList.add(pool.withResource<void>(
+          () => fs
+            .stat(uri.toFilePath(windows: platform.isWindows))
+            .then((FileStat stat) {
+              final DateTime updatedAt = stat.modified;
+              if (updatedAt != null && updatedAt.isAfter(lastCompiled)) {
+                invalidatedFiles.add(uri);
+              }
+            })
+        ));
+      }
+      await Future.wait<void>(waitList);
+    } else {
+      for (final Uri uri in urisToScan) {
+        final DateTime updatedAt = fs.statSync(
+            uri.toFilePath(windows: platform.isWindows)).modified;
+        if (updatedAt != null && updatedAt.isAfter(lastCompiled)) {
+          invalidatedFiles.add(uri);
+        }
       }
     }
     printTrace(
       'Scanned through ${urisToScan.length} files in '
-      '${stopwatch.elapsedMilliseconds}ms',
+      '${stopwatch.elapsedMilliseconds}ms'
+      '${asyncScanning ? " (async)" : ""}',
     );
     return invalidatedFiles;
   }

packages/flutter_tools/test/general.shard/project_file_invalidator_test.dart

@@ -5,6 +5,7 @@
 import 'package:file/memory.dart';
 import 'package:flutter_tools/src/base/file_system.dart';
 import 'package:flutter_tools/src/run_hot.dart';
+import 'package:meta/meta.dart';
 
 import '../src/common.dart';
 import '../src/context.dart';
@@ -14,43 +15,53 @@ final DateTime inFuture = DateTime.now().add(const Duration(days: 100));
 
 void main() {
   group('ProjectFileInvalidator', () {
-    testUsingContext('No last compile', () async {
+    _testProjectFileInvalidator(asyncScanning: false);
-      expect(
+  });
-        ProjectFileInvalidator.findInvalidated(
+  group('ProjectFileInvalidator (async scanning)', () {
-          lastCompiled: null,
+    _testProjectFileInvalidator(asyncScanning: true);
-          urisToMonitor: <Uri>[],
+  });
-          packagesPath: '',
+}
-        ),
+
-        isEmpty,
+void _testProjectFileInvalidator({@required bool asyncScanning}) {
-      );
+  testUsingContext('No last compile', () async {
-    });
+    expect(
-
+      await ProjectFileInvalidator.findInvalidated(
-    testUsingContext('Empty project', () async {
+        lastCompiled: null,
-      expect(
+        urisToMonitor: <Uri>[],
-        ProjectFileInvalidator.findInvalidated(
+        packagesPath: '',
-          lastCompiled: inFuture,
+        asyncScanning: asyncScanning,
-          urisToMonitor: <Uri>[],
+      ),
-          packagesPath: '',
+      isEmpty,
-        ),
+    );
-        isEmpty,
+  });
-      );
+
-    }, overrides: <Type, Generator>{
+  testUsingContext('Empty project', () async {
-      FileSystem: () => MemoryFileSystem(),
+    expect(
-      ProcessManager: () => FakeProcessManager(<FakeCommand>[]),
+      await ProjectFileInvalidator.findInvalidated(
-    });
+        lastCompiled: inFuture,
-
+        urisToMonitor: <Uri>[],
-    testUsingContext('Non-existent files are ignored', () async {
+        packagesPath: '',
-      expect(
+        asyncScanning: asyncScanning,
-        ProjectFileInvalidator.findInvalidated(
+      ),
-          lastCompiled: inFuture,
+      isEmpty,
-          urisToMonitor: <Uri>[Uri.parse('/not-there-anymore'),],
+    );
-          packagesPath: '',
+  }, overrides: <Type, Generator>{
-        ),
+    FileSystem: () => MemoryFileSystem(),
-        isEmpty,
+    ProcessManager: () => FakeProcessManager(<FakeCommand>[]),
-      );
+  });
-    }, overrides: <Type, Generator>{
+
-      FileSystem: () => MemoryFileSystem(),
+  testUsingContext('Non-existent files are ignored', () async {
-      ProcessManager: () => FakeProcessManager(<FakeCommand>[]),
+    expect(
-    });
+      await ProjectFileInvalidator.findInvalidated(
+        lastCompiled: inFuture,
+        urisToMonitor: <Uri>[Uri.parse('/not-there-anymore'),],
+        packagesPath: '',
+        asyncScanning: asyncScanning,
+      ),
+      isEmpty,
+    );
+  }, overrides: <Type, Generator>{
+    FileSystem: () => MemoryFileSystem(),
+    ProcessManager: () => FakeProcessManager(<FakeCommand>[]),
   });
 }