Restructure hot mode test so it runs interactively. (#12519)

* Restructure hot mode test so it runs interactively.

This allows to add a benchmark for hot reload after actual source code change.

* Add curly braces, refactory copyRecursive

dev/devicelab/lib/framework/utils.dart

@@ -98,6 +98,21 @@ void copy(File sourceFile, Directory targetDirectory, {String name}) {
   target.writeAsBytesSync(sourceFile.readAsBytesSync());
 }
 
+void recursiveCopy(Directory source, Directory target) {
+  if (!target.existsSync())
+    target.createSync();
+
+  for (FileSystemEntity entity in source.listSync(followLinks: false)) {
+    final String name = path.basename(entity.path);
+    if (entity is Directory)
+      recursiveCopy(entity, new Directory(path.join(target.path, name)));
+    else if (entity is File) {
+      final File dest = new File(path.join(target.path, name));
+      dest.writeAsBytesSync(entity.readAsBytesSync());
+    }
+  }
+}
+
 FileSystemEntity move(FileSystemEntity whatToMove,
     {Directory to, String name}) {
   return whatToMove

dev/devicelab/lib/tasks/hot_mode_tests.dart

@@ -2,6 +2,8 @@
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
+import 'dart:async';
+import 'dart:convert';
 import 'dart:io';
 
 import 'package:path/path.dart' as path;
@@ -10,30 +12,81 @@ import '../framework/adb.dart';
 import '../framework/framework.dart';
 import '../framework/utils.dart';
 
+final Directory _editedFlutterGalleryDir = dir(path.join(Directory.systemTemp.path, 'edited_flutter_gallery'));
+final Directory flutterGalleryDir = dir(path.join(flutterDirectory.path, 'examples/flutter_gallery'));
+
 TaskFunction createHotModeTest({ bool isPreviewDart2: false }) {
   return () async {
     final Device device = await devices.workingDevice;
     await device.unlock();
-    final Directory appDir =
+    final File benchmarkFile = file(path.join(_editedFlutterGalleryDir.path, 'hot_benchmark.json'));
-    dir(path.join(flutterDirectory.path, 'examples/flutter_gallery'));
-    final File benchmarkFile = file(path.join(appDir.path, 'hot_benchmark.json'));
     rm(benchmarkFile);
     final List<String> options = <String>[
-      '--hot', '-d', device.deviceId, '--benchmark', '--verbose'
+      '--hot', '-d', device.deviceId, '--benchmark', '--verbose', '--resident'
     ];
     if (isPreviewDart2)
       options.add('--preview-dart-2');
-    await inDirectory(appDir, () async {
+    int hotReloadCount = 0;
-      return await flutter('run', options: options, canFail: false);
+    await inDirectory(flutterDirectory, () async {
+      rmTree(_editedFlutterGalleryDir);
+      mkdirs(_editedFlutterGalleryDir);
+      recursiveCopy(flutterGalleryDir, _editedFlutterGalleryDir);
+      await inDirectory(_editedFlutterGalleryDir, () async {
+        final Process process = await startProcess(
+          path.join(flutterDirectory.path, 'bin', 'flutter'),
+          <String>['run']..addAll(options),
+          environment: null
+        );
+
+        final Completer<Null> stdoutDone = new Completer<Null>();
+        final Completer<Null> stderrDone = new Completer<Null>();
+        process.stdout
+          .transform(UTF8.decoder)
+          .transform(const LineSplitter())
+          .listen((String line) {
+          if (line.contains('\] Hot reload performed in')) {
+            if (hotReloadCount == 0) {
+              // Update the file and reload again.
+              final File appDartSource = file(path.join(
+                _editedFlutterGalleryDir.path, 'lib/gallery/app.dart'
+              ));
+              appDartSource.writeAsStringSync(
+                appDartSource.readAsStringSync().replaceFirst(
+                  "'Flutter Gallery'", "'Updated Flutter Gallery'"
+                )
+              );
+              process.stdin.writeln('r');
+              ++hotReloadCount;
+            } else {
+              // Quit after second hot reload.
+              process.stdin.writeln('q');
+            }
+          }
+          print('stdout: $line');
+        }, onDone: () { stdoutDone.complete(); });
+        process.stderr
+          .transform(UTF8.decoder)
+          .transform(const LineSplitter())
+          .listen((String line) {
+          print('stderr: $line');
+        }, onDone: () { stderrDone.complete(); });
+
+        await Future.wait<Null>(<Future<Null>>[stdoutDone.future, stderrDone.future]);
+        return await process.exitCode;
+      });
+    });
+    final Map<String, dynamic> twoReloadsData = JSON.decode(
+      benchmarkFile.readAsStringSync());
+    return new TaskResult.success(<String, dynamic> {
+      'hotReloadInitialDevFSSyncMilliseconds': twoReloadsData['hotReloadInitialDevFSSyncMilliseconds'][0],
+      'hotRestartMillisecondsToFrame': twoReloadsData['hotRestartMillisecondsToFrame'][0],
+      'hotReloadMillisecondsToFrame' : twoReloadsData['hotReloadMillisecondsToFrame'][0],
+      'hotReloadDevFSSyncMilliseconds': twoReloadsData['hotReloadDevFSSyncMilliseconds'][0],
+      'hotReloadFlutterReassembleMilliseconds': twoReloadsData['hotReloadFlutterReassembleMilliseconds'][0],
+      'hotReloadVMReloadMilliseconds': twoReloadsData['hotReloadVMReloadMilliseconds'][0],
+      'hotReloadDevFSSyncMillisecondsAfterChange': twoReloadsData['hotReloadDevFSSyncMilliseconds'][1],
+      'hotReloadFlutterReassembleMillisecondsAfterChange': twoReloadsData['hotReloadFlutterReassembleMilliseconds'][1],
+      'hotReloadVMReloadMillisecondsAfterChange': twoReloadsData['hotReloadVMReloadMilliseconds'][1]
     });
-    return new TaskResult.successFromFile(benchmarkFile,
-        benchmarkScoreKeys: <String>[
-          'hotReloadInitialDevFSSyncMilliseconds',
-          'hotReloadMillisecondsToFrame',
-          'hotRestartMillisecondsToFrame',
-          'hotReloadDevFSSyncMilliseconds',
-          'hotReloadFlutterReassembleMilliseconds',
-          'hotReloadVMReloadMilliseconds',
-        ]);
   };
 }

packages/flutter_tools/lib/src/run_hot.dart

@@ -57,11 +57,16 @@ class HotRunner extends ResidentRunner {
   Set<String> _dartDependencies;
 
   final bool benchmarkMode;
-  final Map<String, int> benchmarkData = <String, int>{};
+  final Map<String, List<int>> benchmarkData = <String, List<int>>{};
   // The initial launch is from a snapshot.
   bool _runningFromSnapshot = true;
   bool previewDart2 = false;
 
+  void _addBenchmarkData(String name, int value) {
+    benchmarkData[name] ??= <int>[];
+    benchmarkData[name].add(value);
+  }
+
   bool _refreshDartDependencies() {
     if (!hotRunnerConfig.computeDartDependencies) {
       // Disabled.
@@ -130,8 +135,8 @@ class HotRunner extends ResidentRunner {
     }
     final Stopwatch initialUpdateDevFSsTimer = new Stopwatch()..start();
     final bool devfsResult = await _updateDevFS();
-    benchmarkData['hotReloadInitialDevFSSyncMilliseconds'] =
+    _addBenchmarkData('hotReloadInitialDevFSSyncMilliseconds',
-        initialUpdateDevFSsTimer.elapsed.inMilliseconds;
+        initialUpdateDevFSsTimer.elapsed.inMilliseconds);
     if (!devfsResult)
       return 3;
 
@@ -162,12 +167,17 @@ class HotRunner extends ResidentRunner {
       printStatus('Benchmarking hot reload');
       // Measure time to perform a hot reload.
       await restart(fullRestart: false);
-      printStatus('Benchmark completed. Exiting application.');
+      if (stayResident) {
-      await _cleanupDevFS();
+        await waitForAppToFinish();
-      await stopEchoingDeviceLog();
+      } else {
-      await stopApp();
+        printStatus('Benchmark completed. Exiting application.');
+        await _cleanupDevFS();
+        await stopEchoingDeviceLog();
+        await stopApp();
+      }
       final File benchmarkOutput = fs.file('hot_benchmark.json');
       benchmarkOutput.writeAsStringSync(toPrettyJson(benchmarkData));
+      return 0;
     }
 
     if (stayResident)
@@ -362,8 +372,8 @@ class HotRunner extends ResidentRunner {
         '${getElapsedAsMilliseconds(restartTimer.elapsed)}.');
     // We are now running from sources.
     _runningFromSnapshot = false;
-    benchmarkData['hotRestartMillisecondsToFrame'] =
+    _addBenchmarkData('hotRestartMillisecondsToFrame',
-        restartTimer.elapsed.inMilliseconds;
+        restartTimer.elapsed.inMilliseconds);
     flutterUsage.sendEvent('hot', 'restart');
     flutterUsage.sendTiming('hot', 'restart', restartTimer.elapsed);
     return OperationResult.ok;
@@ -467,11 +477,10 @@ class HotRunner extends ResidentRunner {
     final Stopwatch devFSTimer = new Stopwatch()..start();
     final bool updatedDevFS = await _updateDevFS();
     // Record time it took to synchronize to DevFS.
-    benchmarkData['hotReloadDevFSSyncMilliseconds'] =
+    _addBenchmarkData('hotReloadDevFSSyncMilliseconds',
-        devFSTimer.elapsed.inMilliseconds;
+        devFSTimer.elapsed.inMilliseconds);
     if (!updatedDevFS)
       return new OperationResult(1, 'DevFS synchronization failed');
-
     String reloadMessage;
     final Stopwatch vmReloadTimer = new Stopwatch()..start();
     try {
@@ -536,8 +545,8 @@ class HotRunner extends ResidentRunner {
       return new OperationResult(errorCode, errorMessage);
     }
     // Record time it took for the VM to reload the sources.
-    benchmarkData['hotReloadVMReloadMilliseconds'] =
+    _addBenchmarkData('hotReloadVMReloadMilliseconds',
-        vmReloadTimer.elapsed.inMilliseconds;
+        vmReloadTimer.elapsed.inMilliseconds);
     final Stopwatch reassembleTimer = new Stopwatch()..start();
     // Reload the isolate.
     for (FlutterDevice device in flutterDevices) {
@@ -595,15 +604,15 @@ class HotRunner extends ResidentRunner {
       }
     }
     // Record time it took for Flutter to reassemble the application.
-    benchmarkData['hotReloadFlutterReassembleMilliseconds'] =
+    _addBenchmarkData('hotReloadFlutterReassembleMilliseconds',
-        reassembleTimer.elapsed.inMilliseconds;
+        reassembleTimer.elapsed.inMilliseconds);
 
     reloadTimer.stop();
     printTrace('Hot reload performed in '
         '${getElapsedAsMilliseconds(reloadTimer.elapsed)}.');
     // Record complete time it took for the reload.
-    benchmarkData['hotReloadMillisecondsToFrame'] =
+    _addBenchmarkData('hotReloadMillisecondsToFrame',
-        reloadTimer.elapsed.inMilliseconds;
+        reloadTimer.elapsed.inMilliseconds);
     // Only report timings if we reloaded a single view without any
     // errors or timeouts.
     if ((reassembleViews.length == 1) &&