print intermediate and raw A/B results when not silent (#54676)

* print intermediate A/B results when not silent
* print raw A/B results when in loud mode
* add tests; handle missing metrics more gracefully
* use less fancy section header on Windows

dev/devicelab/bin/run.dart

@@ -162,7 +162,19 @@ Future<void> _runABTest() async {
     }
 
     abTest.addBResult(localEngineResult);
+
+    if (!silent && i < runsPerTest) {
+      section('A/B results so far');
+      print(abTest.printSummary());
+    }
   }
+
+  if (!silent) {
+    section('Raw results');
+    print(abTest.rawResults());
+  }
+
+  section('Final A/B results');
   print(abTest.printSummary());
 }
 

dev/devicelab/bin/tasks/smoke_test_success.dart

@@ -9,6 +9,13 @@ import 'package:flutter_devicelab/framework/framework.dart';
 /// Smoke test of a successful task.
 Future<void> main() async {
   await task(() async {
-    return TaskResult.success(<String, dynamic>{});
+    return TaskResult.success(<String, dynamic>{
+      'metric1': 42,
+      'metric2': 123,
+      'not_a_metric': 'something',
+    }, benchmarkScoreKeys: <String>[
+      'metric1',
+      'metric2',
+    ]);
   });
 }

dev/devicelab/lib/framework/ab.dart

@@ -30,22 +30,54 @@ class ABTest {
     _addResult(result, _bResults);
   }
 
+  /// Returns unprocessed data collected by the A/B test formatted as
+  /// a tab-separated spreadsheet.
+  String rawResults() {
+    final StringBuffer buffer = StringBuffer();
+    for (final String scoreKey in _allScoreKeys) {
+      buffer.writeln('$scoreKey:');
+      buffer.write('  A:\t');
+      if (_aResults.containsKey(scoreKey)) {
+        for (final double score in _aResults[scoreKey]) {
+          buffer.write('${score.toStringAsFixed(2)}\t');
+        }
+      } else {
+        buffer.write('N/A');
+      }
+      buffer.writeln();
+
+      buffer.write('  B:\t');
+      if (_bResults.containsKey(scoreKey)) {
+        for (final double score in _bResults[scoreKey]) {
+          buffer.write('${score.toStringAsFixed(2)}\t');
+        }
+      } else {
+        buffer.write('N/A');
+      }
+      buffer.writeln();
+    }
+    return buffer.toString();
+  }
+
+  Set<String> get _allScoreKeys {
+    return <String>{
+      ..._aResults.keys,
+      ..._bResults.keys,
+    };
+  }
+
   /// Returns the summary as a tab-separated spreadsheet.
   ///
   /// This value can be copied straight to a Google Spreadsheet for further analysis.
   String printSummary() {
     final Map<String, _ScoreSummary> summariesA = _summarize(_aResults);
     final Map<String, _ScoreSummary> summariesB = _summarize(_bResults);
-    final Set<String> scoreKeyUnion = <String>{
-      ...summariesA.keys,
-      ...summariesB.keys,
-    };
 
     final StringBuffer buffer = StringBuffer(
       'Score\tAverage A (noise)\tAverage B (noise)\tSpeed-up\n',
     );
 
-    for (final String scoreKey in scoreKeyUnion) {
+    for (final String scoreKey in _allScoreKeys) {
       final _ScoreSummary summaryA = summariesA[scoreKey];
       final _ScoreSummary summaryB = summariesB[scoreKey];
       buffer.write('$scoreKey\t');

dev/devicelab/lib/framework/utils.dart

@@ -189,11 +189,18 @@ void mkdirs(Directory directory) {
 bool exists(FileSystemEntity entity) => entity.existsSync();
 
 void section(String title) {
-  title = '╡ ••• $title ••• ╞';
-  final String line = '═' * math.max((80 - title.length) ~/ 2, 2);
-  String output = '$line$title$line';
-  if (output.length == 79)
-    output += '═';
+  String output;
+  if (Platform.isWindows) {
+    // Windows doesn't cope well with characters produced for *nix systems, so
+    // just output the title with no decoration.
+    output = title;
+  } else {
+    title = '╡ ••• $title ••• ╞';
+    final String line = '═' * math.max((80 - title.length) ~/ 2, 2);
+    output = '$line$title$line';
+    if (output.length == 79)
+      output += '═';
+  }
   print('\n\n$output\n');
 }
 

dev/devicelab/test/ab_test.dart

@@ -0,0 +1,51 @@
+// Copyright 2014 The Flutter Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+import 'package:flutter_devicelab/framework/ab.dart';
+
+import 'common.dart';
+
+void main() {
+  test('ABTest', () {
+    final ABTest ab = ABTest();
+
+    for (int i = 0; i < 5; i++) {
+      ab.addAResult(<String, dynamic>{
+        'data': <String, dynamic>{
+          'i': i,
+          'j': 10 * i,
+          'not_a_metric': 'something',
+        },
+        'benchmarkScoreKeys': <String>['i', 'j'],
+      });
+
+      ab.addBResult(<String, dynamic>{
+        'data': <String, dynamic>{
+          'i': i + 1,
+          'k': 10 * i + 1,
+        },
+        'benchmarkScoreKeys': <String>['i', 'k'],
+      });
+    }
+
+    expect(
+      ab.rawResults(),
+      'i:\n'
+      '  A:\t0.00\t1.00\t2.00\t3.00\t4.00\t\n'
+      '  B:\t1.00\t2.00\t3.00\t4.00\t5.00\t\n'
+      'j:\n'
+      '  A:\t0.00\t10.00\t20.00\t30.00\t40.00\t\n'
+      '  B:\tN/A\n'
+      'k:\n'
+      '  A:\tN/A\n'
+      '  B:\t1.00\t11.00\t21.00\t31.00\t41.00\t\n',
+    );
+    expect(
+        ab.printSummary(),
+        'Score\tAverage A (noise)\tAverage B (noise)\tSpeed-up\n'
+        'i\t2.00 (70.71%)\t3.00 (47.14%)\t0.67x\t\n'
+        'j\t20.00 (70.71%)\t\t\n'
+        'k\t\t21.00 (67.34%)\t\n');
+  });
+}

dev/devicelab/test/run_test.dart

@@ -14,21 +14,26 @@ void main() {
   const ProcessManager processManager = LocalProcessManager();
 
   group('run.dart script', () {
-    Future<ProcessResult> runScript(List<String> testNames) async {
-      final String dart = path.absolute(path.join('..', '..', 'bin', 'cache', 'dart-sdk', 'bin', 'dart'));
+    Future<ProcessResult> runScript(List<String> testNames,
+        [List<String> otherArgs = const <String>[]]) async {
+      final String dart = path.absolute(
+          path.join('..', '..', 'bin', 'cache', 'dart-sdk', 'bin', 'dart'));
       final ProcessResult scriptProcess = processManager.runSync(<String>[
         dart,
         'bin/run.dart',
+        ...otherArgs,
         for (final String testName in testNames) ...<String>['-t', testName],
       ]);
       return scriptProcess;
     }
 
-    Future<void> expectScriptResult(List<String> testNames, int expectedExitCode) async {
+    Future<void> expectScriptResult(
+        List<String> testNames, int expectedExitCode) async {
       final ProcessResult result = await runScript(testNames);
       expect(result.exitCode, expectedExitCode,
-          reason: '[ stderr from test process ]\n\n${result.stderr}\n\n[ end of stderr ]'
-          '\n\n[ stdout from test process ]\n\n${result.stdout}\n\n[ end of stdout ]');
+          reason:
+              '[ stderr from test process ]\n\n${result.stderr}\n\n[ end of stderr ]'
+              '\n\n[ stdout from test process ]\n\n${result.stdout}\n\n[ end of stdout ]');
     }
 
     test('exits with code 0 when succeeds', () async {
@@ -36,7 +41,8 @@ void main() {
     });
 
     test('accepts file paths', () async {
-      await expectScriptResult(<String>['bin/tasks/smoke_test_success.dart'], 0);
+      await expectScriptResult(
+          <String>['bin/tasks/smoke_test_success.dart'], 0);
     });
 
     test('rejects invalid file paths', () async {
@@ -56,12 +62,66 @@ void main() {
     }, skip: true); // https://github.com/flutter/flutter/issues/53707
 
     test('exits with code 1 when results are mixed', () async {
-      await expectScriptResult(<String>[
+      await expectScriptResult(
+        <String>[
           'smoke_test_failure',
           'smoke_test_success',
         ],
         1,
       );
     });
+
+    test('runs A/B test', () async {
+      final ProcessResult result = await runScript(
+        <String>['smoke_test_success'],
+        <String>['--ab=2', '--local-engine=host_debug_unopt'],
+      );
+      expect(result.exitCode, 0);
+
+      String sectionHeader = !Platform.isWindows
+          ? '═════════════════════════╡ ••• A/B results so far ••• ╞═════════════════════════'
+          : 'A/B results so far';
+      expect(
+        result.stdout,
+        contains(
+          '$sectionHeader\n'
+          '\n'
+          'Score\tAverage A (noise)\tAverage B (noise)\tSpeed-up\n'
+          'metric1\t42.00 (0.00%)\t42.00 (0.00%)\t1.00x\t\n'
+          'metric2\t123.00 (0.00%)\t123.00 (0.00%)\t1.00x\t\n',
+        ),
+      );
+
+      sectionHeader = !Platform.isWindows
+          ? '════════════════════════════╡ ••• Raw results ••• ╞═════════════════════════════'
+          : 'Raw results';
+      expect(
+        result.stdout,
+        contains(
+          '$sectionHeader\n'
+          '\n'
+          'metric1:\n'
+          '  A:\t42.00\t42.00\t\n'
+          '  B:\t42.00\t42.00\t\n'
+          'metric2:\n'
+          '  A:\t123.00\t123.00\t\n'
+          '  B:\t123.00\t123.00\t\n',
+        ),
+      );
+
+      sectionHeader = !Platform.isWindows
+          ? '═════════════════════════╡ ••• Final A/B results ••• ╞══════════════════════════'
+          : 'Final A/B results';
+      expect(
+        result.stdout,
+        contains(
+          '$sectionHeader\n'
+          '\n'
+          'Score\tAverage A (noise)\tAverage B (noise)\tSpeed-up\n'
+          'metric1\t42.00 (0.00%)\t42.00 (0.00%)\t1.00x\t\n'
+          'metric2\t123.00 (0.00%)\t123.00 (0.00%)\t1.00x\t\n',
+        ),
+      );
+    });
   });
 }