diff --git a/dev/devicelab/bin/run.dart b/dev/devicelab/bin/run.dart index 7a2daa6728..682bc3d387 100644 --- a/dev/devicelab/bin/run.dart +++ b/dev/devicelab/bin/run.dart @@ -162,7 +162,19 @@ Future _runABTest() async { } abTest.addBResult(localEngineResult); + + if (!silent && i < runsPerTest) { + section('A/B results so far'); + print(abTest.printSummary()); + } } + + if (!silent) { + section('Raw results'); + print(abTest.rawResults()); + } + + section('Final A/B results'); print(abTest.printSummary()); } diff --git a/dev/devicelab/bin/tasks/smoke_test_success.dart b/dev/devicelab/bin/tasks/smoke_test_success.dart index 99de0038ef..82da966963 100644 --- a/dev/devicelab/bin/tasks/smoke_test_success.dart +++ b/dev/devicelab/bin/tasks/smoke_test_success.dart @@ -9,6 +9,13 @@ import 'package:flutter_devicelab/framework/framework.dart'; /// Smoke test of a successful task. Future main() async { await task(() async { - return TaskResult.success({}); + return TaskResult.success({ + 'metric1': 42, + 'metric2': 123, + 'not_a_metric': 'something', + }, benchmarkScoreKeys: [ + 'metric1', + 'metric2', + ]); }); } diff --git a/dev/devicelab/lib/framework/ab.dart b/dev/devicelab/lib/framework/ab.dart index ad1de24753..84ce835fa4 100644 --- a/dev/devicelab/lib/framework/ab.dart +++ b/dev/devicelab/lib/framework/ab.dart @@ -30,22 +30,54 @@ class ABTest { _addResult(result, _bResults); } + /// Returns unprocessed data collected by the A/B test formatted as + /// a tab-separated spreadsheet. + String rawResults() { + final StringBuffer buffer = StringBuffer(); + for (final String scoreKey in _allScoreKeys) { + buffer.writeln('$scoreKey:'); + buffer.write(' A:\t'); + if (_aResults.containsKey(scoreKey)) { + for (final double score in _aResults[scoreKey]) { + buffer.write('${score.toStringAsFixed(2)}\t'); + } + } else { + buffer.write('N/A'); + } + buffer.writeln(); + + buffer.write(' B:\t'); + if (_bResults.containsKey(scoreKey)) { + for (final double score in _bResults[scoreKey]) { + buffer.write('${score.toStringAsFixed(2)}\t'); + } + } else { + buffer.write('N/A'); + } + buffer.writeln(); + } + return buffer.toString(); + } + + Set get _allScoreKeys { + return { + ..._aResults.keys, + ..._bResults.keys, + }; + } + /// Returns the summary as a tab-separated spreadsheet. /// /// This value can be copied straight to a Google Spreadsheet for further analysis. String printSummary() { final Map summariesA = _summarize(_aResults); final Map summariesB = _summarize(_bResults); - final Set scoreKeyUnion = { - ...summariesA.keys, - ...summariesB.keys, - }; final StringBuffer buffer = StringBuffer( 'Score\tAverage A (noise)\tAverage B (noise)\tSpeed-up\n', ); - for (final String scoreKey in scoreKeyUnion) { + for (final String scoreKey in _allScoreKeys) { final _ScoreSummary summaryA = summariesA[scoreKey]; final _ScoreSummary summaryB = summariesB[scoreKey]; buffer.write('$scoreKey\t'); diff --git a/dev/devicelab/lib/framework/utils.dart b/dev/devicelab/lib/framework/utils.dart index 077a9522ac..6e7718a1ce 100644 --- a/dev/devicelab/lib/framework/utils.dart +++ b/dev/devicelab/lib/framework/utils.dart @@ -189,11 +189,18 @@ void mkdirs(Directory directory) { bool exists(FileSystemEntity entity) => entity.existsSync(); void section(String title) { - title = '╡ ••• $title ••• ╞'; - final String line = '═' * math.max((80 - title.length) ~/ 2, 2); - String output = '$line$title$line'; - if (output.length == 79) - output += '═'; + String output; + if (Platform.isWindows) { + // Windows doesn't cope well with characters produced for *nix systems, so + // just output the title with no decoration. + output = title; + } else { + title = '╡ ••• $title ••• ╞'; + final String line = '═' * math.max((80 - title.length) ~/ 2, 2); + output = '$line$title$line'; + if (output.length == 79) + output += '═'; + } print('\n\n$output\n'); } diff --git a/dev/devicelab/test/ab_test.dart b/dev/devicelab/test/ab_test.dart new file mode 100644 index 0000000000..d87222aa8c --- /dev/null +++ b/dev/devicelab/test/ab_test.dart @@ -0,0 +1,51 @@ +// Copyright 2014 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'package:flutter_devicelab/framework/ab.dart'; + +import 'common.dart'; + +void main() { + test('ABTest', () { + final ABTest ab = ABTest(); + + for (int i = 0; i < 5; i++) { + ab.addAResult({ + 'data': { + 'i': i, + 'j': 10 * i, + 'not_a_metric': 'something', + }, + 'benchmarkScoreKeys': ['i', 'j'], + }); + + ab.addBResult({ + 'data': { + 'i': i + 1, + 'k': 10 * i + 1, + }, + 'benchmarkScoreKeys': ['i', 'k'], + }); + } + + expect( + ab.rawResults(), + 'i:\n' + ' A:\t0.00\t1.00\t2.00\t3.00\t4.00\t\n' + ' B:\t1.00\t2.00\t3.00\t4.00\t5.00\t\n' + 'j:\n' + ' A:\t0.00\t10.00\t20.00\t30.00\t40.00\t\n' + ' B:\tN/A\n' + 'k:\n' + ' A:\tN/A\n' + ' B:\t1.00\t11.00\t21.00\t31.00\t41.00\t\n', + ); + expect( + ab.printSummary(), + 'Score\tAverage A (noise)\tAverage B (noise)\tSpeed-up\n' + 'i\t2.00 (70.71%)\t3.00 (47.14%)\t0.67x\t\n' + 'j\t20.00 (70.71%)\t\t\n' + 'k\t\t21.00 (67.34%)\t\n'); + }); +} diff --git a/dev/devicelab/test/run_test.dart b/dev/devicelab/test/run_test.dart index 00c5cbd6c9..1b0a52204c 100644 --- a/dev/devicelab/test/run_test.dart +++ b/dev/devicelab/test/run_test.dart @@ -14,21 +14,26 @@ void main() { const ProcessManager processManager = LocalProcessManager(); group('run.dart script', () { - Future runScript(List testNames) async { - final String dart = path.absolute(path.join('..', '..', 'bin', 'cache', 'dart-sdk', 'bin', 'dart')); + Future runScript(List testNames, + [List otherArgs = const []]) async { + final String dart = path.absolute( + path.join('..', '..', 'bin', 'cache', 'dart-sdk', 'bin', 'dart')); final ProcessResult scriptProcess = processManager.runSync([ dart, 'bin/run.dart', + ...otherArgs, for (final String testName in testNames) ...['-t', testName], ]); return scriptProcess; } - Future expectScriptResult(List testNames, int expectedExitCode) async { + Future expectScriptResult( + List testNames, int expectedExitCode) async { final ProcessResult result = await runScript(testNames); expect(result.exitCode, expectedExitCode, - reason: '[ stderr from test process ]\n\n${result.stderr}\n\n[ end of stderr ]' - '\n\n[ stdout from test process ]\n\n${result.stdout}\n\n[ end of stdout ]'); + reason: + '[ stderr from test process ]\n\n${result.stderr}\n\n[ end of stderr ]' + '\n\n[ stdout from test process ]\n\n${result.stdout}\n\n[ end of stdout ]'); } test('exits with code 0 when succeeds', () async { @@ -36,7 +41,8 @@ void main() { }); test('accepts file paths', () async { - await expectScriptResult(['bin/tasks/smoke_test_success.dart'], 0); + await expectScriptResult( + ['bin/tasks/smoke_test_success.dart'], 0); }); test('rejects invalid file paths', () async { @@ -56,12 +62,66 @@ void main() { }, skip: true); // https://github.com/flutter/flutter/issues/53707 test('exits with code 1 when results are mixed', () async { - await expectScriptResult([ + await expectScriptResult( + [ 'smoke_test_failure', 'smoke_test_success', ], 1, ); }); + + test('runs A/B test', () async { + final ProcessResult result = await runScript( + ['smoke_test_success'], + ['--ab=2', '--local-engine=host_debug_unopt'], + ); + expect(result.exitCode, 0); + + String sectionHeader = !Platform.isWindows + ? '═════════════════════════╡ ••• A/B results so far ••• ╞═════════════════════════' + : 'A/B results so far'; + expect( + result.stdout, + contains( + '$sectionHeader\n' + '\n' + 'Score\tAverage A (noise)\tAverage B (noise)\tSpeed-up\n' + 'metric1\t42.00 (0.00%)\t42.00 (0.00%)\t1.00x\t\n' + 'metric2\t123.00 (0.00%)\t123.00 (0.00%)\t1.00x\t\n', + ), + ); + + sectionHeader = !Platform.isWindows + ? '════════════════════════════╡ ••• Raw results ••• ╞═════════════════════════════' + : 'Raw results'; + expect( + result.stdout, + contains( + '$sectionHeader\n' + '\n' + 'metric1:\n' + ' A:\t42.00\t42.00\t\n' + ' B:\t42.00\t42.00\t\n' + 'metric2:\n' + ' A:\t123.00\t123.00\t\n' + ' B:\t123.00\t123.00\t\n', + ), + ); + + sectionHeader = !Platform.isWindows + ? '═════════════════════════╡ ••• Final A/B results ••• ╞══════════════════════════' + : 'Final A/B results'; + expect( + result.stdout, + contains( + '$sectionHeader\n' + '\n' + 'Score\tAverage A (noise)\tAverage B (noise)\tSpeed-up\n' + 'metric1\t42.00 (0.00%)\t42.00 (0.00%)\t1.00x\t\n' + 'metric2\t123.00 (0.00%)\t123.00 (0.00%)\t1.00x\t\n', + ), + ); + }); }); }