From 53cba24de3a09761b66f367f0dd1f6e3390dac88 Mon Sep 17 00:00:00 2001
From: Jenn Magder <magder@google.com>
Date: Thu, 11 Apr 2024 21:07:19 -0700
Subject: [PATCH] Allow A/B tests to run as just aggregator of local engine
 benchmarks (#146479)

Add `--ab-local-engine-only` flag to change A/B benchmark aggregate behavior to instead run the number of tasks specified with the local engine only, not the default engine.  This is useful when you are trying to get average and noise results for a particular engine branch, without needing the default engine every time you change the engine.

Fixes https://github.com/flutter/flutter/issues/143770
---
 dev/devicelab/bin/run.dart | 37 ++++++++++++++++++++++++-------------
 1 file changed, 24 insertions(+), 13 deletions(-)

diff --git a/dev/devicelab/bin/run.dart b/dev/devicelab/bin/run.dart
index 628b16f9e0..af5268d1fe 100644
--- a/dev/devicelab/bin/run.dart
+++ b/dev/devicelab/bin/run.dart
@@ -114,6 +114,7 @@ Future<void> main(List<String> rawArgs) async {
       deviceId: deviceId,
       resultsFile: resultsFile,
       taskName: taskNames.single,
+      onlyLocalEngine: (args['ab-local-engine-only'] as bool?) ?? false,
     );
   } else {
     await runTasks(taskNames,
@@ -142,6 +143,7 @@ Future<void> _runABTest({
   required String? deviceId,
   required String resultsFile,
   required String taskName,
+  bool onlyLocalEngine = false,
 }) async {
   print('$taskName A/B test. Will run $runsPerTest times.');
 
@@ -155,23 +157,27 @@ Future<void> _runABTest({
   for (int i = 1; i <= runsPerTest; i++) {
     section('Run #$i');
 
-    print('Running with the default engine (A)');
-    final TaskResult defaultEngineResult = await runTask(
-      taskName,
-      silent: silent,
-      deviceId: deviceId,
-    );
+    if (onlyLocalEngine) {
+      print('Skipping default engine (A)');
+    } else {
+      print('Running with the default engine (A)');
+      final TaskResult defaultEngineResult = await runTask(
+        taskName,
+        silent: silent,
+        deviceId: deviceId,
+      );
 
-    print('Default engine result:');
-    print(const JsonEncoder.withIndent('  ').convert(defaultEngineResult));
+      print('Default engine result:');
+      print(const JsonEncoder.withIndent('  ').convert(defaultEngineResult));
 
-    if (!defaultEngineResult.succeeded) {
-      stderr.writeln('Task failed on the default engine.');
-      exit(1);
+      if (!defaultEngineResult.succeeded) {
+        stderr.writeln('Task failed on the default engine.');
+        exit(1);
+      }
+
+      abTest.addAResult(defaultEngineResult);
     }
 
-    abTest.addAResult(defaultEngineResult);
-
     print('Running with the local engine (B)');
     final TaskResult localEngineResult = await runTask(
       taskName,
@@ -273,6 +279,11 @@ ArgParser createArgParser(List<String> taskNames) {
             'The filename may contain a single # character to be replaced by a sequence\n'
             'number if the name already exists.',
     )
+    ..addFlag(
+      'ab-local-engine-only',
+      help: 'When running the A/B aggregator, do not run benchmarks with the default engine (A), only the local engine (B).\n'
+            'Shows the averages and noise report for the local engine without comparison to anything else.',
+    )
     ..addFlag(
       'exit',
       help: 'Exit on the first test failure. Currently flakes are intentionally (though '