From c1a2e44c78bd23f4fde05236f01d15190e3389a3 Mon Sep 17 00:00:00 2001
From: Jonah Williams <jonahwilliams@google.com>
Date: Thu, 13 Jun 2019 17:52:04 -0700
Subject: [PATCH] Allow flaky tests to pass or fail and mark web tests as flaky
 (#34456)

---
 dev/bots/run_command.dart | 7 ++++++-
 dev/bots/test.dart        | 5 ++---
 2 files changed, 8 insertions(+), 4 deletions(-)

diff --git a/dev/bots/run_command.dart b/dev/bots/run_command.dart
index 8cf8322ed4..c97a08136d 100644
--- a/dev/bots/run_command.dart
+++ b/dev/bots/run_command.dart
@@ -85,6 +85,7 @@ Future<void> runCommand(String executable, List<String> arguments, {
   String failureMessage,
   bool printOutput = true,
   bool skip = false,
+  bool expectFlaky = false,
   Duration timeout = _kLongTimeout,
 }) async {
   final String commandDescription = '${path.relative(executable, from: workingDirectory)} ${arguments.join(' ')}';
@@ -114,9 +115,13 @@ Future<void> runCommand(String executable, List<String> arguments, {
 
   final int exitCode = await process.exitCode.timeout(timeout, onTimeout: () {
     stderr.writeln('Process timed out after $timeout');
-    return expectNonZeroExit ? 0 : 1;
+    return (expectNonZeroExit || expectFlaky) ? 0 : 1;
   });
   print('$clock ELAPSED TIME: $bold${elapsedTime(start)}$reset for $commandDescription in $relativeWorkingDir: ');
+  // If the test is flaky we don't care about the actual exit.
+  if (expectFlaky) {
+    return;
+  }
   if ((exitCode == 0) == expectNonZeroExit || (expectedExitCode != null && exitCode != expectedExitCode)) {
     if (failureMessage != null) {
       print(failureMessage);
diff --git a/dev/bots/test.dart b/dev/bots/test.dart
index 477c31b663..15713758f5 100644
--- a/dev/bots/test.dart
+++ b/dev/bots/test.dart
@@ -341,7 +341,7 @@ Future<void> _runTests() async {
 }
 
 Future<void> _runWebTests() async {
-  await _runFlutterWebTest(path.join(flutterRoot, 'packages', 'flutter'), expectFailure: false, tests: <String>[
+  await _runFlutterWebTest(path.join(flutterRoot, 'packages', 'flutter'), tests: <String>[
     'test/foundation/',
     'test/physics/',
     'test/rendering/',
@@ -607,7 +607,6 @@ class EvalResult {
 }
 
 Future<void> _runFlutterWebTest(String workingDirectory, {
-  bool expectFailure = false,
   bool printOutput = true,
   bool skip = false,
   Duration timeout = _kLongTimeout,
@@ -627,7 +626,7 @@ Future<void> _runFlutterWebTest(String workingDirectory, {
       flutter,
       args,
       workingDirectory: workingDirectory,
-      expectNonZeroExit: expectFailure,
+      expectFlaky: true,
       timeout: timeout,
       environment: <String, String>{
         'FLUTTER_WEB': 'true',