Pin the customer_tests (#83964)

.cirrus.yml

@@ -145,13 +145,18 @@ task:
 
     - name: customer_testing-linux
       only_if: "$CIRRUS_PR != ''"
-      # environment:
+      environment:
         # Empirically, this shard runs fine at 1 CPU and 4G RAM as of October 2019. We will probably
         # want to grow this container when we invite people to add their tests in large numbers.
+        SHOULD_UPDATE_PACKAGES: "FALSE"
       script:
-        - rm -rf bin/cache/pkg/tests
+        # Cirrus doesn't give us the master branch, so we have to fetch it for ourselves,
-        - git clone https://github.com/flutter/tests.git bin/cache/pkg/tests
+        # otherwise we won't be able to figure out how old or new our current branch is.
-        - dart --enable-asserts dev/customer_testing/run_tests.dart --skip-on-fetch-failure --skip-template bin/cache/pkg/tests/registry/*.test
+        - git config user.email "cirrus-bot@invalid"
+        - git fetch origin master:master
+        - git rebase master
+        # The actual logic is in a shell script so that it can be shared between CIs.
+        - (cd dev/customer_testing/; ./ci.sh)
 
     - name: web_smoke_test
       only_if: "changesInclude('.cirrus.yml', 'examples/hello_world/**' ,'dev/**', 'packages/flutter/**', 'packages/flutter_test/**', 'packages/flutter_tools/lib/src/test/**', 'packages/flutter_web_plugins/**', 'bin/**') && $CIRRUS_PR != ''"

dev/customer_testing/ci.bat

@@ -0,0 +1,19 @@
+@ECHO off
+REM Copyright 2014 The Flutter Authors. All rights reserved.
+REM Use of this source code is governed by a BSD-style license that can be
+REM found in the LICENSE file.
+
+REM This should match the ci.sh file in this directory.
+
+REM This is called from the LUCI recipes:
+REM https://flutter.googlesource.com/recipes/+/refs/heads/master/recipe_modules/adhoc_validation/resources/customer_testing.bat
+
+pub get
+CD ..\tools
+pub get
+CD ..\customer_testing
+
+CMD /S /C "IF EXIST "..\..\bin\cache\pkg\tests\" RMDIR /S /Q ..\..\bin\cache\pkg\tests"
+git clone https://github.com/flutter/tests.git ..\..\bin\cache\pkg\tests
+FOR /F "usebackq tokens=*" %%a IN (`dart --enable-asserts ..\tools\bin\find_commit.dart ..\..\bin\cache\pkg\tests`) DO git -C ..\..\bin\cache\pkg\tests checkout %%a
+dart --enable-asserts run_tests.dart --skip-on-fetch-failure --skip-template ..\..\bin\cache\pkg\tests\registry\*.test

dev/customer_testing/ci.sh

@@ -0,0 +1,37 @@
+#!/usr/bin/env bash
+# Copyright 2014 The Flutter Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+# This should match the ci.bat file in this directory.
+
+# This is called from .cirrus.yml and the LUCI recipes:
+# https://flutter.googlesource.com/recipes/+/refs/heads/master/recipe_modules/adhoc_validation/resources/customer_testing.sh
+
+set -e
+
+# This script does not assume that "flutter update-packages" has been
+# run, to allow CIs to save time by skipping that steps since it's
+# largely not needed to run the flutter/tests tests.
+#
+# However, we do need to update this directory and the tools directory.
+pub get
+(cd ../tools; pub get) # used for find_commit.dart below
+
+# Next we need to update the flutter/tests checkout.
+#
+# We use find_commit.dart so that we pull the version of flutter/tests
+# that was contemporary when the branch we are on was created. That
+# way, we can still run the tests on long-lived branches without being
+# affected by breaking changes on trunk causing changes to the tests
+# that wouldn't work on the long-lived branch.
+#
+# (This also prevents trunk from suddenly failing when tests are
+# revved on flutter/tests -- if you rerun a passing customer_tests
+# shard, it should still pass, even if we rolled one of the tests.)
+rm -rf ../../bin/cache/pkg/tests
+git clone https://github.com/flutter/tests.git ../../bin/cache/pkg/tests
+git -C ../../bin/cache/pkg/tests checkout `dart --enable-asserts ../tools/bin/find_commit.dart ../../bin/cache/pkg/tests`
+
+# Finally, run the tests.
+dart --enable-asserts run_tests.dart --skip-on-fetch-failure --skip-template ../../bin/cache/pkg/tests/registry/*.test

dev/tools/bin/find_commit.dart

@@ -45,10 +45,10 @@ String findCommit({
   final Commit anchor;
   if (primaryBranch == primaryTrunk) {
     log('on $primaryTrunk, using last commit as anchor');
-    anchor = Commit.parse(git(primaryRepoDirectory, <String>['log', Commit.formatArgument, '--max-count=1', primaryBranch]));
+    anchor = Commit.parse(git(primaryRepoDirectory, <String>['log', Commit.formatArgument, '--max-count=1', primaryBranch, '--']));
   } else {
-    final List<Commit> branchCommits = Commit.parseList(git(primaryRepoDirectory, <String>['log', Commit.formatArgument, primaryBranch]));
+    final List<Commit> branchCommits = Commit.parseList(git(primaryRepoDirectory, <String>['log', Commit.formatArgument, primaryBranch, '--']));
-    final List<Commit> trunkCommits = Commit.parseList(git(primaryRepoDirectory, <String>['log', Commit.formatArgument, primaryTrunk]));
+    final List<Commit> trunkCommits = Commit.parseList(git(primaryRepoDirectory, <String>['log', Commit.formatArgument, primaryTrunk, '--']));
     if (branchCommits.isEmpty || trunkCommits.isEmpty || branchCommits.first.hash != trunkCommits.first.hash)
       throw StateError('Branch $primaryBranch does not seem to have a common history with trunk $primaryTrunk.');
     if (branchCommits.last.hash == trunkCommits.last.hash) {
@@ -68,6 +68,7 @@ String findCommit({
     '--until=${anchor.timestamp.toIso8601String()}',
     '--max-count=1',
     secondaryBranch,
+    '--',
   ]);
 }