Set up tests that verify we can build a fresh counter app across our Gradle/AGP/Kotlin support range (#151568)

Sets up tests that verify we can build a fresh counter app across our Gradle/AGP/Kotlin support range.

Post submit only, because the suite takes ~30 minutes to run, and I expect it to be _somewhat_ rare that we break only one of these versions (and therefore it doesn't get caught by existing presubmits).

.ci.yaml

@@ -1215,6 +1215,42 @@ targets:
       - bin/**
       - .ci.yaml
 
+  - name: Linux android_java11_dependency_smoke_tests
+    recipe: devicelab/devicelab_drone
+    presubmit: false
+    bringup: true
+    timeout: 60
+    properties:
+      add_recipes_cq: "true"
+      dependencies: >-
+        [
+          {"dependency": "android_sdk", "version": "version:34v3"},
+          {"dependency": "chrome_and_driver", "version": "version:125.0.6422.141"},
+          {"dependency": "open_jdk", "version": "version:11"}
+        ]
+      task_name: android_java11_dependency_smoke_tests
+      tags: >
+        ["devicelab", "hostonly", "linux"]
+      test_timeout_secs: "2700"
+
+  - name: Linux android_java17_dependency_smoke_tests
+    recipe: devicelab/devicelab_drone
+    presubmit: false
+    bringup: true
+    timeout: 60
+    properties:
+      add_recipes_cq: "true"
+      dependencies: >-
+        [
+          {"dependency": "android_sdk", "version": "version:34v3"},
+          {"dependency": "chrome_and_driver", "version": "version:125.0.6422.141"},
+          {"dependency": "open_jdk", "version": "version:17"}
+        ]
+      task_name: android_java17_dependency_smoke_tests
+      tags: >
+        ["devicelab", "hostonly", "linux"]
+      test_timeout_secs: "2700"
+
   - name: Linux tool_tests_commands
     recipe: flutter/flutter_drone
     timeout: 60

TESTOWNERS

@@ -12,6 +12,8 @@
 /dev/devicelab/bin/tasks/analyzer_benchmark.dart @andrewkolos @flutter/tool
 /dev/devicelab/bin/tasks/android_choreographer_do_frame_test.dart @reidbaker @flutter/engine
 /dev/devicelab/bin/tasks/android_defines_test.dart @andrewkolos @flutter/tool
+/dev/devicelab/bin/tasks/android_java11_dependency_smoke_tests.dart @gmackall @flutter/android
+/dev/devicelab/bin/tasks/android_java17_dependency_smoke_tests.dart @gmackall @flutter/android
 /dev/devicelab/bin/tasks/android_lifecycles_test.dart @reidbaker @flutter/engine
 /dev/devicelab/bin/tasks/android_obfuscate_test.dart @andrewkolos @flutter/tool
 /dev/devicelab/bin/tasks/android_picture_cache_complexity_scoring_perf__timeline_summary.dart @flar @flutter/engine

dev/devicelab/bin/tasks/android_java11_dependency_smoke_tests.dart

@@ -0,0 +1,38 @@
+// Copyright 2014 The Flutter Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+import 'dart:io';
+
+import 'package:file/local.dart';
+import 'package:flutter_devicelab/framework/dependency_smoke_test_task_definition.dart';
+import 'package:flutter_devicelab/framework/framework.dart';
+
+// Methodology:
+// - AGP: all versions within our support range (*).
+// - Gradle: The version that AGP lists as the default Gradle version for that
+//           AGP version under the release notes, e.g.
+//           https://developer.android.com/build/releases/past-releases/agp-8-4-0-release-notes.
+// - Kotlin: No methodology as of yet.
+// (*) - support range defined in packages/flutter_tools/gradle/src/main/kotlin/dependency_version_checker.gradle.kts.
+List<VersionTuple> versionTuples = <VersionTuple>[
+  VersionTuple(agpVersion: '7.0.1', gradleVersion: '7.0.2', kotlinVersion: '1.7.10'),
+  VersionTuple(agpVersion: '7.1.0', gradleVersion: '7.2', kotlinVersion: '1.7.10'),
+  VersionTuple(agpVersion: '7.2.0', gradleVersion: '7.3.3', kotlinVersion: '1.7.10'),
+  VersionTuple(agpVersion: '7.3.0', gradleVersion: '7.4', kotlinVersion: '1.7.10'),
+  VersionTuple(agpVersion: '7.4.0', gradleVersion: '7.5', kotlinVersion: '1.8.10'),
+];
+
+// This test requires a Java version less than 17 due to the intentionally low
+// version of Gradle. We choose 11 because this was the primary version used in
+// CI before 17, and hence it is also hosted on CIPD.
+// https://docs.gradle.org/current/userguide/compatibility.html
+Future<void> main() async {
+  /// The [FileSystem] for the integration test environment.
+  const LocalFileSystem fileSystem = LocalFileSystem();
+
+  final Directory tempDir = fileSystem.systemTempDirectory.createTempSync('flutter_android_dependency_version_tests');
+  await task(() {
+    return buildFlutterApkWithSpecifiedDependencyVersions(versionTuples: versionTuples, tempDir: tempDir, localFileSystem: fileSystem);
+  });
+}

dev/devicelab/bin/tasks/android_java17_dependency_smoke_tests.dart

@@ -0,0 +1,35 @@
+// Copyright 2014 The Flutter Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+import 'dart:io';
+
+import 'package:file/local.dart';
+import 'package:flutter_devicelab/framework/dependency_smoke_test_task_definition.dart';
+import 'package:flutter_devicelab/framework/framework.dart';
+
+// Methodology:
+// - AGP: all versions within our support range (*).
+// - Gradle: The version that AGP lists as the default Gradle version for that
+//           AGP version under the release notes, e.g.
+//           https://developer.android.com/build/releases/past-releases/agp-8-4-0-release-notes.
+// - Kotlin: No methodology as of yet.
+// (*) - support range defined in packages/flutter_tools/gradle/src/main/kotlin/dependency_version_checker.gradle.kts.
+List<VersionTuple> versionTuples = <VersionTuple>[
+  VersionTuple(agpVersion: '8.0.0', gradleVersion: '8.0', kotlinVersion: '1.8.22'),
+  VersionTuple(agpVersion: '8.1.0', gradleVersion: '8.0', kotlinVersion: '1.8.22'),
+  VersionTuple(agpVersion: '8.2.0', gradleVersion: '8.2', kotlinVersion: '1.8.22'),
+  VersionTuple(agpVersion: '8.3.0', gradleVersion: '8.4', kotlinVersion: '1.8.22'),
+  VersionTuple(agpVersion: '8.4.0', gradleVersion: '8.6', kotlinVersion: '1.8.22'),
+  VersionTuple(agpVersion: '8.5.0', gradleVersion: '8.7', kotlinVersion: '1.8.22'),
+];
+
+Future<void> main() async {
+  /// The [FileSystem] for the integration test environment.
+  const LocalFileSystem fileSystem = LocalFileSystem();
+
+  final Directory tempDir = fileSystem.systemTempDirectory.createTempSync('flutter_android_dependency_version_tests');
+  await task(() {
+    return buildFlutterApkWithSpecifiedDependencyVersions(versionTuples: versionTuples, tempDir: tempDir, localFileSystem: fileSystem);
+  });
+}

dev/devicelab/lib/framework/dependency_smoke_test_task_definition.dart

@@ -0,0 +1,142 @@
+// Copyright 2014 The Flutter Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+import 'dart:io';
+
+import 'package:file/local.dart';
+import 'task_result.dart';
+import 'utils.dart';
+
+// The following test outline shares a lot of similarities with
+// the one in packages/flutter_tools/test/src/android_common.dart. When making
+// changes here, consider making the corresponding changes to that file as well.
+
+/// The template settings.gradle content, with AGP and Kotlin versions replaced
+/// by an easily find/replaceable string.
+const String gradleSettingsFileContent = r'''
+pluginManagement {
+    def flutterSdkPath = {
+        def properties = new Properties()
+        file("local.properties").withInputStream { properties.load(it) }
+        def flutterSdkPath = properties.getProperty("flutter.sdk")
+        assert flutterSdkPath != null, "flutter.sdk not set in local.properties"
+        return flutterSdkPath
+    }()
+
+    includeBuild("$flutterSdkPath/packages/flutter_tools/gradle")
+
+    repositories {
+        google()
+        mavenCentral()
+        gradlePluginPortal()
+    }
+}
+
+plugins {
+    id "dev.flutter.flutter-plugin-loader" version "1.0.0"
+    id "com.android.application" version "AGP_REPLACE_ME" apply false
+    id "org.jetbrains.kotlin.android" version "KGP_REPLACE_ME" apply false
+}
+
+include ":app"
+
+''';
+
+const String agpReplacementString = 'AGP_REPLACE_ME';
+const String kgpReplacementString = 'KGP_REPLACE_ME';
+
+/// The template gradle-wrapper.properties content, with the Gradle version replaced
+/// by an easily find/replaceable string.
+const String gradleWrapperPropertiesFileContent = r'''
+distributionBase=GRADLE_USER_HOME
+distributionPath=wrapper/dists
+zipStoreBase=GRADLE_USER_HOME
+zipStorePath=wrapper/dists
+distributionUrl=https\://services.gradle.org/distributions/gradle-GRADLE_REPLACE_ME-all.zip
+
+''';
+
+const String gradleReplacementString = 'GRADLE_REPLACE_ME';
+
+/// A simple class containing a Kotlin, Gradle, and AGP version.
+class VersionTuple {
+
+  VersionTuple({
+    required this.agpVersion,
+    required this.gradleVersion,
+    required this.kotlinVersion
+  });
+
+  String agpVersion;
+  String gradleVersion;
+  String kotlinVersion;
+
+  @override
+  String toString() {
+    return '(AGP version: $agpVersion, Gradle version: $gradleVersion, Kotlin version: $kotlinVersion)';
+  }
+}
+
+/// For each [VersionTuple] in versionTuples:
+/// 1. Calls `flutter create`
+/// 2. Replaces the template AGP, Gradle, and Kotlin versions with those in the
+///    tuple.
+/// 3. Calls `flutter build apk`.
+/// Returns a failed task result if any of the `create` or `build apk` calls
+/// fails, returns a successful result otherwise. Cleans up in either case.
+Future<TaskResult> buildFlutterApkWithSpecifiedDependencyVersions({
+  required List<VersionTuple> versionTuples,
+  required Directory tempDir,
+  required LocalFileSystem localFileSystem,}) async {
+  for (final VersionTuple versions in versionTuples) {
+    final Directory innerTempDir = tempDir.createTempSync(versions.gradleVersion);
+    try {
+      // Create a new flutter project.
+      section('Create new app with Gradle ${versions.gradleVersion}, AGP ${versions.agpVersion}, and Kotlin ${versions.kotlinVersion}');
+      await flutter(
+        'create',
+        options: <String>[
+          'dependency_checker_app',
+          '--platforms=android',
+        ],
+        workingDirectory: innerTempDir.path,
+      );
+
+      final String appPath = '${innerTempDir.absolute.path}/dependency_checker_app';
+
+      // Modify gradle version to passed in version.
+      final File gradleWrapperProperties = localFileSystem.file(localFileSystem.path.join(
+          appPath, 'android', 'gradle', 'wrapper', 'gradle-wrapper.properties'));
+      final String propertyContent = gradleWrapperPropertiesFileContent.replaceFirst(
+        gradleReplacementString,
+        versions.gradleVersion,
+      );
+      await gradleWrapperProperties.writeAsString(propertyContent, flush: true);
+
+      final File gradleSettings = localFileSystem.file(localFileSystem.path.join(
+          appPath, 'android', 'settings.gradle'));
+      final String settingsContent = gradleSettingsFileContent
+          .replaceFirst(agpReplacementString, versions.agpVersion)
+          .replaceFirst(kgpReplacementString, versions.kotlinVersion);
+      await gradleSettings.writeAsString(settingsContent, flush: true);
+
+
+      // Ensure that gradle files exists from templates.
+      section("Ensure 'flutter build apk' succeeds with Gradle ${versions.gradleVersion}, AGP ${versions.agpVersion}, and Kotlin ${versions.kotlinVersion}");
+      await flutter(
+        'build',
+        options: <String>[
+          'apk',
+          '--debug',
+        ],
+        workingDirectory: appPath,
+      );
+    } catch (e) {
+      tempDir.deleteSync(recursive: true);
+      return TaskResult.failure('Failed to build app with Gradle ${versions.gradleVersion}, AGP ${versions.agpVersion}, and Kotlin ${versions.kotlinVersion}, error was:\n$e');
+    }
+  }
+  tempDir.deleteSync(recursive: true);
+  return TaskResult.success(null);
+}

packages/flutter_tools/test/android_java11_integration.shard/android_dependency_version_checking_test.dart

@@ -7,55 +7,13 @@ import 'dart:io';
 import 'package:file/src/interface/file_system_entity.dart';
 
 import '../integration.shard/test_utils.dart';
+import '../src/android_common.dart';
 import '../src/common.dart';
 import '../src/context.dart';
 
-const String gradleSettingsFileContent = r'''
-pluginManagement {
-    def flutterSdkPath = {
-        def properties = new Properties()
-        file("local.properties").withInputStream { properties.load(it) }
-        def flutterSdkPath = properties.getProperty("flutter.sdk")
-        assert flutterSdkPath != null, "flutter.sdk not set in local.properties"
-        return flutterSdkPath
-    }()
 
-    includeBuild("$flutterSdkPath/packages/flutter_tools/gradle")
 
-    repositories {
+// This test requires Java 11 due to the intentionally low version of Gradle.
-        google()
-        mavenCentral()
-        gradlePluginPortal()
-    }
-}
-
-plugins {
-    id "dev.flutter.flutter-plugin-loader" version "1.0.0"
-    id "com.android.application" version "AGP_REPLACE_ME" apply false
-    id "org.jetbrains.kotlin.android" version "KGP_REPLACE_ME" apply false
-}
-
-include ":app"
-
-''';
-
-const String agpReplacementString = 'AGP_REPLACE_ME';
-const String kgpReplacementString = 'KGP_REPLACE_ME';
-
-const String gradleWrapperPropertiesFileContent = r'''
-distributionBase=GRADLE_USER_HOME
-distributionPath=wrapper/dists
-zipStoreBase=GRADLE_USER_HOME
-zipStorePath=wrapper/dists
-distributionUrl=https\://services.gradle.org/distributions/gradle-GRADLE_REPLACE_ME-all.zip
-
-''';
-
-const String gradleReplacementString = 'GRADLE_REPLACE_ME';
-
-// This test is currently on the preview shard (but not using the preview
-// version of Android) because it is the only one using Java 11. This test
-// requires Java 11 due to the intentionally low version of Gradle.
 void main() {
   late Directory tempDir;
 
@@ -67,57 +25,13 @@ void main() {
     tryToDelete(tempDir as FileSystemEntity);
   });
 
-  Future<ProcessResult> buildFlutterApkWithSpecifiedDependencyVersions({
-    required String gradleVersion,
-    required String agpVersion,
-    required String kgpVersion}) async {
-    // Create a new flutter project.
-    final String flutterBin = fileSystem.path.join(getFlutterRoot(), 'bin', 'flutter');
-    ProcessResult result = await processManager.run(<String>[
-      flutterBin,
-      'create',
-      'dependency_checker_app',
-      '--platforms=android',
-    ], workingDirectory: tempDir.path);
-    expect(result, const ProcessResultMatcher());
-
-    final Directory app = Directory(fileSystem.path.join(tempDir.path, 'dependency_checker_app'));
-
-    // Modify gradle version to passed in version.
-    final File gradleWrapperProperties = File(fileSystem.path.join(
-        app.path, 'android', 'gradle', 'wrapper', 'gradle-wrapper.properties'));
-    final String propertyContent = gradleWrapperPropertiesFileContent.replaceFirst(
-      gradleReplacementString,
-      gradleVersion,
-    );
-    await gradleWrapperProperties.writeAsString(propertyContent, flush: true);
-
-    final File gradleSettings = File(fileSystem.path.join(
-        app.path, 'android', 'settings.gradle'));
-    final String settingsContent = gradleSettingsFileContent
-        .replaceFirst(agpReplacementString, agpVersion)
-        .replaceFirst(kgpReplacementString, kgpVersion);
-    await gradleSettings.writeAsString(settingsContent, flush: true);
-
-
-    // Ensure that gradle files exists from templates.
-    result = await processManager.run(<String>[
-      flutterBin,
-      'build',
-      'apk',
-      '--debug',
-    ], workingDirectory: app.path);
-    return result;
-  }
-
   testUsingContext(
       'AGP version out of "warn" support band but in "error" band builds '
           'successfully and prints warning', () async {
-
+    final VersionTuple versionTuple = VersionTuple(agpVersion: '7.0.0', gradleVersion: '7.5', kotlinVersion: '1.7.10');
     final ProcessResult result = await buildFlutterApkWithSpecifiedDependencyVersions(
-        gradleVersion: '7.5',
+        versions: versionTuple,
-        agpVersion: '7.0.0',
+        tempDir: tempDir
-        kgpVersion: '1.7.10'
     );
     expect(result, const ProcessResultMatcher());
     expect(result.stderr, contains('Please upgrade your Android Gradle Plugin version'));
@@ -127,10 +41,10 @@ void main() {
       'Gradle version out of "warn" support band but in "error" band builds '
           'successfully and prints warning', () async {
     // Create a new flutter project.
+    final VersionTuple versionTuple = VersionTuple(agpVersion: '7.0.0', gradleVersion: '7.0.2', kotlinVersion: '1.7.10');
     final ProcessResult result = await buildFlutterApkWithSpecifiedDependencyVersions(
-        gradleVersion: '7.0.2',
+        versions: versionTuple,
-        agpVersion: '7.0.0',
+        tempDir: tempDir
-        kgpVersion: '1.7.10'
     );
     expect(result, const ProcessResultMatcher());
     expect(result.stderr, contains('Please upgrade your Gradle version'));
@@ -139,10 +53,10 @@ void main() {
   testUsingContext(
       'Kotlin version out of "warn" support band but in "error" band builds '
           'successfully and prints warning', () async {
+    final VersionTuple versionTuple = VersionTuple(agpVersion: '7.4.0', gradleVersion: '7.5', kotlinVersion: '1.7.0');
     final ProcessResult result = await buildFlutterApkWithSpecifiedDependencyVersions(
-        gradleVersion: '7.5',
+        versions: versionTuple,
-        agpVersion: '7.4.0',
+        tempDir: tempDir
-        kgpVersion: '1.7.0'
     );
 
     expect(result, const ProcessResultMatcher());

packages/flutter_tools/test/src/android_common.dart

@@ -2,12 +2,17 @@
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
+import 'dart:io';
+
 import 'package:flutter_tools/src/android/android_builder.dart';
-import 'package:flutter_tools/src/base/file_system.dart';
+import 'package:flutter_tools/src/base/file_system.dart' as file_system;
 import 'package:flutter_tools/src/build_info.dart';
 import 'package:flutter_tools/src/globals.dart' as globals;
 import 'package:flutter_tools/src/project.dart';
 
+import '../integration.shard/test_utils.dart';
+import 'common.dart';
+
 /// A fake implementation of [AndroidBuilder].
 class FakeAndroidBuilder implements AndroidBuilder {
   @override
@@ -56,7 +61,7 @@ class FakeFlutterProjectFactory extends FlutterProjectFactory {
       logger: globals.logger,
     );
 
-  final Directory directoryOverride;
+  final file_system.Directory directoryOverride;
 
   @override
   FlutterProject fromDirectory(Directory _) {
@@ -64,3 +69,114 @@ class FakeFlutterProjectFactory extends FlutterProjectFactory {
     return super.fromDirectory(directoryOverride.childDirectory('flutter_project'));
   }
 }
+
+// The following test outline shares a lot of similarities with the one in
+// dev/devicelab/lib/framework/dependency_smoke_test_task_definition.dart
+// When making changes here, consider making the corresponding changes to that
+// file as well.
+
+const String gradleSettingsFileContent = r'''
+pluginManagement {
+    def flutterSdkPath = {
+        def properties = new Properties()
+        file("local.properties").withInputStream { properties.load(it) }
+        def flutterSdkPath = properties.getProperty("flutter.sdk")
+        assert flutterSdkPath != null, "flutter.sdk not set in local.properties"
+        return flutterSdkPath
+    }()
+
+    includeBuild("$flutterSdkPath/packages/flutter_tools/gradle")
+
+    repositories {
+        google()
+        mavenCentral()
+        gradlePluginPortal()
+    }
+}
+
+plugins {
+    id "dev.flutter.flutter-plugin-loader" version "1.0.0"
+    id "com.android.application" version "AGP_REPLACE_ME" apply false
+    id "org.jetbrains.kotlin.android" version "KGP_REPLACE_ME" apply false
+}
+
+include ":app"
+
+''';
+
+const String agpReplacementString = 'AGP_REPLACE_ME';
+const String kgpReplacementString = 'KGP_REPLACE_ME';
+
+const String gradleWrapperPropertiesFileContent = r'''
+distributionBase=GRADLE_USER_HOME
+distributionPath=wrapper/dists
+zipStoreBase=GRADLE_USER_HOME
+zipStorePath=wrapper/dists
+distributionUrl=https\://services.gradle.org/distributions/gradle-GRADLE_REPLACE_ME-all.zip
+
+''';
+
+const String gradleReplacementString = 'GRADLE_REPLACE_ME';
+
+class VersionTuple {
+
+  VersionTuple({
+    required this.agpVersion,
+    required this.gradleVersion,
+    required this.kotlinVersion
+  });
+
+  String agpVersion;
+  String gradleVersion;
+  String kotlinVersion;
+
+  @override
+  String toString() {
+    return '(AGP version: $agpVersion, Gradle version: $gradleVersion, Kotlin version: $kotlinVersion)';
+  }
+}
+
+/// Creates a new Flutter project with the specified AGP, Gradle, and Kotlin
+/// versions and then tries to call `flutter build apk`, returning the
+/// ProcessResult.
+Future<ProcessResult> buildFlutterApkWithSpecifiedDependencyVersions({
+  required VersionTuple versions,
+  required Directory tempDir,}) async {
+  // Create a new flutter project.
+  final String flutterBin = fileSystem.path.join(getFlutterRoot(), 'bin', 'flutter');
+  ProcessResult result = await processManager.run(<String>[
+    flutterBin,
+    'create',
+    'dependency_checker_app',
+    '--platforms=android',
+  ], workingDirectory: tempDir.path);
+  expect(result, const ProcessResultMatcher());
+
+  final Directory app = Directory(fileSystem.path.join(tempDir.path, 'dependency_checker_app'));
+
+  // Modify gradle version to passed in version.
+  final File gradleWrapperProperties = File(fileSystem.path.join(
+      app.path, 'android', 'gradle', 'wrapper', 'gradle-wrapper.properties'));
+  final String propertyContent = gradleWrapperPropertiesFileContent.replaceFirst(
+    gradleReplacementString,
+    versions.gradleVersion,
+  );
+  await gradleWrapperProperties.writeAsString(propertyContent, flush: true);
+
+  final File gradleSettings = File(fileSystem.path.join(
+      app.path, 'android', 'settings.gradle'));
+  final String settingsContent = gradleSettingsFileContent
+      .replaceFirst(agpReplacementString, versions.agpVersion)
+      .replaceFirst(kgpReplacementString, versions.kotlinVersion);
+  await gradleSettings.writeAsString(settingsContent, flush: true);
+
+
+  // Ensure that gradle files exists from templates.
+  result = await processManager.run(<String>[
+    flutterBin,
+    'build',
+    'apk',
+    '--debug',
+  ], workingDirectory: app.path);
+  return result;
+}