Set up tests that verify we can build a fresh counter app across our Gradle/AGP/Kotlin support range (#151568)

Sets up tests that verify we can build a fresh counter app across our Gradle/AGP/Kotlin support range.

Post submit only, because the suite takes ~30 minutes to run, and I expect it to be _somewhat_ rare that we break only one of these versions (and therefore it doesn't get caught by existing presubmits).
This commit is contained in:
Gray Mackall 2024-07-31 12:14:46 -07:00 committed by GitHub
parent 118a015b65
commit a1f03609f7
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
7 changed files with 382 additions and 99 deletions

View File

@ -1215,6 +1215,42 @@ targets:
- bin/**
- .ci.yaml
- name: Linux android_java11_dependency_smoke_tests
recipe: devicelab/devicelab_drone
presubmit: false
bringup: true
timeout: 60
properties:
add_recipes_cq: "true"
dependencies: >-
[
{"dependency": "android_sdk", "version": "version:34v3"},
{"dependency": "chrome_and_driver", "version": "version:125.0.6422.141"},
{"dependency": "open_jdk", "version": "version:11"}
]
task_name: android_java11_dependency_smoke_tests
tags: >
["devicelab", "hostonly", "linux"]
test_timeout_secs: "2700"
- name: Linux android_java17_dependency_smoke_tests
recipe: devicelab/devicelab_drone
presubmit: false
bringup: true
timeout: 60
properties:
add_recipes_cq: "true"
dependencies: >-
[
{"dependency": "android_sdk", "version": "version:34v3"},
{"dependency": "chrome_and_driver", "version": "version:125.0.6422.141"},
{"dependency": "open_jdk", "version": "version:17"}
]
task_name: android_java17_dependency_smoke_tests
tags: >
["devicelab", "hostonly", "linux"]
test_timeout_secs: "2700"
- name: Linux tool_tests_commands
recipe: flutter/flutter_drone
timeout: 60

View File

@ -12,6 +12,8 @@
/dev/devicelab/bin/tasks/analyzer_benchmark.dart @andrewkolos @flutter/tool
/dev/devicelab/bin/tasks/android_choreographer_do_frame_test.dart @reidbaker @flutter/engine
/dev/devicelab/bin/tasks/android_defines_test.dart @andrewkolos @flutter/tool
/dev/devicelab/bin/tasks/android_java11_dependency_smoke_tests.dart @gmackall @flutter/android
/dev/devicelab/bin/tasks/android_java17_dependency_smoke_tests.dart @gmackall @flutter/android
/dev/devicelab/bin/tasks/android_lifecycles_test.dart @reidbaker @flutter/engine
/dev/devicelab/bin/tasks/android_obfuscate_test.dart @andrewkolos @flutter/tool
/dev/devicelab/bin/tasks/android_picture_cache_complexity_scoring_perf__timeline_summary.dart @flar @flutter/engine

View File

@ -0,0 +1,38 @@
// Copyright 2014 The Flutter Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
import 'dart:io';
import 'package:file/local.dart';
import 'package:flutter_devicelab/framework/dependency_smoke_test_task_definition.dart';
import 'package:flutter_devicelab/framework/framework.dart';
// Methodology:
// - AGP: all versions within our support range (*).
// - Gradle: The version that AGP lists as the default Gradle version for that
// AGP version under the release notes, e.g.
// https://developer.android.com/build/releases/past-releases/agp-8-4-0-release-notes.
// - Kotlin: No methodology as of yet.
// (*) - support range defined in packages/flutter_tools/gradle/src/main/kotlin/dependency_version_checker.gradle.kts.
List<VersionTuple> versionTuples = <VersionTuple>[
VersionTuple(agpVersion: '7.0.1', gradleVersion: '7.0.2', kotlinVersion: '1.7.10'),
VersionTuple(agpVersion: '7.1.0', gradleVersion: '7.2', kotlinVersion: '1.7.10'),
VersionTuple(agpVersion: '7.2.0', gradleVersion: '7.3.3', kotlinVersion: '1.7.10'),
VersionTuple(agpVersion: '7.3.0', gradleVersion: '7.4', kotlinVersion: '1.7.10'),
VersionTuple(agpVersion: '7.4.0', gradleVersion: '7.5', kotlinVersion: '1.8.10'),
];
// This test requires a Java version less than 17 due to the intentionally low
// version of Gradle. We choose 11 because this was the primary version used in
// CI before 17, and hence it is also hosted on CIPD.
// https://docs.gradle.org/current/userguide/compatibility.html
Future<void> main() async {
/// The [FileSystem] for the integration test environment.
const LocalFileSystem fileSystem = LocalFileSystem();
final Directory tempDir = fileSystem.systemTempDirectory.createTempSync('flutter_android_dependency_version_tests');
await task(() {
return buildFlutterApkWithSpecifiedDependencyVersions(versionTuples: versionTuples, tempDir: tempDir, localFileSystem: fileSystem);
});
}

View File

@ -0,0 +1,35 @@
// Copyright 2014 The Flutter Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
import 'dart:io';
import 'package:file/local.dart';
import 'package:flutter_devicelab/framework/dependency_smoke_test_task_definition.dart';
import 'package:flutter_devicelab/framework/framework.dart';
// Methodology:
// - AGP: all versions within our support range (*).
// - Gradle: The version that AGP lists as the default Gradle version for that
// AGP version under the release notes, e.g.
// https://developer.android.com/build/releases/past-releases/agp-8-4-0-release-notes.
// - Kotlin: No methodology as of yet.
// (*) - support range defined in packages/flutter_tools/gradle/src/main/kotlin/dependency_version_checker.gradle.kts.
List<VersionTuple> versionTuples = <VersionTuple>[
VersionTuple(agpVersion: '8.0.0', gradleVersion: '8.0', kotlinVersion: '1.8.22'),
VersionTuple(agpVersion: '8.1.0', gradleVersion: '8.0', kotlinVersion: '1.8.22'),
VersionTuple(agpVersion: '8.2.0', gradleVersion: '8.2', kotlinVersion: '1.8.22'),
VersionTuple(agpVersion: '8.3.0', gradleVersion: '8.4', kotlinVersion: '1.8.22'),
VersionTuple(agpVersion: '8.4.0', gradleVersion: '8.6', kotlinVersion: '1.8.22'),
VersionTuple(agpVersion: '8.5.0', gradleVersion: '8.7', kotlinVersion: '1.8.22'),
];
Future<void> main() async {
/// The [FileSystem] for the integration test environment.
const LocalFileSystem fileSystem = LocalFileSystem();
final Directory tempDir = fileSystem.systemTempDirectory.createTempSync('flutter_android_dependency_version_tests');
await task(() {
return buildFlutterApkWithSpecifiedDependencyVersions(versionTuples: versionTuples, tempDir: tempDir, localFileSystem: fileSystem);
});
}

View File

@ -0,0 +1,142 @@
// Copyright 2014 The Flutter Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
import 'dart:io';
import 'package:file/local.dart';
import 'task_result.dart';
import 'utils.dart';
// The following test outline shares a lot of similarities with
// the one in packages/flutter_tools/test/src/android_common.dart. When making
// changes here, consider making the corresponding changes to that file as well.
/// The template settings.gradle content, with AGP and Kotlin versions replaced
/// by an easily find/replaceable string.
const String gradleSettingsFileContent = r'''
pluginManagement {
def flutterSdkPath = {
def properties = new Properties()
file("local.properties").withInputStream { properties.load(it) }
def flutterSdkPath = properties.getProperty("flutter.sdk")
assert flutterSdkPath != null, "flutter.sdk not set in local.properties"
return flutterSdkPath
}()
includeBuild("$flutterSdkPath/packages/flutter_tools/gradle")
repositories {
google()
mavenCentral()
gradlePluginPortal()
}
}
plugins {
id "dev.flutter.flutter-plugin-loader" version "1.0.0"
id "com.android.application" version "AGP_REPLACE_ME" apply false
id "org.jetbrains.kotlin.android" version "KGP_REPLACE_ME" apply false
}
include ":app"
''';
const String agpReplacementString = 'AGP_REPLACE_ME';
const String kgpReplacementString = 'KGP_REPLACE_ME';
/// The template gradle-wrapper.properties content, with the Gradle version replaced
/// by an easily find/replaceable string.
const String gradleWrapperPropertiesFileContent = r'''
distributionBase=GRADLE_USER_HOME
distributionPath=wrapper/dists
zipStoreBase=GRADLE_USER_HOME
zipStorePath=wrapper/dists
distributionUrl=https\://services.gradle.org/distributions/gradle-GRADLE_REPLACE_ME-all.zip
''';
const String gradleReplacementString = 'GRADLE_REPLACE_ME';
/// A simple class containing a Kotlin, Gradle, and AGP version.
class VersionTuple {
VersionTuple({
required this.agpVersion,
required this.gradleVersion,
required this.kotlinVersion
});
String agpVersion;
String gradleVersion;
String kotlinVersion;
@override
String toString() {
return '(AGP version: $agpVersion, Gradle version: $gradleVersion, Kotlin version: $kotlinVersion)';
}
}
/// For each [VersionTuple] in versionTuples:
/// 1. Calls `flutter create`
/// 2. Replaces the template AGP, Gradle, and Kotlin versions with those in the
/// tuple.
/// 3. Calls `flutter build apk`.
/// Returns a failed task result if any of the `create` or `build apk` calls
/// fails, returns a successful result otherwise. Cleans up in either case.
Future<TaskResult> buildFlutterApkWithSpecifiedDependencyVersions({
required List<VersionTuple> versionTuples,
required Directory tempDir,
required LocalFileSystem localFileSystem,}) async {
for (final VersionTuple versions in versionTuples) {
final Directory innerTempDir = tempDir.createTempSync(versions.gradleVersion);
try {
// Create a new flutter project.
section('Create new app with Gradle ${versions.gradleVersion}, AGP ${versions.agpVersion}, and Kotlin ${versions.kotlinVersion}');
await flutter(
'create',
options: <String>[
'dependency_checker_app',
'--platforms=android',
],
workingDirectory: innerTempDir.path,
);
final String appPath = '${innerTempDir.absolute.path}/dependency_checker_app';
// Modify gradle version to passed in version.
final File gradleWrapperProperties = localFileSystem.file(localFileSystem.path.join(
appPath, 'android', 'gradle', 'wrapper', 'gradle-wrapper.properties'));
final String propertyContent = gradleWrapperPropertiesFileContent.replaceFirst(
gradleReplacementString,
versions.gradleVersion,
);
await gradleWrapperProperties.writeAsString(propertyContent, flush: true);
final File gradleSettings = localFileSystem.file(localFileSystem.path.join(
appPath, 'android', 'settings.gradle'));
final String settingsContent = gradleSettingsFileContent
.replaceFirst(agpReplacementString, versions.agpVersion)
.replaceFirst(kgpReplacementString, versions.kotlinVersion);
await gradleSettings.writeAsString(settingsContent, flush: true);
// Ensure that gradle files exists from templates.
section("Ensure 'flutter build apk' succeeds with Gradle ${versions.gradleVersion}, AGP ${versions.agpVersion}, and Kotlin ${versions.kotlinVersion}");
await flutter(
'build',
options: <String>[
'apk',
'--debug',
],
workingDirectory: appPath,
);
} catch (e) {
tempDir.deleteSync(recursive: true);
return TaskResult.failure('Failed to build app with Gradle ${versions.gradleVersion}, AGP ${versions.agpVersion}, and Kotlin ${versions.kotlinVersion}, error was:\n$e');
}
}
tempDir.deleteSync(recursive: true);
return TaskResult.success(null);
}

View File

@ -7,55 +7,13 @@ import 'dart:io';
import 'package:file/src/interface/file_system_entity.dart';
import '../integration.shard/test_utils.dart';
import '../src/android_common.dart';
import '../src/common.dart';
import '../src/context.dart';
const String gradleSettingsFileContent = r'''
pluginManagement {
def flutterSdkPath = {
def properties = new Properties()
file("local.properties").withInputStream { properties.load(it) }
def flutterSdkPath = properties.getProperty("flutter.sdk")
assert flutterSdkPath != null, "flutter.sdk not set in local.properties"
return flutterSdkPath
}()
includeBuild("$flutterSdkPath/packages/flutter_tools/gradle")
repositories {
google()
mavenCentral()
gradlePluginPortal()
}
}
plugins {
id "dev.flutter.flutter-plugin-loader" version "1.0.0"
id "com.android.application" version "AGP_REPLACE_ME" apply false
id "org.jetbrains.kotlin.android" version "KGP_REPLACE_ME" apply false
}
include ":app"
''';
const String agpReplacementString = 'AGP_REPLACE_ME';
const String kgpReplacementString = 'KGP_REPLACE_ME';
const String gradleWrapperPropertiesFileContent = r'''
distributionBase=GRADLE_USER_HOME
distributionPath=wrapper/dists
zipStoreBase=GRADLE_USER_HOME
zipStorePath=wrapper/dists
distributionUrl=https\://services.gradle.org/distributions/gradle-GRADLE_REPLACE_ME-all.zip
''';
const String gradleReplacementString = 'GRADLE_REPLACE_ME';
// This test is currently on the preview shard (but not using the preview
// version of Android) because it is the only one using Java 11. This test
// requires Java 11 due to the intentionally low version of Gradle.
// This test requires Java 11 due to the intentionally low version of Gradle.
void main() {
late Directory tempDir;
@ -67,57 +25,13 @@ void main() {
tryToDelete(tempDir as FileSystemEntity);
});
Future<ProcessResult> buildFlutterApkWithSpecifiedDependencyVersions({
required String gradleVersion,
required String agpVersion,
required String kgpVersion}) async {
// Create a new flutter project.
final String flutterBin = fileSystem.path.join(getFlutterRoot(), 'bin', 'flutter');
ProcessResult result = await processManager.run(<String>[
flutterBin,
'create',
'dependency_checker_app',
'--platforms=android',
], workingDirectory: tempDir.path);
expect(result, const ProcessResultMatcher());
final Directory app = Directory(fileSystem.path.join(tempDir.path, 'dependency_checker_app'));
// Modify gradle version to passed in version.
final File gradleWrapperProperties = File(fileSystem.path.join(
app.path, 'android', 'gradle', 'wrapper', 'gradle-wrapper.properties'));
final String propertyContent = gradleWrapperPropertiesFileContent.replaceFirst(
gradleReplacementString,
gradleVersion,
);
await gradleWrapperProperties.writeAsString(propertyContent, flush: true);
final File gradleSettings = File(fileSystem.path.join(
app.path, 'android', 'settings.gradle'));
final String settingsContent = gradleSettingsFileContent
.replaceFirst(agpReplacementString, agpVersion)
.replaceFirst(kgpReplacementString, kgpVersion);
await gradleSettings.writeAsString(settingsContent, flush: true);
// Ensure that gradle files exists from templates.
result = await processManager.run(<String>[
flutterBin,
'build',
'apk',
'--debug',
], workingDirectory: app.path);
return result;
}
testUsingContext(
'AGP version out of "warn" support band but in "error" band builds '
'successfully and prints warning', () async {
final VersionTuple versionTuple = VersionTuple(agpVersion: '7.0.0', gradleVersion: '7.5', kotlinVersion: '1.7.10');
final ProcessResult result = await buildFlutterApkWithSpecifiedDependencyVersions(
gradleVersion: '7.5',
agpVersion: '7.0.0',
kgpVersion: '1.7.10'
versions: versionTuple,
tempDir: tempDir
);
expect(result, const ProcessResultMatcher());
expect(result.stderr, contains('Please upgrade your Android Gradle Plugin version'));
@ -127,10 +41,10 @@ void main() {
'Gradle version out of "warn" support band but in "error" band builds '
'successfully and prints warning', () async {
// Create a new flutter project.
final VersionTuple versionTuple = VersionTuple(agpVersion: '7.0.0', gradleVersion: '7.0.2', kotlinVersion: '1.7.10');
final ProcessResult result = await buildFlutterApkWithSpecifiedDependencyVersions(
gradleVersion: '7.0.2',
agpVersion: '7.0.0',
kgpVersion: '1.7.10'
versions: versionTuple,
tempDir: tempDir
);
expect(result, const ProcessResultMatcher());
expect(result.stderr, contains('Please upgrade your Gradle version'));
@ -139,10 +53,10 @@ void main() {
testUsingContext(
'Kotlin version out of "warn" support band but in "error" band builds '
'successfully and prints warning', () async {
final VersionTuple versionTuple = VersionTuple(agpVersion: '7.4.0', gradleVersion: '7.5', kotlinVersion: '1.7.0');
final ProcessResult result = await buildFlutterApkWithSpecifiedDependencyVersions(
gradleVersion: '7.5',
agpVersion: '7.4.0',
kgpVersion: '1.7.0'
versions: versionTuple,
tempDir: tempDir
);
expect(result, const ProcessResultMatcher());

View File

@ -2,12 +2,17 @@
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
import 'dart:io';
import 'package:flutter_tools/src/android/android_builder.dart';
import 'package:flutter_tools/src/base/file_system.dart';
import 'package:flutter_tools/src/base/file_system.dart' as file_system;
import 'package:flutter_tools/src/build_info.dart';
import 'package:flutter_tools/src/globals.dart' as globals;
import 'package:flutter_tools/src/project.dart';
import '../integration.shard/test_utils.dart';
import 'common.dart';
/// A fake implementation of [AndroidBuilder].
class FakeAndroidBuilder implements AndroidBuilder {
@override
@ -56,7 +61,7 @@ class FakeFlutterProjectFactory extends FlutterProjectFactory {
logger: globals.logger,
);
final Directory directoryOverride;
final file_system.Directory directoryOverride;
@override
FlutterProject fromDirectory(Directory _) {
@ -64,3 +69,114 @@ class FakeFlutterProjectFactory extends FlutterProjectFactory {
return super.fromDirectory(directoryOverride.childDirectory('flutter_project'));
}
}
// The following test outline shares a lot of similarities with the one in
// dev/devicelab/lib/framework/dependency_smoke_test_task_definition.dart
// When making changes here, consider making the corresponding changes to that
// file as well.
const String gradleSettingsFileContent = r'''
pluginManagement {
def flutterSdkPath = {
def properties = new Properties()
file("local.properties").withInputStream { properties.load(it) }
def flutterSdkPath = properties.getProperty("flutter.sdk")
assert flutterSdkPath != null, "flutter.sdk not set in local.properties"
return flutterSdkPath
}()
includeBuild("$flutterSdkPath/packages/flutter_tools/gradle")
repositories {
google()
mavenCentral()
gradlePluginPortal()
}
}
plugins {
id "dev.flutter.flutter-plugin-loader" version "1.0.0"
id "com.android.application" version "AGP_REPLACE_ME" apply false
id "org.jetbrains.kotlin.android" version "KGP_REPLACE_ME" apply false
}
include ":app"
''';
const String agpReplacementString = 'AGP_REPLACE_ME';
const String kgpReplacementString = 'KGP_REPLACE_ME';
const String gradleWrapperPropertiesFileContent = r'''
distributionBase=GRADLE_USER_HOME
distributionPath=wrapper/dists
zipStoreBase=GRADLE_USER_HOME
zipStorePath=wrapper/dists
distributionUrl=https\://services.gradle.org/distributions/gradle-GRADLE_REPLACE_ME-all.zip
''';
const String gradleReplacementString = 'GRADLE_REPLACE_ME';
class VersionTuple {
VersionTuple({
required this.agpVersion,
required this.gradleVersion,
required this.kotlinVersion
});
String agpVersion;
String gradleVersion;
String kotlinVersion;
@override
String toString() {
return '(AGP version: $agpVersion, Gradle version: $gradleVersion, Kotlin version: $kotlinVersion)';
}
}
/// Creates a new Flutter project with the specified AGP, Gradle, and Kotlin
/// versions and then tries to call `flutter build apk`, returning the
/// ProcessResult.
Future<ProcessResult> buildFlutterApkWithSpecifiedDependencyVersions({
required VersionTuple versions,
required Directory tempDir,}) async {
// Create a new flutter project.
final String flutterBin = fileSystem.path.join(getFlutterRoot(), 'bin', 'flutter');
ProcessResult result = await processManager.run(<String>[
flutterBin,
'create',
'dependency_checker_app',
'--platforms=android',
], workingDirectory: tempDir.path);
expect(result, const ProcessResultMatcher());
final Directory app = Directory(fileSystem.path.join(tempDir.path, 'dependency_checker_app'));
// Modify gradle version to passed in version.
final File gradleWrapperProperties = File(fileSystem.path.join(
app.path, 'android', 'gradle', 'wrapper', 'gradle-wrapper.properties'));
final String propertyContent = gradleWrapperPropertiesFileContent.replaceFirst(
gradleReplacementString,
versions.gradleVersion,
);
await gradleWrapperProperties.writeAsString(propertyContent, flush: true);
final File gradleSettings = File(fileSystem.path.join(
app.path, 'android', 'settings.gradle'));
final String settingsContent = gradleSettingsFileContent
.replaceFirst(agpReplacementString, versions.agpVersion)
.replaceFirst(kgpReplacementString, versions.kotlinVersion);
await gradleSettings.writeAsString(settingsContent, flush: true);
// Ensure that gradle files exists from templates.
result = await processManager.run(<String>[
flutterBin,
'build',
'apk',
'--debug',
], workingDirectory: app.path);
return result;
}