firka/flutter
firka/flutter
5
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

[devicelab] reduce iterations, uninstall at end, and use --application-binary in all startup tests (#68516)

Browse Source 
These devicelab tests are incredibly flaky. Currently they will build & install the same application 15 times. This causes the temp storage to fill up on android, and has a good chance of flaking on iOS due to install issues.

The change from 3 to 15 increased total test time for 2 -> 12 minutes on iOS, or roughly 5 more test equivalents for every single iOS test. Reduce the iteration count back to 5

Uninstall the app after each run so temp storage does not fill up.
This commit is contained in:
Jonah Williams 2020-10-19 14:16:04 -07:00 committed by GitHub
parent 98c519dbe2
commit a4c2075a45
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 38 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

44
dev/devicelab/lib/tasks/perf_tests.dart
View File

@ -454,24 +454,56 @@ class StartupTest {
Future<TaskResult> run() async { Future<TaskResult> run() async {
return await inDirectory<TaskResult>(testDirectory, () async { return await inDirectory<TaskResult>(testDirectory, () async {
final String deviceId = (await devices.workingDevice).deviceId; final Device device = await devices.workingDevice;
await flutter('packages', options: <String>['get']); const int iterations = 5;
const int iterations = 15;
final List<Map<String, dynamic>> results = <Map<String, dynamic>>[]; final List<Map<String, dynamic>> results = <Map<String, dynamic>>[];
for (int i = 0; i < iterations; ++i) {
section('Building application');
String applicationBinaryPath;
switch (deviceOperatingSystem) {
case DeviceOperatingSystem.android:
await flutter('build', options: <String>[
'apk',
'-v',
'--profile',
'--target-platform=android-arm,android-arm64',
]);
applicationBinaryPath = '$testDirectory/build/app/outputs/flutter-apk/app-profile.apk';
break;
case DeviceOperatingSystem.ios:
await flutter('build', options: <String>[
'ios',
'-v',
'--profile',
]);
applicationBinaryPath = '$testDirectory/build/ios/iphoneos/Runner.app';
break;
case DeviceOperatingSystem.fuchsia:
case DeviceOperatingSystem.fake:
break;
}
for (int i = 0; i < iterations; i += 1) {
await flutter('run', options: <String>[ await flutter('run', options: <String>[
'--no-android-gradle-daemon', '--no-android-gradle-daemon',
'--verbose', '--verbose',
'--profile', '--profile',
'--trace-startup', '--trace-startup',
'-d', '-d',
deviceId, device.deviceId,
if (applicationBinaryPath != null)
'--use-application-binary=$applicationBinaryPath',
]); ]);
final Map<String, dynamic> data = json.decode( final Map<String, dynamic> data = json.decode(
file('$testDirectory/build/start_up_info.json').readAsStringSync(), file('$testDirectory/build/start_up_info.json').readAsStringSync(),
) as Map<String, dynamic>; ) as Map<String, dynamic>;
results.add(data); results.add(data);
await flutter('install', options: <String>[
'--uninstall-only',
'-d',
device.deviceId,
]);
} }
final Map<String, dynamic> averageResults = _average(results, iterations); final Map<String, dynamic> averageResults = _average(results, iterations);