firka/flutter
firka/flutter
5
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

[devicelab] delete disabled or not useful benchmarks (#68107)

Browse Source 
- run_without_leak_tests: have been disabled for months
- build_benchmark: the tooling work here is mostly done and we're not tracking further improvements, free up more devicelab capacity
- system_debug_ios: does not work post iOS13
- mac_enable_twc: not adding more mac tests to devicelab
- hello_world_start_up: disabled

Any tests that we think will be valuable in the future can be resurrected from the git history.
This commit is contained in:
Jonah Williams 2020-10-14 10:30:10 -07:00 committed by GitHub
parent 431c599bec
commit 33903c17e1
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
13 changed files with 0 additions and 461 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
dev/devicelab/bin/tasks/build_benchmark.dart
View File

@ -1,10 +0,0 @@
// Copyright 2014 The Flutter Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
import 'package:flutter_devicelab/framework/framework.dart';
import 'package:flutter_devicelab/tasks/build_benchmarks.dart';
Future<void> main() async {
await task(createAndroidBuildBenchmarkTask());
}

10
dev/devicelab/bin/tasks/build_benchmark_ios.dart
View File

@ -1,10 +0,0 @@
// Copyright 2014 The Flutter Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
import 'package:flutter_devicelab/framework/framework.dart';
import 'package:flutter_devicelab/tasks/build_benchmarks.dart';
Future<void> main() async {
await task(createIosBuildBenchmarkTask());
}

12
dev/devicelab/bin/tasks/hello_world__start_up.dart
View File

@ -1,12 +0,0 @@
// Copyright 2014 The Flutter Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
import 'package:flutter_devicelab/tasks/perf_tests.dart';
import 'package:flutter_devicelab/framework/adb.dart';
import 'package:flutter_devicelab/framework/framework.dart';
Future<void> main() async {
deviceOperatingSystem = DeviceOperatingSystem.android;
await task(createHelloWorldStartupTest());
}

11
dev/devicelab/bin/tasks/mac_enable_twc.dart
View File

@ -1,11 +0,0 @@
// Copyright 2014 The Flutter Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
import 'package:flutter_devicelab/framework/framework.dart';
import 'package:flutter_devicelab/tasks/track_widget_creation_enabled_task.dart';
/// Verify that twc can be enabled/disabled on macOS
Future<void> main() async {
await task(TrackWidgetCreationEnabledTask('macos').task);
}

12
dev/devicelab/bin/tasks/run_without_leak_linux.dart
View File

@ -1,12 +0,0 @@
// Copyright 2014 The Flutter Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
import 'package:flutter_devicelab/framework/utils.dart';
import 'package:flutter_devicelab/tasks/run_without_leak.dart';
import 'package:flutter_devicelab/framework/framework.dart';
import 'package:path/path.dart' as path;
Future<void> main() async {
await task(createRunWithoutLeakTest(path.join(flutterDirectory.path, 'examples', 'hello_world')));
}

12
dev/devicelab/bin/tasks/run_without_leak_mac.dart
View File

@ -1,12 +0,0 @@
// Copyright 2014 The Flutter Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
import 'package:flutter_devicelab/framework/utils.dart';
import 'package:flutter_devicelab/tasks/run_without_leak.dart';
import 'package:flutter_devicelab/framework/framework.dart';
import 'package:path/path.dart' as path;
Future<void> main() async {
await task(createRunWithoutLeakTest(path.join(flutterDirectory.path, 'examples', 'hello_world')));
}

12
dev/devicelab/bin/tasks/run_without_leak_win.dart
View File

@ -1,12 +0,0 @@
// Copyright 2014 The Flutter Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
import 'package:flutter_devicelab/framework/utils.dart';
import 'package:flutter_devicelab/tasks/run_without_leak.dart';
import 'package:flutter_devicelab/framework/framework.dart';
import 'package:path/path.dart' as path;
Future<void> main() async {
await task(createRunWithoutLeakTest(path.join(flutterDirectory.path, 'examples', 'hello_world')));
}

90
dev/devicelab/bin/tasks/system_debug_ios.dart
View File

@ -1,90 +0,0 @@
// Copyright 2014 The Flutter Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
import 'dart:io';
import 'package:flutter_devicelab/framework/adb.dart';
import 'package:flutter_devicelab/framework/framework.dart';
import 'package:flutter_devicelab/framework/task_result.dart';
import 'package:flutter_devicelab/framework/utils.dart';
import 'service_extensions_test.dart';
Future<void> main() async {
deviceOperatingSystem = DeviceOperatingSystem.ios;
await task(() async {
final Device device = await devices.workingDevice;
await device.unlock();
final String deviceId = device.deviceId;
final Directory testDirectory =
dir('${flutterDirectory.path}/dev/integration_tests/ui');
await inDirectory<void>(testDirectory, () async {
await flutter('packages', options: <String>['get']);
await checkNoWarningHostLaunch(deviceId);
await checkNoWarningXcodeLaunch(deviceId);
await checkWarningHomeScreenLaunch(deviceId);
});
return TaskResult.success(<String, dynamic>{});
});
}
const String expectedWarning =
'Launching a debug-mode app from the home screen may cause problems.';
// When a debug-mode app is launched from the host there should be no warnings.
Future<void> checkNoWarningHostLaunch(String deviceId) async {
final String output = await evalFlutter('drive', options: <String>[
'--debug',
'--verbose',
'--verbose-system-logs',
'-d',
deviceId,
'lib/empty.dart',
]);
expect(!output.contains(expectedWarning));
}
// When a debug-mode app is launched from Xcode there should be no warnings. The
// Xcode launch is simulated by keeping LLDB attached throughout the lifetime of
// the app.
Future<void> checkNoWarningXcodeLaunch(String deviceId) async {
await flutter('build',
options: <String>['ios', '--debug', '--verbose', 'lib/exit.dart']);
final String output = await eval(
'${flutterDirectory.path}/bin/cache/artifacts/ios-deploy/ios-deploy',
<String>[
'--bundle',
'build/ios/iphoneos/Runner.app',
'-d', // Actually start the app in LLDB, don't just install it.
'--noninteractive',
'--args',
'--verbose-logging',
]);
expect(output.contains('success') && !output.contains(expectedWarning));
}
// When a debug-mode app is launched from the home screen there should be a
// warning every ~100 launches. We lower the threshold from to 1 via
// "--verbose-system-logs" and simulate a home-screen-launch by setting an
// environment variable. The environment variable forces "flutter drive" to not
// pass a flag which it normally passes to debug-mode apps, imitating launchd,
// which doesn't pass any command-line flags.
Future<void> checkWarningHomeScreenLaunch(String deviceId) async {
final String output = await evalFlutter('drive', options: <String>[
'--debug',
'--verbose',
'--verbose-system-logs',
'-d',
deviceId,
'lib/empty.dart',
], environment: <String, String>{
'FLUTTER_TOOLS_DEBUG_WITHOUT_CHECKED_MODE': 'true',
});
expect(output.contains(expectedWarning));
}

81
dev/devicelab/bin/tasks/time_to_development_benchmark__android.dart
View File

@ -1,81 +0,0 @@
// Copyright 2014 The Flutter Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
import 'dart:io';
import 'package:flutter_devicelab/framework/adb.dart';
import 'package:flutter_devicelab/framework/framework.dart';
import 'package:flutter_devicelab/framework/task_result.dart';
import 'package:flutter_devicelab/framework/utils.dart';
import 'package:path/path.dart' as path;
Future<void> main() async {
deviceOperatingSystem = DeviceOperatingSystem.android;
await task(createTimeToDevelopmentCommand);
}
final Directory flutterGalleryDir =
dir(path.join(flutterDirectory.path, 'dev', 'integration_tests', 'flutter_gallery'));
final Directory editedFlutterGalleryDir =
dir(path.join(Directory.systemTemp.path, 'edited_flutter_gallery'));
Future<TaskResult> createTimeToDevelopmentCommand() async {
final Map<String, double> allResults = <String, double>{};
bool failed = false;
await inDirectory<void>(flutterDirectory, () async {
rmTree(editedFlutterGalleryDir);
mkdirs(editedFlutterGalleryDir);
recursiveCopy(flutterGalleryDir, editedFlutterGalleryDir);
await inDirectory<void>(editedFlutterGalleryDir, () async {
final Device device = await devices.workingDevice;
await device.unlock();
final Stopwatch stopwatch = Stopwatch()..start();
final Process initialBuild = await startProcess(
path.join(flutterDirectory.path, 'bin', 'flutter'),
<String>['run', '--debug', '-d', device.deviceId, '--no-resident',],
);
int exitCode = await initialBuild.exitCode;
if (exitCode != 0) {
failed = true;
return;
}
final int initialBuildMilliseconds = stopwatch.elapsedMilliseconds;
stopwatch
..reset()
..start();
// Update a source file.
final File appDartSource = file(path.join(
editedFlutterGalleryDir.path,
'lib/gallery/app.dart',
));
appDartSource.writeAsStringSync(
appDartSource.readAsStringSync().replaceFirst(
"'Flutter Gallery'",
"'Updated Flutter Gallery'",
),
);
final Process secondBuild = await startProcess(
path.join(flutterDirectory.path, 'bin', 'flutter'),
<String>['run', '--debug', '-d', device.deviceId, '--no-resident'],
environment: null,
);
exitCode = await secondBuild.exitCode;
if (exitCode != 0) {
failed = true;
return;
}
stopwatch.stop();
allResults['time_to_development'] = initialBuildMilliseconds.toDouble();
allResults['time_to_development_incremental'] =
stopwatch.elapsedMilliseconds.toDouble();
});
});
if (failed) {
return TaskResult.failure('Failed to build debug app');
}
return TaskResult.success(allResults,
benchmarkScoreKeys: allResults.keys.toList());
}

84
dev/devicelab/lib/tasks/build_benchmarks.dart
View File

@ -1,84 +0,0 @@
// Copyright 2014 The Flutter Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
import 'dart:io';
import 'package:flutter_devicelab/framework/framework.dart';
import 'package:flutter_devicelab/framework/task_result.dart';
import 'package:flutter_devicelab/framework/utils.dart';
import 'package:path/path.dart' as path;
final Directory helloWorldDir = dir(path.join(flutterDirectory.path, 'examples', 'hello_world'));
/// Creates a devicelab build benchmark for Android.
TaskFunction createAndroidBuildBenchmarkTask() {
return () async {
return createBuildCommand('apk');
};
}
/// Creates a devicelab build benchmark for iOS.
TaskFunction createIosBuildBenchmarkTask() {
return () async {
return createBuildCommand('ios');
};
}
Future<TaskResult> createBuildCommand(String buildKind) {
return inDirectory<TaskResult>(helloWorldDir, () async {
final Stopwatch stopwatch = Stopwatch()
..start();
final Process initialBuild = await startProcess(
path.join(flutterDirectory.path, 'bin', 'flutter'),
<String>['build', buildKind, '--debug'],
environment: null,
);
int exitCode = await initialBuild.exitCode;
if (exitCode != 0) {
return TaskResult.failure('Failed to build debug app');
}
final int initialBuildMilliseconds = stopwatch.elapsedMilliseconds;
stopwatch
..reset()
..start();
final Process secondBuild = await startProcess(
path.join(flutterDirectory.path, 'bin', 'flutter'),
<String>['build', buildKind, '--debug'],
environment: null,
);
exitCode = await secondBuild.exitCode;
if (exitCode != 0) {
return TaskResult.failure('Failed to build debug app');
}
final int secondBuildMilliseconds = stopwatch.elapsedMilliseconds;
final Process newBuildConfig = await startProcess(
path.join(flutterDirectory.path, 'bin', 'flutter'),
<String>['build', buildKind, '--profile'],
environment: null,
);
exitCode = await newBuildConfig.exitCode;
if (exitCode != 0) {
return TaskResult.failure('Failed to build profile app');
}
stopwatch
..reset()
..start();
final Process thirdBuild = await startProcess(
path.join(flutterDirectory.path, 'bin', 'flutter'),
<String>['build', buildKind, '--debug'],
environment: null,
);
exitCode = await thirdBuild.exitCode;
if (exitCode != 0) {
return TaskResult.failure('Failed to build debug app');
}
final int thirdBuildMilliseconds = stopwatch.elapsedMilliseconds;
stopwatch.stop();
final Map<String, double> allResults = <String, double>{};
allResults['first_build_debug_millis'] = initialBuildMilliseconds.toDouble();
allResults['second_build_debug_millis'] = secondBuildMilliseconds.toDouble();
allResults['after_config_change_build_debug_millis'] = thirdBuildMilliseconds.toDouble();
return TaskResult.success(allResults, benchmarkScoreKeys: allResults.keys.toList());
});
}

7
dev/devicelab/lib/tasks/perf_tests.dart
View File

@ -218,13 +218,6 @@ TaskFunction createComplexLayoutStartupTest() {
).run;
}
TaskFunction createHelloWorldStartupTest() {
return StartupTest(
'${flutterDirectory.path}/examples/hello_world',
reportMetrics: false,
).run;
}
TaskFunction createFlutterGalleryCompileTest() {
return CompileTest('${flutterDirectory.path}/dev/integration_tests/flutter_gallery').run;
}

61
dev/devicelab/lib/tasks/run_without_leak.dart
View File

@ -1,61 +0,0 @@
// Copyright 2014 The Flutter Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
import 'dart:async';
import 'dart:convert';
import 'dart:io';
import 'package:path/path.dart' as path;
import '../framework/adb.dart';
import '../framework/framework.dart';
import '../framework/task_result.dart';
import '../framework/utils.dart';
TaskFunction createRunWithoutLeakTest(dynamic dir) {
return () async {
final Device device = await devices.workingDevice;
await device.unlock();
final List<String> options = <String>[
'-d', device.deviceId, '--verbose',
];
int exitCode;
await inDirectory<void>(dir, () async {
final Process process = await startProcess(
path.join(flutterDirectory.path, 'bin', 'flutter'),
flutterCommandArgs('run', options),
environment: null,
);
final Completer<void> stdoutDone = Completer<void>();
final Completer<void> stderrDone = Completer<void>();
process.stdout
.transform<String>(utf8.decoder)
.transform<String>(const LineSplitter())
.listen((String line) {
if (line.contains('] For a more detailed help message, press "h". To detach, press "d"; to quit, press "q"')) {
process.stdin.writeln('q');
}
print('stdout: $line');
}, onDone: () {
stdoutDone.complete();
});
process.stderr
.transform<String>(utf8.decoder)
.transform<String>(const LineSplitter())
.listen((String line) {
print('stderr: $line');
}, onDone: () {
stderrDone.complete();
});
await Future.wait<void>(
<Future<void>>[stdoutDone.future, stderrDone.future]);
exitCode = await process.exitCode;
});
return exitCode == 0
? TaskResultCheckProcesses()
: TaskResult.failure('Failed to run $dir');
};
}

59
dev/devicelab/manifest.yaml
View File

@ -389,12 +389,6 @@ tasks:
stage: devicelab
required_agent_capabilities: ["mac/android"]
# hello_world__start_up:
# description: >
# Verifies that Hello World can start on an array of devices.
# stage: devicelab
# required_agent_capabilities: ["linux/android_esoteric"]
microbenchmarks:
description: >
Runs benchmarks from dev/benchmarks/microbenchmarks.
@ -480,18 +474,6 @@ tasks:
stage: devicelab
required_agent_capabilities: ["linux/android"]
build_benchmark:
description: >
Measures APK build performance across config changes.
stage: devicelab
required_agent_capabilities: ["linux/android"]
time_to_development_benchmark__android:
description: >
Measures time from flutter run until resident runner setup is complete.
stage: devicelab
required_agent_capabilities: ["linux/android"]
# iOS on-device tests
ios_defines_test:
@ -644,25 +626,12 @@ tasks:
stage: devicelab_ios
required_agent_capabilities: ["mac/ios"]
# TODO(fujino): does not pass on iOS13 https://github.com/flutter/flutter/issues/43029
# system_debug_ios:
# description: >
# Tests that the Engine correctly initializes the system debugger for debug-mode iOS apps.
# stage: devicelab_ios
# required_agent_capabilities: ["mac/ios"]
macos_chrome_dev_mode:
description: >
Run flutter web on the devicelab and hot restart.
stage: devicelab_ios
required_agent_capabilities: ["mac/ios"]
build_benchmark_ios:
description: >
Measures iOS build performance across config changes.
stage: devicelab_ios
required_agent_capabilities: ["mac/ios"]
simple_animation_perf_ios:
description: >
Measure CPU/GPU usage percentages of a simple animation.
@ -702,14 +671,6 @@ tasks:
stage: devicelab
required_agent_capabilities: ["mac/ios"]
# TODO(jonahwilliams): investigate build failures on devicelab infra to re-enable.
# mac_enable_twc:
# description: >
# Verifies that track-widget-creation can be enabled and disabled.
# stage: devicelab_ios
# required_agent_capabilities: ["mac/ios"]
# flaky: true
ios_app_with_extensions_test:
description: >
Checks that an iOS app with extensions can be built for physical and simulated devices.
@ -742,12 +703,6 @@ tasks:
stage: devicelab_win
required_agent_capabilities: ["windows/android"]
# run_without_leak_win:
# description: >
# Checks that `flutter run` does not leak dart.exe on Windows.
# stage: devicelab_win
# required_agent_capabilities: ["windows/android"]
# Tests running on Linux hosts
hot_mode_dev_cycle_linux__benchmark:
@ -939,20 +894,6 @@ tasks:
stage: devicelab
required_agent_capabilities: ["linux-vm"]
# run_without_leak_linux:
# description: >
# Checks that `flutter run` does not leak dart on Linux.
# stage: devicelab
# required_agent_capabilities: ["linux/android"]
# flaky: true
# run_without_leak_mac:
# description: >
# Checks that `flutter run` does not leak dart on macOS.
# stage: devicelab
# required_agent_capabilities: ["mac/android"]
# flaky: true
# android_splash_screen_integration_test:
# description: >
# Runs end-to-end test of Flutter's Android splash behavior.