diff --git a/dev/devicelab/bin/tasks/build_benchmark.dart b/dev/devicelab/bin/tasks/build_benchmark.dart deleted file mode 100644 index 9dd11bff0f..0000000000 --- a/dev/devicelab/bin/tasks/build_benchmark.dart +++ /dev/null @@ -1,10 +0,0 @@ -// Copyright 2014 The Flutter Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -import 'package:flutter_devicelab/framework/framework.dart'; -import 'package:flutter_devicelab/tasks/build_benchmarks.dart'; - -Future main() async { - await task(createAndroidBuildBenchmarkTask()); -} diff --git a/dev/devicelab/bin/tasks/build_benchmark_ios.dart b/dev/devicelab/bin/tasks/build_benchmark_ios.dart deleted file mode 100644 index 09a6bdbde4..0000000000 --- a/dev/devicelab/bin/tasks/build_benchmark_ios.dart +++ /dev/null @@ -1,10 +0,0 @@ -// Copyright 2014 The Flutter Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -import 'package:flutter_devicelab/framework/framework.dart'; -import 'package:flutter_devicelab/tasks/build_benchmarks.dart'; - -Future main() async { - await task(createIosBuildBenchmarkTask()); -} diff --git a/dev/devicelab/bin/tasks/hello_world__start_up.dart b/dev/devicelab/bin/tasks/hello_world__start_up.dart deleted file mode 100644 index 734123b473..0000000000 --- a/dev/devicelab/bin/tasks/hello_world__start_up.dart +++ /dev/null @@ -1,12 +0,0 @@ -// Copyright 2014 The Flutter Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -import 'package:flutter_devicelab/tasks/perf_tests.dart'; -import 'package:flutter_devicelab/framework/adb.dart'; -import 'package:flutter_devicelab/framework/framework.dart'; - -Future main() async { - deviceOperatingSystem = DeviceOperatingSystem.android; - await task(createHelloWorldStartupTest()); -} diff --git a/dev/devicelab/bin/tasks/mac_enable_twc.dart b/dev/devicelab/bin/tasks/mac_enable_twc.dart deleted file mode 100644 index 368472e8aa..0000000000 --- a/dev/devicelab/bin/tasks/mac_enable_twc.dart +++ /dev/null @@ -1,11 +0,0 @@ -// Copyright 2014 The Flutter Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -import 'package:flutter_devicelab/framework/framework.dart'; -import 'package:flutter_devicelab/tasks/track_widget_creation_enabled_task.dart'; - -/// Verify that twc can be enabled/disabled on macOS -Future main() async { - await task(TrackWidgetCreationEnabledTask('macos').task); -} diff --git a/dev/devicelab/bin/tasks/run_without_leak_linux.dart b/dev/devicelab/bin/tasks/run_without_leak_linux.dart deleted file mode 100644 index 41300991a8..0000000000 --- a/dev/devicelab/bin/tasks/run_without_leak_linux.dart +++ /dev/null @@ -1,12 +0,0 @@ -// Copyright 2014 The Flutter Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -import 'package:flutter_devicelab/framework/utils.dart'; -import 'package:flutter_devicelab/tasks/run_without_leak.dart'; -import 'package:flutter_devicelab/framework/framework.dart'; -import 'package:path/path.dart' as path; - -Future main() async { - await task(createRunWithoutLeakTest(path.join(flutterDirectory.path, 'examples', 'hello_world'))); -} diff --git a/dev/devicelab/bin/tasks/run_without_leak_mac.dart b/dev/devicelab/bin/tasks/run_without_leak_mac.dart deleted file mode 100644 index 41300991a8..0000000000 --- a/dev/devicelab/bin/tasks/run_without_leak_mac.dart +++ /dev/null @@ -1,12 +0,0 @@ -// Copyright 2014 The Flutter Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -import 'package:flutter_devicelab/framework/utils.dart'; -import 'package:flutter_devicelab/tasks/run_without_leak.dart'; -import 'package:flutter_devicelab/framework/framework.dart'; -import 'package:path/path.dart' as path; - -Future main() async { - await task(createRunWithoutLeakTest(path.join(flutterDirectory.path, 'examples', 'hello_world'))); -} diff --git a/dev/devicelab/bin/tasks/run_without_leak_win.dart b/dev/devicelab/bin/tasks/run_without_leak_win.dart deleted file mode 100644 index 41300991a8..0000000000 --- a/dev/devicelab/bin/tasks/run_without_leak_win.dart +++ /dev/null @@ -1,12 +0,0 @@ -// Copyright 2014 The Flutter Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -import 'package:flutter_devicelab/framework/utils.dart'; -import 'package:flutter_devicelab/tasks/run_without_leak.dart'; -import 'package:flutter_devicelab/framework/framework.dart'; -import 'package:path/path.dart' as path; - -Future main() async { - await task(createRunWithoutLeakTest(path.join(flutterDirectory.path, 'examples', 'hello_world'))); -} diff --git a/dev/devicelab/bin/tasks/system_debug_ios.dart b/dev/devicelab/bin/tasks/system_debug_ios.dart deleted file mode 100644 index 4865fb4ecb..0000000000 --- a/dev/devicelab/bin/tasks/system_debug_ios.dart +++ /dev/null @@ -1,90 +0,0 @@ -// Copyright 2014 The Flutter Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -import 'dart:io'; - -import 'package:flutter_devicelab/framework/adb.dart'; -import 'package:flutter_devicelab/framework/framework.dart'; -import 'package:flutter_devicelab/framework/task_result.dart'; -import 'package:flutter_devicelab/framework/utils.dart'; - -import 'service_extensions_test.dart'; - -Future main() async { - deviceOperatingSystem = DeviceOperatingSystem.ios; - await task(() async { - final Device device = await devices.workingDevice; - await device.unlock(); - final String deviceId = device.deviceId; - - final Directory testDirectory = - dir('${flutterDirectory.path}/dev/integration_tests/ui'); - await inDirectory(testDirectory, () async { - await flutter('packages', options: ['get']); - - await checkNoWarningHostLaunch(deviceId); - await checkNoWarningXcodeLaunch(deviceId); - await checkWarningHomeScreenLaunch(deviceId); - }); - return TaskResult.success({}); - }); -} - -const String expectedWarning = - 'Launching a debug-mode app from the home screen may cause problems.'; - -// When a debug-mode app is launched from the host there should be no warnings. -Future checkNoWarningHostLaunch(String deviceId) async { - final String output = await evalFlutter('drive', options: [ - '--debug', - '--verbose', - '--verbose-system-logs', - '-d', - deviceId, - 'lib/empty.dart', - ]); - - expect(!output.contains(expectedWarning)); -} - -// When a debug-mode app is launched from Xcode there should be no warnings. The -// Xcode launch is simulated by keeping LLDB attached throughout the lifetime of -// the app. -Future checkNoWarningXcodeLaunch(String deviceId) async { - await flutter('build', - options: ['ios', '--debug', '--verbose', 'lib/exit.dart']); - - final String output = await eval( - '${flutterDirectory.path}/bin/cache/artifacts/ios-deploy/ios-deploy', - [ - '--bundle', - 'build/ios/iphoneos/Runner.app', - '-d', // Actually start the app in LLDB, don't just install it. - '--noninteractive', - '--args', - '--verbose-logging', - ]); - - expect(output.contains('success') && !output.contains(expectedWarning)); -} - -// When a debug-mode app is launched from the home screen there should be a -// warning every ~100 launches. We lower the threshold from to 1 via -// "--verbose-system-logs" and simulate a home-screen-launch by setting an -// environment variable. The environment variable forces "flutter drive" to not -// pass a flag which it normally passes to debug-mode apps, imitating launchd, -// which doesn't pass any command-line flags. -Future checkWarningHomeScreenLaunch(String deviceId) async { - final String output = await evalFlutter('drive', options: [ - '--debug', - '--verbose', - '--verbose-system-logs', - '-d', - deviceId, - 'lib/empty.dart', - ], environment: { - 'FLUTTER_TOOLS_DEBUG_WITHOUT_CHECKED_MODE': 'true', - }); - expect(output.contains(expectedWarning)); -} diff --git a/dev/devicelab/bin/tasks/time_to_development_benchmark__android.dart b/dev/devicelab/bin/tasks/time_to_development_benchmark__android.dart deleted file mode 100644 index 9b2cb2b1f0..0000000000 --- a/dev/devicelab/bin/tasks/time_to_development_benchmark__android.dart +++ /dev/null @@ -1,81 +0,0 @@ -// Copyright 2014 The Flutter Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -import 'dart:io'; - -import 'package:flutter_devicelab/framework/adb.dart'; -import 'package:flutter_devicelab/framework/framework.dart'; -import 'package:flutter_devicelab/framework/task_result.dart'; -import 'package:flutter_devicelab/framework/utils.dart'; -import 'package:path/path.dart' as path; - -Future main() async { - deviceOperatingSystem = DeviceOperatingSystem.android; - await task(createTimeToDevelopmentCommand); -} - -final Directory flutterGalleryDir = - dir(path.join(flutterDirectory.path, 'dev', 'integration_tests', 'flutter_gallery')); -final Directory editedFlutterGalleryDir = - dir(path.join(Directory.systemTemp.path, 'edited_flutter_gallery')); - -Future createTimeToDevelopmentCommand() async { - final Map allResults = {}; - bool failed = false; - await inDirectory(flutterDirectory, () async { - rmTree(editedFlutterGalleryDir); - mkdirs(editedFlutterGalleryDir); - recursiveCopy(flutterGalleryDir, editedFlutterGalleryDir); - await inDirectory(editedFlutterGalleryDir, () async { - final Device device = await devices.workingDevice; - await device.unlock(); - final Stopwatch stopwatch = Stopwatch()..start(); - final Process initialBuild = await startProcess( - path.join(flutterDirectory.path, 'bin', 'flutter'), - ['run', '--debug', '-d', device.deviceId, '--no-resident',], - ); - int exitCode = await initialBuild.exitCode; - if (exitCode != 0) { - failed = true; - return; - } - final int initialBuildMilliseconds = stopwatch.elapsedMilliseconds; - stopwatch - ..reset() - ..start(); - // Update a source file. - final File appDartSource = file(path.join( - editedFlutterGalleryDir.path, - 'lib/gallery/app.dart', - )); - appDartSource.writeAsStringSync( - appDartSource.readAsStringSync().replaceFirst( - "'Flutter Gallery'", - "'Updated Flutter Gallery'", - ), - ); - - final Process secondBuild = await startProcess( - path.join(flutterDirectory.path, 'bin', 'flutter'), - ['run', '--debug', '-d', device.deviceId, '--no-resident'], - environment: null, - ); - exitCode = await secondBuild.exitCode; - if (exitCode != 0) { - failed = true; - return; - } - stopwatch.stop(); - allResults['time_to_development'] = initialBuildMilliseconds.toDouble(); - allResults['time_to_development_incremental'] = - stopwatch.elapsedMilliseconds.toDouble(); - }); - }); - if (failed) { - return TaskResult.failure('Failed to build debug app'); - } - - return TaskResult.success(allResults, - benchmarkScoreKeys: allResults.keys.toList()); -} diff --git a/dev/devicelab/lib/tasks/build_benchmarks.dart b/dev/devicelab/lib/tasks/build_benchmarks.dart deleted file mode 100644 index c41a77c640..0000000000 --- a/dev/devicelab/lib/tasks/build_benchmarks.dart +++ /dev/null @@ -1,84 +0,0 @@ -// Copyright 2014 The Flutter Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -import 'dart:io'; - -import 'package:flutter_devicelab/framework/framework.dart'; -import 'package:flutter_devicelab/framework/task_result.dart'; -import 'package:flutter_devicelab/framework/utils.dart'; -import 'package:path/path.dart' as path; - -final Directory helloWorldDir = dir(path.join(flutterDirectory.path, 'examples', 'hello_world')); - -/// Creates a devicelab build benchmark for Android. -TaskFunction createAndroidBuildBenchmarkTask() { - return () async { - return createBuildCommand('apk'); - }; -} - -/// Creates a devicelab build benchmark for iOS. -TaskFunction createIosBuildBenchmarkTask() { - return () async { - return createBuildCommand('ios'); - }; -} - -Future createBuildCommand(String buildKind) { - return inDirectory(helloWorldDir, () async { - final Stopwatch stopwatch = Stopwatch() - ..start(); - final Process initialBuild = await startProcess( - path.join(flutterDirectory.path, 'bin', 'flutter'), - ['build', buildKind, '--debug'], - environment: null, - ); - int exitCode = await initialBuild.exitCode; - if (exitCode != 0) { - return TaskResult.failure('Failed to build debug app'); - } - final int initialBuildMilliseconds = stopwatch.elapsedMilliseconds; - stopwatch - ..reset() - ..start(); - final Process secondBuild = await startProcess( - path.join(flutterDirectory.path, 'bin', 'flutter'), - ['build', buildKind, '--debug'], - environment: null, - ); - exitCode = await secondBuild.exitCode; - if (exitCode != 0) { - return TaskResult.failure('Failed to build debug app'); - } - final int secondBuildMilliseconds = stopwatch.elapsedMilliseconds; - final Process newBuildConfig = await startProcess( - path.join(flutterDirectory.path, 'bin', 'flutter'), - ['build', buildKind, '--profile'], - environment: null, - ); - exitCode = await newBuildConfig.exitCode; - if (exitCode != 0) { - return TaskResult.failure('Failed to build profile app'); - } - stopwatch - ..reset() - ..start(); - final Process thirdBuild = await startProcess( - path.join(flutterDirectory.path, 'bin', 'flutter'), - ['build', buildKind, '--debug'], - environment: null, - ); - exitCode = await thirdBuild.exitCode; - if (exitCode != 0) { - return TaskResult.failure('Failed to build debug app'); - } - final int thirdBuildMilliseconds = stopwatch.elapsedMilliseconds; - stopwatch.stop(); - final Map allResults = {}; - allResults['first_build_debug_millis'] = initialBuildMilliseconds.toDouble(); - allResults['second_build_debug_millis'] = secondBuildMilliseconds.toDouble(); - allResults['after_config_change_build_debug_millis'] = thirdBuildMilliseconds.toDouble(); - return TaskResult.success(allResults, benchmarkScoreKeys: allResults.keys.toList()); - }); -} diff --git a/dev/devicelab/lib/tasks/perf_tests.dart b/dev/devicelab/lib/tasks/perf_tests.dart index 290704675c..d96e03cdb2 100644 --- a/dev/devicelab/lib/tasks/perf_tests.dart +++ b/dev/devicelab/lib/tasks/perf_tests.dart @@ -218,13 +218,6 @@ TaskFunction createComplexLayoutStartupTest() { ).run; } -TaskFunction createHelloWorldStartupTest() { - return StartupTest( - '${flutterDirectory.path}/examples/hello_world', - reportMetrics: false, - ).run; -} - TaskFunction createFlutterGalleryCompileTest() { return CompileTest('${flutterDirectory.path}/dev/integration_tests/flutter_gallery').run; } diff --git a/dev/devicelab/lib/tasks/run_without_leak.dart b/dev/devicelab/lib/tasks/run_without_leak.dart deleted file mode 100644 index acb997356e..0000000000 --- a/dev/devicelab/lib/tasks/run_without_leak.dart +++ /dev/null @@ -1,61 +0,0 @@ -// Copyright 2014 The Flutter Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -import 'dart:async'; -import 'dart:convert'; -import 'dart:io'; - -import 'package:path/path.dart' as path; - -import '../framework/adb.dart'; -import '../framework/framework.dart'; -import '../framework/task_result.dart'; -import '../framework/utils.dart'; - -TaskFunction createRunWithoutLeakTest(dynamic dir) { - return () async { - final Device device = await devices.workingDevice; - await device.unlock(); - final List options = [ - '-d', device.deviceId, '--verbose', - ]; - int exitCode; - await inDirectory(dir, () async { - final Process process = await startProcess( - path.join(flutterDirectory.path, 'bin', 'flutter'), - flutterCommandArgs('run', options), - environment: null, - ); - final Completer stdoutDone = Completer(); - final Completer stderrDone = Completer(); - process.stdout - .transform(utf8.decoder) - .transform(const LineSplitter()) - .listen((String line) { - if (line.contains('] For a more detailed help message, press "h". To detach, press "d"; to quit, press "q"')) { - process.stdin.writeln('q'); - } - print('stdout: $line'); - }, onDone: () { - stdoutDone.complete(); - }); - process.stderr - .transform(utf8.decoder) - .transform(const LineSplitter()) - .listen((String line) { - print('stderr: $line'); - }, onDone: () { - stderrDone.complete(); - }); - - await Future.wait( - >[stdoutDone.future, stderrDone.future]); - exitCode = await process.exitCode; - }); - - return exitCode == 0 - ? TaskResultCheckProcesses() - : TaskResult.failure('Failed to run $dir'); - }; -} diff --git a/dev/devicelab/manifest.yaml b/dev/devicelab/manifest.yaml index 1f48582a8d..70d214ddc3 100644 --- a/dev/devicelab/manifest.yaml +++ b/dev/devicelab/manifest.yaml @@ -389,12 +389,6 @@ tasks: stage: devicelab required_agent_capabilities: ["mac/android"] - # hello_world__start_up: - # description: > - # Verifies that Hello World can start on an array of devices. - # stage: devicelab - # required_agent_capabilities: ["linux/android_esoteric"] - microbenchmarks: description: > Runs benchmarks from dev/benchmarks/microbenchmarks. @@ -480,18 +474,6 @@ tasks: stage: devicelab required_agent_capabilities: ["linux/android"] - build_benchmark: - description: > - Measures APK build performance across config changes. - stage: devicelab - required_agent_capabilities: ["linux/android"] - - time_to_development_benchmark__android: - description: > - Measures time from flutter run until resident runner setup is complete. - stage: devicelab - required_agent_capabilities: ["linux/android"] - # iOS on-device tests ios_defines_test: @@ -644,25 +626,12 @@ tasks: stage: devicelab_ios required_agent_capabilities: ["mac/ios"] - # TODO(fujino): does not pass on iOS13 https://github.com/flutter/flutter/issues/43029 - # system_debug_ios: - # description: > - # Tests that the Engine correctly initializes the system debugger for debug-mode iOS apps. - # stage: devicelab_ios - # required_agent_capabilities: ["mac/ios"] - macos_chrome_dev_mode: description: > Run flutter web on the devicelab and hot restart. stage: devicelab_ios required_agent_capabilities: ["mac/ios"] - build_benchmark_ios: - description: > - Measures iOS build performance across config changes. - stage: devicelab_ios - required_agent_capabilities: ["mac/ios"] - simple_animation_perf_ios: description: > Measure CPU/GPU usage percentages of a simple animation. @@ -702,14 +671,6 @@ tasks: stage: devicelab required_agent_capabilities: ["mac/ios"] -# TODO(jonahwilliams): investigate build failures on devicelab infra to re-enable. -# mac_enable_twc: -# description: > -# Verifies that track-widget-creation can be enabled and disabled. -# stage: devicelab_ios -# required_agent_capabilities: ["mac/ios"] -# flaky: true - ios_app_with_extensions_test: description: > Checks that an iOS app with extensions can be built for physical and simulated devices. @@ -742,12 +703,6 @@ tasks: stage: devicelab_win required_agent_capabilities: ["windows/android"] - # run_without_leak_win: - # description: > - # Checks that `flutter run` does not leak dart.exe on Windows. - # stage: devicelab_win - # required_agent_capabilities: ["windows/android"] - # Tests running on Linux hosts hot_mode_dev_cycle_linux__benchmark: @@ -939,20 +894,6 @@ tasks: stage: devicelab required_agent_capabilities: ["linux-vm"] - # run_without_leak_linux: - # description: > - # Checks that `flutter run` does not leak dart on Linux. - # stage: devicelab - # required_agent_capabilities: ["linux/android"] - # flaky: true - - # run_without_leak_mac: - # description: > - # Checks that `flutter run` does not leak dart on macOS. - # stage: devicelab - # required_agent_capabilities: ["mac/android"] - # flaky: true - # android_splash_screen_integration_test: # description: > # Runs end-to-end test of Flutter's Android splash behavior.