[devicelab] delete disabled or not useful benchmarks (#68107)

- run_without_leak_tests: have been disabled for months
- build_benchmark: the tooling work here is mostly done and we're not tracking further improvements, free up more devicelab capacity
- system_debug_ios: does not work post iOS13
- mac_enable_twc: not adding more mac tests to devicelab
- hello_world_start_up: disabled

Any tests that we think will be valuable in the future can be resurrected from the git history.

dev/devicelab/bin/tasks/build_benchmark.dart

@@ -1,10 +0,0 @@
-// Copyright 2014 The Flutter Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file.
-
-import 'package:flutter_devicelab/framework/framework.dart';
-import 'package:flutter_devicelab/tasks/build_benchmarks.dart';
-
-Future<void> main() async {
-  await task(createAndroidBuildBenchmarkTask());
-}

dev/devicelab/bin/tasks/build_benchmark_ios.dart

@@ -1,10 +0,0 @@
-// Copyright 2014 The Flutter Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file.
-
-import 'package:flutter_devicelab/framework/framework.dart';
-import 'package:flutter_devicelab/tasks/build_benchmarks.dart';
-
-Future<void> main() async {
-  await task(createIosBuildBenchmarkTask());
-}

dev/devicelab/bin/tasks/hello_world__start_up.dart

@@ -1,12 +0,0 @@
-// Copyright 2014 The Flutter Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file.
-
-import 'package:flutter_devicelab/tasks/perf_tests.dart';
-import 'package:flutter_devicelab/framework/adb.dart';
-import 'package:flutter_devicelab/framework/framework.dart';
-
-Future<void> main() async {
-  deviceOperatingSystem = DeviceOperatingSystem.android;
-  await task(createHelloWorldStartupTest());
-}

dev/devicelab/bin/tasks/mac_enable_twc.dart

@@ -1,11 +0,0 @@
-// Copyright 2014 The Flutter Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file.
-
-import 'package:flutter_devicelab/framework/framework.dart';
-import 'package:flutter_devicelab/tasks/track_widget_creation_enabled_task.dart';
-
-/// Verify that twc can be enabled/disabled on macOS
-Future<void> main() async {
-  await task(TrackWidgetCreationEnabledTask('macos').task);
-}

dev/devicelab/bin/tasks/run_without_leak_linux.dart

@@ -1,12 +0,0 @@
-// Copyright 2014 The Flutter Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file.
-
-import 'package:flutter_devicelab/framework/utils.dart';
-import 'package:flutter_devicelab/tasks/run_without_leak.dart';
-import 'package:flutter_devicelab/framework/framework.dart';
-import 'package:path/path.dart' as path;
-
-Future<void> main() async {
-  await task(createRunWithoutLeakTest(path.join(flutterDirectory.path, 'examples', 'hello_world')));
-}

dev/devicelab/bin/tasks/run_without_leak_mac.dart

@@ -1,12 +0,0 @@
-// Copyright 2014 The Flutter Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file.
-
-import 'package:flutter_devicelab/framework/utils.dart';
-import 'package:flutter_devicelab/tasks/run_without_leak.dart';
-import 'package:flutter_devicelab/framework/framework.dart';
-import 'package:path/path.dart' as path;
-
-Future<void> main() async {
-  await task(createRunWithoutLeakTest(path.join(flutterDirectory.path, 'examples', 'hello_world')));
-}

dev/devicelab/bin/tasks/run_without_leak_win.dart

@@ -1,12 +0,0 @@
-// Copyright 2014 The Flutter Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file.
-
-import 'package:flutter_devicelab/framework/utils.dart';
-import 'package:flutter_devicelab/tasks/run_without_leak.dart';
-import 'package:flutter_devicelab/framework/framework.dart';
-import 'package:path/path.dart' as path;
-
-Future<void> main() async {
-  await task(createRunWithoutLeakTest(path.join(flutterDirectory.path, 'examples', 'hello_world')));
-}

dev/devicelab/bin/tasks/system_debug_ios.dart

@@ -1,90 +0,0 @@
-// Copyright 2014 The Flutter Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file.
-
-import 'dart:io';
-
-import 'package:flutter_devicelab/framework/adb.dart';
-import 'package:flutter_devicelab/framework/framework.dart';
-import 'package:flutter_devicelab/framework/task_result.dart';
-import 'package:flutter_devicelab/framework/utils.dart';
-
-import 'service_extensions_test.dart';
-
-Future<void> main() async {
-  deviceOperatingSystem = DeviceOperatingSystem.ios;
-  await task(() async {
-    final Device device = await devices.workingDevice;
-    await device.unlock();
-    final String deviceId = device.deviceId;
-
-    final Directory testDirectory =
-        dir('${flutterDirectory.path}/dev/integration_tests/ui');
-    await inDirectory<void>(testDirectory, () async {
-      await flutter('packages', options: <String>['get']);
-
-      await checkNoWarningHostLaunch(deviceId);
-      await checkNoWarningXcodeLaunch(deviceId);
-      await checkWarningHomeScreenLaunch(deviceId);
-    });
-    return TaskResult.success(<String, dynamic>{});
-  });
-}
-
-const String expectedWarning =
-    'Launching a debug-mode app from the home screen may cause problems.';
-
-// When a debug-mode app is launched from the host there should be no warnings.
-Future<void> checkNoWarningHostLaunch(String deviceId) async {
-  final String output = await evalFlutter('drive', options: <String>[
-    '--debug',
-    '--verbose',
-    '--verbose-system-logs',
-    '-d',
-    deviceId,
-    'lib/empty.dart',
-  ]);
-
-  expect(!output.contains(expectedWarning));
-}
-
-// When a debug-mode app is launched from Xcode there should be no warnings. The
-// Xcode launch is simulated by keeping LLDB attached throughout the lifetime of
-// the app.
-Future<void> checkNoWarningXcodeLaunch(String deviceId) async {
-  await flutter('build',
-      options: <String>['ios', '--debug', '--verbose', 'lib/exit.dart']);
-
-  final String output = await eval(
-      '${flutterDirectory.path}/bin/cache/artifacts/ios-deploy/ios-deploy',
-      <String>[
-        '--bundle',
-        'build/ios/iphoneos/Runner.app',
-        '-d', // Actually start the app in LLDB, don't just install it.
-        '--noninteractive',
-        '--args',
-        '--verbose-logging',
-      ]);
-
-  expect(output.contains('success') && !output.contains(expectedWarning));
-}
-
-// When a debug-mode app is launched from the home screen there should be a
-// warning every ~100 launches. We lower the threshold from to 1 via
-// "--verbose-system-logs" and simulate a home-screen-launch by setting an
-// environment variable. The environment variable forces "flutter drive" to not
-// pass a flag which it normally passes to debug-mode apps, imitating launchd,
-// which doesn't pass any command-line flags.
-Future<void> checkWarningHomeScreenLaunch(String deviceId) async {
-  final String output = await evalFlutter('drive', options: <String>[
-    '--debug',
-    '--verbose',
-    '--verbose-system-logs',
-    '-d',
-    deviceId,
-    'lib/empty.dart',
-  ], environment: <String, String>{
-    'FLUTTER_TOOLS_DEBUG_WITHOUT_CHECKED_MODE': 'true',
-  });
-  expect(output.contains(expectedWarning));
-}

dev/devicelab/bin/tasks/time_to_development_benchmark__android.dart

@@ -1,81 +0,0 @@
-// Copyright 2014 The Flutter Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file.
-
-import 'dart:io';
-
-import 'package:flutter_devicelab/framework/adb.dart';
-import 'package:flutter_devicelab/framework/framework.dart';
-import 'package:flutter_devicelab/framework/task_result.dart';
-import 'package:flutter_devicelab/framework/utils.dart';
-import 'package:path/path.dart' as path;
-
-Future<void> main() async {
-  deviceOperatingSystem = DeviceOperatingSystem.android;
-  await task(createTimeToDevelopmentCommand);
-}
-
-final Directory flutterGalleryDir =
-    dir(path.join(flutterDirectory.path, 'dev', 'integration_tests', 'flutter_gallery'));
-final Directory editedFlutterGalleryDir =
-    dir(path.join(Directory.systemTemp.path, 'edited_flutter_gallery'));
-
-Future<TaskResult> createTimeToDevelopmentCommand() async {
-  final Map<String, double> allResults = <String, double>{};
-  bool failed = false;
-  await inDirectory<void>(flutterDirectory, () async {
-    rmTree(editedFlutterGalleryDir);
-    mkdirs(editedFlutterGalleryDir);
-    recursiveCopy(flutterGalleryDir, editedFlutterGalleryDir);
-    await inDirectory<void>(editedFlutterGalleryDir, () async {
-      final Device device = await devices.workingDevice;
-      await device.unlock();
-      final Stopwatch stopwatch = Stopwatch()..start();
-      final Process initialBuild = await startProcess(
-        path.join(flutterDirectory.path, 'bin', 'flutter'),
-        <String>['run', '--debug', '-d', device.deviceId, '--no-resident',],
-      );
-      int exitCode = await initialBuild.exitCode;
-      if (exitCode != 0) {
-        failed = true;
-        return;
-      }
-      final int initialBuildMilliseconds = stopwatch.elapsedMilliseconds;
-      stopwatch
-        ..reset()
-        ..start();
-      // Update a source file.
-      final File appDartSource = file(path.join(
-        editedFlutterGalleryDir.path,
-        'lib/gallery/app.dart',
-      ));
-      appDartSource.writeAsStringSync(
-        appDartSource.readAsStringSync().replaceFirst(
-          "'Flutter Gallery'",
-          "'Updated Flutter Gallery'",
-        ),
-      );
-
-      final Process secondBuild = await startProcess(
-        path.join(flutterDirectory.path, 'bin', 'flutter'),
-        <String>['run', '--debug', '-d', device.deviceId, '--no-resident'],
-        environment: null,
-      );
-      exitCode = await secondBuild.exitCode;
-      if (exitCode != 0) {
-        failed = true;
-        return;
-      }
-      stopwatch.stop();
-      allResults['time_to_development'] = initialBuildMilliseconds.toDouble();
-      allResults['time_to_development_incremental'] =
-          stopwatch.elapsedMilliseconds.toDouble();
-    });
-  });
-  if (failed) {
-    return TaskResult.failure('Failed to build debug app');
-  }
-
-  return TaskResult.success(allResults,
-      benchmarkScoreKeys: allResults.keys.toList());
-}

dev/devicelab/lib/tasks/build_benchmarks.dart

@@ -1,84 +0,0 @@
-// Copyright 2014 The Flutter Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file.
-
-import 'dart:io';
-
-import 'package:flutter_devicelab/framework/framework.dart';
-import 'package:flutter_devicelab/framework/task_result.dart';
-import 'package:flutter_devicelab/framework/utils.dart';
-import 'package:path/path.dart' as path;
-
-final Directory helloWorldDir = dir(path.join(flutterDirectory.path, 'examples', 'hello_world'));
-
-/// Creates a devicelab build benchmark for Android.
-TaskFunction createAndroidBuildBenchmarkTask() {
-  return () async {
-    return createBuildCommand('apk');
-  };
-}
-
-/// Creates a devicelab build benchmark for iOS.
-TaskFunction createIosBuildBenchmarkTask() {
-  return () async {
-    return createBuildCommand('ios');
-  };
-}
-
-Future<TaskResult> createBuildCommand(String buildKind) {
-  return inDirectory<TaskResult>(helloWorldDir, () async {
-    final Stopwatch stopwatch = Stopwatch()
-      ..start();
-    final Process initialBuild = await startProcess(
-      path.join(flutterDirectory.path, 'bin', 'flutter'),
-      <String>['build', buildKind, '--debug'],
-      environment: null,
-    );
-    int exitCode = await initialBuild.exitCode;
-    if (exitCode != 0) {
-      return TaskResult.failure('Failed to build debug app');
-    }
-    final int initialBuildMilliseconds = stopwatch.elapsedMilliseconds;
-    stopwatch
-      ..reset()
-      ..start();
-    final Process secondBuild = await startProcess(
-      path.join(flutterDirectory.path, 'bin', 'flutter'),
-      <String>['build', buildKind, '--debug'],
-      environment: null,
-    );
-    exitCode = await secondBuild.exitCode;
-    if (exitCode != 0) {
-      return TaskResult.failure('Failed to build debug app');
-    }
-    final int secondBuildMilliseconds = stopwatch.elapsedMilliseconds;
-    final Process newBuildConfig = await startProcess(
-      path.join(flutterDirectory.path, 'bin', 'flutter'),
-      <String>['build', buildKind, '--profile'],
-      environment: null,
-    );
-    exitCode = await newBuildConfig.exitCode;
-    if (exitCode != 0) {
-      return TaskResult.failure('Failed to build profile app');
-    }
-    stopwatch
-      ..reset()
-      ..start();
-    final Process thirdBuild = await startProcess(
-      path.join(flutterDirectory.path, 'bin', 'flutter'),
-      <String>['build', buildKind, '--debug'],
-      environment: null,
-    );
-    exitCode = await thirdBuild.exitCode;
-    if (exitCode != 0) {
-      return TaskResult.failure('Failed to build debug app');
-    }
-    final int thirdBuildMilliseconds = stopwatch.elapsedMilliseconds;
-    stopwatch.stop();
-    final Map<String, double> allResults = <String, double>{};
-    allResults['first_build_debug_millis'] = initialBuildMilliseconds.toDouble();
-    allResults['second_build_debug_millis'] = secondBuildMilliseconds.toDouble();
-    allResults['after_config_change_build_debug_millis'] = thirdBuildMilliseconds.toDouble();
-    return TaskResult.success(allResults, benchmarkScoreKeys: allResults.keys.toList());
-  });
-}

dev/devicelab/lib/tasks/perf_tests.dart

@@ -218,13 +218,6 @@ TaskFunction createComplexLayoutStartupTest() {
   ).run;
 }
 
-TaskFunction createHelloWorldStartupTest() {
-  return StartupTest(
-    '${flutterDirectory.path}/examples/hello_world',
-    reportMetrics: false,
-  ).run;
-}
-
 TaskFunction createFlutterGalleryCompileTest() {
   return CompileTest('${flutterDirectory.path}/dev/integration_tests/flutter_gallery').run;
 }

dev/devicelab/lib/tasks/run_without_leak.dart

@@ -1,61 +0,0 @@
-// Copyright 2014 The Flutter Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file.
-
-import 'dart:async';
-import 'dart:convert';
-import 'dart:io';
-
-import 'package:path/path.dart' as path;
-
-import '../framework/adb.dart';
-import '../framework/framework.dart';
-import '../framework/task_result.dart';
-import '../framework/utils.dart';
-
-TaskFunction createRunWithoutLeakTest(dynamic dir) {
-  return () async {
-    final Device device = await devices.workingDevice;
-    await device.unlock();
-    final List<String> options = <String>[
-      '-d', device.deviceId, '--verbose',
-    ];
-    int exitCode;
-    await inDirectory<void>(dir, () async {
-      final Process process = await startProcess(
-          path.join(flutterDirectory.path, 'bin', 'flutter'),
-          flutterCommandArgs('run', options),
-          environment: null,
-      );
-      final Completer<void> stdoutDone = Completer<void>();
-      final Completer<void> stderrDone = Completer<void>();
-      process.stdout
-          .transform<String>(utf8.decoder)
-          .transform<String>(const LineSplitter())
-          .listen((String line) {
-        if (line.contains('] For a more detailed help message, press "h". To detach, press "d"; to quit, press "q"')) {
-          process.stdin.writeln('q');
-        }
-        print('stdout: $line');
-      }, onDone: () {
-        stdoutDone.complete();
-      });
-      process.stderr
-          .transform<String>(utf8.decoder)
-          .transform<String>(const LineSplitter())
-          .listen((String line) {
-        print('stderr: $line');
-      }, onDone: () {
-        stderrDone.complete();
-      });
-
-      await Future.wait<void>(
-          <Future<void>>[stdoutDone.future, stderrDone.future]);
-      exitCode = await process.exitCode;
-    });
-
-    return exitCode == 0
-        ? TaskResultCheckProcesses()
-        : TaskResult.failure('Failed to run $dir');
-  };
-}

dev/devicelab/manifest.yaml

@@ -389,12 +389,6 @@ tasks:
     stage: devicelab
     required_agent_capabilities: ["mac/android"]
 
-  # hello_world__start_up:
-  #   description: >
-  #     Verifies that Hello World can start on an array of devices.
-  #   stage: devicelab
-  #   required_agent_capabilities: ["linux/android_esoteric"]
-
   microbenchmarks:
     description: >
       Runs benchmarks from dev/benchmarks/microbenchmarks.
@@ -480,18 +474,6 @@ tasks:
     stage: devicelab
     required_agent_capabilities: ["linux/android"]
 
-  build_benchmark:
-    description: >
-      Measures APK build performance across config changes.
-    stage: devicelab
-    required_agent_capabilities: ["linux/android"]
-
-  time_to_development_benchmark__android:
-    description: >
-      Measures time from flutter run until resident runner setup is complete.
-    stage: devicelab
-    required_agent_capabilities: ["linux/android"]
-
   # iOS on-device tests
 
   ios_defines_test:
@@ -644,25 +626,12 @@ tasks:
     stage: devicelab_ios
     required_agent_capabilities: ["mac/ios"]
 
-  # TODO(fujino): does not pass on iOS13 https://github.com/flutter/flutter/issues/43029
-  # system_debug_ios:
-  #   description: >
-  #     Tests that the Engine correctly initializes the system debugger for debug-mode iOS apps.
-  #   stage: devicelab_ios
-  #   required_agent_capabilities: ["mac/ios"]
-
   macos_chrome_dev_mode:
     description: >
       Run flutter web on the devicelab and hot restart.
     stage: devicelab_ios
     required_agent_capabilities: ["mac/ios"]
 
-  build_benchmark_ios:
-    description: >
-      Measures iOS build performance across config changes.
-    stage: devicelab_ios
-    required_agent_capabilities: ["mac/ios"]
-
   simple_animation_perf_ios:
     description: >
       Measure CPU/GPU usage percentages of a simple animation.
@@ -702,14 +671,6 @@ tasks:
     stage: devicelab
     required_agent_capabilities: ["mac/ios"]
 
-#  TODO(jonahwilliams): investigate build failures on devicelab infra to re-enable.
-#  mac_enable_twc:
-#    description: >
-#      Verifies that track-widget-creation can be enabled and disabled.
-#    stage: devicelab_ios
-#    required_agent_capabilities: ["mac/ios"]
-#    flaky: true
-
   ios_app_with_extensions_test:
     description: >
       Checks that an iOS app with extensions can be built for physical and simulated devices.
@@ -742,12 +703,6 @@ tasks:
     stage: devicelab_win
     required_agent_capabilities: ["windows/android"]
 
-  # run_without_leak_win:
-  #   description: >
-  #     Checks that `flutter run` does not leak dart.exe on Windows.
-  #   stage: devicelab_win
-  #   required_agent_capabilities: ["windows/android"]
-
   # Tests running on Linux hosts
 
   hot_mode_dev_cycle_linux__benchmark:
@@ -939,20 +894,6 @@ tasks:
      stage: devicelab
      required_agent_capabilities: ["linux-vm"]
 
-  # run_without_leak_linux:
-  #   description: >
-  #     Checks that `flutter run` does not leak dart on Linux.
-  #   stage: devicelab
-  #   required_agent_capabilities: ["linux/android"]
-  #   flaky: true
-
-  # run_without_leak_mac:
-  #   description: >
-  #     Checks that `flutter run` does not leak dart on macOS.
-  #   stage: devicelab
-  #   required_agent_capabilities: ["mac/android"]
-  #   flaky: true
-
   # android_splash_screen_integration_test:
   #   description: >
   #     Runs end-to-end test of Flutter's Android splash behavior.