Technical Debt tracker (#7667)

dev/devicelab/bin/tasks/technical_debt__cost.dart

@@ -0,0 +1,101 @@
+// Copyright 2017 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+import 'dart:async';
+import 'dart:convert';
+import 'dart:io';
+
+import 'package:flutter_devicelab/framework/framework.dart';
+import 'package:flutter_devicelab/framework/utils.dart';
+import 'package:path/path.dart' as path;
+
+// the numbers below are odd, so that the totals don't seem round. :-)
+const double todoCost = 1009.0; // about two average SWE days, in dollars
+const double ignoreCost = 2003.0; // four average SWE days, in dollars
+const double pythonCost = 3001.0; //  six average SWE days, in dollars
+
+final RegExp todoPattern = new RegExp(r'(?://|#) *TODO');
+final RegExp ignorePattern = new RegExp(r'// *ignore:');
+
+Stream<double> findCostsForFile(File file) {
+  if (path.extension(file.path) == '.py')
+    return new Stream<double>.fromIterable(<double>[pythonCost]);
+  if (path.extension(file.path) != '.dart' &&
+      path.extension(file.path) != '.yaml' &&
+      path.extension(file.path) != '.sh')
+    return null;
+  StreamController<double> result = new StreamController<double>();
+  file.openRead().transform(UTF8.decoder).transform(const LineSplitter()).listen((String line) {
+    if (line.contains(todoPattern))
+      result.add(todoCost);
+    if (line.contains(ignorePattern))
+      result.add(ignoreCost);
+  }, onDone: () { result.close(); });
+  return result.stream;
+}
+
+Stream<double> findCostsForDirectory(Directory directory, Set<String> gitFiles) {
+  StreamController<double> result = new StreamController<double>();
+  Set<StreamSubscription<dynamic>> subscriptions = new Set<StreamSubscription<dynamic>>();
+
+  void checkDone(StreamSubscription<dynamic> subscription, String path) {
+    subscriptions.remove(subscription);
+    if (subscriptions.isEmpty)
+      result.close();
+  }
+
+  StreamSubscription<FileSystemEntity> listSubscription;
+  subscriptions.add(listSubscription = directory.list(followLinks: false).listen((FileSystemEntity entity) {
+    String name = path.relative(entity.path, from: flutterDirectory.path);
+    if (gitFiles.contains(name)) {
+      if (entity is File) {
+        StreamSubscription<double> subscription;
+        subscription = findCostsForFile(entity)?.listen((double cost) {
+          result.add(cost);
+        }, onDone: () { checkDone(subscription, name); });
+        if (subscription != null)
+          subscriptions.add(subscription);
+      } else if (entity is Directory) {
+        StreamSubscription<double> subscription;
+        subscription = findCostsForDirectory(entity, gitFiles)?.listen((double cost) {
+          result.add(cost);
+        }, onDone: () { checkDone(subscription, name); });
+        if (subscription != null)
+          subscriptions.add(subscription);
+      }
+    }
+  }, onDone: () { checkDone(listSubscription, directory.path); }));
+  return result.stream;
+}
+
+const String _kBenchmarkKey = 'technical_debt_in_dollars';
+
+Future<Null> main() async {
+  await task(() async {
+    Process git = await startProcess(
+      'git',
+      <String>['ls-files', '--full-name', flutterDirectory.path],
+      workingDirectory: flutterDirectory.path,
+    );
+    Set<String> gitFiles = new Set<String>();
+    await for (String entry in git.stdout.transform(UTF8.decoder).transform(const LineSplitter())) {
+      String subentry = '';
+      for (String component in path.split(entry)) {
+        if (subentry.isNotEmpty)
+          subentry += path.separator;
+        subentry += component;
+        gitFiles.add(subentry);
+      }
+    }
+    int gitExitCode = await git.exitCode;
+    if (gitExitCode != 0)
+      throw new Exception('git exit with unexpected error code $gitExitCode');
+    List<double> costs = await findCostsForDirectory(flutterDirectory, gitFiles).toList();
+    double total = costs.fold(0.0, (double total, double cost) => total + cost);
+    return new TaskResult.success(
+      <String, dynamic>{_kBenchmarkKey: total},
+      benchmarkScoreKeys: <String>[_kBenchmarkKey],
+    );
+  });
+}

dev/devicelab/lib/framework/adb.dart

@@ -246,13 +246,13 @@ class AndroidDevice implements Device {
   }
 
   /// Executes [command] on `adb shell` and returns its exit code.
-  Future<Null> shellExec(String command, List<String> arguments, {Map<String, String> env}) async {
-    await exec(adbPath, <String>['shell', command]..addAll(arguments), env: env, canFail: false);
+  Future<Null> shellExec(String command, List<String> arguments, { Map<String, String> environment }) async {
+    await exec(adbPath, <String>['shell', command]..addAll(arguments), environment: environment, canFail: false);
   }
 
   /// Executes [command] on `adb shell` and returns its standard output as a [String].
-  Future<String> shellEval(String command, List<String> arguments, {Map<String, String> env}) {
-    return eval(adbPath, <String>['shell', command]..addAll(arguments), env: env, canFail: false);
+  Future<String> shellEval(String command, List<String> arguments, { Map<String, String> environment }) {
+    return eval(adbPath, <String>['shell', command]..addAll(arguments), environment: environment, canFail: false);
   }
 
   @override

dev/devicelab/lib/framework/utils.dart

@@ -157,20 +157,28 @@ Future<DateTime> getFlutterRepoCommitTimestamp(String commit) {
   });
 }
 
-Future<Process> startProcess(String executable, List<String> arguments,
-    {Map<String, String> env}) async {
+Future<Process> startProcess(
+  String executable,
+  List<String> arguments, {
+  Map<String, String> environment,
+  String workingDirectory,
+}) async {
   String command = '$executable ${arguments?.join(" ") ?? ""}';
   print('Executing: $command');
-  Process proc = await Process.start(executable, arguments,
-      environment: env, workingDirectory: cwd);
-  ProcessInfo procInfo = new ProcessInfo(command, proc);
-  _runningProcesses.add(procInfo);
+  Process process = await Process.start(
+    executable,
+    arguments,
+    environment: environment,
+    workingDirectory: workingDirectory ?? cwd,
+  );
+  ProcessInfo processInfo = new ProcessInfo(command, process);
+  _runningProcesses.add(processInfo);
 
-  proc.exitCode.whenComplete(() {
-    _runningProcesses.remove(procInfo);
+  process.exitCode.whenComplete(() {
+    _runningProcesses.remove(processInfo);
   });
 
-  return proc;
+  return process;
 }
 
 Future<Null> forceQuitRunningProcesses() async {
@@ -191,20 +199,24 @@ Future<Null> forceQuitRunningProcesses() async {
 }
 
 /// Executes a command and returns its exit code.
-Future<int> exec(String executable, List<String> arguments,
-    {Map<String, String> env, bool canFail: false}) async {
-  Process proc = await startProcess(executable, arguments, env: env);
+Future<int> exec(
+  String executable,
+  List<String> arguments, {
+  Map<String, String> environment,
+  bool canFail: false,
+}) async {
+  Process process = await startProcess(executable, arguments, environment: environment);
 
-  proc.stdout
+  process.stdout
       .transform(UTF8.decoder)
       .transform(const LineSplitter())
       .listen(print);
-  proc.stderr
+  process.stderr
       .transform(UTF8.decoder)
       .transform(const LineSplitter())
       .listen(stderr.writeln);
 
-  int exitCode = await proc.exitCode;
+  int exitCode = await process.exitCode;
 
   if (exitCode != 0 && !canFail)
     fail('Executable failed with exit code $exitCode.');
@@ -215,14 +227,18 @@ Future<int> exec(String executable, List<String> arguments,
 /// Executes a command and returns its standard output as a String.
 ///
 /// Standard error is redirected to the current process' standard error stream.
-Future<String> eval(String executable, List<String> arguments,
-    {Map<String, String> env, bool canFail: false}) async {
-  Process proc = await startProcess(executable, arguments, env: env);
-  proc.stderr.listen((List<int> data) {
+Future<String> eval(
+  String executable,
+  List<String> arguments, {
+  Map<String, String> environment,
+  bool canFail: false,
+}) async {
+  Process process = await startProcess(executable, arguments, environment: environment);
+  process.stderr.listen((List<int> data) {
     stderr.add(data);
   });
-  String output = await UTF8.decodeStream(proc.stdout);
-  int exitCode = await proc.exitCode;
+  String output = await UTF8.decodeStream(process.stdout);
+  int exitCode = await process.exitCode;
 
   if (exitCode != 0 && !canFail)
     fail('Executable failed with exit code $exitCode.');
@@ -230,19 +246,25 @@ Future<String> eval(String executable, List<String> arguments,
   return output.trimRight();
 }
 
-Future<int> flutter(String command,
-    {List<String> options: const <String>[], bool canFail: false, Map<String, String> env}) {
+Future<int> flutter(String command, {
+  List<String> options: const <String>[],
+  bool canFail: false,
+  Map<String, String> environment,
+}) {
   List<String> args = <String>[command]..addAll(options);
   return exec(path.join(flutterDirectory.path, 'bin', 'flutter'), args,
-      canFail: canFail, env: env);
+      canFail: canFail, environment: environment);
 }
 
 /// Runs a `flutter` command and returns the standard output as a string.
-Future<String> evalFlutter(String command,
-    {List<String> options: const <String>[], bool canFail: false, Map<String, String> env}) {
+Future<String> evalFlutter(String command, {
+  List<String> options: const <String>[],
+  bool canFail: false,
+  Map<String, String> environment,
+}) {
   List<String> args = <String>[command]..addAll(options);
   return eval(path.join(flutterDirectory.path, 'bin', 'flutter'), args,
-      canFail: canFail, env: env);
+      canFail: canFail, environment: environment);
 }
 
 String get dartBin =>

dev/devicelab/lib/tasks/microbenchmarks.dart

@@ -47,9 +47,14 @@ TaskFunction createMicrobenchmarkTask() {
   };
 }
 
-Future<Process> _startFlutter({String command = 'run', List<String> options: const <String>[], bool canFail: false, Map<String, String> env}) {
+Future<Process> _startFlutter({
+  String command = 'run',
+  List<String> options: const <String>[],
+  bool canFail: false,
+  Map<String, String> environment,
+}) {
   List<String> args = <String>['run']..addAll(options);
-  return startProcess(path.join(flutterDirectory.path, 'bin', 'flutter'), args, env: env);
+  return startProcess(path.join(flutterDirectory.path, 'bin', 'flutter'), args, environment: environment);
 }
 
 Future<Map<String, double>> _readJsonResults(Process process) {

dev/devicelab/lib/tasks/perf_tests.dart

@@ -255,7 +255,7 @@ class MemoryTest {
           '-d',
           deviceId,
           '--use-existing-app',
-        ], env: <String, String> {
+        ], environment: <String, String> {
           'VM_SERVICE_URL': 'http://localhost:$debugPort'
         });
 

dev/devicelab/manifest.yaml

@@ -68,6 +68,11 @@ tasks:
     stage: devicelab
     required_agent_capabilities: ["has-android-device"]
 
+  technical_debt__cost:
+    description: >
+      Estimates our technical debt (TODOs, analyzer ignores, etc).
+    stage: devicelab
+    required_agent_capabilities: ["has-android-device"]
 
   # Android on-device tests
 

dev/devicelab/test/adb_test.dart

@@ -100,23 +100,29 @@ void expectLog(List<CommandArgs> log) {
   expect(FakeDevice.commandLog, log);
 }
 
-CommandArgs cmd({String command, List<String> arguments, Map<String, String> env}) => new CommandArgs(
-  command: command,
-  arguments: arguments,
-  env: env
-);
+CommandArgs cmd({
+  String command,
+  List<String> arguments,
+  Map<String, String> environment,
+}) {
+  return new CommandArgs(
+    command: command,
+    arguments: arguments,
+    environment: environment,
+  );
+}
 
 typedef dynamic ExitErrorFactory();
 
 class CommandArgs {
-  CommandArgs({this.command, this.arguments, this.env});
+  CommandArgs({ this.command, this.arguments, this.environment });
 
   final String command;
   final List<String> arguments;
-  final Map<String, String> env;
+  final Map<String, String> environment;
 
   @override
-  String toString() => 'CommandArgs(command: $command, arguments: $arguments, env: $env)';
+  String toString() => 'CommandArgs(command: $command, arguments: $arguments, environment: $environment)';
 
   @override
   bool operator==(Object other) {
@@ -126,18 +132,18 @@ class CommandArgs {
     CommandArgs otherCmd = other;
     return otherCmd.command == this.command &&
       const ListEquality<String>().equals(otherCmd.arguments, this.arguments) &&
-      const MapEquality<String, String>().equals(otherCmd.env, this.env);
+      const MapEquality<String, String>().equals(otherCmd.environment, this.environment);
   }
 
   @override
-  int get hashCode => 17 * (17 * command.hashCode + _hashArguments) + _hashEnv;
+  int get hashCode => 17 * (17 * command.hashCode + _hashArguments) + _hashEnvironment;
 
   int get _hashArguments => arguments != null
     ? const ListEquality<String>().hash(arguments)
     : null.hashCode;
 
-  int get _hashEnv => env != null
-    ? const MapEquality<String, String>().hash(env)
+  int get _hashEnvironment => environment != null
+    ? const MapEquality<String, String>().hash(environment)
     : null.hashCode;
 }
 
@@ -166,21 +172,21 @@ class FakeDevice extends AndroidDevice {
   }
 
   @override
-  Future<String> shellEval(String command, List<String> arguments, {Map<String, String> env}) async {
+  Future<String> shellEval(String command, List<String> arguments, { Map<String, String> environment }) async {
     commandLog.add(new CommandArgs(
       command: command,
       arguments: arguments,
-      env: env
+      environment: environment,
     ));
     return output;
   }
 
   @override
-  Future<Null> shellExec(String command, List<String> arguments, {Map<String, String> env}) async {
+  Future<Null> shellExec(String command, List<String> arguments, { Map<String, String> environment }) async {
     commandLog.add(new CommandArgs(
       command: command,
       arguments: arguments,
-      env: env
+      environment: environment,
     ));
     dynamic exitError = exitErrorFactory();
     if (exitError != null)