flutter/packages/flutter_tools/lib/src/base/process.dart

// Copyright 2015 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

import 'dart:async';

import '../convert.dart';
import '../globals.dart';
import 'common.dart';
import 'file_system.dart';
import 'io.dart';
import 'process_manager.dart';
import 'utils.dart';

typedef StringConverter = String Function(String string);

/// A function that will be run before the VM exits.
typedef ShutdownHook = Future<dynamic> Function();

// TODO(ianh): We have way too many ways to run subprocesses in this project.
// Convert most of these into one or more lightweight wrappers around the
// [ProcessManager] API using named parameters for the various options.
// See [here](https://github.com/flutter/flutter/pull/14535#discussion_r167041161)
// for more details.

/// The stage in which a [ShutdownHook] will be run. All shutdown hooks within
/// a given stage will be started in parallel and will be guaranteed to run to
/// completion before shutdown hooks in the next stage are started.
class ShutdownStage implements Comparable<ShutdownStage> {
  const ShutdownStage._(this.priority);

  /// The stage priority. Smaller values will be run before larger values.
  final int priority;

  /// The stage before the invocation recording (if one exists) is serialized
  /// to disk. Tasks performed during this stage *will* be recorded.
  static const ShutdownStage STILL_RECORDING = ShutdownStage._(1);

  /// The stage during which the invocation recording (if one exists) will be
  /// serialized to disk. Invocations performed after this stage will not be
  /// recorded.
  static const ShutdownStage SERIALIZE_RECORDING = ShutdownStage._(2);

  /// The stage during which a serialized recording will be refined (e.g.
  /// cleansed for tests, zipped up for bug reporting purposes, etc.).
  static const ShutdownStage POST_PROCESS_RECORDING = ShutdownStage._(3);

  /// The stage during which temporary files and directories will be deleted.
  static const ShutdownStage CLEANUP = ShutdownStage._(4);

  @override
  int compareTo(ShutdownStage other) => priority.compareTo(other.priority);
}

Map<ShutdownStage, List<ShutdownHook>> _shutdownHooks = <ShutdownStage, List<ShutdownHook>>{};
bool _shutdownHooksRunning = false;

/// Registers a [ShutdownHook] to be executed before the VM exits.
///
/// If [stage] is specified, the shutdown hook will be run during the specified
/// stage. By default, the shutdown hook will be run during the
/// [ShutdownStage.CLEANUP] stage.
void addShutdownHook(
  ShutdownHook shutdownHook, [
  ShutdownStage stage = ShutdownStage.CLEANUP,
]) {
  assert(!_shutdownHooksRunning);
  _shutdownHooks.putIfAbsent(stage, () => <ShutdownHook>[]).add(shutdownHook);
}

/// Runs all registered shutdown hooks and returns a future that completes when
/// all such hooks have finished.
///
/// Shutdown hooks will be run in groups by their [ShutdownStage]. All shutdown
/// hooks within a given stage will be started in parallel and will be
/// guaranteed to run to completion before shutdown hooks in the next stage are
/// started.
Future<void> runShutdownHooks() async {
  printTrace('Running shutdown hooks');
  _shutdownHooksRunning = true;
  try {
    for (ShutdownStage stage in _shutdownHooks.keys.toList()..sort()) {
      printTrace('Shutdown hook priority ${stage.priority}');
      final List<ShutdownHook> hooks = _shutdownHooks.remove(stage);
      final List<Future<dynamic>> futures = <Future<dynamic>>[];
      for (ShutdownHook shutdownHook in hooks)
        futures.add(shutdownHook());
      await Future.wait<dynamic>(futures);
    }
  } finally {
    _shutdownHooksRunning = false;
  }
  assert(_shutdownHooks.isEmpty);
  printTrace('Shutdown hooks complete');
}

Map<String, String> _environment(bool allowReentrantFlutter, [ Map<String, String> environment ]) {
  if (allowReentrantFlutter) {
    if (environment == null)
      environment = <String, String>{'FLUTTER_ALREADY_LOCKED': 'true'};
    else
      environment['FLUTTER_ALREADY_LOCKED'] = 'true';
  }

  return environment;
}

/// This runs the command in the background from the specified working
/// directory. Completes when the process has been started.
Future<Process> runCommand(
  List<String> cmd, {
  String workingDirectory,
  bool allowReentrantFlutter = false,
  Map<String, String> environment,
}) {
  _traceCommand(cmd, workingDirectory: workingDirectory);
  return processManager.start(
    cmd,
    workingDirectory: workingDirectory,
    environment: _environment(allowReentrantFlutter, environment),
  );
}

/// This runs the command and streams stdout/stderr from the child process to
/// this process' stdout/stderr. Completes with the process's exit code.
///
/// If [filter] is null, no lines are removed.
///
/// If [filter] is non-null, all lines that do not match it are removed. If
/// [mapFunction] is present, all lines that match [filter] are also forwarded
/// to [mapFunction] for further processing.
///
/// If [detachFilter] is non-null, the returned future will complete with exit code `0`
/// when the process outputs something matching [detachFilter] to stderr. The process will
/// continue in the background, and the final exit code will not be reported. [filter] is
/// not considered on lines matching [detachFilter].
Future<int> runCommandAndStreamOutput(
  List<String> cmd, {
  String workingDirectory,
  bool allowReentrantFlutter = false,
  String prefix = '',
  bool trace = false,
  RegExp filter,
  StringConverter mapFunction,
  Map<String, String> environment,
  RegExp detachFilter,
}) async {
  final Completer<int> result = Completer<int>();
  final Process process = await runCommand(
    cmd,
    workingDirectory: workingDirectory,
    allowReentrantFlutter: allowReentrantFlutter,
    environment: environment,
  );
  final StreamSubscription<String> stdoutSubscription = process.stdout
    .transform<String>(utf8.decoder)
    .transform<String>(const LineSplitter())
    .map((String line) {
      if (detachFilter != null && detachFilter.hasMatch(line) && !result.isCompleted) {
        // Detach from the process, assuming it will eventually complete successfully.
        // Output printed after detaching (incl. stdout and stderr) will still be
        // processed by [filter] and [mapFunction].
        result.complete(0);
      }
      return line;
    })
    .where((String line) => filter == null || filter.hasMatch(line))
    .listen((String line) {
      if (mapFunction != null)
        line = mapFunction(line);
      if (line != null) {
        final String message = '$prefix$line';
        if (trace)
          printTrace(message);
        else
          printStatus(message, wrap: false);
      }
    });
  final StreamSubscription<String> stderrSubscription = process.stderr
    .transform<String>(utf8.decoder)
    .transform<String>(const LineSplitter())
    .where((String line) => filter == null || filter.hasMatch(line))
    .listen((String line) {
      if (mapFunction != null)
        line = mapFunction(line);
      if (line != null)
        printError('$prefix$line', wrap: false);
    });

  // Wait for stdout to be fully processed before completing with the exit code (non-detached case),
  // because process.exitCode may complete first causing flaky tests. If the process detached,
  // we at least have a predictable output for stdout, although (unavoidably) not for stderr.
  Future<void> readOutput() async {
    await waitGroup<void>(<Future<void>>[
      stdoutSubscription.asFuture<void>(),
      stderrSubscription.asFuture<void>(),
    ]);

    await waitGroup<void>(<Future<void>>[
      stdoutSubscription.cancel(),
      stderrSubscription.cancel(),
    ]);

    // Complete the future if the we did not detach the process yet.
    if (!result.isCompleted) {
      result.complete(process.exitCode);
    }
  }

  unawaited(readOutput());
  return result.future;
}

/// Runs the [command] interactively, connecting the stdin/stdout/stderr
/// streams of this process to those of the child process. Completes with
/// the exit code of the child process.
Future<int> runInteractively(
  List<String> command, {
  String workingDirectory,
  bool allowReentrantFlutter = false,
  Map<String, String> environment,
}) async {
  final Process process = await runCommand(
    command,
    workingDirectory: workingDirectory,
    allowReentrantFlutter: allowReentrantFlutter,
    environment: environment,
  );
  // The real stdin will never finish streaming. Pipe until the child process
  // finishes.
  unawaited(process.stdin.addStream(stdin));
  // Wait for stdout and stderr to be fully processed, because process.exitCode
  // may complete first.
  await Future.wait<dynamic>(<Future<dynamic>>[
    stdout.addStream(process.stdout),
    stderr.addStream(process.stderr),
  ]);
  return await process.exitCode;
}

Future<Process> runDetached(List<String> cmd) {
  _traceCommand(cmd);
  final Future<Process> proc = processManager.start(
    cmd,
    mode: ProcessStartMode.detached,
  );
  return proc;
}

Future<RunResult> runAsync(
  List<String> cmd, {
  String workingDirectory,
  bool allowReentrantFlutter = false,
  Map<String, String> environment,
}) async {
  _traceCommand(cmd, workingDirectory: workingDirectory);
  final ProcessResult results = await processManager.run(
    cmd,
    workingDirectory: workingDirectory,
    environment: _environment(allowReentrantFlutter, environment),
  );
  final RunResult runResults = RunResult(results, cmd);
  printTrace(runResults.toString());
  return runResults;
}

typedef RunResultChecker = bool Function(int);

Future<RunResult> runCheckedAsync(
  List<String> cmd, {
  String workingDirectory,
  bool allowReentrantFlutter = false,
  Map<String, String> environment,
  RunResultChecker whiteListFailures,
}) async {
  final RunResult result = await runAsync(
    cmd,
    workingDirectory: workingDirectory,
    allowReentrantFlutter: allowReentrantFlutter,
    environment: environment,
  );
  if (result.exitCode != 0) {
    if (whiteListFailures == null || !whiteListFailures(result.exitCode)) {
      throw ProcessException(cmd[0], cmd.sublist(1),
          'Process "${cmd[0]}" exited abnormally:\n$result', result.exitCode);
    }
  }
  return result;
}

bool exitsHappy(
  List<String> cli, {
  Map<String, String> environment,
}) {
  _traceCommand(cli);
  try {
    return processManager.runSync(cli, environment: environment).exitCode == 0;
  } catch (error) {
    printTrace('$cli failed with $error');
    return false;
  }
}

Future<bool> exitsHappyAsync(
  List<String> cli, {
  Map<String, String> environment,
}) async {
  _traceCommand(cli);
  try {
    return (await processManager.run(cli, environment: environment)).exitCode == 0;
  } catch (error) {
    printTrace('$cli failed with $error');
    return false;
  }
}

/// Run cmd and return stdout.
///
/// Throws an error if cmd exits with a non-zero value.
String runCheckedSync(
  List<String> cmd, {
  String workingDirectory,
  bool allowReentrantFlutter = false,
  bool hideStdout = false,
  Map<String, String> environment,
  RunResultChecker whiteListFailures,
}) {
  return _runWithLoggingSync(
    cmd,
    workingDirectory: workingDirectory,
    allowReentrantFlutter: allowReentrantFlutter,
    hideStdout: hideStdout,
    checked: true,
    noisyErrors: true,
    environment: environment,
    whiteListFailures: whiteListFailures
  );
}

/// Run cmd and return stdout.
String runSync(
  List<String> cmd, {
  String workingDirectory,
  bool allowReentrantFlutter = false,
}) {
  return _runWithLoggingSync(
    cmd,
    workingDirectory: workingDirectory,
    allowReentrantFlutter: allowReentrantFlutter,
  );
}

void _traceCommand(List<String> args, { String workingDirectory }) {
  final String argsText = args.join(' ');
  if (workingDirectory == null) {
    printTrace('executing: $argsText');
  } else {
    printTrace('executing: [$workingDirectory${fs.path.separator}] $argsText');
  }
}

String _runWithLoggingSync(
  List<String> cmd, {
  bool checked = false,
  bool noisyErrors = false,
  bool throwStandardErrorOnError = false,
  String workingDirectory,
  bool allowReentrantFlutter = false,
  bool hideStdout = false,
  Map<String, String> environment,
  RunResultChecker whiteListFailures,
}) {
  _traceCommand(cmd, workingDirectory: workingDirectory);
  final ProcessResult results = processManager.runSync(
    cmd,
    workingDirectory: workingDirectory,
    environment: _environment(allowReentrantFlutter, environment),
  );

  printTrace('Exit code ${results.exitCode} from: ${cmd.join(' ')}');

  bool failedExitCode = results.exitCode != 0;
  if (whiteListFailures != null && failedExitCode) {
    failedExitCode = !whiteListFailures(results.exitCode);
  }

  if (results.stdout.isNotEmpty && !hideStdout) {
    if (failedExitCode && noisyErrors)
      printStatus(results.stdout.trim());
    else
      printTrace(results.stdout.trim());
  }

  if (failedExitCode) {
    if (results.stderr.isNotEmpty) {
      if (noisyErrors)
        printError(results.stderr.trim());
      else
        printTrace(results.stderr.trim());
    }

    if (throwStandardErrorOnError)
      throw results.stderr.trim();

    if (checked)
      throw 'Exit code ${results.exitCode} from: ${cmd.join(' ')}';
  }

  return results.stdout.trim();
}

class ProcessExit implements Exception {
  ProcessExit(this.exitCode, {this.immediate = false});

  final bool immediate;
  final int exitCode;

  String get message => 'ProcessExit: $exitCode';

  @override
  String toString() => message;
}

class RunResult {
  RunResult(this.processResult, this._command)
    : assert(_command != null),
      assert(_command.isNotEmpty);

  final ProcessResult processResult;

  final List<String> _command;

  int get exitCode => processResult.exitCode;
  String get stdout => processResult.stdout;
  String get stderr => processResult.stderr;

  @override
  String toString() {
    final StringBuffer out = StringBuffer();
    if (processResult.stdout.isNotEmpty)
      out.writeln(processResult.stdout);
    if (processResult.stderr.isNotEmpty)
      out.writeln(processResult.stderr);
    return out.toString().trimRight();
  }

  /// Throws a [ProcessException] with the given `message`.
  void throwException(String message) {
    throw ProcessException(
      _command.first,
      _command.skip(1).toList(),
      message,
      exitCode,
    );
  }
}