From 88710972daf3ca76b6c4a63834ca04d9738df0f6 Mon Sep 17 00:00:00 2001 From: Christopher Fujino Date: Thu, 30 Nov 2023 11:25:50 -0800 Subject: [PATCH] Refactor prepare_package.dart (#139277) I plan to extend the prepare_package.dart script to upload the flutter preview device ([design doc](https://docs.google.com/document/d/1AzI-_Uk2v1LA2kKKFJ7gVD4xcakXJ6yVZiS5Ek6RHtg/edit#heading=h.byp03plw7mg9)). However, given that that script is one large >1k line file, I decided to organize it into smaller libraries in this PR. There should be no behavioral change in this PR, this is a cleanup only. I made the following changes: 1. Created a //dev/bots/prepare_package/ directory to contain helper libraries 2. Moved everything but the `main()` function in //dev/bots/prepare_package.dart into one of 4 helper libraries under the new directory from step 1: a. archive_creator.dart which contains the code that creates archive directory locally on disk b. archive_publisher.dart which contains the code that uploads the archive to cloud storage c. common.dart for shared constants and definitions d. process_runner.dart for an abstraction over running sub-processes 3. Changed all definitions to `File` and `Directory` from `dart:io` to use the testable versions from `package:file`. This allowed me to use the `MemoryFileSystem` in the unit tests, rather than creating real temp file system directories. --- dev/bots/prepare_package.dart | 894 +----------------- dev/bots/prepare_package/archive_creator.dart | 534 +++++++++++ .../prepare_package/archive_publisher.dart | 229 +++++ dev/bots/prepare_package/common.dart | 46 + dev/bots/prepare_package/process_runner.dart | 116 +++ dev/bots/test/prepare_package_test.dart | 73 +- 6 files changed, 986 insertions(+), 906 deletions(-) create mode 100644 dev/bots/prepare_package/archive_creator.dart create mode 100644 dev/bots/prepare_package/archive_publisher.dart create mode 100644 dev/bots/prepare_package/common.dart create mode 100644 dev/bots/prepare_package/process_runner.dart diff --git a/dev/bots/prepare_package.dart b/dev/bots/prepare_package.dart index bf30ae38c4..3bd37b3d65 100644 --- a/dev/bots/prepare_package.dart +++ b/dev/bots/prepare_package.dart @@ -3,885 +3,17 @@ // found in the LICENSE file. import 'dart:async'; -import 'dart:convert'; -import 'dart:io' hide Platform; -import 'dart:typed_data'; +import 'dart:io' show exit, stderr; import 'package:args/args.dart'; -import 'package:convert/convert.dart'; -import 'package:crypto/crypto.dart'; -import 'package:crypto/src/digest_sink.dart'; -import 'package:http/http.dart' as http; -import 'package:path/path.dart' as path; -import 'package:platform/platform.dart' show LocalPlatform, Platform; -import 'package:pool/pool.dart'; -import 'package:process/process.dart'; +import 'package:file/file.dart'; +import 'package:file/local.dart'; -const String gobMirror = 'https://flutter.googlesource.com/mirrors/flutter'; -const String githubRepo = 'https://github.com/flutter/flutter.git'; -const String mingitForWindowsUrl = 'https://storage.googleapis.com/flutter_infra_release/mingit/' - '603511c649b00bbef0a6122a827ac419b656bc19/mingit.zip'; -const String releaseFolder = '/releases'; -const String gsBase = 'gs://flutter_infra_release'; -const String gsReleaseFolder = '$gsBase$releaseFolder'; -const String baseUrl = 'https://storage.googleapis.com/flutter_infra_release'; -const int shortCacheSeconds = 60; -const String frameworkVersionTag = 'frameworkVersionFromGit'; -const String dartVersionTag = 'dartSdkVersion'; -const String dartTargetArchTag = 'dartTargetArch'; +import 'prepare_package/archive_creator.dart'; +import 'prepare_package/archive_publisher.dart'; +import 'prepare_package/common.dart'; -/// Exception class for when a process fails to run, so we can catch -/// it and provide something more readable than a stack trace. -class PreparePackageException implements Exception { - PreparePackageException(this.message, [this.result]); - - final String message; - final ProcessResult? result; - int get exitCode => result?.exitCode ?? -1; - - @override - String toString() { - String output = runtimeType.toString(); - output += ': $message'; - final String stderr = result?.stderr as String? ?? ''; - if (stderr.isNotEmpty) { - output += ':\n$stderr'; - } - return output; - } -} - -enum Branch { - beta, - stable, - master, - main; -} - -/// A helper class for classes that want to run a process, optionally have the -/// stderr and stdout reported as the process runs, and capture the stdout -/// properly without dropping any. -class ProcessRunner { - ProcessRunner({ - ProcessManager? processManager, - this.subprocessOutput = true, - this.defaultWorkingDirectory, - this.platform = const LocalPlatform(), - }) : processManager = processManager ?? const LocalProcessManager() { - environment = Map.from(platform.environment); - } - - /// The platform to use for a starting environment. - final Platform platform; - - /// Set [subprocessOutput] to show output as processes run. Stdout from the - /// process will be printed to stdout, and stderr printed to stderr. - final bool subprocessOutput; - - /// Set the [processManager] in order to inject a test instance to perform - /// testing. - final ProcessManager processManager; - - /// Sets the default directory used when `workingDirectory` is not specified - /// to [runProcess]. - final Directory? defaultWorkingDirectory; - - /// The environment to run processes with. - late Map environment; - - /// Run the command and arguments in `commandLine` as a sub-process from - /// `workingDirectory` if set, or the [defaultWorkingDirectory] if not. Uses - /// [Directory.current] if [defaultWorkingDirectory] is not set. - /// - /// Set `failOk` if [runProcess] should not throw an exception when the - /// command completes with a non-zero exit code. - Future runProcess( - List commandLine, { - Directory? workingDirectory, - bool failOk = false, - }) async { - workingDirectory ??= defaultWorkingDirectory ?? Directory.current; - if (subprocessOutput) { - stderr.write('Running "${commandLine.join(' ')}" in ${workingDirectory.path}.\n'); - } - final List output = []; - final Completer stdoutComplete = Completer(); - final Completer stderrComplete = Completer(); - late Process process; - Future allComplete() async { - await stderrComplete.future; - await stdoutComplete.future; - return process.exitCode; - } - - try { - process = await processManager.start( - commandLine, - workingDirectory: workingDirectory.absolute.path, - environment: environment, - ); - process.stdout.listen( - (List event) { - output.addAll(event); - if (subprocessOutput) { - stdout.add(event); - } - }, - onDone: () async => stdoutComplete.complete(), - ); - if (subprocessOutput) { - process.stderr.listen( - (List event) { - stderr.add(event); - }, - onDone: () async => stderrComplete.complete(), - ); - } else { - stderrComplete.complete(); - } - } on ProcessException catch (e) { - final String message = 'Running "${commandLine.join(' ')}" in ${workingDirectory.path} ' - 'failed with:\n$e'; - throw PreparePackageException(message); - } on ArgumentError catch (e) { - final String message = 'Running "${commandLine.join(' ')}" in ${workingDirectory.path} ' - 'failed with:\n$e'; - throw PreparePackageException(message); - } - - final int exitCode = await allComplete(); - if (exitCode != 0 && !failOk) { - final String message = 'Running "${commandLine.join(' ')}" in ${workingDirectory.path} failed'; - throw PreparePackageException( - message, - ProcessResult(0, exitCode, null, 'returned $exitCode'), - ); - } - return utf8.decoder.convert(output).trim(); - } -} - -typedef HttpReader = Future Function(Uri url, {Map headers}); - -/// Creates a pre-populated Flutter archive from a git repo. -class ArchiveCreator { - /// [tempDir] is the directory to use for creating the archive. The script - /// will place several GiB of data there, so it should have available space. - /// - /// The processManager argument is used to inject a mock of [ProcessManager] for - /// testing purposes. - /// - /// If subprocessOutput is true, then output from processes invoked during - /// archive creation is echoed to stderr and stdout. - factory ArchiveCreator( - Directory tempDir, - Directory outputDir, - String revision, - Branch branch, { - bool strict = true, - ProcessManager? processManager, - bool subprocessOutput = true, - Platform platform = const LocalPlatform(), - HttpReader? httpReader, - }) { - final Directory flutterRoot = Directory(path.join(tempDir.path, 'flutter')); - final ProcessRunner processRunner = ProcessRunner( - processManager: processManager, - subprocessOutput: subprocessOutput, - platform: platform, - )..environment['PUB_CACHE'] = path.join( - tempDir.path, '.pub-cache', - ); - final String flutterExecutable = path.join( - flutterRoot.absolute.path, - 'bin', - 'flutter', - ); - final String dartExecutable = path.join( - flutterRoot.absolute.path, - 'bin', - 'cache', - 'dart-sdk', - 'bin', - 'dart', - ); - - return ArchiveCreator._( - tempDir: tempDir, - platform: platform, - flutterRoot: flutterRoot, - outputDir: outputDir, - revision: revision, - branch: branch, - strict: strict, - processRunner: processRunner, - httpReader: httpReader ?? http.readBytes, - flutterExecutable: flutterExecutable, - dartExecutable: dartExecutable, - ); - } - - ArchiveCreator._({ - required this.tempDir, - required this.platform, - required this.flutterRoot, - required this.outputDir, - required this.revision, - required this.branch, - required this.strict, - required ProcessRunner processRunner, - required this.httpReader, - required String flutterExecutable, - required String dartExecutable, - }) : - assert(revision.length == 40), - _processRunner = processRunner, - _flutter = flutterExecutable, - _dart = dartExecutable; - - /// The platform to use for the environment and determining which - /// platform we're running on. - final Platform platform; - - /// The branch to build the archive for. The branch must contain [revision]. - final Branch branch; - - /// The git revision hash to build the archive for. This revision has - /// to be available in the [branch], although it doesn't have to be - /// at HEAD, since we clone the branch and then reset to this revision - /// to create the archive. - final String revision; - - /// The flutter root directory in the [tempDir]. - final Directory flutterRoot; - - /// The temporary directory used to build the archive in. - final Directory tempDir; - - /// The directory to write the output file to. - final Directory outputDir; - - /// True if the creator should be strict about checking requirements or not. - /// - /// In strict mode, will insist that the [revision] be a tagged revision. - final bool strict; - - final Uri _minGitUri = Uri.parse(mingitForWindowsUrl); - final ProcessRunner _processRunner; - - /// Used to tell the [ArchiveCreator] which function to use for reading - /// bytes from a URL. Used in tests to inject a fake reader. Defaults to - /// [http.readBytes]. - final HttpReader httpReader; - - final Map _version = {}; - late String _flutter; - late String _dart; - - late final Future _dartArch = (() async { - // Parse 'arch' out of a string like '... "os_arch"\n'. - return (await _runDart(['--version'])) - .trim().split(' ').last.replaceAll('"', '').split('_')[1]; - })(); - - /// Returns a default archive name when given a Git revision. - /// Used when an output filename is not given. - Future get _archiveName async { - final String os = platform.operatingSystem.toLowerCase(); - // Include the intended host architecture in the file name for non-x64. - final String arch = await _dartArch == 'x64' ? '' : '${await _dartArch}_'; - // We don't use .tar.xz on Mac because although it can unpack them - // on the command line (with tar), the "Archive Utility" that runs - // when you double-click on them just does some crazy behavior (it - // converts it to a compressed cpio archive, and when you double - // click on that, it converts it back to .tar.xz, without ever - // unpacking it!) So, we use .zip for Mac, and the files are about - // 220MB larger than they need to be. :-( - final String suffix = platform.isLinux ? 'tar.xz' : 'zip'; - final String package = '${os}_$arch${_version[frameworkVersionTag]}'; - return 'flutter_$package-${branch.name}.$suffix'; - } - - /// Checks out the flutter repo and prepares it for other operations. - /// - /// Returns the version for this release as obtained from the git tags, and - /// the dart version as obtained from `flutter --version`. - Future> initializeRepo() async { - await _checkoutFlutter(); - if (_version.isEmpty) { - _version.addAll(await _getVersion()); - } - return _version; - } - - /// Performs all of the steps needed to create an archive. - Future createArchive() async { - assert(_version.isNotEmpty, 'Must run initializeRepo before createArchive'); - final File outputFile = File(path.join( - outputDir.absolute.path, - await _archiveName, - )); - await _installMinGitIfNeeded(); - await _populateCaches(); - await _validate(); - await _archiveFiles(outputFile); - return outputFile; - } - - /// Validates the integrity of the release package. - /// - /// Currently only checks that macOS binaries are codesigned. Will throw a - /// [PreparePackageException] if the test fails. - Future _validate() async { - // Only validate in strict mode, which means `--publish` - if (!strict || !platform.isMacOS) { - return; - } - // Validate that the dart binary is codesigned - try { - // TODO(fujino): Use the conductor https://github.com/flutter/flutter/issues/81701 - await _processRunner.runProcess( - [ - 'codesign', - '-vvvv', - '--check-notarization', - _dart, - ], - workingDirectory: flutterRoot, - ); - } on PreparePackageException catch (e) { - throw PreparePackageException( - 'The binary $_dart was not codesigned!\n${e.message}', - ); - } - } - - /// Returns the version map of this release, according the to tags in the - /// repo and the output of `flutter --version --machine`. - /// - /// This looks for the tag attached to [revision] and, if it doesn't find one, - /// git will give an error. - /// - /// If [strict] is true, the exact [revision] must be tagged to return the - /// version. If [strict] is not true, will look backwards in time starting at - /// [revision] to find the most recent version tag. - /// - /// The version found as a git tag is added to the information given by - /// `flutter --version --machine` with the `frameworkVersionFromGit` tag, and - /// returned. - Future> _getVersion() async { - String gitVersion; - if (strict) { - try { - gitVersion = await _runGit(['describe', '--tags', '--exact-match', revision]); - } on PreparePackageException catch (exception) { - throw PreparePackageException( - 'Git error when checking for a version tag attached to revision $revision.\n' - 'Perhaps there is no tag at that revision?:\n' - '$exception' - ); - } - } else { - gitVersion = await _runGit(['describe', '--tags', '--abbrev=0', revision]); - } - // Run flutter command twice, once to make sure the flutter command is built - // and ready (and thus won't output any junk on stdout the second time), and - // once to capture theJSON output. The second run should be fast. - await _runFlutter(['--version', '--machine']); - final String versionJson = await _runFlutter(['--version', '--machine']); - final Map versionMap = {}; - final Map result = json.decode(versionJson) as Map; - result.forEach((String key, dynamic value) => versionMap[key] = value.toString()); - versionMap[frameworkVersionTag] = gitVersion; - versionMap[dartTargetArchTag] = await _dartArch; - return versionMap; - } - - /// Clone the Flutter repo and make sure that the git environment is sane - /// for when the user will unpack it. - Future _checkoutFlutter() async { - // We want the user to start out the in the specified branch instead of a - // detached head. To do that, we need to make sure the branch points at the - // desired revision. - await _runGit(['clone', '-b', branch.name, gobMirror], workingDirectory: tempDir); - await _runGit(['reset', '--hard', revision]); - - // Make the origin point to github instead of the chromium mirror. - await _runGit(['remote', 'set-url', 'origin', githubRepo]); - - // Minify `.git` footprint (saving about ~100 MB as of Oct 2022) - await _runGit(['gc', '--prune=now', '--aggressive']); - } - - /// Retrieve the MinGit executable from storage and unpack it. - Future _installMinGitIfNeeded() async { - if (!platform.isWindows) { - return; - } - final Uint8List data = await httpReader(_minGitUri); - final File gitFile = File(path.join(tempDir.absolute.path, 'mingit.zip')); - await gitFile.writeAsBytes(data, flush: true); - - final Directory minGitPath = Directory(path.join(flutterRoot.absolute.path, 'bin', 'mingit')); - await minGitPath.create(recursive: true); - await _unzipArchive(gitFile, workingDirectory: minGitPath); - } - - /// Downloads an archive of every package that is present in the temporary - /// pub-cache from pub.dev. Stores the archives in - /// $flutterRoot/.pub-preload-cache. - /// - /// These archives will be installed in the user-level cache on first - /// following flutter command that accesses the cache. - /// - /// Precondition: all packages currently in the PUB_CACHE of [_processRunner] - /// are installed from pub.dev. - Future _downloadPubPackageArchives() async { - final Pool pool = Pool(10); // Number of simultaneous downloads. - final http.Client client = http.Client(); - final Directory preloadCache = Directory(path.join(flutterRoot.path, '.pub-preload-cache')); - preloadCache.createSync(recursive: true); - /// Fetch a single package. - Future fetchPackageArchive(String name, String version) async { - await pool.withResource(() async { - stderr.write('Fetching package archive for $name-$version.\n'); - int retries = 7; - while (true) { - retries-=1; - try { - final Uri packageListingUrl = Uri.parse('https://pub.dev/api/packages/$name'); - // Fetch the package listing to obtain the package download url. - final http.Response packageListingResponse = await client.get(packageListingUrl); - if (packageListingResponse.statusCode != 200) { - throw Exception('Downloading $packageListingUrl failed. Status code ${packageListingResponse.statusCode}.'); - } - final dynamic decodedPackageListing = json.decode(packageListingResponse.body); - if (decodedPackageListing is! Map) { - throw const FormatException('Package listing should be a map'); - } - final dynamic versions = decodedPackageListing['versions']; - if (versions is! List) { - throw const FormatException('.versions should be a list'); - } - final Map versionDescription = versions.firstWhere( - (dynamic description) { - if (description is! Map) { - throw const FormatException('.versions elements should be maps'); - } - return description['version'] == version; - }, - orElse: () => throw FormatException('Could not find $name-$version in package listing') - ) as Map; - final dynamic downloadUrl = versionDescription['archive_url']; - if (downloadUrl is! String) { - throw const FormatException('archive_url should be a string'); - } - final dynamic archiveSha256 = versionDescription['archive_sha256']; - if (archiveSha256 is! String) { - throw const FormatException('archive_sha256 should be a string'); - } - final http.Request request = http.Request('get', Uri.parse(downloadUrl)); - final http.StreamedResponse response = await client.send(request); - if (response.statusCode != 200) { - throw Exception('Downloading ${request.url} failed. Status code ${response.statusCode}.'); - } - final File archiveFile = File( - path.join(preloadCache.path, '$name-$version.tar.gz'), - ); - await response.stream.pipe(archiveFile.openWrite()); - final Stream> archiveStream = archiveFile.openRead(); - final Digest r = await sha256.bind(archiveStream).first; - if (hex.encode(r.bytes) != archiveSha256) { - throw Exception('Hash mismatch of downloaded archive'); - } - } on Exception catch (e) { - stderr.write('Failed downloading $name-$version. $e\n'); - if (retries > 0) { - stderr.write('Retrying download of $name-$version...'); - // Retry. - continue; - } else { - rethrow; - } - } - break; - } - }); - } - final Map cacheDescription = json.decode(await _runFlutter(['pub', 'cache', 'list'])) as Map; - final Map packages = cacheDescription['packages'] as Map; - final List> downloads = >[]; - for (final MapEntry package in packages.entries) { - final String name = package.key; - final Map versions = package.value as Map; - for (final String version in versions.keys) { - downloads.add(fetchPackageArchive(name, version)); - } - } - await Future.wait(downloads); - client.close(); - } - - /// Prepare the archive repo so that it has all of the caches warmed up and - /// is configured for the user to begin working. - Future _populateCaches() async { - await _runFlutter(['doctor']); - await _runFlutter(['update-packages']); - await _runFlutter(['precache']); - await _runFlutter(['ide-config']); - - // Create each of the templates, since they will call 'pub get' on - // themselves when created, and this will warm the cache with their - // dependencies too. - for (final String template in ['app', 'package', 'plugin']) { - final String createName = path.join(tempDir.path, 'create_$template'); - await _runFlutter( - ['create', '--template=$template', createName], - // Run it outside the cloned Flutter repo to not nest git repos, since - // they'll be git repos themselves too. - workingDirectory: tempDir, - ); - } - await _downloadPubPackageArchives(); - // Yes, we could just skip all .packages files when constructing - // the archive, but some are checked in, and we don't want to skip - // those. - await _runGit([ - 'clean', - '-f', - // Do not -X as it could lead to entire bin/cache getting cleaned - '-x', - '--', - '**/.packages', - ]); - /// Remove package_config files and any contents in .dart_tool - await _runGit([ - 'clean', - '-f', - '-x', - '--', - '**/.dart_tool/', - ]); - - // Ensure the above commands do not clean out the cache - final Directory flutterCache = Directory(path.join(flutterRoot.absolute.path, 'bin', 'cache')); - if (!flutterCache.existsSync()) { - throw Exception('The flutter cache was not found at ${flutterCache.path}!'); - } - - /// Remove git subfolder from .pub-cache, this contains the flutter goldens - /// and new flutter_gallery. - final Directory gitCache = Directory(path.join(flutterRoot.absolute.path, '.pub-cache', 'git')); - if (gitCache.existsSync()) { - gitCache.deleteSync(recursive: true); - } - } - - /// Write the archive to the given output file. - Future _archiveFiles(File outputFile) async { - if (outputFile.path.toLowerCase().endsWith('.zip')) { - await _createZipArchive(outputFile, flutterRoot); - } else if (outputFile.path.toLowerCase().endsWith('.tar.xz')) { - await _createTarArchive(outputFile, flutterRoot); - } - } - - Future _runDart(List args, {Directory? workingDirectory}) { - return _processRunner.runProcess( - [_dart, ...args], - workingDirectory: workingDirectory ?? flutterRoot, - ); - } - - Future _runFlutter(List args, {Directory? workingDirectory}) { - return _processRunner.runProcess( - [_flutter, ...args], - workingDirectory: workingDirectory ?? flutterRoot, - ); - } - - Future _runGit(List args, {Directory? workingDirectory}) { - return _processRunner.runProcess( - ['git', ...args], - workingDirectory: workingDirectory ?? flutterRoot, - ); - } - - /// Unpacks the given zip file into the currentDirectory (if set), or the - /// same directory as the archive. - Future _unzipArchive(File archive, {Directory? workingDirectory}) { - workingDirectory ??= Directory(path.dirname(archive.absolute.path)); - List commandLine; - if (platform.isWindows) { - commandLine = [ - '7za', - 'x', - archive.absolute.path, - ]; - } else { - commandLine = [ - 'unzip', - archive.absolute.path, - ]; - } - return _processRunner.runProcess(commandLine, workingDirectory: workingDirectory); - } - - /// Create a zip archive from the directory source. - Future _createZipArchive(File output, Directory source) async { - List commandLine; - if (platform.isWindows) { - // Unhide the .git folder, https://docs.microsoft.com/en-us/windows-server/administration/windows-commands/attrib. - await _processRunner.runProcess( - ['attrib', '-h', '.git'], - workingDirectory: Directory(source.absolute.path), - ); - commandLine = [ - '7za', - 'a', - '-tzip', - '-mx=9', - output.absolute.path, - path.basename(source.path), - ]; - } else { - commandLine = [ - 'zip', - '-r', - '-9', - '--symlinks', - output.absolute.path, - path.basename(source.path), - ]; - } - return _processRunner.runProcess( - commandLine, - workingDirectory: Directory(path.dirname(source.absolute.path)), - ); - } - - /// Create a tar archive from the directory source. - Future _createTarArchive(File output, Directory source) { - return _processRunner.runProcess([ - 'tar', - 'cJf', - output.absolute.path, - path.basename(source.absolute.path), - ], workingDirectory: Directory(path.dirname(source.absolute.path))); - } -} - -class ArchivePublisher { - ArchivePublisher( - this.tempDir, - this.revision, - this.branch, - this.version, - this.outputFile, - this.dryRun, { - ProcessManager? processManager, - bool subprocessOutput = true, - this.platform = const LocalPlatform(), - }) : assert(revision.length == 40), - platformName = platform.operatingSystem.toLowerCase(), - metadataGsPath = '$gsReleaseFolder/${getMetadataFilename(platform)}', - _processRunner = ProcessRunner( - processManager: processManager, - subprocessOutput: subprocessOutput, - ); - - final Platform platform; - final String platformName; - final String metadataGsPath; - final Branch branch; - final String revision; - final Map version; - final Directory tempDir; - final File outputFile; - final ProcessRunner _processRunner; - final bool dryRun; - String get destinationArchivePath => '${branch.name}/$platformName/${path.basename(outputFile.path)}'; - static String getMetadataFilename(Platform platform) => 'releases_${platform.operatingSystem.toLowerCase()}.json'; - - Future _getChecksum(File archiveFile) async { - final DigestSink digestSink = DigestSink(); - final ByteConversionSink sink = sha256.startChunkedConversion(digestSink); - - final Stream> stream = archiveFile.openRead(); - await stream.forEach((List chunk) { - sink.add(chunk); - }); - sink.close(); - return digestSink.value.toString(); - } - - /// Publish the archive to Google Storage. - /// - /// This method will throw if the target archive already exists on cloud - /// storage. - Future publishArchive([bool forceUpload = false]) async { - final String destGsPath = '$gsReleaseFolder/$destinationArchivePath'; - if (!forceUpload) { - if (await _cloudPathExists(destGsPath) && !dryRun) { - throw PreparePackageException( - 'File $destGsPath already exists on cloud storage!', - ); - } - } - await _cloudCopy( - src: outputFile.absolute.path, - dest: destGsPath, - ); - assert(tempDir.existsSync()); - final String gcsPath = '$gsReleaseFolder/${getMetadataFilename(platform)}'; - await _publishMetadata(gcsPath); - } - - /// Downloads and updates the metadata file without publishing it. - Future generateLocalMetadata() async { - await _updateMetadata('$gsReleaseFolder/${getMetadataFilename(platform)}'); - } - - Future> _addRelease(Map jsonData) async { - jsonData['base_url'] = '$baseUrl$releaseFolder'; - if (!jsonData.containsKey('current_release')) { - jsonData['current_release'] = {}; - } - (jsonData['current_release'] as Map)[branch.name] = revision; - if (!jsonData.containsKey('releases')) { - jsonData['releases'] = >[]; - } - - final Map newEntry = {}; - newEntry['hash'] = revision; - newEntry['channel'] = branch.name; - newEntry['version'] = version[frameworkVersionTag]; - newEntry['dart_sdk_version'] = version[dartVersionTag]; - newEntry['dart_sdk_arch'] = version[dartTargetArchTag]; - newEntry['release_date'] = DateTime.now().toUtc().toIso8601String(); - newEntry['archive'] = destinationArchivePath; - newEntry['sha256'] = await _getChecksum(outputFile); - - // Search for any entries with the same hash and channel and remove them. - final List releases = jsonData['releases'] as List; - jsonData['releases'] = >[ - for (final Map entry in releases.cast>()) - if (entry['hash'] != newEntry['hash'] || - entry['channel'] != newEntry['channel'] || - entry['dart_sdk_arch'] != newEntry['dart_sdk_arch']) - entry, - newEntry, - ]..sort((Map a, Map b) { - final DateTime aDate = DateTime.parse(a['release_date'] as String); - final DateTime bDate = DateTime.parse(b['release_date'] as String); - return bDate.compareTo(aDate); - }); - return jsonData; - } - - Future _updateMetadata(String gsPath) async { - // We can't just cat the metadata from the server with 'gsutil cat', because - // Windows wants to echo the commands that execute in gsutil.bat to the - // stdout when we do that. So, we copy the file locally and then read it - // back in. - final File metadataFile = File( - path.join(tempDir.absolute.path, getMetadataFilename(platform)), - ); - await _runGsUtil(['cp', gsPath, metadataFile.absolute.path]); - Map jsonData = {}; - if (!dryRun) { - final String currentMetadata = metadataFile.readAsStringSync(); - if (currentMetadata.isEmpty) { - throw PreparePackageException('Empty metadata received from server'); - } - try { - jsonData = json.decode(currentMetadata) as Map; - } on FormatException catch (e) { - throw PreparePackageException('Unable to parse JSON metadata received from cloud: $e'); - } - } - // Run _addRelease, even on a dry run, so we can inspect the metadata on a - // dry run. On a dry run, the only thing in the metadata file be the new - // release. - jsonData = await _addRelease(jsonData); - - const JsonEncoder encoder = JsonEncoder.withIndent(' '); - metadataFile.writeAsStringSync(encoder.convert(jsonData)); - } - - /// Publishes the metadata file to GCS. - Future _publishMetadata(String gsPath) async { - final File metadataFile = File( - path.join(tempDir.absolute.path, getMetadataFilename(platform)), - ); - await _cloudCopy( - src: metadataFile.absolute.path, - dest: gsPath, - // This metadata file is used by the website, so we don't want a long - // latency between publishing a release and it being available on the - // site. - cacheSeconds: shortCacheSeconds, - ); - } - - Future _runGsUtil( - List args, { - Directory? workingDirectory, - bool failOk = false, - }) async { - if (dryRun) { - print('gsutil.py -- $args'); - return ''; - } - return _processRunner.runProcess( - ['python3', path.join(platform.environment['DEPOT_TOOLS']!, 'gsutil.py'), '--', ...args], - workingDirectory: workingDirectory, - failOk: failOk, - ); - } - - /// Determine if a file exists at a given [cloudPath]. - Future _cloudPathExists(String cloudPath) async { - try { - await _runGsUtil( - ['stat', cloudPath], - ); - } on PreparePackageException { - // `gsutil stat gs://path/to/file` will exit with 1 if file does not exist - return false; - } - return true; - } - - Future _cloudCopy({ - required String src, - required String dest, - int? cacheSeconds, - }) async { - // We often don't have permission to overwrite, but - // we have permission to remove, so that's what we do. - await _runGsUtil(['rm', dest], failOk: true); - String? mimeType; - if (dest.endsWith('.tar.xz')) { - mimeType = 'application/x-gtar'; - } - if (dest.endsWith('.zip')) { - mimeType = 'application/zip'; - } - if (dest.endsWith('.json')) { - mimeType = 'application/json'; - } - return _runGsUtil([ - // Use our preferred MIME type for the files we care about - // and let gsutil figure it out for anything else. - if (mimeType != null) ...['-h', 'Content-Type:$mimeType'], - if (cacheSeconds != null) ...['-h', 'Cache-Control:max-age=$cacheSeconds'], - 'cp', - src, - dest, - ]); - } -} +const FileSystem fs = LocalFileSystem(); /// Prepares a flutter git repo to be packaged up for distribution. It mainly /// serves to populate the .pub-preload-cache with any appropriate Dart @@ -963,23 +95,23 @@ Future main(List rawArguments) async { } final String? tempDirArg = parsedArguments['temp_dir'] as String?; - Directory tempDir; + final Directory tempDir; bool removeTempDir = false; if (tempDirArg == null || tempDirArg.isEmpty) { - tempDir = Directory.systemTemp.createTempSync('flutter_package.'); + tempDir = fs.systemTempDirectory.createTempSync('flutter_package.'); removeTempDir = true; } else { - tempDir = Directory(tempDirArg); + tempDir = fs.directory(tempDirArg); if (!tempDir.existsSync()) { errorExit("Temporary directory $tempDirArg doesn't exist."); } } - Directory outputDir; + final Directory outputDir; if (parsedArguments['output'] == null) { outputDir = tempDir; } else { - outputDir = Directory(parsedArguments['output'] as String); + outputDir = fs.directory(parsedArguments['output'] as String); if (!outputDir.existsSync()) { outputDir.createSync(recursive: true); } @@ -993,6 +125,7 @@ Future main(List rawArguments) async { outputDir, revision, branch, + fs: fs, strict: publish && !dryRun, ); int exitCode = 0; @@ -1007,6 +140,7 @@ Future main(List rawArguments) async { version, outputFile, dryRun, + fs: fs, ); await publisher.generateLocalMetadata(); if (parsedArguments['publish'] as bool) { diff --git a/dev/bots/prepare_package/archive_creator.dart b/dev/bots/prepare_package/archive_creator.dart new file mode 100644 index 0000000000..2415e366fc --- /dev/null +++ b/dev/bots/prepare_package/archive_creator.dart @@ -0,0 +1,534 @@ +// Copyright 2014 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'dart:convert'; +import 'dart:io' show stderr; +import 'dart:typed_data'; + +import 'package:convert/convert.dart'; +import 'package:crypto/crypto.dart'; +import 'package:file/file.dart'; +import 'package:http/http.dart' as http; +import 'package:path/path.dart' as path; +import 'package:platform/platform.dart' show LocalPlatform, Platform; +import 'package:pool/pool.dart'; +import 'package:process/process.dart'; + +import 'common.dart'; +import 'process_runner.dart'; + +typedef HttpReader = Future Function(Uri url, {Map headers}); + +/// Creates a pre-populated Flutter archive from a git repo. +class ArchiveCreator { + /// [tempDir] is the directory to use for creating the archive. The script + /// will place several GiB of data there, so it should have available space. + /// + /// The processManager argument is used to inject a mock of [ProcessManager] for + /// testing purposes. + /// + /// If subprocessOutput is true, then output from processes invoked during + /// archive creation is echoed to stderr and stdout. + factory ArchiveCreator( + Directory tempDir, + Directory outputDir, + String revision, + Branch branch, { + required FileSystem fs, + HttpReader? httpReader, + Platform platform = const LocalPlatform(), + ProcessManager? processManager, + bool strict = true, + bool subprocessOutput = true, + }) { + final Directory flutterRoot = fs.directory(path.join(tempDir.path, 'flutter')); + final ProcessRunner processRunner = ProcessRunner( + processManager: processManager, + subprocessOutput: subprocessOutput, + platform: platform, + )..environment['PUB_CACHE'] = path.join( + tempDir.path, '.pub-cache', + ); + final String flutterExecutable = path.join( + flutterRoot.absolute.path, + 'bin', + 'flutter', + ); + final String dartExecutable = path.join( + flutterRoot.absolute.path, + 'bin', + 'cache', + 'dart-sdk', + 'bin', + 'dart', + ); + + return ArchiveCreator._( + tempDir: tempDir, + platform: platform, + flutterRoot: flutterRoot, + fs: fs, + outputDir: outputDir, + revision: revision, + branch: branch, + strict: strict, + processRunner: processRunner, + httpReader: httpReader ?? http.readBytes, + flutterExecutable: flutterExecutable, + dartExecutable: dartExecutable, + ); + } + + ArchiveCreator._({ + required this.branch, + required String dartExecutable, + required this.fs, + required String flutterExecutable, + required this.flutterRoot, + required this.httpReader, + required this.outputDir, + required this.platform, + required ProcessRunner processRunner, + required this.revision, + required this.strict, + required this.tempDir, + }) : + assert(revision.length == 40), + _processRunner = processRunner, + _flutter = flutterExecutable, + _dart = dartExecutable; + + /// The platform to use for the environment and determining which + /// platform we're running on. + final Platform platform; + + /// The branch to build the archive for. The branch must contain [revision]. + final Branch branch; + + /// The git revision hash to build the archive for. This revision has + /// to be available in the [branch], although it doesn't have to be + /// at HEAD, since we clone the branch and then reset to this revision + /// to create the archive. + final String revision; + + /// The flutter root directory in the [tempDir]. + final Directory flutterRoot; + + /// The temporary directory used to build the archive in. + final Directory tempDir; + + /// The directory to write the output file to. + final Directory outputDir; + + final FileSystem fs; + + /// True if the creator should be strict about checking requirements or not. + /// + /// In strict mode, will insist that the [revision] be a tagged revision. + final bool strict; + + final Uri _minGitUri = Uri.parse(mingitForWindowsUrl); + final ProcessRunner _processRunner; + + /// Used to tell the [ArchiveCreator] which function to use for reading + /// bytes from a URL. Used in tests to inject a fake reader. Defaults to + /// [http.readBytes]. + final HttpReader httpReader; + + final Map _version = {}; + late String _flutter; + late String _dart; + + late final Future _dartArch = (() async { + // Parse 'arch' out of a string like '... "os_arch"\n'. + return (await _runDart(['--version'])) + .trim().split(' ').last.replaceAll('"', '').split('_')[1]; + })(); + + /// Returns a default archive name when given a Git revision. + /// Used when an output filename is not given. + Future get _archiveName async { + final String os = platform.operatingSystem.toLowerCase(); + // Include the intended host architecture in the file name for non-x64. + final String arch = await _dartArch == 'x64' ? '' : '${await _dartArch}_'; + // We don't use .tar.xz on Mac because although it can unpack them + // on the command line (with tar), the "Archive Utility" that runs + // when you double-click on them just does some crazy behavior (it + // converts it to a compressed cpio archive, and when you double + // click on that, it converts it back to .tar.xz, without ever + // unpacking it!) So, we use .zip for Mac, and the files are about + // 220MB larger than they need to be. :-( + final String suffix = platform.isLinux ? 'tar.xz' : 'zip'; + final String package = '${os}_$arch${_version[frameworkVersionTag]}'; + return 'flutter_$package-${branch.name}.$suffix'; + } + + /// Checks out the flutter repo and prepares it for other operations. + /// + /// Returns the version for this release as obtained from the git tags, and + /// the dart version as obtained from `flutter --version`. + Future> initializeRepo() async { + await _checkoutFlutter(); + if (_version.isEmpty) { + _version.addAll(await _getVersion()); + } + return _version; + } + + /// Performs all of the steps needed to create an archive. + Future createArchive() async { + assert(_version.isNotEmpty, 'Must run initializeRepo before createArchive'); + final File outputFile = fs.file(path.join( + outputDir.absolute.path, + await _archiveName, + )); + await _installMinGitIfNeeded(); + await _populateCaches(); + await _validate(); + await _archiveFiles(outputFile); + return outputFile; + } + + /// Validates the integrity of the release package. + /// + /// Currently only checks that macOS binaries are codesigned. Will throw a + /// [PreparePackageException] if the test fails. + Future _validate() async { + // Only validate in strict mode, which means `--publish` + if (!strict || !platform.isMacOS) { + return; + } + // Validate that the dart binary is codesigned + try { + // TODO(fujino): Use the conductor https://github.com/flutter/flutter/issues/81701 + await _processRunner.runProcess( + [ + 'codesign', + '-vvvv', + '--check-notarization', + _dart, + ], + workingDirectory: flutterRoot, + ); + } on PreparePackageException catch (e) { + throw PreparePackageException( + 'The binary $_dart was not codesigned!\n${e.message}', + ); + } + } + + /// Returns the version map of this release, according the to tags in the + /// repo and the output of `flutter --version --machine`. + /// + /// This looks for the tag attached to [revision] and, if it doesn't find one, + /// git will give an error. + /// + /// If [strict] is true, the exact [revision] must be tagged to return the + /// version. If [strict] is not true, will look backwards in time starting at + /// [revision] to find the most recent version tag. + /// + /// The version found as a git tag is added to the information given by + /// `flutter --version --machine` with the `frameworkVersionFromGit` tag, and + /// returned. + Future> _getVersion() async { + String gitVersion; + if (strict) { + try { + gitVersion = await _runGit(['describe', '--tags', '--exact-match', revision]); + } on PreparePackageException catch (exception) { + throw PreparePackageException( + 'Git error when checking for a version tag attached to revision $revision.\n' + 'Perhaps there is no tag at that revision?:\n' + '$exception' + ); + } + } else { + gitVersion = await _runGit(['describe', '--tags', '--abbrev=0', revision]); + } + // Run flutter command twice, once to make sure the flutter command is built + // and ready (and thus won't output any junk on stdout the second time), and + // once to capture theJSON output. The second run should be fast. + await _runFlutter(['--version', '--machine']); + final String versionJson = await _runFlutter(['--version', '--machine']); + final Map versionMap = {}; + final Map result = json.decode(versionJson) as Map; + result.forEach((String key, dynamic value) => versionMap[key] = value.toString()); + versionMap[frameworkVersionTag] = gitVersion; + versionMap[dartTargetArchTag] = await _dartArch; + return versionMap; + } + + /// Clone the Flutter repo and make sure that the git environment is sane + /// for when the user will unpack it. + Future _checkoutFlutter() async { + // We want the user to start out the in the specified branch instead of a + // detached head. To do that, we need to make sure the branch points at the + // desired revision. + await _runGit(['clone', '-b', branch.name, gobMirror], workingDirectory: tempDir); + await _runGit(['reset', '--hard', revision]); + + // Make the origin point to github instead of the chromium mirror. + await _runGit(['remote', 'set-url', 'origin', githubRepo]); + + // Minify `.git` footprint (saving about ~100 MB as of Oct 2022) + await _runGit(['gc', '--prune=now', '--aggressive']); + } + + /// Retrieve the MinGit executable from storage and unpack it. + Future _installMinGitIfNeeded() async { + if (!platform.isWindows) { + return; + } + final Uint8List data = await httpReader(_minGitUri); + final File gitFile = fs.file(path.join(tempDir.absolute.path, 'mingit.zip')); + await gitFile.writeAsBytes(data, flush: true); + + final Directory minGitPath = fs.directory(path.join(flutterRoot.absolute.path, 'bin', 'mingit')); + await minGitPath.create(recursive: true); + await _unzipArchive(gitFile, workingDirectory: minGitPath); + } + + /// Downloads an archive of every package that is present in the temporary + /// pub-cache from pub.dev. Stores the archives in + /// $flutterRoot/.pub-preload-cache. + /// + /// These archives will be installed in the user-level cache on first + /// following flutter command that accesses the cache. + /// + /// Precondition: all packages currently in the PUB_CACHE of [_processRunner] + /// are installed from pub.dev. + Future _downloadPubPackageArchives() async { + final Pool pool = Pool(10); // Number of simultaneous downloads. + final http.Client client = http.Client(); + final Directory preloadCache = fs.directory(path.join(flutterRoot.path, '.pub-preload-cache')); + preloadCache.createSync(recursive: true); + /// Fetch a single package. + Future fetchPackageArchive(String name, String version) async { + await pool.withResource(() async { + stderr.write('Fetching package archive for $name-$version.\n'); + int retries = 7; + while (true) { + retries-=1; + try { + final Uri packageListingUrl = Uri.parse('https://pub.dev/api/packages/$name'); + // Fetch the package listing to obtain the package download url. + final http.Response packageListingResponse = await client.get(packageListingUrl); + if (packageListingResponse.statusCode != 200) { + throw Exception('Downloading $packageListingUrl failed. Status code ${packageListingResponse.statusCode}.'); + } + final dynamic decodedPackageListing = json.decode(packageListingResponse.body); + if (decodedPackageListing is! Map) { + throw const FormatException('Package listing should be a map'); + } + final dynamic versions = decodedPackageListing['versions']; + if (versions is! List) { + throw const FormatException('.versions should be a list'); + } + final Map versionDescription = versions.firstWhere( + (dynamic description) { + if (description is! Map) { + throw const FormatException('.versions elements should be maps'); + } + return description['version'] == version; + }, + orElse: () => throw FormatException('Could not find $name-$version in package listing') + ) as Map; + final dynamic downloadUrl = versionDescription['archive_url']; + if (downloadUrl is! String) { + throw const FormatException('archive_url should be a string'); + } + final dynamic archiveSha256 = versionDescription['archive_sha256']; + if (archiveSha256 is! String) { + throw const FormatException('archive_sha256 should be a string'); + } + final http.Request request = http.Request('get', Uri.parse(downloadUrl)); + final http.StreamedResponse response = await client.send(request); + if (response.statusCode != 200) { + throw Exception('Downloading ${request.url} failed. Status code ${response.statusCode}.'); + } + final File archiveFile = fs.file( + path.join(preloadCache.path, '$name-$version.tar.gz'), + ); + await response.stream.pipe(archiveFile.openWrite()); + final Stream> archiveStream = archiveFile.openRead(); + final Digest r = await sha256.bind(archiveStream).first; + if (hex.encode(r.bytes) != archiveSha256) { + throw Exception('Hash mismatch of downloaded archive'); + } + } on Exception catch (e) { + stderr.write('Failed downloading $name-$version. $e\n'); + if (retries > 0) { + stderr.write('Retrying download of $name-$version...'); + // Retry. + continue; + } else { + rethrow; + } + } + break; + } + }); + } + final Map cacheDescription = json.decode(await _runFlutter(['pub', 'cache', 'list'])) as Map; + final Map packages = cacheDescription['packages'] as Map; + final List> downloads = >[]; + for (final MapEntry package in packages.entries) { + final String name = package.key; + final Map versions = package.value as Map; + for (final String version in versions.keys) { + downloads.add(fetchPackageArchive(name, version)); + } + } + await Future.wait(downloads); + client.close(); + } + + /// Prepare the archive repo so that it has all of the caches warmed up and + /// is configured for the user to begin working. + Future _populateCaches() async { + await _runFlutter(['doctor']); + await _runFlutter(['update-packages']); + await _runFlutter(['precache']); + await _runFlutter(['ide-config']); + + // Create each of the templates, since they will call 'pub get' on + // themselves when created, and this will warm the cache with their + // dependencies too. + for (final String template in ['app', 'package', 'plugin']) { + final String createName = path.join(tempDir.path, 'create_$template'); + await _runFlutter( + ['create', '--template=$template', createName], + // Run it outside the cloned Flutter repo to not nest git repos, since + // they'll be git repos themselves too. + workingDirectory: tempDir, + ); + } + await _downloadPubPackageArchives(); + // Yes, we could just skip all .packages files when constructing + // the archive, but some are checked in, and we don't want to skip + // those. + await _runGit([ + 'clean', + '-f', + // Do not -X as it could lead to entire bin/cache getting cleaned + '-x', + '--', + '**/.packages', + ]); + /// Remove package_config files and any contents in .dart_tool + await _runGit([ + 'clean', + '-f', + '-x', + '--', + '**/.dart_tool/', + ]); + + // Ensure the above commands do not clean out the cache + final Directory flutterCache = fs.directory(path.join(flutterRoot.absolute.path, 'bin', 'cache')); + if (!flutterCache.existsSync()) { + throw Exception('The flutter cache was not found at ${flutterCache.path}!'); + } + + /// Remove git subfolder from .pub-cache, this contains the flutter goldens + /// and new flutter_gallery. + final Directory gitCache = fs.directory(path.join(flutterRoot.absolute.path, '.pub-cache', 'git')); + if (gitCache.existsSync()) { + gitCache.deleteSync(recursive: true); + } + } + + /// Write the archive to the given output file. + Future _archiveFiles(File outputFile) async { + if (outputFile.path.toLowerCase().endsWith('.zip')) { + await _createZipArchive(outputFile, flutterRoot); + } else if (outputFile.path.toLowerCase().endsWith('.tar.xz')) { + await _createTarArchive(outputFile, flutterRoot); + } + } + + Future _runDart(List args, {Directory? workingDirectory}) { + return _processRunner.runProcess( + [_dart, ...args], + workingDirectory: workingDirectory ?? flutterRoot, + ); + } + + Future _runFlutter(List args, {Directory? workingDirectory}) { + return _processRunner.runProcess( + [_flutter, ...args], + workingDirectory: workingDirectory ?? flutterRoot, + ); + } + + Future _runGit(List args, {Directory? workingDirectory}) { + return _processRunner.runProcess( + ['git', ...args], + workingDirectory: workingDirectory ?? flutterRoot, + ); + } + + /// Unpacks the given zip file into the currentDirectory (if set), or the + /// same directory as the archive. + Future _unzipArchive(File archive, {Directory? workingDirectory}) { + workingDirectory ??= fs.directory(path.dirname(archive.absolute.path)); + List commandLine; + if (platform.isWindows) { + commandLine = [ + '7za', + 'x', + archive.absolute.path, + ]; + } else { + commandLine = [ + 'unzip', + archive.absolute.path, + ]; + } + return _processRunner.runProcess(commandLine, workingDirectory: workingDirectory); + } + + /// Create a zip archive from the directory source. + Future _createZipArchive(File output, Directory source) async { + List commandLine; + if (platform.isWindows) { + // Unhide the .git folder, https://docs.microsoft.com/en-us/windows-server/administration/windows-commands/attrib. + await _processRunner.runProcess( + ['attrib', '-h', '.git'], + workingDirectory: fs.directory(source.absolute.path), + ); + commandLine = [ + '7za', + 'a', + '-tzip', + '-mx=9', + output.absolute.path, + path.basename(source.path), + ]; + } else { + commandLine = [ + 'zip', + '-r', + '-9', + '--symlinks', + output.absolute.path, + path.basename(source.path), + ]; + } + return _processRunner.runProcess( + commandLine, + workingDirectory: fs.directory(path.dirname(source.absolute.path)), + ); + } + + /// Create a tar archive from the directory source. + Future _createTarArchive(File output, Directory source) { + return _processRunner.runProcess([ + 'tar', + 'cJf', + output.absolute.path, + path.basename(source.absolute.path), + ], workingDirectory: fs.directory(path.dirname(source.absolute.path))); + } +} diff --git a/dev/bots/prepare_package/archive_publisher.dart b/dev/bots/prepare_package/archive_publisher.dart new file mode 100644 index 0000000000..7e65a90f6a --- /dev/null +++ b/dev/bots/prepare_package/archive_publisher.dart @@ -0,0 +1,229 @@ +// Copyright 2014 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'dart:convert'; + +import 'package:crypto/crypto.dart'; +import 'package:crypto/src/digest_sink.dart'; +import 'package:file/file.dart'; +import 'package:path/path.dart' as path; +import 'package:platform/platform.dart' show LocalPlatform, Platform; +import 'package:process/process.dart'; + +import 'common.dart'; +import 'process_runner.dart'; + +class ArchivePublisher { + ArchivePublisher( + this.tempDir, + this.revision, + this.branch, + this.version, + this.outputFile, + this.dryRun, { + ProcessManager? processManager, + bool subprocessOutput = true, + required this.fs, + this.platform = const LocalPlatform(), + }) : assert(revision.length == 40), + platformName = platform.operatingSystem.toLowerCase(), + metadataGsPath = '$gsReleaseFolder/${getMetadataFilename(platform)}', + _processRunner = ProcessRunner( + processManager: processManager, + subprocessOutput: subprocessOutput, + ); + + final Platform platform; + final FileSystem fs; + final String platformName; + final String metadataGsPath; + final Branch branch; + final String revision; + final Map version; + final Directory tempDir; + final File outputFile; + final ProcessRunner _processRunner; + final bool dryRun; + String get destinationArchivePath => '${branch.name}/$platformName/${path.basename(outputFile.path)}'; + static String getMetadataFilename(Platform platform) => 'releases_${platform.operatingSystem.toLowerCase()}.json'; + + Future _getChecksum(File archiveFile) async { + final DigestSink digestSink = DigestSink(); + final ByteConversionSink sink = sha256.startChunkedConversion(digestSink); + + final Stream> stream = archiveFile.openRead(); + await stream.forEach((List chunk) { + sink.add(chunk); + }); + sink.close(); + return digestSink.value.toString(); + } + + /// Publish the archive to Google Storage. + /// + /// This method will throw if the target archive already exists on cloud + /// storage. + Future publishArchive([bool forceUpload = false]) async { + final String destGsPath = '$gsReleaseFolder/$destinationArchivePath'; + if (!forceUpload) { + if (await _cloudPathExists(destGsPath) && !dryRun) { + throw PreparePackageException( + 'File $destGsPath already exists on cloud storage!', + ); + } + } + await _cloudCopy( + src: outputFile.absolute.path, + dest: destGsPath, + ); + assert(tempDir.existsSync()); + final String gcsPath = '$gsReleaseFolder/${getMetadataFilename(platform)}'; + await _publishMetadata(gcsPath); + } + + /// Downloads and updates the metadata file without publishing it. + Future generateLocalMetadata() async { + await _updateMetadata('$gsReleaseFolder/${getMetadataFilename(platform)}'); + } + + Future> _addRelease(Map jsonData) async { + jsonData['base_url'] = '$baseUrl$releaseFolder'; + if (!jsonData.containsKey('current_release')) { + jsonData['current_release'] = {}; + } + (jsonData['current_release'] as Map)[branch.name] = revision; + if (!jsonData.containsKey('releases')) { + jsonData['releases'] = >[]; + } + + final Map newEntry = {}; + newEntry['hash'] = revision; + newEntry['channel'] = branch.name; + newEntry['version'] = version[frameworkVersionTag]; + newEntry['dart_sdk_version'] = version[dartVersionTag]; + newEntry['dart_sdk_arch'] = version[dartTargetArchTag]; + newEntry['release_date'] = DateTime.now().toUtc().toIso8601String(); + newEntry['archive'] = destinationArchivePath; + newEntry['sha256'] = await _getChecksum(outputFile); + + // Search for any entries with the same hash and channel and remove them. + final List releases = jsonData['releases'] as List; + jsonData['releases'] = >[ + for (final Map entry in releases.cast>()) + if (entry['hash'] != newEntry['hash'] || + entry['channel'] != newEntry['channel'] || + entry['dart_sdk_arch'] != newEntry['dart_sdk_arch']) + entry, + newEntry, + ]..sort((Map a, Map b) { + final DateTime aDate = DateTime.parse(a['release_date'] as String); + final DateTime bDate = DateTime.parse(b['release_date'] as String); + return bDate.compareTo(aDate); + }); + return jsonData; + } + + Future _updateMetadata(String gsPath) async { + // We can't just cat the metadata from the server with 'gsutil cat', because + // Windows wants to echo the commands that execute in gsutil.bat to the + // stdout when we do that. So, we copy the file locally and then read it + // back in. + final File metadataFile = fs.file( + path.join(tempDir.absolute.path, getMetadataFilename(platform)), + ); + await _runGsUtil(['cp', gsPath, metadataFile.absolute.path]); + Map jsonData = {}; + if (!dryRun) { + final String currentMetadata = metadataFile.readAsStringSync(); + if (currentMetadata.isEmpty) { + throw PreparePackageException('Empty metadata received from server'); + } + try { + jsonData = json.decode(currentMetadata) as Map; + } on FormatException catch (e) { + throw PreparePackageException('Unable to parse JSON metadata received from cloud: $e'); + } + } + // Run _addRelease, even on a dry run, so we can inspect the metadata on a + // dry run. On a dry run, the only thing in the metadata file be the new + // release. + jsonData = await _addRelease(jsonData); + + const JsonEncoder encoder = JsonEncoder.withIndent(' '); + metadataFile.writeAsStringSync(encoder.convert(jsonData)); + } + + /// Publishes the metadata file to GCS. + Future _publishMetadata(String gsPath) async { + final File metadataFile = fs.file( + path.join(tempDir.absolute.path, getMetadataFilename(platform)), + ); + await _cloudCopy( + src: metadataFile.absolute.path, + dest: gsPath, + // This metadata file is used by the website, so we don't want a long + // latency between publishing a release and it being available on the + // site. + cacheSeconds: shortCacheSeconds, + ); + } + + Future _runGsUtil( + List args, { + Directory? workingDirectory, + bool failOk = false, + }) async { + if (dryRun) { + print('gsutil.py -- $args'); + return ''; + } + return _processRunner.runProcess( + ['python3', path.join(platform.environment['DEPOT_TOOLS']!, 'gsutil.py'), '--', ...args], + workingDirectory: workingDirectory, + failOk: failOk, + ); + } + + /// Determine if a file exists at a given [cloudPath]. + Future _cloudPathExists(String cloudPath) async { + try { + await _runGsUtil( + ['stat', cloudPath], + ); + } on PreparePackageException { + // `gsutil stat gs://path/to/file` will exit with 1 if file does not exist + return false; + } + return true; + } + + Future _cloudCopy({ + required String src, + required String dest, + int? cacheSeconds, + }) async { + // We often don't have permission to overwrite, but + // we have permission to remove, so that's what we do. + await _runGsUtil(['rm', dest], failOk: true); + String? mimeType; + if (dest.endsWith('.tar.xz')) { + mimeType = 'application/x-gtar'; + } + if (dest.endsWith('.zip')) { + mimeType = 'application/zip'; + } + if (dest.endsWith('.json')) { + mimeType = 'application/json'; + } + return _runGsUtil([ + // Use our preferred MIME type for the files we care about + // and let gsutil figure it out for anything else. + if (mimeType != null) ...['-h', 'Content-Type:$mimeType'], + if (cacheSeconds != null) ...['-h', 'Cache-Control:max-age=$cacheSeconds'], + 'cp', + src, + dest, + ]); + } +} diff --git a/dev/bots/prepare_package/common.dart b/dev/bots/prepare_package/common.dart new file mode 100644 index 0000000000..09136f5a6a --- /dev/null +++ b/dev/bots/prepare_package/common.dart @@ -0,0 +1,46 @@ +// Copyright 2014 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'dart:io' hide Platform; + +const String gobMirror = 'https://flutter.googlesource.com/mirrors/flutter'; +const String githubRepo = 'https://github.com/flutter/flutter.git'; +const String mingitForWindowsUrl = 'https://storage.googleapis.com/flutter_infra_release/mingit/' + '603511c649b00bbef0a6122a827ac419b656bc19/mingit.zip'; +const String releaseFolder = '/releases'; +const String gsBase = 'gs://flutter_infra_release'; +const String gsReleaseFolder = '$gsBase$releaseFolder'; +const String baseUrl = 'https://storage.googleapis.com/flutter_infra_release'; +const int shortCacheSeconds = 60; +const String frameworkVersionTag = 'frameworkVersionFromGit'; +const String dartVersionTag = 'dartSdkVersion'; +const String dartTargetArchTag = 'dartTargetArch'; + +enum Branch { + beta, + stable, + master, + main; +} + +/// Exception class for when a process fails to run, so we can catch +/// it and provide something more readable than a stack trace. +class PreparePackageException implements Exception { + PreparePackageException(this.message, [this.result]); + + final String message; + final ProcessResult? result; + int get exitCode => result?.exitCode ?? -1; + + @override + String toString() { + String output = runtimeType.toString(); + output += ': $message'; + final String stderr = result?.stderr as String? ?? ''; + if (stderr.isNotEmpty) { + output += ':\n$stderr'; + } + return output; + } +} diff --git a/dev/bots/prepare_package/process_runner.dart b/dev/bots/prepare_package/process_runner.dart new file mode 100644 index 0000000000..4fff8b7c61 --- /dev/null +++ b/dev/bots/prepare_package/process_runner.dart @@ -0,0 +1,116 @@ +// Copyright 2014 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'dart:async'; +import 'dart:convert'; +import 'dart:io' hide Platform; + +import 'package:platform/platform.dart' show LocalPlatform, Platform; +import 'package:process/process.dart'; + +import 'common.dart'; + +/// A helper class for classes that want to run a process. +/// +/// The stderr and stdout can optionally be reported as the process runs, and +/// capture the stdout properly without dropping any. +class ProcessRunner { + ProcessRunner({ + ProcessManager? processManager, + this.subprocessOutput = true, + this.defaultWorkingDirectory, + this.platform = const LocalPlatform(), + }) : processManager = processManager ?? const LocalProcessManager() { + environment = Map.from(platform.environment); + } + + /// The platform to use for a starting environment. + final Platform platform; + + /// Set [subprocessOutput] to show output as processes run. Stdout from the + /// process will be printed to stdout, and stderr printed to stderr. + final bool subprocessOutput; + + /// Set the [processManager] in order to inject a test instance to perform + /// testing. + final ProcessManager processManager; + + /// Sets the default directory used when `workingDirectory` is not specified + /// to [runProcess]. + final Directory? defaultWorkingDirectory; + + /// The environment to run processes with. + late Map environment; + + /// Run the command and arguments in `commandLine` as a sub-process from + /// `workingDirectory` if set, or the [defaultWorkingDirectory] if not. Uses + /// [Directory.current] if [defaultWorkingDirectory] is not set. + /// + /// Set `failOk` if [runProcess] should not throw an exception when the + /// command completes with a non-zero exit code. + Future runProcess( + List commandLine, { + Directory? workingDirectory, + bool failOk = false, + }) async { + workingDirectory ??= defaultWorkingDirectory ?? Directory.current; + if (subprocessOutput) { + stderr.write('Running "${commandLine.join(' ')}" in ${workingDirectory.path}.\n'); + } + final List output = []; + final Completer stdoutComplete = Completer(); + final Completer stderrComplete = Completer(); + late Process process; + Future allComplete() async { + await stderrComplete.future; + await stdoutComplete.future; + return process.exitCode; + } + + try { + process = await processManager.start( + commandLine, + workingDirectory: workingDirectory.absolute.path, + environment: environment, + ); + process.stdout.listen( + (List event) { + output.addAll(event); + if (subprocessOutput) { + stdout.add(event); + } + }, + onDone: () async => stdoutComplete.complete(), + ); + if (subprocessOutput) { + process.stderr.listen( + (List event) { + stderr.add(event); + }, + onDone: () async => stderrComplete.complete(), + ); + } else { + stderrComplete.complete(); + } + } on ProcessException catch (e) { + final String message = 'Running "${commandLine.join(' ')}" in ${workingDirectory.path} ' + 'failed with:\n$e'; + throw PreparePackageException(message); + } on ArgumentError catch (e) { + final String message = 'Running "${commandLine.join(' ')}" in ${workingDirectory.path} ' + 'failed with:\n$e'; + throw PreparePackageException(message); + } + + final int exitCode = await allComplete(); + if (exitCode != 0 && !failOk) { + final String message = 'Running "${commandLine.join(' ')}" in ${workingDirectory.path} failed'; + throw PreparePackageException( + message, + ProcessResult(0, exitCode, null, 'returned $exitCode'), + ); + } + return utf8.decoder.convert(output).trim(); + } +} diff --git a/dev/bots/test/prepare_package_test.dart b/dev/bots/test/prepare_package_test.dart index 10ba341cc3..b7fe3c7b76 100644 --- a/dev/bots/test/prepare_package_test.dart +++ b/dev/bots/test/prepare_package_test.dart @@ -3,14 +3,19 @@ // found in the LICENSE file. import 'dart:convert'; -import 'dart:io' hide Platform; +import 'dart:io' show ProcessResult; import 'dart:typed_data'; +import 'package:file/file.dart'; +import 'package:file/memory.dart'; import 'package:path/path.dart' as path; import 'package:platform/platform.dart' show FakePlatform, Platform; import '../../../packages/flutter_tools/test/src/fake_process_manager.dart'; -import '../prepare_package.dart'; +import '../prepare_package/archive_creator.dart'; +import '../prepare_package/archive_publisher.dart'; +import '../prepare_package/common.dart'; +import '../prepare_package/process_runner.dart'; import 'common.dart'; void main() { @@ -73,12 +78,14 @@ void main() { throwsA(isA())); }); }); + group('ArchiveCreator for $platformName', () { late ArchiveCreator creator; late Directory tempDir; Directory flutterDir; Directory cacheDir; late FakeProcessManager processManager; + late FileSystem fs; final List> args = >[]; final List> namedArgs = >[]; late String flutter; @@ -92,16 +99,18 @@ void main() { processManager = FakeProcessManager.list([]); args.clear(); namedArgs.clear(); - tempDir = Directory.systemTemp.createTempSync('flutter_prepage_package_test.'); - flutterDir = Directory(path.join(tempDir.path, 'flutter')); + fs = MemoryFileSystem.test(); + tempDir = fs.systemTempDirectory; + flutterDir = fs.directory(path.join(tempDir.path, 'flutter')); flutterDir.createSync(recursive: true); - cacheDir = Directory(path.join(flutterDir.path, 'bin', 'cache')); + cacheDir = fs.directory(path.join(flutterDir.path, 'bin', 'cache')); cacheDir.createSync(recursive: true); creator = ArchiveCreator( tempDir, tempDir, testRef, Branch.beta, + fs: fs, processManager: processManager, subprocessOutput: false, platform: platform, @@ -196,6 +205,7 @@ void main() { tempDir, testRef, Branch.beta, + fs: fs, processManager: processManager, subprocessOutput: false, platform: platform, @@ -245,6 +255,7 @@ void main() { tempDir, testRef, Branch.beta, + fs: fs, processManager: processManager, subprocessOutput: false, platform: platform, @@ -303,6 +314,7 @@ void main() { tempDir, testRef, Branch.beta, + fs: fs, strict: false, processManager: processManager, subprocessOutput: false, @@ -355,6 +367,7 @@ void main() { tempDir, testRef, Branch.beta, + fs: fs, processManager: processManager, subprocessOutput: false, platform: platform, @@ -376,6 +389,7 @@ void main() { group('ArchivePublisher for $platformName', () { late FakeProcessManager processManager; late Directory tempDir; + late FileSystem fs; final String gsutilCall = platform.isWindows ? 'python3 ${path.join("D:", "depot_tools", "gsutil.py")}' : 'python3 ${path.join("/", "depot_tools", "gsutil.py")}'; @@ -385,8 +399,9 @@ void main() { final String gsArchivePath = 'gs://flutter_infra_release/releases/stable/$platformName/$archiveName'; setUp(() async { + fs = MemoryFileSystem.test(style: platform.isWindows ? FileSystemStyle.windows : FileSystemStyle.posix); processManager = FakeProcessManager.list([]); - tempDir = Directory.systemTemp.createTempSync('flutter_prepage_package_test.'); + tempDir = fs.systemTempDirectory.createTempSync('flutter_prepage_package_test.'); }); tearDown(() async { @@ -435,8 +450,8 @@ void main() { ] } '''; - File(jsonPath).writeAsStringSync(releasesJson); - File(archivePath).writeAsStringSync('archive contents'); + fs.file(jsonPath).writeAsStringSync(releasesJson); + fs.file(archivePath).writeAsStringSync('archive contents'); final Map?> calls = ?>{ // This process fails because the file does NOT already exist '$gsutilCall -- cp $gsJsonPath $jsonPath': null, @@ -447,7 +462,7 @@ void main() { '$gsutilCall -- -h Content-Type:application/json -h Cache-Control:max-age=60 cp $jsonPath $gsJsonPath': null, }; processManager.addCommands(convertResults(calls)); - final File outputFile = File(path.join(tempDir.absolute.path, archiveName)); + final File outputFile = fs.file(path.join(tempDir.absolute.path, archiveName)); outputFile.createSync(); assert(tempDir.existsSync()); final ArchivePublisher publisher = ArchivePublisher( @@ -461,6 +476,7 @@ void main() { }, outputFile, false, + fs: fs, processManager: processManager, subprocessOutput: false, platform: platform, @@ -469,7 +485,7 @@ void main() { await publisher.generateLocalMetadata(); await publisher.publishArchive(); - final File releaseFile = File(jsonPath); + final File releaseFile = fs.file(jsonPath); expect(releaseFile.existsSync(), isTrue); final String contents = releaseFile.readAsStringSync(); // Make sure new data is added. @@ -536,8 +552,8 @@ void main() { ] } '''; - File(jsonPath).writeAsStringSync(releasesJson); - File(archivePath).writeAsStringSync('archive contents'); + fs.file(jsonPath).writeAsStringSync(releasesJson); + fs.file(archivePath).writeAsStringSync('archive contents'); final Map?> calls = ?>{ // This process fails because the file does NOT already exist '$gsutilCall -- cp $gsJsonPath $jsonPath': null, @@ -548,7 +564,7 @@ void main() { '$gsutilCall -- -h Content-Type:application/json -h Cache-Control:max-age=60 cp $jsonPath $gsJsonPath': null, }; processManager.addCommands(convertResults(calls)); - final File outputFile = File(path.join(tempDir.absolute.path, archiveName)); + final File outputFile = fs.file(path.join(tempDir.absolute.path, archiveName)); outputFile.createSync(); assert(tempDir.existsSync()); final ArchivePublisher publisher = ArchivePublisher( @@ -562,6 +578,7 @@ void main() { }, outputFile, false, + fs: fs, processManager: processManager, subprocessOutput: false, platform: platform, @@ -570,7 +587,7 @@ void main() { await publisher.generateLocalMetadata(); await publisher.publishArchive(); - final File releaseFile = File(jsonPath); + final File releaseFile = fs.file(jsonPath); expect(releaseFile.existsSync(), isTrue); final String contents = releaseFile.readAsStringSync(); expect(contents, contains('"dart_sdk_version": "3.2.1"')); @@ -601,8 +618,8 @@ void main() { ] } '''; - File(jsonPath).writeAsStringSync(releasesJson); - File(archivePath).writeAsStringSync('archive contents'); + fs.file(jsonPath).writeAsStringSync(releasesJson); + fs.file(archivePath).writeAsStringSync('archive contents'); final Map?> calls = ?>{ // This process fails because the file does NOT already exist '$gsutilCall -- cp $gsJsonPath $jsonPath': null, @@ -613,7 +630,7 @@ void main() { '$gsutilCall -- -h Content-Type:application/json -h Cache-Control:max-age=60 cp $jsonPath $gsJsonPath': null, }; processManager.addCommands(convertResults(calls)); - final File outputFile = File(path.join(tempDir.absolute.path, archiveName)); + final File outputFile = fs.file(path.join(tempDir.absolute.path, archiveName)); outputFile.createSync(); assert(tempDir.existsSync()); final ArchivePublisher publisher = ArchivePublisher( @@ -627,6 +644,7 @@ void main() { }, outputFile, false, + fs: fs, processManager: processManager, subprocessOutput: false, platform: platform, @@ -635,7 +653,7 @@ void main() { await publisher.generateLocalMetadata(); await publisher.publishArchive(); - final File releaseFile = File(jsonPath); + final File releaseFile = fs.file(jsonPath); expect(releaseFile.existsSync(), isTrue); final String contents = releaseFile.readAsStringSync(); final Map releases = jsonDecode(contents) as Map; @@ -682,8 +700,8 @@ void main() { ] } '''; - File(jsonPath).writeAsStringSync(releasesJson); - File(archivePath).writeAsStringSync('archive contents'); + fs.file(jsonPath).writeAsStringSync(releasesJson); + fs.file(archivePath).writeAsStringSync('archive contents'); final Map?> calls = ?>{ // This process fails because the file does NOT already exist '$gsutilCall -- cp $gsJsonPath $jsonPath': null, @@ -694,7 +712,7 @@ void main() { '$gsutilCall -- -h Content-Type:application/json -h Cache-Control:max-age=60 cp $jsonPath $gsJsonPath': null, }; processManager.addCommands(convertResults(calls)); - final File outputFile = File(path.join(tempDir.absolute.path, archiveName)); + final File outputFile = fs.file(path.join(tempDir.absolute.path, archiveName)); outputFile.createSync(); assert(tempDir.existsSync()); final ArchivePublisher publisher = ArchivePublisher( @@ -708,6 +726,7 @@ void main() { }, outputFile, false, + fs: fs, processManager: processManager, subprocessOutput: false, platform: platform, @@ -716,7 +735,7 @@ void main() { await publisher.generateLocalMetadata(); await publisher.publishArchive(); - final File releaseFile = File(jsonPath); + final File releaseFile = fs.file(jsonPath); expect(releaseFile.existsSync(), isTrue); final String contents = releaseFile.readAsStringSync(); final Map jsonData = json.decode(contents) as Map; @@ -725,7 +744,7 @@ void main() { test('publishArchive throws if forceUpload is false and artifact already exists on cloud storage', () async { final String archiveName = platform.isLinux ? 'archive.tar.xz' : 'archive.zip'; - final File outputFile = File(path.join(tempDir.absolute.path, archiveName)); + final File outputFile = fs.file(path.join(tempDir.absolute.path, archiveName)); final ArchivePublisher publisher = ArchivePublisher( tempDir, testRef, @@ -737,6 +756,7 @@ void main() { }, outputFile, false, + fs: fs, processManager: processManager, subprocessOutput: false, platform: platform, @@ -751,7 +771,7 @@ void main() { test('publishArchive does not throw if forceUpload is true and artifact already exists on cloud storage', () async { final String archiveName = platform.isLinux ? 'archive.tar.xz' : 'archive.zip'; - final File outputFile = File(path.join(tempDir.absolute.path, archiveName)); + final File outputFile = fs.file(path.join(tempDir.absolute.path, archiveName)); final ArchivePublisher publisher = ArchivePublisher( tempDir, testRef, @@ -763,6 +783,7 @@ void main() { }, outputFile, false, + fs: fs, processManager: processManager, subprocessOutput: false, platform: platform, @@ -805,8 +826,8 @@ void main() { ] } '''; - File(jsonPath).writeAsStringSync(releasesJson); - File(archivePath).writeAsStringSync('archive contents'); + fs.file(jsonPath).writeAsStringSync(releasesJson); + fs.file(archivePath).writeAsStringSync('archive contents'); final Map?> calls = ?>{ '$gsutilCall -- cp $gsJsonPath $jsonPath': null, '$gsutilCall -- rm $gsArchivePath': null,