diff --git a/dev/bots/prepare_package.dart b/dev/bots/prepare_package.dart index 0a25c48f97..6feea9c240 100644 --- a/dev/bots/prepare_package.dart +++ b/dev/bots/prepare_package.dart @@ -17,12 +17,15 @@ import 'package:process/process.dart'; const String chromiumRepo = 'https://chromium.googlesource.com/external/github.com/flutter/flutter'; const String githubRepo = 'https://github.com/flutter/flutter.git'; -const String mingitForWindowsUrl = 'https://storage.googleapis.com/flutter_infra/mingit/' +const String mingitForWindowsUrl = 'https://storage.googleapis.com/flutter_infra_release/mingit/' '603511c649b00bbef0a6122a827ac419b656bc19/mingit.zip'; -const String gsBase = 'gs://flutter_infra'; +const String oldGsBase = 'gs://flutter_infra'; const String releaseFolder = '/releases'; -const String gsReleaseFolder = '$gsBase$releaseFolder'; -const String baseUrl = 'https://storage.googleapis.com/flutter_infra'; +const String oldGsReleaseFolder = '$oldGsBase$releaseFolder'; +const String oldBaseUrl = 'https://storage.googleapis.com/flutter_infra'; +const String newGsBase = 'gs://flutter_infra_release'; +const String newGsReleaseFolder = '$newGsBase$releaseFolder'; +const String newBaseUrl = 'https://storage.googleapis.com/flutter_infra_release'; /// Exception class for when a process fails to run, so we can catch /// it and provide something more readable than a stack trace. @@ -470,13 +473,14 @@ class ArchivePublisher { this.revision, this.branch, this.version, - this.outputFile, { + this.outputFile, + this.dryRun, { ProcessManager processManager, bool subprocessOutput = true, this.platform = const LocalPlatform(), }) : assert(revision.length == 40), platformName = platform.operatingSystem.toLowerCase(), - metadataGsPath = '$gsReleaseFolder/${getMetadataFilename(platform)}', + metadataGsPath = '$newGsReleaseFolder/${getMetadataFilename(platform)}', _processRunner = ProcessRunner( processManager: processManager, subprocessOutput: subprocessOutput, @@ -491,6 +495,7 @@ class ArchivePublisher { final Directory tempDir; final File outputFile; final ProcessRunner _processRunner; + final bool dryRun; String get branchName => getBranchName(branch); String get destinationArchivePath => '$branchName/$platformName/${path.basename(outputFile.path)}'; static String getMetadataFilename(Platform platform) => 'releases_${platform.operatingSystem.toLowerCase()}.json'; @@ -512,21 +517,24 @@ class ArchivePublisher { /// This method will throw if the target archive already exists on cloud /// storage. Future publishArchive([bool forceUpload = false]) async { - final String destGsPath = '$gsReleaseFolder/$destinationArchivePath'; - if (!forceUpload) { - if (await _cloudPathExists(destGsPath)) { - throw PreparePackageException( - 'File $destGsPath already exists on cloud storage!', - ); + for (final String releaseFolder in [oldGsReleaseFolder, newGsReleaseFolder]) { + final String destGsPath = '$releaseFolder/$destinationArchivePath'; + if (!forceUpload) { + if (await _cloudPathExists(destGsPath) && !dryRun) { + throw PreparePackageException( + 'File $destGsPath already exists on cloud storage!', + ); + } } + await _cloudCopy(outputFile.absolute.path, destGsPath); + assert(tempDir.existsSync()); + await _updateMetadata('$releaseFolder/${getMetadataFilename(platform)}', newBucket: false); } - await _cloudCopy(outputFile.absolute.path, destGsPath); - assert(tempDir.existsSync()); - await _updateMetadata(); } - Future> _addRelease(Map jsonData) async { - jsonData['base_url'] = '$baseUrl$releaseFolder'; + Future> _addRelease(Map jsonData, {bool newBucket=true}) async { + final String tmpBaseUrl = newBucket ? newBaseUrl : oldBaseUrl; + jsonData['base_url'] = '$tmpBaseUrl$releaseFolder'; if (!jsonData.containsKey('current_release')) { jsonData['current_release'] = {}; } @@ -558,7 +566,7 @@ class ArchivePublisher { return jsonData; } - Future _updateMetadata() async { + Future _updateMetadata(String gsPath, {bool newBucket=true}) async { // We can't just cat the metadata from the server with 'gsutil cat', because // Windows wants to echo the commands that execute in gsutil.bat to the // stdout when we do that. So, we copy the file locally and then read it @@ -566,24 +574,26 @@ class ArchivePublisher { final File metadataFile = File( path.join(tempDir.absolute.path, getMetadataFilename(platform)), ); - await _runGsUtil(['cp', metadataGsPath, metadataFile.absolute.path]); - final String currentMetadata = metadataFile.readAsStringSync(); - if (currentMetadata.isEmpty) { - throw PreparePackageException('Empty metadata received from server'); + await _runGsUtil(['cp', gsPath, metadataFile.absolute.path]); + if (!dryRun) { + final String currentMetadata = metadataFile.readAsStringSync(); + if (currentMetadata.isEmpty) { + throw PreparePackageException('Empty metadata received from server'); + } + + Map jsonData; + try { + jsonData = json.decode(currentMetadata) as Map; + } on FormatException catch (e) { + throw PreparePackageException('Unable to parse JSON metadata received from cloud: $e'); + } + + jsonData = await _addRelease(jsonData, newBucket: newBucket); + + const JsonEncoder encoder = JsonEncoder.withIndent(' '); + metadataFile.writeAsStringSync(encoder.convert(jsonData)); } - - Map jsonData; - try { - jsonData = json.decode(currentMetadata) as Map; - } on FormatException catch (e) { - throw PreparePackageException('Unable to parse JSON metadata received from cloud: $e'); - } - - jsonData = await _addRelease(jsonData); - - const JsonEncoder encoder = JsonEncoder.withIndent(' '); - metadataFile.writeAsStringSync(encoder.convert(jsonData)); - await _cloudCopy(metadataFile.absolute.path, metadataGsPath); + await _cloudCopy(metadataFile.absolute.path, gsPath); } Future _runGsUtil( @@ -591,6 +601,10 @@ class ArchivePublisher { Directory workingDirectory, bool failOk = false, }) async { + if (dryRun) { + print('gsutil.py -- $args'); + return ''; + } if (platform.isWindows) { return _processRunner.runProcess( ['python', path.join(platform.environment['DEPOT_TOOLS'], 'gsutil.py'), '--', ...args], @@ -686,7 +700,7 @@ Future main(List rawArguments) async { defaultsTo: false, help: 'If set, will publish the archive to Google Cloud Storage upon ' 'successful creation of the archive. Will publish under this ' - 'directory: $baseUrl$releaseFolder', + 'directory: $newBaseUrl$releaseFolder', ); argParser.addFlag( 'force', @@ -694,6 +708,12 @@ Future main(List rawArguments) async { defaultsTo: false, help: 'Overwrite a previously uploaded package.', ); + argParser.addFlag( + 'dry_run', + defaultsTo: false, + negatable: false, + help: 'Prints gsutil commands instead of executing them.', + ); argParser.addFlag( 'help', defaultsTo: false, @@ -763,6 +783,7 @@ Future main(List rawArguments) async { branch, version, outputFile, + parsedArguments['dry_run'] as bool, ); await publisher.publishArchive(parsedArguments['force'] as bool); } diff --git a/dev/bots/test/prepare_package_test.dart b/dev/bots/test/prepare_package_test.dart index b41b4348d5..cef553ee1a 100644 --- a/dev/bots/test/prepare_package_test.dart +++ b/dev/bots/test/prepare_package_test.dart @@ -241,6 +241,7 @@ void main() { final String archiveName = platform.isLinux ? 'archive.tar.xz' : 'archive.zip'; final String archiveMime = platform.isLinux ? 'application/x-gtar' : 'application/zip'; final String gsArchivePath = 'gs://flutter_infra/releases/stable/$platformName/$archiveName'; + final String newGsArchivePath = 'gs://flutter_infra_release/releases/stable/$platformName/$archiveName'; setUp(() async { processManager = FakeProcessManager(); @@ -255,6 +256,7 @@ void main() { final String archivePath = path.join(tempDir.absolute.path, archiveName); final String jsonPath = path.join(tempDir.absolute.path, releasesName); final String gsJsonPath = 'gs://flutter_infra/releases/$releasesName'; + final String newGsJsonPath = 'gs://flutter_infra_release/releases/$releasesName'; final String releasesJson = ''' { "base_url": "https://storage.googleapis.com/flutter_infra/releases", @@ -300,9 +302,16 @@ void main() { '$gsutilCall -- cp $gsJsonPath $jsonPath': null, '$gsutilCall -- rm $gsJsonPath': null, '$gsutilCall -- -h Content-Type:application/json cp $jsonPath $gsJsonPath': null, + '$gsutilCall -- stat $newGsArchivePath': [ProcessResult(0, 1, '', '')], + '$gsutilCall -- rm $newGsArchivePath': null, + '$gsutilCall -- -h Content-Type:$archiveMime cp $archivePath $newGsArchivePath': null, + '$gsutilCall -- cp $newGsJsonPath $jsonPath': null, + '$gsutilCall -- rm $newGsJsonPath': null, + '$gsutilCall -- -h Content-Type:application/json cp $jsonPath $newGsJsonPath': null, }; processManager.fakeResults = calls; final File outputFile = File(path.join(tempDir.absolute.path, archiveName)); + outputFile.createSync(); assert(tempDir.existsSync()); final ArchivePublisher publisher = ArchivePublisher( tempDir, @@ -310,6 +319,7 @@ void main() { Branch.stable, 'v1.2.3', outputFile, + false, processManager: processManager, subprocessOutput: false, platform: platform, @@ -354,6 +364,7 @@ void main() { Branch.stable, 'v1.2.3', outputFile, + false, processManager: processManager, subprocessOutput: false, platform: platform, @@ -376,6 +387,7 @@ void main() { Branch.stable, 'v1.2.3', outputFile, + false, processManager: processManager, subprocessOutput: false, platform: platform, @@ -383,6 +395,7 @@ void main() { final String archivePath = path.join(tempDir.absolute.path, archiveName); final String jsonPath = path.join(tempDir.absolute.path, releasesName); final String gsJsonPath = 'gs://flutter_infra/releases/$releasesName'; + final String newGsJsonPath = 'gs://flutter_infra_release/releases/$releasesName'; final String releasesJson = ''' { "base_url": "https://storage.googleapis.com/flutter_infra/releases", @@ -426,6 +439,11 @@ void main() { '$gsutilCall -- cp $gsJsonPath $jsonPath': null, '$gsutilCall -- rm $gsJsonPath': null, '$gsutilCall -- -h Content-Type:application/json cp $jsonPath $gsJsonPath': null, + '$gsutilCall -- rm $newGsArchivePath': null, + '$gsutilCall -- -h Content-Type:$archiveMime cp $archivePath $newGsArchivePath': null, + '$gsutilCall -- cp $newGsJsonPath $jsonPath': null, + '$gsutilCall -- rm $newGsJsonPath': null, + '$gsutilCall -- -h Content-Type:application/json cp $jsonPath $newGsJsonPath': null, }; processManager.fakeResults = calls; assert(tempDir.existsSync());