Update prepare_package to upload artifacts to old and new buckets. (#75389)
* Update prepare_package to upload to old and new infra buckets. Bug: https://github.com/flutter/flutter/issues/75363 * Fix failing tests. * Add duplicated code to a loop.
This commit is contained in:
godofredoc
GitHub
parent
c0ea3bc599
commit
40877c02a7
@ -17,12 +17,15 @@ import 'package:process/process.dart';
|
||||
|
||||
const String chromiumRepo = 'https://chromium.googlesource.com/external/github.com/flutter/flutter';
|
||||
const String githubRepo = 'https://github.com/flutter/flutter.git';
|
||||
const String mingitForWindowsUrl = 'https://storage.googleapis.com/flutter_infra/mingit/'
|
||||
const String mingitForWindowsUrl = 'https://storage.googleapis.com/flutter_infra_release/mingit/'
|
||||
'603511c649b00bbef0a6122a827ac419b656bc19/mingit.zip';
|
||||
const String gsBase = 'gs://flutter_infra';
|
||||
const String oldGsBase = 'gs://flutter_infra';
|
||||
const String releaseFolder = '/releases';
|
||||
const String gsReleaseFolder = '$gsBase$releaseFolder';
|
||||
const String baseUrl = 'https://storage.googleapis.com/flutter_infra';
|
||||
const String oldGsReleaseFolder = '$oldGsBase$releaseFolder';
|
||||
const String oldBaseUrl = 'https://storage.googleapis.com/flutter_infra';
|
||||
const String newGsBase = 'gs://flutter_infra_release';
|
||||
const String newGsReleaseFolder = '$newGsBase$releaseFolder';
|
||||
const String newBaseUrl = 'https://storage.googleapis.com/flutter_infra_release';
|
||||
|
||||
/// Exception class for when a process fails to run, so we can catch
|
||||
/// it and provide something more readable than a stack trace.
|
||||
@ -470,13 +473,14 @@ class ArchivePublisher {
|
||||
this.revision,
|
||||
this.branch,
|
||||
this.version,
|
||||
this.outputFile, {
|
||||
this.outputFile,
|
||||
this.dryRun, {
|
||||
ProcessManager processManager,
|
||||
bool subprocessOutput = true,
|
||||
this.platform = const LocalPlatform(),
|
||||
}) : assert(revision.length == 40),
|
||||
platformName = platform.operatingSystem.toLowerCase(),
|
||||
metadataGsPath = '$gsReleaseFolder/${getMetadataFilename(platform)}',
|
||||
metadataGsPath = '$newGsReleaseFolder/${getMetadataFilename(platform)}',
|
||||
_processRunner = ProcessRunner(
|
||||
processManager: processManager,
|
||||
subprocessOutput: subprocessOutput,
|
||||
@ -491,6 +495,7 @@ class ArchivePublisher {
|
||||
final Directory tempDir;
|
||||
final File outputFile;
|
||||
final ProcessRunner _processRunner;
|
||||
final bool dryRun;
|
||||
String get branchName => getBranchName(branch);
|
||||
String get destinationArchivePath => '$branchName/$platformName/${path.basename(outputFile.path)}';
|
||||
static String getMetadataFilename(Platform platform) => 'releases_${platform.operatingSystem.toLowerCase()}.json';
|
||||
@ -512,21 +517,24 @@ class ArchivePublisher {
|
||||
/// This method will throw if the target archive already exists on cloud
|
||||
/// storage.
|
||||
Future<void> publishArchive([bool forceUpload = false]) async {
|
||||
final String destGsPath = '$gsReleaseFolder/$destinationArchivePath';
|
||||
if (!forceUpload) {
|
||||
if (await _cloudPathExists(destGsPath)) {
|
||||
throw PreparePackageException(
|
||||
'File $destGsPath already exists on cloud storage!',
|
||||
);
|
||||
for (final String releaseFolder in <String>[oldGsReleaseFolder, newGsReleaseFolder]) {
|
||||
final String destGsPath = '$releaseFolder/$destinationArchivePath';
|
||||
if (!forceUpload) {
|
||||
if (await _cloudPathExists(destGsPath) && !dryRun) {
|
||||
throw PreparePackageException(
|
||||
'File $destGsPath already exists on cloud storage!',
|
||||
);
|
||||
}
|
||||
}
|
||||
await _cloudCopy(outputFile.absolute.path, destGsPath);
|
||||
assert(tempDir.existsSync());
|
||||
await _updateMetadata('$releaseFolder/${getMetadataFilename(platform)}', newBucket: false);
|
||||
}
|
||||
await _cloudCopy(outputFile.absolute.path, destGsPath);
|
||||
assert(tempDir.existsSync());
|
||||
await _updateMetadata();
|
||||
}
|
||||
|
||||
Future<Map<String, dynamic>> _addRelease(Map<String, dynamic> jsonData) async {
|
||||
jsonData['base_url'] = '$baseUrl$releaseFolder';
|
||||
Future<Map<String, dynamic>> _addRelease(Map<String, dynamic> jsonData, {bool newBucket=true}) async {
|
||||
final String tmpBaseUrl = newBucket ? newBaseUrl : oldBaseUrl;
|
||||
jsonData['base_url'] = '$tmpBaseUrl$releaseFolder';
|
||||
if (!jsonData.containsKey('current_release')) {
|
||||
jsonData['current_release'] = <String, String>{};
|
||||
}
|
||||
@ -558,7 +566,7 @@ class ArchivePublisher {
|
||||
return jsonData;
|
||||
}
|
||||
|
||||
Future<void> _updateMetadata() async {
|
||||
Future<void> _updateMetadata(String gsPath, {bool newBucket=true}) async {
|
||||
// We can't just cat the metadata from the server with 'gsutil cat', because
|
||||
// Windows wants to echo the commands that execute in gsutil.bat to the
|
||||
// stdout when we do that. So, we copy the file locally and then read it
|
||||
@ -566,24 +574,26 @@ class ArchivePublisher {
|
||||
final File metadataFile = File(
|
||||
path.join(tempDir.absolute.path, getMetadataFilename(platform)),
|
||||
);
|
||||
await _runGsUtil(<String>['cp', metadataGsPath, metadataFile.absolute.path]);
|
||||
final String currentMetadata = metadataFile.readAsStringSync();
|
||||
if (currentMetadata.isEmpty) {
|
||||
throw PreparePackageException('Empty metadata received from server');
|
||||
await _runGsUtil(<String>['cp', gsPath, metadataFile.absolute.path]);
|
||||
if (!dryRun) {
|
||||
final String currentMetadata = metadataFile.readAsStringSync();
|
||||
if (currentMetadata.isEmpty) {
|
||||
throw PreparePackageException('Empty metadata received from server');
|
||||
}
|
||||
|
||||
Map<String, dynamic> jsonData;
|
||||
try {
|
||||
jsonData = json.decode(currentMetadata) as Map<String, dynamic>;
|
||||
} on FormatException catch (e) {
|
||||
throw PreparePackageException('Unable to parse JSON metadata received from cloud: $e');
|
||||
}
|
||||
|
||||
jsonData = await _addRelease(jsonData, newBucket: newBucket);
|
||||
|
||||
const JsonEncoder encoder = JsonEncoder.withIndent(' ');
|
||||
metadataFile.writeAsStringSync(encoder.convert(jsonData));
|
||||
}
|
||||
|
||||
Map<String, dynamic> jsonData;
|
||||
try {
|
||||
jsonData = json.decode(currentMetadata) as Map<String, dynamic>;
|
||||
} on FormatException catch (e) {
|
||||
throw PreparePackageException('Unable to parse JSON metadata received from cloud: $e');
|
||||
}
|
||||
|
||||
jsonData = await _addRelease(jsonData);
|
||||
|
||||
const JsonEncoder encoder = JsonEncoder.withIndent(' ');
|
||||
metadataFile.writeAsStringSync(encoder.convert(jsonData));
|
||||
await _cloudCopy(metadataFile.absolute.path, metadataGsPath);
|
||||
await _cloudCopy(metadataFile.absolute.path, gsPath);
|
||||
}
|
||||
|
||||
Future<String> _runGsUtil(
|
||||
@ -591,6 +601,10 @@ class ArchivePublisher {
|
||||
Directory workingDirectory,
|
||||
bool failOk = false,
|
||||
}) async {
|
||||
if (dryRun) {
|
||||
print('gsutil.py -- $args');
|
||||
return '';
|
||||
}
|
||||
if (platform.isWindows) {
|
||||
return _processRunner.runProcess(
|
||||
<String>['python', path.join(platform.environment['DEPOT_TOOLS'], 'gsutil.py'), '--', ...args],
|
||||
@ -686,7 +700,7 @@ Future<void> main(List<String> rawArguments) async {
|
||||
defaultsTo: false,
|
||||
help: 'If set, will publish the archive to Google Cloud Storage upon '
|
||||
'successful creation of the archive. Will publish under this '
|
||||
'directory: $baseUrl$releaseFolder',
|
||||
'directory: $newBaseUrl$releaseFolder',
|
||||
);
|
||||
argParser.addFlag(
|
||||
'force',
|
||||
@ -694,6 +708,12 @@ Future<void> main(List<String> rawArguments) async {
|
||||
defaultsTo: false,
|
||||
help: 'Overwrite a previously uploaded package.',
|
||||
);
|
||||
argParser.addFlag(
|
||||
'dry_run',
|
||||
defaultsTo: false,
|
||||
negatable: false,
|
||||
help: 'Prints gsutil commands instead of executing them.',
|
||||
);
|
||||
argParser.addFlag(
|
||||
'help',
|
||||
defaultsTo: false,
|
||||
@ -763,6 +783,7 @@ Future<void> main(List<String> rawArguments) async {
|
||||
branch,
|
||||
version,
|
||||
outputFile,
|
||||
parsedArguments['dry_run'] as bool,
|
||||
);
|
||||
await publisher.publishArchive(parsedArguments['force'] as bool);
|
||||
}
|
||||
|
||||
@ -241,6 +241,7 @@ void main() {
|
||||
final String archiveName = platform.isLinux ? 'archive.tar.xz' : 'archive.zip';
|
||||
final String archiveMime = platform.isLinux ? 'application/x-gtar' : 'application/zip';
|
||||
final String gsArchivePath = 'gs://flutter_infra/releases/stable/$platformName/$archiveName';
|
||||
final String newGsArchivePath = 'gs://flutter_infra_release/releases/stable/$platformName/$archiveName';
|
||||
|
||||
setUp(() async {
|
||||
processManager = FakeProcessManager();
|
||||
@ -255,6 +256,7 @@ void main() {
|
||||
final String archivePath = path.join(tempDir.absolute.path, archiveName);
|
||||
final String jsonPath = path.join(tempDir.absolute.path, releasesName);
|
||||
final String gsJsonPath = 'gs://flutter_infra/releases/$releasesName';
|
||||
final String newGsJsonPath = 'gs://flutter_infra_release/releases/$releasesName';
|
||||
final String releasesJson = '''
|
||||
{
|
||||
"base_url": "https://storage.googleapis.com/flutter_infra/releases",
|
||||
@ -300,9 +302,16 @@ void main() {
|
||||
'$gsutilCall -- cp $gsJsonPath $jsonPath': null,
|
||||
'$gsutilCall -- rm $gsJsonPath': null,
|
||||
'$gsutilCall -- -h Content-Type:application/json cp $jsonPath $gsJsonPath': null,
|
||||
'$gsutilCall -- stat $newGsArchivePath': <ProcessResult>[ProcessResult(0, 1, '', '')],
|
||||
'$gsutilCall -- rm $newGsArchivePath': null,
|
||||
'$gsutilCall -- -h Content-Type:$archiveMime cp $archivePath $newGsArchivePath': null,
|
||||
'$gsutilCall -- cp $newGsJsonPath $jsonPath': null,
|
||||
'$gsutilCall -- rm $newGsJsonPath': null,
|
||||
'$gsutilCall -- -h Content-Type:application/json cp $jsonPath $newGsJsonPath': null,
|
||||
};
|
||||
processManager.fakeResults = calls;
|
||||
final File outputFile = File(path.join(tempDir.absolute.path, archiveName));
|
||||
outputFile.createSync();
|
||||
assert(tempDir.existsSync());
|
||||
final ArchivePublisher publisher = ArchivePublisher(
|
||||
tempDir,
|
||||
@ -310,6 +319,7 @@ void main() {
|
||||
Branch.stable,
|
||||
'v1.2.3',
|
||||
outputFile,
|
||||
false,
|
||||
processManager: processManager,
|
||||
subprocessOutput: false,
|
||||
platform: platform,
|
||||
@ -354,6 +364,7 @@ void main() {
|
||||
Branch.stable,
|
||||
'v1.2.3',
|
||||
outputFile,
|
||||
false,
|
||||
processManager: processManager,
|
||||
subprocessOutput: false,
|
||||
platform: platform,
|
||||
@ -376,6 +387,7 @@ void main() {
|
||||
Branch.stable,
|
||||
'v1.2.3',
|
||||
outputFile,
|
||||
false,
|
||||
processManager: processManager,
|
||||
subprocessOutput: false,
|
||||
platform: platform,
|
||||
@ -383,6 +395,7 @@ void main() {
|
||||
final String archivePath = path.join(tempDir.absolute.path, archiveName);
|
||||
final String jsonPath = path.join(tempDir.absolute.path, releasesName);
|
||||
final String gsJsonPath = 'gs://flutter_infra/releases/$releasesName';
|
||||
final String newGsJsonPath = 'gs://flutter_infra_release/releases/$releasesName';
|
||||
final String releasesJson = '''
|
||||
{
|
||||
"base_url": "https://storage.googleapis.com/flutter_infra/releases",
|
||||
@ -426,6 +439,11 @@ void main() {
|
||||
'$gsutilCall -- cp $gsJsonPath $jsonPath': null,
|
||||
'$gsutilCall -- rm $gsJsonPath': null,
|
||||
'$gsutilCall -- -h Content-Type:application/json cp $jsonPath $gsJsonPath': null,
|
||||
'$gsutilCall -- rm $newGsArchivePath': null,
|
||||
'$gsutilCall -- -h Content-Type:$archiveMime cp $archivePath $newGsArchivePath': null,
|
||||
'$gsutilCall -- cp $newGsJsonPath $jsonPath': null,
|
||||
'$gsutilCall -- rm $newGsJsonPath': null,
|
||||
'$gsutilCall -- -h Content-Type:application/json cp $jsonPath $newGsJsonPath': null,
|
||||
};
|
||||
processManager.fakeResults = calls;
|
||||
assert(tempDir.existsSync());
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user