firka/flutter
firka/flutter
5
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Remove logic to save artifacts to old gcs location. (#83526)

Browse Source
This commit is contained in:
godofredoc 2021-05-27 19:04:02 -07:00 committed by GitHub
parent c13c2ee6eb
commit 8bee0834e4
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 28 additions and 56 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

29
dev/bots/prepare_package.dart
View File

@ -20,13 +20,10 @@ const String chromiumRepo = 'https://chromium.googlesource.com/external/github.c
const String githubRepo = 'https://github.com/flutter/flutter.git'; const String githubRepo = 'https://github.com/flutter/flutter.git';
const String mingitForWindowsUrl = 'https://storage.googleapis.com/flutter_infra_release/mingit/' const String mingitForWindowsUrl = 'https://storage.googleapis.com/flutter_infra_release/mingit/'
'603511c649b00bbef0a6122a827ac419b656bc19/mingit.zip'; '603511c649b00bbef0a6122a827ac419b656bc19/mingit.zip';
const String oldGsBase = 'gs://flutter_infra';
const String releaseFolder = '/releases'; const String releaseFolder = '/releases';
const String oldGsReleaseFolder = '$oldGsBase$releaseFolder'; const String gsBase = 'gs://flutter_infra_release';
const String oldBaseUrl = 'https://storage.googleapis.com/flutter_infra'; const String gsReleaseFolder = '$gsBase$releaseFolder';
const String newGsBase = 'gs://flutter_infra_release'; const String baseUrl = 'https://storage.googleapis.com/flutter_infra_release';
const String newGsReleaseFolder = '$newGsBase$releaseFolder';
const String newBaseUrl = 'https://storage.googleapis.com/flutter_infra_release';
const int shortCacheSeconds = 60; const int shortCacheSeconds = 60;
/// Exception class for when a process fails to run, so we can catch /// Exception class for when a process fails to run, so we can catch
@ -540,7 +537,7 @@ class ArchivePublisher {
this.platform = const LocalPlatform(), this.platform = const LocalPlatform(),
}) : assert(revision.length == 40), }) : assert(revision.length == 40),
platformName = platform.operatingSystem.toLowerCase(), platformName = platform.operatingSystem.toLowerCase(),
metadataGsPath = '$newGsReleaseFolder/${getMetadataFilename(platform)}', metadataGsPath = '$gsReleaseFolder/${getMetadataFilename(platform)}',
_processRunner = ProcessRunner( _processRunner = ProcessRunner(
processManager: processManager, processManager: processManager,
subprocessOutput: subprocessOutput, subprocessOutput: subprocessOutput,
@ -577,9 +574,7 @@ class ArchivePublisher {
/// This method will throw if the target archive already exists on cloud /// This method will throw if the target archive already exists on cloud
/// storage. /// storage.
Future<void> publishArchive([bool forceUpload = false]) async { Future<void> publishArchive([bool forceUpload = false]) async {
for (final bool isNew in <bool>[false, true]) { final String destGsPath = '$gsReleaseFolder/$destinationArchivePath';
final String releaseFolder = isNew ? newGsReleaseFolder : oldGsReleaseFolder;
final String destGsPath = '$releaseFolder/$destinationArchivePath';
if (!forceUpload) { if (!forceUpload) {
if (await _cloudPathExists(destGsPath) && !dryRun) { if (await _cloudPathExists(destGsPath) && !dryRun) {
throw PreparePackageException( throw PreparePackageException(
@ -592,13 +587,11 @@ class ArchivePublisher {
dest: destGsPath, dest: destGsPath,
); );
assert(tempDir.existsSync()); assert(tempDir.existsSync());
await _updateMetadata('$releaseFolder/${getMetadataFilename(platform)}', newBucket: isNew); await _updateMetadata('$gsReleaseFolder/${getMetadataFilename(platform)}');
}
} }
Future<Map<String, dynamic>> _addRelease(Map<String, dynamic> jsonData, {bool newBucket=true}) async { Future<Map<String, dynamic>> _addRelease(Map<String, dynamic> jsonData) async {
final String tmpBaseUrl = newBucket ? newBaseUrl : oldBaseUrl; jsonData['base_url'] = '$baseUrl$releaseFolder';
jsonData['base_url'] = '$tmpBaseUrl$releaseFolder';
if (!jsonData.containsKey('current_release')) { if (!jsonData.containsKey('current_release')) {
jsonData['current_release'] = <String, String>{}; jsonData['current_release'] = <String, String>{};
} }
@ -630,7 +623,7 @@ class ArchivePublisher {
return jsonData; return jsonData;
} }
Future<void> _updateMetadata(String gsPath, {bool newBucket=true}) async { Future<void> _updateMetadata(String gsPath) async {
// We can't just cat the metadata from the server with 'gsutil cat', because // We can't just cat the metadata from the server with 'gsutil cat', because
// Windows wants to echo the commands that execute in gsutil.bat to the // Windows wants to echo the commands that execute in gsutil.bat to the
// stdout when we do that. So, we copy the file locally and then read it // stdout when we do that. So, we copy the file locally and then read it
@ -652,7 +645,7 @@ class ArchivePublisher {
throw PreparePackageException('Unable to parse JSON metadata received from cloud: $e'); throw PreparePackageException('Unable to parse JSON metadata received from cloud: $e');
} }
jsonData = await _addRelease(jsonData, newBucket: newBucket); jsonData = await _addRelease(jsonData);
const JsonEncoder encoder = JsonEncoder.withIndent(' '); const JsonEncoder encoder = JsonEncoder.withIndent(' ');
metadataFile.writeAsStringSync(encoder.convert(jsonData)); metadataFile.writeAsStringSync(encoder.convert(jsonData));
@ -776,7 +769,7 @@ Future<void> main(List<String> rawArguments) async {
defaultsTo: false, defaultsTo: false,
help: 'If set, will publish the archive to Google Cloud Storage upon ' help: 'If set, will publish the archive to Google Cloud Storage upon '
'successful creation of the archive. Will publish under this ' 'successful creation of the archive. Will publish under this '
'directory: $newBaseUrl$releaseFolder', 'directory: $baseUrl$releaseFolder',
); );
argParser.addFlag( argParser.addFlag(
'force', 'force',

35
dev/bots/test/prepare_package_test.dart
View File

@ -294,8 +294,7 @@ void main() {
final String releasesName = 'releases_$platformName.json'; final String releasesName = 'releases_$platformName.json';
final String archiveName = platform.isLinux ? 'archive.tar.xz' : 'archive.zip'; final String archiveName = platform.isLinux ? 'archive.tar.xz' : 'archive.zip';
final String archiveMime = platform.isLinux ? 'application/x-gtar' : 'application/zip'; final String archiveMime = platform.isLinux ? 'application/x-gtar' : 'application/zip';
final String gsArchivePath = 'gs://flutter_infra/releases/stable/$platformName/$archiveName'; final String gsArchivePath = 'gs://flutter_infra_release/releases/stable/$platformName/$archiveName';
final String newGsArchivePath = 'gs://flutter_infra_release/releases/stable/$platformName/$archiveName';
setUp(() async { setUp(() async {
processManager = FakeProcessManager.list(<FakeCommand>[]); processManager = FakeProcessManager.list(<FakeCommand>[]);
@ -309,11 +308,10 @@ void main() {
test('calls the right processes', () async { test('calls the right processes', () async {
final String archivePath = path.join(tempDir.absolute.path, archiveName); final String archivePath = path.join(tempDir.absolute.path, archiveName);
final String jsonPath = path.join(tempDir.absolute.path, releasesName); final String jsonPath = path.join(tempDir.absolute.path, releasesName);
final String gsJsonPath = 'gs://flutter_infra/releases/$releasesName'; final String gsJsonPath = 'gs://flutter_infra_release/releases/$releasesName';
final String newGsJsonPath = 'gs://flutter_infra_release/releases/$releasesName';
final String releasesJson = ''' final String releasesJson = '''
{ {
"base_url": "https://storage.googleapis.com/flutter_infra/releases", "base_url": "https://storage.googleapis.com/flutter_infra_release/releases",
"current_release": { "current_release": {
"beta": "3ea4d06340a97a1e9d7cae97567c64e0569dcaa2", "beta": "3ea4d06340a97a1e9d7cae97567c64e0569dcaa2",
"dev": "5a58b36e36b8d7aace89d3950e6deb307956a6a0" "dev": "5a58b36e36b8d7aace89d3950e6deb307956a6a0"
@ -356,12 +354,6 @@ void main() {
'$gsutilCall -- cp $gsJsonPath $jsonPath': null, '$gsutilCall -- cp $gsJsonPath $jsonPath': null,
'$gsutilCall -- rm $gsJsonPath': null, '$gsutilCall -- rm $gsJsonPath': null,
'$gsutilCall -- -h Content-Type:application/json -h Cache-Control:max-age=60 cp $jsonPath $gsJsonPath': null, '$gsutilCall -- -h Content-Type:application/json -h Cache-Control:max-age=60 cp $jsonPath $gsJsonPath': null,
'$gsutilCall -- stat $newGsArchivePath': <ProcessResult>[ProcessResult(0, 1, '', '')],
'$gsutilCall -- rm $newGsArchivePath': null,
'$gsutilCall -- -h Content-Type:$archiveMime cp $archivePath $newGsArchivePath': null,
'$gsutilCall -- cp $newGsJsonPath $jsonPath': null,
'$gsutilCall -- rm $newGsJsonPath': null,
'$gsutilCall -- -h Content-Type:application/json -h Cache-Control:max-age=60 cp $jsonPath $newGsJsonPath': null,
}; };
processManager.addCommands(convertResults(calls)); processManager.addCommands(convertResults(calls));
final File outputFile = File(path.join(tempDir.absolute.path, archiveName)); final File outputFile = File(path.join(tempDir.absolute.path, archiveName));
@ -412,11 +404,10 @@ void main() {
test('updates base_url from old bucket to new bucket', () async { test('updates base_url from old bucket to new bucket', () async {
final String archivePath = path.join(tempDir.absolute.path, archiveName); final String archivePath = path.join(tempDir.absolute.path, archiveName);
final String jsonPath = path.join(tempDir.absolute.path, releasesName); final String jsonPath = path.join(tempDir.absolute.path, releasesName);
final String gsJsonPath = 'gs://flutter_infra/releases/$releasesName'; final String gsJsonPath = 'gs://flutter_infra_release/releases/$releasesName';
final String newGsJsonPath = 'gs://flutter_infra_release/releases/$releasesName';
final String releasesJson = ''' final String releasesJson = '''
{ {
"base_url": "https://storage.googleapis.com/flutter_infra/releases", "base_url": "https://storage.googleapis.com/flutter_infra_release/releases",
"current_release": { "current_release": {
"beta": "3ea4d06340a97a1e9d7cae97567c64e0569dcaa2", "beta": "3ea4d06340a97a1e9d7cae97567c64e0569dcaa2",
"dev": "5a58b36e36b8d7aace89d3950e6deb307956a6a0" "dev": "5a58b36e36b8d7aace89d3950e6deb307956a6a0"
@ -459,12 +450,6 @@ void main() {
'$gsutilCall -- cp $gsJsonPath $jsonPath': null, '$gsutilCall -- cp $gsJsonPath $jsonPath': null,
'$gsutilCall -- rm $gsJsonPath': null, '$gsutilCall -- rm $gsJsonPath': null,
'$gsutilCall -- -h Content-Type:application/json -h Cache-Control:max-age=60 cp $jsonPath $gsJsonPath': null, '$gsutilCall -- -h Content-Type:application/json -h Cache-Control:max-age=60 cp $jsonPath $gsJsonPath': null,
'$gsutilCall -- stat $newGsArchivePath': <ProcessResult>[ProcessResult(0, 1, '', '')],
'$gsutilCall -- rm $newGsArchivePath': null,
'$gsutilCall -- -h Content-Type:$archiveMime cp $archivePath $newGsArchivePath': null,
'$gsutilCall -- cp $newGsJsonPath $jsonPath': null,
'$gsutilCall -- rm $newGsJsonPath': null,
'$gsutilCall -- -h Content-Type:application/json -h Cache-Control:max-age=60 cp $jsonPath $newGsJsonPath': null,
}; };
processManager.addCommands(convertResults(calls)); processManager.addCommands(convertResults(calls));
final File outputFile = File(path.join(tempDir.absolute.path, archiveName)); final File outputFile = File(path.join(tempDir.absolute.path, archiveName));
@ -529,11 +514,10 @@ void main() {
); );
final String archivePath = path.join(tempDir.absolute.path, archiveName); final String archivePath = path.join(tempDir.absolute.path, archiveName);
final String jsonPath = path.join(tempDir.absolute.path, releasesName); final String jsonPath = path.join(tempDir.absolute.path, releasesName);
final String gsJsonPath = 'gs://flutter_infra/releases/$releasesName'; final String gsJsonPath = 'gs://flutter_infra_release/releases/$releasesName';
final String newGsJsonPath = 'gs://flutter_infra_release/releases/$releasesName';
final String releasesJson = ''' final String releasesJson = '''
{ {
"base_url": "https://storage.googleapis.com/flutter_infra/releases", "base_url": "https://storage.googleapis.com/flutter_infra_release/releases",
"current_release": { "current_release": {
"beta": "3ea4d06340a97a1e9d7cae97567c64e0569dcaa2", "beta": "3ea4d06340a97a1e9d7cae97567c64e0569dcaa2",
"dev": "5a58b36e36b8d7aace89d3950e6deb307956a6a0" "dev": "5a58b36e36b8d7aace89d3950e6deb307956a6a0"
@ -574,11 +558,6 @@ void main() {
'$gsutilCall -- cp $gsJsonPath $jsonPath': null, '$gsutilCall -- cp $gsJsonPath $jsonPath': null,
'$gsutilCall -- rm $gsJsonPath': null, '$gsutilCall -- rm $gsJsonPath': null,
'$gsutilCall -- -h Content-Type:application/json -h Cache-Control:max-age=60 cp $jsonPath $gsJsonPath': null, '$gsutilCall -- -h Content-Type:application/json -h Cache-Control:max-age=60 cp $jsonPath $gsJsonPath': null,
'$gsutilCall -- rm $newGsArchivePath': null,
'$gsutilCall -- -h Content-Type:$archiveMime cp $archivePath $newGsArchivePath': null,
'$gsutilCall -- cp $newGsJsonPath $jsonPath': null,
'$gsutilCall -- rm $newGsJsonPath': null,
'$gsutilCall -- -h Content-Type:application/json -h Cache-Control:max-age=60 cp $jsonPath $newGsJsonPath': null,
}; };
processManager.addCommands(convertResults(calls)); processManager.addCommands(convertResults(calls));
assert(tempDir.existsSync()); assert(tempDir.existsSync());