Ensure the packaging script does not overwrite a previous upload (#71597)
This commit is contained in:
Christopher Fujino
GitHub
parent
5f7f4f1aff
commit
94d574fa7f
@ -508,8 +508,18 @@ class ArchivePublisher {
|
||||
}
|
||||
|
||||
/// Publish the archive to Google Storage.
|
||||
Future<void> publishArchive() async {
|
||||
///
|
||||
/// This method will throw if the target archive already exists on cloud
|
||||
/// storage.
|
||||
Future<void> publishArchive([bool forceUpload = false]) async {
|
||||
final String destGsPath = '$gsReleaseFolder/$destinationArchivePath';
|
||||
if (!forceUpload) {
|
||||
if (await _cloudPathExists(destGsPath)) {
|
||||
throw PreparePackageException(
|
||||
'File $destGsPath already exists on cloud storage!',
|
||||
);
|
||||
}
|
||||
}
|
||||
await _cloudCopy(outputFile.absolute.path, destGsPath);
|
||||
assert(tempDir.existsSync());
|
||||
await _updateMetadata();
|
||||
@ -596,6 +606,20 @@ class ArchivePublisher {
|
||||
);
|
||||
}
|
||||
|
||||
/// Determine if a file exists at a given [cloudPath].
|
||||
Future<bool> _cloudPathExists(String cloudPath) async {
|
||||
try {
|
||||
await _runGsUtil(
|
||||
<String>['stat', cloudPath],
|
||||
failOk: false,
|
||||
);
|
||||
} on PreparePackageException {
|
||||
// `gsutil stat gs://path/to/file` will exit with 1 if file does not exist
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
Future<String> _cloudCopy(String src, String dest) async {
|
||||
// We often don't have permission to overwrite, but
|
||||
// we have permission to remove, so that's what we do.
|
||||
@ -664,6 +688,12 @@ Future<void> main(List<String> rawArguments) async {
|
||||
'successful creation of the archive. Will publish under this '
|
||||
'directory: $baseUrl$releaseFolder',
|
||||
);
|
||||
argParser.addFlag(
|
||||
'force',
|
||||
abbr: 'f',
|
||||
defaultsTo: false,
|
||||
help: 'Overwrite a previously uploaded package.',
|
||||
);
|
||||
argParser.addFlag(
|
||||
'help',
|
||||
defaultsTo: false,
|
||||
@ -685,14 +715,14 @@ Future<void> main(List<String> rawArguments) async {
|
||||
}
|
||||
|
||||
final String revision = parsedArguments['revision'] as String;
|
||||
if (revision.isEmpty) {
|
||||
if (!parsedArguments.wasParsed('revision')) {
|
||||
errorExit('Invalid argument: --revision must be specified.');
|
||||
}
|
||||
if (revision.length != 40) {
|
||||
errorExit('Invalid argument: --revision must be the entire hash, not just a prefix.');
|
||||
}
|
||||
|
||||
if ((parsedArguments['branch'] as String).isEmpty) {
|
||||
if (!parsedArguments.wasParsed('branch')) {
|
||||
errorExit('Invalid argument: --branch must be specified.');
|
||||
}
|
||||
|
||||
@ -734,7 +764,7 @@ Future<void> main(List<String> rawArguments) async {
|
||||
version,
|
||||
outputFile,
|
||||
);
|
||||
await publisher.publishArchive();
|
||||
await publisher.publishArchive(parsedArguments['force'] as bool);
|
||||
}
|
||||
} on PreparePackageException catch (e) {
|
||||
exitCode = e.exitCode;
|
||||
|
||||
@ -55,7 +55,6 @@ class FakeProcessManager extends Mock implements ProcessManager {
|
||||
|
||||
ProcessResult _popResult(List<String> command) {
|
||||
final String key = command.join(' ');
|
||||
expect(fakeResults, isNotEmpty);
|
||||
expect(fakeResults, contains(key));
|
||||
expect(fakeResults[key], isNotEmpty);
|
||||
return fakeResults[key].removeAt(0);
|
||||
|
||||
@ -234,6 +234,13 @@ void main() {
|
||||
group('ArchivePublisher for $platformName', () {
|
||||
FakeProcessManager processManager;
|
||||
Directory tempDir;
|
||||
final String gsutilCall = platform.isWindows
|
||||
? 'python ${path.join("D:", "depot_tools", "gsutil.py")}'
|
||||
: 'gsutil.py';
|
||||
final String releasesName = 'releases_$platformName.json';
|
||||
final String archiveName = platform.isLinux ? 'archive.tar.xz' : 'archive.zip';
|
||||
final String archiveMime = platform.isLinux ? 'application/x-gtar' : 'application/zip';
|
||||
final String gsArchivePath = 'gs://flutter_infra/releases/stable/$platformName/$archiveName';
|
||||
|
||||
setUp(() async {
|
||||
processManager = FakeProcessManager();
|
||||
@ -245,11 +252,7 @@ void main() {
|
||||
});
|
||||
|
||||
test('calls the right processes', () async {
|
||||
final String releasesName = 'releases_$platformName.json';
|
||||
final String archiveName = platform.isLinux ? 'archive.tar.xz' : 'archive.zip';
|
||||
final String archiveMime = platform.isLinux ? 'application/x-gtar' : 'application/zip';
|
||||
final String archivePath = path.join(tempDir.absolute.path, archiveName);
|
||||
final String gsArchivePath = 'gs://flutter_infra/releases/stable/$platformName/$archiveName';
|
||||
final String jsonPath = path.join(tempDir.absolute.path, releasesName);
|
||||
final String gsJsonPath = 'gs://flutter_infra/releases/$releasesName';
|
||||
final String releasesJson = '''
|
||||
@ -289,10 +292,9 @@ void main() {
|
||||
''';
|
||||
File(jsonPath).writeAsStringSync(releasesJson);
|
||||
File(archivePath).writeAsStringSync('archive contents');
|
||||
final String gsutilCall = platform.isWindows
|
||||
? 'python ${path.join("D:", "depot_tools", "gsutil.py")}'
|
||||
: 'gsutil.py';
|
||||
final Map<String, List<ProcessResult>> calls = <String, List<ProcessResult>>{
|
||||
// This process fails because the file does NOT already exist
|
||||
'$gsutilCall -- stat $gsArchivePath': <ProcessResult>[ProcessResult(0, 1, '', '')],
|
||||
'$gsutilCall -- rm $gsArchivePath': null,
|
||||
'$gsutilCall -- -h Content-Type:$archiveMime cp $archivePath $gsArchivePath': null,
|
||||
'$gsutilCall -- cp $gsJsonPath $jsonPath': null,
|
||||
@ -342,6 +344,94 @@ void main() {
|
||||
const JsonEncoder encoder = JsonEncoder.withIndent(' ');
|
||||
expect(contents, equals(encoder.convert(jsonData)));
|
||||
});
|
||||
|
||||
test('publishArchive throws if forceUpload is false and artifact already exists on cloud storage', () async {
|
||||
final String archiveName = platform.isLinux ? 'archive.tar.xz' : 'archive.zip';
|
||||
final File outputFile = File(path.join(tempDir.absolute.path, archiveName));
|
||||
final ArchivePublisher publisher = ArchivePublisher(
|
||||
tempDir,
|
||||
testRef,
|
||||
Branch.stable,
|
||||
'v1.2.3',
|
||||
outputFile,
|
||||
processManager: processManager,
|
||||
subprocessOutput: false,
|
||||
platform: platform,
|
||||
);
|
||||
final Map<String, List<ProcessResult>> calls = <String, List<ProcessResult>>{
|
||||
// This process returns 0 because file already exists
|
||||
'$gsutilCall -- stat $gsArchivePath': <ProcessResult>[ProcessResult(0, 0, '', '')],
|
||||
};
|
||||
processManager.fakeResults = calls;
|
||||
expect(() async => await publisher.publishArchive(false), throwsException);
|
||||
processManager.verifyCalls(calls.keys.toList());
|
||||
});
|
||||
|
||||
test('publishArchive does not throw if forceUpload is true and artifact already exists on cloud storage', () async {
|
||||
final String archiveName = platform.isLinux ? 'archive.tar.xz' : 'archive.zip';
|
||||
final File outputFile = File(path.join(tempDir.absolute.path, archiveName));
|
||||
final ArchivePublisher publisher = ArchivePublisher(
|
||||
tempDir,
|
||||
testRef,
|
||||
Branch.stable,
|
||||
'v1.2.3',
|
||||
outputFile,
|
||||
processManager: processManager,
|
||||
subprocessOutput: false,
|
||||
platform: platform,
|
||||
);
|
||||
final String archivePath = path.join(tempDir.absolute.path, archiveName);
|
||||
final String jsonPath = path.join(tempDir.absolute.path, releasesName);
|
||||
final String gsJsonPath = 'gs://flutter_infra/releases/$releasesName';
|
||||
final String releasesJson = '''
|
||||
{
|
||||
"base_url": "https://storage.googleapis.com/flutter_infra/releases",
|
||||
"current_release": {
|
||||
"beta": "3ea4d06340a97a1e9d7cae97567c64e0569dcaa2",
|
||||
"dev": "5a58b36e36b8d7aace89d3950e6deb307956a6a0"
|
||||
},
|
||||
"releases": [
|
||||
{
|
||||
"hash": "5a58b36e36b8d7aace89d3950e6deb307956a6a0",
|
||||
"channel": "dev",
|
||||
"version": "v0.2.3",
|
||||
"release_date": "2018-03-20T01:47:02.851729Z",
|
||||
"archive": "dev/$platformName/flutter_${platformName}_v0.2.3-dev.zip",
|
||||
"sha256": "4fe85a822093e81cb5a66c7fc263f68de39b5797b294191b6d75e7afcc86aff8"
|
||||
},
|
||||
{
|
||||
"hash": "b9bd51cc36b706215915711e580851901faebb40",
|
||||
"channel": "beta",
|
||||
"version": "v0.2.2",
|
||||
"release_date": "2018-03-16T18:48:13.375013Z",
|
||||
"archive": "dev/$platformName/flutter_${platformName}_v0.2.2-dev.zip",
|
||||
"sha256": "6073331168cdb37a4637a5dc073d6a7ef4e466321effa2c529fa27d2253a4d4b"
|
||||
},
|
||||
{
|
||||
"hash": "$testRef",
|
||||
"channel": "stable",
|
||||
"version": "v0.0.0",
|
||||
"release_date": "2018-03-20T01:47:02.851729Z",
|
||||
"archive": "stable/$platformName/flutter_${platformName}_v0.0.0-dev.zip",
|
||||
"sha256": "5dd34873b3a3e214a32fd30c2c319a0f46e608afb72f0d450b2d621a6d02aebd"
|
||||
}
|
||||
]
|
||||
}
|
||||
''';
|
||||
File(jsonPath).writeAsStringSync(releasesJson);
|
||||
File(archivePath).writeAsStringSync('archive contents');
|
||||
final Map<String, List<ProcessResult>> calls = <String, List<ProcessResult>>{
|
||||
'$gsutilCall -- rm $gsArchivePath': null,
|
||||
'$gsutilCall -- -h Content-Type:$archiveMime cp $archivePath $gsArchivePath': null,
|
||||
'$gsutilCall -- cp $gsJsonPath $jsonPath': null,
|
||||
'$gsutilCall -- rm $gsJsonPath': null,
|
||||
'$gsutilCall -- -h Content-Type:application/json cp $jsonPath $gsJsonPath': null,
|
||||
};
|
||||
processManager.fakeResults = calls;
|
||||
assert(tempDir.existsSync());
|
||||
await publisher.publishArchive(true);
|
||||
processManager.verifyCalls(calls.keys.toList());
|
||||
});
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user