diff --git a/dev/bots/prepare_package.dart b/dev/bots/prepare_package.dart index 01a5625104..0b9a988e84 100644 --- a/dev/bots/prepare_package.dart +++ b/dev/bots/prepare_package.dart @@ -26,6 +26,7 @@ const String baseUrl = 'https://storage.googleapis.com/flutter_infra_release'; const int shortCacheSeconds = 60; const String frameworkVersionTag = 'frameworkVersionFromGit'; const String dartVersionTag = 'dartSdkVersion'; +const String dartTargetArchTag = 'dartTargetArch'; /// Exception class for when a process fails to run, so we can catch /// it and provide something more readable than a stack trace. @@ -191,32 +192,72 @@ class ArchiveCreator { /// /// If subprocessOutput is true, then output from processes invoked during /// archive creation is echoed to stderr and stdout. - ArchiveCreator( - this.tempDir, - this.outputDir, - this.revision, - this.branch, { - this.strict = true, + factory ArchiveCreator( + Directory tempDir, + Directory outputDir, + String revision, + Branch branch, { + bool strict = true, ProcessManager? processManager, bool subprocessOutput = true, - this.platform = const LocalPlatform(), + Platform platform = const LocalPlatform(), HttpReader? httpReader, - }) : assert(revision.length == 40), - flutterRoot = Directory(path.join(tempDir.path, 'flutter')), - httpReader = httpReader ?? http.readBytes, - _processRunner = ProcessRunner( - processManager: processManager, - subprocessOutput: subprocessOutput, - platform: platform, - ) { - _flutter = path.join( + }) { + final Directory flutterRoot = Directory(path.join(tempDir.path, 'flutter')); + final ProcessRunner processRunner = ProcessRunner( + processManager: processManager, + subprocessOutput: subprocessOutput, + platform: platform, + )..environment['PUB_CACHE'] = path.join( + flutterRoot.absolute.path, '.pub-cache', + ); + final String flutterExecutable = path.join( flutterRoot.absolute.path, 'bin', 'flutter', ); - _processRunner.environment['PUB_CACHE'] = path.join(flutterRoot.absolute.path, '.pub-cache'); + final String dartExecutable = path.join( + flutterRoot.absolute.path, + 'bin', + 'cache', + 'dart-sdk', + 'bin', + 'dart', + ); + + return ArchiveCreator._( + tempDir: tempDir, + platform: platform, + flutterRoot: flutterRoot, + outputDir: outputDir, + revision: revision, + branch: branch, + strict: strict, + processRunner: processRunner, + httpReader: httpReader ?? http.readBytes, + flutterExecutable: flutterExecutable, + dartExecutable: dartExecutable, + ); } + ArchiveCreator._({ + required this.tempDir, + required this.platform, + required this.flutterRoot, + required this.outputDir, + required this.revision, + required this.branch, + required this.strict, + required ProcessRunner processRunner, + required this.httpReader, + required String flutterExecutable, + required String dartExecutable, + }) : + assert(revision.length == 40), + _processRunner = processRunner, + _flutter = flutterExecutable, + _dart = dartExecutable; + /// The platform to use for the environment and determining which /// platform we're running on. final Platform platform; @@ -252,17 +293,25 @@ class ArchiveCreator { /// [http.readBytes]. final HttpReader httpReader; - late File _outputFile; final Map _version = {}; late String _flutter; + late String _dart; + + late final Future _dartArch = (() async { + // Parse 'arch' out of a string like '... "os_arch"\n'. + return (await _runDart(['--version'])) + .trim().split(' ').last.replaceAll('"', '').split('_')[1]; + })(); /// Get the name of the channel as a string. String get branchName => getBranchName(branch); /// Returns a default archive name when given a Git revision. /// Used when an output filename is not given. - String get _archiveName { + Future get _archiveName async { final String os = platform.operatingSystem.toLowerCase(); + // Include the intended host archetecture in the file name for non-x64. + final String arch = await _dartArch == 'x64' ? '' : '${await _dartArch}_'; // We don't use .tar.xz on Mac because although it can unpack them // on the command line (with tar), the "Archive Utility" that runs // when you double-click on them just does some crazy behavior (it @@ -271,7 +320,7 @@ class ArchiveCreator { // unpacking it!) So, we use .zip for Mac, and the files are about // 220MB larger than they need to be. :-( final String suffix = platform.isLinux ? 'tar.xz' : 'zip'; - return 'flutter_${os}_${_version[frameworkVersionTag]}-$branchName.$suffix'; + return 'flutter_${os}_$arch${_version[frameworkVersionTag]}-$branchName.$suffix'; } /// Checks out the flutter repo and prepares it for other operations. @@ -289,12 +338,15 @@ class ArchiveCreator { /// Performs all of the steps needed to create an archive. Future createArchive() async { assert(_version.isNotEmpty, 'Must run initializeRepo before createArchive'); - _outputFile = File(path.join(outputDir.absolute.path, _archiveName)); + final File outputFile = File(path.join( + outputDir.absolute.path, + await _archiveName, + )); await _installMinGitIfNeeded(); await _populateCaches(); await _validate(); - await _archiveFiles(_outputFile); - return _outputFile; + await _archiveFiles(outputFile); + return outputFile; } /// Validates the integrity of the release package. @@ -307,14 +359,6 @@ class ArchiveCreator { return; } // Validate that the dart binary is codesigned - final String dartPath = path.join( - flutterRoot.absolute.path, - 'bin', - 'cache', - 'dart-sdk', - 'bin', - 'dart', - ); try { // TODO(fujino): Use the conductor https://github.com/flutter/flutter/issues/81701 await _processRunner.runProcess( @@ -322,13 +366,13 @@ class ArchiveCreator { 'codesign', '-vvvv', '--check-notarization', - dartPath, + _dart, ], workingDirectory: flutterRoot, ); } on PreparePackageException catch (e) { throw PreparePackageException( - 'The binary $dartPath was not codesigned!\n${e.message}', + 'The binary $_dart was not codesigned!\n${e.message}', ); } } @@ -370,6 +414,7 @@ class ArchiveCreator { final Map result = json.decode(versionJson) as Map; result.forEach((String key, dynamic value) => versionMap[key] = value.toString()); versionMap[frameworkVersionTag] = gitVersion; + versionMap[dartTargetArchTag] = await _dartArch; return versionMap; } @@ -464,6 +509,13 @@ class ArchiveCreator { } } + Future _runDart(List args, {Directory? workingDirectory}) { + return _processRunner.runProcess( + [_dart, ...args], + workingDirectory: workingDirectory ?? flutterRoot, + ); + } + Future _runFlutter(List args, {Directory? workingDirectory}) { return _processRunner.runProcess( [_flutter, ...args], @@ -623,6 +675,7 @@ class ArchivePublisher { newEntry['channel'] = branchName; newEntry['version'] = version[frameworkVersionTag]; newEntry['dart_sdk_version'] = version[dartVersionTag]; + newEntry['dart_sdk_arch'] = version[dartTargetArchTag]; newEntry['release_date'] = DateTime.now().toUtc().toIso8601String(); newEntry['archive'] = destinationArchivePath; newEntry['sha256'] = await _getChecksum(outputFile); @@ -849,8 +902,16 @@ Future main(List rawArguments) async { } } + final bool publish = parsedArguments['publish'] as bool; + final bool dryRun = parsedArguments['dry_run'] as bool; final Branch branch = fromBranchName(parsedArguments['branch'] as String); - final ArchiveCreator creator = ArchiveCreator(tempDir, outputDir, revision, branch, strict: parsedArguments['publish'] as bool); + final ArchiveCreator creator = ArchiveCreator( + tempDir, + outputDir, + revision, + branch, + strict: publish && !dryRun, + ); int exitCode = 0; late String message; try { @@ -863,7 +924,7 @@ Future main(List rawArguments) async { branch, version, outputFile, - parsedArguments['dry_run'] as bool, + dryRun, ); await publisher.publishArchive(parsedArguments['force'] as bool); } diff --git a/dev/bots/test/prepare_package_test.dart b/dev/bots/test/prepare_package_test.dart index 2c4e9cd512..3b3c3280a6 100644 --- a/dev/bots/test/prepare_package_test.dart +++ b/dev/bots/test/prepare_package_test.dart @@ -82,6 +82,7 @@ void main() { final List> args = >[]; final List> namedArgs = >[]; late String flutter; + late String dart; Future fakeHttpReader(Uri url, {Map? headers}) { return Future.value(Uint8List(0)); @@ -106,7 +107,10 @@ void main() { platform: platform, httpReader: fakeHttpReader, ); - flutter = path.join(creator.flutterRoot.absolute.path, 'bin', 'flutter'); + flutter = path.join(creator.flutterRoot.absolute.path, + 'bin', 'flutter'); + dart = path.join(creator.flutterRoot.absolute.path, + 'bin', 'cache', 'dart-sdk', 'bin', 'dart'); }); tearDown(() async { @@ -127,6 +131,9 @@ void main() { ProcessResult(0, 0, '{"dartSdkVersion": "3.2.1"}', ''), ProcessResult(0, 0, '{"dartSdkVersion": "3.2.1"}', ''), ], + '$dart --version': [ + ProcessResult(0, 0, 'Dart SDK version: 2.17.0-63.0.dev (dev) (Wed Jan 26 03:48:52 2022 -0800) on "${platformName}_x64"', ''), + ], if (platform.isWindows) '7za x ${path.join(tempDir.path, 'mingit.zip')}': null, '$flutter doctor': null, '$flutter update-packages': null, @@ -160,6 +167,56 @@ void main() { ProcessResult(0, 0, '{"dartSdkVersion": "3.2.1"}', ''), ProcessResult(0, 0, '{"dartSdkVersion": "3.2.1"}', ''), ], + '$dart --version': [ + ProcessResult(0, 0, 'Dart SDK version: 2.17.0-63.0.dev (dev) (Wed Jan 26 03:48:52 2022 -0800) on "${platformName}_x64"', ''), + ], + if (platform.isWindows) '7za x ${path.join(tempDir.path, 'mingit.zip')}': null, + '$flutter doctor': null, + '$flutter update-packages': null, + '$flutter precache': null, + '$flutter ide-config': null, + '$flutter create --template=app ${createBase}app': null, + '$flutter create --template=package ${createBase}package': null, + '$flutter create --template=plugin ${createBase}plugin': null, + 'git clean -f -x -- **/.packages': null, + 'git clean -f -x -- **/.dart_tool/': null, + if (platform.isMacOS) 'codesign -vvvv --check-notarization ${path.join(tempDir.path, 'flutter', 'bin', 'cache', 'dart-sdk', 'bin', 'dart')}': null, + if (platform.isWindows) 'attrib -h .git': null, + if (platform.isWindows) '7za a -tzip -mx=9 $archiveName flutter': null + else if (platform.isMacOS) 'zip -r -9 --symlinks $archiveName flutter': null + else if (platform.isLinux) 'tar cJf $archiveName flutter': null, + }; + processManager.addCommands(convertResults(calls)); + creator = ArchiveCreator( + tempDir, + tempDir, + testRef, + Branch.dev, + processManager: processManager, + subprocessOutput: false, + platform: platform, + httpReader: fakeHttpReader, + ); + await creator.initializeRepo(); + await creator.createArchive(); + }); + + test('adds the arch name to the archive for non-x64', () async { + final String createBase = path.join(tempDir.absolute.path, 'create_'); + final String archiveName = path.join(tempDir.absolute.path, + 'flutter_${platformName}_arm64_v1.2.3-dev${platform.isLinux ? '.tar.xz' : '.zip'}'); + final Map?> calls = ?>{ + 'git clone -b dev https://chromium.googlesource.com/external/github.com/flutter/flutter': null, + 'git reset --hard $testRef': null, + 'git remote set-url origin https://github.com/flutter/flutter.git': null, + 'git describe --tags --exact-match $testRef': [ProcessResult(0, 0, 'v1.2.3', '')], + '$flutter --version --machine': [ + ProcessResult(0, 0, '{"dartSdkVersion": "3.2.1"}', ''), + ProcessResult(0, 0, '{"dartSdkVersion": "3.2.1"}', ''), + ], + '$dart --version': [ + ProcessResult(0, 0, 'Dart SDK version: 2.17.0-63.0.dev (dev) (Wed Jan 26 03:48:52 2022 -0800) on "${platformName}_arm64"', ''), + ], if (platform.isWindows) '7za x ${path.join(tempDir.path, 'mingit.zip')}': null, '$flutter doctor': null, '$flutter update-packages': null, @@ -214,6 +271,9 @@ void main() { ProcessResult(0, 0, '{"dartSdkVersion": "3.2.1"}', ''), ProcessResult(0, 0, '{"dartSdkVersion": "3.2.1"}', ''), ], + '$dart --version': [ + ProcessResult(0, 0, 'Dart SDK version: 2.17.0-63.0.dev (dev) (Wed Jan 26 03:48:52 2022 -0800) on "${platformName}_x64"', ''), + ], if (platform.isWindows) '7za x ${path.join(tempDir.path, 'mingit.zip')}': null, '$flutter doctor': null, '$flutter update-packages': null, @@ -260,6 +320,9 @@ void main() { ProcessResult(0, 0, '{"dartSdkVersion": "3.2.1"}', ''), ProcessResult(0, 0, '{"dartSdkVersion": "3.2.1"}', ''), ], + '$dart --version': [ + ProcessResult(0, 0, 'Dart SDK version: 2.17.0-63.0.dev (dev) (Wed Jan 26 03:48:52 2022 -0800) on "${platformName}_x64"', ''), + ], if (platform.isWindows) '7za x ${path.join(tempDir.path, 'mingit.zip')}': null, '$flutter doctor': null, '$flutter update-packages': null, @@ -378,7 +441,11 @@ void main() { tempDir, testRef, Branch.stable, - {'frameworkVersionFromGit': 'v1.2.3', 'dartSdkVersion': '3.2.1'}, + { + 'frameworkVersionFromGit': 'v1.2.3', + 'dartSdkVersion': '3.2.1', + 'dartTargetArch': 'x64', + }, outputFile, false, processManager: processManager, @@ -416,6 +483,85 @@ void main() { expect(contents, equals(encoder.convert(jsonData))); }); + test('contains Dart SDK version info', () async { + final String archivePath = path.join(tempDir.absolute.path, archiveName); + final String jsonPath = path.join(tempDir.absolute.path, releasesName); + final String gsJsonPath = 'gs://flutter_infra_release/releases/$releasesName'; + final String releasesJson = ''' +{ + "base_url": "https://storage.googleapis.com/flutter_infra_release/releases", + "current_release": { + "beta": "3ea4d06340a97a1e9d7cae97567c64e0569dcaa2", + "dev": "5a58b36e36b8d7aace89d3950e6deb307956a6a0" + }, + "releases": [ + { + "hash": "5a58b36e36b8d7aace89d3950e6deb307956a6a0", + "channel": "dev", + "version": "v0.2.3", + "release_date": "2018-03-20T01:47:02.851729Z", + "archive": "dev/$platformName/flutter_${platformName}_v0.2.3-dev.zip", + "sha256": "4fe85a822093e81cb5a66c7fc263f68de39b5797b294191b6d75e7afcc86aff8" + }, + { + "hash": "b9bd51cc36b706215915711e580851901faebb40", + "channel": "beta", + "version": "v0.2.2", + "release_date": "2018-03-16T18:48:13.375013Z", + "archive": "dev/$platformName/flutter_${platformName}_v0.2.2-dev.zip", + "sha256": "6073331168cdb37a4637a5dc073d6a7ef4e466321effa2c529fa27d2253a4d4b" + }, + { + "hash": "$testRef", + "channel": "stable", + "version": "v0.0.0", + "release_date": "2018-03-20T01:47:02.851729Z", + "archive": "stable/$platformName/flutter_${platformName}_v0.0.0-dev.zip", + "sha256": "5dd34873b3a3e214a32fd30c2c319a0f46e608afb72f0d450b2d621a6d02aebd" + } + ] +} +'''; + File(jsonPath).writeAsStringSync(releasesJson); + File(archivePath).writeAsStringSync('archive contents'); + final Map?> calls = ?>{ + // This process fails because the file does NOT already exist + '$gsutilCall -- stat $gsArchivePath': [ProcessResult(0, 1, '', '')], + '$gsutilCall -- rm $gsArchivePath': null, + '$gsutilCall -- -h Content-Type:$archiveMime cp $archivePath $gsArchivePath': null, + '$gsutilCall -- cp $gsJsonPath $jsonPath': null, + '$gsutilCall -- rm $gsJsonPath': null, + '$gsutilCall -- -h Content-Type:application/json -h Cache-Control:max-age=60 cp $jsonPath $gsJsonPath': null, + }; + processManager.addCommands(convertResults(calls)); + final File outputFile = File(path.join(tempDir.absolute.path, archiveName)); + outputFile.createSync(); + assert(tempDir.existsSync()); + final ArchivePublisher publisher = ArchivePublisher( + tempDir, + testRef, + Branch.stable, + { + 'frameworkVersionFromGit': 'v1.2.3', + 'dartSdkVersion': '3.2.1', + 'dartTargetArch': 'x64', + }, + outputFile, + false, + processManager: processManager, + subprocessOutput: false, + platform: platform, + ); + assert(tempDir.existsSync()); + await publisher.publishArchive(); + + final File releaseFile = File(jsonPath); + expect(releaseFile.existsSync(), isTrue); + final String contents = releaseFile.readAsStringSync(); + expect(contents, contains('"dart_sdk_version": "3.2.1"')); + expect(contents, contains('"dart_sdk_arch": "x64"')); + }); + test('updates base_url from old bucket to new bucket', () async { final String archivePath = path.join(tempDir.absolute.path, archiveName); final String jsonPath = path.join(tempDir.absolute.path, releasesName); @@ -474,7 +620,11 @@ void main() { tempDir, testRef, Branch.stable, - {'frameworkVersionFromGit': 'v1.2.3', 'dartSdkVersion': '3.2.1'}, + { + 'frameworkVersionFromGit': 'v1.2.3', + 'dartSdkVersion': '3.2.1', + 'dartTargetArch': 'x64', + }, outputFile, false, processManager: processManager, @@ -498,7 +648,11 @@ void main() { tempDir, testRef, Branch.stable, - {'frameworkVersionFromGit': 'v1.2.3', 'dartSdkVersion': '3.2.1'}, + { + 'frameworkVersionFromGit': 'v1.2.3', + 'dartSdkVersion': '3.2.1', + 'dartTargetArch': 'x64', + }, outputFile, false, processManager: processManager, @@ -520,7 +674,11 @@ void main() { tempDir, testRef, Branch.stable, - {'frameworkVersionFromGit': 'v1.2.3', 'dartSdkVersion': '3.2.1'}, + { + 'frameworkVersionFromGit': 'v1.2.3', + 'dartSdkVersion': '3.2.1', + 'dartTargetArch': 'x64', + }, outputFile, false, processManager: processManager,