diff --git a/dev/bots/docs.sh b/dev/bots/docs.sh
index 6a0cec83c9..a213982482 100755
--- a/dev/bots/docs.sh
+++ b/dev/bots/docs.sh
@@ -16,102 +16,13 @@ function script_location() {
cd -P "$(dirname "$script_location")" >/dev/null && pwd
}
-function generate_docs() {
- # Install and activate dartdoc.
- # When updating to a new dartdoc version, please also update
- # `dartdoc_options.yaml` to include newly introduced error and warning types.
- "$DART" pub global activate dartdoc 6.3.0
-
- # Install and activate the snippets tool, which resides in the
- # assets-for-api-docs repo:
- # https://github.com/flutter/assets-for-api-docs/tree/master/packages/snippets
- "$DART" pub global activate snippets 0.3.1
-
- # This script generates a unified doc set, and creates
- # a custom index.html, placing everything into dev/docs/doc.
- (cd "$FLUTTER_ROOT/dev/tools" && "$FLUTTER" pub get)
- (cd "$FLUTTER_ROOT/dev/tools" && "$DART" pub get)
- (cd "$FLUTTER_ROOT" && "$DART" --disable-dart-dev --enable-asserts "$FLUTTER_ROOT/dev/tools/dartdoc.dart")
- (cd "$FLUTTER_ROOT" && "$DART" --disable-dart-dev --enable-asserts "$FLUTTER_ROOT/dev/tools/java_and_objc_doc.dart")
-}
-
-# Zip up the docs so people can download them for offline usage.
-function create_offline_zip() {
- # Must be run from "$FLUTTER_ROOT/dev/docs"
- echo "$(date): Zipping Flutter offline docs archive."
- rm -rf flutter.docs.zip doc/offline
- (cd ./doc; zip -r -9 -q ../flutter.docs.zip .)
-}
-
-# Generate the docset for Flutter docs for use with Dash, Zeal, and Velocity.
-function create_docset() {
- # Must be run from "$FLUTTER_ROOT/dev/docs"
- # Must have dashing installed: go get -u github.com/technosophos/dashing
- # Dashing produces a LOT of log output (~30MB), so we redirect it, and just
- # show the end of it if there was a problem.
- echo "$(date): Building Flutter docset."
- rm -rf flutter.docset
- # If dashing gets stuck, Cirrus will time out the build after an hour, and we
- # never get to see the logs. Thus, we run it in the background and tail the logs
- # while we wait for it to complete.
- dashing_log=/tmp/dashing.log
- dashing build --source ./doc --config ./dashing.json > $dashing_log 2>&1 &
- dashing_pid=$!
- wait $dashing_pid && \
- cp ./doc/flutter/static-assets/favicon.png ./flutter.docset/icon.png && \
- "$DART" --disable-dart-dev --enable-asserts ./dashing_postprocess.dart && \
- tar cf flutter.docset.tar.gz --use-compress-program="gzip --best" flutter.docset
- if [[ $? -ne 0 ]]; then
- >&2 echo "Dashing docset generation failed"
- tail -200 $dashing_log
- exit 1
- fi
-}
-
-function deploy_docs() {
- case "$LUCI_BRANCH" in
- master)
- echo "$(date): Updating $LUCI_BRANCH docs: https://master-api.flutter.dev/"
- # Disable search indexing on the master staging site so searches get only
- # the stable site.
- echo -e "User-agent: *\nDisallow: /" > "$FLUTTER_ROOT/dev/docs/doc/robots.txt"
- ;;
- stable)
- echo "$(date): Updating $LUCI_BRANCH docs: https://api.flutter.dev/"
- # Enable search indexing on the master staging site so searches get only
- # the stable site.
- echo -e "# All robots welcome!" > "$FLUTTER_ROOT/dev/docs/doc/robots.txt"
- ;;
- *)
- >&2 echo "Docs deployment cannot be run on the $LUCI_BRANCH branch."
- exit 0
- esac
-}
-
-# Move the offline archives into place, after all the processing of the doc
-# directory is done. This avoids the tools recursively processing the archives
-# as part of their process.
-function move_offline_into_place() {
- # Must be run from "$FLUTTER_ROOT/dev/docs"
- echo "$(date): Moving offline data into place."
- mkdir -p doc/offline
- mv flutter.docs.zip doc/offline/flutter.docs.zip
- du -sh doc/offline/flutter.docs.zip
- if [[ "$LUCI_BRANCH" == "stable" ]]; then
- echo -e "\n ${FLUTTER_VERSION_STRING}\n https://api.flutter.dev/offline/flutter.docset.tar.gz\n" > doc/offline/flutter.xml
- else
- echo -e "\n ${FLUTTER_VERSION_STRING}\n https://master-api.flutter.dev/offline/flutter.docset.tar.gz\n" > doc/offline/flutter.xml
- fi
- mv flutter.docset.tar.gz doc/offline/flutter.docset.tar.gz
- du -sh doc/offline/flutter.docset.tar.gz
-}
-
# So that users can run this script from anywhere and it will work as expected.
SCRIPT_LOCATION="$(script_location)"
# Sets the Flutter root to be "$(script_location)/../..": This script assumes
# that it resides two directory levels down from the root, so if that changes,
# then this line will need to as well.
FLUTTER_ROOT="$(dirname "$(dirname "$SCRIPT_LOCATION")")"
+export FLUTTER_ROOT
echo "$(date): Running docs.sh"
@@ -124,31 +35,115 @@ FLUTTER_BIN="$FLUTTER_ROOT/bin"
DART_BIN="$FLUTTER_ROOT/bin/cache/dart-sdk/bin"
FLUTTER="$FLUTTER_BIN/flutter"
DART="$DART_BIN/dart"
-export PATH="$FLUTTER_BIN:$DART_BIN:$PATH"
+PATH="$FLUTTER_BIN:$DART_BIN:$PATH"
-# Make sure dart is installed by invoking Flutter to download it.
-# This also creates the 'version' file.
-FLUTTER_VERSION=$("$FLUTTER" --version --machine)
+# Make sure dart is installed by invoking Flutter to download it if it is missing.
+# Also make sure the flutter command is ready to run before capturing output from
+# it: if it has to rebuild itself or something, it'll spoil our JSON output.
+"$FLUTTER" > /dev/null 2>&1
+FLUTTER_VERSION="$("$FLUTTER" --version --machine)"
export FLUTTER_VERSION
-FLUTTER_VERSION_STRING=$(cat "$FLUTTER_ROOT/version")
# If the pub cache directory exists in the root, then use that.
FLUTTER_PUB_CACHE="$FLUTTER_ROOT/.pub-cache"
if [[ -d "$FLUTTER_PUB_CACHE" ]]; then
# This has to be exported, because pub interprets setting it to the empty
# string in the same way as setting it to ".".
- export PUB_CACHE="${PUB_CACHE:-"$FLUTTER_PUB_CACHE"}"
+ PUB_CACHE="${PUB_CACHE:-"$FLUTTER_PUB_CACHE"}"
+ export PUB_CACHE
fi
-generate_docs
-# Skip publishing docs for PRs and release candidate branches
-if [[ -n "$LUCI_CI" && -z "$LUCI_PR" ]]; then
- (cd "$FLUTTER_ROOT/dev/docs"; create_offline_zip)
- (cd "$FLUTTER_ROOT/dev/docs"; create_docset)
- (cd "$FLUTTER_ROOT/dev/docs"; move_offline_into_place)
- deploy_docs
-fi
+function usage() {
+ echo "Usage: $(basename "${BASH_SOURCE[0]}") [--keep-temp] [--output ]"
+ echo ""
+ echo " --keep-staging Do not delete the staging directory created while generating"
+ echo " docs. Normally the script deletes the staging directory after"
+ echo " generating the output ZIP file."
+ echo " --output specifies where the output ZIP file containing the documentation"
+ echo " data will be written."
+ echo " --staging-dir specifies where the temporary output files will be written while"
+ echo " generating docs. This directory will be deleted after generation"
+ echo " unless --keep-staging is also specified."
+ echo ""
+}
-# Zip docs
-cd "$FLUTTER_ROOT/dev/docs"
-zip -r api_docs.zip doc
+function parse_args() {
+ local arg
+ local args=()
+ STAGING_DIR=
+ KEEP_STAGING=0
+ DESTINATION="$FLUTTER_ROOT/dev/docs/api_docs.zip"
+ while (( "$#" )); do
+ case "$1" in
+ --help)
+ usage
+ exit 0
+ ;;
+ --staging-dir)
+ STAGING_DIR="$2"
+ shift
+ ;;
+ --keep-staging)
+ KEEP_STAGING=1
+ ;;
+ --output)
+ DESTINATION="$2"
+ shift
+ ;;
+ *)
+ args=("${args[@]}" "$1")
+ ;;
+ esac
+ shift
+ done
+ if [[ -z $STAGING_DIR ]]; then
+ STAGING_DIR=$(mktemp -d /tmp/dartdoc.XXXXX)
+ fi
+ DOC_DIR="$STAGING_DIR/doc"
+ if [[ ${#args[@]} != 0 ]]; then
+ >&2 echo "ERROR: Unknown arguments: ${args[@]}"
+ usage
+ exit 1
+ fi
+}
+
+function generate_docs() {
+ # Install and activate dartdoc.
+ # When updating to a new dartdoc version, please also update
+ # `dartdoc_options.yaml` to include newly introduced error and warning types.
+ "$DART" pub global activate dartdoc 6.3.0
+
+ # Install and activate the snippets tool, which resides in the
+ # assets-for-api-docs repo:
+ # https://github.com/flutter/assets-for-api-docs/tree/master/packages/snippets
+ "$DART" pub global activate snippets 0.4.0
+
+ # This script generates a unified doc set, and creates
+ # a custom index.html, placing everything into DOC_DIR.
+
+ # Make sure that create_api_docs.dart has all the dependencies it needs.
+ (cd "$FLUTTER_ROOT/dev/tools" && "$FLUTTER" pub get)
+ (cd "$FLUTTER_ROOT" && "$DART" --disable-dart-dev --enable-asserts "$FLUTTER_ROOT/dev/tools/create_api_docs.dart" --output-dir="$DOC_DIR")
+}
+
+function main() {
+ echo "Writing docs build temporary output to $DOC_DIR"
+ mkdir -p "$DOC_DIR"
+ generate_docs
+ # If the destination isn't an absolute path, make it into one.
+ if ! [[ "$DESTINATION" =~ ^/ ]]; then
+ DESTINATION="$PWD/$DESTINATION"
+ fi
+ # Zip up doc directory and write the output to the destination.
+ (cd "$STAGING_DIR"; zip -r -9 -q "$DESTINATION" ./doc)
+ if [[ $KEEP_STAGING -eq 1 ]]; then
+ echo "Staging documentation output left in $STAGING_DIR"
+ else
+ echo "Removing staging documentation output from $STAGING_DIR"
+ rm -rf "$STAGING_DIR"
+ fi
+ echo "Wrote docs ZIP file to $DESTINATION"
+}
+
+parse_args "$@"
+main
diff --git a/dev/docs/dashing_postprocess.dart b/dev/docs/dashing_postprocess.dart
deleted file mode 100644
index 9f186e4f2c..0000000000
--- a/dev/docs/dashing_postprocess.dart
+++ /dev/null
@@ -1,29 +0,0 @@
-// Copyright 2014 The Flutter Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file.
-
-import 'dart:io';
-
-/// This changes the DocSetPlatformFamily key to be "dartlang" instead of the
-/// name of the package (usually "flutter").
-///
-/// This is so that the IntelliJ plugin for Dash will be able to go directly to
-/// the docs for a symbol from a keystroke. Without this, flutter isn't part
-/// of the list of package names it searches. After this, it finds the flutter
-/// docs because they're declared here to be part of the "dartlang" family of
-/// docs.
-///
-/// Dashing doesn't have a way to configure this, so we modify the Info.plist
-/// directly to make the change.
-void main(List args) {
- final File infoPlist = File('flutter.docset/Contents/Info.plist');
- String contents = infoPlist.readAsStringSync();
-
- // Since I didn't want to add the XML package as a dependency just for this,
- // I just used a regular expression to make this simple change.
- final RegExp findRe = RegExp(r'(\s*DocSetPlatformFamily\s*)[^<]+()', multiLine: true);
- contents = contents.replaceAllMapped(findRe, (Match match) {
- return '${match.group(1)}dartlang${match.group(2)}';
- });
- infoPlist.writeAsStringSync(contents);
-}
diff --git a/dev/tools/create_api_docs.dart b/dev/tools/create_api_docs.dart
new file mode 100644
index 0000000000..4cbef2287f
--- /dev/null
+++ b/dev/tools/create_api_docs.dart
@@ -0,0 +1,1141 @@
+// Copyright 2014 The Flutter Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+import 'dart:convert';
+import 'dart:io';
+import 'dart:math' as math;
+
+import 'package:archive/archive_io.dart';
+import 'package:args/args.dart';
+import 'package:file/file.dart';
+import 'package:file/local.dart';
+import 'package:http/http.dart' as http;
+import 'package:intl/intl.dart';
+import 'package:meta/meta.dart';
+import 'package:path/path.dart' as path;
+import 'package:platform/platform.dart';
+import 'package:process/process.dart';
+import 'package:pub_semver/pub_semver.dart';
+
+import 'dartdoc_checker.dart';
+
+const String kDummyPackageName = 'Flutter';
+const String kPlatformIntegrationPackageName = 'platform_integration';
+
+/// This script will generate documentation for the packages in `packages/` and
+/// write the documentation to the output directory specified on the command
+/// line.
+///
+/// This script also updates the index.html file so that it can be placed at the
+/// root of api.flutter.dev. The files are kept inside of
+/// api.flutter.dev/flutter, so we need to manipulate paths a bit. See
+/// https://github.com/flutter/flutter/issues/3900 for more info.
+///
+/// This will only work on UNIX systems, not Windows. It requires that 'git',
+/// 'zip', and 'tar' be in the PATH. It requires that 'flutter' has been run
+/// previously. It uses the version of Dart downloaded by the 'flutter' tool in
+/// this repository and will fail if that is absent.
+Future main(List arguments) async {
+ const FileSystem filesystem = LocalFileSystem();
+ const ProcessManager processManager = LocalProcessManager();
+ const Platform platform = LocalPlatform();
+
+ // The place to find customization files and configuration files for docs
+ // generation.
+ final Directory docsRoot = filesystem
+ .directory(FlutterInformation.instance.getFlutterRoot().childDirectory('dev').childDirectory('docs'))
+ .absolute;
+ final ArgParser argParser = _createArgsParser(
+ publishDefault: docsRoot.childDirectory('doc').path,
+ );
+ final ArgResults args = argParser.parse(arguments);
+ if (args['help'] as bool) {
+ print('Usage:');
+ print(argParser.usage);
+ exit(0);
+ }
+
+ final Directory publishRoot = filesystem.directory(args['output-dir']! as String).absolute;
+ final Directory packageRoot = publishRoot.parent;
+ if (!filesystem.directory(packageRoot).existsSync()) {
+ filesystem.directory(packageRoot).createSync(recursive: true);
+ }
+
+ if (!filesystem.directory(publishRoot).existsSync()) {
+ filesystem.directory(publishRoot).createSync(recursive: true);
+ }
+
+ final Configurator configurator = Configurator(
+ publishRoot: publishRoot,
+ packageRoot: packageRoot,
+ docsRoot: docsRoot,
+ filesystem: filesystem,
+ processManager: processManager,
+ platform: platform,
+ );
+ configurator.generateConfiguration();
+
+ final PlatformDocGenerator platformGenerator = PlatformDocGenerator(outputDir: publishRoot, filesystem: filesystem);
+ platformGenerator.generatePlatformDocs();
+
+ final DartdocGenerator dartdocGenerator = DartdocGenerator(
+ publishRoot: publishRoot,
+ packageRoot: packageRoot,
+ docsRoot: docsRoot,
+ filesystem: filesystem,
+ processManager: processManager,
+ useJson: args['json'] as bool? ?? true,
+ validateLinks: args['validate-links']! as bool,
+ verbose: args['verbose'] as bool? ?? false,
+ );
+
+ await dartdocGenerator.generateDartdoc();
+ await configurator.generateOfflineAssetsIfNeeded();
+}
+
+ArgParser _createArgsParser({required String publishDefault}) {
+ final ArgParser parser = ArgParser();
+ parser.addFlag('help', abbr: 'h', negatable: false, help: 'Show command help.');
+ parser.addFlag('verbose',
+ defaultsTo: true,
+ help: 'Whether to report all error messages (on) or attempt to '
+ 'filter out some known false positives (off). Shut this off '
+ 'locally if you want to address Flutter-specific issues.');
+ parser.addFlag('json', help: 'Display json-formatted output from dartdoc and skip stdout/stderr prefixing.');
+ parser.addFlag('validate-links', help: 'Display warnings for broken links generated by dartdoc (slow)');
+ parser.addOption('output-dir', defaultsTo: publishDefault, help: 'Sets the output directory for the documentation.');
+ return parser;
+}
+
+/// A class used to configure the staging area for building the docs in.
+///
+/// The [generateConfiguration] function generates a dummy package with a
+/// pubspec. It copies any assets and customization files from the framework
+/// repo. It creates a metadata file for searches.
+///
+/// Once the docs have been generated, [generateOfflineAssetsIfNeeded] will
+/// create offline assets like Dash/Zeal docsets and an offline ZIP file of the
+/// site if the build is a CI build that is not a presubmit build.
+class Configurator {
+ Configurator({
+ required this.docsRoot,
+ required this.publishRoot,
+ required this.packageRoot,
+ required this.filesystem,
+ required this.processManager,
+ required this.platform,
+ });
+
+ /// The root of the directory in the Flutter repo where configuration data is
+ /// stored.
+ final Directory docsRoot;
+
+ /// The root of the output area for the dartdoc docs.
+ ///
+ /// Typically this is a "doc" subdirectory under the [packageRoot].
+ final Directory publishRoot;
+
+ /// The root of the staging area for creating docs.
+ final Directory packageRoot;
+
+ /// The [FileSystem] object used to create [File] and [Directory] objects.
+ final FileSystem filesystem;
+
+ /// The [ProcessManager] object used to invoke external processes.
+ ///
+ /// Can be replaced by tests to have a fake process manager.
+ final ProcessManager processManager;
+
+ /// The [Platform] to use for this run.
+ ///
+ /// Can be replaced by tests to test behavior on different plaforms.
+ final Platform platform;
+
+ void generateConfiguration() {
+ final Version version = FlutterInformation.instance.getFlutterVersion();
+ _createDummyPubspec();
+ _createDummyLibrary();
+ _createPageFooter(packageRoot, version);
+ _copyCustomizations();
+ _createSearchMetadata(
+ docsRoot.childDirectory('lib').childFile('opensearch.xml'), publishRoot.childFile('opensearch.xml'));
+ }
+
+ Future generateOfflineAssetsIfNeeded() async {
+ // Only create the offline docs if we're running in a non-presubmit build:
+ // it takes too long otherwise.
+ if (platform.environment.containsKey('LUCI_CI') && (platform.environment['LUCI_PR'] ?? '').isEmpty) {
+ _createOfflineZipFile();
+ await _createDocset();
+ _moveOfflineIntoPlace();
+ _createRobotsTxt();
+ }
+ }
+
+ /// Returns import or on-disk paths for all libraries in the Flutter SDK.
+ Iterable _libraryRefs() sync* {
+ for (final Directory dir in findPackages(filesystem)) {
+ final String dirName = dir.basename;
+ for (final FileSystemEntity file in dir.childDirectory('lib').listSync()) {
+ if (file is File && file.path.endsWith('.dart')) {
+ yield '$dirName/${file.basename}';
+ }
+ }
+ }
+
+ // Add a fake package for platform integration APIs.
+ yield '$kPlatformIntegrationPackageName/android.dart';
+ yield '$kPlatformIntegrationPackageName/ios.dart';
+ }
+
+ void _createDummyPubspec() {
+ // Create the pubspec.yaml file.
+ final List pubspec = [
+ 'name: $kDummyPackageName',
+ 'homepage: https://flutter.dev',
+ 'version: 0.0.0',
+ 'environment:',
+ " sdk: '>=3.0.0-0 <4.0.0'",
+ 'dependencies:',
+ for (final String package in findPackageNames(filesystem)) ' $package:\n sdk: flutter',
+ ' $kPlatformIntegrationPackageName: 0.0.1',
+ 'dependency_overrides:',
+ ' $kPlatformIntegrationPackageName:',
+ ' path: ${docsRoot.childDirectory(kPlatformIntegrationPackageName).path}',
+ ];
+
+ packageRoot.childFile('pubspec.yaml').writeAsStringSync(pubspec.join('\n'));
+ }
+
+ void _createDummyLibrary() {
+ final Directory libDir = packageRoot.childDirectory('lib');
+ libDir.createSync();
+
+ final StringBuffer contents = StringBuffer('library temp_doc;\n\n');
+ for (final String libraryRef in _libraryRefs()) {
+ contents.writeln("import 'package:$libraryRef';");
+ }
+ packageRoot.childDirectory('lib')
+ ..createSync(recursive: true)
+ ..childFile('temp_doc.dart').writeAsStringSync(contents.toString());
+ }
+
+ void _createPageFooter(Directory footerPath, Version version) {
+ final String timestamp = DateFormat('yyyy-MM-dd HH:mm').format(DateTime.now());
+ final String gitBranch = FlutterInformation.instance.getBranchName();
+ final String gitRevision = FlutterInformation.instance.getFlutterRevision();
+ final String gitBranchOut = gitBranch.isEmpty ? '' : '• $gitBranch';
+ footerPath.childFile('footer.html').writeAsStringSync('');
+ publishRoot.childDirectory('flutter').childFile('footer.js')
+ ..createSync(recursive: true)
+ ..writeAsStringSync('''
+(function() {
+ var span = document.querySelector('footer>span');
+ if (span) {
+ span.innerText = 'Flutter $version • $timestamp • $gitRevision $gitBranchOut';
+ }
+ var sourceLink = document.querySelector('a.source-link');
+ if (sourceLink) {
+ sourceLink.href = sourceLink.href.replace('/master/', '/$gitRevision/');
+ }
+})();
+''');
+ }
+
+ void _copyCustomizations() {
+ final List files = [
+ 'README.md',
+ 'analysis_options.yaml',
+ 'dartdoc_options.yaml',
+ ];
+ for (final String file in files) {
+ final File source = docsRoot.childFile(file);
+ final File destination = packageRoot.childFile(file);
+ // Have to canonicalize because otherwise things like /foo/bar/baz and
+ // /foo/../foo/bar/baz won't compare as identical.
+ if (path.canonicalize(source.absolute.path) != path.canonicalize(destination.absolute.path)) {
+ source.copySync(destination.path);
+ print('Copied ${path.canonicalize(source.absolute.path)} to ${path.canonicalize(destination.absolute.path)}');
+ }
+ }
+ final Directory assetsDir = filesystem.directory(publishRoot.childDirectory('assets'));
+ final Directory assetSource = docsRoot.childDirectory('assets');
+ if (path.canonicalize(assetSource.absolute.path) == path.canonicalize(assetsDir.absolute.path)) {
+ // Don't try and copy the directory over itself.
+ return;
+ }
+ if (assetsDir.existsSync()) {
+ assetsDir.deleteSync(recursive: true);
+ }
+ copyDirectorySync(
+ docsRoot.childDirectory('assets'),
+ assetsDir,
+ onFileCopied: (File src, File dest) {
+ print('Copied ${path.canonicalize(src.absolute.path)} to ${path.canonicalize(dest.absolute.path)}');
+ },
+ filesystem: filesystem,
+ );
+ }
+
+ /// Generates an OpenSearch XML description that can be used to add a custom
+ /// search for Flutter API docs to the browser. Unfortunately, it has to know
+ /// the URL to which site to search, so we customize it here based upon the
+ /// branch name.
+ void _createSearchMetadata(File templatePath, File metadataPath) {
+ final String template = templatePath.readAsStringSync();
+ final String branch = FlutterInformation.instance.getBranchName();
+ final String metadata = template.replaceAll(
+ '{SITE_URL}',
+ branch == 'stable' ? 'https://api.flutter.dev/' : 'https://master-api.flutter.dev/',
+ );
+ metadataPath.parent.create(recursive: true);
+ metadataPath.writeAsStringSync(metadata);
+ }
+
+ Future _createDocset() async {
+ // Must have dashing installed: go get -u github.com/technosophos/dashing
+ // Dashing produces a LOT of log output (~30MB), so we collect it, and just
+ // show the end of it if there was a problem.
+ print('${DateTime.now().toUtc()}: Building Flutter docset.');
+
+ // If dashing gets stuck, Cirrus will time out the build after an hour, and we
+ // never get to see the logs. Thus, we run it in the background and tail the
+ // logs only if it fails.
+ final ProcessWrapper result = ProcessWrapper(
+ await processManager.start(
+ [
+ 'dashing',
+ 'build',
+ '--source',
+ publishRoot.path,
+ '--config',
+ docsRoot.childFile('dashing.json').path,
+ ],
+ workingDirectory: packageRoot.path,
+ ),
+ );
+ final List buffer = [];
+ result.stdout.listen(buffer.addAll);
+ result.stderr.listen(buffer.addAll);
+ // If the dashing process exited with an error, print the last 200 lines of stderr and exit.
+ final int exitCode = await result.done;
+ if (exitCode != 0) {
+ print('Dashing docset generation failed with code $exitCode');
+ final List output = systemEncoding.decode(buffer).split('\n');
+ print(output.sublist(math.max(output.length - 200, 0)).join('\n'));
+ exit(exitCode);
+ }
+ buffer.clear();
+
+ // Copy the favicon file to the output directory.
+ final File faviconFile =
+ publishRoot.childDirectory('flutter').childDirectory('static-assets').childFile('favicon.png');
+ final File iconFile = packageRoot.childDirectory('flutter.docset').childFile('icon.png');
+ faviconFile
+ ..createSync(recursive: true)
+ ..copySync(iconFile.path);
+
+ // Post-process the dashing output.
+ final File infoPlist =
+ packageRoot.childDirectory('flutter.docset').childDirectory('Contents').childFile('Info.plist');
+ String contents = infoPlist.readAsStringSync();
+
+ // Since I didn't want to add the XML package as a dependency just for this,
+ // I just used a regular expression to make this simple change.
+ final RegExp findRe = RegExp(r'(\s*DocSetPlatformFamily\s*)[^<]+()', multiLine: true);
+ contents = contents.replaceAllMapped(findRe, (Match match) {
+ return '${match.group(1)}dartlang${match.group(2)}';
+ });
+ infoPlist.writeAsStringSync(contents);
+ final Directory offlineDir = publishRoot.childDirectory('offline');
+ if (!offlineDir.existsSync()) {
+ offlineDir.createSync(recursive: true);
+ }
+ tarDirectory(packageRoot, offlineDir.childFile('flutter.docset.tar.gz'), processManager: processManager);
+
+ // Write the Dash/Zeal XML feed file.
+ final bool isStable = platform.environment['LUCI_BRANCH'] == 'stable';
+ offlineDir.childFile('flutter.xml').writeAsStringSync('\n'
+ ' ${FlutterInformation.instance.getFlutterVersion()}\n'
+ ' https://${isStable ? '' : 'master-'}api.flutter.dev/offline/flutter.docset.tar.gz\n'
+ '\n');
+ }
+
+ // Creates the offline ZIP file containing all of the website HTML files.
+ void _createOfflineZipFile() {
+ print('${DateTime.now().toLocal()}: Creating offline docs archive.');
+ zipDirectory(publishRoot, packageRoot.childFile('flutter.docs.zip'), processManager: processManager);
+ }
+
+ // Moves the generated offline archives into the publish directory so that
+ // they can be included in the output ZIP file.
+ void _moveOfflineIntoPlace() {
+ print('${DateTime.now().toUtc()}: Moving offline docs into place.');
+ final Directory offlineDir = publishRoot.childDirectory('offline')..createSync(recursive: true);
+ packageRoot.childFile('flutter.docs.zip').renameSync(offlineDir.childFile('flutter.docs.zip').path);
+ }
+
+ // Creates a robots.txt file that disallows indexing unless the branch is the
+ // stable branch.
+ void _createRobotsTxt() {
+ final File robotsTxt = publishRoot.childFile('robots.txt');
+ if (FlutterInformation.instance.getBranchName() == 'stable') {
+ robotsTxt.writeAsStringSync('# All robots welcome!');
+ } else {
+ robotsTxt.writeAsStringSync('User-agent: *\nDisallow: /');
+ }
+ }
+}
+
+/// Runs Dartdoc inside of the given pre-prepared staging area, prepared by
+/// [Configurator.generateConfiguration].
+///
+/// Performs a sanity check of the output once the generation is complete.
+class DartdocGenerator {
+ DartdocGenerator({
+ required this.docsRoot,
+ required this.publishRoot,
+ required this.packageRoot,
+ required this.filesystem,
+ required this.processManager,
+ this.useJson = true,
+ this.validateLinks = true,
+ this.verbose = false,
+ });
+
+ /// The root of the directory in the Flutter repo where configuration data is
+ /// stored.
+ final Directory docsRoot;
+
+ /// The root of the output area for the dartdoc docs.
+ ///
+ /// Typically this is a "doc" subdirectory under the [packageRoot].
+ final Directory publishRoot;
+
+ /// The root of the staging area for creating docs.
+ final Directory packageRoot;
+
+ /// The [FileSystem] object used to create [File] and [Directory] objects.
+ final FileSystem filesystem;
+
+ /// The [ProcessManager] object used to invoke external processes.
+ ///
+ /// Can be replaced by tests to have a fake process manager.
+ final ProcessManager processManager;
+
+ /// Whether or not dartdoc should output an index.json file of the
+ /// documentation.
+ final bool useJson;
+
+ // Whether or not to have dartdoc validate its own links.
+ final bool validateLinks;
+
+ /// Whether or not to filter overly verbose log output from dartdoc.
+ final bool verbose;
+
+ Future generateDartdoc() async {
+ final Directory flutterRoot = FlutterInformation.instance.getFlutterRoot();
+ final Map pubEnvironment = {
+ 'FLUTTER_ROOT': flutterRoot.absolute.path,
+ };
+
+ // If there's a .pub-cache dir in the Flutter root, use that.
+ final File pubCache = flutterRoot.childFile('.pub-cache');
+ if (pubCache.existsSync()) {
+ pubEnvironment['PUB_CACHE'] = pubCache.path;
+ }
+
+ // Run pub.
+ ProcessWrapper process = ProcessWrapper(await runPubProcess(
+ arguments: ['get'],
+ workingDirectory: packageRoot,
+ environment: pubEnvironment,
+ filesystem: filesystem,
+ processManager: processManager,
+ ));
+ printStream(process.stdout, prefix: 'pub:stdout: ');
+ printStream(process.stderr, prefix: 'pub:stderr: ');
+ final int code = await process.done;
+ if (code != 0) {
+ exit(code);
+ }
+
+ final Version version = FlutterInformation.instance.getFlutterVersion();
+
+ // Verify which version of snippets and dartdoc we're using.
+ final ProcessResult snippetsResult = Process.runSync(
+ FlutterInformation.instance.getDartBinaryPath().path,
+ [
+ 'pub',
+ 'global',
+ 'list',
+ ],
+ workingDirectory: packageRoot.path,
+ environment: pubEnvironment,
+ stdoutEncoding: utf8,
+ );
+ print('');
+ final Iterable versionMatches =
+ RegExp(r'^(?snippets|dartdoc) (?[^\s]+)', multiLine: true)
+ .allMatches(snippetsResult.stdout as String);
+ for (final RegExpMatch match in versionMatches) {
+ print('${match.namedGroup('name')} version: ${match.namedGroup('version')}');
+ }
+
+ print('flutter version: $version\n');
+
+ // Dartdoc warnings and errors in these packages are considered fatal.
+ // All packages owned by flutter should be in the list.
+ final List flutterPackages = [
+ kDummyPackageName,
+ kPlatformIntegrationPackageName,
+ ...findPackageNames(filesystem),
+ // TODO(goderbauer): Figure out how to only include `dart:ui` of
+ // `sky_engine` below, https://github.com/dart-lang/dartdoc/issues/2278.
+ // 'sky_engine',
+ ];
+
+ // Generate the documentation. We don't need to exclude flutter_tools in
+ // this list because it's not in the recursive dependencies of the package
+ // defined at packageRoot
+ final List dartdocArgs = [
+ 'global',
+ 'run',
+ '--enable-asserts',
+ 'dartdoc',
+ '--output',
+ publishRoot.childDirectory('flutter').path,
+ '--allow-tools',
+ if (useJson) '--json',
+ if (validateLinks) '--validate-links' else '--no-validate-links',
+ '--link-to-source-excludes',
+ flutterRoot.childDirectory('bin').childDirectory('cache').path,
+ '--link-to-source-root',
+ flutterRoot.path,
+ '--link-to-source-uri-template',
+ 'https://github.com/flutter/flutter/blob/master/%f%#L%l%',
+ '--inject-html',
+ '--use-base-href',
+ '--header',
+ docsRoot.childFile('styles.html').path,
+ '--header',
+ docsRoot.childFile('analytics.html').path,
+ '--header',
+ docsRoot.childFile('survey.html').path,
+ '--header',
+ docsRoot.childFile('snippets.html').path,
+ '--header',
+ docsRoot.childFile('opensearch.html').path,
+ '--footer-text',
+ packageRoot.childFile('footer.html').path,
+ '--allow-warnings-in-packages',
+ flutterPackages.join(','),
+ '--exclude-packages',
+ [
+ 'analyzer',
+ 'args',
+ 'barback',
+ 'csslib',
+ 'flutter_goldens',
+ 'flutter_goldens_client',
+ 'front_end',
+ 'fuchsia_remote_debug_protocol',
+ 'glob',
+ 'html',
+ 'http_multi_server',
+ 'io',
+ 'isolate',
+ 'js',
+ 'kernel',
+ 'logging',
+ 'mime',
+ 'mockito',
+ 'node_preamble',
+ 'plugin',
+ 'shelf',
+ 'shelf_packages_handler',
+ 'shelf_static',
+ 'shelf_web_socket',
+ 'utf',
+ 'watcher',
+ 'yaml',
+ ].join(','),
+ '--exclude',
+ [
+ 'dart:io/network_policy.dart', // dart-lang/dartdoc#2437
+ 'package:Flutter/temp_doc.dart',
+ 'package:http/browser_client.dart',
+ 'package:intl/intl_browser.dart',
+ 'package:matcher/mirror_matchers.dart',
+ 'package:quiver/io.dart',
+ 'package:quiver/mirrors.dart',
+ 'package:vm_service_client/vm_service_client.dart',
+ 'package:web_socket_channel/html.dart',
+ ].join(','),
+ '--favicon',
+ docsRoot.childFile('favicon.ico').absolute.path,
+ '--package-order',
+ 'flutter,Dart,$kPlatformIntegrationPackageName,flutter_test,flutter_driver',
+ '--auto-include-dependencies',
+ ];
+
+ String quote(String arg) => arg.contains(' ') ? "'$arg'" : arg;
+ print('Executing: (cd "${packageRoot.path}" ; '
+ '${FlutterInformation.instance.getDartBinaryPath().path} '
+ '${dartdocArgs.map(quote).join(' ')})');
+
+ process = ProcessWrapper(await runPubProcess(
+ arguments: dartdocArgs,
+ workingDirectory: packageRoot,
+ environment: pubEnvironment,
+ ));
+ printStream(
+ process.stdout,
+ prefix: useJson ? '' : 'dartdoc:stdout: ',
+ filter: [
+ if (!verbose) RegExp(r'^Generating docs for library '), // Unnecessary verbosity
+ ],
+ );
+ printStream(
+ process.stderr,
+ prefix: useJson ? '' : 'dartdoc:stderr: ',
+ filter: [
+ if (!verbose)
+ RegExp(
+ // Remove warnings from packages outside our control
+ r'^ warning: .+: \(.+[\\/]\.pub-cache[\\/]hosted[\\/]pub.dartlang.org[\\/].+\)',
+ ),
+ ],
+ );
+ final int exitCode = await process.done;
+
+ if (exitCode != 0) {
+ exit(exitCode);
+ }
+
+ _sanityCheckDocs();
+ checkForUnresolvedDirectives(publishRoot.childDirectory('flutter').path);
+
+ _createIndexAndCleanup();
+
+ print('Documentation written to ${publishRoot.path}');
+ }
+
+ void _sanityCheckExample(String fileString, String regExpString) {
+ final File file = filesystem.file(fileString);
+ if (file.existsSync()) {
+ final RegExp regExp = RegExp(regExpString, dotAll: true);
+ final String contents = file.readAsStringSync();
+ if (!regExp.hasMatch(contents)) {
+ throw Exception("Missing example code matching '$regExpString' in ${file.path}.");
+ }
+ } else {
+ throw Exception(
+ "Missing example code sanity test file ${file.path}. Either it didn't get published, or you might have to update the test to look at a different file.");
+ }
+ }
+
+ /// Runs a sanity check by running a test.
+ void _sanityCheckDocs([Platform platform = const LocalPlatform()]) {
+ final Directory flutterDirectory = publishRoot.childDirectory('flutter');
+ final Directory widgetsDirectory = flutterDirectory.childDirectory('widgets');
+
+ final List canaries = [
+ publishRoot.childDirectory('assets').childFile('overrides.css'),
+ flutterDirectory.childDirectory('dart-io').childFile('File-class.html'),
+ flutterDirectory.childDirectory('dart-ui').childFile('Canvas-class.html'),
+ flutterDirectory.childDirectory('dart-ui').childDirectory('Canvas').childFile('drawRect.html'),
+ flutterDirectory
+ .childDirectory('flutter_driver')
+ .childDirectory('FlutterDriver')
+ .childFile('FlutterDriver.connectedTo.html'),
+ flutterDirectory.childDirectory('flutter_test').childDirectory('WidgetTester').childFile('pumpWidget.html'),
+ flutterDirectory.childDirectory('material').childFile('Material-class.html'),
+ flutterDirectory.childDirectory('material').childFile('Tooltip-class.html'),
+ widgetsDirectory.childFile('Widget-class.html'),
+ widgetsDirectory.childFile('Listener-class.html'),
+ ];
+ for (final File canary in canaries) {
+ if (!canary.existsSync()) {
+ throw Exception('Missing "${canary.path}", which probably means the documentation failed to build correctly.');
+ }
+ }
+ // Make sure at least one example of each kind includes source code.
+
+ // Check a "sample" example, any one will do.
+ _sanityCheckExample(
+ widgetsDirectory.childFile('showGeneralDialog.html').path,
+ r'\s*\s*import 'package:flutter/material.dart';',
+ );
+
+ // Check a "snippet" example, any one will do.
+ _sanityCheckExample(
+ widgetsDirectory.childDirectory('ModalRoute').childFile('barrierColor.html').path,
+ r'\s*.*Color\s+get\s+barrierColor.*',
+ );
+
+ // Check a "dartpad" example, any one will do, and check for the correct URL
+ // arguments.
+ // Just use "master" for any branch other than the LUCI_BRANCH.
+ final String? luciBranch = platform.environment['LUCI_BRANCH']?.trim();
+ final String expectedBranch = luciBranch != null && luciBranch.isNotEmpty ? luciBranch : 'master';
+ final List argumentRegExps = [
+ r'split=\d+',
+ r'run=true',
+ r'sample_id=widgets\.Listener\.\d+',
+ 'sample_channel=$expectedBranch',
+ 'channel=$expectedBranch',
+ ];
+ for (final String argumentRegExp in argumentRegExps) {
+ _sanityCheckExample(
+ widgetsDirectory.childFile('Listener-class.html').path,
+ r'\s*\s*<\/iframe>',
+ );
+ }
+ }
+
+ /// Creates a custom index.html because we try to maintain old
+ /// paths. Cleanup unused index.html files no longer needed.
+ void _createIndexAndCleanup() {
+ print('\nCreating a custom index.html in ${publishRoot.childFile('index.html').path}');
+ _copyIndexToRootOfDocs();
+ _addHtmlBaseToIndex();
+ _changePackageToSdkInTitlebar();
+ _putRedirectInOldIndexLocation();
+ _writeSnippetsIndexFile();
+ print('\nDocs ready to go!');
+ }
+
+ void _copyIndexToRootOfDocs() {
+ publishRoot.childDirectory('flutter').childFile('index.html').copySync(publishRoot.childFile('index.html').path);
+ }
+
+ void _changePackageToSdkInTitlebar() {
+ final File indexFile = publishRoot.childFile('index.html');
+ String indexContents = indexFile.readAsStringSync();
+ indexContents = indexContents.replaceFirst(
+ '