From 8e0eee9008157df879353d92b394c788d5a7b361 Mon Sep 17 00:00:00 2001 From: Todd Volkert Date: Sun, 26 Jul 2020 23:38:01 -0700 Subject: [PATCH] Sub-shard the docs shard (#62281) Separate out offline docs, docset generation, and docs deployment each into dedicated shards. https://github.com/flutter/flutter/issues/60646 --- .cirrus.yml | 46 ++++++++++++++++- dev/bots/docs.sh | 117 +++++++++++++++++++++++++++--------------- dev/docs/dashing.json | 2 +- 3 files changed, 121 insertions(+), 44 deletions(-) diff --git a/.cirrus.yml b/.cirrus.yml index eebdbcae0b..9f36b21a06 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -278,12 +278,54 @@ task: # Empirically, as of October 2019, the docs-linux shard took about 30 minutes when run with # 1 CPU and 4G of RAM. 2 CPUs reduced that to 20 minutes, more CPUs did not improve matters. CPU: 2 + docs_cache: + folder: dev/docs + fingerprint_script: + - echo "docs_${CIRRUS_CHANGE_IN_REPO}_${CIRRUS_BUILD_ID}" + script: + - ./dev/bots/docs.sh docs + + - name: docs_offline-linux # linux-only + only_if: "$CIRRUS_PR == ''" + depends_on: + - docs-linux + docs_cache: + folder: dev/docs + fingerprint_script: + - echo "docs_${CIRRUS_CHANGE_IN_REPO}_${CIRRUS_BUILD_ID}" + script: + - ./dev/bots/docs.sh offline + + - name: docs_docset-linux # linux-only + only_if: "$CIRRUS_PR == ''" + depends_on: + - docs-linux + environment: + CPU: 2 + docs_cache: + folder: dev/docs + fingerprint_script: + - echo "docs_${CIRRUS_CHANGE_IN_REPO}_${CIRRUS_BUILD_ID}" + script: + - ./dev/bots/docs.sh docset + + - name: docs_deploy-linux # linux-only + only_if: "$CIRRUS_BRANCH == 'master' || $CIRRUS_BRANCH == 'stable'" + depends_on: + - docs-linux + - docs_offline-linux + - docs_docset-linux + environment: # For uploading master docs to Firebase master branch staging site FIREBASE_MASTER_TOKEN: ENCRYPTED[eb768d18798fdc5abfe09b224e1724c4d82831d715ccf90df2c79d618c317216cbd99493278361f6fe7948b409b603f0] - # For uploading beta docs to Firebase public live site + # For uploading stable docs to Firebase public live site FIREBASE_PUBLIC_TOKEN: ENCRYPTED[37e8b82f167864cae9a3f4d2cf3f37dea331d9375c295327c45de524f6c588fa6f6d63e5784f10f6d43ce29689f36c92] + docs_cache: + folder: dev/docs + fingerprint_script: + - echo "docs_${CIRRUS_CHANGE_IN_REPO}_${CIRRUS_BUILD_ID}" script: - - ./dev/bots/docs.sh + - ./dev/bots/docs.sh deploy - name: customer_testing-linux # environment: diff --git a/dev/bots/docs.sh b/dev/bots/docs.sh index f4c1415d63..e710ec7ef4 100755 --- a/dev/bots/docs.sh +++ b/dev/bots/docs.sh @@ -37,6 +37,25 @@ function script_location() { echo "$(cd -P "$(dirname "$script_location")" >/dev/null && pwd)" } +function assert_not_in_pr() { + if [[ -z "$CIRRUS_CI" || -n "$CIRRUS_PR" ]]; then + >&2 echo "The $COMMAND command can only be run in Cirrus for non-PR commits." + exit 1 + fi +} + +function generate_docs() { + # Install and activate dartdoc. + "$PUB" global activate dartdoc 0.32.1 + + # This script generates a unified doc set, and creates + # a custom index.html, placing everything into dev/docs/doc. + (cd "$FLUTTER_ROOT/dev/tools" && "$FLUTTER" pub get) + (cd "$FLUTTER_ROOT/dev/tools" && "$PUB" get) + (cd "$FLUTTER_ROOT" && "$DART" --disable-dart-dev "$FLUTTER_ROOT/dev/tools/dartdoc.dart") + (cd "$FLUTTER_ROOT" && "$DART" --disable-dart-dev "$FLUTTER_ROOT/dev/tools/java_and_objc_doc.dart") +} + # Zip up the docs so people can download them for offline usage. function create_offline_zip() { # Must be run from "$FLUTTER_ROOT/dev/docs" @@ -67,6 +86,40 @@ function create_docset() { kill $tail_pid &> /dev/null } +function deploy_docs() { + (cd "$FLUTTER_ROOT/dev/docs"; move_offline_into_place) + + # Ensure google webmaster tools can verify our site. + cp "$FLUTTER_ROOT/dev/docs/google2ed1af765c529f57.html" "$FLUTTER_ROOT/dev/docs/doc" + + # To help diagnose when things go wrong. + echo "Deploying the following files to Firebase:" + find "$FLUTTER_ROOT/dev/docs" + echo 'EOL' + + case "$CIRRUS_BRANCH" in + master) + echo "$(date): Updating $CIRRUS_BRANCH docs: https://master-api.flutter.dev/" + # Disable search indexing on the master staging site so searches get only + # the stable site. + echo -e "User-agent: *\nDisallow: /" > "$FLUTTER_ROOT/dev/docs/doc/robots.txt" + export FIREBASE_TOKEN="$FIREBASE_MASTER_TOKEN" + deploy 5 master-docs-flutter-dev + ;; + stable) + echo "$(date): Updating $CIRRUS_BRANCH docs: https://api.flutter.dev/" + # Enable search indexing on the master staging site so searches get only + # the stable site. + echo -e "# All robots welcome!" > "$FLUTTER_ROOT/dev/docs/doc/robots.txt" + export FIREBASE_TOKEN="$FIREBASE_PUBLIC_TOKEN" + deploy 5 docs-flutter-dev + ;; + *) + >&2 echo "The $COMMAND command cannot be run on the $CIRRUS_BRANCH branch." + exit 1 + esac +} + # Move the offline archives into place, after all the processing of the doc # directory is done. This avoids the tools recursively processing the archives # as part of their process. @@ -92,10 +145,11 @@ SCRIPT_LOCATION="$(script_location)" # then this line will need to as well. FLUTTER_ROOT="$(dirname "$(dirname "$SCRIPT_LOCATION")")" -echo "$(date): Running docs.sh" +COMMAND="$1" +echo "$(date): Running docs.sh $COMMAND" if [[ ! -d "$FLUTTER_ROOT" || ! -f "$FLUTTER_ROOT/bin/flutter" ]]; then - echo "Unable to locate the Flutter installation (using FLUTTER_ROOT: $FLUTTER_ROOT)" + >&2 echo "Unable to locate the Flutter installation (using FLUTTER_ROOT: $FLUTTER_ROOT)" exit 1 fi @@ -118,42 +172,23 @@ if [[ -d "$FLUTTER_PUB_CACHE" ]]; then export PUB_CACHE="${PUB_CACHE:-"$FLUTTER_PUB_CACHE"}" fi -# Install and activate dartdoc. -"$PUB" global activate dartdoc 0.32.1 - -# This script generates a unified doc set, and creates -# a custom index.html, placing everything into dev/docs/doc. -(cd "$FLUTTER_ROOT/dev/tools" && "$FLUTTER" pub get) -(cd "$FLUTTER_ROOT/dev/tools" && "$PUB" get) -(cd "$FLUTTER_ROOT" && "$DART" --disable-dart-dev "$FLUTTER_ROOT/dev/tools/dartdoc.dart") -(cd "$FLUTTER_ROOT" && "$DART" --disable-dart-dev "$FLUTTER_ROOT/dev/tools/java_and_objc_doc.dart") - -# Upload new API docs when running on Cirrus -if [[ -n "$CIRRUS_CI" && -z "$CIRRUS_PR" ]]; then - # Create offline doc archives. - (cd "$FLUTTER_ROOT/dev/docs"; create_offline_zip) - (cd "$FLUTTER_ROOT/dev/docs"; create_docset) - (cd "$FLUTTER_ROOT/dev/docs"; move_offline_into_place) - - # Ensure google webmaster tools can verify our site. - cp "$FLUTTER_ROOT/dev/docs/google2ed1af765c529f57.html" "$FLUTTER_ROOT/dev/docs/doc" - - echo "This is not a pull request; considering whether to upload docs... (branch=$CIRRUS_BRANCH)" - if [[ "$CIRRUS_BRANCH" == "master" ]]; then - echo "$(date): Updating $CIRRUS_BRANCH docs: https://master-api.flutter.dev/" - # Disable search indexing on the master staging site so searches get only - # the stable site. - echo -e "User-agent: *\nDisallow: /" > "$FLUTTER_ROOT/dev/docs/doc/robots.txt" - export FIREBASE_TOKEN="$FIREBASE_MASTER_TOKEN" - deploy 5 master-docs-flutter-dev - fi - - if [[ "$CIRRUS_BRANCH" == "stable" ]]; then - # Enable search indexing on the master staging site so searches get only - # the stable site. - echo "$(date): Updating $CIRRUS_BRANCH docs: https://api.flutter.dev/" - echo -e "# All robots welcome!" > "$FLUTTER_ROOT/dev/docs/doc/robots.txt" - export FIREBASE_TOKEN="$FIREBASE_PUBLIC_TOKEN" - deploy 5 docs-flutter-dev - fi -fi +case "$COMMAND" in + docs) + generate_docs + ;; + offline) + assert_not_in_pr + (cd "$FLUTTER_ROOT/dev/docs"; create_offline_zip) + ;; + docset) + assert_not_in_pr + (cd "$FLUTTER_ROOT/dev/docs"; create_docset) + ;; + deploy) + assert_not_in_pr + deploy_docs + ;; + *) + >&2 echo "Usage: $0 {docs|offline|docset|deploy}" + exit 1 +esac diff --git a/dev/docs/dashing.json b/dev/docs/dashing.json index 3e96f0bb24..604c83e827 100644 --- a/dev/docs/dashing.json +++ b/dev/docs/dashing.json @@ -8,7 +8,7 @@ "index": "index.html", "icon32x32": "flutter/static-assets/favicon.png", "allowJS": true, - "ExternalURL": "https://docs.flutter.io", + "ExternalURL": "https://api.flutter.dev", "selectors": { "#exceptions span.name a": { "type": "Exception"