Sub-shard the docs shard (#62281)

Separate out offline docs, docset generation, and docs deployment each into dedicated shards. https://github.com/flutter/flutter/issues/60646
2020-07-26 23:38:01 -07:00 · 2020-07-26 23:38:01 -07:00 · 8e0eee9008
commit 8e0eee9008
parent 35e7005184
3 changed files with 121 additions and 44 deletions
--- a/.cirrus.yml
+++ b/.cirrus.yml
@ -278,12 +278,54 @@ task:
        # Empirically, as of October 2019, the docs-linux shard took about 30 minutes when run with
        # 1 CPU and 4G of RAM. 2 CPUs reduced that to 20 minutes, more CPUs did not improve matters.
        CPU: 2
+      docs_cache:
+        folder: dev/docs
+        fingerprint_script:
+          - echo "docs_${CIRRUS_CHANGE_IN_REPO}_${CIRRUS_BUILD_ID}"
+      script:
+        - ./dev/bots/docs.sh docs
+
+    - name: docs_offline-linux # linux-only
+      only_if: "$CIRRUS_PR == ''"
+      depends_on:
+        - docs-linux
+      docs_cache:
+        folder: dev/docs
+        fingerprint_script:
+          - echo "docs_${CIRRUS_CHANGE_IN_REPO}_${CIRRUS_BUILD_ID}"
+      script:
+        - ./dev/bots/docs.sh offline
+
+    - name: docs_docset-linux # linux-only
+      only_if: "$CIRRUS_PR == ''"
+      depends_on:
+        - docs-linux
+      environment:
+        CPU: 2
+      docs_cache:
+        folder: dev/docs
+        fingerprint_script:
+          - echo "docs_${CIRRUS_CHANGE_IN_REPO}_${CIRRUS_BUILD_ID}"
+      script:
+        - ./dev/bots/docs.sh docset
+
+    - name: docs_deploy-linux # linux-only
+      only_if: "$CIRRUS_BRANCH == 'master' || $CIRRUS_BRANCH == 'stable'"
+      depends_on:
+        - docs-linux
+        - docs_offline-linux
+        - docs_docset-linux
+      environment:
        # For uploading master docs to Firebase master branch staging site
        FIREBASE_MASTER_TOKEN: ENCRYPTED[eb768d18798fdc5abfe09b224e1724c4d82831d715ccf90df2c79d618c317216cbd99493278361f6fe7948b409b603f0]
-        # For uploading beta docs to Firebase public live site
+        # For uploading stable docs to Firebase public live site
        FIREBASE_PUBLIC_TOKEN: ENCRYPTED[37e8b82f167864cae9a3f4d2cf3f37dea331d9375c295327c45de524f6c588fa6f6d63e5784f10f6d43ce29689f36c92]
+      docs_cache:
+        folder: dev/docs
+        fingerprint_script:
+          - echo "docs_${CIRRUS_CHANGE_IN_REPO}_${CIRRUS_BUILD_ID}"
      script:
-        - ./dev/bots/docs.sh
+        - ./dev/bots/docs.sh deploy

    - name: customer_testing-linux
      # environment:
--- a/dev/bots/docs.sh
+++ b/dev/bots/docs.sh
@ -37,6 +37,25 @@ function script_location() {
  echo "$(cd -P "$(dirname "$script_location")" >/dev/null && pwd)"
 }

+function assert_not_in_pr() {
+    if [[ -z "$CIRRUS_CI" || -n "$CIRRUS_PR" ]]; then
+        >&2 echo "The $COMMAND command can only be run in Cirrus for non-PR commits."
+        exit 1
+    fi
+}
+
+function generate_docs() {
+    # Install and activate dartdoc.
+    "$PUB" global activate dartdoc 0.32.1
+
+    # This script generates a unified doc set, and creates
+    # a custom index.html, placing everything into dev/docs/doc.
+    (cd "$FLUTTER_ROOT/dev/tools" && "$FLUTTER" pub get)
+    (cd "$FLUTTER_ROOT/dev/tools" && "$PUB" get)
+    (cd "$FLUTTER_ROOT" && "$DART" --disable-dart-dev "$FLUTTER_ROOT/dev/tools/dartdoc.dart")
+    (cd "$FLUTTER_ROOT" && "$DART" --disable-dart-dev "$FLUTTER_ROOT/dev/tools/java_and_objc_doc.dart")
+}
+
 # Zip up the docs so people can download them for offline usage.
 function create_offline_zip() {
  # Must be run from "$FLUTTER_ROOT/dev/docs"
@ -67,6 +86,40 @@ function create_docset() {
  kill $tail_pid &> /dev/null
 }

+function deploy_docs() {
+    (cd "$FLUTTER_ROOT/dev/docs"; move_offline_into_place)
+
+    # Ensure google webmaster tools can verify our site.
+    cp "$FLUTTER_ROOT/dev/docs/google2ed1af765c529f57.html" "$FLUTTER_ROOT/dev/docs/doc"
+
+    # To help diagnose when things go wrong.
+    echo "Deploying the following files to Firebase:"
+    find "$FLUTTER_ROOT/dev/docs"
+    echo 'EOL'
+
+    case "$CIRRUS_BRANCH" in
+        master)
+            echo "$(date): Updating $CIRRUS_BRANCH docs: https://master-api.flutter.dev/"
+            # Disable search indexing on the master staging site so searches get only
+            # the stable site.
+            echo -e "User-agent: *\nDisallow: /" > "$FLUTTER_ROOT/dev/docs/doc/robots.txt"
+            export FIREBASE_TOKEN="$FIREBASE_MASTER_TOKEN"
+            deploy 5 master-docs-flutter-dev
+            ;;
+        stable)
+            echo "$(date): Updating $CIRRUS_BRANCH docs: https://api.flutter.dev/"
+            # Enable search indexing on the master staging site so searches get only
+            # the stable site.
+            echo -e "# All robots welcome!" > "$FLUTTER_ROOT/dev/docs/doc/robots.txt"
+            export FIREBASE_TOKEN="$FIREBASE_PUBLIC_TOKEN"
+            deploy 5 docs-flutter-dev
+            ;;
+        *)
+            >&2 echo "The $COMMAND command cannot be run on the $CIRRUS_BRANCH branch."
+            exit 1
+    esac
+}
+
 # Move the offline archives into place, after all the processing of the doc
 # directory is done. This avoids the tools recursively processing the archives
 # as part of their process.
@ -92,10 +145,11 @@ SCRIPT_LOCATION="$(script_location)"
 # then this line will need to as well.
 FLUTTER_ROOT="$(dirname "$(dirname "$SCRIPT_LOCATION")")"

-echo "$(date): Running docs.sh"
+COMMAND="$1"
+echo "$(date): Running docs.sh $COMMAND"

 if [[ ! -d "$FLUTTER_ROOT" || ! -f "$FLUTTER_ROOT/bin/flutter" ]]; then
-  echo "Unable to locate the Flutter installation (using FLUTTER_ROOT: $FLUTTER_ROOT)"
+  >&2 echo "Unable to locate the Flutter installation (using FLUTTER_ROOT: $FLUTTER_ROOT)"
  exit 1
 fi

@ -118,42 +172,23 @@ if [[ -d "$FLUTTER_PUB_CACHE" ]]; then
  export PUB_CACHE="${PUB_CACHE:-"$FLUTTER_PUB_CACHE"}"
 fi

-# Install and activate dartdoc.
-"$PUB" global activate dartdoc 0.32.1
-
-# This script generates a unified doc set, and creates
-# a custom index.html, placing everything into dev/docs/doc.
-(cd "$FLUTTER_ROOT/dev/tools" && "$FLUTTER" pub get)
-(cd "$FLUTTER_ROOT/dev/tools" && "$PUB" get)
-(cd "$FLUTTER_ROOT" && "$DART" --disable-dart-dev "$FLUTTER_ROOT/dev/tools/dartdoc.dart")
-(cd "$FLUTTER_ROOT" && "$DART" --disable-dart-dev "$FLUTTER_ROOT/dev/tools/java_and_objc_doc.dart")
-
-# Upload new API docs when running on Cirrus
-if [[ -n "$CIRRUS_CI" && -z "$CIRRUS_PR" ]]; then
-  # Create offline doc archives.
-  (cd "$FLUTTER_ROOT/dev/docs"; create_offline_zip)
-  (cd "$FLUTTER_ROOT/dev/docs"; create_docset)
-  (cd "$FLUTTER_ROOT/dev/docs"; move_offline_into_place)
-
-  # Ensure google webmaster tools can verify our site.
-  cp "$FLUTTER_ROOT/dev/docs/google2ed1af765c529f57.html" "$FLUTTER_ROOT/dev/docs/doc"
-
-  echo "This is not a pull request; considering whether to upload docs... (branch=$CIRRUS_BRANCH)"
-  if [[ "$CIRRUS_BRANCH" == "master" ]]; then
-    echo "$(date): Updating $CIRRUS_BRANCH docs: https://master-api.flutter.dev/"
-    # Disable search indexing on the master staging site so searches get only
-    # the stable site.
-    echo -e "User-agent: *\nDisallow: /" > "$FLUTTER_ROOT/dev/docs/doc/robots.txt"
-    export FIREBASE_TOKEN="$FIREBASE_MASTER_TOKEN"
-    deploy 5 master-docs-flutter-dev
-  fi
-
-  if [[ "$CIRRUS_BRANCH" == "stable" ]]; then
-    # Enable search indexing on the master staging site so searches get only
-    # the stable site.
-    echo "$(date): Updating $CIRRUS_BRANCH docs: https://api.flutter.dev/"
-    echo -e "# All robots welcome!" > "$FLUTTER_ROOT/dev/docs/doc/robots.txt"
-    export FIREBASE_TOKEN="$FIREBASE_PUBLIC_TOKEN"
-    deploy 5 docs-flutter-dev
-  fi
-fi
+case "$COMMAND" in
+    docs)
+        generate_docs
+        ;;
+    offline)
+        assert_not_in_pr
+        (cd "$FLUTTER_ROOT/dev/docs"; create_offline_zip)
+        ;;
+    docset)
+        assert_not_in_pr
+        (cd "$FLUTTER_ROOT/dev/docs"; create_docset)
+        ;;
+    deploy)
+        assert_not_in_pr
+        deploy_docs
+        ;;
+    *)
+        >&2 echo "Usage: $0 {docs|offline|docset|deploy}"
+        exit 1
+esac
--- a/dev/docs/dashing.json
+++ b/dev/docs/dashing.json
@ -8,7 +8,7 @@
  "index": "index.html",
  "icon32x32": "flutter/static-assets/favicon.png",
  "allowJS": true,
-  "ExternalURL": "https://docs.flutter.io",
+  "ExternalURL": "https://api.flutter.dev",
  "selectors": {
    "#exceptions span.name a": {
      "type": "Exception"