diff --git a/.cirrus.yml b/.cirrus.yml index 3cd50da481..ea9b7e4e1a 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -1,12 +1,16 @@ container: - image: cirrusci/flutter:base + image: gcr.io/flutter-cirrus/build-flutter-image:latest task: env: + # Name the SDK directory to include a space so that we constantly + # test path names with spaces in them. CIRRUS_WORKING_DIR: "/tmp/flutter sdk" PATH: "$CIRRUS_WORKING_DIR/bin:$CIRRUS_WORKING_DIR/bin/cache/dart-sdk/bin:$PATH" - - git_fetch_script: git fetch origin + ANDROID_HOME: "/opt/android_sdk" + git_fetch_script: + - git fetch origin + - git fetch origin master # To set FETCH_HEAD for "git merge-base" to work pub_cache: folder: $HOME/.pub-cache fingerprint_script: echo $OS; grep -r --include=pubspec.yaml 'PUBSPEC CHECKSUM' "$CIRRUS_WORKING_DIR" @@ -16,22 +20,28 @@ task: artifacts_cache: folder: bin/cache/artifacts fingerprint_script: echo $OS; cat bin/internal/engine.version - setup_script: | - echo "SDK directory is: $PWD" - ./bin/flutter --version - - # disable analytics on the bots and download Flutter dependencies - ./bin/flutter config --no-analytics - - # run pub get in all the repo packages - ./bin/flutter update-packages - - git fetch origin master + setup_script: ./dev/bots/cirrus_setup.sh matrix: - name: docs env: SHARD: docs + # For uploading master docs to Firebase master branch staging site + FIREBASE_MASTER_TOKEN: ENCRYPTED[37e8b82f167864cae9a3f4d2cf3f37dea331d9375c295327c45de524f6c588fa6f6d63e5784f10f6d43ce29689f36c92] + # For uploading beta docs to Firebase public live site + FIREBASE_PUBLIC_TOKEN: ENCRYPTED[c422da192f06da7b4449ca8e7aa866dabeb8a0f8d7488497c2e7e447e6fd31d917e6c813db081dc4e2a7a63afdf41864] docs_script: ./dev/bots/docs.sh + - name: deploy_gallery + only_if: $CIRRUS_BRANCH == 'dev' + depends_on: + - docs + - analyze + - tests-linux + - tool_tests-linux + env: + SHARD: deploy_gallery + GOOGLE_DEVELOPER_SERVICE_ACCOUNT_ACTOR_FASTLANE: ENCRYPTED[d9ac1462c3c556fc2f8165c9d5566a16497d8ebc38a50357f7f3abf136b7f83e1d1d76dde36fee356cb0f9ebf7a89346] + ANDROID_GALLERY_UPLOAD_KEY: ENCRYPTED[0b3e681b4507aec433ef29c79b715f15f8c75ecd25315ea286b0b2bcb8b28d578634eead5aa2c54086a25e8da1bb219a] + test_script: ./dev/bots/deploy_gallery.sh - name: analyze env: SHARD: analyze @@ -61,7 +71,9 @@ task: cpu: 4 env: CIRRUS_WORKING_DIR: "C:\\Windows\\Temp\\flutter sdk" - git_fetch_script: git fetch origin + git_fetch_script: + - git fetch origin + - git fetch origin master # To set FETCH_HEAD for "git merge-base" to work pub_cache: folder: $APPDATA\Pub\Cache fingerprint_script: @@ -87,13 +99,46 @@ task: SHARD: tool_tests task: + name: deploy_gallery-macos + only_if: $CIRRUS_BRANCH == 'dev' + pub_cache: + folder: ~/.pub-cache depends_on: - analyze + - tests-macos + - tool_tests-macos + env: + # Name the SDK directory to include a space so that we constantly + # test path names with spaces in them. + CIRRUS_WORKING_DIR: "/tmp/flutter sdk" + SHARD: deploy_gallery + # Apple Certificates Match Passphrase + MATCH_PASSWORD: ENCRYPTED[db07f252234397090e3ec59152d9ec1831f5ecd0ef97d247b1dca757bbb9ef9b7c832a39bce2caf1949ccdf097e59a73] + # Apple Fastlane Password + FASTLANE_PASSWORD: ENCRYPTED[0bf9bb0cc2cb32a0ed18470cf2c9df0f587cce5f8b04adbd6cff15ca5bde7a74f721ee580227b132ab6b032f08e52ae0] + # Private repo for publishing certificates. + PUBLISHING_MATCH_CERTIFICATE_REPO: git@github.com:flutter/private_publishing_certificates.git osx_instance: image: high-sierra-xcode-9.4.1 + git_fetch_script: + - git fetch origin + - git fetch origin master # To set FETCH_HEAD + setup_script: + - bin/flutter config --no-analytics + - bin/flutter update-packages + test_all_script: + - ./dev/bots/deploy_gallery.sh + +task: + osx_instance: + image: high-sierra-xcode-9.4.1 + depends_on: + - analyze env: CIRRUS_WORKING_DIR: "/tmp/flutter sdk" - git_fetch_script: git fetch origin + git_fetch_script: + - git fetch origin + - git fetch origin master # To set FETCH_HEAD for "git merge-base" to work pub_cache: folder: $HOME/.pub-cache fingerprint_script: echo $OS; grep -r --include=pubspec.yaml 'PUBSPEC CHECKSUM' "$CIRRUS_WORKING_DIR" @@ -116,3 +161,19 @@ task: - name: tool_tests-macos env: SHARD: tool_tests + + +docker_builder: + # Only build a new docker image when we tag a release (for dev, beta, or release.) + only_if: $CIRRUS_TAG != '' + env: + GCLOUD_CREDENTIALS: ENCRYPTED[f7c098d4dd7f5ee1bfee0bb7e944cce72efbe10e97ad6440ae72de4de6a1c24d23f421a2619c668e94377fb64b0bb3e6] + depends_on: + - docs + - analyze + - tests-linux + - tool_tests-linux + build_script: "$CIRRUS_WORKING_DIR/dev/ci/docker_linux/docker_build.sh" + login_script: "$CIRRUS_WORKING_DIR/dev/ci/docker_linux/docker_login.sh" + push_script: "$CIRRUS_WORKING_DIR/dev/ci/docker_linux/docker_push.sh" + diff --git a/dev/bots/cirrus_setup.sh b/dev/bots/cirrus_setup.sh new file mode 100755 index 0000000000..f7c292023a --- /dev/null +++ b/dev/bots/cirrus_setup.sh @@ -0,0 +1,43 @@ +#!/bin/bash +set -e + +# This script is only meant to be run by the Cirrus CI system, not locally. +# It must be run from the root of the Flutter repo. + +# Collects log output in a tmpfile, but only prints it if the command fails. +function log_on_fail() { + local COMMAND="$@" + local TMPDIR="$(mktemp -d)" + local TMPFILE="$TMPDIR/command.log" + local EXIT=0 + if ("$@" > "$TMPFILE" 2>&1); then + echo "'$COMMAND' succeeded." + else + EXIT=$? + cat "$TMPFILE" 1>&2 + echo "FAIL: '$COMMAND' exited with code $EXIT" 1>&2 + fi + rm -rf "$TMPDIR" + return "$EXIT" +} + +function accept_android_licenses() { + yes "y" | flutter doctor --android-licenses +} + +echo "Flutter SDK directory is: $PWD" + +# Run flutter to download dependencies and precompile things, and to disable +# analytics on the bots. +echo "Downloading build dependencies and pre-compiling Flutter snapshot" +log_on_fail ./bin/flutter config --no-analytics + +# Run doctor, to print it to the log for debugging purposes. +./bin/flutter doctor -v + +# Accept licenses. +log_on_fail accept_android_licenses && echo "Android licenses accepted." + +# Run pub get in all the repo packages. +echo "Updating packages for Flutter." +log_on_fail ./bin/flutter update-packages diff --git a/dev/bots/deploy_gallery.sh b/dev/bots/deploy_gallery.sh new file mode 100755 index 0000000000..e05d002a4c --- /dev/null +++ b/dev/bots/deploy_gallery.sh @@ -0,0 +1,77 @@ +#!/bin/bash + +set -e + +function script_location() { + local script_location="${BASH_SOURCE[0]}" + # Resolve symlinks + while [[ -h "$script_location" ]]; do + DIR="$(cd -P "$(dirname "$script_location")" >/dev/null && pwd)" + script_location="$(readlink "$script_location")" + [[ "$script_location" != /* ]] && script_location="$DIR/$script_location" + done + echo "$(cd -P "$(dirname "$script_location")" >/dev/null && pwd)" +} + +# So that users can run this script locally from any directory and it will work as +# expected. +SCRIPT_LOCATION="$(script_location)" +FLUTTER_ROOT="$(dirname "$(dirname "$SCRIPT_LOCATION")")" + +export PATH="$FLUTTER_ROOT/bin:$FLUTTER_ROOT/bin/cache/dart-sdk/bin:$PATH" + +set -x + +cd "$FLUTTER_ROOT" + +if [[ "$SHARD" = "deploy_gallery" ]]; then + version="$( 0 ]]; do + (cd "$FLUTTER_ROOT/dev/docs" && firebase deploy --token "$FIREBASE_TOKEN" --project "$@") && break + remaining_tries=$(($remaining_tries - 1)) + echo "Error: Unable to deploy documentation to Firebase. Retrying in five seconds... ($remaining_tries tries left)" + sleep 5 + done + + [[ "$remaining_tries" == 0 ]] && { + echo "Command still failed after $total_tries tries: '$@'" + return 1 + } + return 0 +} + +function script_location() { + local script_location="${BASH_SOURCE[0]}" + # Resolve symlinks + while [[ -h "$script_location" ]]; do + DIR="$(cd -P "$( dirname "$script_location")" >/dev/null && pwd)" + script_location="$(readlink "$script_location")" + [[ "$script_location" != /* ]] && script_location="$DIR/$script_location" + done + echo "$(cd -P "$(dirname "$script_location")" >/dev/null && pwd)" +} + +# So that users can run this script from anywhere and it will work as expected. +SCRIPT_LOCATION="$(script_location)" +# Sets the Flutter root to be "$(script_location)/../..": This script assumes +# that it resides two directory levels down from the root, so if that changes, +# then this line will need to as well. +FLUTTER_ROOT="$(dirname "$(dirname "$SCRIPT_LOCATION")")" + echo "Running docs.sh" -# If you want to run this script locally, make sure you run it from -# the root of the flutter repository. -export FLUTTER_ROOT="$PWD" -export PATH="$PWD/bin:$PATH" +if [[ ! -d "$FLUTTER_ROOT" || ! -f "$FLUTTER_ROOT/bin/flutter" ]]; then + echo "Unable to locate the Flutter installation (using FLUTTER_ROOT: $FLUTTER_ROOT)" + exit 1 +fi -# This is called from travis_upload.sh on Travis. +FLUTTER_BIN="$FLUTTER_ROOT/bin" +DART_BIN="$FLUTTER_ROOT/bin/cache/dart-sdk/bin" +FLUTTER="$FLUTTER_BIN/flutter" +DART="$DART_BIN/dart" +PUB="$DART_BIN/pub" +export PATH="$FLUTTER_BIN:$DART_BIN:$PATH" -# Make sure dart is installed -bin/flutter --version +# Make sure dart is installed by invoking flutter to download it. +"$FLUTTER" --version # If the pub cache directory exists in the root, then use that. FLUTTER_PUB_CACHE="$FLUTTER_ROOT/.pub-cache" -if [ -d "$FLUTTER_PUB_CACHE" ]; then +if [[ -d "$FLUTTER_PUB_CACHE" ]]; then # This has to be exported, because pub interprets setting it # to the empty string in the same way as setting it to ".". export PUB_CACHE="${PUB_CACHE:-"$FLUTTER_PUB_CACHE"}" fi -# Install dartdoc. -bin/cache/dart-sdk/bin/pub global activate dartdoc 0.20.2 +# Install and activate dartdoc. +"$PUB" global activate dartdoc 0.20.2 # This script generates a unified doc set, and creates # a custom index.html, placing everything into dev/docs/doc. -(cd dev/tools; ../../bin/cache/dart-sdk/bin/pub get) -bin/cache/dart-sdk/bin/dart dev/tools/dartdoc.dart -bin/cache/dart-sdk/bin/dart dev/tools/java_and_objc_doc.dart +(cd "$FLUTTER_ROOT/dev/tools" && "$FLUTTER" packages get) +(cd "$FLUTTER_ROOT/dev/tools" && "$PUB" get) +(cd "$FLUTTER_ROOT" && "$DART" "$FLUTTER_ROOT/dev/tools/dartdoc.dart") +(cd "$FLUTTER_ROOT" && "$DART" "$FLUTTER_ROOT/dev/tools/java_and_objc_doc.dart") # Ensure google webmaster tools can verify our site. -cp dev/docs/google2ed1af765c529f57.html dev/docs/doc +cp "$FLUTTER_ROOT/dev/docs/google2ed1af765c529f57.html" "$FLUTTER_ROOT/dev/docs/doc" # Upload new API docs when on Travis -if [ "$TRAVIS_PULL_REQUEST" == "false" ]; then - echo "This is not a pull request; considering whether to upload docs... (branch=$TRAVIS_BRANCH)" - if [ "$TRAVIS_BRANCH" == "master" -o "$TRAVIS_BRANCH" == "beta" ]; then - cd dev/docs - - if [ "$TRAVIS_BRANCH" == "master" ]; then +if [[ -n "$CIRRUS_CI" && -z "$CIRRUS_PR" ]]; then + echo "This is not a pull request; considering whether to upload docs... (branch=$CIRRUS_BRANCH)" + if [[ "$CIRRUS_BRANCH" == "master" || "$CIRRUS_BRANCH" == "beta" ]]; then + if [[ "$CIRRUS_BRANCH" == "master" ]]; then echo "Updating master docs: https://master-docs-flutter-io.firebaseapp.com/" - echo -e "User-agent: *\nDisallow: /" > doc/robots.txt - while : ; do - firebase deploy --project master-docs-flutter-io && break - echo Error: Unable to deploy documentation to firebase. Retrying in five seconds... - sleep 5 - done + # Disable search indexing on the master staging site so searches get only + # the beta site. + echo -e "User-agent: *\nDisallow: /" > "$FLUTTER_ROOT/dev/docs/doc/robots.txt" + export FIREBASE_TOKEN="$FIREBASE_MASTER_TOKEN" + deploy 5 master-docs-flutter-io fi - if [ "$TRAVIS_BRANCH" == "beta" ]; then + if [[ "$CIRRUS_BRANCH" == "beta" ]]; then echo "Updating beta docs: https://docs.flutter.io/" - while : ; do - firebase deploy --project docs-flutter-io && break - echo Error: Unable to deploy documentation to firebase. Retrying in five seconds... - sleep 5 - done + export FIREBASE_TOKEN="$FIREBASE_PUBLIC_TOKEN" + deploy 5 docs-flutter-io fi fi fi + diff --git a/dev/ci/README.md b/dev/ci/README.md new file mode 100644 index 0000000000..884316896a --- /dev/null +++ b/dev/ci/README.md @@ -0,0 +1 @@ +This directory includes scripts and tools for continuous integration builds and tests. diff --git a/dev/ci/docker_linux/Dockerfile b/dev/ci/docker_linux/Dockerfile new file mode 100644 index 0000000000..a42bd79da7 --- /dev/null +++ b/dev/ci/docker_linux/Dockerfile @@ -0,0 +1,93 @@ +# Flutter (https://flutter.io) Developement Environment for Linux +# =============================================================== +# +# This environment passes all Linux Flutter Doctor checks and is sufficient +# for building Android applications and running Flutter tests. +# +# To build iOS applications, a Mac development environment is necessary. +# +# This includes applications and sdks that are needed only by the CI system +# for performing pushes to production, and so this image is quite a bit larger +# than strictly needed for just building Flutter apps. + +FROM debian:stretch +MAINTAINER Flutter Developers + +RUN apt-get update -y +RUN apt-get upgrade -y + +# Install basics +RUN apt-get install -y --no-install-recommends \ + git \ + wget \ + curl \ + unzip \ + ca-certificates \ + gnupg + +# Add nodejs repository to apt sources and install it. +ENV NODEJS_INSTALL="/opt/nodejs_install" +RUN mkdir -p "${NODEJS_INSTALL}" +RUN wget -q https://deb.nodesource.com/setup_10.x -O "${NODEJS_INSTALL}/nodejs_install.sh" +RUN bash "${NODEJS_INSTALL}/nodejs_install.sh" + +# Install the rest of the dependencies. +RUN apt-get install -y --no-install-recommends \ + locales \ + ruby \ + nodejs \ + lib32stdc++6 \ + libstdc++6 \ + libglu1-mesa \ + build-essential \ + default-jdk-headless + +# Install the Android SDK Dependency. +ENV ANDROID_SDK_URL="https://dl.google.com/android/repository/sdk-tools-linux-4333796.zip" +ENV ANDROID_TOOLS_ROOT="/opt/android_sdk" +RUN mkdir -p "${ANDROID_TOOLS_ROOT}" +RUN mkdir -p ~/.android +# Silence warning. +RUN touch ~/.android/repositories.cfg +ENV ANDROID_SDK_ARCHIVE="${ANDROID_TOOLS_ROOT}/archive" +RUN wget --progress=dot:giga "${ANDROID_SDK_URL}" -O "${ANDROID_SDK_ARCHIVE}" +RUN unzip -q -d "${ANDROID_TOOLS_ROOT}" "${ANDROID_SDK_ARCHIVE}" +# Suppressing output of sdkmanager to keep log size down +# (it prints install progress WAY too often). +RUN yes "y" | "${ANDROID_TOOLS_ROOT}/tools/bin/sdkmanager" "tools" > /dev/null +RUN yes "y" | "${ANDROID_TOOLS_ROOT}/tools/bin/sdkmanager" "build-tools;28.0.0" > /dev/null +RUN yes "y" | "${ANDROID_TOOLS_ROOT}/tools/bin/sdkmanager" "platforms;android-28" > /dev/null +RUN yes "y" | "${ANDROID_TOOLS_ROOT}/tools/bin/sdkmanager" "platform-tools" > /dev/null +RUN yes "y" | "${ANDROID_TOOLS_ROOT}/tools/bin/sdkmanager" "extras;android;m2repository" > /dev/null +RUN yes "y" | "${ANDROID_TOOLS_ROOT}/tools/bin/sdkmanager" "extras;google;m2repository" > /dev/null +RUN yes "y" | "${ANDROID_TOOLS_ROOT}/tools/bin/sdkmanager" "patcher;v4" > /dev/null +RUN rm "${ANDROID_SDK_ARCHIVE}" +ENV PATH="${ANDROID_TOOLS_ROOT}/tools:${PATH}" +ENV PATH="${ANDROID_TOOLS_ROOT}/tools/bin:${PATH}" +# Silence warnings when accepting android licenses. +RUN mkdir -p ~/.android +RUN touch ~/.android/repositories.cfg + +# Setup gradle +ENV GRADLE_ROOT="/opt/gradle" +RUN mkdir -p "${GRADLE_ROOT}" +ENV GRADLE_ARCHIVE="${GRADLE_ROOT}/gradle.zip" +ENV GRADLE_URL="http://services.gradle.org/distributions/gradle-4.4-bin.zip" +RUN wget --progress=dot:giga "$GRADLE_URL" -O "${GRADLE_ARCHIVE}" +RUN unzip -q -d "${GRADLE_ROOT}" "${GRADLE_ARCHIVE}" +ENV PATH="$GRADLE_ROOT/bin:$PATH" + +# Add npm to path. +ENV PATH="/usr/bin:${PATH}" +RUN dpkg-query -L nodejs +# Install Firebase +RUN /usr/bin/npm install -g firebase-tools + +# Set locale to en_US +RUN locale-gen en_US "en_US.UTF-8" && dpkg-reconfigure locales +ENV LANG en_US.UTF-8 + +# Install coveralls and Firebase +RUN gem install coveralls +RUN gem install bundler + diff --git a/dev/ci/docker_linux/README.md b/dev/ci/docker_linux/README.md new file mode 100644 index 0000000000..bed3863a6d --- /dev/null +++ b/dev/ci/docker_linux/README.md @@ -0,0 +1,13 @@ +This directory includes scripts to build the docker container image used for +building flutter/flutter in our CI system (currently [Cirrus](cirrus-ci.org)). + +In order to run the scripts, you have to setup `docker` and `gcloud`. Please +refer to the [internal flutter team doc](go/flutter-team) for how to setup in a +Google internal environment. + +After setup, +* edit `Dockerfile` to change how the container image is built. +* run `./build_docker.sh` to build the container image. +* run `./push_docker.sh` to push the image to google cloud registry. This will + affect our CI tests. + diff --git a/dev/ci/docker_linux/docker_build.sh b/dev/ci/docker_linux/docker_build.sh new file mode 100755 index 0000000000..9133e21771 --- /dev/null +++ b/dev/ci/docker_linux/docker_build.sh @@ -0,0 +1,8 @@ +#!/bin/bash + +TAG="${CIRRUS_TAG:-latest}" + +# pull to make sure we are not rebuilding for nothing +docker pull "gcr.io/flutter-cirrus/build-flutter-image:$TAG" + +docker build --tag "gcr.io/flutter-cirrus/build-flutter-image:$TAG" . diff --git a/dev/ci/docker_linux/docker_login.sh b/dev/ci/docker_linux/docker_login.sh new file mode 100755 index 0000000000..a85a3fc2ab --- /dev/null +++ b/dev/ci/docker_linux/docker_login.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +if [[ -n "$CIRRUS_CI" && -n "$GCLOUD_CREDENTIALS" ]]; then + echo "$GCLOUD_CREDENTIALS" | base64 --decode | docker login -u _json_key --password-stdin https://gcr.io +else + gcloud auth print-access-token | docker login -u oauth2accesstoken --password-stdin https://gcr.io +fi \ No newline at end of file diff --git a/dev/ci/docker_linux/docker_push.sh b/dev/ci/docker_linux/docker_push.sh new file mode 100755 index 0000000000..07f9fa26da --- /dev/null +++ b/dev/ci/docker_linux/docker_push.sh @@ -0,0 +1,6 @@ +#!/bin/bash + +TAG="${CIRRUS_TAG:-latest}" + +docker push "gcr.io/flutter-cirrus/build-flutter-image::$TAG" +