influxdb/.circleci/config.yml

---
# CI Overview
# -----------
#
# Each night:
#
#   A build image is created (ci_image) from `docker/Dockerfile.ci` and is
#   pushed to `quay.io/influxdb/rust:ci`. This build image is then used to run
#   the CI tasks for the day.
#
# Every commit:
#
# The CI for every PR and merge to main runs tests, fmt, lints and compiles debug binaries
#
# On main if all these checks pass it will then additionally compile in "release" mode and
# publish a docker image to quay.io/influxdb/iox:$COMMIT_SHA
#
# Manual CI Image:
#
# It is possible to manually trigger a rebuild of the image used in CI. To do this, navigate to
# https://app.circleci.com/pipelines/github/influxdata/influxdb_iox?branch=main (overriding the
# branch name if desired). Then:
# - Click "Run Pipeline" in the top-right
# - Expand "Add Parameters"
# - Add a "boolean" parameter called "ci_image" with the value true
# - Click "Run Pipeline"
#
# If you refresh the page you should see a newly running ci_image workflow
#

version: 2.1

commands:
  rust_components:
    description: Verify installed components
    steps:
      - run:
          name: Verify installed components
          command: |
            rustup --version
            rustup show
            cargo fmt --version
            cargo clippy --version
            cargo install cargo-hakari && cargo hakari --version

  cache_restore:
    description: Restore Cargo Cache
    steps:
      - restore_cache:
          name: Restoring Cargo Cache
          keys:
            - cargo-cache-{{ arch }}-{{ .Branch }}-{{ checksum "Cargo.lock" }}
            - cargo-cache-{{ arch }}-{{ .Branch }}
            - cargo-cache
  cache_save:
    description: Save Cargo Cache
    steps:
      - save_cache:
          name: Save Cargo Cache
          paths:
            - /usr/local/cargo/registry
          key: cargo-cache-{{ arch }}-{{ .Branch }}-{{ checksum "Cargo.lock" }}

jobs:
  fmt:
    docker:
      - image: quay.io/influxdb/rust:ci
    environment:
      # Disable incremental compilation to avoid overhead. We are not preserving these files anyway.
      CARGO_INCREMENTAL: "0"
      # Disable full debug symbol generation to speed up CI build
      # "1" means line tables only, which is useful for panic tracebacks.
      RUSTFLAGS: "-C debuginfo=1"
      # https://github.com/rust-lang/cargo/issues/10280
      CARGO_NET_GIT_FETCH_WITH_CLI: "true"
    steps:
      - checkout
      - rust_components
      - cache_restore
      - run:
          name: Rust fmt
          command: cargo fmt --all -- --check
      - cache_save
  lint:
    docker:
      - image: quay.io/influxdb/rust:ci
    environment:
      # Disable incremental compilation to avoid overhead. We are not preserving these files anyway.
      CARGO_INCREMENTAL: "0"
      # Disable full debug symbol generation to speed up CI build
      # "1" means line tables only, which is useful for panic tracebacks.
      RUSTFLAGS: "-C debuginfo=1"
      # https://github.com/rust-lang/cargo/issues/10280
      CARGO_NET_GIT_FETCH_WITH_CLI: "true"
    steps:
      - checkout
      - rust_components
      - cache_restore
      - run:
          name: Clippy
          command: cargo clippy --all-targets --all-features --workspace -- -D warnings
      - run:
          name: Shellcheck
          command: find scripts -type f ! \( -iname '*.py' -or -iname '*.supp' \) -exec shellcheck {} +
      - run:
          name: Yamllint
          command: yamllint --config-file .circleci/yamllint.yml --strict .
      - cache_save
  cargo_audit:
    docker:
      - image: quay.io/influxdb/rust:ci
    environment:
      # Disable incremental compilation to avoid overhead. We are not preserving these files anyway.
      CARGO_INCREMENTAL: "0"
      # Disable full debug symbol generation to speed up CI build
      # "1" means line tables only, which is useful for panic tracebacks.
      RUSTFLAGS: "-C debuginfo=1"
      # https://github.com/rust-lang/cargo/issues/10280
      CARGO_NET_GIT_FETCH_WITH_CLI: "true"
    steps:
      - checkout
      - rust_components
      - cache_restore
      - run:
          name: Install Cargo Audit
          command: cargo install --force cargo-audit
      # https://github.com/rust-lang/cargo/issues/10280 but `cargo audit` doesn't support `CARGO_NET_GIT_FETCH_WITH_CLI`
      - run:
          name: Workaround SSH issues
          command: |
            cd /usr/local/cargo
            git clone https://github.com/RustSec/advisory-db.git
      - run:
          name: Cargo Audit
          command: cargo audit --no-fetch
      - cache_save
  doc:
    docker:
      - image: quay.io/influxdb/rust:ci
    environment:
      # Disable incremental compilation to avoid overhead. We are not preserving these files anyway.
      CARGO_INCREMENTAL: "0"
      # Disable full debug symbol generation to speed up CI build
      # "1" means line tables only, which is useful for panic tracebacks.
      RUSTFLAGS: "-C debuginfo=1"
      # https://github.com/rust-lang/cargo/issues/10280
      CARGO_NET_GIT_FETCH_WITH_CLI: "true"
    steps:
      - checkout
      - rust_components
      - cache_restore
      - run:
          name: Cargo doc
          # excluding datafusion because it's effectively a dependency masqueraded as workspace crate.
          command: cargo doc --document-private-items --no-deps --workspace --exclude datafusion
      - cache_save
      - run:
          name: Compress Docs
          command: tar -cvzf rustdoc.tar.gz target/doc/
      - store_artifacts:
          path: rustdoc.tar.gz
  workspace_hack_checks:
    docker:
      - image: quay.io/influxdb/rust:ci
    environment:
      # https://github.com/rust-lang/cargo/issues/10280
      CARGO_NET_GIT_FETCH_WITH_CLI: "true"
    steps:
      - checkout
      - rust_components
      - cache_restore
      - run:
          name: Check that the workspace hack crate contains all features in use
          command: |
            cargo hakari generate --diff || {
              echo "If this fails, fix it by running \`cargo hakari generate\` locally and committing the changes"
              exit 1
            }
      - run:
          name: Check that all crates in the workspace depend on the workspace hack crate
          command: |
            cargo hakari manage-deps --dry-run || {
              echo "If this fails, fix it by running \`cargo hakari manage-deps\` locally and committing the changes"
              exit 1
            }

  test:
    # setup multiple docker images (see https://circleci.com/docs/2.0/configuration-reference/#docker)
    docker:
      - image: quay.io/influxdb/rust:ci
      - image: localstack/localstack
      - image: mcr.microsoft.com/azure-storage/azurite
      - image: vectorized/redpanda:v21.9.2
        command: redpanda start --overprovisioned --smp 1 --memory 1G --reserve-memory 0M
      - image: postgres
        environment:
          POSTGRES_HOST_AUTH_METHOD: trust
    resource_class: xlarge  # use of a smaller executor tends crashes on link
    environment:
      # Disable incremental compilation to avoid overhead. We are not preserving these files anyway.
      CARGO_INCREMENTAL: "0"
      # Disable full debug symbol generation to speed up CI build
      # "1" means line tables only, which is useful for panic tracebacks.
      RUSTFLAGS: "-C debuginfo=1"
      # https://github.com/rust-lang/cargo/issues/10280
      CARGO_NET_GIT_FETCH_WITH_CLI: "true"
      RUST_BACKTRACE: "1"
      # Run integration tests
      TEST_INTEGRATION: 1
      INFLUXDB_IOX_INTEGRATION_LOCAL: 1
      KAFKA_CONNECT: "localhost:9092"
      AWS_DEFAULT_REGION: "us-east-1"
      AWS_ACCESS_KEY_ID: test
      AWS_SECRET_ACCESS_KEY: test
      AWS_ENDPOINT: http://127.0.0.1:4566
      AZURE_USE_EMULATOR: "1"
      INFLUXDB_IOX_BUCKET: iox-test
      POSTGRES_USER: postgres
      DATABASE_URL: "postgres://postgres@localhost/iox_shared"
    steps:
      - run:
          name: Setup localstack (AWS emulation)
          command: |
            cd /tmp
            curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip"
            unzip awscliv2.zip
            sudo ./aws/install
            aws --endpoint-url=http://localhost:4566 s3 mb s3://iox-test
      - run:
          name: Setup Azurite (Azure emulation)
          # the magical connection string is from https://docs.microsoft.com/en-us/azure/storage/common/storage-use-azurite?tabs=visual-studio#http-connection-strings
          command: |
            curl -sL https://aka.ms/InstallAzureCLIDeb | sudo bash
            az storage container create -n iox-test --connection-string 'DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://127.0.0.1:10000/devstoreaccount1;QueueEndpoint=http://127.0.0.1:10001/devstoreaccount1;'
      - checkout
      - rust_components
      - cache_restore
      - run: cargo install sqlx-cli
      - run: sqlx database create
      - run:
          name: Cargo test
          command: cargo test --workspace --features=aws,azure,azure_test
      - cache_save

  # end to end tests with Heappy (heap profiling enabled)
  test_heappy:
    docker:
      - image: quay.io/influxdb/rust:ci
    resource_class: xlarge  # use of a smaller executor tends crashes on link
    environment:
      # Disable incremental compilation to avoid overhead. We are not preserving these files anyway.
      CARGO_INCREMENTAL: "0"
      # Disable full debug symbol generation to speed up CI build
      # "1" means line tables only, which is useful for panic tracebacks.
      RUSTFLAGS: "-C debuginfo=1"
      # https://github.com/rust-lang/cargo/issues/10280
      CARGO_NET_GIT_FETCH_WITH_CLI: "true"
      RUST_BACKTRACE: "1"
    steps:
      - checkout
      - rust_components
      - cache_restore
      - run:
          name: End to end tests (with heappy)
          command: cargo test --no-default-features --features=heappy end_to_end
      - cache_save

  test_perf:
    machine:
      image: ubuntu-2004:202107-02
    resource_class: xlarge
    environment:
      # Disable incremental compilation to avoid overhead. We are not preserving these files anyway.
      CARGO_INCREMENTAL: "0"
      # Disable full debug symbol generation to speed up CI build
      # "1" means line tables only, which is useful for panic tracebacks.
      RUSTFLAGS: "-C debuginfo=1"
      # https://github.com/rust-lang/cargo/issues/10280
      CARGO_NET_GIT_FETCH_WITH_CLI: "true"
      RUST_BACKTRACE: "1"
    steps:
      # Use us.archive.ubuntu.com since it is close. If it is down, try archive.ubuntu.com. If it is
      # down too, temporarily choose something from http://mirrors.ubuntu.com/mirrors.txt that
      # is up to date.
      - run:
          name: Setup ubuntu mirror
          command: |
            mirror="http://us.archive.ubuntu.com/ubuntu"
            sudo sed -i "s;http://.*ubuntu.com/ubuntu/;$mirror;g" /etc/apt/sources.list
      - run: sudo apt update
      - run:
          name: Install required compile tools
          command: sudo apt install curl clang lld pkg-config libssl-dev --yes --no-install-recommends
      - checkout

      - run:
          name: Install rustup
          command: scripts/install_rustup.sh

      - run:
          name: Install perf.py dependencies
          command: python3 -m pip install -r perf/requirements.txt

      - restore_cache:
          keys:
            - cargo-lock-{{ checksum "Cargo.lock" }}
      - run:
          name: Prime Rust build cache
          command: cargo build --package influxdb_iox --bin influxdb_iox --package iox_data_generator --bin iox_data_generator
      - save_cache:
          key: cargo-lock-{{ checksum "Cargo.lock" }}
          paths:
            - target

      - run:
          name: Run basic perf test to ensure that perf.py works
          command: |
            trap "cat perf/logs/test.log" ERR
            perf/perf.py --debug --no-volumes --object-store memory battery-0

  # Build a dev binary.
  #
  # Compiles a binary with the default ("dev") cargo profile from the iox source
  # using the latest ci_image.
  build_dev:
    docker:
      - image: quay.io/influxdb/rust:ci
    resource_class: xlarge  # use of a smaller executor tends crashes on link
    environment:
      # Disable incremental compilation to avoid overhead. We are not preserving these files anyway.
      CARGO_INCREMENTAL: "0"
      # Disable full debug symbol generation to speed up CI build
      # "1" means line tables only, which is useful for panic tracebacks.
      RUSTFLAGS: "-C debuginfo=1"
      # https://github.com/rust-lang/cargo/issues/10280
      CARGO_NET_GIT_FETCH_WITH_CLI: "true"
    steps:
      - checkout
      - rust_components
      - cache_restore
      - run:
          name: Cargo build
          command: cargo build --workspace
      - run:
          name: Build benches
          command: cargo test --workspace --benches --no-run
      - run:
          name: Build with object store + exporter support + HEAP profiling
          command: cargo build --no-default-features --features="aws,gcp,azure,heappy,pprof"
      - cache_save

  # Lint protobufs.
  protobuf-lint:
    docker:
      - image: bufbuild/buf:0.40.0
    environment:
      # Value to look for to skip breaking changes check
      SKIP_LABEL: "https://api.github.com/repos/influxdata/influxdb_iox/labels/incompatible%20protobuf"
    steps:
      - checkout
      - run:
          name: buf lint
          command: buf lint
      - run:
          name: buf breaking changes
          command: |
            echo "If you want to make changes forbidden by this lint, please"
            echo "coordinate with the conductor team, add the 'incompatible protobuf' label"
            echo "to the PR, and rerun this test"
            # Check if label is present using github API:
            # Inspired by https://discuss.circleci.com/t/tag-label-filter/11158
            if wget -O -  https://api.github.com/repos/influxdata/influxdb_iox/issues/$(echo $CIRCLE_PULL_REQUEST | grep -oE "[^/pull]+$") | grep "$SKIP_LABEL" ; then echo "SKIPPING (FOUND LABEL)" && exit ; else echo "CHECKING (NO LABEL FOUND)"; fi
            git fetch origin main
            # compare against only changes in this branch (not against
            # other stuff that may have been added to main since last merge)
            MERGE_BASE=$(git merge-base origin/main $CIRCLE_BRANCH) sh -c 'buf breaking --against ".git#ref=$MERGE_BASE"'

  # Lint docs
  docs-lint:
    docker:
      - image: python:3-slim-bullseye
    steps:
      # need a proper SSH client for checkout
      # https://discuss.circleci.com/t/circleci-user-key-uses-rsa-with-sha1-which-github-has-deprecated/42548
      - run:
          name: Install GIT and SSH
          command: |
            apt-get update
            apt-get install -y git ssh
      - checkout
      - run:
          name: Lint docs
          command: ./scripts/lint_docs.py ./docs

  # Compile cargo "release" profile binaries for iox & data generator
  #
  # Uses the latest ci_image (influxdb/rust below) to build a release binary and
  # copies it to a minimal container image based upon `rust:slim-buster`. This
  # minimal image is then pushed to `quay.io/influxdb/iox:${BRANCH}`.
  build_release:
    # need a machine executor to have a full-powered docker daemon (the `setup_remote_docker` system just provides a
    # kinda small node)
    machine:
      image: ubuntu-2004:202111-01
    resource_class: xlarge  # use of a smaller executor tends crashes on link
    environment:
      # Disable incremental compilation to avoid overhead. We are not preserving these files anyway.
      CARGO_INCREMENTAL: "0"
      # We keep the debug symbols (Enabled in Cargo.toml as debug = true)
      # workaround dynamic CPU detection bug in croaring
      # https://github.com/influxdata/influxdb_iox/pull/2119
      ROARING_ARCH: "haswell"
      # Use avx512 instructions to take full advantage of the CPUs instruction set
      RUSTFLAGS: "-C target-feature=+avx2"
      # https://github.com/rust-lang/cargo/issues/10280
      CARGO_NET_GIT_FETCH_WITH_CLI: "true"
    steps:
      - checkout
      - run: |
          echo "$QUAY_INFLUXDB_IOX_PASS" | docker login quay.io --username $QUAY_INFLUXDB_IOX_USER --password-stdin
      - run:
          name: Cargo release build with target arch set for CRoaring
          command: |
            COMMIT_SHA="$(git rev-parse --short HEAD)"

            RUST_VERSION="$(sed -E -ne 's/channel = "(.*)"/\1/p' rust-toolchain.toml)"
            BRANCH="$(echo "$CIRCLE_BRANCH" | tr '[:upper:]' '[:lower:]' | sed 's/[^a-z0-9]/_/g')"

            docker buildx build \
              --build-arg RUST_VERSION="$RUST_VERSION" \
              --build-arg RUSTFLAGS="-C target-feature=+avx2 -C link-arg=-fuse-ld=lld" \
              --progress plain \
              --tag quay.io/influxdb/iox:"$COMMIT_SHA" \
              --tag quay.io/influxdb/iox:"$BRANCH" \
              .
            docker buildx build \
              --build-arg FEATURES="" \
              --build-arg PACKAGE="iox_data_generator" \
              --build-arg RUST_VERSION="$RUST_VERSION" \
              --build-arg RUSTFLAGS="-C target-feature=+avx2 -C link-arg=-fuse-ld=lld" \
              --progress plain \
              --tag quay.io/influxdb/iox_data_generator:"$COMMIT_SHA" \
              --tag quay.io/influxdb/iox_data_generator:"$BRANCH" \
              .
            docker buildx build \
              --build-arg FEATURES="" \
              --build-arg PACKAGE="iox_gitops_adapter" \
              --build-arg RUST_VERSION="$RUST_VERSION" \
              --build-arg RUSTFLAGS="-C target-feature=+avx2 -C link-arg=-fuse-ld=lld" \
              --progress plain \
              --tag quay.io/influxdb/iox_gitops_adapter:"$COMMIT_SHA" \
              --tag quay.io/influxdb/iox_gitops_adapter:"$BRANCH" \
              .

            docker run -it --rm quay.io/influxdb/iox:$COMMIT_SHA debug print-cpu

            docker push --all-tags quay.io/influxdb/iox
            docker push --all-tags quay.io/influxdb/iox_data_generator
            docker push --all-tags quay.io/influxdb/iox_gitops_adapter

            echo "export COMMIT_SHA=${COMMIT_SHA}" >> $BASH_ENV
          # linking might take a while and doesn't produce CLI output
          no_output_timeout: 20m
      - run:
          name: Deploy tags
          command: |
            echo "$QUAY_PASS" | docker login quay.io --username $QUAY_USER --password-stdin
            ./.circleci/get-deploy-tags.sh "${COMMIT_SHA}"
      - cache_save

  # Prepare the CI image used for other tasks.
  #
  # A nightly job (scheduled below in the `workflows` section) to build the CI
  # image (influxdb/rust) used for the rest of the checks.
  #
  # To modify the contents of the CI image, update docker/Dockerfile.ci
  ci_image:
    # Use a `machine` executor instead of a docker-based because:
    # - we bootstrap our CI images somehow (we could use a docker-hub image as well)
    # - the machine executor also supports multi-arch builds
    # - we only run this job once a day, so the additional startup delay of 30-60s doesn't really matter
    machine:
      # https://circleci.com/docs/2.0/configuration-reference/#available-machine-images
      image: ubuntu-2004:202107-02
    resource_class: xlarge
    steps:
      - checkout
      - run: |
          echo "$QUAY_PASS" | docker login quay.io --username $QUAY_USER --password-stdin
      - run: |
          COMMIT_SHA=$(git rev-parse --short HEAD)
          RUST_VERSION=$(sed -E -ne 's/channel = "(.*)"/\1/p' rust-toolchain.toml)
          docker --version
          docker build -t quay.io/influxdb/rust:$COMMIT_SHA -t quay.io/influxdb/rust:ci -f docker/Dockerfile.ci --build-arg RUST_VERSION=$RUST_VERSION .
          docker push --all-tags quay.io/influxdb/rust


parameters:
  ci_image:
    description: "Trigger build of CI image"
    type: boolean
    default: false
  build_perf:
    # see comments below in build_perf job for usage
    description: "Trigger build of perf image"
    type: boolean
    default: false

workflows:
  version: 2

  # CI for all pull requests.
  ci:
    when:
      not: << pipeline.parameters.ci_image >>
    jobs:
      - fmt
      - lint
      - cargo_audit
      - protobuf-lint
      - docs-lint
      - test
      - test_heappy
      - test_perf
      - build_dev
      - doc
      - workspace_hack_checks
      - build_release:
          filters:
            branches:
              only: main
          requires:  # Only do a release build if all tests have passed
            - fmt
            - lint
            - cargo_audit
            - protobuf-lint
            - docs-lint
            - test
            - test_heappy
            - test_perf
            - build_dev
            - doc

  # Manual build of CI image
  ci_image:
    when: << pipeline.parameters.ci_image >>
    jobs:
      - ci_image

  # Manual build of release image for a branch.
  # Trigger using the CircleCI API, like so:
  #
  #   # e.g. using 'xh' (https://github.com/ducaale/xh)
  #   $ xh -a '<your personal circleCI token>:' POST \
  #       https://circleci.com/api/v2/project/github/influxdata/influxdb_iox/pipeline \
  #       parameters:='{"build_perf": true}' branch=chore/ci-tidy-up
  #
  # ...or equivalent with `curl`:
  #   $ curl -XPOST -H "Content-Type: application/json" -H "Circle-Token: <your personal circleCI token>" \
  #       -d '{"parameters": {"build_perf": true}, "branch": "chore/ci-tidy-up"}' \
  #       https://circleci.com/api/v2/project/github/influxdata/influxdb_iox/pipeline
  build_perf:
    when: << pipeline.parameters.build_perf >>
    jobs:
      - build_release

  # Nightly rebuild of the build container
  ci_image_nightly:
    triggers:
      - schedule:
          cron: "0 5 * * *"
          filters:
            branches:
              only:
                - main
    jobs:
      - ci_image