influxdb/.circleci/config.yml

# CI Overview
# -----------
#
# Each night:
#
#   A build image is created (ci_image) from `docker/Dockerfile.ci` and is
#   pushed to `quay.io/influxdb/rust:ci`. This build image is then used to run
#   the CI tasks for the day.
#
# CI runs for git branches ending in `/perf`:
#
#   Runs tests, fmt, & lints and then compiles binaries using the "release"
#   cargo target and pushes a container with the binary to
#   `quay.io/influxdb/fusion` (see perf_image below).
#
# CI for all other branches:
#
#   - cargo build with the default cargo profile ("dev")
#   - cargo test
#   - cargo fmt
#   - clippy (with warnings denied)
#   - lint protobufs
#   - check if generated flatbuffers code is up to date

version: 2.1

commands:
  rust_components:
    description: Verify installed components
    steps:
      - run:
          name: Verify installed components
          command: |
            rustup --version
            rustup show
            cargo fmt --version
            cargo clippy --version

  cache_restore:
    description: Restore Cargo Cache
    steps:
      - restore_cache:
          name: Restoring Cargo Cache
          keys:
            - cargo-cache-{{ arch }}-{{ .Branch }}-{{ checksum "Cargo.lock" }}
            - cargo-cache-{{ arch }}-{{ .Branch }}
            - cargo-cache
  cache_save:
    description: Save Cargo Cache
    steps:
      - save_cache:
          name: Save Cargo Cache
          paths:
            - /usr/local/cargo/registry
          key: cargo-cache-{{ arch }}-{{ .Branch }}-{{ checksum "Cargo.lock" }}

jobs:
  fmt:
    docker:
      - image: quay.io/influxdb/rust:ci
    environment:
      # Disable incremental compilation to avoid overhead. We are not preserving these files anyway.
      CARGO_INCREMENTAL: "0"
      # Disable full debug symbol generation to speed up CI build
      # "1" means line tables only, which is useful for panic tracebacks.
      RUSTFLAGS: "-C debuginfo=1"
    steps:
      - checkout
      - rust_components
      - cache_restore
      - run:
          name: Rust fmt
          command: cargo fmt --all -- --check
      - cache_save
  lint:
    docker:
      - image: quay.io/influxdb/rust:ci
    environment:
      # Disable incremental compilation to avoid overhead. We are not preserving these files anyway.
      CARGO_INCREMENTAL: "0"
      # Disable full debug symbol generation to speed up CI build
      # "1" means line tables only, which is useful for panic tracebacks.
      RUSTFLAGS: "-C debuginfo=1"
    steps:
      - checkout
      - rust_components
      - cache_restore
      - run:
          name: Clippy
          command: cargo clippy --all-targets --workspace -- -D warnings
      - cache_save
  cargo_audit:
    docker:
      - image: quay.io/influxdb/rust:ci
    environment:
      # Disable incremental compilation to avoid overhead. We are not preserving these files anyway.
      CARGO_INCREMENTAL: "0"
      # Disable full debug symbol generation to speed up CI build
      # "1" means line tables only, which is useful for panic tracebacks.
      RUSTFLAGS: "-C debuginfo=1"
    steps:
      - checkout
      - rust_components
      - cache_restore
      - run:
          name: Install Cargo Audit
          command: cargo install --force cargo-audit
      - run:
          name: Cargo Audit
          command: cargo audit
      - cache_save
  doc:
    docker:
      - image: quay.io/influxdb/rust:ci
    environment:
      # Disable incremental compilation to avoid overhead. We are not preserving these files anyway.
      CARGO_INCREMENTAL: "0"
      # Disable full debug symbol generation to speed up CI build
      # "1" means line tables only, which is useful for panic tracebacks.
      RUSTFLAGS: "-C debuginfo=1"
    steps:
      - checkout
      - rust_components
      - cache_restore
      - run:
          name: Cargo doc
          # excluding datafusion because it's effectively a dependency masqueraded as workspace crate.
          command: cargo doc --document-private-items --no-deps --workspace --exclude datafusion
      - cache_save
      - run:
          name: Compress Docs
          command: tar -cvzf rustdoc.tar.gz target/doc/
      - store_artifacts:
          path: rustdoc.tar.gz

  test:
    docker:
      - image: quay.io/influxdb/rust:ci
    resource_class: xlarge # use of a smaller executor tends crashes on link
    environment:
      # Disable incremental compilation to avoid overhead. We are not preserving these files anyway.
      CARGO_INCREMENTAL: "0"
      # Disable full debug symbol generation to speed up CI build
      # "1" means line tables only, which is useful for panic tracebacks.
      RUSTFLAGS: "-C debuginfo=1"
      RUST_BACKTRACE: "1"
    steps:
      - checkout
      - rust_components
      - cache_restore
      - run:
          name: Cargo test
          command: cargo test --workspace
      - cache_save

  # end to end tests with Heappy (heap profiling enabled)
  test_heappy:
    docker:
      - image: quay.io/influxdb/rust:ci
    resource_class: xlarge # use of a smaller executor tends crashes on link
    environment:
      # Disable incremental compilation to avoid overhead. We are not preserving these files anyway.
      CARGO_INCREMENTAL: "0"
      # Disable full debug symbol generation to speed up CI build
      # "1" means line tables only, which is useful for panic tracebacks.
      RUSTFLAGS: "-C debuginfo=1"
      RUST_BACKTRACE: "1"
    steps:
      - checkout
      - rust_components
      - cache_restore
      - run:
          name: End to end tests (with heappy)
          command: cargo test --no-default-features --features=heappy end_to_end
      - cache_save

  test_perf:
    machine:
      image: ubuntu-2004:202107-02
    resource_class: xlarge
    environment:
      # Disable incremental compilation to avoid overhead. We are not preserving these files anyway.
      CARGO_INCREMENTAL: "0"
      # Disable full debug symbol generation to speed up CI build
      # "1" means line tables only, which is useful for panic tracebacks.
      RUSTFLAGS: "-C debuginfo=1"
      RUST_BACKTRACE: "1"
    steps:
      - run: sudo apt update
      - run:
          name: Install required compile tools
          command: sudo apt install curl clang lld pkg-config libssl-dev --yes --no-install-recommends
      - checkout

      - run:
          name: Install rustup
          command: scripts/install_rustup.sh

      - run:
          name: Install perf.py dependencies
          command: python3 -m pip install -r perf/requirements.txt

      - restore_cache:
          keys:
            - cargo-lock-{{ checksum "Cargo.lock" }}
      - run:
          name: Prime Rust build cache
          command: cargo build --package influxdb_iox --bin influxdb_iox --package iox_data_generator --bin iox_data_generator
      - save_cache:
          key: cargo-lock-{{ checksum "Cargo.lock" }}
          paths:
            - target

      - run:
          name: Run basic perf test to ensure that perf.py works
          command: perf/perf.py --debug --no-volumes --object-store memory battery-0

  test_kafka_integration:
    machine: true
    resource_class: xlarge
    environment:
      # Disable incremental compilation to avoid overhead. We are not preserving these files anyway.
      CARGO_INCREMENTAL: "0"
      # Disable full debug symbol generation to speed up CI build
      # "1" means line tables only, which is useful for panic tracebacks.
      RUSTFLAGS: "-C debuginfo=1"
      RUST_BACKTRACE: "1"
    steps:
      - checkout
      - run:
          name: Run integration tests with Docker Compose
          command: docker-compose -f docker/ci-kafka-docker-compose.yml up --build --force-recreate --exit-code-from rust

  # Integration tests for the influxdb2_client crate against InfluxDB 2.0 OSS.
  test_influxdb2_client:
    docker:
      - image: quay.io/influxdb/rust:ci
    environment:
      # Disable incremental compilation to avoid overhead. We are not preserving these files anyway.
      CARGO_INCREMENTAL: "0"
      # Disable full debug symbol generation to speed up CI build
      # "1" means line tables only, which is useful for panic tracebacks.
      RUSTFLAGS: "-C debuginfo=1"
      RUST_BACKTRACE: "1"
    steps:
      - checkout
      - rust_components
      - cache_restore
      - run:
          name: Cargo test
          command: TEST_INTEGRATION=1 INFLUXDB_IOX_INTEGRATION_LOCAL=1 cargo test -p influxdb2_client
      - cache_save

  # Build a dev binary.
  #
  # Compiles a binary with the default ("dev") cargo profile from the iox source
  # using the latest ci_image.
  build:
    docker:
      - image: quay.io/influxdb/rust:ci
    resource_class: xlarge # use of a smaller executor tends crashes on link
    environment:
      # Disable incremental compilation to avoid overhead. We are not preserving these files anyway.
      CARGO_INCREMENTAL: "0"
      # Disable full debug symbol generation to speed up CI build
      # "1" means line tables only, which is useful for panic tracebacks.
      RUSTFLAGS: "-C debuginfo=1"
    steps:
      - checkout
      - rust_components
      - cache_restore
      - run:
          name: Cargo build
          command: cargo build --workspace
      - run:
          name: Build benches
          command: cargo test --workspace --benches --no-run
      - run:
          name: Build with object store + exporter support + HEAP profiling
          command: cargo build --no-default-features --features="aws,gcp,azure,heappy,pprof"
      - cache_save

  # Lint protobufs.
  protobuf-lint:
    docker:
      - image: bufbuild/buf:0.40.0
    environment:
      # Value to look for to skip breaking changes check
      SKIP_LABEL: "https://api.github.com/repos/influxdata/influxdb_iox/labels/incompatible%20protobuf"
    steps:
      - checkout
      - run:
          name: buf lint
          command: buf lint
      - run:
          name: buf breaking changes
          command: |
            echo "If you want to make changes forbidden by this lint, please"
            echo "coordinate with the conductor team, add the 'incompatible protobuf' label"
            echo "to the PR, and rerun this test"
            # Check if label is present using github API:
            # Inspired by https://discuss.circleci.com/t/tag-label-filter/11158
            if wget -O -  https://api.github.com/repos/influxdata/influxdb_iox/issues/$(echo $CIRCLE_PULL_REQUEST | grep -oE "[^/pull]+$") | grep "$SKIP_LABEL" ; then echo "SKIPPING (FOUND LABEL)" && exit ; else echo "CHECKING (NO LABEL FOUND)"; fi
            git fetch origin main
            # compare against only changes in this branch (not against
            # other stuff that may have been added to master since last merge)
            MERGE_BASE=$(git merge-base origin/main $CIRCLE_BRANCH) sh -c 'buf breaking --against ".git#ref=$MERGE_BASE"'

  # Check that any generated files are  is up-to-date with the changes in this PR.
  # named "check-flatbuffers" because that name is hardcoded into github checks
  check-flatbuffers:
    docker:
      - image: quay.io/influxdb/rust:ci
    resource_class: xlarge # use of a smaller executor tends crashes on link
    steps:
      - checkout
      - rust_components # Regenerating flatbuffers uses rustfmt
      - run:
          name: Check Query Tests
          command: ./query_tests/check-generated.sh
      - run:
          name: Check Flatbuffers
          command: INFLUXDB_IOX_INTEGRATION_LOCAL=1 ./entry/check-flatbuffers.sh

  # Compile a cargo "release" profile binary for branches that end in `/perf`
  #
  # Uses the latest ci_image (influxdb/rust below) to build a release binary and
  # copies it to a minimal container image based upon `rust:slim-buster`. This
  # minimal image is then pushed to `quay.io/influxdb/fusion:${BRANCH}` with '/'
  # repaced by '.' - as an example:
  #
  #   git branch: dom/my-awesome-feature/perf
  #   container: quay.io/influxdb/fusion:dom.my-awesome-feature.perf
  #
  # Subsequent CI runs will overwrite the tag if you push more changes, so watch
  # out for parallel CI runs!
  #
  # To change the contents of the build container, modify docker/Dockerfile.ci
  # To change the final release container, modify docker/Dockerfile.iox
  perf_image:
    docker:
      - image: quay.io/influxdb/rust:ci
    resource_class: xlarge # use of a smaller executor tends crashes on link
    environment:
      # Disable incremental compilation to avoid overhead. We are not preserving these files anyway.
      CARGO_INCREMENTAL: "0"
      # We keep the debug symbols (Enabled in Cargo.toml as debug = true)
      # workaround dynamic CPU detection bug in croaring
      # https://github.com/influxdata/influxdb_iox/pull/2119
      ROARING_ARCH: "haswell"
      # Use avx512 instructions to take full advantage of the CPUs instruction set
      RUSTFLAGS: "-C target-feature=+avx2"
    steps:
      - checkout
      - rust_components
      - cache_restore
      - run:
          name: Cargo release build with target arch set for CRoaring
          command: cargo build --release --no-default-features --features="aws,gcp,azure,jemalloc_replacing_malloc"
      - run: |
          echo sha256sum after build is
          sha256sum target/release/influxdb_iox
      - run:
          name: Print rustc target CPU options
          command: target/release/influxdb_iox debug print-cpu
      - setup_remote_docker:
          # There seems to be a cache invalidation bug in docker
          # or in the way that circleci implements layer caching.
          # Disabling for now, and tracked further investigations
          # in https://github.com/influxdata/k8s-idpe/issues/3038
          docker_layer_caching: false
          version: 19.03.14
      - run: |
          sudo apt-get update
          sudo apt-get install -y docker.io
      - run: |
          echo "$QUAY_PASS" | docker login quay.io --username $QUAY_USER --password-stdin
      - run:
          # Docker has functionality to support per-Dockerfile .dockerignore
          # This was added in https://github.com/moby/buildkit/pull/901
          # And included in 19.03 - https://docs.docker.com/engine/release-notes/19.03/#19030
          # Unfortunately CircleCI only seems to understand a root-level .dockerignore
          # So we need to move it into position for it to not send ~10GB of build context
          name: Fudge CircleCI Docker Context
          command: mv docker/Dockerfile.iox.dockerignore .dockerignore
      - run: |
          BRANCH=$(git rev-parse --abbrev-ref HEAD | tr '/' '.')
          echo sha256sum after build is
          sha256sum target/release/influxdb_iox
          COMMIT_SHA=$(git rev-parse --short HEAD)
          docker build -t quay.io/influxdb/fusion:$COMMIT_SHA -f docker/Dockerfile.iox .
          docker push quay.io/influxdb/fusion:$COMMIT_SHA
          echo "export COMMIT_SHA=${COMMIT_SHA}" >> $BASH_ENV
      - run:
          name: Deploy tags
          command: ./.circleci/get-deploy-tags.sh "${COMMIT_SHA}"
      - cache_save

  # Prepare the CI image used for other tasks.
  #
  # A nightly job (scheduled below in the `workflows` section) to build the CI
  # image (influxdb/rust) used for the rest of the checks.
  #
  # To modify the contents of the CI image, update docker/Dockerfile.ci
  ci_image:
    machine: true
    resource_class: xlarge
    steps:
      - checkout
      - run: |
          echo "$QUAY_PASS" | docker login quay.io --username $QUAY_USER --password-stdin
      - run: |
          COMMIT_SHA=$(git rev-parse --short HEAD)
          RUST_VERSION=$(sed -E -ne 's/channel = "(.*)"/\1/p' rust-toolchain.toml)
          docker build -t quay.io/influxdb/rust:$COMMIT_SHA -f docker/Dockerfile.ci --build-arg RUST_VERSION=$RUST_VERSION .
          docker tag quay.io/influxdb/rust:$COMMIT_SHA quay.io/influxdb/rust:ci
          docker push quay.io/influxdb/rust:$COMMIT_SHA
          docker push quay.io/influxdb/rust:ci

workflows:
  version: 2

  # CI for all pull requests.
  ci:
    jobs:
      - fmt
      - lint
      - cargo_audit
      - protobuf-lint
      - test
      - test_heappy
      - test_perf
      - test_kafka_integration
      - test_influxdb2_client
      - build
      - check-flatbuffers
      - doc

  # Internal pipeline for perf builds.
  #
  # Filter ensures this only runs for git branches ending in `/perf`.
  perf_build:
    jobs:
      - fmt:
          filters:
            branches:
              only: main
      - lint:
          filters:
            branches:
              only: main
      - cargo_audit:
          filters:
            branches:
              only: main
      - test:
          filters:
            branches:
              only: main
      - build:
          filters:
            branches:
              only: main
      - perf_image:
          filters:
            branches:
              only: main
          requires: # Only do a release build if all tests have passed
            - fmt
            - lint
            - cargo_audit
            - test
            - build

  # Nightly rebuild of the build container
  ci_image:
    triggers:
      - schedule:
          cron: "0 5 * * *"
          filters:
            branches:
              only:
                - main
    jobs:
      - ci_image