influxdb/.circleci/config.yml

# CI Overview
# -----------
#
# Each night:
#
#   A build image is created (ci_image) from `docker/Dockerfile.ci` and is
#   pushed to `quay.io/influxdb/rust:ci`. This build image is then used to run
#   the CI tasks for the day.
#
# CI runs for git branches ending in `/perf`:
#
#   Runs tests, fmt, & lints and then compiles binaries using the "release"
#   cargo target and pushes a container with the binary to
#   `quay.io/influxdb/fusion` (see perf_image below).
#
# CI for all other branches:
#
#   - cargo build with the default cargo profile ("dev")
#   - cargo test
#   - cargo fmt
#   - clippy (with warnings denied)
#   - lint protobufs
#   - check if generated flatbuffers code is up to date

version: 2.1

commands:
  rust_components:
    description: Verify installed components
    steps:
      - run:
          name: Verify installed components
          command: |
            rustup --version
            rustup show
            cargo fmt --version
            cargo clippy --version

  cache_restore:
    description: Restore Cargo Cache
    steps:
      - restore_cache:
          name: Restoring Cargo Cache
          keys:
            - cargo-cache-{{ arch }}-{{ .Branch }}-{{ checksum "Cargo.lock" }}
            - cargo-cache-{{ arch }}-{{ .Branch }}
            - cargo-cache
  cache_save:
    description: Save Cargo Cache
    steps:
      - save_cache:
          name: Save Cargo Cache
          paths:
            - /usr/local/cargo/registry
          key: cargo-cache-{{ arch }}-{{ .Branch }}-{{ checksum "Cargo.lock" }}

jobs:
  fmt:
    docker:
      - image: quay.io/influxdb/rust:ci
    environment:
      # Disable incremental compilation to avoid overhead. We are not preserving these files anyway.
      CARGO_INCREMENTAL: "0"
      # Disable full debug symbol generation to speed up CI build
      # "1" means line tables only, which is useful for panic tracebacks.
      RUSTFLAGS: "-C debuginfo=1"
    steps:
      - checkout
      - rust_components
      - cache_restore
      - run:
          name: Rust fmt
          command: cargo fmt --all -- --check
      - cache_save
  lint:
    docker:
      - image: quay.io/influxdb/rust:ci
    environment:
      # Disable incremental compilation to avoid overhead. We are not preserving these files anyway.
      CARGO_INCREMENTAL: "0"
      # Disable full debug symbol generation to speed up CI build
      # "1" means line tables only, which is useful for panic tracebacks.
      RUSTFLAGS: "-C debuginfo=1"
    steps:
      - checkout
      - rust_components
      - cache_restore
      - run:
          name: Clippy
          command: cargo clippy --all-targets --workspace -- -D warnings
      - cache_save
  cargo_audit:
    docker:
      - image: quay.io/influxdb/rust:ci
    environment:
      # Disable incremental compilation to avoid overhead. We are not preserving these files anyway.
      CARGO_INCREMENTAL: "0"
      # Disable full debug symbol generation to speed up CI build
      # "1" means line tables only, which is useful for panic tracebacks.
      RUSTFLAGS: "-C debuginfo=1"
    steps:
      - checkout
      - rust_components
      - cache_restore
      - run:
          name: Install Cargo Audit
          command: cargo install --force cargo-audit
      - run:
          name: Cargo Audit
          command: cargo audit
      - cache_save
  doc:
    docker:
      - image: quay.io/influxdb/rust:ci
    environment:
      # Disable incremental compilation to avoid overhead. We are not preserving these files anyway.
      CARGO_INCREMENTAL: "0"
      # Disable full debug symbol generation to speed up CI build
      # "1" means line tables only, which is useful for panic tracebacks.
      RUSTFLAGS: "-C debuginfo=1"
    steps:
      - checkout
      - rust_components
      - cache_restore
      - run:
          name: Cargo doc
          # excluding datafusion because it's effectively a dependency masqueraded as workspace crate.
          command: cargo doc --document-private-items --no-deps --workspace --exclude datafusion
      - cache_save
      - run:
          name: Compress Docs
          command: tar -cvzf rustdoc.tar.gz target/doc/
      - store_artifacts:
          path: rustdoc.tar.gz

  test:
    docker:
      - image: quay.io/influxdb/rust:ci
    resource_class: xlarge # use of a smaller executor tends crashes on link
    environment:
      # Disable incremental compilation to avoid overhead. We are not preserving these files anyway.
      CARGO_INCREMENTAL: "0"
      # Disable full debug symbol generation to speed up CI build
      # "1" means line tables only, which is useful for panic tracebacks.
      RUSTFLAGS: "-C debuginfo=1"
      RUST_BACKTRACE: "1"
    steps:
      - checkout
      - rust_components
      - cache_restore
      - run:
          name: Cargo test
          command: cargo test --workspace
      - cache_save

  # end to end tests with Heappy (heap profiling enabled)
  test_heappy:
    docker:
      - image: quay.io/influxdb/rust:ci
    resource_class: xlarge # use of a smaller executor tends crashes on link
    environment:
      # Disable incremental compilation to avoid overhead. We are not preserving these files anyway.
      CARGO_INCREMENTAL: "0"
      # Disable full debug symbol generation to speed up CI build
      # "1" means line tables only, which is useful for panic tracebacks.
      RUSTFLAGS: "-C debuginfo=1"
      RUST_BACKTRACE: "1"
    steps:
      - checkout
      - rust_components
      - cache_restore
      - run:
          name: End to end tests (with heappy)
          command: cargo test --no-default-features --features=heappy end_to_end
      - cache_save

  test_perf:
    machine:
      image: ubuntu-2004:202107-02
    resource_class: xlarge
    environment:
      # Disable incremental compilation to avoid overhead. We are not preserving these files anyway.
      CARGO_INCREMENTAL: "0"
      # Disable full debug symbol generation to speed up CI build
      # "1" means line tables only, which is useful for panic tracebacks.
      RUSTFLAGS: "-C debuginfo=1"
      RUST_BACKTRACE: "1"
    steps:
      - run: sudo apt update
      - run:
          name: Install required compile tools
          command: sudo apt install curl clang lld pkg-config libssl-dev --yes --no-install-recommends
      - checkout

      - run:
          name: Install rustup
          command: scripts/install_rustup.sh

      - run:
          name: Install perf.py dependencies
          command: python3 -m pip install -r perf/requirements.txt

      - restore_cache:
          keys:
            - cargo-lock-{{ checksum "Cargo.lock" }}
      - run:
          name: Prime Rust build cache
          command: cargo build --package influxdb_iox --bin influxdb_iox --package iox_data_generator --bin iox_data_generator
      - save_cache:
          key: cargo-lock-{{ checksum "Cargo.lock" }}
          paths:
            - target

      - run:
          name: Run basic perf test to ensure that perf.py works
          command: perf/perf.py --debug --no-volumes --object-store memory battery-0

  test_kafka_integration:
    machine: true
    resource_class: xlarge
    environment:
      # Disable incremental compilation to avoid overhead. We are not preserving these files anyway.
      CARGO_INCREMENTAL: "0"
      # Disable full debug symbol generation to speed up CI build
      # "1" means line tables only, which is useful for panic tracebacks.
      RUSTFLAGS: "-C debuginfo=1"
      RUST_BACKTRACE: "1"
    steps:
      - checkout
      - run:
          name: Run integration tests with Docker Compose
          command: docker-compose -f docker/ci-kafka-docker-compose.yml up --build --force-recreate --exit-code-from rust

  # Integration tests for the influxdb2_client crate against InfluxDB 2.0 OSS.
  test_influxdb2_client:
    docker:
      - image: quay.io/influxdb/rust:ci
    environment:
      # Disable incremental compilation to avoid overhead. We are not preserving these files anyway.
      CARGO_INCREMENTAL: "0"
      # Disable full debug symbol generation to speed up CI build
      # "1" means line tables only, which is useful for panic tracebacks.
      RUSTFLAGS: "-C debuginfo=1"
      RUST_BACKTRACE: "1"
    steps:
      - checkout
      - rust_components
      - cache_restore
      - run:
          name: Cargo test
          command: TEST_INTEGRATION=1 INFLUXDB_IOX_INTEGRATION_LOCAL=1 cargo test -p influxdb2_client
      - cache_save

  # Build a dev binary.
  #
  # Compiles a binary with the default ("dev") cargo profile from the iox source
  # using the latest ci_image.
  build:
    docker:
      - image: quay.io/influxdb/rust:ci
    resource_class: xlarge # use of a smaller executor tends crashes on link
    environment:
      # Disable incremental compilation to avoid overhead. We are not preserving these files anyway.
      CARGO_INCREMENTAL: "0"
      # Disable full debug symbol generation to speed up CI build
      # "1" means line tables only, which is useful for panic tracebacks.
      RUSTFLAGS: "-C debuginfo=1"
    steps:
      - checkout
      - rust_components
      - cache_restore
      - run:
          name: Cargo build
          command: cargo build --workspace
      - run:
          name: Build benches
          command: cargo test --workspace --benches --no-run
      - run:
          name: Build with object store + exporter support + HEAP profiling
          command: cargo build --no-default-features --features="aws,gcp,azure,heappy,pprof"
      - cache_save

  # Lint protobufs.
  protobuf-lint:
    docker:
      - image: bufbuild/buf:0.40.0
    steps:
      - checkout
      - run:
          name: buf lint
          command: buf lint

  # Check that any generated files are  is up-to-date with the changes in this PR.
  # named "check-flatbuffers" because that name is hardcoded into github checks
  check-flatbuffers:
    docker:
      - image: quay.io/influxdb/rust:ci
    resource_class: xlarge # use of a smaller executor tends crashes on link
    steps:
      - checkout
      - rust_components # Regenerating flatbuffers uses rustfmt
      - run:
          name: Check Query Tests
          command: ./query_tests/check-generated.sh
      - run:
          name: Check Flatbuffers
          command: INFLUXDB_IOX_INTEGRATION_LOCAL=1 ./entry/check-flatbuffers.sh

  # Compile a cargo "release" profile binary for branches that end in `/perf`
  #
  # Uses the latest ci_image (influxdb/rust below) to build a release binary and
  # copies it to a minimal container image based upon `rust:slim-buster`. This
  # minimal image is then pushed to `quay.io/influxdb/fusion:${BRANCH}` with '/'
  # repaced by '.' - as an example:
  #
  #   git branch: dom/my-awesome-feature/perf
  #   container: quay.io/influxdb/fusion:dom.my-awesome-feature.perf
  #
  # Subsequent CI runs will overwrite the tag if you push more changes, so watch
  # out for parallel CI runs!
  #
  # To change the contents of the build container, modify docker/Dockerfile.ci
  # To change the final release container, modify docker/Dockerfile.iox
  perf_image:
    docker:
      - image: quay.io/influxdb/rust:ci
    resource_class: xlarge # use of a smaller executor tends crashes on link
    environment:
      # Disable incremental compilation to avoid overhead. We are not preserving these files anyway.
      CARGO_INCREMENTAL: "0"
      # We keep the debug symbols (Enabled in Cargo.toml as debug = true)
      # workaround dynamic CPU detection bug in croaring
      # https://github.com/influxdata/influxdb_iox/pull/2119
      ROARING_ARCH: "haswell"
      # Use avx512 instructions to take full advantage of the CPUs instruction set
      RUSTFLAGS: "-C target-feature=+avx2"
    steps:
      - checkout
      - rust_components
      - cache_restore
      - run:
          name: Print rustc target CPU options
          command: cargo run --release --no-default-features --features="aws,gcp,azure,jemalloc_replacing_malloc" --bin print_cpu
      - run:
          name: Cargo release build with target arch set for CRoaring
          command: cargo build --release --no-default-features --features="aws,gcp,azure,jemalloc_replacing_malloc"
      - run: |
          echo sha256sum after build is
          sha256sum target/release/influxdb_iox
      - setup_remote_docker:
          # There seems to be a cache invalidation bug in docker
          # or in the way that circleci implements layer caching.
          # Disabling for now, and tracked further investigations
          # in https://github.com/influxdata/k8s-idpe/issues/3038
          docker_layer_caching: false
          version: 19.03.14
      - run: |
          sudo apt-get update
          sudo apt-get install -y docker.io
      - run: |
          echo "$QUAY_PASS" | docker login quay.io --username $QUAY_USER --password-stdin
      - run:
          # Docker has functionality to support per-Dockerfile .dockerignore
          # This was added in https://github.com/moby/buildkit/pull/901
          # And included in 19.03 - https://docs.docker.com/engine/release-notes/19.03/#19030
          # Unfortunately CircleCI only seems to understand a root-level .dockerignore
          # So we need to move it into position for it to not send ~10GB of build context
          name: Fudge CircleCI Docker Context
          command: mv docker/Dockerfile.iox.dockerignore .dockerignore
      - run: |
          BRANCH=$(git rev-parse --abbrev-ref HEAD | tr '/' '.')
          echo sha256sum after build is
          sha256sum target/release/influxdb_iox
          COMMIT_SHA=$(git rev-parse --short HEAD)
          docker build -t quay.io/influxdb/fusion:$COMMIT_SHA -f docker/Dockerfile.iox .
          docker push quay.io/influxdb/fusion:$COMMIT_SHA
          echo "export COMMIT_SHA=${COMMIT_SHA}" >> $BASH_ENV
      - run:
          name: Deploy tags
          command: ./.circleci/get-deploy-tags.sh "${COMMIT_SHA}"
      - cache_save

  # Prepare the CI image used for other tasks.
  #
  # A nightly job (scheduled below in the `workflows` section) to build the CI
  # image (influxdb/rust) used for the rest of the checks.
  #
  # To modify the contents of the CI image, update docker/Dockerfile.ci
  ci_image:
    machine: true
    resource_class: xlarge
    steps:
      - checkout
      - run: |
          echo "$QUAY_PASS" | docker login quay.io --username $QUAY_USER --password-stdin
      - run: |
          COMMIT_SHA=$(git rev-parse --short HEAD)
          RUST_VERSION=$(sed -E -ne 's/channel = "(.*)"/\1/p' rust-toolchain.toml)
          docker build -t quay.io/influxdb/rust:$COMMIT_SHA -f docker/Dockerfile.ci --build-arg RUST_VERSION=$RUST_VERSION .
          docker tag quay.io/influxdb/rust:$COMMIT_SHA quay.io/influxdb/rust:ci
          docker push quay.io/influxdb/rust:$COMMIT_SHA
          docker push quay.io/influxdb/rust:ci

workflows:
  version: 2

  # CI for all pull requests.
  ci:
    jobs:
      - fmt
      - lint
      - cargo_audit
      - protobuf-lint
      - test
      - test_heappy
      - test_perf
      - test_kafka_integration
      - test_influxdb2_client
      - build
      - check-flatbuffers
      - doc

  # Internal pipeline for perf builds.
  #
  # Filter ensures this only runs for git branches ending in `/perf`.
  perf_build:
    jobs:
      - fmt:
          filters:
            branches:
              only: main
      - lint:
          filters:
            branches:
              only: main
      - cargo_audit:
          filters:
            branches:
              only: main
      - test:
          filters:
            branches:
              only: main
      - build:
          filters:
            branches:
              only: main
      - perf_image:
          filters:
            branches:
              only: main
          requires: # Only do a release build if all tests have passed
            - fmt
            - lint
            - cargo_audit
            - test
            - build

  # Nightly rebuild of the build container
  ci_image:
    triggers:
      - schedule:
          cron: "0 5 * * *"
          filters:
            branches:
              only:
                - main
    jobs:
      - ci_image