influxdb/.circleci/config.yml

# CI Overview
# -----------
#
# Each night:
#
#   A build image is created (ci_image) from `docker/Dockerfile.ci` and is
#   pushed to `quay.io/influxdb/rust:ci`. This build image is then used to run
#   the CI tasks for the day.
#
# CI runs for git branches ending in `/perf`:
#
#   Runs tests, fmt, & lints and then compiles binaries using the "release"
#   cargo target and pushes a container with the binary to
#   `quay.io/influxdb/fusion` (see perf_image below).
#
# CI for all other branches:
#
#   - cargo build with the default cargo profile ("dev")
#   - cargo test
#   - cargo fmt
#   - clippy (with warnings denied)
#   - lint protobufs
#   - check if generated flatbuffers code is up to date

version: 2.1

commands:
  rust_components:
    description: Install additional toolchain components
    steps:
      - run:
          # Cargo will automatically install the toolchain based on rust-toolchain
          # But won't install optional components
          # This will only matter when the CI image is out of date
          name: Install rustfmt and clippy
          command: rustup component add rustfmt clippy

  cache_restore:
    description: Restore Cargo Cache
    steps:
      - restore_cache:
          name: Restoring Cargo Cache
          keys:
            - cargo-cache-{{ arch }}-{{ .Branch }}-{{ checksum "Cargo.lock" }}
            - cargo-cache-{{ arch }}-{{ .Branch }}
            - cargo-cache
  cache_save:
    description: Save Cargo Cache
    steps:
      - save_cache:
          name: Save Cargo Cache
          paths:
            - /usr/local/cargo/registry
          key: cargo-cache-{{ arch }}-{{ .Branch }}-{{ checksum "Cargo.lock" }}

jobs:
  fmt:
    docker:
      - image: quay.io/influxdb/rust:ci
    environment:
        # Disable full debug symbol generation to speed up CI build
        # "1" means line tables only, which is useful for panic tracebacks.
        RUSTFLAGS: "-C debuginfo=1"
    steps:
      - checkout
      - rust_components
      - cache_restore
      - run:
          name: Rust fmt
          command: cargo fmt --all -- --check
      - cache_save
  lint:
    docker:
      - image: quay.io/influxdb/rust:ci
    environment:
        # Disable full debug symbol generation to speed up CI build
        # "1" means line tables only, which is useful for panic tracebacks.
        RUSTFLAGS: "-C debuginfo=1"
    steps:
      - checkout
      - rust_components
      - cache_restore
      - run:
          name: Clippy
          command: cargo clippy --all-targets --workspace -- -D warnings
      - cache_save
  doc:
    docker:
      - image: quay.io/influxdb/rust:ci
    environment:
        # Disable full debug symbol generation to speed up CI build
        # "1" means line tables only, which is useful for panic tracebacks.
        RUSTFLAGS: "-C debuginfo=1"
    steps:
      - checkout
      - rust_components
      - cache_restore
      - run:
          name: Cargo doc
          command: cargo doc --document-private-items --no-deps --workspace
      - cache_save
  test:
    docker:
      - image: quay.io/influxdb/rust:ci
    resource_class: xlarge # use of a smaller executor tends crashes on link
    environment:
        # Disable full debug symbol generation to speed up CI build
        # "1" means line tables only, which is useful for panic tracebacks.
        RUSTFLAGS: "-C debuginfo=1"
    steps:
      - checkout
      - rust_components
      - cache_restore
      - run:
          name: Cargo test
          command: cargo test --workspace
      - cache_save

  # Integration tests for the influxdb2_client crate against InfluxDB 2.0 OSS.
  test_influxdb2_client:
    docker:
      - image: quay.io/influxdb/rust:ci
    environment:
        # Disable full debug symbol generation to speed up CI build
        # "1" means line tables only, which is useful for panic tracebacks.
        RUSTFLAGS: "-C debuginfo=1"
    steps:
      - checkout
      - rust_components
      - cache_restore
      - run:
          name: Cargo test
          command: TEST_INTEGRATION=1 INFLUXDB_IOX_INTEGRATION_LOCAL=1 cargo test -p influxdb2_client
      - cache_save

  # Build a dev binary.
  #
  # Compiles a binary with the default ("dev") cargo profile from the iox source
  # using the latest ci_image.
  build:
    docker:
      - image: quay.io/influxdb/rust:ci
    resource_class: xlarge # use of a smaller executor tends crashes on link
    environment:
        # Disable full debug symbol generation to speed up CI build
        # "1" means line tables only, which is useful for panic tracebacks.
        RUSTFLAGS: "-C debuginfo=1"
    steps:
      - checkout
      - rust_components
      - cache_restore
      - run:
          name: Cargo build
          command: cargo build --workspace
      - run:
          name: Build benches
          command: cargo test --workspace --benches --no-run
      - cache_save

  # Lint protobufs.
  protobuf-lint:
    docker:
      - image: bufbuild/buf:0.40.0
    steps:
      - checkout
      - run:
          name: buf lint
          command: buf lint

  # Check that the generated flatbuffers code is up-to-date with the changes in this PR.
  check-flatbuffers:
    docker:
      - image: quay.io/influxdb/rust:ci
    resource_class: xlarge # use of a smaller executor tends crashes on link
    steps:
      - checkout
      - rust_components # Regenerating flatbuffers uses rustfmt
      - run:
          name: Check Flatbuffers
          command: INFLUXDB_IOX_INTEGRATION_LOCAL=1 ./generated_types/check-flatbuffers.sh

  # Compile a cargo "release" profile binary for branches that end in `/perf`
  #
  # Uses the latest ci_image (influxdb/rust below) to build a release binary and
  # copies it to a minimal container image based upon `rust:slim-buster`. This
  # minimal image is then pushed to `quay.io/influxdb/fusion:${BRANCH}` with '/'
  # repaced by '.' - as an example:
  #
  #   git branch: dom/my-awesome-feature/perf
  #   container: quay.io/influxdb/fusion:dom.my-awesome-feature.perf
  #
  # Subsequent CI runs will overwrite the tag if you push more changes, so watch
  # out for parallel CI runs!
  #
  # To change the contents of the build container, modify docker/Dockerfile.ci
  # To change the final release container, modify docker/Dockerfile.iox
  perf_image:
    docker:
      - image: quay.io/influxdb/rust:ci
    environment:
        # Disable full debug symbol generation to speed up CI build
        # "1" means line tables only, which is useful for panic tracebacks.
        RUSTFLAGS: "-C debuginfo=1"
    steps:
      - checkout
      - rust_components
      - cache_restore
      - run:
          name: Cargo release build with target arch set for CRoaring
          command: ROARING_ARCH=x86-64 cargo build --release
      - run: |
          echo sha256sum after build is
          sha256sum target/release/influxdb_iox
      - setup_remote_docker:
          # There seems to be a cache invalidation bug in docker
          # or in the way that circleci implements layer caching.
          # Disabling for now, and tracked further investigations
          # in https://github.com/influxdata/k8s-idpe/issues/3038
          docker_layer_caching: false
      - run: |
          sudo apt-get update
          sudo apt-get install -y docker.io
      - run: |
          echo "$QUAY_PASS" | docker login quay.io --username $QUAY_USER --password-stdin
      - run: |
          BRANCH=$(git rev-parse --abbrev-ref HEAD | tr '/' '.')
          echo sha256sum after build is
          sha256sum target/release/influxdb_iox
          docker build -t quay.io/influxdb/fusion:$BRANCH -f docker/Dockerfile.iox .
          docker push quay.io/influxdb/fusion:$BRANCH
          echo "export BRANCH=${BRANCH}" >> $BASH_ENV
      - run:
          name: Deploy tags
          command: ./.circleci/get-deploy-tags.sh "${BRANCH}"
      - cache_save

  # Prepare the CI image used for other tasks.
  #
  # A nightly job (scheduled below in the `workflows` section) to build the CI
  # image (influxdb/rust) used for the rest of the checks.
  #
  # To modify the contents of the CI image, update docker/Dockerfile.ci
  ci_image:
    machine: true
    resource_class: xlarge
    steps:
      - checkout
      - run: |
          echo "$QUAY_PASS" | docker login quay.io --username $QUAY_USER --password-stdin
      - run: |
          COMMIT_SHA=$(git rev-parse --short HEAD)
          docker build -t quay.io/influxdb/rust:$COMMIT_SHA -f docker/Dockerfile.ci --build-arg RUST_VERSION=$(cat rust-toolchain) .
          docker tag quay.io/influxdb/rust:$COMMIT_SHA quay.io/influxdb/rust:ci
          docker push quay.io/influxdb/rust:$COMMIT_SHA
          docker push quay.io/influxdb/rust:ci

workflows:
  version: 2

  # CI for all pull requests.
  ci:
    jobs:
      - fmt
      - lint
      - protobuf-lint
      - test
      - test_influxdb2_client
      - build
      - check-flatbuffers
      - doc

  # Internal pipeline for perf builds.
  #
  # Filter ensures this only runs for git branches ending in `/perf`.
  perf_build:
    jobs:
      - fmt:
          filters:
            branches:
              only: /.*\/perf$/
      - lint:
          filters:
            branches:
              only: /.*\/perf$/
      - test:
          filters:
            branches:
              only: /.*\/perf$/
      - build:
          filters:
            branches:
              only: /.*\/perf$/
      - perf_image:
          filters:
            branches:
              only: /.*\/perf$/
          requires: # Only do a release build if all tests have passed
            - fmt
            - lint
            - test
            - build

  # Nightly rebuild of the build container
  ci_image:
    triggers:
      - schedule:
          cron: "0 5 * * *"
          filters:
            branches:
              only:
                - main
    jobs:
      - ci_image