influxdb/.circleci/config.yml

---
# CI Overview
# -----------
#
# Each night:
#
#   A build image is created (ci_image) from `docker/Dockerfile.ci` and is
#   pushed to `quay.io/influxdb/rust:ci`. This build image is then used to run
#   the CI tasks for the day.
#
# Every commit:
#
# The CI for every PR and merge to main runs tests, fmt, lints and compiles debug binaries
#
# On main if all these checks pass it will then additionally compile in "release" mode and
# publish a docker image to quay.io/influxdb/iox:$COMMIT_SHA
#
# Manually trigger build and push of container image for a branch:
#
# Navigate to https://app.circleci.com/pipelines/github/influxdata/influxdb_iox?branch=<branch-name> (<- change this!)
# Then:
#
# - Click "Run Pipeline" in the top-right
# - Expand "Add Parameters"
# - Add a "boolean" parameter called "release_branch" with the value true
# - Click "Run Pipeline"
#
# You can also do this using the CircleCI API:
#
# Using `xh`:
#
#   # e.g. using 'xh' (https://github.com/ducaale/xh)
#   $ xh -a '<your personal circleCI token>:' POST \
#       https://circleci.com/api/v2/project/github/influxdata/influxdb_iox/pipeline \
#       parameters:='{"release_branch": true}' branch=chore/ci-tidy-up
#
# ...or equivalent with `curl`:
#   $ curl -XPOST -H "Content-Type: application/json" -H "Circle-Token: <your personal circleCI token>" \
#       -d '{"parameters": {"release_branch": true}, "branch": "chore/ci-tidy-up"}' \
#       https://circleci.com/api/v2/project/github/influxdata/influxdb_iox/pipeline
#
# Manual CI Base Image:
#
# It is possible to manually trigger a rebuild of the base image used in CI. To do this, navigate to
# https://app.circleci.com/pipelines/github/influxdata/influxdb_iox?branch=main (overriding the
# branch name if desired). Then:
# - Click "Run Pipeline" in the top-right
# - Expand "Add Parameters"
# - Add a "boolean" parameter called "ci_image" with the value true
# - Click "Run Pipeline"
#
# If you refresh the page you should see a newly running ci_image workflow
#

version: 2.1

commands:
  rust_components:
    description: Verify installed components
    steps:
      - run:
          name: Verify installed components
          command: |
            rustup --version
            rustup show
            cargo fmt --version
            cargo clippy --version
            cargo install cargo-hakari && cargo hakari --version

  login_to_gcloud:
    steps:
      - run:
          name: Login to gcloud
          command: |
            echo "${GCLOUD_SERVICE_ACCOUNT_KEY}" >/tmp/gcloud.json
            gcloud auth activate-service-account "${GCLOUD_SERVICE_ACCOUNT_EMAIL}" --key-file /tmp/gcloud.json --quiet
            rm -f /tmp/gcloud.json
            gcloud auth configure-docker us-docker.pkg.dev

jobs:
  fmt:
    docker:
      - image: quay.io/influxdb/rust:ci
    environment:
      # Disable incremental compilation to avoid overhead. We are not preserving these files anyway.
      CARGO_INCREMENTAL: "0"
      # Disable full debug symbol generation to speed up CI build
      # "1" means line tables only, which is useful for panic tracebacks.
      CARGO_PROFILE_DEV_DEBUG: "1"
      # https://github.com/rust-lang/cargo/issues/10280
      CARGO_NET_GIT_FETCH_WITH_CLI: "true"
    steps:
      - checkout
      - rust_components
      - run:
          name: Rust fmt
          command: cargo fmt --all -- --check
  lint:
    docker:
      - image: quay.io/influxdb/rust:ci
    environment:
      # Disable incremental compilation to avoid overhead. We are not preserving these files anyway.
      CARGO_INCREMENTAL: "0"
      # Disable full debug symbol generation to speed up CI build
      # "1" means line tables only, which is useful for panic tracebacks.
      CARGO_PROFILE_DEV_DEBUG: "1"
      # https://github.com/rust-lang/cargo/issues/10280
      CARGO_NET_GIT_FETCH_WITH_CLI: "true"
    steps:
      - checkout
      - rust_components
      - run:
          name: Clippy
          command: cargo clippy --all-targets --all-features --workspace -- -D warnings
      - run:
          name: Shellcheck
          command: find scripts -type f ! \( -iname '*.py' -or -iname '*.supp' \) -exec shellcheck {} +
      - run:
          name: Yamllint
          command: yamllint --config-file .circleci/yamllint.yml --strict .
      - run:
          name: migrations check
          command: iox_catalog/check_linear_migrations.sh migrations
  cargo_audit:
    docker:
      - image: quay.io/influxdb/rust:ci
    environment:
      # Disable incremental compilation to avoid overhead. We are not preserving these files anyway.
      CARGO_INCREMENTAL: "0"
      # Disable full debug symbol generation to speed up CI build
      # "1" means line tables only, which is useful for panic tracebacks.
      CARGO_PROFILE_DEV_DEBUG: "1"
      # https://github.com/rust-lang/cargo/issues/10280
      CARGO_NET_GIT_FETCH_WITH_CLI: "true"
    steps:
      - checkout
      - rust_components
      - run:
          name: Install cargo-deny
          command: cargo install cargo-deny
      - run:
          name: cargo-deny Checks
          command: cargo deny check -s
  doc:
    docker:
      - image: quay.io/influxdb/rust:ci
    resource_class: large # use of a smaller executor runs out of memory
    environment:
      # Disable incremental compilation to avoid overhead. We are not preserving these files anyway.
      CARGO_INCREMENTAL: "0"
      # Disable full debug symbol generation to speed up CI build
      # "1" means line tables only, which is useful for panic tracebacks.
      CARGO_PROFILE_DEV_DEBUG: "1"
      # https://github.com/rust-lang/cargo/issues/10280
      CARGO_NET_GIT_FETCH_WITH_CLI: "true"
      # Turn warnings into errors
      RUSTDOCFLAGS: "-D warnings -A rustdoc::private-intra-doc-links"
    steps:
      - checkout
      - rust_components
      - run:
          name: Cargo doc
          command: cargo doc --document-private-items --no-deps --workspace
      - run:
          name: Compress Docs
          command: tar -cvzf rustdoc.tar.gz target/doc/
      - store_artifacts:
          path: rustdoc.tar.gz
  workspace_hack_checks:
    docker:
      - image: quay.io/influxdb/rust:ci
    environment:
      # https://github.com/rust-lang/cargo/issues/10280
      CARGO_NET_GIT_FETCH_WITH_CLI: "true"
    steps:
      - add_ssh_keys:
          fingerprints:
            - "77:99:88:4a:ac:1f:55:9e:39:c7:1f:e4:7f:1e:60:4b"
      - checkout
      - rust_components
      - run:
          name: Configure git
          command: |
            git config user.email "circleci@influxdata.com"
            git config user.name "CircleCI[bot]"
      - run:
          name: Commit any changes from cargo hakari
          command: |
            cargo hakari generate
            cargo hakari manage-deps -y
            if [[ $(git status --porcelain) ]] ; then
              if [[ $(git rev-list --count --author=CircleCI origin/main..) -ne 0 ]]; then
                echo "There's already a cargo hakari commit on this branch, but there are cargo hakari changes... something is wrong."
              else
                git commit -am "chore: Run cargo hakari tasks"
                git push origin "$CIRCLE_BRANCH"
                echo "Cargo hakari changes need to be committed before merging this branch."
              fi
              exit 1
            else
              echo "No changes to commit"
            fi

  # Run integration tests that need postgres and other external dependencies
  # such as influxdb v1 and java
  test_integration:
    # setup multiple docker images (see https://circleci.com/docs/2.0/configuration-reference/#docker)
    docker:
      - image: quay.io/influxdb/rust:ci
      - image: postgres
        environment:
          POSTGRES_HOST_AUTH_METHOD: trust
        command:
          - "postgres"
          - "-c"
          - "max_connections=200"
    resource_class: xlarge # use of a smaller executor tends crashes on link
    environment:
      # Disable incremental compilation to avoid overhead. We are not preserving these files anyway.
      CARGO_INCREMENTAL: "0"
      # Disable full debug symbol generation to speed up CI build
      # "1" means line tables only, which is useful for panic tracebacks.
      CARGO_PROFILE_DEV_DEBUG: "1"
      # https://github.com/rust-lang/cargo/issues/10280
      CARGO_NET_GIT_FETCH_WITH_CLI: "true"
      RUST_BACKTRACE: "1"
      # Run integration tests
      TEST_INTEGRATION: 1
      INFLUXDB_IOX_INTEGRATION_LOCAL: 1
      POSTGRES_USER: postgres
      TEST_INFLUXDB_IOX_CATALOG_DSN: "postgres://postgres@localhost/iox_shared"
      # When removing this, also remove the ignore on the test in trogging/src/cli.rs
      RUST_LOG: debug,,hyper::proto::h1=info,h2=info
      LOG_FILTER: debug,,hyper::proto::h1=info,h2=info
      # Run the JDBC tests
      TEST_INFLUXDB_JDBC: "true"
    steps:
      - checkout
      - rust_components
      - run:
          name: Download flight-sql-jdbc-driver-10.0.0.jar
          command: make -C influxdb_iox/tests/jdbc_client flight-sql-jdbc-driver-10.0.0.jar
      - run:
          name: cargo test -p influxdb2_client
          command: cargo test -p influxdb2_client
      - run:
          name: cargo test -p iox_catalog
          command: cargo test -p iox_catalog
      - run:
          name: cargo test --test end_to_end
          command: cargo test --test end_to_end

  # Run all tests (without external dependencies, like a developer would)
  test:
    docker:
      - image: quay.io/influxdb/rust:ci
    resource_class: 2xlarge+ # use of a smaller executor tends crashes on link
    environment:
      # Disable incremental compilation to avoid overhead. We are not preserving these files anyway.
      CARGO_INCREMENTAL: "0"
      # Disable full debug symbol generation to speed up CI build
      # "1" means line tables only, which is useful for panic tracebacks.
      CARGO_PROFILE_DEV_DEBUG: "1"
      # https://github.com/rust-lang/cargo/issues/10280
      CARGO_NET_GIT_FETCH_WITH_CLI: "true"
      RUST_BACKTRACE: "1"
    steps:
      - checkout
      - rust_components
      - run:
          name: cargo test --workspace
          command: cargo test --workspace


  # end to end tests with Heappy (heap profiling enabled)
  test_heappy:
    docker:
      - image: quay.io/influxdb/rust:ci
    resource_class: xlarge # use of a smaller executor tends crashes on link
    environment:
      # Disable incremental compilation to avoid overhead. We are not preserving these files anyway.
      CARGO_INCREMENTAL: "0"
      # Disable full debug symbol generation to speed up CI build
      # "1" means line tables only, which is useful for panic tracebacks.
      CARGO_PROFILE_DEV_DEBUG: "1"
      # https://github.com/rust-lang/cargo/issues/10280
      CARGO_NET_GIT_FETCH_WITH_CLI: "true"
      RUST_BACKTRACE: "1"
    steps:
      - checkout
      - rust_components
      - run:
          name: cargo test --no-default-features --features=heappy end_to_end
          command: cargo test --no-default-features --features=heappy end_to_end

  # Build a dev binary.
  #
  # Compiles a binary with the default ("dev") cargo profile from the iox source
  # using the latest ci_image and ensures various targets compile successfully
  build_dev:
    docker:
      - image: quay.io/influxdb/rust:ci
    resource_class: 2xlarge # use of a smaller executor tends crashes on link
    environment:
      # Disable incremental compilation to avoid overhead. We are not preserving these files anyway.
      CARGO_INCREMENTAL: "0"
      # Disable full debug symbol generation to speed up CI build
      # "1" means line tables only, which is useful for panic tracebacks.
      CARGO_PROFILE_DEV_DEBUG: "1"
      # https://github.com/rust-lang/cargo/issues/10280
      CARGO_NET_GIT_FETCH_WITH_CLI: "true"
      # The `2xlarge` resource class that we use has 32GB RAM but also 16 CPUs. This means we have 2GB RAM per core on
      # avarage. At peak this is a bit tight, so lower the CPU count for cargo a bit.
      CARGO_BUILD_JOBS: "12"
    steps:
      - checkout
      - rust_components
      - run:
          name: Cargo build
          command: cargo build --workspace
      - run:
          name: Check benches compile
          command: cargo check --workspace --benches
      - run:
          name: Check extra features (like prod image)
          command: cargo check --no-default-features --features="aws,gcp,azure,jemalloc_replacing_malloc,tokio_console,pprof"
      - run:
          # Validate that the data generator compiles (in the same way as it does in release builds)
          name: Check iox_data_generator compiles
          command: cargo check  --package="iox_data_generator"  --no-default-features

  # Lint protobufs.
  protobuf-lint:
    docker:
      - image: bufbuild/buf:0.40.0
    environment:
      # Value to look for to skip breaking changes check
      SKIP_LABEL: "https://api.github.com/repos/influxdata/influxdb_iox/labels/incompatible%20protobuf"
    steps:
      - checkout
      - run:
          name: buf lint
          command: buf lint
      - run:
          name: buf breaking changes
          command: |
            echo "If you want to make changes forbidden by this lint, please"
            echo "coordinate with the conductor team, add the 'incompatible protobuf' label"
            echo "to the PR, and rerun this test"
            # Check if label is present using github API:
            # Inspired by https://discuss.circleci.com/t/tag-label-filter/11158
            if wget -O -  https://api.github.com/repos/influxdata/influxdb_iox/issues/$(echo $CIRCLE_PULL_REQUEST | grep -oE "[^/pull]+$") | grep "$SKIP_LABEL" ; then echo "SKIPPING (FOUND LABEL)" && exit ; else echo "CHECKING (NO LABEL FOUND)"; fi
            git fetch origin main
            # compare against only changes in this branch (not against
            # other stuff that may have been added to main since last merge)
            MERGE_BASE=$(git merge-base origin/main $CIRCLE_BRANCH) sh -c 'buf breaking --against ".git#ref=$MERGE_BASE,depth=100"'

  # Lint docs
  docs-lint:
    docker:
      - image: python:3-slim-bullseye
    steps:
      # need a proper SSH client for checkout
      # https://discuss.circleci.com/t/circleci-user-key-uses-rsa-with-sha1-which-github-has-deprecated/42548
      - run:
          name: Install GIT and SSH
          command: |
            apt-get update
            apt-get install -y git ssh
      - checkout
      - run:
          name: Lint docs
          command: ./scripts/lint_docs.py ./docs

  # Compile cargo "release" profile binaries for iox & data generator
  #
  # Uses the latest ci_image (influxdb/rust below) to build a release binary and
  # copies it to a minimal container image based upon `rust:slim-buster`. This
  # minimal image is then pushed to `quay.io/influxdb/iox:${BRANCH}`.
  build_release:
    # need a machine executor to have a full-powered docker daemon (the `setup_remote_docker` system just provides a
    # kinda small node)
    machine:
      image: ubuntu-2004:2022.04.2
    resource_class: 2xlarge # CPU bound, so make it fast
    steps:
      - checkout
      - run:
          name: Cargo release build with target arch set for CRoaring
          command: |
            COMMIT_SHA="$(git rev-parse HEAD)"

            .circleci/docker_build_release.bash \
              "influxdb_iox" \
              "aws,gcp,azure,jemalloc_replacing_malloc,tokio_console,pprof" \
              "quay.io/influxdb/iox:$COMMIT_SHA"

            .circleci/docker_build_release.bash \
              "iox_data_generator" \
              "" \
              "quay.io/influxdb/iox_data_generator:$COMMIT_SHA"

            docker run -it --rm quay.io/influxdb/iox:$COMMIT_SHA debug print-cpu

            mkdir /tmp/images
            docker save quay.io/influxdb/iox:"$COMMIT_SHA" | gzip > /tmp/images/iox.tar.gz
            docker save quay.io/influxdb/iox_data_generator:"$COMMIT_SHA" | gzip > /tmp/images/iox_data_generator.tar.gz
          # linking might take a while and doesn't produce CLI output
          no_output_timeout: 30m
      - store_artifacts:
          path: /tmp/images
      - persist_to_workspace:
          root: /tmp/images
          paths:
            - "*.tar.gz"

  deploy_release:
    docker:
      - image: quay.io/influxdb/rust:ci
    environment:
        # See https://github.com/containers/skopeo/issues/942
        XDG_RUNTIME_DIR: /home/rust/.run
    steps:
      - checkout
      - attach_workspace:
          at: /tmp/images
      - run:
          name: set up runtime dir
          command: |
            mkdir -p "$XDG_RUNTIME_DIR"
      - login_to_gcloud
      - run:
          name: quay login
          command: |
            echo "$QUAY_INFLUXDB_IOX_PASS" | skopeo login quay.io --username $QUAY_INFLUXDB_IOX_USER --password-stdin
      - run:
          name: Upload images
          command: |
            COMMIT_SHA="$(git rev-parse HEAD)"
            BRANCH="$(echo "$CIRCLE_BRANCH" | tr '[:upper:]' '[:lower:]' | sed 's/[^a-z0-9]/_/g')"

            images=( "iox" "iox_data_generator" )
            registries=( "quay.io/influxdb" "us-docker.pkg.dev/influxdb2-artifacts/iox" )
            tags=( "$COMMIT_SHA" "$BRANCH" )

            for image in "${images[@]}"; do
              echo "Image: $image"
              oci_path="oci-archive:///tmp/images/$image.oci.tar"

              # convert the gzipped docker image into OCI
              gzip -d "/tmp/images/$image.tar.gz"
              skopeo copy --format oci --quiet "docker-archive:///tmp/images/$image.tar" "$oci_path"

              for registry in "${registries[@]}"; do
                echo "  Registry: $registry"

                # upload all tags
                # Note: Uploading the 2nd tag for the same image (to the same registry) is very cheap since all layers
                #       exist already (from the previous tag).
                for tag in "${tags[@]}"; do
                  echo "    Upload: tag=$tag"
                  docker_url="docker://$registry/$image:$tag"
                  skopeo copy --quiet "$oci_path" "$docker_url"

                  # print out digest AFTER upload, see https://github.com/containers/skopeo/issues/469
                  echo "      Digest: $(skopeo inspect "$docker_url" | jq ".Digest")"
                done
              done
            done

  # Prepare the CI image used for other tasks.
  #
  # A nightly job (scheduled below in the `workflows` section) to build the CI
  # image (influxdb/rust) used for the rest of the checks.
  #
  # To modify the contents of the CI image, update docker/Dockerfile.ci
  ci_image:
    # Use a `machine` executor instead of a docker-based because:
    # - we bootstrap our CI images somehow (we could use a docker-hub image as well)
    # - the machine executor also supports multi-arch builds
    # - we only run this job once a day, so the additional startup delay of 30-60s doesn't really matter
    machine:
      # https://circleci.com/docs/2.0/configuration-reference/#available-machine-images
      image: ubuntu-2004:2022.04.2
    resource_class: xlarge
    steps:
      - checkout
      - run: |
          echo "$QUAY_PASS" | docker login quay.io --username $QUAY_USER --password-stdin
      - run: |
          COMMIT_SHA=$(git rev-parse --short HEAD)
          RUST_VERSION=$(sed -E -ne 's/channel = "(.*)"/\1/p' rust-toolchain.toml)
          docker --version
          docker build -t quay.io/influxdb/rust:$COMMIT_SHA -t quay.io/influxdb/rust:ci -f docker/Dockerfile.ci --build-arg RUST_VERSION=$RUST_VERSION .
          docker push --all-tags quay.io/influxdb/rust

parameters:
  ci_image:
    description: "Trigger build of CI image"
    type: boolean
    default: false
  release_branch:
    description: "Build and push container image for non-main branch"
    type: boolean
    default: false

workflows:
  version: 2

  # CI for all pull requests.
  ci:
    when:
      and:
        - not: << pipeline.parameters.ci_image >>
        - not: << pipeline.parameters.release_branch >>
    jobs:
      - fmt
      - lint
      - cargo_audit
      - protobuf-lint
      - docs-lint
      - test
      - test_heappy
      - test_integration
      - build_dev
      - doc
      - workspace_hack_checks
      - build_release:
          filters:
            branches:
              only: main
      - deploy_release:
          filters:
            branches:
              only: main
          requires: # Only deploy if all tests have passed
            - fmt
            - lint
            - cargo_audit
            - protobuf-lint
            - docs-lint
            - test
            - test_heappy
            - test_integration
            - build_dev
            - build_release
            - doc

  # Manual build of CI image
  ci_image:
    when: << pipeline.parameters.ci_image >>
    jobs:
      - ci_image

  # Force build and push of container image for non-main branch.
  # See instructions at the top of this file
  release_branch:
    when: << pipeline.parameters.release_branch >>
    jobs:
      - build_release
      - deploy_release:
          requires:
            - build_release

  # Nightly rebuild of the build container
  ci_image_nightly:
    triggers:
      - schedule:
          cron: "0 5 * * *"
          filters:
            branches:
              only:
                - main
    jobs:
      - ci_image