--- # CI Overview # ----------- # # Each night: # # A build image is created (ci_image) from `docker/Dockerfile.ci` and is # pushed to `quay.io/influxdb/rust:ci`. This build image is then used to run # the CI tasks for the day. # # Every commit: # # The CI for every PR and merge to main runs tests, fmt, lints and compiles debug binaries # # On main if all these checks pass it will then additionally compile in "release" mode and # publish a docker image to quay.io/influxdb/iox:$COMMIT_SHA # # Manual CI Image: # # It is possible to manually trigger a rebuild of the image used in CI. To do this, navigate to # https://app.circleci.com/pipelines/github/influxdata/influxdb_iox?branch=main (overriding the # branch name if desired). Then: # - Click "Run Pipeline" in the top-right # - Expand "Add Parameters" # - Add a "boolean" parameter called "ci_image" with the value true # - Click "Run Pipeline" # # If you refresh the page you should see a newly running ci_image workflow # version: 2.1 commands: rust_components: description: Verify installed components steps: - run: name: Verify installed components command: | rustup --version rustup show cargo fmt --version cargo clippy --version cargo install cargo-hakari && cargo hakari --version cache_restore: description: Restore Cargo Cache steps: - restore_cache: name: Restoring Cargo Cache keys: - cargo-cache-{{ arch }}-{{ .Branch }}-{{ checksum "Cargo.lock" }} - cargo-cache-{{ arch }}-{{ .Branch }} - cargo-cache cache_save: description: Save Cargo Cache steps: - save_cache: name: Save Cargo Cache paths: - /usr/local/cargo/registry key: cargo-cache-{{ arch }}-{{ .Branch }}-{{ checksum "Cargo.lock" }} jobs: fmt: docker: - image: quay.io/influxdb/rust:ci environment: # Disable incremental compilation to avoid overhead. We are not preserving these files anyway. CARGO_INCREMENTAL: "0" # Disable full debug symbol generation to speed up CI build # "1" means line tables only, which is useful for panic tracebacks. RUSTFLAGS: "-C debuginfo=1" # https://github.com/rust-lang/cargo/issues/10280 CARGO_NET_GIT_FETCH_WITH_CLI: "true" steps: - checkout - rust_components - cache_restore - run: name: Rust fmt command: cargo fmt --all -- --check - cache_save lint: docker: - image: quay.io/influxdb/rust:ci environment: # Disable incremental compilation to avoid overhead. We are not preserving these files anyway. CARGO_INCREMENTAL: "0" # Disable full debug symbol generation to speed up CI build # "1" means line tables only, which is useful for panic tracebacks. RUSTFLAGS: "-C debuginfo=1" # https://github.com/rust-lang/cargo/issues/10280 CARGO_NET_GIT_FETCH_WITH_CLI: "true" steps: - checkout - rust_components - cache_restore - run: name: Clippy command: cargo clippy --all-targets --all-features --workspace -- -D warnings - run: name: Shellcheck command: find scripts -type f ! \( -iname '*.py' -or -iname '*.supp' \) -exec shellcheck {} + - run: name: Yamllint command: yamllint --config-file .circleci/yamllint.yml --strict . - cache_save cargo_audit: docker: - image: quay.io/influxdb/rust:ci environment: # Disable incremental compilation to avoid overhead. We are not preserving these files anyway. CARGO_INCREMENTAL: "0" # Disable full debug symbol generation to speed up CI build # "1" means line tables only, which is useful for panic tracebacks. RUSTFLAGS: "-C debuginfo=1" # https://github.com/rust-lang/cargo/issues/10280 CARGO_NET_GIT_FETCH_WITH_CLI: "true" steps: - checkout - rust_components - cache_restore - run: name: Install cargo-deny command: cargo install --force cargo-deny - run: name: cargo-deny Checks command: cargo deny check -s - cache_save doc: docker: - image: quay.io/influxdb/rust:ci resource_class: large # use of a smaller executor runs out of memory environment: # Disable incremental compilation to avoid overhead. We are not preserving these files anyway. CARGO_INCREMENTAL: "0" # Disable full debug symbol generation to speed up CI build # "1" means line tables only, which is useful for panic tracebacks. RUSTFLAGS: "-C debuginfo=1" # https://github.com/rust-lang/cargo/issues/10280 CARGO_NET_GIT_FETCH_WITH_CLI: "true" # Turn warnings into errors RUSTDOCFLAGS: "-D warnings -A rustdoc::private-intra-doc-links" steps: - checkout - rust_components - cache_restore - run: name: Cargo doc # excluding datafusion because it's effectively a dependency masqueraded as workspace crate. command: cargo doc --document-private-items --no-deps --workspace --exclude datafusion - cache_save - run: name: Compress Docs command: tar -cvzf rustdoc.tar.gz target/doc/ - store_artifacts: path: rustdoc.tar.gz workspace_hack_checks: docker: - image: quay.io/influxdb/rust:ci environment: # https://github.com/rust-lang/cargo/issues/10280 CARGO_NET_GIT_FETCH_WITH_CLI: "true" steps: - add_ssh_keys: fingerprints: - "77:99:88:4a:ac:1f:55:9e:39:c7:1f:e4:7f:1e:60:4b" - checkout - rust_components - cache_restore - run: name: Configure git command: | git config user.email "circleci@influxdata.com" git config user.name "CircleCI[bot]" - run: name: Commit any changes from cargo hakari command: | cargo hakari generate cargo hakari manage-deps -y if [[ $(git status --porcelain) ]] ; then if [[ $(git rev-list --count --author=CircleCI origin/main..) -ne 0 ]]; then echo "There's already a cargo hakari commit on this branch, but there are cargo hakari changes... something is wrong." else git commit -am "chore: Run cargo hakari tasks" git push origin "$CIRCLE_BRANCH" echo "Cargo hakari changes need to be committed before merging this branch." fi exit 1 else echo "No changes to commit" fi test: # setup multiple docker images (see https://circleci.com/docs/2.0/configuration-reference/#docker) docker: - image: quay.io/influxdb/rust:ci - image: vectorized/redpanda:v21.9.2 command: redpanda start --overprovisioned --smp 1 --memory 1G --reserve-memory 0M - image: postgres environment: POSTGRES_HOST_AUTH_METHOD: trust resource_class: 2xlarge # use of a smaller executor tends crashes on link environment: # Disable incremental compilation to avoid overhead. We are not preserving these files anyway. CARGO_INCREMENTAL: "0" # Disable full debug symbol generation to speed up CI build # "1" means line tables only, which is useful for panic tracebacks. RUSTFLAGS: "-C debuginfo=1" # https://github.com/rust-lang/cargo/issues/10280 CARGO_NET_GIT_FETCH_WITH_CLI: "true" RUST_BACKTRACE: "1" # Run integration tests TEST_INTEGRATION: 1 INFLUXDB_IOX_INTEGRATION_LOCAL: 1 KAFKA_CONNECT: "localhost:9092" POSTGRES_USER: postgres TEST_INFLUXDB_IOX_CATALOG_DSN: "postgres://postgres@localhost/iox_shared" # When removing this, also remove the ignore on the test in trogging/src/cli.rs RUST_LOG: debug LOG_FILTER: debug steps: - checkout - rust_components - cache_restore - run: name: Cargo test command: cargo test --workspace - cache_save # end to end tests with Heappy (heap profiling enabled) test_heappy: docker: - image: quay.io/influxdb/rust:ci resource_class: xlarge # use of a smaller executor tends crashes on link environment: # Disable incremental compilation to avoid overhead. We are not preserving these files anyway. CARGO_INCREMENTAL: "0" # Disable full debug symbol generation to speed up CI build # "1" means line tables only, which is useful for panic tracebacks. RUSTFLAGS: "-C debuginfo=1" # https://github.com/rust-lang/cargo/issues/10280 CARGO_NET_GIT_FETCH_WITH_CLI: "true" RUST_BACKTRACE: "1" steps: - checkout - rust_components - cache_restore - run: name: End to end tests (with heappy) command: cargo test --no-default-features --features=heappy end_to_end - cache_save test_perf: machine: image: ubuntu-2004:202107-02 resource_class: xlarge environment: # Disable incremental compilation to avoid overhead. We are not preserving these files anyway. CARGO_INCREMENTAL: "0" # Disable full debug symbol generation to speed up CI build # "1" means line tables only, which is useful for panic tracebacks. RUSTFLAGS: "-C debuginfo=1" # https://github.com/rust-lang/cargo/issues/10280 CARGO_NET_GIT_FETCH_WITH_CLI: "true" RUST_BACKTRACE: "1" PROTOC: "/home/circleci/protoc/bin/protoc" PROTOC_INCLUDE: "/home/circleci/protoc/include" steps: # Use us.archive.ubuntu.com since it is close. If it is down, try archive.ubuntu.com. If it is # down too, temporarily choose something from http://mirrors.ubuntu.com/mirrors.txt that # is up to date. - run: name: Setup ubuntu mirror command: | mirror="http://us.archive.ubuntu.com/ubuntu" sudo sed -i "s;http://.*ubuntu.com/ubuntu/;$mirror;g" /etc/apt/sources.list - run: sudo apt update - run: name: Install required compile tools command: sudo apt install curl clang lld pkg-config libssl-dev --yes --no-install-recommends - checkout - run: name: Install protoc 3.19.4 to ~/protoc (stock in ubunutu is too old) command: | mkdir /home/circleci/protoc cd /home/circleci/protoc wget https://github.com/protocolbuffers/protobuf/releases/download/v3.19.4/protoc-3.19.4-linux-x86_64.zip unzip protoc-3.19.4-linux-x86_64.zip - run: name: Install rustup command: scripts/install_rustup.sh - run: name: Install perf.py dependencies command: python3 -m pip install -r perf/requirements.txt - restore_cache: keys: - cargo-lock-{{ checksum "Cargo.lock" }} - run: name: Prime Rust build cache command: cargo build --package influxdb_iox --bin influxdb_iox --package iox_data_generator --bin iox_data_generator - save_cache: key: cargo-lock-{{ checksum "Cargo.lock" }} paths: - target - run: name: Run basic perf test to ensure that perf.py works command: | trap "cat perf/logs/test.log" ERR perf/perf.py --debug --no-volumes --object-store memory battery-0 # Build a dev binary. # # Compiles a binary with the default ("dev") cargo profile from the iox source # using the latest ci_image. build_dev: docker: - image: quay.io/influxdb/rust:ci resource_class: 2xlarge # use of a smaller executor tends crashes on link environment: # Disable incremental compilation to avoid overhead. We are not preserving these files anyway. CARGO_INCREMENTAL: "0" # Disable full debug symbol generation to speed up CI build # "1" means line tables only, which is useful for panic tracebacks. RUSTFLAGS: "-C debuginfo=1" # https://github.com/rust-lang/cargo/issues/10280 CARGO_NET_GIT_FETCH_WITH_CLI: "true" steps: - checkout - rust_components - cache_restore - run: name: Cargo build command: cargo build --workspace - run: name: Build benches command: cargo test --workspace --benches --no-run - run: name: Build with object store + exporter support + HEAP profiling command: cargo build --no-default-features --features="aws,gcp,azure,heappy,pprof" - cache_save # Lint protobufs. protobuf-lint: docker: - image: bufbuild/buf:0.40.0 environment: # Value to look for to skip breaking changes check SKIP_LABEL: "https://api.github.com/repos/influxdata/influxdb_iox/labels/incompatible%20protobuf" steps: - checkout - run: name: buf lint command: buf lint - run: name: buf breaking changes command: | echo "If you want to make changes forbidden by this lint, please" echo "coordinate with the conductor team, add the 'incompatible protobuf' label" echo "to the PR, and rerun this test" # Check if label is present using github API: # Inspired by https://discuss.circleci.com/t/tag-label-filter/11158 if wget -O - https://api.github.com/repos/influxdata/influxdb_iox/issues/$(echo $CIRCLE_PULL_REQUEST | grep -oE "[^/pull]+$") | grep "$SKIP_LABEL" ; then echo "SKIPPING (FOUND LABEL)" && exit ; else echo "CHECKING (NO LABEL FOUND)"; fi git fetch origin main # compare against only changes in this branch (not against # other stuff that may have been added to main since last merge) MERGE_BASE=$(git merge-base origin/main $CIRCLE_BRANCH) sh -c 'buf breaking --against ".git#ref=$MERGE_BASE"' # Lint docs docs-lint: docker: - image: python:3-slim-bullseye steps: # need a proper SSH client for checkout # https://discuss.circleci.com/t/circleci-user-key-uses-rsa-with-sha1-which-github-has-deprecated/42548 - run: name: Install GIT and SSH command: | apt-get update apt-get install -y git ssh - checkout - run: name: Lint docs command: ./scripts/lint_docs.py ./docs # Compile cargo "release" profile binaries for iox & data generator # # Uses the latest ci_image (influxdb/rust below) to build a release binary and # copies it to a minimal container image based upon `rust:slim-buster`. This # minimal image is then pushed to `quay.io/influxdb/iox:${BRANCH}`. build_release: # need a machine executor to have a full-powered docker daemon (the `setup_remote_docker` system just provides a # kinda small node) machine: image: ubuntu-2004:202111-01 resource_class: xlarge # use of a smaller executor tends crashes on link environment: # Disable incremental compilation to avoid overhead. We are not preserving these files anyway. CARGO_INCREMENTAL: "0" # We keep the debug symbols (Enabled in Cargo.toml as debug = true) # workaround dynamic CPU detection bug in croaring # https://github.com/influxdata/influxdb_iox/pull/2119 ROARING_ARCH: "haswell" # Use avx512 instructions to take full advantage of the CPUs instruction set RUSTFLAGS: "-C target-feature=+avx2 -C link-arg=-fuse-ld=lld --cfg tokio_unstable" # https://github.com/rust-lang/cargo/issues/10280 CARGO_NET_GIT_FETCH_WITH_CLI: "true" steps: - checkout - run: | echo "$QUAY_INFLUXDB_IOX_PASS" | docker login quay.io --username $QUAY_INFLUXDB_IOX_USER --password-stdin - run: name: Cargo release build with target arch set for CRoaring command: | COMMIT_SHA="$(git rev-parse --short HEAD)" RUST_VERSION="$(sed -E -ne 's/channel = "(.*)"/\1/p' rust-toolchain.toml)" docker buildx build \ --build-arg FEATURES="aws,gcp,azure,jemalloc_replacing_malloc,tokio_console,pprof" \ --build-arg RUST_VERSION="$RUST_VERSION" \ --build-arg RUSTFLAGS="$RUSTFLAGS" \ --progress plain \ --tag quay.io/influxdb/iox:"$COMMIT_SHA" \ . docker buildx build \ --build-arg FEATURES="" \ --build-arg PACKAGE="iox_data_generator" \ --build-arg RUST_VERSION="$RUST_VERSION" \ --build-arg RUSTFLAGS="$RUSTFLAGS" \ --progress plain \ --tag quay.io/influxdb/iox_data_generator:"$COMMIT_SHA" \ . docker run -it --rm quay.io/influxdb/iox:$COMMIT_SHA debug print-cpu docker push quay.io/influxdb/iox:"$COMMIT_SHA" docker push quay.io/influxdb/iox_data_generator:"$COMMIT_SHA" # linking might take a while and doesn't produce CLI output no_output_timeout: 30m - cache_save deploy_release: docker: - image: cimg/base:2021.04 steps: - setup_remote_docker: version: 19.03.13 docker_layer_caching: true - checkout - run: | echo "$QUAY_INFLUXDB_IOX_PASS" | docker login quay.io --username $QUAY_INFLUXDB_IOX_USER --password-stdin - run: name: Update docker branch tags command: | COMMIT_SHA="$(git rev-parse --short HEAD)" BRANCH="$(echo "$CIRCLE_BRANCH" | tr '[:upper:]' '[:lower:]' | sed 's/[^a-z0-9]/_/g')" docker pull quay.io/influxdb/iox:"$COMMIT_SHA" docker pull quay.io/influxdb/iox_data_generator:"$COMMIT_SHA" docker tag quay.io/influxdb/iox:"$COMMIT_SHA" quay.io/influxdb/iox:"$BRANCH" docker tag quay.io/influxdb/iox_data_generator:"$COMMIT_SHA" quay.io/influxdb/iox_data_generator:"$BRANCH" docker push quay.io/influxdb/iox:"$BRANCH" docker push quay.io/influxdb/iox_data_generator:"$BRANCH" echo "export COMMIT_SHA=${COMMIT_SHA}" >> $BASH_ENV - run: name: Deploy tags command: | echo "$QUAY_PASS" | docker login quay.io --username $QUAY_USER --password-stdin ./.circleci/get-deploy-tags.sh "${COMMIT_SHA}" # Prepare the CI image used for other tasks. # # A nightly job (scheduled below in the `workflows` section) to build the CI # image (influxdb/rust) used for the rest of the checks. # # To modify the contents of the CI image, update docker/Dockerfile.ci ci_image: # Use a `machine` executor instead of a docker-based because: # - we bootstrap our CI images somehow (we could use a docker-hub image as well) # - the machine executor also supports multi-arch builds # - we only run this job once a day, so the additional startup delay of 30-60s doesn't really matter machine: # https://circleci.com/docs/2.0/configuration-reference/#available-machine-images image: ubuntu-2004:202107-02 resource_class: xlarge steps: - checkout - run: | echo "$QUAY_PASS" | docker login quay.io --username $QUAY_USER --password-stdin - run: | COMMIT_SHA=$(git rev-parse --short HEAD) RUST_VERSION=$(sed -E -ne 's/channel = "(.*)"/\1/p' rust-toolchain.toml) docker --version docker build -t quay.io/influxdb/rust:$COMMIT_SHA -t quay.io/influxdb/rust:ci -f docker/Dockerfile.ci --build-arg RUST_VERSION=$RUST_VERSION . docker push --all-tags quay.io/influxdb/rust parameters: ci_image: description: "Trigger build of CI image" type: boolean default: false build_perf: # see comments below in build_perf job for usage description: "Trigger build of perf image" type: boolean default: false workflows: version: 2 # CI for all pull requests. ci: when: and: - not: << pipeline.parameters.ci_image >> - not: << pipeline.parameters.build_perf >> jobs: - fmt - lint - cargo_audit - protobuf-lint - docs-lint - test - test_heappy # Temporarily disable until perf can be updated to use NG # https://github.com/influxdata/influxdb_iox/issues/4485 # - test_perf - build_dev - doc - workspace_hack_checks - build_release: filters: branches: only: main - deploy_release: filters: branches: only: main requires: # Only deploy if all tests have passed - fmt - lint - cargo_audit - protobuf-lint - docs-lint - test - test_heappy # Temporarily disable until perf can be updated to use NG # https://github.com/influxdata/influxdb_iox/issues/4485 # - test_perf - build_dev - build_release - doc # Manual build of CI image ci_image: when: << pipeline.parameters.ci_image >> jobs: - ci_image # Manual build of release image for a branch. # Trigger using the CircleCI API, like so: # # # e.g. using 'xh' (https://github.com/ducaale/xh) # $ xh -a ':' POST \ # https://circleci.com/api/v2/project/github/influxdata/influxdb_iox/pipeline \ # parameters:='{"build_perf": true}' branch=chore/ci-tidy-up # # ...or equivalent with `curl`: # $ curl -XPOST -H "Content-Type: application/json" -H "Circle-Token: " \ # -d '{"parameters": {"build_perf": true}, "branch": "chore/ci-tidy-up"}' \ # https://circleci.com/api/v2/project/github/influxdata/influxdb_iox/pipeline build_perf: when: << pipeline.parameters.build_perf >> jobs: - build_release - deploy_release: requires: - build_release # Nightly rebuild of the build container ci_image_nightly: triggers: - schedule: cron: "0 5 * * *" filters: branches: only: - main jobs: - ci_image