486 lines
17 KiB
YAML
486 lines
17 KiB
YAML
# CI Overview
|
||
# -----------
|
||
#
|
||
# Each night:
|
||
#
|
||
# A build image is created (ci_image) from `docker/Dockerfile.ci` and is
|
||
# pushed to `quay.io/influxdb/rust:ci`. This build image is then used to run
|
||
# the CI tasks for the day.
|
||
#
|
||
# CI runs for git branches ending in `/perf`:
|
||
#
|
||
# Runs tests, fmt, & lints and then compiles binaries using the "release"
|
||
# cargo target and pushes a container with the binary to
|
||
# `quay.io/influxdb/fusion` (see perf_image below).
|
||
#
|
||
# CI for all other branches:
|
||
#
|
||
# - cargo build with the default cargo profile ("dev")
|
||
# - cargo test
|
||
# - cargo fmt
|
||
# - clippy (with warnings denied)
|
||
# - lint protobufs
|
||
# - check if generated flatbuffers code is up to date
|
||
|
||
version: 2.1
|
||
|
||
commands:
|
||
rust_components:
|
||
description: Verify installed components
|
||
steps:
|
||
- run:
|
||
name: Verify installed components
|
||
command: |
|
||
rustup --version
|
||
rustup show
|
||
cargo fmt --version
|
||
cargo clippy --version
|
||
|
||
cache_restore:
|
||
description: Restore Cargo Cache
|
||
steps:
|
||
- restore_cache:
|
||
name: Restoring Cargo Cache
|
||
keys:
|
||
- cargo-cache-{{ arch }}-{{ .Branch }}-{{ checksum "Cargo.lock" }}
|
||
- cargo-cache-{{ arch }}-{{ .Branch }}
|
||
- cargo-cache
|
||
cache_save:
|
||
description: Save Cargo Cache
|
||
steps:
|
||
- save_cache:
|
||
name: Save Cargo Cache
|
||
paths:
|
||
- /usr/local/cargo/registry
|
||
key: cargo-cache-{{ arch }}-{{ .Branch }}-{{ checksum "Cargo.lock" }}
|
||
|
||
jobs:
|
||
fmt:
|
||
docker:
|
||
- image: quay.io/influxdb/rust:ci
|
||
environment:
|
||
# Disable incremental compilation to avoid overhead. We are not preserving these files anyway.
|
||
CARGO_INCREMENTAL: "0"
|
||
# Disable full debug symbol generation to speed up CI build
|
||
# "1" means line tables only, which is useful for panic tracebacks.
|
||
RUSTFLAGS: "-C debuginfo=1"
|
||
steps:
|
||
- checkout
|
||
- rust_components
|
||
- cache_restore
|
||
- run:
|
||
name: Rust fmt
|
||
command: cargo fmt --all -- --check
|
||
- cache_save
|
||
lint:
|
||
docker:
|
||
- image: quay.io/influxdb/rust:ci
|
||
environment:
|
||
# Disable incremental compilation to avoid overhead. We are not preserving these files anyway.
|
||
CARGO_INCREMENTAL: "0"
|
||
# Disable full debug symbol generation to speed up CI build
|
||
# "1" means line tables only, which is useful for panic tracebacks.
|
||
RUSTFLAGS: "-C debuginfo=1"
|
||
steps:
|
||
- checkout
|
||
- rust_components
|
||
- cache_restore
|
||
- run:
|
||
name: Clippy
|
||
command: cargo clippy --all-targets --workspace -- -D warnings
|
||
- cache_save
|
||
cargo_audit:
|
||
docker:
|
||
- image: quay.io/influxdb/rust:ci
|
||
environment:
|
||
# Disable incremental compilation to avoid overhead. We are not preserving these files anyway.
|
||
CARGO_INCREMENTAL: "0"
|
||
# Disable full debug symbol generation to speed up CI build
|
||
# "1" means line tables only, which is useful for panic tracebacks.
|
||
RUSTFLAGS: "-C debuginfo=1"
|
||
steps:
|
||
- checkout
|
||
- rust_components
|
||
- cache_restore
|
||
- run:
|
||
name: Install Cargo Audit
|
||
command: cargo install --force cargo-audit
|
||
- run:
|
||
name: Cargo Audit
|
||
command: cargo audit
|
||
- cache_save
|
||
doc:
|
||
docker:
|
||
- image: quay.io/influxdb/rust:ci
|
||
environment:
|
||
# Disable incremental compilation to avoid overhead. We are not preserving these files anyway.
|
||
CARGO_INCREMENTAL: "0"
|
||
# Disable full debug symbol generation to speed up CI build
|
||
# "1" means line tables only, which is useful for panic tracebacks.
|
||
RUSTFLAGS: "-C debuginfo=1"
|
||
steps:
|
||
- checkout
|
||
- rust_components
|
||
- cache_restore
|
||
- run:
|
||
name: Cargo doc
|
||
# excluding datafusion because it's effectively a dependency masqueraded as workspace crate.
|
||
command: cargo doc --document-private-items --no-deps --workspace --exclude datafusion
|
||
- cache_save
|
||
- run:
|
||
name: Compress Docs
|
||
command: tar -cvzf rustdoc.tar.gz target/doc/
|
||
- store_artifacts:
|
||
path: rustdoc.tar.gz
|
||
|
||
test:
|
||
docker:
|
||
- image: quay.io/influxdb/rust:ci
|
||
resource_class: xlarge # use of a smaller executor tends crashes on link
|
||
environment:
|
||
# Disable incremental compilation to avoid overhead. We are not preserving these files anyway.
|
||
CARGO_INCREMENTAL: "0"
|
||
# Disable full debug symbol generation to speed up CI build
|
||
# "1" means line tables only, which is useful for panic tracebacks.
|
||
RUSTFLAGS: "-C debuginfo=1"
|
||
RUST_BACKTRACE: "1"
|
||
steps:
|
||
- checkout
|
||
- rust_components
|
||
- cache_restore
|
||
- run:
|
||
name: Cargo test
|
||
command: cargo test --workspace
|
||
- cache_save
|
||
|
||
# end to end tests with Heappy (heap profiling enabled)
|
||
test_heappy:
|
||
docker:
|
||
- image: quay.io/influxdb/rust:ci
|
||
resource_class: xlarge # use of a smaller executor tends crashes on link
|
||
environment:
|
||
# Disable incremental compilation to avoid overhead. We are not preserving these files anyway.
|
||
CARGO_INCREMENTAL: "0"
|
||
# Disable full debug symbol generation to speed up CI build
|
||
# "1" means line tables only, which is useful for panic tracebacks.
|
||
RUSTFLAGS: "-C debuginfo=1"
|
||
RUST_BACKTRACE: "1"
|
||
steps:
|
||
- checkout
|
||
- rust_components
|
||
- cache_restore
|
||
- run:
|
||
name: End to end tests (with heappy)
|
||
command: cargo test --no-default-features --features=heappy end_to_end
|
||
- cache_save
|
||
|
||
test_perf:
|
||
machine:
|
||
image: ubuntu-2004:202107-02
|
||
resource_class: xlarge
|
||
environment:
|
||
# Disable incremental compilation to avoid overhead. We are not preserving these files anyway.
|
||
CARGO_INCREMENTAL: "0"
|
||
# Disable full debug symbol generation to speed up CI build
|
||
# "1" means line tables only, which is useful for panic tracebacks.
|
||
RUSTFLAGS: "-C debuginfo=1"
|
||
RUST_BACKTRACE: "1"
|
||
steps:
|
||
- run: sudo apt update
|
||
- run:
|
||
name: Install required compile tools
|
||
command: sudo apt install curl clang lld pkg-config libssl-dev --yes --no-install-recommends
|
||
- checkout
|
||
|
||
- run:
|
||
name: Install rustup
|
||
command: scripts/install_rustup.sh
|
||
|
||
- run:
|
||
name: Install perf.py dependencies
|
||
command: python3 -m pip install -r perf/requirements.txt
|
||
|
||
- restore_cache:
|
||
keys:
|
||
- cargo-lock-{{ checksum "Cargo.lock" }}
|
||
- run:
|
||
name: Prime Rust build cache
|
||
command: cargo build --package influxdb_iox --bin influxdb_iox --package iox_data_generator --bin iox_data_generator
|
||
- save_cache:
|
||
key: cargo-lock-{{ checksum "Cargo.lock" }}
|
||
paths:
|
||
- target
|
||
|
||
- run:
|
||
name: Run basic perf test to ensure that perf.py works
|
||
command: perf/perf.py --debug --no-volumes --object-store memory battery-0
|
||
|
||
test_kafka_integration:
|
||
machine: true
|
||
resource_class: xlarge
|
||
environment:
|
||
# Disable incremental compilation to avoid overhead. We are not preserving these files anyway.
|
||
CARGO_INCREMENTAL: "0"
|
||
# Disable full debug symbol generation to speed up CI build
|
||
# "1" means line tables only, which is useful for panic tracebacks.
|
||
RUSTFLAGS: "-C debuginfo=1"
|
||
RUST_BACKTRACE: "1"
|
||
steps:
|
||
- checkout
|
||
- run:
|
||
name: Run integration tests with Docker Compose
|
||
command: docker-compose -f docker/ci-kafka-docker-compose.yml up --build --force-recreate --exit-code-from rust
|
||
|
||
# Integration tests for the influxdb2_client crate against InfluxDB 2.0 OSS.
|
||
test_influxdb2_client:
|
||
docker:
|
||
- image: quay.io/influxdb/rust:ci
|
||
environment:
|
||
# Disable incremental compilation to avoid overhead. We are not preserving these files anyway.
|
||
CARGO_INCREMENTAL: "0"
|
||
# Disable full debug symbol generation to speed up CI build
|
||
# "1" means line tables only, which is useful for panic tracebacks.
|
||
RUSTFLAGS: "-C debuginfo=1"
|
||
RUST_BACKTRACE: "1"
|
||
steps:
|
||
- checkout
|
||
- rust_components
|
||
- cache_restore
|
||
- run:
|
||
name: Cargo test
|
||
command: TEST_INTEGRATION=1 INFLUXDB_IOX_INTEGRATION_LOCAL=1 cargo test -p influxdb2_client
|
||
- cache_save
|
||
|
||
# Build a dev binary.
|
||
#
|
||
# Compiles a binary with the default ("dev") cargo profile from the iox source
|
||
# using the latest ci_image.
|
||
build:
|
||
docker:
|
||
- image: quay.io/influxdb/rust:ci
|
||
resource_class: xlarge # use of a smaller executor tends crashes on link
|
||
environment:
|
||
# Disable incremental compilation to avoid overhead. We are not preserving these files anyway.
|
||
CARGO_INCREMENTAL: "0"
|
||
# Disable full debug symbol generation to speed up CI build
|
||
# "1" means line tables only, which is useful for panic tracebacks.
|
||
RUSTFLAGS: "-C debuginfo=1"
|
||
steps:
|
||
- checkout
|
||
- rust_components
|
||
- cache_restore
|
||
- run:
|
||
name: Cargo build
|
||
command: cargo build --workspace
|
||
- run:
|
||
name: Build benches
|
||
command: cargo test --workspace --benches --no-run
|
||
- run:
|
||
name: Build with object store + exporter support + HEAP profiling
|
||
command: cargo build --no-default-features --features="aws,gcp,azure,heappy,pprof"
|
||
- cache_save
|
||
|
||
# Lint protobufs.
|
||
protobuf-lint:
|
||
docker:
|
||
- image: bufbuild/buf:0.40.0
|
||
environment:
|
||
# Value to look for to skip breaking changes check
|
||
SKIP_LABEL: "https://api.github.com/repos/influxdata/influxdb_iox/labels/incompatible%20protobuf"
|
||
steps:
|
||
- checkout
|
||
- run:
|
||
name: buf lint
|
||
command: buf lint
|
||
- run:
|
||
name: buf breaking changes
|
||
command: |
|
||
echo "If you want to make changes forbidden by this lint, please"
|
||
echo "coordinate with the conductor team, add the 'incompatible protobuf' label"
|
||
echo "to the PR, and rerun this test"
|
||
# Check if label is present using github API:
|
||
# Inspired by https://discuss.circleci.com/t/tag-label-filter/11158
|
||
if wget -O - https://api.github.com/repos/influxdata/influxdb_iox/issues/$(echo $CIRCLE_PULL_REQUEST | grep -oE "[^/pull]+$") | grep "$SKIP_LABEL" ; then echo "SKIPPING (FOUND LABEL)" && exit ; else echo "CHECKING (NO LABEL FOUND)"; fi
|
||
git fetch origin main
|
||
# compare against only changes in this branch (not against
|
||
# other stuff that may have been added to master since last merge)
|
||
MERGE_BASE=$(git merge-base origin/main $CIRCLE_BRANCH) sh -c 'buf breaking --against ".git#ref=$MERGE_BASE"'
|
||
|
||
# Check that any generated files are is up-to-date with the changes in this PR.
|
||
# named "check-flatbuffers" because that name is hardcoded into github checks
|
||
check-flatbuffers:
|
||
docker:
|
||
- image: quay.io/influxdb/rust:ci
|
||
resource_class: xlarge # use of a smaller executor tends crashes on link
|
||
steps:
|
||
- checkout
|
||
- rust_components # Regenerating flatbuffers uses rustfmt
|
||
- run:
|
||
name: Check Query Tests
|
||
command: ./query_tests/check-generated.sh
|
||
- run:
|
||
name: Check Flatbuffers
|
||
command: INFLUXDB_IOX_INTEGRATION_LOCAL=1 ./entry/check-flatbuffers.sh
|
||
|
||
# Compile a cargo "release" profile binary for branches that end in `/perf`
|
||
#
|
||
# Uses the latest ci_image (influxdb/rust below) to build a release binary and
|
||
# copies it to a minimal container image based upon `rust:slim-buster`. This
|
||
# minimal image is then pushed to `quay.io/influxdb/fusion:${BRANCH}` with '/'
|
||
# repaced by '.' - as an example:
|
||
#
|
||
# git branch: dom/my-awesome-feature/perf
|
||
# container: quay.io/influxdb/fusion:dom.my-awesome-feature.perf
|
||
#
|
||
# Subsequent CI runs will overwrite the tag if you push more changes, so watch
|
||
# out for parallel CI runs!
|
||
#
|
||
# To change the contents of the build container, modify docker/Dockerfile.ci
|
||
# To change the final release container, modify docker/Dockerfile.iox
|
||
perf_image:
|
||
docker:
|
||
- image: quay.io/influxdb/rust:ci
|
||
resource_class: xlarge # use of a smaller executor tends crashes on link
|
||
environment:
|
||
# Disable incremental compilation to avoid overhead. We are not preserving these files anyway.
|
||
CARGO_INCREMENTAL: "0"
|
||
# We keep the debug symbols (Enabled in Cargo.toml as debug = true)
|
||
# workaround dynamic CPU detection bug in croaring
|
||
# https://github.com/influxdata/influxdb_iox/pull/2119
|
||
ROARING_ARCH: "haswell"
|
||
# Use avx512 instructions to take full advantage of the CPUs instruction set
|
||
RUSTFLAGS: "-C target-feature=+avx2"
|
||
steps:
|
||
- checkout
|
||
- rust_components
|
||
- cache_restore
|
||
- run:
|
||
name: Cargo release build with target arch set for CRoaring
|
||
command: cargo build --release --no-default-features --features="aws,gcp,azure,jemalloc_replacing_malloc"
|
||
- run: |
|
||
echo sha256sum after build is
|
||
sha256sum target/release/influxdb_iox
|
||
- run:
|
||
name: Print rustc target CPU options
|
||
command: target/release/influxdb_iox debug print-cpu
|
||
- setup_remote_docker:
|
||
# There seems to be a cache invalidation bug in docker
|
||
# or in the way that circleci implements layer caching.
|
||
# Disabling for now, and tracked further investigations
|
||
# in https://github.com/influxdata/k8s-idpe/issues/3038
|
||
docker_layer_caching: false
|
||
version: 19.03.14
|
||
- run: |
|
||
sudo apt-get update
|
||
sudo apt-get install -y docker.io
|
||
- run: |
|
||
echo "$QUAY_PASS" | docker login quay.io --username $QUAY_USER --password-stdin
|
||
- run:
|
||
# Docker has functionality to support per-Dockerfile .dockerignore
|
||
# This was added in https://github.com/moby/buildkit/pull/901
|
||
# And included in 19.03 - https://docs.docker.com/engine/release-notes/19.03/#19030
|
||
# Unfortunately CircleCI only seems to understand a root-level .dockerignore
|
||
# So we need to move it into position for it to not send ~10GB of build context
|
||
name: Fudge CircleCI Docker Context
|
||
command: mv docker/Dockerfile.iox.dockerignore .dockerignore
|
||
- run: |
|
||
BRANCH=$(git rev-parse --abbrev-ref HEAD | tr '/' '.')
|
||
echo sha256sum after build is
|
||
sha256sum target/release/influxdb_iox
|
||
COMMIT_SHA=$(git rev-parse --short HEAD)
|
||
docker build -t quay.io/influxdb/fusion:$COMMIT_SHA -f docker/Dockerfile.iox .
|
||
docker push quay.io/influxdb/fusion:$COMMIT_SHA
|
||
echo "export COMMIT_SHA=${COMMIT_SHA}" >> $BASH_ENV
|
||
- run:
|
||
name: Deploy tags
|
||
command: ./.circleci/get-deploy-tags.sh "${COMMIT_SHA}"
|
||
- cache_save
|
||
|
||
# Prepare the CI image used for other tasks.
|
||
#
|
||
# A nightly job (scheduled below in the `workflows` section) to build the CI
|
||
# image (influxdb/rust) used for the rest of the checks.
|
||
#
|
||
# To modify the contents of the CI image, update docker/Dockerfile.ci
|
||
ci_image:
|
||
machine: true
|
||
resource_class: xlarge
|
||
steps:
|
||
- checkout
|
||
- run: |
|
||
echo "$QUAY_PASS" | docker login quay.io --username $QUAY_USER --password-stdin
|
||
- run: |
|
||
COMMIT_SHA=$(git rev-parse --short HEAD)
|
||
RUST_VERSION=$(sed -E -ne 's/channel = "(.*)"/\1/p' rust-toolchain.toml)
|
||
docker build -t quay.io/influxdb/rust:$COMMIT_SHA -f docker/Dockerfile.ci --build-arg RUST_VERSION=$RUST_VERSION .
|
||
docker tag quay.io/influxdb/rust:$COMMIT_SHA quay.io/influxdb/rust:ci
|
||
docker push quay.io/influxdb/rust:$COMMIT_SHA
|
||
docker push quay.io/influxdb/rust:ci
|
||
|
||
workflows:
|
||
version: 2
|
||
|
||
# CI for all pull requests.
|
||
ci:
|
||
jobs:
|
||
- fmt
|
||
- lint
|
||
- cargo_audit
|
||
- protobuf-lint
|
||
- test
|
||
- test_heappy
|
||
- test_perf
|
||
- test_kafka_integration
|
||
- test_influxdb2_client
|
||
- build
|
||
- check-flatbuffers
|
||
- doc
|
||
|
||
# Internal pipeline for perf builds.
|
||
#
|
||
# Filter ensures this only runs for git branches ending in `/perf`.
|
||
perf_build:
|
||
jobs:
|
||
- fmt:
|
||
filters:
|
||
branches:
|
||
only: main
|
||
- lint:
|
||
filters:
|
||
branches:
|
||
only: main
|
||
- cargo_audit:
|
||
filters:
|
||
branches:
|
||
only: main
|
||
- test:
|
||
filters:
|
||
branches:
|
||
only: main
|
||
- build:
|
||
filters:
|
||
branches:
|
||
only: main
|
||
- perf_image:
|
||
filters:
|
||
branches:
|
||
only: main
|
||
requires: # Only do a release build if all tests have passed
|
||
- fmt
|
||
- lint
|
||
- cargo_audit
|
||
- test
|
||
- build
|
||
|
||
# Nightly rebuild of the build container
|
||
ci_image:
|
||
triggers:
|
||
- schedule:
|
||
cron: "0 5 * * *"
|
||
filters:
|
||
branches:
|
||
only:
|
||
- main
|
||
jobs:
|
||
- ci_image
|