353 lines
12 KiB
YAML
353 lines
12 KiB
YAML
# CI Overview
|
||
# -----------
|
||
#
|
||
# Each night:
|
||
#
|
||
# A build image is created (ci_image) from `docker/Dockerfile.ci` and is
|
||
# pushed to `quay.io/influxdb/rust:ci`. This build image is then used to run
|
||
# the CI tasks for the day.
|
||
#
|
||
# CI runs for git branches ending in `/perf`:
|
||
#
|
||
# Runs tests, fmt, & lints and then compiles binaries using the "release"
|
||
# cargo target and pushes a container with the binary to
|
||
# `quay.io/influxdb/fusion` (see perf_image below).
|
||
#
|
||
# CI for all other branches:
|
||
#
|
||
# - cargo build with the default cargo profile ("dev")
|
||
# - cargo test
|
||
# - cargo fmt
|
||
# - clippy (with warnings denied)
|
||
# - lint protobufs
|
||
# - check if generated flatbuffers code is up to date
|
||
|
||
version: 2.1
|
||
|
||
commands:
|
||
rust_components:
|
||
description: Verify installed components
|
||
steps:
|
||
- run:
|
||
name: Verify installed components
|
||
command: |
|
||
rustup --version
|
||
rustup show
|
||
cargo fmt --version
|
||
cargo clippy --version
|
||
|
||
cache_restore:
|
||
description: Restore Cargo Cache
|
||
steps:
|
||
- restore_cache:
|
||
name: Restoring Cargo Cache
|
||
keys:
|
||
- cargo-cache-{{ arch }}-{{ .Branch }}-{{ checksum "Cargo.lock" }}
|
||
- cargo-cache-{{ arch }}-{{ .Branch }}
|
||
- cargo-cache
|
||
cache_save:
|
||
description: Save Cargo Cache
|
||
steps:
|
||
- save_cache:
|
||
name: Save Cargo Cache
|
||
paths:
|
||
- /usr/local/cargo/registry
|
||
key: cargo-cache-{{ arch }}-{{ .Branch }}-{{ checksum "Cargo.lock" }}
|
||
|
||
jobs:
|
||
fmt:
|
||
docker:
|
||
- image: quay.io/influxdb/rust:ci
|
||
environment:
|
||
# Disable incremental compilation to avoid overhead. We are not preserving these files anyway.
|
||
CARGO_INCREMENTAL: "0"
|
||
# Disable full debug symbol generation to speed up CI build
|
||
# "1" means line tables only, which is useful for panic tracebacks.
|
||
RUSTFLAGS: "-C debuginfo=1"
|
||
steps:
|
||
- checkout
|
||
- rust_components
|
||
- cache_restore
|
||
- run:
|
||
name: Rust fmt
|
||
command: cargo fmt --all -- --check
|
||
- cache_save
|
||
lint:
|
||
docker:
|
||
- image: quay.io/influxdb/rust:ci
|
||
environment:
|
||
# Disable incremental compilation to avoid overhead. We are not preserving these files anyway.
|
||
CARGO_INCREMENTAL: "0"
|
||
# Disable full debug symbol generation to speed up CI build
|
||
# "1" means line tables only, which is useful for panic tracebacks.
|
||
RUSTFLAGS: "-C debuginfo=1"
|
||
steps:
|
||
- checkout
|
||
- rust_components
|
||
- cache_restore
|
||
- run:
|
||
name: Clippy
|
||
command: cargo clippy --all-targets --workspace -- -D warnings
|
||
- cache_save
|
||
doc:
|
||
docker:
|
||
- image: quay.io/influxdb/rust:ci
|
||
environment:
|
||
# Disable incremental compilation to avoid overhead. We are not preserving these files anyway.
|
||
CARGO_INCREMENTAL: "0"
|
||
# Disable full debug symbol generation to speed up CI build
|
||
# "1" means line tables only, which is useful for panic tracebacks.
|
||
RUSTFLAGS: "-C debuginfo=1"
|
||
steps:
|
||
- checkout
|
||
- rust_components
|
||
- cache_restore
|
||
- run:
|
||
name: Cargo doc
|
||
# excluding datafusion because it's effectively a dependency masqueraded as workspace crate.
|
||
command: cargo doc --document-private-items --no-deps --workspace --exclude datafusion
|
||
- cache_save
|
||
- store_artifacts:
|
||
path: target/doc/
|
||
destination: rustdoc
|
||
test:
|
||
docker:
|
||
- image: quay.io/influxdb/rust:ci
|
||
resource_class: xlarge # use of a smaller executor tends crashes on link
|
||
environment:
|
||
# Disable incremental compilation to avoid overhead. We are not preserving these files anyway.
|
||
CARGO_INCREMENTAL: "0"
|
||
# Disable full debug symbol generation to speed up CI build
|
||
# "1" means line tables only, which is useful for panic tracebacks.
|
||
RUSTFLAGS: "-C debuginfo=1"
|
||
RUST_BACKTRACE: "1"
|
||
steps:
|
||
- checkout
|
||
- rust_components
|
||
- cache_restore
|
||
- run:
|
||
name: Cargo test
|
||
command: cargo test --workspace
|
||
- cache_save
|
||
|
||
test_kafka_integration:
|
||
machine: true
|
||
resource_class: xlarge
|
||
environment:
|
||
# Disable incremental compilation to avoid overhead. We are not preserving these files anyway.
|
||
CARGO_INCREMENTAL: "0"
|
||
# Disable full debug symbol generation to speed up CI build
|
||
# "1" means line tables only, which is useful for panic tracebacks.
|
||
RUSTFLAGS: "-C debuginfo=1"
|
||
RUST_BACKTRACE: "1"
|
||
steps:
|
||
- checkout
|
||
- run:
|
||
name: Run integration tests with Docker Compose
|
||
command: docker-compose -f docker/ci-kafka-docker-compose.yml up --build --force-recreate --exit-code-from rust
|
||
|
||
# Integration tests for the influxdb2_client crate against InfluxDB 2.0 OSS.
|
||
test_influxdb2_client:
|
||
docker:
|
||
- image: quay.io/influxdb/rust:ci
|
||
environment:
|
||
# Disable incremental compilation to avoid overhead. We are not preserving these files anyway.
|
||
CARGO_INCREMENTAL: "0"
|
||
# Disable full debug symbol generation to speed up CI build
|
||
# "1" means line tables only, which is useful for panic tracebacks.
|
||
RUSTFLAGS: "-C debuginfo=1"
|
||
RUST_BACKTRACE: "1"
|
||
steps:
|
||
- checkout
|
||
- rust_components
|
||
- cache_restore
|
||
- run:
|
||
name: Cargo test
|
||
command: TEST_INTEGRATION=1 INFLUXDB_IOX_INTEGRATION_LOCAL=1 cargo test -p influxdb2_client
|
||
- cache_save
|
||
|
||
# Build a dev binary.
|
||
#
|
||
# Compiles a binary with the default ("dev") cargo profile from the iox source
|
||
# using the latest ci_image.
|
||
build:
|
||
docker:
|
||
- image: quay.io/influxdb/rust:ci
|
||
resource_class: xlarge # use of a smaller executor tends crashes on link
|
||
environment:
|
||
# Disable incremental compilation to avoid overhead. We are not preserving these files anyway.
|
||
CARGO_INCREMENTAL: "0"
|
||
# Disable full debug symbol generation to speed up CI build
|
||
# "1" means line tables only, which is useful for panic tracebacks.
|
||
RUSTFLAGS: "-C debuginfo=1"
|
||
steps:
|
||
- checkout
|
||
- rust_components
|
||
- cache_restore
|
||
- run:
|
||
name: Cargo build
|
||
command: cargo build --workspace
|
||
- run:
|
||
name: Build benches
|
||
command: cargo test --workspace --benches --no-run
|
||
- cache_save
|
||
|
||
# Lint protobufs.
|
||
protobuf-lint:
|
||
docker:
|
||
- image: bufbuild/buf:0.40.0
|
||
steps:
|
||
- checkout
|
||
- run:
|
||
name: buf lint
|
||
command: buf lint
|
||
|
||
# Check that the generated flatbuffers code is up-to-date with the changes in this PR.
|
||
check-flatbuffers:
|
||
docker:
|
||
- image: quay.io/influxdb/rust:ci
|
||
resource_class: xlarge # use of a smaller executor tends crashes on link
|
||
steps:
|
||
- checkout
|
||
- rust_components # Regenerating flatbuffers uses rustfmt
|
||
- run:
|
||
name: Check Flatbuffers
|
||
command: INFLUXDB_IOX_INTEGRATION_LOCAL=1 ./entry/check-flatbuffers.sh
|
||
|
||
# Compile a cargo "release" profile binary for branches that end in `/perf`
|
||
#
|
||
# Uses the latest ci_image (influxdb/rust below) to build a release binary and
|
||
# copies it to a minimal container image based upon `rust:slim-buster`. This
|
||
# minimal image is then pushed to `quay.io/influxdb/fusion:${BRANCH}` with '/'
|
||
# repaced by '.' - as an example:
|
||
#
|
||
# git branch: dom/my-awesome-feature/perf
|
||
# container: quay.io/influxdb/fusion:dom.my-awesome-feature.perf
|
||
#
|
||
# Subsequent CI runs will overwrite the tag if you push more changes, so watch
|
||
# out for parallel CI runs!
|
||
#
|
||
# To change the contents of the build container, modify docker/Dockerfile.ci
|
||
# To change the final release container, modify docker/Dockerfile.iox
|
||
perf_image:
|
||
docker:
|
||
- image: quay.io/influxdb/rust:ci
|
||
resource_class: xlarge # use of a smaller executor tends crashes on link
|
||
environment:
|
||
# Disable incremental compilation to avoid overhead. We are not preserving these files anyway.
|
||
CARGO_INCREMENTAL: "0"
|
||
# We keep the debug symbols (Enabled in Cargo.toml as debug = true)
|
||
steps:
|
||
- checkout
|
||
- rust_components
|
||
- cache_restore
|
||
- run:
|
||
name: Cargo release build with target arch set for CRoaring
|
||
command: ROARING_ARCH=x86-64 cargo build --release
|
||
- run: |
|
||
echo sha256sum after build is
|
||
sha256sum target/release/influxdb_iox
|
||
- setup_remote_docker:
|
||
# There seems to be a cache invalidation bug in docker
|
||
# or in the way that circleci implements layer caching.
|
||
# Disabling for now, and tracked further investigations
|
||
# in https://github.com/influxdata/k8s-idpe/issues/3038
|
||
docker_layer_caching: false
|
||
- run: |
|
||
sudo apt-get update
|
||
sudo apt-get install -y docker.io
|
||
- run: |
|
||
echo "$QUAY_PASS" | docker login quay.io --username $QUAY_USER --password-stdin
|
||
- run: |
|
||
BRANCH=$(git rev-parse --abbrev-ref HEAD | tr '/' '.')
|
||
echo sha256sum after build is
|
||
sha256sum target/release/influxdb_iox
|
||
COMMIT_SHA=$(git rev-parse --short HEAD)
|
||
docker build -t quay.io/influxdb/fusion:$COMMIT_SHA -f docker/Dockerfile.iox .
|
||
docker push quay.io/influxdb/fusion:$COMMIT_SHA
|
||
echo "export COMMIT_SHA=${COMMIT_SHA}" >> $BASH_ENV
|
||
- run:
|
||
name: Deploy tags
|
||
command: ./.circleci/get-deploy-tags.sh "${COMMIT_SHA}"
|
||
- cache_save
|
||
|
||
# Prepare the CI image used for other tasks.
|
||
#
|
||
# A nightly job (scheduled below in the `workflows` section) to build the CI
|
||
# image (influxdb/rust) used for the rest of the checks.
|
||
#
|
||
# To modify the contents of the CI image, update docker/Dockerfile.ci
|
||
ci_image:
|
||
machine: true
|
||
resource_class: xlarge
|
||
steps:
|
||
- checkout
|
||
- run: |
|
||
echo "$QUAY_PASS" | docker login quay.io --username $QUAY_USER --password-stdin
|
||
- run: |
|
||
COMMIT_SHA=$(git rev-parse --short HEAD)
|
||
RUST_VERSION=$(sed -E -ne 's/channel = "(.*)"/\1/p' rust-toolchain.toml)
|
||
docker build -t quay.io/influxdb/rust:$COMMIT_SHA -f docker/Dockerfile.ci --build-arg RUST_VERSION=$RUST_VERSION .
|
||
docker tag quay.io/influxdb/rust:$COMMIT_SHA quay.io/influxdb/rust:ci
|
||
docker push quay.io/influxdb/rust:$COMMIT_SHA
|
||
docker push quay.io/influxdb/rust:ci
|
||
|
||
workflows:
|
||
version: 2
|
||
|
||
# CI for all pull requests.
|
||
ci:
|
||
jobs:
|
||
- fmt
|
||
- lint
|
||
- protobuf-lint
|
||
- test
|
||
- test_kafka_integration
|
||
- test_influxdb2_client
|
||
- build
|
||
- check-flatbuffers
|
||
- doc
|
||
|
||
# Internal pipeline for perf builds.
|
||
#
|
||
# Filter ensures this only runs for git branches ending in `/perf`.
|
||
perf_build:
|
||
jobs:
|
||
- fmt:
|
||
filters:
|
||
branches:
|
||
only: main
|
||
- lint:
|
||
filters:
|
||
branches:
|
||
only: main
|
||
- test:
|
||
filters:
|
||
branches:
|
||
only: main
|
||
- build:
|
||
filters:
|
||
branches:
|
||
only: main
|
||
- perf_image:
|
||
filters:
|
||
branches:
|
||
only: main
|
||
requires: # Only do a release build if all tests have passed
|
||
- fmt
|
||
- lint
|
||
- test
|
||
- build
|
||
|
||
# Nightly rebuild of the build container
|
||
ci_image:
|
||
triggers:
|
||
- schedule:
|
||
cron: "0 5 * * *"
|
||
filters:
|
||
branches:
|
||
only:
|
||
- main
|
||
jobs:
|
||
- ci_image
|