influxdb/.circleci/config.yml

353 lines
12 KiB
YAML
Raw Blame History

This file contains invisible Unicode characters!

This file contains invisible Unicode characters that may be processed differently from what appears below. If your use case is intentional and legitimate, you can safely ignore this warning. Use the Escape button to reveal hidden characters.

# CI Overview
# -----------
#
# Each night:
#
# A build image is created (ci_image) from `docker/Dockerfile.ci` and is
# pushed to `quay.io/influxdb/rust:ci`. This build image is then used to run
# the CI tasks for the day.
#
# CI runs for git branches ending in `/perf`:
#
# Runs tests, fmt, & lints and then compiles binaries using the "release"
# cargo target and pushes a container with the binary to
# `quay.io/influxdb/fusion` (see perf_image below).
#
# CI for all other branches:
#
# - cargo build with the default cargo profile ("dev")
# - cargo test
# - cargo fmt
# - clippy (with warnings denied)
# - lint protobufs
# - check if generated flatbuffers code is up to date
version: 2.1
commands:
rust_components:
description: Verify installed components
steps:
- run:
name: Verify installed components
command: |
rustup --version
rustup show
cargo fmt --version
cargo clippy --version
cache_restore:
description: Restore Cargo Cache
steps:
- restore_cache:
name: Restoring Cargo Cache
keys:
- cargo-cache-{{ arch }}-{{ .Branch }}-{{ checksum "Cargo.lock" }}
- cargo-cache-{{ arch }}-{{ .Branch }}
- cargo-cache
cache_save:
description: Save Cargo Cache
steps:
- save_cache:
name: Save Cargo Cache
paths:
- /usr/local/cargo/registry
key: cargo-cache-{{ arch }}-{{ .Branch }}-{{ checksum "Cargo.lock" }}
jobs:
fmt:
docker:
- image: quay.io/influxdb/rust:ci
environment:
# Disable incremental compilation to avoid overhead. We are not preserving these files anyway.
CARGO_INCREMENTAL: "0"
# Disable full debug symbol generation to speed up CI build
# "1" means line tables only, which is useful for panic tracebacks.
RUSTFLAGS: "-C debuginfo=1"
steps:
- checkout
- rust_components
- cache_restore
- run:
name: Rust fmt
command: cargo fmt --all -- --check
- cache_save
lint:
docker:
- image: quay.io/influxdb/rust:ci
environment:
# Disable incremental compilation to avoid overhead. We are not preserving these files anyway.
CARGO_INCREMENTAL: "0"
# Disable full debug symbol generation to speed up CI build
# "1" means line tables only, which is useful for panic tracebacks.
RUSTFLAGS: "-C debuginfo=1"
steps:
- checkout
- rust_components
- cache_restore
- run:
name: Clippy
command: cargo clippy --all-targets --workspace -- -D warnings
- cache_save
doc:
docker:
- image: quay.io/influxdb/rust:ci
environment:
# Disable incremental compilation to avoid overhead. We are not preserving these files anyway.
CARGO_INCREMENTAL: "0"
# Disable full debug symbol generation to speed up CI build
# "1" means line tables only, which is useful for panic tracebacks.
RUSTFLAGS: "-C debuginfo=1"
steps:
- checkout
- rust_components
- cache_restore
- run:
name: Cargo doc
# excluding datafusion because it's effectively a dependency masqueraded as workspace crate.
command: cargo doc --document-private-items --no-deps --workspace --exclude datafusion
- cache_save
- store_artifacts:
path: target/doc/
destination: rustdoc
test:
docker:
- image: quay.io/influxdb/rust:ci
resource_class: xlarge # use of a smaller executor tends crashes on link
environment:
# Disable incremental compilation to avoid overhead. We are not preserving these files anyway.
CARGO_INCREMENTAL: "0"
# Disable full debug symbol generation to speed up CI build
# "1" means line tables only, which is useful for panic tracebacks.
RUSTFLAGS: "-C debuginfo=1"
RUST_BACKTRACE: "1"
steps:
- checkout
- rust_components
- cache_restore
- run:
name: Cargo test
command: cargo test --workspace
- cache_save
test_kafka_integration:
machine: true
resource_class: xlarge
environment:
# Disable incremental compilation to avoid overhead. We are not preserving these files anyway.
CARGO_INCREMENTAL: "0"
# Disable full debug symbol generation to speed up CI build
# "1" means line tables only, which is useful for panic tracebacks.
RUSTFLAGS: "-C debuginfo=1"
RUST_BACKTRACE: "1"
steps:
- checkout
- run:
name: Run integration tests with Docker Compose
command: docker-compose -f docker/ci-kafka-docker-compose.yml up --build --force-recreate --exit-code-from rust
# Integration tests for the influxdb2_client crate against InfluxDB 2.0 OSS.
test_influxdb2_client:
docker:
- image: quay.io/influxdb/rust:ci
environment:
# Disable incremental compilation to avoid overhead. We are not preserving these files anyway.
CARGO_INCREMENTAL: "0"
# Disable full debug symbol generation to speed up CI build
# "1" means line tables only, which is useful for panic tracebacks.
RUSTFLAGS: "-C debuginfo=1"
RUST_BACKTRACE: "1"
steps:
- checkout
- rust_components
- cache_restore
- run:
name: Cargo test
command: TEST_INTEGRATION=1 INFLUXDB_IOX_INTEGRATION_LOCAL=1 cargo test -p influxdb2_client
- cache_save
# Build a dev binary.
#
# Compiles a binary with the default ("dev") cargo profile from the iox source
# using the latest ci_image.
build:
docker:
- image: quay.io/influxdb/rust:ci
resource_class: xlarge # use of a smaller executor tends crashes on link
environment:
# Disable incremental compilation to avoid overhead. We are not preserving these files anyway.
CARGO_INCREMENTAL: "0"
# Disable full debug symbol generation to speed up CI build
# "1" means line tables only, which is useful for panic tracebacks.
RUSTFLAGS: "-C debuginfo=1"
steps:
- checkout
- rust_components
- cache_restore
- run:
name: Cargo build
command: cargo build --workspace
- run:
name: Build benches
command: cargo test --workspace --benches --no-run
- cache_save
# Lint protobufs.
protobuf-lint:
docker:
- image: bufbuild/buf:0.40.0
steps:
- checkout
- run:
name: buf lint
command: buf lint
# Check that the generated flatbuffers code is up-to-date with the changes in this PR.
check-flatbuffers:
docker:
- image: quay.io/influxdb/rust:ci
resource_class: xlarge # use of a smaller executor tends crashes on link
steps:
- checkout
- rust_components # Regenerating flatbuffers uses rustfmt
- run:
name: Check Flatbuffers
command: INFLUXDB_IOX_INTEGRATION_LOCAL=1 ./entry/check-flatbuffers.sh
# Compile a cargo "release" profile binary for branches that end in `/perf`
#
# Uses the latest ci_image (influxdb/rust below) to build a release binary and
# copies it to a minimal container image based upon `rust:slim-buster`. This
# minimal image is then pushed to `quay.io/influxdb/fusion:${BRANCH}` with '/'
# repaced by '.' - as an example:
#
# git branch: dom/my-awesome-feature/perf
# container: quay.io/influxdb/fusion:dom.my-awesome-feature.perf
#
# Subsequent CI runs will overwrite the tag if you push more changes, so watch
# out for parallel CI runs!
#
# To change the contents of the build container, modify docker/Dockerfile.ci
# To change the final release container, modify docker/Dockerfile.iox
perf_image:
docker:
- image: quay.io/influxdb/rust:ci
resource_class: xlarge # use of a smaller executor tends crashes on link
environment:
# Disable incremental compilation to avoid overhead. We are not preserving these files anyway.
CARGO_INCREMENTAL: "0"
# We keep the debug symbols (Enabled in Cargo.toml as debug = true)
steps:
- checkout
- rust_components
- cache_restore
- run:
name: Cargo release build with target arch set for CRoaring
command: ROARING_ARCH=x86-64 cargo build --release
- run: |
echo sha256sum after build is
sha256sum target/release/influxdb_iox
- setup_remote_docker:
# There seems to be a cache invalidation bug in docker
# or in the way that circleci implements layer caching.
# Disabling for now, and tracked further investigations
# in https://github.com/influxdata/k8s-idpe/issues/3038
docker_layer_caching: false
- run: |
sudo apt-get update
sudo apt-get install -y docker.io
- run: |
echo "$QUAY_PASS" | docker login quay.io --username $QUAY_USER --password-stdin
- run: |
BRANCH=$(git rev-parse --abbrev-ref HEAD | tr '/' '.')
echo sha256sum after build is
sha256sum target/release/influxdb_iox
COMMIT_SHA=$(git rev-parse --short HEAD)
docker build -t quay.io/influxdb/fusion:$COMMIT_SHA -f docker/Dockerfile.iox .
docker push quay.io/influxdb/fusion:$COMMIT_SHA
echo "export COMMIT_SHA=${COMMIT_SHA}" >> $BASH_ENV
- run:
name: Deploy tags
command: ./.circleci/get-deploy-tags.sh "${COMMIT_SHA}"
- cache_save
# Prepare the CI image used for other tasks.
#
# A nightly job (scheduled below in the `workflows` section) to build the CI
# image (influxdb/rust) used for the rest of the checks.
#
# To modify the contents of the CI image, update docker/Dockerfile.ci
ci_image:
machine: true
resource_class: xlarge
steps:
- checkout
- run: |
echo "$QUAY_PASS" | docker login quay.io --username $QUAY_USER --password-stdin
- run: |
COMMIT_SHA=$(git rev-parse --short HEAD)
RUST_VERSION=$(sed -E -ne 's/channel = "(.*)"/\1/p' rust-toolchain.toml)
docker build -t quay.io/influxdb/rust:$COMMIT_SHA -f docker/Dockerfile.ci --build-arg RUST_VERSION=$RUST_VERSION .
docker tag quay.io/influxdb/rust:$COMMIT_SHA quay.io/influxdb/rust:ci
docker push quay.io/influxdb/rust:$COMMIT_SHA
docker push quay.io/influxdb/rust:ci
workflows:
version: 2
# CI for all pull requests.
ci:
jobs:
- fmt
- lint
- protobuf-lint
- test
- test_kafka_integration
- test_influxdb2_client
- build
- check-flatbuffers
- doc
# Internal pipeline for perf builds.
#
# Filter ensures this only runs for git branches ending in `/perf`.
perf_build:
jobs:
- fmt:
filters:
branches:
only: main
- lint:
filters:
branches:
only: main
- test:
filters:
branches:
only: main
- build:
filters:
branches:
only: main
- perf_image:
filters:
branches:
only: main
requires: # Only do a release build if all tests have passed
- fmt
- lint
- test
- build
# Nightly rebuild of the build container
ci_image:
triggers:
- schedule:
cron: "0 5 * * *"
filters:
branches:
only:
- main
jobs:
- ci_image