Influxdata
/
influxdb
mirror of https://github.com/influxdata/influxdb.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity
influxdb/.circleci/config.yml

658 lines
25 KiB
YAML
Raw Blame History

---
# CI Overview
# -----------
#
# Each night:
#
# A build image is created (ci_image) from `docker/Dockerfile.ci` and is
# pushed to `quay.io/influxdb/rust:ci`. This build image is then used to run
# the CI tasks for the day.
#
# Every commit:
#
# The CI for every PR and merge to main runs tests, fmt, lints and compiles debug binaries
#
# On main if all these checks pass it will then additionally compile in "release" mode and
# publish a docker image to quay.io/influxdb/iox:$COMMIT_SHA
#
# Manually trigger build and push of container image for a branch:
#
# Navigate to https://app.circleci.com/pipelines/github/influxdata/influxdb_iox?branch=<branch-name> (<- change this!)
# Then:
#
# - Click "Run Pipeline" in the top-right
# - Expand "Add Parameters"
# - Add a "boolean" parameter called "release_branch" with the value true
# - Click "Run Pipeline"
#
# You can also do this using the CircleCI API:
#
# Using `xh`:
#
# # e.g. using 'xh' (https://github.com/ducaale/xh)
# $ xh -a '<your personal circleCI token>:' POST \
# https://circleci.com/api/v2/project/github/influxdata/influxdb_iox/pipeline \
# parameters:='{"release_branch": true}' branch=chore/ci-tidy-up
#
# ...or equivalent with `curl`:
# $ curl -XPOST -H "Content-Type: application/json" -H "Circle-Token: <your personal circleCI token>" \
# -d '{"parameters": {"release_branch": true}, "branch": "chore/ci-tidy-up"}' \
# https://circleci.com/api/v2/project/github/influxdata/influxdb_iox/pipeline
#
# Manual CI Base Image:
#
# It is possible to manually trigger a rebuild of the base image used in CI. To do this, navigate to
# https://app.circleci.com/pipelines/github/influxdata/influxdb_iox?branch=main (overriding the
# branch name if desired). Then:
# - Click "Run Pipeline" in the top-right
# - Expand "Add Parameters"
# - Add a "boolean" parameter called "ci_image" with the value true
# - Click "Run Pipeline"
#
# If you refresh the page you should see a newly running ci_image workflow
#
version: 2.1
commands:
rust_components:
description: Verify installed components
steps:
- run:
name: Verify installed components
command: |
rustup --version
rustup show
cargo fmt --version
cargo clippy --version
cargo install cargo-hakari && cargo hakari --version
cache_restore:
description: Restore Cargo Cache
steps:
- restore_cache:
name: Restoring Cargo Cache
keys:
- cargo-cache-{{ arch }}-{{ .Branch }}-{{ checksum "Cargo.lock" }}
- cargo-cache-{{ arch }}-{{ .Branch }}
- cargo-cache
cache_save:
description: Save Cargo Cache
steps:
- save_cache:
name: Save Cargo Cache
paths:
- /usr/local/cargo/registry
key: cargo-cache-{{ arch }}-{{ .Branch }}-{{ checksum "Cargo.lock" }}
login_to_gcloud:
steps:
- run:
name: Login to gcloud
command: |
echo "${GCLOUD_SERVICE_ACCOUNT_KEY}" >/tmp/gcloud.json
gcloud auth activate-service-account "${GCLOUD_SERVICE_ACCOUNT_EMAIL}" --key-file /tmp/gcloud.json --quiet
rm -f /tmp/gcloud.json
gcloud auth configure-docker us-docker.pkg.dev
jobs:
fmt:
docker:
- image: quay.io/influxdb/rust:ci
environment:
# Disable incremental compilation to avoid overhead. We are not preserving these files anyway.
CARGO_INCREMENTAL: "0"
# Disable full debug symbol generation to speed up CI build
# "1" means line tables only, which is useful for panic tracebacks.
RUSTFLAGS: "-C debuginfo=1"
# https://github.com/rust-lang/cargo/issues/10280
CARGO_NET_GIT_FETCH_WITH_CLI: "true"
steps:
- checkout
- rust_components
- cache_restore
- run:
name: Rust fmt
command: cargo fmt --all -- --check
- cache_save
lint:
docker:
- image: quay.io/influxdb/rust:ci
environment:
# Disable incremental compilation to avoid overhead. We are not preserving these files anyway.
CARGO_INCREMENTAL: "0"
# Disable full debug symbol generation to speed up CI build
# "1" means line tables only, which is useful for panic tracebacks.
RUSTFLAGS: "-C debuginfo=1"
# https://github.com/rust-lang/cargo/issues/10280
CARGO_NET_GIT_FETCH_WITH_CLI: "true"
steps:
- checkout
- rust_components
- cache_restore
- run:
name: Clippy
command: cargo clippy --all-targets --all-features --workspace -- -D warnings
- run:
name: Shellcheck
command: find scripts -type f ! \( -iname '*.py' -or -iname '*.supp' \) -exec shellcheck {} +
- run:
name: Yamllint
command: yamllint --config-file .circleci/yamllint.yml --strict .
- cache_save
cargo_audit:
docker:
- image: quay.io/influxdb/rust:ci
environment:
# Disable incremental compilation to avoid overhead. We are not preserving these files anyway.
CARGO_INCREMENTAL: "0"
# Disable full debug symbol generation to speed up CI build
# "1" means line tables only, which is useful for panic tracebacks.
RUSTFLAGS: "-C debuginfo=1"
# https://github.com/rust-lang/cargo/issues/10280
CARGO_NET_GIT_FETCH_WITH_CLI: "true"
steps:
- checkout
- rust_components
- cache_restore
- run:
name: Install cargo-deny
command: cargo install cargo-deny
- run:
name: cargo-deny Checks
command: cargo deny check -s
- cache_save
doc:
docker:
- image: quay.io/influxdb/rust:ci
resource_class: large # use of a smaller executor runs out of memory
environment:
# Disable incremental compilation to avoid overhead. We are not preserving these files anyway.
CARGO_INCREMENTAL: "0"
# Disable full debug symbol generation to speed up CI build
# "1" means line tables only, which is useful for panic tracebacks.
RUSTFLAGS: "-C debuginfo=1"
# https://github.com/rust-lang/cargo/issues/10280
CARGO_NET_GIT_FETCH_WITH_CLI: "true"
# Turn warnings into errors
RUSTDOCFLAGS: "-D warnings -A rustdoc::private-intra-doc-links"
steps:
- checkout
- rust_components
- cache_restore
- run:
name: Cargo doc
command: cargo doc --document-private-items --no-deps --workspace
- cache_save
- run:
name: Compress Docs
command: tar -cvzf rustdoc.tar.gz target/doc/
- store_artifacts:
path: rustdoc.tar.gz
workspace_hack_checks:
docker:
- image: quay.io/influxdb/rust:ci
environment:
# https://github.com/rust-lang/cargo/issues/10280
CARGO_NET_GIT_FETCH_WITH_CLI: "true"
steps:
- add_ssh_keys:
fingerprints:
- "77:99:88:4a:ac:1f:55:9e:39:c7:1f:e4:7f:1e:60:4b"
- checkout
- rust_components
- cache_restore
- run:
name: Configure git
command: |
git config user.email "circleci@influxdata.com"
git config user.name "CircleCI[bot]"
- run:
name: Commit any changes from cargo hakari
command: |
cargo hakari generate
cargo hakari manage-deps -y
if [[ $(git status --porcelain) ]] ; then
if [[ $(git rev-list --count --author=CircleCI origin/main..) -ne 0 ]]; then
echo "There's already a cargo hakari commit on this branch, but there are cargo hakari changes... something is wrong."
else
git commit -am "chore: Run cargo hakari tasks"
git push origin "$CIRCLE_BRANCH"
echo "Cargo hakari changes need to be committed before merging this branch."
fi
exit 1
else
echo "No changes to commit"
fi
test:
# setup multiple docker images (see https://circleci.com/docs/2.0/configuration-reference/#docker)
docker:
- image: quay.io/influxdb/rust:ci
- image: vectorized/redpanda:v22.1.5
command: redpanda start --overprovisioned --smp 1 --memory 1G --reserve-memory 0M
- image: postgres
environment:
POSTGRES_HOST_AUTH_METHOD: trust
resource_class: 2xlarge+ # use of a smaller executor tends crashes on link
environment:
# Disable incremental compilation to avoid overhead. We are not preserving these files anyway.
CARGO_INCREMENTAL: "0"
# Disable full debug symbol generation to speed up CI build
# "1" means line tables only, which is useful for panic tracebacks.
RUSTFLAGS: "-C debuginfo=1"
# https://github.com/rust-lang/cargo/issues/10280
CARGO_NET_GIT_FETCH_WITH_CLI: "true"
RUST_BACKTRACE: "1"
# Run integration tests
TEST_INTEGRATION: 1
INFLUXDB_IOX_INTEGRATION_LOCAL: 1
KAFKA_CONNECT: "localhost:9092"
POSTGRES_USER: postgres
TEST_INFLUXDB_IOX_CATALOG_DSN: "postgres://postgres@localhost/iox_shared"
# When removing this, also remove the ignore on the test in trogging/src/cli.rs
RUST_LOG: debug,,hyper::proto::h1=info,h2=info
LOG_FILTER: debug,,hyper::proto::h1=info,h2=info
# TEMPORARY: Can be removed when the ingester that uses the write buffer is removed. Tests
# need to spin up separate servers because I've only been able to implement a "persist
# everything" API, and if tests run in parallel using a shared server, they interfere with
# each other. Starting separate servers with the maximum number of Rust test threads uses up
# all the Postgres connections in CI, so limit the parallelization until we switch completely
# to ingester2, which does have a "persist-per-namespace" API that means tests can run on
# shared MiniClusters.
RUST_TEST_THREADS: 8
steps:
- checkout
- rust_components
- cache_restore
- run:
name: Cargo test
command: cargo test --workspace
- cache_save
# end to end test for JDBC
test-flight-sql-jdbc:
# setup multiple docker images (see https://circleci.com/docs/2.0/configuration-reference/#docker)
docker:
- image: quay.io/influxdb/rust:ci
- image: postgres
environment:
POSTGRES_HOST_AUTH_METHOD: trust
resource_class: 2xlarge+ # use of a smaller executor tends crashes on link
environment:
# Disable incremental compilation to avoid overhead. We are not preserving these files anyway.
CARGO_INCREMENTAL: "0"
# Disable full debug symbol generation to speed up CI build
# "1" means line tables only, which is useful for panic tracebacks.
RUSTFLAGS: "-C debuginfo=1"
# https://github.com/rust-lang/cargo/issues/10280
CARGO_NET_GIT_FETCH_WITH_CLI: "true"
RUST_BACKTRACE: "1"
# Run integration tests
TEST_INTEGRATION: 1
INFLUXDB_IOX_INTEGRATION_LOCAL: 1
POSTGRES_USER: postgres
TEST_INFLUXDB_IOX_CATALOG_DSN: "postgres://postgres@localhost/iox_shared"
# When removing this, also remove the ignore on the test in trogging/src/cli.rs
RUST_LOG: debug,,hyper::proto::h1=info,h2=info
LOG_FILTER: debug,,hyper::proto::h1=info,h2=info
# Run the actual test
TEST_INFLUXDB_JDBC: "true"
steps:
- checkout
- run:
name: Install javac
command: |
sudo apt-get update
sudo apt-get install openjdk-11-jdk -y
- rust_components
- cache_restore
- run:
name: Cargo test
command: cargo test --test end_to_end jdbc
# end to end tests with Heappy (heap profiling enabled)
test_heappy:
docker:
- image: quay.io/influxdb/rust:ci
resource_class: xlarge # use of a smaller executor tends crashes on link
environment:
# Disable incremental compilation to avoid overhead. We are not preserving these files anyway.
CARGO_INCREMENTAL: "0"
# Disable full debug symbol generation to speed up CI build
# "1" means line tables only, which is useful for panic tracebacks.
RUSTFLAGS: "-C debuginfo=1"
# https://github.com/rust-lang/cargo/issues/10280
CARGO_NET_GIT_FETCH_WITH_CLI: "true"
RUST_BACKTRACE: "1"
steps:
- checkout
- rust_components
- cache_restore
- run:
name: End to end tests (with heappy)
command: cargo test --no-default-features --features=heappy end_to_end
- cache_save
# Build a dev binary.
#
# Compiles a binary with the default ("dev") cargo profile from the iox source
# using the latest ci_image and ensures various targets compile successfully
build_dev:
docker:
- image: quay.io/influxdb/rust:ci
resource_class: 2xlarge # use of a smaller executor tends crashes on link
environment:
# Disable incremental compilation to avoid overhead. We are not preserving these files anyway.
CARGO_INCREMENTAL: "0"
# - Disable full debug symbol generation to speed up CI build
# "1" means line tables only, which is useful for panic tracebacks.
# - Enable tokio-console
RUSTFLAGS: "-C debuginfo=1 --cfg tokio_unstable"
# https://github.com/rust-lang/cargo/issues/10280
CARGO_NET_GIT_FETCH_WITH_CLI: "true"
# The `2xlarge` resource class that we use has 32GB RAM but also 16 CPUs. This means we have 2GB RAM per core on
# avarage. At peak this is a bit tight, so lower the CPU count for cargo a bit.
CARGO_BUILD_JOBS: "12"
steps:
- checkout
- rust_components
- cache_restore
- run:
name: Cargo build
command: cargo build --workspace
- run:
name: Check benches compile
command: cargo check --workspace --benches
- run:
name: Check extra features (like prod image)
command: cargo check --no-default-features --features="aws,gcp,azure,jemalloc_replacing_malloc,tokio_console,pprof"
- run:
# Validate that the data generator compiles (in the same way as it does in release builds)
name: Check iox_data_generator compiles
command: cargo check --package="iox_data_generator" --no-default-features
- cache_save
# Lint protobufs.
protobuf-lint:
docker:
- image: bufbuild/buf:0.40.0
environment:
# Value to look for to skip breaking changes check
SKIP_LABEL: "https://api.github.com/repos/influxdata/influxdb_iox/labels/incompatible%20protobuf"
steps:
- checkout
- run:
name: buf lint
command: buf lint
- run:
name: buf breaking changes
command: |
echo "If you want to make changes forbidden by this lint, please"
echo "coordinate with the conductor team, add the 'incompatible protobuf' label"
echo "to the PR, and rerun this test"
# Check if label is present using github API:
# Inspired by https://discuss.circleci.com/t/tag-label-filter/11158
if wget -O - https://api.github.com/repos/influxdata/influxdb_iox/issues/$(echo $CIRCLE_PULL_REQUEST | grep -oE "[^/pull]+$") | grep "$SKIP_LABEL" ; then echo "SKIPPING (FOUND LABEL)" && exit ; else echo "CHECKING (NO LABEL FOUND)"; fi
git fetch origin main
# compare against only changes in this branch (not against
# other stuff that may have been added to main since last merge)
MERGE_BASE=$(git merge-base origin/main $CIRCLE_BRANCH) sh -c 'buf breaking --against ".git#ref=$MERGE_BASE,depth=100"'
# Lint docs
docs-lint:
docker:
- image: python:3-slim-bullseye
steps:
# need a proper SSH client for checkout
# https://discuss.circleci.com/t/circleci-user-key-uses-rsa-with-sha1-which-github-has-deprecated/42548
- run:
name: Install GIT and SSH
command: |
apt-get update
apt-get install -y git ssh
- checkout
- run:
name: Lint docs
command: ./scripts/lint_docs.py ./docs
# Compile cargo "release" profile binaries for iox & data generator
#
# Uses the latest ci_image (influxdb/rust below) to build a release binary and
# copies it to a minimal container image based upon `rust:slim-buster`. This
# minimal image is then pushed to `quay.io/influxdb/iox:${BRANCH}`.
build_release:
# need a machine executor to have a full-powered docker daemon (the `setup_remote_docker` system just provides a
# kinda small node)
machine:
image: ubuntu-2004:2022.04.2
resource_class: xlarge # use of a smaller executor tends crashes on link
environment:
# Disable incremental compilation to avoid overhead. We are not preserving these files anyway.
CARGO_INCREMENTAL: "0"
# Enable tokio-console
RUSTFLAGS: "--cfg tokio_unstable"
# https://github.com/rust-lang/cargo/issues/10280
CARGO_NET_GIT_FETCH_WITH_CLI: "true"
steps:
- checkout
- run:
name: Cargo release build with target arch set for CRoaring
command: |
COMMIT_SHA="$(git rev-parse HEAD)"
RUST_VERSION="$(sed -E -ne 's/channel = "(.*)"/\1/p' rust-toolchain.toml)"
docker buildx build \
--build-arg FEATURES="aws,gcp,azure,jemalloc_replacing_malloc,tokio_console,pprof" \
--build-arg RUST_VERSION="$RUST_VERSION" \
--build-arg RUSTFLAGS="$RUSTFLAGS" \
--progress plain \
--tag quay.io/influxdb/iox:"$COMMIT_SHA" \
.
docker buildx build \
--build-arg FEATURES="" \
--build-arg PACKAGE="iox_data_generator" \
--build-arg RUST_VERSION="$RUST_VERSION" \
--build-arg RUSTFLAGS="$RUSTFLAGS" \
--progress plain \
--tag quay.io/influxdb/iox_data_generator:"$COMMIT_SHA" \
.
docker run -it --rm quay.io/influxdb/iox:$COMMIT_SHA debug print-cpu
mkdir /tmp/images
docker save quay.io/influxdb/iox:"$COMMIT_SHA" | gzip > /tmp/images/iox.tar.gz
docker save quay.io/influxdb/iox_data_generator:"$COMMIT_SHA" | gzip > /tmp/images/iox_data_generator.tar.gz
# linking might take a while and doesn't produce CLI output
no_output_timeout: 30m
- store_artifacts:
path: /tmp/images
- persist_to_workspace:
root: /tmp/images
paths:
- "*.tar.gz"
- cache_save
deploy_release:
docker:
- image: cimg/gcp:2022.11.1
steps:
- setup_remote_docker:
version: 19.03.13
docker_layer_caching: true
- checkout
- attach_workspace:
at: /tmp/images
- login_to_gcloud
- run:
name: quay login
command: |
echo "$QUAY_INFLUXDB_IOX_PASS" | docker login quay.io --username $QUAY_INFLUXDB_IOX_USER --password-stdin
- run:
name: install skopeo
command: |
#!/bin/bash
set -eu -o pipefail
readonly attempts=5
install() {
# install skopeo via kubic on Ubuntu 20.04
if [ "$(sh -c '. /etc/os-release && echo $VERSION_ID')" = "20.04" ] ; then
sudo sh -c '. /etc/os-release && echo "deb https://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable/xUbuntu_${VERSION_ID}/ /" >/etc/apt/sources.list.d/kubic-stable.list'
sudo sh -c '. /etc/os-release && curl -L "https://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable/xUbuntu_${VERSION_ID}/Release.key" | apt-key add -'
fi
sudo apt-get update
sudo apt-get -y install skopeo
}
# retry installing skopeo up to 5 times to avoid temporary networking issues
for attempt in $(seq 1 "${attempts}") ; do
# run installation, if successful, exit and return 0
if install ; then
exit 0
else
if [ "${attempt}" -ne ${attempts} ] ; then
# if installing failed, wait for a bit and retry
sleep $(( attempt * 5 ))
else
# if this was the last attempt, exit with an error
echo >&2 "Failed to install skopeo"
exit 1
fi
fi
done
- run:
name: Upload images
command: |
COMMIT_SHA="$(git rev-parse HEAD)"
BRANCH="$(echo "$CIRCLE_BRANCH" | tr '[:upper:]' '[:lower:]' | sed 's/[^a-z0-9]/_/g')"
docker load -i /tmp/images/iox.tar.gz
docker load -i /tmp/images/iox_data_generator.tar.gz
docker tag quay.io/influxdb/iox:"$COMMIT_SHA" quay.io/influxdb/iox:"$BRANCH"
docker tag quay.io/influxdb/iox_data_generator:"$COMMIT_SHA" quay.io/influxdb/iox_data_generator:"$BRANCH"
docker push quay.io/influxdb/iox:"$COMMIT_SHA"
docker push quay.io/influxdb/iox_data_generator:"$COMMIT_SHA"
docker push quay.io/influxdb/iox:"$BRANCH"
docker push quay.io/influxdb/iox_data_generator:"$BRANCH"
# copy image to Google as well
skopeo copy docker://quay.io/influxdb/iox:${COMMIT_SHA} docker://us-docker.pkg.dev/influxdb2-artifacts/iox/iox:${COMMIT_SHA} --quiet
skopeo copy docker://quay.io/influxdb/iox:${BRANCH} docker://us-docker.pkg.dev/influxdb2-artifacts/iox/iox:${BRANCH} --quiet
skopeo copy docker://quay.io/influxdb/iox_data_generator:${COMMIT_SHA} docker://us-docker.pkg.dev/influxdb2-artifacts/iox/iox_data_generator:${COMMIT_SHA} --quiet
skopeo copy docker://quay.io/influxdb/iox_data_generator:${BRANCH} docker://us-docker.pkg.dev/influxdb2-artifacts/iox/iox_data_generator:${BRANCH} --quiet
# for next step
echo "export COMMIT_SHA=${COMMIT_SHA}" >> $BASH_ENV
# Prepare the CI image used for other tasks.
#
# A nightly job (scheduled below in the `workflows` section) to build the CI
# image (influxdb/rust) used for the rest of the checks.
#
# To modify the contents of the CI image, update docker/Dockerfile.ci
ci_image:
# Use a `machine` executor instead of a docker-based because:
# - we bootstrap our CI images somehow (we could use a docker-hub image as well)
# - the machine executor also supports multi-arch builds
# - we only run this job once a day, so the additional startup delay of 30-60s doesn't really matter
machine:
# https://circleci.com/docs/2.0/configuration-reference/#available-machine-images
image: ubuntu-2004:2022.04.2
resource_class: xlarge
steps:
- checkout
- run: |
echo "$QUAY_PASS" | docker login quay.io --username $QUAY_USER --password-stdin
- run: |
COMMIT_SHA=$(git rev-parse --short HEAD)
RUST_VERSION=$(sed -E -ne 's/channel = "(.*)"/\1/p' rust-toolchain.toml)
docker --version
docker build -t quay.io/influxdb/rust:$COMMIT_SHA -t quay.io/influxdb/rust:ci -f docker/Dockerfile.ci --build-arg RUST_VERSION=$RUST_VERSION .
docker push --all-tags quay.io/influxdb/rust
parameters:
ci_image:
description: "Trigger build of CI image"
type: boolean
default: false
release_branch:
description: "Build and push container image for non-main branch"
type: boolean
default: false
workflows:
version: 2
# CI for all pull requests.
ci:
when:
and:
- not: << pipeline.parameters.ci_image >>
- not: << pipeline.parameters.release_branch >>
jobs:
- fmt
- lint
- cargo_audit
- protobuf-lint
- docs-lint
- test
- test_heappy
- build_dev
- doc
- workspace_hack_checks
- test-flight-sql-jdbc
- build_release:
filters:
branches:
only: main
- deploy_release:
filters:
branches:
only: main
requires: # Only deploy if all tests have passed
- fmt
- lint
- cargo_audit
- protobuf-lint
- docs-lint
- test
- test_heappy
- build_dev
- build_release
- doc
# Manual build of CI image
ci_image:
when: << pipeline.parameters.ci_image >>
jobs:
- ci_image
# Force build and push of container image for non-main branch.
# See instructions at the top of this file
release_branch:
when: << pipeline.parameters.release_branch >>
jobs:
- build_release
- deploy_release:
requires:
- build_release
# Nightly rebuild of the build container
ci_image_nightly:
triggers:
- schedule:
cron: "0 5 * * *"
filters:
branches:
only:
- main
jobs:
- ci_image
Reference in New Issue View Git Blame Copy Permalink