Influxdata
/
influxdb
mirror of https://github.com/influxdata/influxdb.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity
influxdb/.circleci/config.yml

614 lines
20 KiB
YAML
Raw Blame History

---
# CI Overview
# -----------
#
# Every commit:
#
# The CI for every PR and merge to main runs tests, fmt, lints and compiles debug binaries
#
# On main if all these checks pass it will then additionally compile in "release" mode and
# publish a docker image to quay.io/influxdb/influxdb3:$COMMIT_SHA
#
# Manually trigger build and push of container image for a branch:
#
# Navigate to https://app.circleci.com/pipelines/github/influxdata/influxdb?branch=<branch-name> (<- change this!)
# Then:
#
# - Click "Run Pipeline" in the top-right
# - Expand "Add Parameters"
# - Add a "boolean" parameter called "release_branch" with the value true
# - Click "Run Pipeline"
#
# You can also do this using the CircleCI API:
#
# Using `xh`:
#
# # e.g. using 'xh' (https://github.com/ducaale/xh)
# $ xh -a '<your personal circleCI token>:' POST \
# https://circleci.com/api/v2/project/github/influxdata/influxdb/pipeline \
# parameters:='{"release_branch": true}' branch=chore/ci-tidy-up
#
# ...or equivalent with `curl`:
# $ curl -XPOST -H "Content-Type: application/json" -H "Circle-Token: <your personal circleCI token>" \
# -d '{"parameters": {"release_branch": true}, "branch": "chore/ci-tidy-up"}' \
# https://circleci.com/api/v2/project/github/influxdata/influxdb/pipeline
version: 2.1
orbs:
aws-s3: circleci/aws-s3@2.0.0
terraform: circleci/terraform@2.1.0
rust: circleci/rust@1.6.1
# Unlike when a commit is pushed to a branch, CircleCI does not automatically
# execute a workflow when a tag is pushed to a repository. These filters
# allow the corresponding workflow to execute on any branch or tag.
any_filter: &any_filter
filters:
tags:
only: /.*/
branches:
only: /.*/
release_filter: &release_filter
filters:
tags:
only: /^v(\d+)(?:\.(\d+))?(?:\.(\d+))?$/
branches:
ignore: /.*/
main_filter: &main_filter
filters:
branches:
only: main
nofork_filter: &nofork_filter
filters:
branches:
ignore: /pull\/[0-9]+/
commands:
rust_components:
description: Verify installed components
steps:
- run:
name: Verify installed components
command: |
rustup --version
rustup show
cargo fmt --version
cargo clippy --version
gcloud-docker-login:
steps:
- run:
name: configure-gar
command: |
gcloud auth activate-service-account "${GCLOUD_SERVICE_ACCOUNT_EMAIL}" --key-file <(echo "${GCLOUD_SERVICE_ACCOUNT_KEY}")
gcloud auth configure-docker us-docker.pkg.dev
jobs:
fmt:
docker:
- image: quay.io/influxdb/rust:ci
environment:
# Disable incremental compilation to avoid overhead. We are not preserving these files anyway.
CARGO_INCREMENTAL: "0"
# Disable full debug symbol generation to speed up CI build
# "1" means line tables only, which is useful for panic tracebacks.
CARGO_PROFILE_DEV_DEBUG: "1"
# https://github.com/rust-lang/cargo/issues/10280
CARGO_NET_GIT_FETCH_WITH_CLI: "true"
steps:
- checkout
- rust_components
- run:
name: Rust fmt
command: cargo fmt --all -- --check
lint:
docker:
- image: quay.io/influxdb/rust:ci
environment:
# Disable incremental compilation to avoid overhead. We are not preserving these files anyway.
CARGO_INCREMENTAL: "0"
# Disable full debug symbol generation to speed up CI build
# "1" means line tables only, which is useful for panic tracebacks.
CARGO_PROFILE_DEV_DEBUG: "1"
# https://github.com/rust-lang/cargo/issues/10280
CARGO_NET_GIT_FETCH_WITH_CLI: "true"
steps:
- checkout
- rust_components
- run:
name: Clippy
command: cargo clippy --all-targets --all-features --workspace -- -D warnings
- run:
name: Yamllint
command: yamllint --config-file .circleci/yamllint.yml --strict .
inclusivity:
docker:
- image: cimg/go:1.23
steps:
- checkout
- run:
name: Build and run inclusivity checker
command: |
go install github.com/jdstrand/language-checker@latest
language-checker --exit-1-on-failure .
cargo-audit:
docker:
- image: quay.io/influxdb/rust:ci
environment:
# Disable incremental compilation to avoid overhead. We are not preserving these files anyway.
CARGO_INCREMENTAL: "0"
# Disable full debug symbol generation to speed up CI build
# "1" means line tables only, which is useful for panic tracebacks.
CARGO_PROFILE_DEV_DEBUG: "1"
# https://github.com/rust-lang/cargo/issues/10280
CARGO_NET_GIT_FETCH_WITH_CLI: "true"
steps:
- checkout
- rust_components
- run:
name: Install cargo-deny
command: cargo install cargo-deny --locked
- run:
name: cargo-deny Checks
command: cargo deny check -s
doc:
docker:
- image: quay.io/influxdb/rust:ci
resource_class: large # use of a smaller executor runs out of memory
environment:
# Disable incremental compilation to avoid overhead. We are not preserving these files anyway.
CARGO_INCREMENTAL: "0"
# Disable full debug symbol generation to speed up CI build
# "1" means line tables only, which is useful for panic tracebacks.
CARGO_PROFILE_DEV_DEBUG: "1"
# https://github.com/rust-lang/cargo/issues/10280
CARGO_NET_GIT_FETCH_WITH_CLI: "true"
# Turn warnings into errors
RUSTDOCFLAGS: "-D warnings -A rustdoc::private-intra-doc-links"
steps:
- checkout
- rust_components
- run:
name: Cargo doc
command: cargo doc --document-private-items --no-deps --workspace
- run:
name: Compress Docs
command: tar -cvzf rustdoc.tar.gz target/doc/
- store_artifacts:
path: rustdoc.tar.gz
# Run all tests
test:
docker:
- image: quay.io/influxdb/rust:ci
resource_class: 2xlarge+ # use of a smaller executor tends crashes on link
environment:
# Disable incremental compilation to avoid overhead. We are not preserving these files anyway.
CARGO_INCREMENTAL: "0"
# Disable full debug symbol generation to speed up CI build
# "1" means line tables only, which is useful for panic tracebacks.
CARGO_PROFILE_DEV_DEBUG: "1"
# https://github.com/rust-lang/cargo/issues/10280
CARGO_NET_GIT_FETCH_WITH_CLI: "true"
RUST_BACKTRACE: "1"
steps:
- checkout
- rust_components
- run:
name: cargo nextest
command: TEST_LOG= RUST_LOG=info RUST_LOG_SPAN_EVENTS=full RUST_BACKTRACE=1 cargo nextest run --workspace --failure-output immediate-final --no-fail-fast
# Build a dev binary.
#
# Compiles a binary with the default ("dev") cargo profile from the influxdb3 source
# using the latest ci_image (influxdb/rust) and ensures various targets compile successfully
# Build a dev binary.
build-dev:
docker:
- image: us-east1-docker.pkg.dev/influxdata-team-edge/ci-support/ci-cross-influxdb3:latest
auth:
username: _json_key
password: $CISUPPORT_GCS_AUTHORIZATION
resource_class: 2xlarge+ # use of a smaller executor tends crashes on link
environment:
TARGET: << parameters.target >>
# Disable incremental compilation to avoid overhead. We are not preserving these files anyway.
CARGO_INCREMENTAL: "0"
# Disable full debug symbol generation to speed up CI build
# "1" means line tables only, which is useful for panic tracebacks.
CARGO_PROFILE_DEV_DEBUG: "1"
# https://github.com/rust-lang/cargo/issues/10280
CARGO_NET_GIT_FETCH_WITH_CLI: "true"
# The `2xlarge` resource class that we use has 32GB RAM but also 16 CPUs. This means we have 2GB RAM per core on
# avarage. At peak this is a bit tight, so lower the CPU count for cargo a bit.
CARGO_BUILD_JOBS: "12"
parameters:
target:
type: string
steps:
- checkout
- run:
name: Install Target
command: rustup target add << parameters.target >>
- run:
name: Cargo build
command: target-env cargo build --target=<< parameters.target >> --workspace
- when:
condition:
not:
equal: [ << parameters.target >>, x86_64-pc-windows-gnu ]
steps:
- run:
name: Check benches compile
command: target-env cargo check --target=<< parameters.target >> --workspace --benches
- run:
name: Check extra features (like prod image)
command: target-env cargo check --target=<< parameters.target >> --no-default-features --features="aws,gcp,azure,jemalloc_replacing_malloc,tokio_console"
- when:
condition:
equal: [ << parameters.target >>, x86_64-pc-windows-gnu ]
steps:
- run:
name: Check extra features (like prod image)
command: target-env cargo check --target=<< parameters.target >> --no-default-features --features="aws,gcp,azure,jemalloc_replacing_malloc,tokio_console"
# Compile cargo "release" profile binaries for influxdb3 edge releases
build-release:
docker:
- image: us-east1-docker.pkg.dev/influxdata-team-edge/ci-support/ci-cross-influxdb3:latest
auth:
username: _json_key
password: $CISUPPORT_GCS_AUTHORIZATION
resource_class: 2xlarge+
environment:
TARGET: << parameters.target >>
# Disable incremental compilation to avoid overhead. We are not preserving these files anyway.
CARGO_INCREMENTAL: "0"
# Disable full debug symbol generation to speed up CI build
# "1" means line tables only, which is useful for panic tracebacks.
CARGO_PROFILE_DEV_DEBUG: "1"
# https://github.com/rust-lang/cargo/issues/10280
CARGO_NET_GIT_FETCH_WITH_CLI: "true"
# The `2xlarge` resource class that we use has 32GB RAM but also 16 CPUs. This means we have 2GB RAM per core on
# avarage. At peak this is a bit tight, so lower the CPU count for cargo a bit.
CARGO_BUILD_JOBS: "12"
parameters:
target:
type: string
profile:
type: string
default: release
steps:
- checkout
- run:
name: Install Target
command: rustup target add << parameters.target >>
- run:
name: Cargo release build
command: target-env cargo build --target=<< parameters.target >> --profile=<< parameters.profile >> --workspace
# linking might take a while and doesn't produce CLI output
no_output_timeout: 30m
- run:
name: tar and gzip build artifacts
command: |
mkdir -p artifacts
tar --ignore-failed-read -czvf "${PWD}/artifacts/influxdb3-edge_<< parameters.target >>.tar.gz" -C "${PWD}/target/<< parameters.target >>/<< parameters.profile >>" influxdb3{,.exe}
- store_artifacts:
path: artifacts
- persist_to_workspace:
root: .
paths:
- artifacts
build-packages:
docker:
- image: us-east1-docker.pkg.dev/influxdata-team-edge/ci-support/ci-packager-next:latest
auth:
username: _json_key
password: $CISUPPORT_GCS_AUTHORIZATION
steps:
- checkout
- attach_workspace:
at: /tmp/workspace
- run: packager .circleci/packages/config.yaml
- persist_to_workspace:
root: .
paths:
- artifacts
- store_artifacts:
path: artifacts/
check_package_deb_amd64:
machine:
image: ubuntu-2204:current
resource_class: medium
steps:
- attach_workspace:
at: /tmp/workspace
- checkout
- run:
name: Validate Debian Package (AMD64)
command: |
sudo .circleci/scripts/package-validation/debian \
/tmp/workspace/artifacts/influxdb3*amd64.deb
check_package_deb_arm64:
machine:
image: ubuntu-2204:current
resource_class: arm.medium
steps:
- attach_workspace:
at: /tmp/workspace
- checkout
- run:
name: Validate Debian Package (ARM64)
command: |
sudo .circleci/scripts/package-validation/debian \
/tmp/workspace/artifacts/influxdb3*arm64.deb
check_package_rpm:
machine:
image: ubuntu-2204:current
resource_class: arm.medium
parameters:
arch:
type: string
steps:
- attach_workspace:
at: /tmp/workspace
- add_ssh_keys:
fingerprints:
- 3a:d1:7a:b7:57:d7:85:0b:76:79:85:51:38:f3:e4:67
- checkout
- run: |
AWS_ACCESS_KEY_ID=$TEST_AWS_ACCESS_KEY_ID \
AWS_SECRET_ACCESS_KEY=$TEST_AWS_SECRET_ACCESS_KEY \
.circleci/scripts/package-validation/redhat << parameters.arch >> /tmp/workspace/artifacts/influxdb3*.<< parameters.arch >>.rpm
sign-packages:
circleci_ip_ranges: true
docker:
- image: quay.io/influxdb/rsign:latest
auth:
username: $QUAY_RSIGN_USERNAME
password: $QUAY_RSIGN_PASSWORD
steps:
- add_ssh_keys:
fingerprints:
- fc:7b:6e:a6:38:7c:63:5a:13:be:cb:bb:fa:33:b3:3c
- attach_workspace:
at: /tmp/workspace
- run: |
# We need this base so that we can filter it out of our checksums
# output and if that changes at all we only need to update it here
WORK_DIR="/tmp/workspace/artifacts/"
for target in ${WORK_DIR}*
do
case "${target}"
in
# rsign is shipped on Alpine Linux which uses "busybox ash" instead
# of bash. ash is somewhat more posix compliant and is missing some
# extensions and niceties from bash.
*.deb|*.rpm|*.tar.gz|*.zip)
rsign "${target}"
;;
esac
if [ -f "${target}" ]
then
# Since all artifacts are present, sign them here. This saves Circle
# credits over spinning up another instance just to separate out the
# checksum job.
sha256sum "${target}" | sed "s#$WORK_DIR##" >> "/tmp/workspace/artifacts/influxdb3-edge.${CIRCLE_TAG}.digests"
# write individual checksums
md5sum "${target}" | sed "s#$WORK_DIR##" >> "${target}.md5"
sha256sum "${target}" | sed "s#$WORK_DIR##" >> "${target}.sha256"
fi
done
- persist_to_workspace:
root: /tmp/workspace
paths:
- artifacts
- store_artifacts:
path: /tmp/workspace/artifacts
publish-packages:
docker:
- image: cimg/python:3.12.2
parameters:
# "destination" should be one of:
# - releases
# - nightlies
# - snapshots
destination:
type: string
steps:
- attach_workspace:
at: /tmp/workspace
- aws-s3/sync:
arguments: --acl public-read
aws-region: RELEASE_AWS_REGION
aws-access-key-id: RELEASE_AWS_ACCESS_KEY_ID
aws-secret-access-key: RELEASE_AWS_SECRET_ACCESS_KEY
from: /tmp/workspace/artifacts
to: s3://dl.influxdata.com/influxdb/<< parameters.destination >>
- run:
command: |
export AWS_REGION="${RELEASE_AWS_REGION}"
export AWS_ACCESS_KEY_ID="${RELEASE_AWS_ACCESS_KEY_ID}"
export AWS_SECRET_ACCESS_KEY="${RELEASE_AWS_SECRET_ACCESS_KEY}"
aws cloudfront create-invalidation --distribution-id "${RELEASE_ARTIFACTS_CLOUDFRONT}" --paths '/influxdb/<< parameters.destination >>/*'
build-docker:
# need a machine executor to have a full-powered docker daemon (the `setup_remote_docker` system just provides a
# kinda small node)
machine:
image: default
resource_class: 2xlarge+ # CPU bound, so make it fast
steps:
- checkout
- run:
name: Build the docker image
command: |
.circleci/scripts/docker_build_release.bash \
"influxdb3" \
"aws,gcp,azure,jemalloc_replacing_malloc,tokio_console" \
"influxdb3-edge:latest"
# linking might take a while and doesn't produce CLI output
no_output_timeout: 30m
- run: |
docker save influxdb3-edge:latest >influxdb3-edge.tar
- persist_to_workspace:
root: .
paths:
- influxdb3-edge.tar
publish-docker:
docker:
- image: cimg/gcp:2023.02
resource_class: medium
steps:
- checkout
- setup_remote_docker
- gcloud-docker-login
- attach_workspace:
at: .
- run: |
docker load <influxdb3-edge.tar
.circleci/scripts/publish.bash influxdb3-edge
wait-for-docker:
resource_class: small
docker:
- image: busybox
steps:
- run: |
echo build executed successfully
workflows:
version: 2
snapshot:
jobs:
- build-release:
<<: *main_filter
name: build-snapshot-<< matrix.target >>
matrix:
parameters:
profile:
- quick-release
target:
- aarch64-apple-darwin
- aarch64-unknown-linux-gnu
- aarch64-unknown-linux-musl
- x86_64-pc-windows-gnu
- x86_64-unknown-linux-gnu
- x86_64-unknown-linux-musl
- build-packages:
<<: *main_filter
requires:
- build-release
- sign-packages:
<<: *main_filter
requires:
- build-packages
- publish-packages:
<<: *main_filter
matrix:
parameters:
destination: [ snapshots ]
requires:
- build-release
- sign-packages
ci:
jobs:
- fmt:
<<: *any_filter
- lint:
<<: *any_filter
- inclusivity:
<<: *any_filter
- cargo-audit:
<<: *any_filter
- test:
<<: *any_filter
- build-dev:
# This workflow requires secrets stored in the environment.
# These are not passed to workflows executed on forked
# repositories. In this case, skip the workflow, as it
# will be executed on merge to main anyways.
<<: *nofork_filter
name: build-dev-<< matrix.target >>
matrix:
parameters:
target:
- aarch64-apple-darwin
- aarch64-unknown-linux-gnu
- aarch64-unknown-linux-musl
- x86_64-pc-windows-gnu
- x86_64-unknown-linux-gnu
- x86_64-unknown-linux-musl
- doc:
<<: *any_filter
- build-release:
<<: *release_filter
name: build-release-<< matrix.target >>
matrix:
parameters:
target:
- aarch64-apple-darwin
- aarch64-unknown-linux-gnu
- aarch64-unknown-linux-musl
- x86_64-pc-windows-gnu
- x86_64-unknown-linux-gnu
- x86_64-unknown-linux-musl
- build-packages:
<<: *release_filter
requires:
- build-release
- check_package_deb_arm64:
<<: *release_filter
requires:
- build-packages
- check_package_deb_amd64:
<<: *release_filter
requires:
- build-packages
- check_package_rpm:
<<: *nofork_filter
name:
check_package_rpm-<< matrix.arch >>
matrix:
parameters:
arch: [ x86_64, aarch64 ]
requires:
- build-packages
- sign-packages:
<<: *release_filter
requires:
- build-packages
- check_package_rpm
- check_package_deb_arm64
- check_package_deb_amd64
- test
- publish-packages:
<<: *release_filter
matrix:
parameters:
destination: [ releases ]
requires:
- build-release
- sign-packages
- test
- doc
- lint
- fmt
- cargo-audit
- build-docker:
<<: *release_filter
- publish-docker:
<<: *release_filter
requires:
- build-docker
- wait-for-docker:
<<: *release_filter
requires:
- build-docker
- publish-docker
Reference in New Issue View Git Blame Copy Permalink