influxdb/.circleci/config.yml

678 lines
22 KiB
YAML

version: "2.1"
orbs:
terraform: circleci/terraform@2.1.0
executors:
cross-builder:
docker:
# NOTE: To upgrade the Go version, first push the upgrade to the cross-builder Dockerfile
# in the edge repo, then update the version here to match.
- image: quay.io/influxdb/cross-builder:go1.17.2-c9371c24dc3e9daabf19786cf9638428dfe2637a
resource_class: large
linux-amd64:
machine:
image: ubuntu-2004:202107-02
resource_class: large
linux-arm64:
machine:
image: ubuntu-2004:202101-01
resource_class: arm.large
darwin:
macos:
xcode: 12.4.0
resource_class: medium
shell: /bin/bash -eo pipefail
windows:
machine:
image: windows-server-2019-vs2019:stable
resource_class: windows.medium
shell: bash.exe -eo pipefail
workflows:
version: 2
build:
jobs:
- test-race
- test-build:
name: test-build-<< matrix.os >>-<< matrix.arch >>
matrix:
parameters:
os: [ linux, darwin, windows ]
arch: [ amd64, arm64 ]
exclude:
- os: darwin
arch: arm64
- os: windows
arch: arm64
# linux/amd64 can be tested directly from our cross-builder image
# to save time & enable running with the race detector.
- os: linux
arch: amd64
- test-prebuilt:
name: test-linux-arm64
executor: linux-arm64
requires:
- test-build-linux-arm64
- test-prebuilt:
name: test-darwin
executor: darwin
requires:
- test-build-darwin-amd64
- test-prebuilt:
name: test-windows
executor: windows
requires:
- test-build-windows-amd64
- fluxtest
- tlstest
- lint
- build:
name: build-snapshot-<< matrix.os >>-<< matrix.arch >>
build-type: snapshot
matrix:
parameters:
os: [ linux, darwin, windows ]
arch: [ amd64, arm64 ]
exclude:
- os: darwin
arch: arm64
- os: windows
arch: arm64
- package-and-release:
name: package-snapshot
build-type: snapshot
filters:
branches:
ignore: /pull\/[0-9]+/
requires:
- build
- test-downgrade:
requires:
- build-snapshot-linux-amd64
- e2e-monitor-ci:
filters:
branches:
ignore: /pull\/[0-9]+/
requires:
- build-snapshot-linux-amd64
- test-linux-packages:
filters:
branches:
ignore: /pull\/[0-9]+/
requires:
- package-snapshot
- perf-test:
record_results: true
requires:
- package-snapshot
filters:
branches:
only:
- master
- grace-test:
requires:
- build-snapshot-linux-amd64
- litmus-smoke-test:
requires:
- build-snapshot-linux-amd64
- litmus-full-test:
requires:
- build-snapshot-linux-amd64
filters:
branches:
only: master
- share-testing-image:
filters:
branches:
only:
- master
requires:
- e2e-monitor-ci
aws-destroy-daily:
triggers:
- schedule:
# run every day at 10pm -- note: use spaces, not tabs
cron: "0 22 * * *"
filters:
branches:
only:
- "master"
jobs:
- aws-destroy-by-date
nightly:
triggers:
- schedule:
cron: "0 5 * * *"
filters:
branches:
only:
- master
jobs:
- changelog
- test-race
- test-build:
name: test-build-<< matrix.os >>-<< matrix.arch >>
matrix:
parameters:
os: [ linux, darwin, windows ]
arch: [ amd64, arm64 ]
exclude:
- os: darwin
arch: arm64
- os: windows
arch: arm64
# linux/amd64 can be tested directly from our cross-builder image
# to save time & enable running with the race detector.
- os: linux
arch: amd64
- test-prebuilt:
name: test-linux-arm64
executor: linux-arm64
requires:
- test-build-linux-arm64
- test-prebuilt:
name: test-darwin
executor: darwin
requires:
- test-build-darwin-amd64
- test-prebuilt:
name: test-windows
executor: windows
requires:
- test-build-windows-amd64
- lint
- fluxtest
- tlstest
- build:
name: build-nightly-<< matrix.os >>-<< matrix.arch >>
build-type: nightly
matrix:
parameters:
os: [ linux, darwin, windows ]
arch: [ amd64, arm64 ]
exclude:
- os: darwin
arch: arm64
- os: windows
arch: arm64
- package-and-release:
name: package-and-upload-nightly
build-type: nightly
requires:
- build
- changelog
- litmus-full-test:
requires:
- build-nightly-linux-amd64
- grace-test:
requires:
- build-nightly-linux-amd64
commands:
quay_login:
steps:
- run:
name: Log in to Quay
command: docker login -u "$QUAY_USER" -p $QUAY_PASS quay.io
jobs:
lint:
executor: cross-builder
steps:
- checkout
- run:
name: Check go version
command: ./scripts/ci/check-system-go-matches-go-mod.sh
when: always
- run:
name: Check flag generation
command: ./scripts/ci/lint/flags.bash
when: always
- run:
name: Check SQL migrations
command: make checksqlmigrations
when: always
- run:
name: Check formatting
command: make checkfmt
when: always
- run:
name: Check codegen
command: make checkgenerate
when: always
- run:
name: vet
command: make vet
when: always
- run:
name: staticcheck
command: |
go install honnef.co/go/tools/cmd/staticcheck
staticcheck ./...
when: always
test-race:
executor: cross-builder
parallelism: 8
steps:
- checkout
- run:
name: Run race tests
command: ./scripts/ci/run-race-tests.sh $(pwd)/test-results
- store_test_results:
path: ./test-results
- store_artifacts:
path: ./test-results
destination: raw-test-results
test-build:
executor: cross-builder
resource_class: large
parameters:
os:
type: string
arch:
type: string
steps:
- checkout
- run:
name: Build test binaries
command: |
export GOOS=<< parameters.os >>
export GOARCH=<< parameters.arch >>
./scripts/ci/build-tests.sh ./test-bin
- persist_to_workspace:
root: .
paths:
- ./test-bin
test-prebuilt:
parameters:
executor:
type: executor
executor: << parameters.executor >>
parallelism: 8
steps:
- checkout
- attach_workspace:
at: .
- run:
name: Run tests
command: ./scripts/ci/run-prebuilt-tests.sh $(pwd)/test-bin $(pwd)/test-results
- store_test_results:
path: ./test-results
- store_artifacts:
path: ./test-results
destination: raw-test-results
fluxtest:
executor: cross-builder
steps:
- checkout
- run: make test-flux
tlstest:
executor: cross-builder
steps:
- checkout
- run: make test-tls
build:
executor: cross-builder
parameters:
os:
type: string
arch:
type: string
build-type:
type: string
steps:
- checkout
- run:
name: Generate UI assets
command: make generate-web-assets
- run:
name: Build binaries
command: |
export GOOS=<< parameters.os >>
export GOARCH=<< parameters.arch >>
./scripts/ci/build.sh "bin/influxd_$(go env GOOS)_$(go env GOARCH)" "<< parameters.build-type >>" ./cmd/influxd
- store_artifacts:
path: bin
- persist_to_workspace:
root: .
paths:
- bin
package-and-release:
executor: cross-builder
parameters:
build-type:
type: string
steps:
- checkout
- attach_workspace:
at: .
- run:
name: Compute artifact version
command: echo "export GORELEASER_CURRENT_TAG='$(build-version.sh << parameters.build-type >>)'" >> $BASH_ENV
- unless:
condition:
equal: [ release, << parameters.build-type >> ]
steps:
- run:
name: Tag commit locally to pass goreleaser validation
command: git tag "$GORELEASER_CURRENT_TAG"
- run:
name: Import GPG key
command: |
echo -e "$GPG_PRIVATE_KEY" > private.key
gpg --batch --import private.key
- run:
name: Install goreleaser-pro
environment:
GORELEASER_VERSION: 0.184.0
# checksum from `checksums.txt` file at https://github.com/goreleaser/goreleaser-pro/releases
GORELEASER_SHA: def0b9882e0fbb91727c73bcc033917c4562aa5f9e89317884faff2b114c6b9c
command: |
curl --proto '=https' --tlsv1.2 -sSfL --max-redirs 1 -O \
https://github.com/goreleaser/goreleaser-pro/releases/download/v${GORELEASER_VERSION}-pro/goreleaser-pro_Linux_x86_64.tar.gz
echo "${GORELEASER_SHA} goreleaser-pro_Linux_x86_64.tar.gz" | sha256sum --check -
# Extract goreleaser binary only.
tar --extract --file=goreleaser-pro_Linux_x86_64.tar.gz goreleaser
rm goreleaser-pro_Linux_x86_64.tar.gz
# Move goreleaser binary out of project dir so it doesn't dirty the git state.
mv goreleaser /go/bin/
- run:
name: Run goreleaser
command: goreleaser --debug release --config .goreleaser/<< parameters.build-type >>.yml
# goreleaser generates some temp files in the dist/
# directory alongside the artifacts we want to save.
- run:
name: Clean up artifacts
command: |
mkdir artifacts
mv dist/influx* artifacts/
- store_artifacts:
path: artifacts
- persist_to_workspace:
root: .
paths:
- artifacts
test-linux-packages:
executor: terraform/default
steps:
- attach_workspace:
at: /tmp/workspace
- checkout
- add_ssh_keys:
fingerprints:
- "91:0a:5b:a7:f9:46:77:f3:5d:4a:cf:d2:44:c8:2c:5a"
- terraform/validate:
path: scripts/ci/
- run:
name: Terraform apply
command: |
set -x
export DEBNAME="$(find /tmp/workspace/artifacts/influxdb2-*-amd64.deb)"
terraform -chdir=scripts/ci init -input=false
AWS_ACCESS_KEY_ID=$TEST_AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY=$TEST_AWS_SECRET_ACCESS_KEY terraform \
-chdir=scripts/ci \
apply \
-auto-approve \
-var package_path=${DEBNAME} \
-var instance_name=circleci-terraform-${CIRCLE_SHA1} \
-var additional_files_dir=${PWD}/scripts/ci/tests/
- run:
name: Install deb
command: |
set -x
export ec2_ip=$(terraform -chdir=scripts/ci output -raw test_node_ssh)
ssh -o "StrictHostKeyChecking=no" ubuntu@$ec2_ip \<< EOF
sudo apt-get update && sudo apt-get install -y /home/ubuntu/influxdb.deb
EOF
- run:
name: Run tests
command: |
set -x
export ec2_ip=$(terraform -chdir=scripts/ci output -raw test_node_ssh)
files=$(ssh -o "StrictHostKeyChecking=no" ubuntu@$ec2_ip 'find /home/ubuntu/files/ -maxdepth 1 -mindepth 1 | sort')
for file in $files; do
ssh -o "StrictHostKeyChecking=no" ubuntu@$ec2_ip "sudo $file"
done
- run:
name: Terraform destroy
when: always
command: |
AWS_ACCESS_KEY_ID=$TEST_AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY=$TEST_AWS_SECRET_ACCESS_KEY terraform \
-chdir=scripts/ci \
destroy \
-auto-approve
perf-test:
docker:
- image: cimg/base:2021.04
resource_class: small
parameters:
record_results:
type: boolean
steps:
- attach_workspace:
at: /tmp/workspace
- checkout
# To ssh into aws without failing host key checks
- add_ssh_keys:
fingerprints:
- "91:0a:5b:a7:f9:46:77:f3:5d:4a:cf:d2:44:c8:2c:5a"
- run:
name: Set up AWS CLI
command: |
./scripts/ci/install-aws-cli.sh
echo "export AWS_ACCESS_KEY_ID=${TEST_AWS_ACCESS_KEY_ID}" >> vars.sh
echo "export AWS_SECRET_ACCESS_KEY=${TEST_AWS_SECRET_ACCESS_KEY}" >> vars.sh
echo "export TEST_RECORD_RESULTS=<< parameters.record_results >>" >> vars.sh
echo 'export DATA_I_TYPE="r5.2xlarge"' >> vars.sh
- run:
name: Set up AWS instance
command: |
scripts/ci/perf_test.sh
- run:
name: Run perf test
no_output_timeout: 20m
command: |
source vars.sh
set -x
ssh "ubuntu@$EC2_IP" "sudo ./run_perftest.sh"
- run:
name: Tear down AWS instance
when: always
command: |
source vars.sh
set -x
if [[ -n "$EC2_INSTANCE_ID" ]] ; then
aws --region us-west-2 ec2 terminate-instances --instance-ids "$EC2_INSTANCE_ID"
fi
aws-destroy-by-date:
executor: linux-amd64
steps:
- attach_workspace:
at: /tmp/workspace
- checkout
- add_ssh_keys:
fingerprints:
- "91:0a:5b:a7:f9:46:77:f3:5d:4a:cf:d2:44:c8:2c:5a"
- run:
name: Destroy AWS instances with datestring more than a day old
no_output_timeout: 20m
command: |
set -x
yesterday_date=$(date --date "yesterday" +%Y%m%d)
instance_info=$(AWS_ACCESS_KEY_ID=${TEST_AWS_ACCESS_KEY_ID} \
AWS_SECRET_ACCESS_KEY=${TEST_AWS_SECRET_ACCESS_KEY} \
aws --region us-west-2 ec2 describe-instances \
--filters "Name=tag:Name,Values=oss-perftest-*" \
--query "Reservations[].Instances[].[InstanceId, Tags[?Key=='Name']|[0].Value]" \
--output text)
while [ -n "$instance_info" ]; do
instance_id=$(echo $instance_info | tr -s ' ' | cut -d ' ' -f1)
name=$(echo $instance_info | tr -s ' ' | cut -d ' ' -f2)
instance_info=$(echo $instance_info | tr -s ' ' | cut -d ' ' -f3-)
date=$(echo $name | cut -d '-' -f3)
if [ $date -le $yesterday_date ]; then
AWS_ACCESS_KEY_ID=${TEST_AWS_ACCESS_KEY_ID} AWS_SECRET_ACCESS_KEY=${TEST_AWS_SECRET_ACCESS_KEY} aws --region us-west-2 ec2 terminate-instances --instance-ids $instance_id
fi
done
e2e-monitor-ci:
docker:
- image: cimg/base:stable
steps:
- checkout
- attach_workspace:
at: .
- setup_remote_docker
- quay_login
- run:
name: Build and push candidate docker image
command: |
cp bin/influxd_linux_amd64/influxd .
docker build -f docker/influxd/Dockerfile -t quay.io/influxdb/oss-acceptance:${CIRCLE_SHA1} .
docker push quay.io/influxdb/oss-acceptance:${CIRCLE_SHA1}
- run:
name: Run the tests
command: API_KEY=${MONITOR_CI_API_KEY} MONITOR_CI_BRANCH="master" OSS_SHA=${CIRCLE_SHA1} ./scripts/ci/run-monitor-ci-tests.bash
- store_artifacts:
path: monitor-ci/test-artifacts/results/build-oss-image
destination: test_artifacts/results/build-oss-image
- store_artifacts:
path: monitor-ci/test-artifacts/results/oss-e2e
destination: test_artifacts/results/oss-e2e
- store_artifacts:
path: monitor-ci/test-artifacts/results/shared
destination: test_artifacts/results/shared
litmus-smoke-test:
executor: linux-amd64
steps:
- attach_workspace:
at: ~/project
- run: docker run --entrypoint "./run_litmus_tests_oss.sh" -e ONE_TEST=src/cloud/rest_api/smoke/test_smoke.py -e BINARYPATH=/Litmus/result/bin/influxd_linux_amd64/influxd -e BOLTPATH=/Litmus/result/influxd_test/influxd.bolt -e ENGINEPATH=/Litmus/result/influxd_test --net host -v /var/run/docker.sock:/var/run/docker.sock -v ~/project:/Litmus/result quay.io/influxdb/litmus:latest
- store_artifacts:
path: ~/project
destination: raw-daily-output
- store_test_results:
path: ~/project
litmus-full-test:
executor: linux-amd64
steps:
- attach_workspace:
at: ~/project
- run: docker run --entrypoint "./run_litmus_tests_oss.sh" -e TEST_LIST=tests_lists/gateway_api_tests.list -e BINARYPATH=/Litmus/result/bin/influxd_linux_amd64/influxd -e BOLTPATH=/tmp/influxd_test/influxd.bolt -e ENGINEPATH=/tmp/influxd_test --net host -v /var/run/docker.sock:/var/run/docker.sock -v ~/project:/Litmus/result quay.io/influxdb/litmus:latest
- store_artifacts:
path: ~/project
destination: raw-daily-output
- store_test_results:
path: ~/project
grace-test:
executor: linux-amd64
steps:
- attach_workspace:
at: ~/project
- run:
command: ./bin/influxd_linux_amd64/influxd --store=memory --log-level=debug
background: true
- run: mkdir -p ~/project/results
- run:
name: Wait for influxd to bind HTTP port
command: |
attempts=0
max_attempts=30
while ! curl localhost:8086/health; do
attempts=$((attempts+1))
if [[ $attempts = $max_attempts ]]; then
>&2 echo influxd "didn't" start in time
exit 1
fi
sleep 1
done
- run: docker run --net host -v /var/run/docker.sock:/var/run/docker.sock -v ~/project/results:/grace/test-results/grace-results quay.io/influxdb/grace:daily
- store_artifacts:
path: ~/project/results
- store_test_results:
path: ~/project/results
test-downgrade:
executor: cross-builder
steps:
- checkout
- attach_workspace:
at: .
- run:
name: Run downgrade tests
command: |
./scripts/ci/test-downgrade.sh $(pwd)/bin/influxd_linux_amd64/influxd
share-testing-image:
docker:
- image: cimg/base:stable
steps:
- setup_remote_docker
- quay_login
- run:
name: Push the image to Quay
command: |
docker pull quay.io/influxdb/oss-acceptance:${CIRCLE_SHA1}
docker tag quay.io/influxdb/oss-acceptance:${CIRCLE_SHA1} quay.io/influxdb/oss-acceptance:latest
docker push quay.io/influxdb/oss-acceptance:latest
changelog:
docker:
- image: quay.io/influxdb/changelogger:cf2648551b43797458d21e061501f60590b31daa
steps:
- checkout
- run:
name: Generate changelog
command: |
BRANCH="<< pipeline.git.branch >>"
VERSION="v$BRANCH"
DOCS_LINKS_VERSION="$VERSION"
# Runs triggered by pushed tags do not have the pipeline.git.branch parameter available.
if [ -n "<< pipeline.git.tag >>" ]; then
VERSION="<< pipeline.git.tag >>"
DOCS_LINKS_VERSION="$(echo "$VERSION" | sed 's/\.[^.]*$//')"
BRANCH="$(echo "$DOCS_LINKS_VERSION" | sed 's/v//' )"
fi
mkdir changelog_artifacts
echo -e "CHANGELOG for InfluxDB OSS $VERSION commit ${CIRCLE_SHA1}\n" > changelog_artifacts/CHANGELOG.md
if [[ ! "$BRANCH" == "master" ]] ; then
echo -e "In addition to the list of changes below, please also see the [official release notes](https://docs.influxdata.com/influxdb/$DOCS_LINKS_VERSION/reference/release-notes/influxdb/) for other important information about this release.\n" >> changelog_artifacts/CHANGELOG.md
fi
if [[ "$BRANCH" == "master" ]] ; then
LATEST_21X="$(git tag | grep '^v2\.1\.[0-9]*$' | sort -V | tail -1)"
# build 2.1.0 to 2.1.x changelog, plus unreleased from master
build-multibranch-changelog.sh HEAD "$LATEST_21X" v2.1.0 >> changelog_artifacts/CHANGELOG.md
elif [[ "$BRANCH" == "2.1" ]] ; then
# build 2.1.0 to current 2.1 changelog
build-simple-changelog.sh HEAD v2.1.0 >> changelog_artifacts/CHANGELOG.md
fi
cat ./scripts/ci/CHANGELOG_frozen.md >> changelog_artifacts/CHANGELOG.md
- store_artifacts:
path: changelog_artifacts/
- persist_to_workspace:
root: .
paths:
- changelog_artifacts