influxdb/.circleci/config.yml

794 lines
26 KiB
YAML

version: "2.1"
workflows:
version: 2
build:
jobs:
- godeps
- jsdeps
- gotest:
requires:
- godeps
- jstest:
requires:
- jsdeps
- influxql_validation:
requires:
- godeps
- influxql_integration:
requires:
- godeps
- golint:
requires:
- godeps
- jslint:
requires:
- jsdeps
- build:
requires:
- godeps
- jsdeps
- cross_build:
requires:
- godeps
- jsdeps
- e2e:
requires:
- build
- grace_daily:
requires:
- build
filters:
branches:
# Forked pull requests have CIRCLE_BRANCH set to pull/XXX
ignore: /pull\/[0-9]+/
- litmus_daily:
requires:
- build
filters:
branches:
# Forked pull requests have CIRCLE_BRANCH set to pull/XXX
ignore: /pull\/[0-9]+/
- litmus_integration:
requires:
- build
filters:
branches:
only: master
hourly-e2e:
triggers:
- schedule:
cron: "0 * * * *"
filters:
branches:
only:
- master
jobs:
- godeps
- jsdeps
- build:
requires:
- godeps
- jsdeps
- e2e:
requires:
- build
nightly:
triggers:
- schedule:
cron: "0 5 * * *"
filters:
branches:
only:
- master
jobs:
- godeps
- jsdeps
- gotest:
requires:
- godeps
- golint:
requires:
- godeps
- influxql_validation:
requires:
- godeps
- influxql_integration:
requires:
- godeps
- jstest:
requires:
- jsdeps
- jslint:
requires:
- jsdeps
- deploy_nightly:
requires:
- gotest
- golint
- jstest
- jslint
- influxql_validation
- influxql_integration
filters:
branches:
only:
- master
- litmus_nightly:
requires:
- deploy_nightly
- grace_nightly:
requires:
- deploy_nightly
orbs:
# Needed to install chrome for e2e testing.
browser-tools: circleci/browser-tools@1.1
commands:
# Log into quay.io, with a pre-check to ensure we aren't running from a forked PR.
quay_login:
steps:
- run:
name: Ensure not running from a fork
command: |
if [ -n "$CIRCLE_PR_NUMBER" ]; then
echo 'Error: build from fork detected, exiting!'
exit 1
fi
- run:
name: Log into Quay
command: docker login -u "$QUAY_USER" -p $QUAY_PASS quay.io
# Install system dependencies needed to run a native build of influxd
install_core_deps:
steps:
- run:
name: Install system dependencies
command: |
sudo apt-get update
sudo apt-get install -y --no-install-recommends \
bzr \
clang \
libprotobuf-dev \
pkg-config \
protobuf-compiler
- run:
name: Install Rust toolchain
command: |
curl https://sh.rustup.rs -sSf | sh -s -- --default-toolchain stable -y
echo 'export PATH=${HOME}/.cargo/bin:${PATH}' >> $BASH_ENV
# Run goreleaser to cross-build or cross-publish influxd
run_goreleaser:
parameters:
publish_release:
type: boolean
steps:
- when:
condition: << parameters.publish_release >>
steps:
- run:
name: Ensure not running from a fork
command: |
if [ -n "$CIRCLE_PR_NUMBER" ]; then
echo 'Error: release from fork detected, exiting!'
exit 1
fi
- run:
name: Import GPG key
command: |
echo -e "$GPG_PRIVATE_KEY" > private.key
gpg --batch --import private.key
- run:
name: Log into Quay
command: docker login -u "$QUAY_USER" -p $QUAY_PASS quay.io
- run:
name: Set GOPATH
# Machine executors use a different GOPATH from the cimg/go Docker executors.
command: |
echo 'export GOPATH=/home/circleci/go' >> $BASH_ENV
echo 'export PATH=${GOPATH}/bin:${PATH}' >> $BASH_ENV
- run:
name: Install updated Docker
command: |
export BUILDKIT_PROGRESS=plain
export DOCKER_BUILDKIT=1
export DOCKER_CLI_EXPERIMENTAL=enabled
echo 'export BUILDKIT_PROGRESS=plain' >> $BASH_ENV
echo 'export DOCKER_BUILDKIT=1' >> $BASH_ENV
echo 'export DOCKER_CLI_EXPERIMENTAL=enabled' >> $BASH_ENV
curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add -
sudo add-apt-repository \
"deb [arch=amd64] https://download.docker.com/linux/ubuntu \
$(lsb_release -cs) \
stable"
sudo apt-get update
sudo apt-get install -y \
containerd.io \
docker-ce \
docker-ce-cli
sudo service docker restart
- run:
name: Set up Docker cross-builder
command: |
# Get jq to parse binfmt output.
sudo apt-get update && sudo apt-get install -y jq
# Uninstall any emulators provided by the system.
emulators=($(docker run --rm --privileged tonistiigi/binfmt:latest | jq -r .emulators[]))
for e in ${emulators[@]}; do
docker run --rm --privileged tonistiigi/binfmt:latest --uninstall ${e}
done
# Install the QEMU emulators we need to cross-build.
docker run --rm --privileged tonistiigi/binfmt:latest --install all
# Create a new buildx context using the freshly-installed emulators.
docker buildx create --name cross-builder
docker buildx use --default cross-builder
docker buildx inspect --bootstrap
# Build the 1st stage of our Docker(s) on our target platforms, to flush out
# any problems in our emulator setup.
docker buildx build --target dependency-base --platform linux/amd64,linux/arm64 docker/influxd
- run:
name: Install cross-build system dependencies
command: |
sudo apt-get update
sudo apt-get install -y \
cmake \
gnupg \
libssl-dev \
libxml2-dev \
llvm-dev \
lzma-dev \
zlib1g-dev
- run:
name: Install cross-compilers
environment:
MUSL_VERSION: 1.1.24
MUSL_BUILD_TIME: 20210108172549
MUSL_CROSS_MAKE_VERSION: 0.9.9
MUSL_CROSS_ARM64_BUILD_TIME: 20210108174735
OSXCROSS_VERSION: c2ad5e859d12a295c3f686a15bd7181a165bfa82
OSXCROSS_BUILD_TIME: 20210108174800
command: |
MUSL_ARCHIVE=musl-${MUSL_VERSION}-${MUSL_BUILD_TIME}.tar.gz
curl https://dl.influxdata.com/influxdb-ci/musl/${MUSL_VERSION}/${MUSL_ARCHIVE} -O && \
sudo tar xzf ${MUSL_ARCHIVE} -C /usr/local && \
rm ${MUSL_ARCHIVE}
echo 'export PATH=/usr/local/musl/bin:${PATH}' >> $BASH_ENV
MUSL_CROSS_ARM64_ARCHIVE=musl-${MUSL_VERSION}-cross-aarch64-${MUSL_CROSS_MAKE_VERSION}-${MUSL_CROSS_ARM64_BUILD_TIME}.tar.gz
curl https://dl.influxdata.com/influxdb-ci/musl/${MUSL_VERSION}/musl-cross/${MUSL_CROSS_MAKE_VERSION}/${MUSL_CROSS_ARM64_ARCHIVE} -O && \
sudo tar xzf ${MUSL_CROSS_ARM64_ARCHIVE} -C /usr/local && \
rm ${MUSL_CROSS_ARM64_ARCHIVE}
echo 'export PATH=/usr/local/musl-cross/bin:${PATH}' >> $BASH_ENV
OSXCROSS_ARCHIVE=osxcross-${OSXCROSS_VERSION}-${OSXCROSS_BUILD_TIME}.tar.gz
curl https://dl.influxdata.com/influxdb-ci/osxcross/${OSXCROSS_VERSION}/${OSXCROSS_ARCHIVE} -O && \
sudo tar xzf ${OSXCROSS_ARCHIVE} -C /usr/local && \
rm ${OSXCROSS_ARCHIVE}
echo 'export PATH=/usr/local/osxcross/target/bin:${PATH}' >> $BASH_ENV
mkdir -p ${GOPATH}/bin
cp scripts/ci/xcc.sh ${GOPATH}/bin/xcc
chmod a+x ${GOPATH}/bin/xcc
- run:
name: Install Rust cross-targets
command: |
rustup target add \
x86_64-unknown-linux-musl \
aarch64-unknown-linux-musl \
x86_64-apple-darwin
echo 'export CARGO_TARGET_X86_64_UNKNOWN_LINUX_MUSL_LINKER=/usr/local/musl/bin/musl-gcc' >> $BASH_ENV
echo 'export CARGO_TARGET_AARCH64_UNKNOWN_LINUX_MUSL_LINKER=/usr/local/musl-cross/bin/aarch64-unknown-linux-musl-gcc' >> $BASH_ENV
echo 'export CARGO_TARGET_X86_64_APPLE_DARWIN_LINKER=/usr/local/osxcross/target/bin/x86_64-apple-darwin15-clang' >> $BASH_ENV
- run:
name: Install goreleaser
environment:
GORELEASER_VERSION: 0.152.0
command: |
curl -sfL -o goreleaser-install https://install.goreleaser.com/github.com/goreleaser/goreleaser.sh && \
sh goreleaser-install -b ${GOPATH}/bin v${GORELEASER_VERSION} && \
rm goreleaser-install
- run:
name: Install pkg-config
command: make pkg-config
- unless:
condition: << parameters.publish_release >>
steps:
- run:
name: Build release
# `goreleaser release --skip-publish` builds Docker images, but doesn't push them.
# As opposed to `goreleaser build`, which stops before building Dockers.
command: goreleaser --debug release --skip-publish --skip-sign -p 1 --rm-dist --skip-validate
- when:
condition: << parameters.publish_release >>
steps:
- run:
name: Publish release
command: goreleaser --debug release -p 1 --rm-dist --skip-validate
jobs:
####################
### UI-only jobs ###
####################
jsdeps:
docker:
- image: cimg/go:1.15.6-node
working_directory: /home/circleci/go/src/github.com/influxdata/influxdb
steps:
- checkout
- restore_cache:
name: Restore Yarn Cache
keys:
- yarn-deps-lock-{{ checksum "ui/yarn.lock" }}
- yarn-deps-lock-
- run:
name: Install Dependencies
command: |
set +e
cd ui
yarn install --immutable
- save_cache:
name: Save Yarn Cache
key: yarn-deps-lock-{{ checksum "ui/yarn.lock" }}
paths:
- /home/circleci/go/src/github.com/influxdata/influxdb/ui/node_modules
- ~/.cache/yarn
- ~/.cache/Cypress
jstest:
docker:
- image: cimg/go:1.15.6-node
working_directory: /home/circleci/go/src/github.com/influxdata/influxdb
steps:
- checkout
- restore_cache:
name: Restore Yarn Cache
keys:
- yarn-deps-lock-{{ checksum "ui/yarn.lock" }}
- run:
name: install dependencies
command: |
cd ui
# This should usually be a no-op (fully contained in the Yarn cache above), but we
# include it to be safe since `yarn test` won't auto-install missing modules.
yarn install --immutable
- run: make ui_client
- run:
name: run tests
command: |
cd ui
yarn test:circleci
- store_test_results:
path: ui/coverage
- store_artifacts:
path: ui/coverage
destination: raw-test-output
jslint:
docker:
- image: cimg/go:1.15.6-node
working_directory: /home/circleci/go/src/github.com/influxdata/influxdb
parallelism: 8
steps:
- checkout
- restore_cache:
name: Restore Yarn Cache
keys:
- yarn-deps-lock-{{ checksum "ui/yarn.lock" }}
- run:
name: install dependencies
command: |
cd ui
# This should usually be a no-op (fully contained in the Yarn cache above), but we
# include it to be safe since the lint commands won't auto-install missing modules.
yarn install --immutable
- run: make ui_client
- run:
name: parallel eslint
command: |
cd ui
TESTFILES=$(circleci tests glob "src/**/*.ts*" "cypress/**/*.ts*" | circleci tests split --split-by=filesize)
yarn prettier:circleci ${TESTFILES[@]}
yarn eslint:circleci ${TESTFILES[@]}
#########################
### Backend-only jobs ###
#########################
godeps:
docker:
- image: cimg/go:1.15.6
environment:
GOCACHE: /tmp/go-cache
working_directory: /home/circleci/go/src/github.com/influxdata/influxdb
steps:
- checkout
- restore_cache:
name: Restore GOPATH/pkg/mod
keys:
- influxdb-gomod-sum-{{ checksum "go.sum" }}
- influxdb-gomod-sum-
- run:
name: Install Dependencies
command: go mod download -x
- save_cache:
name: Save GOPATH/pkg/mod
key: influxdb-gomod-sum-{{ checksum "go.sum" }}
paths:
- /home/circleci/go/pkg/mod
golint:
docker:
- image: cimg/go:1.15.6
environment:
GOCACHE: /tmp/go-cache
working_directory: /home/circleci/go/src/github.com/influxdata/influxdb
steps:
- checkout
- restore_cache:
keys:
- influxdb-gomod-sum-{{ checksum "go.sum" }}
- run: |
# this is not in a seperate bash script because it isn't meant to be run on local.
# it just checks to make sure that the same major/minor version of go is used in the mod file as on ci
# to prevent accidentally checking in a wrong go mod version.
gomodversiondiff=$( go mod edit -go=$( go version | sed -n 's/^.*go\([0-9]*.[0-9]*\).*$/\1/p') -print |diff - go.mod )
if [ "$gomodversiondiff" ]
then
echo unexpected go version $gomodversiondiff
exit 1
fi
- install_core_deps
- run: make vet
- run: make checkfmt
- run: make checktidy
- run: GO111MODULE=on go mod vendor # staticcheck looks in vendor for dependencies.
- run: GO111MODULE=on go install honnef.co/go/tools/cmd/staticcheck # Install staticcheck from the version we specify in go.mod.
- run: GO111MODULE=on ./env staticcheck ./...
gotest:
docker:
- image: cimg/go:1.15.6
resource_class: large
environment:
GOCACHE: /tmp/go-cache
TEST_RESULTS: /tmp/test-results
working_directory: /home/circleci/go/src/github.com/influxdata/influxdb
parallelism: 8
steps:
- checkout
- restore_cache:
name: Restore GOCACHE
keys:
- influxdb-gotest-{{ .Branch }}-{{ .Revision }}
- influxdb-gotest-{{ .Branch }}-
- restore_cache:
name: Restore GOPATH/pkg/mod
keys:
- influxdb-gomod-sum-{{ checksum "go.sum" }}
- run: mkdir -p $TEST_RESULTS
- install_core_deps
- run:
name: run parallel race tests
command: |
GO_TEST_CMD="gotestsum --format standard-quiet --junitfile /tmp/test-results/gotestsum.xml -- -p=4"
TESTFILES=($(go list ./... | circleci tests split --split-by=timings))
make GO_TEST_CMD="$GO_TEST_CMD" GO_TEST_PATHS="${TESTFILES[*]}" test-go-race
- save_cache:
name: Save GOCACHE
key: influxdb-gotest-{{ .Branch }}-{{ .Revision }}
paths:
- /tmp/go-cache
- store_artifacts:
path: /tmp/test-results
destination: raw-test-output
- store_test_results:
path: /tmp/test-results
influxql_validation:
docker:
- image: cimg/go:1.15.6
resource_class: large
environment:
GOCACHE: /tmp/go-cache
TEST_RESULTS: /tmp/test-results
working_directory: /home/circleci/go/src/github.com/influxdata/influxdb
steps:
- checkout
- restore_cache:
name: Restore GOCACHE
keys:
- influxdb-iqlvalidation-{{ .Branch }}-{{ .Revision }}
- influxdb-iqlvalidation-{{ .Branch }}-
- restore_cache:
name: Restore GOPATH/pkg/mod
keys:
- influxdb-gomod-sum-{{ checksum "go.sum" }}
- run: mkdir -p $TEST_RESULTS
- install_core_deps
- run: make GO_TEST_CMD="gotestsum --format standard-quiet --junitfile /tmp/test-results/gotestsum.xml --" test-influxql-validation
- save_cache:
name: Save GOCACHE
key: influxdb-iqlvalidation-{{ .Branch }}-{{ .Revision }}
paths:
- /tmp/go-cache
- store_artifacts:
path: /tmp/test-results
destination: raw-test-output
- store_test_results:
path: /tmp/test-results
influxql_integration:
docker:
- image: cimg/go:1.15.6
resource_class: large
environment:
GOCACHE: /tmp/go-cache
TEST_RESULTS: /tmp/test-results
working_directory: /home/circleci/go/src/github.com/influxdata/influxdb
steps:
- checkout
- restore_cache:
name: Restore GOCACHE
keys:
- influxdb-iqlintegration-{{ .Branch }}-{{ .Revision }}
- influxdb-iqlintegration-{{ .Branch }}-
- restore_cache:
name: Restore GOPATH/pkg/mod
keys:
- influxdb-gomod-sum-{{ checksum "go.sum" }}
- run: mkdir -p $TEST_RESULTS
- install_core_deps
- run: make GO_TEST_CMD="gotestsum --format standard-quiet --junitfile /tmp/test-results/gotestsum.xml --" test-influxql-integration
- save_cache:
name: Save GOCACHE
key: influxdb-iqlintegration-{{ .Branch }}-{{ .Revision }}
paths:
- /tmp/go-cache
- store_artifacts:
path: /tmp/test-results
destination: raw-test-output
- store_test_results:
path: /tmp/test-results
#####################################
### UI+backend build/release jobs ###
#####################################
build:
docker:
- image: cimg/go:1.15.6-node
resource_class: large
environment:
GOCACHE: /tmp/go-cache
working_directory: /home/circleci/go/src/github.com/influxdata/influxdb
steps:
- checkout
- restore_cache:
name: Restore GOPATH/pkg/mod
keys:
- influxdb-gomod-sum-{{ checksum "go.sum" }}
- restore_cache:
name: Restore GOCACHE
keys:
- influxdb-build-{{ .Branch }}-{{ .Revision }}
- influxdb-build-{{ .Branch }}-
- restore_cache:
name: Restore Yarn Cache
keys:
- yarn-deps-lock-{{ checksum "ui/yarn.lock" }}
- install_core_deps
- run: make build
- save_cache:
name: Save GOCACHE
key: influxdb-build-{{ .Branch }}-{{ .Revision }}
paths:
- /tmp/go-cache
- persist_to_workspace:
root: .
paths:
- project
- bin/linux/influxd
- bin/linux/influx
- etc/litmus_success_notify.sh
- etc/litmus_fail_notify.sh
cross_build:
machine:
image: ubuntu-2004:202010-01
resource_class: large
environment:
GOCACHE: /tmp/go-cache
steps:
- checkout
- restore_cache:
name: Restore GOPATH/pkg/mod
keys:
- influxdb-gomod-sum-{{ checksum "go.sum" }}
- restore_cache:
name: Restore GOCACHE
keys:
- influxdb-cross-build-{{ .Branch }}-{{ .Revision }}
- influxdb-cross-build-{{ .Branch }}-
- restore_cache:
name: Restore Yarn Cache
keys:
- yarn-deps-lock-{{ checksum "ui/yarn.lock" }}
- install_core_deps
- run_goreleaser:
publish_release: false
- save_cache:
name: Save GOCACHE
key: influxdb-cross-build-{{ .Branch }}-{{ .Revision }}
paths:
- /tmp/go-cache
deploy_nightly:
machine:
image: ubuntu-2004:202010-01
resource_class: large
environment:
GOCACHE: /tmp/go-cache
working_directory: /home/circleci/go/src/github.com/influxdata/influxdb
steps:
- checkout
- restore_cache:
name: Restore GOPATH/pkg/mod
keys:
- influxdb-gomod-sum-{{ checksum "go.sum" }}
- restore_cache:
name: Restore GOCACHE
keys:
- influxdb-nightly-{{ .Branch }}-{{ .Revision }}
- influxdb-nightly-{{ .Branch }}-
- restore_cache:
name: Restore Yarn Cache
keys:
- yarn-deps-lock-{{ checksum "ui/yarn.lock" }}
- quay_login
- install_core_deps
- run_goreleaser:
publish_release: true
- save_cache:
name: Save GOCACHE
key: influxdb-nightly-{{ .Branch }}-{{ .Revision }}
paths:
- /tmp/go-cache
- persist_to_workspace:
root: .
paths:
- etc/litmus_success_notify.sh
- etc/litmus_fail_notify.sh
#################################
### e2e/integration test jobs ###
#################################
e2e:
docker:
- image: cimg/go:1.15.6-browsers
environment:
GOCACHE: /tmp/go-cache
working_directory: /home/circleci/go/src/github.com/influxdata/influxdb
steps:
- browser-tools/install-browser-tools
- checkout
- attach_workspace:
at: /home/circleci/go/src/github.com/influxdata/influxdb
- restore_cache:
name: Restore Yarn Cache
keys:
- yarn-deps-lock-{{ checksum "ui/yarn.lock" }}
- run: sudo apt-get update && sudo apt-get install netcat-openbsd
- run:
command: ./bin/linux/influxd --store=memory --e2e-testing=true --feature-flags=communityTemplates=true
background: true
- run: make e2e
- store_test_results:
path: ui/junit-results
- store_artifacts:
path: ui/cypress/videos
destination: videos
- store_artifacts:
path: ui/cypress/screenshots
destination: screenshots
litmus_daily:
machine: true
steps:
- attach_workspace:
at: ~/project
- quay_login
- run: docker run --entrypoint "./run_litmus_tests_oss.sh" -e ONE_TEST=src/cloud/rest_api/smoke/test_smoke.py -e BINARYPATH=/Litmus/result/bin/linux/influxd -e BOLTPATH=/Litmus/result/influxd_test/influxd.bolt -e ENGINEPATH=/Litmus/result/influxd_test --net host -v /var/run/docker.sock:/var/run/docker.sock -v ~/project:/Litmus/result quay.io/influxdb/litmus:latest
- run:
name: Litmus Smoke Tests Success
when: on_success
command: bash ~/project/etc/litmus_success_notify.sh Smoke
path: ~/project
- run:
name: Litmus Smoke Tests Fail
when: on_fail
command: bash ~/project/etc/litmus_fail_notify.sh Smoke
- store_artifacts:
path: ~/project
destination: raw-daily-output
- store_test_results:
path: ~/project
litmus_integration:
machine: true
steps:
- attach_workspace:
at: ~/project
- quay_login
- run: docker run --entrypoint "./run_litmus_tests_oss.sh" -e TEST_LIST=tests_lists/gateway_api_tests.list -e INFLUXPATH=/Litmus/result/bin/linux/influx -e BINARYPATH=/Litmus/result/bin/linux/influxd -e BOLTPATH=/tmp/influxd_test/influxd.bolt -e ENGINEPATH=/tmp/influxd_test --net host -v /var/run/docker.sock:/var/run/docker.sock -v ~/project:/Litmus/result quay.io/influxdb/litmus:latest
- run:
name: Litmus Integration Tests Success
when: on_success
command: bash ~/project/etc/litmus_success_notify.sh Integration
- run:
name: Litmus Integration Tests Failure
when: on_fail
command: bash ~/project/etc/litmus_fail_notify.sh Integration
- store_artifacts:
path: ~/project
destination: raw-daily-output
- store_test_results:
path: ~/project
litmus_nightly:
machine: true
steps:
- attach_workspace:
at: ~/project
- quay_login
- run: docker run --entrypoint "./run_litmus_tests_oss.sh" -e TEST_LIST=tests_lists/gateway_api_tests.list -e DOCKERIMAGE=true --net host -v /var/run/docker.sock:/var/run/docker.sock -v ~/project:/Litmus/result quay.io/influxdb/litmus:latest
- run:
name: Litmus Nightly Tests Success
when: on_success
command: bash ~/project/etc/litmus_success_notify.sh Nightly
- run:
name: Litmus Nightly Tests Fail
when: on_fail
command: bash ~/project/etc/litmus_fail_notify.sh Nightly
- store_artifacts:
path: ~/project
destination: raw-nightly-output
- store_test_results:
path: ~/project
grace_nightly:
machine: true
steps:
- attach_workspace:
at: ~/project
- quay_login
- run: docker run --net host -v /var/run/docker.sock:/var/run/docker.sock -e TEST_RESULTS=~/project quay.io/influxdb/grace:latest
- store_artifacts:
path: ~/project
- store_test_results:
path: ~/project
grace_daily:
machine: true
steps:
- attach_workspace:
at: ~/project
- quay_login
- run:
command: ./bin/linux/influxd --store=memory --log-level=debug
background: true
- run: docker run --net host -v /var/run/docker.sock:/var/run/docker.sock -v ~/project:/grace/test-results/grace-results quay.io/influxdb/grace:daily
- store_artifacts:
path: ~/project