influxdb/.circleci/config.yml

618 lines
22 KiB
YAML

version: "2.1"
commands:
install_rust_compiler:
description: >
This will install the rust compiler with the rust tools.
steps:
- run:
name: Install clang
command: sudo apt-get install -y --no-install-recommends clang
- run:
name: Install rust compiler
command: |
curl https://sh.rustup.rs -sSf | \
sh -s -- --default-toolchain stable -y
echo 'source $HOME/.cargo/env' >> $BASH_ENV
install_release_tools:
description: >
This will install the cross compilers necessary to build release binaries.
steps:
- run:
name: Install linux cross compilers
command: >
sudo apt-get install -y --no-install-recommends
gcc-arm-linux-gnueabihf libc6-dev-armhf-cross
gcc-aarch64-linux-gnu libc6-dev-arm64-cross
- run:
name: Install macOS cross compilers
command: |
sudo apt-get install -y --no-install-recommends \
cmake patch libxml2-dev libssl-dev zlib1g-dev
sudo mkdir -p /opt/osxcross
cd /opt
sudo git clone https://github.com/tpoechtrager/osxcross.git
cd osxcross
sudo git checkout c2ad5e859d12a295c3f686a15bd7181a165bfa82
sudo curl -L -o \
./tarballs/MacOSX10.11.sdk.tar.xz \
https://macos-sdks.s3.amazonaws.com/MacOSX10.11.sdk.tar.xz
sudo UNATTENDED=1 PORTABLE=true ./build.sh
- run:
name: Install additional rust targets
command: |
rustup target add aarch64-unknown-linux-gnu \
arm-unknown-linux-gnueabihf \
armv7-unknown-linux-gnueabihf \
x86_64-apple-darwin
echo 'export CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_LINKER=gcc' >> $HOME/.cargo/env
echo 'export CARGO_TARGET_ARM_UNKNOWN_LINUX_GNUEABIHF_LINKER=arm-linux-gnueabihf-gcc' >> $HOME/.cargo/env
echo 'export CARGO_TARGET_ARMV7_UNKNOWN_LINUX_GNUEABIHF_LINKER=arm-linux-gnueabihf-gcc' >> $HOME/.cargo/env
echo 'export CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_LINKER=aarch64-linux-gnu-gcc' >> $HOME/.cargo/env
echo 'export CARGO_TARGET_X86_64_APPLE_DARWIN_LINKER=/opt/osxcross/target/bin/o64-clang' >> $HOME/.cargo/env
- run:
name: Install pkg-config wrapper
command: go build -o /go/bin/pkg-config github.com/influxdata/pkg-config
- run:
name: Copy xcc wrapper script for cross compilation
command: install xcc.sh /go/bin/xcc
jobs:
litmus_daily:
machine: true
steps:
- attach_workspace:
at: ~/project
- run: docker login -u=$QUAY_USER -p=$QUAY_PASS quay.io
- run: docker run --entrypoint "./run_litmus_tests_oss.sh" -e ONE_TEST=src/cloud/rest_api/smoke/test_smoke.py -e BINARYPATH=/Litmus/result/bin/linux/influxd -e BOLTPATH=/Litmus/result/influxd_test/influxd.bolt -e ENGINEPATH=/Litmus/result/influxd_test --net host -v /var/run/docker.sock:/var/run/docker.sock -v ~/project:/Litmus/result quay.io/influxdb/litmus:latest
- run:
name: Litmus Smoke Tests Success
when: on_success
command: bash ~/project/etc/litmus_success_notify.sh Smoke
path: ~/project
- run:
name: Litmus Smoke Tests Fail
when: on_fail
command: bash ~/project/etc/litmus_fail_notify.sh Smoke
- store_artifacts:
path: ~/project
destination: raw-daily-output
- store_test_results:
path: ~/project
destination: daily-junit
litmus_integration:
machine: true
steps:
- attach_workspace:
at: ~/project
- run: docker login -u=$QUAY_USER -p=$QUAY_PASS quay.io
- run: docker run --entrypoint "./run_litmus_tests_oss.sh" -e TEST_LIST=tests_lists/gateway_api_tests.list -e INFLUXPATH=/Litmus/result/bin/linux/influx -e BINARYPATH=/Litmus/result/bin/linux/influxd -e BOLTPATH=/tmp/influxd_test/influxd.bolt -e ENGINEPATH=/tmp/influxd_test --net host -v /var/run/docker.sock:/var/run/docker.sock -v ~/project:/Litmus/result quay.io/influxdb/litmus:latest
- run:
name: Litmus Integration Tests Success
when: on_success
command: bash ~/project/etc/litmus_success_notify.sh Integration
- run:
name: Litmus Integration Tests Failure
when: on_fail
command: bash ~/project/etc/litmus_fail_notify.sh Integration
- store_artifacts:
path: ~/project
destination: raw-daily-output
- store_test_results:
path: ~/project
destination: daily-junit
litmus_nightly:
machine: true
steps:
- attach_workspace:
at: ~/project
- run: docker login -u=$QUAY_USER -p=$QUAY_PASS quay.io
- run: docker run --entrypoint "./run_litmus_tests_oss.sh" -e TEST_LIST=tests_lists/gateway_api_tests.list -e DOCKERIMAGE=true --net host -v /var/run/docker.sock:/var/run/docker.sock -v ~/project:/Litmus/result quay.io/influxdb/litmus:latest
- run:
name: Litmus Nightly Tests Success
when: on_success
command: bash ~/project/etc/litmus_success_notify.sh Nightly
- run:
name: Litmus Nightly Tests Fail
when: on_fail
command: bash ~/project/etc/litmus_fail_notify.sh Nightly
- store_artifacts:
path: ~/project
destination: raw-nightly-output
- store_test_results:
path: ~/project
destination: nightly-junit
e2e:
docker:
- image: circleci/golang:1.13-node-browsers
environment:
GOCACHE: /tmp/go-cache
GOFLAGS: "-mod=readonly -p=4" # Go on Circle thinks 32 CPUs are available, but there aren't.
working_directory: /go/src/github.com/influxdata/influxdb
steps:
- checkout
# Speed up `make build` by restoring caches from previous runs.
- restore_cache:
name: Restoring GOCACHE
keys:
- influxdb-gocache- # Just match the most recent Go cache.
- restore_cache:
name: Restoring GOPATH/pkg/mod
keys:
- influxdb-gomod-{{ checksum "go.sum" }} # Just match the go.sum checksum cache.
- run: sudo apt-get install -y netcat-openbsd
- run: sudo apt-get install -y bzr
- install_rust_compiler
- run: make protoc
- run: make build
- run:
command: ./bin/linux/influxd --store=memory --e2e-testing=true
background: true
- run: make e2e
- store_test_results:
path: ui/junit-results
destination: junit-results
- store_artifacts:
path: ui/cypress/videos
destination: videos
- store_artifacts:
path: ui/cypress/screenshots
destination: screenshots
selenium_accept:
docker:
- image: circleci/node:lts-stretch-browsers
- image: quay.io/influxdb/influx:nightly
command: [--e2e-testing=true]
steps:
- checkout
- run:
name: Environment check
command: |
git --version
node --version && npm --version
docker --version
google-chrome --version && which google-chrome && chromedriver --version && which chromedriver
timeout 300 bash -c 'while [[ "$(curl -s -o /dev/null -w ''%{http_code}'' localhost:9999)" != "200" ]]; do sleep 5; done' || false
- run:
name: Selenium tests
command: |
set +e
cd e2e
npm install
sed -i "s/\"headless\": false/\"headless\": true/g" e2e.conf.json
npm test; TEST_RESULT=$?
npm run report:html
npm run report:junit
mkdir -p ~/e2e/test-results/cucumber
mkdir -p ~/e2e/artifacts/html
cp ~/project/e2e/report/cucumber_report.html ~/e2e/artifacts/html/cucumber_report.html
cp ~/project/e2e/report/cucumber_junit.xml ~/e2e/test-results/cucumber/report.xml
cp ~/project/e2e/report/cucumber_junit.xml ~/e2e/artifacts/report.xml
cp -r ~/project/e2e/screenshots ~/e2e/artifacts
ls -al
exit $TEST_RESULT
- store_test_results:
path: ~/e2e/test-results
- store_artifacts:
path: ~/e2e/artifacts
jstest:
docker:
- image: circleci/golang:1.13-node-browsers
working_directory: /go/src/github.com/influxdata/influxdb
parallelism: 8
steps:
- checkout
- restore_cache:
keys:
- 'yarn-cached-packages-{{ checksum "ui/yarn.lock" }}'
name: "Restore Yarn Package Cache"
- run:
command: |
set +e
cd ui
yarn install
yarn prettier
name: "Install Dependencies"
- run: make ui_client
- run:
name: parallel jest tests
command: |
set +e
cd ui
TESTFILES=$(circleci tests glob "src/**/*.test.ts*" | circleci tests split --split-by=timings)
yarn test:circleci $TESTFILES
- store_test_results:
path: coverage
- store_artifacts:
path: coverage
destination: raw-test-output
- save_cache:
key: 'yarn-cached-packages-{{ checksum "ui/yarn.lock" }}'
name: "Save Yarn Package Cache"
paths:
- ~/.cache/yarn
jslint:
docker:
- image: circleci/golang:1.13-node-browsers
working_directory: /go/src/github.com/influxdata/influxdb
parallelism: 8
steps:
- checkout
- restore_cache:
keys:
- 'yarn-cached-packages-{{ checksum "ui/yarn.lock" }}'
name: "Restore Yarn Package Cache"
- run:
command: |
set +e
cd ui
yarn install
name: "Install Dependencies"
- run: make ui_client
- run:
name: parallel eslint
command: |
set +e
cd ui
TESTFILES=$(circleci tests glob "src/**/*.ts*" "cypress/**/*.ts*" | circleci tests split --split-by=filesize)
yarn eslint:circleci $TESTFILES
- save_cache:
key: 'yarn-cached-packages-{{ checksum "ui/yarn.lock" }}'
name: "Save Yarn Package Cache"
paths:
- ~/.cache/yarn
gotest:
docker:
- image: circleci/golang:1.13
resource_class: large
environment:
GOCACHE: /tmp/go-cache
GOFLAGS: "-mod=readonly -p=4" # Go on Circle thinks 32 CPUs are available, but there aren't.
TEST_RESULTS: /tmp/test-results
working_directory: /go/src/github.com/influxdata/influxdb
parallelism: 8
steps:
- checkout
# Populate GOCACHE.
- restore_cache:
name: Restoring GOCACHE
keys:
- influxdb-gocache-{{ .Branch }}-{{ .Revision }} # Matches when retrying a single run.
- influxdb-gocache-{{ .Branch }}- # Matches a new commit on an existing branch.
- influxdb-gocache- # Matches a new branch.
# Populate GOPATH/pkg.
- restore_cache:
name: Restoring GOPATH/pkg/mod
keys:
- influxdb-gomod-{{ checksum "go.sum" }} # Matches based on go.sum checksum.
- run: sudo apt-get install -y bzr
- install_rust_compiler
- run: mkdir -p $TEST_RESULTS
- run: make test-go # This uses the test cache so it may succeed or fail quickly.
- run:
name: parallel go race tests
command: |
set +e
# filter internal/promqltests because it has special go.mod
TESTFILES=$(go list ./... | grep -v internal/promqltests | circleci tests split --split-by=timings)
echo $TESTFILES
GOTRACEBACK=all GO111MODULE=on ./env gotestsum --format standard-quiet --junitfile /tmp/test-results/gotestsum.xml -- -race -count=1 $TESTFILES
- save_cache:
name: Saving GOCACHE
key: influxdb-gocache-{{ .Branch }}-{{ .Revision }}
paths:
- /tmp/go-cache
when: always
- save_cache:
name: Saving GOPATH/pkg/mod
key: influxdb-gomod-{{ checksum "go.sum" }}
paths:
- /go/pkg/mod
when: always
- store_artifacts: # Upload test summary for display in Artifacts: https://circleci.com/docs/2.0/artifacts/
path: /tmp/test-results
destination: raw-test-output
- store_test_results: # Upload test results for display in Test Summary: https://circleci.com/docs/2.0/collect-test-data/
path: /tmp/test-results
lint-feature-flags:
docker:
- image: circleci/golang:1.13
environment:
GOCACHE: /tmp/go-cache
GOFLAGS: "-mod=readonly -p=2" # Go on Circle thinks 32 CPUs are available, but there aren't.
working_directory: /go/src/github.com/influxdata/influxdb
steps:
- checkout
# Populate GOCACHE.
- restore_cache:
name: Restoring GOCACHE
keys:
- influxdb-gocache-{{ .Branch }}-{{ .Revision }} # Matches when retrying a single run.
- influxdb-gocache-{{ .Branch }}- # Matches a new commit on an existing branch.
- influxdb-gocache- # Matches a new branch.
# Populate GOPATH/pkg.
- restore_cache:
name: Restoring GOPATH/pkg/mod
keys:
- influxdb-gomod-{{ checksum "go.sum" }} # Matches based on go.sum checksum.
- run: ./scripts/ci/lint/flags.bash
- skip_if_not_master
- save_cache:
name: Saving GOCACHE
key: influxdb-gocache-{{ .Branch }}-{{ .Revision }}
paths:
- /tmp/go-cache
when: always
- save_cache:
name: Saving GOPATH/pkg/mod
key: influxdb-gomod-{{ checksum "go.sum" }}
paths:
- /go/pkg/mod
when: always
golint:
docker:
- image: circleci/golang:1.13
environment:
GOCACHE: /tmp/go-cache
GOFLAGS: "-mod=readonly -p=2" # Go on Circle thinks 32 CPUs are available, but there aren't.
TEST_RESULTS: /tmp/test-results
working_directory: /go/src/github.com/influxdata/influxdb
steps:
- checkout
# Populate GOCACHE.
- restore_cache:
name: Restoring GOCACHE
keys:
- influxdb-gocache-{{ .Branch }}-{{ .Revision }} # Matches when retrying a single run.
- influxdb-gocache-{{ .Branch }}- # Matches a new commit on an existing branch.
- influxdb-gocache- # Matches a new branch.
# Populate GOPATH/pkg.
- restore_cache:
name: Restoring GOPATH/pkg/mod
keys:
- influxdb-gomod-{{ checksum "go.sum" }} # Matches based on go.sum checksum.
- run: sudo apt-get install -y bzr
- install_rust_compiler
- run: mkdir -p $TEST_RESULTS
- run: |
# this is not in a seperate bash script because it isn't meant to be run on local.
# it just checks to make sure that the same major/minor version of go is used in the mod file as on ci
# to prevent accidentally checking in a wrong go mod version.
gomodversiondiff=$( go mod edit -go=$( go version | sed -n 's/^.*go\([0-9]*.[0-9]*\).*$/\1/p') -print |diff - go.mod )
if [ "$gomodversiondiff" ]
then
echo unexpected go version $gomodversiondiff
exit 1
fi
- run: make vet
- run: make checkfmt
- run: make checktidy
- run: GO111MODULE=on go mod vendor # staticcheck looks in vendor for dependencies.
- run: GO111MODULE=on go install honnef.co/go/tools/cmd/staticcheck # Install staticcheck from the version we specify in go.mod.
- run: GO111MODULE=on ./env staticcheck ./...
- save_cache:
name: Saving GOCACHE
key: influxdb-gocache-{{ .Branch }}-{{ .Revision }}
paths:
- /tmp/go-cache
when: always
- save_cache:
name: Saving GOPATH/pkg/mod
key: influxdb-gomod-{{ checksum "go.sum" }}
paths:
- /go/pkg/mod
when: always
- store_artifacts:
path: /tmp/test-results
destination: raw-test-output
- store_test_results:
path: /tmp/test-results
build:
docker:
- image: circleci/golang:1.13-node-browsers
environment:
GOCACHE: /tmp/go-cache
GOFLAGS: "-mod=readonly -p=4" # Go on Circle thinks 32 CPUs are available, but there aren't.
working_directory: /go/src/github.com/influxdata/influxdb
steps:
- checkout
- run: sudo apt-get install -y bzr
- install_rust_compiler
- run: make checkcommit
# Speed up `make build` by restoring caches from previous runs.
- restore_cache:
name: Restoring GOCACHE
keys:
- influxdb-gocache- # Just match the most recent Go cache.
- restore_cache:
name: Restoring GOPATH/pkg/mod
keys:
- influxdb-gomod-{{ checksum "go.sum" }} # Just match the go.sum checksum cache.
- run: make protoc
- run: make build
- persist_to_workspace:
root: .
paths:
- project
- bin/linux/influxd
- bin/linux/influx
- etc/litmus_success_notify.sh
- etc/litmus_fail_notify.sh
deploy-nightly:
docker:
- image: circleci/golang:1.13-node-browsers
environment:
GOCACHE: /tmp/go-cache
GOFLAGS: "-mod=readonly -p=4" # Go on Circle thinks 32 CPUs are available, but there aren't.
working_directory: /go/src/github.com/influxdata/influxdb
steps:
- checkout
# Speed up `make nightly` by restoring caches from previous runs.
- restore_cache:
name: Restoring GOCACHE
keys:
- influxdb-gocache- # Just match the most recent Go cache.
- restore_cache:
name: Restoring GOPATH/pkg/mod
keys:
- influxdb-gomod-{{ checksum "go.sum" }} # Just match the go.sum checksum cache.
- setup_remote_docker
- run:
name: "Docker Login"
command: docker login -u "$QUAY_USER" -p $QUAY_PASS quay.io
- run: sudo apt-get install -y bzr
- install_rust_compiler
- install_release_tools
- run: make protoc # installs protoc
- run:
name: "Build nightly"
command: make nightly
- persist_to_workspace:
root: .
paths:
- etc/litmus_success_notify.sh
- etc/litmus_fail_notify.sh
release:
docker:
- image: circleci/golang:1.13-node-browsers
environment:
GOCACHE: /tmp/go-cache
GOFLAGS: "-mod=readonly -p=4" # Go on Circle thinks 32 CPUs are available, but there aren't.
DOCKER_VERSION: 2.0.0-beta
working_directory: /go/src/github.com/influxdata/influxdb
steps:
- checkout
# Speed up `make nightly` by restoring caches from previous runs.
- restore_cache:
name: Restoring GOCACHE
keys:
- influxdb-gocache- # Just match the most recent Go cache.
- restore_cache:
name: Restoring GOPATH/pkg/mod
keys:
- influxdb-gomod-{{ checksum "go.sum" }} # Just match the go.sum checksum cache.
- setup_remote_docker
- run:
name: "Docker Login"
command: docker login -u "$QUAY_USER" -p $QUAY_PASS quay.io
- run: sudo apt-get install -y bzr
- install_rust_compiler
- install_release_tools
- run: make protoc # installs protoc
- run:
name: "Build release"
command: make release
workflows:
version: 2
build:
jobs:
- gotest
- golint
- jstest
- jslint
- build
- litmus_daily:
requires:
- build
filters:
branches:
only: /^(?!pull\/).*$/
- litmus_integration:
requires:
- litmus_daily
filters:
branches:
only:
- master
e2e:
jobs:
- e2e
hourly-e2e:
triggers:
- schedule:
cron: "0 * * * *"
filters:
branches:
only:
- master
jobs:
- e2e
nightly:
triggers:
- schedule:
cron: "0 5 * * *"
filters:
branches:
only:
- master
- feat/use-algo-w
jobs:
- gotest
- golint
- jstest
- jslint
- deploy-nightly:
requires:
- gotest
- golint
- jstest
- jslint
filters:
branches:
only:
- master
- feat/use-algo-w
- litmus_nightly:
requires:
- deploy-nightly
- selenium_accept:
requires:
- deploy-nightly
release:
jobs:
- gotest:
filters:
branches:
ignore: /.*/
tags:
only: /^v[0-9]+\.[0-9]+\.[0-9]+-(alpha|beta)\.[0-9]+$/
- golint:
filters:
branches:
ignore: /.*/
tags:
only: /^v[0-9]+\.[0-9]+\.[0-9]+-(alpha|beta)\.[0-9]+$/
- jstest:
filters:
branches:
ignore: /.*/
tags:
only: /^v[0-9]+\.[0-9]+\.[0-9]+-(alpha|beta)\.[0-9]+$/
- jslint:
filters:
branches:
ignore: /.*/
tags:
only: /^v[0-9]+\.[0-9]+\.[0-9]+-(alpha|beta)\.[0-9]+$/
- release:
requires:
- gotest
- golint
- jstest
- jslint
filters:
branches:
ignore: /.*/
tags:
only: /^v[0-9]+\.[0-9]+\.[0-9]+-(alpha|beta)\.[0-9]+$/