build: refactor lint and race-test jobs to use cross-builder executor (#22584)

.circleci/config.yml

@@ -9,7 +9,7 @@ executors:
       # NOTE: To upgrade the Go version, first push the upgrade to the cross-builder Dockerfile in the edge repo,
       # then update the version here to match. Until we finish the migration to using the cross-builder image,
       # you'll also need to update references to `cimg/go` and `GO_VERSION` in this file.
-      - image: quay.io/influxdb/cross-builder:go1.17.1-41d944d749b2c130119eff7efaaa9883ccffe5a7
+      - image: quay.io/influxdb/cross-builder:go1.17.1-f8379e935394b78ab2977471b55974d0d617d289
     resource_class: large
   linux-amd64:
     machine:
@@ -54,9 +54,9 @@ workflows:
       not: << pipeline.parameters.aws_teardown >>
     jobs:
       - godeps
-      - gotest:
+      - test-race:
-          requires:
+          # TODO: Remove this alias as part of https://github.com/influxdata/influxdb/issues/22623
-            - godeps
+          name: gotest
       - test-build:
           matrix:
             parameters:
@@ -92,9 +92,9 @@ workflows:
       - tlstest:
           requires:
             - godeps
-      - golint:
+      - lint:
-          requires:
+          # TODO: Remove this alias as part of https://github.com/influxdata/influxdb/issues/22623
-            - godeps
+          name: golint
       - build:
           requires:
             - godeps
@@ -180,9 +180,9 @@ workflows:
     jobs:
       - changelog
       - godeps
-      - gotest:
+      - test-race:
-          requires:
+          # TODO: Remove this alias as part of https://github.com/influxdata/influxdb/issues/22623
-            - godeps
+          name: gotest
       - test-build:
           matrix:
             parameters:
@@ -212,9 +212,9 @@ workflows:
           executor: windows
           requires:
             - test-build-amd64-windows
-      - golint:
+      - lint:
-          requires:
+          # TODO: Remove this alias as part of https://github.com/influxdata/influxdb/issues/22623
-            - godeps
+          name: golint
       - fluxtest:
           requires:
             - godeps
@@ -449,70 +449,50 @@ jobs:
           paths:
             - /home/circleci/go/pkg/mod
 
-  golint:
+  lint:
-    docker:
+    executor: cross-builder
-      - image: cimg/go:1.17.1
-    environment:
-      TMPDIR: /mnt/ramdisk
-    working_directory: /home/circleci/go/src/github.com/influxdata/influxdb
     steps:
       - checkout
-      - restore_cache:
+      - run:
-          keys:
+          name: Check go version
-            - influxdb-gomod-sum-{{ checksum "go.sum" }}
+          command: ./scripts/ci/check-system-go-matches-go-mod.sh
-      - run: |
+          when: always
-          # this is not in a seperate bash script because it isn't meant to be run on local.
-          # it just checks to make sure that the same major/minor version of go is used in the mod file as on ci
-          # to prevent accidentally checking in a wrong go mod version.
-          gomodversiondiff=$( go mod edit -go=$( go version | sed -n 's/^.*go\([0-9]*.[0-9]*\).*$/\1/p') -print |diff - go.mod )
-          if [ "$gomodversiondiff" ]
-          then
-            echo unexpected go version $gomodversiondiff
-            exit 1
-          fi
-      - install_core_deps
       - run:
           name: Check flag generation
           command: ./scripts/ci/lint/flags.bash
-      - run: make checkgenerate
+          when: always
       - run:
-          name: Check generation when vendoring deps
+          name: Check formatting
+          command: make checkfmt
+          when: always
+      - run:
+          name: Check codegen
+          command: make checkgenerate
+          when: always
+      - run:
+          name: vet
+          command: make vet
+          when: always
+      - run:
+          name: staticcheck
           command: |
-            go mod vendor
+            go install honnef.co/go/tools/cmd/staticcheck
-            make checkgenerate
+            staticcheck ./...
-            rm -rf vendor
+          when: always
-      - run: make vet
-      - run: make checkfmt
-      - run: go install honnef.co/go/tools/cmd/staticcheck
-      - run: staticcheck ./...
 
-  gotest:
+  test-race:
-    docker:
+    executor: cross-builder
-      - image: cimg/go:1.17.1
-    resource_class: large
-    environment:
-      TMPDIR: /mnt/ramdisk
-    working_directory: /home/circleci/go/src/github.com/influxdata/influxdb
     parallelism: 8
     steps:
       - checkout
-      - restore_cache:
-          name: Restore GOPATH/pkg/mod
-          keys:
-            - influxdb-gomod-sum-{{ checksum "go.sum" }}
-      - run: mkdir -p /tmp/test-results
-      - install_core_deps
       - run:
-          name: run parallel race tests
+          name: Run race tests
-          command: |
+          command: ./scripts/ci/run-race-tests.sh $(pwd)/test-results
-            GO_TEST_CMD="gotestsum --format standard-quiet --junitfile /tmp/test-results/gotestsum.xml -- -p=4"
-            TESTFILES=($(go list ./... | circleci tests split --split-by=timings))
-            make GO_TEST_CMD="$GO_TEST_CMD" GO_TEST_PATHS="${TESTFILES[*]}" test-go-race
-      - store_artifacts:
-          path: /tmp/test-results
-          destination: raw-test-output
       - store_test_results:
-          path: /tmp/test-results
+          path: ./test-results
+      - store_artifacts:
+          path: ./test-results
+          destination: raw-test-results
 
   test-build:
     executor: cross-builder

scripts/ci/build-tests.sh

@@ -1,30 +1,6 @@
 #!/usr/bin/env bash
 set -exo pipefail
 
-declare -r GOTESTSUM_VERSION=1.7.0
-
-# Values are from https://github.com/gotestyourself/gotestsum/releases/download/v${VERSION}/gotestsum-${VERSION}-checksums.txt
-function gotestsum_expected_sha () {
-    case "$(go env GOOS)_$(go env GOARCH)" in
-        linux_amd64)
-            echo b5c98cc408c75e76a097354d9487dca114996e821b3af29a0442aa6c9159bd40
-            ;;
-        linux_arm64)
-            echo ee57c91abadc464a7cd9f8abbffe94a673aab7deeb9af8c17b96de4bb8f37e41
-            ;;
-        darwin_amd64)
-            echo a8e2351604882af1a67601cbeeacdcfa9b17fc2f6fbac291cf5d434efdf2d85b
-            ;;
-        windows_amd64)
-            echo 7ae12ddb171375f0c14d6a09dd27a5c1d1fc72edeea674e3d6e7489a533b40c1
-            ;;
-        *)
-            >&2 echo Error: Unsupported OS/arch pair for gotestsum: "$(go env GOOS)_$(go env GOARCH)"
-            exit 1
-            ;;
-    esac
-}
-
 function build_linux () {
     local tags=osusergo,netgo,sqlite_foreign_keys,sqlite_json,static_build
     local cc
@@ -58,14 +34,12 @@ function build_windows () {
 }
 
 function build_test_tools () {
-    # Download gotestsum from its releases (faster than building it).
+    # Copy pre-built gotestsum out of the cross-builder.
-    local -r gotestsum_archive="gotestsum_${GOTESTSUM_VERSION}_$(go env GOOS)_$(go env GOARCH).tar.gz"
+    local ext=""
-    local -r gotestsum_url="https://github.com/gotestyourself/gotestsum/releases/download/v${GOTESTSUM_VERSION}/${gotestsum_archive}"
+    if [ "$(go env GOOS)" = windows ]; then
-
+        ext=".exe"
-    curl -L "$gotestsum_url" -O
+    fi
-    echo "$(gotestsum_expected_sha)  ${gotestsum_archive}" | sha256sum --check --
+    cp "/usr/local/bin/gotestsum_$(go env GOOS)_$(go env GOARCH)${ext}" "$1/gotestsum${ext}"
-    tar xzf "$gotestsum_archive" -C "${1}/"
-    rm "$gotestsum_archive"
 
     # Build test2json from the installed Go distribution.
     CGO_ENABLED=0 go build -o "${1}/" -ldflags="-s -w" cmd/test2json

scripts/ci/check-system-go-matches-go-mod.sh

@@ -0,0 +1,15 @@
+#!/usr/bin/env bash
+set -eo pipefail
+
+# Check that the same major/minor version of go is used in the mod file as on CI to prevent
+# the two from accidentally getting out-of-sync.
+function main () {
+    local -r version_diff=$(go mod edit -go=$(go version | sed -n 's/^.*go\([0-9]*.[0-9]*\).*$/\1/p') -print | diff - go.mod)
+    if [ -n "$version_diff" ]; then
+        >&2 echo Error: unexpected difference in go version:
+        >&2 echo "$version_diff"
+        exit 1
+    fi
+}
+
+main

scripts/ci/run-race-tests.sh

@@ -0,0 +1,25 @@
+#!/usr/bin/env bash
+set -exo pipefail
+
+function main () {
+    if [[ $# != 1 ]]; then
+        >&2 echo Usage: $0 '<output-dir>'
+        exit 1
+    fi
+    if [[ $(go env GOOS) != linux || $(go env GOARCH) != amd64 ]]; then
+        >&2 echo Race tests only supported on linux/amd64
+        exit 1
+    fi
+
+    local -r out_dir="$1"
+    mkdir -p "$out_dir"
+
+    # Get list of packages to test on this node according to Circle's timings.
+    local -r test_packages="$(go list ./... | circleci tests split --split-by=timings --timings-type=classname)"
+
+    # Run tests
+    local -r tags=osuergo,netgo,sqlite_foreign_keys,sqlite_json
+    gotestsum --junitfile "${out_dir}/report.xml" -- -tags "$tags" -race ${test_packages[@]}
+}
+
+main ${@}