build: refactor lint and race-test jobs to use cross-builder executor (#22584)

2021-10-05 16:27:00 -04:00 · 2021-10-05 16:27:00 -04:00 · a4ee58e451
parent 7c19225bed
commit a4ee58e451
4 changed files with 90 additions and 96 deletions
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@ -9,7 +9,7 @@ executors:
      # NOTE: To upgrade the Go version, first push the upgrade to the cross-builder Dockerfile in the edge repo,
      # then update the version here to match. Until we finish the migration to using the cross-builder image,
      # you'll also need to update references to `cimg/go` and `GO_VERSION` in this file.
-      - image: quay.io/influxdb/cross-builder:go1.17.1-41d944d749b2c130119eff7efaaa9883ccffe5a7
+      - image: quay.io/influxdb/cross-builder:go1.17.1-f8379e935394b78ab2977471b55974d0d617d289
    resource_class: large
  linux-amd64:
    machine:
@ -54,9 +54,9 @@ workflows:
      not: << pipeline.parameters.aws_teardown >>
    jobs:
      - godeps
-      - gotest:
-          requires:
-            - godeps
+      - test-race:
+          # TODO: Remove this alias as part of https://github.com/influxdata/influxdb/issues/22623
+          name: gotest
      - test-build:
          matrix:
            parameters:
@ -92,9 +92,9 @@ workflows:
      - tlstest:
          requires:
            - godeps
-      - golint:
-          requires:
-            - godeps
+      - lint:
+          # TODO: Remove this alias as part of https://github.com/influxdata/influxdb/issues/22623
+          name: golint
      - build:
          requires:
            - godeps
@ -180,9 +180,9 @@ workflows:
    jobs:
      - changelog
      - godeps
-      - gotest:
-          requires:
-            - godeps
+      - test-race:
+          # TODO: Remove this alias as part of https://github.com/influxdata/influxdb/issues/22623
+          name: gotest
      - test-build:
          matrix:
            parameters:
@ -212,9 +212,9 @@ workflows:
          executor: windows
          requires:
            - test-build-amd64-windows
-      - golint:
-          requires:
-            - godeps
+      - lint:
+          # TODO: Remove this alias as part of https://github.com/influxdata/influxdb/issues/22623
+          name: golint
      - fluxtest:
          requires:
            - godeps
@ -449,70 +449,50 @@ jobs:
          paths:
            - /home/circleci/go/pkg/mod

-  golint:
-    docker:
-      - image: cimg/go:1.17.1
-    environment:
-      TMPDIR: /mnt/ramdisk
-    working_directory: /home/circleci/go/src/github.com/influxdata/influxdb
+  lint:
+    executor: cross-builder
    steps:
      - checkout
-      - restore_cache:
-          keys:
-            - influxdb-gomod-sum-{{ checksum "go.sum" }}
-      - run: |
-          # this is not in a seperate bash script because it isn't meant to be run on local.
-          # it just checks to make sure that the same major/minor version of go is used in the mod file as on ci
-          # to prevent accidentally checking in a wrong go mod version.
-          gomodversiondiff=$( go mod edit -go=$( go version | sed -n 's/^.*go\([0-9]*.[0-9]*\).*$/\1/p') -print |diff - go.mod )
-          if [ "$gomodversiondiff" ]
-          then
-            echo unexpected go version $gomodversiondiff
-            exit 1
-          fi
-      - install_core_deps
+      - run:
+          name: Check go version
+          command: ./scripts/ci/check-system-go-matches-go-mod.sh
+          when: always
      - run:
          name: Check flag generation
          command: ./scripts/ci/lint/flags.bash
-      - run: make checkgenerate
+          when: always
      - run:
-          name: Check generation when vendoring deps
+          name: Check formatting
+          command: make checkfmt
+          when: always
+      - run:
+          name: Check codegen
+          command: make checkgenerate
+          when: always
+      - run:
+          name: vet
+          command: make vet
+          when: always
+      - run:
+          name: staticcheck
          command: |
-            go mod vendor
-            make checkgenerate
-            rm -rf vendor
-      - run: make vet
-      - run: make checkfmt
-      - run: go install honnef.co/go/tools/cmd/staticcheck
-      - run: staticcheck ./...
+            go install honnef.co/go/tools/cmd/staticcheck
+            staticcheck ./...
+          when: always

-  gotest:
-    docker:
-      - image: cimg/go:1.17.1
-    resource_class: large
-    environment:
-      TMPDIR: /mnt/ramdisk
-    working_directory: /home/circleci/go/src/github.com/influxdata/influxdb
+  test-race:
+    executor: cross-builder
    parallelism: 8
    steps:
      - checkout
-      - restore_cache:
-          name: Restore GOPATH/pkg/mod
-          keys:
-            - influxdb-gomod-sum-{{ checksum "go.sum" }}
-      - run: mkdir -p /tmp/test-results
-      - install_core_deps
      - run:
-          name: run parallel race tests
-          command: |
-            GO_TEST_CMD="gotestsum --format standard-quiet --junitfile /tmp/test-results/gotestsum.xml -- -p=4"
-            TESTFILES=($(go list ./... | circleci tests split --split-by=timings))
-            make GO_TEST_CMD="$GO_TEST_CMD" GO_TEST_PATHS="${TESTFILES[*]}" test-go-race
-      - store_artifacts:
-          path: /tmp/test-results
-          destination: raw-test-output
+          name: Run race tests
+          command: ./scripts/ci/run-race-tests.sh $(pwd)/test-results
      - store_test_results:
-          path: /tmp/test-results
+          path: ./test-results
+      - store_artifacts:
+          path: ./test-results
+          destination: raw-test-results

  test-build:
    executor: cross-builder
--- a/scripts/ci/build-tests.sh
+++ b/scripts/ci/build-tests.sh
@ -1,30 +1,6 @@
 #!/usr/bin/env bash
 set -exo pipefail

-declare -r GOTESTSUM_VERSION=1.7.0
-
-# Values are from https://github.com/gotestyourself/gotestsum/releases/download/v${VERSION}/gotestsum-${VERSION}-checksums.txt
-function gotestsum_expected_sha () {
-    case "$(go env GOOS)_$(go env GOARCH)" in
-        linux_amd64)
-            echo b5c98cc408c75e76a097354d9487dca114996e821b3af29a0442aa6c9159bd40
-            ;;
-        linux_arm64)
-            echo ee57c91abadc464a7cd9f8abbffe94a673aab7deeb9af8c17b96de4bb8f37e41
-            ;;
-        darwin_amd64)
-            echo a8e2351604882af1a67601cbeeacdcfa9b17fc2f6fbac291cf5d434efdf2d85b
-            ;;
-        windows_amd64)
-            echo 7ae12ddb171375f0c14d6a09dd27a5c1d1fc72edeea674e3d6e7489a533b40c1
-            ;;
-        *)
-            >&2 echo Error: Unsupported OS/arch pair for gotestsum: "$(go env GOOS)_$(go env GOARCH)"
-            exit 1
-            ;;
-    esac
-}
-
 function build_linux () {
    local tags=osusergo,netgo,sqlite_foreign_keys,sqlite_json,static_build
    local cc
@ -58,14 +34,12 @@ function build_windows () {
 }

 function build_test_tools () {
-    # Download gotestsum from its releases (faster than building it).
-    local -r gotestsum_archive="gotestsum_${GOTESTSUM_VERSION}_$(go env GOOS)_$(go env GOARCH).tar.gz"
-    local -r gotestsum_url="https://github.com/gotestyourself/gotestsum/releases/download/v${GOTESTSUM_VERSION}/${gotestsum_archive}"
-
-    curl -L "$gotestsum_url" -O
-    echo "$(gotestsum_expected_sha)  ${gotestsum_archive}" | sha256sum --check --
-    tar xzf "$gotestsum_archive" -C "${1}/"
-    rm "$gotestsum_archive"
+    # Copy pre-built gotestsum out of the cross-builder.
+    local ext=""
+    if [ "$(go env GOOS)" = windows ]; then
+        ext=".exe"
+    fi
+    cp "/usr/local/bin/gotestsum_$(go env GOOS)_$(go env GOARCH)${ext}" "$1/gotestsum${ext}"

    # Build test2json from the installed Go distribution.
    CGO_ENABLED=0 go build -o "${1}/" -ldflags="-s -w" cmd/test2json
--- a/scripts/ci/check-system-go-matches-go-mod.sh
+++ b/scripts/ci/check-system-go-matches-go-mod.sh
@ -0,0 +1,15 @@
+#!/usr/bin/env bash
+set -eo pipefail
+
+# Check that the same major/minor version of go is used in the mod file as on CI to prevent
+# the two from accidentally getting out-of-sync.
+function main () {
+    local -r version_diff=$(go mod edit -go=$(go version | sed -n 's/^.*go\([0-9]*.[0-9]*\).*$/\1/p') -print | diff - go.mod)
+    if [ -n "$version_diff" ]; then
+        >&2 echo Error: unexpected difference in go version:
+        >&2 echo "$version_diff"
+        exit 1
+    fi
+}
+
+main
--- a/scripts/ci/run-race-tests.sh
+++ b/scripts/ci/run-race-tests.sh
@ -0,0 +1,25 @@
+#!/usr/bin/env bash
+set -exo pipefail
+
+function main () {
+    if [[ $# != 1 ]]; then
+        >&2 echo Usage: $0 '<output-dir>'
+        exit 1
+    fi
+    if [[ $(go env GOOS) != linux || $(go env GOARCH) != amd64 ]]; then
+        >&2 echo Race tests only supported on linux/amd64
+        exit 1
+    fi
+
+    local -r out_dir="$1"
+    mkdir -p "$out_dir"
+
+    # Get list of packages to test on this node according to Circle's timings.
+    local -r test_packages="$(go list ./... | circleci tests split --split-by=timings --timings-type=classname)"
+
+    # Run tests
+    local -r tags=osuergo,netgo,sqlite_foreign_keys,sqlite_json
+    gotestsum --junitfile "${out_dir}/report.xml" -- -tags "$tags" -race ${test_packages[@]}
+}
+
+main ${@}