chore: fix release build scripts (#20970)

Note: this commit fixes linux/amd64 and darwin/amd64 builds. We are still
working on fixes for linux/arm64 and windows/amd64.

Partial fix of https://github.com/influxdata/influxdb/issues/20854

releng/README.md

@@ -20,7 +20,6 @@ The `fs` folder is overlaid on the Docker image so that is clear where each scri
 Those scripts make assumptions about the environment which are controlled in the outer scripts (i.e. `build.bash`), so the scripts not intended to be run outside of Docker.
 
 By default, these scripts will use the "current" Go version as determined by `_go_versions.sh`.
-To use the "next" version of Go, set the environment variable GO_NEXT to a non-empty value.
 
 ## source-tarball
 

releng/_go_versions.sh

@@ -1,5 +1,3 @@
-# These are the current and "next" Go versions used to build influxdb.
 # This file is meant to be sourced from other scripts.
 
 export GO_CURRENT_VERSION=1.13.8
-export GO_NEXT_VERSION=1.13.8

releng/packages/fs/usr/local/bin/influxdb_packages.bash

@@ -35,9 +35,10 @@ if [ -z "$OS" ] || [ -z "$ARCH" ]; then
   exit 1
 fi
 
-mkdir -p /go
+WORK=/influxdata
-tar x -C /go -zf /influxdb-src.tar.gz
+mkdir -p ${WORK}
-ln -s /go/src/github.com/influxdata/influxdb /isrc # Shorthand for influxdb source.
+tar x -C ${WORK} -zf /influxdb-src.tar.gz
+ln -s ${WORK}/influxdb /isrc # Shorthand for influxdb source.
 SHA=$(jq -r .sha < "/isrc/.metadata.json")
 VERSION=$(jq -r .version < "/isrc/.metadata.json")
 ARCHIVE_ROOT_NAME="influxdb-${VERSION}-1"

releng/raw-binaries/Dockerfile

@@ -1,10 +1,54 @@
 ARG GO_VERSION
 FROM golang:${GO_VERSION}
 
+# Install system dependencies + cross build system dependencies
 RUN apt-get update && apt-get install -y --no-install-recommends \
+    asciidoc \
+    bzr \
+    clang \
+    cmake \
     jq \
+    libprotobuf-dev \
+    libssl-dev \
+    libxml2-dev \
+    llvm-dev \
+    lzma-dev \
+    patch \
+    protobuf-compiler \
+    xmlto \
+    xz-utils \
+    zlib1g-dev \
   && rm -rf /var/lib/apt/lists/*
 
+# Install rust
+RUN curl https://sh.rustup.rs -sSf | sh -s -- --default-toolchain stable -y \
+ && . $HOME/.cargo/env && rustup target add x86_64-apple-darwin x86_64-unknown-linux-musl
+
+ENV PATH="/root/.cargo/bin:${PATH}"
+
+# Install osxcross
+# The default osxcross deployment target is 10.6, which doesn't support thread local variables.
+ENV MACOSX_DEPLOYMENT_TARGET=10.11
+# The `git checkout` line specifies the same commit of osxcross that flux CI runs against.
+# It doesn't necessarily have to match but it would be good to keep things similar.
+RUN mkdir -p /opt/osxcross && \
+    cd /opt && \
+    git clone https://github.com/tpoechtrager/osxcross.git && \
+    cd osxcross && \
+    git checkout c2ad5e859d12a295c3f686a15bd7181a165bfa82 && \
+    curl -L -o ./tarballs/MacOSX${MACOSX_DEPLOYMENT_TARGET}.sdk.tar.xz \
+        https://macos-sdks.s3.amazonaws.com/MacOSX${MACOSX_DEPLOYMENT_TARGET}.sdk.tar.xz && \
+    UNATTENDED=1 PORTABLE=true OCDEBUG=1 ./build.sh && \
+    rm -rf .git build tarballs
+
+ENV PATH="/opt/osxcross/target/bin:${PATH}"
+
+# Wrapper to pull a version of pkg-config at runtime
+ENV PKG_CONFIG=/usr/local/bin/pkg-config.sh
+
+# install go-junit-report for unit tests
+RUN GO111MODULE=on go get github.com/jstemmer/go-junit-report@v0.9.1
+
 COPY fs/ /
 
 ENTRYPOINT ["influxdb_raw_binaries.bash"]

releng/raw-binaries/build.bash

@@ -4,9 +4,6 @@ function printHelp() {
   >&2 echo "USAGE: $0 -i PATH_TO_SOURCE_TARBALL -o OUTDIR
 
 Emits an archive of influxdb binaries based on the current environment's GOOS and GOARCH.
-Respects CGO_ENABLED.
-
-If the environment variable GO_NEXT is not empty, builds the binaries with the 'next' version of Go.
 "
 }
 
@@ -26,13 +23,15 @@ source "$SRCDIR/../_go_versions.sh"
 OUTDIR=""
 TARBALL=""
 RACE_FLAG=""
+STATIC_FLAG=""
 
-while getopts hi:o:r arg; do
+while getopts hi:o:rs arg; do
   case "$arg" in
     h) printHelp; exit 1;;
     i) TARBALL="$OPTARG";;
     o) OUTDIR="$OPTARG";;
     r) RACE_FLAG="-r";;
+    s) STATIC_FLAG="-s";;
   esac
 done
 
@@ -41,13 +40,8 @@ if [ -z "$OUTDIR" ] || [ -z "$TARBALL" ]; then
   exit 1
 fi
 
-if [ -z "$GO_NEXT" ]; then
+DOCKER_TAG=latest
-  DOCKER_TAG=latest
+GO_VERSION="$GO_CURRENT_VERSION"
-  GO_VERSION="$GO_CURRENT_VERSION"
-else
-  DOCKER_TAG=next
-  GO_VERSION="$GO_NEXT_VERSION"
-fi
 docker build --build-arg "GO_VERSION=$GO_VERSION" -t influxdata/influxdb/releng/raw-binaries:"$DOCKER_TAG" "$SRCDIR"
 
 mkdir -p "$OUTDIR"
@@ -55,5 +49,5 @@ mkdir -p "$OUTDIR"
 docker run --rm \
    --mount type=bind,source="${OUTDIR}",destination=/out \
    --mount type=bind,source="${TARBALL}",destination=/influxdb-src.tar.gz,ro=1 \
-   -e GOOS -e GOARCH -e CGO_ENABLED \
+   -e GOOS -e GOARCH \
-  influxdata/influxdb/releng/raw-binaries:"$DOCKER_TAG" $RACE_FLAG
+  influxdata/influxdb/releng/raw-binaries:"$DOCKER_TAG" $RACE_FLAG $STATIC_FLAG

releng/raw-binaries/fs/root/.cargo/config

@@ -0,0 +1,6 @@
+# Eventually rust might support cross compilation out of the box, but until then
+# we have to tell it which external linker to use:
+# https://rust-lang.github.io/rustup/cross-compilation.html
+[target.x86_64-apple-darwin]
+linker = "x86_64-apple-darwin15-clang"
+ar = "x86_64-apple-darwin15-ar"

releng/raw-binaries/fs/usr/local/bin/influxdb_raw_binaries.bash

@@ -1,5 +1,7 @@
 #!/bin/bash
 
+set -e
+
 function printHelp() {
   >&2 echo "USAGE: $0 [-r]
 
@@ -15,29 +17,62 @@ To build with race detection enabled, pass the -r flag.
 }
 
 RACE_FLAG=""
+STATIC=""
 
-while getopts hr arg; do
+while getopts w:hrs arg; do
   case "$arg" in
     h) printHelp; exit 1;;
     r) RACE_FLAG="-race";;
+    s) STATIC=1;;
   esac
 done
 
-
 if [ -z "$GOOS" ] || [ -z "$GOARCH" ]; then
   >&2 echo 'The environment variables $GOOS and $GOARCH must both be set.'
   exit 1
 fi
 
+if [[ "$GOOS" == darwin ]] ; then
+  # Control the compiler for go. Rust linker is set in $HOME/.cargo/config
+  export CC=x86_64-apple-darwin15-clang
+fi
 
-# Extract tarball into GOPATH.
+WORKSPACE=/influxdata
-tar xz -C "$GOPATH" -f /influxdb-src.tar.gz
 
-SHA=$(jq -r .sha < "$GOPATH/src/github.com/influxdata/influxdb/.metadata.json")
+mkdir -p ${WORKSPACE}
 
+echo "Extracting influxdb tarball"
+# Extract tarball into WORKSPACE.
+
+tar -vxz -C ${WORKSPACE} -f /influxdb-src.tar.gz
+
+SHA=$(jq -r .sha < "${WORKSPACE}/influxdb/.metadata.json")
+
+OUTDIR=$(mktemp -d)
+(
+	cd ${WORKSPACE}/influxdb
+
+	BINARY_PACKAGES="
+		github.com/influxdata/influxdb/cmd/influxd
+		github.com/influxdata/influxdb/cmd/influx
+		github.com/influxdata/influxdb/cmd/influx_inspect
+		github.com/influxdata/influxdb/cmd/influx_tsm"
+
+	for cmd in $BINARY_PACKAGES; do
+		export CGO_ENABLED=1
+		echo "env for go build: GOOS=$GOOS GOARCH=$GOARCH CGO_ENABLED=$CGO_ENABLED"
+		if [[ -n "$STATIC" ]] ; then
+			echo go build -i -o "$OUTDIR/$(basename $cmd)" -tags "netgo osusergo static_build" $cmd
+			go build -i -o "$OUTDIR/$(basename $cmd)" -tags "netgo osusergo static_build" $cmd
+		else
+			echo go build $RACE_FLAG -i -o "$OUTDIR/$(basename $cmd)" $cmd
+			go build $RACE_FLAG -i -o "$OUTDIR/$(basename $cmd)" $cmd
+		fi
+	done
+)
 
 SUFFIX=
-if [ "$CGO_ENABLED" == "0" ]; then
+if [[ -n "$STATIC" ]]; then
   # Only add the static suffix to the filename when explicitly requested.
   SUFFIX=_static
 elif [ -n "$RACE_FLAG" ]; then
@@ -46,36 +81,6 @@ elif [ -n "$RACE_FLAG" ]; then
 fi
 
 TARBALL_NAME="influxdb_bin_${GOOS}_${GOARCH}${SUFFIX}-${SHA}.tar.gz"
-
+TARBALL_PATH="/out/${TARBALL_NAME}"
-# note: according to https://github.com/golang/go/wiki/GoArm
+echo tar -C ${OUTDIR} -cvzf ${TARBALL_PATH} .
-# we want to support armel using GOARM=5
+tar -C ${OUTDIR} -cvzf ${TARBALL_PATH} .
-# and we want to support armhf using GOARM=6
-# no GOARM setting is necessary for arm64
-if [ $GOARCH == "armel" ]; then
-  GOARCH=arm
-  GOARM=5
-fi
-
-if [ $GOARCH == "armhf" ]; then
-  GOARCH=arm
-  GOARM=6
-fi
-
-
-
-OUTDIR=$(mktemp -d)
-for cmd in \
-  influxdb/cmd/influxd \
-  influxdb/cmd/influx \
-  influxdb/cmd/influx_inspect \
-  influxdb/cmd/influx_tsm \
-  ; do
-    # Build all the binaries into $OUTDIR.
-    # Windows binaries will get the .exe suffix as expected.
-    (cd "$OUTDIR" && go build $RACE_FLAG -i "github.com/influxdata/$cmd")
-done
-
-
-(cd "$OUTDIR" && tar czf "/out/$TARBALL_NAME" ./*)
-(cd /out && md5sum "$TARBALL_NAME" > "$TARBALL_NAME.md5")
-(cd /out && sha256sum "$TARBALL_NAME" > "$TARBALL_NAME.sha256")

releng/raw-binaries/fs/usr/local/bin/pkg-config.sh

@@ -0,0 +1,14 @@
+#!/usr/bin/env bash
+
+tmpdir=$(mktemp -d)
+trap "{ rm -rf ${tmpdir}; }" EXIT
+
+# "go build" can be noisy, and when Go invokes pkg-config (by calling this script) it will merge stdout and stderr.
+# Discard any output unless "go build" terminates with an error.
+CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -o ${tmpdir}/pkg-config github.com/influxdata/pkg-config &> ${tmpdir}/go_build_output
+if [ "$?" -ne 0 ]; then
+    cat ${tmpdir}/go_build_output 1>&2
+    exit 1
+fi
+
+${tmpdir}/pkg-config "$@"

releng/source-tarball/Dockerfile

@@ -1,19 +1,53 @@
 ARG GO_VERSION
-FROM golang:${GO_VERSION}-alpine
+FROM golang:${GO_VERSION}
 
-RUN apk add --no-cache \
+# Install system dependencies + cross build system dependencies
+RUN apt-get update && apt-get install -y --no-install-recommends \
     asciidoc \
-      bash \
+    bzr \
-      git \
+    clang \
-      openssh-client \
+    cmake \
-      make \
+    jq \
-      tar \
+    libprotobuf-dev \
-      xmlto
+    libssl-dev \
+    libxml2-dev \
+    llvm-dev \
+    lzma-dev \
+    patch \
+    protobuf-compiler \
+    xmlto \
+    xz-utils \
+    zlib1g-dev \
+  && rm -rf /var/lib/apt/lists/*
 
-# Build the dep binary and then clean out /go.
+# Install rust
-RUN go get github.com/golang/dep/cmd/dep && \
+RUN curl https://sh.rustup.rs -sSf | sh -s -- --default-toolchain stable -y \
-      mv /go/bin/dep /usr/local/bin/dep && \
+ && . $HOME/.cargo/env && rustup target add x86_64-apple-darwin x86_64-unknown-linux-musl
-      rm -rf /go/*
+
+ENV PATH="/root/.cargo/bin:${PATH}"
+
+# Install osxcross
+# The default osxcross deployment target is 10.6, which doesn't support thread local variables.
+ENV MACOSX_DEPLOYMENT_TARGET=10.11
+# The `git checkout` line specifies the same commit of osxcross that flux CI runs against.
+# It doesn't necessarily have to match but it would be good to keep things similar.
+RUN mkdir -p /opt/osxcross && \
+    cd /opt && \
+    git clone https://github.com/tpoechtrager/osxcross.git && \
+    cd osxcross && \
+    git checkout c2ad5e859d12a295c3f686a15bd7181a165bfa82 && \
+    curl -L -o ./tarballs/MacOSX${MACOSX_DEPLOYMENT_TARGET}.sdk.tar.xz \
+        https://macos-sdks.s3.amazonaws.com/MacOSX${MACOSX_DEPLOYMENT_TARGET}.sdk.tar.xz && \
+    UNATTENDED=1 PORTABLE=true OCDEBUG=1 ./build.sh && \
+    rm -rf .git build tarballs
+
+ENV PATH="/opt/osxcross/target/bin:${PATH}"
+
+# Wrapper to pull a version of pkg-config at runtime
+ENV PKG_CONFIG=/usr/local/bin/pkg-config.sh
+
+# install go-junit-report for unit tests
+RUN GO111MODULE=on go get github.com/jstemmer/go-junit-report@v0.9.1
 
 COPY fs/ /
 

releng/source-tarball/fs/root/.cargo/config

@@ -0,0 +1,6 @@
+# Eventually rust might support cross compilation out of the box, but until then
+# we have to tell it which external linker to use:
+# https://rust-lang.github.io/rustup/cross-compilation.html
+[target.x86_64-apple-darwin]
+linker = "x86_64-apple-darwin15-clang"
+ar = "x86_64-apple-darwin15-ar"

releng/source-tarball/fs/usr/local/bin/influxdb_tarball.bash

@@ -36,19 +36,19 @@ if [ -z "$SHA" ] || [ -z "$BRANCH" ] || [ -z "$VERSION" ]; then
   exit 1
 fi
 
-IPATH=/go/src/github.com/influxdata
+IPATH=/influxdb-source
 mkdir -p "$IPATH" && cd "$IPATH"
 if [ -d /influxdb-git ]; then
   git clone /influxdb-git "$IPATH/influxdb"
 else
-
+  echo "Influxdb .git directory required" >&2
-  git clone https://github.com/influxdata/influxdb.git
+  exit 1
 fi
 
-cd influxdb
+(
-git checkout "$SHA"
+	cd influxdb
-go mod vendor
+	git checkout "$SHA"
-cd ..
+)
 
 # Emit version metadata to appropriate files.
 
@@ -71,20 +71,19 @@ func init() {
 	commit = "%s"
 }' "$VERSION" "$BRANCH" "$SHA" > "./influxdb/cmd/influxd/version.generated.go"
 
-
 # influx uses just version.
 printf 'package main
 
 // Code generated by influxdata/releng tooling. DO NOT EDIT.
 
-	func init() {
+func init() {
 	version = "%s"
 }' "$VERSION" > "./influxdb/cmd/influx/version.generated.go"
 
 # Prebuild the man pages so that consumers of the source tarball don't have to build it themselves.
-(cd /go/src/github.com/influxdata/influxdb/man && make build && gzip -9 ./*.1)
+(cd "$IPATH"/influxdb/man && make build && gzip -9 ./*.1)
 
 TARBALL_NAME="influxdb-src-$SHA.tar.gz"
-(cd /go && tar czf "/out/$TARBALL_NAME" --exclude-vcs ./*) # --exclude-vcs is a GNU tar option.
+tar -vC ${IPATH} -czf "/out/${TARBALL_NAME}" --exclude-vcs ./* # --exclude-vcs is a GNU tar option.
 (cd /out && md5sum "$TARBALL_NAME" > "$TARBALL_NAME.md5")
 (cd /out && sha256sum "$TARBALL_NAME" > "$TARBALL_NAME.sha256")

releng/source-tarball/fs/usr/local/bin/pkg-config.sh

@@ -0,0 +1,14 @@
+#!/usr/bin/env bash
+
+tmpdir=$(mktemp -d)
+trap "{ rm -rf ${tmpdir}; }" EXIT
+
+# "go build" can be noisy, and when Go invokes pkg-config (by calling this script) it will merge stdout and stderr.
+# Discard any output unless "go build" terminates with an error.
+CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -o ${tmpdir}/pkg-config github.com/influxdata/pkg-config &> ${tmpdir}/go_build_output
+if [ "$?" -ne 0 ]; then
+    cat ${tmpdir}/go_build_output 1>&2
+    exit 1
+fi
+
+${tmpdir}/pkg-config "$@"

releng/unit-tests/Dockerfile

@@ -1,14 +1,53 @@
 ARG GO_VERSION
-FROM golang:${GO_VERSION}-alpine
+FROM golang:${GO_VERSION}
 
-RUN apk add --no-cache \
+# Install system dependencies + cross build system dependencies
-      bash \
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    asciidoc \
+    bzr \
+    clang \
+    cmake \
     jq \
-      git
+    libprotobuf-dev \
+    libssl-dev \
+    libxml2-dev \
+    llvm-dev \
+    lzma-dev \
+    patch \
+    protobuf-compiler \
+    xmlto \
+    xz-utils \
+    zlib1g-dev \
+  && rm -rf /var/lib/apt/lists/*
 
-RUN go get -u github.com/jstemmer/go-junit-report && \
+# Install rust
-      mv /go/bin/go-junit-report /usr/bin/go-junit-report && \
+RUN curl https://sh.rustup.rs -sSf | sh -s -- --default-toolchain stable -y \
-      rm -rf /go/*
+ && . $HOME/.cargo/env && rustup target add x86_64-apple-darwin x86_64-unknown-linux-musl
+
+ENV PATH="/root/.cargo/bin:${PATH}"
+
+# Install osxcross
+# The default osxcross deployment target is 10.6, which doesn't support thread local variables.
+ENV MACOSX_DEPLOYMENT_TARGET=10.11
+# The `git checkout` line specifies the same commit of osxcross that flux CI runs against.
+# It doesn't necessarily have to match but it would be good to keep things similar.
+RUN mkdir -p /opt/osxcross && \
+    cd /opt && \
+    git clone https://github.com/tpoechtrager/osxcross.git && \
+    cd osxcross && \
+    git checkout c2ad5e859d12a295c3f686a15bd7181a165bfa82 && \
+    curl -L -o ./tarballs/MacOSX${MACOSX_DEPLOYMENT_TARGET}.sdk.tar.xz \
+        https://macos-sdks.s3.amazonaws.com/MacOSX${MACOSX_DEPLOYMENT_TARGET}.sdk.tar.xz && \
+    UNATTENDED=1 PORTABLE=true OCDEBUG=1 ./build.sh && \
+    rm -rf .git build tarballs
+
+ENV PATH="/opt/osxcross/target/bin:${PATH}"
+
+# Wrapper to pull a version of pkg-config at runtime
+ENV PKG_CONFIG=/usr/local/bin/pkg-config.sh
+
+# install go-junit-report for unit tests
+RUN GO111MODULE=on go get github.com/jstemmer/go-junit-report@v0.9.1
 
 COPY fs/ /
 

releng/unit-tests/fs/root/.cargo/config

@@ -0,0 +1,6 @@
+# Eventually rust might support cross compilation out of the box, but until then
+# we have to tell it which external linker to use:
+# https://rust-lang.github.io/rustup/cross-compilation.html
+[target.x86_64-apple-darwin]
+linker = "x86_64-apple-darwin15-clang"
+ar = "x86_64-apple-darwin15-ar"

releng/unit-tests/fs/usr/local/bin/pkg-config.sh

@@ -0,0 +1,14 @@
+#!/usr/bin/env bash
+
+tmpdir=$(mktemp -d)
+trap "{ rm -rf ${tmpdir}; }" EXIT
+
+# "go build" can be noisy, and when Go invokes pkg-config (by calling this script) it will merge stdout and stderr.
+# Discard any output unless "go build" terminates with an error.
+CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -o ${tmpdir}/pkg-config github.com/influxdata/pkg-config &> ${tmpdir}/go_build_output
+if [ "$?" -ne 0 ]; then
+    cat ${tmpdir}/go_build_output 1>&2
+    exit 1
+fi
+
+${tmpdir}/pkg-config "$@"

releng/unit-tests/run.bash

@@ -4,8 +4,6 @@ function printHelp() {
   >&2 echo "USAGE: $0 -i PATH_TO_SOURCE_TARBALL -o OUTDIR
 
 Runs unit tests for influxdb.
-
-If the environment variable GO_NEXT is not empty, tests run with the 'next' version of Go.
 "
 }
 
@@ -33,13 +31,8 @@ if [ -z "$TARBALL" ] || [ -z "$OUTDIR" ]; then
   exit 1
 fi
 
-if [ -z "$GO_NEXT" ]; then
+DOCKER_TAG=latest
-  DOCKER_TAG=latest
+GO_VERSION="$GO_CURRENT_VERSION"
-  GO_VERSION="$GO_CURRENT_VERSION"
-else
-  DOCKER_TAG=next
-  GO_VERSION="$GO_NEXT_VERSION"
-fi
 docker build --build-arg "GO_VERSION=$GO_VERSION" -t influxdata/influxdb/releng/unit-tests:"$DOCKER_TAG" "$SRCDIR"
 
 docker run --rm \