chore: fix release build scripts (#20970)

Note: this commit fixes linux/amd64 and darwin/amd64 builds. We are still
working on fixes for linux/arm64 and windows/amd64.

Partial fix of https://github.com/influxdata/influxdb/issues/20854
pull/20972/head
Sam Arnold 2021-03-16 10:23:55 -03:00 committed by GitHub
parent 178a1d0c3b
commit 1e53bf13e0
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
16 changed files with 266 additions and 100 deletions

View File

@ -20,7 +20,6 @@ The `fs` folder is overlaid on the Docker image so that is clear where each scri
Those scripts make assumptions about the environment which are controlled in the outer scripts (i.e. `build.bash`), so the scripts not intended to be run outside of Docker.
By default, these scripts will use the "current" Go version as determined by `_go_versions.sh`.
To use the "next" version of Go, set the environment variable GO_NEXT to a non-empty value.
## source-tarball

View File

@ -1,5 +1,3 @@
# These are the current and "next" Go versions used to build influxdb.
# This file is meant to be sourced from other scripts.
export GO_CURRENT_VERSION=1.13.8
export GO_NEXT_VERSION=1.13.8

View File

@ -35,9 +35,10 @@ if [ -z "$OS" ] || [ -z "$ARCH" ]; then
exit 1
fi
mkdir -p /go
tar x -C /go -zf /influxdb-src.tar.gz
ln -s /go/src/github.com/influxdata/influxdb /isrc # Shorthand for influxdb source.
WORK=/influxdata
mkdir -p ${WORK}
tar x -C ${WORK} -zf /influxdb-src.tar.gz
ln -s ${WORK}/influxdb /isrc # Shorthand for influxdb source.
SHA=$(jq -r .sha < "/isrc/.metadata.json")
VERSION=$(jq -r .version < "/isrc/.metadata.json")
ARCHIVE_ROOT_NAME="influxdb-${VERSION}-1"

View File

@ -1,10 +1,54 @@
ARG GO_VERSION
FROM golang:${GO_VERSION}
# Install system dependencies + cross build system dependencies
RUN apt-get update && apt-get install -y --no-install-recommends \
asciidoc \
bzr \
clang \
cmake \
jq \
libprotobuf-dev \
libssl-dev \
libxml2-dev \
llvm-dev \
lzma-dev \
patch \
protobuf-compiler \
xmlto \
xz-utils \
zlib1g-dev \
&& rm -rf /var/lib/apt/lists/*
# Install rust
RUN curl https://sh.rustup.rs -sSf | sh -s -- --default-toolchain stable -y \
&& . $HOME/.cargo/env && rustup target add x86_64-apple-darwin x86_64-unknown-linux-musl
ENV PATH="/root/.cargo/bin:${PATH}"
# Install osxcross
# The default osxcross deployment target is 10.6, which doesn't support thread local variables.
ENV MACOSX_DEPLOYMENT_TARGET=10.11
# The `git checkout` line specifies the same commit of osxcross that flux CI runs against.
# It doesn't necessarily have to match but it would be good to keep things similar.
RUN mkdir -p /opt/osxcross && \
cd /opt && \
git clone https://github.com/tpoechtrager/osxcross.git && \
cd osxcross && \
git checkout c2ad5e859d12a295c3f686a15bd7181a165bfa82 && \
curl -L -o ./tarballs/MacOSX${MACOSX_DEPLOYMENT_TARGET}.sdk.tar.xz \
https://macos-sdks.s3.amazonaws.com/MacOSX${MACOSX_DEPLOYMENT_TARGET}.sdk.tar.xz && \
UNATTENDED=1 PORTABLE=true OCDEBUG=1 ./build.sh && \
rm -rf .git build tarballs
ENV PATH="/opt/osxcross/target/bin:${PATH}"
# Wrapper to pull a version of pkg-config at runtime
ENV PKG_CONFIG=/usr/local/bin/pkg-config.sh
# install go-junit-report for unit tests
RUN GO111MODULE=on go get github.com/jstemmer/go-junit-report@v0.9.1
COPY fs/ /
ENTRYPOINT ["influxdb_raw_binaries.bash"]

View File

@ -4,9 +4,6 @@ function printHelp() {
>&2 echo "USAGE: $0 -i PATH_TO_SOURCE_TARBALL -o OUTDIR
Emits an archive of influxdb binaries based on the current environment's GOOS and GOARCH.
Respects CGO_ENABLED.
If the environment variable GO_NEXT is not empty, builds the binaries with the 'next' version of Go.
"
}
@ -26,13 +23,15 @@ source "$SRCDIR/../_go_versions.sh"
OUTDIR=""
TARBALL=""
RACE_FLAG=""
STATIC_FLAG=""
while getopts hi:o:r arg; do
while getopts hi:o:rs arg; do
case "$arg" in
h) printHelp; exit 1;;
i) TARBALL="$OPTARG";;
o) OUTDIR="$OPTARG";;
r) RACE_FLAG="-r";;
s) STATIC_FLAG="-s";;
esac
done
@ -41,13 +40,8 @@ if [ -z "$OUTDIR" ] || [ -z "$TARBALL" ]; then
exit 1
fi
if [ -z "$GO_NEXT" ]; then
DOCKER_TAG=latest
GO_VERSION="$GO_CURRENT_VERSION"
else
DOCKER_TAG=next
GO_VERSION="$GO_NEXT_VERSION"
fi
DOCKER_TAG=latest
GO_VERSION="$GO_CURRENT_VERSION"
docker build --build-arg "GO_VERSION=$GO_VERSION" -t influxdata/influxdb/releng/raw-binaries:"$DOCKER_TAG" "$SRCDIR"
mkdir -p "$OUTDIR"
@ -55,5 +49,5 @@ mkdir -p "$OUTDIR"
docker run --rm \
--mount type=bind,source="${OUTDIR}",destination=/out \
--mount type=bind,source="${TARBALL}",destination=/influxdb-src.tar.gz,ro=1 \
-e GOOS -e GOARCH -e CGO_ENABLED \
influxdata/influxdb/releng/raw-binaries:"$DOCKER_TAG" $RACE_FLAG
-e GOOS -e GOARCH \
influxdata/influxdb/releng/raw-binaries:"$DOCKER_TAG" $RACE_FLAG $STATIC_FLAG

View File

@ -0,0 +1,6 @@
# Eventually rust might support cross compilation out of the box, but until then
# we have to tell it which external linker to use:
# https://rust-lang.github.io/rustup/cross-compilation.html
[target.x86_64-apple-darwin]
linker = "x86_64-apple-darwin15-clang"
ar = "x86_64-apple-darwin15-ar"

View File

@ -1,5 +1,7 @@
#!/bin/bash
set -e
function printHelp() {
>&2 echo "USAGE: $0 [-r]
@ -15,29 +17,62 @@ To build with race detection enabled, pass the -r flag.
}
RACE_FLAG=""
STATIC=""
while getopts hr arg; do
while getopts w:hrs arg; do
case "$arg" in
h) printHelp; exit 1;;
r) RACE_FLAG="-race";;
s) STATIC=1;;
esac
done
if [ -z "$GOOS" ] || [ -z "$GOARCH" ]; then
>&2 echo 'The environment variables $GOOS and $GOARCH must both be set.'
exit 1
fi
if [[ "$GOOS" == darwin ]] ; then
# Control the compiler for go. Rust linker is set in $HOME/.cargo/config
export CC=x86_64-apple-darwin15-clang
fi
# Extract tarball into GOPATH.
tar xz -C "$GOPATH" -f /influxdb-src.tar.gz
WORKSPACE=/influxdata
SHA=$(jq -r .sha < "$GOPATH/src/github.com/influxdata/influxdb/.metadata.json")
mkdir -p ${WORKSPACE}
echo "Extracting influxdb tarball"
# Extract tarball into WORKSPACE.
tar -vxz -C ${WORKSPACE} -f /influxdb-src.tar.gz
SHA=$(jq -r .sha < "${WORKSPACE}/influxdb/.metadata.json")
OUTDIR=$(mktemp -d)
(
cd ${WORKSPACE}/influxdb
BINARY_PACKAGES="
github.com/influxdata/influxdb/cmd/influxd
github.com/influxdata/influxdb/cmd/influx
github.com/influxdata/influxdb/cmd/influx_inspect
github.com/influxdata/influxdb/cmd/influx_tsm"
for cmd in $BINARY_PACKAGES; do
export CGO_ENABLED=1
echo "env for go build: GOOS=$GOOS GOARCH=$GOARCH CGO_ENABLED=$CGO_ENABLED"
if [[ -n "$STATIC" ]] ; then
echo go build -i -o "$OUTDIR/$(basename $cmd)" -tags "netgo osusergo static_build" $cmd
go build -i -o "$OUTDIR/$(basename $cmd)" -tags "netgo osusergo static_build" $cmd
else
echo go build $RACE_FLAG -i -o "$OUTDIR/$(basename $cmd)" $cmd
go build $RACE_FLAG -i -o "$OUTDIR/$(basename $cmd)" $cmd
fi
done
)
SUFFIX=
if [ "$CGO_ENABLED" == "0" ]; then
if [[ -n "$STATIC" ]]; then
# Only add the static suffix to the filename when explicitly requested.
SUFFIX=_static
elif [ -n "$RACE_FLAG" ]; then
@ -46,36 +81,6 @@ elif [ -n "$RACE_FLAG" ]; then
fi
TARBALL_NAME="influxdb_bin_${GOOS}_${GOARCH}${SUFFIX}-${SHA}.tar.gz"
# note: according to https://github.com/golang/go/wiki/GoArm
# we want to support armel using GOARM=5
# and we want to support armhf using GOARM=6
# no GOARM setting is necessary for arm64
if [ $GOARCH == "armel" ]; then
GOARCH=arm
GOARM=5
fi
if [ $GOARCH == "armhf" ]; then
GOARCH=arm
GOARM=6
fi
OUTDIR=$(mktemp -d)
for cmd in \
influxdb/cmd/influxd \
influxdb/cmd/influx \
influxdb/cmd/influx_inspect \
influxdb/cmd/influx_tsm \
; do
# Build all the binaries into $OUTDIR.
# Windows binaries will get the .exe suffix as expected.
(cd "$OUTDIR" && go build $RACE_FLAG -i "github.com/influxdata/$cmd")
done
(cd "$OUTDIR" && tar czf "/out/$TARBALL_NAME" ./*)
(cd /out && md5sum "$TARBALL_NAME" > "$TARBALL_NAME.md5")
(cd /out && sha256sum "$TARBALL_NAME" > "$TARBALL_NAME.sha256")
TARBALL_PATH="/out/${TARBALL_NAME}"
echo tar -C ${OUTDIR} -cvzf ${TARBALL_PATH} .
tar -C ${OUTDIR} -cvzf ${TARBALL_PATH} .

View File

@ -0,0 +1,14 @@
#!/usr/bin/env bash
tmpdir=$(mktemp -d)
trap "{ rm -rf ${tmpdir}; }" EXIT
# "go build" can be noisy, and when Go invokes pkg-config (by calling this script) it will merge stdout and stderr.
# Discard any output unless "go build" terminates with an error.
CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -o ${tmpdir}/pkg-config github.com/influxdata/pkg-config &> ${tmpdir}/go_build_output
if [ "$?" -ne 0 ]; then
cat ${tmpdir}/go_build_output 1>&2
exit 1
fi
${tmpdir}/pkg-config "$@"

View File

@ -1,19 +1,53 @@
ARG GO_VERSION
FROM golang:${GO_VERSION}-alpine
FROM golang:${GO_VERSION}
RUN apk add --no-cache \
# Install system dependencies + cross build system dependencies
RUN apt-get update && apt-get install -y --no-install-recommends \
asciidoc \
bash \
git \
openssh-client \
make \
tar \
xmlto
bzr \
clang \
cmake \
jq \
libprotobuf-dev \
libssl-dev \
libxml2-dev \
llvm-dev \
lzma-dev \
patch \
protobuf-compiler \
xmlto \
xz-utils \
zlib1g-dev \
&& rm -rf /var/lib/apt/lists/*
# Build the dep binary and then clean out /go.
RUN go get github.com/golang/dep/cmd/dep && \
mv /go/bin/dep /usr/local/bin/dep && \
rm -rf /go/*
# Install rust
RUN curl https://sh.rustup.rs -sSf | sh -s -- --default-toolchain stable -y \
&& . $HOME/.cargo/env && rustup target add x86_64-apple-darwin x86_64-unknown-linux-musl
ENV PATH="/root/.cargo/bin:${PATH}"
# Install osxcross
# The default osxcross deployment target is 10.6, which doesn't support thread local variables.
ENV MACOSX_DEPLOYMENT_TARGET=10.11
# The `git checkout` line specifies the same commit of osxcross that flux CI runs against.
# It doesn't necessarily have to match but it would be good to keep things similar.
RUN mkdir -p /opt/osxcross && \
cd /opt && \
git clone https://github.com/tpoechtrager/osxcross.git && \
cd osxcross && \
git checkout c2ad5e859d12a295c3f686a15bd7181a165bfa82 && \
curl -L -o ./tarballs/MacOSX${MACOSX_DEPLOYMENT_TARGET}.sdk.tar.xz \
https://macos-sdks.s3.amazonaws.com/MacOSX${MACOSX_DEPLOYMENT_TARGET}.sdk.tar.xz && \
UNATTENDED=1 PORTABLE=true OCDEBUG=1 ./build.sh && \
rm -rf .git build tarballs
ENV PATH="/opt/osxcross/target/bin:${PATH}"
# Wrapper to pull a version of pkg-config at runtime
ENV PKG_CONFIG=/usr/local/bin/pkg-config.sh
# install go-junit-report for unit tests
RUN GO111MODULE=on go get github.com/jstemmer/go-junit-report@v0.9.1
COPY fs/ /

View File

@ -0,0 +1,6 @@
# Eventually rust might support cross compilation out of the box, but until then
# we have to tell it which external linker to use:
# https://rust-lang.github.io/rustup/cross-compilation.html
[target.x86_64-apple-darwin]
linker = "x86_64-apple-darwin15-clang"
ar = "x86_64-apple-darwin15-ar"

View File

@ -36,19 +36,19 @@ if [ -z "$SHA" ] || [ -z "$BRANCH" ] || [ -z "$VERSION" ]; then
exit 1
fi
IPATH=/go/src/github.com/influxdata
IPATH=/influxdb-source
mkdir -p "$IPATH" && cd "$IPATH"
if [ -d /influxdb-git ]; then
git clone /influxdb-git "$IPATH/influxdb"
else
git clone https://github.com/influxdata/influxdb.git
echo "Influxdb .git directory required" >&2
exit 1
fi
cd influxdb
git checkout "$SHA"
go mod vendor
cd ..
(
cd influxdb
git checkout "$SHA"
)
# Emit version metadata to appropriate files.
@ -71,20 +71,19 @@ func init() {
commit = "%s"
}' "$VERSION" "$BRANCH" "$SHA" > "./influxdb/cmd/influxd/version.generated.go"
# influx uses just version.
printf 'package main
// Code generated by influxdata/releng tooling. DO NOT EDIT.
func init() {
func init() {
version = "%s"
}' "$VERSION" > "./influxdb/cmd/influx/version.generated.go"
# Prebuild the man pages so that consumers of the source tarball don't have to build it themselves.
(cd /go/src/github.com/influxdata/influxdb/man && make build && gzip -9 ./*.1)
(cd "$IPATH"/influxdb/man && make build && gzip -9 ./*.1)
TARBALL_NAME="influxdb-src-$SHA.tar.gz"
(cd /go && tar czf "/out/$TARBALL_NAME" --exclude-vcs ./*) # --exclude-vcs is a GNU tar option.
tar -vC ${IPATH} -czf "/out/${TARBALL_NAME}" --exclude-vcs ./* # --exclude-vcs is a GNU tar option.
(cd /out && md5sum "$TARBALL_NAME" > "$TARBALL_NAME.md5")
(cd /out && sha256sum "$TARBALL_NAME" > "$TARBALL_NAME.sha256")

View File

@ -0,0 +1,14 @@
#!/usr/bin/env bash
tmpdir=$(mktemp -d)
trap "{ rm -rf ${tmpdir}; }" EXIT
# "go build" can be noisy, and when Go invokes pkg-config (by calling this script) it will merge stdout and stderr.
# Discard any output unless "go build" terminates with an error.
CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -o ${tmpdir}/pkg-config github.com/influxdata/pkg-config &> ${tmpdir}/go_build_output
if [ "$?" -ne 0 ]; then
cat ${tmpdir}/go_build_output 1>&2
exit 1
fi
${tmpdir}/pkg-config "$@"

View File

@ -1,14 +1,53 @@
ARG GO_VERSION
FROM golang:${GO_VERSION}-alpine
FROM golang:${GO_VERSION}
RUN apk add --no-cache \
bash \
# Install system dependencies + cross build system dependencies
RUN apt-get update && apt-get install -y --no-install-recommends \
asciidoc \
bzr \
clang \
cmake \
jq \
git
libprotobuf-dev \
libssl-dev \
libxml2-dev \
llvm-dev \
lzma-dev \
patch \
protobuf-compiler \
xmlto \
xz-utils \
zlib1g-dev \
&& rm -rf /var/lib/apt/lists/*
RUN go get -u github.com/jstemmer/go-junit-report && \
mv /go/bin/go-junit-report /usr/bin/go-junit-report && \
rm -rf /go/*
# Install rust
RUN curl https://sh.rustup.rs -sSf | sh -s -- --default-toolchain stable -y \
&& . $HOME/.cargo/env && rustup target add x86_64-apple-darwin x86_64-unknown-linux-musl
ENV PATH="/root/.cargo/bin:${PATH}"
# Install osxcross
# The default osxcross deployment target is 10.6, which doesn't support thread local variables.
ENV MACOSX_DEPLOYMENT_TARGET=10.11
# The `git checkout` line specifies the same commit of osxcross that flux CI runs against.
# It doesn't necessarily have to match but it would be good to keep things similar.
RUN mkdir -p /opt/osxcross && \
cd /opt && \
git clone https://github.com/tpoechtrager/osxcross.git && \
cd osxcross && \
git checkout c2ad5e859d12a295c3f686a15bd7181a165bfa82 && \
curl -L -o ./tarballs/MacOSX${MACOSX_DEPLOYMENT_TARGET}.sdk.tar.xz \
https://macos-sdks.s3.amazonaws.com/MacOSX${MACOSX_DEPLOYMENT_TARGET}.sdk.tar.xz && \
UNATTENDED=1 PORTABLE=true OCDEBUG=1 ./build.sh && \
rm -rf .git build tarballs
ENV PATH="/opt/osxcross/target/bin:${PATH}"
# Wrapper to pull a version of pkg-config at runtime
ENV PKG_CONFIG=/usr/local/bin/pkg-config.sh
# install go-junit-report for unit tests
RUN GO111MODULE=on go get github.com/jstemmer/go-junit-report@v0.9.1
COPY fs/ /

View File

@ -0,0 +1,6 @@
# Eventually rust might support cross compilation out of the box, but until then
# we have to tell it which external linker to use:
# https://rust-lang.github.io/rustup/cross-compilation.html
[target.x86_64-apple-darwin]
linker = "x86_64-apple-darwin15-clang"
ar = "x86_64-apple-darwin15-ar"

View File

@ -0,0 +1,14 @@
#!/usr/bin/env bash
tmpdir=$(mktemp -d)
trap "{ rm -rf ${tmpdir}; }" EXIT
# "go build" can be noisy, and when Go invokes pkg-config (by calling this script) it will merge stdout and stderr.
# Discard any output unless "go build" terminates with an error.
CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -o ${tmpdir}/pkg-config github.com/influxdata/pkg-config &> ${tmpdir}/go_build_output
if [ "$?" -ne 0 ]; then
cat ${tmpdir}/go_build_output 1>&2
exit 1
fi
${tmpdir}/pkg-config "$@"

View File

@ -4,8 +4,6 @@ function printHelp() {
>&2 echo "USAGE: $0 -i PATH_TO_SOURCE_TARBALL -o OUTDIR
Runs unit tests for influxdb.
If the environment variable GO_NEXT is not empty, tests run with the 'next' version of Go.
"
}
@ -33,13 +31,8 @@ if [ -z "$TARBALL" ] || [ -z "$OUTDIR" ]; then
exit 1
fi
if [ -z "$GO_NEXT" ]; then
DOCKER_TAG=latest
GO_VERSION="$GO_CURRENT_VERSION"
else
DOCKER_TAG=next
GO_VERSION="$GO_NEXT_VERSION"
fi
DOCKER_TAG=latest
GO_VERSION="$GO_CURRENT_VERSION"
docker build --build-arg "GO_VERSION=$GO_VERSION" -t influxdata/influxdb/releng/unit-tests:"$DOCKER_TAG" "$SRCDIR"
docker run --rm \