From c969c041e2cda926e6d582175bea565353aed040 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Lucas=20K=C3=A4ldstr=C3=B6m?= Date: Sun, 7 Feb 2016 20:35:14 +0200 Subject: [PATCH] Add support for building arm, arm64 and ppc64le server and client targets --- build/build-image/Dockerfile | 8 +-- build/build-image/cross/Dockerfile | 16 +++++- build/common.sh | 88 ++++++++++++++++++++++++------ hack/build-cross.sh | 5 +- hack/lib/golang.sh | 37 ++++++++++--- 5 files changed, 124 insertions(+), 30 deletions(-) diff --git a/build/build-image/Dockerfile b/build/build-image/Dockerfile index b02af4e19e..7a6f8c6d0f 100644 --- a/build/build-image/Dockerfile +++ b/build/build-image/Dockerfile @@ -20,8 +20,6 @@ FROM KUBE_BUILD_IMAGE_CROSS MAINTAINER Joe Beda -# (set an explicit GOARM of 5 for maximum compatibility) -ENV GOARM 5 ENV GOOS linux ENV GOARCH amd64 @@ -34,8 +32,10 @@ RUN mkdir $TMPDIR && \ # We use rsync to copy some binaries around. It is faster (0.3s vs. 1.1s) on my # machine vs. `install` -RUN rm -rf /var/lib/apt/lists/ -RUN apt-get -o Acquire::Check-Valid-Until=false update && apt-get install -y rsync +RUN rm -rf /var/lib/apt/lists/ \ + && apt-get -o Acquire::Check-Valid-Until=false update \ + && apt-get install -y rsync \ + && rm -rf /var/lib/apt/lists/ # Download and symlink etcd. We need this for our integration tests. RUN export ETCD_VERSION=v2.2.1; \ diff --git a/build/build-image/cross/Dockerfile b/build/build-image/cross/Dockerfile index 38fcb3b9ad..e5d70a9c8c 100644 --- a/build/build-image/cross/Dockerfile +++ b/build/build-image/cross/Dockerfile @@ -18,8 +18,13 @@ FROM golang:1.4.2 MAINTAINER Joe Beda +ENV GOARM 6 +ENV KUBE_DYNAMIC_CROSSPLATFORMS \ + armel + ENV KUBE_CROSSPLATFORMS \ - linux/386 linux/arm \ + linux/386 \ + linux/arm \ darwin/amd64 darwin/386 \ windows/amd64 windows/386 @@ -38,3 +43,12 @@ RUN apt-get install -y g++ && apt-get clean && rm -rf /var/lib/apt/lists/* &&\ rm -rf protobuf-3.0.0-beta-2 &&\ protoc --version +# Use dynamic cgo linking for architectures other than amd64 for the server platforms +# More info here: https://wiki.debian.org/CrossToolchains +RUN echo "deb http://emdebian.org/tools/debian/ jessie main" > /etc/apt/sources.list.d/cgocrosscompiling.list \ + && curl -s http://emdebian.org/tools/debian/emdebian-toolchain-archive.key | apt-key add - \ + && for platform in ${KUBE_DYNAMIC_CROSSPLATFORMS}; do dpkg --add-architecture ${platform}; done \ + && apt-get update \ + && apt-get install -y build-essential \ + && for platform in ${KUBE_DYNAMIC_CROSSPLATFORMS}; do apt-get install -y crossbuild-essential-${platform}; done \ + && apt-get clean && rm -rf /var/lib/apt/lists/* diff --git a/build/common.sh b/build/common.sh index 56ae96cc2e..d16daf3243 100755 --- a/build/common.sh +++ b/build/common.sh @@ -49,7 +49,7 @@ readonly KUBE_GCS_DELETE_EXISTING="${KUBE_GCS_DELETE_EXISTING:-n}" # Constants readonly KUBE_BUILD_IMAGE_REPO=kube-build readonly KUBE_BUILD_GOLANG_VERSION=1.4.2 -readonly KUBE_BUILD_IMAGE_CROSS_TAG="cross-${KUBE_BUILD_GOLANG_VERSION}-1" +readonly KUBE_BUILD_IMAGE_CROSS_TAG="cross-${KUBE_BUILD_GOLANG_VERSION}-2" readonly KUBE_BUILD_IMAGE_CROSS="${KUBE_BUILD_IMAGE_REPO}:${KUBE_BUILD_IMAGE_CROSS_TAG}" # KUBE_BUILD_DATA_CONTAINER_NAME=kube-build-data- @@ -90,17 +90,47 @@ readonly RELEASE_STAGE="${LOCAL_OUTPUT_ROOT}/release-stage" readonly RELEASE_DIR="${LOCAL_OUTPUT_ROOT}/release-tars" readonly GCS_STAGE="${LOCAL_OUTPUT_ROOT}/gcs-stage" -# The set of master binaries that run in Docker (on Linux) +# Get the set of master binaries that run in Docker (on Linux) # Entry format is ",". # Binaries are placed in /usr/local/bin inside the image. -readonly KUBE_DOCKER_WRAPPED_BINARIES=( - kube-apiserver,busybox - kube-controller-manager,busybox - kube-scheduler,busybox - kube-proxy,gcr.io/google_containers/debian-iptables:v1 -) +# +# $1 - server architecture +kube::build::get_docker_wrapped_binaries() { + case $1 in + "amd64") + local targets=( + kube-apiserver,busybox + kube-controller-manager,busybox + kube-scheduler,busybox + kube-proxy,gcr.io/google_containers/debian-iptables:v1 + );; + "arm") # TODO: Use image with iptables installed for kube-proxy for arm, arm64 and ppc64le + local targets=( + kube-apiserver,hypriot/armhf-busybox + kube-controller-manager,hypriot/armhf-busybox + kube-scheduler,hypriot/armhf-busybox + kube-proxy,hypriot/armhf-busybox + );; + "arm64") + local targets=( + kube-apiserver,aarch64/busybox + kube-controller-manager,aarch64/busybox + kube-scheduler,aarch64/busybox + kube-proxy,aarch64/busybox + );; + "ppc64le") + local targets=( + kube-apiserver,ppc64le/busybox + kube-controller-manager,ppc64le/busybox + kube-scheduler,ppc64le/busybox + kube-proxy,ppc64le/busybox + );; + esac -# The set of addons images that should be prepopulated + echo "${targets[@]}" +} + +# The set of addons images that should be prepopulated on linux/amd64 readonly KUBE_ADDON_PATHS=( gcr.io/google_containers/pause:2.0 gcr.io/google_containers/kube-registry-proxy:0.3 @@ -109,7 +139,7 @@ readonly KUBE_ADDON_PATHS=( # --------------------------------------------------------------------------- # Basic setup functions -# Verify that the right utilities and such are installed for building Kube. Set +# Verify that the right utilities and such are installed for building Kube. Set # up some dynamic constants. # # Vars set: @@ -669,7 +699,7 @@ function kube::release::package_tarballs() { kube::util::wait-for-jobs || { kube::log::error "previous tarball phase failed"; return 1; } } -# Package up all of the cross compiled clients. Over time this should grow into +# Package up all of the cross compiled clients. Over time this should grow into # a full SDK function kube::release::package_client_tarballs() { # Find all of the built client binaries @@ -726,7 +756,11 @@ function kube::release::package_server_tarballs() { "${release_stage}/server/bin/" kube::release::create_docker_images_for_server "${release_stage}/server/bin" "${arch}" - kube::release::write_addon_docker_images_for_server "${release_stage}/addons" + + # Only release addon images for linux/amd64. These addon images aren't necessary for other architectures + if [[ ${platform} == "linux/amd64" ]]; then + kube::release::write_addon_docker_images_for_server "${release_stage}/addons" + fi # Include the client binaries here too as they are useful debugging tools. local client_bins=("${KUBE_CLIENT_BINARIES[@]}") @@ -764,15 +798,15 @@ function kube::release::sha1() { # Args: # $1 - binary_dir, the directory to save the tared images to. # $2 - arch, architecture for which we are building docker images. -# Globals: -# KUBE_DOCKER_WRAPPED_BINARIES function kube::release::create_docker_images_for_server() { # Create a sub-shell so that we don't pollute the outer environment ( local binary_dir="$1" local arch="$2" local binary_name - for wrappable in "${KUBE_DOCKER_WRAPPED_BINARIES[@]}"; do + local binaries=($(kube::build::get_docker_wrapped_binaries ${arch})) + + for wrappable in "${binaries[@]}"; do local oldifs=$IFS IFS="," @@ -797,7 +831,14 @@ function kube::release::create_docker_images_for_server() { ln ${binary_dir}/${binary_name} ${docker_build_path}/${binary_name} printf " FROM ${base_image} \n ADD ${binary_name} /usr/local/bin/${binary_name}\n" > ${docker_file_path} - local docker_image_tag=gcr.io/google_containers/$binary_name:$md5_sum + if [[ ${arch} == "amd64" ]]; then + # If we are building a amd64 docker image, preserve the original image name + local docker_image_tag=gcr.io/google_containers/${binary_name}:${md5_sum} + else + # If we are building a docker image for another architecture, append the arch in the image tag + local docker_image_tag=gcr.io/google_containers/${binary_name}-${arch}:${md5_sum} + fi + "${DOCKER[@]}" build -q -t "${docker_image_tag}" ${docker_build_path} >/dev/null "${DOCKER[@]}" save ${docker_image_tag} > ${binary_dir}/${binary_name}.tar echo $md5_sum > ${binary_dir}/${binary_name}.docker_tag @@ -930,7 +971,7 @@ function kube::release::package_test_tarball() { mkdir -p "${release_stage}" local platform - for platform in "${KUBE_CLIENT_PLATFORMS[@]}"; do + for platform in "${KUBE_TEST_PLATFORMS[@]}"; do local test_bins=("${KUBE_TEST_BINARIES[@]}") if [[ "${platform%/*}" == "windows" ]]; then test_bins=("${KUBE_TEST_BINARIES_WIN[@]}") @@ -1482,6 +1523,9 @@ function kube::release::docker::release() { local archs=( "amd64" + "arm" + "arm64" + "ppc64le" ) local docker_push_cmd=("${DOCKER[@]}") @@ -1494,6 +1538,16 @@ function kube::release::docker::release() { local docker_target="${KUBE_DOCKER_REGISTRY}/${binary}-${arch}:${KUBE_DOCKER_IMAGE_TAG}" kube::log::status "Pushing ${binary} to ${docker_target}" "${docker_push_cmd[@]}" push "${docker_target}" + + # If we have a amd64 docker image. Tag it without -amd64 also and push it for compability with earlier versions + if [[ ${arch} == "amd64" ]]; then + local legacy_docker_target="${KUBE_DOCKER_REGISTRY}/${binary}:${KUBE_DOCKER_IMAGE_TAG}" + + "${DOCKER[@]}" tag -f "${docker_target}" "${legacy_docker_target}" 2>/dev/null + + kube::log::status "Pushing ${binary} to ${legacy_docker_target}" + "${docker_push_cmd[@]}" push "${legacy_docker_target}" + fi done done } diff --git a/hack/build-cross.sh b/hack/build-cross.sh index 14fa501d7f..98a9c60392 100755 --- a/hack/build-cross.sh +++ b/hack/build-cross.sh @@ -28,6 +28,9 @@ KUBE_BUILD_PLATFORMS=("${KUBE_SERVER_PLATFORMS[@]}") kube::golang::build_binaries "${KUBE_SERVER_TARGETS[@]}" KUBE_BUILD_PLATFORMS=("${KUBE_CLIENT_PLATFORMS[@]}") -kube::golang::build_binaries "${KUBE_CLIENT_TARGETS[@]}" "${KUBE_TEST_TARGETS[@]}" +kube::golang::build_binaries "${KUBE_CLIENT_TARGETS[@]}" + +KUBE_BUILD_PLATFORMS=("${KUBE_TEST_PLATFORMS[@]}") +kube::golang::build_binaries "${KUBE_TEST_TARGETS[@]}" kube::golang::place_bins diff --git a/hack/lib/golang.sh b/hack/lib/golang.sh index bf881caf38..c1b97d7678 100755 --- a/hack/lib/golang.sh +++ b/hack/lib/golang.sh @@ -60,6 +60,18 @@ readonly KUBE_CLIENT_TARGETS=( readonly KUBE_CLIENT_BINARIES=("${KUBE_CLIENT_TARGETS[@]##*/}") readonly KUBE_CLIENT_BINARIES_WIN=("${KUBE_CLIENT_BINARIES[@]/%/.exe}") +# If we update this we should also update the set of golang compilers we build +# in 'build/build-image/cross/Dockerfile'. However, it's only a bit faster since go 1.5, not mandatory +readonly KUBE_CLIENT_PLATFORMS=( + linux/amd64 + linux/386 + linux/arm + darwin/amd64 + darwin/386 + windows/amd64 + windows/386 +) + # The set of test targets that we are building for all platforms kube::golang::test_targets() { local targets=( @@ -97,14 +109,10 @@ readonly KUBE_TEST_PORTABLE=( hack/lib ) -# If we update this we need to also update the set of golang compilers we build -# in 'build/build-image/Dockerfile' -readonly KUBE_CLIENT_PLATFORMS=( +# Which platforms we should compile test targets for. Not all client platforms need these tests +readonly KUBE_TEST_PLATFORMS=( linux/amd64 - linux/386 - linux/arm darwin/amd64 - darwin/386 windows/amd64 ) @@ -161,7 +169,7 @@ kube::golang::binaries_from_targets() { done } -# Asks golang what it thinks the host platform is. The go tool chain does some +# Asks golang what it thinks the host platform is. The go tool chain does some # slightly different things when the target platform matches the host platform. kube::golang::host_platform() { echo "$(go env GOHOSTOS)/$(go env GOHOSTARCH)" @@ -190,11 +198,26 @@ kube::golang::set_platform_envs() { export GOOS=${platform%/*} export GOARCH=${platform##*/} + + # Dynamic CGO linking for other server architectures than linux/amd64 goes here + # If you want to include support for more server platforms than these, add arch-specific gcc names here + if [[ ${platform} == "linux/arm" ]]; then + export CGO_ENABLED=1 + export CC=arm-linux-gnueabi-gcc + elif [[ ${platform} == "linux/arm64" ]]; then + export CGO_ENABLED=1 + export CC=aarch64-linux-gnu-gcc + elif [[ ${platform} == "linux/ppc64le" ]]; then + export CGO_ENABLED=1 + export CC=powerpc64le-linux-gnu-gcc + fi } kube::golang::unset_platform_envs() { unset GOOS unset GOARCH + unset CGO_ENABLED + unset CC } # Create the GOPATH tree under $KUBE_OUTPUT