From 8e05f69067d2fb1b5ec3e906fc1e33d74757a899 Mon Sep 17 00:00:00 2001 From: Steven Powell Date: Wed, 9 Aug 2023 16:09:51 -0700 Subject: [PATCH] CI: Add prow multiarch --- Makefile | 24 ++++++++++++++---------- deploy/prow/Dockerfile | 9 +++++---- 2 files changed, 19 insertions(+), 14 deletions(-) diff --git a/Makefile b/Makefile index b618b67ef0..0aee7fcb27 100644 --- a/Makefile +++ b/Makefile @@ -113,7 +113,9 @@ GVISOR_TAG ?= latest AUTOPAUSE_HOOK_TAG ?= v0.0.4 # prow-test tag to push changes to -PROW_TEST_TAG ?= v0.0.3 +PROW_TEST_TAG ?= v0.0.4 + +BUILDX_BUILDER ?= multiarch # storage provisioner tag to push changes to # NOTE: you will need to bump the PreloadVersion if you change this @@ -702,6 +704,7 @@ storage-provisioner-image-%: out/storage-provisioner-% docker-multi-arch-build: # installs QEMU static binaries to allow docker multi-arch build, see: https://github.com/docker/setup-qemu-action docker run --rm --privileged tonistiigi/binfmt:latest --install all + docker buildx create --name $(BUILDX_BUILDER) --bootstrap KICBASE_ARCH ?= linux/amd64,linux/arm64,linux/s390x,linux/arm,linux/ppc64le KICBASE_IMAGE_GCR ?= $(REGISTRY)/kicbase:$(KIC_VERSION) @@ -731,7 +734,8 @@ local-kicbase-debug: local-kicbase ## Builds a local kicbase image and switches .PHONY: build-kic-base-image build-kic-base-image: docker-multi-arch-build ## Build multi-arch local/kicbase:latest - docker buildx build -f ./deploy/kicbase/Dockerfile --platform $(KICBASE_ARCH) $(addprefix -t ,$(KICBASE_IMAGE_REGISTRIES)) --build-arg VERSION_JSON=$(VERSION_JSON) --build-arg COMMIT_SHA=${VERSION}-$(COMMIT_NOQUOTES) . + docker buildx build --builder $(BUILDX_BUILDER) -f ./deploy/kicbase/Dockerfile --platform $(KICBASE_ARCH) $(addprefix -t ,$(KICBASE_IMAGE_REGISTRIES)) --build-arg VERSION_JSON=$(VERSION_JSON) --build-arg COMMIT_SHA=${VERSION}-$(COMMIT_NOQUOTES) . + docker buildx rm $(BUILDX_BUILDER) .PHONY: push-kic-base-image push-kic-base-image: docker-multi-arch-build ## Push multi-arch local/kicbase:latest to all remote registries @@ -746,7 +750,8 @@ ifndef CIBUILD $(call user_confirm, 'Are you sure you want to push $(KICBASE_IMAGE_REGISTRIES) ?') endif ./deploy/kicbase/build_auto_pause.sh $(KICBASE_ARCH) - docker buildx build -f ./deploy/kicbase/Dockerfile --platform $(KICBASE_ARCH) $(addprefix -t ,$(KICBASE_IMAGE_REGISTRIES)) --push --build-arg VERSION_JSON=$(VERSION_JSON) --build-arg COMMIT_SHA=${VERSION}-$(COMMIT_NOQUOTES) --build-arg PREBUILT_AUTO_PAUSE=true . + docker buildx build --builder $(BUILDX_BUILDER) -f ./deploy/kicbase/Dockerfile --platform $(KICBASE_ARCH) $(addprefix -t ,$(KICBASE_IMAGE_REGISTRIES)) --push --build-arg VERSION_JSON=$(VERSION_JSON) --build-arg COMMIT_SHA=${VERSION}-$(COMMIT_NOQUOTES) --build-arg PREBUILT_AUTO_PAUSE=true . + docker buildx rm $(BUILDX_BUILDER) out/preload-tool: go build -ldflags="$(MINIKUBE_LDFLAGS)" -o $@ ./hack/preload-images/*.go @@ -871,7 +876,8 @@ kvm-image-amd64: installers/linux/kvm/Dockerfile.amd64 ## Convenient alias to b .PHONY: kvm-image-arm64 kvm-image-arm64: installers/linux/kvm/Dockerfile.arm64 docker-multi-arch-build ## Convenient alias to build the docker container - docker buildx build --platform linux/arm64 --build-arg "GO_VERSION=$(KVM_GO_VERSION)" -t $(KVM_BUILD_IMAGE_ARM64) -f $< $(dir $<) + docker buildx build --builder $(BUILDX_BUILDER) --platform linux/arm64 --build-arg "GO_VERSION=$(KVM_GO_VERSION)" -t $(KVM_BUILD_IMAGE_ARM64) -f $< $(dir $<) + docker buildx rm $(BUILDX_BUILDER) @echo "" @echo "$(@) successfully built" @@ -963,14 +969,12 @@ push-auto-pause-hook-image: auto-pause-hook-image docker login gcr.io/k8s-minikube $(MAKE) push-docker IMAGE=$(REGISTRY)/auto-pause-hook:$(AUTOPAUSE_HOOK_TAG) -.PHONY: prow-test-image -prow-test-image: - docker build --build-arg "GO_VERSION=$(GO_VERSION)" -t $(REGISTRY)/prow-test:$(PROW_TEST_TAG) ./deploy/prow - .PHONY: push-prow-test-image -push-prow-test-image: prow-test-image +push-prow-test-image: docker-multi-arch-build docker login gcr.io/k8s-minikube - $(MAKE) push-docker IMAGE=$(REGISTRY)/prow-test:$(PROW_TEST_TAG) + docker buildx build --push --builder $(BUILDX_BUILDER) --build-arg "GO_VERSION=$(GO_VERSION)" --platform linux/amd64,linux/arm64 -t $(REGISTRY)/prow-test:$(PROW_TEST_TAG) ./deploy/prow + docker buildx build --push --builder $(BUILDX_BUILDER) --build-arg "GO_VERSION=$(GO_VERSION)" --platform linux/amd64,linux/arm64 -t $(REGISTRY)/prow-test:latest ./deploy/prow + docker buildx rm $(BUILDX_BUILDER) .PHONY: out/performance-bot out/performance-bot: diff --git a/deploy/prow/Dockerfile b/deploy/prow/Dockerfile index 9b95ad8d4d..f4395deaa9 100644 --- a/deploy/prow/Dockerfile +++ b/deploy/prow/Dockerfile @@ -35,7 +35,8 @@ ENV GOPATH=/home/prow/go \ # We're already inside docker though so we can be sure these are already mounted. # Trying to remount these makes for a very noisy error block in the beginning of # the pod logs, so we just comment out the call to it... :shrug: -RUN echo "Installing Packages ..." \ +RUN export ARCH=$(dpkg --print-architecture) \ + && echo "Installing Packages ..." \ && apt-get update \ && apt-get install -y --no-install-recommends \ apt-transport-https \ @@ -58,7 +59,7 @@ RUN echo "Installing Packages ..." \ unzip \ && rm -rf /var/lib/apt/lists/* \ && echo "Installing Go ..." \ - && export GO_TARBALL="go${GO_VERSION}.linux-amd64.tar.gz"\ + && export GO_TARBALL="go${GO_VERSION}.linux-${ARCH}.tar.gz"\ && curl -fsSL "https://storage.googleapis.com/golang/${GO_TARBALL}" --output "${GO_TARBALL}" \ && tar xzf "${GO_TARBALL}" -C /usr/local \ && rm "${GO_TARBALL}"\ @@ -66,7 +67,7 @@ RUN echo "Installing Packages ..." \ && echo "Installing Docker ..." \ && curl -fsSL https://download.docker.com/linux/$(. /etc/os-release; echo "$ID")/gpg | apt-key add - \ && add-apt-repository \ - "deb [arch=amd64] https://download.docker.com/linux/$(. /etc/os-release; echo "$ID") \ + "deb [arch="$(dpkg --print-architecture)"] https://download.docker.com/linux/$(. /etc/os-release; echo "$ID") \ $(lsb_release -cs) stable" \ && apt-get update \ && apt-get install -y --no-install-recommends docker-ce \ @@ -76,7 +77,7 @@ RUN echo "Installing Packages ..." \ && update-alternatives --set iptables /usr/sbin/iptables-legacy \ && update-alternatives --set ip6tables /usr/sbin/ip6tables-legacy \ && echo "Installing Kubectl ..." \ - && curl -LO "https://dl.k8s.io/$(curl -sSL https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl" \ + && curl -LO "https://dl.k8s.io/$(curl -sSL https://dl.k8s.io/release/stable.txt)/bin/linux/${ARCH}/kubectl" \ && chmod +x ./kubectl \ && cp kubectl /usr/local/bin # copy in image utility scripts