diff --git a/Makefile b/Makefile index f293caf12e..0641d92bb1 100644 --- a/Makefile +++ b/Makefile @@ -100,6 +100,9 @@ GVISOR_TAG ?= latest # auto-pause-hook tag to push changes to AUTOPAUSE_HOOK_TAG ?= 1.13 +# prow-test tag to push changes to +PROW_TEST_TAG ?= v0.0.1 + # storage provisioner tag to push changes to # NOTE: you will need to bump the PreloadVersion if you change this STORAGE_PROVISIONER_TAG ?= v5 @@ -878,6 +881,15 @@ push-auto-pause-hook-image: auto-pause-hook-image docker login docker.io/azhao155 $(MAKE) push-docker IMAGE=docker.io/azhao155/auto-pause-hook:$(AUTOPAUSE_HOOK_TAG) +.PHONY: prow-test-image +prow-test-image: + docker build --build-arg "GO_VERSION=$(GO_VERSION)" -t docker.io/azhao155/prow-test:$(PROW_TEST_TAG) ./deploy/prow + +.PHONY: push-prow-test-image +push-prow-test-image: prow-test-image + docker login docker.io/azhao155 + $(MAKE) push-docker IMAGE=docker.io/azhao155/prow-test:$(PROW_TEST_TAG) + .PHONY: out/performance-bot out/performance-bot: GOOS=$(GOOS) GOARCH=$(GOARCH) go build -o $@ cmd/performance/pr-bot/bot.go diff --git a/deploy/prow/Dockerfile b/deploy/prow/Dockerfile new file mode 100644 index 0000000000..7a99642b02 --- /dev/null +++ b/deploy/prow/Dockerfile @@ -0,0 +1,87 @@ +# Copyright 2021 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Includes tools used for kubernetes/minikube CI +# NOTE: we attempt to avoid unnecessary tools and image layers while +# supporting kubernetes builds, minikube installation, etc. +FROM debian:buster + +# arg that specifies the go version to install +ARG GO_VERSION + +# add envs: +# - hinting that we are in a docker container +ENV GOPATH=/home/prow/go \ + PATH=/usr/local/go/bin:${PATH} \ + CONTAINER=docker + + +# Install tools needed to: +# - install docker +# +# TODO: the `sed` is a bit of a hack, look into alternatives. +# Why this exists: `docker service start` on debian runs a `cgroupfs_mount` method, +# We're already inside docker though so we can be sure these are already mounted. +# Trying to remount these makes for a very noisy error block in the beginning of +# the pod logs, so we just comment out the call to it... :shrug: +RUN echo "Installing Packages ..." \ + && apt-get update \ + && apt-get install -y --no-install-recommends \ + apt-transport-https \ + build-essential \ + ca-certificates \ + curl \ + file \ + git \ + gnupg2 \ + kmod \ + lsb-release \ + mercurial \ + pkg-config \ + procps \ + python \ + python-dev \ + python-pip \ + rsync \ + software-properties-common \ + unzip \ + && rm -rf /var/lib/apt/lists/* \ + && echo "Installing Go ..." \ + && export GO_TARBALL="go${GO_VERSION}.linux-amd64.tar.gz"\ + && curl -fsSL "https://storage.googleapis.com/golang/${GO_TARBALL}" --output "${GO_TARBALL}" \ + && tar xzf "${GO_TARBALL}" -C /usr/local \ + && rm "${GO_TARBALL}"\ + && mkdir -p "${GOPATH}/bin" \ + && echo "Installing Docker ..." \ + && curl -fsSL https://download.docker.com/linux/$(. /etc/os-release; echo "$ID")/gpg | apt-key add - \ + && add-apt-repository \ + "deb [arch=amd64] https://download.docker.com/linux/$(. /etc/os-release; echo "$ID") \ + $(lsb_release -cs) stable" \ + && apt-get update \ + && apt-get install -y --no-install-recommends docker-ce \ + && rm -rf /var/lib/apt/lists/* \ + && sed -i 's/cgroupfs_mount$/#cgroupfs_mount\n/' /etc/init.d/docker \ + && echo "Ensuring Legacy Iptables ..." \ + && update-alternatives --set iptables /usr/sbin/iptables-legacy \ + && update-alternatives --set ip6tables /usr/sbin/ip6tables-legacy \ + && echo "Installing Kubectl ..." \ + && curl -LO "https://storage.googleapis.com/kubernetes-release/release/$(curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt)/bin/linux/amd64/kubectl" \ + && chmod +x ./kubectl \ + && cp kubectl /usr/local/bin +# copy in image utility scripts +COPY wrapper.sh /usr/local/bin/ +# entrypoint is our wrapper script, in Prow you will need to explicitly re-specify this +ENTRYPOINT ["wrapper.sh", "/bin/bash"] +# volume for docker in docker, use an emptyDir in Prow +VOLUME ["/var/lib/docker"] diff --git a/deploy/prow/README.md b/deploy/prow/README.md new file mode 100644 index 0000000000..9345f66fe4 --- /dev/null +++ b/deploy/prow/README.md @@ -0,0 +1,8 @@ +Minikube prow testing docker image + +This image contains things we need to run minikube in Kubernetes CI, and +is maintained for the sole purpose of testing Kubernetes with MINIKUBE. + +## WARNING + +This image is _not_ supported for other use cases. Use at your own risk. diff --git a/deploy/prow/wrapper.sh b/deploy/prow/wrapper.sh new file mode 100755 index 0000000000..c52f37da69 --- /dev/null +++ b/deploy/prow/wrapper.sh @@ -0,0 +1,95 @@ +#!/usr/bin/env bash + +# Copyright 2021 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# wrapper.sh handles setting up things before / after the test command $@ +# +# usage: wrapper.sh my-test-command [my-test-args] +# +# Things wrapper.sh handles: +# - starting / stopping docker-in-docker +# - ensuring GOPATH/bin is in PATH +# +# After handling these things / before cleanup, my-test-command will be invoked, +# and the exit code of my-test-command will be preserved by wrapper.sh + +set -o errexit +set -o pipefail +set -o nounset + +>&2 echo "wrapper.sh] [INFO] Wrapping Test Command: \`$*\`" +printf '%0.s=' {1..80} >&2; echo >&2 +>&2 echo "wrapper.sh] [SETUP] Performing pre-test setup ..." + +cleanup(){ + >&2 echo "wrapper.sh] [CLEANUP] Cleaning up after Docker in Docker ..." + docker ps -aq | xargs -r docker rm -f || true + service docker stop || true + >&2 echo "wrapper.sh] [CLEANUP] Done cleaning up after Docker in Docker." +} + +early_exit_handler() { + >&2 echo "wrapper.sh] [EARLY EXIT] Interrupted, entering handler ..." + if [ -n "${EXIT_VALUE:-}" ]; then + >&2 echo "Original exit code was ${EXIT_VALUE}, not preserving due to interrupt signal" + fi + cleanup + >&2 echo "wrapper.sh] [EARLY EXIT] Completed handler ..." + exit 1 +} + +trap early_exit_handler TERM INT + + +>&2 echo "wrapper.sh] [SETUP] Docker in Docker enabled, initializing ..." +# If we have opted in to docker in docker, start the docker daemon, +service docker start +# the service can be started but the docker socket not ready, wait for ready +WAIT_N=0 +while true; do + # docker ps -q should only work if the daemon is ready + docker ps -q > /dev/null 2>&1 && break + if [[ ${WAIT_N} -lt 5 ]]; then + WAIT_N=$((WAIT_N+1)) + echo "wrapper.sh] [SETUP] Waiting for Docker to be ready, sleeping for ${WAIT_N} seconds ..." + sleep ${WAIT_N} + else + echo "wrapper.sh] [SETUP] Reached maximum attempts, not waiting any longer ..." + break + fi +done +echo "wrapper.sh] [SETUP] Done setting up Docker in Docker." + +# add $GOPATH/bin to $PATH +export GOPATH="${GOPATH:-${HOME}/go}" +export PATH="${GOPATH}/bin:${PATH}" +mkdir -p "${GOPATH}/bin" + +# actually run the user supplied command +printf '%0.s=' {1..80}; echo +>&2 echo "wrapper.sh] [TEST] Running Test Command: \`$*\` ..." +set +o errexit +"$@" +EXIT_VALUE=$? +set -o errexit +>&2 echo "wrapper.sh] [TEST] Test Command exit code: ${EXIT_VALUE}" + +# cleanup +cleanup + +# preserve exit value from user supplied command +printf '%0.s=' {1..80} >&2; echo >&2 +>&2 echo "wrapper.sh] Exiting ${EXIT_VALUE}" +exit ${EXIT_VALUE}