Refactor test harness to isolate test outputs
Thomas Stromberg
parent
b9a4f9459f
commit
81cdbae8d5
|
|
@ -20,107 +20,178 @@
|
|||
# The script expects the following env variables:
|
||||
# OS_ARCH: The operating system and the architecture separated by a hyphen '-' (e.g. darwin-amd64, linux-amd64, windows-amd64)
|
||||
# VM_DRIVER: the vm-driver to use for the test
|
||||
# EXTRA_BUILD_ARGS: additional flags to pass into minikube start
|
||||
# EXTRA_START_ARGS: additional flags to pass into minikube start
|
||||
# EXTRA_ARGS: additional flags to pass into minikube
|
||||
# JOB_NAME: the name of the logfile and check name to update on github
|
||||
#
|
||||
|
||||
mkdir -p out/ testdata/
|
||||
|
||||
readonly TEST_ROOT="${HOME}/minikube-integration"
|
||||
readonly TEST_HOME="${TEST_ROOT}/${OS_ARCH}-${VM_DRIVER}-${MINIKUBE_LOCATION}-$$-${COMMIT}"
|
||||
echo ">> Starting at $(date)"
|
||||
echo ""
|
||||
echo "arch: ${OS_ARCH}"
|
||||
echo "build: ${MINIKUBE_LOCATION}"
|
||||
echo "driver: ${VM_DRIVER}"
|
||||
echo "job: ${JOB_NAME}"
|
||||
echo "test home: ${TEST_HOME}"
|
||||
echo "sudo: ${SUDO_PREFIX}"
|
||||
echo "kernel: $(uname -v)"
|
||||
# Setting KUBECONFIG prevents the version ceck from erroring out due to permission issues
|
||||
echo "kubectl: $(env KUBECONFIG=${TEST_HOME} kubectl version --client --short=true)"
|
||||
echo "docker: $(docker version --format '{{ .Client.Version }}')"
|
||||
|
||||
case "${VM_DRIVER}" in
|
||||
kvm2)
|
||||
echo "virsh: $(virsh --version)"
|
||||
;;
|
||||
virtualbox)
|
||||
echo "vbox: $(vboxmanage --version)"
|
||||
;;
|
||||
esac
|
||||
|
||||
echo ""
|
||||
mkdir -p out/ testdata/ "${TEST_HOME}"
|
||||
|
||||
# Install gsutil if necessary.
|
||||
if ! type -P gsutil; then
|
||||
if ! type -P gsutil >/dev/null; then
|
||||
if [[ ! -x "out/gsutil/gsutil" ]]; then
|
||||
echo "Installing gsutil to $(pwd)/out ..."
|
||||
curl -s https://storage.googleapis.com/pub/gsutil.tar.gz | tar -C out/ -zxf -
|
||||
fi
|
||||
export PATH="$(pwd)/out/gsutil:$PATH"
|
||||
PATH="$(pwd)/out/gsutil:$PATH"
|
||||
fi
|
||||
|
||||
# Add the out/ directory to the PATH, for using new drivers.
|
||||
export PATH="$(pwd)/out/":$PATH
|
||||
gsutil -m cp gs://minikube-builds/${MINIKUBE_LOCATION}/minikube-${OS_ARCH} out/
|
||||
gsutil -m cp gs://minikube-builds/${MINIKUBE_LOCATION}/docker-machine-driver-* out/
|
||||
gsutil -m cp gs://minikube-builds/${MINIKUBE_LOCATION}/e2e-${OS_ARCH} out/
|
||||
gsutil -m cp gs://minikube-builds/${MINIKUBE_LOCATION}/testdata/* testdata/
|
||||
PATH="$(pwd)/out/":$PATH
|
||||
export PATH
|
||||
|
||||
echo ""
|
||||
echo ">> Downloading test inputs from ${MINIKUBE_LOCATION} ..."
|
||||
gsutil -qm cp \
|
||||
"gs://minikube-builds/${MINIKUBE_LOCATION}/minikube-${OS_ARCH}" \
|
||||
"gs://minikube-builds/${MINIKUBE_LOCATION}/docker-machine-driver"-* \
|
||||
"gs://minikube-builds/${MINIKUBE_LOCATION}/e2e-${OS_ARCH}" out
|
||||
|
||||
gsutil -qm cp "gs://minikube-builds/${MINIKUBE_LOCATION}/testdata"/* testdata/
|
||||
|
||||
# Set the executable bit on the e2e binary and out binary
|
||||
chmod +x out/e2e-${OS_ARCH}
|
||||
chmod +x out/minikube-${OS_ARCH}
|
||||
chmod +x out/docker-machine-driver-*
|
||||
export MINIKUBE_BIN="out/minikube-${OS_ARCH}"
|
||||
export E2E_BIN="out/e2e-${OS_ARCH}"
|
||||
chmod +x "${MINIKUBE_BIN}" "${E2E_BIN}" out/docker-machine-driver-*
|
||||
|
||||
# Fix permissions in $HOME
|
||||
sudo chown -R $USER $HOME/.kube || true
|
||||
sudo chown -R $USER $HOME/.minikube || true
|
||||
echo ""
|
||||
echo ">> Cleaning up after previous test runs ..."
|
||||
|
||||
export MINIKUBE_WANTREPORTERRORPROMPT=False
|
||||
sudo ./out/minikube-${OS_ARCH} delete || true
|
||||
./out/minikube-${OS_ARCH} delete || true
|
||||
# Cleanup stale test outputs.
|
||||
pgrep minikube && echo "WARNING: other instances of minikube may be running"
|
||||
|
||||
for stale_dir in "${TEST_ROOT}"/*; do
|
||||
echo "* Cleaning stale test: ${stale_dir}"
|
||||
export MINIKUBE_HOME="${stale_dir}/.minikube"
|
||||
export KUBECONFIG="${stale_dir}/kubeconfig"
|
||||
|
||||
# Linux cleanup
|
||||
virsh -c qemu:///system list --all \
|
||||
| sed -n '3,$ p' \
|
||||
| cut -d' ' -f 7 \
|
||||
| xargs -I {} sh -c "virsh -c qemu:///system destroy {}; virsh -c qemu:///system undefine {}" \
|
||||
|| true
|
||||
if [[ -d "${MINIKUBE_HOME}" ]]; then
|
||||
echo "Shutting down stale minikube instance ..."
|
||||
if [[ -w "${MINIKUBE_HOME}" ]]; then
|
||||
"${MINIKUBE_BIN}" delete || true
|
||||
rm -Rf "${MINIKUBE_HOME}"
|
||||
else
|
||||
sudo -E "${MINIKUBE_BIN}" delete || true
|
||||
sudo rm -Rf "${MINIKUBE_HOME}"
|
||||
fi
|
||||
fi
|
||||
|
||||
# Clean up virtualbox VMs
|
||||
vboxmanage list vms \
|
||||
| cut -d "{" -f2 \
|
||||
| cut -d "}" -f1 \
|
||||
| xargs -I {} sh -c "vboxmanage startvm {} --type emergencystop; vboxmanage unregistervm {} --delete" \
|
||||
|| true
|
||||
if [[ -f "${KUBECONFIG}" ]]; then
|
||||
if [[ -w "${KUBECONFIG}" ]]; then
|
||||
rm -f "${KUBECONFIG}"
|
||||
else
|
||||
sudo rm -f "${KUBECONFIG}" || true
|
||||
fi
|
||||
fi
|
||||
|
||||
# Clean up xhyve disks
|
||||
hdiutil info \
|
||||
| egrep \/dev\/disk[1-9][^s] \
|
||||
rmdir "${stale_dir}" || true
|
||||
done
|
||||
|
||||
if type -P virsh; then
|
||||
virsh -c qemu:///system list --all
|
||||
virsh -c qemu:///system list --all \
|
||||
| grep minikube \
|
||||
| awk '{ print $2 }' \
|
||||
| xargs -I {} sh -c "virsh -c qemu:///system destroy {}; virsh -c qemu:///system undefine {}"
|
||||
fi
|
||||
|
||||
if type -P vboxmanage; then
|
||||
vboxmanage list vms
|
||||
vboxmanage list vms \
|
||||
| cut -d'"' -f2 \
|
||||
| xargs -I {} sh -c "vboxmanage startvm {} --type emergencystop; vboxmanage unregistervm {} --delete" \
|
||||
|| true
|
||||
fi
|
||||
|
||||
if type -P hdiutil; then
|
||||
hdiutil info | grep -E "/dev/disk[1-9][^s]"
|
||||
hdiutil info \
|
||||
| grep -E "/dev/disk[1-9][^s]" \
|
||||
| awk '{print $1}' \
|
||||
| xargs -I {} sh -c "hdiutil detach {}" \
|
||||
|| true
|
||||
|
||||
# Clean up xhyve processes
|
||||
pgrep xhyve | xargs kill || true
|
||||
|
||||
|
||||
if [ -e out/docker-machine-driver-hyperkit ]; then
|
||||
sudo chown root:wheel out/docker-machine-driver-hyperkit || true
|
||||
sudo chmod u+s out/docker-machine-driver-hyperkit || true
|
||||
fi
|
||||
|
||||
# See the default image
|
||||
./out/minikube-${OS_ARCH} start -h | grep iso
|
||||
if [[ "${OS_ARCH}" == "Darwin" ]]; then
|
||||
pgrep xhyve | xargs kill || true
|
||||
if [[ -e out/docker-machine-driver-hyperkit ]]; then
|
||||
sudo chown root:wheel out/docker-machine-driver-hyperkit || true
|
||||
sudo chmod u+s out/docker-machine-driver-hyperkit || true
|
||||
fi
|
||||
fi
|
||||
|
||||
# see what driver we are using
|
||||
which docker-machine-driver-${VM_DRIVER} || true
|
||||
find ~/.minikube || true
|
||||
if pgrep kubectl; then
|
||||
echo "stale kubectl processes found: $(pgrep kubectl)"
|
||||
pgrep kubectl | xargs kill
|
||||
fi
|
||||
|
||||
# Allow this to fail, we'll switch on the return code below.
|
||||
set +e
|
||||
${SUDO_PREFIX}out/e2e-${OS_ARCH} \
|
||||
# Respected by minikube
|
||||
export MINIKUBE_HOME="${TEST_HOME}/.minikube"
|
||||
export MINIKUBE_WANTREPORTERRORPROMPT=False
|
||||
|
||||
# Respected by kubectl
|
||||
export KUBECONFIG="${TEST_HOME}/kubeconfig"
|
||||
|
||||
# Display the default image URL
|
||||
echo ""
|
||||
echo ">> ISO URL"
|
||||
"${MINIKUBE_BIN}" start -h | grep iso-url
|
||||
|
||||
echo ""
|
||||
echo ">> Starting ${E2E_BIN} at $(date)"
|
||||
${SUDO_PREFIX}${E2E_BIN} \
|
||||
-minikube-start-args="--vm-driver=${VM_DRIVER} ${EXTRA_START_ARGS}" \
|
||||
-minikube-args="--v=10 --logtostderr ${EXTRA_ARGS}" \
|
||||
-test.v -test.timeout=30m -binary=out/minikube-${OS_ARCH}
|
||||
result=$?
|
||||
set -e
|
||||
|
||||
# See the KUBECONFIG file for debugging
|
||||
sudo cat $KUBECONFIG
|
||||
|
||||
MINIKUBE_WANTREPORTERRORPROMPT=False sudo ./out/minikube-${OS_ARCH} delete \
|
||||
|| MINIKUBE_WANTREPORTERRORPROMPT=False ./out/minikube-${OS_ARCH} delete \
|
||||
|| true
|
||||
-test.v -test.timeout=30m -binary="${MINIKUBE_BIN}" && result=$? || result=$?
|
||||
echo ">> ${E2E_BIN} exited with ${result} at $(date)"
|
||||
echo ""
|
||||
|
||||
if [[ $result -eq 0 ]]; then
|
||||
status="success"
|
||||
echo "minikube: SUCCESS"
|
||||
else
|
||||
status="failure"
|
||||
echo "minikube: FAIL"
|
||||
source print-debug-info.sh
|
||||
fi
|
||||
|
||||
set +x
|
||||
target_url="https://storage.googleapis.com/minikube-builds/logs/${MINIKUBE_LOCATION}/${JOB_NAME}.txt"
|
||||
echo ">> Cleaning up after ourselves ..."
|
||||
${SUDO_PREFIX}${MINIKUBE_BIN} delete >/dev/null 2>/dev/null || true
|
||||
${SUDO_PREFIX} rm -Rf "${MINIKUBE_HOME}"
|
||||
${SUDO_PREFIX} rm -f "${KUBECONFIG}"
|
||||
rmdir "${TEST_HOME}"
|
||||
echo ">> ${TEST_HOME} completed at $(date)"
|
||||
|
||||
readonly target_url="https://storage.googleapis.com/minikube-builds/logs/${MINIKUBE_LOCATION}/${JOB_NAME}.txt"
|
||||
curl "https://api.github.com/repos/kubernetes/minikube/statuses/${COMMIT}?access_token=$access_token" \
|
||||
-H "Content-Type: application/json" \
|
||||
-X POST \
|
||||
-d "{\"state\": \"$status\", \"description\": \"Jenkins\", \"target_url\": \"$target_url\", \"context\": \"${JOB_NAME}\"}"
|
||||
set -x
|
||||
|
||||
exit $result
|
||||
exit $result
|
||||
|
|
@ -20,22 +20,40 @@
|
|||
# the cluster state.
|
||||
set +e
|
||||
|
||||
env
|
||||
${SUDO_PREFIX} cat $KUBECONFIG
|
||||
|
||||
kubectl get pods --all-namespaces
|
||||
kubectl cluster-info dump
|
||||
|
||||
cat $HOME/.kube/config
|
||||
echo $PATH
|
||||
echo ""
|
||||
echo ">>> print-debug-info at $(date):"
|
||||
echo ""
|
||||
${SUDO_PREFIX} cat "${KUBECONFIG}"
|
||||
kubectl version \
|
||||
&& kubectl get pods --all-namespaces \
|
||||
&& kubectl cluster-info dump
|
||||
|
||||
docker ps
|
||||
|
||||
MINIKUBE=${SUDO_PREFIX}out/minikube-${OS_ARCH}
|
||||
${MINIKUBE} status
|
||||
${MINIKUBE} ip
|
||||
${MINIKUBE} ssh -- cat /etc/VERSION
|
||||
${MINIKUBE} ssh -- docker ps
|
||||
${MINIKUBE} logs
|
||||
|
||||
# minikube has probably been shut down, so iterate forward each command rather than spamming.
|
||||
${MINIKUBE} status \
|
||||
&& ${MINIKUBE} ip \
|
||||
&& ${MINIKUBE} ssh -- cat /etc/VERSION \
|
||||
&& ${MINIKUBE} logs \
|
||||
&& ${MINIKUBE} ssh -- docker ps
|
||||
|
||||
if type -P virsh; then
|
||||
virsh -c qemu:///system list --all
|
||||
fi
|
||||
|
||||
if type -P vboxmanage; then
|
||||
vboxmanage list vms
|
||||
fi
|
||||
|
||||
if type -P hdiutil; then
|
||||
hdiutil info | grep -E "/dev/disk[1-9][^s]"
|
||||
fi
|
||||
|
||||
ps -afe | grep kube
|
||||
|
||||
echo ""
|
||||
echo ">>> end print-debug-info"
|
||||
echo ""
|
||||
set -e
|
||||
|
|
|
|||
Loading…
Reference in New Issue