diff --git a/hack/jenkins/osx_cleanup_and_reboot.sh b/hack/jenkins/cron/cleanup_and_reboot_Darwin.sh similarity index 53% rename from hack/jenkins/osx_cleanup_and_reboot.sh rename to hack/jenkins/cron/cleanup_and_reboot_Darwin.sh index 503796c212..993d6c82f6 100755 --- a/hack/jenkins/osx_cleanup_and_reboot.sh +++ b/hack/jenkins/cron/cleanup_and_reboot_Darwin.sh @@ -19,21 +19,29 @@ set -uf -o pipefail PATH=/usr/local/bin:/sbin:/usr/local/sbin:$PATH -exit_if_jenkins() { - jenkins=$(pgrep java) - if [[ "$jenkins" -- "" ]]; then - echo "no java, no jenkins" - return 0 +# cleanup shared between Linux and macOS +function check_jenkins() { + jenkins_pid="$(pidof java)" + if [[ "${jenkins_pid}" = "" ]]; then + return fi - pstree $jenkins | grep -v java && echo "jenkins is running..." && exit 1 + pstree "${jenkins_pid}" \ + | egrep -i 'bash|integration|e2e|minikube' \ + && echo "tests are is running on pid ${jenkins_pid} ..." \ + && exit 1 } -exit_if_jenkins -echo "waiting to see if any jobs are coming in..." -sleep 15 -exit_if_jenkins -echo "doing it" +check_jenkins +logger "cleanup_and_reboot running - may shutdown in 60 seconds" +echo "cleanup_and_reboot running - may shutdown in 60 seconds" | wall +sleep 10 +check_jenkins +logger "cleanup_and_reboot is happening!" + +# kill jenkins to avoid an incoming request killall java -sudo rm -Rf ~jenkins/.minikube || echo "could not delete minikube" -sudo rm -Rf ~/jenkins/minikube-integration/* || true + +# macOS specific cleanup +sudo rm /var/db/dhcpd_leases || echo "could not clear dhcpd leases" +sudo softwareupdate -i -a -R sudo reboot diff --git a/hack/jenkins/cron/cleanup_and_reboot_Linux.sh b/hack/jenkins/cron/cleanup_and_reboot_Linux.sh new file mode 100755 index 0000000000..b2e067c58e --- /dev/null +++ b/hack/jenkins/cron/cleanup_and_reboot_Linux.sh @@ -0,0 +1,48 @@ +#!/bin/bash + +# Copyright 2019 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# cleanup shared between Linux and macOS +function check_jenkins() { + jenkins_pid="$(pidof java)" + if [[ "${jenkins_pid}" = "" ]]; then + return + fi + pstree "${jenkins_pid}" \ + | egrep -i 'bash|integration|e2e|minikube' \ + && echo "tests are is running on pid ${jenkins_pid} ..." \ + && exit 1 +} + +check_jenkins +logger "cleanup_and_reboot running - may shutdown in 60 seconds" +echo "cleanup_and_reboot running - may shutdown in 60 seconds" | wall +sleep 10 +check_jenkins +logger "cleanup_and_reboot is happening!" + +# kill jenkins to avoid an incoming request +killall java + +# Linux-specific cleanup + +# disable localkube, kubelet +systemctl list-unit-files --state=enabled \ + | grep kube \ + | awk '{ print $1 }' \ + | xargs systemctl disable + +# update and reboot +apt update -y && apt upgrade -y && reboot diff --git a/hack/jenkins/linux_integration_tests_kvm.sh b/hack/jenkins/linux_integration_tests_kvm.sh index 19181b70e2..5b51eb5c0f 100755 --- a/hack/jenkins/linux_integration_tests_kvm.sh +++ b/hack/jenkins/linux_integration_tests_kvm.sh @@ -33,5 +33,8 @@ EXPECTED_DEFAULT_DRIVER="kvm2" # We pick kvm as our gvisor testbed because it is fast & reliable EXTRA_TEST_ARGS="-gvisor" +mkdir cron && gsutil -qm rsync "gs://minikube-builds/${MINIKUBE_LOCATION}/cron" cron || echo "FAILED TO GET CRON FILES" +sudo install cron/cleanup_and_reboot_Linux.sh /etc/cron.hourly/cleanup_and_reboot || echo "FAILED TO INSTALL CLEANUP" + # Download files and set permissions source ./common.sh diff --git a/hack/jenkins/linux_integration_tests_none.sh b/hack/jenkins/linux_integration_tests_none.sh index cb1cc4d3a2..5cb2c778f8 100755 --- a/hack/jenkins/linux_integration_tests_none.sh +++ b/hack/jenkins/linux_integration_tests_none.sh @@ -51,5 +51,8 @@ systemctl is-active --quiet kubelet \ && echo "stopping kubelet" \ && sudo systemctl stop kubelet +mkdir -p cron && gsutil -m rsync "gs://minikube-builds/${MINIKUBE_LOCATION}/cron" cron || echo "FAILED TO GET CRON FILES" +sudo install cron/cleanup_and_reboot_Linux.sh /etc/cron.hourly/cleanup_and_reboot || echo "FAILED TO INSTALL CLEANUP" + # Download files and set permissions source ./common.sh diff --git a/hack/jenkins/linux_integration_tests_virtualbox.sh b/hack/jenkins/linux_integration_tests_virtualbox.sh index d159afef5c..1603a42be7 100755 --- a/hack/jenkins/linux_integration_tests_virtualbox.sh +++ b/hack/jenkins/linux_integration_tests_virtualbox.sh @@ -30,5 +30,8 @@ VM_DRIVER="virtualbox" JOB_NAME="VirtualBox_Linux" EXPECTED_DEFAULT_DRIVER="kvm2" +mkdir -p cron && gsutil -m rsync "gs://minikube-builds/${MINIKUBE_LOCATION}/cron" cron +sudo install cleanup_and_reboot_Linux.sh /etc/cron.hourly/cleanup_and_reboot + # Download files and set permissions source ./common.sh diff --git a/hack/jenkins/osx_integration_tests_hyperkit.sh b/hack/jenkins/osx_integration_tests_hyperkit.sh index 8dc49ab367..8805e8a2eb 100755 --- a/hack/jenkins/osx_integration_tests_hyperkit.sh +++ b/hack/jenkins/osx_integration_tests_hyperkit.sh @@ -34,5 +34,10 @@ EXTRA_START_ARGS="" EXPECTED_DEFAULT_DRIVER="hyperkit" +mkdir -p cron && gsutil -qm rsync "gs://minikube-builds/${MINIKUBE_LOCATION}/cron" cron || echo "FAILED TO GET CRON FILES" +install cron/cleanup_and_reboot_Darwin.sh $HOME/cleanup_and_reboot.sh || echo "FAILED TO INSTALL CLEANUP" +echo "*/30 * * * * $HOME/cleanup_and_reboot.sh" | crontab +crontab -l + # Download files and set permissions source common.sh diff --git a/hack/jenkins/osx_integration_tests_virtualbox.sh b/hack/jenkins/osx_integration_tests_virtualbox.sh index 8ebdb846c1..49dcba37e7 100755 --- a/hack/jenkins/osx_integration_tests_virtualbox.sh +++ b/hack/jenkins/osx_integration_tests_virtualbox.sh @@ -34,5 +34,10 @@ EXTRA_ARGS="--bootstrapper=kubeadm" EXPECTED_DEFAULT_DRIVER="hyperkit" +mkdir -p cron && gsutil -qm rsync "gs://minikube-builds/${MINIKUBE_LOCATION}/cron" cron || echo "FAILED TO GET CRON FILES" +install cron/cleanup_and_reboot_Darwin.sh $HOME/cleanup_and_reboot.sh || echo "FAILED TO GET INSTALL CLEANUP" +echo "*/30 * * * * $HOME/cleanup_and_reboot.sh" | crontab +crontab -l + # Download files and set permissions source common.sh