Merge pull request #5648 from tstromberg/cleanup-scripts

Automatically install periodic cleanup script on macOS and Linux

hack/jenkins/cron/cleanup_and_reboot_Darwin.sh

@@ -19,21 +19,29 @@ set -uf -o pipefail
 
 PATH=/usr/local/bin:/sbin:/usr/local/sbin:$PATH
 
-exit_if_jenkins() {
-  jenkins=$(pgrep java)
-  if [[ "$jenkins" -- "" ]]; then
-    echo "no java, no jenkins"
-    return 0
+# cleanup shared between Linux and macOS
+function check_jenkins() {
+  jenkins_pid="$(pidof java)"
+  if [[ "${jenkins_pid}" = "" ]]; then
+          return
   fi
-  pstree $jenkins | grep -v java && echo "jenkins is running..." && exit 1
+  pstree "${jenkins_pid}" \
+        | egrep -i 'bash|integration|e2e|minikube' \
+        && echo "tests are is running on pid ${jenkins_pid} ..." \
+        && exit 1
 }
 
-exit_if_jenkins
-echo "waiting to see if any jobs are coming in..."
-sleep 15
-exit_if_jenkins
-echo "doing it"
+check_jenkins
+logger "cleanup_and_reboot running - may shutdown in 60 seconds"
+echo "cleanup_and_reboot running - may shutdown in 60 seconds" | wall
+sleep 10
+check_jenkins
+logger "cleanup_and_reboot is happening!"
+
+# kill jenkins to avoid an incoming request
 killall java
-sudo rm -Rf ~jenkins/.minikube || echo "could not delete minikube"
-sudo rm -Rf ~/jenkins/minikube-integration/* || true
+
+# macOS specific cleanup
+sudo rm /var/db/dhcpd_leases || echo "could not clear dhcpd leases"
+sudo softwareupdate -i -a -R
 sudo reboot

hack/jenkins/cron/cleanup_and_reboot_Linux.sh

@@ -0,0 +1,48 @@
+#!/bin/bash
+
+# Copyright 2019 The Kubernetes Authors All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# cleanup shared between Linux and macOS
+function check_jenkins() {
+  jenkins_pid="$(pidof java)"
+  if [[ "${jenkins_pid}" = "" ]]; then
+          return
+  fi
+  pstree "${jenkins_pid}" \
+        | egrep -i 'bash|integration|e2e|minikube' \
+        && echo "tests are is running on pid ${jenkins_pid} ..." \
+        && exit 1
+}
+
+check_jenkins
+logger "cleanup_and_reboot running - may shutdown in 60 seconds"
+echo "cleanup_and_reboot running - may shutdown in 60 seconds" | wall
+sleep 10
+check_jenkins
+logger "cleanup_and_reboot is happening!"
+
+# kill jenkins to avoid an incoming request
+killall java
+
+# Linux-specific cleanup
+
+# disable localkube, kubelet
+systemctl list-unit-files --state=enabled \
+        | grep kube \
+        | awk '{ print $1 }' \
+        | xargs systemctl disable
+
+# update and reboot
+apt update -y && apt upgrade -y && reboot

hack/jenkins/linux_integration_tests_kvm.sh

@@ -33,5 +33,8 @@ EXPECTED_DEFAULT_DRIVER="kvm2"
 # We pick kvm as our gvisor testbed because it is fast & reliable
 EXTRA_TEST_ARGS="-gvisor"
 
+mkdir cron && gsutil -qm rsync "gs://minikube-builds/${MINIKUBE_LOCATION}/cron" cron || echo "FAILED TO GET CRON FILES"
+sudo install cron/cleanup_and_reboot_Linux.sh /etc/cron.hourly/cleanup_and_reboot || echo "FAILED TO INSTALL CLEANUP"
+
 # Download files and set permissions
 source ./common.sh

hack/jenkins/linux_integration_tests_none.sh

@@ -51,5 +51,8 @@ systemctl is-active --quiet kubelet \
   && echo "stopping kubelet" \
   && sudo systemctl stop kubelet
 
+mkdir -p cron && gsutil -m rsync "gs://minikube-builds/${MINIKUBE_LOCATION}/cron" cron || echo "FAILED TO GET CRON FILES"
+sudo install cron/cleanup_and_reboot_Linux.sh /etc/cron.hourly/cleanup_and_reboot || echo "FAILED TO INSTALL CLEANUP"
+
 # Download files and set permissions
 source ./common.sh

hack/jenkins/linux_integration_tests_virtualbox.sh

@@ -30,5 +30,8 @@ VM_DRIVER="virtualbox"
 JOB_NAME="VirtualBox_Linux"
 EXPECTED_DEFAULT_DRIVER="kvm2"
 
+mkdir -p cron && gsutil -m rsync "gs://minikube-builds/${MINIKUBE_LOCATION}/cron" cron
+sudo install cleanup_and_reboot_Linux.sh /etc/cron.hourly/cleanup_and_reboot
+
 # Download files and set permissions
 source ./common.sh

hack/jenkins/osx_integration_tests_hyperkit.sh

@@ -34,5 +34,10 @@ EXTRA_START_ARGS=""
 EXPECTED_DEFAULT_DRIVER="hyperkit"
 
 
+mkdir -p cron && gsutil -qm rsync "gs://minikube-builds/${MINIKUBE_LOCATION}/cron" cron || echo "FAILED TO GET CRON FILES"
+install cron/cleanup_and_reboot_Darwin.sh $HOME/cleanup_and_reboot.sh || echo "FAILED TO INSTALL CLEANUP"
+echo "*/30 * * * * $HOME/cleanup_and_reboot.sh" | crontab
+crontab -l
+
 # Download files and set permissions
 source common.sh

hack/jenkins/osx_integration_tests_virtualbox.sh

@@ -34,5 +34,10 @@ EXTRA_ARGS="--bootstrapper=kubeadm"
 EXPECTED_DEFAULT_DRIVER="hyperkit"
 
 
+mkdir -p cron && gsutil -qm rsync "gs://minikube-builds/${MINIKUBE_LOCATION}/cron" cron || echo "FAILED TO GET CRON FILES"
+install cron/cleanup_and_reboot_Darwin.sh $HOME/cleanup_and_reboot.sh  || echo "FAILED TO GET INSTALL CLEANUP"
+echo "*/30 * * * * $HOME/cleanup_and_reboot.sh" | crontab
+crontab -l
+
 # Download files and set permissions
 source common.sh