From 305209e869bd40040de1237ee98061349e425d32 Mon Sep 17 00:00:00 2001
From: Thomas Stromberg <tstromberg@google.com>
Date: Thu, 17 Oct 2019 12:19:43 -0700
Subject: [PATCH] Automatically install periodic cleanup script on macOS and
 Linux

---
 hack/jenkins/cron/Darwin.crontab              |  1 +
 .../cleanup-and-reboot_Darwin.sh}             |  0
 hack/jenkins/cron/cleanup-and-reboot_Linux.sh | 39 +++++++++++++++++++
 hack/jenkins/linux_integration_tests_kvm.sh   |  2 +
 hack/jenkins/linux_integration_tests_none.sh  |  2 +
 .../linux_integration_tests_virtualbox.sh     |  2 +
 .../jenkins/osx_integration_tests_hyperkit.sh |  3 ++
 .../osx_integration_tests_virtualbox.sh       |  3 ++
 8 files changed, 52 insertions(+)
 create mode 100644 hack/jenkins/cron/Darwin.crontab
 rename hack/jenkins/{osx_cleanup_and_reboot.sh => cron/cleanup-and-reboot_Darwin.sh} (100%)
 create mode 100755 hack/jenkins/cron/cleanup-and-reboot_Linux.sh

diff --git a/hack/jenkins/cron/Darwin.crontab b/hack/jenkins/cron/Darwin.crontab
new file mode 100644
index 0000000000..58dd2d5477
--- /dev/null
+++ b/hack/jenkins/cron/Darwin.crontab
@@ -0,0 +1 @@
+*/20 * * * * /Users/jenkins/cleanup-and-reboot.sh
diff --git a/hack/jenkins/osx_cleanup_and_reboot.sh b/hack/jenkins/cron/cleanup-and-reboot_Darwin.sh
similarity index 100%
rename from hack/jenkins/osx_cleanup_and_reboot.sh
rename to hack/jenkins/cron/cleanup-and-reboot_Darwin.sh
diff --git a/hack/jenkins/cron/cleanup-and-reboot_Linux.sh b/hack/jenkins/cron/cleanup-and-reboot_Linux.sh
new file mode 100755
index 0000000000..246b0e0bea
--- /dev/null
+++ b/hack/jenkins/cron/cleanup-and-reboot_Linux.sh
@@ -0,0 +1,39 @@
+#!/bin/bash
+# Periodically cleanup and reboot if no Jenkins subprocesses are running.
+#
+# Installation:
+#   install cleanup-and-reboot.linux /etc/cron.hourly/cleanup-and-reboot
+
+function check_jenkins() {
+  jenkins_pid="$(pidof java)"
+  if [[ "${jenkins_pid}" = "" ]]; then
+          return
+  fi
+  pstree "${jenkins_pid}" \
+        | grep -v java \
+        && echo "jenkins is running at pid ${jenkins_pid} ..." \
+        && exit 1
+}
+
+check_jenkins
+logger "cleanup-and-reboot running - may shutdown in 60 seconds"
+wall "cleanup-and-reboot running - may shutdown in 60 seconds"
+
+sleep 60
+
+check_jenkins
+logger "cleanup-and-reboot is happening!"
+
+# kill jenkins to avoid an incoming request
+killall java
+
+# disable localkube, kubelet
+systemctl list-unit-files --state=enabled \
+        | grep kube \
+        | awk '{ print $1 }' \
+        | xargs systemctl disable
+
+# update and reboot
+apt update -y \
+        && apt upgrade -y \
+        && reboot
diff --git a/hack/jenkins/linux_integration_tests_kvm.sh b/hack/jenkins/linux_integration_tests_kvm.sh
index 5af0e48b30..5137b76a02 100755
--- a/hack/jenkins/linux_integration_tests_kvm.sh
+++ b/hack/jenkins/linux_integration_tests_kvm.sh
@@ -30,5 +30,7 @@ VM_DRIVER="kvm2"
 JOB_NAME="KVM_Linux"
 PARALLEL_COUNT=4
 
+install cron/cleanup-and-reboot.linux /etc/cron.hourly/cleanup-and-reboot
+
 # Download files and set permissions
 source ./common.sh
diff --git a/hack/jenkins/linux_integration_tests_none.sh b/hack/jenkins/linux_integration_tests_none.sh
index 6721d15e24..eb83215460 100755
--- a/hack/jenkins/linux_integration_tests_none.sh
+++ b/hack/jenkins/linux_integration_tests_none.sh
@@ -50,5 +50,7 @@ systemctl is-active --quiet kubelet \
   && echo "stopping kubelet" \
   && sudo systemctl stop kubelet
 
+install cron/cleanup-and-reboot.linux /etc/cron.hourly/cleanup-and-reboot
+
 # Download files and set permissions
 source ./common.sh
diff --git a/hack/jenkins/linux_integration_tests_virtualbox.sh b/hack/jenkins/linux_integration_tests_virtualbox.sh
index bd413c1586..a4b61833e1 100755
--- a/hack/jenkins/linux_integration_tests_virtualbox.sh
+++ b/hack/jenkins/linux_integration_tests_virtualbox.sh
@@ -30,5 +30,7 @@ VM_DRIVER="virtualbox"
 JOB_NAME="VirtualBox_Linux"
 PARALLEL_COUNT=4
 
+install cron/cleanup-and-reboot.linux /etc/cron.hourly/cleanup-and-reboot
+
 # Download files and set permissions
 source ./common.sh
diff --git a/hack/jenkins/osx_integration_tests_hyperkit.sh b/hack/jenkins/osx_integration_tests_hyperkit.sh
index 7091d6512b..6c98564d60 100755
--- a/hack/jenkins/osx_integration_tests_hyperkit.sh
+++ b/hack/jenkins/osx_integration_tests_hyperkit.sh
@@ -33,5 +33,8 @@ EXTRA_ARGS="--bootstrapper=kubeadm"
 EXTRA_START_ARGS=""
 PARALLEL_COUNT=3
 
+install cron/cleanup_and_reboot_Darwin.sh $HOME/cleanup_and_reboot.sh
+crontab < cron/Darwin.crontab
+
 # Download files and set permissions
 source common.sh
diff --git a/hack/jenkins/osx_integration_tests_virtualbox.sh b/hack/jenkins/osx_integration_tests_virtualbox.sh
index cb48a389ed..ad67abdcc8 100755
--- a/hack/jenkins/osx_integration_tests_virtualbox.sh
+++ b/hack/jenkins/osx_integration_tests_virtualbox.sh
@@ -31,5 +31,8 @@ JOB_NAME="VirtualBox_macOS"
 EXTRA_ARGS="--bootstrapper=kubeadm"
 PARALLEL_COUNT=3
 
+install cron/cleanup_and_reboot_Darwin.sh $HOME/cleanup_and_reboot.sh
+crontab < cron/Darwin.crontab
+
 # Download files and set permissions
 source common.sh