311 lines
9.3 KiB
Bash
Executable File
311 lines
9.3 KiB
Bash
Executable File
#!/bin/bash
|
|
|
|
# Copyright 2021 The Kubernetes Authors All rights reserved.
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
# you may not use this file except in compliance with the License.
|
|
# You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
|
|
# Gather data comparing the overhead of multiple local Kubernetes (macOS and linux)
|
|
readonly TESTS=$1
|
|
|
|
# How many iterations to cycle through
|
|
readonly TEST_ITERATIONS=10
|
|
|
|
# How long to poll CPU usage for (each point is an average over this period)
|
|
readonly POLL_DURATION=5s
|
|
|
|
# How long to measure background usage for. 5 minutes too short, 10 minutes too long
|
|
readonly TOTAL_DURATION=5m
|
|
|
|
# How all tests will be identified
|
|
readonly SESSION_ID="$(date +%Y%m%d-%H%M%S)-$$"
|
|
|
|
# OS Type
|
|
readonly OS=$(uname)
|
|
|
|
measure() {
|
|
local name=$1
|
|
local iteration=$2
|
|
local totalduration=$3
|
|
local filename="benchmark-results/${SESSION_ID}/cstat.${name}.$$-${iteration}"
|
|
|
|
echo ""
|
|
echo " >> Current top processes by CPU:"
|
|
if [[ "${OS}" == "Darwin" ]]; then
|
|
top -n 3 -l 2 -s 2 -o cpu | tail -n4 | awk '{ print $1 " " $2 " " $3 " " $4 }'
|
|
elif [[ "${OS}" == "Linux" ]]; then
|
|
top -b -n 3 -o %CPU | head -n 9
|
|
fi
|
|
|
|
if [[ "${iteration}" == 0 ]]; then
|
|
echo "NOTE: dry-run iteration: will not record measurements"
|
|
cstat --poll "${POLL_DURATION}" --for "${POLL_DURATION}" --busy
|
|
return
|
|
fi
|
|
|
|
echo ""
|
|
echo " >> Measuring ${name} and saving to out/${filename} ..."
|
|
if [[ "${totalduration}" != "" ]]; then
|
|
cstat --poll "${POLL_DURATION}" --for "${totalduration}" --busy --header=false | tee "$(pwd)/out/${filename}"
|
|
else
|
|
cstat --poll "${POLL_DURATION}" --for "${TOTAL_DURATION}" --busy --header=false | tee "$(pwd)/out/${filename}"
|
|
fi
|
|
}
|
|
|
|
|
|
cleanup() {
|
|
echo " >> Deleting local clusters and Docker containers ..."
|
|
out/minikube delete --all 2>/dev/null >/dev/null
|
|
k3d cluster delete 2>/dev/null >/dev/null
|
|
kind delete cluster 2>/dev/null >/dev/null
|
|
docker stop $(docker ps -q) 2>/dev/null
|
|
docker kill $(docker ps -q) 2>/dev/null
|
|
docker rm $(docker ps -a -q) 2>/dev/null
|
|
sleep 2
|
|
}
|
|
|
|
pause_if_running_apps() {
|
|
while true; do
|
|
local apps=$(osascript -e 'tell application "System Events" to get name of (processes where background only is false)' | tr ',' '\n' | sed s/"^ "//g)
|
|
local quiet=0
|
|
|
|
for app in $apps; do
|
|
quiet=1
|
|
if [[ "${app}" != "Terminal" && "${app}" != "Finder" ]]; then
|
|
echo "Unexpected application running: \"${app}\" - will sleep"
|
|
quiet=0
|
|
fi
|
|
done
|
|
|
|
pmset -g batt | grep 'AC Power'
|
|
if [[ "$?" != 0 ]]; then
|
|
echo "waiting to be plugged in ..."
|
|
sleep 5
|
|
continue
|
|
fi
|
|
|
|
if [[ "${quiet}" == 1 ]]; then
|
|
break
|
|
else
|
|
echo "waiting for apps to be closed ..."
|
|
sleep 5
|
|
fi
|
|
|
|
done
|
|
}
|
|
|
|
fail() {
|
|
local name=$1
|
|
local iteration=$2
|
|
|
|
echo '***********************************************************************'
|
|
echo "${name} failed on iteration ${iteration} - will not record measurement"
|
|
echo '***********************************************************************'
|
|
|
|
if [[ "${iteration}" == 0 ]]; then
|
|
echo "test environment appears invalid, exiting"
|
|
exit 90
|
|
fi
|
|
}
|
|
|
|
start_docker() {
|
|
local docker_up=0
|
|
local started=0
|
|
|
|
while [[ "${docker_up}" == 0 ]]; do
|
|
docker info >/dev/null && docker_up=1 || docker_up=0
|
|
|
|
if [[ "${docker_up}" == 0 && "${started}" == 0 ]]; then
|
|
if [[ "${OS}" == "Darwin" ]]; then
|
|
echo ""
|
|
echo " >> Starting Docker for Desktop ..."
|
|
open -a Docker
|
|
started=1
|
|
elif [[ "${OS}" == "Linux" ]]; then
|
|
echo ""
|
|
echo " >> Starting Docker Engine ..."
|
|
sudo systemctl start docker
|
|
started=1
|
|
fi
|
|
fi
|
|
|
|
sleep 1
|
|
done
|
|
|
|
# Give time for d4d Kubernetes to begin, if it's around
|
|
if [[ "${started}" == 1 ]]; then
|
|
sleep 60
|
|
fi
|
|
}
|
|
|
|
|
|
main() {
|
|
# check if cstat is installed
|
|
CSTAT=$(which cstat)
|
|
if [[ "$?" != 0 ]]; then
|
|
echo "cstat in not installed. Install cstat at https://github.com/tstromberg/cstat"
|
|
exit 1
|
|
fi
|
|
|
|
echo "----[ versions ]------------------------------------"
|
|
k3d version || { echo "k3d version failed. Please install latest k3d"; exit 1; }
|
|
kind version || { echo "kind version failed. Please install latest kind"; exit 1; }
|
|
out/minikube version || { echo "minikube version failed"; exit 1; }
|
|
docker version
|
|
echo "----------------------------------------------------"
|
|
echo ""
|
|
|
|
echo "Session ID: ${SESSION_ID}"
|
|
mkdir -p "out/benchmark-results/${SESSION_ID}"
|
|
|
|
echo ""
|
|
|
|
if [[ "${OS}" == "Darwin" ]]; then
|
|
echo "Turning on Wi-Fi for initial downloads"
|
|
networksetup -setairportpower Wi-Fi on
|
|
fi
|
|
|
|
for i in $(seq 0 ${TEST_ITERATIONS}); do
|
|
echo ""
|
|
echo "==> session ${SESSION_ID}, iteration $i"
|
|
|
|
cleanup
|
|
|
|
if [[ "$i" = 0 ]]; then
|
|
echo "NOTE: The 0 iteration is an unmeasured dry run!"
|
|
else
|
|
if [[ "${OS}" == "Darwin" ]]; then
|
|
pause_if_running_apps
|
|
echo "Turning off Wi-Fi to remove background noise"
|
|
networksetup -setairportpower Wi-Fi off
|
|
|
|
echo " >> Killing Docker for Desktop ..."
|
|
osascript -e 'quit app "Docker"'
|
|
elif [[ "${OS}" == "Linux" ]]; then
|
|
echo " >> Killing Docker Engine ..."
|
|
sudo systemctl stop docker
|
|
fi
|
|
|
|
# Measure the background noise on this system
|
|
sleep 15
|
|
measure idle $i
|
|
fi
|
|
|
|
# Run cleanup once we can assert that Docker is up
|
|
start_docker
|
|
cleanup
|
|
|
|
docker_k8s=0
|
|
# depending on whether Docker for Mac Kubernetes is enabled
|
|
if [[ "${OS}" == "Darwin" ]]; then
|
|
# wait kubernetes system pods for Docker for Mac, if it is enabled
|
|
sleep 60
|
|
kubectl --context docker-desktop version
|
|
|
|
# measure Docker for Mac Kubernetes
|
|
if [[ $? == 0 ]]; then
|
|
echo "Kubernetes is running in Docker for Desktop - adjusting tests"
|
|
docker_k8s=1
|
|
kubectl create deployment nginx --image=nginx:1.20.0
|
|
measure docker_k8s $i
|
|
echo "end of measurement for Docker for Desktop"
|
|
kubectl delete deployment nginx
|
|
# measure Docker idle
|
|
else
|
|
kubectl create deployment nginx --image=nginx:1.20.0
|
|
measure docker $i
|
|
echo "end of measurement for Docker for Desktop Kubernetes"
|
|
kubectl delete deployment nginx
|
|
fi
|
|
# measure Docker idle only
|
|
elif [[ "${OS}" == "Linux" ]]; then
|
|
measure docker $i
|
|
fi
|
|
|
|
echo ""
|
|
echo "-> k3d"
|
|
time k3d cluster create
|
|
echo "-> deploy nginx deployment"
|
|
kubectl create deployment nginx --image=nginx:1.20.0
|
|
measure k3d $i || fail k3d $i
|
|
cleanup
|
|
|
|
echo ""
|
|
echo "-> kind"
|
|
time kind create cluster
|
|
echo "-> deploy nginx deployment"
|
|
kubectl create deployment nginx --image=nginx:1.20.0
|
|
measure kind $i || fail kind $i
|
|
cleanup
|
|
|
|
# test different drivers
|
|
if [[ "${OS}" == "Darwin" ]]; then
|
|
drivers=(docker hyperkit virtualbox)
|
|
elif [[ "${OS}" == "Linux" ]]; then
|
|
drivers=(docker kvm2 virtualbox)
|
|
fi
|
|
|
|
for driver in "${drivers[@]}"; do
|
|
echo ""
|
|
# 1. start minikube cluster
|
|
echo "-> out/minikube --driver=${driver}"
|
|
time out/minikube start --driver "${driver}"
|
|
|
|
#2. deploy sample application(nginx deployment)
|
|
echo "-> deploy nginx deployment"
|
|
kubectl create deployment nginx --image=nginx:1.20.0
|
|
|
|
#3. wait 1 minute without anything and 4. measure No.3 idle CPU usage
|
|
measure "minikube.${driver}.nonautopause" $i "1m" || fail "minikube.${driver}.nonautopause" $i
|
|
|
|
# 5. enable auto-pause addons
|
|
echo "-> enable auto-pause to control plane"
|
|
out/minikube addons enable auto-pause
|
|
|
|
# 6. wait 1 minute so that control plane will become Paused status
|
|
pause=0
|
|
while [ "${pause}" = 0 ]
|
|
do
|
|
# It takes 1 minute to become Pause status from Stopped status. 70s is a number with a margin
|
|
sleep 70
|
|
# 7. verify if minikube control plane is paused
|
|
PAUSE=$(out/minikube status)
|
|
echo $PAUSE | grep "apiserver: Paused"
|
|
if [[ "$?" == 0 ]]; then
|
|
echo "kube-apiserver is paused"
|
|
pause=1
|
|
else
|
|
echo "...status is not Paused. wait for becoming Pause..."
|
|
fi
|
|
done
|
|
|
|
# 8. wait 3 minute without anything and 9. measure No.8 idle CPU usage
|
|
measure "minikube.${driver}.autopause" $i "3m" || fail "minikube.${driver}.autopause" $i
|
|
cleanup
|
|
|
|
# We won't be needing docker for the remaining tests this iteration
|
|
if [[ "${OS}" == "Darwin" && "${driver}" == "docker" ]]; then
|
|
echo " >> Quitting Docker for Desktop ..."
|
|
osascript -e 'quit app "Docker"'
|
|
elif [[ "${OS}" == "Linux" && ${driver} == "docker" ]]; then
|
|
echo " >> Quitting Docker Engine ..."
|
|
sudo systemctl stop docker
|
|
fi
|
|
done ## driver
|
|
done ## iteration
|
|
}
|
|
|
|
main "$@"
|
|
# update benchmark result into docs contents
|
|
./hack/benchmark/cpu_usage/auto_pause/update_summary.sh "${SESSION_ID}"
|
|
go run ./hack/benchmark/cpu_usage/auto_pause/chart.go "${SESSION_ID}"
|