milvus-io
/
milvus
mirror of https://github.com/milvus-io/milvus.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity
milvus/shards/kubernetes_demo/start.sh

369 lines
13 KiB
Bash
Executable File
Raw Blame History

#!/bin/bash
UL=`tput smul`
NOUL=`tput rmul`
BOLD=`tput bold`
NORMAL=`tput sgr0`
RED='\033[0;31m'
GREEN='\033[0;32m'
BLUE='\033[0;34m'
YELLOW='\033[1;33m'
ENDC='\033[0m'
function showHelpMessage () {
echo -e "${BOLD}Usage:${NORMAL} ${RED}$0${ENDC} [option...] {cleanup${GREEN}|${ENDC}baseup${GREEN}|${ENDC}appup${GREEN}|${ENDC}appdown${GREEN}|${ENDC}allup}" >&2
echo
echo " -h, --help show help message"
echo " ${BOLD}cleanup, delete all resources${NORMAL}"
echo " ${BOLD}baseup, start all required base resources${NORMAL}"
echo " ${BOLD}appup, start all pods${NORMAL}"
echo " ${BOLD}appdown, remove all pods${NORMAL}"
echo " ${BOLD}allup, start all base resources and pods${NORMAL}"
echo " ${BOLD}scale-proxy, scale proxy${NORMAL}"
echo " ${BOLD}scale-ro-server, scale readonly servers${NORMAL}"
echo " ${BOLD}scale-worker, scale calculation workers${NORMAL}"
}
function showscaleHelpMessage () {
echo -e "${BOLD}Usage:${NORMAL} ${RED}$0 $1${ENDC} [option...] {1|2|3|4|...}" >&2
echo
echo " -h, --help show help message"
echo " ${BOLD}number, (int) target scale number"
}
function PrintScaleSuccessMessage() {
echo -e "${BLUE}${BOLD}Successfully Scaled: ${1} --> ${2}${ENDC}"
}
function PrintPodStatusMessage() {
echo -e "${BOLD}${1}${NORMAL}"
}
timeout=60
function setUpMysql () {
mysqlUserName=$(kubectl describe configmap -n milvus milvus-roserver-configmap |
grep backend_url |
awk '{print $2}' |
awk '{split($0, level1, ":");
split(level1[2], level2, "/");
print level2[3]}')
mysqlPassword=$(kubectl describe configmap -n milvus milvus-roserver-configmap |
grep backend_url |
awk '{print $2}' |
awk '{split($0, level1, ":");
split(level1[3], level3, "@");
print level3[1]}')
mysqlDBName=$(kubectl describe configmap -n milvus milvus-roserver-configmap |
grep backend_url |
awk '{print $2}' |
awk '{split($0, level1, ":");
split(level1[4], level4, "/");
print level4[2]}')
mysqlContainer=$(kubectl get pods -n milvus | grep milvus-mysql | awk '{print $1}')
kubectl exec -n milvus $mysqlContainer -- mysql -h milvus-mysql -u$mysqlUserName -p$mysqlPassword -e "CREATE DATABASE IF NOT EXISTS $mysqlDBName;"
checkDBExists=$(kubectl exec -n milvus $mysqlContainer -- mysql -h milvus-mysql -u$mysqlUserName -p$mysqlPassword -e "SELECT schema_name FROM information_schema.schemata WHERE schema_name = '$mysqlDBName';" | grep -o $mysqlDBName | wc -l)
counter=0
while [ $checkDBExists -lt 1 ]; do
sleep 1
let counter=counter+1
if [ $counter == $timeout ]; then
echo "Creating MySQL database $mysqlDBName timeout"
return 1
fi
checkDBExists=$(kubectl exec -n milvus $mysqlContainer -- mysql -h milvus-mysql -u$mysqlUserName -p$mysqlPassword -e "SELECT schema_name FROM information_schema.schemata WHERE schema_name = '$mysqlDBName';" | grep -o $mysqlDBName | wc -l)
done;
kubectl exec -n milvus $mysqlContainer -- mysql -h milvus-mysql -u$mysqlUserName -p$mysqlPassword -e "GRANT ALL PRIVILEGES ON $mysqlDBName.* TO '$mysqlUserName'@'%';"
kubectl exec -n milvus $mysqlContainer -- mysql -h milvus-mysql -u$mysqlUserName -p$mysqlPassword -e "FLUSH PRIVILEGES;"
checkGrant=$(kubectl exec -n milvus $mysqlContainer -- mysql -h milvus-mysql -u$mysqlUserName -p$mysqlPassword -e "SHOW GRANTS for $mysqlUserName;" | grep -o "GRANT ALL PRIVILEGES ON \`$mysqlDBName\`\.\*" | wc -l)
counter=0
while [ $checkGrant -lt 1 ]; do
sleep 1
let counter=counter+1
if [ $counter == $timeout ]; then
echo "Granting all privileges on $mysqlDBName to $mysqlUserName timeout"
return 1
fi
checkGrant=$(kubectl exec -n milvus $mysqlContainer -- mysql -h milvus-mysql -u$mysqlUserName -p$mysqlPassword -e "SHOW GRANTS for $mysqlUserName;" | grep -o "GRANT ALL PRIVILEGES ON \`$mysqlDBName\`\.\*" | wc -l)
done;
}
function checkStatefulSevers() {
stateful_replicas=$(kubectl describe statefulset -n milvus milvus-ro-servers | grep "Replicas:" | awk '{print $2}')
stateful_running_pods=$(kubectl describe statefulset -n milvus milvus-ro-servers | grep "Pods Status:" | awk '{print $3}')
counter=0
prev=$stateful_running_pods
PrintPodStatusMessage "Running milvus-ro-servers Pods: $stateful_running_pods/$stateful_replicas"
while [ $stateful_replicas != $stateful_running_pods ]; do
echo -e "${YELLOW}Wait another 1 sec --- ${counter}${ENDC}"
sleep 1;
let counter=counter+1
if [ $counter -eq $timeout ]; then
return 1;
fi
stateful_running_pods=$(kubectl describe statefulset -n milvus milvus-ro-servers | grep "Pods Status:" | awk '{print $3}')
if [ $stateful_running_pods -ne $prev ]; then
PrintPodStatusMessage "Running milvus-ro-servers Pods: $stateful_running_pods/$stateful_replicas"
fi
prev=$stateful_running_pods
done;
return 0;
}
function checkDeployment() {
deployment_name=$1
replicas=$(kubectl describe deployment -n milvus $deployment_name | grep "Replicas:" | awk '{print $2}')
running=$(kubectl get pods -n milvus | grep $deployment_name | grep Running | wc -l)
counter=0
prev=$running
PrintPodStatusMessage "Running $deployment_name Pods: $running/$replicas"
while [ $replicas != $running ]; do
echo -e "${YELLOW}Wait another 1 sec --- ${counter}${ENDC}"
sleep 1;
let counter=counter+1
if [ $counter == $timeout ]; then
return 1
fi
running=$(kubectl get pods -n milvus | grep "$deployment_name" | grep Running | wc -l)
if [ $running -ne $prev ]; then
PrintPodStatusMessage "Running $deployment_name Pods: $running/$replicas"
fi
prev=$running
done
}
function startDependencies() {
kubectl apply -f milvus_data_pvc.yaml
kubectl apply -f milvus_configmap.yaml
kubectl apply -f milvus_auxiliary.yaml
counter=0
while [ $(kubectl get pvc -n milvus | grep Bound | wc -l) != 4 ]; do
sleep 1;
let counter=counter+1
if [ $counter == $timeout ]; then
echo "baseup timeout"
return 1
fi
done
checkDeployment "milvus-mysql"
}
function startApps() {
counter=0
errmsg=""
echo -e "${GREEN}${BOLD}Checking required resouces...${NORMAL}${ENDC}"
while [ $counter -lt $timeout ]; do
sleep 1;
if [ $(kubectl get pvc -n milvus 2>/dev/null | grep Bound | wc -l) != 4 ]; then
echo -e "${YELLOW}No pvc. Wait another sec... $counter${ENDC}";
errmsg='No pvc';
let counter=counter+1;
continue
fi
if [ $(kubectl get configmap -n milvus 2>/dev/null | grep milvus | wc -l) != 4 ]; then
echo -e "${YELLOW}No configmap. Wait another sec... $counter${ENDC}";
errmsg='No configmap';
let counter=counter+1;
continue
fi
if [ $(kubectl get ep -n milvus 2>/dev/null | grep milvus-mysql | awk '{print $2}') == "<none>" ]; then
echo -e "${YELLOW}No mysql. Wait another sec... $counter${ENDC}";
errmsg='No mysql';
let counter=counter+1;
continue
fi
# if [ $(kubectl get ep -n milvus 2>/dev/null | grep milvus-redis | awk '{print $2}') == "<none>" ]; then
# echo -e "${NORMAL}${YELLOW}No redis. Wait another sec... $counter${ENDC}";
# errmsg='No redis';
# let counter=counter+1;
# continue
# fi
break;
done
if [ $counter -ge $timeout ]; then
echo -e "${RED}${BOLD}Start APP Error: $errmsg${NORMAL}${ENDC}"
exit 1;
fi
echo -e "${GREEN}${BOLD}Setup requried database ...${NORMAL}${ENDC}"
setUpMysql
if [ $? -ne 0 ]; then
echo -e "${RED}${BOLD}Setup MySQL database timeout${NORMAL}${ENDC}"
exit 1
fi
echo -e "${GREEN}${BOLD}Start servers ...${NORMAL}${ENDC}"
kubectl apply -f milvus_stateful_servers.yaml
kubectl apply -f milvus_write_servers.yaml
checkStatefulSevers
if [ $? -ne 0 ]; then
echo -e "${RED}${BOLD}Starting milvus-ro-servers timeout${NORMAL}${ENDC}"
exit 1
fi
checkDeployment "milvus-wo-servers"
if [ $? -ne 0 ]; then
echo -e "${RED}${BOLD}Starting milvus-wo-servers timeout${NORMAL}${ENDC}"
exit 1
fi
echo -e "${GREEN}${BOLD}Start rolebinding ...${NORMAL}${ENDC}"
kubectl apply -f milvus_rbac.yaml
echo -e "${GREEN}${BOLD}Start proxies ...${NORMAL}${ENDC}"
kubectl apply -f milvus_proxy.yaml
checkDeployment "milvus-proxy"
if [ $? -ne 0 ]; then
echo -e "${RED}${BOLD}Starting milvus-proxy timeout${NORMAL}${ENDC}"
exit 1
fi
# echo -e "${GREEN}${BOLD}Start flower ...${NORMAL}${ENDC}"
# kubectl apply -f milvus_flower.yaml
# checkDeployment "milvus-flower"
# if [ $? -ne 0 ]; then
# echo -e "${RED}${BOLD}Starting milvus-flower timeout${NORMAL}${ENDC}"
# exit 1
# fi
}
function removeApps () {
# kubectl delete -f milvus_flower.yaml 2>/dev/null
kubectl delete -f milvus_proxy.yaml 2>/dev/null
kubectl delete -f milvus_stateful_servers.yaml 2>/dev/null
kubectl delete -f milvus_write_servers.yaml 2>/dev/null
kubectl delete -f milvus_rbac.yaml 2>/dev/null
# kubectl delete -f milvus_monitor.yaml 2>/dev/null
}
function scaleDeployment() {
deployment_name=$1
subcommand=$2
des=$3
case $des in
-h|--help|"")
showscaleHelpMessage $subcommand
exit 3
;;
esac
cur=$(kubectl get deployment -n milvus $deployment_name |grep $deployment_name |awk '{split($2, status, "/"); print status[2];}')
echo -e "${GREEN}Current Running ${BOLD}$cur ${GREEN}${deployment_name}, Scaling to ${BOLD}$des ...${ENDC}";
scalecmd="kubectl scale deployment -n milvus ${deployment_name} --replicas=${des}"
${scalecmd}
if [ $? -ne 0 ]; then
echo -e "${RED}${BOLD}Scale Error: ${GREEN}${scalecmd}${ENDC}"
exit 1
fi
checkDeployment $deployment_name
if [ $? -ne 0 ]; then
echo -e "${RED}${BOLD}Scale ${deployment_name} timeout${NORMAL}${ENDC}"
scalecmd="kubectl scale deployment -n milvus ${deployment_name} --replicas=${cur}"
${scalecmd}
if [ $? -ne 0 ]; then
echo -e "${RED}${BOLD}Scale Rollback Error: ${GREEN}${scalecmd}${ENDC}"
exit 2
fi
echo -e "${BLUE}${BOLD}Scale Rollback to ${cur}${ENDC}"
exit 1
fi
PrintScaleSuccessMessage $cur $des
}
function scaleROServers() {
subcommand=$1
des=$2
case $des in
-h|--help|"")
showscaleHelpMessage $subcommand
exit 3
;;
esac
cur=$(kubectl get statefulset -n milvus milvus-ro-servers |tail -n 1 |awk '{split($2, status, "/"); print status[2];}')
echo -e "${GREEN}Current Running ${BOLD}$cur ${GREEN}Readonly Servers, Scaling to ${BOLD}$des ...${ENDC}";
scalecmd="kubectl scale sts milvus-ro-servers -n milvus --replicas=${des}"
${scalecmd}
if [ $? -ne 0 ]; then
echo -e "${RED}${BOLD}Scale Error: ${GREEN}${scalecmd}${ENDC}"
exit 1
fi
checkStatefulSevers
if [ $? -ne 0 ]; then
echo -e "${RED}${BOLD}Scale milvus-ro-servers timeout${NORMAL}${ENDC}"
scalecmd="kubectl scale sts milvus-ro-servers -n milvus --replicas=${cur}"
${scalecmd}
if [ $? -ne 0 ]; then
echo -e "${RED}${BOLD}Scale Rollback Error: ${GREEN}${scalecmd}${ENDC}"
exit 2
fi
echo -e "${BLUE}${BOLD}Scale Rollback to ${cur}${ENDC}"
exit 1
fi
PrintScaleSuccessMessage $cur $des
}
case "$1" in
cleanup)
kubectl delete -f . 2>/dev/null
echo -e "${BLUE}${BOLD}All resources are removed${NORMAL}${ENDC}"
;;
appdown)
removeApps;
echo -e "${BLUE}${BOLD}All pods are removed${NORMAL}${ENDC}"
;;
baseup)
startDependencies;
echo -e "${BLUE}${BOLD}All pvc, configmap and services up${NORMAL}${ENDC}"
;;
appup)
startApps;
echo -e "${BLUE}${BOLD}All pods up${NORMAL}${ENDC}"
;;
allup)
startDependencies;
sleep 2
startApps;
echo -e "${BLUE}${BOLD}All resources and pods up${NORMAL}${ENDC}"
;;
scale-ro-server)
scaleROServers $1 $2
;;
scale-proxy)
scaleDeployment "milvus-proxy" $1 $2
;;
-h|--help|*)
showHelpMessage
;;
esac
Reference in New Issue View Git Blame Copy Permalink