mirror of https://github.com/milvus-io/milvus.git
Add storageclass check for meta migration (#20804)
Signed-off-by: Edward Zeng <jie.zeng@zilliz.com> Signed-off-by: Edward Zeng <jie.zeng@zilliz.com>pull/20791/head
edward.zeng
GitHub
parent
f588a73719
commit
170bbde0de
|
|
@ -6,22 +6,30 @@ Milvus 2.2 has changed the meta structure for segment index. To upgrade a Milvus
|
|||
|
||||
|
||||
> Note:
|
||||
> 1. This script only applies to Milvus installed on a K8s cluster.
|
||||
> 1. This script only applies to Milvus installed on a K8s cluster which has storageclass.
|
||||
> You can verify storageclass by running `kubectl get storageclass`. If you don't have a
|
||||
> default storageclass, you need specify a storageclass via `-c <storage-class>` when running
|
||||
> this script.
|
||||
> 2. This script only supports upgrading from Milvus v2.1.x to v2.2.0.
|
||||
> 3. If your milvus cluster use an external etcd service (which isn't installed with milvus),
|
||||
> you need specify etcd service explicitly via `-e <etcd-service-ip:etcd-service-port>` otherwise
|
||||
> we use the default etcd service which installed with milvus.
|
||||
|
||||
## Parameters
|
||||
|
||||
| Parameters | Description | Default value | Required |
|
||||
| ------------ | ---------------------------------------------------------------- | ---------------------------- | ----------------------- |
|
||||
| `i` | The Milvus instance name. | `None` | True |
|
||||
| `n` | The namespace that Milvus is installed in. | `default` | False |
|
||||
| `s` | The source Milvus version. | `None` | True |
|
||||
| `t` | The target Milvus version. | `None` | True |
|
||||
| `r` | The root path of Milvus meta. | `by-dev` | False |
|
||||
| `w` | The new Milvus image tag. | `milvusdb/milvus:v2.2.0` | False |
|
||||
| `m` | The meta migration image tag. | `milvusdb/meta-migration:v2.2.0` | False |
|
||||
| `o` | The meta migration operation. | `migrate` | False |
|
||||
| `d` | Whether to delete migration pod after the migration is completed. | `false` | False |
|
||||
| Parameters | Description | Default value | Required |
|
||||
| ------------ | ----------------------------------------------------------| -------------------------------- | ----------------------- |
|
||||
| `i` | The Milvus instance name. | `None` | True |
|
||||
| `n` | The namespace that Milvus is installed in. | `default` | False |
|
||||
| `s` | The source Milvus version. | `None` | True |
|
||||
| `t` | The target Milvus version. | `None` | True |
|
||||
| `r` | The root path of Milvus meta. | `by-dev` | False |
|
||||
| `w` | The new Milvus image tag. | `milvusdb/milvus:v2.2.0` | False |
|
||||
| `m` | The meta migration image tag. | `milvusdb/meta-migration:v2.2.0` | False |
|
||||
| `o` | The meta migration operation. | `migrate` | False |
|
||||
| `d` | Whether to cleanup after migration is completed. | `false` | False |
|
||||
| `c` | The storage class for meta migration pvc. | `default storage class` | False |
|
||||
| `e` | The endpoints for etcd which used by milvus. | `etcd svc installed with milvus` | False |
|
||||
> By default, the script only applies to migration from v2.1.x to v2.2.x. Rollback to the older version with the `rollback` operation first if an error occurs.
|
||||
|
||||
|
||||
|
|
@ -70,3 +78,15 @@ Milvus 2.2 has changed the meta structure for segment index. To upgrade a Milvus
|
|||
./migrate.sh -i my-release -n milvus -s 2.1.1 -t 2.2.0 -r by-dev -o rollback -w <milvus-2-1-1-image>
|
||||
./migrate.sh -i my-release -n milvus -s 2.1.1 -t 2.2.0 -r by-dev -o migrate -w <milvus-2-2-0-image>
|
||||
```
|
||||
|
||||
7. Specify the storage class explicitly.
|
||||
|
||||
```shell
|
||||
./migrate.sh -i my-release -n milvus -s 2.1.4 -t 2.2.0 -c <special-storage-class>
|
||||
```
|
||||
|
||||
8. Specify external etcd service.
|
||||
|
||||
```shell
|
||||
./migrate.sh -i my-release -n milvus -s 2.1.4 -t 2.2.0 -e <etcd-svc-ip:etcd-svc-port>
|
||||
```
|
||||
|
|
|
|||
|
|
@ -6,6 +6,9 @@ operation="migrate"
|
|||
image_tag="milvusdb/milvus:v2.2.0"
|
||||
meta_migration_pod_tag="milvusdb/meta-migration:v2.2.0"
|
||||
remove_migrate_pod_after_migrate="false"
|
||||
storage_class=""
|
||||
external_etcd_svc=""
|
||||
etcd_svc=""
|
||||
|
||||
#-n namespace: The namespace that Milvus is installed in.
|
||||
#-i milvus_instance: The name of milvus instance.
|
||||
|
|
@ -14,8 +17,11 @@ remove_migrate_pod_after_migrate="false"
|
|||
#-r root_path: The milvus meta root path.
|
||||
#-w image_tag: The new milvus image tag.
|
||||
#-o operation: The operation: migrate/rollback.
|
||||
#-m meta_migration_pod_tag: The image for meta migration pod.
|
||||
#-d remove_migrate_pod_after_migrate: Remove migration pod after successful migration.
|
||||
while getopts "n:i:s:t:r:w:o:d:m:" opt_name
|
||||
#-c storage_class: The storage class for meta migration pvc.
|
||||
#-e external_etcd_svc: The endpoints for etcd which used by milvus.
|
||||
while getopts "n:i:s:t:r:w:o:m:c:e:d:" opt_name
|
||||
do
|
||||
case $opt_name in
|
||||
n) namespace=$OPTARG;;
|
||||
|
|
@ -27,6 +33,8 @@ do
|
|||
o) operation=$OPTARG;;
|
||||
m) meta_migration_pod_tag=$OPTARG;;
|
||||
d) remove_migrate_pod_after_migrate="true";;
|
||||
c) storage_class=$OPTARG;;
|
||||
e) external_etcd_svc=$OPTARG;;
|
||||
*) echo "Unkonwen parameters";;
|
||||
esac
|
||||
done
|
||||
|
|
@ -51,6 +59,72 @@ if [ ! $image_tag ]; then
|
|||
exit 1
|
||||
fi
|
||||
|
||||
deployments=$(kubectl get deploy -n $namespace -l app.kubernetes.io/instance=$instance_name,app.kubernetes.io/name=milvus --output=jsonpath={.items..metadata.name})
|
||||
deploys=()
|
||||
replicas=()
|
||||
for d in $deployments; do
|
||||
component=$(kubectl get deploy $d -n $namespace --output=jsonpath={.metadata.labels.component})
|
||||
replica=$(kubectl get deploy $d -n $namespace --output=jsonpath={.spec.replicas})
|
||||
if [ "$component" = "attu" ]; then
|
||||
continue
|
||||
fi
|
||||
deploys+=("$d")
|
||||
done
|
||||
|
||||
if [ ${#deploys[@]} -eq 0 ]; then
|
||||
echo "There is no Milvus instance $instance_name in the namespace $namespace"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ ! $external_etcd_svc ]; then
|
||||
svc=$(kubectl get svc -n $namespace -l app.kubernetes.io/instance=$instance_name,app.kubernetes.io/name=etcd -o name | grep -v headless | cut -d '/' -f 2)
|
||||
|
||||
if [ ! $svc ]; then
|
||||
echo "Missing etcd service, please add it. For example:'./migrate.sh -e <etcd-svc-ip>:<etcd-svc-port>'"
|
||||
exit 1
|
||||
else
|
||||
port=$(kubectl get svc -n $namespace $svc --output=jsonpath='{.spec.ports[?(@.name == "client")].port}')
|
||||
if [ ! $port ]; then
|
||||
echo "Missing etcd service port..."
|
||||
exit 1
|
||||
fi
|
||||
etcd_svc="$svc:$port"
|
||||
fi
|
||||
else
|
||||
etcd_svc="$external_etcd_svc"
|
||||
fi
|
||||
|
||||
scs=$(kubectl get storageclass --output=jsonpath='{.items..metadata.name}')
|
||||
if [ ! $storage_class ]; then
|
||||
# check whether there a default storageclass
|
||||
exists=0
|
||||
for sc in $scs; do
|
||||
default=$(kubectl get storageclass $sc --output=jsonpath='{.metadata.annotations.storageclass\.kubernetes\.io/is-default-class}')
|
||||
if [ "$default" = "true" ] ; then
|
||||
exists=1
|
||||
break
|
||||
fi
|
||||
done
|
||||
|
||||
if [ $exists -eq 0 ] ; then
|
||||
echo "No default storageclass, please specify a storageclass via -c <storage-class-name>"
|
||||
exit 1
|
||||
fi
|
||||
else
|
||||
exists=0
|
||||
for sc in $scs; do
|
||||
if [ "$sc" = "$storage_class" ] ; then
|
||||
exists=1
|
||||
break
|
||||
fi
|
||||
done
|
||||
|
||||
if [ $exists -eq 0 ]; then
|
||||
echo "Nonexistent storageclass, please make sure specified storageclass exists..."
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
|
||||
echo "Migration milvus meta will take four steps:"
|
||||
echo "1. Stop the milvus components"
|
||||
echo "2. Backup the milvus meta"
|
||||
|
|
@ -59,19 +133,34 @@ echo "4. Startup milvus components in new image version"
|
|||
echo
|
||||
|
||||
|
||||
replicas=()
|
||||
function fetch_deploy_replica(){
|
||||
for deploy in $1;
|
||||
do
|
||||
replica=$(kubectl get deploy $deploy -n $namespace --output=jsonpath={.spec.replicas})
|
||||
replicas+=("$deploy:$replica")
|
||||
done
|
||||
function store_deploy_replicas(){
|
||||
# store deploy replicas in a configmap
|
||||
# first check whether config exists
|
||||
data=$(kubectl get configmap "milvus-deploy-replicas-$instance_name" -n $namespace --output=jsonpath={.data.replicas} 2>/dev/null)
|
||||
if [ ! "$data" ]; then
|
||||
for d in ${deploys[@]}; do
|
||||
replica=$(kubectl get deploy $d -n $namespace --output=jsonpath={.spec.replicas})
|
||||
replicas+=("$d:$replica")
|
||||
done
|
||||
cat <<EOF | kubectl apply -n $namespace -f -
|
||||
apiVersion: v1
|
||||
kind: ConfigMap
|
||||
metadata:
|
||||
name: milvus-deploy-replicas-${instance_name}
|
||||
data:
|
||||
replicas: "${replicas[@]}"
|
||||
EOF
|
||||
else
|
||||
for d in $data; do
|
||||
replicas+=("$d")
|
||||
done
|
||||
fi
|
||||
}
|
||||
|
||||
function stop_milvus_deploy(){
|
||||
echo "Stop milvus deployments: $1"
|
||||
kubectl scale deploy -n $namespace $1 --replicas=0 > /dev/null
|
||||
wait_for_milvus_stopped $1
|
||||
echo "Stop milvus deployments: ${deploys[@]}"
|
||||
kubectl scale deploy -n $namespace "${deploys[@]}" --replicas=0 > /dev/null
|
||||
wait_for_milvus_stopped "${deploys[@]}"
|
||||
echo "Stopped..."
|
||||
}
|
||||
|
||||
|
|
@ -172,20 +261,25 @@ function wait_for_rollback_done(){
|
|||
esac
|
||||
done
|
||||
}
|
||||
|
||||
function generate_migrate_meta_cm_and_pvc(){
|
||||
etcd_svc=$(kubectl get svc -n $namespace -l app.kubernetes.io/instance=$instance_name,app.kubernetes.io/name=etcd -o name | grep -v headless | cut -d '/' -f 2)
|
||||
cat <<EOF | kubectl apply -n $namespace -f -
|
||||
apiVersion: v1
|
||||
kind: PersistentVolumeClaim
|
||||
metadata:
|
||||
name: milvus-meta-migration-backup-${instance_name}
|
||||
spec:
|
||||
accessModes:
|
||||
- ReadWriteOnce
|
||||
resources:
|
||||
requests:
|
||||
storage: 10Gi
|
||||
# check whether pvc exists
|
||||
exists=$(kubectl get pvc "milvus-meta-migration-backup-${instance_name}" -n $namespace --output=jsonpath={.metadata.name} 2>/dev/null)
|
||||
if [ ! $exists ]; then
|
||||
cat <<EOF | kubectl apply -n $namespace -f -
|
||||
apiVersion: v1
|
||||
kind: PersistentVolumeClaim
|
||||
metadata:
|
||||
name: milvus-meta-migration-backup-${instance_name}
|
||||
spec:
|
||||
accessModes:
|
||||
- ReadWriteOnce
|
||||
storageClassName: $storage_class
|
||||
resources:
|
||||
requests:
|
||||
storage: 10Gi
|
||||
EOF
|
||||
fi
|
||||
cat <<EOF | kubectl apply -n $namespace -f -
|
||||
apiVersion: v1
|
||||
kind: ConfigMap
|
||||
|
|
@ -206,7 +300,7 @@ EOF
|
|||
|
||||
etcd:
|
||||
endpoints:
|
||||
- $etcd_svc:2379
|
||||
- $etcd_svc
|
||||
rootPath: $root_path
|
||||
metaSubPath: meta
|
||||
kvSubPath: kv
|
||||
|
|
@ -224,7 +318,7 @@ EOF
|
|||
|
||||
etcd:
|
||||
endpoints:
|
||||
- $etcd_svc:2379
|
||||
- $etcd_svc
|
||||
rootPath: $root_path
|
||||
metaSubPath: meta
|
||||
kvSubPath: kv
|
||||
|
|
@ -242,7 +336,7 @@ EOF
|
|||
|
||||
etcd:
|
||||
endpoints:
|
||||
- $etcd_svc:2379
|
||||
- $etcd_svc
|
||||
rootPath: $root_path
|
||||
metaSubPath: meta
|
||||
kvSubPath: kv
|
||||
|
|
@ -254,7 +348,7 @@ function backup_meta(){
|
|||
echo "Checking whether backup exists..."
|
||||
configmap_name="milvus-meta-migration-config-${instance_name}"
|
||||
backup_exists=$(kubectl get configmap $configmap_name -n $namespace --output=jsonpath={.metadata.annotations.backup})
|
||||
if [ "$backup_exists" == "succeed" ]; then
|
||||
if [ "$backup_exists" = "succeed" ]; then
|
||||
echo "Found previous backups, skip meta backup..."
|
||||
else
|
||||
cat <<EOF | kubectl apply -n $namespace -f -
|
||||
|
|
@ -292,7 +386,7 @@ function rollback_meta(){
|
|||
generate_migrate_meta_cm_and_pvc
|
||||
echo "Checking whether backup exists..."
|
||||
backup_exists=$(kubectl get configmap milvus-meta-migration-config -n $namespace --output=jsonpath={.metadata.annotations.backup})
|
||||
if [ "$backup_exists" == "succeed" ]; then
|
||||
if [ "$backup_exists" = "succeed" ]; then
|
||||
echo "Found previous backups, start meta rollback..."
|
||||
cat <<EOF | kubectl apply -n $namespace -f -
|
||||
apiVersion: v1
|
||||
|
|
@ -365,23 +459,16 @@ EOF
|
|||
|
||||
function start_milvus_deploy(){
|
||||
echo "Starting milvus components..."
|
||||
for deploy in "${replicas[@]}"
|
||||
for d in "${replicas[@]}"
|
||||
do
|
||||
IFS=':'
|
||||
read -a item <<< "$deploy"
|
||||
read -a item <<< "$d"
|
||||
deploy_name="${item[0]}"
|
||||
replica_count="${item[1]}"
|
||||
echo "Starting $deploy_name..."
|
||||
component=$(kubectl get deploy -n $namespace $deploy_name --output=jsonpath={.metadata.labels.component})
|
||||
kubectl patch deployment/$deploy_name -n $namespace --patch-file=/dev/stdin <<-EOF
|
||||
spec:
|
||||
replicas: $replica_count
|
||||
template:
|
||||
spec:
|
||||
containers:
|
||||
- name: $component
|
||||
image: $image_tag
|
||||
EOF
|
||||
kubectl patch deployment $deploy_name -n $namespace -p "{\"spec\":{\"template\": {\"spec\": {\"containers\":[{\"name\": \"$component\", \"image\": \"$image_tag\"}]}}}}"
|
||||
kubectl scale deployment $deploy_name -n $namespace --replicas="$replica_count"
|
||||
done
|
||||
}
|
||||
|
||||
|
|
@ -397,51 +484,47 @@ function wait_for_milvus_ready(){
|
|||
deploy_name="${item[0]}"
|
||||
replica_count="${item[1]}"
|
||||
count=$(kubectl get deploy $deploy_name -n $namespace --output=jsonpath={.status.readyReplicas})
|
||||
if [ "$count" == "$replica_count" ] ; then
|
||||
if [ "$count" = "$replica_count" ] ; then
|
||||
ready_count=`expr $ready_count + 1`
|
||||
fi
|
||||
done
|
||||
if [ "$total_component_count" == "$ready_count" ]; then
|
||||
if [ "$total_component_count" = "$ready_count" ]; then
|
||||
break
|
||||
fi
|
||||
sleep 5
|
||||
done
|
||||
}
|
||||
|
||||
deployments=$(kubectl get deploy -n $namespace -l app.kubernetes.io/instance=$instance_name,app.kubernetes.io/name=milvus --output=jsonpath={.items..metadata.name})
|
||||
if [ ${#deployments} == 0 ]; then
|
||||
echo "There is no Milvus instance $instance_name in the namespace $namespace"
|
||||
else
|
||||
fetch_deploy_replica "${deployments[*]}"
|
||||
stop_milvus_deploy "$deployments"
|
||||
echo
|
||||
case $operation in
|
||||
migrate)
|
||||
echo "Starting to migrate milvus meta..."
|
||||
migrate_meta
|
||||
start_milvus_deploy
|
||||
echo
|
||||
echo "Upgrading is done. Waiting for milvus components to be ready again..."
|
||||
wait_for_milvus_ready
|
||||
if [ "$remove_migrate_pod_after_migrate" == "true" ]; then
|
||||
kubectl delete pods -n $namespace "milvus-meta-migration-backup-${instance_name}" "milvus-meta-migration-${instance_name}"
|
||||
kubectl delete pvc -n $namespace "milvus-meta-migration-backup-${instance_name}"
|
||||
kubectl delete configmap -n $namespace "milvus-meta-migration-config-${instance_name}"
|
||||
fi
|
||||
echo "All milvus components are running. Enjoy your vector search..."
|
||||
;;
|
||||
rollback)
|
||||
echo "Starting to rollback milvus meta..."
|
||||
rollback_meta
|
||||
start_milvus_deploy
|
||||
echo
|
||||
echo "Rollbacking is done. Waiting for milvus components to be ready again..."
|
||||
wait_for_milvus_ready
|
||||
echo "All milvus components are running. Enjoy your vector search..."
|
||||
;;
|
||||
*)
|
||||
echo "Invalid operation..."
|
||||
exit 6
|
||||
;;
|
||||
esac
|
||||
fi
|
||||
store_deploy_replicas
|
||||
stop_milvus_deploy
|
||||
echo
|
||||
case $operation in
|
||||
migrate)
|
||||
echo "Starting to migrate milvus meta..."
|
||||
migrate_meta
|
||||
start_milvus_deploy
|
||||
echo
|
||||
echo "Upgrading is done. Waiting for milvus components to be ready again..."
|
||||
wait_for_milvus_ready
|
||||
if [ "$remove_migrate_pod_after_migrate" = "true" ]; then
|
||||
kubectl delete pods -n $namespace "milvus-meta-migration-backup-${instance_name}" "milvus-meta-migration-${instance_name}"
|
||||
kubectl delete pvc -n $namespace "milvus-meta-migration-backup-${instance_name}"
|
||||
kubectl delete configmap -n $namespace "milvus-meta-migration-config-${instance_name}" "milvus-deploy-replicas-${instance_name}"
|
||||
echo
|
||||
fi
|
||||
echo "All milvus components are running. Enjoy your vector search..."
|
||||
;;
|
||||
rollback)
|
||||
echo "Starting to rollback milvus meta..."
|
||||
rollback_meta
|
||||
start_milvus_deploy
|
||||
echo
|
||||
echo "Rollbacking is done. Waiting for milvus components to be ready again..."
|
||||
wait_for_milvus_ready
|
||||
echo "All milvus components are running. Enjoy your vector search..."
|
||||
;;
|
||||
*)
|
||||
echo "Invalid operation..."
|
||||
exit 6
|
||||
;;
|
||||
esac
|
||||
|
|
|
|||
Loading…
Reference in New Issue