mirror of https://github.com/milvus-io/milvus.git
Add meta migration script for milvus installation in kubernetes (#19861)
Signed-off-by: Edward Zeng <jie.zeng@zilliz.com> Signed-off-by: Edward Zeng <jie.zeng@zilliz.com>pull/19914/head
edward.zeng
GitHub
parent
26fa72fac4
commit
c9d4f10ce5
|
|
@ -0,0 +1,66 @@
|
|||
# README
|
||||
|
||||
## Overview
|
||||
|
||||
Milvus 2.2 has changed the meta structure for segment index. If you have installed a milvus cluster of version v2.1.x, you'd run this script to migrate the meta and upgrade the milvus image version.
|
||||
|
||||
> Note:
|
||||
> 1. This script only works with Milvus installed on k8s cluster.
|
||||
> 2. You can only upgrade from v2.1.x to v2.2.
|
||||
|
||||
## Parameter Description
|
||||
|
||||
| Parameters | Description | Default | Required |
|
||||
| ---------- | ------------------------------------------------- | ---------------------------- | ----------------------- |
|
||||
| i | Specify the milvus instance name | None | true |
|
||||
| n | Specify the namespace that milvus is installed in | default | false |
|
||||
| s | Specify the milvus source version | None | true |
|
||||
| t | Specify the milvus target version | None | true |
|
||||
| r | Specify the milvus meta root path | by-dev | false |
|
||||
| w | Specify the milvus new image tag | milvusdb/milvus:v2.2.0 | false |
|
||||
| o | Specify the meta migration operation | migrate | false |
|
||||
> By default, the script only migrate from v2.1.x to v2.2.x. If there is anything wrong, you'd first rollback to the older version using the `rollback` operation.
|
||||
|
||||
## Migrate Procedures
|
||||
The migration will take four steps:
|
||||
1. Stop the milvus components; if there are any live session in milvus etcd, the migration will fail.
|
||||
2. Backup the milvus meta;
|
||||
3. Migrate the milvus meta;
|
||||
4. Startup milvus components with provided new image;
|
||||
|
||||
> Note:
|
||||
> 1. if step 1) or 2) are failed, you can rerun the migration again
|
||||
> 2. if step 3) or 4) are failed, you'd first rollback and migrate again
|
||||
|
||||
## Usage
|
||||
|
||||
1. Milvus instance name, milvus source version, milvus target vresion are required to be specified.
|
||||
|
||||
```shell
|
||||
./migrate.sh -i my-release -s 2.1.1 -t 2.2.0
|
||||
```
|
||||
|
||||
2. If your milvus is not installed in the k8s default namespace, please specify namespace with `-n`.
|
||||
|
||||
```shell
|
||||
./migrate.sh -i my-release -n milvus -s 2.1.1 -t 2.2.0
|
||||
```
|
||||
|
||||
3. If your milvus is installed with custom `rootpath`, please specify rootpath with `-r`.
|
||||
|
||||
```shell
|
||||
./migrate.sh -i my-release -n milvus -s 2.1.1 -t 2.2.0 -r by-dev
|
||||
```
|
||||
|
||||
4. If your milvus is installed with custom `image`, please specify the image tag with `-w`.
|
||||
|
||||
```shell
|
||||
./migrate.sh -i my-release -n milvus -s 2.1.1 -t 2.2.0 -r by-dev -w milvusdb/milvus:master-20221016-15878781
|
||||
```
|
||||
|
||||
5. If the migrate is failed, you'd first rollback migration and rerun migrate again.
|
||||
|
||||
```
|
||||
./migrate.sh -i my-release -n milvus -s 2.1.1 -t 2.2.0 -r by-dev -o rollback -w <milvus-2-1-1-image>
|
||||
./migrate.sh -i my-release -n milvus -s 2.1.1 -t 2.2.0 -r by-dev -o migrate -w <milvus-2-2-0-image>
|
||||
```
|
||||
|
|
@ -0,0 +1,434 @@
|
|||
#!/bin/bash
|
||||
|
||||
namespace="default"
|
||||
root_path="by-dev"
|
||||
operation="migrate"
|
||||
image_tag="milvusdb/milvus:v2.2.0"
|
||||
|
||||
#-n namespace: The namespace that Milvus is installed in.
|
||||
#-i milvus_instance: The name of milvus instance.
|
||||
#-s source_version: The milvus source version.
|
||||
#-t target_version: The milvus target version.
|
||||
#-r root_path: The milvus meta root path.
|
||||
#-w image_tag: The new milvus image tag.
|
||||
#-o operation: The operation: migrate/rollback
|
||||
while getopts "n:i:s:t:r:w:o" opt_name
|
||||
do
|
||||
case $opt_name in
|
||||
n) namespace=$OPTARG;;
|
||||
i) instance_name=$OPTARG;;
|
||||
s) source_version=$OPTARG;;
|
||||
t) target_version=$OPTARG;;
|
||||
r) root_path=$OPTARG;;
|
||||
w) image_tag=$OPTARG;;
|
||||
o) operation=$OPTARG;;
|
||||
*) echo "Unkonwen parameters";;
|
||||
esac
|
||||
done
|
||||
|
||||
if [ ! $instance_name ]; then
|
||||
echo "Missing argument instance_name, please add it. For example:'./migrate.sh -i milvus-instance-name'"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ ! $source_version ]; then
|
||||
echo "Missing argument source_version, please add it. For example:'./migrate.sh -s 2.1.4'"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ ! $target_version ]; then
|
||||
echo "Missing argument target_version, please add it. For example:'./migrate.sh -t 2.2.0'"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ ! $image_tag ]; then
|
||||
echo "Missing argument image_tag, please add it. For example:'./migrate.sh -w milvusdb/milvus:v2.2.0'"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "Migration milvus meta will take four steps:"
|
||||
echo "1. Stop the milvus components"
|
||||
echo "2. Backup the milvus meta"
|
||||
echo "3. Migrate the milvus meta"
|
||||
echo "4. Startup milvus components in new image version"
|
||||
echo
|
||||
|
||||
|
||||
replicas=()
|
||||
function fetch_deploy_replica(){
|
||||
for deploy in $1;
|
||||
do
|
||||
replica=$(kubectl get deploy $deploy -n $namespace --output=jsonpath={.spec.replicas})
|
||||
replicas+=("$deploy:$replica")
|
||||
done
|
||||
}
|
||||
|
||||
function stop_milvus_deploy(){
|
||||
echo "Stop milvus deployments: $1"
|
||||
kubectl scale deploy -n $namespace $1 --replicas=0 > /dev/null
|
||||
wait_for_milvus_stopped $1
|
||||
echo "Stopped..."
|
||||
}
|
||||
|
||||
function wait_for_milvus_stopped(){
|
||||
while true
|
||||
do
|
||||
total=0
|
||||
for deploy in $1;
|
||||
do
|
||||
count=$(kubectl get deploy $deploy -n $namespace --output=jsonpath={.spec.replicas})
|
||||
total=`expr $total + $count`
|
||||
done
|
||||
if [ $total -eq 0 ]; then
|
||||
break
|
||||
fi
|
||||
sleep 5
|
||||
done
|
||||
}
|
||||
|
||||
function wait_for_backup_done(){
|
||||
backup_pod_name="milvus-meta-migration-backup-${instance_name}"
|
||||
configmap_name="milvus-meta-migration-config-${instance_name}"
|
||||
while true
|
||||
do
|
||||
status=$(kubectl get pod -n $namespace $backup_pod_name --output=jsonpath={.status.phase})
|
||||
case $status in
|
||||
Succeeded)
|
||||
echo "Meta backup is done..."
|
||||
kubectl annotate configmap $configmap_name -n $namespace backup=succeed
|
||||
break
|
||||
;;
|
||||
Failed)
|
||||
echo "Meta backup is failed..."
|
||||
echo "Here is the log:"
|
||||
kubectl logs $backup_pod_name -n $namespace
|
||||
echo
|
||||
exit 2
|
||||
;;
|
||||
*)
|
||||
sleep 10
|
||||
continue
|
||||
;;
|
||||
esac
|
||||
done
|
||||
}
|
||||
|
||||
function wait_for_migrate_done(){
|
||||
migrate_pod_name="milvus-meta-migration-${instance_name}"
|
||||
while true
|
||||
do
|
||||
status=$(kubectl get pod -n $namespace $migrate_pod_name --output=jsonpath={.status.phase})
|
||||
case $status in
|
||||
Succeeded)
|
||||
echo "Migration is done..."
|
||||
echo
|
||||
break
|
||||
;;
|
||||
Failed)
|
||||
echo "Migration is failed..."
|
||||
echo "Here is the log:"
|
||||
kubectl logs $migrate_pod_name -n $namespace
|
||||
echo
|
||||
exit 3
|
||||
;;
|
||||
*)
|
||||
sleep 10
|
||||
continue
|
||||
;;
|
||||
esac
|
||||
done
|
||||
}
|
||||
|
||||
function wait_for_rollback_done(){
|
||||
rollback_pod_name="milvus-meta-migration-rollback-${instance_name}"
|
||||
while true
|
||||
do
|
||||
status=$(kubectl get pod -n $namespace $rollback_pod_name --output=jsonpath={.status.phase})
|
||||
case $status in
|
||||
Succeeded)
|
||||
echo "Rollback is done..."
|
||||
echo
|
||||
break
|
||||
;;
|
||||
Failed)
|
||||
echo "Rollback is failed..."
|
||||
echo "Here is the log:"
|
||||
kubectl logs $rollback_pod_name -n $namespace
|
||||
echo
|
||||
exit 5
|
||||
;;
|
||||
*)
|
||||
sleep 10
|
||||
continue
|
||||
;;
|
||||
esac
|
||||
done
|
||||
}
|
||||
function generate_migrate_meta_cm_and_pvc(){
|
||||
etcd_svc=$(kubectl get svc -n $namespace -l app.kubernetes.io/instance=$instance_name,app.kubernetes.io/name=etcd -o name | grep -v headless | cut -d '/' -f 2)
|
||||
cat <<EOF | kubectl apply -f -
|
||||
apiVersion: v1
|
||||
kind: PersistentVolumeClaim
|
||||
metadata:
|
||||
name: milvus-meta-migration-backup-${instance_name}
|
||||
spec:
|
||||
accessModes:
|
||||
- ReadWriteOnce
|
||||
resources:
|
||||
requests:
|
||||
storage: 10Gi
|
||||
EOF
|
||||
cat <<EOF | kubectl apply -f -
|
||||
apiVersion: v1
|
||||
kind: ConfigMap
|
||||
metadata:
|
||||
name: milvus-meta-migration-config-${instance_name}
|
||||
data:
|
||||
backup.yaml: |+
|
||||
cmd:
|
||||
type: backup
|
||||
|
||||
config:
|
||||
sourceVersion: $source_version
|
||||
targetVersion: $target_version
|
||||
backupFilePath: /milvus/data/migration.bak
|
||||
|
||||
metastore:
|
||||
type: etcd
|
||||
|
||||
etcd:
|
||||
endpoints:
|
||||
- $etcd_svc:2379
|
||||
rootPath: $root_path
|
||||
metaSubPath: meta
|
||||
kvSubPath: kv
|
||||
migration.yaml: |+
|
||||
cmd:
|
||||
type: run
|
||||
|
||||
config:
|
||||
sourceVersion: $source_version
|
||||
targetVersion: $target_version
|
||||
backupFilePath: /milvus/data/migration.bak
|
||||
|
||||
metastore:
|
||||
type: etcd
|
||||
|
||||
etcd:
|
||||
endpoints:
|
||||
- $etcd_svc:2379
|
||||
rootPath: $root_path
|
||||
metaSubPath: meta
|
||||
kvSubPath: kv
|
||||
rollback.yaml: |+
|
||||
cmd:
|
||||
type: rollback
|
||||
|
||||
config:
|
||||
sourceVersion: $source_version
|
||||
targetVersion: $target_version
|
||||
backupFilePath: /milvus/data/migration.bak
|
||||
|
||||
metastore:
|
||||
type: etcd
|
||||
|
||||
etcd:
|
||||
endpoints:
|
||||
- $etcd_svc:2379
|
||||
rootPath: $root_path
|
||||
metaSubPath: meta
|
||||
kvSubPath: kv
|
||||
EOF
|
||||
}
|
||||
|
||||
function backup_meta(){
|
||||
echo "Backuping meta..."
|
||||
echo "Checking whether backup exists..."
|
||||
configmap_name="milvus-meta-migration-config-${instance_name}"
|
||||
backup_exists=$(kubectl get configmap $configmap_name -n $namespace --output=jsonpath={.metadata.annotations.backup})
|
||||
if [ "$backup_exists" == "succeed" ]; then
|
||||
echo "Found previous backups, skip meta backup..."
|
||||
else
|
||||
cat <<EOF | kubectl apply -f -
|
||||
apiVersion: v1
|
||||
kind: Pod
|
||||
metadata:
|
||||
name: milvus-meta-migration-backup-${instance_name}
|
||||
spec:
|
||||
restartPolicy: Never
|
||||
containers:
|
||||
- name: meta-migration
|
||||
image: harbor.milvus.io/milvus/meta-migration:20221019-b79687687
|
||||
command: ["/bin/sh"]
|
||||
args:
|
||||
- -c
|
||||
- /milvus/bin/meta-migration -config=/milvus/configs/meta/backup.yaml
|
||||
volumeMounts:
|
||||
- name: backup
|
||||
mountPath: /milvus/data
|
||||
- name: config
|
||||
mountPath: /milvus/configs/meta
|
||||
volumes:
|
||||
- name: backup
|
||||
persistentVolumeClaim:
|
||||
claimName: milvus-meta-migration-backup-${instance_name}
|
||||
- name: config
|
||||
configMap:
|
||||
name: milvus-meta-migration-config-${instance_name}
|
||||
EOF
|
||||
wait_for_backup_done
|
||||
fi
|
||||
}
|
||||
|
||||
function rollback_meta(){
|
||||
generate_migrate_meta_cm_and_pvc
|
||||
echo "Checking whether backup exists..."
|
||||
backup_exists=$(kubectl get configmap milvus-meta-migration-config -n $namespace --output=jsonpath={.metadata.annotations.backup})
|
||||
if [ "$backup_exists" == "succeed" ]; then
|
||||
echo "Found previous backups, start meta rollback..."
|
||||
cat <<EOF | kubectl apply -f -
|
||||
apiVersion: v1
|
||||
kind: Pod
|
||||
metadata:
|
||||
name: milvus-meta-migration-rollback-${instance_name}
|
||||
spec:
|
||||
restartPolicy: Never
|
||||
containers:
|
||||
- name: meta-migration
|
||||
image: harbor.milvus.io/milvus/meta-migration:20221019-b79687687
|
||||
command: ["/bin/sh"]
|
||||
args:
|
||||
- -c
|
||||
- /milvus/bin/meta-migration -config=/milvus/configs/meta/rollback.yaml
|
||||
volumeMounts:
|
||||
- name: backup
|
||||
mountPath: /milvus/data
|
||||
- name: config
|
||||
mountPath: /milvus/configs/meta
|
||||
volumes:
|
||||
- name: backup
|
||||
persistentVolumeClaim:
|
||||
claimName: milvus-meta-migration-backup-${instance_name}
|
||||
- name: config
|
||||
configMap:
|
||||
name: milvus-meta-migration-config-${instance_name}
|
||||
EOF
|
||||
wait_for_rollback_done
|
||||
else
|
||||
echo "No backup exists, abort..."
|
||||
exit 4
|
||||
fi
|
||||
}
|
||||
|
||||
function migrate_meta(){
|
||||
generate_migrate_meta_cm_and_pvc
|
||||
|
||||
backup_meta
|
||||
echo "Migrating meta..."
|
||||
cat <<EOF | kubectl apply -f -
|
||||
apiVersion: v1
|
||||
kind: Pod
|
||||
metadata:
|
||||
name: milvus-meta-migration-${instance_name}
|
||||
spec:
|
||||
restartPolicy: Never
|
||||
containers:
|
||||
- name: meta-migration
|
||||
image: harbor.milvus.io/milvus/meta-migration:20221019-b79687687
|
||||
command: ["/bin/sh"]
|
||||
args:
|
||||
- -c
|
||||
- /milvus/bin/meta-migration -config=/milvus/configs/meta/migration.yaml
|
||||
volumeMounts:
|
||||
- name: backup
|
||||
mountPath: /milvus/data
|
||||
- name: config
|
||||
mountPath: /milvus/configs/meta
|
||||
volumes:
|
||||
- name: backup
|
||||
persistentVolumeClaim:
|
||||
claimName: milvus-meta-migration-backup-${instance_name}
|
||||
- name: config
|
||||
configMap:
|
||||
name: milvus-meta-migration-config-${instance_name}
|
||||
EOF
|
||||
wait_for_migrate_done
|
||||
}
|
||||
|
||||
function start_milvus_deploy(){
|
||||
echo "Starting milvus components..."
|
||||
for deploy in "${replicas[@]}"
|
||||
do
|
||||
IFS=':'
|
||||
read -a item <<< "$deploy"
|
||||
deploy_name="${item[0]}"
|
||||
replica_count="${item[1]}"
|
||||
echo "Starting $deploy_name..."
|
||||
component=$(kubectl get deploy -n $namespace $deploy_name --output=jsonpath={.metadata.labels.component})
|
||||
kubectl patch deployment/$deploy_name -n $namespace --patch-file=/dev/stdin <<-EOF
|
||||
spec:
|
||||
replicas: $replica_count
|
||||
template:
|
||||
spec:
|
||||
containers:
|
||||
- name: $component
|
||||
image: $image_tag
|
||||
EOF
|
||||
done
|
||||
}
|
||||
|
||||
function wait_for_milvus_ready(){
|
||||
total_component_count=${#replicas[@]}
|
||||
while true
|
||||
do
|
||||
ready_count=0
|
||||
for deploy in "${replicas[@]}"
|
||||
do
|
||||
IFS=':'
|
||||
read -a item <<< "$deploy"
|
||||
deploy_name="${item[0]}"
|
||||
replica_count="${item[1]}"
|
||||
count=$(kubectl get deploy $deploy_name -n $namespace --output=jsonpath={.status.readyReplicas})
|
||||
if [ "$count" == "$replica_count" ] ; then
|
||||
ready_count=`expr $ready_count + 1`
|
||||
fi
|
||||
done
|
||||
if [ "$total_component_count" == "$ready_count" ]; then
|
||||
break
|
||||
fi
|
||||
sleep 5
|
||||
done
|
||||
}
|
||||
|
||||
deployments=$(kubectl get deploy -n $namespace -l app.kubernetes.io/instance=$instance_name,app.kubernetes.io/name=milvus --output=jsonpath={.items..metadata.name})
|
||||
if [ ${#deployments} == 0 ]; then
|
||||
echo "There is no Milvus instance $instance_name in the namespace $namespace"
|
||||
else
|
||||
fetch_deploy_replica "${deployments[*]}"
|
||||
stop_milvus_deploy "$deployments"
|
||||
echo
|
||||
case $operation in
|
||||
migrate)
|
||||
echo "Starting to migrate milvus meta..."
|
||||
migrate_meta
|
||||
start_milvus_deploy
|
||||
echo
|
||||
echo "Upgrading is done. Waiting for milvus components to be ready again..."
|
||||
wait_for_milvus_ready
|
||||
echo "All milvus components are running. Enjoy your vector search..."
|
||||
;;
|
||||
rollback)
|
||||
echo "Starting to rollback milvus meta..."
|
||||
rollback_meta
|
||||
start_milvus_deploy
|
||||
echo
|
||||
echo "Rollbacking is done. Waiting for milvus components to be ready again..."
|
||||
wait_for_milvus_ready
|
||||
echo "All milvus components are running. Enjoy your vector search..."
|
||||
;;
|
||||
*)
|
||||
echo "Invalid operation..."
|
||||
exit 6
|
||||
;;
|
||||
esac
|
||||
fi
|
||||
Loading…
Reference in New Issue