mirror of https://github.com/milvus-io/milvus.git
[skip e2e]Add kafka standalone for chaos and deploy test (#17707)
Signed-off-by: zhuwenxing <wenxing.zhu@zilliz.com>pull/17693/head
zhuwenxing
GitHub
parent
8cf54137cf
commit
69ad70b35b
|
|
@ -190,8 +190,6 @@ jobs:
|
|||
mode: [standalone,cluster]
|
||||
task: [reinstall,upgrade]
|
||||
exclude:
|
||||
- mq_type: kafka
|
||||
mode: standalone
|
||||
- mq_type: kafka
|
||||
task: upgrade
|
||||
|
||||
|
|
@ -229,7 +227,10 @@ jobs:
|
|||
run: |
|
||||
yq -i ".kafka.enabled = false" cluster-values.yaml
|
||||
yq -i ".pulsar.enabled = false" cluster-values.yaml
|
||||
yq -i ".${{ matrix.mq_type }}.enabled = true" cluster-values.yaml
|
||||
yq -i ".${{ matrix.mq_type }}.enabled = true" cluster-values.yaml
|
||||
if [ ${{ matrix.mq_type }} == "kafka" ]; then
|
||||
yq -i ".kafka.enabled = true" standalone-values.yaml;
|
||||
fi
|
||||
|
||||
- name: First Milvus Deployment
|
||||
timeout-minutes: 15
|
||||
|
|
|
|||
|
|
@ -22,7 +22,7 @@ jobs:
|
|||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
pod: [allcluster, datacoord, datanode, indexcoord, indexnode, proxy, kafka, querycoord, querynode, rootcoord, etcd, minio]
|
||||
pod: [allstandalone, allcluster, standalone, datacoord, datanode, indexcoord, indexnode, proxy, kafka, querycoord, querynode, rootcoord, etcd, minio]
|
||||
|
||||
steps:
|
||||
|
||||
|
|
@ -84,7 +84,7 @@ jobs:
|
|||
helm repo add milvus https://milvus-io.github.io/milvus-helm
|
||||
helm repo update
|
||||
if [[ ${{ matrix.pod }} != *"standalone"* ]]; then helm install --wait --timeout 720s ${{ env.RELEASE }} milvus/milvus --set pulsar.enabled=false --set kafka.enabled=true --set image.all.repository=${{ env.IMAGE_REPO }} --set image.all.tag=${{ env.IMAGE_TAG }} -f cluster-values.yaml -n=chaos-testing; fi
|
||||
if [[ ${{ matrix.pod }} == *"standalone"* ]]; then helm install --wait --timeout 720s ${{ env.RELEASE }} milvus/milvus --set image.all.repository=${{ env.IMAGE_REPO }} --set image.all.tag=${{ env.IMAGE_TAG }} -f standalone-values.yaml -n=chaos-testing; fi
|
||||
if [[ ${{ matrix.pod }} == *"standalone"* ]]; then helm install --wait --timeout 720s ${{ env.RELEASE }} milvus/milvus --set pulsar.enabled=false --set kafka.enabled=true --set image.all.repository=${{ env.IMAGE_REPO }} --set image.all.tag=${{ env.IMAGE_TAG }} -f standalone-values.yaml -n=chaos-testing; fi
|
||||
kubectl get pods -n chaos-testing
|
||||
sleep 20s
|
||||
kubectl get pods -n chaos-testing
|
||||
|
|
|
|||
|
|
@ -24,7 +24,7 @@ pipeline {
|
|||
mem-stress: datanode, etcd, indexnode, minio, proxy, pulsar, querynode, standalone \
|
||||
io-fault & io-latency: minio, pulsar, etcd ',
|
||||
name: 'pod_name',
|
||||
choices: ["standalone", "datacoord", "datanode", "indexcoord", "indexnode", "proxy", "pulsar", "querycoord", "querynode", "rootcoord", "etcd", "minio"]
|
||||
choices: ["allstandalone", "allcluster", "standalone", "datacoord", "datanode", "indexcoord", "indexnode", "proxy", "pulsar", "querycoord", "querynode", "rootcoord", "etcd", "minio"]
|
||||
)
|
||||
choice(
|
||||
description: 'Chaos Test Task',
|
||||
|
|
|
|||
|
|
@ -21,10 +21,10 @@ pipeline {
|
|||
)
|
||||
choice(
|
||||
description: 'Chaos Test Target: \
|
||||
mem-stress: datanode, etcd, indexnode, minio, proxy, kakfa, querynode, standalone \
|
||||
mem-stress: datanode, etcd, indexnode, minio, proxy, kafka, querynode, standalone \
|
||||
io-fault & io-latency: minio, kafka, etcd ',
|
||||
name: 'pod_name',
|
||||
choices: ["datacoord", "datanode", "indexcoord", "indexnode", "proxy", "kafka", "querycoord", "querynode", "rootcoord", "etcd", "minio"]
|
||||
choices: ["allstandalone", "allcluster", "standalone", "datacoord", "datanode", "indexcoord", "indexnode", "proxy", "kafka", "querycoord", "querynode", "rootcoord", "etcd", "minio"]
|
||||
)
|
||||
choice(
|
||||
description: 'Chaos Test Task',
|
||||
|
|
@ -70,14 +70,14 @@ pipeline {
|
|||
description: 'Proxy Nums',
|
||||
name: 'proxy_nums',
|
||||
defaultValue: '1'
|
||||
)
|
||||
)
|
||||
booleanParam(
|
||||
description: 'Keep Env',
|
||||
name: 'keep_env',
|
||||
defaultValue: 'false'
|
||||
)
|
||||
}
|
||||
|
||||
|
||||
environment {
|
||||
ARTIFACTS = "${env.WORKSPACE}/_artifacts"
|
||||
RELEASE_NAME = "${params.pod_name}-${params.chaos_type}-${env.BUILD_ID}"
|
||||
|
|
@ -90,7 +90,7 @@ pipeline {
|
|||
container('main') {
|
||||
dir ('tests/python_client') {
|
||||
script {
|
||||
sh "pip install -r requirements.txt --trusted-host https://test.pypi.org"
|
||||
sh "pip install -r requirements.txt --trusted-host https://test.pypi.org"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -101,10 +101,9 @@ pipeline {
|
|||
container('main') {
|
||||
dir ('tests/python_client/chaos') {
|
||||
script {
|
||||
sh"""
|
||||
sh """
|
||||
yq -i '.kafka.enabled = true' standalone-values.yaml
|
||||
yq -i '.kafka.enabled = true' cluster-values.yaml
|
||||
yq -i '.pulsar.enabled = false' cluster-values.yaml
|
||||
|
||||
yq -i '.queryNode.replicas = "${params.querynode_nums}"' cluster-values.yaml
|
||||
yq -i '.dataNode.replicas = "${params.datanode_nums}"' cluster-values.yaml
|
||||
yq -i '.indexNode.replicas = "${params.indexnode_nums}"' cluster-values.yaml
|
||||
|
|
@ -129,7 +128,7 @@ pipeline {
|
|||
script {
|
||||
def image_tag_modified = ""
|
||||
if ("${params.image_tag}" == "master-latest") {
|
||||
image_tag_modified = sh(returnStdout: true, script: 'bash ../../../../scripts/docker_image_find_tag.sh -n milvusdb/milvus-dev -t master-latest -f master- -F -L -q').trim()
|
||||
image_tag_modified = sh(returnStdout: true, script: 'bash ../../../../scripts/docker_image_find_tag.sh -n milvusdb/milvus-dev -t master-latest -f master- -F -L -q').trim()
|
||||
}
|
||||
else {
|
||||
image_tag_modified = "${params.image_tag}"
|
||||
|
|
@ -144,8 +143,8 @@ pipeline {
|
|||
REPOSITORY="${params.image_repository}" \
|
||||
RELEASE_NAME="${env.RELEASE_NAME}" \
|
||||
bash install_milvus_standalone.sh
|
||||
"""
|
||||
}else{
|
||||
"""
|
||||
}else{
|
||||
sh"""
|
||||
IMAGE_TAG="${image_tag_modified}" \
|
||||
REPOSITORY="${params.image_repository}" \
|
||||
|
|
@ -164,18 +163,18 @@ pipeline {
|
|||
stage ('Run e2e test before chaos') {
|
||||
options {
|
||||
timeout(time: 5, unit: 'MINUTES') // timeout on this stage
|
||||
}
|
||||
}
|
||||
steps {
|
||||
container('main') {
|
||||
dir ('tests/python_client/chaos') {
|
||||
script {
|
||||
def host = sh(returnStdout: true, script: "kubectl get svc/${env.RELEASE_NAME}-milvus -o jsonpath=\"{.spec.clusterIP}\"").trim()
|
||||
sh "pytest -s -v ../testcases/test_e2e.py --host $host --log-cli-level=INFO --capture=no"
|
||||
sh "pytest -s -v ../testcases/test_e2e.py --host $host --log-cli-level=INFO --capture=no"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
||||
stage ('Run hello_milvus before chaos') {
|
||||
|
|
@ -187,12 +186,12 @@ pipeline {
|
|||
dir ('tests/python_client/chaos') {
|
||||
script {
|
||||
def host = sh(returnStdout: true, script: "kubectl get svc/${env.RELEASE_NAME}-milvus -o jsonpath=\"{.spec.clusterIP}\"").trim()
|
||||
sh "python3 scripts/hello_milvus.py --host $host"
|
||||
sh "python3 scripts/hello_milvus.py --host $host"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
||||
|
||||
|
|
@ -217,17 +216,17 @@ pipeline {
|
|||
}
|
||||
if ("${params.chaos_task}" == "data-consist-test"){
|
||||
def host = sh(returnStdout: true, script: "kubectl get svc/${env.RELEASE_NAME}-milvus -o jsonpath=\"{.spec.clusterIP}\"").trim()
|
||||
sh "pytest -s -v test_chaos_data_consist.py --host $host --log-cli-level=INFO --capture=no || echo 'chaos test fail' "
|
||||
sh "pytest -s -v test_chaos_data_consist.py --host $host --log-cli-level=INFO --capture=no || echo 'chaos test fail' "
|
||||
}
|
||||
echo "chaos test done"
|
||||
sh "kubectl wait --for=condition=Ready pod -l app.kubernetes.io/instance=${env.RELEASE_NAME} -n ${env.NAMESPACE} --timeout=360s"
|
||||
sh "kubectl wait --for=condition=Ready pod -l release=${env.RELEASE_NAME} -n ${env.NAMESPACE} --timeout=360s"
|
||||
sh "kubectl wait --for=condition=Ready pod -l release=${env.RELEASE_NAME} -n ${env.NAMESPACE} --timeout=360s"
|
||||
sh "kubectl get pods -o wide|grep ${env.RELEASE_NAME}"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
stage ('result analysis') {
|
||||
steps {
|
||||
|
|
@ -240,24 +239,24 @@ pipeline {
|
|||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
stage ('run e2e test after chaos') {
|
||||
options {
|
||||
timeout(time: 5, unit: 'MINUTES') // timeout on this stage
|
||||
}
|
||||
}
|
||||
steps {
|
||||
container('main') {
|
||||
dir ('tests/python_client/chaos') {
|
||||
script {
|
||||
def host = sh(returnStdout: true, script: "kubectl get svc/${env.RELEASE_NAME}-milvus -o jsonpath=\"{.spec.clusterIP}\"").trim()
|
||||
sh "pytest -s -v ../testcases/test_e2e.py --host $host --log-cli-level=INFO --capture=no"
|
||||
sh "pytest -s -v ../testcases/test_e2e.py --host $host --log-cli-level=INFO --capture=no"
|
||||
sh "kubectl get pods -o wide|grep ${env.RELEASE_NAME}"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
|
||||
stage ('Run hello_milvus after chaos') {
|
||||
|
|
@ -269,12 +268,12 @@ pipeline {
|
|||
dir ('tests/python_client/chaos') {
|
||||
script {
|
||||
def host = sh(returnStdout: true, script: "kubectl get svc/${env.RELEASE_NAME}-milvus -o jsonpath=\"{.spec.clusterIP}\"").trim()
|
||||
sh "python3 scripts/hello_milvus.py --host $host"
|
||||
sh "python3 scripts/hello_milvus.py --host $host"
|
||||
sh "kubectl get pods -o wide|grep ${env.RELEASE_NAME}"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
stage ('Verify all collections after chaos') {
|
||||
options {
|
||||
|
|
@ -290,8 +289,8 @@ pipeline {
|
|||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
post {
|
||||
always {
|
||||
|
|
@ -302,7 +301,7 @@ pipeline {
|
|||
echo "get pod status"
|
||||
sh "kubectl get pods -o wide|grep ${env.RELEASE_NAME} || true"
|
||||
echo "collecte logs"
|
||||
sh "bash ../../scripts/export_log_k8s.sh ${env.NAMESPACE} ${env.RELEASE_NAME} k8s_log/${env.RELEASE_NAME} || true"
|
||||
sh "bash ../../scripts/export_log_k8s.sh ${env.NAMESPACE} ${env.RELEASE_NAME} k8s_log/${env.RELEASE_NAME} || true"
|
||||
sh "tar -zcvf artifacts-${env.RELEASE_NAME}-pytest-logs.tar.gz /tmp/ci_logs/ --remove-files || true"
|
||||
sh "tar -zcvf artifacts-${env.RELEASE_NAME}-server-logs.tar.gz k8s_log/ --remove-files || true"
|
||||
archiveArtifacts artifacts: "artifacts-${env.RELEASE_NAME}-pytest-logs.tar.gz", allowEmptyArchive: true
|
||||
|
|
@ -313,7 +312,7 @@ pipeline {
|
|||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
success {
|
||||
echo 'I succeeeded!'
|
||||
|
|
@ -323,7 +322,7 @@ pipeline {
|
|||
sh "bash uninstall_milvus.sh ${env.RELEASE_NAME} || true"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
unstable {
|
||||
|
|
@ -336,4 +335,4 @@ pipeline {
|
|||
echo 'Things were different before...'
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -16,7 +16,7 @@ pipeline {
|
|||
choice(
|
||||
description: 'Milvus Mode',
|
||||
name: 'milvus_mode',
|
||||
choices: ["cluster"]
|
||||
choices: ["standalone", "cluster"]
|
||||
)
|
||||
choice(
|
||||
description: 'MQ Type',
|
||||
|
|
@ -127,6 +127,7 @@ pipeline {
|
|||
sh "yq -i '.pulsar.enabled = true' cluster-values.yaml"
|
||||
} else if ("${params.mq_type}" == "kafka") {
|
||||
sh "yq -i '.kafka.enabled = true' cluster-values.yaml"
|
||||
sh "yq -i '.kafka.enabled = true' standalone-values.yaml"
|
||||
}
|
||||
sh"""
|
||||
yq -i '.queryNode.replicas = "${params.querynode_nums}"' cluster-values.yaml
|
||||
|
|
@ -134,7 +135,13 @@ pipeline {
|
|||
yq -i '.indexNode.replicas = "${params.indexnode_nums}"' cluster-values.yaml
|
||||
yq -i '.proxy.replicas = "${params.proxy_nums}"' cluster-values.yaml
|
||||
"""
|
||||
sh "cat cluster-values.yaml"
|
||||
if ("${params.milvus_mode}" == "cluster"){
|
||||
sh "cat cluster-values.yaml"
|
||||
}
|
||||
if ("${params.mq_type}" == "standalone"){
|
||||
sh "cat standalone-values.yaml"
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -5,6 +5,13 @@ image:
|
|||
repository: milvusdb/milvus-dev
|
||||
tag: master-latest
|
||||
pullPolicy: IfNotPresent
|
||||
|
||||
kafka:
|
||||
enabled: false
|
||||
name: kafka
|
||||
replicaCount: 3
|
||||
defaultReplicationFactor: 2
|
||||
|
||||
etcd:
|
||||
replicaCount: 1
|
||||
image:
|
||||
|
|
|
|||
|
|
@ -5,6 +5,13 @@ image:
|
|||
repository: milvusdb/milvus-dev
|
||||
tag: master-latest
|
||||
pullPolicy: IfNotPresent
|
||||
|
||||
kafka:
|
||||
enabled: false
|
||||
name: kafka
|
||||
replicaCount: 3
|
||||
defaultReplicationFactor: 2
|
||||
|
||||
etcd:
|
||||
replicaCount: 1
|
||||
image:
|
||||
|
|
|
|||
Loading…
Reference in New Issue