[skip e2e]Add jenkins file for chaos test (#15262)

Signed-off-by: zhuwenxing <wenxing.zhu@zilliz.com>

build/ci/jenkins/ChaosTest.groovy

@@ -0,0 +1,270 @@
+pipeline {
+    options {
+        timestamps()
+        timeout(time: 30, unit: 'MINUTES')   // timeout on this stage
+    }
+    agent {
+        kubernetes {
+            label "milvus-chaos-test"
+            defaultContainer 'main'
+            yamlFile "build/ci/jenkins/pod/chaos-test.yaml"
+            customWorkspace '/home/jenkins/agent/workspace'
+            // idle 5 minutes to wait clean up tasks
+            idleMinutes 5
+        }
+    }
+    parameters{
+        choice(
+            description: 'Chaos Test Type',
+            name: 'chaos_type',
+            choices: ['pod-kill', 'pod-failure']
+        )
+        choice(
+            description: 'Chaos Test Target',
+            name: 'pod_name',
+            choices: ["standalone", "datacoord", "datanode", "indexcoord", "indexnode", "proxy", "pulsar", "querycoord", "querynode", "rootcoord", "etcd", "minio"]
+        )
+        choice(
+            description: 'Chaos Test Task',
+            name: 'chaos_task',
+            choices: ['chaos-test', 'data-consist-test']
+        )
+        string(
+            description: 'Image Repository',
+            name: 'image_repository',
+            defaultValue: 'milvusdb/milvus-dev'
+        )
+        string(
+            description: 'Image Tag',
+            name: 'image_tag',
+            defaultValue: 'master-latest'
+        )
+        string(
+            description: 'Pod Nums',
+            name: 'pod_nums',
+            defaultValue: '1'
+        )
+        booleanParam(
+            description: 'Keep Env',
+            name: 'keep_env',
+            defaultValue: 'false'
+        )
+    }
+    
+    environment {
+        ARTIFACTS = "${env.WORKSPACE}/_artifacts"
+        RELEASE_NAME = "test-${params.pod_name}-${params.chaos_type}-${env.BUILD_ID}"
+        NAMESPACE = "chaos-testing"
+    }
+
+    stages {
+        stage ('Install Dependency') {
+            steps {
+                container('main') {
+                    dir ('tests/python_client') {
+                        script {
+                        sh "pip install -r requirements.txt --trusted-host https://test.pypi.org"       
+                        }
+                    }
+                }
+            }
+        }
+        stage ('Deploy Milvus') {
+            options {
+              timeout(time: 10, unit: 'MINUTES')   // timeout on this stage
+            }
+            steps {
+                container('main') {
+                    
+                    dir ('tests/python_client/chaos/scripts') {
+                        script {
+                            def image_tag_modified = ""
+                            if ("${params.image_tag}" == "master-latest") {
+                                image_tag_modified = sh(returnStdout: true, script: 'bash ../../../../scripts/docker_image_find_tag.sh -n milvusdb/milvus-dev -t master-latest -f master- -F -L -q').trim()    
+                            }
+                            else {
+                                image_tag_modified = "${params.image_tag}"
+                            }
+                            sh "echo ${image_tag_modified}"
+                            sh "echo ${params.chaos_type}"
+                            sh "helm repo add milvus https://milvus-io.github.io/milvus-helm"
+                            sh "helm repo update"
+                            if ("${params.pod_name}" == "standalone"){
+                                sh"""
+                                IMAGE_TAG="${image_tag_modified}" \
+                                REPOSITORY="${params.image_repository}" \
+                                RELEASE_NAME="${env.RELEASE_NAME}" \
+                                bash install_milvus_standalone.sh
+                                """    
+                            }else{   
+                                sh"""
+                                IMAGE_TAG="${image_tag_modified}" \
+                                REPOSITORY="${params.image_repository}" \
+                                RELEASE_NAME="${env.RELEASE_NAME}" \
+                                bash install_milvus_cluster.sh
+                                """
+                            }
+                            sh "kubectl wait --for=condition=Ready pod -l app.kubernetes.io/instance=${env.RELEASE_NAME} -n ${env.NAMESPACE} --timeout=360s"
+                            sh "kubectl wait --for=condition=Ready pod -l release=${env.RELEASE_NAME} -n ${env.NAMESPACE} --timeout=360s"
+                            sh "kubectl get pods|grep ${env.RELEASE_NAME}"
+                            }
+                        }
+                    }
+                }
+        }
+        stage ('Run e2e test before chaos') {
+            steps {
+                container('main') {
+                    dir ('tests/python_client/chaos') {
+                        script {
+                        def host = sh(returnStdout: true, script: "kubectl get svc/${env.RELEASE_NAME}-milvus -o jsonpath=\"{.spec.clusterIP}\"").trim()
+                        sh "pytest -s -v ../testcases/test_e2e.py --host $host --log-cli-level=INFO --capture=no"       
+                        }
+                    }
+                }
+            }
+            
+        }
+
+        stage ('Run hello_milvus before chaos') {
+            options {
+              timeout(time: 5, unit: 'MINUTES')   // timeout on this stage
+            }
+            steps {
+                container('main') {
+                    dir ('tests/python_client/chaos') {
+                        script {
+                        def host = sh(returnStdout: true, script: "kubectl get svc/${env.RELEASE_NAME}-milvus -o jsonpath=\"{.spec.clusterIP}\"").trim()
+                        sh "python3 scripts/hello_milvus.py --host $host"          
+                        }
+                    }
+                }
+            }
+            
+        }
+
+
+        stage ('Run chaos test'){
+            options {
+              timeout(time: 15, unit: 'MINUTES')   // timeout on this stage
+            }
+            steps {
+                container('main') {
+                    dir ('tests/python_client/chaos') {                                     
+                        script {
+                        sh"""
+                        POD_NAME="${params.pod_name}" \
+                        CHAOS_TYPE="${params.chaos_type}" \
+                        RELEASE_NAME="${env.RELEASE_NAME}" \
+                        bash scripts/modify_config.sh
+                        """
+
+                        if ("${params.chaos_task}" == "chaos-test"){
+                            def host = sh(returnStdout: true, script: "kubectl get svc/${env.RELEASE_NAME}-milvus -o jsonpath=\"{.spec.clusterIP}\"").trim()
+                            sh "pytest -s -v test_chaos.py --host $host --log-cli-level=INFO --capture=no || echo 'chaos test fail' "
+                        }
+                        if ("${params.chaos_task}" == "data-consist-test"){
+                            def host = sh(returnStdout: true, script: "kubectl get svc/${env.RELEASE_NAME}-milvus -o jsonpath=\"{.spec.clusterIP}\"").trim()
+                            sh "pytest -s -v test_chaos_data_consist.py --host $host --log-cli-level=INFO --capture=no || echo "chaos test fail" "                           
+                        }
+                        echo "chaos test done"
+                        sh "kubectl wait --for=condition=Ready pod -l app.kubernetes.io/instance=${env.RELEASE_NAME} -n ${env.NAMESPACE} --timeout=360s"
+                        sh "kubectl wait --for=condition=Ready pod -l release=${env.RELEASE_NAME} -n ${env.NAMESPACE} --timeout=360s"                               
+                        sh "kubectl get pods|grep ${env.RELEASE_NAME}"
+                        }
+                    }
+                }
+            }
+            
+        }
+
+        stage ('run e2e test after chaos') {
+            options {
+              timeout(time: 5, unit: 'MINUTES')   // timeout on this stage
+            }            
+            steps {
+                container('main') {
+                    dir ('tests/python_client/chaos') {
+                        script {
+                        def host = sh(returnStdout: true, script: "kubectl get svc/${env.RELEASE_NAME}-milvus -o jsonpath=\"{.spec.clusterIP}\"").trim()
+                        sh "pytest -s -v ../testcases/test_e2e.py --host $host --log-cli-level=INFO --capture=no"        
+                        sh "kubectl get pods|grep ${env.RELEASE_NAME}"
+                        }
+                    }
+                }
+            }
+            
+        }
+
+        stage ('Run hello_milvus after chaos') {
+            options {
+              timeout(time: 5, unit: 'MINUTES')   // timeout on this stage
+            }
+            steps {
+                container('main') {
+                    dir ('tests/python_client/chaos') {
+                        script {
+                        def host = sh(returnStdout: true, script: "kubectl get svc/${env.RELEASE_NAME}-milvus -o jsonpath=\"{.spec.clusterIP}\"").trim()
+                        sh "python3 scripts/hello_milvus.py --host $host"        
+                        sh "kubectl get pods|grep ${env.RELEASE_NAME}"
+                        }
+                    }
+                }
+            }
+            
+        }
+        stage ('result analysis') {
+            steps {
+                container('main') {
+                    dir ('tests/python_client/chaos/reports') {
+                        script {
+                            echo "result analysis"
+                            sh "cat ${env.RELEASE_NAME}.log || echo 'no log file'"
+                        }
+                    }
+                }
+            }
+        } 
+    }
+    post {
+        always {
+            echo 'upload logs'
+            container('main') {
+                dir ('tests/python_client/chaos') {
+                    script {
+                        echo "collecte logs"
+                        sh "bash ../../scripts/export_log_k8s.sh ${env.NAMESPACE} ${env.RELEASE_NAME} k8s_log/${env.RELEASE_NAME}"                        
+                        sh "tar -zcvf artifacts-${env.RELEASE_NAME}-pytest-logs.tar.gz /tmp/ci_logs/ --remove-files || true"
+                        sh "tar -zcvf artifacts-${env.RELEASE_NAME}-server-logs.tar.gz k8s_log/ --remove-files || true"
+                        archiveArtifacts artifacts: "artifacts-${env.RELEASE_NAME}-pytest-logs.tar.gz", allowEmptyArchive: true
+                        archiveArtifacts artifacts: "artifacts-${env.RELEASE_NAME}-server-logs.tar.gz", allowEmptyArchive: true
+                        if ("${params.keep_env}" == "false"){
+                            sh "bash scripts/uninstall_milvus.sh ${env.RELEASE_NAME}"
+                        }
+                    }
+                }
+            }
+        
+        }
+        success {
+            echo 'I succeeeded!'
+            container('main') {
+                dir ('tests/python_client/chaos/scripts') {
+                    script {
+                        sh "bash uninstall_milvus.sh ${env.RELEASE_NAME} || true"
+                    }
+                }
+            }  
+
+        }
+        unstable {
+            echo 'I am unstable :/'
+        }
+        failure {
+            echo 'I failed :('
+        }
+        changed {
+            echo 'Things were different before...'
+        }
+    }
+}

build/ci/jenkins/pod/chaos-test.yaml

@@ -0,0 +1,22 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  labels:
+    app: milvus-chaos-test
+  namespace: chaos-testing
+spec:
+  enableServiceLinks: false
+  containers:
+  - name: main
+    image: registry.milvus.io/test/krte:dev-1
+    securityContext:
+      privileged: true
+    tty: true
+    args: ["cat"]
+    resources:
+      limits:
+        cpu: "6"
+        memory: 12Gi
+      requests:
+        cpu: "2"
+        memory: 10Gi

tests/python_client/chaos/chaos_test.sh

@@ -1,9 +1,10 @@
+#!/bin/bash
 set -e
 set -x
 
 
 echo "check os env"
-platform='unknown'
+platform='Linux'
 unamestr=$(uname)
 if [[ "$unamestr" == 'Linux' ]]; then
    platform='Linux'
@@ -15,7 +16,8 @@ echo "platform: $platform"
 ns="chaos-testing"
 
 # switch namespace
-kubectl config set-context --current --namespace=${ns}
+# kubectl config set-context --current --namespace=${ns}
+# kubectl get pod
 
 # set parameters
 pod=${1:-"querynode"}
@@ -24,8 +26,13 @@ chaos_task=${3:-"chaos-test"} # chaos-test or data-consist-test
 node_num=${4:-1} # cluster_1_node or cluster_n_nodes
 
 cur_time=$(date +%H-%M-%S)
-release="test"-${pod}-${chaos_type/_/-}-${cur_time} # replace pod_kill to pod-kill
+release_name="test"-${pod}-${chaos_type/_/-}-${cur_time} # replace pod_kill to pod-kill
+release=${RELEASE_NAME:-"${release_name}"}
 
+# replace separator to default
+chaos_type=${chaos_type/-/_} # default separator of chaos_type is _
+chaos_task=${chaos_task/_/-} # default separator of chaos_task is -
+echo "chaos_type: ${chaos_type}"
 # install milvus cluster for chaos testing
 pushd ./scripts
 echo "uninstall milvus if exist"
@@ -36,13 +43,13 @@ echo "install milvus"
 if [ ${pod} != "standalone" ];
 then
     echo "insatll cluster"
-    helm install --wait --timeout 360s ${release} milvus/milvus --set image.all.tag=${image_tag:-"master-latest"} --set ${pod_map[${pod}]}.replicas=$node_num -f ../cluster-values.yaml -n=${ns}
+    helm install --wait --timeout 360s ${release} milvus/milvus --set image.all.repository=${REPOSITORY:-"milvusdb/milvus-dev"} --set image.all.tag=${IMAGE_TAG:-"master-latest"} --set ${pod_map[${pod}]}.replicas=$node_num -f ../cluster-values.yaml -n=${ns}
 fi
 
 if [ ${pod} == "standalone" ];
 then
     echo "install standalone"
-    helm install --wait --timeout 360s ${release} milvus/milvus --set image.all.tag=${image_tag:-"master-latest"} -f ../standalone-values.yaml -n=${ns}
+    helm install --wait --timeout 360s ${release} milvus/milvus --set image.all.repository=${REPOSITORY:-"milvusdb/milvus-dev"} --set image.all.tag=${IMAGE_TAG:-"master-latest"} -f ../standalone-values.yaml -n=${ns}
 fi
 
 # wait all pod ready
@@ -72,7 +79,7 @@ else
     host=$(kubectl get svc/${release}-milvus -o jsonpath="{.spec.clusterIP}")
 fi
 pytest -s -v ../testcases/test_e2e.py --host "$host" --log-cli-level=INFO --capture=no
-python scripts/hello_milvus.py --host "$host"
+python3 scripts/hello_milvus.py --host "$host"
 
 # chaos test
 if [ "$chaos_task" == "chaos-test" ];
@@ -90,8 +97,8 @@ kubectl wait --for=condition=Ready pod -l app.kubernetes.io/instance=${release}
 kubectl wait --for=condition=Ready pod -l release=${release} -n ${ns} --timeout=360s
 
 pytest -s -v ../testcases/test_e2e.py --host "$host" --log-cli-level=INFO --capture=no || echo "e2e test fail"
-python scripts/hello_milvus.py --host "$host" || echo "e2e test fail"
+python3 scripts/hello_milvus.py --host "$host" || echo "e2e test fail"
 
 # save logs
 cur_time=$(date +%Y-%m-%d-%H-%M-%S)
-bash ../../scripts/export_log_k8s.sh ${ns} ${release} k8s_log/${pod}-${chaos_type}-${chaos_task}-${cur_time}
+bash ../../scripts/export_log_k8s.sh ${ns} ${release} k8s_log/${pod}-${chaos_type}-${chaos_task}-${cur_time}

tests/python_client/chaos/scripts/install_milvus_cluster.sh

@@ -0,0 +1,16 @@
+#!/bin/bash
+set -e
+
+release=${1:-"milvs-chaos"}
+ns=${2:-"chaos-testing"}
+pod=${3:-"querynode"}
+node_num=${4:-1}
+bash uninstall_milvus.sh ${release} ${ns}|| true
+
+declare -A pod_map=(["querynode"]="queryNode" ["indexnode"]="indexNode" ["datanode"]="dataNode" ["proxy"]="proxy")
+echo "insatll cluster"
+helm install --wait --timeout 360s ${RELEASE_NAME:-$release} milvus/milvus \
+                    --set image.all.repository=${REPOSITORY:-"milvusdb/milvus-dev"} \
+                    --set image.all.tag=${IMAGE_TAG:-"master-latest"} \
+                    --set ${pod_map[${pod}]}.replicas=$node_num \
+                    -f ../cluster-values.yaml -n=${ns}

tests/python_client/chaos/scripts/install_milvus_standalone.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+set -e
+
+release=${1:-"milvs-chaos"}
+ns=${2:-"chaos-testing"}
+bash uninstall_milvus.sh ${release} ${ns}|| true
+echo "insatll standalone"
+helm install --wait --timeout 360s ${RELEASE_NAME:-$release} milvus/milvus \
+                    --set image.all.repository=${REPOSITORY:-"milvusdb/milvus-dev"} \
+                    --set image.all.tag=${IMAGE_TAG:-"master-latest"} \
+                    -f ../standalone-values.yaml -n=${ns}

tests/python_client/chaos/scripts/modify_config.sh

@@ -0,0 +1,25 @@
+#!/bin/bash
+set -e
+set -x
+
+
+echo "check os env"
+platform='Linux'
+unamestr=$(uname)
+if [[ "$unamestr" == 'Linux' ]]; then
+   platform='Linux'
+elif [[ "$unamestr" == 'Darwin' ]]; then
+   platform='Mac'
+fi
+echo "platform: $platform"
+
+if [ "$platform" == "Mac" ];
+then
+    sed -i "" "s/TESTS_CONFIG_LOCATION =.*/TESTS_CONFIG_LOCATION = \'chaos_objects\/${CHAOS_TYPE/-/_}\/'/g" constants.py
+    sed -i "" "s/ALL_CHAOS_YAMLS =.*/ALL_CHAOS_YAMLS = \'chaos_${POD_NAME}_${CHAOS_TYPE/-/_}.yaml\'/g" constants.py
+    sed -i "" "s/RELEASE_NAME =.*/RELEASE_NAME = \'${RELEASE_NAME}\'/g" constants.py
+else
+    sed -i "s/TESTS_CONFIG_LOCATION =.*/TESTS_CONFIG_LOCATION = \'chaos_objects\/${CHAOS_TYPE/-/_}\/'/g" constants.py
+    sed -i "s/ALL_CHAOS_YAMLS =.*/ALL_CHAOS_YAMLS = \'chaos_${POD_NAME}_${CHAOS_TYPE/-/_}.yaml\'/g" constants.py
+    sed -i "s/RELEASE_NAME =.*/RELEASE_NAME = \'${RELEASE_NAME}\'/g" constants.py
+fi

tests/python_client/chaos/test_chaos.py

@@ -40,7 +40,7 @@ def record_results(checkers):
         # expect succ if no expectations
         succ_rate = checkers[k].succ_rate()
         total = checkers[k].total()
-        res += f"{str(k)} succ rate {succ_rate}, total: {total}\n"
+        res += f"{str(k):10} succ rate: {succ_rate:.2f}  total: {total:02d}\n"
     return res