Remove KinD for new Nightly CI (#11919)

Signed-off-by: Jenny Li <jing.li@zilliz.com>

build/ci/jenkins/Nightly.groovy

@@ -0,0 +1,226 @@
+#!/usr/bin/env groovy
+
+// When scheduling a job that gets automatically triggered by changes,
+// you need to include a [cronjob] tag within the commit message.
+String cron_timezone = 'TZ=Asia/Shanghai'
+String cron_string = BRANCH_NAME == "master" ? "50 22 * * * " : ""
+
+int total_timeout_minutes = 660
+
+pipeline {
+    triggers {
+        cron """${cron_timezone}
+            ${cron_string}"""
+    }
+    options {
+        timestamps()
+        timeout(time: total_timeout_minutes, unit: 'MINUTES')
+        buildDiscarder logRotator(artifactDaysToKeepStr: '30')
+    // parallelsAlwaysFailFast()
+    }
+    agent {
+        kubernetes {
+            label "milvus-e2e-test-nightly"             
+            inheritFrom 'default'
+            defaultContainer 'main'
+            yamlFile "build/ci/jenkins/pod/rte.yaml"
+            customWorkspace '/home/jenkins/agent/workspace'
+        }
+    }
+    environment {
+        PROJECT_NAME = "milvus"
+        SEMVER = "${BRANCH_NAME.contains('/') ? BRANCH_NAME.substring(BRANCH_NAME.lastIndexOf('/') + 1) : BRANCH_NAME}"
+        IMAGE_REPO = "dockerhub-mirror-sh.zilliz.cc/milvusdb"
+        DOCKER_BUILDKIT = 1
+        ARTIFACTS = "${env.WORKSPACE}/_artifacts"
+        DOCKER_CREDENTIALS_ID = "f0aacc8e-33f2-458a-ba9e-2c44f431b4d2"
+        TARGET_REPO = "milvusdb"
+        CI_DOCKER_CREDENTIAL_ID = "ci-docker-registry"
+        MILVUS_HELM_NAMESPACE = "milvus-ci"
+        DISABLE_KIND = true
+        HUB = "registry.milvus.io/milvus"
+        JENKINS_BUILD_ID = "${env.BUILD_ID}"
+    }
+
+    stages {
+        stage ('Build'){
+            steps {
+                container('main') {
+                    dir ('tests/scripts') {
+                        script {
+                            sh 'printenv'
+                            def date = sh(returnStdout: true, script: 'date +%Y%m%d').trim()
+                            def gitShortCommit = sh(returnStdout: true, script: 'git rev-parse --short HEAD').trim()                           
+                            withCredentials([usernamePassword(credentialsId: "${env.CI_DOCKER_CREDENTIAL_ID}", usernameVariable: 'CI_REGISTRY_USERNAME', passwordVariable: 'CI_REGISTRY_PASSWORD')]){
+                                sh """
+                                TAG="${env.BRANCH_NAME}-${date}-${gitShortCommit}" \
+                                ./e2e-k8s.sh \
+                                --skip-export-logs \
+                                --skip-install \
+                                --skip-cleanup \
+                                --skip-setup \
+                                --skip-test                               
+                                """
+                    
+                            }
+                        }
+                    }
+                }
+            }
+        }
+
+
+        stage ('Install & E2E Test') {
+                matrix {
+                    axes {
+                        axis {
+                            name 'MILVUS_SERVER_TYPE'
+                            values 'standalone', 'distributed'
+                        }
+                        axis {
+                            name 'MILVUS_CLIENT'
+                            values 'pymilvus'
+                        }
+                    }
+
+                stages {
+                        stage('Install') {
+                            steps {
+                                container('main') {
+                                    dir ('tests/scripts') {
+                                        script {
+                                            sh 'printenv'
+                                            def clusterEnabled = "false"
+                                            def setMemoryResourceLimitArgs="--set standalone.resources.limits.memory=4Gi"
+                                            if ("${MILVUS_SERVER_TYPE}" == "distributed") {
+                                                clusterEnabled = "true"
+                                                setMemoryResourceLimitArgs="--set queryNode.resources.limits.memory=4Gi"
+                                            }
+
+                                            def date = sh(returnStdout: true, script: 'date +%Y%m%d').trim()
+                                            def gitShortCommit = sh(returnStdout: true, script: 'git rev-parse --short HEAD').trim()
+                                            if ("${MILVUS_CLIENT}" == "pymilvus") {
+                                                withCredentials([usernamePassword(credentialsId: "${env.CI_DOCKER_CREDENTIAL_ID}", usernameVariable: 'CI_REGISTRY_USERNAME', passwordVariable: 'CI_REGISTRY_PASSWORD')]){
+                                                    sh """
+                                                    MILVUS_CLUSTER_ENABLED=${clusterEnabled} \
+                                                    TAG="${env.BRANCH_NAME}-${date}-${gitShortCommit}" \
+                                                    ./e2e-k8s.sh \
+                                                    --skip-export-logs \
+                                                    --skip-cleanup \
+                                                    --skip-setup \
+                                                    --skip-test \
+                                                    --skip-build \
+                                                    --skip-build-image \
+                                                    --install-extra-arg "--set etcd.persistence.storageClass=local-path ${setMemoryResourceLimitArgs} \
+                                                    --set metrics.serviceMonitor.enabled=true" 
+                                                    """
+                                                }
+                                            } else {
+                                                error "Error: Unsupported Milvus client: ${MILVUS_CLIENT}"
+                                            }
+                                        }
+                                    }
+                                }
+                            }
+                        }
+                    stage('E2E Test'){
+                            steps {
+                                container('pytest') {
+                                    dir ('tests/scripts') {
+                                        script {
+                                                    def release_name=sh(returnStdout: true, script: './get_release_name.sh')
+                                                    def clusterEnabled = "false"
+                                                    int e2e_timeout_seconds = 6 * 60 * 60
+                                                    if ("${MILVUS_SERVER_TYPE}" == "distributed") {
+                                                        clusterEnabled = "true"
+                                                        e2e_timeout_seconds = 10 * 60 * 60
+                                                    }
+                                                    if ("${MILVUS_CLIENT}" == "pymilvus") {
+                                                        sh """ 
+                                                        MILVUS_HELM_RELEASE_NAME="${release_name}" \
+                                                        MILVUS_CLUSTER_ENABLED="${clusterEnabled}" \
+                                                        ./e2e-k8s.sh \
+                                                        --skip-export-logs \
+                                                        --skip-install \
+                                                        --skip-cleanup \
+                                                        --skip-setup \
+                                                        --skip-build \
+                                                        --skip-build-image --test-extra-arg "--tags L0 L1 L2 --repeat-scope=session" \
+                                                        --test-timeout ${e2e_timeout_seconds}
+                                                        """
+                                                    } else {
+                                                    error "Error: Unsupported Milvus client: ${MILVUS_CLIENT}"
+                                                    }
+                                        }
+                                    }
+                                }
+                            }
+
+                    }
+                }
+                post {
+                    unsuccessful {
+                        container('jnlp') {
+                            script {
+                                emailext subject: '$DEFAULT_SUBJECT',
+                                body: '$DEFAULT_CONTENT',
+                                recipientProviders: [requestor()],
+                                replyTo: '$DEFAULT_REPLYTO',
+                                to: "${authorEmail}"
+                            }
+                        }
+                    }
+                    success {
+                        container('main') {
+                            script {
+
+                                def date = sh(returnStdout: true, script: 'date +%Y%m%d').trim()
+                                def gitShortCommit = sh(returnStdout: true, script: 'git rev-parse --short HEAD').trim()
+                                def imageTag= "${env.BRANCH_NAME}-${date}-${gitShortCommit}"
+                                withCredentials([usernamePassword(credentialsId: "${env.DOCKER_CREDENTIALS_ID}", usernameVariable: 'DOCKER_USERNAME', passwordVariable: 'DOCKER_PASSWORD')]) {
+                                    sh 'docker login -u ${DOCKER_USERNAME} -p ${DOCKER_PASSWORD}'
+
+                                    //  Use ci registry instead of local registry
+                                    sh """
+                                        docker tag ${HUB}/milvus:${imageTag} ${TARGET_REPO}/milvus-nightly:${imageTag}
+                                        docker tag ${HUB}/milvus:${imageTag} ${TARGET_REPO}/milvus-nightly:${env.BRANCH_NAME}-latest
+                                        docker push ${TARGET_REPO}/milvus-nightly:${imageTag}
+                                        docker push ${TARGET_REPO}/milvus-nightly:${env.BRANCH_NAME}-latest
+                                    """
+                                    sh 'docker logout'
+                                }
+                            }
+                        }
+                    }
+                    always {
+                        container('pytest') {
+                            dir ('tests/scripts') {
+                            script {
+                                    def release_name=sh(returnStdout: true, script: './get_release_name.sh')
+                                    sh "./ci_logs.sh --log-dir /ci-logs  --artifacts-name ${env.ARTIFACTS}/artifacts-${PROJECT_NAME}-${MILVUS_SERVER_TYPE}-${SEMVER}-${env.BUILD_NUMBER}-${MILVUS_CLIENT}-e2e-logs \
+                                    --release-name ${release_name} "
+                                    dir("${env.ARTIFACTS}") {
+                                          if ("${MILVUS_CLIENT}" == "pymilvus") {
+                                            sh "tar -zcvf artifacts-${PROJECT_NAME}-${MILVUS_SERVER_TYPE}-${MILVUS_CLIENT}-pytest-logs.tar.gz /tmp/ci_logs/test --remove-files || true"
+                                        }
+                                        archiveArtifacts artifacts: "**.tar.gz", allowEmptyArchive: true
+                                    }
+                            }
+                            }
+                        }
+                    }
+                    cleanup {
+                        container('main') {
+                            dir ('tests/scripts') {  
+                                script {
+                                def release_name=sh(returnStdout: true, script: './get_release_name.sh')
+                                sh "./uninstall_milvus.sh --release-name ${release_name}"
+                                }
+                            }
+                        }
+                    }
+                }
+                }
+        }
+    }
+}

build/ci/jenkins/pod/rte.yaml

@@ -0,0 +1,59 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  labels:
+    app: milvus-e2e
+  namespace: jenkins
+spec:
+  enableServiceLinks: false
+  containers:
+  - name: main
+    image: milvusdb/krte:20210722-806d13f
+    env:
+    - name: DOCKER_IN_DOCKER_ENABLED
+      value: "true"
+    - name: DOCKER_VOLUME_DIRECTORY
+      value: "/mnt/disk/.docker"
+    tty: true
+    securityContext:
+      privileged: true
+    args: ["cat"]
+    volumeMounts:
+    - mountPath: /docker-graph
+      name: docker-graph
+    - mountPath: /var/lib/docker
+      name: docker-root
+    - mountPath: /lib/modules
+      name: modules
+      readOnly: true
+    - mountPath: /sys/fs/cgroup
+      name: cgroup
+    - mountPath: /mnt/disk/.docker
+      name: build-cache
+      subPath: docker-volume   
+  - name: pytest
+    image: milvusdb/pytest:20211108-d14cf36
+    volumeMounts:
+    - mountPath: /ci-logs
+      name: ci-logs  
+  volumes:
+  - emptyDir: {}
+    name: docker-graph
+  - emptyDir: {}
+    name: docker-root
+  - hostPath:
+      path: /tmp/krte/cache
+      type: DirectoryOrCreate
+    name: build-cache
+  - hostPath:
+      path: /lib/modules
+      type: Directory
+    name: modules
+  - hostPath:
+      path: /sys/fs/cgroup
+      type: Directory
+    name: cgroup
+  - name: ci-logs
+    nfs:
+      path: /volume1/ci-logs
+      server: 192.168.1.126

tests/scripts/ci-util.sh

@@ -0,0 +1,46 @@
+#!/bin/bash
+
+# Copyright 2018 Istio Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -e
+set -u
+set -x
+
+SOURCE="${BASH_SOURCE[0]}"
+while [ -h "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a symlink
+  DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )"
+  SOURCE="$(readlink "$SOURCE")"
+  [[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE" # if $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located
+done
+ROOT="$( cd -P "$( dirname "$SOURCE" )/../.." && pwd )"
+
+
+# Install pytest requirements
+function install_pytest_requirements(){
+ echo "Install pytest requirements"
+ cd ${ROOT}/tests/python_client
+ python3 -m pip install --no-cache-dir -r requirements.txt
+}
+
+function docker_login_ci_registry(){
+
+    if [[ -z "${CI_REGISTRY_USERNAME:-}" || -z "${CI_REGISTRY_PASSWORD:-}" ]]; then 
+       echo "Please setup docker credential for ci registry-${HUB}"
+    else
+        echo "docker login ci registry"
+        docker login  -u ${CI_REGISTRY_USERNAME} -p ${CI_REGISTRY_PASSWORD} ${HUB}
+    fi
+}
+

tests/scripts/ci_logs.sh

@@ -0,0 +1,79 @@
+#!/bin/bash
+
+# Copyright 2018 Istio Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -e
+set -u
+set -x
+
+SOURCE="${BASH_SOURCE[0]}"
+while [ -h "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a symlink
+  DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )"
+  SOURCE="$(readlink "$SOURCE")"
+  [[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE" # if $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located
+done
+ROOT="$( cd -P "$( dirname "$SOURCE" )/../.." && pwd )"
+
+while (( "$#" )); do
+  case "$1" in
+
+    --release-name)
+      RELEASE_NAME=$2
+      shift 2
+    ;;
+
+    --artifacts-name)
+      ARTIFACTS_NAME=$2
+      shift 2
+    ;;
+
+    --log-dir)
+      LOG_DIR=$2
+      shift 2
+    ;;
+
+    -h|--help)
+      { set +x; } 2>/dev/null
+      HELP="
+Usage:
+  $0 [flags] [Arguments]
+    
+    --log-dir                  Log Path
+
+    --artifacts-name           Artifacts Name
+
+    --release-name             Milvus helm release name
+
+    -h or --help                Print help information
+
+
+Use \"$0  --help\" for more information about a given command.
+"
+      echo -e "${HELP}" ; exit 0
+    ;;
+    -*)
+      echo "Error: Unsupported flag $1" >&2
+      exit 1
+      ;;
+    *) # preserve positional arguments
+      PARAMS+=("$1")
+      shift
+      ;;
+  esac
+done
+
+
+find ${LOG_DIR:-/ci-logs/} -type f  -name "*${RELEASE_NAME:-milvus-testing}*" \
+| xargs tar -zcvf ${ARTIFACTS_NAME:-artifacts}.tar.gz --remove-files || true

tests/scripts/e2e-k8s.sh

@@ -33,6 +33,10 @@ setup_and_export_git_sha
 # shellcheck source=build/kind_provisioner.sh
 source "${ROOT}/build/kind_provisioner.sh"
 
+# shellcheck source=ci-util.sh
+source "${ROOT}/tests/scripts/ci-util.sh"
+
+
 TOPOLOGY=SINGLE_CLUSTER
 NODE_IMAGE="kindest/node:v1.20.2"
 KIND_CONFIG=""
@@ -97,6 +101,10 @@ while (( "$#" )); do
       SKIP_EXPORT_LOGS=true
       shift
     ;;
+    --disable-kind)
+      DISABLE_KIND=true
+      shift
+    ;;
     --manual)
       MANUAL=true
       shift
@@ -164,6 +172,8 @@ Usage:
 
     --manual                    Manual Mode
 
+    --disable-kind              Remove KinD cluster
+
     -h or --help                Print help information
 
 
@@ -217,15 +227,6 @@ if [[ ! -d "${ARTIFACTS}" ]];then
   mkdir -p "${ARTIFACTS}"
 fi
 
-if [[ ! -x "$(command -v kind)" ]]; then
-  KIND_DIR="${KIND_DIR:-"${HOME}/tool_cache/kind"}"
-  KIND_VERSION="v0.11.1"
-
-  export PATH="${KIND_DIR}:${PATH}"
-  if [[ ! -x "$(command -v kind)" ]]; then
-    install_kind "${KIND_DIR}" "${KIND_VERSION}"
-  fi
-fi
 
 if [[ ! -x "$(command -v kubectl)" ]]; then
   KUBECTL_DIR="${KUBECTL_DIR:-"${HOME}/tool_cache/kubectl"}"
@@ -237,17 +238,20 @@ if [[ ! -x "$(command -v kubectl)" ]]; then
   fi
 fi
 
-if [[ ! -x "$(command -v helm)" ]]; then
+if [[  -z "${DISABLE_KIND:-}" ]];then
-  HELM_DIR="${HELM_DIR:-"${HOME}/tool_cache/helm"}"
+  if [[ ! -x "$(command -v kind)" ]]; then
-  HELM_VERSION="v3.5.4"
+    KIND_DIR="${KIND_DIR:-"${HOME}/tool_cache/kind"}"
+    KIND_VERSION="v0.11.1"
 
-  export PATH="${HELM_DIR}:${PATH}"
+    export PATH="${KIND_DIR}:${PATH}"
-  if [[ ! -x "$(command -v helm)" ]]; then
+    if [[ ! -x "$(command -v kind)" ]]; then
-    install_helm "${HELM_DIR}" "${HELM_VERSION}"
+      install_kind "${KIND_DIR}" "${KIND_VERSION}"
+    fi
   fi
 fi
 
 if [[ -z "${SKIP_SETUP:-}" ]]; then
+
   export DEFAULT_CLUSTER_YAML="${ROOT}/build/config/topology/trustworthy-jwt.yaml"
   export METRICS_SERVER_CONFIG_DIR="${ROOT}/build/config/metrics"
 
@@ -275,21 +279,37 @@ if [[ -z "${SKIP_SETUP:-}" ]]; then
 fi
 
 if [[ -z "${SKIP_BUILD:-}" ]]; then
-  trace "setup kind registry" setup_kind_registry
+  #If disable_kind exist, do not need kind registry
-  pushd "${ROOT}"
+  if [[ -n "${DISABLE_KIND:-}" ]]; then
-    trace "build milvus" "${ROOT}/build/builder.sh" /bin/bash -c "${BUILD_COMMAND}"
+    echo "do not set up kind registry when disable_kind exist"
-  popd
+  else
+    trace "setup kind registry" setup_kind_registry
+  fi 
+    pushd "${ROOT}"
+      trace "build milvus" "${ROOT}/build/builder.sh" /bin/bash -c "${BUILD_COMMAND}"
+    popd
+  
 fi
 
+export MILVUS_IMAGE_TAG="${TAG}"
+export MILVUS_IMAGE_REPO="${HUB}/milvus"
+
 if [[ -z "${SKIP_BUILD_IMAGE:-}" ]]; then
-  # If we're not intending to pull from an actual remote registry, use the local kind registry
+
-  running="$(docker inspect -f '{{.State.Running}}' "${KIND_REGISTRY_NAME}" 2>/dev/null || true)"
+  #[remove-kind] if disable_kind exist, do not need kind registry
-  if [[ "${running}" == 'true' ]]; then
+  if [[ -n "${DISABLE_KIND:-}" ]]; then
-    HUB="${KIND_REGISTRY}"
+      trace "docker login in ci registry" docker_login_ci_registry
-    export HUB
+  else
+    # If we're not intending to pull from an actual remote registry, use the local kind registry
+      running="$(docker inspect -f '{{.State.Running}}' "${KIND_REGISTRY_NAME}" 2>/dev/null || true)"
+      if [[ "${running}" == 'true' ]]; then
+        HUB="${KIND_REGISTRY}"
+        export HUB
+        export MILVUS_IMAGE_REPO="${HUB}/milvus"
+      fi
   fi
-  export MILVUS_IMAGE_REPO="${HUB}/milvus"
+  
-  export MILVUS_IMAGE_TAG="${TAG}"
+ 
 
   pushd "${ROOT}"
     # Build Milvus Docker Image
@@ -299,16 +319,38 @@ if [[ -z "${SKIP_BUILD_IMAGE:-}" ]]; then
 fi
 
 if [[ -z "${SKIP_INSTALL:-}" ]]; then
+  if [[ ! -x "$(command -v helm)" ]]; then
+    HELM_DIR="${HELM_DIR:-"${HOME}/tool_cache/helm"}"
+    HELM_VERSION="v3.5.4"
+
+    export PATH="${HELM_DIR}:${PATH}"
+    if [[ ! -x "$(command -v helm)" ]]; then
+      install_helm "${HELM_DIR}" "${HELM_VERSION}"
+    fi
+  fi
+  export MILVUS_HELM_RELEASE_NAME=$(./get_release_name.sh)
+  echo "[debug] helm install ${MILVUS_HELM_RELEASE_NAME}"
   trace "install milvus helm chart" "${ROOT}/tests/scripts/install_milvus.sh" "${INSTALL_EXTRA_ARG}"
 fi
 
 if [[ -z "${SKIP_TEST:-}" ]]; then
-  trace "prepare e2e test" "${ROOT}/tests/scripts/prepare_e2e.sh"
+# Prepare e2e test,install pytest requirements 
+  if [[ -n "${DISABLE_KIND:-}" ]]; then
+    trace "prepare e2e test"  install_pytest_requirements  
+  else
+    trace "prepare e2e test" "${ROOT}/tests/scripts/prepare_e2e.sh"
+  fi
+
   if [[ -n "${TEST_TIMEOUT:-}" ]]; then
-    trace "e2e test" "timeout" "-v" "${TEST_TIMEOUT}" "${ROOT}/tests/scripts/e2e.sh" "${TEST_EXTRA_ARG}"
+    if [[ -n "${DISABLE_KIND:-}" ]]; then
+      trace "e2e test" "timeout" "${TEST_TIMEOUT}" "${ROOT}/tests/scripts/e2e.sh" "${TEST_EXTRA_ARG}"
+    else
+      trace "e2e test" "timeout" "-v" "${TEST_TIMEOUT}" "${ROOT}/tests/scripts/e2e.sh" "${TEST_EXTRA_ARG}"
+    fi
   else
     trace "e2e test" "${ROOT}/tests/scripts/e2e.sh" "${TEST_EXTRA_ARG}"
   fi
+
 fi
 
 # Check if the user is running the clusters in manual mode.

tests/scripts/e2e.sh

@@ -48,23 +48,40 @@ if [[ "${TEST_ENV:-}" =~ ^kind*  ]]; then
     MILVUS_SERVICE_IP=$(kubectl get nodes --namespace "${MILVUS_HELM_NAMESPACE}" -o jsonpath='{.items[0].status.addresses[0].address}')
     MILVUS_SERVICE_PORT=$(kubectl get service --namespace "${MILVUS_HELM_NAMESPACE}" -l "${MILVUS_LABELS}" -o jsonpath='{.items[0].spec.ports[0].nodePort}')
   else
-    MILVUS_SERVICE_IP="127.0.0.1"
+    #[remove-kind] use service ip when disable kind to run ci test
-    POD_NAME=$(kubectl get pods --namespace "${MILVUS_HELM_NAMESPACE}" -l "${MILVUS_LABELS}" -o jsonpath='{.items[0].metadata.name}')
+    if [[ -n "${DISABLE_KIND:-}" ]]; then
-    MILVUS_SERVICE_PORT=$(kubectl get service --namespace "${MILVUS_HELM_NAMESPACE}" -l "${MILVUS_LABELS}" -o jsonpath='{.items[0].spec.ports[0].port}')
+      MILVUS_SERVICE_IP=$(kubectl get service --namespace "${MILVUS_HELM_NAMESPACE}" -l "${MILVUS_LABELS}" -o jsonpath='{.items[0].spec.clusterIP}')
-    kubectl --namespace "${MILVUS_HELM_NAMESPACE}" port-forward "${POD_NAME}" "${MILVUS_SERVICE_PORT}" &
+      MILVUS_SERVICE_PORT=$(kubectl get service --namespace "${MILVUS_HELM_NAMESPACE}" -l "${MILVUS_LABELS}" -o jsonpath='{.items[0].spec.ports[0].port}')
-    PORT_FORWARD_PID=$!
+    else
-    trap "kill -TERM ${PORT_FORWARD_PID}" EXIT
+      MILVUS_SERVICE_IP="127.0.0.1"
+      POD_NAME=$(kubectl get pods --namespace "${MILVUS_HELM_NAMESPACE}" -l "${MILVUS_LABELS}" -o jsonpath='{.items[0].metadata.name}')
+      MILVUS_SERVICE_PORT=$(kubectl get service --namespace "${MILVUS_HELM_NAMESPACE}" -l "${MILVUS_LABELS}" -o jsonpath='{.items[0].spec.ports[0].port}')
+      kubectl --namespace "${MILVUS_HELM_NAMESPACE}" port-forward "${POD_NAME}" "${MILVUS_SERVICE_PORT}" &
+      PORT_FORWARD_PID=$!
+      trap "kill -TERM ${PORT_FORWARD_PID}" EXIT
+    fi
   fi
 fi
-
+echo "[debug] DISABLE_KIND is ${DISABLE_KIND:-},PARALLEL_NUM is ${PARALLEL_NUM},MILVUS_SERVICE_IP is ${MILVUS_SERVICE_IP},MILVUS_SERVICE_PORT is ${MILVUS_SERVICE_PORT}"
-pushd "${ROOT}/tests/docker"
+#[remove-kind] use pytest run in the krte when remove kinD cluster
-  if [[ "${TEST_ENV:-}" =~ ^kind*  ]]; then
+if [[ -n "${DISABLE_KIND:-}" ]]; then
-    export PRE_EXIST_NETWORK="true"
+  cd ${ROOT}/tests/python_client
-    export PYTEST_NETWORK="kind"
+  python3 -V
+  export CI_LOG_PATH=/tmp/ci_logs/test
+  if [ ! -d "${CI_LOG_PATH}" ]; then
+    mkdir -p ${CI_LOG_PATH}
   fi
+  pytest -n ${PARALLEL_NUM} --host ${MILVUS_SERVICE_IP} --port ${MILVUS_SERVICE_PORT} \
+                                            --html=${CI_LOG_PATH}/report.html --self-contained-html ${@:-}
+else
+  pushd "${ROOT}/tests/docker"
+    if [[ "${TEST_ENV:-}" =~ ^kind*  ]]; then
+      export PRE_EXIST_NETWORK="true"
+      export PYTEST_NETWORK="kind"
+    fi
 
-  export MILVUS_SERVICE_IP="${MILVUS_SERVICE_IP:-127.0.0.1}"
+    export MILVUS_SERVICE_IP="${MILVUS_SERVICE_IP:-127.0.0.1}"
-  export MILVUS_SERVICE_PORT="${MILVUS_SERVICE_PORT:-19530}"
+    export MILVUS_SERVICE_PORT="${MILVUS_SERVICE_PORT:-19530}"
 
   if [[ "${MANUAL:-}" == "true" ]]; then
     docker-compose up -d
@@ -75,4 +92,5 @@ pushd "${ROOT}/tests/docker"
                                                    --html=\${CI_LOG_PATH}/report.html --self-contained-html ${@:-}"
     fi
   fi
-popd
+  popd
+fi

tests/scripts/get_release_name.sh

@@ -0,0 +1,48 @@
+#!/bin/bash
+
+# Copyright 2018 Istio Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -e
+set -u
+set -x
+function milvus_ci_release_name(){
+    # rules for helm release name 
+    local name="m"
+    if [[ "${MILVUS_SERVER_TYPE:-}" == "distributed" ]]; then
+        # distributed mode
+       name+="d"
+    else 
+       #standalone mode      
+        name+="s"
+
+    fi 
+    #[debug] add pr number into release name 
+    if [[ -n ${CHANGE_ID:-} ]]; then 
+        name+="-${CHANGE_ID:-}"
+    fi 
+
+    if [[ -n ${GIT_COMMIT:-} ]]; then 
+            name+="-${GIT_COMMIT:0:4}"
+    fi 
+
+    # [remove-kind] Add Jenkins BUILD_ID into Name
+    if [[ -n ${JENKINS_BUILD_ID:-} ]]; then 
+            name+="-${JENKINS_BUILD_ID}"
+    fi 
+
+    export MILVUS_HELM_RELEASE_NAME=${name}
+    echo ${name}
+}
+milvus_ci_release_name

tests/scripts/install_milvus.sh

@@ -17,6 +17,7 @@ set -e
 # Print commands
 set -x
 
+
 MILVUS_HELM_REPO="https://github.com/milvus-io/milvus-helm.git"
 MILVUS_HELM_RELEASE_NAME="${MILVUS_HELM_RELEASE_NAME:-milvus-testing}"
 MILVUS_CLUSTER_ENABLED="${MILVUS_CLUSTER_ENABLED:-false}"
@@ -42,6 +43,11 @@ else
   MILVUS_SERVICE_TYPE="${MILVUS_SERVICE_TYPE:-ClusterIP}"
 fi
 
+if [[ -n "${DISABLE_KIND:-}" ]]; then
+  # Use cluster IP to deploy milvus when kinD cluster is removed
+  MILVUS_SERVICE_TYPE="ClusterIP"
+fi
+
 # Get Milvus Chart from git
 if [[ ! -d "${MILVUS_HELM_CHART_PATH:-}" ]]; then
   TMP_DIR="$(mktemp -d)"
@@ -52,6 +58,7 @@ fi
 # Create namespace when it does not exist
 kubectl create namespace "${MILVUS_HELM_NAMESPACE}" > /dev/null 2>&1 || true
 
+echo "[debug]  cluster type is ${MILVUS_SERVICE_TYPE}"
 if [[ "${MILVUS_CLUSTER_ENABLED}" == "true" ]]; then
   helm install --wait --timeout "${MILVUS_INSTALL_TIMEOUT}" \
                                --set image.all.repository="${MILVUS_IMAGE_REPO}" \

tests/scripts/uninstall_milvus.sh

@@ -16,9 +16,47 @@ set -e
 # Print commands
 set -x
 
+while (( "$#" )); do
+  case "$1" in
+
+    --release-name)
+      RELEASE_NAME=$2
+      shift 2
+    ;;
+  
+    -h|--help)
+      { set +x; } 2>/dev/null
+      HELP="
+Usage:
+  $0 [flags] [Arguments]
+
+    --release-name             Milvus helm release name
+
+    -h or --help                Print help information
+
+
+Use \"$0  --help\" for more information about a given command.
+"
+      echo -e "${HELP}" ; exit 0
+    ;;
+    -*)
+      echo "Error: Unsupported flag $1" >&2
+      exit 1
+      ;;
+    *) # preserve positional arguments
+      PARAMS+=("$1")
+      shift
+      ;;
+  esac
+done
+
 MILVUS_HELM_RELEASE_NAME="${MILVUS_HELM_RELEASE_NAME:-milvus-testing}"
 MILVUS_HELM_NAMESPACE="${MILVUS_HELM_NAMESPACE:-default}"
 
+if [[ -n "${RELEASE_NAME:-}" ]]; then
+    MILVUS_HELM_RELEASE_NAME="${RELEASE_NAME}"
+fi
+
 # Uninstall Milvus Helm Release
 helm uninstall -n "${MILVUS_HELM_NAMESPACE}" "${MILVUS_HELM_RELEASE_NAME}"