Update CI for QA cluster (#15461)

Signed-off-by: Jenny Li <jing.li@zilliz.com>
2022-02-09 09:57:46 +08:00 · 2022-02-09 09:57:46 +08:00 · 5bd0de4d81
parent 3573c28af5
commit 5bd0de4d81
3 changed files with 190 additions and 23 deletions
--- a/build/ci/jenkins/qa/PR.groovy
+++ b/build/ci/jenkins/qa/PR.groovy
@ -4,22 +4,21 @@ int total_timeout_minutes = 120
 int e2e_timeout_seconds = 70 * 60
 def imageTag=''
 int case_timeout_seconds = 10 * 60
-def chart_version='2.4.25'
+def chart_version='3.0.1'
+
 pipeline {
    options {
-        timestamps()
        timeout(time: total_timeout_minutes, unit: 'MINUTES')
        buildDiscarder logRotator(artifactDaysToKeepStr: '30')
        parallelsAlwaysFailFast()
        preserveStashes(buildCount: 5)
-
    }
    agent {
            kubernetes {
                label 'milvus-qa-e2e-test-pipeline'
                inheritFrom 'default'
                defaultContainer 'main'
-                yamlFile 'build/ci/jenkins/pod/qa/rte.yaml'
+                yamlFile 'build/ci/jenkins/qa/pod/rte.yaml'
                customWorkspace '/home/jenkins/agent/workspace'
            }
    }
@ -28,17 +27,12 @@ pipeline {
        SEMVER = "${BRANCH_NAME.contains('/') ? BRANCH_NAME.substring(BRANCH_NAME.lastIndexOf('/') + 1) : BRANCH_NAME}"
        DOCKER_BUILDKIT = 1
        ARTIFACTS = "${env.WORKSPACE}/_artifacts"
-        DOCKER_CREDENTIALS_ID = "f0aacc8e-33f2-458a-ba9e-2c44f431b4d2"
-        TARGET_REPO = "milvusdb"
-        // CI_DOCKER_CREDENTIAL_ID = "ci-docker-registry"
        CI_DOCKER_CREDENTIAL_ID = "qa-ci-docker-registry"
        MILVUS_HELM_NAMESPACE = "milvus-ci"
        DISABLE_KIND = true
-        // HUB = 'registry.milvus.io/milvus'
        HUB = 'harbor.zilliz.cc/milvus-ci'
        JENKINS_BUILD_ID = "${env.BUILD_ID}"
        CI_MODE="pr"
-        MIRROR_URL="http://10.201.20.246:5000"
    }

    stages {
@ -108,7 +102,7 @@ pipeline {
                                                    try{
                                                        unstash 'imageTag'
                                                        imageTag=sh(returnStdout: true, script: 'cat imageTag.txt | tr -d \'\n\r\'')
-                                                    }catch(e){
+                                                    }catch (e){
                                                        print "No Image Tag info remained ,please rerun build to build new image."
                                                        exit 1
                                                    }
@ -142,13 +136,13 @@ pipeline {
                            }
                        }
                    }
-                    stage('E2E Test'){
+                     stage('E2E Test'){
                        agent {
                                kubernetes {
                                    label 'milvus-qa-e2e-test-pr'
                                    inheritFrom 'default'
                                    defaultContainer 'main'
-                                    yamlFile 'build/ci/jenkins/pod/qa/rte.yaml'
+                                    yamlFile 'build/ci/jenkins/qa/pod/rte.yaml'
                                    customWorkspace '/home/jenkins/agent/workspace'
                                }
                        }
@ -177,23 +171,26 @@ pipeline {
                                }
                            }
                        }
-
+                        post{
+                            always {
+                                container('pytest'){
+                                    dir("${env.ARTIFACTS}") {
+                                            sh "tar -zcvf ${PROJECT_NAME}-${MILVUS_SERVER_TYPE}-${MILVUS_CLIENT}-pytest-logs.tar.gz /tmp/ci_logs/test --remove-files || true"
+                                            archiveArtifacts artifacts: "${PROJECT_NAME}-${MILVUS_SERVER_TYPE}-${MILVUS_CLIENT}-pytest-logs.tar.gz ", allowEmptyArchive: true
+                                    }
+                                }
+                            }
+                        }
                    }
                }
                post{
                    always {
-                        container('pytest'){
-                            dir("${env.ARTIFACTS}") {
-                                    sh "tar -zcvf artifacts-${PROJECT_NAME}-${MILVUS_SERVER_TYPE}-${MILVUS_CLIENT}-pytest-logs.tar.gz /tmp/ci_logs/test --remove-files || true"
-                                    archiveArtifacts artifacts: "artifacts-${PROJECT_NAME}-${MILVUS_SERVER_TYPE}-${MILVUS_CLIENT}-pytest-logs.tar.gz ", allowEmptyArchive: true
-                            }
-                        }
                        container('main') {
                            dir ('tests/scripts') {  
                                script {
                                    def release_name=sh(returnStdout: true, script: './get_release_name.sh')
-                                    sh "./qa/uninstall_milvus.sh --release-name ${release_name}"
-                                    sh "./qa/ci_logs.sh --log-dir /ci-logs  --artifacts-name ${env.ARTIFACTS}/artifacts-${PROJECT_NAME}-${MILVUS_SERVER_TYPE}-${SEMVER}-${env.BUILD_NUMBER}-${MILVUS_CLIENT}-e2e-logs \
+                                    sh "./uninstall_milvus.sh --release-name ${release_name}"
+                                    sh "./ci_logs.sh --log-dir /ci-logs  --artifacts-name ${env.ARTIFACTS}/artifacts-${PROJECT_NAME}-${MILVUS_SERVER_TYPE}-${SEMVER}-${env.BUILD_NUMBER}-${MILVUS_CLIENT}-e2e-logs \
                                    --release-name ${release_name}"
                                    dir("${env.ARTIFACTS}") {
                                        archiveArtifacts artifacts: "artifacts-${PROJECT_NAME}-${MILVUS_SERVER_TYPE}-${SEMVER}-${env.BUILD_NUMBER}-${MILVUS_CLIENT}-e2e-logs.tar.gz", allowEmptyArchive: true
--- a/build/ci/jenkins/qa/pod/rte.yaml
+++ b/build/ci/jenkins/qa/pod/rte.yaml
@ -8,7 +8,7 @@ spec:
  enableServiceLinks: false
  containers:
  - name: main
-    image: harbor.zilliz.cc/dockerhub/milvusdb/krte:20211213-dcc15e9
+    image: harbor.zilliz.cc/ci/krte:20211213-dcc15e9
    env:
    - name: DOCKER_IN_DOCKER_ENABLED
      value: "true"
@ -38,8 +38,10 @@ spec:
    - mountPath: /mnt/disk/.docker
      name: build-cache
      subPath: docker-volume
+    - mountPath: /ci-logs
+      name: ci-logs  
  - name: pytest
-    image: harbor.zilliz.cc/dockerhub/milvusdb/pytest:20220118-3eb5cc4
+    image: harbor.zilliz.cc/ci/pytest:20220128-6d9612d
    resources:
      limits:
        cpu: "6"
@ -47,6 +49,9 @@ spec:
      requests:
        cpu: "0.5"
        memory: 5Gi
+    volumeMounts:
+    - mountPath: /ci-logs
+      name: ci-logs
  volumes:
  - emptyDir: {}
    name: docker-graph
@ -64,3 +69,7 @@ spec:
      path: /sys/fs/cgroup
      type: Directory
    name: cgroup
+  - name: ci-logs
+    nfs:
+      path: /volume1/ci-logs
+      server: 172.16.70.249
--- a/tests/scripts/values/qa/pr.yaml
+++ b/tests/scripts/values/qa/pr.yaml
@ -0,0 +1,161 @@
+metrics:
+  serviceMonitor:
+    enabled: true
+    interval: "5s"
+    scrapeTimeout: "5s"
+proxy:
+  resources:
+    requests:
+      cpu: "0.3"
+      memory: "256Mi"
+    # limits:
+      # cpu: "0.5"
+      # memory: "512Mi"
+
+rootCoordinator:
+  resources:
+    requests:
+      cpu: "0.3"
+      memory: "256Mi"
+    # limits:
+      # cpu: "0.6"
+      # memory: "512Mi"
+
+queryCoordinator:
+  resources:
+    requests:
+      cpu: "0.4"
+      memory: "100Mi"
+    # limits:
+      # cpu: "1.1"
+      # memory: "256Mi"
+
+queryNode:
+  resources:
+    requests:
+      cpu: "2"
+      memory: "500Mi"
+    limits:
+      # cpu: "2"
+      # memory: "1.5Gi"
+indexCoordinator:
+  resources:
+    requests:
+      cpu: "0.1"
+      memory: "50Mi"
+    # limits:
+      # cpu: "0.1"
+      # memory: "100Mi"
+indexNode:
+  resources:
+    requests:
+      cpu: "4"
+      memory: "500Mi"
+    # limits:
+      # cpu: "4"
+      # memory: "1.5Gi"
+
+dataCoordinator:
+  resources:
+    requests:
+      cpu: "0.1"
+      memory: "50Mi"
+    # limits:
+      # cpu: "0.2"
+      # memory: "100Mi"
+
+dataNode:
+  resources:
+    requests:
+      cpu: "0.5"
+      memory: "500Mi"
+    # limits:
+      # cpu: "1.2"
+      # memory: "1.5Gi"
+pulsar:
+  proxy:
+    resources:
+      requests:
+        cpu: "1"
+        memory: "2Gi"
+      # limits:
+        # cpu: "1"
+        # memory: "4Gi"
+  # Resources for the websocket proxy
+    wsResources:
+      requests:
+        memory: "512Mi"
+        cpu: "0.3"
+      # limits:
+      #   memory: "512Mi"
+        # cpu: "0.3"
+  broker:
+    resources:
+      requests:
+        cpu: "1.5"
+        memory: "4Gi"
+      # limits:
+        # cpu: "1.5"
+        # memory: "8Gi"
+  bookkeeper:
+    volumes:
+      ledgers:
+        existingStorageClassName: local-path
+    resources:
+      requests:
+        cpu: 1
+        memory: "4Gi"
+      # limits:
+        # cpu: 1
+        # memory: "8Gi"
+  bastion:
+    resources:
+      requests:
+        cpu: "0.3"
+        memory: "50Mi"
+      # limits:
+        # cpu: "0.3"
+        # memory: "50Mi"
+  autorecovery:
+    resources:
+      requests:
+        cpu: "1"
+        memory: "512Mi"
+      # limits:
+        # cpu: "1"
+        # memory: "1Gi"
+  zookeeper:
+    volumes:
+      data:
+        existingStorageClassName: local-path
+    resources:
+      requests:
+        cpu: "0.3"
+        memory: "1Gi"
+      # limits:
+        # cpu: "0.3"
+        # memory: "2Gi"
+etcd:
+  resources:
+    requests:
+      cpu: "0.1"
+      memory: "100Mi"
+    # limits:
+      # cpu: "0.1"
+      # memory: "300Mi"
+minio:
+  resources:
+    requests:
+      cpu: "0.3"
+      memory: "512Mi"
+    # limits:
+      # cpu: "0.6"
+      # memory: "512Mi"
+standalone:
+  resources:
+    requests:
+      cpu: "1"
+      memory: "3.5Gi"
+    # limits:
+      # cpu: "2.6"
+      # memory: "8.5Gi"