From 299b0c6b660fadc78978d04339311d71041ff294 Mon Sep 17 00:00:00 2001 From: quicksilver Date: Sat, 24 Aug 2019 17:35:29 +0800 Subject: [PATCH] add staging test stage Former-commit-id: 081f1cc16b276ff4695024953ac3458ae53c2ead --- ci/jenkinsfile/cleanup_staging.groovy | 13 +++ ci/jenkinsfile/deploy2staging.groovy | 16 +++ ci/jenkinsfile/dev_test.groovy | 6 +- ci/jenkinsfile/staging_test.groovy | 31 +++++ ci/jenkinsfile/upload_dev_test_out.groovy | 4 +- ci/jenkinsfile/upload_staging_test_out.groovy | 13 +++ ci/nightly_main_jenkinsfile | 109 +++++++++++++++++- 7 files changed, 181 insertions(+), 11 deletions(-) create mode 100644 ci/jenkinsfile/cleanup_staging.groovy create mode 100644 ci/jenkinsfile/deploy2staging.groovy create mode 100644 ci/jenkinsfile/staging_test.groovy create mode 100644 ci/jenkinsfile/upload_staging_test_out.groovy diff --git a/ci/jenkinsfile/cleanup_staging.groovy b/ci/jenkinsfile/cleanup_staging.groovy new file mode 100644 index 0000000000..2e9332fa6e --- /dev/null +++ b/ci/jenkinsfile/cleanup_staging.groovy @@ -0,0 +1,13 @@ +try { + def result = sh script: "helm status ${env.JOB_NAME}-${env.BUILD_NUMBER}", returnStatus: true + if (!result) { + sh "helm del --purge ${env.JOB_NAME}-${env.BUILD_NUMBER}" + } +} catch (exc) { + def result = sh script: "helm status ${env.JOB_NAME}-${env.BUILD_NUMBER}", returnStatus: true + if (!result) { + sh "helm del --purge ${env.JOB_NAME}-${env.BUILD_NUMBER}" + } + throw exc +} + diff --git a/ci/jenkinsfile/deploy2staging.groovy b/ci/jenkinsfile/deploy2staging.groovy new file mode 100644 index 0000000000..8079698894 --- /dev/null +++ b/ci/jenkinsfile/deploy2staging.groovy @@ -0,0 +1,16 @@ +try { + sh 'helm init --client-only --skip-refresh --stable-repo-url https://kubernetes.oss-cn-hangzhou.aliyuncs.com/charts' + sh 'helm repo add milvus https://registry.zilliz.com/chartrepo/milvus' + sh 'helm repo update' + dir ("milvus-helm") { + checkout([$class: 'GitSCM', branches: [[name: "${SEMVER}"]], doGenerateSubmoduleConfigurations: false, extensions: [], submoduleCfg: [], userRemoteConfigs: [[credentialsId: "${params.GIT_USER}", url: "git@192.168.1.105:megasearch/milvus-helm.git", name: 'origin', refspec: "+refs/heads/${SEMVER}:refs/remotes/origin/${SEMVER}"]]]) + dir ("milvus/milvus-gpu") { + sh "helm install --wait --timeout 300 --set engine.image.repository="zilliz.azurecr.cn/milvus/engine" --set engine.image.tag=${DOCKER_VERSION} --set expose.type=loadBalancer --name ${env.JOB_NAME}-${env.BUILD_NUMBER} -f ci/values.yaml --namespace milvus-1 --version 0.4.0 ." + } + } +} catch (exc) { + echo 'Helm running failed!' + sh "helm del --purge ${env.JOB_NAME}-${env.BUILD_NUMBER}" + throw exc +} + diff --git a/ci/jenkinsfile/dev_test.groovy b/ci/jenkinsfile/dev_test.groovy index fc3badae0c..8ae7e4c478 100644 --- a/ci/jenkinsfile/dev_test.groovy +++ b/ci/jenkinsfile/dev_test.groovy @@ -3,7 +3,7 @@ timeout(time: 40, unit: 'MINUTES') { dir ("${PROJECT_NAME}_test") { checkout([$class: 'GitSCM', branches: [[name: "${SEMVER}"]], doGenerateSubmoduleConfigurations: false, extensions: [], submoduleCfg: [], userRemoteConfigs: [[credentialsId: "${params.GIT_USER}", url: "git@192.168.1.105:Test/milvus_test.git", name: 'origin', refspec: "+refs/heads/${SEMVER}:refs/remotes/origin/${SEMVER}"]]]) sh 'python3 -m pip install -r requirements.txt' - sh "pytest . --alluredir=test_out --ip ${env.JOB_NAME}-${env.BUILD_NUMBER}-milvus-gpu-engine.milvus-1.svc.cluster.local" + sh "pytest . --alluredir="test_out/dev/single/sqlite" --ip ${env.JOB_NAME}-${env.BUILD_NUMBER}-milvus-gpu-engine.milvus-1.svc.cluster.local" } // mysql database backend test @@ -16,11 +16,11 @@ timeout(time: 40, unit: 'MINUTES') { } dir ("milvus-helm") { dir ("milvus/milvus-gpu") { - sh "helm install --wait --timeout 300 --set engine.image.tag=${DOCKER_VERSION} --set expose.type=clusterIP --name ${env.JOB_NAME}-${env.BUILD_NUMBER} -f ci/db_backend/mysql_values.yaml --namespace milvus-2 --version 0.3.1 ." + sh "helm install --wait --timeout 300 --set engine.image.tag=${DOCKER_VERSION} --set expose.type=clusterIP --name ${env.JOB_NAME}-${env.BUILD_NUMBER} -f ci/db_backend/mysql_values.yaml --namespace milvus-2 --version 0.4.0 ." } } dir ("${PROJECT_NAME}_test") { - sh "pytest . --alluredir=test_out --ip ${env.JOB_NAME}-${env.BUILD_NUMBER}-milvus-gpu-engine.milvus-2.svc.cluster.local" + sh "pytest . --alluredir="test_out/dev/single/mysql" --ip ${env.JOB_NAME}-${env.BUILD_NUMBER}-milvus-gpu-engine.milvus-2.svc.cluster.local" } } catch (exc) { echo 'Milvus Test Failed !' diff --git a/ci/jenkinsfile/staging_test.groovy b/ci/jenkinsfile/staging_test.groovy new file mode 100644 index 0000000000..39fceb1538 --- /dev/null +++ b/ci/jenkinsfile/staging_test.groovy @@ -0,0 +1,31 @@ +timeout(time: 40, unit: 'MINUTES') { + try { + dir ("${PROJECT_NAME}_test") { + checkout([$class: 'GitSCM', branches: [[name: "${SEMVER}"]], doGenerateSubmoduleConfigurations: false, extensions: [], submoduleCfg: [], userRemoteConfigs: [[credentialsId: "${params.GIT_USER}", url: "git@192.168.1.105:Test/milvus_test.git", name: 'origin', refspec: "+refs/heads/${SEMVER}:refs/remotes/origin/${SEMVER}"]]]) + sh 'python3 -m pip install -r requirements.txt' + def service_ip = sh (script: "kubectl get svc --namespace milvus-1 ${env.JOB_NAME}-${env.BUILD_NUMBER}-milvus-gpu-engine --template \"{{range .status.loadBalancer.ingress}}{{.ip}}{{end}}\"",returnStdout: true).trim() + sh "pytest . --alluredir="test_out/staging/single/sqlite" --ip ${service_ip}" + } + + // mysql database backend test + load "${env.WORKSPACE}/ci/jenkinsfile/cleanup_staging.groovy" + + if (!fileExists('milvus-helm')) { + dir ("milvus-helm") { + checkout([$class: 'GitSCM', branches: [[name: "${SEMVER}"]], doGenerateSubmoduleConfigurations: false, extensions: [], submoduleCfg: [], userRemoteConfigs: [[credentialsId: "${params.GIT_USER}", url: "git@192.168.1.105:megasearch/milvus-helm.git", name: 'origin', refspec: "+refs/heads/${SEMVER}:refs/remotes/origin/${SEMVER}"]]]) + } + } + dir ("milvus-helm") { + dir ("milvus/milvus-gpu") { + sh "helm install --wait --timeout 300 --set engine.image.repository="zilliz.azurecr.cn/milvus/engine" --set engine.image.tag=${DOCKER_VERSION} --set expose.type=loadBalancer --name ${env.JOB_NAME}-${env.BUILD_NUMBER} -f ci/db_backend/mysql_values.yaml --namespace milvus-2 --version 0.4.0 ." + } + } + dir ("${PROJECT_NAME}_test") { + def service_ip = sh (script: "kubectl get svc --namespace milvus-2 ${env.JOB_NAME}-${env.BUILD_NUMBER}-milvus-gpu-engine --template \"{{range .status.loadBalancer.ingress}}{{.ip}}{{end}}\"",returnStdout: true).trim() + sh "pytest . --alluredir="test_out/staging/single/mysql" --ip ${service_ip}" + } + } catch (exc) { + echo 'Milvus Test Failed !' + throw exc + } +} diff --git a/ci/jenkinsfile/upload_dev_test_out.groovy b/ci/jenkinsfile/upload_dev_test_out.groovy index 7e106c0296..017b887334 100644 --- a/ci/jenkinsfile/upload_dev_test_out.groovy +++ b/ci/jenkinsfile/upload_dev_test_out.groovy @@ -1,8 +1,8 @@ timeout(time: 5, unit: 'MINUTES') { dir ("${PROJECT_NAME}_test") { - if (fileExists('test_out')) { + if (fileExists('test_out/dev')) { def fileTransfer = load "${env.WORKSPACE}/ci/function/file_transfer.groovy" - fileTransfer.FileTransfer("test_out/", "${PROJECT_NAME}/test/${JOB_NAME}-${BUILD_ID}", 'nas storage') + fileTransfer.FileTransfer("test_out/dev/", "${PROJECT_NAME}/test/${JOB_NAME}-${BUILD_ID}", 'nas storage') if (currentBuild.resultIsBetterOrEqualTo('SUCCESS')) { echo "Milvus Dev Test Out Viewer \"ftp://192.168.1.126/data/${PROJECT_NAME}/test/${JOB_NAME}-${BUILD_ID}\"" } diff --git a/ci/jenkinsfile/upload_staging_test_out.groovy b/ci/jenkinsfile/upload_staging_test_out.groovy new file mode 100644 index 0000000000..1f1e66ab1b --- /dev/null +++ b/ci/jenkinsfile/upload_staging_test_out.groovy @@ -0,0 +1,13 @@ +timeout(time: 5, unit: 'MINUTES') { + dir ("${PROJECT_NAME}_test") { + if (fileExists('test_out/staging')) { + def fileTransfer = load "${env.WORKSPACE}/ci/function/file_transfer.groovy" + fileTransfer.FileTransfer("test_out/staging/", "${PROJECT_NAME}/test/${JOB_NAME}-${BUILD_ID}", 'nas storage') + if (currentBuild.resultIsBetterOrEqualTo('SUCCESS')) { + echo "Milvus Dev Test Out Viewer \"ftp://192.168.1.126/data/${PROJECT_NAME}/test/${JOB_NAME}-${BUILD_ID}\"" + } + } else { + error("Milvus Dev Test Out directory don't exists!") + } + } +} diff --git a/ci/nightly_main_jenkinsfile b/ci/nightly_main_jenkinsfile index e92d4f14f8..6f6773b6e7 100644 --- a/ci/nightly_main_jenkinsfile +++ b/ci/nightly_main_jenkinsfile @@ -208,17 +208,17 @@ spec: } success { script { - echo "Milvus Single Node CI/CD success !" + echo "Milvus Deploy to Dev Single Node CI/CD success !" } } aborted { script { - echo "Milvus Single Node CI/CD aborted !" + echo "Milvus Deploy to Dev Single Node CI/CD aborted !" } } failure { script { - echo "Milvus Single Node CI/CD failure !" + echo "Milvus Deploy to Dev Single Node CI/CD failure !" } } } @@ -300,17 +300,114 @@ spec: } success { script { - echo "Milvus Cluster CI/CD success !" + echo "Milvus Deploy to Dev Cluster CI/CD success !" } } aborted { script { - echo "Milvus Cluster CI/CD aborted !" + echo "Milvus Deploy to Dev Cluster CI/CD aborted !" } } failure { script { - echo "Milvus Cluster CI/CD failure !" + echo "Milvus Deploy to Dev Cluster CI/CD failure !" + } + } + } + } + } + } + + stage("Deploy to Staging") { + parallel { + stage("Single Node") { + agent { + kubernetes { + label 'dev-test' + defaultContainer 'jnlp' + yaml """ +apiVersion: v1 +kind: Pod +metadata: + labels: + app: milvus + componet: test +spec: + containers: + - name: milvus-testframework + image: registry.zilliz.com/milvus/milvus-test:v0.2 + command: + - cat + tty: true + volumeMounts: + - name: kubeconf + mountPath: /root/.kube/ + readOnly: true + volumes: + - name: kubeconf + secret: + secretName: aks-gpu-cluster-config +""" + } + } + + stages { + stage("Deploy to Staging") { + steps { + gitlabCommitStatus(name: 'Deloy to Staging') { + container('milvus-testframework') { + script { + load "${env.WORKSPACE}/ci/jenkinsfile/deploy2staging.groovy" + } + } + } + } + } + stage("Staging Test") { + steps { + gitlabCommitStatus(name: 'Staging Test') { + container('milvus-testframework') { + script { + load "${env.WORKSPACE}/ci/jenkinsfile/staging_test.groovy" + load "${env.WORKSPACE}/ci/jenkinsfile/upload_staging_test_out.groovy" + } + } + } + } + } + stage ("Cleanup Staging") { + steps { + gitlabCommitStatus(name: 'Cleanup Staging') { + container('milvus-testframework') { + script { + load "${env.WORKSPACE}/ci/jenkinsfile/cleanup_staging.groovy" + } + } + } + } + } + } + post { + always { + container('milvus-testframework') { + script { + load "${env.WORKSPACE}/ci/jenkinsfile/cleanup_staging.groovy" + } + } + } + success { + script { + echo "Milvus Deploy to Staging Single Node CI/CD success !" + } + } + aborted { + script { + echo "Milvus Deploy to Staging Single Node CI/CD aborted !" + } + } + failure { + script { + echo "Milvus Deploy to Staging Single Node CI/CD failure !" } } }