From bb6aac99e8208d3bcc8d6dfece4899d1cc39eaaa Mon Sep 17 00:00:00 2001 From: quicksilver Date: Fri, 12 Jul 2019 23:26:03 +0800 Subject: [PATCH 1/3] add milvus cluster CI/CD groovy file Former-commit-id: ee77bea9949bb1dbb58bf1d7de1f4af5c0237b1f --- ci/jenkinsfile/cleanup_dev.groovy | 1 + ci/jenkinsfile/cluster_cleanup_dev.groovy | 13 + ci/jenkinsfile/cluster_deploy2dev.groovy | 21 ++ ci/jenkinsfile/cluster_dev_test.groovy | 18 ++ ci/jenkinsfile/deploy2dev.groovy | 7 +- ci/jenkinsfile/dev_test.groovy | 3 +- ci/jenkinsfile/milvus_build.groovy | 3 +- ci/jenkinsfile/milvus_build_no_ut.groovy | 5 +- ci/jenkinsfile/publish_docker.groovy | 3 +- .../upload_dev_cluster_test_out.groovy | 26 ++ ci/main_jenkinsfile | 234 +++++++++++----- ci/main_jenkinsfile_no_ut | 250 +++++++++++++----- ci/nightly_main_jenkinsfile | 250 +++++++++++++----- 13 files changed, 634 insertions(+), 200 deletions(-) create mode 100644 ci/jenkinsfile/cluster_cleanup_dev.groovy create mode 100644 ci/jenkinsfile/cluster_deploy2dev.groovy create mode 100644 ci/jenkinsfile/cluster_dev_test.groovy create mode 100644 ci/jenkinsfile/upload_dev_cluster_test_out.groovy diff --git a/ci/jenkinsfile/cleanup_dev.groovy b/ci/jenkinsfile/cleanup_dev.groovy index 32ee43d3b1..40f354275f 100644 --- a/ci/jenkinsfile/cleanup_dev.groovy +++ b/ci/jenkinsfile/cleanup_dev.groovy @@ -10,3 +10,4 @@ try { updateGitlabCommitStatus name: 'Cleanup Dev', state: 'failed' throw exc } + diff --git a/ci/jenkinsfile/cluster_cleanup_dev.groovy b/ci/jenkinsfile/cluster_cleanup_dev.groovy new file mode 100644 index 0000000000..57328c4524 --- /dev/null +++ b/ci/jenkinsfile/cluster_cleanup_dev.groovy @@ -0,0 +1,13 @@ +try { + sh "helm del --purge ${env.JOB_NAME}-${env.BUILD_NUMBER}-cluster" + + if (currentBuild.result == 'ABORTED') { + throw new hudson.AbortException("Cluster Dev Test Aborted !") + } else if (currentBuild.result == 'FAILURE') { + error("Dev Test Failure !") + } +} catch (exc) { + updateGitlabCommitStatus name: 'Cleanup Dev', state: 'failed' + throw exc +} + diff --git a/ci/jenkinsfile/cluster_deploy2dev.groovy b/ci/jenkinsfile/cluster_deploy2dev.groovy new file mode 100644 index 0000000000..97062922ca --- /dev/null +++ b/ci/jenkinsfile/cluster_deploy2dev.groovy @@ -0,0 +1,21 @@ +try { + sh 'helm init --client-only --skip-refresh --stable-repo-url https://kubernetes.oss-cn-hangzhou.aliyuncs.com/charts' + sh 'helm repo add milvus https://registry.zilliz.com/chartrepo/milvus' + sh 'helm repo update' + dir ("milvus-helm") { + checkout([$class: 'GitSCM', branches: [[name: "${SEMVER}"]], doGenerateSubmoduleConfigurations: false, extensions: [[$class: 'SubmoduleOption',disableSubmodules: false,parentCredentials: true,recursiveSubmodules: true,reference: '',trackingSubmodules: false]], submoduleCfg: [], userRemoteConfigs: [[credentialsId: "${params.GIT_USER}", url: "git@192.168.1.105:megasearch/milvus-helm.git", name: 'origin', refspec: "+refs/heads/${SEMVER}:refs/remotes/origin/${SEMVER}"]]]) + dir ("milvus/milvus-cluster") { + sh "helm install --set roServers.image.tag=${DOCKER_VERSION} --set woServers.image.tag=${DOCKER_VERSION} --set expose.type=clusterIP -f ci/values.yaml --name ${env.JOB_NAME}-${env.BUILD_NUMBER}-cluster --namespace milvus-cluster --version 0.1.0 . " + } + waitUntil { + def result = sh script: "nc -z -w 2 ${env.JOB_NAME}-${env.BUILD_NUMBER}-cluster-milvus-cluster-proxy.milvus-cluster.svc.cluster.local 19530", returnStatus: true + return !result + } + } +} catch (exc) { + updateGitlabCommitStatus name: 'Deloy to Dev', state: 'failed' + echo 'Helm running failed!' + sh "helm del --purge ${env.JOB_NAME}-${env.BUILD_NUMBER}-cluster" + throw exc +} + diff --git a/ci/jenkinsfile/cluster_dev_test.groovy b/ci/jenkinsfile/cluster_dev_test.groovy new file mode 100644 index 0000000000..50fd67f803 --- /dev/null +++ b/ci/jenkinsfile/cluster_dev_test.groovy @@ -0,0 +1,18 @@ +container('milvus-testframework') { + timeout(time: 10, unit: 'MINUTES') { + gitlabCommitStatus(name: 'Dev Test') { + try { + dir ("${PROJECT_NAME}_test") { + checkout([$class: 'GitSCM', branches: [[name: "${SEMVER}"]], doGenerateSubmoduleConfigurations: false, extensions: [], submoduleCfg: [], userRemoteConfigs: [[credentialsId: "${params.GIT_USER}", url: "git@192.168.1.105:Test/milvus_test.git", name: 'origin', refspec: "+refs/heads/${SEMVER}:refs/remotes/origin/${SEMVER}"]]]) + sh 'python3 -m pip install -r requirements.txt' + sh "pytest . --alluredir=cluster_test_out --ip ${env.JOB_NAME}-${env.BUILD_NUMBER}-cluster-milvus-cluster-proxy.milvus-cluster.svc.cluster.local" + } + } catch (exc) { + updateGitlabCommitStatus name: 'Dev Test', state: 'failed' + currentBuild.result = 'FAILURE' + echo 'Milvus Test Failed !' + } + } + } +} + diff --git a/ci/jenkinsfile/deploy2dev.groovy b/ci/jenkinsfile/deploy2dev.groovy index 47497e1648..f594d0ba57 100644 --- a/ci/jenkinsfile/deploy2dev.groovy +++ b/ci/jenkinsfile/deploy2dev.groovy @@ -2,10 +2,15 @@ try { sh 'helm init --client-only --skip-refresh --stable-repo-url https://kubernetes.oss-cn-hangzhou.aliyuncs.com/charts' sh 'helm repo add milvus https://registry.zilliz.com/chartrepo/milvus' sh 'helm repo update' - sh "helm install --set engine.image.repository=registry.zilliz.com/${PROJECT_NAME}/engine --set engine.image.tag=${DOCKER_VERSION} --set expose.type=clusterIP --name ${env.JOB_NAME}-${env.BUILD_NUMBER} --version 0.3.0 milvus/milvus-gpu" + sh "helm install --set engine.image.tag=${DOCKER_VERSION} --set expose.type=clusterIP --name ${env.JOB_NAME}-${env.BUILD_NUMBER} --version 0.3.0 milvus/milvus-gpu" + waitUntil { + def result = sh script: "nc -z -w 2 ${env.JOB_NAME}-${env.BUILD_NUMBER}-milvus-gpu-engine.kube-opt.svc.cluster.local 19530", returnStatus: true + return !result + } } catch (exc) { updateGitlabCommitStatus name: 'Deloy to Dev', state: 'failed' echo 'Helm running failed!' sh "helm del --purge ${env.JOB_NAME}-${env.BUILD_NUMBER}" throw exc } + diff --git a/ci/jenkinsfile/dev_test.groovy b/ci/jenkinsfile/dev_test.groovy index f5808cef40..13e1c868b8 100644 --- a/ci/jenkinsfile/dev_test.groovy +++ b/ci/jenkinsfile/dev_test.groovy @@ -3,7 +3,7 @@ container('milvus-testframework') { gitlabCommitStatus(name: 'Dev Test') { try { dir ("${PROJECT_NAME}_test") { - checkout([$class: 'GitSCM', branches: [[name: "${SEMVER}"]], doGenerateSubmoduleConfigurations: false, extensions: [], submoduleCfg: [], userRemoteConfigs: [[credentialsId: "${params.GIT_USER}", url: "git@192.168.1.105:Test/milvus_test.git"]]]) + checkout([$class: 'GitSCM', branches: [[name: "${SEMVER}"]], doGenerateSubmoduleConfigurations: false, extensions: [], submoduleCfg: [], userRemoteConfigs: [[credentialsId: "${params.GIT_USER}", url: "git@192.168.1.105:Test/milvus_test.git", name: 'origin', refspec: "+refs/heads/${SEMVER}:refs/remotes/origin/${SEMVER}"]]]) sh 'python3 -m pip install -r requirements.txt' sh "pytest . --alluredir=test_out --ip ${env.JOB_NAME}-${env.BUILD_NUMBER}-milvus-gpu-engine.kube-opt.svc.cluster.local" } @@ -15,3 +15,4 @@ container('milvus-testframework') { } } } + diff --git a/ci/jenkinsfile/milvus_build.groovy b/ci/jenkinsfile/milvus_build.groovy index 243b6dc2da..99892c90ef 100644 --- a/ci/jenkinsfile/milvus_build.groovy +++ b/ci/jenkinsfile/milvus_build.groovy @@ -3,7 +3,7 @@ container('milvus-build-env') { gitlabCommitStatus(name: 'Build Engine') { dir ("milvus_engine") { try { - checkout([$class: 'GitSCM', branches: [[name: "${SEMVER}"]], doGenerateSubmoduleConfigurations: false, extensions: [[$class: 'SubmoduleOption',disableSubmodules: false,parentCredentials: true,recursiveSubmodules: true,reference: '',trackingSubmodules: false]], submoduleCfg: [], userRemoteConfigs: [[credentialsId: "${params.GIT_USER}", url: "git@192.168.1.105:megasearch/milvus.git"]]]) + checkout([$class: 'GitSCM', branches: [[name: "${SEMVER}"]], doGenerateSubmoduleConfigurations: false, extensions: [[$class: 'SubmoduleOption',disableSubmodules: false,parentCredentials: true,recursiveSubmodules: true,reference: '',trackingSubmodules: false]], submoduleCfg: [], userRemoteConfigs: [[credentialsId: "${params.GIT_USER}", url: "git@192.168.1.105:megasearch/milvus.git", name: 'origin', refspec: "+refs/heads/${SEMVER}:refs/remotes/origin/${SEMVER}"]]]) dir ("cpp") { sh "git config --global user.email \"test@zilliz.com\"" sh "git config --global user.name \"test\"" @@ -17,3 +17,4 @@ container('milvus-build-env') { } } } + diff --git a/ci/jenkinsfile/milvus_build_no_ut.groovy b/ci/jenkinsfile/milvus_build_no_ut.groovy index 02b971de2f..6c30ce491c 100644 --- a/ci/jenkinsfile/milvus_build_no_ut.groovy +++ b/ci/jenkinsfile/milvus_build_no_ut.groovy @@ -3,8 +3,10 @@ container('milvus-build-env') { gitlabCommitStatus(name: 'Build Engine') { dir ("milvus_engine") { try { - checkout([$class: 'GitSCM', branches: [[name: "${SEMVER}"]], doGenerateSubmoduleConfigurations: false, extensions: [[$class: 'SubmoduleOption',disableSubmodules: false,parentCredentials: true,recursiveSubmodules: true,reference: '',trackingSubmodules: false]], submoduleCfg: [], userRemoteConfigs: [[credentialsId: "${params.GIT_USER}", url: "git@192.168.1.105:megasearch/milvus.git"]]]) + checkout([$class: 'GitSCM', branches: [[name: "${SEMVER}"]], doGenerateSubmoduleConfigurations: false, extensions: [[$class: 'SubmoduleOption',disableSubmodules: false,parentCredentials: true,recursiveSubmodules: true,reference: '',trackingSubmodules: false]], submoduleCfg: [], userRemoteConfigs: [[credentialsId: "${params.GIT_USER}", url: "git@192.168.1.105:megasearch/milvus.git", name: 'origin', refspec: "+refs/heads/${SEMVER}:refs/remotes/origin/${SEMVER}"]]]) dir ("cpp") { + sh "git config --global user.email \"test@zilliz.com\"" + sh "git config --global user.name \"test\"" sh "./build.sh -t ${params.BUILD_TYPE}" } } catch (exc) { @@ -15,3 +17,4 @@ container('milvus-build-env') { } } } + diff --git a/ci/jenkinsfile/publish_docker.groovy b/ci/jenkinsfile/publish_docker.groovy index 04f1a8567d..7416c9cb17 100644 --- a/ci/jenkinsfile/publish_docker.groovy +++ b/ci/jenkinsfile/publish_docker.groovy @@ -3,7 +3,7 @@ container('publish-docker') { gitlabCommitStatus(name: 'Publish Engine Docker') { try { dir ("${PROJECT_NAME}_build") { - checkout([$class: 'GitSCM', branches: [[name: "${SEMVER}"]], doGenerateSubmoduleConfigurations: false, extensions: [], submoduleCfg: [], userRemoteConfigs: [[credentialsId: "${params.GIT_USER}", url: "git@192.168.1.105:build/milvus_build.git"]]]) + checkout([$class: 'GitSCM', branches: [[name: "${SEMVER}"]], doGenerateSubmoduleConfigurations: false, extensions: [], submoduleCfg: [], userRemoteConfigs: [[credentialsId: "${params.GIT_USER}", url: "git@192.168.1.105:build/milvus_build.git", name: 'origin', refspec: "+refs/heads/${SEMVER}:refs/remotes/origin/${SEMVER}"]]]) dir ("docker/deploy/ubuntu16.04/free_version") { sh "curl -O -u anonymous: ftp://192.168.1.126/data/${PROJECT_NAME}/engine/${JOB_NAME}-${BUILD_ID}/${PROJECT_NAME}-engine-${PACKAGE_VERSION}.tar.gz" sh "tar zxvf ${PROJECT_NAME}-engine-${PACKAGE_VERSION}.tar.gz" @@ -29,3 +29,4 @@ container('publish-docker') { } } } + diff --git a/ci/jenkinsfile/upload_dev_cluster_test_out.groovy b/ci/jenkinsfile/upload_dev_cluster_test_out.groovy new file mode 100644 index 0000000000..52fecc6c1d --- /dev/null +++ b/ci/jenkinsfile/upload_dev_cluster_test_out.groovy @@ -0,0 +1,26 @@ +container('milvus-testframework') { + timeout(time: 5, unit: 'MINUTES') { + dir ("${PROJECT_NAME}_test") { + gitlabCommitStatus(name: 'Upload Dev Test Out') { + if (fileExists('cluster_test_out')) { + try { + def fileTransfer = load "${env.WORKSPACE}/ci/function/file_transfer.groovy" + fileTransfer.FileTransfer("cluster_test_out/", "${PROJECT_NAME}/test/${JOB_NAME}-${BUILD_ID}", 'nas storage') + if (currentBuild.resultIsBetterOrEqualTo('SUCCESS')) { + echo "Milvus Dev Test Out Viewer \"ftp://192.168.1.126/data/${PROJECT_NAME}/test/${JOB_NAME}-${BUILD_ID}\"" + } + } catch (hudson.AbortException ae) { + updateGitlabCommitStatus name: 'Upload Dev Test Out', state: 'canceled' + currentBuild.result = 'ABORTED' + } catch (exc) { + updateGitlabCommitStatus name: 'Upload Dev Test Out', state: 'failed' + currentBuild.result = 'FAILURE' + } + } else { + updateGitlabCommitStatus name: 'Upload Dev Test Out', state: 'failed' + echo "Milvus Dev Test Out directory don't exists!" + } + } + } + } +} diff --git a/ci/main_jenkinsfile b/ci/main_jenkinsfile index 1c7d8be75c..1f0d0c3d7f 100644 --- a/ci/main_jenkinsfile +++ b/ci/main_jenkinsfile @@ -130,97 +130,200 @@ spec: } stage("Deploy to Development") { - stages { - stage("Deploy to Dev") { - agent { - kubernetes { - label 'jenkins-slave' - defaultContainer 'jnlp' - } - } + parallel { + stage("Single") { stages { - stage('Deploy') { - steps { - gitlabCommitStatus(name: 'Deloy to Dev') { + stage("Deploy to Dev") { + agent { + kubernetes { + label 'jenkins-slave' + defaultContainer 'jnlp' + } + } + stages { + stage('Deploy') { + steps { + gitlabCommitStatus(name: 'Deloy to Dev') { + script { + load "${env.WORKSPACE}/ci/jenkinsfile/deploy2dev.groovy" + } + } + } + } + } + post { + aborted { script { - load "${env.WORKSPACE}/ci/jenkinsfile/deploy2dev.groovy" + updateGitlabCommitStatus name: 'Deloy to Dev', state: 'canceled' + echo "Milvus Deloy to Dev aborted !" + } + } + + failure { + script { + updateGitlabCommitStatus name: 'Deloy to Dev', state: 'failed' + echo "Milvus Deloy to Dev failure !" } } } } - } - post { - aborted { - script { - updateGitlabCommitStatus name: 'Deloy to Dev', state: 'canceled' - echo "Milvus Deloy to Dev aborted !" + + stage("Dev Test") { + agent { + kubernetes { + label 'test' + defaultContainer 'jnlp' + containerTemplate { + name 'milvus-testframework' + image 'registry.zilliz.com/milvus/milvus-test:v0.1' + ttyEnabled true + command 'cat' + } + } + } + stages { + stage('Test') { + steps { + script { + load "${env.WORKSPACE}/ci/jenkinsfile/dev_test.groovy" + load "${env.WORKSPACE}/ci/jenkinsfile/upload_dev_test_out.groovy" + } + } + } } } - failure { - script { - updateGitlabCommitStatus name: 'Deloy to Dev', state: 'failed' - echo "Milvus Deloy to Dev failure !" + stage ("Cleanup Dev") { + agent { + kubernetes { + label 'jenkins-slave' + defaultContainer 'jnlp' + } } - } - } - } + stages { + stage('Cleanup') { + steps { + gitlabCommitStatus(name: 'Cleanup Dev') { + script { + load "${env.WORKSPACE}/ci/jenkinsfile/cleanup_dev.groovy" + } + } + } + } + } + post { + aborted { + script { + updateGitlabCommitStatus name: 'Cleanup Dev', state: 'canceled' + echo "Milvus Cleanup Dev aborted !" + } + } - stage("Dev Test") { - agent { - kubernetes { - label 'test' - defaultContainer 'jnlp' - containerTemplate { - name 'milvus-testframework' - image 'registry.zilliz.com/milvus/milvus-test:v0.1' - ttyEnabled true - command 'cat' - } - } - } - stages { - stage('Test') { - steps { - script { - load "${env.WORKSPACE}/ci/jenkinsfile/dev_test.groovy" - load "${env.WORKSPACE}/ci/jenkinsfile/upload_dev_test_out.groovy" + failure { + script { + updateGitlabCommitStatus name: 'Cleanup Dev', state: 'failed' + echo "Milvus Cleanup Dev failure !" + } } } } } } - stage ("Cleanup Dev") { - agent { - kubernetes { - label 'jenkins-slave' - defaultContainer 'jnlp' - } - } + stage("Cluster") { stages { - stage('Cleanup') { - steps { - gitlabCommitStatus(name: 'Cleanup Dev') { + stage("Deploy to Dev") { + agent { + kubernetes { + label 'jenkins-slave' + defaultContainer 'jnlp' + } + } + stages { + stage('Deploy') { + steps { + gitlabCommitStatus(name: 'Deloy to Dev') { + script { + load "${env.WORKSPACE}/ci/jenkinsfile/cluster_deploy2dev.groovy" + } + } + } + } + } + post { + aborted { script { - load "${env.WORKSPACE}/ci/jenkinsfile/cleanup_dev.groovy" + updateGitlabCommitStatus name: 'Deloy to Dev', state: 'canceled' + echo "Milvus Deloy to Dev aborted !" + } + } + + failure { + script { + updateGitlabCommitStatus name: 'Deloy to Dev', state: 'failed' + echo "Milvus Deloy to Dev failure !" } } } } - } - post { - aborted { - script { - updateGitlabCommitStatus name: 'Cleanup Dev', state: 'canceled' - echo "Milvus Cleanup Dev aborted !" + + stage("Dev Test") { + agent { + kubernetes { + label 'test' + defaultContainer 'jnlp' + containerTemplate { + name 'milvus-testframework' + image 'registry.zilliz.com/milvus/milvus-test:v0.1' + ttyEnabled true + command 'cat' + } + } + } + stages { + stage('Test') { + steps { + script { + load "${env.WORKSPACE}/ci/jenkinsfile/cluster_dev_test.groovy" + load "${env.WORKSPACE}/ci/jenkinsfile/upload_cluster_dev_test_out.groovy" + } + } + } } } - failure { - script { - updateGitlabCommitStatus name: 'Cleanup Dev', state: 'failed' - echo "Milvus Cleanup Dev failure !" + stage ("Cleanup Dev") { + agent { + kubernetes { + label 'jenkins-slave' + defaultContainer 'jnlp' + } + } + stages { + stage('Cleanup') { + steps { + gitlabCommitStatus(name: 'Cleanup Dev') { + script { + load "${env.WORKSPACE}/ci/jenkinsfile/cluster_cleanup_dev.groovy" + } + } + } + } + } + post { + aborted { + script { + updateGitlabCommitStatus name: 'Cleanup Dev', state: 'canceled' + echo "Milvus Cleanup Dev aborted !" + } + } + + failure { + script { + updateGitlabCommitStatus name: 'Cleanup Dev', state: 'failed' + echo "Milvus Cleanup Dev failure !" + } + } } } } @@ -270,3 +373,4 @@ spec: } } } + diff --git a/ci/main_jenkinsfile_no_ut b/ci/main_jenkinsfile_no_ut index 277ec155a5..203b185e14 100644 --- a/ci/main_jenkinsfile_no_ut +++ b/ci/main_jenkinsfile_no_ut @@ -130,97 +130,200 @@ spec: } stage("Deploy to Development") { - stages { - stage("Deploy to Dev") { - agent { - kubernetes { - label 'jenkins-slave' - defaultContainer 'jnlp' - } - } + parallel { + stage("Single") { stages { - stage('Deploy') { - steps { - gitlabCommitStatus(name: 'Deloy to Dev') { + stage("Deploy to Dev") { + agent { + kubernetes { + label 'jenkins-slave' + defaultContainer 'jnlp' + } + } + stages { + stage('Deploy') { + steps { + gitlabCommitStatus(name: 'Deloy to Dev') { + script { + load "${env.WORKSPACE}/ci/jenkinsfile/deploy2dev.groovy" + } + } + } + } + } + post { + aborted { script { - load "${env.WORKSPACE}/ci/jenkinsfile/deploy2dev.groovy" + updateGitlabCommitStatus name: 'Deloy to Dev', state: 'canceled' + echo "Milvus Deloy to Dev aborted !" + } + } + + failure { + script { + updateGitlabCommitStatus name: 'Deloy to Dev', state: 'failed' + echo "Milvus Deloy to Dev failure !" } } } } - } - post { - aborted { - script { - updateGitlabCommitStatus name: 'Deloy to Dev', state: 'canceled' - echo "Milvus Deloy to Dev aborted !" + + stage("Dev Test") { + agent { + kubernetes { + label 'test' + defaultContainer 'jnlp' + containerTemplate { + name 'milvus-testframework' + image 'registry.zilliz.com/milvus/milvus-test:v0.1' + ttyEnabled true + command 'cat' + } + } + } + stages { + stage('Test') { + steps { + script { + load "${env.WORKSPACE}/ci/jenkinsfile/dev_test.groovy" + load "${env.WORKSPACE}/ci/jenkinsfile/upload_dev_test_out.groovy" + } + } + } } } - failure { - script { - updateGitlabCommitStatus name: 'Deloy to Dev', state: 'failed' - echo "Milvus Deloy to Dev failure !" + stage ("Cleanup Dev") { + agent { + kubernetes { + label 'jenkins-slave' + defaultContainer 'jnlp' + } } - } - } - } + stages { + stage('Cleanup') { + steps { + gitlabCommitStatus(name: 'Cleanup Dev') { + script { + load "${env.WORKSPACE}/ci/jenkinsfile/cleanup_dev.groovy" + } + } + } + } + } + post { + aborted { + script { + updateGitlabCommitStatus name: 'Cleanup Dev', state: 'canceled' + echo "Milvus Cleanup Dev aborted !" + } + } - stage("Dev Test") { - agent { - kubernetes { - label 'test' - defaultContainer 'jnlp' - containerTemplate { - name 'milvus-testframework' - image 'registry.zilliz.com/milvus/milvus-test:v0.1' - ttyEnabled true - command 'cat' - } - } - } - stages { - stage('Test') { - steps { - script { - load "${env.WORKSPACE}/ci/jenkinsfile/dev_test.groovy" - load "${env.WORKSPACE}/ci/jenkinsfile/upload_dev_test_out.groovy" + failure { + script { + updateGitlabCommitStatus name: 'Cleanup Dev', state: 'failed' + echo "Milvus Cleanup Dev failure !" + } } } } } } - stage ("Cleanup Dev") { - agent { - kubernetes { - label 'jenkins-slave' - defaultContainer 'jnlp' - } - } + stage("Cluster") { stages { - stage('Cleanup') { - steps { - gitlabCommitStatus(name: 'Cleanup Dev') { + stage("Deploy to Dev") { + agent { + kubernetes { + label 'jenkins-slave' + defaultContainer 'jnlp' + } + } + stages { + stage('Deploy') { + steps { + gitlabCommitStatus(name: 'Deloy to Dev') { + script { + load "${env.WORKSPACE}/ci/jenkinsfile/cluster_deploy2dev.groovy" + } + } + } + } + } + post { + aborted { script { - load "${env.WORKSPACE}/ci/jenkinsfile/cleanup_dev.groovy" + updateGitlabCommitStatus name: 'Deloy to Dev', state: 'canceled' + echo "Milvus Deloy to Dev aborted !" + } + } + + failure { + script { + updateGitlabCommitStatus name: 'Deloy to Dev', state: 'failed' + echo "Milvus Deloy to Dev failure !" } } } } - } - post { - aborted { - script { - updateGitlabCommitStatus name: 'Cleanup Dev', state: 'canceled' - echo "Milvus Cleanup Dev aborted !" + + stage("Dev Test") { + agent { + kubernetes { + label 'test' + defaultContainer 'jnlp' + containerTemplate { + name 'milvus-testframework' + image 'registry.zilliz.com/milvus/milvus-test:v0.1' + ttyEnabled true + command 'cat' + } + } + } + stages { + stage('Test') { + steps { + script { + load "${env.WORKSPACE}/ci/jenkinsfile/cluster_dev_test.groovy" + load "${env.WORKSPACE}/ci/jenkinsfile/upload_cluster_dev_test_out.groovy" + } + } + } } } - failure { - script { - updateGitlabCommitStatus name: 'Cleanup Dev', state: 'failed' - echo "Milvus Cleanup Dev failure !" + stage ("Cleanup Dev") { + agent { + kubernetes { + label 'jenkins-slave' + defaultContainer 'jnlp' + } + } + stages { + stage('Cleanup') { + steps { + gitlabCommitStatus(name: 'Cleanup Dev') { + script { + load "${env.WORKSPACE}/ci/jenkinsfile/cluster_cleanup_dev.groovy" + } + } + } + } + } + post { + aborted { + script { + updateGitlabCommitStatus name: 'Cleanup Dev', state: 'canceled' + echo "Milvus Cleanup Dev aborted !" + } + } + + failure { + script { + updateGitlabCommitStatus name: 'Cleanup Dev', state: 'failed' + echo "Milvus Cleanup Dev failure !" + } + } } } } @@ -232,6 +335,22 @@ spec: } post { + always { + script { + if (!currentBuild.resultIsBetterOrEqualTo('SUCCESS')) { + // Send an email only if the build status has changed from green/unstable to red + emailext subject: '$DEFAULT_SUBJECT', + body: '$DEFAULT_CONTENT', + recipientProviders: [ + [$class: 'DevelopersRecipientProvider'], + [$class: 'RequesterRecipientProvider'] + ], + replyTo: '$DEFAULT_REPLYTO', + to: '$DEFAULT_RECIPIENTS' + } + } + } + success { script { updateGitlabCommitStatus name: 'CI/CD', state: 'success' @@ -254,3 +373,4 @@ spec: } } } + diff --git a/ci/nightly_main_jenkinsfile b/ci/nightly_main_jenkinsfile index eff34a5de2..c375cc87ea 100644 --- a/ci/nightly_main_jenkinsfile +++ b/ci/nightly_main_jenkinsfile @@ -130,97 +130,200 @@ spec: } stage("Deploy to Development") { - stages { - stage("Deploy to Dev") { - agent { - kubernetes { - label 'jenkins-slave' - defaultContainer 'jnlp' - } - } + parallel { + stage("Single") { stages { - stage('Deploy') { - steps { - gitlabCommitStatus(name: 'Deloy to Dev') { + stage("Deploy to Dev") { + agent { + kubernetes { + label 'jenkins-slave' + defaultContainer 'jnlp' + } + } + stages { + stage('Deploy') { + steps { + gitlabCommitStatus(name: 'Deloy to Dev') { + script { + load "${env.WORKSPACE}/ci/jenkinsfile/deploy2dev.groovy" + } + } + } + } + } + post { + aborted { script { - load "${env.WORKSPACE}/ci/jenkinsfile/deploy2dev.groovy" + updateGitlabCommitStatus name: 'Deloy to Dev', state: 'canceled' + echo "Milvus Deloy to Dev aborted !" + } + } + + failure { + script { + updateGitlabCommitStatus name: 'Deloy to Dev', state: 'failed' + echo "Milvus Deloy to Dev failure !" } } } } - } - post { - aborted { - script { - updateGitlabCommitStatus name: 'Deloy to Dev', state: 'canceled' - echo "Milvus Deloy to Dev aborted !" + + stage("Dev Test") { + agent { + kubernetes { + label 'test' + defaultContainer 'jnlp' + containerTemplate { + name 'milvus-testframework' + image 'registry.zilliz.com/milvus/milvus-test:v0.1' + ttyEnabled true + command 'cat' + } + } + } + stages { + stage('Test') { + steps { + script { + load "${env.WORKSPACE}/ci/jenkinsfile/dev_test.groovy" + load "${env.WORKSPACE}/ci/jenkinsfile/upload_dev_test_out.groovy" + } + } + } } } - failure { - script { - updateGitlabCommitStatus name: 'Deloy to Dev', state: 'failed' - echo "Milvus Deloy to Dev failure !" + stage ("Cleanup Dev") { + agent { + kubernetes { + label 'jenkins-slave' + defaultContainer 'jnlp' + } } - } - } - } + stages { + stage('Cleanup') { + steps { + gitlabCommitStatus(name: 'Cleanup Dev') { + script { + load "${env.WORKSPACE}/ci/jenkinsfile/cleanup_dev.groovy" + } + } + } + } + } + post { + aborted { + script { + updateGitlabCommitStatus name: 'Cleanup Dev', state: 'canceled' + echo "Milvus Cleanup Dev aborted !" + } + } - stage("Dev Test") { - agent { - kubernetes { - label 'test' - defaultContainer 'jnlp' - containerTemplate { - name 'milvus-testframework' - image 'registry.zilliz.com/milvus/milvus-test:v0.1' - ttyEnabled true - command 'cat' - } - } - } - stages { - stage('Test') { - steps { - script { - load "${env.WORKSPACE}/ci/jenkinsfile/dev_test.groovy" - load "${env.WORKSPACE}/ci/jenkinsfile/upload_dev_test_out.groovy" + failure { + script { + updateGitlabCommitStatus name: 'Cleanup Dev', state: 'failed' + echo "Milvus Cleanup Dev failure !" + } } } } } } - stage ("Cleanup Dev") { - agent { - kubernetes { - label 'jenkins-slave' - defaultContainer 'jnlp' - } - } + stage("Cluster") { stages { - stage('Cleanup') { - steps { - gitlabCommitStatus(name: 'Cleanup Dev') { + stage("Deploy to Dev") { + agent { + kubernetes { + label 'jenkins-slave' + defaultContainer 'jnlp' + } + } + stages { + stage('Deploy') { + steps { + gitlabCommitStatus(name: 'Deloy to Dev') { + script { + load "${env.WORKSPACE}/ci/jenkinsfile/cluster_deploy2dev.groovy" + } + } + } + } + } + post { + aborted { script { - load "${env.WORKSPACE}/ci/jenkinsfile/cleanup_dev.groovy" + updateGitlabCommitStatus name: 'Deloy to Dev', state: 'canceled' + echo "Milvus Deloy to Dev aborted !" + } + } + + failure { + script { + updateGitlabCommitStatus name: 'Deloy to Dev', state: 'failed' + echo "Milvus Deloy to Dev failure !" } } } } - } - post { - aborted { - script { - updateGitlabCommitStatus name: 'Cleanup Dev', state: 'canceled' - echo "Milvus Cleanup Dev aborted !" + + stage("Dev Test") { + agent { + kubernetes { + label 'test' + defaultContainer 'jnlp' + containerTemplate { + name 'milvus-testframework' + image 'registry.zilliz.com/milvus/milvus-test:v0.1' + ttyEnabled true + command 'cat' + } + } + } + stages { + stage('Test') { + steps { + script { + load "${env.WORKSPACE}/ci/jenkinsfile/cluster_dev_test.groovy" + load "${env.WORKSPACE}/ci/jenkinsfile/upload_cluster_dev_test_out.groovy" + } + } + } } } - failure { - script { - updateGitlabCommitStatus name: 'Cleanup Dev', state: 'failed' - echo "Milvus Cleanup Dev failure !" + stage ("Cleanup Dev") { + agent { + kubernetes { + label 'jenkins-slave' + defaultContainer 'jnlp' + } + } + stages { + stage('Cleanup') { + steps { + gitlabCommitStatus(name: 'Cleanup Dev') { + script { + load "${env.WORKSPACE}/ci/jenkinsfile/cluster_cleanup_dev.groovy" + } + } + } + } + } + post { + aborted { + script { + updateGitlabCommitStatus name: 'Cleanup Dev', state: 'canceled' + echo "Milvus Cleanup Dev aborted !" + } + } + + failure { + script { + updateGitlabCommitStatus name: 'Cleanup Dev', state: 'failed' + echo "Milvus Cleanup Dev failure !" + } + } } } } @@ -232,6 +335,22 @@ spec: } post { + always { + script { + if (!currentBuild.resultIsBetterOrEqualTo('SUCCESS')) { + // Send an email only if the build status has changed from green/unstable to red + emailext subject: '$DEFAULT_SUBJECT', + body: '$DEFAULT_CONTENT', + recipientProviders: [ + [$class: 'DevelopersRecipientProvider'], + [$class: 'RequesterRecipientProvider'] + ], + replyTo: '$DEFAULT_REPLYTO', + to: '$DEFAULT_RECIPIENTS' + } + } + } + success { script { updateGitlabCommitStatus name: 'CI/CD', state: 'success' @@ -254,3 +373,4 @@ spec: } } } + From 8c844d587d4c230b3de8166d715ad85c722f8cde Mon Sep 17 00:00:00 2001 From: quicksilver Date: Fri, 12 Jul 2019 23:42:40 +0800 Subject: [PATCH 2/3] helm clean up Former-commit-id: 0cf0b16f1a736d001b96399bfd16d97db0e5fa40 --- ci/jenkinsfile/cleanup_dev.groovy | 1 + ci/jenkinsfile/cluster_cleanup_dev.groovy | 1 + ci/jenkinsfile/cluster_dev_test.groovy | 1 + ci/jenkinsfile/dev_test.groovy | 1 + 4 files changed, 4 insertions(+) diff --git a/ci/jenkinsfile/cleanup_dev.groovy b/ci/jenkinsfile/cleanup_dev.groovy index 40f354275f..056b8fa8c5 100644 --- a/ci/jenkinsfile/cleanup_dev.groovy +++ b/ci/jenkinsfile/cleanup_dev.groovy @@ -7,6 +7,7 @@ try { error("Dev Test Failure !") } } catch (exc) { + sh "helm del --purge ${env.JOB_NAME}-${env.BUILD_NUMBER}" updateGitlabCommitStatus name: 'Cleanup Dev', state: 'failed' throw exc } diff --git a/ci/jenkinsfile/cluster_cleanup_dev.groovy b/ci/jenkinsfile/cluster_cleanup_dev.groovy index 57328c4524..dc18067cdd 100644 --- a/ci/jenkinsfile/cluster_cleanup_dev.groovy +++ b/ci/jenkinsfile/cluster_cleanup_dev.groovy @@ -7,6 +7,7 @@ try { error("Dev Test Failure !") } } catch (exc) { + sh "helm del --purge ${env.JOB_NAME}-${env.BUILD_NUMBER}-cluster" updateGitlabCommitStatus name: 'Cleanup Dev', state: 'failed' throw exc } diff --git a/ci/jenkinsfile/cluster_dev_test.groovy b/ci/jenkinsfile/cluster_dev_test.groovy index 50fd67f803..3bb10cf288 100644 --- a/ci/jenkinsfile/cluster_dev_test.groovy +++ b/ci/jenkinsfile/cluster_dev_test.groovy @@ -8,6 +8,7 @@ container('milvus-testframework') { sh "pytest . --alluredir=cluster_test_out --ip ${env.JOB_NAME}-${env.BUILD_NUMBER}-cluster-milvus-cluster-proxy.milvus-cluster.svc.cluster.local" } } catch (exc) { + sh "helm del --purge ${env.JOB_NAME}-${env.BUILD_NUMBER}-cluster" updateGitlabCommitStatus name: 'Dev Test', state: 'failed' currentBuild.result = 'FAILURE' echo 'Milvus Test Failed !' diff --git a/ci/jenkinsfile/dev_test.groovy b/ci/jenkinsfile/dev_test.groovy index 13e1c868b8..5594a2f807 100644 --- a/ci/jenkinsfile/dev_test.groovy +++ b/ci/jenkinsfile/dev_test.groovy @@ -8,6 +8,7 @@ container('milvus-testframework') { sh "pytest . --alluredir=test_out --ip ${env.JOB_NAME}-${env.BUILD_NUMBER}-milvus-gpu-engine.kube-opt.svc.cluster.local" } } catch (exc) { + sh "helm del --purge ${env.JOB_NAME}-${env.BUILD_NUMBER}" updateGitlabCommitStatus name: 'Dev Test', state: 'failed' currentBuild.result = 'FAILURE' echo 'Milvus Test Failed !' From b0380864767d381fff6a2c1754e85896cb1db969 Mon Sep 17 00:00:00 2001 From: quicksilver Date: Fri, 12 Jul 2019 23:59:19 +0800 Subject: [PATCH 3/3] disable helm clean up Former-commit-id: fc4a6cc8e7f66110e160b17aacedae03786efc15 --- ci/jenkinsfile/cluster_dev_test.groovy | 1 - ci/jenkinsfile/dev_test.groovy | 1 - 2 files changed, 2 deletions(-) diff --git a/ci/jenkinsfile/cluster_dev_test.groovy b/ci/jenkinsfile/cluster_dev_test.groovy index 3bb10cf288..50fd67f803 100644 --- a/ci/jenkinsfile/cluster_dev_test.groovy +++ b/ci/jenkinsfile/cluster_dev_test.groovy @@ -8,7 +8,6 @@ container('milvus-testframework') { sh "pytest . --alluredir=cluster_test_out --ip ${env.JOB_NAME}-${env.BUILD_NUMBER}-cluster-milvus-cluster-proxy.milvus-cluster.svc.cluster.local" } } catch (exc) { - sh "helm del --purge ${env.JOB_NAME}-${env.BUILD_NUMBER}-cluster" updateGitlabCommitStatus name: 'Dev Test', state: 'failed' currentBuild.result = 'FAILURE' echo 'Milvus Test Failed !' diff --git a/ci/jenkinsfile/dev_test.groovy b/ci/jenkinsfile/dev_test.groovy index 5594a2f807..13e1c868b8 100644 --- a/ci/jenkinsfile/dev_test.groovy +++ b/ci/jenkinsfile/dev_test.groovy @@ -8,7 +8,6 @@ container('milvus-testframework') { sh "pytest . --alluredir=test_out --ip ${env.JOB_NAME}-${env.BUILD_NUMBER}-milvus-gpu-engine.kube-opt.svc.cluster.local" } } catch (exc) { - sh "helm del --purge ${env.JOB_NAME}-${env.BUILD_NUMBER}" updateGitlabCommitStatus name: 'Dev Test', state: 'failed' currentBuild.result = 'FAILURE' echo 'Milvus Test Failed !'