milvus-io
/
milvus
mirror of https://github.com/milvus-io/milvus.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'branch-0.4.0' into 'branch-0.4.0' 

add staging test stage

See merge request megasearch/milvus!426

Former-commit-id: 19be6fd04c2b62fb44f3a848faef0d0bed38da74
pull/191/head
Jeff 2019-08-24 17:35:56 +08:00
parent 842bc5932e 299b0c6b66
commit b4aa489ddc
7 changed files with 181 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
ci/jenkinsfile/cleanup_staging.groovy Normal file
View File

@ -0,0 +1,13 @@
try {
def result = sh script: "helm status ${env.JOB_NAME}-${env.BUILD_NUMBER}", returnStatus: true
if (!result) {
sh "helm del --purge ${env.JOB_NAME}-${env.BUILD_NUMBER}"
}
} catch (exc) {
def result = sh script: "helm status ${env.JOB_NAME}-${env.BUILD_NUMBER}", returnStatus: true
if (!result) {
sh "helm del --purge ${env.JOB_NAME}-${env.BUILD_NUMBER}"
}
throw exc
}

16
ci/jenkinsfile/deploy2staging.groovy Normal file
View File

@ -0,0 +1,16 @@
try {
sh 'helm init --client-only --skip-refresh --stable-repo-url https://kubernetes.oss-cn-hangzhou.aliyuncs.com/charts'
sh 'helm repo add milvus https://registry.zilliz.com/chartrepo/milvus'
sh 'helm repo update'
dir ("milvus-helm") {
checkout([$class: 'GitSCM', branches: [[name: "${SEMVER}"]], doGenerateSubmoduleConfigurations: false, extensions: [], submoduleCfg: [], userRemoteConfigs: [[credentialsId: "${params.GIT_USER}", url: "git@192.168.1.105:megasearch/milvus-helm.git", name: 'origin', refspec: "+refs/heads/${SEMVER}:refs/remotes/origin/${SEMVER}"]]])
dir ("milvus/milvus-gpu") {
sh "helm install --wait --timeout 300 --set engine.image.repository="zilliz.azurecr.cn/milvus/engine" --set engine.image.tag=${DOCKER_VERSION} --set expose.type=loadBalancer --name ${env.JOB_NAME}-${env.BUILD_NUMBER} -f ci/values.yaml --namespace milvus-1 --version 0.4.0 ."
}
}
} catch (exc) {
echo 'Helm running failed!'
sh "helm del --purge ${env.JOB_NAME}-${env.BUILD_NUMBER}"
throw exc
}

6
ci/jenkinsfile/dev_test.groovy
View File

@ -3,7 +3,7 @@ timeout(time: 40, unit: 'MINUTES') {
dir ("${PROJECT_NAME}_test") { dir ("${PROJECT_NAME}_test") {
checkout([$class: 'GitSCM', branches: [[name: "${SEMVER}"]], doGenerateSubmoduleConfigurations: false, extensions: [], submoduleCfg: [], userRemoteConfigs: [[credentialsId: "${params.GIT_USER}", url: "git@192.168.1.105:Test/milvus_test.git", name: 'origin', refspec: "+refs/heads/${SEMVER}:refs/remotes/origin/${SEMVER}"]]]) checkout([$class: 'GitSCM', branches: [[name: "${SEMVER}"]], doGenerateSubmoduleConfigurations: false, extensions: [], submoduleCfg: [], userRemoteConfigs: [[credentialsId: "${params.GIT_USER}", url: "git@192.168.1.105:Test/milvus_test.git", name: 'origin', refspec: "+refs/heads/${SEMVER}:refs/remotes/origin/${SEMVER}"]]])
sh 'python3 -m pip install -r requirements.txt' sh 'python3 -m pip install -r requirements.txt'
sh "pytest . --alluredir=test_out --ip ${env.JOB_NAME}-${env.BUILD_NUMBER}-milvus-gpu-engine.milvus-1.svc.cluster.local" sh "pytest . --alluredir="test_out/dev/single/sqlite" --ip ${env.JOB_NAME}-${env.BUILD_NUMBER}-milvus-gpu-engine.milvus-1.svc.cluster.local"
} }
// mysql database backend test // mysql database backend test
@ -16,11 +16,11 @@ timeout(time: 40, unit: 'MINUTES') {
} }
dir ("milvus-helm") { dir ("milvus-helm") {
dir ("milvus/milvus-gpu") { dir ("milvus/milvus-gpu") {
sh "helm install --wait --timeout 300 --set engine.image.tag=${DOCKER_VERSION} --set expose.type=clusterIP --name ${env.JOB_NAME}-${env.BUILD_NUMBER} -f ci/db_backend/mysql_values.yaml --namespace milvus-2 --version 0.3.1 ." sh "helm install --wait --timeout 300 --set engine.image.tag=${DOCKER_VERSION} --set expose.type=clusterIP --name ${env.JOB_NAME}-${env.BUILD_NUMBER} -f ci/db_backend/mysql_values.yaml --namespace milvus-2 --version 0.4.0 ."
} }
} }
dir ("${PROJECT_NAME}_test") { dir ("${PROJECT_NAME}_test") {
sh "pytest . --alluredir=test_out --ip ${env.JOB_NAME}-${env.BUILD_NUMBER}-milvus-gpu-engine.milvus-2.svc.cluster.local" sh "pytest . --alluredir="test_out/dev/single/mysql" --ip ${env.JOB_NAME}-${env.BUILD_NUMBER}-milvus-gpu-engine.milvus-2.svc.cluster.local"
} }
} catch (exc) { } catch (exc) {
echo 'Milvus Test Failed !' echo 'Milvus Test Failed !'

31
ci/jenkinsfile/staging_test.groovy Normal file
View File

@ -0,0 +1,31 @@
timeout(time: 40, unit: 'MINUTES') {
try {
dir ("${PROJECT_NAME}_test") {
checkout([$class: 'GitSCM', branches: [[name: "${SEMVER}"]], doGenerateSubmoduleConfigurations: false, extensions: [], submoduleCfg: [], userRemoteConfigs: [[credentialsId: "${params.GIT_USER}", url: "git@192.168.1.105:Test/milvus_test.git", name: 'origin', refspec: "+refs/heads/${SEMVER}:refs/remotes/origin/${SEMVER}"]]])
sh 'python3 -m pip install -r requirements.txt'
def service_ip = sh (script: "kubectl get svc --namespace milvus-1 ${env.JOB_NAME}-${env.BUILD_NUMBER}-milvus-gpu-engine --template \"{{range .status.loadBalancer.ingress}}{{.ip}}{{end}}\"",returnStdout: true).trim()
sh "pytest . --alluredir="test_out/staging/single/sqlite" --ip ${service_ip}"
}
// mysql database backend test
load "${env.WORKSPACE}/ci/jenkinsfile/cleanup_staging.groovy"
if (!fileExists('milvus-helm')) {
dir ("milvus-helm") {
checkout([$class: 'GitSCM', branches: [[name: "${SEMVER}"]], doGenerateSubmoduleConfigurations: false, extensions: [], submoduleCfg: [], userRemoteConfigs: [[credentialsId: "${params.GIT_USER}", url: "git@192.168.1.105:megasearch/milvus-helm.git", name: 'origin', refspec: "+refs/heads/${SEMVER}:refs/remotes/origin/${SEMVER}"]]])
}
}
dir ("milvus-helm") {
dir ("milvus/milvus-gpu") {
sh "helm install --wait --timeout 300 --set engine.image.repository="zilliz.azurecr.cn/milvus/engine" --set engine.image.tag=${DOCKER_VERSION} --set expose.type=loadBalancer --name ${env.JOB_NAME}-${env.BUILD_NUMBER} -f ci/db_backend/mysql_values.yaml --namespace milvus-2 --version 0.4.0 ."
}
}
dir ("${PROJECT_NAME}_test") {
def service_ip = sh (script: "kubectl get svc --namespace milvus-2 ${env.JOB_NAME}-${env.BUILD_NUMBER}-milvus-gpu-engine --template \"{{range .status.loadBalancer.ingress}}{{.ip}}{{end}}\"",returnStdout: true).trim()
sh "pytest . --alluredir="test_out/staging/single/mysql" --ip ${service_ip}"
}
} catch (exc) {
echo 'Milvus Test Failed !'
throw exc
}
}

4
ci/jenkinsfile/upload_dev_test_out.groovy
View File

@ -1,8 +1,8 @@
timeout(time: 5, unit: 'MINUTES') { timeout(time: 5, unit: 'MINUTES') {
dir ("${PROJECT_NAME}_test") { dir ("${PROJECT_NAME}_test") {
if (fileExists('test_out')) { if (fileExists('test_out/dev')) {
def fileTransfer = load "${env.WORKSPACE}/ci/function/file_transfer.groovy" def fileTransfer = load "${env.WORKSPACE}/ci/function/file_transfer.groovy"
fileTransfer.FileTransfer("test_out/", "${PROJECT_NAME}/test/${JOB_NAME}-${BUILD_ID}", 'nas storage') fileTransfer.FileTransfer("test_out/dev/", "${PROJECT_NAME}/test/${JOB_NAME}-${BUILD_ID}", 'nas storage')
if (currentBuild.resultIsBetterOrEqualTo('SUCCESS')) { if (currentBuild.resultIsBetterOrEqualTo('SUCCESS')) {
echo "Milvus Dev Test Out Viewer \"ftp://192.168.1.126/data/${PROJECT_NAME}/test/${JOB_NAME}-${BUILD_ID}\"" echo "Milvus Dev Test Out Viewer \"ftp://192.168.1.126/data/${PROJECT_NAME}/test/${JOB_NAME}-${BUILD_ID}\""
} }

13
ci/jenkinsfile/upload_staging_test_out.groovy Normal file
View File

@ -0,0 +1,13 @@
timeout(time: 5, unit: 'MINUTES') {
dir ("${PROJECT_NAME}_test") {
if (fileExists('test_out/staging')) {
def fileTransfer = load "${env.WORKSPACE}/ci/function/file_transfer.groovy"
fileTransfer.FileTransfer("test_out/staging/", "${PROJECT_NAME}/test/${JOB_NAME}-${BUILD_ID}", 'nas storage')
if (currentBuild.resultIsBetterOrEqualTo('SUCCESS')) {
echo "Milvus Dev Test Out Viewer \"ftp://192.168.1.126/data/${PROJECT_NAME}/test/${JOB_NAME}-${BUILD_ID}\""
}
} else {
error("Milvus Dev Test Out directory don't exists!")
}
}
}

109
ci/nightly_main_jenkinsfile
View File

@ -208,17 +208,17 @@ spec:
} }
success { success {
script { script {
echo "Milvus Single Node CI/CD success !" echo "Milvus Deploy to Dev Single Node CI/CD success !"
} }
} }
aborted { aborted {
script { script {
echo "Milvus Single Node CI/CD aborted !" echo "Milvus Deploy to Dev Single Node CI/CD aborted !"
} }
} }
failure { failure {
script { script {
echo "Milvus Single Node CI/CD failure !" echo "Milvus Deploy to Dev Single Node CI/CD failure !"
} }
} }
} }
@ -300,17 +300,114 @@ spec:
} }
success { success {
script { script {
echo "Milvus Cluster CI/CD success !" echo "Milvus Deploy to Dev Cluster CI/CD success !"
} }
} }
aborted { aborted {
script { script {
echo "Milvus Cluster CI/CD aborted !" echo "Milvus Deploy to Dev Cluster CI/CD aborted !"
} }
} }
failure { failure {
script { script {
echo "Milvus Cluster CI/CD failure !" echo "Milvus Deploy to Dev Cluster CI/CD failure !"
}
}
}
}
}
}
stage("Deploy to Staging") {
parallel {
stage("Single Node") {
agent {
kubernetes {
label 'dev-test'
defaultContainer 'jnlp'
yaml """
apiVersion: v1
kind: Pod
metadata:
labels:
app: milvus
componet: test
spec:
containers:
- name: milvus-testframework
image: registry.zilliz.com/milvus/milvus-test:v0.2
command:
- cat
tty: true
volumeMounts:
- name: kubeconf
mountPath: /root/.kube/
readOnly: true
volumes:
- name: kubeconf
secret:
secretName: aks-gpu-cluster-config
"""
}
}
stages {
stage("Deploy to Staging") {
steps {
gitlabCommitStatus(name: 'Deloy to Staging') {
container('milvus-testframework') {
script {
load "${env.WORKSPACE}/ci/jenkinsfile/deploy2staging.groovy"
}
}
}
}
}
stage("Staging Test") {
steps {
gitlabCommitStatus(name: 'Staging Test') {
container('milvus-testframework') {
script {
load "${env.WORKSPACE}/ci/jenkinsfile/staging_test.groovy"
load "${env.WORKSPACE}/ci/jenkinsfile/upload_staging_test_out.groovy"
}
}
}
}
}
stage ("Cleanup Staging") {
steps {
gitlabCommitStatus(name: 'Cleanup Staging') {
container('milvus-testframework') {
script {
load "${env.WORKSPACE}/ci/jenkinsfile/cleanup_staging.groovy"
}
}
}
}
}
}
post {
always {
container('milvus-testframework') {
script {
load "${env.WORKSPACE}/ci/jenkinsfile/cleanup_staging.groovy"
}
}
}
success {
script {
echo "Milvus Deploy to Staging Single Node CI/CD success !"
}
}
aborted {
script {
echo "Milvus Deploy to Staging Single Node CI/CD aborted !"
}
}
failure {
script {
echo "Milvus Deploy to Staging Single Node CI/CD failure !"
} }
} }
} }