Merge pull request #22 from milvus-ci-robot/0.5.0

Add Jenkins CI for Milvus Former-commit-id: cfb72c09d41e47538937033004e28385404b3e4b
2019-10-17 09:37:10 +08:00 · 2019-10-17 09:37:10 +08:00 · 1dbc3d9301
parent 710be6d161 2909cfcfd9
commit 1dbc3d9301
14 changed files with 397 additions and 0 deletions
--- a/ci/jenkins/Jenkinsfile
+++ b/ci/jenkins/Jenkinsfile
@ -0,0 +1,142 @@
+pipeline {
+    agent none
+
+    options {
+        timestamps()
+    }
+
+    parameters{
+        choice choices: ['Release', 'Debug'], description: '', name: 'BUILD_TYPE'
+        string defaultValue: 'cf1434e7-5a4b-4d25-82e8-88d667aef9e5', description: 'GIT CREDENTIALS ID', name: 'GIT_CREDENTIALS_ID', trim: true
+        string defaultValue: '1a527823-d2b7-44fd-834b-9844350baf14', description: 'JFROG CREDENTIALS ID', name: 'JFROG_CREDENTIALS_ID', trim: true
+        string defaultValue: 'ba070c98-c8cc-4f7c-b657-897715f359fc', description: 'DOCKER CREDENTIALS ID', name: 'DOCKER_CREDENTIALS_ID', trim: true
+        string defaultValue: 'http://192.168.1.202/artifactory/milvus', description: 'JFROG ARTFACTORY URL', name: 'JFROG_ARTFACTORY_URL', trim: true
+    }
+
+    environment {
+        PROJECT_NAME = "milvus"
+        LOWER_BUILD_TYPE = params.BUILD_TYPE.toLowerCase()
+        SEMVER = "${BRANCH_NAME}"
+        JOBNAMES = env.JOB_NAME.split('/')
+        PIPELIEN_NAME = "${JOBNAMES[0]}"
+    }
+
+    stages {
+        stage("Ubuntu 16.04") {
+            environment {
+                OS_NAME = "ubuntu16.04"
+                PACKAGE_VERSION = VersionNumber([
+                    versionNumberString : '${SEMVER}-${LOWER_BUILD_TYPE}-${OS_NAME}-x86_64-${BUILD_DATE_FORMATTED, "yyyyMMdd"}-${BUILDS_TODAY}'
+                ]);
+                DOCKER_VERSION = "${SEMVER}-${OS_NAME}-${LOWER_BUILD_TYPE}"
+            }
+
+            stages {
+                stage("Run Build") {
+                    agent {
+                        kubernetes {
+                            label 'build'
+                            defaultContainer 'jnlp'
+                            yamlFile 'ci/jenkins/pod/milvus-build-env-pod.yaml'
+                        }
+                    }
+
+                    stages {
+                        stage('Build') {
+                            steps {
+                                container('milvus-build-env') {
+                                    script {
+                                        load "${env.WORKSPACE}/ci/jenkins/jenkinsfile/build.groovy"
+                                    }
+                                }
+                            }
+                        }
+                        stage('Upload Package') {
+                            steps {
+                                container('milvus-build-env') {
+                                    script {
+                                        load "${env.WORKSPACE}/ci/jenkins/jenkinsfile/package.groovy"
+                                    }
+                                }
+                            }
+                        }
+                    }
+                }
+
+                stage("Publish docker images") {
+                    agent {
+                        kubernetes {
+                            label 'publish'
+                            defaultContainer 'jnlp'
+                            yamlFile 'ci/jenkins/pod/docker-pod.yaml'
+                        }
+                    }
+
+                    stages {
+                        stage('Publish') {
+                            steps {
+                                container('publish-images'){
+                                    script {
+                                        load "${env.WORKSPACE}/ci/jenkins/jenkinsfile/publishImages.groovy"
+                                    }
+                                }
+                            }
+                        }
+                    }
+                }
+
+                stage("Deploy to Development") {
+                    agent {
+                        kubernetes {
+                            label 'dev-test'
+                            defaultContainer 'jnlp'
+                            yamlFile 'ci/jenkins/pod/testEnvironment.yaml'
+                        }
+                    }
+
+                    stages {
+                        stage("Deploy to Dev") {
+                            steps {
+                                container('milvus-test-env') {
+                                    script {
+                                        load "${env.WORKSPACE}/ci/jenkins/jenkinsfile/deploySingle2Dev.groovy"
+                                    }
+                                }
+                            }
+                        }
+
+                        stage("Dev Test") {
+                            steps {
+                                container('milvus-test-env') {
+                                    script {
+                                        load "${env.WORKSPACE}/ci/jenkins/jenkinsfile/singleDevTest.groovy"
+                                    }
+                                }
+                            }
+                        }
+
+                        stage ("Cleanup Dev") {
+                            steps {
+                                container('milvus-test-env') {
+                                    script {
+                                        load "${env.WORKSPACE}/ci/jenkins/jenkinsfile/cleanupSingleDev.groovy"
+                                    }
+                                }
+                            }
+                        }
+                    }
+                    post {
+                        unsuccessful {
+                            container('milvus-test-env') {
+                                script {
+                                    load "${env.WORKSPACE}/ci/jenkins/jenkinsfile/cleanupSingleDev.groovy"
+                                }
+                            }
+                        }
+                    }
+                }
+            }
+        }
+    }
+}
+
--- a/ci/jenkins/jenkinsfile/build.groovy
+++ b/ci/jenkins/jenkinsfile/build.groovy
@ -0,0 +1,11 @@
+timeout(time: 60, unit: 'MINUTES') {
+    dir ("core") {
+        sh "git config --global user.email \"test@zilliz.com\""
+        sh "git config --global user.name \"test\""
+        sh "./build.sh -l"
+        withCredentials([usernamePassword(credentialsId: "${params.JFROG_CREDENTIALS_ID}", usernameVariable: 'USERNAME', passwordVariable: 'PASSWORD')]) {
+            sh "export JFROG_ARTFACTORY_URL='${params.JFROG_ARTFACTORY_URL}' && export JFROG_USER_NAME='${USERNAME}' && export JFROG_PASSWORD='${PASSWORD}' && ./build.sh -t ${params.BUILD_TYPE} -d /opt/milvus -j -u"
+        }
+    }
+}
+
--- a/ci/jenkins/jenkinsfile/cleanupSingleDev.groovy
+++ b/ci/jenkins/jenkinsfile/cleanupSingleDev.groovy
@ -0,0 +1,10 @@
+try {
+    def jobNames = env.JOB_NAME.split('/')
+    sh "helm del --purge ${env.PIPELIEN_NAME}-${env.BUILD_NUMBER}-single-gpu"
+} catch (exc) {
+    def helmResult = sh script: "helm status ${env.PIPELIEN_NAME}-${env.BUILD_NUMBER}-single-gpu", returnStatus: true
+    if (!helmResult) {
+        sh "helm del --purge ${env.PIPELIEN_NAME}-${env.BUILD_NUMBER}-single-gpu"
+    }
+    throw exc
+}
--- a/ci/jenkins/jenkinsfile/deploySingle2Dev.groovy
+++ b/ci/jenkins/jenkinsfile/deploySingle2Dev.groovy
@ -0,0 +1,14 @@
+try {
+    sh 'helm init --client-only --skip-refresh --stable-repo-url https://kubernetes.oss-cn-hangzhou.aliyuncs.com/charts'
+    sh 'helm repo update'
+    dir ('milvus-helm') {
+        checkout([$class: 'GitSCM', branches: [[name: "0.5.0"]], doGenerateSubmoduleConfigurations: false, extensions: [], submoduleCfg: [], userRemoteConfigs: [[credentialsId: "${params.GIT_CREDENTIALS_ID}", url: "https://github.com/milvus-io/milvus-helm.git", name: 'origin', refspec: "+refs/heads/0.5.0:refs/remotes/origin/0.5.0"]]])
+        dir ("milvus-gpu") {
+            sh "helm install --wait --timeout 300 --set engine.image.tag=${DOCKER_VERSION} --set expose.type=clusterIP --name ${env.PIPELIEN_NAME}-${env.BUILD_NUMBER}-single-gpu -f ci/values.yaml --namespace milvus ."
+        }
+    }
+} catch (exc) {
+    echo 'Helm running failed!'
+    sh "helm del --purge ${env.PIPELIEN_NAME}-${env.BUILD_NUMBER}-single-gpu"
+    throw exc
+}
--- a/ci/jenkins/jenkinsfile/package.groovy
+++ b/ci/jenkins/jenkinsfile/package.groovy
@ -0,0 +1,15 @@
+timeout(time: 5, unit: 'MINUTES') {
+    dir ("core") {
+        if (fileExists('milvus')) {
+            sh "tar -zcvf ./${PROJECT_NAME}-${PACKAGE_VERSION}.tar.gz ./milvus"
+            withCredentials([usernamePassword(credentialsId: "${params.JFROG_CREDENTIALS_ID}", usernameVariable: 'JFROG_USERNAME', passwordVariable: 'JFROG_PASSWORD')]) {
+                def uploadStatus = sh(returnStatus: true, script: "curl -u${JFROG_USERNAME}:${JFROG_PASSWORD} -T ./${PROJECT_NAME}-${PACKAGE_VERSION}.tar.gz ${params.JFROG_ARTFACTORY_URL}/milvus/package/${PROJECT_NAME}-${PACKAGE_VERSION}.tar.gz")
+                if (uploadStatus != 0) {
+                    error("\" ${PROJECT_NAME}-${PACKAGE_VERSION}.tar.gz \" upload to \" ${params.JFROG_ARTFACTORY_URL}/milvus/package/${PROJECT_NAME}-${PACKAGE_VERSION}.tar.gz \" failed!")
+                }
+            }
+        } else {
+            error("Milvus binary directory don't exists!")
+        }
+    }
+}
--- a/ci/jenkins/jenkinsfile/publishImages.groovy
+++ b/ci/jenkins/jenkinsfile/publishImages.groovy
@ -0,0 +1,48 @@
+container('publish-images') {
+    timeout(time: 15, unit: 'MINUTES') {
+        dir ("docker/deploy/${OS_NAME}") {
+            def binaryPackage = "${PROJECT_NAME}-${PACKAGE_VERSION}.tar.gz"
+            def dockerRegistryURL = "registry.zilliz.com"
+
+            withCredentials([usernamePassword(credentialsId: "${params.JFROG_CREDENTIALS_ID}", usernameVariable: 'JFROG_USERNAME', passwordVariable: 'JFROG_PASSWORD')]) {
+                def downloadStatus = sh(returnStatus: true, script: "curl -u${JFROG_USERNAME}:${JFROG_PASSWORD} -O ${params.JFROG_ARTFACTORY_URL}/milvus/package/${binaryPackage}")
+
+                if (downloadStatus != 0) {
+                    error("\" Download \" ${params.JFROG_ARTFACTORY_URL}/milvus/package/${binaryPackage} \" failed!")
+                }
+            }
+            sh "tar zxvf ${binaryPackage}"
+            def imageName = "${PROJECT_NAME}/engine:${DOCKER_VERSION}"
+
+            try {
+                def isExistImage = sh(returnStatus: true, script: "docker inspect --type=image ${imageName} 2>&1 > /dev/null")
+                if (isExistImage == 0) {
+                    sh "docker rmi ${imageName}"
+                }
+
+                def customImage = docker.build("${imageName}")
+
+                isExistImage = sh(returnStatus: true, script: "docker inspect --type=image ${dockerRegistryURL}/${imageName} 2>&1 > /dev/null")
+                if (isExistImage == 0) {
+                    sh "docker rmi ${dockerRegistryURL}/${imageName}"
+                }
+
+                docker.withRegistry("https://${dockerRegistryURL}", "${params.DOCKER_CREDENTIALS_ID}") {
+                    customImage.push()
+                }
+            } catch (exc) {
+                throw exc
+            } finally {
+                def isExistImage = sh(returnStatus: true, script: "docker inspect --type=image ${imageName} 2>&1 > /dev/null")
+                if (isExistImage == 0) {
+                    sh "docker rmi ${imageName}"
+                }
+
+                isExistImage = sh(returnStatus: true, script: "docker inspect --type=image ${dockerRegistryURL}/${imageName} 2>&1 > /dev/null")
+                if (isExistImage == 0) {
+                    sh "docker rmi ${dockerRegistryURL}/${imageName}"
+                }
+            }
+        } 
+    }
+}
--- a/ci/jenkins/jenkinsfile/singleDevTest.groovy
+++ b/ci/jenkins/jenkinsfile/singleDevTest.groovy
@ -0,0 +1,22 @@
+timeout(time: 30, unit: 'MINUTES') {
+    dir ("tests/milvus_python_test") {
+        sh 'python3 -m pip install -r requirements.txt'
+        sh "pytest . --alluredir=\"test_out/dev/single/sqlite\" --level=1 --ip ${env.PIPELIEN_NAME}-${env.BUILD_NUMBER}-single-gpu-milvus-gpu-engine.milvus.svc.cluster.local"
+    }
+    // mysql database backend test
+    load "${env.WORKSPACE}/ci/jenkins/jenkinsfile/cleanupSingleDev.groovy"
+
+    if (!fileExists('milvus-helm')) {
+        dir ("milvus-helm") {
+            checkout([$class: 'GitSCM', branches: [[name: "0.5.0"]], doGenerateSubmoduleConfigurations: false, extensions: [], submoduleCfg: [], userRemoteConfigs: [[credentialsId: "${params.GIT_CREDENTIALS_ID}", url: "https://github.com/milvus-io/milvus-helm.git", name: 'origin', refspec: "+refs/heads/0.5.0:refs/remotes/origin/0.5.0"]]])
+        }
+    }
+    dir ("milvus-helm") {
+        dir ("milvus-gpu") {
+            sh "helm install --wait --timeout 300 --set engine.image.tag=${DOCKER_VERSION} --set expose.type=clusterIP --name ${env.PIPELIEN_NAME}-${env.BUILD_NUMBER}-single-gpu -f ci/db_backend/mysql_values.yaml --namespace milvus ."
+        }
+    }
+    dir ("tests/milvus_python_test") {
+        sh "pytest . --alluredir=\"test_out/dev/single/mysql\" --level=1 --ip ${env.PIPELIEN_NAME}-${env.BUILD_NUMBER}-single-gpu-milvus-gpu-engine.milvus.svc.cluster.local"
+    }
+}
--- a/ci/jenkins/pod/docker-pod.yaml
+++ b/ci/jenkins/pod/docker-pod.yaml
@ -0,0 +1,23 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  labels:
+    app: publish
+    componet: docker
+spec:
+  containers:
+  - name: publish-images
+    image: registry.zilliz.com/library/docker:v1.0.0
+    securityContext:
+      privileged: true
+    command:
+    - cat
+    tty: true
+    volumeMounts:
+    - name: docker-sock
+      mountPath: /var/run/docker.sock
+  volumes:
+  - name: docker-sock
+    hostPath:
+      path: /var/run/docker.sock
+
--- a/ci/jenkins/pod/milvus-build-env-pod.yaml
+++ b/ci/jenkins/pod/milvus-build-env-pod.yaml
@ -0,0 +1,22 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  name: milvus-build-env
+  labels:
+    app: milvus
+    componet: build-env
+spec:
+  containers:
+  - name: milvus-build-env
+    image: registry.zilliz.com/milvus/milvus-build-env:v0.5.0
+    command:
+    - cat
+    tty: true
+    resources:
+      limits:
+        memory: "32Gi"
+        cpu: "8.0"
+        nvidia.com/gpu: 1
+      requests:
+        memory: "16Gi"
+        cpu: "4.0"
--- a/ci/jenkins/pod/testEnvironment.yaml
+++ b/ci/jenkins/pod/testEnvironment.yaml
@ -0,0 +1,22 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  labels:
+    app: milvus
+    componet: test-env
+spec:
+  containers:
+  - name: milvus-test-env
+    image: registry.zilliz.com/milvus/milvus-test-env:v0.1
+    command:
+    - cat
+    tty: true
+    volumeMounts:
+    - name: kubeconf
+      mountPath: /root/.kube/
+      readOnly: true
+  volumes:
+  - name: kubeconf
+    secret:
+      secretName: test-cluster-config
+
--- a/docker/build_env/ubuntu16.04/Dockerfile
+++ b/docker/build_env/ubuntu16.04/Dockerfile
@ -0,0 +1,25 @@
+FROM nvidia/cuda:10.1-devel-ubuntu16.04
+
+ENV NVIDIA_DRIVER_CAPABILITIES compute,utility
+
+RUN apt-get update && apt-get install -y --no-install-recommends wget && \
+    wget -qO- "https://cmake.org/files/v3.14/cmake-3.14.3-Linux-x86_64.tar.gz" | tar --strip-components=1 -xz -C /usr/local && \
+    wget -P /tmp https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS-2019.PUB && \
+    apt-key add /tmp/GPG-PUB-KEY-INTEL-SW-PRODUCTS-2019.PUB && \
+    sh -c 'echo deb https://apt.repos.intel.com/mkl all main > /etc/apt/sources.list.d/intel-mkl.list' && \
+    apt-get update && apt-get install -y --no-install-recommends \
+    git flex bison gfortran \
+    curl libtool automake libboost1.58-all-dev libssl-dev pkg-config libcurl4-openssl-dev \
+    clang-format-6.0 clang-tidy-6.0 \
+    lcov mysql-client libmysqlclient-dev intel-mkl-gnu-2019.4-243 intel-mkl-core-2019.4-243 && \
+    apt-get remove --purge -y && \
+    rm -rf /var/lib/apt/lists/*
+
+RUN ln -s /usr/lib/x86_64-linux-gnu/libmysqlclient.so /usr/lib/x86_64-linux-gnu/libmysqlclient_r.so
+
+RUN sh -c 'echo export LD_LIBRARY_PATH=/opt/intel/compilers_and_libraries_2019.4.243/linux/mkl/lib/intel64:\$LD_LIBRARY_PATH > /etc/profile.d/mkl.sh'
+
+COPY docker-entrypoint.sh /app/docker-entrypoint.sh
+ENTRYPOINT [ "/app/docker-entrypoint.sh" ]
+CMD [ "start" ]
+
--- a/docker/build_env/ubuntu16.04/docker-entrypoint.sh
+++ b/docker/build_env/ubuntu16.04/docker-entrypoint.sh
@ -0,0 +1,10 @@
+#!/bin/bash
+
+set -e
+
+if [ "$1" = 'start' ]; then
+   tail -f /dev/null
+fi
+
+exec "$@"
+
--- a/docker/deploy/ubuntu16.04/Dockerfile
+++ b/docker/deploy/ubuntu16.04/Dockerfile
@ -0,0 +1,23 @@
+FROM nvidia/cuda:10.1-devel-ubuntu16.04
+
+ENV NVIDIA_DRIVER_CAPABILITIES compute,utility
+
+RUN rm -rf /etc/apt/sources.list.d/nvidia-ml.list && rm -rf /etc/apt/sources.list.d/cuda.list
+
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    gfortran libsqlite3-dev libmysqlclient-dev libcurl4-openssl-dev python3 && \
+    apt-get remove --purge -y && \
+    rm -rf /var/lib/apt/lists/*
+
+RUN ln -s /usr/lib/x86_64-linux-gnu/libmysqlclient.so /usr/lib/x86_64-linux-gnu/libmysqlclient_r.so
+
+COPY ./docker-entrypoint.sh /opt
+COPY ./milvus /opt/milvus
+ENV LD_LIBRARY_PATH="$LD_LIBRARY_PATH:/opt/milvus/lib"
+
+ENTRYPOINT [ "/opt/docker-entrypoint.sh" ]
+
+CMD [ "start" ]
+
+EXPOSE 19530
+
--- a/docker/deploy/ubuntu16.04/docker-entrypoint.sh
+++ b/docker/deploy/ubuntu16.04/docker-entrypoint.sh
@ -0,0 +1,10 @@
+#!/bin/bash
+
+set -e
+
+if [ "$1" == 'start' ]; then
+    cd /opt/milvus/scripts && ./start_server.sh
+fi
+
+exec "$@"
+