diff --git a/ci/jenkins/Jenkinsfile b/ci/jenkins/Jenkinsfile index 6150fd3005..697d7592e7 100644 --- a/ci/jenkins/Jenkinsfile +++ b/ci/jenkins/Jenkinsfile @@ -54,7 +54,7 @@ pipeline { } stages { - stage ("${OS_NAME} ${CPU_ARCH} ${BINRARY_VERSION}") { + stage("Milvus ${OS_NAME} ${CPU_ARCH} ${BINRARY_VERSION}") { environment { PACKAGE_VERSION = VersionNumber([ versionNumberString : '${SEMVER}-${BINRARY_VERSION}-${OS_NAME}-${CPU_ARCH}-${LOWER_BUILD_TYPE}-${BUILD_DATE_FORMATTED, "yyyyMMdd"}-${BUILDS_TODAY}' @@ -63,7 +63,7 @@ pipeline { } stages { - stage ("Run Build") { + stage('Run Build') { agent { kubernetes { label "${BINRARY_VERSION}-build" @@ -73,7 +73,7 @@ pipeline { } stages { - stage ('Build') { + stage('Build') { steps { container("milvus-${BINRARY_VERSION}-build-env") { script { @@ -105,7 +105,7 @@ pipeline { } } - stage("Publish docker images") { + stage('Publish docker images') { agent { kubernetes { label "${BINRARY_VERSION}-publish" @@ -127,7 +127,7 @@ pipeline { } } - stage("Deploy to Development") { + stage('Deploy to Development') { environment { FROMAT_SEMVER = "${env.SEMVER}".replaceAll("\\.", "-") HELM_RELEASE_NAME = "${env.PIPELINE_NAME}-${env.FROMAT_SEMVER}-${env.BUILD_NUMBER}-single-${BINRARY_VERSION}".toLowerCase() @@ -142,7 +142,7 @@ pipeline { } stages { - stage("Deploy to Dev") { + stage('Deploy to Dev') { steps { container('milvus-test-env') { script { @@ -152,7 +152,7 @@ pipeline { } } - stage("Dev Test") { + stage('Dev Test') { steps { container('milvus-test-env') { script { @@ -167,7 +167,7 @@ pipeline { } } - stage ("Cleanup Dev") { + stage('Cleanup Dev') { steps { container('milvus-test-env') { script { diff --git a/ci/jenkins/pod/milvus-cpu-version-centos7-build-env-pod.yaml b/ci/jenkins/pod/milvus-cpu-version-centos7-build-env-pod.yaml index 37f9cb3598..6091a01eeb 100644 --- a/ci/jenkins/pod/milvus-cpu-version-centos7-build-env-pod.yaml +++ b/ci/jenkins/pod/milvus-cpu-version-centos7-build-env-pod.yaml @@ -14,8 +14,10 @@ spec: valueFrom: fieldRef: fieldPath: status.podIP + - name: OS_NAME + value: "centos7" - name: BUILD_ENV_IMAGE_ID - value: "23476391bec80c64f10d44a6370c73c71f011a6b95114b10ff82a60e771e11c7" + value: "225b4d9c26d67b70b476964b4dd6e216de4b464d7a973a8c0c7ed1313c4d81ad" command: - cat tty: true diff --git a/ci/jenkins/pod/milvus-cpu-version-ubuntu18.04-build-env-pod.yaml b/ci/jenkins/pod/milvus-cpu-version-ubuntu18.04-build-env-pod.yaml index 7301be0692..4689da5db2 100644 --- a/ci/jenkins/pod/milvus-cpu-version-ubuntu18.04-build-env-pod.yaml +++ b/ci/jenkins/pod/milvus-cpu-version-ubuntu18.04-build-env-pod.yaml @@ -14,6 +14,8 @@ spec: valueFrom: fieldRef: fieldPath: status.podIP + - name: OS_NAME + value: "ubuntu18.04" - name: BUILD_ENV_IMAGE_ID value: "23476391bec80c64f10d44a6370c73c71f011a6b95114b10ff82a60e771e11c7" command: diff --git a/ci/jenkins/pod/milvus-gpu-version-centos7-build-env-pod.yaml b/ci/jenkins/pod/milvus-gpu-version-centos7-build-env-pod.yaml index 5bc95b7cb7..34da55a93d 100644 --- a/ci/jenkins/pod/milvus-gpu-version-centos7-build-env-pod.yaml +++ b/ci/jenkins/pod/milvus-gpu-version-centos7-build-env-pod.yaml @@ -14,8 +14,10 @@ spec: valueFrom: fieldRef: fieldPath: status.podIP + - name: OS_NAME + value: "centos7" - name: BUILD_ENV_IMAGE_ID - value: "da9023b0f858f072672f86483a869aa87e90a5140864f89e5a012ec766d96dea" + value: "a5ec9914737ea4727d88ae36b4a73ca5d817f19438ba913cc1de6a1ee2ed2336" command: - cat tty: true diff --git a/ci/jenkins/pod/milvus-gpu-version-ubuntu18.04-build-env-pod.yaml b/ci/jenkins/pod/milvus-gpu-version-ubuntu18.04-build-env-pod.yaml index f41dda8b09..95b47c9111 100644 --- a/ci/jenkins/pod/milvus-gpu-version-ubuntu18.04-build-env-pod.yaml +++ b/ci/jenkins/pod/milvus-gpu-version-ubuntu18.04-build-env-pod.yaml @@ -14,6 +14,8 @@ spec: valueFrom: fieldRef: fieldPath: status.podIP + - name: OS_NAME + value: "ubuntu18.04" - name: BUILD_ENV_IMAGE_ID value: "da9023b0f858f072672f86483a869aa87e90a5140864f89e5a012ec766d96dea" command: diff --git a/ci/jenkins/step/build.groovy b/ci/jenkins/step/build.groovy index 05924f7308..ef8ce5be90 100644 --- a/ci/jenkins/step/build.groovy +++ b/ci/jenkins/step/build.groovy @@ -2,7 +2,7 @@ timeout(time: 60, unit: 'MINUTES') { dir ("ci/scripts") { withCredentials([usernamePassword(credentialsId: "${params.JFROG_CREDENTIALS_ID}", usernameVariable: 'USERNAME', passwordVariable: 'PASSWORD')]) { def checkResult = sh(script: "./check_ccache.sh -l ${params.JFROG_ARTFACTORY_URL}/ccache", returnStatus: true) - if ("${env.BINRARY_VERSION}" == "gpu") { + if ("${BINRARY_VERSION}" == "gpu") { sh ". ./before-install.sh && ./build.sh -t ${params.BUILD_TYPE} -o ${env.MILVUS_INSTALL_PREFIX} -l -g -u -c" } else { sh ". ./before-install.sh && ./build.sh -t ${params.BUILD_TYPE} -o ${env.MILVUS_INSTALL_PREFIX} -l -u -c" diff --git a/ci/jenkins/step/deploySingle2Dev.groovy b/ci/jenkins/step/deploySingle2Dev.groovy index cb2ad2b1cb..e566c3da0c 100644 --- a/ci/jenkins/step/deploySingle2Dev.groovy +++ b/ci/jenkins/step/deploySingle2Dev.groovy @@ -3,7 +3,7 @@ sh 'helm repo update' dir ('milvus-helm') { checkout([$class: 'GitSCM', branches: [[name: "0.6.0"]], userRemoteConfigs: [[url: "https://github.com/milvus-io/milvus-helm.git", name: 'origin', refspec: "+refs/heads/0.6.0:refs/remotes/origin/0.6.0"]]]) dir ("milvus") { - sh "helm install --wait --timeout 300 --set engine.image.tag=${DOCKER_VERSION} --set expose.type=clusterIP --name ${env.HELM_RELEASE_NAME} -f ci/db_backend/sqlite_${env.BINRARY_VERSION}_values.yaml -f ci/filebeat/values.yaml --namespace milvus ." + sh "helm install --wait --timeout 300 --set engine.image.tag=${DOCKER_VERSION} --set expose.type=clusterIP --name ${env.HELM_RELEASE_NAME} -f ci/db_backend/sqlite_${BINRARY_VERSION}_values.yaml -f ci/filebeat/values.yaml --namespace milvus ." } } diff --git a/ci/jenkins/step/publishImages.groovy b/ci/jenkins/step/publishImages.groovy index 5449bcedd8..fe1c211abc 100644 --- a/ci/jenkins/step/publishImages.groovy +++ b/ci/jenkins/step/publishImages.groovy @@ -1,5 +1,5 @@ timeout(time: 15, unit: 'MINUTES') { - dir ("docker/deploy/${env.BINRARY_VERSION}/${env.OS_NAME}") { + dir ("docker/deploy/${BINRARY_VERSION}/${OS_NAME}") { def binaryPackage = "${PROJECT_NAME}-${PACKAGE_VERSION}.tar.gz" withCredentials([usernamePassword(credentialsId: "${params.JFROG_CREDENTIALS_ID}", usernameVariable: 'JFROG_USERNAME', passwordVariable: 'JFROG_PASSWORD')]) { diff --git a/ci/jenkins/step/singleDevNightlyTest.groovy b/ci/jenkins/step/singleDevNightlyTest.groovy index d14ba1b66c..618c2457b3 100644 --- a/ci/jenkins/step/singleDevNightlyTest.groovy +++ b/ci/jenkins/step/singleDevNightlyTest.groovy @@ -13,7 +13,7 @@ timeout(time: 90, unit: 'MINUTES') { } dir ("milvus-helm") { dir ("milvus") { - sh "helm install --wait --timeout 300 --set engine.image.tag=${DOCKER_VERSION} --set expose.type=clusterIP --name ${env.HELM_RELEASE_NAME} -f ci/db_backend/mysql_${env.BINRARY_VERSION}_values.yaml -f ci/filebeat/values.yaml --namespace milvus ." + sh "helm install --wait --timeout 300 --set engine.image.tag=${DOCKER_VERSION} --set expose.type=clusterIP --name ${env.HELM_RELEASE_NAME} -f ci/db_backend/mysql_${BINRARY_VERSION}_values.yaml -f ci/filebeat/values.yaml --namespace milvus ." } } dir ("tests/milvus_python_test") { diff --git a/ci/jenkins/step/singleDevTest.groovy b/ci/jenkins/step/singleDevTest.groovy index 7b72eaacde..255f748996 100644 --- a/ci/jenkins/step/singleDevTest.groovy +++ b/ci/jenkins/step/singleDevTest.groovy @@ -14,7 +14,7 @@ timeout(time: 60, unit: 'MINUTES') { // } // dir ("milvus-helm") { // dir ("milvus") { - // sh "helm install --wait --timeout 300 --set engine.image.tag=${DOCKER_VERSION} --set expose.type=clusterIP --name ${env.HELM_RELEASE_NAME} -f ci/db_backend/mysql_${env.BINRARY_VERSION}_values.yaml -f ci/filebeat/values.yaml --namespace milvus ." + // sh "helm install --wait --timeout 300 --set engine.image.tag=${DOCKER_VERSION} --set expose.type=clusterIP --name ${env.HELM_RELEASE_NAME} -f ci/db_backend/mysql_${BINRARY_VERSION}_values.yaml -f ci/filebeat/values.yaml --namespace milvus ." // } // } // dir ("tests/milvus_python_test") { diff --git a/ci/scripts/check_ccache.sh b/ci/scripts/check_ccache.sh index 2350bd558d..cc0708505e 100755 --- a/ci/scripts/check_ccache.sh +++ b/ci/scripts/check_ccache.sh @@ -1,7 +1,6 @@ #!/bin/bash -OS_NAME="linux" -CODE_NAME=$(lsb_release -sc) +OS_NAME="${OS_NAME}" BUILD_ENV_DOCKER_IMAGE_ID="${BUILD_ENV_IMAGE_ID}" BRANCH_NAMES=$(git log --decorate | head -n 1 | sed 's/.*(\(.*\))/\1/' | sed 's=[a-zA-Z]*\/==g' | awk -F", " '{$1=""; print $0}') ARTIFACTORY_URL="" @@ -43,13 +42,14 @@ fi check_ccache() { BRANCH=$1 - echo "fetching ${BRANCH}/ccache-${OS_NAME}-${CODE_NAME}-${BUILD_ENV_DOCKER_IMAGE_ID}.tar.gz" - wget -q --method HEAD "${ARTIFACTORY_URL}/${BRANCH}/ccache-${OS_NAME}-${CODE_NAME}-${BUILD_ENV_DOCKER_IMAGE_ID}.tar.gz" + PACKAGE_FILE="ccache-${OS_NAME}-${BUILD_ENV_DOCKER_IMAGE_ID}.tar.gz" + echo "fetching ${BRANCH}/${PACKAGE_FILE}" + wget -q --method HEAD "${ARTIFACTORY_URL}/${BRANCH}/${PACKAGE_FILE}" if [[ $? == 0 ]];then - wget -q "${ARTIFACTORY_URL}/${BRANCH}/ccache-${OS_NAME}-${CODE_NAME}-${BUILD_ENV_DOCKER_IMAGE_ID}.tar.gz" && \ + wget -q "${ARTIFACTORY_URL}/${BRANCH}/${PACKAGE_FILE}" && \ mkdir -p ${CCACHE_DIRECTORY} && \ - tar zxf ccache-${OS_NAME}-${CODE_NAME}-${BUILD_ENV_DOCKER_IMAGE_ID}.tar.gz -C ${CCACHE_DIRECTORY} && \ - rm ccache-${OS_NAME}-${CODE_NAME}-${BUILD_ENV_DOCKER_IMAGE_ID}.tar.gz + tar zxf ${PACKAGE_FILE} -C ${CCACHE_DIRECTORY} && \ + rm ${PACKAGE_FILE} if [[ $? == 0 ]];then echo "found cache" exit 0 diff --git a/ci/scripts/update_ccache.sh b/ci/scripts/update_ccache.sh index f4afc29d1e..ec52bd1156 100755 --- a/ci/scripts/update_ccache.sh +++ b/ci/scripts/update_ccache.sh @@ -1,7 +1,6 @@ #!/bin/bash -OS_NAME="linux" -CODE_NAME=$(lsb_release -sc) +OS_NAME="${OS_NAME}" BUILD_ENV_DOCKER_IMAGE_ID="${BUILD_ENV_IMAGE_ID}" BRANCH_NAME=$(git log --decorate | head -n 1 | sed 's/.*(\(.*\))/\1/' | sed 's/.*, //' | sed 's=[a-zA-Z]*\/==g') ARTIFACTORY_URL="" @@ -51,7 +50,7 @@ if [[ -z "${ARTIFACTORY_URL}" || "${ARTIFACTORY_URL}" == "" ]];then exit 1 fi -PACKAGE_FILE="ccache-${OS_NAME}-${CODE_NAME}-${BUILD_ENV_DOCKER_IMAGE_ID}.tar.gz" +PACKAGE_FILE="ccache-${OS_NAME}-${BUILD_ENV_DOCKER_IMAGE_ID}.tar.gz" REMOTE_PACKAGE_PATH="${ARTIFACTORY_URL}/${BRANCH_NAME}" ccache --show-stats