milvus-io
/
milvus
mirror of https://github.com/milvus-io/milvus.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Add Cron job jenkins pipeline (#5634) 

* Add Cron job jenkins pipeline

Signed-off-by: quicksilver <zhifeng.zhang@zilliz.com>

Add Cron job jenkins pipeline

Signed-off-by: quicksilver <zhifeng.zhang@zilliz.com>

Add Cron job jenkins pipeline

Signed-off-by: quicksilver <zhifeng.zhang@zilliz.com>

Add Cron job jenkins pipeline

Signed-off-by: quicksilver <zhifeng.zhang@zilliz.com>

Add Cron job jenkins pipeline

Signed-off-by: quicksilver <zhifeng.zhang@zilliz.com>

Add Cron job jenkins pipeline

Signed-off-by: quicksilver <zhifeng.zhang@zilliz.com>

Add Cron job jenkins pipeline

Signed-off-by: quicksilver <zhifeng.zhang@zilliz.com>

* Add Cron job jenkins pipeline

Signed-off-by: quicksilver <zhifeng.zhang@zilliz.com>

* Add Cron job jenkins pipeline

Signed-off-by: quicksilver <zhifeng.zhang@zilliz.com>
pull/5647/head
quicksilver 2021-06-07 14:17:38 +08:00 committed by GitHub
parent 50b3d39370
commit 7d9d524539
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 122 additions and 45 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

46
build/ci/jenkins/Jenkinsfile vendored
View File

@ -1,17 +1,10 @@
#!/usr/bin/env groovy
String cron_timezone = "TZ=Asia/Shanghai"
String cron_string = BRANCH_NAME == "master" ? "50 20,22,0,6,11,16 * * * " : ""
pipeline {
agent none
triggers {
cron """${cron_timezone}
${cron_string}"""
}
options {
timestamps()
timeout(time: 1, unit: 'HOURS')
timeout(time: 30, unit: 'MINUTES')
buildDiscarder logRotator(artifactDaysToKeepStr: '30')
// parallelsAlwaysFailFast()
}
@ -41,9 +34,6 @@ pipeline {
IMAGE_REPO = "dockerhub-mirror-sh.zilliz.cc/milvusdb"
DOCKER_BUILDKIT = 1
ARTIFACTS = "${env.WORKSPACE}/artifacts"
DOCKER_CREDENTIALS_ID = "ba070c98-c8cc-4f7c-b657-897715f359fc"
DOKCER_REGISTRY_URL = "registry.zilliz.com"
TARGET_REPO = "${DOKCER_REGISTRY_URL}/milvus"
}
stages {
stage('Test') {
@ -56,12 +46,7 @@ pipeline {
standaloneEnabled = "false"
}
if (isTimeTriggeredBuild()) {
echo "This is Cron Job!"
sh "MILVUS_STANDALONE_ENABLED=${standaloneEnabled} ./e2e-k8s.sh --node-image registry.zilliz.com/kindest/node:v1.20.2"
} else {
sh "MILVUS_STANDALONE_ENABLED=${standaloneEnabled} ./e2e-k8s.sh --node-image registry.zilliz.com/kindest/node:v1.20.2 --test-extra-arg \"--tags=smoke\""
}
sh "MILVUS_STANDALONE_ENABLED=${standaloneEnabled} ./e2e-k8s.sh --node-image registry.zilliz.com/kindest/node:v1.20.2 --test-extra-arg \"--tags=smoke\""
}
}
}
@ -69,27 +54,6 @@ pipeline {
}
}
post {
success {
container('main') {
script {
if ( env.CHANGE_ID == null ){
def date = sh(returnStdout: true, script: 'date +%Y%m%d').trim()
def gitShortCommit = sh(returnStdout: true, script: 'git rev-parse --short HEAD').trim()
withCredentials([usernamePassword(credentialsId: "${env.DOCKER_CREDENTIALS_ID}", usernameVariable: 'DOCKER_USERNAME', passwordVariable: 'DOCKER_PASSWORD')]) {
sh 'docker login -u ${DOCKER_USERNAME} -p ${DOCKER_PASSWORD} ${DOKCER_REGISTRY_URL}'
sh """
docker tag localhost:5000/milvus:latest ${TARGET_REPO}/milvus:${env.BRANCH_NAME}-${date}-${gitShortCommit}
docker tag localhost:5000/milvus:latest ${TARGET_REPO}/milvus:${env.BRANCH_NAME}-latest
docker push ${TARGET_REPO}/milvus:${env.BRANCH_NAME}-${date}-${gitShortCommit}
docker push ${TARGET_REPO}/milvus:${env.BRANCH_NAME}-latest
"""
sh 'docker logout ${DOKCER_REGISTRY_URL}'
}
}
}
}
}
always {
container('main') {
script {
@ -98,7 +62,7 @@ pipeline {
archiveArtifacts artifacts: "**.tar.gz", allowEmptyArchive: true
sh 'rm -rf ./*'
sh 'docker rm -f \$(docker network inspect -f \'{{ range \$key, \$value := .Containers }}{{ printf "%s " \$key}}{{ end }}\' kind) || true'
sh 'docker network rm kind || true'
sh 'docker network rm kind 2>&1 > /dev/null || true'
}
}
}
@ -108,7 +72,3 @@ pipeline {
}
}
}
boolean isTimeTriggeredBuild() {
return (currentBuild.getBuildCauses('hudson.triggers.TimerTrigger$TimerTriggerCause').size() != 0) ? true : false;
}

117
build/ci/jenkins/NightlyCI.groovy Normal file
View File

@ -0,0 +1,117 @@
#!/usr/bin/env groovy
String cron_timezone = "TZ=Asia/Shanghai"
String cron_string = BRANCH_NAME == "master" ? "50 20,22,0,6,11,16 * * * " : ""
pipeline {
agent none
triggers {
pollSCM ignorePostCommitHooks: true, scmpoll_spec: """${cron_timezone}
${cron_string}"""
}
options {
timestamps()
timeout(time: 1, unit: 'HOURS')
buildDiscarder logRotator(artifactDaysToKeepStr: '30')
// parallelsAlwaysFailFast()
}
stages {
stage ('E2E Test') {
matrix {
axes {
axis {
name 'MILVUS_SERVER_TYPE'
values 'standalone', 'distributed'
}
}
agent {
kubernetes {
label "milvus-e2e-test-kind-nightly"
inheritFrom 'default'
defaultContainer 'main'
yamlFile "build/ci/jenkins/pod/krte.yaml"
customWorkspace '/home/jenkins/agent/workspace'
}
}
environment {
PROJECT_NAME = "milvus"
SEMVER = "${BRANCH_NAME.contains('/') ? BRANCH_NAME.substring(BRANCH_NAME.lastIndexOf('/') + 1) : BRANCH_NAME}"
IMAGE_REPO = "dockerhub-mirror-sh.zilliz.cc/milvusdb"
DOCKER_BUILDKIT = 1
ARTIFACTS = "${env.WORKSPACE}/artifacts"
DOCKER_CREDENTIALS_ID = "ba070c98-c8cc-4f7c-b657-897715f359fc"
DOKCER_REGISTRY_URL = "registry.zilliz.com"
TARGET_REPO = "${DOKCER_REGISTRY_URL}/milvus"
}
stages {
stage('Test') {
steps {
container('main') {
dir ('tests/scripts') {
script {
def standaloneEnabled = "true"
if ("${MILVUS_SERVER_TYPE}" == "distributed") {
standaloneEnabled = "false"
}
sh "MILVUS_STANDALONE_ENABLED=${standaloneEnabled} ./e2e-k8s.sh --node-image registry.zilliz.com/kindest/node:v1.20.2"
}
}
}
}
}
}
post {
unsuccessful {
container('jnlp') {
script {
emailext subject: '$DEFAULT_SUBJECT',
body: '$DEFAULT_CONTENT',
recipientProviders: [
[$class: 'DevelopersRecipientProvider'],
[$class: 'RequesterRecipientProvider']
],
replyTo: '$DEFAULT_REPLYTO',
to: 'qa@zilliz.com'
}
}
}
success {
container('main') {
script {
if ( env.CHANGE_ID == null ){
def date = sh(returnStdout: true, script: 'date +%Y%m%d').trim()
def gitShortCommit = sh(returnStdout: true, script: 'git rev-parse --short HEAD').trim()
withCredentials([usernamePassword(credentialsId: "${env.DOCKER_CREDENTIALS_ID}", usernameVariable: 'DOCKER_USERNAME', passwordVariable: 'DOCKER_PASSWORD')]) {
sh 'docker login -u ${DOCKER_USERNAME} -p ${DOCKER_PASSWORD} ${DOKCER_REGISTRY_URL}'
sh """
docker tag localhost:5000/milvus:latest ${TARGET_REPO}/milvus:${env.BRANCH_NAME}-${date}-${gitShortCommit}
docker tag localhost:5000/milvus:latest ${TARGET_REPO}/milvus:${env.BRANCH_NAME}-latest
docker push ${TARGET_REPO}/milvus:${env.BRANCH_NAME}-${date}-${gitShortCommit}
docker push ${TARGET_REPO}/milvus:${env.BRANCH_NAME}-latest
"""
sh 'docker logout ${DOKCER_REGISTRY_URL}'
}
}
}
}
}
always {
container('main') {
script {
dir("${env.ARTIFACTS}") {
sh "find ./kind -path '*/history/*' -type f | xargs tar -zcvf artifacts-${PROJECT_NAME}-${MILVUS_SERVER_TYPE}-${SEMVER}-${env.BUILD_NUMBER}-e2e-nightly-logs.tar.gz --transform='s:^[^/]*/[^/]*/[^/]*/[^/]*/::g' || true"
archiveArtifacts artifacts: "**.tar.gz", allowEmptyArchive: true
sh 'rm -rf ./*'
sh 'docker rm -f \$(docker network inspect -f \'{{ range \$key, \$value := .Containers }}{{ printf "%s " \$key}}{{ end }}\' kind) || true'
sh 'docker network rm kind 2>&1 > /dev/null || true'
}
}
}
}
}
}
}
}
}

4
build/kind_provisioner.sh
View File

@ -128,7 +128,7 @@ function cleanup_kind_cluster() {
if [[ -z "${SKIP_CLEANUP:-}" ]]; then
echo "Cleaning up kind cluster"
kind delete cluster --name "${NAME}" -v9 || true
docker network rm kind || true
docker network rm kind 2>&1 > /dev/null || true
fi
}
@ -163,7 +163,7 @@ function setup_kind_cluster() {
if ! (kind delete cluster --name="${NAME}" -v9) > /dev/null; then
echo "No existing kind cluster with name ${NAME}. Continue..."
else
docker network rm kind || true
docker network rm kind 2>&1 > /dev/null || true
fi
# explicitly disable shellcheck since we actually want $NAME to expand now