milvus-io
/
milvus
mirror of https://github.com/milvus-io/milvus.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Simplify testing Kubernetes application deployment within CI pipeline by using Helm 

Signed-off-by: quicksilver <zhifeng.zhang@zilliz.com>
pull/4973/head^2
quicksilver 2021-03-11 12:01:05 +08:00 committed by yefu.chen
parent 02796cf961
commit ba3270112a
5 changed files with 123 additions and 59 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
.jenkins/modules/Regression/CleanupPythonRegression.groovy Normal file
View File

@ -0,0 +1,8 @@
container('deploy-env') {
def helmStatus = sh script: "helm status -n ${env.HELM_RELEASE_NAMESPACE} ${env.HELM_RELEASE_NAME}", returnStatus: true
if (!helmStatus) {
sh "helm uninstall -n ${env.HELM_RELEASE_NAMESPACE} ${env.HELM_RELEASE_NAME}"
def labels = "app.kubernetes.io/instance=${env.HELM_RELEASE_NAME}"
sh "kubectl delete pvc -n ${env.HELM_RELEASE_NAMESPACE} \$(kubectl get pvc -n ${env.HELM_RELEASE_NAMESPACE} -l ${labels} -o jsonpath='{range.items[*]}{.metadata.name} ')"
}
}

113
.jenkins/modules/Regression/PythonRegression.groovy
View File

@ -1,59 +1,70 @@
timeout(time: 60, unit: 'MINUTES') {
try {
if ("${REGRESSION_SERVICE_NAME}" == "regression_distributed") {
sh 'docker-compose -p ${DOCKER_COMPOSE_PROJECT_NAME}-${REGRESSION_SERVICE_NAME} up -d pulsar'
}
sh 'docker-compose -p ${DOCKER_COMPOSE_PROJECT_NAME}-${REGRESSION_SERVICE_NAME} up -d etcd'
sh 'docker-compose -p ${DOCKER_COMPOSE_PROJECT_NAME}-${REGRESSION_SERVICE_NAME} up -d minio'
dir ('build/docker/deploy') {
sh 'docker pull ${TARGET_REPO}/milvus-distributed:${TARGET_TAG}'
if ("${REGRESSION_SERVICE_NAME}" == "regression_distributed") {
sh 'docker-compose -p ${DOCKER_COMPOSE_PROJECT_NAME}-${REGRESSION_SERVICE_NAME} up -d master'
sh 'docker-compose -p ${DOCKER_COMPOSE_PROJECT_NAME}-${REGRESSION_SERVICE_NAME} up -d indexservice'
sh 'docker-compose -p ${DOCKER_COMPOSE_PROJECT_NAME}-${REGRESSION_SERVICE_NAME} up -d indexnode'
sh 'docker-compose -p ${DOCKER_COMPOSE_PROJECT_NAME}-${REGRESSION_SERVICE_NAME} up -d proxyservice'
sh 'docker-compose -p ${DOCKER_COMPOSE_PROJECT_NAME}-${REGRESSION_SERVICE_NAME} up -d dataservice'
sh 'docker-compose -p ${DOCKER_COMPOSE_PROJECT_NAME}-${REGRESSION_SERVICE_NAME} up -d queryservice'
sh 'docker-compose -p ${DOCKER_COMPOSE_PROJECT_NAME}-${REGRESSION_SERVICE_NAME} run -e DATA_NODE_ID=3 -d datanode'
sh 'docker-compose -p ${DOCKER_COMPOSE_PROJECT_NAME}-${REGRESSION_SERVICE_NAME} up -d proxynode'
sh 'docker-compose -p ${DOCKER_COMPOSE_PROJECT_NAME}-${REGRESSION_SERVICE_NAME} run -e QUERY_NODE_ID=1 -d querynode'
sh 'docker-compose -p ${DOCKER_COMPOSE_PROJECT_NAME}-${REGRESSION_SERVICE_NAME} run -e QUERY_NODE_ID=2 -d querynode'
} else {
sh 'docker-compose -p ${DOCKER_COMPOSE_PROJECT_NAME}-${REGRESSION_SERVICE_NAME} up -d standalone'
}
}
container('deploy-env') {
dir ('milvus-helm-chart') {
sh " helm version && \
helm repo add stable https://kubernetes.oss-cn-hangzhou.aliyuncs.com/charts && \
helm repo add bitnami https://charts.bitnami.com/bitnami && \
helm repo add minio https://helm.min.io/ && \
helm repo update"
dir ('build/docker/test') {
sh 'docker pull ${SOURCE_REPO}/pytest:${SOURCE_TAG} || true'
sh 'docker-compose build --force-rm ${REGRESSION_SERVICE_NAME}'
sh 'docker-compose -p ${DOCKER_COMPOSE_PROJECT_NAME}-${REGRESSION_SERVICE_NAME} run --rm ${REGRESSION_SERVICE_NAME}'
try {
withCredentials([usernamePassword(credentialsId: "${env.DOCKER_CREDENTIALS_ID}", usernameVariable: 'DOCKER_USERNAME', passwordVariable: 'DOCKER_PASSWORD')]) {
sh 'docker login -u ${DOCKER_USERNAME} -p ${DOCKER_PASSWORD} ${DOKCER_REGISTRY_URL}'
sh 'docker-compose push ${REGRESSION_SERVICE_NAME} || true'
def milvusHelmURL = "https://github.com/zilliztech/milvus-helm-charts.git"
checkout([$class: 'GitSCM', branches: [[name: "${env.HELM_BRANCH}"]], userRemoteConfigs: [[url: "${milvusHelmURL}"]]])
dir ('charts/milvus-ha') {
sh script: "kubectl create namespace ${env.HELM_RELEASE_NAMESPACE}", returnStatus: true
def helmCMD = ""
if ("${REGRESSION_SERVICE_TYPE}" == "distributed") {
helmCMD = "helm install --wait --timeout 300s \
--set standalone.enabled=false \
--set image.all.repository=${env.TARGET_REPO}/milvus-distributed \
--set image.all.tag=${env.TARGET_TAG} \
--set image.all.pullPolicy=Always \
--set logsPersistence.enabled=true \
--set logsPersistence.mountPath=/milvus-distributed/logs \
--namespace ${env.HELM_RELEASE_NAMESPACE} ${env.HELM_RELEASE_NAME} ."
} else {
helmCMD = "helm install --wait --timeout 300s \
--set image.all.repository=${env.TARGET_REPO}/milvus-distributed \
--set image.all.tag=${env.TARGET_TAG} \
--set image.all.pullPolicy=Always \
--set logsPersistence.enabled=true \
--set logsPersistence.mountPath=/milvus-distributed/logs \
--namespace ${env.HELM_RELEASE_NAMESPACE} ${env.HELM_RELEASE_NAME} ."
}
try {
sh "${helmCMD}"
} catch (exc) {
def helmStatusCMD = "helm get manifest -n ${env.HELM_RELEASE_NAMESPACE} ${env.HELM_RELEASE_NAME} | kubectl describe -n ${env.HELM_RELEASE_NAMESPACE} -f - && \
helm status -n ${env.HELM_RELEASE_NAMESPACE} ${env.HELM_RELEASE_NAME}"
sh script: helmStatusCMD, returnStatus: true
throw exc
}
} catch (exc) {
throw exc
} finally {
sh 'docker logout ${DOKCER_REGISTRY_URL}'
}
}
} catch(exc) {
throw exc
} finally {
dir ('build/docker/deploy') {
sh 'docker-compose -p ${DOCKER_COMPOSE_PROJECT_NAME}-${REGRESSION_SERVICE_NAME} ps -a | tail -n +3 | awk \'{ print $1 }\' | ( while read arg; do docker logs -t $arg > $arg.log 2>&1; done )'
archiveArtifacts artifacts: "**.log", allowEmptyArchive: true
sh 'docker-compose -p ${DOCKER_COMPOSE_PROJECT_NAME}-${REGRESSION_SERVICE_NAME} down --rmi all -v || true'
}
if ("${REGRESSION_SERVICE_NAME}" == "regression_distributed") {
sh 'docker-compose -p ${DOCKER_COMPOSE_PROJECT_NAME}-${REGRESSION_SERVICE_NAME} rm -f -s -v pulsar'
}
sh 'docker-compose -p ${DOCKER_COMPOSE_PROJECT_NAME}-${REGRESSION_SERVICE_NAME} rm -f -s -v etcd'
sh 'docker-compose -p ${DOCKER_COMPOSE_PROJECT_NAME}-${REGRESSION_SERVICE_NAME} rm -f -s -v minio'
dir ('build/docker/test') {
sh 'docker-compose -p ${DOCKER_COMPOSE_PROJECT_NAME}-${REGRESSION_SERVICE_NAME} run --rm ${REGRESSION_SERVICE_NAME} /bin/bash -c "find . -name \'__pycache__\' | xargs rm -rf && find . -name \'.pytest_cache\' | xargs rm -rf"'
sh 'docker-compose -p ${DOCKER_COMPOSE_PROJECT_NAME}-${REGRESSION_SERVICE_NAME} down --rmi all -v || true'
}
container('test-env') {
try {
dir ('tests/python_test') {
sh "pytest --tags=0331 --ip ${env.HELM_RELEASE_NAME}-milvus-ha.${env.HELM_RELEASE_NAMESPACE}.svc.cluster.local"
}
} catch (exc) {
echo 'PyTest Regression Failed !'
throw exc
} finally {
container('deploy-env') {
def labels = ""
if ("${REGRESSION_SERVICE_TYPE}" == "distributed") {
labels = "app.kubernetes.io/instance=${env.HELM_RELEASE_NAME},component=proxyservice"
} else {
labels = "app.kubernetes.io/instance=${env.HELM_RELEASE_NAME},component=standalone"
}
sh "mkdir -p ${env.DEV_TEST_ARTIFACTS_PATH}"
sh "kubectl cp -n ${env.HELM_RELEASE_NAMESPACE} \$(kubectl get pod -n ${env.HELM_RELEASE_NAMESPACE} -l ${labels} -o jsonpath='{range.items[0]}{.metadata.name}'):logs ${env.DEV_TEST_ARTIFACTS_PATH}"
archiveArtifacts artifacts: "${env.DEV_TEST_ARTIFACTS_PATH}/**", allowEmptyArchive: true
}
}
}
}

19
build/ci/jenkins/Jenkinsfile vendored
View File

@ -7,7 +7,7 @@ pipeline {
timestamps()
}
environment {
PROJECT_NAME = "milvus-distributed"
PROJECT_NAME = "milvus"
SEMVER = "${BRANCH_NAME.contains('/') ? BRANCH_NAME.substring(BRANCH_NAME.lastIndexOf('/') + 1) : BRANCH_NAME}"
BUILD_TYPE = "Release"
LOWER_BUILD_TYPE = BUILD_TYPE.toLowerCase()
@ -71,15 +71,22 @@ pipeline {
matrix {
axes {
axis {
name 'REGRESSION_SERVICE_NAME'
values 'regression_standalone', 'regression_distributed'
name 'REGRESSION_SERVICE_TYPE'
values 'standalone', 'distributed'
}
}
environment {
DOCKER_COMPOSE_PROJECT_NAME = "${PROJECT_NAME}-${SEMVER}-${env.BUILD_NUMBER}".replaceAll("\\.", "-").replaceAll("_", "-")
HELM_BRANCH = "main"
HELM_RELEASE_NAMESPACE = "dev"
HELM_RELEASE_NAME = "${PROJECT_NAME}-${SEMVER}-${env.BUILD_NUMBER}-${REGRESSION_SERVICE_TYPE}".replaceAll("\\.", "-").replaceAll("_", "-").toLowerCase()
DEV_TEST_ARTIFACTS_PATH = "artifacts"
}
agent {
label 'performance'
kubernetes {
label "${env.PROJECT_NAME}-${SEMVER}-${env.BUILD_NUMBER}-${REGRESSION_SERVICE_TYPE}-devtest"
defaultContainer 'test-env'
yamlFile "build/ci/jenkins/pod/test-env.yaml"
}
}
stages {
stage('Test') {
@ -90,7 +97,7 @@ pipeline {
}
post {
cleanup {
deleteDir() /* clean up our workspace */
MPLModule('Cleanup Python Regression')
}
}
}

38
build/ci/jenkins/pod/test-env.yaml Normal file
View File

@ -0,0 +1,38 @@
apiVersion: v1
kind: Pod
metadata:
labels:
app: milvus
componet: test-env
spec:
containers:
- name: test-env
image: milvusdb/pytest:20210310-5ea22d7
command:
- cat
tty: true
resources:
limits:
memory: "8Gi"
cpu: "4.0"
requests:
memory: "4Gi"
cpu: "2.0"
- name: deploy-env
image: registry.zilliz.com/library/helm3:v0.1
command:
- cat
tty: true
volumeMounts:
- name: kubeconf
mountPath: /root/.kube/
readOnly: true
volumes:
- name: kubeconf
secret:
secretName: test-cluster-config
tolerations:
- key: dedicated
operator: Equal
value: milvus
effect: NoSchedule

4
build/docker/test/.env
View File

@ -1,4 +1,4 @@
SOURCE_REPO=milvusdb
TARGET_REPO=milvusdb
SOURCE_TAG=latest
TARGET_TAG=latest
SOURCE_TAG=20210310-5ea22d7
TARGET_TAG=20210310-5ea22d7