milvus-io
/
milvus
mirror of https://github.com/milvus-io/milvus.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Make pytest run in separate pods (#14779) 

Signed-off-by: Jenny Li <jing.li@zilliz.com>
pull/14799/head
Jenny Li 2022-01-05 09:41:32 +08:00 committed by GitHub
parent 96a4450f2e
commit 870b2ed131
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 35 additions and 24 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

39
build/ci/jenkins/PR.groovy
View File

@ -15,7 +15,7 @@ pipeline {
}
agent {
kubernetes {
label 'milvus-e2e-test-pr'
label 'milvus-e2e-test-pipeline'
inheritFrom 'default'
defaultContainer 'main'
yamlFile 'build/ci/jenkins/pod/rte.yaml'
@ -137,6 +137,15 @@ pipeline {
}
}
stage('E2E Test'){
agent {
kubernetes {
label 'milvus-e2e-test'
inheritFrom 'default'
defaultContainer 'main'
yamlFile 'build/ci/jenkins/pod/rte.yaml'
customWorkspace '/home/jenkins/agent/workspace'
}
}
steps {
container('pytest') {
dir ('tests/scripts') {
@ -154,6 +163,7 @@ pipeline {
TEST_TIMEOUT="${e2e_timeout_seconds}" \
./ci_e2e.sh "-n 6 -x --tags L0 L1 --timeout ${case_timeout_seconds}"
"""
} else {
error "Error: Unsupported Milvus client: ${MILVUS_CLIENT}"
}
@ -166,27 +176,22 @@ pipeline {
}
post{
always {
container('pytest'){
dir("${env.ARTIFACTS}") {
sh "tar -zcvf artifacts-${PROJECT_NAME}-${MILVUS_SERVER_TYPE}-${MILVUS_CLIENT}-pytest-logs.tar.gz /tmp/ci_logs/test --remove-files || true"
archiveArtifacts artifacts: "artifacts-${PROJECT_NAME}-${MILVUS_SERVER_TYPE}-${MILVUS_CLIENT}-pytest-logs.tar.gz ", allowEmptyArchive: true
}
}
container('main') {
dir ('tests/scripts') {
script {
def release_name=sh(returnStdout: true, script: './get_release_name.sh')
sh "./uninstall_milvus.sh --release-name ${release_name}"
}
}
}
container('pytest') {
dir ('tests/scripts') {
script {
def release_name = sh(returnStdout: true, script: './get_release_name.sh ')
sh "./ci_logs.sh --log-dir /ci-logs --artifacts-name ${env.ARTIFACTS}/artifacts-${PROJECT_NAME}-${MILVUS_SERVER_TYPE}-${SEMVER}-${env.BUILD_NUMBER}-${MILVUS_CLIENT}-e2e-logs \
--release-name ${release_name}"
dir("${env.ARTIFACTS}") {
if ("${MILVUS_CLIENT}" == "pymilvus") {
sh "tar -zcvf artifacts-${PROJECT_NAME}-${MILVUS_SERVER_TYPE}-${MILVUS_CLIENT}-pytest-logs.tar.gz /tmp/ci_logs/test --remove-files || true"
}
archiveArtifacts artifacts: "artifacts-${PROJECT_NAME}-${MILVUS_SERVER_TYPE}-${MILVUS_CLIENT}-pytest-logs.tar.gz ", allowEmptyArchive: true
archiveArtifacts artifacts: "artifacts-${PROJECT_NAME}-${MILVUS_SERVER_TYPE}-${SEMVER}-${env.BUILD_NUMBER}-${MILVUS_CLIENT}-e2e-logs.tar.gz", allowEmptyArchive: true
}
sh "./ci_logs.sh --log-dir /ci-logs --artifacts-name ${env.ARTIFACTS}/artifacts-${PROJECT_NAME}-${MILVUS_SERVER_TYPE}-${SEMVER}-${env.BUILD_NUMBER}-${MILVUS_CLIENT}-e2e-logs \
--release-name ${release_name}"
dir("${env.ARTIFACTS}") {
archiveArtifacts artifacts: "artifacts-${PROJECT_NAME}-${MILVUS_SERVER_TYPE}-${SEMVER}-${env.BUILD_NUMBER}-${MILVUS_CLIENT}-e2e-logs.tar.gz", allowEmptyArchive: true
}
}
}
}

14
build/ci/jenkins/pod/rte.yaml
View File

@ -37,12 +37,18 @@ spec:
name: cgroup
- mountPath: /mnt/disk/.docker
name: build-cache
subPath: docker-volume
- name: pytest
image: milvusdb/pytest:20211229-b4c3cd8
volumeMounts:
subPath: docker-volume
- mountPath: /ci-logs
name: ci-logs
- name: pytest
image: milvusdb/pytest:20211229-b4c3cd8
resources:
limits:
cpu: "6"
memory: 12Gi
requests:
cpu: "2"
memory: 10Gi
volumes:
- emptyDir: {}
name: docker-graph

6
tests/scripts/ci_e2e.sh
View File

@ -40,8 +40,6 @@ MILVUS_CLIENT="${MILVUS_CLIENT:-pymilvus}"
MILVUS_SERVICE_NAME=$(echo "${MILVUS_HELM_RELEASE_NAME}-milvus.${MILVUS_HELM_NAMESPACE}" | tr -d '\n')
MILVUS_SERVICE_PORT="19530"
# Shellcheck source=build/lib.sh
source "${ROOT}/build/lib.sh"
# Shellcheck source=ci-util.sh
@ -60,7 +58,9 @@ if [ ! -d "${CI_LOG_PATH}" ]; then
# Create dir for ci log path when it does not exist
mkdir -p ${CI_LOG_PATH}
fi
trace "prepare e2e test" install_pytest_requirements
echo "prepare e2e test"
install_pytest_requirements
# Pytest is not able to have both --timeout & --workers, so do not add --timeout or --workers in the shell script