milvus-io
/
milvus
mirror of https://github.com/milvus-io/milvus.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Add kafka support in nightly CI (#16494) 

Signed-off-by: Binbin Lv <binbin.lv@zilliz.com>
pull/16498/head
binbin 2022-04-14 22:29:34 +08:00 committed by GitHub
parent cf9b9aba79
commit f24d06ade1
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 32 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
build/ci/jenkins/Nightly.groovy
View File

@ -8,7 +8,7 @@ String cron_string = BRANCH_NAME == "master" ? "50 22,2 * * * " : ""
// Make timeout 4 hours so that we can run two nightly during the ci // Make timeout 4 hours so that we can run two nightly during the ci
int total_timeout_minutes = 4 * 60 int total_timeout_minutes = 4 * 60
def imageTag='' def imageTag=''
def chart_version='3.0.19' def chart_version='3.0.20'
pipeline { pipeline {
triggers { triggers {
cron """${cron_timezone} cron """${cron_timezone}
@ -86,7 +86,7 @@ pipeline {
axes { axes {
axis { axis {
name 'MILVUS_SERVER_TYPE' name 'MILVUS_SERVER_TYPE'
values 'standalone', 'distributed' values 'standalone', 'distributed-pulsar', 'distributed-kafka'
} }
axis { axis {
name 'MILVUS_CLIENT' name 'MILVUS_CLIENT'
@ -103,9 +103,12 @@ pipeline {
sh 'printenv' sh 'printenv'
def clusterEnabled = "false" def clusterEnabled = "false"
// def setMemoryResourceLimitArgs="--set standalone.resources.limits.memory=4Gi" // def setMemoryResourceLimitArgs="--set standalone.resources.limits.memory=4Gi"
if ("${MILVUS_SERVER_TYPE}" == "distributed") { def mqMode='pulsar' // default using is pulsar
if ("${MILVUS_SERVER_TYPE}" == "distributed-pulsar") {
clusterEnabled = "true" clusterEnabled = "true"
// setMemoryResourceLimitArgs="--set queryNode.resources.limits.memory=4Gi" } else if ("${MILVUS_SERVER_TYPE}" == "distributed-kafka") {
clusterEnabled = "true"
mqMode='kafka'
} }
if ("${MILVUS_CLIENT}" == "pymilvus") { if ("${MILVUS_CLIENT}" == "pymilvus") {
if ("${imageTag}"==''){ if ("${imageTag}"==''){
@ -140,6 +143,7 @@ pipeline {
--set indexNode.replicas=2 \ --set indexNode.replicas=2 \
--set dataNode.replicas=2 \ --set dataNode.replicas=2 \
--version ${chart_version} \ --version ${chart_version} \
-f values/${mqMode}.yaml \
-f values/nightly.yaml " -f values/nightly.yaml "
""" """
} }
@ -167,10 +171,15 @@ pipeline {
script { script {
def release_name=sh(returnStdout: true, script: './get_release_name.sh') def release_name=sh(returnStdout: true, script: './get_release_name.sh')
def clusterEnabled = "false" def clusterEnabled = "false"
def mqMode='pulsar'
int e2e_timeout_seconds = 6 * 60 * 60 int e2e_timeout_seconds = 6 * 60 * 60
if ("${MILVUS_SERVER_TYPE}" == "distributed") { if ("${MILVUS_SERVER_TYPE}" == "distributed-pulsar") {
clusterEnabled = "true" clusterEnabled = "true"
e2e_timeout_seconds = 10 * 60 * 60 e2e_timeout_seconds = 10 * 60 * 60
} else if("${MILVUS_SERVER_TYPE}" == "distributed-kafka" ) {
clusterEnabled = "true"
mqMode='kafka'
e2e_timeout_seconds = 10 * 60 * 60
} }
if ("${MILVUS_CLIENT}" == "pymilvus") { if ("${MILVUS_CLIENT}" == "pymilvus") {
sh """ sh """
@ -178,7 +187,8 @@ pipeline {
MILVUS_HELM_NAMESPACE="milvus-ci" \ MILVUS_HELM_NAMESPACE="milvus-ci" \
MILVUS_CLUSTER_ENABLED="${clusterEnabled}" \ MILVUS_CLUSTER_ENABLED="${clusterEnabled}" \
TEST_TIMEOUT="${e2e_timeout_seconds}" \ TEST_TIMEOUT="${e2e_timeout_seconds}" \
./ci_e2e.sh "-n 6 --tags L0 L1 L2 " MQ_MODE="${mqMode}" \
./ci_e2e.sh "-n 6 --tags L0 L1 L2"
""" """
} else { } else {
error "Error: Unsupported Milvus client: ${MILVUS_CLIENT}" error "Error: Unsupported Milvus client: ${MILVUS_CLIENT}"

10
tests/scripts/get_release_name.sh
View File

@ -25,10 +25,16 @@ set -x
function milvus_ci_release_name(){ function milvus_ci_release_name(){
# Rules for helm release name # Rules for helm release name
local name="m" local name="m"
if [[ "${MILVUS_SERVER_TYPE:-}" == "distributed" ]]; then if [[ "${MILVUS_SERVER_TYPE:-}" == "distributed-pulsar" ]]; then
# Distributed pulsar mode
name+="dp"
elif [[ "${MILVUS_SERVER_TYPE:-}" == "distributed-kafka" ]]; then
# Distributed kafka mode
name+="dk"
elif [[ "${MILVUS_SERVER_TYPE:-}" == "distributed" ]]; then
# Distributed mode # Distributed mode
name+="d" name+="d"
else else
# Standalone mode # Standalone mode
name+="s" name+="s"

4
tests/scripts/values/kafka.yaml Normal file
View File

@ -0,0 +1,4 @@
kafka:
enabled: true
pulsar:
enabled: false

4
tests/scripts/values/pulsar.yaml Normal file
View File

@ -0,0 +1,4 @@
kafka:
enabled: false
pulsar:
enabled: true