mirror of https://github.com/milvus-io/milvus.git
Chanege cluster for e2e to 4am (#26253)
Signed-off-by: Bennu-Li <yunmei.li@zilliz.com>pull/26335/head
Bennu
GitHub
parent
4c4b2903f9
commit
ae780e0313
|
|
@ -17,7 +17,8 @@ pipeline {
|
|||
}
|
||||
agent {
|
||||
kubernetes {
|
||||
inheritFrom 'milvus-e2e'
|
||||
cloud '4am'
|
||||
inheritFrom 'milvus-e2e-4am'
|
||||
defaultContainer 'main'
|
||||
yamlFile 'ci/jenkins/pod/rte-build.yaml'
|
||||
customWorkspace '/home/jenkins/agent/workspace'
|
||||
|
|
@ -42,7 +43,9 @@ pipeline {
|
|||
steps {
|
||||
container('main') {
|
||||
dir ('build'){
|
||||
sh './set_docker_mirror.sh'
|
||||
sh """
|
||||
MIRROR_URL="https://docker-nexus-ci.zilliz.cc" ./set_docker_mirror.sh
|
||||
"""
|
||||
}
|
||||
dir ('tests/scripts') {
|
||||
script {
|
||||
|
|
@ -96,7 +99,7 @@ pipeline {
|
|||
script {
|
||||
sh 'printenv'
|
||||
def clusterEnabled = "false"
|
||||
def valuesFile = "pr.yaml"
|
||||
def valuesFile = "pr-4am.yaml"
|
||||
if ("${MILVUS_SERVER_TYPE}".contains('distributed')) {
|
||||
clusterEnabled = "true"
|
||||
}
|
||||
|
|
@ -125,7 +128,7 @@ pipeline {
|
|||
chmod +x /usr/bin/yq
|
||||
'''
|
||||
sh """
|
||||
cp values/ci/pr.yaml values/ci/pr_kafka.yaml
|
||||
cp values/ci/pr-4am.yaml values/ci/pr_kafka.yaml
|
||||
yq -i '.pulsar.enabled=false' values/ci/pr_kafka.yaml
|
||||
yq -i '.kafka.enabled=true' values/ci/pr_kafka.yaml
|
||||
yq -i '.kafka.metrics.kafka.enabled=true' values/ci/pr_kafka.yaml
|
||||
|
|
@ -136,7 +139,7 @@ pipeline {
|
|||
withCredentials([usernamePassword(credentialsId: "${env.CI_DOCKER_CREDENTIAL_ID}", usernameVariable: 'CI_REGISTRY_USERNAME', passwordVariable: 'CI_REGISTRY_PASSWORD')]){
|
||||
sh """
|
||||
MILVUS_CLUSTER_ENABLED=${clusterEnabled} \
|
||||
MILVUS_HELM_REPO="http://nexus-nexus-repository-manager.nexus:8081/repository/milvus-proxy" \
|
||||
MILVUS_HELM_REPO="https://nexus-ci.zilliz.cc/repository/milvus-proxy" \
|
||||
TAG=${imageTag}\
|
||||
./e2e-k8s.sh \
|
||||
--skip-export-logs \
|
||||
|
|
@ -171,6 +174,7 @@ pipeline {
|
|||
}
|
||||
agent {
|
||||
kubernetes {
|
||||
cloud '4am'
|
||||
inheritFrom 'default'
|
||||
defaultContainer 'main'
|
||||
yamlFile 'ci/jenkins/pod/e2e.yaml'
|
||||
|
|
@ -196,7 +200,7 @@ pipeline {
|
|||
MILVUS_HELM_NAMESPACE="milvus-ci" \
|
||||
MILVUS_CLUSTER_ENABLED="${clusterEnabled}" \
|
||||
TEST_TIMEOUT="${e2e_timeout_seconds}" \
|
||||
./ci_e2e.sh "-n 6 -x --tags L0 L1 --timeout ${case_timeout_seconds}"
|
||||
./ci_e2e_4am.sh "-n 6 -x --tags L0 L1 --timeout ${case_timeout_seconds}"
|
||||
"""
|
||||
|
||||
} else {
|
||||
|
|
|
|||
|
|
@ -23,5 +23,6 @@ spec:
|
|||
volumes:
|
||||
- name: ci-logs
|
||||
nfs:
|
||||
path: /ci-logs
|
||||
server: 172.16.70.239
|
||||
path: /volume1/ci-logs
|
||||
# path: /volume1/4am-logs
|
||||
server: 172.16.70.249
|
||||
|
|
@ -64,5 +64,6 @@ spec:
|
|||
name: cgroup
|
||||
- name: ci-logs
|
||||
nfs:
|
||||
path: /ci-logs
|
||||
server: 172.16.70.239
|
||||
path: /volume1/ci-logs
|
||||
# path: /volume1/4am-logs
|
||||
server: 172.16.70.249
|
||||
|
|
|
|||
|
|
@ -0,0 +1,60 @@
|
|||
#!/bin/bash
|
||||
|
||||
# Licensed to the LF AI & Data foundation under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# Exit immediately for non zero status
|
||||
set -e
|
||||
# Check unset variables
|
||||
set -u
|
||||
# Print commands
|
||||
set -x
|
||||
|
||||
SOURCE="${BASH_SOURCE[0]}"
|
||||
while [ -h "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a symlink
|
||||
DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )"
|
||||
SOURCE="$(readlink "$SOURCE")"
|
||||
[[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE" # if $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located
|
||||
done
|
||||
ROOT="$( cd -P "$( dirname "$SOURCE" )/../.." && pwd )"
|
||||
|
||||
|
||||
# Install pytest requirements
|
||||
function install_pytest_requirements(){
|
||||
echo "Install pytest requirements"
|
||||
cd ${ROOT}/tests/python_client
|
||||
|
||||
# MIRROR_HOST="${MIRROR_HOST:-nexus-nexus-repository-manager.nexus}"
|
||||
|
||||
|
||||
export PIP_TRUSTED_HOST="nexus-ci.zilliz.cc"
|
||||
export PIP_INDEX_URL="https://nexus-ci.zilliz.cc/repository/pypi-all/simple"
|
||||
export PIP_INDEX="https://nexus-ci.zilliz.cc/repository/pypi-all/pypi"
|
||||
export PIP_FIND_LINKS="https://nexus-ci.zilliz.cc/repository/pypi-all/pypi"
|
||||
python3 -m pip install --no-cache-dir -r requirements.txt --timeout 30 --retries 6
|
||||
}
|
||||
|
||||
# Login in ci docker registry
|
||||
function docker_login_ci_registry(){
|
||||
|
||||
if [[ -z "${CI_REGISTRY_USERNAME:-}" || -z "${CI_REGISTRY_PASSWORD:-}" ]]; then
|
||||
echo "Please setup docker credential for ci registry-${HUB}"
|
||||
else
|
||||
echo "docker login ci registry"
|
||||
docker login -u ${CI_REGISTRY_USERNAME} -p ${CI_REGISTRY_PASSWORD} ${HUB}
|
||||
fi
|
||||
}
|
||||
|
||||
|
|
@ -0,0 +1,86 @@
|
|||
#!/bin/bash
|
||||
|
||||
# Licensed to the LF AI & Data foundation under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
SOURCE="${BASH_SOURCE[0]}"
|
||||
while [ -h "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a symlink
|
||||
DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )"
|
||||
SOURCE="$(readlink "$SOURCE")"
|
||||
[[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE" # if $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located
|
||||
done
|
||||
ROOT="$( cd -P "$( dirname "$SOURCE" )/../.." && pwd )"
|
||||
|
||||
# Exit immediately for non zero status
|
||||
set -e
|
||||
# Check unset variables
|
||||
set -u
|
||||
# Print commands
|
||||
set -x
|
||||
|
||||
|
||||
|
||||
MILVUS_HELM_RELEASE_NAME="${MILVUS_HELM_RELEASE_NAME:-milvus-testing}"
|
||||
MILVUS_CLUSTER_ENABLED="${MILVUS_CLUSTER_ENABLED:-false}"
|
||||
MILVUS_HELM_NAMESPACE="${MILVUS_HELM_NAMESPACE:-default}"
|
||||
PARALLEL_NUM="${PARALLEL_NUM:-6}"
|
||||
# Use service name instead of IP to test
|
||||
MILVUS_SERVICE_NAME=$(echo "${MILVUS_HELM_RELEASE_NAME}-milvus.${MILVUS_HELM_NAMESPACE}" | tr -d '\n')
|
||||
MILVUS_SERVICE_PORT="19530"
|
||||
# Minio service name
|
||||
MINIO_SERVICE_NAME=$(echo "${MILVUS_HELM_RELEASE_NAME}-minio.${MILVUS_HELM_NAMESPACE}" | tr -d '\n')
|
||||
|
||||
|
||||
# Shellcheck source=ci-util.sh
|
||||
source "${ROOT}/tests/scripts/ci-util-4am.sh"
|
||||
|
||||
|
||||
cd ${ROOT}/tests/python_client
|
||||
|
||||
# Print python3 version, python version 3.6.8 is more stable for test
|
||||
python3 -V
|
||||
|
||||
# Pytest will try to get ${CI_LOG_PATH} from environment variables first,then use default path
|
||||
export CI_LOG_PATH=/tmp/ci_logs/test
|
||||
|
||||
if [ ! -d "${CI_LOG_PATH}" ]; then
|
||||
# Create dir for ci log path when it does not exist
|
||||
mkdir -p ${CI_LOG_PATH}
|
||||
fi
|
||||
|
||||
echo "prepare e2e test"
|
||||
install_pytest_requirements
|
||||
|
||||
|
||||
# Pytest is not able to have both --timeout & --workers, so do not add --timeout or --workers in the shell script
|
||||
if [[ -n "${TEST_TIMEOUT:-}" ]]; then
|
||||
|
||||
timeout "${TEST_TIMEOUT}" pytest --host ${MILVUS_SERVICE_NAME} --port ${MILVUS_SERVICE_PORT} \
|
||||
--html=${CI_LOG_PATH}/report.html --self-contained-html ${@:-}
|
||||
else
|
||||
pytest --host ${MILVUS_SERVICE_NAME} --port ${MILVUS_SERVICE_PORT} \
|
||||
--html=${CI_LOG_PATH}/report.html --self-contained-html ${@:-}
|
||||
fi
|
||||
|
||||
# Run bulk insert test
|
||||
if [[ -n "${TEST_TIMEOUT:-}" ]]; then
|
||||
|
||||
timeout "${TEST_TIMEOUT}" pytest testcases/test_bulk_insert.py --host ${MILVUS_SERVICE_NAME} --port ${MILVUS_SERVICE_PORT} --minio_host ${MINIO_SERVICE_NAME} \
|
||||
--html=${CI_LOG_PATH}/report_bulk_insert.html --self-contained-html
|
||||
else
|
||||
pytest testcases/test_bulk_insert.py --host ${MILVUS_SERVICE_NAME} --port ${MILVUS_SERVICE_PORT} --minio_host ${MINIO_SERVICE_NAME} \
|
||||
--html=${CI_LOG_PATH}/report_bulk_insert.html --self-contained-html
|
||||
fi
|
||||
|
|
@ -87,14 +87,19 @@ if [[ ! -d ${RELEASE_LOG_DIR} ]] ;then
|
|||
fi
|
||||
# Try to found logs file from mount disk /volume1/ci-logs
|
||||
log_files=$(find ${LOG_DIR} -type f -name "*${RELEASE_NAME}*" )
|
||||
for log_file in ${log_files}
|
||||
do
|
||||
file_name=$(basename ${log_file})
|
||||
mv ${log_file} ${RELEASE_LOG_DIR}/`echo ${file_name} | sed 's/jenkins.var.log.containers.//g' `
|
||||
done
|
||||
|
||||
tar -zcvf ${ARTIFACTS_NAME:-artifacts}.tar.gz ${RELEASE_LOG_DIR}/*
|
||||
rm -rf ${RELEASE_LOG_DIR}
|
||||
if [ -z "${log_files:-}" ]; then
|
||||
echo "No log files find"
|
||||
else
|
||||
for log_file in ${log_files}
|
||||
do
|
||||
file_name=$(basename ${log_file})
|
||||
mv ${log_file} ${RELEASE_LOG_DIR}/`echo ${file_name} | sed 's/jenkins.var.log.containers.//g' `
|
||||
done
|
||||
|
||||
tar -zcvf ${ARTIFACTS_NAME:-artifacts}.tar.gz ${RELEASE_LOG_DIR}/*
|
||||
rm -rf ${RELEASE_LOG_DIR}
|
||||
fi
|
||||
|
||||
remain_log_files=$(find ${LOG_DIR} -type f -name "*${RELEASE_NAME}*")
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,186 @@
|
|||
metrics:
|
||||
serviceMonitor:
|
||||
enabled: true
|
||||
log:
|
||||
level: debug
|
||||
proxy:
|
||||
resources:
|
||||
requests:
|
||||
cpu: "0.3"
|
||||
memory: "256Mi"
|
||||
limits:
|
||||
cpu: "1"
|
||||
rootCoordinator:
|
||||
resources:
|
||||
requests:
|
||||
cpu: "0.2"
|
||||
memory: "256Mi"
|
||||
limits:
|
||||
cpu: "1"
|
||||
queryCoordinator:
|
||||
resources:
|
||||
requests:
|
||||
cpu: "0.2"
|
||||
memory: "100Mi"
|
||||
limits:
|
||||
cpu: "1"
|
||||
queryNode:
|
||||
resources:
|
||||
requests:
|
||||
cpu: "0.5"
|
||||
memory: "500Mi"
|
||||
limits:
|
||||
cpu: "2"
|
||||
indexCoordinator:
|
||||
resources:
|
||||
requests:
|
||||
cpu: "0.1"
|
||||
memory: "50Mi"
|
||||
limits:
|
||||
cpu: "1"
|
||||
indexNode:
|
||||
resources:
|
||||
requests:
|
||||
cpu: "0.5"
|
||||
memory: "500Mi"
|
||||
limits:
|
||||
cpu: "2"
|
||||
dataCoordinator:
|
||||
resources:
|
||||
requests:
|
||||
cpu: "0.1"
|
||||
memory: "50Mi"
|
||||
limits:
|
||||
cpu: "1"
|
||||
dataNode:
|
||||
resources:
|
||||
requests:
|
||||
cpu: "0.5"
|
||||
memory: "500Mi"
|
||||
limits:
|
||||
cpu: "2"
|
||||
|
||||
pulsar:
|
||||
components:
|
||||
autorecovery: false
|
||||
proxy:
|
||||
configData:
|
||||
PULSAR_MEM: >
|
||||
-Xms1024m -Xmx1024m
|
||||
PULSAR_GC: >
|
||||
-XX:MaxDirectMemorySize=2048m
|
||||
httpNumThreads: "50"
|
||||
resources:
|
||||
requests:
|
||||
cpu: "0.5"
|
||||
memory: "1Gi"
|
||||
# Resources for the websocket proxy
|
||||
wsResources:
|
||||
requests:
|
||||
memory: "100Mi"
|
||||
cpu: "0.1"
|
||||
broker:
|
||||
resources:
|
||||
requests:
|
||||
cpu: "0.5"
|
||||
memory: "4Gi"
|
||||
configData:
|
||||
PULSAR_MEM: >
|
||||
-Xms4096m
|
||||
-Xmx4096m
|
||||
-XX:MaxDirectMemorySize=8192m
|
||||
PULSAR_GC: >
|
||||
-Dio.netty.leakDetectionLevel=disabled
|
||||
-Dio.netty.recycler.linkCapacity=1024
|
||||
-XX:+ParallelRefProcEnabled
|
||||
-XX:+UnlockExperimentalVMOptions
|
||||
-XX:+DoEscapeAnalysis
|
||||
-XX:ParallelGCThreads=32
|
||||
-XX:ConcGCThreads=32
|
||||
-XX:G1NewSizePercent=50
|
||||
-XX:+DisableExplicitGC
|
||||
-XX:-ResizePLAB
|
||||
-XX:+ExitOnOutOfMemoryError
|
||||
maxMessageSize: "104857600"
|
||||
defaultRetentionTimeInMinutes: "10080"
|
||||
defaultRetentionSizeInMB: "8192"
|
||||
backlogQuotaDefaultLimitGB: "8"
|
||||
backlogQuotaDefaultRetentionPolicy: producer_exception
|
||||
|
||||
bookkeeper:
|
||||
configData:
|
||||
PULSAR_MEM: >
|
||||
-Xms4096m
|
||||
-Xmx4096m
|
||||
-XX:MaxDirectMemorySize=8192m
|
||||
PULSAR_GC: >
|
||||
-Dio.netty.leakDetectionLevel=disabled
|
||||
-Dio.netty.recycler.linkCapacity=1024
|
||||
-XX:+UseG1GC -XX:MaxGCPauseMillis=10
|
||||
-XX:+ParallelRefProcEnabled
|
||||
-XX:+UnlockExperimentalVMOptions
|
||||
-XX:+DoEscapeAnalysis
|
||||
-XX:ParallelGCThreads=32
|
||||
-XX:ConcGCThreads=32
|
||||
-XX:G1NewSizePercent=50
|
||||
-XX:+DisableExplicitGC
|
||||
-XX:-ResizePLAB
|
||||
-XX:+ExitOnOutOfMemoryError
|
||||
-XX:+PerfDisableSharedMem
|
||||
-XX:+PrintGCDetails
|
||||
nettyMaxFrameSizeBytes: "104867840"
|
||||
resources:
|
||||
requests:
|
||||
cpu: "0.5"
|
||||
memory: "4Gi"
|
||||
|
||||
zookeeper:
|
||||
replicaCount: 1
|
||||
configData:
|
||||
PULSAR_MEM: >
|
||||
-Xms1024m
|
||||
-Xmx1024m
|
||||
PULSAR_GC: >
|
||||
-Dcom.sun.management.jmxremote
|
||||
-Djute.maxbuffer=10485760
|
||||
-XX:+ParallelRefProcEnabled
|
||||
-XX:+UnlockExperimentalVMOptions
|
||||
-XX:+DoEscapeAnalysis
|
||||
-XX:+DisableExplicitGC
|
||||
-XX:+PerfDisableSharedMem
|
||||
-Dzookeeper.forceSync=no
|
||||
resources:
|
||||
requests:
|
||||
cpu: "0.3"
|
||||
memory: "512Mi"
|
||||
kafka:
|
||||
resources:
|
||||
requests:
|
||||
cpu: "0.5"
|
||||
memory: "1Gi"
|
||||
zookeeper:
|
||||
replicaCount: 1
|
||||
resources:
|
||||
requests:
|
||||
cpu: "0.3"
|
||||
memory: "512Mi"
|
||||
etcd:
|
||||
nodeSelector:
|
||||
nvme: "true"
|
||||
replicaCount: 1
|
||||
resources:
|
||||
requests:
|
||||
cpu: "0.3"
|
||||
memory: "100Mi"
|
||||
minio:
|
||||
resources:
|
||||
requests:
|
||||
cpu: "0.3"
|
||||
memory: "512Mi"
|
||||
standalone:
|
||||
resources:
|
||||
requests:
|
||||
cpu: "1"
|
||||
memory: "3.5Gi"
|
||||
limits:
|
||||
cpu: "4"
|
||||
Loading…
Reference in New Issue