From 136cd7a70416e0e5e6cdff6291025c8900b186e3 Mon Sep 17 00:00:00 2001 From: quicksilver Date: Thu, 29 Apr 2021 16:45:17 +0800 Subject: [PATCH] Run KinD Inside A Kubernetes Cluster For Continuous Integration (#5085) **What type of PR is this?** - [x] Improvement **Which issue(s) this PR fixes:** Fixes #5038 **What this PR does / why we need it:** Run KinD Inside A Kubernetes Cluster For Continuous Integration --- .jenkins/modules/Build/Build.groovy | 20 --- .jenkins/modules/Package/Package.groovy | 7 - .jenkins/modules/Publish/Publish.groovy | 25 ---- .../Regression/CleanupPythonRegression.groovy | 8 -- .../Regression/PythonRegression.groovy | 88 ------------- .jenkins/modules/UnitTest/UnitTest.groovy | 3 - build/ci/jenkins/Jenkinsfile | 121 +++++++----------- build/ci/jenkins/pod/build-env.yaml | 43 ------- build/ci/jenkins/pod/docker-pod.yaml | 34 ----- build/ci/jenkins/pod/krte.yaml | 66 ++++++++++ build/ci/jenkins/pod/test-env.yaml | 38 ------ build/config/topology/trustworthy-jwt.yaml | 18 +++ cmd/standalone/main.go | 14 +- scripts/before-install.sh | 12 -- scripts/check_cache.sh | 114 ----------------- scripts/update_cache.sh | 104 --------------- tests/scripts/e2e-k8s.sh | 1 - 17 files changed, 137 insertions(+), 579 deletions(-) delete mode 100644 .jenkins/modules/Build/Build.groovy delete mode 100644 .jenkins/modules/Package/Package.groovy delete mode 100644 .jenkins/modules/Publish/Publish.groovy delete mode 100644 .jenkins/modules/Regression/CleanupPythonRegression.groovy delete mode 100644 .jenkins/modules/Regression/PythonRegression.groovy delete mode 100644 .jenkins/modules/UnitTest/UnitTest.groovy delete mode 100644 build/ci/jenkins/pod/build-env.yaml delete mode 100644 build/ci/jenkins/pod/docker-pod.yaml create mode 100644 build/ci/jenkins/pod/krte.yaml delete mode 100644 build/ci/jenkins/pod/test-env.yaml delete mode 100755 scripts/before-install.sh delete mode 100755 scripts/check_cache.sh delete mode 100755 scripts/update_cache.sh diff --git a/.jenkins/modules/Build/Build.groovy b/.jenkins/modules/Build/Build.groovy deleted file mode 100644 index f106ec864c..0000000000 --- a/.jenkins/modules/Build/Build.groovy +++ /dev/null @@ -1,20 +0,0 @@ -timeout(time: 20, unit: 'MINUTES') { - - sh '. ./scripts/before-install.sh && ./scripts/check_cache.sh -l $CCACHE_ARTFACTORY_URL --cache_dir=\$CCACHE_DIR -f ccache-\$OS_NAME-\$BUILD_ENV_IMAGE_ID.tar.gz || echo \"Ccache artfactory files not found!\"' - sh '. ./scripts/before-install.sh && ./scripts/check_cache.sh -l $GO_CACHE_ARTFACTORY_URL --cache_dir=\$(go env GOCACHE) -f go-cache-\$OS_NAME-\$BUILD_ENV_IMAGE_ID.tar.gz || echo \"Go cache artfactory files not found!\"' - sh '. ./scripts/before-install.sh && ./scripts/check_cache.sh -l $THIRDPARTY_ARTFACTORY_URL --cache_dir=$CUSTOM_THIRDPARTY_PATH -f thirdparty-download.tar.gz || echo \"Thirdparty artfactory files not found!\"' - sh '. ./scripts/before-install.sh && go clean --modcache && ./scripts/check_cache.sh -l $GO_MOD_ARTFACTORY_URL --cache_dir=\$GOPATH/pkg/mod -f milvus-go-mod-\$(md5sum go.mod).tar.gz || echo \"Go mod artfactory files not found!\"' - - // Zero the cache statistics (but not the configuration options) - sh 'ccache -z' - sh '. ./scripts/before-install.sh && make install' - sh 'echo -e "===\n=== ccache statistics after build\n===" && ccache --show-stats' - - - withCredentials([usernamePassword(credentialsId: "${env.JFROG_CREDENTIALS_ID}", usernameVariable: 'USERNAME', passwordVariable: 'PASSWORD')]) { - sh '. ./scripts/before-install.sh && ./scripts/update_cache.sh -l $CCACHE_ARTFACTORY_URL --cache_dir=\$CCACHE_DIR -f ccache-\$OS_NAME-\$BUILD_ENV_IMAGE_ID.tar.gz -u ${USERNAME} -p ${PASSWORD}' - sh '. ./scripts/before-install.sh && ./scripts/update_cache.sh -l $GO_CACHE_ARTFACTORY_URL --cache_dir=\$(go env GOCACHE) -f go-cache-\$OS_NAME-\$BUILD_ENV_IMAGE_ID.tar.gz -u ${USERNAME} -p ${PASSWORD}' - sh '. ./scripts/before-install.sh && ./scripts/update_cache.sh -l $THIRDPARTY_ARTFACTORY_URL --cache_dir=$CUSTOM_THIRDPARTY_PATH -f thirdparty-download.tar.gz -u ${USERNAME} -p ${PASSWORD}' - sh '. ./scripts/before-install.sh && ./scripts/update_cache.sh -l $GO_MOD_ARTFACTORY_URL --cache_dir=\$GOPATH/pkg/mod -f milvus-go-mod-\$(md5sum go.mod).tar.gz -u ${USERNAME} -p ${PASSWORD}' - } -} diff --git a/.jenkins/modules/Package/Package.groovy b/.jenkins/modules/Package/Package.groovy deleted file mode 100644 index 256b38fec9..0000000000 --- a/.jenkins/modules/Package/Package.groovy +++ /dev/null @@ -1,7 +0,0 @@ -sh 'tar -zcvf ./${PACKAGE_NAME} ./bin ./configs ./lib' -withCredentials([usernamePassword(credentialsId: "${env.JFROG_CREDENTIALS_ID}", usernameVariable: 'JFROG_USERNAME', passwordVariable: 'JFROG_PASSWORD')]) { - def uploadStatus = sh(returnStatus: true, script: 'curl -u${JFROG_USERNAME}:${JFROG_PASSWORD} -T ./${PACKAGE_NAME} ${PACKAGE_ARTFACTORY_URL}') - if (uploadStatus != 0) { - error("\" ${PACKAGE_NAME} \" upload to \" ${PACKAGE_ARTFACTORY_URL} \" failed!") - } -} diff --git a/.jenkins/modules/Publish/Publish.groovy b/.jenkins/modules/Publish/Publish.groovy deleted file mode 100644 index cba5be12d4..0000000000 --- a/.jenkins/modules/Publish/Publish.groovy +++ /dev/null @@ -1,25 +0,0 @@ -withCredentials([usernamePassword(credentialsId: "${env.JFROG_CREDENTIALS_ID}", usernameVariable: 'JFROG_USERNAME', passwordVariable: 'JFROG_PASSWORD')]) { - def downloadStatus = sh(returnStatus: true, script: 'curl -u${JFROG_USERNAME}:${JFROG_PASSWORD} -O ${PACKAGE_ARTFACTORY_URL}') - - if (downloadStatus != 0) { - error("\" Download \" ${PACKAGE_ARTFACTORY_URL} \" failed!") - } -} - -sh 'tar zxvf ${PACKAGE_NAME}' - -try { - withCredentials([usernamePassword(credentialsId: "${env.DOCKER_CREDENTIALS_ID}", usernameVariable: 'DOCKER_USERNAME', passwordVariable: 'DOCKER_PASSWORD')]) { - sh 'docker login -u ${DOCKER_USERNAME} -p ${DOCKER_PASSWORD} ${DOKCER_REGISTRY_URL}' - - sh 'docker pull registry.zilliz.com/milvus/openblas:latest || true' - sh "docker build -f build/docker/milvus/Dockerfile -t ${TARGET_REPO}/milvus:${TARGET_TAG} ." - sh "docker push ${TARGET_REPO}/milvus:${TARGET_TAG}" - } -} catch (exc) { - throw exc -} finally { - sh 'docker logout ${DOKCER_REGISTRY_URL}' - sh "docker rmi -f \$(docker images | grep '' | awk '{print \$3}') || true" - sh "docker rmi ${TARGET_REPO}/milvus:${TARGET_TAG}" -} diff --git a/.jenkins/modules/Regression/CleanupPythonRegression.groovy b/.jenkins/modules/Regression/CleanupPythonRegression.groovy deleted file mode 100644 index d454c99664..0000000000 --- a/.jenkins/modules/Regression/CleanupPythonRegression.groovy +++ /dev/null @@ -1,8 +0,0 @@ -container('deploy-env') { - def helmStatus = sh script: "helm status -n ${env.HELM_RELEASE_NAMESPACE} ${env.HELM_RELEASE_NAME}", returnStatus: true - if (!helmStatus) { - sh "helm uninstall -n ${env.HELM_RELEASE_NAMESPACE} ${env.HELM_RELEASE_NAME}" - def labels = "app.kubernetes.io/instance=${env.HELM_RELEASE_NAME}" - sh "kubectl delete pvc -n ${env.HELM_RELEASE_NAMESPACE} \$(kubectl get pvc -n ${env.HELM_RELEASE_NAMESPACE} -l ${labels} -o jsonpath='{range.items[*]}{.metadata.name} ')" - } -} diff --git a/.jenkins/modules/Regression/PythonRegression.groovy b/.jenkins/modules/Regression/PythonRegression.groovy deleted file mode 100644 index 1c400d9c78..0000000000 --- a/.jenkins/modules/Regression/PythonRegression.groovy +++ /dev/null @@ -1,88 +0,0 @@ -def isTimeTriggeredBuild = currentBuild.getBuildCauses('hudson.triggers.TimerTrigger$TimerTriggerCause').size() != 0 -def regressionTimeout = isTimeTriggeredBuild ? "120" : "60" -timeout(time: "${regressionTimeout}", unit: 'MINUTES') { - container('deploy-env') { - dir ('milvus-helm-chart') { - sh " helm version && \ - helm repo add stable https://kubernetes.oss-cn-hangzhou.aliyuncs.com/charts && \ - helm repo add bitnami https://charts.bitnami.com/bitnami && \ - helm repo add minio https://helm.min.io/ && \ - helm repo update" - - def milvusHelmURL = "https://github.com/zilliztech/milvus-helm-charts.git" - checkout([$class: 'GitSCM', branches: [[name: "${env.HELM_BRANCH}"]], userRemoteConfigs: [[url: "${milvusHelmURL}"]]]) - - dir ('charts/milvus-ha') { - sh script: "kubectl create namespace ${env.HELM_RELEASE_NAMESPACE}", returnStatus: true - - def helmCMD = "" - if ("${REGRESSION_SERVICE_TYPE}" == "distributed") { - helmCMD = "helm install --wait --timeout 300s \ - --set standalone.enabled=false \ - --set image.all.repository=${env.TARGET_REPO}/milvus \ - --set image.all.tag=${env.TARGET_TAG} \ - --set image.all.pullPolicy=Always \ - --set logsPersistence.enabled=true \ - --set logsPersistence.mountPath=/milvus/logs \ - --namespace ${env.HELM_RELEASE_NAMESPACE} ${env.HELM_RELEASE_NAME} ." - } else { - helmCMD = "helm install --wait --timeout 300s \ - --set standalone.enabled=true \ - --set image.all.repository=${env.TARGET_REPO}/milvus \ - --set image.all.tag=${env.TARGET_TAG} \ - --set image.all.pullPolicy=Always \ - --set logsPersistence.enabled=true \ - --set logsPersistence.mountPath=/milvus/logs \ - --namespace ${env.HELM_RELEASE_NAMESPACE} ${env.HELM_RELEASE_NAME} ." - } - - try { - sh "${helmCMD}" - } catch (exc) { - def helmStatusCMD = "helm get manifest -n ${env.HELM_RELEASE_NAMESPACE} ${env.HELM_RELEASE_NAME} | kubectl describe -n ${env.HELM_RELEASE_NAMESPACE} -f - && \ - helm status -n ${env.HELM_RELEASE_NAMESPACE} ${env.HELM_RELEASE_NAME}" - sh script: helmStatusCMD, returnStatus: true - throw exc - } - } - } - } - - container('test-env') { - try { - dir ('tests/python_test') { - sh "python3 -m pip install --no-cache-dir -r requirements.txt" - if (isTimeTriggeredBuild) { - echo "This is Cron Job!" - sh "pytest -n 4 --ip ${env.HELM_RELEASE_NAME}-milvus-ha.${env.HELM_RELEASE_NAMESPACE}.svc.cluster.local" - } else { - sh "pytest --tags=smoke -n 4 --ip ${env.HELM_RELEASE_NAME}-milvus-ha.${env.HELM_RELEASE_NAMESPACE}.svc.cluster.local" - } - } - } catch (exc) { - echo 'PyTest Regression Failed !' - throw exc - } finally { - container('deploy-env') { - def milvusLabels = "" - if ("${REGRESSION_SERVICE_TYPE}" == "distributed") { - milvusLabels = "app.kubernetes.io/instance=${env.HELM_RELEASE_NAME},component=proxyservice" - } else { - milvusLabels = "app.kubernetes.io/instance=${env.HELM_RELEASE_NAME},component=standalone" - } - def etcdLabels = "app.kubernetes.io/instance=${env.HELM_RELEASE_NAME},app.kubernetes.io/name=etcd" - def pulsarLabels = "app.kubernetes.io/instance=${env.HELM_RELEASE_NAME},component=pulsar" - def componentLabels = "release=${env.HELM_RELEASE_NAME}" - def namespace = "${env.HELM_RELEASE_NAMESPACE}" - def artifactsPath = "${env.DEV_TEST_ARTIFACTS_PATH}" - - sh "mkdir -p $artifactsPath" - sh "kubectl cp -n ${env.HELM_RELEASE_NAMESPACE} \$(kubectl get pod -n ${env.HELM_RELEASE_NAMESPACE} -l ${milvusLabels} -o jsonpath='{range.items[0]}{.metadata.name}'):logs $artifactsPath" - sh "for pod in \$(kubectl get pod -n $namespace -l ${etcdLabels} -o jsonpath='{range.items[*]}{.metadata.name} '); do kubectl logs --all-containers -n $namespace \$pod > $artifactsPath/\$pod.log; done" - sh "for pod in \$(kubectl get pod -n $namespace -l ${pulsarLabels} -o jsonpath='{range.items[*]}{.metadata.name} '); do kubectl logs --all-containers -n $namespace \$pod > $artifactsPath/\$pod.log; done" - sh "for pod in \$(kubectl get pod -n $namespace -l ${componentLabels} -o jsonpath='{range.items[*]}{.metadata.name} '); do kubectl logs --all-containers -n $namespace \$pod > $artifactsPath/\$pod.log; done" - archiveArtifacts artifacts: "$artifactsPath/**", allowEmptyArchive: true - } - } - } -} diff --git a/.jenkins/modules/UnitTest/UnitTest.groovy b/.jenkins/modules/UnitTest/UnitTest.groovy deleted file mode 100644 index f78834e99c..0000000000 --- a/.jenkins/modules/UnitTest/UnitTest.groovy +++ /dev/null @@ -1,3 +0,0 @@ -timeout(time: 5, unit: 'MINUTES') { - sh 'make unittest' -} \ No newline at end of file diff --git a/build/ci/jenkins/Jenkinsfile b/build/ci/jenkins/Jenkinsfile index 6215139ccf..4648cdef09 100644 --- a/build/ci/jenkins/Jenkinsfile +++ b/build/ci/jenkins/Jenkinsfile @@ -1,5 +1,4 @@ #!/usr/bin/env groovy -@Library('mpl') _ String cron_timezone = "TZ=Asia/Shanghai" String cron_string = BRANCH_NAME == "r0.3-test" ? "50 20,22,0,2,4,6,11,16 * * * " : "" @@ -12,103 +11,75 @@ pipeline { } options { timestamps() + timeout(time: 1, unit: 'HOURS') // parallelsAlwaysFailFast() } - environment { - PROJECT_NAME = "milvus" - SEMVER = "${BRANCH_NAME.contains('/') ? BRANCH_NAME.substring(BRANCH_NAME.lastIndexOf('/') + 1) : BRANCH_NAME}" - BUILD_TYPE = "Release" - LOWER_BUILD_TYPE = BUILD_TYPE.toLowerCase() - PACKAGE_VERSION = "${SEMVER}-${LOWER_BUILD_TYPE}" - PACKAGE_NAME = "${PROJECT_NAME}-${PACKAGE_VERSION}.tar.gz" - JFROG_CREDENTIALS_ID = "1a527823-d2b7-44fd-834b-9844350baf14" - JFROG_ARTFACTORY_URL = "http://192.168.1.201/artifactory/milvus" - PACKAGE_ARTFACTORY_URL = "${JFROG_ARTFACTORY_URL}/${PROJECT_NAME}/package/${PACKAGE_NAME}" - DOCKER_CREDENTIALS_ID = "ba070c98-c8cc-4f7c-b657-897715f359fc" - DOKCER_REGISTRY_URL = "registry.zilliz.com" - SOURCE_REPO = "${DOKCER_REGISTRY_URL}/milvus" - TARGET_REPO = "${DOKCER_REGISTRY_URL}/milvus" - SOURCE_TAG = "${CHANGE_TARGET ? CHANGE_TARGET : SEMVER}-${LOWER_BUILD_TYPE}" - TARGET_TAG = "${SEMVER}-${LOWER_BUILD_TYPE}" - DOCKER_BUILDKIT = 1 - } stages { - stage ('Build and UnitTest') { - agent { - kubernetes { - label "${env.PROJECT_NAME}-${SEMVER}-${env.BUILD_NUMBER}-build" - defaultContainer 'build-env' - customWorkspace '/home/jenkins/agent/workspace' - yamlFile "build/ci/jenkins/pod/build-env.yaml" - } - } - environment { - PULSAR_ADDRESS = "pulsar://127.0.0.1:6650" - ETCD_ADDRESS = "127.0.0.1:2379" - CCACHE_ARTFACTORY_URL = "${JFROG_ARTFACTORY_URL}/milvus/ccache" - THIRDPARTY_ARTFACTORY_URL = "${JFROG_ARTFACTORY_URL}/milvus/thirdparty" - CUSTOM_THIRDPARTY_PATH = "${WORKSPACE}/3rdparty_download" - GO_CACHE_ARTFACTORY_URL = "${JFROG_ARTFACTORY_URL}/milvus/go-cache" - GO_MOD_ARTFACTORY_URL = "${JFROG_ARTFACTORY_URL}/milvus/go-mod" - } - steps { - container('build-env') { - MPLModule('Build') - MPLModule('Package') - // MPLModule('UnitTest') - } - } - } - - stage ('Publish Docker Images') { - agent { - kubernetes { - label "${env.PROJECT_NAME}-${SEMVER}-${env.BUILD_NUMBER}-publish" - defaultContainer 'publish-images' - yamlFile "build/ci/jenkins/pod/docker-pod.yaml" - } - } - steps { - container('publish-images') { - MPLModule('Publish') - } - } - } - - stage ('Dev Test') { + stage ('E2E Test') { matrix { axes { axis { - name 'REGRESSION_SERVICE_TYPE' + name 'MILVUS_SERVER_TYPE' values 'standalone', 'distributed' } } - environment { - HELM_BRANCH = "2.0" - HELM_RELEASE_NAMESPACE = "dev" - HELM_RELEASE_NAME = "${PROJECT_NAME}-${SEMVER}-${env.BUILD_NUMBER}-${REGRESSION_SERVICE_TYPE}".replaceAll("\\.", "-").replaceAll("_", "-").toLowerCase() - DEV_TEST_ARTIFACTS_PATH = "artifacts" - } agent { kubernetes { - label "${env.PROJECT_NAME}-${SEMVER}-${env.BUILD_NUMBER}-${REGRESSION_SERVICE_TYPE}-devtest" - defaultContainer 'test-env' - yamlFile "build/ci/jenkins/pod/test-env.yaml" + label "milvus-e2e-test-kind" + defaultContainer 'main' + yamlFile "build/ci/jenkins/pod/krte.yaml" + customWorkspace '/home/jenkins/agent/workspace' + // We allow this pod to remain active for a while, later jobs can + // reuse cache in previous created nodes. + idleMinutes 120 } } + environment { + IMAGE_REPO = "registry.zilliz.com/milvus" + DOCKER_BUILDKIT = 1 + DOCKER_VOLUME_DIRECTORY = "/mnt/disk/.docker" + ARTIFACTS = "${env.WORKSPACE}/artifacts" + } stages { stage('Test') { steps { - MPLModule('Python Regression') + container('main') { + dir ('tests/scripts') { + script { + def standaloneEnabled = "true" + if ("${MILVUS_SERVER_TYPE}" == "distributed") { + standaloneEnabled = "false" + } + + if (isTimeTriggeredBuild()) { + echo "This is Cron Job!" + sh "MILVUS_STANDALONE_ENABLED=${standaloneEnabled} ./e2e-k8s.sh --node-image registry.zilliz.com/kindest/node:v1.20.2" + } else { + sh "MILVUS_STANDALONE_ENABLED=${standaloneEnabled} ./e2e-k8s.sh --node-image registry.zilliz.com/kindest/node:v1.20.2 --test-extra-arg \"--tags=smoke\"" + } + } + } + } } } } post { - cleanup { - MPLModule('Cleanup Python Regression') + always { + container('main') { + script { + dir("${env.ARTIFACTS}") { + sh "tar -zcvf artifacts-milvus-${MILVUS_SERVER_TYPE}-e2e-kind.tar.gz ./kind --remove-files || true" + archiveArtifacts artifacts: "artifacts-milvus-${MILVUS_SERVER_TYPE}-e2e-kind.tar.gz", allowEmptyArchive: true + } + } + } } } } } } } + +boolean isTimeTriggeredBuild() { + return (currentBuild.getBuildCauses('hudson.triggers.TimerTrigger$TimerTriggerCause').size() != 0) ? true : false; +} diff --git a/build/ci/jenkins/pod/build-env.yaml b/build/ci/jenkins/pod/build-env.yaml deleted file mode 100644 index b8cf863f13..0000000000 --- a/build/ci/jenkins/pod/build-env.yaml +++ /dev/null @@ -1,43 +0,0 @@ -apiVersion: v1 -kind: Pod -metadata: - name: build-env - labels: - app: milvus - componet: build-env -spec: - containers: - - name: build-env - image: milvusdb/milvus-dev:amd64-ubuntu18.04-20210426-055946 - env: - - name: OS_NAME - value: "ubuntu18.04" - - name: BUILD_ENV_IMAGE_ID - value: "14fd5a2cff5485257e79421a119ece1d6613587c2336fc18c7a8d12c791db0ef" - securityContext: - runAsUser: 1000 - runAsGroup: 1000 - tty: true - resources: - limits: - memory: "16Gi" - cpu: "8.0" - requests: - memory: "8Gi" - cpu: "4.0" - - name: etcd - image: quay.io/coreos/etcd:v3.4.13 - env: - - name: ETCD_LISTEN_CLIENT_URLS - value: "http://0.0.0.0:2379" - - name: ETCD_ADVERTISE_CLIENT_URLS - value: "http://0.0.0.0:2379" - ports: - - containerPort: 2379 - name: etcd - - name: pulsar - image: apachepulsar/pulsar:2.6.1 - ports: - - containerPort: 6650 - name: pulsar - command: ["bin/pulsar", "standalone"] diff --git a/build/ci/jenkins/pod/docker-pod.yaml b/build/ci/jenkins/pod/docker-pod.yaml deleted file mode 100644 index bd74e55e9f..0000000000 --- a/build/ci/jenkins/pod/docker-pod.yaml +++ /dev/null @@ -1,34 +0,0 @@ -apiVersion: v1 -kind: Pod -metadata: - labels: - app: publish - componet: docker -spec: - containers: - - name: publish-images - image: registry.zilliz.com/library/dind-compose:v0.1 - imagePullPolicy: Always - securityContext: - privileged: true - resources: - limits: - memory: "8Gi" - cpu: "2" - requests: - memory: "2Gi" - cpu: "1" - volumeMounts: - - name: docker-graph-storage - mountPath: /var/lib/docker - volumes: - - name: docker-graph-storage - emptyDir: {} - tolerations: - - key: dedicated - operator: Equal - value: milvus - effect: NoSchedule - - - diff --git a/build/ci/jenkins/pod/krte.yaml b/build/ci/jenkins/pod/krte.yaml new file mode 100644 index 0000000000..3159a09f42 --- /dev/null +++ b/build/ci/jenkins/pod/krte.yaml @@ -0,0 +1,66 @@ +apiVersion: v1 +kind: Pod +metadata: + labels: + app: milvus-e2e +spec: + containers: + - name: main + image: registry.zilliz.com/milvus/krte:v20210414-bd36693 + env: + - name: DOCKER_IN_DOCKER_ENABLED + value: "true" + securityContext: + privileged: true + tty: true + args: ["cat"] + resources: + limits: + memory: 14Gi + ephemeral-storage: "20Gi" + requests: + cpu: "4" + memory: 8Gi + ephemeral-storage: "20Gi" + volumeMounts: + - mountPath: /docker-graph + name: docker-graph + - mountPath: /var/lib/docker + name: docker-root + - mountPath: /lib/modules + name: modules + readOnly: true + - mountPath: /sys/fs/cgroup + name: cgroup + - mountPath: /mnt/disk/.docker + name: build-cache + subPath: docker-volume + volumes: + - emptyDir: {} + name: docker-graph + - emptyDir: {} + name: docker-root + - hostPath: + path: /tmp/krte/cache + type: DirectoryOrCreate + name: build-cache + - hostPath: + path: /lib/modules + type: Directory + name: modules + - hostPath: + path: /sys/fs/cgroup + type: Directory + name: cgroup + affinity: + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + podAffinityTerm: + labelSelector: + matchExpressions: + - key: app + operator: In + values: + - milvus-e2e + topologyKey: kubernetes.io/hostname diff --git a/build/ci/jenkins/pod/test-env.yaml b/build/ci/jenkins/pod/test-env.yaml deleted file mode 100644 index f19dd8c183..0000000000 --- a/build/ci/jenkins/pod/test-env.yaml +++ /dev/null @@ -1,38 +0,0 @@ -apiVersion: v1 -kind: Pod -metadata: - labels: - app: milvus - componet: test-env -spec: - containers: - - name: test-env - image: milvusdb/pytest:20210322-201a528 - command: - - cat - tty: true - resources: - limits: - memory: "8Gi" - cpu: "4.0" - requests: - memory: "4Gi" - cpu: "2.0" - - name: deploy-env - image: registry.zilliz.com/library/helm3:v0.1 - command: - - cat - tty: true - volumeMounts: - - name: kubeconf - mountPath: /root/.kube/ - readOnly: true - volumes: - - name: kubeconf - secret: - secretName: test-cluster-config - tolerations: - - key: dedicated - operator: Equal - value: milvus - effect: NoSchedule diff --git a/build/config/topology/trustworthy-jwt.yaml b/build/config/topology/trustworthy-jwt.yaml index a552e7dc5f..fe271fbdd1 100644 --- a/build/config/topology/trustworthy-jwt.yaml +++ b/build/config/topology/trustworthy-jwt.yaml @@ -4,11 +4,29 @@ apiVersion: kind.x-k8s.io/v1alpha4 networking: ipFamily: "ipv4" kubeProxyMode: "iptables" + podSubnet: "10.245.0.0/16" + serviceSubnet: "10.246.0.0/16" nodes: - role: control-plane - role: worker - role: worker kubeadmConfigPatches: + - | + apiVersion: kubeadm.k8s.io/v1beta2 + kind: JoinConfiguration + metadata: + name: config + nodeRegistration: + kubeletExtraArgs: + cgroup-root: "/kubelet" + - | + apiVersion: kubeadm.k8s.io/v1beta2 + kind: InitConfiguration + metadata: + name: config + nodeRegistration: + kubeletExtraArgs: + cgroup-root: "/kubelet" - | apiVersion: kubeadm.k8s.io/v1beta2 kind: ClusterConfiguration diff --git a/cmd/standalone/main.go b/cmd/standalone/main.go index 1269a5f00c..9592c0281e 100644 --- a/cmd/standalone/main.go +++ b/cmd/standalone/main.go @@ -42,13 +42,13 @@ func initLogCfg() log.Config { logCfg.File.MaxDays = 10 // FIXME(wxyu): Load from config files - //logCfg.File.Filename = "" - ciFileDir := "/milvus/logs/" - if _, err := os.Stat(ciFileDir); err == nil { - logCfg.File.Filename = ciFileDir + "standalone.log" - } else { - logCfg.File.Filename = "/tmp/milvus/standalone.log" - } + logCfg.File.Filename = "" + // ciFileDir := "/milvus/logs/" + // if _, err := os.Stat(ciFileDir); err == nil { + // logCfg.File.Filename = ciFileDir + "standalone.log" + // } else { + // logCfg.File.Filename = "/tmp/milvus/standalone.log" + // } return logCfg } diff --git a/scripts/before-install.sh b/scripts/before-install.sh deleted file mode 100755 index e3891820ac..0000000000 --- a/scripts/before-install.sh +++ /dev/null @@ -1,12 +0,0 @@ -#!/bin/bash - -set -ex - -export CCACHE_COMPRESS=${CCACHE_COMPRESS:="1"} -export CCACHE_COMPRESSLEVEL=${CCACHE_COMPRESSLEVEL:="5"} -export CCACHE_COMPILERCHECK=${CCACHE_COMPILERCHECK:="content"} -export CCACHE_MAXSIZE=${CCACHE_MAXSIZE:="2G"} -export CCACHE_DIR=${CCACHE_DIR:="${HOME}/.ccache"} -export GOPROXY="https://goproxy.cn" - -set +ex diff --git a/scripts/check_cache.sh b/scripts/check_cache.sh deleted file mode 100755 index d675181745..0000000000 --- a/scripts/check_cache.sh +++ /dev/null @@ -1,114 +0,0 @@ -#!/bin/bash - -HELP=" -Usage: - $0 [flags] [Arguments] - - -l [ARTIFACTORY_URL] Artifactory URL - --cache_dir=[CACHE_DIR] Cache directory - -f [FILE] or --file=[FILE] Cache compress package file - -h or --help Print help information - - -Use \"$0 --help\" for more information about a given command. -" - -ARGS=$(getopt -o "l:f:h" -l "cache_dir::,file::,help" -n "$0" -- "$@") - -eval set -- "${ARGS}" - -while true ; do - case "$1" in - -l) - # o has an optional argument. As we are in quoted mode, - # an empty parameter will be generated if its optional - # argument is not found. - case "$2" in - "") echo "Option Artifactory URL, no argument"; exit 1 ;; - *) ARTIFACTORY_URL=$2 ; shift 2 ;; - esac ;; - --cache_dir) - case "$2" in - "") echo "Option cache_dir, no argument"; exit 1 ;; - *) CACHE_DIR=$2 ; shift 2 ;; - esac ;; - -f|--file) - case "$2" in - "") echo "Option file, no argument"; exit 1 ;; - *) PACKAGE_FILE=$2 ; shift 2 ;; - esac ;; - -h|--help) echo -e "${HELP}" ; exit 0 ;; - --) shift ; break ;; - *) echo "Internal error!" ; exit 1 ;; - esac -done - -# Set defaults for vars modified by flags to this script -BRANCH_NAMES=$(git log --decorate | head -n 1 | sed 's/.*(\(.*\))/\1/' | sed 's=[a-zA-Z]*\/==g' | awk -F", " '{$1=""; print $0}') - -if [[ -z "${ARTIFACTORY_URL}" || "${ARTIFACTORY_URL}" == "" ]];then - echo "You have not input ARTIFACTORY_URL !" - exit 1 -fi - -if [[ -z "${CACHE_DIR}" ]]; then - echo "You have not input CACHE_DIR !" - exit 1 -fi - -if [[ -z "${PACKAGE_FILE}" ]]; then - echo "You have not input PACKAGE_FILE !" - exit 1 -fi - -function check_cache() { - BRANCH=$1 - echo "fetching ${BRANCH}/${PACKAGE_FILE}" - wget -q --spider "${ARTIFACTORY_URL}/${BRANCH}/${PACKAGE_FILE}" - return $? -} - -function download_file() { - BRANCH=$1 - wget -q "${ARTIFACTORY_URL}/${BRANCH}/${PACKAGE_FILE}" && \ - mkdir -p "${CACHE_DIR}" && \ - tar zxf "${PACKAGE_FILE}" -C "${CACHE_DIR}" && \ - rm ${PACKAGE_FILE} - return $? -} - -if [[ -n "${CHANGE_TARGET}" && "${BRANCH_NAME}" =~ "PR-" ]];then - check_cache ${CHANGE_TARGET} - if [[ $? == 0 ]];then - download_file ${CHANGE_TARGET} - if [[ $? == 0 ]];then - echo "found cache" - exit 0 - fi - fi - - check_cache ${BRANCH_NAME} - if [[ $? == 0 ]];then - download_file ${BRANCH_NAME} - if [[ $? == 0 ]];then - echo "found cache" - exit 0 - fi - fi -fi - -for CURRENT_BRANCH in ${BRANCH_NAMES} -do - if [[ "${CURRENT_BRANCH}" != "HEAD" ]];then - check_cache ${CURRENT_BRANCH} - if [[ $? == 0 ]];then - download_file ${CURRENT_BRANCH} - if [[ $? == 0 ]];then - echo "found cache" - exit 0 - fi - fi - fi -done - -echo "could not download cache" && exit 1 diff --git a/scripts/update_cache.sh b/scripts/update_cache.sh deleted file mode 100755 index 85985b741d..0000000000 --- a/scripts/update_cache.sh +++ /dev/null @@ -1,104 +0,0 @@ -#!/bin/bash - -HELP=" -Usage: - $0 [flags] [Arguments] - - -l [ARTIFACTORY_URL] Artifactory URL - --cache_dir=[CACHE_DIR] Cache directory - -f [FILE] or --file=[FILE] Cache compress package file - -u [USERNAME] Artifactory Username - -p [PASSWORD] Artifactory Password - -h or --help Print help information - - -Use \"$0 --help\" for more information about a given command. -" - -ARGS=$(getopt -o "l:f:u:p:h" -l "cache_dir::,file::,help" -n "$0" -- "$@") - -eval set -- "${ARGS}" - -while true ; do - case "$1" in - -l) - # o has an optional argument. As we are in quoted mode, - # an empty parameter will be generated if its optional - # argument is not found. - case "$2" in - "") echo "Option Artifactory URL, no argument"; exit 1 ;; - *) ARTIFACTORY_URL=$2 ; shift 2 ;; - esac ;; - --cache_dir) - case "$2" in - "") echo "Option cache_dir, no argument"; exit 1 ;; - *) CACHE_DIR=$2 ; shift 2 ;; - esac ;; - -u) - case "$2" in - "") echo "Option Username, no argument"; exit 1 ;; - *) USERNAME=$2 ; shift 2 ;; - esac ;; - -p) - case "$2" in - "") echo "Option Password, no argument"; exit 1 ;; - *) PASSWORD=$2 ; shift 2 ;; - esac ;; - -f|--file) - case "$2" in - "") echo "Option file, no argument"; exit 1 ;; - *) PACKAGE_FILE=$2 ; shift 2 ;; - esac ;; - -h|--help) echo -e "${HELP}" ; exit 0 ;; - --) shift ; break ;; - *) echo "Internal error!" ; exit 1 ;; - esac -done - -# Set defaults for vars modified by flags to this script -BRANCH_NAME=$(git log --decorate | head -n 1 | sed 's/.*(\(.*\))/\1/' | sed 's/.*, //' | sed 's=[a-zA-Z]*\/==g') - -if [[ -z "${ARTIFACTORY_URL}" || "${ARTIFACTORY_URL}" == "" ]];then - echo "You have not input ARTIFACTORY_URL !" - exit 1 -fi - -if [[ ! -d "${CACHE_DIR}" ]]; then - echo "\"${CACHE_DIR}\" directory does not exist !" - exit 1 -fi - -if [[ -z "${PACKAGE_FILE}" ]]; then - echo "You have not input PACKAGE_FILE !" - exit 1 -fi - -function check_cache() { - BRANCH=$1 - wget -q --spider "${ARTIFACTORY_URL}/${BRANCH}/${PACKAGE_FILE}" - return $? -} - -if [[ -n "${CHANGE_TARGET}" && "${BRANCH_NAME}" =~ "PR-" ]]; then - check_cache ${CHANGE_TARGET} - if [[ $? == 0 ]];then - echo "Skip Update cache package ..." && exit 0 - fi -fi - -if [[ "${BRANCH_NAME}" != "HEAD" ]];then - REMOTE_PACKAGE_PATH="${ARTIFACTORY_URL}/${BRANCH_NAME}" - echo "Updating cache package file: ${PACKAGE_FILE}" - tar zcf ./"${PACKAGE_FILE}" -C "${CACHE_DIR}" . - echo "Uploading cache package file ${PACKAGE_FILE} to ${REMOTE_PACKAGE_PATH}" - curl -u"${USERNAME}":"${PASSWORD}" -T "${PACKAGE_FILE}" "${REMOTE_PACKAGE_PATH}"/"${PACKAGE_FILE}" - if [[ $? == 0 ]];then - echo "Uploading cache package file success !" - exit 0 - else - echo "Uploading cache package file fault !" - exit 1 - fi -fi - -echo "Skip Update cache package ..." diff --git a/tests/scripts/e2e-k8s.sh b/tests/scripts/e2e-k8s.sh index 7777e53630..a8409d5f23 100755 --- a/tests/scripts/e2e-k8s.sh +++ b/tests/scripts/e2e-k8s.sh @@ -272,7 +272,6 @@ if [[ -z "${SKIP_BUILD:-}" ]]; then trace "setup kind registry" setup_kind_registry pushd "${ROOT}" - source "${ROOT}/scripts/before-install.sh" trace "build milvus" "${ROOT}/build/builder.sh" /bin/bash -c "${BUILD_COMMAND}" trace "build milvus image" docker build -f "${ROOT}/build/docker/milvus/Dockerfile" -t "${MILVUS_IMAGE_REPO}:${MILVUS_IMAGE_TAG}" . trace "push milvus image" docker push "${MILVUS_IMAGE_REPO}:${MILVUS_IMAGE_TAG}"