mirror of https://github.com/milvus-io/milvus.git
Refactor the build and scripts folders
Signed-off-by: quicksilver <zhifeng.zhang@zilliz.com>pull/4973/head^2
quicksilver
yefu.chen
parent
52932da551
commit
1b31b85ef7
|
|
@ -6,25 +6,23 @@ on:
|
||||||
push:
|
push:
|
||||||
# file paths to consider in the event. Optional; defaults to all.
|
# file paths to consider in the event. Optional; defaults to all.
|
||||||
paths:
|
paths:
|
||||||
- 'ci/**'
|
- 'scripts/**'
|
||||||
- 'internal/**'
|
- 'internal/**'
|
||||||
- 'cmd/**'
|
- 'cmd/**'
|
||||||
- '.github/workflows/main.yml'
|
- '.github/workflows/main.yml'
|
||||||
- docker-compose.yml
|
- docker-compose.yml
|
||||||
- '!**.md'
|
- '!**.md'
|
||||||
- '!**_test.go'
|
- '!**_test.go'
|
||||||
- '!ci/jenkins/**'
|
|
||||||
pull_request:
|
pull_request:
|
||||||
# file paths to consider in the event. Optional; defaults to all.
|
# file paths to consider in the event. Optional; defaults to all.
|
||||||
paths:
|
paths:
|
||||||
- 'ci/**'
|
- 'scripts/**'
|
||||||
- 'internal/**'
|
- 'internal/**'
|
||||||
- 'cmd/**'
|
- 'cmd/**'
|
||||||
- '.github/workflows/main.yml'
|
- '.github/workflows/main.yml'
|
||||||
- docker-compose.yml
|
- docker-compose.yml
|
||||||
- '!**.md'
|
- '!**.md'
|
||||||
- '!**_test.go'
|
- '!**_test.go'
|
||||||
- '!ci/jenkins/**'
|
|
||||||
|
|
||||||
jobs:
|
jobs:
|
||||||
ubuntu:
|
ubuntu:
|
||||||
|
|
@ -35,7 +33,7 @@ jobs:
|
||||||
uses: actions/checkout@v2
|
uses: actions/checkout@v2
|
||||||
- name: Install Dependency
|
- name: Install Dependency
|
||||||
run: |
|
run: |
|
||||||
./ci/scripts/install_deps.sh
|
./scripts/install_deps.sh
|
||||||
go get github.com/golang/protobuf/protoc-gen-go@v1.3.2
|
go get github.com/golang/protobuf/protoc-gen-go@v1.3.2
|
||||||
- name: Cache Core Thirdparty
|
- name: Cache Core Thirdparty
|
||||||
id: cache-core
|
id: cache-core
|
||||||
|
|
@ -46,14 +44,14 @@ jobs:
|
||||||
key: ${{ runner.os }}-core-thirdparty
|
key: ${{ runner.os }}-core-thirdparty
|
||||||
- name: Build Cpp
|
- name: Build Cpp
|
||||||
run: |
|
run: |
|
||||||
./ci/scripts/core_build.sh -u
|
./scripts/core_build.sh -u
|
||||||
- name: Generat Proto GO File
|
- name: Generat Proto GO File
|
||||||
run: |
|
run: |
|
||||||
echo `pwd`
|
echo `pwd`
|
||||||
pwd_dir=`pwd`
|
pwd_dir=`pwd`
|
||||||
export PATH=$PATH:$(go env GOPATH)/bin
|
export PATH=$PATH:$(go env GOPATH)/bin
|
||||||
export protoc=${pwd_dir}/internal/core/cmake_build/thirdparty/protobuf/protobuf-build/protoc
|
export protoc=${pwd_dir}/internal/core/cmake_build/thirdparty/protobuf/protobuf-build/protoc
|
||||||
./ci/scripts/proto_gen_go.sh
|
./scripts/proto_gen_go.sh
|
||||||
- name: Build GO
|
- name: Build GO
|
||||||
run: |
|
run: |
|
||||||
go build -o ./cmd/writer/writer ./cmd/writer/writer.go
|
go build -o ./cmd/writer/writer ./cmd/writer/writer.go
|
||||||
|
|
@ -65,4 +63,4 @@ jobs:
|
||||||
docker-compose up -d
|
docker-compose up -d
|
||||||
- name: Run Unittest
|
- name: Run Unittest
|
||||||
run: |
|
run: |
|
||||||
./ci/scripts/run_unittest.sh
|
./scripts/run_unittest.sh
|
||||||
|
|
|
||||||
|
|
@ -1,196 +0,0 @@
|
||||||
#!/usr/bin/env groovy
|
|
||||||
@Library('mpl') _
|
|
||||||
|
|
||||||
String cron_timezone = "TZ=Asia/Shanghai"
|
|
||||||
String cron_string = BRANCH_NAME == "master" ? "50 3 * * * " : ""
|
|
||||||
|
|
||||||
pipeline {
|
|
||||||
agent none
|
|
||||||
triggers {
|
|
||||||
cron """${cron_timezone}
|
|
||||||
${cron_string}"""
|
|
||||||
}
|
|
||||||
options {
|
|
||||||
timestamps()
|
|
||||||
}
|
|
||||||
parameters{
|
|
||||||
choice choices: ['Release', 'Debug'], description: 'Build Type', name: 'BUILD_TYPE'
|
|
||||||
choice choices: ['False', 'True'], description: 'Is Manual Trigger Or Not', name: 'IS_MANUAL_TRIGGER_TYPE'
|
|
||||||
string defaultValue: 'registry.zilliz.com', description: 'DOCKER REGISTRY URL', name: 'DOKCER_REGISTRY_URL', trim: true
|
|
||||||
string defaultValue: 'ba070c98-c8cc-4f7c-b657-897715f359fc', description: 'DOCKER CREDENTIALS ID', name: 'DOCKER_CREDENTIALS_ID', trim: true
|
|
||||||
string defaultValue: 'http://192.168.1.201/artifactory/milvus', description: 'JFROG ARTFACTORY URL', name: 'JFROG_ARTFACTORY_URL', trim: true
|
|
||||||
string defaultValue: '1a527823-d2b7-44fd-834b-9844350baf14', description: 'JFROG CREDENTIALS ID', name: 'JFROG_CREDENTIALS_ID', trim: true
|
|
||||||
}
|
|
||||||
environment {
|
|
||||||
PROJECT_NAME = "milvus"
|
|
||||||
MILVUS_ROOT_PATH="/var/lib"
|
|
||||||
MILVUS_INSTALL_PREFIX="${env.MILVUS_ROOT_PATH}/${env.PROJECT_NAME}"
|
|
||||||
LOWER_BUILD_TYPE = params.BUILD_TYPE.toLowerCase()
|
|
||||||
SEMVER = "${BRANCH_NAME.contains('/') ? BRANCH_NAME.substring(BRANCH_NAME.lastIndexOf('/') + 1) : BRANCH_NAME}"
|
|
||||||
PIPELINE_NAME = "milvus-ci"
|
|
||||||
HELM_BRANCH = "0.11.0"
|
|
||||||
}
|
|
||||||
stages {
|
|
||||||
stage ('Milvus Build and Unittest') {
|
|
||||||
matrix {
|
|
||||||
axes {
|
|
||||||
axis {
|
|
||||||
name 'OS_NAME'
|
|
||||||
values 'centos7'
|
|
||||||
}
|
|
||||||
axis {
|
|
||||||
name 'CPU_ARCH'
|
|
||||||
values 'amd64'
|
|
||||||
}
|
|
||||||
axis {
|
|
||||||
name 'BINARY_VERSION'
|
|
||||||
values 'cpu'
|
|
||||||
}
|
|
||||||
}
|
|
||||||
environment {
|
|
||||||
PACKAGE_VERSION = VersionNumber([
|
|
||||||
versionNumberString : '${SEMVER}-${BINARY_VERSION}-${OS_NAME}-${CPU_ARCH}-${LOWER_BUILD_TYPE}'
|
|
||||||
]);
|
|
||||||
}
|
|
||||||
agent {
|
|
||||||
kubernetes {
|
|
||||||
label "${OS_NAME}-${BINARY_VERSION}-build-${SEMVER}-${env.PIPELINE_NAME}-${env.BUILD_NUMBER}"
|
|
||||||
defaultContainer 'jnlp'
|
|
||||||
customWorkspace '/home/jenkins/agent/workspace'
|
|
||||||
yamlFile "ci/jenkins/pod/milvus-${BINARY_VERSION}-version-${OS_NAME}-build-env-pod.yaml"
|
|
||||||
}
|
|
||||||
}
|
|
||||||
stages {
|
|
||||||
stage('Build and Unittest') {
|
|
||||||
steps {
|
|
||||||
container("milvus-${BINARY_VERSION}-build-env") {
|
|
||||||
MPLModule('Milvus Build')
|
|
||||||
MPLModule('Unittest')
|
|
||||||
MPLModule('Package Build')
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
stage ('Publish Docker Images') {
|
|
||||||
matrix {
|
|
||||||
axes {
|
|
||||||
axis {
|
|
||||||
name 'OS_NAME'
|
|
||||||
values 'centos7'
|
|
||||||
}
|
|
||||||
axis {
|
|
||||||
name 'CPU_ARCH'
|
|
||||||
values 'amd64'
|
|
||||||
}
|
|
||||||
axis {
|
|
||||||
name 'BINARY_VERSION'
|
|
||||||
values 'cpu'
|
|
||||||
}
|
|
||||||
}
|
|
||||||
environment {
|
|
||||||
PACKAGE_VERSION = VersionNumber([
|
|
||||||
versionNumberString : '${SEMVER}-${BINARY_VERSION}-${OS_NAME}-${CPU_ARCH}-${LOWER_BUILD_TYPE}'
|
|
||||||
]);
|
|
||||||
SOURCE_REPO = "${params.DOKCER_REGISTRY_URL}/milvus/engine"
|
|
||||||
TARGET_REPO = "${params.DOKCER_REGISTRY_URL}/milvus/engine"
|
|
||||||
SOURCE_TAG = "${CHANGE_TARGET ? CHANGE_TARGET : SEMVER}-${BINARY_VERSION}-${OS_NAME}-${LOWER_BUILD_TYPE}"
|
|
||||||
TARGET_TAG = "${SEMVER}-${BINARY_VERSION}-${OS_NAME}-${LOWER_BUILD_TYPE}"
|
|
||||||
DOCKER_BUILDKIT = 1
|
|
||||||
}
|
|
||||||
agent {
|
|
||||||
kubernetes {
|
|
||||||
label "${OS_NAME}-${BINARY_VERSION}-publish-${SEMVER}-${env.PIPELINE_NAME}-${env.BUILD_NUMBER}"
|
|
||||||
defaultContainer 'jnlp'
|
|
||||||
yamlFile 'ci/jenkins/pod/docker-pod.yaml'
|
|
||||||
}
|
|
||||||
}
|
|
||||||
stages {
|
|
||||||
stage('Publish') {
|
|
||||||
steps {
|
|
||||||
container('publish-images') {
|
|
||||||
MPLModule('Publish')
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
stage ('Dev Test') {
|
|
||||||
matrix {
|
|
||||||
axes {
|
|
||||||
axis {
|
|
||||||
name 'OS_NAME'
|
|
||||||
values 'centos7'
|
|
||||||
}
|
|
||||||
|
|
||||||
axis {
|
|
||||||
name 'CPU_ARCH'
|
|
||||||
values 'amd64'
|
|
||||||
}
|
|
||||||
|
|
||||||
axis {
|
|
||||||
name 'BINARY_VERSION'
|
|
||||||
values 'cpu'
|
|
||||||
}
|
|
||||||
}
|
|
||||||
environment {
|
|
||||||
DOCKER_VERSION = "${SEMVER}-${BINARY_VERSION}-${OS_NAME}-${LOWER_BUILD_TYPE}"
|
|
||||||
FROMAT_SEMVER = "${env.SEMVER}".replaceAll("\\.", "-").replaceAll("_", "-")
|
|
||||||
FORMAT_OS_NAME = "${OS_NAME}".replaceAll("\\.", "-").replaceAll("_", "-")
|
|
||||||
HELM_RELEASE_NAME = "${env.PIPELINE_NAME}-${env.FROMAT_SEMVER}-${env.BUILD_NUMBER}-single-${FORMAT_OS_NAME}-${BINARY_VERSION}".toLowerCase()
|
|
||||||
SHARDS_HELM_RELEASE_NAME = "${env.PIPELINE_NAME}-${env.FROMAT_SEMVER}-${env.BUILD_NUMBER}-shards-${FORMAT_OS_NAME}-${BINARY_VERSION}".toLowerCase()
|
|
||||||
DEV_TEST_ARTIFACTS = "_artifacts/${FORMAT_OS_NAME}/${BINARY_VERSION}"
|
|
||||||
}
|
|
||||||
agent {
|
|
||||||
kubernetes {
|
|
||||||
label "${OS_NAME}-${BINARY_VERSION}-dev-test-${SEMVER}-${env.PIPELINE_NAME}-${env.BUILD_NUMBER}"
|
|
||||||
defaultContainer 'jnlp'
|
|
||||||
yamlFile 'ci/jenkins/pod/testEnvironment.yaml'
|
|
||||||
}
|
|
||||||
}
|
|
||||||
stages {
|
|
||||||
stage('Test') {
|
|
||||||
steps {
|
|
||||||
container('milvus-test-env') {
|
|
||||||
MPLModule('Single Node DevTest')
|
|
||||||
MPLModule('Mishards DevTest')
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
post {
|
|
||||||
cleanup {
|
|
||||||
container('milvus-test-env') {
|
|
||||||
archiveArtifacts artifacts: "${env.DEV_TEST_ARTIFACTS}/**", allowEmptyArchive: true
|
|
||||||
MPLModule('Cleanup Single Node DevTest')
|
|
||||||
MPLModule('Cleanup Mishards DevTest')
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
post {
|
|
||||||
unsuccessful {
|
|
||||||
script {
|
|
||||||
if (isTimeTriggeredBuild()) {
|
|
||||||
// Send an email only if the build status has changed from green/unstable to red
|
|
||||||
emailext subject: '$DEFAULT_SUBJECT',
|
|
||||||
body: '$DEFAULT_CONTENT',
|
|
||||||
recipientProviders: [
|
|
||||||
[$class: 'DevelopersRecipientProvider'],
|
|
||||||
[$class: 'RequesterRecipientProvider']
|
|
||||||
],
|
|
||||||
replyTo: '$DEFAULT_REPLYTO',
|
|
||||||
to: 'dev.milvus@zilliz.com'
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
boolean isTimeTriggeredBuild() {
|
|
||||||
return (currentBuild.getBuildCauses('hudson.triggers.TimerTrigger$TimerTriggerCause').size() != 0) ? true : false;
|
|
||||||
}
|
|
||||||
|
|
@ -1,35 +0,0 @@
|
||||||
apiVersion: v1
|
|
||||||
kind: Pod
|
|
||||||
metadata:
|
|
||||||
labels:
|
|
||||||
app: publish
|
|
||||||
componet: docker
|
|
||||||
spec:
|
|
||||||
containers:
|
|
||||||
- name: publish-images
|
|
||||||
image: registry.zilliz.com/library/docker:v1.1.0
|
|
||||||
imagePullPolicy: Always
|
|
||||||
securityContext:
|
|
||||||
privileged: true
|
|
||||||
command:
|
|
||||||
- cat
|
|
||||||
tty: true
|
|
||||||
resources:
|
|
||||||
limits:
|
|
||||||
memory: "8Gi"
|
|
||||||
cpu: "2"
|
|
||||||
requests:
|
|
||||||
memory: "2Gi"
|
|
||||||
cpu: "1"
|
|
||||||
volumeMounts:
|
|
||||||
- name: docker-sock
|
|
||||||
mountPath: /var/run/docker.sock
|
|
||||||
volumes:
|
|
||||||
- name: docker-sock
|
|
||||||
hostPath:
|
|
||||||
path: /var/run/docker.sock
|
|
||||||
tolerations:
|
|
||||||
- key: dedicated
|
|
||||||
operator: Equal
|
|
||||||
value: milvus
|
|
||||||
effect: NoSchedule
|
|
||||||
|
|
@ -1,35 +0,0 @@
|
||||||
apiVersion: v1
|
|
||||||
kind: Pod
|
|
||||||
metadata:
|
|
||||||
name: milvus-cpu-build-env
|
|
||||||
labels:
|
|
||||||
app: milvus
|
|
||||||
componet: cpu-build-env
|
|
||||||
spec:
|
|
||||||
containers:
|
|
||||||
- name: milvus-cpu-build-env
|
|
||||||
image: registry.zilliz.com/milvus/milvus-cpu-build-env:v0.10.2-ubuntu18.04
|
|
||||||
env:
|
|
||||||
- name: POD_IP
|
|
||||||
valueFrom:
|
|
||||||
fieldRef:
|
|
||||||
fieldPath: status.podIP
|
|
||||||
- name: OS_NAME
|
|
||||||
value: "ubuntu18.04"
|
|
||||||
- name: BUILD_ENV_IMAGE_ID
|
|
||||||
value: "4e30ebe398d1a10150c625e52f44d5426c71a557afbb3f16ee9cea20e52e1b9d"
|
|
||||||
command:
|
|
||||||
- cat
|
|
||||||
tty: true
|
|
||||||
resources:
|
|
||||||
limits:
|
|
||||||
memory: "14Gi"
|
|
||||||
cpu: "6.0"
|
|
||||||
requests:
|
|
||||||
memory: "8Gi"
|
|
||||||
cpu: "4.0"
|
|
||||||
tolerations:
|
|
||||||
- key: dedicated
|
|
||||||
operator: Equal
|
|
||||||
value: milvus
|
|
||||||
effect: NoSchedule
|
|
||||||
|
|
@ -1,33 +0,0 @@
|
||||||
apiVersion: v1
|
|
||||||
kind: Pod
|
|
||||||
metadata:
|
|
||||||
labels:
|
|
||||||
app: milvus
|
|
||||||
componet: test-env
|
|
||||||
spec:
|
|
||||||
containers:
|
|
||||||
- name: milvus-test-env
|
|
||||||
image: registry.zilliz.com/milvus/milvus-test-env:v0.2
|
|
||||||
command:
|
|
||||||
- cat
|
|
||||||
tty: true
|
|
||||||
resources:
|
|
||||||
limits:
|
|
||||||
memory: "8Gi"
|
|
||||||
cpu: "4.0"
|
|
||||||
requests:
|
|
||||||
memory: "4Gi"
|
|
||||||
cpu: "2.0"
|
|
||||||
volumeMounts:
|
|
||||||
- name: kubeconf
|
|
||||||
mountPath: /root/.kube/
|
|
||||||
readOnly: true
|
|
||||||
volumes:
|
|
||||||
- name: kubeconf
|
|
||||||
secret:
|
|
||||||
secretName: test-cluster-config
|
|
||||||
tolerations:
|
|
||||||
- key: dedicated
|
|
||||||
operator: Equal
|
|
||||||
value: milvus
|
|
||||||
effect: NoSchedule
|
|
||||||
|
|
@ -1,114 +0,0 @@
|
||||||
#!/bin/bash
|
|
||||||
|
|
||||||
HELP="
|
|
||||||
Usage:
|
|
||||||
$0 [flags] [Arguments]
|
|
||||||
|
|
||||||
-l [ARTIFACTORY_URL] Artifactory URL
|
|
||||||
--cache_dir=[CACHE_DIR] Cache directory
|
|
||||||
-f [FILE] or --file=[FILE] Cache compress package file
|
|
||||||
-h or --help Print help information
|
|
||||||
|
|
||||||
|
|
||||||
Use \"$0 --help\" for more information about a given command.
|
|
||||||
"
|
|
||||||
|
|
||||||
ARGS=$(getopt -o "l:f:h" -l "cache_dir::,file::,help" -n "$0" -- "$@")
|
|
||||||
|
|
||||||
eval set -- "${ARGS}"
|
|
||||||
|
|
||||||
while true ; do
|
|
||||||
case "$1" in
|
|
||||||
-l)
|
|
||||||
# o has an optional argument. As we are in quoted mode,
|
|
||||||
# an empty parameter will be generated if its optional
|
|
||||||
# argument is not found.
|
|
||||||
case "$2" in
|
|
||||||
"") echo "Option Artifactory URL, no argument"; exit 1 ;;
|
|
||||||
*) ARTIFACTORY_URL=$2 ; shift 2 ;;
|
|
||||||
esac ;;
|
|
||||||
--cache_dir)
|
|
||||||
case "$2" in
|
|
||||||
"") echo "Option cache_dir, no argument"; exit 1 ;;
|
|
||||||
*) CACHE_DIR=$2 ; shift 2 ;;
|
|
||||||
esac ;;
|
|
||||||
-f|--file)
|
|
||||||
case "$2" in
|
|
||||||
"") echo "Option file, no argument"; exit 1 ;;
|
|
||||||
*) PACKAGE_FILE=$2 ; shift 2 ;;
|
|
||||||
esac ;;
|
|
||||||
-h|--help) echo -e "${HELP}" ; exit 0 ;;
|
|
||||||
--) shift ; break ;;
|
|
||||||
*) echo "Internal error!" ; exit 1 ;;
|
|
||||||
esac
|
|
||||||
done
|
|
||||||
|
|
||||||
# Set defaults for vars modified by flags to this script
|
|
||||||
BRANCH_NAMES=$(git log --decorate | head -n 1 | sed 's/.*(\(.*\))/\1/' | sed 's=[a-zA-Z]*\/==g' | awk -F", " '{$1=""; print $0}')
|
|
||||||
|
|
||||||
if [[ -z "${ARTIFACTORY_URL}" || "${ARTIFACTORY_URL}" == "" ]];then
|
|
||||||
echo "You have not input ARTIFACTORY_URL !"
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
|
|
||||||
if [[ -z "${CACHE_DIR}" ]]; then
|
|
||||||
echo "You have not input CACHE_DIR !"
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
|
|
||||||
if [[ -z "${PACKAGE_FILE}" ]]; then
|
|
||||||
echo "You have not input PACKAGE_FILE !"
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
|
|
||||||
function check_cache() {
|
|
||||||
BRANCH=$1
|
|
||||||
echo "fetching ${BRANCH}/${PACKAGE_FILE}"
|
|
||||||
wget -q --spider "${ARTIFACTORY_URL}/${BRANCH}/${PACKAGE_FILE}"
|
|
||||||
return $?
|
|
||||||
}
|
|
||||||
|
|
||||||
function download_file() {
|
|
||||||
BRANCH=$1
|
|
||||||
wget -q "${ARTIFACTORY_URL}/${BRANCH}/${PACKAGE_FILE}" && \
|
|
||||||
mkdir -p "${CACHE_DIR}" && \
|
|
||||||
tar zxf "${PACKAGE_FILE}" -C "${CACHE_DIR}" && \
|
|
||||||
rm ${PACKAGE_FILE}
|
|
||||||
return $?
|
|
||||||
}
|
|
||||||
|
|
||||||
if [[ -n "${CHANGE_TARGET}" && "${BRANCH_NAME}" =~ "PR-" ]];then
|
|
||||||
check_cache ${CHANGE_TARGET}
|
|
||||||
if [[ $? == 0 ]];then
|
|
||||||
download_file ${CHANGE_TARGET}
|
|
||||||
if [[ $? == 0 ]];then
|
|
||||||
echo "found cache"
|
|
||||||
exit 0
|
|
||||||
fi
|
|
||||||
fi
|
|
||||||
|
|
||||||
check_cache ${BRANCH_NAME}
|
|
||||||
if [[ $? == 0 ]];then
|
|
||||||
download_file ${BRANCH_NAME}
|
|
||||||
if [[ $? == 0 ]];then
|
|
||||||
echo "found cache"
|
|
||||||
exit 0
|
|
||||||
fi
|
|
||||||
fi
|
|
||||||
fi
|
|
||||||
|
|
||||||
for CURRENT_BRANCH in ${BRANCH_NAMES}
|
|
||||||
do
|
|
||||||
if [[ "${CURRENT_BRANCH}" != "HEAD" ]];then
|
|
||||||
check_cache ${CURRENT_BRANCH}
|
|
||||||
if [[ $? == 0 ]];then
|
|
||||||
download_file ${CURRENT_BRANCH}
|
|
||||||
if [[ $? == 0 ]];then
|
|
||||||
echo "found cache"
|
|
||||||
exit 0
|
|
||||||
fi
|
|
||||||
fi
|
|
||||||
fi
|
|
||||||
done
|
|
||||||
|
|
||||||
echo "could not download cache" && exit 1
|
|
||||||
|
|
@ -1,104 +0,0 @@
|
||||||
#!/bin/bash
|
|
||||||
|
|
||||||
HELP="
|
|
||||||
Usage:
|
|
||||||
$0 [flags] [Arguments]
|
|
||||||
|
|
||||||
-l [ARTIFACTORY_URL] Artifactory URL
|
|
||||||
--cache_dir=[CACHE_DIR] Cache directory
|
|
||||||
-f [FILE] or --file=[FILE] Cache compress package file
|
|
||||||
-u [USERNAME] Artifactory Username
|
|
||||||
-p [PASSWORD] Artifactory Password
|
|
||||||
-h or --help Print help information
|
|
||||||
|
|
||||||
|
|
||||||
Use \"$0 --help\" for more information about a given command.
|
|
||||||
"
|
|
||||||
|
|
||||||
ARGS=$(getopt -o "l:f:u:p:h" -l "cache_dir::,file::,help" -n "$0" -- "$@")
|
|
||||||
|
|
||||||
eval set -- "${ARGS}"
|
|
||||||
|
|
||||||
while true ; do
|
|
||||||
case "$1" in
|
|
||||||
-l)
|
|
||||||
# o has an optional argument. As we are in quoted mode,
|
|
||||||
# an empty parameter will be generated if its optional
|
|
||||||
# argument is not found.
|
|
||||||
case "$2" in
|
|
||||||
"") echo "Option Artifactory URL, no argument"; exit 1 ;;
|
|
||||||
*) ARTIFACTORY_URL=$2 ; shift 2 ;;
|
|
||||||
esac ;;
|
|
||||||
--cache_dir)
|
|
||||||
case "$2" in
|
|
||||||
"") echo "Option cache_dir, no argument"; exit 1 ;;
|
|
||||||
*) CACHE_DIR=$2 ; shift 2 ;;
|
|
||||||
esac ;;
|
|
||||||
-u)
|
|
||||||
case "$2" in
|
|
||||||
"") echo "Option Username, no argument"; exit 1 ;;
|
|
||||||
*) USERNAME=$2 ; shift 2 ;;
|
|
||||||
esac ;;
|
|
||||||
-p)
|
|
||||||
case "$2" in
|
|
||||||
"") echo "Option Password, no argument"; exit 1 ;;
|
|
||||||
*) PASSWORD=$2 ; shift 2 ;;
|
|
||||||
esac ;;
|
|
||||||
-f|--file)
|
|
||||||
case "$2" in
|
|
||||||
"") echo "Option file, no argument"; exit 1 ;;
|
|
||||||
*) PACKAGE_FILE=$2 ; shift 2 ;;
|
|
||||||
esac ;;
|
|
||||||
-h|--help) echo -e "${HELP}" ; exit 0 ;;
|
|
||||||
--) shift ; break ;;
|
|
||||||
*) echo "Internal error!" ; exit 1 ;;
|
|
||||||
esac
|
|
||||||
done
|
|
||||||
|
|
||||||
# Set defaults for vars modified by flags to this script
|
|
||||||
BRANCH_NAME=$(git log --decorate | head -n 1 | sed 's/.*(\(.*\))/\1/' | sed 's/.*, //' | sed 's=[a-zA-Z]*\/==g')
|
|
||||||
|
|
||||||
if [[ -z "${ARTIFACTORY_URL}" || "${ARTIFACTORY_URL}" == "" ]];then
|
|
||||||
echo "You have not input ARTIFACTORY_URL !"
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
|
|
||||||
if [[ ! -d "${CACHE_DIR}" ]]; then
|
|
||||||
echo "\"${CACHE_DIR}\" directory does not exist !"
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
|
|
||||||
if [[ -z "${PACKAGE_FILE}" ]]; then
|
|
||||||
echo "You have not input PACKAGE_FILE !"
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
|
|
||||||
function check_cache() {
|
|
||||||
BRANCH=$1
|
|
||||||
wget -q --spider "${ARTIFACTORY_URL}/${BRANCH}/${PACKAGE_FILE}"
|
|
||||||
return $?
|
|
||||||
}
|
|
||||||
|
|
||||||
if [[ -n "${CHANGE_TARGET}" && "${BRANCH_NAME}" =~ "PR-" ]]; then
|
|
||||||
check_cache ${CHANGE_TARGET}
|
|
||||||
if [[ $? == 0 ]];then
|
|
||||||
echo "Skip Update cache package ..." && exit 0
|
|
||||||
fi
|
|
||||||
fi
|
|
||||||
|
|
||||||
if [[ "${BRANCH_NAME}" != "HEAD" ]];then
|
|
||||||
REMOTE_PACKAGE_PATH="${ARTIFACTORY_URL}/${BRANCH_NAME}"
|
|
||||||
echo "Updating cache package file: ${PACKAGE_FILE}"
|
|
||||||
tar zcf ./"${PACKAGE_FILE}" -C "${CACHE_DIR}" .
|
|
||||||
echo "Uploading cache package file ${PACKAGE_FILE} to ${REMOTE_PACKAGE_PATH}"
|
|
||||||
curl -u"${USERNAME}":"${PASSWORD}" -T "${PACKAGE_FILE}" "${REMOTE_PACKAGE_PATH}"/"${PACKAGE_FILE}"
|
|
||||||
if [[ $? == 0 ]];then
|
|
||||||
echo "Uploading cache package file success !"
|
|
||||||
exit 0
|
|
||||||
else
|
|
||||||
echo "Uploading cache package file fault !"
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
fi
|
|
||||||
|
|
||||||
echo "Skip Update cache package ..."
|
|
||||||
|
|
@ -1,33 +0,0 @@
|
||||||
# Copyright (C) 2019-2020 Zilliz. All rights reserved.
|
|
||||||
#
|
|
||||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
|
|
||||||
# with the License. You may obtain a copy of the License at
|
|
||||||
#
|
|
||||||
# http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
#
|
|
||||||
# Unless required by applicable law or agreed to in writing, software distributed under the License
|
|
||||||
# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
|
|
||||||
# or implied. See the License for the specific language governing permissions and limitations under the License.
|
|
||||||
|
|
||||||
FROM milvusdb/milvus-dev:amd64-centos-7-core AS openblas
|
|
||||||
|
|
||||||
FROM centos:centos7
|
|
||||||
|
|
||||||
RUN yum install -y wget && \
|
|
||||||
wget -P /etc/yum.repos.d/ http://mirrors.aliyun.com/repo/epel-7.repo && \
|
|
||||||
yum clean all && yum makecache && \
|
|
||||||
yum install -y libgomp libgfortran4 mysql-devel && \
|
|
||||||
rm -rf /var/cache/yum/*
|
|
||||||
|
|
||||||
COPY ./milvus /var/lib/milvus
|
|
||||||
ENV LD_LIBRARY_PATH="$LD_LIBRARY_PATH:/var/lib/milvus/lib"
|
|
||||||
|
|
||||||
COPY --from=openblas /usr/lib/libopenblas-r0.3.9.so /var/lib/milvus/lib/
|
|
||||||
RUN ln -s /var/lib/milvus/lib/libopenblas-r0.3.9.so /var/lib/milvus/lib/libopenblas.so.0 && \
|
|
||||||
ln -s /var/lib/milvus/lib/libopenblas.so.0 /var/lib/milvus/lib/libopenblas.so
|
|
||||||
|
|
||||||
WORKDIR /var/lib/milvus
|
|
||||||
|
|
||||||
CMD [ "/var/lib/milvus/bin/milvus_server", "-c", "/var/lib/milvus/conf/milvus.yaml" ]
|
|
||||||
|
|
||||||
EXPOSE 19530
|
|
||||||
|
|
@ -1,35 +0,0 @@
|
||||||
# Copyright (C) 2019-2020 Zilliz. All rights reserved.
|
|
||||||
#
|
|
||||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
|
|
||||||
# with the License. You may obtain a copy of the License at
|
|
||||||
#
|
|
||||||
# http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
#
|
|
||||||
# Unless required by applicable law or agreed to in writing, software distributed under the License
|
|
||||||
# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
|
|
||||||
# or implied. See the License for the specific language governing permissions and limitations under the License.
|
|
||||||
|
|
||||||
FROM milvusdb/milvus-dev:amd64-ubuntu-18.04-core AS openblas
|
|
||||||
|
|
||||||
FROM ubuntu:18.04
|
|
||||||
|
|
||||||
RUN apt-get update && apt-get install -y --no-install-recommends \
|
|
||||||
gfortran libsqlite3-dev libmysqlclient-dev libcurl4-openssl-dev python3 && \
|
|
||||||
apt-get remove --purge -y && \
|
|
||||||
rm -rf /var/lib/apt/lists/*
|
|
||||||
|
|
||||||
RUN ln -s /usr/lib/x86_64-linux-gnu/libmysqlclient.so /usr/lib/x86_64-linux-gnu/libmysqlclient_r.so
|
|
||||||
|
|
||||||
COPY ./milvus /var/lib/milvus
|
|
||||||
ENV LD_LIBRARY_PATH="$LD_LIBRARY_PATH:/var/lib/milvus/lib"
|
|
||||||
|
|
||||||
COPY --from=openblas /usr/lib/libopenblas-r0.3.9.so /var/lib/milvus/lib/
|
|
||||||
RUN ln -s /var/lib/milvus/lib/libopenblas-r0.3.9.so /var/lib/milvus/lib/libopenblas.so.0 && \
|
|
||||||
ln -s /var/lib/milvus/lib/libopenblas.so.0 /var/lib/milvus/lib/libopenblas.so
|
|
||||||
|
|
||||||
WORKDIR /var/lib/milvus
|
|
||||||
|
|
||||||
CMD [ "/var/lib/milvus/bin/milvus_server", "-c", "/var/lib/milvus/conf/milvus.yaml" ]
|
|
||||||
|
|
||||||
EXPOSE 19530
|
|
||||||
|
|
||||||
|
|
@ -1,17 +0,0 @@
|
||||||
version: '2.3'
|
|
||||||
|
|
||||||
services:
|
|
||||||
cpu_centos7:
|
|
||||||
image: ${TARGET_REPO}:${TARGET_TAG}
|
|
||||||
build:
|
|
||||||
context: ./
|
|
||||||
dockerfile: cpu/centos7/Dockerfile
|
|
||||||
cache_from:
|
|
||||||
- ${SOURCE_REPO}:${SOURCE_TAG}
|
|
||||||
cpu_ubuntu18.04:
|
|
||||||
image: ${TARGET_REPO}:${TARGET_TAG}
|
|
||||||
build:
|
|
||||||
context: ./
|
|
||||||
dockerfile: cpu/ubuntu18.04/Dockerfile
|
|
||||||
cache_from:
|
|
||||||
- ${SOURCE_REPO}:${SOURCE_TAG}
|
|
||||||
|
|
@ -1,21 +0,0 @@
|
||||||
version: '2.3'
|
|
||||||
|
|
||||||
networks:
|
|
||||||
monitor:
|
|
||||||
driver: bridge
|
|
||||||
|
|
||||||
services:
|
|
||||||
milvus_server:
|
|
||||||
runtime: nvidia
|
|
||||||
image: milvusdb/milvus:latest
|
|
||||||
restart: always
|
|
||||||
environment:
|
|
||||||
WEB_APP: host.docker.internal
|
|
||||||
volumes:
|
|
||||||
- ../core/conf/milvus.yaml:/var/lib/milvus/conf/milvus.yaml
|
|
||||||
- ../core/conf/log_config.conf:/var/lib/milvus/conf/log_config.conf
|
|
||||||
ports:
|
|
||||||
- "8080:8080"
|
|
||||||
- "19530:19530"
|
|
||||||
networks:
|
|
||||||
- monitor
|
|
||||||
|
|
@ -1,30 +0,0 @@
|
||||||
# Copyright (C) 2019-2020 Zilliz. All rights reserved.
|
|
||||||
#
|
|
||||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
|
|
||||||
# with the License. You may obtain a copy of the License at
|
|
||||||
#
|
|
||||||
# http://www.apache.org/licenses/LICENSE-2.0
|
|
||||||
#
|
|
||||||
# Unless required by applicable law or agreed to in writing, software distributed under the License
|
|
||||||
# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
|
|
||||||
# or implied. See the License for the specific language governing permissions and limitations under the License.
|
|
||||||
|
|
||||||
FROM python:3.6.8-jessie
|
|
||||||
|
|
||||||
SHELL ["/bin/bash", "-o", "pipefail", "-c"]
|
|
||||||
|
|
||||||
RUN apt-get update && apt-get install -y --no-install-recommends wget apt-transport-https && \
|
|
||||||
wget -qO- "https://get.helm.sh/helm-v3.0.2-linux-amd64.tar.gz" | tar --strip-components=1 -xz -C /usr/local/bin linux-amd64/helm && \
|
|
||||||
wget -P /tmp https://mirrors.aliyun.com/kubernetes/apt/doc/apt-key.gpg && \
|
|
||||||
apt-key add /tmp/apt-key.gpg && \
|
|
||||||
sh -c 'echo deb https://mirrors.aliyun.com/kubernetes/apt/ kubernetes-xenial main > /etc/apt/sources.list.d/kubernetes.list' && \
|
|
||||||
apt-get update && apt-get install -y --no-install-recommends \
|
|
||||||
build-essential kubectl && \
|
|
||||||
apt-get remove --purge -y && \
|
|
||||||
rm -rf /var/lib/apt/lists/*
|
|
||||||
|
|
||||||
COPY docker-entrypoint.sh /app/docker-entrypoint.sh
|
|
||||||
WORKDIR /root
|
|
||||||
|
|
||||||
ENTRYPOINT [ "/app/docker-entrypoint.sh" ]
|
|
||||||
CMD [ "start" ]
|
|
||||||
|
|
@ -1,9 +0,0 @@
|
||||||
#!/bin/bash
|
|
||||||
|
|
||||||
set -e
|
|
||||||
|
|
||||||
if [ "$1" = 'start' ]; then
|
|
||||||
tail -f /dev/null
|
|
||||||
fi
|
|
||||||
|
|
||||||
exec "$@"
|
|
||||||
|
|
@ -13,7 +13,7 @@ while [ -h "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a symli
|
||||||
done
|
done
|
||||||
SCRIPTS_DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )"
|
SCRIPTS_DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )"
|
||||||
|
|
||||||
CPP_SRC_DIR="${SCRIPTS_DIR}/../../internal/core"
|
CPP_SRC_DIR="${SCRIPTS_DIR}/../internal/core"
|
||||||
CPP_BUILD_DIR="${CPP_SRC_DIR}/cmake_build"
|
CPP_BUILD_DIR="${CPP_SRC_DIR}/cmake_build"
|
||||||
|
|
||||||
BUILD_OUTPUT_DIR=${CPP_BUILD_DIR}
|
BUILD_OUTPUT_DIR=${CPP_BUILD_DIR}
|
||||||
|
|
@ -26,7 +26,7 @@ usage:
|
||||||
done
|
done
|
||||||
|
|
||||||
|
|
||||||
PROTO_DIR=$SCRIPTS_DIR/../../internal/proto/
|
PROTO_DIR=$SCRIPTS_DIR/../internal/proto/
|
||||||
|
|
||||||
PROGRAM=$(basename "$0")
|
PROGRAM=$(basename "$0")
|
||||||
GOPATH=$(go env GOPATH)
|
GOPATH=$(go env GOPATH)
|
||||||
|
|
@ -10,7 +10,7 @@ while [ -h "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a symli
|
||||||
done
|
done
|
||||||
SCRIPTS_DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )"
|
SCRIPTS_DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )"
|
||||||
|
|
||||||
MILVUS_CORE_DIR="${SCRIPTS_DIR}/../../internal/core"
|
MILVUS_CORE_DIR="${SCRIPTS_DIR}/../internal/core"
|
||||||
CORE_INSTALL_PREFIX="${MILVUS_CORE_DIR}/output"
|
CORE_INSTALL_PREFIX="${MILVUS_CORE_DIR}/output"
|
||||||
UNITTEST_DIRS=("${CORE_INSTALL_PREFIX}/unittest")
|
UNITTEST_DIRS=("${CORE_INSTALL_PREFIX}/unittest")
|
||||||
|
|
||||||
Loading…
Reference in New Issue