milvus-io
/
milvus
mirror of https://github.com/milvus-io/milvus.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Improve efficiency in Docker build stage (#2334) 

Signed-off-by: quicksilver <zhifeng.zhang@zilliz.com>
pull/2335/head
quicksilver 2020-05-13 22:26:11 +08:00 committed by GitHub
parent 5e777450b3
commit 1a6592f115
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
19 changed files with 123 additions and 121 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.github/workflows/core.yml vendored
View File

@ -8,6 +8,7 @@ on:
- 'ci/**'
- 'core/**'
- '.github/workflows/core.yml'
- docker-compose.yml
- '!**.md'
- '!ci/jenkins/**'
pull_request:
@ -16,6 +17,7 @@ on:
- 'ci/**'
- 'core/**'
- '.github/workflows/core.yml'
- docker-compose.yml
- '!**.md'
- '!ci/jenkins/**'

21
ci/jenkins/Jenkinsfile vendored
View File

@ -64,6 +64,8 @@ pipeline {
versionNumberString : '${SEMVER}-${BINARY_VERSION}-${OS_NAME}-${CPU_ARCH}-${LOWER_BUILD_TYPE}-${BUILD_DATE_FORMATTED, "yyyyMMdd"}-${BUILDS_TODAY}'
]);
DOCKER_VERSION = "${SEMVER}-${BINARY_VERSION}-${OS_NAME}-${LOWER_BUILD_TYPE}"
CHANGE_DOCKER_VERSION = "${CHANGE_TARGET}-${BINARY_VERSION}-${OS_NAME}-${LOWER_BUILD_TYPE}"
CACHA_DOCKER_VERSION = "${CHANGE_TARGET ? CHANGE_DOCKER_VERSION : DOCKER_VERSION}"
}
stages {
@ -91,11 +93,7 @@ pipeline {
steps {
container("milvus-${BINARY_VERSION}-build-env") {
script {
if ("${BINARY_VERSION}" == "gpu") {
load "${env.WORKSPACE}/ci/jenkins/step/unittest.groovy"
} else {
echo "Skip Unittest"
}
load "${env.WORKSPACE}/ci/jenkins/step/unittest.groovy"
}
}
}
@ -105,11 +103,7 @@ pipeline {
steps {
container("milvus-${BINARY_VERSION}-build-env") {
script {
if ("${BINARY_VERSION}" == "gpu") {
load "${env.WORKSPACE}/ci/jenkins/step/coverage.groovy"
} else {
echo "Skip Code Coverage"
}
load "${env.WORKSPACE}/ci/jenkins/step/coverage.groovy"
}
}
}
@ -128,6 +122,13 @@ pipeline {
}
stage('Publish Docker images') {
environment {
SOURCE_REPO = "${params.DOKCER_REGISTRY_URL}/milvus/engine"
TARGET_REPO = "${params.DOKCER_REGISTRY_URL}/milvus/engine"
SOURCE_TAG = "${CACHA_DOCKER_VERSION}"
TARGET_TAG = "${DOCKER_VERSION}"
}
agent {
kubernetes {
label "${OS_NAME}-${BINARY_VERSION}-publish-${SEMVER}-${env.PIPELINE_NAME}-${env.BUILD_NUMBER}"

3
ci/jenkins/pod/docker-pod.yaml
View File

@ -7,7 +7,8 @@ metadata:
spec:
containers:
- name: publish-images
image: registry.zilliz.com/library/docker:v1.0.0
image: registry.zilliz.com/library/docker:v1.1.0
imagePullPolicy: Always
securityContext:
privileged: true
command:

29
ci/jenkins/step/publishImages.groovy
View File

@ -1,4 +1,4 @@
dir ("docker/deploy/${BINARY_VERSION}/${OS_NAME}") {
dir ("docker/deploy") {
def binaryPackage = "${PROJECT_NAME}-${PACKAGE_VERSION}.tar.gz"
withCredentials([usernamePassword(credentialsId: "${params.JFROG_CREDENTIALS_ID}", usernameVariable: 'JFROG_USERNAME', passwordVariable: 'JFROG_PASSWORD')]) {
@ -9,23 +9,27 @@ dir ("docker/deploy/${BINARY_VERSION}/${OS_NAME}") {
}
}
sh "tar zxvf ${binaryPackage}"
def imageName = "${PROJECT_NAME}/engine:${DOCKER_VERSION}"
def sourceImage = "${params.DOKCER_REGISTRY_URL}/${PROJECT_NAME}/engine:${SOURCE_TAG}"
try {
deleteImages("${imageName}", true)
def customImage = docker.build("${imageName}")
deleteImages("${params.DOKCER_REGISTRY_URL}/${imageName}", true)
docker.withRegistry("https://${params.DOKCER_REGISTRY_URL}", "${params.DOCKER_CREDENTIALS_ID}") {
customImage.push()
sh(returnStatus: true, script: "docker pull ${sourceImage}")
sh "docker-compose build --force-rm ${BINARY_VERSION}_${OS_NAME}"
try {
withCredentials([usernamePassword(credentialsId: "${params.DOCKER_CREDENTIALS_ID}", usernameVariable: 'DOCKER_USERNAME', passwordVariable: 'DOCKER_PASSWORD')]) {
sh "docker login -u ${DOCKER_USERNAME} -p ${DOCKER_PASSWORD} ${params.DOKCER_REGISTRY_URL}"
sh "docker-compose push ${BINARY_VERSION}_${OS_NAME}"
}
} catch (exc) {
throw exc
} finally {
sh "docker logout ${params.DOKCER_REGISTRY_URL}"
}
} catch (exc) {
throw exc
} finally {
deleteImages("${imageName}", true)
deleteImages("${params.DOKCER_REGISTRY_URL}/${imageName}", true)
deleteImages("${sourceImage}", true)
sh "docker-compose down --rmi all"
sh(returnStatus: true, script: "docker rmi -f \$(docker images | grep '<none>' | awk '{print \$3}')")
}
}
@ -47,4 +51,3 @@ boolean deleteImages(String imageName, boolean force) {
}
return true
}

4
docker-compose.yml
View File

@ -39,7 +39,7 @@ services:
networks:
- milvus
command: &ubuntu-command
["/milvus/ci/scripts/build.sh -t Release -i ${MILVUS_INSTALL_PREFIX} --with_fiu --coverage -l -u &&
["/milvus/ci/scripts/build.sh -t Release -j4 -i ${MILVUS_INSTALL_PREFIX} --with_fiu --coverage -l -u &&
/milvus/ci/scripts/run_unittest.sh -i ${MILVUS_INSTALL_PREFIX} --mysql_user=root --mysql_password=123456 --mysql_host=mysql &&
/milvus/ci/scripts/coverage.sh"]
@ -59,7 +59,7 @@ services:
networks:
- milvus
command: &centos-command
["/milvus/ci/scripts/build.sh -t Release -i ${MILVUS_INSTALL_PREFIX} --with_fiu --coverage -l -u &&
["/milvus/ci/scripts/build.sh -t Release -j4 -i ${MILVUS_INSTALL_PREFIX} --with_fiu --coverage -l -u &&
/milvus/ci/scripts/run_unittest.sh -i ${MILVUS_INSTALL_PREFIX} --mysql_user=root --mysql_password=123456 --mysql_host=mysql &&
/milvus/ci/scripts/coverage.sh"]

10
docker/build_env/cpu/ubuntu16.04/docker-entrypoint.sh
View File

@ -1,10 +0,0 @@
#!/bin/bash
set -e
if [ "$1" = 'start' ]; then
tail -f /dev/null
fi
exec "$@"

10
docker/build_env/cpu/ubuntu18.04/docker-entrypoint.sh
View File

@ -1,10 +0,0 @@
#!/bin/bash
set -e
if [ "$1" = 'start' ]; then
tail -f /dev/null
fi
exec "$@"

45
docker/build_env/docker-compose.yaml Normal file
View File

@ -0,0 +1,45 @@
version: '2.3'
services:
cpu_centos7:
image: ${TARGET_REPO}:${TARGET_TAG}
build:
context: ./
dockerfile: cpu/centos7/Dockerfile
cache_from:
- ${SOURCE_REPO}:${SOURCE_TAG}
gpu_centos7:
image: ${TARGET_REPO}:${TARGET_TAG}
build:
context: ./
dockerfile: gpu/centos7/Dockerfile
cache_from:
- ${SOURCE_REPO}:${SOURCE_TAG}
cpu_ubuntu16.04:
image: ${TARGET_REPO}:${TARGET_TAG}
build:
context: ./
dockerfile: cpu/ubuntu16.04/Dockerfile
cache_from:
- ${SOURCE_REPO}:${SOURCE_TAG}
gpu_ubuntu16.04:
image: ${TARGET_REPO}:${TARGET_TAG}
build:
context: ./
dockerfile: gpu/ubuntu16.04/Dockerfile
cache_from:
- ${SOURCE_REPO}:${SOURCE_TAG}
cpu_ubuntu18.04:
image: ${TARGET_REPO}:${TARGET_TAG}
build:
context: ./
dockerfile: cpu/ubuntu18.04/Dockerfile
cache_from:
- ${SOURCE_REPO}:${SOURCE_TAG}
gpu_ubuntu18.04:
image: ${TARGET_REPO}:${TARGET_TAG}
build:
context: ./
dockerfile: gpu/ubuntu18.04/Dockerfile
cache_from:
- ${SOURCE_REPO}:${SOURCE_TAG}

0
docker/build_env/cpu/centos7/docker-entrypoint.sh → docker/build_env/docker-entrypoint.sh
View File

10
docker/build_env/gpu/centos7/docker-entrypoint.sh
View File

@ -1,10 +0,0 @@
#!/bin/bash
set -e
if [ "$1" = 'start' ]; then
tail -f /dev/null
fi
exec "$@"

10
docker/build_env/gpu/ubuntu16.04/docker-entrypoint.sh
View File

@ -1,10 +0,0 @@
#!/bin/bash
set -e
if [ "$1" = 'start' ]; then
tail -f /dev/null
fi
exec "$@"

10
docker/build_env/gpu/ubuntu18.04/docker-entrypoint.sh
View File

@ -1,10 +0,0 @@
#!/bin/bash
set -e
if [ "$1" = 'start' ]; then
tail -f /dev/null
fi
exec "$@"

9
docker/deploy/cpu/ubuntu16.04/docker-entrypoint.sh
View File

@ -1,9 +0,0 @@
#!/bin/bash
set -e
if [ "$1" == 'start' ]; then
cd /var/lib/milvus/scripts && ./start_server.sh
fi
exec "$@"

9
docker/deploy/cpu/ubuntu18.04/docker-entrypoint.sh
View File

@ -1,9 +0,0 @@
#!/bin/bash
set -e
if [ "$1" == 'start' ]; then
cd /var/lib/milvus/scripts && ./start_server.sh
fi
exec "$@"

45
docker/deploy/docker-compose.yaml Normal file
View File

@ -0,0 +1,45 @@
version: '2.3'
services:
cpu_centos7:
image: ${TARGET_REPO}:${TARGET_TAG}
build:
context: ./
dockerfile: cpu/centos7/Dockerfile
cache_from:
- ${SOURCE_REPO}:${SOURCE_TAG}
gpu_centos7:
image: ${TARGET_REPO}:${TARGET_TAG}
build:
context: ./
dockerfile: gpu/centos7/Dockerfile
cache_from:
- ${SOURCE_REPO}:${SOURCE_TAG}
cpu_ubuntu16.04:
image: ${TARGET_REPO}:${TARGET_TAG}
build:
context: ./
dockerfile: cpu/ubuntu16.04/Dockerfile
cache_from:
- ${SOURCE_REPO}:${SOURCE_TAG}
gpu_ubuntu16.04:
image: ${TARGET_REPO}:${TARGET_TAG}
build:
context: ./
dockerfile: gpu/ubuntu16.04/Dockerfile
cache_from:
- ${SOURCE_REPO}:${SOURCE_TAG}
cpu_ubuntu18.04:
image: ${TARGET_REPO}:${TARGET_TAG}
build:
context: ./
dockerfile: cpu/ubuntu18.04/Dockerfile
cache_from:
- ${SOURCE_REPO}:${SOURCE_TAG}
gpu_ubuntu18.04:
image: ${TARGET_REPO}:${TARGET_TAG}
build:
context: ./
dockerfile: gpu/ubuntu18.04/Dockerfile
cache_from:
- ${SOURCE_REPO}:${SOURCE_TAG}

0
docker/deploy/cpu/centos7/docker-entrypoint.sh → docker/deploy/docker-entrypoint.sh
View File

9
docker/deploy/gpu/centos7/docker-entrypoint.sh
View File

@ -1,9 +0,0 @@
#!/bin/bash
set -e
if [ "$1" == 'start' ]; then
cd /var/lib/milvus/scripts && ./start_server.sh
fi
exec "$@"

9
docker/deploy/gpu/ubuntu16.04/docker-entrypoint.sh
View File

@ -1,9 +0,0 @@
#!/bin/bash
set -e
if [ "$1" == 'start' ]; then
cd /var/lib/milvus/scripts && ./start_server.sh
fi
exec "$@"

9
docker/deploy/gpu/ubuntu18.04/docker-entrypoint.sh
View File

@ -1,9 +0,0 @@
#!/bin/bash
set -e
if [ "$1" == 'start' ]; then
cd /var/lib/milvus/scripts && ./start_server.sh
fi
exec "$@"