Milvus build stage parallel processing (#3423)

* Milvus build stage parallel processing Signed-off-by: quicksilver <zhifeng.zhang@zilliz.com> * Milvus build stage parallel processing Signed-off-by: quicksilver <zhifeng.zhang@zilliz.com> * Milvus build stage parallel processing Signed-off-by: quicksilver <zhifeng.zhang@zilliz.com> * Milvus build stage parallel processing Signed-off-by: quicksilver <zhifeng.zhang@zilliz.com> * Milvus build stage parallel processing Signed-off-by: quicksilver <zhifeng.zhang@zilliz.com> * Milvus build stage parallel processing Signed-off-by: quicksilver <zhifeng.zhang@zilliz.com>
2020-08-24 15:30:52 +08:00 · 2020-08-24 15:30:52 +08:00 · ff9fe6fa4a
parent ed1195edb2
commit ff9fe6fa4a
4 changed files with 132 additions and 16 deletions
--- a/.github/workflows/cpplint.yml
+++ b/.github/workflows/cpplint.yml
@ -0,0 +1,64 @@
+name: Cpplint
+
+# This workflow is triggered on pushes or pull request to the repository.
+on:
+  push:
+    # file paths to consider in the event. Optional; defaults to all.
+    paths:
+      - 'ci/**'
+      - 'core/**'
+      - 'docker/build_env/cpu/**'
+      - '.github/workflows/cpplint.yml'
+      - docker-compose.yml
+      - '!**.md'
+      - '!ci/jenkins/**'
+  pull_request:
+    # file paths to consider in the event. Optional; defaults to all.
+    paths:
+      - 'ci/**'
+      - 'core/**'
+      - '.github/workflows/cpplint.yml'
+      - docker-compose.yml
+      - '!**.md'
+      - '!ci/jenkins/**'
+
+jobs:
+  centos:
+    name: ${{ matrix.title }}
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        name:
+          - cpplint
+          - cpplint-gpu
+        include:
+          - name: cpplint
+            service: cpplint
+            title: Milvus CppLint
+          - name: cpplint-gpu
+            service: cpplint-gpu
+            title: Milvus GPU Version CppLint
+    steps:
+      # This step checks out a copy of your repository.
+      - name: Checkout Milvus
+        uses: actions/checkout@v2
+      - name: Docker Pull
+        shell: bash
+        run: |
+          docker-compose pull --ignore-pull-failures ${{ matrix.service }}
+      - name: Docker Build
+        shell: bash
+        run: |
+          docker-compose build ${{ matrix.service }}
+      - name: Docker Run
+        run: |
+          docker-compose run ${{ matrix.service }}
+      - name: Docker Push
+        if: success() && github.event_name == 'push' && github.repository == 'milvus-io/milvus'
+        continue-on-error: true
+        shell: bash
+        run: |
+          docker login -u ${{ secrets.DOCKERHUB_USER }} \
+                       -p ${{ secrets.DOCKERHUB_TOKEN }}
+          docker-compose push ${{ matrix.service }}
--- a/.jenkins/modules/Build/MilvusBuild.groovy
+++ b/.jenkins/modules/Build/MilvusBuild.groovy
@ -3,11 +3,11 @@ timeout(time: 60, unit: 'MINUTES') {
 		def isTimeTriggeredBuild = currentBuild.getBuildCauses('hudson.triggers.TimerTrigger$TimerTriggerCause').size() != 0
 		if (!isTimeTriggeredBuild) {
 			sh ". ./before-install.sh && ./check_cache.sh -l ${params.JFROG_ARTFACTORY_URL}/ccache --cache_dir=\${CCACHE_DIR} -f \${CCACHE_COMPRESS_PACKAGE_FILE} || echo \"ccache files not found!\""
-		    sh ". ./before-install.sh && ./check_cache.sh -l ${params.JFROG_ARTFACTORY_URL}/thirdparty --cache_dir=\${CUSTOM_THIRDPARTY_DOWNLOAD_PATH} -f \${THIRDPARTY_COMPRESS_PACKAGE_FILE} || echo \"thirdparty files not found!\""
+			sh ". ./before-install.sh && ./check_cache.sh -l ${params.JFROG_ARTFACTORY_URL}/thirdparty --cache_dir=\${CUSTOM_THIRDPARTY_DOWNLOAD_PATH} -f \${THIRDPARTY_COMPRESS_PACKAGE_FILE} || echo \"thirdparty files not found!\""
 		}

 		if ("${BINARY_VERSION}" == "gpu") {
-			sh "/bin/bash --login -c \". ./before-install.sh && ./build.sh -t ${params.BUILD_TYPE} -j4 -i ${env.MILVUS_INSTALL_PREFIX} --coverage -l -g -u\""
+			sh "/bin/bash --login -c \". ./before-install.sh && ./build.sh -t ${params.BUILD_TYPE} -j4 -i ${env.MILVUS_INSTALL_PREFIX} --coverage -g -l -u\""
 		} else {
 			sh "/bin/bash --login -c \". ./before-install.sh && ./build.sh -t ${params.BUILD_TYPE} -j4 -i ${env.MILVUS_INSTALL_PREFIX} --coverage -l -u\""
 		}
--- a/ci/scripts/build.sh
+++ b/ci/scripts/build.sh
@ -31,6 +31,7 @@ Usage:
    -g                        Building for the architecture of the GPU in the system
    --with_mkl                Build with MKL (default: OFF)
    --with_fiu                Build with FIU (default: OFF)
+    --tidy                    Run clang-tidy
    -c or --coverage          Build Code Coverage
    -u or --tests             Build unittest case
    -p or --privileges        Install command with elevated privileges
@ -41,7 +42,7 @@ Usage:
 Use \"$0  --help\" for more information about a given command.
 "

-ARGS=`getopt -o "i:t:s:j::lngcupvh" -l "install_prefix::,build_type::,custom_thirdparty::,jobs::,with_mkl,with_fiu,coverage,tests,privileges,help" -n "$0" -- "$@"`
+ARGS=`getopt -o "i:t:s:j::lngcupvh" -l "install_prefix::,build_type::,custom_thirdparty::,jobs::,tidy,with_mkl,with_fiu,coverage,tests,privileges,help" -n "$0" -- "$@"`

 eval set -- "${ARGS}"

@ -73,6 +74,7 @@ while true ; do
                -g) echo "Building for the architecture of the GPU in the system..." ; GPU_VERSION="ON" ; shift ;;
                --with_mkl) echo "Build with MKL" ; WITH_MKL="ON" ; shift ;;
                --with_fiu) echo "Build with FIU" ; FIU_ENABLE="ON" ; shift ;;
+                --tidy) echo "Run clang-tidy" ; RUN_CLANG_TIDY="ON" ; shift ;;
                --coverage) echo "Build code coverage" ; BUILD_COVERAGE="ON" ; shift ;;
                -u|--tests) echo "Build unittest cases" ; BUILD_UNITTEST="ON" ; shift ;;
                -n) echo "No build and install step" ; COMPILE_BUILD="OFF" ; shift ;;
@ -101,6 +103,7 @@ COMPILE_BUILD=${COMPILE_BUILD:="ON"}
 GPU_VERSION=${GPU_VERSION:="OFF"}
 CUDA_ARCH=${CUDA_ARCH:="DEFAULT"}
 RUN_CPPLINT=${RUN_CPPLINT:="OFF"}
+RUN_CLANG_TIDY=${RUN_CLANG_TIDY:="OFF"}
 WITH_MKL=${WITH_MKL:="OFF"}
 FIU_ENABLE=${FIU_ENABLE:="OFF"}
 PRIVILEGES=${PRIVILEGES:="OFF"}
@ -159,19 +162,17 @@ if [[ ${RUN_CPPLINT} == "ON" ]]; then
        exit 1
    fi
    echo "clang-format check passed!"
+fi

+if [[ ${RUN_CLANG_TIDY} == "ON" ]]; then
    # clang-tidy check
-    if [[ ${GPU_VERSION} == "ON" ]]; then
-        make check-clang-tidy
-        if [ $? -ne 0 ]; then
-            echo "ERROR! clang-tidy check failed"
-            rm -f CMakeCache.txt
-            exit 1
-        fi
-        echo "clang-tidy check passed!"
-    else
-        echo "CPU version skip clang-tidy check!"
+    make check-clang-tidy
+    if [ $? -ne 0 ]; then
+        echo "ERROR! clang-tidy check failed"
+        rm -f CMakeCache.txt
+        exit 1
    fi
+    echo "clang-tidy check passed!"
 fi

 if [[ ${COMPILE_BUILD} == "ON" ]];then
--- a/docker-compose.yml
+++ b/docker-compose.yml
@ -3,6 +3,7 @@ version: '3.5'
 volumes:
  amd64-ubuntu-18.04-cache:
  amd64-centos-7-cache:
+  amd64-ubuntu-18.04-cuda-cache:
  amd64-centos-7-cuda-cache:

 x-ccache: &ccache
@ -40,7 +41,7 @@ services:
    networks:
      - milvus
    command: &ubuntu-command
-      ["/milvus/ci/scripts/build.sh -t Release -j2 -i ${MILVUS_INSTALL_PREFIX} --with_fiu --coverage -l -u &&
+      ["/milvus/ci/scripts/build.sh -t Release -j2 -i ${MILVUS_INSTALL_PREFIX} --with_fiu --coverage -u &&
        /milvus/ci/scripts/run_unittest.sh -i ${MILVUS_INSTALL_PREFIX} --mysql_user=root --mysql_password=123456 --mysql_host=mysql &&
        /milvus/ci/scripts/coverage.sh"]

@ -60,10 +61,28 @@ services:
    networks:
      - milvus
    command: &centos-command
-      ["/milvus/ci/scripts/build.sh -t Release -j2 -i ${MILVUS_INSTALL_PREFIX} --with_fiu --coverage -l -u &&
+      ["/milvus/ci/scripts/build.sh -t Release -j2 -i ${MILVUS_INSTALL_PREFIX} --with_fiu --coverage -u &&
        /milvus/ci/scripts/run_unittest.sh -i ${MILVUS_INSTALL_PREFIX} --mysql_user=root --mysql_password=123456 --mysql_host=mysql &&
        /milvus/ci/scripts/coverage.sh"]

+  ubuntu-cuda-core:
+    image: ${REPO}:${ARCH}-ubuntu-${UBUNTU}-cuda-core
+    build:
+      context: .
+      dockerfile: docker/build_env/gpu/ubuntu${UBUNTU}/Dockerfile
+      cache_from:
+        - ${REPO}:${ARCH}-ubuntu-${UBUNTU}-cuda-core
+    shm_size: 2G
+    environment:
+      <<: *ccache
+    volumes: &ubuntu-cuda-volumes
+      - .:/milvus:delegated
+      - ${ARCH}-ubuntu-${UBUNTU}-cuda-cache:/build:delegated
+    networks:
+      - milvus
+    command: &ubuntu-cuda-command
+      ["/milvus/ci/scripts/build.sh -t Release -j2 -i ${MILVUS_INSTALL_PREFIX} -g"]
+
  centos-cuda-core:
    image: ${REPO}:${ARCH}-centos-${CENTOS}-cuda-core
    build:
@ -80,7 +99,39 @@ services:
    networks:
      - milvus
    command: &centos-cuda-command
-      ["/milvus/ci/scripts/build.sh -t Release -j2 -i ${MILVUS_INSTALL_PREFIX} -l -g"]
+      ["/milvus/ci/scripts/build.sh -t Release -j2 -i ${MILVUS_INSTALL_PREFIX} -g"]
+
+  cpplint:
+    image: ${REPO}:${ARCH}-centos-${CENTOS}-core
+    build:
+      context: .
+      dockerfile: docker/build_env/cpu/centos${CENTOS}/Dockerfile
+      cache_from:
+        - ${REPO}:${ARCH}-centos-${CENTOS}-core
+    shm_size: 2G
+    volumes:
+      - .:/milvus:delegated
+      - ${ARCH}-centos-${CENTOS}-cache:/build:delegated
+    networks:
+      - milvus
+    command:
+      ["/milvus/ci/scripts/build.sh -t Release -j2 -i ${MILVUS_INSTALL_PREFIX} -l --tidy -n"]
+
+  cpplint-gpu:
+    image: ${REPO}:${ARCH}-centos-${CENTOS}-cuda-core
+    build:
+      context: .
+      dockerfile: docker/build_env/gpu/centos${CENTOS}/Dockerfile
+      cache_from:
+        - ${REPO}:${ARCH}-centos-${CENTOS}-cuda-core
+    shm_size: 2G
+    volumes:
+      - .:/milvus:delegated
+      - ${ARCH}-centos-${CENTOS}-cuda-cache:/build:delegated
+    networks:
+      - milvus
+    command:
+      ["/milvus/ci/scripts/build.sh -t Release -j2 -i ${MILVUS_INSTALL_PREFIX} -l -g --tidy -n"]

 networks:
  milvus: