Add offline build logic for arrow dependency (#17266)

Signed-off-by: zhenshan.cao <zhenshan.cao@zilliz.com>

.github/workflows/code-checker.yaml

@@ -43,37 +43,37 @@ jobs:
         uses: actions/checkout@v2
       - name: Cache CCache Volumes
         if: ${{  matrix.os == 'ubuntu-18.04' }}
-        uses: actions/cache@v1
+        uses: actions/cache@v3
         with:
           path: .docker/amd64-ubuntu18.04-ccache
           key: ubuntu18.04-ccache-${{ hashFiles('internal/core/**') }}
           restore-keys: ubuntu18.04-ccache-
       - name: Cache Third Party
         if: ${{  matrix.os == 'ubuntu-18.04' }}
-        uses: actions/cache@v1
+        uses: actions/cache@v3
         with:
           path: .docker/thirdparty
-          key: ubuntu18.04-thirdparty-${{ hashFiles('internal/core/**') }}
+          key: ubuntu18.04-thirdparty-${{ hashFiles('internal/core/thirdparty/**') }}
           restore-keys: ubuntu18.04-thirdparty-
       - name: Mac Cache Third Party
         if: ${{  matrix.os == 'macos-11' }}
-        uses: actions/cache@v1
+        uses: actions/cache@v3
         with:
           path: |
             /tmp/thirdparty 
-            cmake_build/3rdparty_download
-          key: macos-11-thirdparty-${{ hashFiles('internal/core/**') }}
+            cmake_build/3rdparty_download/download
+          key: macos-11-thirdparty-${{ hashFiles('internal/core/thirdparty/**') }}
           restore-keys: macos-11-thirdparty-
       - name: Mac Cache CCache Volumes
         if: ${{  matrix.os == 'macos-11' }}
-        uses: actions/cache@v1
+        uses: actions/cache@v3
         with:
           path: /var/tmp/ccache
           key: macos-11-ccache-${{ hashFiles('internal/core/**') }}
           restore-keys: macos-11-ccache-
       - name: Cache Go Mod Volumes
         if: ${{  matrix.os == 'ubuntu-18.04' }}
-        uses: actions/cache@v1
+        uses: actions/cache@v3
         with:
           path: .docker/amd64-ubuntu18.04-go-mod
           key: ubuntu18.04-go-mod-${{ hashFiles('**/go.sum') }}
@@ -85,7 +85,7 @@ jobs:
           go-version: '~1.17.1'
       - name: Mac Cache Go Mod Volumes
         if: ${{  matrix.os == 'macos-11' }}
-        uses: actions/cache@v1
+        uses: actions/cache@v3
         with:
           path: ~/go/pkg/mod
           key: macos-11-go-mod-${{ hashFiles('**/go.sum') }}
@@ -110,7 +110,7 @@ jobs:
             mkdir -p /var/tmp/ccache
           fi
           brew install boost libomp ninja tbb openblas ccache
-          make check-proto-product && make verifiers
+          make check-proto-product && make verifiers && sh scripts/collect_arrow_dep.sh -f /tmp/thirdparty
   centos:
     name: Code Checker CentOS 7
     # Run in centos docker 
@@ -120,19 +120,19 @@ jobs:
       - name: Checkout
         uses: actions/checkout@v2
       - name: Cache CCache Volumes
-        uses: actions/cache@v2
+        uses: actions/cache@v3
         with:
           path: .docker/amd64-centos7-ccache
           key: centos7-ccache-${{ hashFiles('internal/core/**') }}
           restore-keys: centos7-ccache-
       - name: Cache Third Party
-        uses: actions/cache@v2
+        uses: actions/cache@v3
         with:
           path: .docker/thirdparty
-          key: centos7-thirdparty-${{ hashFiles('internal/core/**') }}
+          key: centos7-thirdparty-${{ hashFiles('internal/core/thirdparty/**') }}
           restore-keys: centos7-thirdparty-
       - name: Cache Go Mod Volumes
-        uses: actions/cache@v2
+        uses: actions/cache@v3
         with:
           path: .docker/amd64-centos7-go-mod
           key: centos7-go-mod-${{ hashFiles('**/go.sum') }}
@@ -161,7 +161,7 @@ jobs:
       - name: Checkout
         uses: actions/checkout@v2
       - name: Cache go
-        uses: actions/cache@v2
+        uses: actions/cache@v3
         with:
           path: |
             %LocalAppData%\go-build
@@ -169,7 +169,7 @@ jobs:
           key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
           restore-keys: ${{ runner.os }}-go-
       - name: Cache ccache
-        uses: actions/cache@v2
+        uses: actions/cache@v3
         with:
           path: |
             ${{ github.workspace }}/.ccache

scripts/collect_arrow_dep.sh

@@ -0,0 +1,106 @@
+#!/usr/bin/env bash
+
+# Licensed to the LF AI & Data foundation under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# assume we are in milvus directory
+
+CUSTOM_THIRDPARTY_PATH=
+
+while getopts "f:h" arg; do
+  case $arg in
+  f)
+    CUSTOM_THIRDPARTY_PATH=$OPTARG
+    ;;
+
+  h) # help
+    echo "
+
+parameter:
+-f: custom paths of thirdparty downloaded files(default: NULL)
+-h: help
+
+usage:
+./collect_arrow_dep.sh -f\${CUSTOM_THIRDPARTY_PATH} [-h]
+                "
+    exit 0
+    ;;
+  ?)
+    echo "ERROR! unknown argument"
+    exit 1
+    ;;
+  esac
+done
+
+
+if [ -z "$CUSTOM_THIRDPARTY_PATH" ]
+then
+	echo "
+	parameter:
+	-f: custom paths of thirdparty downloaded files(default: NULL)
+	-h: help
+
+	usage:
+	./core_build.sh -f\${CUSTOM_THIRDPARTY_PATH} [-h]
+			"
+    	exit 0
+fi
+
+if [ ! -d "$CUSTOM_THIRDPARTY_PATH" ]
+then
+	echo "${CUSTOM_THIRDPARTY_PATH} is not a directory"
+	exit 0
+fi
+
+SOURCE="${BASH_SOURCE[0]}"
+while [ -h "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a symlink
+  DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )"
+  SOURCE="$(readlink "$SOURCE")"
+  [[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE" # if $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located
+done
+SCRIPTS_DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )"
+
+BUILD_OUTPUT_DIR="${SCRIPTS_DIR}/../cmake_build"
+ARROW_DIR=${BUILD_OUTPUT_DIR}/thirdparty/arrow 
+ARROWBIN_DIR=${BUILD_OUTPUT_DIR}/thirdparty/arrow/arrow-bin
+
+echo ${ARROWBIN_DIR}
+
+pushd ${ARROWBIN_DIR}
+targetNames=(
+"jemalloc-5.2.1.tar.bz2"
+"thrift-0.13.0.tar.gz"
+"utf8proc-v2.7.0.tar.gz"
+"xsimd-7d1778c3b38d63db7cec7145d939f40bc5d859d1.tar.gz"
+"zstd-v1.5.1.tar.gz"
+)
+
+srcNames=(
+"jemalloc_ep-prefix/src/jemalloc-5.2.1.tar.bz2"
+"thrift_ep-prefix/src/thrift-0.13.0.tar.gz"
+"utf8proc_ep-prefix/src/v2.7.0.tar.gz"
+"src/7d1778c3b38d63db7cec7145d939f40bc5d859d1.tar.gz"
+"zstd_ep-prefix/src/v1.5.1.tar.gz"
+)
+
+for i in "${!srcNames[@]}"; do
+   if test -f "${srcNames[i]}"; then
+	echo "${srcNames[i]} exists. start to copy to ${CUSTOM_THIRDPARTY_PATH}/${targetNames[i]}"
+	echo cp -f "${srcNames[i]}"  "${CUSTOM_THIRDPARTY_PATH}/${targetNames[i]}"
+	cp  -f "${srcNames[i]}" "${CUSTOM_THIRDPARTY_PATH}/${targetNames[i]}"
+   fi
+done
+popd

scripts/core_build.sh

@@ -204,6 +204,30 @@ echo ${CMAKE_CMD}
 ${CMAKE_CMD} -G "${CMAKE_GENERATOR}"
 
 
+# enable offline build of arrow dependency if files exist.
+arrowDepKeys=(
+"ARROW_JEMALLOC_URL"
+"ARROW_THRIFT_URL"
+"ARROW_UTF8PROC_URL"
+"ARROW_XSIMD_URL"
+"ARROW_ZSTD_URL"
+)
+arrowDepValues=(
+"jemalloc-5.2.1.tar.bz2"
+"thrift-0.13.0.tar.gz"
+"utf8proc-v2.7.0.tar.gz"
+"xsimd-7d1778c3b38d63db7cec7145d939f40bc5d859d1.tar.gz"
+"zstd-v1.5.1.tar.gz"
+)
+for i in "${!arrowDepValues[@]}"; do
+   if test -f "${CUSTOM_THIRDPARTY_PATH}/${arrowDepValues[$i]}"; then
+	echo "${arrowDepValues[$i]} exists."
+	export ${arrowDepKeys[$i]}=${CUSTOM_THIRDPARTY_PATH}/${arrowDepValues[$i]}
+   fi
+done
+
+set
+
 if [[ ${RUN_CPPLINT} == "ON" ]]; then
   # cpplint check
   make lint
@@ -233,4 +257,10 @@ else
   make -j ${jobs} install || exit 1
 fi
 
+if command -v ccache &> /dev/null
+then
+	ccache -s
+    exit
+fi
+
 popd