Fix jemalloc.so not found (#21106)

Signed-off-by: Enwei Jiao <enwei.jiao@zilliz.com>

Signed-off-by: Enwei Jiao <enwei.jiao@zilliz.com>

.github/mergify.yml

@@ -85,7 +85,7 @@ pull_request_rules:
       - or:
         - base=master
         - base~=^2(\.\d+){2}$
-      - -files~=^(?=.*((\.(go|h|cpp)|CMakeLists.txt))).*$
+      - -files~=^(?=.*((\.(go|h|cpp)|CMakeLists.txt|conanfile\.*))).*$
       - "status-success=cpu-e2e"
     actions:
       label:
@@ -320,4 +320,4 @@ pull_request_rules:
     actions:
       label:
         remove:
-          - needs-rebase
+          - needs-rebase

.github/workflows/code-checker.yaml

@@ -51,12 +51,6 @@ jobs:
           path: .docker/amd64-ubuntu20.04-ccache
           key: ubuntu20.04-ccache-${{ env.corehash }}
           restore-keys: ubuntu20.04-ccache-
-      - name: Cache Third Party
-        uses: actions/cache@v3
-        with:
-          path: .docker/thirdparty
-          key: ubuntu20.04-thirdparty-${{ hashFiles('internal/core/thirdparty/**') }}
-          restore-keys: ubuntu20.04-thirdparty-
       - name: Cache Go Mod Volumes
         uses: actions/cache@v3
         with:
@@ -67,7 +61,7 @@ jobs:
         uses: actions/cache@v3
         with:
           path: .docker/amd64-ubuntu20.04-conan
-          key: ubuntu20.04-conan-${{ hashFiles('internal/core/conanfile.txt') }}
+          key: ubuntu20.04-conan-${{ hashFiles('internal/core/conanfile.*') }}
           restore-keys: ubuntu20.04-conan-
       - name: Code Check
         env:

.github/workflows/main.yaml

@@ -58,12 +58,6 @@ jobs:
           path: .docker/amd64-ubuntu${{ matrix.ubuntu }}-ccache
           key: ubuntu${{ matrix.ubuntu }}-ccache-${{ env.corehash }}
           restore-keys: ubuntu${{ matrix.ubuntu }}-ccache-
-      - name: Cache Third Party
-        uses: actions/cache@v3
-        with:
-          path: .docker/thirdparty
-          key: ubuntu${{ matrix.ubuntu }}-thirdparty-${{ hashFiles('internal/core/thirdparty/**') }}
-          restore-keys: ubuntu${{ matrix.ubuntu }}-thirdparty-
       - name: Cache Go Mod Volumes
         uses: actions/cache@v3
         with:
@@ -74,7 +68,7 @@ jobs:
         uses: actions/cache@v3
         with:
           path: .docker/amd64-ubuntu${{ matrix.ubuntu }}-conan
-          key: ubuntu${{ matrix.ubuntu }}-conan-${{ hashFiles('internal/core/conanfile.txt') }}
+          key: ubuntu${{ matrix.ubuntu }}-conan-${{ hashFiles('internal/core/conanfile.*') }}
           restore-keys: ubuntu${{ matrix.ubuntu }}-conan-
       - name: Start Service
         shell: bash
@@ -114,7 +108,7 @@ jobs:
       - name: Display structure of code coverage results
         run: |
           ls -lah
-          
+
       - name: Upload coverage to Codecov
         if: "github.repository == 'milvus-io/milvus'"
         uses: codecov/codecov-action@v3.1.1

internal/core/conanfile.py

@@ -0,0 +1,46 @@
+from conans import ConanFile, CMake
+
+
+class MilvusConan(ConanFile):
+
+    settings = "os", "compiler", "build_type", "arch"
+    requires = (
+        "rocksdb/6.29.5",
+        "boost/1.80.0",
+        "onetbb/2021.7.0",
+        "zstd/1.5.2",
+        "arrow/8.0.1",
+        "openssl/1.1.1q",
+        "aws-sdk-cpp/1.9.234",
+        "benchmark/1.7.0",
+        "gtest/1.8.1",
+        "protobuf/3.9.1",
+        "rapidxml/1.13",
+        "yaml-cpp/0.7.0",
+        "marisa/0.2.6",
+        "zlib/1.2.13",
+    )
+    generators = "cmake"
+    default_options = {
+        "rocksdb:shared": True,
+        "arrow:parquet": True,
+        "arrow:compute": True,
+        "arrow:with_zstd": True,
+        "aws-sdk-cpp:text-to-speech": False,
+        "aws-sdk-cpp:transfer": False,
+        "gtest:build_gmock": False,
+        "jemalloc:shared": True,
+    }
+    should_build = False
+
+    def configure(self):
+
+        # Macos M1 cannot use jemalloc
+        if self.settings.os != "Macos" or self.settings.arch in ("x86_64", "x86"):
+            self.requires("jemalloc/5.3.0")
+
+    def imports(self):
+        self.copy("*.dylib", "../lib", "lib")
+        self.copy("*.dll", "../lib", "lib")
+        self.copy("*.so*", "../lib", "lib")
+        self.copy("*", "../bin", "bin")

internal/core/conanfile.txt

@@ -1,33 +0,0 @@
-[requires]
-rocksdb/6.29.5
-boost/1.80.0
-onetbb/2021.3.0
-zstd/1.5.2
-arrow/8.0.1
-openssl/1.1.1q
-aws-sdk-cpp/1.9.234
-benchmark/1.7.0
-gtest/1.8.1
-protobuf/3.9.1
-rapidxml/1.13
-yaml-cpp/0.7.0
-marisa/0.2.6
-zlib/1.2.13
-
-[generators]
-cmake
-
-[options]
-rocksdb:shared=True
-arrow:parquet=True
-arrow:compute=True
-arrow:with_zstd=True
-aws-sdk-cpp:text-to-speech=False
-aws-sdk-cpp:transfer=False
-gtest:build_gmock=False
-
-[imports]
-bin, * -> ../bin
-lib, *.dylib -> ../lib
-lib, *.dll -> ../lib
-lib, *.so* -> ../lib

scripts/setenv.sh

@@ -34,9 +34,14 @@ ROOT_DIR="$( cd -P "$( dirname "$SOURCE" )/.." && pwd )"
 
 unameOut="$(uname -s)"
 
-
 case "${unameOut}" in
     Linux*)     
+      LIBJEMALLOC=$PWD/internal/core/output/lib/libjemalloc.so
+      if test -f "$LIBJEMALLOC"; then
+        export LD_PRELOAD="$LIBJEMALLOC"
+      else
+        echo "WARN: Cannot find $LIBJEMALLOC"
+      fi
       export PKG_CONFIG_PATH="${PKG_CONFIG_PATH}:$ROOT_DIR/internal/core/output/lib/pkgconfig:$ROOT_DIR/internal/core/output/lib64/pkgconfig"
       export LD_LIBRARY_PATH="${LD_LIBRARY_PATH}:$ROOT_DIR/internal/core/output/lib:$ROOT_DIR/internal/core/output/lib64"
       export RPATH=$LD_LIBRARY_PATH;;