enhance: enable asan for cpp unittest (#37041)

https://github.com/milvus-io/milvus/issues/35854 Signed-off-by: chyezh <chyezh@outlook.com> Co-authored-by: chyezh <chyezh@outlook.com>
2024-10-23 17:21:27 +08:00 · 2024-10-23 17:21:27 +08:00 · 8902e2220e
parent 94febdf7d7
commit 8902e2220e
10 changed files with 57 additions and 46 deletions
--- a/.github/workflows/main.yaml
+++ b/.github/workflows/main.yaml
@ -112,6 +112,8 @@ jobs:
          kind: 'cpp'

  UT-Cpp:
+    # skip the UT-Cpp job in github workflow, it run in jenkins now see: UT-CPP.groovy
+    if: false 
    name: UT for Cpp
    needs: Build
    runs-on: ubuntu-latest
@ -268,7 +270,7 @@ jobs:

  codecov:
    name: Upload Code Coverage
-    needs: [UT-Cpp, UT-Go, integration-test]
+    needs: [UT-Go, integration-test]
    runs-on: ubuntu-latest
    timeout-minutes: 5
    steps:
@ -276,10 +278,6 @@ jobs:
        uses: actions/checkout@v2
        with:
          fetch-depth: 0
-      - name: Download Cpp code coverage results
-        uses: actions/download-artifact@v4.1.0
-        with:
-          name: cpp-result
      - name: Download Go code coverage results
        uses: actions/download-artifact@v4.1.0
        with:
@ -297,7 +295,7 @@ jobs:
        id: upload_cov
        with:
          token: ${{ secrets.CODECOV_TOKEN }}
-          files: ./go_coverage.txt,./lcov_output.info,./it_coverage.txt
+          files: ./go_coverage.txt,./it_coverage.txt
          name: ubuntu-20.04-unittests
          fail_ci_if_error: true
          disable_safe_directory: true
@ -307,7 +305,7 @@ jobs:
        id: retry_upload_cov
        with:
          token: ${{ secrets.CODECOV_TOKEN }}
-          files: ./go_coverage.txt,./lcov_output.info,./it_coverage.txt
+          files: ./go_coverage.txt,./it_coverage.txt
          name: ubuntu-20.04-unittests
          fail_ci_if_error: true
          disable_safe_directory: true
--- a/4
+++ b/4
@ -30,8 +30,8 @@ endif
 use_asan = OFF
 ifdef USE_ASAN
 	use_asan =${USE_ASAN}
-	CGO_LDFLAGS := $(shell go env CGO_LDFLAGS) -fsanitize=address -fno-omit-frame-pointer
-	CGO_CFLAGS := $(shell go env CGO_CFLAGS) -fsanitize=address -fno-omit-frame-pointer
+	CGO_LDFLAGS := $(shell go env CGO_LDFLAGS) -fno-stack-protector -fno-omit-frame-pointer -fno-var-tracking -fsanitize=address
+	CGO_CFLAGS := $(shell go env CGO_CFLAGS) -fno-stack-protector -fno-omit-frame-pointer -fno-var-tracking -fsanitize=address
 	MILVUS_GO_BUILD_TAGS := $(MILVUS_GO_BUILD_TAGS),use_asan
 endif

--- a/ci/jenkins/UT-CPP.groovy
+++ b/ci/jenkins/UT-CPP.groovy
@ -1,4 +1,4 @@
-@Library('jenkins-shared-library@tekton') _
+@Library('jenkins-shared-library@v0.62.0') _

 def pod = libraryResource 'io/milvus/pod/tekton-4am.yaml'
 def milvus_helm_chart_version = '4.2.8'
@ -38,7 +38,7 @@ pipeline {
                                              gitMode: gitMode ,
                                              gitBaseRef: gitBaseRef,
                                              pullRequestNumber: "$env.CHANGE_ID",
-                                              make_cmd: "make clean && make USE_ASAN=OFF build-cpp-with-coverage",
+                                              make_cmd: "make clean && make USE_ASAN=ON build-cpp-with-coverage",
                                              test_entrypoint: "./scripts/run_cpp_codecov.sh",
                                              codecov_files: "./lcov_output.info,./it_coverage.txt"
                    }
--- a/internal/core/CMakeLists.txt
+++ b/internal/core/CMakeLists.txt
@ -131,8 +131,8 @@ if (LINUX OR MSYS)
                  )
    if (USE_ASAN STREQUAL "ON") 
        message( STATUS "Building Milvus Core Using AddressSanitizer")
-        add_compile_options(-fno-omit-frame-pointer -fsanitize=address)
-        add_link_options(-fno-omit-frame-pointer -fsanitize=address)
+        add_compile_options(-fno-stack-protector -fno-omit-frame-pointer -fno-var-tracking -fsanitize=address)
+        add_link_options(-fno-stack-protector -fno-omit-frame-pointer -fno-var-tracking -fsanitize=address)
    endif()
    if (CMAKE_BUILD_TYPE STREQUAL "Release")
        append_flags( CMAKE_CXX_FLAGS
--- a/internal/core/src/segcore/vector_index_c.cpp
+++ b/internal/core/src/segcore/vector_index_c.cpp
@ -27,7 +27,7 @@ GetIndexListSize() {

 void
 GetIndexFeatures(void* index_key_list, uint64_t* index_feature_list) {
-    auto features = knowhere::IndexFactory::Instance().GetIndexFeatures();
+    auto& features = knowhere::IndexFactory::Instance().GetIndexFeatures();
    int idx = 0;

    const char** index_keys = (const char**)index_key_list;
--- a/internal/core/unittest/test_storage.cpp
+++ b/internal/core/unittest/test_storage.cpp
@ -38,21 +38,24 @@ get_azure_storage_config() {
        "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/"
        "K1SZFPTOtr/KBHBeksoGMGw==";

-    return CStorageConfig{endpoint,
-                          bucketName.c_str(),
-                          accessKey,
-                          accessValue,
-                          rootPath.c_str(),
-                          "remote",
-                          "azure",
-                          "",
-                          "error",
-                          "",
-                          false,
-                          "",
-                          false,
-                          false,
-                          30000};
+    return CStorageConfig{
+        endpoint,
+        bucketName.c_str(),
+        accessKey,
+        accessValue,
+        rootPath.c_str(),
+        "remote",
+        "azure",
+        "",
+        "error",
+        "",
+        false,
+        "",
+        false,
+        false,
+        30000,
+        "",
+    };
 }

 class StorageTest : public testing::Test {
@ -93,7 +96,9 @@ TEST_F(StorageTest, GetLocalUsedSize) {

 TEST_F(StorageTest, InitRemoteChunkManagerSingleton) {
    CStorageConfig storageConfig = get_azure_storage_config();
-    InitRemoteChunkManagerSingleton(storageConfig);
+    auto status = InitRemoteChunkManagerSingleton(storageConfig);
+    EXPECT_STREQ(status.error_msg, "");
+    EXPECT_EQ(status.error_code, Success);
    auto rcm =
        RemoteChunkManagerSingleton::GetInstance().GetRemoteChunkManager();
    EXPECT_EQ(rcm->GetRootPath(), "/tmp/milvus/remote_data");
--- a/scripts/core_build.sh
+++ b/scripts/core_build.sh
@ -138,7 +138,6 @@ while getopts "p:d:t:s:f:n:i:y:a:x:o:ulrcghzmebZ" arg; do
    if [[ ${ENV_VAL} == 'ON' ]]; then
        echo "Set USE_ASAN to ON"
        USE_ASAN="ON"
-        BUILD_TYPE=Debug
    fi
    ;;
  y)
--- a/scripts/install_milvus.sh
+++ b/scripts/install_milvus.sh
@ -24,8 +24,6 @@ cp $PWD"/internal/core/output/lib/"*.dylib* "$LIBRARY_PATH" 2>/dev/null || true
 cp $PWD"/internal/core/output/lib/"*.so* "$LIBRARY_PATH" || true
 cp $PWD"/internal/core/output/lib64/"*.so* "$LIBRARY_PATH" 2>/dev/null || true

-if [ "$USE_ASAN" == "ON" ]; then
-    for LIB_PATH in $(ldconfig -p | grep -E '(asan|atomic)' | awk '{print $NF}'); do
-        cp "$LIB_PATH" "$LIBRARY_PATH" 2>/dev/null
-    done
-fi
+for LIB_PATH in $(ldd ./bin/milvus | grep -E '(asan|atomic)' | awk '{print $3}'); do
+    cp "$LIB_PATH" "$LIBRARY_PATH" 2>/dev/null
+done
--- a/scripts/run_cpp_codecov.sh
+++ b/scripts/run_cpp_codecov.sh
@ -21,11 +21,11 @@ set -e

 SOURCE="${BASH_SOURCE[0]}"
 while [ -h "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a symlink
-  DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )"
-  SOURCE="$(readlink "$SOURCE")"
-  [[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE" # if $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located
+    DIR="$(cd -P "$(dirname "$SOURCE")" && pwd)"
+    SOURCE="$(readlink "$SOURCE")"
+    [[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE" # if $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located
 done
-ROOT_DIR="$( cd -P "$( dirname "$SOURCE" )/.." && pwd )"
+ROOT_DIR="$(cd -P "$(dirname "$SOURCE")/.." && pwd)"
 source ${ROOT_DIR}/scripts/setenv.sh

 MILVUS_CORE_DIR="${ROOT_DIR}/internal/core"
@ -59,13 +59,18 @@ if [ $? -ne 0 ]; then
    exit -1
 fi
 # starting the timer
-beginTime=`date +%s`
+beginTime=$(date +%s)

 # run unittest
-for test in `ls ${MILVUS_CORE_UNITTEST_DIR}`; do
+for test in $(ls ${MILVUS_CORE_UNITTEST_DIR}); do
    echo "Running cpp unittest: ${MILVUS_CORE_UNITTEST_DIR}/$test"
    # run unittest
-    ${MILVUS_CORE_UNITTEST_DIR}/${test}
+    if [ -n "$MILVUS_ENABLE_ASAN_LIB" ]; then
+        echo "ASAN is enabled with env MILVUS_ENABLE_ASAN_LIB, set {$MILVUS_ENABLE_ASAN_LIB} at the front of LD_PRELOAD"
+        LD_PRELOAD="$MILVUS_ENABLE_ASAN_LIB:$LD_PRELOAD" ${MILVUS_CORE_UNITTEST_DIR}/${test}
+    else
+        ${MILVUS_CORE_UNITTEST_DIR}/${test}
+    fi
    if [ $? -ne 0 ]; then
        echo ${args}
        echo "${MILVUS_CORE_UNITTEST_DIR}/${test} run failed"
@ -94,7 +99,6 @@ ${LCOV_CMD} -r "${FILE_INFO_COMBINE}" -o "${FILE_INFO_OUTPUT}" \
 ${LCOV_GEN_CMD} ${FILE_INFO_OUTPUT} --output-directory ${DIR_LCOV_OUTPUT}/
 echo "Generate cpp code coverage report to ${DIR_LCOV_OUTPUT}"

+endTime=$(date +%s)

-endTime=`date +%s`
-
-echo "Total time for cpp unittest:" $(($endTime-$beginTime)) "s"
+echo "Total time for cpp unittest:" $(($endTime - $beginTime)) "s"
--- a/scripts/setenv.sh
+++ b/scripts/setenv.sh
@ -36,6 +36,13 @@ unameOut="$(uname -s)"

 case "${unameOut}" in
    Linux*)
+      # check if use asan.
+      MILVUS_ENABLE_ASAN_LIB=$(ldd $ROOT_DIR/internal/core/output/lib/libmilvus_core.so | grep asan | awk '{print $3}')
+      if [ -n "$MILVUS_ENABLE_ASAN_LIB" ]; then
+          echo "Enable ASAN With ${MILVUS_ENABLE_ASAN_LIB}"
+          export MILVUS_ENABLE_ASAN_LIB="$MILVUS_ENABLE_ASAN_LIB"
+      fi
+
      LIBJEMALLOC=$PWD/internal/core/output/lib/libjemalloc.so
      if test -f "$LIBJEMALLOC"; then
        export LD_PRELOAD="$LIBJEMALLOC"