From a7ea9b281765d17c961e0f921aacc05bab5ca4bf Mon Sep 17 00:00:00 2001 From: starlord Date: Thu, 12 Sep 2019 11:25:21 +0800 Subject: [PATCH 1/2] remove gpu_cache_capacity config Former-commit-id: 94abdd662201c9b19c79d7b7170f3e9af2c39e81 --- cpp/build.sh | 11 +---------- cpp/conf/server_config.template | 2 -- cpp/coverage.sh | 6 +++++- cpp/src/cache/GpuCacheMgr.cpp | 2 +- cpp/src/server/ServerConfig.cpp | 2 +- 5 files changed, 8 insertions(+), 15 deletions(-) diff --git a/cpp/build.sh b/cpp/build.sh index f21e2234fa..86af2d42e0 100755 --- a/cpp/build.sh +++ b/cpp/build.sh @@ -9,15 +9,12 @@ DB_PATH="/opt/milvus" PROFILING="OFF" BUILD_FAISS_WITH_MKL="OFF" USE_JFROG_CACHE="OFF" -KNOWHERE_BUILD_DIR="`pwd`/src/core/cmake_build" -KNOWHERE_OPTIONS="-t ${BUILD_TYPE}" -while getopts "p:d:t:k:uhrcgmj" arg +while getopts "p:d:t:uhrcgmj" arg do case $arg in t) BUILD_TYPE=$OPTARG # BUILD_TYPE - KNOWHERE_OPTIONS="-t ${BUILD_TYPE}" ;; u) echo "Build and run unittest cases" ; @@ -41,15 +38,11 @@ do g) PROFILING="ON" ;; - k) - KNOWHERE_BUILD_DIR=$OPTARG - ;; m) BUILD_FAISS_WITH_MKL="ON" ;; j) USE_JFROG_CACHE="ON" - KNOWHERE_OPTIONS="${KNOWHERE_OPTIONS} -j" ;; h) # help echo " @@ -62,7 +55,6 @@ parameter: -r: remove previous build directory(default: OFF) -c: code coverage(default: OFF) -g: profiling(default: OFF) --k: specify knowhere header/binary path -m: build faiss with MKL(default: OFF) -j: use jfrog cache build directory @@ -96,7 +88,6 @@ if [[ ${MAKE_CLEAN} == "ON" ]]; then -DMILVUS_DB_PATH=${DB_PATH} \ -DMILVUS_ENABLE_PROFILING=${PROFILING} \ -DBUILD_FAISS_WITH_MKL=${BUILD_FAISS_WITH_MKL} \ - -DKNOWHERE_BUILD_DIR=${KNOWHERE_BUILD_DIR} \ -DUSE_JFROG_CACHE=${USE_JFROG_CACHE} \ ../" echo ${CMAKE_CMD} diff --git a/cpp/conf/server_config.template b/cpp/conf/server_config.template index f1615b360c..804025b3b5 100644 --- a/cpp/conf/server_config.template +++ b/cpp/conf/server_config.template @@ -31,8 +31,6 @@ cache_config: cpu_cache_capacity: 16 # how many memory are used as cache, unit: GB, range: 0 ~ less than total memory cpu_cache_free_percent: 0.85 # old data will be erased from cache when cache is full, this value specify how much memory should be kept, range: greater than zero ~ 1.0 insert_cache_immediately: false # insert data will be load into cache immediately for hot query - gpu_cache_capacity: 5 # how many memory are used as cache in gpu, unit: GB, RANGE: 0 ~ less than total memory - gpu_cache_free_percent: 0.85 # old data will be erased from cache when cache is full, this value specify how much memory should be kept, range: greater than zero ~ 1.0 engine_config: use_blas_threshold: 20 diff --git a/cpp/coverage.sh b/cpp/coverage.sh index 701416921f..156da43a4b 100755 --- a/cpp/coverage.sh +++ b/cpp/coverage.sh @@ -13,6 +13,10 @@ DIR_LCOV_OUTPUT="lcov_out" DIR_GCNO="cmake_build" DIR_UNITTEST="milvus/unittest" + +# delete old code coverage info files +rm -rf lcov_out +rm -f FILE_INFO_BASE FILE_INFO_MILVUS FILE_INFO_OUTPUT FILE_INFO_OUTPUT_NEW MYSQL_USER_NAME=root MYSQL_PASSWORD=Fantast1c @@ -84,7 +88,7 @@ done mysql_exc "DROP DATABASE IF EXISTS ${MYSQL_DB_NAME};" -# gen test converage +# gen code coverage ${LCOV_CMD} -d ${DIR_GCNO} -o "${FILE_INFO_MILVUS}" -c # merge coverage ${LCOV_CMD} -a ${FILE_INFO_BASE} -a ${FILE_INFO_MILVUS} -o "${FILE_INFO_OUTPUT}" diff --git a/cpp/src/cache/GpuCacheMgr.cpp b/cpp/src/cache/GpuCacheMgr.cpp index 0b6a6132d8..5c1afa3f5c 100644 --- a/cpp/src/cache/GpuCacheMgr.cpp +++ b/cpp/src/cache/GpuCacheMgr.cpp @@ -23,7 +23,7 @@ namespace { GpuCacheMgr::GpuCacheMgr() { server::ConfigNode& config = server::ServerConfig::GetInstance().GetConfig(server::CONFIG_CACHE); - int64_t cap = config.GetInt64Value(server::CONFIG_GPU_CACHE_CAPACITY, 2); + int64_t cap = config.GetInt64Value(server::CONFIG_GPU_CACHE_CAPACITY, 0); cap *= G_BYTE; cache_ = std::make_shared(cap, 1UL<<32); diff --git a/cpp/src/server/ServerConfig.cpp b/cpp/src/server/ServerConfig.cpp index da898d2239..9934cdc5ad 100644 --- a/cpp/src/server/ServerConfig.cpp +++ b/cpp/src/server/ServerConfig.cpp @@ -305,7 +305,7 @@ ServerConfig::CheckCacheConfig() { okay = false; } - std::string gpu_cache_capacity_str = cache_config.GetValue(CONFIG_GPU_CACHE_CAPACITY, "5"); + std::string gpu_cache_capacity_str = cache_config.GetValue(CONFIG_GPU_CACHE_CAPACITY, "0"); if (ValidationUtil::ValidateStringIsNumber(gpu_cache_capacity_str) != SERVER_SUCCESS) { std::cerr << "ERROR: gpu_cache_capacity " << gpu_cache_capacity_str << " is not a number" << std::endl; okay = false; From b2a20f68c00a4ac19c06fce17b3079b098cd3351 Mon Sep 17 00:00:00 2001 From: starlord Date: Thu, 12 Sep 2019 11:39:16 +0800 Subject: [PATCH 2/2] remove knowhere path from jenkins build script Former-commit-id: ce5cb318fc129a0dea7596dc2133fdc740700167 --- ci/jenkinsfile/milvus_build.groovy | 4 +--- ci/jenkinsfile/milvus_build_no_ut.groovy | 4 +--- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/ci/jenkinsfile/milvus_build.groovy b/ci/jenkinsfile/milvus_build.groovy index 0eb9cd7c99..0e24c63a34 100644 --- a/ci/jenkinsfile/milvus_build.groovy +++ b/ci/jenkinsfile/milvus_build.groovy @@ -3,8 +3,6 @@ container('milvus-build-env') { gitlabCommitStatus(name: 'Build Engine') { dir ("milvus_engine") { try { - def knowhere_build_dir = "${env.WORKSPACE}/milvus_engine/cpp/src/core/cmake_build" - checkout([$class: 'GitSCM', branches: [[name: "${SEMVER}"]], doGenerateSubmoduleConfigurations: false, extensions: [[$class: 'SubmoduleOption',disableSubmodules: false,parentCredentials: true,recursiveSubmodules: true,reference: '',trackingSubmodules: false]], submoduleCfg: [], userRemoteConfigs: [[credentialsId: "${params.GIT_USER}", url: "git@192.168.1.105:megasearch/milvus.git", name: 'origin', refspec: "+refs/heads/${SEMVER}:refs/remotes/origin/${SEMVER}"]]]) /* @@ -17,7 +15,7 @@ container('milvus-build-env') { dir ("cpp") { sh "git config --global user.email \"test@zilliz.com\"" sh "git config --global user.name \"test\"" - sh "./build.sh -t ${params.BUILD_TYPE} -k ${knowhere_build_dir} -j -u -c" + sh "./build.sh -t ${params.BUILD_TYPE} -j -u -c" } } catch (exc) { updateGitlabCommitStatus name: 'Build Engine', state: 'failed' diff --git a/ci/jenkinsfile/milvus_build_no_ut.groovy b/ci/jenkinsfile/milvus_build_no_ut.groovy index 8bbb568fee..dd0fe4a442 100644 --- a/ci/jenkinsfile/milvus_build_no_ut.groovy +++ b/ci/jenkinsfile/milvus_build_no_ut.groovy @@ -3,8 +3,6 @@ container('milvus-build-env') { gitlabCommitStatus(name: 'Build Engine') { dir ("milvus_engine") { try { - def knowhere_build_dir = "${env.WORKSPACE}/milvus_engine/cpp/src/core/cmake_build" - checkout([$class: 'GitSCM', branches: [[name: "${SEMVER}"]], doGenerateSubmoduleConfigurations: false, extensions: [[$class: 'SubmoduleOption',disableSubmodules: false,parentCredentials: true,recursiveSubmodules: true,reference: '',trackingSubmodules: false]], submoduleCfg: [], userRemoteConfigs: [[credentialsId: "${params.GIT_USER}", url: "git@192.168.1.105:megasearch/milvus.git", name: 'origin', refspec: "+refs/heads/${SEMVER}:refs/remotes/origin/${SEMVER}"]]]) /* @@ -17,7 +15,7 @@ container('milvus-build-env') { dir ("cpp") { sh "git config --global user.email \"test@zilliz.com\"" sh "git config --global user.name \"test\"" - sh "./build.sh -t ${params.BUILD_TYPE} -k ${knowhere_build_dir} -j" + sh "./build.sh -t ${params.BUILD_TYPE} -j" } } catch (exc) { updateGitlabCommitStatus name: 'Build Engine', state: 'failed'