From a5f4484f9d760385a3f972c84eb4377c670fd9ec Mon Sep 17 00:00:00 2001 From: "G.Y Feng" <40229765+GuanyunFeng@users.noreply.github.com> Date: Mon, 18 Nov 2019 18:31:12 +0800 Subject: [PATCH 1/8] Update README_CN.md Add a link to C++ SDK in README_CN #383 --- README_CN.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README_CN.md b/README_CN.md index 5dad64af9b..8f4d4b05aa 100644 --- a/README_CN.md +++ b/README_CN.md @@ -20,7 +20,7 @@ Milvus 是一款开源的、针对海量特征向量的相似性搜索引擎。基于异构众核计算框架设计，成本更低，性能更好。在有限的计算资源下，十亿向量搜索仅毫秒响应。 -Milvus 提供稳定的 Python、Java 以及 C++ 的 API 接口。 +Milvus 提供稳定的 [Python](https://github.com/milvus-io/pymilvus)、[Java](https://github.com/milvus-io/milvus-sdk-java) 以及 [C++](https://github.com/milvus-io/milvus/tree/master/core/src/sdk) 的 API 接口。 通过 [版本发布说明](https://milvus.io/docs/zh-CN/release/v0.5.0/) 获取最新发行版本的 Milvus。 From 039f23e08205831017022d2dcab560d5f279431c Mon Sep 17 00:00:00 2001 From: "G.Y Feng" <40229765+GuanyunFeng@users.noreply.github.com> Date: Mon, 18 Nov 2019 18:32:32 +0800 Subject: [PATCH 2/8] Update README.md Add a link to C++ SDK in README --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 311d0d6843..4a1151dbf8 100644 --- a/README.md +++ b/README.md @@ -16,7 +16,7 @@ Milvus is the world's fastest similarity search engine for massive-scale feature For more detailed introduction of Milvus and its architecture, see [Milvus overview](https://www.milvus.io/docs/en/aboutmilvus/overview/). -Milvus provides stable [Python](https://github.com/milvus-io/pymilvus), [Java](https://github.com/milvus-io/milvus-sdk-java) and C++ APIs. +Milvus provides stable [Python](https://github.com/milvus-io/pymilvus), [Java](https://github.com/milvus-io/milvus-sdk-java) and [C++](https://github.com/milvus-io/milvus/tree/master/core/src/sdk) APIs. Keep up-to-date with newest releases and latest updates by reading Milvus [release notes](https://www.milvus.io/docs/en/release/v0.5.0/). From 7e67c76e2c5c9a5791de18e9d5ed131f03ef1681 Mon Sep 17 00:00:00 2001 From: "G.Y Feng" <40229765+GuanyunFeng@users.noreply.github.com> Date: Mon, 18 Nov 2019 18:34:31 +0800 Subject: [PATCH 3/8] Update ubuntu_build_deps.sh solve issue #387, "failed to create symbolic link '/usr/lib/x86_64-linux-gnu/libmysqlclient_r.so': File exists" --- core/ubuntu_build_deps.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/core/ubuntu_build_deps.sh b/core/ubuntu_build_deps.sh index 911046aa1f..6f3fabc084 100755 --- a/core/ubuntu_build_deps.sh +++ b/core/ubuntu_build_deps.sh @@ -9,4 +9,6 @@ sudo apt-get -y update && sudo apt-get -y install intel-mkl-gnu-2019.5-281 intel sudo apt-get install -y gfortran libmysqlclient-dev mysql-client libcurl4-openssl-dev libboost-system-dev \ libboost-filesystem-dev libboost-serialization-dev libboost-regex-dev -sudo ln -s /usr/lib/x86_64-linux-gnu/libmysqlclient.so /usr/lib/x86_64-linux-gnu/libmysqlclient_r.so +if [ ! -f "/usr/lib/x86_64-linux-gnu/libmysqlclient_r.so" ]; then + sudo ln -s /usr/lib/x86_64-linux-gnu/libmysqlclient.so /usr/lib/x86_64-linux-gnu/libmysqlclient_r.so +fi From 37df2b6fc42e1647f8dd760eacdfeb08e60210a4 Mon Sep 17 00:00:00 2001 From: "yudong.cai" Date: Wed, 20 Nov 2019 10:22:42 +0800 Subject: [PATCH 4/8] #346 rename gpu_resource_config.enable_gpu to gpu_resource_config.enable --- core/conf/server_cpu_config.template | 2 +- core/conf/server_gpu_config.template | 2 +- core/src/server/Config.cpp | 57 ++++++++++++++-------------- core/src/server/Config.h | 10 ++--- core/unittest/server/test_config.cpp | 6 +-- 5 files changed, 38 insertions(+), 39 deletions(-) diff --git a/core/conf/server_cpu_config.template b/core/conf/server_cpu_config.template index 1a384f19a1..41889f5cef 100644 --- a/core/conf/server_cpu_config.template +++ b/core/conf/server_cpu_config.template @@ -36,7 +36,7 @@ engine_config: gpu_search_threshold: 1000 # threshold beyond which the search computation is executed on GPUs only gpu_resource_config: - enable_gpu: false # whether to enable GPU resources + enable: false # whether to enable GPU resources cache_capacity: 4 # GB, size of GPU memory per card used for cache, must be a positive integer search_resources: # define the GPU devices used for search computation, must be in format gpux - gpu0 diff --git a/core/conf/server_gpu_config.template b/core/conf/server_gpu_config.template index 9790c93491..531c633da7 100644 --- a/core/conf/server_gpu_config.template +++ b/core/conf/server_gpu_config.template @@ -36,7 +36,7 @@ engine_config: gpu_search_threshold: 1000 # threshold beyond which the search computation is executed on GPUs only gpu_resource_config: - enable_gpu: true # whether to enable GPU resources + enable: true # whether to enable GPU resources cache_capacity: 4 # GB, size of GPU memory per card used for cache, must be a positive integer search_resources: # define the GPU devices used for search computation, must be in format gpux - gpu0 diff --git a/core/src/server/Config.cpp b/core/src/server/Config.cpp index e3a48b6ae4..2ff91bfe3a 100644 --- a/core/src/server/Config.cpp +++ b/core/src/server/Config.cpp @@ -190,8 +190,8 @@ Config::ValidateConfig() { /* gpu resource config */ #ifdef MILVUS_GPU_VERSION - bool resource_enable_gpu; - s = GetGpuResourceConfigEnableGpu(resource_enable_gpu); + bool gpu_resource_enable; + s = GetGpuResourceConfigEnable(gpu_resource_enable); if (!s.ok()) { return s; } @@ -330,7 +330,7 @@ Config::ResetDefaultConfig() { /* gpu resource config */ #ifdef MILVUS_GPU_VERSION - s = SetGpuResourceConfigEnableGpu(CONFIG_GPU_RESOURCE_ENABLE_GPU_DEFAULT); + s = SetGpuResourceConfigEnable(CONFIG_GPU_RESOURCE_ENABLE_DEFAULT); if (!s.ok()) { return s; } @@ -641,10 +641,10 @@ Config::CheckEngineConfigGpuSearchThreshold(const std::string& value) { } Status -Config::CheckGpuResourceConfigEnableGpu(const std::string& value) { +Config::CheckGpuResourceConfigEnable(const std::string& value) { if (!ValidationUtil::ValidateStringIsBool(value).ok()) { - std::string msg = "Invalid gpu resource config: " + value + - ". Possible reason: gpu_resource_config.enable_gpu is not a boolean."; + std::string msg = + "Invalid gpu resource config: " + value + ". Possible reason: gpu_resource_config.enable is not a boolean."; return Status(SERVER_INVALID_ARGUMENT, msg); } return Status::OK(); @@ -992,10 +992,9 @@ Config::GetEngineConfigGpuSearchThreshold(int32_t& value) { } Status -Config::GetGpuResourceConfigEnableGpu(bool& value) { - std::string str = - GetConfigStr(CONFIG_GPU_RESOURCE, CONFIG_GPU_RESOURCE_ENABLE_GPU, CONFIG_GPU_RESOURCE_ENABLE_GPU_DEFAULT); - Status s = CheckGpuResourceConfigEnableGpu(str); +Config::GetGpuResourceConfigEnable(bool& value) { + std::string str = GetConfigStr(CONFIG_GPU_RESOURCE, CONFIG_GPU_RESOURCE_ENABLE, CONFIG_GPU_RESOURCE_ENABLE_DEFAULT); + Status s = CheckGpuResourceConfigEnable(str); if (!s.ok()) { return s; } @@ -1006,13 +1005,13 @@ Config::GetGpuResourceConfigEnableGpu(bool& value) { Status Config::GetGpuResourceConfigCacheCapacity(int64_t& value) { - bool enable_gpu = false; - Status s = GetGpuResourceConfigEnableGpu(enable_gpu); + bool gpu_resource_enable = false; + Status s = GetGpuResourceConfigEnable(gpu_resource_enable); if (!s.ok()) { return s; } - if (!enable_gpu) { - std::string msg = "GPU not supported. Possible reason: gpu_resource_config.enable_gpu is set to false."; + if (!gpu_resource_enable) { + std::string msg = "GPU not supported. Possible reason: gpu_resource_config.enable is set to false."; return Status(SERVER_UNSUPPORTED_ERROR, msg); } std::string str = GetConfigStr(CONFIG_GPU_RESOURCE, CONFIG_GPU_RESOURCE_CACHE_CAPACITY, @@ -1027,13 +1026,13 @@ Config::GetGpuResourceConfigCacheCapacity(int64_t& value) { Status Config::GetGpuResourceConfigCacheThreshold(float& value) { - bool enable_gpu = false; - Status s = GetGpuResourceConfigEnableGpu(enable_gpu); + bool gpu_resource_enable = false; + Status s = GetGpuResourceConfigEnable(gpu_resource_enable); if (!s.ok()) { return s; } - if (!enable_gpu) { - std::string msg = "GPU not supported. Possible reason: gpu_resource_config.enable_gpu is set to false."; + if (!gpu_resource_enable) { + std::string msg = "GPU not supported. Possible reason: gpu_resource_config.enable is set to false."; return Status(SERVER_UNSUPPORTED_ERROR, msg); } std::string str = GetConfigStr(CONFIG_GPU_RESOURCE, CONFIG_GPU_RESOURCE_CACHE_THRESHOLD, @@ -1048,13 +1047,13 @@ Config::GetGpuResourceConfigCacheThreshold(float& value) { Status Config::GetGpuResourceConfigSearchResources(std::vector& value) { - bool enable_gpu = false; - Status s = GetGpuResourceConfigEnableGpu(enable_gpu); + bool gpu_resource_enable = false; + Status s = GetGpuResourceConfigEnable(gpu_resource_enable); if (!s.ok()) { return s; } - if (!enable_gpu) { - std::string msg = "GPU not supported. Possible reason: gpu_resource_config.enable_gpu is set to false."; + if (!gpu_resource_enable) { + std::string msg = "GPU not supported. Possible reason: gpu_resource_config.enable is set to false."; return Status(SERVER_UNSUPPORTED_ERROR, msg); } std::string str = GetConfigSequenceStr(CONFIG_GPU_RESOURCE, CONFIG_GPU_RESOURCE_SEARCH_RESOURCES, @@ -1073,13 +1072,13 @@ Config::GetGpuResourceConfigSearchResources(std::vector& value) { Status Config::GetGpuResourceConfigBuildIndexResources(std::vector& value) { - bool enable_gpu = false; - Status s = GetGpuResourceConfigEnableGpu(enable_gpu); + bool gpu_resource_enable = false; + Status s = GetGpuResourceConfigEnable(gpu_resource_enable); if (!s.ok()) { return s; } - if (!enable_gpu) { - std::string msg = "GPU not supported. Possible reason: gpu_resource_config.enable_gpu is set to false."; + if (!gpu_resource_enable) { + std::string msg = "GPU not supported. Possible reason: gpu_resource_config.enable is set to false."; return Status(SERVER_UNSUPPORTED_ERROR, msg); } std::string str = @@ -1295,12 +1294,12 @@ Config::SetEngineConfigGpuSearchThreshold(const std::string& value) { /* gpu resource config */ Status -Config::SetGpuResourceConfigEnableGpu(const std::string& value) { - Status s = CheckGpuResourceConfigEnableGpu(value); +Config::SetGpuResourceConfigEnable(const std::string& value) { + Status s = CheckGpuResourceConfigEnable(value); if (!s.ok()) { return s; } - SetConfigValueInMem(CONFIG_GPU_RESOURCE, CONFIG_GPU_RESOURCE_ENABLE_GPU, value); + SetConfigValueInMem(CONFIG_GPU_RESOURCE, CONFIG_GPU_RESOURCE_ENABLE, value); return Status::OK(); } diff --git a/core/src/server/Config.h b/core/src/server/Config.h index 0442ae0626..7c141be57e 100644 --- a/core/src/server/Config.h +++ b/core/src/server/Config.h @@ -85,9 +85,9 @@ static const char* CONFIG_ENGINE_GPU_SEARCH_THRESHOLD_DEFAULT = "1000"; /* gpu resource config */ static const char* CONFIG_GPU_RESOURCE = "gpu_resource_config"; -static const char* CONFIG_GPU_RESOURCE_ENABLE_GPU = "enable_gpu"; +static const char* CONFIG_GPU_RESOURCE_ENABLE = "enable"; #ifdef MILVUS_GPU_VERSION -static const char* CONFIG_GPU_RESOURCE_ENABLE_GPU_DEFAULT = "true"; +static const char* CONFIG_GPU_RESOURCE_ENABLE_DEFAULT = "true"; #else static const char* CONFIG_GPU_RESOURCE_ENABLE_GPU_DEFAULT = "false"; #endif @@ -175,7 +175,7 @@ class Config { /* gpu resource config */ Status - CheckGpuResourceConfigEnableGpu(const std::string& value); + CheckGpuResourceConfigEnable(const std::string& value); Status CheckGpuResourceConfigCacheCapacity(const std::string& value); Status @@ -244,7 +244,7 @@ class Config { /* gpu resource config */ Status - GetGpuResourceConfigEnableGpu(bool& value); + GetGpuResourceConfigEnable(bool& value); Status GetGpuResourceConfigCacheCapacity(int64_t& value); Status @@ -305,7 +305,7 @@ class Config { /* gpu resource config */ Status - SetGpuResourceConfigEnableGpu(const std::string& value); + SetGpuResourceConfigEnable(const std::string& value); Status SetGpuResourceConfigCacheCapacity(const std::string& value); Status diff --git a/core/unittest/server/test_config.cpp b/core/unittest/server/test_config.cpp index 6ba7f19160..e0fde62916 100644 --- a/core/unittest/server/test_config.cpp +++ b/core/unittest/server/test_config.cpp @@ -246,9 +246,9 @@ TEST_F(ConfigTest, SERVER_CONFIG_VALID_TEST) { /* gpu resource config */ bool resource_enable_gpu = true; - s = config.SetGpuResourceConfigEnableGpu(std::to_string(resource_enable_gpu)); + s = config.SetGpuResourceConfigEnable(std::to_string(resource_enable_gpu)); ASSERT_TRUE(s.ok()); - s = config.GetGpuResourceConfigEnableGpu(bool_val); + s = config.GetGpuResourceConfigEnable(bool_val); ASSERT_TRUE(s.ok()); ASSERT_TRUE(bool_val == resource_enable_gpu); @@ -394,7 +394,7 @@ TEST_F(ConfigTest, SERVER_CONFIG_INVALID_TEST) { ASSERT_FALSE(s.ok()); /* gpu resource config */ - s = config.SetGpuResourceConfigEnableGpu("ok"); + s = config.SetGpuResourceConfigEnable("ok"); ASSERT_FALSE(s.ok()); #ifdef MILVUS_GPU_VERSION From 631cf7143976f6e08bc4afb7064b46b773af8010 Mon Sep 17 00:00:00 2001 From: "yudong.cai" Date: Wed, 20 Nov 2019 10:57:01 +0800 Subject: [PATCH 5/8] #346 fix CPU version build error --- core/src/server/Config.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/server/Config.h b/core/src/server/Config.h index 7c141be57e..a6e78671b3 100644 --- a/core/src/server/Config.h +++ b/core/src/server/Config.h @@ -89,7 +89,7 @@ static const char* CONFIG_GPU_RESOURCE_ENABLE = "enable"; #ifdef MILVUS_GPU_VERSION static const char* CONFIG_GPU_RESOURCE_ENABLE_DEFAULT = "true"; #else -static const char* CONFIG_GPU_RESOURCE_ENABLE_GPU_DEFAULT = "false"; +static const char* CONFIG_GPU_RESOURCE_ENABLE_DEFAULT = "false"; #endif static const char* CONFIG_GPU_RESOURCE_CACHE_CAPACITY = "cache_capacity"; static const char* CONFIG_GPU_RESOURCE_CACHE_CAPACITY_DEFAULT = "4"; From 58e591ae7c1d24b929f7a542fead68fdae803e3c Mon Sep 17 00:00:00 2001 From: quicksilver Date: Wed, 20 Nov 2019 14:19:31 +0800 Subject: [PATCH 6/8] Update Jenkinsfile --- ci/jenkins/Jenkinsfile | 40 ++++++++++++++++++++-------------------- ci/scripts/build.sh | 10 ++++++++-- 2 files changed, 28 insertions(+), 22 deletions(-) diff --git a/ci/jenkins/Jenkinsfile b/ci/jenkins/Jenkinsfile index 8d3953b112..bf7a70d11a 100644 --- a/ci/jenkins/Jenkinsfile +++ b/ci/jenkins/Jenkinsfile @@ -50,7 +50,7 @@ pipeline { } stages { - stage("Run Build") { + stage("Run GPU Version Build") { agent { kubernetes { label "${BINRARY_VERSION}-build" @@ -60,7 +60,7 @@ pipeline { } stages { - stage('Build') { + stage('GPU Version Build') { steps { container('milvus-build-env') { script { @@ -69,7 +69,7 @@ pipeline { } } } - stage('Code Coverage') { + stage('GPU Version Code Coverage') { steps { container('milvus-build-env') { script { @@ -78,7 +78,7 @@ pipeline { } } } - stage('Upload Package') { + stage('Upload GPU Version Package') { steps { container('milvus-build-env') { script { @@ -90,7 +90,7 @@ pipeline { } } - stage("Publish docker images") { + stage("Publish GPU Version docker images") { agent { kubernetes { label "${BINRARY_VERSION}-publish" @@ -100,7 +100,7 @@ pipeline { } stages { - stage('Publish') { + stage('Publish GPU Version') { steps { container('publish-images'){ script { @@ -112,7 +112,7 @@ pipeline { } } - stage("Deploy to Development") { + stage("Deploy GPU Version to Development") { agent { kubernetes { label "${BINRARY_VERSION}-dev-test" @@ -122,7 +122,7 @@ pipeline { } stages { - stage("Deploy to Dev") { + stage("Deploy GPU Version to Dev") { steps { container('milvus-test-env') { script { @@ -132,7 +132,7 @@ pipeline { } } - stage("Dev Test") { + stage("GPU Version Dev Test") { steps { container('milvus-test-env') { script { @@ -147,7 +147,7 @@ pipeline { } } - stage ("Cleanup Dev") { + stage ("Cleanup GPU Version Dev") { steps { container('milvus-test-env') { script { @@ -180,7 +180,7 @@ pipeline { } stages { - stage("Run Build") { + stage("Run CPU Version Build") { agent { kubernetes { label "${BINRARY_VERSION}-build" @@ -190,7 +190,7 @@ pipeline { } stages { - stage('Build') { + stage('Build CPU Version') { steps { container('milvus-build-env') { script { @@ -199,7 +199,7 @@ pipeline { } } } - stage('Code Coverage') { + stage('CPU Version Code Coverage') { steps { container('milvus-build-env') { script { @@ -208,7 +208,7 @@ pipeline { } } } - stage('Upload Package') { + stage('Upload CPU Version Package') { steps { container('milvus-build-env') { script { @@ -220,7 +220,7 @@ pipeline { } } - stage("Publish docker images") { + stage("Publish CPU Version docker images") { agent { kubernetes { label "${BINRARY_VERSION}-publish" @@ -230,7 +230,7 @@ pipeline { } stages { - stage('Publish') { + stage('Publish CPU Version') { steps { container('publish-images'){ script { @@ -242,7 +242,7 @@ pipeline { } } - stage("Deploy to Development") { + stage("Deploy CPU Version to Development") { agent { kubernetes { label "${BINRARY_VERSION}-dev-test" @@ -252,7 +252,7 @@ pipeline { } stages { - stage("Deploy to Dev") { + stage("Deploy CPU Version to Dev") { steps { container('milvus-test-env') { script { @@ -262,7 +262,7 @@ pipeline { } } - stage("Dev Test") { + stage("CPU Version Dev Test") { steps { container('milvus-test-env') { script { @@ -277,7 +277,7 @@ pipeline { } } - stage ("Cleanup Dev") { + stage ("Cleanup CPU Version Dev") { steps { container('milvus-test-env') { script { diff --git a/ci/scripts/build.sh b/ci/scripts/build.sh index 27962ccb54..eb5205ad26 100755 --- a/ci/scripts/build.sh +++ b/ci/scripts/build.sh @@ -16,6 +16,7 @@ BUILD_TYPE="Debug" BUILD_UNITTEST="OFF" INSTALL_PREFIX="/opt/milvus" FAISS_ROOT="" +CUSTOMIZATION="OFF" # default use origin faiss BUILD_COVERAGE="OFF" USE_JFROG_CACHE="OFF" RUN_CPPLINT="OFF" @@ -23,7 +24,7 @@ GPU_VERSION="OFF" WITH_MKL="OFF" CUDA_COMPILER=/usr/local/cuda/bin/nvcc -while getopts "o:t:b:f:gulcjmh" arg +while getopts "o:t:b:f:gxulcjmh" arg do case $arg in o) @@ -41,6 +42,9 @@ do g) GPU_VERSION="ON"; ;; + x) + CUSTOMIZATION="ON"; + ;; u) echo "Build and run unittest cases" ; BUILD_UNITTEST="ON"; @@ -66,6 +70,7 @@ parameter: -b: core code build directory -f: faiss root path -g: gpu version +-x: milvus customization (default: OFF) -u: building unit test options(default: OFF) -l: run cpplint, clang-format and clang-tidy(default: OFF) -c: code coverage(default: OFF) @@ -74,7 +79,7 @@ parameter: -h: help usage: -./build.sh -o \${INSTALL_PREFIX} -t \${BUILD_TYPE} -b \${CORE_BUILD_DIR} -f \${FAISS_ROOT} [-u] [-l] [-c] [-j] [-m] [-h] +./build.sh -o \${INSTALL_PREFIX} -t \${BUILD_TYPE} -b \${CORE_BUILD_DIR} -f \${FAISS_ROOT} [-g] [-x] [-u] [-l] [-c] [-j] [-m] [-h] " exit 0 ;; @@ -96,6 +101,7 @@ CMAKE_CMD="cmake \ -DCMAKE_BUILD_TYPE=${BUILD_TYPE} \ -DCMAKE_CUDA_COMPILER=${CUDA_COMPILER} \ -DMILVUS_GPU_VERSION=${GPU_VERSION} \ +-DCUSTOMIZATION=${CUSTOMIZATION} \ -DBUILD_UNIT_TEST=${BUILD_UNITTEST} \ -DBUILD_COVERAGE=${BUILD_COVERAGE} \ -DUSE_JFROG_CACHE=${USE_JFROG_CACHE} \ From c89bb6699ebcfdc1239635b7775bf3a827e0970b Mon Sep 17 00:00:00 2001 From: "yudong.cai" Date: Wed, 20 Nov 2019 11:24:54 +0800 Subject: [PATCH 7/8] #306 Use int64 for all config integer --- core/src/db/Options.h | 2 +- core/src/db/engine/ExecutionEngineImpl.cpp | 4 +- core/src/db/engine/ExecutionEngineImpl.h | 4 +- core/src/scheduler/SchedInst.cpp | 4 +- .../scheduler/optimizer/BuildIndexPass.cpp | 2 +- core/src/scheduler/optimizer/FaissFlatPass.h | 4 +- .../scheduler/optimizer/FaissIVFFlatPass.h | 4 +- .../scheduler/optimizer/FaissIVFSQ8HPass.cpp | 2 +- .../scheduler/optimizer/FaissIVFSQ8HPass.h | 4 +- .../src/scheduler/optimizer/FaissIVFSQ8Pass.h | 4 +- core/src/server/Config.cpp | 70 +++++++++---------- core/src/server/Config.h | 16 ++--- core/src/server/DBWrapper.cpp | 8 +-- core/src/utils/CommonUtil.cpp | 2 +- core/src/utils/CommonUtil.h | 2 +- core/src/wrapper/KnowhereResource.cpp | 4 +- core/unittest/server/test_config.cpp | 45 ++++++------ core/unittest/server/test_util.cpp | 2 +- 18 files changed, 91 insertions(+), 92 deletions(-) diff --git a/core/src/db/Options.h b/core/src/db/Options.h index ebecb4de5a..91459a8d5f 100644 --- a/core/src/db/Options.h +++ b/core/src/db/Options.h @@ -33,7 +33,7 @@ static const char* ARCHIVE_CONF_DISK = "disk"; static const char* ARCHIVE_CONF_DAYS = "days"; struct ArchiveConf { - using CriteriaT = std::map; + using CriteriaT = std::map; explicit ArchiveConf(const std::string& type, const std::string& criterias = std::string()); diff --git a/core/src/db/engine/ExecutionEngineImpl.cpp b/core/src/db/engine/ExecutionEngineImpl.cpp index 4f2f542254..973d8b5912 100644 --- a/core/src/db/engine/ExecutionEngineImpl.cpp +++ b/core/src/db/engine/ExecutionEngineImpl.cpp @@ -146,7 +146,7 @@ ExecutionEngineImpl::HybridLoad() const { const std::string key = location_ + ".quantizer"; server::Config& config = server::Config::GetInstance(); - std::vector gpus; + std::vector gpus; Status s = config.GetGpuResourceConfigSearchResources(gpus); if (!s.ok()) { ENGINE_LOG_ERROR << s.message(); @@ -586,7 +586,7 @@ ExecutionEngineImpl::GpuCache(uint64_t gpu_id) { Status ExecutionEngineImpl::Init() { server::Config& config = server::Config::GetInstance(); - std::vector gpu_ids; + std::vector gpu_ids; Status s = config.GetGpuResourceConfigBuildIndexResources(gpu_ids); for (auto id : gpu_ids) { if (gpu_num_ == id) { diff --git a/core/src/db/engine/ExecutionEngineImpl.h b/core/src/db/engine/ExecutionEngineImpl.h index 7eb304426e..da0e7cfb64 100644 --- a/core/src/db/engine/ExecutionEngineImpl.h +++ b/core/src/db/engine/ExecutionEngineImpl.h @@ -122,8 +122,8 @@ class ExecutionEngineImpl : public ExecutionEngine { int64_t dim_; std::string location_; - int32_t nlist_ = 0; - int32_t gpu_num_ = 0; + int64_t nlist_ = 0; + int64_t gpu_num_ = 0; }; } // namespace engine diff --git a/core/src/scheduler/SchedInst.cpp b/core/src/scheduler/SchedInst.cpp index 867052ef5d..69d293f986 100644 --- a/core/src/scheduler/SchedInst.cpp +++ b/core/src/scheduler/SchedInst.cpp @@ -55,9 +55,9 @@ load_simple_config() { // get resources #ifdef MILVUS_GPU_VERSION server::Config& config = server::Config::GetInstance(); - std::vector gpu_ids; + std::vector gpu_ids; config.GetGpuResourceConfigSearchResources(gpu_ids); - std::vector build_gpu_ids; + std::vector build_gpu_ids; config.GetGpuResourceConfigBuildIndexResources(build_gpu_ids); auto pcie = Connection("pcie", 12000); diff --git a/core/src/scheduler/optimizer/BuildIndexPass.cpp b/core/src/scheduler/optimizer/BuildIndexPass.cpp index cd1b7550d8..d535b9675f 100644 --- a/core/src/scheduler/optimizer/BuildIndexPass.cpp +++ b/core/src/scheduler/optimizer/BuildIndexPass.cpp @@ -26,7 +26,7 @@ namespace scheduler { void BuildIndexPass::Init() { server::Config& config = server::Config::GetInstance(); - std::vector build_resources; + std::vector build_resources; Status s = config.GetGpuResourceConfigBuildIndexResources(build_resources); if (!s.ok()) { throw; diff --git a/core/src/scheduler/optimizer/FaissFlatPass.h b/core/src/scheduler/optimizer/FaissFlatPass.h index 60adf62a07..f219bebdf3 100644 --- a/core/src/scheduler/optimizer/FaissFlatPass.h +++ b/core/src/scheduler/optimizer/FaissFlatPass.h @@ -45,9 +45,9 @@ class FaissFlatPass : public Pass { Run(const TaskPtr& task) override; private: - int32_t threshold_ = std::numeric_limits::max(); + int64_t threshold_ = std::numeric_limits::max(); int64_t count_ = 0; - std::vector gpus; + std::vector gpus; }; using FaissFlatPassPtr = std::shared_ptr; diff --git a/core/src/scheduler/optimizer/FaissIVFFlatPass.h b/core/src/scheduler/optimizer/FaissIVFFlatPass.h index 9264db32a8..2d15539014 100644 --- a/core/src/scheduler/optimizer/FaissIVFFlatPass.h +++ b/core/src/scheduler/optimizer/FaissIVFFlatPass.h @@ -45,9 +45,9 @@ class FaissIVFFlatPass : public Pass { Run(const TaskPtr& task) override; private: - int32_t threshold_ = std::numeric_limits::max(); + int64_t threshold_ = std::numeric_limits::max(); int64_t count_ = 0; - std::vector gpus; + std::vector gpus; }; using FaissIVFFlatPassPtr = std::shared_ptr; diff --git a/core/src/scheduler/optimizer/FaissIVFSQ8HPass.cpp b/core/src/scheduler/optimizer/FaissIVFSQ8HPass.cpp index b63a0f7490..a99e861e03 100644 --- a/core/src/scheduler/optimizer/FaissIVFSQ8HPass.cpp +++ b/core/src/scheduler/optimizer/FaissIVFSQ8HPass.cpp @@ -32,7 +32,7 @@ FaissIVFSQ8HPass::Init() { server::Config& config = server::Config::GetInstance(); Status s = config.GetEngineConfigGpuSearchThreshold(threshold_); if (!s.ok()) { - threshold_ = std::numeric_limits::max(); + threshold_ = std::numeric_limits::max(); } s = config.GetGpuResourceConfigSearchResources(gpus); } diff --git a/core/src/scheduler/optimizer/FaissIVFSQ8HPass.h b/core/src/scheduler/optimizer/FaissIVFSQ8HPass.h index fa830ff08a..0d2892809f 100644 --- a/core/src/scheduler/optimizer/FaissIVFSQ8HPass.h +++ b/core/src/scheduler/optimizer/FaissIVFSQ8HPass.h @@ -45,9 +45,9 @@ class FaissIVFSQ8HPass : public Pass { Run(const TaskPtr& task) override; private: - int32_t threshold_ = std::numeric_limits::max(); + int64_t threshold_ = std::numeric_limits::max(); int64_t count_ = 0; - std::vector gpus; + std::vector gpus; }; using FaissIVFSQ8HPassPtr = std::shared_ptr; diff --git a/core/src/scheduler/optimizer/FaissIVFSQ8Pass.h b/core/src/scheduler/optimizer/FaissIVFSQ8Pass.h index c5853283f7..e92ea2fe4e 100644 --- a/core/src/scheduler/optimizer/FaissIVFSQ8Pass.h +++ b/core/src/scheduler/optimizer/FaissIVFSQ8Pass.h @@ -45,9 +45,9 @@ class FaissIVFSQ8Pass : public Pass { Run(const TaskPtr& task) override; private: - int32_t threshold_ = std::numeric_limits::max(); + int64_t threshold_ = std::numeric_limits::max(); int64_t count_ = 0; - std::vector gpus; + std::vector gpus; }; using FaissIVFSQ8PassPtr = std::shared_ptr; diff --git a/core/src/server/Config.cpp b/core/src/server/Config.cpp index 2ff91bfe3a..f3efcff0cc 100644 --- a/core/src/server/Config.cpp +++ b/core/src/server/Config.cpp @@ -113,19 +113,19 @@ Config::ValidateConfig() { return s; } - int32_t db_archive_disk_threshold; + int64_t db_archive_disk_threshold; s = GetDBConfigArchiveDiskThreshold(db_archive_disk_threshold); if (!s.ok()) { return s; } - int32_t db_archive_days_threshold; + int64_t db_archive_days_threshold; s = GetDBConfigArchiveDaysThreshold(db_archive_days_threshold); if (!s.ok()) { return s; } - int32_t db_insert_buffer_size; + int64_t db_insert_buffer_size; s = GetDBConfigInsertBufferSize(db_insert_buffer_size); if (!s.ok()) { return s; @@ -170,19 +170,19 @@ Config::ValidateConfig() { } /* engine config */ - int32_t engine_use_blas_threshold; + int64_t engine_use_blas_threshold; s = GetEngineConfigUseBlasThreshold(engine_use_blas_threshold); if (!s.ok()) { return s; } - int32_t engine_omp_thread_num; + int64_t engine_omp_thread_num; s = GetEngineConfigOmpThreadNum(engine_omp_thread_num); if (!s.ok()) { return s; } - int32_t engine_gpu_search_threshold; + int64_t engine_gpu_search_threshold; s = GetEngineConfigGpuSearchThreshold(engine_gpu_search_threshold); if (!s.ok()) { return s; @@ -208,13 +208,13 @@ Config::ValidateConfig() { return s; } - std::vector search_resources; + std::vector search_resources; s = GetGpuResourceConfigSearchResources(search_resources); if (!s.ok()) { return s; } - std::vector index_build_resources; + std::vector index_build_resources; s = GetGpuResourceConfigBuildIndexResources(index_build_resources); if (!s.ok()) { return s; @@ -485,7 +485,7 @@ Config::CheckDBConfigInsertBufferSize(const std::string& value) { ". Possible reason: db_config.insert_buffer_size is not a positive integer."; return Status(SERVER_INVALID_ARGUMENT, msg); } else { - int64_t buffer_size = std::stoi(value) * GB; + int64_t buffer_size = std::stoll(value) * GB; if (buffer_size <= 0) { std::string msg = "Invalid insert buffer size: " + value + ". Possible reason: db_config.insert_buffer_size is not a positive integer."; @@ -540,7 +540,7 @@ Config::CheckCacheConfigCpuCacheCapacity(const std::string& value) { ". Possible reason: cache_config.cpu_cache_capacity is not a positive integer."; return Status(SERVER_INVALID_ARGUMENT, msg); } else { - int64_t cpu_cache_capacity = std::stoi(value) * GB; + int64_t cpu_cache_capacity = std::stoll(value) * GB; if (cpu_cache_capacity <= 0) { std::string msg = "Invalid cpu cache capacity: " + value + ". Possible reason: cache_config.cpu_cache_capacity is not a positive integer."; @@ -557,7 +557,7 @@ Config::CheckCacheConfigCpuCacheCapacity(const std::string& value) { std::cerr << "WARNING: cpu cache capacity value is too big" << std::endl; } - int32_t buffer_value; + int64_t buffer_value; Status s = GetDBConfigInsertBufferSize(buffer_value); if (!s.ok()) { return s; @@ -619,10 +619,10 @@ Config::CheckEngineConfigOmpThreadNum(const std::string& value) { return Status(SERVER_INVALID_ARGUMENT, msg); } - int32_t omp_thread = std::stoi(value); - uint32_t sys_thread_cnt = 8; + int64_t omp_thread = std::stoll(value); + int64_t sys_thread_cnt = 8; CommonUtil::GetSystemAvailableThreads(sys_thread_cnt); - if (omp_thread > static_cast(sys_thread_cnt)) { + if (omp_thread > sys_thread_cnt) { std::string msg = "Invalid omp thread num: " + value + ". Possible reason: engine_config.omp_thread_num exceeds system cpu cores."; return Status(SERVER_INVALID_ARGUMENT, msg); @@ -657,14 +657,14 @@ Config::CheckGpuResourceConfigCacheCapacity(const std::string& value) { ". Possible reason: gpu_resource_config.cache_capacity is not a positive integer."; return Status(SERVER_INVALID_ARGUMENT, msg); } else { - uint64_t gpu_cache_capacity = std::stoi(value) * GB; - std::vector gpu_ids; + int64_t gpu_cache_capacity = std::stoll(value) * GB; + std::vector gpu_ids; Status s = GetGpuResourceConfigBuildIndexResources(gpu_ids); if (!s.ok()) { return s; } - for (int32_t gpu_id : gpu_ids) { + for (int64_t gpu_id : gpu_ids) { size_t gpu_memory; if (!ValidationUtil::GetGpuMemory(gpu_id, gpu_memory).ok()) { std::string msg = "Fail to get GPU memory for GPU device: " + std::to_string(gpu_id); @@ -855,37 +855,37 @@ Config::GetDBConfigBackendUrl(std::string& value) { } Status -Config::GetDBConfigArchiveDiskThreshold(int32_t& value) { +Config::GetDBConfigArchiveDiskThreshold(int64_t& value) { std::string str = GetConfigStr(CONFIG_DB, CONFIG_DB_ARCHIVE_DISK_THRESHOLD, CONFIG_DB_ARCHIVE_DISK_THRESHOLD_DEFAULT); Status s = CheckDBConfigArchiveDiskThreshold(str); if (!s.ok()) { return s; } - value = std::stoi(str); + value = std::stoll(str); return Status::OK(); } Status -Config::GetDBConfigArchiveDaysThreshold(int32_t& value) { +Config::GetDBConfigArchiveDaysThreshold(int64_t& value) { std::string str = GetConfigStr(CONFIG_DB, CONFIG_DB_ARCHIVE_DAYS_THRESHOLD, CONFIG_DB_ARCHIVE_DAYS_THRESHOLD_DEFAULT); Status s = CheckDBConfigArchiveDaysThreshold(str); if (!s.ok()) { return s; } - value = std::stoi(str); + value = std::stoll(str); return Status::OK(); } Status -Config::GetDBConfigInsertBufferSize(int32_t& value) { +Config::GetDBConfigInsertBufferSize(int64_t& value) { std::string str = GetConfigStr(CONFIG_DB, CONFIG_DB_INSERT_BUFFER_SIZE, CONFIG_DB_INSERT_BUFFER_SIZE_DEFAULT); Status s = CheckDBConfigInsertBufferSize(str); if (!s.ok()) { return s; } - value = std::stoi(str); + value = std::stoll(str); return Status::OK(); } @@ -927,7 +927,7 @@ Config::GetCacheConfigCpuCacheCapacity(int64_t& value) { if (!s.ok()) { return s; } - value = std::stoi(str); + value = std::stoll(str); return Status::OK(); } @@ -957,37 +957,37 @@ Config::GetCacheConfigCacheInsertData(bool& value) { } Status -Config::GetEngineConfigUseBlasThreshold(int32_t& value) { +Config::GetEngineConfigUseBlasThreshold(int64_t& value) { std::string str = GetConfigStr(CONFIG_ENGINE, CONFIG_ENGINE_USE_BLAS_THRESHOLD, CONFIG_ENGINE_USE_BLAS_THRESHOLD_DEFAULT); Status s = CheckEngineConfigUseBlasThreshold(str); if (!s.ok()) { return s; } - value = std::stoi(str); + value = std::stoll(str); return Status::OK(); } Status -Config::GetEngineConfigOmpThreadNum(int32_t& value) { +Config::GetEngineConfigOmpThreadNum(int64_t& value) { std::string str = GetConfigStr(CONFIG_ENGINE, CONFIG_ENGINE_OMP_THREAD_NUM, CONFIG_ENGINE_OMP_THREAD_NUM_DEFAULT); Status s = CheckEngineConfigOmpThreadNum(str); if (!s.ok()) { return s; } - value = std::stoi(str); + value = std::stoll(str); return Status::OK(); } Status -Config::GetEngineConfigGpuSearchThreshold(int32_t& value) { +Config::GetEngineConfigGpuSearchThreshold(int64_t& value) { std::string str = GetConfigStr(CONFIG_ENGINE, CONFIG_ENGINE_GPU_SEARCH_THRESHOLD, CONFIG_ENGINE_GPU_SEARCH_THRESHOLD_DEFAULT); Status s = CheckEngineConfigGpuSearchThreshold(str); if (!s.ok()) { return s; } - value = std::stoi(str); + value = std::stoll(str); return Status::OK(); } @@ -1020,7 +1020,7 @@ Config::GetGpuResourceConfigCacheCapacity(int64_t& value) { if (!s.ok()) { return s; } - value = std::stoi(str); + value = std::stoll(str); return Status::OK(); } @@ -1046,7 +1046,7 @@ Config::GetGpuResourceConfigCacheThreshold(float& value) { } Status -Config::GetGpuResourceConfigSearchResources(std::vector& value) { +Config::GetGpuResourceConfigSearchResources(std::vector& value) { bool gpu_resource_enable = false; Status s = GetGpuResourceConfigEnable(gpu_resource_enable); if (!s.ok()) { @@ -1065,13 +1065,13 @@ Config::GetGpuResourceConfigSearchResources(std::vector& value) { return s; } for (std::string& res : res_vec) { - value.push_back(std::stoi(res.substr(3))); + value.push_back(std::stoll(res.substr(3))); } return Status::OK(); } Status -Config::GetGpuResourceConfigBuildIndexResources(std::vector& value) { +Config::GetGpuResourceConfigBuildIndexResources(std::vector& value) { bool gpu_resource_enable = false; Status s = GetGpuResourceConfigEnable(gpu_resource_enable); if (!s.ok()) { @@ -1091,7 +1091,7 @@ Config::GetGpuResourceConfigBuildIndexResources(std::vector& value) { return s; } for (std::string& res : res_vec) { - value.push_back(std::stoi(res.substr(3))); + value.push_back(std::stoll(res.substr(3))); } return Status::OK(); } diff --git a/core/src/server/Config.h b/core/src/server/Config.h index a6e78671b3..0907080a6f 100644 --- a/core/src/server/Config.h +++ b/core/src/server/Config.h @@ -210,11 +210,11 @@ class Config { Status GetDBConfigBackendUrl(std::string& value); Status - GetDBConfigArchiveDiskThreshold(int32_t& value); + GetDBConfigArchiveDiskThreshold(int64_t& value); Status - GetDBConfigArchiveDaysThreshold(int32_t& value); + GetDBConfigArchiveDaysThreshold(int64_t& value); Status - GetDBConfigInsertBufferSize(int32_t& value); + GetDBConfigInsertBufferSize(int64_t& value); Status GetDBConfigPreloadTable(std::string& value); @@ -236,11 +236,11 @@ class Config { /* engine config */ Status - GetEngineConfigUseBlasThreshold(int32_t& value); + GetEngineConfigUseBlasThreshold(int64_t& value); Status - GetEngineConfigOmpThreadNum(int32_t& value); + GetEngineConfigOmpThreadNum(int64_t& value); Status - GetEngineConfigGpuSearchThreshold(int32_t& value); + GetEngineConfigGpuSearchThreshold(int64_t& value); /* gpu resource config */ Status @@ -250,9 +250,9 @@ class Config { Status GetGpuResourceConfigCacheThreshold(float& value); Status - GetGpuResourceConfigSearchResources(std::vector& value); + GetGpuResourceConfigSearchResources(std::vector& value); Status - GetGpuResourceConfigBuildIndexResources(std::vector& value); + GetGpuResourceConfigBuildIndexResources(std::vector& value); public: /* server config */ diff --git a/core/src/server/DBWrapper.cpp b/core/src/server/DBWrapper.cpp index 1ff914dc69..f7a43d671c 100644 --- a/core/src/server/DBWrapper.cpp +++ b/core/src/server/DBWrapper.cpp @@ -89,7 +89,7 @@ DBWrapper::StartService() { } // engine config - int32_t omp_thread; + int64_t omp_thread; s = config.GetEngineConfigOmpThreadNum(omp_thread); if (!s.ok()) { std::cerr << s.ToString() << std::endl; @@ -100,7 +100,7 @@ DBWrapper::StartService() { omp_set_num_threads(omp_thread); SERVER_LOG_DEBUG << "Specify openmp thread number: " << omp_thread; } else { - uint32_t sys_thread_cnt = 8; + int64_t sys_thread_cnt = 8; if (CommonUtil::GetSystemAvailableThreads(sys_thread_cnt)) { omp_thread = static_cast(ceil(sys_thread_cnt * 0.5)); omp_set_num_threads(omp_thread); @@ -108,7 +108,7 @@ DBWrapper::StartService() { } // init faiss global variable - int32_t use_blas_threshold; + int64_t use_blas_threshold; s = config.GetEngineConfigUseBlasThreshold(use_blas_threshold); if (!s.ok()) { std::cerr << s.ToString() << std::endl; @@ -119,7 +119,7 @@ DBWrapper::StartService() { // set archive config engine::ArchiveConf::CriteriaT criterial; - int32_t disk, days; + int64_t disk, days; s = config.GetDBConfigArchiveDiskThreshold(disk); if (!s.ok()) { std::cerr << s.ToString() << std::endl; diff --git a/core/src/utils/CommonUtil.cpp b/core/src/utils/CommonUtil.cpp index fbf3112aeb..26e43619fb 100644 --- a/core/src/utils/CommonUtil.cpp +++ b/core/src/utils/CommonUtil.cpp @@ -54,7 +54,7 @@ CommonUtil::GetSystemMemInfo(uint64_t& total_mem, uint64_t& free_mem) { } bool -CommonUtil::GetSystemAvailableThreads(uint32_t& thread_count) { +CommonUtil::GetSystemAvailableThreads(int64_t& thread_count) { // threadCnt = std::thread::hardware_concurrency(); thread_count = sysconf(_SC_NPROCESSORS_CONF); thread_count *= THREAD_MULTIPLY_CPU; diff --git a/core/src/utils/CommonUtil.h b/core/src/utils/CommonUtil.h index 939bdd6d31..121196986a 100644 --- a/core/src/utils/CommonUtil.h +++ b/core/src/utils/CommonUtil.h @@ -30,7 +30,7 @@ class CommonUtil { static bool GetSystemMemInfo(uint64_t& total_mem, uint64_t& free_mem); static bool - GetSystemAvailableThreads(uint32_t& thread_count); + GetSystemAvailableThreads(int64_t& thread_count); static bool IsFileExist(const std::string& path); diff --git a/core/src/wrapper/KnowhereResource.cpp b/core/src/wrapper/KnowhereResource.cpp index bf6675cafa..5a2296b16e 100644 --- a/core/src/wrapper/KnowhereResource.cpp +++ b/core/src/wrapper/KnowhereResource.cpp @@ -48,7 +48,7 @@ KnowhereResource::Initialize() { // get build index gpu resource server::Config& config = server::Config::GetInstance(); - std::vector build_index_gpus; + std::vector build_index_gpus; s = config.GetGpuResourceConfigBuildIndexResources(build_index_gpus); if (!s.ok()) return s; @@ -58,7 +58,7 @@ KnowhereResource::Initialize() { } // get search gpu resource - std::vector search_gpus; + std::vector search_gpus; s = config.GetGpuResourceConfigSearchResources(search_gpus); if (!s.ok()) return s; diff --git a/core/unittest/server/test_config.cpp b/core/unittest/server/test_config.cpp index e0fde62916..664a08d631 100644 --- a/core/unittest/server/test_config.cpp +++ b/core/unittest/server/test_config.cpp @@ -104,7 +104,6 @@ TEST_F(ConfigTest, SERVER_CONFIG_VALID_TEST) { milvus::server::Config& config = milvus::server::Config::GetInstance(); milvus::Status s; std::string str_val; - int32_t int32_val; int64_t int64_val; float float_val; bool bool_val; @@ -160,26 +159,26 @@ TEST_F(ConfigTest, SERVER_CONFIG_VALID_TEST) { ASSERT_TRUE(s.ok()); ASSERT_TRUE(str_val == db_backend_url); - int32_t db_archive_disk_threshold = 100; + int64_t db_archive_disk_threshold = 100; s = config.SetDBConfigArchiveDiskThreshold(std::to_string(db_archive_disk_threshold)); ASSERT_TRUE(s.ok()); - s = config.GetDBConfigArchiveDiskThreshold(int32_val); + s = config.GetDBConfigArchiveDiskThreshold(int64_val); ASSERT_TRUE(s.ok()); - ASSERT_TRUE(int32_val == db_archive_disk_threshold); + ASSERT_TRUE(int64_val == db_archive_disk_threshold); - int32_t db_archive_days_threshold = 365; + int64_t db_archive_days_threshold = 365; s = config.SetDBConfigArchiveDaysThreshold(std::to_string(db_archive_days_threshold)); ASSERT_TRUE(s.ok()); - s = config.GetDBConfigArchiveDaysThreshold(int32_val); + s = config.GetDBConfigArchiveDaysThreshold(int64_val); ASSERT_TRUE(s.ok()); - ASSERT_TRUE(int32_val == db_archive_days_threshold); + ASSERT_TRUE(int64_val == db_archive_days_threshold); - int32_t db_insert_buffer_size = 2; + int64_t db_insert_buffer_size = 2; s = config.SetDBConfigInsertBufferSize(std::to_string(db_insert_buffer_size)); ASSERT_TRUE(s.ok()); - s = config.GetDBConfigInsertBufferSize(int32_val); + s = config.GetDBConfigInsertBufferSize(int64_val); ASSERT_TRUE(s.ok()); - ASSERT_TRUE(int32_val == db_insert_buffer_size); + ASSERT_TRUE(int64_val == db_insert_buffer_size); /* metric config */ bool metric_enable_monitor = false; @@ -223,26 +222,26 @@ TEST_F(ConfigTest, SERVER_CONFIG_VALID_TEST) { ASSERT_TRUE(bool_val == cache_insert_data); /* engine config */ - int32_t engine_use_blas_threshold = 50; + int64_t engine_use_blas_threshold = 50; s = config.SetEngineConfigUseBlasThreshold(std::to_string(engine_use_blas_threshold)); ASSERT_TRUE(s.ok()); - s = config.GetEngineConfigUseBlasThreshold(int32_val); + s = config.GetEngineConfigUseBlasThreshold(int64_val); ASSERT_TRUE(s.ok()); - ASSERT_TRUE(int32_val == engine_use_blas_threshold); + ASSERT_TRUE(int64_val == engine_use_blas_threshold); - int32_t engine_omp_thread_num = 8; + int64_t engine_omp_thread_num = 8; s = config.SetEngineConfigOmpThreadNum(std::to_string(engine_omp_thread_num)); ASSERT_TRUE(s.ok()); - s = config.GetEngineConfigOmpThreadNum(int32_val); + s = config.GetEngineConfigOmpThreadNum(int64_val); ASSERT_TRUE(s.ok()); - ASSERT_TRUE(int32_val == engine_omp_thread_num); + ASSERT_TRUE(int64_val == engine_omp_thread_num); - int32_t engine_gpu_search_threshold = 800; + int64_t engine_gpu_search_threshold = 800; s = config.SetEngineConfigGpuSearchThreshold(std::to_string(engine_gpu_search_threshold)); ASSERT_TRUE(s.ok()); - s = config.GetEngineConfigGpuSearchThreshold(int32_val); + s = config.GetEngineConfigGpuSearchThreshold(int64_val); ASSERT_TRUE(s.ok()); - ASSERT_TRUE(int32_val == engine_gpu_search_threshold); + ASSERT_TRUE(int64_val == engine_gpu_search_threshold); /* gpu resource config */ bool resource_enable_gpu = true; @@ -267,7 +266,7 @@ TEST_F(ConfigTest, SERVER_CONFIG_VALID_TEST) { ASSERT_TRUE(float_val == gpu_cache_threshold); std::vector search_resources = {"gpu0"}; - std::vector search_res_vec; + std::vector search_res_vec; std::string search_res_str; milvus::server::StringHelpFunctions::MergeStringWithDelimeter( search_resources, milvus::server::CONFIG_GPU_RESOURCE_DELIMITER, search_res_str); @@ -276,11 +275,11 @@ TEST_F(ConfigTest, SERVER_CONFIG_VALID_TEST) { s = config.GetGpuResourceConfigSearchResources(search_res_vec); ASSERT_TRUE(s.ok()); for (size_t i = 0; i < search_resources.size(); i++) { - ASSERT_TRUE(std::stoi(search_resources[i].substr(3)) == search_res_vec[i]); + ASSERT_TRUE(std::stoll(search_resources[i].substr(3)) == search_res_vec[i]); } std::vector build_index_resources = {"gpu0"}; - std::vector build_index_res_vec; + std::vector build_index_res_vec; std::string build_index_res_str; milvus::server::StringHelpFunctions::MergeStringWithDelimeter( build_index_resources, milvus::server::CONFIG_GPU_RESOURCE_DELIMITER, build_index_res_str); @@ -289,7 +288,7 @@ TEST_F(ConfigTest, SERVER_CONFIG_VALID_TEST) { s = config.GetGpuResourceConfigBuildIndexResources(build_index_res_vec); ASSERT_TRUE(s.ok()); for (size_t i = 0; i < build_index_resources.size(); i++) { - ASSERT_TRUE(std::stoi(build_index_resources[i].substr(3)) == build_index_res_vec[i]); + ASSERT_TRUE(std::stoll(build_index_resources[i].substr(3)) == build_index_res_vec[i]); } #endif } diff --git a/core/unittest/server/test_util.cpp b/core/unittest/server/test_util.cpp index 68400f2454..e5884cac65 100644 --- a/core/unittest/server/test_util.cpp +++ b/core/unittest/server/test_util.cpp @@ -60,7 +60,7 @@ TEST(UtilTest, COMMON_TEST) { ASSERT_GT(total_mem, 0); ASSERT_GT(free_mem, 0); - uint32_t thread_cnt = 0; + int64_t thread_cnt = 0; milvus::server::CommonUtil::GetSystemAvailableThreads(thread_cnt); ASSERT_GT(thread_cnt, 0); From 82e6b2738b9b72af25b1495ff50a697cff2c5b1b Mon Sep 17 00:00:00 2001 From: "yudong.cai" Date: Wed, 20 Nov 2019 14:35:10 +0800 Subject: [PATCH 8/8] #306 update changelog --- CHANGELOG.md | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 42d5b8978f..7d88bc270e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,15 +22,16 @@ Please mark all change in change log and use the ticket from JIRA. - \#346 - Support build index with multiple gpu ## Improvement -- \#275 - Rename C++ SDK IndexType -- \#284 - Change C++ SDK to shared library +- \#255 - Add ivfsq8 test report detailed version - \#260 - C++ SDK README - \#266 - Rpc request source code refactor -- \#314 - add Find FAISS in CMake +- \#275 - Rename C++ SDK IndexType +- \#284 - Change C++ SDK to shared library +- \#306 - Use int64 for all config integer - \#310 - Add Q&A for 'protocol https not supported or disable in libcurl' issue +- \#314 - add Find FAISS in CMake - \#322 - Add option to enable / disable prometheus - \#358 - Add more information in build.sh and install.md -- \#255 - Add ivfsq8 test report detailed version - \#404 - Add virtual method Init() in Pass abstract class - \#409 - Add a Fallback pass in optimizer