From b285c64e96928e48d36a6cd9760dcfbb9e298ed7 Mon Sep 17 00:00:00 2001 From: starlord Date: Mon, 22 Jul 2019 10:44:24 +0800 Subject: [PATCH 1/7] Check machine hardware during initialize Former-commit-id: fe3efe3b519235d9579e0c311a7da3f7a299154f --- cpp/CHANGELOG.md | 1 + cpp/conf/server_config.template | 2 +- cpp/src/CMakeLists.txt | 1 + cpp/src/server/RequestTask.cpp | 20 +++---- cpp/src/server/Server.cpp | 4 ++ cpp/src/server/ServerConfig.cpp | 64 +++++++++++++++++++++++ cpp/src/server/ServerConfig.h | 2 + cpp/src/utils/ValidationUtil.cpp | 38 ++++++++++++-- cpp/src/utils/ValidationUtil.h | 21 +++++--- cpp/src/wrapper/IndexBuilder.cpp | 2 +- cpp/unittest/CMakeLists.txt | 1 + cpp/unittest/db/CMakeLists.txt | 4 ++ cpp/unittest/faiss_wrapper/CMakeLists.txt | 4 ++ cpp/unittest/metrics/CMakeLists.txt | 4 ++ cpp/unittest/server/CMakeLists.txt | 1 + cpp/unittest/storage/CMakeLists.txt | 7 ++- cpp/unittest/utils/ValidationUtilTest.cpp | 30 +++++------ 17 files changed, 168 insertions(+), 38 deletions(-) diff --git a/cpp/CHANGELOG.md b/cpp/CHANGELOG.md index 56c5c5faa8..6e1fff4a2d 100644 --- a/cpp/CHANGELOG.md +++ b/cpp/CHANGELOG.md @@ -35,6 +35,7 @@ Please mark all change in change log and use the ticket from JIRA. - MS-248 - Support AddVector/SearchVector profiling - MS-256 - Add more cache config - MS-260 - Refine log +- MS-249 - Check machine hardware during initialize ## New Feature - MS-180 - Add new mem manager diff --git a/cpp/conf/server_config.template b/cpp/conf/server_config.template index b079d603d2..215ca45027 100644 --- a/cpp/conf/server_config.template +++ b/cpp/conf/server_config.template @@ -17,7 +17,7 @@ db_config: archive_disk_threshold: 0 # triger archive action if storage size exceed this value, 0 means no limit, unit: GB archive_days_threshold: 0 # files older than x days will be archived, 0 means no limit, unit: day insert_buffer_size: 4 # maximum insert buffer size allowed, default: 4, unit: GB, should be at least 1 GB. - # the sum of insert_buffer_size and cpu_cache_capacity should be less than total memory + # the sum of insert_buffer_size and cpu_cache_capacity should be less than total memory, unit: GB metric_config: is_startup: off # if monitoring start: on, off diff --git a/cpp/src/CMakeLists.txt b/cpp/src/CMakeLists.txt index 64edf5b91e..5784734dd6 100644 --- a/cpp/src/CMakeLists.txt +++ b/cpp/src/CMakeLists.txt @@ -88,6 +88,7 @@ set(third_party_libs mysqlpp ${PROFILER_LIB} ${CUDA_TOOLKIT_ROOT_DIR}/lib64/stubs/libnvidia-ml.so + cudart ) if (MEGASEARCH_WITH_ARROW STREQUAL "ON") diff --git a/cpp/src/server/RequestTask.cpp b/cpp/src/server/RequestTask.cpp index d6c1a2fb9c..0ab6a6a5e4 100644 --- a/cpp/src/server/RequestTask.cpp +++ b/cpp/src/server/RequestTask.cpp @@ -162,17 +162,17 @@ ServerError CreateTableTask::OnExecute() { try { //step 1: check arguments ServerError res = SERVER_SUCCESS; - res = ValidateTableName(schema_.table_name); + res = ValidationUtil::ValidateTableName(schema_.table_name); if(res != SERVER_SUCCESS) { return SetError(res, "Invalid table name: " + schema_.table_name); } - res = ValidateTableDimension(schema_.dimension); + res = ValidationUtil::ValidateTableDimension(schema_.dimension); if(res != SERVER_SUCCESS) { return SetError(res, "Invalid table dimension: " + std::to_string(schema_.dimension)); } - res = ValidateTableIndexType(schema_.index_type); + res = ValidationUtil::ValidateTableIndexType(schema_.index_type); if(res != SERVER_SUCCESS) { return SetError(res, "Invalid index type: " + std::to_string(schema_.index_type)); } @@ -217,7 +217,7 @@ ServerError DescribeTableTask::OnExecute() { try { //step 1: check arguments ServerError res = SERVER_SUCCESS; - res = ValidateTableName(table_name_); + res = ValidationUtil::ValidateTableName(table_name_); if(res != SERVER_SUCCESS) { return SetError(res, "Invalid table name: " + table_name_); } @@ -260,7 +260,7 @@ ServerError BuildIndexTask::OnExecute() { //step 1: check arguments ServerError res = SERVER_SUCCESS; - res = ValidateTableName(table_name_); + res = ValidationUtil::ValidateTableName(table_name_); if(res != SERVER_SUCCESS) { return SetError(res, "Invalid table name: " + table_name_); } @@ -303,7 +303,7 @@ ServerError HasTableTask::OnExecute() { //step 1: check arguments ServerError res = SERVER_SUCCESS; - res = ValidateTableName(table_name_); + res = ValidationUtil::ValidateTableName(table_name_); if(res != SERVER_SUCCESS) { return SetError(res, "Invalid table name: " + table_name_); } @@ -339,7 +339,7 @@ ServerError DeleteTableTask::OnExecute() { //step 1: check arguments ServerError res = SERVER_SUCCESS; - res = ValidateTableName(table_name_); + res = ValidationUtil::ValidateTableName(table_name_); if(res != SERVER_SUCCESS) { return SetError(res, "Invalid table name: " + table_name_); } @@ -420,7 +420,7 @@ ServerError AddVectorTask::OnExecute() { //step 1: check arguments ServerError res = SERVER_SUCCESS; - res = ValidateTableName(table_name_); + res = ValidationUtil::ValidateTableName(table_name_); if(res != SERVER_SUCCESS) { return SetError(res, "Invalid table name: " + table_name_); } @@ -508,7 +508,7 @@ ServerError SearchVectorTaskBase::OnExecute() { //step 1: check arguments ServerError res = SERVER_SUCCESS; - res = ValidateTableName(table_name_); + res = ValidationUtil::ValidateTableName(table_name_); if(res != SERVER_SUCCESS) { return SetError(res, "Invalid table name: " + table_name_); } @@ -720,7 +720,7 @@ ServerError GetTableRowCountTask::OnExecute() { //step 1: check arguments ServerError res = SERVER_SUCCESS; - res = ValidateTableName(table_name_); + res = ValidationUtil::ValidateTableName(table_name_); if(res != SERVER_SUCCESS) { return SetError(res, "Invalid table name: " + table_name_); } diff --git a/cpp/src/server/Server.cpp b/cpp/src/server/Server.cpp index a79a8c7357..aeccc1e5be 100644 --- a/cpp/src/server/Server.cpp +++ b/cpp/src/server/Server.cpp @@ -226,6 +226,10 @@ Server::Stop() { ServerError Server::LoadConfig() { ServerConfig::GetInstance().LoadConfigFile(config_filename_); + ServerError err = ServerConfig::GetInstance().ValidateConfig(); + if(err != SERVER_SUCCESS){ + exit(0); + } return SERVER_SUCCESS; } diff --git a/cpp/src/server/ServerConfig.cpp b/cpp/src/server/ServerConfig.cpp index 736a249e25..bb1879e0b4 100644 --- a/cpp/src/server/ServerConfig.cpp +++ b/cpp/src/server/ServerConfig.cpp @@ -12,11 +12,16 @@ #include #include "config/IConfigMgr.h" +#include "utils/CommonUtil.h" +#include "utils/ValidationUtil.h" namespace zilliz { namespace milvus { namespace server { +constexpr uint64_t MB = 1024*1024; +constexpr uint64_t GB = MB*1024; + ServerConfig& ServerConfig::GetInstance() { static ServerConfig config; @@ -53,6 +58,65 @@ ServerConfig::LoadConfigFile(const std::string& config_filename) { return SERVER_SUCCESS; } +ServerError ServerConfig::ValidateConfig() const { + //server config validation + ConfigNode server_config = GetConfig(CONFIG_SERVER); + uint32_t gpu_index = (uint32_t)server_config.GetInt32Value(CONFIG_GPU_INDEX); + if(ValidationUtil::ValidateGpuIndex(gpu_index) != SERVER_SUCCESS) { + std::cout << "Error: invalid gpu_index " << std::to_string(gpu_index) << std::endl; + return SERVER_INVALID_ARGUMENT; + } + + //db config validation + unsigned long total_mem = 0, free_mem = 0; + CommonUtil::GetSystemMemInfo(total_mem, free_mem); + + ConfigNode db_config = GetConfig(CONFIG_DB); + uint64_t insert_buffer_size = (uint64_t)db_config.GetInt32Value(CONFIG_DB_INSERT_BUFFER_SIZE); + insert_buffer_size *= GB; + if(insert_buffer_size >= total_mem) { + std::cout << "Error: insert_buffer_size execeed system memory" << std::endl; + return SERVER_INVALID_ARGUMENT; + } + + uint64_t index_building_threshold = (uint64_t)db_config.GetInt32Value(CONFIG_DB_INDEX_TRIGGER_SIZE); + index_building_threshold *= MB; + + size_t gpu_mem = 0; + ValidationUtil::GetGpuMemory(gpu_index, gpu_mem); + if(index_building_threshold >= gpu_mem/3) { + std::cout << "Warnning: index_building_threshold is greater than 1/3 of gpu memory, " + << "some index type(such as IVFLAT) may cause cuda::bad_alloc() error" << std::endl; + } else if(index_building_threshold >= gpu_mem) { + std::cout << "Error: index_building_threshold execeed gpu memory" << std::endl; + return SERVER_INVALID_ARGUMENT; + } + + //cache config validation + ConfigNode cache_config = GetConfig(CONFIG_CACHE); + uint64_t cache_cap = (uint64_t)cache_config.GetInt64Value(CONFIG_CPU_CACHE_CAPACITY); + cache_cap *= GB; + if(cache_cap >= total_mem) { + std::cout << "Error: cpu_cache_capacity execeed system memory" << std::endl; + return SERVER_INVALID_ARGUMENT; + } if(cache_cap > (double)total_mem*0.9) { + std::cout << "Warnning: cpu_cache_capacity value is too aggressive" << std::endl; + } + + if(insert_buffer_size + cache_cap >= total_mem) if(cache_cap >= total_mem) { + std::cout << "Error: sum of cpu_cache_capacity and insert_buffer_size execeed system memory" << std::endl; + return SERVER_INVALID_ARGUMENT; + } + + double free_percent = cache_config.GetDoubleValue(server::CACHE_FREE_PERCENT); + if(free_percent < std::numeric_limits::epsilon() || free_percent > 1.0) { + std::cout << "Error: invalid cache_free_percent " << std::to_string(free_percent) << std::endl; + return SERVER_INVALID_ARGUMENT; + } + + return SERVER_SUCCESS; +} + void ServerConfig::PrintAll() const { if(const IConfigMgr* mgr = IConfigMgr::GetInstance()) { diff --git a/cpp/src/server/ServerConfig.h b/cpp/src/server/ServerConfig.h index 12d1e8b889..4a1b089810 100644 --- a/cpp/src/server/ServerConfig.h +++ b/cpp/src/server/ServerConfig.h @@ -19,6 +19,7 @@ static const std::string CONFIG_SERVER_ADDRESS = "address"; static const std::string CONFIG_SERVER_PORT = "port"; static const std::string CONFIG_SERVER_PROTOCOL = "transfer_protocol"; static const std::string CONFIG_CLUSTER_MODE = "mode"; +static const std::string CONFIG_GPU_INDEX = "gpu_index"; static const std::string CONFIG_DB = "db_config"; static const std::string CONFIG_DB_URL = "db_backend_url"; @@ -57,6 +58,7 @@ class ServerConfig { static ServerConfig &GetInstance(); ServerError LoadConfigFile(const std::string& config_filename); + ServerError ValidateConfig() const; void PrintAll() const; ConfigNode GetConfig(const std::string& name) const; diff --git a/cpp/src/utils/ValidationUtil.cpp b/cpp/src/utils/ValidationUtil.cpp index 89b5573999..bfdf35cee2 100644 --- a/cpp/src/utils/ValidationUtil.cpp +++ b/cpp/src/utils/ValidationUtil.cpp @@ -1,7 +1,8 @@ -#include +#include "db/ExecutionEngine.h" #include "ValidationUtil.h" #include "Log.h" +#include namespace zilliz { namespace milvus { @@ -11,7 +12,7 @@ constexpr size_t table_name_size_limit = 255; constexpr int64_t table_dimension_limit = 16384; ServerError -ValidateTableName(const std::string &table_name) { +ValidationUtil::ValidateTableName(const std::string &table_name) { // Table name shouldn't be empty. if (table_name.empty()) { @@ -45,7 +46,7 @@ ValidateTableName(const std::string &table_name) { } ServerError -ValidateTableDimension(int64_t dimension) { +ValidationUtil::ValidateTableDimension(int64_t dimension) { if (dimension <= 0 || dimension > table_dimension_limit) { SERVER_LOG_ERROR << "Table dimension excceed the limitation: " << table_dimension_limit; return SERVER_INVALID_VECTOR_DIMENSION; @@ -55,7 +56,7 @@ ValidateTableDimension(int64_t dimension) { } ServerError -ValidateTableIndexType(int32_t index_type) { +ValidationUtil::ValidateTableIndexType(int32_t index_type) { int engine_type = (int)engine::EngineType(index_type); if(engine_type <= 0 || engine_type > (int)engine::EngineType::MAX_VALUE) { return SERVER_INVALID_INDEX_TYPE; @@ -64,6 +65,35 @@ ValidateTableIndexType(int32_t index_type) { return SERVER_SUCCESS; } +ServerError +ValidationUtil::ValidateGpuIndex(uint32_t gpu_index) { + int num_devices = 0; + auto cuda_err = cudaGetDeviceCount(&num_devices); + if (cuda_err) { + SERVER_LOG_ERROR << "Failed to count video card: " << std::to_string(cuda_err); + return SERVER_UNEXPECTED_ERROR; + } + + if(gpu_index >= num_devices) { + return SERVER_INVALID_ARGUMENT; + } + + return SERVER_SUCCESS; +} + +ServerError +ValidationUtil::GetGpuMemory(uint32_t gpu_index, size_t& memory) { + cudaDeviceProp deviceProp; + auto cuda_err = cudaGetDeviceProperties(&deviceProp, gpu_index); + if (cuda_err) { + SERVER_LOG_ERROR << "Failed to get video card properties: " << std::to_string(cuda_err); + return SERVER_UNEXPECTED_ERROR; + } + + memory = deviceProp.totalGlobalMem; + return SERVER_SUCCESS; +} + } } } \ No newline at end of file diff --git a/cpp/src/utils/ValidationUtil.h b/cpp/src/utils/ValidationUtil.h index 608ac22682..1f90fac273 100644 --- a/cpp/src/utils/ValidationUtil.h +++ b/cpp/src/utils/ValidationUtil.h @@ -6,14 +6,23 @@ namespace zilliz { namespace milvus { namespace server { -ServerError -ValidateTableName(const std::string& table_name); +class ValidationUtil { +public: + static ServerError + ValidateTableName(const std::string &table_name); -ServerError -ValidateTableDimension(int64_t dimension); + static ServerError + ValidateTableDimension(int64_t dimension); -ServerError -ValidateTableIndexType(int32_t index_type); + static ServerError + ValidateTableIndexType(int32_t index_type); + + static ServerError + ValidateGpuIndex(uint32_t gpu_index); + + static ServerError + GetGpuMemory(uint32_t gpu_index, size_t &memory); +}; } } diff --git a/cpp/src/wrapper/IndexBuilder.cpp b/cpp/src/wrapper/IndexBuilder.cpp index 41859907a7..62781751e2 100644 --- a/cpp/src/wrapper/IndexBuilder.cpp +++ b/cpp/src/wrapper/IndexBuilder.cpp @@ -37,7 +37,7 @@ class GpuResources { using namespace zilliz::milvus::server; ServerConfig &config = ServerConfig::GetInstance(); ConfigNode server_config = config.GetConfig(CONFIG_SERVER); - gpu_num = server_config.GetInt32Value("gpu_index", 0); + gpu_num = server_config.GetInt32Value(server::CONFIG_GPU_INDEX, 0); } int32_t GetGpu() { diff --git a/cpp/unittest/CMakeLists.txt b/cpp/unittest/CMakeLists.txt index 6a6f94a632..f4a670acf3 100644 --- a/cpp/unittest/CMakeLists.txt +++ b/cpp/unittest/CMakeLists.txt @@ -35,6 +35,7 @@ set(unittest_libs dl z ${CUDA_TOOLKIT_ROOT_DIR}/lib64/stubs/libnvidia-ml.so + cudart ) add_subdirectory(server) diff --git a/cpp/unittest/db/CMakeLists.txt b/cpp/unittest/db/CMakeLists.txt index 736219952c..a7bed578dd 100644 --- a/cpp/unittest/db/CMakeLists.txt +++ b/cpp/unittest/db/CMakeLists.txt @@ -9,6 +9,9 @@ aux_source_directory(${MILVUS_ENGINE_SRC}/cache cache_srcs) aux_source_directory(${MILVUS_ENGINE_SRC}/wrapper wrapper_src) aux_source_directory(./ test_srcs) +set(util_files + ${MILVUS_ENGINE_SRC}/utils/ValidationUtil.cpp) + aux_source_directory(${MILVUS_ENGINE_SRC}/db/scheduler scheduler_files) aux_source_directory(${MILVUS_ENGINE_SRC}/db/scheduler/context scheduler_context_files) aux_source_directory(${MILVUS_ENGINE_SRC}/db/scheduler/task scheduler_task_files) @@ -29,6 +32,7 @@ set(db_test_src ${db_srcs} ${db_scheduler_srcs} ${wrapper_src} + ${util_files} ${require_files} ${test_srcs}) diff --git a/cpp/unittest/faiss_wrapper/CMakeLists.txt b/cpp/unittest/faiss_wrapper/CMakeLists.txt index c439250544..c906c8ec57 100644 --- a/cpp/unittest/faiss_wrapper/CMakeLists.txt +++ b/cpp/unittest/faiss_wrapper/CMakeLists.txt @@ -6,6 +6,9 @@ aux_source_directory(${MILVUS_ENGINE_SRC}/wrapper wrapper_src) aux_source_directory(${MILVUS_ENGINE_SRC}/config config_files) +set(util_files + ${MILVUS_ENGINE_SRC}/utils/ValidationUtil.cpp) + # Make sure that your call to link_directories takes place before your call to the relevant add_executable. include_directories(/usr/local/cuda/include) link_directories("/usr/local/cuda/lib64") @@ -14,6 +17,7 @@ set(wrapper_test_src ${unittest_srcs} ${wrapper_src} ${config_files} + ${util_files} ${require_files} wrapper_test.cpp ) diff --git a/cpp/unittest/metrics/CMakeLists.txt b/cpp/unittest/metrics/CMakeLists.txt index 418544d0ca..661201fbc0 100644 --- a/cpp/unittest/metrics/CMakeLists.txt +++ b/cpp/unittest/metrics/CMakeLists.txt @@ -17,6 +17,9 @@ aux_source_directory(../../src/wrapper wrapper_src) aux_source_directory(../../src/metrics metrics_src) aux_source_directory(./ test_srcs) +set(util_files + ${MILVUS_ENGINE_SRC}/utils/ValidationUtil.cpp) + aux_source_directory(${MILVUS_ENGINE_SRC}/db/scheduler scheduler_files) aux_source_directory(${MILVUS_ENGINE_SRC}/db/scheduler/context scheduler_context_files) aux_source_directory(${MILVUS_ENGINE_SRC}/db/scheduler/task scheduler_task_files) @@ -43,6 +46,7 @@ set(count_test_src ${wrapper_src} ${metrics_src} ${test_srcs} + ${util_files} ) diff --git a/cpp/unittest/server/CMakeLists.txt b/cpp/unittest/server/CMakeLists.txt index 40f5b10128..0eb6c7cd10 100644 --- a/cpp/unittest/server/CMakeLists.txt +++ b/cpp/unittest/server/CMakeLists.txt @@ -19,6 +19,7 @@ set(utils_srcs ${MILVUS_ENGINE_SRC}/utils/TimeRecorder.cpp ${MILVUS_ENGINE_SRC}/utils/CommonUtil.cpp ${MILVUS_ENGINE_SRC}/utils/LogUtil.cpp + ${MILVUS_ENGINE_SRC}/utils/ValidationUtil.cpp ) cuda_add_executable(server_test diff --git a/cpp/unittest/storage/CMakeLists.txt b/cpp/unittest/storage/CMakeLists.txt index d4deaefab8..7529028e3c 100644 --- a/cpp/unittest/storage/CMakeLists.txt +++ b/cpp/unittest/storage/CMakeLists.txt @@ -5,6 +5,9 @@ #------------------------------------------------------------------------------- aux_source_directory(${MILVUS_ENGINE_SRC}/storage/s3 s3_client_src) +set(util_files + ${MILVUS_ENGINE_SRC}/utils/ValidationUtil.cpp) + # Make sure that your call to link_directories takes place before your call to the relevant add_executable. include_directories("${CUDA_TOOLKIT_ROOT_DIR}/include") link_directories("${CUDA_TOOLKIT_ROOT_DIR}/lib64") @@ -19,7 +22,9 @@ set(s3_client_test_src add_executable(s3_test ${s3_client_test_src} - ${config_files}) + ${config_files} + ${util_files} + ) set(s3_client_libs stdc++ diff --git a/cpp/unittest/utils/ValidationUtilTest.cpp b/cpp/unittest/utils/ValidationUtilTest.cpp index 38fc63a10d..871172e6c9 100644 --- a/cpp/unittest/utils/ValidationUtilTest.cpp +++ b/cpp/unittest/utils/ValidationUtilTest.cpp @@ -16,48 +16,48 @@ using namespace zilliz::milvus::server; TEST(ValidationUtilTest, TableNameTest) { std::string table_name = "Normal123_"; - ServerError res = ValidateTableName(table_name); + ServerError res = ValidationUtil::ValidateTableName(table_name); ASSERT_EQ(res, SERVER_SUCCESS); table_name = "12sds"; - res = ValidateTableName(table_name); + res = ValidationUtil::ValidateTableName(table_name); ASSERT_EQ(res, SERVER_INVALID_TABLE_NAME); table_name = ""; - res = ValidateTableName(table_name); + res = ValidationUtil::ValidateTableName(table_name); ASSERT_EQ(res, SERVER_INVALID_TABLE_NAME); table_name = "_asdasd"; - res = ValidateTableName(table_name); + res = ValidationUtil::ValidateTableName(table_name); ASSERT_EQ(res, SERVER_SUCCESS); table_name = "!@#!@"; - res = ValidateTableName(table_name); + res = ValidationUtil::ValidateTableName(table_name); ASSERT_EQ(res, SERVER_INVALID_TABLE_NAME); table_name = "中文"; - res = ValidateTableName(table_name); + res = ValidationUtil::ValidateTableName(table_name); ASSERT_EQ(res, SERVER_INVALID_TABLE_NAME); table_name = std::string('a', 32768); - res = ValidateTableName(table_name); + res = ValidationUtil::ValidateTableName(table_name); ASSERT_EQ(res, SERVER_INVALID_TABLE_NAME); } TEST(ValidationUtilTest, TableDimensionTest) { - ASSERT_EQ(ValidateTableDimension(-1), SERVER_INVALID_VECTOR_DIMENSION); - ASSERT_EQ(ValidateTableDimension(0), SERVER_INVALID_VECTOR_DIMENSION); - ASSERT_EQ(ValidateTableDimension(16385), SERVER_INVALID_VECTOR_DIMENSION); - ASSERT_EQ(ValidateTableDimension(16384), SERVER_SUCCESS); - ASSERT_EQ(ValidateTableDimension(1), SERVER_SUCCESS); + ASSERT_EQ(ValidationUtil::ValidateTableDimension(-1), SERVER_INVALID_VECTOR_DIMENSION); + ASSERT_EQ(ValidationUtil::ValidateTableDimension(0), SERVER_INVALID_VECTOR_DIMENSION); + ASSERT_EQ(ValidationUtil::ValidateTableDimension(16385), SERVER_INVALID_VECTOR_DIMENSION); + ASSERT_EQ(ValidationUtil::ValidateTableDimension(16384), SERVER_SUCCESS); + ASSERT_EQ(ValidationUtil::ValidateTableDimension(1), SERVER_SUCCESS); } TEST(ValidationUtilTest, TableIndexTypeTest) { - ASSERT_EQ(ValidateTableIndexType((int)engine::EngineType::INVALID), SERVER_INVALID_INDEX_TYPE); + ASSERT_EQ(ValidationUtil::ValidateTableIndexType((int)engine::EngineType::INVALID), SERVER_INVALID_INDEX_TYPE); for(int i = 1; i <= (int)engine::EngineType::MAX_VALUE; i++) { - ASSERT_EQ(ValidateTableIndexType(i), SERVER_SUCCESS); + ASSERT_EQ(ValidationUtil::ValidateTableIndexType(i), SERVER_SUCCESS); } - ASSERT_EQ(ValidateTableIndexType((int)engine::EngineType::MAX_VALUE + 1), SERVER_INVALID_INDEX_TYPE); + ASSERT_EQ(ValidationUtil::ValidateTableIndexType((int)engine::EngineType::MAX_VALUE + 1), SERVER_INVALID_INDEX_TYPE); } From eb1d1e393f9cb56205ac2ab86a8250bf94b1348b Mon Sep 17 00:00:00 2001 From: starlord Date: Mon, 22 Jul 2019 11:15:11 +0800 Subject: [PATCH 2/7] MS-249 Check machine hardware during initialize Former-commit-id: e513499fb015a6867a762a10b629dd774182f21a --- cpp/src/server/ServerConfig.cpp | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/cpp/src/server/ServerConfig.cpp b/cpp/src/server/ServerConfig.cpp index bb1879e0b4..4a5d0d0931 100644 --- a/cpp/src/server/ServerConfig.cpp +++ b/cpp/src/server/ServerConfig.cpp @@ -84,12 +84,12 @@ ServerError ServerConfig::ValidateConfig() const { size_t gpu_mem = 0; ValidationUtil::GetGpuMemory(gpu_index, gpu_mem); - if(index_building_threshold >= gpu_mem/3) { - std::cout << "Warnning: index_building_threshold is greater than 1/3 of gpu memory, " - << "some index type(such as IVFLAT) may cause cuda::bad_alloc() error" << std::endl; - } else if(index_building_threshold >= gpu_mem) { + if(index_building_threshold >= gpu_mem) { std::cout << "Error: index_building_threshold execeed gpu memory" << std::endl; return SERVER_INVALID_ARGUMENT; + } else if(index_building_threshold >= gpu_mem/3) { + std::cout << "Warnning: index_building_threshold is greater than 1/3 of gpu memory, " + << "some index type(such as IVFLAT) may cause cuda::bad_alloc() error" << std::endl; } //cache config validation @@ -103,7 +103,7 @@ ServerError ServerConfig::ValidateConfig() const { std::cout << "Warnning: cpu_cache_capacity value is too aggressive" << std::endl; } - if(insert_buffer_size + cache_cap >= total_mem) if(cache_cap >= total_mem) { + if(insert_buffer_size + cache_cap >= total_mem) { std::cout << "Error: sum of cpu_cache_capacity and insert_buffer_size execeed system memory" << std::endl; return SERVER_INVALID_ARGUMENT; } From 2671c711f6ada431ca829b2a288567e62b89874e Mon Sep 17 00:00:00 2001 From: quicksilver Date: Mon, 22 Jul 2019 14:00:09 +0800 Subject: [PATCH 3/7] update test namespace Former-commit-id: 6adcb38e41730728e6df89be6da343c82e6f6f31 --- ci/jenkinsfile/deploy2dev.groovy | 8 -------- ci/jenkinsfile/dev_test.groovy | 4 ++-- 2 files changed, 2 insertions(+), 10 deletions(-) diff --git a/ci/jenkinsfile/deploy2dev.groovy b/ci/jenkinsfile/deploy2dev.groovy index 52cbae2bfe..e980699f1f 100644 --- a/ci/jenkinsfile/deploy2dev.groovy +++ b/ci/jenkinsfile/deploy2dev.groovy @@ -8,14 +8,6 @@ try { sh "helm install --wait --timeout 300 --set engine.image.tag=${DOCKER_VERSION} --set expose.type=clusterIP --name ${env.JOB_NAME}-${env.BUILD_NUMBER} -f ci/values.yaml --namespace milvus-1 --version 0.3.1 ." } } - /* - timeout(time: 2, unit: 'MINUTES') { - waitUntil { - def result = sh script: "nc -z -w 3 ${env.JOB_NAME}-${env.BUILD_NUMBER}-milvus-gpu-engine.kube-opt.svc.cluster.local 19530", returnStatus: true - return !result - } - } - */ } catch (exc) { echo 'Helm running failed!' sh "helm del --purge ${env.JOB_NAME}-${env.BUILD_NUMBER}" diff --git a/ci/jenkinsfile/dev_test.groovy b/ci/jenkinsfile/dev_test.groovy index fb5b4749cc..1b6ef4de13 100644 --- a/ci/jenkinsfile/dev_test.groovy +++ b/ci/jenkinsfile/dev_test.groovy @@ -3,7 +3,7 @@ timeout(time: 20, unit: 'MINUTES') { dir ("${PROJECT_NAME}_test") { checkout([$class: 'GitSCM', branches: [[name: "${SEMVER}"]], doGenerateSubmoduleConfigurations: false, extensions: [], submoduleCfg: [], userRemoteConfigs: [[credentialsId: "${params.GIT_USER}", url: "git@192.168.1.105:Test/milvus_test.git", name: 'origin', refspec: "+refs/heads/${SEMVER}:refs/remotes/origin/${SEMVER}"]]]) sh 'python3 -m pip install -r requirements.txt' - sh "pytest . --alluredir=test_out --ip ${env.JOB_NAME}-${env.BUILD_NUMBER}-milvus-gpu-engine.kube-opt.svc.cluster.local" + sh "pytest . --alluredir=test_out --ip ${env.JOB_NAME}-${env.BUILD_NUMBER}-milvus-gpu-engine.milvus-1.svc.cluster.local" } // mysql database backend test @@ -20,7 +20,7 @@ timeout(time: 20, unit: 'MINUTES') { } } dir ("${PROJECT_NAME}_test") { - sh "pytest . --alluredir=test_out --ip ${env.JOB_NAME}-${env.BUILD_NUMBER}-milvus-gpu-engine.kube-opt.svc.cluster.local" + sh "pytest . --alluredir=test_out --ip ${env.JOB_NAME}-${env.BUILD_NUMBER}-milvus-gpu-engine.milvus-2.svc.cluster.local" } } catch (exc) { echo 'Milvus Test Failed !' From f7742f7281c937373946505c840de9ba0705a7c9 Mon Sep 17 00:00:00 2001 From: starlord Date: Mon, 22 Jul 2019 14:25:33 +0800 Subject: [PATCH 4/7] MS-249 Check machine hardware during initialize Former-commit-id: 874dcaa8f6000158409de2865ca28050e4ceeac3 --- cpp/conf/server_config.template | 2 +- cpp/src/cache/Cache.cpp | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/cpp/conf/server_config.template b/cpp/conf/server_config.template index 215ca45027..2942ffa179 100644 --- a/cpp/conf/server_config.template +++ b/cpp/conf/server_config.template @@ -34,7 +34,7 @@ license_config: # license configure cache_config: # cache configure cpu_cache_capacity: 16 # how many memory are used as cache, unit: GB, range: 0 ~ less than total memory - cache_free_percent: 0.85 # how much memory should be free when cache is full, range: greater than zero ~ 1.0 + cache_free_percent: 0.85 # old data will be erased from cache when cache is full, this value specify how much memory should be kept, range: greater than zero ~ 1.0 insert_cache_immediately: false # insert data will be load into cache immediately for hot query engine_config: diff --git a/cpp/src/cache/Cache.cpp b/cpp/src/cache/Cache.cpp index 0cc804ac5f..a1f2520302 100644 --- a/cpp/src/cache/Cache.cpp +++ b/cpp/src/cache/Cache.cpp @@ -163,6 +163,9 @@ void Cache::free_memory() { int64_t threshhold = capacity_ * freemem_percent_; int64_t delta_size = usage_ - threshhold; + if(delta_size <= 0) { + delta_size = 1;//ensure at least one item erased + } std::set key_array; int64_t released_size = 0; From add911ae22aed014a7f0785106fa1222a538f55c Mon Sep 17 00:00:00 2001 From: starlord Date: Mon, 22 Jul 2019 15:33:36 +0800 Subject: [PATCH 5/7] MS-249 Check machine hardware during initialize Former-commit-id: 9f42d9c41dcfac9bc1716737368826bfcd777251 --- cpp/src/server/ServerConfig.cpp | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/cpp/src/server/ServerConfig.cpp b/cpp/src/server/ServerConfig.cpp index 4a5d0d0931..71d32452ac 100644 --- a/cpp/src/server/ServerConfig.cpp +++ b/cpp/src/server/ServerConfig.cpp @@ -61,7 +61,7 @@ ServerConfig::LoadConfigFile(const std::string& config_filename) { ServerError ServerConfig::ValidateConfig() const { //server config validation ConfigNode server_config = GetConfig(CONFIG_SERVER); - uint32_t gpu_index = (uint32_t)server_config.GetInt32Value(CONFIG_GPU_INDEX); + uint32_t gpu_index = (uint32_t)server_config.GetInt32Value(CONFIG_GPU_INDEX, 0); if(ValidationUtil::ValidateGpuIndex(gpu_index) != SERVER_SUCCESS) { std::cout << "Error: invalid gpu_index " << std::to_string(gpu_index) << std::endl; return SERVER_INVALID_ARGUMENT; @@ -72,14 +72,14 @@ ServerError ServerConfig::ValidateConfig() const { CommonUtil::GetSystemMemInfo(total_mem, free_mem); ConfigNode db_config = GetConfig(CONFIG_DB); - uint64_t insert_buffer_size = (uint64_t)db_config.GetInt32Value(CONFIG_DB_INSERT_BUFFER_SIZE); + uint64_t insert_buffer_size = (uint64_t)db_config.GetInt32Value(CONFIG_DB_INSERT_BUFFER_SIZE, 4); insert_buffer_size *= GB; if(insert_buffer_size >= total_mem) { std::cout << "Error: insert_buffer_size execeed system memory" << std::endl; return SERVER_INVALID_ARGUMENT; } - uint64_t index_building_threshold = (uint64_t)db_config.GetInt32Value(CONFIG_DB_INDEX_TRIGGER_SIZE); + uint64_t index_building_threshold = (uint64_t)db_config.GetInt32Value(CONFIG_DB_INDEX_TRIGGER_SIZE, 1024); index_building_threshold *= MB; size_t gpu_mem = 0; @@ -94,7 +94,7 @@ ServerError ServerConfig::ValidateConfig() const { //cache config validation ConfigNode cache_config = GetConfig(CONFIG_CACHE); - uint64_t cache_cap = (uint64_t)cache_config.GetInt64Value(CONFIG_CPU_CACHE_CAPACITY); + uint64_t cache_cap = (uint64_t)cache_config.GetInt64Value(CONFIG_CPU_CACHE_CAPACITY, 16); cache_cap *= GB; if(cache_cap >= total_mem) { std::cout << "Error: cpu_cache_capacity execeed system memory" << std::endl; @@ -108,7 +108,7 @@ ServerError ServerConfig::ValidateConfig() const { return SERVER_INVALID_ARGUMENT; } - double free_percent = cache_config.GetDoubleValue(server::CACHE_FREE_PERCENT); + double free_percent = cache_config.GetDoubleValue(server::CACHE_FREE_PERCENT, 0.85); if(free_percent < std::numeric_limits::epsilon() || free_percent > 1.0) { std::cout << "Error: invalid cache_free_percent " << std::to_string(free_percent) << std::endl; return SERVER_INVALID_ARGUMENT; From 5f86066e8d51a5eaa18fb3eb15361e1fe7ed15ec Mon Sep 17 00:00:00 2001 From: starlord Date: Mon, 22 Jul 2019 17:49:11 +0800 Subject: [PATCH 6/7] MS-246 refine log Former-commit-id: d8c30756db5666f964fd4fcf63fdc4a7ab08a4ea --- cpp/src/server/RequestTask.cpp | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/cpp/src/server/RequestTask.cpp b/cpp/src/server/RequestTask.cpp index 0ab6a6a5e4..d9e50a82bd 100644 --- a/cpp/src/server/RequestTask.cpp +++ b/cpp/src/server/RequestTask.cpp @@ -504,7 +504,9 @@ SearchVectorTaskBase::SearchVectorTaskBase(const std::string &table_name, ServerError SearchVectorTaskBase::OnExecute() { try { - TimeRecorder rc("SearchVectorTask"); + std::string title = "SearchVectorTask(n=" + std::to_string(record_array_.size()) + + " k=" + std::to_string(top_k_) + ")"; + TimeRecorder rc(title); //step 1: check arguments ServerError res = SERVER_SUCCESS; @@ -596,7 +598,7 @@ ServerError SearchVectorTaskBase::OnExecute() { //step 6: print time cost percent double total_cost = span_check + span_prepare + span_search + span_result; - SERVER_LOG_DEBUG << "SearchVectorTask: " << "check validation(" << (span_check/total_cost)*100.0 << "%)" + SERVER_LOG_DEBUG << title << ": check validation(" << (span_check/total_cost)*100.0 << "%)" << " prepare data(" << (span_prepare/total_cost)*100.0 << "%)" << " search(" << (span_search/total_cost)*100.0 << "%)" << " construct result(" << (span_result/total_cost)*100.0 << "%)"; From cdb031e62bd9781b3c051f52e46bda8d27706487 Mon Sep 17 00:00:00 2001 From: "peng.xu" Date: Mon, 22 Jul 2019 20:03:59 +0800 Subject: [PATCH 7/7] Revert "Merge branch 'branch-0.3.1' into 'branch-0.3.1'" This reverts merge request !264 Former-commit-id: 501e92946ab87fb9a21c55545f84ab8da499e1aa --- cpp/build.sh | 2 +- cpp/cmake/ThirdPartyPackages.cmake | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cpp/build.sh b/cpp/build.sh index edfe9305be..ef66a0c375 100755 --- a/cpp/build.sh +++ b/cpp/build.sh @@ -8,7 +8,7 @@ MAKE_CLEAN="OFF" BUILD_COVERAGE="OFF" DB_PATH="/opt/milvus" PROFILING="OFF" -BUILD_FAISS_WITH_MKL="OFF" +BUILD_FAISS_WITH_MKL="ON" while getopts "p:d:t:uhlrcgm" arg do diff --git a/cpp/cmake/ThirdPartyPackages.cmake b/cpp/cmake/ThirdPartyPackages.cmake index fed5dece88..837d6bd470 100644 --- a/cpp/cmake/ThirdPartyPackages.cmake +++ b/cpp/cmake/ThirdPartyPackages.cmake @@ -700,7 +700,7 @@ endmacro() # FAISS if(NOT DEFINED BUILD_FAISS_WITH_MKL) - set(BUILD_FAISS_WITH_MKL OFF) + set(BUILD_FAISS_WITH_MKL ON) endif() if(EXISTS "/proc/cpuinfo")