milvus-io
/
milvus
mirror of https://github.com/milvus-io/milvus.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'branch-0.3.1' into 'branch-0.3.1' 

MS-249 Check machine hardware during initialize

See merge request megasearch/milvus!257

Former-commit-id: 87d28fed9a4d3d09213d114745fddad420734d1b
pull/191/head
jinhai 2019-07-22 11:17:31 +08:00
parent b404ba8a6d eb1d1e393f
commit e09158d26d
17 changed files with 169 additions and 39 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
cpp/CHANGELOG.md
View File

@ -35,7 +35,8 @@ Please mark all change in change log and use the ticket from JIRA.
- MS-248 - Support AddVector/SearchVector profiling
- MS-256 - Add more cache config
- MS-260 - Refine log
- MS-261 - update faiss version to 1.5.3 and add BUILD_FAISS_WITH_MKL as an option
- MS-249 - Check machine hardware during initialize
- MS-261 - Update faiss version to 1.5.3 and add BUILD_FAISS_WITH_MKL as an option
## New Feature
- MS-180 - Add new mem manager

2
cpp/conf/server_config.template
View File

@ -17,7 +17,7 @@ db_config:
archive_disk_threshold: 0 # triger archive action if storage size exceed this value, 0 means no limit, unit: GB
archive_days_threshold: 0 # files older than x days will be archived, 0 means no limit, unit: day
insert_buffer_size: 4 # maximum insert buffer size allowed, default: 4, unit: GB, should be at least 1 GB.
# the sum of insert_buffer_size and cpu_cache_capacity should be less than total memory
# the sum of insert_buffer_size and cpu_cache_capacity should be less than total memory, unit: GB
metric_config:
is_startup: off # if monitoring start: on, off

1
cpp/src/CMakeLists.txt
View File

@ -87,6 +87,7 @@ set(third_party_libs
mysqlpp
${PROFILER_LIB}
${CUDA_TOOLKIT_ROOT_DIR}/lib64/stubs/libnvidia-ml.so
cudart
)
if (MEGASEARCH_WITH_ARROW STREQUAL "ON")

20
cpp/src/server/RequestTask.cpp
View File

@ -162,17 +162,17 @@ ServerError CreateTableTask::OnExecute() {
try {
//step 1: check arguments
ServerError res = SERVER_SUCCESS;
res = ValidateTableName(schema_.table_name);
res = ValidationUtil::ValidateTableName(schema_.table_name);
if(res != SERVER_SUCCESS) {
return SetError(res, "Invalid table name: " + schema_.table_name);
}
res = ValidateTableDimension(schema_.dimension);
res = ValidationUtil::ValidateTableDimension(schema_.dimension);
if(res != SERVER_SUCCESS) {
return SetError(res, "Invalid table dimension: " + std::to_string(schema_.dimension));
}
res = ValidateTableIndexType(schema_.index_type);
res = ValidationUtil::ValidateTableIndexType(schema_.index_type);
if(res != SERVER_SUCCESS) {
return SetError(res, "Invalid index type: " + std::to_string(schema_.index_type));
}
@ -217,7 +217,7 @@ ServerError DescribeTableTask::OnExecute() {
try {
//step 1: check arguments
ServerError res = SERVER_SUCCESS;
res = ValidateTableName(table_name_);
res = ValidationUtil::ValidateTableName(table_name_);
if(res != SERVER_SUCCESS) {
return SetError(res, "Invalid table name: " + table_name_);
}
@ -260,7 +260,7 @@ ServerError BuildIndexTask::OnExecute() {
//step 1: check arguments
ServerError res = SERVER_SUCCESS;
res = ValidateTableName(table_name_);
res = ValidationUtil::ValidateTableName(table_name_);
if(res != SERVER_SUCCESS) {
return SetError(res, "Invalid table name: " + table_name_);
}
@ -303,7 +303,7 @@ ServerError HasTableTask::OnExecute() {
//step 1: check arguments
ServerError res = SERVER_SUCCESS;
res = ValidateTableName(table_name_);
res = ValidationUtil::ValidateTableName(table_name_);
if(res != SERVER_SUCCESS) {
return SetError(res, "Invalid table name: " + table_name_);
}
@ -339,7 +339,7 @@ ServerError DeleteTableTask::OnExecute() {
//step 1: check arguments
ServerError res = SERVER_SUCCESS;
res = ValidateTableName(table_name_);
res = ValidationUtil::ValidateTableName(table_name_);
if(res != SERVER_SUCCESS) {
return SetError(res, "Invalid table name: " + table_name_);
}
@ -420,7 +420,7 @@ ServerError AddVectorTask::OnExecute() {
//step 1: check arguments
ServerError res = SERVER_SUCCESS;
res = ValidateTableName(table_name_);
res = ValidationUtil::ValidateTableName(table_name_);
if(res != SERVER_SUCCESS) {
return SetError(res, "Invalid table name: " + table_name_);
}
@ -508,7 +508,7 @@ ServerError SearchVectorTaskBase::OnExecute() {
//step 1: check arguments
ServerError res = SERVER_SUCCESS;
res = ValidateTableName(table_name_);
res = ValidationUtil::ValidateTableName(table_name_);
if(res != SERVER_SUCCESS) {
return SetError(res, "Invalid table name: " + table_name_);
}
@ -720,7 +720,7 @@ ServerError GetTableRowCountTask::OnExecute() {
//step 1: check arguments
ServerError res = SERVER_SUCCESS;
res = ValidateTableName(table_name_);
res = ValidationUtil::ValidateTableName(table_name_);
if(res != SERVER_SUCCESS) {
return SetError(res, "Invalid table name: " + table_name_);
}

4
cpp/src/server/Server.cpp
View File

@ -226,6 +226,10 @@ Server::Stop() {
ServerError
Server::LoadConfig() {
ServerConfig::GetInstance().LoadConfigFile(config_filename_);
ServerError err = ServerConfig::GetInstance().ValidateConfig();
if(err != SERVER_SUCCESS){
exit(0);
}
return SERVER_SUCCESS;
}

64
cpp/src/server/ServerConfig.cpp
View File

@ -12,11 +12,16 @@
#include <iostream>
#include "config/IConfigMgr.h"
#include "utils/CommonUtil.h"
#include "utils/ValidationUtil.h"
namespace zilliz {
namespace milvus {
namespace server {
constexpr uint64_t MB = 1024*1024;
constexpr uint64_t GB = MB*1024;
ServerConfig&
ServerConfig::GetInstance() {
static ServerConfig config;
@ -53,6 +58,65 @@ ServerConfig::LoadConfigFile(const std::string& config_filename) {
return SERVER_SUCCESS;
}
ServerError ServerConfig::ValidateConfig() const {
//server config validation
ConfigNode server_config = GetConfig(CONFIG_SERVER);
uint32_t gpu_index = (uint32_t)server_config.GetInt32Value(CONFIG_GPU_INDEX);
if(ValidationUtil::ValidateGpuIndex(gpu_index) != SERVER_SUCCESS) {
std::cout << "Error: invalid gpu_index " << std::to_string(gpu_index) << std::endl;
return SERVER_INVALID_ARGUMENT;
}
//db config validation
unsigned long total_mem = 0, free_mem = 0;
CommonUtil::GetSystemMemInfo(total_mem, free_mem);
ConfigNode db_config = GetConfig(CONFIG_DB);
uint64_t insert_buffer_size = (uint64_t)db_config.GetInt32Value(CONFIG_DB_INSERT_BUFFER_SIZE);
insert_buffer_size *= GB;
if(insert_buffer_size >= total_mem) {
std::cout << "Error: insert_buffer_size execeed system memory" << std::endl;
return SERVER_INVALID_ARGUMENT;
}
uint64_t index_building_threshold = (uint64_t)db_config.GetInt32Value(CONFIG_DB_INDEX_TRIGGER_SIZE);
index_building_threshold *= MB;
size_t gpu_mem = 0;
ValidationUtil::GetGpuMemory(gpu_index, gpu_mem);
if(index_building_threshold >= gpu_mem) {
std::cout << "Error: index_building_threshold execeed gpu memory" << std::endl;
return SERVER_INVALID_ARGUMENT;
} else if(index_building_threshold >= gpu_mem/3) {
std::cout << "Warnning: index_building_threshold is greater than 1/3 of gpu memory, "
<< "some index type(such as IVFLAT) may cause cuda::bad_alloc() error" << std::endl;
}
//cache config validation
ConfigNode cache_config = GetConfig(CONFIG_CACHE);
uint64_t cache_cap = (uint64_t)cache_config.GetInt64Value(CONFIG_CPU_CACHE_CAPACITY);
cache_cap *= GB;
if(cache_cap >= total_mem) {
std::cout << "Error: cpu_cache_capacity execeed system memory" << std::endl;
return SERVER_INVALID_ARGUMENT;
} if(cache_cap > (double)total_mem*0.9) {
std::cout << "Warnning: cpu_cache_capacity value is too aggressive" << std::endl;
}
if(insert_buffer_size + cache_cap >= total_mem) {
std::cout << "Error: sum of cpu_cache_capacity and insert_buffer_size execeed system memory" << std::endl;
return SERVER_INVALID_ARGUMENT;
}
double free_percent = cache_config.GetDoubleValue(server::CACHE_FREE_PERCENT);
if(free_percent < std::numeric_limits<double>::epsilon() || free_percent > 1.0) {
std::cout << "Error: invalid cache_free_percent " << std::to_string(free_percent) << std::endl;
return SERVER_INVALID_ARGUMENT;
}
return SERVER_SUCCESS;
}
void
ServerConfig::PrintAll() const {
if(const IConfigMgr* mgr = IConfigMgr::GetInstance()) {

2
cpp/src/server/ServerConfig.h
View File

@ -19,6 +19,7 @@ static const std::string CONFIG_SERVER_ADDRESS = "address";
static const std::string CONFIG_SERVER_PORT = "port";
static const std::string CONFIG_SERVER_PROTOCOL = "transfer_protocol";
static const std::string CONFIG_CLUSTER_MODE = "mode";
static const std::string CONFIG_GPU_INDEX = "gpu_index";
static const std::string CONFIG_DB = "db_config";
static const std::string CONFIG_DB_URL = "db_backend_url";
@ -57,6 +58,7 @@ class ServerConfig {
static ServerConfig &GetInstance();
ServerError LoadConfigFile(const std::string& config_filename);
ServerError ValidateConfig() const;
void PrintAll() const;
ConfigNode GetConfig(const std::string& name) const;

38
cpp/src/utils/ValidationUtil.cpp
View File

@ -1,7 +1,8 @@
#include <src/db/ExecutionEngine.h>
#include "db/ExecutionEngine.h"
#include "ValidationUtil.h"
#include "Log.h"
#include <cuda_runtime.h>
namespace zilliz {
namespace milvus {
@ -11,7 +12,7 @@ constexpr size_t table_name_size_limit = 255;
constexpr int64_t table_dimension_limit = 16384;
ServerError
ValidateTableName(const std::string &table_name) {
ValidationUtil::ValidateTableName(const std::string &table_name) {
// Table name shouldn't be empty.
if (table_name.empty()) {
@ -45,7 +46,7 @@ ValidateTableName(const std::string &table_name) {
}
ServerError
ValidateTableDimension(int64_t dimension) {
ValidationUtil::ValidateTableDimension(int64_t dimension) {
if (dimension <= 0 || dimension > table_dimension_limit) {
SERVER_LOG_ERROR << "Table dimension excceed the limitation: " << table_dimension_limit;
return SERVER_INVALID_VECTOR_DIMENSION;
@ -55,7 +56,7 @@ ValidateTableDimension(int64_t dimension) {
}
ServerError
ValidateTableIndexType(int32_t index_type) {
ValidationUtil::ValidateTableIndexType(int32_t index_type) {
int engine_type = (int)engine::EngineType(index_type);
if(engine_type <= 0 || engine_type > (int)engine::EngineType::MAX_VALUE) {
return SERVER_INVALID_INDEX_TYPE;
@ -64,6 +65,35 @@ ValidateTableIndexType(int32_t index_type) {
return SERVER_SUCCESS;
}
ServerError
ValidationUtil::ValidateGpuIndex(uint32_t gpu_index) {
int num_devices = 0;
auto cuda_err = cudaGetDeviceCount(&num_devices);
if (cuda_err) {
SERVER_LOG_ERROR << "Failed to count video card: " << std::to_string(cuda_err);
return SERVER_UNEXPECTED_ERROR;
}
if(gpu_index >= num_devices) {
return SERVER_INVALID_ARGUMENT;
}
return SERVER_SUCCESS;
}
ServerError
ValidationUtil::GetGpuMemory(uint32_t gpu_index, size_t& memory) {
cudaDeviceProp deviceProp;
auto cuda_err = cudaGetDeviceProperties(&deviceProp, gpu_index);
if (cuda_err) {
SERVER_LOG_ERROR << "Failed to get video card properties: " << std::to_string(cuda_err);
return SERVER_UNEXPECTED_ERROR;
}
memory = deviceProp.totalGlobalMem;
return SERVER_SUCCESS;
}
}
}
}

21
cpp/src/utils/ValidationUtil.h
View File

@ -6,14 +6,23 @@ namespace zilliz {
namespace milvus {
namespace server {
ServerError
ValidateTableName(const std::string& table_name);
class ValidationUtil {
public:
static ServerError
ValidateTableName(const std::string &table_name);
ServerError
ValidateTableDimension(int64_t dimension);
static ServerError
ValidateTableDimension(int64_t dimension);
ServerError
ValidateTableIndexType(int32_t index_type);
static ServerError
ValidateTableIndexType(int32_t index_type);
static ServerError
ValidateGpuIndex(uint32_t gpu_index);
static ServerError
GetGpuMemory(uint32_t gpu_index, size_t &memory);
};
}
}

2
cpp/src/wrapper/IndexBuilder.cpp
View File

@ -37,7 +37,7 @@ class GpuResources {
using namespace zilliz::milvus::server;
ServerConfig &config = ServerConfig::GetInstance();
ConfigNode server_config = config.GetConfig(CONFIG_SERVER);
gpu_num = server_config.GetInt32Value("gpu_index", 0);
gpu_num = server_config.GetInt32Value(server::CONFIG_GPU_INDEX, 0);
}
int32_t GetGpu() {

1
cpp/unittest/CMakeLists.txt
View File

@ -35,6 +35,7 @@ set(unittest_libs
dl
z
${CUDA_TOOLKIT_ROOT_DIR}/lib64/stubs/libnvidia-ml.so
cudart
)
add_subdirectory(server)

4
cpp/unittest/db/CMakeLists.txt
View File

@ -9,6 +9,9 @@ aux_source_directory(${MILVUS_ENGINE_SRC}/cache cache_srcs)
aux_source_directory(${MILVUS_ENGINE_SRC}/wrapper wrapper_src)
aux_source_directory(./ test_srcs)
set(util_files
${MILVUS_ENGINE_SRC}/utils/ValidationUtil.cpp)
aux_source_directory(${MILVUS_ENGINE_SRC}/db/scheduler scheduler_files)
aux_source_directory(${MILVUS_ENGINE_SRC}/db/scheduler/context scheduler_context_files)
aux_source_directory(${MILVUS_ENGINE_SRC}/db/scheduler/task scheduler_task_files)
@ -29,6 +32,7 @@ set(db_test_src
${db_srcs}
${db_scheduler_srcs}
${wrapper_src}
${util_files}
${require_files}
${test_srcs})

4
cpp/unittest/faiss_wrapper/CMakeLists.txt
View File

@ -6,6 +6,9 @@
aux_source_directory(${MILVUS_ENGINE_SRC}/wrapper wrapper_src)
aux_source_directory(${MILVUS_ENGINE_SRC}/config config_files)
set(util_files
${MILVUS_ENGINE_SRC}/utils/ValidationUtil.cpp)
# Make sure that your call to link_directories takes place before your call to the relevant add_executable.
include_directories(/usr/local/cuda/include)
link_directories("/usr/local/cuda/lib64")
@ -14,6 +17,7 @@ set(wrapper_test_src
${unittest_srcs}
${wrapper_src}
${config_files}
${util_files}
${require_files}
wrapper_test.cpp
)

4
cpp/unittest/metrics/CMakeLists.txt
View File

@ -17,6 +17,9 @@ aux_source_directory(../../src/wrapper wrapper_src)
aux_source_directory(../../src/metrics metrics_src)
aux_source_directory(./ test_srcs)
set(util_files
${MILVUS_ENGINE_SRC}/utils/ValidationUtil.cpp)
aux_source_directory(${MILVUS_ENGINE_SRC}/db/scheduler scheduler_files)
aux_source_directory(${MILVUS_ENGINE_SRC}/db/scheduler/context scheduler_context_files)
aux_source_directory(${MILVUS_ENGINE_SRC}/db/scheduler/task scheduler_task_files)
@ -43,6 +46,7 @@ set(count_test_src
${wrapper_src}
${metrics_src}
${test_srcs}
${util_files}
)

1
cpp/unittest/server/CMakeLists.txt
View File

@ -19,6 +19,7 @@ set(utils_srcs
${MILVUS_ENGINE_SRC}/utils/TimeRecorder.cpp
${MILVUS_ENGINE_SRC}/utils/CommonUtil.cpp
${MILVUS_ENGINE_SRC}/utils/LogUtil.cpp
${MILVUS_ENGINE_SRC}/utils/ValidationUtil.cpp
)
cuda_add_executable(server_test

7
cpp/unittest/storage/CMakeLists.txt
View File

@ -5,6 +5,9 @@
#-------------------------------------------------------------------------------
aux_source_directory(${MILVUS_ENGINE_SRC}/storage/s3 s3_client_src)
set(util_files
${MILVUS_ENGINE_SRC}/utils/ValidationUtil.cpp)
# Make sure that your call to link_directories takes place before your call to the relevant add_executable.
include_directories("${CUDA_TOOLKIT_ROOT_DIR}/include")
link_directories("${CUDA_TOOLKIT_ROOT_DIR}/lib64")
@ -19,7 +22,9 @@ set(s3_client_test_src
add_executable(s3_test
${s3_client_test_src}
${config_files})
${config_files}
${util_files}
)
set(s3_client_libs
stdc++

30
cpp/unittest/utils/ValidationUtilTest.cpp
View File

@ -16,48 +16,48 @@ using namespace zilliz::milvus::server;
TEST(ValidationUtilTest, TableNameTest) {
std::string table_name = "Normal123_";
ServerError res = ValidateTableName(table_name);
ServerError res = ValidationUtil::ValidateTableName(table_name);
ASSERT_EQ(res, SERVER_SUCCESS);
table_name = "12sds";
res = ValidateTableName(table_name);
res = ValidationUtil::ValidateTableName(table_name);
ASSERT_EQ(res, SERVER_INVALID_TABLE_NAME);
table_name = "";
res = ValidateTableName(table_name);
res = ValidationUtil::ValidateTableName(table_name);
ASSERT_EQ(res, SERVER_INVALID_TABLE_NAME);
table_name = "_asdasd";
res = ValidateTableName(table_name);
res = ValidationUtil::ValidateTableName(table_name);
ASSERT_EQ(res, SERVER_SUCCESS);
table_name = "!@#!@";
res = ValidateTableName(table_name);
res = ValidationUtil::ValidateTableName(table_name);
ASSERT_EQ(res, SERVER_INVALID_TABLE_NAME);
table_name = "中文";
res = ValidateTableName(table_name);
res = ValidationUtil::ValidateTableName(table_name);
ASSERT_EQ(res, SERVER_INVALID_TABLE_NAME);
table_name = std::string('a', 32768);
res = ValidateTableName(table_name);
res = ValidationUtil::ValidateTableName(table_name);
ASSERT_EQ(res, SERVER_INVALID_TABLE_NAME);
}
TEST(ValidationUtilTest, TableDimensionTest) {
ASSERT_EQ(ValidateTableDimension(-1), SERVER_INVALID_VECTOR_DIMENSION);
ASSERT_EQ(ValidateTableDimension(0), SERVER_INVALID_VECTOR_DIMENSION);
ASSERT_EQ(ValidateTableDimension(16385), SERVER_INVALID_VECTOR_DIMENSION);
ASSERT_EQ(ValidateTableDimension(16384), SERVER_SUCCESS);
ASSERT_EQ(ValidateTableDimension(1), SERVER_SUCCESS);
ASSERT_EQ(ValidationUtil::ValidateTableDimension(-1), SERVER_INVALID_VECTOR_DIMENSION);
ASSERT_EQ(ValidationUtil::ValidateTableDimension(0), SERVER_INVALID_VECTOR_DIMENSION);
ASSERT_EQ(ValidationUtil::ValidateTableDimension(16385), SERVER_INVALID_VECTOR_DIMENSION);
ASSERT_EQ(ValidationUtil::ValidateTableDimension(16384), SERVER_SUCCESS);
ASSERT_EQ(ValidationUtil::ValidateTableDimension(1), SERVER_SUCCESS);
}
TEST(ValidationUtilTest, TableIndexTypeTest) {
ASSERT_EQ(ValidateTableIndexType((int)engine::EngineType::INVALID), SERVER_INVALID_INDEX_TYPE);
ASSERT_EQ(ValidationUtil::ValidateTableIndexType((int)engine::EngineType::INVALID), SERVER_INVALID_INDEX_TYPE);
for(int i = 1; i <= (int)engine::EngineType::MAX_VALUE; i++) {
ASSERT_EQ(ValidateTableIndexType(i), SERVER_SUCCESS);
ASSERT_EQ(ValidationUtil::ValidateTableIndexType(i), SERVER_SUCCESS);
}
ASSERT_EQ(ValidateTableIndexType((int)engine::EngineType::MAX_VALUE + 1), SERVER_INVALID_INDEX_TYPE);
ASSERT_EQ(ValidationUtil::ValidateTableIndexType((int)engine::EngineType::MAX_VALUE + 1), SERVER_INVALID_INDEX_TYPE);
}