milvus-io
/
milvus
mirror of https://github.com/milvus-io/milvus.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Config file validation 

Former-commit-id: 1590db61ab823d9b4b142e2d4476d2ff8bd8c2ed
pull/191/head
zhiru 2019-09-10 10:59:20 +08:00
parent b42dbcaa7d
commit 55f27dee7c
4 changed files with 73 additions and 56 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
cpp/CMakeLists.txt
View File

@ -31,7 +31,6 @@ if(CMAKE_BUILD_TYPE STREQUAL "Release")
set(BUILD_TYPE "release") set(BUILD_TYPE "release")
else() else()
set(BUILD_TYPE "debug") set(BUILD_TYPE "debug")
SET(CMAKE_VERBOSE_MAKEFILE on)
endif() endif()
message(STATUS "Build type = ${BUILD_TYPE}") message(STATUS "Build type = ${BUILD_TYPE}")

82
cpp/src/server/ServerConfig.cpp
View File

@ -15,23 +15,24 @@
#include "utils/CommonUtil.h" #include "utils/CommonUtil.h"
#include "utils/ValidationUtil.h" #include "utils/ValidationUtil.h"
namespace zilliz { namespace zilliz {
namespace milvus { namespace milvus {
namespace server { namespace server {
constexpr uint64_t MB = 1024*1024; constexpr uint64_t MB = 1024 * 1024;
constexpr uint64_t GB = MB*1024; constexpr uint64_t GB = MB * 1024;
ServerConfig& ServerConfig &
ServerConfig::GetInstance() { ServerConfig::GetInstance() {
static ServerConfig config; static ServerConfig config;
return config; return config;
} }
ErrorCode ErrorCode
ServerConfig::LoadConfigFile(const std::string& config_filename) { ServerConfig::LoadConfigFile(const std::string &config_filename) {
std::string filename = config_filename; std::string filename = config_filename;
if(filename.empty()){ if (filename.empty()) {
std::cout << "ERROR: a config file is required" << std::endl; std::cout << "ERROR: a config file is required" << std::endl;
exit(1);//directly exit program if config file not specified exit(1);//directly exit program if config file not specified
} }
@ -43,14 +44,14 @@ ServerConfig::LoadConfigFile(const std::string& config_filename) {
} }
try { try {
ConfigMgr* mgr = const_cast<ConfigMgr*>(ConfigMgr::GetInstance()); ConfigMgr *mgr = const_cast<ConfigMgr *>(ConfigMgr::GetInstance());
ErrorCode err = mgr->LoadConfigFile(filename); ErrorCode err = mgr->LoadConfigFile(filename);
if(err != 0) { if (err != 0) {
std::cout << "Server failed to load config file" << std::endl; std::cout << "Server failed to load config file" << std::endl;
exit(1);//directly exit program if the config file is illegal exit(1);//directly exit program if the config file is illegal
} }
} }
catch (YAML::Exception& e) { catch (YAML::Exception &e) {
std::cout << "Server failed to load config file: " << std::endl; std::cout << "Server failed to load config file: " << std::endl;
return SERVER_UNEXPECTED_ERROR; return SERVER_UNEXPECTED_ERROR;
} }
@ -120,7 +121,7 @@ ServerConfig::CheckServerConfig() {
} }
else { else {
int32_t gpu_index = std::stol(gpu_index_str); int32_t gpu_index = std::stol(gpu_index_str);
if(ValidationUtil::ValidateGpuIndex(gpu_index) != SERVER_SUCCESS) { if (ValidationUtil::ValidateGpuIndex(gpu_index) != SERVER_SUCCESS) {
std::cerr << "Error: invalid gpu_index " << gpu_index_str << std::endl; std::cerr << "Error: invalid gpu_index " << gpu_index_str << std::endl;
okay = false; okay = false;
} }
@ -192,11 +193,11 @@ ServerConfig::CheckDBConfig() {
okay = false; okay = false;
} }
else { else {
uint64_t insert_buffer_size = (uint64_t)std::stol(insert_buffer_size_str); uint64_t insert_buffer_size = (uint64_t) std::stol(insert_buffer_size_str);
insert_buffer_size *= GB; insert_buffer_size *= GB;
unsigned long total_mem = 0, free_mem = 0; unsigned long total_mem = 0, free_mem = 0;
CommonUtil::GetSystemMemInfo(total_mem, free_mem); CommonUtil::GetSystemMemInfo(total_mem, free_mem);
if(insert_buffer_size >= total_mem) { if (insert_buffer_size >= total_mem) {
std::cerr << "Error: insert_buffer_size exceed system memory" << std::endl; std::cerr << "Error: insert_buffer_size exceed system memory" << std::endl;
okay = false; okay = false;
} }
@ -256,7 +257,7 @@ ServerConfig::CheckCacheConfig() {
okay = false; okay = false;
} }
else { else {
uint64_t cpu_cache_capacity = (uint64_t)std::stol(cpu_cache_capacity_str); uint64_t cpu_cache_capacity = (uint64_t) std::stol(cpu_cache_capacity_str);
cpu_cache_capacity *= GB; cpu_cache_capacity *= GB;
unsigned long total_mem = 0, free_mem = 0; unsigned long total_mem = 0, free_mem = 0;
CommonUtil::GetSystemMemInfo(total_mem, free_mem); CommonUtil::GetSystemMemInfo(total_mem, free_mem);
@ -264,11 +265,11 @@ ServerConfig::CheckCacheConfig() {
std::cerr << "Error: cpu_cache_capacity exceed system memory" << std::endl; std::cerr << "Error: cpu_cache_capacity exceed system memory" << std::endl;
okay = false; okay = false;
} }
else if(cpu_cache_capacity > (double)total_mem*0.9) { else if (cpu_cache_capacity > (double) total_mem * 0.9) {
std::cerr << "Warning: cpu_cache_capacity value is too aggressive" << std::endl; std::cerr << "Warning: cpu_cache_capacity value is too aggressive" << std::endl;
} }
uint64_t insert_buffer_size = (uint64_t)GetConfig(CONFIG_DB).GetInt32Value(CONFIG_DB_INSERT_BUFFER_SIZE, 4); uint64_t insert_buffer_size = (uint64_t) GetConfig(CONFIG_DB).GetInt32Value(CONFIG_DB_INSERT_BUFFER_SIZE, 4);
insert_buffer_size *= GB; insert_buffer_size *= GB;
if (insert_buffer_size + cpu_cache_capacity >= total_mem) { if (insert_buffer_size + cpu_cache_capacity >= total_mem) {
std::cerr << "Error: sum of cpu_cache_capacity and insert_buffer_size exceed system memory" << std::endl; std::cerr << "Error: sum of cpu_cache_capacity and insert_buffer_size exceed system memory" << std::endl;
@ -299,7 +300,7 @@ ServerConfig::CheckCacheConfig() {
okay = false; okay = false;
} }
else { else {
uint64_t gpu_cache_capacity = (uint64_t)std::stol(gpu_cache_capacity_str); uint64_t gpu_cache_capacity = (uint64_t) std::stol(gpu_cache_capacity_str);
gpu_cache_capacity *= GB; gpu_cache_capacity *= GB;
int gpu_index = GetConfig(CONFIG_SERVER).GetInt32Value(CONFIG_GPU_INDEX, 0); int gpu_index = GetConfig(CONFIG_SERVER).GetInt32Value(CONFIG_GPU_INDEX, 0);
size_t gpu_memory; size_t gpu_memory;
@ -312,7 +313,7 @@ ServerConfig::CheckCacheConfig() {
<< " exceed total gpu memory " << gpu_memory << std::endl; << " exceed total gpu memory " << gpu_memory << std::endl;
okay = false; okay = false;
} }
else if(gpu_cache_capacity > (double)gpu_memory*0.9) { else if (gpu_cache_capacity > (double) gpu_memory * 0.9) {
std::cerr << "Warning: gpu_cache_capacity value is too aggressive" << std::endl; std::cerr << "Warning: gpu_cache_capacity value is too aggressive" << std::endl;
} }
} }
@ -336,7 +337,7 @@ ServerConfig::CheckCacheConfig() {
okay = false; okay = false;
} }
else if (ValidationUtil::ValidateGpuIndex(std::stol(gpu_id)) != SERVER_SUCCESS) { else if (ValidationUtil::ValidateGpuIndex(std::stol(gpu_id)) != SERVER_SUCCESS) {
std::cerr << "Error: gpu_id " << gpu_id << " is valid" << std::endl; std::cerr << "Error: gpu_id " << gpu_id << " is invalid" << std::endl;
okay = false; okay = false;
} }
} }
@ -368,8 +369,9 @@ ServerConfig::CheckEngineConfig() {
else { else {
int32_t omp_thread = std::stol(omp_thread_num_str); int32_t omp_thread = std::stol(omp_thread_num_str);
uint32_t sys_thread_cnt = 8; uint32_t sys_thread_cnt = 8;
if(omp_thread > CommonUtil::GetSystemAvailableThreads(sys_thread_cnt)) { if (omp_thread > CommonUtil::GetSystemAvailableThreads(sys_thread_cnt)) {
std::cerr << "Error: omp_thread_num " << omp_thread_num_str << " > system available thread " << sys_thread_cnt << std::endl; std::cerr << "Error: omp_thread_num " << omp_thread_num_str << " > system available thread "
<< sys_thread_cnt << std::endl;
okay = false; okay = false;
} }
} }
@ -448,7 +450,7 @@ ServerConfig::CheckResourceConfig() {
auto type = resource_conf.GetValue(CONFIG_RESOURCE_TYPE); auto type = resource_conf.GetValue(CONFIG_RESOURCE_TYPE);
std::string device_id_str = resource_conf.GetValue(CONFIG_RESOURCE_DEVICE_ID, "0"); std::string device_id_str = resource_conf.GetValue(CONFIG_RESOURCE_DEVICE_ID, "0");
int32_t device_id; int32_t device_id = -1;
if (ValidationUtil::ValidateStringIsNumber(device_id_str) != SERVER_SUCCESS) { if (ValidationUtil::ValidateStringIsNumber(device_id_str) != SERVER_SUCCESS) {
std::cerr << "Error: device_id " << device_id_str << " is not a number" << std::endl; std::cerr << "Error: device_id " << device_id_str << " is not a number" << std::endl;
okay = false; okay = false;
@ -472,9 +474,9 @@ ServerConfig::CheckResourceConfig() {
hasExecutor = true; hasExecutor = true;
} }
} }
else if(type == "GPU") { else if (type == "GPU") {
int build_index_gpu_index = GetConfig(CONFIG_SERVER).GetInt32Value(CONFIG_GPU_INDEX, 0); int build_index_gpu_index = GetConfig(CONFIG_SERVER).GetInt32Value(CONFIG_GPU_INDEX, 0);
if(device_id == build_index_gpu_index) { if (device_id == build_index_gpu_index) {
resource_valid_flag = true; resource_valid_flag = true;
} }
if (resource_conf.GetBoolValue(CONFIG_RESOURCE_ENABLE_EXECUTOR, false)) { if (resource_conf.GetBoolValue(CONFIG_RESOURCE_ENABLE_EXECUTOR, false)) {
@ -485,20 +487,38 @@ ServerConfig::CheckResourceConfig() {
std::cerr << "Error: gpu_resource_num " << gpu_resource_num_str << " is not a number" << std::endl; std::cerr << "Error: gpu_resource_num " << gpu_resource_num_str << " is not a number" << std::endl;
okay = false; okay = false;
} }
bool mem_valid = true;
std::string pinned_memory_str = resource_conf.GetValue(CONFIG_RESOURCE_PIN_MEMORY, "300"); std::string pinned_memory_str = resource_conf.GetValue(CONFIG_RESOURCE_PIN_MEMORY, "300");
if (ValidationUtil::ValidateStringIsNumber(pinned_memory_str) != SERVER_SUCCESS) { if (ValidationUtil::ValidateStringIsNumber(pinned_memory_str) != SERVER_SUCCESS) {
std::cerr << "Error: pinned_memory " << pinned_memory_str << " is not a number" << std::endl; std::cerr << "Error: pinned_memory " << pinned_memory_str << " is not a number" << std::endl;
okay = false; okay = false;
mem_valid = false;
} }
std::string temp_memory_str = resource_conf.GetValue(CONFIG_RESOURCE_TEMP_MEMORY, "300"); std::string temp_memory_str = resource_conf.GetValue(CONFIG_RESOURCE_TEMP_MEMORY, "300");
if (ValidationUtil::ValidateStringIsNumber(temp_memory_str) != SERVER_SUCCESS) { if (ValidationUtil::ValidateStringIsNumber(temp_memory_str) != SERVER_SUCCESS) {
std::cerr << "Error: temp_memory " << temp_memory_str << " is not a number" << std::endl; std::cerr << "Error: temp_memory " << temp_memory_str << " is not a number" << std::endl;
okay = false; okay = false;
mem_valid = false;
}
if (mem_valid) {
size_t gpu_memory;
if (ValidationUtil::GetGpuMemory(device_id, gpu_memory) != SERVER_SUCCESS) {
std::cerr << "Error: could not get gpu memory for device " << device_id << std::endl;
okay = false;
}
else {
size_t prealoc_mem = std::stol(pinned_memory_str) + std::stol(temp_memory_str);
if (prealoc_mem >= gpu_memory) {
std::cerr << "Error: sum of pinned_memory and temp_memory " << prealoc_mem
<< " exceeds total gpu memory " << gpu_memory << " for device " << device_id << std::endl;
okay = false;
}
}
} }
} }
} }
if(!resource_valid_flag) { if (!resource_valid_flag) {
std::cerr << "Building index GPU can't be found in resource config." << std::endl; std::cerr << "Building index GPU can't be found in resource config." << std::endl;
okay = false; okay = false;
} }
@ -547,7 +567,7 @@ ServerConfig::CheckResourceConfig() {
void void
ServerConfig::PrintAll() const { ServerConfig::PrintAll() const {
if(const ConfigMgr* mgr = ConfigMgr::GetInstance()) { if (const ConfigMgr *mgr = ConfigMgr::GetInstance()) {
std::string str = mgr->DumpString(); std::string str = mgr->DumpString();
// SERVER_LOG_INFO << "\n" << str; // SERVER_LOG_INFO << "\n" << str;
std::cout << "\n" << str << std::endl; std::cout << "\n" << str << std::endl;
@ -555,16 +575,16 @@ ServerConfig::PrintAll() const {
} }
ConfigNode ConfigNode
ServerConfig::GetConfig(const std::string& name) const { ServerConfig::GetConfig(const std::string &name) const {
const ConfigMgr* mgr = ConfigMgr::GetInstance(); const ConfigMgr *mgr = ConfigMgr::GetInstance();
const ConfigNode& root_node = mgr->GetRootNode(); const ConfigNode &root_node = mgr->GetRootNode();
return root_node.GetChild(name); return root_node.GetChild(name);
} }
ConfigNode& ConfigNode &
ServerConfig::GetConfig(const std::string& name) { ServerConfig::GetConfig(const std::string &name) {
ConfigMgr* mgr = ConfigMgr::GetInstance(); ConfigMgr *mgr = ConfigMgr::GetInstance();
ConfigNode& root_node = mgr->GetRootNode(); ConfigNode &root_node = mgr->GetRootNode();
return root_node.GetChild(name); return root_node.GetChild(name);
} }

2
cpp/src/server/ServerConfig.h
View File

@ -52,9 +52,7 @@ static const char* CONFIG_OMP_THREAD_NUM = "omp_thread_num";
static const char* CONFIG_RESOURCE = "resource_config"; static const char* CONFIG_RESOURCE = "resource_config";
static const char* CONFIG_RESOURCES = "resources"; static const char* CONFIG_RESOURCES = "resources";
static const char* CONFIG_RESOURCE_TYPE = "type"; static const char* CONFIG_RESOURCE_TYPE = "type";
static const char* CONFIG_RESOURCE_MEMORY = "memory";
static const char* CONFIG_RESOURCE_DEVICE_ID = "device_id"; static const char* CONFIG_RESOURCE_DEVICE_ID = "device_id";
static const char* CONFIG_RESOURCE_ENABLE_LOADER = "enable_loader";
static const char* CONFIG_RESOURCE_ENABLE_EXECUTOR = "enable_executor"; static const char* CONFIG_RESOURCE_ENABLE_EXECUTOR = "enable_executor";
static const char* CONFIG_RESOURCE_NUM = "gpu_resource_num"; static const char* CONFIG_RESOURCE_NUM = "gpu_resource_num";
static const char* CONFIG_RESOURCE_PIN_MEMORY = "pinned_memory"; static const char* CONFIG_RESOURCE_PIN_MEMORY = "pinned_memory";

40
cpp/src/utils/ValidationUtil.cpp
View File

@ -9,6 +9,7 @@
#include <regex> #include <regex>
#include <algorithm> #include <algorithm>
namespace zilliz { namespace zilliz {
namespace milvus { namespace milvus {
namespace server { namespace server {
@ -56,15 +57,16 @@ ValidationUtil::ValidateTableDimension(int64_t dimension) {
if (dimension <= 0 || dimension > TABLE_DIMENSION_LIMIT) { if (dimension <= 0 || dimension > TABLE_DIMENSION_LIMIT) {
SERVER_LOG_ERROR << "Table dimension excceed the limitation: " << TABLE_DIMENSION_LIMIT; SERVER_LOG_ERROR << "Table dimension excceed the limitation: " << TABLE_DIMENSION_LIMIT;
return SERVER_INVALID_VECTOR_DIMENSION; return SERVER_INVALID_VECTOR_DIMENSION;
} else { }
else {
return SERVER_SUCCESS; return SERVER_SUCCESS;
} }
} }
ErrorCode ErrorCode
ValidationUtil::ValidateTableIndexType(int32_t index_type) { ValidationUtil::ValidateTableIndexType(int32_t index_type) {
int engine_type = (int)engine::EngineType(index_type); int engine_type = (int) engine::EngineType(index_type);
if(engine_type <= 0 || engine_type > (int)engine::EngineType::MAX_VALUE) { if (engine_type <= 0 || engine_type > (int) engine::EngineType::MAX_VALUE) {
return SERVER_INVALID_INDEX_TYPE; return SERVER_INVALID_INDEX_TYPE;
} }
@ -73,7 +75,7 @@ ValidationUtil::ValidateTableIndexType(int32_t index_type) {
ErrorCode ErrorCode
ValidationUtil::ValidateTableIndexNlist(int32_t nlist) { ValidationUtil::ValidateTableIndexNlist(int32_t nlist) {
if(nlist <= 0) { if (nlist <= 0) {
return SERVER_INVALID_INDEX_NLIST; return SERVER_INVALID_INDEX_NLIST;
} }
@ -82,7 +84,7 @@ ValidationUtil::ValidateTableIndexNlist(int32_t nlist) {
ErrorCode ErrorCode
ValidationUtil::ValidateTableIndexFileSize(int64_t index_file_size) { ValidationUtil::ValidateTableIndexFileSize(int64_t index_file_size) {
if(index_file_size <= 0 || index_file_size > INDEX_FILE_SIZE_LIMIT) { if (index_file_size <= 0 || index_file_size > INDEX_FILE_SIZE_LIMIT) {
return SERVER_INVALID_INDEX_FILE_SIZE; return SERVER_INVALID_INDEX_FILE_SIZE;
} }
@ -91,14 +93,14 @@ ValidationUtil::ValidateTableIndexFileSize(int64_t index_file_size) {
ErrorCode ErrorCode
ValidationUtil::ValidateTableIndexMetricType(int32_t metric_type) { ValidationUtil::ValidateTableIndexMetricType(int32_t metric_type) {
if(metric_type != (int32_t)engine::MetricType::L2 && metric_type != (int32_t)engine::MetricType::IP) { if (metric_type != (int32_t) engine::MetricType::L2 && metric_type != (int32_t) engine::MetricType::IP) {
return SERVER_INVALID_INDEX_METRIC_TYPE; return SERVER_INVALID_INDEX_METRIC_TYPE;
} }
return SERVER_SUCCESS; return SERVER_SUCCESS;
} }
ErrorCode ErrorCode
ValidationUtil::ValidateSearchTopk(int64_t top_k, const engine::meta::TableSchema& table_schema) { ValidationUtil::ValidateSearchTopk(int64_t top_k, const engine::meta::TableSchema &table_schema) {
if (top_k <= 0 || top_k > 2048) { if (top_k <= 0 || top_k > 2048) {
return SERVER_INVALID_TOPK; return SERVER_INVALID_TOPK;
} }
@ -107,7 +109,7 @@ ValidationUtil::ValidateSearchTopk(int64_t top_k, const engine::meta::TableSchem
} }
ErrorCode ErrorCode
ValidationUtil::ValidateSearchNprobe(int64_t nprobe, const engine::meta::TableSchema& table_schema) { ValidationUtil::ValidateSearchNprobe(int64_t nprobe, const engine::meta::TableSchema &table_schema) {
if (nprobe <= 0 || nprobe > table_schema.nlist_) { if (nprobe <= 0 || nprobe > table_schema.nlist_) {
return SERVER_INVALID_NPROBE; return SERVER_INVALID_NPROBE;
} }
@ -124,7 +126,7 @@ ValidationUtil::ValidateGpuIndex(uint32_t gpu_index) {
return SERVER_UNEXPECTED_ERROR; return SERVER_UNEXPECTED_ERROR;
} }
if(gpu_index >= num_devices) { if (gpu_index >= num_devices) {
return SERVER_INVALID_ARGUMENT; return SERVER_INVALID_ARGUMENT;
} }
@ -132,7 +134,7 @@ ValidationUtil::ValidateGpuIndex(uint32_t gpu_index) {
} }
ErrorCode ErrorCode
ValidationUtil::GetGpuMemory(uint32_t gpu_index, size_t& memory) { ValidationUtil::GetGpuMemory(uint32_t gpu_index, size_t &memory) {
cudaDeviceProp deviceProp; cudaDeviceProp deviceProp;
auto cuda_err = cudaGetDeviceProperties(&deviceProp, gpu_index); auto cuda_err = cudaGetDeviceProperties(&deviceProp, gpu_index);
if (cuda_err) { if (cuda_err) {
@ -145,20 +147,17 @@ ValidationUtil::GetGpuMemory(uint32_t gpu_index, size_t& memory) {
} }
ErrorCode ErrorCode
ValidationUtil::ValidateIpAddress(const std::string &ip_address) { ValidationUtil::ValidateIpAddress(const std::string &ip_address) {
struct in_addr address; struct in_addr address;
int result = inet_pton(AF_INET, ip_address.c_str(), &address); int result = inet_pton(AF_INET, ip_address.c_str(), &address);
switch(result) { switch (result) {
case 1: case 1:return SERVER_SUCCESS;
return SERVER_SUCCESS; case 0:SERVER_LOG_ERROR << "Invalid IP address: " << ip_address;
case 0:
SERVER_LOG_ERROR << "Invalid IP address: " << ip_address;
return SERVER_INVALID_ARGUMENT; return SERVER_INVALID_ARGUMENT;
default: default:SERVER_LOG_ERROR << "inet_pton conversion error";
SERVER_LOG_ERROR << "inet_pton conversion error";
return SERVER_UNEXPECTED_ERROR; return SERVER_UNEXPECTED_ERROR;
} }
} }
@ -188,7 +187,7 @@ ValidationUtil::ValidateStringIsBool(std::string &str) {
ErrorCode ErrorCode
ValidationUtil::ValidateStringIsDouble(const std::string &str, double &val) { ValidationUtil::ValidateStringIsDouble(const std::string &str, double &val) {
char* end = nullptr; char *end = nullptr;
val = std::strtod(str.c_str(), &end); val = std::strtod(str.c_str(), &end);
if (end != str.c_str() && *end == '\0' && val != HUGE_VAL) { if (end != str.c_str() && *end == '\0' && val != HUGE_VAL) {
return SERVER_SUCCESS; return SERVER_SUCCESS;
@ -240,7 +239,8 @@ ValidationUtil::ValidateDbURI(const std::string &uri) {
okay = false; okay = false;
} }
} }
} else { }
else {
SERVER_LOG_ERROR << "Wrong URI format: URI = " << uri; SERVER_LOG_ERROR << "Wrong URI format: URI = " << uri;
okay = false; okay = false;
} }