milvus-io
/
milvus
mirror of https://github.com/milvus-io/milvus.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Config file validation 

Former-commit-id: 1590db61ab823d9b4b142e2d4476d2ff8bd8c2ed
pull/191/head
zhiru 2019-09-10 10:59:20 +08:00
parent b42dbcaa7d
commit 55f27dee7c
4 changed files with 73 additions and 56 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
cpp/CMakeLists.txt
View File

@ -31,7 +31,6 @@ if(CMAKE_BUILD_TYPE STREQUAL "Release")
set(BUILD_TYPE "release")
else()
set(BUILD_TYPE "debug")
SET(CMAKE_VERBOSE_MAKEFILE on)
endif()
message(STATUS "Build type = ${BUILD_TYPE}")

86
cpp/src/server/ServerConfig.cpp
View File

@ -15,23 +15,24 @@
#include "utils/CommonUtil.h"
#include "utils/ValidationUtil.h"
namespace zilliz {
namespace milvus {
namespace server {
constexpr uint64_t MB = 1024*1024;
constexpr uint64_t GB = MB*1024;
constexpr uint64_t MB = 1024 * 1024;
constexpr uint64_t GB = MB * 1024;
ServerConfig&
ServerConfig &
ServerConfig::GetInstance() {
static ServerConfig config;
return config;
}
ErrorCode
ServerConfig::LoadConfigFile(const std::string& config_filename) {
ServerConfig::LoadConfigFile(const std::string &config_filename) {
std::string filename = config_filename;
if(filename.empty()){
if (filename.empty()) {
std::cout << "ERROR: a config file is required" << std::endl;
exit(1);//directly exit program if config file not specified
}
@ -43,14 +44,14 @@ ServerConfig::LoadConfigFile(const std::string& config_filename) {
}
try {
ConfigMgr* mgr = const_cast<ConfigMgr*>(ConfigMgr::GetInstance());
ConfigMgr *mgr = const_cast<ConfigMgr *>(ConfigMgr::GetInstance());
ErrorCode err = mgr->LoadConfigFile(filename);
if(err != 0) {
if (err != 0) {
std::cout << "Server failed to load config file" << std::endl;
exit(1);//directly exit program if the config file is illegal
}
}
catch (YAML::Exception& e) {
catch (YAML::Exception &e) {
std::cout << "Server failed to load config file: " << std::endl;
return SERVER_UNEXPECTED_ERROR;
}
@ -120,7 +121,7 @@ ServerConfig::CheckServerConfig() {
}
else {
int32_t gpu_index = std::stol(gpu_index_str);
if(ValidationUtil::ValidateGpuIndex(gpu_index) != SERVER_SUCCESS) {
if (ValidationUtil::ValidateGpuIndex(gpu_index) != SERVER_SUCCESS) {
std::cerr << "Error: invalid gpu_index " << gpu_index_str << std::endl;
okay = false;
}
@ -192,11 +193,11 @@ ServerConfig::CheckDBConfig() {
okay = false;
}
else {
uint64_t insert_buffer_size = (uint64_t)std::stol(insert_buffer_size_str);
uint64_t insert_buffer_size = (uint64_t) std::stol(insert_buffer_size_str);
insert_buffer_size *= GB;
unsigned long total_mem = 0, free_mem = 0;
CommonUtil::GetSystemMemInfo(total_mem, free_mem);
if(insert_buffer_size >= total_mem) {
if (insert_buffer_size >= total_mem) {
std::cerr << "Error: insert_buffer_size exceed system memory" << std::endl;
okay = false;
}
@ -256,7 +257,7 @@ ServerConfig::CheckCacheConfig() {
okay = false;
}
else {
uint64_t cpu_cache_capacity = (uint64_t)std::stol(cpu_cache_capacity_str);
uint64_t cpu_cache_capacity = (uint64_t) std::stol(cpu_cache_capacity_str);
cpu_cache_capacity *= GB;
unsigned long total_mem = 0, free_mem = 0;
CommonUtil::GetSystemMemInfo(total_mem, free_mem);
@ -264,11 +265,11 @@ ServerConfig::CheckCacheConfig() {
std::cerr << "Error: cpu_cache_capacity exceed system memory" << std::endl;
okay = false;
}
else if(cpu_cache_capacity > (double)total_mem*0.9) {
else if (cpu_cache_capacity > (double) total_mem * 0.9) {
std::cerr << "Warning: cpu_cache_capacity value is too aggressive" << std::endl;
}
uint64_t insert_buffer_size = (uint64_t)GetConfig(CONFIG_DB).GetInt32Value(CONFIG_DB_INSERT_BUFFER_SIZE, 4);
uint64_t insert_buffer_size = (uint64_t) GetConfig(CONFIG_DB).GetInt32Value(CONFIG_DB_INSERT_BUFFER_SIZE, 4);
insert_buffer_size *= GB;
if (insert_buffer_size + cpu_cache_capacity >= total_mem) {
std::cerr << "Error: sum of cpu_cache_capacity and insert_buffer_size exceed system memory" << std::endl;
@ -299,7 +300,7 @@ ServerConfig::CheckCacheConfig() {
okay = false;
}
else {
uint64_t gpu_cache_capacity = (uint64_t)std::stol(gpu_cache_capacity_str);
uint64_t gpu_cache_capacity = (uint64_t) std::stol(gpu_cache_capacity_str);
gpu_cache_capacity *= GB;
int gpu_index = GetConfig(CONFIG_SERVER).GetInt32Value(CONFIG_GPU_INDEX, 0);
size_t gpu_memory;
@ -312,7 +313,7 @@ ServerConfig::CheckCacheConfig() {
<< " exceed total gpu memory " << gpu_memory << std::endl;
okay = false;
}
else if(gpu_cache_capacity > (double)gpu_memory*0.9) {
else if (gpu_cache_capacity > (double) gpu_memory * 0.9) {
std::cerr << "Warning: gpu_cache_capacity value is too aggressive" << std::endl;
}
}
@ -336,7 +337,7 @@ ServerConfig::CheckCacheConfig() {
okay = false;
}
else if (ValidationUtil::ValidateGpuIndex(std::stol(gpu_id)) != SERVER_SUCCESS) {
std::cerr << "Error: gpu_id " << gpu_id << " is valid" << std::endl;
std::cerr << "Error: gpu_id " << gpu_id << " is invalid" << std::endl;
okay = false;
}
}
@ -368,8 +369,9 @@ ServerConfig::CheckEngineConfig() {
else {
int32_t omp_thread = std::stol(omp_thread_num_str);
uint32_t sys_thread_cnt = 8;
if(omp_thread > CommonUtil::GetSystemAvailableThreads(sys_thread_cnt)) {
std::cerr << "Error: omp_thread_num " << omp_thread_num_str << " > system available thread " << sys_thread_cnt << std::endl;
if (omp_thread > CommonUtil::GetSystemAvailableThreads(sys_thread_cnt)) {
std::cerr << "Error: omp_thread_num " << omp_thread_num_str << " > system available thread "
<< sys_thread_cnt << std::endl;
okay = false;
}
}
@ -446,9 +448,9 @@ ServerConfig::CheckResourceConfig() {
resource_list.emplace(resource.first);
auto &resource_conf = resource.second;
auto type = resource_conf.GetValue(CONFIG_RESOURCE_TYPE);
std::string device_id_str = resource_conf.GetValue(CONFIG_RESOURCE_DEVICE_ID, "0");
int32_t device_id;
int32_t device_id = -1;
if (ValidationUtil::ValidateStringIsNumber(device_id_str) != SERVER_SUCCESS) {
std::cerr << "Error: device_id " << device_id_str << " is not a number" << std::endl;
okay = false;
@ -472,9 +474,9 @@ ServerConfig::CheckResourceConfig() {
hasExecutor = true;
}
}
else if(type == "GPU") {
else if (type == "GPU") {
int build_index_gpu_index = GetConfig(CONFIG_SERVER).GetInt32Value(CONFIG_GPU_INDEX, 0);
if(device_id == build_index_gpu_index) {
if (device_id == build_index_gpu_index) {
resource_valid_flag = true;
}
if (resource_conf.GetBoolValue(CONFIG_RESOURCE_ENABLE_EXECUTOR, false)) {
@ -485,20 +487,38 @@ ServerConfig::CheckResourceConfig() {
std::cerr << "Error: gpu_resource_num " << gpu_resource_num_str << " is not a number" << std::endl;
okay = false;
}
bool mem_valid = true;
std::string pinned_memory_str = resource_conf.GetValue(CONFIG_RESOURCE_PIN_MEMORY, "300");
if (ValidationUtil::ValidateStringIsNumber(pinned_memory_str) != SERVER_SUCCESS) {
std::cerr << "Error: pinned_memory " << pinned_memory_str << " is not a number" << std::endl;
okay = false;
mem_valid = false;
}
std::string temp_memory_str = resource_conf.GetValue(CONFIG_RESOURCE_TEMP_MEMORY, "300");
if (ValidationUtil::ValidateStringIsNumber(temp_memory_str) != SERVER_SUCCESS) {
std::cerr << "Error: temp_memory " << temp_memory_str << " is not a number" << std::endl;
okay = false;
mem_valid = false;
}
if (mem_valid) {
size_t gpu_memory;
if (ValidationUtil::GetGpuMemory(device_id, gpu_memory) != SERVER_SUCCESS) {
std::cerr << "Error: could not get gpu memory for device " << device_id << std::endl;
okay = false;
}
else {
size_t prealoc_mem = std::stol(pinned_memory_str) + std::stol(temp_memory_str);
if (prealoc_mem >= gpu_memory) {
std::cerr << "Error: sum of pinned_memory and temp_memory " << prealoc_mem
<< " exceeds total gpu memory " << gpu_memory << " for device " << device_id << std::endl;
okay = false;
}
}
}
}
}
if(!resource_valid_flag) {
if (!resource_valid_flag) {
std::cerr << "Building index GPU can't be found in resource config." << std::endl;
okay = false;
}
@ -510,7 +530,7 @@ ServerConfig::CheckResourceConfig() {
std::cerr << "No CPU or GPU resource has executor enabled" << std::endl;
okay = false;
}
auto connections = resource_config.GetChild(CONFIG_RESOURCE_CONNECTIONS).GetChildren();
for (auto &connection : connections) {
auto &connection_conf = connection.second;
@ -547,7 +567,7 @@ ServerConfig::CheckResourceConfig() {
void
ServerConfig::PrintAll() const {
if(const ConfigMgr* mgr = ConfigMgr::GetInstance()) {
if (const ConfigMgr *mgr = ConfigMgr::GetInstance()) {
std::string str = mgr->DumpString();
// SERVER_LOG_INFO << "\n" << str;
std::cout << "\n" << str << std::endl;
@ -555,16 +575,16 @@ ServerConfig::PrintAll() const {
}
ConfigNode
ServerConfig::GetConfig(const std::string& name) const {
const ConfigMgr* mgr = ConfigMgr::GetInstance();
const ConfigNode& root_node = mgr->GetRootNode();
ServerConfig::GetConfig(const std::string &name) const {
const ConfigMgr *mgr = ConfigMgr::GetInstance();
const ConfigNode &root_node = mgr->GetRootNode();
return root_node.GetChild(name);
}
ConfigNode&
ServerConfig::GetConfig(const std::string& name) {
ConfigMgr* mgr = ConfigMgr::GetInstance();
ConfigNode& root_node = mgr->GetRootNode();
ConfigNode &
ServerConfig::GetConfig(const std::string &name) {
ConfigMgr *mgr = ConfigMgr::GetInstance();
ConfigNode &root_node = mgr->GetRootNode();
return root_node.GetChild(name);
}

2
cpp/src/server/ServerConfig.h
View File

@ -52,9 +52,7 @@ static const char* CONFIG_OMP_THREAD_NUM = "omp_thread_num";
static const char* CONFIG_RESOURCE = "resource_config";
static const char* CONFIG_RESOURCES = "resources";
static const char* CONFIG_RESOURCE_TYPE = "type";
static const char* CONFIG_RESOURCE_MEMORY = "memory";
static const char* CONFIG_RESOURCE_DEVICE_ID = "device_id";
static const char* CONFIG_RESOURCE_ENABLE_LOADER = "enable_loader";
static const char* CONFIG_RESOURCE_ENABLE_EXECUTOR = "enable_executor";
static const char* CONFIG_RESOURCE_NUM = "gpu_resource_num";
static const char* CONFIG_RESOURCE_PIN_MEMORY = "pinned_memory";

40
cpp/src/utils/ValidationUtil.cpp
View File

@ -9,6 +9,7 @@
#include <regex>
#include <algorithm>
namespace zilliz {
namespace milvus {
namespace server {
@ -56,15 +57,16 @@ ValidationUtil::ValidateTableDimension(int64_t dimension) {
if (dimension <= 0 || dimension > TABLE_DIMENSION_LIMIT) {
SERVER_LOG_ERROR << "Table dimension excceed the limitation: " << TABLE_DIMENSION_LIMIT;
return SERVER_INVALID_VECTOR_DIMENSION;
} else {
}
else {
return SERVER_SUCCESS;
}
}
ErrorCode
ValidationUtil::ValidateTableIndexType(int32_t index_type) {
int engine_type = (int)engine::EngineType(index_type);
if(engine_type <= 0 || engine_type > (int)engine::EngineType::MAX_VALUE) {
int engine_type = (int) engine::EngineType(index_type);
if (engine_type <= 0 || engine_type > (int) engine::EngineType::MAX_VALUE) {
return SERVER_INVALID_INDEX_TYPE;
}
@ -73,7 +75,7 @@ ValidationUtil::ValidateTableIndexType(int32_t index_type) {
ErrorCode
ValidationUtil::ValidateTableIndexNlist(int32_t nlist) {
if(nlist <= 0) {
if (nlist <= 0) {
return SERVER_INVALID_INDEX_NLIST;
}
@ -82,7 +84,7 @@ ValidationUtil::ValidateTableIndexNlist(int32_t nlist) {
ErrorCode
ValidationUtil::ValidateTableIndexFileSize(int64_t index_file_size) {
if(index_file_size <= 0 || index_file_size > INDEX_FILE_SIZE_LIMIT) {
if (index_file_size <= 0 || index_file_size > INDEX_FILE_SIZE_LIMIT) {
return SERVER_INVALID_INDEX_FILE_SIZE;
}
@ -91,14 +93,14 @@ ValidationUtil::ValidateTableIndexFileSize(int64_t index_file_size) {
ErrorCode
ValidationUtil::ValidateTableIndexMetricType(int32_t metric_type) {
if(metric_type != (int32_t)engine::MetricType::L2 && metric_type != (int32_t)engine::MetricType::IP) {
if (metric_type != (int32_t) engine::MetricType::L2 && metric_type != (int32_t) engine::MetricType::IP) {
return SERVER_INVALID_INDEX_METRIC_TYPE;
}
return SERVER_SUCCESS;
}
ErrorCode
ValidationUtil::ValidateSearchTopk(int64_t top_k, const engine::meta::TableSchema& table_schema) {
ValidationUtil::ValidateSearchTopk(int64_t top_k, const engine::meta::TableSchema &table_schema) {
if (top_k <= 0 || top_k > 2048) {
return SERVER_INVALID_TOPK;
}
@ -107,7 +109,7 @@ ValidationUtil::ValidateSearchTopk(int64_t top_k, const engine::meta::TableSchem
}
ErrorCode
ValidationUtil::ValidateSearchNprobe(int64_t nprobe, const engine::meta::TableSchema& table_schema) {
ValidationUtil::ValidateSearchNprobe(int64_t nprobe, const engine::meta::TableSchema &table_schema) {
if (nprobe <= 0 || nprobe > table_schema.nlist_) {
return SERVER_INVALID_NPROBE;
}
@ -124,7 +126,7 @@ ValidationUtil::ValidateGpuIndex(uint32_t gpu_index) {
return SERVER_UNEXPECTED_ERROR;
}
if(gpu_index >= num_devices) {
if (gpu_index >= num_devices) {
return SERVER_INVALID_ARGUMENT;
}
@ -132,7 +134,7 @@ ValidationUtil::ValidateGpuIndex(uint32_t gpu_index) {
}
ErrorCode
ValidationUtil::GetGpuMemory(uint32_t gpu_index, size_t& memory) {
ValidationUtil::GetGpuMemory(uint32_t gpu_index, size_t &memory) {
cudaDeviceProp deviceProp;
auto cuda_err = cudaGetDeviceProperties(&deviceProp, gpu_index);
if (cuda_err) {
@ -145,20 +147,17 @@ ValidationUtil::GetGpuMemory(uint32_t gpu_index, size_t& memory) {
}
ErrorCode
ValidationUtil::ValidateIpAddress(const std::string &ip_address) {
ValidationUtil::ValidateIpAddress(const std::string &ip_address) {
struct in_addr address;
int result = inet_pton(AF_INET, ip_address.c_str(), &address);
switch(result) {
case 1:
return SERVER_SUCCESS;
case 0:
SERVER_LOG_ERROR << "Invalid IP address: " << ip_address;
switch (result) {
case 1:return SERVER_SUCCESS;
case 0:SERVER_LOG_ERROR << "Invalid IP address: " << ip_address;
return SERVER_INVALID_ARGUMENT;
default:
SERVER_LOG_ERROR << "inet_pton conversion error";
default:SERVER_LOG_ERROR << "inet_pton conversion error";
return SERVER_UNEXPECTED_ERROR;
}
}
@ -188,7 +187,7 @@ ValidationUtil::ValidateStringIsBool(std::string &str) {
ErrorCode
ValidationUtil::ValidateStringIsDouble(const std::string &str, double &val) {
char* end = nullptr;
char *end = nullptr;
val = std::strtod(str.c_str(), &end);
if (end != str.c_str() && *end == '\0' && val != HUGE_VAL) {
return SERVER_SUCCESS;
@ -240,7 +239,8 @@ ValidationUtil::ValidateDbURI(const std::string &uri) {
okay = false;
}
}
} else {
}
else {
SERVER_LOG_ERROR << "Wrong URI format: URI = " << uri;
okay = false;
}