milvus-io
/
milvus
mirror of https://github.com/milvus-io/milvus.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

#90 The server start error messages could be improved to enhance user experience 

Former-commit-id: e3ad89ab67f2c46bc4cb4e6d094a7763fb098664
pull/191/head
starlord 2019-10-29 15:16:04 +08:00
parent e743adb373
commit a78e928dfa
3 changed files with 91 additions and 80 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
core/conf/server_config.template
View File

@ -4,7 +4,7 @@ server_config:
address: 0.0.0.0 # milvus server ip address (IPv4)
port: 19530 # port range: 1025 ~ 65534
deploy_mode: single # deployment type: single, cluster_readonly, cluster_writable
time_zone: UTC+8
time_zone: UTC+8 # time zone, must be in format: UTC+X
db_config:
primary_path: @MILVUS_DB_PATH@ # path used to store data and meta
@ -14,30 +14,30 @@ db_config:
# Keep 'dialect://:@:/', and replace other texts with real values
# Replace 'dialect' with 'mysql' or 'sqlite'
insert_buffer_size: 4 # GB, maximum insert buffer size allowed
insert_buffer_size: 4 # GB, maximum insert buffer size allowed, must be a positive integer
# sum of insert_buffer_size and cpu_cache_capacity cannot exceed total memory
preload_table: # preload data at startup, '*' means load all tables, empty value means no preload
# you can specify preload tables like this: table1,table2,table3
metric_config:
enable_monitor: false # enable monitoring or not
enable_monitor: false # enable monitoring or not, must be a boolean
collector: prometheus # prometheus
prometheus_config:
port: 8080 # port prometheus uses to fetch metrics
port: 8080 # port prometheus uses to fetch metrics, range: 1025 ~ 65534
cache_config:
cpu_cache_capacity: 16 # GB, CPU memory used for cache
cpu_cache_threshold: 0.85 # percentage of data that will be kept when cache cleanup is triggered
gpu_cache_capacity: 4 # GB, GPU memory used for cache
gpu_cache_threshold: 0.85 # percentage of data that will be kept when cache cleanup is triggered
cache_insert_data: false # whether to load inserted data into cache
cpu_cache_capacity: 16 # GB, CPU memory used for cache, must be a positive integer
cpu_cache_threshold: 0.85 # percentage of data that will be kept when cache cleanup is triggered, must be in range (0.0, 1.0]
gpu_cache_capacity: 4 # GB, GPU memory used for cache, must be a positive integer
gpu_cache_threshold: 0.85 # percentage of data that will be kept when cache cleanup is triggered, must be in range (0.0, 1.0]
cache_insert_data: false # whether to load inserted data into cache, must be a boolean
engine_config:
use_blas_threshold: 20 # if nq < use_blas_threshold, use SSE, faster with fluctuated response times
# if nq >= use_blas_threshold, use OpenBlas, slower with stable response times
resource_config:
search_resources: # define the GPUs used for search computation, valid value: gpux
search_resources: # define the GPUs used for search computation, must be in format: gpux
- gpu0
index_build_device: gpu0 # GPU used for building index

145
core/src/server/Config.cpp
View File

@ -363,8 +363,8 @@ Config::PrintAll() {
Status
Config::CheckServerConfigAddress(const std::string& value) {
if (!ValidationUtil::ValidateIpAddress(value).ok()) {
std::string msg = "Invalid server IP address: " + value
+ ". Possible reason: server_config.address is invalid in server_config.yaml.";
std::string msg = "Invalid server IP address: " + value +
". Possible reason: server_config.address is invalid in server_config.yaml.";
return Status(SERVER_INVALID_ARGUMENT, msg);
}
return Status::OK();
@ -373,14 +373,14 @@ Config::CheckServerConfigAddress(const std::string& value) {
Status
Config::CheckServerConfigPort(const std::string& value) {
if (!ValidationUtil::ValidateStringIsNumber(value).ok()) {
std::string msg = "Port " + value + " is not a number. "
+ "Possible reason: server_config.port in server_config.yaml is invalid.";
std::string msg = "Port " + value + " is not a number. " +
"Possible reason: server_config.port in server_config.yaml is invalid.";
return Status(SERVER_INVALID_ARGUMENT, msg);
} else {
int32_t port = std::stoi(value);
if (!(port > 1024 && port < 65535)) {
std::string msg = "Port " + value + " is not in range [1025, 65534]. "
+ "Possible reason: server_config.port in server_config.yaml is invalid.";
std::string msg = "Port " + value + " is not in range [1025, 65534]. " +
"Possible reason: server_config.port in server_config.yaml is invalid.";
return Status(SERVER_INVALID_ARGUMENT, msg);
}
}
@ -433,9 +433,9 @@ Status
Config::CheckDBConfigBackendUrl(const std::string& value) {
if (!ValidationUtil::ValidateDbURI(value).ok()) {
std::string msg =
"Invalid db_backend_url: " + value
+ ". Possible reason: db_config.db_backend_url is invalid in server_config.yaml. "
+ "The correct format should be like sqlite://:@:/ or mysql://root:123456@127.0.0.1:3306/milvus.";
"Invalid db_backend_url: " + value +
". Possible reason: db_config.db_backend_url is invalid in server_config.yaml. " +
"The correct format should be like sqlite://:@:/ or mysql://root:123456@127.0.0.1:3306/milvus.";
return Status(SERVER_INVALID_ARGUMENT, "invalid db_backend_url: " + value);
}
return Status::OK();
@ -444,8 +444,8 @@ Config::CheckDBConfigBackendUrl(const std::string& value) {
Status
Config::CheckDBConfigArchiveDiskThreshold(const std::string& value) {
if (!ValidationUtil::ValidateStringIsNumber(value).ok()) {
std::string msg = "Invalid archive disk threshold: " + value
+ "Possible reason: db_config.archive_disk_threshold in server_config.yaml is invalid.";
std::string msg = "Invalid archive disk threshold: " + value +
"Possible reason: db_config.archive_disk_threshold in server_config.yaml is invalid.";
return Status(SERVER_INVALID_ARGUMENT, msg);
}
return Status::OK();
@ -454,8 +454,8 @@ Config::CheckDBConfigArchiveDiskThreshold(const std::string& value) {
Status
Config::CheckDBConfigArchiveDaysThreshold(const std::string& value) {
if (!ValidationUtil::ValidateStringIsNumber(value).ok()) {
std::string msg = "Invalid archive days threshold: " + value
+ "Possible reason: db_config.archive_disk_threshold in server_config.yaml is invalid.";
std::string msg = "Invalid archive days threshold: " + value +
"Possible reason: db_config.archive_disk_threshold in server_config.yaml is invalid.";
return Status(SERVER_INVALID_ARGUMENT, msg);
}
return Status::OK();
@ -464,16 +464,24 @@ Config::CheckDBConfigArchiveDaysThreshold(const std::string& value) {
Status
Config::CheckDBConfigInsertBufferSize(const std::string& value) {
if (!ValidationUtil::ValidateStringIsNumber(value).ok()) {
std::string msg = "Invalid insert buffer size: " + value
+ "Possible reason: db_config.insert_buffer_size in server_config.yaml is invalid.";
std::string msg = "Invalid insert buffer size: " + value +
"Possible reason: db_config.insert_buffer_size in server_config.yaml "
"is not a positive integer.";
return Status(SERVER_INVALID_ARGUMENT, msg);
} else {
int64_t buffer_size = std::stoi(value) * GB;
if (buffer_size <= 0) {
std::string msg = "Invalid insert buffer size: " + value +
"Possible reason: db_config.insert_buffer_size in server_config.yaml "
"is not a positive integer.";
return Status(SERVER_INVALID_ARGUMENT, msg);
}
uint64_t total_mem = 0, free_mem = 0;
CommonUtil::GetSystemMemInfo(total_mem, free_mem);
if (buffer_size >= total_mem) {
std::string msg = "Invalid insert buffer size: " + value
+ "Possible reason: insert buffer size exceed system memory.";
std::string msg =
"Invalid insert buffer size: " + value + "Possible reason: insert buffer size exceeds system memory.";
return Status(SERVER_INVALID_ARGUMENT, msg);
}
}
@ -483,8 +491,8 @@ Config::CheckDBConfigInsertBufferSize(const std::string& value) {
Status
Config::CheckMetricConfigEnableMonitor(const std::string& value) {
if (!ValidationUtil::ValidateStringIsBool(value).ok()) {
std::string msg = "Invalid metric config: " + value
+ "Possible reason: metric_config.enable_monitor is invalid.";
std::string msg =
"Invalid metric config: " + value + "Possible reason: metric_config.enable_monitor is not a boolean.";
return Status(SERVER_INVALID_ARGUMENT, msg);
}
return Status::OK();
@ -493,8 +501,7 @@ Config::CheckMetricConfigEnableMonitor(const std::string& value) {
Status
Config::CheckMetricConfigCollector(const std::string& value) {
if (value != "prometheus") {
std::string msg = "Invalid metric config: " + value
+ "Possible reason: metric_config.collector is invalid.";
std::string msg = "Invalid metric config: " + value + "Possible reason: metric_config.collector is invalid.";
return Status(SERVER_INVALID_ARGUMENT, msg);
}
return Status::OK();
@ -503,8 +510,8 @@ Config::CheckMetricConfigCollector(const std::string& value) {
Status
Config::CheckMetricConfigPrometheusPort(const std::string& value) {
if (!ValidationUtil::ValidateStringIsNumber(value).ok()) {
std::string msg = "Invalid metric config: " + value
+ "Possible reason: metric_config.prometheus_config.port is invalid.";
std::string msg = "Invalid metric config: " + value +
"Possible reason: metric_config.prometheus_config.port is not in range [1025, 65534].";
return Status(SERVER_INVALID_ARGUMENT, "Invalid metric config prometheus_port: " + value);
}
return Status::OK();
@ -513,18 +520,24 @@ Config::CheckMetricConfigPrometheusPort(const std::string& value) {
Status
Config::CheckCacheConfigCpuCacheCapacity(const std::string& value) {
if (!ValidationUtil::ValidateStringIsNumber(value).ok()) {
std::string msg = "Invalid cpu cache capacity: " + value
+ "Possible reason: cache_config.cpu_cache_capacity is invalid.";
std::string msg = "Invalid cpu cache capacity: " + value +
"Possible reason: cache_config.cpu_cache_capacity is not a positive integer.";
return Status(SERVER_INVALID_ARGUMENT, msg);
} else {
uint64_t cpu_cache_capacity = std::stoi(value) * GB;
int64_t cpu_cache_capacity = std::stoi(value) * GB;
if (cpu_cache_capacity <= 0) {
std::string msg = "Invalid cpu cache capacity: " + value +
"Possible reason: cache_config.cpu_cache_capacity is not a positive integer.";
return Status(SERVER_INVALID_ARGUMENT, msg);
}
uint64_t total_mem = 0, free_mem = 0;
CommonUtil::GetSystemMemInfo(total_mem, free_mem);
if (cpu_cache_capacity >= total_mem) {
std::string msg = "Invalid cpu cache capacity: " + value
+ "Possible reason: Cache config cpu_cache_capacity exceed system memory.";
if (static_cast<uint64_t>(cpu_cache_capacity) >= total_mem) {
std::string msg = "Invalid cpu cache capacity: " + value +
"Possible reason: Cache config cpu_cache_capacity exceeds system memory.";
return Status(SERVER_INVALID_ARGUMENT, msg);
} else if (cpu_cache_capacity > static_cast<double>(total_mem * 0.9)) {
} else if (static_cast<double>(cpu_cache_capacity) > static_cast<double>(total_mem * 0.9)) {
std::cerr << "WARNING: cpu cache capacity value is too big" << std::endl;
}
@ -536,9 +549,9 @@ Config::CheckCacheConfigCpuCacheCapacity(const std::string& value) {
int64_t insert_buffer_size = buffer_value * GB;
if (insert_buffer_size + cpu_cache_capacity >= total_mem) {
std::string msg = "Invalid cpu cache capacity: " + value
+ "Possible reason: sum of cache_config.cpu_cache_capacity and "
"db_config.insert_buffer_size exceeds system memory.";
std::string msg = "Invalid cpu cache capacity: " + value +
"Possible reason: sum of cache_config.cpu_cache_capacity and "
"db_config.insert_buffer_size exceeds system memory.";
return Status(SERVER_INVALID_ARGUMENT, msg);
}
}
@ -548,14 +561,14 @@ Config::CheckCacheConfigCpuCacheCapacity(const std::string& value) {
Status
Config::CheckCacheConfigCpuCacheThreshold(const std::string& value) {
if (!ValidationUtil::ValidateStringIsFloat(value).ok()) {
std::string msg = "Invalid cpu cache threshold: " + value
+ "Possible reason: cache_config.cpu_cache_threshold is invalid.";
std::string msg = "Invalid cpu cache threshold: " + value +
"Possible reason: cache_config.cpu_cache_threshold is not in range (0.0, 1.0].";
return Status(SERVER_INVALID_ARGUMENT, msg);
} else {
float cpu_cache_threshold = std::stof(value);
if (cpu_cache_threshold <= 0.0 || cpu_cache_threshold >= 1.0) {
std::string msg = "Invalid cpu cache threshold: " + value
+ "Possible reason: cache_config.cpu_cache_threshold is invalid.";
std::string msg = "Invalid cpu cache threshold: " + value +
"Possible reason: cache_config.cpu_cache_threshold is not in range (0.0, 1.0].";
return Status(SERVER_INVALID_ARGUMENT, msg);
}
}
@ -565,8 +578,8 @@ Config::CheckCacheConfigCpuCacheThreshold(const std::string& value) {
Status
Config::CheckCacheConfigGpuCacheCapacity(const std::string& value) {
if (!ValidationUtil::ValidateStringIsNumber(value).ok()) {
std::string msg = "Invalid gpu cache capacity: " + value
+ "Possible reason: cache_config.gpu_cache_capacity is invalid.";
std::string msg = "Invalid gpu cache capacity: " + value +
"Possible reason: cache_config.gpu_cache_capacity is not a positive integer.";
return Status(SERVER_INVALID_ARGUMENT, msg);
} else {
uint64_t gpu_cache_capacity = std::stoi(value) * GB;
@ -581,8 +594,8 @@ Config::CheckCacheConfigGpuCacheCapacity(const std::string& value) {
std::string msg = "Fail to get GPU memory for GPU device: " + std::to_string(gpu_index);
return Status(SERVER_UNEXPECTED_ERROR, msg);
} else if (gpu_cache_capacity >= gpu_memory) {
std::string msg = "Invalid gpu cache capacity: " + value
+ "Possible reason: cache_config.gpu_cache_capacity exceed GPU memory.";
std::string msg = "Invalid gpu cache capacity: " + value +
"Possible reason: cache_config.gpu_cache_capacity exceeds GPU memory.";
return Status(SERVER_INVALID_ARGUMENT, msg);
} else if (gpu_cache_capacity > (double)gpu_memory * 0.9) {
std::cerr << "Warning: gpu cache capacity value is too big" << std::endl;
@ -594,14 +607,14 @@ Config::CheckCacheConfigGpuCacheCapacity(const std::string& value) {
Status
Config::CheckCacheConfigGpuCacheThreshold(const std::string& value) {
if (!ValidationUtil::ValidateStringIsFloat(value).ok()) {
std::string msg = "Invalid gpu cache threshold: " + value
+ "Possible reason: cache_config.gpu_cache_threshold is invalid.";
std::string msg = "Invalid gpu cache threshold: " + value +
"Possible reason: cache_config.gpu_cache_threshold is not in range (0.0, 1.0].";
return Status(SERVER_INVALID_ARGUMENT, msg);
} else {
float gpu_cache_threshold = std::stof(value);
if (gpu_cache_threshold <= 0.0 || gpu_cache_threshold >= 1.0) {
std::string msg = "Invalid gpu cache threshold: " + value
+ "Possible reason: cache_config.gpu_cache_threshold is invalid.";
std::string msg = "Invalid gpu cache threshold: " + value +
"Possible reason: cache_config.gpu_cache_threshold is not in range (0.0, 1.0].";
return Status(SERVER_INVALID_ARGUMENT, msg);
}
}
@ -611,8 +624,8 @@ Config::CheckCacheConfigGpuCacheThreshold(const std::string& value) {
Status
Config::CheckCacheConfigCacheInsertData(const std::string& value) {
if (!ValidationUtil::ValidateStringIsBool(value).ok()) {
std::string msg = "Invalid cache insert option: " + value
+ "Possible reason: cache_config.cache_insert_data is invalid.";
std::string msg = "Invalid cache insert option: " + value +
"Possible reason: cache_config.cache_insert_data is not a boolean.";
return Status(SERVER_INVALID_ARGUMENT, msg);
}
return Status::OK();
@ -621,8 +634,8 @@ Config::CheckCacheConfigCacheInsertData(const std::string& value) {
Status
Config::CheckEngineConfigUseBlasThreshold(const std::string& value) {
if (!ValidationUtil::ValidateStringIsNumber(value).ok()) {
std::string msg = "Invalid blas threshold: " + value
+ "Possible reason: engine_config.use_blas_threshold is invalid.";
std::string msg = "Invalid blas threshold: " + value +
"Possible reason: engine_config.use_blas_threshold is not a positive integer.";
return Status(SERVER_INVALID_ARGUMENT, msg);
}
return Status::OK();
@ -631,8 +644,8 @@ Config::CheckEngineConfigUseBlasThreshold(const std::string& value) {
Status
Config::CheckEngineConfigOmpThreadNum(const std::string& value) {
if (!ValidationUtil::ValidateStringIsNumber(value).ok()) {
std::string msg = "Invalid omp thread number: " + value
+ "Possible reason: engine_config.omp_thread_num is invalid.";
std::string msg = "Invalid omp thread number: " + value +
"Possible reason: engine_config.omp_thread_num is not a positive integer.";
return Status(SERVER_INVALID_ARGUMENT, msg);
}
@ -640,8 +653,8 @@ Config::CheckEngineConfigOmpThreadNum(const std::string& value) {
uint32_t sys_thread_cnt = 8;
CommonUtil::GetSystemAvailableThreads(sys_thread_cnt);
if (omp_thread > static_cast<int32_t>(sys_thread_cnt)) {
std::string msg = "Invalid omp thread number: " + value
+ "Possible reason: engine_config.omp_thread_num is invalid.";
std::string msg = "Invalid omp thread number: " + value +
"Possible reason: engine_config.omp_thread_num exceeds system cpu cores.";
return Status(SERVER_INVALID_ARGUMENT, msg);
}
return Status::OK();
@ -650,8 +663,7 @@ Config::CheckEngineConfigOmpThreadNum(const std::string& value) {
Status
Config::CheckResourceConfigMode(const std::string& value) {
if (value != "simple") {
std::string msg = "Invalid resource mode: " + value
+ "Possible reason: resource_config.mode is invalid.";
std::string msg = "Invalid resource mode: " + value + "Possible reason: resource_config.mode is invalid.";
return Status(SERVER_INVALID_ARGUMENT, msg);
}
return Status::OK();
@ -662,15 +674,15 @@ CheckGpuDevice(const std::string& value) {
const std::regex pat("gpu(\\d+)");
std::cmatch m;
if (!std::regex_match(value.c_str(), m, pat)) {
std::string msg = "Invalid gpu device: " + value
+ "Possible reason: resource_config.search_resources is invalid.";
std::string msg = "Invalid gpu device: " + value +
"Possible reason: resource_config.search_resources does not match your hardware.";
return Status(SERVER_INVALID_ARGUMENT, msg);
}
int32_t gpu_index = std::stoi(value.substr(3));
if (!ValidationUtil::ValidateGpuIndex(gpu_index).ok()) {
std::string msg = "Invalid gpu device: " + value
+ "Possible reason: resource_config.search_resources is invalid.";
std::string msg = "Invalid gpu device: " + value +
"Possible reason: resource_config.search_resources does not match your hardware.";
return Status(SERVER_INVALID_ARGUMENT, msg);
}
return Status::OK();
@ -679,16 +691,17 @@ CheckGpuDevice(const std::string& value) {
Status
Config::CheckResourceConfigSearchResources(const std::vector<std::string>& value) {
if (value.empty()) {
std::string msg = "Invalid search resource. "
"Possible reason: resource_config.search_resources is empty.";
std::string msg =
"Invalid search resource. "
"Possible reason: resource_config.search_resources is empty.";
return Status(SERVER_INVALID_ARGUMENT, msg);
}
for (auto& gpu_device : value) {
if (!CheckGpuDevice(gpu_device).ok()) {
std::string msg = "Invalid search resource: " + gpu_device
+ "Possible reason: resource_config.search_resources is invalid.";
return Status(SERVER_INVALID_ARGUMENT, "Invalid resource config search_resources: " + gpu_device);
std::string msg = "Invalid search resource: " + gpu_device +
"Possible reason: resource_config.search_resources does not match your hardware.";
return Status(SERVER_INVALID_ARGUMENT, msg);
}
}
return Status::OK();
@ -697,8 +710,8 @@ Config::CheckResourceConfigSearchResources(const std::vector<std::string>& value
Status
Config::CheckResourceConfigIndexBuildDevice(const std::string& value) {
if (!CheckGpuDevice(value).ok()) {
std::string msg = "Invalid index build device: " + value
+ "Possible reason: resource_config.index_build_device is invalid.";
std::string msg = "Invalid index build device: " + value +
"Possible reason: resource_config.index_build_device does not match your hardware.";
return Status(SERVER_INVALID_ARGUMENT, msg);
}
return Status::OK();

6
core/src/server/DBWrapper.cpp
View File

@ -82,8 +82,7 @@ DBWrapper::StartService() {
opt.mode_ = engine::DBOptions::MODE::CLUSTER_WRITABLE;
} else {
std::cerr << "Error: server_config.deploy_mode in server_config.yaml is not one of "
<< "single, cluster_readonly, and cluster_writable."
<< std::endl;
<< "single, cluster_readonly, and cluster_writable." << std::endl;
kill(0, SIGUSR1);
}
@ -164,8 +163,7 @@ DBWrapper::StartService() {
db_ = engine::DBFactory::Build(opt);
} catch (std::exception& ex) {
std::cerr << "Error: failed to open database: " << ex.what()
<< ". Possible reason: the meta system does not work."
<< std::endl;
<< ". Possible reason: the meta system does not work." << std::endl;
kill(0, SIGUSR1);
}