milvus-io
/
milvus
mirror of https://github.com/milvus-io/milvus.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

MS-574 save config in map 

Former-commit-id: 961dc6749c1b262e489a825cce81586bf27f2c18
pull/191/head
yudong.cai 2019-09-21 18:14:29 +08:00
parent ebda6cb4ef
commit b0fe84cd81
2 changed files with 204 additions and 71 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

255
cpp/src/server/Config.cpp
View File

@ -22,6 +22,7 @@
#include <unistd.h> #include <unistd.h>
#include <stdlib.h> #include <stdlib.h>
#include <iostream> #include <iostream>
#include <algorithm>
#include "config/ConfigMgr.h" #include "config/ConfigMgr.h"
#include "utils/CommonUtil.h" #include "utils/CommonUtil.h"
@ -32,8 +33,8 @@ namespace zilliz {
namespace milvus { namespace milvus {
namespace server { namespace server {
constexpr uint64_t MB = 1024 * 1024; constexpr uint64_t MB = 1UL << 20;
constexpr uint64_t GB = MB * 1024; constexpr uint64_t GB = 1UL << 30;
Config & Config &
Config::GetInstance() { Config::GetInstance() {
@ -626,164 +627,282 @@ Config::GetConfig(const std::string &name) {
return root_node.GetChild(name); return root_node.GetChild(name);
} }
Status
Config::GetConfigValueInMem(const std::string &parent_key,
const std::string &child_key,
std::string &value) {
if (config_map_.find(parent_key) != config_map_.end() &&
config_map_[parent_key].find(child_key) != config_map_[parent_key].end()) {
std::lock_guard<std::mutex> lock(mutex_);
value = config_map_[parent_key][child_key];
return Status::OK();
} else {
return Status(SERVER_UNEXPECTED_ERROR, "key not exist");
}
}
void
Config::SetConfigValueInMem(const std::string &parent_key,
const std::string &child_key,
std::string &value) {
std::lock_guard<std::mutex> lock(mutex_);
config_map_[parent_key][child_key] = value;
}
////////////////////////////////////////////////////////////////////////////////
/* server config */ /* server config */
std::string std::string
Config::GetServerConfigAddress() { Config::GetServerConfigAddress() {
ConfigNode server_config = GetConfig(CONFIG_SERVER); std::string value;
return server_config.GetValue(CONFIG_SERVER_ADDRESS, if (!GetConfigValueInMem(CONFIG_SERVER, CONFIG_SERVER_ADDRESS, value).ok()) {
CONFIG_SERVER_ADDRESS_DEFAULT); value = GetConfig(CONFIG_SERVER).GetValue(CONFIG_SERVER_ADDRESS,
CONFIG_SERVER_ADDRESS_DEFAULT);
SetConfigValueInMem(CONFIG_SERVER, CONFIG_SERVER_ADDRESS, value);
}
return value;
} }
std::string std::string
Config::GetServerConfigPort() { Config::GetServerConfigPort() {
ConfigNode server_config = GetConfig(CONFIG_SERVER); std::string value;
return server_config.GetValue(CONFIG_SERVER_PORT, if (!GetConfigValueInMem(CONFIG_SERVER, CONFIG_SERVER_PORT, value).ok()) {
CONFIG_SERVER_PORT_DEFAULT); value = GetConfig(CONFIG_SERVER).GetValue(CONFIG_SERVER_PORT,
CONFIG_SERVER_PORT_DEFAULT);
SetConfigValueInMem(CONFIG_SERVER, CONFIG_SERVER_PORT, value);
}
return value;
} }
std::string std::string
Config::GetServerConfigMode() { Config::GetServerConfigMode() {
ConfigNode server_config = GetConfig(CONFIG_SERVER); std::string value;
return server_config.GetValue(CONFIG_SERVER_MODE, if (!GetConfigValueInMem(CONFIG_SERVER, CONFIG_SERVER_MODE, value).ok()) {
CONFIG_SERVER_MODE_DEFAULT); value = GetConfig(CONFIG_SERVER).GetValue(CONFIG_SERVER_MODE,
CONFIG_SERVER_MODE_DEFAULT);
SetConfigValueInMem(CONFIG_SERVER, CONFIG_SERVER_MODE, value);
}
return value;
} }
std::string std::string
Config::GetServerConfigTimeZone() { Config::GetServerConfigTimeZone() {
ConfigNode server_config = GetConfig(CONFIG_SERVER); std::string value;
return server_config.GetValue(CONFIG_SERVER_TIME_ZONE, if (!GetConfigValueInMem(CONFIG_SERVER, CONFIG_SERVER_TIME_ZONE, value).ok()) {
CONFIG_SERVER_TIME_ZONE_DEFAULT); value = GetConfig(CONFIG_SERVER).GetValue(CONFIG_SERVER_TIME_ZONE,
CONFIG_SERVER_TIME_ZONE_DEFAULT);
SetConfigValueInMem(CONFIG_SERVER, CONFIG_SERVER_TIME_ZONE, value);
}
return value;
} }
////////////////////////////////////////////////////////////////////////////////
/* db config */ /* db config */
std::string std::string
Config::GetDBConfigPath() { Config::GetDBConfigPath() {
ConfigNode db_config = GetConfig(CONFIG_DB); std::string value;
return db_config.GetValue(CONFIG_DB_PATH, if (!GetConfigValueInMem(CONFIG_DB, CONFIG_DB_PATH, value).ok()) {
CONFIG_DB_PATH_DEFAULT); value = GetConfig(CONFIG_DB).GetValue(CONFIG_DB_PATH,
CONFIG_DB_PATH_DEFAULT);
SetConfigValueInMem(CONFIG_DB, CONFIG_DB_PATH, value);
}
return value;
} }
std::string std::string
Config::GetDBConfigSlavePath() { Config::GetDBConfigSlavePath() {
ConfigNode db_config = GetConfig(CONFIG_DB); std::string value;
return db_config.GetValue(CONFIG_DB_SLAVE_PATH, if (!GetConfigValueInMem(CONFIG_DB, CONFIG_DB_SLAVE_PATH, value).ok()) {
CONFIG_DB_SLAVE_PATH_DEFAULT); value = GetConfig(CONFIG_DB).GetValue(CONFIG_DB_SLAVE_PATH,
CONFIG_DB_SLAVE_PATH_DEFAULT);
SetConfigValueInMem(CONFIG_DB, CONFIG_DB_SLAVE_PATH, value);
}
return value;
} }
std::string std::string
Config::GetDBConfigBackendUrl() { Config::GetDBConfigBackendUrl() {
ConfigNode db_config = GetConfig(CONFIG_DB); std::string value;
return db_config.GetValue(CONFIG_DB_BACKEND_URL, if (!GetConfigValueInMem(CONFIG_DB, CONFIG_DB_BACKEND_URL, value).ok()) {
CONFIG_DB_BACKEND_URL_DEFAULT); value = GetConfig(CONFIG_DB).GetValue(CONFIG_DB_BACKEND_URL,
CONFIG_DB_BACKEND_URL_DEFAULT);
SetConfigValueInMem(CONFIG_DB, CONFIG_DB_BACKEND_URL, value);
}
return value;
} }
int32_t int32_t
Config::GetDBConfigArchiveDiskThreshold() { Config::GetDBConfigArchiveDiskThreshold() {
ConfigNode db_config = GetConfig(CONFIG_DB); std::string value;
return db_config.GetInt32Value(CONFIG_DB_ARCHIVE_DISK_THRESHOLD, if (!GetConfigValueInMem(CONFIG_DB, CONFIG_DB_ARCHIVE_DISK_THRESHOLD, value).ok()) {
std::stoi(CONFIG_DB_ARCHIVE_DISK_THRESHOLD_DEFAULT)); value = GetConfig(CONFIG_DB).GetValue(CONFIG_DB_ARCHIVE_DISK_THRESHOLD,
CONFIG_DB_ARCHIVE_DISK_THRESHOLD_DEFAULT);
SetConfigValueInMem(CONFIG_DB, CONFIG_DB_ARCHIVE_DISK_THRESHOLD, value);
}
return std::stoi(value);
} }
int32_t int32_t
Config::GetDBConfigArchiveDaysThreshold() { Config::GetDBConfigArchiveDaysThreshold() {
ConfigNode db_config = GetConfig(CONFIG_DB); std::string value;
return db_config.GetInt32Value(CONFIG_DB_ARCHIVE_DAYS_THRESHOLD, if (!GetConfigValueInMem(CONFIG_DB, CONFIG_DB_ARCHIVE_DAYS_THRESHOLD, value).ok()) {
std::stoi(CONFIG_DB_ARCHIVE_DAYS_THRESHOLD_DEFAULT)); value = GetConfig(CONFIG_DB).GetValue(CONFIG_DB_ARCHIVE_DAYS_THRESHOLD,
CONFIG_DB_ARCHIVE_DAYS_THRESHOLD_DEFAULT);
SetConfigValueInMem(CONFIG_DB, CONFIG_DB_ARCHIVE_DAYS_THRESHOLD, value);
}
return std::stoi(value);
} }
int32_t int32_t
Config::GetDBConfigBufferSize() { Config::GetDBConfigBufferSize() {
ConfigNode db_config = GetConfig(CONFIG_DB); std::string value;
return db_config.GetInt32Value(CONFIG_DB_BUFFER_SIZE, if (!GetConfigValueInMem(CONFIG_DB, CONFIG_DB_BUFFER_SIZE, value).ok()) {
std::stoi(CONFIG_DB_BUFFER_SIZE_DEFAULT)); value = GetConfig(CONFIG_DB).GetValue(CONFIG_DB_BUFFER_SIZE,
CONFIG_DB_BUFFER_SIZE_DEFAULT);
SetConfigValueInMem(CONFIG_DB, CONFIG_DB_BUFFER_SIZE, value);
}
return std::stoi(value);
} }
int32_t int32_t
Config::GetDBConfigBuildIndexGPU() { Config::GetDBConfigBuildIndexGPU() {
ConfigNode db_config = GetConfig(CONFIG_DB); std::string value;
return db_config.GetInt32Value(CONFIG_DB_BUILD_INDEX_GPU, if (!GetConfigValueInMem(CONFIG_DB, CONFIG_DB_BUILD_INDEX_GPU, value).ok()) {
std::stoi(CONFIG_DB_BUILD_INDEX_GPU_DEFAULT)); value = GetConfig(CONFIG_DB).GetValue(CONFIG_DB_BUILD_INDEX_GPU,
CONFIG_DB_BUILD_INDEX_GPU_DEFAULT);
SetConfigValueInMem(CONFIG_DB, CONFIG_DB_BUILD_INDEX_GPU, value);
}
return std::stoi(value);
} }
////////////////////////////////////////////////////////////////////////////////
/* metric config */ /* metric config */
bool bool
Config::GetMetricConfigAutoBootup() { Config::GetMetricConfigAutoBootup() {
ConfigNode metric_config = GetConfig(CONFIG_METRIC); std::string value;
return metric_config.GetBoolValue(CONFIG_METRIC_AUTO_BOOTUP, if (!GetConfigValueInMem(CONFIG_METRIC, CONFIG_METRIC_AUTO_BOOTUP, value).ok()) {
std::stoi(CONFIG_METRIC_AUTO_BOOTUP_DEFAULT)); value = GetConfig(CONFIG_METRIC).GetValue(CONFIG_METRIC_AUTO_BOOTUP,
CONFIG_METRIC_AUTO_BOOTUP_DEFAULT);
SetConfigValueInMem(CONFIG_METRIC, CONFIG_METRIC_AUTO_BOOTUP, value);
}
std::transform(value.begin(), value.end(), value.begin(), ::tolower);
return (value == "true" || value == "on" || value == "yes" || value == "1");
} }
std::string std::string
Config::GetMetricConfigCollector() { Config::GetMetricConfigCollector() {
ConfigNode metric_config = GetConfig(CONFIG_METRIC); std::string value;
return metric_config.GetValue(CONFIG_METRIC_COLLECTOR, if (!GetConfigValueInMem(CONFIG_METRIC, CONFIG_METRIC_COLLECTOR, value).ok()) {
CONFIG_METRIC_COLLECTOR_DEFAULT); value = GetConfig(CONFIG_METRIC).GetValue(CONFIG_METRIC_COLLECTOR,
CONFIG_METRIC_COLLECTOR_DEFAULT);
SetConfigValueInMem(CONFIG_METRIC, CONFIG_METRIC_COLLECTOR, value);
}
return value;
} }
std::string std::string
Config::GetMetricConfigPrometheusPort() { Config::GetMetricConfigPrometheusPort() {
ConfigNode metric_config = GetConfig(CONFIG_METRIC); std::string value;
return metric_config.GetValue(CONFIG_METRIC_PROMETHEUS_PORT, if (!GetConfigValueInMem(CONFIG_METRIC, CONFIG_METRIC_PROMETHEUS_PORT, value).ok()) {
CONFIG_METRIC_PROMETHEUS_PORT_DEFAULT); value = GetConfig(CONFIG_METRIC).GetValue(CONFIG_METRIC_PROMETHEUS_PORT,
CONFIG_METRIC_PROMETHEUS_PORT_DEFAULT);
SetConfigValueInMem(CONFIG_METRIC, CONFIG_METRIC_PROMETHEUS_PORT, value);
}
return value;
} }
////////////////////////////////////////////////////////////////////////////////
/* cache config */ /* cache config */
int32_t int32_t
Config::GetCacheConfigCpuMemCapacity() { Config::GetCacheConfigCpuMemCapacity() {
ConfigNode cache_config = GetConfig(CONFIG_CACHE); std::string value;
return cache_config.GetInt32Value(CONFIG_CACHE_CPU_MEM_CAPACITY, if (!GetConfigValueInMem(CONFIG_CACHE, CONFIG_CACHE_CPU_MEM_CAPACITY, value).ok()) {
std::stoi(CONFIG_CACHE_CPU_MEM_CAPACITY_DEFAULT)); value = GetConfig(CONFIG_CACHE).GetValue(CONFIG_CACHE_CPU_MEM_CAPACITY,
CONFIG_CACHE_CPU_MEM_CAPACITY_DEFAULT);
SetConfigValueInMem(CONFIG_CACHE, CONFIG_CACHE_CPU_MEM_CAPACITY, value);
}
return std::stoi(value);
} }
float float
Config::GetCacheConfigCpuMemThreshold() { Config::GetCacheConfigCpuMemThreshold() {
ConfigNode cache_config = GetConfig(CONFIG_CACHE); std::string value;
return cache_config.GetFloatValue(CONFIG_CACHE_CPU_MEM_THRESHOLD, if (!GetConfigValueInMem(CONFIG_CACHE, CONFIG_CACHE_CPU_MEM_THRESHOLD, value).ok()) {
std::stof(CONFIG_CACHE_CPU_MEM_THRESHOLD_DEFAULT)); value = GetConfig(CONFIG_CACHE).GetValue(CONFIG_CACHE_CPU_MEM_THRESHOLD,
CONFIG_CACHE_CPU_MEM_THRESHOLD_DEFAULT);
SetConfigValueInMem(CONFIG_CACHE, CONFIG_CACHE_CPU_MEM_THRESHOLD, value);
}
return std::stof(value);
} }
int32_t int32_t
Config::GetCacheConfigGpuMemCapacity() { Config::GetCacheConfigGpuMemCapacity() {
ConfigNode cache_config = GetConfig(CONFIG_CACHE); std::string value;
return cache_config.GetInt32Value(CONFIG_CACHE_GPU_MEM_CAPACITY, if (!GetConfigValueInMem(CONFIG_CACHE, CONFIG_CACHE_GPU_MEM_CAPACITY, value).ok()) {
std::stoi(CONFIG_CACHE_GPU_MEM_CAPACITY_DEFAULT)); value = GetConfig(CONFIG_CACHE).GetValue(CONFIG_CACHE_GPU_MEM_CAPACITY,
CONFIG_CACHE_GPU_MEM_CAPACITY_DEFAULT);
SetConfigValueInMem(CONFIG_CACHE, CONFIG_CACHE_GPU_MEM_CAPACITY, value);
}
return std::stoi(value);
} }
float float
Config::GetCacheConfigGpuMemThreshold() { Config::GetCacheConfigGpuMemThreshold() {
ConfigNode cache_config = GetConfig(CONFIG_CACHE); std::string value;
return cache_config.GetFloatValue(CONFIG_CACHE_GPU_MEM_THRESHOLD, if (!GetConfigValueInMem(CONFIG_CACHE, CONFIG_CACHE_GPU_MEM_THRESHOLD, value).ok()) {
std::stof(CONFIG_CACHE_GPU_MEM_THRESHOLD_DEFAULT)); value = GetConfig(CONFIG_CACHE).GetValue(CONFIG_CACHE_GPU_MEM_THRESHOLD,
CONFIG_CACHE_GPU_MEM_THRESHOLD_DEFAULT);
SetConfigValueInMem(CONFIG_CACHE, CONFIG_CACHE_GPU_MEM_THRESHOLD, value);
}
return std::stof(value);
} }
bool bool
Config::GetCacheConfigCacheInsertData() { Config::GetCacheConfigCacheInsertData() {
ConfigNode cache_config = GetConfig(CONFIG_CACHE); std::string value;
return cache_config.GetBoolValue(CONFIG_CACHE_CACHE_INSERT_DATA, if (!GetConfigValueInMem(CONFIG_CACHE, CONFIG_CACHE_CACHE_INSERT_DATA, value).ok()) {
std::stoi(CONFIG_CACHE_CACHE_INSERT_DATA_DEFAULT)); value = GetConfig(CONFIG_CACHE).GetValue(CONFIG_CACHE_CACHE_INSERT_DATA,
CONFIG_CACHE_CACHE_INSERT_DATA_DEFAULT);
SetConfigValueInMem(CONFIG_CACHE, CONFIG_CACHE_CACHE_INSERT_DATA, value);
}
std::transform(value.begin(), value.end(), value.begin(), ::tolower);
return (value == "true" || value == "on" || value == "yes" || value == "1");
} }
////////////////////////////////////////////////////////////////////////////////
/* engine config */ /* engine config */
int32_t int32_t
Config::GetEngineConfigBlasThreshold() { Config::GetEngineConfigBlasThreshold() {
ConfigNode engine_config = GetConfig(CONFIG_ENGINE); std::string value;
return engine_config.GetInt32Value(CONFIG_ENGINE_BLAS_THRESHOLD, if (!GetConfigValueInMem(CONFIG_ENGINE, CONFIG_ENGINE_BLAS_THRESHOLD, value).ok()) {
std::stoi(CONFIG_ENGINE_BLAS_THRESHOLD_DEFAULT)); value = GetConfig(CONFIG_ENGINE).GetValue(CONFIG_ENGINE_BLAS_THRESHOLD,
CONFIG_ENGINE_BLAS_THRESHOLD_DEFAULT);
SetConfigValueInMem(CONFIG_ENGINE, CONFIG_ENGINE_BLAS_THRESHOLD, value);
}
return std::stoi(value);
} }
int32_t int32_t
Config::GetEngineConfigOmpThreadNum() { Config::GetEngineConfigOmpThreadNum() {
ConfigNode engine_config = GetConfig(CONFIG_ENGINE); std::string value;
return engine_config.GetInt32Value(CONFIG_ENGINE_OMP_THREAD_NUM, if (!GetConfigValueInMem(CONFIG_ENGINE, CONFIG_ENGINE_OMP_THREAD_NUM, value).ok()) {
std::stoi(CONFIG_ENGINE_OMP_THREAD_NUM_DEFAULT)); value = GetConfig(CONFIG_ENGINE).GetValue(CONFIG_ENGINE_OMP_THREAD_NUM,
CONFIG_ENGINE_OMP_THREAD_NUM_DEFAULT);
SetConfigValueInMem(CONFIG_ENGINE, CONFIG_ENGINE_OMP_THREAD_NUM, value);
}
return std::stoi(value);
} }
////////////////////////////////////////////////////////////////////////////////
/* resource config */ /* resource config */
std::string std::string
Config::GetResourceConfigMode() { Config::GetResourceConfigMode() {
ConfigNode resource_config = GetConfig(CONFIG_RESOURCE); std::string value;
return resource_config.GetValue(CONFIG_RESOURCE_MODE, if (!GetConfigValueInMem(CONFIG_RESOURCE, CONFIG_RESOURCE_MODE, value).ok()) {
CONFIG_RESOURCE_MODE_DEFAULT); value = GetConfig(CONFIG_RESOURCE).GetValue(CONFIG_RESOURCE_MODE,
CONFIG_RESOURCE_MODE_DEFAULT);
SetConfigValueInMem(CONFIG_RESOURCE, CONFIG_RESOURCE_MODE, value);
}
return value;
} }
std::vector<std::string> std::vector<std::string>

20
cpp/src/server/Config.h
View File

@ -17,10 +17,12 @@
#pragma once #pragma once
#include <mutex>
#include <unordered_map>
#include "yaml-cpp/yaml.h"
#include "utils/Status.h" #include "utils/Status.h"
#include "config/ConfigNode.h" #include "config/ConfigNode.h"
#include "yaml-cpp/yaml.h"
namespace zilliz { namespace zilliz {
namespace milvus { namespace milvus {
@ -65,12 +67,12 @@ static const char* CONFIG_CACHE_CPU_MEM_THRESHOLD_DEFAULT = "0.85";
static const char* CONFIG_CACHE_GPU_MEM_THRESHOLD = "gpu_mem_threshold"; static const char* CONFIG_CACHE_GPU_MEM_THRESHOLD = "gpu_mem_threshold";
static const char* CONFIG_CACHE_GPU_MEM_THRESHOLD_DEFAULT = "0.85"; static const char* CONFIG_CACHE_GPU_MEM_THRESHOLD_DEFAULT = "0.85";
static const char* CONFIG_CACHE_CACHE_INSERT_DATA = "cache_insert_data"; static const char* CONFIG_CACHE_CACHE_INSERT_DATA = "cache_insert_data";
static const char* CONFIG_CACHE_CACHE_INSERT_DATA_DEFAULT = "0"; static const char* CONFIG_CACHE_CACHE_INSERT_DATA_DEFAULT = "false";
/* metric config */ /* metric config */
static const char* CONFIG_METRIC = "metric_config"; static const char* CONFIG_METRIC = "metric_config";
static const char* CONFIG_METRIC_AUTO_BOOTUP = "auto_bootup"; static const char* CONFIG_METRIC_AUTO_BOOTUP = "auto_bootup";
static const char* CONFIG_METRIC_AUTO_BOOTUP_DEFAULT = "0"; static const char* CONFIG_METRIC_AUTO_BOOTUP_DEFAULT = "false";
static const char* CONFIG_METRIC_COLLECTOR = "collector"; static const char* CONFIG_METRIC_COLLECTOR = "collector";
static const char* CONFIG_METRIC_COLLECTOR_DEFAULT = "prometheus"; static const char* CONFIG_METRIC_COLLECTOR_DEFAULT = "prometheus";
static const char* CONFIG_METRIC_PROMETHEUS = "prometheus_config"; static const char* CONFIG_METRIC_PROMETHEUS = "prometheus_config";
@ -110,6 +112,14 @@ class Config {
Status CheckEngineConfig(); Status CheckEngineConfig();
Status CheckResourceConfig(); Status CheckResourceConfig();
Status GetConfigValueInMem(const std::string& parent_key,
const std::string& child_key,
std::string& value);
void SetConfigValueInMem(const std::string& parent_key,
const std::string& child_key,
std::string& value);
public: public:
std::string GetServerConfigAddress(); std::string GetServerConfigAddress();
std::string GetServerConfigPort(); std::string GetServerConfigPort();
@ -140,6 +150,10 @@ class Config {
std::string GetResourceConfigMode(); std::string GetResourceConfigMode();
std::vector<std::string> std::vector<std::string>
GetResourceConfigPool(); GetResourceConfigPool();
private:
std::unordered_map<std::string, std::unordered_map<std::string, std::string>> config_map_;
std::mutex mutex_;
}; };
} }