From 82e37e11ba02640dbb6ae26cbd2dbb6bdf6370b0 Mon Sep 17 00:00:00 2001 From: "yudong.cai" Date: Fri, 20 Sep 2019 17:08:26 +0800 Subject: [PATCH 1/7] MS-574 uniform config name format, add config default value Former-commit-id: 2d11027d4b70632e22b3100fb2bde91c86972969 --- cpp/conf/server_config.template | 43 ++++++----- cpp/src/cache/CpuCacheMgr.cpp | 6 +- cpp/src/cache/GpuCacheMgr.cpp | 6 +- cpp/src/db/engine/ExecutionEngineImpl.cpp | 2 +- cpp/src/main.cpp | 2 +- cpp/src/metrics/Metrics.cpp | 3 +- cpp/src/metrics/PrometheusMetrics.cpp | 16 +++-- cpp/src/scheduler/SchedInst.cpp | 6 +- cpp/src/server/DBWrapper.cpp | 20 ++++-- cpp/src/server/Server.cpp | 11 +-- cpp/src/server/Server.h | 6 +- cpp/src/server/ServerConfig.cpp | 88 +++++++++++++---------- cpp/src/server/ServerConfig.h | 73 +++++++++++-------- cpp/src/server/grpc_impl/GrpcServer.cpp | 4 +- cpp/src/utils/SignalUtil.cpp | 4 +- cpp/src/wrapper/KnowhereResource.cpp | 7 +- cpp/unittest/server/rpc_test.cpp | 4 +- 17 files changed, 167 insertions(+), 134 deletions(-) diff --git a/cpp/conf/server_config.template b/cpp/conf/server_config.template index 8253788cb7..a96d2d81cf 100644 --- a/cpp/conf/server_config.template +++ b/cpp/conf/server_config.template @@ -1,42 +1,39 @@ server_config: - address: 0.0.0.0 # milvus server ip address (IPv4) - port: 19530 # the port milvus listen to, default: 19530, range: 1025 ~ 65534 - mode: single # milvus deployment type: single, cluster, read_only - time_zone: UTC+8 # Use the UTC-x or UTC+x to specify a time zone. eg. UTC+8 for China Standard Time + address: 0.0.0.0 # milvus server ip address (IPv4) + port: 19530 # port range: 1025 ~ 65534 + mode: single # deployment type: single, cluster, read_only + time_zone: UTC+8 db_config: - db_path: @MILVUS_DB_PATH@ # milvus data storage path - db_slave_path: # secondry data storage path, split by semicolon + db_path: @MILVUS_DB_PATH@ # milvus database path + db_slave_path: # secondary database path, split by semicolon # URI format: dialect://username:password@host:port/database # All parts except dialect are optional, but you MUST include the delimiters # Currently dialect supports mysql or sqlite db_backend_url: sqlite://:@:/ - archive_disk_threshold: 0 # triger archive action if storage size exceed this value, 0 means no limit, unit: GB - archive_days_threshold: 0 # files older than x days will be archived, 0 means no limit, unit: day - insert_buffer_size: 4 # maximum insert buffer size allowed, default: 4, unit: GB, should be at least 1 GB. - # the sum of insert_buffer_size and cpu_cache_capacity should be less than total memory, unit: GB - build_index_gpu: 0 # which gpu is used to build index, default: 0, range: 0 ~ gpu number - 1 + archive_disk_threshold: 0 # GB, file will be archived when disk usage exceed, 0 for no limit + archive_days_threshold: 0 # DAYS, older files will be archived, 0 for no limit + buffer_size: 4 # GB, maximum insert buffer size allowed + build_index_gpu: 0 # gpu id used for building index metric_config: - is_startup: off # if monitoring start: on, off - collector: prometheus # metrics collector: prometheus - prometheus_config: # following are prometheus configure - port: 8080 # the port prometheus use to fetch metrics - push_gateway_ip_address: 127.0.0.1 # push method configure: push gateway ip address - push_gateway_port: 9091 # push method configure: push gateway port + auto_bootup: off # whether enable monitoring when bootup + collector: prometheus # prometheus + prometheus_config: + port: 8080 # port prometheus used to fetch metrics cache_config: - cpu_cache_capacity: 16 # how many memory are used as cache, unit: GB, range: 0 ~ less than total memory - cpu_cache_free_percent: 0.85 # old data will be erased from cache when cache is full, this value specify how much memory should be kept, range: greater than zero ~ 1.0 - insert_cache_immediately: false # insert data will be load into cache immediately for hot query + cpu_mem_capacity: 16 # GB, CPU memory size used for cache + cpu_mem_threshold: 0.85 # percent of data kept when cache cleanup triggered + insert_immediately: false # whether load data into cache when insert engine_config: - use_blas_threshold: 20 + blas_threshold: 20 resource_config: mode: simple - resources: -# - cpu + pool: + - cpu - gpu0 diff --git a/cpp/src/cache/CpuCacheMgr.cpp b/cpp/src/cache/CpuCacheMgr.cpp index e47d768f34..ee30494192 100644 --- a/cpp/src/cache/CpuCacheMgr.cpp +++ b/cpp/src/cache/CpuCacheMgr.cpp @@ -30,11 +30,13 @@ namespace { CpuCacheMgr::CpuCacheMgr() { server::ConfigNode& config = server::ServerConfig::GetInstance().GetConfig(server::CONFIG_CACHE); - int64_t cap = config.GetInt64Value(server::CONFIG_CPU_CACHE_CAPACITY, 16); + int64_t cap = + config.GetInt64Value(server::CONFIG_CACHE_CPU_MEM_CAPACITY, std::stoi(server::CONFIG_CACHE_CPU_MEM_CAPACITY_DEFAULT)); cap *= unit; cache_ = std::make_shared>(cap, 1UL<<32); - double free_percent = config.GetDoubleValue(server::CACHE_FREE_PERCENT, 0.85); + double free_percent = + config.GetDoubleValue(server::CONFIG_CACHE_CPU_MEM_THRESHOLD, std::stof(server::CONFIG_CACHE_CPU_MEM_THRESHOLD_DEFAULT)); if(free_percent > 0.0 && free_percent <= 1.0) { cache_->set_freemem_percent(free_percent); } else { diff --git a/cpp/src/cache/GpuCacheMgr.cpp b/cpp/src/cache/GpuCacheMgr.cpp index bd6f2adfe1..58c7db3fd6 100644 --- a/cpp/src/cache/GpuCacheMgr.cpp +++ b/cpp/src/cache/GpuCacheMgr.cpp @@ -35,11 +35,13 @@ namespace { GpuCacheMgr::GpuCacheMgr() { server::ConfigNode& config = server::ServerConfig::GetInstance().GetConfig(server::CONFIG_CACHE); - int64_t cap = config.GetInt64Value(server::CONFIG_GPU_CACHE_CAPACITY, 0); + int64_t cap = + config.GetInt64Value(server::CONFIG_CACHE_GPU_MEM_CAPACITY, std::stoi(server::CONFIG_CACHE_GPU_MEM_CAPACITY_DEFAULT)); cap *= G_BYTE; cache_ = std::make_shared>(cap, 1UL<<32); - double free_percent = config.GetDoubleValue(server::GPU_CACHE_FREE_PERCENT, 0.85); + double free_percent = + config.GetDoubleValue(server::CONFIG_CACHE_GPU_MEM_THRESHOLD, std::stof(server::CONFIG_CACHE_GPU_MEM_THRESHOLD_DEFAULT)); if (free_percent > 0.0 && free_percent <= 1.0) { cache_->set_freemem_percent(free_percent); } else { diff --git a/cpp/src/db/engine/ExecutionEngineImpl.cpp b/cpp/src/db/engine/ExecutionEngineImpl.cpp index 7b9bac2e3a..65483690d6 100644 --- a/cpp/src/db/engine/ExecutionEngineImpl.cpp +++ b/cpp/src/db/engine/ExecutionEngineImpl.cpp @@ -336,7 +336,7 @@ Status ExecutionEngineImpl::Init() { using namespace zilliz::milvus::server; ServerConfig &config = ServerConfig::GetInstance(); ConfigNode server_config = config.GetConfig(CONFIG_DB); - gpu_num_ = server_config.GetInt32Value(CONFIG_DB_BUILD_INDEX_GPU, 0); + gpu_num_ = server_config.GetInt32Value(CONFIG_DB_BUILD_INDEX_GPU, std::stoi(CONFIG_DB_BUILD_INDEX_GPU_DEFAULT)); return Status::OK(); } diff --git a/cpp/src/main.cpp b/cpp/src/main.cpp index 5a61c5bfb4..dc17733a8b 100644 --- a/cpp/src/main.cpp +++ b/cpp/src/main.cpp @@ -110,7 +110,7 @@ main(int argc, char *argv[]) { signal(SIGUSR2, server::SignalUtil::HandleSignal); signal(SIGTERM, server::SignalUtil::HandleSignal); - server::Server &server = server::Server::Instance(); + server::Server &server = server::Server::GetInstance(); server.Init(start_daemonized, pid_filename, config_filename, log_config_file); server.Start(); diff --git a/cpp/src/metrics/Metrics.cpp b/cpp/src/metrics/Metrics.cpp index 10ddd7ee3c..ba51d7fd94 100644 --- a/cpp/src/metrics/Metrics.cpp +++ b/cpp/src/metrics/Metrics.cpp @@ -32,7 +32,8 @@ Metrics::GetInstance() { MetricsBase & Metrics::CreateMetricsCollector() { ConfigNode &config = ServerConfig::GetInstance().GetConfig(CONFIG_METRIC); - std::string collector_type_str = config.GetValue(CONFIG_METRIC_COLLECTOR); + std::string collector_type_str = + config.GetValue(CONFIG_METRIC_COLLECTOR, CONFIG_METRIC_COLLECTOR_DEFAULT); if (collector_type_str == "prometheus") { return PrometheusMetrics::GetInstance(); diff --git a/cpp/src/metrics/PrometheusMetrics.cpp b/cpp/src/metrics/PrometheusMetrics.cpp index f4d70a21c3..cc8aca204f 100644 --- a/cpp/src/metrics/PrometheusMetrics.cpp +++ b/cpp/src/metrics/PrometheusMetrics.cpp @@ -26,15 +26,19 @@ namespace zilliz { namespace milvus { namespace server { - ErrorCode +ErrorCode PrometheusMetrics::Init() { try { - ConfigNode &configNode = ServerConfig::GetInstance().GetConfig(CONFIG_METRIC); - startup_ = configNode.GetValue(CONFIG_METRIC_IS_STARTUP) == "on"; - if(!startup_) return SERVER_SUCCESS; + ConfigNode &metric_config = ServerConfig::GetInstance().GetConfig(CONFIG_METRIC); + startup_ = + metric_config.GetBoolValue(CONFIG_METRIC_AUTO_BOOTUP, std::stoi(CONFIG_METRIC_AUTO_BOOTUP_DEFAULT)); + if (!startup_) return SERVER_SUCCESS; + // Following should be read from config file. - const std::string bind_address = configNode.GetChild(CONFIG_PROMETHEUS).GetValue(CONFIG_METRIC_PROMETHEUS_PORT); - const std::string uri = std::string("/metrics"); + ConfigNode &prometheus_config = metric_config.GetChild(CONFIG_METRIC_PROMETHEUS); + const std::string bind_address = + prometheus_config.GetValue(CONFIG_METRIC_PROMETHEUS_PORT, CONFIG_METRIC_PROMETHEUS_PORT_DEFAULT); + const std::string uri = std::string("/tmp/metrics"); const std::size_t num_threads = 2; // Init Exposer diff --git a/cpp/src/scheduler/SchedInst.cpp b/cpp/src/scheduler/SchedInst.cpp index 3b14158e58..0b1e7cd302 100644 --- a/cpp/src/scheduler/SchedInst.cpp +++ b/cpp/src/scheduler/SchedInst.cpp @@ -39,12 +39,12 @@ std::mutex JobMgrInst::mutex_; void load_simple_config() { server::ConfigNode &config = server::ServerConfig::GetInstance().GetConfig(server::CONFIG_RESOURCE); - auto mode = config.GetValue("mode", "simple"); + auto mode = config.GetValue(server::CONFIG_RESOURCE_MODE, server::CONFIG_RESOURCE_MODE_DEFAULT); - auto resources = config.GetSequence("resources"); + auto pool = config.GetSequence(server::CONFIG_RESOURCE_POOL); bool cpu = false; std::set gpu_ids; - for (auto &resource : resources) { + for (auto &resource : pool) { if (resource == "cpu") { cpu = true; break; diff --git a/cpp/src/server/DBWrapper.cpp b/cpp/src/server/DBWrapper.cpp index 1409fff597..756438dfb3 100644 --- a/cpp/src/server/DBWrapper.cpp +++ b/cpp/src/server/DBWrapper.cpp @@ -38,7 +38,7 @@ Status DBWrapper::StartService() { //db config engine::DBOptions opt; ConfigNode& db_config = ServerConfig::GetInstance().GetConfig(CONFIG_DB); - opt.meta.backend_uri = db_config.GetValue(CONFIG_DB_URL); + opt.meta.backend_uri = db_config.GetValue(CONFIG_DB_BACKEND_URL); std::string db_path = db_config.GetValue(CONFIG_DB_PATH); opt.meta.path = db_path + "/db"; @@ -47,10 +47,11 @@ Status DBWrapper::StartService() { // cache config ConfigNode& cache_config = ServerConfig::GetInstance().GetConfig(CONFIG_CACHE); - opt.insert_cache_immediately_ = cache_config.GetBoolValue(CONFIG_INSERT_CACHE_IMMEDIATELY, false); + opt.insert_cache_immediately_ = + cache_config.GetBoolValue(CONFIG_CACHE_INSERT_IMMEDIATELY, std::stoi(CONFIG_CACHE_INSERT_IMMEDIATELY_DEFAULT)); ConfigNode& serverConfig = ServerConfig::GetInstance().GetConfig(CONFIG_SERVER); - std::string mode = serverConfig.GetValue(CONFIG_CLUSTER_MODE, "single"); + std::string mode = serverConfig.GetValue(CONFIG_SERVER_MODE, CONFIG_SERVER_MODE_DEFAULT); if (mode == "single") { opt.mode = engine::DBOptions::MODE::SINGLE; } @@ -67,7 +68,8 @@ Status DBWrapper::StartService() { // engine config ConfigNode& engine_config = ServerConfig::GetInstance().GetConfig(CONFIG_ENGINE); - int32_t omp_thread = engine_config.GetInt32Value(CONFIG_OMP_THREAD_NUM, 0); + int32_t omp_thread = + engine_config.GetInt32Value(CONFIG_ENGINE_OMP_THREAD_NUM, std::stoi(CONFIG_ENGINE_OMP_THREAD_NUM_DEFAULT)); if(omp_thread > 0) { omp_set_num_threads(omp_thread); SERVER_LOG_DEBUG << "Specify openmp thread number: " << omp_thread; @@ -79,12 +81,16 @@ Status DBWrapper::StartService() { } } - faiss::distance_compute_blas_threshold = engine_config.GetInt32Value(CONFIG_DCBT, 20);//init faiss global variable + //init faiss global variable + faiss::distance_compute_blas_threshold = + engine_config.GetInt32Value(CONFIG_ENGINE_BLAS_THRESHOLD, std::stoi(CONFIG_ENGINE_BLAS_THRESHOLD_DEFAULT)); //set archive config engine::ArchiveConf::CriteriaT criterial; - int64_t disk = db_config.GetInt64Value(CONFIG_DB_ARCHIVE_DISK, 0); - int64_t days = db_config.GetInt64Value(CONFIG_DB_ARCHIVE_DAYS, 0); + int64_t disk = + db_config.GetInt64Value(CONFIG_DB_ARCHIVE_DISK_THRESHOLD, std::stoi(CONFIG_DB_ARCHIVE_DISK_THRESHOLD_DEFAULT)); + int64_t days = + db_config.GetInt64Value(CONFIG_DB_ARCHIVE_DAYS_THRESHOLD, std::stoi(CONFIG_DB_ARCHIVE_DAYS_THRESHOLD_DEFAULT)); if(disk > 0) { criterial[engine::ARCHIVE_CONF_DISK] = disk; } diff --git a/cpp/src/server/Server.cpp b/cpp/src/server/Server.cpp index 2d0a4e5c99..7f85ed6509 100644 --- a/cpp/src/server/Server.cpp +++ b/cpp/src/server/Server.cpp @@ -43,18 +43,11 @@ namespace milvus { namespace server { Server & -Server::Instance() { +Server::GetInstance() { static Server server; return server; } -Server::Server() { - -} -Server::~Server() { - -} - void Server::Init(int64_t daemonized, const std::string &pid_filename, @@ -171,7 +164,7 @@ Server::Start() { ServerConfig &config = ServerConfig::GetInstance(); ConfigNode server_config = config.GetConfig(CONFIG_SERVER); - std::string time_zone = server_config.GetValue(CONFIG_TIME_ZONE, "UTC+8"); + std::string time_zone = server_config.GetValue(CONFIG_SERVER_TIME_ZONE, CONFIG_SERVER_TIME_ZONE_DEFAULT); if (time_zone.length() == 3) { time_zone = "CUT"; } else { diff --git a/cpp/src/server/Server.h b/cpp/src/server/Server.h index 30ea95a7bb..cb323434fe 100644 --- a/cpp/src/server/Server.h +++ b/cpp/src/server/Server.h @@ -29,7 +29,7 @@ namespace server { class Server { public: - static Server &Instance(); + static Server &GetInstance(); void Init(int64_t daemonized, const std::string &pid_filename, @@ -40,8 +40,8 @@ class Server { void Stop(); private: - Server(); - ~Server(); + Server() = default; + ~Server() = default; void Daemonize(); diff --git a/cpp/src/server/ServerConfig.cpp b/cpp/src/server/ServerConfig.cpp index 687490cf5e..0f51ecf6e2 100644 --- a/cpp/src/server/ServerConfig.cpp +++ b/cpp/src/server/ServerConfig.cpp @@ -109,50 +109,51 @@ ServerConfig::CheckServerConfig() { bool okay = true; ConfigNode server_config = GetConfig(CONFIG_SERVER); - std::string ip_address = server_config.GetValue(CONFIG_SERVER_ADDRESS, "127.0.0.1"); + std::string ip_address = server_config.GetValue(CONFIG_SERVER_ADDRESS, CONFIG_SERVER_ADDRESS_DEFAULT); if (!ValidationUtil::ValidateIpAddress(ip_address).ok()) { std::cerr << "ERROR: invalid server IP address: " << ip_address << std::endl; okay = false; } - std::string port_str = server_config.GetValue(CONFIG_SERVER_PORT, "19530"); + std::string port_str = server_config.GetValue(CONFIG_SERVER_PORT, CONFIG_SERVER_PORT_DEFAULT); if (!ValidationUtil::ValidateStringIsNumber(port_str).ok()) { std::cerr << "ERROR: port " << port_str << " is not a number" << std::endl; okay = false; } else { int32_t port = std::stol(port_str); - if (port < 1025 | port > 65534) { - std::cerr << "ERROR: port " << port_str << " out of range [1025, 65534]" << std::endl; + if (!(port > 1024 && port < 65535)) { + std::cerr << "ERROR: port " << port_str << " out of range (1024, 65535)" << std::endl; okay = false; } } - std::string mode = server_config.GetValue(CONFIG_CLUSTER_MODE, "single"); + std::string mode = server_config.GetValue(CONFIG_SERVER_MODE, CONFIG_SERVER_MODE_DEFAULT); if (mode != "single" && mode != "cluster" && mode != "read_only") { std::cerr << "ERROR: mode " << mode << " is not one of ['single', 'cluster', 'read_only']" << std::endl; okay = false; } - std::string time_zone = server_config.GetValue(CONFIG_TIME_ZONE, "UTC+8"); + std::string time_zone = server_config.GetValue(CONFIG_SERVER_TIME_ZONE, CONFIG_SERVER_TIME_ZONE_DEFAULT); int flag = 0; - if(time_zone.length() < 3) + if (time_zone.length() <= 3) { flag = 1; - else if(time_zone.substr(0, 3) != "UTC") - flag = 1; - else if(time_zone.length() > 3){ - try { - stoi(time_zone.substr(3, std::string::npos)); - } - catch (std::invalid_argument &) { + } else { + if (time_zone.substr(0, 3) != "UTC") { flag = 1; + } else { + try { + stoi(time_zone.substr(3)); + } catch (...) { + flag = 1; + } } } - if(flag == 1){ - std::cerr << "ERROR: time_zone " << time_zone << " is not in a right format" << std::endl; + if (flag == 1) { + std::cerr << "ERROR: time_zone " << time_zone << " format wrong" << std::endl; okay = false; } - return (okay ? Status::OK() : Status(SERVER_INVALID_ARGUMENT, "Server config is illegal")); + return (okay ? Status::OK() : Status(SERVER_INVALID_ARGUMENT, "Illegal server config")); } Status @@ -182,25 +183,25 @@ ServerConfig::CheckDBConfig() { okay = false; } - std::string db_backend_url = db_config.GetValue(CONFIG_DB_URL); + std::string db_backend_url = db_config.GetValue(CONFIG_DB_BACKEND_URL); if (!ValidationUtil::ValidateDbURI(db_backend_url).ok()) { std::cerr << "ERROR: invalid db_backend_url: " << db_backend_url << std::endl; okay = false; } - std::string archive_disk_threshold_str = db_config.GetValue(CONFIG_DB_INSERT_BUFFER_SIZE, "0"); + std::string archive_disk_threshold_str = db_config.GetValue(CONFIG_DB_ARCHIVE_DISK_THRESHOLD, CONFIG_DB_ARCHIVE_DISK_THRESHOLD_DEFAULT); if (!ValidationUtil::ValidateStringIsNumber(archive_disk_threshold_str).ok()) { std::cerr << "ERROR: archive_disk_threshold " << archive_disk_threshold_str << " is not a number" << std::endl; okay = false; } - std::string archive_days_threshold_str = db_config.GetValue(CONFIG_DB_INSERT_BUFFER_SIZE, "0"); + std::string archive_days_threshold_str = db_config.GetValue(CONFIG_DB_ARCHIVE_DAYS_THRESHOLD, CONFIG_DB_ARCHIVE_DAYS_THRESHOLD_DEFAULT); if (!ValidationUtil::ValidateStringIsNumber(archive_days_threshold_str).ok()) { std::cerr << "ERROR: archive_days_threshold " << archive_days_threshold_str << " is not a number" << std::endl; okay = false; } - std::string insert_buffer_size_str = db_config.GetValue(CONFIG_DB_INSERT_BUFFER_SIZE, "4"); + std::string insert_buffer_size_str = db_config.GetValue(CONFIG_DB_BUFFER_SIZE, CONFIG_DB_BUFFER_SIZE_DEFAULT); if (!ValidationUtil::ValidateStringIsNumber(insert_buffer_size_str).ok()) { std::cerr << "ERROR: insert_buffer_size " << insert_buffer_size_str << " is not a number" << std::endl; okay = false; @@ -216,7 +217,7 @@ ServerConfig::CheckDBConfig() { } } - std::string gpu_index_str = db_config.GetValue(CONFIG_DB_BUILD_INDEX_GPU, "0"); + std::string gpu_index_str = db_config.GetValue(CONFIG_DB_BUILD_INDEX_GPU, CONFIG_DB_BUILD_INDEX_GPU_DEFAULT); if (!ValidationUtil::ValidateStringIsNumber(gpu_index_str).ok()) { std::cerr << "ERROR: gpu_index " << gpu_index_str << " is not a number" << std::endl; okay = false; @@ -245,13 +246,14 @@ ServerConfig::CheckMetricConfig() { bool okay = true; ConfigNode metric_config = GetConfig(CONFIG_METRIC); - std::string is_startup_str = metric_config.GetValue(CONFIG_METRIC_IS_STARTUP, "off"); + std::string is_startup_str = + metric_config.GetValue(CONFIG_METRIC_AUTO_BOOTUP, CONFIG_METRIC_AUTO_BOOTUP_DEFAULT); if (!ValidationUtil::ValidateStringIsBool(is_startup_str).ok()) { std::cerr << "ERROR: invalid is_startup config: " << is_startup_str << std::endl; okay = false; } - std::string port_str = metric_config.GetChild(CONFIG_PROMETHEUS).GetValue(CONFIG_METRIC_PROMETHEUS_PORT, "8080"); + std::string port_str = metric_config.GetChild(CONFIG_METRIC_PROMETHEUS).GetValue(CONFIG_METRIC_PROMETHEUS_PORT, "8080"); if (!ValidationUtil::ValidateStringIsNumber(port_str).ok()) { std::cerr << "ERROR: port specified in prometheus_config " << port_str << " is not a number" << std::endl; okay = false; @@ -274,7 +276,8 @@ ServerConfig::CheckCacheConfig() { bool okay = true; ConfigNode cache_config = GetConfig(CONFIG_CACHE); - std::string cpu_cache_capacity_str = cache_config.GetValue(CONFIG_CPU_CACHE_CAPACITY, "16"); + std::string cpu_cache_capacity_str = + cache_config.GetValue(CONFIG_CACHE_CPU_MEM_CAPACITY, CONFIG_CACHE_CPU_MEM_CAPACITY_DEFAULT); if (!ValidationUtil::ValidateStringIsNumber(cpu_cache_capacity_str).ok()) { std::cerr << "ERROR: cpu_cache_capacity " << cpu_cache_capacity_str << " is not a number" << std::endl; okay = false; @@ -292,15 +295,17 @@ ServerConfig::CheckCacheConfig() { std::cerr << "Warning: cpu_cache_capacity value is too aggressive" << std::endl; } - uint64_t insert_buffer_size = (uint64_t) GetConfig(CONFIG_DB).GetInt32Value(CONFIG_DB_INSERT_BUFFER_SIZE, 4); - insert_buffer_size *= GB; - if (insert_buffer_size + cpu_cache_capacity >= total_mem) { + uint64_t buffer_size = + (uint64_t) GetConfig(CONFIG_DB).GetInt32Value(CONFIG_DB_BUFFER_SIZE, std::stoi(CONFIG_DB_BUFFER_SIZE_DEFAULT)); + buffer_size *= GB; + if (buffer_size + cpu_cache_capacity >= total_mem) { std::cerr << "ERROR: sum of cpu_cache_capacity and insert_buffer_size exceed system memory" << std::endl; okay = false; } } - std::string cpu_cache_free_percent_str = cache_config.GetValue(CACHE_FREE_PERCENT, "0.85"); + std::string cpu_cache_free_percent_str = + cache_config.GetValue(CONFIG_CACHE_CPU_MEM_THRESHOLD, CONFIG_CACHE_CPU_MEM_THRESHOLD_DEFAULT); double cpu_cache_free_percent; if (!ValidationUtil::ValidateStringIsDouble(cpu_cache_free_percent_str, cpu_cache_free_percent).ok()) { std::cerr << "ERROR: cpu_cache_free_percent " << cpu_cache_free_percent_str << " is not a double" << std::endl; @@ -311,13 +316,15 @@ ServerConfig::CheckCacheConfig() { okay = false; } - std::string insert_cache_immediately_str = cache_config.GetValue(CONFIG_INSERT_CACHE_IMMEDIATELY, "false"); + std::string insert_cache_immediately_str = + cache_config.GetValue(CONFIG_CACHE_INSERT_IMMEDIATELY, CONFIG_CACHE_INSERT_IMMEDIATELY_DEFAULT); if (!ValidationUtil::ValidateStringIsBool(insert_cache_immediately_str).ok()) { std::cerr << "ERROR: invalid insert_cache_immediately config: " << insert_cache_immediately_str << std::endl; okay = false; } - std::string gpu_cache_capacity_str = cache_config.GetValue(CONFIG_GPU_CACHE_CAPACITY, "0"); + std::string gpu_cache_capacity_str = + cache_config.GetValue(CONFIG_CACHE_GPU_MEM_CAPACITY, CONFIG_CACHE_GPU_MEM_CAPACITY_DEFAULT); if (!ValidationUtil::ValidateStringIsNumber(gpu_cache_capacity_str).ok()) { std::cerr << "ERROR: gpu_cache_capacity " << gpu_cache_capacity_str << " is not a number" << std::endl; okay = false; @@ -325,7 +332,7 @@ ServerConfig::CheckCacheConfig() { else { uint64_t gpu_cache_capacity = (uint64_t) std::stol(gpu_cache_capacity_str); gpu_cache_capacity *= GB; - int gpu_index = GetConfig(CONFIG_DB).GetInt32Value(CONFIG_DB_BUILD_INDEX_GPU, 0); + int gpu_index = GetConfig(CONFIG_DB).GetInt32Value(CONFIG_DB_BUILD_INDEX_GPU, std::stoi(CONFIG_DB_BUILD_INDEX_GPU_DEFAULT)); size_t gpu_memory; if (!ValidationUtil::GetGpuMemory(gpu_index, gpu_memory).ok()) { std::cerr << "ERROR: could not get gpu memory for device " << gpu_index << std::endl; @@ -341,7 +348,8 @@ ServerConfig::CheckCacheConfig() { } } - std::string gpu_cache_free_percent_str = cache_config.GetValue(GPU_CACHE_FREE_PERCENT, "0.85"); + std::string gpu_cache_free_percent_str = + cache_config.GetValue(CONFIG_CACHE_GPU_MEM_THRESHOLD, CONFIG_CACHE_GPU_MEM_THRESHOLD_DEFAULT); double gpu_cache_free_percent; if (!ValidationUtil::ValidateStringIsDouble(gpu_cache_free_percent_str, gpu_cache_free_percent).ok()) { std::cerr << "ERROR: gpu_cache_free_percent " << gpu_cache_free_percent_str << " is not a double" << std::endl; @@ -365,13 +373,15 @@ ServerConfig::CheckEngineConfig() { bool okay = true; ConfigNode engine_config = GetConfig(CONFIG_ENGINE); - std::string use_blas_threshold_str = engine_config.GetValue(CONFIG_DCBT, "20"); + std::string use_blas_threshold_str = + engine_config.GetValue(CONFIG_ENGINE_BLAS_THRESHOLD, CONFIG_ENGINE_BLAS_THRESHOLD_DEFAULT); if (!ValidationUtil::ValidateStringIsNumber(use_blas_threshold_str).ok()) { std::cerr << "ERROR: use_blas_threshold " << use_blas_threshold_str << " is not a number" << std::endl; okay = false; } - std::string omp_thread_num_str = engine_config.GetValue(CONFIG_OMP_THREAD_NUM, "0"); + std::string omp_thread_num_str = + engine_config.GetValue(CONFIG_ENGINE_OMP_THREAD_NUM, CONFIG_ENGINE_OMP_THREAD_NUM_DEFAULT); if (!ValidationUtil::ValidateStringIsNumber(omp_thread_num_str).ok()) { std::cerr << "ERROR: omp_thread_num " << omp_thread_num_str << " is not a number" << std::endl; okay = false; @@ -393,20 +403,20 @@ ServerConfig::CheckResourceConfig() { /* resource_config: mode: simple - resources: + pool: - cpu - gpu0 - gpu100 */ bool okay = true; server::ConfigNode &config = server::ServerConfig::GetInstance().GetConfig(server::CONFIG_RESOURCE); - auto mode = config.GetValue("mode", "simple"); + auto mode = config.GetValue(CONFIG_RESOURCE_MODE, CONFIG_RESOURCE_MODE_DEFAULT); if (mode != "simple") { std::cerr << "ERROR: invalid resource config: mode is " << mode << std::endl; okay = false; } - auto resources = config.GetSequence("resources"); - if (resources.empty()) { + auto pool = config.GetSequence(CONFIG_RESOURCE_POOL); + if (pool.empty()) { std::cerr << "ERROR: invalid resource config: resources empty" << std::endl; okay = false; } diff --git a/cpp/src/server/ServerConfig.h b/cpp/src/server/ServerConfig.h index d0f3e30d6b..e515b74645 100644 --- a/cpp/src/server/ServerConfig.h +++ b/cpp/src/server/ServerConfig.h @@ -26,52 +26,69 @@ namespace zilliz { namespace milvus { namespace server { +/* server config */ static const char* CONFIG_SERVER = "server_config"; static const char* CONFIG_SERVER_ADDRESS = "address"; +static const char* CONFIG_SERVER_ADDRESS_DEFAULT = "127.0.0.1"; static const char* CONFIG_SERVER_PORT = "port"; -static const char* CONFIG_CLUSTER_MODE = "mode"; -static const char* CONFIG_TIME_ZONE = "time_zone"; +static const char* CONFIG_SERVER_PORT_DEFAULT = "19530"; +static const char* CONFIG_SERVER_MODE = "mode"; +static const char* CONFIG_SERVER_MODE_DEFAULT = "single"; +static const char* CONFIG_SERVER_TIME_ZONE = "time_zone"; +static const char* CONFIG_SERVER_TIME_ZONE_DEFAULT = "UTC+8"; +/* db config */ static const char* CONFIG_DB = "db_config"; -static const char* CONFIG_DB_URL = "db_backend_url"; static const char* CONFIG_DB_PATH = "db_path"; +static const char* CONFIG_DB_PATH_DEFAULT = "/tmp/milvus"; static const char* CONFIG_DB_SLAVE_PATH = "db_slave_path"; -static const char* CONFIG_DB_ARCHIVE_DISK = "archive_disk_threshold"; -static const char* CONFIG_DB_ARCHIVE_DAYS = "archive_days_threshold"; -static const char* CONFIG_DB_INSERT_BUFFER_SIZE = "insert_buffer_size"; -static const char* CONFIG_DB_PARALLEL_REDUCE = "parallel_reduce"; +static const char* CONFIG_DB_SLAVE_PATH_DEFAULT = ""; +static const char* CONFIG_DB_BACKEND_URL = "db_backend_url"; +static const char* CONFIG_DB_BACKEND_URL_DEFAULT = "sqlite://:@:/"; +static const char* CONFIG_DB_ARCHIVE_DISK_THRESHOLD = "archive_disk_threshold"; +static const char* CONFIG_DB_ARCHIVE_DISK_THRESHOLD_DEFAULT = "0"; +static const char* CONFIG_DB_ARCHIVE_DAYS_THRESHOLD = "archive_days_threshold"; +static const char* CONFIG_DB_ARCHIVE_DAYS_THRESHOLD_DEFAULT = "0"; +static const char* CONFIG_DB_BUFFER_SIZE = "buffer_size"; +static const char* CONFIG_DB_BUFFER_SIZE_DEFAULT = "4"; static const char* CONFIG_DB_BUILD_INDEX_GPU = "build_index_gpu"; +static const char* CONFIG_DB_BUILD_INDEX_GPU_DEFAULT = "0"; -static const char* CONFIG_LOG = "log_config"; - +/* cache config */ static const char* CONFIG_CACHE = "cache_config"; -static const char* CONFIG_CPU_CACHE_CAPACITY = "cpu_cache_capacity"; -static const char* CONFIG_GPU_CACHE_CAPACITY = "gpu_cache_capacity"; -static const char* CACHE_FREE_PERCENT = "cpu_cache_free_percent"; -static const char* CONFIG_INSERT_CACHE_IMMEDIATELY = "insert_cache_immediately"; -static const char *GPU_CACHE_FREE_PERCENT = "gpu_cache_free_percent"; +static const char* CONFIG_CACHE_CPU_MEM_CAPACITY = "cpu_mem_capacity"; +static const char* CONFIG_CACHE_CPU_MEM_CAPACITY_DEFAULT = "16"; +static const char* CONFIG_CACHE_GPU_MEM_CAPACITY = "gpu_mem_capacity"; +static const char* CONFIG_CACHE_GPU_MEM_CAPACITY_DEFAULT = "0"; +static const char* CONFIG_CACHE_CPU_MEM_THRESHOLD = "cpu_mem_threshold"; +static const char* CONFIG_CACHE_CPU_MEM_THRESHOLD_DEFAULT = "0.85"; +static const char* CONFIG_CACHE_GPU_MEM_THRESHOLD = "gpu_mem_threshold"; +static const char* CONFIG_CACHE_GPU_MEM_THRESHOLD_DEFAULT = "0.85"; +static const char* CONFIG_CACHE_INSERT_IMMEDIATELY = "insert_immediately"; +static const char* CONFIG_CACHE_INSERT_IMMEDIATELY_DEFAULT = "0"; +/* metric config */ static const char* CONFIG_METRIC = "metric_config"; -static const char* CONFIG_METRIC_IS_STARTUP = "is_startup"; +static const char* CONFIG_METRIC_AUTO_BOOTUP = "auto_bootup"; +static const char* CONFIG_METRIC_AUTO_BOOTUP_DEFAULT = "0"; static const char* CONFIG_METRIC_COLLECTOR = "collector"; -static const char* CONFIG_PROMETHEUS = "prometheus_config"; +static const char* CONFIG_METRIC_COLLECTOR_DEFAULT = "prometheus"; +static const char* CONFIG_METRIC_PROMETHEUS = "prometheus_config"; static const char* CONFIG_METRIC_PROMETHEUS_PORT = "port"; +static const char* CONFIG_METRIC_PROMETHEUS_PORT_DEFAULT = "8080"; +/* engine config */ static const char* CONFIG_ENGINE = "engine_config"; -static const char* CONFIG_DCBT = "use_blas_threshold"; -static const char* CONFIG_OMP_THREAD_NUM = "omp_thread_num"; +static const char* CONFIG_ENGINE_BLAS_THRESHOLD = "blas_threshold"; +static const char* CONFIG_ENGINE_BLAS_THRESHOLD_DEFAULT = "20"; +static const char* CONFIG_ENGINE_OMP_THREAD_NUM = "omp_thread_num"; +static const char* CONFIG_ENGINE_OMP_THREAD_NUM_DEFAULT = "0"; +/* resource config */ static const char* CONFIG_RESOURCE = "resource_config"; -static const char* CONFIG_RESOURCES = "resources"; -static const char* CONFIG_RESOURCE_TYPE = "type"; -static const char* CONFIG_RESOURCE_DEVICE_ID = "device_id"; -static const char* CONFIG_RESOURCE_ENABLE_EXECUTOR = "enable_executor"; -static const char* CONFIG_RESOURCE_NUM = "gpu_resource_num"; -static const char* CONFIG_RESOURCE_PIN_MEMORY = "pinned_memory"; -static const char* CONFIG_RESOURCE_TEMP_MEMORY = "temp_memory"; -static const char* CONFIG_RESOURCE_CONNECTIONS = "connections"; -static const char* CONFIG_SPEED_CONNECTIONS = "speed"; -static const char* CONFIG_ENDPOINT_CONNECTIONS = "endpoint"; +static const char* CONFIG_RESOURCE_MODE = "mode"; +static const char* CONFIG_RESOURCE_MODE_DEFAULT = "simple"; +static const char* CONFIG_RESOURCE_POOL = "pool"; class ServerConfig { diff --git a/cpp/src/server/grpc_impl/GrpcServer.cpp b/cpp/src/server/grpc_impl/GrpcServer.cpp index 4ff7a8200f..5160e0c8ad 100644 --- a/cpp/src/server/grpc_impl/GrpcServer.cpp +++ b/cpp/src/server/grpc_impl/GrpcServer.cpp @@ -76,8 +76,8 @@ GrpcServer::StartService() { ServerConfig &config = ServerConfig::GetInstance(); ConfigNode server_config = config.GetConfig(CONFIG_SERVER); ConfigNode engine_config = config.GetConfig(CONFIG_ENGINE); - std::string address = server_config.GetValue(CONFIG_SERVER_ADDRESS, "127.0.0.1"); - int32_t port = server_config.GetInt32Value(CONFIG_SERVER_PORT, 19530); + std::string address = server_config.GetValue(CONFIG_SERVER_ADDRESS, CONFIG_SERVER_ADDRESS_DEFAULT); + int32_t port = server_config.GetInt32Value(CONFIG_SERVER_PORT, std::stoi(CONFIG_SERVER_PORT_DEFAULT)); std::string server_address(address + ":" + std::to_string(port)); diff --git a/cpp/src/utils/SignalUtil.cpp b/cpp/src/utils/SignalUtil.cpp index 6c68f702d2..92678cfe6b 100644 --- a/cpp/src/utils/SignalUtil.cpp +++ b/cpp/src/utils/SignalUtil.cpp @@ -34,7 +34,7 @@ void SignalUtil::HandleSignal(int signum) { case SIGUSR2: { SERVER_LOG_INFO << "Server received signal: " << signum; - server::Server &server = server::Server::Instance(); + server::Server &server = server::Server::GetInstance(); server.Stop(); exit(0); @@ -43,7 +43,7 @@ void SignalUtil::HandleSignal(int signum) { SERVER_LOG_INFO << "Server received critical signal: " << signum; SignalUtil::PrintStacktrace(); - server::Server &server = server::Server::Instance(); + server::Server &server = server::Server::GetInstance(); server.Stop(); exit(1); diff --git a/cpp/src/wrapper/KnowhereResource.cpp b/cpp/src/wrapper/KnowhereResource.cpp index 007858de37..e59b56b574 100644 --- a/cpp/src/wrapper/KnowhereResource.cpp +++ b/cpp/src/wrapper/KnowhereResource.cpp @@ -41,14 +41,15 @@ ErrorCode KnowhereResource::Initialize() { server::ServerConfig& root_config = server::ServerConfig::GetInstance(); server::ConfigNode& db_config = root_config.GetConfig(server::CONFIG_DB); - int32_t build_index_gpu = db_config.GetInt32Value(server::CONFIG_DB_BUILD_INDEX_GPU, 0); + int32_t build_index_gpu = + db_config.GetInt32Value(server::CONFIG_DB_BUILD_INDEX_GPU, std::stoi(server::CONFIG_DB_BUILD_INDEX_GPU_DEFAULT)); gpu_resources.insert(std::make_pair(build_index_gpu, GpuResourceSetting())); //get search gpu resource server::ConfigNode& res_config = root_config.GetConfig(server::CONFIG_RESOURCE); - auto resources = res_config.GetSequence("resources"); + auto pool = res_config.GetSequence(server::CONFIG_RESOURCE_POOL); std::set gpu_ids; - for (auto &resource : resources) { + for (auto &resource : pool) { if (resource.length() < 4 || resource.substr(0, 3) != "gpu") { // invalid continue; diff --git a/cpp/unittest/server/rpc_test.cpp b/cpp/unittest/server/rpc_test.cpp index 69b258aa75..5ad6005a36 100644 --- a/cpp/unittest/server/rpc_test.cpp +++ b/cpp/unittest/server/rpc_test.cpp @@ -67,7 +67,7 @@ class RpcHandlerTest : public testing::Test { engine::DBOptions opt; server::ConfigNode &db_config = server::ServerConfig::GetInstance().GetConfig(server::CONFIG_DB); - db_config.SetValue(server::CONFIG_DB_URL, "sqlite://:@:/"); + db_config.SetValue(server::CONFIG_DB_BACKEND_URL, "sqlite://:@:/"); db_config.SetValue(server::CONFIG_DB_PATH, "/tmp/milvus_test"); db_config.SetValue(server::CONFIG_DB_SLAVE_PATH, ""); db_config.SetValue(server::CONFIG_DB_ARCHIVE_DISK, ""); @@ -77,7 +77,7 @@ class RpcHandlerTest : public testing::Test { cache_config.SetValue(server::CONFIG_INSERT_CACHE_IMMEDIATELY, ""); server::ConfigNode &engine_config = server::ServerConfig::GetInstance().GetConfig(server::CONFIG_ENGINE); - engine_config.SetValue(server::CONFIG_OMP_THREAD_NUM, ""); + engine_config.SetValue(server::CONFIG_ENGINE_OMP_THREAD_NUM, ""); server::ConfigNode &serverConfig = server::ServerConfig::GetInstance().GetConfig(server::CONFIG_SERVER); // serverConfig.SetValue(server::CONFIG_CLUSTER_MODE, "cluster"); From 22563d83d7efce599799ae57ac52b736e61220a2 Mon Sep 17 00:00:00 2001 From: "yudong.cai" Date: Sat, 21 Sep 2019 15:02:35 +0800 Subject: [PATCH 2/7] MS-574 add GET config APIs Former-commit-id: 6011f32d650d26367ab84f4bf8521889881f2629 --- cpp/conf/server_config.template | 8 +- cpp/src/cache/CpuCacheMgr.cpp | 17 +- cpp/src/cache/GpuCacheMgr.cpp | 17 +- cpp/src/db/engine/ExecutionEngineImpl.cpp | 3 +- cpp/src/metrics/Metrics.cpp | 6 +- cpp/src/metrics/PrometheusMetrics.cpp | 9 +- cpp/src/scheduler/SchedInst.cpp | 7 +- cpp/src/server/DBWrapper.cpp | 34 ++-- cpp/src/server/Server.cpp | 9 +- cpp/src/server/ServerConfig.cpp | 220 +++++++++++++++++++--- cpp/src/server/ServerConfig.h | 45 ++++- cpp/src/server/grpc_impl/GrpcServer.cpp | 8 +- cpp/src/wrapper/KnowhereResource.cpp | 9 +- 13 files changed, 280 insertions(+), 112 deletions(-) diff --git a/cpp/conf/server_config.template b/cpp/conf/server_config.template index a96d2d81cf..6364698100 100644 --- a/cpp/conf/server_config.template +++ b/cpp/conf/server_config.template @@ -5,13 +5,13 @@ server_config: time_zone: UTC+8 db_config: - db_path: @MILVUS_DB_PATH@ # milvus database path - db_slave_path: # secondary database path, split by semicolon + path: @MILVUS_DB_PATH@ # milvus database path + slave_path: # secondary database path, split by semicolon # URI format: dialect://username:password@host:port/database # All parts except dialect are optional, but you MUST include the delimiters # Currently dialect supports mysql or sqlite - db_backend_url: sqlite://:@:/ + backend_url: sqlite://:@:/ archive_disk_threshold: 0 # GB, file will be archived when disk usage exceed, 0 for no limit archive_days_threshold: 0 # DAYS, older files will be archived, 0 for no limit @@ -27,7 +27,7 @@ metric_config: cache_config: cpu_mem_capacity: 16 # GB, CPU memory size used for cache cpu_mem_threshold: 0.85 # percent of data kept when cache cleanup triggered - insert_immediately: false # whether load data into cache when insert + cache_insert_data: false # whether load data into cache when insert engine_config: blas_threshold: 20 diff --git a/cpp/src/cache/CpuCacheMgr.cpp b/cpp/src/cache/CpuCacheMgr.cpp index ee30494192..7141394fa1 100644 --- a/cpp/src/cache/CpuCacheMgr.cpp +++ b/cpp/src/cache/CpuCacheMgr.cpp @@ -29,19 +29,16 @@ namespace { } CpuCacheMgr::CpuCacheMgr() { - server::ConfigNode& config = server::ServerConfig::GetInstance().GetConfig(server::CONFIG_CACHE); - int64_t cap = - config.GetInt64Value(server::CONFIG_CACHE_CPU_MEM_CAPACITY, std::stoi(server::CONFIG_CACHE_CPU_MEM_CAPACITY_DEFAULT)); - cap *= unit; + server::ServerConfig& config = server::ServerConfig::GetInstance(); + int64_t cap = config.GetCacheConfigCpuMemCapacity() * unit; cache_ = std::make_shared>(cap, 1UL<<32); - double free_percent = - config.GetDoubleValue(server::CONFIG_CACHE_CPU_MEM_THRESHOLD, std::stof(server::CONFIG_CACHE_CPU_MEM_THRESHOLD_DEFAULT)); - if(free_percent > 0.0 && free_percent <= 1.0) { - cache_->set_freemem_percent(free_percent); + float cpu_mem_threshold = config.GetCacheConfigCpuMemThreshold(); + if (cpu_mem_threshold > 0.0 && cpu_mem_threshold <= 1.0) { + cache_->set_freemem_percent(cpu_mem_threshold); } else { - SERVER_LOG_ERROR << "Invalid cache_free_percent: " << free_percent << - ", defaultly set to " << cache_->freemem_percent(); + SERVER_LOG_ERROR << "Invalid cpu_mem_threshold: " << cpu_mem_threshold + << ", by default set to " << cache_->freemem_percent(); } } diff --git a/cpp/src/cache/GpuCacheMgr.cpp b/cpp/src/cache/GpuCacheMgr.cpp index 58c7db3fd6..ba5665f5f2 100644 --- a/cpp/src/cache/GpuCacheMgr.cpp +++ b/cpp/src/cache/GpuCacheMgr.cpp @@ -33,20 +33,17 @@ namespace { } GpuCacheMgr::GpuCacheMgr() { - server::ConfigNode& config = server::ServerConfig::GetInstance().GetConfig(server::CONFIG_CACHE); + server::ServerConfig& config = server::ServerConfig::GetInstance(); - int64_t cap = - config.GetInt64Value(server::CONFIG_CACHE_GPU_MEM_CAPACITY, std::stoi(server::CONFIG_CACHE_GPU_MEM_CAPACITY_DEFAULT)); - cap *= G_BYTE; + int32_t cap = config.GetCacheConfigGpuMemCapacity() * G_BYTE; cache_ = std::make_shared>(cap, 1UL<<32); - double free_percent = - config.GetDoubleValue(server::CONFIG_CACHE_GPU_MEM_THRESHOLD, std::stof(server::CONFIG_CACHE_GPU_MEM_THRESHOLD_DEFAULT)); - if (free_percent > 0.0 && free_percent <= 1.0) { - cache_->set_freemem_percent(free_percent); + float gpu_mem_threshold = config.GetCacheConfigGpuMemThreshold(); + if (gpu_mem_threshold > 0.0 && gpu_mem_threshold <= 1.0) { + cache_->set_freemem_percent(gpu_mem_threshold); } else { - SERVER_LOG_ERROR << "Invalid gpu_cache_free_percent: " << free_percent << - ", defaultly set to " << cache_->freemem_percent(); + SERVER_LOG_ERROR << "Invalid gpu_mem_threshold: " << gpu_mem_threshold + << ", by default set to " << cache_->freemem_percent(); } } diff --git a/cpp/src/db/engine/ExecutionEngineImpl.cpp b/cpp/src/db/engine/ExecutionEngineImpl.cpp index 65483690d6..960f3baa89 100644 --- a/cpp/src/db/engine/ExecutionEngineImpl.cpp +++ b/cpp/src/db/engine/ExecutionEngineImpl.cpp @@ -335,8 +335,7 @@ Status ExecutionEngineImpl::GpuCache(uint64_t gpu_id) { Status ExecutionEngineImpl::Init() { using namespace zilliz::milvus::server; ServerConfig &config = ServerConfig::GetInstance(); - ConfigNode server_config = config.GetConfig(CONFIG_DB); - gpu_num_ = server_config.GetInt32Value(CONFIG_DB_BUILD_INDEX_GPU, std::stoi(CONFIG_DB_BUILD_INDEX_GPU_DEFAULT)); + gpu_num_ = config.GetDBConfigBuildIndexGPU(); return Status::OK(); } diff --git a/cpp/src/metrics/Metrics.cpp b/cpp/src/metrics/Metrics.cpp index ba51d7fd94..59eaea6355 100644 --- a/cpp/src/metrics/Metrics.cpp +++ b/cpp/src/metrics/Metrics.cpp @@ -16,6 +16,7 @@ // under the License. #include "Metrics.h" +#include "server/ServerConfig.h" #include "PrometheusMetrics.h" @@ -31,9 +32,8 @@ Metrics::GetInstance() { MetricsBase & Metrics::CreateMetricsCollector() { - ConfigNode &config = ServerConfig::GetInstance().GetConfig(CONFIG_METRIC); - std::string collector_type_str = - config.GetValue(CONFIG_METRIC_COLLECTOR, CONFIG_METRIC_COLLECTOR_DEFAULT); + ServerConfig &config = ServerConfig::GetInstance(); + std::string collector_type_str = config.GetMetricConfigCollector(); if (collector_type_str == "prometheus") { return PrometheusMetrics::GetInstance(); diff --git a/cpp/src/metrics/PrometheusMetrics.cpp b/cpp/src/metrics/PrometheusMetrics.cpp index cc8aca204f..b07e91ea32 100644 --- a/cpp/src/metrics/PrometheusMetrics.cpp +++ b/cpp/src/metrics/PrometheusMetrics.cpp @@ -29,15 +29,12 @@ namespace server { ErrorCode PrometheusMetrics::Init() { try { - ConfigNode &metric_config = ServerConfig::GetInstance().GetConfig(CONFIG_METRIC); - startup_ = - metric_config.GetBoolValue(CONFIG_METRIC_AUTO_BOOTUP, std::stoi(CONFIG_METRIC_AUTO_BOOTUP_DEFAULT)); + ServerConfig &config = ServerConfig::GetInstance(); + startup_ = config.GetMetricConfigAutoBootup(); if (!startup_) return SERVER_SUCCESS; // Following should be read from config file. - ConfigNode &prometheus_config = metric_config.GetChild(CONFIG_METRIC_PROMETHEUS); - const std::string bind_address = - prometheus_config.GetValue(CONFIG_METRIC_PROMETHEUS_PORT, CONFIG_METRIC_PROMETHEUS_PORT_DEFAULT); + const std::string bind_address = config.GetMetricConfigPrometheusPort(); const std::string uri = std::string("/tmp/metrics"); const std::size_t num_threads = 2; diff --git a/cpp/src/scheduler/SchedInst.cpp b/cpp/src/scheduler/SchedInst.cpp index 0b1e7cd302..54b870100a 100644 --- a/cpp/src/scheduler/SchedInst.cpp +++ b/cpp/src/scheduler/SchedInst.cpp @@ -38,10 +38,9 @@ std::mutex JobMgrInst::mutex_; void load_simple_config() { - server::ConfigNode &config = server::ServerConfig::GetInstance().GetConfig(server::CONFIG_RESOURCE); - auto mode = config.GetValue(server::CONFIG_RESOURCE_MODE, server::CONFIG_RESOURCE_MODE_DEFAULT); - - auto pool = config.GetSequence(server::CONFIG_RESOURCE_POOL); + server::ServerConfig &config = server::ServerConfig::GetInstance(); + auto mode = config.GetResourceConfigMode(); + auto pool = config.GetResourceConfigPool(); bool cpu = false; std::set gpu_ids; for (auto &resource : pool) { diff --git a/cpp/src/server/DBWrapper.cpp b/cpp/src/server/DBWrapper.cpp index 756438dfb3..3d958b15a4 100644 --- a/cpp/src/server/DBWrapper.cpp +++ b/cpp/src/server/DBWrapper.cpp @@ -35,23 +35,20 @@ DBWrapper::DBWrapper() { } Status DBWrapper::StartService() { + ServerConfig& config = ServerConfig::GetInstance(); + //db config engine::DBOptions opt; - ConfigNode& db_config = ServerConfig::GetInstance().GetConfig(CONFIG_DB); - opt.meta.backend_uri = db_config.GetValue(CONFIG_DB_BACKEND_URL); - std::string db_path = db_config.GetValue(CONFIG_DB_PATH); - opt.meta.path = db_path + "/db"; + opt.meta.backend_uri = config.GetDBConfigBackendUrl(); + opt.meta.path = config.GetDBConfigPath() + "/db"; - std::string db_slave_path = db_config.GetValue(CONFIG_DB_SLAVE_PATH); + std::string db_slave_path = config.GetDBConfigSlavePath(); StringHelpFunctions::SplitStringByDelimeter(db_slave_path, ";", opt.meta.slave_paths); // cache config - ConfigNode& cache_config = ServerConfig::GetInstance().GetConfig(CONFIG_CACHE); - opt.insert_cache_immediately_ = - cache_config.GetBoolValue(CONFIG_CACHE_INSERT_IMMEDIATELY, std::stoi(CONFIG_CACHE_INSERT_IMMEDIATELY_DEFAULT)); + opt.insert_cache_immediately_ = config.GetCacheConfigCacheInsertData(); - ConfigNode& serverConfig = ServerConfig::GetInstance().GetConfig(CONFIG_SERVER); - std::string mode = serverConfig.GetValue(CONFIG_SERVER_MODE, CONFIG_SERVER_MODE_DEFAULT); + std::string mode = config.GetServerConfigMode(); if (mode == "single") { opt.mode = engine::DBOptions::MODE::SINGLE; } @@ -67,9 +64,7 @@ Status DBWrapper::StartService() { } // engine config - ConfigNode& engine_config = ServerConfig::GetInstance().GetConfig(CONFIG_ENGINE); - int32_t omp_thread = - engine_config.GetInt32Value(CONFIG_ENGINE_OMP_THREAD_NUM, std::stoi(CONFIG_ENGINE_OMP_THREAD_NUM_DEFAULT)); + int32_t omp_thread = config.GetEngineConfigOmpThreadNum(); if(omp_thread > 0) { omp_set_num_threads(omp_thread); SERVER_LOG_DEBUG << "Specify openmp thread number: " << omp_thread; @@ -82,19 +77,16 @@ Status DBWrapper::StartService() { } //init faiss global variable - faiss::distance_compute_blas_threshold = - engine_config.GetInt32Value(CONFIG_ENGINE_BLAS_THRESHOLD, std::stoi(CONFIG_ENGINE_BLAS_THRESHOLD_DEFAULT)); + faiss::distance_compute_blas_threshold = config.GetEngineConfigBlasThreshold(); //set archive config engine::ArchiveConf::CriteriaT criterial; - int64_t disk = - db_config.GetInt64Value(CONFIG_DB_ARCHIVE_DISK_THRESHOLD, std::stoi(CONFIG_DB_ARCHIVE_DISK_THRESHOLD_DEFAULT)); - int64_t days = - db_config.GetInt64Value(CONFIG_DB_ARCHIVE_DAYS_THRESHOLD, std::stoi(CONFIG_DB_ARCHIVE_DAYS_THRESHOLD_DEFAULT)); - if(disk > 0) { + int32_t disk = config.GetDBConfigArchiveDiskThreshold(); + int32_t days = config.GetDBConfigArchiveDaysThreshold(); + if (disk > 0) { criterial[engine::ARCHIVE_CONF_DISK] = disk; } - if(days > 0) { + if (days > 0) { criterial[engine::ARCHIVE_CONF_DAYS] = days; } opt.meta.archive_conf.SetCriterias(criterial); diff --git a/cpp/src/server/Server.cpp b/cpp/src/server/Server.cpp index 7f85ed6509..d8924e9d7d 100644 --- a/cpp/src/server/Server.cpp +++ b/cpp/src/server/Server.cpp @@ -162,9 +162,7 @@ Server::Start() { /* log path is defined in Config file, so InitLog must be called after LoadConfig */ ServerConfig &config = ServerConfig::GetInstance(); - ConfigNode server_config = config.GetConfig(CONFIG_SERVER); - - std::string time_zone = server_config.GetValue(CONFIG_SERVER_TIME_ZONE, CONFIG_SERVER_TIME_ZONE_DEFAULT); + std::string time_zone = config.GetServerConfigTimeZone(); if (time_zone.length() == 3) { time_zone = "CUT"; } else { @@ -232,8 +230,9 @@ Server::Stop() { ErrorCode Server::LoadConfig() { - ServerConfig::GetInstance().LoadConfigFile(config_filename_); - auto status = ServerConfig::GetInstance().ValidateConfig(); + ServerConfig& server_config = ServerConfig::GetInstance(); + server_config.LoadConfigFile(config_filename_); + auto status = server_config.ValidateConfig(); if (!status.ok()) { std::cerr << "Failed to load config file: " << config_filename_ << std::endl; exit(0); diff --git a/cpp/src/server/ServerConfig.cpp b/cpp/src/server/ServerConfig.cpp index 0f51ecf6e2..35012192e8 100644 --- a/cpp/src/server/ServerConfig.cpp +++ b/cpp/src/server/ServerConfig.cpp @@ -42,30 +42,29 @@ ServerConfig::GetInstance() { } Status -ServerConfig::LoadConfigFile(const std::string &config_filename) { - std::string filename = config_filename; +ServerConfig::LoadConfigFile(const std::string &filename) { if (filename.empty()) { - std::cerr << "ERROR: a config file is required" << std::endl; - exit(1);//directly exit program if config file not specified + std::cerr << "ERROR: need specify config file" << std::endl; + exit(1); } - struct stat directoryStat; - int statOK = stat(filename.c_str(), &directoryStat); + struct stat dirStat; + int statOK = stat(filename.c_str(), &dirStat); if (statOK != 0) { - std::cerr << "ERROR: " << filename << " not found!" << std::endl; - exit(1);//directly exit program if config file not found + std::cerr << "ERROR: Config file not exist: " << filename << std::endl; + exit(1); } try { ConfigMgr *mgr = const_cast(ConfigMgr::GetInstance()); ErrorCode err = mgr->LoadConfigFile(filename); if (err != 0) { - std::cerr << "Server failed to load config file" << std::endl; - exit(1);//directly exit program if the config file is illegal + std::cerr << "Server failed to load config file: " << filename << std::endl; + exit(1); } } catch (YAML::Exception &e) { - std::cerr << "Server failed to load config file: " << std::endl; - exit(1);//directly exit program if the config file is illegal + std::cerr << "Server failed to load config file: " << filename << std::endl; + exit(1); } return Status::OK(); @@ -73,27 +72,25 @@ ServerConfig::LoadConfigFile(const std::string &config_filename) { Status ServerConfig::ValidateConfig() { - - bool okay = true; if (!CheckServerConfig().ok()) { - okay = false; + return Status(SERVER_INVALID_ARGUMENT, "Server config validation check fail"); } if (!CheckDBConfig().ok()) { - okay = false; + return Status(SERVER_INVALID_ARGUMENT, "DB config validation check fail"); } if (!CheckMetricConfig().ok()) { - okay = false; + return Status(SERVER_INVALID_ARGUMENT, "Metric config validation check fail"); } if (!CheckCacheConfig().ok()) { - okay = false; + return Status(SERVER_INVALID_ARGUMENT, "Cache config validation check fail"); } if (!CheckEngineConfig().ok()) { - okay = false; + return Status(SERVER_INVALID_ARGUMENT, "Engine config validation check fail"); } if (!CheckResourceConfig().ok()) { - okay = false; + return Status(SERVER_INVALID_ARGUMENT, "Resource config validation check fail"); } - return (okay ? Status::OK() : Status(SERVER_INVALID_ARGUMENT, "Config validation not pass")); + return Status::OK(); } Status @@ -201,18 +198,18 @@ ServerConfig::CheckDBConfig() { okay = false; } - std::string insert_buffer_size_str = db_config.GetValue(CONFIG_DB_BUFFER_SIZE, CONFIG_DB_BUFFER_SIZE_DEFAULT); - if (!ValidationUtil::ValidateStringIsNumber(insert_buffer_size_str).ok()) { - std::cerr << "ERROR: insert_buffer_size " << insert_buffer_size_str << " is not a number" << std::endl; + std::string buffer_size_str = db_config.GetValue(CONFIG_DB_BUFFER_SIZE, CONFIG_DB_BUFFER_SIZE_DEFAULT); + if (!ValidationUtil::ValidateStringIsNumber(buffer_size_str).ok()) { + std::cerr << "ERROR: buffer_size " << buffer_size_str << " is not a number" << std::endl; okay = false; } else { - uint64_t insert_buffer_size = (uint64_t) std::stol(insert_buffer_size_str); - insert_buffer_size *= GB; + uint64_t buffer_size = (uint64_t) std::stol(buffer_size_str); + buffer_size *= GB; unsigned long total_mem = 0, free_mem = 0; CommonUtil::GetSystemMemInfo(total_mem, free_mem); - if (insert_buffer_size >= total_mem) { - std::cerr << "ERROR: insert_buffer_size exceed system memory" << std::endl; + if (buffer_size >= total_mem) { + std::cerr << "ERROR: buffer_size exceed system memory" << std::endl; okay = false; } } @@ -317,7 +314,7 @@ ServerConfig::CheckCacheConfig() { } std::string insert_cache_immediately_str = - cache_config.GetValue(CONFIG_CACHE_INSERT_IMMEDIATELY, CONFIG_CACHE_INSERT_IMMEDIATELY_DEFAULT); + cache_config.GetValue(CONFIG_CACHE_CACHE_INSERT_DATA, CONFIG_CACHE_CACHE_INSERT_DATA_DEFAULT); if (!ValidationUtil::ValidateStringIsBool(insert_cache_immediately_str).ok()) { std::cerr << "ERROR: invalid insert_cache_immediately config: " << insert_cache_immediately_str << std::endl; okay = false; @@ -629,6 +626,171 @@ ServerConfig::GetConfig(const std::string &name) { return root_node.GetChild(name); } +/* server config */ +std::string +ServerConfig::GetServerConfigAddress() { + ConfigNode server_config = GetConfig(CONFIG_SERVER); + return server_config.GetValue(CONFIG_SERVER_ADDRESS, + CONFIG_SERVER_ADDRESS_DEFAULT); +} + +std::string +ServerConfig::GetServerConfigPort() { + ConfigNode server_config = GetConfig(CONFIG_SERVER); + return server_config.GetValue(CONFIG_SERVER_PORT, + CONFIG_SERVER_PORT_DEFAULT); +} + +std::string +ServerConfig::GetServerConfigMode() { + ConfigNode server_config = GetConfig(CONFIG_SERVER); + return server_config.GetValue(CONFIG_SERVER_MODE, + CONFIG_SERVER_MODE_DEFAULT); +} + +std::string +ServerConfig::GetServerConfigTimeZone() { + ConfigNode server_config = GetConfig(CONFIG_SERVER); + return server_config.GetValue(CONFIG_SERVER_TIME_ZONE, + CONFIG_SERVER_TIME_ZONE_DEFAULT); +} + +/* db config */ +std::string +ServerConfig::GetDBConfigPath() { + ConfigNode db_config = GetConfig(CONFIG_DB); + return db_config.GetValue(CONFIG_DB_PATH, + CONFIG_DB_PATH_DEFAULT); +} + +std::string +ServerConfig::GetDBConfigSlavePath() { + ConfigNode db_config = GetConfig(CONFIG_DB); + return db_config.GetValue(CONFIG_DB_SLAVE_PATH, + CONFIG_DB_SLAVE_PATH_DEFAULT); +} + +std::string +ServerConfig::GetDBConfigBackendUrl() { + ConfigNode db_config = GetConfig(CONFIG_DB); + return db_config.GetValue(CONFIG_DB_BACKEND_URL, + CONFIG_DB_BACKEND_URL_DEFAULT); +} + +int32_t +ServerConfig::GetDBConfigArchiveDiskThreshold() { + ConfigNode db_config = GetConfig(CONFIG_DB); + return db_config.GetInt32Value(CONFIG_DB_ARCHIVE_DISK_THRESHOLD, + std::stoi(CONFIG_DB_ARCHIVE_DISK_THRESHOLD_DEFAULT)); +} + +int32_t +ServerConfig::GetDBConfigArchiveDaysThreshold() { + ConfigNode db_config = GetConfig(CONFIG_DB); + return db_config.GetInt32Value(CONFIG_DB_ARCHIVE_DAYS_THRESHOLD, + std::stoi(CONFIG_DB_ARCHIVE_DAYS_THRESHOLD_DEFAULT)); +} + +int32_t +ServerConfig::GetDBConfigBufferSize() { + ConfigNode db_config = GetConfig(CONFIG_DB); + return db_config.GetInt32Value(CONFIG_DB_BUFFER_SIZE, + std::stoi(CONFIG_DB_BUFFER_SIZE_DEFAULT)); +} + +int32_t +ServerConfig::GetDBConfigBuildIndexGPU() { + ConfigNode db_config = GetConfig(CONFIG_DB); + return db_config.GetInt32Value(CONFIG_DB_BUILD_INDEX_GPU, + std::stoi(CONFIG_DB_BUILD_INDEX_GPU_DEFAULT)); +} + +/* metric config */ +bool +ServerConfig::GetMetricConfigAutoBootup() { + ConfigNode metric_config = GetConfig(CONFIG_METRIC); + return metric_config.GetBoolValue(CONFIG_METRIC_AUTO_BOOTUP, + std::stoi(CONFIG_METRIC_AUTO_BOOTUP_DEFAULT)); +} + +std::string +ServerConfig::GetMetricConfigCollector() { + ConfigNode metric_config = GetConfig(CONFIG_METRIC); + return metric_config.GetValue(CONFIG_METRIC_COLLECTOR, + CONFIG_METRIC_COLLECTOR_DEFAULT); +} + +std::string +ServerConfig::GetMetricConfigPrometheusPort() { + ConfigNode metric_config = GetConfig(CONFIG_METRIC); + return metric_config.GetValue(CONFIG_METRIC_PROMETHEUS_PORT, + CONFIG_METRIC_PROMETHEUS_PORT_DEFAULT); +} + +/* cache config */ +int32_t +ServerConfig::GetCacheConfigCpuMemCapacity() { + ConfigNode cache_config = GetConfig(CONFIG_CACHE); + return cache_config.GetInt32Value(CONFIG_CACHE_CPU_MEM_CAPACITY, + std::stoi(CONFIG_CACHE_CPU_MEM_CAPACITY_DEFAULT)); +} + +float +ServerConfig::GetCacheConfigCpuMemThreshold() { + ConfigNode cache_config = GetConfig(CONFIG_CACHE); + return cache_config.GetFloatValue(CONFIG_CACHE_CPU_MEM_THRESHOLD, + std::stof(CONFIG_CACHE_CPU_MEM_THRESHOLD_DEFAULT)); +} + +int32_t +ServerConfig::GetCacheConfigGpuMemCapacity() { + ConfigNode cache_config = GetConfig(CONFIG_CACHE); + return cache_config.GetInt32Value(CONFIG_CACHE_GPU_MEM_CAPACITY, + std::stoi(CONFIG_CACHE_GPU_MEM_CAPACITY_DEFAULT)); +} + +float +ServerConfig::GetCacheConfigGpuMemThreshold() { + ConfigNode cache_config = GetConfig(CONFIG_CACHE); + return cache_config.GetFloatValue(CONFIG_CACHE_GPU_MEM_THRESHOLD, + std::stof(CONFIG_CACHE_GPU_MEM_THRESHOLD_DEFAULT)); +} + +bool +ServerConfig::GetCacheConfigCacheInsertData() { + ConfigNode cache_config = GetConfig(CONFIG_CACHE); + return cache_config.GetBoolValue(CONFIG_CACHE_CACHE_INSERT_DATA, + std::stoi(CONFIG_CACHE_CACHE_INSERT_DATA_DEFAULT)); +} + +/* engine config */ +int32_t +ServerConfig::GetEngineConfigBlasThreshold() { + ConfigNode engine_config = GetConfig(CONFIG_ENGINE); + return engine_config.GetInt32Value(CONFIG_ENGINE_BLAS_THRESHOLD, + std::stoi(CONFIG_ENGINE_BLAS_THRESHOLD_DEFAULT)); +} + +int32_t +ServerConfig::GetEngineConfigOmpThreadNum() { + ConfigNode engine_config = GetConfig(CONFIG_ENGINE); + return engine_config.GetInt32Value(CONFIG_ENGINE_OMP_THREAD_NUM, + std::stoi(CONFIG_ENGINE_OMP_THREAD_NUM_DEFAULT)); +} + +/* resource config */ +std::string +ServerConfig::GetResourceConfigMode() { + ConfigNode resource_config = GetConfig(CONFIG_RESOURCE); + return resource_config.GetValue(CONFIG_RESOURCE_MODE, + CONFIG_RESOURCE_MODE_DEFAULT); +} + +std::vector +ServerConfig::GetResourceConfigPool() { + ConfigNode resource_config = GetConfig(CONFIG_RESOURCE); + return resource_config.GetSequence(CONFIG_RESOURCE_POOL); +} } } diff --git a/cpp/src/server/ServerConfig.h b/cpp/src/server/ServerConfig.h index e515b74645..1453adde49 100644 --- a/cpp/src/server/ServerConfig.h +++ b/cpp/src/server/ServerConfig.h @@ -39,11 +39,11 @@ static const char* CONFIG_SERVER_TIME_ZONE_DEFAULT = "UTC+8"; /* db config */ static const char* CONFIG_DB = "db_config"; -static const char* CONFIG_DB_PATH = "db_path"; +static const char* CONFIG_DB_PATH = "path"; static const char* CONFIG_DB_PATH_DEFAULT = "/tmp/milvus"; -static const char* CONFIG_DB_SLAVE_PATH = "db_slave_path"; +static const char* CONFIG_DB_SLAVE_PATH = "slave_path"; static const char* CONFIG_DB_SLAVE_PATH_DEFAULT = ""; -static const char* CONFIG_DB_BACKEND_URL = "db_backend_url"; +static const char* CONFIG_DB_BACKEND_URL = "backend_url"; static const char* CONFIG_DB_BACKEND_URL_DEFAULT = "sqlite://:@:/"; static const char* CONFIG_DB_ARCHIVE_DISK_THRESHOLD = "archive_disk_threshold"; static const char* CONFIG_DB_ARCHIVE_DISK_THRESHOLD_DEFAULT = "0"; @@ -64,8 +64,8 @@ static const char* CONFIG_CACHE_CPU_MEM_THRESHOLD = "cpu_mem_threshold"; static const char* CONFIG_CACHE_CPU_MEM_THRESHOLD_DEFAULT = "0.85"; static const char* CONFIG_CACHE_GPU_MEM_THRESHOLD = "gpu_mem_threshold"; static const char* CONFIG_CACHE_GPU_MEM_THRESHOLD_DEFAULT = "0.85"; -static const char* CONFIG_CACHE_INSERT_IMMEDIATELY = "insert_immediately"; -static const char* CONFIG_CACHE_INSERT_IMMEDIATELY_DEFAULT = "0"; +static const char* CONFIG_CACHE_CACHE_INSERT_DATA = "cache_insert_data"; +static const char* CONFIG_CACHE_CACHE_INSERT_DATA_DEFAULT = "0"; /* metric config */ static const char* CONFIG_METRIC = "metric_config"; @@ -95,20 +95,51 @@ class ServerConfig { public: static ServerConfig &GetInstance(); - Status LoadConfigFile(const std::string& config_filename); + Status LoadConfigFile(const std::string& filename); Status ValidateConfig(); void PrintAll() const; + private: ConfigNode GetConfig(const std::string& name) const; ConfigNode& GetConfig(const std::string& name); - private: Status CheckServerConfig(); Status CheckDBConfig(); Status CheckMetricConfig(); Status CheckCacheConfig(); Status CheckEngineConfig(); Status CheckResourceConfig(); + + public: + std::string GetServerConfigAddress(); + std::string GetServerConfigPort(); + std::string GetServerConfigMode(); + std::string GetServerConfigTimeZone(); + + std::string GetDBConfigPath(); + std::string GetDBConfigSlavePath(); + std::string GetDBConfigBackendUrl(); + int32_t GetDBConfigArchiveDiskThreshold(); + int32_t GetDBConfigArchiveDaysThreshold(); + int32_t GetDBConfigBufferSize(); + int32_t GetDBConfigBuildIndexGPU(); + + bool GetMetricConfigAutoBootup(); + std::string GetMetricConfigCollector(); + std::string GetMetricConfigPrometheusPort(); + + int32_t GetCacheConfigCpuMemCapacity(); + float GetCacheConfigCpuMemThreshold(); + int32_t GetCacheConfigGpuMemCapacity(); + float GetCacheConfigGpuMemThreshold(); + bool GetCacheConfigCacheInsertData(); + + int32_t GetEngineConfigBlasThreshold(); + int32_t GetEngineConfigOmpThreadNum(); + + std::string GetResourceConfigMode(); + std::vector + GetResourceConfigPool(); }; } diff --git a/cpp/src/server/grpc_impl/GrpcServer.cpp b/cpp/src/server/grpc_impl/GrpcServer.cpp index 5160e0c8ad..e330f5483c 100644 --- a/cpp/src/server/grpc_impl/GrpcServer.cpp +++ b/cpp/src/server/grpc_impl/GrpcServer.cpp @@ -74,12 +74,10 @@ GrpcServer::Stop() { Status GrpcServer::StartService() { ServerConfig &config = ServerConfig::GetInstance(); - ConfigNode server_config = config.GetConfig(CONFIG_SERVER); - ConfigNode engine_config = config.GetConfig(CONFIG_ENGINE); - std::string address = server_config.GetValue(CONFIG_SERVER_ADDRESS, CONFIG_SERVER_ADDRESS_DEFAULT); - int32_t port = server_config.GetInt32Value(CONFIG_SERVER_PORT, std::stoi(CONFIG_SERVER_PORT_DEFAULT)); + std::string address = config.GetServerConfigAddress(); + std::string port = config.GetServerConfigPort(); - std::string server_address(address + ":" + std::to_string(port)); + std::string server_address(address + ":" + port); ::grpc::ServerBuilder builder; builder.SetOption(std::unique_ptr<::grpc::ServerBuilderOption>(new NoReusePortOption)); diff --git a/cpp/src/wrapper/KnowhereResource.cpp b/cpp/src/wrapper/KnowhereResource.cpp index e59b56b574..dfd75df253 100644 --- a/cpp/src/wrapper/KnowhereResource.cpp +++ b/cpp/src/wrapper/KnowhereResource.cpp @@ -38,16 +38,13 @@ ErrorCode KnowhereResource::Initialize() { GpuResourcesArray gpu_resources; //get build index gpu resource - server::ServerConfig& root_config = server::ServerConfig::GetInstance(); - server::ConfigNode& db_config = root_config.GetConfig(server::CONFIG_DB); + server::ServerConfig& config = server::ServerConfig::GetInstance(); - int32_t build_index_gpu = - db_config.GetInt32Value(server::CONFIG_DB_BUILD_INDEX_GPU, std::stoi(server::CONFIG_DB_BUILD_INDEX_GPU_DEFAULT)); + int32_t build_index_gpu = config.GetDBConfigBuildIndexGPU(); gpu_resources.insert(std::make_pair(build_index_gpu, GpuResourceSetting())); //get search gpu resource - server::ConfigNode& res_config = root_config.GetConfig(server::CONFIG_RESOURCE); - auto pool = res_config.GetSequence(server::CONFIG_RESOURCE_POOL); + auto pool = config.GetResourceConfigPool(); std::set gpu_ids; for (auto &resource : pool) { if (resource.length() < 4 || resource.substr(0, 3) != "gpu") { From ebda6cb4ef63ea74ce9a20f6bc80550301ec0a3b Mon Sep 17 00:00:00 2001 From: "yudong.cai" Date: Sat, 21 Sep 2019 16:39:48 +0800 Subject: [PATCH 3/7] MS-574 rename ServerConfig to Config Former-commit-id: 2b0ef94293017c64a2f17616a3089e7f0cdd6b2b --- cpp/src/cache/CpuCacheMgr.cpp | 4 +- cpp/src/cache/GpuCacheMgr.cpp | 4 +- cpp/src/db/engine/ExecutionEngineImpl.cpp | 3 +- cpp/src/metrics/MetricBase.h | 6 -- cpp/src/metrics/Metrics.cpp | 4 +- cpp/src/metrics/PrometheusMetrics.cpp | 5 +- cpp/src/metrics/PrometheusMetrics.h | 19 +---- cpp/src/scheduler/SchedInst.cpp | 6 +- .../server/{ServerConfig.cpp => Config.cpp} | 82 +++++++++---------- cpp/src/server/{ServerConfig.h => Config.h} | 4 +- cpp/src/server/DBWrapper.cpp | 4 +- cpp/src/server/Server.cpp | 23 +++--- cpp/src/server/grpc_impl/GrpcRequestTask.cpp | 6 +- cpp/src/server/grpc_impl/GrpcServer.cpp | 4 +- cpp/src/utils/LogUtil.cpp | 4 +- cpp/src/wrapper/KnowhereResource.cpp | 4 +- cpp/unittest/db/engine_test.cpp | 1 - cpp/unittest/db/search_test.cpp | 7 +- cpp/unittest/main.cpp | 7 +- cpp/unittest/scheduler/schedinst_test.cpp | 4 +- cpp/unittest/scheduler/scheduler_test.cpp | 7 +- cpp/unittest/server/cache_test.cpp | 4 +- cpp/unittest/server/config_test.cpp | 1 - cpp/unittest/server/rpc_test.cpp | 1 - 24 files changed, 91 insertions(+), 123 deletions(-) rename cpp/src/server/{ServerConfig.cpp => Config.cpp} (94%) rename cpp/src/server/{ServerConfig.h => Config.h} (98%) diff --git a/cpp/src/cache/CpuCacheMgr.cpp b/cpp/src/cache/CpuCacheMgr.cpp index 7141394fa1..9a69bf599f 100644 --- a/cpp/src/cache/CpuCacheMgr.cpp +++ b/cpp/src/cache/CpuCacheMgr.cpp @@ -17,7 +17,7 @@ #include "CpuCacheMgr.h" -#include "server/ServerConfig.h" +#include "server/Config.h" #include "utils/Log.h" namespace zilliz { @@ -29,7 +29,7 @@ namespace { } CpuCacheMgr::CpuCacheMgr() { - server::ServerConfig& config = server::ServerConfig::GetInstance(); + server::Config& config = server::Config::GetInstance(); int64_t cap = config.GetCacheConfigCpuMemCapacity() * unit; cache_ = std::make_shared>(cap, 1UL<<32); diff --git a/cpp/src/cache/GpuCacheMgr.cpp b/cpp/src/cache/GpuCacheMgr.cpp index ba5665f5f2..bc6c3e13db 100644 --- a/cpp/src/cache/GpuCacheMgr.cpp +++ b/cpp/src/cache/GpuCacheMgr.cpp @@ -19,7 +19,7 @@ #include #include "utils/Log.h" #include "GpuCacheMgr.h" -#include "server/ServerConfig.h" +#include "server/Config.h" namespace zilliz { namespace milvus { @@ -33,7 +33,7 @@ namespace { } GpuCacheMgr::GpuCacheMgr() { - server::ServerConfig& config = server::ServerConfig::GetInstance(); + server::Config& config = server::Config::GetInstance(); int32_t cap = config.GetCacheConfigGpuMemCapacity() * G_BYTE; cache_ = std::make_shared>(cap, 1UL<<32); diff --git a/cpp/src/db/engine/ExecutionEngineImpl.cpp b/cpp/src/db/engine/ExecutionEngineImpl.cpp index 960f3baa89..742d21a0be 100644 --- a/cpp/src/db/engine/ExecutionEngineImpl.cpp +++ b/cpp/src/db/engine/ExecutionEngineImpl.cpp @@ -26,6 +26,7 @@ #include "src/wrapper/vec_index.h" #include "src/wrapper/vec_impl.h" #include "knowhere/common/Exception.h" +#include "server/Config.h" #include @@ -334,7 +335,7 @@ Status ExecutionEngineImpl::GpuCache(uint64_t gpu_id) { // TODO(linxj): remove. Status ExecutionEngineImpl::Init() { using namespace zilliz::milvus::server; - ServerConfig &config = ServerConfig::GetInstance(); + server::Config &config = server::Config::GetInstance(); gpu_num_ = config.GetDBConfigBuildIndexGPU(); return Status::OK(); diff --git a/cpp/src/metrics/MetricBase.h b/cpp/src/metrics/MetricBase.h index aa181fb3f3..8a3f11d7df 100644 --- a/cpp/src/metrics/MetricBase.h +++ b/cpp/src/metrics/MetricBase.h @@ -19,7 +19,6 @@ #pragma once #include "utils/Error.h" -#include "server/ServerConfig.h" #include "SystemInfo.h" namespace zilliz { @@ -83,11 +82,6 @@ class MetricsBase{ virtual void CPUTemperature() {}; }; - - - - - } } } \ No newline at end of file diff --git a/cpp/src/metrics/Metrics.cpp b/cpp/src/metrics/Metrics.cpp index 59eaea6355..6c6df56b17 100644 --- a/cpp/src/metrics/Metrics.cpp +++ b/cpp/src/metrics/Metrics.cpp @@ -16,7 +16,7 @@ // under the License. #include "Metrics.h" -#include "server/ServerConfig.h" +#include "server/Config.h" #include "PrometheusMetrics.h" @@ -32,7 +32,7 @@ Metrics::GetInstance() { MetricsBase & Metrics::CreateMetricsCollector() { - ServerConfig &config = ServerConfig::GetInstance(); + Config &config = Config::GetInstance(); std::string collector_type_str = config.GetMetricConfigCollector(); if (collector_type_str == "prometheus") { diff --git a/cpp/src/metrics/PrometheusMetrics.cpp b/cpp/src/metrics/PrometheusMetrics.cpp index b07e91ea32..14d2de6732 100644 --- a/cpp/src/metrics/PrometheusMetrics.cpp +++ b/cpp/src/metrics/PrometheusMetrics.cpp @@ -16,8 +16,9 @@ // under the License. -#include +#include "cache/GpuCacheMgr.h" #include "PrometheusMetrics.h" +#include "server/Config.h" #include "utils/Log.h" #include "SystemInfo.h" @@ -29,7 +30,7 @@ namespace server { ErrorCode PrometheusMetrics::Init() { try { - ServerConfig &config = ServerConfig::GetInstance(); + Config &config = Config::GetInstance(); startup_ = config.GetMetricConfigAutoBootup(); if (!startup_) return SERVER_SUCCESS; diff --git a/cpp/src/metrics/PrometheusMetrics.h b/cpp/src/metrics/PrometheusMetrics.h index 5a3379305a..4ee6699b77 100644 --- a/cpp/src/metrics/PrometheusMetrics.h +++ b/cpp/src/metrics/PrometheusMetrics.h @@ -17,15 +17,13 @@ #pragma once -#include "utils/Error.h" #include #include - - #include #include #include -#include "server/ServerConfig.h" + +#include "utils/Error.h" #include "MetricBase.h" @@ -38,10 +36,6 @@ namespace zilliz { namespace milvus { namespace server { - - - - class PrometheusMetrics: public MetricsBase { public: @@ -107,11 +101,6 @@ class PrometheusMetrics: public MetricsBase { void GPUTemperature() override; void CPUTemperature() override; - - - - - std::shared_ptr &exposer_ptr() {return exposer_ptr_; } // prometheus::Exposer& exposer() { return exposer_;} std::shared_ptr ®istry_ptr() {return registry_; } @@ -125,8 +114,6 @@ class PrometheusMetrics: public MetricsBase { // .Register(*registry_); // prometheus::Counter &connection_total_ = connect_request_.Add({}); - - ////all from DBImpl.cpp using BucketBoundaries = std::vector; //record add_group request @@ -472,10 +459,8 @@ class PrometheusMetrics: public MetricsBase { .Name("CPU_temperature") .Help("CPU temperature") .Register(*registry_); - }; - } } } diff --git a/cpp/src/scheduler/SchedInst.cpp b/cpp/src/scheduler/SchedInst.cpp index 54b870100a..5c4ec522da 100644 --- a/cpp/src/scheduler/SchedInst.cpp +++ b/cpp/src/scheduler/SchedInst.cpp @@ -17,7 +17,7 @@ #include "SchedInst.h" -#include "server/ServerConfig.h" +#include "server/Config.h" #include "ResourceFactory.h" #include "knowhere/index/vector_index/IndexGPUIVF.h" #include "Utils.h" @@ -38,7 +38,7 @@ std::mutex JobMgrInst::mutex_; void load_simple_config() { - server::ServerConfig &config = server::ServerConfig::GetInstance(); + server::Config &config = server::Config::GetInstance(); auto mode = config.GetResourceConfigMode(); auto pool = config.GetResourceConfigPool(); bool cpu = false; @@ -81,7 +81,7 @@ load_simple_config() { void load_advance_config() { // try { -// server::ConfigNode &config = server::ServerConfig::GetInstance().GetConfig(server::CONFIG_RESOURCE); +// server::ConfigNode &config = server::Config::GetInstance().GetConfig(server::CONFIG_RESOURCE); // // if (config.GetChildren().empty()) throw "resource_config null exception"; // diff --git a/cpp/src/server/ServerConfig.cpp b/cpp/src/server/Config.cpp similarity index 94% rename from cpp/src/server/ServerConfig.cpp rename to cpp/src/server/Config.cpp index 35012192e8..9f4f456452 100644 --- a/cpp/src/server/ServerConfig.cpp +++ b/cpp/src/server/Config.cpp @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -#include "ServerConfig.h" +#include "Config.h" #include #include @@ -35,14 +35,14 @@ namespace server { constexpr uint64_t MB = 1024 * 1024; constexpr uint64_t GB = MB * 1024; -ServerConfig & -ServerConfig::GetInstance() { - static ServerConfig config; - return config; +Config & +Config::GetInstance() { + static Config config_inst; + return config_inst; } Status -ServerConfig::LoadConfigFile(const std::string &filename) { +Config::LoadConfigFile(const std::string &filename) { if (filename.empty()) { std::cerr << "ERROR: need specify config file" << std::endl; exit(1); @@ -71,7 +71,7 @@ ServerConfig::LoadConfigFile(const std::string &filename) { } Status -ServerConfig::ValidateConfig() { +Config::ValidateConfig() { if (!CheckServerConfig().ok()) { return Status(SERVER_INVALID_ARGUMENT, "Server config validation check fail"); } @@ -94,7 +94,7 @@ ServerConfig::ValidateConfig() { } Status -ServerConfig::CheckServerConfig() { +Config::CheckServerConfig() { /* server_config: address: 0.0.0.0 # milvus server ip address (IPv4) @@ -154,7 +154,7 @@ ServerConfig::CheckServerConfig() { } Status -ServerConfig::CheckDBConfig() { +Config::CheckDBConfig() { /* db_config: db_path: @MILVUS_DB_PATH@ # milvus data storage path @@ -230,7 +230,7 @@ ServerConfig::CheckDBConfig() { } Status -ServerConfig::CheckMetricConfig() { +Config::CheckMetricConfig() { /* metric_config: is_startup: off # if monitoring start: on, off @@ -260,7 +260,7 @@ ServerConfig::CheckMetricConfig() { } Status -ServerConfig::CheckCacheConfig() { +Config::CheckCacheConfig() { /* cache_config: cpu_cache_capacity: 16 # how many memory are used as cache, unit: GB, range: 0 ~ less than total memory @@ -361,7 +361,7 @@ ServerConfig::CheckCacheConfig() { } Status -ServerConfig::CheckEngineConfig() { +Config::CheckEngineConfig() { /* engine_config: use_blas_threshold: 20 @@ -396,7 +396,7 @@ ServerConfig::CheckEngineConfig() { } Status -ServerConfig::CheckResourceConfig() { +Config::CheckResourceConfig() { /* resource_config: mode: simple @@ -406,7 +406,7 @@ ServerConfig::CheckResourceConfig() { - gpu100 */ bool okay = true; - server::ConfigNode &config = server::ServerConfig::GetInstance().GetConfig(server::CONFIG_RESOURCE); + server::ConfigNode &config = GetConfig(server::CONFIG_RESOURCE); auto mode = config.GetValue(CONFIG_RESOURCE_MODE, CONFIG_RESOURCE_MODE_DEFAULT); if (mode != "simple") { std::cerr << "ERROR: invalid resource config: mode is " << mode << std::endl; @@ -422,7 +422,7 @@ ServerConfig::CheckResourceConfig() { } //Status -//ServerConfig::CheckResourceConfig() { +//Config::CheckResourceConfig() { /* resource_config: # resource list, length: 0~N @@ -604,7 +604,7 @@ ServerConfig::CheckResourceConfig() { //} void -ServerConfig::PrintAll() const { +Config::PrintAll() const { if (const ConfigMgr *mgr = ConfigMgr::GetInstance()) { std::string str = mgr->DumpString(); // SERVER_LOG_INFO << "\n" << str; @@ -613,14 +613,14 @@ ServerConfig::PrintAll() const { } ConfigNode -ServerConfig::GetConfig(const std::string &name) const { +Config::GetConfig(const std::string &name) const { const ConfigMgr *mgr = ConfigMgr::GetInstance(); const ConfigNode &root_node = mgr->GetRootNode(); return root_node.GetChild(name); } ConfigNode & -ServerConfig::GetConfig(const std::string &name) { +Config::GetConfig(const std::string &name) { ConfigMgr *mgr = ConfigMgr::GetInstance(); ConfigNode &root_node = mgr->GetRootNode(); return root_node.GetChild(name); @@ -628,28 +628,28 @@ ServerConfig::GetConfig(const std::string &name) { /* server config */ std::string -ServerConfig::GetServerConfigAddress() { +Config::GetServerConfigAddress() { ConfigNode server_config = GetConfig(CONFIG_SERVER); return server_config.GetValue(CONFIG_SERVER_ADDRESS, CONFIG_SERVER_ADDRESS_DEFAULT); } std::string -ServerConfig::GetServerConfigPort() { +Config::GetServerConfigPort() { ConfigNode server_config = GetConfig(CONFIG_SERVER); return server_config.GetValue(CONFIG_SERVER_PORT, CONFIG_SERVER_PORT_DEFAULT); } std::string -ServerConfig::GetServerConfigMode() { +Config::GetServerConfigMode() { ConfigNode server_config = GetConfig(CONFIG_SERVER); return server_config.GetValue(CONFIG_SERVER_MODE, CONFIG_SERVER_MODE_DEFAULT); } std::string -ServerConfig::GetServerConfigTimeZone() { +Config::GetServerConfigTimeZone() { ConfigNode server_config = GetConfig(CONFIG_SERVER); return server_config.GetValue(CONFIG_SERVER_TIME_ZONE, CONFIG_SERVER_TIME_ZONE_DEFAULT); @@ -657,49 +657,49 @@ ServerConfig::GetServerConfigTimeZone() { /* db config */ std::string -ServerConfig::GetDBConfigPath() { +Config::GetDBConfigPath() { ConfigNode db_config = GetConfig(CONFIG_DB); return db_config.GetValue(CONFIG_DB_PATH, CONFIG_DB_PATH_DEFAULT); } std::string -ServerConfig::GetDBConfigSlavePath() { +Config::GetDBConfigSlavePath() { ConfigNode db_config = GetConfig(CONFIG_DB); return db_config.GetValue(CONFIG_DB_SLAVE_PATH, CONFIG_DB_SLAVE_PATH_DEFAULT); } std::string -ServerConfig::GetDBConfigBackendUrl() { +Config::GetDBConfigBackendUrl() { ConfigNode db_config = GetConfig(CONFIG_DB); return db_config.GetValue(CONFIG_DB_BACKEND_URL, CONFIG_DB_BACKEND_URL_DEFAULT); } int32_t -ServerConfig::GetDBConfigArchiveDiskThreshold() { +Config::GetDBConfigArchiveDiskThreshold() { ConfigNode db_config = GetConfig(CONFIG_DB); return db_config.GetInt32Value(CONFIG_DB_ARCHIVE_DISK_THRESHOLD, std::stoi(CONFIG_DB_ARCHIVE_DISK_THRESHOLD_DEFAULT)); } int32_t -ServerConfig::GetDBConfigArchiveDaysThreshold() { +Config::GetDBConfigArchiveDaysThreshold() { ConfigNode db_config = GetConfig(CONFIG_DB); return db_config.GetInt32Value(CONFIG_DB_ARCHIVE_DAYS_THRESHOLD, std::stoi(CONFIG_DB_ARCHIVE_DAYS_THRESHOLD_DEFAULT)); } int32_t -ServerConfig::GetDBConfigBufferSize() { +Config::GetDBConfigBufferSize() { ConfigNode db_config = GetConfig(CONFIG_DB); return db_config.GetInt32Value(CONFIG_DB_BUFFER_SIZE, std::stoi(CONFIG_DB_BUFFER_SIZE_DEFAULT)); } int32_t -ServerConfig::GetDBConfigBuildIndexGPU() { +Config::GetDBConfigBuildIndexGPU() { ConfigNode db_config = GetConfig(CONFIG_DB); return db_config.GetInt32Value(CONFIG_DB_BUILD_INDEX_GPU, std::stoi(CONFIG_DB_BUILD_INDEX_GPU_DEFAULT)); @@ -707,21 +707,21 @@ ServerConfig::GetDBConfigBuildIndexGPU() { /* metric config */ bool -ServerConfig::GetMetricConfigAutoBootup() { +Config::GetMetricConfigAutoBootup() { ConfigNode metric_config = GetConfig(CONFIG_METRIC); return metric_config.GetBoolValue(CONFIG_METRIC_AUTO_BOOTUP, std::stoi(CONFIG_METRIC_AUTO_BOOTUP_DEFAULT)); } std::string -ServerConfig::GetMetricConfigCollector() { +Config::GetMetricConfigCollector() { ConfigNode metric_config = GetConfig(CONFIG_METRIC); return metric_config.GetValue(CONFIG_METRIC_COLLECTOR, CONFIG_METRIC_COLLECTOR_DEFAULT); } std::string -ServerConfig::GetMetricConfigPrometheusPort() { +Config::GetMetricConfigPrometheusPort() { ConfigNode metric_config = GetConfig(CONFIG_METRIC); return metric_config.GetValue(CONFIG_METRIC_PROMETHEUS_PORT, CONFIG_METRIC_PROMETHEUS_PORT_DEFAULT); @@ -729,35 +729,35 @@ ServerConfig::GetMetricConfigPrometheusPort() { /* cache config */ int32_t -ServerConfig::GetCacheConfigCpuMemCapacity() { +Config::GetCacheConfigCpuMemCapacity() { ConfigNode cache_config = GetConfig(CONFIG_CACHE); return cache_config.GetInt32Value(CONFIG_CACHE_CPU_MEM_CAPACITY, std::stoi(CONFIG_CACHE_CPU_MEM_CAPACITY_DEFAULT)); } float -ServerConfig::GetCacheConfigCpuMemThreshold() { +Config::GetCacheConfigCpuMemThreshold() { ConfigNode cache_config = GetConfig(CONFIG_CACHE); return cache_config.GetFloatValue(CONFIG_CACHE_CPU_MEM_THRESHOLD, std::stof(CONFIG_CACHE_CPU_MEM_THRESHOLD_DEFAULT)); } int32_t -ServerConfig::GetCacheConfigGpuMemCapacity() { +Config::GetCacheConfigGpuMemCapacity() { ConfigNode cache_config = GetConfig(CONFIG_CACHE); return cache_config.GetInt32Value(CONFIG_CACHE_GPU_MEM_CAPACITY, std::stoi(CONFIG_CACHE_GPU_MEM_CAPACITY_DEFAULT)); } float -ServerConfig::GetCacheConfigGpuMemThreshold() { +Config::GetCacheConfigGpuMemThreshold() { ConfigNode cache_config = GetConfig(CONFIG_CACHE); return cache_config.GetFloatValue(CONFIG_CACHE_GPU_MEM_THRESHOLD, std::stof(CONFIG_CACHE_GPU_MEM_THRESHOLD_DEFAULT)); } bool -ServerConfig::GetCacheConfigCacheInsertData() { +Config::GetCacheConfigCacheInsertData() { ConfigNode cache_config = GetConfig(CONFIG_CACHE); return cache_config.GetBoolValue(CONFIG_CACHE_CACHE_INSERT_DATA, std::stoi(CONFIG_CACHE_CACHE_INSERT_DATA_DEFAULT)); @@ -765,14 +765,14 @@ ServerConfig::GetCacheConfigCacheInsertData() { /* engine config */ int32_t -ServerConfig::GetEngineConfigBlasThreshold() { +Config::GetEngineConfigBlasThreshold() { ConfigNode engine_config = GetConfig(CONFIG_ENGINE); return engine_config.GetInt32Value(CONFIG_ENGINE_BLAS_THRESHOLD, std::stoi(CONFIG_ENGINE_BLAS_THRESHOLD_DEFAULT)); } int32_t -ServerConfig::GetEngineConfigOmpThreadNum() { +Config::GetEngineConfigOmpThreadNum() { ConfigNode engine_config = GetConfig(CONFIG_ENGINE); return engine_config.GetInt32Value(CONFIG_ENGINE_OMP_THREAD_NUM, std::stoi(CONFIG_ENGINE_OMP_THREAD_NUM_DEFAULT)); @@ -780,14 +780,14 @@ ServerConfig::GetEngineConfigOmpThreadNum() { /* resource config */ std::string -ServerConfig::GetResourceConfigMode() { +Config::GetResourceConfigMode() { ConfigNode resource_config = GetConfig(CONFIG_RESOURCE); return resource_config.GetValue(CONFIG_RESOURCE_MODE, CONFIG_RESOURCE_MODE_DEFAULT); } std::vector -ServerConfig::GetResourceConfigPool() { +Config::GetResourceConfigPool() { ConfigNode resource_config = GetConfig(CONFIG_RESOURCE); return resource_config.GetSequence(CONFIG_RESOURCE_POOL); } diff --git a/cpp/src/server/ServerConfig.h b/cpp/src/server/Config.h similarity index 98% rename from cpp/src/server/ServerConfig.h rename to cpp/src/server/Config.h index 1453adde49..2ad4c382cc 100644 --- a/cpp/src/server/ServerConfig.h +++ b/cpp/src/server/Config.h @@ -91,9 +91,9 @@ static const char* CONFIG_RESOURCE_MODE_DEFAULT = "simple"; static const char* CONFIG_RESOURCE_POOL = "pool"; -class ServerConfig { +class Config { public: - static ServerConfig &GetInstance(); + static Config &GetInstance(); Status LoadConfigFile(const std::string& filename); Status ValidateConfig(); diff --git a/cpp/src/server/DBWrapper.cpp b/cpp/src/server/DBWrapper.cpp index 3d958b15a4..bc0b080625 100644 --- a/cpp/src/server/DBWrapper.cpp +++ b/cpp/src/server/DBWrapper.cpp @@ -17,7 +17,7 @@ #include "DBWrapper.h" -#include "ServerConfig.h" +#include "Config.h" #include "db/DBFactory.h" #include "utils/CommonUtil.h" #include "utils/Log.h" @@ -35,7 +35,7 @@ DBWrapper::DBWrapper() { } Status DBWrapper::StartService() { - ServerConfig& config = ServerConfig::GetInstance(); + Config& config = Config::GetInstance(); //db config engine::DBOptions opt; diff --git a/cpp/src/server/Server.cpp b/cpp/src/server/Server.cpp index d8924e9d7d..7ca0b2a7c9 100644 --- a/cpp/src/server/Server.cpp +++ b/cpp/src/server/Server.cpp @@ -16,14 +16,6 @@ // under the License. #include -#include "Server.h" -#include "server/grpc_impl/GrpcServer.h" -#include "utils/Log.h" -#include "utils/LogUtil.h" -#include "utils/SignalUtil.h" -#include "utils/TimeRecorder.h" -#include "metrics/Metrics.h" - #include #include #include @@ -31,10 +23,17 @@ //#include #include #include -#include -#include "src/wrapper/KnowhereResource.h" +#include "Server.h" +#include "server/grpc_impl/GrpcServer.h" +#include "server/Config.h" +#include "utils/Log.h" +#include "utils/LogUtil.h" +#include "utils/SignalUtil.h" +#include "utils/TimeRecorder.h" #include "metrics/Metrics.h" +#include "scheduler/SchedInst.h" +#include "wrapper/KnowhereResource.h" #include "DBWrapper.h" @@ -161,7 +160,7 @@ Server::Start() { } /* log path is defined in Config file, so InitLog must be called after LoadConfig */ - ServerConfig &config = ServerConfig::GetInstance(); + Config &config = Config::GetInstance(); std::string time_zone = config.GetServerConfigTimeZone(); if (time_zone.length() == 3) { time_zone = "CUT"; @@ -230,7 +229,7 @@ Server::Stop() { ErrorCode Server::LoadConfig() { - ServerConfig& server_config = ServerConfig::GetInstance(); + Config& server_config = Config::GetInstance(); server_config.LoadConfigFile(config_filename_); auto status = server_config.ValidateConfig(); if (!status.ok()) { diff --git a/cpp/src/server/grpc_impl/GrpcRequestTask.cpp b/cpp/src/server/grpc_impl/GrpcRequestTask.cpp index ad0abd713d..e699dd04b7 100644 --- a/cpp/src/server/grpc_impl/GrpcRequestTask.cpp +++ b/cpp/src/server/grpc_impl/GrpcRequestTask.cpp @@ -15,8 +15,9 @@ // specific language governing permissions and limitations // under the License. +#include + #include "GrpcRequestTask.h" -#include "../ServerConfig.h" #include "utils/CommonUtil.h" #include "utils/Log.h" #include "utils/TimeRecorder.h" @@ -28,9 +29,8 @@ #include "scheduler/SchedInst.h" //#include -#include "src/server/Server.h" +#include "server/Server.h" -#include namespace zilliz { namespace milvus { diff --git a/cpp/src/server/grpc_impl/GrpcServer.cpp b/cpp/src/server/grpc_impl/GrpcServer.cpp index e330f5483c..c10d7b5379 100644 --- a/cpp/src/server/grpc_impl/GrpcServer.cpp +++ b/cpp/src/server/grpc_impl/GrpcServer.cpp @@ -17,7 +17,7 @@ #include "milvus.grpc.pb.h" #include "GrpcServer.h" -#include "server/ServerConfig.h" +#include "server/Config.h" #include "server/DBWrapper.h" #include "utils/Log.h" #include "GrpcRequestHandler.h" @@ -73,7 +73,7 @@ GrpcServer::Stop() { Status GrpcServer::StartService() { - ServerConfig &config = ServerConfig::GetInstance(); + Config &config = Config::GetInstance(); std::string address = config.GetServerConfigAddress(); std::string port = config.GetServerConfigPort(); diff --git a/cpp/src/utils/LogUtil.cpp b/cpp/src/utils/LogUtil.cpp index cfdbbdc768..3a3fe0c7e2 100644 --- a/cpp/src/utils/LogUtil.cpp +++ b/cpp/src/utils/LogUtil.cpp @@ -15,13 +15,11 @@ // specific language governing permissions and limitations // under the License. -#include "LogUtil.h" -#include "server/ServerConfig.h" - #include #include #include +#include "LogUtil.h" namespace zilliz { namespace milvus { diff --git a/cpp/src/wrapper/KnowhereResource.cpp b/cpp/src/wrapper/KnowhereResource.cpp index dfd75df253..1e4248f6e1 100644 --- a/cpp/src/wrapper/KnowhereResource.cpp +++ b/cpp/src/wrapper/KnowhereResource.cpp @@ -18,7 +18,7 @@ #include "KnowhereResource.h" #include "knowhere/index/vector_index/helpers/FaissGpuResourceMgr.h" -#include "server/ServerConfig.h" +#include "server/Config.h" #include @@ -38,7 +38,7 @@ ErrorCode KnowhereResource::Initialize() { GpuResourcesArray gpu_resources; //get build index gpu resource - server::ServerConfig& config = server::ServerConfig::GetInstance(); + server::Config& config = server::Config::GetInstance(); int32_t build_index_gpu = config.GetDBConfigBuildIndexGPU(); gpu_resources.insert(std::make_pair(build_index_gpu, GpuResourceSetting())); diff --git a/cpp/unittest/db/engine_test.cpp b/cpp/unittest/db/engine_test.cpp index e06e9038c8..f55d8d9e27 100644 --- a/cpp/unittest/db/engine_test.cpp +++ b/cpp/unittest/db/engine_test.cpp @@ -21,7 +21,6 @@ #include "db/engine/EngineFactory.h" #include "db/engine/ExecutionEngineImpl.h" -#include "server/ServerConfig.h" #include "utils.h" using namespace zilliz::milvus; diff --git a/cpp/unittest/db/search_test.cpp b/cpp/unittest/db/search_test.cpp index 295650f2d2..d492256534 100644 --- a/cpp/unittest/db/search_test.cpp +++ b/cpp/unittest/db/search_test.cpp @@ -15,13 +15,12 @@ // specific language governing permissions and limitations // under the License. -#include "server/ServerConfig.h" -#include "utils/TimeRecorder.h" - #include #include #include -#include + +#include "scheduler/task/SearchTask.h" +#include "utils/TimeRecorder.h" using namespace zilliz::milvus; diff --git a/cpp/unittest/main.cpp b/cpp/unittest/main.cpp index 33a860945d..7a28bbf45f 100644 --- a/cpp/unittest/main.cpp +++ b/cpp/unittest/main.cpp @@ -15,13 +15,12 @@ // specific language governing permissions and limitations // under the License. -#include "utils/easylogging++.h" -#include "server/ServerConfig.h" -#include "utils/CommonUtil.h" - #include #include +#include "utils/easylogging++.h" +#include "utils/CommonUtil.h" + INITIALIZE_EASYLOGGINGPP using namespace zilliz::milvus; diff --git a/cpp/unittest/scheduler/schedinst_test.cpp b/cpp/unittest/scheduler/schedinst_test.cpp index 27b6a17bbb..cb56264525 100644 --- a/cpp/unittest/scheduler/schedinst_test.cpp +++ b/cpp/unittest/scheduler/schedinst_test.cpp @@ -15,12 +15,10 @@ // specific language governing permissions and limitations // under the License. - -#include "scheduler/SchedInst.h" -#include "server/ServerConfig.h" #include #include +#include "scheduler/SchedInst.h" namespace zilliz { namespace milvus { diff --git a/cpp/unittest/scheduler/scheduler_test.cpp b/cpp/unittest/scheduler/scheduler_test.cpp index 1cfbda1e14..b8899ebdc2 100644 --- a/cpp/unittest/scheduler/scheduler_test.cpp +++ b/cpp/unittest/scheduler/scheduler_test.cpp @@ -15,17 +15,16 @@ // specific language governing permissions and limitations // under the License. -#include "scheduler/Scheduler.h" #include -#include -#include +#include "scheduler/tasklabel/DefaultLabel.h" #include "cache/DataObj.h" #include "cache/GpuCacheMgr.h" +#include "scheduler/Scheduler.h" #include "scheduler/task/TestTask.h" #include "scheduler/ResourceFactory.h" #include "scheduler/resource/Resource.h" #include "utils/Error.h" -#include "src/wrapper/vec_index.h" +#include "wrapper/vec_index.h" #include "scheduler/tasklabel/SpecResLabel.h" diff --git a/cpp/unittest/server/cache_test.cpp b/cpp/unittest/server/cache_test.cpp index bf9b6edf7d..94eae5dbf0 100644 --- a/cpp/unittest/server/cache_test.cpp +++ b/cpp/unittest/server/cache_test.cpp @@ -18,10 +18,8 @@ #include #include "cache/CpuCacheMgr.h" #include "cache/GpuCacheMgr.h" -#include "server/ServerConfig.h" - #include "utils/Error.h" -#include "src/wrapper/vec_index.h" +#include "wrapper/vec_index.h" using namespace zilliz::milvus; diff --git a/cpp/unittest/server/config_test.cpp b/cpp/unittest/server/config_test.cpp index 13089e914b..65ce562722 100644 --- a/cpp/unittest/server/config_test.cpp +++ b/cpp/unittest/server/config_test.cpp @@ -19,7 +19,6 @@ #include #include "config/ConfigMgr.h" -#include "server/ServerConfig.h" #include "utils/CommonUtil.h" #include "utils/ValidationUtil.h" diff --git a/cpp/unittest/server/rpc_test.cpp b/cpp/unittest/server/rpc_test.cpp index 5ad6005a36..c196c082e5 100644 --- a/cpp/unittest/server/rpc_test.cpp +++ b/cpp/unittest/server/rpc_test.cpp @@ -29,7 +29,6 @@ #include "grpc/gen-status/status.pb.h" #include "server/DBWrapper.h" -#include "server/ServerConfig.h" #include "scheduler/SchedInst.h" #include "scheduler/ResourceFactory.h" #include "utils/CommonUtil.h" From b0fe84cd816fc2548ad776f875c485d61ef66cc6 Mon Sep 17 00:00:00 2001 From: "yudong.cai" Date: Sat, 21 Sep 2019 18:14:29 +0800 Subject: [PATCH 4/7] MS-574 save config in map Former-commit-id: 961dc6749c1b262e489a825cce81586bf27f2c18 --- cpp/src/server/Config.cpp | 255 ++++++++++++++++++++++++++++---------- cpp/src/server/Config.h | 20 ++- 2 files changed, 204 insertions(+), 71 deletions(-) diff --git a/cpp/src/server/Config.cpp b/cpp/src/server/Config.cpp index 9f4f456452..9f7b9fd12e 100644 --- a/cpp/src/server/Config.cpp +++ b/cpp/src/server/Config.cpp @@ -22,6 +22,7 @@ #include #include #include +#include #include "config/ConfigMgr.h" #include "utils/CommonUtil.h" @@ -32,8 +33,8 @@ namespace zilliz { namespace milvus { namespace server { -constexpr uint64_t MB = 1024 * 1024; -constexpr uint64_t GB = MB * 1024; +constexpr uint64_t MB = 1UL << 20; +constexpr uint64_t GB = 1UL << 30; Config & Config::GetInstance() { @@ -626,164 +627,282 @@ Config::GetConfig(const std::string &name) { return root_node.GetChild(name); } +Status +Config::GetConfigValueInMem(const std::string &parent_key, + const std::string &child_key, + std::string &value) { + if (config_map_.find(parent_key) != config_map_.end() && + config_map_[parent_key].find(child_key) != config_map_[parent_key].end()) { + std::lock_guard lock(mutex_); + value = config_map_[parent_key][child_key]; + return Status::OK(); + } else { + return Status(SERVER_UNEXPECTED_ERROR, "key not exist"); + } +} + +void +Config::SetConfigValueInMem(const std::string &parent_key, + const std::string &child_key, + std::string &value) { + std::lock_guard lock(mutex_); + config_map_[parent_key][child_key] = value; +} + +//////////////////////////////////////////////////////////////////////////////// /* server config */ std::string Config::GetServerConfigAddress() { - ConfigNode server_config = GetConfig(CONFIG_SERVER); - return server_config.GetValue(CONFIG_SERVER_ADDRESS, - CONFIG_SERVER_ADDRESS_DEFAULT); + std::string value; + if (!GetConfigValueInMem(CONFIG_SERVER, CONFIG_SERVER_ADDRESS, value).ok()) { + value = GetConfig(CONFIG_SERVER).GetValue(CONFIG_SERVER_ADDRESS, + CONFIG_SERVER_ADDRESS_DEFAULT); + SetConfigValueInMem(CONFIG_SERVER, CONFIG_SERVER_ADDRESS, value); + } + return value; } std::string Config::GetServerConfigPort() { - ConfigNode server_config = GetConfig(CONFIG_SERVER); - return server_config.GetValue(CONFIG_SERVER_PORT, - CONFIG_SERVER_PORT_DEFAULT); + std::string value; + if (!GetConfigValueInMem(CONFIG_SERVER, CONFIG_SERVER_PORT, value).ok()) { + value = GetConfig(CONFIG_SERVER).GetValue(CONFIG_SERVER_PORT, + CONFIG_SERVER_PORT_DEFAULT); + SetConfigValueInMem(CONFIG_SERVER, CONFIG_SERVER_PORT, value); + } + return value; } std::string Config::GetServerConfigMode() { - ConfigNode server_config = GetConfig(CONFIG_SERVER); - return server_config.GetValue(CONFIG_SERVER_MODE, - CONFIG_SERVER_MODE_DEFAULT); + std::string value; + if (!GetConfigValueInMem(CONFIG_SERVER, CONFIG_SERVER_MODE, value).ok()) { + value = GetConfig(CONFIG_SERVER).GetValue(CONFIG_SERVER_MODE, + CONFIG_SERVER_MODE_DEFAULT); + SetConfigValueInMem(CONFIG_SERVER, CONFIG_SERVER_MODE, value); + } + return value; } std::string Config::GetServerConfigTimeZone() { - ConfigNode server_config = GetConfig(CONFIG_SERVER); - return server_config.GetValue(CONFIG_SERVER_TIME_ZONE, - CONFIG_SERVER_TIME_ZONE_DEFAULT); + std::string value; + if (!GetConfigValueInMem(CONFIG_SERVER, CONFIG_SERVER_TIME_ZONE, value).ok()) { + value = GetConfig(CONFIG_SERVER).GetValue(CONFIG_SERVER_TIME_ZONE, + CONFIG_SERVER_TIME_ZONE_DEFAULT); + SetConfigValueInMem(CONFIG_SERVER, CONFIG_SERVER_TIME_ZONE, value); + } + return value; } +//////////////////////////////////////////////////////////////////////////////// /* db config */ std::string Config::GetDBConfigPath() { - ConfigNode db_config = GetConfig(CONFIG_DB); - return db_config.GetValue(CONFIG_DB_PATH, - CONFIG_DB_PATH_DEFAULT); + std::string value; + if (!GetConfigValueInMem(CONFIG_DB, CONFIG_DB_PATH, value).ok()) { + value = GetConfig(CONFIG_DB).GetValue(CONFIG_DB_PATH, + CONFIG_DB_PATH_DEFAULT); + SetConfigValueInMem(CONFIG_DB, CONFIG_DB_PATH, value); + } + return value; } std::string Config::GetDBConfigSlavePath() { - ConfigNode db_config = GetConfig(CONFIG_DB); - return db_config.GetValue(CONFIG_DB_SLAVE_PATH, - CONFIG_DB_SLAVE_PATH_DEFAULT); + std::string value; + if (!GetConfigValueInMem(CONFIG_DB, CONFIG_DB_SLAVE_PATH, value).ok()) { + value = GetConfig(CONFIG_DB).GetValue(CONFIG_DB_SLAVE_PATH, + CONFIG_DB_SLAVE_PATH_DEFAULT); + SetConfigValueInMem(CONFIG_DB, CONFIG_DB_SLAVE_PATH, value); + } + return value; } std::string Config::GetDBConfigBackendUrl() { - ConfigNode db_config = GetConfig(CONFIG_DB); - return db_config.GetValue(CONFIG_DB_BACKEND_URL, - CONFIG_DB_BACKEND_URL_DEFAULT); + std::string value; + if (!GetConfigValueInMem(CONFIG_DB, CONFIG_DB_BACKEND_URL, value).ok()) { + value = GetConfig(CONFIG_DB).GetValue(CONFIG_DB_BACKEND_URL, + CONFIG_DB_BACKEND_URL_DEFAULT); + SetConfigValueInMem(CONFIG_DB, CONFIG_DB_BACKEND_URL, value); + } + return value; } int32_t Config::GetDBConfigArchiveDiskThreshold() { - ConfigNode db_config = GetConfig(CONFIG_DB); - return db_config.GetInt32Value(CONFIG_DB_ARCHIVE_DISK_THRESHOLD, - std::stoi(CONFIG_DB_ARCHIVE_DISK_THRESHOLD_DEFAULT)); + std::string value; + if (!GetConfigValueInMem(CONFIG_DB, CONFIG_DB_ARCHIVE_DISK_THRESHOLD, value).ok()) { + value = GetConfig(CONFIG_DB).GetValue(CONFIG_DB_ARCHIVE_DISK_THRESHOLD, + CONFIG_DB_ARCHIVE_DISK_THRESHOLD_DEFAULT); + SetConfigValueInMem(CONFIG_DB, CONFIG_DB_ARCHIVE_DISK_THRESHOLD, value); + } + return std::stoi(value); } int32_t Config::GetDBConfigArchiveDaysThreshold() { - ConfigNode db_config = GetConfig(CONFIG_DB); - return db_config.GetInt32Value(CONFIG_DB_ARCHIVE_DAYS_THRESHOLD, - std::stoi(CONFIG_DB_ARCHIVE_DAYS_THRESHOLD_DEFAULT)); + std::string value; + if (!GetConfigValueInMem(CONFIG_DB, CONFIG_DB_ARCHIVE_DAYS_THRESHOLD, value).ok()) { + value = GetConfig(CONFIG_DB).GetValue(CONFIG_DB_ARCHIVE_DAYS_THRESHOLD, + CONFIG_DB_ARCHIVE_DAYS_THRESHOLD_DEFAULT); + SetConfigValueInMem(CONFIG_DB, CONFIG_DB_ARCHIVE_DAYS_THRESHOLD, value); + } + return std::stoi(value); } int32_t Config::GetDBConfigBufferSize() { - ConfigNode db_config = GetConfig(CONFIG_DB); - return db_config.GetInt32Value(CONFIG_DB_BUFFER_SIZE, - std::stoi(CONFIG_DB_BUFFER_SIZE_DEFAULT)); + std::string value; + if (!GetConfigValueInMem(CONFIG_DB, CONFIG_DB_BUFFER_SIZE, value).ok()) { + value = GetConfig(CONFIG_DB).GetValue(CONFIG_DB_BUFFER_SIZE, + CONFIG_DB_BUFFER_SIZE_DEFAULT); + SetConfigValueInMem(CONFIG_DB, CONFIG_DB_BUFFER_SIZE, value); + } + return std::stoi(value); } int32_t Config::GetDBConfigBuildIndexGPU() { - ConfigNode db_config = GetConfig(CONFIG_DB); - return db_config.GetInt32Value(CONFIG_DB_BUILD_INDEX_GPU, - std::stoi(CONFIG_DB_BUILD_INDEX_GPU_DEFAULT)); + std::string value; + if (!GetConfigValueInMem(CONFIG_DB, CONFIG_DB_BUILD_INDEX_GPU, value).ok()) { + value = GetConfig(CONFIG_DB).GetValue(CONFIG_DB_BUILD_INDEX_GPU, + CONFIG_DB_BUILD_INDEX_GPU_DEFAULT); + SetConfigValueInMem(CONFIG_DB, CONFIG_DB_BUILD_INDEX_GPU, value); + } + return std::stoi(value); } +//////////////////////////////////////////////////////////////////////////////// /* metric config */ bool Config::GetMetricConfigAutoBootup() { - ConfigNode metric_config = GetConfig(CONFIG_METRIC); - return metric_config.GetBoolValue(CONFIG_METRIC_AUTO_BOOTUP, - std::stoi(CONFIG_METRIC_AUTO_BOOTUP_DEFAULT)); + std::string value; + if (!GetConfigValueInMem(CONFIG_METRIC, CONFIG_METRIC_AUTO_BOOTUP, value).ok()) { + value = GetConfig(CONFIG_METRIC).GetValue(CONFIG_METRIC_AUTO_BOOTUP, + CONFIG_METRIC_AUTO_BOOTUP_DEFAULT); + SetConfigValueInMem(CONFIG_METRIC, CONFIG_METRIC_AUTO_BOOTUP, value); + } + std::transform(value.begin(), value.end(), value.begin(), ::tolower); + return (value == "true" || value == "on" || value == "yes" || value == "1"); } std::string Config::GetMetricConfigCollector() { - ConfigNode metric_config = GetConfig(CONFIG_METRIC); - return metric_config.GetValue(CONFIG_METRIC_COLLECTOR, - CONFIG_METRIC_COLLECTOR_DEFAULT); + std::string value; + if (!GetConfigValueInMem(CONFIG_METRIC, CONFIG_METRIC_COLLECTOR, value).ok()) { + value = GetConfig(CONFIG_METRIC).GetValue(CONFIG_METRIC_COLLECTOR, + CONFIG_METRIC_COLLECTOR_DEFAULT); + SetConfigValueInMem(CONFIG_METRIC, CONFIG_METRIC_COLLECTOR, value); + } + return value; } std::string Config::GetMetricConfigPrometheusPort() { - ConfigNode metric_config = GetConfig(CONFIG_METRIC); - return metric_config.GetValue(CONFIG_METRIC_PROMETHEUS_PORT, - CONFIG_METRIC_PROMETHEUS_PORT_DEFAULT); + std::string value; + if (!GetConfigValueInMem(CONFIG_METRIC, CONFIG_METRIC_PROMETHEUS_PORT, value).ok()) { + value = GetConfig(CONFIG_METRIC).GetValue(CONFIG_METRIC_PROMETHEUS_PORT, + CONFIG_METRIC_PROMETHEUS_PORT_DEFAULT); + SetConfigValueInMem(CONFIG_METRIC, CONFIG_METRIC_PROMETHEUS_PORT, value); + } + return value; } +//////////////////////////////////////////////////////////////////////////////// /* cache config */ int32_t Config::GetCacheConfigCpuMemCapacity() { - ConfigNode cache_config = GetConfig(CONFIG_CACHE); - return cache_config.GetInt32Value(CONFIG_CACHE_CPU_MEM_CAPACITY, - std::stoi(CONFIG_CACHE_CPU_MEM_CAPACITY_DEFAULT)); + std::string value; + if (!GetConfigValueInMem(CONFIG_CACHE, CONFIG_CACHE_CPU_MEM_CAPACITY, value).ok()) { + value = GetConfig(CONFIG_CACHE).GetValue(CONFIG_CACHE_CPU_MEM_CAPACITY, + CONFIG_CACHE_CPU_MEM_CAPACITY_DEFAULT); + SetConfigValueInMem(CONFIG_CACHE, CONFIG_CACHE_CPU_MEM_CAPACITY, value); + } + return std::stoi(value); } float Config::GetCacheConfigCpuMemThreshold() { - ConfigNode cache_config = GetConfig(CONFIG_CACHE); - return cache_config.GetFloatValue(CONFIG_CACHE_CPU_MEM_THRESHOLD, - std::stof(CONFIG_CACHE_CPU_MEM_THRESHOLD_DEFAULT)); + std::string value; + if (!GetConfigValueInMem(CONFIG_CACHE, CONFIG_CACHE_CPU_MEM_THRESHOLD, value).ok()) { + value = GetConfig(CONFIG_CACHE).GetValue(CONFIG_CACHE_CPU_MEM_THRESHOLD, + CONFIG_CACHE_CPU_MEM_THRESHOLD_DEFAULT); + SetConfigValueInMem(CONFIG_CACHE, CONFIG_CACHE_CPU_MEM_THRESHOLD, value); + } + return std::stof(value); } int32_t Config::GetCacheConfigGpuMemCapacity() { - ConfigNode cache_config = GetConfig(CONFIG_CACHE); - return cache_config.GetInt32Value(CONFIG_CACHE_GPU_MEM_CAPACITY, - std::stoi(CONFIG_CACHE_GPU_MEM_CAPACITY_DEFAULT)); + std::string value; + if (!GetConfigValueInMem(CONFIG_CACHE, CONFIG_CACHE_GPU_MEM_CAPACITY, value).ok()) { + value = GetConfig(CONFIG_CACHE).GetValue(CONFIG_CACHE_GPU_MEM_CAPACITY, + CONFIG_CACHE_GPU_MEM_CAPACITY_DEFAULT); + SetConfigValueInMem(CONFIG_CACHE, CONFIG_CACHE_GPU_MEM_CAPACITY, value); + } + return std::stoi(value); } float Config::GetCacheConfigGpuMemThreshold() { - ConfigNode cache_config = GetConfig(CONFIG_CACHE); - return cache_config.GetFloatValue(CONFIG_CACHE_GPU_MEM_THRESHOLD, - std::stof(CONFIG_CACHE_GPU_MEM_THRESHOLD_DEFAULT)); + std::string value; + if (!GetConfigValueInMem(CONFIG_CACHE, CONFIG_CACHE_GPU_MEM_THRESHOLD, value).ok()) { + value = GetConfig(CONFIG_CACHE).GetValue(CONFIG_CACHE_GPU_MEM_THRESHOLD, + CONFIG_CACHE_GPU_MEM_THRESHOLD_DEFAULT); + SetConfigValueInMem(CONFIG_CACHE, CONFIG_CACHE_GPU_MEM_THRESHOLD, value); + } + return std::stof(value); } bool Config::GetCacheConfigCacheInsertData() { - ConfigNode cache_config = GetConfig(CONFIG_CACHE); - return cache_config.GetBoolValue(CONFIG_CACHE_CACHE_INSERT_DATA, - std::stoi(CONFIG_CACHE_CACHE_INSERT_DATA_DEFAULT)); + std::string value; + if (!GetConfigValueInMem(CONFIG_CACHE, CONFIG_CACHE_CACHE_INSERT_DATA, value).ok()) { + value = GetConfig(CONFIG_CACHE).GetValue(CONFIG_CACHE_CACHE_INSERT_DATA, + CONFIG_CACHE_CACHE_INSERT_DATA_DEFAULT); + SetConfigValueInMem(CONFIG_CACHE, CONFIG_CACHE_CACHE_INSERT_DATA, value); + } + std::transform(value.begin(), value.end(), value.begin(), ::tolower); + return (value == "true" || value == "on" || value == "yes" || value == "1"); } +//////////////////////////////////////////////////////////////////////////////// /* engine config */ int32_t Config::GetEngineConfigBlasThreshold() { - ConfigNode engine_config = GetConfig(CONFIG_ENGINE); - return engine_config.GetInt32Value(CONFIG_ENGINE_BLAS_THRESHOLD, - std::stoi(CONFIG_ENGINE_BLAS_THRESHOLD_DEFAULT)); + std::string value; + if (!GetConfigValueInMem(CONFIG_ENGINE, CONFIG_ENGINE_BLAS_THRESHOLD, value).ok()) { + value = GetConfig(CONFIG_ENGINE).GetValue(CONFIG_ENGINE_BLAS_THRESHOLD, + CONFIG_ENGINE_BLAS_THRESHOLD_DEFAULT); + SetConfigValueInMem(CONFIG_ENGINE, CONFIG_ENGINE_BLAS_THRESHOLD, value); + } + return std::stoi(value); } int32_t Config::GetEngineConfigOmpThreadNum() { - ConfigNode engine_config = GetConfig(CONFIG_ENGINE); - return engine_config.GetInt32Value(CONFIG_ENGINE_OMP_THREAD_NUM, - std::stoi(CONFIG_ENGINE_OMP_THREAD_NUM_DEFAULT)); + std::string value; + if (!GetConfigValueInMem(CONFIG_ENGINE, CONFIG_ENGINE_OMP_THREAD_NUM, value).ok()) { + value = GetConfig(CONFIG_ENGINE).GetValue(CONFIG_ENGINE_OMP_THREAD_NUM, + CONFIG_ENGINE_OMP_THREAD_NUM_DEFAULT); + SetConfigValueInMem(CONFIG_ENGINE, CONFIG_ENGINE_OMP_THREAD_NUM, value); + } + return std::stoi(value); } +//////////////////////////////////////////////////////////////////////////////// /* resource config */ std::string Config::GetResourceConfigMode() { - ConfigNode resource_config = GetConfig(CONFIG_RESOURCE); - return resource_config.GetValue(CONFIG_RESOURCE_MODE, - CONFIG_RESOURCE_MODE_DEFAULT); + std::string value; + if (!GetConfigValueInMem(CONFIG_RESOURCE, CONFIG_RESOURCE_MODE, value).ok()) { + value = GetConfig(CONFIG_RESOURCE).GetValue(CONFIG_RESOURCE_MODE, + CONFIG_RESOURCE_MODE_DEFAULT); + SetConfigValueInMem(CONFIG_RESOURCE, CONFIG_RESOURCE_MODE, value); + } + return value; } std::vector diff --git a/cpp/src/server/Config.h b/cpp/src/server/Config.h index 2ad4c382cc..992ccf0965 100644 --- a/cpp/src/server/Config.h +++ b/cpp/src/server/Config.h @@ -17,10 +17,12 @@ #pragma once +#include +#include +#include "yaml-cpp/yaml.h" #include "utils/Status.h" #include "config/ConfigNode.h" -#include "yaml-cpp/yaml.h" namespace zilliz { namespace milvus { @@ -65,12 +67,12 @@ static const char* CONFIG_CACHE_CPU_MEM_THRESHOLD_DEFAULT = "0.85"; static const char* CONFIG_CACHE_GPU_MEM_THRESHOLD = "gpu_mem_threshold"; static const char* CONFIG_CACHE_GPU_MEM_THRESHOLD_DEFAULT = "0.85"; static const char* CONFIG_CACHE_CACHE_INSERT_DATA = "cache_insert_data"; -static const char* CONFIG_CACHE_CACHE_INSERT_DATA_DEFAULT = "0"; +static const char* CONFIG_CACHE_CACHE_INSERT_DATA_DEFAULT = "false"; /* metric config */ static const char* CONFIG_METRIC = "metric_config"; static const char* CONFIG_METRIC_AUTO_BOOTUP = "auto_bootup"; -static const char* CONFIG_METRIC_AUTO_BOOTUP_DEFAULT = "0"; +static const char* CONFIG_METRIC_AUTO_BOOTUP_DEFAULT = "false"; static const char* CONFIG_METRIC_COLLECTOR = "collector"; static const char* CONFIG_METRIC_COLLECTOR_DEFAULT = "prometheus"; static const char* CONFIG_METRIC_PROMETHEUS = "prometheus_config"; @@ -110,6 +112,14 @@ class Config { Status CheckEngineConfig(); Status CheckResourceConfig(); + Status GetConfigValueInMem(const std::string& parent_key, + const std::string& child_key, + std::string& value); + + void SetConfigValueInMem(const std::string& parent_key, + const std::string& child_key, + std::string& value); + public: std::string GetServerConfigAddress(); std::string GetServerConfigPort(); @@ -140,6 +150,10 @@ class Config { std::string GetResourceConfigMode(); std::vector GetResourceConfigPool(); + + private: + std::unordered_map> config_map_; + std::mutex mutex_; }; } From 0d0d08c256fb7ef99265c394d5bfb5dc1c41c653 Mon Sep 17 00:00:00 2001 From: "yudong.cai" Date: Wed, 25 Sep 2019 16:16:08 +0800 Subject: [PATCH 5/7] MS-574 let all config APIs return Status Former-commit-id: 7a843fa57779637761a80def4705a37f9805ceaf --- cpp/src/cache/CpuCacheMgr.cpp | 15 +- cpp/src/cache/GpuCacheMgr.cpp | 14 +- cpp/src/db/engine/ExecutionEngineImpl.cpp | 3 +- cpp/src/db/engine/ExecutionEngineImpl.h | 2 +- cpp/src/metrics/Metrics.cpp | 4 +- cpp/src/metrics/PrometheusMetrics.cpp | 7 +- cpp/src/scheduler/SchedInst.cpp | 7 +- cpp/src/server/Config.cpp | 588 ++++++++++++++-------- cpp/src/server/Config.h | 93 +++- cpp/src/server/DBWrapper.cpp | 45 +- cpp/src/server/Server.cpp | 8 +- cpp/src/server/grpc_impl/GrpcServer.cpp | 13 +- cpp/src/wrapper/KnowhereResource.cpp | 11 +- 13 files changed, 538 insertions(+), 272 deletions(-) diff --git a/cpp/src/cache/CpuCacheMgr.cpp b/cpp/src/cache/CpuCacheMgr.cpp index 9a69bf599f..1f93ec92bd 100644 --- a/cpp/src/cache/CpuCacheMgr.cpp +++ b/cpp/src/cache/CpuCacheMgr.cpp @@ -30,10 +30,21 @@ namespace { CpuCacheMgr::CpuCacheMgr() { server::Config& config = server::Config::GetInstance(); - int64_t cap = config.GetCacheConfigCpuMemCapacity() * unit; + Status s; + + int32_t cpu_mem_cap; + s = config.GetCacheConfigCpuMemCapacity(cpu_mem_cap); + if (!s.ok()) { + SERVER_LOG_ERROR << s.message(); + } + int64_t cap = cpu_mem_cap * unit; cache_ = std::make_shared>(cap, 1UL<<32); - float cpu_mem_threshold = config.GetCacheConfigCpuMemThreshold(); + float cpu_mem_threshold; + s = config.GetCacheConfigCpuMemThreshold(cpu_mem_threshold); + if (!s.ok()) { + SERVER_LOG_ERROR << s.message(); + } if (cpu_mem_threshold > 0.0 && cpu_mem_threshold <= 1.0) { cache_->set_freemem_percent(cpu_mem_threshold); } else { diff --git a/cpp/src/cache/GpuCacheMgr.cpp b/cpp/src/cache/GpuCacheMgr.cpp index bc6c3e13db..5c104ba0e7 100644 --- a/cpp/src/cache/GpuCacheMgr.cpp +++ b/cpp/src/cache/GpuCacheMgr.cpp @@ -34,11 +34,21 @@ namespace { GpuCacheMgr::GpuCacheMgr() { server::Config& config = server::Config::GetInstance(); + Status s; - int32_t cap = config.GetCacheConfigGpuMemCapacity() * G_BYTE; + int32_t gpu_mem_cap; + s = config.GetCacheConfigGpuMemCapacity(gpu_mem_cap); + if (!s.ok()) { + SERVER_LOG_ERROR << s.message(); + } + int32_t cap = gpu_mem_cap * G_BYTE; cache_ = std::make_shared>(cap, 1UL<<32); - float gpu_mem_threshold = config.GetCacheConfigGpuMemThreshold(); + float gpu_mem_threshold; + s = config.GetCacheConfigGpuMemThreshold(gpu_mem_threshold); + if (!s.ok()) { + SERVER_LOG_ERROR << s.message(); + } if (gpu_mem_threshold > 0.0 && gpu_mem_threshold <= 1.0) { cache_->set_freemem_percent(gpu_mem_threshold); } else { diff --git a/cpp/src/db/engine/ExecutionEngineImpl.cpp b/cpp/src/db/engine/ExecutionEngineImpl.cpp index 742d21a0be..a2640cb527 100644 --- a/cpp/src/db/engine/ExecutionEngineImpl.cpp +++ b/cpp/src/db/engine/ExecutionEngineImpl.cpp @@ -336,7 +336,8 @@ Status ExecutionEngineImpl::GpuCache(uint64_t gpu_id) { Status ExecutionEngineImpl::Init() { using namespace zilliz::milvus::server; server::Config &config = server::Config::GetInstance(); - gpu_num_ = config.GetDBConfigBuildIndexGPU(); + Status s = config.GetDBConfigBuildIndexGPU(gpu_num_); + if (!s.ok()) return s; return Status::OK(); } diff --git a/cpp/src/db/engine/ExecutionEngineImpl.h b/cpp/src/db/engine/ExecutionEngineImpl.h index cb08c50ad4..44d59d83e8 100644 --- a/cpp/src/db/engine/ExecutionEngineImpl.h +++ b/cpp/src/db/engine/ExecutionEngineImpl.h @@ -101,7 +101,7 @@ protected: std::string location_; int32_t nlist_ = 0; - int64_t gpu_num_ = 0; + int32_t gpu_num_ = 0; }; diff --git a/cpp/src/metrics/Metrics.cpp b/cpp/src/metrics/Metrics.cpp index 6c6df56b17..612f5e7fce 100644 --- a/cpp/src/metrics/Metrics.cpp +++ b/cpp/src/metrics/Metrics.cpp @@ -33,7 +33,9 @@ Metrics::GetInstance() { MetricsBase & Metrics::CreateMetricsCollector() { Config &config = Config::GetInstance(); - std::string collector_type_str = config.GetMetricConfigCollector(); + std::string collector_type_str; + + config.GetMetricConfigCollector(collector_type_str); if (collector_type_str == "prometheus") { return PrometheusMetrics::GetInstance(); diff --git a/cpp/src/metrics/PrometheusMetrics.cpp b/cpp/src/metrics/PrometheusMetrics.cpp index 14d2de6732..b607dbb020 100644 --- a/cpp/src/metrics/PrometheusMetrics.cpp +++ b/cpp/src/metrics/PrometheusMetrics.cpp @@ -31,11 +31,14 @@ ErrorCode PrometheusMetrics::Init() { try { Config &config = Config::GetInstance(); - startup_ = config.GetMetricConfigAutoBootup(); + Status s = config.GetMetricConfigAutoBootup(startup_); + if (!s.ok()) return s.code(); if (!startup_) return SERVER_SUCCESS; // Following should be read from config file. - const std::string bind_address = config.GetMetricConfigPrometheusPort(); + std::string bind_address; + s = config.GetMetricConfigPrometheusPort(bind_address); + if (!s.ok()) return s.code(); const std::string uri = std::string("/tmp/metrics"); const std::size_t num_threads = 2; diff --git a/cpp/src/scheduler/SchedInst.cpp b/cpp/src/scheduler/SchedInst.cpp index 5c4ec522da..8fe19f5b3e 100644 --- a/cpp/src/scheduler/SchedInst.cpp +++ b/cpp/src/scheduler/SchedInst.cpp @@ -39,8 +39,11 @@ std::mutex JobMgrInst::mutex_; void load_simple_config() { server::Config &config = server::Config::GetInstance(); - auto mode = config.GetResourceConfigMode(); - auto pool = config.GetResourceConfigPool(); + std::string mode; + config.GetResourceConfigMode(mode); + std::vector pool; + config.GetResourceConfigPool(pool); + bool cpu = false; std::set gpu_ids; for (auto &resource : pool) { diff --git a/cpp/src/server/Config.cpp b/cpp/src/server/Config.cpp index 9f7b9fd12e..9d76fa72c6 100644 --- a/cpp/src/server/Config.cpp +++ b/cpp/src/server/Config.cpp @@ -105,7 +105,7 @@ Config::CheckServerConfig() { */ bool okay = true; - ConfigNode server_config = GetConfig(CONFIG_SERVER); + ConfigNode server_config = GetConfigNode(CONFIG_SERVER); std::string ip_address = server_config.GetValue(CONFIG_SERVER_ADDRESS, CONFIG_SERVER_ADDRESS_DEFAULT); if (!ValidationUtil::ValidateIpAddress(ip_address).ok()) { @@ -173,7 +173,7 @@ Config::CheckDBConfig() { build_index_gpu: 0 # which gpu is used to build index, default: 0, range: 0 ~ gpu number - 1 */ bool okay = true; - ConfigNode db_config = GetConfig(CONFIG_DB); + ConfigNode db_config = GetConfigNode(CONFIG_DB); std::string db_path = db_config.GetValue(CONFIG_DB_PATH); if (db_path.empty()) { @@ -187,13 +187,15 @@ Config::CheckDBConfig() { okay = false; } - std::string archive_disk_threshold_str = db_config.GetValue(CONFIG_DB_ARCHIVE_DISK_THRESHOLD, CONFIG_DB_ARCHIVE_DISK_THRESHOLD_DEFAULT); + std::string archive_disk_threshold_str = + db_config.GetValue(CONFIG_DB_ARCHIVE_DISK_THRESHOLD, CONFIG_DB_ARCHIVE_DISK_THRESHOLD_DEFAULT); if (!ValidationUtil::ValidateStringIsNumber(archive_disk_threshold_str).ok()) { std::cerr << "ERROR: archive_disk_threshold " << archive_disk_threshold_str << " is not a number" << std::endl; okay = false; } - std::string archive_days_threshold_str = db_config.GetValue(CONFIG_DB_ARCHIVE_DAYS_THRESHOLD, CONFIG_DB_ARCHIVE_DAYS_THRESHOLD_DEFAULT); + std::string archive_days_threshold_str = + db_config.GetValue(CONFIG_DB_ARCHIVE_DAYS_THRESHOLD, CONFIG_DB_ARCHIVE_DAYS_THRESHOLD_DEFAULT); if (!ValidationUtil::ValidateStringIsNumber(archive_days_threshold_str).ok()) { std::cerr << "ERROR: archive_days_threshold " << archive_days_threshold_str << " is not a number" << std::endl; okay = false; @@ -203,8 +205,7 @@ Config::CheckDBConfig() { if (!ValidationUtil::ValidateStringIsNumber(buffer_size_str).ok()) { std::cerr << "ERROR: buffer_size " << buffer_size_str << " is not a number" << std::endl; okay = false; - } - else { + } else { uint64_t buffer_size = (uint64_t) std::stol(buffer_size_str); buffer_size *= GB; unsigned long total_mem = 0, free_mem = 0; @@ -242,7 +243,7 @@ Config::CheckMetricConfig() { (not used) push_gateway_port: 9091 # push method configure: push gateway port */ bool okay = true; - ConfigNode metric_config = GetConfig(CONFIG_METRIC); + ConfigNode metric_config = GetConfigNode(CONFIG_METRIC); std::string is_startup_str = metric_config.GetValue(CONFIG_METRIC_AUTO_BOOTUP, CONFIG_METRIC_AUTO_BOOTUP_DEFAULT); @@ -251,7 +252,8 @@ Config::CheckMetricConfig() { okay = false; } - std::string port_str = metric_config.GetChild(CONFIG_METRIC_PROMETHEUS).GetValue(CONFIG_METRIC_PROMETHEUS_PORT, "8080"); + std::string + port_str = metric_config.GetChild(CONFIG_METRIC_PROMETHEUS).GetValue(CONFIG_METRIC_PROMETHEUS_PORT, "8080"); if (!ValidationUtil::ValidateStringIsNumber(port_str).ok()) { std::cerr << "ERROR: port specified in prometheus_config " << port_str << " is not a number" << std::endl; okay = false; @@ -272,15 +274,14 @@ Config::CheckCacheConfig() { */ bool okay = true; - ConfigNode cache_config = GetConfig(CONFIG_CACHE); + ConfigNode cache_config = GetConfigNode(CONFIG_CACHE); std::string cpu_cache_capacity_str = cache_config.GetValue(CONFIG_CACHE_CPU_MEM_CAPACITY, CONFIG_CACHE_CPU_MEM_CAPACITY_DEFAULT); if (!ValidationUtil::ValidateStringIsNumber(cpu_cache_capacity_str).ok()) { std::cerr << "ERROR: cpu_cache_capacity " << cpu_cache_capacity_str << " is not a number" << std::endl; okay = false; - } - else { + } else { uint64_t cpu_cache_capacity = (uint64_t) std::stol(cpu_cache_capacity_str); cpu_cache_capacity *= GB; unsigned long total_mem = 0, free_mem = 0; @@ -288,13 +289,13 @@ Config::CheckCacheConfig() { if (cpu_cache_capacity >= total_mem) { std::cerr << "ERROR: cpu_cache_capacity exceed system memory" << std::endl; okay = false; - } - else if (cpu_cache_capacity > (double) total_mem * 0.9) { + } else if (cpu_cache_capacity > (double) total_mem * 0.9) { std::cerr << "Warning: cpu_cache_capacity value is too aggressive" << std::endl; } uint64_t buffer_size = - (uint64_t) GetConfig(CONFIG_DB).GetInt32Value(CONFIG_DB_BUFFER_SIZE, std::stoi(CONFIG_DB_BUFFER_SIZE_DEFAULT)); + (uint64_t) GetConfigNode(CONFIG_DB).GetInt32Value(CONFIG_DB_BUFFER_SIZE, + std::stoi(CONFIG_DB_BUFFER_SIZE_DEFAULT)); buffer_size *= GB; if (buffer_size + cpu_cache_capacity >= total_mem) { std::cerr << "ERROR: sum of cpu_cache_capacity and insert_buffer_size exceed system memory" << std::endl; @@ -308,8 +309,7 @@ Config::CheckCacheConfig() { if (!ValidationUtil::ValidateStringIsDouble(cpu_cache_free_percent_str, cpu_cache_free_percent).ok()) { std::cerr << "ERROR: cpu_cache_free_percent " << cpu_cache_free_percent_str << " is not a double" << std::endl; okay = false; - } - else if (cpu_cache_free_percent < std::numeric_limits::epsilon() || cpu_cache_free_percent > 1.0) { + } else if (cpu_cache_free_percent < std::numeric_limits::epsilon() || cpu_cache_free_percent > 1.0) { std::cerr << "ERROR: invalid cpu_cache_free_percent " << cpu_cache_free_percent_str << std::endl; okay = false; } @@ -326,22 +326,20 @@ Config::CheckCacheConfig() { if (!ValidationUtil::ValidateStringIsNumber(gpu_cache_capacity_str).ok()) { std::cerr << "ERROR: gpu_cache_capacity " << gpu_cache_capacity_str << " is not a number" << std::endl; okay = false; - } - else { + } else { uint64_t gpu_cache_capacity = (uint64_t) std::stol(gpu_cache_capacity_str); gpu_cache_capacity *= GB; - int gpu_index = GetConfig(CONFIG_DB).GetInt32Value(CONFIG_DB_BUILD_INDEX_GPU, std::stoi(CONFIG_DB_BUILD_INDEX_GPU_DEFAULT)); + int gpu_index = GetConfigNode(CONFIG_DB).GetInt32Value(CONFIG_DB_BUILD_INDEX_GPU, + std::stoi(CONFIG_DB_BUILD_INDEX_GPU_DEFAULT)); size_t gpu_memory; if (!ValidationUtil::GetGpuMemory(gpu_index, gpu_memory).ok()) { std::cerr << "ERROR: could not get gpu memory for device " << gpu_index << std::endl; okay = false; - } - else if (gpu_cache_capacity >= gpu_memory) { + } else if (gpu_cache_capacity >= gpu_memory) { std::cerr << "ERROR: gpu_cache_capacity " << gpu_cache_capacity << " exceed total gpu memory " << gpu_memory << std::endl; okay = false; - } - else if (gpu_cache_capacity > (double) gpu_memory * 0.9) { + } else if (gpu_cache_capacity > (double) gpu_memory * 0.9) { std::cerr << "Warning: gpu_cache_capacity value is too aggressive" << std::endl; } } @@ -352,8 +350,7 @@ Config::CheckCacheConfig() { if (!ValidationUtil::ValidateStringIsDouble(gpu_cache_free_percent_str, gpu_cache_free_percent).ok()) { std::cerr << "ERROR: gpu_cache_free_percent " << gpu_cache_free_percent_str << " is not a double" << std::endl; okay = false; - } - else if (gpu_cache_free_percent < std::numeric_limits::epsilon() || gpu_cache_free_percent > 1.0) { + } else if (gpu_cache_free_percent < std::numeric_limits::epsilon() || gpu_cache_free_percent > 1.0) { std::cerr << "ERROR: invalid gpu_cache_free_percent " << gpu_cache_free_percent << std::endl; okay = false; } @@ -369,7 +366,7 @@ Config::CheckEngineConfig() { omp_thread_num: 0 # how many compute threads be used by engine, 0 means use all cpu core to compute */ bool okay = true; - ConfigNode engine_config = GetConfig(CONFIG_ENGINE); + ConfigNode engine_config = GetConfigNode(CONFIG_ENGINE); std::string use_blas_threshold_str = engine_config.GetValue(CONFIG_ENGINE_BLAS_THRESHOLD, CONFIG_ENGINE_BLAS_THRESHOLD_DEFAULT); @@ -407,7 +404,7 @@ Config::CheckResourceConfig() { - gpu100 */ bool okay = true; - server::ConfigNode &config = GetConfig(server::CONFIG_RESOURCE); + server::ConfigNode &config = GetConfigNode(server::CONFIG_RESOURCE); auto mode = config.GetValue(CONFIG_RESOURCE_MODE, CONFIG_RESOURCE_MODE_DEFAULT); if (mode != "simple") { std::cerr << "ERROR: invalid resource config: mode is " << mode << std::endl; @@ -613,15 +610,8 @@ Config::PrintAll() const { } } -ConfigNode -Config::GetConfig(const std::string &name) const { - const ConfigMgr *mgr = ConfigMgr::GetInstance(); - const ConfigNode &root_node = mgr->GetRootNode(); - return root_node.GetChild(name); -} - ConfigNode & -Config::GetConfig(const std::string &name) { +Config::GetConfigNode(const std::string &name) { ConfigMgr *mgr = ConfigMgr::GetInstance(); ConfigNode &root_node = mgr->GetRootNode(); return root_node.GetChild(name); @@ -641,274 +631,436 @@ Config::GetConfigValueInMem(const std::string &parent_key, } } -void +Status Config::SetConfigValueInMem(const std::string &parent_key, const std::string &child_key, std::string &value) { std::lock_guard lock(mutex_); config_map_[parent_key][child_key] = value; + return Status::OK(); } //////////////////////////////////////////////////////////////////////////////// /* server config */ -std::string -Config::GetServerConfigAddress() { - std::string value; - if (!GetConfigValueInMem(CONFIG_SERVER, CONFIG_SERVER_ADDRESS, value).ok()) { - value = GetConfig(CONFIG_SERVER).GetValue(CONFIG_SERVER_ADDRESS, - CONFIG_SERVER_ADDRESS_DEFAULT); - SetConfigValueInMem(CONFIG_SERVER, CONFIG_SERVER_ADDRESS, value); +Status +Config::GetServerConfigStrAddress(std::string& value) { + if (GetConfigValueInMem(CONFIG_SERVER, CONFIG_SERVER_ADDRESS, value).ok()) { + return Status::OK(); + } else { + value = GetConfigNode(CONFIG_SERVER).GetValue(CONFIG_SERVER_ADDRESS, + CONFIG_SERVER_ADDRESS_DEFAULT); + return SetConfigValueInMem(CONFIG_SERVER, CONFIG_SERVER_ADDRESS, value); } - return value; } -std::string -Config::GetServerConfigPort() { - std::string value; - if (!GetConfigValueInMem(CONFIG_SERVER, CONFIG_SERVER_PORT, value).ok()) { - value = GetConfig(CONFIG_SERVER).GetValue(CONFIG_SERVER_PORT, - CONFIG_SERVER_PORT_DEFAULT); - SetConfigValueInMem(CONFIG_SERVER, CONFIG_SERVER_PORT, value); +Status +Config::GetServerConfigStrPort(std::string& value) { + if (GetConfigValueInMem(CONFIG_SERVER, CONFIG_SERVER_PORT, value).ok()) { + return Status::OK(); + } else { + value = GetConfigNode(CONFIG_SERVER).GetValue(CONFIG_SERVER_PORT, + CONFIG_SERVER_PORT_DEFAULT); + return SetConfigValueInMem(CONFIG_SERVER, CONFIG_SERVER_PORT, value); } - return value; } -std::string -Config::GetServerConfigMode() { - std::string value; - if (!GetConfigValueInMem(CONFIG_SERVER, CONFIG_SERVER_MODE, value).ok()) { - value = GetConfig(CONFIG_SERVER).GetValue(CONFIG_SERVER_MODE, - CONFIG_SERVER_MODE_DEFAULT); - SetConfigValueInMem(CONFIG_SERVER, CONFIG_SERVER_MODE, value); +Status +Config::GetServerConfigStrMode(std::string& value) { + if (GetConfigValueInMem(CONFIG_SERVER, CONFIG_SERVER_MODE, value).ok()) { + return Status::OK(); + } else { + value = GetConfigNode(CONFIG_SERVER).GetValue(CONFIG_SERVER_MODE, + CONFIG_SERVER_MODE_DEFAULT); + return SetConfigValueInMem(CONFIG_SERVER, CONFIG_SERVER_MODE, value); } - return value; } -std::string -Config::GetServerConfigTimeZone() { - std::string value; - if (!GetConfigValueInMem(CONFIG_SERVER, CONFIG_SERVER_TIME_ZONE, value).ok()) { - value = GetConfig(CONFIG_SERVER).GetValue(CONFIG_SERVER_TIME_ZONE, - CONFIG_SERVER_TIME_ZONE_DEFAULT); - SetConfigValueInMem(CONFIG_SERVER, CONFIG_SERVER_TIME_ZONE, value); +Status +Config::GetServerConfigStrTimeZone(std::string& value) { + if (GetConfigValueInMem(CONFIG_SERVER, CONFIG_SERVER_TIME_ZONE, value).ok()) { + return Status::OK(); + } else { + value = GetConfigNode(CONFIG_SERVER).GetValue(CONFIG_SERVER_TIME_ZONE, + CONFIG_SERVER_TIME_ZONE_DEFAULT); + return SetConfigValueInMem(CONFIG_SERVER, CONFIG_SERVER_TIME_ZONE, value); } - return value; } //////////////////////////////////////////////////////////////////////////////// /* db config */ -std::string -Config::GetDBConfigPath() { - std::string value; - if (!GetConfigValueInMem(CONFIG_DB, CONFIG_DB_PATH, value).ok()) { - value = GetConfig(CONFIG_DB).GetValue(CONFIG_DB_PATH, - CONFIG_DB_PATH_DEFAULT); - SetConfigValueInMem(CONFIG_DB, CONFIG_DB_PATH, value); +Status +Config::GetDBConfigStrPath(std::string& value) { + if (GetConfigValueInMem(CONFIG_DB, CONFIG_DB_PATH, value).ok()) { + return Status::OK(); + } else { + value = GetConfigNode(CONFIG_DB).GetValue(CONFIG_DB_PATH, + CONFIG_DB_PATH_DEFAULT); + return SetConfigValueInMem(CONFIG_DB, CONFIG_DB_PATH, value); } - return value; } -std::string -Config::GetDBConfigSlavePath() { - std::string value; - if (!GetConfigValueInMem(CONFIG_DB, CONFIG_DB_SLAVE_PATH, value).ok()) { - value = GetConfig(CONFIG_DB).GetValue(CONFIG_DB_SLAVE_PATH, - CONFIG_DB_SLAVE_PATH_DEFAULT); - SetConfigValueInMem(CONFIG_DB, CONFIG_DB_SLAVE_PATH, value); +Status +Config::GetDBConfigStrSlavePath(std::string& value) { + if (GetConfigValueInMem(CONFIG_DB, CONFIG_DB_SLAVE_PATH, value).ok()) { + return Status::OK(); + } else { + value = GetConfigNode(CONFIG_DB).GetValue(CONFIG_DB_SLAVE_PATH, + CONFIG_DB_SLAVE_PATH_DEFAULT); + return SetConfigValueInMem(CONFIG_DB, CONFIG_DB_SLAVE_PATH, value); } - return value; } -std::string -Config::GetDBConfigBackendUrl() { - std::string value; - if (!GetConfigValueInMem(CONFIG_DB, CONFIG_DB_BACKEND_URL, value).ok()) { - value = GetConfig(CONFIG_DB).GetValue(CONFIG_DB_BACKEND_URL, - CONFIG_DB_BACKEND_URL_DEFAULT); - SetConfigValueInMem(CONFIG_DB, CONFIG_DB_BACKEND_URL, value); +Status +Config::GetDBConfigStrBackendUrl(std::string& value) { + if (GetConfigValueInMem(CONFIG_DB, CONFIG_DB_BACKEND_URL, value).ok()) { + return Status::OK(); + } else { + value = GetConfigNode(CONFIG_DB).GetValue(CONFIG_DB_BACKEND_URL, + CONFIG_DB_BACKEND_URL_DEFAULT); + return SetConfigValueInMem(CONFIG_DB, CONFIG_DB_BACKEND_URL, value); } - return value; } -int32_t -Config::GetDBConfigArchiveDiskThreshold() { - std::string value; - if (!GetConfigValueInMem(CONFIG_DB, CONFIG_DB_ARCHIVE_DISK_THRESHOLD, value).ok()) { - value = GetConfig(CONFIG_DB).GetValue(CONFIG_DB_ARCHIVE_DISK_THRESHOLD, - CONFIG_DB_ARCHIVE_DISK_THRESHOLD_DEFAULT); - SetConfigValueInMem(CONFIG_DB, CONFIG_DB_ARCHIVE_DISK_THRESHOLD, value); +Status +Config::GetDBConfigStrArchiveDiskThreshold(std::string& value) { + if (GetConfigValueInMem(CONFIG_DB, CONFIG_DB_ARCHIVE_DISK_THRESHOLD, value).ok()) { + return Status::OK(); + } else { + value = GetConfigNode(CONFIG_DB).GetValue(CONFIG_DB_ARCHIVE_DISK_THRESHOLD, + CONFIG_DB_ARCHIVE_DISK_THRESHOLD_DEFAULT); + return SetConfigValueInMem(CONFIG_DB, CONFIG_DB_ARCHIVE_DISK_THRESHOLD, value); } - return std::stoi(value); } -int32_t -Config::GetDBConfigArchiveDaysThreshold() { - std::string value; - if (!GetConfigValueInMem(CONFIG_DB, CONFIG_DB_ARCHIVE_DAYS_THRESHOLD, value).ok()) { - value = GetConfig(CONFIG_DB).GetValue(CONFIG_DB_ARCHIVE_DAYS_THRESHOLD, - CONFIG_DB_ARCHIVE_DAYS_THRESHOLD_DEFAULT); - SetConfigValueInMem(CONFIG_DB, CONFIG_DB_ARCHIVE_DAYS_THRESHOLD, value); +Status +Config::GetDBConfigStrArchiveDaysThreshold(std::string& value) { + if (GetConfigValueInMem(CONFIG_DB, CONFIG_DB_ARCHIVE_DAYS_THRESHOLD, value).ok()) { + return Status::OK(); + } else { + value = GetConfigNode(CONFIG_DB).GetValue(CONFIG_DB_ARCHIVE_DAYS_THRESHOLD, + CONFIG_DB_ARCHIVE_DAYS_THRESHOLD_DEFAULT); + return SetConfigValueInMem(CONFIG_DB, CONFIG_DB_ARCHIVE_DAYS_THRESHOLD, value); } - return std::stoi(value); } -int32_t -Config::GetDBConfigBufferSize() { - std::string value; - if (!GetConfigValueInMem(CONFIG_DB, CONFIG_DB_BUFFER_SIZE, value).ok()) { - value = GetConfig(CONFIG_DB).GetValue(CONFIG_DB_BUFFER_SIZE, - CONFIG_DB_BUFFER_SIZE_DEFAULT); - SetConfigValueInMem(CONFIG_DB, CONFIG_DB_BUFFER_SIZE, value); +Status +Config::GetDBConfigStrBufferSize(std::string& value) { + if (GetConfigValueInMem(CONFIG_DB, CONFIG_DB_BUFFER_SIZE, value).ok()) { + return Status::OK(); + } else { + value = GetConfigNode(CONFIG_DB).GetValue(CONFIG_DB_BUFFER_SIZE, + CONFIG_DB_BUFFER_SIZE_DEFAULT); + return SetConfigValueInMem(CONFIG_DB, CONFIG_DB_BUFFER_SIZE, value); } - return std::stoi(value); } -int32_t -Config::GetDBConfigBuildIndexGPU() { - std::string value; - if (!GetConfigValueInMem(CONFIG_DB, CONFIG_DB_BUILD_INDEX_GPU, value).ok()) { - value = GetConfig(CONFIG_DB).GetValue(CONFIG_DB_BUILD_INDEX_GPU, - CONFIG_DB_BUILD_INDEX_GPU_DEFAULT); - SetConfigValueInMem(CONFIG_DB, CONFIG_DB_BUILD_INDEX_GPU, value); +Status +Config::GetDBConfigStrBuildIndexGPU(std::string& value) { + if (GetConfigValueInMem(CONFIG_DB, CONFIG_DB_BUILD_INDEX_GPU, value).ok()) { + return Status::OK(); + } else { + value = GetConfigNode(CONFIG_DB).GetValue(CONFIG_DB_BUILD_INDEX_GPU, + CONFIG_DB_BUILD_INDEX_GPU_DEFAULT); + return SetConfigValueInMem(CONFIG_DB, CONFIG_DB_BUILD_INDEX_GPU, value); } - return std::stoi(value); } //////////////////////////////////////////////////////////////////////////////// /* metric config */ -bool -Config::GetMetricConfigAutoBootup() { - std::string value; - if (!GetConfigValueInMem(CONFIG_METRIC, CONFIG_METRIC_AUTO_BOOTUP, value).ok()) { - value = GetConfig(CONFIG_METRIC).GetValue(CONFIG_METRIC_AUTO_BOOTUP, - CONFIG_METRIC_AUTO_BOOTUP_DEFAULT); - SetConfigValueInMem(CONFIG_METRIC, CONFIG_METRIC_AUTO_BOOTUP, value); +Status +Config::GetMetricConfigStrAutoBootup(std::string& value) { + if (GetConfigValueInMem(CONFIG_METRIC, CONFIG_METRIC_AUTO_BOOTUP, value).ok()) { + return Status::OK(); + } else { + value = GetConfigNode(CONFIG_METRIC).GetValue(CONFIG_METRIC_AUTO_BOOTUP, + CONFIG_METRIC_AUTO_BOOTUP_DEFAULT); + return SetConfigValueInMem(CONFIG_METRIC, CONFIG_METRIC_AUTO_BOOTUP, value); } - std::transform(value.begin(), value.end(), value.begin(), ::tolower); - return (value == "true" || value == "on" || value == "yes" || value == "1"); } -std::string -Config::GetMetricConfigCollector() { - std::string value; - if (!GetConfigValueInMem(CONFIG_METRIC, CONFIG_METRIC_COLLECTOR, value).ok()) { - value = GetConfig(CONFIG_METRIC).GetValue(CONFIG_METRIC_COLLECTOR, - CONFIG_METRIC_COLLECTOR_DEFAULT); - SetConfigValueInMem(CONFIG_METRIC, CONFIG_METRIC_COLLECTOR, value); +Status +Config::GetMetricConfigStrCollector(std::string& value) { + if (GetConfigValueInMem(CONFIG_METRIC, CONFIG_METRIC_COLLECTOR, value).ok()) { + return Status::OK(); + } else { + value = GetConfigNode(CONFIG_METRIC).GetValue(CONFIG_METRIC_COLLECTOR, + CONFIG_METRIC_COLLECTOR_DEFAULT); + return SetConfigValueInMem(CONFIG_METRIC, CONFIG_METRIC_COLLECTOR, value); } - return value; } -std::string -Config::GetMetricConfigPrometheusPort() { - std::string value; - if (!GetConfigValueInMem(CONFIG_METRIC, CONFIG_METRIC_PROMETHEUS_PORT, value).ok()) { - value = GetConfig(CONFIG_METRIC).GetValue(CONFIG_METRIC_PROMETHEUS_PORT, - CONFIG_METRIC_PROMETHEUS_PORT_DEFAULT); - SetConfigValueInMem(CONFIG_METRIC, CONFIG_METRIC_PROMETHEUS_PORT, value); +Status +Config::GetMetricConfigStrPrometheusPort(std::string& value) { + if (GetConfigValueInMem(CONFIG_METRIC, CONFIG_METRIC_PROMETHEUS_PORT, value).ok()) { + return Status::OK(); + } else { + value = GetConfigNode(CONFIG_METRIC).GetValue(CONFIG_METRIC_PROMETHEUS_PORT, + CONFIG_METRIC_PROMETHEUS_PORT_DEFAULT); + return SetConfigValueInMem(CONFIG_METRIC, CONFIG_METRIC_PROMETHEUS_PORT, value); } - return value; } //////////////////////////////////////////////////////////////////////////////// /* cache config */ -int32_t -Config::GetCacheConfigCpuMemCapacity() { - std::string value; - if (!GetConfigValueInMem(CONFIG_CACHE, CONFIG_CACHE_CPU_MEM_CAPACITY, value).ok()) { - value = GetConfig(CONFIG_CACHE).GetValue(CONFIG_CACHE_CPU_MEM_CAPACITY, - CONFIG_CACHE_CPU_MEM_CAPACITY_DEFAULT); - SetConfigValueInMem(CONFIG_CACHE, CONFIG_CACHE_CPU_MEM_CAPACITY, value); +Status +Config::GetCacheConfigStrCpuMemCapacity(std::string& value) { + if (GetConfigValueInMem(CONFIG_CACHE, CONFIG_CACHE_CPU_MEM_CAPACITY, value).ok()) { + return Status::OK(); + } else { + value = GetConfigNode(CONFIG_CACHE).GetValue(CONFIG_CACHE_CPU_MEM_CAPACITY, + CONFIG_CACHE_CPU_MEM_CAPACITY_DEFAULT); + return SetConfigValueInMem(CONFIG_CACHE, CONFIG_CACHE_CPU_MEM_CAPACITY, value); } - return std::stoi(value); } -float -Config::GetCacheConfigCpuMemThreshold() { - std::string value; - if (!GetConfigValueInMem(CONFIG_CACHE, CONFIG_CACHE_CPU_MEM_THRESHOLD, value).ok()) { - value = GetConfig(CONFIG_CACHE).GetValue(CONFIG_CACHE_CPU_MEM_THRESHOLD, - CONFIG_CACHE_CPU_MEM_THRESHOLD_DEFAULT); - SetConfigValueInMem(CONFIG_CACHE, CONFIG_CACHE_CPU_MEM_THRESHOLD, value); +Status +Config::GetCacheConfigStrCpuMemThreshold(std::string& value) { + if (GetConfigValueInMem(CONFIG_CACHE, CONFIG_CACHE_CPU_MEM_THRESHOLD, value).ok()) { + return Status::OK(); + } else { + value = GetConfigNode(CONFIG_CACHE).GetValue(CONFIG_CACHE_CPU_MEM_THRESHOLD, + CONFIG_CACHE_CPU_MEM_THRESHOLD_DEFAULT); + return SetConfigValueInMem(CONFIG_CACHE, CONFIG_CACHE_CPU_MEM_THRESHOLD, value); } - return std::stof(value); } -int32_t -Config::GetCacheConfigGpuMemCapacity() { - std::string value; - if (!GetConfigValueInMem(CONFIG_CACHE, CONFIG_CACHE_GPU_MEM_CAPACITY, value).ok()) { - value = GetConfig(CONFIG_CACHE).GetValue(CONFIG_CACHE_GPU_MEM_CAPACITY, - CONFIG_CACHE_GPU_MEM_CAPACITY_DEFAULT); - SetConfigValueInMem(CONFIG_CACHE, CONFIG_CACHE_GPU_MEM_CAPACITY, value); +Status +Config::GetCacheConfigStrGpuMemCapacity(std::string& value) { + if (GetConfigValueInMem(CONFIG_CACHE, CONFIG_CACHE_GPU_MEM_CAPACITY, value).ok()) { + return Status::OK(); + } else { + value = GetConfigNode(CONFIG_CACHE).GetValue(CONFIG_CACHE_GPU_MEM_CAPACITY, + CONFIG_CACHE_GPU_MEM_CAPACITY_DEFAULT); + return SetConfigValueInMem(CONFIG_CACHE, CONFIG_CACHE_GPU_MEM_CAPACITY, value); } - return std::stoi(value); } -float -Config::GetCacheConfigGpuMemThreshold() { - std::string value; - if (!GetConfigValueInMem(CONFIG_CACHE, CONFIG_CACHE_GPU_MEM_THRESHOLD, value).ok()) { - value = GetConfig(CONFIG_CACHE).GetValue(CONFIG_CACHE_GPU_MEM_THRESHOLD, - CONFIG_CACHE_GPU_MEM_THRESHOLD_DEFAULT); - SetConfigValueInMem(CONFIG_CACHE, CONFIG_CACHE_GPU_MEM_THRESHOLD, value); +Status +Config::GetCacheConfigStrGpuMemThreshold(std::string& value) { + if (GetConfigValueInMem(CONFIG_CACHE, CONFIG_CACHE_GPU_MEM_THRESHOLD, value).ok()) { + return Status::OK(); + } else { + value = GetConfigNode(CONFIG_CACHE).GetValue(CONFIG_CACHE_GPU_MEM_THRESHOLD, + CONFIG_CACHE_GPU_MEM_THRESHOLD_DEFAULT); + return SetConfigValueInMem(CONFIG_CACHE, CONFIG_CACHE_GPU_MEM_THRESHOLD, value); } - return std::stof(value); } -bool -Config::GetCacheConfigCacheInsertData() { - std::string value; - if (!GetConfigValueInMem(CONFIG_CACHE, CONFIG_CACHE_CACHE_INSERT_DATA, value).ok()) { - value = GetConfig(CONFIG_CACHE).GetValue(CONFIG_CACHE_CACHE_INSERT_DATA, - CONFIG_CACHE_CACHE_INSERT_DATA_DEFAULT); - SetConfigValueInMem(CONFIG_CACHE, CONFIG_CACHE_CACHE_INSERT_DATA, value); +Status +Config::GetCacheConfigStrCacheInsertData(std::string& value) { + if (GetConfigValueInMem(CONFIG_CACHE, CONFIG_CACHE_CACHE_INSERT_DATA, value).ok()) { + return Status::OK(); + } else { + value = GetConfigNode(CONFIG_CACHE).GetValue(CONFIG_CACHE_CACHE_INSERT_DATA, + CONFIG_CACHE_CACHE_INSERT_DATA_DEFAULT); + return SetConfigValueInMem(CONFIG_CACHE, CONFIG_CACHE_CACHE_INSERT_DATA, value); } - std::transform(value.begin(), value.end(), value.begin(), ::tolower); - return (value == "true" || value == "on" || value == "yes" || value == "1"); } //////////////////////////////////////////////////////////////////////////////// /* engine config */ -int32_t -Config::GetEngineConfigBlasThreshold() { - std::string value; - if (!GetConfigValueInMem(CONFIG_ENGINE, CONFIG_ENGINE_BLAS_THRESHOLD, value).ok()) { - value = GetConfig(CONFIG_ENGINE).GetValue(CONFIG_ENGINE_BLAS_THRESHOLD, - CONFIG_ENGINE_BLAS_THRESHOLD_DEFAULT); - SetConfigValueInMem(CONFIG_ENGINE, CONFIG_ENGINE_BLAS_THRESHOLD, value); +Status +Config::GetEngineConfigStrBlasThreshold(std::string& value) { + if (GetConfigValueInMem(CONFIG_ENGINE, CONFIG_ENGINE_BLAS_THRESHOLD, value).ok()) { + return Status::OK(); + } else { + value = GetConfigNode(CONFIG_ENGINE).GetValue(CONFIG_ENGINE_BLAS_THRESHOLD, + CONFIG_ENGINE_BLAS_THRESHOLD_DEFAULT); + return SetConfigValueInMem(CONFIG_ENGINE, CONFIG_ENGINE_BLAS_THRESHOLD, value); } - return std::stoi(value); } -int32_t -Config::GetEngineConfigOmpThreadNum() { - std::string value; - if (!GetConfigValueInMem(CONFIG_ENGINE, CONFIG_ENGINE_OMP_THREAD_NUM, value).ok()) { - value = GetConfig(CONFIG_ENGINE).GetValue(CONFIG_ENGINE_OMP_THREAD_NUM, - CONFIG_ENGINE_OMP_THREAD_NUM_DEFAULT); - SetConfigValueInMem(CONFIG_ENGINE, CONFIG_ENGINE_OMP_THREAD_NUM, value); +Status +Config::GetEngineConfigStrOmpThreadNum(std::string& value) { + if (GetConfigValueInMem(CONFIG_ENGINE, CONFIG_ENGINE_OMP_THREAD_NUM, value).ok()) { + return Status::OK(); + } else { + value = GetConfigNode(CONFIG_ENGINE).GetValue(CONFIG_ENGINE_OMP_THREAD_NUM, + CONFIG_ENGINE_OMP_THREAD_NUM_DEFAULT); + return SetConfigValueInMem(CONFIG_ENGINE, CONFIG_ENGINE_OMP_THREAD_NUM, value); } - return std::stoi(value); } //////////////////////////////////////////////////////////////////////////////// /* resource config */ -std::string -Config::GetResourceConfigMode() { - std::string value; - if (!GetConfigValueInMem(CONFIG_RESOURCE, CONFIG_RESOURCE_MODE, value).ok()) { - value = GetConfig(CONFIG_RESOURCE).GetValue(CONFIG_RESOURCE_MODE, - CONFIG_RESOURCE_MODE_DEFAULT); - SetConfigValueInMem(CONFIG_RESOURCE, CONFIG_RESOURCE_MODE, value); +Status +Config::GetResourceConfigStrMode(std::string& value) { + if (GetConfigValueInMem(CONFIG_RESOURCE, CONFIG_RESOURCE_MODE, value).ok()) { + return Status::OK(); + } else { + value = GetConfigNode(CONFIG_RESOURCE).GetValue(CONFIG_RESOURCE_MODE, + CONFIG_RESOURCE_MODE_DEFAULT); + return SetConfigValueInMem(CONFIG_RESOURCE, CONFIG_RESOURCE_MODE, value); } - return value; } -std::vector -Config::GetResourceConfigPool() { - ConfigNode resource_config = GetConfig(CONFIG_RESOURCE); - return resource_config.GetSequence(CONFIG_RESOURCE_POOL); + +//////////////////////////////////////////////////////////////////////////////// +Status +Config::GetServerConfigAddress(std::string& value) { + return GetServerConfigStrAddress(value); +} + +Status +Config::GetServerConfigPort(std::string& value) { + return GetServerConfigStrPort(value); +} + +Status +Config::GetServerConfigMode(std::string& value) { + return GetServerConfigStrMode(value); +} + +Status +Config::GetServerConfigTimeZone(std::string& value) { + return GetServerConfigStrTimeZone(value); +} + +Status +Config::GetDBConfigPath(std::string& value) { + return GetDBConfigStrPath(value); +} + +Status +Config::GetDBConfigSlavePath(std::string& value) { + return GetDBConfigStrSlavePath(value); +} + +Status +Config::GetDBConfigBackendUrl(std::string& value) { + return GetDBConfigStrBackendUrl(value); +} + +Status +Config::GetDBConfigArchiveDiskThreshold(int32_t& value) { + std::string str; + Status s = GetDBConfigStrArchiveDiskThreshold(str); + if (!s.ok()) return s; + value = std::stoi(str); + return Status::OK(); +} + +Status +Config::GetDBConfigArchiveDaysThreshold(int32_t& value) { + std::string str; + Status s = GetDBConfigStrArchiveDaysThreshold(str); + if (!s.ok()) return s; + value = std::stoi(str); + return Status::OK(); +} + +Status +Config::GetDBConfigBufferSize(int32_t& value) { + std::string str; + Status s = GetDBConfigStrBufferSize(str); + if (!s.ok()) return s; + value = std::stoi(str); + return Status::OK(); +} + +Status +Config::GetDBConfigBuildIndexGPU(int32_t& value) { + std::string str; + Status s = GetDBConfigStrBuildIndexGPU(str); + if (!s.ok()) return s; + value = std::stoi(str); + return Status::OK(); +} + +Status +Config::GetMetricConfigAutoBootup(bool& value) { + std::string str; + Status s = GetMetricConfigStrAutoBootup(str); + if (!s.ok()) return s; + std::transform(str.begin(), str.end(), str.begin(), ::tolower); + value = (str == "true" || str == "on" || str == "yes" || str == "1"); + return Status::OK(); +} + +Status +Config::GetMetricConfigCollector(std::string& value) { + return GetMetricConfigStrCollector(value); +} + +Status +Config::GetMetricConfigPrometheusPort(std::string& value) { + return GetMetricConfigStrPrometheusPort(value); +} + +Status +Config::GetCacheConfigCpuMemCapacity(int32_t& value) { + std::string str; + Status s = GetCacheConfigStrCpuMemCapacity(str); + if (!s.ok()) return s; + value = std::stoi(str); + return Status::OK(); +} + +Status +Config::GetCacheConfigCpuMemThreshold(float& value) { + std::string str; + Status s = GetCacheConfigStrCpuMemThreshold(str); + if (!s.ok()) return s; + value = std::stof(str); + return Status::OK(); +} + +Status +Config::GetCacheConfigGpuMemCapacity(int32_t& value) { + std::string str; + Status s = GetCacheConfigStrGpuMemCapacity(str); + if (!s.ok()) return s; + value = std::stoi(str); + return Status::OK(); +} + +Status +Config::GetCacheConfigGpuMemThreshold(float& value) { + std::string str; + Status s = GetCacheConfigStrGpuMemThreshold(str); + if (!s.ok()) return s; + value = std::stof(str); + return Status::OK(); +} + +Status +Config::GetCacheConfigCacheInsertData(bool& value) { + std::string str; + Status s = GetCacheConfigStrCacheInsertData(str); + if (!s.ok()) return s; + std::transform(str.begin(), str.end(), str.begin(), ::tolower); + value = (str == "true" || str == "on" || str == "yes" || str == "1"); + return Status::OK(); +} + +Status +Config::GetEngineConfigBlasThreshold(int32_t& value) { + std::string str; + Status s = GetEngineConfigStrBlasThreshold(str); + if (!s.ok()) return s; + value = std::stoi(str); + return Status::OK(); +} + +Status +Config::GetEngineConfigOmpThreadNum(int32_t& value) { + std::string str; + Status s = GetEngineConfigStrOmpThreadNum(str); + if (!s.ok()) return s; + value = std::stoi(str); + return Status::OK(); +} + +Status +Config::GetResourceConfigMode(std::string& value) { + return GetResourceConfigStrMode(value); +} + +Status +Config::GetResourceConfigPool(std::vector& value) { + ConfigNode resource_config = GetConfigNode(CONFIG_RESOURCE); + value = resource_config.GetSequence(CONFIG_RESOURCE_POOL); + return Status::OK(); } } diff --git a/cpp/src/server/Config.h b/cpp/src/server/Config.h index 992ccf0965..3eeda32546 100644 --- a/cpp/src/server/Config.h +++ b/cpp/src/server/Config.h @@ -102,8 +102,7 @@ class Config { void PrintAll() const; private: - ConfigNode GetConfig(const std::string& name) const; - ConfigNode& GetConfig(const std::string& name); + ConfigNode& GetConfigNode(const std::string& name); Status CheckServerConfig(); Status CheckDBConfig(); @@ -116,40 +115,80 @@ class Config { const std::string& child_key, std::string& value); - void SetConfigValueInMem(const std::string& parent_key, + Status SetConfigValueInMem(const std::string& parent_key, const std::string& child_key, std::string& value); + private: + /* server config */ + Status GetServerConfigStrAddress(std::string& value); + Status GetServerConfigStrPort(std::string& value); + Status GetServerConfigStrMode(std::string& value); + Status GetServerConfigStrTimeZone(std::string& value); + + /* db config */ + Status GetDBConfigStrPath(std::string& value); + Status GetDBConfigStrSlavePath(std::string& value); + Status GetDBConfigStrBackendUrl(std::string& value); + Status GetDBConfigStrArchiveDiskThreshold(std::string& value); + Status GetDBConfigStrArchiveDaysThreshold(std::string& value); + Status GetDBConfigStrBufferSize(std::string& value); + Status GetDBConfigStrBuildIndexGPU(std::string& value); + + /* metric config */ + Status GetMetricConfigStrAutoBootup(std::string& value); + Status GetMetricConfigStrCollector(std::string& value); + Status GetMetricConfigStrPrometheusPort(std::string& value); + + /* cache config */ + Status GetCacheConfigStrCpuMemCapacity(std::string& value); + Status GetCacheConfigStrCpuMemThreshold(std::string& value); + Status GetCacheConfigStrGpuMemCapacity(std::string& value); + Status GetCacheConfigStrGpuMemThreshold(std::string& value); + Status GetCacheConfigStrCacheInsertData(std::string& value); + + /* engine config */ + Status GetEngineConfigStrBlasThreshold(std::string& value); + Status GetEngineConfigStrOmpThreadNum(std::string& value); + + /* resource config */ + Status GetResourceConfigStrMode(std::string& value); + public: - std::string GetServerConfigAddress(); - std::string GetServerConfigPort(); - std::string GetServerConfigMode(); - std::string GetServerConfigTimeZone(); + /* server config */ + Status GetServerConfigAddress(std::string& value); + Status GetServerConfigPort(std::string& value); + Status GetServerConfigMode(std::string& value); + Status GetServerConfigTimeZone(std::string& value); - std::string GetDBConfigPath(); - std::string GetDBConfigSlavePath(); - std::string GetDBConfigBackendUrl(); - int32_t GetDBConfigArchiveDiskThreshold(); - int32_t GetDBConfigArchiveDaysThreshold(); - int32_t GetDBConfigBufferSize(); - int32_t GetDBConfigBuildIndexGPU(); + /* db config */ + Status GetDBConfigPath(std::string& value); + Status GetDBConfigSlavePath(std::string& value); + Status GetDBConfigBackendUrl(std::string& value); + Status GetDBConfigArchiveDiskThreshold(int32_t& value); + Status GetDBConfigArchiveDaysThreshold(int32_t& value); + Status GetDBConfigBufferSize(int32_t& value); + Status GetDBConfigBuildIndexGPU(int32_t& value); - bool GetMetricConfigAutoBootup(); - std::string GetMetricConfigCollector(); - std::string GetMetricConfigPrometheusPort(); + /* metric config */ + Status GetMetricConfigAutoBootup(bool& value); + Status GetMetricConfigCollector(std::string& value); + Status GetMetricConfigPrometheusPort(std::string& value); - int32_t GetCacheConfigCpuMemCapacity(); - float GetCacheConfigCpuMemThreshold(); - int32_t GetCacheConfigGpuMemCapacity(); - float GetCacheConfigGpuMemThreshold(); - bool GetCacheConfigCacheInsertData(); + /* cache config */ + Status GetCacheConfigCpuMemCapacity(int32_t& value); + Status GetCacheConfigCpuMemThreshold(float& value); + Status GetCacheConfigGpuMemCapacity(int32_t& value); + Status GetCacheConfigGpuMemThreshold(float& value); + Status GetCacheConfigCacheInsertData(bool& value); - int32_t GetEngineConfigBlasThreshold(); - int32_t GetEngineConfigOmpThreadNum(); + /* engine config */ + Status GetEngineConfigBlasThreshold(int32_t& value); + Status GetEngineConfigOmpThreadNum(int32_t& value); - std::string GetResourceConfigMode(); - std::vector - GetResourceConfigPool(); + /* resource config */ + Status GetResourceConfigMode(std::string& value); + Status GetResourceConfigPool(std::vector& value); private: std::unordered_map> config_map_; diff --git a/cpp/src/server/DBWrapper.cpp b/cpp/src/server/DBWrapper.cpp index bc0b080625..bb628a0c63 100644 --- a/cpp/src/server/DBWrapper.cpp +++ b/cpp/src/server/DBWrapper.cpp @@ -36,19 +36,33 @@ DBWrapper::DBWrapper() { Status DBWrapper::StartService() { Config& config = Config::GetInstance(); - + Status s; //db config engine::DBOptions opt; - opt.meta.backend_uri = config.GetDBConfigBackendUrl(); - opt.meta.path = config.GetDBConfigPath() + "/db"; - std::string db_slave_path = config.GetDBConfigSlavePath(); + s = config.GetDBConfigBackendUrl(opt.meta.backend_uri); + if (!s.ok()) return s; + + std::string path; + s = config.GetDBConfigPath(path); + if (!s.ok()) return s; + + opt.meta.path = path + "/db"; + + std::string db_slave_path; + s = config.GetDBConfigSlavePath(db_slave_path); + if (!s.ok()) return s; + StringHelpFunctions::SplitStringByDelimeter(db_slave_path, ";", opt.meta.slave_paths); // cache config - opt.insert_cache_immediately_ = config.GetCacheConfigCacheInsertData(); + s = config.GetCacheConfigCacheInsertData(opt.insert_cache_immediately_); + if (!s.ok()) return s; + + std::string mode; + s = config.GetServerConfigMode(mode); + if (!s.ok()) return s; - std::string mode = config.GetServerConfigMode(); if (mode == "single") { opt.mode = engine::DBOptions::MODE::SINGLE; } @@ -64,8 +78,10 @@ Status DBWrapper::StartService() { } // engine config - int32_t omp_thread = config.GetEngineConfigOmpThreadNum(); - if(omp_thread > 0) { + int32_t omp_thread; + s = config.GetEngineConfigOmpThreadNum(omp_thread); + if (!s.ok()) return s; + if (omp_thread > 0) { omp_set_num_threads(omp_thread); SERVER_LOG_DEBUG << "Specify openmp thread number: " << omp_thread; } else { @@ -77,15 +93,22 @@ Status DBWrapper::StartService() { } //init faiss global variable - faiss::distance_compute_blas_threshold = config.GetEngineConfigBlasThreshold(); + int32_t blas_threshold; + s = config.GetEngineConfigBlasThreshold(blas_threshold); + if (!s.ok()) return s; + faiss::distance_compute_blas_threshold = blas_threshold; //set archive config engine::ArchiveConf::CriteriaT criterial; - int32_t disk = config.GetDBConfigArchiveDiskThreshold(); - int32_t days = config.GetDBConfigArchiveDaysThreshold(); + int32_t disk, days; + s = config.GetDBConfigArchiveDiskThreshold(disk); + if (!s.ok()) return s; if (disk > 0) { criterial[engine::ARCHIVE_CONF_DISK] = disk; } + + s = config.GetDBConfigArchiveDaysThreshold(days); + if (!s.ok()) return s; if (days > 0) { criterial[engine::ARCHIVE_CONF_DAYS] = days; } diff --git a/cpp/src/server/Server.cpp b/cpp/src/server/Server.cpp index 7ca0b2a7c9..d926ca5dac 100644 --- a/cpp/src/server/Server.cpp +++ b/cpp/src/server/Server.cpp @@ -161,7 +161,13 @@ Server::Start() { /* log path is defined in Config file, so InitLog must be called after LoadConfig */ Config &config = Config::GetInstance(); - std::string time_zone = config.GetServerConfigTimeZone(); + std::string time_zone; + Status s = config.GetServerConfigTimeZone(time_zone); + if (!s.ok()) { + std::cerr << "Fail to get server config timezone" << std::endl; + return; + } + if (time_zone.length() == 3) { time_zone = "CUT"; } else { diff --git a/cpp/src/server/grpc_impl/GrpcServer.cpp b/cpp/src/server/grpc_impl/GrpcServer.cpp index c10d7b5379..7e20921549 100644 --- a/cpp/src/server/grpc_impl/GrpcServer.cpp +++ b/cpp/src/server/grpc_impl/GrpcServer.cpp @@ -74,8 +74,17 @@ GrpcServer::Stop() { Status GrpcServer::StartService() { Config &config = Config::GetInstance(); - std::string address = config.GetServerConfigAddress(); - std::string port = config.GetServerConfigPort(); + std::string address, port; + Status s; + + s = config.GetServerConfigAddress(address); + if (!s.ok()) { + return s; + } + s = config.GetServerConfigPort(port); + if (!s.ok()) { + return s; + } std::string server_address(address + ":" + port); diff --git a/cpp/src/wrapper/KnowhereResource.cpp b/cpp/src/wrapper/KnowhereResource.cpp index 1e4248f6e1..a54ff1e84d 100644 --- a/cpp/src/wrapper/KnowhereResource.cpp +++ b/cpp/src/wrapper/KnowhereResource.cpp @@ -36,15 +36,22 @@ ErrorCode KnowhereResource::Initialize() { }; using GpuResourcesArray = std::map; GpuResourcesArray gpu_resources; + Status s; //get build index gpu resource server::Config& config = server::Config::GetInstance(); - int32_t build_index_gpu = config.GetDBConfigBuildIndexGPU(); + int32_t build_index_gpu; + s = config.GetDBConfigBuildIndexGPU(build_index_gpu); + if (!s.ok()) return s.code(); + gpu_resources.insert(std::make_pair(build_index_gpu, GpuResourceSetting())); //get search gpu resource - auto pool = config.GetResourceConfigPool(); + std::vector pool; + s = config.GetResourceConfigPool(pool); + if (!s.ok()) return s.code(); + std::set gpu_ids; for (auto &resource : pool) { if (resource.length() < 4 || resource.substr(0, 3) != "gpu") { From ee7fb3684ad70e8263b92a7070b97cc5c7dd0e91 Mon Sep 17 00:00:00 2001 From: "yudong.cai" Date: Wed, 25 Sep 2019 19:16:12 +0800 Subject: [PATCH 6/7] MS-574 add config check Former-commit-id: ac1ed68cc7fcd08cde02bccbd574ac34fdbfdb97 --- cpp/CHANGELOG.md | 3 +- cpp/src/server/Config.cpp | 985 +++++++++++-------------------- cpp/src/server/Config.h | 98 +-- cpp/src/server/Server.cpp | 8 +- cpp/src/utils/ValidationUtil.cpp | 50 +- cpp/src/utils/ValidationUtil.h | 4 +- 6 files changed, 456 insertions(+), 692 deletions(-) diff --git a/cpp/CHANGELOG.md b/cpp/CHANGELOG.md index 667399cc5d..2281494fed 100644 --- a/cpp/CHANGELOG.md +++ b/cpp/CHANGELOG.md @@ -18,7 +18,8 @@ Please mark all change in change log and use the ticket from JIRA. - MS-562 - Add JobMgr and TaskCreator in Scheduler - MS-566 - Refactor cmake - MS-555 - Remove old scheduler -- MS-578 - Makesure milvus5.0 don't crack 0.3.1 data +- MS-574 - Milvus configuration refactor +- MS-578 - Make sure milvus5.0 don't crack 0.3.1 data ## New Feature diff --git a/cpp/src/server/Config.cpp b/cpp/src/server/Config.cpp index 9d76fa72c6..b89f1e3e02 100644 --- a/cpp/src/server/Config.cpp +++ b/cpp/src/server/Config.cpp @@ -33,7 +33,6 @@ namespace zilliz { namespace milvus { namespace server { -constexpr uint64_t MB = 1UL << 20; constexpr uint64_t GB = 1UL << 30; Config & @@ -71,545 +70,272 @@ Config::LoadConfigFile(const std::string &filename) { return Status::OK(); } +void +Config::PrintConfigSection(const std::string& config_node_name) { + std::cout << std::endl; + std::cout << config_node_name << ":" << std::endl; + if (config_map_.find(config_node_name) != config_map_.end()) { + for (auto item: config_map_[config_node_name]) { + std::cout << item.first << ": " << item.second << std::endl; + } + } +} + +void +Config::PrintAll() { + PrintConfigSection(CONFIG_SERVER); + PrintConfigSection(CONFIG_DB); + PrintConfigSection(CONFIG_CACHE); + PrintConfigSection(CONFIG_METRIC); + PrintConfigSection(CONFIG_ENGINE); + PrintConfigSection(CONFIG_RESOURCE); +} + +//////////////////////////////////////////////////////////////////////////////// Status -Config::ValidateConfig() { - if (!CheckServerConfig().ok()) { - return Status(SERVER_INVALID_ARGUMENT, "Server config validation check fail"); - } - if (!CheckDBConfig().ok()) { - return Status(SERVER_INVALID_ARGUMENT, "DB config validation check fail"); - } - if (!CheckMetricConfig().ok()) { - return Status(SERVER_INVALID_ARGUMENT, "Metric config validation check fail"); - } - if (!CheckCacheConfig().ok()) { - return Status(SERVER_INVALID_ARGUMENT, "Cache config validation check fail"); - } - if (!CheckEngineConfig().ok()) { - return Status(SERVER_INVALID_ARGUMENT, "Engine config validation check fail"); - } - if (!CheckResourceConfig().ok()) { - return Status(SERVER_INVALID_ARGUMENT, "Resource config validation check fail"); +Config::CheckServerConfigAddress(std::string &value) { + if (!ValidationUtil::ValidateIpAddress(value).ok()) { + return Status(SERVER_INVALID_ARGUMENT, "Invalid server config address: " + value); } return Status::OK(); } Status -Config::CheckServerConfig() { -/* - server_config: - address: 0.0.0.0 # milvus server ip address (IPv4) - port: 19530 # the port milvus listen to, default: 19530, range: 1025 ~ 65534 - mode: single # milvus deployment type: single, cluster, read_only - time_zone: UTC+8 # Use the UTC-x or UTC+x to specify a time zone. eg. UTC+8 for China Standard Time - -*/ - bool okay = true; - ConfigNode server_config = GetConfigNode(CONFIG_SERVER); - - std::string ip_address = server_config.GetValue(CONFIG_SERVER_ADDRESS, CONFIG_SERVER_ADDRESS_DEFAULT); - if (!ValidationUtil::ValidateIpAddress(ip_address).ok()) { - std::cerr << "ERROR: invalid server IP address: " << ip_address << std::endl; - okay = false; - } - - std::string port_str = server_config.GetValue(CONFIG_SERVER_PORT, CONFIG_SERVER_PORT_DEFAULT); - if (!ValidationUtil::ValidateStringIsNumber(port_str).ok()) { - std::cerr << "ERROR: port " << port_str << " is not a number" << std::endl; - okay = false; +Config::CheckServerConfigPort(std::string &value) { + if (!ValidationUtil::ValidateStringIsNumber(value).ok()) { + return Status(SERVER_INVALID_ARGUMENT, "Invalid server config port: " + value); } else { - int32_t port = std::stol(port_str); + int32_t port = std::stoi(value); if (!(port > 1024 && port < 65535)) { - std::cerr << "ERROR: port " << port_str << " out of range (1024, 65535)" << std::endl; - okay = false; + return Status(SERVER_INVALID_ARGUMENT, "Server config port out of range (1024, 65535): " + value); } } + return Status::OK(); +} - std::string mode = server_config.GetValue(CONFIG_SERVER_MODE, CONFIG_SERVER_MODE_DEFAULT); - if (mode != "single" && mode != "cluster" && mode != "read_only") { - std::cerr << "ERROR: mode " << mode << " is not one of ['single', 'cluster', 'read_only']" << std::endl; - okay = false; +Status +Config::CheckServerConfigMode(std::string &value) { + if (value != "single" && value != "cluster" && value != "read_only") { + return Status(SERVER_INVALID_ARGUMENT, "Invalid server config mode [single, cluster, read_only]: " + value); } + return Status::OK(); +} - std::string time_zone = server_config.GetValue(CONFIG_SERVER_TIME_ZONE, CONFIG_SERVER_TIME_ZONE_DEFAULT); - int flag = 0; - if (time_zone.length() <= 3) { - flag = 1; +Status +Config::CheckServerConfigTimeZone(std::string &value) { + if (value.length() <= 3) { + return Status(SERVER_INVALID_ARGUMENT, "Invalid server config time_zone: " + value); } else { - if (time_zone.substr(0, 3) != "UTC") { - flag = 1; + if (value.substr(0, 3) != "UTC") { + return Status(SERVER_INVALID_ARGUMENT, "Invalid server config time_zone: " + value); } else { try { - stoi(time_zone.substr(3)); + stoi(value.substr(3)); } catch (...) { - flag = 1; + return Status(SERVER_INVALID_ARGUMENT, "Invalid server config time_zone: " + value); } } } - if (flag == 1) { - std::cerr << "ERROR: time_zone " << time_zone << " format wrong" << std::endl; - okay = false; - } - - return (okay ? Status::OK() : Status(SERVER_INVALID_ARGUMENT, "Illegal server config")); + return Status::OK(); } Status -Config::CheckDBConfig() { -/* - db_config: - db_path: @MILVUS_DB_PATH@ # milvus data storage path - db_slave_path: # secondry data storage path, split by semicolon - - # URI format: dialect://username:password@host:port/database - # All parts except dialect are optional, but you MUST include the delimiters - # Currently dialect supports mysql or sqlite - db_backend_url: sqlite://:@:/ - - archive_disk_threshold: 0 # triger archive action if storage size exceed this value, 0 means no limit, unit: GB - archive_days_threshold: 0 # files older than x days will be archived, 0 means no limit, unit: day - insert_buffer_size: 4 # maximum insert buffer size allowed, default: 4, unit: GB, should be at least 1 GB. - # the sum of insert_buffer_size and cpu_cache_capacity should be less than total memory, unit: GB - build_index_gpu: 0 # which gpu is used to build index, default: 0, range: 0 ~ gpu number - 1 -*/ - bool okay = true; - ConfigNode db_config = GetConfigNode(CONFIG_DB); - - std::string db_path = db_config.GetValue(CONFIG_DB_PATH); - if (db_path.empty()) { - std::cerr << "ERROR: db_path is empty" << std::endl; - okay = false; +Config::CheckDBConfigPath(const std::string &value) { + if (value.empty()) { + return Status(SERVER_INVALID_ARGUMENT, "DB config path empty"); } + return Status::OK(); +} - std::string db_backend_url = db_config.GetValue(CONFIG_DB_BACKEND_URL); - if (!ValidationUtil::ValidateDbURI(db_backend_url).ok()) { - std::cerr << "ERROR: invalid db_backend_url: " << db_backend_url << std::endl; - okay = false; +Status +Config::CheckDBConfigBackendUrl(const std::string &value) { + if (!ValidationUtil::ValidateDbURI(value).ok()) { + return Status(SERVER_INVALID_ARGUMENT, "Invalid DB config backend_url: " + value); } + return Status::OK(); +} - std::string archive_disk_threshold_str = - db_config.GetValue(CONFIG_DB_ARCHIVE_DISK_THRESHOLD, CONFIG_DB_ARCHIVE_DISK_THRESHOLD_DEFAULT); - if (!ValidationUtil::ValidateStringIsNumber(archive_disk_threshold_str).ok()) { - std::cerr << "ERROR: archive_disk_threshold " << archive_disk_threshold_str << " is not a number" << std::endl; - okay = false; +Status +Config::CheckDBConfigArchiveDiskThreshold(const std::string &value) { + if (!ValidationUtil::ValidateStringIsNumber(value).ok()) { + return Status(SERVER_INVALID_ARGUMENT, "Invalid DB config archive_disk_threshold: " + value); } + return Status::OK(); +} - std::string archive_days_threshold_str = - db_config.GetValue(CONFIG_DB_ARCHIVE_DAYS_THRESHOLD, CONFIG_DB_ARCHIVE_DAYS_THRESHOLD_DEFAULT); - if (!ValidationUtil::ValidateStringIsNumber(archive_days_threshold_str).ok()) { - std::cerr << "ERROR: archive_days_threshold " << archive_days_threshold_str << " is not a number" << std::endl; - okay = false; +Status +Config::CheckDBConfigArchiveDaysThreshold(const std::string &value) { + if (!ValidationUtil::ValidateStringIsNumber(value).ok()) { + return Status(SERVER_INVALID_ARGUMENT, "Invalid DB config archive_days_threshold: " + value); } + return Status::OK(); +} - std::string buffer_size_str = db_config.GetValue(CONFIG_DB_BUFFER_SIZE, CONFIG_DB_BUFFER_SIZE_DEFAULT); - if (!ValidationUtil::ValidateStringIsNumber(buffer_size_str).ok()) { - std::cerr << "ERROR: buffer_size " << buffer_size_str << " is not a number" << std::endl; - okay = false; +Status +Config::CheckDBConfigBufferSize(const std::string &value) { + if (!ValidationUtil::ValidateStringIsNumber(value).ok()) { + return Status(SERVER_INVALID_ARGUMENT, "Invalid DB config buffer_size: " + value); } else { - uint64_t buffer_size = (uint64_t) std::stol(buffer_size_str); - buffer_size *= GB; + int64_t buffer_size = std::stoi(value) * GB; unsigned long total_mem = 0, free_mem = 0; CommonUtil::GetSystemMemInfo(total_mem, free_mem); if (buffer_size >= total_mem) { - std::cerr << "ERROR: buffer_size exceed system memory" << std::endl; - okay = false; + return Status(SERVER_INVALID_ARGUMENT, "DB config buffer_size exceed system memory: " + value); } } + return Status::OK(); +} - std::string gpu_index_str = db_config.GetValue(CONFIG_DB_BUILD_INDEX_GPU, CONFIG_DB_BUILD_INDEX_GPU_DEFAULT); - if (!ValidationUtil::ValidateStringIsNumber(gpu_index_str).ok()) { - std::cerr << "ERROR: gpu_index " << gpu_index_str << " is not a number" << std::endl; - okay = false; +Status +Config::CheckDBConfigBuildIndexGPU(const std::string &value) { + if (!ValidationUtil::ValidateStringIsNumber(value).ok()) { + return Status(SERVER_INVALID_ARGUMENT, "Invalid DB config build_index_gpu: " + value); } else { - int32_t gpu_index = std::stol(gpu_index_str); + int32_t gpu_index = std::stoi(value); if (!ValidationUtil::ValidateGpuIndex(gpu_index).ok()) { - std::cerr << "ERROR: invalid gpu_index " << gpu_index_str << std::endl; - okay = false; + return Status(SERVER_INVALID_ARGUMENT, "Invalid DB config build_index_gpu: " + value); } } - - return (okay ? Status::OK() : Status(SERVER_INVALID_ARGUMENT, "DB config is illegal")); + return Status::OK(); } Status -Config::CheckMetricConfig() { -/* - metric_config: - is_startup: off # if monitoring start: on, off - collector: prometheus # metrics collector: prometheus - prometheus_config: # following are prometheus configure - port: 8080 # the port prometheus use to fetch metrics - (not used) push_gateway_ip_address: 127.0.0.1 # push method configure: push gateway ip address - (not used) push_gateway_port: 9091 # push method configure: push gateway port -*/ - bool okay = true; - ConfigNode metric_config = GetConfigNode(CONFIG_METRIC); - - std::string is_startup_str = - metric_config.GetValue(CONFIG_METRIC_AUTO_BOOTUP, CONFIG_METRIC_AUTO_BOOTUP_DEFAULT); - if (!ValidationUtil::ValidateStringIsBool(is_startup_str).ok()) { - std::cerr << "ERROR: invalid is_startup config: " << is_startup_str << std::endl; - okay = false; +Config::CheckMetricConfigAutoBootup(const std::string& value) { + if (!ValidationUtil::ValidateStringIsBool(value).ok()) { + return Status(SERVER_INVALID_ARGUMENT, "Invalid metric config auto_bootup: " + value); } - - std::string - port_str = metric_config.GetChild(CONFIG_METRIC_PROMETHEUS).GetValue(CONFIG_METRIC_PROMETHEUS_PORT, "8080"); - if (!ValidationUtil::ValidateStringIsNumber(port_str).ok()) { - std::cerr << "ERROR: port specified in prometheus_config " << port_str << " is not a number" << std::endl; - okay = false; - } - - return (okay ? Status::OK() : Status(SERVER_INVALID_ARGUMENT, "Metric config is illegal")); + return Status::OK(); } Status -Config::CheckCacheConfig() { -/* - cache_config: - cpu_cache_capacity: 16 # how many memory are used as cache, unit: GB, range: 0 ~ less than total memory - cpu_cache_free_percent: 0.85 # old data will be erased from cache when cache is full, this value specify how much memory should be kept, range: greater than zero ~ 1.0 - insert_cache_immediately: false # insert data will be load into cache immediately for hot query - gpu_cache_capacity: 5 # how many memory are used as cache in gpu, unit: GB, RANGE: 0 ~ less than total memory - gpu_cache_free_percent: 0.85 # old data will be erased from cache when cache is full, this value specify how much memory should be kept, range: greater than zero ~ 1.0 +Config::CheckMetricConfigPrometheusPort(const std::string& value) { + if (!ValidationUtil::ValidateStringIsNumber(value).ok()) { + return Status(SERVER_INVALID_ARGUMENT, "Invalid metric config prometheus_port: " + value); + } + return Status::OK(); +} -*/ - bool okay = true; - ConfigNode cache_config = GetConfigNode(CONFIG_CACHE); - - std::string cpu_cache_capacity_str = - cache_config.GetValue(CONFIG_CACHE_CPU_MEM_CAPACITY, CONFIG_CACHE_CPU_MEM_CAPACITY_DEFAULT); - if (!ValidationUtil::ValidateStringIsNumber(cpu_cache_capacity_str).ok()) { - std::cerr << "ERROR: cpu_cache_capacity " << cpu_cache_capacity_str << " is not a number" << std::endl; - okay = false; +Status +Config::CheckCacheConfigCpuMemCapacity(const std::string& value) { + if (!ValidationUtil::ValidateStringIsNumber(value).ok()) { + return Status(SERVER_INVALID_ARGUMENT, "Invalid cache config cpu_mem_capacity: " + value); } else { - uint64_t cpu_cache_capacity = (uint64_t) std::stol(cpu_cache_capacity_str); - cpu_cache_capacity *= GB; + uint64_t cpu_cache_capacity = std::stoi(value) * GB; unsigned long total_mem = 0, free_mem = 0; CommonUtil::GetSystemMemInfo(total_mem, free_mem); if (cpu_cache_capacity >= total_mem) { - std::cerr << "ERROR: cpu_cache_capacity exceed system memory" << std::endl; - okay = false; + return Status(SERVER_INVALID_ARGUMENT, "Cache config cpu_mem_capacity exceed system memory: " + value); } else if (cpu_cache_capacity > (double) total_mem * 0.9) { - std::cerr << "Warning: cpu_cache_capacity value is too aggressive" << std::endl; + std::cerr << "Warning: cpu_mem_capacity value is too big" << std::endl; } - uint64_t buffer_size = - (uint64_t) GetConfigNode(CONFIG_DB).GetInt32Value(CONFIG_DB_BUFFER_SIZE, - std::stoi(CONFIG_DB_BUFFER_SIZE_DEFAULT)); - buffer_size *= GB; - if (buffer_size + cpu_cache_capacity >= total_mem) { - std::cerr << "ERROR: sum of cpu_cache_capacity and insert_buffer_size exceed system memory" << std::endl; - okay = false; + int32_t buffer_size; + Status s = GetDBConfigBufferSize(buffer_size); + if (!s.ok()) return s; + int64_t insert_buffer_size = buffer_size * GB; + if (insert_buffer_size + cpu_cache_capacity >= total_mem) { + return Status(SERVER_INVALID_ARGUMENT, "Sum of cpu_mem_capacity and buffer_size exceed system memory"); } } + return Status::OK(); +} - std::string cpu_cache_free_percent_str = - cache_config.GetValue(CONFIG_CACHE_CPU_MEM_THRESHOLD, CONFIG_CACHE_CPU_MEM_THRESHOLD_DEFAULT); - double cpu_cache_free_percent; - if (!ValidationUtil::ValidateStringIsDouble(cpu_cache_free_percent_str, cpu_cache_free_percent).ok()) { - std::cerr << "ERROR: cpu_cache_free_percent " << cpu_cache_free_percent_str << " is not a double" << std::endl; - okay = false; - } else if (cpu_cache_free_percent < std::numeric_limits::epsilon() || cpu_cache_free_percent > 1.0) { - std::cerr << "ERROR: invalid cpu_cache_free_percent " << cpu_cache_free_percent_str << std::endl; - okay = false; - } - - std::string insert_cache_immediately_str = - cache_config.GetValue(CONFIG_CACHE_CACHE_INSERT_DATA, CONFIG_CACHE_CACHE_INSERT_DATA_DEFAULT); - if (!ValidationUtil::ValidateStringIsBool(insert_cache_immediately_str).ok()) { - std::cerr << "ERROR: invalid insert_cache_immediately config: " << insert_cache_immediately_str << std::endl; - okay = false; - } - - std::string gpu_cache_capacity_str = - cache_config.GetValue(CONFIG_CACHE_GPU_MEM_CAPACITY, CONFIG_CACHE_GPU_MEM_CAPACITY_DEFAULT); - if (!ValidationUtil::ValidateStringIsNumber(gpu_cache_capacity_str).ok()) { - std::cerr << "ERROR: gpu_cache_capacity " << gpu_cache_capacity_str << " is not a number" << std::endl; - okay = false; +Status +Config::CheckCacheConfigCpuMemThreshold(const std::string& value) { + if (!ValidationUtil::ValidateStringIsFloat(value).ok()) { + return Status(SERVER_INVALID_ARGUMENT, "Invalid cache config cpu_mem_threshold: " + value); } else { - uint64_t gpu_cache_capacity = (uint64_t) std::stol(gpu_cache_capacity_str); - gpu_cache_capacity *= GB; - int gpu_index = GetConfigNode(CONFIG_DB).GetInt32Value(CONFIG_DB_BUILD_INDEX_GPU, - std::stoi(CONFIG_DB_BUILD_INDEX_GPU_DEFAULT)); + float cpu_mem_threshold = std::stof(value); + if (cpu_mem_threshold <= 0.0 || cpu_mem_threshold >= 1.0) { + return Status(SERVER_INVALID_ARGUMENT, "Invalid cache config cpu_mem_threshold: " + value); + } + } + return Status::OK(); +} + +Status +Config::CheckCacheConfigGpuMemCapacity(const std::string& value) { + if (!ValidationUtil::ValidateStringIsNumber(value).ok()) { + std::cerr << "ERROR: gpu_cache_capacity " << value << " is not a number" << std::endl; + } else { + uint64_t gpu_cache_capacity = std::stoi(value) * GB; + int gpu_index; + Status s = GetDBConfigBuildIndexGPU(gpu_index); + if (!s.ok()) return s; size_t gpu_memory; if (!ValidationUtil::GetGpuMemory(gpu_index, gpu_memory).ok()) { - std::cerr << "ERROR: could not get gpu memory for device " << gpu_index << std::endl; - okay = false; + return Status(SERVER_UNEXPECTED_ERROR, + "Fail to get GPU memory for GPU device: " + std::to_string(gpu_index)); } else if (gpu_cache_capacity >= gpu_memory) { - std::cerr << "ERROR: gpu_cache_capacity " << gpu_cache_capacity - << " exceed total gpu memory " << gpu_memory << std::endl; - okay = false; + return Status(SERVER_INVALID_ARGUMENT, + "Cache config gpu_mem_capacity exceed GPU memory: " + std::to_string(gpu_memory)); } else if (gpu_cache_capacity > (double) gpu_memory * 0.9) { - std::cerr << "Warning: gpu_cache_capacity value is too aggressive" << std::endl; + std::cerr << "Warning: gpu_mem_capacity value is too big" << std::endl; } } - - std::string gpu_cache_free_percent_str = - cache_config.GetValue(CONFIG_CACHE_GPU_MEM_THRESHOLD, CONFIG_CACHE_GPU_MEM_THRESHOLD_DEFAULT); - double gpu_cache_free_percent; - if (!ValidationUtil::ValidateStringIsDouble(gpu_cache_free_percent_str, gpu_cache_free_percent).ok()) { - std::cerr << "ERROR: gpu_cache_free_percent " << gpu_cache_free_percent_str << " is not a double" << std::endl; - okay = false; - } else if (gpu_cache_free_percent < std::numeric_limits::epsilon() || gpu_cache_free_percent > 1.0) { - std::cerr << "ERROR: invalid gpu_cache_free_percent " << gpu_cache_free_percent << std::endl; - okay = false; - } - - return (okay ? Status::OK() : Status(SERVER_INVALID_ARGUMENT, "Cache config is illegal")); + return Status::OK(); } Status -Config::CheckEngineConfig() { -/* - engine_config: - use_blas_threshold: 20 - omp_thread_num: 0 # how many compute threads be used by engine, 0 means use all cpu core to compute -*/ - bool okay = true; - ConfigNode engine_config = GetConfigNode(CONFIG_ENGINE); - - std::string use_blas_threshold_str = - engine_config.GetValue(CONFIG_ENGINE_BLAS_THRESHOLD, CONFIG_ENGINE_BLAS_THRESHOLD_DEFAULT); - if (!ValidationUtil::ValidateStringIsNumber(use_blas_threshold_str).ok()) { - std::cerr << "ERROR: use_blas_threshold " << use_blas_threshold_str << " is not a number" << std::endl; - okay = false; - } - - std::string omp_thread_num_str = - engine_config.GetValue(CONFIG_ENGINE_OMP_THREAD_NUM, CONFIG_ENGINE_OMP_THREAD_NUM_DEFAULT); - if (!ValidationUtil::ValidateStringIsNumber(omp_thread_num_str).ok()) { - std::cerr << "ERROR: omp_thread_num " << omp_thread_num_str << " is not a number" << std::endl; - okay = false; +Config::CheckCacheConfigGpuMemThreshold(const std::string& value) { + if (!ValidationUtil::ValidateStringIsFloat(value).ok()) { + return Status(SERVER_INVALID_ARGUMENT, "Invalid cache config gpu_mem_threshold: " + value); } else { - int32_t omp_thread = std::stol(omp_thread_num_str); + float gpu_mem_threshold = std::stof(value); + if (gpu_mem_threshold <= 0.0 || gpu_mem_threshold >= 1.0) { + return Status(SERVER_INVALID_ARGUMENT, "Invalid cache config gpu_mem_threshold: " + value); + } + } + return Status::OK(); +} + +Status +Config::CheckCacheConfigCacheInsertData(const std::string& value) { + if (!ValidationUtil::ValidateStringIsBool(value).ok()) { + return Status(SERVER_INVALID_ARGUMENT, "Invalid cache config cache_insert_data: " + value); + } + return Status::OK(); +} + +Status +Config::CheckEngineConfigBlasThreshold(const std::string& value) { + if (!ValidationUtil::ValidateStringIsNumber(value).ok()) { + return Status(SERVER_INVALID_ARGUMENT, "Invalid engine config blas threshold: " + value); + } + return Status::OK(); +} + +Status +Config::CheckEngineConfigOmpThreadNum(const std::string& value) { + if (!ValidationUtil::ValidateStringIsNumber(value).ok()) { + return Status(SERVER_INVALID_ARGUMENT, "Invalid engine config omp_thread_num: " + value); + } else { + int32_t omp_thread = std::stoi(value); uint32_t sys_thread_cnt = 8; if (omp_thread > CommonUtil::GetSystemAvailableThreads(sys_thread_cnt)) { - std::cerr << "ERROR: omp_thread_num " << omp_thread_num_str << " > system available thread " - << sys_thread_cnt << std::endl; - okay = false; + return Status(SERVER_INVALID_ARGUMENT, "Invalid engine config omp_thread_num: " + value); } } - - return (okay ? Status::OK() : Status(SERVER_INVALID_ARGUMENT, "Engine config is illegal")); + return Status::OK(); } Status -Config::CheckResourceConfig() { - /* - resource_config: - mode: simple - pool: - - cpu - - gpu0 - - gpu100 - */ - bool okay = true; - server::ConfigNode &config = GetConfigNode(server::CONFIG_RESOURCE); - auto mode = config.GetValue(CONFIG_RESOURCE_MODE, CONFIG_RESOURCE_MODE_DEFAULT); - if (mode != "simple") { - std::cerr << "ERROR: invalid resource config: mode is " << mode << std::endl; - okay = false; +Config::CheckResourceConfigMode(const std::string& value) { + if (value != "simple") { + return Status(SERVER_INVALID_ARGUMENT, "Invalid resource config mode: " + value); } - auto pool = config.GetSequence(CONFIG_RESOURCE_POOL); - if (pool.empty()) { - std::cerr << "ERROR: invalid resource config: resources empty" << std::endl; - okay = false; - } - - return (okay ? Status::OK() : Status(SERVER_INVALID_ARGUMENT, "Resource config is illegal")); + return Status::OK(); } -//Status -//Config::CheckResourceConfig() { -/* - resource_config: - # resource list, length: 0~N - # please set a DISK resource and a CPU resource least, or system will not return query result. - # - # example: - # resource_name: # resource name, just using in connections below - # type: DISK # resource type, optional: DISK/CPU/GPU - # device_id: 0 - # enable_executor: false # if is enable executor, optional: true, false - - resources: - ssda: - type: DISK - device_id: 0 - enable_executor: false - - cpu: - type: CPU - device_id: 0 - enable_executor: true - - gpu0: - type: GPU - device_id: 0 - enable_executor: false - gpu_resource_num: 2 - pinned_memory: 300 - temp_memory: 300 - - # connection list, length: 0~N - # example: - # connection_name: - # speed: 100 # unit: MS/s - # endpoint: ${resource_name}===${resource_name} - connections: - io: - speed: 500 - endpoint: ssda===cpu - pcie0: - speed: 11000 - endpoint: cpu===gpu0 -*/ -// bool okay = true; -// server::ConfigNode resource_config = GetConfig(CONFIG_RESOURCE); -// if (resource_config.GetChildren().empty()) { -// std::cerr << "ERROR: no context under resource" << std::endl; -// okay = false; -// } -// -// auto resources = resource_config.GetChild(CONFIG_RESOURCES).GetChildren(); -// -// if (resources.empty()) { -// std::cerr << "no resources specified" << std::endl; -// okay = false; -// } -// -// bool resource_valid_flag = false; -// bool hasDisk = false; -// bool hasCPU = false; -// bool hasExecutor = false; -// std::set resource_list; -// for (auto &resource : resources) { -// resource_list.emplace(resource.first); -// auto &resource_conf = resource.second; -// auto type = resource_conf.GetValue(CONFIG_RESOURCE_TYPE); -// -// std::string device_id_str = resource_conf.GetValue(CONFIG_RESOURCE_DEVICE_ID, "0"); -// int32_t device_id = -1; -// if (!ValidationUtil::ValidateStringIsNumber(device_id_str).ok()) { -// std::cerr << "ERROR: device_id " << device_id_str << " is not a number" << std::endl; -// okay = false; -// } else { -// device_id = std::stol(device_id_str); -// } -// -// std::string enable_executor_str = resource_conf.GetValue(CONFIG_RESOURCE_ENABLE_EXECUTOR, "off"); -// if (!ValidationUtil::ValidateStringIsBool(enable_executor_str).ok()) { -// std::cerr << "ERROR: invalid enable_executor config: " << enable_executor_str << std::endl; -// okay = false; -// } -// -// if (type == "DISK") { -// hasDisk = true; -// } else if (type == "CPU") { -// hasCPU = true; -// if (resource_conf.GetBoolValue(CONFIG_RESOURCE_ENABLE_EXECUTOR, false)) { -// hasExecutor = true; -// } -// } -// else if (type == "GPU") { -// int build_index_gpu_index = GetConfig(CONFIG_DB).GetInt32Value(CONFIG_DB_BUILD_INDEX_GPU, 0); -// if (device_id == build_index_gpu_index) { -// resource_valid_flag = true; -// } -// if (resource_conf.GetBoolValue(CONFIG_RESOURCE_ENABLE_EXECUTOR, false)) { -// hasExecutor = true; -// } -// std::string gpu_resource_num_str = resource_conf.GetValue(CONFIG_RESOURCE_NUM, "2"); -// if (!ValidationUtil::ValidateStringIsNumber(gpu_resource_num_str).ok()) { -// std::cerr << "ERROR: gpu_resource_num " << gpu_resource_num_str << " is not a number" << std::endl; -// okay = false; -// } -// bool mem_valid = true; -// std::string pinned_memory_str = resource_conf.GetValue(CONFIG_RESOURCE_PIN_MEMORY, "300"); -// if (!ValidationUtil::ValidateStringIsNumber(pinned_memory_str).ok()) { -// std::cerr << "ERROR: pinned_memory " << pinned_memory_str << " is not a number" << std::endl; -// okay = false; -// mem_valid = false; -// } -// std::string temp_memory_str = resource_conf.GetValue(CONFIG_RESOURCE_TEMP_MEMORY, "300"); -// if (!ValidationUtil::ValidateStringIsNumber(temp_memory_str).ok()) { -// std::cerr << "ERROR: temp_memory " << temp_memory_str << " is not a number" << std::endl; -// okay = false; -// mem_valid = false; -// } -// if (mem_valid) { -// size_t gpu_memory; -// if (!ValidationUtil::GetGpuMemory(device_id, gpu_memory).ok()) { -// std::cerr << "ERROR: could not get gpu memory for device " << device_id << std::endl; -// okay = false; -// } -// else { -// size_t prealoc_mem = std::stol(pinned_memory_str) + std::stol(temp_memory_str); -// if (prealoc_mem >= gpu_memory) { -// std::cerr << "ERROR: sum of pinned_memory and temp_memory " << prealoc_mem -// << " exceeds total gpu memory " << gpu_memory << " for device " << device_id << std::endl; -// okay = false; -// } -// } -// } -// } -// } -// -// if (!resource_valid_flag) { -// std::cerr << "Building index GPU can't be found in resource config." << std::endl; -// okay = false; -// } -// if (!hasDisk || !hasCPU) { -// std::cerr << "No DISK or CPU resource" << std::endl; -// okay = false; -// } -// if (!hasExecutor) { -// std::cerr << "No CPU or GPU resource has executor enabled" << std::endl; -// okay = false; -// } -// -// auto connections = resource_config.GetChild(CONFIG_RESOURCE_CONNECTIONS).GetChildren(); -// for (auto &connection : connections) { -// auto &connection_conf = connection.second; -// -// std::string speed_str = connection_conf.GetValue(CONFIG_SPEED_CONNECTIONS); -// if (ValidationUtil::ValidateStringIsNumber(speed_str) != SERVER_SUCCESS) { -// std::cerr << "ERROR: speed " << speed_str << " is not a number" << std::endl; -// okay = false; -// } -// -// std::string endpoint_str = connection_conf.GetValue(CONFIG_ENDPOINT_CONNECTIONS); -// std::string delimiter = "==="; -// auto delimiter_pos = endpoint_str.find(delimiter); -// if (delimiter_pos == std::string::npos) { -// std::cerr << "ERROR: invalid endpoint format: " << endpoint_str << std::endl; -// okay = false; -// } else { -// std::string left_resource = endpoint_str.substr(0, delimiter_pos); -// if (resource_list.find(left_resource) == resource_list.end()) { -// std::cerr << "ERROR: left resource " << left_resource << " does not exist" << std::endl; -// okay = false; -// } -// std::string right_resource = endpoint_str.substr(delimiter_pos + delimiter.length(), endpoint_str.length()); -// if (resource_list.find(right_resource) == resource_list.end()) { -// std::cerr << "ERROR: right resource " << right_resource << " does not exist" << std::endl; -// okay = false; -// } -// } -// } -// -// return (okay ? Status::OK() : Status(SERVER_INVALID_ARGUMENT, "Resource config is illegal")); -//} - -void -Config::PrintAll() const { - if (const ConfigMgr *mgr = ConfigMgr::GetInstance()) { - std::string str = mgr->DumpString(); -// SERVER_LOG_INFO << "\n" << str; - std::cout << "\n" << str << std::endl; +Status +Config::CheckResourceConfigPool(const std::vector& value) { + if (value.empty()) { + return Status(SERVER_INVALID_ARGUMENT, "Invalid resource config pool"); } + return Status::OK(); } +//////////////////////////////////////////////////////////////////////////////// ConfigNode & Config::GetConfigNode(const std::string &name) { ConfigMgr *mgr = ConfigMgr::GetInstance(); @@ -631,310 +357,316 @@ Config::GetConfigValueInMem(const std::string &parent_key, } } -Status +void Config::SetConfigValueInMem(const std::string &parent_key, const std::string &child_key, std::string &value) { std::lock_guard lock(mutex_); config_map_[parent_key][child_key] = value; - return Status::OK(); } //////////////////////////////////////////////////////////////////////////////// /* server config */ -Status -Config::GetServerConfigStrAddress(std::string& value) { - if (GetConfigValueInMem(CONFIG_SERVER, CONFIG_SERVER_ADDRESS, value).ok()) { - return Status::OK(); - } else { +std::string +Config::GetServerConfigStrAddress() { + std::string value; + if (!GetConfigValueInMem(CONFIG_SERVER, CONFIG_SERVER_ADDRESS, value).ok()) { value = GetConfigNode(CONFIG_SERVER).GetValue(CONFIG_SERVER_ADDRESS, CONFIG_SERVER_ADDRESS_DEFAULT); - return SetConfigValueInMem(CONFIG_SERVER, CONFIG_SERVER_ADDRESS, value); + SetConfigValueInMem(CONFIG_SERVER, CONFIG_SERVER_ADDRESS, value); } + return value; } -Status -Config::GetServerConfigStrPort(std::string& value) { - if (GetConfigValueInMem(CONFIG_SERVER, CONFIG_SERVER_PORT, value).ok()) { - return Status::OK(); - } else { +std::string +Config::GetServerConfigStrPort() { + std::string value; + if (!GetConfigValueInMem(CONFIG_SERVER, CONFIG_SERVER_PORT, value).ok()) { value = GetConfigNode(CONFIG_SERVER).GetValue(CONFIG_SERVER_PORT, CONFIG_SERVER_PORT_DEFAULT); - return SetConfigValueInMem(CONFIG_SERVER, CONFIG_SERVER_PORT, value); + SetConfigValueInMem(CONFIG_SERVER, CONFIG_SERVER_PORT, value); } + return value; } -Status -Config::GetServerConfigStrMode(std::string& value) { - if (GetConfigValueInMem(CONFIG_SERVER, CONFIG_SERVER_MODE, value).ok()) { - return Status::OK(); - } else { +std::string +Config::GetServerConfigStrMode() { + std::string value; + if (!GetConfigValueInMem(CONFIG_SERVER, CONFIG_SERVER_MODE, value).ok()) { value = GetConfigNode(CONFIG_SERVER).GetValue(CONFIG_SERVER_MODE, CONFIG_SERVER_MODE_DEFAULT); - return SetConfigValueInMem(CONFIG_SERVER, CONFIG_SERVER_MODE, value); + SetConfigValueInMem(CONFIG_SERVER, CONFIG_SERVER_MODE, value); } + return value; } -Status -Config::GetServerConfigStrTimeZone(std::string& value) { - if (GetConfigValueInMem(CONFIG_SERVER, CONFIG_SERVER_TIME_ZONE, value).ok()) { - return Status::OK(); - } else { +std::string +Config::GetServerConfigStrTimeZone() { + std::string value; + if (!GetConfigValueInMem(CONFIG_SERVER, CONFIG_SERVER_TIME_ZONE, value).ok()) { value = GetConfigNode(CONFIG_SERVER).GetValue(CONFIG_SERVER_TIME_ZONE, CONFIG_SERVER_TIME_ZONE_DEFAULT); - return SetConfigValueInMem(CONFIG_SERVER, CONFIG_SERVER_TIME_ZONE, value); + SetConfigValueInMem(CONFIG_SERVER, CONFIG_SERVER_TIME_ZONE, value); } + return value; } //////////////////////////////////////////////////////////////////////////////// /* db config */ -Status -Config::GetDBConfigStrPath(std::string& value) { - if (GetConfigValueInMem(CONFIG_DB, CONFIG_DB_PATH, value).ok()) { - return Status::OK(); - } else { +std::string +Config::GetDBConfigStrPath() { + std::string value; + if (!GetConfigValueInMem(CONFIG_DB, CONFIG_DB_PATH, value).ok()) { value = GetConfigNode(CONFIG_DB).GetValue(CONFIG_DB_PATH, CONFIG_DB_PATH_DEFAULT); - return SetConfigValueInMem(CONFIG_DB, CONFIG_DB_PATH, value); + SetConfigValueInMem(CONFIG_DB, CONFIG_DB_PATH, value); } + return value; } -Status -Config::GetDBConfigStrSlavePath(std::string& value) { - if (GetConfigValueInMem(CONFIG_DB, CONFIG_DB_SLAVE_PATH, value).ok()) { - return Status::OK(); - } else { +std::string +Config::GetDBConfigStrSlavePath() { + std::string value; + if (!GetConfigValueInMem(CONFIG_DB, CONFIG_DB_SLAVE_PATH, value).ok()) { value = GetConfigNode(CONFIG_DB).GetValue(CONFIG_DB_SLAVE_PATH, CONFIG_DB_SLAVE_PATH_DEFAULT); - return SetConfigValueInMem(CONFIG_DB, CONFIG_DB_SLAVE_PATH, value); + SetConfigValueInMem(CONFIG_DB, CONFIG_DB_SLAVE_PATH, value); } + return value; } -Status -Config::GetDBConfigStrBackendUrl(std::string& value) { - if (GetConfigValueInMem(CONFIG_DB, CONFIG_DB_BACKEND_URL, value).ok()) { - return Status::OK(); - } else { +std::string +Config::GetDBConfigStrBackendUrl() { + std::string value; + if (!GetConfigValueInMem(CONFIG_DB, CONFIG_DB_BACKEND_URL, value).ok()) { value = GetConfigNode(CONFIG_DB).GetValue(CONFIG_DB_BACKEND_URL, CONFIG_DB_BACKEND_URL_DEFAULT); - return SetConfigValueInMem(CONFIG_DB, CONFIG_DB_BACKEND_URL, value); + SetConfigValueInMem(CONFIG_DB, CONFIG_DB_BACKEND_URL, value); } + return value; } -Status -Config::GetDBConfigStrArchiveDiskThreshold(std::string& value) { - if (GetConfigValueInMem(CONFIG_DB, CONFIG_DB_ARCHIVE_DISK_THRESHOLD, value).ok()) { - return Status::OK(); - } else { +std::string +Config::GetDBConfigStrArchiveDiskThreshold() { + std::string value; + if (!GetConfigValueInMem(CONFIG_DB, CONFIG_DB_ARCHIVE_DISK_THRESHOLD, value).ok()) { value = GetConfigNode(CONFIG_DB).GetValue(CONFIG_DB_ARCHIVE_DISK_THRESHOLD, CONFIG_DB_ARCHIVE_DISK_THRESHOLD_DEFAULT); - return SetConfigValueInMem(CONFIG_DB, CONFIG_DB_ARCHIVE_DISK_THRESHOLD, value); + SetConfigValueInMem(CONFIG_DB, CONFIG_DB_ARCHIVE_DISK_THRESHOLD, value); } + return value; } -Status -Config::GetDBConfigStrArchiveDaysThreshold(std::string& value) { - if (GetConfigValueInMem(CONFIG_DB, CONFIG_DB_ARCHIVE_DAYS_THRESHOLD, value).ok()) { - return Status::OK(); - } else { +std::string +Config::GetDBConfigStrArchiveDaysThreshold() { + std::string value; + if (!GetConfigValueInMem(CONFIG_DB, CONFIG_DB_ARCHIVE_DAYS_THRESHOLD, value).ok()) { value = GetConfigNode(CONFIG_DB).GetValue(CONFIG_DB_ARCHIVE_DAYS_THRESHOLD, CONFIG_DB_ARCHIVE_DAYS_THRESHOLD_DEFAULT); - return SetConfigValueInMem(CONFIG_DB, CONFIG_DB_ARCHIVE_DAYS_THRESHOLD, value); + SetConfigValueInMem(CONFIG_DB, CONFIG_DB_ARCHIVE_DAYS_THRESHOLD, value); } + return value; } -Status -Config::GetDBConfigStrBufferSize(std::string& value) { - if (GetConfigValueInMem(CONFIG_DB, CONFIG_DB_BUFFER_SIZE, value).ok()) { - return Status::OK(); - } else { +std::string +Config::GetDBConfigStrBufferSize() { + std::string value; + if (!GetConfigValueInMem(CONFIG_DB, CONFIG_DB_BUFFER_SIZE, value).ok()) { value = GetConfigNode(CONFIG_DB).GetValue(CONFIG_DB_BUFFER_SIZE, CONFIG_DB_BUFFER_SIZE_DEFAULT); - return SetConfigValueInMem(CONFIG_DB, CONFIG_DB_BUFFER_SIZE, value); + SetConfigValueInMem(CONFIG_DB, CONFIG_DB_BUFFER_SIZE, value); } + return value; } -Status -Config::GetDBConfigStrBuildIndexGPU(std::string& value) { - if (GetConfigValueInMem(CONFIG_DB, CONFIG_DB_BUILD_INDEX_GPU, value).ok()) { - return Status::OK(); - } else { +std::string +Config::GetDBConfigStrBuildIndexGPU() { + std::string value; + if (!GetConfigValueInMem(CONFIG_DB, CONFIG_DB_BUILD_INDEX_GPU, value).ok()) { value = GetConfigNode(CONFIG_DB).GetValue(CONFIG_DB_BUILD_INDEX_GPU, CONFIG_DB_BUILD_INDEX_GPU_DEFAULT); - return SetConfigValueInMem(CONFIG_DB, CONFIG_DB_BUILD_INDEX_GPU, value); + SetConfigValueInMem(CONFIG_DB, CONFIG_DB_BUILD_INDEX_GPU, value); } + return value; } //////////////////////////////////////////////////////////////////////////////// /* metric config */ -Status -Config::GetMetricConfigStrAutoBootup(std::string& value) { - if (GetConfigValueInMem(CONFIG_METRIC, CONFIG_METRIC_AUTO_BOOTUP, value).ok()) { - return Status::OK(); - } else { +std::string +Config::GetMetricConfigStrAutoBootup() { + std::string value; + if (!GetConfigValueInMem(CONFIG_METRIC, CONFIG_METRIC_AUTO_BOOTUP, value).ok()) { value = GetConfigNode(CONFIG_METRIC).GetValue(CONFIG_METRIC_AUTO_BOOTUP, CONFIG_METRIC_AUTO_BOOTUP_DEFAULT); - return SetConfigValueInMem(CONFIG_METRIC, CONFIG_METRIC_AUTO_BOOTUP, value); + SetConfigValueInMem(CONFIG_METRIC, CONFIG_METRIC_AUTO_BOOTUP, value); } + return value; } -Status -Config::GetMetricConfigStrCollector(std::string& value) { - if (GetConfigValueInMem(CONFIG_METRIC, CONFIG_METRIC_COLLECTOR, value).ok()) { - return Status::OK(); - } else { +std::string +Config::GetMetricConfigStrCollector() { + std::string value; + if (!GetConfigValueInMem(CONFIG_METRIC, CONFIG_METRIC_COLLECTOR, value).ok()) { value = GetConfigNode(CONFIG_METRIC).GetValue(CONFIG_METRIC_COLLECTOR, CONFIG_METRIC_COLLECTOR_DEFAULT); - return SetConfigValueInMem(CONFIG_METRIC, CONFIG_METRIC_COLLECTOR, value); + SetConfigValueInMem(CONFIG_METRIC, CONFIG_METRIC_COLLECTOR, value); } + return value; } -Status -Config::GetMetricConfigStrPrometheusPort(std::string& value) { - if (GetConfigValueInMem(CONFIG_METRIC, CONFIG_METRIC_PROMETHEUS_PORT, value).ok()) { - return Status::OK(); - } else { +std::string +Config::GetMetricConfigStrPrometheusPort() { + std::string value; + if (!GetConfigValueInMem(CONFIG_METRIC, CONFIG_METRIC_PROMETHEUS_PORT, value).ok()) { value = GetConfigNode(CONFIG_METRIC).GetValue(CONFIG_METRIC_PROMETHEUS_PORT, CONFIG_METRIC_PROMETHEUS_PORT_DEFAULT); - return SetConfigValueInMem(CONFIG_METRIC, CONFIG_METRIC_PROMETHEUS_PORT, value); + SetConfigValueInMem(CONFIG_METRIC, CONFIG_METRIC_PROMETHEUS_PORT, value); } + return value; } //////////////////////////////////////////////////////////////////////////////// /* cache config */ -Status -Config::GetCacheConfigStrCpuMemCapacity(std::string& value) { - if (GetConfigValueInMem(CONFIG_CACHE, CONFIG_CACHE_CPU_MEM_CAPACITY, value).ok()) { - return Status::OK(); - } else { +std::string +Config::GetCacheConfigStrCpuMemCapacity() { + std::string value; + if (!GetConfigValueInMem(CONFIG_CACHE, CONFIG_CACHE_CPU_MEM_CAPACITY, value).ok()) { value = GetConfigNode(CONFIG_CACHE).GetValue(CONFIG_CACHE_CPU_MEM_CAPACITY, CONFIG_CACHE_CPU_MEM_CAPACITY_DEFAULT); - return SetConfigValueInMem(CONFIG_CACHE, CONFIG_CACHE_CPU_MEM_CAPACITY, value); + SetConfigValueInMem(CONFIG_CACHE, CONFIG_CACHE_CPU_MEM_CAPACITY, value); } + return value; } -Status -Config::GetCacheConfigStrCpuMemThreshold(std::string& value) { - if (GetConfigValueInMem(CONFIG_CACHE, CONFIG_CACHE_CPU_MEM_THRESHOLD, value).ok()) { - return Status::OK(); - } else { +std::string +Config::GetCacheConfigStrCpuMemThreshold() { + std::string value; + if (!GetConfigValueInMem(CONFIG_CACHE, CONFIG_CACHE_CPU_MEM_THRESHOLD, value).ok()) { value = GetConfigNode(CONFIG_CACHE).GetValue(CONFIG_CACHE_CPU_MEM_THRESHOLD, CONFIG_CACHE_CPU_MEM_THRESHOLD_DEFAULT); - return SetConfigValueInMem(CONFIG_CACHE, CONFIG_CACHE_CPU_MEM_THRESHOLD, value); + SetConfigValueInMem(CONFIG_CACHE, CONFIG_CACHE_CPU_MEM_THRESHOLD, value); } + return value; } -Status -Config::GetCacheConfigStrGpuMemCapacity(std::string& value) { - if (GetConfigValueInMem(CONFIG_CACHE, CONFIG_CACHE_GPU_MEM_CAPACITY, value).ok()) { - return Status::OK(); - } else { +std::string +Config::GetCacheConfigStrGpuMemCapacity() { + std::string value; + if (!GetConfigValueInMem(CONFIG_CACHE, CONFIG_CACHE_GPU_MEM_CAPACITY, value).ok()) { value = GetConfigNode(CONFIG_CACHE).GetValue(CONFIG_CACHE_GPU_MEM_CAPACITY, CONFIG_CACHE_GPU_MEM_CAPACITY_DEFAULT); - return SetConfigValueInMem(CONFIG_CACHE, CONFIG_CACHE_GPU_MEM_CAPACITY, value); + SetConfigValueInMem(CONFIG_CACHE, CONFIG_CACHE_GPU_MEM_CAPACITY, value); } + return value; } -Status -Config::GetCacheConfigStrGpuMemThreshold(std::string& value) { - if (GetConfigValueInMem(CONFIG_CACHE, CONFIG_CACHE_GPU_MEM_THRESHOLD, value).ok()) { - return Status::OK(); - } else { +std::string +Config::GetCacheConfigStrGpuMemThreshold() { + std::string value; + if (!GetConfigValueInMem(CONFIG_CACHE, CONFIG_CACHE_GPU_MEM_THRESHOLD, value).ok()) { value = GetConfigNode(CONFIG_CACHE).GetValue(CONFIG_CACHE_GPU_MEM_THRESHOLD, CONFIG_CACHE_GPU_MEM_THRESHOLD_DEFAULT); - return SetConfigValueInMem(CONFIG_CACHE, CONFIG_CACHE_GPU_MEM_THRESHOLD, value); + SetConfigValueInMem(CONFIG_CACHE, CONFIG_CACHE_GPU_MEM_THRESHOLD, value); } + return value; } -Status -Config::GetCacheConfigStrCacheInsertData(std::string& value) { - if (GetConfigValueInMem(CONFIG_CACHE, CONFIG_CACHE_CACHE_INSERT_DATA, value).ok()) { - return Status::OK(); - } else { +std::string +Config::GetCacheConfigStrCacheInsertData() { + std::string value; + if (!GetConfigValueInMem(CONFIG_CACHE, CONFIG_CACHE_CACHE_INSERT_DATA, value).ok()) { value = GetConfigNode(CONFIG_CACHE).GetValue(CONFIG_CACHE_CACHE_INSERT_DATA, CONFIG_CACHE_CACHE_INSERT_DATA_DEFAULT); - return SetConfigValueInMem(CONFIG_CACHE, CONFIG_CACHE_CACHE_INSERT_DATA, value); + SetConfigValueInMem(CONFIG_CACHE, CONFIG_CACHE_CACHE_INSERT_DATA, value); } + return value; } //////////////////////////////////////////////////////////////////////////////// /* engine config */ -Status -Config::GetEngineConfigStrBlasThreshold(std::string& value) { - if (GetConfigValueInMem(CONFIG_ENGINE, CONFIG_ENGINE_BLAS_THRESHOLD, value).ok()) { - return Status::OK(); - } else { +std::string +Config::GetEngineConfigStrBlasThreshold() { + std::string value; + if (!GetConfigValueInMem(CONFIG_ENGINE, CONFIG_ENGINE_BLAS_THRESHOLD, value).ok()) { value = GetConfigNode(CONFIG_ENGINE).GetValue(CONFIG_ENGINE_BLAS_THRESHOLD, CONFIG_ENGINE_BLAS_THRESHOLD_DEFAULT); - return SetConfigValueInMem(CONFIG_ENGINE, CONFIG_ENGINE_BLAS_THRESHOLD, value); + SetConfigValueInMem(CONFIG_ENGINE, CONFIG_ENGINE_BLAS_THRESHOLD, value); } + return value; } -Status -Config::GetEngineConfigStrOmpThreadNum(std::string& value) { - if (GetConfigValueInMem(CONFIG_ENGINE, CONFIG_ENGINE_OMP_THREAD_NUM, value).ok()) { - return Status::OK(); - } else { +std::string +Config::GetEngineConfigStrOmpThreadNum() { + std::string value; + if (!GetConfigValueInMem(CONFIG_ENGINE, CONFIG_ENGINE_OMP_THREAD_NUM, value).ok()) { value = GetConfigNode(CONFIG_ENGINE).GetValue(CONFIG_ENGINE_OMP_THREAD_NUM, CONFIG_ENGINE_OMP_THREAD_NUM_DEFAULT); - return SetConfigValueInMem(CONFIG_ENGINE, CONFIG_ENGINE_OMP_THREAD_NUM, value); + SetConfigValueInMem(CONFIG_ENGINE, CONFIG_ENGINE_OMP_THREAD_NUM, value); } + return value; } //////////////////////////////////////////////////////////////////////////////// /* resource config */ -Status -Config::GetResourceConfigStrMode(std::string& value) { - if (GetConfigValueInMem(CONFIG_RESOURCE, CONFIG_RESOURCE_MODE, value).ok()) { - return Status::OK(); - } else { +std::string +Config::GetResourceConfigStrMode() { + std::string value; + if (!GetConfigValueInMem(CONFIG_RESOURCE, CONFIG_RESOURCE_MODE, value).ok()) { value = GetConfigNode(CONFIG_RESOURCE).GetValue(CONFIG_RESOURCE_MODE, CONFIG_RESOURCE_MODE_DEFAULT); - return SetConfigValueInMem(CONFIG_RESOURCE, CONFIG_RESOURCE_MODE, value); + SetConfigValueInMem(CONFIG_RESOURCE, CONFIG_RESOURCE_MODE, value); } + return value; } //////////////////////////////////////////////////////////////////////////////// Status Config::GetServerConfigAddress(std::string& value) { - return GetServerConfigStrAddress(value); + value = GetServerConfigStrAddress(); + return CheckServerConfigAddress(value); } Status Config::GetServerConfigPort(std::string& value) { - return GetServerConfigStrPort(value); + value = GetServerConfigStrPort(); + return CheckServerConfigPort(value); } Status Config::GetServerConfigMode(std::string& value) { - return GetServerConfigStrMode(value); + value = GetServerConfigStrMode(); + return CheckServerConfigMode(value); } Status Config::GetServerConfigTimeZone(std::string& value) { - return GetServerConfigStrTimeZone(value); + value = GetServerConfigStrTimeZone(); + return CheckServerConfigTimeZone(value); } Status Config::GetDBConfigPath(std::string& value) { - return GetDBConfigStrPath(value); + value = GetDBConfigStrPath(); + return CheckDBConfigPath(value); } Status Config::GetDBConfigSlavePath(std::string& value) { - return GetDBConfigStrSlavePath(value); + value = GetDBConfigStrSlavePath(); + return Status::OK(); } Status Config::GetDBConfigBackendUrl(std::string& value) { - return GetDBConfigStrBackendUrl(value); + value = GetDBConfigStrBackendUrl(); + return CheckDBConfigBackendUrl(value); } Status Config::GetDBConfigArchiveDiskThreshold(int32_t& value) { - std::string str; - Status s = GetDBConfigStrArchiveDiskThreshold(str); + std::string str = GetDBConfigStrArchiveDiskThreshold(); + Status s = CheckDBConfigArchiveDiskThreshold(str); if (!s.ok()) return s; value = std::stoi(str); return Status::OK(); @@ -942,8 +674,8 @@ Config::GetDBConfigArchiveDiskThreshold(int32_t& value) { Status Config::GetDBConfigArchiveDaysThreshold(int32_t& value) { - std::string str; - Status s = GetDBConfigStrArchiveDaysThreshold(str); + std::string str = GetDBConfigStrArchiveDaysThreshold(); + Status s = CheckDBConfigArchiveDaysThreshold(str); if (!s.ok()) return s; value = std::stoi(str); return Status::OK(); @@ -951,8 +683,8 @@ Config::GetDBConfigArchiveDaysThreshold(int32_t& value) { Status Config::GetDBConfigBufferSize(int32_t& value) { - std::string str; - Status s = GetDBConfigStrBufferSize(str); + std::string str = GetDBConfigStrBufferSize(); + Status s = CheckDBConfigBufferSize(str); if (!s.ok()) return s; value = std::stoi(str); return Status::OK(); @@ -960,8 +692,8 @@ Config::GetDBConfigBufferSize(int32_t& value) { Status Config::GetDBConfigBuildIndexGPU(int32_t& value) { - std::string str; - Status s = GetDBConfigStrBuildIndexGPU(str); + std::string str = GetDBConfigStrBuildIndexGPU(); + Status s = CheckDBConfigBuildIndexGPU(str); if (!s.ok()) return s; value = std::stoi(str); return Status::OK(); @@ -969,8 +701,8 @@ Config::GetDBConfigBuildIndexGPU(int32_t& value) { Status Config::GetMetricConfigAutoBootup(bool& value) { - std::string str; - Status s = GetMetricConfigStrAutoBootup(str); + std::string str = GetMetricConfigStrAutoBootup(); + Status s = CheckMetricConfigPrometheusPort(str); if (!s.ok()) return s; std::transform(str.begin(), str.end(), str.begin(), ::tolower); value = (str == "true" || str == "on" || str == "yes" || str == "1"); @@ -979,18 +711,20 @@ Config::GetMetricConfigAutoBootup(bool& value) { Status Config::GetMetricConfigCollector(std::string& value) { - return GetMetricConfigStrCollector(value); + value = GetMetricConfigStrCollector(); + return Status::OK(); } Status Config::GetMetricConfigPrometheusPort(std::string& value) { - return GetMetricConfigStrPrometheusPort(value); + value = GetMetricConfigStrPrometheusPort(); + return CheckMetricConfigPrometheusPort(value); } Status Config::GetCacheConfigCpuMemCapacity(int32_t& value) { - std::string str; - Status s = GetCacheConfigStrCpuMemCapacity(str); + std::string str = GetCacheConfigStrCpuMemCapacity(); + Status s = CheckCacheConfigCpuMemCapacity(str); if (!s.ok()) return s; value = std::stoi(str); return Status::OK(); @@ -998,8 +732,8 @@ Config::GetCacheConfigCpuMemCapacity(int32_t& value) { Status Config::GetCacheConfigCpuMemThreshold(float& value) { - std::string str; - Status s = GetCacheConfigStrCpuMemThreshold(str); + std::string str = GetCacheConfigStrCpuMemThreshold(); + Status s = CheckCacheConfigCpuMemThreshold(str); if (!s.ok()) return s; value = std::stof(str); return Status::OK(); @@ -1007,8 +741,8 @@ Config::GetCacheConfigCpuMemThreshold(float& value) { Status Config::GetCacheConfigGpuMemCapacity(int32_t& value) { - std::string str; - Status s = GetCacheConfigStrGpuMemCapacity(str); + std::string str = GetCacheConfigStrGpuMemCapacity(); + Status s = CheckCacheConfigGpuMemCapacity(str); if (!s.ok()) return s; value = std::stoi(str); return Status::OK(); @@ -1016,8 +750,8 @@ Config::GetCacheConfigGpuMemCapacity(int32_t& value) { Status Config::GetCacheConfigGpuMemThreshold(float& value) { - std::string str; - Status s = GetCacheConfigStrGpuMemThreshold(str); + std::string str = GetCacheConfigStrGpuMemThreshold(); + Status s = CheckCacheConfigGpuMemThreshold(str); if (!s.ok()) return s; value = std::stof(str); return Status::OK(); @@ -1025,8 +759,8 @@ Config::GetCacheConfigGpuMemThreshold(float& value) { Status Config::GetCacheConfigCacheInsertData(bool& value) { - std::string str; - Status s = GetCacheConfigStrCacheInsertData(str); + std::string str = GetCacheConfigStrCacheInsertData(); + Status s = CheckCacheConfigCacheInsertData(str); if (!s.ok()) return s; std::transform(str.begin(), str.end(), str.begin(), ::tolower); value = (str == "true" || str == "on" || str == "yes" || str == "1"); @@ -1035,8 +769,8 @@ Config::GetCacheConfigCacheInsertData(bool& value) { Status Config::GetEngineConfigBlasThreshold(int32_t& value) { - std::string str; - Status s = GetEngineConfigStrBlasThreshold(str); + std::string str = GetEngineConfigStrBlasThreshold(); + Status s = CheckEngineConfigBlasThreshold(str); if (!s.ok()) return s; value = std::stoi(str); return Status::OK(); @@ -1044,8 +778,8 @@ Config::GetEngineConfigBlasThreshold(int32_t& value) { Status Config::GetEngineConfigOmpThreadNum(int32_t& value) { - std::string str; - Status s = GetEngineConfigStrOmpThreadNum(str); + std::string str = GetEngineConfigStrOmpThreadNum(); + Status s = CheckEngineConfigOmpThreadNum(str); if (!s.ok()) return s; value = std::stoi(str); return Status::OK(); @@ -1053,14 +787,15 @@ Config::GetEngineConfigOmpThreadNum(int32_t& value) { Status Config::GetResourceConfigMode(std::string& value) { - return GetResourceConfigStrMode(value); + value = GetResourceConfigStrMode(); + return CheckResourceConfigMode(value); } Status Config::GetResourceConfigPool(std::vector& value) { ConfigNode resource_config = GetConfigNode(CONFIG_RESOURCE); value = resource_config.GetSequence(CONFIG_RESOURCE_POOL); - return Status::OK(); + return CheckResourceConfigPool(value); } } diff --git a/cpp/src/server/Config.h b/cpp/src/server/Config.h index 3eeda32546..911ec67462 100644 --- a/cpp/src/server/Config.h +++ b/cpp/src/server/Config.h @@ -95,64 +95,92 @@ static const char* CONFIG_RESOURCE_POOL = "pool"; class Config { public: - static Config &GetInstance(); - + static Config& GetInstance(); Status LoadConfigFile(const std::string& filename); - Status ValidateConfig(); - void PrintAll() const; + void PrintAll(); private: ConfigNode& GetConfigNode(const std::string& name); - Status CheckServerConfig(); - Status CheckDBConfig(); - Status CheckMetricConfig(); - Status CheckCacheConfig(); - Status CheckEngineConfig(); - Status CheckResourceConfig(); - Status GetConfigValueInMem(const std::string& parent_key, const std::string& child_key, std::string& value); - Status SetConfigValueInMem(const std::string& parent_key, + void SetConfigValueInMem(const std::string& parent_key, const std::string& child_key, std::string& value); - private: + void PrintConfigSection(const std::string& config_node_name); + + /////////////////////////////////////////////////////////////////////////// /* server config */ - Status GetServerConfigStrAddress(std::string& value); - Status GetServerConfigStrPort(std::string& value); - Status GetServerConfigStrMode(std::string& value); - Status GetServerConfigStrTimeZone(std::string& value); + Status CheckServerConfigAddress(std::string& value); + Status CheckServerConfigPort(std::string& value); + Status CheckServerConfigMode(std::string& value); + Status CheckServerConfigTimeZone(std::string& value); /* db config */ - Status GetDBConfigStrPath(std::string& value); - Status GetDBConfigStrSlavePath(std::string& value); - Status GetDBConfigStrBackendUrl(std::string& value); - Status GetDBConfigStrArchiveDiskThreshold(std::string& value); - Status GetDBConfigStrArchiveDaysThreshold(std::string& value); - Status GetDBConfigStrBufferSize(std::string& value); - Status GetDBConfigStrBuildIndexGPU(std::string& value); + Status CheckDBConfigPath(const std::string& value); + Status CheckDBConfigSlavePath(const std::string& value); + Status CheckDBConfigBackendUrl(const std::string& value); + Status CheckDBConfigArchiveDiskThreshold(const std::string& value); + Status CheckDBConfigArchiveDaysThreshold(const std::string& value); + Status CheckDBConfigBufferSize(const std::string& value); + Status CheckDBConfigBuildIndexGPU(const std::string& value); /* metric config */ - Status GetMetricConfigStrAutoBootup(std::string& value); - Status GetMetricConfigStrCollector(std::string& value); - Status GetMetricConfigStrPrometheusPort(std::string& value); + Status CheckMetricConfigAutoBootup(const std::string& value); + Status CheckMetricConfigPrometheusPort(const std::string& value); /* cache config */ - Status GetCacheConfigStrCpuMemCapacity(std::string& value); - Status GetCacheConfigStrCpuMemThreshold(std::string& value); - Status GetCacheConfigStrGpuMemCapacity(std::string& value); - Status GetCacheConfigStrGpuMemThreshold(std::string& value); - Status GetCacheConfigStrCacheInsertData(std::string& value); + Status CheckCacheConfigCpuMemCapacity(const std::string& value); + Status CheckCacheConfigCpuMemThreshold(const std::string& value); + Status CheckCacheConfigGpuMemCapacity(const std::string& value); + Status CheckCacheConfigGpuMemThreshold(const std::string& value); + Status CheckCacheConfigCacheInsertData(const std::string& value); /* engine config */ - Status GetEngineConfigStrBlasThreshold(std::string& value); - Status GetEngineConfigStrOmpThreadNum(std::string& value); + Status CheckEngineConfigBlasThreshold(const std::string& value); + Status CheckEngineConfigOmpThreadNum(const std::string& value); /* resource config */ - Status GetResourceConfigStrMode(std::string& value); + Status CheckResourceConfigMode(const std::string& value); + Status CheckResourceConfigPool(const std::vector& value); + + /////////////////////////////////////////////////////////////////////////// + /* server config */ + std::string GetServerConfigStrAddress(); + std::string GetServerConfigStrPort(); + std::string GetServerConfigStrMode(); + std::string GetServerConfigStrTimeZone(); + + /* db config */ + std::string GetDBConfigStrPath(); + std::string GetDBConfigStrSlavePath(); + std::string GetDBConfigStrBackendUrl(); + std::string GetDBConfigStrArchiveDiskThreshold(); + std::string GetDBConfigStrArchiveDaysThreshold(); + std::string GetDBConfigStrBufferSize(); + std::string GetDBConfigStrBuildIndexGPU(); + + /* metric config */ + std::string GetMetricConfigStrAutoBootup(); + std::string GetMetricConfigStrCollector(); + std::string GetMetricConfigStrPrometheusPort(); + + /* cache config */ + std::string GetCacheConfigStrCpuMemCapacity(); + std::string GetCacheConfigStrCpuMemThreshold(); + std::string GetCacheConfigStrGpuMemCapacity(); + std::string GetCacheConfigStrGpuMemThreshold(); + std::string GetCacheConfigStrCacheInsertData(); + + /* engine config */ + std::string GetEngineConfigStrBlasThreshold(); + std::string GetEngineConfigStrOmpThreadNum(); + + /* resource config */ + std::string GetResourceConfigStrMode(); public: /* server config */ diff --git a/cpp/src/server/Server.cpp b/cpp/src/server/Server.cpp index d926ca5dac..355f7903f2 100644 --- a/cpp/src/server/Server.cpp +++ b/cpp/src/server/Server.cpp @@ -235,14 +235,12 @@ Server::Stop() { ErrorCode Server::LoadConfig() { - Config& server_config = Config::GetInstance(); - server_config.LoadConfigFile(config_filename_); - auto status = server_config.ValidateConfig(); - if (!status.ok()) { + Config& config = Config::GetInstance(); + Status s = config.LoadConfigFile(config_filename_); + if (!s.ok()) { std::cerr << "Failed to load config file: " << config_filename_ << std::endl; exit(0); } - return SERVER_SUCCESS; } diff --git a/cpp/src/utils/ValidationUtil.cpp b/cpp/src/utils/ValidationUtil.cpp index c993a4ae14..c9ec2d046b 100644 --- a/cpp/src/utils/ValidationUtil.cpp +++ b/cpp/src/utils/ValidationUtil.cpp @@ -206,38 +206,40 @@ ValidationUtil::ValidateIpAddress(const std::string &ip_address) { } Status -ValidationUtil::ValidateStringIsNumber(const std::string &string) { - if (!string.empty() && std::all_of(string.begin(), string.end(), ::isdigit)) { +ValidationUtil::ValidateStringIsNumber(const std::string &str) { + if (str.empty() || !std::all_of(str.begin(), str.end(), ::isdigit)) { + return Status(SERVER_INVALID_ARGUMENT, "Invalid number"); + } + try { + int32_t value = std::stoi(str); + } catch(...) { + return Status(SERVER_INVALID_ARGUMENT, "Invalid number"); + } + return Status::OK(); +} + +Status +ValidationUtil::ValidateStringIsBool(const std::string &str) { + std::string s = str; + std::transform(s.begin(), s.end(), s.begin(), ::tolower); + if (s == "true" || s == "on" || s == "yes" || s == "1" || + s == "false" || s == "off" || s == "no" || s == "0" || + s.empty()) { return Status::OK(); } else { - return Status(SERVER_INVALID_ARGUMENT, "Not a number"); + return Status(SERVER_INVALID_ARGUMENT, "Invalid boolean: " + str); } } Status -ValidationUtil::ValidateStringIsBool(std::string &str) { - std::transform(str.begin(), str.end(), str.begin(), ::tolower); - if (str == "true" || str == "on" || str == "yes" || str == "1" || - str == "false" || str == "off" || str == "no" || str == "0" || - str.empty()) { - return Status::OK(); - } - else { - return Status(SERVER_INVALID_ARGUMENT, "Not a boolean: " + str); - } -} - -Status -ValidationUtil::ValidateStringIsDouble(const std::string &str, double &val) { - char *end = nullptr; - val = std::strtod(str.c_str(), &end); - if (end != str.c_str() && *end == '\0' && val != HUGE_VAL) { - return Status::OK(); - } - else { - return Status(SERVER_INVALID_ARGUMENT, "Not a double value: " + str); +ValidationUtil::ValidateStringIsFloat(const std::string &str) { + try { + float val = std::stof(str); + } catch(...) { + return Status(SERVER_INVALID_ARGUMENT, "Invalid float: " + str); } + return Status::OK(); } Status diff --git a/cpp/src/utils/ValidationUtil.h b/cpp/src/utils/ValidationUtil.h index ad8ba5fee9..789da0de40 100644 --- a/cpp/src/utils/ValidationUtil.h +++ b/cpp/src/utils/ValidationUtil.h @@ -67,10 +67,10 @@ public: ValidateStringIsNumber(const std::string &str); static Status - ValidateStringIsBool(std::string &str); + ValidateStringIsBool(const std::string &str); static Status - ValidateStringIsDouble(const std::string &str, double &val); + ValidateStringIsFloat(const std::string &str); static Status ValidateDbURI(const std::string &uri); From 91daaf7e7e2faed9af9aa574c91fda838eaf545c Mon Sep 17 00:00:00 2001 From: "yudong.cai" Date: Wed, 25 Sep 2019 19:49:22 +0800 Subject: [PATCH 7/7] MS-574 add config set APIs Former-commit-id: e5465a07270b4ffbdf9dcacfd9912ecfb88c528c --- cpp/src/server/Config.cpp | 208 ++++++++++++++++++++++++++++++++++++-- cpp/src/server/Config.h | 46 ++++++++- 2 files changed, 243 insertions(+), 11 deletions(-) diff --git a/cpp/src/server/Config.cpp b/cpp/src/server/Config.cpp index b89f1e3e02..8ee2f7c62b 100644 --- a/cpp/src/server/Config.cpp +++ b/cpp/src/server/Config.cpp @@ -93,7 +93,7 @@ Config::PrintAll() { //////////////////////////////////////////////////////////////////////////////// Status -Config::CheckServerConfigAddress(std::string &value) { +Config::CheckServerConfigAddress(const std::string &value) { if (!ValidationUtil::ValidateIpAddress(value).ok()) { return Status(SERVER_INVALID_ARGUMENT, "Invalid server config address: " + value); } @@ -101,7 +101,7 @@ Config::CheckServerConfigAddress(std::string &value) { } Status -Config::CheckServerConfigPort(std::string &value) { +Config::CheckServerConfigPort(const std::string &value) { if (!ValidationUtil::ValidateStringIsNumber(value).ok()) { return Status(SERVER_INVALID_ARGUMENT, "Invalid server config port: " + value); } else { @@ -114,7 +114,7 @@ Config::CheckServerConfigPort(std::string &value) { } Status -Config::CheckServerConfigMode(std::string &value) { +Config::CheckServerConfigMode(const std::string &value) { if (value != "single" && value != "cluster" && value != "read_only") { return Status(SERVER_INVALID_ARGUMENT, "Invalid server config mode [single, cluster, read_only]: " + value); } @@ -122,7 +122,7 @@ Config::CheckServerConfigMode(std::string &value) { } Status -Config::CheckServerConfigTimeZone(std::string &value) { +Config::CheckServerConfigTimeZone(const std::string &value) { if (value.length() <= 3) { return Status(SERVER_INVALID_ARGUMENT, "Invalid server config time_zone: " + value); } else { @@ -147,6 +147,11 @@ Config::CheckDBConfigPath(const std::string &value) { return Status::OK(); } +Status +Config::CheckDBConfigSlavePath(const std::string &value) { + return Status::OK(); +} + Status Config::CheckDBConfigBackendUrl(const std::string &value) { if (!ValidationUtil::ValidateDbURI(value).ok()) { @@ -207,6 +212,14 @@ Config::CheckMetricConfigAutoBootup(const std::string& value) { return Status::OK(); } +Status +Config::CheckMetricConfigCollector(const std::string& value) { + if (value != "prometheus") { + return Status(SERVER_INVALID_ARGUMENT, "Invalid metric config collector: " + value); + } + return Status::OK(); +} + Status Config::CheckMetricConfigPrometheusPort(const std::string& value) { if (!ValidationUtil::ValidateStringIsNumber(value).ok()) { @@ -347,9 +360,9 @@ Status Config::GetConfigValueInMem(const std::string &parent_key, const std::string &child_key, std::string &value) { + std::lock_guard lock(mutex_); if (config_map_.find(parent_key) != config_map_.end() && config_map_[parent_key].find(child_key) != config_map_[parent_key].end()) { - std::lock_guard lock(mutex_); value = config_map_[parent_key][child_key]; return Status::OK(); } else { @@ -360,7 +373,7 @@ Config::GetConfigValueInMem(const std::string &parent_key, void Config::SetConfigValueInMem(const std::string &parent_key, const std::string &child_key, - std::string &value) { + const std::string &value) { std::lock_guard lock(mutex_); config_map_[parent_key][child_key] = value; } @@ -798,6 +811,189 @@ Config::GetResourceConfigPool(std::vector& value) { return CheckResourceConfigPool(value); } +/////////////////////////////////////////////////////////////////////////////// +/* server config */ +Status +Config::SetServerConfigAddress(const std::string& value) { + Status s = CheckServerConfigAddress(value); + if (!s.ok()) return s; + SetConfigValueInMem(CONFIG_SERVER, CONFIG_SERVER_ADDRESS, value); + return Status::OK(); +} + +Status +Config::SetServerConfigPort(const std::string& value) { + Status s = CheckServerConfigPort(value); + if (!s.ok()) return s; + SetConfigValueInMem(CONFIG_SERVER, CONFIG_SERVER_PORT, value); + return Status::OK(); +} + +Status +Config::SetServerConfigMode(const std::string& value) { + Status s = CheckServerConfigMode(value); + if (!s.ok()) return s; + SetConfigValueInMem(CONFIG_SERVER, CONFIG_SERVER_MODE, value); + return Status::OK(); +} + +Status +Config::SetServerConfigTimeZone(const std::string& value) { + Status s = CheckServerConfigTimeZone(value); + if (!s.ok()) return s; + SetConfigValueInMem(CONFIG_SERVER, CONFIG_SERVER_TIME_ZONE, value); + return Status::OK(); +} + +/* db config */ +Status +Config::SetDBConfigPath(const std::string& value) { + Status s = CheckDBConfigPath(value); + if (!s.ok()) return s; + SetConfigValueInMem(CONFIG_DB, CONFIG_DB_PATH, value); + return Status::OK(); +} + +Status +Config::SetDBConfigSlavePath(const std::string& value) { + Status s = CheckDBConfigSlavePath(value); + if (!s.ok()) return s; + SetConfigValueInMem(CONFIG_DB, CONFIG_DB_SLAVE_PATH, value); + return Status::OK(); +} + +Status +Config::SetDBConfigBackendUrl(const std::string& value) { + Status s = CheckDBConfigBackendUrl(value); + if (!s.ok()) return s; + SetConfigValueInMem(CONFIG_DB, CONFIG_DB_BACKEND_URL, value); + return Status::OK(); +} + +Status +Config::SetDBConfigArchiveDiskThreshold(const std::string& value) { + Status s = CheckDBConfigArchiveDiskThreshold(value); + if (!s.ok()) return s; + SetConfigValueInMem(CONFIG_DB, CONFIG_DB_ARCHIVE_DISK_THRESHOLD, value); + return Status::OK(); +} + +Status +Config::SetDBConfigArchiveDaysThreshold(const std::string& value) { + Status s = CheckDBConfigArchiveDaysThreshold(value); + if (!s.ok()) return s; + SetConfigValueInMem(CONFIG_DB, CONFIG_DB_ARCHIVE_DAYS_THRESHOLD, value); + return Status::OK(); +} + +Status +Config::SetDBConfigBufferSize(const std::string& value) { + Status s = CheckDBConfigBufferSize(value); + if (!s.ok()) return s; + SetConfigValueInMem(CONFIG_DB, CONFIG_DB_BUFFER_SIZE, value); + return Status::OK(); +} + +Status +Config::SetDBConfigBuildIndexGPU(const std::string& value) { + Status s = CheckDBConfigBuildIndexGPU(value); + if (!s.ok()) return s; + SetConfigValueInMem(CONFIG_DB, CONFIG_DB_BUILD_INDEX_GPU, value); + return Status::OK(); +} + +/* metric config */ +Status +Config::SetMetricConfigAutoBootup(const std::string& value) { + Status s = CheckMetricConfigAutoBootup(value); + if (!s.ok()) return s; + SetConfigValueInMem(CONFIG_DB, CONFIG_METRIC_AUTO_BOOTUP, value); + return Status::OK(); +} + +Status +Config::SetMetricConfigCollector(const std::string& value) { + Status s = CheckMetricConfigCollector(value); + if (!s.ok()) return s; + SetConfigValueInMem(CONFIG_DB, CONFIG_METRIC_COLLECTOR, value); + return Status::OK(); +} + +Status +Config::SetMetricConfigPrometheusPort(const std::string& value) { + Status s = CheckMetricConfigPrometheusPort(value); + if (!s.ok()) return s; + SetConfigValueInMem(CONFIG_DB, CONFIG_METRIC_PROMETHEUS_PORT, value); + return Status::OK(); +} + +/* cache config */ +Status +Config::SetCacheConfigCpuMemCapacity(const std::string& value) { + Status s = CheckCacheConfigCpuMemCapacity(value); + if (!s.ok()) return s; + SetConfigValueInMem(CONFIG_DB, CONFIG_CACHE_CPU_MEM_CAPACITY, value); + return Status::OK(); +} + +Status +Config::SetCacheConfigCpuMemThreshold(const std::string& value) { + Status s = CheckCacheConfigCpuMemThreshold(value); + if (!s.ok()) return s; + SetConfigValueInMem(CONFIG_DB, CONFIG_CACHE_CPU_MEM_THRESHOLD, value); + return Status::OK(); +} + +Status +Config::SetCacheConfigGpuMemCapacity(const std::string& value) { + Status s = CheckCacheConfigGpuMemCapacity(value); + if (!s.ok()) return s; + SetConfigValueInMem(CONFIG_DB, CONFIG_CACHE_GPU_MEM_CAPACITY, value); + return Status::OK(); +} + +Status +Config::SetCacheConfigGpuMemThreshold(const std::string& value) { + Status s = CheckCacheConfigGpuMemThreshold(value); + if (!s.ok()) return s; + SetConfigValueInMem(CONFIG_DB, CONFIG_CACHE_GPU_MEM_THRESHOLD, value); + return Status::OK(); +} + +Status +Config::SetCacheConfigCacheInsertData(const std::string& value) { + Status s = CheckCacheConfigCacheInsertData(value); + if (!s.ok()) return s; + SetConfigValueInMem(CONFIG_DB, CONFIG_CACHE_CACHE_INSERT_DATA, value); + return Status::OK(); +} + +/* engine config */ +Status +Config::SetEngineConfigBlasThreshold(const std::string& value) { + Status s = CheckEngineConfigBlasThreshold(value); + if (!s.ok()) return s; + SetConfigValueInMem(CONFIG_DB, CONFIG_ENGINE_BLAS_THRESHOLD, value); + return Status::OK(); +} + +Status +Config::SetEngineConfigOmpThreadNum(const std::string& value) { + Status s = CheckEngineConfigOmpThreadNum(value); + if (!s.ok()) return s; + SetConfigValueInMem(CONFIG_DB, CONFIG_ENGINE_OMP_THREAD_NUM, value); + return Status::OK(); +} + +/* resource config */ +Status +Config::SetResourceConfigMode(const std::string& value) { + Status s = CheckResourceConfigMode(value); + if (!s.ok()) return s; + SetConfigValueInMem(CONFIG_DB, CONFIG_RESOURCE_MODE, value); + return Status::OK(); +} + } } } diff --git a/cpp/src/server/Config.h b/cpp/src/server/Config.h index 911ec67462..956c08100f 100644 --- a/cpp/src/server/Config.h +++ b/cpp/src/server/Config.h @@ -108,16 +108,16 @@ class Config { void SetConfigValueInMem(const std::string& parent_key, const std::string& child_key, - std::string& value); + const std::string& value); void PrintConfigSection(const std::string& config_node_name); /////////////////////////////////////////////////////////////////////////// /* server config */ - Status CheckServerConfigAddress(std::string& value); - Status CheckServerConfigPort(std::string& value); - Status CheckServerConfigMode(std::string& value); - Status CheckServerConfigTimeZone(std::string& value); + Status CheckServerConfigAddress(const std::string& value); + Status CheckServerConfigPort(const std::string& value); + Status CheckServerConfigMode(const std::string& value); + Status CheckServerConfigTimeZone(const std::string& value); /* db config */ Status CheckDBConfigPath(const std::string& value); @@ -130,6 +130,7 @@ class Config { /* metric config */ Status CheckMetricConfigAutoBootup(const std::string& value); + Status CheckMetricConfigCollector(const std::string& value); Status CheckMetricConfigPrometheusPort(const std::string& value); /* cache config */ @@ -218,6 +219,41 @@ class Config { Status GetResourceConfigMode(std::string& value); Status GetResourceConfigPool(std::vector& value); + public: + /* server config */ + Status SetServerConfigAddress(const std::string& value); + Status SetServerConfigPort(const std::string& value); + Status SetServerConfigMode(const std::string& value); + Status SetServerConfigTimeZone(const std::string& value); + + /* db config */ + Status SetDBConfigPath(const std::string& value); + Status SetDBConfigSlavePath(const std::string& value); + Status SetDBConfigBackendUrl(const std::string& value); + Status SetDBConfigArchiveDiskThreshold(const std::string& value); + Status SetDBConfigArchiveDaysThreshold(const std::string& value); + Status SetDBConfigBufferSize(const std::string& value); + Status SetDBConfigBuildIndexGPU(const std::string& value); + + /* metric config */ + Status SetMetricConfigAutoBootup(const std::string& value); + Status SetMetricConfigCollector(const std::string& value); + Status SetMetricConfigPrometheusPort(const std::string& value); + + /* cache config */ + Status SetCacheConfigCpuMemCapacity(const std::string& value); + Status SetCacheConfigCpuMemThreshold(const std::string& value); + Status SetCacheConfigGpuMemCapacity(const std::string& value); + Status SetCacheConfigGpuMemThreshold(const std::string& value); + Status SetCacheConfigCacheInsertData(const std::string& value); + + /* engine config */ + Status SetEngineConfigBlasThreshold(const std::string& value); + Status SetEngineConfigOmpThreadNum(const std::string& value); + + /* resource config */ + Status SetResourceConfigMode(const std::string& value); + private: std::unordered_map> config_map_; std::mutex mutex_;