milvus-io
/
milvus
mirror of https://github.com/milvus-io/milvus.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

MS-665 rename build_index_gpu to index_build_device, and move it to resource config 

Former-commit-id: f3da7963531406da7a22fbf5ab95caa8e920344a
pull/191/head
yudong.cai 2019-10-18 16:32:28 +08:00
parent 39d433e3ab
commit 4d1fb143e1
10 changed files with 109 additions and 85 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
CHANGELOG.md
View File

@ -26,6 +26,7 @@ Please mark all change in change log and use the ticket from JIRA.
- MS-653 - When config check fail, Milvus close without message
- MS-654 - Describe index timeout when building index
- MS-658 - Fix SQ8 Hybrid can't search
- MS-665 - IVF_SQ8H search crash when no GPU resource in search_resources
- \#20 - C++ sdk example get grpc error
- \#23 - Add unittest to improve code coverage
- \#31 - make clang-format failed after run build.sh -l

5
core/conf/server_config.template
View File

@ -16,7 +16,6 @@ db_config:
insert_buffer_size: 4 # GB, maximum insert buffer size allowed
# sum of insert_buffer_size and cpu_cache_capacity cannot exceed total memory
build_index_gpu: 0 # gpu id used for building index
preload_table: # preload data at startup, '*' means load all tables, empty value means no preload
# you can specify preload tables like this: table1,table2,table3
@ -39,6 +38,6 @@ engine_config:
# if nq >= use_blas_threshold, use OpenBlas, slower with stable response times
resource_config:
resource_pool:
- cpu
search_resources: # define the GPUs used for search computation, valid value: gpux
- gpu0
index_build_device: gpu0 # GPU used for building index

2
core/src/db/engine/ExecutionEngineImpl.cpp
View File

@ -503,7 +503,7 @@ ExecutionEngineImpl::GpuCache(uint64_t gpu_id) {
Status
ExecutionEngineImpl::Init() {
server::Config& config = server::Config::GetInstance();
Status s = config.GetDBConfigBuildIndexGPU(gpu_num_);
Status s = config.GetResourceConfigIndexBuildDevice(gpu_num_);
if (!s.ok()) {
return s;
}

2
core/src/scheduler/SchedInst.cpp
View File

@ -50,7 +50,7 @@ load_simple_config() {
std::string mode;
config.GetResourceConfigMode(mode);
std::vector<std::string> pool;
config.GetResourceConfigPool(pool);
config.GetResourceConfigSearchResources(pool);
// get resources
bool use_cpu_to_compute = false;

2
core/src/scheduler/Utils.cpp
View File

@ -48,7 +48,7 @@ get_gpu_pool() {
server::Config& config = server::Config::GetInstance();
std::vector<std::string> pool;
Status s = config.GetResourceConfigPool(pool);
Status s = config.GetResourceConfigSearchResources(pool);
if (!s.ok()) {
SERVER_LOG_ERROR << s.message();
}

2
core/src/scheduler/action/PushTaskToNeighbour.cpp
View File

@ -184,7 +184,7 @@ Action::SpecifiedResourceLabelTaskScheduler(ResourceMgrWPtr res_mgr, ResourcePtr
// get build index gpu resource
server::Config& config = server::Config::GetInstance();
int32_t build_index_gpu;
Status stat = config.GetDBConfigBuildIndexGPU(build_index_gpu);
Status stat = config.GetResourceConfigIndexBuildDevice(build_index_gpu);
bool find_gpu_res = false;
for (uint64_t i = 0; i < compute_resources.size(); ++i) {

128
core/src/server/Config.cpp
View File

@ -18,6 +18,7 @@
#include <sys/stat.h>
#include <algorithm>
#include <iostream>
#include <regex>
#include <string>
#include <vector>
@ -129,12 +130,6 @@ Config::ValidateConfig() {
return s;
}
int32_t db_build_index_gpu;
s = GetDBConfigBuildIndexGPU(db_build_index_gpu);
if (!s.ok()) {
return s;
}
/* metric config */
bool metric_enable_monitor;
s = GetMetricConfigEnableMonitor(metric_enable_monitor);
@ -205,8 +200,14 @@ Config::ValidateConfig() {
return s;
}
std::vector<std::string> resource_pool;
s = GetResourceConfigPool(resource_pool);
std::vector<std::string> search_resources;
s = GetResourceConfigSearchResources(search_resources);
if (!s.ok()) {
return s;
}
int32_t resource_index_build_device;
s = GetResourceConfigIndexBuildDevice(resource_index_build_device);
if (!s.ok()) {
return s;
}
@ -270,11 +271,6 @@ Config::ResetDefaultConfig() {
return s;
}
s = SetDBConfigBuildIndexGPU(CONFIG_DB_BUILD_INDEX_GPU_DEFAULT);
if (!s.ok()) {
return s;
}
/* metric config */
s = SetMetricConfigEnableMonitor(CONFIG_METRIC_ENABLE_MONITOR_DEFAULT);
if (!s.ok()) {
@ -334,6 +330,11 @@ Config::ResetDefaultConfig() {
return s;
}
s = SetResourceConfigIndexBuildDevice(CONFIG_RESOURCE_INDEX_BUILD_DEVICE_DEFAULT);
if (!s.ok()) {
return s;
}
return Status::OK();
}
@ -459,19 +460,6 @@ Config::CheckDBConfigInsertBufferSize(const std::string& value) {
return Status::OK();
}
Status
Config::CheckDBConfigBuildIndexGPU(const std::string& value) {
if (!ValidationUtil::ValidateStringIsNumber(value).ok()) {
return Status(SERVER_INVALID_ARGUMENT, "Invalid DB config build_index_gpu: " + value);
} else {
int32_t gpu_index = std::stoi(value);
if (!ValidationUtil::ValidateGpuIndex(gpu_index).ok()) {
return Status(SERVER_INVALID_ARGUMENT, "Invalid DB config build_index_gpu: " + value);
}
}
return Status::OK();
}
Status
Config::CheckMetricConfigEnableMonitor(const std::string& value) {
if (!ValidationUtil::ValidateStringIsBool(value).ok()) {
@ -544,7 +532,7 @@ Config::CheckCacheConfigGpuCacheCapacity(const std::string& value) {
} else {
uint64_t gpu_cache_capacity = std::stoi(value) * GB;
int gpu_index;
Status s = GetDBConfigBuildIndexGPU(gpu_index);
Status s = GetResourceConfigIndexBuildDevice(gpu_index);
if (!s.ok()) {
return s;
}
@ -616,9 +604,38 @@ Config::CheckResourceConfigMode(const std::string& value) {
}
Status
Config::CheckResourceConfigPool(const std::vector<std::string>& value) {
CheckGpuDevice(const std::string& value) {
const std::regex pat("gpu(\\d+)");
std::cmatch m;
if (!std::regex_match(value.c_str(), m, pat)) {
return Status(SERVER_INVALID_ARGUMENT, "Invalid gpu device: " + value);
}
int32_t gpu_index = std::stoi(value.substr(3));
if (!ValidationUtil::ValidateGpuIndex(gpu_index).ok()) {
return Status(SERVER_INVALID_ARGUMENT, "Invalid gpu device: " + value);
}
return Status::OK();
}
Status
Config::CheckResourceConfigSearchResources(const std::vector<std::string>& value) {
if (value.empty()) {
return Status(SERVER_INVALID_ARGUMENT, "Invalid resource config pool");
return Status(SERVER_INVALID_ARGUMENT, "Empty resource config search_resources");
}
for (auto& gpu_device : value) {
if (!CheckGpuDevice(gpu_device).ok()) {
return Status(SERVER_INVALID_ARGUMENT, "Invalid resource config search_resources: " + gpu_device);
}
}
return Status::OK();
}
Status
Config::CheckResourceConfigIndexBuildDevice(const std::string& value) {
if (!CheckGpuDevice(value).ok()) {
return Status(SERVER_INVALID_ARGUMENT, "Invalid resource config index_build_device: " + value);
}
return Status::OK();
}
@ -739,18 +756,6 @@ Config::GetDBConfigInsertBufferSize(int32_t& value) {
return Status::OK();
}
Status
Config::GetDBConfigBuildIndexGPU(int32_t& value) {
std::string str = GetConfigStr(CONFIG_DB, CONFIG_DB_BUILD_INDEX_GPU, CONFIG_DB_BUILD_INDEX_GPU_DEFAULT);
Status s = CheckDBConfigBuildIndexGPU(str);
if (!s.ok()) {
return s;
}
value = std::stoi(str);
return Status::OK();
}
Status
Config::GetDBConfigPreloadTable(std::string& value) {
value = GetConfigStr(CONFIG_DB, CONFIG_DB_PRELOAD_TABLE);
@ -880,10 +885,23 @@ Config::GetResourceConfigMode(std::string& value) {
}
Status
Config::GetResourceConfigPool(std::vector<std::string>& value) {
Config::GetResourceConfigSearchResources(std::vector<std::string>& value) {
ConfigNode resource_config = GetConfigNode(CONFIG_RESOURCE);
value = resource_config.GetSequence(CONFIG_RESOURCE_POOL);
return CheckResourceConfigPool(value);
value = resource_config.GetSequence(CONFIG_RESOURCE_SEARCH_RESOURCES);
return CheckResourceConfigSearchResources(value);
}
Status
Config::GetResourceConfigIndexBuildDevice(int32_t& value) {
std::string str =
GetConfigStr(CONFIG_RESOURCE, CONFIG_RESOURCE_INDEX_BUILD_DEVICE, CONFIG_RESOURCE_INDEX_BUILD_DEVICE_DEFAULT);
Status s = CheckResourceConfigIndexBuildDevice(str);
if (!s.ok()) {
return s;
}
value = std::stoi(str.substr(3));
return Status::OK();
}
///////////////////////////////////////////////////////////////////////////////
@ -999,17 +1017,6 @@ Config::SetDBConfigInsertBufferSize(const std::string& value) {
return Status::OK();
}
Status
Config::SetDBConfigBuildIndexGPU(const std::string& value) {
Status s = CheckDBConfigBuildIndexGPU(value);
if (!s.ok()) {
return s;
}
SetConfigValueInMem(CONFIG_DB, CONFIG_DB_BUILD_INDEX_GPU, value);
return Status::OK();
}
/* metric config */
Status
Config::SetMetricConfigEnableMonitor(const std::string& value) {
@ -1135,5 +1142,16 @@ Config::SetResourceConfigMode(const std::string& value) {
return Status::OK();
}
Status
Config::SetResourceConfigIndexBuildDevice(const std::string& value) {
Status s = CheckResourceConfigIndexBuildDevice(value);
if (!s.ok()) {
return s;
}
SetConfigValueInMem(CONFIG_DB, CONFIG_RESOURCE_INDEX_BUILD_DEVICE, value);
return Status::OK();
}
} // namespace server
} // namespace milvus

22
core/src/server/Config.h
View File

@ -53,8 +53,6 @@ static const char* CONFIG_DB_ARCHIVE_DAYS_THRESHOLD = "archive_days_threshold";
static const char* CONFIG_DB_ARCHIVE_DAYS_THRESHOLD_DEFAULT = "0";
static const char* CONFIG_DB_INSERT_BUFFER_SIZE = "insert_buffer_size";
static const char* CONFIG_DB_INSERT_BUFFER_SIZE_DEFAULT = "4";
static const char* CONFIG_DB_BUILD_INDEX_GPU = "build_index_gpu";
static const char* CONFIG_DB_BUILD_INDEX_GPU_DEFAULT = "0";
static const char* CONFIG_DB_PRELOAD_TABLE = "preload_table";
/* cache config */
@ -91,7 +89,9 @@ static const char* CONFIG_ENGINE_OMP_THREAD_NUM_DEFAULT = "0";
static const char* CONFIG_RESOURCE = "resource_config";
static const char* CONFIG_RESOURCE_MODE = "mode";
static const char* CONFIG_RESOURCE_MODE_DEFAULT = "simple";
static const char* CONFIG_RESOURCE_POOL = "resource_pool";
static const char* CONFIG_RESOURCE_SEARCH_RESOURCES = "search_resources";
static const char* CONFIG_RESOURCE_INDEX_BUILD_DEVICE = "index_build_device";
static const char* CONFIG_RESOURCE_INDEX_BUILD_DEVICE_DEFAULT = "gpu0";
class Config {
public:
@ -140,8 +140,6 @@ class Config {
CheckDBConfigArchiveDaysThreshold(const std::string& value);
Status
CheckDBConfigInsertBufferSize(const std::string& value);
Status
CheckDBConfigBuildIndexGPU(const std::string& value);
/* metric config */
Status
@ -173,7 +171,9 @@ class Config {
Status
CheckResourceConfigMode(const std::string& value);
Status
CheckResourceConfigPool(const std::vector<std::string>& value);
CheckResourceConfigSearchResources(const std::vector<std::string>& value);
Status
CheckResourceConfigIndexBuildDevice(const std::string& value);
std::string
GetConfigStr(const std::string& parent_key, const std::string& child_key, const std::string& default_value = "");
@ -203,8 +203,6 @@ class Config {
Status
GetDBConfigInsertBufferSize(int32_t& value);
Status
GetDBConfigBuildIndexGPU(int32_t& value);
Status
GetDBConfigPreloadTable(std::string& value);
/* metric config */
@ -237,7 +235,9 @@ class Config {
Status
GetResourceConfigMode(std::string& value);
Status
GetResourceConfigPool(std::vector<std::string>& value);
GetResourceConfigSearchResources(std::vector<std::string>& value);
Status
GetResourceConfigIndexBuildDevice(int32_t& value);
public:
/* server config */
@ -263,8 +263,6 @@ class Config {
SetDBConfigArchiveDaysThreshold(const std::string& value);
Status
SetDBConfigInsertBufferSize(const std::string& value);
Status
SetDBConfigBuildIndexGPU(const std::string& value);
/* metric config */
Status
@ -295,6 +293,8 @@ class Config {
/* resource config */
Status
SetResourceConfigMode(const std::string& value);
Status
SetResourceConfigIndexBuildDevice(const std::string& value);
private:
std::unordered_map<std::string, std::unordered_map<std::string, std::string>> config_map_;

4
core/src/wrapper/KnowhereResource.cpp
View File

@ -45,7 +45,7 @@ KnowhereResource::Initialize() {
server::Config& config = server::Config::GetInstance();
int32_t build_index_gpu;
s = config.GetDBConfigBuildIndexGPU(build_index_gpu);
s = config.GetResourceConfigIndexBuildDevice(build_index_gpu);
if (!s.ok())
return s;
@ -53,7 +53,7 @@ KnowhereResource::Initialize() {
// get search gpu resource
std::vector<std::string> pool;
s = config.GetResourceConfigPool(pool);
s = config.GetResourceConfigSearchResources(pool);
if (!s.ok())
return s;

26
core/unittest/server/appendix/server_config.yaml
View File

@ -1,4 +1,4 @@
# All the following configurations are default values.
# Default values are used when you make no changes to the following parameters.
server_config:
address: 0.0.0.0 # milvus server ip address (IPv4)
@ -11,27 +11,33 @@ db_config:
secondary_path: # path used to store data only, split by semicolon
backend_url: sqlite://:@:/ # URI format: dialect://username:password@host:port/database
# Keep 'dialect://:@:/', and replace other texts with real values.
# Keep 'dialect://:@:/', and replace other texts with real values
# Replace 'dialect' with 'mysql' or 'sqlite'
insert_buffer_size: 4 # GB, maximum insert buffer size allowed
build_index_gpu: 0 # gpu id used for building index
# sum of insert_buffer_size and cpu_cache_capacity cannot exceed total memory
preload_table: # preload data at startup, '*' means load all tables, empty value means no preload
# you can specify preload tables like this: table1,table2,table3
metric_config:
enable_monitor: false # enable monitoring or not
collector: prometheus # prometheus
prometheus_config:
port: 8080 # port prometheus used to fetch metrics
port: 8080 # port prometheus uses to fetch metrics
cache_config:
cpu_mem_capacity: 16 # GB, CPU memory used for cache
cpu_mem_threshold: 0.85 # percentage of data kept when cache cleanup triggered
cache_insert_data: false # whether load inserted data into cache
cpu_cache_capacity: 16 # GB, CPU memory used for cache
cpu_cache_threshold: 0.85 # percentage of data that will be kept when cache cleanup is triggered
gpu_cache_capacity: 4 # GB, GPU memory used for cache
gpu_cache_threshold: 0.85 # percentage of data that will be kept when cache cleanup is triggered
cache_insert_data: false # whether to load inserted data into cache
engine_config:
blas_threshold: 20
use_blas_threshold: 20 # if nq < use_blas_threshold, use SSE, faster with fluctuated response times
# if nq >= use_blas_threshold, use OpenBlas, slower with stable response times
resource_config:
resource_pool:
- cpu
search_resources: # define the GPUs used for search computation, valid value: gpux
- gpu0
index_build_device: gpu0 # GPU used for building index