milvus-io
/
milvus
mirror of https://github.com/milvus-io/milvus.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

#771 add ProcessCommand() to process pre-defined commands (#777)

pull/774/head
Cai Yudong 2019-12-17 10:48:28 +08:00 committed by Jin Hai
parent 952f34352a
commit 7c02c3760e
11 changed files with 96 additions and 109 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
CHANGELOG.md
View File

@ -10,6 +10,7 @@ Please mark all change in change log and use the issue from GitHub
## Feature
- \#343 - Add Opentracing
- \#665 - Support get/set config via CLI
- \#771 - Add server build commit info interface
## Improvement

17
core/src/sdk/examples/simple/src/ClientTest.cpp
View File

@ -76,6 +76,23 @@ ClientTest::Test(const std::string& address, const std::string& port) {
std::cout << "SDK version: " << version << std::endl;
}
{ // build commit
std::string build_commit_str;
std::vector<std::string> params;
stat = conn->ProcessCommand(build_commit_str, "build_commit_id", params);
std::cout << "ProcessCommand status: " << stat.message() << std::endl;
std::cout << "Milvus build commit id: " << build_commit_str << std::endl;
}
{ // build commit
std::string config_str;
std::vector<std::string> params;
params.push_back("*");
stat = conn->ProcessCommand(config_str, "get_config", params);
std::cout << "ProcessCommand status: " << stat.message() << std::endl;
std::cout << "Milvus config json: " << config_str << std::endl;
}
{ // show tables
std::vector<std::string> tables;
stat = conn->ShowTables(tables);

20
core/src/sdk/grpc/ClientProxy.cpp
View File

@ -457,21 +457,15 @@ ClientProxy::DropPartition(const PartitionParam& partition_param) {
}
Status
ClientProxy::GetConfig(const std::string& node_name, std::string& value) const {
ClientProxy::ProcessCommand(std::string& result, const std::string& cmd, const std::vector<std::string>& params) const {
try {
return client_ptr_->Cmd(value, "get " + node_name);
std::string cmd_line = cmd;
for (const std::string& param : params) {
cmd_line += " " + param;
}
return client_ptr_->Cmd(result, cmd_line);
} catch (std::exception& ex) {
return Status(StatusCode::UnknownError, "Fail to get config: " + node_name);
}
}
Status
ClientProxy::SetConfig(const std::string& node_name, const std::string& value) const {
try {
std::string dummy;
return client_ptr_->Cmd(dummy, "set " + node_name + " " + value);
} catch (std::exception& ex) {
return Status(StatusCode::UnknownError, "Fail to set config: " + node_name);
return Status(StatusCode::UnknownError, "Fail to process command: " + cmd);
}
}

5
core/src/sdk/grpc/ClientProxy.h
View File

@ -105,10 +105,7 @@ class ClientProxy : public Connection {
DropPartition(const PartitionParam& partition_param) override;
Status
GetConfig(const std::string& node_name, std::string& value) const override;
Status
SetConfig(const std::string& node_name, const std::string& value) const override;
ProcessCommand(std::string& result, const std::string& cmd, const std::vector<std::string>& params) const override;
private:
std::shared_ptr<::grpc::Channel> channel_;

24
core/src/sdk/include/MilvusApi.h
View File

@ -446,30 +446,18 @@ class Connection {
DropPartition(const PartitionParam& param) = 0;
/**
* @brief Get config method
* @brief Process command
*
* This method is used to set config.
* This method is used to process pre-defined commands.
*
* @param node_name, config node name.
* @param value, config value.
* @param result, result string.
* @param cmd, command string.
* @param params, command parameters.
*
* @return Indicate if this operation is successful.
*/
virtual Status
GetConfig(const std::string& node_name, std::string& value) const = 0;
/**
* @brief Set config method
*
* This method is used to set config.
*
* @param node_name, config node name.
* @param value, config value.
*
* @return Indicate if this operation is successful.
*/
virtual Status
SetConfig(const std::string& node_name, const std::string& value) const = 0;
ProcessCommand(std::string& result, const std::string& cmd, const std::vector<std::string>& params) const = 0;
};
} // namespace milvus

9
core/src/sdk/interface/ConnectionImpl.cpp
View File

@ -162,12 +162,9 @@ ConnectionImpl::DropPartition(const PartitionParam& param) {
}
Status
ConnectionImpl::GetConfig(const std::string& node_name, std::string& value) const {
return client_proxy_->GetConfig(node_name, value);
ConnectionImpl::ProcessCommand(std::string& result, const std::string& cmd,
const std::vector<std::string>& params) const {
return client_proxy_->ProcessCommand(result, cmd, params);
}
Status
ConnectionImpl::SetConfig(const std::string& node_name, const std::string& value) const {
return client_proxy_->SetConfig(node_name, value);
}
} // namespace milvus

5
core/src/sdk/interface/ConnectionImpl.h
View File

@ -107,10 +107,7 @@ class ConnectionImpl : public Connection {
DropPartition(const PartitionParam& param) override;
Status
GetConfig(const std::string& node_name, std::string& value) const override;
Status
SetConfig(const std::string& node_name, const std::string& value) const override;
ProcessCommand(std::string& result, const std::string& cmd, const std::vector<std::string>& params) const override;
private:
std::shared_ptr<ClientProxy> client_proxy_;

43
core/src/server/Config.cpp
View File

@ -26,6 +26,7 @@
#include "config/YamlConfigMgr.h"
#include "server/Config.h"
#include "thirdparty/nlohmann/json.hpp"
#include "utils/CommonUtil.h"
#include "utils/StringHelpFunctions.h"
#include "utils/ValidationUtil.h"
@ -380,28 +381,13 @@ Config::ResetDefaultConfig() {
}
void
Config::PrintConfigSection(const std::string& config_node_name) {
std::cout << std::endl;
std::cout << config_node_name << ":" << std::endl;
if (config_map_.find(config_node_name) != config_map_.end()) {
for (auto item : config_map_[config_node_name]) {
std::cout << item.first << ": " << item.second << std::endl;
}
}
}
void
Config::PrintAll() {
PrintConfigSection(CONFIG_SERVER);
PrintConfigSection(CONFIG_DB);
PrintConfigSection(CONFIG_CACHE);
PrintConfigSection(CONFIG_METRIC);
PrintConfigSection(CONFIG_ENGINE);
PrintConfigSection(CONFIG_GPU_RESOURCE);
Config::GetConfigJsonStr(std::string& result) {
nlohmann::json config_json(config_map_);
result = config_json.dump();
}
Status
Config::GetConfigCli(const std::string& parent_key, const std::string& child_key, std::string& value) {
Config::GetConfigCli(std::string& value, const std::string& parent_key, const std::string& child_key) {
if (!ConfigNodeValid(parent_key, child_key)) {
std::string str = "Config node invalid: " + parent_key + CONFIG_NODE_DELIMITER + child_key;
return Status(SERVER_UNEXPECTED_ERROR, str);
@ -459,20 +445,25 @@ Config::SetConfigCli(const std::string& parent_key, const std::string& child_key
}
Status
Config::HandleConfigCli(std::string& result, const std::string& cmd) {
Config::ProcessConfigCli(std::string& result, const std::string& cmd) {
std::vector<std::string> tokens;
std::vector<std::string> nodes;
server::StringHelpFunctions::SplitStringByDelimeter(cmd, " ", tokens);
if (tokens[0] == "get") {
if (tokens[0] == "get_config") {
if (tokens.size() != 2) {
return Status(SERVER_UNEXPECTED_ERROR, "Invalid command: " + cmd);
}
server::StringHelpFunctions::SplitStringByDelimeter(tokens[1], CONFIG_NODE_DELIMITER, nodes);
if (nodes.size() != 2) {
return Status(SERVER_UNEXPECTED_ERROR, "Invalid command: " + cmd);
if (tokens[1] == "*") {
GetConfigJsonStr(result);
return Status::OK();
} else {
server::StringHelpFunctions::SplitStringByDelimeter(tokens[1], CONFIG_NODE_DELIMITER, nodes);
if (nodes.size() != 2) {
return Status(SERVER_UNEXPECTED_ERROR, "Invalid command: " + cmd);
}
return GetConfigCli(result, nodes[0], nodes[1]);
}
return GetConfigCli(nodes[0], nodes[1], result);
} else if (tokens[0] == "set") {
} else if (tokens[0] == "set_config") {
if (tokens.size() != 3) {
return Status(SERVER_UNEXPECTED_ERROR, "Invalid command: " + cmd);
}

8
core/src/server/Config.h
View File

@ -121,9 +121,9 @@ class Config {
Status
ResetDefaultConfig();
void
PrintAll();
GetConfigJsonStr(std::string& result);
Status
HandleConfigCli(std::string& result, const std::string& cmd);
ProcessConfigCli(std::string& result, const std::string& cmd);
private:
ConfigNode&
@ -136,10 +136,8 @@ class Config {
GetConfigValueInMem(const std::string& parent_key, const std::string& child_key, std::string& value);
Status
SetConfigValueInMem(const std::string& parent_key, const std::string& child_key, const std::string& value);
void
PrintConfigSection(const std::string& config_node_name);
Status
GetConfigCli(const std::string& parent_key, const std::string& child_key, std::string& value);
GetConfigCli(std::string& value, const std::string& parent_key, const std::string& child_key);
Status
SetConfigCli(const std::string& parent_key, const std::string& child_key, const std::string& value);

6
core/src/server/delivery/request/CmdRequest.cpp
View File

@ -52,9 +52,11 @@ CmdRequest::OnExecute() {
#else
result_ = "CPU";
#endif
} else if (cmd_.substr(0, 3) == "set" || cmd_.substr(0, 3) == "get") {
} else if (cmd_ == "build_commit_id") {
result_ = LAST_COMMIT_ID;
} else if (cmd_.substr(0, 10) == "set_config" || cmd_.substr(0, 10) == "get_config") {
server::Config& config = server::Config::GetInstance();
stat = config.HandleConfigCli(result_, cmd_);
stat = config.ProcessConfigCli(result_, cmd_);
} else {
result_ = "Unknown command";
}

67
core/unittest/server/test_config.cpp
View File

@ -311,12 +311,12 @@ TEST_F(ConfigTest, SERVER_CONFIG_VALID_TEST) {
}
std::string gen_get_command(const std::string& parent_node, const std::string& child_node) {
std::string cmd = "get " + parent_node + ms::CONFIG_NODE_DELIMITER + child_node;
std::string cmd = "get_config " + parent_node + ms::CONFIG_NODE_DELIMITER + child_node;
return cmd;
}
std::string gen_set_command(const std::string& parent_node, const std::string& child_node, const std::string& value) {
std::string cmd = "set " + parent_node + ms::CONFIG_NODE_DELIMITER + child_node + " " + value;
std::string cmd = "set_config " + parent_node + ms::CONFIG_NODE_DELIMITER + child_node + " " + value;
return cmd;
}
@ -328,75 +328,78 @@ TEST_F(ConfigTest, SERVER_CONFIG_CLI_TEST) {
std::string get_cmd, set_cmd;
std::string result, dummy;
s = config.ProcessConfigCli(result, "get_config *");
ASSERT_TRUE(s.ok());
/* server config */
std::string server_addr = "192.168.1.155";
get_cmd = gen_get_command(ms::CONFIG_SERVER, ms::CONFIG_SERVER_ADDRESS);
set_cmd = gen_set_command(ms::CONFIG_SERVER, ms::CONFIG_SERVER_ADDRESS, server_addr);
s = config.HandleConfigCli(dummy, set_cmd);
s = config.ProcessConfigCli(dummy, set_cmd);
ASSERT_FALSE(s.ok());
s = config.HandleConfigCli(result, get_cmd);
s = config.ProcessConfigCli(result, get_cmd);
ASSERT_TRUE(s.ok());
/* db config */
std::string db_primary_path = "/home/zilliz";
get_cmd = gen_get_command(ms::CONFIG_DB, ms::CONFIG_DB_PRIMARY_PATH);
set_cmd = gen_set_command(ms::CONFIG_DB, ms::CONFIG_DB_PRIMARY_PATH, db_primary_path);
s = config.HandleConfigCli(dummy, set_cmd);
s = config.ProcessConfigCli(dummy, set_cmd);
ASSERT_FALSE(s.ok());
s = config.HandleConfigCli(result, get_cmd);
s = config.ProcessConfigCli(result, get_cmd);
ASSERT_TRUE(s.ok());
/* metric config */
std::string metric_enable_monitor = "false";
get_cmd = gen_get_command(ms::CONFIG_METRIC, ms::CONFIG_METRIC_ENABLE_MONITOR);
set_cmd = gen_set_command(ms::CONFIG_METRIC, ms::CONFIG_METRIC_ENABLE_MONITOR, metric_enable_monitor);
s = config.HandleConfigCli(dummy, set_cmd);
s = config.ProcessConfigCli(dummy, set_cmd);
ASSERT_FALSE(s.ok());
s = config.HandleConfigCli(result, get_cmd);
s = config.ProcessConfigCli(result, get_cmd);
ASSERT_TRUE(s.ok());
/* cache config */
std::string cache_cpu_cache_capacity = "5";
get_cmd = gen_get_command(ms::CONFIG_CACHE, ms::CONFIG_CACHE_CPU_CACHE_CAPACITY);
set_cmd = gen_set_command(ms::CONFIG_CACHE, ms::CONFIG_CACHE_CPU_CACHE_CAPACITY, cache_cpu_cache_capacity);
s = config.HandleConfigCli(dummy, set_cmd);
s = config.ProcessConfigCli(dummy, set_cmd);
ASSERT_TRUE(s.ok());
s = config.HandleConfigCli(result, get_cmd);
s = config.ProcessConfigCli(result, get_cmd);
ASSERT_TRUE(s.ok());
ASSERT_TRUE(result == cache_cpu_cache_capacity);
std::string cache_cpu_cache_threshold = "0.1";
get_cmd = gen_get_command(ms::CONFIG_CACHE, ms::CONFIG_CACHE_CPU_CACHE_THRESHOLD);
set_cmd = gen_set_command(ms::CONFIG_CACHE, ms::CONFIG_CACHE_CPU_CACHE_THRESHOLD, cache_cpu_cache_threshold);
s = config.HandleConfigCli(dummy, set_cmd);
s = config.ProcessConfigCli(dummy, set_cmd);
ASSERT_TRUE(s.ok());
s = config.HandleConfigCli(result, get_cmd);
s = config.ProcessConfigCli(result, get_cmd);
ASSERT_TRUE(result == cache_cpu_cache_threshold);
std::string cache_insert_data = "true";
get_cmd = gen_get_command(ms::CONFIG_CACHE, ms::CONFIG_CACHE_CACHE_INSERT_DATA);
set_cmd = gen_set_command(ms::CONFIG_CACHE, ms::CONFIG_CACHE_CACHE_INSERT_DATA, cache_insert_data);
s = config.HandleConfigCli(dummy, set_cmd);
s = config.ProcessConfigCli(dummy, set_cmd);
ASSERT_TRUE(s.ok());
s = config.HandleConfigCli(result, get_cmd);
s = config.ProcessConfigCli(result, get_cmd);
ASSERT_TRUE(result == cache_insert_data);
/* engine config */
std::string engine_use_blas_threshold = "50";
get_cmd = gen_get_command(ms::CONFIG_ENGINE, ms::CONFIG_ENGINE_USE_BLAS_THRESHOLD);
set_cmd = gen_set_command(ms::CONFIG_ENGINE, ms::CONFIG_ENGINE_USE_BLAS_THRESHOLD, engine_use_blas_threshold);
s = config.HandleConfigCli(dummy, set_cmd);
s = config.ProcessConfigCli(dummy, set_cmd);
ASSERT_TRUE(s.ok());
s = config.HandleConfigCli(result, get_cmd);
s = config.ProcessConfigCli(result, get_cmd);
ASSERT_TRUE(s.ok());
ASSERT_TRUE(result == engine_use_blas_threshold);
std::string engine_omp_thread_num = "8";
get_cmd = gen_get_command(ms::CONFIG_ENGINE, ms::CONFIG_ENGINE_OMP_THREAD_NUM);
set_cmd = gen_set_command(ms::CONFIG_ENGINE, ms::CONFIG_ENGINE_OMP_THREAD_NUM, engine_omp_thread_num);
s = config.HandleConfigCli(dummy, set_cmd);
s = config.ProcessConfigCli(dummy, set_cmd);
ASSERT_TRUE(s.ok());
s = config.HandleConfigCli(result, get_cmd);
s = config.ProcessConfigCli(result, get_cmd);
ASSERT_TRUE(s.ok());
ASSERT_TRUE(result == engine_omp_thread_num);
@ -404,9 +407,9 @@ TEST_F(ConfigTest, SERVER_CONFIG_CLI_TEST) {
std::string engine_gpu_search_threshold = "800";
get_cmd = gen_get_command(ms::CONFIG_ENGINE, ms::CONFIG_ENGINE_GPU_SEARCH_THRESHOLD);
set_cmd = gen_set_command(ms::CONFIG_ENGINE, ms::CONFIG_ENGINE_GPU_SEARCH_THRESHOLD, engine_gpu_search_threshold);
s = config.HandleConfigCli(dummy, set_cmd);
s = config.ProcessConfigCli(dummy, set_cmd);
ASSERT_TRUE(s.ok());
s = config.HandleConfigCli(result, get_cmd);
s = config.ProcessConfigCli(result, get_cmd);
ASSERT_TRUE(s.ok());
ASSERT_TRUE(result == engine_gpu_search_threshold);
@ -414,35 +417,35 @@ TEST_F(ConfigTest, SERVER_CONFIG_CLI_TEST) {
std::string resource_enable_gpu = "true";
get_cmd = gen_get_command(ms::CONFIG_GPU_RESOURCE, ms::CONFIG_GPU_RESOURCE_ENABLE);
set_cmd = gen_set_command(ms::CONFIG_GPU_RESOURCE, ms::CONFIG_GPU_RESOURCE_ENABLE, resource_enable_gpu);
s = config.HandleConfigCli(dummy, set_cmd);
s = config.ProcessConfigCli(dummy, set_cmd);
ASSERT_TRUE(s.ok());
s = config.HandleConfigCli(result, get_cmd);
s = config.ProcessConfigCli(result, get_cmd);
ASSERT_TRUE(s.ok());
ASSERT_TRUE(result == resource_enable_gpu);
std::string gpu_cache_capacity = "1";
get_cmd = gen_get_command(ms::CONFIG_GPU_RESOURCE, ms::CONFIG_GPU_RESOURCE_CACHE_CAPACITY);
set_cmd = gen_set_command(ms::CONFIG_GPU_RESOURCE, ms::CONFIG_GPU_RESOURCE_CACHE_CAPACITY, gpu_cache_capacity);
s = config.HandleConfigCli(dummy, set_cmd);
s = config.ProcessConfigCli(dummy, set_cmd);
ASSERT_TRUE(s.ok());
s = config.HandleConfigCli(result, get_cmd);
s = config.ProcessConfigCli(result, get_cmd);
ASSERT_TRUE(s.ok());
ASSERT_TRUE(result == gpu_cache_capacity);
std::string gpu_cache_threshold = "0.2";
get_cmd = gen_get_command(ms::CONFIG_GPU_RESOURCE, ms::CONFIG_GPU_RESOURCE_CACHE_THRESHOLD);
set_cmd = gen_set_command(ms::CONFIG_GPU_RESOURCE, ms::CONFIG_GPU_RESOURCE_CACHE_THRESHOLD, gpu_cache_threshold);
s = config.HandleConfigCli(dummy, set_cmd);
s = config.ProcessConfigCli(dummy, set_cmd);
ASSERT_TRUE(s.ok());
s = config.HandleConfigCli(result, get_cmd);
s = config.ProcessConfigCli(result, get_cmd);
ASSERT_TRUE(result == gpu_cache_threshold);
std::string search_resources = "gpu0";
get_cmd = gen_get_command(ms::CONFIG_GPU_RESOURCE, ms::CONFIG_GPU_RESOURCE_SEARCH_RESOURCES);
set_cmd = gen_set_command(ms::CONFIG_GPU_RESOURCE, ms::CONFIG_GPU_RESOURCE_SEARCH_RESOURCES, search_resources);
s = config.HandleConfigCli(dummy, set_cmd);
s = config.ProcessConfigCli(dummy, set_cmd);
ASSERT_TRUE(s.ok());
s = config.HandleConfigCli(result, get_cmd);
s = config.ProcessConfigCli(result, get_cmd);
ASSERT_TRUE(s.ok());
ASSERT_TRUE(result == search_resources);
@ -450,9 +453,9 @@ TEST_F(ConfigTest, SERVER_CONFIG_CLI_TEST) {
get_cmd = gen_get_command(ms::CONFIG_GPU_RESOURCE, ms::CONFIG_GPU_RESOURCE_BUILD_INDEX_RESOURCES);
set_cmd =
gen_set_command(ms::CONFIG_GPU_RESOURCE, ms::CONFIG_GPU_RESOURCE_BUILD_INDEX_RESOURCES, build_index_resources);
s = config.HandleConfigCli(dummy, set_cmd);
s = config.ProcessConfigCli(dummy, set_cmd);
ASSERT_TRUE(s.ok());
s = config.HandleConfigCli(result, get_cmd);
s = config.ProcessConfigCli(result, get_cmd);
ASSERT_TRUE(s.ok());
ASSERT_TRUE(result == build_index_resources);
#endif
@ -591,7 +594,9 @@ TEST_F(ConfigTest, SERVER_CONFIG_TEST) {
s = config.ValidateConfig();
ASSERT_TRUE(s.ok());
config.PrintAll();
std::string config_json_str;
config.GetConfigJsonStr(config_json_str);
std::cout << config_json_str << std::endl;
s = config.ResetDefaultConfig();
ASSERT_TRUE(s.ok());