mirror of https://github.com/milvus-io/milvus.git
Add cfg fun (#1501)
* save work Signed-off-by: lichengming <chengming.li@zilliz.com> * wal config>est Signed-off-by: lichengming <chengming.li@zilliz.com> * fix dco and lint error Signed-off-by: lichengming <chengming.li@zilliz.com> * try to fix lint error Signed-off-by: lichengming <chengming.li@zilliz.com> * try to fix clang-format issue Signed-off-by: lichengming <chengming.li@zilliz.com> * update change log Signed-off-by: lichengming <chengming.li@zilliz.com> * ajdjust the code by yudong's advice Signed-off-by: lichengming <chengming.li@zilliz.com> * fix error Signed-off-by: lichengming <chengming.li@zilliz.com> * recommit pr 1501,local gtest is ok Signed-off-by: lichengming <chengming.li@zilliz.com>pull/1514/head^2
Jin Hai
GitHub
parent
8c7ef47f20
commit
570ecf4266
|
|
@ -32,6 +32,7 @@ Please mark all change in change log and use the issue from GitHub
|
|||
- \#1499 Fix duplicated ID number issue
|
||||
- \#1491 Server crashed during adding vectors
|
||||
- \#1504 Avoid possible race condition between delete and search
|
||||
- \#1510 Add set interfaces for WAL configurations
|
||||
|
||||
## Feature
|
||||
- \#216 Add CLI to get server info
|
||||
|
|
@ -44,7 +45,7 @@ Please mark all change in change log and use the issue from GitHub
|
|||
- \#813 Add push mode for prometheus monitor
|
||||
- \#815 Support MinIO storage
|
||||
- \#823 Support binary vector tanimoto/jaccard/hamming metric
|
||||
- \#830 - Support WAL(write-ahead logging)
|
||||
- \#830 Support WAL(write-ahead logging)
|
||||
- \#853 Support HNSW
|
||||
- \#861 Support DeleteById / SearchByID / GetVectorById / Flush
|
||||
- \#910 Change Milvus c++ standard to c++17
|
||||
|
|
|
|||
|
|
@ -284,6 +284,12 @@ Config::ResetDefaultConfig() {
|
|||
CONFIG_CHECK(SetEngineConfigUseBlasThreshold(CONFIG_ENGINE_USE_BLAS_THRESHOLD_DEFAULT));
|
||||
CONFIG_CHECK(SetEngineConfigOmpThreadNum(CONFIG_ENGINE_OMP_THREAD_NUM_DEFAULT));
|
||||
CONFIG_CHECK(SetEngineConfigUseAVX512(CONFIG_ENGINE_USE_AVX512_DEFAULT));
|
||||
|
||||
/* wal config */
|
||||
CONFIG_CHECK(SetWalConfigEnable(CONFIG_WAL_ENABLE_DEFAULT));
|
||||
CONFIG_CHECK(SetWalConfigRecoveryErrorIgnore(CONFIG_WAL_RECOVERY_ERROR_IGNORE_DEFAULT));
|
||||
CONFIG_CHECK(SetWalConfigBufferSize(CONFIG_WAL_BUFFER_SIZE_DEFAULT));
|
||||
CONFIG_CHECK(SetWalConfigWalPath(CONFIG_WAL_WAL_PATH_DEFAULT));
|
||||
#ifdef MILVUS_GPU_VERSION
|
||||
CONFIG_CHECK(SetEngineConfigGpuSearchThreshold(CONFIG_ENGINE_GPU_SEARCH_THRESHOLD_DEFAULT));
|
||||
#endif
|
||||
|
|
@ -402,6 +408,16 @@ Config::SetConfigCli(const std::string& parent_key, const std::string& child_key
|
|||
#endif
|
||||
} else if (parent_key == CONFIG_TRACING) {
|
||||
return Status(SERVER_UNSUPPORTED_ERROR, "Not support set tracing_config currently");
|
||||
} else if (parent_key == CONFIG_WAL) {
|
||||
if (child_key == CONFIG_WAL_ENABLE) {
|
||||
status = SetWalConfigEnable(value);
|
||||
} else if (child_key == CONFIG_WAL_RECOVERY_ERROR_IGNORE) {
|
||||
status = SetWalConfigRecoveryErrorIgnore(value);
|
||||
} else if (child_key == CONFIG_WAL_BUFFER_SIZE) {
|
||||
status = SetWalConfigBufferSize(value);
|
||||
} else if (child_key == CONFIG_WAL_WAL_PATH) {
|
||||
status = SetWalConfigWalPath(value);
|
||||
}
|
||||
}
|
||||
|
||||
if (status.ok()) {
|
||||
|
|
@ -1201,7 +1217,10 @@ Config::CheckGpuResourceConfigBuildIndexResources(const std::vector<std::string>
|
|||
/* wal config */
|
||||
Status
|
||||
Config::CheckWalConfigEnable(const std::string& value) {
|
||||
if (!ValidationUtil::ValidateStringIsBool(value).ok()) {
|
||||
auto exist_error = !ValidationUtil::ValidateStringIsBool(value).ok();
|
||||
fiu_do_on("check_config_wal_enable_fail", exist_error = true);
|
||||
|
||||
if (exist_error) {
|
||||
std::string msg = "Invalid wal config: " + value + ". Possible reason: wal_config.enable is not a boolean.";
|
||||
return Status(SERVER_INVALID_ARGUMENT, msg);
|
||||
}
|
||||
|
|
@ -1210,7 +1229,10 @@ Config::CheckWalConfigEnable(const std::string& value) {
|
|||
|
||||
Status
|
||||
Config::CheckWalConfigRecoveryErrorIgnore(const std::string& value) {
|
||||
if (!ValidationUtil::ValidateStringIsBool(value).ok()) {
|
||||
auto exist_error = !ValidationUtil::ValidateStringIsBool(value).ok();
|
||||
fiu_do_on("check_config_wal_recovery_error_ignore_fail", exist_error = true);
|
||||
|
||||
if (exist_error) {
|
||||
std::string msg =
|
||||
"Invalid wal config: " + value + ". Possible reason: wal_config.recovery_error_ignore is not a boolean.";
|
||||
return Status(SERVER_INVALID_ARGUMENT, msg);
|
||||
|
|
@ -1220,7 +1242,10 @@ Config::CheckWalConfigRecoveryErrorIgnore(const std::string& value) {
|
|||
|
||||
Status
|
||||
Config::CheckWalConfigBufferSize(const std::string& value) {
|
||||
if (!ValidationUtil::ValidateStringIsNumber(value).ok()) {
|
||||
auto exist_error = !ValidationUtil::ValidateStringIsNumber(value).ok();
|
||||
fiu_do_on("check_config_wal_buffer_size_fail", exist_error = true);
|
||||
|
||||
if (exist_error) {
|
||||
std::string msg = "Invalid wal buffer size: " + value +
|
||||
". Possible reason: wal_config.buffer_size is not a positive integer.";
|
||||
return Status(SERVER_INVALID_ARGUMENT, msg);
|
||||
|
|
@ -1228,6 +1253,16 @@ Config::CheckWalConfigBufferSize(const std::string& value) {
|
|||
return Status::OK();
|
||||
}
|
||||
|
||||
Status
|
||||
Config::CheckWalConfigWalPath(const std::string& value) {
|
||||
fiu_return_on("check_wal_path_fail", Status(SERVER_INVALID_ARGUMENT, ""));
|
||||
if (value.empty()) {
|
||||
return Status(SERVER_INVALID_ARGUMENT, "wal_config.wal_path is empty!");
|
||||
}
|
||||
|
||||
return ValidationUtil::ValidateStoragePath(value);
|
||||
}
|
||||
|
||||
////////////////////////////////////////////////////////////////////////////////
|
||||
ConfigNode&
|
||||
Config::GetConfigRoot() {
|
||||
|
|
@ -1688,6 +1723,7 @@ Config::GetWalConfigBufferSize(int64_t& buffer_size) {
|
|||
Status
|
||||
Config::GetWalConfigWalPath(std::string& wal_path) {
|
||||
wal_path = GetConfigStr(CONFIG_WAL, CONFIG_WAL_WAL_PATH, CONFIG_WAL_WAL_PATH_DEFAULT);
|
||||
CONFIG_CHECK(CheckWalConfigWalPath(wal_path));
|
||||
return Status::OK();
|
||||
}
|
||||
|
||||
|
|
@ -1865,6 +1901,31 @@ Config::SetEngineConfigUseAVX512(const std::string& value) {
|
|||
return SetConfigValueInMem(CONFIG_ENGINE, CONFIG_ENGINE_USE_AVX512, value);
|
||||
}
|
||||
|
||||
/* wal config */
|
||||
Status
|
||||
Config::SetWalConfigEnable(const std::string& value) {
|
||||
CONFIG_CHECK(CheckWalConfigEnable(value));
|
||||
return SetConfigValueInMem(CONFIG_WAL, CONFIG_WAL_ENABLE, value);
|
||||
}
|
||||
|
||||
Status
|
||||
Config::SetWalConfigRecoveryErrorIgnore(const std::string& value) {
|
||||
CONFIG_CHECK(CheckWalConfigRecoveryErrorIgnore(value));
|
||||
return SetConfigValueInMem(CONFIG_WAL, CONFIG_WAL_RECOVERY_ERROR_IGNORE_DEFAULT, value);
|
||||
}
|
||||
|
||||
Status
|
||||
Config::SetWalConfigBufferSize(const std::string& value) {
|
||||
CONFIG_CHECK(CheckWalConfigBufferSize(value));
|
||||
return SetConfigValueInMem(CONFIG_WAL, CONFIG_WAL_BUFFER_SIZE, value);
|
||||
}
|
||||
|
||||
Status
|
||||
Config::SetWalConfigWalPath(const std::string& value) {
|
||||
CONFIG_CHECK(CheckWalConfigWalPath(value));
|
||||
return SetConfigValueInMem(CONFIG_WAL, CONFIG_WAL_WAL_PATH, value);
|
||||
}
|
||||
|
||||
#ifdef MILVUS_GPU_VERSION
|
||||
Status
|
||||
Config::SetEngineConfigGpuSearchThreshold(const std::string& value) {
|
||||
|
|
|
|||
|
|
@ -288,6 +288,8 @@ class Config {
|
|||
CheckWalConfigRecoveryErrorIgnore(const std::string& value);
|
||||
Status
|
||||
CheckWalConfigBufferSize(const std::string& value);
|
||||
Status
|
||||
CheckWalConfigWalPath(const std::string& value);
|
||||
|
||||
std::string
|
||||
GetConfigStr(const std::string& parent_key, const std::string& child_key, const std::string& default_value = "");
|
||||
|
|
@ -392,13 +394,13 @@ class Config {
|
|||
|
||||
/* wal config */
|
||||
Status
|
||||
GetWalConfigEnable(bool& wal_enable);
|
||||
GetWalConfigEnable(bool& value);
|
||||
Status
|
||||
GetWalConfigRecoveryErrorIgnore(bool& recovery_error_ignore);
|
||||
GetWalConfigRecoveryErrorIgnore(bool& value);
|
||||
Status
|
||||
GetWalConfigBufferSize(int64_t& buffer_size);
|
||||
GetWalConfigBufferSize(int64_t& value);
|
||||
Status
|
||||
GetWalConfigWalPath(std::string& wal_path);
|
||||
GetWalConfigWalPath(std::string& value);
|
||||
|
||||
Status
|
||||
GetServerRestartRequired(bool& required);
|
||||
|
|
@ -468,6 +470,16 @@ class Config {
|
|||
Status
|
||||
SetEngineConfigUseAVX512(const std::string& value);
|
||||
|
||||
/* wal config */
|
||||
Status
|
||||
SetWalConfigEnable(const std::string& value);
|
||||
Status
|
||||
SetWalConfigRecoveryErrorIgnore(const std::string& value);
|
||||
Status
|
||||
SetWalConfigBufferSize(const std::string& value);
|
||||
Status
|
||||
SetWalConfigWalPath(const std::string& value);
|
||||
|
||||
#ifdef MILVUS_GPU_VERSION
|
||||
Status
|
||||
SetEngineConfigGpuSearchThreshold(const std::string& value);
|
||||
|
|
|
|||
|
|
@ -56,6 +56,7 @@ TEST_F(ConfigTest, CONFIG_TEST) {
|
|||
milvus::server::ConfigNode& db_config = root_config.GetChild("db_config");
|
||||
milvus::server::ConfigNode& metric_config = root_config.GetChild("metric_config");
|
||||
milvus::server::ConfigNode& cache_config = root_config.GetChild("cache_config");
|
||||
milvus::server::ConfigNode& wal_config = root_config.GetChild("wal_config");
|
||||
milvus::server::ConfigNode invalid_config = root_config.GetChild("invalid_config");
|
||||
|
||||
const auto& im_config_mgr = *static_cast<milvus::server::YamlConfigMgr*>(config_mgr);
|
||||
|
|
@ -323,6 +324,27 @@ TEST_F(ConfigTest, SERVER_CONFIG_VALID_TEST) {
|
|||
ASSERT_TRUE(std::stoll(build_index_resources[i].substr(3)) == build_index_res_vec[i]);
|
||||
}
|
||||
#endif
|
||||
|
||||
/* wal config */
|
||||
bool wal_enable = false;
|
||||
ASSERT_TRUE(config.SetWalConfigEnable(std::to_string(wal_enable)).ok());
|
||||
ASSERT_TRUE(config.GetWalConfigEnable(bool_val).ok());
|
||||
ASSERT_TRUE(bool_val == wal_enable);
|
||||
|
||||
bool wal_recovery_ignore = true;
|
||||
ASSERT_TRUE(config.SetWalConfigRecoveryErrorIgnore(std::to_string(wal_recovery_ignore)).ok());
|
||||
ASSERT_TRUE(config.GetWalConfigRecoveryErrorIgnore(bool_val).ok());
|
||||
ASSERT_TRUE(bool_val == wal_recovery_ignore);
|
||||
|
||||
int64_t wal_buffer_size = 128;
|
||||
ASSERT_TRUE(config.SetWalConfigBufferSize(std::to_string(wal_buffer_size)).ok());
|
||||
ASSERT_TRUE(config.GetWalConfigBufferSize(int64_val).ok());
|
||||
ASSERT_TRUE(int64_val == wal_buffer_size);
|
||||
|
||||
std::string wal_path = "/tmp/aaa/wal";
|
||||
ASSERT_TRUE(config.SetWalConfigWalPath(wal_path).ok());
|
||||
ASSERT_TRUE(config.GetWalConfigWalPath(str_val).ok());
|
||||
ASSERT_TRUE(str_val == wal_path);
|
||||
}
|
||||
|
||||
std::string
|
||||
|
|
@ -625,6 +647,12 @@ TEST_F(ConfigTest, SERVER_CONFIG_INVALID_TEST) {
|
|||
ASSERT_FALSE(config.SetGpuResourceConfigBuildIndexResources("gpu16").ok());
|
||||
ASSERT_FALSE(config.SetGpuResourceConfigBuildIndexResources("gpu0, gpu0, gpu1").ok());
|
||||
#endif
|
||||
|
||||
/* wal config */
|
||||
ASSERT_FALSE(config.SetWalConfigWalPath("hello/world").ok());
|
||||
ASSERT_FALSE(config.SetWalConfigWalPath("").ok());
|
||||
ASSERT_FALSE(config.SetWalConfigBufferSize("-1").ok());
|
||||
ASSERT_FALSE(config.SetWalConfigBufferSize("a").ok());
|
||||
}
|
||||
|
||||
TEST_F(ConfigTest, SERVER_CONFIG_TEST) {
|
||||
|
|
@ -844,6 +872,27 @@ TEST_F(ConfigTest, SERVER_CONFIG_VALID_FAIL_TEST) {
|
|||
ASSERT_FALSE(s.ok());
|
||||
fiu_disable("Config.GetGpuResourceConfigCacheThreshold.diable_gpu_resource");
|
||||
#endif
|
||||
|
||||
/* wal config */
|
||||
fiu_enable("check_config_wal_enable_fail", 1, NULL, 0);
|
||||
s = config.ValidateConfig();
|
||||
ASSERT_FALSE(s.ok());
|
||||
fiu_disable("check_config_wal_enable_fail");
|
||||
|
||||
fiu_enable("check_config_wal_recovery_error_ignore_fail", 1, NULL, 0);
|
||||
s = config.ValidateConfig();
|
||||
ASSERT_FALSE(s.ok());
|
||||
fiu_disable("check_config_wal_recovery_error_ignore_fail");
|
||||
|
||||
fiu_enable("check_config_wal_buffer_size_fail", 1, NULL, 0);
|
||||
s = config.ValidateConfig();
|
||||
ASSERT_FALSE(s.ok());
|
||||
fiu_disable("check_config_wal_buffer_size_fail");
|
||||
|
||||
fiu_enable("check_wal_path_fail", 1, NULL, 0);
|
||||
s = config.ValidateConfig();
|
||||
ASSERT_FALSE(s.ok());
|
||||
fiu_disable("check_wal_path_fail");
|
||||
}
|
||||
|
||||
TEST_F(ConfigTest, SERVER_CONFIG_RESET_DEFAULT_CONFIG_FAIL_TEST) {
|
||||
|
|
@ -977,6 +1026,27 @@ TEST_F(ConfigTest, SERVER_CONFIG_RESET_DEFAULT_CONFIG_FAIL_TEST) {
|
|||
|
||||
s = config.ResetDefaultConfig();
|
||||
ASSERT_TRUE(s.ok());
|
||||
|
||||
/* wal config */
|
||||
fiu_enable("check_config_wal_enable_fail", 1, NULL, 0);
|
||||
s = config.ResetDefaultConfig();
|
||||
ASSERT_FALSE(s.ok());
|
||||
fiu_disable("check_config_wal_enable_fail");
|
||||
|
||||
fiu_enable("check_config_wal_recovery_error_ignore_fail", 1, NULL, 0);
|
||||
s = config.ResetDefaultConfig();
|
||||
ASSERT_FALSE(s.ok());
|
||||
fiu_disable("check_config_wal_recovery_error_ignore_fail");
|
||||
|
||||
fiu_enable("check_config_wal_buffer_size_fail", 1, NULL, 0);
|
||||
s = config.ResetDefaultConfig();
|
||||
ASSERT_FALSE(s.ok());
|
||||
fiu_disable("check_config_wal_buffer_size_fail");
|
||||
|
||||
fiu_enable("check_wal_path_fail", 1, NULL, 0);
|
||||
s = config.ResetDefaultConfig();
|
||||
ASSERT_FALSE(s.ok());
|
||||
fiu_disable("check_wal_path_fail");
|
||||
}
|
||||
|
||||
TEST_F(ConfigTest, SERVER_CONFIG_OTHER_CONFIGS_FAIL_TEST) {
|
||||
|
|
|
|||
Loading…
Reference in New Issue