diff --git a/CHANGELOG.md b/CHANGELOG.md index 5b994e48ba..24e33cdd10 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -36,6 +36,7 @@ Please mark all change in change log and use the issue from GitHub - \#1510 Add set interfaces for WAL configurations - \#1511 Fix big integer cannot pass to server correctly - \#1518 Table count did not match after deleting vectors and compact +- \#1525 Add setter API for config preload_table ## Feature - \#216 Add CLI to get server info diff --git a/core/src/CMakeLists.txt b/core/src/CMakeLists.txt index ddeb6207e3..3978a407fd 100644 --- a/core/src/CMakeLists.txt +++ b/core/src/CMakeLists.txt @@ -30,6 +30,7 @@ endforeach () aux_source_directory(${MILVUS_ENGINE_SRC}/cache cache_files) aux_source_directory(${MILVUS_ENGINE_SRC}/config config_files) +aux_source_directory(${MILVUS_ENGINE_SRC}/config/handler config_handler_files) aux_source_directory(${MILVUS_ENGINE_SRC}/metrics metrics_files) aux_source_directory(${MILVUS_ENGINE_SRC}/metrics/prometheus metrics_prometheus_files) aux_source_directory(${MILVUS_ENGINE_SRC}/db db_main_files) @@ -50,7 +51,6 @@ aux_source_directory(${MILVUS_ENGINE_SRC}/scheduler/action scheduler_action_file aux_source_directory(${MILVUS_ENGINE_SRC}/scheduler/event scheduler_event_files) aux_source_directory(${MILVUS_ENGINE_SRC}/scheduler/job scheduler_job_files) aux_source_directory(${MILVUS_ENGINE_SRC}/scheduler/optimizer scheduler_optimizer_files) -aux_source_directory(${MILVUS_ENGINE_SRC}/scheduler/optimizer/handler scheduler_optimizer_handler_files) aux_source_directory(${MILVUS_ENGINE_SRC}/scheduler/resource scheduler_resource_files) aux_source_directory(${MILVUS_ENGINE_SRC}/scheduler/task scheduler_task_files) set(scheduler_files @@ -59,7 +59,6 @@ set(scheduler_files ${scheduler_event_files} ${scheduler_job_files} ${scheduler_optimizer_files} - ${scheduler_optimizer_handler_files} ${scheduler_resource_files} ${scheduler_task_files} ) @@ -279,6 +278,7 @@ set(server_libs add_executable(milvus_server ${config_files} + ${config_handler_files} ${metrics_files} ${scheduler_files} ${server_files} diff --git a/core/src/scheduler/optimizer/handler/GpuBuildResHandler.cpp b/core/src/config/handler/GpuBuildResHandler.cpp similarity index 85% rename from core/src/scheduler/optimizer/handler/GpuBuildResHandler.cpp rename to core/src/config/handler/GpuBuildResHandler.cpp index 66fbc269aa..6a57812b85 100644 --- a/core/src/scheduler/optimizer/handler/GpuBuildResHandler.cpp +++ b/core/src/config/handler/GpuBuildResHandler.cpp @@ -9,13 +9,13 @@ // is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express // or implied. See the License for the specific language governing permissions and limitations under the License. #ifdef MILVUS_GPU_VERSION -#include "scheduler/optimizer/handler/GpuBuildResHandler.h" +#include "config/handler/GpuBuildResHandler.h" #include #include namespace milvus { -namespace scheduler { +namespace server { GpuBuildResHandler::GpuBuildResHandler() { server::Config& config = server::Config::GetInstance(); @@ -50,6 +50,12 @@ GpuBuildResHandler::AddGpuBuildResListener() { lambda); } -} // namespace scheduler +void +GpuBuildResHandler::RemoveGpuBuildResListener() { + auto& config = server::Config::GetInstance(); + config.CancelCallBack(server::CONFIG_GPU_RESOURCE, server::CONFIG_GPU_RESOURCE_BUILD_INDEX_RESOURCES, identity_); +} + +} // namespace server } // namespace milvus #endif diff --git a/core/src/scheduler/optimizer/handler/GpuBuildResHandler.h b/core/src/config/handler/GpuBuildResHandler.h similarity index 88% rename from core/src/scheduler/optimizer/handler/GpuBuildResHandler.h rename to core/src/config/handler/GpuBuildResHandler.h index 9448f20f92..6047836bcb 100644 --- a/core/src/scheduler/optimizer/handler/GpuBuildResHandler.h +++ b/core/src/config/handler/GpuBuildResHandler.h @@ -11,12 +11,12 @@ #ifdef MILVUS_GPU_VERSION #pragma once -#include "scheduler/optimizer/handler/GpuResourcesHandler.h" - #include +#include "config/handler/GpuResourcesHandler.h" + namespace milvus { -namespace scheduler { +namespace server { class GpuBuildResHandler : virtual public GpuResourcesHandler { public: @@ -32,10 +32,13 @@ class GpuBuildResHandler : virtual public GpuResourcesHandler { void AddGpuBuildResListener(); + void + RemoveGpuBuildResListener(); + protected: std::vector build_gpus_; }; -} // namespace scheduler +} // namespace server } // namespace milvus #endif diff --git a/core/src/scheduler/optimizer/handler/GpuResourcesHandler.cpp b/core/src/config/handler/GpuResourcesHandler.cpp similarity index 91% rename from core/src/scheduler/optimizer/handler/GpuResourcesHandler.cpp rename to core/src/config/handler/GpuResourcesHandler.cpp index aa9b6e883b..65659985d6 100644 --- a/core/src/scheduler/optimizer/handler/GpuResourcesHandler.cpp +++ b/core/src/config/handler/GpuResourcesHandler.cpp @@ -10,10 +10,10 @@ // is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express // or implied. See the License for the specific language governing permissions and limitations under the License. #ifdef MILVUS_GPU_VERSION -#include "scheduler/optimizer/handler/GpuResourcesHandler.h" +#include "config/handler/GpuResourcesHandler.h" namespace milvus { -namespace scheduler { +namespace server { GpuResourcesHandler::GpuResourcesHandler() { server::Config& config = server::Config::GetInstance(); @@ -21,8 +21,7 @@ GpuResourcesHandler::GpuResourcesHandler() { } GpuResourcesHandler::~GpuResourcesHandler() { - server::Config& config = server::Config::GetInstance(); - config.CancelCallBack(server::CONFIG_GPU_RESOURCE, server::CONFIG_GPU_RESOURCE_ENABLE, identity_); + RemoveGpuEnableListener(); } ////////////////////////////////////////////////////////////// @@ -54,6 +53,12 @@ GpuResourcesHandler::AddGpuEnableListener() { config.RegisterCallBack(server::CONFIG_GPU_RESOURCE, server::CONFIG_GPU_RESOURCE_ENABLE, identity_, lambda); } -} // namespace scheduler +void +GpuResourcesHandler::RemoveGpuEnableListener() { + server::Config& config = server::Config::GetInstance(); + config.CancelCallBack(server::CONFIG_GPU_RESOURCE, server::CONFIG_GPU_RESOURCE_ENABLE, identity_); +} + +} // namespace server } // namespace milvus #endif diff --git a/core/src/scheduler/optimizer/handler/GpuResourcesHandler.h b/core/src/config/handler/GpuResourcesHandler.h similarity index 92% rename from core/src/scheduler/optimizer/handler/GpuResourcesHandler.h rename to core/src/config/handler/GpuResourcesHandler.h index 6c36280a18..d9e5dc601e 100644 --- a/core/src/scheduler/optimizer/handler/GpuResourcesHandler.h +++ b/core/src/config/handler/GpuResourcesHandler.h @@ -18,7 +18,7 @@ #include "server/Config.h" namespace milvus { -namespace scheduler { +namespace server { class GpuResourcesHandler { public: @@ -26,7 +26,7 @@ class GpuResourcesHandler { ~GpuResourcesHandler(); - public: + protected: virtual void OnGpuEnableChanged(bool enable); @@ -37,11 +37,14 @@ class GpuResourcesHandler { void AddGpuEnableListener(); + void + RemoveGpuEnableListener(); + protected: bool gpu_enable_ = true; std::string identity_; }; -} // namespace scheduler +} // namespace server } // namespace milvus #endif diff --git a/core/src/scheduler/optimizer/handler/GpuSearchResHandler.cpp b/core/src/config/handler/GpuSearchResHandler.cpp similarity index 89% rename from core/src/scheduler/optimizer/handler/GpuSearchResHandler.cpp rename to core/src/config/handler/GpuSearchResHandler.cpp index 5a1e94f32e..781e5aa527 100644 --- a/core/src/scheduler/optimizer/handler/GpuSearchResHandler.cpp +++ b/core/src/config/handler/GpuSearchResHandler.cpp @@ -9,7 +9,7 @@ // is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express // or implied. See the License for the specific language governing permissions and limitations under the License. #ifdef MILVUS_GPU_VERSION -#include "scheduler/optimizer/handler/GpuSearchResHandler.h" +#include "config/handler/GpuSearchResHandler.h" #include #include @@ -17,7 +17,7 @@ #include "server/Config.h" namespace milvus { -namespace scheduler { +namespace server { GpuSearchResHandler::GpuSearchResHandler() { server::Config& config = server::Config::GetInstance(); @@ -31,9 +31,8 @@ GpuSearchResHandler::GpuSearchResHandler() { } GpuSearchResHandler::~GpuSearchResHandler() { - server::Config& config = server::Config::GetInstance(); - config.CancelCallBack(server::CONFIG_ENGINE, server::CONFIG_ENGINE_GPU_SEARCH_THRESHOLD, identity_); - config.CancelCallBack(server::CONFIG_GPU_RESOURCE, server::CONFIG_GPU_RESOURCE_SEARCH_RESOURCES, identity_); + RemoveGpuSearchThresholdListener(); + RemoveGpuSearchResListener(); } //////////////////////////////////////////////////////////////////////// @@ -83,6 +82,18 @@ GpuSearchResHandler::AddGpuSearchResListener() { lambda_gpu_search_res); } -} // namespace scheduler +void +GpuSearchResHandler::RemoveGpuSearchThresholdListener() { + server::Config& config = server::Config::GetInstance(); + config.CancelCallBack(server::CONFIG_ENGINE, server::CONFIG_ENGINE_GPU_SEARCH_THRESHOLD, identity_); +} + +void +GpuSearchResHandler::RemoveGpuSearchResListener() { + auto& config = server::Config::GetInstance(); + config.CancelCallBack(server::CONFIG_GPU_RESOURCE, server::CONFIG_GPU_RESOURCE_SEARCH_RESOURCES, identity_); +} + +} // namespace server } // namespace milvus #endif diff --git a/core/src/scheduler/optimizer/handler/GpuSearchResHandler.h b/core/src/config/handler/GpuSearchResHandler.h similarity index 87% rename from core/src/scheduler/optimizer/handler/GpuSearchResHandler.h rename to core/src/config/handler/GpuSearchResHandler.h index b86fbc1b55..390db85bb0 100644 --- a/core/src/scheduler/optimizer/handler/GpuSearchResHandler.h +++ b/core/src/config/handler/GpuSearchResHandler.h @@ -11,13 +11,13 @@ #ifdef MILVUS_GPU_VERSION #pragma once -#include "scheduler/optimizer/handler/GpuResourcesHandler.h" - #include #include +#include "config/handler/GpuResourcesHandler.h" + namespace milvus { -namespace scheduler { +namespace server { class GpuSearchResHandler : virtual public GpuResourcesHandler { public: @@ -39,11 +39,17 @@ class GpuSearchResHandler : virtual public GpuResourcesHandler { void AddGpuSearchResListener(); + void + RemoveGpuSearchThresholdListener(); + + void + RemoveGpuSearchResListener(); + protected: int64_t threshold_ = std::numeric_limits::max(); std::vector search_gpus_; }; -} // namespace scheduler +} // namespace server } // namespace milvus #endif diff --git a/core/src/scheduler/optimizer/BuildIndexPass.h b/core/src/scheduler/optimizer/BuildIndexPass.h index 92a9fc9633..d745876902 100644 --- a/core/src/scheduler/optimizer/BuildIndexPass.h +++ b/core/src/scheduler/optimizer/BuildIndexPass.h @@ -22,13 +22,13 @@ #include #include +#include "config/handler/GpuBuildResHandler.h" #include "scheduler/optimizer/Pass.h" -#include "scheduler/optimizer/handler/GpuBuildResHandler.h" namespace milvus { namespace scheduler { -class BuildIndexPass : public Pass, public GpuBuildResHandler { +class BuildIndexPass : public Pass, public server::GpuBuildResHandler { public: BuildIndexPass() = default; diff --git a/core/src/scheduler/optimizer/FaissFlatPass.h b/core/src/scheduler/optimizer/FaissFlatPass.h index 3377266e23..6c78257b34 100644 --- a/core/src/scheduler/optimizer/FaissFlatPass.h +++ b/core/src/scheduler/optimizer/FaissFlatPass.h @@ -23,13 +23,13 @@ #include #include +#include "config/handler/GpuSearchResHandler.h" #include "scheduler/optimizer/Pass.h" -#include "src/scheduler/optimizer/handler/GpuSearchResHandler.h" namespace milvus { namespace scheduler { -class FaissFlatPass : public Pass, public GpuSearchResHandler { +class FaissFlatPass : public Pass, public server::GpuSearchResHandler { public: FaissFlatPass() = default; diff --git a/core/src/scheduler/optimizer/FaissIVFFlatPass.h b/core/src/scheduler/optimizer/FaissIVFFlatPass.h index 128c193722..8686628567 100644 --- a/core/src/scheduler/optimizer/FaissIVFFlatPass.h +++ b/core/src/scheduler/optimizer/FaissIVFFlatPass.h @@ -23,13 +23,13 @@ #include #include +#include "config/handler/GpuSearchResHandler.h" #include "scheduler/optimizer/Pass.h" -#include "scheduler/optimizer/handler/GpuSearchResHandler.h" namespace milvus { namespace scheduler { -class FaissIVFFlatPass : public Pass, public GpuSearchResHandler { +class FaissIVFFlatPass : public Pass, public server::GpuSearchResHandler { public: FaissIVFFlatPass() = default; diff --git a/core/src/scheduler/optimizer/FaissIVFPQPass.h b/core/src/scheduler/optimizer/FaissIVFPQPass.h index 9c88c6913b..3b5ffd90ba 100644 --- a/core/src/scheduler/optimizer/FaissIVFPQPass.h +++ b/core/src/scheduler/optimizer/FaissIVFPQPass.h @@ -23,13 +23,13 @@ #include #include +#include "config/handler/GpuSearchResHandler.h" #include "scheduler/optimizer/Pass.h" -#include "scheduler/optimizer/handler/GpuSearchResHandler.h" namespace milvus { namespace scheduler { -class FaissIVFPQPass : public Pass, public GpuSearchResHandler { +class FaissIVFPQPass : public Pass, public server::GpuSearchResHandler { public: FaissIVFPQPass() = default; diff --git a/core/src/scheduler/optimizer/FaissIVFSQ8HPass.h b/core/src/scheduler/optimizer/FaissIVFSQ8HPass.h index 6d25906499..f23be2da79 100644 --- a/core/src/scheduler/optimizer/FaissIVFSQ8HPass.h +++ b/core/src/scheduler/optimizer/FaissIVFSQ8HPass.h @@ -23,13 +23,13 @@ #include #include +#include "config/handler/GpuSearchResHandler.h" #include "scheduler/optimizer/Pass.h" -#include "scheduler/optimizer/handler/GpuSearchResHandler.h" namespace milvus { namespace scheduler { -class FaissIVFSQ8HPass : public Pass, public GpuSearchResHandler { +class FaissIVFSQ8HPass : public Pass, public server::GpuSearchResHandler { public: FaissIVFSQ8HPass() = default; diff --git a/core/src/scheduler/optimizer/FaissIVFSQ8Pass.h b/core/src/scheduler/optimizer/FaissIVFSQ8Pass.h index 26bcfd0477..5bc283d4e2 100644 --- a/core/src/scheduler/optimizer/FaissIVFSQ8Pass.h +++ b/core/src/scheduler/optimizer/FaissIVFSQ8Pass.h @@ -23,13 +23,13 @@ #include #include +#include "config/handler/GpuSearchResHandler.h" #include "scheduler/optimizer/Pass.h" -#include "scheduler/optimizer/handler/GpuSearchResHandler.h" namespace milvus { namespace scheduler { -class FaissIVFSQ8Pass : public Pass, public GpuSearchResHandler { +class FaissIVFSQ8Pass : public Pass, public server::GpuSearchResHandler { public: FaissIVFSQ8Pass() = default; diff --git a/core/src/server/Config.cpp b/core/src/server/Config.cpp index 3340531f2a..62d8dcf04e 100644 --- a/core/src/server/Config.cpp +++ b/core/src/server/Config.cpp @@ -13,7 +13,6 @@ #include #include #include -#include #include #include #include @@ -24,6 +23,7 @@ #include "config/YamlConfigMgr.h" #include "server/Config.h" +#include "server/DBWrapper.h" #include "thirdparty/nlohmann/json.hpp" #include "utils/CommonUtil.h" #include "utils/StringHelpFunctions.h" @@ -131,6 +131,9 @@ Config::ValidateConfig() { std::string db_backend_url; CONFIG_CHECK(GetDBConfigBackendUrl(db_backend_url)); + std::string db_preload_table; + CONFIG_CHECK(GetDBConfigPreloadTable(db_preload_table)); + int64_t db_archive_disk_threshold; CONFIG_CHECK(GetDBConfigArchiveDiskThreshold(db_archive_disk_threshold)); @@ -256,6 +259,7 @@ Config::ResetDefaultConfig() { /* db config */ CONFIG_CHECK(SetDBConfigBackendUrl(CONFIG_DB_BACKEND_URL_DEFAULT)); + CONFIG_CHECK(SetDBConfigPreloadTable(CONFIG_DB_PRELOAD_TABLE_DEFAULT)); CONFIG_CHECK(SetDBConfigArchiveDiskThreshold(CONFIG_DB_ARCHIVE_DISK_THRESHOLD_DEFAULT)); CONFIG_CHECK(SetDBConfigArchiveDaysThreshold(CONFIG_DB_ARCHIVE_DAYS_THRESHOLD_DEFAULT)); @@ -323,9 +327,10 @@ Config::GetConfigCli(std::string& value, const std::string& parent_key, const st Status Config::SetConfigCli(const std::string& parent_key, const std::string& child_key, const std::string& value) { + std::string invalid_node_str = "Config node invalid: " + parent_key + CONFIG_NODE_DELIMITER + child_key; + if (!ConfigNodeValid(parent_key, child_key)) { - std::string str = "Config node invalid: " + parent_key + CONFIG_NODE_DELIMITER + child_key; - return Status(SERVER_UNEXPECTED_ERROR, str); + return Status(SERVER_UNEXPECTED_ERROR, invalid_node_str); } auto status = Status::OK(); if (parent_key == CONFIG_SERVER) { @@ -339,10 +344,16 @@ Config::SetConfigCli(const std::string& parent_key, const std::string& child_key status = SetServerConfigTimeZone(value); } else if (child_key == CONFIG_SERVER_WEB_PORT) { status = SetServerConfigWebPort(value); + } else { + status = Status(SERVER_UNEXPECTED_ERROR, invalid_node_str); } } else if (parent_key == CONFIG_DB) { if (child_key == CONFIG_DB_BACKEND_URL) { status = SetDBConfigBackendUrl(value); + } else if (child_key == CONFIG_DB_PRELOAD_TABLE) { + status = SetDBConfigPreloadTable(value); + } else { + status = Status(SERVER_UNEXPECTED_ERROR, invalid_node_str); } } else if (parent_key == CONFIG_STORAGE) { if (child_key == CONFIG_STORAGE_PRIMARY_PATH) { @@ -361,6 +372,8 @@ Config::SetConfigCli(const std::string& parent_key, const std::string& child_key status = SetStorageConfigS3SecretKey(value); } else if (child_key == CONFIG_STORAGE_S3_BUCKET) { status = SetStorageConfigS3Bucket(value); + } else { + status = Status(SERVER_UNEXPECTED_ERROR, invalid_node_str); } } else if (parent_key == CONFIG_METRIC) { if (child_key == CONFIG_METRIC_ENABLE_MONITOR) { @@ -369,6 +382,8 @@ Config::SetConfigCli(const std::string& parent_key, const std::string& child_key status = SetMetricConfigAddress(value); } else if (child_key == CONFIG_METRIC_PORT) { status = SetMetricConfigPort(value); + } else { + status = Status(SERVER_UNEXPECTED_ERROR, invalid_node_str); } } else if (parent_key == CONFIG_CACHE) { if (child_key == CONFIG_CACHE_CPU_CACHE_CAPACITY) { @@ -379,6 +394,8 @@ Config::SetConfigCli(const std::string& parent_key, const std::string& child_key status = SetCacheConfigCacheInsertData(value); } else if (child_key == CONFIG_CACHE_INSERT_BUFFER_SIZE) { status = SetCacheConfigInsertBufferSize(value); + } else { + status = Status(SERVER_UNEXPECTED_ERROR, invalid_node_str); } } else if (parent_key == CONFIG_ENGINE) { if (child_key == CONFIG_ENGINE_USE_BLAS_THRESHOLD) { @@ -391,6 +408,8 @@ Config::SetConfigCli(const std::string& parent_key, const std::string& child_key } else if (child_key == CONFIG_ENGINE_GPU_SEARCH_THRESHOLD) { status = SetEngineConfigGpuSearchThreshold(value); #endif + } else { + status = Status(SERVER_UNEXPECTED_ERROR, invalid_node_str); } #ifdef MILVUS_GPU_VERSION } else if (parent_key == CONFIG_GPU_RESOURCE) { @@ -404,6 +423,8 @@ Config::SetConfigCli(const std::string& parent_key, const std::string& child_key status = SetGpuResourceConfigSearchResources(value); } else if (child_key == CONFIG_GPU_RESOURCE_BUILD_INDEX_RESOURCES) { status = SetGpuResourceConfigBuildIndexResources(value); + } else { + status = Status(SERVER_UNEXPECTED_ERROR, invalid_node_str); } #endif } else if (parent_key == CONFIG_TRACING) { @@ -417,6 +438,8 @@ Config::SetConfigCli(const std::string& parent_key, const std::string& child_key status = SetWalConfigBufferSize(value); } else if (child_key == CONFIG_WAL_WAL_PATH) { status = SetWalConfigWalPath(value); + } else { + status = Status(SERVER_UNEXPECTED_ERROR, invalid_node_str); } } @@ -738,6 +761,28 @@ Config::CheckDBConfigBackendUrl(const std::string& value) { return Status::OK(); } +Status +Config::CheckDBConfigPreloadTable(const std::string& value) { + if (value.empty() || value == "*") { + return Status::OK(); + } + + std::vector tables; + StringHelpFunctions::SplitStringByDelimeter(value, ",", tables); + for (auto& table : tables) { + if (!ValidationUtil::ValidateTableName(table).ok()) { + return Status(SERVER_INVALID_ARGUMENT, "Invalid table name: " + table); + } + bool exist = false; + auto status = DBWrapper::DB()->HasNativeTable(table, exist); + if (!(status.ok() && exist)) { + return Status(SERVER_TABLE_NOT_EXIST, "Table " + table + " not exist"); + } + } + + return Status::OK(); +} + Status Config::CheckDBConfigArchiveDiskThreshold(const std::string& value) { auto exist_error = !ValidationUtil::ValidateStringIsNumber(value).ok(); @@ -1772,6 +1817,12 @@ Config::SetDBConfigBackendUrl(const std::string& value) { return SetConfigValueInMem(CONFIG_DB, CONFIG_DB_BACKEND_URL, value); } +Status +Config::SetDBConfigPreloadTable(const std::string& value) { + CONFIG_CHECK(CheckDBConfigPreloadTable(value)); + return SetConfigValueInMem(CONFIG_DB, CONFIG_DB_PRELOAD_TABLE, value); +} + Status Config::SetDBConfigArchiveDiskThreshold(const std::string& value) { CONFIG_CHECK(CheckDBConfigArchiveDiskThreshold(value)); diff --git a/core/src/server/Config.h b/core/src/server/Config.h index 5964f91106..697c9df105 100644 --- a/core/src/server/Config.h +++ b/core/src/server/Config.h @@ -214,6 +214,8 @@ class Config { Status CheckDBConfigBackendUrl(const std::string& value); Status + CheckDBConfigPreloadTable(const std::string& value); + Status CheckDBConfigArchiveDiskThreshold(const std::string& value); Status CheckDBConfigArchiveDaysThreshold(const std::string& value); @@ -422,6 +424,8 @@ class Config { Status SetDBConfigBackendUrl(const std::string& value); Status + SetDBConfigPreloadTable(const std::string& value); + Status SetDBConfigArchiveDiskThreshold(const std::string& value); Status SetDBConfigArchiveDaysThreshold(const std::string& value); diff --git a/core/src/server/web_impl/controller/WebController.hpp b/core/src/server/web_impl/controller/WebController.hpp index 56137dbc35..7b74c86e64 100644 --- a/core/src/server/web_impl/controller/WebController.hpp +++ b/core/src/server/web_impl/controller/WebController.hpp @@ -527,12 +527,12 @@ class WebController : public oatpp::web::server::api::ApiController { auto status_dto = handler.ShowSegments(table_name, query_params, response); switch (status_dto->code->getValue()) { - case StatusCode::SUCCESS:{ + case StatusCode::SUCCESS: return createResponse(Status::CODE_200, response); - } - default:{ + case StatusCode::TABLE_NOT_EXISTS: + return createDtoResponse(Status::CODE_404, status_dto); + default: return createDtoResponse(Status::CODE_400, status_dto); - } } } @@ -551,12 +551,12 @@ class WebController : public oatpp::web::server::api::ApiController { auto status_dto = handler.GetSegmentInfo(table_name, segment_name, info, query_params, response); switch (status_dto->code->getValue()) { - case StatusCode::SUCCESS:{ + case StatusCode::SUCCESS: return createResponse(Status::CODE_200, response); - } - default:{ + case StatusCode::TABLE_NOT_EXISTS: + return createDtoResponse(Status::CODE_404, status_dto); + default: return createDtoResponse(Status::CODE_400, status_dto); - } } } @@ -578,12 +578,12 @@ class WebController : public oatpp::web::server::api::ApiController { auto status_dto = handler.GetVector(table_name, query_params, response); switch (status_dto->code->getValue()) { - case StatusCode::SUCCESS:{ + case StatusCode::SUCCESS: return createResponse(Status::CODE_200, response); - } - default:{ + case StatusCode::TABLE_NOT_EXISTS: + return createDtoResponse(Status::CODE_404, status_dto); + default: return createDtoResponse(Status::CODE_400, status_dto); - } } } diff --git a/core/src/server/web_impl/handler/WebRequestHandler.cpp b/core/src/server/web_impl/handler/WebRequestHandler.cpp index cff9fd4d17..7abf2c0784 100644 --- a/core/src/server/web_impl/handler/WebRequestHandler.cpp +++ b/core/src/server/web_impl/handler/WebRequestHandler.cpp @@ -726,7 +726,7 @@ WebRequestHandler::SetAdvancedConfig(const AdvancedConfigDto::ObjectWrapper& adv #ifdef MILVUS_GPU_VERSION engine_cmd_string = engine_cmd_prefix + std::string(CONFIG_ENGINE_GPU_SEARCH_THRESHOLD) + " " + std::to_string(advanced_config->gpu_search_threshold->getValue()); - CommandLine(engine_cmd_string, reply); + status = CommandLine(engine_cmd_string, reply); if (!status.ok()) { ASSIGN_RETURN_STATUS_DTO(status) } @@ -1434,7 +1434,7 @@ WebRequestHandler::SystemOp(const OString& op, const OString& body_str, OString& status = Compact(j["compact"], result_str); } } else if (op->equals("config")) { - SetConfig(j, result_str); + status = SetConfig(j, result_str); } else { status = Status(UNKNOWN_PATH, "Unknown path: /system/" + op->std_str()); } diff --git a/core/unittest/CMakeLists.txt b/core/unittest/CMakeLists.txt index 6b862a9cf8..95b63523d5 100644 --- a/core/unittest/CMakeLists.txt +++ b/core/unittest/CMakeLists.txt @@ -22,6 +22,7 @@ include_directories(${CMAKE_CURRENT_SOURCE_DIR}) aux_source_directory(${MILVUS_ENGINE_SRC}/cache cache_files) aux_source_directory(${MILVUS_ENGINE_SRC}/config config_files) +aux_source_directory(${MILVUS_ENGINE_SRC}/config/handler config_handler_files) aux_source_directory(${MILVUS_ENGINE_SRC}/metrics metrics_files) aux_source_directory(${MILVUS_ENGINE_SRC}/db db_main_files) aux_source_directory(${MILVUS_ENGINE_SRC}/db/engine db_engine_files) @@ -43,7 +44,6 @@ aux_source_directory(${MILVUS_ENGINE_SRC}/scheduler/job scheduler_job_files) aux_source_directory(${MILVUS_ENGINE_SRC}/scheduler/resource scheduler_resource_files) aux_source_directory(${MILVUS_ENGINE_SRC}/scheduler/task scheduler_task_files) aux_source_directory(${MILVUS_ENGINE_SRC}/scheduler/optimizer scheduler_optimizer_files) -aux_source_directory(${MILVUS_ENGINE_SRC}/scheduler/optimizer/handler scheduler_optimizer_handler_files) set(scheduler_files ${scheduler_main_files} ${scheduler_action_files} @@ -52,7 +52,6 @@ set(scheduler_files ${scheduler_resource_files} ${scheduler_task_files} ${scheduler_optimizer_files} - ${scheduler_optimizer_handler_files} ) aux_source_directory(${MILVUS_THIRDPARTY_SRC}/easyloggingpp thirdparty_easyloggingpp_files) @@ -130,6 +129,7 @@ set(helper_files set(common_files ${cache_files} ${config_files} + ${config_handler_files} ${db_main_files} ${db_engine_files} ${db_insert_files} diff --git a/core/unittest/server/test_web.cpp b/core/unittest/server/test_web.cpp index 79c376b365..cfafc84490 100644 --- a/core/unittest/server/test_web.cpp +++ b/core/unittest/server/test_web.cpp @@ -1296,6 +1296,10 @@ TEST_F(WebControllerTest, GET_SEGMENT_INFO) { auto vecs_json = vecs_result_json["vectors"]; ASSERT_TRUE(vecs_json.is_array()); ASSERT_EQ(10, vecs_json.size()); + + // non-existent table + response = client_ptr->getSegmentInfo(table_name + "_non_existent", segment_name.c_str(), "ids", "0", "10"); + ASSERT_EQ(OStatus::CODE_404.code, response->getStatusCode()) << response->readBodyToString()->c_str(); } TEST_F(WebControllerTest, SEGMENT_FILTER) { @@ -1536,6 +1540,10 @@ TEST_F(WebControllerTest, GET_VECTOR_BY_ID) { } ASSERT_EQ(64, vec.size()); + + // non-existent table + response = client_ptr->getVectors(table_name + "_non_existent", id_str.c_str(), conncetion_ptr); + ASSERT_EQ(OStatus::CODE_404.code, response->getStatusCode()) << response->readBodyToString()->c_str(); } TEST_F(WebControllerTest, DELETE_BY_ID) { @@ -1565,6 +1573,10 @@ TEST_F(WebControllerTest, DELETE_BY_ID) { response = client_ptr->vectorsOp(table_name, delete_json.dump().c_str(), conncetion_ptr); ASSERT_EQ(OStatus::CODE_200.code, response->getStatusCode()) << response->readBodyToString()->c_str(); + + // non-existent table + response = client_ptr->vectorsOp(table_name + "_non_existent", delete_json.dump().c_str(), conncetion_ptr); + ASSERT_EQ(OStatus::CODE_404.code, response->getStatusCode()) << response->readBodyToString()->c_str(); } TEST_F(WebControllerTest, CMD) { @@ -1609,6 +1621,31 @@ TEST_F(WebControllerTest, CONFIG) { auto response = client_ptr->cmd("config", "", "", conncetion_ptr); ASSERT_EQ(OStatus::CODE_200.code, response->getStatusCode()) << response->readBodyToString()->c_str(); + auto result_json = nlohmann::json::parse(response->readBodyToString()->c_str()); + ASSERT_TRUE(result_json.contains("restart_required")); + + OString table_name = "milvus_test_webcontroller_test_preload_table"; + GenTable(table_name, 16, 10, "L2"); + + OString table_name_s = "milvus_test_webcontroller_test_preload_table"; + GenTable(table_name_s, 16, 10, "L2"); + + OString body_str = "{\"db_config\": {\"preload_table\": \"" + table_name + "\"}}"; + response = client_ptr->op("config", body_str, conncetion_ptr); + ASSERT_EQ(OStatus::CODE_200.code, response->getStatusCode()) << response->readBodyToString()->c_str(); + + body_str = "{\"db_config\": {\"preload_table\": \"" + table_name + "," + table_name_s + "\"}}"; + response = client_ptr->op("config", body_str, conncetion_ptr); + ASSERT_EQ(OStatus::CODE_200.code, response->getStatusCode()) << response->readBodyToString()->c_str(); + auto set_result_json = nlohmann::json::parse(response->readBodyToString()->c_str()); + ASSERT_TRUE(set_result_json.contains("restart_required")); + ASSERT_EQ(true, set_result_json["restart_required"].get()); + + response = client_ptr->cmd("config", "", "", conncetion_ptr); + ASSERT_EQ(OStatus::CODE_200.code, response->getStatusCode()) << response->readBodyToString()->c_str(); + auto get_result_json = nlohmann::json::parse(response->readBodyToString()->c_str()); + ASSERT_TRUE(get_result_json.contains("restart_required")); + ASSERT_EQ(true, get_result_json["restart_required"].get()); } TEST_F(WebControllerTest, ADVANCED_CONFIG) {