From 2aad94b678a9b60642b20f4932e40b2414d15332 Mon Sep 17 00:00:00 2001 From: "yudong.cai" Date: Thu, 28 Nov 2019 16:12:39 +0800 Subject: [PATCH 1/4] #560 add version in server config file --- CHANGELOG.md | 1 + core/conf/server_cpu_config.template | 2 ++ core/conf/server_gpu_config.template | 2 ++ core/src/server/Config.cpp | 35 +++++++++++++++++++++++++--- core/src/server/Config.h | 9 +++++++ core/unittest/server/utils.cpp | 2 ++ 6 files changed, 48 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 808a019204..2b6be1a6d2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -44,6 +44,7 @@ Please mark all change in change log and use the ticket from JIRA. - \#420 - Update shards merge part to match v0.5.3 - \#488 - Add log in scheduler/optimizer - \#502 - C++ SDK support IVFPQ and SPTAG +- \#560 - Add version in server config file ## Improvement - \#255 - Add ivfsq8 test report detailed version diff --git a/core/conf/server_cpu_config.template b/core/conf/server_cpu_config.template index 41889f5cef..40b8107d39 100644 --- a/core/conf/server_cpu_config.template +++ b/core/conf/server_cpu_config.template @@ -1,5 +1,7 @@ # Default values are used when you make no changes to the following parameters. +version: 0.1 # config version + server_config: address: 0.0.0.0 # milvus server ip address (IPv4) port: 19530 # milvus server port, must in range [1025, 65534] diff --git a/core/conf/server_gpu_config.template b/core/conf/server_gpu_config.template index 531c633da7..678915ca9d 100644 --- a/core/conf/server_gpu_config.template +++ b/core/conf/server_gpu_config.template @@ -1,5 +1,7 @@ # Default values are used when you make no changes to the following parameters. +version: 0.1 # config version + server_config: address: 0.0.0.0 # milvus server ip address (IPv4) port: 19530 # milvus server port, must in range [1025, 65534] diff --git a/core/src/server/Config.cpp b/core/src/server/Config.cpp index 95bab84391..cd9b264fcd 100644 --- a/core/src/server/Config.cpp +++ b/core/src/server/Config.cpp @@ -20,6 +20,7 @@ #include #include #include +#include #include #include "config/YamlConfigMgr.h" @@ -33,6 +34,8 @@ namespace server { constexpr uint64_t GB = 1UL << 30; +static const std::unordered_map milvus_config_version_map({{"0.6.0", "0.1"}}); + Config& Config::GetInstance() { static Config config_inst; @@ -69,6 +72,12 @@ Status Config::ValidateConfig() { Status s; + std::string config_version; + s = GetConfigVersion(config_version); + if (!s.ok()) { + return s; + } + /* server config */ std::string server_addr; s = GetServerConfigAddress(server_addr); @@ -383,6 +392,16 @@ Config::PrintAll() { } //////////////////////////////////////////////////////////////////////////////// +Status +Config::CheckConfigVersion(const std::string& value) { + if (milvus_config_version_map.at(MILVUS_VERSION) != value) { + std::string msg = "Invalid config version: " + value + + ". Expected config version: " + milvus_config_version_map.at(MILVUS_VERSION); + return Status(SERVER_INVALID_ARGUMENT, msg); + } + return Status::OK(); +} + Status Config::CheckServerConfigAddress(const std::string& value) { if (!ValidationUtil::ValidateIpAddress(value).ok()) { @@ -766,10 +785,14 @@ Config::CheckGpuResourceConfigBuildIndexResources(const std::vector //////////////////////////////////////////////////////////////////////////////// ConfigNode& -Config::GetConfigNode(const std::string& name) { +Config::GetConfigRoot() { ConfigMgr* mgr = YamlConfigMgr::GetInstance(); - ConfigNode& root_node = mgr->GetRootNode(); - return root_node.GetChild(name); + return mgr->GetRootNode(); +} + +ConfigNode& +Config::GetConfigNode(const std::string& name) { + return GetConfigRoot().GetChild(name); } Status @@ -816,6 +839,12 @@ Config::GetConfigSequenceStr(const std::string& parent_key, const std::string& c return value; } +Status +Config::GetConfigVersion(std::string& value) { + value = GetConfigRoot().GetValue(CONFIG_VERSION); + return CheckConfigVersion(value); +} + Status Config::GetServerConfigAddress(std::string& value) { value = GetConfigStr(CONFIG_SERVER, CONFIG_SERVER_ADDRESS, CONFIG_SERVER_ADDRESS_DEFAULT); diff --git a/core/src/server/Config.h b/core/src/server/Config.h index 281a832d57..4361cd1e37 100644 --- a/core/src/server/Config.h +++ b/core/src/server/Config.h @@ -28,6 +28,8 @@ namespace milvus { namespace server { +static const char* CONFIG_VERSION = "version"; + /* server config */ static const char* CONFIG_SERVER = "server_config"; static const char* CONFIG_SERVER_ADDRESS = "address"; @@ -115,6 +117,8 @@ class Config { PrintAll(); private: + ConfigNode& + GetConfigRoot(); ConfigNode& GetConfigNode(const std::string& name); Status @@ -125,6 +129,9 @@ class Config { PrintConfigSection(const std::string& config_node_name); /////////////////////////////////////////////////////////////////////////// + Status + CheckConfigVersion(const std::string& value); + /* server config */ Status CheckServerConfigAddress(const std::string& value); @@ -193,6 +200,8 @@ class Config { std::string GetConfigSequenceStr(const std::string& parent_key, const std::string& child_key, const std::string& delim = ",", const std::string& default_value = ""); + Status + GetConfigVersion(std::string& value); public: /* server config */ diff --git a/core/unittest/server/utils.cpp b/core/unittest/server/utils.cpp index 0545d34575..6fb424356d 100644 --- a/core/unittest/server/utils.cpp +++ b/core/unittest/server/utils.cpp @@ -28,6 +28,8 @@ namespace { static const char* VALID_CONFIG_STR = "# Default values are used when you make no changes to the following parameters.\n" "\n" + "version: 0.1" + "\n" "server_config:\n" " address: 0.0.0.0 # milvus server ip address (IPv4)\n" " port: 19530 # port range: 1025 ~ 65534\n" From 09d6165290b6266b578e591008726b2ded5db8b4 Mon Sep 17 00:00:00 2001 From: Yukikaze-CZR Date: Thu, 28 Nov 2019 19:37:36 +0800 Subject: [PATCH 2/4] Increase code coverage rate #485 --- core/src/db/engine/ExecutionEngineImpl.cpp | 2 ++ core/unittest/db/test_engine.cpp | 28 +++++++++++++++++++++- 2 files changed, 29 insertions(+), 1 deletion(-) diff --git a/core/src/db/engine/ExecutionEngineImpl.cpp b/core/src/db/engine/ExecutionEngineImpl.cpp index 397ca89d68..5a3d6e5e2a 100644 --- a/core/src/db/engine/ExecutionEngineImpl.cpp +++ b/core/src/db/engine/ExecutionEngineImpl.cpp @@ -112,10 +112,12 @@ ExecutionEngineImpl::CreatetVecIndex(EngineType type) { index = GetVecIndexFactory(IndexType::NSG_MIX); break; } +#ifdef CUSTOMIZATION case EngineType::FAISS_IVFSQ8H: { index = GetVecIndexFactory(IndexType::FAISS_IVFSQ8_HYBRID); break; } +#endif case EngineType::FAISS_PQ: { #ifdef MILVUS_CPU_VERSION index = GetVecIndexFactory(IndexType::FAISS_IVFPQ_CPU); diff --git a/core/unittest/db/test_engine.cpp b/core/unittest/db/test_engine.cpp index 67070df41b..e0edd05537 100644 --- a/core/unittest/db/test_engine.cpp +++ b/core/unittest/db/test_engine.cpp @@ -59,6 +59,29 @@ TEST_F(EngineTest, FACTORY_TEST) { ASSERT_TRUE(engine_ptr != nullptr); } + + { + auto engine_ptr = milvus::engine::EngineFactory::Build( + 512, "/tmp/milvus_index_1", milvus::engine::EngineType::FAISS_PQ, milvus::engine::MetricType::IP, 1024); + + ASSERT_TRUE(engine_ptr != nullptr); + } + + { + auto engine_ptr = milvus::engine::EngineFactory::Build( + 512, "/tmp/milvus_index_1", milvus::engine::EngineType::SPTAG_KDT, + milvus::engine::MetricType::L2, 1024); + + ASSERT_TRUE(engine_ptr != nullptr); + } + + { + auto engine_ptr = milvus::engine::EngineFactory::Build( + 512, "/tmp/milvus_index_1", milvus::engine::EngineType::SPTAG_KDT, + milvus::engine::MetricType::L2, 1024); + + ASSERT_TRUE(engine_ptr != nullptr); + } } TEST_F(EngineTest, ENGINE_IMPL_TEST) { @@ -69,7 +92,7 @@ TEST_F(EngineTest, ENGINE_IMPL_TEST) { std::vector data; std::vector ids; - const int row_count = 10000; + const int row_count = 500; data.reserve(row_count * dimension); ids.reserve(row_count); for (int64_t i = 0; i < row_count; i++) { @@ -95,5 +118,8 @@ TEST_F(EngineTest, ENGINE_IMPL_TEST) { // ASSERT_TRUE(status.ok()); auto engine_build = engine_ptr->BuildIndex("/tmp/milvus_index_2", milvus::engine::EngineType::FAISS_IVFSQ8); + engine_build = engine_ptr->BuildIndex("/tmp/milvus_index_3", milvus::engine::EngineType::FAISS_PQ); + engine_build = engine_ptr->BuildIndex("/tmp/milvus_index_4", milvus::engine::EngineType::SPTAG_KDT); + engine_build = engine_ptr->BuildIndex("/tmp/milvus_index_5", milvus::engine::EngineType::SPTAG_BKT); // ASSERT_TRUE(status.ok()); } From 3a8a83369bb3bde526781e61b68d39ddde312b7a Mon Sep 17 00:00:00 2001 From: Yukikaze-CZR Date: Thu, 28 Nov 2019 19:39:43 +0800 Subject: [PATCH 3/4] add changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index fab5bf3bd0..2e2c1ddea4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,6 +21,7 @@ Please mark all change in change log and use the ticket from JIRA. - \#440 - Server cannot startup with gpu_resource_config.enable=false in GPU version - \#458 - Index data is not compatible between 0.5 and 0.6 - \#465 - Server hang caused by searching with nsg index +- \#485 - Increase code coverage rate - \#486 - gpu no usage during index building - \#497 - CPU-version search performance decreased - \#504 - The code coverage rate of core/src/scheduler/optimizer is too low From 05bfaa191b01b853bad0f970c5a19c4fa3aea90d Mon Sep 17 00:00:00 2001 From: Yukikaze-CZR Date: Thu, 28 Nov 2019 19:46:47 +0800 Subject: [PATCH 4/4] clang-format --- core/src/db/meta/MySQLMetaImpl.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/core/src/db/meta/MySQLMetaImpl.cpp b/core/src/db/meta/MySQLMetaImpl.cpp index 7b53e6361a..dcf3824fe1 100644 --- a/core/src/db/meta/MySQLMetaImpl.cpp +++ b/core/src/db/meta/MySQLMetaImpl.cpp @@ -1639,7 +1639,8 @@ MySQLMetaImpl::FilesByType(const std::string& table_id, const std::vector& case (int)TableFileSchema::BACKUP: msg = msg + " backup files:" + std::to_string(backup_count); break; - default:break; + default: + break; } } ENGINE_LOG_DEBUG << msg;