Merge remote-tracking branch 'upstream/0.6.0' into 0.6.0-yk-refactor-scheduler

CHANGELOG.md

@@ -21,6 +21,7 @@ Please mark all change in change log and use the ticket from JIRA.
 - \#440 - Server cannot startup with gpu_resource_config.enable=false in GPU version
 - \#458 - Index data is not compatible between 0.5 and 0.6
 - \#465 - Server hang caused by searching with nsg index
+- \#485 - Increase code coverage rate
 - \#486 - gpu no usage during index building
 - \#497 - CPU-version search performance decreased
 - \#504 - The code coverage rate of core/src/scheduler/optimizer is too low
@@ -49,6 +50,7 @@ Please mark all change in change log and use the ticket from JIRA.
 - \#420 - Update shards merge part to match v0.5.3
 - \#488 - Add log in scheduler/optimizer
 - \#502 - C++ SDK support IVFPQ and SPTAG
+- \#560 - Add version in server config file
 
 ## Improvement
 - \#255 - Add ivfsq8 test report detailed version

core/conf/server_cpu_config.template

@@ -1,5 +1,7 @@
 # Default values are used when you make no changes to the following parameters.
 
+version: 0.1                        # config version
+
 server_config:
   address: 0.0.0.0                  # milvus server ip address (IPv4)
   port: 19530                       # milvus server port, must in range [1025, 65534]

core/conf/server_gpu_config.template

@@ -1,5 +1,7 @@
 # Default values are used when you make no changes to the following parameters.
 
+version: 0.1                        # config version
+
 server_config:
   address: 0.0.0.0                  # milvus server ip address (IPv4)
   port: 19530                       # milvus server port, must in range [1025, 65534]

core/src/db/engine/ExecutionEngineImpl.cpp

@@ -112,10 +112,12 @@ ExecutionEngineImpl::CreatetVecIndex(EngineType type) {
             index = GetVecIndexFactory(IndexType::NSG_MIX);
             break;
         }
+#ifdef CUSTOMIZATION
         case EngineType::FAISS_IVFSQ8H: {
             index = GetVecIndexFactory(IndexType::FAISS_IVFSQ8_HYBRID);
             break;
         }
+#endif
         case EngineType::FAISS_PQ: {
 #ifdef MILVUS_CPU_VERSION
             index = GetVecIndexFactory(IndexType::FAISS_IVFPQ_CPU);

core/src/server/Config.cpp

@@ -20,6 +20,7 @@
 #include <iostream>
 #include <regex>
 #include <string>
+#include <unordered_map>
 #include <vector>
 
 #include "config/YamlConfigMgr.h"
@@ -33,6 +34,8 @@ namespace server {
 
 constexpr uint64_t GB = 1UL << 30;
 
+static const std::unordered_map<std::string, std::string> milvus_config_version_map({{"0.6.0", "0.1"}});
+
 Config&
 Config::GetInstance() {
     static Config config_inst;
@@ -69,6 +72,12 @@ Status
 Config::ValidateConfig() {
     Status s;
 
+    std::string config_version;
+    s = GetConfigVersion(config_version);
+    if (!s.ok()) {
+        return s;
+    }
+
     /* server config */
     std::string server_addr;
     s = GetServerConfigAddress(server_addr);
@@ -383,6 +392,16 @@ Config::PrintAll() {
 }
 
 ////////////////////////////////////////////////////////////////////////////////
+Status
+Config::CheckConfigVersion(const std::string& value) {
+    if (milvus_config_version_map.at(MILVUS_VERSION) != value) {
+        std::string msg = "Invalid config version: " + value +
+                          ". Expected config version: " + milvus_config_version_map.at(MILVUS_VERSION);
+        return Status(SERVER_INVALID_ARGUMENT, msg);
+    }
+    return Status::OK();
+}
+
 Status
 Config::CheckServerConfigAddress(const std::string& value) {
     if (!ValidationUtil::ValidateIpAddress(value).ok()) {
@@ -766,10 +785,14 @@ Config::CheckGpuResourceConfigBuildIndexResources(const std::vector<std::string>
 
 ////////////////////////////////////////////////////////////////////////////////
 ConfigNode&
-Config::GetConfigNode(const std::string& name) {
+Config::GetConfigRoot() {
     ConfigMgr* mgr = YamlConfigMgr::GetInstance();
-    ConfigNode& root_node = mgr->GetRootNode();
-    return root_node.GetChild(name);
+    return mgr->GetRootNode();
+}
+
+ConfigNode&
+Config::GetConfigNode(const std::string& name) {
+    return GetConfigRoot().GetChild(name);
 }
 
 Status
@@ -816,6 +839,12 @@ Config::GetConfigSequenceStr(const std::string& parent_key, const std::string& c
     return value;
 }
 
+Status
+Config::GetConfigVersion(std::string& value) {
+    value = GetConfigRoot().GetValue(CONFIG_VERSION);
+    return CheckConfigVersion(value);
+}
+
 Status
 Config::GetServerConfigAddress(std::string& value) {
     value = GetConfigStr(CONFIG_SERVER, CONFIG_SERVER_ADDRESS, CONFIG_SERVER_ADDRESS_DEFAULT);

core/src/server/Config.h

@@ -28,6 +28,8 @@
 namespace milvus {
 namespace server {
 
+static const char* CONFIG_VERSION = "version";
+
 /* server config */
 static const char* CONFIG_SERVER = "server_config";
 static const char* CONFIG_SERVER_ADDRESS = "address";
@@ -115,6 +117,8 @@ class Config {
     PrintAll();
 
  private:
+    ConfigNode&
+    GetConfigRoot();
     ConfigNode&
     GetConfigNode(const std::string& name);
     Status
@@ -125,6 +129,9 @@ class Config {
     PrintConfigSection(const std::string& config_node_name);
 
     ///////////////////////////////////////////////////////////////////////////
+    Status
+    CheckConfigVersion(const std::string& value);
+
     /* server config */
     Status
     CheckServerConfigAddress(const std::string& value);
@@ -193,6 +200,8 @@ class Config {
     std::string
     GetConfigSequenceStr(const std::string& parent_key, const std::string& child_key, const std::string& delim = ",",
                          const std::string& default_value = "");
+    Status
+    GetConfigVersion(std::string& value);
 
  public:
     /* server config */

core/unittest/db/test_engine.cpp

@@ -59,6 +59,29 @@ TEST_F(EngineTest, FACTORY_TEST) {
 
         ASSERT_TRUE(engine_ptr != nullptr);
     }
+
+    {
+        auto engine_ptr = milvus::engine::EngineFactory::Build(
+                512, "/tmp/milvus_index_1", milvus::engine::EngineType::FAISS_PQ, milvus::engine::MetricType::IP, 1024);
+
+        ASSERT_TRUE(engine_ptr != nullptr);
+    }
+
+    {
+        auto engine_ptr = milvus::engine::EngineFactory::Build(
+                512, "/tmp/milvus_index_1", milvus::engine::EngineType::SPTAG_KDT,
+                milvus::engine::MetricType::L2, 1024);
+
+        ASSERT_TRUE(engine_ptr != nullptr);
+    }
+
+    {
+        auto engine_ptr = milvus::engine::EngineFactory::Build(
+                512, "/tmp/milvus_index_1", milvus::engine::EngineType::SPTAG_KDT,
+                milvus::engine::MetricType::L2, 1024);
+
+        ASSERT_TRUE(engine_ptr != nullptr);
+    }
 }
 
 TEST_F(EngineTest, ENGINE_IMPL_TEST) {
@@ -69,7 +92,7 @@ TEST_F(EngineTest, ENGINE_IMPL_TEST) {
 
     std::vector<float> data;
     std::vector<int64_t> ids;
-    const int row_count = 10000;
+    const int row_count = 500;
     data.reserve(row_count * dimension);
     ids.reserve(row_count);
     for (int64_t i = 0; i < row_count; i++) {
@@ -95,5 +118,8 @@ TEST_F(EngineTest, ENGINE_IMPL_TEST) {
     // ASSERT_TRUE(status.ok());
 
     auto engine_build = engine_ptr->BuildIndex("/tmp/milvus_index_2", milvus::engine::EngineType::FAISS_IVFSQ8);
+    engine_build = engine_ptr->BuildIndex("/tmp/milvus_index_3", milvus::engine::EngineType::FAISS_PQ);
+    engine_build = engine_ptr->BuildIndex("/tmp/milvus_index_4", milvus::engine::EngineType::SPTAG_KDT);
+    engine_build = engine_ptr->BuildIndex("/tmp/milvus_index_5", milvus::engine::EngineType::SPTAG_BKT);
     // ASSERT_TRUE(status.ok());
 }

core/unittest/server/utils.cpp

@@ -28,6 +28,8 @@ namespace {
 static const char* VALID_CONFIG_STR =
     "# Default values are used when you make no changes to the following parameters.\n"
     "\n"
+    "version: 0.1"
+    "\n"
     "server_config:\n"
     "  address: 0.0.0.0                  # milvus server ip address (IPv4)\n"
     "  port: 19530                       # port range: 1025 ~ 65534\n"