Merge branch 'branch-0.3.0' into 'branch-0.3.1'

MS - 144 Add engine conf

See merge request megasearch/milvus!150

Former-commit-id: a035448700a96fc317f4ba43df89a521bfdf23c9

cpp/CHANGELOG.md

@@ -61,6 +61,7 @@ Please mark all change in change log and use the ticket from JIRA.
 - MS-97 - Add S3 SDK for MinIO Storage
 - MS-105 - Add MySQL
 - MS-130 - Add prometheus_test
+- MS-144 - Add nprobe config
 
 ## Task
 - MS-74 - Change README.md in cpp

cpp/src/server/ServerConfig.h

@@ -43,6 +43,9 @@ static const std::string CONFIG_METRIC_COLLECTOR = "collector";
 static const std::string CONFIG_PROMETHEUS = "prometheus_config";
 static const std::string CONFIG_METRIC_PROMETHEUS_PORT = "port";
 
+static const std::string CONFIG_ENGINE = "engine_config";
+static const std::string CONFIG_NPROBE = "nprobe";
+
 class ServerConfig {
  public:
     static ServerConfig &GetInstance();

cpp/src/wrapper/Index.cpp

@@ -14,6 +14,8 @@
 #include "Index.h"
 #include "faiss/index_io.h"
 #include "faiss/IndexIVF.h"
+#include "faiss/IVFlib.h"
+#include "server/ServerConfig.h"
 
 namespace zilliz {
 namespace milvus {
@@ -23,6 +25,32 @@ using std::string;
 using std::unordered_map;
 using std::vector;
 
+class Nprobe {
+ public:
+    static Nprobe &GetInstance() {
+        static Nprobe instance;
+        return instance;
+    }
+
+    void SelectNprobe() {
+        using namespace zilliz::milvus::server;
+        ServerConfig &config = ServerConfig::GetInstance();
+        ConfigNode engine_config = config.GetConfig(CONFIG_ENGINE);
+        nprobe_ = engine_config.GetInt32Value(CONFIG_NPROBE, 1000);
+    }
+
+    size_t GetNprobe() {
+        return nprobe_;
+    }
+
+ private:
+    Nprobe() : nprobe_(1000) { SelectNprobe(); }
+
+ private:
+    size_t nprobe_;
+};
+
+
 Index::Index(const std::shared_ptr<faiss::Index> &raw_index) {
     index_ = raw_index;
     dim = index_->d;
@@ -57,7 +85,7 @@ bool Index::add_with_ids(idx_t n, const float *xdata, const long *xids) {
 bool Index::search(idx_t n, const float *data, idx_t k, float *distances, long *labels) const {
     try {
         if(auto ivf_index = std::dynamic_pointer_cast<faiss::IndexIVF>(index_)) {
-            ivf_index->nprobe = 100;
+            ivf_index->nprobe = Nprobe::GetInstance().GetNprobe();
         }
         index_->search(n, data, k, distances, labels);
     }

cpp/src/wrapper/Operand.cpp

@@ -39,7 +39,7 @@ string Operand::get_index_type(const int &nb) {
         }
         case IVF: {
             index_str += (ncent != 0 ? index_type + std::to_string(ncent) :
-                          index_type + std::to_string(int(nb / 1000000.0 * 1638)));
+                          index_type + std::to_string(int(nb / 1000000.0 * 16384)));
             break;
         }
         case IDMAP: {

cpp/unittest/server/appendix/server_config.yaml

@@ -29,3 +29,6 @@ license_config:         # license configure
 
 cache_config:           # cache configure
   cpu_cache_capacity: 16   # how many memory are used as cache, unit: GB, range: 0 ~ less than total memory
+
+engine_config:
+  nprobe: 3000