Add ivf index nprobe logging (#3070)

* add nprobe logging

Signed-off-by: sahuang <xiaohai.xu@zilliz.com>

* add all cases

Signed-off-by: sahuang <xiaohai.xu@zilliz.com>

core/src/db/DBImpl.cpp

@@ -35,6 +35,7 @@
 #include "db/merge/MergeManagerFactory.h"
 #include "engine/EngineFactory.h"
 #include "index/knowhere/knowhere/index/vector_index/helpers/BuilderSuspend.h"
+#include "index/knowhere/knowhere/index/vector_index/helpers/FaissIO.h"
 #include "index/thirdparty/faiss/utils/distances.h"
 #include "insert/MemManagerFactory.h"
 #include "meta/MetaConsts.h"
@@ -95,6 +96,7 @@ DBImpl::DBImpl(const DBOptions& options)
     SetIdentity("DBImpl");
     AddCacheInsertDataListener();
     AddUseBlasThresholdListener();
+    knowhere::enable_faiss_logging();
 
     Start();
 }

core/src/index/knowhere/knowhere/common/Log.cpp

@@ -51,5 +51,35 @@ GetThreadName() {
     return thread_name;
 }
 
+void
+log_trace_(const std::string& s) {
+    LOG_KNOWHERE_TRACE_ << s;
+}
+
+void
+log_debug_(const std::string& s) {
+    LOG_KNOWHERE_DEBUG_ << s;
+}
+
+void
+log_info_(const std::string& s) {
+    LOG_KNOWHERE_INFO_ << s;
+}
+
+void
+log_warning_(const std::string& s) {
+    LOG_KNOWHERE_WARNING_ << s;
+}
+
+void
+log_error_(const std::string& s) {
+    LOG_KNOWHERE_ERROR_ << s;
+}
+
+void
+log_fatal_(const std::string& s) {
+    LOG_KNOWHERE_FATAL_ << s;
+}
+
 }  // namespace knowhere
 }  // namespace milvus

core/src/index/knowhere/knowhere/common/Log.h

@@ -27,6 +27,24 @@ SetThreadName(const std::string& name);
 std::string
 GetThreadName();
 
+void
+log_trace_(const std::string&);
+
+void
+log_debug_(const std::string&);
+
+void
+log_info_(const std::string&);
+
+void
+log_warning_(const std::string&);
+
+void
+log_error_(const std::string&);
+
+void
+log_fatal_(const std::string&);
+
 /*
  * Please use LOG_MODULE_LEVEL_C macro in member function of class
  * and LOG_MODULE_LEVEL_ macro in other functions.

core/src/index/knowhere/knowhere/index/vector_index/IndexIVF.cpp

@@ -35,6 +35,7 @@
 #include "knowhere/common/Log.h"
 #include "knowhere/index/vector_index/IndexIVF.h"
 #include "knowhere/index/vector_index/adapter/VectorAdapter.h"
+#include "knowhere/index/vector_index/helpers/FaissIO.h"
 #include "knowhere/index/vector_index/helpers/IndexParameter.h"
 #ifdef MILVUS_GPU_VERSION
 #include "knowhere/index/vector_index/gpu/IndexGPUIVF.h"

core/src/index/knowhere/knowhere/index/vector_index/helpers/FaissIO.cpp

@@ -11,6 +11,7 @@
 
 #include <cstring>
 
+#include "knowhere/common/Log.h"
 #include "knowhere/index/vector_index/helpers/FaissIO.h"
 
 namespace milvus {
@@ -60,5 +61,10 @@ MemoryIOReader::operator()(void* ptr, size_t size, size_t nitems) {
     return nitems;
 }
 
+void
+enable_faiss_logging() {
+    faiss::LOG_DEBUG_ = &log_debug_;
+}
+
 }  // namespace knowhere
 }  // namespace milvus

core/src/index/knowhere/knowhere/index/vector_index/helpers/FaissIO.h

@@ -12,6 +12,7 @@
 #pragma once
 
 #include <faiss/impl/io.h>
+#include <faiss/utils/utils.h>
 
 namespace milvus {
 namespace knowhere {
@@ -46,5 +47,8 @@ struct MemoryIOReader : public faiss::IOReader {
     }
 };
 
+void
+enable_faiss_logging();
+
 }  // namespace knowhere
 }  // namespace milvus

core/src/index/thirdparty/faiss/IndexIVF.cpp

@@ -15,6 +15,7 @@
 #include <cstdio>
 #include <memory>
 #include <iostream>
+#include <sstream>
 
 #include <faiss/utils/utils.h>
 #include <faiss/utils/hamming.h>
@@ -314,6 +315,22 @@ void IndexIVF::search (idx_t n, const float *x, idx_t k,
     search_preassigned (n, x, k, idx.get(), coarse_dis.get(),
                         distances, labels, false, nullptr, bitset);
     indexIVF_stats.search_time += getmillisecs() - t0;
+
+    // string
+    if (LOG_DEBUG_) {
+        auto ids = idx.get();
+        for (size_t i = 0; i < n; i++) {
+            std::stringstream ss;
+            ss << "Query #" << i << ", nprobe list: ";
+            for (size_t j = 0; j < nprobe; j++) {
+                if (j != 0) {
+                    ss << ",";
+                }
+                ss << ids[i * nprobe + j];
+            }
+            (*LOG_DEBUG_)(ss.str());
+        }
+    }
 }
 
 #if 0

core/src/index/thirdparty/faiss/utils/utils.cpp

@@ -708,4 +708,16 @@ int64_t get_L3_Size() {
     return l3_size;
 }
 
+void (*LOG_TRACE_)(const std::string&);
+
+void (*LOG_DEBUG_)(const std::string&);
+
+void (*LOG_INFO_)(const std::string&);
+
+void (*LOG_WARNING_)(const std::string&);
+
+void (*LOG_FATAL_)(const std::string&);
+
+void (*LOG_ERROR_)(const std::string&);
+
 } // namespace faiss

core/src/index/thirdparty/faiss/utils/utils.h

@@ -17,6 +17,7 @@
 #define FAISS_utils_h
 
 #include <stdint.h>
+#include <string>
 
 #include <faiss/utils/Heap.h>
 
@@ -163,6 +164,18 @@ bool check_openmp();
 /** get the size of L3 cache  */
 int64_t get_L3_Size();
 
+extern void (*LOG_TRACE_)(const std::string&);
+
+extern void (*LOG_DEBUG_)(const std::string&);
+
+extern void (*LOG_INFO_)(const std::string&);
+
+extern void (*LOG_WARNING_)(const std::string&);
+
+extern void (*LOG_FATAL_)(const std::string&);
+
+extern void (*LOG_ERROR_)(const std::string&);
+
 } // namspace faiss