Limit faiss ivf index build thread num and fix ut (#27567)

Signed-off-by: chasingegg <chao.gao@zilliz.com>

internal/core/thirdparty/knowhere/CMakeLists.txt

@@ -12,7 +12,7 @@
 #-------------------------------------------------------------------------------
 
 # Update KNOWHERE_VERSION for the first occurrence
-set( KNOWHERE_VERSION 6225493 )
+set( KNOWHERE_VERSION 7144b89 )
 set( GIT_REPOSITORY  "https://github.com/zilliztech/knowhere.git")
 if ( INDEX_ENGINE STREQUAL "cardinal" )
     set( KNOWHERE_VERSION main )

internal/core/unittest/test_indexing.cpp

@@ -415,6 +415,7 @@ TEST_P(IndexTest, BuildAndQuery) {
     for (auto& binary : binary_set.binary_map_) {
         index_files.emplace_back(binary.first);
     }
+    load_conf = generate_load_conf(index_type, metric_type, 0);
     load_conf["index_files"] = index_files;
     ASSERT_NO_THROW(vec_index->Load(load_conf));
     EXPECT_EQ(vec_index->Count(), NB);
@@ -471,6 +472,7 @@ TEST_P(IndexTest, Mmap) {
     for (auto& binary : binary_set.binary_map_) {
         index_files.emplace_back(binary.first);
     }
+    load_conf = generate_load_conf(index_type, metric_type, 0);
     load_conf["index_files"] = index_files;
     load_conf["mmap_filepath"] = "mmap/test_index_mmap_" + index_type;
     vec_index->Load(load_conf);
@@ -514,25 +516,23 @@ TEST_P(IndexTest, GetVector) {
     milvus::index::IndexBasePtr new_index;
     milvus::index::VectorIndex* vec_index = nullptr;
 
+    auto binary_set = index->Upload();
+    index.reset();
+    std::vector<std::string> index_files;
+    for (auto& binary : binary_set.binary_map_) {
+        index_files.emplace_back(binary.first);
+    }
+    new_index = milvus::index::IndexFactory::GetInstance().CreateIndex(
+        create_index_info, file_manager_context);
+    load_conf = generate_load_conf(index_type, metric_type, 0);
+    load_conf["index_files"] = index_files;
+
+    vec_index = dynamic_cast<milvus::index::VectorIndex*>(new_index.get());
     if (index_type == knowhere::IndexEnum::INDEX_DISKANN) {
-        // TODO ::diskann.query need load first, ugly
-        auto binary_set = index->Serialize(milvus::Config{});
-        index.reset();
-
-        new_index = milvus::index::IndexFactory::GetInstance().CreateIndex(
-            create_index_info, file_manager_context);
-
-        vec_index = dynamic_cast<milvus::index::VectorIndex*>(new_index.get());
-
-        std::vector<std::string> index_files;
-        for (auto& binary : binary_set.binary_map_) {
-            index_files.emplace_back(binary.first);
-        }
-        load_conf["index_files"] = index_files;
         vec_index->Load(binary_set, load_conf);
         EXPECT_EQ(vec_index->Count(), NB);
     } else {
-        vec_index = dynamic_cast<milvus::index::VectorIndex*>(index.get());
+        vec_index->Load(load_conf);
     }
     EXPECT_EQ(vec_index->GetDim(), DIM);
     EXPECT_EQ(vec_index->Count(), NB);

internal/core/unittest/test_utils/DataGen.h

@@ -902,18 +902,36 @@ SealedCreator(SchemaPtr schema, const GeneratedData& dataset) {
 
 inline std::unique_ptr<milvus::index::VectorIndex>
 GenVecIndexing(int64_t N, int64_t dim, const float* vec) {
-    // {knowhere::IndexParams::nprobe, 10},
     auto conf =
         knowhere::Json{{knowhere::meta::METRIC_TYPE, knowhere::metric::L2},
                        {knowhere::meta::DIM, std::to_string(dim)},
                        {knowhere::indexparam::NLIST, "1024"},
                        {knowhere::meta::DEVICE_ID, 0}};
     auto database = knowhere::GenDataSet(N, dim, vec);
+    milvus::storage::FieldDataMeta field_data_meta{1, 2, 3, 100};
+    milvus::storage::IndexMeta index_meta{3, 100, 1000, 1};
+    milvus::storage::StorageConfig storage_config;
+    storage_config.storage_type = "local";
+    storage_config.root_path = TestRemotePath;
+    auto chunk_manager = milvus::storage::CreateChunkManager(storage_config);
+    milvus::storage::FileManagerContext file_manager_context(
+        field_data_meta, index_meta, chunk_manager);
     auto indexing = std::make_unique<index::VectorMemIndex>(
         knowhere::IndexEnum::INDEX_FAISS_IVFFLAT,
         knowhere::metric::L2,
-        knowhere::Version::GetCurrentVersion().VersionNumber());
+        knowhere::Version::GetCurrentVersion().VersionNumber(),
+        file_manager_context);
     indexing->BuildWithDataset(database, conf);
+    auto binary_set = indexing->Upload();
+
+    std::vector<std::string> index_files;
+    for (auto& binary : binary_set.binary_map_) {
+        index_files.emplace_back(binary.first);
+    }
+    conf["index_files"] = index_files;
+    // we need a load stage to use index as the producation does
+    // knowhere would do some data preparation in this stage
+    indexing->Load(conf);
     return indexing;
 }
 

internal/core/unittest/test_utils/indexbuilder_test_utils.h

@@ -114,7 +114,10 @@ generate_load_conf(const milvus::IndexType& index_type,
              std::to_string(0.0002)},
         };
     }
-    return knowhere::Json();
+    return knowhere::Json{
+        {knowhere::meta::METRIC_TYPE, metric_type},
+        {knowhere::meta::DIM, std::to_string(DIM)},
+    };
 }
 
 std::vector<milvus::IndexType>

tests/python_client/testcases/test_search.py

@@ -3294,7 +3294,6 @@ class TestCollectionSearch(TestcaseBase):
                                          "output_fields": output_fields})
 
     @pytest.mark.tags(CaseLabel.L1)
-    @pytest.mark.skip(reason="issue #27462")
     @pytest.mark.parametrize("index, params",
                              zip(ct.all_index_types[:7],
                                  ct.default_index_params[:7]))