milvus-io
/
milvus
mirror of https://github.com/milvus-io/milvus.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Fix search failed on disk index when search_list equal to limit (#21113) 

Signed-off-by: xige-16 <xi.ge@zilliz.com>
pull/22003/head
xige-16 2023-02-16 11:02:34 +08:00 committed by GitHub
parent 87a4ddc7e2
commit 8ec5ba13dc
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 73 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
internal/core/src/index/VectorDiskIndex.cpp
View File

@ -129,7 +129,7 @@ VectorDiskAnnIndex<T>::Query(const DatasetPtr dataset, const SearchInfo& search_
// set search list size
auto search_list_size = GetValueFromConfig<uint32_t>(search_info.search_params_, DISK_ANN_QUERY_LIST);
AssertInfo(search_list_size.has_value(), "param " + std::string(DISK_ANN_QUERY_LIST) + "is empty");
AssertInfo(search_list_size.value() > topk, "search_list should be greater than topk");
AssertInfo(search_list_size.value() >= topk, "search_list should be greater than or equal to topk");
AssertInfo(search_list_size.value() <= std::max(uint32_t(topk * 10), uint32_t(kSearchListMaxValue1)) &&
search_list_size.value() <= uint32_t(kSearchListMaxValue2),
"search_list should be less than max(topk*10, 200) and less than 65535");

72
internal/core/unittest/test_indexing.cpp
View File

@ -423,3 +423,75 @@ TEST_P(IndexTest, BuildAndQuery) {
EXPECT_EQ(result->seg_offsets_[0], query_offset);
}
}
//#ifdef BUILD_DISK_ANN
// TEST(Indexing, SearchDiskAnnWithInvalidParam) {
// int64_t NB = 10000;
// IndexType index_type = knowhere::IndexEnum::INDEX_DISKANN;
// MetricType metric_type = knowhere::metric::L2;
// milvus::index::CreateIndexInfo create_index_info;
// create_index_info.index_type = index_type;
// create_index_info.metric_type = metric_type;
// create_index_info.field_type = milvus::DataType::VECTOR_FLOAT;
//
// StorageConfig storage_config = get_default_storage_config();
// auto rcm = std::make_shared<storage::MinioChunkManager>(storage_config);
// if (!rcm->BucketExists(storage_config.bucket_name)) {
// rcm->CreateBucket(storage_config.bucket_name);
// }
// milvus::storage::FieldDataMeta field_data_meta{1, 2, 3, 100};
// milvus::storage::IndexMeta index_meta{3, 100, 1000, 1};
// auto file_manager =
// std::make_shared<milvus::storage::DiskFileManagerImpl>(field_data_meta, index_meta, storage_config);
// auto index = milvus::index::IndexFactory::GetInstance().CreateIndex(create_index_info, file_manager);
//
// auto build_conf = knowhere::Config{
// {knowhere::meta::METRIC_TYPE, metric_type},
// {knowhere::meta::DIM, std::to_string(DIM)},
// {milvus::index::DISK_ANN_MAX_DEGREE, std::to_string(48)},
// {milvus::index::DISK_ANN_SEARCH_LIST_SIZE, std::to_string(128)},
// {milvus::index::DISK_ANN_PQ_CODE_BUDGET, std::to_string(0.001)},
// {milvus::index::DISK_ANN_BUILD_DRAM_BUDGET, std::to_string(2)},
// };
//
// // build disk ann index
// auto dataset = GenDataset(NB, metric_type, false);
// std::vector<float> xb_data = dataset.get_col<float>(milvus::FieldId(100));
// knowhere::DatasetPtr xb_dataset = knowhere::GenDataset(NB, DIM, xb_data.data());
// ASSERT_NO_THROW(index->BuildWithDataset(xb_dataset, build_conf));
//
// // serialize and load disk index, disk index can only be search after loading for now
// auto binary_set = index->Serialize(milvus::Config{});
// index.reset();
// // clean local file dir
// file_manager.reset();
//
// auto new_file_manager =
// std::make_shared<milvus::storage::DiskFileManagerImpl>(field_data_meta, index_meta, storage_config);
// auto new_index = milvus::index::IndexFactory::GetInstance().CreateIndex(create_index_info, new_file_manager);
// auto vec_index = dynamic_cast<milvus::index::VectorIndex*>(new_index.get());
// std::vector<std::string> index_files;
// for (auto& binary : binary_set.binary_map_) {
// index_files.emplace_back(binary.first);
// }
// auto load_conf = generate_load_conf(index_type, metric_type, NB);
// load_conf["index_files"] = index_files;
// vec_index->Load(binary_set, load_conf);
// EXPECT_EQ(vec_index->Count(), NB);
//
// // search disk index with search_list == limit
// int query_offset = 100;
// knowhere::DatasetPtr xq_dataset = knowhere::GenDataset(NQ, DIM, xb_data.data() + DIM * query_offset);
//
// milvus::SearchInfo search_info;
// search_info.topk_ = K;
// search_info.metric_type_ = metric_type;
// search_info.search_params_ = milvus::Config{
// {knowhere::meta::METRIC_TYPE, metric_type},
// {milvus::index::DISK_ANN_QUERY_LIST, K - 1},
// };
// EXPECT_THROW(vec_index->Query(xq_dataset, search_info, nullptr), std::runtime_error);
// // vec_index->Query(xq_dataset, search_info, nullptr);
//}
//#endif