From abd91eab82a4a02fb6ec4073ceecfa1184f2bda5 Mon Sep 17 00:00:00 2001 From: Tinkerrr Date: Wed, 19 Feb 2020 17:16:48 +0800 Subject: [PATCH] Fix HNSW crash (#1262) * fix Signed-off-by: xiaojun.lin * update. Signed-off-by: xiaojun.lin --- CHANGELOG.md | 1 + .../knowhere/index/vector_index/IndexHNSW.cpp | 5 ++++- core/src/wrapper/ConfAdapter.cpp | 4 ++-- core/unittest/wrapper/test_wrapper.cpp | 17 +++++++++-------- 4 files changed, 16 insertions(+), 11 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5c7defb576..d29a819e8e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,6 +22,7 @@ Please mark all change in change log and use the issue from GitHub - \#1075 - improve error message when page size or offset is illegal - \#1082 - check page_size or offset value to avoid float - \#1115 - http server support load table into memory +- \#1211 - Server down caused by searching with index_type: HNSW - \#1240 - Update license declaration ## Feature diff --git a/core/src/index/knowhere/knowhere/index/vector_index/IndexHNSW.cpp b/core/src/index/knowhere/knowhere/index/vector_index/IndexHNSW.cpp index ee12562c38..4d9deb0c73 100644 --- a/core/src/index/knowhere/knowhere/index/vector_index/IndexHNSW.cpp +++ b/core/src/index/knowhere/knowhere/index/vector_index/IndexHNSW.cpp @@ -77,11 +77,14 @@ IndexHNSW::Search(const DatasetPtr& dataset, const Config& config) { auto p_dist = (float*)malloc(dist_size * rows); using P = std::pair; - auto compare = [](P v1, P v2) { return v1.first < v2.first; }; + auto compare = [](P& v1, P& v2) { return v1.first < v2.first; }; #pragma omp parallel for for (unsigned int i = 0; i < rows; ++i) { const float* single_query = p_data + i * dim; std::vector> ret = index_->searchKnn(single_query, config->k, compare); + while (ret.size() < config->k) { + ret.push_back(std::make_pair(-1, -1)); + } std::vector dist; std::vector ids; std::transform(ret.begin(), ret.end(), std::back_inserter(dist), diff --git a/core/src/wrapper/ConfAdapter.cpp b/core/src/wrapper/ConfAdapter.cpp index bce10b3687..96500e1b71 100644 --- a/core/src/wrapper/ConfAdapter.cpp +++ b/core/src/wrapper/ConfAdapter.cpp @@ -268,8 +268,8 @@ HNSWConfAdapter::Match(const TempMetaConf& metaconf) { conf->d = metaconf.dim; conf->metric_type = metaconf.metric_type; - conf->ef = 100; // ef can be auto-configured by using sample data. - conf->M = 16; // A reasonable range of M is from 5 to 48. + conf->ef = 200; // ef can be auto-configured by using sample data. + conf->M = 32; // A reasonable range of M is from 5 to 48. return conf; } diff --git a/core/unittest/wrapper/test_wrapper.cpp b/core/unittest/wrapper/test_wrapper.cpp index 70c59e5f32..c740fa5c7f 100644 --- a/core/unittest/wrapper/test_wrapper.cpp +++ b/core/unittest/wrapper/test_wrapper.cpp @@ -77,18 +77,19 @@ INSTANTIATE_TEST_CASE_P( Values( //["Index type", "Generator type", "dim", "nb", "nq", "k", "build config", "search config"] #ifdef MILVUS_GPU_VERSION - std::make_tuple(milvus::engine::IndexType::FAISS_IVFFLAT_GPU, "Default", DIM, NB, 10, 10), - std::make_tuple(milvus::engine::IndexType::FAISS_IVFFLAT_MIX, "Default", 64, 1000, 10, 10), - std::make_tuple(milvus::engine::IndexType::FAISS_IVFSQ8_GPU, "Default", DIM, NB, 10, 10), - std::make_tuple(milvus::engine::IndexType::FAISS_IVFSQ8_MIX, "Default", DIM, NB, 10, 10), - std::make_tuple(milvus::engine::IndexType::FAISS_IVFPQ_MIX, "Default", 64, 1000, 10, 10), + std::make_tuple(milvus::engine::IndexType::FAISS_IVFFLAT_GPU, "Default", DIM, NB, 10, 10), + std::make_tuple(milvus::engine::IndexType::FAISS_IVFFLAT_MIX, "Default", 64, 1000, 10, 10), + std::make_tuple(milvus::engine::IndexType::FAISS_IVFSQ8_GPU, "Default", DIM, NB, 10, 10), + std::make_tuple(milvus::engine::IndexType::FAISS_IVFSQ8_MIX, "Default", DIM, NB, 10, 10), + std::make_tuple(milvus::engine::IndexType::FAISS_IVFPQ_MIX, "Default", 64, 1000, 10, 10), // std::make_tuple(milvus::engine::IndexType::NSG_MIX, "Default", 128, 250000, 10, 10), #endif // std::make_tuple(milvus::engine::IndexType::SPTAG_KDT_RNT_CPU, "Default", 128, 100, 10, 10), // std::make_tuple(milvus::engine::IndexType::SPTAG_BKT_RNT_CPU, "Default", 128, 100, 10, 10), - std::make_tuple(milvus::engine::IndexType::FAISS_IDMAP, "Default", 64, 1000, 10, 10), - std::make_tuple(milvus::engine::IndexType::FAISS_IVFFLAT_CPU, "Default", 64, 1000, 10, 10), - std::make_tuple(milvus::engine::IndexType::FAISS_IVFSQ8_CPU, "Default", DIM, NB, 10, 10))); + std::make_tuple(milvus::engine::IndexType::HNSW, "Default", 64, 10000, 10, 10), + std::make_tuple(milvus::engine::IndexType::FAISS_IDMAP, "Default", 64, 1000, 10, 10), + std::make_tuple(milvus::engine::IndexType::FAISS_IVFFLAT_CPU, "Default", 64, 1000, 10, 10), + std::make_tuple(milvus::engine::IndexType::FAISS_IVFSQ8_CPU, "Default", DIM, NB, 10, 10))); #ifdef MILVUS_GPU_VERSION TEST_P(KnowhereWrapperTest, WRAPPER_EXCEPTION_TEST) {