From 04f00bbe91cdc400279bd2aff18501a847636289 Mon Sep 17 00:00:00 2001 From: Zhiru Zhu Date: Thu, 12 Mar 2020 16:43:18 +0800 Subject: [PATCH 1/3] update Signed-off-by: Zhiru Zhu --- core/src/db/engine/ExecutionEngineImpl.cpp | 9 ++++++++- core/unittest/db/utils.cpp | 3 ++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/core/src/db/engine/ExecutionEngineImpl.cpp b/core/src/db/engine/ExecutionEngineImpl.cpp index 49f3c73b25..9352a34382 100644 --- a/core/src/db/engine/ExecutionEngineImpl.cpp +++ b/core/src/db/engine/ExecutionEngineImpl.cpp @@ -738,15 +738,22 @@ ExecutionEngineImpl::BuildIndex(const std::string& location, EngineType engine_t auto status = Status::OK(); std::vector uids; + faiss::ConcurrentBitsetPtr blacklist; if (from_index) { status = to_index->BuildAll(Count(), from_index->GetRawVectors(), from_index->GetRawIds(), conf); uids = from_index->GetUids(); + from_index->GetBlacklist(blacklist); } else if (bin_from_index) { status = to_index->BuildAll(Count(), bin_from_index->GetRawVectors(), bin_from_index->GetRawIds(), conf); uids = bin_from_index->GetUids(); + bin_from_index->GetBlacklist(blacklist); } to_index->SetUids(uids); - ENGINE_LOG_DEBUG << "set uids " << to_index->GetUids().size() << " for " << location; + ENGINE_LOG_DEBUG << "Set " << to_index->GetUids().size() << "uids for " << location; + if (blacklist != nullptr) { + to_index->SetBlacklist(blacklist); + ENGINE_LOG_DEBUG << "Set blacklist for index " << location; + } if (!status.ok()) { throw Exception(DB_ERROR, status.message()); diff --git a/core/unittest/db/utils.cpp b/core/unittest/db/utils.cpp index 77fe031d26..3d9863f159 100644 --- a/core/unittest/db/utils.cpp +++ b/core/unittest/db/utils.cpp @@ -65,7 +65,7 @@ static const char *CONFIG_STR = " cpu_cache_capacity: 4 # GB, CPU memory used for cache\n" " cpu_cache_threshold: 0.85 # percentage of data kept when cache cleanup triggered\n" " insert_buffer_size: 4 # GB, maximum insert buffer size allowed\n" - " cache_insert_data: false # whether load inserted data into cache\n" + " cache_insert_data: true # whether load inserted data into cache\n" "\n" "engine_config:\n" " use_blas_threshold: 20\n" @@ -179,6 +179,7 @@ DBTest::SetUp() { milvus::scheduler::JobMgrInst::GetInstance()->Start(); auto options = GetOptions(); + options.insert_cache_immediately_ = true; db_ = milvus::engine::DBFactory::Build(options); std::string config_path(options.meta_.path_ + CONFIG_FILE); From 7ac4e95e86067d02ce3bb8eeb23af88dad532538 Mon Sep 17 00:00:00 2001 From: Zhiru Zhu Date: Thu, 12 Mar 2020 16:44:43 +0800 Subject: [PATCH 2/3] update Signed-off-by: Zhiru Zhu --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index cb778c2c8a..6c881bc7d0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -60,6 +60,7 @@ Please mark all change in change log and use the issue from GitHub - \#1598 Server down during mixed operations - \#1601 External link bug in HTTP doc - \#1609 Refine Compact function +- \#1635 Vectors can be returned by searching after vectors deleted if `cache_insert_data` set true ## Feature - \#216 Add CLI to get server info From 7ec0444dc0238da1aae396e7e1401d4551772675 Mon Sep 17 00:00:00 2001 From: Zhiru Zhu Date: Thu, 12 Mar 2020 16:57:52 +0800 Subject: [PATCH 3/3] update Signed-off-by: Zhiru Zhu --- CHANGELOG.md | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6c881bc7d0..9938505788 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,12 @@ Please mark all change in change log and use the issue from GitHub -# Milvus 0.7.0 (TBD) +# Milvus 0.7.1 (TBD) + +## Bug +- \#1635 Vectors can be returned by searching after vectors deleted if `cache_insert_data` set true + +# Milvus 0.7.0 (2020-03-11) ## Bug - \#715 Milvus crash when searching and building index simultaneously using SQ8H @@ -60,7 +65,6 @@ Please mark all change in change log and use the issue from GitHub - \#1598 Server down during mixed operations - \#1601 External link bug in HTTP doc - \#1609 Refine Compact function -- \#1635 Vectors can be returned by searching after vectors deleted if `cache_insert_data` set true ## Feature - \#216 Add CLI to get server info