From e108476601903d8cdb32294704b209bb7c8cf32c Mon Sep 17 00:00:00 2001 From: yukun Date: Tue, 19 Oct 2021 11:54:34 +0800 Subject: [PATCH] Support delete in SegmentGrowing (#10154) Signed-off-by: fishpenguin --- .../src/query/visitors/ExecPlanNodeVisitor.cpp | 4 +++- internal/core/src/segcore/SegmentGrowingImpl.cpp | 14 ++++++++++---- internal/core/src/segcore/SegmentGrowingImpl.h | 11 +++++++---- internal/core/src/segcore/SegmentInterface.h | 3 +++ internal/core/src/segcore/SegmentSealedImpl.cpp | 6 ++++++ internal/core/src/segcore/SegmentSealedImpl.h | 3 +++ 6 files changed, 32 insertions(+), 9 deletions(-) diff --git a/internal/core/src/query/visitors/ExecPlanNodeVisitor.cpp b/internal/core/src/query/visitors/ExecPlanNodeVisitor.cpp index 130e1a4b46..9ce984c946 100644 --- a/internal/core/src/query/visitors/ExecPlanNodeVisitor.cpp +++ b/internal/core/src/query/visitors/ExecPlanNodeVisitor.cpp @@ -110,7 +110,9 @@ ExecPlanNodeVisitor::VectorVisitorImpl(VectorPlanNode& node) { view = BitsetView((uint8_t*)boost_ext::get_data(bitset_holder), bitset_holder.size()); } - segment->vector_search(active_count, node.search_info_, src_data, num_queries, MAX_TIMESTAMP, view, ret); + auto final_bitset = segment->get_filtered_bitmap(view, active_count, MAX_TIMESTAMP); + + segment->vector_search(active_count, node.search_info_, src_data, num_queries, MAX_TIMESTAMP, final_bitset, ret); ret_ = ret; } diff --git a/internal/core/src/segcore/SegmentGrowingImpl.cpp b/internal/core/src/segcore/SegmentGrowingImpl.cpp index 9e2ba086c3..ec61dcfb87 100644 --- a/internal/core/src/segcore/SegmentGrowingImpl.cpp +++ b/internal/core/src/segcore/SegmentGrowingImpl.cpp @@ -43,11 +43,11 @@ SegmentGrowingImpl::PreDelete(int64_t size) { return reserved_begin; } -auto +std::shared_ptr SegmentGrowingImpl::get_deleted_bitmap(int64_t del_barrier, Timestamp query_timestamp, int64_t insert_barrier, - bool force) -> std::shared_ptr { + bool force) const { auto old = deleted_record_.get_lru_entry(); if (!force || old->bitmap_ptr->count() == insert_barrier) { @@ -115,10 +115,16 @@ SegmentGrowingImpl::get_deleted_bitmap(int64_t del_barrier, return current; } -const BitsetView -SegmentGrowingImpl::get_filtered_bitmap(BitsetView& bitset, int64_t ins_barrier, Timestamp timestamp) { +BitsetView +SegmentGrowingImpl::get_filtered_bitmap(const BitsetView& bitset, int64_t ins_barrier, Timestamp timestamp) const { auto del_barrier = get_barrier(get_deleted_record(), timestamp); + if (del_barrier == 0) { + return bitset; + } auto bitmap_holder = get_deleted_bitmap(del_barrier, timestamp, ins_barrier); + if (bitmap_holder == nullptr) { + return bitset; + } AssertInfo(bitmap_holder, "bitmap_holder is null"); auto deleted_bitmap = bitmap_holder->bitmap_ptr; AssertInfo(deleted_bitmap->count() == bitset.u8size(), "Deleted bitmap count not equal to filtered bitmap count"); diff --git a/internal/core/src/segcore/SegmentGrowingImpl.h b/internal/core/src/segcore/SegmentGrowingImpl.h index e38ca69aa9..b1115e54b2 100644 --- a/internal/core/src/segcore/SegmentGrowingImpl.h +++ b/internal/core/src/segcore/SegmentGrowingImpl.h @@ -177,10 +177,13 @@ class SegmentGrowingImpl : public SegmentGrowing { public: std::shared_ptr - get_deleted_bitmap(int64_t del_barrier, Timestamp query_timestamp, int64_t insert_barrier, bool force = false); + get_deleted_bitmap(int64_t del_barrier, + Timestamp query_timestamp, + int64_t insert_barrier, + bool force = false) const; - const BitsetView - get_filtered_bitmap(BitsetView& bitset, int64_t ins_barrier, Timestamp timestamp); + BitsetView + get_filtered_bitmap(const BitsetView& bitset, int64_t ins_barrier, Timestamp timestamp) const override; std::pair, std::vector> search_ids(const IdArray& id_array, Timestamp timestamp) const override; @@ -213,7 +216,7 @@ class SegmentGrowingImpl : public SegmentGrowing { SchemaPtr schema_; InsertRecord record_; - DeletedRecord deleted_record_; + mutable DeletedRecord deleted_record_; IndexingRecord indexing_record_; SealedIndexingRecord sealed_indexing_record_; diff --git a/internal/core/src/segcore/SegmentInterface.h b/internal/core/src/segcore/SegmentInterface.h index c795b6350b..6d63488d69 100644 --- a/internal/core/src/segcore/SegmentInterface.h +++ b/internal/core/src/segcore/SegmentInterface.h @@ -101,6 +101,9 @@ class SegmentInternalInterface : public SegmentInterface { const BitsetView& bitset, SearchResult& output) const = 0; + virtual BitsetView + get_filtered_bitmap(const BitsetView& bitset, int64_t ins_barrier, Timestamp timestamp) const = 0; + // count of chunk that has index available virtual int64_t num_chunk_index(FieldOffset field_offset) const = 0; diff --git a/internal/core/src/segcore/SegmentSealedImpl.cpp b/internal/core/src/segcore/SegmentSealedImpl.cpp index c1de3f9129..7f1d220207 100644 --- a/internal/core/src/segcore/SegmentSealedImpl.cpp +++ b/internal/core/src/segcore/SegmentSealedImpl.cpp @@ -212,6 +212,12 @@ SegmentSealedImpl::get_schema() const { return *schema_; } +BitsetView +SegmentSealedImpl::get_filtered_bitmap(const BitsetView& bitset, int64_t ins_barrier, Timestamp timestamp) const { + // TODO(yukun) + return bitset; +} + void SegmentSealedImpl::vector_search(int64_t vec_count, query::SearchInfo search_info, diff --git a/internal/core/src/segcore/SegmentSealedImpl.h b/internal/core/src/segcore/SegmentSealedImpl.h index 77e3a0d73e..e5aff043f4 100644 --- a/internal/core/src/segcore/SegmentSealedImpl.h +++ b/internal/core/src/segcore/SegmentSealedImpl.h @@ -118,6 +118,9 @@ class SegmentSealedImpl : public SegmentSealed { const BitsetView& bitset, SearchResult& output) const override; + BitsetView + get_filtered_bitmap(const BitsetView& bitset, int64_t ins_barrier, Timestamp timestamp) const override; + bool is_system_field_ready() const { return system_ready_count_ == 2;