milvus-io
/
milvus
mirror of https://github.com/milvus-io/milvus.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Support delete in SegmentGrowing (#10154) 

Signed-off-by: fishpenguin <kun.yu@zilliz.com>
pull/10159/head
yukun 2021-10-19 11:54:34 +08:00 committed by GitHub
parent 39bf58d8cb
commit e108476601
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 32 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
internal/core/src/query/visitors/ExecPlanNodeVisitor.cpp
View File

@ -110,7 +110,9 @@ ExecPlanNodeVisitor::VectorVisitorImpl(VectorPlanNode& node) {
view = BitsetView((uint8_t*)boost_ext::get_data(bitset_holder), bitset_holder.size());
}
segment->vector_search(active_count, node.search_info_, src_data, num_queries, MAX_TIMESTAMP, view, ret);
auto final_bitset = segment->get_filtered_bitmap(view, active_count, MAX_TIMESTAMP);
segment->vector_search(active_count, node.search_info_, src_data, num_queries, MAX_TIMESTAMP, final_bitset, ret);
ret_ = ret;
}

14
internal/core/src/segcore/SegmentGrowingImpl.cpp
View File

@ -43,11 +43,11 @@ SegmentGrowingImpl::PreDelete(int64_t size) {
return reserved_begin;
}
auto
std::shared_ptr<DeletedRecord::TmpBitmap>
SegmentGrowingImpl::get_deleted_bitmap(int64_t del_barrier,
Timestamp query_timestamp,
int64_t insert_barrier,
bool force) -> std::shared_ptr<DeletedRecord::TmpBitmap> {
bool force) const {
auto old = deleted_record_.get_lru_entry();
if (!force || old->bitmap_ptr->count() == insert_barrier) {
@ -115,10 +115,16 @@ SegmentGrowingImpl::get_deleted_bitmap(int64_t del_barrier,
return current;
}
const BitsetView
SegmentGrowingImpl::get_filtered_bitmap(BitsetView& bitset, int64_t ins_barrier, Timestamp timestamp) {
BitsetView
SegmentGrowingImpl::get_filtered_bitmap(const BitsetView& bitset, int64_t ins_barrier, Timestamp timestamp) const {
auto del_barrier = get_barrier(get_deleted_record(), timestamp);
if (del_barrier == 0) {
return bitset;
}
auto bitmap_holder = get_deleted_bitmap(del_barrier, timestamp, ins_barrier);
if (bitmap_holder == nullptr) {
return bitset;
}
AssertInfo(bitmap_holder, "bitmap_holder is null");
auto deleted_bitmap = bitmap_holder->bitmap_ptr;
AssertInfo(deleted_bitmap->count() == bitset.u8size(), "Deleted bitmap count not equal to filtered bitmap count");

11
internal/core/src/segcore/SegmentGrowingImpl.h
View File

@ -177,10 +177,13 @@ class SegmentGrowingImpl : public SegmentGrowing {
public:
std::shared_ptr<DeletedRecord::TmpBitmap>
get_deleted_bitmap(int64_t del_barrier, Timestamp query_timestamp, int64_t insert_barrier, bool force = false);
get_deleted_bitmap(int64_t del_barrier,
Timestamp query_timestamp,
int64_t insert_barrier,
bool force = false) const;
const BitsetView
get_filtered_bitmap(BitsetView& bitset, int64_t ins_barrier, Timestamp timestamp);
BitsetView
get_filtered_bitmap(const BitsetView& bitset, int64_t ins_barrier, Timestamp timestamp) const override;
std::pair<std::unique_ptr<IdArray>, std::vector<SegOffset>>
search_ids(const IdArray& id_array, Timestamp timestamp) const override;
@ -213,7 +216,7 @@ class SegmentGrowingImpl : public SegmentGrowing {
SchemaPtr schema_;
InsertRecord record_;
DeletedRecord deleted_record_;
mutable DeletedRecord deleted_record_;
IndexingRecord indexing_record_;
SealedIndexingRecord sealed_indexing_record_;

3
internal/core/src/segcore/SegmentInterface.h
View File

@ -101,6 +101,9 @@ class SegmentInternalInterface : public SegmentInterface {
const BitsetView& bitset,
SearchResult& output) const = 0;
virtual BitsetView
get_filtered_bitmap(const BitsetView& bitset, int64_t ins_barrier, Timestamp timestamp) const = 0;
// count of chunk that has index available
virtual int64_t
num_chunk_index(FieldOffset field_offset) const = 0;

6
internal/core/src/segcore/SegmentSealedImpl.cpp
View File

@ -212,6 +212,12 @@ SegmentSealedImpl::get_schema() const {
return *schema_;
}
BitsetView
SegmentSealedImpl::get_filtered_bitmap(const BitsetView& bitset, int64_t ins_barrier, Timestamp timestamp) const {
// TODO(yukun)
return bitset;
}
void
SegmentSealedImpl::vector_search(int64_t vec_count,
query::SearchInfo search_info,

3
internal/core/src/segcore/SegmentSealedImpl.h
View File

@ -118,6 +118,9 @@ class SegmentSealedImpl : public SegmentSealed {
const BitsetView& bitset,
SearchResult& output) const override;
BitsetView
get_filtered_bitmap(const BitsetView& bitset, int64_t ins_barrier, Timestamp timestamp) const override;
bool
is_system_field_ready() const {
return system_ready_count_ == 2;