milvus-io
/
milvus
mirror of https://github.com/milvus-io/milvus.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

fix: fix several bugs and refactor some codes related with chunked segment (#37168) 

issue: https://github.com/milvus-io/milvus/issues/37147

---------

Signed-off-by: sunby <sunbingyi1992@gmail.com>
pull/37195/head
Bingyi Sun 2024-10-28 14:17:30 +08:00 committed by GitHub
parent cdee149191
commit b81f162f6a
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
4 changed files with 18 additions and 29 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
internal/core/src/common/Chunk.h
View File

@ -66,6 +66,11 @@ class Chunk {
return data_;
}
const char*
RawData() const {
return data_;
}
virtual bool
isValid(int offset) {
return valid_[offset];

7
internal/core/src/exec/expression/CompareExpr.cpp
View File

@ -64,15 +64,16 @@ PhyCompareFilterExpr::ExecCompareExprDispatcher(OpType op) {
right_current_chunk_id_,
right_current_chunk_pos_);
for (int i = 0; i < real_batch_size; ++i) {
if (!left().has_value() || !right().has_value()) {
auto left_value = left(), right_value = right();
if (!left_value.has_value() || !right_value.has_value()) {
res[i] = false;
valid_res[i] = false;
continue;
}
res[i] =
boost::apply_visitor(milvus::query::Relational<decltype(op)>{},
left().value(),
right().value());
left_value.value(),
right_value.value());
}
return res_vec;
} else {

2
internal/core/src/mmap/ChunkedColumn.h
View File

@ -83,7 +83,7 @@ class ChunkedColumnBase : public ColumnBase {
AssertInfo(chunks_.size() == 1,
"only support one chunk, but got {} chunk(s)",
chunks_.size());
return chunks_[0]->Data();
return chunks_[0]->RawData();
}
bool

33
internal/core/src/segcore/SegmentChunkReader.cpp
View File

@ -25,21 +25,13 @@ SegmentChunkReader::GetChunkDataAccessor(FieldId field_id,
if (index) {
auto& indexing = const_cast<index::ScalarIndex<T>&>(
segment_->chunk_scalar_index<T>(field_id, current_chunk_id));
auto current_chunk_size = segment_->type() == SegmentType::Growing
? SizePerChunk()
: active_count_;
if (indexing.HasRawData()) {
return [&, current_chunk_size]() -> const data_access_type {
if (current_chunk_pos >= current_chunk_size) {
current_chunk_id++;
current_chunk_pos = 0;
indexing = const_cast<index::ScalarIndex<T>&>(
segment_->chunk_scalar_index<T>(field_id,
current_chunk_id));
return [&]() -> const data_access_type {
if (current_chunk_pos >= active_count_) {
return std::nullopt;
}
auto raw = indexing.Reverse_Lookup(current_chunk_pos);
current_chunk_pos++;
auto raw = indexing.Reverse_Lookup(current_chunk_pos++);
if (!raw.has_value()) {
return std::nullopt;
}
@ -85,21 +77,12 @@ SegmentChunkReader::GetChunkDataAccessor<std::string>(
auto& indexing = const_cast<index::ScalarIndex<std::string>&>(
segment_->chunk_scalar_index<std::string>(field_id,
current_chunk_id));
auto current_chunk_size = segment_->type() == SegmentType::Growing
? SizePerChunk()
: active_count_;
if (indexing.HasRawData()) {
return [&, current_chunk_size]() mutable -> const data_access_type {
if (current_chunk_pos >= current_chunk_size) {
current_chunk_id++;
current_chunk_pos = 0;
indexing = const_cast<index::ScalarIndex<std::string>&>(
segment_->chunk_scalar_index<std::string>(
field_id, current_chunk_id));
return [&]() mutable -> const data_access_type {
if (current_chunk_pos >= active_count_) {
return std::nullopt;
}
auto raw = indexing.Reverse_Lookup(current_chunk_pos);
current_chunk_pos++;
auto raw = indexing.Reverse_Lookup(current_chunk_pos++);
if (!raw.has_value()) {
return std::nullopt;
}