milvus-io
/
milvus
mirror of https://github.com/milvus-io/milvus.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Optimize load segment, make load fields concurrent (#22856) 

Signed-off-by: yah01 <yang.cen@zilliz.com>
pull/22900/head
yah01 2023-03-21 16:27:57 +08:00 committed by GitHub
parent 532d12c4cf
commit 9efebb3df7
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 5 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
internal/core/src/segcore/InsertRecord.h
View File

@ -13,6 +13,7 @@
#include <algorithm>
#include <memory>
#include <mutex>
#include <string>
#include <unordered_map>
#include <utility>
@ -264,6 +265,7 @@ struct InsertRecord {
void
seal_pks() {
std::lock_guard lck(shared_mutex_);
pk2offset_->seal();
}

7
internal/core/src/segcore/SegmentSealedImpl.cpp
View File

@ -221,9 +221,6 @@ SegmentSealedImpl::LoadFieldData(const LoadFieldDataInfo& info) {
AssertInfo(data_type == DataType(info.field_data->type()),
"field type of load data is inconsistent with the schema");
// write data under lock
std::unique_lock lck(mutex_);
// Don't allow raw data and index exist at the same time
AssertInfo(!get_bit(index_ready_bitset_, field_id),
"field data can't be loaded when indexing exists");
@ -234,11 +231,13 @@ SegmentSealedImpl::LoadFieldData(const LoadFieldDataInfo& info) {
VariableField field(get_segment_id(), field_meta, info);
size = field.size();
field_data = reinterpret_cast<void*>(field.data());
std::unique_lock lck(mutex_);
variable_fields_.emplace(field_id, std::move(field));
} else {
field_data = CreateMap(get_segment_id(), field_meta, info);
fixed_fields_[field_id] = field_data;
size = field_meta.get_sizeof() * info.row_count;
std::unique_lock lck(mutex_);
fixed_fields_[field_id] = field_data;
}
// set pks to offset