milvus-io
/
milvus
mirror of https://github.com/milvus-io/milvus.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Optimize load segment, make load fields concurrent (#22856) 

Signed-off-by: yah01 <yang.cen@zilliz.com>
pull/22900/head
yah01 2023-03-21 16:27:57 +08:00 committed by GitHub
parent 532d12c4cf
commit 9efebb3df7
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 5 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
internal/core/src/segcore/InsertRecord.h
View File

@ -13,6 +13,7 @@
#include <algorithm> #include <algorithm>
#include <memory> #include <memory>
#include <mutex>
#include <string> #include <string>
#include <unordered_map> #include <unordered_map>
#include <utility> #include <utility>
@ -264,6 +265,7 @@ struct InsertRecord {
void void
seal_pks() { seal_pks() {
std::lock_guard lck(shared_mutex_);
pk2offset_->seal(); pk2offset_->seal();
} }

7
internal/core/src/segcore/SegmentSealedImpl.cpp
View File

@ -221,9 +221,6 @@ SegmentSealedImpl::LoadFieldData(const LoadFieldDataInfo& info) {
AssertInfo(data_type == DataType(info.field_data->type()), AssertInfo(data_type == DataType(info.field_data->type()),
"field type of load data is inconsistent with the schema"); "field type of load data is inconsistent with the schema");
// write data under lock
std::unique_lock lck(mutex_);
// Don't allow raw data and index exist at the same time // Don't allow raw data and index exist at the same time
AssertInfo(!get_bit(index_ready_bitset_, field_id), AssertInfo(!get_bit(index_ready_bitset_, field_id),
"field data can't be loaded when indexing exists"); "field data can't be loaded when indexing exists");
@ -234,11 +231,13 @@ SegmentSealedImpl::LoadFieldData(const LoadFieldDataInfo& info) {
VariableField field(get_segment_id(), field_meta, info); VariableField field(get_segment_id(), field_meta, info);
size = field.size(); size = field.size();
field_data = reinterpret_cast<void*>(field.data()); field_data = reinterpret_cast<void*>(field.data());
std::unique_lock lck(mutex_);
variable_fields_.emplace(field_id, std::move(field)); variable_fields_.emplace(field_id, std::move(field));
} else { } else {
field_data = CreateMap(get_segment_id(), field_meta, info); field_data = CreateMap(get_segment_id(), field_meta, info);
fixed_fields_[field_id] = field_data;
size = field_meta.get_sizeof() * info.row_count; size = field_meta.get_sizeof() * info.row_count;
std::unique_lock lck(mutex_);
fixed_fields_[field_id] = field_data;
} }
// set pks to offset // set pks to offset