milvus-io
/
milvus
mirror of https://github.com/milvus-io/milvus.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

enhance: reduce mmap_rss after chunkcache warmup (#35974) 

related pr: https://github.com/milvus-io/milvus/pull/35965

Signed-off-by: cqy123456 <qianya.cheng@zilliz.com>
pull/36034/head
cqy123456 2024-09-05 18:07:05 +08:00 committed by GitHub
parent 8b043f58dc
commit 560e8e70b0
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
3 changed files with 27 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
internal/core/src/segcore/SegmentSealedImpl.cpp
View File

@ -152,9 +152,13 @@ SegmentSealedImpl::WarmupChunkCache(const FieldId field_id, bool mmap_enabled) {
auto field_info = it->second;
auto cc = storage::MmapManager::GetInstance().GetChunkCache();
bool mmap_rss_not_need = true;
for (const auto& data_path : field_info.insert_files) {
auto column =
cc->Read(data_path, mmap_descriptor_, field_meta, mmap_enabled);
auto column = cc->Read(data_path,
mmap_descriptor_,
field_meta,
mmap_enabled,
mmap_rss_not_need);
}
}

21
internal/core/src/storage/ChunkCache.cpp
View File

@ -24,7 +24,8 @@ std::shared_ptr<ColumnBase>
ChunkCache::Read(const std::string& filepath,
const MmapChunkDescriptorPtr& descriptor,
const FieldMeta& field_meta,
bool mmap_enabled) {
bool mmap_enabled,
bool mmap_rss_not_need) {
// use rlock to get future
{
std::shared_lock lck(mutex_);
@ -64,6 +65,22 @@ ChunkCache::Read(const std::string& filepath,
field_data = DownloadAndDecodeRemoteFile(cm_.get(), filepath);
column = Mmap(
field_data->GetFieldData(), descriptor, field_meta, mmap_enabled);
if (mmap_enabled && mmap_rss_not_need) {
auto ok = madvise(reinterpret_cast<void*>(
const_cast<char*>(column->MmappedData())),
column->ByteSize(),
ReadAheadPolicy_Map["dontneed"]);
if (ok != 0) {
LOG_WARN(
"failed to madvise to the data file {}, addr {}, size {}, "
"err: "
"{}",
filepath,
static_cast<const void*>(column->MmappedData()),
column->ByteSize(),
strerror(errno));
}
}
allocate_success = true;
} catch (const SegcoreError& e) {
err_code = e.get_error_code();
@ -113,7 +130,7 @@ ChunkCache::Prefetch(const std::string& filepath) {
LOG_WARN(
"failed to madvise to the data file {}, addr {}, size {}, err: {}",
filepath,
column->MmappedData(),
static_cast<const void*>(column->MmappedData()),
column->ByteSize(),
strerror(errno));
}

3
internal/core/src/storage/ChunkCache.h
View File

@ -48,7 +48,8 @@ class ChunkCache {
Read(const std::string& filepath,
const MmapChunkDescriptorPtr& descriptor,
const FieldMeta& field_meta,
bool mmap_enabled);
bool mmap_enabled,
bool mmap_rss_not_need = false);
void
Remove(const std::string& filepath);