milvus-io
/
milvus
mirror of https://github.com/milvus-io/milvus.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

enhance: [Cherry-Pick] Add back load memory factor when esitmating memory resource (#30999) 

Cherry-pick from master
pr: #30994
Segment load memory usage is underestimated due to removing the load
memroy factor. This PR adds it back to protect querynode OOM during some
extreme memory cases.

Signed-off-by: Congqi Xia <congqi.xia@zilliz.com>
pull/31024/head
congqixia 2024-03-05 09:15:00 +08:00 committed by GitHub
parent 336e0ae45e
commit 81b197267a
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 25 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

35
internal/querynodev2/segments/segment_loader.go
View File

@ -970,11 +970,13 @@ func (loader *segmentLoader) checkSegmentSize(ctx context.Context, segmentLoadIn
diskUsage := uint64(localDiskUsage) + loader.committedResource.DiskSize
mmapEnabled := len(paramtable.Get().QueryNodeCfg.MmapDirPath.GetValue()) > 0
memoryUsageFactor := paramtable.Get().QueryNodeCfg.LoadMemoryUsageFactor.GetAsFloat()
maxSegmentSize := uint64(0)
predictMemUsage := memUsage
predictDiskUsage := diskUsage
for _, loadInfo := range segmentLoadInfos {
oldUsedMem := predictMemUsage
var segmentMemorySize, segmentDiskSize uint64
vecFieldID2IndexInfo := make(map[int64]*querypb.FieldIndexInfo)
for _, fieldIndexInfo := range loadInfo.IndexInfos {
if fieldIndexInfo.EnableIndex {
@ -997,20 +999,21 @@ func (loader *segmentLoader) checkSegmentSize(ctx context.Context, segmentLoadIn
return 0, 0, err
}
if mmapEnabled {
predictDiskUsage += neededMemSize + neededDiskSize
segmentDiskSize += neededMemSize + neededDiskSize
} else {
predictMemUsage += neededMemSize
predictDiskUsage += neededDiskSize
segmentMemorySize += neededMemSize
segmentDiskSize += neededDiskSize
}
} else {
binlogSize := uint64(getBinlogDataSize(fieldBinlog))
if mmapEnabled {
predictDiskUsage += uint64(getBinlogDataSize(fieldBinlog))
segmentDiskSize += binlogSize
} else {
predictMemUsage += uint64(getBinlogDataSize(fieldBinlog))
segmentMemorySize += binlogSize
enableBinlogIndex := paramtable.Get().QueryNodeCfg.EnableInterimSegmentIndex.GetAsBool()
if enableBinlogIndex {
buildBinlogIndexRate := paramtable.Get().QueryNodeCfg.InterimIndexMemExpandRate.GetAsFloat()
predictMemUsage += uint64(float32(getBinlogDataSize(fieldBinlog)) * float32(buildBinlogIndexRate))
segmentMemorySize += uint64(float64(binlogSize) * buildBinlogIndexRate)
}
}
}
@ -1018,7 +1021,7 @@ func (loader *segmentLoader) checkSegmentSize(ctx context.Context, segmentLoadIn
// get size of stats data
for _, fieldBinlog := range loadInfo.Statslogs {
predictMemUsage += uint64(getBinlogDataSize(fieldBinlog))
segmentMemorySize += uint64(getBinlogDataSize(fieldBinlog))
}
// get size of delete data
@ -1026,9 +1029,21 @@ func (loader *segmentLoader) checkSegmentSize(ctx context.Context, segmentLoadIn
predictMemUsage += uint64(getBinlogDataSize(fieldBinlog))
}
if predictMemUsage-oldUsedMem > maxSegmentSize {
maxSegmentSize = predictMemUsage - oldUsedMem
segmentMemorySize = uint64(float64(segmentMemorySize) * memoryUsageFactor)
if segmentMemorySize > maxSegmentSize {
maxSegmentSize = segmentMemorySize
}
predictMemUsage += segmentMemorySize
predictDiskUsage += segmentDiskSize
log.Debug("segment resource for loading",
zap.Int64("segmentID", loadInfo.GetSegmentID()),
zap.Float64("memoryUsage(MB)", toMB(segmentMemorySize)),
zap.Float64("diskUsage(MB)", toMB(segmentDiskSize)),
zap.Float64("memoryLoadFactor", memoryUsageFactor),
)
}
log.Info("predict memory and disk usage while loading (in MiB)",