milvus-io
/
milvus
mirror of https://github.com/milvus-io/milvus.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Predict the loading memory usage more accurate (#17451) 

Signed-off-by: yah01 <yang.cen@zilliz.com>
pull/17452/head
yah01 2022-06-09 18:16:07 +08:00 committed by GitHub
parent 2ca81620ea
commit a38790f586
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 28 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
configs/milvus.yaml
View File

@ -162,6 +162,7 @@ queryCoord:
queryNode:
cacheSize: 32 # GB, default 32 GB, `cacheSize` is the memory used for caching data for faster query. The `cacheSize` must be less than system memory size.
port: 21123
loadMemoryUsageFactor: 3 # The multiply factor of calculating the memory usage while loading segments
stats:
publishInterval: 1000 # Interval for querynode to report node information (milliseconds)

21
internal/querynode/segment_loader.go
View File

@ -790,14 +790,25 @@ func (loader *segmentLoader) checkSegmentSize(collectionID UniqueID, segmentLoad
}
}
toMB := func(mem uint64) float64 {
return float64(mem) / 1024 / 1024
toMB := func(mem uint64) uint64 {
return mem / 1024 / 1024
}
// when load segment, data will be copied from go memory to c++ memory
if usedMemAfterLoad+maxSegmentSize*uint64(concurrency) > uint64(float64(totalMem)*Params.QueryNodeCfg.OverloadedMemoryThresholdPercentage) {
return fmt.Errorf("load segment failed, OOM if load, collectionID = %d, maxSegmentSize = %.2f MB, concurrency = %d, usedMemAfterLoad = %.2f MB, totalMem = %.2f MB, thresholdFactor = %f",
collectionID, toMB(maxSegmentSize), concurrency, toMB(usedMemAfterLoad), toMB(totalMem), Params.QueryNodeCfg.OverloadedMemoryThresholdPercentage)
loadingUsage := usedMemAfterLoad + uint64(
float64(maxSegmentSize)*float64(concurrency)*Params.QueryNodeCfg.LoadMemoryUsageFactor)
log.Debug("predict memory usage while loading (in MiB)",
zap.Uint64("usage", toMB(loadingUsage)),
zap.Uint64("usageAfterLoad", toMB(usedMemAfterLoad)))
if loadingUsage > uint64(float64(totalMem)*Params.QueryNodeCfg.OverloadedMemoryThresholdPercentage) {
return fmt.Errorf("load segment failed, OOM if load, collectionID = %d, maxSegmentSize = %v MB, concurrency = %d, usedMemAfterLoad = %v MB, totalMem = %v MB, thresholdFactor = %f",
collectionID,
toMB(maxSegmentSize),
concurrency,
toMB(usedMemAfterLoad),
toMB(totalMem),
Params.QueryNodeCfg.OverloadedMemoryThresholdPercentage)
}
return nil

11
internal/util/paramtable/component_param.go
View File

@ -693,6 +693,7 @@ type queryNodeConfig struct {
UpdatedTime time.Time
// memory limit
LoadMemoryUsageFactor float64
OverloadedMemoryThresholdPercentage float64
// cache limit
@ -718,6 +719,7 @@ func (p *queryNodeConfig) init(base *BaseTable) {
p.initSmallIndexParams()
p.initLoadMemoryUsageFactor()
p.initOverloadedMemoryThresholdPercentage()
p.initCacheMemoryLimit()
@ -801,6 +803,15 @@ func (p *queryNodeConfig) initSmallIndexParams() {
}
}
func (p *queryNodeConfig) initLoadMemoryUsageFactor() {
loadMemoryUsageFactor := p.Base.LoadWithDefault("queryNode.loadMemoryUsageFactor", "3")
factor, err := strconv.ParseFloat(loadMemoryUsageFactor, 64)
if err != nil {
panic(err)
}
p.LoadMemoryUsageFactor = factor
}
func (p *queryNodeConfig) initOverloadedMemoryThresholdPercentage() {
overloadedMemoryThresholdPercentage := p.Base.LoadWithDefault("queryCoord.overloadedMemoryThresholdPercentage", "90")
thresholdPercentage, err := strconv.ParseInt(overloadedMemoryThresholdPercentage, 10, 64)