milvus-io
/
milvus
mirror of https://github.com/milvus-io/milvus.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

enhance: Add param item for datanode l0 batch/linear mode memory ratio (#30523) 

See also #27606

---------

Signed-off-by: Congqi Xia <congqi.xia@zilliz.com>
pull/30632/head
congqixia 2024-02-18 13:02:50 +08:00 committed by GitHub
parent bb63e324b2
commit 91b02b5d22
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
2 changed files with 14 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
internal/datanode/l0_compactor.go
View File

@ -162,8 +162,8 @@ func (t *levelZeroCompactionTask) compact() (*datapb.CompactionPlanResult, error
}
var resultSegments []*datapb.CompactionSegment
// TODO
if hardware.GetFreeMemoryCount() < uint64(totalSize)*20 {
if float64(hardware.GetFreeMemoryCount())*paramtable.Get().DataNodeCfg.L0BatchMemoryRatio.GetAsFloat() < float64(totalSize) {
resultSegments, err = t.linearProcess(ctxTimeout, targetSegIDs, totalDeltalogs)
} else {
resultSegments, err = t.batchProcess(ctxTimeout, targetSegIDs, lo.Values(totalDeltalogs)...)

12
pkg/util/paramtable/component_param.go
View File

@ -2932,6 +2932,9 @@ type dataNodeConfig struct {
UpdateChannelCheckpointRPCTimeout ParamItem `refreshable:"true"`
MaxConcurrentImportTaskNum ParamItem `refreshable:"true"`
// Compaction
L0BatchMemoryRatio ParamItem `refreshable:"true"`
}
func (p *dataNodeConfig) init(base *BaseTable) {
@ -3192,6 +3195,15 @@ func (p *dataNodeConfig) init(base *BaseTable) {
Export: true,
}
p.MaxConcurrentImportTaskNum.Init(base.mgr)
p.L0BatchMemoryRatio = ParamItem{
Key: "datanode.compaction.levelZeroBatchMemoryRatio",
Version: "2.4.0",
Doc: "The minimal memory ratio of free memory for level zero compaction executing in batch mode",
DefaultValue: "0.05",
Export: true,
}
p.L0BatchMemoryRatio.Init(base.mgr)
}
// /////////////////////////////////////////////////////////////////////////////