milvus-io
/
milvus
mirror of https://github.com/milvus-io/milvus.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

enhance: Change sync manager parallel config item (#29216) 

Since the sync manager is global in datanode now, the old
`maxParallelSyncTaskNum` does not fit into current implementation
anymore.

This PR add a new param item for sync mgr parallel control and enlarge
default value

Signed-off-by: Congqi Xia <congqi.xia@zilliz.com>
pull/29007/head
congqixia 2023-12-14 20:46:41 +08:00 committed by GitHub
parent 6efb7afd3f
commit 25a4525297
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 12 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
internal/datanode/data_node.go
View File

@ -273,7 +273,7 @@ func (node *DataNode) Init() error {
}
node.chunkManager = chunkManager
syncMgr, err := syncmgr.NewSyncManager(paramtable.Get().DataNodeCfg.MaxParallelSyncTaskNum.GetAsInt(),
syncMgr, err := syncmgr.NewSyncManager(paramtable.Get().DataNodeCfg.MaxParallelSyncMgrTasks.GetAsInt(),
node.chunkManager, node.allocator)
if err != nil {
initError = err

12
pkg/util/paramtable/component_param.go
View File

@ -2589,6 +2589,7 @@ type dataNodeConfig struct {
FlowGraphMaxQueueLength ParamItem `refreshable:"false"`
FlowGraphMaxParallelism ParamItem `refreshable:"false"`
MaxParallelSyncTaskNum ParamItem `refreshable:"false"`
MaxParallelSyncMgrTasks ParamItem `refreshable:"false"`
// skip mode
FlowGraphSkipModeEnable ParamItem `refreshable:"true"`
@ -2686,11 +2687,20 @@ func (p *dataNodeConfig) init(base *BaseTable) {
Key: "dataNode.dataSync.maxParallelSyncTaskNum",
Version: "2.3.0",
DefaultValue: "6",
Doc: "Maximum number of sync tasks executed in parallel in each flush manager",
Doc: "deprecated, legacy flush manager max conurrency number",
Export: true,
}
p.MaxParallelSyncTaskNum.Init(base.mgr)
p.MaxParallelSyncMgrTasks = ParamItem{
Key: "dataNode.dataSync.maxParallelSyncMgrTasks",
Version: "2.3.4",
DefaultValue: "64",
Doc: "The max concurrent sync task number of datanode sync mgr globally",
Export: true,
}
p.MaxParallelSyncMgrTasks.Init(base.mgr)
p.FlushInsertBufferSize = ParamItem{
Key: "dataNode.segment.insertBufSize",
Version: "2.0.0",