milvus-io
/
milvus
mirror of https://github.com/milvus-io/milvus.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

enhance: Add metrics for proxy queue (#40070) 

Signed-off-by: Cai Zhang <cai.zhang@zilliz.com>
pull/39654/head
cai.zhang 2025-02-27 17:30:01 +08:00 committed by GitHub
parent a148a78629
commit eb04686348
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
3 changed files with 51 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

40
internal/proxy/task_scheduler.go
View File

@ -228,6 +228,18 @@ type ddTaskQueue struct {
lock sync.Mutex lock sync.Mutex
} }
func (queue *ddTaskQueue) updateMetrics() {
queue.utLock.RLock()
unissuedTasksNum := queue.unissuedTasks.Len()
queue.utLock.RUnlock()
queue.atLock.RLock()
activateTaskNum := len(queue.activeTasks)
queue.atLock.RUnlock()
metrics.ProxyQueueTaskNum.WithLabelValues(strconv.FormatInt(paramtable.GetNodeID(), 10), "ddl", metrics.UnissuedIndexTaskLabel).Set(float64(unissuedTasksNum))
metrics.ProxyQueueTaskNum.WithLabelValues(strconv.FormatInt(paramtable.GetNodeID(), 10), "ddl", metrics.InProgressIndexTaskLabel).Set(float64(activateTaskNum))
}
type pChanStatInfo struct { type pChanStatInfo struct {
pChanStatistics pChanStatistics
tsSet map[Timestamp]struct{} tsSet map[Timestamp]struct{}
@ -241,6 +253,18 @@ type dmTaskQueue struct {
pChanStatisticsInfos map[pChan]*pChanStatInfo pChanStatisticsInfos map[pChan]*pChanStatInfo
} }
func (queue *dmTaskQueue) updateMetrics() {
queue.utLock.RLock()
unissuedTasksNum := queue.unissuedTasks.Len()
queue.utLock.RUnlock()
queue.atLock.RLock()
activateTaskNum := len(queue.activeTasks)
queue.atLock.RUnlock()
metrics.ProxyQueueTaskNum.WithLabelValues(strconv.FormatInt(paramtable.GetNodeID(), 10), "dml", metrics.UnissuedIndexTaskLabel).Set(float64(unissuedTasksNum))
metrics.ProxyQueueTaskNum.WithLabelValues(strconv.FormatInt(paramtable.GetNodeID(), 10), "dml", metrics.InProgressIndexTaskLabel).Set(float64(activateTaskNum))
}
func (queue *dmTaskQueue) Enqueue(t task) error { func (queue *dmTaskQueue) Enqueue(t task) error {
// This statsLock has two functions: // This statsLock has two functions:
// 1) Protect member pChanStatisticsInfos // 1) Protect member pChanStatisticsInfos
@ -361,6 +385,18 @@ type dqTaskQueue struct {
*baseTaskQueue *baseTaskQueue
} }
func (queue *dqTaskQueue) updateMetrics() {
queue.utLock.RLock()
unissuedTasksNum := queue.unissuedTasks.Len()
queue.utLock.RUnlock()
queue.atLock.RLock()
activateTaskNum := len(queue.activeTasks)
queue.atLock.RUnlock()
metrics.ProxyQueueTaskNum.WithLabelValues(strconv.FormatInt(paramtable.GetNodeID(), 10), "dql", metrics.UnissuedIndexTaskLabel).Set(float64(unissuedTasksNum))
metrics.ProxyQueueTaskNum.WithLabelValues(strconv.FormatInt(paramtable.GetNodeID(), 10), "dql", metrics.InProgressIndexTaskLabel).Set(float64(activateTaskNum))
}
func (queue *ddTaskQueue) Enqueue(t task) error { func (queue *ddTaskQueue) Enqueue(t task) error {
queue.lock.Lock() queue.lock.Lock()
defer queue.lock.Unlock() defer queue.lock.Unlock()
@ -507,6 +543,7 @@ func (sched *taskScheduler) definitionLoop() {
return struct{}{}, nil return struct{}{}, nil
}) })
} }
sched.ddQueue.updateMetrics()
} }
} }
} }
@ -528,6 +565,7 @@ func (sched *taskScheduler) controlLoop() {
return struct{}{}, nil return struct{}{}, nil
}) })
} }
sched.dcQueue.updateMetrics()
} }
} }
} }
@ -547,6 +585,7 @@ func (sched *taskScheduler) manipulationLoop() {
return struct{}{}, nil return struct{}{}, nil
}) })
} }
sched.dmQueue.updateMetrics()
} }
} }
} }
@ -577,6 +616,7 @@ func (sched *taskScheduler) queryLoop() {
} else { } else {
log.Ctx(context.TODO()).Debug("query queue is empty ...") log.Ctx(context.TODO()).Debug("query queue is empty ...")
} }
sched.dqQueue.updateMetrics()
} }
} }
} }

1
pkg/metrics/metrics.go
View File

@ -124,6 +124,7 @@ const (
pathLabelName = "path" pathLabelName = "path"
cgoNameLabelName = `cgo_name` cgoNameLabelName = `cgo_name`
cgoTypeLabelName = `cgo_type` cgoTypeLabelName = `cgo_type`
queueTypeLabelName = `queue_type`
// entities label // entities label
LoadedLabel = "loaded" LoadedLabel = "loaded"

10
pkg/metrics/proxy_metrics.go
View File

@ -427,6 +427,15 @@ var (
Help: "the number of non-zeros in each sparse search task", Help: "the number of non-zeros in each sparse search task",
Buckets: buckets, Buckets: buckets,
}, []string{nodeIDLabelName, collectionName}) }, []string{nodeIDLabelName, collectionName})
// ProxyQueueTaskNum records task number of queue in Proxy.
ProxyQueueTaskNum = prometheus.NewGaugeVec(
prometheus.GaugeOpts{
Namespace: milvusNamespace,
Subsystem: typeutil.ProxyRole,
Name: "queue_task_num",
Help: "",
}, []string{nodeIDLabelName, queueTypeLabelName, taskStateLabel})
) )
// RegisterProxy registers Proxy metrics // RegisterProxy registers Proxy metrics
@ -490,6 +499,7 @@ func RegisterProxy(registry *prometheus.Registry) {
registry.MustRegister(ProxyRecallSearchCount) registry.MustRegister(ProxyRecallSearchCount)
registry.MustRegister(ProxySearchSparseNumNonZeros) registry.MustRegister(ProxySearchSparseNumNonZeros)
registry.MustRegister(ProxyQueueTaskNum)
RegisterStreamingServiceClient(registry) RegisterStreamingServiceClient(registry)
} }