milvus-io
/
milvus
mirror of https://github.com/milvus-io/milvus.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

enhance: add metric to record maxInsertRate and queryNodeMemoryHighWaterLevel (#35185) 

fix: https://github.com/milvus-io/milvus/issues/35177

---------

Signed-off-by: longjiquan <jiquan.long@zilliz.com>
pull/34491/head
Jiquan Long 2024-08-02 10:14:13 +08:00 committed by GitHub
parent 196a7986b3
commit 4dbd023402
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
4 changed files with 46 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
internal/rootcoord/quota_center.go
View File

@ -41,10 +41,12 @@ import (
"github.com/milvus-io/milvus/internal/util/quota"
rlinternal "github.com/milvus-io/milvus/internal/util/ratelimitutil"
"github.com/milvus-io/milvus/pkg/common"
"github.com/milvus-io/milvus/pkg/config"
"github.com/milvus-io/milvus/pkg/log"
"github.com/milvus-io/milvus/pkg/metrics"
"github.com/milvus-io/milvus/pkg/util/commonpbutil"
"github.com/milvus-io/milvus/pkg/util/metricsinfo"
"github.com/milvus-io/milvus/pkg/util/paramtable"
"github.com/milvus-io/milvus/pkg/util/ratelimitutil"
"github.com/milvus-io/milvus/pkg/util/tsoutil"
"github.com/milvus-io/milvus/pkg/util/typeutil"
@ -276,10 +278,18 @@ func (q *QuotaCenter) Start() {
}()
}
func (q *QuotaCenter) watchQuotaAndLimit() {
pt := paramtable.Get()
pt.Watch(pt.QuotaConfig.QueryNodeMemoryHighWaterLevel.Key, config.NewHandler(pt.QuotaConfig.QueryNodeMemoryHighWaterLevel.Key, func(event *config.Event) {
metrics.QueryNodeMemoryHighWaterLevel.Set(pt.QuotaConfig.QueryNodeMemoryHighWaterLevel.GetAsFloat())
}))
}
// run starts the service of QuotaCenter.
func (q *QuotaCenter) run() {
interval := Params.QuotaConfig.QuotaCenterCollectInterval.GetAsDuration(time.Second)
log.Info("Start QuotaCenter", zap.Duration("collectInterval", interval))
q.watchQuotaAndLimit()
ticker := time.NewTicker(interval)
defer ticker.Stop()
for {

16
internal/util/quota/quota_constant.go
View File

@ -25,7 +25,9 @@ import (
"go.uber.org/zap"
"github.com/milvus-io/milvus/internal/proto/internalpb"
"github.com/milvus-io/milvus/pkg/config"
"github.com/milvus-io/milvus/pkg/log"
"github.com/milvus-io/milvus/pkg/metrics"
"github.com/milvus-io/milvus/pkg/util/paramtable"
)
@ -82,6 +84,20 @@ func initLimitConfigMaps() {
internalpb.RateType_DQLQuery: &quotaConfig.DQLMaxQueryRatePerPartition,
},
}
pt := paramtable.Get()
pt.Watch(quotaConfig.DMLMaxInsertRate.Key, config.NewHandler(quotaConfig.DMLMaxInsertRate.Key, func(event *config.Event) {
metrics.MaxInsertRate.WithLabelValues(paramtable.GetStringNodeID(), "cluster").Set(quotaConfig.DMLMaxInsertRate.GetAsFloat())
}))
pt.Watch(quotaConfig.DMLMaxInsertRatePerDB.Key, config.NewHandler(quotaConfig.DMLMaxInsertRatePerDB.Key, func(event *config.Event) {
metrics.MaxInsertRate.WithLabelValues(paramtable.GetStringNodeID(), "db").Set(quotaConfig.DMLMaxInsertRatePerDB.GetAsFloat())
}))
pt.Watch(quotaConfig.DMLMaxInsertRatePerCollection.Key, config.NewHandler(quotaConfig.DMLMaxInsertRatePerCollection.Key, func(event *config.Event) {
metrics.MaxInsertRate.WithLabelValues(paramtable.GetStringNodeID(), "collection").Set(quotaConfig.DMLMaxInsertRatePerCollection.GetAsFloat())
}))
pt.Watch(quotaConfig.DMLMaxInsertRatePerPartition.Key, config.NewHandler(quotaConfig.DMLMaxInsertRatePerPartition.Key, func(event *config.Event) {
metrics.MaxInsertRate.WithLabelValues(paramtable.GetStringNodeID(), "partition").Set(quotaConfig.DMLMaxInsertRatePerPartition.GetAsFloat())
}))
})
}

10
pkg/metrics/proxy_metrics.go
View File

@ -381,6 +381,14 @@ var (
Help: "latency which request waits in the queue",
Buckets: buckets, // unit: ms
}, []string{nodeIDLabelName, functionLabelName})
MaxInsertRate = prometheus.NewGaugeVec(
prometheus.GaugeOpts{
Namespace: milvusNamespace,
Subsystem: typeutil.ProxyRole,
Name: "max_insert_rate",
Help: "max insert rate",
}, []string{"node_id", "scope"})
)
// RegisterProxy registers Proxy metrics
@ -437,6 +445,8 @@ func RegisterProxy(registry *prometheus.Registry) {
registry.MustRegister(ProxySlowQueryCount)
registry.MustRegister(ProxyReportValue)
registry.MustRegister(ProxyReqInQueueLatency)
registry.MustRegister(MaxInsertRate)
}
func CleanupProxyDBMetrics(nodeID int64, dbName string) {

10
pkg/metrics/rootcoord_metrics.go
View File

@ -218,6 +218,14 @@ var (
indexName,
isVectorIndex,
})
QueryNodeMemoryHighWaterLevel = prometheus.NewGauge(
prometheus.GaugeOpts{
Namespace: milvusNamespace,
Subsystem: typeutil.RootCoordRole,
Name: "qn_mem_high_water_level",
Help: "querynode memory high water level",
})
)
// RegisterRootCoord registers RootCoord metrics
@ -256,6 +264,8 @@ func RegisterRootCoord(registry *prometheus.Registry) {
registry.MustRegister(RootCoordNumEntities)
registry.MustRegister(RootCoordIndexedNumEntities)
registry.MustRegister(QueryNodeMemoryHighWaterLevel)
}
func CleanupRootCoordDBMetrics(dbName string) {