milvus-io
/
milvus
mirror of https://github.com/milvus-io/milvus.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

enhance: Add a counter monitoring for the rate-limit requests (#30109) 

Add a counter monitoring metric for the ratelimited rpc requests with
labels: proxy nodeID, rpc request type, and state.

issue: https://github.com/milvus-io/milvus/issues/30052

Signed-off-by: bigsheeper <yihao.dai@zilliz.com>
pull/30164/head
yihao.dai 2024-01-21 14:44:59 +08:00 committed by GitHub
parent 4f44942c80
commit c8a78465b4
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
2 changed files with 17 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
internal/proxy/rate_limit_interceptor.go
View File

@ -20,6 +20,7 @@ import (
"context"
"fmt"
"reflect"
"strconv"
"github.com/golang/protobuf/proto"
"google.golang.org/grpc"
@ -27,7 +28,9 @@ import (
"github.com/milvus-io/milvus-proto/go-api/v2/milvuspb"
"github.com/milvus-io/milvus/internal/proto/internalpb"
"github.com/milvus-io/milvus/internal/types"
"github.com/milvus-io/milvus/pkg/metrics"
"github.com/milvus-io/milvus/pkg/util/merr"
"github.com/milvus-io/milvus/pkg/util/paramtable"
)
// RateLimitInterceptor returns a new unary server interceptors that performs request rate limiting.
@ -39,12 +42,16 @@ func RateLimitInterceptor(limiter types.Limiter) grpc.UnaryServerInterceptor {
}
err = limiter.Check(collectionID, rt, n)
nodeID := strconv.FormatInt(paramtable.GetNodeID(), 10)
metrics.ProxyRateLimitReqCount.WithLabelValues(nodeID, rt.String(), metrics.TotalLabel).Inc()
if err != nil {
metrics.ProxyRateLimitReqCount.WithLabelValues(nodeID, rt.String(), metrics.FailLabel).Inc()
rsp := getFailedResponse(req, err)
if rsp != nil {
return rsp, nil
}
}
metrics.ProxyRateLimitReqCount.WithLabelValues(nodeID, rt.String(), metrics.SuccessLabel).Inc()
return handler(ctx, req)
}
}

10
pkg/metrics/proxy_metrics.go
View File

@ -297,6 +297,15 @@ var (
}, []string{
nodeIDLabelName,
})
// ProxyRateLimitReqCount integrates a counter monitoring metric for the rate-limit rpc requests.
ProxyRateLimitReqCount = prometheus.NewCounterVec(
prometheus.CounterOpts{
Namespace: milvusNamespace,
Subsystem: typeutil.ProxyRole,
Name: "rate_limit_req_count",
Help: "count of operation executed",
}, []string{nodeIDLabelName, msgTypeLabelName, statusLabelName})
)
// RegisterProxy registers Proxy metrics
@ -341,6 +350,7 @@ func RegisterProxy(registry *prometheus.Registry) {
registry.MustRegister(ProxyWorkLoadScore)
registry.MustRegister(ProxyExecutingTotalNq)
registry.MustRegister(ProxyRateLimitReqCount)
}
func CleanupCollectionMetrics(nodeID int64, collection string) {