diff --git a/internal/distributed/proxy/service.go b/internal/distributed/proxy/service.go index e132fa6fa1..445d0dc836 100644 --- a/internal/distributed/proxy/service.go +++ b/internal/distributed/proxy/service.go @@ -66,6 +66,7 @@ import ( "github.com/milvus-io/milvus/internal/util/dependency" _ "github.com/milvus-io/milvus/internal/util/grpcclient" "github.com/milvus-io/milvus/pkg/log" + "github.com/milvus-io/milvus/pkg/metrics" "github.com/milvus-io/milvus/pkg/tracer" "github.com/milvus-io/milvus/pkg/util" "github.com/milvus-io/milvus/pkg/util/etcd" @@ -172,8 +173,31 @@ func (s *Server) registerHTTPServer() { func (s *Server) startHTTPServer(errChan chan error) { defer s.wg.Done() ginHandler := gin.New() - ginHandler.Use(accesslog.AccessLogMiddleware) + ginHandler.Use(func(c *gin.Context) { + path := c.Request.URL.Path + metrics.RestfulFunctionCall.WithLabelValues( + strconv.FormatInt(paramtable.GetNodeID(), 10), path, + ).Inc() + if c.Request.ContentLength >= 0 { + metrics.RestfulReceiveBytes.WithLabelValues( + strconv.FormatInt(paramtable.GetNodeID(), 10), path, + ).Add(float64(c.Request.ContentLength)) + } + start := time.Now() + // Process request + c.Next() + + latency := time.Now().Sub(start) + metrics.RestfulReqLatency.WithLabelValues( + strconv.FormatInt(paramtable.GetNodeID(), 10), path, + ).Observe(float64(latency.Milliseconds())) + metrics.RestfulSendBytes.WithLabelValues( + strconv.FormatInt(paramtable.GetNodeID(), 10), path, + ).Add(float64(c.Writer.Size())) + }) + + ginHandler.Use(accesslog.AccessLogMiddleware) ginLogger := gin.LoggerWithConfig(gin.LoggerConfig{ SkipPaths: proxy.Params.ProxyCfg.GinLogSkipPaths.GetAsStrings(), Formatter: func(param gin.LogFormatterParams) string { diff --git a/pkg/metrics/metrics.go b/pkg/metrics/metrics.go index 0183c446da..23c847c5a5 100644 --- a/pkg/metrics/metrics.go +++ b/pkg/metrics/metrics.go @@ -110,6 +110,7 @@ const ( lockType = "lock_type" lockOp = "lock_op" loadTypeName = "load_type" + pathLabelName = "path" // entities label LoadedLabel = "loaded" diff --git a/pkg/metrics/proxy_metrics.go b/pkg/metrics/proxy_metrics.go index 43fbd49c83..c22bbed15d 100644 --- a/pkg/metrics/proxy_metrics.go +++ b/pkg/metrics/proxy_metrics.go @@ -263,6 +263,43 @@ var ( Help: "count of bytes sent back to sdk", }, []string{nodeIDLabelName}) + // RestfulFunctionCall records the number of times the restful apis was called. + RestfulFunctionCall = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: milvusNamespace, + Subsystem: typeutil.ProxyRole, + Name: "restful_api_req_count", + Help: "count of operation executed", + }, []string{nodeIDLabelName, pathLabelName}) + + // RestfulReqLatency records the latency that for all requests. + RestfulReqLatency = prometheus.NewHistogramVec( + prometheus.HistogramOpts{ + Namespace: milvusNamespace, + Subsystem: typeutil.ProxyRole, + Name: "restful_api_req_latency", + Help: "latency of each request", + Buckets: buckets, // unit: ms + }, []string{nodeIDLabelName, pathLabelName}) + + // RestfulReceiveBytes record the received bytes of messages in Proxy + RestfulReceiveBytes = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: milvusNamespace, + Subsystem: typeutil.ProxyRole, + Name: "restful_api_receive_bytes_count", + Help: "count of bytes received from sdk", + }, []string{nodeIDLabelName, pathLabelName}) + + // RestfulSendBytes record the bytes sent back to client by Proxy + RestfulSendBytes = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: milvusNamespace, + Subsystem: typeutil.ProxyRole, + Name: "restful_api_send_bytes_count", + Help: "count of bytes sent back to sdk", + }, []string{nodeIDLabelName, pathLabelName}) + // ProxyReportValue records value about the request ProxyReportValue = prometheus.NewCounterVec( prometheus.CounterOpts{ @@ -383,6 +420,12 @@ func RegisterProxy(registry *prometheus.Registry) { registry.MustRegister(ProxyReceiveBytes) registry.MustRegister(ProxyReadReqSendBytes) + registry.MustRegister(RestfulFunctionCall) + registry.MustRegister(RestfulReqLatency) + + registry.MustRegister(RestfulReceiveBytes) + registry.MustRegister(RestfulSendBytes) + registry.MustRegister(ProxyLimiterRate) registry.MustRegister(ProxyHookFunc) registry.MustRegister(UserRPCCounter)