enhance: add metrics for restful apis (#34969)

issue: #34968

Signed-off-by: PowderLi <min.li@zilliz.com>

internal/distributed/proxy/service.go

@@ -66,6 +66,7 @@ import (
 	"github.com/milvus-io/milvus/internal/util/dependency"
 	_ "github.com/milvus-io/milvus/internal/util/grpcclient"
 	"github.com/milvus-io/milvus/pkg/log"
+	"github.com/milvus-io/milvus/pkg/metrics"
 	"github.com/milvus-io/milvus/pkg/tracer"
 	"github.com/milvus-io/milvus/pkg/util"
 	"github.com/milvus-io/milvus/pkg/util/etcd"
@@ -172,8 +173,31 @@ func (s *Server) registerHTTPServer() {
 func (s *Server) startHTTPServer(errChan chan error) {
 	defer s.wg.Done()
 	ginHandler := gin.New()
-	ginHandler.Use(accesslog.AccessLogMiddleware)
+	ginHandler.Use(func(c *gin.Context) {
+		path := c.Request.URL.Path
+		metrics.RestfulFunctionCall.WithLabelValues(
+			strconv.FormatInt(paramtable.GetNodeID(), 10), path,
+		).Inc()
+		if c.Request.ContentLength >= 0 {
+			metrics.RestfulReceiveBytes.WithLabelValues(
+				strconv.FormatInt(paramtable.GetNodeID(), 10), path,
+			).Add(float64(c.Request.ContentLength))
+		}
+		start := time.Now()
 
+		// Process request
+		c.Next()
+
+		latency := time.Now().Sub(start)
+		metrics.RestfulReqLatency.WithLabelValues(
+			strconv.FormatInt(paramtable.GetNodeID(), 10), path,
+		).Observe(float64(latency.Milliseconds()))
+		metrics.RestfulSendBytes.WithLabelValues(
+			strconv.FormatInt(paramtable.GetNodeID(), 10), path,
+		).Add(float64(c.Writer.Size()))
+	})
+
+	ginHandler.Use(accesslog.AccessLogMiddleware)
 	ginLogger := gin.LoggerWithConfig(gin.LoggerConfig{
 		SkipPaths: proxy.Params.ProxyCfg.GinLogSkipPaths.GetAsStrings(),
 		Formatter: func(param gin.LogFormatterParams) string {

pkg/metrics/metrics.go

@@ -110,6 +110,7 @@ const (
 	lockType                 = "lock_type"
 	lockOp                   = "lock_op"
 	loadTypeName             = "load_type"
+	pathLabelName            = "path"
 
 	// entities label
 	LoadedLabel         = "loaded"

pkg/metrics/proxy_metrics.go

@@ -263,6 +263,43 @@ var (
 			Help:      "count of bytes sent back to sdk",
 		}, []string{nodeIDLabelName})
 
+	// RestfulFunctionCall records the number of times the restful apis was called.
+	RestfulFunctionCall = prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Namespace: milvusNamespace,
+			Subsystem: typeutil.ProxyRole,
+			Name:      "restful_api_req_count",
+			Help:      "count of operation executed",
+		}, []string{nodeIDLabelName, pathLabelName})
+
+	// RestfulReqLatency records the latency that for all requests.
+	RestfulReqLatency = prometheus.NewHistogramVec(
+		prometheus.HistogramOpts{
+			Namespace: milvusNamespace,
+			Subsystem: typeutil.ProxyRole,
+			Name:      "restful_api_req_latency",
+			Help:      "latency of each request",
+			Buckets:   buckets, // unit: ms
+		}, []string{nodeIDLabelName, pathLabelName})
+
+	// RestfulReceiveBytes record the received bytes of messages in Proxy
+	RestfulReceiveBytes = prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Namespace: milvusNamespace,
+			Subsystem: typeutil.ProxyRole,
+			Name:      "restful_api_receive_bytes_count",
+			Help:      "count of bytes received  from sdk",
+		}, []string{nodeIDLabelName, pathLabelName})
+
+	// RestfulSendBytes record the bytes sent back to client by Proxy
+	RestfulSendBytes = prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Namespace: milvusNamespace,
+			Subsystem: typeutil.ProxyRole,
+			Name:      "restful_api_send_bytes_count",
+			Help:      "count of bytes sent back to sdk",
+		}, []string{nodeIDLabelName, pathLabelName})
+
 	// ProxyReportValue records value about the request
 	ProxyReportValue = prometheus.NewCounterVec(
 		prometheus.CounterOpts{
@@ -383,6 +420,12 @@ func RegisterProxy(registry *prometheus.Registry) {
 	registry.MustRegister(ProxyReceiveBytes)
 	registry.MustRegister(ProxyReadReqSendBytes)
 
+	registry.MustRegister(RestfulFunctionCall)
+	registry.MustRegister(RestfulReqLatency)
+
+	registry.MustRegister(RestfulReceiveBytes)
+	registry.MustRegister(RestfulSendBytes)
+
 	registry.MustRegister(ProxyLimiterRate)
 	registry.MustRegister(ProxyHookFunc)
 	registry.MustRegister(UserRPCCounter)