Add prometheus metrics (#5342)

Add prometheus metrics for components

Resolves: #5325

Signed-off-by: Weida Zhu <weida.zhu@zilliz.com>

cmd/distributed/roles/roles.go

@@ -25,6 +25,7 @@ import (
 	"github.com/milvus-io/milvus/internal/indexservice"
 	"github.com/milvus-io/milvus/internal/log"
 	"github.com/milvus-io/milvus/internal/masterservice"
+	"github.com/milvus-io/milvus/internal/metrics"
 	"github.com/milvus-io/milvus/internal/proxynode"
 	"github.com/milvus-io/milvus/internal/proxyservice"
 	"github.com/milvus-io/milvus/internal/querynode"
@@ -93,6 +94,8 @@ func (mr *MilvusRoles) Run(localMsg bool) {
 		if ms != nil {
 			defer ms.Stop()
 		}
+
+		metrics.RegisterMaster()
 	}
 
 	if mr.EnableProxyService {
@@ -115,6 +118,8 @@ func (mr *MilvusRoles) Run(localMsg bool) {
 		if ps != nil {
 			defer ps.Stop()
 		}
+
+		metrics.RegisterProxyService()
 	}
 
 	if mr.EnableProxyNode {
@@ -137,6 +142,8 @@ func (mr *MilvusRoles) Run(localMsg bool) {
 		if pn != nil {
 			defer pn.Stop()
 		}
+
+		metrics.RegisterProxyNode()
 	}
 
 	if mr.EnableQueryService {
@@ -159,6 +166,8 @@ func (mr *MilvusRoles) Run(localMsg bool) {
 		if qs != nil {
 			defer qs.Stop()
 		}
+
+		metrics.RegisterQueryService()
 	}
 
 	if mr.EnableQueryNode {
@@ -181,6 +190,8 @@ func (mr *MilvusRoles) Run(localMsg bool) {
 		if qn != nil {
 			defer qn.Stop()
 		}
+
+		metrics.RegisterQueryNode()
 	}
 
 	if mr.EnableDataService {
@@ -203,6 +214,8 @@ func (mr *MilvusRoles) Run(localMsg bool) {
 		if ds != nil {
 			defer ds.Stop()
 		}
+
+		metrics.RegisterDataService()
 	}
 
 	if mr.EnableDataNode {
@@ -225,6 +238,8 @@ func (mr *MilvusRoles) Run(localMsg bool) {
 		if dn != nil {
 			defer dn.Stop()
 		}
+
+		metrics.RegisterDataNode()
 	}
 
 	if mr.EnableIndexService {
@@ -246,6 +261,8 @@ func (mr *MilvusRoles) Run(localMsg bool) {
 		if is != nil {
 			defer is.Stop()
 		}
+
+		metrics.RegisterIndexService()
 	}
 
 	if mr.EnableIndexNode {
@@ -267,6 +284,8 @@ func (mr *MilvusRoles) Run(localMsg bool) {
 		if in != nil {
 			in.Stop()
 		}
+
+		metrics.RegisterIndexNode()
 	}
 
 	if mr.EnableMsgStreamService {
@@ -284,8 +303,12 @@ func (mr *MilvusRoles) Run(localMsg bool) {
 		if mss != nil {
 			defer mss.Stop()
 		}
+
+		metrics.RegisterMsgStreamService()
 	}
 
+	metrics.ServeHTTP()
+
 	sc := make(chan os.Signal, 1)
 	signal.Notify(sc,
 		syscall.SIGHUP,

go.mod

@@ -23,6 +23,7 @@ require (
 	github.com/opentracing/opentracing-go v1.2.0
 	github.com/pierrec/lz4 v2.5.2+incompatible // indirect
 	github.com/pkg/errors v0.9.1
+	github.com/prometheus/client_golang v1.7.1
 	github.com/quasilyte/go-ruleguard v0.2.1 // indirect
 	github.com/sirupsen/logrus v1.6.0 // indirect
 	github.com/spaolacci/murmur3 v1.1.0

internal/metrics/metrics.go

@@ -0,0 +1,82 @@
+package metrics
+
+import (
+	"net/http"
+
+	"github.com/milvus-io/milvus/internal/log"
+	"github.com/prometheus/client_golang/prometheus/promhttp"
+
+	"go.uber.org/zap"
+)
+
+/*
+var (
+	PanicCounter = prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Namespace: "milvus",
+			Subsystem: "server",
+			Name:      "panic_total",
+			Help:      "Counter of panic.",
+		}, []string{"type"})
+)
+*/
+
+//RegisterMaster register Master metrics
+func RegisterMaster() {
+	//prometheus.MustRegister(PanicCounter)
+}
+
+//RegisterProxyService register ProxyService metrics
+func RegisterProxyService() {
+
+}
+
+//RegisterProxyNode register ProxyNode metrics
+func RegisterProxyNode() {
+
+}
+
+//RegisterQueryService register QueryService metrics
+func RegisterQueryService() {
+
+}
+
+//RegisterQueryNode register QueryNode metrics
+func RegisterQueryNode() {
+
+}
+
+//RegisterDataService register DataService metrics
+func RegisterDataService() {
+
+}
+
+//RegisterDataNode register DataNode metrics
+func RegisterDataNode() {
+
+}
+
+//RegisterIndexService register IndexService metrics
+func RegisterIndexService() {
+
+}
+
+//RegisterIndexNode register IndexNode metrics
+func RegisterIndexNode() {
+
+}
+
+//RegisterMsgStreamService register MsgStreamService metrics
+func RegisterMsgStreamService() {
+
+}
+
+//ServeHTTP serve prometheus http service
+func ServeHTTP() {
+	http.Handle("/metrics", promhttp.Handler())
+	go func() {
+		if err := http.ListenAndServe(":9091", nil); err != nil {
+			log.Error("handle metrics failed", zap.Error(err))
+		}
+	}()
+}

internal/metrics/metrics_test.go

@@ -0,0 +1,19 @@
+package metrics
+
+import (
+	"testing"
+)
+
+func TestRegisterMetrics(t *testing.T) {
+	// Make sure it doesn't panic.
+	RegisterMaster()
+	RegisterDataNode()
+	RegisterDataService()
+	RegisterIndexNode()
+	RegisterIndexService()
+	RegisterProxyNode()
+	RegisterProxyService()
+	RegisterQueryNode()
+	RegisterQueryService()
+	RegisterMsgStreamService()
+}