enhance: Add metrics for querycoord current target cp lag (#31391)

See also #31390

Signed-off-by: Congqi Xia <congqi.xia@zilliz.com>

internal/querycoordv2/meta/target_manager.go

@@ -18,6 +18,7 @@ package meta
 
 import (
 	"context"
+	"fmt"
 	"sync"
 
 	"github.com/cockroachdb/errors"
@@ -29,8 +30,11 @@ import (
 	"github.com/milvus-io/milvus/internal/proto/datapb"
 	"github.com/milvus-io/milvus/pkg/common"
 	"github.com/milvus-io/milvus/pkg/log"
+	"github.com/milvus-io/milvus/pkg/metrics"
 	"github.com/milvus-io/milvus/pkg/util/merr"
+	"github.com/milvus-io/milvus/pkg/util/paramtable"
 	"github.com/milvus-io/milvus/pkg/util/retry"
+	"github.com/milvus-io/milvus/pkg/util/tsoutil"
 	"github.com/milvus-io/milvus/pkg/util/typeutil"
 )
 
@@ -87,6 +91,13 @@ func (mgr *TargetManager) UpdateCollectionCurrentTarget(collectionID int64) bool
 		zap.Strings("channels", newTarget.GetAllDmChannelNames()),
 		zap.Int64("version", newTarget.GetTargetVersion()),
 	)
+	for channelName, dmlChannel := range newTarget.dmChannels {
+		ts, _ := tsoutil.ParseTS(dmlChannel.GetSeekPosition().GetTimestamp())
+		metrics.QueryCoordCurrentTargetCheckpointUnixSeconds.WithLabelValues(
+			fmt.Sprint(paramtable.GetNodeID()),
+			channelName,
+		).Set(float64(ts.Unix()))
+	}
 	return true
 }
 

pkg/metrics/querycoord_metrics.go

@@ -104,6 +104,17 @@ var (
 			Name:      "querynode_num",
 			Help:      "number of QueryNodes managered by QueryCoord",
 		}, []string{})
+
+	QueryCoordCurrentTargetCheckpointUnixSeconds = prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Namespace: milvusNamespace,
+			Subsystem: typeutil.QueryCoordRole,
+			Name:      "current_target_checkpoint_unix_seconds",
+			Help:      "current target checkpoint timestamp in unix seconds",
+		}, []string{
+			nodeIDLabelName,
+			channelNameLabelName,
+		})
 )
 
 // RegisterQueryCoord registers QueryCoord metrics
@@ -116,4 +127,5 @@ func RegisterQueryCoord(registry *prometheus.Registry) {
 	registry.MustRegister(QueryCoordReleaseLatency)
 	registry.MustRegister(QueryCoordTaskNum)
 	registry.MustRegister(QueryCoordNumQueryNodes)
+	registry.MustRegister(QueryCoordCurrentTargetCheckpointUnixSeconds)
 }