Separate segments/shards reduce latency metrics (#25475)

Signed-off-by: Congqi Xia <congqi.xia@zilliz.com>

internal/querynodev2/services.go

@@ -882,7 +882,7 @@ func (node *QueryNode) Search(ctx context.Context, req *querypb.SearchRequest) (
 		failRet.Status.Reason = err.Error()
 		return failRet, nil
 	}
-	metrics.QueryNodeReduceLatency.WithLabelValues(fmt.Sprint(paramtable.GetNodeID()), metrics.SearchLabel).
+	metrics.QueryNodeReduceLatency.WithLabelValues(fmt.Sprint(paramtable.GetNodeID()), metrics.SearchLabel, metrics.ReduceShards).
 		Observe(float64(tr.ElapseSpan().Milliseconds()))
 
 	collector.Rate.Add(metricsinfo.NQPerSecond, float64(req.GetReq().GetNq()))
@@ -1036,7 +1036,7 @@ func (node *QueryNode) Query(ctx context.Context, req *querypb.QueryRequest) (*i
 	if err != nil {
 		return WrapRetrieveResult(commonpb.ErrorCode_UnexpectedError, "failed to query channel", err), nil
 	}
-	metrics.QueryNodeReduceLatency.WithLabelValues(fmt.Sprint(paramtable.GetNodeID()), metrics.QueryLabel).
+	metrics.QueryNodeReduceLatency.WithLabelValues(fmt.Sprint(paramtable.GetNodeID()), metrics.QueryLabel, metrics.ReduceShards).
 		Observe(float64(tr.ElapseSpan().Milliseconds()))
 
 	if !req.FromShardLeader {

internal/querynodev2/tasks/task.go

@@ -206,7 +206,8 @@ func (t *SearchTask) Execute() error {
 
 		metrics.QueryNodeReduceLatency.WithLabelValues(
 			fmt.Sprint(paramtable.GetNodeID()),
-			metrics.SearchLabel).
+			metrics.SearchLabel,
+			metrics.ReduceSegments).
 			Observe(float64(reduceRecord.ElapseSpan().Milliseconds()))
 
 		task.result = &internalpb.SearchResults{

pkg/metrics/metrics.go

@@ -61,6 +61,9 @@ const (
 	HookAfter  = "after"
 	HookMock   = "mock"
 
+	ReduceSegments = "segments"
+	ReduceShards   = "shards"
+
 	nodeIDLabelName          = "node_id"
 	statusLabelName          = "status"
 	indexTaskStatusLabelName = "index_task_status"
@@ -80,6 +83,7 @@ const (
 	indexCountLabelName      = "indexed_field_count"
 	requestScope             = "scope"
 	fullMethodLabelName      = "full_method"
+	reduceLevelName          = "reduce_level"
 )
 
 var (

pkg/metrics/querynode_metrics.go

@@ -202,6 +202,7 @@ var (
 		}, []string{
 			nodeIDLabelName,
 			queryTypeLabelName,
+			reduceLevelName,
 		})
 
 	QueryNodeLoadSegmentLatency = prometheus.NewHistogramVec(