fix: some missing metrics for WebUI restful API (#37747)

issue: #36621

Signed-off-by: jaime <yun.zhang@zilliz.com>

internal/proxy/http_req_impl.go

@@ -28,6 +28,7 @@ import (
 	"github.com/milvus-io/milvus-proto/go-api/v2/milvuspb"
 	mhttp "github.com/milvus-io/milvus/internal/http"
 	"github.com/milvus-io/milvus/internal/json"
+	"github.com/milvus-io/milvus/internal/proto/querypb"
 	"github.com/milvus-io/milvus/internal/proxy/connection"
 	"github.com/milvus-io/milvus/internal/types"
 	"github.com/milvus-io/milvus/internal/util/dependency"
@@ -45,6 +46,7 @@ var (
 	defaultDB          = "default"
 	httpDBName         = "db_name"
 	HTTPCollectionName = "collection_name"
+	UnknownData        = "unknown"
 )
 
 func getConfigs(configs map[string]string) gin.HandlerFunc {
@@ -208,39 +210,71 @@ func getDataComponentMetrics(node *Proxy, metricsType string) gin.HandlerFunc {
 
 // The Get request should be used to get the query parameters, not the body, such as Javascript
 // fetch API only support GET request with query parameter.
-func listCollection(node types.ProxyComponent) gin.HandlerFunc {
+func listCollection(rootCoord types.RootCoordClient, queryCoord types.QueryCoordClient) gin.HandlerFunc {
 	return func(c *gin.Context) {
 		dbName := c.Query(httpDBName)
 		if len(dbName) == 0 {
 			dbName = defaultDB
 		}
 
-		showCollectionResp, err := node.ShowCollections(c, &milvuspb.ShowCollectionsRequest{
+		rootCollectionListResp, err := rootCoord.ShowCollections(c, &milvuspb.ShowCollectionsRequest{
 			Base: &commonpb.MsgBase{
 				MsgType: commonpb.MsgType_ShowCollections,
 			},
 			DbName: dbName,
 		})
-		if err := merr.CheckRPCCall(showCollectionResp, err); err != nil {
+
+		if err := merr.CheckRPCCall(rootCollectionListResp, err); err != nil {
 			c.AbortWithStatusJSON(http.StatusInternalServerError, gin.H{
 				mhttp.HTTPReturnMessage: err.Error(),
 			})
 			return
 		}
 
+		queryCollectionListResp, err := queryCoord.ShowCollections(c, &querypb.ShowCollectionsRequest{
+			Base: &commonpb.MsgBase{
+				MsgType: commonpb.MsgType_ShowCollections,
+			},
+		})
+
+		if err := merr.CheckRPCCall(queryCollectionListResp, err); err != nil {
+			c.AbortWithStatusJSON(http.StatusInternalServerError, gin.H{
+				mhttp.HTTPReturnMessage: err.Error(),
+			})
+			return
+		}
+
+		collectionID2Offset := make(map[int64]int, len(queryCollectionListResp.CollectionIDs))
+		for collectionID, offset := range queryCollectionListResp.CollectionIDs {
+			collectionID2Offset[offset] = collectionID
+		}
+
 		// Convert the response to Collections struct
 		collections := &metricsinfo.Collections{
-			CollectionIDs: lo.Map(showCollectionResp.CollectionIds, func(t int64, i int) string {
+			CollectionIDs: lo.Map(rootCollectionListResp.CollectionIds, func(t int64, i int) string {
 				return strconv.FormatInt(t, 10)
 			}),
-			CollectionNames: showCollectionResp.CollectionNames,
-			CreatedUtcTimestamps: lo.Map(showCollectionResp.CreatedUtcTimestamps, func(t uint64, i int) string {
+			CollectionNames: rootCollectionListResp.CollectionNames,
+			CreatedUtcTimestamps: lo.Map(rootCollectionListResp.CreatedUtcTimestamps, func(t uint64, i int) string {
 				return typeutil.TimestampToString(t)
 			}),
-			InMemoryPercentages: lo.Map(showCollectionResp.InMemoryPercentages, func(t int64, i int) int {
-				return int(t)
+			InMemoryPercentages: lo.Map(rootCollectionListResp.CollectionIds, func(collectionID int64, i int) string {
+				offset, ok := collectionID2Offset[collectionID]
+				if !ok {
+					return UnknownData
+				}
+
+				loadPercentage := queryCollectionListResp.InMemoryPercentages[offset]
+				return strconv.FormatInt(loadPercentage, 10)
+			}),
+			QueryServiceAvailable: lo.Map(rootCollectionListResp.CollectionIds, func(collectionID int64, i int) bool {
+				offset, ok := collectionID2Offset[collectionID]
+				if !ok {
+					return false
+				}
+
+				return queryCollectionListResp.QueryServiceAvailable[offset]
 			}),
-			QueryServiceAvailable: showCollectionResp.QueryServiceAvailable,
 		}
 
 		// Marshal the collections struct to JSON

internal/proxy/http_req_impl_test.go

@@ -15,6 +15,7 @@ import (
 	"github.com/milvus-io/milvus-proto/go-api/v2/milvuspb"
 	"github.com/milvus-io/milvus-proto/go-api/v2/schemapb"
 	"github.com/milvus-io/milvus/internal/mocks"
+	"github.com/milvus-io/milvus/internal/proto/querypb"
 	"github.com/milvus-io/milvus/internal/proxy/connection"
 	"github.com/milvus-io/milvus/pkg/util/metricsinfo"
 	"github.com/milvus-io/milvus/pkg/util/paramtable"
@@ -159,8 +160,8 @@ func TestListCollection(t *testing.T) {
 		c, _ := gin.CreateTestContext(w)
 		c.Request, _ = http.NewRequest("GET", "/?db_name=default", nil)
 
-		mockProxy := mocks.NewMockProxy(t)
-		mockProxy.EXPECT().ShowCollections(mock.Anything, mock.Anything).Return(&milvuspb.ShowCollectionsResponse{
+		mockRoortCoordClient := mocks.NewMockRootCoordClient(t)
+		mockRoortCoordClient.EXPECT().ShowCollections(mock.Anything, mock.Anything).Return(&milvuspb.ShowCollectionsResponse{
 			Status:                &commonpb.Status{ErrorCode: commonpb.ErrorCode_Success},
 			CollectionIds:         []int64{1, 2},
 			CollectionNames:       []string{"collection1", "collection2"},
@@ -169,7 +170,15 @@ func TestListCollection(t *testing.T) {
 			QueryServiceAvailable: []bool{true, true},
 		}, nil)
 
-		handler := listCollection(mockProxy)
+		mockQueryCoordClient := mocks.NewMockQueryCoordClient(t)
+		mockQueryCoordClient.EXPECT().ShowCollections(mock.Anything, mock.Anything).Return(&querypb.ShowCollectionsResponse{
+			Status:                &commonpb.Status{ErrorCode: commonpb.ErrorCode_Success},
+			CollectionIDs:         []int64{1},
+			InMemoryPercentages:   []int64{100, 100},
+			QueryServiceAvailable: []bool{true, true},
+		}, nil)
+
+		handler := listCollection(mockRoortCoordClient, mockQueryCoordClient)
 		handler(c)
 
 		assert.Equal(t, http.StatusOK, w.Code)
@@ -177,17 +186,34 @@ func TestListCollection(t *testing.T) {
 		assert.Contains(t, w.Body.String(), "collection2")
 	})
 
-	t.Run("list collections with error", func(t *testing.T) {
+	t.Run("list collections with error in RC response", func(t *testing.T) {
 		w := httptest.NewRecorder()
 		c, _ := gin.CreateTestContext(w)
 		c.Request, _ = http.NewRequest("GET", "/?db_name=default", nil)
 
-		mockProxy := mocks.NewMockProxy(t)
-		mockProxy.EXPECT().ShowCollections(mock.Anything, mock.Anything).Return(nil, errors.New("error"))
+		mockRoortCoordClient := mocks.NewMockRootCoordClient(t)
+		mockRoortCoordClient.EXPECT().ShowCollections(mock.Anything, mock.Anything).Return(nil, errors.New("error"))
 
-		handler := listCollection(mockProxy)
+		handler := listCollection(mockRoortCoordClient, nil)
 		handler(c)
+		assert.Equal(t, http.StatusInternalServerError, w.Code)
+		assert.Contains(t, w.Body.String(), "error")
+	})
 
+	t.Run("list collections with error in QC response", func(t *testing.T) {
+		w := httptest.NewRecorder()
+		c, _ := gin.CreateTestContext(w)
+		c.Request, _ = http.NewRequest("GET", "/?db_name=default", nil)
+
+		mockRoortCoordClient := mocks.NewMockRootCoordClient(t)
+		mockRoortCoordClient.EXPECT().ShowCollections(mock.Anything, mock.Anything).Return(&milvuspb.ShowCollectionsResponse{
+			Status: &commonpb.Status{ErrorCode: commonpb.ErrorCode_Success},
+		}, nil)
+		mockQueryCoordClient := mocks.NewMockQueryCoordClient(t)
+		mockQueryCoordClient.EXPECT().ShowCollections(mock.Anything, mock.Anything).Return(nil, errors.New("error"))
+
+		handler := listCollection(mockRoortCoordClient, mockQueryCoordClient)
+		handler(c)
 		assert.Equal(t, http.StatusInternalServerError, w.Code)
 		assert.Contains(t, w.Body.String(), "error")
 	})

internal/proxy/impl.go

@@ -6540,7 +6540,7 @@ func (node *Proxy) RegisterRestRouter(router gin.IRouter) {
 	router.GET(http.DatabaseDescPath, describeDatabase(node))
 
 	// Collection requests
-	router.GET(http.CollectionListPath, listCollection(node))
+	router.GET(http.CollectionListPath, listCollection(node.rootCoord, node.queryCoord))
 	router.GET(http.CollectionDescPath, describeCollection(node, node.rootCoord))
 }
 

internal/rootcoord/list_db_task.go

@@ -112,15 +112,18 @@ func (t *listDatabaseTask) Execute(ctx context.Context) error {
 	}
 
 	dbNames := make([]string, 0, len(ret))
+	dbIDs := make([]int64, 0, len(ret))
 	createdTimes := make([]uint64, 0, len(ret))
 	for _, db := range ret {
 		if !isVisibleDBForCurUser(db.Name, visibleDBs) {
 			continue
 		}
 		dbNames = append(dbNames, db.Name)
+		dbIDs = append(dbIDs, db.ID)
 		createdTimes = append(createdTimes, db.CreatedTime)
 	}
 	t.Resp.DbNames = dbNames
+	t.Resp.DbIds = dbIDs
 	t.Resp.CreatedTimestamp = createdTimes
 	return nil
 }

pkg/util/metricsinfo/metrics_info.go

@@ -405,7 +405,7 @@ type Collections struct {
 	CollectionIDs        []string `json:"collection_ids,omitempty"`
 	CreatedUtcTimestamps []string `json:"created_utc_timestamps,omitempty"`
 	// Load percentage on querynode when type is InMemory
-	InMemoryPercentages []int `json:"inMemory_percentages,omitempty"`
+	InMemoryPercentages []string `json:"inMemory_percentages,omitempty"`
 	// Indicate whether query service is available
 	QueryServiceAvailable []bool `json:"query_service_available,omitempty"`
 }