diff --git a/internal/proxy/http_req_impl.go b/internal/proxy/http_req_impl.go index ea21a7464a..d986b9f061 100644 --- a/internal/proxy/http_req_impl.go +++ b/internal/proxy/http_req_impl.go @@ -28,6 +28,7 @@ import ( "github.com/milvus-io/milvus-proto/go-api/v2/milvuspb" mhttp "github.com/milvus-io/milvus/internal/http" "github.com/milvus-io/milvus/internal/json" + "github.com/milvus-io/milvus/internal/proto/querypb" "github.com/milvus-io/milvus/internal/proxy/connection" "github.com/milvus-io/milvus/internal/types" "github.com/milvus-io/milvus/internal/util/dependency" @@ -45,6 +46,7 @@ var ( defaultDB = "default" httpDBName = "db_name" HTTPCollectionName = "collection_name" + UnknownData = "unknown" ) func getConfigs(configs map[string]string) gin.HandlerFunc { @@ -208,39 +210,71 @@ func getDataComponentMetrics(node *Proxy, metricsType string) gin.HandlerFunc { // The Get request should be used to get the query parameters, not the body, such as Javascript // fetch API only support GET request with query parameter. -func listCollection(node types.ProxyComponent) gin.HandlerFunc { +func listCollection(rootCoord types.RootCoordClient, queryCoord types.QueryCoordClient) gin.HandlerFunc { return func(c *gin.Context) { dbName := c.Query(httpDBName) if len(dbName) == 0 { dbName = defaultDB } - showCollectionResp, err := node.ShowCollections(c, &milvuspb.ShowCollectionsRequest{ + rootCollectionListResp, err := rootCoord.ShowCollections(c, &milvuspb.ShowCollectionsRequest{ Base: &commonpb.MsgBase{ MsgType: commonpb.MsgType_ShowCollections, }, DbName: dbName, }) - if err := merr.CheckRPCCall(showCollectionResp, err); err != nil { + + if err := merr.CheckRPCCall(rootCollectionListResp, err); err != nil { c.AbortWithStatusJSON(http.StatusInternalServerError, gin.H{ mhttp.HTTPReturnMessage: err.Error(), }) return } + queryCollectionListResp, err := queryCoord.ShowCollections(c, &querypb.ShowCollectionsRequest{ + Base: &commonpb.MsgBase{ + MsgType: commonpb.MsgType_ShowCollections, + }, + }) + + if err := merr.CheckRPCCall(queryCollectionListResp, err); err != nil { + c.AbortWithStatusJSON(http.StatusInternalServerError, gin.H{ + mhttp.HTTPReturnMessage: err.Error(), + }) + return + } + + collectionID2Offset := make(map[int64]int, len(queryCollectionListResp.CollectionIDs)) + for collectionID, offset := range queryCollectionListResp.CollectionIDs { + collectionID2Offset[offset] = collectionID + } + // Convert the response to Collections struct collections := &metricsinfo.Collections{ - CollectionIDs: lo.Map(showCollectionResp.CollectionIds, func(t int64, i int) string { + CollectionIDs: lo.Map(rootCollectionListResp.CollectionIds, func(t int64, i int) string { return strconv.FormatInt(t, 10) }), - CollectionNames: showCollectionResp.CollectionNames, - CreatedUtcTimestamps: lo.Map(showCollectionResp.CreatedUtcTimestamps, func(t uint64, i int) string { + CollectionNames: rootCollectionListResp.CollectionNames, + CreatedUtcTimestamps: lo.Map(rootCollectionListResp.CreatedUtcTimestamps, func(t uint64, i int) string { return typeutil.TimestampToString(t) }), - InMemoryPercentages: lo.Map(showCollectionResp.InMemoryPercentages, func(t int64, i int) int { - return int(t) + InMemoryPercentages: lo.Map(rootCollectionListResp.CollectionIds, func(collectionID int64, i int) string { + offset, ok := collectionID2Offset[collectionID] + if !ok { + return UnknownData + } + + loadPercentage := queryCollectionListResp.InMemoryPercentages[offset] + return strconv.FormatInt(loadPercentage, 10) + }), + QueryServiceAvailable: lo.Map(rootCollectionListResp.CollectionIds, func(collectionID int64, i int) bool { + offset, ok := collectionID2Offset[collectionID] + if !ok { + return false + } + + return queryCollectionListResp.QueryServiceAvailable[offset] }), - QueryServiceAvailable: showCollectionResp.QueryServiceAvailable, } // Marshal the collections struct to JSON diff --git a/internal/proxy/http_req_impl_test.go b/internal/proxy/http_req_impl_test.go index 09b72b4239..531951b7ff 100644 --- a/internal/proxy/http_req_impl_test.go +++ b/internal/proxy/http_req_impl_test.go @@ -15,6 +15,7 @@ import ( "github.com/milvus-io/milvus-proto/go-api/v2/milvuspb" "github.com/milvus-io/milvus-proto/go-api/v2/schemapb" "github.com/milvus-io/milvus/internal/mocks" + "github.com/milvus-io/milvus/internal/proto/querypb" "github.com/milvus-io/milvus/internal/proxy/connection" "github.com/milvus-io/milvus/pkg/util/metricsinfo" "github.com/milvus-io/milvus/pkg/util/paramtable" @@ -159,8 +160,8 @@ func TestListCollection(t *testing.T) { c, _ := gin.CreateTestContext(w) c.Request, _ = http.NewRequest("GET", "/?db_name=default", nil) - mockProxy := mocks.NewMockProxy(t) - mockProxy.EXPECT().ShowCollections(mock.Anything, mock.Anything).Return(&milvuspb.ShowCollectionsResponse{ + mockRoortCoordClient := mocks.NewMockRootCoordClient(t) + mockRoortCoordClient.EXPECT().ShowCollections(mock.Anything, mock.Anything).Return(&milvuspb.ShowCollectionsResponse{ Status: &commonpb.Status{ErrorCode: commonpb.ErrorCode_Success}, CollectionIds: []int64{1, 2}, CollectionNames: []string{"collection1", "collection2"}, @@ -169,7 +170,15 @@ func TestListCollection(t *testing.T) { QueryServiceAvailable: []bool{true, true}, }, nil) - handler := listCollection(mockProxy) + mockQueryCoordClient := mocks.NewMockQueryCoordClient(t) + mockQueryCoordClient.EXPECT().ShowCollections(mock.Anything, mock.Anything).Return(&querypb.ShowCollectionsResponse{ + Status: &commonpb.Status{ErrorCode: commonpb.ErrorCode_Success}, + CollectionIDs: []int64{1}, + InMemoryPercentages: []int64{100, 100}, + QueryServiceAvailable: []bool{true, true}, + }, nil) + + handler := listCollection(mockRoortCoordClient, mockQueryCoordClient) handler(c) assert.Equal(t, http.StatusOK, w.Code) @@ -177,17 +186,34 @@ func TestListCollection(t *testing.T) { assert.Contains(t, w.Body.String(), "collection2") }) - t.Run("list collections with error", func(t *testing.T) { + t.Run("list collections with error in RC response", func(t *testing.T) { w := httptest.NewRecorder() c, _ := gin.CreateTestContext(w) c.Request, _ = http.NewRequest("GET", "/?db_name=default", nil) - mockProxy := mocks.NewMockProxy(t) - mockProxy.EXPECT().ShowCollections(mock.Anything, mock.Anything).Return(nil, errors.New("error")) + mockRoortCoordClient := mocks.NewMockRootCoordClient(t) + mockRoortCoordClient.EXPECT().ShowCollections(mock.Anything, mock.Anything).Return(nil, errors.New("error")) - handler := listCollection(mockProxy) + handler := listCollection(mockRoortCoordClient, nil) handler(c) + assert.Equal(t, http.StatusInternalServerError, w.Code) + assert.Contains(t, w.Body.String(), "error") + }) + t.Run("list collections with error in QC response", func(t *testing.T) { + w := httptest.NewRecorder() + c, _ := gin.CreateTestContext(w) + c.Request, _ = http.NewRequest("GET", "/?db_name=default", nil) + + mockRoortCoordClient := mocks.NewMockRootCoordClient(t) + mockRoortCoordClient.EXPECT().ShowCollections(mock.Anything, mock.Anything).Return(&milvuspb.ShowCollectionsResponse{ + Status: &commonpb.Status{ErrorCode: commonpb.ErrorCode_Success}, + }, nil) + mockQueryCoordClient := mocks.NewMockQueryCoordClient(t) + mockQueryCoordClient.EXPECT().ShowCollections(mock.Anything, mock.Anything).Return(nil, errors.New("error")) + + handler := listCollection(mockRoortCoordClient, mockQueryCoordClient) + handler(c) assert.Equal(t, http.StatusInternalServerError, w.Code) assert.Contains(t, w.Body.String(), "error") }) diff --git a/internal/proxy/impl.go b/internal/proxy/impl.go index 23c4489864..83626ed5b9 100644 --- a/internal/proxy/impl.go +++ b/internal/proxy/impl.go @@ -6540,7 +6540,7 @@ func (node *Proxy) RegisterRestRouter(router gin.IRouter) { router.GET(http.DatabaseDescPath, describeDatabase(node)) // Collection requests - router.GET(http.CollectionListPath, listCollection(node)) + router.GET(http.CollectionListPath, listCollection(node.rootCoord, node.queryCoord)) router.GET(http.CollectionDescPath, describeCollection(node, node.rootCoord)) } diff --git a/internal/rootcoord/list_db_task.go b/internal/rootcoord/list_db_task.go index 847b5928c5..14ad53d690 100644 --- a/internal/rootcoord/list_db_task.go +++ b/internal/rootcoord/list_db_task.go @@ -112,15 +112,18 @@ func (t *listDatabaseTask) Execute(ctx context.Context) error { } dbNames := make([]string, 0, len(ret)) + dbIDs := make([]int64, 0, len(ret)) createdTimes := make([]uint64, 0, len(ret)) for _, db := range ret { if !isVisibleDBForCurUser(db.Name, visibleDBs) { continue } dbNames = append(dbNames, db.Name) + dbIDs = append(dbIDs, db.ID) createdTimes = append(createdTimes, db.CreatedTime) } t.Resp.DbNames = dbNames + t.Resp.DbIds = dbIDs t.Resp.CreatedTimestamp = createdTimes return nil } diff --git a/pkg/util/metricsinfo/metrics_info.go b/pkg/util/metricsinfo/metrics_info.go index dda5d1b90d..b321316885 100644 --- a/pkg/util/metricsinfo/metrics_info.go +++ b/pkg/util/metricsinfo/metrics_info.go @@ -405,7 +405,7 @@ type Collections struct { CollectionIDs []string `json:"collection_ids,omitempty"` CreatedUtcTimestamps []string `json:"created_utc_timestamps,omitempty"` // Load percentage on querynode when type is InMemory - InMemoryPercentages []int `json:"inMemory_percentages,omitempty"` + InMemoryPercentages []string `json:"inMemory_percentages,omitempty"` // Indicate whether query service is available QueryServiceAvailable []bool `json:"query_service_available,omitempty"` }