From 22b917a1e619c100d653d4267128de21756cb864 Mon Sep 17 00:00:00 2001 From: aoiasd <45024769+aoiasd@users.noreply.github.com> Date: Fri, 25 Oct 2024 14:29:47 +0800 Subject: [PATCH] enhance: Add collection name label for some metric (#36951) Signed-off-by: aoiasd --- internal/datacoord/index_meta.go | 2 +- internal/datacoord/meta.go | 2 +- internal/querynodev2/metrics_info.go | 10 ++-- internal/querynodev2/segments/collection.go | 11 +++++ .../segments/mock_collection_manager.go | 47 +++++++++++++++++++ pkg/metrics/datacoord_metrics.go | 4 +- pkg/metrics/querynode_metrics.go | 1 + 7 files changed, 70 insertions(+), 7 deletions(-) diff --git a/internal/datacoord/index_meta.go b/internal/datacoord/index_meta.go index 2525517ef5..973a8d7aed 100644 --- a/internal/datacoord/index_meta.go +++ b/internal/datacoord/index_meta.go @@ -824,7 +824,7 @@ func (m *indexMeta) SetStoredIndexFileSizeMetric(collections map[UniqueID]*colle for _, segmentIdx := range m.buildID2SegmentIndex { coll, ok := collections[segmentIdx.CollectionID] if ok { - metrics.DataCoordStoredIndexFilesSize.WithLabelValues(coll.DatabaseName, + metrics.DataCoordStoredIndexFilesSize.WithLabelValues(coll.DatabaseName, coll.Schema.GetName(), fmt.Sprint(segmentIdx.CollectionID), fmt.Sprint(segmentIdx.SegmentID)).Set(float64(segmentIdx.IndexSize)) total += segmentIdx.IndexSize } diff --git a/internal/datacoord/meta.go b/internal/datacoord/meta.go index a323805a6d..dec075915c 100644 --- a/internal/datacoord/meta.go +++ b/internal/datacoord/meta.go @@ -446,7 +446,7 @@ func (m *meta) GetQuotaInfo() *metricsinfo.DataCoordQuotaMetrics { coll, ok := m.collections[collectionID] if ok { for state, rows := range statesRows { - metrics.DataCoordNumStoredRows.WithLabelValues(coll.DatabaseName, fmt.Sprint(collectionID), state.String()).Set(float64(rows)) + metrics.DataCoordNumStoredRows.WithLabelValues(coll.DatabaseName, fmt.Sprint(collectionID), coll.Schema.GetName(), state.String()).Set(float64(rows)) } } } diff --git a/internal/querynodev2/metrics_info.go b/internal/querynodev2/metrics_info.go index f60a07e036..151b4e03fb 100644 --- a/internal/querynodev2/metrics_info.go +++ b/internal/querynodev2/metrics_info.go @@ -59,7 +59,7 @@ func getQuotaMetrics(node *QueryNode) (*metricsinfo.QueryNodeQuotaMetrics, error } minTsafeChannel, minTsafe := node.tSafeManager.Min() - collections := node.manager.Collection.List() + collections := node.manager.Collection.ListWithName() nodeID := fmt.Sprint(node.GetNodeID()) metrics.QueryNodeNumEntities.Reset() @@ -70,7 +70,7 @@ func getQuotaMetrics(node *QueryNode) (*metricsinfo.QueryNodeQuotaMetrics, error growingGroupByCollection := lo.GroupBy(growingSegments, func(seg segments.Segment) int64 { return seg.Collection() }) - for _, collection := range collections { + for collection := range collections { segs := growingGroupByCollection[collection] size := lo.SumBy(segs, func(seg segments.Segment) int64 { return seg.MemSize() @@ -90,6 +90,7 @@ func getQuotaMetrics(node *QueryNode) (*metricsinfo.QueryNodeQuotaMetrics, error segment := segs[0] metrics.QueryNodeNumEntities.WithLabelValues( segment.DatabaseName(), + collections[segment.Collection()], nodeID, fmt.Sprint(segment.Collection()), fmt.Sprint(segment.Partition()), @@ -101,7 +102,7 @@ func getQuotaMetrics(node *QueryNode) (*metricsinfo.QueryNodeQuotaMetrics, error sealedGroupByCollection := lo.GroupBy(sealedSegments, func(seg segments.Segment) int64 { return seg.Collection() }) - for _, collection := range collections { + for collection := range collections { segs := sealedGroupByCollection[collection] size := lo.SumBy(segs, func(seg segments.Segment) int64 { return seg.MemSize() @@ -119,6 +120,7 @@ func getQuotaMetrics(node *QueryNode) (*metricsinfo.QueryNodeQuotaMetrics, error segment := segs[0] metrics.QueryNodeNumEntities.WithLabelValues( segment.DatabaseName(), + collections[segment.Collection()], nodeID, fmt.Sprint(segment.Collection()), fmt.Sprint(segment.Partition()), @@ -148,7 +150,7 @@ func getQuotaMetrics(node *QueryNode) (*metricsinfo.QueryNodeQuotaMetrics, error GrowingSegmentsSize: totalGrowingSize, Effect: metricsinfo.NodeEffect{ NodeID: node.GetNodeID(), - CollectionIDs: collections, + CollectionIDs: lo.Keys(collections), }, DeleteBufferInfo: metricsinfo.DeleteBufferInfo{ CollectionDeleteBufferNum: deleteBufferNum, diff --git a/internal/querynodev2/segments/collection.go b/internal/querynodev2/segments/collection.go index c9cb4c3e4c..38ea2b2438 100644 --- a/internal/querynodev2/segments/collection.go +++ b/internal/querynodev2/segments/collection.go @@ -47,6 +47,7 @@ import ( type CollectionManager interface { List() []int64 + ListWithName() map[int64]string Get(collectionID int64) *Collection PutOrRef(collectionID int64, schema *schemapb.CollectionSchema, meta *segcorepb.CollectionIndexMeta, loadMeta *querypb.LoadMetaInfo) Ref(collectionID int64, count uint32) bool @@ -74,6 +75,16 @@ func (m *collectionManager) List() []int64 { return lo.Keys(m.collections) } +// return all collections by map id --> name +func (m *collectionManager) ListWithName() map[int64]string { + m.mut.RLock() + defer m.mut.RUnlock() + + return lo.MapValues(m.collections, func(coll *Collection, _ int64) string { + return coll.Schema().GetName() + }) +} + func (m *collectionManager) Get(collectionID int64) *Collection { m.mut.RLock() defer m.mut.RUnlock() diff --git a/internal/querynodev2/segments/mock_collection_manager.go b/internal/querynodev2/segments/mock_collection_manager.go index 496134100f..9440d7faee 100644 --- a/internal/querynodev2/segments/mock_collection_manager.go +++ b/internal/querynodev2/segments/mock_collection_manager.go @@ -118,6 +118,53 @@ func (_c *MockCollectionManager_List_Call) RunAndReturn(run func() []int64) *Moc return _c } +// ListWithName provides a mock function with given fields: +func (_m *MockCollectionManager) ListWithName() map[int64]string { + ret := _m.Called() + + if len(ret) == 0 { + panic("no return value specified for ListWithName") + } + + var r0 map[int64]string + if rf, ok := ret.Get(0).(func() map[int64]string); ok { + r0 = rf() + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(map[int64]string) + } + } + + return r0 +} + +// MockCollectionManager_ListWithName_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'ListWithName' +type MockCollectionManager_ListWithName_Call struct { + *mock.Call +} + +// ListWithName is a helper method to define mock.On call +func (_e *MockCollectionManager_Expecter) ListWithName() *MockCollectionManager_ListWithName_Call { + return &MockCollectionManager_ListWithName_Call{Call: _e.mock.On("ListWithName")} +} + +func (_c *MockCollectionManager_ListWithName_Call) Run(run func()) *MockCollectionManager_ListWithName_Call { + _c.Call.Run(func(args mock.Arguments) { + run() + }) + return _c +} + +func (_c *MockCollectionManager_ListWithName_Call) Return(_a0 map[int64]string) *MockCollectionManager_ListWithName_Call { + _c.Call.Return(_a0) + return _c +} + +func (_c *MockCollectionManager_ListWithName_Call) RunAndReturn(run func() map[int64]string) *MockCollectionManager_ListWithName_Call { + _c.Call.Return(run) + return _c +} + // PutOrRef provides a mock function with given fields: collectionID, schema, meta, loadMeta func (_m *MockCollectionManager) PutOrRef(collectionID int64, schema *schemapb.CollectionSchema, meta *segcorepb.CollectionIndexMeta, loadMeta *querypb.LoadMetaInfo) { _m.Called(collectionID, schema, meta, loadMeta) diff --git a/pkg/metrics/datacoord_metrics.go b/pkg/metrics/datacoord_metrics.go index 493c958f18..65320398e5 100644 --- a/pkg/metrics/datacoord_metrics.go +++ b/pkg/metrics/datacoord_metrics.go @@ -96,6 +96,7 @@ var ( }, []string{ databaseLabelName, collectionIDLabelName, + collectionName, segmentStateLabelName, }) @@ -163,6 +164,7 @@ var ( Help: "index files size of the segments", }, []string{ databaseLabelName, + collectionName, collectionIDLabelName, segmentIDLabelName, }) @@ -403,7 +405,7 @@ func CleanupDataCoordSegmentMetrics(dbName string, collectionID int64, segmentID collectionIDLabelName: fmt.Sprint(collectionID), segmentIDLabelName: fmt.Sprint(segmentID), }) - DataCoordStoredIndexFilesSize.Delete(prometheus.Labels{ + DataCoordStoredIndexFilesSize.DeletePartialMatch(prometheus.Labels{ databaseLabelName: dbName, collectionIDLabelName: fmt.Sprint(collectionID), segmentIDLabelName: fmt.Sprint(segmentID), diff --git a/pkg/metrics/querynode_metrics.go b/pkg/metrics/querynode_metrics.go index d7cb9a35ad..b8e8b55c0e 100644 --- a/pkg/metrics/querynode_metrics.go +++ b/pkg/metrics/querynode_metrics.go @@ -439,6 +439,7 @@ var ( Help: "number of entities which can be searched/queried, clustered by collection, partition and state", }, []string{ databaseLabelName, + collectionName, nodeIDLabelName, collectionIDLabelName, partitionIDLabelName,