milvus-io
/
milvus
mirror of https://github.com/milvus-io/milvus.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

enhance: [2.5] improve sparse query nnz metric (#40714) 

add query type and field id label; add metric for hybrid search

issue: https://github.com/milvus-io/milvus/issues/35853
pr: https://github.com/milvus-io/milvus/pull/40713

Signed-off-by: Buqian Zheng <zhengbuqian@gmail.com>
pull/40010/head
Buqian Zheng 2025-03-20 14:14:21 +08:00 committed by GitHub
parent 281260e48a
commit cff0e82f57
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
6 changed files with 15 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
internal/proxy/task_search.go
View File

@ -414,6 +414,9 @@ func (t *searchTask) initAdvancedSearchRequest(ctx context.Context) error {
if err != nil {
return err
}
if typeutil.IsFieldSparseFloatVector(t.schema.CollectionSchema, internalSubReq.FieldId) {
metrics.ProxySearchSparseNumNonZeros.WithLabelValues(strconv.FormatInt(paramtable.GetNodeID(), 10), t.collectionName, metrics.HybridSearchLabel, strconv.FormatInt(internalSubReq.FieldId, 10)).Observe(float64(typeutil.EstimateSparseVectorNNZFromPlaceholderGroup(internalSubReq.PlaceholderGroup, int(internalSubReq.GetNq()))))
}
t.SearchRequest.SubReqs[index] = internalSubReq
t.queryInfos[index] = queryInfo
log.Debug("proxy init search request",
@ -491,7 +494,9 @@ func (t *searchTask) initSearchRequest(ctx context.Context) error {
if err != nil {
return err
}
metrics.ProxySearchSparseNumNonZeros.WithLabelValues(strconv.FormatInt(paramtable.GetNodeID(), 10), t.collectionName).Observe(float64(typeutil.EstimateSparseVectorNNZFromPlaceholderGroup(t.request.PlaceholderGroup, int(t.request.GetNq()))))
if typeutil.IsFieldSparseFloatVector(t.schema.CollectionSchema, t.SearchRequest.FieldId) {
metrics.ProxySearchSparseNumNonZeros.WithLabelValues(strconv.FormatInt(paramtable.GetNodeID(), 10), t.collectionName, metrics.SearchLabel, strconv.FormatInt(t.SearchRequest.FieldId, 10)).Observe(float64(typeutil.EstimateSparseVectorNNZFromPlaceholderGroup(t.request.PlaceholderGroup, int(t.request.GetNq()))))
}
t.SearchRequest.PlaceholderGroup = t.request.PlaceholderGroup
t.SearchRequest.Topk = queryInfo.GetTopk()
t.SearchRequest.MetricType = queryInfo.GetMetricType()

2
internal/querynodev2/delegator/delegator_data.go
View File

@ -1030,7 +1030,7 @@ func (sd *shardDelegator) buildBM25IDF(req *internalpb.SearchRequest) (float64,
}
for _, idf := range idfSparseVector {
metrics.QueryNodeSearchFTSNumTokens.WithLabelValues(fmt.Sprint(paramtable.GetNodeID()), fmt.Sprint(sd.collectionID)).Observe(float64(typeutil.SparseFloatRowElementCount(idf)))
metrics.QueryNodeSearchFTSNumTokens.WithLabelValues(fmt.Sprint(paramtable.GetNodeID()), fmt.Sprint(sd.collectionID), fmt.Sprint(req.GetFieldId())).Observe(float64(typeutil.SparseFloatRowElementCount(idf)))
}
err = SetBM25Params(req, avgdl)

1
pkg/metrics/metrics.go
View File

@ -94,6 +94,7 @@ const (
indexTaskStatusLabelName = "index_task_status"
msgTypeLabelName = "msg_type"
collectionIDLabelName = "collection_id"
fieldIDLabelName = "field_id"
channelNameLabelName = "channel_name"
functionLabelName = "function_name"
queryTypeLabelName = "query_type"

2
pkg/metrics/proxy_metrics.go
View File

@ -426,7 +426,7 @@ var (
Name: "search_sparse_num_non_zeros",
Help: "the number of non-zeros in each sparse search task",
Buckets: buckets,
}, []string{nodeIDLabelName, collectionName})
}, []string{nodeIDLabelName, collectionName, queryTypeLabelName, fieldIDLabelName})
ProxyParseExpressionLatency = prometheus.NewHistogramVec(
prometheus.HistogramOpts{

1
pkg/metrics/querynode_metrics.go
View File

@ -360,6 +360,7 @@ var (
}, []string{
nodeIDLabelName,
collectionIDLabelName,
fieldIDLabelName,
})
QueryNodeSearchGroupSize = prometheus.NewHistogramVec(

5
pkg/util/typeutil/schema.go
View File

@ -1183,6 +1183,11 @@ func GetPrimaryFieldSchema(schema *schemapb.CollectionSchema) (*schemapb.FieldSc
return nil, errors.New("primary field is not found")
}
func IsFieldSparseFloatVector(schema *schemapb.CollectionSchema, fieldID int64) bool {
fieldSchema := GetField(schema, fieldID)
return fieldSchema != nil && IsSparseFloatVectorType(fieldSchema.DataType)
}
// GetPartitionKeyFieldSchema get partition field schema from collection schema
func GetPartitionKeyFieldSchema(schema *schemapb.CollectionSchema) (*schemapb.FieldSchema, error) {
for _, fieldSchema := range schema.GetFields() {