From b9bafc76b4dab412563b88bca0ddf1193a50c821 Mon Sep 17 00:00:00 2001 From: aoiasd <45024769+aoiasd@users.noreply.github.com> Date: Wed, 5 Jun 2024 17:13:50 +0800 Subject: [PATCH] enhance: access log support print with consistency level (#33503) relate: https://github.com/milvus-io/milvus/issues/33563 Signed-off-by: aoiasd --- internal/proxy/accesslog/info/grpc_info.go | 8 ++++ .../proxy/accesslog/info/grpc_info_test.go | 11 +++++ internal/proxy/accesslog/info/info.go | 44 +++++++++++-------- internal/proxy/accesslog/info/restful_info.go | 16 ++++--- .../proxy/accesslog/info/restful_info_test.go | 13 ++++++ pkg/util/requestutil/getter.go | 12 +++++ 6 files changed, 80 insertions(+), 24 deletions(-) diff --git a/internal/proxy/accesslog/info/grpc_info.go b/internal/proxy/accesslog/info/grpc_info.go index 9d94078f72..7dcd7c3bd7 100644 --- a/internal/proxy/accesslog/info/grpc_info.go +++ b/internal/proxy/accesslog/info/grpc_info.go @@ -262,3 +262,11 @@ func (i *GrpcAccessInfo) OutputFields() string { } return Unknown } + +func (i *GrpcAccessInfo) ConsistencyLevel() string { + level, ok := requestutil.GetConsistencyLevelFromRequst(i.req) + if ok { + return level.String() + } + return Unknown +} diff --git a/internal/proxy/accesslog/info/grpc_info_test.go b/internal/proxy/accesslog/info/grpc_info_test.go index 0a4ecf031e..6d4eb7f968 100644 --- a/internal/proxy/accesslog/info/grpc_info_test.go +++ b/internal/proxy/accesslog/info/grpc_info_test.go @@ -190,6 +190,17 @@ func (s *GrpcAccessInfoSuite) TestOutputFields() { s.Equal(fmt.Sprint(fields), result[0]) } +func (s *GrpcAccessInfoSuite) TestConsistencyLevel() { + result := Get(s.info, "$consistency_level") + s.Equal(Unknown, result[0]) + + s.info.req = &milvuspb.QueryRequest{ + ConsistencyLevel: commonpb.ConsistencyLevel_Bounded, + } + result = Get(s.info, "$consistency_level") + s.Equal(commonpb.ConsistencyLevel_Bounded.String(), result[0]) +} + func (s *GrpcAccessInfoSuite) TestClusterPrefix() { cluster := "instance-test" paramtable.Init() diff --git a/internal/proxy/accesslog/info/info.go b/internal/proxy/accesslog/info/info.go index 06431db898..3f69fc8519 100644 --- a/internal/proxy/accesslog/info/info.go +++ b/internal/proxy/accesslog/info/info.go @@ -26,25 +26,26 @@ type getMetricFunc func(i AccessInfo) string // supported metrics var MetricFuncMap = map[string]getMetricFunc{ - "$method_name": getMethodName, - "$method_status": getMethodStatus, - "$trace_id": getTraceID, - "$user_addr": getAddr, - "$user_name": getUserName, - "$response_size": getResponseSize, - "$error_code": getErrorCode, - "$error_msg": getErrorMsg, - "$database_name": getDbName, - "$collection_name": getCollectionName, - "$partition_name": getPartitionName, - "$time_cost": getTimeCost, - "$time_now": getTimeNow, - "$time_start": getTimeStart, - "$time_end": getTimeEnd, - "$method_expr": getExpr, - "$output_fields": getOutputFields, - "$sdk_version": getSdkVersion, - "$cluster_prefix": getClusterPrefix, + "$method_name": getMethodName, + "$method_status": getMethodStatus, + "$trace_id": getTraceID, + "$user_addr": getAddr, + "$user_name": getUserName, + "$response_size": getResponseSize, + "$error_code": getErrorCode, + "$error_msg": getErrorMsg, + "$database_name": getDbName, + "$collection_name": getCollectionName, + "$partition_name": getPartitionName, + "$time_cost": getTimeCost, + "$time_now": getTimeNow, + "$time_start": getTimeStart, + "$time_end": getTimeEnd, + "$method_expr": getExpr, + "$output_fields": getOutputFields, + "$sdk_version": getSdkVersion, + "$cluster_prefix": getClusterPrefix, + "$consistency_level": getConsistencyLevel, } type AccessInfo interface { @@ -66,6 +67,7 @@ type AccessInfo interface { Expression() string OutputFields() string SdkVersion() string + ConsistencyLevel() string } func Get(i AccessInfo, keys ...string) []any { @@ -153,6 +155,10 @@ func getOutputFields(i AccessInfo) string { return i.OutputFields() } +func getConsistencyLevel(i AccessInfo) string { + return i.ConsistencyLevel() +} + func getClusterPrefix(i AccessInfo) string { return ClusterPrefix.Load() } diff --git a/internal/proxy/accesslog/info/restful_info.go b/internal/proxy/accesslog/info/restful_info.go index cd7e4eba3b..a585e3e5e7 100644 --- a/internal/proxy/accesslog/info/restful_info.go +++ b/internal/proxy/accesslog/info/restful_info.go @@ -19,7 +19,6 @@ package info import ( "fmt" "net/http" - "sync" "time" "github.com/gin-gonic/gin" @@ -35,10 +34,9 @@ const ( ) type RestfulInfo struct { - params *gin.LogFormatterParams - start time.Time - req interface{} - reqInitOnce sync.Once + params *gin.LogFormatterParams + start time.Time + req interface{} } func NewRestfulInfo() *RestfulInfo { @@ -187,3 +185,11 @@ func (i *RestfulInfo) OutputFields() string { } return Unknown } + +func (i *RestfulInfo) ConsistencyLevel() string { + level, ok := requestutil.GetConsistencyLevelFromRequst(i.req) + if ok { + return level.String() + } + return Unknown +} diff --git a/internal/proxy/accesslog/info/restful_info_test.go b/internal/proxy/accesslog/info/restful_info_test.go index 8a12ad1e93..c07099a1e7 100644 --- a/internal/proxy/accesslog/info/restful_info_test.go +++ b/internal/proxy/accesslog/info/restful_info_test.go @@ -25,6 +25,7 @@ import ( "github.com/gin-gonic/gin" "github.com/stretchr/testify/suite" + "github.com/milvus-io/milvus-proto/go-api/v2/commonpb" "github.com/milvus-io/milvus-proto/go-api/v2/milvuspb" "github.com/milvus-io/milvus/pkg/util/merr" "github.com/milvus-io/milvus/pkg/util/paramtable" @@ -178,6 +179,18 @@ func (s *RestfulAccessInfoSuite) TestOutputFields() { s.Equal(fmt.Sprint(fields), result[0]) } +func (s *RestfulAccessInfoSuite) TestConsistencyLevel() { + result := Get(s.info, "$consistency_level") + s.Equal(Unknown, result[0]) + + s.info.params.Keys[ContextRequest] = &milvuspb.QueryRequest{ + ConsistencyLevel: commonpb.ConsistencyLevel_Bounded, + } + s.info.InitReq() + result = Get(s.info, "$consistency_level") + s.Equal(commonpb.ConsistencyLevel_Bounded.String(), result[0]) +} + func (s *RestfulAccessInfoSuite) TestClusterPrefix() { cluster := "instance-test" paramtable.Init() diff --git a/pkg/util/requestutil/getter.go b/pkg/util/requestutil/getter.go index 85a273287c..8bcec4a815 100644 --- a/pkg/util/requestutil/getter.go +++ b/pkg/util/requestutil/getter.go @@ -156,6 +156,18 @@ func GetStatusFromResponse(resp interface{}) (*commonpb.Status, bool) { return getter.GetStatus(), true } +type ConsistencyLevelGetter interface { + GetConsistencyLevel() commonpb.ConsistencyLevel +} + +func GetConsistencyLevelFromRequst(req interface{}) (commonpb.ConsistencyLevel, bool) { + getter, ok := req.(ConsistencyLevelGetter) + if !ok { + return 0, false + } + return getter.GetConsistencyLevel(), true +} + var TraceLogBaseInfoFuncMap = map[string]func(interface{}) (any, bool){ "collection_name": GetCollectionNameFromRequest, "db_name": GetDbNameFromRequest,