milvus-io
/
milvus
mirror of https://github.com/milvus-io/milvus.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Add error log and metrics for the hook and privilege interceptor (#22111) 

Signed-off-by: SimFG <bang.fu@zilliz.com>
pull/22124/head
SimFG 2023-02-10 15:52:32 +08:00 committed by GitHub
parent 31de6cfb3c
commit bd559b3148
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 46 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
internal/metrics/metrics.go
View File

@ -63,6 +63,10 @@ const (
Leader = "OnLeader"
FromLeader = "FromLeader"
HookBefore = "before"
HookAfter = "after"
HookMock = "mock"
nodeIDLabelName = "node_id"
statusLabelName = "status"
indexTaskStatusLabelName = "index_task_status"
@ -82,6 +86,7 @@ const (
cacheStateLabelName = "cache_state"
indexCountLabelName = "indexed_field_count"
requestScope = "scope"
fullMethodLabelName = "full_method"
)
var (

9
internal/metrics/proxy_metrics.go
View File

@ -224,6 +224,14 @@ var (
Name: "limiter_rate",
Help: "",
}, []string{nodeIDLabelName, msgTypeLabelName})
ProxyHookFunc = prometheus.NewCounterVec(
prometheus.CounterOpts{
Namespace: milvusNamespace,
Subsystem: typeutil.ProxyRole,
Name: "hook_func_count",
Help: "the hook function count",
}, []string{functionLabelName, fullMethodLabelName})
)
//RegisterProxy registers Proxy metrics
@ -258,6 +266,7 @@ func RegisterProxy(registry *prometheus.Registry) {
registry.MustRegister(ProxyReadReqSendBytes)
registry.MustRegister(ProxyLimiterRate)
registry.MustRegister(ProxyHookFunc)
}
// SetRateGaugeByRateType sets ProxyLimiterRate metrics.

21
internal/proxy/hook_interceptor.go
View File

@ -6,9 +6,9 @@ import (
"plugin"
"github.com/milvus-io/milvus-proto/go-api/hook"
"github.com/milvus-io/milvus/internal/log"
"github.com/milvus-io/milvus/internal/metrics"
"go.uber.org/zap"
"google.golang.org/grpc"
)
@ -82,16 +82,33 @@ func UnaryServerHookInterceptor() grpc.UnaryServerInterceptor {
)
if isMock, mockResp, err = hoo.Mock(ctx, req, fullMethod); isMock {
log.Info("hook mock", zap.String("user", getCurrentUser(ctx)), zap.String("full method", fullMethod),
zap.Any("request", req), zap.Error(err))
metrics.ProxyHookFunc.WithLabelValues(metrics.HookMock, fullMethod).Inc()
return mockResp, err
}
if newCtx, err = hoo.Before(ctx, req, fullMethod); err != nil {
log.Warn("hook before error", zap.String("user", getCurrentUser(ctx)), zap.String("full method", fullMethod),
zap.Any("request", req), zap.Error(err))
metrics.ProxyHookFunc.WithLabelValues(metrics.HookBefore, fullMethod).Inc()
return nil, err
}
realResp, realErr = handler(newCtx, req)
if err = hoo.After(newCtx, realResp, realErr, fullMethod); err != nil {
log.Warn("hook after error", zap.String("user", getCurrentUser(ctx)), zap.String("full method", fullMethod),
zap.Any("request", req), zap.Error(err))
metrics.ProxyHookFunc.WithLabelValues(metrics.HookAfter, fullMethod).Inc()
return nil, err
}
return realResp, realErr
}
}
func getCurrentUser(ctx context.Context) string {
username, err := GetCurUserFromContext(ctx)
if err != nil {
log.Warn("fail to get current user", zap.Error(err))
}
return username
}

12
internal/proxy/privilege_interceptor.go
View File

@ -73,7 +73,7 @@ func PrivilegeInterceptor(ctx context.Context, req interface{}) (context.Context
}
username, err := GetCurUserFromContext(ctx)
if err != nil {
log.Error("GetCurUserFromContext fail", zap.Error(err))
log.Warn("GetCurUserFromContext fail", zap.Error(err))
return ctx, err
}
if username == util.UserRoot {
@ -81,7 +81,7 @@ func PrivilegeInterceptor(ctx context.Context, req interface{}) (context.Context
}
roleNames, err := GetRole(username)
if err != nil {
log.Error("GetRole fail", zap.String("username", username), zap.Error(err))
log.Warn("GetRole fail", zap.String("username", username), zap.Error(err))
return ctx, err
}
roleNames = append(roleNames, util.RolePublic)
@ -96,7 +96,7 @@ func PrivilegeInterceptor(ctx context.Context, req interface{}) (context.Context
objectPrivilege := privilegeExt.ObjectPrivilege.String()
policyInfo := strings.Join(globalMetaCache.GetPrivilegeInfo(ctx), ",")
log.Debug("current request info", zap.String("username", username), zap.Strings("role_names", roleNames),
logWithCurrentRequestInfo := log.With(zap.String("username", username), zap.Strings("role_names", roleNames),
zap.String("object_type", objectType), zap.String("object_privilege", objectPrivilege),
zap.Int32("object_index", objectNameIndex), zap.String("object_name", objectName),
zap.Int32("object_indexs", objectNameIndexs), zap.Strings("object_names", objectNames),
@ -109,7 +109,7 @@ func PrivilegeInterceptor(ctx context.Context, req interface{}) (context.Context
casbinModel := templateModel.Copy()
e, err := casbin.NewEnforcer(casbinModel, a)
if err != nil {
log.Error("NewEnforcer fail", zap.String("policy", policy), zap.Error(err))
logWithCurrentRequestInfo.Warn("NewEnforcer fail", zap.String("policy", policy), zap.Error(err))
return ctx, err
}
for _, roleName := range roleNames {
@ -126,6 +126,7 @@ func PrivilegeInterceptor(ctx context.Context, req interface{}) (context.Context
// handle the api which refers one resource
permitObject, err := permitFunc(objectName)
if err != nil {
logWithCurrentRequestInfo.Warn("fail to execute permit func", zap.String("name", objectName), zap.Error(err))
return ctx, err
}
if permitObject {
@ -139,6 +140,7 @@ func PrivilegeInterceptor(ctx context.Context, req interface{}) (context.Context
for _, name := range objectNames {
p, err := permitFunc(name)
if err != nil {
logWithCurrentRequestInfo.Warn("fail to execute permit func", zap.String("name", name), zap.Error(err))
return ctx, err
}
if !p {
@ -152,7 +154,7 @@ func PrivilegeInterceptor(ctx context.Context, req interface{}) (context.Context
}
}
log.Info("permission deny", zap.String("policy", policy), zap.Strings("roles", roleNames))
logWithCurrentRequestInfo.Info("permission deny", zap.String("policy", policy), zap.Strings("roles", roleNames))
return ctx, status.Error(codes.PermissionDenied, fmt.Sprintf("%s: permission deny", objectPrivilege))
}

6
internal/util/paramtable/hook_config.go
View File

@ -1,5 +1,10 @@
package paramtable
import (
"github.com/milvus-io/milvus/internal/log"
"go.uber.org/zap"
)
const hookYamlFile = "hook.yaml"
type HookConfig struct {
@ -11,6 +16,7 @@ type HookConfig struct {
func (h *HookConfig) init() {
h.Base = &BaseTable{YamlFile: hookYamlFile}
h.Base.Init()
log.Info("hook config", zap.Any("hook", h.Base.Configs()))
h.initSoPath()
h.initSoConfig()