Add hook error log and metrics (#22063)

Signed-off-by: SimFG <bang.fu@zilliz.com>
hotfix-2.2.2
SimFG 2023-02-08 19:07:14 +08:00 committed by GitHub
parent 14858adc95
commit ec4784b393
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 37 additions and 2 deletions

View File

@ -58,6 +58,9 @@ const (
Leader = "OnLeader"
FromLeader = "FromLeader"
HookErrorBefore = "before"
HookErrorAfter = "after"
nodeIDLabelName = "node_id"
statusLabelName = "status"
indexTaskStatusLabelName = "index_task_status"
@ -75,6 +78,7 @@ const (
cacheStateLabelName = "cache_state"
indexCountLabelName = "indexed_field_count"
requestScope = "scope"
hookErrorFuncLabelName = "error_func"
)
var (

View File

@ -224,6 +224,14 @@ var (
Name: "limiter_rate",
Help: "",
}, []string{nodeIDLabelName, msgTypeLabelName})
ProxyHookErr = prometheus.NewCounterVec(
prometheus.CounterOpts{
Namespace: milvusNamespace,
Subsystem: typeutil.ProxyRole,
Name: "hook_error_count",
Help: "the hook error count",
}, []string{hookErrorFuncLabelName})
)
//RegisterProxy registers Proxy metrics
@ -258,6 +266,7 @@ func RegisterProxy(registry *prometheus.Registry) {
registry.MustRegister(ProxyReadReqSendBytes)
registry.MustRegister(ProxyLimiterRate)
registry.MustRegister(ProxyHookErr)
}
// SetRateGaugeByRateType sets ProxyLimiterRate metrics.

View File

@ -6,9 +6,9 @@ import (
"plugin"
"github.com/milvus-io/milvus-proto/go-api/hook"
"github.com/milvus-io/milvus/internal/log"
"github.com/milvus-io/milvus/internal/metrics"
"go.uber.org/zap"
"google.golang.org/grpc"
)
@ -82,16 +82,32 @@ func UnaryServerHookInterceptor() grpc.UnaryServerInterceptor {
)
if isMock, mockResp, err = hoo.Mock(ctx, req, fullMethod); isMock {
log.Info("hook mock", zap.String("user", getCurrentUser(ctx)), zap.String("full method", fullMethod),
zap.Any("request", req), zap.Error(err))
return mockResp, err
}
if newCtx, err = hoo.Before(ctx, req, fullMethod); err != nil {
log.Warn("hook before error", zap.String("user", getCurrentUser(ctx)), zap.String("full method", fullMethod),
zap.Any("request", req), zap.Error(err))
metrics.ProxyHookErr.WithLabelValues(metrics.HookErrorBefore).Inc()
return nil, err
}
realResp, realErr = handler(newCtx, req)
if err = hoo.After(newCtx, realResp, realErr, fullMethod); err != nil {
log.Warn("hook after error", zap.String("user", getCurrentUser(ctx)), zap.String("full method", fullMethod),
zap.Any("request", req), zap.Error(err))
metrics.ProxyHookErr.WithLabelValues(metrics.HookErrorAfter).Inc()
return nil, err
}
return realResp, realErr
}
}
func getCurrentUser(ctx context.Context) string {
username, err := GetCurUserFromContext(ctx)
if err != nil {
log.Warn("fail to get current user", zap.Error(err))
}
return username
}

View File

@ -1,5 +1,10 @@
package paramtable
import (
"github.com/milvus-io/milvus/internal/log"
"go.uber.org/zap"
)
const hookYamlFile = "hook.yaml"
type HookConfig struct {
@ -11,6 +16,7 @@ type HookConfig struct {
func (h *HookConfig) init() {
h.Base = &BaseTable{YamlFile: hookYamlFile}
h.Base.Init()
log.Info("hook config", zap.Any("hook", h.Base.Configs()))
h.initSoPath()
h.initSoConfig()