mirror of https://github.com/milvus-io/milvus.git
129 lines
4.5 KiB
Go
129 lines
4.5 KiB
Go
package metricsutil
|
|
|
|
import (
|
|
"strconv"
|
|
"time"
|
|
|
|
"github.com/prometheus/client_golang/prometheus"
|
|
"go.uber.org/zap/zapcore"
|
|
|
|
"github.com/milvus-io/milvus/internal/util/streamingutil/status"
|
|
"github.com/milvus-io/milvus/pkg/v2/log"
|
|
"github.com/milvus-io/milvus/pkg/v2/metrics"
|
|
"github.com/milvus-io/milvus/pkg/v2/streaming/util/message"
|
|
"github.com/milvus-io/milvus/pkg/v2/streaming/util/types"
|
|
"github.com/milvus-io/milvus/pkg/v2/util/paramtable"
|
|
)
|
|
|
|
// NewWriteMetrics creates a new WriteMetrics.
|
|
func NewWriteMetrics(pchannel types.PChannelInfo, walName string) *WriteMetrics {
|
|
constLabel := prometheus.Labels{
|
|
metrics.NodeIDLabelName: paramtable.GetStringNodeID(),
|
|
metrics.WALChannelLabelName: pchannel.Name,
|
|
}
|
|
metrics.WALInfo.WithLabelValues(
|
|
paramtable.GetStringNodeID(),
|
|
pchannel.Name,
|
|
strconv.FormatInt(pchannel.Term, 10),
|
|
walName).Set(1)
|
|
|
|
return &WriteMetrics{
|
|
walName: walName,
|
|
pchannel: pchannel,
|
|
constLabel: constLabel,
|
|
bytes: metrics.WALAppendMessageBytes.MustCurryWith(constLabel),
|
|
total: metrics.WALAppendMessageTotal.MustCurryWith(constLabel),
|
|
walDuration: metrics.WALAppendMessageDurationSeconds.MustCurryWith(constLabel),
|
|
walimplsDuration: metrics.WALImplsAppendMessageDurationSeconds.MustCurryWith(constLabel),
|
|
walBeforeInterceptorDuration: metrics.WALAppendMessageBeforeInterceptorDurationSeconds.MustCurryWith(constLabel),
|
|
walAfterInterceptorDuration: metrics.WALAppendMessageAfterInterceptorDurationSeconds.MustCurryWith(constLabel),
|
|
}
|
|
}
|
|
|
|
type WriteMetrics struct {
|
|
log.Binder
|
|
|
|
walName string
|
|
pchannel types.PChannelInfo
|
|
constLabel prometheus.Labels
|
|
bytes prometheus.ObserverVec
|
|
total *prometheus.CounterVec
|
|
walDuration prometheus.ObserverVec
|
|
walimplsDuration prometheus.ObserverVec
|
|
walBeforeInterceptorDuration prometheus.ObserverVec
|
|
walAfterInterceptorDuration prometheus.ObserverVec
|
|
}
|
|
|
|
func (m *WriteMetrics) StartAppend(msg message.MutableMessage) *AppendMetrics {
|
|
return &AppendMetrics{
|
|
wm: m,
|
|
messageType: msg.MessageType(),
|
|
bytes: msg.EstimateSize(),
|
|
persisted: true,
|
|
interceptors: make(map[string][]*InterceptorMetrics),
|
|
}
|
|
}
|
|
|
|
func (m *WriteMetrics) done(appendMetrics *AppendMetrics) {
|
|
if !appendMetrics.persisted {
|
|
// ignore all the metrics if the message is not persisted.
|
|
return
|
|
}
|
|
|
|
status := parseError(appendMetrics.err)
|
|
if appendMetrics.implAppendDuration != 0 {
|
|
m.walimplsDuration.WithLabelValues(status).Observe(appendMetrics.implAppendDuration.Seconds())
|
|
}
|
|
m.bytes.WithLabelValues(status).Observe(float64(appendMetrics.bytes))
|
|
m.total.WithLabelValues(appendMetrics.messageType.String(), status).Inc()
|
|
m.walDuration.WithLabelValues(status).Observe(appendMetrics.appendDuration.Seconds())
|
|
for name, ims := range appendMetrics.interceptors {
|
|
for _, im := range ims {
|
|
if im.Before != 0 {
|
|
m.walBeforeInterceptorDuration.WithLabelValues(name).Observe(im.Before.Seconds())
|
|
}
|
|
if im.After != 0 {
|
|
m.walAfterInterceptorDuration.WithLabelValues(name).Observe(im.After.Seconds())
|
|
}
|
|
}
|
|
}
|
|
if appendMetrics.err != nil {
|
|
m.Logger().Warn("append message into wal failed", appendMetrics.IntoLogFields()...)
|
|
return
|
|
}
|
|
if appendMetrics.appendDuration >= time.Second {
|
|
// log slow append catch
|
|
m.Logger().Warn("append message into wal too slow", appendMetrics.IntoLogFields()...)
|
|
return
|
|
}
|
|
if m.Logger().Level().Enabled(zapcore.DebugLevel) {
|
|
m.Logger().Debug("append message into wal", appendMetrics.IntoLogFields()...)
|
|
}
|
|
}
|
|
|
|
func (m *WriteMetrics) Close() {
|
|
metrics.WALAppendMessageBeforeInterceptorDurationSeconds.DeletePartialMatch(m.constLabel)
|
|
metrics.WALAppendMessageAfterInterceptorDurationSeconds.DeletePartialMatch(m.constLabel)
|
|
metrics.WALAppendMessageBytes.DeletePartialMatch(m.constLabel)
|
|
metrics.WALAppendMessageTotal.DeletePartialMatch(m.constLabel)
|
|
metrics.WALAppendMessageDurationSeconds.DeletePartialMatch(m.constLabel)
|
|
metrics.WALImplsAppendMessageDurationSeconds.DeletePartialMatch(m.constLabel)
|
|
metrics.WALInfo.DeleteLabelValues(
|
|
paramtable.GetStringNodeID(),
|
|
m.pchannel.Name,
|
|
strconv.FormatInt(m.pchannel.Term, 10),
|
|
m.walName,
|
|
)
|
|
}
|
|
|
|
// parseError parses the error to status.
|
|
func parseError(err error) string {
|
|
if err == nil {
|
|
return metrics.WALStatusOK
|
|
}
|
|
if status.IsCanceled(err) {
|
|
return metrics.WALStatusCancel
|
|
}
|
|
return metrics.WALStatusError
|
|
}
|