milvus-io
/
milvus
mirror of https://github.com/milvus-io/milvus.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

enhance: fix serialization record span & flushed buffer size metrics (#29482) 

See also #27675 #29413

---------

Signed-off-by: Congqi Xia <congqi.xia@zilliz.com>
pull/29517/head
congqixia 2023-12-27 10:20:48 +08:00 committed by GitHub
parent 033456ea2c
commit f6cff25712
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 25 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
internal/datanode/syncmgr/storage_serializer.go
View File

@ -18,6 +18,7 @@ package syncmgr
import (
"context"
"fmt"
"strconv"
"github.com/samber/lo"
@ -28,7 +29,10 @@ import (
"github.com/milvus-io/milvus/internal/proto/etcdpb"
"github.com/milvus-io/milvus/internal/storage"
"github.com/milvus-io/milvus/pkg/log"
"github.com/milvus-io/milvus/pkg/metrics"
"github.com/milvus-io/milvus/pkg/util/merr"
"github.com/milvus-io/milvus/pkg/util/paramtable"
"github.com/milvus-io/milvus/pkg/util/timerecord"
)
type storageV1Serializer struct {
@ -69,6 +73,8 @@ func NewStorageSerializer(metacache metacache.MetaCache, metaWriter MetaWriter)
func (s *storageV1Serializer) EncodeBuffer(ctx context.Context, pack *SyncPack) (Task, error) {
task := NewSyncTask()
tr := timerecord.NewTimeRecorder("storage_serializer")
metricSegLevel := pack.level.String()
log := log.Ctx(ctx).With(
zap.Int64("segmentID", pack.segmentID),
@ -125,6 +131,8 @@ func (s *storageV1Serializer) EncodeBuffer(ctx context.Context, pack *SyncPack)
}
s.setTaskMeta(task, pack)
metrics.DataNodeEncodeBufferLatency.WithLabelValues(fmt.Sprint(paramtable.GetNodeID()), metricSegLevel).Observe(float64(tr.RecordSpan().Milliseconds()))
return task, nil
}

6
internal/datanode/syncmgr/storage_v2_serializer.go
View File

@ -35,7 +35,10 @@ import (
iTypeutil "github.com/milvus-io/milvus/internal/util/typeutil"
"github.com/milvus-io/milvus/pkg/common"
"github.com/milvus-io/milvus/pkg/log"
"github.com/milvus-io/milvus/pkg/metrics"
"github.com/milvus-io/milvus/pkg/util/merr"
"github.com/milvus-io/milvus/pkg/util/paramtable"
"github.com/milvus-io/milvus/pkg/util/timerecord"
"github.com/milvus-io/milvus/pkg/util/typeutil"
)
@ -68,6 +71,8 @@ func NewStorageV2Serializer(
func (s *storageV2Serializer) EncodeBuffer(ctx context.Context, pack *SyncPack) (Task, error) {
task := NewSyncTaskV2()
tr := timerecord.NewTimeRecorder("storage_serializer_v2")
metricSegLevel := pack.level.String()
space, err := s.storageV2Cache.GetOrCreateSpace(pack.segmentID, SpaceCreatorFunc(pack.segmentID, s.schema, s.arrowSchema))
if err != nil {
@ -120,6 +125,7 @@ func (s *storageV2Serializer) EncodeBuffer(ctx context.Context, pack *SyncPack)
}
s.setTaskMeta(task, pack)
metrics.DataNodeEncodeBufferLatency.WithLabelValues(fmt.Sprint(paramtable.GetNodeID()), metricSegLevel).Observe(float64(tr.RecordSpan().Milliseconds()))
return task, nil
}

23
internal/datanode/syncmgr/task.go
View File

@ -21,6 +21,7 @@ import (
"fmt"
"path"
"github.com/samber/lo"
"go.uber.org/zap"
"github.com/milvus-io/milvus-proto/go-api/v2/commonpb"
@ -162,18 +163,6 @@ func (t *SyncTask) Run() (err error) {
return err
}
/*
var totalSize float64 = 0
if t.deleteData != nil {
totalSize += float64(t.deleteData.Size())
}
if t.insertData != nil {
totalSize += float64(t.insertData.GetMemorySize())
}
metrics.DataNodeFlushedSize.WithLabelValues(fmt.Sprint(paramtable.GetNodeID()), metrics.AllLabel, metricSegLevel).Add(totalSize)
metrics.DataNodeEncodeBufferLatency.WithLabelValues(fmt.Sprint(paramtable.GetNodeID()), metricSegLevel).Observe(float64(t.tr.RecordSpan().Milliseconds()))*/
err = t.writeLogs()
if err != nil {
log.Warn("failed to save serialized data into storage", zap.Error(err))
@ -181,6 +170,16 @@ func (t *SyncTask) Run() (err error) {
return err
}
var totalSize float64
totalSize += lo.SumBy(lo.Values(t.binlogMemsize), func(fieldSize int64) float64 {
return float64(fieldSize)
})
if t.deltaBlob != nil {
totalSize += float64(len(t.deltaBlob.Value))
}
metrics.DataNodeFlushedSize.WithLabelValues(fmt.Sprint(paramtable.GetNodeID()), metrics.AllLabel, metricSegLevel).Add(totalSize)
metrics.DataNodeSave2StorageLatency.WithLabelValues(fmt.Sprint(paramtable.GetNodeID()), metricSegLevel).Observe(float64(t.tr.RecordSpan().Milliseconds()))
if t.metaWriter != nil {