milvus-io
/
milvus
mirror of https://github.com/milvus-io/milvus.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Use binlog size from meta when estimate SegmentSize (#13891) 

Signed-off-by: Congqi Xia <congqi.xia@zilliz.com>
pull/11956/head
congqixia 2021-12-21 20:01:03 +08:00 committed by GitHub
parent 8a63dba149
commit 55ab425010
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 6 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
internal/querycoord/cluster.go
View File

@ -741,19 +741,8 @@ func estimateSegmentsSize(segments *querypb.LoadSegmentsRequest, kvClient kv.Dat
// get binlog size
for _, binlogPath := range loadInfo.BinlogPaths {
for _, path := range binlogPath.Binlogs {
binlogSize, err := storage.EstimateMemorySize(kvClient, path.GetLogPath())
if err != nil {
binlogSize, err = storage.GetBinlogSize(kvClient, path.GetLogPath())
if err != nil {
log.Warn("estimate binlog size wrong",
zap.Int64("segmentID", loadInfo.GetSegmentID()),
zap.String("binlog path", path.GetLogPath()),
zap.Error(err))
return 0, err
}
}
segmentSize += binlogSize
for _, binlog := range binlogPath.Binlogs {
segmentSize += binlog.GetLogSize()
}
}
}

8
internal/querycoord/cluster_test.go
View File

@ -664,7 +664,7 @@ func TestEstimateSegmentSize(t *testing.T) {
binlog := []*datapb.FieldBinlog{
{
FieldID: simpleConstField.id,
Binlogs: []*datapb.Binlog{{LogPath: "^&^%*&%&&(*^*&"}},
Binlogs: []*datapb.Binlog{{LogPath: "by-dev/rand/path", LogSize: 1024}},
},
}
@ -683,8 +683,8 @@ func TestEstimateSegmentSize(t *testing.T) {
}
size, err := estimateSegmentsSize(loadReq, dataKV)
assert.Error(t, err)
assert.Equal(t, int64(0), size)
assert.NoError(t, err)
assert.Equal(t, int64(1024), size)
binlog, err = saveSimpleBinLog(baseCtx, schema, dataKV)
assert.NoError(t, err)
@ -693,7 +693,7 @@ func TestEstimateSegmentSize(t *testing.T) {
size, err = estimateSegmentsSize(loadReq, dataKV)
assert.NoError(t, err)
assert.NotEqual(t, int64(0), size)
assert.NotEqual(t, int64(1024), size)
indexPath, err := generateIndex(defaultSegmentID)
assert.NoError(t, err)