milvus-io
/
milvus
mirror of https://github.com/milvus-io/milvus.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

enhance: Support gc with bm25 stats logs (#37408) 

Signed-off-by: aoiasd <zhicheng.yue@zilliz.com>
pull/37561/head
aoiasd 2024-11-12 11:30:28 +08:00 committed by GitHub
parent 24c6a4bb29
commit 4682bba586
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
3 changed files with 9 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
internal/datacoord/garbage_collector.go
View File

@ -473,6 +473,7 @@ func (gc *garbageCollector) recycleDroppedSegments(ctx context.Context) {
log.Info("GC segment start...", zap.Int("insert_logs", len(segment.GetBinlogs())),
zap.Int("delta_logs", len(segment.GetDeltalogs())),
zap.Int("stats_logs", len(segment.GetStatslogs())),
zap.Int("bm25_logs", len(segment.GetBm25Statslogs())),
zap.Int("text_logs", len(segment.GetTextStatsLogs())))
if err := gc.removeObjectFiles(ctx, logs); err != nil {
log.Warn("GC segment remove logs failed", zap.Error(err))
@ -564,6 +565,11 @@ func getLogs(sinfo *SegmentInfo) map[string]struct{} {
logs[l.GetLogPath()] = struct{}{}
}
}
for _, flog := range sinfo.GetBm25Statslogs() {
for _, l := range flog.GetBinlogs() {
logs[l.GetLogPath()] = struct{}{}
}
}
return logs
}

3
internal/metastore/kv/datacoord/kv_catalog.go
View File

@ -436,8 +436,9 @@ func (kc *Catalog) DropSegment(ctx context.Context, segment *datapb.SegmentInfo)
binlogPreix := fmt.Sprintf("%s/%d/%d/%d", SegmentBinlogPathPrefix, segment.GetCollectionID(), segment.GetPartitionID(), segment.GetID())
deltalogPreix := fmt.Sprintf("%s/%d/%d/%d", SegmentDeltalogPathPrefix, segment.GetCollectionID(), segment.GetPartitionID(), segment.GetID())
statelogPreix := fmt.Sprintf("%s/%d/%d/%d", SegmentStatslogPathPrefix, segment.GetCollectionID(), segment.GetPartitionID(), segment.GetID())
bm25logPrefix := fmt.Sprintf("%s/%d/%d/%d", SegmentBM25logPathPrefix, segment.GetCollectionID(), segment.GetPartitionID(), segment.GetID())
keys := []string{segKey, binlogPreix, deltalogPreix, statelogPreix}
keys := []string{segKey, binlogPreix, deltalogPreix, statelogPreix, bm25logPrefix}
if err := kc.MetaKv.MultiSaveAndRemoveWithPrefix(nil, keys); err != nil {
return err
}

2
internal/metastore/kv/datacoord/kv_catalog_test.go
View File

@ -533,7 +533,7 @@ func Test_DropSegment(t *testing.T) {
deltalogPreix := fmt.Sprintf("%s/%d/%d/%d", SegmentDeltalogPathPrefix, segment1.GetCollectionID(), segment1.GetPartitionID(), segment1.GetID())
statelogPreix := fmt.Sprintf("%s/%d/%d/%d", SegmentStatslogPathPrefix, segment1.GetCollectionID(), segment1.GetPartitionID(), segment1.GetID())
assert.Equal(t, 4, len(removedKvs))
assert.Equal(t, 5, len(removedKvs))
for _, k := range []string{segKey, binlogPreix, deltalogPreix, statelogPreix} {
_, ok := removedKvs[k]
assert.True(t, ok)