milvus-io
/
milvus
mirror of https://github.com/milvus-io/milvus.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Make garbage collector fetch meta after list from storage (#27203) 

Signed-off-by: Congqi Xia <congqi.xia@zilliz.com>
pull/27235/head
congqixia 2023-09-19 17:05:34 +08:00 committed by GitHub
parent aa947d49ff
commit 19e4deb792
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 11 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
internal/datacoord/garbage_collector.go
View File

@ -122,16 +122,18 @@ func (gc *garbageCollector) scan() {
total = 0
valid = 0
missing = 0
segmentMap = typeutil.NewUniqueSet()
filesMap = typeutil.NewSet[string]()
)
segments := gc.meta.GetAllSegmentsUnsafe()
for _, segment := range segments {
segmentMap.Insert(segment.GetID())
for _, log := range getLogs(segment) {
filesMap.Insert(log.GetLogPath())
getMetaMap := func() (typeutil.UniqueSet, typeutil.Set[string]) {
segmentMap := typeutil.NewUniqueSet()
filesMap := typeutil.NewSet[string]()
segments := gc.meta.GetAllSegmentsUnsafe()
for _, segment := range segments {
segmentMap.Insert(segment.GetID())
for _, log := range getLogs(segment) {
filesMap.Insert(log.GetLogPath())
}
}
return segmentMap, filesMap
}
// walk only data cluster related prefixes
@ -150,6 +152,7 @@ func (gc *garbageCollector) scan() {
zap.Error(err),
)
}
segmentMap, filesMap := getMetaMap()
log.Info("gc scan finish list object", zap.String("prefix", prefix), zap.Duration("time spent", time.Since(startTs)), zap.Int("keys", len(infoKeys)))
for i, infoKey := range infoKeys {
total++