milvus-io
/
milvus
mirror of https://github.com/milvus-io/milvus.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Add log when remove keys in garbage collector. (#15453) 

Signed-off-by: sunby <bingyi.sun@zilliz.com>

Co-authored-by: sunby <bingyi.sun@zilliz.com>
pull/15626/head
Bingyi Sun 2022-02-18 11:59:49 +08:00 committed by GitHub
parent 03bb9e7251
commit daf25bcb81
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 4 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
internal/datacoord/garbage_collector.go
View File

@ -110,7 +110,7 @@ func (gc *garbageCollector) close() {
// scan load meta file info and compares OSS keys
// if missing found, performs gc cleanup
func (gc *garbageCollector) scan() {
var v, m, e int
var v, m int
valid := gc.meta.ListSegmentFiles()
vm := make(map[string]struct{})
for _, k := range valid {
@ -122,6 +122,7 @@ func (gc *garbageCollector) scan() {
prefixes = append(prefixes, path.Join(gc.option.rootPath, insertLogPrefix))
prefixes = append(prefixes, path.Join(gc.option.rootPath, statsLogPrefix))
prefixes = append(prefixes, path.Join(gc.option.rootPath, deltaLogPrefix))
var removedKeys []string
for _, prefix := range prefixes {
for info := range gc.option.cli.ListObjects(context.TODO(), gc.option.bucketName, minio.ListObjectsOptions{
@ -136,13 +137,13 @@ func (gc *garbageCollector) scan() {
m++
// not found in meta, check last modified time exceeds tolerance duration
if time.Since(info.LastModified) > gc.option.missingTolerance {
e++
// ignore error since it could be cleaned up next time
removedKeys = append(removedKeys, info.Key)
_ = gc.option.cli.RemoveObject(context.TODO(), gc.option.bucketName, info.Key, minio.RemoveObjectOptions{})
}
}
}
log.Warn("scan result", zap.Int("valid", v), zap.Int("missing", m), zap.Int("removed", e))
log.Warn("scan result", zap.Int("valid", v), zap.Int("missing", m), zap.Strings("removed keys", removedKeys))
}
func (gc *garbageCollector) clearEtcd() {

1
internal/datacoord/meta.go
View File

@ -163,7 +163,6 @@ func (m *meta) AddSegment(segment *SegmentInfo) error {
return nil
}
// Deprecated
// DropSegment remove segment with provided id, etcd persistence also removed
func (m *meta) DropSegment(segmentID UniqueID) error {
m.Lock()