Add datacoord GC configuration (#12061)

Signed-off-by: Congqi Xia <congqi.xia@zilliz.com>

configs/milvus.yaml

@@ -179,6 +179,12 @@ dataCoord:
   compaction:
     retentionDuration: 432000 # 5 days in seconds
 
+  gc:
+    interval: 3600 # gc interval in seconds
+    missingTolerance: 86400 # file meta missing tolerance duration in seconds, 60*24
+    dropTolerance: 86400 # file belongs to dropped entity tolerance duration in seconds, 60*24
+
+
 dataNode:
   port: 21124
 

internal/datacoord/garbage_collector.go

@@ -26,14 +26,6 @@ import (
 	"go.uber.org/zap"
 )
 
-const (
-	// temp solution use const
-	// maybe put garbage config into config files in the future
-	defaultGcInterval       = 1 * time.Hour
-	defaultMissingTolerance = 24 * time.Hour // 1day
-	defaultDropTolerance    = 24 * time.Hour // 1day
-)
-
 // GcOption garbage collection options
 type GcOption struct {
 	cli              *minio.Client // OSS client
@@ -59,6 +51,8 @@ type garbageCollector struct {
 
 // newGarbageCollector create garbage collector with meta and option
 func newGarbageCollector(meta *meta, opt GcOption) *garbageCollector {
+	log.Info("GC with option", zap.Bool("enabled", opt.enabled), zap.Duration("interval", opt.checkInterval),
+		zap.Duration("missingTolerance", opt.missingTolerance), zap.Duration("dropTolerance", opt.dropTolerance))
 	return &garbageCollector{
 		meta:    meta,
 		option:  opt,

internal/datacoord/param_table.go

@@ -81,6 +81,11 @@ type ParamTable struct {
 	EnableGarbageCollection bool
 
 	CompactionRetentionDuration int64
+
+	// Garbage Collection
+	GCInterval         time.Duration
+	GCMissingTolerance time.Duration
+	GCDropTolerance    time.Duration
 }
 
 // Params is a package scoped variable of type ParamTable.
@@ -130,6 +135,11 @@ func (p *ParamTable) Init() {
 	p.initMinioRootPath()
 
 	p.initCompactionRetentionDuration()
+
+	p.initEnableGarbageCollection()
+	p.initGCInterval()
+	p.initGCMissingTolerance()
+	p.initGCDropTolerance()
 }
 
 // InitOnce ensures param table is a singleton
@@ -298,10 +308,24 @@ func (p *ParamTable) initEnableCompaction() {
 	p.EnableCompaction = p.ParseBool("dataCoord.enableCompaction", false)
 }
 
+// -- GC --
+
 func (p *ParamTable) initEnableGarbageCollection() {
 	p.EnableGarbageCollection = p.ParseBool("dataCoord.enableGarbageCollection", false)
 }
 
+func (p *ParamTable) initGCInterval() {
+	p.GCInterval = time.Duration(p.ParseInt64WithDefault("dataCoord.gc.interval", 60*60)) * time.Second
+}
+
+func (p *ParamTable) initGCMissingTolerance() {
+	p.GCMissingTolerance = time.Duration(p.ParseInt64WithDefault("dataCoord.gc.missingTolerance", 24*60*60)) * time.Second
+}
+
+func (p *ParamTable) initGCDropTolerance() {
+	p.GCDropTolerance = time.Duration(p.ParseInt64WithDefault("dataCoord.gc.dropTolerance", 24*60*60)) * time.Second
+}
+
 // --- MinIO ---
 func (p *ParamTable) initMinioAddress() {
 	endpoint, err := p.Load("_MinioAddress")

internal/datacoord/server.go

@@ -336,9 +336,9 @@ func (s *Server) initGarbageCollection() error {
 		bucketName: Params.MinioBucketName,
 		rootPath:   Params.MinioRootPath,
 
-		checkInterval:    defaultGcInterval,
-		missingTolerance: defaultMissingTolerance,
-		dropTolerance:    defaultMissingTolerance,
+		checkInterval:    Params.GCInterval,
+		missingTolerance: Params.GCMissingTolerance,
+		dropTolerance:    Params.GCDropTolerance,
 	})
 	return nil
 }