diff --git a/configs/milvus.yaml b/configs/milvus.yaml index aafa3e84f0..ce872672b5 100644 --- a/configs/milvus.yaml +++ b/configs/milvus.yaml @@ -273,6 +273,10 @@ dataCoord: # `minSizeFromIdleToSealed`, Milvus will automatically seal it. maxIdleTime: 600 # The max idle time of segment in seconds, 10*60. minSizeFromIdleToSealed: 16 # The min size in MB of segment which can be idle from sealed. + smallProportion: 0.5 # The segment is considered as "small segment" when its # of rows is smaller than + # (smallProportion * segment max # of rows). + compactableProportion: 0.5 # A compaction will happen on small segments if the segment after compaction will have + # over (compactableProportion * segment max # of rows) rows. compaction: enableAutoCompaction: true diff --git a/internal/datacoord/compaction_trigger.go b/internal/datacoord/compaction_trigger.go index c557f3f049..0c96374cd4 100644 --- a/internal/datacoord/compaction_trigger.go +++ b/internal/datacoord/compaction_trigger.go @@ -533,7 +533,7 @@ func (t *compactionTrigger) generatePlans(segments []*SegmentInfo, force bool, c targetRow += s.GetNumOfRows() } // only merge if candidate number is large than MinSegmentToMerge or if target row is large enough - if len(bucket) >= Params.DataCoordCfg.MinSegmentToMerge || targetRow > int64(float64(segment.GetMaxRowNum())*Params.DataCoordCfg.SegmentSmallProportion) { + if len(bucket) >= Params.DataCoordCfg.MinSegmentToMerge || targetRow > int64(float64(segment.GetMaxRowNum())*Params.DataCoordCfg.SegmentCompactableProportion) { plan := segmentsToPlan(bucket, compactTime) log.Info("generate a plan for small candidates", zap.Any("plan", plan), zap.Int64("target segment row", targetRow), zap.Int64("target segment size", size)) diff --git a/internal/util/paramtable/component_param.go b/internal/util/paramtable/component_param.go index 37d6a43620..2238ae7131 100644 --- a/internal/util/paramtable/component_param.go +++ b/internal/util/paramtable/component_param.go @@ -1194,6 +1194,7 @@ type dataCoordConfig struct { MinSegmentToMerge int MaxSegmentToMerge int SegmentSmallProportion float64 + SegmentCompactableProportion float64 CompactionTimeoutInSeconds int32 CompactionCheckIntervalInSeconds int64 SingleCompactionRatioThreshold float32 @@ -1228,6 +1229,7 @@ func (p *dataCoordConfig) init(base *BaseTable) { p.initCompactionMinSegment() p.initCompactionMaxSegment() p.initSegmentSmallProportion() + p.initSegmentCompactableProportion() p.initCompactionTimeoutInSeconds() p.initCompactionCheckIntervalInSeconds() p.initSingleCompactionRatioThreshold() @@ -1299,6 +1301,10 @@ func (p *dataCoordConfig) initSegmentSmallProportion() { p.SegmentSmallProportion = p.Base.ParseFloatWithDefault("dataCoord.segment.smallProportion", 0.5) } +func (p *dataCoordConfig) initSegmentCompactableProportion() { + p.SegmentCompactableProportion = p.Base.ParseFloatWithDefault("dataCoord.segment.compactableProportion", 0.5) +} + // compaction execution timeout func (p *dataCoordConfig) initCompactionTimeoutInSeconds() { p.CompactionTimeoutInSeconds = p.Base.ParseInt32WithDefault("dataCoord.compaction.timeout", 60*3)