milvus-io
/
milvus
mirror of https://github.com/milvus-io/milvus.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Split small segment proportion and small segment compactable proportion (#21012) 

/kind improvement

Signed-off-by: Yuchen Gao <yuchen.gao@zilliz.com>

Signed-off-by: Yuchen Gao <yuchen.gao@zilliz.com>
pull/21029/head
Ten Thousand Leaves 2022-12-07 10:15:19 +08:00 committed by GitHub
parent b8be79fe27
commit 3e62688eb6
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 11 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
configs/milvus.yaml
View File

@ -273,6 +273,10 @@ dataCoord:
# `minSizeFromIdleToSealed`, Milvus will automatically seal it.
maxIdleTime: 600 # The max idle time of segment in seconds, 10*60.
minSizeFromIdleToSealed: 16 # The min size in MB of segment which can be idle from sealed.
smallProportion: 0.5 # The segment is considered as "small segment" when its # of rows is smaller than
# (smallProportion * segment max # of rows).
compactableProportion: 0.5 # A compaction will happen on small segments if the segment after compaction will have
# over (compactableProportion * segment max # of rows) rows.
compaction:
enableAutoCompaction: true

2
internal/datacoord/compaction_trigger.go
View File

@ -533,7 +533,7 @@ func (t *compactionTrigger) generatePlans(segments []*SegmentInfo, force bool, c
targetRow += s.GetNumOfRows()
}
// only merge if candidate number is large than MinSegmentToMerge or if target row is large enough
if len(bucket) >= Params.DataCoordCfg.MinSegmentToMerge || targetRow > int64(float64(segment.GetMaxRowNum())*Params.DataCoordCfg.SegmentSmallProportion) {
if len(bucket) >= Params.DataCoordCfg.MinSegmentToMerge || targetRow > int64(float64(segment.GetMaxRowNum())*Params.DataCoordCfg.SegmentCompactableProportion) {
plan := segmentsToPlan(bucket, compactTime)
log.Info("generate a plan for small candidates", zap.Any("plan", plan),
zap.Int64("target segment row", targetRow), zap.Int64("target segment size", size))

6
internal/util/paramtable/component_param.go
View File

@ -1194,6 +1194,7 @@ type dataCoordConfig struct {
MinSegmentToMerge int
MaxSegmentToMerge int
SegmentSmallProportion float64
SegmentCompactableProportion float64
CompactionTimeoutInSeconds int32
CompactionCheckIntervalInSeconds int64
SingleCompactionRatioThreshold float32
@ -1228,6 +1229,7 @@ func (p *dataCoordConfig) init(base *BaseTable) {
p.initCompactionMinSegment()
p.initCompactionMaxSegment()
p.initSegmentSmallProportion()
p.initSegmentCompactableProportion()
p.initCompactionTimeoutInSeconds()
p.initCompactionCheckIntervalInSeconds()
p.initSingleCompactionRatioThreshold()
@ -1299,6 +1301,10 @@ func (p *dataCoordConfig) initSegmentSmallProportion() {
p.SegmentSmallProportion = p.Base.ParseFloatWithDefault("dataCoord.segment.smallProportion", 0.5)
}
func (p *dataCoordConfig) initSegmentCompactableProportion() {
p.SegmentCompactableProportion = p.Base.ParseFloatWithDefault("dataCoord.segment.compactableProportion", 0.5)
}
// compaction execution timeout
func (p *dataCoordConfig) initCompactionTimeoutInSeconds() {
p.CompactionTimeoutInSeconds = p.Base.ParseInt32WithDefault("dataCoord.compaction.timeout", 60*3)