milvus-io
/
milvus
mirror of https://github.com/milvus-io/milvus.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

enhance: Send flush signal when the water level reaches the high watermark (#34907) 

issue: #30633

Signed-off-by: Cai Zhang <cai.zhang@zilliz.com>
pull/34913/head
cai.zhang 2024-07-23 10:29:42 +08:00 committed by GitHub
parent 575ce91039
commit 260a6e2ba9
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 5 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
internal/datanode/compaction/clustering_compactor.go
View File

@ -188,7 +188,7 @@ func (t *clusteringCompactionTask) init() error {
segIDAlloc := allocator.NewLocalAllocator(t.plan.GetPreAllocatedSegments().GetBegin(), t.plan.GetPreAllocatedSegments().GetEnd())
t.logIDAlloc = logIDAlloc
t.segIDAlloc = segIDAlloc
var pkField *schemapb.FieldSchema
if t.plan.Schema == nil {
return merr.WrapErrIllegalCompactionPlan("empty schema in compactionPlan")
@ -616,7 +616,7 @@ func (t *clusteringCompactionTask) mappingSegment(
pack: pack,
id: clusterBuffer.id,
}
} else if currentBufferTotalMemorySize > t.getMemoryBufferBlockFlushThreshold() && !t.hasSignal.Load() {
} else if currentBufferTotalMemorySize > t.getMemoryBufferHighWatermark() && !t.hasSignal.Load() {
// reach flushBinlog trigger threshold
log.Debug("largest buffer need to flush",
zap.Int64("currentBufferTotalMemorySize", currentBufferTotalMemorySize))
@ -640,8 +640,8 @@ func (t *clusteringCompactionTask) mappingSegment(
default:
// currentSize := t.getCurrentBufferWrittenMemorySize()
currentSize := t.getBufferTotalUsedMemorySize()
if currentSize < t.getMemoryBufferBlockFlushThreshold() {
log.Debug("memory is already below the block watermark, continue writing",
if currentSize < t.getMemoryBufferHighWatermark() {
log.Debug("memory is already below the high watermark, continue writing",
zap.Int64("currentSize", currentSize))
break loop
}
@ -693,7 +693,7 @@ func (t *clusteringCompactionTask) getMemoryBufferLowWatermark() int64 {
}
func (t *clusteringCompactionTask) getMemoryBufferHighWatermark() int64 {
return int64(float64(t.memoryBufferSize) * 0.9)
return int64(float64(t.memoryBufferSize) * 0.7)
}
func (t *clusteringCompactionTask) getMemoryBufferBlockFlushThreshold() int64 {