enhance: Add deltalog expansion rate in segment loader (#30704)

See also #30191

It turned out that in auto id and batch delete scenario actual memory
size of deltalog maybe way larger than deltalog file size. This PR add a
configurable expansion rate for deltalog memory usage to prevent
out-of-memory panicking during loading deltalogs.

Signed-off-by: Congqi Xia <congqi.xia@zilliz.com>

internal/querynodev2/segments/segment_loader.go

@@ -1442,7 +1442,7 @@ func (loader *segmentLoader) checkSegmentSize(ctx context.Context, segmentLoadIn
 
 		// get size of delete data
 		for _, fieldBinlog := range loadInfo.Deltalogs {
-			predictMemUsage += uint64(getBinlogDataSize(fieldBinlog))
+			predictMemUsage += uint64(float64(getBinlogDataSize(fieldBinlog)) * paramtable.Get().QueryNodeCfg.DeltaDataExpansionRate.GetAsFloat())
 		}
 
 		if predictMemUsage-oldUsedMem > maxSegmentSize {

pkg/util/paramtable/component_param.go

@@ -1930,7 +1930,8 @@ type queryNodeConfig struct {
 	DeleteBufferBlockSize  ParamItem `refreshable:"false"`
 
 	// loader
-	IoPoolSize ParamItem `refreshable:"false"`
+	IoPoolSize             ParamItem `refreshable:"false"`
+	DeltaDataExpansionRate ParamItem `refreshable:"true"`
 
 	// schedule task policy.
 	SchedulePolicyName                    ParamItem `refreshable:"false"`
@@ -2314,6 +2315,14 @@ Max read concurrency must greater than or equal to 1, and less than or equal to
 	}
 	p.IoPoolSize.Init(base.mgr)
 
+	p.DeltaDataExpansionRate = ParamItem{
+		Key:          "querynode.deltaDataExpansionRate",
+		Version:      "2.4.0",
+		DefaultValue: "50",
+		Doc:          "the expansion rate for deltalog physical size to actual memory usage",
+	}
+	p.DeltaDataExpansionRate.Init(base.mgr)
+
 	// schedule read task policy.
 	p.SchedulePolicyName = ParamItem{
 		Key:          "queryNode.scheduler.scheduleReadPolicy.name",