Increase default ttProtection max delay (#20125)

Signed-off-by: bigsheeper <yihao.dai@zilliz.com>

Signed-off-by: bigsheeper <yihao.dai@zilliz.com>

configs/milvus.yaml

@@ -435,7 +435,7 @@ quotaAndLimits:
       # maxTimeTickDelay indicates the backpressure for DML Operations.
       # DML rates would be reduced according to the ratio of time tick delay to maxTimeTickDelay,
       # if time tick delay is greater than maxTimeTickDelay, all DML requests would be rejected.
-      maxTimeTickDelay: 30 # in seconds
+      maxTimeTickDelay: 300 # in seconds
     memProtection:
       enabled: true
       # When memory usage > memoryHighWaterLevel, all dml requests would be rejected;

internal/datanode/data_node.go

@@ -772,7 +772,7 @@ func (node *DataNode) GetMetrics(ctx context.Context, req *milvuspb.GetMetricsRe
 		return &milvuspb.GetMetricsResponse{
 			Status: &commonpb.Status{
 				ErrorCode: commonpb.ErrorCode_UnexpectedError,
-				Reason:    err.Error(),
+				Reason:    fmt.Sprintf("datanode GetMetrics failed, nodeID=%d, err=%s", Params.DataNodeCfg.GetNodeID(), err.Error()),
 			},
 		}, nil
 	}
@@ -784,7 +784,7 @@ func (node *DataNode) GetMetrics(ctx context.Context, req *milvuspb.GetMetricsRe
 			return &milvuspb.GetMetricsResponse{
 				Status: &commonpb.Status{
 					ErrorCode: commonpb.ErrorCode_UnexpectedError,
-					Reason:    err.Error(),
+					Reason:    fmt.Sprintf("datanode GetMetrics failed, nodeID=%d, err=%s", Params.DataNodeCfg.GetNodeID(), err.Error()),
 				},
 			}, nil
 		}

internal/metrics/proxy_metrics.go

@@ -19,13 +19,10 @@ package metrics
 import (
 	"strconv"
 
-	"github.com/prometheus/client_golang/prometheus"
-	"go.uber.org/zap"
-
-	"github.com/milvus-io/milvus/internal/log"
 	"github.com/milvus-io/milvus/internal/proto/internalpb"
 	"github.com/milvus-io/milvus/internal/util/ratelimitutil"
 	"github.com/milvus-io/milvus/internal/util/typeutil"
+	"github.com/prometheus/client_golang/prometheus"
 )
 
 var (
@@ -269,7 +266,9 @@ func SetRateGaugeByRateType(rateType internalpb.RateType, nodeID int64, rate flo
 		return
 	}
 	nodeIDStr := strconv.FormatInt(nodeID, 10)
-	log.Debug("set rates", zap.Int64("nodeID", nodeID), zap.String("rateType", rateType.String()), zap.Float64("rate", rate))
+	//log.Debug("set rates", zap.Int64("nodeID", nodeID),
+	//	zap.String("rateType", rateType.String()),
+	//	zap.String("rate", fmt.Sprintf("%v", rate)))
 	switch rateType {
 	case internalpb.RateType_DMLInsert:
 		ProxyLimiterRate.WithLabelValues(nodeIDStr, InsertLabel).Set(rate)

internal/proxy/multi_rate_limiter.go

@@ -90,7 +90,7 @@ func (rl *rateLimiter) setRates(rates []*internalpb.Rate) error {
 func (rl *rateLimiter) printRates(rates []*internalpb.Rate) {
 	//fmt.Printf("RateLimiter set rates:\n---------------------------------\n")
 	//for _, r := range rates {
-	//	fmt.Printf("%s -> %f\n", r.GetRt().String(), r.GetR())
+	//	fmt.Printf("%s -> %v\n", r.GetRt().String(), r.GetR())
 	//}
 	//fmt.Printf("---------------------------------\n")
 	log.Debug("RateLimiter setRates", zap.Any("rates", rates))

internal/rootcoord/quota_center.go

@@ -468,7 +468,18 @@ func (q *QuotaCenter) getTimeTickDelayFactor(ts Timestamp) float64 {
 			zap.Any("DataNodeMetrics", q.dataNodeMetrics))
 		return 0
 	}
-	return float64(maxDelay.Nanoseconds()-curMaxDelay.Nanoseconds()) / float64(maxDelay.Nanoseconds())
+	factor := float64(maxDelay.Nanoseconds()-curMaxDelay.Nanoseconds()) / float64(maxDelay.Nanoseconds())
+	if factor <= 0.9 {
+		log.Warn("QuotaCenter: limit writing due to long timeTick delay",
+			zap.String("node", role),
+			zap.String("vchannel", vchannel),
+			zap.Time("curTs", t1),
+			zap.Time("minTs", minTt),
+			zap.Duration("delay", curMaxDelay),
+			zap.Duration("MaxDelay", maxDelay),
+			zap.Float64("factor", factor))
+	}
+	return factor
 }
 
 // getNQInQueryFactor checks search&query nq in QueryNode,

internal/util/paramtable/quota_param.go

@@ -17,6 +17,7 @@
 package paramtable
 
 import (
+	"fmt"
 	"math"
 	"sync"
 	"time"
@@ -257,7 +258,7 @@ func megaBytes2Bytes(f float64) float64 {
 func (p *quotaConfig) checkMinMaxLegal(min, max float64) bool {
 	if min > max {
 		log.Warn("init QuotaConfig failed, max/high must be greater than or equal to min/low, use default values",
-			zap.Float64("min", min), zap.Float64("max", max), zap.Float64("defaultMin", defaultMin), zap.Float64("defaultMax", defaultMax))
+			zap.String("msg", fmt.Sprintf("min: %v, max: %v, defaultMin: %v, defaultMax: %v", min, max, defaultMin, defaultMax)))
 		return false
 	}
 	return true
@@ -440,7 +441,7 @@ func (p *quotaConfig) initMaxTimeTickDelay() {
 		p.MaxTimeTickDelay = math.MaxInt64
 		return
 	}
-	const defaultMaxTtDelay = 30.0
+	const defaultMaxTtDelay = 300.0
 	delay := p.Base.ParseFloatWithDefault("quotaAndLimits.limitWriting.ttProtection.maxTimeTickDelay", defaultMaxTtDelay)
 	// (0, 65536)
 	if delay <= 0 || delay >= 65536 {
@@ -526,7 +527,7 @@ func (p *quotaConfig) initDiskQuota() {
 		p.DiskQuota = defaultDiskQuotaInMB
 	}
 	if p.DiskQuota < defaultDiskQuotaInMB {
-		log.Debug("init disk quota", zap.Float64("diskQuota(MB)", p.DiskQuota))
+		log.Debug("init disk quota", zap.String("diskQuota(MB)", fmt.Sprintf("%v", p.DiskQuota)))
 	} else {
 		log.Debug("init disk quota", zap.String("diskQuota(MB)", "+inf"))
 	}

internal/util/paramtable/quota_param_test.go

@@ -73,7 +73,7 @@ func TestQuotaParam(t *testing.T) {
 	t.Run("test limit writing", func(t *testing.T) {
 		assert.False(t, qc.ForceDenyWriting)
 		assert.Equal(t, true, qc.TtProtectionEnabled)
-		assert.Equal(t, 30*time.Second, qc.MaxTimeTickDelay)
+		assert.Equal(t, 300*time.Second, qc.MaxTimeTickDelay)
 		assert.Equal(t, defaultLowWaterLevel, qc.DataNodeMemoryLowWaterLevel)
 		assert.Equal(t, defaultHighWaterLevel, qc.DataNodeMemoryHighWaterLevel)
 		assert.Equal(t, defaultLowWaterLevel, qc.QueryNodeMemoryLowWaterLevel)

internal/util/ratelimitutil/limiter.go

@@ -138,7 +138,7 @@ func (limit Limit) String() string {
 	if limit == Inf {
 		return "+inf"
 	}
-	return fmt.Sprintf("%.4f", limit)
+	return fmt.Sprintf("%v", float64(limit))
 }
 
 // tokensFromDuration is a unit conversion function from a time duration to the number of tokens