Reduce the number of retries and add error log (#16754)

Signed-off-by: cai.zhang <cai.zhang@zilliz.com>

internal/datacoord/server.go

@@ -55,7 +55,7 @@ import (
 )
 
 const (
-	connEtcdMaxRetryTime = 100000
+	connEtcdMaxRetryTime = 100
 	allPartitionID       = 0 // paritionID means no filtering
 )
 

internal/datanode/data_node.go

@@ -76,7 +76,7 @@ const (
 	MetricRequestsSuccess = "success"
 
 	// ConnectEtcdMaxRetryTime is used to limit the max retry time for connection etcd
-	ConnectEtcdMaxRetryTime = 1000
+	ConnectEtcdMaxRetryTime = 100
 )
 
 const illegalRequestErrStr = "Illegal request"

internal/distributed/rootcoord/service_test.go

@@ -948,7 +948,7 @@ func initEtcd(etcdEndpoints []string) (*clientv3.Client, error) {
 		etcdCli = etcd
 		return nil
 	}
-	err := retry.Do(context.TODO(), connectEtcdFn, retry.Attempts(300))
+	err := retry.Do(context.TODO(), connectEtcdFn, retry.Attempts(100))
 	if err != nil {
 		return nil, err
 	}

internal/indexcoord/index_coord.go

@@ -183,7 +183,7 @@ func (i *IndexCoord) Init() error {
 			return err
 		}
 		log.Debug("IndexCoord try to connect etcd")
-		err = retry.Do(i.loopCtx, connectEtcdFn, retry.Attempts(300))
+		err = retry.Do(i.loopCtx, connectEtcdFn, retry.Attempts(100))
 		if err != nil {
 			log.Error("IndexCoord try to connect etcd failed", zap.Error(err))
 			initErr = err

internal/rootcoord/root_coord.go

@@ -1074,7 +1074,7 @@ func (c *Core) Init() error {
 			return nil
 		}
 		log.Debug("RootCoord, Connecting to Etcd", zap.String("kv root", Params.EtcdCfg.KvRootPath), zap.String("meta root", Params.EtcdCfg.MetaRootPath))
-		err := retry.Do(c.ctx, connectEtcdFn, retry.Attempts(300))
+		err := retry.Do(c.ctx, connectEtcdFn, retry.Attempts(100))
 		if err != nil {
 			return
 		}

internal/storage/minio_chunk_manager.go

@@ -81,7 +81,7 @@ func newMinioChunkManagerWithConfig(ctx context.Context, c *config) (*MinioChunk
 		}
 		return nil
 	}
-	err = retry.Do(ctx, checkBucketFn, retry.Attempts(300))
+	err = retry.Do(ctx, checkBucketFn, retry.Attempts(100))
 	if err != nil {
 		return nil, err
 	}

internal/util/retry/retry.go

@@ -15,6 +15,9 @@ import (
 	"context"
 	"time"
 
+	"go.uber.org/zap"
+
+	"github.com/milvus-io/milvus/internal/log"
 	"github.com/milvus-io/milvus/internal/util/errorutil"
 )
 
@@ -32,6 +35,9 @@ func Do(ctx context.Context, fn func() error, opts ...Option) error {
 
 	for i := uint(0); i < c.attempts; i++ {
 		if err := fn(); err != nil {
+			if i%10 == 0 {
+				log.Debug("retry func failed", zap.Uint("retry time", i), zap.Error(err))
+			}
 
 			el = append(el, err)
 

internal/util/sessionutil/session_util.go

@@ -92,7 +92,7 @@ func NewSession(ctx context.Context, metaRoot string, client *clientv3.Client) *
 		session.etcdCli = client
 		return nil
 	}
-	err := retry.Do(ctx, connectEtcdFn, retry.Attempts(300))
+	err := retry.Do(ctx, connectEtcdFn, retry.Attempts(100))
 	if err != nil {
 		log.Warn("failed to initialize session",
 			zap.Error(err))
@@ -241,7 +241,7 @@ func (s *Session) registerService() (<-chan *clientv3.LeaseKeepAliveResponse, er
 		log.Debug("Session register successfully", zap.Int64("ServerID", s.ServerID))
 		return nil
 	}
-	err := retry.Do(s.ctx, registerFn, retry.Attempts(DefaultRetryTimes), retry.Sleep(500*time.Millisecond))
+	err := retry.Do(s.ctx, registerFn, retry.Attempts(DefaultRetryTimes))
 	if err != nil {
 		return nil, err
 	}