fix retry on unimplemented rpc error (#27639)

Signed-off-by: Wei Liu <wei.liu@zilliz.com>

internal/util/grpcclient/client.go

@@ -110,7 +110,8 @@ type ClientBase[T interface {
 
 func NewClientBase[T interface {
 	GetComponentStates(ctx context.Context, in *milvuspb.GetComponentStatesRequest, opts ...grpc.CallOption) (*milvuspb.ComponentStates, error)
-}](config *paramtable.GrpcClientConfig, serviceName string) *ClientBase[T] {
+}](config *paramtable.GrpcClientConfig, serviceName string,
+) *ClientBase[T] {
 	return &ClientBase[T]{
 		ClientMaxRecvSize:       config.ClientMaxRecvSize.GetAsInt(),
 		ClientMaxSendSize:       config.ClientMaxSendSize.GetAsInt(),
@@ -368,6 +369,10 @@ func (c *ClientBase[T]) checkErr(ctx context.Context, err error) (needRetry, nee
 			// canceled or deadline exceeded
 			return true, c.needResetCancel()
 		}
+
+		if funcutil.IsGrpcErr(err, codes.Unimplemented) {
+			return false, false
+		}
 		return true, true
 	case IsServerIDMismatchErr(err):
 		fallthrough

internal/util/grpcclient/client_test.go

@@ -309,6 +309,21 @@ func TestClientBase_Recall(t *testing.T) {
 	})
 }
 
+func TestClientBase_CheckError(t *testing.T) {
+	base := ClientBase[*mockClient]{}
+	base.grpcClient = &mockClient{}
+	base.MaxAttempts = 1
+
+	ctx := context.Background()
+	retry, reset := base.checkErr(ctx, status.Errorf(codes.Canceled, "fake context canceled"))
+	assert.True(t, retry)
+	assert.True(t, reset)
+
+	retry, reset = base.checkErr(ctx, status.Errorf(codes.Unimplemented, "fake context canceled"))
+	assert.False(t, retry)
+	assert.False(t, reset)
+}
+
 type server struct {
 	helloworld.UnimplementedGreeterServer
 	reqCounter   uint