Improve logic when watch dm channel fails (#8386)

Signed-off-by: Congqi Xia <congqi.xia@zilliz.com>

internal/datanode/data_node.go

@@ -405,7 +405,12 @@ func (node *DataNode) WatchDmChannels(ctx context.Context, in *datapb.WatchDmCha
 				zap.Any("channal Info", chanInfo),
 			)
 			if err := node.NewDataSyncService(chanInfo); err != nil {
-				log.Warn("Failed to new data sync service", zap.Any("channel", chanInfo))
+				log.Warn("Failed to new data sync service",
+					zap.Any("channel", chanInfo),
+					zap.Error(err))
+				// return error even partial success
+				status.Reason = err.Error()
+				return status, nil
 			}
 		}
 

internal/datanode/data_node_test.go

@@ -120,6 +120,41 @@ func TestDataNode(t *testing.T) {
 		}
 	})
 
+	t.Run("Test WatchDmChannels fails", func(t *testing.T) {
+		ctx, cancel := context.WithCancel(context.Background())
+		defer cancel()
+
+		node := newIDLEDataNodeMock(ctx)
+
+		// before healthy
+		status, err := node.WatchDmChannels(ctx, &datapb.WatchDmChannelsRequest{})
+		assert.Nil(t, err)
+		assert.Equal(t, commonpb.ErrorCode_UnexpectedError, status.ErrorCode)
+
+		node.Init()
+		node.Start()
+		node.Register()
+		defer func() {
+			err := node.Stop()
+			assert.Nil(t, err)
+		}()
+
+		node.msFactory = &FailMessageStreamFactory{
+			Factory: node.msFactory,
+		}
+
+		status, err = node.WatchDmChannels(ctx, &datapb.WatchDmChannelsRequest{
+			Vchannels: []*datapb.VchannelInfo{
+				{
+					CollectionID: collectionID0,
+					ChannelName:  "test_channel_name",
+				},
+			},
+		})
+		assert.Nil(t, err)
+		assert.Equal(t, commonpb.ErrorCode_UnexpectedError, status.ErrorCode)
+	})
+
 	t.Run("Test GetComponentStates", func(t *testing.T) {
 		stat, err := node.GetComponentStates(node.ctx)
 		assert.NoError(t, err)
@@ -505,4 +540,8 @@ func TestWatchChannel(t *testing.T) {
 		node.chanMut.RUnlock()
 		assert.False(t, has)
 	})
+
+	t.Run("watch dm channel fails", func(t *testing.T) {
+		node.WatchDmChannels(context.Background(), &datapb.WatchDmChannelsRequest{})
+	})
 }

internal/datanode/data_sync_service.go

@@ -163,13 +163,17 @@ func (dsService *dataSyncService) initNodes(vchanInfo *datapb.VchannelInfo) erro
 
 	dsService.saveBinlog = saveBinlog
 
-	var dmStreamNode Node = newDmInputNode(
+	var dmStreamNode Node
+	dmStreamNode, err = newDmInputNode(
 		dsService.ctx,
 		dsService.msFactory,
 		vchanInfo.CollectionID,
 		vchanInfo.GetChannelName(),
 		vchanInfo.GetSeekPosition(),
 	)
+	if err != nil {
+		return err
+	}
 	var ddNode Node = newDDNode(dsService.clearSignal, dsService.collectionID, vchanInfo)
 	var insertBufferNode Node
 	insertBufferNode, err = newInsertBufferNode(

internal/datanode/flow_graph_dmstream_input_node.go

@@ -22,13 +22,16 @@ import (
 	"github.com/milvus-io/milvus/internal/util/flowgraph"
 )
 
-func newDmInputNode(ctx context.Context, factory msgstream.Factory, collID UniqueID, chanName string, seekPos *internalpb.MsgPosition) *flowgraph.InputNode {
+func newDmInputNode(ctx context.Context, factory msgstream.Factory, collID UniqueID, chanName string, seekPos *internalpb.MsgPosition) (*flowgraph.InputNode, error) {
 	maxQueueLength := Params.FlowGraphMaxQueueLength
 	maxParallelism := Params.FlowGraphMaxParallelism
 
 	// subName should be unique, since pchannelName is shared among several collections
 	consumeSubName := Params.MsgChannelSubName + "-" + strconv.FormatInt(collID, 10)
-	insertStream, _ := factory.NewTtMsgStream(ctx)
+	insertStream, err := factory.NewTtMsgStream(ctx)
+	if err != nil {
+		return nil, err
+	}
 
 	pchannelName := rootcoord.ToPhysicalChannel(chanName)
 	insertStream.AsConsumer([]string{pchannelName}, consumeSubName)
@@ -43,5 +46,5 @@ func newDmInputNode(ctx context.Context, factory msgstream.Factory, collID Uniqu
 
 	var stream msgstream.MsgStream = insertStream
 	node := flowgraph.NewInputNode(&stream, "dmInputNode", maxQueueLength, maxParallelism)
-	return node
+	return node, nil
 }

internal/datanode/mock_test.go

@@ -543,3 +543,16 @@ func (m *RootCoordFactory) GetComponentStates(ctx context.Context) (*internalpb.
 		},
 	}, nil
 }
+
+// FailMessageStreamFactory mock MessageStreamFactory failure
+type FailMessageStreamFactory struct {
+	msgstream.Factory
+}
+
+func (f *FailMessageStreamFactory) NewMsgStream(ctx context.Context) (msgstream.MsgStream, error) {
+	return nil, errors.New("mocked failure")
+}
+
+func (f *FailMessageStreamFactory) NewTtMsgStream(ctx context.Context) (msgstream.MsgStream, error) {
+	return nil, errors.New("mocked failure")
+}