milvus-io
/
milvus
mirror of https://github.com/milvus-io/milvus.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Add time tick in wite node 

Signed-off-by: neza2017 <yefu.chen@zilliz.com>
pull/4973/head^2
neza2017 2020-12-26 17:05:24 +08:00 committed by yefu.chen
parent e316533a3a
commit 7e8b203fe2
3 changed files with 46 additions and 16 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

58
internal/writenode/flow_graph_insert_buffer_node.go
View File

@ -16,7 +16,9 @@ import (
"github.com/zilliztech/milvus-distributed/internal/kv" "github.com/zilliztech/milvus-distributed/internal/kv"
etcdkv "github.com/zilliztech/milvus-distributed/internal/kv/etcd" etcdkv "github.com/zilliztech/milvus-distributed/internal/kv/etcd"
miniokv "github.com/zilliztech/milvus-distributed/internal/kv/minio" miniokv "github.com/zilliztech/milvus-distributed/internal/kv/minio"
"github.com/zilliztech/milvus-distributed/internal/msgstream"
"github.com/zilliztech/milvus-distributed/internal/proto/etcdpb" "github.com/zilliztech/milvus-distributed/internal/proto/etcdpb"
"github.com/zilliztech/milvus-distributed/internal/proto/internalpb"
"github.com/zilliztech/milvus-distributed/internal/proto/schemapb" "github.com/zilliztech/milvus-distributed/internal/proto/schemapb"
"github.com/zilliztech/milvus-distributed/internal/storage" "github.com/zilliztech/milvus-distributed/internal/storage"
"github.com/zilliztech/milvus-distributed/internal/util/typeutil" "github.com/zilliztech/milvus-distributed/internal/util/typeutil"
@ -34,12 +36,13 @@ type (
insertBufferNode struct { insertBufferNode struct {
BaseNode BaseNode
kvClient *etcdkv.EtcdKV kvClient *etcdkv.EtcdKV
insertBuffer *insertBuffer insertBuffer *insertBuffer
minIOKV kv.Base minIOKV kv.Base
minioPrifex string minioPrifex string
idAllocator *allocator.IDAllocator idAllocator *allocator.IDAllocator
outCh chan *insertFlushSyncMsg outCh chan *insertFlushSyncMsg
pulsarWriteNodeTimeTickStream *msgstream.PulsarMsgStream
} }
insertBuffer struct { insertBuffer struct {
@ -426,11 +429,33 @@ func (ibNode *insertBufferNode) Operate(in []*Msg) []*Msg {
// Return // Return
} }
if err := ibNode.writeHardTimeTick(iMsg.timeRange.timestampMax); err != nil {
log.Printf("Error: send hard time tick into pulsar channel failed, %s\n", err.Error())
}
return nil return nil
} }
func newInsertBufferNode(ctx context.Context, outCh chan *insertFlushSyncMsg) *insertBufferNode { func (ibNode *insertBufferNode) writeHardTimeTick(ts Timestamp) error {
msgPack := msgstream.MsgPack{}
timeTickMsg := msgstream.TimeTickMsg{
BaseMsg: msgstream.BaseMsg{
BeginTimestamp: ts,
EndTimestamp: ts,
HashValues: []uint32{0},
},
TimeTickMsg: internalpb.TimeTickMsg{
MsgType: internalpb.MsgType_kTimeTick,
PeerID: Params.WriteNodeID,
Timestamp: ts,
},
}
msgPack.Msgs = append(msgPack.Msgs, &timeTickMsg)
return ibNode.pulsarWriteNodeTimeTickStream.Produce(&msgPack)
}
func newInsertBufferNode(ctx context.Context, outCh chan *insertFlushSyncMsg) *insertBufferNode {
maxQueueLength := Params.FlowGraphMaxQueueLength maxQueueLength := Params.FlowGraphMaxQueueLength
maxParallelism := Params.FlowGraphMaxParallelism maxParallelism := Params.FlowGraphMaxParallelism
@ -482,13 +507,18 @@ func newInsertBufferNode(ctx context.Context, outCh chan *insertFlushSyncMsg) *i
panic(err) panic(err)
} }
wTt := msgstream.NewPulsarMsgStream(ctx, 1024) //input stream, write node time tick
wTt.SetPulsarClient(Params.PulsarAddress)
wTt.CreatePulsarProducers([]string{Params.WriteNodeTimeTickChannelName})
return &insertBufferNode{ return &insertBufferNode{
BaseNode: baseNode, BaseNode: baseNode,
kvClient: kvClient, kvClient: kvClient,
insertBuffer: iBuffer, insertBuffer: iBuffer,
minIOKV: minIOKV, minIOKV: minIOKV,
minioPrifex: minioPrefix, minioPrifex: minioPrefix,
idAllocator: idAllocator, idAllocator: idAllocator,
outCh: outCh, outCh: outCh,
pulsarWriteNodeTimeTickStream: wTt,
} }
} }

2
internal/writenode/param_table.go
View File

@ -260,7 +260,7 @@ func (p *ParamTable) initWriteNodeTimeTickChannelName() {
if err != nil { if err != nil {
panic(err) panic(err)
} }
p.WriteNodeTimeTickChannelName = channels p.WriteNodeTimeTickChannelName = channels + "-" + strconv.FormatInt(p.WriteNodeID, 10)
} }
func (p *ParamTable) initSliceIndex() { func (p *ParamTable) initSliceIndex() {

2
internal/writenode/param_table_test.go
View File

@ -64,7 +64,7 @@ func TestParamTable_WriteNode(t *testing.T) {
t.Run("Test timeTickChannelName", func(t *testing.T) { t.Run("Test timeTickChannelName", func(t *testing.T) {
name := Params.WriteNodeTimeTickChannelName name := Params.WriteNodeTimeTickChannelName
assert.Equal(t, name, "writeNodeTimeTick") assert.Equal(t, name, "writeNodeTimeTick-3")
}) })
t.Run("Test minioAccessKeyID", func(t *testing.T) { t.Run("Test minioAccessKeyID", func(t *testing.T) {