diff --git a/docs/developer_guides/appendix_a_basic_components.md b/docs/developer_guides/appendix_a_basic_components.md index 6e6eb3638f..d64e2e0b6c 100644 --- a/docs/developer_guides/appendix_a_basic_components.md +++ b/docs/developer_guides/appendix_a_basic_components.md @@ -70,111 +70,6 @@ etcd: -#### A.3 Message Stream - -``` go -type MsgType uint32 -const { - kInsert MsgType = 400 - kDelete MsgType = 401 - kSearch MsgType = 500 - KSearchResult MsgType = 1000 - - kSegStatistics MsgType = 1100 - - kTimeTick MsgType = 1200 - kTimeSync MsgType = 1201 -} - -type TsMsg interface { - SetTs(ts Timestamp) - BeginTs() Timestamp - EndTs() Timestamp - Type() MsgType - Marshal(*TsMsg) []byte - Unmarshal([]byte) *TsMsg -} - -type MsgPack struct { - BeginTs Timestamp - EndTs Timestamp - Msgs []TsMsg -} - - -type MsgStream interface { - Produce(*MsgPack) error - Broadcast(*MsgPack) error - Consume() *MsgPack // message can be consumed exactly once -} - -type RepackFunc(msgs []* TsMsg, hashKeys [][]int32) map[int32] *MsgPack - -type PulsarMsgStream struct { - client *pulsar.Client - repackFunc RepackFunc - producers []*pulsar.Producer - consumers []*pulsar.Consumer - unmarshal *UnmarshalDispatcher -} - -func (ms *PulsarMsgStream) CreatePulsarProducers(topics []string) -func (ms *PulsarMsgStream) CreatePulsarConsumers(subname string, topics []string, unmarshal *UnmarshalDispatcher) -func (ms *PulsarMsgStream) SetRepackFunc(repackFunc RepackFunc) -func (ms *PulsarMsgStream) Produce(msgs *MsgPack) error -func (ms *PulsarMsgStream) Broadcast(msgs *MsgPack) error -func (ms *PulsarMsgStream) Consume() (*MsgPack, error) -func (ms *PulsarMsgStream) Start() error -func (ms *PulsarMsgStream) Close() error - -func NewPulsarMsgStream(ctx context.Context, pulsarAddr string) *PulsarMsgStream - - -type PulsarTtMsgStream struct { - client *pulsar.Client - repackFunc RepackFunc - producers []*pulsar.Producer - consumers []*pulsar.Consumer - unmarshal *UnmarshalDispatcher - inputBuf []*TsMsg - unsolvedBuf []*TsMsg - msgPacks []*MsgPack -} - -func (ms *PulsarTtMsgStream) CreatePulsarProducers(topics []string) -func (ms *PulsarTtMsgStream) CreatePulsarConsumers(subname string, topics []string, unmarshal *UnmarshalDispatcher) -func (ms *PulsarTtMsgStream) SetRepackFunc(repackFunc RepackFunc) -func (ms *PulsarTtMsgStream) Produce(msgs *MsgPack) error -func (ms *PulsarTtMsgStream) Broadcast(msgs *MsgPack) error -func (ms *PulsarTtMsgStream) Consume() *MsgPack //return messages in one time tick -func (ms *PulsarTtMsgStream) Start() error -func (ms *PulsarTtMsgStream) Close() error - -func NewPulsarTtMsgStream(ctx context.Context, pulsarAddr string) *PulsarTtMsgStream -``` - - - -```go -type MarshalFunc func(*TsMsg) []byte -type UnmarshalFunc func([]byte) *TsMsg - - -type UnmarshalDispatcher struct { - tempMap map[ReqType]UnmarshalFunc -} - -func (dispatcher *MarshalDispatcher) Unmarshal([]byte) *TsMsg -func (dispatcher *MarshalDispatcher) AddMsgTemplate(msgType MsgType, marshal MarshalFunc) -func (dispatcher *MarshalDispatcher) addDefaultMsgTemplates() - -func NewUnmarshalDispatcher() *UnmarshalDispatcher -``` - - - - - #### A.4 Time Ticked Flow Graph ###### A.4.1 Flow Graph States diff --git a/docs/developer_guides/chap03_index_builder.md b/docs/developer_guides/chap03_index_builder.md index b342433681..a3e7495330 100644 --- a/docs/developer_guides/chap03_index_builder.md +++ b/docs/developer_guides/chap03_index_builder.md @@ -6,7 +6,7 @@ #### 8.1 Overview - + #### 8.2 API diff --git a/docs/developer_guides/chap04_message_stream.md b/docs/developer_guides/chap04_message_stream.md index 38db852f0a..19fefbd344 100644 --- a/docs/developer_guides/chap04_message_stream.md +++ b/docs/developer_guides/chap04_message_stream.md @@ -18,8 +18,6 @@ type Client interface { } ``` - - * *CreateChannels* ```go @@ -50,3 +48,109 @@ type ChannelDescriptions struct { } ``` + + + + +#### A.3 Message Stream + +``` go +type MsgType uint32 +const { + kInsert MsgType = 400 + kDelete MsgType = 401 + kSearch MsgType = 500 + KSearchResult MsgType = 1000 + + kSegStatistics MsgType = 1100 + + kTimeTick MsgType = 1200 + kTimeSync MsgType = 1201 +} + +type TsMsg interface { + SetTs(ts Timestamp) + BeginTs() Timestamp + EndTs() Timestamp + Type() MsgType + Marshal(*TsMsg) []byte + Unmarshal([]byte) *TsMsg +} + +type MsgPack struct { + BeginTs Timestamp + EndTs Timestamp + Msgs []TsMsg +} + + +type MsgStream interface { + Produce(*MsgPack) error + Broadcast(*MsgPack) error + Consume() *MsgPack // message can be consumed exactly once +} + +type RepackFunc(msgs []* TsMsg, hashKeys [][]int32) map[int32] *MsgPack + +type PulsarMsgStream struct { + client *pulsar.Client + repackFunc RepackFunc + producers []*pulsar.Producer + consumers []*pulsar.Consumer + unmarshal *UnmarshalDispatcher +} + +func (ms *PulsarMsgStream) CreatePulsarProducers(topics []string) +func (ms *PulsarMsgStream) CreatePulsarConsumers(subname string, topics []string, unmarshal *UnmarshalDispatcher) +func (ms *PulsarMsgStream) SetRepackFunc(repackFunc RepackFunc) +func (ms *PulsarMsgStream) Produce(msgs *MsgPack) error +func (ms *PulsarMsgStream) Broadcast(msgs *MsgPack) error +func (ms *PulsarMsgStream) Consume() (*MsgPack, error) +func (ms *PulsarMsgStream) Start() error +func (ms *PulsarMsgStream) Close() error + +func NewPulsarMsgStream(ctx context.Context, pulsarAddr string) *PulsarMsgStream + + +type PulsarTtMsgStream struct { + client *pulsar.Client + repackFunc RepackFunc + producers []*pulsar.Producer + consumers []*pulsar.Consumer + unmarshal *UnmarshalDispatcher + inputBuf []*TsMsg + unsolvedBuf []*TsMsg + msgPacks []*MsgPack +} + +func (ms *PulsarTtMsgStream) CreatePulsarProducers(topics []string) +func (ms *PulsarTtMsgStream) CreatePulsarConsumers(subname string, topics []string, unmarshal *UnmarshalDispatcher) +func (ms *PulsarTtMsgStream) SetRepackFunc(repackFunc RepackFunc) +func (ms *PulsarTtMsgStream) Produce(msgs *MsgPack) error +func (ms *PulsarTtMsgStream) Broadcast(msgs *MsgPack) error +func (ms *PulsarTtMsgStream) Consume() *MsgPack //return messages in one time tick +func (ms *PulsarTtMsgStream) Start() error +func (ms *PulsarTtMsgStream) Close() error + +func NewPulsarTtMsgStream(ctx context.Context, pulsarAddr string) *PulsarTtMsgStream +``` + + + +```go +type MarshalFunc func(*TsMsg) []byte +type UnmarshalFunc func([]byte) *TsMsg + + +type UnmarshalDispatcher struct { + tempMap map[ReqType]UnmarshalFunc +} + +func (dispatcher *MarshalDispatcher) Unmarshal([]byte) *TsMsg +func (dispatcher *MarshalDispatcher) AddMsgTemplate(msgType MsgType, marshal MarshalFunc) +func (dispatcher *MarshalDispatcher) addDefaultMsgTemplates() + +func NewUnmarshalDispatcher() *UnmarshalDispatcher +``` + + diff --git a/docs/developer_guides/chap05_proxy.md b/docs/developer_guides/chap05_proxy.md index 29a0dec98b..24ec12e94f 100644 --- a/docs/developer_guides/chap05_proxy.md +++ b/docs/developer_guides/chap05_proxy.md @@ -2,6 +2,12 @@ ## 6. Proxy + + + + + + #### 6.0 Proxy Service API ```go @@ -28,8 +34,6 @@ type Client interface { - - #### 6.1 Proxy Instance ```go diff --git a/docs/developer_guides/chap06_master.md b/docs/developer_guides/chap06_master.md index 30508b1f38..28c3b599de 100644 --- a/docs/developer_guides/chap06_master.md +++ b/docs/developer_guides/chap06_master.md @@ -2,7 +2,8 @@ ## 10. Master -, + + #### 10.1 API @@ -28,8 +29,6 @@ type Client interface { - - #### 10.1 Interfaces (RPC) | RPC | description | diff --git a/docs/developer_guides/chap07_query_service.md b/docs/developer_guides/chap07_query_service.md index d30e5e3d17..52295480ab 100644 --- a/docs/developer_guides/chap07_query_service.md +++ b/docs/developer_guides/chap07_query_service.md @@ -6,6 +6,8 @@ #### 8.1 Overview + + #### 8.2 API @@ -120,6 +122,28 @@ type ReleasePartitionRequest struct { +#### 8.2 Query Node + +```go +type QueryNode interface { + Start() error + Close() error + + AddQueryStream(requestStream MsgStream, resultStream MsgStream) error + RemoveQueryStream(requestStreamID string) error + WatchDmStreams(insertStreams MsgStream) error + WatchDdStream(stream MsgStream) error + SetTimeTickStream(stream MsgStream) error + SetStatsStream(stream MsgStream) error + + LoadSegments(DbID UniqueID, CollID UniqueID, PartitionID UniqueID, SegIDs []UniqueID, FieldIDs []int64) error + ReleaseSegments(DbID UniqueID, CollID UniqueID, PartitionID UniqueID, SegIDs []UniqueID) error + DescribeParition(DbID UniqueID, CollID UniqueID, PartitionID UniqueID) (PartitionDescription, error) +} +``` + + + #### 8.2 Collection Replica diff --git a/docs/developer_guides/chap09_data_service.md b/docs/developer_guides/chap09_data_service.md index 2f1a2f3f2c..084ac5a003 100644 --- a/docs/developer_guides/chap09_data_service.md +++ b/docs/developer_guides/chap09_data_service.md @@ -6,6 +6,7 @@ #### 8.1 Overview + #### 8.2 API diff --git a/docs/developer_guides/figs/data_service.jpeg b/docs/developer_guides/figs/data_service.jpeg new file mode 100644 index 0000000000..8cc0dcd4b9 Binary files /dev/null and b/docs/developer_guides/figs/data_service.jpeg differ diff --git a/docs/developer_guides/figs/index_service.jpeg b/docs/developer_guides/figs/index_service.jpeg new file mode 100644 index 0000000000..ec42848549 Binary files /dev/null and b/docs/developer_guides/figs/index_service.jpeg differ diff --git a/docs/developer_guides/figs/local_distributed_cloud.jpeg b/docs/developer_guides/figs/local_distributed_cloud.jpeg new file mode 100644 index 0000000000..476921dc9c Binary files /dev/null and b/docs/developer_guides/figs/local_distributed_cloud.jpeg differ diff --git a/docs/developer_guides/figs/master.jpeg b/docs/developer_guides/figs/master.jpeg new file mode 100644 index 0000000000..00b0adc5e8 Binary files /dev/null and b/docs/developer_guides/figs/master.jpeg differ diff --git a/docs/developer_guides/figs/proxy.jpeg b/docs/developer_guides/figs/proxy.jpeg new file mode 100644 index 0000000000..724849228d Binary files /dev/null and b/docs/developer_guides/figs/proxy.jpeg differ diff --git a/docs/developer_guides/figs/query_service.jpeg b/docs/developer_guides/figs/query_service.jpeg new file mode 100644 index 0000000000..8df1166e2d Binary files /dev/null and b/docs/developer_guides/figs/query_service.jpeg differ