2020-12-29 10:02:44 +00:00
|
|
|
|
|
|
|
|
|
|
|
## 8. Data Service
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
#### 8.1 Overview
|
|
|
|
|
2021-01-14 07:12:08 +00:00
|
|
|
<img src="./figs/data_service.png" width=700>
|
2020-12-29 10:02:44 +00:00
|
|
|
|
2021-01-13 03:08:03 +00:00
|
|
|
#### 8.2 Data Service Interface
|
2020-12-29 10:02:44 +00:00
|
|
|
|
|
|
|
```go
|
2021-01-13 03:08:03 +00:00
|
|
|
type DataService interface {
|
|
|
|
Service
|
|
|
|
|
|
|
|
RegisterNode(req RegisterNodeRequest) (RegisterNodeResponse, error)
|
2021-01-11 10:35:54 +00:00
|
|
|
Flush(req FlushRequest) error
|
2021-01-13 03:08:03 +00:00
|
|
|
|
|
|
|
AssignSegmentID(req AssignSegIDRequest) (AssignSegIDResponse, error)
|
2021-01-12 10:03:24 +00:00
|
|
|
ShowSegments(req ShowSegmentRequest) (ShowSegmentResponse, error)
|
|
|
|
GetSegmentStates(req SegmentStatesRequest) (SegmentStatesResponse, error)
|
2020-12-29 10:02:44 +00:00
|
|
|
GetInsertBinlogPaths(req InsertBinlogPathRequest) (InsertBinlogPathsResponse, error)
|
2021-01-12 10:03:24 +00:00
|
|
|
|
2020-12-29 10:02:44 +00:00
|
|
|
GetInsertChannels(req InsertChannelRequest) ([]string, error)
|
|
|
|
}
|
|
|
|
```
|
|
|
|
|
|
|
|
|
|
|
|
|
2021-01-15 06:38:36 +00:00
|
|
|
* *MsgBase*
|
2021-01-13 03:08:03 +00:00
|
|
|
|
|
|
|
```go
|
2021-01-15 06:38:36 +00:00
|
|
|
type MsgBase struct {
|
2021-01-13 03:08:03 +00:00
|
|
|
MsgType MsgType
|
2021-01-15 06:38:36 +00:00
|
|
|
MsgID UniqueID
|
2021-01-13 03:08:03 +00:00
|
|
|
Timestamp Timestamp
|
2021-01-15 06:38:36 +00:00
|
|
|
SourceID UniqueID
|
2021-01-13 03:08:03 +00:00
|
|
|
}
|
|
|
|
```
|
|
|
|
|
2021-01-11 10:35:54 +00:00
|
|
|
* *RegisterNode*
|
|
|
|
|
|
|
|
```go
|
2021-01-13 03:08:03 +00:00
|
|
|
type RegisterNodeRequest struct {
|
2021-01-15 06:38:36 +00:00
|
|
|
MsgBase
|
2021-01-13 03:08:03 +00:00
|
|
|
Address string
|
|
|
|
Port int64
|
|
|
|
}
|
2021-01-11 10:35:54 +00:00
|
|
|
|
2021-01-13 03:08:03 +00:00
|
|
|
type RegisterNodeResponse struct {
|
|
|
|
//InitParams
|
|
|
|
}
|
2021-01-11 10:35:54 +00:00
|
|
|
```
|
|
|
|
|
2020-12-29 10:02:44 +00:00
|
|
|
* *AssignSegmentID*
|
|
|
|
|
|
|
|
```go
|
|
|
|
type SegIDRequest struct {
|
|
|
|
Count uint32
|
2021-01-14 12:30:27 +00:00
|
|
|
ChannelName string
|
2020-12-29 10:02:44 +00:00
|
|
|
CollectionID UniqueID
|
|
|
|
PartitionID UniqueID
|
|
|
|
}
|
|
|
|
|
|
|
|
type AssignSegIDRequest struct {
|
2021-01-15 06:38:36 +00:00
|
|
|
MsgBase
|
2020-12-29 10:02:44 +00:00
|
|
|
PerChannelRequest []SegIDRequest
|
|
|
|
}
|
|
|
|
|
|
|
|
type SegIDAssignment struct {
|
2021-01-13 03:08:03 +00:00
|
|
|
SegmentID UniqueID
|
2021-01-14 12:30:27 +00:00
|
|
|
ChannelName string
|
2020-12-29 10:02:44 +00:00
|
|
|
Count uint32
|
|
|
|
CollectionID UniqueID
|
|
|
|
PartitionID UniqueID
|
|
|
|
ExpireTime Timestamp
|
|
|
|
}
|
|
|
|
|
|
|
|
type AssignSegIDResponse struct {
|
|
|
|
PerChannelResponse []SegIDAssignment
|
|
|
|
}
|
|
|
|
```
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
* *Flush*
|
|
|
|
|
|
|
|
```go
|
|
|
|
type FlushRequest struct {
|
2021-01-15 06:38:36 +00:00
|
|
|
MsgBase
|
2020-12-29 10:02:44 +00:00
|
|
|
DbID UniqueID
|
|
|
|
CollectionID UniqueID
|
|
|
|
}
|
|
|
|
```
|
|
|
|
|
|
|
|
|
|
|
|
|
2021-01-12 10:03:24 +00:00
|
|
|
* *ShowSegments*
|
|
|
|
|
|
|
|
```go
|
|
|
|
type ShowSegmentRequest struct {
|
2021-01-15 06:38:36 +00:00
|
|
|
MsgBase
|
2021-01-12 10:03:24 +00:00
|
|
|
CollectionID UniqueID
|
|
|
|
PartitionID UniqueID
|
|
|
|
}
|
|
|
|
|
|
|
|
type ShowSegmentResponse struct {
|
|
|
|
SegmentIDs []UniqueID
|
|
|
|
}
|
|
|
|
```
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
* *GetSegmentStates*
|
|
|
|
|
|
|
|
```go
|
|
|
|
enum SegmentState {
|
|
|
|
NONE = 0;
|
|
|
|
NOT_EXIST = 1;
|
|
|
|
GROWING = 2;
|
|
|
|
SEALED = 3;
|
|
|
|
}
|
|
|
|
|
|
|
|
type SegmentStatesRequest struct {
|
2021-01-15 06:38:36 +00:00
|
|
|
MsgBase
|
2021-01-12 10:03:24 +00:00
|
|
|
SegmentID UniqueID
|
|
|
|
}
|
|
|
|
|
|
|
|
type SegmentStatesResponse struct {
|
|
|
|
State SegmentState
|
2021-01-18 02:09:17 +00:00
|
|
|
OpenTime Timestamp
|
2021-01-12 10:03:24 +00:00
|
|
|
SealedTime Timestamp
|
2021-01-18 02:09:17 +00:00
|
|
|
MsgStartPositions []msgstream.MsgPosition
|
|
|
|
MsgEndPositions []msgstream.MsgPosition
|
2021-01-12 10:03:24 +00:00
|
|
|
}
|
|
|
|
```
|
|
|
|
|
|
|
|
|
|
|
|
|
2020-12-29 10:02:44 +00:00
|
|
|
* *GetInsertBinlogPaths*
|
|
|
|
|
|
|
|
```go
|
|
|
|
type InsertBinlogPathRequest struct {
|
2021-01-15 06:38:36 +00:00
|
|
|
MsgBase
|
2021-01-12 10:03:24 +00:00
|
|
|
SegmentID UniqueID
|
2020-12-29 10:02:44 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
type InsertBinlogPathsResponse struct {
|
2021-01-12 10:03:24 +00:00
|
|
|
FieldIDToPaths map[int64][]string
|
2020-12-29 10:02:44 +00:00
|
|
|
}
|
|
|
|
```
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
* *GetInsertChannels*
|
|
|
|
|
|
|
|
```go
|
|
|
|
type InsertChannelRequest struct {
|
2021-01-15 06:38:36 +00:00
|
|
|
MsgBase
|
|
|
|
DbID UniqueID
|
|
|
|
CollectionID UniqueID
|
|
|
|
}
|
|
|
|
```
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
#### 8.2 Insert Channel
|
|
|
|
|
|
|
|
```go
|
|
|
|
type InsertRequest struct {
|
2021-01-15 10:03:16 +00:00
|
|
|
MsgBase
|
2021-01-15 06:38:36 +00:00
|
|
|
DbName string
|
|
|
|
CollectionName string
|
|
|
|
PartitionName string
|
2021-01-15 06:02:12 +00:00
|
|
|
DbID UniqueID
|
|
|
|
CollectionID UniqueID
|
2021-01-15 06:38:36 +00:00
|
|
|
PartitionID UniqueID
|
|
|
|
RowData []Blob
|
|
|
|
HashKeys []uint32
|
2021-01-15 06:02:12 +00:00
|
|
|
}
|
|
|
|
```
|
|
|
|
|
|
|
|
|
|
|
|
|
2021-01-13 03:08:03 +00:00
|
|
|
#### 8.2 Data Node Interface
|
2021-01-11 10:35:54 +00:00
|
|
|
|
|
|
|
```go
|
|
|
|
type DataNode interface {
|
2021-01-13 03:08:03 +00:00
|
|
|
Service
|
2021-01-11 10:35:54 +00:00
|
|
|
|
2021-01-13 03:08:03 +00:00
|
|
|
WatchDmChannels(req WatchDmChannelRequest) error
|
2021-01-14 12:30:27 +00:00
|
|
|
//WatchDdChannel(channelName string) error
|
|
|
|
//SetTimeTickChannel(channelName string) error
|
2021-01-16 07:47:33 +00:00
|
|
|
//SetStatisticsChannel(channelName string) error
|
2021-01-11 10:35:54 +00:00
|
|
|
|
2021-01-13 03:08:03 +00:00
|
|
|
FlushSegments(req FlushSegRequest) error
|
|
|
|
}
|
|
|
|
```
|
|
|
|
|
|
|
|
|
2021-01-20 02:02:59 +00:00
|
|
|
|
2021-01-13 03:08:03 +00:00
|
|
|
* *WatchDmChannels*
|
|
|
|
|
|
|
|
```go
|
|
|
|
type WatchDmChannelRequest struct {
|
2021-01-15 06:38:36 +00:00
|
|
|
MsgBase
|
2021-01-14 12:30:27 +00:00
|
|
|
InsertChannelNames []string
|
2021-01-13 03:08:03 +00:00
|
|
|
}
|
|
|
|
```
|
|
|
|
|
|
|
|
* *FlushSegments*
|
|
|
|
|
|
|
|
```go
|
|
|
|
type FlushSegRequest struct {
|
2021-01-15 06:38:36 +00:00
|
|
|
MsgBase
|
2021-01-13 03:08:03 +00:00
|
|
|
DbID UniqueID
|
|
|
|
CollectionID UniqueID
|
|
|
|
SegmentID []UniqueID
|
2021-01-11 10:35:54 +00:00
|
|
|
}
|
|
|
|
```
|
|
|
|
|
2021-01-16 07:47:33 +00:00
|
|
|
* *SegmentStatistics*
|
|
|
|
|
|
|
|
```go
|
|
|
|
type SegmentStatisticsUpdates struct {
|
|
|
|
SegmentID UniqueID
|
|
|
|
MemorySize int64
|
|
|
|
NumRows int64
|
|
|
|
}
|
|
|
|
|
|
|
|
type SegmentStatistics struct{
|
|
|
|
MsgBase
|
|
|
|
SegStats []*SegmentStatisticsUpdates
|
|
|
|
}
|
|
|
|
```
|
|
|
|
|