milvus-io
/
milvus
mirror of https://github.com/milvus-io/milvus.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Remove Load Delta Channel Request (#18394) 

Fix #18393

remove watch delta channel task related logic

Co-authored-by: xiaofan-luan <xiaofan.luan@zilliz.com>
Signed-off-by: Congqi Xia <congqi.xia@zilliz.com>

Co-authored-by: Congqi Xia <congqi.xia@zilliz.com>
pull/18542/head
Xiaofan 2022-08-04 17:50:37 +08:00 committed by GitHub
parent 936da41f5f
commit 9899a5be98
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
30 changed files with 474 additions and 1101 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
cmd/milvus/mck.go
View File

@ -624,21 +624,8 @@ func (c *mck) unmarshalTask(taskID int64, t string) (string, []int64, []int64, e
log.Info("WatchDmChannels", zap.String("detail", fmt.Sprintf("+%v", loadReq)))
return "WatchDmChannels", removeRepeatElement(partitionIDs), removeRepeatElement(segmentIDs), nil
case commonpb.MsgType_WatchDeltaChannels:
loadReq := querypb.WatchDeltaChannelsRequest{}
err = proto.Unmarshal([]byte(t), &loadReq)
if err != nil {
return errReturn(taskID, "WatchDeltaChannelsRequest", err)
}
var partitionIDs []int64
var segmentIDs []int64
if loadReq.LoadMeta != nil {
partitionIDs = append(partitionIDs, loadReq.LoadMeta.PartitionIDs...)
}
pids, sids := c.extractVchannelInfo(taskID, loadReq.Infos)
partitionIDs = append(partitionIDs, pids...)
segmentIDs = append(segmentIDs, sids...)
log.Info("WatchDeltaChannels", zap.String("detail", fmt.Sprintf("+%v", loadReq)))
return "WatchDeltaChannels", removeRepeatElement(partitionIDs), removeRepeatElement(segmentIDs), nil
log.Warn("legacy WatchDeltaChannels type found, ignore")
return "WatchQueryChannels", emptyInt64(), emptyInt64(), nil
case commonpb.MsgType_WatchQueryChannels:
log.Warn("legacy WatchQueryChannels type found, ignore")
return "WatchQueryChannels", emptyInt64(), emptyInt64(), nil

14
internal/distributed/querynode/client/client.go
View File

@ -152,20 +152,6 @@ func (c *Client) WatchDmChannels(ctx context.Context, req *querypb.WatchDmChanne
return ret.(*commonpb.Status), err
}
// WatchDeltaChannels watches the channels about data manipulation.
func (c *Client) WatchDeltaChannels(ctx context.Context, req *querypb.WatchDeltaChannelsRequest) (*commonpb.Status, error) {
ret, err := c.grpcClient.ReCall(ctx, func(client interface{}) (interface{}, error) {
if !funcutil.CheckCtxValid(ctx) {
return nil, ctx.Err()
}
return client.(querypb.QueryNodeClient).WatchDeltaChannels(ctx, req)
})
if err != nil || ret == nil {
return nil, err
}
return ret.(*commonpb.Status), err
}
// LoadSegments loads the segments to search.
func (c *Client) LoadSegments(ctx context.Context, req *querypb.LoadSegmentsRequest) (*commonpb.Status, error) {
ret, err := c.grpcClient.ReCall(ctx, func(client interface{}) (interface{}, error) {

3
internal/distributed/querynode/client/client_test.go
View File

@ -90,9 +90,6 @@ func Test_NewClient(t *testing.T) {
r12, err := client.GetMetrics(ctx, nil)
retCheck(retNotNil, r12, err)
r13, err := client.WatchDeltaChannels(ctx, nil)
retCheck(retNotNil, r13, err)
r14, err := client.Search(ctx, nil)
retCheck(retNotNil, r14, err)

6
internal/distributed/querynode/service.go
View File

@ -258,12 +258,6 @@ func (s *Server) WatchDmChannels(ctx context.Context, req *querypb.WatchDmChanne
return s.querynode.WatchDmChannels(ctx, req)
}
// WatchDeltaChannels watches the channels about data manipulation.
func (s *Server) WatchDeltaChannels(ctx context.Context, req *querypb.WatchDeltaChannelsRequest) (*commonpb.Status, error) {
// ignore ctx
return s.querynode.WatchDeltaChannels(ctx, req)
}
// LoadSegments loads the segments to search.
func (s *Server) LoadSegments(ctx context.Context, req *querypb.LoadSegmentsRequest) (*commonpb.Status, error) {
// ignore ctx

4
internal/distributed/querynode/service_test.go
View File

@ -80,10 +80,6 @@ func (m *MockQueryNode) WatchDmChannels(ctx context.Context, req *querypb.WatchD
return m.status, m.err
}
func (m *MockQueryNode) WatchDeltaChannels(ctx context.Context, req *querypb.WatchDeltaChannelsRequest) (*commonpb.Status, error) {
return m.status, m.err
}
func (m *MockQueryNode) LoadSegments(ctx context.Context, req *querypb.LoadSegmentsRequest) (*commonpb.Status, error) {
return m.status, m.err
}

15
internal/mq/msgstream/mq_msgstream.go
View File

@ -21,6 +21,7 @@ import (
"errors"
"fmt"
"path/filepath"
"strings"
"sync"
"sync/atomic"
"time"
@ -117,7 +118,7 @@ func (ms *mqMsgStream) AsProducer(channels []string) {
ms.producerChannels = append(ms.producerChannels, channel)
return nil
}
err := retry.Do(context.TODO(), fn, retry.Attempts(20), retry.Sleep(time.Millisecond*200), retry.MaxSleepTime(5*time.Second))
err := retry.Do(context.TODO(), fn, retry.Attempts(20), retry.Sleep(time.Millisecond*200), retry.MaxSleepTime(2*time.Second))
if err != nil {
errMsg := "Failed to create producer " + channel + ", error = " + err.Error()
panic(errMsg)
@ -167,7 +168,7 @@ func (ms *mqMsgStream) AsConsumerWithPosition(channels []string, subName string,
return nil
}
// TODO if know the former subscribe is invalid, should we use pulsarctl to accelerate recovery speed
err := retry.Do(context.TODO(), fn, retry.Attempts(50), retry.Sleep(time.Millisecond*200), retry.MaxSleepTime(5*time.Second))
err := retry.Do(context.TODO(), fn, retry.Attempts(20), retry.Sleep(time.Millisecond*200), retry.MaxSleepTime(2*time.Second))
if err != nil {
errMsg := "Failed to create consumer " + channel + ", error = " + err.Error()
panic(errMsg)
@ -883,7 +884,10 @@ func (ms *MqTtMsgStream) Seek(msgPositions []*internalpb.MsgPosition) error {
log.Info("MsgStream begin to seek start msg: ", zap.String("channel", mp.ChannelName), zap.Any("MessageID", seekMsgID))
err = consumer.Seek(seekMsgID, true)
if err != nil {
log.Warn("Failed to seek", zap.String("channel", mp.ChannelName), zap.Error(err))
// if topic not exist
if strings.Contains(err.Error(), "not exist") {
return retry.Unrecoverable(err)
}
return err
}
log.Info("MsgStream seek finished", zap.String("channel", mp.ChannelName))
@ -899,9 +903,10 @@ func (ms *MqTtMsgStream) Seek(msgPositions []*internalpb.MsgPosition) error {
if len(mp.MsgID) == 0 {
return fmt.Errorf("when msgID's length equal to 0, please use AsConsumer interface")
}
err = retry.Do(context.TODO(), fn, retry.Attempts(20), retry.Sleep(time.Millisecond*200), retry.MaxSleepTime(5*time.Second))
err = retry.Do(context.TODO(), fn, retry.Attempts(20), retry.Sleep(time.Millisecond*200), retry.MaxSleepTime(2*time.Second))
if err != nil {
return fmt.Errorf("failed to seek, error %s", err.Error())
log.Warn("failed to seek", zap.Error(err))
return err
}
ms.addConsumer(consumer, mp.ChannelName)
ms.chanMsgPos[consumer] = (proto.Clone(mp)).(*MsgPosition)

10
internal/proto/query_coord.proto
View File

@ -41,7 +41,6 @@ service QueryNode {
rpc GetStatisticsChannel(internal.GetStatisticsChannelRequest) returns(milvus.StringResponse){}
rpc WatchDmChannels(WatchDmChannelsRequest) returns (common.Status) {}
rpc WatchDeltaChannels(WatchDeltaChannelsRequest) returns (common.Status) {}
rpc LoadSegments(LoadSegmentsRequest) returns (common.Status) {}
rpc ReleaseCollection(ReleaseCollectionRequest) returns (common.Status) {}
rpc ReleasePartitions(ReleasePartitionsRequest) returns (common.Status) {}
@ -175,15 +174,6 @@ message WatchDmChannelsRequest {
map<int64, data.SegmentInfo> segment_infos = 10;
}
message WatchDeltaChannelsRequest {
common.MsgBase base = 1;
int64 nodeID = 2;
int64 collectionID = 3;
repeated data.VchannelInfo infos = 4;
int64 replica_id = 5;
LoadMetaInfo load_meta = 9;
}
message SegmentLoadInfo {
int64 segmentID = 1;
int64 partitionID = 2;

520
internal/proto/querypb/query_coord.pb.go
View File

@ -1212,85 +1212,6 @@ func (m *WatchDmChannelsRequest) GetSegmentInfos() map[int64]*datapb.SegmentInfo
return nil
}
type WatchDeltaChannelsRequest struct {
Base *commonpb.MsgBase `protobuf:"bytes,1,opt,name=base,proto3" json:"base,omitempty"`
NodeID int64 `protobuf:"varint,2,opt,name=nodeID,proto3" json:"nodeID,omitempty"`
CollectionID int64 `protobuf:"varint,3,opt,name=collectionID,proto3" json:"collectionID,omitempty"`
Infos []*datapb.VchannelInfo `protobuf:"bytes,4,rep,name=infos,proto3" json:"infos,omitempty"`
ReplicaId int64 `protobuf:"varint,5,opt,name=replica_id,json=replicaId,proto3" json:"replica_id,omitempty"`
LoadMeta *LoadMetaInfo `protobuf:"bytes,9,opt,name=load_meta,json=loadMeta,proto3" json:"load_meta,omitempty"`
XXX_NoUnkeyedLiteral struct{} `json:"-"`
XXX_unrecognized []byte `json:"-"`
XXX_sizecache int32 `json:"-"`
}
func (m *WatchDeltaChannelsRequest) Reset() { *m = WatchDeltaChannelsRequest{} }
func (m *WatchDeltaChannelsRequest) String() string { return proto.CompactTextString(m) }
func (*WatchDeltaChannelsRequest) ProtoMessage() {}
func (*WatchDeltaChannelsRequest) Descriptor() ([]byte, []int) {
return fileDescriptor_aab7cc9a69ed26e8, []int{17}
}
func (m *WatchDeltaChannelsRequest) XXX_Unmarshal(b []byte) error {
return xxx_messageInfo_WatchDeltaChannelsRequest.Unmarshal(m, b)
}
func (m *WatchDeltaChannelsRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
return xxx_messageInfo_WatchDeltaChannelsRequest.Marshal(b, m, deterministic)
}
func (m *WatchDeltaChannelsRequest) XXX_Merge(src proto.Message) {
xxx_messageInfo_WatchDeltaChannelsRequest.Merge(m, src)
}
func (m *WatchDeltaChannelsRequest) XXX_Size() int {
return xxx_messageInfo_WatchDeltaChannelsRequest.Size(m)
}
func (m *WatchDeltaChannelsRequest) XXX_DiscardUnknown() {
xxx_messageInfo_WatchDeltaChannelsRequest.DiscardUnknown(m)
}
var xxx_messageInfo_WatchDeltaChannelsRequest proto.InternalMessageInfo
func (m *WatchDeltaChannelsRequest) GetBase() *commonpb.MsgBase {
if m != nil {
return m.Base
}
return nil
}
func (m *WatchDeltaChannelsRequest) GetNodeID() int64 {
if m != nil {
return m.NodeID
}
return 0
}
func (m *WatchDeltaChannelsRequest) GetCollectionID() int64 {
if m != nil {
return m.CollectionID
}
return 0
}
func (m *WatchDeltaChannelsRequest) GetInfos() []*datapb.VchannelInfo {
if m != nil {
return m.Infos
}
return nil
}
func (m *WatchDeltaChannelsRequest) GetReplicaId() int64 {
if m != nil {
return m.ReplicaId
}
return 0
}
func (m *WatchDeltaChannelsRequest) GetLoadMeta() *LoadMetaInfo {
if m != nil {
return m.LoadMeta
}
return nil
}
type SegmentLoadInfo struct {
SegmentID int64 `protobuf:"varint,1,opt,name=segmentID,proto3" json:"segmentID,omitempty"`
PartitionID int64 `protobuf:"varint,2,opt,name=partitionID,proto3" json:"partitionID,omitempty"`
@ -1314,7 +1235,7 @@ func (m *SegmentLoadInfo) Reset() { *m = SegmentLoadInfo{} }
func (m *SegmentLoadInfo) String() string { return proto.CompactTextString(m) }
func (*SegmentLoadInfo) ProtoMessage() {}
func (*SegmentLoadInfo) Descriptor() ([]byte, []int) {
return fileDescriptor_aab7cc9a69ed26e8, []int{18}
return fileDescriptor_aab7cc9a69ed26e8, []int{17}
}
func (m *SegmentLoadInfo) XXX_Unmarshal(b []byte) error {
@ -1444,7 +1365,7 @@ func (m *FieldIndexInfo) Reset() { *m = FieldIndexInfo{} }
func (m *FieldIndexInfo) String() string { return proto.CompactTextString(m) }
func (*FieldIndexInfo) ProtoMessage() {}
func (*FieldIndexInfo) Descriptor() ([]byte, []int) {
return fileDescriptor_aab7cc9a69ed26e8, []int{19}
return fileDescriptor_aab7cc9a69ed26e8, []int{18}
}
func (m *FieldIndexInfo) XXX_Unmarshal(b []byte) error {
@ -1540,7 +1461,7 @@ func (m *LoadSegmentsRequest) Reset() { *m = LoadSegmentsRequest{} }
func (m *LoadSegmentsRequest) String() string { return proto.CompactTextString(m) }
func (*LoadSegmentsRequest) ProtoMessage() {}
func (*LoadSegmentsRequest) Descriptor() ([]byte, []int) {
return fileDescriptor_aab7cc9a69ed26e8, []int{20}
return fileDescriptor_aab7cc9a69ed26e8, []int{19}
}
func (m *LoadSegmentsRequest) XXX_Unmarshal(b []byte) error {
@ -1642,7 +1563,7 @@ func (m *ReleaseSegmentsRequest) Reset() { *m = ReleaseSegmentsRequest{}
func (m *ReleaseSegmentsRequest) String() string { return proto.CompactTextString(m) }
func (*ReleaseSegmentsRequest) ProtoMessage() {}
func (*ReleaseSegmentsRequest) Descriptor() ([]byte, []int) {
return fileDescriptor_aab7cc9a69ed26e8, []int{21}
return fileDescriptor_aab7cc9a69ed26e8, []int{20}
}
func (m *ReleaseSegmentsRequest) XXX_Unmarshal(b []byte) error {
@ -1727,7 +1648,7 @@ func (m *SearchRequest) Reset() { *m = SearchRequest{} }
func (m *SearchRequest) String() string { return proto.CompactTextString(m) }
func (*SearchRequest) ProtoMessage() {}
func (*SearchRequest) Descriptor() ([]byte, []int) {
return fileDescriptor_aab7cc9a69ed26e8, []int{22}
return fileDescriptor_aab7cc9a69ed26e8, []int{21}
}
func (m *SearchRequest) XXX_Unmarshal(b []byte) error {
@ -1798,7 +1719,7 @@ func (m *QueryRequest) Reset() { *m = QueryRequest{} }
func (m *QueryRequest) String() string { return proto.CompactTextString(m) }
func (*QueryRequest) ProtoMessage() {}
func (*QueryRequest) Descriptor() ([]byte, []int) {
return fileDescriptor_aab7cc9a69ed26e8, []int{23}
return fileDescriptor_aab7cc9a69ed26e8, []int{22}
}
func (m *QueryRequest) XXX_Unmarshal(b []byte) error {
@ -1867,7 +1788,7 @@ func (m *SyncReplicaSegmentsRequest) Reset() { *m = SyncReplicaSegmentsR
func (m *SyncReplicaSegmentsRequest) String() string { return proto.CompactTextString(m) }
func (*SyncReplicaSegmentsRequest) ProtoMessage() {}
func (*SyncReplicaSegmentsRequest) Descriptor() ([]byte, []int) {
return fileDescriptor_aab7cc9a69ed26e8, []int{24}
return fileDescriptor_aab7cc9a69ed26e8, []int{23}
}
func (m *SyncReplicaSegmentsRequest) XXX_Unmarshal(b []byte) error {
@ -1922,7 +1843,7 @@ func (m *ReplicaSegmentsInfo) Reset() { *m = ReplicaSegmentsInfo{} }
func (m *ReplicaSegmentsInfo) String() string { return proto.CompactTextString(m) }
func (*ReplicaSegmentsInfo) ProtoMessage() {}
func (*ReplicaSegmentsInfo) Descriptor() ([]byte, []int) {
return fileDescriptor_aab7cc9a69ed26e8, []int{25}
return fileDescriptor_aab7cc9a69ed26e8, []int{24}
}
func (m *ReplicaSegmentsInfo) XXX_Unmarshal(b []byte) error {
@ -1978,7 +1899,7 @@ func (m *HandoffSegmentsRequest) Reset() { *m = HandoffSegmentsRequest{}
func (m *HandoffSegmentsRequest) String() string { return proto.CompactTextString(m) }
func (*HandoffSegmentsRequest) ProtoMessage() {}
func (*HandoffSegmentsRequest) Descriptor() ([]byte, []int) {
return fileDescriptor_aab7cc9a69ed26e8, []int{26}
return fileDescriptor_aab7cc9a69ed26e8, []int{25}
}
func (m *HandoffSegmentsRequest) XXX_Unmarshal(b []byte) error {
@ -2036,7 +1957,7 @@ func (m *LoadBalanceRequest) Reset() { *m = LoadBalanceRequest{} }
func (m *LoadBalanceRequest) String() string { return proto.CompactTextString(m) }
func (*LoadBalanceRequest) ProtoMessage() {}
func (*LoadBalanceRequest) Descriptor() ([]byte, []int) {
return fileDescriptor_aab7cc9a69ed26e8, []int{27}
return fileDescriptor_aab7cc9a69ed26e8, []int{26}
}
func (m *LoadBalanceRequest) XXX_Unmarshal(b []byte) error {
@ -2114,7 +2035,7 @@ func (m *DmChannelWatchInfo) Reset() { *m = DmChannelWatchInfo{} }
func (m *DmChannelWatchInfo) String() string { return proto.CompactTextString(m) }
func (*DmChannelWatchInfo) ProtoMessage() {}
func (*DmChannelWatchInfo) Descriptor() ([]byte, []int) {
return fileDescriptor_aab7cc9a69ed26e8, []int{28}
return fileDescriptor_aab7cc9a69ed26e8, []int{27}
}
func (m *DmChannelWatchInfo) XXX_Unmarshal(b []byte) error {
@ -2185,7 +2106,7 @@ func (m *QueryChannelInfo) Reset() { *m = QueryChannelInfo{} }
func (m *QueryChannelInfo) String() string { return proto.CompactTextString(m) }
func (*QueryChannelInfo) ProtoMessage() {}
func (*QueryChannelInfo) Descriptor() ([]byte, []int) {
return fileDescriptor_aab7cc9a69ed26e8, []int{29}
return fileDescriptor_aab7cc9a69ed26e8, []int{28}
}
func (m *QueryChannelInfo) XXX_Unmarshal(b []byte) error {
@ -2254,7 +2175,7 @@ func (m *PartitionStates) Reset() { *m = PartitionStates{} }
func (m *PartitionStates) String() string { return proto.CompactTextString(m) }
func (*PartitionStates) ProtoMessage() {}
func (*PartitionStates) Descriptor() ([]byte, []int) {
return fileDescriptor_aab7cc9a69ed26e8, []int{30}
return fileDescriptor_aab7cc9a69ed26e8, []int{29}
}
func (m *PartitionStates) XXX_Unmarshal(b []byte) error {
@ -2322,7 +2243,7 @@ func (m *SegmentInfo) Reset() { *m = SegmentInfo{} }
func (m *SegmentInfo) String() string { return proto.CompactTextString(m) }
func (*SegmentInfo) ProtoMessage() {}
func (*SegmentInfo) Descriptor() ([]byte, []int) {
return fileDescriptor_aab7cc9a69ed26e8, []int{31}
return fileDescriptor_aab7cc9a69ed26e8, []int{30}
}
func (m *SegmentInfo) XXX_Unmarshal(b []byte) error {
@ -2467,7 +2388,7 @@ func (m *CollectionInfo) Reset() { *m = CollectionInfo{} }
func (m *CollectionInfo) String() string { return proto.CompactTextString(m) }
func (*CollectionInfo) ProtoMessage() {}
func (*CollectionInfo) Descriptor() ([]byte, []int) {
return fileDescriptor_aab7cc9a69ed26e8, []int{32}
return fileDescriptor_aab7cc9a69ed26e8, []int{31}
}
func (m *CollectionInfo) XXX_Unmarshal(b []byte) error {
@ -2563,7 +2484,7 @@ func (m *UnsubscribeChannels) Reset() { *m = UnsubscribeChannels{} }
func (m *UnsubscribeChannels) String() string { return proto.CompactTextString(m) }
func (*UnsubscribeChannels) ProtoMessage() {}
func (*UnsubscribeChannels) Descriptor() ([]byte, []int) {
return fileDescriptor_aab7cc9a69ed26e8, []int{33}
return fileDescriptor_aab7cc9a69ed26e8, []int{32}
}
func (m *UnsubscribeChannels) XXX_Unmarshal(b []byte) error {
@ -2610,7 +2531,7 @@ func (m *UnsubscribeChannelInfo) Reset() { *m = UnsubscribeChannelInfo{}
func (m *UnsubscribeChannelInfo) String() string { return proto.CompactTextString(m) }
func (*UnsubscribeChannelInfo) ProtoMessage() {}
func (*UnsubscribeChannelInfo) Descriptor() ([]byte, []int) {
return fileDescriptor_aab7cc9a69ed26e8, []int{34}
return fileDescriptor_aab7cc9a69ed26e8, []int{33}
}
func (m *UnsubscribeChannelInfo) XXX_Unmarshal(b []byte) error {
@ -2660,7 +2581,7 @@ func (m *SegmentChangeInfo) Reset() { *m = SegmentChangeInfo{} }
func (m *SegmentChangeInfo) String() string { return proto.CompactTextString(m) }
func (*SegmentChangeInfo) ProtoMessage() {}
func (*SegmentChangeInfo) Descriptor() ([]byte, []int) {
return fileDescriptor_aab7cc9a69ed26e8, []int{35}
return fileDescriptor_aab7cc9a69ed26e8, []int{34}
}
func (m *SegmentChangeInfo) XXX_Unmarshal(b []byte) error {
@ -2721,7 +2642,7 @@ func (m *SealedSegmentsChangeInfo) Reset() { *m = SealedSegmentsChangeIn
func (m *SealedSegmentsChangeInfo) String() string { return proto.CompactTextString(m) }
func (*SealedSegmentsChangeInfo) ProtoMessage() {}
func (*SealedSegmentsChangeInfo) Descriptor() ([]byte, []int) {
return fileDescriptor_aab7cc9a69ed26e8, []int{36}
return fileDescriptor_aab7cc9a69ed26e8, []int{35}
}
func (m *SealedSegmentsChangeInfo) XXX_Unmarshal(b []byte) error {
@ -2779,7 +2700,6 @@ func init() {
proto.RegisterType((*LoadMetaInfo)(nil), "milvus.proto.query.LoadMetaInfo")
proto.RegisterType((*WatchDmChannelsRequest)(nil), "milvus.proto.query.WatchDmChannelsRequest")
proto.RegisterMapType((map[int64]*datapb.SegmentInfo)(nil), "milvus.proto.query.WatchDmChannelsRequest.SegmentInfosEntry")
proto.RegisterType((*WatchDeltaChannelsRequest)(nil), "milvus.proto.query.WatchDeltaChannelsRequest")
proto.RegisterType((*SegmentLoadInfo)(nil), "milvus.proto.query.SegmentLoadInfo")
proto.RegisterType((*FieldIndexInfo)(nil), "milvus.proto.query.FieldIndexInfo")
proto.RegisterType((*LoadSegmentsRequest)(nil), "milvus.proto.query.LoadSegmentsRequest")
@ -2804,191 +2724,187 @@ func init() {
func init() { proto.RegisterFile("query_coord.proto", fileDescriptor_aab7cc9a69ed26e8) }
var fileDescriptor_aab7cc9a69ed26e8 = []byte{
// 2929 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xec, 0x3a, 0xc9, 0x8f, 0x1c, 0x57,
0xf9, 0x53, 0xbd, 0x4d, 0xf7, 0xd7, 0x5b, 0xf9, 0x8d, 0x3d, 0xee, 0xf4, 0xcf, 0x4e, 0x9c, 0x72,
0x9c, 0xcc, 0x6f, 0x42, 0xc6, 0x66, 0x1c, 0xa2, 0x04, 0x82, 0x84, 0x3d, 0x13, 0x4f, 0x06, 0xdb,
0x93, 0xa1, 0xda, 0x0e, 0xc8, 0x8a, 0x54, 0x54, 0x57, 0xbd, 0xe9, 0x29, 0xb9, 0x96, 0x76, 0xbd,
0xea, 0x71, 0x26, 0x67, 0x84, 0xc4, 0x76, 0x40, 0x5c, 0x51, 0xc4, 0x01, 0x0e, 0x48, 0x44, 0x5c,
0xb8, 0x20, 0x21, 0xc4, 0x8d, 0x2b, 0x12, 0x7f, 0x00, 0x47, 0x04, 0x77, 0xc4, 0x15, 0xbd, 0xa5,
0xf6, 0x2a, 0x77, 0x8f, 0x27, 0x4e, 0x82, 0xc4, 0xad, 0xea, 0x7b, 0xcb, 0xb7, 0x6f, 0xef, 0x3d,
0x38, 0xf3, 0x68, 0x86, 0xfd, 0x63, 0xcd, 0xf0, 0x3c, 0xdf, 0xdc, 0x98, 0xfa, 0x5e, 0xe0, 0x21,
0xe4, 0x58, 0xf6, 0xd1, 0x8c, 0xf0, 0xbf, 0x0d, 0x36, 0x3e, 0xec, 0x18, 0x9e, 0xe3, 0x78, 0x2e,
0x87, 0x0d, 0x3b, 0xc9, 0x19, 0xc3, 0x9e, 0xe5, 0x06, 0xd8, 0x77, 0x75, 0x3b, 0x1c, 0x25, 0xc6,
0x21, 0x76, 0x74, 0xf1, 0x27, 0x9b, 0x7a, 0xa0, 0x27, 0xf7, 0x57, 0xbe, 0x27, 0xc1, 0xea, 0xe8,
0xd0, 0x7b, 0xbc, 0xe5, 0xd9, 0x36, 0x36, 0x02, 0xcb, 0x73, 0x89, 0x8a, 0x1f, 0xcd, 0x30, 0x09,
0xd0, 0x35, 0xa8, 0x8d, 0x75, 0x82, 0x07, 0xd2, 0x25, 0x69, 0xad, 0xbd, 0x79, 0x61, 0x23, 0x45,
0x89, 0x20, 0xe1, 0x2e, 0x99, 0xdc, 0xd4, 0x09, 0x56, 0xd9, 0x4c, 0x84, 0xa0, 0x66, 0x8e, 0x77,
0xb7, 0x07, 0x95, 0x4b, 0xd2, 0x5a, 0x55, 0x65, 0xdf, 0xe8, 0x25, 0xe8, 0x1a, 0xd1, 0xde, 0xbb,
0xdb, 0x64, 0x50, 0xbd, 0x54, 0x5d, 0xab, 0xaa, 0x69, 0xa0, 0xf2, 0x37, 0x09, 0xce, 0xe7, 0xc8,
0x20, 0x53, 0xcf, 0x25, 0x18, 0x5d, 0x87, 0x06, 0x09, 0xf4, 0x60, 0x46, 0x04, 0x25, 0xff, 0x57,
0x48, 0xc9, 0x88, 0x4d, 0x51, 0xc5, 0xd4, 0x3c, 0xda, 0x4a, 0x01, 0x5a, 0xf4, 0x65, 0x38, 0x6b,
0xb9, 0x77, 0xb1, 0xe3, 0xf9, 0xc7, 0xda, 0x14, 0xfb, 0x06, 0x76, 0x03, 0x7d, 0x82, 0x43, 0x1a,
0x57, 0xc2, 0xb1, 0xfd, 0x78, 0x08, 0xbd, 0x01, 0xe7, 0xb9, 0x96, 0x08, 0xf6, 0x8f, 0x2c, 0x03,
0x6b, 0xfa, 0x91, 0x6e, 0xd9, 0xfa, 0xd8, 0xc6, 0x83, 0xda, 0xa5, 0xea, 0x5a, 0x53, 0x3d, 0xc7,
0x86, 0x47, 0x7c, 0xf4, 0x46, 0x38, 0xa8, 0xfc, 0x4a, 0x82, 0x73, 0x94, 0xc3, 0x7d, 0xdd, 0x0f,
0xac, 0x67, 0x20, 0x67, 0x05, 0x3a, 0x49, 0xde, 0x06, 0x55, 0x36, 0x96, 0x82, 0xd1, 0x39, 0xd3,
0x10, 0x3d, 0x95, 0x49, 0x8d, 0xb1, 0x99, 0x82, 0x29, 0xbf, 0x14, 0x06, 0x91, 0xa4, 0xf3, 0x34,
0x8a, 0xc8, 0xe2, 0xac, 0xe4, 0x71, 0x3e, 0x85, 0x1a, 0x94, 0xbf, 0x4b, 0x70, 0xee, 0x8e, 0xa7,
0x9b, 0xb1, 0xc1, 0x7c, 0xf6, 0xe2, 0xfc, 0x3a, 0x34, 0xb8, 0x77, 0x0d, 0x6a, 0x0c, 0xd7, 0x95,
0x34, 0x2e, 0xe1, 0x79, 0x31, 0x85, 0x23, 0x06, 0x50, 0xc5, 0x22, 0x74, 0x05, 0x7a, 0x3e, 0x9e,
0xda, 0x96, 0xa1, 0x6b, 0xee, 0xcc, 0x19, 0x63, 0x7f, 0x50, 0xbf, 0x24, 0xad, 0xd5, 0xd5, 0xae,
0x80, 0xee, 0x31, 0xa0, 0xf2, 0x73, 0x09, 0x06, 0x2a, 0xb6, 0xb1, 0x4e, 0xf0, 0xe7, 0xc9, 0xec,
0x2a, 0x34, 0x5c, 0xcf, 0xc4, 0xbb, 0xdb, 0x8c, 0xd9, 0xaa, 0x2a, 0xfe, 0x94, 0x1f, 0x55, 0xb8,
0x22, 0xbe, 0xe0, 0x76, 0x9d, 0x50, 0x56, 0xfd, 0xd3, 0x51, 0x56, 0xa3, 0x48, 0x59, 0x7f, 0x8a,
0x95, 0xf5, 0x45, 0x17, 0x48, 0xac, 0xd0, 0x7a, 0x4a, 0xa1, 0xbf, 0x96, 0xe0, 0xb9, 0x1d, 0x1c,
0x44, 0xe4, 0x53, 0x7f, 0xc6, 0x5f, 0xd0, 0x60, 0xf5, 0x89, 0x04, 0xc3, 0x22, 0x5a, 0x4f, 0x13,
0xb0, 0x1e, 0xc0, 0x6a, 0x84, 0x43, 0x33, 0x31, 0x31, 0x7c, 0x6b, 0xca, 0xd4, 0xc8, 0x42, 0x57,
0x7b, 0xf3, 0xf2, 0x46, 0x3e, 0x25, 0x6f, 0x64, 0x29, 0x38, 0x17, 0x6d, 0xb1, 0x9d, 0xd8, 0x41,
0xf9, 0x89, 0x04, 0xe7, 0x76, 0x70, 0x30, 0xc2, 0x13, 0x07, 0xbb, 0xc1, 0xae, 0x7b, 0xe0, 0x3d,
0xbd, 0x5c, 0x9f, 0x07, 0x20, 0x62, 0x9f, 0x28, 0xac, 0x26, 0x20, 0x8b, 0xc8, 0x98, 0x65, 0xff,
0x2c, 0x3d, 0xa7, 0x91, 0xdd, 0x57, 0xa0, 0x6e, 0xb9, 0x07, 0x5e, 0x28, 0xaa, 0x17, 0x8a, 0x44,
0x95, 0x44, 0xc6, 0x67, 0x2b, 0x2e, 0xa7, 0xe2, 0x50, 0xf7, 0xcd, 0x3b, 0x58, 0x37, 0xb1, 0x7f,
0x0a, 0x73, 0xcb, 0xb2, 0x5d, 0x29, 0x60, 0xfb, 0xc7, 0x12, 0x9c, 0xcf, 0x21, 0x3c, 0x0d, 0xdf,
0x6f, 0x43, 0x83, 0xd0, 0xcd, 0x42, 0xc6, 0x5f, 0x2a, 0x64, 0x3c, 0x81, 0xee, 0x8e, 0x45, 0x02,
0x55, 0xac, 0x51, 0x3c, 0x90, 0xb3, 0x63, 0xe8, 0x45, 0xe8, 0x18, 0x87, 0xba, 0xeb, 0x62, 0x5b,
0x73, 0x75, 0x87, 0x0b, 0xa0, 0xa5, 0xb6, 0x05, 0x6c, 0x4f, 0x77, 0x30, 0x7a, 0x0e, 0x9a, 0xd4,
0x65, 0x35, 0xcb, 0x0c, 0xd5, 0xbf, 0xcc, 0x5c, 0xd8, 0x24, 0xe8, 0x22, 0x00, 0x1b, 0xd2, 0x4d,
0xd3, 0xe7, 0x69, 0xb4, 0xa5, 0xb6, 0x28, 0xe4, 0x06, 0x05, 0x28, 0x3f, 0x95, 0xa0, 0x43, 0x63,
0xf6, 0x5d, 0x1c, 0xe8, 0x54, 0x0f, 0xe8, 0x2d, 0x68, 0xd9, 0x9e, 0x6e, 0x6a, 0xc1, 0xf1, 0x94,
0xa3, 0xea, 0x65, 0x65, 0xcd, 0x59, 0xa0, 0x8b, 0xee, 0x1d, 0x4f, 0xb1, 0xda, 0xb4, 0xc5, 0xd7,
0x22, 0xf2, 0xce, 0xb9, 0x72, 0xb5, 0xc0, 0x95, 0xbf, 0x5f, 0x87, 0xd5, 0x6f, 0xeb, 0x81, 0x71,
0xb8, 0xed, 0x6c, 0x71, 0x26, 0x4f, 0x61, 0x04, 0x71, 0x6c, 0xab, 0x24, 0x63, 0xdb, 0xa7, 0x16,
0x3b, 0x23, 0x3b, 0xaf, 0x17, 0xd9, 0x39, 0x2d, 0xb2, 0x37, 0xde, 0x17, 0xaa, 0x4a, 0xd8, 0x79,
0x22, 0x07, 0x35, 0x9e, 0x26, 0x07, 0x6d, 0x41, 0x17, 0x7f, 0x68, 0xd8, 0x33, 0xaa, 0x73, 0x86,
0x7d, 0x99, 0x61, 0x7f, 0xbe, 0x00, 0x7b, 0xd2, 0xc9, 0x3a, 0x62, 0xd1, 0xae, 0xa0, 0x81, 0xab,
0xda, 0xc1, 0x81, 0x3e, 0x68, 0x32, 0x32, 0x2e, 0x95, 0xa9, 0x3a, 0xb4, 0x0f, 0xae, 0x6e, 0xfa,
0x87, 0x2e, 0x40, 0x4b, 0x64, 0xbc, 0xdd, 0xed, 0x41, 0x8b, 0x89, 0x2f, 0x06, 0x20, 0x1d, 0xba,
0x22, 0x02, 0x09, 0x0a, 0x81, 0x51, 0xf8, 0x76, 0x11, 0x82, 0x62, 0x65, 0x27, 0x29, 0x27, 0xef,
0xb8, 0x81, 0x7f, 0xac, 0x76, 0x48, 0x02, 0x34, 0xd4, 0xe0, 0x4c, 0x6e, 0x0a, 0x92, 0xa1, 0xfa,
0x10, 0x1f, 0x33, 0x03, 0xa9, 0xaa, 0xf4, 0x13, 0xbd, 0x0e, 0xf5, 0x23, 0xdd, 0x9e, 0x61, 0x66,
0x00, 0xf3, 0x65, 0xc4, 0x27, 0x7f, 0xb5, 0xf2, 0xa6, 0xa4, 0x7c, 0x5c, 0x81, 0xe7, 0x38, 0x6d,
0xd8, 0x0e, 0xf4, 0xcf, 0xd7, 0x16, 0x23, 0x3b, 0xab, 0x9d, 0xc8, 0xce, 0x2e, 0x02, 0x84, 0xc5,
0x8a, 0x65, 0x8a, 0xf4, 0x1e, 0x69, 0xc9, 0x4c, 0x9b, 0x40, 0xeb, 0xa4, 0x26, 0xa0, 0xfc, 0xb1,
0x06, 0x7d, 0x21, 0x3b, 0x3a, 0x83, 0x05, 0x90, 0x0b, 0xd0, 0x8a, 0x52, 0x8f, 0x50, 0x43, 0x0c,
0x40, 0x97, 0xa0, 0x9d, 0x70, 0x1f, 0x21, 0x87, 0x24, 0x68, 0x21, 0x61, 0x84, 0x85, 0x44, 0x2d,
0x51, 0x48, 0x5c, 0x04, 0x38, 0xb0, 0x67, 0xe4, 0x50, 0x0b, 0x2c, 0x07, 0x87, 0x9c, 0x32, 0xc8,
0x3d, 0xcb, 0xc1, 0xe8, 0x06, 0x74, 0xc6, 0x96, 0x6b, 0x7b, 0x13, 0x6d, 0xaa, 0x07, 0x87, 0x64,
0xd0, 0x28, 0x75, 0x98, 0x5b, 0x16, 0xb6, 0xcd, 0x9b, 0x6c, 0xae, 0xda, 0xe6, 0x6b, 0xf6, 0xe9,
0x12, 0xf4, 0x3c, 0xb4, 0xdd, 0x99, 0xa3, 0x79, 0x07, 0x9a, 0xef, 0x3d, 0xa6, 0x2e, 0xc7, 0x50,
0xb8, 0x33, 0xe7, 0xbd, 0x03, 0xd5, 0x7b, 0x4c, 0x43, 0x7f, 0x8b, 0x26, 0x01, 0x62, 0x7b, 0x13,
0x32, 0x68, 0x2e, 0xb4, 0x7f, 0xbc, 0x80, 0xae, 0x36, 0xa9, 0x99, 0xb1, 0xd5, 0xad, 0xc5, 0x56,
0x47, 0x0b, 0xd0, 0xcb, 0xd0, 0x33, 0x3c, 0x67, 0xaa, 0x33, 0x09, 0xdd, 0xf2, 0x3d, 0x87, 0xf9,
0x5b, 0x55, 0xcd, 0x40, 0xd1, 0x16, 0xb4, 0x2d, 0xd7, 0xc4, 0x1f, 0x0a, 0xa7, 0x6c, 0x33, 0x3c,
0x4a, 0x91, 0xca, 0x19, 0xa2, 0x5d, 0x3a, 0x97, 0x29, 0x1d, 0xac, 0xf0, 0x93, 0xd0, 0x8c, 0x14,
0xfa, 0x36, 0xb1, 0x3e, 0xc2, 0x83, 0x0e, 0xd7, 0xa2, 0x80, 0x8d, 0xac, 0x8f, 0x30, 0x2d, 0x92,
0x2d, 0x97, 0x60, 0x3f, 0xd0, 0x84, 0x51, 0x0e, 0xba, 0x2c, 0x6d, 0x75, 0x39, 0x54, 0xf8, 0x92,
0xf2, 0xdb, 0x0a, 0xf4, 0xd2, 0x88, 0xd0, 0x00, 0x96, 0x0f, 0x18, 0x24, 0xb4, 0x9e, 0xf0, 0x97,
0xa2, 0xc5, 0x2e, 0xed, 0xa0, 0x35, 0x46, 0x0b, 0x33, 0x9e, 0xa6, 0xda, 0xe6, 0x30, 0xb6, 0x01,
0x35, 0x02, 0xce, 0x1e, 0xcb, 0x94, 0x55, 0x86, 0xb2, 0xc5, 0x20, 0x2c, 0x4f, 0x0e, 0x60, 0x99,
0xb3, 0x11, 0x9a, 0x4e, 0xf8, 0x4b, 0x47, 0xc6, 0x33, 0x8b, 0x61, 0xe5, 0xa6, 0x13, 0xfe, 0xa2,
0x6d, 0xe8, 0xf0, 0x2d, 0xa7, 0xba, 0xaf, 0x3b, 0xa1, 0xe1, 0xbc, 0x58, 0xe8, 0xee, 0xb7, 0xf1,
0xf1, 0xfb, 0x34, 0x7a, 0xec, 0xeb, 0x96, 0xaf, 0x72, 0x41, 0xef, 0xb3, 0x55, 0x68, 0x0d, 0x64,
0xbe, 0xcb, 0x81, 0x65, 0x63, 0x61, 0x82, 0xcb, 0x2c, 0x19, 0xf7, 0x18, 0xfc, 0x96, 0x65, 0x63,
0x6e, 0x65, 0x11, 0x0b, 0x4c, 0xb4, 0x4d, 0x6e, 0x64, 0x0c, 0x42, 0x05, 0xab, 0xfc, 0xb5, 0x0a,
0x2b, 0xd4, 0xd7, 0x84, 0xdb, 0x9d, 0x22, 0x1a, 0x5d, 0x04, 0x30, 0x49, 0xa0, 0xa5, 0x22, 0x52,
0xcb, 0x24, 0xc1, 0x1e, 0x0f, 0x4a, 0x6f, 0x85, 0x01, 0xa7, 0x5a, 0x5e, 0xeb, 0x66, 0x7c, 0x3f,
0x9f, 0xdc, 0x9e, 0xaa, 0x1b, 0xbe, 0x0c, 0x5d, 0xe2, 0xcd, 0x7c, 0x03, 0x6b, 0xa9, 0xae, 0xa4,
0xc3, 0x81, 0x7b, 0xc5, 0x31, 0xb3, 0x51, 0xd8, 0x95, 0x27, 0xa2, 0xdb, 0xf2, 0xe9, 0x12, 0x5c,
0x33, 0x9b, 0xe0, 0x6e, 0x43, 0x9f, 0xb9, 0x9f, 0x36, 0xf5, 0x08, 0x6f, 0xee, 0x84, 0xd7, 0x66,
0xbc, 0x29, 0x3a, 0x85, 0xbb, 0x4b, 0x26, 0xfb, 0x62, 0xaa, 0xda, 0x63, 0x4b, 0xc3, 0x5f, 0xa2,
0xfc, 0xac, 0x02, 0xab, 0xa2, 0x59, 0x3c, 0xbd, 0x62, 0xcb, 0xd2, 0x4c, 0x18, 0x35, 0xab, 0x4f,
0x68, 0xbf, 0x6a, 0x0b, 0x94, 0x41, 0xf5, 0x82, 0x32, 0x28, 0xdd, 0x82, 0x34, 0x72, 0x2d, 0xc8,
0x75, 0xa8, 0x13, 0xc3, 0x9b, 0x62, 0xa6, 0x86, 0xde, 0xe6, 0xc5, 0x22, 0x35, 0x6c, 0xeb, 0x81,
0x3e, 0xa2, 0x93, 0x54, 0x3e, 0x57, 0xf9, 0x87, 0x04, 0xdd, 0x11, 0xd6, 0x7d, 0xe3, 0x30, 0x14,
0xc6, 0x1b, 0x50, 0xf5, 0xf1, 0x23, 0x21, 0x8b, 0x97, 0x4a, 0x04, 0x9d, 0x5a, 0xa2, 0xd2, 0x05,
0xe8, 0x05, 0x68, 0x9b, 0x8e, 0x1d, 0xc5, 0xa2, 0x0a, 0x0b, 0x0c, 0x60, 0x3a, 0xb6, 0x08, 0x44,
0x19, 0xfa, 0xab, 0x39, 0xfa, 0xd7, 0xe1, 0xcc, 0x81, 0xef, 0x39, 0x1a, 0xab, 0xd3, 0x35, 0x9b,
0x95, 0xe7, 0x4c, 0x58, 0x4d, 0xb5, 0x4f, 0x07, 0x12, 0x55, 0x7b, 0xcc, 0x6b, 0xfd, 0x04, 0xbc,
0xfe, 0x53, 0x82, 0xce, 0xb7, 0xe8, 0x50, 0xc8, 0xea, 0x9b, 0x49, 0x56, 0x5f, 0x2e, 0x61, 0x55,
0xc5, 0x81, 0x6f, 0xe1, 0x23, 0xfc, 0x5f, 0xc6, 0xec, 0x9f, 0x25, 0x18, 0x8e, 0x8e, 0x5d, 0x43,
0xe5, 0xde, 0x74, 0x7a, 0x93, 0xbf, 0x0c, 0xdd, 0xa3, 0x54, 0x93, 0xc4, 0x99, 0xee, 0x1c, 0x25,
0xbb, 0x24, 0x15, 0xe4, 0xb0, 0x16, 0x12, 0xcc, 0x86, 0xc1, 0xed, 0x95, 0x22, 0xaa, 0x33, 0xc4,
0xb1, 0xe0, 0xd0, 0xf7, 0xd3, 0x40, 0xc5, 0x87, 0x95, 0x82, 0x79, 0xe8, 0x3c, 0x2c, 0x8b, 0x86,
0x4c, 0x24, 0x31, 0xee, 0x83, 0x26, 0xcd, 0x61, 0xf1, 0x91, 0x82, 0x65, 0xe6, 0x0b, 0x20, 0x93,
0xaa, 0x2f, 0xaa, 0x9c, 0xcd, 0x9c, 0x7a, 0x4c, 0xa2, 0xfc, 0x41, 0x82, 0xd5, 0x77, 0x75, 0xd7,
0xf4, 0x0e, 0x0e, 0x4e, 0x2f, 0xb9, 0x2d, 0x48, 0x15, 0xd5, 0x8b, 0xb6, 0xeb, 0xa9, 0x45, 0xe8,
0x55, 0x38, 0xe3, 0xf3, 0xe8, 0x65, 0xa6, 0x45, 0x5b, 0x55, 0xe5, 0x70, 0x20, 0x12, 0xd9, 0x6f,
0x2a, 0x80, 0x68, 0xc4, 0xbd, 0xa9, 0xdb, 0xba, 0x6b, 0xe0, 0xa7, 0x27, 0xfd, 0x0a, 0xf4, 0x52,
0x79, 0x22, 0x3a, 0xd9, 0x4f, 0x26, 0x0a, 0x82, 0x6e, 0x43, 0x6f, 0xcc, 0x51, 0x69, 0x3e, 0xd6,
0x89, 0xe7, 0xb2, 0x00, 0xd8, 0x2b, 0xee, 0xcc, 0xef, 0xf9, 0xd6, 0x64, 0x82, 0xfd, 0x2d, 0xcf,
0x35, 0x79, 0xa4, 0xee, 0x8e, 0x43, 0x32, 0xe9, 0x52, 0xe6, 0x5b, 0x51, 0xd2, 0x0c, 0x3b, 0x42,
0x88, 0xb2, 0x26, 0x13, 0x05, 0xc1, 0xba, 0x1d, 0x0b, 0x22, 0x8e, 0x98, 0x32, 0x1f, 0x18, 0x95,
0x1f, 0xcc, 0x14, 0x24, 0x31, 0xe5, 0x77, 0x12, 0xa0, 0xa8, 0x37, 0x62, 0xdd, 0x08, 0xb3, 0xb0,
0xec, 0x52, 0xa9, 0x20, 0x70, 0x5f, 0x80, 0x96, 0x19, 0xae, 0x14, 0x1e, 0x11, 0x03, 0xa8, 0xcf,
0x70, 0x36, 0x34, 0x9a, 0xf1, 0xb0, 0x19, 0x56, 0xda, 0x1c, 0x78, 0x87, 0xc1, 0xd2, 0x39, 0xb0,
0x96, 0xcd, 0x81, 0xc9, 0x73, 0x87, 0x7a, 0xea, 0xdc, 0x41, 0xf9, 0xa4, 0x02, 0x32, 0x8b, 0x67,
0x5b, 0x71, 0x53, 0xb2, 0x10, 0xd1, 0x97, 0xa1, 0x2b, 0xee, 0xbe, 0x52, 0x84, 0x77, 0x1e, 0x25,
0x36, 0x43, 0xd7, 0xe0, 0x2c, 0x9f, 0xe4, 0x63, 0x32, 0xb3, 0xe3, 0x22, 0x93, 0x57, 0x7c, 0xe8,
0x11, 0x0f, 0xa4, 0x74, 0x28, 0x5c, 0x71, 0x1f, 0x56, 0x27, 0xb6, 0x37, 0xd6, 0x6d, 0x2d, 0xad,
0x9e, 0x92, 0x86, 0x2a, 0x6f, 0xf1, 0x67, 0xf9, 0xf2, 0x51, 0x52, 0x87, 0x04, 0xed, 0xd0, 0x36,
0x17, 0x3f, 0x8c, 0x8a, 0x00, 0x71, 0xa4, 0xbc, 0x48, 0x0d, 0xd0, 0xa1, 0x0b, 0xc3, 0x3f, 0xe5,
0x63, 0x09, 0xfa, 0x99, 0xa3, 0xc3, 0x6c, 0xb3, 0x24, 0xe5, 0x9b, 0xa5, 0x37, 0xa1, 0x4e, 0x3b,
0x08, 0x1e, 0xef, 0x7a, 0xc5, 0x85, 0x7c, 0x7a, 0x57, 0x95, 0x2f, 0x40, 0x57, 0x61, 0xa5, 0xe0,
0xa2, 0x45, 0xd8, 0x00, 0xca, 0xdf, 0xb3, 0x28, 0xbf, 0xaf, 0x41, 0x3b, 0x21, 0x8f, 0x39, 0x7d,
0xde, 0x22, 0x67, 0x41, 0x19, 0xf6, 0xaa, 0x79, 0xf6, 0x4a, 0x6e, 0x1a, 0xa8, 0xdd, 0x39, 0xd8,
0xe1, 0x15, 0xb2, 0x28, 0xd7, 0x1d, 0xec, 0xb0, 0xc6, 0x83, 0x9a, 0xe4, 0xcc, 0xe1, 0x1d, 0x1a,
0x77, 0xa7, 0x65, 0x77, 0xe6, 0xb0, 0xfe, 0x2c, 0xdd, 0x1c, 0x2c, 0x3f, 0xa1, 0x39, 0x68, 0xa6,
0x9b, 0x83, 0x94, 0x1f, 0xb5, 0xb2, 0x7e, 0xb4, 0x68, 0xeb, 0x75, 0x0d, 0x56, 0x0c, 0x1f, 0xeb,
0x01, 0x36, 0x6f, 0x1e, 0x6f, 0x45, 0x43, 0x83, 0x36, 0xcb, 0xab, 0x45, 0x43, 0xe8, 0x56, 0x7c,
0x86, 0xc2, 0xb5, 0xdc, 0x61, 0x5a, 0x2e, 0xee, 0x3d, 0x84, 0x6e, 0xb8, 0x92, 0xc3, 0xf0, 0xcc,
0xfe, 0xb2, 0x4d, 0x5f, 0xf7, 0xa9, 0x9a, 0xbe, 0x17, 0xa0, 0x1d, 0x9f, 0x24, 0x90, 0x41, 0x8f,
0x47, 0xbe, 0xe8, 0x28, 0x81, 0xa4, 0x82, 0x41, 0x3f, 0x1d, 0x0c, 0xfe, 0x52, 0x85, 0x5e, 0x5c,
0xee, 0x2f, 0x1c, 0x0a, 0x16, 0xb9, 0x30, 0xdc, 0x03, 0x39, 0x4e, 0xa8, 0x4c, 0x4a, 0x4f, 0xec,
0x58, 0xb2, 0xa7, 0xf3, 0xfd, 0x69, 0xc6, 0xe7, 0x52, 0xe7, 0x9f, 0xb5, 0x13, 0x9d, 0x7f, 0x9e,
0xf2, 0x5e, 0xe9, 0x3a, 0x9c, 0x8b, 0x92, 0x68, 0x8a, 0x6d, 0x5e, 0x4d, 0x9f, 0x0d, 0x07, 0xf7,
0x93, 0xec, 0x97, 0xb8, 0xf1, 0x72, 0x99, 0x1b, 0x67, 0xd5, 0xd8, 0xcc, 0xa9, 0x31, 0x7f, 0xbd,
0xd5, 0x2a, 0xba, 0xde, 0xba, 0x0f, 0x2b, 0xf7, 0x5d, 0x32, 0x1b, 0x13, 0xc3, 0xb7, 0xc6, 0x38,
0x3c, 0x1b, 0x5b, 0x48, 0xad, 0x43, 0x68, 0x8a, 0x78, 0xcd, 0x55, 0xda, 0x52, 0xa3, 0x7f, 0xe5,
0x87, 0x12, 0xac, 0xe6, 0xf7, 0x65, 0x16, 0x13, 0x07, 0x03, 0x29, 0x15, 0x0c, 0xbe, 0x03, 0x2b,
0xf1, 0xf6, 0x5a, 0x6a, 0xe7, 0x92, 0xca, 0xae, 0x80, 0x70, 0x15, 0xc5, 0x7b, 0x84, 0x30, 0xe5,
0x5f, 0x52, 0x74, 0xc2, 0x48, 0x61, 0x13, 0x76, 0x6e, 0x4a, 0x13, 0x94, 0xe7, 0xda, 0x96, 0x1b,
0xb5, 0xa7, 0x82, 0x47, 0x0e, 0x14, 0xed, 0xe9, 0xbb, 0xd0, 0x17, 0x93, 0xa2, 0x3c, 0xb3, 0x60,
0x65, 0xd5, 0xe3, 0xeb, 0xa2, 0x0c, 0x73, 0x05, 0x7a, 0xde, 0xc1, 0x41, 0x12, 0x1f, 0x0f, 0x94,
0x5d, 0x01, 0x15, 0x08, 0xbf, 0x09, 0x72, 0x38, 0xed, 0xa4, 0x99, 0xad, 0x2f, 0x16, 0x46, 0x15,
0xda, 0x0f, 0x24, 0x18, 0xa4, 0xf3, 0x5c, 0x82, 0xfd, 0x93, 0xd7, 0x69, 0x5f, 0x4b, 0x5f, 0x05,
0x5d, 0x79, 0x02, 0x3d, 0x31, 0x1e, 0x71, 0x96, 0xb0, 0xfe, 0x0d, 0x68, 0x45, 0xed, 0x03, 0x6a,
0xc3, 0xf2, 0x7d, 0xf7, 0xb6, 0xeb, 0x3d, 0x76, 0xe5, 0x25, 0xb4, 0x0c, 0xd5, 0x1b, 0xb6, 0x2d,
0x4b, 0xa8, 0x0b, 0xad, 0x51, 0xe0, 0x63, 0xdd, 0xb1, 0xdc, 0x89, 0x5c, 0x41, 0x3d, 0x80, 0x77,
0x2d, 0x12, 0x78, 0xbe, 0x65, 0xe8, 0xb6, 0x5c, 0x5d, 0xff, 0x08, 0x7a, 0x69, 0xaf, 0x47, 0x1d,
0x68, 0xee, 0x79, 0xc1, 0x3b, 0x1f, 0x5a, 0x24, 0x90, 0x97, 0xe8, 0xfc, 0x3d, 0x2f, 0xd8, 0xf7,
0x31, 0xc1, 0x6e, 0x20, 0x4b, 0x08, 0xa0, 0xf1, 0x9e, 0xbb, 0x6d, 0x91, 0x87, 0x72, 0x05, 0xad,
0x88, 0xa4, 0xac, 0xdb, 0xbb, 0xc2, 0x95, 0xe4, 0x2a, 0x5d, 0x1e, 0xfd, 0xd5, 0x90, 0x0c, 0x9d,
0x68, 0xca, 0xce, 0xfe, 0x7d, 0xb9, 0x8e, 0x5a, 0x50, 0xe7, 0x9f, 0x8d, 0x75, 0x13, 0xe4, 0x6c,
0x45, 0x49, 0xf7, 0xe4, 0x4c, 0x44, 0x20, 0x79, 0x89, 0x72, 0x26, 0x4a, 0x7a, 0x59, 0x42, 0x7d,
0x68, 0x27, 0x0a, 0x64, 0xb9, 0x42, 0x01, 0x3b, 0xfe, 0xd4, 0x10, 0xa5, 0x32, 0x27, 0x81, 0xea,
0x7d, 0x9b, 0x4a, 0xa2, 0xb6, 0x7e, 0x13, 0x9a, 0x61, 0x38, 0xa2, 0x53, 0x85, 0x88, 0xe8, 0xaf,
0xbc, 0x84, 0xce, 0x40, 0x37, 0x75, 0x29, 0x2f, 0x4b, 0x08, 0x41, 0x2f, 0xfd, 0x60, 0x42, 0xae,
0x6c, 0xfe, 0xa2, 0x03, 0xc0, 0xeb, 0x35, 0xcf, 0xf3, 0x4d, 0x34, 0x05, 0xb4, 0x83, 0x03, 0x9a,
0x8b, 0x3c, 0x37, 0xcc, 0x23, 0x04, 0x5d, 0x2b, 0x29, 0x6b, 0xf2, 0x53, 0x05, 0xa9, 0xc3, 0xb2,
0xc6, 0x35, 0x33, 0x5d, 0x59, 0x42, 0x0e, 0xc3, 0x78, 0xcf, 0x72, 0xf0, 0x3d, 0xcb, 0x78, 0x18,
0x15, 0x7a, 0xe5, 0x18, 0x33, 0x53, 0x43, 0x8c, 0x99, 0xb0, 0x2f, 0x7e, 0x46, 0x81, 0x6f, 0xb9,
0x93, 0xf0, 0x6a, 0x4f, 0x59, 0x42, 0x8f, 0xe0, 0xec, 0x0e, 0x66, 0xd8, 0x2d, 0x12, 0x58, 0x06,
0x09, 0x11, 0x6e, 0x96, 0x23, 0xcc, 0x4d, 0x3e, 0x21, 0x4a, 0x1b, 0xfa, 0x99, 0x87, 0x4d, 0x68,
0xbd, 0xf8, 0x76, 0xb0, 0xe8, 0x11, 0xd6, 0xf0, 0xd5, 0x85, 0xe6, 0x46, 0xd8, 0x2c, 0xe8, 0xa5,
0x1f, 0xef, 0xa0, 0xff, 0x2f, 0xdb, 0x20, 0xf7, 0x3e, 0x61, 0xb8, 0xbe, 0xc8, 0xd4, 0x08, 0xd5,
0x03, 0x6e, 0x4f, 0xf3, 0x50, 0x15, 0xbe, 0x0d, 0x19, 0x3e, 0xe9, 0x56, 0x55, 0x59, 0x42, 0xdf,
0x85, 0x33, 0xb9, 0x57, 0x14, 0xe8, 0x4b, 0xc5, 0xfd, 0x7a, 0xf1, 0x63, 0x8b, 0x79, 0x18, 0x1e,
0x64, 0xbd, 0xa1, 0x9c, 0xfa, 0xdc, 0xab, 0x9b, 0xc5, 0xa9, 0x4f, 0x6c, 0xff, 0x24, 0xea, 0x4f,
0x8c, 0x61, 0xc6, 0xdc, 0x26, 0xdb, 0x39, 0xbc, 0x56, 0x84, 0xa2, 0xf4, 0x29, 0xc7, 0x70, 0x63,
0xd1, 0xe9, 0x49, 0xeb, 0x4a, 0xbf, 0x16, 0x28, 0x16, 0x5a, 0xe1, 0x0b, 0x87, 0x62, 0xeb, 0x2a,
0x7e, 0x7c, 0xa0, 0x2c, 0xa1, 0x7b, 0xa9, 0x68, 0x88, 0x5e, 0x2e, 0x53, 0x4e, 0xfa, 0x3c, 0x61,
0x9e, 0xdc, 0x34, 0x80, 0x1d, 0x1c, 0xdc, 0xc5, 0x81, 0x6f, 0x19, 0x24, 0xbb, 0xa9, 0xf8, 0x89,
0x27, 0x84, 0x9b, 0xbe, 0x32, 0x77, 0x5e, 0x44, 0xf6, 0x18, 0xda, 0x3b, 0x38, 0x10, 0x87, 0x43,
0x04, 0x95, 0xae, 0x0c, 0x67, 0x84, 0x28, 0xd6, 0xe6, 0x4f, 0x4c, 0x46, 0x94, 0xcc, 0xe3, 0x05,
0x54, 0x2a, 0xdb, 0xfc, 0x93, 0x8a, 0xe2, 0x88, 0x52, 0xf2, 0x1a, 0x42, 0x59, 0xda, 0xfc, 0x37,
0x40, 0x8b, 0xa5, 0x08, 0x9a, 0x7a, 0xfe, 0x97, 0x21, 0x9e, 0x41, 0x86, 0xf8, 0x00, 0xfa, 0x99,
0xbb, 0xf0, 0x62, 0x7d, 0x16, 0x5f, 0x98, 0xcf, 0x33, 0xf9, 0x31, 0xa0, 0xfc, 0x6d, 0x76, 0x71,
0xa8, 0x28, 0xbd, 0xf5, 0x9e, 0x87, 0xe3, 0x7d, 0xfe, 0x9c, 0x24, 0xaa, 0x5e, 0x5f, 0x29, 0xf3,
0xd6, 0xcc, 0xc9, 0xe5, 0xe7, 0x1f, 0x48, 0x9f, 0x7d, 0xa2, 0xf9, 0x00, 0xfa, 0x99, 0x3b, 0x9e,
0x62, 0xed, 0x16, 0x5f, 0x04, 0xcd, 0xdb, 0xfd, 0x33, 0x8c, 0xc8, 0x26, 0xac, 0x14, 0x9c, 0xde,
0xa3, 0xc2, 0x2c, 0x52, 0x7e, 0xcc, 0x3f, 0x8f, 0xa1, 0x11, 0x34, 0xf8, 0x4d, 0x0e, 0x7a, 0xb1,
0xb8, 0x63, 0x48, 0xdc, 0xf2, 0x0c, 0xe7, 0xdd, 0x05, 0x91, 0x99, 0x1d, 0xf0, 0x4d, 0xeb, 0x2c,
0x84, 0xa1, 0xc2, 0x8b, 0xc0, 0xe4, 0x05, 0xcc, 0x70, 0xfe, 0x9d, 0x4b, 0xb8, 0xe9, 0xb3, 0xce,
0x25, 0x37, 0x5f, 0x7f, 0xb0, 0x39, 0xb1, 0x82, 0xc3, 0xd9, 0x98, 0x0a, 0xe9, 0x2a, 0x9f, 0xf9,
0x9a, 0xe5, 0x89, 0xaf, 0xab, 0x21, 0x69, 0x57, 0xd9, 0x4e, 0x57, 0x19, 0x2f, 0xd3, 0xf1, 0xb8,
0xc1, 0x7e, 0xaf, 0xff, 0x27, 0x00, 0x00, 0xff, 0xff, 0x98, 0x70, 0x46, 0x0e, 0x4c, 0x30, 0x00,
0x00,
// 2879 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xec, 0x1a, 0xcb, 0x6f, 0x1c, 0x67,
0xdd, 0xb3, 0x2f, 0xef, 0xfe, 0xf6, 0x35, 0xf9, 0x9c, 0x38, 0xdb, 0x25, 0x69, 0xdd, 0x49, 0xd3,
0x1a, 0x97, 0x3a, 0xc1, 0x29, 0x55, 0x0b, 0x45, 0x22, 0xb1, 0x1b, 0xd7, 0x24, 0x71, 0xcd, 0x6c,
0x52, 0x50, 0x54, 0x69, 0x98, 0x9d, 0xf9, 0xbc, 0x1e, 0x65, 0x1e, 0x9b, 0xf9, 0x66, 0x9d, 0xba,
0x67, 0x84, 0xc4, 0xeb, 0x80, 0x7a, 0x45, 0x88, 0x03, 0x1c, 0x90, 0xa8, 0xb8, 0x70, 0x41, 0x42,
0x88, 0x1b, 0x57, 0x24, 0xfe, 0x00, 0x8e, 0x08, 0xee, 0xdc, 0xd1, 0xf7, 0x9a, 0x9d, 0x99, 0x9d,
0x89, 0x37, 0xd9, 0xa6, 0x2d, 0x12, 0xb7, 0xf9, 0x7e, 0xdf, 0xe3, 0xf7, 0x7e, 0x7d, 0xdf, 0xc0,
0x99, 0x87, 0x13, 0x1c, 0x9e, 0x18, 0x56, 0x10, 0x84, 0xf6, 0xe6, 0x38, 0x0c, 0xa2, 0x00, 0x21,
0xcf, 0x71, 0x8f, 0x27, 0x84, 0x8f, 0x36, 0xd9, 0x7c, 0xbf, 0x65, 0x05, 0x9e, 0x17, 0xf8, 0x1c,
0xd6, 0x6f, 0x25, 0x57, 0xf4, 0x3b, 0x8e, 0x1f, 0xe1, 0xd0, 0x37, 0x5d, 0x39, 0x4b, 0xac, 0x23,
0xec, 0x99, 0x62, 0xa4, 0xda, 0x66, 0x64, 0x26, 0xcf, 0xd7, 0x7e, 0xa0, 0xc0, 0xea, 0xe0, 0x28,
0x78, 0xb4, 0x1d, 0xb8, 0x2e, 0xb6, 0x22, 0x27, 0xf0, 0x89, 0x8e, 0x1f, 0x4e, 0x30, 0x89, 0xd0,
0x55, 0xa8, 0x0c, 0x4d, 0x82, 0x7b, 0xca, 0x9a, 0xb2, 0xde, 0xdc, 0xba, 0xb0, 0x99, 0xa2, 0x44,
0x90, 0x70, 0x87, 0x8c, 0x6e, 0x98, 0x04, 0xeb, 0x6c, 0x25, 0x42, 0x50, 0xb1, 0x87, 0x7b, 0x3b,
0xbd, 0xd2, 0x9a, 0xb2, 0x5e, 0xd6, 0xd9, 0x37, 0x7a, 0x09, 0xda, 0x56, 0x7c, 0xf6, 0xde, 0x0e,
0xe9, 0x95, 0xd7, 0xca, 0xeb, 0x65, 0x3d, 0x0d, 0xd4, 0xfe, 0xa1, 0xc0, 0xf9, 0x19, 0x32, 0xc8,
0x38, 0xf0, 0x09, 0x46, 0xd7, 0xa0, 0x46, 0x22, 0x33, 0x9a, 0x10, 0x41, 0xc9, 0x97, 0x72, 0x29,
0x19, 0xb0, 0x25, 0xba, 0x58, 0x3a, 0x8b, 0xb6, 0x94, 0x83, 0x16, 0x7d, 0x15, 0xce, 0x3a, 0xfe,
0x1d, 0xec, 0x05, 0xe1, 0x89, 0x31, 0xc6, 0xa1, 0x85, 0xfd, 0xc8, 0x1c, 0x61, 0x49, 0xe3, 0x8a,
0x9c, 0x3b, 0x98, 0x4e, 0xa1, 0x37, 0xe0, 0x3c, 0xd7, 0x12, 0xc1, 0xe1, 0xb1, 0x63, 0x61, 0xc3,
0x3c, 0x36, 0x1d, 0xd7, 0x1c, 0xba, 0xb8, 0x57, 0x59, 0x2b, 0xaf, 0xd7, 0xf5, 0x73, 0x6c, 0x7a,
0xc0, 0x67, 0xaf, 0xcb, 0x49, 0xed, 0x37, 0x0a, 0x9c, 0xa3, 0x1c, 0x1e, 0x98, 0x61, 0xe4, 0x3c,
0x03, 0x39, 0x6b, 0xd0, 0x4a, 0xf2, 0xd6, 0x2b, 0xb3, 0xb9, 0x14, 0x8c, 0xae, 0x19, 0x4b, 0xf4,
0x54, 0x26, 0x15, 0xc6, 0x66, 0x0a, 0xa6, 0xfd, 0x5a, 0x18, 0x44, 0x92, 0xce, 0x45, 0x14, 0x91,
0xc5, 0x59, 0x9a, 0xc5, 0xf9, 0x14, 0x6a, 0xd0, 0xfe, 0xa9, 0xc0, 0xb9, 0xdb, 0x81, 0x69, 0x4f,
0x0d, 0xe6, 0xb3, 0x17, 0xe7, 0x37, 0xa1, 0xc6, 0xbd, 0xab, 0x57, 0x61, 0xb8, 0x2e, 0xa7, 0x71,
0x09, 0xcf, 0x9b, 0x52, 0x38, 0x60, 0x00, 0x5d, 0x6c, 0x42, 0x97, 0xa1, 0x13, 0xe2, 0xb1, 0xeb,
0x58, 0xa6, 0xe1, 0x4f, 0xbc, 0x21, 0x0e, 0x7b, 0xd5, 0x35, 0x65, 0xbd, 0xaa, 0xb7, 0x05, 0x74,
0x9f, 0x01, 0xb5, 0x5f, 0x28, 0xd0, 0xd3, 0xb1, 0x8b, 0x4d, 0x82, 0x3f, 0x4f, 0x66, 0x57, 0xa1,
0xe6, 0x07, 0x36, 0xde, 0xdb, 0x61, 0xcc, 0x96, 0x75, 0x31, 0xd2, 0x7e, 0x52, 0xe2, 0x8a, 0xf8,
0x82, 0xdb, 0x75, 0x42, 0x59, 0xd5, 0x4f, 0x47, 0x59, 0xb5, 0x3c, 0x65, 0xfd, 0x65, 0xaa, 0xac,
0x2f, 0xba, 0x40, 0xa6, 0x0a, 0xad, 0xa6, 0x14, 0xfa, 0x5b, 0x05, 0x9e, 0xdb, 0xc5, 0x51, 0x4c,
0x3e, 0xf5, 0x67, 0xfc, 0x05, 0x0d, 0x56, 0x9f, 0x28, 0xd0, 0xcf, 0xa3, 0x75, 0x91, 0x80, 0x75,
0x1f, 0x56, 0x63, 0x1c, 0x86, 0x8d, 0x89, 0x15, 0x3a, 0x63, 0xa6, 0x46, 0x16, 0xba, 0x9a, 0x5b,
0x97, 0x36, 0x67, 0x53, 0xf2, 0x66, 0x96, 0x82, 0x73, 0xf1, 0x11, 0x3b, 0x89, 0x13, 0xb4, 0x9f,
0x29, 0x70, 0x6e, 0x17, 0x47, 0x03, 0x3c, 0xf2, 0xb0, 0x1f, 0xed, 0xf9, 0x87, 0xc1, 0xd3, 0xcb,
0xf5, 0x79, 0x00, 0x22, 0xce, 0x89, 0xc3, 0x6a, 0x02, 0x32, 0x8f, 0x8c, 0x59, 0xf6, 0xcf, 0xd2,
0xb3, 0x88, 0xec, 0xbe, 0x06, 0x55, 0xc7, 0x3f, 0x0c, 0xa4, 0xa8, 0x5e, 0xc8, 0x13, 0x55, 0x12,
0x19, 0x5f, 0xad, 0xf9, 0x9c, 0x8a, 0x23, 0x33, 0xb4, 0x6f, 0x63, 0xd3, 0xc6, 0xe1, 0x02, 0xe6,
0x96, 0x65, 0xbb, 0x94, 0xc3, 0xf6, 0x4f, 0x15, 0x38, 0x3f, 0x83, 0x70, 0x11, 0xbe, 0xdf, 0x86,
0x1a, 0xa1, 0x87, 0x49, 0xc6, 0x5f, 0xca, 0x65, 0x3c, 0x81, 0xee, 0xb6, 0x43, 0x22, 0x5d, 0xec,
0xd1, 0x02, 0x50, 0xb3, 0x73, 0xe8, 0x45, 0x68, 0x59, 0x47, 0xa6, 0xef, 0x63, 0xd7, 0xf0, 0x4d,
0x8f, 0x0b, 0xa0, 0xa1, 0x37, 0x05, 0x6c, 0xdf, 0xf4, 0x30, 0x7a, 0x0e, 0xea, 0xd4, 0x65, 0x0d,
0xc7, 0x96, 0xea, 0x5f, 0x66, 0x2e, 0x6c, 0x13, 0x74, 0x11, 0x80, 0x4d, 0x99, 0xb6, 0x1d, 0xf2,
0x34, 0xda, 0xd0, 0x1b, 0x14, 0x72, 0x9d, 0x02, 0xb4, 0x9f, 0x2b, 0xd0, 0xa2, 0x31, 0xfb, 0x0e,
0x8e, 0x4c, 0xaa, 0x07, 0xf4, 0x16, 0x34, 0xdc, 0xc0, 0xb4, 0x8d, 0xe8, 0x64, 0xcc, 0x51, 0x75,
0xb2, 0xb2, 0xe6, 0x2c, 0xd0, 0x4d, 0x77, 0x4f, 0xc6, 0x58, 0xaf, 0xbb, 0xe2, 0x6b, 0x1e, 0x79,
0xcf, 0xb8, 0x72, 0x39, 0xc7, 0x95, 0x7f, 0x58, 0x85, 0xd5, 0xef, 0x9a, 0x91, 0x75, 0xb4, 0xe3,
0x6d, 0x73, 0x26, 0x17, 0x30, 0x82, 0x69, 0x6c, 0x2b, 0x25, 0x63, 0xdb, 0xa7, 0x16, 0x3b, 0x63,
0x3b, 0xaf, 0xe6, 0xd9, 0x39, 0x2d, 0xb2, 0x37, 0xdf, 0x17, 0xaa, 0x4a, 0xd8, 0x79, 0x22, 0x07,
0xd5, 0x9e, 0x26, 0x07, 0x6d, 0x43, 0x1b, 0x7f, 0x68, 0xb9, 0x13, 0xaa, 0x73, 0x86, 0x7d, 0x99,
0x61, 0x7f, 0x3e, 0x07, 0x7b, 0xd2, 0xc9, 0x5a, 0x62, 0xd3, 0x9e, 0xa0, 0x81, 0xab, 0xda, 0xc3,
0x91, 0xd9, 0xab, 0x33, 0x32, 0xd6, 0x8a, 0x54, 0x2d, 0xed, 0x83, 0xab, 0x9b, 0x8e, 0xd0, 0x05,
0x68, 0x88, 0x8c, 0xb7, 0xb7, 0xd3, 0x6b, 0x30, 0xf1, 0x4d, 0x01, 0xc8, 0x84, 0xb6, 0x88, 0x40,
0x82, 0x42, 0x60, 0x14, 0xbe, 0x9d, 0x87, 0x20, 0x5f, 0xd9, 0x49, 0xca, 0xc9, 0x3b, 0x7e, 0x14,
0x9e, 0xe8, 0x2d, 0x92, 0x00, 0xf5, 0x0d, 0x38, 0x33, 0xb3, 0x04, 0xa9, 0x50, 0x7e, 0x80, 0x4f,
0x98, 0x81, 0x94, 0x75, 0xfa, 0x89, 0x5e, 0x87, 0xea, 0xb1, 0xe9, 0x4e, 0x30, 0x33, 0x80, 0xd3,
0x65, 0xc4, 0x17, 0x7f, 0xbd, 0xf4, 0xa6, 0xa2, 0xfd, 0xb9, 0x02, 0x5d, 0x31, 0x45, 0x65, 0xc0,
0xfc, 0xe3, 0x02, 0x34, 0xe2, 0xc8, 0x2a, 0xb0, 0x4c, 0x01, 0x68, 0x0d, 0x9a, 0x09, 0xeb, 0x10,
0x26, 0x97, 0x04, 0xcd, 0x65, 0x77, 0x32, 0x4f, 0x56, 0x12, 0x79, 0xf2, 0x22, 0xc0, 0xa1, 0x3b,
0x21, 0x47, 0x46, 0xe4, 0x78, 0x58, 0xe4, 0xe9, 0x06, 0x83, 0xdc, 0x75, 0x3c, 0x8c, 0xae, 0x43,
0x6b, 0xe8, 0xf8, 0x6e, 0x30, 0x32, 0xc6, 0x66, 0x74, 0x44, 0x7a, 0xb5, 0x42, 0x7b, 0xb8, 0xe9,
0x60, 0xd7, 0xbe, 0xc1, 0xd6, 0xea, 0x4d, 0xbe, 0xe7, 0x80, 0x6e, 0x41, 0xcf, 0x43, 0xd3, 0x9f,
0x78, 0x46, 0x70, 0x68, 0x84, 0xc1, 0x23, 0x6a, 0x51, 0x0c, 0x85, 0x3f, 0xf1, 0xde, 0x3b, 0xd4,
0x83, 0x47, 0x34, 0xb2, 0x35, 0x68, 0x8c, 0x23, 0x6e, 0x30, 0x22, 0xbd, 0xfa, 0x5c, 0xe7, 0x4f,
0x37, 0xd0, 0xdd, 0x36, 0x76, 0x23, 0x93, 0xed, 0x6e, 0xcc, 0xb7, 0x3b, 0xde, 0x80, 0x5e, 0x86,
0x8e, 0x15, 0x78, 0x63, 0x93, 0x49, 0xe8, 0x66, 0x18, 0x78, 0xcc, 0x9c, 0xca, 0x7a, 0x06, 0x8a,
0xb6, 0xa1, 0xe9, 0xf8, 0x36, 0xfe, 0x50, 0xd8, 0x5c, 0x93, 0xe1, 0xd1, 0xf2, 0x6c, 0x8e, 0x21,
0xda, 0xa3, 0x6b, 0x99, 0xd6, 0xc1, 0x91, 0x9f, 0x84, 0x06, 0x5c, 0x69, 0xba, 0xc4, 0xf9, 0x08,
0xf7, 0x5a, 0x5c, 0x8b, 0x02, 0x36, 0x70, 0x3e, 0xc2, 0xb4, 0x06, 0x74, 0x7c, 0x82, 0xc3, 0xc8,
0x10, 0xbe, 0xdd, 0x6b, 0xb3, 0xa8, 0xdc, 0xe6, 0x50, 0x61, 0xc9, 0xda, 0xef, 0x4b, 0xd0, 0x49,
0x23, 0x42, 0x3d, 0x58, 0x3e, 0x64, 0x10, 0x69, 0x3d, 0x72, 0x48, 0xd1, 0x62, 0x9f, 0x36, 0x88,
0x06, 0xa3, 0x85, 0x19, 0x4f, 0x5d, 0x6f, 0x72, 0x18, 0x3b, 0x80, 0x1a, 0x01, 0x67, 0x8f, 0x25,
0x82, 0x32, 0x43, 0xd9, 0x60, 0x10, 0x96, 0x06, 0x7a, 0xb0, 0xcc, 0xd9, 0x90, 0xa6, 0x23, 0x87,
0x74, 0x66, 0x38, 0x71, 0x18, 0x56, 0x6e, 0x3a, 0x72, 0x88, 0x76, 0xa0, 0xc5, 0x8f, 0x1c, 0x9b,
0xa1, 0xe9, 0x49, 0xc3, 0x79, 0x31, 0x37, 0xb2, 0xde, 0xc2, 0x27, 0xef, 0x53, 0xe7, 0x38, 0x30,
0x9d, 0x50, 0xe7, 0x82, 0x3e, 0x60, 0xbb, 0xd0, 0x3a, 0xa8, 0xfc, 0x94, 0x43, 0xc7, 0xc5, 0xc2,
0x04, 0x97, 0x59, 0xae, 0xe9, 0x30, 0xf8, 0x4d, 0xc7, 0xc5, 0xdc, 0xca, 0x62, 0x16, 0x98, 0x68,
0xeb, 0xdc, 0xc8, 0x18, 0x84, 0x0a, 0x56, 0xfb, 0x7b, 0x19, 0x56, 0xa8, 0xaf, 0x09, 0xb7, 0x5b,
0x20, 0xf0, 0x5f, 0x04, 0xb0, 0x49, 0x64, 0xa4, 0x82, 0x7f, 0xc3, 0x26, 0xd1, 0x3e, 0x8f, 0xff,
0x6f, 0xc9, 0xb8, 0x5d, 0x2e, 0x2e, 0xe5, 0x32, 0xbe, 0x3f, 0x1b, 0xbb, 0x9f, 0xaa, 0xd9, 0xbb,
0x04, 0x6d, 0x12, 0x4c, 0x42, 0x0b, 0x1b, 0xa9, 0xa2, 0xbb, 0xc5, 0x81, 0xfb, 0xf9, 0xe9, 0xa9,
0x96, 0xdb, 0x74, 0x26, 0xe2, 0xf7, 0xf2, 0x62, 0xf1, 0xbb, 0x9e, 0x8d, 0xdf, 0xb7, 0xa0, 0xcb,
0xdc, 0xcf, 0x18, 0x07, 0x84, 0xf7, 0x2e, 0xc2, 0x6b, 0x33, 0xde, 0x14, 0x5f, 0x32, 0xdd, 0x21,
0xa3, 0x03, 0xb1, 0x54, 0xef, 0xb0, 0xad, 0x72, 0x48, 0xb4, 0x8f, 0x4b, 0xb0, 0x2a, 0x7a, 0xa1,
0xc5, 0x15, 0x5b, 0x94, 0xd1, 0x65, 0xd4, 0x2c, 0x3f, 0xa6, 0xbb, 0xa8, 0xcc, 0x91, 0xe5, 0xab,
0x39, 0x59, 0x3e, 0x5d, 0x61, 0xd7, 0x66, 0x2a, 0xec, 0x6b, 0x50, 0x25, 0x56, 0x30, 0xc6, 0x4c,
0x0d, 0x9d, 0xad, 0x8b, 0x79, 0x6a, 0xd8, 0x31, 0x23, 0x73, 0x40, 0x17, 0xe9, 0x7c, 0xad, 0xf6,
0x2f, 0x05, 0xda, 0x03, 0x6c, 0x86, 0xd6, 0x91, 0x14, 0xc6, 0x1b, 0x50, 0x0e, 0xf1, 0x43, 0x21,
0x8b, 0x97, 0x0a, 0x04, 0x9d, 0xda, 0xa2, 0xd3, 0x0d, 0xe8, 0x05, 0x68, 0xda, 0x9e, 0x1b, 0xc7,
0xa2, 0x12, 0x0b, 0x0c, 0x60, 0x7b, 0xae, 0x08, 0x44, 0x19, 0xfa, 0xcb, 0x33, 0xf4, 0x6f, 0xc0,
0x99, 0xc3, 0x30, 0xf0, 0x0c, 0x56, 0x86, 0x1a, 0x2e, 0xab, 0x3e, 0x99, 0xb0, 0xea, 0x7a, 0x97,
0x4e, 0x24, 0x8a, 0xd2, 0x29, 0xaf, 0xd5, 0x27, 0xe0, 0xf5, 0xdf, 0x0a, 0xb4, 0xbe, 0x43, 0xa7,
0x24, 0xab, 0x6f, 0x26, 0x59, 0x7d, 0xb9, 0x80, 0x55, 0x1d, 0x47, 0xa1, 0x83, 0x8f, 0xf1, 0xff,
0x18, 0xb3, 0x7f, 0x55, 0xa0, 0x3f, 0x38, 0xf1, 0x2d, 0x9d, 0x7b, 0xd3, 0xe2, 0x26, 0x7f, 0x09,
0xda, 0xc7, 0xa9, 0x1e, 0x80, 0x33, 0xdd, 0x3a, 0x4e, 0x36, 0x01, 0x3a, 0xa8, 0xf2, 0x5e, 0x42,
0x30, 0x2b, 0x83, 0xdb, 0x2b, 0x79, 0x54, 0x67, 0x88, 0x63, 0xc1, 0xa1, 0x1b, 0xa6, 0x81, 0x5a,
0x08, 0x2b, 0x39, 0xeb, 0xd0, 0x79, 0x58, 0x16, 0xfd, 0x86, 0x48, 0x62, 0xdc, 0x07, 0x6d, 0x9a,
0xc3, 0xa6, 0x1d, 0xb3, 0x63, 0xcf, 0x16, 0x40, 0x36, 0x55, 0x5f, 0x5c, 0x18, 0xda, 0x33, 0xea,
0xb1, 0x89, 0xf6, 0x27, 0x05, 0x56, 0xdf, 0x35, 0x7d, 0x3b, 0x38, 0x3c, 0x5c, 0x5c, 0x72, 0xdb,
0x90, 0xaa, 0x19, 0xe7, 0xed, 0x46, 0x53, 0x9b, 0xd0, 0xab, 0x70, 0x26, 0xe4, 0xd1, 0xcb, 0x4e,
0x8b, 0xb6, 0xac, 0xab, 0x72, 0x22, 0x16, 0xd9, 0xef, 0x4a, 0x80, 0x68, 0xc4, 0xbd, 0x61, 0xba,
0xa6, 0x6f, 0xe1, 0xa7, 0x27, 0xfd, 0x32, 0x74, 0x52, 0x79, 0x22, 0xbe, 0xb8, 0x4e, 0x26, 0x0a,
0x82, 0x6e, 0x41, 0x67, 0xc8, 0x51, 0x19, 0x21, 0x36, 0x49, 0xe0, 0xb3, 0x00, 0xd8, 0xc9, 0x6f,
0x3c, 0xef, 0x86, 0xce, 0x68, 0x84, 0xc3, 0xed, 0xc0, 0xb7, 0x79, 0xa4, 0x6e, 0x0f, 0x25, 0x99,
0x74, 0x2b, 0xf3, 0xad, 0x38, 0x69, 0xca, 0x86, 0x07, 0xe2, 0xac, 0xc9, 0x44, 0x41, 0xb0, 0xe9,
0x4e, 0x05, 0x31, 0x8d, 0x98, 0x2a, 0x9f, 0x18, 0x14, 0xdf, 0x3b, 0xe4, 0x24, 0x31, 0xed, 0x0f,
0x0a, 0xa0, 0xb8, 0xf4, 0x67, 0x8d, 0x00, 0xb3, 0xb0, 0xec, 0x56, 0x25, 0x27, 0x70, 0x5f, 0x80,
0x86, 0x2d, 0x77, 0x0a, 0x8f, 0x98, 0x02, 0xa8, 0xcf, 0x70, 0x36, 0x0c, 0x9a, 0xf1, 0xb0, 0x2d,
0x2b, 0x6d, 0x0e, 0xbc, 0xcd, 0x60, 0xe9, 0x1c, 0x58, 0xc9, 0xe6, 0xc0, 0x64, 0x5b, 0x5d, 0x4d,
0xb5, 0xd5, 0xda, 0x27, 0x25, 0x50, 0x59, 0x3c, 0xdb, 0x9e, 0xf6, 0x76, 0x73, 0x11, 0x7d, 0x09,
0xda, 0xe2, 0x69, 0x27, 0x45, 0x78, 0xeb, 0x61, 0xe2, 0x30, 0x74, 0x15, 0xce, 0xf2, 0x45, 0x21,
0x26, 0x13, 0x77, 0x5a, 0x64, 0xf2, 0x8a, 0x0f, 0x3d, 0xe4, 0x81, 0x94, 0x4e, 0xc9, 0x1d, 0xf7,
0x60, 0x75, 0xe4, 0x06, 0x43, 0xd3, 0x35, 0xd2, 0xea, 0xe1, 0x3a, 0x9c, 0xc3, 0xe2, 0xcf, 0xf2,
0xed, 0x83, 0xa4, 0x0e, 0x09, 0xda, 0xa5, 0x5d, 0x1c, 0x7e, 0x10, 0x17, 0x01, 0xe2, 0xc6, 0x74,
0x9e, 0x1a, 0xa0, 0x45, 0x37, 0xca, 0x91, 0xf6, 0x4b, 0x05, 0xba, 0x99, 0x9b, 0xb1, 0x6c, 0xb3,
0xa4, 0xcc, 0x36, 0x4b, 0x6f, 0x42, 0x95, 0x76, 0x10, 0x3c, 0xde, 0x75, 0xf2, 0x0b, 0xf9, 0xf4,
0xa9, 0x3a, 0xdf, 0x80, 0xae, 0xc0, 0x4a, 0xce, 0x3b, 0x82, 0xb0, 0x01, 0x34, 0xfb, 0x8c, 0xa0,
0xfd, 0xb1, 0x02, 0xcd, 0x84, 0x3c, 0x4e, 0xe9, 0xf3, 0xe6, 0xb9, 0xea, 0xc8, 0xb0, 0x57, 0x9e,
0x65, 0xaf, 0xe0, 0x22, 0x9d, 0xda, 0x9d, 0x87, 0x3d, 0x5e, 0x21, 0x8b, 0x72, 0xdd, 0xc3, 0x1e,
0x6b, 0x3c, 0xa8, 0x49, 0x4e, 0x3c, 0xde, 0xa1, 0x71, 0x77, 0x5a, 0xf6, 0x27, 0x1e, 0xeb, 0xcf,
0xd2, 0xcd, 0xc1, 0xf2, 0x63, 0x9a, 0x83, 0x7a, 0xba, 0x39, 0x48, 0xf9, 0x51, 0x23, 0xeb, 0x47,
0xf3, 0xb6, 0x5e, 0x57, 0x61, 0xc5, 0x0a, 0xb1, 0x19, 0x61, 0xfb, 0xc6, 0xc9, 0x76, 0x3c, 0xd5,
0x6b, 0xb2, 0xbc, 0x9a, 0x37, 0x85, 0x6e, 0x4e, 0xaf, 0x08, 0xb8, 0x96, 0x5b, 0x4c, 0xcb, 0xf9,
0xbd, 0x87, 0xd0, 0x0d, 0x57, 0xb2, 0x0c, 0xcf, 0x6c, 0x94, 0x6d, 0xfa, 0xda, 0x4f, 0xd5, 0xf4,
0xbd, 0x00, 0x4d, 0x99, 0x3d, 0xa9, 0xbb, 0x77, 0x78, 0xe4, 0x93, 0xb1, 0xc0, 0x26, 0xa9, 0x60,
0xd0, 0x4d, 0x07, 0x83, 0xbf, 0x95, 0xa1, 0x33, 0x2d, 0xf7, 0xe7, 0x0e, 0x05, 0xf3, 0xbc, 0x87,
0xed, 0x83, 0x3a, 0x4d, 0xa8, 0x4c, 0x4a, 0x8f, 0xed, 0x58, 0xb2, 0x97, 0xcf, 0xdd, 0x71, 0xc6,
0xe7, 0x52, 0xd7, 0x7b, 0x95, 0x27, 0xba, 0xde, 0x5b, 0xf0, 0xd9, 0xe4, 0x1a, 0x9c, 0x8b, 0x93,
0x68, 0x8a, 0x6d, 0x5e, 0x4d, 0x9f, 0x95, 0x93, 0x07, 0x49, 0xf6, 0x0b, 0xdc, 0x78, 0xb9, 0xc8,
0x8d, 0xb3, 0x6a, 0xac, 0xcf, 0xa8, 0x71, 0xf6, 0xf5, 0xa6, 0x91, 0xf7, 0x7a, 0x73, 0x0f, 0x56,
0xee, 0xf9, 0x64, 0x32, 0x24, 0x56, 0xe8, 0x0c, 0xb1, 0xbc, 0x99, 0x9a, 0x4b, 0xad, 0x7d, 0xa8,
0x8b, 0x78, 0xcd, 0x55, 0xda, 0xd0, 0xe3, 0xb1, 0xf6, 0x63, 0x05, 0x56, 0x67, 0xcf, 0x65, 0x16,
0x33, 0x0d, 0x06, 0x4a, 0x2a, 0x18, 0x7c, 0x0f, 0x56, 0xa6, 0xc7, 0x1b, 0xa9, 0x93, 0x0b, 0x2a,
0xbb, 0x1c, 0xc2, 0x75, 0x34, 0x3d, 0x43, 0xc2, 0xb4, 0xff, 0x28, 0xf1, 0x05, 0x1a, 0x85, 0x8d,
0xd8, 0xb5, 0x20, 0x4d, 0x50, 0x81, 0xef, 0x3a, 0x7e, 0xdc, 0x9e, 0x0a, 0x1e, 0x39, 0x50, 0xb4,
0xa7, 0xef, 0x42, 0x57, 0x2c, 0x8a, 0xf3, 0xcc, 0x9c, 0x95, 0x55, 0x87, 0xef, 0x8b, 0x33, 0xcc,
0x65, 0xe8, 0x04, 0x87, 0x87, 0x49, 0x7c, 0x3c, 0x50, 0xb6, 0x05, 0x54, 0x20, 0xfc, 0x36, 0xa8,
0x72, 0xd9, 0x93, 0x66, 0xb6, 0xae, 0xd8, 0x18, 0x57, 0x68, 0x3f, 0x52, 0xa0, 0x97, 0xce, 0x73,
0x09, 0xf6, 0x9f, 0xbc, 0x4e, 0xfb, 0x46, 0xfa, 0xa5, 0xe3, 0xf2, 0x63, 0xe8, 0x99, 0xe2, 0x11,
0x77, 0x09, 0x1b, 0xdf, 0x82, 0x46, 0xdc, 0x3e, 0xa0, 0x26, 0x2c, 0xdf, 0xf3, 0x6f, 0xf9, 0xc1,
0x23, 0x5f, 0x5d, 0x42, 0xcb, 0x50, 0xbe, 0xee, 0xba, 0xaa, 0x82, 0xda, 0xd0, 0x18, 0x44, 0x21,
0x36, 0x3d, 0xc7, 0x1f, 0xa9, 0x25, 0xd4, 0x01, 0x78, 0xd7, 0x21, 0x51, 0x10, 0x3a, 0x96, 0xe9,
0xaa, 0xe5, 0x8d, 0x8f, 0xa0, 0x93, 0xf6, 0x7a, 0xd4, 0x82, 0xfa, 0x7e, 0x10, 0xbd, 0xf3, 0xa1,
0x43, 0x22, 0x75, 0x89, 0xae, 0xdf, 0x0f, 0xa2, 0x83, 0x10, 0x13, 0xec, 0x47, 0xaa, 0x82, 0x00,
0x6a, 0xef, 0xf9, 0x3b, 0x0e, 0x79, 0xa0, 0x96, 0xd0, 0x8a, 0x48, 0xca, 0xa6, 0xbb, 0x27, 0x5c,
0x49, 0x2d, 0xd3, 0xed, 0xf1, 0xa8, 0x82, 0x54, 0x68, 0xc5, 0x4b, 0x76, 0x0f, 0xee, 0xa9, 0x55,
0xd4, 0x80, 0x2a, 0xff, 0xac, 0x6d, 0xd8, 0xa0, 0x66, 0x2b, 0x4a, 0x7a, 0x26, 0x67, 0x22, 0x06,
0xa9, 0x4b, 0x94, 0x33, 0x51, 0xd2, 0xab, 0x0a, 0xea, 0x42, 0x33, 0x51, 0x20, 0xab, 0x25, 0x0a,
0xd8, 0x0d, 0xc7, 0x96, 0x28, 0x95, 0x39, 0x09, 0x54, 0xef, 0x3b, 0x54, 0x12, 0x95, 0x8d, 0x1b,
0x50, 0x97, 0xe1, 0x88, 0x2e, 0x15, 0x22, 0xa2, 0x43, 0x75, 0x09, 0x9d, 0x81, 0x76, 0xea, 0xcd,
0x59, 0x55, 0x10, 0x82, 0x4e, 0xfa, 0x7f, 0x00, 0xb5, 0xb4, 0xf5, 0xab, 0x16, 0x00, 0xaf, 0xd7,
0x82, 0x20, 0xb4, 0xd1, 0x18, 0xd0, 0x2e, 0x8e, 0x68, 0x2e, 0x0a, 0x7c, 0x99, 0x47, 0x08, 0xba,
0x5a, 0x50, 0xd6, 0xcc, 0x2e, 0x15, 0xa4, 0xf6, 0x8b, 0x1a, 0xd7, 0xcc, 0x72, 0x6d, 0x09, 0x79,
0x0c, 0xe3, 0x5d, 0xc7, 0xc3, 0x77, 0x1d, 0xeb, 0x41, 0x5c, 0xe8, 0x15, 0x63, 0xcc, 0x2c, 0x95,
0x18, 0x33, 0x61, 0x5f, 0x0c, 0x06, 0x51, 0xe8, 0xf8, 0x23, 0xf9, 0x72, 0xa5, 0x2d, 0xa1, 0x87,
0x70, 0x76, 0x17, 0x33, 0xec, 0x0e, 0x89, 0x1c, 0x8b, 0x48, 0x84, 0x5b, 0xc5, 0x08, 0x67, 0x16,
0x3f, 0x21, 0x4a, 0x17, 0xba, 0x99, 0xff, 0x76, 0xd0, 0x46, 0xfe, 0xe3, 0x57, 0xde, 0x3f, 0x46,
0xfd, 0x57, 0xe7, 0x5a, 0x1b, 0x63, 0x73, 0xa0, 0x93, 0xfe, 0x37, 0x05, 0x7d, 0xb9, 0xe8, 0x80,
0x99, 0xe7, 0xf7, 0xfe, 0xc6, 0x3c, 0x4b, 0x63, 0x54, 0xf7, 0xb9, 0x3d, 0x9d, 0x86, 0x2a, 0xf7,
0xd7, 0x87, 0xfe, 0xe3, 0x1e, 0x0d, 0xb5, 0x25, 0xf4, 0x7d, 0x38, 0x33, 0xf3, 0x93, 0x00, 0xfa,
0x4a, 0x7e, 0xbf, 0x9e, 0xff, 0x2f, 0xc1, 0x69, 0x18, 0xee, 0x67, 0xbd, 0xa1, 0x98, 0xfa, 0x99,
0x9f, 0x4a, 0xe6, 0xa7, 0x3e, 0x71, 0xfc, 0xe3, 0xa8, 0x7f, 0x62, 0x0c, 0x13, 0xe6, 0x36, 0xd9,
0xce, 0xe1, 0xb5, 0x3c, 0x14, 0x85, 0x7f, 0x2a, 0xf4, 0x37, 0xe7, 0x5d, 0x9e, 0xb4, 0xae, 0xf4,
0x63, 0x78, 0xbe, 0xd0, 0x72, 0x1f, 0xf0, 0xf3, 0xad, 0x2b, 0xff, 0x6d, 0x5d, 0x5b, 0x42, 0x77,
0x53, 0xd1, 0x10, 0xbd, 0x5c, 0xa4, 0x9c, 0xf4, 0x7d, 0xc2, 0x69, 0x72, 0x33, 0x00, 0x76, 0x71,
0x74, 0x07, 0x47, 0xa1, 0x63, 0x91, 0xec, 0xa1, 0x62, 0x30, 0x5d, 0x20, 0x0f, 0x7d, 0xe5, 0xd4,
0x75, 0x31, 0xd9, 0x43, 0x68, 0xee, 0xe2, 0x48, 0x5c, 0x0e, 0x11, 0x54, 0xb8, 0x53, 0xae, 0x90,
0x28, 0xd6, 0x4f, 0x5f, 0x98, 0x8c, 0x28, 0x99, 0xb7, 0x79, 0x54, 0x28, 0xdb, 0xd9, 0x3f, 0x06,
0xf2, 0x23, 0x4a, 0xc1, 0x63, 0xbf, 0xb6, 0xb4, 0xf5, 0x31, 0x40, 0x83, 0xa5, 0x08, 0x9a, 0x7a,
0xfe, 0x9f, 0x21, 0x9e, 0x41, 0x86, 0xf8, 0x00, 0xba, 0x99, 0xa7, 0xde, 0x7c, 0x7d, 0xe6, 0xbf,
0x07, 0x9f, 0x66, 0xf2, 0xef, 0xf3, 0x3f, 0x19, 0xe2, 0xca, 0xf2, 0x95, 0x22, 0x4f, 0xca, 0xdc,
0x2a, 0x7e, 0xfe, 0x41, 0xee, 0xd9, 0x27, 0x81, 0x0f, 0xa0, 0x9b, 0x79, 0x7f, 0xc9, 0x97, 0x7c,
0xfe, 0x23, 0xcd, 0x69, 0xa7, 0x7f, 0x86, 0xd1, 0xd2, 0x86, 0x95, 0x9c, 0x9b, 0x75, 0x94, 0x1b,
0xe1, 0x8b, 0xaf, 0xe0, 0x4f, 0x63, 0x68, 0x00, 0x35, 0xfe, 0xca, 0x82, 0x5e, 0xcc, 0xaf, 0xe6,
0x13, 0x2f, 0x30, 0xfd, 0xd3, 0xde, 0x69, 0xc8, 0xc4, 0x8d, 0xf8, 0xa1, 0x55, 0x16, 0x5e, 0x50,
0xee, 0x23, 0x5d, 0xf2, 0x71, 0xa4, 0x7f, 0xfa, 0x7b, 0x88, 0x3c, 0xf4, 0x59, 0xc7, 0xf9, 0x1b,
0xaf, 0xdf, 0xdf, 0x1a, 0x39, 0xd1, 0xd1, 0x64, 0x48, 0x85, 0x74, 0x85, 0xaf, 0x7c, 0xcd, 0x09,
0xc4, 0xd7, 0x15, 0x49, 0xda, 0x15, 0x76, 0xd2, 0x15, 0xc6, 0xcb, 0x78, 0x38, 0xac, 0xb1, 0xe1,
0xb5, 0xff, 0x06, 0x00, 0x00, 0xff, 0xff, 0x68, 0x67, 0x67, 0x75, 0xc7, 0x2e, 0x00, 0x00,
}
// Reference imports to suppress errors if they are not otherwise used.
@ -3587,7 +3503,6 @@ type QueryNodeClient interface {
GetTimeTickChannel(ctx context.Context, in *internalpb.GetTimeTickChannelRequest, opts ...grpc.CallOption) (*milvuspb.StringResponse, error)
GetStatisticsChannel(ctx context.Context, in *internalpb.GetStatisticsChannelRequest, opts ...grpc.CallOption) (*milvuspb.StringResponse, error)
WatchDmChannels(ctx context.Context, in *WatchDmChannelsRequest, opts ...grpc.CallOption) (*commonpb.Status, error)
WatchDeltaChannels(ctx context.Context, in *WatchDeltaChannelsRequest, opts ...grpc.CallOption) (*commonpb.Status, error)
LoadSegments(ctx context.Context, in *LoadSegmentsRequest, opts ...grpc.CallOption) (*commonpb.Status, error)
ReleaseCollection(ctx context.Context, in *ReleaseCollectionRequest, opts ...grpc.CallOption) (*commonpb.Status, error)
ReleasePartitions(ctx context.Context, in *ReleasePartitionsRequest, opts ...grpc.CallOption) (*commonpb.Status, error)
@ -3644,15 +3559,6 @@ func (c *queryNodeClient) WatchDmChannels(ctx context.Context, in *WatchDmChanne
return out, nil
}
func (c *queryNodeClient) WatchDeltaChannels(ctx context.Context, in *WatchDeltaChannelsRequest, opts ...grpc.CallOption) (*commonpb.Status, error) {
out := new(commonpb.Status)
err := c.cc.Invoke(ctx, "/milvus.proto.query.QueryNode/WatchDeltaChannels", in, out, opts...)
if err != nil {
return nil, err
}
return out, nil
}
func (c *queryNodeClient) LoadSegments(ctx context.Context, in *LoadSegmentsRequest, opts ...grpc.CallOption) (*commonpb.Status, error) {
out := new(commonpb.Status)
err := c.cc.Invoke(ctx, "/milvus.proto.query.QueryNode/LoadSegments", in, out, opts...)
@ -3740,7 +3646,6 @@ type QueryNodeServer interface {
GetTimeTickChannel(context.Context, *internalpb.GetTimeTickChannelRequest) (*milvuspb.StringResponse, error)
GetStatisticsChannel(context.Context, *internalpb.GetStatisticsChannelRequest) (*milvuspb.StringResponse, error)
WatchDmChannels(context.Context, *WatchDmChannelsRequest) (*commonpb.Status, error)
WatchDeltaChannels(context.Context, *WatchDeltaChannelsRequest) (*commonpb.Status, error)
LoadSegments(context.Context, *LoadSegmentsRequest) (*commonpb.Status, error)
ReleaseCollection(context.Context, *ReleaseCollectionRequest) (*commonpb.Status, error)
ReleasePartitions(context.Context, *ReleasePartitionsRequest) (*commonpb.Status, error)
@ -3769,9 +3674,6 @@ func (*UnimplementedQueryNodeServer) GetStatisticsChannel(ctx context.Context, r
func (*UnimplementedQueryNodeServer) WatchDmChannels(ctx context.Context, req *WatchDmChannelsRequest) (*commonpb.Status, error) {
return nil, status.Errorf(codes.Unimplemented, "method WatchDmChannels not implemented")
}
func (*UnimplementedQueryNodeServer) WatchDeltaChannels(ctx context.Context, req *WatchDeltaChannelsRequest) (*commonpb.Status, error) {
return nil, status.Errorf(codes.Unimplemented, "method WatchDeltaChannels not implemented")
}
func (*UnimplementedQueryNodeServer) LoadSegments(ctx context.Context, req *LoadSegmentsRequest) (*commonpb.Status, error) {
return nil, status.Errorf(codes.Unimplemented, "method LoadSegments not implemented")
}
@ -3876,24 +3778,6 @@ func _QueryNode_WatchDmChannels_Handler(srv interface{}, ctx context.Context, de
return interceptor(ctx, in, info, handler)
}
func _QueryNode_WatchDeltaChannels_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
in := new(WatchDeltaChannelsRequest)
if err := dec(in); err != nil {
return nil, err
}
if interceptor == nil {
return srv.(QueryNodeServer).WatchDeltaChannels(ctx, in)
}
info := &grpc.UnaryServerInfo{
Server: srv,
FullMethod: "/milvus.proto.query.QueryNode/WatchDeltaChannels",
}
handler := func(ctx context.Context, req interface{}) (interface{}, error) {
return srv.(QueryNodeServer).WatchDeltaChannels(ctx, req.(*WatchDeltaChannelsRequest))
}
return interceptor(ctx, in, info, handler)
}
func _QueryNode_LoadSegments_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
in := new(LoadSegmentsRequest)
if err := dec(in); err != nil {
@ -4076,10 +3960,6 @@ var _QueryNode_serviceDesc = grpc.ServiceDesc{
MethodName: "WatchDmChannels",
Handler: _QueryNode_WatchDmChannels_Handler,
},
{
MethodName: "WatchDeltaChannels",
Handler: _QueryNode_WatchDeltaChannels_Handler,
},
{
MethodName: "LoadSegments",
Handler: _QueryNode_LoadSegments_Handler,

5
internal/proxy/query_node_mock_test.go
View File

@ -58,11 +58,6 @@ func (m *QueryNodeMock) WatchDmChannels(ctx context.Context, req *querypb.WatchD
return nil, nil
}
// TODO
func (m *QueryNodeMock) WatchDeltaChannels(ctx context.Context, req *querypb.WatchDeltaChannelsRequest) (*commonpb.Status, error) {
return nil, nil
}
// TODO
func (m *QueryNodeMock) LoadSegments(ctx context.Context, req *querypb.LoadSegmentsRequest) (*commonpb.Status, error) {
return nil, nil

93
internal/querycoord/cluster.go
View File

@ -62,8 +62,6 @@ type Cluster interface {
// Channel
WatchDmChannels(ctx context.Context, nodeID int64, in *querypb.WatchDmChannelsRequest) error
WatchDeltaChannels(ctx context.Context, nodeID int64, in *querypb.WatchDeltaChannelsRequest) error
HasWatchedDeltaChannel(ctx context.Context, nodeID int64, collectionID UniqueID) bool
// Node
RegisterNode(ctx context.Context, session *sessionutil.Session, id UniqueID, state nodeState) error
@ -213,48 +211,44 @@ func (c *queryNodeCluster) LoadSegments(ctx context.Context, nodeID int64, in *q
if targetNode != nil {
collectionID := in.CollectionID
// if node has watched the collection's deltaChannel
// then the node should recover part delete log from dmChanel
if c.HasWatchedDeltaChannel(ctx, nodeID, collectionID) {
// get all deltaChannelInfo of the collection from meta
deltaChannelInfos, err := c.clusterMeta.getDeltaChannelsByCollectionID(collectionID)
// get all deltaChannelInfo of the collection from meta
deltaChannelInfos, err := c.clusterMeta.getDeltaChannelsByCollectionID(collectionID)
if err != nil {
// this case should not happen
// deltaChannelInfos should have been set to meta before executing child tasks
log.Error("loadSegments: failed to get deltaChannelInfo from meta", zap.Error(err))
return err
}
deltaChannel2Info := make(map[string]*datapb.VchannelInfo, len(deltaChannelInfos))
for _, info := range deltaChannelInfos {
deltaChannel2Info[info.ChannelName] = info
}
// check delta channel which should be reloaded
reloadDeltaChannels := make(map[string]struct{})
for _, segment := range in.Infos {
// convert vChannel to deltaChannel
deltaChannelName, err := funcutil.ConvertChannelName(segment.InsertChannel, Params.CommonCfg.RootCoordDml, Params.CommonCfg.RootCoordDelta)
if err != nil {
// this case should not happen
// deltaChannelInfos should have been set to meta before executing child tasks
log.Error("loadSegments: failed to get deltaChannelInfo from meta", zap.Error(err))
return err
}
deltaChannel2Info := make(map[string]*datapb.VchannelInfo, len(deltaChannelInfos))
for _, info := range deltaChannelInfos {
deltaChannel2Info[info.ChannelName] = info
}
// check delta channel which should be reloaded
reloadDeltaChannels := make(map[string]struct{})
for _, segment := range in.Infos {
// convert vChannel to deltaChannel
deltaChannelName, err := funcutil.ConvertChannelName(segment.InsertChannel, Params.CommonCfg.RootCoordDml, Params.CommonCfg.RootCoordDelta)
if err != nil {
return err
}
reloadDeltaChannels[deltaChannelName] = struct{}{}
}
reloadDeltaChannels[deltaChannelName] = struct{}{}
}
in.DeltaPositions = make([]*internalpb.MsgPosition, 0)
for deltaChannelName := range reloadDeltaChannels {
if info, ok := deltaChannel2Info[deltaChannelName]; ok {
in.DeltaPositions = append(in.DeltaPositions, info.SeekPosition)
} else {
// this case should not happen
err = fmt.Errorf("loadSegments: can't find deltaChannelInfo, channel name = %s", deltaChannelName)
log.Error(err.Error())
return err
}
in.DeltaPositions = make([]*internalpb.MsgPosition, 0)
for deltaChannelName := range reloadDeltaChannels {
if info, ok := deltaChannel2Info[deltaChannelName]; ok {
in.DeltaPositions = append(in.DeltaPositions, info.SeekPosition)
} else {
// this case should not happen
err = fmt.Errorf("loadSegments: can't find deltaChannelInfo, channel name = %s", deltaChannelName)
log.Error(err.Error())
return err
}
}
err := targetNode.loadSegments(ctx, in)
err = targetNode.loadSegments(ctx, in)
if err != nil {
log.Warn("loadSegments: queryNode load segments error", zap.Int64("nodeID", nodeID), zap.String("error info", err.Error()))
return err
@ -332,33 +326,6 @@ func (c *queryNodeCluster) WatchDmChannels(ctx context.Context, nodeID int64, in
return fmt.Errorf("watchDmChannels: can't find QueryNode by nodeID, nodeID = %d", nodeID)
}
func (c *queryNodeCluster) WatchDeltaChannels(ctx context.Context, nodeID int64, in *querypb.WatchDeltaChannelsRequest) error {
c.RLock()
var targetNode Node
if node, ok := c.nodes[nodeID]; ok {
targetNode = node
}
c.RUnlock()
if targetNode != nil {
err := targetNode.watchDeltaChannels(ctx, in)
if err != nil {
return err
}
return nil
}
return fmt.Errorf("watchDeltaChannels: can't find QueryNode by nodeID, nodeID = %d", nodeID)
}
func (c *queryNodeCluster) HasWatchedDeltaChannel(ctx context.Context, nodeID int64, collectionID UniqueID) bool {
c.RLock()
defer c.RUnlock()
return c.nodes[nodeID].hasWatchedDeltaChannel(collectionID)
}
func (c *queryNodeCluster) ReleaseCollection(ctx context.Context, nodeID int64, in *querypb.ReleaseCollectionRequest) error {
c.RLock()
var targetNode Node

12
internal/querycoord/cluster_test.go
View File

@ -495,20 +495,10 @@ func TestGrpcRequest(t *testing.T) {
CollectionID: defaultCollectionID,
}
err := cluster.LoadSegments(baseCtx, nodeID, loadSegmentReq)
assert.Equal(t, 0, len(loadSegmentReq.DeltaPositions))
assert.Equal(t, 1, len(loadSegmentReq.DeltaPositions))
assert.Nil(t, err)
})
t.Run("Test WatchDeletaChannel", func(t *testing.T) {
watchDeltaChannelReq := &querypb.WatchDeltaChannelsRequest{
CollectionID: defaultCollectionID,
Infos: deltaChannelInfo,
}
err := cluster.WatchDeltaChannels(baseCtx, nodeID, watchDeltaChannelReq)
assert.Nil(t, err)
assert.Equal(t, true, cluster.HasWatchedDeltaChannel(baseCtx, nodeID, defaultCollectionID))
})
t.Run("Test LoadSegmentsAfterWatchDeltaChannel", func(t *testing.T) {
segmentLoadInfo := &querypb.SegmentLoadInfo{
SegmentID: defaultSegmentID,

19
internal/querycoord/mock_querynode_client_test.go
View File

@ -26,7 +26,6 @@ import (
etcdkv "github.com/milvus-io/milvus/internal/kv/etcd"
"github.com/milvus-io/milvus/internal/log"
"github.com/milvus-io/milvus/internal/proto/commonpb"
"github.com/milvus-io/milvus/internal/proto/datapb"
"github.com/milvus-io/milvus/internal/proto/internalpb"
"github.com/milvus-io/milvus/internal/proto/milvuspb"
"github.com/milvus-io/milvus/internal/proto/querypb"
@ -43,7 +42,6 @@ type queryNodeClientMock struct {
}
func newQueryNodeTest(ctx context.Context, address string, id UniqueID, kv *etcdkv.EtcdKV) (Node, error) {
watchedDeltaChannels := make(map[UniqueID][]*datapb.VchannelInfo)
childCtx, cancel := context.WithCancel(ctx)
client, err := newQueryNodeClientMock(childCtx, address)
if err != nil {
@ -51,13 +49,12 @@ func newQueryNodeTest(ctx context.Context, address string, id UniqueID, kv *etcd
return nil, err
}
node := &queryNode{
ctx: childCtx,
cancel: cancel,
id: id,
address: address,
client: client,
kvClient: kv,
watchedDeltaChannels: watchedDeltaChannels,
ctx: childCtx,
cancel: cancel,
id: id,
address: address,
client: client,
kvClient: kv,
}
return node, nil
@ -120,10 +117,6 @@ func (client *queryNodeClientMock) WatchDmChannels(ctx context.Context, req *que
return client.grpcClient.WatchDmChannels(ctx, req)
}
func (client *queryNodeClientMock) WatchDeltaChannels(ctx context.Context, req *querypb.WatchDeltaChannelsRequest) (*commonpb.Status, error) {
return client.grpcClient.WatchDeltaChannels(ctx, req)
}
func (client *queryNodeClientMock) LoadSegments(ctx context.Context, req *querypb.LoadSegmentsRequest) (*commonpb.Status, error) {
return client.grpcClient.LoadSegments(ctx, req)
}

4
internal/querycoord/mock_querynode_server_test.go
View File

@ -207,10 +207,6 @@ func (qs *queryNodeServerMock) WatchDmChannels(ctx context.Context, req *querypb
return qs.watchDmChannels()
}
func (qs *queryNodeServerMock) WatchDeltaChannels(ctx context.Context, req *querypb.WatchDeltaChannelsRequest) (*commonpb.Status, error) {
return qs.watchDeltaChannels()
}
func (qs *queryNodeServerMock) LoadSegments(ctx context.Context, req *querypb.LoadSegmentsRequest) (*commonpb.Status, error) {
sizePerRecord, err := typeutil.EstimateSizePerRecord(req.Schema)
if err != nil {

66
internal/querycoord/querynode.go
View File

@ -29,7 +29,6 @@ import (
etcdkv "github.com/milvus-io/milvus/internal/kv/etcd"
"github.com/milvus-io/milvus/internal/log"
"github.com/milvus-io/milvus/internal/proto/commonpb"
"github.com/milvus-io/milvus/internal/proto/datapb"
"github.com/milvus-io/milvus/internal/proto/internalpb"
"github.com/milvus-io/milvus/internal/proto/milvuspb"
"github.com/milvus-io/milvus/internal/proto/querypb"
@ -47,12 +46,7 @@ type Node interface {
releasePartitions(ctx context.Context, in *querypb.ReleasePartitionsRequest) error
watchDmChannels(ctx context.Context, in *querypb.WatchDmChannelsRequest) error
watchDeltaChannels(ctx context.Context, in *querypb.WatchDeltaChannelsRequest) error
syncReplicaSegments(ctx context.Context, in *querypb.SyncReplicaSegmentsRequest) error
//removeDmChannel(collectionID UniqueID, channels []string) error
hasWatchedDeltaChannel(collectionID UniqueID) bool
setState(state nodeState)
getState() nodeState
isOnline() bool
@ -75,9 +69,8 @@ type queryNode struct {
kvClient *etcdkv.EtcdKV
sync.RWMutex
watchedDeltaChannels map[UniqueID][]*datapb.VchannelInfo
state nodeState
stateLock sync.RWMutex
state nodeState
stateLock sync.RWMutex
totalMem uint64
memUsage uint64
@ -86,7 +79,6 @@ type queryNode struct {
}
func newQueryNode(ctx context.Context, address string, id UniqueID, kv *etcdkv.EtcdKV) (Node, error) {
watchedDeltaChannels := make(map[UniqueID][]*datapb.VchannelInfo)
childCtx, cancel := context.WithCancel(ctx)
client, err := nodeclient.NewClient(childCtx, address)
if err != nil {
@ -94,14 +86,13 @@ func newQueryNode(ctx context.Context, address string, id UniqueID, kv *etcdkv.E
return nil, err
}
node := &queryNode{
ctx: childCtx,
cancel: cancel,
id: id,
address: address,
client: client,
kvClient: kv,
watchedDeltaChannels: watchedDeltaChannels,
state: disConnect,
ctx: childCtx,
cancel: cancel,
id: id,
address: address,
client: client,
kvClient: kv,
state: disConnect,
}
return node, nil
@ -127,30 +118,12 @@ func (qn *queryNode) start() error {
}
func (qn *queryNode) stop() {
//qn.stateLock.Lock()
//defer qn.stateLock.Unlock()
//qn.state = offline
if qn.client != nil {
qn.client.Stop()
}
qn.cancel()
}
func (qn *queryNode) hasWatchedDeltaChannel(collectionID UniqueID) bool {
qn.RLock()
defer qn.RUnlock()
_, ok := qn.watchedDeltaChannels[collectionID]
return ok
}
func (qn *queryNode) setDeltaChannelInfo(collectionID int64, infos []*datapb.VchannelInfo) {
qn.Lock()
defer qn.Unlock()
qn.watchedDeltaChannels[collectionID] = infos
}
func (qn *queryNode) setState(state nodeState) {
qn.stateLock.Lock()
defer qn.stateLock.Unlock()
@ -196,22 +169,6 @@ func (qn *queryNode) watchDmChannels(ctx context.Context, in *querypb.WatchDmCha
return nil
}
func (qn *queryNode) watchDeltaChannels(ctx context.Context, in *querypb.WatchDeltaChannelsRequest) error {
if !qn.isOnline() {
return errors.New("WatchDeltaChannels: queryNode is offline")
}
status, err := qn.client.WatchDeltaChannels(qn.ctx, in)
if err != nil {
return err
}
if status.ErrorCode != commonpb.ErrorCode_Success {
return errors.New(status.Reason)
}
qn.setDeltaChannelInfo(in.CollectionID, in.Infos)
return err
}
func (qn *queryNode) releaseCollection(ctx context.Context, in *querypb.ReleaseCollectionRequest) error {
if !qn.isOnline() {
log.Warn("ReleaseCollection: the QueryNode has been offline, the release request is no longer needed", zap.Int64("nodeID", qn.id))
@ -225,11 +182,6 @@ func (qn *queryNode) releaseCollection(ctx context.Context, in *querypb.ReleaseC
if status.ErrorCode != commonpb.ErrorCode_Success {
return errors.New(status.Reason)
}
qn.Lock()
delete(qn.watchedDeltaChannels, in.CollectionID)
qn.Unlock()
return nil
}

90
internal/querycoord/task.go
View File

@ -1382,7 +1382,7 @@ func (lst *loadSegmentTask) execute(ctx context.Context) error {
err := lst.cluster.LoadSegments(ctx, lst.DstNodeID, lst.LoadSegmentsRequest)
if err != nil {
log.Warn("loadSegmentTask: loadSegment occur error", zap.Int64("taskID", lst.getTaskID()))
log.Warn("loadSegmentTask: loadSegment occur error", zap.Int64("taskID", lst.getTaskID()), zap.Error(err))
lst.setResultInfo(err)
return err
}
@ -1629,84 +1629,6 @@ func (wdt *watchDmChannelTask) reschedule(ctx context.Context) ([]task, error) {
return reScheduledTasks, nil
}
type watchDeltaChannelTask struct {
*baseTask
*querypb.WatchDeltaChannelsRequest
cluster Cluster
}
func (wdt *watchDeltaChannelTask) msgBase() *commonpb.MsgBase {
return wdt.Base
}
func (wdt *watchDeltaChannelTask) marshal() ([]byte, error) {
return proto.Marshal(wdt.WatchDeltaChannelsRequest)
}
func (wdt *watchDeltaChannelTask) isValid() bool {
online, err := wdt.cluster.IsOnline(wdt.NodeID)
if err != nil {
return false
}
return wdt.ctx != nil && online
}
func (wdt *watchDeltaChannelTask) msgType() commonpb.MsgType {
return wdt.Base.MsgType
}
func (wdt *watchDeltaChannelTask) timestamp() Timestamp {
return wdt.Base.Timestamp
}
func (wdt *watchDeltaChannelTask) updateTaskProcess() {
parentTask := wdt.getParentTask()
if parentTask == nil {
log.Warn("watchDeltaChannel: parentTask should not be nil")
return
}
parentTask.updateTaskProcess()
}
func (wdt *watchDeltaChannelTask) preExecute(context.Context) error {
channelInfos := wdt.Infos
channels := make([]string, 0)
for _, info := range channelInfos {
channels = append(channels, info.ChannelName)
}
wdt.setResultInfo(nil)
log.Info("start do watchDeltaChannelTask",
zap.Strings("deltaChannels", channels),
zap.Int64("loaded nodeID", wdt.NodeID),
zap.Int64("taskID", wdt.getTaskID()),
zap.Int64("msgID", wdt.GetBase().GetMsgID()))
return nil
}
func (wdt *watchDeltaChannelTask) execute(ctx context.Context) error {
defer wdt.reduceRetryCount()
err := wdt.cluster.WatchDeltaChannels(wdt.ctx, wdt.NodeID, wdt.WatchDeltaChannelsRequest)
if err != nil {
log.Warn("watchDeltaChannelTask: watchDeltaChannel occur error", zap.Int64("taskID", wdt.getTaskID()), zap.Error(err))
wdt.setResultInfo(err)
return err
}
log.Info("watchDeltaChannelsTask Execute done",
zap.Int64("taskID", wdt.getTaskID()),
zap.Int64("msgID", wdt.GetBase().GetMsgID()))
return nil
}
func (wdt *watchDeltaChannelTask) postExecute(context.Context) error {
log.Info("watchDeltaChannelTask postExecute done",
zap.Int64("taskID", wdt.getTaskID()),
zap.Int64("msgID", wdt.GetBase().GetMsgID()))
return nil
}
//****************************handoff task********************************//
type handoffTask struct {
*baseTask
@ -2648,13 +2570,13 @@ func mergeWatchDeltaChannelInfo(infos []*datapb.VchannelInfo) []*datapb.Vchannel
}
}
var result []*datapb.VchannelInfo
for _, index := range minPositions {
for channel, index := range minPositions {
result = append(result, infos[index])
log.Info("merge delta channels finished", zap.String("channel", channel),
zap.Any("merged info", infos[index]),
)
}
log.Info("merge delta channels finished",
zap.Any("origin info length", len(infos)),
zap.Any("merged info length", len(result)),
)
return result
}

81
internal/querycoord/task_scheduler.go
View File

@ -402,19 +402,7 @@ func (scheduler *TaskScheduler) unmarshalTask(taskID UniqueID, t string) (task,
}
newTask = watchDmChannelTask
case commonpb.MsgType_WatchDeltaChannels:
//TODO::trigger condition may be different
loadReq := querypb.WatchDeltaChannelsRequest{}
err = proto.Unmarshal([]byte(t), &loadReq)
reviseWatchDeltaChannelsRequest(&loadReq)
if err != nil {
return nil, err
}
watchDeltaChannelTask := &watchDeltaChannelTask{
baseTask: baseTask,
WatchDeltaChannelsRequest: &loadReq,
cluster: scheduler.cluster,
}
newTask = watchDeltaChannelTask
log.Warn("legacy WatchDeltaChannels type found, ignore")
case commonpb.MsgType_WatchQueryChannels:
//Deprecated WatchQueryChannel
log.Warn("legacy WatchQueryChannels type found, ignore")
@ -515,8 +503,6 @@ func (scheduler *TaskScheduler) processTask(t task) error {
protoSize = proto.Size(childTask.(*loadSegmentTask).LoadSegmentsRequest)
case commonpb.MsgType_WatchDmChannels:
protoSize = proto.Size(childTask.(*watchDmChannelTask).WatchDmChannelsRequest)
case commonpb.MsgType_WatchDeltaChannels:
protoSize = proto.Size(childTask.(*watchDeltaChannelTask).WatchDeltaChannelsRequest)
default:
//TODO::
}
@ -710,17 +696,6 @@ func (scheduler *TaskScheduler) scheduleLoop() {
if len(childTasks) != 0 {
// include loadSegment, watchDmChannel, releaseCollection, releasePartition, releaseSegment
processInternalTaskFn(childTasks, triggerTask)
if triggerTask.getResultInfo().ErrorCode == commonpb.ErrorCode_Success {
// derivedInternalTasks include watchDeltaChannel, watchQueryChannel
// derivedInternalTasks generate from loadSegment and watchDmChannel reqs
derivedInternalTasks, err := generateDerivedInternalTasks(triggerTask, scheduler.meta, scheduler.cluster)
if err != nil {
log.Error("scheduleLoop: generate derived watchDeltaChannel and watchcQueryChannel tasks failed", zap.Int64("triggerTaskID", triggerTask.getTaskID()), zap.Error(err))
triggerTask.setResultInfo(err)
} else {
processInternalTaskFn(derivedInternalTasks, triggerTask)
}
}
}
// triggerTask may be LoadCollection, LoadPartitions, LoadBalance, Handoff
@ -1076,57 +1051,3 @@ func updateSegmentInfoFromTask(ctx context.Context, triggerTask task, meta Meta)
return nil
}
// generateDerivedInternalTasks generate watchDeltaChannel and watchQueryChannel tasks
func generateDerivedInternalTasks(triggerTask task, meta Meta, cluster Cluster) ([]task, error) {
var derivedInternalTasks []task
watchDeltaChannelInfo := make(map[int64]map[UniqueID]UniqueID)
addChannelWatchInfoFn := func(nodeID int64, collectionID UniqueID, replicaID UniqueID, watchInfo map[int64]map[UniqueID]UniqueID) {
if _, ok := watchInfo[nodeID]; !ok {
watchInfo[nodeID] = make(map[UniqueID]UniqueID)
}
watchInfo[nodeID][collectionID] = replicaID
}
for _, childTask := range triggerTask.getChildTask() {
if childTask.msgType() == commonpb.MsgType_LoadSegments {
loadSegmentTask := childTask.(*loadSegmentTask)
collectionID := loadSegmentTask.CollectionID
replicaID := loadSegmentTask.GetReplicaID()
nodeID := loadSegmentTask.DstNodeID
if !cluster.HasWatchedDeltaChannel(triggerTask.traceCtx(), nodeID, collectionID) {
addChannelWatchInfoFn(nodeID, collectionID, replicaID, watchDeltaChannelInfo)
}
}
}
for nodeID, collectionIDs := range watchDeltaChannelInfo {
for collectionID, replicaID := range collectionIDs {
deltaChannelInfo, err := meta.getDeltaChannelsByCollectionID(collectionID)
if err != nil {
return nil, err
}
msgBase := proto.Clone(triggerTask.msgBase()).(*commonpb.MsgBase)
msgBase.MsgType = commonpb.MsgType_WatchDeltaChannels
watchDeltaRequest := &querypb.WatchDeltaChannelsRequest{
Base: msgBase,
CollectionID: collectionID,
Infos: deltaChannelInfo,
ReplicaId: replicaID,
}
watchDeltaRequest.NodeID = nodeID
baseTask := newBaseTask(triggerTask.traceCtx(), triggerTask.getTriggerCondition())
baseTask.setParentTask(triggerTask)
watchDeltaTask := &watchDeltaChannelTask{
baseTask: baseTask,
WatchDeltaChannelsRequest: watchDeltaRequest,
cluster: cluster,
}
derivedInternalTasks = append(derivedInternalTasks, watchDeltaTask)
}
}
return derivedInternalTasks, nil
}

64
internal/querycoord/task_scheduler_test.go
View File

@ -365,27 +365,6 @@ func TestUnMarshalTask(t *testing.T) {
assert.Nil(t, task2)
})
t.Run("Test watchDeltaChannelTask", func(t *testing.T) {
watchTask := &watchDeltaChannelTask{
WatchDeltaChannelsRequest: &querypb.WatchDeltaChannelsRequest{
Base: &commonpb.MsgBase{
MsgType: commonpb.MsgType_WatchDeltaChannels,
},
},
}
blobs, err := watchTask.marshal()
assert.Nil(t, err)
err = kv.Save("testMarshalWatchDeltaChannel", string(blobs))
assert.Nil(t, err)
defer kv.RemoveWithPrefix("testMarshalWatchDeltaChannel")
value, err := kv.Load("testMarshalWatchDeltaChannel")
assert.Nil(t, err)
task, err := taskScheduler.unmarshalTask(1007, value)
assert.Nil(t, err)
assert.Equal(t, task.msgType(), commonpb.MsgType_WatchDeltaChannels)
})
t.Run("Test loadBalanceTask", func(t *testing.T) {
loadBalanceTask := &loadBalanceTask{
LoadBalanceRequest: &querypb.LoadBalanceRequest{
@ -525,49 +504,6 @@ func Test_saveInternalTaskToEtcd(t *testing.T) {
})
}
func Test_generateDerivedInternalTasks(t *testing.T) {
refreshParams()
baseCtx := context.Background()
queryCoord, err := startQueryCoord(baseCtx)
assert.Nil(t, err)
node1, err := startQueryNodeServer(baseCtx)
assert.Nil(t, err)
waitQueryNodeOnline(queryCoord.cluster, node1.queryNodeID)
vChannelInfos, _, err := queryCoord.broker.getRecoveryInfo(baseCtx, defaultCollectionID, defaultPartitionID)
assert.NoError(t, err)
deltaChannelInfos := make([]*datapb.VchannelInfo, len(vChannelInfos))
for i, info := range vChannelInfos {
deltaInfo, err := generateWatchDeltaChannelInfo(info)
assert.NoError(t, err)
deltaChannelInfos[i] = deltaInfo
}
queryCoord.meta.setDeltaChannel(defaultCollectionID, deltaChannelInfos)
loadCollectionTask := genLoadCollectionTask(baseCtx, queryCoord)
loadSegmentTask := genLoadSegmentTask(baseCtx, queryCoord, node1.queryNodeID)
loadCollectionTask.addChildTask(loadSegmentTask)
loadSegmentTask.setParentTask(loadCollectionTask)
watchDmChannelTask := genWatchDmChannelTask(baseCtx, queryCoord, node1.queryNodeID)
loadCollectionTask.addChildTask(watchDmChannelTask)
watchDmChannelTask.setParentTask(loadCollectionTask)
derivedTasks, err := generateDerivedInternalTasks(loadCollectionTask, queryCoord.meta, queryCoord.cluster)
assert.Nil(t, err)
assert.Equal(t, 1, len(derivedTasks))
for _, internalTask := range derivedTasks {
matchType := internalTask.msgType() == commonpb.MsgType_WatchDeltaChannels
assert.Equal(t, true, matchType)
if internalTask.msgType() == commonpb.MsgType_WatchDeltaChannels {
assert.Equal(t, node1.queryNodeID, internalTask.(*watchDeltaChannelTask).NodeID)
}
}
queryCoord.Stop()
err = removeAllSession()
assert.Nil(t, err)
}
func TestTaskScheduler_BindContext(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
s := &TaskScheduler{

40
internal/querycoord/task_test.go
View File

@ -193,12 +193,14 @@ func genWatchDmChannelTask(ctx context.Context, queryCoord *QueryCoord, nodeID i
queryCoord.meta.addCollection(defaultCollectionID, querypb.LoadType_LoadCollection, schema)
return watchDmChannelTask
}
func genLoadSegmentTask(ctx context.Context, queryCoord *QueryCoord, nodeID int64) *loadSegmentTask {
schema := genDefaultCollectionSchema(false)
segmentInfo := &querypb.SegmentLoadInfo{
SegmentID: defaultSegmentID,
PartitionID: defaultPartitionID,
CollectionID: defaultCollectionID,
SegmentID: defaultSegmentID,
PartitionID: defaultPartitionID,
CollectionID: defaultCollectionID,
InsertChannel: "by-dev-rootcoord-dml_1_2021v1",
}
req := &querypb.LoadSegmentsRequest{
Base: &commonpb.MsgBase{
@ -249,29 +251,18 @@ func genLoadSegmentTask(ctx context.Context, queryCoord *QueryCoord, nodeID int6
loadSegmentTask.setParentTask(parentTask)
queryCoord.meta.addCollection(defaultCollectionID, querypb.LoadType_LoadCollection, schema)
return loadSegmentTask
}
func genWatchDeltaChannelTask(ctx context.Context, queryCoord *QueryCoord, nodeID int64) *watchDeltaChannelTask {
req := &querypb.WatchDeltaChannelsRequest{
Base: &commonpb.MsgBase{
MsgType: commonpb.MsgType_WatchDeltaChannels,
},
NodeID: nodeID,
CollectionID: defaultCollectionID,
LoadMeta: &querypb.LoadMetaInfo{
LoadType: querypb.LoadType_LoadCollection,
deltaChannelInfo := []*datapb.VchannelInfo{
{
CollectionID: defaultCollectionID,
PartitionIDs: []int64{defaultPartitionID},
ChannelName: "by-dev-rootcoord-delta_1_2021v1",
SeekPosition: &internalpb.MsgPosition{
ChannelName: "by-dev-rootcoord-dml_1",
},
},
}
baseTask := newBaseTask(ctx, querypb.TriggerCondition_GrpcRequest)
baseTask.taskID = 300
return &watchDeltaChannelTask{
baseTask: baseTask,
WatchDeltaChannelsRequest: req,
cluster: queryCoord.cluster,
}
queryCoord.meta.setDeltaChannel(defaultCollectionID, deltaChannelInfo)
return loadSegmentTask
}
func waitTaskFinalState(t task, state taskState) {
@ -281,7 +272,7 @@ func waitTaskFinalState(t task, state taskState) {
break
}
log.Debug("task state not match es",
log.Debug("task state not matches",
zap.Int("actual", int(currentState)),
zap.Int("expected", int(state)))
}
@ -1397,9 +1388,6 @@ func TestUpdateTaskProcessWhenLoadSegment(t *testing.T) {
assert.Nil(t, err)
assert.Equal(t, int64(0), collectionInfo.InMemoryPercentage)
watchDeltaChannel := genWatchDeltaChannelTask(ctx, queryCoord, node1.queryNodeID)
watchDeltaChannel.setParentTask(loadCollectionTask)
queryCoord.scheduler.processTask(watchDeltaChannel)
collectionInfo, err = queryCoord.meta.getCollectionInfoByID(defaultCollectionID)
assert.Nil(t, err)
assert.Equal(t, int64(0), collectionInfo.InMemoryPercentage)

7
internal/querycoord/task_util.go
View File

@ -63,13 +63,6 @@ func thinWatchDmChannelsRequest(request *querypb.WatchDmChannelsRequest) *queryp
return cloned
}
// reviseWatchDeltaChannelsRequest will revise the WatchDeltaChannelsRequest for upgrade compatibility from 2.0.2
func reviseWatchDeltaChannelsRequest(req *querypb.WatchDeltaChannelsRequest) {
for _, vChannel := range req.GetInfos() {
reviseVChannelInfo(vChannel)
}
}
// reviseVChannelInfo will revise the datapb.VchannelInfo for upgrade compatibility from 2.0.2
func reviseVChannelInfo(vChannel *datapb.VchannelInfo) {
removeDuplicateSegmentIDFn := func(ids []int64) []int64 {

29
internal/querycoord/util.go
View File

@ -21,7 +21,6 @@ import (
"sort"
"github.com/milvus-io/milvus/internal/log"
"github.com/milvus-io/milvus/internal/proto/commonpb"
"github.com/milvus-io/milvus/internal/proto/datapb"
"github.com/milvus-io/milvus/internal/proto/milvuspb"
"github.com/milvus-io/milvus/internal/proto/querypb"
@ -81,34 +80,6 @@ func getFieldSizeFromFieldBinlog(fieldBinlog *datapb.FieldBinlog) int64 {
}
func getDstNodeIDByTask(t task) int64 {
var nodeID int64
switch t.msgType() {
case commonpb.MsgType_LoadSegments:
loadSegment := t.(*loadSegmentTask)
nodeID = loadSegment.DstNodeID
case commonpb.MsgType_WatchDmChannels:
watchDmChannel := t.(*watchDmChannelTask)
nodeID = watchDmChannel.NodeID
case commonpb.MsgType_WatchDeltaChannels:
watchDeltaChannel := t.(*watchDeltaChannelTask)
nodeID = watchDeltaChannel.NodeID
case commonpb.MsgType_ReleaseCollection:
releaseCollection := t.(*releaseCollectionTask)
nodeID = releaseCollection.NodeID
case commonpb.MsgType_ReleasePartitions:
releasePartition := t.(*releasePartitionTask)
nodeID = releasePartition.NodeID
case commonpb.MsgType_ReleaseSegments:
releaseSegment := t.(*releaseSegmentTask)
nodeID = releaseSegment.NodeID
default:
//TODO::
}
return nodeID
}
// syncReplicaSegments syncs the segments distribution of replica to shard leaders
// only syncs the segments in shards if not nil
func syncReplicaSegments(ctx context.Context, meta Meta, cluster Cluster, replicaID UniqueID, shards ...string) error {

52
internal/querynode/impl.go
View File

@ -138,58 +138,6 @@ func (node *QueryNode) WatchDmChannels(ctx context.Context, in *queryPb.WatchDmC
return waitFunc()
}
// WatchDeltaChannels create consumers on dmChannels to receive Incremental data，which is the important part of real-time query
func (node *QueryNode) WatchDeltaChannels(ctx context.Context, in *queryPb.WatchDeltaChannelsRequest) (*commonpb.Status, error) {
code := node.stateCode.Load().(internalpb.StateCode)
if code != internalpb.StateCode_Healthy {
err := fmt.Errorf("query node %d is not ready", Params.QueryNodeCfg.GetNodeID())
status := &commonpb.Status{
ErrorCode: commonpb.ErrorCode_UnexpectedError,
Reason: err.Error(),
}
return status, nil
}
dct := &watchDeltaChannelsTask{
baseTask: baseTask{
ctx: ctx,
done: make(chan error),
},
req: in,
node: node,
}
err := node.scheduler.queue.Enqueue(dct)
if err != nil {
status := &commonpb.Status{
ErrorCode: commonpb.ErrorCode_UnexpectedError,
Reason: err.Error(),
}
log.Warn(err.Error())
return status, nil
}
log.Info("watchDeltaChannelsTask Enqueue done", zap.Int64("collectionID", in.CollectionID), zap.Int64("nodeID", Params.QueryNodeCfg.GetNodeID()))
waitFunc := func() (*commonpb.Status, error) {
err = dct.WaitToFinish()
if err != nil {
status := &commonpb.Status{
ErrorCode: commonpb.ErrorCode_UnexpectedError,
Reason: err.Error(),
}
log.Warn(err.Error())
return status, nil
}
log.Info("watchDeltaChannelsTask WaitToFinish done", zap.Int64("collectionID", in.CollectionID), zap.Int64("nodeID", Params.QueryNodeCfg.GetNodeID()))
return &commonpb.Status{
ErrorCode: commonpb.ErrorCode_Success,
}, nil
}
return waitFunc()
}
// LoadSegments load historical data into query node, historical data can be vector data or index
func (node *QueryNode) LoadSegments(ctx context.Context, in *queryPb.LoadSegmentsRequest) (*commonpb.Status, error) {
code := node.stateCode.Load().(internalpb.StateCode)

2
internal/querynode/query_shard_service.go
View File

@ -75,7 +75,7 @@ func (q *queryShardService) addQueryShard(collectionID UniqueID, channel Channel
q.queryShardsMu.Lock()
defer q.queryShardsMu.Unlock()
if _, ok := q.queryShards[channel]; ok {
return errors.New(fmt.Sprintln("query shard(channel) ", channel, " already exists"))
return nil
}
qs, err := newQueryShard(
q.ctx,

54
internal/querynode/segment_loader.go
View File

@ -24,7 +24,6 @@ import (
"runtime"
"runtime/debug"
"strconv"
"sync"
"github.com/panjf2000/ants/v2"
"go.uber.org/zap"
@ -84,6 +83,7 @@ func (loader *segmentLoader) LoadSegment(req *querypb.LoadSegmentsRequest, segme
segmentNum := len(req.Infos)
if segmentNum == 0 {
log.Warn("find no valid segment target, skip load segment", zap.Any("request", req))
return nil
}
@ -156,7 +156,7 @@ func (loader *segmentLoader) LoadSegment(req *querypb.LoadSegmentsRequest, segme
newSegments[segmentID] = segment
}
loadSegmentFunc := func(idx int) error {
loadFileFunc := func(idx int) error {
loadInfo := req.Infos[idx]
collectionID := loadInfo.CollectionID
partitionID := loadInfo.PartitionID
@ -164,7 +164,7 @@ func (loader *segmentLoader) LoadSegment(req *querypb.LoadSegmentsRequest, segme
segment := newSegments[segmentID]
tr := timerecord.NewTimeRecorder("loadDurationPerSegment")
err := loader.loadSegmentInternal(segment, loadInfo)
err := loader.loadFiles(segment, loadInfo)
if err != nil {
log.Error("load segment failed when load data into memory",
zap.Int64("collectionID", collectionID),
@ -186,8 +186,7 @@ func (loader *segmentLoader) LoadSegment(req *querypb.LoadSegmentsRequest, segme
zap.Int("segmentNum", segmentNum),
zap.Int("concurrencyLevel", concurrencyLevel))
err = funcutil.ProcessFuncParallel(segmentNum,
concurrencyLevel,
loadSegmentFunc, "loadSegmentFunc")
concurrencyLevel, loadFileFunc, "loadSegmentFunc")
if err != nil {
segmentGC()
return err
@ -211,7 +210,7 @@ func (loader *segmentLoader) LoadSegment(req *querypb.LoadSegmentsRequest, segme
return nil
}
func (loader *segmentLoader) loadSegmentInternal(segment *Segment,
func (loader *segmentLoader) loadFiles(segment *Segment,
loadInfo *querypb.SegmentLoadInfo) error {
collectionID := loadInfo.CollectionID
partitionID := loadInfo.PartitionID
@ -643,7 +642,7 @@ func (loader *segmentLoader) loadDeltaLogs(segment *Segment, deltaLogs []*datapb
}
func (loader *segmentLoader) FromDmlCPLoadDelete(ctx context.Context, collectionID int64, position *internalpb.MsgPosition) error {
log.Info("from dml check point load delete", zap.Any("position", position), zap.Any("msg id", position.MsgID))
log.Info("from dml check point load delete", zap.Any("position", position))
stream, err := loader.factory.NewMsgStream(ctx)
if err != nil {
return err
@ -682,7 +681,7 @@ func (loader *segmentLoader) FromDmlCPLoadDelete(ctx context.Context, collection
}
log.Info("start read delta msg from seek position to last position",
zap.Int64("Collection ID", collectionID), zap.String("channel", pChannelName))
zap.Int64("Collection ID", collectionID), zap.String("channel", pChannelName), zap.Any("seek pos", position), zap.Any("last msg", lastMsgID))
hasMore := true
for hasMore {
select {
@ -744,43 +743,18 @@ func (loader *segmentLoader) FromDmlCPLoadDelete(ctx context.Context, collection
}
offset := segment.segmentPreDelete(len(pks))
delData.deleteOffset[segmentID] = offset
timestamps := delData.deleteTimestamps[segmentID]
err = segment.segmentDelete(offset, pks, timestamps)
if err != nil {
log.Warn("QueryNode: segment delete failed", zap.Int64("segment", segmentID), zap.Error(err))
return err
}
}
wg := sync.WaitGroup{}
for segmentID := range delData.deleteOffset {
wg.Add(1)
go deletePk(loader.metaReplica, delData, segmentID, &wg)
}
wg.Wait()
log.Info("from dml check point load done", zap.Any("msg id", position.GetMsgID()))
return nil
}
func deletePk(replica ReplicaInterface, deleteData *deleteData, segmentID UniqueID, wg *sync.WaitGroup) {
defer wg.Done()
log.Debug("QueryNode::iNode::delete", zap.Any("SegmentID", segmentID))
targetSegment, err := replica.getSegmentByID(segmentID, segmentTypeSealed)
if err != nil {
log.Error(err.Error())
return
}
if targetSegment.segmentType != segmentTypeSealed {
return
}
ids := deleteData.deleteIDs[segmentID]
timestamps := deleteData.deleteTimestamps[segmentID]
offset := deleteData.deleteOffset[segmentID]
err = targetSegment.segmentDelete(offset, ids, timestamps)
if err != nil {
log.Warn("QueryNode: targetSegmentDelete failed", zap.Error(err))
return
}
log.Debug("Do delete done", zap.Int("len", len(deleteData.deleteIDs[segmentID])), zap.Int64("segmentID", segmentID), zap.Any("segmentType", targetSegment.segmentType))
}
// JoinIDPath joins ids to path format.
func JoinIDPath(ids ...UniqueID) string {
idStr := make([]string, 0, len(ids))
@ -819,6 +793,8 @@ func (loader *segmentLoader) checkSegmentSize(collectionID UniqueID, segmentLoad
loadingUsage := usedMemAfterLoad + uint64(
float64(maxSegmentSize)*float64(concurrency)*Params.QueryNodeCfg.LoadMemoryUsageFactor)
log.Debug("predict memory usage while loading (in MiB)",
zap.Int64("collectionID", collectionID),
zap.Int("concurrency", concurrency),
zap.Uint64("usage", toMB(loadingUsage)),
zap.Uint64("usageAfterLoad", toMB(usedMemAfterLoad)))

2
internal/querynode/shard_cluster.go
View File

@ -808,7 +808,7 @@ func (sc *ShardCluster) Search(ctx context.Context, req *querypb.SearchRequest,
wg.Wait()
if err != nil {
log.Error(err.Error())
log.Error("failed to do search", zap.Any("req", req), zap.Error(err))
return nil, err
}

151
internal/querynode/task.go
View File

@ -90,12 +90,6 @@ type watchDmChannelsTask struct {
node *QueryNode
}
type watchDeltaChannelsTask struct {
baseTask
req *queryPb.WatchDeltaChannelsRequest
node *QueryNode
}
type loadSegmentsTask struct {
baseTask
req *queryPb.LoadSegmentsRequest
@ -386,9 +380,7 @@ func (w *watchDmChannelsTask) Execute(ctx context.Context) (err error) {
// add tsafe watch in query shard if exists
for _, dmlChannel := range vChannels {
if !w.node.queryShardService.hasQueryShard(dmlChannel) {
w.node.queryShardService.addQueryShard(collectionID, dmlChannel, w.req.GetReplicaID())
}
w.node.queryShardService.addQueryShard(collectionID, dmlChannel, w.req.GetReplicaID())
}
// start flow graphs
@ -400,71 +392,49 @@ func (w *watchDmChannelsTask) Execute(ctx context.Context) (err error) {
return nil
}
// watchDeltaChannelsTask
func (w *watchDeltaChannelsTask) Execute(ctx context.Context) error {
collectionID := w.req.CollectionID
// get all vChannels
vDeltaChannels := make([]Channel, 0)
pDeltaChannels := make([]Channel, 0)
VPDeltaChannels := make(map[string]string) // map[vChannel]pChannel
vChannel2SeekPosition := make(map[string]*internalpb.MsgPosition)
for _, info := range w.req.Infos {
v := info.ChannelName
p := funcutil.ToPhysicalChannel(info.ChannelName)
vDeltaChannels = append(vDeltaChannels, v)
// internal helper function to subscribe delta channel
func (l *loadSegmentsTask) watchDeltaChannel(vchanName []string) error {
collectionID := l.req.CollectionID
var vDeltaChannels, pDeltaChannels []string
VPDeltaChannels := make(map[string]string)
for _, v := range vchanName {
dc, err := funcutil.ConvertChannelName(v, Params.CommonCfg.RootCoordDml, Params.CommonCfg.RootCoordDelta)
if err != nil {
log.Warn("watchDeltaChannels, failed to convert deltaChannel from dmlChannel", zap.String("DmlChannel", v), zap.Error(err))
return err
}
p := funcutil.ToPhysicalChannel(dc)
vDeltaChannels = append(vDeltaChannels, dc)
pDeltaChannels = append(pDeltaChannels, p)
VPDeltaChannels[v] = p
vChannel2SeekPosition[v] = info.SeekPosition
VPDeltaChannels[dc] = p
}
log.Info("Starting WatchDeltaChannels ...",
zap.Int64("collectionID", collectionID),
zap.Strings("vDeltaChannels", vDeltaChannels),
zap.Strings("pChannels", pDeltaChannels),
)
if len(VPDeltaChannels) != len(vDeltaChannels) {
return errors.New("get physical channels failed, illegal channel length, collectionID = " + fmt.Sprintln(collectionID))
}
log.Info("Get physical channels done",
zap.Int64("collectionID", collectionID),
zap.Any("channels", VPDeltaChannels),
)
if hasColl := w.node.metaReplica.hasCollection(collectionID); !hasColl {
return fmt.Errorf("cannot find collection with collectionID, %d", collectionID)
}
coll, err := w.node.metaReplica.getCollectionByID(collectionID)
coll, err := l.node.metaReplica.getCollectionByID(collectionID)
if err != nil {
return err
}
channel2FlowGraph, err := w.node.dataSyncService.addFlowGraphsForDeltaChannels(collectionID, vDeltaChannels)
channel2FlowGraph, err := l.node.dataSyncService.addFlowGraphsForDeltaChannels(collectionID, vDeltaChannels)
if err != nil {
log.Warn("watchDeltaChannel, add flowGraph for deltaChannel failed", zap.Int64("collectionID", collectionID), zap.Strings("vDeltaChannels", vDeltaChannels), zap.Error(err))
return err
}
consumeSubName := funcutil.GenChannelSubName(Params.CommonCfg.QueryNodeSubName, collectionID, Params.QueryNodeCfg.GetNodeID())
runningGroup, groupCtx := errgroup.WithContext(w.ctx)
// channels as consumer
for channel, fg := range channel2FlowGraph {
channel := channel
fg := fg
runningGroup.Go(func() error {
// use pChannel to consume
err := fg.consumeFlowGraphFromLatest(VPDeltaChannels[channel], consumeSubName)
if err != nil {
log.Error("msgStream as consumer failed for deltaChannels", zap.Int64("collectionID", collectionID), zap.Strings("vDeltaChannels", vDeltaChannels))
return err
}
err = w.node.loader.FromDmlCPLoadDelete(groupCtx, collectionID, vChannel2SeekPosition[channel])
if err != nil {
log.Error("watchDeltaChannelsTask from dml cp load delete failed", zap.Int64("collectionID", collectionID), zap.Strings("vDeltaChannels", vDeltaChannels))
return err
}
return nil
})
pchannel := VPDeltaChannels[channel]
// use pChannel to consume
err = fg.consumeFlowGraphFromLatest(pchannel, consumeSubName)
if err != nil {
log.Error("msgStream as consumer failed for deltaChannels", zap.Int64("collectionID", collectionID), zap.Strings("vDeltaChannels", vDeltaChannels))
break
}
}
err = runningGroup.Wait()
if err != nil {
log.Warn("watchDeltaChannel, add flowGraph for deltaChannel failed", zap.Int64("collectionID", collectionID), zap.Strings("vDeltaChannels", vDeltaChannels), zap.Error(err))
@ -475,7 +445,7 @@ func (w *watchDeltaChannelsTask) Execute(ctx context.Context) error {
for channel := range channel2FlowGraph {
gcChannels = append(gcChannels, channel)
}
w.node.dataSyncService.removeFlowGraphsByDeltaChannels(gcChannels)
l.node.dataSyncService.removeFlowGraphsByDeltaChannels(gcChannels)
return err
}
@ -487,18 +457,20 @@ func (w *watchDeltaChannelsTask) Execute(ctx context.Context) error {
// create tSafe
for _, channel := range vDeltaChannels {
w.node.tSafeReplica.addTSafe(channel)
l.node.tSafeReplica.addTSafe(channel)
}
// add tsafe watch in query shard if exists
// add tsafe watch in query shard if exists, we find no way to handle it if query shard not exist
for _, channel := range vDeltaChannels {
dmlChannel, err := funcutil.ConvertChannelName(channel, Params.CommonCfg.RootCoordDelta, Params.CommonCfg.RootCoordDml)
if err != nil {
log.Warn("failed to convert delta channel to dml", zap.String("channel", channel), zap.Error(err))
continue
log.Error("failed to convert delta channel to dml", zap.String("channel", channel), zap.Error(err))
panic(err)
}
if !w.node.queryShardService.hasQueryShard(dmlChannel) {
w.node.queryShardService.addQueryShard(collectionID, dmlChannel, w.req.GetReplicaId())
err = l.node.queryShardService.addQueryShard(collectionID, dmlChannel, l.req.GetReplicaID())
if err != nil {
log.Error("failed to add shard Service to query shard", zap.String("channel", channel), zap.Error(err))
panic(err)
}
}
@ -512,7 +484,6 @@ func (w *watchDeltaChannelsTask) Execute(ctx context.Context) error {
}
// loadSegmentsTask
func (l *loadSegmentsTask) PreExecute(ctx context.Context) error {
log.Info("LoadSegmentTask PreExecute start", zap.Int64("msgID", l.req.Base.MsgID))
var err error
@ -536,7 +507,7 @@ func (l *loadSegmentsTask) PreExecute(ctx context.Context) error {
if !has {
filteredInfos = append(filteredInfos, info)
} else {
log.Debug("ignore segment that is already loaded", zap.Int64("segmentID", info.SegmentID))
log.Debug("ignore segment that is already loaded", zap.Int64("collectionID", info.SegmentID), zap.Int64("segmentID", info.SegmentID))
}
}
l.req.Infos = filteredInfos
@ -545,38 +516,52 @@ func (l *loadSegmentsTask) PreExecute(ctx context.Context) error {
}
func (l *loadSegmentsTask) Execute(ctx context.Context) error {
// TODO: support db
log.Info("LoadSegmentTask Execute start", zap.Int64("msgID", l.req.Base.MsgID))
err := l.node.loader.LoadSegment(l.req, segmentTypeSealed)
if err != nil {
log.Warn(err.Error())
log.Warn("failed to load segment", zap.Int64("collectionID", l.req.CollectionID),
zap.Int64("replicaID", l.req.ReplicaID), zap.Error(err))
return err
}
vchanName := make([]string, 0)
for _, deltaPosition := range l.req.DeltaPositions {
vchanName = append(vchanName, deltaPosition.ChannelName)
}
// TODO delta channel need to released 1. if other watchDeltaChannel fail 2. when segment release
err = l.watchDeltaChannel(vchanName)
if err != nil {
// roll back
for _, segment := range l.req.Infos {
l.node.metaReplica.removeSegment(segment.SegmentID, segmentTypeSealed)
}
log.Warn("failed to watch Delta channel while load segment", zap.Int64("collectionID", l.req.CollectionID),
zap.Int64("replicaID", l.req.ReplicaID), zap.Error(err))
return err
}
// reload delete log from cp to latest position
runningGroup, groupCtx := errgroup.WithContext(l.ctx)
for _, deltaPosition := range l.req.DeltaPositions {
err = l.node.loader.FromDmlCPLoadDelete(ctx, l.req.CollectionID, deltaPosition)
if err != nil {
for _, segment := range l.req.Infos {
l.node.metaReplica.removeSegment(segment.SegmentID, segmentTypeSealed)
}
log.Warn("LoadSegmentTask from delta check point load delete failed", zap.Int64("msgID", l.req.Base.MsgID), zap.Error(err))
return err
pos := deltaPosition
runningGroup.Go(func() error {
// reload data from dml channel
return l.node.loader.FromDmlCPLoadDelete(groupCtx, l.req.CollectionID, pos)
})
}
err = runningGroup.Wait()
if err != nil {
for _, segment := range l.req.Infos {
l.node.metaReplica.removeSegment(segment.SegmentID, segmentTypeSealed)
}
log.Warn("failed to load delete data while load segment", zap.Int64("collectionID", l.req.CollectionID),
zap.Int64("replicaID", l.req.ReplicaID), zap.Error(err))
return err
}
log.Info("LoadSegmentTask Execute done", zap.Int64("msgID", l.req.Base.MsgID))
log.Info("LoadSegmentTask Execute done", zap.Int64("collectionID", l.req.CollectionID),
zap.Int64("replicaID", l.req.ReplicaID), zap.Int64("msgID", l.req.Base.MsgID))
return nil
}
type ReplicaType int
const (
replicaNone ReplicaType = iota
replicaStreaming
replicaHistorical
)
func (r *releaseCollectionTask) Execute(ctx context.Context) error {
log.Info("Execute release collection task", zap.Any("collectionID", r.req.CollectionID))

200
internal/querynode/task_test.go
View File

@ -21,10 +21,8 @@ import (
"testing"
"time"
"github.com/apache/pulsar-client-go/pulsar"
"github.com/stretchr/testify/assert"
"github.com/milvus-io/milvus/internal/mq/msgstream"
"github.com/milvus-io/milvus/internal/mq/msgstream/mqwrapper/rmq"
"github.com/milvus-io/milvus/internal/proto/commonpb"
"github.com/milvus-io/milvus/internal/proto/datapb"
"github.com/milvus-io/milvus/internal/proto/internalpb"
@ -32,6 +30,7 @@ import (
"github.com/milvus-io/milvus/internal/proto/schemapb"
"github.com/milvus-io/milvus/internal/util/funcutil"
"github.com/milvus-io/milvus/internal/util/typeutil"
"github.com/stretchr/testify/assert"
)
func TestTask_watchDmChannelsTask(t *testing.T) {
@ -268,93 +267,6 @@ func TestTask_watchDmChannelsTask(t *testing.T) {
})
}
func TestTask_watchDeltaChannelsTask(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
genWatchDeltaChannelsRequest := func() *querypb.WatchDeltaChannelsRequest {
req := &querypb.WatchDeltaChannelsRequest{
Base: genCommonMsgBase(commonpb.MsgType_WatchDeltaChannels),
CollectionID: defaultCollectionID,
}
return req
}
t.Run("test timestamp", func(t *testing.T) {
timestamp := Timestamp(1000)
task := watchDeltaChannelsTask{
baseTask: baseTask{
ts: timestamp,
},
req: genWatchDeltaChannelsRequest(),
}
resT := task.Timestamp()
assert.Equal(t, timestamp, resT)
})
t.Run("test OnEnqueue", func(t *testing.T) {
task := watchDeltaChannelsTask{
req: genWatchDeltaChannelsRequest(),
}
err := task.OnEnqueue()
assert.NoError(t, err)
task.req.Base = nil
err = task.OnEnqueue()
assert.NoError(t, err)
})
t.Run("test execute", func(t *testing.T) {
node, err := genSimpleQueryNode(ctx)
assert.NoError(t, err)
task := watchDeltaChannelsTask{
req: genWatchDeltaChannelsRequest(),
node: node,
}
task.ctx = ctx
task.req.Infos = []*datapb.VchannelInfo{
{
CollectionID: defaultCollectionID,
ChannelName: defaultDeltaChannel,
SeekPosition: &internalpb.MsgPosition{
ChannelName: defaultDMLChannel,
MsgID: pulsar.EarliestMessageID().Serialize(),
MsgGroup: defaultSubName,
Timestamp: 0,
},
},
}
err = task.Execute(ctx)
assert.NoError(t, err)
})
t.Run("test execute without init collection", func(t *testing.T) {
node, err := genSimpleQueryNode(ctx)
assert.NoError(t, err)
task := watchDeltaChannelsTask{
req: genWatchDeltaChannelsRequest(),
node: node,
}
task.ctx = ctx
task.req.Infos = []*datapb.VchannelInfo{
{
CollectionID: defaultCollectionID,
ChannelName: defaultDeltaChannel,
SeekPosition: &internalpb.MsgPosition{
ChannelName: defaultDeltaChannel,
MsgID: []byte{1, 2, 3, 4, 5, 6, 7, 8},
MsgGroup: defaultSubName,
Timestamp: 0,
},
},
}
task.req.CollectionID++
err = task.Execute(ctx)
assert.Error(t, err)
})
}
func TestTask_loadSegmentsTask(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
@ -373,7 +285,8 @@ func TestTask_loadSegmentsTask(t *testing.T) {
timestamp := Timestamp(1000)
task := loadSegmentsTask{
baseTask: baseTask{
ts: timestamp,
ts: timestamp,
ctx: ctx,
},
req: genLoadEmptySegmentsRequest(),
}
@ -383,6 +296,9 @@ func TestTask_loadSegmentsTask(t *testing.T) {
t.Run("test OnEnqueue", func(t *testing.T) {
task := loadSegmentsTask{
baseTask: baseTask{
ctx: ctx,
},
req: genLoadEmptySegmentsRequest(),
}
err := task.OnEnqueue()
@ -416,6 +332,9 @@ func TestTask_loadSegmentsTask(t *testing.T) {
}
task := loadSegmentsTask{
baseTask: baseTask{
ctx: ctx,
},
req: req,
node: node,
}
@ -427,6 +346,8 @@ func TestTask_loadSegmentsTask(t *testing.T) {
node, err := genSimpleQueryNode(ctx)
assert.NoError(t, err)
node.metaReplica.removeSegment(defaultSegmentID, segmentTypeSealed)
fieldBinlog, statsLog, err := saveBinLog(ctx, defaultCollectionID, defaultPartitionID, defaultSegmentID, defaultMsgLength, schema)
assert.NoError(t, err)
@ -445,6 +366,9 @@ func TestTask_loadSegmentsTask(t *testing.T) {
}
task := loadSegmentsTask{
baseTask: baseTask{
ctx: ctx,
},
req: req,
node: node,
}
@ -555,6 +479,9 @@ func TestTask_loadSegmentsTask(t *testing.T) {
}
task := loadSegmentsTask{
baseTask: baseTask{
ctx: ctx,
},
req: req,
node: node,
}
@ -583,6 +510,9 @@ func TestTask_loadSegmentsTask(t *testing.T) {
assert.NoError(t, err)
task := loadSegmentsTask{
baseTask: baseTask{
ctx: ctx,
},
req: genLoadEmptySegmentsRequest(),
node: node,
}
@ -605,6 +535,94 @@ func TestTask_loadSegmentsTask(t *testing.T) {
})
}
func TestTask_loadSegmentsTaskLoadDelta(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
schema := genTestCollectionSchema()
t.Run("test repeated load delta channel", func(t *testing.T) {
node, err := genSimpleQueryNode(ctx)
assert.NoError(t, err)
vDmChannel := "by-dev-rootcoord-dml-test_2_2021v2"
segmentLoadInfo := &querypb.SegmentLoadInfo{
SegmentID: UniqueID(1000),
PartitionID: defaultPartitionID,
CollectionID: defaultCollectionID,
}
loadReq := &querypb.LoadSegmentsRequest{
Base: genCommonMsgBase(commonpb.MsgType_LoadSegments),
Schema: schema,
Infos: []*querypb.SegmentLoadInfo{segmentLoadInfo},
DeltaPositions: []*internalpb.MsgPosition{
{
ChannelName: vDmChannel,
MsgID: rmq.SerializeRmqID(0),
Timestamp: 100,
},
},
}
task := loadSegmentsTask{
baseTask: baseTask{
ctx: ctx,
},
req: loadReq,
node: node,
}
// execute loadSegmentsTask twice
err = task.PreExecute(ctx)
assert.NoError(t, err)
err = task.Execute(ctx)
assert.NoError(t, err)
// expected only one segment in replica
num := node.metaReplica.getSegmentNum(segmentTypeSealed)
assert.Equal(t, 2, num)
// load second segments with same channel
loadReq = &querypb.LoadSegmentsRequest{
Base: genCommonMsgBase(commonpb.MsgType_LoadSegments),
Schema: schema,
Infos: []*querypb.SegmentLoadInfo{
{
SegmentID: UniqueID(1001),
PartitionID: defaultPartitionID,
CollectionID: defaultCollectionID,
},
},
DeltaPositions: []*internalpb.MsgPosition{
{
ChannelName: vDmChannel,
MsgID: rmq.SerializeRmqID(0),
Timestamp: 100,
},
},
}
task = loadSegmentsTask{
baseTask: baseTask{
ctx: ctx,
},
req: loadReq,
node: node,
}
// execute loadSegmentsTask twice
err = task.PreExecute(ctx)
assert.NoError(t, err)
err = task.Execute(ctx)
assert.NoError(t, err)
num = node.metaReplica.getSegmentNum(segmentTypeSealed)
assert.Equal(t, 3, num)
ok := node.queryShardService.hasQueryShard(vDmChannel)
assert.True(t, ok)
assert.Equal(t, len(node.dataSyncService.dmlChannel2FlowGraph), 0)
assert.Equal(t, len(node.dataSyncService.deltaChannel2FlowGraph), 1)
})
}
func TestTask_releaseCollectionTask(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()

1
internal/types/types.go
View File

@ -1204,7 +1204,6 @@ type QueryNode interface {
TimeTickProvider
WatchDmChannels(ctx context.Context, req *querypb.WatchDmChannelsRequest) (*commonpb.Status, error)
WatchDeltaChannels(ctx context.Context, req *querypb.WatchDeltaChannelsRequest) (*commonpb.Status, error)
// LoadSegments notifies QueryNode to load the sealed segments from storage. The load tasks are sync to this
// rpc, QueryNode will return after all the sealed segments are loaded.
//

6
internal/util/funcutil/parallel.go
View File

@ -48,12 +48,10 @@ func ProcessFuncParallel(total, maxParallel int, f ProcessFunc, fname string) er
t := time.Now()
defer func() {
log.Debug(fname, zap.Any("time cost", time.Since(t)))
log.Debug(fname, zap.Any("total", total), zap.Any("time cost", time.Since(t)))
}()
nPerBatch := (total + maxParallel - 1) / maxParallel
log.Debug(fname, zap.Any("total", total))
log.Debug(fname, zap.Any("nPerBatch", nPerBatch))
quit := make(chan bool)
errc := make(chan error)
@ -105,8 +103,6 @@ func ProcessFuncParallel(total, maxParallel int, f ProcessFunc, fname string) er
routineNum++
}
log.Debug(fname, zap.Any("NumOfGoRoutines", routineNum))
if routineNum <= 0 {
return nil
}

4
internal/util/mock/grpc_querynode_client.go
View File

@ -48,10 +48,6 @@ func (m *GrpcQueryNodeClient) WatchDmChannels(ctx context.Context, in *querypb.W
return &commonpb.Status{}, m.Err
}
func (m *GrpcQueryNodeClient) WatchDeltaChannels(ctx context.Context, in *querypb.WatchDeltaChannelsRequest, opts ...grpc.CallOption) (*commonpb.Status, error) {
return &commonpb.Status{}, m.Err
}
func (m *GrpcQueryNodeClient) LoadSegments(ctx context.Context, in *querypb.LoadSegmentsRequest, opts ...grpc.CallOption) (*commonpb.Status, error) {
return &commonpb.Status{}, m.Err
}

4
internal/util/mock/querynode_client.go
View File

@ -65,10 +65,6 @@ func (q QueryNodeClient) WatchDmChannels(ctx context.Context, req *querypb.Watch
return q.grpcClient.WatchDmChannels(ctx, req)
}
func (q QueryNodeClient) WatchDeltaChannels(ctx context.Context, req *querypb.WatchDeltaChannelsRequest) (*commonpb.Status, error) {
return q.grpcClient.WatchDeltaChannels(ctx, req)
}
func (q QueryNodeClient) LoadSegments(ctx context.Context, req *querypb.LoadSegmentsRequest) (*commonpb.Status, error) {
return q.grpcClient.LoadSegments(ctx, req)
}