milvus-io
/
milvus
mirror of https://github.com/milvus-io/milvus.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Add globalSegmentManager and consume changeInfoMsg (#9575) 

Signed-off-by: bigsheeper <yihao.dai@zilliz.com>
pull/9625/head v2.0.0-rc7
bigsheeper 2021-10-11 15:40:38 +08:00 committed by GitHub
parent 29af477c5e
commit d567b21fd4
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 447 additions and 138 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
internal/msgstream/msg_test.go
View File

@ -831,6 +831,12 @@ func TestDataNodeTtMsg_Unmarshal_IllegalParameter(t *testing.T) {
}
func TestSealedSegmentsChangeInfoMsg(t *testing.T) {
genSimpleSegmentInfo := func(segmentID UniqueID) *querypb.SegmentInfo {
return &querypb.SegmentInfo{
SegmentID: segmentID,
}
}
changeInfoMsg := &SealedSegmentsChangeInfoMsg{
BaseMsg: generateBaseMsg(),
SealedSegmentsChangeInfo: querypb.SealedSegmentsChangeInfo{
@ -840,10 +846,18 @@ func TestSealedSegmentsChangeInfoMsg(t *testing.T) {
Timestamp: 2,
SourceID: 3,
},
OnlineNodeID: int64(1),
OnlineSegmentIDs: []int64{1, 2, 3},
OfflineNodeID: int64(2),
OfflineSegmentIDs: []int64{4, 5, 6},
OnlineNodeID: int64(1),
OnlineSegments: []*querypb.SegmentInfo{
genSimpleSegmentInfo(1),
genSimpleSegmentInfo(2),
genSimpleSegmentInfo(3),
},
OfflineNodeID: int64(2),
OfflineSegments: []*querypb.SegmentInfo{
genSimpleSegmentInfo(4),
genSimpleSegmentInfo(5),
genSimpleSegmentInfo(6),
},
},
}

4
internal/proto/query_coord.proto
View File

@ -308,7 +308,7 @@ message LoadBalanceRequest {
message SealedSegmentsChangeInfo {
common.MsgBase base = 1;
int64 online_nodeID = 2;
repeated int64 online_segmentIDs = 3;
repeated SegmentInfo online_segments = 3;
int64 offline_nodeID = 4;
repeated int64 offline_segmentIDs = 5;
repeated SegmentInfo offline_segments = 5;
}

258
internal/proto/querypb/query_coord.pb.go
View File

@ -1997,9 +1997,9 @@ func (m *LoadBalanceRequest) GetBalanceReason() TriggerCondition {
type SealedSegmentsChangeInfo struct {
Base *commonpb.MsgBase `protobuf:"bytes,1,opt,name=base,proto3" json:"base,omitempty"`
OnlineNodeID int64 `protobuf:"varint,2,opt,name=online_nodeID,json=onlineNodeID,proto3" json:"online_nodeID,omitempty"`
OnlineSegmentIDs []int64 `protobuf:"varint,3,rep,packed,name=online_segmentIDs,json=onlineSegmentIDs,proto3" json:"online_segmentIDs,omitempty"`
OnlineSegments []*SegmentInfo `protobuf:"bytes,3,rep,name=online_segments,json=onlineSegments,proto3" json:"online_segments,omitempty"`
OfflineNodeID int64 `protobuf:"varint,4,opt,name=offline_nodeID,json=offlineNodeID,proto3" json:"offline_nodeID,omitempty"`
OfflineSegmentIDs []int64 `protobuf:"varint,5,rep,packed,name=offline_segmentIDs,json=offlineSegmentIDs,proto3" json:"offline_segmentIDs,omitempty"`
OfflineSegments []*SegmentInfo `protobuf:"bytes,5,rep,name=offline_segments,json=offlineSegments,proto3" json:"offline_segments,omitempty"`
XXX_NoUnkeyedLiteral struct{} `json:"-"`
XXX_unrecognized []byte `json:"-"`
XXX_sizecache int32 `json:"-"`
@ -2044,9 +2044,9 @@ func (m *SealedSegmentsChangeInfo) GetOnlineNodeID() int64 {
return 0
}
func (m *SealedSegmentsChangeInfo) GetOnlineSegmentIDs() []int64 {
func (m *SealedSegmentsChangeInfo) GetOnlineSegments() []*SegmentInfo {
if m != nil {
return m.OnlineSegmentIDs
return m.OnlineSegments
}
return nil
}
@ -2058,9 +2058,9 @@ func (m *SealedSegmentsChangeInfo) GetOfflineNodeID() int64 {
return 0
}
func (m *SealedSegmentsChangeInfo) GetOfflineSegmentIDs() []int64 {
func (m *SealedSegmentsChangeInfo) GetOfflineSegments() []*SegmentInfo {
if m != nil {
return m.OfflineSegmentIDs
return m.OfflineSegments
}
return nil
}
@ -2104,136 +2104,136 @@ func init() {
func init() { proto.RegisterFile("query_coord.proto", fileDescriptor_aab7cc9a69ed26e8) }
var fileDescriptor_aab7cc9a69ed26e8 = []byte{
// 2052 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xec, 0x59, 0xcd, 0x73, 0x1b, 0x49,
0x15, 0xf7, 0x48, 0xb2, 0x3e, 0x9e, 0xbe, 0xc6, 0x9d, 0xc4, 0x28, 0x22, 0xc9, 0x9a, 0xc9, 0x66,
0x93, 0x75, 0x88, 0xbd, 0xeb, 0x2c, 0x55, 0xe4, 0xc0, 0x61, 0x63, 0x6d, 0x8c, 0x60, 0xe3, 0x98,
0xb1, 0x59, 0x8a, 0x54, 0xaa, 0x86, 0x91, 0xa6, 0x25, 0x4f, 0xed, 0xcc, 0xb4, 0x32, 0x3d, 0x8a,
0xe3, 0x1c, 0x38, 0x71, 0xe3, 0xcc, 0x09, 0x8a, 0x2a, 0xaa, 0xf8, 0x28, 0x0e, 0xfc, 0x03, 0x9c,
0xf6, 0xc2, 0x9d, 0x7f, 0x00, 0xaa, 0x28, 0xb8, 0x73, 0xe1, 0x0f, 0xa0, 0xfa, 0x63, 0x46, 0xf3,
0x25, 0x5b, 0xb1, 0x31, 0x49, 0x6d, 0xed, 0x4d, 0xfd, 0xfa, 0xf5, 0x7b, 0xaf, 0xdf, 0xc7, 0xaf,
0xdf, 0x3c, 0xc1, 0xca, 0xf3, 0x29, 0xf6, 0x8f, 0x8d, 0x21, 0x21, 0xbe, 0xb5, 0x31, 0xf1, 0x49,
0x40, 0x10, 0x72, 0x6d, 0xe7, 0xc5, 0x94, 0x8a, 0xd5, 0x06, 0xdf, 0xef, 0x36, 0x86, 0xc4, 0x75,
0x89, 0x27, 0x68, 0xdd, 0x46, 0x9c, 0xa3, 0xdb, 0xb2, 0xbd, 0x00, 0xfb, 0x9e, 0xe9, 0x84, 0xbb,
0x74, 0x78, 0x88, 0x5d, 0x53, 0xae, 0x54, 0xcb, 0x0c, 0xcc, 0xb8, 0x7c, 0xed, 0x67, 0x0a, 0xac,
0xee, 0x1f, 0x92, 0xa3, 0x6d, 0xe2, 0x38, 0x78, 0x18, 0xd8, 0xc4, 0xa3, 0x3a, 0x7e, 0x3e, 0xc5,
// 2056 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xec, 0x59, 0x4b, 0x6f, 0x1b, 0xc9,
0x11, 0xd6, 0x90, 0x14, 0x1f, 0xc5, 0xd7, 0xb8, 0x6d, 0x29, 0x34, 0x63, 0x7b, 0x95, 0xf1, 0x7a,
0xed, 0xd5, 0x66, 0xa5, 0x5d, 0x79, 0x03, 0xc4, 0x87, 0x1c, 0xd6, 0xe2, 0x5a, 0xcb, 0xcd, 0x5a,
0x56, 0x46, 0xca, 0x06, 0x31, 0x0c, 0x4c, 0x86, 0x9c, 0x26, 0x35, 0xd8, 0x99, 0x69, 0x7a, 0x7a,
0x68, 0x59, 0x3e, 0xe4, 0x94, 0x5b, 0xce, 0x39, 0x25, 0x08, 0x10, 0x20, 0x0f, 0xe4, 0x90, 0xfc,
0x80, 0x9c, 0xf6, 0x92, 0x7b, 0xfe, 0x40, 0x02, 0x04, 0xc9, 0x3d, 0x7f, 0x21, 0xe8, 0xc7, 0x0c,
0xe7, 0x45, 0x89, 0x96, 0xa2, 0xd8, 0x30, 0xf6, 0xc6, 0xa9, 0xae, 0xae, 0xaa, 0xae, 0xc7, 0xd7,
0xd5, 0x45, 0xb8, 0xf4, 0x74, 0x8a, 0xfd, 0x63, 0x63, 0x48, 0x88, 0x6f, 0x6d, 0x4c, 0x7c, 0x12,
0x10, 0x84, 0x5c, 0xdb, 0x79, 0x36, 0xa5, 0xe2, 0x6b, 0x83, 0xaf, 0x77, 0x1b, 0x43, 0xe2, 0xba,
0xc4, 0x13, 0xb4, 0x6e, 0x23, 0xce, 0xd1, 0x6d, 0xd9, 0x5e, 0x80, 0x7d, 0xcf, 0x74, 0xc2, 0x55,
0x3a, 0x3c, 0xc4, 0xae, 0x29, 0xbf, 0x54, 0xcb, 0x0c, 0xcc, 0xb8, 0x7c, 0xed, 0x67, 0x0a, 0xac,
0xee, 0x1f, 0x92, 0xa3, 0x6d, 0xe2, 0x38, 0x78, 0x18, 0xd8, 0xc4, 0xa3, 0x3a, 0x7e, 0x3a, 0xc5,
0x34, 0x40, 0x1f, 0x40, 0x69, 0x60, 0x52, 0xdc, 0x51, 0xd6, 0x94, 0x3b, 0xf5, 0xad, 0x6b, 0x1b,
0x09, 0x4b, 0xa4, 0x09, 0x8f, 0xe9, 0xf8, 0xa1, 0x49, 0xb1, 0xce, 0x39, 0x11, 0x82, 0x92, 0x35,
0xe8, 0xf7, 0x3a, 0x85, 0x35, 0xe5, 0x4e, 0x51, 0xe7, 0xbf, 0xd1, 0xbb, 0xd0, 0x1c, 0x46, 0xb2,
0xfb, 0x3d, 0xda, 0x29, 0xae, 0x15, 0xef, 0x14, 0xf5, 0x24, 0x51, 0xfb, 0x83, 0x02, 0x5f, 0xcb,
0x98, 0x41, 0x27, 0xc4, 0xa3, 0x18, 0xdd, 0x87, 0x32, 0x0d, 0xcc, 0x60, 0x4a, 0xa5, 0x25, 0x5f,
0xcf, 0xb5, 0x64, 0x9f, 0xb3, 0xe8, 0x92, 0x35, 0xab, 0xb6, 0x90, 0xa3, 0x16, 0x7d, 0x08, 0x97,
0x6d, 0xef, 0x31, 0x76, 0x89, 0x7f, 0x6c, 0x4c, 0xb0, 0x3f, 0xc4, 0x5e, 0x60, 0x8e, 0x71, 0x68,
0xe3, 0xa5, 0x70, 0x6f, 0x6f, 0xb6, 0xa5, 0xfd, 0x4e, 0x81, 0x2b, 0xcc, 0xd2, 0x3d, 0xd3, 0x0f,
0xec, 0x0b, 0xf0, 0x97, 0x06, 0x8d, 0xb8, 0x8d, 0x9d, 0x22, 0xdf, 0x4b, 0xd0, 0x18, 0xcf, 0x24,
0x54, 0xcf, 0xee, 0x56, 0xe2, 0xe6, 0x26, 0x68, 0xda, 0x6f, 0x65, 0x60, 0xe3, 0x76, 0x9e, 0xc7,
0xa1, 0x69, 0x9d, 0x85, 0xac, 0xce, 0xb3, 0xb8, 0xf3, 0x0b, 0x05, 0xae, 0x7c, 0x4a, 0x4c, 0x6b,
0x16, 0xf8, 0xff, 0xbf, 0x3b, 0xbf, 0x03, 0x65, 0x51, 0x25, 0x9d, 0x12, 0xd7, 0x75, 0x2b, 0xa9,
0x4b, 0x56, 0xd0, 0xcc, 0xc2, 0x7d, 0x4e, 0xd0, 0xe5, 0x21, 0xed, 0x57, 0x0a, 0x74, 0x74, 0xec,
0x60, 0x93, 0xe2, 0x37, 0x79, 0x8b, 0x55, 0x28, 0x7b, 0xc4, 0xc2, 0xfd, 0x1e, 0xbf, 0x45, 0x51,
0x97, 0x2b, 0xed, 0x5f, 0xd2, 0xc3, 0x6f, 0x79, 0xc2, 0xc6, 0xa2, 0xb0, 0x7c, 0x96, 0x28, 0x7c,
0x31, 0x8b, 0xc2, 0xdb, 0x7e, 0xd3, 0x59, 0xa4, 0x96, 0x13, 0x91, 0xfa, 0x31, 0x5c, 0xdd, 0xf6,
0xb1, 0x19, 0xe0, 0x1f, 0x30, 0x98, 0xdf, 0x3e, 0x34, 0x3d, 0x0f, 0x3b, 0xe1, 0x15, 0xd2, 0xca,
0x95, 0x1c, 0xe5, 0x1d, 0xa8, 0x4c, 0x7c, 0xf2, 0xf2, 0x38, 0xb2, 0x3b, 0x5c, 0x6a, 0xbf, 0x51,
0x09, 0x4b, 0xa4, 0x09, 0x0f, 0xe9, 0xf8, 0xbe, 0x49, 0xb1, 0xce, 0x39, 0x11, 0x82, 0x92, 0x35,
0xe8, 0xf7, 0x3a, 0x85, 0x35, 0xe5, 0x4e, 0x51, 0xe7, 0xbf, 0xd1, 0xdb, 0xd0, 0x1c, 0x46, 0xb2,
0xfb, 0x3d, 0xda, 0x29, 0xae, 0x15, 0xef, 0x14, 0xf5, 0x24, 0x51, 0xfb, 0x83, 0x02, 0xdf, 0xc8,
0x98, 0x41, 0x27, 0xc4, 0xa3, 0x18, 0xdd, 0x85, 0x32, 0x0d, 0xcc, 0x60, 0x4a, 0xa5, 0x25, 0xdf,
0xcc, 0xb5, 0x64, 0x9f, 0xb3, 0xe8, 0x92, 0x35, 0xab, 0xb6, 0x90, 0xa3, 0x16, 0x7d, 0x08, 0x57,
0x6c, 0xef, 0x21, 0x76, 0x89, 0x7f, 0x6c, 0x4c, 0xb0, 0x3f, 0xc4, 0x5e, 0x60, 0x8e, 0x71, 0x68,
0xe3, 0xe5, 0x70, 0x6d, 0x6f, 0xb6, 0xa4, 0xfd, 0x4e, 0x81, 0x15, 0x66, 0xe9, 0x9e, 0xe9, 0x07,
0xf6, 0x05, 0xf8, 0x4b, 0x83, 0x46, 0xdc, 0xc6, 0x4e, 0x91, 0xaf, 0x25, 0x68, 0x8c, 0x67, 0x12,
0xaa, 0x67, 0x67, 0x2b, 0x71, 0x73, 0x13, 0x34, 0xed, 0xb7, 0x32, 0xb0, 0x71, 0x3b, 0xcf, 0xe3,
0xd0, 0xb4, 0xce, 0x42, 0x56, 0xe7, 0x59, 0xdc, 0xf9, 0x95, 0x02, 0x2b, 0x9f, 0x13, 0xd3, 0x9a,
0x05, 0xfe, 0xff, 0xef, 0xce, 0xef, 0x41, 0x59, 0x54, 0x49, 0xa7, 0xc4, 0x75, 0xdd, 0x4a, 0xea,
0x92, 0x15, 0x34, 0xb3, 0x70, 0x9f, 0x13, 0x74, 0xb9, 0x49, 0xfb, 0x95, 0x02, 0x1d, 0x1d, 0x3b,
0xd8, 0xa4, 0xf8, 0x55, 0x9e, 0x62, 0x15, 0xca, 0x1e, 0xb1, 0x70, 0xbf, 0xc7, 0x4f, 0x51, 0xd4,
0xe5, 0x97, 0xf6, 0x6f, 0xe9, 0xe1, 0xd7, 0x3c, 0x61, 0x63, 0x51, 0x58, 0x3e, 0x4b, 0x14, 0xbe,
0x9a, 0x45, 0xe1, 0x75, 0x3f, 0xe9, 0x2c, 0x52, 0xcb, 0x89, 0x48, 0xfd, 0x18, 0xae, 0x6e, 0xfb,
0xd8, 0x0c, 0xf0, 0x0f, 0x18, 0xcc, 0x6f, 0x1f, 0x9a, 0x9e, 0x87, 0x9d, 0xf0, 0x08, 0x69, 0xe5,
0x4a, 0x8e, 0xf2, 0x0e, 0x54, 0x26, 0x3e, 0x79, 0x7e, 0x1c, 0xd9, 0x1d, 0x7e, 0x6a, 0xbf, 0x51,
0xa0, 0x9b, 0x27, 0xfb, 0x3c, 0x88, 0x70, 0x1b, 0xda, 0xbe, 0x30, 0xce, 0x18, 0x0a, 0x79, 0x5c,
0x6b, 0x4d, 0x6f, 0x49, 0xb2, 0xd4, 0x82, 0x6e, 0x41, 0xcb, 0xc7, 0x74, 0xea, 0xcc, 0xf8, 0x8a,
0x9c, 0xaf, 0x29, 0xa8, 0x92, 0x4d, 0xfb, 0xa3, 0x02, 0x57, 0x77, 0x70, 0x10, 0x45, 0x8f, 0xa9,
0xc3, 0x6f, 0x29, 0xba, 0xfe, 0x5a, 0x81, 0x76, 0xca, 0x50, 0xb4, 0x06, 0xf5, 0x18, 0x8f, 0x0c,
0x50, 0x9c, 0x84, 0xbe, 0x0d, 0xcb, 0xcc, 0x77, 0x98, 0x9b, 0xd4, 0xda, 0xd2, 0x36, 0xb2, 0x8f,
0xfb, 0x46, 0x52, 0xaa, 0x2e, 0x0e, 0xa0, 0x4d, 0xb8, 0x94, 0x83, 0xac, 0xd2, 0x7c, 0x94, 0x05,
0x56, 0xed, 0x4f, 0x0a, 0x74, 0xf3, 0x9c, 0x79, 0x9e, 0x80, 0x3f, 0x85, 0xd5, 0xe8, 0x36, 0x86,
0x85, 0xe9, 0xd0, 0xb7, 0x27, 0xbc, 0xcc, 0xf8, 0x63, 0x50, 0xdf, 0xba, 0x79, 0xfa, 0x7d, 0xa8,
0x7e, 0x25, 0x12, 0xd1, 0x8b, 0x49, 0xd0, 0x6c, 0xb8, 0xb2, 0x83, 0x83, 0x7d, 0x3c, 0x76, 0xb1,
0x17, 0xf4, 0xbd, 0x11, 0x39, 0x7b, 0xdc, 0x6f, 0x00, 0x50, 0x29, 0x27, 0x7a, 0xa7, 0x62, 0x14,
0xed, 0x6f, 0x05, 0xa8, 0xc7, 0x14, 0xa1, 0x6b, 0x50, 0x8b, 0x76, 0x65, 0xd4, 0x66, 0x84, 0x4c,
0xc6, 0x14, 0x72, 0x32, 0x26, 0x15, 0xf9, 0x62, 0x36, 0xf2, 0x73, 0xc0, 0x19, 0x5d, 0x85, 0xaa,
0x8b, 0x5d, 0x83, 0xda, 0xaf, 0xb0, 0x04, 0x83, 0x8a, 0x8b, 0xdd, 0x7d, 0xfb, 0x15, 0x66, 0x5b,
0xde, 0xd4, 0x35, 0x7c, 0x72, 0x44, 0x3b, 0x65, 0xb1, 0xe5, 0x4d, 0x5d, 0x9d, 0x1c, 0x51, 0x74,
0x1d, 0xc0, 0xf6, 0x2c, 0xfc, 0xd2, 0xf0, 0x4c, 0x17, 0x77, 0x2a, 0xbc, 0x98, 0x6a, 0x9c, 0xb2,
0x6b, 0xba, 0x98, 0xc1, 0x00, 0x5f, 0xf4, 0x7b, 0x9d, 0xaa, 0x38, 0x28, 0x97, 0xec, 0xaa, 0xb2,
0x04, 0xfb, 0xbd, 0x4e, 0x4d, 0x9c, 0x8b, 0x08, 0xe8, 0x13, 0x68, 0xca, 0x7b, 0x1b, 0x22, 0x4d,
0x81, 0xa7, 0xe9, 0x5a, 0x5e, 0x58, 0xa5, 0x03, 0x45, 0x92, 0x36, 0x68, 0x6c, 0xc5, 0x5b, 0xca,
0x74, 0x2c, 0xcf, 0x93, 0x76, 0xdf, 0x82, 0x65, 0xdb, 0x1b, 0x91, 0x30, 0xcb, 0xde, 0x39, 0xc1,
0x1c, 0xae, 0x4c, 0x70, 0x6b, 0x7f, 0x57, 0x60, 0xf5, 0x63, 0xcb, 0xca, 0xc3, 0xd2, 0xd7, 0xcf,
0xa9, 0x59, 0xfc, 0x0a, 0x89, 0xf8, 0x2d, 0x82, 0x27, 0x77, 0x61, 0x25, 0x85, 0x93, 0x32, 0x0d,
0x6a, 0xba, 0x9a, 0x44, 0xca, 0x7e, 0x0f, 0xbd, 0x0f, 0x6a, 0x12, 0x2b, 0xe5, 0x2b, 0x51, 0xd3,
0xdb, 0x09, 0xb4, 0xec, 0xf7, 0xb4, 0x7f, 0x28, 0x70, 0x55, 0xc7, 0x2e, 0x79, 0x81, 0xbf, 0xbc,
0x77, 0xfc, 0x67, 0x01, 0x56, 0x7f, 0x64, 0x06, 0xc3, 0xc3, 0x9e, 0x2b, 0x89, 0xf4, 0xcd, 0x5c,
0x30, 0x55, 0xe2, 0xa5, 0x6c, 0x89, 0x47, 0x69, 0xba, 0x9c, 0x97, 0xa6, 0xec, 0xc3, 0x6b, 0xe3,
0xb3, 0xf0, 0xbe, 0xb3, 0x34, 0x8d, 0xb5, 0x3d, 0xe5, 0x33, 0xb4, 0x3d, 0x68, 0x1b, 0x9a, 0xf8,
0xe5, 0xd0, 0x99, 0x5a, 0xd8, 0x10, 0xda, 0x2b, 0x5c, 0xfb, 0x8d, 0x1c, 0xed, 0xf1, 0x1a, 0x69,
0xc8, 0x43, 0x7d, 0x5e, 0x2a, 0x3f, 0x2f, 0x40, 0x5b, 0xee, 0xb2, 0x4e, 0x71, 0x01, 0x54, 0x4c,
0xb9, 0xa3, 0x90, 0x75, 0xc7, 0x22, 0x4e, 0x0d, 0x5f, 0xe8, 0x52, 0xec, 0x85, 0xbe, 0x0e, 0x30,
0x72, 0xa6, 0xf4, 0xd0, 0x08, 0x6c, 0x37, 0xc4, 0xc4, 0x1a, 0xa7, 0x1c, 0xd8, 0x2e, 0x46, 0x1f,
0x43, 0x63, 0x60, 0x7b, 0x0e, 0x19, 0x1b, 0x13, 0x33, 0x38, 0x64, 0xc8, 0x38, 0xef, 0xba, 0x8f,
0x6c, 0xec, 0x58, 0x0f, 0x39, 0xaf, 0x5e, 0x17, 0x67, 0xf6, 0xd8, 0x11, 0x74, 0x03, 0xea, 0x0c,
0x58, 0xc9, 0x48, 0x60, 0x6b, 0x45, 0xa8, 0xf0, 0xa6, 0xee, 0x93, 0x11, 0x43, 0x57, 0xed, 0xf7,
0x05, 0xb8, 0xc4, 0xdc, 0x20, 0x3d, 0x72, 0x01, 0x09, 0xf7, 0x20, 0x4c, 0x95, 0xe2, 0xfc, 0x77,
0x33, 0x15, 0x8f, 0x6c, 0xba, 0x9c, 0xe5, 0x5b, 0x05, 0x7d, 0x1f, 0x5a, 0x0e, 0x31, 0x2d, 0x63,
0x48, 0x3c, 0x8b, 0x47, 0x8a, 0x7b, 0xb8, 0xb5, 0xf5, 0x6e, 0x9e, 0x09, 0x07, 0xbe, 0x3d, 0x1e,
0x63, 0x7f, 0x3b, 0xe4, 0xd5, 0x9b, 0x0e, 0xff, 0x52, 0x93, 0x4b, 0x8e, 0xb0, 0xb2, 0xe5, 0xbe,
0xc3, 0xaf, 0x29, 0xba, 0xfe, 0x5a, 0x81, 0x76, 0xca, 0x50, 0xb4, 0x06, 0xf5, 0x18, 0x8f, 0x0c,
0x50, 0x9c, 0x84, 0xbe, 0x0b, 0xcb, 0xcc, 0x77, 0x98, 0x9b, 0xd4, 0xda, 0xd2, 0x36, 0xb2, 0x97,
0xfb, 0x46, 0x52, 0xaa, 0x2e, 0x36, 0xa0, 0x4d, 0xb8, 0x9c, 0x83, 0xac, 0xd2, 0x7c, 0x94, 0x05,
0x56, 0xed, 0x4f, 0x0a, 0x74, 0xf3, 0x9c, 0x79, 0x9e, 0x80, 0x3f, 0x86, 0xd5, 0xe8, 0x34, 0x86,
0x85, 0xe9, 0xd0, 0xb7, 0x27, 0xbc, 0xcc, 0xf8, 0x65, 0x50, 0xdf, 0xba, 0x79, 0xfa, 0x79, 0xa8,
0xbe, 0x12, 0x89, 0xe8, 0xc5, 0x24, 0x68, 0x36, 0xac, 0xec, 0xe0, 0x60, 0x1f, 0x8f, 0x5d, 0xec,
0x05, 0x7d, 0x6f, 0x44, 0xce, 0x1e, 0xf7, 0x1b, 0x00, 0x54, 0xca, 0x89, 0xee, 0xa9, 0x18, 0x45,
0xfb, 0x7b, 0x01, 0xea, 0x31, 0x45, 0xe8, 0x1a, 0xd4, 0xa2, 0x55, 0x19, 0xb5, 0x19, 0x21, 0x93,
0x31, 0x85, 0x9c, 0x8c, 0x49, 0x45, 0xbe, 0x98, 0x8d, 0xfc, 0x1c, 0x70, 0x46, 0x57, 0xa1, 0xea,
0x62, 0xd7, 0xa0, 0xf6, 0x0b, 0x2c, 0xc1, 0xa0, 0xe2, 0x62, 0x77, 0xdf, 0x7e, 0x81, 0xd9, 0x92,
0x37, 0x75, 0x0d, 0x9f, 0x1c, 0xd1, 0x4e, 0x59, 0x2c, 0x79, 0x53, 0x57, 0x27, 0x47, 0x14, 0x5d,
0x07, 0xb0, 0x3d, 0x0b, 0x3f, 0x37, 0x3c, 0xd3, 0xc5, 0x9d, 0x0a, 0x2f, 0xa6, 0x1a, 0xa7, 0xec,
0x9a, 0x2e, 0x66, 0x30, 0xc0, 0x3f, 0xfa, 0xbd, 0x4e, 0x55, 0x6c, 0x94, 0x9f, 0xec, 0xa8, 0xb2,
0x04, 0xfb, 0xbd, 0x4e, 0x4d, 0xec, 0x8b, 0x08, 0xe8, 0x13, 0x68, 0xca, 0x73, 0x1b, 0x22, 0x4d,
0x81, 0xa7, 0xe9, 0x5a, 0x5e, 0x58, 0xa5, 0x03, 0x45, 0x92, 0x36, 0x68, 0xec, 0x8b, 0xb7, 0x94,
0xe9, 0x58, 0x9e, 0x27, 0xed, 0xbe, 0x03, 0xcb, 0xb6, 0x37, 0x22, 0x61, 0x96, 0xbd, 0x75, 0x82,
0x39, 0x5c, 0x99, 0xe0, 0xd6, 0xfe, 0xa1, 0xc0, 0xea, 0xc7, 0x96, 0x95, 0x87, 0xa5, 0x2f, 0x9f,
0x53, 0xb3, 0xf8, 0x15, 0x12, 0xf1, 0x5b, 0x04, 0x4f, 0xde, 0x83, 0x4b, 0x29, 0x9c, 0x94, 0x69,
0x50, 0xd3, 0xd5, 0x24, 0x52, 0xf6, 0x7b, 0xe8, 0x5d, 0x50, 0x93, 0x58, 0x29, 0x6f, 0x89, 0x9a,
0xde, 0x4e, 0xa0, 0x65, 0xbf, 0xa7, 0xfd, 0x53, 0x81, 0xab, 0x3a, 0x76, 0xc9, 0x33, 0xfc, 0xe6,
0x9e, 0xf1, 0x5f, 0x05, 0x58, 0xfd, 0x91, 0x19, 0x0c, 0x0f, 0x7b, 0xae, 0x24, 0xd2, 0x57, 0x73,
0xc0, 0x54, 0x89, 0x97, 0xb2, 0x25, 0x1e, 0xa5, 0xe9, 0x72, 0x5e, 0x9a, 0xb2, 0x87, 0xd7, 0xc6,
0x17, 0xe1, 0x79, 0x67, 0x69, 0x1a, 0x6b, 0x7b, 0xca, 0x67, 0x68, 0x7b, 0xd0, 0x36, 0x34, 0xf1,
0xf3, 0xa1, 0x33, 0xb5, 0xb0, 0x21, 0xb4, 0x57, 0xb8, 0xf6, 0x1b, 0x39, 0xda, 0xe3, 0x35, 0xd2,
0x90, 0x9b, 0xfa, 0xbc, 0x54, 0x7e, 0x5e, 0x80, 0xb6, 0x5c, 0x65, 0x9d, 0xe2, 0x02, 0xa8, 0x98,
0x72, 0x47, 0x21, 0xeb, 0x8e, 0x45, 0x9c, 0x1a, 0xde, 0xd0, 0xa5, 0xd8, 0x0d, 0x7d, 0x1d, 0x60,
0xe4, 0x4c, 0xe9, 0xa1, 0x11, 0xd8, 0x6e, 0x88, 0x89, 0x35, 0x4e, 0x39, 0xb0, 0x5d, 0x8c, 0x3e,
0x86, 0xc6, 0xc0, 0xf6, 0x1c, 0x32, 0x36, 0x26, 0x66, 0x70, 0xc8, 0x90, 0x71, 0xde, 0x71, 0x1f,
0xd8, 0xd8, 0xb1, 0xee, 0x73, 0x5e, 0xbd, 0x2e, 0xf6, 0xec, 0xb1, 0x2d, 0xe8, 0x06, 0xd4, 0x19,
0xb0, 0x92, 0x91, 0xc0, 0xd6, 0x8a, 0x50, 0xe1, 0x4d, 0xdd, 0x47, 0x23, 0x86, 0xae, 0xda, 0xef,
0x0b, 0x70, 0x99, 0xb9, 0x41, 0x7a, 0xe4, 0x02, 0x12, 0xee, 0x5e, 0x98, 0x2a, 0xc5, 0xf9, 0xf7,
0x66, 0x2a, 0x1e, 0xd9, 0x74, 0x39, 0xcb, 0x5b, 0x05, 0x7d, 0x1f, 0x5a, 0x0e, 0x31, 0x2d, 0x63,
0x48, 0x3c, 0x8b, 0x47, 0x8a, 0x7b, 0xb8, 0xb5, 0xf5, 0x76, 0x9e, 0x09, 0x07, 0xbe, 0x3d, 0x1e,
0x63, 0x7f, 0x3b, 0xe4, 0xd5, 0x9b, 0x0e, 0x7f, 0xa9, 0xc9, 0x4f, 0x8e, 0xb0, 0xb2, 0xe5, 0xbe,
0x38, 0x5f, 0x85, 0x39, 0x52, 0x3c, 0xa1, 0x8b, 0x2b, 0x2d, 0xd0, 0xc5, 0x2d, 0xe7, 0x34, 0xe2,
0xc9, 0x4e, 0xa1, 0x9c, 0xe9, 0x14, 0x0e, 0xa0, 0x19, 0xe1, 0x0e, 0x2f, 0x8a, 0x9b, 0xd0, 0x14,
0x66, 0x19, 0xcc, 0x13, 0xd8, 0x0a, 0xbb, 0x70, 0x41, 0xfc, 0x94, 0xd3, 0x98, 0xd4, 0x08, 0xd7,
0xc4, 0xa3, 0x55, 0xd3, 0x63, 0x14, 0xed, 0x17, 0x0a, 0xa8, 0x71, 0xc4, 0xe6, 0x92, 0x17, 0x69,
0xef, 0x6f, 0x43, 0x5b, 0x0e, 0x88, 0x22, 0xd8, 0x94, 0x0d, 0xf7, 0xf3, 0xb8, 0xb8, 0x1e, 0xfa,
0x08, 0x56, 0x05, 0x63, 0x06, 0x66, 0x45, 0xe3, 0x7d, 0x99, 0xef, 0xea, 0x29, 0xac, 0xfd, 0x6b,
0x66, 0x19, 0xcc, 0x13, 0xd8, 0x0a, 0xbb, 0x70, 0x41, 0xfc, 0x9c, 0xd3, 0x98, 0xd4, 0x08, 0xd7,
0xc4, 0xa5, 0x55, 0xd3, 0x63, 0x14, 0xed, 0x17, 0x0a, 0xa8, 0x71, 0xc4, 0xe6, 0x92, 0x17, 0x69,
0xef, 0x6f, 0x43, 0x5b, 0x0e, 0x88, 0x22, 0xd8, 0x94, 0x0d, 0xf7, 0xd3, 0xb8, 0xb8, 0x1e, 0xfa,
0x08, 0x56, 0x05, 0x63, 0x06, 0x66, 0x45, 0xe3, 0x7d, 0x85, 0xaf, 0xea, 0x29, 0xac, 0xfd, 0x5b,
0x11, 0x5a, 0xb3, 0xc4, 0x59, 0xd8, 0xaa, 0x45, 0x06, 0x03, 0xbb, 0xa0, 0xce, 0x3a, 0x47, 0xde,
0x5b, 0x9c, 0x98, 0xfb, 0xe9, 0x9e, 0xb1, 0x3d, 0x49, 0xb5, 0xda, 0x8f, 0xa0, 0x29, 0xef, 0x24,
0x51, 0xaf, 0xc4, 0x85, 0x7d, 0x23, 0x4f, 0x58, 0x22, 0x82, 0x7a, 0x23, 0x06, 0xc1, 0x14, 0x3d,
0x80, 0x1a, 0x2f, 0x87, 0xe0, 0x78, 0x82, 0x65, 0x25, 0x5c, 0xcb, 0x93, 0xc1, 0x22, 0x7b, 0x70,
0x3c, 0xc1, 0x7a, 0xd5, 0x91, 0xbf, 0xce, 0x8b, 0xdb, 0xf7, 0xe1, 0x8a, 0x2f, 0x4a, 0xc7, 0x32,
0x12, 0xee, 0xab, 0x70, 0xf7, 0x5d, 0x0e, 0x37, 0xf7, 0xe2, 0x6e, 0x9c, 0xf3, 0x15, 0x50, 0x9d,
0xfb, 0x15, 0xf0, 0x53, 0x68, 0x7f, 0xd7, 0xf4, 0x2c, 0x32, 0x1a, 0x85, 0x05, 0x7a, 0x86, 0xca,
0x7c, 0x90, 0xec, 0xbf, 0x5e, 0x03, 0xad, 0xb4, 0x5f, 0x16, 0x60, 0x95, 0xd1, 0x1e, 0x9a, 0x8e,
0xe9, 0x0d, 0xf1, 0xe2, 0x5d, 0xf7, 0xff, 0xe6, 0x7d, 0xb9, 0x09, 0x4d, 0x4a, 0xa6, 0xfe, 0x10,
0x1b, 0x89, 0xe6, 0xbb, 0x21, 0x88, 0xbb, 0x02, 0x60, 0xae, 0x03, 0x58, 0x34, 0x30, 0x12, 0x5f,
0xe4, 0x35, 0x8b, 0x06, 0x72, 0xfb, 0x1d, 0xa8, 0x4b, 0x19, 0x16, 0xf1, 0x30, 0x0f, 0x76, 0x55,
0x07, 0x41, 0xea, 0x11, 0x8f, 0xf7, 0xe9, 0xec, 0x3c, 0xdf, 0xad, 0xf0, 0xdd, 0x8a, 0x45, 0x03,
0xbe, 0x75, 0x1d, 0xe0, 0x85, 0xe9, 0xd8, 0x16, 0x4f, 0x52, 0x1e, 0xa6, 0xaa, 0x5e, 0xe3, 0x14,
0xe6, 0x02, 0xed, 0xcf, 0x0a, 0xa0, 0x98, 0x77, 0xce, 0x8e, 0x9d, 0xb7, 0xa0, 0x95, 0xb8, 0x67,
0x34, 0xed, 0x8c, 0x5f, 0x94, 0x32, 0xf0, 0x1f, 0x08, 0x55, 0x86, 0x8f, 0x4d, 0x4a, 0x3c, 0xee,
0xb4, 0x85, 0xc1, 0x7f, 0x10, 0x9a, 0xc9, 0x8e, 0x6a, 0xff, 0x51, 0xa0, 0xb3, 0x8f, 0x4d, 0x07,
0x47, 0xef, 0x24, 0x2b, 0xb3, 0x31, 0xef, 0x28, 0xce, 0x70, 0x85, 0x9b, 0xd0, 0x24, 0x9e, 0x63,
0x7b, 0x51, 0xa8, 0xe4, 0x77, 0x96, 0x20, 0xca, 0x58, 0xdc, 0x85, 0x15, 0xc9, 0x14, 0x83, 0x6d,
0x31, 0x5c, 0x54, 0xc5, 0xc6, 0x7e, 0x44, 0x67, 0x4e, 0x21, 0xa3, 0x51, 0x5c, 0xa4, 0x88, 0x7e,
0x53, 0x52, 0xa5, 0xcc, 0x7b, 0x80, 0x42, 0xb6, 0x98, 0x50, 0xf1, 0x5a, 0xac, 0xc8, 0x9d, 0x99,
0xd4, 0xf5, 0x57, 0xd0, 0x4a, 0xa2, 0x13, 0x6a, 0x40, 0x75, 0x97, 0x04, 0x9f, 0xbc, 0xb4, 0x69,
0xa0, 0x2e, 0xa1, 0x16, 0xc0, 0x2e, 0x09, 0xf6, 0x7c, 0x4c, 0xb1, 0x17, 0xa8, 0x0a, 0x02, 0x28,
0x3f, 0xf1, 0x7a, 0x36, 0xfd, 0x5c, 0x2d, 0xa0, 0x4b, 0x72, 0x66, 0x60, 0x3a, 0x7d, 0x59, 0xaa,
0x6a, 0x91, 0x1d, 0x8f, 0x56, 0x25, 0xa4, 0x42, 0x23, 0x62, 0xd9, 0xd9, 0xfb, 0xa1, 0xba, 0x8c,
0x6a, 0xb0, 0x2c, 0x7e, 0x96, 0xd7, 0x9f, 0x80, 0x9a, 0x8e, 0x0a, 0xaa, 0x43, 0xe5, 0x50, 0x54,
0xb8, 0xba, 0x84, 0xda, 0x50, 0x77, 0x66, 0xf9, 0xa4, 0x2a, 0x8c, 0x30, 0xf6, 0x27, 0x43, 0x99,
0x59, 0x6a, 0x81, 0x69, 0x63, 0xce, 0xe8, 0x91, 0x23, 0x4f, 0x2d, 0xae, 0x7f, 0x0f, 0x1a, 0xf1,
0xef, 0x38, 0x54, 0x85, 0xd2, 0x2e, 0xf1, 0xb0, 0xba, 0xc4, 0xc4, 0xee, 0xf8, 0xe4, 0xc8, 0xf6,
0xc6, 0xe2, 0x0e, 0x8f, 0x7c, 0xf2, 0x0a, 0x7b, 0x6a, 0x81, 0x6d, 0x50, 0x6c, 0x3a, 0x6c, 0xa3,
0xc8, 0x36, 0x28, 0x4f, 0x01, 0xb5, 0xb4, 0xfe, 0x21, 0x54, 0x43, 0x94, 0x44, 0x2b, 0xd0, 0x4c,
0x4c, 0x1c, 0xd5, 0x25, 0x84, 0x44, 0xe3, 0x31, 0xc3, 0x43, 0x55, 0xd9, 0xfa, 0x37, 0x00, 0x88,
0x87, 0x90, 0x10, 0xdf, 0x42, 0x13, 0x40, 0x3b, 0x38, 0xd8, 0x26, 0xee, 0x84, 0x78, 0xa1, 0x49,
0x14, 0x7d, 0x90, 0x4c, 0x9e, 0xe8, 0xef, 0x8d, 0x2c, 0xab, 0xbc, 0x65, 0xf7, 0xbd, 0x39, 0x27,
0x52, 0xec, 0xda, 0x12, 0x72, 0xb9, 0x46, 0xd6, 0x57, 0x1e, 0xd8, 0xc3, 0xcf, 0xc3, 0x71, 0xd5,
0x09, 0x1a, 0x53, 0xac, 0xa1, 0xc6, 0x14, 0x24, 0xca, 0xc5, 0x7e, 0xe0, 0xdb, 0xde, 0x38, 0xfc,
0xf6, 0xd5, 0x96, 0xd0, 0x73, 0xb8, 0xcc, 0xbe, 0x8b, 0x03, 0x33, 0xb0, 0x69, 0x60, 0x0f, 0x69,
0xa8, 0x70, 0x6b, 0xbe, 0xc2, 0x0c, 0xf3, 0x6b, 0xaa, 0x74, 0xa0, 0x9d, 0xfa, 0x5b, 0x05, 0xad,
0xe7, 0xe2, 0x77, 0xee, 0x5f, 0x40, 0xdd, 0xbb, 0x0b, 0xf1, 0x46, 0xda, 0x6c, 0x68, 0x25, 0xff,
0x72, 0x40, 0xef, 0xcf, 0x13, 0x90, 0x99, 0xd1, 0x76, 0xd7, 0x17, 0x61, 0x8d, 0x54, 0x3d, 0x85,
0x56, 0x72, 0xa8, 0x9d, 0xaf, 0x2a, 0x77, 0xf0, 0xdd, 0x3d, 0x69, 0xec, 0xa0, 0x2d, 0xa1, 0x9f,
0xc0, 0x4a, 0x66, 0x92, 0x8c, 0xbe, 0x99, 0x27, 0x7e, 0xde, 0xc0, 0xf9, 0x34, 0x0d, 0xd2, 0xfa,
0x99, 0x17, 0xe7, 0x5b, 0x9f, 0xf9, 0x4b, 0x61, 0x71, 0xeb, 0x63, 0xe2, 0x4f, 0xb2, 0xfe, 0xb5,
0x35, 0x4c, 0x01, 0x65, 0x67, 0xc9, 0xe8, 0x5e, 0x9e, 0x8a, 0xb9, 0xf3, 0xec, 0xee, 0xc6, 0xa2,
0xec, 0x51, 0xc8, 0xa7, 0xbc, 0x5a, 0xd3, 0x53, 0xd7, 0x5c, 0xb5, 0x73, 0xc7, 0xc8, 0xf9, 0x6a,
0xe7, 0x0f, 0x4a, 0x45, 0x52, 0x27, 0xa7, 0x59, 0xf9, 0xb1, 0xca, 0x9d, 0x5e, 0xe6, 0x27, 0x75,
0xfe, 0x70, 0x4c, 0x5b, 0x42, 0x06, 0xc0, 0x0e, 0x0e, 0x1e, 0xe3, 0xc0, 0xb7, 0x87, 0x14, 0xbd,
0x97, 0x5b, 0xe2, 0x33, 0x86, 0x50, 0xc7, 0xed, 0x53, 0xf9, 0x42, 0x05, 0x5b, 0x7f, 0xa9, 0x41,
0x8d, 0x7b, 0x97, 0x3d, 0x7e, 0x5f, 0x01, 0xee, 0x05, 0x00, 0xee, 0x33, 0x68, 0xa7, 0x86, 0x8e,
0xf9, 0x80, 0x9b, 0x3f, 0x99, 0x3c, 0xad, 0xf2, 0x06, 0x80, 0xb2, 0x13, 0xbf, 0xfc, 0x12, 0x98,
0x3b, 0x19, 0x3c, 0x4d, 0xc7, 0x33, 0x68, 0xa7, 0x26, 0x6e, 0xf9, 0x37, 0xc8, 0x1f, 0xcb, 0x9d,
0x26, 0xfd, 0x33, 0x68, 0xc4, 0x67, 0x2b, 0xe8, 0xf6, 0x3c, 0xdc, 0x4b, 0x4d, 0x14, 0xde, 0x3c,
0xea, 0x5d, 0xfc, 0xab, 0xf0, 0x0c, 0xda, 0xa9, 0x71, 0x4a, 0xbe, 0xe7, 0xf3, 0x67, 0x2e, 0xa7,
0x49, 0xff, 0x12, 0xe1, 0xd8, 0xc3, 0x8f, 0x9e, 0x6e, 0x8d, 0xed, 0xe0, 0x70, 0x3a, 0x60, 0xb7,
0xdc, 0x14, 0x9c, 0xf7, 0x6c, 0x22, 0x7f, 0x6d, 0x86, 0x05, 0xbd, 0xc9, 0x25, 0x6d, 0x72, 0x6b,
0x27, 0x83, 0x41, 0x99, 0x2f, 0xef, 0xff, 0x37, 0x00, 0x00, 0xff, 0xff, 0x7b, 0xfb, 0x73, 0x08,
0x67, 0x23, 0x00, 0x00,
0x5b, 0x9c, 0x98, 0xfb, 0xe9, 0x9e, 0xb1, 0x3d, 0x49, 0xb5, 0xda, 0x0f, 0xa0, 0x29, 0xcf, 0x24,
0x51, 0xaf, 0xc4, 0x85, 0x7d, 0x2b, 0x4f, 0x58, 0x22, 0x82, 0x7a, 0x23, 0x06, 0xc1, 0x14, 0xdd,
0x83, 0x1a, 0x2f, 0x87, 0xe0, 0x78, 0x82, 0x65, 0x25, 0x5c, 0xcb, 0x93, 0xc1, 0x22, 0x7b, 0x70,
0x3c, 0xc1, 0x7a, 0xd5, 0x91, 0xbf, 0xce, 0x8b, 0xdb, 0x77, 0x61, 0xc5, 0x17, 0xa5, 0x63, 0x19,
0x09, 0xf7, 0x55, 0xb8, 0xfb, 0xae, 0x84, 0x8b, 0x7b, 0x71, 0x37, 0xce, 0x79, 0x05, 0x54, 0xe7,
0xbe, 0x02, 0x7e, 0x0a, 0xed, 0x4f, 0x4d, 0xcf, 0x22, 0xa3, 0x51, 0x58, 0xa0, 0x67, 0xa8, 0xcc,
0x7b, 0xc9, 0xfe, 0xeb, 0x25, 0xd0, 0x4a, 0xfb, 0x65, 0x01, 0x56, 0x19, 0xed, 0xbe, 0xe9, 0x98,
0xde, 0x10, 0x2f, 0xde, 0x75, 0xff, 0x6f, 0xee, 0x97, 0x9b, 0xd0, 0xa4, 0x64, 0xea, 0x0f, 0xb1,
0x91, 0x68, 0xbe, 0x1b, 0x82, 0xb8, 0x2b, 0x00, 0xe6, 0x3a, 0x80, 0x45, 0x03, 0x23, 0xf1, 0x22,
0xaf, 0x59, 0x34, 0x90, 0xcb, 0x6f, 0x41, 0x5d, 0xca, 0xb0, 0x88, 0x87, 0x79, 0xb0, 0xab, 0x3a,
0x08, 0x52, 0x8f, 0x78, 0xbc, 0x4f, 0x67, 0xfb, 0xf9, 0x6a, 0x85, 0xaf, 0x56, 0x2c, 0x1a, 0xf0,
0xa5, 0xeb, 0x00, 0xcf, 0x4c, 0xc7, 0xb6, 0x78, 0x92, 0xf2, 0x30, 0x55, 0xf5, 0x1a, 0xa7, 0x30,
0x17, 0x68, 0x7f, 0x51, 0x00, 0xc5, 0xbc, 0x73, 0x76, 0xec, 0xbc, 0x05, 0xad, 0xc4, 0x39, 0xa3,
0x69, 0x67, 0xfc, 0xa0, 0x94, 0x81, 0xff, 0x40, 0xa8, 0x32, 0x7c, 0x6c, 0x52, 0xe2, 0x71, 0xa7,
0x2d, 0x0c, 0xfe, 0x83, 0xd0, 0x4c, 0xb6, 0x55, 0xfb, 0x73, 0x01, 0x3a, 0xfb, 0xd8, 0x74, 0x70,
0x74, 0x4f, 0xb2, 0x32, 0x1b, 0xf3, 0x8e, 0xe2, 0x0c, 0x47, 0xb8, 0x09, 0x4d, 0xe2, 0x39, 0xb6,
0x17, 0x85, 0x4a, 0xbe, 0xb3, 0x04, 0x51, 0xc6, 0xe2, 0x53, 0x68, 0x4b, 0x26, 0x99, 0x29, 0x21,
0x8a, 0x9c, 0xfa, 0x26, 0x68, 0x89, 0x7d, 0x51, 0x15, 0xdc, 0x82, 0x16, 0x19, 0x8d, 0xe2, 0xfa,
0x44, 0x6a, 0x34, 0x25, 0x55, 0x2a, 0xfc, 0x0c, 0xd4, 0x90, 0x2d, 0xd2, 0xb8, 0xbc, 0x98, 0xc6,
0xb6, 0xdc, 0x18, 0xaa, 0x5c, 0x7f, 0x01, 0xad, 0x24, 0xae, 0xa1, 0x06, 0x54, 0x77, 0x49, 0xf0,
0xc9, 0x73, 0x9b, 0x06, 0xea, 0x12, 0x6a, 0x01, 0xec, 0x92, 0x60, 0xcf, 0xc7, 0x14, 0x7b, 0x81,
0xaa, 0x20, 0x80, 0xf2, 0x23, 0xaf, 0x67, 0xd3, 0x2f, 0xd5, 0x02, 0xba, 0x2c, 0xa7, 0x0d, 0xa6,
0xd3, 0x97, 0x45, 0xae, 0x16, 0xd9, 0xf6, 0xe8, 0xab, 0x84, 0x54, 0x68, 0x44, 0x2c, 0x3b, 0x7b,
0x3f, 0x54, 0x97, 0x51, 0x0d, 0x96, 0xc5, 0xcf, 0xf2, 0xfa, 0x23, 0x50, 0xd3, 0xf1, 0x44, 0x75,
0xa8, 0x1c, 0x0a, 0x6c, 0x50, 0x97, 0x50, 0x1b, 0xea, 0xce, 0x2c, 0x13, 0x55, 0x85, 0x11, 0xc6,
0xfe, 0x64, 0x28, 0x73, 0x52, 0x2d, 0x30, 0x6d, 0xcc, 0x53, 0x3d, 0x72, 0xe4, 0xa9, 0xc5, 0xf5,
0xcf, 0xa0, 0x11, 0x7f, 0x01, 0xa2, 0x2a, 0x94, 0x76, 0x89, 0x87, 0xd5, 0x25, 0x26, 0x76, 0xc7,
0x27, 0x47, 0xb6, 0x37, 0x16, 0x67, 0x78, 0xe0, 0x93, 0x17, 0xd8, 0x53, 0x0b, 0x6c, 0x81, 0x62,
0xd3, 0x61, 0x0b, 0x45, 0xb6, 0x40, 0x79, 0xf2, 0xa8, 0xa5, 0xf5, 0x0f, 0xa1, 0x1a, 0xe2, 0x2b,
0xba, 0x04, 0xcd, 0xc4, 0xac, 0x52, 0x5d, 0x42, 0x48, 0xb4, 0x2c, 0x33, 0x24, 0x55, 0x95, 0xad,
0xff, 0x00, 0x80, 0xb8, 0x42, 0x09, 0xf1, 0x2d, 0x34, 0x01, 0xb4, 0x83, 0x83, 0x6d, 0xe2, 0x4e,
0x88, 0x17, 0x9a, 0x44, 0xd1, 0x07, 0xc9, 0x10, 0x45, 0x7f, 0x8c, 0x64, 0x59, 0xe5, 0x29, 0xbb,
0xef, 0xcc, 0xd9, 0x91, 0x62, 0xd7, 0x96, 0x90, 0xcb, 0x35, 0xb2, 0x8e, 0xf4, 0xc0, 0x1e, 0x7e,
0x19, 0x0e, 0xba, 0x4e, 0xd0, 0x98, 0x62, 0x0d, 0x35, 0xa6, 0xc0, 0x54, 0x7e, 0xec, 0x07, 0xbe,
0xed, 0x8d, 0xc3, 0x57, 0xb3, 0xb6, 0x84, 0x9e, 0xc2, 0x15, 0xf6, 0xa2, 0x0e, 0xcc, 0xc0, 0xa6,
0x81, 0x3d, 0xa4, 0xa1, 0xc2, 0xad, 0xf9, 0x0a, 0x33, 0xcc, 0x2f, 0xa9, 0xd2, 0x81, 0x76, 0xea,
0x0f, 0x19, 0xb4, 0x9e, 0x9b, 0xf3, 0xb9, 0x7f, 0x1e, 0x75, 0xdf, 0x5b, 0x88, 0x37, 0xd2, 0x66,
0x43, 0x2b, 0xf9, 0x67, 0x05, 0x7a, 0x77, 0x9e, 0x80, 0xcc, 0x74, 0xb7, 0xbb, 0xbe, 0x08, 0x6b,
0xa4, 0xea, 0x31, 0xb4, 0x92, 0xe3, 0xf0, 0x7c, 0x55, 0xb9, 0x23, 0xf3, 0xee, 0x49, 0x03, 0x0b,
0x6d, 0x09, 0xfd, 0x04, 0x2e, 0x65, 0x66, 0xd0, 0xe8, 0xdb, 0x79, 0xe2, 0xe7, 0x8d, 0xaa, 0x4f,
0xd3, 0x20, 0xad, 0x9f, 0x79, 0x71, 0xbe, 0xf5, 0x99, 0x3f, 0x23, 0x16, 0xb7, 0x3e, 0x26, 0xfe,
0x24, 0xeb, 0x5f, 0x5a, 0xc3, 0x14, 0x50, 0x76, 0x0a, 0x8d, 0xde, 0xcf, 0x53, 0x31, 0x77, 0x12,
0xde, 0xdd, 0x58, 0x94, 0x3d, 0x0a, 0xf9, 0x94, 0x57, 0x6b, 0x7a, 0x5e, 0x9b, 0xab, 0x76, 0xee,
0x00, 0x3a, 0x5f, 0xed, 0xfc, 0x11, 0xab, 0x48, 0xea, 0xe4, 0x1c, 0x2c, 0x3f, 0x56, 0xb9, 0x73,
0xcf, 0xfc, 0xa4, 0xce, 0x1f, 0xab, 0x69, 0x4b, 0xc8, 0x00, 0xd8, 0xc1, 0xc1, 0x43, 0x1c, 0xf8,
0xf6, 0x90, 0xa2, 0x77, 0x72, 0x4b, 0x7c, 0xc6, 0x10, 0xea, 0xb8, 0x7d, 0x2a, 0x5f, 0xa8, 0x60,
0xeb, 0xaf, 0x35, 0xa8, 0x71, 0xef, 0xb2, 0x9b, 0xf1, 0x6b, 0xc0, 0xbd, 0x00, 0xc0, 0x7d, 0x02,
0xed, 0xd4, 0xb8, 0x32, 0x1f, 0x70, 0xf3, 0x67, 0x9a, 0xa7, 0x55, 0xde, 0x00, 0x50, 0x76, 0x56,
0x98, 0x5f, 0x02, 0x73, 0x67, 0x8a, 0xa7, 0xe9, 0x78, 0x02, 0xed, 0xd4, 0xac, 0x2e, 0xff, 0x04,
0xf9, 0x03, 0xbd, 0xd3, 0xa4, 0x7f, 0x01, 0x8d, 0xf8, 0x54, 0x06, 0xdd, 0x9e, 0x87, 0x7b, 0xa9,
0x59, 0xc4, 0xab, 0x47, 0xbd, 0x8b, 0xbf, 0x15, 0x9e, 0x40, 0x3b, 0x35, 0x88, 0xc9, 0xf7, 0x7c,
0xfe, 0xb4, 0xe6, 0x34, 0xe9, 0x6f, 0x10, 0x8e, 0xdd, 0xff, 0xe8, 0xf1, 0xd6, 0xd8, 0x0e, 0x0e,
0xa7, 0x03, 0x76, 0xca, 0x4d, 0xc1, 0xf9, 0xbe, 0x4d, 0xe4, 0xaf, 0xcd, 0xb0, 0xa0, 0x37, 0xb9,
0xa4, 0x4d, 0x6e, 0xed, 0x64, 0x30, 0x28, 0xf3, 0xcf, 0xbb, 0xff, 0x0d, 0x00, 0x00, 0xff, 0xff,
0x3c, 0xe9, 0xbf, 0xfe, 0xa1, 0x23, 0x00, 0x00,
}
// Reference imports to suppress errors if they are not otherwise used.

93
internal/querynode/global_sealed_segment_manager.go Normal file
View File

@ -0,0 +1,93 @@
// Copyright (C) 2019-2020 Zilliz. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software distributed under the License
// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
// or implied. See the License for the specific language governing permissions and limitations under the License.
package querynode
import (
"errors"
"fmt"
"sync"
"github.com/milvus-io/milvus/internal/proto/querypb"
)
type globalSealedSegmentManager struct {
collectionID UniqueID
mu sync.Mutex // guards globalSealedSegments
globalSealedSegments map[UniqueID]*querypb.SegmentInfo // map[segmentID]SegmentInfo
}
func newGlobalSealedSegmentManager(collectionID UniqueID) *globalSealedSegmentManager {
return &globalSealedSegmentManager{
collectionID: collectionID,
globalSealedSegments: make(map[UniqueID]*querypb.SegmentInfo),
}
}
func (g *globalSealedSegmentManager) addGlobalSegmentInfo(segmentInfo *querypb.SegmentInfo) error {
g.mu.Lock()
defer g.mu.Unlock()
if segmentInfo.CollectionID != g.collectionID {
return errors.New(fmt.Sprintln("mismatch collectionID when addGlobalSegmentInfo, ",
"manager collectionID = ", g.collectionID, ", ",
"segmentInfo collectionID = ", segmentInfo.CollectionID))
}
g.globalSealedSegments[segmentInfo.SegmentID] = segmentInfo
return nil
}
func (g *globalSealedSegmentManager) getGlobalSegmentIDs() []UniqueID {
g.mu.Lock()
defer g.mu.Unlock()
resIDs := make([]UniqueID, 0)
for _, v := range g.globalSealedSegments {
resIDs = append(resIDs, v.SegmentID)
}
return resIDs
}
func (g *globalSealedSegmentManager) getGlobalSegmentIDsByPartitionIds(partitionIDs []UniqueID) []UniqueID {
g.mu.Lock()
defer g.mu.Unlock()
resIDs := make([]UniqueID, 0)
for _, v := range g.globalSealedSegments {
for _, partitionID := range partitionIDs {
if v.PartitionID == partitionID {
resIDs = append(resIDs, v.SegmentID)
}
}
}
return resIDs
}
func (g *globalSealedSegmentManager) removeGlobalSegmentInfo(segmentID UniqueID) {
g.mu.Lock()
defer g.mu.Unlock()
delete(g.globalSealedSegments, segmentID)
}
func (g *globalSealedSegmentManager) removeGlobalSegmentIDsByPartitionIds(partitionIDs []UniqueID) {
g.mu.Lock()
defer g.mu.Unlock()
for _, v := range g.globalSealedSegments {
for _, partitionID := range partitionIDs {
if v.PartitionID == partitionID {
delete(g.globalSealedSegments, v.SegmentID)
}
}
}
}
func (g *globalSealedSegmentManager) close() {
g.mu.Lock()
defer g.mu.Unlock()
g.globalSealedSegments = make(map[UniqueID]*querypb.SegmentInfo)
}

66
internal/querynode/global_sealed_segment_manager_test.go Normal file
View File

@ -0,0 +1,66 @@
// Copyright (C) 2019-2020 Zilliz. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software distributed under the License
// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
// or implied. See the License for the specific language governing permissions and limitations under the License.
package querynode
import (
"testing"
"github.com/stretchr/testify/assert"
"github.com/milvus-io/milvus/internal/proto/querypb"
)
func TestGlobalSealedSegmentManager(t *testing.T) {
manager := newGlobalSealedSegmentManager(defaultCollectionID)
assert.NotNil(t, manager)
segmentInfo := &querypb.SegmentInfo{
SegmentID: defaultSegmentID,
CollectionID: defaultCollectionID,
PartitionID: defaultPartitionID,
}
err := manager.addGlobalSegmentInfo(segmentInfo)
assert.NoError(t, err)
segmentInfo.CollectionID = 1000
err = manager.addGlobalSegmentInfo(segmentInfo)
assert.Error(t, err)
ids := manager.getGlobalSegmentIDs()
assert.Len(t, ids, 1)
assert.Equal(t, segmentInfo.SegmentID, ids[0])
ids = manager.getGlobalSegmentIDsByPartitionIds([]UniqueID{defaultPartitionID})
assert.Len(t, ids, 1)
assert.Equal(t, segmentInfo.SegmentID, ids[0])
manager.removeGlobalSegmentIDsByPartitionIds([]UniqueID{defaultPartitionID})
ids = manager.getGlobalSegmentIDs()
assert.Len(t, ids, 0)
segmentInfo.CollectionID = defaultCollectionID
err = manager.addGlobalSegmentInfo(segmentInfo)
assert.NoError(t, err)
manager.removeGlobalSegmentInfo(defaultSegmentID)
ids = manager.getGlobalSegmentIDs()
assert.Len(t, ids, 0)
segmentInfo.CollectionID = defaultCollectionID
err = manager.addGlobalSegmentInfo(segmentInfo)
assert.NoError(t, err)
manager.close()
ids = manager.getGlobalSegmentIDs()
assert.Len(t, ids, 0)
}

53
internal/querynode/query_collection.go
View File

@ -71,6 +71,8 @@ type queryCollection struct {
remoteChunkManager storage.ChunkManager
vectorChunkManager storage.ChunkManager
localCacheEnabled bool
globalSegmentManager *globalSealedSegmentManager
}
type ResultEntityIds []UniqueID
@ -110,9 +112,10 @@ func newQueryCollection(releaseCtx context.Context,
queryMsgStream: queryStream,
queryResultMsgStream: queryResultStream,
localChunkManager: localChunkManager,
remoteChunkManager: remoteChunkManager,
localCacheEnabled: localCacheEnabled,
localChunkManager: localChunkManager,
remoteChunkManager: remoteChunkManager,
localCacheEnabled: localCacheEnabled,
globalSegmentManager: newGlobalSealedSegmentManager(collectionID),
}
err := qc.registerCollectionTSafe()
@ -136,6 +139,7 @@ func (q *queryCollection) close() {
if q.queryResultMsgStream != nil {
q.queryResultMsgStream.Close()
}
q.globalSegmentManager.close()
}
// registerCollectionTSafe registers tSafe watcher if vChannels exists
@ -287,6 +291,12 @@ func (q *queryCollection) consumeQuery() {
if err != nil {
log.Warn(err.Error())
}
case *msgstream.SealedSegmentsChangeInfoMsg:
err := q.adjustByChangeInfo(sm)
if err != nil {
// should not happen
log.Error(err.Error())
}
default:
log.Warn("unsupported msg type in search channel", zap.Any("msg", sm))
}
@ -299,6 +309,43 @@ func (q *queryCollection) loadBalance(msg *msgstream.LoadBalanceSegmentsMsg) {
//TODO:: get loadBalance info from etcd
}
func (q *queryCollection) adjustByChangeInfo(msg *msgstream.SealedSegmentsChangeInfoMsg) error {
// for OnlineSegments:
for _, segment := range msg.OnlineSegments {
// 1. update global sealed segments
err := q.globalSegmentManager.addGlobalSegmentInfo(segment)
if err != nil {
return err
}
// 2. delete growing segment because these segments are loaded
hasGrowingSegment := q.streaming.replica.hasSegment(segment.SegmentID)
if hasGrowingSegment {
err = q.streaming.replica.removeSegment(segment.SegmentID)
if err != nil {
return err
}
log.Debug("remove growing segment in adjustByChangeInfo",
zap.Any("collectionID", q.collectionID),
zap.Any("segmentID", segment.SegmentID),
)
}
}
// for OfflineSegments:
for _, segment := range msg.OfflineSegments {
// 1. update global sealed segments
q.globalSegmentManager.removeGlobalSegmentInfo(segment.SegmentID)
// 2. load balance, remove old sealed segments
if msg.OfflineNodeID == Params.QueryNodeID {
err := q.historical.replica.removeSegment(segment.SegmentID)
if err != nil {
return err
}
}
}
return nil
}
func (q *queryCollection) receiveQueryMsg(msg queryMsg) error {
msgType := msg.Type()
var collectionID UniqueID

89
internal/querynode/query_collection_test.go
View File

@ -18,6 +18,7 @@ import (
"github.com/milvus-io/milvus/internal/proto/commonpb"
"github.com/milvus-io/milvus/internal/proto/internalpb"
"github.com/milvus-io/milvus/internal/proto/milvuspb"
"github.com/milvus-io/milvus/internal/proto/querypb"
"github.com/milvus-io/milvus/internal/proto/schemapb"
"github.com/milvus-io/milvus/internal/proto/segcorepb"
"github.com/milvus-io/milvus/internal/util/tsoutil"
@ -61,6 +62,31 @@ func genSimpleQueryCollection(ctx context.Context, cancel context.CancelFunc) (*
return queryCollection, err
}
func genSimpleSegmentInfo() *querypb.SegmentInfo {
return &querypb.SegmentInfo{
SegmentID: defaultSegmentID,
CollectionID: defaultCollectionID,
PartitionID: defaultPartitionID,
}
}
func genSimpleSealedSegmentsChangeInfo() *querypb.SealedSegmentsChangeInfo {
return &querypb.SealedSegmentsChangeInfo{
Base: genCommonMsgBase(commonpb.MsgType_SealedSegmentsChangeInfo),
OnlineNodeID: Params.QueryNodeID,
OnlineSegments: []*querypb.SegmentInfo{},
OfflineNodeID: Params.QueryNodeID,
OfflineSegments: []*querypb.SegmentInfo{},
}
}
func genSimpleSealedSegmentsChangeInfoMsg() *msgstream.SealedSegmentsChangeInfoMsg {
return &msgstream.SealedSegmentsChangeInfoMsg{
BaseMsg: genMsgStreamBaseMsg(),
SealedSegmentsChangeInfo: *genSimpleSealedSegmentsChangeInfo(),
}
}
func updateTSafe(queryCollection *queryCollection, timestamp Timestamp) {
// register
queryCollection.tSafeWatchers[defaultVChannel] = newTSafeWatcher()
@ -306,6 +332,15 @@ func TestQueryCollection_consumeQuery(t *testing.T) {
runConsumeQuery(msg)
})
t.Run("consume SimpleSealedSegmentsChangeInfoMsg", func(t *testing.T) {
// test is success if it doesn't block
msg := genSimpleSealedSegmentsChangeInfoMsg()
simpleInfo := genSimpleSegmentInfo()
simpleInfo.CollectionID = 1000
msg.OnlineSegments = append(msg.OnlineSegments, simpleInfo)
runConsumeQuery(msg)
})
t.Run("consume invalid msg", func(t *testing.T) {
msg, err := genSimpleRetrieveMsg()
assert.NoError(t, err)
@ -631,3 +666,57 @@ func TestQueryCollection_AddPopUnsolvedMsg(t *testing.T) {
assert.EqualValues(t, i, unsolved[i].ID())
}
}
func TestQueryCollection_adjustByChangeInfo(t *testing.T) {
ctx, cancel := context.WithCancel(context.TODO())
t.Run("test adjustByChangeInfo", func(t *testing.T) {
qc, err := genSimpleQueryCollection(ctx, cancel)
assert.Nil(t, err)
info := genSimpleSealedSegmentsChangeInfoMsg()
// test online
info.OnlineSegments = append(info.OnlineSegments, genSimpleSegmentInfo())
err = qc.adjustByChangeInfo(info)
assert.NoError(t, err)
ids := qc.globalSegmentManager.getGlobalSegmentIDs()
assert.Len(t, ids, 1)
// test offline
info.OnlineSegments = make([]*querypb.SegmentInfo, 0)
info.OfflineSegments = append(info.OfflineSegments, genSimpleSegmentInfo())
err = qc.adjustByChangeInfo(info)
assert.NoError(t, err)
ids = qc.globalSegmentManager.getGlobalSegmentIDs()
assert.Len(t, ids, 0)
})
t.Run("test mismatch collectionID when adjustByChangeInfo", func(t *testing.T) {
qc, err := genSimpleQueryCollection(ctx, cancel)
assert.Nil(t, err)
info := genSimpleSealedSegmentsChangeInfoMsg()
// test online
simpleInfo := genSimpleSegmentInfo()
simpleInfo.CollectionID = 1000
info.OnlineSegments = append(info.OnlineSegments, simpleInfo)
err = qc.adjustByChangeInfo(info)
assert.Error(t, err)
})
t.Run("test no segment when adjustByChangeInfo", func(t *testing.T) {
qc, err := genSimpleQueryCollection(ctx, cancel)
assert.Nil(t, err)
err = qc.historical.replica.removeSegment(defaultSegmentID)
assert.NoError(t, err)
info := genSimpleSealedSegmentsChangeInfoMsg()
info.OfflineSegments = append(info.OfflineSegments, genSimpleSegmentInfo())
err = qc.adjustByChangeInfo(info)
assert.Error(t, err)
})
}