diff --git a/internal/proto/query_coord.proto b/internal/proto/query_coord.proto index d3db8f71b5..f588c10a03 100644 --- a/internal/proto/query_coord.proto +++ b/internal/proto/query_coord.proto @@ -490,6 +490,7 @@ message LeaderView { map segment_dist = 3; repeated int64 growing_segmentIDs = 4; map growing_segments = 5; + int64 TargetVersion = 6; } message SegmentDist { @@ -547,6 +548,7 @@ enum SyncType { Remove = 0; Set = 1; Amend = 2; + UpdateVersion = 3; } message SyncAction { @@ -556,6 +558,9 @@ message SyncAction { int64 nodeID = 4; int64 version = 5; SegmentLoadInfo info = 6; + repeated int64 growingInTarget = 7; + repeated int64 sealedInTarget = 8; + int64 TargetVersion = 9; } message SyncDistributionRequest { diff --git a/internal/proto/querypb/query_coord.pb.go b/internal/proto/querypb/query_coord.pb.go index 04daa110bb..ab6c90fb52 100644 --- a/internal/proto/querypb/query_coord.pb.go +++ b/internal/proto/querypb/query_coord.pb.go @@ -220,21 +220,24 @@ func (LoadStatus) EnumDescriptor() ([]byte, []int) { type SyncType int32 const ( - SyncType_Remove SyncType = 0 - SyncType_Set SyncType = 1 - SyncType_Amend SyncType = 2 + SyncType_Remove SyncType = 0 + SyncType_Set SyncType = 1 + SyncType_Amend SyncType = 2 + SyncType_UpdateVersion SyncType = 3 ) var SyncType_name = map[int32]string{ 0: "Remove", 1: "Set", 2: "Amend", + 3: "UpdateVersion", } var SyncType_value = map[string]int32{ - "Remove": 0, - "Set": 1, - "Amend": 2, + "Remove": 0, + "Set": 1, + "Amend": 2, + "UpdateVersion": 3, } func (x SyncType) String() string { @@ -3398,6 +3401,7 @@ type LeaderView struct { SegmentDist map[int64]*SegmentDist `protobuf:"bytes,3,rep,name=segment_dist,json=segmentDist,proto3" json:"segment_dist,omitempty" protobuf_key:"varint,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` GrowingSegmentIDs []int64 `protobuf:"varint,4,rep,packed,name=growing_segmentIDs,json=growingSegmentIDs,proto3" json:"growing_segmentIDs,omitempty"` GrowingSegments map[int64]*msgpb.MsgPosition `protobuf:"bytes,5,rep,name=growing_segments,json=growingSegments,proto3" json:"growing_segments,omitempty" protobuf_key:"varint,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` + TargetVersion int64 `protobuf:"varint,6,opt,name=TargetVersion,proto3" json:"TargetVersion,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -3463,6 +3467,13 @@ func (m *LeaderView) GetGrowingSegments() map[int64]*msgpb.MsgPosition { return nil } +func (m *LeaderView) GetTargetVersion() int64 { + if m != nil { + return m.TargetVersion + } + return 0 +} + type SegmentDist struct { NodeID int64 `protobuf:"varint,1,opt,name=nodeID,proto3" json:"nodeID,omitempty"` Version int64 `protobuf:"varint,2,opt,name=version,proto3" json:"version,omitempty"` @@ -3864,6 +3875,9 @@ type SyncAction struct { NodeID int64 `protobuf:"varint,4,opt,name=nodeID,proto3" json:"nodeID,omitempty"` Version int64 `protobuf:"varint,5,opt,name=version,proto3" json:"version,omitempty"` Info *SegmentLoadInfo `protobuf:"bytes,6,opt,name=info,proto3" json:"info,omitempty"` + GrowingInTarget []int64 `protobuf:"varint,7,rep,packed,name=growingInTarget,proto3" json:"growingInTarget,omitempty"` + SealedInTarget []int64 `protobuf:"varint,8,rep,packed,name=sealedInTarget,proto3" json:"sealedInTarget,omitempty"` + TargetVersion int64 `protobuf:"varint,9,opt,name=TargetVersion,proto3" json:"TargetVersion,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -3936,6 +3950,27 @@ func (m *SyncAction) GetInfo() *SegmentLoadInfo { return nil } +func (m *SyncAction) GetGrowingInTarget() []int64 { + if m != nil { + return m.GrowingInTarget + } + return nil +} + +func (m *SyncAction) GetSealedInTarget() []int64 { + if m != nil { + return m.SealedInTarget + } + return nil +} + +func (m *SyncAction) GetTargetVersion() int64 { + if m != nil { + return m.TargetVersion + } + return 0 +} + type SyncDistributionRequest struct { Base *commonpb.MsgBase `protobuf:"bytes,1,opt,name=base,proto3" json:"base,omitempty"` CollectionID int64 `protobuf:"varint,2,opt,name=collectionID,proto3" json:"collectionID,omitempty"` @@ -4503,297 +4538,301 @@ func init() { func init() { proto.RegisterFile("query_coord.proto", fileDescriptor_aab7cc9a69ed26e8) } var fileDescriptor_aab7cc9a69ed26e8 = []byte{ - // 4638 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xec, 0x3c, 0x4b, 0x6f, 0x1c, 0x47, - 0x7a, 0xea, 0x79, 0x71, 0xe6, 0x9b, 0x27, 0x8b, 0xa4, 0x34, 0x3b, 0x2b, 0xc9, 0x74, 0xcb, 0xb2, - 0xb9, 0xb4, 0x4d, 0x6a, 0xa9, 0x5d, 0xaf, 0x76, 0xed, 0x85, 0x23, 0x91, 0x96, 0xcc, 0xb5, 0x4c, - 0x73, 0x9b, 0x92, 0x36, 0x70, 0xbc, 0x3b, 0xdb, 0x9c, 0x2e, 0x0e, 0x1b, 0xec, 0xc7, 0xa8, 0xbb, - 0x87, 0x14, 0x1d, 0x20, 0xa7, 0x5c, 0xb2, 0xc8, 0x26, 0xb9, 0x25, 0x87, 0x20, 0x87, 0x04, 0x01, - 0x36, 0x41, 0x72, 0x09, 0x12, 0x20, 0x87, 0x1c, 0x72, 0x4b, 0x90, 0x20, 0x8f, 0x5b, 0xfe, 0x40, - 0x72, 0x08, 0x90, 0x20, 0xa7, 0x45, 0xe0, 0x5b, 0x50, 0x8f, 0x7e, 0x54, 0x77, 0x0d, 0xa7, 0xc9, - 0x91, 0x5f, 0xc1, 0xde, 0xa6, 0xbf, 0x7a, 0x7c, 0x5f, 0x7d, 0xf5, 0xbd, 0xab, 0x6a, 0x60, 0xfe, - 0xe9, 0x18, 0x7b, 0xa7, 0xfd, 0x81, 0xeb, 0x7a, 0xc6, 0xda, 0xc8, 0x73, 0x03, 0x17, 0x21, 0xdb, - 0xb4, 0x8e, 0xc7, 0x3e, 0xfb, 0x5a, 0xa3, 0xed, 0xbd, 0xc6, 0xc0, 0xb5, 0x6d, 0xd7, 0x61, 0xb0, - 0x5e, 0x23, 0xd9, 0xa3, 0xd7, 0x32, 0x9d, 0x00, 0x7b, 0x8e, 0x6e, 0x85, 0xad, 0xfe, 0xe0, 0x10, - 0xdb, 0x3a, 0xff, 0xaa, 0xd9, 0xfe, 0x90, 0xff, 0xec, 0x18, 0x7a, 0xa0, 0x27, 0x51, 0xf5, 0xe6, - 0x4d, 0xc7, 0xc0, 0xcf, 0x92, 0x20, 0xf5, 0xd7, 0x15, 0xb8, 0xbc, 0x77, 0xe8, 0x9e, 0x6c, 0xba, - 0x96, 0x85, 0x07, 0x81, 0xe9, 0x3a, 0xbe, 0x86, 0x9f, 0x8e, 0xb1, 0x1f, 0xa0, 0x5b, 0x50, 0xda, - 0xd7, 0x7d, 0xdc, 0x55, 0x96, 0x95, 0x95, 0xfa, 0xc6, 0xd5, 0x35, 0x81, 0x4e, 0x4e, 0xe0, 0xfb, - 0xfe, 0xf0, 0x9e, 0xee, 0x63, 0x8d, 0xf6, 0x44, 0x08, 0x4a, 0xc6, 0xfe, 0xf6, 0x56, 0xb7, 0xb0, - 0xac, 0xac, 0x14, 0x35, 0xfa, 0x1b, 0xbd, 0x04, 0xcd, 0x41, 0x34, 0xf7, 0xf6, 0x96, 0xdf, 0x2d, - 0x2e, 0x17, 0x57, 0x8a, 0x9a, 0x08, 0x54, 0x7f, 0x52, 0x80, 0x2b, 0x19, 0x32, 0xfc, 0x91, 0xeb, - 0xf8, 0x18, 0xdd, 0x86, 0x8a, 0x1f, 0xe8, 0xc1, 0xd8, 0xe7, 0x94, 0x7c, 0x55, 0x4a, 0xc9, 0x1e, - 0xed, 0xa2, 0xf1, 0xae, 0x59, 0xb4, 0x05, 0x09, 0x5a, 0xf4, 0x75, 0x58, 0x34, 0x9d, 0xf7, 0xb1, - 0xed, 0x7a, 0xa7, 0xfd, 0x11, 0xf6, 0x06, 0xd8, 0x09, 0xf4, 0x21, 0x0e, 0x69, 0x5c, 0x08, 0xdb, - 0x76, 0xe3, 0x26, 0xf4, 0x06, 0x5c, 0x61, 0x7b, 0xe8, 0x63, 0xef, 0xd8, 0x1c, 0xe0, 0xbe, 0x7e, - 0xac, 0x9b, 0x96, 0xbe, 0x6f, 0xe1, 0x6e, 0x69, 0xb9, 0xb8, 0x52, 0xd5, 0x96, 0x68, 0xf3, 0x1e, - 0x6b, 0xbd, 0x1b, 0x36, 0xa2, 0xaf, 0x41, 0xc7, 0xc3, 0x07, 0x1e, 0xf6, 0x0f, 0xfb, 0x23, 0xcf, - 0x1d, 0x7a, 0xd8, 0xf7, 0xbb, 0x65, 0x8a, 0xa6, 0xcd, 0xe1, 0xbb, 0x1c, 0xac, 0xfe, 0xb1, 0x02, - 0x4b, 0x84, 0x19, 0xbb, 0xba, 0x17, 0x98, 0x9f, 0xc2, 0x96, 0xa8, 0xd0, 0x48, 0xb2, 0xa1, 0x5b, - 0xa4, 0x6d, 0x02, 0x8c, 0xf4, 0x19, 0x85, 0xe8, 0x09, 0xfb, 0x4a, 0x94, 0x54, 0x01, 0xa6, 0xfe, - 0x0b, 0x97, 0x9d, 0x24, 0x9d, 0xb3, 0xec, 0x59, 0x1a, 0x67, 0x21, 0x8b, 0xf3, 0x22, 0x3b, 0x26, - 0xe3, 0x7c, 0x49, 0xce, 0xf9, 0x7f, 0x2a, 0xc2, 0xd2, 0x43, 0x57, 0x37, 0x62, 0x31, 0xfc, 0xec, - 0x39, 0xff, 0x5d, 0xa8, 0x30, 0x8d, 0xee, 0x96, 0x28, 0xae, 0x9b, 0x22, 0x2e, 0xae, 0xed, 0x31, - 0x85, 0x7b, 0x14, 0xa0, 0xf1, 0x41, 0xe8, 0x26, 0xb4, 0x3c, 0x3c, 0xb2, 0xcc, 0x81, 0xde, 0x77, - 0xc6, 0xf6, 0x3e, 0xf6, 0xba, 0xe5, 0x65, 0x65, 0xa5, 0xac, 0x35, 0x39, 0x74, 0x87, 0x02, 0xd1, - 0x8f, 0xa1, 0x79, 0x60, 0x62, 0xcb, 0xe8, 0x53, 0x93, 0xb0, 0xbd, 0xd5, 0xad, 0x2c, 0x17, 0x57, - 0xea, 0x1b, 0x6f, 0xae, 0x65, 0xad, 0xd1, 0x9a, 0x94, 0x23, 0x6b, 0xf7, 0xc9, 0xf0, 0x6d, 0x36, - 0xfa, 0x1d, 0x27, 0xf0, 0x4e, 0xb5, 0xc6, 0x41, 0x02, 0x84, 0xba, 0x30, 0xc7, 0xd9, 0xdb, 0x9d, - 0x5b, 0x56, 0x56, 0xaa, 0x5a, 0xf8, 0x89, 0x5e, 0x81, 0xb6, 0x87, 0x7d, 0x77, 0xec, 0x0d, 0x70, - 0x7f, 0xe8, 0xb9, 0xe3, 0x91, 0xdf, 0xad, 0x2e, 0x17, 0x57, 0x6a, 0x5a, 0x2b, 0x04, 0x3f, 0xa0, - 0xd0, 0xde, 0xdb, 0x30, 0x9f, 0xc1, 0x82, 0x3a, 0x50, 0x3c, 0xc2, 0xa7, 0x74, 0x23, 0x8a, 0x1a, - 0xf9, 0x89, 0x16, 0xa1, 0x7c, 0xac, 0x5b, 0x63, 0xcc, 0x59, 0xcd, 0x3e, 0xbe, 0x53, 0xb8, 0xa3, - 0xa8, 0xbf, 0xaf, 0x40, 0x57, 0xc3, 0x16, 0xd6, 0x7d, 0xfc, 0x79, 0x6e, 0xe9, 0x65, 0xa8, 0x38, - 0xae, 0x81, 0xb7, 0xb7, 0xe8, 0x96, 0x16, 0x35, 0xfe, 0xa5, 0x7e, 0xa2, 0xc0, 0xe2, 0x03, 0x1c, - 0x10, 0x35, 0x30, 0xfd, 0xc0, 0x1c, 0x44, 0x7a, 0xfe, 0x5d, 0x28, 0x7a, 0xf8, 0x29, 0xa7, 0xec, - 0x55, 0x91, 0xb2, 0xc8, 0xfc, 0xcb, 0x46, 0x6a, 0x64, 0x1c, 0x7a, 0x11, 0x1a, 0x86, 0x6d, 0xf5, - 0x07, 0x87, 0xba, 0xe3, 0x60, 0x8b, 0x29, 0x52, 0x4d, 0xab, 0x1b, 0xb6, 0xb5, 0xc9, 0x41, 0xe8, - 0x3a, 0x80, 0x8f, 0x87, 0x36, 0x76, 0x82, 0xd8, 0x26, 0x27, 0x20, 0x68, 0x15, 0xe6, 0x0f, 0x3c, - 0xd7, 0xee, 0xfb, 0x87, 0xba, 0x67, 0xf4, 0x2d, 0xac, 0x1b, 0xd8, 0xa3, 0xd4, 0x57, 0xb5, 0x36, - 0x69, 0xd8, 0x23, 0xf0, 0x87, 0x14, 0x8c, 0x6e, 0x43, 0xd9, 0x1f, 0xb8, 0x23, 0x4c, 0x25, 0xad, - 0xb5, 0x71, 0x4d, 0x26, 0x43, 0x5b, 0x7a, 0xa0, 0xef, 0x91, 0x4e, 0x1a, 0xeb, 0xab, 0xfe, 0x75, - 0x89, 0xa9, 0xda, 0x17, 0xdc, 0xc8, 0x25, 0xd4, 0xb1, 0xfc, 0x7c, 0xd4, 0xb1, 0x92, 0x4b, 0x1d, - 0xe7, 0xce, 0x56, 0xc7, 0x0c, 0xd7, 0xce, 0xa3, 0x8e, 0xd5, 0xa9, 0xea, 0x58, 0x93, 0xa9, 0x23, - 0x7a, 0x07, 0xda, 0x2c, 0x80, 0x30, 0x9d, 0x03, 0xb7, 0x6f, 0x99, 0x7e, 0xd0, 0x05, 0x4a, 0xe6, - 0xb5, 0xb4, 0x84, 0x1a, 0xf8, 0xd9, 0x1a, 0x43, 0xec, 0x1c, 0xb8, 0x5a, 0xd3, 0x0c, 0x7f, 0x3e, - 0x34, 0xfd, 0x60, 0x76, 0xad, 0xfe, 0xdb, 0x58, 0xab, 0xbf, 0xe8, 0xd2, 0x13, 0x6b, 0x7e, 0x59, - 0xd0, 0xfc, 0x3f, 0x51, 0xe0, 0x2b, 0x0f, 0x70, 0x10, 0x91, 0x4f, 0x14, 0x19, 0x7f, 0x41, 0xdd, - 0xfc, 0x9f, 0x2b, 0xd0, 0x93, 0xd1, 0x3a, 0x8b, 0xab, 0xff, 0x10, 0x2e, 0x47, 0x38, 0xfa, 0x06, - 0xf6, 0x07, 0x9e, 0x39, 0xa2, 0xdb, 0x48, 0x6d, 0x55, 0x7d, 0xe3, 0x86, 0x4c, 0xf0, 0xd3, 0x14, - 0x2c, 0x45, 0x53, 0x6c, 0x25, 0x66, 0x50, 0x7f, 0xaa, 0xc0, 0x12, 0xb1, 0x8d, 0xdc, 0x98, 0x11, - 0x09, 0xbc, 0x30, 0x5f, 0x45, 0x33, 0x59, 0xc8, 0x98, 0xc9, 0x1c, 0x3c, 0xa6, 0x21, 0x76, 0x9a, - 0x9e, 0x59, 0x78, 0xf7, 0x4d, 0x28, 0x13, 0x05, 0x0c, 0x59, 0xf5, 0x82, 0x8c, 0x55, 0x49, 0x64, - 0xac, 0xb7, 0xea, 0x30, 0x2a, 0x62, 0xbb, 0x3d, 0x83, 0xb8, 0xa5, 0x97, 0x5d, 0x90, 0x2c, 0xfb, - 0x37, 0x15, 0xb8, 0x92, 0x41, 0x38, 0xcb, 0xba, 0xdf, 0x82, 0x0a, 0xf5, 0x46, 0xe1, 0xc2, 0x5f, - 0x92, 0x2e, 0x3c, 0x81, 0x8e, 0x58, 0x1b, 0x8d, 0x8f, 0x51, 0x5d, 0xe8, 0xa4, 0xdb, 0x88, 0x9f, - 0xe4, 0x3e, 0xb2, 0xef, 0xe8, 0x36, 0x63, 0x40, 0x4d, 0xab, 0x73, 0xd8, 0x8e, 0x6e, 0x63, 0xf4, - 0x15, 0xa8, 0x12, 0x95, 0xed, 0x9b, 0x46, 0xb8, 0xfd, 0x73, 0x54, 0x85, 0x0d, 0x1f, 0x5d, 0x03, - 0xa0, 0x4d, 0xba, 0x61, 0x78, 0xcc, 0x85, 0xd6, 0xb4, 0x1a, 0x81, 0xdc, 0x25, 0x00, 0xf5, 0xf7, - 0x14, 0xb8, 0xbe, 0x77, 0xea, 0x0c, 0x76, 0xf0, 0xc9, 0xa6, 0x87, 0xf5, 0x00, 0xc7, 0x46, 0xfb, - 0x53, 0x65, 0x3c, 0x5a, 0x86, 0x7a, 0x42, 0x7f, 0xb9, 0x48, 0x26, 0x41, 0xea, 0x5f, 0x28, 0xd0, - 0x20, 0x5e, 0xe4, 0x7d, 0x1c, 0xe8, 0x44, 0x44, 0xd0, 0xb7, 0xa1, 0x66, 0xb9, 0xba, 0xd1, 0x0f, - 0x4e, 0x47, 0x8c, 0x9a, 0x56, 0x9a, 0x9a, 0xd8, 0xf5, 0x3c, 0x3a, 0x1d, 0x61, 0xad, 0x6a, 0xf1, - 0x5f, 0xb9, 0x28, 0x4a, 0x5b, 0x99, 0xa2, 0xc4, 0x52, 0xbe, 0x00, 0x75, 0x1b, 0x07, 0x9e, 0x39, - 0x60, 0x44, 0x94, 0xe8, 0x56, 0x00, 0x03, 0x11, 0x44, 0xea, 0x4f, 0x2b, 0x70, 0xf9, 0x07, 0x7a, - 0x30, 0x38, 0xdc, 0xb2, 0xc3, 0x28, 0xe6, 0xe2, 0x7c, 0x8c, 0xed, 0x72, 0x21, 0x69, 0x97, 0x9f, - 0x9b, 0xdd, 0x8f, 0x74, 0xb4, 0x2c, 0xd3, 0x51, 0x92, 0x98, 0xaf, 0x3d, 0xe1, 0x62, 0x96, 0xd0, - 0xd1, 0x44, 0xb0, 0x51, 0xb9, 0x48, 0xb0, 0xb1, 0x09, 0x4d, 0xfc, 0x6c, 0x60, 0x8d, 0x89, 0xbc, - 0x52, 0xec, 0x2c, 0x8a, 0xb8, 0x2e, 0xc1, 0x9e, 0x34, 0x10, 0x0d, 0x3e, 0x68, 0x9b, 0xd3, 0xc0, - 0x64, 0xc1, 0xc6, 0x81, 0x4e, 0x43, 0x85, 0xfa, 0xc6, 0xf2, 0x24, 0x59, 0x08, 0x05, 0x88, 0xc9, - 0x03, 0xf9, 0x42, 0x57, 0xa1, 0xc6, 0x43, 0x9b, 0xed, 0xad, 0x6e, 0x8d, 0xb2, 0x2f, 0x06, 0x20, - 0x1d, 0x9a, 0xdc, 0x7a, 0x72, 0x0a, 0x59, 0x00, 0xf1, 0x96, 0x0c, 0x81, 0x7c, 0xb3, 0x93, 0x94, - 0xfb, 0x3c, 0xd0, 0xf1, 0x13, 0x20, 0x92, 0xf9, 0xbb, 0x07, 0x07, 0x96, 0xe9, 0xe0, 0x1d, 0xb6, - 0xc3, 0x75, 0x4a, 0x84, 0x08, 0x24, 0xe1, 0xd0, 0x31, 0xf6, 0x7c, 0xd3, 0x75, 0xba, 0x0d, 0xda, - 0x1e, 0x7e, 0xca, 0xa2, 0x9c, 0xe6, 0x05, 0xa2, 0x9c, 0x3e, 0xcc, 0x67, 0x28, 0x95, 0x44, 0x39, - 0xdf, 0x48, 0x46, 0x39, 0xd3, 0xb7, 0x2a, 0x11, 0x05, 0xfd, 0x4c, 0x81, 0xa5, 0xc7, 0x8e, 0x3f, - 0xde, 0x8f, 0x58, 0xf4, 0xf9, 0xa8, 0x43, 0xda, 0x88, 0x96, 0x32, 0x46, 0x54, 0xfd, 0x87, 0x32, - 0xb4, 0xf9, 0x2a, 0x88, 0xd4, 0x50, 0x93, 0x73, 0x15, 0x6a, 0x91, 0x1f, 0xe5, 0x0c, 0x89, 0x01, - 0x69, 0x1b, 0x56, 0xc8, 0xd8, 0xb0, 0x5c, 0xa4, 0x85, 0x51, 0x51, 0x29, 0x11, 0x15, 0x5d, 0x03, - 0x38, 0xb0, 0xc6, 0xfe, 0x61, 0x3f, 0x30, 0x6d, 0xcc, 0xa3, 0xb2, 0x1a, 0x85, 0x3c, 0x32, 0x6d, - 0x8c, 0xee, 0x42, 0x63, 0xdf, 0x74, 0x2c, 0x77, 0xd8, 0x1f, 0xe9, 0xc1, 0xa1, 0xcf, 0xd3, 0x62, - 0xd9, 0xb6, 0xd0, 0x18, 0xf6, 0x1e, 0xed, 0xab, 0xd5, 0xd9, 0x98, 0x5d, 0x32, 0x04, 0x5d, 0x87, - 0xba, 0x33, 0xb6, 0xfb, 0xee, 0x41, 0xdf, 0x73, 0x4f, 0x7c, 0x9a, 0xfc, 0x16, 0xb5, 0x9a, 0x33, - 0xb6, 0x3f, 0x38, 0xd0, 0xdc, 0x13, 0xe2, 0xc7, 0x6a, 0xc4, 0xa3, 0xf9, 0x96, 0x3b, 0x64, 0x89, - 0xef, 0xf4, 0xf9, 0xe3, 0x01, 0x64, 0xb4, 0x81, 0xad, 0x40, 0xa7, 0xa3, 0x6b, 0xf9, 0x46, 0x47, - 0x03, 0xd0, 0xcb, 0xd0, 0x1a, 0xb8, 0xf6, 0x48, 0xa7, 0x1c, 0xba, 0xef, 0xb9, 0x36, 0x55, 0xc0, - 0xa2, 0x96, 0x82, 0xa2, 0x4d, 0xa8, 0xc7, 0x4a, 0xe0, 0x77, 0xeb, 0x14, 0x8f, 0x2a, 0xd3, 0xd2, - 0x44, 0x28, 0x4f, 0x04, 0x14, 0x22, 0x2d, 0xf0, 0x89, 0x64, 0x84, 0xca, 0xee, 0x9b, 0x1f, 0x63, - 0xae, 0x68, 0x75, 0x0e, 0xdb, 0x33, 0x3f, 0xc6, 0x24, 0x3d, 0x32, 0x1d, 0x1f, 0x7b, 0x41, 0x98, - 0xac, 0x76, 0x9b, 0x54, 0x7c, 0x9a, 0x0c, 0xca, 0x05, 0x1b, 0x6d, 0x41, 0xcb, 0x0f, 0x74, 0x2f, - 0xe8, 0x8f, 0x5c, 0x9f, 0x0a, 0x40, 0xb7, 0x45, 0x65, 0x3b, 0xa5, 0x92, 0xb6, 0x3f, 0x24, 0x82, - 0xbd, 0xcb, 0x3b, 0x69, 0x4d, 0x3a, 0x28, 0xfc, 0x24, 0xb3, 0x50, 0x4e, 0xc4, 0xb3, 0xb4, 0x73, - 0xcd, 0x42, 0x07, 0x85, 0x9f, 0xea, 0xff, 0x14, 0xa0, 0x25, 0x2e, 0x9a, 0x18, 0x13, 0x96, 0x6b, - 0x85, 0x92, 0x1c, 0x7e, 0x12, 0x16, 0x60, 0x47, 0xdf, 0xb7, 0x30, 0x4b, 0xec, 0xa8, 0x20, 0x57, - 0xb5, 0x3a, 0x83, 0xd1, 0x09, 0x88, 0x40, 0x32, 0x56, 0x53, 0xed, 0x29, 0xd2, 0xe5, 0xd7, 0x28, - 0x84, 0x06, 0x20, 0x5d, 0x98, 0x0b, 0x73, 0x42, 0x26, 0xc6, 0xe1, 0x27, 0x69, 0xd9, 0x1f, 0x9b, - 0x14, 0x2b, 0x13, 0xe3, 0xf0, 0x13, 0x6d, 0x41, 0x83, 0x4d, 0x39, 0xd2, 0x3d, 0xdd, 0x0e, 0x85, - 0xf8, 0x45, 0xa9, 0x21, 0x78, 0x0f, 0x9f, 0x3e, 0x21, 0x36, 0x65, 0x57, 0x37, 0x3d, 0x8d, 0x6d, - 0xfa, 0x2e, 0x1d, 0x85, 0x56, 0xa0, 0xc3, 0x66, 0x39, 0x30, 0x2d, 0xcc, 0xd5, 0x61, 0x8e, 0x25, - 0x86, 0x14, 0x7e, 0xdf, 0xb4, 0x30, 0x93, 0xf8, 0x68, 0x09, 0x74, 0x9b, 0xab, 0x4c, 0xe0, 0x29, - 0x84, 0x6e, 0xf2, 0x0d, 0x60, 0xb6, 0xb1, 0x1f, 0x5a, 0x5c, 0xe6, 0x16, 0x18, 0x8d, 0x4f, 0xb8, - 0xd9, 0x25, 0x81, 0xd6, 0xd8, 0x66, 0x2a, 0x03, 0x6c, 0x39, 0xce, 0xd8, 0x26, 0x0a, 0xa3, 0xfe, - 0x63, 0x09, 0x16, 0x88, 0xdd, 0xe0, 0x26, 0x64, 0x06, 0xb7, 0x7f, 0x0d, 0xc0, 0xf0, 0x83, 0xbe, - 0x60, 0xeb, 0x6a, 0x86, 0x1f, 0x70, 0xa7, 0xf0, 0xed, 0xd0, 0x6b, 0x17, 0x27, 0x27, 0x21, 0x29, - 0x3b, 0x96, 0xf5, 0xdc, 0x17, 0xaa, 0xda, 0xdd, 0x80, 0x26, 0xcf, 0xc0, 0x85, 0x74, 0xb1, 0xc1, - 0x80, 0x3b, 0x72, 0x6b, 0x5c, 0x91, 0x56, 0x0f, 0x13, 0xde, 0x7b, 0x6e, 0x36, 0xef, 0x5d, 0x4d, - 0x7b, 0xef, 0xfb, 0xd0, 0x16, 0x15, 0x28, 0xb4, 0x40, 0x53, 0x34, 0xa8, 0x25, 0x68, 0x90, 0x9f, - 0x74, 0xbe, 0x20, 0x3a, 0xdf, 0x1b, 0xd0, 0x74, 0x30, 0x36, 0xfa, 0x81, 0xa7, 0x3b, 0xfe, 0x01, - 0xf6, 0xa8, 0xf3, 0xae, 0x6a, 0x0d, 0x02, 0x7c, 0xc4, 0x61, 0xe8, 0x2d, 0x00, 0xba, 0x46, 0x56, - 0x74, 0x6a, 0x4c, 0x2e, 0x3a, 0x51, 0xa1, 0xa1, 0x45, 0x27, 0xca, 0x14, 0xfa, 0x53, 0xfd, 0xe7, - 0x02, 0x5c, 0xe6, 0xd5, 0x83, 0xd9, 0x05, 0x6a, 0x92, 0xe3, 0x0c, 0x3d, 0x4f, 0xf1, 0x8c, 0x7c, - 0xbc, 0x94, 0x23, 0xb6, 0x2c, 0x4b, 0x62, 0x4b, 0x31, 0x27, 0xad, 0x64, 0x72, 0xd2, 0xa8, 0x1c, - 0x37, 0x97, 0xbf, 0x1c, 0x87, 0x16, 0xa1, 0x4c, 0x13, 0x25, 0xba, 0xe9, 0x35, 0x8d, 0x7d, 0xe4, - 0xda, 0x0e, 0xf5, 0x77, 0x0b, 0xd0, 0xdc, 0xc3, 0xba, 0x37, 0x38, 0x0c, 0xf9, 0xf8, 0x46, 0xb2, - 0x7c, 0xf9, 0xd2, 0x84, 0xf2, 0xa5, 0x30, 0xe4, 0x4b, 0x53, 0xb7, 0x24, 0x08, 0x02, 0x37, 0xd0, - 0x23, 0x2a, 0xfb, 0xce, 0xd8, 0xe6, 0x35, 0xbd, 0x36, 0x6d, 0xe0, 0xa4, 0xee, 0x8c, 0x6d, 0xf5, - 0xbf, 0x14, 0x68, 0x7c, 0x9f, 0x4c, 0x13, 0x32, 0xe6, 0x4e, 0x92, 0x31, 0x2f, 0x4f, 0x60, 0x8c, - 0x46, 0x72, 0x1e, 0x7c, 0x8c, 0xbf, 0x74, 0x25, 0xdd, 0xbf, 0x53, 0xa0, 0x47, 0x32, 0x5e, 0x8d, - 0x19, 0x8c, 0xd9, 0xb5, 0xeb, 0x06, 0x34, 0x8f, 0x85, 0xd8, 0xb2, 0x40, 0x85, 0xb3, 0x71, 0x9c, - 0xcc, 0xd0, 0x35, 0xe8, 0x84, 0x15, 0x56, 0xbe, 0xd8, 0xd0, 0x7e, 0xbf, 0x22, 0xa3, 0x3a, 0x45, - 0x1c, 0xb5, 0x7f, 0x6d, 0x4f, 0x04, 0xaa, 0xbf, 0xa5, 0xc0, 0x82, 0xa4, 0x23, 0xba, 0x02, 0x73, - 0xbc, 0x1a, 0xc0, 0x1d, 0x3d, 0xd3, 0x77, 0x83, 0x6c, 0x4f, 0x5c, 0xcf, 0x32, 0x8d, 0x6c, 0xc0, - 0x6a, 0x90, 0x04, 0x37, 0x4a, 0x7d, 0x8c, 0xcc, 0xfe, 0x18, 0x3e, 0xea, 0x41, 0x95, 0x9b, 0xc1, - 0x30, 0xa7, 0x8c, 0xbe, 0xd5, 0x23, 0x40, 0x0f, 0x70, 0xec, 0x74, 0x66, 0xe1, 0x68, 0x6c, 0x6f, - 0x62, 0x42, 0x93, 0x46, 0xc8, 0x50, 0xff, 0x5d, 0x81, 0x05, 0x01, 0xdb, 0x2c, 0x55, 0x9b, 0xd8, - 0x31, 0x16, 0x2e, 0xe2, 0x18, 0x85, 0xca, 0x44, 0xf1, 0x5c, 0x95, 0x89, 0xeb, 0x00, 0x11, 0xff, - 0x43, 0x8e, 0x26, 0x20, 0xea, 0xdf, 0x28, 0x70, 0xf9, 0x5d, 0xdd, 0x31, 0xdc, 0x83, 0x83, 0xd9, - 0x45, 0x75, 0x13, 0x84, 0x2c, 0x34, 0x6f, 0x6d, 0x4e, 0x4c, 0x5d, 0x5f, 0x85, 0x79, 0x8f, 0x79, - 0x26, 0x43, 0x94, 0xe5, 0xa2, 0xd6, 0x09, 0x1b, 0x22, 0x19, 0xfd, 0xb3, 0x02, 0x20, 0xb2, 0xea, - 0x7b, 0xba, 0xa5, 0x3b, 0x03, 0x7c, 0x71, 0xd2, 0x6f, 0x42, 0x4b, 0x88, 0x3d, 0xa2, 0xb3, 0xf2, - 0x64, 0xf0, 0xe1, 0xa3, 0xf7, 0xa0, 0xb5, 0xcf, 0x50, 0xf5, 0x3d, 0xac, 0xfb, 0xae, 0xc3, 0xb7, - 0x43, 0x5a, 0x86, 0x7b, 0xe4, 0x99, 0xc3, 0x21, 0xf6, 0x36, 0x5d, 0xc7, 0xe0, 0x41, 0xf4, 0x7e, - 0x48, 0x26, 0x19, 0x4a, 0x94, 0x21, 0x0e, 0xc4, 0xa2, 0xcd, 0x89, 0x22, 0x31, 0xca, 0x0a, 0x1f, - 0xeb, 0x56, 0xcc, 0x88, 0xd8, 0x1b, 0x76, 0x58, 0xc3, 0xde, 0xe4, 0x2a, 0xac, 0x24, 0x30, 0x52, - 0xff, 0x52, 0x01, 0x14, 0x65, 0xca, 0xb4, 0xb4, 0x40, 0x35, 0x3a, 0x3d, 0x54, 0x91, 0x38, 0xe5, - 0xab, 0x50, 0x33, 0xc2, 0x91, 0xdc, 0x04, 0xc5, 0x00, 0xea, 0x23, 0x29, 0xd1, 0x7d, 0x22, 0x79, - 0xd8, 0x08, 0x33, 0x51, 0x06, 0x7c, 0x48, 0x61, 0x62, 0x5c, 0x55, 0x4a, 0xc7, 0x55, 0xc9, 0x22, - 0x63, 0x59, 0x28, 0x32, 0xaa, 0x3f, 0x2b, 0x40, 0x87, 0xba, 0x90, 0xcd, 0xb8, 0x5a, 0x94, 0x8b, - 0xe8, 0x1b, 0xd0, 0xe4, 0x77, 0x4d, 0x04, 0xc2, 0x1b, 0x4f, 0x13, 0x93, 0xa1, 0x5b, 0xb0, 0xc8, - 0x3a, 0x79, 0xd8, 0x1f, 0x5b, 0x71, 0x12, 0xc6, 0xb2, 0x10, 0xf4, 0x94, 0xf9, 0x2e, 0xd2, 0x14, - 0x8e, 0x78, 0x0c, 0x97, 0x87, 0x96, 0xbb, 0xaf, 0x5b, 0x7d, 0x71, 0x7b, 0xd8, 0x1e, 0xe6, 0x90, - 0xf8, 0x45, 0x36, 0x7c, 0x2f, 0xb9, 0x87, 0x3e, 0xba, 0x07, 0x4d, 0x1f, 0xe3, 0xa3, 0x38, 0x33, - 0x2b, 0xe7, 0xc9, 0xcc, 0x1a, 0x64, 0x4c, 0x94, 0x98, 0xfd, 0x81, 0x02, 0xed, 0xd4, 0x11, 0x41, - 0xba, 0x8e, 0xa0, 0x64, 0xeb, 0x08, 0x77, 0xa0, 0x4c, 0x2c, 0x15, 0xf3, 0x2d, 0x2d, 0x79, 0x8e, - 0x2b, 0xce, 0xaa, 0xb1, 0x01, 0x68, 0x1d, 0x16, 0x24, 0x57, 0x11, 0xf8, 0xf6, 0xa3, 0xec, 0x4d, - 0x04, 0xf5, 0xe7, 0x25, 0xa8, 0x27, 0x58, 0x31, 0xa5, 0x04, 0xf2, 0x5c, 0x4a, 0xbd, 0x93, 0x8e, - 0x9e, 0x89, 0xc8, 0xd9, 0xd8, 0x66, 0x09, 0x1b, 0xcf, 0x1e, 0x6d, 0x6c, 0xd3, 0x74, 0x2d, 0x99, - 0x89, 0x55, 0x84, 0x4c, 0x2c, 0x95, 0xab, 0xce, 0x9d, 0x91, 0xab, 0x56, 0xc5, 0x5c, 0x55, 0x50, - 0xa1, 0x5a, 0x5a, 0x85, 0xf2, 0x56, 0x25, 0x6e, 0xc1, 0xc2, 0x80, 0x95, 0xd2, 0xef, 0x9d, 0x6e, - 0x46, 0x4d, 0x3c, 0x28, 0x95, 0x35, 0xa1, 0xfb, 0x71, 0xbd, 0x91, 0xed, 0x32, 0xcb, 0x16, 0xe4, - 0xa9, 0x30, 0xdf, 0x1b, 0xb6, 0xc9, 0xa1, 0x65, 0xa6, 0x5f, 0xe9, 0x7a, 0x48, 0xf3, 0x42, 0xf5, - 0x90, 0x17, 0xa0, 0x1e, 0x46, 0x2a, 0x44, 0xd3, 0x5b, 0xcc, 0xe8, 0x85, 0x66, 0xc0, 0xf0, 0x05, - 0x3b, 0xd0, 0x16, 0x0f, 0x1b, 0xd2, 0x85, 0x84, 0x4e, 0xb6, 0x90, 0x70, 0x05, 0xe6, 0x4c, 0xbf, - 0x7f, 0xa0, 0x1f, 0xe1, 0xee, 0x3c, 0x6d, 0xad, 0x98, 0xfe, 0x7d, 0xfd, 0x08, 0xab, 0xff, 0x5a, - 0x84, 0x56, 0xec, 0x60, 0x73, 0x5b, 0x90, 0x3c, 0xd7, 0x71, 0x76, 0xa0, 0x13, 0xc7, 0x3d, 0x94, - 0xc3, 0x67, 0x26, 0xcf, 0xe9, 0x13, 0xbc, 0xf6, 0x28, 0xa5, 0xaf, 0x82, 0xbb, 0x2f, 0x9d, 0xcb, - 0xdd, 0xcf, 0x78, 0x50, 0x7f, 0x1b, 0x96, 0x22, 0xdf, 0x2b, 0x2c, 0x9b, 0x25, 0x58, 0x8b, 0x61, - 0xe3, 0x6e, 0x72, 0xf9, 0x13, 0x4c, 0xc0, 0xdc, 0x24, 0x13, 0x90, 0x16, 0x81, 0x6a, 0x46, 0x04, - 0xb2, 0xf7, 0x05, 0x6a, 0x92, 0xfb, 0x02, 0xea, 0x63, 0x58, 0xa0, 0xb5, 0x5f, 0x7f, 0xe0, 0x99, - 0xfb, 0x38, 0x4a, 0x01, 0xf2, 0x6c, 0x6b, 0x0f, 0xaa, 0xa9, 0x2c, 0x22, 0xfa, 0x56, 0x7f, 0xa2, - 0xc0, 0xe5, 0xec, 0xbc, 0x54, 0x62, 0x62, 0x43, 0xa2, 0x08, 0x86, 0xe4, 0x97, 0x61, 0x21, 0x11, - 0x51, 0x0a, 0x33, 0x4f, 0x88, 0xc0, 0x25, 0x84, 0x6b, 0x28, 0x9e, 0x23, 0x84, 0xa9, 0x3f, 0x57, - 0xa2, 0x12, 0x3a, 0x81, 0x0d, 0xe9, 0xf9, 0x04, 0xf1, 0x6b, 0xae, 0x63, 0x99, 0x4e, 0x54, 0x29, - 0xe1, 0x6b, 0x64, 0x40, 0x5e, 0x29, 0x79, 0x17, 0xda, 0xbc, 0x53, 0xe4, 0x9e, 0x72, 0x06, 0x64, - 0x2d, 0x36, 0x2e, 0x72, 0x4c, 0x37, 0xa1, 0xc5, 0x0f, 0x0e, 0x42, 0x7c, 0x45, 0xd9, 0x71, 0xc2, - 0xf7, 0xa0, 0x13, 0x76, 0x3b, 0xaf, 0x43, 0x6c, 0xf3, 0x81, 0x51, 0x60, 0xf7, 0x1b, 0x0a, 0x74, - 0x45, 0xf7, 0x98, 0x58, 0xfe, 0xf9, 0xc3, 0xbb, 0x37, 0xc5, 0xe3, 0xe2, 0x9b, 0x67, 0xd0, 0x13, - 0xe3, 0x09, 0x0f, 0x8d, 0x7f, 0xa7, 0x40, 0xcf, 0xfe, 0x49, 0xaa, 0xb7, 0x65, 0xfa, 0x81, 0x67, - 0xee, 0x8f, 0x67, 0x3b, 0xc0, 0xd4, 0xa1, 0x3e, 0x38, 0xc4, 0x83, 0xa3, 0x91, 0x6b, 0xc6, 0xbb, - 0xf2, 0xb6, 0x8c, 0xa6, 0xc9, 0x68, 0xd7, 0x36, 0xe3, 0x19, 0xd8, 0x09, 0x50, 0x72, 0xce, 0xde, - 0x0f, 0xa1, 0x93, 0xee, 0x90, 0x3c, 0x78, 0xa9, 0xb1, 0x83, 0x97, 0xdb, 0xe2, 0xc1, 0xcb, 0x94, - 0x48, 0x23, 0x71, 0xee, 0xf2, 0x57, 0x05, 0xf8, 0xaa, 0x94, 0xb6, 0x59, 0xb2, 0xa4, 0x49, 0x75, - 0xa4, 0x7b, 0x50, 0x4d, 0x25, 0xb5, 0x2f, 0x9f, 0xb1, 0x7f, 0xbc, 0x96, 0xca, 0x6a, 0x7a, 0x7e, - 0x1c, 0x5b, 0xc5, 0x0a, 0x5f, 0x9a, 0x3c, 0x07, 0xd7, 0x3b, 0x61, 0x8e, 0x70, 0x1c, 0xba, 0x0b, - 0x0d, 0x56, 0x30, 0xe8, 0x1f, 0x9b, 0xf8, 0x24, 0x3c, 0xd6, 0xbc, 0x2e, 0x35, 0xcd, 0xb4, 0xdf, - 0x13, 0x13, 0x9f, 0x68, 0x75, 0x2b, 0xfa, 0xed, 0xab, 0xff, 0x5d, 0x04, 0x88, 0xdb, 0x48, 0x76, - 0x16, 0xeb, 0x3c, 0x57, 0xe2, 0x04, 0x84, 0xc4, 0x12, 0x62, 0xe4, 0x1a, 0x7e, 0x22, 0x2d, 0x3e, - 0x56, 0x30, 0x4c, 0x3f, 0xe0, 0x7c, 0x59, 0x3f, 0x9b, 0x96, 0x90, 0x45, 0x64, 0xcb, 0xb8, 0xcc, - 0xf8, 0x31, 0x04, 0xbd, 0x0e, 0x68, 0xe8, 0xb9, 0x27, 0xa6, 0x33, 0x4c, 0xe6, 0x1b, 0x2c, 0x2d, - 0x99, 0xe7, 0x2d, 0x89, 0x84, 0xe3, 0x47, 0xd0, 0x49, 0x75, 0x0f, 0x59, 0x72, 0x7b, 0x0a, 0x19, - 0x0f, 0x84, 0xb9, 0xb8, 0xf8, 0xb6, 0x45, 0x0c, 0x7e, 0xaf, 0x0f, 0x9d, 0x34, 0xbd, 0x92, 0xb3, - 0xc3, 0x6f, 0x8a, 0x22, 0x7c, 0x96, 0xa5, 0x21, 0xd3, 0x24, 0x84, 0xb8, 0xa7, 0xc3, 0xa2, 0x8c, - 0x12, 0x09, 0x92, 0x0b, 0xeb, 0xc9, 0xdb, 0x51, 0xb0, 0x4b, 0x39, 0x3c, 0xc9, 0x7f, 0x24, 0x6a, - 0xc1, 0x05, 0xa1, 0x16, 0xac, 0xfe, 0xbd, 0x02, 0x28, 0x2b, 0xd8, 0xa8, 0x05, 0x85, 0x68, 0x92, - 0xc2, 0xf6, 0x56, 0x4a, 0x90, 0x0a, 0x19, 0x41, 0xba, 0x0a, 0xb5, 0xc8, 0x9f, 0x73, 0xe3, 0x1d, - 0x03, 0x92, 0x62, 0x56, 0x12, 0xc5, 0x2c, 0x41, 0x58, 0x59, 0x2c, 0x52, 0xdf, 0x82, 0x45, 0x4b, - 0xf7, 0x83, 0x3e, 0xab, 0x85, 0x07, 0xa6, 0x8d, 0xfd, 0x40, 0xb7, 0x47, 0x34, 0x58, 0x2e, 0x69, - 0x88, 0xb4, 0x6d, 0x91, 0xa6, 0x47, 0x61, 0x8b, 0x7a, 0x08, 0x28, 0xab, 0x5e, 0x49, 0xdc, 0x8a, - 0x88, 0x7b, 0xda, 0x9a, 0x12, 0xb4, 0x15, 0x45, 0xa6, 0xfd, 0x51, 0x11, 0x50, 0x1c, 0xe3, 0x44, - 0xa7, 0xad, 0x79, 0x02, 0x83, 0x75, 0x58, 0xc8, 0x46, 0x40, 0x61, 0xd8, 0x87, 0x32, 0xf1, 0x8f, - 0x2c, 0x56, 0x29, 0xca, 0xee, 0x36, 0xbe, 0x11, 0x19, 0x44, 0x16, 0xd0, 0x5d, 0x9f, 0x58, 0xaa, - 0x17, 0x6d, 0xe2, 0x0f, 0xd3, 0x77, 0x22, 0x99, 0x86, 0xdd, 0x91, 0x1a, 0xaf, 0xcc, 0x92, 0xa7, - 0x5e, 0x88, 0x14, 0x42, 0xcd, 0xca, 0x79, 0x42, 0xcd, 0xd9, 0x6f, 0x30, 0xfe, 0x5b, 0x01, 0xe6, - 0x23, 0x46, 0x9e, 0x6b, 0x93, 0xa6, 0x1f, 0x8c, 0x7f, 0xca, 0xbb, 0xf2, 0x91, 0x7c, 0x57, 0xbe, - 0x75, 0x66, 0xb8, 0x9f, 0x77, 0x53, 0x66, 0xe7, 0xec, 0xc7, 0x30, 0xc7, 0x0b, 0xb7, 0x19, 0x43, - 0x91, 0x27, 0xa1, 0x5e, 0x84, 0x32, 0xb1, 0x4b, 0x61, 0xd5, 0x8d, 0x7d, 0x30, 0x96, 0x26, 0x6f, - 0xc8, 0x72, 0x5b, 0xd1, 0x14, 0x2e, 0xc8, 0xaa, 0xff, 0xa9, 0x00, 0xec, 0x9d, 0x3a, 0x83, 0xbb, - 0x4c, 0x49, 0x6f, 0x41, 0x69, 0xda, 0x7d, 0x2a, 0xd2, 0x9b, 0xca, 0x16, 0xed, 0x99, 0x63, 0x73, - 0x85, 0x92, 0x41, 0x31, 0x5d, 0x32, 0x98, 0x94, 0xec, 0x4f, 0x36, 0x65, 0xdf, 0x82, 0x12, 0x09, - 0xf4, 0xf8, 0x75, 0xa3, 0x5c, 0x07, 0x9e, 0x74, 0x80, 0xfa, 0x49, 0x01, 0xae, 0x10, 0xea, 0x9f, - 0x4f, 0x54, 0x98, 0x67, 0x6b, 0x12, 0xd6, 0xb2, 0x28, 0x5a, 0xcb, 0x3b, 0x30, 0xc7, 0xd2, 0xfd, - 0x30, 0xbe, 0xb9, 0x3e, 0x89, 0xd7, 0x6c, 0x67, 0xb4, 0xb0, 0xfb, 0xac, 0x39, 0xa3, 0x70, 0xd8, - 0x5a, 0x99, 0xed, 0xb0, 0x75, 0x2e, 0x5d, 0x14, 0x4c, 0x6c, 0x5a, 0x55, 0xb4, 0xf1, 0x8f, 0xa1, - 0xa9, 0x25, 0x05, 0x0f, 0x21, 0x28, 0x25, 0xee, 0x2f, 0xd2, 0xdf, 0x34, 0xcd, 0xd3, 0x47, 0xfa, - 0xc0, 0x0c, 0x4e, 0x29, 0x3b, 0xcb, 0x5a, 0xf4, 0x2d, 0x97, 0x72, 0xf5, 0x7f, 0x15, 0xb8, 0x1c, - 0x1e, 0xea, 0x71, 0x1d, 0xba, 0xf8, 0x8e, 0x6e, 0xc0, 0x12, 0x57, 0x98, 0x94, 0xe6, 0xb0, 0x60, - 0x6e, 0x81, 0xc1, 0xc4, 0x65, 0x6c, 0xc0, 0x52, 0xa0, 0x7b, 0x43, 0x1c, 0xa4, 0xc7, 0xb0, 0xfd, - 0x5e, 0x60, 0x8d, 0xe2, 0x98, 0x3c, 0x87, 0xaa, 0x2f, 0xb0, 0x0b, 0x39, 0x9c, 0xb5, 0x5c, 0x05, - 0xc0, 0x19, 0xdb, 0x7c, 0x95, 0xea, 0x09, 0x5c, 0x65, 0x37, 0x88, 0xf7, 0x45, 0x8a, 0x66, 0xaa, - 0xa9, 0x4b, 0xd7, 0x9d, 0xb2, 0x18, 0x7f, 0xa8, 0xc0, 0xb5, 0x09, 0x98, 0x67, 0xc9, 0x26, 0x1e, - 0x4a, 0xb1, 0x4f, 0xc8, 0xfd, 0x04, 0xbc, 0xec, 0x26, 0x9b, 0x48, 0xe4, 0x27, 0x25, 0x98, 0xcf, - 0x74, 0x3a, 0xb7, 0xcc, 0xbd, 0x06, 0x88, 0x6c, 0x42, 0xf4, 0x5a, 0x8e, 0xa6, 0xd3, 0xdc, 0x35, - 0x75, 0x9c, 0xb1, 0x1d, 0xbd, 0x94, 0x23, 0x19, 0x35, 0x32, 0x59, 0x6f, 0x56, 0x51, 0x8f, 0x76, - 0xae, 0x34, 0xf9, 0x51, 0x44, 0x86, 0xc0, 0xb5, 0x9d, 0xb1, 0xcd, 0x8a, 0xef, 0x7c, 0x97, 0x99, - 0xbb, 0x21, 0xa8, 0x04, 0x30, 0x3a, 0x80, 0x79, 0x7a, 0x5d, 0x6b, 0x1c, 0x0c, 0x5d, 0x12, 0xd0, - 0x53, 0xba, 0x98, 0x53, 0xfb, 0x4e, 0x6e, 0x4c, 0x1f, 0xf0, 0xd1, 0x84, 0x78, 0x1e, 0xd3, 0x3b, - 0x22, 0x34, 0xc4, 0x63, 0x3a, 0x03, 0xd7, 0x8e, 0xf0, 0x54, 0xce, 0x89, 0x67, 0x9b, 0x8f, 0x16, - 0xf1, 0x24, 0xa1, 0xbd, 0x4d, 0x58, 0x92, 0x2e, 0x7d, 0x9a, 0x1b, 0x2d, 0x27, 0xf3, 0x83, 0x7b, - 0xb0, 0x28, 0x5b, 0xd5, 0x05, 0xe6, 0xc8, 0x50, 0x7c, 0x9e, 0x39, 0xd4, 0x3f, 0x2d, 0x40, 0x73, - 0x0b, 0x5b, 0x38, 0xc0, 0x9f, 0xee, 0x99, 0x67, 0xe6, 0x00, 0xb7, 0x98, 0x3d, 0xc0, 0xcd, 0x9c, - 0x46, 0x97, 0x24, 0xa7, 0xd1, 0xd7, 0xa2, 0x43, 0x78, 0x32, 0x4b, 0x59, 0xf4, 0xd0, 0x06, 0x7a, - 0x13, 0x1a, 0x23, 0xcf, 0xb4, 0x75, 0xef, 0xb4, 0x7f, 0x84, 0x4f, 0x7d, 0xee, 0x34, 0xba, 0x52, - 0xb7, 0xb3, 0xbd, 0xe5, 0x6b, 0x75, 0xde, 0xfb, 0x3d, 0x7c, 0x4a, 0x0f, 0xf8, 0xa3, 0x64, 0x83, - 0x5d, 0xc5, 0x2a, 0x69, 0x09, 0xc8, 0xea, 0x32, 0xd4, 0xa2, 0x1b, 0x2f, 0xa8, 0x0a, 0xa5, 0xfb, - 0x63, 0xcb, 0xea, 0x5c, 0x42, 0x35, 0x28, 0xd3, 0x74, 0xa4, 0xa3, 0xac, 0xfe, 0x12, 0xd4, 0xa2, - 0x53, 0x7b, 0x54, 0x87, 0xb9, 0xc7, 0xce, 0x7b, 0x8e, 0x7b, 0xe2, 0x74, 0x2e, 0xa1, 0x39, 0x28, - 0xde, 0xb5, 0xac, 0x8e, 0x82, 0x9a, 0x50, 0xdb, 0x0b, 0x3c, 0xac, 0x93, 0x3d, 0xeb, 0x14, 0x50, - 0x0b, 0xe0, 0x5d, 0xd3, 0x0f, 0x5c, 0xcf, 0x1c, 0xe8, 0x56, 0xa7, 0xb8, 0xfa, 0x31, 0xb4, 0xc4, - 0x22, 0x2e, 0x6a, 0x40, 0x75, 0xc7, 0x0d, 0xde, 0x79, 0x66, 0xfa, 0x41, 0xe7, 0x12, 0xe9, 0xbf, - 0xe3, 0x06, 0xbb, 0x1e, 0xf6, 0xb1, 0x13, 0x74, 0x14, 0x04, 0x50, 0xf9, 0xc0, 0xd9, 0x32, 0xfd, - 0xa3, 0x4e, 0x01, 0x2d, 0xf0, 0xf3, 0x19, 0xdd, 0xda, 0xe6, 0x95, 0xd1, 0x4e, 0x91, 0x0c, 0x8f, - 0xbe, 0x4a, 0xa8, 0x03, 0x8d, 0xa8, 0xcb, 0x83, 0xdd, 0xc7, 0x9d, 0x32, 0xa1, 0x9e, 0xfd, 0xac, - 0xac, 0x1a, 0xd0, 0x49, 0x9f, 0x2b, 0x92, 0x39, 0xd9, 0x22, 0x22, 0x50, 0xe7, 0x12, 0x59, 0x19, - 0x3f, 0xd8, 0xed, 0x28, 0xa8, 0x0d, 0xf5, 0xc4, 0x31, 0x69, 0xa7, 0x40, 0x00, 0x0f, 0xbc, 0xd1, - 0x80, 0x0b, 0x14, 0x23, 0x81, 0x48, 0xe7, 0x16, 0xe1, 0x44, 0x69, 0xf5, 0x1e, 0x54, 0xc3, 0x90, - 0x9f, 0x74, 0xe5, 0x2c, 0x22, 0x9f, 0x9d, 0x4b, 0x68, 0x1e, 0x9a, 0xc2, 0xf3, 0xab, 0x8e, 0x82, - 0x10, 0xb4, 0xc4, 0x07, 0x92, 0x9d, 0xc2, 0xea, 0x06, 0x40, 0x1c, 0x3a, 0x13, 0x72, 0xb6, 0x9d, - 0x63, 0xdd, 0x32, 0x0d, 0x46, 0x1b, 0x69, 0x22, 0xdc, 0xa5, 0xdc, 0x61, 0x8a, 0xda, 0x29, 0xac, - 0xae, 0x42, 0x35, 0x0c, 0x07, 0x09, 0x5c, 0xc3, 0xb6, 0x7b, 0x8c, 0xd9, 0xce, 0xec, 0x61, 0xc2, - 0xca, 0x1a, 0x94, 0xef, 0xda, 0xd8, 0x31, 0x3a, 0x85, 0x8d, 0xff, 0x58, 0x00, 0x60, 0xa7, 0x82, - 0xae, 0xeb, 0x19, 0xc8, 0xa2, 0xb7, 0x03, 0x36, 0x5d, 0x7b, 0xe4, 0x3a, 0xe1, 0x91, 0x85, 0x8f, - 0xd6, 0x52, 0xa9, 0x3a, 0xfb, 0xc8, 0x76, 0xe4, 0x8c, 0xe8, 0xbd, 0x24, 0xed, 0x9f, 0xea, 0xac, - 0x5e, 0x42, 0x36, 0xc5, 0x46, 0x92, 0xdb, 0x47, 0xe6, 0xe0, 0x28, 0x3a, 0x4a, 0x9c, 0xfc, 0x4a, - 0x31, 0xd5, 0x35, 0xc4, 0x77, 0x43, 0x8a, 0x6f, 0x2f, 0xf0, 0x4c, 0x67, 0x18, 0xfa, 0x3f, 0xf5, - 0x12, 0x7a, 0x9a, 0x7a, 0x23, 0x19, 0x22, 0xdc, 0xc8, 0xf3, 0x2c, 0xf2, 0x62, 0x28, 0x2d, 0x68, - 0xa7, 0x1e, 0xa3, 0xa3, 0x55, 0xf9, 0x63, 0x13, 0xd9, 0xc3, 0xf9, 0xde, 0xab, 0xb9, 0xfa, 0x46, - 0xd8, 0x4c, 0x68, 0x89, 0xaf, 0xa8, 0xd1, 0xd7, 0x26, 0x4d, 0x90, 0x79, 0xee, 0xd6, 0x5b, 0xcd, - 0xd3, 0x35, 0x42, 0xf5, 0x21, 0x93, 0xd5, 0x69, 0xa8, 0xa4, 0x2f, 0x0c, 0x7b, 0x67, 0x85, 0x1e, - 0xea, 0x25, 0xf4, 0x63, 0x12, 0x25, 0xa4, 0x1e, 0xe5, 0xa1, 0xd7, 0xe4, 0x9e, 0x4d, 0xfe, 0x76, - 0x6f, 0x1a, 0x86, 0x0f, 0xd3, 0x9a, 0x36, 0x99, 0xfa, 0xcc, 0x6b, 0xdf, 0xfc, 0xd4, 0x27, 0xa6, - 0x3f, 0x8b, 0xfa, 0x73, 0x63, 0xb0, 0x58, 0xc2, 0x24, 0x79, 0x0e, 0x94, 0x16, 0xe5, 0x38, 0x5f, - 0x99, 0xfc, 0x76, 0x68, 0x1a, 0xb6, 0x31, 0x55, 0xd2, 0xf4, 0x71, 0xf8, 0xeb, 0x13, 0x0a, 0xed, - 0xf2, 0x77, 0x88, 0xbd, 0xb5, 0xbc, 0xdd, 0x93, 0xb2, 0x2c, 0x3e, 0x75, 0x93, 0x6f, 0x91, 0xf4, - 0x79, 0x9e, 0x5c, 0x96, 0xe5, 0x2f, 0xe7, 0xd4, 0x4b, 0xe8, 0x91, 0x60, 0xd7, 0xd1, 0xcb, 0x93, - 0x44, 0x41, 0xbc, 0x1f, 0x33, 0x8d, 0x6f, 0xbf, 0x0a, 0x88, 0x69, 0xaa, 0x73, 0x60, 0x0e, 0xc7, - 0x9e, 0xce, 0xc4, 0x78, 0x92, 0x71, 0xcb, 0x76, 0x0d, 0xd1, 0x7c, 0xfd, 0x1c, 0x23, 0xa2, 0x25, - 0xf5, 0x01, 0x1e, 0xe0, 0xe0, 0x7d, 0xfa, 0xe6, 0xc9, 0x4f, 0xaf, 0x28, 0xb6, 0xdf, 0xbc, 0x43, - 0x88, 0xea, 0x95, 0xa9, 0xfd, 0x22, 0x04, 0xfb, 0x50, 0x7f, 0x40, 0x32, 0x28, 0x1a, 0x15, 0xfa, - 0x68, 0xe2, 0xc8, 0xb0, 0x47, 0x88, 0x62, 0x65, 0x7a, 0xc7, 0xa4, 0xf1, 0x4c, 0x3d, 0xfb, 0x43, - 0x13, 0x37, 0x36, 0xfb, 0x18, 0x51, 0x6e, 0x3c, 0x27, 0xbc, 0x23, 0x64, 0x2b, 0xa2, 0x87, 0x3d, - 0xef, 0x62, 0xdd, 0x0a, 0x0e, 0x27, 0xac, 0x28, 0xd1, 0xe3, 0xec, 0x15, 0x09, 0x1d, 0x23, 0x1c, - 0x18, 0x16, 0x98, 0x16, 0x8a, 0xa9, 0xe7, 0xba, 0x7c, 0x8a, 0x6c, 0xcf, 0x9c, 0xa2, 0xa7, 0xc3, - 0xfc, 0x96, 0xe7, 0x8e, 0x44, 0x24, 0xaf, 0x4b, 0x91, 0x64, 0xfa, 0xe5, 0x44, 0xf1, 0x03, 0x68, - 0x84, 0x19, 0x3e, 0xcd, 0x49, 0xe4, 0x5c, 0x48, 0x76, 0xc9, 0x39, 0xf1, 0x47, 0xd0, 0x4e, 0x95, - 0x0e, 0xe4, 0x9b, 0x2e, 0xaf, 0x2f, 0x4c, 0x9b, 0xfd, 0x04, 0x10, 0x7d, 0xcb, 0x29, 0x3e, 0x47, - 0x97, 0xc7, 0x37, 0xd9, 0x8e, 0x21, 0x92, 0xf5, 0xdc, 0xfd, 0xa3, 0x9d, 0xff, 0x35, 0x58, 0x92, - 0xa6, 0xe7, 0x69, 0x83, 0xc0, 0x2f, 0xc4, 0x9e, 0x51, 0x43, 0x48, 0x1b, 0x84, 0x33, 0x47, 0x84, - 0xf8, 0x37, 0x7e, 0x7b, 0x1e, 0x6a, 0x34, 0xce, 0xa3, 0xbb, 0xf5, 0x8b, 0x30, 0xef, 0xf9, 0x86, - 0x79, 0x1f, 0x41, 0x3b, 0xf5, 0xc6, 0x50, 0x2e, 0xb4, 0xf2, 0x87, 0x88, 0x39, 0xa2, 0x15, 0xf1, - 0x79, 0x9e, 0xdc, 0x15, 0x4a, 0x9f, 0xf0, 0x4d, 0x9b, 0xfb, 0x09, 0x7b, 0xbf, 0x1b, 0xdd, 0x52, - 0x78, 0x65, 0x62, 0xf1, 0x5e, 0xbc, 0xd8, 0xfa, 0xf9, 0x47, 0x41, 0x5f, 0xee, 0x08, 0xf4, 0x23, - 0x68, 0xa7, 0x9e, 0x8e, 0xc8, 0x25, 0x46, 0xfe, 0xbe, 0x64, 0xda, 0xec, 0x9f, 0x61, 0xf0, 0x64, - 0xc0, 0x82, 0xe4, 0xa6, 0x3e, 0x5a, 0x9b, 0x14, 0x88, 0xca, 0xaf, 0xf4, 0x4f, 0x5f, 0x50, 0x53, - 0x50, 0xd3, 0xb4, 0xbf, 0x89, 0x89, 0x4c, 0xff, 0x8f, 0x4d, 0xef, 0xb5, 0x7c, 0x7f, 0x7a, 0x13, - 0x2d, 0x68, 0x0f, 0x2a, 0xec, 0x41, 0x09, 0x7a, 0x51, 0x7e, 0x88, 0x91, 0x78, 0x6c, 0xd2, 0x9b, - 0xf6, 0x24, 0xc5, 0x1f, 0x5b, 0x01, 0xa1, 0xff, 0x57, 0xa0, 0xc5, 0x40, 0x11, 0x83, 0x9e, 0xe3, - 0xe4, 0x7b, 0x50, 0xa6, 0xa6, 0x1d, 0x49, 0x8f, 0x0c, 0x92, 0xcf, 0x46, 0x7a, 0xd3, 0x5f, 0x8a, - 0xc4, 0x14, 0x37, 0xbf, 0xcf, 0xfe, 0x7e, 0x8c, 0x13, 0xfc, 0x3c, 0x27, 0xff, 0xff, 0x1d, 0x1b, - 0x3f, 0xa3, 0x8f, 0x1e, 0xd2, 0xd7, 0x7a, 0xd0, 0xda, 0xf9, 0xee, 0x26, 0xf5, 0xd6, 0x73, 0xf7, - 0x8f, 0x30, 0xff, 0x08, 0x3a, 0xe9, 0xa3, 0x34, 0xf4, 0xea, 0x24, 0x4d, 0x94, 0xe1, 0x9c, 0xa2, - 0x86, 0xdf, 0x83, 0x0a, 0xab, 0xa1, 0xca, 0xc5, 0x57, 0xa8, 0xaf, 0x4e, 0x99, 0xeb, 0xde, 0x37, - 0x3e, 0xdc, 0x18, 0x9a, 0xc1, 0xe1, 0x78, 0x9f, 0xb4, 0xac, 0xb3, 0xae, 0xaf, 0x9b, 0x2e, 0xff, - 0xb5, 0x1e, 0xee, 0xe5, 0x3a, 0x1d, 0xbd, 0x4e, 0x11, 0x8c, 0xf6, 0xf7, 0x2b, 0xf4, 0xf3, 0xf6, - 0xff, 0x05, 0x00, 0x00, 0xff, 0xff, 0xaf, 0x53, 0x45, 0xc6, 0xff, 0x50, 0x00, 0x00, + // 4694 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xec, 0x3c, 0x4b, 0x6c, 0x1c, 0x47, + 0x76, 0xea, 0xf9, 0x71, 0xe6, 0xcd, 0xaf, 0x59, 0x24, 0xa5, 0xd9, 0x59, 0x49, 0xa6, 0x5b, 0xb6, + 0xcc, 0xa5, 0x6d, 0x52, 0x4b, 0xed, 0x7a, 0xb5, 0x6b, 0x1b, 0x8e, 0x44, 0x5a, 0x32, 0xd7, 0x32, + 0xcd, 0x6d, 0x4a, 0xda, 0xc0, 0xf1, 0xee, 0x6c, 0x73, 0xba, 0x38, 0x6c, 0xb0, 0x3f, 0xa3, 0xee, + 0x1e, 0x52, 0x74, 0x80, 0x9c, 0x72, 0xc9, 0x22, 0x9b, 0x04, 0xc8, 0x21, 0x39, 0x04, 0x39, 0x24, + 0x08, 0xb0, 0x09, 0x92, 0x4b, 0x90, 0x00, 0x39, 0xe4, 0x90, 0x5b, 0x82, 0x04, 0xf9, 0xdc, 0x72, + 0x0e, 0x90, 0xdc, 0x02, 0xe4, 0xb4, 0x08, 0x7c, 0x0b, 0xea, 0xd3, 0x9f, 0xea, 0xae, 0xe1, 0x0c, + 0x39, 0xf2, 0xda, 0x0e, 0xf6, 0x36, 0xfd, 0xea, 0xf3, 0x5e, 0xd5, 0xfb, 0xbf, 0xaa, 0x1a, 0x98, + 0x7f, 0x3a, 0xc2, 0xfe, 0x69, 0xaf, 0xef, 0x79, 0xbe, 0xb9, 0x36, 0xf4, 0xbd, 0xd0, 0x43, 0xc8, + 0xb1, 0xec, 0xe3, 0x51, 0xc0, 0xbe, 0xd6, 0x68, 0x7b, 0xb7, 0xd1, 0xf7, 0x1c, 0xc7, 0x73, 0x19, + 0xac, 0xdb, 0x48, 0xf7, 0xe8, 0xb6, 0x2c, 0x37, 0xc4, 0xbe, 0x6b, 0xd8, 0x51, 0x6b, 0xd0, 0x3f, + 0xc4, 0x8e, 0xc1, 0xbf, 0x6a, 0x4e, 0x30, 0xe0, 0x3f, 0x55, 0xd3, 0x08, 0x8d, 0x34, 0xaa, 0xee, + 0xbc, 0xe5, 0x9a, 0xf8, 0x59, 0x1a, 0xa4, 0xfd, 0xba, 0x02, 0x97, 0xf7, 0x0e, 0xbd, 0x93, 0x4d, + 0xcf, 0xb6, 0x71, 0x3f, 0xb4, 0x3c, 0x37, 0xd0, 0xf1, 0xd3, 0x11, 0x0e, 0x42, 0x74, 0x0b, 0x4a, + 0xfb, 0x46, 0x80, 0x3b, 0xca, 0xb2, 0xb2, 0x52, 0xdf, 0xb8, 0xba, 0x26, 0xd0, 0xc9, 0x09, 0xfc, + 0x20, 0x18, 0xdc, 0x33, 0x02, 0xac, 0xd3, 0x9e, 0x08, 0x41, 0xc9, 0xdc, 0xdf, 0xde, 0xea, 0x14, + 0x96, 0x95, 0x95, 0xa2, 0x4e, 0x7f, 0xa3, 0x97, 0xa0, 0xd9, 0x8f, 0xe7, 0xde, 0xde, 0x0a, 0x3a, + 0xc5, 0xe5, 0xe2, 0x4a, 0x51, 0x17, 0x81, 0xda, 0x8f, 0x0b, 0x70, 0x25, 0x47, 0x46, 0x30, 0xf4, + 0xdc, 0x00, 0xa3, 0xdb, 0x50, 0x09, 0x42, 0x23, 0x1c, 0x05, 0x9c, 0x92, 0xaf, 0x4a, 0x29, 0xd9, + 0xa3, 0x5d, 0x74, 0xde, 0x35, 0x8f, 0xb6, 0x20, 0x41, 0x8b, 0xbe, 0x0e, 0x8b, 0x96, 0xfb, 0x01, + 0x76, 0x3c, 0xff, 0xb4, 0x37, 0xc4, 0x7e, 0x1f, 0xbb, 0xa1, 0x31, 0xc0, 0x11, 0x8d, 0x0b, 0x51, + 0xdb, 0x6e, 0xd2, 0x84, 0xde, 0x80, 0x2b, 0x8c, 0x87, 0x01, 0xf6, 0x8f, 0xad, 0x3e, 0xee, 0x19, + 0xc7, 0x86, 0x65, 0x1b, 0xfb, 0x36, 0xee, 0x94, 0x96, 0x8b, 0x2b, 0x55, 0x7d, 0x89, 0x36, 0xef, + 0xb1, 0xd6, 0xbb, 0x51, 0x23, 0xfa, 0x1a, 0xa8, 0x3e, 0x3e, 0xf0, 0x71, 0x70, 0xd8, 0x1b, 0xfa, + 0xde, 0xc0, 0xc7, 0x41, 0xd0, 0x29, 0x53, 0x34, 0x6d, 0x0e, 0xdf, 0xe5, 0x60, 0xed, 0x4f, 0x14, + 0x58, 0x22, 0x9b, 0xb1, 0x6b, 0xf8, 0xa1, 0xf5, 0x19, 0xb0, 0x44, 0x83, 0x46, 0x7a, 0x1b, 0x3a, + 0x45, 0xda, 0x26, 0xc0, 0x48, 0x9f, 0x61, 0x84, 0x9e, 0x6c, 0x5f, 0x89, 0x92, 0x2a, 0xc0, 0xb4, + 0x7f, 0xe5, 0xb2, 0x93, 0xa6, 0x73, 0x16, 0x9e, 0x65, 0x71, 0x16, 0xf2, 0x38, 0x2f, 0xc2, 0x31, + 0xd9, 0xce, 0x97, 0xe4, 0x3b, 0xff, 0xcf, 0x45, 0x58, 0x7a, 0xe8, 0x19, 0x66, 0x22, 0x86, 0x3f, + 0xff, 0x9d, 0x7f, 0x1b, 0x2a, 0x4c, 0xa3, 0x3b, 0x25, 0x8a, 0xeb, 0x65, 0x11, 0x17, 0xd7, 0xf6, + 0x84, 0xc2, 0x3d, 0x0a, 0xd0, 0xf9, 0x20, 0xf4, 0x32, 0xb4, 0x7c, 0x3c, 0xb4, 0xad, 0xbe, 0xd1, + 0x73, 0x47, 0xce, 0x3e, 0xf6, 0x3b, 0xe5, 0x65, 0x65, 0xa5, 0xac, 0x37, 0x39, 0x74, 0x87, 0x02, + 0xd1, 0x8f, 0xa0, 0x79, 0x60, 0x61, 0xdb, 0xec, 0x51, 0x93, 0xb0, 0xbd, 0xd5, 0xa9, 0x2c, 0x17, + 0x57, 0xea, 0x1b, 0x6f, 0xae, 0xe5, 0xad, 0xd1, 0x9a, 0x74, 0x47, 0xd6, 0xee, 0x93, 0xe1, 0xdb, + 0x6c, 0xf4, 0xbb, 0x6e, 0xe8, 0x9f, 0xea, 0x8d, 0x83, 0x14, 0x08, 0x75, 0x60, 0x8e, 0x6f, 0x6f, + 0x67, 0x6e, 0x59, 0x59, 0xa9, 0xea, 0xd1, 0x27, 0x7a, 0x05, 0xda, 0x3e, 0x0e, 0xbc, 0x91, 0xdf, + 0xc7, 0xbd, 0x81, 0xef, 0x8d, 0x86, 0x41, 0xa7, 0xba, 0x5c, 0x5c, 0xa9, 0xe9, 0xad, 0x08, 0xfc, + 0x80, 0x42, 0xbb, 0xef, 0xc0, 0x7c, 0x0e, 0x0b, 0x52, 0xa1, 0x78, 0x84, 0x4f, 0x29, 0x23, 0x8a, + 0x3a, 0xf9, 0x89, 0x16, 0xa1, 0x7c, 0x6c, 0xd8, 0x23, 0xcc, 0xb7, 0x9a, 0x7d, 0x7c, 0xa7, 0x70, + 0x47, 0xd1, 0xfe, 0x40, 0x81, 0x8e, 0x8e, 0x6d, 0x6c, 0x04, 0xf8, 0xf3, 0x64, 0xe9, 0x65, 0xa8, + 0xb8, 0x9e, 0x89, 0xb7, 0xb7, 0x28, 0x4b, 0x8b, 0x3a, 0xff, 0xd2, 0x3e, 0x55, 0x60, 0xf1, 0x01, + 0x0e, 0x89, 0x1a, 0x58, 0x41, 0x68, 0xf5, 0x63, 0x3d, 0x7f, 0x1b, 0x8a, 0x3e, 0x7e, 0xca, 0x29, + 0x7b, 0x55, 0xa4, 0x2c, 0x36, 0xff, 0xb2, 0x91, 0x3a, 0x19, 0x87, 0x5e, 0x84, 0x86, 0xe9, 0xd8, + 0xbd, 0xfe, 0xa1, 0xe1, 0xba, 0xd8, 0x66, 0x8a, 0x54, 0xd3, 0xeb, 0xa6, 0x63, 0x6f, 0x72, 0x10, + 0xba, 0x0e, 0x10, 0xe0, 0x81, 0x83, 0xdd, 0x30, 0xb1, 0xc9, 0x29, 0x08, 0x5a, 0x85, 0xf9, 0x03, + 0xdf, 0x73, 0x7a, 0xc1, 0xa1, 0xe1, 0x9b, 0x3d, 0x1b, 0x1b, 0x26, 0xf6, 0x29, 0xf5, 0x55, 0xbd, + 0x4d, 0x1a, 0xf6, 0x08, 0xfc, 0x21, 0x05, 0xa3, 0xdb, 0x50, 0x0e, 0xfa, 0xde, 0x10, 0x53, 0x49, + 0x6b, 0x6d, 0x5c, 0x93, 0xc9, 0xd0, 0x96, 0x11, 0x1a, 0x7b, 0xa4, 0x93, 0xce, 0xfa, 0x6a, 0x7f, + 0x53, 0x62, 0xaa, 0xf6, 0x05, 0x37, 0x72, 0x29, 0x75, 0x2c, 0x3f, 0x1f, 0x75, 0xac, 0x4c, 0xa5, + 0x8e, 0x73, 0x67, 0xab, 0x63, 0x6e, 0xd7, 0xce, 0xa3, 0x8e, 0xd5, 0x89, 0xea, 0x58, 0x93, 0xa9, + 0x23, 0x7a, 0x17, 0xda, 0x2c, 0x80, 0xb0, 0xdc, 0x03, 0xaf, 0x67, 0x5b, 0x41, 0xd8, 0x01, 0x4a, + 0xe6, 0xb5, 0xac, 0x84, 0x9a, 0xf8, 0xd9, 0x1a, 0x43, 0xec, 0x1e, 0x78, 0x7a, 0xd3, 0x8a, 0x7e, + 0x3e, 0xb4, 0x82, 0x70, 0x76, 0xad, 0xfe, 0xbb, 0x44, 0xab, 0xbf, 0xe8, 0xd2, 0x93, 0x68, 0x7e, + 0x59, 0xd0, 0xfc, 0x3f, 0x55, 0xe0, 0x2b, 0x0f, 0x70, 0x18, 0x93, 0x4f, 0x14, 0x19, 0x7f, 0x41, + 0xdd, 0xfc, 0x5f, 0x28, 0xd0, 0x95, 0xd1, 0x3a, 0x8b, 0xab, 0xff, 0x08, 0x2e, 0xc7, 0x38, 0x7a, + 0x26, 0x0e, 0xfa, 0xbe, 0x35, 0xa4, 0x6c, 0xa4, 0xb6, 0xaa, 0xbe, 0x71, 0x43, 0x26, 0xf8, 0x59, + 0x0a, 0x96, 0xe2, 0x29, 0xb6, 0x52, 0x33, 0x68, 0x3f, 0x51, 0x60, 0x89, 0xd8, 0x46, 0x6e, 0xcc, + 0x88, 0x04, 0x5e, 0x78, 0x5f, 0x45, 0x33, 0x59, 0xc8, 0x99, 0xc9, 0x29, 0xf6, 0x98, 0x86, 0xd8, + 0x59, 0x7a, 0x66, 0xd9, 0xbb, 0x6f, 0x42, 0x99, 0x28, 0x60, 0xb4, 0x55, 0x2f, 0xc8, 0xb6, 0x2a, + 0x8d, 0x8c, 0xf5, 0xd6, 0x5c, 0x46, 0x45, 0x62, 0xb7, 0x67, 0x10, 0xb7, 0xec, 0xb2, 0x0b, 0x92, + 0x65, 0xff, 0xa6, 0x02, 0x57, 0x72, 0x08, 0x67, 0x59, 0xf7, 0x5b, 0x50, 0xa1, 0xde, 0x28, 0x5a, + 0xf8, 0x4b, 0xd2, 0x85, 0xa7, 0xd0, 0x11, 0x6b, 0xa3, 0xf3, 0x31, 0x9a, 0x07, 0x6a, 0xb6, 0x8d, + 0xf8, 0x49, 0xee, 0x23, 0x7b, 0xae, 0xe1, 0xb0, 0x0d, 0xa8, 0xe9, 0x75, 0x0e, 0xdb, 0x31, 0x1c, + 0x8c, 0xbe, 0x02, 0x55, 0xa2, 0xb2, 0x3d, 0xcb, 0x8c, 0xd8, 0x3f, 0x47, 0x55, 0xd8, 0x0c, 0xd0, + 0x35, 0x00, 0xda, 0x64, 0x98, 0xa6, 0xcf, 0x5c, 0x68, 0x4d, 0xaf, 0x11, 0xc8, 0x5d, 0x02, 0xd0, + 0x7e, 0x5f, 0x81, 0xeb, 0x7b, 0xa7, 0x6e, 0x7f, 0x07, 0x9f, 0x6c, 0xfa, 0xd8, 0x08, 0x71, 0x62, + 0xb4, 0x3f, 0xd3, 0x8d, 0x47, 0xcb, 0x50, 0x4f, 0xe9, 0x2f, 0x17, 0xc9, 0x34, 0x48, 0xfb, 0x4b, + 0x05, 0x1a, 0xc4, 0x8b, 0x7c, 0x80, 0x43, 0x83, 0x88, 0x08, 0xfa, 0x36, 0xd4, 0x6c, 0xcf, 0x30, + 0x7b, 0xe1, 0xe9, 0x90, 0x51, 0xd3, 0xca, 0x52, 0x93, 0xb8, 0x9e, 0x47, 0xa7, 0x43, 0xac, 0x57, + 0x6d, 0xfe, 0x6b, 0x2a, 0x8a, 0xb2, 0x56, 0xa6, 0x28, 0xb1, 0x94, 0x2f, 0x40, 0xdd, 0xc1, 0xa1, + 0x6f, 0xf5, 0x19, 0x11, 0x25, 0xca, 0x0a, 0x60, 0x20, 0x82, 0x48, 0xfb, 0x49, 0x05, 0x2e, 0x7f, + 0xdf, 0x08, 0xfb, 0x87, 0x5b, 0x4e, 0x14, 0xc5, 0x5c, 0x7c, 0x1f, 0x13, 0xbb, 0x5c, 0x48, 0xdb, + 0xe5, 0xe7, 0x66, 0xf7, 0x63, 0x1d, 0x2d, 0xcb, 0x74, 0x94, 0x24, 0xe6, 0x6b, 0x4f, 0xb8, 0x98, + 0xa5, 0x74, 0x34, 0x15, 0x6c, 0x54, 0x2e, 0x12, 0x6c, 0x6c, 0x42, 0x13, 0x3f, 0xeb, 0xdb, 0x23, + 0x22, 0xaf, 0x14, 0x3b, 0x8b, 0x22, 0xae, 0x4b, 0xb0, 0xa7, 0x0d, 0x44, 0x83, 0x0f, 0xda, 0xe6, + 0x34, 0x30, 0x59, 0x70, 0x70, 0x68, 0xd0, 0x50, 0xa1, 0xbe, 0xb1, 0x3c, 0x4e, 0x16, 0x22, 0x01, + 0x62, 0xf2, 0x40, 0xbe, 0xd0, 0x55, 0xa8, 0xf1, 0xd0, 0x66, 0x7b, 0xab, 0x53, 0xa3, 0xdb, 0x97, + 0x00, 0x90, 0x01, 0x4d, 0x6e, 0x3d, 0x39, 0x85, 0x2c, 0x80, 0x78, 0x4b, 0x86, 0x40, 0xce, 0xec, + 0x34, 0xe5, 0x01, 0x0f, 0x74, 0x82, 0x14, 0x88, 0x64, 0xfe, 0xde, 0xc1, 0x81, 0x6d, 0xb9, 0x78, + 0x87, 0x71, 0xb8, 0x4e, 0x89, 0x10, 0x81, 0x24, 0x1c, 0x3a, 0xc6, 0x7e, 0x60, 0x79, 0x6e, 0xa7, + 0x41, 0xdb, 0xa3, 0x4f, 0x59, 0x94, 0xd3, 0xbc, 0x40, 0x94, 0xd3, 0x83, 0xf9, 0x1c, 0xa5, 0x92, + 0x28, 0xe7, 0x1b, 0xe9, 0x28, 0x67, 0x32, 0xab, 0x52, 0x51, 0xd0, 0x4f, 0x15, 0x58, 0x7a, 0xec, + 0x06, 0xa3, 0xfd, 0x78, 0x8b, 0x3e, 0x1f, 0x75, 0xc8, 0x1a, 0xd1, 0x52, 0xce, 0x88, 0x6a, 0xff, + 0x58, 0x86, 0x36, 0x5f, 0x05, 0x91, 0x1a, 0x6a, 0x72, 0xae, 0x42, 0x2d, 0xf6, 0xa3, 0x7c, 0x43, + 0x12, 0x40, 0xd6, 0x86, 0x15, 0x72, 0x36, 0x6c, 0x2a, 0xd2, 0xa2, 0xa8, 0xa8, 0x94, 0x8a, 0x8a, + 0xae, 0x01, 0x1c, 0xd8, 0xa3, 0xe0, 0xb0, 0x17, 0x5a, 0x0e, 0xe6, 0x51, 0x59, 0x8d, 0x42, 0x1e, + 0x59, 0x0e, 0x46, 0x77, 0xa1, 0xb1, 0x6f, 0xb9, 0xb6, 0x37, 0xe8, 0x0d, 0x8d, 0xf0, 0x30, 0xe0, + 0x69, 0xb1, 0x8c, 0x2d, 0x34, 0x86, 0xbd, 0x47, 0xfb, 0xea, 0x75, 0x36, 0x66, 0x97, 0x0c, 0x41, + 0xd7, 0xa1, 0xee, 0x8e, 0x9c, 0x9e, 0x77, 0xd0, 0xf3, 0xbd, 0x93, 0x80, 0x26, 0xbf, 0x45, 0xbd, + 0xe6, 0x8e, 0x9c, 0x0f, 0x0f, 0x74, 0xef, 0x84, 0xf8, 0xb1, 0x1a, 0xf1, 0x68, 0x81, 0xed, 0x0d, + 0x58, 0xe2, 0x3b, 0x79, 0xfe, 0x64, 0x00, 0x19, 0x6d, 0x62, 0x3b, 0x34, 0xe8, 0xe8, 0xda, 0x74, + 0xa3, 0xe3, 0x01, 0xe8, 0x26, 0xb4, 0xfa, 0x9e, 0x33, 0x34, 0xe8, 0x0e, 0xdd, 0xf7, 0x3d, 0x87, + 0x2a, 0x60, 0x51, 0xcf, 0x40, 0xd1, 0x26, 0xd4, 0x13, 0x25, 0x08, 0x3a, 0x75, 0x8a, 0x47, 0x93, + 0x69, 0x69, 0x2a, 0x94, 0x27, 0x02, 0x0a, 0xb1, 0x16, 0x04, 0x44, 0x32, 0x22, 0x65, 0x0f, 0xac, + 0x4f, 0x30, 0x57, 0xb4, 0x3a, 0x87, 0xed, 0x59, 0x9f, 0x60, 0x92, 0x1e, 0x59, 0x6e, 0x80, 0xfd, + 0x30, 0x4a, 0x56, 0x3b, 0x4d, 0x2a, 0x3e, 0x4d, 0x06, 0xe5, 0x82, 0x8d, 0xb6, 0xa0, 0x15, 0x84, + 0x86, 0x1f, 0xf6, 0x86, 0x5e, 0x40, 0x05, 0xa0, 0xd3, 0xa2, 0xb2, 0x9d, 0x51, 0x49, 0x27, 0x18, + 0x10, 0xc1, 0xde, 0xe5, 0x9d, 0xf4, 0x26, 0x1d, 0x14, 0x7d, 0x92, 0x59, 0xe8, 0x4e, 0x24, 0xb3, + 0xb4, 0xa7, 0x9a, 0x85, 0x0e, 0x8a, 0x3e, 0xb5, 0xff, 0x29, 0x40, 0x4b, 0x5c, 0x34, 0x31, 0x26, + 0x2c, 0xd7, 0x8a, 0x24, 0x39, 0xfa, 0x24, 0x5b, 0x80, 0x5d, 0x63, 0xdf, 0xc6, 0x2c, 0xb1, 0xa3, + 0x82, 0x5c, 0xd5, 0xeb, 0x0c, 0x46, 0x27, 0x20, 0x02, 0xc9, 0xb6, 0x9a, 0x6a, 0x4f, 0x91, 0x2e, + 0xbf, 0x46, 0x21, 0x34, 0x00, 0xe9, 0xc0, 0x5c, 0x94, 0x13, 0x32, 0x31, 0x8e, 0x3e, 0x49, 0xcb, + 0xfe, 0xc8, 0xa2, 0x58, 0x99, 0x18, 0x47, 0x9f, 0x68, 0x0b, 0x1a, 0x6c, 0xca, 0xa1, 0xe1, 0x1b, + 0x4e, 0x24, 0xc4, 0x2f, 0x4a, 0x0d, 0xc1, 0xfb, 0xf8, 0xf4, 0x09, 0xb1, 0x29, 0xbb, 0x86, 0xe5, + 0xeb, 0x8c, 0xe9, 0xbb, 0x74, 0x14, 0x5a, 0x01, 0x95, 0xcd, 0x72, 0x60, 0xd9, 0x98, 0xab, 0xc3, + 0x1c, 0x4b, 0x0c, 0x29, 0xfc, 0xbe, 0x65, 0x63, 0x26, 0xf1, 0xf1, 0x12, 0x28, 0x9b, 0xab, 0x4c, + 0xe0, 0x29, 0x84, 0x32, 0xf9, 0x06, 0x30, 0xdb, 0xd8, 0x8b, 0x2c, 0x2e, 0x73, 0x0b, 0x8c, 0xc6, + 0x27, 0xdc, 0xec, 0x92, 0x40, 0x6b, 0xe4, 0x30, 0x95, 0x01, 0xb6, 0x1c, 0x77, 0xe4, 0x10, 0x85, + 0xd1, 0xfe, 0xa9, 0x04, 0x0b, 0xc4, 0x6e, 0x70, 0x13, 0x32, 0x83, 0xdb, 0xbf, 0x06, 0x60, 0x06, + 0x61, 0x4f, 0xb0, 0x75, 0x35, 0x33, 0x08, 0xb9, 0x53, 0xf8, 0x76, 0xe4, 0xb5, 0x8b, 0xe3, 0x93, + 0x90, 0x8c, 0x1d, 0xcb, 0x7b, 0xee, 0x0b, 0x55, 0xed, 0x6e, 0x40, 0x93, 0x67, 0xe0, 0x42, 0xba, + 0xd8, 0x60, 0xc0, 0x1d, 0xb9, 0x35, 0xae, 0x48, 0xab, 0x87, 0x29, 0xef, 0x3d, 0x37, 0x9b, 0xf7, + 0xae, 0x66, 0xbd, 0xf7, 0x7d, 0x68, 0x8b, 0x0a, 0x14, 0x59, 0xa0, 0x09, 0x1a, 0xd4, 0x12, 0x34, + 0x28, 0x48, 0x3b, 0x5f, 0x10, 0x9d, 0xef, 0x0d, 0x68, 0xba, 0x18, 0x9b, 0xbd, 0xd0, 0x37, 0xdc, + 0xe0, 0x00, 0xfb, 0xd4, 0x79, 0x57, 0xf5, 0x06, 0x01, 0x3e, 0xe2, 0x30, 0xf4, 0x16, 0x00, 0x5d, + 0x23, 0x2b, 0x3a, 0x35, 0xc6, 0x17, 0x9d, 0xa8, 0xd0, 0xd0, 0xa2, 0x13, 0xdd, 0x14, 0xfa, 0x53, + 0xfb, 0x97, 0x02, 0x5c, 0xe6, 0xd5, 0x83, 0xd9, 0x05, 0x6a, 0x9c, 0xe3, 0x8c, 0x3c, 0x4f, 0xf1, + 0x8c, 0x7c, 0xbc, 0x34, 0x45, 0x6c, 0x59, 0x96, 0xc4, 0x96, 0x62, 0x4e, 0x5a, 0xc9, 0xe5, 0xa4, + 0x71, 0x39, 0x6e, 0x6e, 0xfa, 0x72, 0x1c, 0x5a, 0x84, 0x32, 0x4d, 0x94, 0x28, 0xd3, 0x6b, 0x3a, + 0xfb, 0x98, 0x8a, 0x1d, 0xda, 0xef, 0x15, 0xa0, 0xb9, 0x87, 0x0d, 0xbf, 0x7f, 0x18, 0xed, 0xe3, + 0x1b, 0xe9, 0xf2, 0xe5, 0x4b, 0x63, 0xca, 0x97, 0xc2, 0x90, 0x2f, 0x4d, 0xdd, 0x92, 0x20, 0x08, + 0xbd, 0xd0, 0x88, 0xa9, 0xec, 0xb9, 0x23, 0x87, 0xd7, 0xf4, 0xda, 0xb4, 0x81, 0x93, 0xba, 0x33, + 0x72, 0xb4, 0xff, 0x56, 0xa0, 0xf1, 0x3d, 0x32, 0x4d, 0xb4, 0x31, 0x77, 0xd2, 0x1b, 0x73, 0x73, + 0xcc, 0xc6, 0xe8, 0x24, 0xe7, 0xc1, 0xc7, 0xf8, 0x4b, 0x57, 0xd2, 0xfd, 0x7b, 0x05, 0xba, 0x24, + 0xe3, 0xd5, 0x99, 0xc1, 0x98, 0x5d, 0xbb, 0x6e, 0x40, 0xf3, 0x58, 0x88, 0x2d, 0x0b, 0x54, 0x38, + 0x1b, 0xc7, 0xe9, 0x0c, 0x5d, 0x07, 0x35, 0xaa, 0xb0, 0xf2, 0xc5, 0x46, 0xf6, 0xfb, 0x15, 0x19, + 0xd5, 0x19, 0xe2, 0xa8, 0xfd, 0x6b, 0xfb, 0x22, 0x50, 0xfb, 0x2d, 0x05, 0x16, 0x24, 0x1d, 0xd1, + 0x15, 0x98, 0xe3, 0xd5, 0x00, 0xee, 0xe8, 0x99, 0xbe, 0x9b, 0x84, 0x3d, 0x49, 0x3d, 0xcb, 0x32, + 0xf3, 0x01, 0xab, 0x49, 0x12, 0xdc, 0x38, 0xf5, 0x31, 0x73, 0xfc, 0x31, 0x03, 0xd4, 0x85, 0x2a, + 0x37, 0x83, 0x51, 0x4e, 0x19, 0x7f, 0x6b, 0x47, 0x80, 0x1e, 0xe0, 0xc4, 0xe9, 0xcc, 0xb2, 0xa3, + 0x89, 0xbd, 0x49, 0x08, 0x4d, 0x1b, 0x21, 0x53, 0xfb, 0x4f, 0x05, 0x16, 0x04, 0x6c, 0xb3, 0x54, + 0x6d, 0x12, 0xc7, 0x58, 0xb8, 0x88, 0x63, 0x14, 0x2a, 0x13, 0xc5, 0x73, 0x55, 0x26, 0xae, 0x03, + 0xc4, 0xfb, 0x1f, 0xed, 0x68, 0x0a, 0xa2, 0xfd, 0xad, 0x02, 0x97, 0xdf, 0x33, 0x5c, 0xd3, 0x3b, + 0x38, 0x98, 0x5d, 0x54, 0x37, 0x41, 0xc8, 0x42, 0xa7, 0xad, 0xcd, 0x89, 0xa9, 0xeb, 0xab, 0x30, + 0xef, 0x33, 0xcf, 0x64, 0x8a, 0xb2, 0x5c, 0xd4, 0xd5, 0xa8, 0x21, 0x96, 0xd1, 0x3f, 0x2f, 0x00, + 0x22, 0xab, 0xbe, 0x67, 0xd8, 0x86, 0xdb, 0xc7, 0x17, 0x27, 0xfd, 0x65, 0x68, 0x09, 0xb1, 0x47, + 0x7c, 0x56, 0x9e, 0x0e, 0x3e, 0x02, 0xf4, 0x3e, 0xb4, 0xf6, 0x19, 0xaa, 0x9e, 0x8f, 0x8d, 0xc0, + 0x73, 0x39, 0x3b, 0xa4, 0x65, 0xb8, 0x47, 0xbe, 0x35, 0x18, 0x60, 0x7f, 0xd3, 0x73, 0x4d, 0x1e, + 0x44, 0xef, 0x47, 0x64, 0x92, 0xa1, 0x44, 0x19, 0x92, 0x40, 0x2c, 0x66, 0x4e, 0x1c, 0x89, 0xd1, + 0xad, 0x08, 0xb0, 0x61, 0x27, 0x1b, 0x91, 0x78, 0x43, 0x95, 0x35, 0xec, 0x8d, 0xaf, 0xc2, 0x4a, + 0x02, 0x23, 0xed, 0xaf, 0x14, 0x40, 0x71, 0xa6, 0x4c, 0x4b, 0x0b, 0x54, 0xa3, 0xb3, 0x43, 0x15, + 0x89, 0x53, 0xbe, 0x0a, 0x35, 0x33, 0x1a, 0xc9, 0x4d, 0x50, 0x02, 0xa0, 0x3e, 0x92, 0x12, 0xdd, + 0x23, 0x92, 0x87, 0xcd, 0x28, 0x13, 0x65, 0xc0, 0x87, 0x14, 0x26, 0xc6, 0x55, 0xa5, 0x6c, 0x5c, + 0x95, 0x2e, 0x32, 0x96, 0x85, 0x22, 0xa3, 0xf6, 0xd3, 0x02, 0xa8, 0xd4, 0x85, 0x6c, 0x26, 0xd5, + 0xa2, 0xa9, 0x88, 0xbe, 0x01, 0x4d, 0x7e, 0xd7, 0x44, 0x20, 0xbc, 0xf1, 0x34, 0x35, 0x19, 0xba, + 0x05, 0x8b, 0xac, 0x93, 0x8f, 0x83, 0x91, 0x9d, 0x24, 0x61, 0x2c, 0x0b, 0x41, 0x4f, 0x99, 0xef, + 0x22, 0x4d, 0xd1, 0x88, 0xc7, 0x70, 0x79, 0x60, 0x7b, 0xfb, 0x86, 0xdd, 0x13, 0xd9, 0xc3, 0x78, + 0x38, 0x85, 0xc4, 0x2f, 0xb2, 0xe1, 0x7b, 0x69, 0x1e, 0x06, 0xe8, 0x1e, 0x34, 0x03, 0x8c, 0x8f, + 0x92, 0xcc, 0xac, 0x3c, 0x4d, 0x66, 0xd6, 0x20, 0x63, 0xe2, 0xc4, 0xec, 0x0f, 0x15, 0x68, 0x67, + 0x8e, 0x08, 0xb2, 0x75, 0x04, 0x25, 0x5f, 0x47, 0xb8, 0x03, 0x65, 0x62, 0xa9, 0x98, 0x6f, 0x69, + 0xc9, 0x73, 0x5c, 0x71, 0x56, 0x9d, 0x0d, 0x40, 0xeb, 0xb0, 0x20, 0xb9, 0x8a, 0xc0, 0xd9, 0x8f, + 0xf2, 0x37, 0x11, 0xb4, 0x9f, 0x95, 0xa0, 0x9e, 0xda, 0x8a, 0x09, 0x25, 0x90, 0xe7, 0x52, 0xea, + 0x1d, 0x77, 0xf4, 0x4c, 0x44, 0xce, 0xc1, 0x0e, 0x4b, 0xd8, 0x78, 0xf6, 0xe8, 0x60, 0x87, 0xa6, + 0x6b, 0xe9, 0x4c, 0xac, 0x22, 0x64, 0x62, 0x99, 0x5c, 0x75, 0xee, 0x8c, 0x5c, 0xb5, 0x2a, 0xe6, + 0xaa, 0x82, 0x0a, 0xd5, 0xb2, 0x2a, 0x34, 0x6d, 0x55, 0xe2, 0x16, 0x2c, 0xf4, 0x59, 0x29, 0xfd, + 0xde, 0xe9, 0x66, 0xdc, 0xc4, 0x83, 0x52, 0x59, 0x13, 0xba, 0x9f, 0xd4, 0x1b, 0x19, 0x97, 0x59, + 0xb6, 0x20, 0x4f, 0x85, 0x39, 0x6f, 0x18, 0x93, 0x23, 0xcb, 0x4c, 0xbf, 0xb2, 0xf5, 0x90, 0xe6, + 0x85, 0xea, 0x21, 0x2f, 0x40, 0x3d, 0x8a, 0x54, 0x88, 0xa6, 0xb7, 0x98, 0xd1, 0x8b, 0xcc, 0x80, + 0x19, 0x08, 0x76, 0xa0, 0x2d, 0x1e, 0x36, 0x64, 0x0b, 0x09, 0x6a, 0xbe, 0x90, 0x70, 0x05, 0xe6, + 0xac, 0xa0, 0x77, 0x60, 0x1c, 0xe1, 0xce, 0x3c, 0x6d, 0xad, 0x58, 0xc1, 0x7d, 0xe3, 0x08, 0x6b, + 0xff, 0x56, 0x84, 0x56, 0xe2, 0x60, 0xa7, 0xb6, 0x20, 0xd3, 0x5c, 0xc7, 0xd9, 0x01, 0x35, 0x89, + 0x7b, 0xe8, 0x0e, 0x9f, 0x99, 0x3c, 0x67, 0x4f, 0xf0, 0xda, 0xc3, 0x8c, 0xbe, 0x0a, 0xee, 0xbe, + 0x74, 0x2e, 0x77, 0x3f, 0xe3, 0x41, 0xfd, 0x6d, 0x58, 0x8a, 0x7d, 0xaf, 0xb0, 0x6c, 0x96, 0x60, + 0x2d, 0x46, 0x8d, 0xbb, 0xe9, 0xe5, 0x8f, 0x31, 0x01, 0x73, 0xe3, 0x4c, 0x40, 0x56, 0x04, 0xaa, + 0x39, 0x11, 0xc8, 0xdf, 0x17, 0xa8, 0x49, 0xee, 0x0b, 0x68, 0x8f, 0x61, 0x81, 0xd6, 0x7e, 0x83, + 0xbe, 0x6f, 0xed, 0xe3, 0x38, 0x05, 0x98, 0x86, 0xad, 0x5d, 0xa8, 0x66, 0xb2, 0x88, 0xf8, 0x5b, + 0xfb, 0xb1, 0x02, 0x97, 0xf3, 0xf3, 0x52, 0x89, 0x49, 0x0c, 0x89, 0x22, 0x18, 0x92, 0x5f, 0x86, + 0x85, 0x54, 0x44, 0x29, 0xcc, 0x3c, 0x26, 0x02, 0x97, 0x10, 0xae, 0xa3, 0x64, 0x8e, 0x08, 0xa6, + 0xfd, 0x4c, 0x89, 0x4b, 0xe8, 0x04, 0x36, 0xa0, 0xe7, 0x13, 0xc4, 0xaf, 0x79, 0xae, 0x6d, 0xb9, + 0x71, 0xa5, 0x84, 0xaf, 0x91, 0x01, 0x79, 0xa5, 0xe4, 0x3d, 0x68, 0xf3, 0x4e, 0xb1, 0x7b, 0x9a, + 0x32, 0x20, 0x6b, 0xb1, 0x71, 0xb1, 0x63, 0x7a, 0x19, 0x5a, 0xfc, 0xe0, 0x20, 0xc2, 0x57, 0x94, + 0x1d, 0x27, 0x7c, 0x17, 0xd4, 0xa8, 0xdb, 0x79, 0x1d, 0x62, 0x9b, 0x0f, 0x8c, 0x03, 0xbb, 0xdf, + 0x50, 0xa0, 0x23, 0xba, 0xc7, 0xd4, 0xf2, 0xcf, 0x1f, 0xde, 0xbd, 0x29, 0x1e, 0x17, 0xbf, 0x7c, + 0x06, 0x3d, 0x09, 0x9e, 0xe8, 0xd0, 0xf8, 0x77, 0x0a, 0xf4, 0xec, 0x9f, 0xa4, 0x7a, 0x5b, 0x56, + 0x10, 0xfa, 0xd6, 0xfe, 0x68, 0xb6, 0x03, 0x4c, 0x03, 0xea, 0xfd, 0x43, 0xdc, 0x3f, 0x1a, 0x7a, + 0x56, 0xc2, 0x95, 0x77, 0x64, 0x34, 0x8d, 0x47, 0xbb, 0xb6, 0x99, 0xcc, 0xc0, 0x4e, 0x80, 0xd2, + 0x73, 0x76, 0x7f, 0x00, 0x6a, 0xb6, 0x43, 0xfa, 0xe0, 0xa5, 0xc6, 0x0e, 0x5e, 0x6e, 0x8b, 0x07, + 0x2f, 0x13, 0x22, 0x8d, 0xd4, 0xb9, 0xcb, 0x5f, 0x17, 0xe0, 0xab, 0x52, 0xda, 0x66, 0xc9, 0x92, + 0xc6, 0xd5, 0x91, 0xee, 0x41, 0x35, 0x93, 0xd4, 0xde, 0x3c, 0x83, 0x7f, 0xbc, 0x96, 0xca, 0x6a, + 0x7a, 0x41, 0x12, 0x5b, 0x25, 0x0a, 0x5f, 0x1a, 0x3f, 0x07, 0xd7, 0x3b, 0x61, 0x8e, 0x68, 0x1c, + 0xba, 0x0b, 0x0d, 0x56, 0x30, 0xe8, 0x1d, 0x5b, 0xf8, 0x24, 0x3a, 0xd6, 0xbc, 0x2e, 0x35, 0xcd, + 0xb4, 0xdf, 0x13, 0x0b, 0x9f, 0xe8, 0x75, 0x3b, 0xfe, 0x1d, 0x68, 0xbf, 0x5b, 0x02, 0x48, 0xda, + 0x48, 0x76, 0x96, 0xe8, 0x3c, 0x57, 0xe2, 0x14, 0x84, 0xc4, 0x12, 0x62, 0xe4, 0x1a, 0x7d, 0x22, + 0x3d, 0x39, 0x56, 0x30, 0xad, 0x20, 0xe4, 0xfb, 0xb2, 0x7e, 0x36, 0x2d, 0xd1, 0x16, 0x11, 0x96, + 0x71, 0x99, 0x09, 0x12, 0x08, 0x7a, 0x1d, 0xd0, 0xc0, 0xf7, 0x4e, 0x2c, 0x77, 0x90, 0xce, 0x37, + 0x58, 0x5a, 0x32, 0xcf, 0x5b, 0x52, 0x09, 0xc7, 0x0f, 0x41, 0xcd, 0x74, 0x8f, 0xb6, 0xe4, 0xf6, + 0x04, 0x32, 0x1e, 0x08, 0x73, 0x71, 0xf1, 0x6d, 0x8b, 0x18, 0xe8, 0x19, 0xe6, 0x23, 0xc3, 0x1f, + 0xe0, 0x88, 0xa3, 0x3c, 0x0e, 0x13, 0x81, 0xdd, 0x1e, 0xa8, 0xd9, 0x55, 0x49, 0x4e, 0x18, 0xbf, + 0x29, 0x0a, 0xfa, 0x59, 0xf6, 0x88, 0x4c, 0x93, 0x12, 0xf5, 0xae, 0x01, 0x8b, 0x32, 0x7a, 0x25, + 0x48, 0x2e, 0xac, 0x4d, 0xef, 0xc4, 0x21, 0x31, 0xe5, 0xc3, 0x38, 0x2f, 0x93, 0xaa, 0x18, 0x17, + 0x84, 0x8a, 0xb1, 0xf6, 0x0f, 0x0a, 0xa0, 0xbc, 0xf8, 0xa3, 0x16, 0x14, 0xe2, 0x49, 0x0a, 0xdb, + 0x5b, 0x19, 0x71, 0x2b, 0xe4, 0xc4, 0xed, 0x2a, 0xd4, 0x62, 0xaf, 0xcf, 0x4d, 0x7c, 0x02, 0x48, + 0x0b, 0x63, 0x49, 0x14, 0xc6, 0x14, 0x61, 0x65, 0xb1, 0x94, 0x7d, 0x0b, 0x16, 0x6d, 0x23, 0x08, + 0x7b, 0xac, 0x62, 0x1e, 0x5a, 0x0e, 0x0e, 0x42, 0xc3, 0x19, 0x52, 0x56, 0x96, 0x74, 0x44, 0xda, + 0xb6, 0x48, 0xd3, 0xa3, 0xa8, 0x45, 0x3b, 0x04, 0x94, 0x57, 0xc2, 0x34, 0x6e, 0x45, 0xc4, 0x3d, + 0x69, 0x4d, 0x29, 0xda, 0x8a, 0xe2, 0xa6, 0xfd, 0x71, 0x11, 0x50, 0x12, 0x09, 0xc5, 0x67, 0xb2, + 0xd3, 0x84, 0x0f, 0xeb, 0xb0, 0x90, 0x8f, 0x93, 0xa2, 0xe0, 0x10, 0xe5, 0xa2, 0x24, 0x59, 0x44, + 0x53, 0x94, 0xdd, 0x80, 0x7c, 0x23, 0x36, 0x9b, 0x2c, 0xec, 0xbb, 0x3e, 0xb6, 0xa0, 0x2f, 0x5a, + 0xce, 0x1f, 0x64, 0x6f, 0x4e, 0x32, 0x3d, 0xbc, 0x23, 0x35, 0x71, 0xb9, 0x25, 0x4f, 0xbc, 0x36, + 0x29, 0x04, 0xa4, 0x95, 0xf3, 0x04, 0xa4, 0xb3, 0xdf, 0x73, 0xfc, 0xf7, 0x02, 0xcc, 0xc7, 0x1b, + 0x79, 0x2e, 0x26, 0x4d, 0x3e, 0x3e, 0xff, 0x8c, 0xb9, 0xf2, 0xb1, 0x9c, 0x2b, 0xdf, 0x3a, 0x33, + 0x29, 0x98, 0x96, 0x29, 0xb3, 0xef, 0xec, 0x27, 0x30, 0xc7, 0xcb, 0xbb, 0x39, 0x43, 0x31, 0x4d, + 0xda, 0xbd, 0x08, 0x65, 0x62, 0x97, 0xa2, 0xda, 0x1c, 0xfb, 0x60, 0x5b, 0x9a, 0xbe, 0x47, 0xcb, + 0x6d, 0x45, 0x53, 0xb8, 0x46, 0xab, 0xfd, 0x47, 0x01, 0x60, 0xef, 0xd4, 0xed, 0xdf, 0x65, 0x4a, + 0x7a, 0x0b, 0x4a, 0x93, 0x6e, 0x5d, 0x91, 0xde, 0x54, 0xb6, 0x68, 0xcf, 0x29, 0x98, 0x2b, 0x14, + 0x16, 0x8a, 0xd9, 0xc2, 0xc2, 0xb8, 0x92, 0xc0, 0x78, 0x53, 0xf6, 0x2d, 0x28, 0x91, 0x70, 0x90, + 0x5f, 0x4a, 0x9a, 0xea, 0x58, 0x94, 0x0e, 0x40, 0x2b, 0x10, 0xb9, 0xb6, 0x6d, 0x97, 0xf9, 0x2e, + 0x7a, 0x82, 0x5c, 0xd4, 0xb3, 0x60, 0x74, 0x13, 0x5a, 0xac, 0xa0, 0x14, 0x77, 0x64, 0xb9, 0x51, + 0x06, 0x9a, 0xf7, 0x8c, 0x35, 0x89, 0x67, 0xd4, 0x3e, 0x2d, 0xc0, 0x15, 0xb2, 0x6b, 0xcf, 0x27, + 0x66, 0x9d, 0x46, 0x24, 0x52, 0x56, 0xba, 0x28, 0x5a, 0xe9, 0x3b, 0x30, 0xc7, 0x8a, 0x11, 0x51, + 0xf4, 0x75, 0x7d, 0x1c, 0x8f, 0x99, 0x44, 0xe8, 0x51, 0xf7, 0x59, 0x33, 0x5a, 0xe1, 0x28, 0xb8, + 0x32, 0xdb, 0x51, 0xf0, 0x5c, 0xb6, 0x64, 0x99, 0x12, 0x96, 0xaa, 0xe8, 0x5b, 0x1e, 0x43, 0x53, + 0x4f, 0x0b, 0x3c, 0x42, 0x50, 0x4a, 0xdd, 0xae, 0xa4, 0xbf, 0x69, 0x12, 0x6a, 0x0c, 0x8d, 0xbe, + 0x15, 0x9e, 0xd2, 0xed, 0x2c, 0xeb, 0xf1, 0xb7, 0x5c, 0xbb, 0xb4, 0xff, 0x55, 0xe0, 0x72, 0x74, + 0xe4, 0xc8, 0x75, 0xf7, 0xe2, 0x1c, 0xdd, 0x80, 0x25, 0xae, 0xa8, 0x19, 0x8d, 0x65, 0xa1, 0xe6, + 0x02, 0x83, 0x89, 0xcb, 0xd8, 0x80, 0xa5, 0x90, 0x0a, 0x59, 0x76, 0x0c, 0xe3, 0xf7, 0x02, 0x6b, + 0x14, 0xc7, 0x4c, 0x73, 0xe4, 0xfb, 0x02, 0xbb, 0x2e, 0xc4, 0xb7, 0x96, 0xab, 0x1e, 0xb8, 0x23, + 0x87, 0xaf, 0x52, 0x3b, 0x81, 0xab, 0xec, 0x7e, 0xf3, 0xbe, 0x48, 0xd1, 0x4c, 0x15, 0x7f, 0xe9, + 0xba, 0x33, 0x96, 0xea, 0x8f, 0x14, 0xb8, 0x36, 0x06, 0xf3, 0x2c, 0xb9, 0xce, 0x43, 0x29, 0xf6, + 0x31, 0x99, 0xa9, 0x80, 0x97, 0xdd, 0xb3, 0x13, 0x89, 0xfc, 0xb4, 0x04, 0xf3, 0xb9, 0x4e, 0xe7, + 0x96, 0xb9, 0xd7, 0x00, 0x11, 0x26, 0xc4, 0x6f, 0xf9, 0x68, 0xb2, 0xcf, 0x5d, 0xa2, 0xea, 0x8e, + 0x9c, 0xf8, 0x1d, 0x1f, 0xc9, 0xf7, 0x91, 0xc5, 0x7a, 0xb3, 0x7a, 0x7f, 0xcc, 0xb9, 0xd2, 0xf8, + 0x27, 0x1b, 0x39, 0x02, 0xd7, 0x76, 0x46, 0x0e, 0x3b, 0x1a, 0xe0, 0x5c, 0x66, 0x6e, 0x8e, 0xa0, + 0x12, 0xc0, 0xe8, 0x00, 0xe6, 0xe9, 0x65, 0xb2, 0x51, 0x38, 0xf0, 0x48, 0xba, 0x41, 0xe9, 0x62, + 0xce, 0xf4, 0x3b, 0x53, 0x63, 0xfa, 0x90, 0x8f, 0x26, 0xc4, 0xf3, 0x8c, 0xc3, 0x15, 0xa1, 0x11, + 0x1e, 0xcb, 0xed, 0x7b, 0x4e, 0x8c, 0xa7, 0x72, 0x4e, 0x3c, 0xdb, 0x7c, 0xb4, 0x88, 0x27, 0x0d, + 0xed, 0x6e, 0xc2, 0x92, 0x74, 0xe9, 0x93, 0xdc, 0x77, 0x39, 0x9d, 0x97, 0xdc, 0x83, 0x45, 0xd9, + 0xaa, 0x2e, 0x30, 0x47, 0x8e, 0xe2, 0xf3, 0xcc, 0xa1, 0xfd, 0x59, 0x01, 0x9a, 0x5b, 0xd8, 0xc6, + 0x21, 0xfe, 0x6c, 0x4f, 0x64, 0x73, 0xc7, 0xcb, 0xc5, 0xfc, 0xf1, 0x72, 0xee, 0xac, 0xbc, 0x24, + 0x39, 0x2b, 0xbf, 0x16, 0x5f, 0x11, 0x20, 0xb3, 0x94, 0xc5, 0xc8, 0xc0, 0x44, 0x6f, 0x42, 0x63, + 0xe8, 0x5b, 0x8e, 0xe1, 0x9f, 0xf6, 0x8e, 0xf0, 0x69, 0xc0, 0x9d, 0x46, 0x47, 0xea, 0x76, 0xb6, + 0xb7, 0x02, 0xbd, 0xce, 0x7b, 0xbf, 0x8f, 0x4f, 0xe9, 0xf5, 0x83, 0x38, 0xc9, 0x61, 0x17, 0xc5, + 0x4a, 0x7a, 0x0a, 0xb2, 0xba, 0x0c, 0xb5, 0xf8, 0x3e, 0x0e, 0xaa, 0x42, 0xe9, 0xfe, 0xc8, 0xb6, + 0xd5, 0x4b, 0xa8, 0x06, 0x65, 0x9a, 0x06, 0xa9, 0xca, 0xea, 0x2f, 0x41, 0x2d, 0xbe, 0x53, 0x80, + 0xea, 0x30, 0xf7, 0xd8, 0x7d, 0xdf, 0xf5, 0x4e, 0x5c, 0xf5, 0x12, 0x9a, 0x83, 0xe2, 0x5d, 0xdb, + 0x56, 0x15, 0xd4, 0x84, 0xda, 0x5e, 0xe8, 0x63, 0x83, 0xf0, 0x4c, 0x2d, 0xa0, 0x16, 0xc0, 0x7b, + 0x56, 0x10, 0x7a, 0xbe, 0xd5, 0x37, 0x6c, 0xb5, 0xb8, 0xfa, 0x09, 0xb4, 0xc4, 0x12, 0x33, 0x6a, + 0x40, 0x75, 0xc7, 0x0b, 0xdf, 0x7d, 0x66, 0x05, 0xa1, 0x7a, 0x89, 0xf4, 0xdf, 0xf1, 0xc2, 0x5d, + 0x1f, 0x07, 0xd8, 0x0d, 0x55, 0x05, 0x01, 0x54, 0x3e, 0x74, 0xb7, 0xac, 0xe0, 0x48, 0x2d, 0xa0, + 0x05, 0x7e, 0x7a, 0x64, 0xd8, 0xdb, 0xbc, 0x6e, 0xab, 0x16, 0xc9, 0xf0, 0xf8, 0xab, 0x84, 0x54, + 0x68, 0xc4, 0x5d, 0x1e, 0xec, 0x3e, 0x56, 0xcb, 0x84, 0x7a, 0xf6, 0xb3, 0xb2, 0x6a, 0x82, 0x9a, + 0x3d, 0xf5, 0x24, 0x73, 0xb2, 0x45, 0xc4, 0x20, 0xf5, 0x12, 0x59, 0x19, 0x3f, 0x76, 0x56, 0x15, + 0xd4, 0x86, 0x7a, 0xea, 0x10, 0x57, 0x2d, 0x10, 0xc0, 0x03, 0x7f, 0xd8, 0xe7, 0x02, 0xc5, 0x48, + 0x20, 0xd2, 0xb9, 0x45, 0x76, 0xa2, 0xb4, 0x7a, 0x0f, 0xaa, 0x51, 0xaa, 0x41, 0xba, 0xf2, 0x2d, + 0x22, 0x9f, 0xea, 0x25, 0x34, 0x0f, 0x4d, 0xe1, 0x71, 0x98, 0xaa, 0x20, 0x04, 0x2d, 0xf1, 0xf9, + 0xa6, 0x5a, 0x58, 0xdd, 0x00, 0x48, 0x42, 0x76, 0x42, 0xce, 0xb6, 0x7b, 0x6c, 0xd8, 0x96, 0xc9, + 0x68, 0x23, 0x4d, 0x64, 0x77, 0xe9, 0xee, 0x30, 0x45, 0x55, 0x0b, 0xab, 0x6f, 0x43, 0x35, 0x0a, + 0x43, 0x09, 0x5c, 0xc7, 0x8e, 0x77, 0x8c, 0x19, 0x67, 0xf6, 0x30, 0xd9, 0xca, 0x1a, 0x94, 0xef, + 0x3a, 0xd8, 0x35, 0xd5, 0x02, 0x21, 0xe3, 0xf1, 0xd0, 0x34, 0x42, 0xcc, 0xc3, 0x31, 0xb5, 0xb8, + 0xf1, 0x5f, 0x0b, 0x00, 0xec, 0x18, 0xd3, 0xf3, 0x7c, 0x13, 0xd9, 0xf4, 0x3a, 0xc3, 0xa6, 0xe7, + 0x0c, 0x3d, 0x37, 0x3a, 0x63, 0x09, 0xd0, 0x5a, 0xa6, 0x6a, 0xc0, 0x3e, 0xf2, 0x1d, 0xf9, 0xde, + 0x74, 0x5f, 0x92, 0xf6, 0xcf, 0x74, 0xd6, 0x2e, 0x21, 0x87, 0x62, 0x23, 0x79, 0xf6, 0x23, 0xab, + 0x7f, 0x14, 0x9f, 0x7d, 0x8e, 0x7f, 0x56, 0x99, 0xe9, 0x1a, 0xe1, 0xbb, 0x21, 0xc5, 0xb7, 0x17, + 0xfa, 0x96, 0x3b, 0x88, 0x5c, 0xa2, 0x76, 0x09, 0x3d, 0xcd, 0x3c, 0xea, 0x8c, 0x10, 0x6e, 0x4c, + 0xf3, 0x8e, 0xf3, 0x62, 0x28, 0x6d, 0x68, 0x67, 0x5e, 0xcf, 0xa3, 0x55, 0xf9, 0xeb, 0x18, 0xd9, + 0x4b, 0xff, 0xee, 0xab, 0x53, 0xf5, 0x8d, 0xb1, 0x59, 0xd0, 0x12, 0x9f, 0x7d, 0xa3, 0xaf, 0x8d, + 0x9b, 0x20, 0xf7, 0x3e, 0xaf, 0xbb, 0x3a, 0x4d, 0xd7, 0x18, 0xd5, 0x47, 0x4c, 0x7c, 0x27, 0xa1, + 0x92, 0x3e, 0x89, 0xec, 0x9e, 0x15, 0x8d, 0x68, 0x97, 0xd0, 0x8f, 0x48, 0xe0, 0x90, 0x79, 0x45, + 0x88, 0x5e, 0x93, 0x3b, 0x3b, 0xf9, 0x63, 0xc3, 0x49, 0x18, 0x3e, 0xca, 0x2a, 0xdf, 0x78, 0xea, + 0x73, 0xcf, 0x93, 0xa7, 0xa7, 0x3e, 0x35, 0xfd, 0x59, 0xd4, 0x9f, 0x1b, 0x83, 0xcd, 0x72, 0x28, + 0xc9, 0xfb, 0xa5, 0xac, 0x28, 0x27, 0x29, 0xcc, 0xf8, 0xc7, 0x4e, 0x93, 0xb0, 0x8d, 0xa8, 0x92, + 0x66, 0xcf, 0xef, 0x5f, 0x1f, 0x73, 0x32, 0x20, 0x7f, 0x38, 0xd9, 0x5d, 0x9b, 0xb6, 0x7b, 0x5a, + 0x96, 0xc5, 0xb7, 0x79, 0x72, 0x16, 0x49, 0xdf, 0x13, 0xca, 0x65, 0x59, 0xfe, 0xd4, 0x4f, 0xbb, + 0x84, 0x1e, 0x09, 0xa6, 0x1e, 0xdd, 0x1c, 0x27, 0x0a, 0xe2, 0x85, 0x9e, 0x49, 0xfb, 0xf6, 0xab, + 0x80, 0x98, 0xa6, 0xba, 0x07, 0xd6, 0x60, 0xe4, 0x1b, 0x4c, 0x8c, 0xc7, 0x19, 0xb7, 0x7c, 0xd7, + 0x08, 0xcd, 0xd7, 0xcf, 0x31, 0x22, 0x5e, 0x52, 0x0f, 0xe0, 0x01, 0x0e, 0x3f, 0xa0, 0x8f, 0xb4, + 0x82, 0xec, 0x8a, 0x12, 0xfb, 0xcd, 0x3b, 0x44, 0xa8, 0x5e, 0x99, 0xd8, 0x2f, 0x46, 0xb0, 0x0f, + 0xf5, 0x07, 0x24, 0xa9, 0xa2, 0x81, 0x62, 0x80, 0xc6, 0x8e, 0x8c, 0x7a, 0x44, 0x28, 0x56, 0x26, + 0x77, 0x4c, 0x1b, 0xcf, 0xcc, 0x3b, 0x45, 0x34, 0x96, 0xb1, 0xf9, 0xd7, 0x93, 0x72, 0xe3, 0x39, + 0xe6, 0xe1, 0x23, 0x5b, 0x11, 0x3d, 0x9d, 0x7a, 0x0f, 0x1b, 0x76, 0x78, 0x38, 0x66, 0x45, 0xa9, + 0x1e, 0x67, 0xaf, 0x48, 0xe8, 0x18, 0xe3, 0xc0, 0xb0, 0xc0, 0xb4, 0x50, 0xcc, 0x46, 0xd7, 0xe5, + 0x53, 0xe4, 0x7b, 0x4e, 0x29, 0x7a, 0x06, 0xcc, 0x6f, 0xf9, 0xde, 0x50, 0x44, 0xf2, 0xba, 0x14, + 0x49, 0xae, 0xdf, 0x94, 0x28, 0xbe, 0x0f, 0x8d, 0x28, 0xe9, 0xa7, 0x69, 0x8a, 0x7c, 0x17, 0xd2, + 0x5d, 0xa6, 0x9c, 0xf8, 0x63, 0x68, 0x67, 0xaa, 0x09, 0x72, 0xa6, 0xcb, 0x4b, 0x0e, 0x93, 0x66, + 0x3f, 0x01, 0x44, 0x1f, 0x9f, 0x8a, 0xef, 0xe7, 0xe5, 0xf1, 0x4d, 0xbe, 0x63, 0x84, 0x64, 0x7d, + 0xea, 0xfe, 0x31, 0xe7, 0x7f, 0x0d, 0x96, 0xa4, 0x19, 0x7b, 0xd6, 0x20, 0xf0, 0x1b, 0xbc, 0x67, + 0x94, 0x15, 0xb2, 0x06, 0xe1, 0xcc, 0x11, 0x11, 0xfe, 0x8d, 0xdf, 0x9e, 0x87, 0x1a, 0x8d, 0xf3, + 0x28, 0xb7, 0x7e, 0x11, 0xe6, 0x3d, 0xdf, 0x30, 0xef, 0x63, 0x68, 0x67, 0x1e, 0x45, 0xca, 0x85, + 0x56, 0xfe, 0x72, 0x72, 0x8a, 0x68, 0x45, 0x7c, 0x4f, 0x28, 0x77, 0x85, 0xd2, 0x37, 0x87, 0x93, + 0xe6, 0x7e, 0xc2, 0x1e, 0x1c, 0xc7, 0x07, 0x9c, 0xaf, 0x8c, 0x3d, 0x47, 0x10, 0x6f, 0xe2, 0x7e, + 0xfe, 0x51, 0xd0, 0x97, 0x3b, 0x02, 0xfd, 0x18, 0xda, 0x99, 0xb7, 0x2e, 0x72, 0x89, 0x91, 0x3f, + 0x88, 0x99, 0x34, 0xfb, 0xcf, 0x31, 0x78, 0x32, 0x61, 0x41, 0xf2, 0xb4, 0x00, 0xad, 0x8d, 0x0b, + 0x44, 0xe5, 0x6f, 0x10, 0x26, 0x2f, 0xa8, 0x29, 0xa8, 0x69, 0xd6, 0xdf, 0x24, 0x44, 0x66, 0xff, + 0x78, 0xa7, 0xfb, 0xda, 0x74, 0xff, 0xd2, 0x13, 0x2f, 0x68, 0x0f, 0x2a, 0xec, 0x05, 0x0c, 0x7a, + 0x51, 0x7e, 0x9e, 0x92, 0x7a, 0x1d, 0xd3, 0x9d, 0xf4, 0x86, 0x26, 0x18, 0xd9, 0x21, 0xa1, 0xff, + 0x57, 0xa0, 0xc5, 0x40, 0xf1, 0x06, 0x3d, 0xc7, 0xc9, 0xf7, 0xa0, 0x4c, 0x4d, 0x3b, 0x92, 0x9e, + 0x22, 0xa4, 0xdf, 0xb9, 0x74, 0x27, 0x3f, 0x6d, 0x49, 0x28, 0x6e, 0x7e, 0x8f, 0xfd, 0x5f, 0x1a, + 0x27, 0xf8, 0x79, 0x4e, 0xfe, 0xff, 0x3b, 0x36, 0x7e, 0x46, 0x5f, 0x69, 0x64, 0xef, 0x21, 0xa1, + 0xb5, 0xf3, 0x5d, 0xa6, 0xea, 0xae, 0x4f, 0xdd, 0x3f, 0xc6, 0xfc, 0x43, 0x50, 0xb3, 0xa7, 0x6b, + 0xe8, 0xd5, 0x71, 0x9a, 0x28, 0xc3, 0x39, 0x41, 0x0d, 0xbf, 0x0b, 0x15, 0x56, 0x56, 0x95, 0x8b, + 0xaf, 0x50, 0x72, 0x9d, 0x30, 0xd7, 0xbd, 0x6f, 0x7c, 0xb4, 0x31, 0xb0, 0xc2, 0xc3, 0xd1, 0x3e, + 0x69, 0x59, 0x67, 0x5d, 0x5f, 0xb7, 0x3c, 0xfe, 0x6b, 0x3d, 0xe2, 0xe5, 0x3a, 0x1d, 0xbd, 0x4e, + 0x11, 0x0c, 0xf7, 0xf7, 0x2b, 0xf4, 0xf3, 0xf6, 0xff, 0x05, 0x00, 0x00, 0xff, 0xff, 0xf5, 0xd5, + 0x38, 0x83, 0xb0, 0x51, 0x00, 0x00, } // Reference imports to suppress errors if they are not otherwise used. diff --git a/internal/querycoordv2/checkers/segment_checker.go b/internal/querycoordv2/checkers/segment_checker.go index f6772f0b20..f1f9faa740 100644 --- a/internal/querycoordv2/checkers/segment_checker.go +++ b/internal/querycoordv2/checkers/segment_checker.go @@ -91,6 +91,7 @@ func (c *SegmentChecker) checkReplica(ctx context.Context, replica *meta.Replica task.SetReason("lacks of segment", tasks...) ret = append(ret, tasks...) + redundancies = c.filterSegmentInUse(replica, redundancies) tasks = c.createSegmentReduceTasks(ctx, redundancies, replica.GetID(), querypb.DataScope_All) task.SetReason("segment not exists in target", tasks...) ret = append(ret, tasks...) @@ -122,28 +123,43 @@ func (c *SegmentChecker) getStreamingSegmentDiff(targetMgr *meta.TargetManager, log.Info("replica does not exist, skip it") return } - dist := c.getStreamingSegmentsDist(distMgr, replica) - distMap := typeutil.NewUniqueSet() - for _, s := range dist { - distMap.Insert(s.GetID()) - } - nextTargetSegmentIDs := targetMgr.GetStreamingSegmentsByCollection(collectionID, meta.NextTarget) - currentTargetSegmentIDs := targetMgr.GetStreamingSegmentsByCollection(collectionID, meta.CurrentTarget) - currentTargetChannelMap := targetMgr.GetDmChannelsByCollection(collectionID, meta.CurrentTarget) + log := log.Ctx(context.TODO()).WithRateGroup("qcv2.SegmentChecker", 60, 1).With( + zap.Int64("collectionID", collectionID), + zap.Int64("replicaID", replica.ID)) - // get segment which exist on dist, but not on current target and next target - for _, segment := range dist { - if !currentTargetSegmentIDs.Contain(segment.GetID()) && !nextTargetSegmentIDs.Contain(segment.GetID()) { - if channel, ok := currentTargetChannelMap[segment.InsertChannel]; ok { - timestampInSegment := segment.GetStartPosition().GetTimestamp() - timestampInTarget := channel.GetSeekPosition().GetTimestamp() - // filter toRelease which seekPosition is newer than next target dmChannel - if timestampInSegment < timestampInTarget { - log.Info("growing segment not exist in target, so release it", - zap.Int64("segmentID", segment.GetID()), - ) - toRelease = append(toRelease, segment) + leaders := distMgr.ChannelDistManager.GetShardLeadersByReplica(replica) + for leader, node := range leaders { + view := distMgr.LeaderViewManager.GetLeaderShardView(node, leader) + targetVersion := targetMgr.GetCollectionTargetVersion(collectionID, meta.CurrentTarget) + if view.TargetVersion != targetVersion { + // before shard delegator update it's readable version, skip release segment + log.RatedInfo(20, "before shard delegator update it's readable version, skip release segment", + zap.String("channelName", leader), + zap.Int64("nodeID", node), + zap.Int64("leaderVersion", view.TargetVersion), + zap.Int64("currentVersion", targetVersion), + ) + continue + } + + nextTargetSegmentIDs := targetMgr.GetStreamingSegmentsByCollection(collectionID, meta.NextTarget) + currentTargetSegmentIDs := targetMgr.GetStreamingSegmentsByCollection(collectionID, meta.CurrentTarget) + currentTargetChannelMap := targetMgr.GetDmChannelsByCollection(collectionID, meta.CurrentTarget) + + // get segment which exist on leader view, but not on current target and next target + for _, segment := range view.GrowingSegments { + if !currentTargetSegmentIDs.Contain(segment.GetID()) && !nextTargetSegmentIDs.Contain(segment.GetID()) { + if channel, ok := currentTargetChannelMap[segment.InsertChannel]; ok { + timestampInSegment := segment.GetStartPosition().GetTimestamp() + timestampInTarget := channel.GetSeekPosition().GetTimestamp() + // filter toRelease which seekPosition is newer than next target dmChannel + if timestampInSegment < timestampInTarget { + log.Info("growing segment not exist in target, so release it", + zap.Int64("segmentID", segment.GetID()), + ) + toRelease = append(toRelease, segment) + } } } } @@ -152,18 +168,6 @@ func (c *SegmentChecker) getStreamingSegmentDiff(targetMgr *meta.TargetManager, return } -func (c *SegmentChecker) getStreamingSegmentsDist(distMgr *meta.DistributionManager, replica *meta.Replica) map[int64]*meta.Segment { - segments := make(map[int64]*meta.Segment, 0) - for _, node := range replica.GetNodes() { - segmentsOnNodes := distMgr.LeaderViewManager.GetGrowingSegmentDistByCollectionAndNode(replica.CollectionID, node) - for k, v := range segmentsOnNodes { - segments[k] = v - } - } - - return segments -} - // GetHistoricalSegmentDiff get historical segment diff between target and dist func (c *SegmentChecker) getHistoricalSegmentDiff( targetMgr *meta.TargetManager, @@ -248,16 +252,10 @@ func (c *SegmentChecker) filterExistedOnLeader(replica *meta.Replica, segments [ if !ok { continue } - onLeader := false - leaderViews := c.dist.LeaderViewManager.GetLeaderView(leaderID) - for _, view := range leaderViews { - version, ok := view.Segments[s.GetID()] - if ok && version.NodeID == s.Node { - onLeader = true - break - } - } - if onLeader { + + view := c.dist.LeaderViewManager.GetLeaderShardView(leaderID, s.GetInsertChannel()) + seg, ok := view.Segments[s.GetID()] + if ok && seg.NodeID == s.Node { // if this segment is serving on leader, do not remove it for search available continue } @@ -266,6 +264,26 @@ func (c *SegmentChecker) filterExistedOnLeader(replica *meta.Replica, segments [ return filtered } +func (c *SegmentChecker) filterSegmentInUse(replica *meta.Replica, segments []*meta.Segment) []*meta.Segment { + filtered := make([]*meta.Segment, 0, len(segments)) + for _, s := range segments { + leaderID, ok := c.dist.ChannelDistManager.GetShardLeader(replica, s.GetInsertChannel()) + if !ok { + continue + } + + view := c.dist.LeaderViewManager.GetLeaderShardView(leaderID, s.GetInsertChannel()) + currentTargetVersion := c.targetMgr.GetCollectionTargetVersion(s.CollectionID, meta.CurrentTarget) + partition := c.meta.CollectionManager.GetPartition(s.PartitionID) + if partition != nil && view.TargetVersion != currentTargetVersion { + // leader view version hasn't been updated, segment maybe still in use + continue + } + filtered = append(filtered, s) + } + return filtered +} + func (c *SegmentChecker) createSegmentLoadTasks(ctx context.Context, segments []*datapb.SegmentInfo, replica *meta.Replica) []task.Task { if len(segments) == 0 { return nil diff --git a/internal/querycoordv2/checkers/segment_checker_test.go b/internal/querycoordv2/checkers/segment_checker_test.go index bf97d8a193..8b67825d12 100644 --- a/internal/querycoordv2/checkers/segment_checker_test.go +++ b/internal/querycoordv2/checkers/segment_checker_test.go @@ -202,6 +202,52 @@ func (suite *SegmentCheckerTestSuite) TestReleaseRepeatedSegments() { suite.Len(tasks, 0) } +func (suite *SegmentCheckerTestSuite) TestSkipReleaseSealedSegments() { + checker := suite.checker + + collectionID := int64(1) + partitionID := int64(1) + // set meta + checker.meta.CollectionManager.PutCollection(utils.CreateTestCollection(collectionID, 1)) + checker.meta.CollectionManager.PutPartition(utils.CreateTestPartition(collectionID, partitionID)) + checker.meta.ReplicaManager.Put(utils.CreateTestReplica(1, collectionID, []int64{1, 2})) + + // set target + segments := []*datapb.SegmentInfo{} + suite.broker.EXPECT().GetRecoveryInfoV2(mock.Anything, int64(1)).Return( + nil, segments, nil) + checker.targetMgr.UpdateCollectionNextTargetWithPartitions(collectionID, partitionID) + checker.targetMgr.UpdateCollectionCurrentTarget(collectionID) + readableVersion := checker.targetMgr.GetCollectionTargetVersion(collectionID, meta.CurrentTarget) + + // set dist + nodeID := int64(2) + segmentID := int64(1) + checker.dist.ChannelDistManager.Update(nodeID, utils.CreateTestChannel(collectionID, nodeID, segmentID, "test-insert-channel")) + view := utils.CreateTestLeaderView(nodeID, collectionID, "test-insert-channel", map[int64]int64{segmentID: 2}, map[int64]*meta.Segment{}) + view.TargetVersion = readableVersion - 1 + checker.dist.LeaderViewManager.Update(nodeID, view) + checker.dist.SegmentDistManager.Update(nodeID, utils.CreateTestSegment(collectionID, partitionID, segmentID, nodeID, 2, "test-insert-channel")) + + tasks := checker.Check(context.TODO()) + suite.Len(tasks, 0) + + // test less version exist on leader + view = utils.CreateTestLeaderView(nodeID, collectionID, "test-insert-channel", map[int64]int64{1: 3}, map[int64]*meta.Segment{}) + view.TargetVersion = readableVersion + checker.dist.LeaderViewManager.Update(2, view) + tasks = checker.Check(context.TODO()) + suite.Len(tasks, 1) + suite.Len(tasks[0].Actions(), 1) + action, ok := tasks[0].Actions()[0].(*task.SegmentAction) + suite.True(ok) + suite.EqualValues(1, tasks[0].ReplicaID()) + suite.Equal(task.ActionTypeReduce, action.Type()) + suite.EqualValues(segmentID, action.SegmentID()) + suite.EqualValues(nodeID, action.Node()) + suite.Equal(tasks[0].Priority(), task.TaskPriorityNormal) +} + func (suite *SegmentCheckerTestSuite) TestReleaseGrowingSegments() { checker := suite.checker // segment3 is compacted from segment2, and node2 has growing segments 2 and 3. checker should generate @@ -239,7 +285,9 @@ func (suite *SegmentCheckerTestSuite) TestReleaseGrowingSegments() { dmChannel := utils.CreateTestChannel(1, 2, 1, "test-insert-channel") dmChannel.UnflushedSegmentIds = []int64{2, 3} checker.dist.ChannelDistManager.Update(2, dmChannel) - checker.dist.LeaderViewManager.Update(2, utils.CreateTestLeaderView(2, 1, "test-insert-channel", map[int64]int64{3: 2}, growingSegments)) + view := utils.CreateTestLeaderView(2, 1, "test-insert-channel", map[int64]int64{3: 2}, growingSegments) + view.TargetVersion = checker.targetMgr.GetCollectionTargetVersion(int64(1), meta.CurrentTarget) + checker.dist.LeaderViewManager.Update(2, view) checker.dist.SegmentDistManager.Update(2, utils.CreateTestSegment(1, 1, 3, 2, 2, "test-insert-channel")) tasks := checker.Check(context.TODO()) @@ -266,6 +314,52 @@ func (suite *SegmentCheckerTestSuite) TestReleaseGrowingSegments() { suite.Equal(tasks[1].Priority(), task.TaskPriorityNormal) } +func (suite *SegmentCheckerTestSuite) TestSkipReleaseGrowingSegments() { + checker := suite.checker + checker.meta.CollectionManager.PutCollection(utils.CreateTestCollection(1, 1)) + checker.meta.ReplicaManager.Put(utils.CreateTestReplica(1, 1, []int64{1, 2})) + + segments := []*datapb.SegmentInfo{} + channels := []*datapb.VchannelInfo{ + { + CollectionID: 1, + ChannelName: "test-insert-channel", + SeekPosition: &msgpb.MsgPosition{Timestamp: 10}, + }, + } + suite.broker.EXPECT().GetRecoveryInfoV2(mock.Anything, int64(1)).Return( + channels, segments, nil) + checker.targetMgr.UpdateCollectionNextTargetWithPartitions(int64(1), int64(1)) + checker.targetMgr.UpdateCollectionCurrentTarget(int64(1), int64(1)) + + growingSegments := make(map[int64]*meta.Segment) + growingSegments[2] = utils.CreateTestSegment(1, 1, 2, 2, 0, "test-insert-channel") + growingSegments[2].SegmentInfo.StartPosition = &msgpb.MsgPosition{Timestamp: 2} + + dmChannel := utils.CreateTestChannel(1, 2, 1, "test-insert-channel") + dmChannel.UnflushedSegmentIds = []int64{2, 3} + checker.dist.ChannelDistManager.Update(2, dmChannel) + view := utils.CreateTestLeaderView(2, 1, "test-insert-channel", map[int64]int64{}, growingSegments) + view.TargetVersion = checker.targetMgr.GetCollectionTargetVersion(int64(1), meta.CurrentTarget) - 1 + checker.dist.LeaderViewManager.Update(2, view) + + tasks := checker.Check(context.TODO()) + suite.Len(tasks, 0) + + view.TargetVersion = checker.targetMgr.GetCollectionTargetVersion(int64(1), meta.CurrentTarget) + checker.dist.LeaderViewManager.Update(2, view) + tasks = checker.Check(context.TODO()) + suite.Len(tasks, 1) + suite.Len(tasks[0].Actions(), 1) + action, ok := tasks[0].Actions()[0].(*task.SegmentAction) + suite.True(ok) + suite.EqualValues(1, tasks[0].ReplicaID()) + suite.Equal(task.ActionTypeReduce, action.Type()) + suite.EqualValues(2, action.SegmentID()) + suite.EqualValues(2, action.Node()) + suite.Equal(tasks[0].Priority(), task.TaskPriorityNormal) +} + func (suite *SegmentCheckerTestSuite) TestReleaseDroppedSegments() { checker := suite.checker checker.dist.SegmentDistManager.Update(1, utils.CreateTestSegment(1, 1, 1, 1, 1, "test-insert-channel")) diff --git a/internal/querycoordv2/dist/dist_handler.go b/internal/querycoordv2/dist/dist_handler.go index 0f6751747b..957ad280e1 100644 --- a/internal/querycoordv2/dist/dist_handler.go +++ b/internal/querycoordv2/dist/dist_handler.go @@ -201,6 +201,7 @@ func (dh *distHandler) updateLeaderView(resp *querypb.GetDataDistributionRespons Version: version, Segments: lview.GetSegmentDist(), GrowingSegments: segments, + TargetVersion: lview.TargetVersion, } updates = append(updates, view) } diff --git a/internal/querycoordv2/meta/leader_view_manager.go b/internal/querycoordv2/meta/leader_view_manager.go index 5fcb28f71e..8d5078492b 100644 --- a/internal/querycoordv2/meta/leader_view_manager.go +++ b/internal/querycoordv2/meta/leader_view_manager.go @@ -31,6 +31,7 @@ type LeaderView struct { Version int64 Segments map[int64]*querypb.SegmentDist GrowingSegments map[int64]*Segment + TargetVersion int64 } func (view *LeaderView) Clone() *LeaderView { @@ -51,6 +52,7 @@ func (view *LeaderView) Clone() *LeaderView { Version: view.Version, Segments: segments, GrowingSegments: growings, + TargetVersion: view.TargetVersion, } } diff --git a/internal/querycoordv2/meta/target.go b/internal/querycoordv2/meta/target.go index ea3d729401..de3e460343 100644 --- a/internal/querycoordv2/meta/target.go +++ b/internal/querycoordv2/meta/target.go @@ -17,6 +17,8 @@ package meta import ( + "time" + "github.com/samber/lo" "github.com/milvus-io/milvus/internal/proto/datapb" @@ -26,12 +28,14 @@ import ( type CollectionTarget struct { segments map[int64]*datapb.SegmentInfo dmChannels map[string]*DmChannel + version int64 } func NewCollectionTarget(segments map[int64]*datapb.SegmentInfo, dmChannels map[string]*DmChannel) *CollectionTarget { return &CollectionTarget{ segments: segments, dmChannels: dmChannels, + version: time.Now().UnixNano(), } } @@ -39,6 +43,10 @@ func (p *CollectionTarget) GetAllSegments() map[int64]*datapb.SegmentInfo { return p.segments } +func (p *CollectionTarget) GetTargetVersion() int64 { + return p.version +} + func (p *CollectionTarget) GetAllDmChannels() map[string]*DmChannel { return p.dmChannels } diff --git a/internal/querycoordv2/meta/target_manager.go b/internal/querycoordv2/meta/target_manager.go index 9d1cba92a5..3e7de9dec1 100644 --- a/internal/querycoordv2/meta/target_manager.go +++ b/internal/querycoordv2/meta/target_manager.go @@ -80,7 +80,9 @@ func (mgr *TargetManager) UpdateCollectionCurrentTarget(collectionID int64, part log.Debug("finish to update current target for collection", zap.Int64s("segments", newTarget.GetAllSegmentIDs()), - zap.Strings("channels", newTarget.GetAllDmChannelNames())) + zap.Strings("channels", newTarget.GetAllDmChannelNames()), + zap.Int64("version", newTarget.GetTargetVersion()), + ) return true } @@ -457,6 +459,18 @@ func (mgr *TargetManager) GetHistoricalSegment(collectionID int64, id int64, sco return collectionTarget.GetAllSegments()[id] } +func (mgr *TargetManager) GetCollectionTargetVersion(collectionID int64, scope TargetScope) int64 { + mgr.rwMutex.RLock() + defer mgr.rwMutex.RUnlock() + targetMap := mgr.getTarget(scope) + collectionTarget := targetMap.getCollectionTarget(collectionID) + + if collectionTarget == nil { + return 0 + } + return collectionTarget.GetTargetVersion() +} + func (mgr *TargetManager) IsCurrentTargetExist(collectionID int64) bool { newChannels := mgr.GetDmChannelsByCollection(collectionID, CurrentTarget) diff --git a/internal/querycoordv2/meta/target_manager_test.go b/internal/querycoordv2/meta/target_manager_test.go index ae3871eeda..2a68eeeff5 100644 --- a/internal/querycoordv2/meta/target_manager_test.go +++ b/internal/querycoordv2/meta/target_manager_test.go @@ -18,6 +18,7 @@ package meta import ( "testing" + "time" "github.com/cockroachdb/errors" "github.com/samber/lo" @@ -322,6 +323,25 @@ func (suite *TargetManagerSuite) assertSegments(expected []int64, actual map[int return suite.Len(set, 0) } +func (suite *TargetManagerSuite) TestGetCollectionTargetVersion() { + t1 := time.Now().UnixNano() + target := NewCollectionTarget(nil, nil) + t2 := time.Now().UnixNano() + + version := target.GetTargetVersion() + suite.True(t1 <= version) + suite.True(t2 >= version) + + collectionID := int64(1) + t3 := time.Now().UnixNano() + suite.mgr.updateCollectionNextTarget(collectionID) + t4 := time.Now().UnixNano() + + collectionVersion := suite.mgr.GetCollectionTargetVersion(collectionID, NextTarget) + suite.True(t3 <= collectionVersion) + suite.True(t4 >= collectionVersion) +} + func TestTargetManager(t *testing.T) { suite.Run(t, new(TargetManagerSuite)) } diff --git a/internal/querycoordv2/observers/collection_observer.go b/internal/querycoordv2/observers/collection_observer.go index 380a4bd4f6..87c905440b 100644 --- a/internal/querycoordv2/observers/collection_observer.go +++ b/internal/querycoordv2/observers/collection_observer.go @@ -38,6 +38,7 @@ type CollectionObserver struct { meta *meta.Meta targetMgr *meta.TargetManager targetObserver *TargetObserver + leaderObserver *LeaderObserver checkerController *checkers.CheckerController partitionLoadedCount map[int64]int @@ -49,6 +50,7 @@ func NewCollectionObserver( meta *meta.Meta, targetMgr *meta.TargetManager, targetObserver *TargetObserver, + leaderObserver *LeaderObserver, checherController *checkers.CheckerController, ) *CollectionObserver { return &CollectionObserver{ @@ -57,6 +59,7 @@ func NewCollectionObserver( meta: meta, targetMgr: targetMgr, targetObserver: targetObserver, + leaderObserver: leaderObserver, checkerController: checherController, partitionLoadedCount: make(map[int64]int), } @@ -203,7 +206,7 @@ func (ob *CollectionObserver) observePartitionLoadStatus(partition *meta.Partiti } ob.partitionLoadedCount[partition.GetPartitionID()] = loadedCount - if loadPercentage == 100 && ob.targetObserver.Check(partition.GetCollectionID()) { + if loadPercentage == 100 && ob.targetObserver.Check(partition.GetCollectionID()) && ob.leaderObserver.CheckTargetVersion(partition.GetCollectionID()) { delete(ob.partitionLoadedCount, partition.GetPartitionID()) } collectionPercentage, err := ob.meta.CollectionManager.UpdateLoadPercent(partition.PartitionID, loadPercentage) diff --git a/internal/querycoordv2/observers/collection_observer_test.go b/internal/querycoordv2/observers/collection_observer_test.go index b2a85f3382..7e0386a968 100644 --- a/internal/querycoordv2/observers/collection_observer_test.go +++ b/internal/querycoordv2/observers/collection_observer_test.go @@ -26,6 +26,7 @@ import ( "github.com/stretchr/testify/suite" clientv3 "go.etcd.io/etcd/client/v3" + "github.com/milvus-io/milvus-proto/go-api/v2/commonpb" "github.com/milvus-io/milvus/internal/kv" etcdkv "github.com/milvus-io/milvus/internal/kv/etcd" "github.com/milvus-io/milvus/internal/proto/datapb" @@ -63,6 +64,7 @@ type CollectionObserverSuite struct { meta *meta.Meta targetMgr *meta.TargetManager targetObserver *TargetObserver + leaderObserver *LeaderObserver checkerController *checkers.CheckerController // Test object @@ -192,12 +194,19 @@ func (suite *CollectionObserverSuite) SetupTest() { ) suite.checkerController = &checkers.CheckerController{} + mockCluster := session.NewMockCluster(suite.T()) + suite.leaderObserver = NewLeaderObserver(suite.dist, suite.meta, suite.targetMgr, suite.broker, mockCluster) + mockCluster.EXPECT().SyncDistribution(mock.Anything, mock.Anything, mock.Anything).Return(&commonpb.Status{ + ErrorCode: commonpb.ErrorCode_Success, + }, nil).Maybe() + // Test object suite.ob = NewCollectionObserver( suite.dist, suite.meta, suite.targetMgr, suite.targetObserver, + suite.leaderObserver, suite.checkerController, ) @@ -205,6 +214,7 @@ func (suite *CollectionObserverSuite) SetupTest() { suite.broker.EXPECT().GetPartitions(mock.Anything, collection).Return(suite.partitions[collection], nil).Maybe() } suite.targetObserver.Start(context.Background()) + suite.leaderObserver.Start(context.TODO()) suite.ob.Start(context.Background()) suite.loadAll() } diff --git a/internal/querycoordv2/observers/leader_observer.go b/internal/querycoordv2/observers/leader_observer.go index 2ba35c0bd2..3d8fdff302 100644 --- a/internal/querycoordv2/observers/leader_observer.go +++ b/internal/querycoordv2/observers/leader_observer.go @@ -30,6 +30,7 @@ import ( "github.com/milvus-io/milvus/internal/querycoordv2/utils" "github.com/milvus-io/milvus/pkg/log" "github.com/milvus-io/milvus/pkg/util/commonpbutil" + "github.com/samber/lo" ) const ( @@ -39,13 +40,14 @@ const ( // LeaderObserver is to sync the distribution with leader type LeaderObserver struct { - wg sync.WaitGroup - closeCh chan struct{} - dist *meta.DistributionManager - meta *meta.Meta - target *meta.TargetManager - broker meta.Broker - cluster session.Cluster + wg sync.WaitGroup + closeCh chan struct{} + dist *meta.DistributionManager + meta *meta.Meta + target *meta.TargetManager + broker meta.Broker + cluster session.Cluster + manualCheck chan checkRequest stopOnce sync.Once } @@ -64,6 +66,12 @@ func (o *LeaderObserver) Start(ctx context.Context) { case <-ctx.Done(): log.Info("stop leader observer due to ctx done") return + case req := <-o.manualCheck: + log.Info("triggering manual check") + ret := o.observeCollection(ctx, req.CollectionID) + req.Notifier <- ret + log.Info("manual check done", zap.Bool("result", ret)) + case <-ticker.C: o.observe(ctx) } @@ -89,8 +97,9 @@ func (o *LeaderObserver) observeSegmentsDist(ctx context.Context) { } } -func (o *LeaderObserver) observeCollection(ctx context.Context, collection int64) { +func (o *LeaderObserver) observeCollection(ctx context.Context, collection int64) bool { replicas := o.meta.ReplicaManager.GetByCollection(collection) + result := true for _, replica := range replicas { leaders := o.dist.ChannelDistManager.GetShardLeadersByReplica(replica) for ch, leaderID := range leaders { @@ -99,11 +108,55 @@ func (o *LeaderObserver) observeCollection(ctx context.Context, collection int64 continue } dists := o.dist.SegmentDistManager.GetByShardWithReplica(ch, replica) - needLoaded, needRemoved := o.findNeedLoadedSegments(leaderView, dists), - o.findNeedRemovedSegments(leaderView, dists) - o.sync(ctx, replica.GetID(), leaderView, append(needLoaded, needRemoved...)) + + actions := o.findNeedLoadedSegments(leaderView, dists) + actions = append(actions, o.findNeedRemovedSegments(leaderView, dists)...) + updateVersionAction := o.checkNeedUpdateTargetVersion(leaderView) + if updateVersionAction != nil { + actions = append(actions, updateVersionAction) + } + success := o.sync(ctx, replica.GetID(), leaderView, actions) + if !success { + result = false + } } } + return result +} + +func (ob *LeaderObserver) CheckTargetVersion(collectionID int64) bool { + notifier := make(chan bool) + ob.manualCheck <- checkRequest{ + CollectionID: collectionID, + Notifier: notifier, + } + return <-notifier +} + +func (o *LeaderObserver) checkNeedUpdateTargetVersion(leaderView *meta.LeaderView) *querypb.SyncAction { + targetVersion := o.target.GetCollectionTargetVersion(leaderView.CollectionID, meta.CurrentTarget) + + if targetVersion <= leaderView.TargetVersion { + return nil + } + + log.Info("Update readable segment version", + zap.Int64("collectionID", leaderView.CollectionID), + zap.String("channelName", leaderView.Channel), + zap.Int64("nodeID", leaderView.ID), + zap.Int64("oldVersion", leaderView.TargetVersion), + zap.Int64("newVersion", targetVersion), + ) + + sealedSegments := o.target.GetHistoricalSegmentsByCollection(leaderView.CollectionID, meta.CurrentTarget) + growingSegments := o.target.GetStreamingSegmentsByCollection(leaderView.CollectionID, meta.CurrentTarget) + + return &querypb.SyncAction{ + Type: querypb.SyncType_UpdateVersion, + GrowingInTarget: growingSegments.Collect(), + SealedInTarget: lo.Keys(sealedSegments), + TargetVersion: targetVersion, + } } func (o *LeaderObserver) findNeedLoadedSegments(leaderView *meta.LeaderView, dists []*meta.Segment) []*querypb.SyncAction { @@ -168,9 +221,9 @@ func (o *LeaderObserver) findNeedRemovedSegments(leaderView *meta.LeaderView, di return ret } -func (o *LeaderObserver) sync(ctx context.Context, replicaID int64, leaderView *meta.LeaderView, diffs []*querypb.SyncAction) { +func (o *LeaderObserver) sync(ctx context.Context, replicaID int64, leaderView *meta.LeaderView, diffs []*querypb.SyncAction) bool { if len(diffs) == 0 { - return + return true } log := log.With( @@ -182,12 +235,12 @@ func (o *LeaderObserver) sync(ctx context.Context, replicaID int64, leaderView * schema, err := o.broker.GetCollectionSchema(ctx, leaderView.CollectionID) if err != nil { log.Error("sync distribution failed, cannot get schema of collection", zap.Error(err)) - return + return false } partitions, err := utils.GetPartitions(o.meta.CollectionManager, leaderView.CollectionID) if err != nil { log.Error("sync distribution failed, cannot get partitions of collection", zap.Error(err)) - return + return false } req := &querypb.SyncDistributionRequest{ @@ -209,12 +262,15 @@ func (o *LeaderObserver) sync(ctx context.Context, replicaID int64, leaderView * resp, err := o.cluster.SyncDistribution(ctx, leaderView.ID, req) if err != nil { log.Error("failed to sync distribution", zap.Error(err)) - return + return false } if resp.ErrorCode != commonpb.ErrorCode_Success { log.Error("failed to sync distribution", zap.String("reason", resp.GetReason())) + return false } + + return true } func NewLeaderObserver( @@ -225,11 +281,12 @@ func NewLeaderObserver( cluster session.Cluster, ) *LeaderObserver { return &LeaderObserver{ - closeCh: make(chan struct{}), - dist: dist, - meta: meta, - target: targetMgr, - broker: broker, - cluster: cluster, + closeCh: make(chan struct{}), + dist: dist, + meta: meta, + target: targetMgr, + broker: broker, + cluster: cluster, + manualCheck: make(chan checkRequest, 10), } } diff --git a/internal/querycoordv2/observers/leader_observer_test.go b/internal/querycoordv2/observers/leader_observer_test.go index 00d95fdf98..a1e86bba20 100644 --- a/internal/querycoordv2/observers/leader_observer_test.go +++ b/internal/querycoordv2/observers/leader_observer_test.go @@ -73,6 +73,9 @@ func (suite *LeaderObserverTestSuite) SetupTest() { suite.broker = meta.NewMockBroker(suite.T()) suite.mockCluster = session.NewMockCluster(suite.T()) + // suite.mockCluster.EXPECT().SyncDistribution(mock.Anything, mock.Anything, mock.Anything).Return(&commonpb.Status{ + // ErrorCode: commonpb.ErrorCode_Success, + // }, nil).Maybe() distManager := meta.NewDistributionManager() targetManager := meta.NewTargetManager(suite.broker, suite.meta) suite.observer = NewLeaderObserver(distManager, suite.meta, targetManager, suite.broker, suite.mockCluster) @@ -120,7 +123,9 @@ func (suite *LeaderObserverTestSuite) TestSyncLoadedSegments() { observer.target.UpdateCollectionCurrentTarget(1) observer.dist.SegmentDistManager.Update(1, utils.CreateTestSegment(1, 1, 1, 2, 1, "test-insert-channel")) observer.dist.ChannelDistManager.Update(2, utils.CreateTestChannel(1, 2, 1, "test-insert-channel")) - observer.dist.LeaderViewManager.Update(2, utils.CreateTestLeaderView(2, 1, "test-insert-channel", map[int64]int64{}, map[int64]*meta.Segment{})) + view := utils.CreateTestLeaderView(2, 1, "test-insert-channel", map[int64]int64{}, map[int64]*meta.Segment{}) + view.TargetVersion = observer.target.GetCollectionTargetVersion(1, meta.CurrentTarget) + observer.dist.LeaderViewManager.Update(2, view) expectReqeustFunc := func(version int64) *querypb.SyncDistributionRequest { return &querypb.SyncDistributionRequest{ @@ -151,11 +156,10 @@ func (suite *LeaderObserverTestSuite) TestSyncLoadedSegments() { called := atomic.NewBool(false) suite.mockCluster.EXPECT().SyncDistribution(context.TODO(), int64(2), - mock.AnythingOfType("*querypb.SyncDistributionRequest")).Once(). - Run(func(args mock.Arguments) { - inputReq := (args[2]).(*querypb.SyncDistributionRequest) - assert.ElementsMatch(suite.T(), []*querypb.SyncDistributionRequest{inputReq}, - []*querypb.SyncDistributionRequest{expectReqeustFunc(inputReq.GetVersion())}) + mock.AnythingOfType("*querypb.SyncDistributionRequest")). + Run(func(ctx context.Context, nodeID int64, req *querypb.SyncDistributionRequest) { + assert.ElementsMatch(suite.T(), []*querypb.SyncDistributionRequest{req}, + []*querypb.SyncDistributionRequest{expectReqeustFunc(req.GetVersion())}) called.Store(true) }). Return(&commonpb.Status{}, nil) @@ -209,7 +213,9 @@ func (suite *LeaderObserverTestSuite) TestIgnoreSyncLoadedSegments() { observer.dist.SegmentDistManager.Update(1, utils.CreateTestSegment(1, 1, 1, 2, 1, "test-insert-channel"), utils.CreateTestSegment(1, 1, 2, 2, 1, "test-insert-channel")) observer.dist.ChannelDistManager.Update(2, utils.CreateTestChannel(1, 2, 1, "test-insert-channel")) - observer.dist.LeaderViewManager.Update(2, utils.CreateTestLeaderView(2, 1, "test-insert-channel", map[int64]int64{}, map[int64]*meta.Segment{})) + view := utils.CreateTestLeaderView(2, 1, "test-insert-channel", map[int64]int64{}, map[int64]*meta.Segment{}) + view.TargetVersion = observer.target.GetCollectionTargetVersion(1, meta.CurrentTarget) + observer.dist.LeaderViewManager.Update(2, view) expectReqeustFunc := func(version int64) *querypb.SyncDistributionRequest { return &querypb.SyncDistributionRequest{ @@ -238,11 +244,10 @@ func (suite *LeaderObserverTestSuite) TestIgnoreSyncLoadedSegments() { } } called := atomic.NewBool(false) - suite.mockCluster.EXPECT().SyncDistribution(context.TODO(), int64(2), mock.AnythingOfType("*querypb.SyncDistributionRequest")).Once(). - Run(func(args mock.Arguments) { - inputReq := (args[2]).(*querypb.SyncDistributionRequest) - assert.ElementsMatch(suite.T(), []*querypb.SyncDistributionRequest{inputReq}, - []*querypb.SyncDistributionRequest{expectReqeustFunc(inputReq.GetVersion())}) + suite.mockCluster.EXPECT().SyncDistribution(context.TODO(), int64(2), mock.AnythingOfType("*querypb.SyncDistributionRequest")). + Run(func(ctx context.Context, nodeID int64, req *querypb.SyncDistributionRequest) { + assert.ElementsMatch(suite.T(), []*querypb.SyncDistributionRequest{req}, + []*querypb.SyncDistributionRequest{expectReqeustFunc(req.GetVersion())}) called.Store(true) }). Return(&commonpb.Status{}, nil) @@ -290,6 +295,7 @@ func (suite *LeaderObserverTestSuite) TestIgnoreBalancedSegment() { NodeID: 2, Version: 2, } + leaderView.TargetVersion = observer.target.GetCollectionTargetVersion(1, meta.CurrentTarget) observer.dist.LeaderViewManager.Update(2, leaderView) observer.Start(context.TODO()) @@ -336,8 +342,12 @@ func (suite *LeaderObserverTestSuite) TestSyncLoadedSegmentsWithReplicas() { observer.dist.SegmentDistManager.Update(1, utils.CreateTestSegment(1, 1, 1, 1, 1, "test-insert-channel")) observer.dist.SegmentDistManager.Update(4, utils.CreateTestSegment(1, 1, 1, 4, 2, "test-insert-channel")) observer.dist.ChannelDistManager.Update(2, utils.CreateTestChannel(1, 2, 1, "test-insert-channel")) - observer.dist.LeaderViewManager.Update(2, utils.CreateTestLeaderView(2, 1, "test-insert-channel", map[int64]int64{}, map[int64]*meta.Segment{})) - observer.dist.LeaderViewManager.Update(4, utils.CreateTestLeaderView(4, 1, "test-insert-channel", map[int64]int64{1: 4}, map[int64]*meta.Segment{})) + view := utils.CreateTestLeaderView(2, 1, "test-insert-channel", map[int64]int64{}, map[int64]*meta.Segment{}) + view.TargetVersion = observer.target.GetCollectionTargetVersion(1, meta.CurrentTarget) + observer.dist.LeaderViewManager.Update(2, view) + view2 := utils.CreateTestLeaderView(4, 1, "test-insert-channel", map[int64]int64{1: 4}, map[int64]*meta.Segment{}) + view.TargetVersion = observer.target.GetCollectionTargetVersion(1, meta.CurrentTarget) + observer.dist.LeaderViewManager.Update(4, view2) expectReqeustFunc := func(version int64) *querypb.SyncDistributionRequest { return &querypb.SyncDistributionRequest{ @@ -367,11 +377,10 @@ func (suite *LeaderObserverTestSuite) TestSyncLoadedSegmentsWithReplicas() { } called := atomic.NewBool(false) suite.mockCluster.EXPECT().SyncDistribution(context.TODO(), int64(2), - mock.AnythingOfType("*querypb.SyncDistributionRequest")).Once(). - Run(func(args mock.Arguments) { - inputReq := (args[2]).(*querypb.SyncDistributionRequest) - assert.ElementsMatch(suite.T(), []*querypb.SyncDistributionRequest{inputReq}, - []*querypb.SyncDistributionRequest{expectReqeustFunc(inputReq.GetVersion())}) + mock.AnythingOfType("*querypb.SyncDistributionRequest")). + Run(func(ctx context.Context, nodeID int64, req *querypb.SyncDistributionRequest) { + assert.ElementsMatch(suite.T(), []*querypb.SyncDistributionRequest{req}, + []*querypb.SyncDistributionRequest{expectReqeustFunc(req.GetVersion())}) called.Store(true) }). Return(&commonpb.Status{}, nil) @@ -423,11 +432,10 @@ func (suite *LeaderObserverTestSuite) TestSyncRemovedSegments() { } ch := make(chan struct{}) suite.mockCluster.EXPECT().SyncDistribution(context.TODO(), int64(2), - mock.AnythingOfType("*querypb.SyncDistributionRequest")).Once(). - Run(func(args mock.Arguments) { - inputReq := (args[2]).(*querypb.SyncDistributionRequest) - assert.ElementsMatch(suite.T(), []*querypb.SyncDistributionRequest{inputReq}, - []*querypb.SyncDistributionRequest{expectReqeustFunc(inputReq.GetVersion())}) + mock.AnythingOfType("*querypb.SyncDistributionRequest")). + Run(func(ctx context.Context, nodeID int64, req *querypb.SyncDistributionRequest) { + assert.ElementsMatch(suite.T(), []*querypb.SyncDistributionRequest{req}, + []*querypb.SyncDistributionRequest{expectReqeustFunc(req.GetVersion())}) close(ch) }). Return(&commonpb.Status{}, nil) @@ -492,11 +500,10 @@ func (suite *LeaderObserverTestSuite) TestIgnoreSyncRemovedSegments() { } } called := atomic.NewBool(false) - suite.mockCluster.EXPECT().SyncDistribution(context.TODO(), int64(2), mock.AnythingOfType("*querypb.SyncDistributionRequest")).Once(). - Run(func(args mock.Arguments) { - inputReq := (args[2]).(*querypb.SyncDistributionRequest) - assert.ElementsMatch(suite.T(), []*querypb.SyncDistributionRequest{inputReq}, - []*querypb.SyncDistributionRequest{expectReqeustFunc(inputReq.GetVersion())}) + suite.mockCluster.EXPECT().SyncDistribution(context.TODO(), int64(2), mock.AnythingOfType("*querypb.SyncDistributionRequest")). + Run(func(ctx context.Context, nodeID int64, req *querypb.SyncDistributionRequest) { + assert.ElementsMatch(suite.T(), []*querypb.SyncDistributionRequest{req}, + []*querypb.SyncDistributionRequest{expectReqeustFunc(req.GetVersion())}) called.Store(true) }). Return(&commonpb.Status{}, nil) @@ -510,6 +517,47 @@ func (suite *LeaderObserverTestSuite) TestIgnoreSyncRemovedSegments() { ) } +func (suite *LeaderObserverTestSuite) TestSyncTargetVersion() { + collectionID := int64(1001) + + observer := suite.observer + observer.meta.CollectionManager.PutCollection(utils.CreateTestCollection(collectionID, 1)) + observer.meta.ReplicaManager.Put(utils.CreateTestReplica(1, collectionID, []int64{1, 2})) + + nextTargetChannels := []*datapb.VchannelInfo{ + { + CollectionID: collectionID, + ChannelName: "channel-1", + UnflushedSegmentIds: []int64{22, 33}, + }, + } + + nextTargetSegments := []*datapb.SegmentInfo{ + { + ID: 11, + PartitionID: 1, + InsertChannel: "channel-1", + }, + } + + suite.broker.EXPECT().GetRecoveryInfoV2(mock.Anything, collectionID).Return(nextTargetChannels, nextTargetSegments, nil) + suite.observer.target.UpdateCollectionNextTargetWithPartitions(collectionID, 1) + suite.observer.target.UpdateCollectionCurrentTarget(collectionID) + TargetVersion := suite.observer.target.GetCollectionTargetVersion(collectionID, meta.CurrentTarget) + + view := utils.CreateTestLeaderView(1, collectionID, "test-channel", nil, nil) + view.TargetVersion = TargetVersion + action := observer.checkNeedUpdateTargetVersion(view) + suite.Nil(action) + + view.TargetVersion = TargetVersion - 1 + action = observer.checkNeedUpdateTargetVersion(view) + suite.NotNil(action) + suite.Equal(querypb.SyncType_UpdateVersion, action.Type) + suite.Len(action.GrowingInTarget, 2) + suite.Len(action.SealedInTarget, 1) +} + func TestLeaderObserverSuite(t *testing.T) { suite.Run(t, new(LeaderObserverTestSuite)) } diff --git a/internal/querycoordv2/server.go b/internal/querycoordv2/server.go index 42ccc37d8d..3b162a2d47 100644 --- a/internal/querycoordv2/server.go +++ b/internal/querycoordv2/server.go @@ -360,6 +360,7 @@ func (s *Server) initObserver() { s.meta, s.targetMgr, s.targetObserver, + s.leaderObserver, s.checkerController, ) diff --git a/internal/querycoordv2/server_test.go b/internal/querycoordv2/server_test.go index e477b1322e..93b92b1fa5 100644 --- a/internal/querycoordv2/server_test.go +++ b/internal/querycoordv2/server_test.go @@ -517,6 +517,7 @@ func (suite *ServerSuite) hackServer() { suite.server.meta, suite.server.targetMgr, suite.server.targetObserver, + suite.server.leaderObserver, suite.server.checkerController, ) diff --git a/internal/querycoordv2/services_test.go b/internal/querycoordv2/services_test.go index c600627dd6..05da4e1f38 100644 --- a/internal/querycoordv2/services_test.go +++ b/internal/querycoordv2/services_test.go @@ -189,6 +189,7 @@ func (suite *ServiceSuite) SetupTest() { suite.server.meta, suite.server.targetMgr, suite.targetObserver, + suite.server.leaderObserver, &checkers.CheckerController{}, ) diff --git a/internal/querynodev2/delegator/delegator.go b/internal/querynodev2/delegator/delegator.go index dc948990fc..80c519f1b6 100644 --- a/internal/querynodev2/delegator/delegator.go +++ b/internal/querynodev2/delegator/delegator.go @@ -77,7 +77,7 @@ func newLifetime() *lifetime { type ShardDelegator interface { Collection() int64 Version() int64 - GetSegmentInfo() (sealed []SnapshotItem, growing []SegmentEntry) + GetSegmentInfo(readable bool) (sealed []SnapshotItem, growing []SegmentEntry) SyncDistribution(ctx context.Context, entries ...SegmentEntry) Search(ctx context.Context, req *querypb.SearchRequest) ([]*internalpb.SearchResults, error) Query(ctx context.Context, req *querypb.QueryRequest) ([]*internalpb.RetrieveResults, error) @@ -89,6 +89,8 @@ type ShardDelegator interface { LoadGrowing(ctx context.Context, infos []*querypb.SegmentLoadInfo, version int64) error LoadSegments(ctx context.Context, req *querypb.LoadSegmentsRequest) error ReleaseSegments(ctx context.Context, req *querypb.ReleaseSegmentsRequest, force bool) error + SyncTargetVersion(newVersion int64, growingInTarget []int64, sealedInTarget []int64) + GetTargetVersion() int64 // control Serviceable() bool @@ -164,8 +166,8 @@ func (sd *shardDelegator) Version() int64 { } // GetSegmentInfo returns current segment distribution snapshot. -func (sd *shardDelegator) GetSegmentInfo() ([]SnapshotItem, []SegmentEntry) { - return sd.distribution.Peek() +func (sd *shardDelegator) GetSegmentInfo(readable bool) ([]SnapshotItem, []SegmentEntry) { + return sd.distribution.PeekSegments(readable) } // SyncDistribution revises distribution. @@ -227,7 +229,7 @@ func (sd *shardDelegator) Search(ctx context.Context, req *querypb.SearchRequest fmt.Sprint(paramtable.GetNodeID()), metrics.SearchLabel). Observe(float64(waitTr.ElapseSpan().Milliseconds())) - sealed, growing, version := sd.distribution.GetCurrent(req.GetReq().GetPartitionIDs()...) + sealed, growing, version := sd.distribution.GetSegments(true, req.GetReq().GetPartitionIDs()...) defer sd.distribution.FinishUsage(version) existPartitions := sd.collection.GetPartitions() growing = lo.Filter(growing, func(segment SegmentEntry, _ int) bool { @@ -286,7 +288,7 @@ func (sd *shardDelegator) Query(ctx context.Context, req *querypb.QueryRequest) fmt.Sprint(paramtable.GetNodeID()), metrics.QueryLabel). Observe(float64(waitTr.ElapseSpan().Milliseconds())) - sealed, growing, version := sd.distribution.GetCurrent(req.GetReq().GetPartitionIDs()...) + sealed, growing, version := sd.distribution.GetSegments(true, req.GetReq().GetPartitionIDs()...) defer sd.distribution.FinishUsage(version) existPartitions := sd.collection.GetPartitions() growing = lo.Filter(growing, func(segment SegmentEntry, _ int) bool { @@ -340,7 +342,7 @@ func (sd *shardDelegator) GetStatistics(ctx context.Context, req *querypb.GetSta return nil, err } - sealed, growing, version := sd.distribution.GetCurrent(req.Req.GetPartitionIDs()...) + sealed, growing, version := sd.distribution.GetSegments(true, req.Req.GetPartitionIDs()...) defer sd.distribution.FinishUsage(version) tasks, err := organizeSubTask(req, sealed, growing, sd.workerManager, func(req *querypb.GetStatisticsRequest, scope querypb.DataScope, segmentIDs []int64, targetID int64) *querypb.GetStatisticsRequest { diff --git a/internal/querynodev2/delegator/delegator_data.go b/internal/querynodev2/delegator/delegator_data.go index caa75a2ea4..bb3025122e 100644 --- a/internal/querynodev2/delegator/delegator_data.go +++ b/internal/querynodev2/delegator/delegator_data.go @@ -86,10 +86,11 @@ func (sd *shardDelegator) newGrowing(segmentID int64, insertData *InsertData) se sd.segmentManager.Put(segments.SegmentTypeGrowing, segment) sd.addGrowing(SegmentEntry{ - NodeID: paramtable.GetNodeID(), - SegmentID: segmentID, - PartitionID: insertData.PartitionID, - Version: 0, + NodeID: paramtable.GetNodeID(), + SegmentID: segmentID, + PartitionID: insertData.PartitionID, + Version: 0, + TargetVersion: initialTargetVersion, }) return segment } @@ -173,7 +174,7 @@ func (sd *shardDelegator) ProcessDelete(deleteData []*DeleteData, ts uint64) { offlineSegments := typeutil.NewConcurrentSet[int64]() - sealed, growing, version := sd.distribution.GetCurrent() + sealed, growing, version := sd.distribution.GetSegments(false) eg, ctx := errgroup.WithContext(context.Background()) for _, entry := range sealed { @@ -296,10 +297,11 @@ func (sd *shardDelegator) LoadGrowing(ctx context.Context, infos []*querypb.Segm } sd.addGrowing(lo.Map(loaded, func(segment segments.Segment, _ int) SegmentEntry { return SegmentEntry{ - NodeID: paramtable.GetNodeID(), - SegmentID: segment.ID(), - PartitionID: segment.Partition(), - Version: version, + NodeID: paramtable.GetNodeID(), + SegmentID: segment.ID(), + PartitionID: segment.Partition(), + Version: version, + TargetVersion: sd.distribution.getTargetVersion(), } })...) return nil @@ -351,37 +353,14 @@ func (sd *shardDelegator) LoadSegments(ctx context.Context, req *querypb.LoadSeg // alter distribution entries := lo.Map(req.GetInfos(), func(info *querypb.SegmentLoadInfo, _ int) SegmentEntry { return SegmentEntry{ - SegmentID: info.GetSegmentID(), - PartitionID: info.GetPartitionID(), - NodeID: req.GetDstNodeID(), - Version: req.GetVersion(), + SegmentID: info.GetSegmentID(), + PartitionID: info.GetPartitionID(), + NodeID: req.GetDstNodeID(), + Version: req.GetVersion(), + TargetVersion: sd.distribution.getTargetVersion(), } }) - removed, signal := sd.distribution.AddDistributions(entries...) - - // release possible matched growing segments async - if len(removed) > 0 { - go func() { - <-signal - worker, err := sd.workerManager.GetWorker(paramtable.GetNodeID()) - if err != nil { - log.Warn("failed to get local worker when try to release related growing", zap.Error(err)) - return - } - err = worker.ReleaseSegments(context.Background(), &querypb.ReleaseSegmentsRequest{ - Base: commonpbutil.NewMsgBase(commonpbutil.WithTargetID(paramtable.GetNodeID())), - CollectionID: sd.collectionID, - NodeID: paramtable.GetNodeID(), - Scope: querypb.DataScope_Streaming, - SegmentIDs: removed, - Shard: sd.vchannelName, - NeedTransfer: false, - }) - if err != nil { - log.Warn("failed to call release segments(local)", zap.Error(err)) - } - }() - } + sd.distribution.AddDistributions(entries...) return nil } @@ -615,3 +594,11 @@ func (sd *shardDelegator) ReleaseSegments(ctx context.Context, req *querypb.Rele return nil } + +func (sd *shardDelegator) SyncTargetVersion(newVersion int64, growingInTarget []int64, sealedInTarget []int64) { + sd.distribution.SyncTargetVersion(newVersion, growingInTarget, sealedInTarget) +} + +func (sd *shardDelegator) GetTargetVersion() int64 { + return sd.distribution.getTargetVersion() +} diff --git a/internal/querynodev2/delegator/delegator_data_test.go b/internal/querynodev2/delegator/delegator_data_test.go index 31428d3216..5a8cffada4 100644 --- a/internal/querynodev2/delegator/delegator_data_test.go +++ b/internal/querynodev2/delegator/delegator_data_test.go @@ -345,7 +345,7 @@ func (s *DelegatorDataSuite) TestLoadSegments() { }) s.NoError(err) - sealed, _ := s.delegator.GetSegmentInfo() + sealed, _ := s.delegator.GetSegmentInfo(false) s.Require().Equal(1, len(sealed)) s.Equal(int64(1), sealed[0].NodeID) s.ElementsMatch([]SegmentEntry{ @@ -421,7 +421,7 @@ func (s *DelegatorDataSuite) TestLoadSegments() { }) s.NoError(err) - sealed, _ := s.delegator.GetSegmentInfo() + sealed, _ := s.delegator.GetSegmentInfo(false) s.Require().Equal(1, len(sealed)) s.Equal(int64(1), sealed[0].NodeID) s.ElementsMatch([]SegmentEntry{ @@ -625,7 +625,7 @@ func (s *DelegatorDataSuite) TestReleaseSegment() { }) s.Require().NoError(err) - sealed, growing := s.delegator.GetSegmentInfo() + sealed, growing := s.delegator.GetSegmentInfo(false) s.Require().Equal(1, len(sealed)) s.Equal(int64(1), sealed[0].NodeID) s.ElementsMatch([]SegmentEntry{ @@ -652,7 +652,7 @@ func (s *DelegatorDataSuite) TestReleaseSegment() { }, false) s.NoError(err) - sealed, _ = s.delegator.GetSegmentInfo() + sealed, _ = s.delegator.GetSegmentInfo(false) s.Equal(0, len(sealed)) err = s.delegator.ReleaseSegments(ctx, &querypb.ReleaseSegmentsRequest{ @@ -663,7 +663,7 @@ func (s *DelegatorDataSuite) TestReleaseSegment() { }, false) s.NoError(err) - _, growing = s.delegator.GetSegmentInfo() + _, growing = s.delegator.GetSegmentInfo(false) s.Equal(0, len(growing)) err = s.delegator.ReleaseSegments(ctx, &querypb.ReleaseSegmentsRequest{ @@ -687,6 +687,11 @@ func (s *DelegatorDataSuite) TestReleaseSegment() { s.NoError(err) } +func (s *DelegatorSuite) TestSyncTargetVersion() { + s.delegator.SyncTargetVersion(int64(5), []int64{}, []int64{}) + s.Equal(int64(5), s.delegator.GetTargetVersion()) +} + func TestDelegatorDataSuite(t *testing.T) { suite.Run(t, new(DelegatorDataSuite)) } diff --git a/internal/querynodev2/delegator/delegator_test.go b/internal/querynodev2/delegator/delegator_test.go index b3b65e5d28..a108b41450 100644 --- a/internal/querynodev2/delegator/delegator_test.go +++ b/internal/querynodev2/delegator/delegator_test.go @@ -174,7 +174,7 @@ func (s *DelegatorSuite) TestBasicInfo() { } func (s *DelegatorSuite) TestGetSegmentInfo() { - sealed, growing := s.delegator.GetSegmentInfo() + sealed, growing := s.delegator.GetSegmentInfo(false) s.Equal(0, len(sealed)) s.Equal(0, len(growing)) @@ -185,7 +185,7 @@ func (s *DelegatorSuite) TestGetSegmentInfo() { Version: 2001, }) - sealed, growing = s.delegator.GetSegmentInfo() + sealed, growing = s.delegator.GetSegmentInfo(false) s.EqualValues([]SnapshotItem{ { NodeID: 1, @@ -241,6 +241,7 @@ func (s *DelegatorSuite) TestSearch() { Version: 2001, }, ) + s.delegator.SyncTargetVersion(2001, []int64{1004}, []int64{1000, 1001, 1002, 1003}) s.Run("normal", func() { defer func() { s.workerManager.ExpectedCalls = nil @@ -493,6 +494,7 @@ func (s *DelegatorSuite) TestQuery() { Version: 2001, }, ) + s.delegator.SyncTargetVersion(2001, []int64{1004}, []int64{1000, 1001, 1002, 1003}) s.Run("normal", func() { defer func() { s.workerManager.ExpectedCalls = nil @@ -712,6 +714,8 @@ func (s *DelegatorSuite) TestGetStats() { Version: 2001, }, ) + + s.delegator.SyncTargetVersion(2001, []int64{1004}, []int64{1000, 1001, 1002, 1003}) s.Run("normal", func() { defer func() { s.workerManager.ExpectedCalls = nil diff --git a/internal/querynodev2/delegator/distribution.go b/internal/querynodev2/delegator/distribution.go index 544cf0ce36..da064bd96e 100644 --- a/internal/querynodev2/delegator/distribution.go +++ b/internal/querynodev2/delegator/distribution.go @@ -20,13 +20,17 @@ import ( "sync" "go.uber.org/atomic" + "go.uber.org/zap" + "github.com/milvus-io/milvus/pkg/log" "github.com/milvus-io/milvus/pkg/util/typeutil" + "github.com/samber/lo" ) const ( // wildcardNodeID matches any nodeID, used for force distribution correction. - wildcardNodeID = int64(-1) + wildcardNodeID = int64(-1) + initialTargetVersion = int64(0) ) var ( @@ -47,11 +51,12 @@ func getClosedCh() chan struct{} { type distribution struct { // segments information // map[SegmentID]=>segmentEntry + targetVersion *atomic.Int64 growingSegments map[UniqueID]SegmentEntry sealedSegments map[UniqueID]SegmentEntry - // version indicator - version int64 + // snapshotVersion indicator + snapshotVersion int64 // quick flag for current snapshot is serviceable serviceable *atomic.Bool offlines typeutil.Set[int64] @@ -66,10 +71,11 @@ type distribution struct { // SegmentEntry stores the segment meta information. type SegmentEntry struct { - NodeID int64 - SegmentID UniqueID - PartitionID UniqueID - Version int64 + NodeID int64 + SegmentID UniqueID + PartitionID UniqueID + Version int64 + TargetVersion int64 } // NewDistribution creates a new distribution instance with all field initialized. @@ -81,20 +87,59 @@ func NewDistribution() *distribution { snapshots: typeutil.NewConcurrentMap[int64, *snapshot](), current: atomic.NewPointer[snapshot](nil), offlines: typeutil.NewSet[int64](), + targetVersion: atomic.NewInt64(initialTargetVersion), } dist.genSnapshot() return dist } -// GetCurrent returns current snapshot. -func (d *distribution) GetCurrent(partitions ...int64) (sealed []SnapshotItem, growing []SegmentEntry, version int64) { +// GetAllSegments returns segments in current snapshot, filter readable segment when readable is true +func (d *distribution) GetSegments(readable bool, partitions ...int64) (sealed []SnapshotItem, growing []SegmentEntry, version int64) { d.mut.RLock() defer d.mut.RUnlock() current := d.current.Load() sealed, growing = current.Get(partitions...) version = current.version + + if readable { + TargetVersion := current.GetTargetVersion() + sealed, growing = d.filterReadableSegments(sealed, growing, TargetVersion) + return + } + + return +} + +func (d *distribution) filterReadableSegments(sealed []SnapshotItem, growing []SegmentEntry, targetVersion int64) ([]SnapshotItem, []SegmentEntry) { + filterReadable := func(entry SegmentEntry, _ int) bool { + return entry.TargetVersion == targetVersion || entry.TargetVersion == initialTargetVersion + } + + growing = lo.Filter(growing, filterReadable) + sealed = lo.Map(sealed, func(item SnapshotItem, _ int) SnapshotItem { + return SnapshotItem{ + NodeID: item.NodeID, + Segments: lo.Filter(item.Segments, filterReadable), + } + }) + + return sealed, growing +} + +// PeekAllSegments returns current snapshot without increasing inuse count +// show only used by GetDataDistribution. +func (d *distribution) PeekSegments(readable bool, partitions ...int64) (sealed []SnapshotItem, growing []SegmentEntry) { + current := d.current.Load() + sealed, growing = current.Peek(partitions...) + + if readable { + TargetVersion := current.GetTargetVersion() + sealed, growing = d.filterReadableSegments(sealed, growing, TargetVersion) + return + } + return } @@ -106,12 +151,9 @@ func (d *distribution) FinishUsage(version int64) { } } -// Peek returns current snapshot without increasing inuse count -// show only used by GetDataDistribution. -func (d *distribution) Peek(partitions ...int64) (sealed []SnapshotItem, growing []SegmentEntry) { +func (d *distribution) getTargetVersion() int64 { current := d.current.Load() - sealed, growing = current.Peek(partitions...) - return sealed, growing + return current.GetTargetVersion() } // Serviceable returns wether current snapshot is serviceable. @@ -120,28 +162,16 @@ func (d *distribution) Serviceable() bool { } // AddDistributions add multiple segment entries. -func (d *distribution) AddDistributions(entries ...SegmentEntry) ([]int64, chan struct{}) { +func (d *distribution) AddDistributions(entries ...SegmentEntry) { d.mut.Lock() defer d.mut.Unlock() - // remove growing if sealed is loaded - var removed []int64 for _, entry := range entries { d.sealedSegments[entry.SegmentID] = entry d.offlines.Remove(entry.SegmentID) - _, ok := d.growingSegments[entry.SegmentID] - if ok { - removed = append(removed, entry.SegmentID) - delete(d.growingSegments, entry.SegmentID) - } } - ch := d.genSnapshot() - // no offline growing, return closed ch to skip wait - if len(removed) == 0 { - return removed, getClosedCh() - } - return removed, ch + d.genSnapshot() } // AddGrowing adds growing segment distribution. @@ -176,6 +206,45 @@ func (d *distribution) AddOfflines(segmentIDs ...int64) { } } +// UpdateTargetVersion update readable segment version +func (d *distribution) SyncTargetVersion(newVersion int64, growingInTarget []int64, sealedInTarget []int64) { + d.mut.Lock() + defer d.mut.Unlock() + for _, segmentID := range growingInTarget { + entry, ok := d.growingSegments[segmentID] + if !ok { + log.Error("readable growing segment lost, make it unserviceable", + zap.Int64("segmentID", segmentID)) + d.serviceable.Store(false) + continue + } + entry.TargetVersion = newVersion + d.growingSegments[segmentID] = entry + } + + for _, segmentID := range sealedInTarget { + entry, ok := d.sealedSegments[segmentID] + if !ok { + log.Error("readable sealed segment lost, make it unserviceable", + zap.Int64("segmentID", segmentID)) + d.serviceable.Store(false) + continue + } + entry.TargetVersion = newVersion + d.sealedSegments[segmentID] = entry + } + + oldValue := d.targetVersion.Load() + d.targetVersion.Store(newVersion) + d.genSnapshot() + log.Info("Update readable segment version", + zap.Int64("oldVersion", oldValue), + zap.Int64("newVersion", newVersion), + zap.Int64s("growing", growingInTarget), + zap.Int64s("sealed", sealedInTarget), + ) +} + // RemoveDistributions remove segments distributions and returns the clear signal channel. func (d *distribution) RemoveDistributions(sealedSegments []SegmentEntry, growingSegments []SegmentEntry) chan struct{} { d.mut.Lock() @@ -219,7 +288,6 @@ func (d *distribution) RemoveDistributions(sealedSegments []SegmentEntry, growin // in which, user could use found nodeID=>segmentID list. // mutex RLock is required before calling this method. func (d *distribution) genSnapshot() chan struct{} { - nodeSegments := make(map[int64][]SegmentEntry) for _, entry := range d.sealedSegments { nodeSegments[entry.NodeID] = append(nodeSegments[entry.NodeID], entry) @@ -243,12 +311,12 @@ func (d *distribution) genSnapshot() chan struct{} { // stores last snapshot // ok to be nil last := d.current.Load() - // increase version - d.version++ - newSnapShot := NewSnapshot(dist, growing, last, d.version) + // update snapshot version + d.snapshotVersion++ + newSnapShot := NewSnapshot(dist, growing, last, d.snapshotVersion, d.targetVersion.Load()) d.current.Store(newSnapShot) // shall be a new one - d.snapshots.GetOrInsert(d.version, newSnapShot) + d.snapshots.GetOrInsert(d.snapshotVersion, newSnapShot) // first snapshot, return closed chan if last == nil { diff --git a/internal/querynodev2/delegator/distribution_test.go b/internal/querynodev2/delegator/distribution_test.go index 8dae155810..d449b82b4e 100644 --- a/internal/querynodev2/delegator/distribution_test.go +++ b/internal/querynodev2/delegator/distribution_test.go @@ -30,6 +30,7 @@ type DistributionSuite struct { func (s *DistributionSuite) SetupTest() { s.dist = NewDistribution() + s.Equal(initialTargetVersion, s.dist.getTargetVersion()) } func (s *DistributionSuite) TearDownTest() { @@ -145,11 +146,10 @@ func (s *DistributionSuite) TestAddDistribution() { s.SetupTest() defer s.TearDownTest() s.dist.AddGrowing(tc.growing...) - _, _, version := s.dist.GetCurrent() - _, signal := s.dist.AddDistributions(tc.input...) - sealed, _ := s.dist.Peek() + _, _, version := s.dist.GetSegments(false) + s.dist.AddDistributions(tc.input...) + sealed, _ := s.dist.PeekSegments(false) s.compareSnapshotItems(tc.expected, sealed) - s.Equal(tc.expectedSignalClosed, s.isClosedCh(signal)) s.dist.FinishUsage(version) }) } @@ -215,7 +215,7 @@ func (s *DistributionSuite) TestAddGrowing() { defer s.TearDownTest() s.dist.AddGrowing(tc.input...) - _, growing, version := s.dist.GetCurrent() + _, growing, version := s.dist.GetSegments(false) defer s.dist.FinishUsage(version) s.ElementsMatch(tc.expected, growing) @@ -394,7 +394,7 @@ func (s *DistributionSuite) TestRemoveDistribution() { var version int64 if tc.withMockRead { - _, _, version = s.dist.GetCurrent() + _, _, version = s.dist.GetSegments(false) } ch := s.dist.RemoveDistributions(tc.removalSealed, tc.removalGrowing) @@ -418,7 +418,7 @@ func (s *DistributionSuite) TestRemoveDistribution() { case <-ch: } - sealed, growing, version := s.dist.GetCurrent() + sealed, growing, version := s.dist.GetSegments(false) defer s.dist.FinishUsage(version) s.compareSnapshotItems(tc.expectSealed, sealed) s.ElementsMatch(tc.expectGrowing, growing) @@ -514,7 +514,7 @@ func (s *DistributionSuite) TestPeek() { // peek during lock s.dist.AddDistributions(tc.input...) s.dist.mut.Lock() - sealed, _ := s.dist.Peek() + sealed, _ := s.dist.PeekSegments(false) s.compareSnapshotItems(tc.expected, sealed) s.dist.mut.Unlock() }) @@ -577,6 +577,62 @@ func (s *DistributionSuite) TestAddOfflines() { } } +func (s *DistributionSuite) Test_SyncTargetVersion() { + growing := []SegmentEntry{ + { + NodeID: 1, + SegmentID: 1, + PartitionID: 1, + TargetVersion: 1, + }, + { + NodeID: 1, + SegmentID: 2, + PartitionID: 1, + TargetVersion: 1, + }, + { + NodeID: 1, + SegmentID: 3, + PartitionID: 1, + TargetVersion: 1, + }, + } + + sealed := []SegmentEntry{ + { + NodeID: 1, + SegmentID: 4, + PartitionID: 1, + TargetVersion: 1, + }, + { + NodeID: 1, + SegmentID: 5, + PartitionID: 1, + TargetVersion: 1, + }, + { + NodeID: 1, + SegmentID: 6, + PartitionID: 1, + TargetVersion: 1, + }, + } + + s.dist.AddGrowing(growing...) + s.dist.AddDistributions(sealed...) + s.dist.SyncTargetVersion(2, []int64{2, 3}, []int64{6}) + + s1, s2, _ := s.dist.GetSegments(true) + s.Len(s1[0].Segments, 1) + s.Len(s2, 2) + + s1, s2, _ = s.dist.GetSegments(false) + s.Len(s1[0].Segments, 3) + s.Len(s2, 3) +} + func TestDistributionSuite(t *testing.T) { suite.Run(t, new(DistributionSuite)) } diff --git a/internal/querynodev2/delegator/mock_delegator.go b/internal/querynodev2/delegator/mock_delegator.go index f922a1018a..e4333012fa 100644 --- a/internal/querynodev2/delegator/mock_delegator.go +++ b/internal/querynodev2/delegator/mock_delegator.go @@ -1,4 +1,4 @@ -// Code generated by mockery v2.16.0. DO NOT EDIT. +// Code generated by mockery v2.21.1. DO NOT EDIT. package delegator @@ -51,6 +51,11 @@ func (_c *MockShardDelegator_Close_Call) Return() *MockShardDelegator_Close_Call return _c } +func (_c *MockShardDelegator_Close_Call) RunAndReturn(run func()) *MockShardDelegator_Close_Call { + _c.Call.Return(run) + return _c +} + // Collection provides a mock function with given fields: func (_m *MockShardDelegator) Collection() int64 { ret := _m.Called() @@ -87,22 +92,30 @@ func (_c *MockShardDelegator_Collection_Call) Return(_a0 int64) *MockShardDelega return _c } -// GetSegmentInfo provides a mock function with given fields: -func (_m *MockShardDelegator) GetSegmentInfo() ([]SnapshotItem, []SegmentEntry) { - ret := _m.Called() +func (_c *MockShardDelegator_Collection_Call) RunAndReturn(run func() int64) *MockShardDelegator_Collection_Call { + _c.Call.Return(run) + return _c +} + +// GetSegmentInfo provides a mock function with given fields: readable +func (_m *MockShardDelegator) GetSegmentInfo(readable bool) ([]SnapshotItem, []SegmentEntry) { + ret := _m.Called(readable) var r0 []SnapshotItem - if rf, ok := ret.Get(0).(func() []SnapshotItem); ok { - r0 = rf() + var r1 []SegmentEntry + if rf, ok := ret.Get(0).(func(bool) ([]SnapshotItem, []SegmentEntry)); ok { + return rf(readable) + } + if rf, ok := ret.Get(0).(func(bool) []SnapshotItem); ok { + r0 = rf(readable) } else { if ret.Get(0) != nil { r0 = ret.Get(0).([]SnapshotItem) } } - var r1 []SegmentEntry - if rf, ok := ret.Get(1).(func() []SegmentEntry); ok { - r1 = rf() + if rf, ok := ret.Get(1).(func(bool) []SegmentEntry); ok { + r1 = rf(readable) } else { if ret.Get(1) != nil { r1 = ret.Get(1).([]SegmentEntry) @@ -118,13 +131,14 @@ type MockShardDelegator_GetSegmentInfo_Call struct { } // GetSegmentInfo is a helper method to define mock.On call -func (_e *MockShardDelegator_Expecter) GetSegmentInfo() *MockShardDelegator_GetSegmentInfo_Call { - return &MockShardDelegator_GetSegmentInfo_Call{Call: _e.mock.On("GetSegmentInfo")} +// - readable bool +func (_e *MockShardDelegator_Expecter) GetSegmentInfo(readable interface{}) *MockShardDelegator_GetSegmentInfo_Call { + return &MockShardDelegator_GetSegmentInfo_Call{Call: _e.mock.On("GetSegmentInfo", readable)} } -func (_c *MockShardDelegator_GetSegmentInfo_Call) Run(run func()) *MockShardDelegator_GetSegmentInfo_Call { +func (_c *MockShardDelegator_GetSegmentInfo_Call) Run(run func(readable bool)) *MockShardDelegator_GetSegmentInfo_Call { _c.Call.Run(func(args mock.Arguments) { - run() + run(args[0].(bool)) }) return _c } @@ -134,11 +148,20 @@ func (_c *MockShardDelegator_GetSegmentInfo_Call) Return(sealed []SnapshotItem, return _c } +func (_c *MockShardDelegator_GetSegmentInfo_Call) RunAndReturn(run func(bool) ([]SnapshotItem, []SegmentEntry)) *MockShardDelegator_GetSegmentInfo_Call { + _c.Call.Return(run) + return _c +} + // GetStatistics provides a mock function with given fields: ctx, req func (_m *MockShardDelegator) GetStatistics(ctx context.Context, req *querypb.GetStatisticsRequest) ([]*internalpb.GetStatisticsResponse, error) { ret := _m.Called(ctx, req) var r0 []*internalpb.GetStatisticsResponse + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, *querypb.GetStatisticsRequest) ([]*internalpb.GetStatisticsResponse, error)); ok { + return rf(ctx, req) + } if rf, ok := ret.Get(0).(func(context.Context, *querypb.GetStatisticsRequest) []*internalpb.GetStatisticsResponse); ok { r0 = rf(ctx, req) } else { @@ -147,7 +170,6 @@ func (_m *MockShardDelegator) GetStatistics(ctx context.Context, req *querypb.Ge } } - var r1 error if rf, ok := ret.Get(1).(func(context.Context, *querypb.GetStatisticsRequest) error); ok { r1 = rf(ctx, req) } else { @@ -181,6 +203,52 @@ func (_c *MockShardDelegator_GetStatistics_Call) Return(_a0 []*internalpb.GetSta return _c } +func (_c *MockShardDelegator_GetStatistics_Call) RunAndReturn(run func(context.Context, *querypb.GetStatisticsRequest) ([]*internalpb.GetStatisticsResponse, error)) *MockShardDelegator_GetStatistics_Call { + _c.Call.Return(run) + return _c +} + +// GetTargetVersion provides a mock function with given fields: +func (_m *MockShardDelegator) GetTargetVersion() int64 { + ret := _m.Called() + + var r0 int64 + if rf, ok := ret.Get(0).(func() int64); ok { + r0 = rf() + } else { + r0 = ret.Get(0).(int64) + } + + return r0 +} + +// MockShardDelegator_GetTargetVersion_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'GetTargetVersion' +type MockShardDelegator_GetTargetVersion_Call struct { + *mock.Call +} + +// GetTargetVersion is a helper method to define mock.On call +func (_e *MockShardDelegator_Expecter) GetTargetVersion() *MockShardDelegator_GetTargetVersion_Call { + return &MockShardDelegator_GetTargetVersion_Call{Call: _e.mock.On("GetTargetVersion")} +} + +func (_c *MockShardDelegator_GetTargetVersion_Call) Run(run func()) *MockShardDelegator_GetTargetVersion_Call { + _c.Call.Run(func(args mock.Arguments) { + run() + }) + return _c +} + +func (_c *MockShardDelegator_GetTargetVersion_Call) Return(_a0 int64) *MockShardDelegator_GetTargetVersion_Call { + _c.Call.Return(_a0) + return _c +} + +func (_c *MockShardDelegator_GetTargetVersion_Call) RunAndReturn(run func() int64) *MockShardDelegator_GetTargetVersion_Call { + _c.Call.Return(run) + return _c +} + // LoadGrowing provides a mock function with given fields: ctx, infos, version func (_m *MockShardDelegator) LoadGrowing(ctx context.Context, infos []*querypb.SegmentLoadInfo, version int64) error { ret := _m.Called(ctx, infos, version) @@ -220,6 +288,11 @@ func (_c *MockShardDelegator_LoadGrowing_Call) Return(_a0 error) *MockShardDeleg return _c } +func (_c *MockShardDelegator_LoadGrowing_Call) RunAndReturn(run func(context.Context, []*querypb.SegmentLoadInfo, int64) error) *MockShardDelegator_LoadGrowing_Call { + _c.Call.Return(run) + return _c +} + // LoadSegments provides a mock function with given fields: ctx, req func (_m *MockShardDelegator) LoadSegments(ctx context.Context, req *querypb.LoadSegmentsRequest) error { ret := _m.Called(ctx, req) @@ -258,6 +331,11 @@ func (_c *MockShardDelegator_LoadSegments_Call) Return(_a0 error) *MockShardDele return _c } +func (_c *MockShardDelegator_LoadSegments_Call) RunAndReturn(run func(context.Context, *querypb.LoadSegmentsRequest) error) *MockShardDelegator_LoadSegments_Call { + _c.Call.Return(run) + return _c +} + // ProcessDelete provides a mock function with given fields: deleteData, ts func (_m *MockShardDelegator) ProcessDelete(deleteData []*DeleteData, ts uint64) { _m.Called(deleteData, ts) @@ -287,6 +365,11 @@ func (_c *MockShardDelegator_ProcessDelete_Call) Return() *MockShardDelegator_Pr return _c } +func (_c *MockShardDelegator_ProcessDelete_Call) RunAndReturn(run func([]*DeleteData, uint64)) *MockShardDelegator_ProcessDelete_Call { + _c.Call.Return(run) + return _c +} + // ProcessInsert provides a mock function with given fields: insertRecords func (_m *MockShardDelegator) ProcessInsert(insertRecords map[int64]*InsertData) { _m.Called(insertRecords) @@ -315,11 +398,20 @@ func (_c *MockShardDelegator_ProcessInsert_Call) Return() *MockShardDelegator_Pr return _c } +func (_c *MockShardDelegator_ProcessInsert_Call) RunAndReturn(run func(map[int64]*InsertData)) *MockShardDelegator_ProcessInsert_Call { + _c.Call.Return(run) + return _c +} + // Query provides a mock function with given fields: ctx, req func (_m *MockShardDelegator) Query(ctx context.Context, req *querypb.QueryRequest) ([]*internalpb.RetrieveResults, error) { ret := _m.Called(ctx, req) var r0 []*internalpb.RetrieveResults + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, *querypb.QueryRequest) ([]*internalpb.RetrieveResults, error)); ok { + return rf(ctx, req) + } if rf, ok := ret.Get(0).(func(context.Context, *querypb.QueryRequest) []*internalpb.RetrieveResults); ok { r0 = rf(ctx, req) } else { @@ -328,7 +420,6 @@ func (_m *MockShardDelegator) Query(ctx context.Context, req *querypb.QueryReque } } - var r1 error if rf, ok := ret.Get(1).(func(context.Context, *querypb.QueryRequest) error); ok { r1 = rf(ctx, req) } else { @@ -362,6 +453,11 @@ func (_c *MockShardDelegator_Query_Call) Return(_a0 []*internalpb.RetrieveResult return _c } +func (_c *MockShardDelegator_Query_Call) RunAndReturn(run func(context.Context, *querypb.QueryRequest) ([]*internalpb.RetrieveResults, error)) *MockShardDelegator_Query_Call { + _c.Call.Return(run) + return _c +} + // ReleaseSegments provides a mock function with given fields: ctx, req, force func (_m *MockShardDelegator) ReleaseSegments(ctx context.Context, req *querypb.ReleaseSegmentsRequest, force bool) error { ret := _m.Called(ctx, req, force) @@ -401,11 +497,20 @@ func (_c *MockShardDelegator_ReleaseSegments_Call) Return(_a0 error) *MockShardD return _c } +func (_c *MockShardDelegator_ReleaseSegments_Call) RunAndReturn(run func(context.Context, *querypb.ReleaseSegmentsRequest, bool) error) *MockShardDelegator_ReleaseSegments_Call { + _c.Call.Return(run) + return _c +} + // Search provides a mock function with given fields: ctx, req func (_m *MockShardDelegator) Search(ctx context.Context, req *querypb.SearchRequest) ([]*internalpb.SearchResults, error) { ret := _m.Called(ctx, req) var r0 []*internalpb.SearchResults + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, *querypb.SearchRequest) ([]*internalpb.SearchResults, error)); ok { + return rf(ctx, req) + } if rf, ok := ret.Get(0).(func(context.Context, *querypb.SearchRequest) []*internalpb.SearchResults); ok { r0 = rf(ctx, req) } else { @@ -414,7 +519,6 @@ func (_m *MockShardDelegator) Search(ctx context.Context, req *querypb.SearchReq } } - var r1 error if rf, ok := ret.Get(1).(func(context.Context, *querypb.SearchRequest) error); ok { r1 = rf(ctx, req) } else { @@ -448,6 +552,11 @@ func (_c *MockShardDelegator_Search_Call) Return(_a0 []*internalpb.SearchResults return _c } +func (_c *MockShardDelegator_Search_Call) RunAndReturn(run func(context.Context, *querypb.SearchRequest) ([]*internalpb.SearchResults, error)) *MockShardDelegator_Search_Call { + _c.Call.Return(run) + return _c +} + // Serviceable provides a mock function with given fields: func (_m *MockShardDelegator) Serviceable() bool { ret := _m.Called() @@ -484,6 +593,11 @@ func (_c *MockShardDelegator_Serviceable_Call) Return(_a0 bool) *MockShardDelega return _c } +func (_c *MockShardDelegator_Serviceable_Call) RunAndReturn(run func() bool) *MockShardDelegator_Serviceable_Call { + _c.Call.Return(run) + return _c +} + // Start provides a mock function with given fields: func (_m *MockShardDelegator) Start() { _m.Called() @@ -511,6 +625,11 @@ func (_c *MockShardDelegator_Start_Call) Return() *MockShardDelegator_Start_Call return _c } +func (_c *MockShardDelegator_Start_Call) RunAndReturn(run func()) *MockShardDelegator_Start_Call { + _c.Call.Return(run) + return _c +} + // SyncDistribution provides a mock function with given fields: ctx, entries func (_m *MockShardDelegator) SyncDistribution(ctx context.Context, entries ...SegmentEntry) { _va := make([]interface{}, len(entries)) @@ -554,6 +673,46 @@ func (_c *MockShardDelegator_SyncDistribution_Call) Return() *MockShardDelegator return _c } +func (_c *MockShardDelegator_SyncDistribution_Call) RunAndReturn(run func(context.Context, ...SegmentEntry)) *MockShardDelegator_SyncDistribution_Call { + _c.Call.Return(run) + return _c +} + +// SyncTargetVersion provides a mock function with given fields: newVersion, growingInTarget, sealedInTarget +func (_m *MockShardDelegator) SyncTargetVersion(newVersion int64, growingInTarget []int64, sealedInTarget []int64) { + _m.Called(newVersion, growingInTarget, sealedInTarget) +} + +// MockShardDelegator_SyncTargetVersion_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'SyncTargetVersion' +type MockShardDelegator_SyncTargetVersion_Call struct { + *mock.Call +} + +// SyncTargetVersion is a helper method to define mock.On call +// - newVersion int64 +// - growingInTarget []int64 +// - sealedInTarget []int64 +func (_e *MockShardDelegator_Expecter) SyncTargetVersion(newVersion interface{}, growingInTarget interface{}, sealedInTarget interface{}) *MockShardDelegator_SyncTargetVersion_Call { + return &MockShardDelegator_SyncTargetVersion_Call{Call: _e.mock.On("SyncTargetVersion", newVersion, growingInTarget, sealedInTarget)} +} + +func (_c *MockShardDelegator_SyncTargetVersion_Call) Run(run func(newVersion int64, growingInTarget []int64, sealedInTarget []int64)) *MockShardDelegator_SyncTargetVersion_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(int64), args[1].([]int64), args[2].([]int64)) + }) + return _c +} + +func (_c *MockShardDelegator_SyncTargetVersion_Call) Return() *MockShardDelegator_SyncTargetVersion_Call { + _c.Call.Return() + return _c +} + +func (_c *MockShardDelegator_SyncTargetVersion_Call) RunAndReturn(run func(int64, []int64, []int64)) *MockShardDelegator_SyncTargetVersion_Call { + _c.Call.Return(run) + return _c +} + // Version provides a mock function with given fields: func (_m *MockShardDelegator) Version() int64 { ret := _m.Called() @@ -590,6 +749,11 @@ func (_c *MockShardDelegator_Version_Call) Return(_a0 int64) *MockShardDelegator return _c } +func (_c *MockShardDelegator_Version_Call) RunAndReturn(run func() int64) *MockShardDelegator_Version_Call { + _c.Call.Return(run) + return _c +} + type mockConstructorTestingTNewMockShardDelegator interface { mock.TestingT Cleanup(func()) diff --git a/internal/querynodev2/delegator/snapshot.go b/internal/querynodev2/delegator/snapshot.go index d98e967410..ab4b1a6979 100644 --- a/internal/querynodev2/delegator/snapshot.go +++ b/internal/querynodev2/delegator/snapshot.go @@ -36,8 +36,9 @@ type snapshotCleanup func() // snapshot records segment distribution with ref count. type snapshot struct { - dist []SnapshotItem - growing []SegmentEntry + dist []SnapshotItem + growing []SegmentEntry + targetVersion int64 // version ID for tracking version int64 @@ -57,13 +58,14 @@ type snapshot struct { } // NewSnapshot returns a prepared snapshot with channel initialized. -func NewSnapshot(sealed []SnapshotItem, growing []SegmentEntry, last *snapshot, version int64) *snapshot { +func NewSnapshot(sealed []SnapshotItem, growing []SegmentEntry, last *snapshot, version int64, targetVersion int64) *snapshot { return &snapshot{ - version: version, - growing: growing, - dist: sealed, - last: last, - cleared: make(chan struct{}), + version: version, + growing: growing, + dist: sealed, + last: last, + cleared: make(chan struct{}), + targetVersion: targetVersion, } } @@ -80,6 +82,10 @@ func (s *snapshot) Get(partitions ...int64) (sealed []SnapshotItem, growing []Se return s.filter(partitions...) } +func (s *snapshot) GetTargetVersion() int64 { + return s.targetVersion +} + // Peek returns segment distributions without increasing inUse. func (s *snapshot) Peek(partitions ...int64) (sealed []SnapshotItem, growing []SegmentEntry) { return s.filter(partitions...) diff --git a/internal/querynodev2/delegator/snapshot_test.go b/internal/querynodev2/delegator/snapshot_test.go index 2e085a9421..7f9d3987cf 100644 --- a/internal/querynodev2/delegator/snapshot_test.go +++ b/internal/querynodev2/delegator/snapshot_test.go @@ -30,8 +30,9 @@ type SnapshotSuite struct { } func (s *SnapshotSuite) SetupTest() { - last := NewSnapshot(nil, nil, nil, 0) + last := NewSnapshot(nil, nil, nil, 0, initialTargetVersion) last.Expire(func() {}) + s.Equal(initialTargetVersion, last.GetTargetVersion()) dist := []SnapshotItem{ { @@ -72,7 +73,8 @@ func (s *SnapshotSuite) SetupTest() { }, } - s.snapshot = NewSnapshot(dist, growing, last, 1) + s.snapshot = NewSnapshot(dist, growing, last, 1, 1) + s.Equal(int64(1), s.snapshot.GetTargetVersion()) } func (s *SnapshotSuite) TearDownTest() { diff --git a/internal/querynodev2/handlers.go b/internal/querynodev2/handlers.go index ff20201fbd..33373b4943 100644 --- a/internal/querynodev2/handlers.go +++ b/internal/querynodev2/handlers.go @@ -271,7 +271,7 @@ func (node *QueryNode) optimizeSearchParams(ctx context.Context, req *querypb.Se switch plan.GetNode().(type) { case *planpb.PlanNode_VectorAnns: // ignore growing ones for now since they will always be brute force - sealed, _ := deleg.GetSegmentInfo() + sealed, _ := deleg.GetSegmentInfo(true) sealedNum := lo.Reduce(sealed, func(sum int, item delegator.SnapshotItem, _ int) int { return sum + len(item.Segments) }, 0) diff --git a/internal/querynodev2/handlers_test.go b/internal/querynodev2/handlers_test.go index fc50513358..774586a556 100644 --- a/internal/querynodev2/handlers_test.go +++ b/internal/querynodev2/handlers_test.go @@ -164,7 +164,7 @@ func (suite *OptimizeSearchParamSuite) SetupSuite() { suite.channel = "test-channel" suite.delegator = &delegator.MockShardDelegator{} - suite.delegator.EXPECT().GetSegmentInfo().Return([]delegator.SnapshotItem{{NodeID: 1, Segments: []delegator.SegmentEntry{{SegmentID: 100}}}}, []delegator.SegmentEntry{}) + suite.delegator.EXPECT().GetSegmentInfo(mock.Anything).Return([]delegator.SnapshotItem{{NodeID: 1, Segments: []delegator.SegmentEntry{{SegmentID: 100}}}}, []delegator.SegmentEntry{}) } func (suite *OptimizeSearchParamSuite) SetupTest() { diff --git a/internal/querynodev2/services.go b/internal/querynodev2/services.go index f869b46f0b..4e38e9d342 100644 --- a/internal/querynodev2/services.go +++ b/internal/querynodev2/services.go @@ -1184,7 +1184,7 @@ func (node *QueryNode) GetDataDistribution(ctx context.Context, req *querypb.Get Version: value.Version(), }) - sealed, growing := value.GetSegmentInfo() + sealed, growing := value.GetSegmentInfo(false) sealedSegments := make(map[int64]*querypb.SegmentDist) for _, item := range sealed { for _, segment := range item.Segments { @@ -1210,6 +1210,7 @@ func (node *QueryNode) GetDataDistribution(ctx context.Context, req *querypb.Get Channel: key, SegmentDist: sealedSegments, GrowingSegments: growingSegments, + TargetVersion: value.GetTargetVersion(), }) return true }) @@ -1260,6 +1261,7 @@ func (node *QueryNode) SyncDistribution(ctx context.Context, req *querypb.SyncDi log := log.With(zap.String("Action", action.GetType().String()), zap.Int64("segmentID", action.SegmentID), + zap.Int64("TargetVersion", action.GetTargetVersion()), ) log.Info("sync action") switch action.GetType() { @@ -1267,6 +1269,8 @@ func (node *QueryNode) SyncDistribution(ctx context.Context, req *querypb.SyncDi removeActions = append(removeActions, action) case querypb.SyncType_Set: addSegments[action.GetNodeID()] = append(addSegments[action.GetNodeID()], action.GetInfo()) + case querypb.SyncType_UpdateVersion: + shardDelegator.SyncTargetVersion(action.GetTargetVersion(), action.GetGrowingInTarget(), action.GetSealedInTarget()) default: return &commonpb.Status{ ErrorCode: commonpb.ErrorCode_UnexpectedError, diff --git a/internal/querynodev2/services_test.go b/internal/querynodev2/services_test.go index faa3996482..97e8d3a5b5 100644 --- a/internal/querynodev2/services_test.go +++ b/internal/querynodev2/services_test.go @@ -1368,7 +1368,7 @@ func (suite *ServiceSuite) TestSyncDistribution_ReleaseResultCheck() { delegator, ok := suite.node.delegators.Get(suite.vchannel) suite.True(ok) - sealedSegments, _ := delegator.GetSegmentInfo() + sealedSegments, _ := delegator.GetSegmentInfo(false) suite.Len(sealedSegments[0].Segments, 3) // data @@ -1392,7 +1392,7 @@ func (suite *ServiceSuite) TestSyncDistribution_ReleaseResultCheck() { status, err := suite.node.SyncDistribution(ctx, req) suite.NoError(err) suite.Equal(commonpb.ErrorCode_Success, status.ErrorCode) - sealedSegments, _ = delegator.GetSegmentInfo() + sealedSegments, _ = delegator.GetSegmentInfo(false) suite.Len(sealedSegments[0].Segments, 3) releaseAction = &querypb.SyncAction{ @@ -1406,7 +1406,7 @@ func (suite *ServiceSuite) TestSyncDistribution_ReleaseResultCheck() { status, err = suite.node.SyncDistribution(ctx, req) suite.NoError(err) suite.Equal(commonpb.ErrorCode_Success, status.ErrorCode) - sealedSegments, _ = delegator.GetSegmentInfo() + sealedSegments, _ = delegator.GetSegmentInfo(false) suite.Len(sealedSegments[0].Segments, 2) }