milvus-io
/
milvus
mirror of https://github.com/milvus-io/milvus.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Add recovery logic for querynode and queryservice (#5843) 

* merge milvus/recovery2

Signed-off-by: xige-16 <xi.ge@zilliz.com>

* add recovery logic in queryservice

Signed-off-by: xige-16 <xi.ge@zilliz.com>

* debug smoke case

Signed-off-by: xige-16 <xi.ge@zilliz.com>

* add etcd to querynode

Signed-off-by: xige-16 <xi.ge@zilliz.com>

* fix release partition error

Signed-off-by: xige-16 <xi.ge@zilliz.com>

* fix load balance error

Signed-off-by: xige-16 <xi.ge@zilliz.com>

* debug querynode down and recovery

Signed-off-by: xige-16 <xi.ge@zilliz.com>

* add log

Signed-off-by: xige-16 <xi.ge@zilliz.com>

* fix showCollection

Signed-off-by: xige-16 <xi.ge@zilliz.com>

* skip smoke test search without insert

Signed-off-by: xige-16 <xi.ge@zilliz.com>
pull/5878/head
xige-16 2021-06-19 11:45:09 +08:00 committed by GitHub
parent 26b8d5966f
commit e4c51aae36
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
27 changed files with 2495 additions and 944 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
internal/distributed/querynode/service.go
View File

@ -89,10 +89,6 @@ func (s *Server) init() error {
closer := trace.InitTracing(fmt.Sprintf("query_node ip: %s, port: %d", Params.QueryNodeIP, Params.QueryNodePort))
s.closer = closer
if err := s.querynode.Register(); err != nil {
return err
}
log.Debug("QueryNode", zap.Int("port", Params.QueryNodePort))
s.wg.Add(1)
go s.startGrpcLoop(Params.QueryNodePort)
@ -101,6 +97,10 @@ func (s *Server) init() error {
if err != nil {
return err
}
if err := s.querynode.Register(); err != nil {
return err
}
// --- QueryService ---
log.Debug("QueryNode start to new QueryServiceClient", zap.Any("QueryServiceAddress", Params.QueryServiceAddress))
queryService, err := qsc.NewClient(qn.Params.MetaRootPath, qn.Params.EtcdEndpoints, 3*time.Second)

9
internal/msgstream/msg.go
View File

@ -20,7 +20,6 @@ import (
"github.com/milvus-io/milvus/internal/proto/commonpb"
"github.com/milvus-io/milvus/internal/proto/datapb"
"github.com/milvus-io/milvus/internal/proto/internalpb"
"github.com/milvus-io/milvus/internal/proto/querypb"
)
type MsgType = commonpb.MsgType
@ -888,7 +887,7 @@ func (sim *SegmentInfoMsg) Unmarshal(input MarshalType) (TsMsg, error) {
/////////////////////////////////////////LoadBalanceSegments//////////////////////////////////////////
type LoadBalanceSegmentsMsg struct {
BaseMsg
querypb.LoadBalanceSegments
internalpb.LoadBalanceSegmentsRequest
}
func (l *LoadBalanceSegmentsMsg) TraceCtx() context.Context {
@ -909,7 +908,7 @@ func (l *LoadBalanceSegmentsMsg) Type() MsgType {
func (l *LoadBalanceSegmentsMsg) Marshal(input TsMsg) (MarshalType, error) {
load := input.(*LoadBalanceSegmentsMsg)
loadReq := &load.LoadBalanceSegments
loadReq := &load.LoadBalanceSegmentsRequest
mb, err := proto.Marshal(loadReq)
if err != nil {
return nil, err
@ -918,7 +917,7 @@ func (l *LoadBalanceSegmentsMsg) Marshal(input TsMsg) (MarshalType, error) {
}
func (l *LoadBalanceSegmentsMsg) Unmarshal(input MarshalType) (TsMsg, error) {
loadReq := querypb.LoadBalanceSegments{}
loadReq := internalpb.LoadBalanceSegmentsRequest{}
in, err := ConvertToByteArray(input)
if err != nil {
return nil, err
@ -927,7 +926,7 @@ func (l *LoadBalanceSegmentsMsg) Unmarshal(input MarshalType) (TsMsg, error) {
if err != nil {
return nil, err
}
loadMsg := &LoadBalanceSegmentsMsg{LoadBalanceSegments: loadReq}
loadMsg := &LoadBalanceSegmentsMsg{LoadBalanceSegmentsRequest: loadReq}
loadMsg.BeginTimestamp = loadReq.Base.Timestamp
loadMsg.EndTimestamp = loadReq.Base.Timestamp

5
internal/proto/internal.proto
View File

@ -174,6 +174,11 @@ message DeleteRequest {
repeated int64 primary_keys = 5;
}
message LoadBalanceSegmentsRequest {
common.MsgBase base = 1;
repeated int64 segmentIDs = 2;
}
message LoadIndex {
common.MsgBase base = 1;
int64 segmentID = 2;

298
internal/proto/internalpb/internal.pb.go
View File

@ -1487,6 +1487,53 @@ func (m *DeleteRequest) GetPrimaryKeys() []int64 {
return nil
}
type LoadBalanceSegmentsRequest struct {
Base *commonpb.MsgBase `protobuf:"bytes,1,opt,name=base,proto3" json:"base,omitempty"`
SegmentIDs []int64 `protobuf:"varint,2,rep,packed,name=segmentIDs,proto3" json:"segmentIDs,omitempty"`
XXX_NoUnkeyedLiteral struct{} `json:"-"`
XXX_unrecognized []byte `json:"-"`
XXX_sizecache int32 `json:"-"`
}
func (m *LoadBalanceSegmentsRequest) Reset() { *m = LoadBalanceSegmentsRequest{} }
func (m *LoadBalanceSegmentsRequest) String() string { return proto.CompactTextString(m) }
func (*LoadBalanceSegmentsRequest) ProtoMessage() {}
func (*LoadBalanceSegmentsRequest) Descriptor() ([]byte, []int) {
return fileDescriptor_41f4a519b878ee3b, []int{21}
}
func (m *LoadBalanceSegmentsRequest) XXX_Unmarshal(b []byte) error {
return xxx_messageInfo_LoadBalanceSegmentsRequest.Unmarshal(m, b)
}
func (m *LoadBalanceSegmentsRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
return xxx_messageInfo_LoadBalanceSegmentsRequest.Marshal(b, m, deterministic)
}
func (m *LoadBalanceSegmentsRequest) XXX_Merge(src proto.Message) {
xxx_messageInfo_LoadBalanceSegmentsRequest.Merge(m, src)
}
func (m *LoadBalanceSegmentsRequest) XXX_Size() int {
return xxx_messageInfo_LoadBalanceSegmentsRequest.Size(m)
}
func (m *LoadBalanceSegmentsRequest) XXX_DiscardUnknown() {
xxx_messageInfo_LoadBalanceSegmentsRequest.DiscardUnknown(m)
}
var xxx_messageInfo_LoadBalanceSegmentsRequest proto.InternalMessageInfo
func (m *LoadBalanceSegmentsRequest) GetBase() *commonpb.MsgBase {
if m != nil {
return m.Base
}
return nil
}
func (m *LoadBalanceSegmentsRequest) GetSegmentIDs() []int64 {
if m != nil {
return m.SegmentIDs
}
return nil
}
type LoadIndex struct {
Base *commonpb.MsgBase `protobuf:"bytes,1,opt,name=base,proto3" json:"base,omitempty"`
SegmentID int64 `protobuf:"varint,2,opt,name=segmentID,proto3" json:"segmentID,omitempty"`
@ -1503,7 +1550,7 @@ func (m *LoadIndex) Reset() { *m = LoadIndex{} }
func (m *LoadIndex) String() string { return proto.CompactTextString(m) }
func (*LoadIndex) ProtoMessage() {}
func (*LoadIndex) Descriptor() ([]byte, []int) {
return fileDescriptor_41f4a519b878ee3b, []int{21}
return fileDescriptor_41f4a519b878ee3b, []int{22}
}
func (m *LoadIndex) XXX_Unmarshal(b []byte) error {
@ -1583,7 +1630,7 @@ func (m *SegmentStatisticsUpdates) Reset() { *m = SegmentStatisticsUpdat
func (m *SegmentStatisticsUpdates) String() string { return proto.CompactTextString(m) }
func (*SegmentStatisticsUpdates) ProtoMessage() {}
func (*SegmentStatisticsUpdates) Descriptor() ([]byte, []int) {
return fileDescriptor_41f4a519b878ee3b, []int{22}
return fileDescriptor_41f4a519b878ee3b, []int{23}
}
func (m *SegmentStatisticsUpdates) XXX_Unmarshal(b []byte) error {
@ -1665,7 +1712,7 @@ func (m *SegmentStatistics) Reset() { *m = SegmentStatistics{} }
func (m *SegmentStatistics) String() string { return proto.CompactTextString(m) }
func (*SegmentStatistics) ProtoMessage() {}
func (*SegmentStatistics) Descriptor() ([]byte, []int) {
return fileDescriptor_41f4a519b878ee3b, []int{23}
return fileDescriptor_41f4a519b878ee3b, []int{24}
}
func (m *SegmentStatistics) XXX_Unmarshal(b []byte) error {
@ -1712,7 +1759,7 @@ func (m *SegmentFlushCompletedMsg) Reset() { *m = SegmentFlushCompletedM
func (m *SegmentFlushCompletedMsg) String() string { return proto.CompactTextString(m) }
func (*SegmentFlushCompletedMsg) ProtoMessage() {}
func (*SegmentFlushCompletedMsg) Descriptor() ([]byte, []int) {
return fileDescriptor_41f4a519b878ee3b, []int{24}
return fileDescriptor_41f4a519b878ee3b, []int{25}
}
func (m *SegmentFlushCompletedMsg) XXX_Unmarshal(b []byte) error {
@ -1759,7 +1806,7 @@ func (m *IndexStats) Reset() { *m = IndexStats{} }
func (m *IndexStats) String() string { return proto.CompactTextString(m) }
func (*IndexStats) ProtoMessage() {}
func (*IndexStats) Descriptor() ([]byte, []int) {
return fileDescriptor_41f4a519b878ee3b, []int{25}
return fileDescriptor_41f4a519b878ee3b, []int{26}
}
func (m *IndexStats) XXX_Unmarshal(b []byte) error {
@ -1807,7 +1854,7 @@ func (m *FieldStats) Reset() { *m = FieldStats{} }
func (m *FieldStats) String() string { return proto.CompactTextString(m) }
func (*FieldStats) ProtoMessage() {}
func (*FieldStats) Descriptor() ([]byte, []int) {
return fileDescriptor_41f4a519b878ee3b, []int{26}
return fileDescriptor_41f4a519b878ee3b, []int{27}
}
func (m *FieldStats) XXX_Unmarshal(b []byte) error {
@ -1863,7 +1910,7 @@ func (m *SegmentStats) Reset() { *m = SegmentStats{} }
func (m *SegmentStats) String() string { return proto.CompactTextString(m) }
func (*SegmentStats) ProtoMessage() {}
func (*SegmentStats) Descriptor() ([]byte, []int) {
return fileDescriptor_41f4a519b878ee3b, []int{27}
return fileDescriptor_41f4a519b878ee3b, []int{28}
}
func (m *SegmentStats) XXX_Unmarshal(b []byte) error {
@ -1925,7 +1972,7 @@ func (m *QueryNodeStats) Reset() { *m = QueryNodeStats{} }
func (m *QueryNodeStats) String() string { return proto.CompactTextString(m) }
func (*QueryNodeStats) ProtoMessage() {}
func (*QueryNodeStats) Descriptor() ([]byte, []int) {
return fileDescriptor_41f4a519b878ee3b, []int{28}
return fileDescriptor_41f4a519b878ee3b, []int{29}
}
func (m *QueryNodeStats) XXX_Unmarshal(b []byte) error {
@ -1981,7 +2028,7 @@ func (m *MsgPosition) Reset() { *m = MsgPosition{} }
func (m *MsgPosition) String() string { return proto.CompactTextString(m) }
func (*MsgPosition) ProtoMessage() {}
func (*MsgPosition) Descriptor() ([]byte, []int) {
return fileDescriptor_41f4a519b878ee3b, []int{29}
return fileDescriptor_41f4a519b878ee3b, []int{30}
}
func (m *MsgPosition) XXX_Unmarshal(b []byte) error {
@ -2044,7 +2091,7 @@ func (m *ChannelTimeTickMsg) Reset() { *m = ChannelTimeTickMsg{} }
func (m *ChannelTimeTickMsg) String() string { return proto.CompactTextString(m) }
func (*ChannelTimeTickMsg) ProtoMessage() {}
func (*ChannelTimeTickMsg) Descriptor() ([]byte, []int) {
return fileDescriptor_41f4a519b878ee3b, []int{30}
return fileDescriptor_41f4a519b878ee3b, []int{31}
}
func (m *ChannelTimeTickMsg) XXX_Unmarshal(b []byte) error {
@ -2116,6 +2163,7 @@ func init() {
proto.RegisterType((*RetrieveRequest)(nil), "milvus.proto.internal.RetrieveRequest")
proto.RegisterType((*RetrieveResults)(nil), "milvus.proto.internal.RetrieveResults")
proto.RegisterType((*DeleteRequest)(nil), "milvus.proto.internal.DeleteRequest")
proto.RegisterType((*LoadBalanceSegmentsRequest)(nil), "milvus.proto.internal.LoadBalanceSegmentsRequest")
proto.RegisterType((*LoadIndex)(nil), "milvus.proto.internal.LoadIndex")
proto.RegisterType((*SegmentStatisticsUpdates)(nil), "milvus.proto.internal.SegmentStatisticsUpdates")
proto.RegisterType((*SegmentStatistics)(nil), "milvus.proto.internal.SegmentStatistics")
@ -2131,118 +2179,120 @@ func init() {
func init() { proto.RegisterFile("internal.proto", fileDescriptor_41f4a519b878ee3b) }
var fileDescriptor_41f4a519b878ee3b = []byte{
// 1807 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xec, 0x58, 0xcd, 0x93, 0x1b, 0x47,
0x15, 0x67, 0x24, 0xed, 0x4a, 0x7a, 0x9a, 0x5d, 0xcb, 0xed, 0xb5, 0x33, 0x6b, 0x3b, 0x89, 0x32,
0xe1, 0x63, 0x89, 0x0b, 0xaf, 0xd9, 0x00, 0x49, 0x51, 0x14, 0x4e, 0xbc, 0x4a, 0x8c, 0xca, 0x59,
0xd7, 0x32, 0x72, 0x52, 0x05, 0x97, 0xa9, 0x96, 0xa6, 0x57, 0x9a, 0x64, 0xbe, 0xe8, 0x6e, 0xd9,
0x56, 0x4e, 0x1c, 0xb8, 0x51, 0x70, 0xa0, 0x8a, 0x13, 0xff, 0x03, 0x57, 0x4e, 0x7c, 0x54, 0x4e,
0x54, 0xf1, 0x17, 0xf0, 0x6f, 0x70, 0xe4, 0x02, 0xd5, 0xaf, 0x7b, 0x3e, 0xa4, 0x9d, 0x5d, 0x36,
0xeb, 0xa2, 0x20, 0x05, 0xb7, 0xe9, 0xf7, 0x5e, 0x7f, 0xfc, 0x7e, 0xef, 0xf5, 0x7b, 0x6f, 0x1a,
0xb6, 0xc3, 0x44, 0x32, 0x9e, 0xd0, 0xe8, 0x6e, 0xc6, 0x53, 0x99, 0x92, 0xeb, 0x71, 0x18, 0x3d,
0x5d, 0x08, 0x3d, 0xba, 0x9b, 0x2b, 0x6f, 0xda, 0xd3, 0x34, 0x8e, 0xd3, 0x44, 0x8b, 0x6f, 0xda,
0x62, 0x3a, 0x67, 0x31, 0xd5, 0x23, 0xf7, 0x0f, 0x16, 0x6c, 0x1d, 0xa6, 0x71, 0x96, 0x26, 0x2c,
0x91, 0xa3, 0xe4, 0x24, 0x25, 0x37, 0x60, 0x33, 0x49, 0x03, 0x36, 0x1a, 0x3a, 0xd6, 0xc0, 0xda,
0x6b, 0x7a, 0x66, 0x44, 0x08, 0xb4, 0x78, 0x1a, 0x31, 0xa7, 0x31, 0xb0, 0xf6, 0xba, 0x1e, 0x7e,
0x93, 0xfb, 0x00, 0x42, 0x52, 0xc9, 0xfc, 0x69, 0x1a, 0x30, 0xa7, 0x39, 0xb0, 0xf6, 0xb6, 0x0f,
0x06, 0x77, 0x6b, 0x4f, 0x71, 0x77, 0xac, 0x0c, 0x0f, 0xd3, 0x80, 0x79, 0x5d, 0x91, 0x7f, 0x92,
0x77, 0x00, 0xd8, 0x73, 0xc9, 0xa9, 0x1f, 0x26, 0x27, 0xa9, 0xd3, 0x1a, 0x34, 0xf7, 0x7a, 0x07,
0xaf, 0xad, 0x2e, 0x60, 0x0e, 0xff, 0x88, 0x2d, 0x3f, 0xa2, 0xd1, 0x82, 0x1d, 0xd3, 0x90, 0x7b,
0x5d, 0x9c, 0xa4, 0x8e, 0xeb, 0xfe, 0xd5, 0x82, 0x2b, 0x05, 0x00, 0xdc, 0x43, 0x90, 0xef, 0xc2,
0x06, 0x6e, 0x81, 0x08, 0x7a, 0x07, 0x5f, 0x3e, 0xe3, 0x44, 0x2b, 0xb8, 0x3d, 0x3d, 0x85, 0x7c,
0x08, 0xd7, 0xc4, 0x62, 0x32, 0xcd, 0x55, 0x3e, 0x4a, 0x85, 0xd3, 0xc0, 0xa3, 0x5d, 0x6c, 0x25,
0x52, 0x5d, 0xc0, 0x1c, 0xe9, 0x4d, 0xd8, 0x54, 0x2b, 0x2d, 0x04, 0xb2, 0xd4, 0x3b, 0xb8, 0x55,
0x0b, 0x72, 0x8c, 0x26, 0x9e, 0x31, 0x75, 0x6f, 0xc1, 0xee, 0x43, 0x26, 0xd7, 0xd0, 0x79, 0xec,
0x27, 0x0b, 0x26, 0xa4, 0x51, 0x3e, 0x09, 0x63, 0xf6, 0x24, 0x9c, 0x7e, 0x72, 0x38, 0xa7, 0x49,
0xc2, 0xa2, 0x5c, 0xf9, 0x32, 0xdc, 0x7a, 0xc8, 0x70, 0x42, 0x28, 0x64, 0x38, 0x15, 0x6b, 0xea,
0xeb, 0x70, 0xed, 0x21, 0x93, 0xc3, 0x60, 0x4d, 0xfc, 0x11, 0x74, 0x1e, 0x2b, 0x67, 0xab, 0x30,
0xf8, 0x0e, 0xb4, 0x69, 0x10, 0x70, 0x26, 0x84, 0x61, 0xf1, 0x76, 0xed, 0x89, 0xdf, 0xd5, 0x36,
0x5e, 0x6e, 0x5c, 0x17, 0x26, 0xee, 0xc7, 0x00, 0xa3, 0x24, 0x94, 0xc7, 0x94, 0xd3, 0x58, 0x9c,
0x19, 0x60, 0x43, 0xb0, 0x85, 0xa4, 0x5c, 0xfa, 0x19, 0xda, 0x19, 0xca, 0x2f, 0x10, 0x0d, 0x3d,
0x9c, 0xa6, 0x57, 0x77, 0x7f, 0x04, 0x30, 0x96, 0x3c, 0x4c, 0x66, 0x1f, 0x84, 0x42, 0xaa, 0xbd,
0x9e, 0x2a, 0x3b, 0x05, 0xa2, 0xb9, 0xd7, 0xf5, 0xcc, 0xa8, 0xe2, 0x8e, 0xc6, 0xc5, 0xdd, 0x71,
0x1f, 0x7a, 0x39, 0xdd, 0x47, 0x62, 0x46, 0xee, 0x41, 0x6b, 0x42, 0x05, 0x3b, 0x97, 0x9e, 0x23,
0x31, 0x7b, 0x40, 0x05, 0xf3, 0xd0, 0xd2, 0xfd, 0xac, 0x01, 0x2f, 0x1d, 0x72, 0x86, 0xc1, 0x1f,
0x45, 0x6c, 0x2a, 0xc3, 0x34, 0x31, 0xdc, 0x7f, 0xfe, 0xd5, 0xc8, 0x4b, 0xd0, 0x0e, 0x26, 0x7e,
0x42, 0xe3, 0x9c, 0xec, 0xcd, 0x60, 0xf2, 0x98, 0xc6, 0x8c, 0x7c, 0x15, 0xb6, 0xa7, 0xc5, 0xfa,
0x4a, 0x82, 0x31, 0xd7, 0xf5, 0xd6, 0xa4, 0xca, 0x55, 0xc1, 0x64, 0x34, 0x74, 0x5a, 0xe8, 0x06,
0xfc, 0x26, 0x2e, 0xd8, 0xa5, 0xd5, 0x68, 0xe8, 0x6c, 0xa0, 0x6e, 0x45, 0xa6, 0x48, 0xd5, 0x39,
0xc4, 0xd9, 0x1c, 0x58, 0x7b, 0xb6, 0x67, 0x46, 0xe4, 0x1e, 0x5c, 0x7b, 0x1a, 0x72, 0xb9, 0xa0,
0x91, 0x89, 0x2b, 0xb5, 0x8b, 0x70, 0xda, 0xc8, 0x7c, 0x9d, 0x8a, 0x1c, 0xc0, 0x4e, 0x36, 0x5f,
0x8a, 0x70, 0xba, 0x36, 0xa5, 0x83, 0x53, 0x6a, 0x75, 0xee, 0x67, 0x16, 0x5c, 0x1f, 0xf2, 0x34,
0xfb, 0x22, 0x53, 0xe8, 0xfe, 0xa2, 0x01, 0x37, 0x74, 0x24, 0x1c, 0x53, 0x2e, 0xc3, 0x7f, 0x13,
0x8a, 0xaf, 0xc1, 0x95, 0x72, 0x57, 0x6d, 0x50, 0x0f, 0xe3, 0x2b, 0xb0, 0x9d, 0xe5, 0xe7, 0xd0,
0x76, 0x2d, 0xb4, 0xdb, 0x2a, 0xa4, 0x2b, 0x68, 0x37, 0xce, 0x41, 0xbb, 0x59, 0x13, 0x30, 0x03,
0xe8, 0x15, 0x0b, 0x8d, 0x86, 0x4e, 0x1b, 0x4d, 0xaa, 0x22, 0xf7, 0xe7, 0x0d, 0xd8, 0x51, 0x4e,
0xfd, 0x3f, 0x1b, 0x8a, 0x8d, 0x3f, 0x36, 0x80, 0xe8, 0xe8, 0x18, 0x25, 0x01, 0x7b, 0xfe, 0x9f,
0xe4, 0xe2, 0x65, 0x80, 0x93, 0x90, 0x45, 0x41, 0x95, 0x87, 0x2e, 0x4a, 0x5e, 0x88, 0x03, 0x07,
0xda, 0xb8, 0x48, 0x81, 0x3f, 0x1f, 0xaa, 0x2a, 0xa0, 0x3b, 0x02, 0x53, 0x05, 0x3a, 0x17, 0xae,
0x02, 0x38, 0xcd, 0x54, 0x81, 0xdf, 0x36, 0x61, 0x6b, 0x94, 0x08, 0xc6, 0xe5, 0xff, 0x72, 0x20,
0x91, 0xdb, 0xd0, 0x15, 0x6c, 0x16, 0xab, 0xc6, 0x64, 0xe8, 0x74, 0x50, 0x5f, 0x0a, 0x94, 0x76,
0xaa, 0x33, 0xeb, 0x68, 0xe8, 0x74, 0xb5, 0x6b, 0x0b, 0x01, 0x79, 0x05, 0x40, 0x86, 0x31, 0x13,
0x92, 0xc6, 0x99, 0x70, 0x60, 0xd0, 0xdc, 0x6b, 0x79, 0x15, 0x89, 0xaa, 0x02, 0x3c, 0x7d, 0x36,
0x1a, 0x0a, 0xa7, 0x37, 0x68, 0xaa, 0x32, 0xae, 0x47, 0xe4, 0x5b, 0xd0, 0xe1, 0xe9, 0x33, 0x3f,
0xa0, 0x92, 0x3a, 0x36, 0x3a, 0x6f, 0xb7, 0x96, 0xec, 0x07, 0x51, 0x3a, 0xf1, 0xda, 0x3c, 0x7d,
0x36, 0xa4, 0x92, 0xba, 0x7f, 0x6b, 0xc0, 0xd6, 0x98, 0x51, 0x3e, 0x9d, 0x5f, 0xde, 0x61, 0x5f,
0x87, 0x3e, 0x67, 0x62, 0x11, 0x49, 0xbf, 0x84, 0xa5, 0x3d, 0x77, 0x45, 0xcb, 0x0f, 0x0b, 0x70,
0x39, 0xe5, 0xcd, 0x73, 0x28, 0x6f, 0xd5, 0x50, 0xee, 0x82, 0x5d, 0xe1, 0x57, 0x38, 0x1b, 0x08,
0x7d, 0x45, 0x46, 0xfa, 0xd0, 0x0c, 0x44, 0x84, 0x1e, 0xeb, 0x7a, 0xea, 0x93, 0xdc, 0x81, 0xab,
0x59, 0x44, 0xa7, 0x6c, 0x9e, 0x46, 0x01, 0xe3, 0xfe, 0x8c, 0xa7, 0x8b, 0x0c, 0xdd, 0x65, 0x7b,
0xfd, 0x8a, 0xe2, 0xa1, 0x92, 0x93, 0xb7, 0xa0, 0x13, 0x88, 0xc8, 0x97, 0xcb, 0x8c, 0xa1, 0xcb,
0xb6, 0xcf, 0xc0, 0x3e, 0x14, 0xd1, 0x93, 0x65, 0xc6, 0xbc, 0x76, 0xa0, 0x3f, 0xc8, 0x3d, 0xd8,
0x11, 0x8c, 0x87, 0x34, 0x0a, 0x3f, 0x65, 0x81, 0xcf, 0x9e, 0x67, 0xdc, 0xcf, 0x22, 0x9a, 0xa0,
0x67, 0x6d, 0x8f, 0x94, 0xba, 0xf7, 0x9e, 0x67, 0xfc, 0x38, 0xa2, 0x89, 0xfb, 0x8f, 0x0a, 0xe9,
0x8a, 0x1f, 0x71, 0x09, 0xd2, 0x2f, 0xd3, 0x49, 0xd5, 0x7a, 0xaa, 0x59, 0xef, 0xa9, 0x57, 0xa1,
0x17, 0x33, 0xc9, 0xc3, 0xa9, 0x66, 0x44, 0x5f, 0x20, 0xd0, 0x22, 0x84, 0x4d, 0xa0, 0x35, 0x0f,
0xa5, 0x76, 0x85, 0xed, 0xe1, 0x37, 0xf9, 0x1e, 0xdc, 0x14, 0x8c, 0x46, 0x2c, 0xf0, 0x8b, 0x68,
0x17, 0xbe, 0x40, 0xa4, 0x2c, 0x70, 0x36, 0xd1, 0x69, 0x8e, 0xb6, 0x18, 0x17, 0x06, 0x63, 0xa3,
0x27, 0xfb, 0x70, 0xad, 0x38, 0x56, 0x65, 0x9a, 0xee, 0x63, 0x48, 0xa9, 0x2a, 0x26, 0xbc, 0x0d,
0xce, 0x2c, 0x4a, 0x27, 0x34, 0xf2, 0x4f, 0xed, 0x8a, 0xf9, 0xab, 0xe9, 0xdd, 0xd0, 0xfa, 0xf1,
0xda, 0x96, 0xee, 0x6f, 0x1a, 0x70, 0xc5, 0x53, 0x58, 0xd8, 0x53, 0xf6, 0x85, 0x0f, 0xfc, 0x37,
0xa0, 0x19, 0x06, 0x02, 0x03, 0xbf, 0x77, 0xe0, 0xac, 0x9e, 0xdb, 0xfc, 0x74, 0x8e, 0x86, 0xc2,
0x53, 0x46, 0xe4, 0x75, 0xd8, 0x4a, 0x17, 0x32, 0x5b, 0x48, 0x1f, 0x13, 0x7f, 0xde, 0x25, 0xda,
0x5a, 0xf8, 0x3e, 0xca, 0xdc, 0xdf, 0x37, 0xab, 0xec, 0xfc, 0xb7, 0x46, 0xa8, 0x81, 0xdd, 0xba,
0x08, 0xec, 0xfb, 0xd0, 0xd3, 0x78, 0x75, 0x7e, 0xdc, 0xc0, 0xfc, 0xf8, 0x4a, 0xed, 0x1c, 0xe4,
0x40, 0xe5, 0x46, 0x4f, 0x57, 0x60, 0xa1, 0xbe, 0xc9, 0xf7, 0xe1, 0xd6, 0xe9, 0xc8, 0xe6, 0x86,
0xa3, 0x3c, 0xb4, 0x77, 0xd7, 0x43, 0x3b, 0x27, 0x31, 0x20, 0xdf, 0x84, 0x9d, 0x4a, 0x6c, 0x97,
0x13, 0x4d, 0x93, 0x5e, 0xea, 0xca, 0x29, 0x97, 0x8f, 0xee, 0xbf, 0x58, 0xb0, 0x35, 0x64, 0x11,
0x93, 0x2f, 0x10, 0xdb, 0x35, 0xc5, 0xb6, 0x51, 0x5b, 0x6c, 0x57, 0xaa, 0x59, 0xf3, 0xfc, 0x6a,
0xd6, 0x3a, 0x55, 0xcd, 0x5e, 0x03, 0x3b, 0xe3, 0x61, 0x4c, 0xf9, 0xd2, 0xff, 0x84, 0x2d, 0xf3,
0xf8, 0xee, 0x19, 0xd9, 0x23, 0xb6, 0x14, 0xee, 0xdf, 0x2d, 0xe8, 0x7e, 0x90, 0xd2, 0x00, 0x7b,
0xb2, 0x4b, 0x20, 0x59, 0x29, 0xc6, 0x8d, 0x9a, 0x62, 0x5c, 0xb4, 0x55, 0xf9, 0xf1, 0xcb, 0x3e,
0xab, 0xd2, 0x2f, 0xb5, 0x56, 0xfb, 0xa5, 0x57, 0xa1, 0x17, 0xaa, 0x03, 0xf9, 0x19, 0x95, 0x73,
0x7d, 0xee, 0xae, 0x07, 0x28, 0x3a, 0x56, 0x12, 0xd5, 0x50, 0xe5, 0x06, 0xd8, 0x50, 0x6d, 0x5e,
0xb8, 0xa1, 0x32, 0x8b, 0x60, 0x43, 0xf5, 0xa7, 0x06, 0x38, 0xc6, 0xb3, 0xe5, 0xab, 0xc2, 0x87,
0x59, 0x80, 0x8f, 0x1b, 0xb7, 0xa1, 0x5b, 0x78, 0xdd, 0xfc, 0xd4, 0x97, 0x02, 0x45, 0xfd, 0x11,
0x8b, 0x53, 0xbe, 0x1c, 0x87, 0x9f, 0x32, 0x03, 0xbc, 0x22, 0x51, 0xd8, 0x1e, 0x2f, 0x62, 0x2f,
0x7d, 0x26, 0x4c, 0x56, 0xca, 0x87, 0x0a, 0xdb, 0x14, 0xdb, 0x60, 0x5f, 0x79, 0x0a, 0x91, 0xb7,
0x3c, 0xd0, 0x22, 0xf5, 0x27, 0x4e, 0x76, 0xa1, 0xc3, 0x92, 0x40, 0x6b, 0x37, 0x50, 0xdb, 0x66,
0x49, 0x80, 0xaa, 0x11, 0x6c, 0x9b, 0xd7, 0x84, 0x54, 0x60, 0x86, 0x32, 0x79, 0xc9, 0x3d, 0xe3,
0x09, 0xe7, 0x48, 0xcc, 0x8e, 0x8d, 0xa5, 0xb7, 0xa5, 0x1f, 0x14, 0xcc, 0x90, 0xbc, 0x07, 0xb6,
0xda, 0xa5, 0x58, 0xa8, 0x7d, 0xe1, 0x85, 0x7a, 0x2c, 0x09, 0xf2, 0x81, 0xfb, 0x2b, 0x0b, 0xae,
0x9e, 0xa2, 0xf0, 0x12, 0x71, 0xf4, 0x08, 0x3a, 0x63, 0x36, 0x53, 0x4b, 0xe4, 0x6f, 0x24, 0xfb,
0x67, 0x3d, 0xb9, 0x9d, 0xe1, 0x30, 0xaf, 0x58, 0xc0, 0xfd, 0xb8, 0x70, 0xeb, 0xfb, 0xd1, 0x42,
0xcc, 0x0f, 0xd3, 0x38, 0x53, 0xf7, 0x35, 0xb8, 0xd4, 0x03, 0xc7, 0xf9, 0x21, 0xee, 0xfe, 0xcc,
0x02, 0xc0, 0xcb, 0x83, 0x5b, 0x9f, 0x0a, 0x4c, 0xeb, 0x32, 0x81, 0xa9, 0xba, 0x9e, 0x64, 0x11,
0xfb, 0x9c, 0x45, 0x54, 0x96, 0xb9, 0x49, 0x98, 0xdd, 0x49, 0xb2, 0x88, 0x3d, 0xad, 0x32, 0x30,
0x85, 0xfb, 0x4b, 0x0b, 0x00, 0x93, 0xab, 0x3e, 0xc6, 0x7a, 0xf5, 0xb3, 0xce, 0xff, 0x5d, 0x69,
0xac, 0x5e, 0xbf, 0x07, 0xf9, 0xf5, 0x13, 0xe8, 0x8f, 0x66, 0x1d, 0x86, 0xc2, 0x1f, 0x25, 0x78,
0x73, 0x43, 0xb5, 0x0f, 0x7e, 0x6d, 0x81, 0x5d, 0x71, 0x95, 0x58, 0xa5, 0xd1, 0x5a, 0xcf, 0x14,
0xd8, 0x11, 0xa9, 0xdb, 0xe3, 0x8b, 0xca, 0x85, 0x8a, 0xcb, 0x0b, 0xb5, 0x0b, 0x1d, 0xa4, 0xa4,
0x72, 0xa3, 0x12, 0x73, 0xa3, 0xee, 0xc0, 0x55, 0xce, 0xa6, 0x2c, 0x91, 0xd1, 0xd2, 0x8f, 0xd3,
0x20, 0x3c, 0x09, 0x59, 0x80, 0xf7, 0xaa, 0xe3, 0xf5, 0x73, 0xc5, 0x91, 0x91, 0xbb, 0x7f, 0xb6,
0x60, 0xfb, 0x87, 0x0b, 0xc6, 0x97, 0x8f, 0xd3, 0x80, 0xe9, 0x93, 0x7d, 0xfe, 0x90, 0x78, 0x07,
0xb1, 0x18, 0x7a, 0x74, 0xb8, 0xbe, 0xfe, 0xaf, 0xc3, 0x55, 0x78, 0x1d, 0x61, 0x42, 0x54, 0x51,
0xac, 0x7f, 0x41, 0x2f, 0x42, 0x71, 0xe9, 0x58, 0x53, 0x36, 0x35, 0xc5, 0x3f, 0xb5, 0xa0, 0x57,
0xb9, 0x98, 0x2a, 0xdd, 0x9b, 0xda, 0xa0, 0x4b, 0x8a, 0x85, 0x09, 0xb7, 0x37, 0x2d, 0x1f, 0x9a,
0xc8, 0x0e, 0x6c, 0xc4, 0x62, 0x66, 0x3c, 0x6e, 0x7b, 0x7a, 0x40, 0x6e, 0x42, 0x27, 0x16, 0x33,
0xec, 0xd4, 0x4d, 0x96, 0x2e, 0xc6, 0xca, 0x6d, 0x45, 0x45, 0x31, 0xc9, 0xaa, 0x14, 0xb8, 0xbf,
0xb3, 0x80, 0x98, 0xa6, 0xe1, 0x85, 0x5e, 0x11, 0x31, 0x60, 0xab, 0x8f, 0x65, 0x0d, 0xdd, 0x39,
0x55, 0x65, 0x6b, 0xe5, 0xae, 0x79, 0xaa, 0xdc, 0xdd, 0x81, 0xab, 0x01, 0x3b, 0xa1, 0xaa, 0xbf,
0x59, 0x3f, 0x72, 0xdf, 0x28, 0x9e, 0xe4, 0xf2, 0x37, 0xde, 0x86, 0x6e, 0xf1, 0x78, 0x4f, 0xfa,
0x60, 0x8f, 0x92, 0x50, 0xe2, 0xaf, 0x44, 0x98, 0xcc, 0xfa, 0x5f, 0x22, 0x3d, 0x68, 0xff, 0x80,
0xd1, 0x48, 0xce, 0x97, 0x7d, 0x8b, 0xd8, 0xd0, 0x79, 0x77, 0x92, 0xa4, 0x3c, 0xa6, 0x51, 0xbf,
0xf1, 0xe0, 0xad, 0x1f, 0x7f, 0x7b, 0x16, 0xca, 0xf9, 0x62, 0xa2, 0x90, 0xec, 0x6b, 0x68, 0xdf,
0x08, 0x53, 0xf3, 0xb5, 0x9f, 0x7b, 0x6d, 0x1f, 0xd1, 0x16, 0xc3, 0x6c, 0x32, 0xd9, 0x44, 0xc9,
0x9b, 0xff, 0x0c, 0x00, 0x00, 0xff, 0xff, 0xef, 0x4f, 0x87, 0x49, 0xe2, 0x18, 0x00, 0x00,
// 1832 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xec, 0x59, 0x4b, 0x93, 0x23, 0x47,
0x11, 0xa6, 0x25, 0xcd, 0x48, 0x4a, 0xf5, 0xcc, 0x6a, 0x6b, 0xc7, 0xeb, 0x9e, 0xd9, 0xf5, 0x5a,
0x6e, 0xf3, 0x18, 0xbc, 0xc1, 0xce, 0x32, 0x06, 0xec, 0x20, 0x08, 0xd6, 0x9e, 0x91, 0xbd, 0x28,
0xd6, 0xb3, 0x31, 0xb4, 0xd6, 0x8e, 0x80, 0x4b, 0x47, 0xa9, 0xbb, 0x46, 0x6a, 0xbb, 0x5f, 0x74,
0x95, 0x76, 0x57, 0x3e, 0x71, 0xe0, 0x46, 0xc0, 0x81, 0x08, 0x4e, 0xfc, 0x07, 0xae, 0x9c, 0x78,
0x84, 0x4f, 0x44, 0xf0, 0x0b, 0xf8, 0x1b, 0x1c, 0xb9, 0x40, 0x54, 0x56, 0xf5, 0x43, 0x9a, 0x9e,
0x61, 0xac, 0x0d, 0x02, 0x1c, 0x70, 0xeb, 0xca, 0xcc, 0x7a, 0xe4, 0x97, 0x5f, 0x65, 0xa6, 0x4a,
0xb0, 0x1d, 0xc4, 0x82, 0x65, 0x31, 0x0d, 0xef, 0xa5, 0x59, 0x22, 0x12, 0xf2, 0x52, 0x14, 0x84,
0x4f, 0xe7, 0x5c, 0x8d, 0xee, 0xe5, 0xca, 0x3d, 0xd3, 0x4b, 0xa2, 0x28, 0x89, 0x95, 0x78, 0xcf,
0xe4, 0xde, 0x8c, 0x45, 0x54, 0x8d, 0xec, 0x3f, 0x18, 0xb0, 0x75, 0x9c, 0x44, 0x69, 0x12, 0xb3,
0x58, 0x8c, 0xe2, 0xb3, 0x84, 0xdc, 0x84, 0xcd, 0x38, 0xf1, 0xd9, 0x68, 0x68, 0x19, 0x03, 0x63,
0xbf, 0xe9, 0xe8, 0x11, 0x21, 0xd0, 0xca, 0x92, 0x90, 0x59, 0x8d, 0x81, 0xb1, 0xdf, 0x75, 0xf0,
0x9b, 0x3c, 0x00, 0xe0, 0x82, 0x0a, 0xe6, 0x7a, 0x89, 0xcf, 0xac, 0xe6, 0xc0, 0xd8, 0xdf, 0x3e,
0x1c, 0xdc, 0xab, 0x3d, 0xc5, 0xbd, 0xb1, 0x34, 0x3c, 0x4e, 0x7c, 0xe6, 0x74, 0x79, 0xfe, 0x49,
0xde, 0x01, 0x60, 0xcf, 0x45, 0x46, 0xdd, 0x20, 0x3e, 0x4b, 0xac, 0xd6, 0xa0, 0xb9, 0xdf, 0x3b,
0x7c, 0x6d, 0x79, 0x01, 0x7d, 0xf8, 0x47, 0x6c, 0xf1, 0x11, 0x0d, 0xe7, 0xec, 0x94, 0x06, 0x99,
0xd3, 0xc5, 0x49, 0xf2, 0xb8, 0xf6, 0x5f, 0x0d, 0xb8, 0x56, 0x38, 0x80, 0x7b, 0x70, 0xf2, 0x5d,
0xd8, 0xc0, 0x2d, 0xd0, 0x83, 0xde, 0xe1, 0x97, 0x2f, 0x38, 0xd1, 0x92, 0xdf, 0x8e, 0x9a, 0x42,
0x3e, 0x84, 0x1b, 0x7c, 0x3e, 0xf1, 0x72, 0x95, 0x8b, 0x52, 0x6e, 0x35, 0xf0, 0x68, 0x57, 0x5b,
0x89, 0x54, 0x17, 0xd0, 0x47, 0x7a, 0x13, 0x36, 0xe5, 0x4a, 0x73, 0x8e, 0x28, 0xf5, 0x0e, 0x6f,
0xd5, 0x3a, 0x39, 0x46, 0x13, 0x47, 0x9b, 0xda, 0xb7, 0x60, 0xf7, 0x21, 0x13, 0x2b, 0xde, 0x39,
0xec, 0x27, 0x73, 0xc6, 0x85, 0x56, 0x3e, 0x09, 0x22, 0xf6, 0x24, 0xf0, 0x3e, 0x39, 0x9e, 0xd1,
0x38, 0x66, 0x61, 0xae, 0x7c, 0x05, 0x6e, 0x3d, 0x64, 0x38, 0x21, 0xe0, 0x22, 0xf0, 0xf8, 0x8a,
0xfa, 0x25, 0xb8, 0xf1, 0x90, 0x89, 0xa1, 0xbf, 0x22, 0xfe, 0x08, 0x3a, 0x8f, 0x65, 0xb0, 0x25,
0x0d, 0xbe, 0x03, 0x6d, 0xea, 0xfb, 0x19, 0xe3, 0x5c, 0xa3, 0x78, 0xbb, 0xf6, 0xc4, 0xef, 0x2a,
0x1b, 0x27, 0x37, 0xae, 0xa3, 0x89, 0xfd, 0x31, 0xc0, 0x28, 0x0e, 0xc4, 0x29, 0xcd, 0x68, 0xc4,
0x2f, 0x24, 0xd8, 0x10, 0x4c, 0x2e, 0x68, 0x26, 0xdc, 0x14, 0xed, 0x34, 0xe4, 0x57, 0x60, 0x43,
0x0f, 0xa7, 0xa9, 0xd5, 0xed, 0x1f, 0x01, 0x8c, 0x45, 0x16, 0xc4, 0xd3, 0x0f, 0x02, 0x2e, 0xe4,
0x5e, 0x4f, 0xa5, 0x9d, 0x74, 0xa2, 0xb9, 0xdf, 0x75, 0xf4, 0xa8, 0x12, 0x8e, 0xc6, 0xd5, 0xc3,
0xf1, 0x00, 0x7a, 0x39, 0xdc, 0x27, 0x7c, 0x4a, 0xee, 0x43, 0x6b, 0x42, 0x39, 0xbb, 0x14, 0x9e,
0x13, 0x3e, 0x3d, 0xa2, 0x9c, 0x39, 0x68, 0x69, 0x7f, 0xd6, 0x80, 0x97, 0x8f, 0x33, 0x86, 0xe4,
0x0f, 0x43, 0xe6, 0x89, 0x20, 0x89, 0x35, 0xf6, 0x9f, 0x7f, 0x35, 0xf2, 0x32, 0xb4, 0xfd, 0x89,
0x1b, 0xd3, 0x28, 0x07, 0x7b, 0xd3, 0x9f, 0x3c, 0xa6, 0x11, 0x23, 0x5f, 0x85, 0x6d, 0xaf, 0x58,
0x5f, 0x4a, 0x90, 0x73, 0x5d, 0x67, 0x45, 0x2a, 0x43, 0xe5, 0x4f, 0x46, 0x43, 0xab, 0x85, 0x61,
0xc0, 0x6f, 0x62, 0x83, 0x59, 0x5a, 0x8d, 0x86, 0xd6, 0x06, 0xea, 0x96, 0x64, 0x12, 0x54, 0x95,
0x43, 0xac, 0xcd, 0x81, 0xb1, 0x6f, 0x3a, 0x7a, 0x44, 0xee, 0xc3, 0x8d, 0xa7, 0x41, 0x26, 0xe6,
0x34, 0xd4, 0xbc, 0x92, 0xbb, 0x70, 0xab, 0x8d, 0xc8, 0xd7, 0xa9, 0xc8, 0x21, 0xec, 0xa4, 0xb3,
0x05, 0x0f, 0xbc, 0x95, 0x29, 0x1d, 0x9c, 0x52, 0xab, 0xb3, 0x3f, 0x33, 0xe0, 0xa5, 0x61, 0x96,
0xa4, 0x5f, 0x64, 0x08, 0xed, 0x5f, 0x34, 0xe0, 0xa6, 0x62, 0xc2, 0x29, 0xcd, 0x44, 0xf0, 0x6f,
0xf2, 0xe2, 0x6b, 0x70, 0xad, 0xdc, 0x55, 0x19, 0xd4, 0xbb, 0xf1, 0x15, 0xd8, 0x4e, 0xf3, 0x73,
0x28, 0xbb, 0x16, 0xda, 0x6d, 0x15, 0xd2, 0x25, 0x6f, 0x37, 0x2e, 0xf1, 0x76, 0xb3, 0x86, 0x30,
0x03, 0xe8, 0x15, 0x0b, 0x8d, 0x86, 0x56, 0x1b, 0x4d, 0xaa, 0x22, 0xfb, 0xe7, 0x0d, 0xd8, 0x91,
0x41, 0xfd, 0x3f, 0x1a, 0x12, 0x8d, 0x3f, 0x36, 0x80, 0x28, 0x76, 0x8c, 0x62, 0x9f, 0x3d, 0xff,
0x4f, 0x62, 0xf1, 0x0a, 0xc0, 0x59, 0xc0, 0x42, 0xbf, 0x8a, 0x43, 0x17, 0x25, 0x2f, 0x84, 0x81,
0x05, 0x6d, 0x5c, 0xa4, 0xf0, 0x3f, 0x1f, 0xca, 0x2a, 0xa0, 0x3a, 0x02, 0x5d, 0x05, 0x3a, 0x57,
0xae, 0x02, 0x38, 0x4d, 0x57, 0x81, 0xdf, 0x36, 0x61, 0x6b, 0x14, 0x73, 0x96, 0x89, 0xff, 0x65,
0x22, 0x91, 0xdb, 0xd0, 0xe5, 0x6c, 0x1a, 0xc9, 0xc6, 0x64, 0x68, 0x75, 0x50, 0x5f, 0x0a, 0xa4,
0xd6, 0x53, 0x99, 0x75, 0x34, 0xb4, 0xba, 0x2a, 0xb4, 0x85, 0x80, 0xdc, 0x01, 0x10, 0x41, 0xc4,
0xb8, 0xa0, 0x51, 0xca, 0x2d, 0x18, 0x34, 0xf7, 0x5b, 0x4e, 0x45, 0x22, 0xab, 0x40, 0x96, 0x3c,
0x1b, 0x0d, 0xb9, 0xd5, 0x1b, 0x34, 0x65, 0x19, 0x57, 0x23, 0xf2, 0x2d, 0xe8, 0x64, 0xc9, 0x33,
0xd7, 0xa7, 0x82, 0x5a, 0x26, 0x06, 0x6f, 0xb7, 0x16, 0xec, 0xa3, 0x30, 0x99, 0x38, 0xed, 0x2c,
0x79, 0x36, 0xa4, 0x82, 0xda, 0x7f, 0x6b, 0xc0, 0xd6, 0x98, 0xd1, 0xcc, 0x9b, 0xad, 0x1f, 0xb0,
0xaf, 0x43, 0x3f, 0x63, 0x7c, 0x1e, 0x0a, 0xb7, 0x74, 0x4b, 0x45, 0xee, 0x9a, 0x92, 0x1f, 0x17,
0xce, 0xe5, 0x90, 0x37, 0x2f, 0x81, 0xbc, 0x55, 0x03, 0xb9, 0x0d, 0x66, 0x05, 0x5f, 0x6e, 0x6d,
0xa0, 0xeb, 0x4b, 0x32, 0xd2, 0x87, 0xa6, 0xcf, 0x43, 0x8c, 0x58, 0xd7, 0x91, 0x9f, 0xe4, 0x2e,
0x5c, 0x4f, 0x43, 0xea, 0xb1, 0x59, 0x12, 0xfa, 0x2c, 0x73, 0xa7, 0x59, 0x32, 0x4f, 0x31, 0x5c,
0xa6, 0xd3, 0xaf, 0x28, 0x1e, 0x4a, 0x39, 0x79, 0x0b, 0x3a, 0x3e, 0x0f, 0x5d, 0xb1, 0x48, 0x19,
0x86, 0x6c, 0xfb, 0x02, 0xdf, 0x87, 0x3c, 0x7c, 0xb2, 0x48, 0x99, 0xd3, 0xf6, 0xd5, 0x07, 0xb9,
0x0f, 0x3b, 0x9c, 0x65, 0x01, 0x0d, 0x83, 0x4f, 0x99, 0xef, 0xb2, 0xe7, 0x69, 0xe6, 0xa6, 0x21,
0x8d, 0x31, 0xb2, 0xa6, 0x43, 0x4a, 0xdd, 0x7b, 0xcf, 0xd3, 0xec, 0x34, 0xa4, 0xb1, 0xfd, 0x8f,
0x0a, 0xe8, 0x12, 0x1f, 0xbe, 0x06, 0xe8, 0xeb, 0x74, 0x52, 0xb5, 0x91, 0x6a, 0xd6, 0x47, 0xea,
0x55, 0xe8, 0x45, 0x4c, 0x64, 0x81, 0xa7, 0x10, 0x51, 0x17, 0x08, 0x94, 0x08, 0xdd, 0x26, 0xd0,
0x9a, 0x05, 0x42, 0x85, 0xc2, 0x74, 0xf0, 0x9b, 0x7c, 0x0f, 0xf6, 0x38, 0xa3, 0x21, 0xf3, 0xdd,
0x82, 0xed, 0xdc, 0xe5, 0xe8, 0x29, 0xf3, 0xad, 0x4d, 0x0c, 0x9a, 0xa5, 0x2c, 0xc6, 0x85, 0xc1,
0x58, 0xeb, 0xc9, 0x01, 0xdc, 0x28, 0x8e, 0x55, 0x99, 0xa6, 0xfa, 0x18, 0x52, 0xaa, 0x8a, 0x09,
0x6f, 0x83, 0x35, 0x0d, 0x93, 0x09, 0x0d, 0xdd, 0x73, 0xbb, 0x62, 0xfe, 0x6a, 0x3a, 0x37, 0x95,
0x7e, 0xbc, 0xb2, 0xa5, 0xfd, 0x9b, 0x06, 0x5c, 0x73, 0xa4, 0x2f, 0xec, 0x29, 0xfb, 0xc2, 0x13,
0xff, 0x0d, 0x68, 0x06, 0x3e, 0x47, 0xe2, 0xf7, 0x0e, 0xad, 0xe5, 0x73, 0xeb, 0x1f, 0x9d, 0xa3,
0x21, 0x77, 0xa4, 0x11, 0x79, 0x1d, 0xb6, 0x92, 0xb9, 0x48, 0xe7, 0xc2, 0xc5, 0xc4, 0x9f, 0x77,
0x89, 0xa6, 0x12, 0xbe, 0x8f, 0x32, 0xfb, 0xf7, 0xcd, 0x2a, 0x3a, 0xff, 0xad, 0x0c, 0xd5, 0x6e,
0xb7, 0xae, 0xe2, 0xf6, 0x03, 0xe8, 0x29, 0x7f, 0x55, 0x7e, 0xdc, 0xc0, 0xfc, 0x78, 0xa7, 0x76,
0x0e, 0x62, 0x20, 0x73, 0xa3, 0xa3, 0x2a, 0x30, 0x97, 0xdf, 0xe4, 0xfb, 0x70, 0xeb, 0x3c, 0xb3,
0x33, 0x8d, 0x51, 0x4e, 0xed, 0xdd, 0x55, 0x6a, 0xe7, 0x20, 0xfa, 0xe4, 0x9b, 0xb0, 0x53, 0xe1,
0x76, 0x39, 0x51, 0x37, 0xe9, 0xa5, 0xae, 0x9c, 0xb2, 0x3e, 0xbb, 0xff, 0x62, 0xc0, 0xd6, 0x90,
0x85, 0x4c, 0xbc, 0x00, 0xb7, 0x6b, 0x8a, 0x6d, 0xa3, 0xb6, 0xd8, 0x2e, 0x55, 0xb3, 0xe6, 0xe5,
0xd5, 0xac, 0x75, 0xae, 0x9a, 0xbd, 0x06, 0x66, 0x9a, 0x05, 0x11, 0xcd, 0x16, 0xee, 0x27, 0x6c,
0x91, 0xf3, 0xbb, 0xa7, 0x65, 0x8f, 0xd8, 0x82, 0xdb, 0x31, 0xec, 0x7d, 0x90, 0x50, 0xff, 0x88,
0x86, 0x34, 0xf6, 0x98, 0x76, 0x93, 0xaf, 0xef, 0xd9, 0x1d, 0x80, 0x0a, 0x92, 0x0d, 0xdc, 0xb0,
0x22, 0xb1, 0xff, 0x6e, 0x40, 0x57, 0x6e, 0x88, 0x3d, 0xe0, 0x1a, 0xeb, 0x2f, 0x15, 0xff, 0x46,
0x4d, 0xf1, 0x2f, 0xda, 0xb8, 0x1c, 0xae, 0xb2, 0xaf, 0xab, 0xf4, 0x67, 0xad, 0xe5, 0xfe, 0xec,
0x55, 0xe8, 0x05, 0xf2, 0x40, 0x6e, 0x4a, 0xc5, 0x4c, 0xe1, 0xd4, 0x75, 0x00, 0x45, 0xa7, 0x52,
0x22, 0x1b, 0xb8, 0xdc, 0x00, 0x1b, 0xb8, 0xcd, 0x2b, 0x37, 0x70, 0x7a, 0x11, 0x6c, 0xe0, 0xfe,
0xd4, 0x00, 0x4b, 0x43, 0x5c, 0xbe, 0x62, 0x7c, 0x98, 0xfa, 0xf8, 0x98, 0x72, 0x1b, 0xba, 0x05,
0xcb, 0xf4, 0x23, 0x42, 0x29, 0x90, 0xb8, 0x9e, 0xb0, 0x28, 0xc9, 0x16, 0xe3, 0xe0, 0x53, 0xa6,
0x1d, 0xaf, 0x48, 0xa4, 0x6f, 0x8f, 0xe7, 0x91, 0x93, 0x3c, 0xe3, 0x3a, 0x0b, 0xe6, 0x43, 0xe9,
0x9b, 0x87, 0x6d, 0xb7, 0x2b, 0x99, 0x81, 0x9e, 0xb7, 0x1c, 0x50, 0x22, 0xf9, 0xcb, 0x9f, 0xec,
0x42, 0x87, 0xc5, 0xbe, 0xd2, 0x6e, 0xa0, 0xb6, 0xcd, 0x62, 0x1f, 0x55, 0x23, 0xd8, 0xd6, 0xaf,
0x17, 0x09, 0xc7, 0x8c, 0xa8, 0xf3, 0xa0, 0x7d, 0xc1, 0x93, 0xd1, 0x09, 0x9f, 0x9e, 0x6a, 0x4b,
0x67, 0x4b, 0x3d, 0x60, 0xe8, 0x21, 0x79, 0x0f, 0x4c, 0xb9, 0x4b, 0xb1, 0x50, 0xfb, 0xca, 0x0b,
0xf5, 0x58, 0xec, 0xe7, 0x03, 0xfb, 0x57, 0x06, 0x5c, 0x3f, 0x07, 0xe1, 0x1a, 0x3c, 0x7a, 0x04,
0x9d, 0x31, 0x9b, 0xca, 0x25, 0xf2, 0x37, 0x99, 0x83, 0x8b, 0x9e, 0xf8, 0x2e, 0x08, 0x98, 0x53,
0x2c, 0x60, 0x7f, 0x5c, 0x84, 0xf5, 0xfd, 0x70, 0xce, 0x67, 0xc7, 0x49, 0x94, 0xca, 0xfc, 0xe0,
0xaf, 0xf5, 0xa0, 0x72, 0x39, 0xc5, 0xed, 0x9f, 0x19, 0x00, 0x78, 0x79, 0x70, 0xeb, 0x73, 0xc4,
0x34, 0xd6, 0x21, 0xa6, 0xec, 0xb2, 0xe2, 0x79, 0xe4, 0x66, 0x2c, 0xa4, 0xa2, 0xcc, 0x85, 0x5c,
0xef, 0x4e, 0xe2, 0x79, 0xe4, 0x28, 0x55, 0x9e, 0x20, 0xec, 0x5f, 0x1a, 0x00, 0x98, 0xcc, 0xd5,
0x31, 0x56, 0xab, 0xad, 0x71, 0xf9, 0xcf, 0xa3, 0xc6, 0xf2, 0xf5, 0x3b, 0xca, 0xaf, 0x1f, 0xc7,
0x78, 0x34, 0xeb, 0x7c, 0x28, 0xe2, 0x51, 0x3a, 0xaf, 0x6f, 0xa8, 0x8a, 0xc1, 0xaf, 0x0d, 0x30,
0x2b, 0xa1, 0xe2, 0xcb, 0x30, 0x1a, 0xab, 0x99, 0x02, 0x3b, 0x30, 0x79, 0x7b, 0x5c, 0x5e, 0xb9,
0x50, 0x51, 0x79, 0xa1, 0x76, 0xa1, 0x83, 0x90, 0x54, 0x6e, 0x54, 0xac, 0x6f, 0xd4, 0x5d, 0xb8,
0x9e, 0x31, 0x8f, 0xc5, 0x22, 0x5c, 0xb8, 0x51, 0xe2, 0x07, 0x67, 0x01, 0xf3, 0xf1, 0x5e, 0x75,
0x9c, 0x7e, 0xae, 0x38, 0xd1, 0x72, 0xfb, 0xcf, 0x06, 0x6c, 0xff, 0x70, 0xce, 0xb2, 0xc5, 0xe3,
0xc4, 0x67, 0xea, 0x64, 0x9f, 0x9f, 0x12, 0xef, 0xa0, 0x2f, 0x1a, 0x1e, 0x45, 0xd7, 0xd7, 0xff,
0x35, 0x5d, 0xb9, 0xd3, 0xe1, 0x9a, 0xa2, 0x12, 0x62, 0xf5, 0x93, 0xf7, 0x2a, 0x10, 0x97, 0x81,
0xd5, 0x65, 0x5a, 0x41, 0xfc, 0x53, 0x03, 0x7a, 0x95, 0x8b, 0x29, 0xcb, 0x8b, 0xae, 0x45, 0xaa,
0x84, 0x19, 0x98, 0x70, 0x7b, 0x5e, 0xf9, 0xb0, 0x45, 0x76, 0x60, 0x23, 0xe2, 0x53, 0x1d, 0x71,
0xd3, 0x51, 0x03, 0xb2, 0x07, 0x9d, 0x88, 0x4f, 0xf1, 0x97, 0x81, 0xce, 0xd2, 0xc5, 0x58, 0x86,
0xad, 0xa8, 0x60, 0x3a, 0x59, 0x95, 0x02, 0xfb, 0x77, 0x06, 0x10, 0xdd, 0xa4, 0xbc, 0xd0, 0xab,
0x25, 0x12, 0xb6, 0xfa, 0x38, 0xd7, 0x50, 0x9d, 0x5a, 0x55, 0xb6, 0x52, 0x5e, 0x9b, 0xe7, 0xca,
0xeb, 0x5d, 0xb8, 0xee, 0xb3, 0x33, 0x2a, 0xfb, 0xa9, 0xd5, 0x23, 0xf7, 0xb5, 0xe2, 0x49, 0x2e,
0x7f, 0xe3, 0x6d, 0xe8, 0x16, 0x7f, 0x16, 0x90, 0x3e, 0x98, 0xa3, 0x38, 0x10, 0xf8, 0xd3, 0x25,
0x88, 0xa7, 0xfd, 0x2f, 0x91, 0x1e, 0xb4, 0x7f, 0xc0, 0x68, 0x28, 0x66, 0x8b, 0xbe, 0x41, 0x4c,
0xe8, 0xbc, 0x3b, 0x89, 0x93, 0x2c, 0xa2, 0x61, 0xbf, 0x71, 0xf4, 0xd6, 0x8f, 0xbf, 0x3d, 0x0d,
0xc4, 0x6c, 0x3e, 0x91, 0x9e, 0x1c, 0x28, 0xd7, 0xbe, 0x11, 0x24, 0xfa, 0xeb, 0x20, 0x8f, 0xda,
0x01, 0x7a, 0x5b, 0x0c, 0xd3, 0xc9, 0x64, 0x13, 0x25, 0x6f, 0xfe, 0x33, 0x00, 0x00, 0xff, 0xff,
0x57, 0x6c, 0x28, 0x16, 0x52, 0x19, 0x00, 0x00,
}

14
internal/proto/query_service.proto
View File

@ -88,6 +88,7 @@ message ReleaseCollectionRequest {
common.MsgBase base = 1;
int64 dbID = 2;
int64 collectionID = 3;
int64 nodeID = 4;
}
message LoadPartitionsRequest {
@ -103,6 +104,7 @@ message ReleasePartitionsRequest {
int64 dbID = 2;
int64 collectionID = 3;
repeated int64 partitionIDs = 4;
int64 nodeID = 5;
}
message CreateQueryChannelRequest {
@ -266,7 +268,8 @@ message CollectionInfo {
int64 collectionID = 1;
repeated int64 partitionIDs = 2;
repeated DmChannelInfo channel_infos = 3;
schema.CollectionSchema schema = 6;
bool load_collection = 4;
schema.CollectionSchema schema = 5;
}
message HandoffSegments {
@ -285,9 +288,8 @@ message LoadBalanceSegmentInfo {
bool valid_info = 8;
}
message LoadBalanceSegments {
message LoadBalanceRequest {
common.MsgBase base = 1;
repeated LoadBalanceSegmentInfo infos = 2;
}
repeated int64 source_nodeIDs = 2;
TriggerCondition balance_reason = 3;
}

319
internal/proto/querypb/query_service.pb.go
View File

@ -493,6 +493,7 @@ type ReleaseCollectionRequest struct {
Base *commonpb.MsgBase `protobuf:"bytes,1,opt,name=base,proto3" json:"base,omitempty"`
DbID int64 `protobuf:"varint,2,opt,name=dbID,proto3" json:"dbID,omitempty"`
CollectionID int64 `protobuf:"varint,3,opt,name=collectionID,proto3" json:"collectionID,omitempty"`
NodeID int64 `protobuf:"varint,4,opt,name=nodeID,proto3" json:"nodeID,omitempty"`
XXX_NoUnkeyedLiteral struct{} `json:"-"`
XXX_unrecognized []byte `json:"-"`
XXX_sizecache int32 `json:"-"`
@ -544,6 +545,13 @@ func (m *ReleaseCollectionRequest) GetCollectionID() int64 {
return 0
}
func (m *ReleaseCollectionRequest) GetNodeID() int64 {
if m != nil {
return m.NodeID
}
return 0
}
type LoadPartitionsRequest struct {
Base *commonpb.MsgBase `protobuf:"bytes,1,opt,name=base,proto3" json:"base,omitempty"`
DbID int64 `protobuf:"varint,2,opt,name=dbID,proto3" json:"dbID,omitempty"`
@ -620,6 +628,7 @@ type ReleasePartitionsRequest struct {
DbID int64 `protobuf:"varint,2,opt,name=dbID,proto3" json:"dbID,omitempty"`
CollectionID int64 `protobuf:"varint,3,opt,name=collectionID,proto3" json:"collectionID,omitempty"`
PartitionIDs []int64 `protobuf:"varint,4,rep,packed,name=partitionIDs,proto3" json:"partitionIDs,omitempty"`
NodeID int64 `protobuf:"varint,5,opt,name=nodeID,proto3" json:"nodeID,omitempty"`
XXX_NoUnkeyedLiteral struct{} `json:"-"`
XXX_unrecognized []byte `json:"-"`
XXX_sizecache int32 `json:"-"`
@ -678,6 +687,13 @@ func (m *ReleasePartitionsRequest) GetPartitionIDs() []int64 {
return nil
}
func (m *ReleasePartitionsRequest) GetNodeID() int64 {
if m != nil {
return m.NodeID
}
return 0
}
type CreateQueryChannelRequest struct {
CollectionID int64 `protobuf:"varint,1,opt,name=collectionID,proto3" json:"collectionID,omitempty"`
ProxyID int64 `protobuf:"varint,2,opt,name=proxyID,proto3" json:"proxyID,omitempty"`
@ -1708,7 +1724,8 @@ type CollectionInfo struct {
CollectionID int64 `protobuf:"varint,1,opt,name=collectionID,proto3" json:"collectionID,omitempty"`
PartitionIDs []int64 `protobuf:"varint,2,rep,packed,name=partitionIDs,proto3" json:"partitionIDs,omitempty"`
ChannelInfos []*DmChannelInfo `protobuf:"bytes,3,rep,name=channel_infos,json=channelInfos,proto3" json:"channel_infos,omitempty"`
Schema *schemapb.CollectionSchema `protobuf:"bytes,6,opt,name=schema,proto3" json:"schema,omitempty"`
LoadCollection bool `protobuf:"varint,4,opt,name=load_collection,json=loadCollection,proto3" json:"load_collection,omitempty"`
Schema *schemapb.CollectionSchema `protobuf:"bytes,5,opt,name=schema,proto3" json:"schema,omitempty"`
XXX_NoUnkeyedLiteral struct{} `json:"-"`
XXX_unrecognized []byte `json:"-"`
XXX_sizecache int32 `json:"-"`
@ -1760,6 +1777,13 @@ func (m *CollectionInfo) GetChannelInfos() []*DmChannelInfo {
return nil
}
func (m *CollectionInfo) GetLoadCollection() bool {
if m != nil {
return m.LoadCollection
}
return false
}
func (m *CollectionInfo) GetSchema() *schemapb.CollectionSchema {
if m != nil {
return m.Schema
@ -1909,53 +1933,61 @@ func (m *LoadBalanceSegmentInfo) GetValidInfo() bool {
return false
}
type LoadBalanceSegments struct {
Base *commonpb.MsgBase `protobuf:"bytes,1,opt,name=base,proto3" json:"base,omitempty"`
Infos []*LoadBalanceSegmentInfo `protobuf:"bytes,2,rep,name=infos,proto3" json:"infos,omitempty"`
XXX_NoUnkeyedLiteral struct{} `json:"-"`
XXX_unrecognized []byte `json:"-"`
XXX_sizecache int32 `json:"-"`
type LoadBalanceRequest struct {
Base *commonpb.MsgBase `protobuf:"bytes,1,opt,name=base,proto3" json:"base,omitempty"`
SourceNodeIDs []int64 `protobuf:"varint,2,rep,packed,name=source_nodeIDs,json=sourceNodeIDs,proto3" json:"source_nodeIDs,omitempty"`
BalanceReason TriggerCondition `protobuf:"varint,3,opt,name=balance_reason,json=balanceReason,proto3,enum=milvus.proto.query.TriggerCondition" json:"balance_reason,omitempty"`
XXX_NoUnkeyedLiteral struct{} `json:"-"`
XXX_unrecognized []byte `json:"-"`
XXX_sizecache int32 `json:"-"`
}
func (m *LoadBalanceSegments) Reset() { *m = LoadBalanceSegments{} }
func (m *LoadBalanceSegments) String() string { return proto.CompactTextString(m) }
func (*LoadBalanceSegments) ProtoMessage() {}
func (*LoadBalanceSegments) Descriptor() ([]byte, []int) {
func (m *LoadBalanceRequest) Reset() { *m = LoadBalanceRequest{} }
func (m *LoadBalanceRequest) String() string { return proto.CompactTextString(m) }
func (*LoadBalanceRequest) ProtoMessage() {}
func (*LoadBalanceRequest) Descriptor() ([]byte, []int) {
return fileDescriptor_5fcb6756dc1afb8d, []int{29}
}
func (m *LoadBalanceSegments) XXX_Unmarshal(b []byte) error {
return xxx_messageInfo_LoadBalanceSegments.Unmarshal(m, b)
func (m *LoadBalanceRequest) XXX_Unmarshal(b []byte) error {
return xxx_messageInfo_LoadBalanceRequest.Unmarshal(m, b)
}
func (m *LoadBalanceSegments) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
return xxx_messageInfo_LoadBalanceSegments.Marshal(b, m, deterministic)
func (m *LoadBalanceRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
return xxx_messageInfo_LoadBalanceRequest.Marshal(b, m, deterministic)
}
func (m *LoadBalanceSegments) XXX_Merge(src proto.Message) {
xxx_messageInfo_LoadBalanceSegments.Merge(m, src)
func (m *LoadBalanceRequest) XXX_Merge(src proto.Message) {
xxx_messageInfo_LoadBalanceRequest.Merge(m, src)
}
func (m *LoadBalanceSegments) XXX_Size() int {
return xxx_messageInfo_LoadBalanceSegments.Size(m)
func (m *LoadBalanceRequest) XXX_Size() int {
return xxx_messageInfo_LoadBalanceRequest.Size(m)
}
func (m *LoadBalanceSegments) XXX_DiscardUnknown() {
xxx_messageInfo_LoadBalanceSegments.DiscardUnknown(m)
func (m *LoadBalanceRequest) XXX_DiscardUnknown() {
xxx_messageInfo_LoadBalanceRequest.DiscardUnknown(m)
}
var xxx_messageInfo_LoadBalanceSegments proto.InternalMessageInfo
var xxx_messageInfo_LoadBalanceRequest proto.InternalMessageInfo
func (m *LoadBalanceSegments) GetBase() *commonpb.MsgBase {
func (m *LoadBalanceRequest) GetBase() *commonpb.MsgBase {
if m != nil {
return m.Base
}
return nil
}
func (m *LoadBalanceSegments) GetInfos() []*LoadBalanceSegmentInfo {
func (m *LoadBalanceRequest) GetSourceNodeIDs() []int64 {
if m != nil {
return m.Infos
return m.SourceNodeIDs
}
return nil
}
func (m *LoadBalanceRequest) GetBalanceReason() TriggerCondition {
if m != nil {
return m.BalanceReason
}
return TriggerCondition_handoff
}
func init() {
proto.RegisterEnum("milvus.proto.query.PartitionState", PartitionState_name, PartitionState_value)
proto.RegisterEnum("milvus.proto.query.TriggerCondition", TriggerCondition_name, TriggerCondition_value)
@ -1989,129 +2021,132 @@ func init() {
proto.RegisterType((*CollectionInfo)(nil), "milvus.proto.query.CollectionInfo")
proto.RegisterType((*HandoffSegments)(nil), "milvus.proto.query.HandoffSegments")
proto.RegisterType((*LoadBalanceSegmentInfo)(nil), "milvus.proto.query.LoadBalanceSegmentInfo")
proto.RegisterType((*LoadBalanceSegments)(nil), "milvus.proto.query.LoadBalanceSegments")
proto.RegisterType((*LoadBalanceRequest)(nil), "milvus.proto.query.LoadBalanceRequest")
}
func init() { proto.RegisterFile("query_service.proto", fileDescriptor_5fcb6756dc1afb8d) }
var fileDescriptor_5fcb6756dc1afb8d = []byte{
// 1856 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xec, 0x58, 0xcd, 0x73, 0x1c, 0x47,
// 1898 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xec, 0x59, 0xcd, 0x73, 0x1c, 0x47,
0x15, 0xd7, 0xec, 0xae, 0x56, 0xda, 0xb7, 0x5f, 0x93, 0xb6, 0x2d, 0xd6, 0x4b, 0x9c, 0x38, 0xe3,
0x04, 0x3b, 0x0a, 0x91, 0x53, 0x4a, 0xa0, 0xc8, 0x81, 0x2a, 0x2c, 0x6d, 0x2c, 0x16, 0x12, 0x45,
0x8c, 0x4c, 0x28, 0x5c, 0x2e, 0x96, 0xd1, 0x4c, 0x6b, 0x35, 0x95, 0x99, 0xee, 0xd5, 0x74, 0xaf,
0x65, 0xfb, 0x00, 0x45, 0x15, 0x17, 0x4e, 0x9c, 0x38, 0x71, 0xe2, 0x42, 0xc1, 0x81, 0xff, 0x81,
0x7f, 0x04, 0x28, 0x8a, 0xdc, 0x38, 0x73, 0xa7, 0xa6, 0xbb, 0x67, 0x76, 0x3e, 0x7a, 0xa5, 0xb5,
0x64, 0x47, 0x29, 0x8a, 0xdb, 0xf4, 0xeb, 0xd7, 0xef, 0xbb, 0x7f, 0xf3, 0xfa, 0xc1, 0x95, 0xe3,
0x29, 0x8e, 0x9e, 0x8e, 0x18, 0x8e, 0x1e, 0xfb, 0x2e, 0xde, 0x98, 0x44, 0x94, 0x53, 0x84, 0x42,
0x3f, 0x78, 0x3c, 0x65, 0x72, 0xb5, 0x21, 0x38, 0xfa, 0x2d, 0x97, 0x86, 0x21, 0x25, 0x92, 0xd6,
0x6f, 0x65, 0x39, 0xfa, 0x1d, 0x9f, 0x70, 0x1c, 0x11, 0x27, 0x48, 0x76, 0x99, 0x7b, 0x84, 0x43,
0x47, 0xad, 0x90, 0xe7, 0x70, 0x27, 0xaf, 0xc1, 0xfa, 0x25, 0x5c, 0xb1, 0xf1, 0xd8, 0x67, 0x1c,
0x47, 0xbb, 0xd4, 0xc3, 0x36, 0x3e, 0x9e, 0x62, 0xc6, 0xd1, 0x7b, 0x50, 0x3b, 0x70, 0x18, 0xee,
0x19, 0x37, 0x8d, 0x3b, 0xcd, 0xcd, 0x57, 0x37, 0x72, 0x76, 0x28, 0x03, 0x3e, 0x61, 0xe3, 0x2d,
0x87, 0x61, 0x5b, 0x70, 0xa2, 0x6f, 0xc3, 0x8a, 0xe3, 0x79, 0x11, 0x66, 0xac, 0x57, 0x39, 0xe5,
0xd0, 0x3d, 0xc9, 0x63, 0x27, 0xcc, 0xd6, 0x6f, 0x0d, 0xb8, 0x9a, 0xb7, 0x80, 0x4d, 0x28, 0x61,
0x18, 0xbd, 0x0f, 0x75, 0xc6, 0x1d, 0x3e, 0x65, 0xca, 0x88, 0xaf, 0x6b, 0xe5, 0xed, 0x0b, 0x16,
0x5b, 0xb1, 0xa2, 0x2d, 0x68, 0xfa, 0xc4, 0xe7, 0xa3, 0x89, 0x13, 0x39, 0x61, 0x62, 0xc9, 0x1b,
0xf9, 0x93, 0x69, 0x8c, 0x86, 0xc4, 0xe7, 0x7b, 0x82, 0xd1, 0x06, 0x3f, 0xfd, 0xb6, 0x7e, 0x06,
0x6b, 0xfb, 0x47, 0xf4, 0x64, 0x9b, 0x06, 0x01, 0x76, 0xb9, 0x4f, 0x09, 0x3b, 0x7f, 0x54, 0x10,
0xd4, 0xbc, 0x83, 0xe1, 0x40, 0x18, 0x52, 0xb5, 0xc5, 0xb7, 0xc5, 0xe1, 0x6b, 0x25, 0xf9, 0x17,
0xf1, 0xf9, 0x4d, 0x68, 0xbb, 0xa9, 0xac, 0xe1, 0x20, 0xf6, 0xba, 0x7a, 0xa7, 0x6a, 0xe7, 0x89,
0xd6, 0xaf, 0x0c, 0xb8, 0x16, 0xab, 0xdd, 0x73, 0x22, 0xee, 0xbf, 0x78, 0xaf, 0x90, 0x05, 0xad,
0xac, 0xc2, 0x5e, 0x55, 0xec, 0xe5, 0x68, 0xd6, 0xb1, 0x8c, 0x6c, 0xd6, 0x84, 0x8b, 0x38, 0x6e,
0x41, 0x6b, 0x92, 0x88, 0x9a, 0xf9, 0x9d, 0xa3, 0x59, 0x7f, 0x35, 0xe0, 0xda, 0xc7, 0xd4, 0xf1,
0x66, 0xd1, 0xfe, 0xd2, 0xdd, 0x46, 0xdf, 0x85, 0xba, 0xbc, 0x87, 0xbd, 0x9a, 0xd0, 0xf5, 0x56,
0x5e, 0x97, 0xba, 0xa3, 0x33, 0x0b, 0xf7, 0x05, 0xc1, 0x56, 0x87, 0xac, 0x5f, 0x1b, 0xd0, 0xb3,
0x71, 0x80, 0x1d, 0x86, 0x2f, 0xd1, 0x0b, 0xeb, 0x0b, 0x15, 0xc9, 0x4b, 0x2c, 0xa0, 0x52, 0xc6,
0x6b, 0xe5, 0x8c, 0x67, 0xa2, 0xbd, 0x7c, 0x9e, 0x68, 0xff, 0x69, 0x16, 0xed, 0xaf, 0xb8, 0xa7,
0xd6, 0x4f, 0xe1, 0xfa, 0x76, 0x84, 0x1d, 0x8e, 0x7f, 0x14, 0xff, 0x18, 0xb6, 0x8f, 0x1c, 0x42,
0x70, 0x90, 0x98, 0x5a, 0x54, 0x62, 0x68, 0x94, 0xf4, 0x60, 0x65, 0x12, 0xd1, 0x27, 0x4f, 0x53,
0xfb, 0x92, 0xa5, 0xf5, 0x07, 0x03, 0xfa, 0x3a, 0xd9, 0x17, 0xb9, 0xae, 0xb7, 0xa1, 0x1b, 0x49,
0xe3, 0x46, 0xae, 0x94, 0x27, 0xb4, 0x36, 0xec, 0x8e, 0x22, 0x2b, 0x2d, 0xe8, 0x2d, 0xe8, 0x44,
0x98, 0x4d, 0x83, 0x19, 0x5f, 0x55, 0xf0, 0xb5, 0x25, 0x55, 0xb1, 0x59, 0x7f, 0x36, 0xe0, 0xfa,
0x0e, 0xe6, 0x69, 0x96, 0x62, 0x75, 0xf8, 0x2b, 0x9a, 0xaa, 0x10, 0xba, 0x05, 0x3b, 0xd1, 0x4d,
0x68, 0x66, 0x58, 0x54, 0x7e, 0xb2, 0x24, 0xf4, 0x1d, 0x58, 0x8e, 0x43, 0x87, 0x85, 0x45, 0x9d,
0x4d, 0x6b, 0xa3, 0xdc, 0x0d, 0x6c, 0xe4, 0xa5, 0xda, 0xf2, 0x80, 0xf5, 0x17, 0x03, 0xfa, 0xba,
0xd0, 0x5c, 0x24, 0x7d, 0x0f, 0x61, 0x2d, 0x35, 0x6e, 0xe4, 0x61, 0xe6, 0x46, 0xfe, 0x44, 0x5c,
0x0e, 0x81, 0xbb, 0xcd, 0xcd, 0x5b, 0x67, 0x9b, 0xc7, 0xec, 0x6b, 0xa9, 0x88, 0x41, 0x46, 0x82,
0xe5, 0xc3, 0xb5, 0x1d, 0xcc, 0xf7, 0xf1, 0x38, 0xc4, 0x84, 0x0f, 0xc9, 0x21, 0x3d, 0x7f, 0x16,
0x5f, 0x03, 0x60, 0x4a, 0x4e, 0xfa, 0x4b, 0xc8, 0x50, 0xac, 0xbf, 0x55, 0xa0, 0x99, 0x51, 0x84,
0x5e, 0x85, 0x46, 0xba, 0xab, 0x92, 0x30, 0x23, 0x94, 0xf2, 0x5f, 0xd1, 0xe4, 0xbf, 0x90, 0xc8,
0x6a, 0x39, 0x91, 0x6b, 0x50, 0x27, 0xd4, 0xc3, 0xc3, 0x81, 0xf8, 0x01, 0x54, 0x6d, 0xb5, 0x42,
0xd7, 0x61, 0x35, 0xc4, 0xe1, 0x88, 0xf9, 0xcf, 0xb0, 0x00, 0xab, 0xaa, 0xbd, 0x12, 0xe2, 0x70,
0xdf, 0x7f, 0x86, 0xe3, 0x2d, 0x32, 0x0d, 0x47, 0x11, 0x3d, 0x61, 0xbd, 0xba, 0xdc, 0x22, 0xd3,
0xd0, 0xa6, 0x27, 0x0c, 0xdd, 0x00, 0xf0, 0x89, 0x87, 0x9f, 0x8c, 0x88, 0x13, 0xe2, 0xde, 0x8a,
0xb8, 0x1a, 0x0d, 0x41, 0xd9, 0x75, 0x42, 0x1c, 0x5f, 0x6a, 0xb1, 0x18, 0x0e, 0x7a, 0xab, 0xf2,
0xa0, 0x5a, 0xc6, 0xae, 0xaa, 0x0b, 0x35, 0x1c, 0xf4, 0x1a, 0xf2, 0x5c, 0x4a, 0x40, 0x1f, 0x41,
0x5b, 0xf9, 0x3d, 0x92, 0x55, 0x07, 0xa2, 0xea, 0x6e, 0xea, 0xd2, 0xaa, 0x02, 0x28, 0x6b, 0xae,
0xc5, 0x32, 0xab, 0xf8, 0x6f, 0xb5, 0x56, 0xcc, 0xe5, 0x45, 0xca, 0xee, 0x5b, 0xb0, 0xec, 0x93,
0x43, 0x9a, 0x54, 0xd9, 0xeb, 0xa7, 0x98, 0x23, 0x94, 0x49, 0x6e, 0xeb, 0xef, 0x06, 0xac, 0xdd,
0xf3, 0x3c, 0x1d, 0x32, 0x3e, 0x7f, 0x4d, 0xcd, 0xf2, 0x57, 0xc9, 0xe5, 0x6f, 0x11, 0x74, 0x78,
0x07, 0x5e, 0x29, 0xa0, 0x9e, 0x2a, 0x83, 0x86, 0x6d, 0xe6, 0x71, 0x6f, 0x38, 0x40, 0x6f, 0x83,
0x99, 0x47, 0xbe, 0xe1, 0x40, 0x14, 0x46, 0xc3, 0xee, 0xe6, 0xb0, 0x6f, 0x38, 0xb0, 0xfe, 0x69,
0xc0, 0x75, 0x1b, 0x87, 0xf4, 0x31, 0xfe, 0xdf, 0xf5, 0xf1, 0x5f, 0x15, 0x58, 0xfb, 0x89, 0xc3,
0xdd, 0xa3, 0x41, 0xa8, 0x88, 0xec, 0x72, 0x1c, 0x2c, 0x5c, 0xf1, 0x5a, 0xf9, 0x8a, 0xa7, 0x65,
0xba, 0xac, 0x2b, 0xd3, 0xf8, 0xe1, 0xb5, 0xf1, 0x59, 0xe2, 0xef, 0xac, 0x4c, 0x33, 0xcd, 0x4a,
0xfd, 0x1c, 0xcd, 0x0a, 0xda, 0x86, 0x36, 0x7e, 0xe2, 0x06, 0x53, 0x0f, 0x8f, 0xa4, 0xf6, 0x15,
0xa1, 0xfd, 0x35, 0x8d, 0xf6, 0xec, 0x1d, 0x69, 0xa9, 0x43, 0x43, 0x71, 0x55, 0xfe, 0x6d, 0x40,
0x57, 0xed, 0xc6, 0xfd, 0xdd, 0x02, 0xa8, 0x58, 0x08, 0x47, 0xa5, 0x1c, 0x8e, 0x45, 0x82, 0x9a,
0xfc, 0x6f, 0x6b, 0x99, 0xff, 0xed, 0x0d, 0x80, 0xc3, 0x60, 0xca, 0x8e, 0x46, 0xdc, 0x0f, 0x13,
0x4c, 0x6c, 0x08, 0xca, 0x03, 0x3f, 0xc4, 0xe8, 0x1e, 0xb4, 0x0e, 0x7c, 0x12, 0xd0, 0xf1, 0x68,
0xe2, 0xf0, 0xa3, 0x18, 0x19, 0xe7, 0xb9, 0x7b, 0xdf, 0xc7, 0x81, 0xb7, 0x25, 0x78, 0xed, 0xa6,
0x3c, 0xb3, 0x17, 0x1f, 0xb1, 0xfe, 0x58, 0x81, 0x2b, 0xb1, 0x9b, 0xca, 0xe3, 0x97, 0x50, 0x50,
0x1f, 0x26, 0xa5, 0x50, 0x9d, 0xff, 0x5f, 0x2c, 0xc4, 0xbb, 0x5c, 0x0e, 0xe7, 0x79, 0x29, 0xa0,
0x1f, 0x42, 0x27, 0xa0, 0x8e, 0x37, 0x72, 0x29, 0xf1, 0x44, 0x26, 0x44, 0x04, 0x3b, 0x9b, 0x6f,
0xea, 0x4c, 0x78, 0x10, 0xf9, 0xe3, 0x31, 0x8e, 0xb6, 0x13, 0x5e, 0xbb, 0x1d, 0x88, 0x77, 0x92,
0x5a, 0x0a, 0x04, 0x55, 0x8d, 0xf0, 0xcb, 0x8b, 0x55, 0x52, 0x03, 0xd5, 0x53, 0x7a, 0xae, 0xda,
0x02, 0x3d, 0xd7, 0xb2, 0xe6, 0x21, 0x90, 0xef, 0x04, 0xea, 0xa5, 0x4e, 0xe0, 0x01, 0xb4, 0x53,
0x5c, 0x11, 0x45, 0x7f, 0x0b, 0xda, 0xd2, 0xac, 0x51, 0x1c, 0x09, 0xec, 0x25, 0x3d, 0xb3, 0x24,
0x7e, 0x2c, 0x68, 0xb1, 0xd4, 0x14, 0xb7, 0xe4, 0x4f, 0xa9, 0x61, 0x67, 0x28, 0xd6, 0xef, 0x0c,
0x30, 0xb3, 0x88, 0x2c, 0x24, 0x2f, 0xd2, 0x8c, 0xdf, 0x86, 0xae, 0x1c, 0x01, 0xcd, 0x60, 0x51,
0xb5, 0xc7, 0xc7, 0x59, 0x71, 0x03, 0xf4, 0x01, 0xac, 0x49, 0xc6, 0x12, 0x8c, 0xca, 0x36, 0xf9,
0xaa, 0xd8, 0xb5, 0x0b, 0x58, 0xfa, 0x0f, 0x03, 0x3a, 0xb3, 0xc2, 0x59, 0xd8, 0xaa, 0x05, 0xde,
0xd8, 0xe8, 0x3e, 0xb4, 0x95, 0x0d, 0xa3, 0x6c, 0xe1, 0xbf, 0xa1, 0xab, 0xba, 0x5c, 0xc4, 0xed,
0x56, 0x06, 0x12, 0x2f, 0x0a, 0x86, 0xd6, 0x2f, 0xa0, 0xfb, 0x7d, 0x87, 0x78, 0xf4, 0xf0, 0x30,
0xa9, 0xd7, 0x73, 0x14, 0xea, 0x87, 0xf9, 0x76, 0xe3, 0x39, 0x2e, 0xaf, 0xf5, 0xfb, 0x0a, 0xac,
0xc5, 0xb4, 0x2d, 0x27, 0x70, 0x88, 0x8b, 0x17, 0x6f, 0x32, 0x5f, 0x0c, 0x9c, 0xde, 0x82, 0x36,
0xa3, 0xd3, 0xc8, 0xc5, 0xa3, 0x5c, 0xaf, 0xd9, 0x92, 0xc4, 0x5d, 0x79, 0xdf, 0x6e, 0x00, 0x78,
0x8c, 0x27, 0x1c, 0x0a, 0x5f, 0x3d, 0xc6, 0xd5, 0xf6, 0xeb, 0xd0, 0x54, 0x32, 0x3c, 0x4a, 0xb0,
0x48, 0xc3, 0xaa, 0x0d, 0x92, 0x34, 0xa0, 0x44, 0xb4, 0xa5, 0xf1, 0x79, 0xb1, 0xbb, 0x22, 0x76,
0x57, 0x3c, 0xc6, 0xc5, 0xd6, 0x0d, 0x80, 0xc7, 0x4e, 0xe0, 0x7b, 0xa2, 0x06, 0x44, 0xeb, 0xb9,
0x6a, 0x37, 0x04, 0x25, 0x0e, 0x81, 0xf5, 0x1b, 0x43, 0xe2, 0x6e, 0x3e, 0x3a, 0xe7, 0x49, 0xd1,
0xf7, 0xf2, 0x29, 0x5a, 0xd7, 0xa5, 0x48, 0x9f, 0x07, 0x95, 0xa9, 0xf5, 0x67, 0xd0, 0xc9, 0x3f,
0x4c, 0x50, 0x0b, 0x56, 0x77, 0x29, 0xff, 0xe8, 0x89, 0xcf, 0xb8, 0xb9, 0x84, 0x3a, 0x00, 0xbb,
0x94, 0xef, 0x45, 0x98, 0x61, 0xc2, 0x4d, 0x03, 0x01, 0xd4, 0x3f, 0x25, 0x03, 0x9f, 0x7d, 0x6e,
0x56, 0xd0, 0x15, 0xf5, 0x92, 0x73, 0x82, 0x21, 0xf9, 0x04, 0x87, 0x34, 0x7a, 0x6a, 0x56, 0xe3,
0xe3, 0xe9, 0xaa, 0x86, 0x4c, 0x68, 0xa5, 0x2c, 0x3b, 0x7b, 0x3f, 0x36, 0x97, 0x51, 0x03, 0x96,
0xe5, 0x67, 0x7d, 0xfd, 0x53, 0x30, 0x8b, 0xc8, 0x8b, 0x9a, 0xb0, 0x72, 0x24, 0x2b, 0xd7, 0x5c,
0x42, 0x5d, 0x68, 0x06, 0x33, 0xeb, 0x4d, 0x23, 0x26, 0x8c, 0xa3, 0x89, 0xab, 0xc0, 0xd7, 0xac,
0xc4, 0xda, 0xe2, 0x04, 0x0e, 0xe8, 0x09, 0x31, 0xab, 0xeb, 0x3f, 0x80, 0x56, 0xb6, 0x1d, 0x47,
0xab, 0x50, 0xdb, 0xa5, 0x04, 0x9b, 0x4b, 0xb1, 0xd8, 0x9d, 0x88, 0x9e, 0xf8, 0x64, 0x2c, 0x7d,
0xb8, 0x1f, 0xd1, 0x67, 0x98, 0x98, 0x95, 0x78, 0x83, 0x61, 0x27, 0x88, 0x37, 0xaa, 0xf1, 0x46,
0xbc, 0xc0, 0x9e, 0x59, 0xdb, 0xfc, 0x0f, 0x40, 0x4b, 0x80, 0xd7, 0xbe, 0x1c, 0x12, 0xa3, 0x09,
0xa0, 0x1d, 0xcc, 0xb7, 0x69, 0x38, 0xa1, 0x24, 0xd1, 0xc0, 0xd0, 0x7b, 0x73, 0x06, 0xaa, 0x65,
0x56, 0x65, 0x74, 0xff, 0x1b, 0x73, 0x4e, 0x14, 0xd8, 0xad, 0x25, 0x14, 0x0a, 0x8d, 0xf1, 0xdf,
0xfe, 0x81, 0xef, 0x7e, 0x9e, 0x8c, 0x04, 0x4e, 0xd1, 0x58, 0x60, 0x4d, 0x34, 0x16, 0x6e, 0xae,
0x5a, 0xec, 0xf3, 0xc8, 0x27, 0xe3, 0xe4, 0x45, 0x62, 0x2d, 0xa1, 0x63, 0xb8, 0x1a, 0xbf, 0x56,
0xb8, 0xc3, 0x7d, 0xc6, 0x7d, 0x97, 0x25, 0x0a, 0x37, 0xe7, 0x2b, 0x2c, 0x31, 0x3f, 0xa7, 0x4a,
0x17, 0x5a, 0xd9, 0x81, 0x37, 0xba, 0xad, 0x2b, 0x60, 0xcd, 0x50, 0xbe, 0x7f, 0xe7, 0x6c, 0xc6,
0x54, 0x49, 0x00, 0xdd, 0xc2, 0x90, 0x19, 0x69, 0x2f, 0x8a, 0x7e, 0xd2, 0xdd, 0x7f, 0x67, 0x21,
0xde, 0x54, 0x9b, 0x0f, 0x9d, 0xfc, 0x60, 0x17, 0xbd, 0x3d, 0x4f, 0x40, 0x69, 0xa8, 0xd6, 0x5f,
0x5f, 0x84, 0x35, 0x55, 0xf5, 0x10, 0x3a, 0xf9, 0x29, 0xa4, 0x5e, 0x95, 0x76, 0x52, 0xd9, 0x3f,
0xed, 0xc5, 0x69, 0x2d, 0xa1, 0x9f, 0xc3, 0x2b, 0xa5, 0xd1, 0x1f, 0xfa, 0xa6, 0x3e, 0xea, 0xfa,
0x09, 0xe1, 0x59, 0x1a, 0x94, 0xf5, 0xb3, 0x28, 0xce, 0xb7, 0xbe, 0x34, 0xeb, 0x5d, 0xdc, 0xfa,
0x8c, 0xf8, 0xd3, 0xac, 0x7f, 0x6e, 0x0d, 0x53, 0x40, 0xe5, 0xa1, 0x20, 0x7a, 0x57, 0xa7, 0x62,
0xee, 0x60, 0xb2, 0xbf, 0xb1, 0x28, 0x7b, 0x9a, 0xf2, 0xa9, 0x80, 0x84, 0xe2, 0xfc, 0x4c, 0xab,
0x76, 0xee, 0x3c, 0x50, 0xaf, 0x76, 0xfe, 0x8c, 0x4c, 0x16, 0x75, 0x7e, 0x90, 0xa1, 0xcf, 0x95,
0x76, 0x70, 0xa5, 0x2f, 0x6a, 0xfd, 0x5c, 0xc4, 0x5a, 0xda, 0xfc, 0x62, 0x15, 0x1a, 0xc2, 0x79,
0x01, 0x08, 0xff, 0x07, 0xdd, 0x17, 0x0f, 0xba, 0x8f, 0xa0, 0x5b, 0x18, 0x07, 0xe9, 0xf1, 0x50,
0x3f, 0x33, 0x3a, 0xeb, 0x62, 0x1c, 0x00, 0x2a, 0xcf, 0x62, 0xf4, 0x15, 0x3a, 0x77, 0x66, 0x73,
0x96, 0x8e, 0x47, 0xd0, 0x2d, 0xcc, 0x42, 0xf4, 0x1e, 0xe8, 0x07, 0x26, 0x67, 0x49, 0xff, 0x0c,
0x5a, 0xd9, 0x57, 0xb1, 0xfe, 0xa7, 0xa4, 0x79, 0x37, 0x5f, 0x3e, 0x28, 0xbd, 0x7c, 0xd0, 0x7e,
0x04, 0xdd, 0xc2, 0x43, 0x58, 0x1f, 0x79, 0xfd, 0x6b, 0xf9, 0x2c, 0xe9, 0x5f, 0x1e, 0xcc, 0x6c,
0x7d, 0xf0, 0x70, 0x73, 0xec, 0xf3, 0xa3, 0xe9, 0x41, 0x6c, 0xc4, 0x5d, 0x79, 0xf2, 0x5d, 0x9f,
0xaa, 0xaf, 0xbb, 0xc9, 0x7d, 0xbb, 0x2b, 0x84, 0xdd, 0x15, 0xc2, 0x26, 0x07, 0x07, 0x75, 0xb1,
0x7c, 0xff, 0xbf, 0x01, 0x00, 0x00, 0xff, 0xff, 0x28, 0x13, 0x87, 0x0e, 0xa2, 0x20, 0x00, 0x00,
0x18, 0x3b, 0x0a, 0x91, 0x53, 0x4a, 0xa0, 0xc8, 0x81, 0x83, 0xa5, 0x8d, 0xc5, 0x42, 0xa2, 0x88,
0x91, 0x09, 0x85, 0xcb, 0xc5, 0x30, 0x3b, 0xd3, 0xda, 0x9d, 0xca, 0x4c, 0xf7, 0x6a, 0x7a, 0xd6,
0xb2, 0x7d, 0x80, 0xa2, 0x8a, 0x03, 0x37, 0x4e, 0x9c, 0xe0, 0xc2, 0x85, 0x2a, 0x0e, 0xfc, 0x03,
0x9c, 0xf2, 0x8f, 0x40, 0x15, 0x45, 0x6e, 0x9c, 0xb9, 0x53, 0xd3, 0xdd, 0x33, 0x3b, 0x1f, 0xbd,
0xd2, 0x5a, 0xb2, 0x63, 0x17, 0xc5, 0x6d, 0xfb, 0xf5, 0xeb, 0xf7, 0xdd, 0xbf, 0x79, 0xfd, 0x16,
0x2e, 0x1d, 0xcf, 0x70, 0xf8, 0xc4, 0x62, 0x38, 0x7c, 0xe4, 0x39, 0x78, 0x6b, 0x1a, 0xd2, 0x88,
0x22, 0x14, 0x78, 0xfe, 0xa3, 0x19, 0x13, 0xab, 0x2d, 0xce, 0xd1, 0x6f, 0x39, 0x34, 0x08, 0x28,
0x11, 0xb4, 0x7e, 0x2b, 0xcb, 0xd1, 0xef, 0x78, 0x24, 0xc2, 0x21, 0xb1, 0xfd, 0x64, 0x97, 0x39,
0x13, 0x1c, 0xd8, 0x72, 0x85, 0x5c, 0x3b, 0xb2, 0xf3, 0x1a, 0x8c, 0x5f, 0xc1, 0x25, 0x13, 0x8f,
0x3d, 0x16, 0xe1, 0x70, 0x9f, 0xba, 0xd8, 0xc4, 0xc7, 0x33, 0xcc, 0x22, 0xf4, 0x3e, 0xd4, 0x46,
0x36, 0xc3, 0x3d, 0xed, 0xba, 0x76, 0xbb, 0xb9, 0xfd, 0xfa, 0x56, 0xce, 0x0e, 0x69, 0xc0, 0xa7,
0x6c, 0xbc, 0x63, 0x33, 0x6c, 0x72, 0x4e, 0xf4, 0x5d, 0x58, 0xb3, 0x5d, 0x37, 0xc4, 0x8c, 0xf5,
0x2a, 0xa7, 0x1c, 0xba, 0x2b, 0x78, 0xcc, 0x84, 0xd9, 0xf8, 0x9d, 0x06, 0x97, 0xf3, 0x16, 0xb0,
0x29, 0x25, 0x0c, 0xa3, 0x0f, 0xa0, 0xce, 0x22, 0x3b, 0x9a, 0x31, 0x69, 0xc4, 0x37, 0x95, 0xf2,
0x0e, 0x39, 0x8b, 0x29, 0x59, 0xd1, 0x0e, 0x34, 0x3d, 0xe2, 0x45, 0xd6, 0xd4, 0x0e, 0xed, 0x20,
0xb1, 0xe4, 0xad, 0xfc, 0xc9, 0x34, 0x46, 0x43, 0xe2, 0x45, 0x07, 0x9c, 0xd1, 0x04, 0x2f, 0xfd,
0x6d, 0xfc, 0x1c, 0x36, 0x0e, 0x27, 0xf4, 0x64, 0x97, 0xfa, 0x3e, 0x76, 0x22, 0x8f, 0x12, 0x76,
0xfe, 0xa8, 0x20, 0xa8, 0xb9, 0xa3, 0xe1, 0x80, 0x1b, 0x52, 0x35, 0xf9, 0x6f, 0x23, 0x82, 0x6f,
0x94, 0xe4, 0x5f, 0xc4, 0xe7, 0xb7, 0xa1, 0xed, 0xa4, 0xb2, 0x86, 0x83, 0xd8, 0xeb, 0xea, 0xed,
0xaa, 0x99, 0x27, 0x1a, 0xbf, 0xd6, 0xe0, 0x4a, 0xac, 0xf6, 0xc0, 0x0e, 0x23, 0xef, 0xf9, 0x7b,
0x85, 0x0c, 0x68, 0x65, 0x15, 0xf6, 0xaa, 0x7c, 0x2f, 0x47, 0x33, 0x8e, 0x45, 0x64, 0xb3, 0x26,
0x5c, 0xc4, 0x71, 0x03, 0x5a, 0xd3, 0x44, 0xd4, 0xdc, 0xef, 0x1c, 0xcd, 0xf8, 0x52, 0x83, 0x2b,
0x9f, 0x50, 0xdb, 0x9d, 0x47, 0xfb, 0x6b, 0x77, 0x1b, 0x7d, 0x1f, 0xea, 0xe2, 0x1e, 0xf6, 0x6a,
0x5c, 0xd7, 0xcd, 0xbc, 0x2e, 0x79, 0x47, 0xe7, 0x16, 0x1e, 0x72, 0x82, 0x29, 0x0f, 0x19, 0x7f,
0xd4, 0xa0, 0x67, 0x62, 0x1f, 0xdb, 0x0c, 0xbf, 0x4c, 0x2f, 0x36, 0xa0, 0x4e, 0xa8, 0x8b, 0x87,
0x03, 0xee, 0x45, 0xd5, 0x94, 0x2b, 0xe3, 0x2b, 0x19, 0xe1, 0x97, 0x58, 0x58, 0xa5, 0x4a, 0xa8,
0x95, 0x2b, 0x21, 0x93, 0x85, 0xd5, 0xf3, 0x64, 0xe1, 0xcb, 0x79, 0x16, 0x5e, 0x75, 0x4f, 0xe7,
0x99, 0x5a, 0xcd, 0x65, 0xea, 0x67, 0x70, 0x75, 0x37, 0xc4, 0x76, 0x84, 0x7f, 0x1c, 0x7f, 0x48,
0x76, 0x27, 0x36, 0x21, 0xd8, 0x4f, 0x5c, 0x28, 0x2a, 0xd7, 0x14, 0xca, 0x7b, 0xb0, 0x36, 0x0d,
0xe9, 0xe3, 0x27, 0xa9, 0xdd, 0xc9, 0xd2, 0xf8, 0x93, 0x06, 0x7d, 0x95, 0xec, 0x8b, 0x5c, 0xef,
0x5b, 0xd0, 0x0d, 0x85, 0x71, 0x96, 0x23, 0xe4, 0x71, 0xad, 0x0d, 0xb3, 0x23, 0xc9, 0x52, 0x0b,
0xba, 0x09, 0x9d, 0x10, 0xb3, 0x99, 0x3f, 0xe7, 0xab, 0x72, 0xbe, 0xb6, 0xa0, 0x4a, 0x36, 0xe3,
0x2f, 0x1a, 0x5c, 0xdd, 0xc3, 0x51, 0x9a, 0xbd, 0x58, 0x1d, 0x7e, 0x35, 0x53, 0x68, 0x04, 0xd0,
0x2d, 0xd8, 0x89, 0xae, 0x43, 0x33, 0xc3, 0x22, 0xf3, 0x93, 0x25, 0xa1, 0xef, 0xc1, 0x6a, 0x1c,
0x3a, 0xcc, 0x2d, 0xea, 0x6c, 0x1b, 0x5b, 0xe5, 0xee, 0x61, 0x2b, 0x2f, 0xd5, 0x14, 0x07, 0x8c,
0xbf, 0x6a, 0xd0, 0x57, 0x85, 0xe6, 0x22, 0xe9, 0x7b, 0x00, 0x1b, 0xa9, 0x71, 0x96, 0x8b, 0x99,
0x13, 0x7a, 0x53, 0x7e, 0x69, 0x38, 0x4e, 0x37, 0xb7, 0x6f, 0x9c, 0x6d, 0x1e, 0x33, 0xaf, 0xa4,
0x22, 0x06, 0x19, 0x09, 0x86, 0x07, 0x57, 0xf6, 0x70, 0x74, 0x88, 0xc7, 0x01, 0x26, 0xd1, 0x90,
0x1c, 0xd1, 0xf3, 0x67, 0xf1, 0x0d, 0x00, 0x26, 0xe5, 0xa4, 0x9f, 0x90, 0x0c, 0xc5, 0xf8, 0x7b,
0x05, 0x9a, 0x19, 0x45, 0xe8, 0x75, 0x68, 0xa4, 0xbb, 0x32, 0x09, 0x73, 0x42, 0x29, 0xff, 0x15,
0x45, 0xfe, 0x0b, 0x89, 0xac, 0x96, 0x13, 0xb9, 0x00, 0x6a, 0xd1, 0x55, 0x58, 0x0f, 0x70, 0x60,
0x31, 0xef, 0x29, 0x96, 0x57, 0x7b, 0x2d, 0xc0, 0xc1, 0xa1, 0xf7, 0x14, 0xc7, 0x5b, 0x64, 0x16,
0x58, 0x21, 0x3d, 0x61, 0xbd, 0xba, 0xd8, 0x22, 0xb3, 0xc0, 0xa4, 0x27, 0x0c, 0x5d, 0x03, 0xf0,
0x88, 0x8b, 0x1f, 0x5b, 0xc4, 0x0e, 0x70, 0x6f, 0x8d, 0x5f, 0x8d, 0x06, 0xa7, 0xec, 0xdb, 0x01,
0x8e, 0x2f, 0x35, 0x5f, 0x0c, 0x07, 0xbd, 0x75, 0x71, 0x50, 0x2e, 0x63, 0x57, 0xe5, 0x85, 0x1a,
0x0e, 0x7a, 0x0d, 0x71, 0x2e, 0x25, 0xa0, 0x8f, 0xa1, 0x2d, 0xfd, 0xb6, 0x44, 0xd5, 0x01, 0xaf,
0xba, 0xeb, 0xaa, 0xb4, 0xca, 0x00, 0x8a, 0x9a, 0x6b, 0xb1, 0xcc, 0xca, 0xf8, 0x8d, 0x06, 0x1b,
0xc5, 0x5c, 0x5e, 0xa4, 0xec, 0xbe, 0x03, 0xab, 0x1e, 0x39, 0xa2, 0x49, 0x95, 0xbd, 0x79, 0x8a,
0x39, 0x5c, 0x99, 0xe0, 0x36, 0xfe, 0xa1, 0xc1, 0xc6, 0x5d, 0xd7, 0x55, 0x21, 0xe3, 0xb3, 0xd7,
0xd4, 0x3c, 0x7f, 0x95, 0x5c, 0xfe, 0x96, 0x41, 0x87, 0x77, 0xe1, 0xb5, 0x02, 0xea, 0xc9, 0x32,
0x68, 0x98, 0x7a, 0x1e, 0xf7, 0x86, 0x03, 0xf4, 0x0e, 0xe8, 0x79, 0xe4, 0x93, 0x98, 0xdf, 0x30,
0xbb, 0x39, 0xec, 0x1b, 0x0e, 0x8c, 0x7f, 0x6a, 0x70, 0xd5, 0xc4, 0x01, 0x7d, 0x84, 0xff, 0x77,
0x7d, 0xfc, 0x57, 0x05, 0x36, 0x7e, 0x6a, 0x47, 0xce, 0x64, 0x10, 0x48, 0x22, 0x7b, 0x39, 0x0e,
0x16, 0xae, 0x78, 0xad, 0x7c, 0xc5, 0xd3, 0x32, 0x5d, 0x55, 0x95, 0x69, 0xfc, 0x50, 0xdb, 0xfa,
0x3c, 0xf1, 0x77, 0x5e, 0xa6, 0x99, 0x26, 0xa6, 0x7e, 0x8e, 0x26, 0x06, 0xed, 0x42, 0x1b, 0x3f,
0x76, 0xfc, 0x99, 0x8b, 0x2d, 0xa1, 0x7d, 0x8d, 0x6b, 0x7f, 0x43, 0xa1, 0x3d, 0x7b, 0x47, 0x5a,
0xf2, 0xd0, 0x90, 0x5f, 0x95, 0x7f, 0x6b, 0xd0, 0x95, 0xbb, 0x71, 0xdf, 0xb7, 0x04, 0x2a, 0x16,
0xc2, 0x51, 0x29, 0x87, 0x63, 0x99, 0xa0, 0x26, 0xdf, 0xdb, 0x5a, 0xe6, 0x7b, 0x7b, 0x0d, 0xe0,
0xc8, 0x9f, 0xb1, 0x89, 0x15, 0x79, 0x41, 0x82, 0x89, 0x0d, 0x4e, 0xb9, 0xef, 0x05, 0x18, 0xdd,
0x85, 0xd6, 0xc8, 0x23, 0x3e, 0x1d, 0x5b, 0x53, 0x3b, 0x9a, 0xc4, 0xc8, 0xb8, 0xc8, 0xdd, 0x7b,
0x1e, 0xf6, 0xdd, 0x1d, 0xce, 0x6b, 0x36, 0xc5, 0x99, 0x83, 0xf8, 0x88, 0xf1, 0xe7, 0x0a, 0x5c,
0x8a, 0xdd, 0x94, 0x1e, 0xbf, 0x80, 0x82, 0xfa, 0x28, 0x29, 0x85, 0xea, 0xe2, 0xef, 0x62, 0x21,
0xde, 0xe5, 0x72, 0x38, 0xcf, 0xcb, 0x02, 0xfd, 0x08, 0x3a, 0x3e, 0xb5, 0x5d, 0xcb, 0xa1, 0xc4,
0xe5, 0x99, 0xe0, 0x11, 0xec, 0x6c, 0xbf, 0xad, 0x32, 0xe1, 0x7e, 0xe8, 0x8d, 0xc7, 0x38, 0xdc,
0x4d, 0x78, 0xcd, 0xb6, 0xcf, 0xdf, 0x55, 0x72, 0xc9, 0x11, 0x54, 0x36, 0xc8, 0x2f, 0x2e, 0x56,
0x49, 0x0d, 0x54, 0x4f, 0xe9, 0xb9, 0x6a, 0x4b, 0xf4, 0x5c, 0xab, 0x8a, 0xb6, 0x39, 0xdf, 0x09,
0xd4, 0x4b, 0x9d, 0xc0, 0x7d, 0x68, 0xa7, 0xb8, 0xc2, 0x8b, 0xfe, 0x06, 0xb4, 0x85, 0x59, 0x56,
0x1c, 0x09, 0xec, 0x26, 0x3d, 0xb3, 0x20, 0x7e, 0xc2, 0x69, 0xb1, 0xd4, 0x14, 0xb7, 0xc4, 0x47,
0xa9, 0x61, 0x66, 0x28, 0xc6, 0xef, 0x35, 0xd0, 0xb3, 0x88, 0xcc, 0x25, 0x2f, 0xd3, 0x8c, 0xdf,
0x82, 0xae, 0x18, 0x19, 0xcd, 0x61, 0x51, 0xb6, 0xc7, 0xc7, 0x59, 0x71, 0x03, 0xf4, 0x21, 0x6c,
0x08, 0xc6, 0x12, 0x8c, 0x8a, 0x36, 0xf9, 0x32, 0xdf, 0x35, 0x0b, 0x58, 0xfa, 0xdb, 0x0a, 0x74,
0xe6, 0x85, 0xb3, 0xb4, 0x55, 0x4b, 0xbc, 0xc9, 0xd1, 0x3d, 0x68, 0x4b, 0x1b, 0xac, 0x6c, 0xe1,
0xbf, 0xa5, 0xaa, 0xba, 0x5c, 0xc4, 0xcd, 0x56, 0x06, 0x12, 0xf9, 0x03, 0x41, 0x96, 0x6f, 0x62,
0x00, 0xcf, 0xfd, 0xba, 0xd9, 0xf1, 0x73, 0x2f, 0xfe, 0x8b, 0x3e, 0xfd, 0x7e, 0x09, 0xdd, 0x1f,
0xd8, 0xc4, 0xa5, 0x47, 0x47, 0x49, 0x61, 0x9f, 0xa3, 0xa2, 0x3f, 0xca, 0xf7, 0x25, 0xcf, 0x70,
0xcb, 0x8d, 0x3f, 0x54, 0x60, 0x23, 0xa6, 0xed, 0xd8, 0xbe, 0x4d, 0x1c, 0xbc, 0x7c, 0x37, 0xfa,
0x7c, 0x70, 0xf7, 0x06, 0xb4, 0x19, 0x9d, 0x85, 0x0e, 0xb6, 0x72, 0x4d, 0x69, 0x4b, 0x10, 0xf7,
0xc5, 0xc5, 0xbc, 0x06, 0xe0, 0xb2, 0xc8, 0xca, 0xbd, 0x3b, 0x1b, 0x2e, 0x8b, 0xe4, 0xf6, 0x9b,
0xd0, 0x94, 0x32, 0x5c, 0x4a, 0x30, 0xff, 0x78, 0xad, 0x9b, 0x20, 0x48, 0x03, 0x4a, 0x78, 0xff,
0x1a, 0x9f, 0xe7, 0xbb, 0x6b, 0x7c, 0x77, 0xcd, 0x65, 0x11, 0xdf, 0xba, 0x06, 0xf0, 0xc8, 0xf6,
0x3d, 0x97, 0x17, 0x0b, 0xef, 0x51, 0xd7, 0xcd, 0x06, 0xa7, 0xc4, 0x21, 0x30, 0xfe, 0xa6, 0x01,
0xca, 0x44, 0xe7, 0xfc, 0x98, 0x73, 0x13, 0x3a, 0x39, 0x3f, 0xd3, 0x41, 0x5a, 0xd6, 0x51, 0x16,
0x83, 0xe6, 0x48, 0xa8, 0xb2, 0x42, 0x6c, 0x33, 0x4a, 0x78, 0xd0, 0x96, 0x06, 0xcd, 0x51, 0x62,
0x66, 0x7c, 0x74, 0xf3, 0x29, 0x74, 0xf2, 0x4f, 0x1e, 0xd4, 0x82, 0xf5, 0x7d, 0x1a, 0x7d, 0xfc,
0xd8, 0x63, 0x91, 0xbe, 0x82, 0x3a, 0x00, 0xfb, 0x34, 0x3a, 0x08, 0x31, 0xc3, 0x24, 0xd2, 0x35,
0x04, 0x50, 0xff, 0x8c, 0x0c, 0x3c, 0xf6, 0x85, 0x5e, 0x41, 0x97, 0xe4, 0x1b, 0xd1, 0xf6, 0x87,
0xe4, 0x53, 0x1c, 0xd0, 0xf0, 0x89, 0x5e, 0x8d, 0x8f, 0xa7, 0xab, 0x1a, 0xd2, 0xa1, 0x95, 0xb2,
0xec, 0x1d, 0xfc, 0x44, 0x5f, 0x45, 0x0d, 0x58, 0x15, 0x3f, 0xeb, 0x9b, 0x9f, 0x81, 0x5e, 0x34,
0x0f, 0x35, 0x61, 0x6d, 0x22, 0x4a, 0x5d, 0x5f, 0x41, 0x5d, 0x68, 0xfa, 0xf3, 0xc0, 0xea, 0x5a,
0x4c, 0x18, 0x87, 0x53, 0x47, 0x86, 0x58, 0xaf, 0xc4, 0xda, 0xe2, 0x58, 0x0d, 0xe8, 0x09, 0xd1,
0xab, 0x9b, 0x3f, 0x84, 0x56, 0xb6, 0xd1, 0x47, 0xeb, 0x50, 0xdb, 0xa7, 0x04, 0xeb, 0x2b, 0xb1,
0xd8, 0xbd, 0x90, 0x9e, 0x78, 0x64, 0x2c, 0x7c, 0xb8, 0x17, 0xd2, 0xa7, 0x98, 0xe8, 0x95, 0x78,
0x83, 0x61, 0xdb, 0x8f, 0x37, 0xaa, 0xf1, 0x46, 0xbc, 0xc0, 0xae, 0x5e, 0xdb, 0xfe, 0x0f, 0x40,
0x8b, 0xc3, 0xe2, 0xa1, 0x18, 0x57, 0xa3, 0x29, 0xa0, 0x3d, 0x1c, 0xed, 0xd2, 0x60, 0x4a, 0x49,
0xa2, 0x81, 0xa1, 0xf7, 0x17, 0x8c, 0x76, 0xcb, 0xac, 0xd2, 0xe8, 0xfe, 0xb7, 0x16, 0x9c, 0x28,
0xb0, 0x1b, 0x2b, 0x28, 0xe0, 0x1a, 0xe3, 0x3e, 0xe2, 0xbe, 0xe7, 0x7c, 0x91, 0x0c, 0x1b, 0x4e,
0xd1, 0x58, 0x60, 0x4d, 0x34, 0x16, 0xae, 0xba, 0x5c, 0x1c, 0x46, 0xa1, 0x47, 0xc6, 0xc9, 0x5b,
0xc7, 0x58, 0x41, 0xc7, 0x70, 0x39, 0x7e, 0x07, 0x45, 0x76, 0xe4, 0xb1, 0xc8, 0x73, 0x58, 0xa2,
0x70, 0x7b, 0xb1, 0xc2, 0x12, 0xf3, 0x33, 0xaa, 0x74, 0xa0, 0x95, 0x1d, 0xbd, 0xa3, 0x5b, 0xaa,
0x12, 0x56, 0xfc, 0x3d, 0xd0, 0xbf, 0x7d, 0x36, 0x63, 0xaa, 0xc4, 0x87, 0x6e, 0x61, 0xdc, 0x8d,
0x36, 0x95, 0xe0, 0xa7, 0x9c, 0xb9, 0xf7, 0xdf, 0x5d, 0x8a, 0x37, 0xd5, 0xe6, 0x41, 0x27, 0x3f,
0x62, 0x46, 0xef, 0x2c, 0x12, 0x50, 0x1a, 0xe3, 0xf5, 0x37, 0x97, 0x61, 0x4d, 0x55, 0x3d, 0x80,
0x4e, 0x7e, 0xee, 0xa9, 0x56, 0xa5, 0x9c, 0x8d, 0xf6, 0x4f, 0x7b, 0xcb, 0x1a, 0x2b, 0xe8, 0x17,
0xf0, 0x5a, 0x69, 0xd8, 0x88, 0xbe, 0xad, 0x8e, 0xba, 0x7a, 0x26, 0x79, 0x96, 0x06, 0x69, 0x7d,
0xe6, 0x2b, 0xb9, 0xd0, 0xfa, 0xd2, 0xd4, 0x79, 0x79, 0xeb, 0x33, 0xe2, 0x4f, 0xb3, 0xfe, 0x99,
0x35, 0xcc, 0x00, 0x95, 0xc7, 0x8d, 0xe8, 0x3d, 0x95, 0x8a, 0x85, 0x23, 0xcf, 0xfe, 0xd6, 0xb2,
0xec, 0x69, 0xca, 0x67, 0x1c, 0x12, 0x8a, 0x93, 0x39, 0xa5, 0xda, 0x85, 0x93, 0x46, 0xb5, 0xda,
0xc5, 0xd3, 0x37, 0x51, 0xd4, 0xf9, 0x11, 0x89, 0x3a, 0x57, 0xca, 0x91, 0x98, 0xba, 0xa8, 0xd5,
0x13, 0x17, 0x63, 0x65, 0xfb, 0xab, 0x75, 0x68, 0x70, 0xe7, 0x39, 0x20, 0xfc, 0x1f, 0x74, 0x9f,
0x3f, 0xe8, 0x3e, 0x84, 0x6e, 0x61, 0xd0, 0xa4, 0xc6, 0x43, 0xf5, 0x34, 0xea, 0xac, 0x8b, 0x31,
0x02, 0x54, 0x9e, 0xf2, 0xa8, 0x2b, 0x74, 0xe1, 0x34, 0xe8, 0x2c, 0x1d, 0x0f, 0xa1, 0x5b, 0x98,
0xb2, 0xa8, 0x3d, 0x50, 0x8f, 0x62, 0xce, 0x92, 0xfe, 0x39, 0xb4, 0xb2, 0xef, 0x6d, 0xf5, 0x47,
0x49, 0xf1, 0x22, 0x7f, 0xf9, 0xa0, 0xf4, 0xe2, 0x41, 0xfb, 0x21, 0x74, 0x0b, 0x4f, 0x6c, 0x75,
0xe4, 0xd5, 0xef, 0xf0, 0xb3, 0xa4, 0x7f, 0x7d, 0x30, 0xb3, 0xf3, 0xe1, 0x83, 0xed, 0xb1, 0x17,
0x4d, 0x66, 0xa3, 0xd8, 0x88, 0x3b, 0xe2, 0xe4, 0x7b, 0x1e, 0x95, 0xbf, 0xee, 0x24, 0xf7, 0xed,
0x0e, 0x17, 0x76, 0x87, 0x0b, 0x9b, 0x8e, 0x46, 0x75, 0xbe, 0xfc, 0xe0, 0xbf, 0x01, 0x00, 0x00,
0xff, 0xff, 0xcb, 0x4e, 0xe3, 0xca, 0x2c, 0x21, 0x00, 0x00,
}
// Reference imports to suppress errors if they are not otherwise used.

11
internal/querynode/collection_replica.go
View File

@ -25,6 +25,7 @@ import "C"
import (
"errors"
"fmt"
etcdkv "github.com/milvus-io/milvus/internal/kv/etcd"
"github.com/milvus-io/milvus/internal/proto/datapb"
"strconv"
"sync"
@ -92,6 +93,8 @@ type collectionReplica struct {
segments map[UniqueID]*Segment
excludedSegments map[UniqueID][]*datapb.SegmentInfo // map[collectionID]segmentIDs
etcdKV *etcdkv.EtcdKV
}
//----------------------------------------------------------------------------------------------------- collection
@ -396,6 +399,11 @@ func (colReplica *collectionReplica) removeSegmentPrivate(segmentID UniqueID) er
partition.removeSegmentID(segmentID)
delete(colReplica.segments, segmentID)
deleteSegment(segment)
key := fmt.Sprintf("%s/%d", queryNodeSegmentMetaPrefix, segmentID)
err = colReplica.etcdKV.Remove(key)
if err != nil {
log.Error("error when remove segment info from etcd")
}
return nil
}
@ -551,7 +559,7 @@ func (colReplica *collectionReplica) freeAll() {
colReplica.segments = make(map[UniqueID]*Segment)
}
func newCollectionReplica() ReplicaInterface {
func newCollectionReplica(etcdKv *etcdkv.EtcdKV) ReplicaInterface {
collections := make(map[UniqueID]*Collection)
partitions := make(map[UniqueID]*Partition)
segments := make(map[UniqueID]*Segment)
@ -563,6 +571,7 @@ func newCollectionReplica() ReplicaInterface {
segments: segments,
excludedSegments: excludedSegments,
etcdKV: etcdKv,
}
return replica

2
internal/querynode/flow_graph_service_time_node.go
View File

@ -69,7 +69,7 @@ func (stNode *serviceTimeNode) Operate(in []flowgraph.Msg) []flowgraph.Msg {
// zap.Int64("tSafe", int64(serviceTimeMsg.timeRange.timestampMax)),
// zap.Any("collectionID", stNode.collectionID),
// zap.Any("id", id),
// zap.Any("channel", channelTmp),
// zap.Any("channel", stNode.vChannel),
//)
//if err := stNode.sendTimeTick(serviceTimeMsg.timeRange.timestampMax); err != nil {

11
internal/querynode/historical.go
View File

@ -16,6 +16,7 @@ import (
"errors"
"fmt"
etcdkv "github.com/milvus-io/milvus/internal/kv/etcd"
"go.uber.org/zap"
"github.com/milvus-io/milvus/internal/log"
@ -27,15 +28,19 @@ type historical struct {
replica ReplicaInterface
loader *segmentLoader
statsService *statsService
//TODO
globalSealedSegments []UniqueID
}
func newHistorical(ctx context.Context,
masterService types.MasterService,
dataService types.DataService,
indexService types.IndexService,
factory msgstream.Factory) *historical {
replica := newCollectionReplica()
loader := newSegmentLoader(ctx, masterService, indexService, dataService, replica)
factory msgstream.Factory,
etcdKV *etcdkv.EtcdKV) *historical {
replica := newCollectionReplica(etcdKV)
loader := newSegmentLoader(ctx, masterService, indexService, dataService, replica, etcdKV)
ss := newStatsService(ctx, replica, loader.indexLoader.fieldStatsChan, factory)
return &historical{

111
internal/querynode/query_node.go
View File

@ -27,19 +27,19 @@ import "C"
import (
"context"
"errors"
"fmt"
"github.com/milvus-io/milvus/internal/kv"
etcdkv "github.com/milvus-io/milvus/internal/kv/etcd"
"github.com/milvus-io/milvus/internal/util/retry"
"go.etcd.io/etcd/clientv3"
"go.uber.org/zap"
"math/rand"
"strconv"
"sync/atomic"
"time"
"go.uber.org/zap"
"github.com/milvus-io/milvus/internal/log"
"github.com/milvus-io/milvus/internal/msgstream"
"github.com/milvus-io/milvus/internal/proto/commonpb"
"github.com/milvus-io/milvus/internal/proto/internalpb"
queryPb "github.com/milvus-io/milvus/internal/proto/querypb"
"github.com/milvus-io/milvus/internal/types"
"github.com/milvus-io/milvus/internal/util/sessionutil"
"github.com/milvus-io/milvus/internal/util/typeutil"
@ -70,6 +70,9 @@ type QueryNode struct {
scheduler *taskScheduler
session *sessionutil.Session
minioKV kv.BaseKV // minio minioKV
etcdKV *etcdkv.EtcdKV
}
func NewQueryNode(ctx context.Context, queryNodeID UniqueID, factory msgstream.Factory) *QueryNode {
@ -111,57 +114,75 @@ func (node *QueryNode) Register() error {
node.session = sessionutil.NewSession(node.queryNodeLoopCtx, Params.MetaRootPath, Params.EtcdEndpoints)
node.session.Init(typeutil.QueryNodeRole, Params.QueryNodeIP+":"+strconv.FormatInt(Params.QueryNodePort, 10), false)
Params.QueryNodeID = node.session.ServerID
log.Debug("query nodeID", zap.Int64("nodeID", Params.QueryNodeID))
return nil
}
func (node *QueryNode) Init() error {
ctx := context.Background()
//ctx := context.Background()
connectEtcdFn := func() error {
etcdClient, err := clientv3.New(clientv3.Config{Endpoints: Params.EtcdEndpoints})
if err != nil {
return err
}
etcdKV := etcdkv.NewEtcdKV(etcdClient, Params.MetaRootPath)
node.etcdKV = etcdKV
return err
}
log.Debug("queryNode try to connect etcd")
err := retry.Retry(100000, time.Millisecond*200, connectEtcdFn)
if err != nil {
log.Debug("queryNode try to connect etcd failed", zap.Error(err))
return err
}
log.Debug("queryNode try to connect etcd success")
node.historical = newHistorical(node.queryNodeLoopCtx,
node.masterService,
node.dataService,
node.indexService,
node.msFactory)
node.streaming = newStreaming(node.queryNodeLoopCtx, node.msFactory)
node.msFactory,
node.etcdKV)
node.streaming = newStreaming(node.queryNodeLoopCtx, node.msFactory, node.etcdKV)
C.SegcoreInit()
registerReq := &queryPb.RegisterNodeRequest{
Base: &commonpb.MsgBase{
SourceID: Params.QueryNodeID,
},
Address: &commonpb.Address{
Ip: Params.QueryNodeIP,
Port: Params.QueryNodePort,
},
}
resp, err := node.queryService.RegisterNode(ctx, registerReq)
if err != nil {
log.Debug("QueryNode RegisterNode failed", zap.Error(err))
panic(err)
}
if resp.Status.ErrorCode != commonpb.ErrorCode_Success {
log.Debug("QueryNode RegisterNode failed", zap.Any("Reason", resp.Status.Reason))
panic(resp.Status.Reason)
}
log.Debug("QueryNode RegisterNode success")
for _, kv := range resp.InitParams.StartParams {
switch kv.Key {
case "StatsChannelName":
Params.StatsChannelName = kv.Value
case "TimeTickChannelName":
Params.QueryTimeTickChannelName = kv.Value
case "SearchChannelName":
Params.SearchChannelNames = append(Params.SearchChannelNames, kv.Value)
case "SearchResultChannelName":
Params.SearchResultChannelNames = append(Params.SearchResultChannelNames, kv.Value)
default:
return fmt.Errorf("Invalid key: %v", kv.Key)
}
}
log.Debug("QueryNode Init ", zap.Int64("QueryNodeID", Params.QueryNodeID), zap.Any("searchChannelNames", Params.SearchChannelNames))
//registerReq := &queryPb.RegisterNodeRequest{
// Base: &commonpb.MsgBase{
// SourceID: Params.QueryNodeID,
// },
// Address: &commonpb.Address{
// Ip: Params.QueryNodeIP,
// Port: Params.QueryNodePort,
// },
//}
//
//resp, err := node.queryService.RegisterNode(ctx, registerReq)
//if err != nil {
// log.Debug("QueryNode RegisterNode failed", zap.Error(err))
// panic(err)
//}
//if resp.Status.ErrorCode != commonpb.ErrorCode_Success {
// log.Debug("QueryNode RegisterNode failed", zap.Any("Reason", resp.Status.Reason))
// panic(resp.Status.Reason)
//}
//log.Debug("QueryNode RegisterNode success")
//
//for _, kv := range resp.InitParams.StartParams {
// switch kv.Key {
// case "StatsChannelName":
// Params.StatsChannelName = kv.Value
// case "TimeTickChannelName":
// Params.QueryTimeTickChannelName = kv.Value
// case "SearchChannelName":
// Params.SearchChannelNames = append(Params.SearchChannelNames, kv.Value)
// case "SearchResultChannelName":
// Params.SearchResultChannelNames = append(Params.SearchResultChannelNames, kv.Value)
// default:
// return fmt.Errorf("Invalid key: %v", kv.Key)
// }
//}
//
//log.Debug("QueryNode Init ", zap.Int64("QueryNodeID", Params.QueryNodeID), zap.Any("searchChannelNames", Params.SearchChannelNames))
if node.masterService == nil {
log.Error("null master service detected")

12
internal/querynode/query_node_test.go
View File

@ -20,7 +20,9 @@ import (
"time"
"github.com/stretchr/testify/assert"
"go.etcd.io/etcd/clientv3"
etcdkv "github.com/milvus-io/milvus/internal/kv/etcd"
"github.com/milvus-io/milvus/internal/msgstream"
"github.com/milvus-io/milvus/internal/proto/commonpb"
"github.com/milvus-io/milvus/internal/proto/etcdpb"
@ -163,6 +165,12 @@ func newQueryNodeMock() *QueryNode {
}()
}
etcdClient, err := clientv3.New(clientv3.Config{Endpoints: Params.EtcdEndpoints})
if err != nil {
panic(err)
}
etcdKV := etcdkv.NewEtcdKV(etcdClient, Params.MetaRootPath)
msFactory, err := newMessageStreamFactory()
if err != nil {
panic(err)
@ -172,8 +180,8 @@ func newQueryNodeMock() *QueryNode {
if err != nil {
panic(err)
}
svr.historical = newHistorical(svr.queryNodeLoopCtx, nil, nil, nil, svr.msFactory)
svr.streaming = newStreaming(ctx, msFactory)
svr.historical = newHistorical(svr.queryNodeLoopCtx, nil, nil, nil, svr.msFactory, etcdKV)
svr.streaming = newStreaming(ctx, msFactory, etcdKV)
return svr
}

61
internal/querynode/search_collection.go
View File

@ -227,33 +227,34 @@ func (s *searchCollection) consumeSearch() {
}
func (s *searchCollection) loadBalance(msg *msgstream.LoadBalanceSegmentsMsg) {
log.Debug("consume load balance message",
zap.Int64("msgID", msg.ID()))
nodeID := Params.QueryNodeID
for _, info := range msg.Infos {
segmentID := info.SegmentID
if nodeID == info.SourceNodeID {
err := s.historical.replica.removeSegment(segmentID)
if err != nil {
log.Error("loadBalance failed when remove segment",
zap.Error(err),
zap.Any("segmentID", segmentID))
}
}
if nodeID == info.DstNodeID {
segment, err := s.historical.replica.getSegmentByID(segmentID)
if err != nil {
log.Error("loadBalance failed when making segment on service",
zap.Error(err),
zap.Any("segmentID", segmentID))
continue // not return, try to load balance all segment
}
segment.setOnService(true)
}
}
log.Debug("load balance done",
zap.Int64("msgID", msg.ID()),
zap.Int("num of segment", len(msg.Infos)))
//TODO:: get loadBalance info from etcd
//log.Debug("consume load balance message",
// zap.Int64("msgID", msg.ID()))
//nodeID := Params.QueryNodeID
//for _, info := range msg.Infos {
// segmentID := info.SegmentID
// if nodeID == info.SourceNodeID {
// err := s.historical.replica.removeSegment(segmentID)
// if err != nil {
// log.Error("loadBalance failed when remove segment",
// zap.Error(err),
// zap.Any("segmentID", segmentID))
// }
// }
// if nodeID == info.DstNodeID {
// segment, err := s.historical.replica.getSegmentByID(segmentID)
// if err != nil {
// log.Error("loadBalance failed when making segment on service",
// zap.Error(err),
// zap.Any("segmentID", segmentID))
// continue // not return, try to load balance all segment
// }
// segment.setOnService(true)
// }
//}
//log.Debug("load balance done",
// zap.Int64("msgID", msg.ID()),
// zap.Int("num of segment", len(msg.Infos)))
}
func (s *searchCollection) receiveSearch(msg *msgstream.SearchMsg) {
@ -346,9 +347,9 @@ func (s *searchCollection) doUnsolvedMsgSearch() {
zap.Any("tSafe", st))
s.setServiceableTime(serviceTime)
log.Debug("query node::doUnsolvedMsgSearch: setServiceableTime",
zap.Any("serviceTime", st),
)
//log.Debug("query node::doUnsolvedMsgSearch: setServiceableTime",
// zap.Any("serviceTime", st),
//)
searchMsg := make([]*msgstream.SearchMsg, 0)
tempMsg := s.popAllUnsolvedMsg()

41
internal/querynode/segment_loader.go
View File

@ -17,9 +17,11 @@ import (
"fmt"
"strconv"
"github.com/golang/protobuf/proto"
"go.uber.org/zap"
"github.com/milvus-io/milvus/internal/kv"
etcdkv "github.com/milvus-io/milvus/internal/kv/etcd"
minioKV "github.com/milvus-io/milvus/internal/kv/minio"
"github.com/milvus-io/milvus/internal/log"
"github.com/milvus-io/milvus/internal/proto/commonpb"
@ -29,13 +31,19 @@ import (
"github.com/milvus-io/milvus/internal/types"
)
const (
queryServiceSegmentMetaPrefix = "queryService-segmentMeta"
queryNodeSegmentMetaPrefix = "queryNode-segmentMeta"
)
// segmentLoader is only responsible for loading the field data from binlog
type segmentLoader struct {
historicalReplica ReplicaInterface
dataService types.DataService
kv kv.BaseKV // minio kv
minioKV kv.BaseKV // minio minioKV
etcdKV *etcdkv.EtcdKV
indexLoader *indexLoader
}
@ -100,6 +108,30 @@ func (loader *segmentLoader) loadSegment(req *queryPb.LoadSegmentsRequest, onSer
if err != nil {
deleteSegment(segment)
log.Error(err.Error())
continue
}
if onService {
key := fmt.Sprintf("%s/%d", queryServiceSegmentMetaPrefix, segmentID)
value, err := loader.etcdKV.Load(key)
if err != nil {
deleteSegment(segment)
log.Error("error when load segment info from etcd", zap.Any("error", err.Error()))
continue
}
segmentInfo := &queryPb.SegmentInfo{}
err = proto.UnmarshalText(value, segmentInfo)
if err != nil {
deleteSegment(segment)
log.Error("error when unmarshal segment info from etcd", zap.Any("error", err.Error()))
continue
}
segmentInfo.SegmentState = queryPb.SegmentState_sealed
newKey := fmt.Sprintf("%s/%d", queryNodeSegmentMetaPrefix, segmentID)
err = loader.etcdKV.Save(newKey, proto.MarshalTextString(segmentInfo))
if err != nil {
deleteSegment(segment)
log.Error("error when update segment info to etcd", zap.Any("error", err.Error()))
}
}
}
@ -211,7 +243,7 @@ func (loader *segmentLoader) loadSegmentFieldsData(segment *Segment, binlogPaths
Value: make([]byte, 0),
}
for _, path := range paths {
binLog, err := loader.kv.Load(path)
binLog, err := loader.minioKV.Load(path)
if err != nil {
// TODO: return or continue?
return err
@ -273,7 +305,7 @@ func (loader *segmentLoader) loadSegmentFieldsData(segment *Segment, binlogPaths
return nil
}
func newSegmentLoader(ctx context.Context, masterService types.MasterService, indexService types.IndexService, dataService types.DataService, replica ReplicaInterface) *segmentLoader {
func newSegmentLoader(ctx context.Context, masterService types.MasterService, indexService types.IndexService, dataService types.DataService, replica ReplicaInterface, etcdKV *etcdkv.EtcdKV) *segmentLoader {
option := &minioKV.Option{
Address: Params.MinioEndPoint,
AccessKeyID: Params.MinioAccessKeyID,
@ -294,7 +326,8 @@ func newSegmentLoader(ctx context.Context, masterService types.MasterService, in
dataService: dataService,
kv: client,
minioKV: client,
etcdKV: etcdKV,
indexLoader: iLoader,
}

5
internal/querynode/streaming.go
View File

@ -16,6 +16,7 @@ import (
"errors"
"fmt"
etcdkv "github.com/milvus-io/milvus/internal/kv/etcd"
"go.uber.org/zap"
"github.com/milvus-io/milvus/internal/log"
@ -32,8 +33,8 @@ type streaming struct {
msFactory msgstream.Factory
}
func newStreaming(ctx context.Context, factory msgstream.Factory) *streaming {
replica := newCollectionReplica()
func newStreaming(ctx context.Context, factory msgstream.Factory, etcdKV *etcdkv.EtcdKV) *streaming {
replica := newCollectionReplica(etcdKV)
tReplica := newTSafeReplica()
newDS := newDataSyncService(ctx, replica, tReplica, factory)

12
internal/querynode/tsafe.go
View File

@ -16,8 +16,6 @@ import (
"math"
"sync"
"go.uber.org/zap"
"github.com/milvus-io/milvus/internal/log"
)
@ -107,11 +105,11 @@ func (ts *tSafe) start() {
watcher.notify()
}
log.Debug("set tSafe done",
zap.Any("id", m.id),
zap.Any("channel", ts.channel),
zap.Any("t", m.t),
zap.Any("tSafe", ts.tSafe))
//log.Debug("set tSafe done",
// zap.Any("id", m.id),
// zap.Any("channel", ts.channel),
// zap.Any("t", m.t),
// zap.Any("tSafe", ts.tSafe))
ts.tSafeMu.Unlock()
}
}

233
internal/queryservice/cluster.go
View File

@ -13,37 +13,106 @@ package queryservice
import (
"context"
"encoding/json"
"errors"
"fmt"
"path/filepath"
"strconv"
"sync"
"github.com/golang/protobuf/proto"
"go.uber.org/zap"
etcdkv "github.com/milvus-io/milvus/internal/kv/etcd"
"github.com/milvus-io/milvus/internal/log"
"github.com/milvus-io/milvus/internal/proto/commonpb"
"github.com/milvus-io/milvus/internal/proto/internalpb"
"github.com/milvus-io/milvus/internal/proto/querypb"
"github.com/milvus-io/milvus/internal/util/sessionutil"
)
const (
queryNodeMetaPrefix = "queryService-queryNodeMeta"
queryNodeInfoPrefix = "queryService-queryNodeInfo"
)
type queryNodeCluster struct {
client *etcdkv.EtcdKV
sync.RWMutex
clusterMeta *meta
nodes map[int64]*queryNode
}
func newQueryNodeCluster(clusterMeta *meta) *queryNodeCluster {
func newQueryNodeCluster(clusterMeta *meta, kv *etcdkv.EtcdKV) (*queryNodeCluster, error) {
nodes := make(map[int64]*queryNode)
return &queryNodeCluster{
c := &queryNodeCluster{
client: kv,
clusterMeta: clusterMeta,
nodes: nodes,
}
err := c.reloadFromKV()
if err != nil {
return nil, err
}
return c, nil
}
func (c *queryNodeCluster) GetComponentInfos(ctx context.Context) []*internalpb.ComponentInfo {
func (c *queryNodeCluster) reloadFromKV() error {
nodeIDs := make([]UniqueID, 0)
keys, values, err := c.client.LoadWithPrefix(queryNodeInfoPrefix)
if err != nil {
return err
}
for index := range keys {
nodeID, err := strconv.ParseInt(filepath.Base(keys[index]), 10, 64)
if err != nil {
return err
}
nodeIDs = append(nodeIDs, nodeID)
session := &sessionutil.Session{}
err = json.Unmarshal([]byte(values[index]), session)
if err != nil {
return err
}
err = c.RegisterNode(session, nodeID)
if err != nil {
return err
}
}
for _, nodeID := range nodeIDs {
infoPrefix := fmt.Sprintf("%s/%d", queryNodeMetaPrefix, nodeID)
collectionKeys, collectionValues, err := c.client.LoadWithPrefix(infoPrefix)
if err != nil {
return err
}
for index := range collectionKeys {
collectionID, err := strconv.ParseInt(filepath.Base(collectionKeys[index]), 10, 64)
if err != nil {
return err
}
collectionInfo := &querypb.CollectionInfo{}
err = proto.UnmarshalText(collectionValues[index], collectionInfo)
if err != nil {
return err
}
c.nodes[nodeID].collectionInfos[collectionID] = collectionInfo
}
}
return nil
}
func (c *queryNodeCluster) GetComponentInfos(ctx context.Context) ([]*internalpb.ComponentInfo, error) {
c.RLock()
defer c.RUnlock()
subComponentInfos := make([]*internalpb.ComponentInfo, 0)
for nodeID, node := range c.nodes {
nodeIDs, err := c.getOnServiceNodeIDs()
if err != nil {
return nil, err
}
for _, nodeID := range nodeIDs {
node := c.nodes[nodeID]
componentStates, err := node.client.GetComponentStates(ctx)
if err != nil {
subComponentInfos = append(subComponentInfos, &internalpb.ComponentInfo{
@ -55,49 +124,59 @@ func (c *queryNodeCluster) GetComponentInfos(ctx context.Context) []*internalpb.
subComponentInfos = append(subComponentInfos, componentStates.State)
}
return subComponentInfos
return subComponentInfos, nil
}
func (c *queryNodeCluster) LoadSegments(ctx context.Context, nodeID int64, in *querypb.LoadSegmentsRequest) (*commonpb.Status, error) {
c.Lock()
defer c.Unlock()
if node, ok := c.nodes[nodeID]; ok {
//TODO::etcd
log.Debug("load segment infos", zap.Any("infos", in))
if !node.isOnService() {
return nil, errors.New("node offline")
}
segmentInfos := make(map[UniqueID]*querypb.SegmentInfo)
for _, info := range in.Infos {
segmentID := info.SegmentID
if info, ok := c.clusterMeta.segmentInfos[segmentID]; ok {
info.SegmentState = querypb.SegmentState_sealing
segmentInfo, err := c.clusterMeta.getSegmentInfoByID(segmentID)
if err == nil {
segmentInfos[segmentID] = proto.Clone(segmentInfo).(*querypb.SegmentInfo)
if in.LoadCondition != querypb.TriggerCondition_loadBalance {
segmentInfo.SegmentState = querypb.SegmentState_sealing
segmentInfo.NodeID = nodeID
}
} else {
segmentInfo = &querypb.SegmentInfo{
SegmentID: segmentID,
CollectionID: info.CollectionID,
PartitionID: info.PartitionID,
NodeID: nodeID,
SegmentState: querypb.SegmentState_sealing,
}
}
segmentInfo := &querypb.SegmentInfo{
SegmentID: segmentID,
CollectionID: info.CollectionID,
PartitionID: info.PartitionID,
NodeID: nodeID,
SegmentState: querypb.SegmentState_sealing,
}
c.clusterMeta.segmentInfos[segmentID] = segmentInfo
c.clusterMeta.setSegmentInfo(segmentID, segmentInfo)
}
status, err := node.client.LoadSegments(ctx, in)
if err == nil && status.ErrorCode == commonpb.ErrorCode_Success {
for _, info := range in.Infos {
if !c.clusterMeta.hasCollection(info.CollectionID) {
c.clusterMeta.addCollection(info.CollectionID, in.Schema)
}
c.clusterMeta.addCollection(info.CollectionID, in.Schema)
c.clusterMeta.addPartition(info.CollectionID, info.PartitionID)
if !node.hasCollection(info.CollectionID) {
node.addCollection(info.CollectionID, in.Schema)
}
node.addCollection(info.CollectionID, in.Schema)
node.addPartition(info.CollectionID, info.PartitionID)
}
return status, err
}
for _, info := range in.Infos {
segmentID := info.SegmentID
c.clusterMeta.deleteSegmentInfoByID(segmentID)
} else {
for _, info := range in.Infos {
segmentID := info.SegmentID
if _, ok = segmentInfos[segmentID]; ok {
c.clusterMeta.setSegmentInfo(segmentID, segmentInfos[segmentID])
continue
}
c.clusterMeta.removeSegmentInfo(segmentID)
c.clusterMeta.deleteSegmentInfoByID(segmentID)
}
}
return status, err
}
return nil, errors.New("Can't find query node by nodeID ")
@ -108,6 +187,15 @@ func (c *queryNodeCluster) ReleaseSegments(ctx context.Context, nodeID int64, in
defer c.Unlock()
if node, ok := c.nodes[nodeID]; ok {
if !node.isOnService() {
return nil, errors.New("node offline")
}
for _, segmentID := range in.SegmentIDs {
err := c.clusterMeta.removeSegmentInfo(segmentID)
if err != nil {
log.Error("remove segmentInfo Error", zap.Any("error", err.Error()), zap.Int64("segmentID", segmentID))
}
}
status, err := node.client.ReleaseSegments(ctx, in)
if err == nil && status.ErrorCode == commonpb.ErrorCode_Success {
for _, segmentID := range in.SegmentIDs {
@ -123,7 +211,11 @@ func (c *queryNodeCluster) ReleaseSegments(ctx context.Context, nodeID int64, in
func (c *queryNodeCluster) WatchDmChannels(ctx context.Context, nodeID int64, in *querypb.WatchDmChannelsRequest) (*commonpb.Status, error) {
c.Lock()
defer c.Unlock()
if node, ok := c.nodes[nodeID]; ok {
if !node.isOnService() {
return nil, errors.New("node offline")
}
channels := make([]string, 0)
for _, info := range in.Infos {
channels = append(channels, info.ChannelName)
@ -133,14 +225,13 @@ func (c *queryNodeCluster) WatchDmChannels(ctx context.Context, nodeID int64, in
log.Debug("queryNode watch dm channel done")
if err == nil && status.ErrorCode == commonpb.ErrorCode_Success {
collectionID := in.CollectionID
if !c.clusterMeta.hasCollection(collectionID) {
c.clusterMeta.addCollection(collectionID, in.Schema)
}
c.clusterMeta.addCollection(collectionID, in.Schema)
c.clusterMeta.addDmChannel(collectionID, nodeID, channels)
if !node.hasCollection(collectionID) {
node.addCollection(collectionID, in.Schema)
}
node.addCollection(collectionID, in.Schema)
node.addDmChannel(collectionID, channels)
} else {
}
return status, err
}
@ -151,8 +242,6 @@ func (c *queryNodeCluster) hasWatchedQueryChannel(ctx context.Context, nodeID in
c.Lock()
defer c.Unlock()
//TODO::should reopen
//collectionID = 0
return c.nodes[nodeID].hasWatchedQueryChannel(collectionID)
}
@ -237,8 +326,13 @@ func (c *queryNodeCluster) getSegmentInfo(ctx context.Context, in *querypb.GetSe
defer c.Unlock()
segmentInfos := make([]*querypb.SegmentInfo, 0)
for _, node := range c.nodes {
res, err := node.client.GetSegmentInfo(ctx, in)
nodes, err := c.getOnServiceNodeIDs()
if err != nil {
log.Warn(err.Error())
return segmentInfos, nil
}
for _, nodeID := range nodes {
res, err := c.nodes[nodeID].client.GetSegmentInfo(ctx, in)
if err != nil {
return nil, err
}
@ -284,16 +378,69 @@ func (c *queryNodeCluster) getNumSegments(nodeID int64) (int, error) {
return numSegment, nil
}
func (c *queryNodeCluster) RegisterNode(ip string, port int64, id UniqueID) error {
node, err := newQueryNode(ip, port, id)
func (c *queryNodeCluster) RegisterNode(session *sessionutil.Session, id UniqueID) error {
c.Lock()
defer c.Unlock()
sessionJSON, err := json.Marshal(session)
if err != nil {
return err
}
c.Lock()
defer c.Unlock()
key := fmt.Sprintf("%s/%d", queryNodeInfoPrefix, id)
err = c.client.Save(key, string(sessionJSON))
if err != nil {
return err
}
node, err := newQueryNode(session.Address, id, c.client)
if err != nil {
return err
}
log.Debug("register a new query node", zap.Int64("nodeID", id), zap.String("address", session.Address))
if _, ok := c.nodes[id]; !ok {
c.nodes[id] = node
return nil
}
return fmt.Errorf("node %d alredy exists in cluster", id)
}
func (c *queryNodeCluster) removeNodeInfo(nodeID int64) error {
key := fmt.Sprintf("%s/%d", queryNodeInfoPrefix, nodeID)
return c.client.Remove(key)
}
func (c *queryNodeCluster) onServiceNodeIDs() ([]int64, error) {
c.Lock()
defer c.Unlock()
return c.getOnServiceNodeIDs()
}
func (c *queryNodeCluster) getOnServiceNodeIDs() ([]int64, error) {
nodeIDs := make([]int64, 0)
for nodeID, node := range c.nodes {
if node.isOnService() {
nodeIDs = append(nodeIDs, nodeID)
}
}
if len(nodeIDs) == 0 {
return nil, errors.New("no queryNode is alive")
}
return nodeIDs, nil
}
func (c *queryNodeCluster) printMeta() {
for id, node := range c.nodes {
if node.isOnService() {
for collectionID, info := range node.collectionInfos {
log.Debug("queryService cluster info: collectionInfo", zap.Int64("nodeID", id), zap.Int64("collectionID", collectionID), zap.Any("info", info))
}
for collectionID, info := range node.watchedQueryChannels {
log.Debug("queryService cluster info: watchedQueryChannelInfo", zap.Int64("nodeID", id), zap.Int64("collectionID", collectionID), zap.Any("info", info))
}
}
}
}

161
internal/queryservice/impl.go
View File

@ -15,7 +15,6 @@ import (
"context"
"errors"
"fmt"
"strconv"
"go.uber.org/zap"
@ -24,6 +23,7 @@ import (
"github.com/milvus-io/milvus/internal/proto/internalpb"
"github.com/milvus-io/milvus/internal/proto/milvuspb"
"github.com/milvus-io/milvus/internal/proto/querypb"
"github.com/milvus-io/milvus/internal/util/sessionutil"
)
func (qs *QueryService) GetComponentStates(ctx context.Context) (*internalpb.ComponentStates, error) {
@ -31,13 +31,22 @@ func (qs *QueryService) GetComponentStates(ctx context.Context) (*internalpb.Com
NodeID: Params.QueryServiceID,
StateCode: qs.stateCode.Load().(internalpb.StateCode),
}
subComponentInfos := qs.cluster.GetComponentInfos(ctx)
//subComponentInfos, err := qs.cluster.GetComponentInfos(ctx)
//if err != nil {
// return &internalpb.ComponentStates{
// Status: &commonpb.Status{
// ErrorCode: commonpb.ErrorCode_UnexpectedError,
// Reason: err.Error(),
// },
// }, err
//}
return &internalpb.ComponentStates{
Status: &commonpb.Status{
ErrorCode: commonpb.ErrorCode_Success,
},
State: serviceComponentInfo,
SubcomponentStates: subComponentInfos,
State: serviceComponentInfo,
//SubcomponentStates: subComponentInfos,
}, nil
}
@ -77,7 +86,11 @@ func (qs *QueryService) RegisterNode(ctx context.Context, req *querypb.RegisterN
}, err
}
err := qs.cluster.RegisterNode(req.Address.Ip, req.Address.Port, req.Base.SourceID)
session := &sessionutil.Session{
ServerID: nodeID,
Address: fmt.Sprintf("%s:%d", req.Address.Ip, req.Address.Port),
}
err := qs.cluster.RegisterNode(session, req.Base.SourceID)
if err != nil {
log.Debug("register query node new NodeClient failed", zap.Any("QueryNodeID", nodeID), zap.String("address", req.Address.String()))
return &querypb.RegisterNodeResponse{
@ -101,9 +114,9 @@ func (qs *QueryService) RegisterNode(ctx context.Context, req *querypb.RegisterN
func (qs *QueryService) ShowCollections(ctx context.Context, req *querypb.ShowCollectionsRequest) (*querypb.ShowCollectionsResponse, error) {
dbID := req.DbID
log.Debug("show collection start, dbID = ", zap.String("dbID", strconv.FormatInt(dbID, 10)))
log.Debug("show collection start", zap.Int64("dbID", dbID))
collectionIDs := qs.meta.showCollections()
log.Debug("show collection end")
log.Debug("show collection end", zap.Int64s("collections", collectionIDs))
return &querypb.ShowCollectionsResponse{
Status: &commonpb.Status{
ErrorCode: commonpb.ErrorCode_Success,
@ -114,7 +127,7 @@ func (qs *QueryService) ShowCollections(ctx context.Context, req *querypb.ShowCo
func (qs *QueryService) LoadCollection(ctx context.Context, req *querypb.LoadCollectionRequest) (*commonpb.Status, error) {
collectionID := req.CollectionID
schema := req.Schema
//schema := req.Schema
log.Debug("LoadCollectionRequest received", zap.String("role", Params.RoleName), zap.Int64("msgID", req.Base.MsgID), zap.Int64("collectionID", collectionID),
zap.Stringer("schema", req.Schema))
status := &commonpb.Status{
@ -122,19 +135,21 @@ func (qs *QueryService) LoadCollection(ctx context.Context, req *querypb.LoadCol
}
hasCollection := qs.meta.hasCollection(collectionID)
if !hasCollection {
err := qs.meta.addCollection(collectionID, schema)
if err != nil {
log.Error(err.Error())
return status, err
}
if hasCollection {
status.ErrorCode = commonpb.ErrorCode_Success
status.Reason = "collection has been loaded"
return status, nil
}
//err := qs.meta.addCollection(collectionID, schema)
//if err != nil {
// log.Error(err.Error())
// return status, err
//}
loadCollectionTask := &LoadCollectionTask{
BaseTask: BaseTask{
ctx: qs.loopCtx,
Condition: NewTaskCondition(qs.loopCtx),
ctx: qs.loopCtx,
Condition: NewTaskCondition(qs.loopCtx),
triggerCondition: querypb.TriggerCondition_grpcRequest,
},
LoadCollectionRequest: req,
@ -145,11 +160,12 @@ func (qs *QueryService) LoadCollection(ctx context.Context, req *querypb.LoadCol
}
qs.scheduler.Enqueue([]task{loadCollectionTask})
err := loadCollectionTask.WaitToFinish()
if err != nil {
status.Reason = err.Error()
return status, err
}
//err := loadCollectionTask.WaitToFinish()
//if err != nil {
// status.Reason = err.Error()
// return status, err
//}
//qs.meta.setLoadCollection(collectionID, true)
log.Debug("LoadCollectionRequest completed", zap.String("role", Params.RoleName), zap.Int64("msgID", req.Base.MsgID), zap.Int64("collectionID", collectionID))
status.ErrorCode = commonpb.ErrorCode_Success
@ -171,9 +187,8 @@ func (qs *QueryService) ReleaseCollection(ctx context.Context, req *querypb.Rele
releaseCollectionTask := &ReleaseCollectionTask{
BaseTask: BaseTask{
ctx: qs.loopCtx,
Condition: NewTaskCondition(qs.loopCtx),
ctx: qs.loopCtx,
Condition: NewTaskCondition(qs.loopCtx),
triggerCondition: querypb.TriggerCondition_grpcRequest,
},
ReleaseCollectionRequest: req,
@ -189,11 +204,14 @@ func (qs *QueryService) ReleaseCollection(ctx context.Context, req *querypb.Rele
}
log.Debug("ReleaseCollectionRequest completed", zap.String("role", Params.RoleName), zap.Int64("msgID", req.Base.MsgID), zap.Int64("collectionID", collectionID))
qs.meta.printMeta()
qs.cluster.printMeta()
return status, nil
}
func (qs *QueryService) ShowPartitions(ctx context.Context, req *querypb.ShowPartitionsRequest) (*querypb.ShowPartitionsResponse, error) {
collectionID := req.CollectionID
log.Debug("show partitions start, ", zap.Int64("collectionID", collectionID))
partitionIDs, err := qs.meta.showPartitions(collectionID)
if err != nil {
return &querypb.ShowPartitionsResponse{
@ -204,6 +222,8 @@ func (qs *QueryService) ShowPartitions(ctx context.Context, req *querypb.ShowPar
}, err
}
log.Debug("show partitions end", zap.Int64("collectionID", collectionID), zap.Int64s("partitionIDs", partitionIDs))
return &querypb.ShowPartitionsResponse{
Status: &commonpb.Status{
ErrorCode: commonpb.ErrorCode_Success,
@ -213,50 +233,18 @@ func (qs *QueryService) ShowPartitions(ctx context.Context, req *querypb.ShowPar
}
func (qs *QueryService) LoadPartitions(ctx context.Context, req *querypb.LoadPartitionsRequest) (*commonpb.Status, error) {
//TODO::suggest different partitions have different dm channel
collectionID := req.CollectionID
partitionIDs := req.PartitionIDs
log.Debug("LoadPartitionRequest received", zap.String("role", Params.RoleName), zap.Int64("msgID", req.Base.MsgID), zap.Int64("collectionID", collectionID), zap.Int64s("partitionIDs", partitionIDs))
status, err := LoadPartitionMetaCheck(qs.meta, req)
if err != nil {
return status, err
}
loadPartitionTask := &LoadPartitionTask{
BaseTask: BaseTask{
ctx: qs.loopCtx,
Condition: NewTaskCondition(qs.loopCtx),
triggerCondition: querypb.TriggerCondition_grpcRequest,
},
LoadPartitionsRequest: req,
masterService: qs.masterServiceClient,
dataService: qs.dataServiceClient,
cluster: qs.cluster,
meta: qs.meta,
}
qs.scheduler.Enqueue([]task{loadPartitionTask})
err = loadPartitionTask.WaitToFinish()
if err != nil {
status.ErrorCode = commonpb.ErrorCode_UnexpectedError
status.Reason = err.Error()
return status, err
}
log.Debug("LoadPartitionRequest completed", zap.String("role", Params.RoleName), zap.Int64("msgID", req.Base.MsgID), zap.Int64("collectionID", req.CollectionID))
return status, nil
}
func LoadPartitionMetaCheck(meta *meta, req *querypb.LoadPartitionsRequest) (*commonpb.Status, error) {
//dbID := req.DbID
collectionID := req.CollectionID
partitionIDs := req.PartitionIDs
schema := req.Schema
status := &commonpb.Status{
ErrorCode: commonpb.ErrorCode_UnexpectedError,
}
hasCollection := qs.meta.hasCollection(collectionID)
if hasCollection && qs.meta.collectionInfos[collectionID].LoadCollection {
status.ErrorCode = commonpb.ErrorCode_Success
status.Reason = "collection has been loaded"
return status, nil
}
if len(partitionIDs) == 0 {
err := errors.New("partitionIDs are empty")
@ -264,30 +252,39 @@ func LoadPartitionMetaCheck(meta *meta, req *querypb.LoadPartitionsRequest) (*co
return status, err
}
hasCollection := meta.hasCollection(collectionID)
if !hasCollection {
err := meta.addCollection(collectionID, schema)
if err != nil {
status.Reason = err.Error()
return status, err
}
}
partitionIDsToLoad := make([]UniqueID, 0)
for _, partitionID := range partitionIDs {
hasPartition := meta.hasPartition(collectionID, partitionID)
hasPartition := qs.meta.hasPartition(collectionID, partitionID)
if !hasPartition {
err := meta.addPartition(collectionID, partitionID)
if err != nil {
status.Reason = err.Error()
return status, err
}
partitionIDsToLoad = append(partitionIDsToLoad, partitionID)
}
}
req.PartitionIDs = partitionIDsToLoad
if len(req.PartitionIDs) > 0 {
loadPartitionTask := &LoadPartitionTask{
BaseTask: BaseTask{
ctx: qs.loopCtx,
Condition: NewTaskCondition(qs.loopCtx),
triggerCondition: querypb.TriggerCondition_grpcRequest,
},
LoadPartitionsRequest: req,
dataService: qs.dataServiceClient,
cluster: qs.cluster,
meta: qs.meta,
}
qs.scheduler.Enqueue([]task{loadPartitionTask})
//err := loadPartitionTask.WaitToFinish()
//if err != nil {
// status.ErrorCode = commonpb.ErrorCode_UnexpectedError
// status.Reason = err.Error()
// return status, err
//}
}
status.ErrorCode = commonpb.ErrorCode_Success
log.Debug("LoadPartitionRequest completed", zap.String("role", Params.RoleName), zap.Int64("msgID", req.Base.MsgID), zap.Int64("collectionID", req.CollectionID))
return status, nil
}
@ -311,9 +308,8 @@ func (qs *QueryService) ReleasePartitions(ctx context.Context, req *querypb.Rele
req.PartitionIDs = toReleasedPartitionID
releasePartitionTask := &ReleasePartitionTask{
BaseTask: BaseTask{
ctx: qs.loopCtx,
Condition: NewTaskCondition(qs.loopCtx),
ctx: qs.loopCtx,
Condition: NewTaskCondition(qs.loopCtx),
triggerCondition: querypb.TriggerCondition_grpcRequest,
},
ReleasePartitionsRequest: req,
@ -329,7 +325,8 @@ func (qs *QueryService) ReleasePartitions(ctx context.Context, req *querypb.Rele
}
}
log.Debug("ReleasePartitionRequest completed", zap.String("role", Params.RoleName), zap.Int64("msgID", req.Base.MsgID), zap.Int64("collectionID", collectionID), zap.Int64s("partitionIDs", partitionIDs))
//TODO:: queryNodeCluster cancel subscribe dmChannels
qs.meta.printMeta()
qs.cluster.printMeta()
return status, nil
}

269
internal/queryservice/meta.go
View File

@ -13,17 +13,29 @@ package queryservice
import (
"errors"
"fmt"
"path/filepath"
"strconv"
"sync"
"github.com/golang/protobuf/proto"
"go.uber.org/zap"
etcdkv "github.com/milvus-io/milvus/internal/kv/etcd"
"github.com/milvus-io/milvus/internal/log"
"github.com/milvus-io/milvus/internal/proto/querypb"
"github.com/milvus-io/milvus/internal/proto/schemapb"
)
const (
collectionMetaPrefix = "queryService-collectionMeta"
segmentMetaPrefix = "queryService-segmentMeta"
queryChannelMetaPrefix = "queryService-queryChannel"
)
type meta struct {
client *etcdkv.EtcdKV // client of a reliable kv service, i.e. etcd client
sync.RWMutex
collectionInfos map[UniqueID]*querypb.CollectionInfo
segmentInfos map[UniqueID]*querypb.SegmentInfo
@ -32,17 +44,82 @@ type meta struct {
partitionStates map[UniqueID]querypb.PartitionState
}
func newMeta() *meta {
func newMeta(kv *etcdkv.EtcdKV) (*meta, error) {
collectionInfos := make(map[UniqueID]*querypb.CollectionInfo)
segmentInfos := make(map[UniqueID]*querypb.SegmentInfo)
queryChannelInfos := make(map[UniqueID]*querypb.QueryChannelInfo)
partitionStates := make(map[UniqueID]querypb.PartitionState)
return &meta{
m := &meta{
client: kv,
collectionInfos: collectionInfos,
segmentInfos: segmentInfos,
queryChannelInfos: queryChannelInfos,
partitionStates: partitionStates,
}
err := m.reloadFromKV()
if err != nil {
return nil, err
}
return m, nil
}
func (m *meta) reloadFromKV() error {
collectionKeys, collectionValues, err := m.client.LoadWithPrefix(collectionMetaPrefix)
if err != nil {
return err
}
for index := range collectionKeys {
collectionID, err := strconv.ParseInt(filepath.Base(collectionKeys[index]), 10, 64)
if err != nil {
return err
}
collectionInfo := &querypb.CollectionInfo{}
err = proto.UnmarshalText(collectionValues[index], collectionInfo)
if err != nil {
return err
}
m.collectionInfos[collectionID] = collectionInfo
}
segmentKeys, segmentValues, err := m.client.LoadWithPrefix(segmentMetaPrefix)
if err != nil {
return err
}
for index := range segmentKeys {
segmentID, err := strconv.ParseInt(filepath.Base(segmentKeys[index]), 10, 64)
if err != nil {
return err
}
segmentInfo := &querypb.SegmentInfo{}
err = proto.UnmarshalText(segmentValues[index], segmentInfo)
if err != nil {
return err
}
m.segmentInfos[segmentID] = segmentInfo
}
queryChannelKeys, queryChannelValues, err := m.client.LoadWithPrefix(queryChannelMetaPrefix)
if err != nil {
return nil
}
for index := range queryChannelKeys {
collectionID, err := strconv.ParseInt(filepath.Base(queryChannelKeys[index]), 10, 64)
if err != nil {
return err
}
queryChannelInfo := &querypb.QueryChannelInfo{}
err = proto.UnmarshalText(queryChannelValues[index], queryChannelInfo)
if err != nil {
return err
}
m.queryChannelInfos[collectionID] = queryChannelInfo
}
//TODO::update partition states
return nil
}
func (m *meta) showCollections() []UniqueID {
@ -107,6 +184,10 @@ func (m *meta) addCollection(collectionID UniqueID, schema *schemapb.CollectionS
Schema: schema,
}
m.collectionInfos[collectionID] = newCollection
err := m.saveCollectionInfo(collectionID, newCollection)
if err != nil {
log.Error("save collectionInfo error", zap.Any("error", err.Error()), zap.Int64("collectionID", collectionID))
}
return nil
}
@ -126,6 +207,10 @@ func (m *meta) addPartition(collectionID UniqueID, partitionID UniqueID) error {
col.PartitionIDs = append(col.PartitionIDs, partitionID)
m.partitionStates[partitionID] = querypb.PartitionState_NotPresent
log.Debug("add a partition to meta", zap.Int64s("partitionIDs", col.PartitionIDs))
err := m.saveCollectionInfo(collectionID, col)
if err != nil {
log.Error("save collectionInfo error", zap.Any("error", err.Error()), zap.Int64("collectionID", collectionID))
}
return nil
}
return errors.New("addPartition: can't find collection when add partition")
@ -134,21 +219,50 @@ func (m *meta) addPartition(collectionID UniqueID, partitionID UniqueID) error {
func (m *meta) deleteSegmentInfoByID(segmentID UniqueID) {
m.Lock()
defer m.Unlock()
delete(m.segmentInfos, segmentID)
if _, ok := m.segmentInfos[segmentID]; ok {
err := m.removeSegmentInfo(segmentID)
if err != nil {
log.Error("remove segmentInfo error", zap.Any("error", err.Error()), zap.Int64("segmentID", segmentID))
}
delete(m.segmentInfos, segmentID)
}
}
func (m *meta) deleteSegmentInfoByNodeID(nodeID UniqueID) {
m.Lock()
defer m.Unlock()
for segmentID, info := range m.segmentInfos {
if info.NodeID == nodeID {
err := m.removeSegmentInfo(segmentID)
if err != nil {
log.Error("remove segmentInfo error", zap.Any("error", err.Error()), zap.Int64("segmentID", segmentID))
}
delete(m.segmentInfos, segmentID)
}
}
}
func (m *meta) setSegmentInfo(segmentID UniqueID, info *querypb.SegmentInfo) {
m.Lock()
defer m.Unlock()
err := m.saveSegmentInfo(segmentID, info)
if err != nil {
log.Error("save segmentInfo error", zap.Any("error", err.Error()), zap.Int64("segmentID", segmentID))
}
m.segmentInfos[segmentID] = info
}
func (m *meta) getSegmentInfos(segmentIDs []UniqueID) ([]*querypb.SegmentInfo, error) {
m.Lock()
defer m.Unlock()
segmentInfos := make([]*querypb.SegmentInfo, 0)
for _, segmentID := range segmentIDs {
if info, ok := m.segmentInfos[segmentID]; ok {
segmentInfos = append(segmentInfos, info)
segmentInfos = append(segmentInfos, proto.Clone(info).(*querypb.SegmentInfo))
continue
}
return nil, errors.New("segment not exist")
@ -156,17 +270,51 @@ func (m *meta) getSegmentInfos(segmentIDs []UniqueID) ([]*querypb.SegmentInfo, e
return segmentInfos, nil
}
func (m *meta) hasSegmentInfo(segmentID UniqueID) bool {
m.RLock()
defer m.RUnlock()
if _, ok := m.segmentInfos[segmentID]; ok {
return true
}
return false
}
func (m *meta) getSegmentInfoByID(segmentID UniqueID) (*querypb.SegmentInfo, error) {
m.Lock()
defer m.Unlock()
if info, ok := m.segmentInfos[segmentID]; ok {
return info, nil
return proto.Clone(info).(*querypb.SegmentInfo), nil
}
return nil, errors.New("getSegmentInfoByID: can't find segmentID in segmentInfos")
}
func (m *meta) getCollectionInfoByID(collectionID UniqueID) (*querypb.CollectionInfo, error) {
m.Lock()
defer m.Unlock()
if info, ok := m.collectionInfos[collectionID]; ok {
return proto.Clone(info).(*querypb.CollectionInfo), nil
}
return nil, errors.New("getCollectionInfoByID: can't find collectionID in collectionInfo")
}
func (m *meta) getQueryChannelInfoByID(collectionID UniqueID) (*querypb.QueryChannelInfo, error) {
m.Lock()
defer m.Unlock()
if info, ok := m.queryChannelInfos[collectionID]; ok {
return proto.Clone(info).(*querypb.QueryChannelInfo), nil
}
return nil, errors.New("getQueryChannelInfoByID: can't find collectionID in queryChannelInfo")
}
func (m *meta) updatePartitionState(partitionID UniqueID, state querypb.PartitionState) error {
m.Lock()
defer m.Unlock()
@ -201,10 +349,19 @@ func (m *meta) releaseCollection(collectionID UniqueID) {
}
for id, info := range m.segmentInfos {
if info.CollectionID == collectionID {
err := m.removeSegmentInfo(id)
if err != nil {
log.Error("remove segmentInfo error", zap.Any("error", err.Error()), zap.Int64("segmentID", id))
}
delete(m.segmentInfos, id)
}
}
delete(m.queryChannelInfos, collectionID)
err := m.removeCollectionInfo(collectionID)
if err != nil {
log.Error("remove collectionInfo error", zap.Any("error", err.Error()), zap.Int64("collectionID", collectionID))
}
}
func (m *meta) releasePartition(collectionID UniqueID, partitionID UniqueID) {
@ -300,6 +457,43 @@ func (m *meta) addDmChannel(collectionID UniqueID, nodeID int64, channels []stri
}
info.ChannelInfos = append(info.ChannelInfos, newChannelInfo)
}
err := m.saveCollectionInfo(collectionID, info)
if err != nil {
log.Error("save collectionInfo error", zap.Any("error", err.Error()), zap.Int64("collectionID", collectionID))
}
}
return errors.New("addDmChannels: can't find collection in collectionInfos")
}
func (m *meta) removeDmChannel(collectionID UniqueID, nodeID int64, channels []string) error {
m.Lock()
defer m.Unlock()
if info, ok := m.collectionInfos[collectionID]; ok {
for _, channelInfo := range info.ChannelInfos {
if channelInfo.NodeIDLoaded == nodeID {
newChannelIDs := make([]string, 0)
for _, channelID := range channelInfo.ChannelIDs {
findChannel := false
for _, channel := range channels {
if channelID == channel {
findChannel = true
}
}
if !findChannel {
newChannelIDs = append(newChannelIDs, channelID)
}
}
channelInfo.ChannelIDs = newChannelIDs
}
}
err := m.saveCollectionInfo(collectionID, info)
if err != nil {
log.Error("save collectionInfo error", zap.Any("error", err.Error()), zap.Int64("collectionID", collectionID))
}
}
return errors.New("addDmChannels: can't find collection in collectionInfos")
@ -330,3 +524,68 @@ func (m *meta) GetQueryChannel(collectionID UniqueID) (string, string) {
//TODO::return channel according collectionID
return allocatedQueryChannel, allocatedQueryResultChannel
}
func (m *meta) saveCollectionInfo(collectionID UniqueID, info *querypb.CollectionInfo) error {
infoBytes := proto.MarshalTextString(info)
key := fmt.Sprintf("%s/%d", collectionMetaPrefix, collectionID)
return m.client.Save(key, infoBytes)
}
func (m *meta) removeCollectionInfo(collectionID UniqueID) error {
key := fmt.Sprintf("%s/%d", collectionMetaPrefix, collectionID)
return m.client.Remove(key)
}
func (m *meta) saveSegmentInfo(segmentID UniqueID, info *querypb.SegmentInfo) error {
infoBytes := proto.MarshalTextString(info)
key := fmt.Sprintf("%s/%d", segmentMetaPrefix, segmentID)
return m.client.Save(key, infoBytes)
}
func (m *meta) removeSegmentInfo(segmentID UniqueID) error {
key := fmt.Sprintf("%s/%d", segmentMetaPrefix, segmentID)
return m.client.Remove(key)
}
func (m *meta) saveQueryChannelInfo(collectionID UniqueID, info *querypb.QueryChannelInfo) error {
infoBytes := proto.MarshalTextString(info)
key := fmt.Sprintf("%s/%d", queryChannelMetaPrefix, collectionID)
return m.client.Save(key, infoBytes)
}
func (m *meta) removeQueryChannelInfo(collectionID UniqueID) error {
key := fmt.Sprintf("%s/%d", queryChannelMetaPrefix, collectionID)
return m.client.Remove(key)
}
func (m *meta) setLoadCollection(collectionID UniqueID, state bool) error {
m.Lock()
defer m.Unlock()
if info, ok := m.collectionInfos[collectionID]; ok {
info.LoadCollection = state
err := m.saveCollectionInfo(collectionID, info)
if err != nil {
log.Error("save collectionInfo error", zap.Any("error", err.Error()), zap.Int64("collectionID", collectionID))
}
}
return errors.New("setLoadCollection: can't find collection in collectionInfos")
}
func (m *meta) printMeta() {
for id, info := range m.collectionInfos {
log.Debug("queryService meta: collectionInfo", zap.Int64("collectionID", id), zap.Any("info", info))
}
for id, info := range m.segmentInfos {
log.Debug("queryService meta: segmentInfo", zap.Int64("segmentID", id), zap.Any("info", info))
}
for id, info := range m.queryChannelInfos {
log.Debug("queryService meta: queryChannelInfo", zap.Int64("collectionID", id), zap.Any("info", info))
}
}

5
internal/queryservice/meta_test.go
View File

@ -19,8 +19,9 @@ import (
)
func TestReplica_Release(t *testing.T) {
meta := newMeta()
err := meta.addCollection(1, nil)
meta, err := newMeta(nil)
assert.Nil(t, err)
err = meta.addCollection(1, nil)
require.NoError(t, err)
collections := meta.showCollections()

14
internal/queryservice/param_table.go
View File

@ -50,6 +50,7 @@ type ParamTable struct {
// --- ETCD ---
EtcdEndpoints []string
MetaRootPath string
KvRootPath string
}
var Params ParamTable
@ -86,6 +87,7 @@ func (p *ParamTable) Init() {
// --- ETCD ---
p.initEtcdEndpoints()
p.initMetaRootPath()
p.initKvRootPath()
})
}
@ -194,3 +196,15 @@ func (p *ParamTable) initMetaRootPath() {
}
p.MetaRootPath = path.Join(rootPath, subPath)
}
func (p *ParamTable) initKvRootPath() {
rootPath, err := p.Load("etcd.rootPath")
if err != nil {
panic(err)
}
subPath, err := p.Load("etcd.kvSubPath")
if err != nil {
panic(err)
}
p.KvRootPath = path.Join(rootPath, subPath)
}

133
internal/queryservice/querynode.go
View File

@ -17,53 +17,50 @@ import (
"sync"
"time"
"github.com/golang/protobuf/proto"
"go.uber.org/zap"
nodeclient "github.com/milvus-io/milvus/internal/distributed/querynode/client"
etcdkv "github.com/milvus-io/milvus/internal/kv/etcd"
"github.com/milvus-io/milvus/internal/log"
"github.com/milvus-io/milvus/internal/proto/querypb"
"github.com/milvus-io/milvus/internal/proto/schemapb"
"github.com/milvus-io/milvus/internal/types"
)
type queryNode struct {
id int64
address struct {
ip string
port int64
}
client types.QueryNode
id int64
address string
client types.QueryNode
kvClient *etcdkv.EtcdKV
//mu sync.Mutex // guards segments and channels2Col
//nodeMeta *meta
sync.RWMutex
collectionInfos map[UniqueID]*querypb.CollectionInfo
watchedQueryChannels map[UniqueID]*querypb.QueryChannelInfo
//segments map[UniqueID][]UniqueID
//channels2Col map[UniqueID][]string
onService bool
}
func newQueryNode(ip string, port int64, id UniqueID) (*queryNode, error) {
client, err := nodeclient.NewClient(fmt.Sprintf("%s:%d", ip, port), 3*time.Second)
func newQueryNode(address string, id UniqueID, kv *etcdkv.EtcdKV) (*queryNode, error) {
client, err := nodeclient.NewClient(address, 3*time.Second)
if err != nil {
return nil, err
}
if err := client.Init(); err != nil {
return nil, err
}
if err := client.Start(); err != nil {
return nil, err
}
collectionInfo := make(map[UniqueID]*querypb.CollectionInfo)
watchedChannels := make(map[UniqueID]*querypb.QueryChannelInfo)
return &queryNode{
id: id,
address: struct {
ip string
port int64
}{ip: ip, port: port},
id: id,
address: address,
client: client,
kvClient: kv,
collectionInfos: collectionInfo,
watchedQueryChannels: watchedChannels,
//nodeMeta: newMetaReplica(),
onService: true,
}, nil
}
@ -107,12 +104,26 @@ func (qn *queryNode) addCollection(collectionID UniqueID, schema *schemapb.Colle
Schema: schema,
}
qn.collectionInfos[collectionID] = newCollection
err := qn.saveCollectionInfo(collectionID, newCollection)
if err != nil {
log.Error("save collectionInfo error", zap.Any("error", err.Error()), zap.Int64("collectionID", collectionID))
}
return nil
}
return errors.New("addCollection: collection already exists")
}
func (qn *queryNode) getCollectionInfoByID(collectionID UniqueID) (*querypb.CollectionInfo, error) {
qn.Lock()
defer qn.Lock()
if _, ok := qn.collectionInfos[collectionID]; ok {
return proto.Clone(qn.collectionInfos[collectionID]).(*querypb.CollectionInfo), nil
}
return nil, errors.New("addPartition: can't find collection")
}
func (qn *queryNode) addPartition(collectionID UniqueID, partitionID UniqueID) error {
qn.Lock()
defer qn.Unlock()
@ -123,6 +134,10 @@ func (qn *queryNode) addPartition(collectionID UniqueID, partitionID UniqueID) e
}
}
col.PartitionIDs = append(col.PartitionIDs, partitionID)
err := qn.saveCollectionInfo(collectionID, col)
if err != nil {
log.Error("save collectionInfo error", zap.Any("error", err.Error()), zap.Int64("collectionID", collectionID))
}
return nil
}
return errors.New("addPartition: can't find collection when add partition")
@ -131,9 +146,14 @@ func (qn *queryNode) addPartition(collectionID UniqueID, partitionID UniqueID) e
func (qn *queryNode) releaseCollection(collectionID UniqueID) {
qn.Lock()
defer qn.Unlock()
delete(qn.collectionInfos, collectionID)
//TODO::should reopen
//collectionID = 0
if _, ok := qn.collectionInfos[collectionID]; ok {
err := qn.removeCollectionInfo(collectionID)
if err != nil {
log.Error("remove collectionInfo error", zap.Any("error", err.Error()), zap.Int64("collectionID", collectionID))
}
delete(qn.collectionInfos, collectionID)
}
delete(qn.watchedQueryChannels, collectionID)
}
@ -149,6 +169,10 @@ func (qn *queryNode) releasePartition(collectionID UniqueID, partitionID UniqueI
}
}
info.PartitionIDs = newPartitionIDs
err := qn.removeCollectionInfo(collectionID)
if err != nil {
log.Error("remove collectionInfo error", zap.Any("error", err.Error()), zap.Int64("collectionID", collectionID))
}
}
}
@ -206,12 +230,46 @@ func (qn *queryNode) addDmChannel(collectionID UniqueID, channels []string) erro
}
info.ChannelInfos = append(info.ChannelInfos, newChannelInfo)
}
err := qn.saveCollectionInfo(collectionID, info)
if err != nil {
log.Error("save collectionInfo error", zap.Any("error", err.Error()), zap.Int64("collectionID", collectionID))
}
}
return errors.New("addDmChannels: can't find collection in watchedQueryChannel")
}
//TODO::removeDmChannels
func (qn *queryNode) removeDmChannel(collectionID UniqueID, channels []string) error {
qn.Lock()
defer qn.Unlock()
if info, ok := qn.collectionInfos[collectionID]; ok {
for _, channelInfo := range info.ChannelInfos {
if channelInfo.NodeIDLoaded == qn.id {
newChannelIDs := make([]string, 0)
for _, channelID := range channelInfo.ChannelIDs {
findChannel := false
for _, channel := range channels {
if channelID == channel {
findChannel = true
}
}
if !findChannel {
newChannelIDs = append(newChannelIDs, channelID)
}
}
channelInfo.ChannelIDs = newChannelIDs
}
}
err := qn.saveCollectionInfo(collectionID, info)
if err != nil {
log.Error("save collectionInfo error", zap.Any("error", err.Error()), zap.Int64("collectionID", collectionID))
}
}
return errors.New("addDmChannels: can't find collection in watchedQueryChannel")
}
func (qn *queryNode) hasWatchedQueryChannel(collectionID UniqueID) bool {
qn.RLock()
@ -236,5 +294,32 @@ func (qn *queryNode) removeQueryChannel(collectionID UniqueID) error {
defer qn.Unlock()
delete(qn.watchedQueryChannels, collectionID)
return nil
return errors.New("removeQueryChannel: can't find collection in watchedQueryChannel")
}
func (qn *queryNode) saveCollectionInfo(collectionID UniqueID, info *querypb.CollectionInfo) error {
infoBytes := proto.MarshalTextString(info)
key := fmt.Sprintf("%s/%d/%d", queryNodeMetaPrefix, qn.id, collectionID)
return qn.kvClient.Save(key, infoBytes)
}
func (qn *queryNode) removeCollectionInfo(collectionID UniqueID) error {
key := fmt.Sprintf("%s/%d/%d", queryNodeMetaPrefix, qn.id, collectionID)
return qn.kvClient.Remove(key)
}
func (qn *queryNode) setNodeState(onService bool) {
qn.Lock()
defer qn.Unlock()
qn.onService = onService
}
func (qn *queryNode) isOnService() bool {
qn.Lock()
defer qn.Unlock()
return qn.onService
}

192
internal/queryservice/queryservice.go
View File

@ -14,17 +14,25 @@ package queryservice
import (
"context"
"math/rand"
"path/filepath"
"strconv"
"sync"
"sync/atomic"
"time"
"github.com/coreos/etcd/mvcc/mvccpb"
"github.com/golang/protobuf/proto"
"go.etcd.io/etcd/clientv3"
"go.uber.org/zap"
etcdkv "github.com/milvus-io/milvus/internal/kv/etcd"
"github.com/milvus-io/milvus/internal/log"
"github.com/milvus-io/milvus/internal/msgstream"
"github.com/milvus-io/milvus/internal/proto/commonpb"
"github.com/milvus-io/milvus/internal/proto/internalpb"
"github.com/milvus-io/milvus/internal/proto/querypb"
"github.com/milvus-io/milvus/internal/types"
"github.com/milvus-io/milvus/internal/util/retry"
"github.com/milvus-io/milvus/internal/util/sessionutil"
"github.com/milvus-io/milvus/internal/util/typeutil"
)
@ -39,7 +47,8 @@ type queryChannelInfo struct {
type QueryService struct {
loopCtx context.Context
loopCancel context.CancelFunc
kvBase *etcdkv.EtcdKV
loopWg sync.WaitGroup
kvClient *etcdkv.EtcdKV
queryServiceID uint64
meta *meta
@ -49,7 +58,8 @@ type QueryService struct {
dataServiceClient types.DataService
masterServiceClient types.MasterService
session *sessionutil.Session
session *sessionutil.Session
eventChan <-chan *sessionutil.SessionEvent
stateCode atomic.Value
isInit atomic.Value
@ -67,7 +77,33 @@ func (qs *QueryService) Register() error {
}
func (qs *QueryService) Init() error {
connectEtcdFn := func() error {
etcdClient, err := clientv3.New(clientv3.Config{Endpoints: Params.EtcdEndpoints})
if err != nil {
return err
}
etcdKV := etcdkv.NewEtcdKV(etcdClient, Params.MetaRootPath)
qs.kvClient = etcdKV
metaKV, err := newMeta(etcdKV)
if err != nil {
return err
}
qs.meta = metaKV
qs.cluster, err = newQueryNodeCluster(metaKV, etcdKV)
if err != nil {
return err
}
qs.scheduler, err = NewTaskScheduler(qs.loopCtx, metaKV, qs.cluster, etcdKV, qs.masterServiceClient, qs.dataServiceClient)
return err
}
log.Debug("queryService try to connect etcd")
err := retry.Retry(100000, time.Millisecond*200, connectEtcdFn)
if err != nil {
log.Debug("queryService try to connect etcd failed", zap.Error(err))
return err
}
log.Debug("queryService try to connect etcd success")
return nil
}
@ -75,6 +111,13 @@ func (qs *QueryService) Start() error {
qs.scheduler.Start()
log.Debug("start scheduler ...")
qs.UpdateStateCode(internalpb.StateCode_Healthy)
qs.loopWg.Add(1)
go qs.watchNodeLoop()
qs.loopWg.Add(1)
go qs.watchMetaLoop()
return nil
}
@ -83,6 +126,8 @@ func (qs *QueryService) Stop() error {
log.Debug("close scheduler ...")
qs.loopCancel()
qs.UpdateStateCode(internalpb.StateCode_Abnormal)
qs.loopWg.Wait()
return nil
}
@ -105,16 +150,11 @@ func NewQueryService(ctx context.Context, factory msgstream.Factory) (*QueryServ
})
ctx1, cancel := context.WithCancel(ctx)
meta := newMeta()
service := &QueryService{
loopCtx: ctx1,
loopCancel: cancel,
meta: meta,
msFactory: factory,
}
//TODO::set etcd kvbase
service.scheduler = NewTaskScheduler(ctx1, meta, service.kvBase)
service.cluster = newQueryNodeCluster(meta)
service.UpdateStateCode(internalpb.StateCode_Abnormal)
log.Debug("QueryService", zap.Any("queryChannels", queryChannels))
@ -128,3 +168,141 @@ func (qs *QueryService) SetMasterService(masterService types.MasterService) {
func (qs *QueryService) SetDataService(dataService types.DataService) {
qs.dataServiceClient = dataService
}
func (qs *QueryService) watchNodeLoop() {
ctx, cancel := context.WithCancel(qs.loopCtx)
defer cancel()
defer qs.loopWg.Done()
log.Debug("QueryService start watch node loop")
clusterStartSession, version, _ := qs.session.GetSessions(typeutil.QueryNodeRole)
sessionMap := make(map[int64]*sessionutil.Session)
for _, session := range clusterStartSession {
nodeID := session.ServerID
sessionMap[nodeID] = session
}
for nodeID, session := range sessionMap {
if _, ok := qs.cluster.nodes[nodeID]; !ok {
serverID := session.ServerID
err := qs.cluster.RegisterNode(session, serverID)
if err != nil {
log.Error("register queryNode error", zap.Any("error", err.Error()))
}
log.Debug("QueryService", zap.Any("Add QueryNode, session serverID", serverID))
}
}
for nodeID := range qs.cluster.nodes {
if _, ok := sessionMap[nodeID]; !ok {
qs.cluster.nodes[nodeID].setNodeState(false)
loadBalanceSegment := &querypb.LoadBalanceRequest{
Base: &commonpb.MsgBase{
MsgType: commonpb.MsgType_LoadBalanceSegments,
SourceID: qs.session.ServerID,
},
SourceNodeIDs: []int64{nodeID},
}
loadBalanceTask := &LoadBalanceTask{
BaseTask: BaseTask{
ctx: qs.loopCtx,
Condition: NewTaskCondition(qs.loopCtx),
triggerCondition: querypb.TriggerCondition_nodeDown,
},
LoadBalanceRequest: loadBalanceSegment,
master: qs.masterServiceClient,
dataService: qs.dataServiceClient,
cluster: qs.cluster,
meta: qs.meta,
}
qs.scheduler.Enqueue([]task{loadBalanceTask})
}
}
qs.eventChan = qs.session.WatchServices(typeutil.QueryNodeRole, version+1)
for {
select {
case <-ctx.Done():
return
case event := <-qs.eventChan:
switch event.EventType {
case sessionutil.SessionAddEvent:
serverID := event.Session.ServerID
err := qs.cluster.RegisterNode(event.Session, serverID)
if err != nil {
log.Error(err.Error())
}
log.Debug("QueryService", zap.Any("Add QueryNode, session serverID", serverID))
case sessionutil.SessionDelEvent:
serverID := event.Session.ServerID
log.Debug("QueryService", zap.Any("The QueryNode crashed with ID", serverID))
qs.cluster.nodes[serverID].setNodeState(false)
loadBalanceSegment := &querypb.LoadBalanceRequest{
Base: &commonpb.MsgBase{
MsgType: commonpb.MsgType_LoadBalanceSegments,
SourceID: qs.session.ServerID,
},
SourceNodeIDs: []int64{serverID},
BalanceReason: querypb.TriggerCondition_nodeDown,
}
loadBalanceTask := &LoadBalanceTask{
BaseTask: BaseTask{
ctx: qs.loopCtx,
Condition: NewTaskCondition(qs.loopCtx),
triggerCondition: querypb.TriggerCondition_nodeDown,
},
LoadBalanceRequest: loadBalanceSegment,
master: qs.masterServiceClient,
dataService: qs.dataServiceClient,
cluster: qs.cluster,
meta: qs.meta,
}
qs.scheduler.Enqueue([]task{loadBalanceTask})
err := loadBalanceTask.WaitToFinish()
if err != nil {
log.Error(err.Error())
}
log.Debug("load balance done after queryNode down", zap.Int64s("nodeIDs", loadBalanceTask.SourceNodeIDs))
//TODO::remove nodeInfo and clear etcd
}
}
}
}
func (qs *QueryService) watchMetaLoop() {
ctx, cancel := context.WithCancel(qs.loopCtx)
defer cancel()
defer qs.loopWg.Done()
log.Debug("QueryService start watch meta loop")
watchChan := qs.meta.client.WatchWithPrefix("queryNode-segmentMeta")
for {
select {
case <-ctx.Done():
return
case resp := <-watchChan:
log.Debug("segment meta updated.")
for _, event := range resp.Events {
segmentID, err := strconv.ParseInt(filepath.Base(string(event.Kv.Key)), 10, 64)
if err != nil {
log.Error("watch meta loop error when get segmentID", zap.Any("error", err.Error()))
}
segmentInfo := &querypb.SegmentInfo{}
err = proto.UnmarshalText(string(event.Kv.Value), segmentInfo)
if err != nil {
log.Error("watch meta loop error when unmarshal", zap.Any("error", err.Error()))
}
switch event.Type {
case mvccpb.PUT:
//TODO::
qs.meta.setSegmentInfo(segmentID, segmentInfo)
case mvccpb.DELETE:
//TODO::
}
}
}
}
}

936
internal/queryservice/task.go
View File

File diff suppressed because it is too large Load Diff

574
internal/queryservice/task_scheduler.go
View File

@ -14,14 +14,24 @@ package queryservice
import (
"container/list"
"context"
"errors"
"fmt"
"path/filepath"
"strconv"
"sync"
"github.com/golang/protobuf/proto"
"github.com/opentracing/opentracing-go"
"go.uber.org/zap"
"github.com/milvus-io/milvus/internal/allocator"
etcdkv "github.com/milvus-io/milvus/internal/kv/etcd"
"github.com/milvus-io/milvus/internal/log"
"github.com/milvus-io/milvus/internal/proto/commonpb"
"github.com/milvus-io/milvus/internal/proto/querypb"
"github.com/milvus-io/milvus/internal/types"
"github.com/milvus-io/milvus/internal/util/trace"
"github.com/milvus-io/milvus/internal/util/tsoutil"
oplog "github.com/opentracing/opentracing-go/log"
)
@ -31,7 +41,6 @@ type TaskQueue struct {
maxTask int64
taskChan chan int // to block scheduler
scheduler *TaskScheduler
sync.Mutex
}
@ -77,6 +86,15 @@ func (queue *TaskQueue) addTask(tasks []task) {
}
}
func (queue *TaskQueue) addTaskToFront(t task) {
queue.taskChan <- 1
if queue.tasks.Len() == 0 {
queue.tasks.PushBack(t)
} else {
queue.tasks.PushFront(t)
}
}
func (queue *TaskQueue) PopTask() task {
queue.Lock()
defer queue.Unlock()
@ -92,12 +110,11 @@ func (queue *TaskQueue) PopTask() task {
return ft.Value.(task)
}
func NewTaskQueue(scheduler *TaskScheduler) *TaskQueue {
func NewTaskQueue() *TaskQueue {
return &TaskQueue{
tasks: list.New(),
maxTask: 1024,
taskChan: make(chan int, 1024),
scheduler: scheduler,
tasks: list.New(),
maxTask: 1024,
taskChan: make(chan int, 1024),
}
}
@ -105,144 +122,557 @@ type TaskScheduler struct {
triggerTaskQueue *TaskQueue
activateTaskChan chan task
meta *meta
cluster *queryNodeCluster
taskIDAllocator func() (UniqueID, error)
kvBase *etcdkv.EtcdKV
client *etcdkv.EtcdKV
master types.MasterService
dataService types.DataService
wg sync.WaitGroup
ctx context.Context
cancel context.CancelFunc
}
func NewTaskScheduler(ctx context.Context, meta *meta, kv *etcdkv.EtcdKV) *TaskScheduler {
func NewTaskScheduler(ctx context.Context, meta *meta, cluster *queryNodeCluster, kv *etcdkv.EtcdKV, master types.MasterService, dataService types.DataService) (*TaskScheduler, error) {
ctx1, cancel := context.WithCancel(ctx)
taskChan := make(chan task, 1024)
s := &TaskScheduler{
ctx: ctx1,
cancel: cancel,
meta: meta,
cluster: cluster,
activateTaskChan: taskChan,
kvBase: kv,
client: kv,
master: master,
dataService: dataService,
}
s.triggerTaskQueue = NewTaskQueue()
idAllocator := allocator.NewGlobalIDAllocator("idTimestamp", tsoutil.NewTSOKVBase(Params.EtcdEndpoints, Params.KvRootPath, "queryService task id"))
if err := idAllocator.Initialize(); err != nil {
log.Debug("QueryService idAllocator initialize failed", zap.Error(err))
return nil, err
}
s.taskIDAllocator = func() (UniqueID, error) {
return idAllocator.AllocOne()
}
err := s.reloadFromKV()
if err != nil {
return nil, err
}
s.triggerTaskQueue = NewTaskQueue(s)
//TODO::add etcd
//idAllocator := allocator.NewGlobalIDAllocator("queryService taskID", s.kvBase)
//s.taskIDAllocator = func() (UniqueID, error) {
// return idAllocator.AllocOne()
//}
return s
return s, nil
}
func (scheduler *TaskScheduler) reloadFromKV() error {
triggerTaskIDKeys, triggerTaskValues, err := scheduler.client.LoadWithPrefix(triggerTaskPrefix)
if err != nil {
return err
}
activeTaskIDKeys, activeTaskValues, err := scheduler.client.LoadWithPrefix(activeTaskPrefix)
if err != nil {
return err
}
taskInfoKeys, taskInfoValues, err := scheduler.client.LoadWithPrefix(taskInfoPrefix)
if err != nil {
return err
}
triggerTasks := make(map[int64]task)
for index := range triggerTaskIDKeys {
taskID, err := strconv.ParseInt(filepath.Base(triggerTaskIDKeys[index]), 10, 64)
if err != nil {
return err
}
t, err := scheduler.unmarshalTask(triggerTaskValues[index])
if err != nil {
return err
}
triggerTasks[taskID] = t
}
activeTasks := make(map[int64]task)
for index := range activeTaskIDKeys {
taskID, err := strconv.ParseInt(filepath.Base(activeTaskIDKeys[index]), 10, 64)
if err != nil {
return err
}
t, err := scheduler.unmarshalTask(activeTaskValues[index])
if err != nil {
return err
}
activeTasks[taskID] = t
}
taskInfos := make(map[int64]taskState)
for index := range taskInfoKeys {
taskID, err := strconv.ParseInt(filepath.Base(taskInfoKeys[index]), 10, 64)
if err != nil {
return err
}
value, err := strconv.ParseInt(taskInfoValues[index], 10, 64)
if err != nil {
return err
}
state := taskState(value)
taskInfos[taskID] = state
}
var doneTriggerTask task = nil
for id, t := range triggerTasks {
if taskInfos[id] == taskDone {
doneTriggerTask = t
for _, childTask := range activeTasks {
t.AddChildTask(childTask)
}
continue
}
scheduler.triggerTaskQueue.addTask([]task{t})
}
if doneTriggerTask != nil {
scheduler.triggerTaskQueue.addTaskToFront(doneTriggerTask)
}
return nil
}
func (scheduler *TaskScheduler) unmarshalTask(t string) (task, error) {
header := commonpb.MsgHeader{}
err := proto.UnmarshalText(t, &header)
if err != nil {
return nil, fmt.Errorf("Failed to unmarshal message header, err %s ", err.Error())
}
var newTask task
switch header.Base.MsgType {
case commonpb.MsgType_LoadCollection:
loadReq := querypb.LoadCollectionRequest{}
err = proto.UnmarshalText(t, &loadReq)
if err != nil {
log.Error(err.Error())
}
loadCollectionTask := &LoadCollectionTask{
BaseTask: BaseTask{
ctx: scheduler.ctx,
Condition: NewTaskCondition(scheduler.ctx),
triggerCondition: querypb.TriggerCondition_grpcRequest,
},
LoadCollectionRequest: &loadReq,
masterService: scheduler.master,
dataService: scheduler.dataService,
cluster: scheduler.cluster,
meta: scheduler.meta,
}
newTask = loadCollectionTask
case commonpb.MsgType_LoadPartitions:
loadReq := querypb.LoadPartitionsRequest{}
err = proto.UnmarshalText(t, &loadReq)
if err != nil {
log.Error(err.Error())
}
loadPartitionTask := &LoadPartitionTask{
BaseTask: BaseTask{
ctx: scheduler.ctx,
Condition: NewTaskCondition(scheduler.ctx),
triggerCondition: querypb.TriggerCondition_grpcRequest,
},
LoadPartitionsRequest: &loadReq,
dataService: scheduler.dataService,
cluster: scheduler.cluster,
meta: scheduler.meta,
}
newTask = loadPartitionTask
case commonpb.MsgType_ReleaseCollection:
loadReq := querypb.ReleaseCollectionRequest{}
err = proto.UnmarshalText(t, &loadReq)
if err != nil {
log.Error(err.Error())
}
releaseCollectionTask := &ReleaseCollectionTask{
BaseTask: BaseTask{
ctx: scheduler.ctx,
Condition: NewTaskCondition(scheduler.ctx),
triggerCondition: querypb.TriggerCondition_grpcRequest,
},
ReleaseCollectionRequest: &loadReq,
cluster: scheduler.cluster,
}
newTask = releaseCollectionTask
case commonpb.MsgType_ReleasePartitions:
loadReq := querypb.ReleasePartitionsRequest{}
err = proto.UnmarshalText(t, &loadReq)
if err != nil {
log.Error(err.Error())
}
releasePartitionTask := &ReleasePartitionTask{
BaseTask: BaseTask{
ctx: scheduler.ctx,
Condition: NewTaskCondition(scheduler.ctx),
triggerCondition: querypb.TriggerCondition_grpcRequest,
},
ReleasePartitionsRequest: &loadReq,
cluster: scheduler.cluster,
}
newTask = releasePartitionTask
case commonpb.MsgType_LoadSegments:
loadReq := querypb.LoadSegmentsRequest{}
err = proto.UnmarshalText(t, &loadReq)
if err != nil {
log.Error(err.Error())
}
loadSegmentTask := &LoadSegmentTask{
BaseTask: BaseTask{
ctx: scheduler.ctx,
Condition: NewTaskCondition(scheduler.ctx),
triggerCondition: querypb.TriggerCondition_grpcRequest,
},
LoadSegmentsRequest: &loadReq,
cluster: scheduler.cluster,
meta: scheduler.meta,
}
newTask = loadSegmentTask
case commonpb.MsgType_ReleaseSegments:
loadReq := querypb.ReleaseSegmentsRequest{}
err = proto.UnmarshalText(t, &loadReq)
if err != nil {
log.Error(err.Error())
}
releaseSegmentTask := &ReleaseSegmentTask{
BaseTask: BaseTask{
ctx: scheduler.ctx,
Condition: NewTaskCondition(scheduler.ctx),
triggerCondition: querypb.TriggerCondition_grpcRequest,
},
ReleaseSegmentsRequest: &loadReq,
cluster: scheduler.cluster,
}
newTask = releaseSegmentTask
case commonpb.MsgType_WatchDmChannels:
loadReq := querypb.WatchDmChannelsRequest{}
err = proto.UnmarshalText(t, &loadReq)
if err != nil {
log.Error(err.Error())
}
watchDmChannelTask := &WatchDmChannelTask{
BaseTask: BaseTask{
ctx: scheduler.ctx,
Condition: NewTaskCondition(scheduler.ctx),
triggerCondition: querypb.TriggerCondition_grpcRequest,
},
WatchDmChannelsRequest: &loadReq,
cluster: scheduler.cluster,
meta: scheduler.meta,
}
newTask = watchDmChannelTask
case commonpb.MsgType_WatchQueryChannels:
loadReq := querypb.AddQueryChannelRequest{}
err = proto.UnmarshalText(t, &loadReq)
if err != nil {
log.Error(err.Error())
}
watchQueryChannelTask := &WatchQueryChannelTask{
BaseTask: BaseTask{
ctx: scheduler.ctx,
Condition: NewTaskCondition(scheduler.ctx),
triggerCondition: querypb.TriggerCondition_grpcRequest,
},
AddQueryChannelRequest: &loadReq,
cluster: scheduler.cluster,
}
newTask = watchQueryChannelTask
case commonpb.MsgType_LoadBalanceSegments:
loadReq := querypb.LoadBalanceRequest{}
err = proto.UnmarshalText(t, &loadReq)
if err != nil {
log.Error(err.Error())
}
loadBalanceTask := &LoadBalanceTask{
BaseTask: BaseTask{
ctx: scheduler.ctx,
Condition: NewTaskCondition(scheduler.ctx),
triggerCondition: loadReq.BalanceReason,
},
LoadBalanceRequest: &loadReq,
master: scheduler.master,
dataService: scheduler.dataService,
cluster: scheduler.cluster,
meta: scheduler.meta,
}
newTask = loadBalanceTask
default:
err = errors.New("inValid msg type when unMarshal task")
log.Error(err.Error())
return nil, err
}
return newTask, nil
}
func (scheduler *TaskScheduler) Enqueue(tasks []task) {
//TODO::open when add etcd
//for _, t := range tasks {
// id, err := scheduler.taskIDAllocator()
// if err != nil {
// log.Error(err.Error())
// }
// t.SetID(id)
//}
for _, t := range tasks {
id, err := scheduler.taskIDAllocator()
if err != nil {
log.Error(err.Error())
}
t.SetID(id)
kvs := make(map[string]string)
taskKey := fmt.Sprintf("%s/%d", triggerTaskPrefix, t.ID())
kvs[taskKey] = t.Marshal()
stateKey := fmt.Sprintf("%s/%d", taskInfoPrefix, t.ID())
kvs[stateKey] = string(taskUndo)
err = scheduler.client.MultiSave(kvs)
if err != nil {
log.Error("error when save trigger task to etcd", zap.Int64("taskID", t.ID()))
}
log.Debug("EnQueue a triggerTask and save to etcd", zap.Int64("taskID", t.ID()))
}
scheduler.triggerTaskQueue.addTask(tasks)
}
func (scheduler *TaskScheduler) processTask(t task) {
func (scheduler *TaskScheduler) processTask(t task) error {
span, ctx := trace.StartSpanFromContext(t.TraceCtx(),
opentracing.Tags{
"Type": t.Type(),
"ID": t.ID(),
})
defer span.Finish()
span.LogFields(oplog.Int64("scheduler process PreExecute", t.ID()))
err := t.PreExecute(ctx)
span.LogFields(oplog.Int64("processTask: scheduler process PreExecute", t.ID()))
key := fmt.Sprintf("%s/%d", taskInfoPrefix, t.ID())
err := scheduler.client.Save(key, string(taskDoing))
defer func() {
t.Notify(err)
}()
if err != nil {
log.Debug("preExecute err", zap.String("reason", err.Error()))
log.Debug("processTask: update task state err", zap.String("reason", err.Error()))
trace.LogError(span, err)
return
return err
}
span.LogFields(oplog.Int64("scheduler process Execute", t.ID()))
err = t.PreExecute(ctx)
if err != nil {
log.Debug("processTask: preExecute err", zap.String("reason", err.Error()))
trace.LogError(span, err)
return err
}
span.LogFields(oplog.Int64("processTask: scheduler process Execute", t.ID()))
err = t.Execute(ctx)
if err != nil {
log.Debug("execute err", zap.String("reason", err.Error()))
log.Debug("processTask: execute err", zap.String("reason", err.Error()))
trace.LogError(span, err)
return
return err
}
span.LogFields(oplog.Int64("scheduler process PostExecute", t.ID()))
span.LogFields(oplog.Int64("processTask: scheduler process PostExecute", t.ID()))
err = t.PostExecute(ctx)
if err != nil {
log.Debug("processTask: postExecute err", zap.String("reason", err.Error()))
trace.LogError(span, err)
return err
}
for _, childTask := range t.GetChildTask() {
if childTask == nil {
log.Error("processTask: child task equal nil")
continue
}
id, err := scheduler.taskIDAllocator()
if err != nil {
return err
}
childTask.SetID(id)
kvs := make(map[string]string)
taskKey := fmt.Sprintf("%s/%d", activeTaskPrefix, childTask.ID())
kvs[taskKey] = t.Marshal()
stateKey := fmt.Sprintf("%s/%d", taskInfoPrefix, childTask.ID())
kvs[stateKey] = string(taskUndo)
err = scheduler.client.MultiSave(kvs)
if err != nil {
return err
}
log.Debug("processTask: save active task to etcd", zap.Int64("parent taskID", t.ID()), zap.Int64("child taskID", childTask.ID()))
}
err = scheduler.client.Save(key, string(taskDone))
if err != nil {
log.Debug("processTask: update task state err", zap.String("reason", err.Error()))
trace.LogError(span, err)
return err
}
return err
}
func (scheduler *TaskScheduler) scheduleLoop() {
defer scheduler.wg.Done()
var w sync.WaitGroup
activeTaskWg := &sync.WaitGroup{}
var err error = nil
for {
select {
case <-scheduler.ctx.Done():
return
case <-scheduler.triggerTaskQueue.Chan():
t := scheduler.triggerTaskQueue.PopTask()
log.Debug("pop a triggerTask from triggerTaskQueue")
scheduler.processTask(t)
//TODO::add active task to etcd
w.Add(2)
go scheduler.addActivateTask(&w, t)
//TODO::handle active task return error, maybe node down...
go scheduler.processActivateTask(&w)
w.Wait()
//TODO:: delete trigger task from etcd
log.Debug("scheduleLoop: pop a triggerTask from triggerTaskQueue", zap.Int64("taskID", t.ID()))
if t.State() < taskDone {
err = scheduler.processTask(t)
if err != nil {
log.Error("scheduleLoop: process task error", zap.Any("error", err.Error()))
continue
}
}
log.Debug("scheduleLoop: num of child task", zap.Int("num child task", len(t.GetChildTask())))
for _, childTask := range t.GetChildTask() {
if childTask != nil {
log.Debug("scheduleLoop: add a activate task to activateChan", zap.Int64("taskID", childTask.ID()))
scheduler.activateTaskChan <- childTask
activeTaskWg.Add(1)
go scheduler.waitActivateTaskDone(activeTaskWg, childTask)
}
}
activeTaskWg.Wait()
if t.Type() == commonpb.MsgType_ReleaseCollection || t.Type() == commonpb.MsgType_ReleasePartitions {
t.Notify(err)
}
keys := make([]string, 0)
taskKey := fmt.Sprintf("%s/%d", triggerTaskPrefix, t.ID())
stateKey := fmt.Sprintf("%s/%d", taskInfoPrefix, t.ID())
keys = append(keys, taskKey)
keys = append(keys, stateKey)
err = scheduler.client.MultiRemove(keys)
if err != nil {
log.Error("scheduleLoop: error when remove trigger task to etcd", zap.Int64("taskID", t.ID()))
}
log.Debug("scheduleLoop: trigger task done and delete from etcd", zap.Int64("taskID", t.ID()))
}
}
}
func (scheduler *TaskScheduler) addActivateTask(wg *sync.WaitGroup, t task) {
defer wg.Done()
var activeTaskWg sync.WaitGroup
log.Debug("num of child task", zap.Int("num child task", len(t.GetChildTask())))
for _, childTask := range t.GetChildTask() {
if childTask != nil {
log.Debug("add a activate task to activateChan")
scheduler.activateTaskChan <- childTask
activeTaskWg.Add(1)
go scheduler.waitActivateTaskDone(&activeTaskWg, childTask)
}
}
scheduler.activateTaskChan <- nil
activeTaskWg.Wait()
}
func (scheduler *TaskScheduler) waitActivateTaskDone(wg *sync.WaitGroup, t task) {
defer wg.Done()
err := t.WaitToFinish()
if err != nil {
//TODO:: redo task
log.Error("waitActivateTaskDone: activate task return err")
log.Debug("waitActivateTaskDone: activate task return err", zap.Any("error", err.Error()), zap.Int64("taskID", t.ID()))
redoFunc1 := func() {
if !t.IsValid() {
reScheduledTasks, err := t.Reschedule()
if err != nil {
log.Error(err.Error())
return
}
removes := make([]string, 0)
taskKey := fmt.Sprintf("%s/%d", activeTaskPrefix, t.ID())
removes = append(removes, taskKey)
stateKey := fmt.Sprintf("%s/%d", taskInfoPrefix, t.ID())
removes = append(removes, stateKey)
saves := make(map[string]string)
reSchedID := make([]int64, 0)
for _, rt := range reScheduledTasks {
if rt != nil {
id, err := scheduler.taskIDAllocator()
if err != nil {
log.Error(err.Error())
continue
}
rt.SetID(id)
taskKey := fmt.Sprintf("%s/%d", activeTaskPrefix, rt.ID())
saves[taskKey] = rt.Marshal()
stateKey := fmt.Sprintf("%s/%d", taskInfoPrefix, rt.ID())
saves[stateKey] = string(taskUndo)
reSchedID = append(reSchedID, rt.ID())
}
}
err = scheduler.client.MultiSaveAndRemove(saves, removes)
if err != nil {
log.Error("waitActivateTaskDone: error when save and remove task from etcd")
}
log.Debug("waitActivateTaskDone: delete failed active task and save reScheduled task to etcd", zap.Int64("failed taskID", t.ID()), zap.Int64s("reScheduled taskIDs", reSchedID))
for _, rt := range reScheduledTasks {
if rt != nil {
log.Debug("waitActivateTaskDone: add a reScheduled active task to activateChan", zap.Int64("taskID", rt.ID()))
scheduler.activateTaskChan <- rt
wg.Add(1)
go scheduler.waitActivateTaskDone(wg, rt)
}
}
//delete task from etcd
} else {
log.Debug("waitActivateTaskDone: retry the active task", zap.Int64("taskID", t.ID()))
scheduler.activateTaskChan <- t
wg.Add(1)
go scheduler.waitActivateTaskDone(wg, t)
}
}
redoFunc2 := func() {
if t.IsValid() {
scheduler.activateTaskChan <- t
wg.Add(1)
go scheduler.waitActivateTaskDone(wg, t)
}
}
switch t.Type() {
case commonpb.MsgType_LoadSegments:
redoFunc1()
case commonpb.MsgType_WatchDmChannels:
redoFunc1()
case commonpb.MsgType_WatchQueryChannels:
redoFunc2()
case commonpb.MsgType_ReleaseSegments:
redoFunc2()
case commonpb.MsgType_ReleaseCollection:
redoFunc2()
case commonpb.MsgType_ReleasePartitions:
redoFunc2()
default:
//TODO:: case commonpb.MsgType_RemoveDmChannels:
}
} else {
keys := make([]string, 0)
taskKey := fmt.Sprintf("%s/%d", activeTaskPrefix, t.ID())
stateKey := fmt.Sprintf("%s/%d", taskInfoPrefix, t.ID())
keys = append(keys, taskKey)
keys = append(keys, stateKey)
err = scheduler.client.MultiRemove(keys)
if err != nil {
log.Error("waitActivateTaskDone: error when remove task from etcd", zap.Int64("taskID", t.ID()))
}
log.Debug("waitActivateTaskDone: delete activate task from etcd", zap.Int64("taskID", t.ID()))
}
log.Debug("one activate task done")
log.Debug("waitActivateTaskDone: one activate task done", zap.Int64("taskID", t.ID()))
}
func (scheduler *TaskScheduler) processActivateTask(wg *sync.WaitGroup) {
defer wg.Done()
func (scheduler *TaskScheduler) processActivateTaskLoop() {
defer scheduler.wg.Done()
for {
select {
case <-scheduler.ctx.Done():
return
case t := <-scheduler.activateTaskChan:
if t == nil {
return
log.Error("processActivateTaskLoop: pop a nil active task", zap.Int64("taskID", t.ID()))
continue
}
log.Debug("pop a activate task from activateChan")
scheduler.processTask(t)
//TODO:: delete active task from etcd
stateKey := fmt.Sprintf("%s/%d", taskInfoPrefix, t.ID())
err := scheduler.client.Save(stateKey, string(taskDoing))
if err != nil {
t.Notify(err)
continue
}
log.Debug("processActivateTaskLoop: pop a active task from activateChan", zap.Int64("taskID", t.ID()))
go func() {
err := scheduler.processTask(t)
t.Notify(err)
}()
}
}
}
func (scheduler *TaskScheduler) Start() error {
scheduler.wg.Add(1)
scheduler.wg.Add(2)
go scheduler.scheduleLoop()
go scheduler.processActivateTaskLoop()
return nil
}

1
tests/python_test/collection/test_load_collection.py
View File

@ -499,6 +499,7 @@ class TestLoadPartition:
connect.create_index(binary_collection, default_binary_vec_field_name, get_binary_index)
connect.load_partitions(binary_collection, [default_tag])
@pytest.mark.skip("xige-16-search-without-insert")
@pytest.mark.tags(CaseLabel.tags_smoke)
def test_load_empty_partition(self, connect, collection):
'''

1
tests/python_test/entity/test_insert.py
View File

@ -885,6 +885,7 @@ class TestInsertMultiCollections:
stats = connect.get_collection_stats(collection_name)
assert stats[row_count] == 1
@pytest.mark.skip("xige-16-search-without-insert")
@pytest.mark.timeout(ADD_TIMEOUT)
@pytest.mark.tags(CaseLabel.tags_smoke)
def test_insert_entity_search_entity_another(self, connect, collection):

1
tests/python_test/entity/test_search.py
View File

@ -1154,6 +1154,7 @@ class TestSearchDSL(object):
assert len(res) == nq
assert len(res[0]) == 0
@pytest.mark.skip("xige-16-search-without-insert")
@pytest.mark.tags(CaseLabel.tags_smoke)
def test_query_complex_dsl(self, connect, collection):
'''