diff --git a/configs/milvus.yaml b/configs/milvus.yaml index 0181c4c80b..d7df286aa8 100644 --- a/configs/milvus.yaml +++ b/configs/milvus.yaml @@ -72,6 +72,7 @@ proxy: queryCoord: address: localhost port: 19531 + autoHandoff: true grpc: serverMaxRecvSize: 2147483647 # math.MaxInt32 diff --git a/internal/datacoord/meta.go b/internal/datacoord/meta.go index 47d1cc6977..1e8f199369 100644 --- a/internal/datacoord/meta.go +++ b/internal/datacoord/meta.go @@ -17,19 +17,20 @@ import ( "sync" "time" - "github.com/milvus-io/milvus/internal/log" "go.uber.org/zap" "github.com/golang/protobuf/proto" "github.com/milvus-io/milvus/internal/kv" - + "github.com/milvus-io/milvus/internal/log" "github.com/milvus-io/milvus/internal/proto/commonpb" "github.com/milvus-io/milvus/internal/proto/datapb" + "github.com/milvus-io/milvus/internal/proto/querypb" ) const ( - metaPrefix = "datacoord-meta" - segmentPrefix = metaPrefix + "/s" + metaPrefix = "datacoord-meta" + segmentPrefix = metaPrefix + "/s" + handoffSegmentPrefix = "querycoord-handoff" ) type meta struct { @@ -452,8 +453,27 @@ func (m *meta) saveSegmentInfo(segment *SegmentInfo) error { log.Error("DataCoord saveSegmentInfo marshal failed", zap.Int64("segmentID", segment.GetID()), zap.Error(err)) return fmt.Errorf("DataCoord saveSegmentInfo segmentID:%d, marshal failed:%w", segment.GetID(), err) } - key := buildSegmentPath(segment.GetCollectionID(), segment.GetPartitionID(), segment.GetID()) - return m.client.Save(key, string(segBytes)) + kvs := make(map[string]string) + dataKey := buildSegmentPath(segment.GetCollectionID(), segment.GetPartitionID(), segment.GetID()) + kvs[dataKey] = string(segBytes) + if segment.State == commonpb.SegmentState_Flushed { + handoffSegmentInfo := &querypb.SegmentInfo{ + SegmentID: segment.ID, + CollectionID: segment.CollectionID, + PartitionID: segment.PartitionID, + ChannelID: segment.InsertChannel, + SegmentState: querypb.SegmentState_sealed, + } + handoffSegBytes, err := proto.Marshal(handoffSegmentInfo) + if err != nil { + log.Error("DataCoord saveSegmentInfo marshal handoffSegInfo failed", zap.Int64("segmentID", segment.GetID()), zap.Error(err)) + return fmt.Errorf("DataCoord saveSegmentInfo segmentID:%d, marshal handoffSegInfo failed:%w", segment.GetID(), err) + } + queryKey := buildQuerySegmentPath(segment.GetCollectionID(), segment.GetPartitionID(), segment.GetID()) + kvs[queryKey] = string(handoffSegBytes) + } + + return m.client.MultiSave(kvs) } // removeSegmentInfo utility function removing segment info from kv store @@ -473,6 +493,11 @@ func buildSegmentPath(collectionID UniqueID, partitionID UniqueID, segmentID Uni return fmt.Sprintf("%s/%d/%d/%d", segmentPrefix, collectionID, partitionID, segmentID) } +// buildQuerySegmentPath common logic mapping segment info to corresponding key of queryCoord in kv store +func buildQuerySegmentPath(collectionID UniqueID, partitionID UniqueID, segmentID UniqueID) string { + return fmt.Sprintf("%s/%d/%d/%d", handoffSegmentPrefix, collectionID, partitionID, segmentID) +} + // buildSegment utility function for compose datapb.SegmentInfo struct with provided info func buildSegment(collectionID UniqueID, partitionID UniqueID, segmentID UniqueID, channelName string) *SegmentInfo { info := &datapb.SegmentInfo{ diff --git a/internal/datacoord/meta_test.go b/internal/datacoord/meta_test.go index c757aa3c61..0134783b07 100644 --- a/internal/datacoord/meta_test.go +++ b/internal/datacoord/meta_test.go @@ -12,6 +12,8 @@ package datacoord import ( "context" + "path/filepath" + "strconv" "testing" "github.com/golang/protobuf/proto" @@ -284,3 +286,26 @@ func TestUpdateFlushSegmentsInfo(t *testing.T) { assert.Nil(t, segmentInfo.StartPosition) }) } + +func TestSaveHandoffMeta(t *testing.T) { + meta, err := newMeta(memkv.NewMemoryKV()) + assert.Nil(t, err) + + info := &datapb.SegmentInfo{ + ID: 100, + State: commonpb.SegmentState_Flushed, + } + segmentInfo := &SegmentInfo{ + SegmentInfo: info, + } + + err = meta.saveSegmentInfo(segmentInfo) + assert.Nil(t, err) + + keys, _, err := meta.client.LoadWithPrefix(handoffSegmentPrefix) + assert.Nil(t, err) + assert.Equal(t, 1, len(keys)) + segmentID, err := strconv.ParseInt(filepath.Base(keys[0]), 10, 64) + assert.Nil(t, err) + assert.Equal(t, 100, int(segmentID)) +} diff --git a/internal/proto/query_coord.proto b/internal/proto/query_coord.proto index c941b5ca31..e0fcbd4e36 100644 --- a/internal/proto/query_coord.proto +++ b/internal/proto/query_coord.proto @@ -292,11 +292,6 @@ message CollectionInfo { int64 inMemory_percentage = 8; } -message HandoffSegments { - common.MsgBase base = 1; - repeated SegmentLoadInfo infos = 2; -} - message LoadBalanceSegmentInfo { int64 segmentID = 1; int64 partitionID = 2; @@ -308,6 +303,11 @@ message LoadBalanceSegmentInfo { bool valid_info = 8; } +message HandoffSegmentsRequest { + common.MsgBase base = 1; + repeated SegmentInfo segmentInfos = 2; +} + message LoadBalanceRequest { common.MsgBase base = 1; repeated int64 source_nodeIDs = 2; diff --git a/internal/proto/querypb/query_coord.pb.go b/internal/proto/querypb/query_coord.pb.go index d66b3ced7c..23af6c451a 100644 --- a/internal/proto/querypb/query_coord.pb.go +++ b/internal/proto/querypb/query_coord.pb.go @@ -1876,53 +1876,6 @@ func (m *CollectionInfo) GetInMemoryPercentage() int64 { return 0 } -type HandoffSegments struct { - Base *commonpb.MsgBase `protobuf:"bytes,1,opt,name=base,proto3" json:"base,omitempty"` - Infos []*SegmentLoadInfo `protobuf:"bytes,2,rep,name=infos,proto3" json:"infos,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` -} - -func (m *HandoffSegments) Reset() { *m = HandoffSegments{} } -func (m *HandoffSegments) String() string { return proto.CompactTextString(m) } -func (*HandoffSegments) ProtoMessage() {} -func (*HandoffSegments) Descriptor() ([]byte, []int) { - return fileDescriptor_aab7cc9a69ed26e8, []int{25} -} - -func (m *HandoffSegments) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_HandoffSegments.Unmarshal(m, b) -} -func (m *HandoffSegments) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_HandoffSegments.Marshal(b, m, deterministic) -} -func (m *HandoffSegments) XXX_Merge(src proto.Message) { - xxx_messageInfo_HandoffSegments.Merge(m, src) -} -func (m *HandoffSegments) XXX_Size() int { - return xxx_messageInfo_HandoffSegments.Size(m) -} -func (m *HandoffSegments) XXX_DiscardUnknown() { - xxx_messageInfo_HandoffSegments.DiscardUnknown(m) -} - -var xxx_messageInfo_HandoffSegments proto.InternalMessageInfo - -func (m *HandoffSegments) GetBase() *commonpb.MsgBase { - if m != nil { - return m.Base - } - return nil -} - -func (m *HandoffSegments) GetInfos() []*SegmentLoadInfo { - if m != nil { - return m.Infos - } - return nil -} - type LoadBalanceSegmentInfo struct { SegmentID int64 `protobuf:"varint,1,opt,name=segmentID,proto3" json:"segmentID,omitempty"` PartitionID int64 `protobuf:"varint,2,opt,name=partitionID,proto3" json:"partitionID,omitempty"` @@ -1941,7 +1894,7 @@ func (m *LoadBalanceSegmentInfo) Reset() { *m = LoadBalanceSegmentInfo{} func (m *LoadBalanceSegmentInfo) String() string { return proto.CompactTextString(m) } func (*LoadBalanceSegmentInfo) ProtoMessage() {} func (*LoadBalanceSegmentInfo) Descriptor() ([]byte, []int) { - return fileDescriptor_aab7cc9a69ed26e8, []int{26} + return fileDescriptor_aab7cc9a69ed26e8, []int{25} } func (m *LoadBalanceSegmentInfo) XXX_Unmarshal(b []byte) error { @@ -2018,6 +1971,53 @@ func (m *LoadBalanceSegmentInfo) GetValidInfo() bool { return false } +type HandoffSegmentsRequest struct { + Base *commonpb.MsgBase `protobuf:"bytes,1,opt,name=base,proto3" json:"base,omitempty"` + SegmentInfos []*SegmentInfo `protobuf:"bytes,2,rep,name=segmentInfos,proto3" json:"segmentInfos,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *HandoffSegmentsRequest) Reset() { *m = HandoffSegmentsRequest{} } +func (m *HandoffSegmentsRequest) String() string { return proto.CompactTextString(m) } +func (*HandoffSegmentsRequest) ProtoMessage() {} +func (*HandoffSegmentsRequest) Descriptor() ([]byte, []int) { + return fileDescriptor_aab7cc9a69ed26e8, []int{26} +} + +func (m *HandoffSegmentsRequest) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_HandoffSegmentsRequest.Unmarshal(m, b) +} +func (m *HandoffSegmentsRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_HandoffSegmentsRequest.Marshal(b, m, deterministic) +} +func (m *HandoffSegmentsRequest) XXX_Merge(src proto.Message) { + xxx_messageInfo_HandoffSegmentsRequest.Merge(m, src) +} +func (m *HandoffSegmentsRequest) XXX_Size() int { + return xxx_messageInfo_HandoffSegmentsRequest.Size(m) +} +func (m *HandoffSegmentsRequest) XXX_DiscardUnknown() { + xxx_messageInfo_HandoffSegmentsRequest.DiscardUnknown(m) +} + +var xxx_messageInfo_HandoffSegmentsRequest proto.InternalMessageInfo + +func (m *HandoffSegmentsRequest) GetBase() *commonpb.MsgBase { + if m != nil { + return m.Base + } + return nil +} + +func (m *HandoffSegmentsRequest) GetSegmentInfos() []*SegmentInfo { + if m != nil { + return m.SegmentInfos + } + return nil +} + type LoadBalanceRequest struct { Base *commonpb.MsgBase `protobuf:"bytes,1,opt,name=base,proto3" json:"base,omitempty"` SourceNodeIDs []int64 `protobuf:"varint,2,rep,packed,name=source_nodeIDs,json=sourceNodeIDs,proto3" json:"source_nodeIDs,omitempty"` @@ -2175,8 +2175,8 @@ func init() { proto.RegisterType((*DmChannelInfo)(nil), "milvus.proto.query.DmChannelInfo") proto.RegisterType((*QueryChannelInfo)(nil), "milvus.proto.query.QueryChannelInfo") proto.RegisterType((*CollectionInfo)(nil), "milvus.proto.query.CollectionInfo") - proto.RegisterType((*HandoffSegments)(nil), "milvus.proto.query.HandoffSegments") proto.RegisterType((*LoadBalanceSegmentInfo)(nil), "milvus.proto.query.LoadBalanceSegmentInfo") + proto.RegisterType((*HandoffSegmentsRequest)(nil), "milvus.proto.query.HandoffSegmentsRequest") proto.RegisterType((*LoadBalanceRequest)(nil), "milvus.proto.query.LoadBalanceRequest") proto.RegisterType((*SealedSegmentsChangeInfo)(nil), "milvus.proto.query.SealedSegmentsChangeInfo") } @@ -2184,145 +2184,145 @@ func init() { func init() { proto.RegisterFile("query_coord.proto", fileDescriptor_aab7cc9a69ed26e8) } var fileDescriptor_aab7cc9a69ed26e8 = []byte{ - // 2200 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xec, 0x19, 0x4d, 0x6f, 0x1c, 0x49, - 0xd5, 0x3d, 0xdf, 0xf3, 0xe6, 0xab, 0x53, 0x89, 0xbd, 0x93, 0x21, 0xc9, 0x86, 0xce, 0x66, 0x93, - 0xf5, 0xb2, 0xce, 0xae, 0xb3, 0x20, 0x22, 0xb1, 0x87, 0x8d, 0x67, 0xe3, 0x9d, 0x25, 0x71, 0x4c, - 0xdb, 0xbb, 0x88, 0x28, 0x52, 0xd3, 0x33, 0x5d, 0x1e, 0xb7, 0xd2, 0xdd, 0x35, 0xe9, 0xea, 0x89, - 0xe3, 0x1c, 0x38, 0x71, 0xe0, 0xc2, 0x2f, 0x00, 0x21, 0x21, 0x81, 0x10, 0x07, 0xb8, 0xc1, 0x81, - 0xd3, 0x5e, 0xb8, 0xf3, 0x07, 0x40, 0x42, 0x70, 0xe7, 0xc6, 0x19, 0x75, 0x55, 0x75, 0x4f, 0x7f, - 0xd4, 0xd8, 0x63, 0x9b, 0x90, 0x68, 0xc5, 0xad, 0xfb, 0xd5, 0xab, 0xf7, 0x5e, 0xbd, 0xef, 0x7a, - 0x05, 0xe7, 0x9e, 0x4e, 0xb1, 0x7f, 0x68, 0x8c, 0x08, 0xf1, 0xad, 0xb5, 0x89, 0x4f, 0x02, 0x82, - 0x90, 0x6b, 0x3b, 0xcf, 0xa6, 0x94, 0xff, 0xad, 0xb1, 0xf5, 0x5e, 0x73, 0x44, 0x5c, 0x97, 0x78, - 0x1c, 0xd6, 0x6b, 0x26, 0x31, 0x7a, 0x6d, 0xdb, 0x0b, 0xb0, 0xef, 0x99, 0x4e, 0xb4, 0x4a, 0x47, - 0xfb, 0xd8, 0x35, 0xc5, 0x9f, 0x6a, 0x99, 0x81, 0x99, 0xa4, 0xaf, 0xfd, 0x58, 0x81, 0x95, 0x9d, - 0x7d, 0x72, 0xb0, 0x41, 0x1c, 0x07, 0x8f, 0x02, 0x9b, 0x78, 0x54, 0xc7, 0x4f, 0xa7, 0x98, 0x06, - 0xe8, 0x7d, 0x28, 0x0d, 0x4d, 0x8a, 0xbb, 0xca, 0x55, 0xe5, 0x66, 0x63, 0xfd, 0xd2, 0x5a, 0x4a, - 0x12, 0x21, 0xc2, 0x03, 0x3a, 0xbe, 0x6b, 0x52, 0xac, 0x33, 0x4c, 0x84, 0xa0, 0x64, 0x0d, 0x07, - 0xfd, 0x6e, 0xe1, 0xaa, 0x72, 0xb3, 0xa8, 0xb3, 0x6f, 0xf4, 0x16, 0xb4, 0x46, 0x31, 0xed, 0x41, - 0x9f, 0x76, 0x8b, 0x57, 0x8b, 0x37, 0x8b, 0x7a, 0x1a, 0xa8, 0xfd, 0x46, 0x81, 0x37, 0x72, 0x62, - 0xd0, 0x09, 0xf1, 0x28, 0x46, 0xb7, 0xa1, 0x42, 0x03, 0x33, 0x98, 0x52, 0x21, 0xc9, 0xd7, 0xa4, - 0x92, 0xec, 0x30, 0x14, 0x5d, 0xa0, 0xe6, 0xd9, 0x16, 0x24, 0x6c, 0xd1, 0x07, 0x70, 0xc1, 0xf6, - 0x1e, 0x60, 0x97, 0xf8, 0x87, 0xc6, 0x04, 0xfb, 0x23, 0xec, 0x05, 0xe6, 0x18, 0x47, 0x32, 0x9e, - 0x8f, 0xd6, 0xb6, 0x67, 0x4b, 0xda, 0xaf, 0x15, 0x58, 0x0e, 0x25, 0xdd, 0x36, 0xfd, 0xc0, 0x7e, - 0x09, 0xfa, 0xd2, 0xa0, 0x99, 0x94, 0xb1, 0x5b, 0x64, 0x6b, 0x29, 0x58, 0x88, 0x33, 0x89, 0xd8, - 0x87, 0x67, 0x2b, 0x31, 0x71, 0x53, 0x30, 0xed, 0x57, 0xc2, 0xb0, 0x49, 0x39, 0xcf, 0xa2, 0xd0, - 0x2c, 0xcf, 0x42, 0x9e, 0xe7, 0x69, 0xd4, 0xf9, 0xa5, 0x02, 0xcb, 0xf7, 0x89, 0x69, 0xcd, 0x0c, - 0xff, 0xbf, 0x57, 0xe7, 0x47, 0x50, 0xe1, 0x51, 0xd2, 0x2d, 0x31, 0x5e, 0xd7, 0xd3, 0xbc, 0x44, - 0x04, 0xcd, 0x24, 0xdc, 0x61, 0x00, 0x5d, 0x6c, 0xd2, 0x7e, 0xae, 0x40, 0x57, 0xc7, 0x0e, 0x36, - 0x29, 0x7e, 0x95, 0xa7, 0x58, 0x81, 0x8a, 0x47, 0x2c, 0x3c, 0xe8, 0xb3, 0x53, 0x14, 0x75, 0xf1, - 0xa7, 0xfd, 0x53, 0x68, 0xf8, 0x35, 0x77, 0xd8, 0x84, 0x15, 0xca, 0xa7, 0xb1, 0xc2, 0x97, 0x33, - 0x2b, 0xbc, 0xee, 0x27, 0x9d, 0x59, 0xaa, 0x9c, 0xb2, 0xd4, 0x0f, 0xe0, 0xe2, 0x86, 0x8f, 0xcd, - 0x00, 0x7f, 0x2f, 0x4c, 0xf3, 0x1b, 0xfb, 0xa6, 0xe7, 0x61, 0x27, 0x3a, 0x42, 0x96, 0xb9, 0x22, - 0x61, 0xde, 0x85, 0xea, 0xc4, 0x27, 0xcf, 0x0f, 0x63, 0xb9, 0xa3, 0x5f, 0xed, 0x97, 0x0a, 0xf4, - 0x64, 0xb4, 0xcf, 0x92, 0x11, 0x6e, 0x40, 0xc7, 0xe7, 0xc2, 0x19, 0x23, 0x4e, 0x8f, 0x71, 0xad, - 0xeb, 0x6d, 0x01, 0x16, 0x5c, 0xd0, 0x75, 0x68, 0xfb, 0x98, 0x4e, 0x9d, 0x19, 0x5e, 0x91, 0xe1, - 0xb5, 0x38, 0x54, 0xa0, 0x69, 0xbf, 0x55, 0xe0, 0xe2, 0x26, 0x0e, 0x62, 0xeb, 0x85, 0xec, 0xf0, - 0x6b, 0x9a, 0x5d, 0x7f, 0xa1, 0x40, 0x27, 0x23, 0x28, 0xba, 0x0a, 0x8d, 0x04, 0x8e, 0x30, 0x50, - 0x12, 0x84, 0xbe, 0x0d, 0xe5, 0x50, 0x77, 0x98, 0x89, 0xd4, 0x5e, 0xd7, 0xd6, 0xf2, 0xc5, 0x7d, - 0x2d, 0x4d, 0x55, 0xe7, 0x1b, 0xd0, 0x2d, 0x38, 0x2f, 0xc9, 0xac, 0x42, 0x7c, 0x94, 0x4f, 0xac, - 0xda, 0xef, 0x14, 0xe8, 0xc9, 0x94, 0x79, 0x16, 0x83, 0x3f, 0x82, 0x95, 0xf8, 0x34, 0x86, 0x85, - 0xe9, 0xc8, 0xb7, 0x27, 0x2c, 0xcc, 0x58, 0x31, 0x68, 0xac, 0x5f, 0x3b, 0xfe, 0x3c, 0x54, 0x5f, - 0x8e, 0x49, 0xf4, 0x13, 0x14, 0xb4, 0x9f, 0x2a, 0xb0, 0xbc, 0x89, 0x83, 0x1d, 0x3c, 0x76, 0xb1, - 0x17, 0x0c, 0xbc, 0x3d, 0x72, 0x7a, 0xc3, 0x5f, 0x01, 0xa0, 0x82, 0x4e, 0x5c, 0xa8, 0x12, 0x90, - 0x45, 0x9c, 0x40, 0xfb, 0x6b, 0x01, 0x1a, 0x09, 0x61, 0xd0, 0x25, 0xa8, 0xc7, 0x14, 0x84, 0x69, - 0x67, 0x80, 0x1c, 0xc5, 0x82, 0xc4, 0xad, 0x32, 0xee, 0x51, 0xcc, 0xbb, 0xc7, 0x9c, 0x0c, 0x8e, - 0x2e, 0x42, 0xcd, 0xc5, 0xae, 0x41, 0xed, 0x17, 0x58, 0x64, 0x8c, 0xaa, 0x8b, 0xdd, 0x1d, 0xfb, - 0x05, 0x0e, 0x97, 0xbc, 0xa9, 0x6b, 0xf8, 0xe4, 0x80, 0x76, 0x2b, 0x7c, 0xc9, 0x9b, 0xba, 0x3a, - 0x39, 0xa0, 0xe8, 0x32, 0x80, 0xed, 0x59, 0xf8, 0xb9, 0xe1, 0x99, 0x2e, 0xee, 0x56, 0x59, 0xc4, - 0xd5, 0x19, 0x64, 0xcb, 0x74, 0x71, 0x98, 0x2b, 0xd8, 0xcf, 0xa0, 0xdf, 0xad, 0xf1, 0x8d, 0xe2, - 0x37, 0x3c, 0xaa, 0x88, 0xd3, 0x41, 0xbf, 0x5b, 0xe7, 0xfb, 0x62, 0x00, 0xfa, 0x04, 0x5a, 0xe2, - 0xdc, 0x06, 0xf7, 0x65, 0x60, 0xbe, 0x7c, 0x55, 0x66, 0x7b, 0xa1, 0x40, 0xee, 0xc9, 0x4d, 0x9a, - 0xf8, 0x63, 0x7d, 0x67, 0xd6, 0xde, 0x67, 0xf1, 0xcd, 0x6f, 0x42, 0xd9, 0xf6, 0xf6, 0x48, 0xe4, - 0x8a, 0x6f, 0x1e, 0x21, 0x0e, 0x63, 0xc6, 0xb1, 0xb5, 0x3f, 0x16, 0x61, 0xe5, 0x63, 0xcb, 0x92, - 0x25, 0xdc, 0x93, 0xfb, 0xdd, 0xcc, 0x7e, 0x85, 0x94, 0xfd, 0x16, 0x49, 0x3a, 0xef, 0xc2, 0xb9, - 0x4c, 0x32, 0x15, 0x6e, 0x50, 0xd7, 0xd5, 0x74, 0x3a, 0x1d, 0xf4, 0xd1, 0x3b, 0xa0, 0xa6, 0x13, - 0xaa, 0x28, 0x25, 0x75, 0xbd, 0x93, 0x4a, 0xa9, 0x83, 0x3e, 0xfa, 0x16, 0xbc, 0x31, 0x76, 0xc8, - 0xd0, 0x74, 0x0c, 0x8a, 0x4d, 0x07, 0x5b, 0xc6, 0xcc, 0x8b, 0x2b, 0x2c, 0x30, 0x96, 0xf9, 0xf2, - 0x0e, 0x5b, 0xdd, 0x89, 0x3d, 0x7a, 0x33, 0x34, 0x33, 0x7e, 0x62, 0x4c, 0x08, 0x65, 0xee, 0xc9, - 0x1c, 0xa8, 0x91, 0x4d, 0x59, 0xf1, 0x65, 0xe3, 0x01, 0x1d, 0x6f, 0x0b, 0xcc, 0xd0, 0xd0, 0xf8, - 0x49, 0xf4, 0x87, 0x3e, 0x87, 0x15, 0xa9, 0x00, 0xb4, 0x5b, 0x5b, 0xcc, 0x52, 0x17, 0x24, 0x02, - 0x52, 0xed, 0xef, 0x0a, 0x5c, 0xd4, 0xb1, 0x4b, 0x9e, 0xe1, 0xaf, 0xac, 0xed, 0xb4, 0x7f, 0x14, - 0x60, 0xe5, 0xfb, 0x66, 0x30, 0xda, 0xef, 0xbb, 0x02, 0x48, 0x5f, 0xcd, 0x01, 0x33, 0xa9, 0xab, - 0x94, 0x4f, 0x5d, 0x71, 0xf8, 0x95, 0x65, 0x46, 0x0d, 0x6f, 0x9d, 0x6b, 0x5f, 0x44, 0xe7, 0x9d, - 0x85, 0x5f, 0xa2, 0xe7, 0xab, 0x9c, 0xa2, 0xe7, 0x43, 0x1b, 0xd0, 0xc2, 0xcf, 0x47, 0xce, 0xd4, - 0xc2, 0x06, 0xe7, 0x5e, 0x65, 0xdc, 0xaf, 0x48, 0xb8, 0x27, 0x3d, 0xaa, 0x29, 0x36, 0x0d, 0x58, - 0x0a, 0xf8, 0x49, 0x11, 0x3a, 0x62, 0x35, 0x6c, 0x93, 0x17, 0xc8, 0xf6, 0x19, 0x75, 0x14, 0xf2, - 0xea, 0x58, 0x44, 0xa9, 0x51, 0x7b, 0x52, 0x4a, 0xb4, 0x27, 0x97, 0x01, 0xf6, 0x9c, 0x29, 0xdd, - 0x37, 0x02, 0xdb, 0x8d, 0x72, 0x7d, 0x9d, 0x41, 0x76, 0x6d, 0x17, 0xa3, 0x8f, 0xa1, 0x39, 0xb4, - 0x3d, 0x87, 0x8c, 0x8d, 0x89, 0x19, 0xec, 0x53, 0x16, 0xc1, 0xf2, 0xe3, 0xde, 0xb3, 0xb1, 0x63, - 0xdd, 0x65, 0xb8, 0x7a, 0x83, 0xef, 0xd9, 0x0e, 0xb7, 0xa0, 0x2b, 0xd0, 0x08, 0x0b, 0x06, 0xd9, - 0xe3, 0x35, 0xa3, 0xca, 0x59, 0x78, 0x53, 0xf7, 0xe1, 0x1e, 0xab, 0x1a, 0xdf, 0x81, 0x7a, 0x98, - 0x51, 0xa9, 0x43, 0xc6, 0x51, 0x84, 0x1e, 0x47, 0x7f, 0xb6, 0x01, 0x7d, 0x04, 0x75, 0x0b, 0x3b, - 0x81, 0xc9, 0x76, 0xd7, 0xe7, 0xba, 0x42, 0x3f, 0xc4, 0xb9, 0x4f, 0xc6, 0xcc, 0x1a, 0xb3, 0x1d, - 0xda, 0xbf, 0x0b, 0x70, 0x3e, 0xb4, 0x41, 0x14, 0xe5, 0xa7, 0xf7, 0xf6, 0xcb, 0x00, 0x16, 0x0d, - 0x8c, 0x94, 0xc7, 0xd7, 0x2d, 0x1a, 0x6c, 0x71, 0xa7, 0xbf, 0x13, 0xb9, 0x6b, 0x71, 0x7e, 0xe3, - 0x92, 0xf1, 0x89, 0xbc, 0xcb, 0x9e, 0xe6, 0xb2, 0x88, 0xbe, 0x0b, 0x6d, 0x87, 0x98, 0x96, 0x31, - 0x22, 0x9e, 0xc5, 0x13, 0x6b, 0x99, 0xd5, 0xcf, 0xb7, 0x64, 0x22, 0xec, 0xfa, 0xf6, 0x78, 0x8c, - 0xfd, 0x8d, 0x08, 0x57, 0x6f, 0x39, 0xec, 0xaa, 0x2c, 0x7e, 0xd1, 0x35, 0x68, 0x51, 0x32, 0xf5, - 0x47, 0x38, 0x3a, 0x28, 0x6f, 0x01, 0x9a, 0x1c, 0xb8, 0x25, 0x0f, 0xf0, 0xaa, 0xa4, 0xdb, 0xf9, - 0x9b, 0x02, 0x2b, 0xe2, 0xf2, 0x74, 0x76, 0xdd, 0xcf, 0xcb, 0x34, 0x91, 0xc3, 0x17, 0x8f, 0xe8, - 0xc7, 0x4b, 0x0b, 0xf4, 0xe3, 0x65, 0xc9, 0x95, 0x2a, 0xdd, 0xf2, 0x55, 0xb2, 0x2d, 0x9f, 0xb6, - 0x0b, 0xad, 0x38, 0x89, 0xb2, 0x08, 0xbf, 0x06, 0x2d, 0x2e, 0x96, 0x11, 0xaa, 0x14, 0x5b, 0xd1, - 0x7d, 0x8a, 0x03, 0xef, 0x33, 0x58, 0x48, 0x35, 0x4e, 0xd2, 0xbc, 0xb3, 0xa8, 0xeb, 0x09, 0x88, - 0xf6, 0x87, 0x02, 0xa8, 0xc9, 0xf2, 0xc3, 0x28, 0x2f, 0x72, 0x51, 0xbb, 0x01, 0x1d, 0x31, 0xea, - 0x8b, 0x6b, 0x80, 0xb8, 0x3a, 0x3d, 0x4d, 0x92, 0xeb, 0xa3, 0x0f, 0x61, 0x85, 0x23, 0xe6, 0x6a, - 0x06, 0xbf, 0x42, 0x5d, 0x60, 0xab, 0x7a, 0xa6, 0xe8, 0xcf, 0xaf, 0xb9, 0xa5, 0x33, 0xd4, 0xdc, - 0x7c, 0x4f, 0x50, 0x3e, 0x5d, 0x4f, 0xa0, 0xfd, 0xa5, 0x08, 0xed, 0x59, 0x84, 0x2c, 0xac, 0xb5, - 0x45, 0x46, 0x50, 0x5b, 0xa0, 0xce, 0xee, 0x28, 0xac, 0x41, 0x3d, 0x32, 0xc8, 0xb3, 0xb7, 0x93, - 0xce, 0x24, 0x73, 0xa9, 0xbb, 0x07, 0x2d, 0xa1, 0x73, 0x51, 0x62, 0xb8, 0x06, 0xbf, 0x2e, 0x23, - 0x96, 0xf2, 0x30, 0xbd, 0x99, 0xa8, 0x77, 0x14, 0xdd, 0x81, 0x3a, 0x8b, 0xfb, 0xe0, 0x70, 0x82, - 0x45, 0xc8, 0x5f, 0x92, 0xd1, 0x08, 0x3d, 0x6f, 0xf7, 0x70, 0x82, 0xf5, 0x9a, 0x23, 0xbe, 0xce, - 0x5a, 0x24, 0x6f, 0xc3, 0xb2, 0xcf, 0x43, 0xdb, 0x32, 0x52, 0xea, 0xab, 0x32, 0xf5, 0x5d, 0x88, - 0x16, 0xb7, 0x93, 0x6a, 0x9c, 0x73, 0xdf, 0xac, 0xcd, 0xbd, 0x6f, 0xfe, 0x08, 0x3a, 0x9f, 0x9a, - 0x9e, 0x45, 0xf6, 0xf6, 0x62, 0x77, 0x39, 0x79, 0xe6, 0xb8, 0x93, 0x6e, 0xe2, 0x4f, 0x90, 0x96, - 0xb5, 0x9f, 0x15, 0x60, 0x25, 0x84, 0xdd, 0x35, 0x1d, 0xd3, 0x1b, 0xe1, 0xc5, 0xaf, 0x6e, 0xff, - 0x9d, 0x62, 0x9e, 0xcb, 0xc4, 0x25, 0x49, 0x26, 0x4e, 0x17, 0xa5, 0x72, 0xb6, 0x28, 0xbd, 0x09, - 0x0d, 0x41, 0xc3, 0x22, 0x1e, 0x66, 0xc6, 0xae, 0xe9, 0xc0, 0x41, 0x7d, 0xe2, 0xb1, 0xcb, 0x5e, - 0xb8, 0x9f, 0xad, 0x56, 0xd9, 0x6a, 0xd5, 0xa2, 0x01, 0x5b, 0xba, 0x0c, 0xf0, 0xcc, 0x74, 0x6c, - 0x8b, 0x39, 0x29, 0x33, 0x53, 0x4d, 0xaf, 0x33, 0x48, 0xa8, 0x02, 0xed, 0x4f, 0x0a, 0xa0, 0x84, - 0x76, 0x4e, 0x9f, 0xdb, 0xaf, 0x43, 0x3b, 0x75, 0xce, 0x78, 0xae, 0x9e, 0x3c, 0x28, 0x0d, 0xab, - 0xdc, 0x90, 0xb3, 0x32, 0x7c, 0x6c, 0x52, 0xe2, 0x31, 0xa5, 0x2d, 0x5c, 0xe5, 0x86, 0x91, 0x98, - 0xe1, 0x56, 0xed, 0xf7, 0x05, 0xe8, 0xa6, 0x33, 0x51, 0x18, 0x66, 0x63, 0xd6, 0xbe, 0x9d, 0xe2, - 0x08, 0xd7, 0xa0, 0x45, 0x3c, 0xc7, 0xf6, 0x70, 0xba, 0x3b, 0x68, 0x72, 0xa0, 0xb0, 0xc5, 0xa7, - 0xd0, 0x11, 0x48, 0x71, 0xea, 0x2c, 0x2e, 0x96, 0x3a, 0xdb, 0x7c, 0x5f, 0x1c, 0x05, 0xd7, 0xa1, - 0x4d, 0xf6, 0xf6, 0x92, 0xfc, 0xb8, 0x6b, 0xb4, 0x04, 0x54, 0x30, 0xfc, 0x0c, 0xd4, 0x08, 0x2d, - 0xe6, 0x58, 0x5e, 0x8c, 0x63, 0x47, 0x6c, 0x8c, 0x58, 0xae, 0xbe, 0x80, 0x76, 0x3a, 0xaf, 0xa1, - 0x26, 0xd4, 0xb6, 0x48, 0xf0, 0xc9, 0x73, 0x9b, 0x06, 0xea, 0x12, 0x6a, 0x03, 0x6c, 0x91, 0x60, - 0xdb, 0xc7, 0x14, 0x7b, 0x81, 0xaa, 0x20, 0x80, 0xca, 0x43, 0xaf, 0x6f, 0xd3, 0x27, 0x6a, 0x01, - 0x9d, 0x17, 0x73, 0x2d, 0xd3, 0x19, 0x88, 0x20, 0x57, 0x8b, 0xe1, 0xf6, 0xf8, 0xaf, 0x84, 0x54, - 0x68, 0xc6, 0x28, 0x9b, 0xdb, 0x9f, 0xab, 0x65, 0x54, 0x87, 0x32, 0xff, 0xac, 0xac, 0x3e, 0x04, - 0x35, 0x6b, 0x4f, 0xd4, 0x80, 0xea, 0x3e, 0xcf, 0x0d, 0xea, 0x12, 0xea, 0x40, 0xc3, 0x99, 0x79, - 0xa2, 0xaa, 0x84, 0x80, 0xb1, 0x3f, 0x19, 0x09, 0x9f, 0x54, 0x0b, 0x21, 0xb7, 0x50, 0x53, 0x7d, - 0x72, 0xe0, 0xa9, 0xc5, 0xd5, 0xcf, 0xa0, 0x99, 0x1c, 0x23, 0xa0, 0x1a, 0x94, 0xb6, 0x88, 0x87, - 0xd5, 0xa5, 0x90, 0xec, 0xa6, 0x4f, 0x0e, 0x6c, 0x6f, 0xcc, 0xcf, 0x70, 0xcf, 0x27, 0x2f, 0xb0, - 0xa7, 0x16, 0xc2, 0x85, 0xb0, 0xee, 0x85, 0x0b, 0xc5, 0x70, 0x81, 0x17, 0x41, 0xb5, 0xb4, 0xfa, - 0x01, 0xd4, 0xa2, 0xfc, 0x8a, 0xce, 0x41, 0x2b, 0x35, 0x15, 0x57, 0x97, 0x10, 0xe2, 0xbd, 0xd9, - 0x2c, 0x93, 0xaa, 0xca, 0xfa, 0xbf, 0x00, 0x80, 0x97, 0x78, 0x42, 0x7c, 0x0b, 0x4d, 0x00, 0x6d, - 0xe2, 0x60, 0x83, 0xb8, 0x13, 0xe2, 0x45, 0x22, 0x51, 0xf4, 0xfe, 0x9c, 0x0a, 0x98, 0x47, 0x15, - 0xa7, 0xec, 0xbd, 0x3d, 0x67, 0x47, 0x06, 0x5d, 0x5b, 0x42, 0x2e, 0xe3, 0x18, 0xb6, 0xff, 0xbb, - 0xf6, 0xe8, 0x49, 0x34, 0x52, 0x3d, 0x82, 0x63, 0x06, 0x35, 0xe2, 0x98, 0x49, 0xa6, 0xe2, 0x67, - 0x27, 0xf0, 0x6d, 0x6f, 0x1c, 0x8d, 0x5e, 0xb4, 0x25, 0xf4, 0x14, 0x2e, 0x6c, 0x62, 0xc6, 0xdd, - 0xa6, 0x81, 0x3d, 0xa2, 0x11, 0xc3, 0xf5, 0xf9, 0x0c, 0x73, 0xc8, 0x27, 0x64, 0xe9, 0x40, 0x27, - 0xf3, 0xf4, 0x87, 0x56, 0xa5, 0x3e, 0x2f, 0x7d, 0xa6, 0xec, 0xbd, 0xbb, 0x10, 0x6e, 0xcc, 0xcd, - 0x86, 0x76, 0xfa, 0x59, 0x0c, 0xbd, 0x33, 0x8f, 0x40, 0xee, 0x1d, 0xa1, 0xb7, 0xba, 0x08, 0x6a, - 0xcc, 0xea, 0x11, 0xb4, 0xd3, 0x0f, 0x2f, 0x72, 0x56, 0xd2, 0xc7, 0x99, 0xde, 0x51, 0x53, 0x2f, - 0x6d, 0x09, 0xfd, 0x10, 0xce, 0xe5, 0x5e, 0x3b, 0xd0, 0x37, 0x64, 0xe4, 0xe7, 0x3d, 0x8a, 0x1c, - 0xc7, 0x41, 0x48, 0x3f, 0xd3, 0xe2, 0x7c, 0xe9, 0x73, 0xcf, 0x5e, 0x8b, 0x4b, 0x9f, 0x20, 0x7f, - 0x94, 0xf4, 0x27, 0xe6, 0x30, 0x05, 0x94, 0x7f, 0xef, 0x40, 0xef, 0xc9, 0x58, 0xcc, 0x7d, 0x73, - 0xe9, 0xad, 0x2d, 0x8a, 0x1e, 0x9b, 0x7c, 0xca, 0xa2, 0x35, 0xfb, 0x32, 0x20, 0x65, 0x3b, 0xf7, - 0xa9, 0x43, 0xce, 0x76, 0xfe, 0x30, 0x9f, 0x3b, 0x75, 0x7a, 0x98, 0x2a, 0xb7, 0x95, 0x74, 0xc0, - 0x2e, 0x77, 0x6a, 0xf9, 0x6c, 0x56, 0x5b, 0x42, 0x06, 0xc0, 0x26, 0x0e, 0x1e, 0xe0, 0xc0, 0xb7, - 0x47, 0x14, 0xbd, 0x2d, 0x0d, 0xf1, 0x19, 0x42, 0xc4, 0xe3, 0xc6, 0xb1, 0x78, 0x11, 0x83, 0xf5, - 0x3f, 0xd7, 0xa1, 0xce, 0xb4, 0x1b, 0x56, 0xc6, 0xff, 0x27, 0xdc, 0x97, 0x90, 0x70, 0x1f, 0x43, - 0x27, 0x33, 0xf3, 0x96, 0x27, 0x5c, 0xf9, 0x60, 0xfc, 0xb8, 0xc8, 0x1b, 0x02, 0xca, 0x0f, 0x66, - 0xe5, 0x21, 0x30, 0x77, 0x80, 0x7b, 0x1c, 0x8f, 0xc7, 0xd0, 0xc9, 0x0c, 0x46, 0xe5, 0x27, 0x90, - 0x4f, 0x4f, 0x8f, 0xa3, 0xfe, 0x05, 0x34, 0x93, 0x53, 0x28, 0x74, 0x63, 0x5e, 0xde, 0xcb, 0xcc, - 0x4a, 0x5e, 0x7d, 0xd6, 0x7b, 0xf9, 0x55, 0xe1, 0x31, 0x74, 0x32, 0x83, 0x22, 0xb9, 0xe6, 0xe5, - 0xd3, 0xa4, 0xe3, 0xa8, 0x7f, 0x85, 0xf2, 0xd8, 0xdd, 0x0f, 0x1f, 0xad, 0x8f, 0xed, 0x60, 0x7f, - 0x3a, 0x0c, 0x4f, 0x79, 0x8b, 0x63, 0xbe, 0x67, 0x13, 0xf1, 0x75, 0x2b, 0x0a, 0xe8, 0x5b, 0x8c, - 0xd2, 0x2d, 0x26, 0xed, 0x64, 0x38, 0xac, 0xb0, 0xdf, 0xdb, 0xff, 0x09, 0x00, 0x00, 0xff, 0xff, - 0xa8, 0x96, 0x54, 0x9d, 0x0b, 0x26, 0x00, 0x00, + // 2202 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xec, 0x19, 0x4d, 0x8f, 0x1c, 0x47, + 0x75, 0x7b, 0xbe, 0xe7, 0xcd, 0x57, 0xbb, 0xec, 0x9d, 0x8c, 0x87, 0xd8, 0x31, 0xed, 0x38, 0x76, + 0x36, 0x64, 0x9d, 0xac, 0x03, 0x22, 0x12, 0x39, 0xc4, 0x3b, 0xf1, 0x66, 0x82, 0xbd, 0x5e, 0x7a, + 0x37, 0x41, 0x58, 0x96, 0x9a, 0x9e, 0xe9, 0xda, 0xd9, 0x96, 0xbb, 0xbb, 0xc6, 0x5d, 0x3d, 0x5e, + 0xaf, 0xcf, 0x1c, 0xb8, 0x20, 0x7e, 0x00, 0x08, 0x09, 0x09, 0x84, 0x38, 0xc0, 0x0d, 0x0e, 0x9c, + 0x72, 0xe1, 0xce, 0x1f, 0x00, 0x09, 0xc1, 0x9d, 0x1b, 0x67, 0xd4, 0x55, 0xd5, 0x3d, 0xfd, 0x51, + 0xb3, 0x3b, 0xde, 0xc5, 0xd8, 0x8a, 0xb8, 0x75, 0xbf, 0x7a, 0xf5, 0xde, 0xab, 0xf7, 0x5d, 0xaf, + 0xe0, 0xdc, 0xe3, 0x19, 0xf6, 0x8f, 0x8c, 0x31, 0x21, 0xbe, 0xb5, 0x3e, 0xf5, 0x49, 0x40, 0x10, + 0x72, 0x6d, 0xe7, 0xc9, 0x8c, 0xf2, 0xbf, 0x75, 0xb6, 0xde, 0x6f, 0x8e, 0x89, 0xeb, 0x12, 0x8f, + 0xc3, 0xfa, 0xcd, 0x24, 0x46, 0xbf, 0x6d, 0x7b, 0x01, 0xf6, 0x3d, 0xd3, 0x89, 0x56, 0xe9, 0xf8, + 0x00, 0xbb, 0xa6, 0xf8, 0x53, 0x2d, 0x33, 0x30, 0x93, 0xf4, 0xb5, 0x1f, 0x29, 0xd0, 0xdd, 0x3d, + 0x20, 0x87, 0x9b, 0xc4, 0x71, 0xf0, 0x38, 0xb0, 0x89, 0x47, 0x75, 0xfc, 0x78, 0x86, 0x69, 0x80, + 0xde, 0x83, 0xd2, 0xc8, 0xa4, 0xb8, 0xa7, 0x5c, 0x51, 0x6e, 0x34, 0x36, 0x5e, 0x5f, 0x4f, 0x49, + 0x22, 0x44, 0xb8, 0x47, 0x27, 0xb7, 0x4d, 0x8a, 0x75, 0x86, 0x89, 0x10, 0x94, 0xac, 0xd1, 0x70, + 0xd0, 0x2b, 0x5c, 0x51, 0x6e, 0x14, 0x75, 0xf6, 0x8d, 0xde, 0x84, 0xd6, 0x38, 0xa6, 0x3d, 0x1c, + 0xd0, 0x5e, 0xf1, 0x4a, 0xf1, 0x46, 0x51, 0x4f, 0x03, 0xb5, 0xdf, 0x28, 0xf0, 0x5a, 0x4e, 0x0c, + 0x3a, 0x25, 0x1e, 0xc5, 0xe8, 0x16, 0x54, 0x68, 0x60, 0x06, 0x33, 0x2a, 0x24, 0xf9, 0x9a, 0x54, + 0x92, 0x5d, 0x86, 0xa2, 0x0b, 0xd4, 0x3c, 0xdb, 0x82, 0x84, 0x2d, 0x7a, 0x1f, 0x2e, 0xd8, 0xde, + 0x3d, 0xec, 0x12, 0xff, 0xc8, 0x98, 0x62, 0x7f, 0x8c, 0xbd, 0xc0, 0x9c, 0xe0, 0x48, 0xc6, 0xf3, + 0xd1, 0xda, 0xce, 0x7c, 0x49, 0xfb, 0xb5, 0x02, 0xab, 0xa1, 0xa4, 0x3b, 0xa6, 0x1f, 0xd8, 0x2f, + 0x40, 0x5f, 0x1a, 0x34, 0x93, 0x32, 0xf6, 0x8a, 0x6c, 0x2d, 0x05, 0x0b, 0x71, 0xa6, 0x11, 0xfb, + 0xf0, 0x6c, 0x25, 0x26, 0x6e, 0x0a, 0xa6, 0xfd, 0x4a, 0x18, 0x36, 0x29, 0xe7, 0x59, 0x14, 0x9a, + 0xe5, 0x59, 0xc8, 0xf3, 0x3c, 0x8d, 0x3a, 0xbf, 0x54, 0x60, 0xf5, 0x2e, 0x31, 0xad, 0xb9, 0xe1, + 0xff, 0xf7, 0xea, 0xfc, 0x08, 0x2a, 0x3c, 0x4a, 0x7a, 0x25, 0xc6, 0xeb, 0x5a, 0x9a, 0x97, 0x88, + 0xa0, 0xb9, 0x84, 0xbb, 0x0c, 0xa0, 0x8b, 0x4d, 0xda, 0xcf, 0x15, 0xe8, 0xe9, 0xd8, 0xc1, 0x26, + 0xc5, 0x2f, 0xf3, 0x14, 0x5d, 0xa8, 0x78, 0xc4, 0xc2, 0xc3, 0x01, 0x3b, 0x45, 0x51, 0x17, 0x7f, + 0xda, 0x3f, 0x85, 0x86, 0x5f, 0x71, 0x87, 0x4d, 0x58, 0xa1, 0x7c, 0x1a, 0x2b, 0x7c, 0x39, 0xb7, + 0xc2, 0xab, 0x7e, 0xd2, 0xb9, 0xa5, 0xca, 0x29, 0x4b, 0xfd, 0x00, 0x2e, 0x6e, 0xfa, 0xd8, 0x0c, + 0xf0, 0xf7, 0xc2, 0x34, 0xbf, 0x79, 0x60, 0x7a, 0x1e, 0x76, 0xa2, 0x23, 0x64, 0x99, 0x2b, 0x12, + 0xe6, 0x3d, 0xa8, 0x4e, 0x7d, 0xf2, 0xf4, 0x28, 0x96, 0x3b, 0xfa, 0xd5, 0x7e, 0xa9, 0x40, 0x5f, + 0x46, 0xfb, 0x2c, 0x19, 0xe1, 0x3a, 0x74, 0x7c, 0x2e, 0x9c, 0x31, 0xe6, 0xf4, 0x18, 0xd7, 0xba, + 0xde, 0x16, 0x60, 0xc1, 0x05, 0x5d, 0x83, 0xb6, 0x8f, 0xe9, 0xcc, 0x99, 0xe3, 0x15, 0x19, 0x5e, + 0x8b, 0x43, 0x05, 0x9a, 0xf6, 0x5b, 0x05, 0x2e, 0x6e, 0xe1, 0x20, 0xb6, 0x5e, 0xc8, 0x0e, 0xbf, + 0xa2, 0xd9, 0xf5, 0x17, 0x0a, 0x74, 0x32, 0x82, 0xa2, 0x2b, 0xd0, 0x48, 0xe0, 0x08, 0x03, 0x25, + 0x41, 0xe8, 0xdb, 0x50, 0x0e, 0x75, 0x87, 0x99, 0x48, 0xed, 0x0d, 0x6d, 0x3d, 0x5f, 0xdc, 0xd7, + 0xd3, 0x54, 0x75, 0xbe, 0x01, 0xdd, 0x84, 0xf3, 0x92, 0xcc, 0x2a, 0xc4, 0x47, 0xf9, 0xc4, 0xaa, + 0xfd, 0x4e, 0x81, 0xbe, 0x4c, 0x99, 0x67, 0x31, 0xf8, 0x03, 0xe8, 0xc6, 0xa7, 0x31, 0x2c, 0x4c, + 0xc7, 0xbe, 0x3d, 0x65, 0x61, 0xc6, 0x8a, 0x41, 0x63, 0xe3, 0xea, 0xc9, 0xe7, 0xa1, 0xfa, 0x6a, + 0x4c, 0x62, 0x90, 0xa0, 0xa0, 0xfd, 0x44, 0x81, 0xd5, 0x2d, 0x1c, 0xec, 0xe2, 0x89, 0x8b, 0xbd, + 0x60, 0xe8, 0xed, 0x93, 0xd3, 0x1b, 0xfe, 0x32, 0x00, 0x15, 0x74, 0xe2, 0x42, 0x95, 0x80, 0x2c, + 0xe3, 0x04, 0xda, 0x5f, 0x0b, 0xd0, 0x48, 0x08, 0x83, 0x5e, 0x87, 0x7a, 0x4c, 0x41, 0x98, 0x76, + 0x0e, 0xc8, 0x51, 0x2c, 0x48, 0xdc, 0x2a, 0xe3, 0x1e, 0xc5, 0xbc, 0x7b, 0x2c, 0xc8, 0xe0, 0xe8, + 0x22, 0xd4, 0x5c, 0xec, 0x1a, 0xd4, 0x7e, 0x86, 0x45, 0xc6, 0xa8, 0xba, 0xd8, 0xdd, 0xb5, 0x9f, + 0xe1, 0x70, 0xc9, 0x9b, 0xb9, 0x86, 0x4f, 0x0e, 0x69, 0xaf, 0xc2, 0x97, 0xbc, 0x99, 0xab, 0x93, + 0x43, 0x8a, 0x2e, 0x01, 0xd8, 0x9e, 0x85, 0x9f, 0x1a, 0x9e, 0xe9, 0xe2, 0x5e, 0x95, 0x45, 0x5c, + 0x9d, 0x41, 0xb6, 0x4d, 0x17, 0x87, 0xb9, 0x82, 0xfd, 0x0c, 0x07, 0xbd, 0x1a, 0xdf, 0x28, 0x7e, + 0xc3, 0xa3, 0x8a, 0x38, 0x1d, 0x0e, 0x7a, 0x75, 0xbe, 0x2f, 0x06, 0xa0, 0x4f, 0xa0, 0x25, 0xce, + 0x6d, 0x70, 0x5f, 0x06, 0xe6, 0xcb, 0x57, 0x64, 0xb6, 0x17, 0x0a, 0xe4, 0x9e, 0xdc, 0xa4, 0x89, + 0x3f, 0xd6, 0x77, 0x66, 0xed, 0x7d, 0x16, 0xdf, 0xfc, 0x26, 0x94, 0x6d, 0x6f, 0x9f, 0x44, 0xae, + 0xf8, 0xc6, 0x31, 0xe2, 0x30, 0x66, 0x1c, 0x5b, 0xfb, 0x63, 0x11, 0xba, 0x1f, 0x5b, 0x96, 0x2c, + 0xe1, 0x3e, 0xbf, 0xdf, 0xcd, 0xed, 0x57, 0x48, 0xd9, 0x6f, 0x99, 0xa4, 0xf3, 0x0e, 0x9c, 0xcb, + 0x24, 0x53, 0xe1, 0x06, 0x75, 0x5d, 0x4d, 0xa7, 0xd3, 0xe1, 0x00, 0xbd, 0x0d, 0x6a, 0x3a, 0xa1, + 0x8a, 0x52, 0x52, 0xd7, 0x3b, 0xa9, 0x94, 0x3a, 0x1c, 0xa0, 0x6f, 0xc1, 0x6b, 0x13, 0x87, 0x8c, + 0x4c, 0xc7, 0xa0, 0xd8, 0x74, 0xb0, 0x65, 0xcc, 0xbd, 0xb8, 0xc2, 0x02, 0x63, 0x95, 0x2f, 0xef, + 0xb2, 0xd5, 0xdd, 0xd8, 0xa3, 0xb7, 0x42, 0x33, 0xe3, 0x47, 0xc6, 0x94, 0x50, 0xe6, 0x9e, 0xcc, + 0x81, 0x1a, 0xd9, 0x94, 0x15, 0x5f, 0x36, 0xee, 0xd1, 0xc9, 0x8e, 0xc0, 0x0c, 0x0d, 0x8d, 0x1f, + 0x45, 0x7f, 0xe8, 0x73, 0xe8, 0x4a, 0x05, 0xa0, 0xbd, 0xda, 0x72, 0x96, 0xba, 0x20, 0x11, 0x90, + 0x6a, 0x7f, 0x57, 0xe0, 0xa2, 0x8e, 0x5d, 0xf2, 0x04, 0x7f, 0x65, 0x6d, 0xa7, 0xfd, 0xa3, 0x00, + 0xdd, 0xef, 0x9b, 0xc1, 0xf8, 0x60, 0xe0, 0x0a, 0x20, 0x7d, 0x39, 0x07, 0xcc, 0xa4, 0xae, 0x52, + 0x3e, 0x75, 0xc5, 0xe1, 0x57, 0x96, 0x19, 0x35, 0xbc, 0x75, 0xae, 0x7f, 0x11, 0x9d, 0x77, 0x1e, + 0x7e, 0x89, 0x9e, 0xaf, 0x72, 0x8a, 0x9e, 0x0f, 0x6d, 0x42, 0x0b, 0x3f, 0x1d, 0x3b, 0x33, 0x0b, + 0x1b, 0x9c, 0x7b, 0x95, 0x71, 0xbf, 0x2c, 0xe1, 0x9e, 0xf4, 0xa8, 0xa6, 0xd8, 0x34, 0x64, 0x29, + 0xe0, 0xc7, 0x45, 0xe8, 0x88, 0xd5, 0xb0, 0x4d, 0x5e, 0x22, 0xdb, 0x67, 0xd4, 0x51, 0xc8, 0xab, + 0x63, 0x19, 0xa5, 0x46, 0xed, 0x49, 0x29, 0xd1, 0x9e, 0x5c, 0x02, 0xd8, 0x77, 0x66, 0xf4, 0xc0, + 0x08, 0x6c, 0x37, 0xca, 0xf5, 0x75, 0x06, 0xd9, 0xb3, 0x5d, 0x8c, 0x3e, 0x86, 0xe6, 0xc8, 0xf6, + 0x1c, 0x32, 0x31, 0xa6, 0x66, 0x70, 0x40, 0x59, 0x04, 0xcb, 0x8f, 0x7b, 0xc7, 0xc6, 0x8e, 0x75, + 0x9b, 0xe1, 0xea, 0x0d, 0xbe, 0x67, 0x27, 0xdc, 0x82, 0x2e, 0x43, 0x23, 0x2c, 0x18, 0x64, 0x9f, + 0xd7, 0x8c, 0x2a, 0x67, 0xe1, 0xcd, 0xdc, 0xfb, 0xfb, 0xac, 0x6a, 0x7c, 0x07, 0xea, 0x61, 0x46, + 0xa5, 0x0e, 0x99, 0x44, 0x11, 0x7a, 0x12, 0xfd, 0xf9, 0x06, 0xf4, 0x11, 0xd4, 0x2d, 0xec, 0x04, + 0x26, 0xdb, 0x5d, 0x5f, 0xe8, 0x0a, 0x83, 0x10, 0xe7, 0x2e, 0x99, 0x30, 0x6b, 0xcc, 0x77, 0x68, + 0xff, 0x2e, 0xc0, 0xf9, 0xd0, 0x06, 0x51, 0x94, 0x9f, 0xde, 0xdb, 0x2f, 0x01, 0x58, 0x34, 0x30, + 0x52, 0x1e, 0x5f, 0xb7, 0x68, 0xb0, 0xcd, 0x9d, 0xfe, 0xc3, 0xc8, 0x5d, 0x8b, 0x8b, 0x1b, 0x97, + 0x8c, 0x4f, 0xe4, 0x5d, 0xf6, 0x34, 0x97, 0x45, 0xf4, 0x5d, 0x68, 0x3b, 0xc4, 0xb4, 0x8c, 0x31, + 0xf1, 0x2c, 0x9e, 0x58, 0xcb, 0xac, 0x7e, 0xbe, 0x29, 0x13, 0x61, 0xcf, 0xb7, 0x27, 0x13, 0xec, + 0x6f, 0x46, 0xb8, 0x7a, 0xcb, 0x61, 0x57, 0x65, 0xf1, 0x8b, 0xae, 0x42, 0x8b, 0x92, 0x99, 0x3f, + 0xc6, 0xd1, 0x41, 0x79, 0x0b, 0xd0, 0xe4, 0xc0, 0x6d, 0x79, 0x80, 0x57, 0x25, 0xdd, 0xce, 0xdf, + 0x14, 0xe8, 0x8a, 0xcb, 0xd3, 0xd9, 0x75, 0xbf, 0x28, 0xd3, 0x44, 0x0e, 0x5f, 0x3c, 0xa6, 0x1f, + 0x2f, 0x2d, 0xd1, 0x8f, 0x97, 0x25, 0x57, 0xaa, 0x74, 0xcb, 0x57, 0xc9, 0xb6, 0x7c, 0xda, 0x1e, + 0xb4, 0xe2, 0x24, 0xca, 0x22, 0xfc, 0x2a, 0xb4, 0xb8, 0x58, 0x46, 0xa8, 0x52, 0x6c, 0x45, 0xf7, + 0x29, 0x0e, 0xbc, 0xcb, 0x60, 0x21, 0xd5, 0x38, 0x49, 0xf3, 0xce, 0xa2, 0xae, 0x27, 0x20, 0xda, + 0x1f, 0x0a, 0xa0, 0x26, 0xcb, 0x0f, 0xa3, 0xbc, 0xcc, 0x45, 0xed, 0x3a, 0x74, 0xc4, 0xa8, 0x2f, + 0xae, 0x01, 0xe2, 0xea, 0xf4, 0x38, 0x49, 0x6e, 0x80, 0x3e, 0x80, 0x2e, 0x47, 0xcc, 0xd5, 0x0c, + 0x7e, 0x85, 0xba, 0xc0, 0x56, 0xf5, 0x4c, 0xd1, 0x5f, 0x5c, 0x73, 0x4b, 0x67, 0xa8, 0xb9, 0xf9, + 0x9e, 0xa0, 0x7c, 0xba, 0x9e, 0x40, 0xfb, 0x4b, 0x11, 0xda, 0xf3, 0x08, 0x59, 0x5a, 0x6b, 0xcb, + 0x8c, 0xa0, 0xb6, 0x41, 0x9d, 0xdf, 0x51, 0x58, 0x83, 0x7a, 0x6c, 0x90, 0x67, 0x6f, 0x27, 0x9d, + 0x69, 0xe6, 0x52, 0x77, 0x07, 0x5a, 0x42, 0xe7, 0xa2, 0xc4, 0x70, 0x0d, 0x7e, 0x5d, 0x46, 0x2c, + 0xe5, 0x61, 0x7a, 0x33, 0x51, 0xef, 0x28, 0xfa, 0x10, 0xea, 0x2c, 0xee, 0x83, 0xa3, 0x29, 0x16, + 0x21, 0xff, 0xba, 0x8c, 0x46, 0xe8, 0x79, 0x7b, 0x47, 0x53, 0xac, 0xd7, 0x1c, 0xf1, 0x75, 0xd6, + 0x22, 0x79, 0x0b, 0x56, 0x7d, 0x1e, 0xda, 0x96, 0x91, 0x52, 0x5f, 0x95, 0xa9, 0xef, 0x42, 0xb4, + 0xb8, 0x93, 0x54, 0xe3, 0x82, 0xfb, 0x66, 0x6d, 0xe1, 0x7d, 0xf3, 0x67, 0x05, 0xe8, 0x86, 0xb2, + 0xdf, 0x36, 0x1d, 0xd3, 0x1b, 0xe3, 0xe5, 0xaf, 0x4e, 0xff, 0x9d, 0x62, 0x9a, 0xcb, 0x84, 0x25, + 0x49, 0x26, 0x4c, 0x17, 0x85, 0x72, 0xb6, 0x28, 0xbc, 0x01, 0x0d, 0x41, 0xc3, 0x22, 0x1e, 0x66, + 0xca, 0xae, 0xe9, 0xc0, 0x41, 0x03, 0xe2, 0xb1, 0xcb, 0x56, 0xb8, 0x9f, 0xad, 0x56, 0xd9, 0x6a, + 0xd5, 0xa2, 0x01, 0x5b, 0xba, 0x04, 0xf0, 0xc4, 0x74, 0x6c, 0x8b, 0x39, 0x09, 0x53, 0x53, 0x4d, + 0xaf, 0x33, 0x48, 0xa8, 0x02, 0xed, 0xa7, 0x0a, 0x74, 0x3f, 0x35, 0x3d, 0x8b, 0xec, 0xef, 0x9f, + 0x3d, 0xbf, 0x6e, 0x42, 0x74, 0x95, 0x1a, 0x3e, 0xcf, 0x8d, 0x27, 0xb5, 0x49, 0xfb, 0x93, 0x02, + 0x28, 0x61, 0xaf, 0xd3, 0x4b, 0x73, 0x0d, 0xda, 0x29, 0xcd, 0xc7, 0x93, 0xf6, 0xa4, 0xea, 0x69, + 0x58, 0xf7, 0x46, 0x9c, 0x95, 0xe1, 0x63, 0x93, 0x12, 0x8f, 0x99, 0x71, 0xe9, 0xba, 0x37, 0x8a, + 0xc4, 0x0c, 0xb7, 0x6a, 0xbf, 0x2f, 0x40, 0x2f, 0x9d, 0x9b, 0xc2, 0xc0, 0x9b, 0xb0, 0x86, 0xee, + 0x14, 0x47, 0xb8, 0x0a, 0x2d, 0xe2, 0x39, 0xb6, 0x87, 0xd3, 0xfd, 0x42, 0x93, 0x03, 0x85, 0x77, + 0x7c, 0x0a, 0x1d, 0x81, 0x14, 0x27, 0xd3, 0xe2, 0x72, 0x8a, 0x6f, 0xf3, 0x7d, 0x71, 0x1a, 0xbd, + 0x06, 0x6d, 0xb2, 0xbf, 0x9f, 0xe4, 0xc7, 0x9d, 0xb5, 0x25, 0xa0, 0x82, 0xe1, 0x67, 0xa0, 0x46, + 0x68, 0x31, 0xc7, 0xf2, 0x72, 0x1c, 0x3b, 0x62, 0x63, 0xc4, 0x72, 0xed, 0x19, 0xb4, 0xd3, 0x99, + 0x0e, 0x35, 0xa1, 0xb6, 0x4d, 0x82, 0x4f, 0x9e, 0xda, 0x34, 0x50, 0x57, 0x50, 0x1b, 0x60, 0x9b, + 0x04, 0x3b, 0x3e, 0xa6, 0xd8, 0x0b, 0x54, 0x05, 0x01, 0x54, 0xee, 0x7b, 0x03, 0x9b, 0x3e, 0x52, + 0x0b, 0xe8, 0xbc, 0x98, 0x74, 0x99, 0xce, 0x50, 0x84, 0xbd, 0x5a, 0x0c, 0xb7, 0xc7, 0x7f, 0x25, + 0xa4, 0x42, 0x33, 0x46, 0xd9, 0xda, 0xf9, 0x5c, 0x2d, 0xa3, 0x3a, 0x94, 0xf9, 0x67, 0x65, 0xed, + 0x3e, 0xa8, 0x59, 0x7b, 0xa2, 0x06, 0x54, 0x0f, 0x78, 0x38, 0xa8, 0x2b, 0xa8, 0x03, 0x0d, 0x67, + 0xee, 0x89, 0xaa, 0x12, 0x02, 0x26, 0xfe, 0x74, 0x2c, 0x7c, 0x52, 0x2d, 0x84, 0xdc, 0x42, 0x4d, + 0x0d, 0xc8, 0xa1, 0xa7, 0x16, 0xd7, 0x3e, 0x83, 0x66, 0x72, 0xb0, 0x80, 0x6a, 0x50, 0xda, 0x26, + 0x1e, 0x56, 0x57, 0x42, 0xb2, 0x5b, 0x3e, 0x39, 0xb4, 0xbd, 0x09, 0x3f, 0xc3, 0x1d, 0x9f, 0x3c, + 0xc3, 0x9e, 0x5a, 0x08, 0x17, 0xc2, 0x4a, 0x18, 0x2e, 0x14, 0xc3, 0x05, 0x5e, 0x16, 0xd5, 0xd2, + 0xda, 0xfb, 0x50, 0x8b, 0x32, 0x2e, 0x3a, 0x07, 0xad, 0xd4, 0x9c, 0x5c, 0x5d, 0x41, 0x88, 0x77, + 0x6b, 0xf3, 0xdc, 0xaa, 0x2a, 0x1b, 0xff, 0x02, 0x00, 0x5e, 0xf4, 0x09, 0xf1, 0x2d, 0x34, 0x05, + 0xb4, 0x85, 0x83, 0x4d, 0xe2, 0x4e, 0x89, 0x17, 0x89, 0x44, 0xd1, 0x7b, 0x0b, 0x6a, 0x62, 0x1e, + 0x55, 0x9c, 0xb2, 0xff, 0xd6, 0x82, 0x1d, 0x19, 0x74, 0x6d, 0x05, 0xb9, 0x8c, 0x63, 0x78, 0x21, + 0xd8, 0xb3, 0xc7, 0x8f, 0xa2, 0x21, 0xeb, 0x31, 0x1c, 0x33, 0xa8, 0x11, 0xc7, 0x4c, 0x41, 0x14, + 0x3f, 0xbb, 0x81, 0x6f, 0x7b, 0x93, 0x68, 0x18, 0xa3, 0xad, 0xa0, 0xc7, 0x70, 0x61, 0x0b, 0x33, + 0xee, 0x36, 0x0d, 0xec, 0x31, 0x8d, 0x18, 0x6e, 0x2c, 0x66, 0x98, 0x43, 0x7e, 0x4e, 0x96, 0x0e, + 0x74, 0x32, 0x8f, 0x81, 0x68, 0x4d, 0xea, 0xf3, 0xd2, 0x87, 0xcb, 0xfe, 0x3b, 0x4b, 0xe1, 0xc6, + 0xdc, 0x6c, 0x68, 0xa7, 0x1f, 0xca, 0xd0, 0xdb, 0x8b, 0x08, 0xe4, 0x5e, 0x16, 0xfa, 0x6b, 0xcb, + 0xa0, 0xc6, 0xac, 0x1e, 0x40, 0x3b, 0xfd, 0x14, 0x23, 0x67, 0x25, 0x7d, 0xae, 0xe9, 0x1f, 0x37, + 0x07, 0xd3, 0x56, 0xd0, 0x0f, 0xe1, 0x5c, 0xee, 0xfd, 0x03, 0x7d, 0x43, 0x46, 0x7e, 0xd1, 0x33, + 0xc9, 0x49, 0x1c, 0x84, 0xf4, 0x73, 0x2d, 0x2e, 0x96, 0x3e, 0xf7, 0x10, 0xb6, 0xbc, 0xf4, 0x09, + 0xf2, 0xc7, 0x49, 0xff, 0xdc, 0x1c, 0x66, 0x80, 0xf2, 0x2f, 0x20, 0xe8, 0x5d, 0x19, 0x8b, 0x85, + 0xaf, 0x30, 0xfd, 0xf5, 0x65, 0xd1, 0x63, 0x93, 0xcf, 0x58, 0xb4, 0x66, 0xdf, 0x0a, 0xa4, 0x6c, + 0x17, 0x3e, 0x7e, 0xc8, 0xd9, 0x2e, 0x1e, 0xef, 0x73, 0xa7, 0x4e, 0x8f, 0x57, 0xe5, 0xb6, 0x92, + 0x8e, 0xdc, 0xe5, 0x4e, 0x2d, 0x9f, 0xd6, 0x6a, 0x2b, 0xc8, 0x00, 0xd8, 0xc2, 0xc1, 0x3d, 0x1c, + 0xf8, 0xf6, 0x98, 0xa2, 0xb7, 0xa4, 0x21, 0x3e, 0x47, 0x88, 0x78, 0x5c, 0x3f, 0x11, 0x2f, 0x62, + 0xb0, 0xf1, 0xe7, 0x3a, 0xd4, 0x99, 0x76, 0xc3, 0xca, 0xf8, 0xff, 0x84, 0xfb, 0x02, 0x12, 0xee, + 0x43, 0xe8, 0x64, 0xa6, 0xe0, 0xf2, 0x84, 0x2b, 0x1f, 0x95, 0x9f, 0x14, 0x79, 0x23, 0x40, 0xf9, + 0x51, 0xad, 0x3c, 0x04, 0x16, 0x8e, 0x74, 0x4f, 0xe2, 0xf1, 0x10, 0x3a, 0x99, 0x51, 0xa9, 0xfc, + 0x04, 0xf2, 0x79, 0xea, 0x49, 0xd4, 0xbf, 0x80, 0x66, 0x72, 0x2e, 0x85, 0xae, 0x2f, 0xca, 0x7b, + 0x99, 0xee, 0xfe, 0xe5, 0x67, 0xbd, 0x17, 0x5f, 0x15, 0x1e, 0x42, 0x27, 0x33, 0x3a, 0x92, 0x6b, + 0x5e, 0x3e, 0x5f, 0x3a, 0x89, 0xfa, 0x57, 0x28, 0x8f, 0xdd, 0xfe, 0xe0, 0xc1, 0xc6, 0xc4, 0x0e, + 0x0e, 0x66, 0xa3, 0xf0, 0x94, 0x37, 0x39, 0xe6, 0xbb, 0x36, 0x11, 0x5f, 0x37, 0xa3, 0x80, 0xbe, + 0xc9, 0x28, 0xdd, 0x64, 0xd2, 0x4e, 0x47, 0xa3, 0x0a, 0xfb, 0xbd, 0xf5, 0x9f, 0x00, 0x00, 0x00, + 0xff, 0xff, 0xe4, 0x62, 0x7a, 0xd7, 0x1d, 0x26, 0x00, 0x00, } // Reference imports to suppress errors if they are not otherwise used. diff --git a/internal/querycoord/mock_3rd_component_test.go b/internal/querycoord/mock_3rd_component_test.go index 214152804e..15e4b49a95 100644 --- a/internal/querycoord/mock_3rd_component_test.go +++ b/internal/querycoord/mock_3rd_component_test.go @@ -42,6 +42,7 @@ const ( defaultPartitionID = UniqueID(2021) defaultSegmentID = UniqueID(2021) defaultQueryNodeID = int64(100) + defaultChannelNum = 2 ) func genCollectionSchema(collectionID UniqueID, isBinary bool) *schemapb.CollectionSchema { @@ -318,7 +319,7 @@ func newDataCoordMock(ctx context.Context) (*dataCoordMock, error) { partitionID2Segment: partitionID2Segments, Segment2Binlog: segment2Binglog, baseSegmentID: defaultSegmentID, - channelNumPerCol: 2, + channelNumPerCol: defaultChannelNum, }, nil } diff --git a/internal/querycoord/param_table.go b/internal/querycoord/param_table.go index a96975bf4c..87b3faa623 100644 --- a/internal/querycoord/param_table.go +++ b/internal/querycoord/param_table.go @@ -64,6 +64,9 @@ type ParamTable struct { // --- Pulsar --- PulsarAddress string + + //---- Handoff --- + AutoHandoff bool } // Params are variables of the ParamTable type @@ -114,6 +117,9 @@ func (p *ParamTable) Init() { //--- Pulsar ---- p.initPulsarAddress() + + //---- Handoff --- + p.initAutoHandoff() } func (p *ParamTable) initQueryCoordAddress() { @@ -256,3 +262,14 @@ func (p *ParamTable) initPulsarAddress() { } p.PulsarAddress = addr } + +func (p *ParamTable) initAutoHandoff() { + handoff, err := p.Load("queryCoord.autoHandoff") + if err != nil { + panic(err) + } + p.AutoHandoff, err = strconv.ParseBool(handoff) + if err != nil { + panic(err) + } +} diff --git a/internal/querycoord/query_coord.go b/internal/querycoord/query_coord.go index f638773caf..c0fb08719a 100644 --- a/internal/querycoord/query_coord.go +++ b/internal/querycoord/query_coord.go @@ -15,12 +15,15 @@ import ( "context" "errors" + "fmt" "math/rand" "strconv" "sync" "sync/atomic" "time" + "github.com/golang/protobuf/proto" + "go.etcd.io/etcd/api/v3/mvccpb" "go.uber.org/zap" "github.com/milvus-io/milvus/internal/allocator" @@ -38,6 +41,10 @@ import ( "github.com/milvus-io/milvus/internal/util/typeutil" ) +const ( + handoffSegmentPrefix = "querycoord-handoff" +) + // Timestamp is an alias for the Int64 type type Timestamp = typeutil.Timestamp @@ -172,6 +179,9 @@ func (qc *QueryCoord) Start() error { qc.loopWg.Add(1) go qc.watchNodeLoop() + qc.loopWg.Add(1) + go qc.watchHandoffSegmentLoop() + go qc.session.LivenessCheck(qc.loopCtx, func() { qc.Stop() }) @@ -331,3 +341,86 @@ func (qc *QueryCoord) watchNodeLoop() { } } } + +func (qc *QueryCoord) watchHandoffSegmentLoop() { + ctx, cancel := context.WithCancel(qc.loopCtx) + + defer cancel() + defer qc.loopWg.Done() + log.Debug("query coordinator start watch segment loop") + + // TODO:: recover handoff task when coord down + watchChan := qc.kvClient.WatchWithPrefix(handoffSegmentPrefix) + + for { + select { + case <-ctx.Done(): + return + case resp := <-watchChan: + for _, event := range resp.Events { + segmentInfo := &querypb.SegmentInfo{} + err := proto.Unmarshal(event.Kv.Value, segmentInfo) + if err != nil { + log.Error("watchHandoffSegmentLoop: unmarshal failed", zap.Any("error", err.Error())) + continue + } + switch event.Type { + case mvccpb.PUT: + collectionID := segmentInfo.CollectionID + partitionID := segmentInfo.PartitionID + segmentID := segmentInfo.SegmentID + if Params.AutoHandoff { + log.Debug("watchHandoffSegmentLoop: handoff segment received", + zap.Any("collectionID", collectionID), + zap.Any("partitionID", partitionID), + zap.Any("segmentID", segmentID), + zap.Any("channel", segmentInfo.ChannelID), + ) + baseTask := newBaseTask(qc.loopCtx, querypb.TriggerCondition_handoff) + handoffReq := &querypb.HandoffSegmentsRequest{ + Base: &commonpb.MsgBase{ + MsgType: commonpb.MsgType_HandoffSegments, + }, + SegmentInfos: []*querypb.SegmentInfo{segmentInfo}, + } + handoffTask := &handoffTask{ + baseTask: baseTask, + HandoffSegmentsRequest: handoffReq, + dataCoord: qc.dataCoordClient, + cluster: qc.cluster, + meta: qc.meta, + } + err = qc.scheduler.Enqueue(handoffTask) + if err != nil { + log.Error("watchHandoffSegmentLoop: handoffTask enqueue failed", zap.Error(err)) + break + } + + go func() { + err := handoffTask.waitToFinish() + if err != nil { + log.Error("watchHandoffSegmentLoop: handoffTask failed", zap.Error(err)) + } + }() + + log.Debug("watchHandoffSegmentLoop: handoffTask completed", + zap.Any("collectionID", collectionID), + zap.Any("partitionID", partitionID), + zap.Any("segmentID", segmentID), + zap.Any("channel", segmentInfo.ChannelID), + ) + } + + buildQuerySegmentPath := fmt.Sprintf("%s/%d/%d/%d", handoffSegmentPrefix, collectionID, partitionID, segmentID) + err = qc.kvClient.Remove(buildQuerySegmentPath) + if err != nil { + log.Error("watchHandoffSegmentLoop: remove handoff segment from etcd failed", zap.Error(err)) + } + default: + // do nothing + } + } + } + } + +} diff --git a/internal/querycoord/query_coord_test.go b/internal/querycoord/query_coord_test.go index e7e25f97f5..848228ae66 100644 --- a/internal/querycoord/query_coord_test.go +++ b/internal/querycoord/query_coord_test.go @@ -26,6 +26,7 @@ import ( etcdkv "github.com/milvus-io/milvus/internal/kv/etcd" "github.com/milvus-io/milvus/internal/msgstream" + "github.com/milvus-io/milvus/internal/proto/commonpb" "github.com/milvus-io/milvus/internal/proto/querypb" "github.com/milvus-io/milvus/internal/util/sessionutil" ) @@ -219,3 +220,162 @@ func TestWatchNodeLoop(t *testing.T) { assert.Nil(t, err) }) } + +func TestHandoffSegmentLoop(t *testing.T) { + refreshParams() + baseCtx := context.Background() + + queryCoord, err := startQueryCoord(baseCtx) + assert.Nil(t, err) + + queryNode1, err := startQueryNodeServer(baseCtx) + assert.Nil(t, err) + waitQueryNodeOnline(queryCoord.cluster, queryNode1.queryNodeID) + + t.Run("Test watchHandoffLoop", func(t *testing.T) { + segmentInfo := &querypb.SegmentInfo{ + SegmentID: defaultSegmentID, + CollectionID: defaultCollectionID, + PartitionID: defaultPartitionID, + SegmentState: querypb.SegmentState_sealed, + } + + key := fmt.Sprintf("%s/%d/%d/%d", handoffSegmentPrefix, defaultCollectionID, defaultPartitionID, defaultSegmentID) + value, err := proto.Marshal(segmentInfo) + assert.Nil(t, err) + err = queryCoord.kvClient.Save(key, string(value)) + assert.Nil(t, err) + for { + _, err := queryCoord.kvClient.Load(key) + if err != nil { + break + } + } + }) + + loadPartitionTask := genLoadPartitionTask(baseCtx, queryCoord) + err = queryCoord.scheduler.Enqueue(loadPartitionTask) + assert.Nil(t, err) + waitTaskFinalState(loadPartitionTask, taskExpired) + + t.Run("Test partitionNotLoaded", func(t *testing.T) { + baseTask := newBaseTask(baseCtx, querypb.TriggerCondition_handoff) + segmentInfo := &querypb.SegmentInfo{ + SegmentID: defaultSegmentID, + CollectionID: defaultCollectionID, + PartitionID: defaultPartitionID + 1, + SegmentState: querypb.SegmentState_sealed, + } + handoffReq := &querypb.HandoffSegmentsRequest{ + Base: &commonpb.MsgBase{ + MsgType: commonpb.MsgType_HandoffSegments, + }, + SegmentInfos: []*querypb.SegmentInfo{segmentInfo}, + } + handoffTask := &handoffTask{ + baseTask: baseTask, + HandoffSegmentsRequest: handoffReq, + dataCoord: queryCoord.dataCoordClient, + cluster: queryCoord.cluster, + meta: queryCoord.meta, + } + err = queryCoord.scheduler.Enqueue(handoffTask) + assert.Nil(t, err) + + waitTaskFinalState(handoffTask, taskExpired) + }) + + loadCollectionTask := genLoadCollectionTask(baseCtx, queryCoord) + err = queryCoord.scheduler.Enqueue(loadCollectionTask) + assert.Nil(t, err) + waitTaskFinalState(loadCollectionTask, taskExpired) + + t.Run("Test handoffGrowingSegment", func(t *testing.T) { + infos := queryCoord.meta.showSegmentInfos(defaultCollectionID, nil) + assert.NotEqual(t, 0, len(infos)) + segmentID := defaultSegmentID + 4 + baseTask := newBaseTask(baseCtx, querypb.TriggerCondition_handoff) + + segmentInfo := &querypb.SegmentInfo{ + SegmentID: segmentID, + CollectionID: defaultCollectionID, + PartitionID: defaultPartitionID + 2, + SegmentState: querypb.SegmentState_sealed, + } + handoffReq := &querypb.HandoffSegmentsRequest{ + Base: &commonpb.MsgBase{ + MsgType: commonpb.MsgType_HandoffSegments, + }, + SegmentInfos: []*querypb.SegmentInfo{segmentInfo}, + } + handoffTask := &handoffTask{ + baseTask: baseTask, + HandoffSegmentsRequest: handoffReq, + dataCoord: queryCoord.dataCoordClient, + cluster: queryCoord.cluster, + meta: queryCoord.meta, + } + err = queryCoord.scheduler.Enqueue(handoffTask) + assert.Nil(t, err) + + waitTaskFinalState(handoffTask, taskExpired) + }) + + t.Run("Test binlogNotExist", func(t *testing.T) { + baseTask := newBaseTask(baseCtx, querypb.TriggerCondition_handoff) + segmentInfo := &querypb.SegmentInfo{ + SegmentID: defaultSegmentID + 100, + CollectionID: defaultCollectionID, + PartitionID: defaultPartitionID, + SegmentState: querypb.SegmentState_sealed, + } + handoffReq := &querypb.HandoffSegmentsRequest{ + Base: &commonpb.MsgBase{ + MsgType: commonpb.MsgType_HandoffSegments, + }, + SegmentInfos: []*querypb.SegmentInfo{segmentInfo}, + } + handoffTask := &handoffTask{ + baseTask: baseTask, + HandoffSegmentsRequest: handoffReq, + dataCoord: queryCoord.dataCoordClient, + cluster: queryCoord.cluster, + meta: queryCoord.meta, + } + err = queryCoord.scheduler.Enqueue(handoffTask) + assert.Nil(t, err) + + waitTaskFinalState(handoffTask, taskFailed) + }) + + t.Run("Test sealedSegmentExist", func(t *testing.T) { + baseTask := newBaseTask(baseCtx, querypb.TriggerCondition_handoff) + segmentInfo := &querypb.SegmentInfo{ + SegmentID: defaultSegmentID, + CollectionID: defaultCollectionID, + PartitionID: defaultPartitionID, + SegmentState: querypb.SegmentState_sealed, + } + handoffReq := &querypb.HandoffSegmentsRequest{ + Base: &commonpb.MsgBase{ + MsgType: commonpb.MsgType_HandoffSegments, + }, + SegmentInfos: []*querypb.SegmentInfo{segmentInfo}, + } + handoffTask := &handoffTask{ + baseTask: baseTask, + HandoffSegmentsRequest: handoffReq, + dataCoord: queryCoord.dataCoordClient, + cluster: queryCoord.cluster, + meta: queryCoord.meta, + } + err = queryCoord.scheduler.Enqueue(handoffTask) + assert.Nil(t, err) + + waitTaskFinalState(handoffTask, taskFailed) + }) + + queryCoord.Stop() + err = removeAllSession() + assert.Nil(t, err) +} diff --git a/internal/querycoord/task.go b/internal/querycoord/task.go index fdc7354d16..7afabb91b0 100644 --- a/internal/querycoord/task.go +++ b/internal/querycoord/task.go @@ -1378,7 +1378,152 @@ func (wqt *watchQueryChannelTask) postExecute(context.Context) error { return nil } +//****************************handoff task********************************// type handoffTask struct { + *baseTask + *querypb.HandoffSegmentsRequest + dataCoord types.DataCoord + cluster Cluster + meta Meta +} + +func (ht *handoffTask) msgBase() *commonpb.MsgBase { + return ht.Base +} + +func (ht *handoffTask) marshal() ([]byte, error) { + return proto.Marshal(ht.HandoffSegmentsRequest) +} + +func (ht *handoffTask) msgType() commonpb.MsgType { + return ht.Base.MsgType +} + +func (ht *handoffTask) timestamp() Timestamp { + return ht.Base.Timestamp +} + +func (ht *handoffTask) preExecute(context.Context) error { + ht.setResultInfo(nil) + segmentIDs := make([]UniqueID, 0) + segmentInfos := ht.HandoffSegmentsRequest.SegmentInfos + for _, info := range segmentInfos { + segmentIDs = append(segmentIDs, info.SegmentID) + } + log.Debug("start do handoff segments task", + zap.Int64s("segmentIDs", segmentIDs)) + return nil +} + +func (ht *handoffTask) execute(ctx context.Context) error { + segmentInfos := ht.HandoffSegmentsRequest.SegmentInfos + for _, segmentInfo := range segmentInfos { + collectionID := segmentInfo.CollectionID + partitionID := segmentInfo.PartitionID + segmentID := segmentInfo.SegmentID + + collectionInfo, err := ht.meta.getCollectionInfoByID(collectionID) + if err != nil { + log.Debug("handoffTask: collection has not been loaded into memory", zap.Int64("collectionID", collectionID), zap.Int64("segmentID", segmentID)) + continue + } + + partitionLoaded := false + for _, id := range collectionInfo.PartitionIDs { + if id == partitionID { + partitionLoaded = true + } + } + + if collectionInfo.LoadType != querypb.LoadType_loadCollection && !partitionLoaded { + log.Debug("handoffTask: partition has not been loaded into memory", zap.Int64("collectionID", collectionID), zap.Int64("partitionID", partitionID), zap.Int64("segmentID", segmentID)) + continue + } + + _, err = ht.meta.getSegmentInfoByID(segmentID) + if err != nil { + getRecoveryInfoRequest := &datapb.GetRecoveryInfoRequest{ + Base: ht.Base, + CollectionID: collectionID, + PartitionID: partitionID, + } + recoveryInfo, err := ht.dataCoord.GetRecoveryInfo(ctx, getRecoveryInfoRequest) + if err != nil { + ht.setResultInfo(err) + return err + } + + findBinlog := false + var loadSegmentReq *querypb.LoadSegmentsRequest + for _, segmentBinlogs := range recoveryInfo.Binlogs { + if segmentBinlogs.SegmentID == segmentID { + findBinlog = true + segmentLoadInfo := &querypb.SegmentLoadInfo{ + SegmentID: segmentID, + PartitionID: partitionID, + CollectionID: collectionID, + BinlogPaths: segmentBinlogs.FieldBinlogs, + NumOfRows: segmentBinlogs.NumOfRows, + } + + msgBase := proto.Clone(ht.Base).(*commonpb.MsgBase) + msgBase.MsgType = commonpb.MsgType_LoadSegments + loadSegmentReq = &querypb.LoadSegmentsRequest{ + Base: msgBase, + Infos: []*querypb.SegmentLoadInfo{segmentLoadInfo}, + Schema: collectionInfo.Schema, + LoadCondition: querypb.TriggerCondition_handoff, + } + } + } + + if !findBinlog { + err = fmt.Errorf("segmnet has not been flushed, segmentID is %d", segmentID) + ht.setResultInfo(err) + return err + } + err = assignInternalTask(ctx, collectionID, ht, ht.meta, ht.cluster, []*querypb.LoadSegmentsRequest{loadSegmentReq}, nil, true) + if err != nil { + log.Error("handoffTask: assign child task failed", zap.Any("segmentInfo", segmentInfo)) + ht.setResultInfo(err) + return err + } + } else { + err = fmt.Errorf("sealed segment has been exist on query node, segmentID is %d", segmentID) + log.Error("handoffTask: sealed segment has been exist on query node", zap.Int64("segmentID", segmentID)) + ht.setResultInfo(err) + return err + } + } + + log.Debug("handoffTask: assign child task done", zap.Any("segmentInfos", segmentInfos)) + + log.Debug("handoffTask Execute done", + zap.Int64("taskID", ht.getTaskID())) + return nil +} + +func (ht *handoffTask) postExecute(context.Context) error { + if ht.result.ErrorCode != commonpb.ErrorCode_Success { + ht.childTasks = []task{} + } + + log.Debug("handoffTask postExecute done", + zap.Int64("taskID", ht.getTaskID())) + + return nil +} + +func (ht *handoffTask) rollBack(ctx context.Context) []task { + resultTasks := make([]task, 0) + childTasks := ht.getChildTask() + for _, childTask := range childTasks { + if childTask.msgType() == commonpb.MsgType_LoadSegments { + // TODO:: add release segment to rollBack, no release does not affect correctness of query + } + } + + return resultTasks } type loadBalanceTask struct { diff --git a/internal/querycoord/task_scheduler.go b/internal/querycoord/task_scheduler.go index 1136174015..8cdbbe0ee7 100644 --- a/internal/querycoord/task_scheduler.go +++ b/internal/querycoord/task_scheduler.go @@ -387,6 +387,20 @@ func (scheduler *TaskScheduler) unmarshalTask(taskID UniqueID, t string) (task, meta: scheduler.meta, } newTask = loadBalanceTask + case commonpb.MsgType_HandoffSegments: + handoffReq := querypb.HandoffSegmentsRequest{} + err = proto.Unmarshal([]byte(t), &handoffReq) + if err != nil { + return nil, err + } + handoffTask := &handoffTask{ + baseTask: baseTask, + HandoffSegmentsRequest: &handoffReq, + dataCoord: scheduler.dataCoord, + cluster: scheduler.cluster, + meta: scheduler.meta, + } + newTask = handoffTask default: err = errors.New("inValid msg type when unMarshal task") log.Error(err.Error()) diff --git a/internal/querycoord/task_scheduler_test.go b/internal/querycoord/task_scheduler_test.go index 3a00de5b37..71861f930f 100644 --- a/internal/querycoord/task_scheduler_test.go +++ b/internal/querycoord/task_scheduler_test.go @@ -398,6 +398,28 @@ func TestUnMarshalTask(t *testing.T) { assert.Equal(t, task.msgType(), commonpb.MsgType_LoadBalanceSegments) }) + t.Run("Test handoffTask", func(t *testing.T) { + handoffTask := &handoffTask{ + HandoffSegmentsRequest: &querypb.HandoffSegmentsRequest{ + Base: &commonpb.MsgBase{ + MsgType: commonpb.MsgType_HandoffSegments, + }, + }, + } + + blobs, err := handoffTask.marshal() + assert.Nil(t, err) + err = kv.Save("testMarshalHandoffTask", string(blobs)) + assert.Nil(t, err) + defer kv.RemoveWithPrefix("testMarshalHandoffTask") + value, err := kv.Load("testMarshalHandoffTask") + assert.Nil(t, err) + + task, err := taskScheduler.unmarshalTask(1008, value) + assert.Nil(t, err) + assert.Equal(t, task.msgType(), commonpb.MsgType_HandoffSegments) + }) + taskScheduler.Close() } diff --git a/internal/querycoord/task_test.go b/internal/querycoord/task_test.go index 695048d77b..8cfe46ac15 100644 --- a/internal/querycoord/task_test.go +++ b/internal/querycoord/task_test.go @@ -737,3 +737,54 @@ func Test_reverseSealedSegmentChangeInfo(t *testing.T) { err = removeAllSession() assert.Nil(t, err) } + +func Test_handoffSegmentFail(t *testing.T) { + refreshParams() + ctx := context.Background() + queryCoord, err := startQueryCoord(ctx) + assert.Nil(t, err) + + node1, err := startQueryNodeServer(ctx) + assert.Nil(t, err) + waitQueryNodeOnline(queryCoord.cluster, node1.queryNodeID) + + loadCollectionTask := genLoadCollectionTask(ctx, queryCoord) + err = queryCoord.scheduler.Enqueue(loadCollectionTask) + assert.Nil(t, err) + waitTaskFinalState(loadCollectionTask, taskExpired) + + node1.loadSegment = returnFailedResult + + infos := queryCoord.meta.showSegmentInfos(defaultCollectionID, nil) + assert.NotEqual(t, 0, len(infos)) + segmentID := defaultSegmentID + 4 + baseTask := newBaseTask(ctx, querypb.TriggerCondition_handoff) + + segmentInfo := &querypb.SegmentInfo{ + SegmentID: segmentID, + CollectionID: defaultCollectionID, + PartitionID: defaultPartitionID + 2, + SegmentState: querypb.SegmentState_sealed, + } + handoffReq := &querypb.HandoffSegmentsRequest{ + Base: &commonpb.MsgBase{ + MsgType: commonpb.MsgType_HandoffSegments, + }, + SegmentInfos: []*querypb.SegmentInfo{segmentInfo}, + } + handoffTask := &handoffTask{ + baseTask: baseTask, + HandoffSegmentsRequest: handoffReq, + dataCoord: queryCoord.dataCoordClient, + cluster: queryCoord.cluster, + meta: queryCoord.meta, + } + err = queryCoord.scheduler.Enqueue(handoffTask) + assert.Nil(t, err) + + waitTaskFinalState(handoffTask, taskFailed) + + queryCoord.Stop() + err = removeAllSession() + assert.Nil(t, err) +}