milvus-io
/
milvus
mirror of https://github.com/milvus-io/milvus.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Decouple mq module from internal proto definition (#22536) 

Signed-off-by: jaime <yun.zhang@zilliz.com>
pull/22573/head
jaime 2023-03-04 23:21:50 +08:00 committed by GitHub
parent d302632b2e
commit d126f06946
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
104 changed files with 2236 additions and 2822 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
cmd/tools/migration/legacy/legacypb/legacy.pb.go
View File

@ -5,11 +5,10 @@ package legacypb
import (
fmt "fmt"
math "math"
proto "github.com/golang/protobuf/proto"
commonpb "github.com/milvus-io/milvus-proto/go-api/commonpb"
schemapb "github.com/milvus-io/milvus-proto/go-api/schemapb"
math "math"
)
// Reference imports to suppress errors if they are not otherwise used.

2
docs/design_docs/datanode_flowgraph_recovery_design_0604_2021.md
View File

@ -48,7 +48,7 @@ Proto design for WatchDmChannelReq:
message VchannelInfo {
int64 collectionID = 1;
string channelName = 2;
internal.MsgPosition seek_position = 3;
msgpb.MsgPosition seek_position = 3;
repeated SegmentInfo unflushedSegments = 4;
repeated int64 flushedSegments = 5;
}

4
docs/design_docs/datanode_recovery_design_0513_2021.md
View File

@ -67,7 +67,7 @@ message ID2PathList {
message CheckPoint {
int64 segmentID = 1;
internal.MsgPosition position = 2;
msgpb.MsgPosition position = 2;
int64 num_of_rows = 3;
}
@ -97,7 +97,7 @@ Change `WatchDmChannelsRequest` proto.
message VchannelInfo {
int64 collectionID = 1;
string channelName = 2;
internal.MsgPosition seek_position = 3;
msgpb.MsgPosition seek_position = 3;
repeated SegmentInfo unflushedSegments = 4;
repeated int64 flushedSegments = 5;
}

4
docs/design_docs/milvus_create_index_en.md
View File

@ -213,8 +213,8 @@ message SegmentInfo {
common.SegmentState state = 6;
int64 max_row_num = 7;
uint64 last_expire_time = 8;
internal.MsgPosition start_position = 9;
internal.MsgPosition dml_position = 10;
msgpb.MsgPosition start_position = 9;
msgpb.MsgPosition dml_position = 10;
repeated FieldBinlog binlogs = 11;
}

4
docs/design_docs/milvus_flush_collections_en.md
View File

@ -117,10 +117,10 @@ message SegmentInfo {
string insert_channel = 4;
int64 num_of_rows = 5;
common.SegmentState state = 6;
internal.MsgPosition dml_position = 7;
msgpb.MsgPosition dml_position = 7;
int64 max_row_num = 8;
uint64 last_expire_time = 9;
internal.MsgPosition start_position = 10;
msgpb.MsgPosition start_position = 10;
}
enum SegmentState {

10
docs/developer_guides/chap06_root_coordinator.md
View File

@ -363,7 +363,7 @@ type CreateCollectionRequest struct {
type CreateCollectionMsg struct {
BaseMsg
internalpb.CreateCollectionRequest
msgpb.CreateCollectionRequest
}
```
@ -450,16 +450,16 @@ type Core struct {
SendTimeTick func(t typeutil.Timestamp) error
//setMsgStreams, send create collection into dd channel
SendDdCreateCollectionReq func(ctx context.Context, req *internalpb.CreateCollectionRequest, channelNames []string) error
SendDdCreateCollectionReq func(ctx context.Context, req *msgpb.CreateCollectionRequest, channelNames []string) error
//setMsgStreams, send drop collection into dd channel, and notify the proxy to delete this collection
SendDdDropCollectionReq func(ctx context.Context, req *internalpb.DropCollectionRequest, channelNames []string) error
SendDdDropCollectionReq func(ctx context.Context, req *msgpb.DropCollectionRequest, channelNames []string) error
//setMsgStreams, send create partition into dd channel
SendDdCreatePartitionReq func(ctx context.Context, req *internalpb.CreatePartitionRequest, channelNames []string) error
SendDdCreatePartitionReq func(ctx context.Context, req *msgpb.CreatePartitionRequest, channelNames []string) error
//setMsgStreams, send drop partition into dd channel
SendDdDropPartitionReq func(ctx context.Context, req *internalpb.DropPartitionRequest, channelNames []string) error
SendDdDropPartitionReq func(ctx context.Context, req *msgpb.DropPartitionRequest, channelNames []string) error
// if RootCoord create segment, DataCoord will put segment msg into this channel
DataCoordSegmentChan <-chan *ms.MsgPack

18
docs/developer_guides/chap09_data_coord.md
View File

@ -116,8 +116,8 @@ const (
type SegmentStateInfo struct {
SegmentID UniqueID
State commonpb.SegmentState
StartPosition *internalpb.MsgPosition
EndPosition *internalpb.MsgPosition
StartPosition *msgpb.MsgPosition
EndPosition *msgpb.MsgPosition
Status *commonpb.Status
}
@ -188,11 +188,11 @@ type SegmentInfo struct {
InsertChannel string
NumOfRows int64
State commonpb.SegmentState
DmlPosition *internalpb.MsgPosition
DmlPosition *msgpb.MsgPosition
MaxRowNum int64
LastExpireTime uint64
StartPosition *internalpb.MsgPosition
DmlPosition *internalpb.MsgPosition
StartPosition *msgpb.MsgPosition
DmlPosition *msgpb.MsgPosition
Binlogs []*FieldBinlog
}
@ -215,7 +215,7 @@ type GetRecoveryInfoRequest struct {
type VchannelInfo struct {
CollectionID int64
ChannelName string
SeekPosition *internalpb.MsgPosition
SeekPosition *msgpb.MsgPosition
UnflushedSegments []*SegmentInfo
FlushedSegments []int64
}
@ -236,7 +236,7 @@ type GetRecoveryInfoResponse struct {
```go
type SegmentStartPosition struct {
StartPosition *internalpb.MsgPosition
StartPosition *msgpb.MsgPosition
SegmentID int64
}
@ -325,8 +325,8 @@ type SegmentStatisticsUpdates struct {
NumRows int64
CreateTime uint64
EndTime uint64
StartPosition *internalpb.MsgPosition
EndPosition *internalpb.MsgPosition
StartPosition *msgpb.MsgPosition
EndPosition *msgpb.MsgPosition
}
type SegmentStatistics struct {

4
go.mod
View File

@ -27,7 +27,7 @@ require (
github.com/klauspost/compress v1.14.4
github.com/lingdor/stackerror v0.0.0-20191119040541-976d8885ed76
github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d
github.com/milvus-io/milvus-proto/go-api v0.0.0-20230220081023-3cf200738ae7
github.com/milvus-io/milvus-proto/go-api v0.0.0-20230303054144-16f081962572
github.com/minio/minio-go/v7 v7.0.17
github.com/panjf2000/ants/v2 v2.4.8
github.com/prometheus/client_golang v1.11.1
@ -179,8 +179,8 @@ require (
require github.com/ianlancetaylor/cgosymbolizer v0.0.0-20221217025313-27d3c9f66b6a // indirect
require (
github.com/golang/mock v1.5.0
github.com/cockroachdb/errors v1.9.1
github.com/golang/mock v1.5.0
github.com/uber/jaeger-client-go v2.30.0+incompatible
go.opentelemetry.io/otel/exporters/jaeger v1.11.2
go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.11.2

8
go.sum
View File

@ -562,8 +562,12 @@ github.com/microcosm-cc/bluemonday v1.0.2/go.mod h1:iVP4YcDBq+n/5fb23BhYFvIMq/le
github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg=
github.com/milvus-io/gorocksdb v0.0.0-20220624081344-8c5f4212846b h1:TfeY0NxYxZzUfIfYe5qYDBzt4ZYRqzUjTR6CvUzjat8=
github.com/milvus-io/gorocksdb v0.0.0-20220624081344-8c5f4212846b/go.mod h1:iwW+9cWfIzzDseEBCCeDSN5SD16Tidvy8cwQ7ZY8Qj4=
github.com/milvus-io/milvus-proto/go-api v0.0.0-20230220081023-3cf200738ae7 h1:G3qXTVWaHXQkgnPOxuCb/NIxb2oD4xHHViyuTGs5KQU=
github.com/milvus-io/milvus-proto/go-api v0.0.0-20230220081023-3cf200738ae7/go.mod h1:148qnlmZ0Fdm1Fq+Mj/OW2uDoEP25g3mjh0vMGtkgmk=
github.com/milvus-io/milvus-proto/go-api v0.0.0-20230129073344-87a125853a0b h1:HoJ3J70COnaR3WQTA4gN70DkiaMRPkyLI6yXrPqpFiU=
github.com/milvus-io/milvus-proto/go-api v0.0.0-20230129073344-87a125853a0b/go.mod h1:148qnlmZ0Fdm1Fq+Mj/OW2uDoEP25g3mjh0vMGtkgmk=
github.com/milvus-io/milvus-proto/go-api v0.0.0-20230302072344-f5dca5d8857b h1:O8ueZJ150EZ78naAjeTkqtoNpI2Pw7YtG3Qg19EMmX0=
github.com/milvus-io/milvus-proto/go-api v0.0.0-20230302072344-f5dca5d8857b/go.mod h1:148qnlmZ0Fdm1Fq+Mj/OW2uDoEP25g3mjh0vMGtkgmk=
github.com/milvus-io/milvus-proto/go-api v0.0.0-20230303054144-16f081962572 h1:QOimLfT1VwjH9jUkq2SN3uT/WGfgcD4pYBoFOGUMwjM=
github.com/milvus-io/milvus-proto/go-api v0.0.0-20230303054144-16f081962572/go.mod h1:148qnlmZ0Fdm1Fq+Mj/OW2uDoEP25g3mjh0vMGtkgmk=
github.com/milvus-io/pulsar-client-go v0.6.10 h1:eqpJjU+/QX0iIhEo3nhOqMNXL+TyInAs1IAHZCrCM/A=
github.com/milvus-io/pulsar-client-go v0.6.10/go.mod h1:lQqCkgwDF8YFYjKA+zOheTk1tev2B+bKj5j7+nm8M1w=
github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8 h1:AMFGa4R4MiIpspGNG7Z948v4n35fFGB3RR3G/ry4FWs=

509
internal/core/src/pb/common.pb.cc
View File

@ -180,10 +180,24 @@ struct PrivilegeExtDefaultTypeInternal {
};
};
PROTOBUF_ATTRIBUTE_NO_DESTROY PROTOBUF_CONSTINIT PROTOBUF_ATTRIBUTE_INIT_PRIORITY1 PrivilegeExtDefaultTypeInternal _PrivilegeExt_default_instance_;
PROTOBUF_CONSTEXPR SegmentStats::SegmentStats(
::_pbi::ConstantInitialized): _impl_{
/*decltype(_impl_.segmentid_)*/int64_t{0}
, /*decltype(_impl_.numrows_)*/int64_t{0}
, /*decltype(_impl_._cached_size_)*/{}} {}
struct SegmentStatsDefaultTypeInternal {
PROTOBUF_CONSTEXPR SegmentStatsDefaultTypeInternal()
: _instance(::_pbi::ConstantInitialized{}) {}
~SegmentStatsDefaultTypeInternal() {}
union {
SegmentStats _instance;
};
};
PROTOBUF_ATTRIBUTE_NO_DESTROY PROTOBUF_CONSTINIT PROTOBUF_ATTRIBUTE_INIT_PRIORITY1 SegmentStatsDefaultTypeInternal _SegmentStats_default_instance_;
} // namespace common
} // namespace proto
} // namespace milvus
static ::_pb::Metadata file_level_metadata_common_2eproto[11];
static ::_pb::Metadata file_level_metadata_common_2eproto[12];
static const ::_pb::EnumDescriptor* file_level_enum_descriptors_common_2eproto[13];
static constexpr ::_pb::ServiceDescriptor const** file_level_service_descriptors_common_2eproto = nullptr;
@ -279,6 +293,14 @@ const uint32_t TableStruct_common_2eproto::offsets[] PROTOBUF_SECTION_VARIABLE(p
PROTOBUF_FIELD_OFFSET(::milvus::proto::common::PrivilegeExt, _impl_.object_privilege_),
PROTOBUF_FIELD_OFFSET(::milvus::proto::common::PrivilegeExt, _impl_.object_name_index_),
PROTOBUF_FIELD_OFFSET(::milvus::proto::common::PrivilegeExt, _impl_.object_name_indexs_),
~0u, // no _has_bits_
PROTOBUF_FIELD_OFFSET(::milvus::proto::common::SegmentStats, _internal_metadata_),
~0u, // no _extensions_
~0u, // no _oneof_case_
~0u, // no _weak_field_map_
~0u, // no _inlined_string_donated_
PROTOBUF_FIELD_OFFSET(::milvus::proto::common::SegmentStats, _impl_.segmentid_),
PROTOBUF_FIELD_OFFSET(::milvus::proto::common::SegmentStats, _impl_.numrows_),
};
static const ::_pbi::MigrationSchema schemas[] PROTOBUF_SECTION_VARIABLE(protodesc_cold) = {
{ 0, -1, -1, sizeof(::milvus::proto::common::Status)},
@ -292,6 +314,7 @@ static const ::_pbi::MigrationSchema schemas[] PROTOBUF_SECTION_VARIABLE(protode
{ 66, -1, -1, sizeof(::milvus::proto::common::MsgHeader)},
{ 73, -1, -1, sizeof(::milvus::proto::common::DMLMsgHeader)},
{ 81, -1, -1, sizeof(::milvus::proto::common::PrivilegeExt)},
{ 91, -1, -1, sizeof(::milvus::proto::common::SegmentStats)},
};
static const ::_pb::Message* const file_default_instances[] = {
@ -306,6 +329,7 @@ static const ::_pb::Message* const file_default_instances[] = {
&::milvus::proto::common::_MsgHeader_default_instance_._instance,
&::milvus::proto::common::_DMLMsgHeader_default_instance_._instance,
&::milvus::proto::common::_PrivilegeExt_default_instance_._instance,
&::milvus::proto::common::_SegmentStats_default_instance_._instance,
};
const char descriptor_table_protodef_common_2eproto[] PROTOBUF_SECTION_VARIABLE(protodesc_cold) =
@ -332,146 +356,148 @@ const char descriptor_table_protodef_common_2eproto[] PROTOBUF_SECTION_VARIABLE(
".ObjectType\022>\n\020object_privilege\030\002 \001(\0162$."
"milvus.proto.common.ObjectPrivilege\022\031\n\021o"
"bject_name_index\030\003 \001(\005\022\032\n\022object_name_in"
"dexs\030\004 \001(\005*\303\n\n\tErrorCode\022\013\n\007Success\020\000\022\023\n"
"\017UnexpectedError\020\001\022\021\n\rConnectFailed\020\002\022\024\n"
"\020PermissionDenied\020\003\022\027\n\023CollectionNotExis"
"ts\020\004\022\023\n\017IllegalArgument\020\005\022\024\n\020IllegalDime"
"nsion\020\007\022\024\n\020IllegalIndexType\020\010\022\031\n\025Illegal"
"CollectionName\020\t\022\017\n\013IllegalTOPK\020\n\022\024\n\020Ill"
"egalRowRecord\020\013\022\023\n\017IllegalVectorID\020\014\022\027\n\023"
"IllegalSearchResult\020\r\022\020\n\014FileNotFound\020\016\022"
"\016\n\nMetaFailed\020\017\022\017\n\013CacheFailed\020\020\022\026\n\022Cann"
"otCreateFolder\020\021\022\024\n\020CannotCreateFile\020\022\022\026"
"\n\022CannotDeleteFolder\020\023\022\024\n\020CannotDeleteFi"
"le\020\024\022\023\n\017BuildIndexError\020\025\022\020\n\014IllegalNLIS"
"T\020\026\022\025\n\021IllegalMetricType\020\027\022\017\n\013OutOfMemor"
"y\020\030\022\021\n\rIndexNotExist\020\031\022\023\n\017EmptyCollectio"
"n\020\032\022\033\n\027UpdateImportTaskFailure\020\033\022\032\n\026Coll"
"ectionNameNotFound\020\034\022\033\n\027CreateCredential"
"Failure\020\035\022\033\n\027UpdateCredentialFailure\020\036\022\033"
"\n\027DeleteCredentialFailure\020\037\022\030\n\024GetCreden"
"tialFailure\020 \022\030\n\024ListCredUsersFailure\020!\022"
"\022\n\016GetUserFailure\020\"\022\025\n\021CreateRoleFailure"
"\020#\022\023\n\017DropRoleFailure\020$\022\032\n\026OperateUserRo"
"leFailure\020%\022\025\n\021SelectRoleFailure\020&\022\025\n\021Se"
"lectUserFailure\020\'\022\031\n\025SelectResourceFailu"
"re\020(\022\033\n\027OperatePrivilegeFailure\020)\022\026\n\022Sel"
"ectGrantFailure\020*\022!\n\035RefreshPolicyInfoCa"
"cheFailure\020+\022\025\n\021ListPolicyFailure\020,\022\022\n\016N"
"otShardLeader\020-\022\026\n\022NoReplicaAvailable\020.\022"
"\023\n\017SegmentNotFound\020/\022\r\n\tForceDeny\0200\022\r\n\tR"
"ateLimit\0201\022\022\n\016NodeIDNotMatch\0202\022\024\n\020Upsert"
"AutoIDTrue\0203\022\034\n\030InsufficientMemoryToLoad"
"\0204\022\030\n\024MemoryQuotaExhausted\0205\022\026\n\022DiskQuot"
"aExhausted\0206\022\025\n\021TimeTickLongDelay\0207\022\021\n\rN"
"otReadyServe\0208\022\033\n\027NotReadyCoordActivatin"
"g\0209\022\017\n\013DataCoordNA\020d\022\022\n\rDDRequestRace\020\350\007"
"*c\n\nIndexState\022\022\n\016IndexStateNone\020\000\022\014\n\010Un"
"issued\020\001\022\016\n\nInProgress\020\002\022\014\n\010Finished\020\003\022\n"
"\n\006Failed\020\004\022\t\n\005Retry\020\005*\202\001\n\014SegmentState\022\024"
"\n\020SegmentStateNone\020\000\022\014\n\010NotExist\020\001\022\013\n\007Gr"
"owing\020\002\022\n\n\006Sealed\020\003\022\013\n\007Flushed\020\004\022\014\n\010Flus"
"hing\020\005\022\013\n\007Dropped\020\006\022\r\n\tImporting\020\007*>\n\017Pl"
"aceholderType\022\010\n\004None\020\000\022\020\n\014BinaryVector\020"
"d\022\017\n\013FloatVector\020e*\300\016\n\007MsgType\022\r\n\tUndefi"
"ned\020\000\022\024\n\020CreateCollection\020d\022\022\n\016DropColle"
"ction\020e\022\021\n\rHasCollection\020f\022\026\n\022DescribeCo"
"llection\020g\022\023\n\017ShowCollections\020h\022\024\n\020GetSy"
"stemConfigs\020i\022\022\n\016LoadCollection\020j\022\025\n\021Rel"
"easeCollection\020k\022\017\n\013CreateAlias\020l\022\r\n\tDro"
"pAlias\020m\022\016\n\nAlterAlias\020n\022\023\n\017AlterCollect"
"ion\020o\022\024\n\020RenameCollection\020p\022\024\n\017CreatePar"
"tition\020\310\001\022\022\n\rDropPartition\020\311\001\022\021\n\014HasPart"
"ition\020\312\001\022\026\n\021DescribePartition\020\313\001\022\023\n\016Show"
"Partitions\020\314\001\022\023\n\016LoadPartitions\020\315\001\022\026\n\021Re"
"leasePartitions\020\316\001\022\021\n\014ShowSegments\020\372\001\022\024\n"
"\017DescribeSegment\020\373\001\022\021\n\014LoadSegments\020\374\001\022\024"
"\n\017ReleaseSegments\020\375\001\022\024\n\017HandoffSegments\020"
"\376\001\022\030\n\023LoadBalanceSegments\020\377\001\022\025\n\020Describe"
"Segments\020\200\002\022\020\n\013CreateIndex\020\254\002\022\022\n\rDescrib"
"eIndex\020\255\002\022\016\n\tDropIndex\020\256\002\022\013\n\006Insert\020\220\003\022\013"
"\n\006Delete\020\221\003\022\n\n\005Flush\020\222\003\022\027\n\022ResendSegment"
"Stats\020\223\003\022\013\n\006Upsert\020\224\003\022\013\n\006Search\020\364\003\022\021\n\014Se"
"archResult\020\365\003\022\022\n\rGetIndexState\020\366\003\022\032\n\025Get"
"IndexBuildProgress\020\367\003\022\034\n\027GetCollectionSt"
"atistics\020\370\003\022\033\n\026GetPartitionStatistics\020\371\003"
"\022\r\n\010Retrieve\020\372\003\022\023\n\016RetrieveResult\020\373\003\022\024\n\017"
"WatchDmChannels\020\374\003\022\025\n\020RemoveDmChannels\020\375"
"\003\022\027\n\022WatchQueryChannels\020\376\003\022\030\n\023RemoveQuer"
"yChannels\020\377\003\022\035\n\030SealedSegmentsChangeInfo"
"\020\200\004\022\027\n\022WatchDeltaChannels\020\201\004\022\024\n\017GetShard"
"Leaders\020\202\004\022\020\n\013GetReplicas\020\203\004\022\023\n\016UnsubDmC"
"hannel\020\204\004\022\024\n\017GetDistribution\020\205\004\022\025\n\020SyncD"
"istribution\020\206\004\022\020\n\013SegmentInfo\020\330\004\022\017\n\nSyst"
"emInfo\020\331\004\022\024\n\017GetRecoveryInfo\020\332\004\022\024\n\017GetSe"
"gmentState\020\333\004\022\r\n\010TimeTick\020\260\t\022\023\n\016QueryNod"
"eStats\020\261\t\022\016\n\tLoadIndex\020\262\t\022\016\n\tRequestID\020\263"
"\t\022\017\n\nRequestTSO\020\264\t\022\024\n\017AllocateSegment\020\265\t"
"\022\026\n\021SegmentStatistics\020\266\t\022\025\n\020SegmentFlush"
"Done\020\267\t\022\017\n\nDataNodeTt\020\270\t\022\025\n\020CreateCreden"
"tial\020\334\013\022\022\n\rGetCredential\020\335\013\022\025\n\020DeleteCre"
"dential\020\336\013\022\025\n\020UpdateCredential\020\337\013\022\026\n\021Lis"
"tCredUsernames\020\340\013\022\017\n\nCreateRole\020\300\014\022\r\n\010Dr"
"opRole\020\301\014\022\024\n\017OperateUserRole\020\302\014\022\017\n\nSelec"
"tRole\020\303\014\022\017\n\nSelectUser\020\304\014\022\023\n\016SelectResou"
"rce\020\305\014\022\025\n\020OperatePrivilege\020\306\014\022\020\n\013SelectG"
"rant\020\307\014\022\033\n\026RefreshPolicyInfoCache\020\310\014\022\017\n\n"
"ListPolicy\020\311\014\022\030\n\023CreateResourceGroup\020\244\r\022"
"\026\n\021DropResourceGroup\020\245\r\022\027\n\022ListResourceG"
"roups\020\246\r\022\032\n\025DescribeResourceGroup\020\247\r\022\021\n\014"
"TransferNode\020\250\r\022\024\n\017TransferReplica\020\251\r*\"\n"
"\007DslType\022\007\n\003Dsl\020\000\022\016\n\nBoolExprV1\020\001*B\n\017Com"
"pactionState\022\021\n\rUndefiedState\020\000\022\r\n\tExecu"
"ting\020\001\022\r\n\tCompleted\020\002*X\n\020ConsistencyLeve"
"l\022\n\n\006Strong\020\000\022\013\n\007Session\020\001\022\013\n\007Bounded\020\002\022"
"\016\n\nEventually\020\003\022\016\n\nCustomized\020\004*\236\001\n\013Impo"
"rtState\022\021\n\rImportPending\020\000\022\020\n\014ImportFail"
"ed\020\001\022\021\n\rImportStarted\020\002\022\023\n\017ImportPersist"
"ed\020\005\022\021\n\rImportFlushed\020\010\022\023\n\017ImportComplet"
"ed\020\006\022\032\n\026ImportFailedAndCleaned\020\007*2\n\nObje"
"ctType\022\016\n\nCollection\020\000\022\n\n\006Global\020\001\022\010\n\004Us"
"er\020\002*\227\007\n\017ObjectPrivilege\022\020\n\014PrivilegeAll"
"\020\000\022\035\n\031PrivilegeCreateCollection\020\001\022\033\n\027Pri"
"vilegeDropCollection\020\002\022\037\n\033PrivilegeDescr"
"ibeCollection\020\003\022\034\n\030PrivilegeShowCollecti"
"ons\020\004\022\021\n\rPrivilegeLoad\020\005\022\024\n\020PrivilegeRel"
"ease\020\006\022\027\n\023PrivilegeCompaction\020\007\022\023\n\017Privi"
"legeInsert\020\010\022\023\n\017PrivilegeDelete\020\t\022\032\n\026Pri"
"vilegeGetStatistics\020\n\022\030\n\024PrivilegeCreate"
"Index\020\013\022\030\n\024PrivilegeIndexDetail\020\014\022\026\n\022Pri"
"vilegeDropIndex\020\r\022\023\n\017PrivilegeSearch\020\016\022\022"
"\n\016PrivilegeFlush\020\017\022\022\n\016PrivilegeQuery\020\020\022\030"
"\n\024PrivilegeLoadBalance\020\021\022\023\n\017PrivilegeImp"
"ort\020\022\022\034\n\030PrivilegeCreateOwnership\020\023\022\027\n\023P"
"rivilegeUpdateUser\020\024\022\032\n\026PrivilegeDropOwn"
"ership\020\025\022\034\n\030PrivilegeSelectOwnership\020\026\022\034"
"\n\030PrivilegeManageOwnership\020\027\022\027\n\023Privileg"
"eSelectUser\020\030\022\023\n\017PrivilegeUpsert\020\031\022 \n\034Pr"
"ivilegeCreateResourceGroup\020\032\022\036\n\032Privileg"
"eDropResourceGroup\020\033\022\"\n\036PrivilegeDescrib"
"eResourceGroup\020\034\022\037\n\033PrivilegeListResourc"
"eGroups\020\035\022\031\n\025PrivilegeTransferNode\020\036\022\034\n\030"
"PrivilegeTransferReplica\020\037\022\037\n\033PrivilegeG"
"etLoadingProgress\020 \022\031\n\025PrivilegeGetLoadS"
"tate\020!*S\n\tStateCode\022\020\n\014Initializing\020\000\022\013\n"
"\007Healthy\020\001\022\014\n\010Abnormal\020\002\022\013\n\007StandBy\020\003\022\014\n"
"\010Stopping\020\004*c\n\tLoadState\022\025\n\021LoadStateNot"
"Exist\020\000\022\024\n\020LoadStateNotLoad\020\001\022\024\n\020LoadSta"
"teLoading\020\002\022\023\n\017LoadStateLoaded\020\003:^\n\021priv"
"ilege_ext_obj\022\037.google.protobuf.MessageO"
"ptions\030\351\007 \001(\0132!.milvus.proto.common.Priv"
"ilegeExtBf\n\016io.milvus.grpcB\013CommonProtoP"
"\001Z1github.com/milvus-io/milvus-proto/go-"
"api/commonpb\240\001\001\252\002\016IO.Milvus.Grpcb\006proto3"
"dexs\030\004 \001(\005\"2\n\014SegmentStats\022\021\n\tSegmentID\030"
"\001 \001(\003\022\017\n\007NumRows\030\002 \001(\003*\303\n\n\tErrorCode\022\013\n\007"
"Success\020\000\022\023\n\017UnexpectedError\020\001\022\021\n\rConnec"
"tFailed\020\002\022\024\n\020PermissionDenied\020\003\022\027\n\023Colle"
"ctionNotExists\020\004\022\023\n\017IllegalArgument\020\005\022\024\n"
"\020IllegalDimension\020\007\022\024\n\020IllegalIndexType\020"
"\010\022\031\n\025IllegalCollectionName\020\t\022\017\n\013IllegalT"
"OPK\020\n\022\024\n\020IllegalRowRecord\020\013\022\023\n\017IllegalVe"
"ctorID\020\014\022\027\n\023IllegalSearchResult\020\r\022\020\n\014Fil"
"eNotFound\020\016\022\016\n\nMetaFailed\020\017\022\017\n\013CacheFail"
"ed\020\020\022\026\n\022CannotCreateFolder\020\021\022\024\n\020CannotCr"
"eateFile\020\022\022\026\n\022CannotDeleteFolder\020\023\022\024\n\020Ca"
"nnotDeleteFile\020\024\022\023\n\017BuildIndexError\020\025\022\020\n"
"\014IllegalNLIST\020\026\022\025\n\021IllegalMetricType\020\027\022\017"
"\n\013OutOfMemory\020\030\022\021\n\rIndexNotExist\020\031\022\023\n\017Em"
"ptyCollection\020\032\022\033\n\027UpdateImportTaskFailu"
"re\020\033\022\032\n\026CollectionNameNotFound\020\034\022\033\n\027Crea"
"teCredentialFailure\020\035\022\033\n\027UpdateCredentia"
"lFailure\020\036\022\033\n\027DeleteCredentialFailure\020\037\022"
"\030\n\024GetCredentialFailure\020 \022\030\n\024ListCredUse"
"rsFailure\020!\022\022\n\016GetUserFailure\020\"\022\025\n\021Creat"
"eRoleFailure\020#\022\023\n\017DropRoleFailure\020$\022\032\n\026O"
"perateUserRoleFailure\020%\022\025\n\021SelectRoleFai"
"lure\020&\022\025\n\021SelectUserFailure\020\'\022\031\n\025SelectR"
"esourceFailure\020(\022\033\n\027OperatePrivilegeFail"
"ure\020)\022\026\n\022SelectGrantFailure\020*\022!\n\035Refresh"
"PolicyInfoCacheFailure\020+\022\025\n\021ListPolicyFa"
"ilure\020,\022\022\n\016NotShardLeader\020-\022\026\n\022NoReplica"
"Available\020.\022\023\n\017SegmentNotFound\020/\022\r\n\tForc"
"eDeny\0200\022\r\n\tRateLimit\0201\022\022\n\016NodeIDNotMatch"
"\0202\022\024\n\020UpsertAutoIDTrue\0203\022\034\n\030Insufficient"
"MemoryToLoad\0204\022\030\n\024MemoryQuotaExhausted\0205"
"\022\026\n\022DiskQuotaExhausted\0206\022\025\n\021TimeTickLong"
"Delay\0207\022\021\n\rNotReadyServe\0208\022\033\n\027NotReadyCo"
"ordActivating\0209\022\017\n\013DataCoordNA\020d\022\022\n\rDDRe"
"questRace\020\350\007*c\n\nIndexState\022\022\n\016IndexState"
"None\020\000\022\014\n\010Unissued\020\001\022\016\n\nInProgress\020\002\022\014\n\010"
"Finished\020\003\022\n\n\006Failed\020\004\022\t\n\005Retry\020\005*\202\001\n\014Se"
"gmentState\022\024\n\020SegmentStateNone\020\000\022\014\n\010NotE"
"xist\020\001\022\013\n\007Growing\020\002\022\n\n\006Sealed\020\003\022\013\n\007Flush"
"ed\020\004\022\014\n\010Flushing\020\005\022\013\n\007Dropped\020\006\022\r\n\tImpor"
"ting\020\007*>\n\017PlaceholderType\022\010\n\004None\020\000\022\020\n\014B"
"inaryVector\020d\022\017\n\013FloatVector\020e*\300\016\n\007MsgTy"
"pe\022\r\n\tUndefined\020\000\022\024\n\020CreateCollection\020d\022"
"\022\n\016DropCollection\020e\022\021\n\rHasCollection\020f\022\026"
"\n\022DescribeCollection\020g\022\023\n\017ShowCollection"
"s\020h\022\024\n\020GetSystemConfigs\020i\022\022\n\016LoadCollect"
"ion\020j\022\025\n\021ReleaseCollection\020k\022\017\n\013CreateAl"
"ias\020l\022\r\n\tDropAlias\020m\022\016\n\nAlterAlias\020n\022\023\n\017"
"AlterCollection\020o\022\024\n\020RenameCollection\020p\022"
"\024\n\017CreatePartition\020\310\001\022\022\n\rDropPartition\020\311"
"\001\022\021\n\014HasPartition\020\312\001\022\026\n\021DescribePartitio"
"n\020\313\001\022\023\n\016ShowPartitions\020\314\001\022\023\n\016LoadPartiti"
"ons\020\315\001\022\026\n\021ReleasePartitions\020\316\001\022\021\n\014ShowSe"
"gments\020\372\001\022\024\n\017DescribeSegment\020\373\001\022\021\n\014LoadS"
"egments\020\374\001\022\024\n\017ReleaseSegments\020\375\001\022\024\n\017Hand"
"offSegments\020\376\001\022\030\n\023LoadBalanceSegments\020\377\001"
"\022\025\n\020DescribeSegments\020\200\002\022\020\n\013CreateIndex\020\254"
"\002\022\022\n\rDescribeIndex\020\255\002\022\016\n\tDropIndex\020\256\002\022\013\n"
"\006Insert\020\220\003\022\013\n\006Delete\020\221\003\022\n\n\005Flush\020\222\003\022\027\n\022R"
"esendSegmentStats\020\223\003\022\013\n\006Upsert\020\224\003\022\013\n\006Sea"
"rch\020\364\003\022\021\n\014SearchResult\020\365\003\022\022\n\rGetIndexSta"
"te\020\366\003\022\032\n\025GetIndexBuildProgress\020\367\003\022\034\n\027Get"
"CollectionStatistics\020\370\003\022\033\n\026GetPartitionS"
"tatistics\020\371\003\022\r\n\010Retrieve\020\372\003\022\023\n\016RetrieveR"
"esult\020\373\003\022\024\n\017WatchDmChannels\020\374\003\022\025\n\020Remove"
"DmChannels\020\375\003\022\027\n\022WatchQueryChannels\020\376\003\022\030"
"\n\023RemoveQueryChannels\020\377\003\022\035\n\030SealedSegmen"
"tsChangeInfo\020\200\004\022\027\n\022WatchDeltaChannels\020\201\004"
"\022\024\n\017GetShardLeaders\020\202\004\022\020\n\013GetReplicas\020\203\004"
"\022\023\n\016UnsubDmChannel\020\204\004\022\024\n\017GetDistribution"
"\020\205\004\022\025\n\020SyncDistribution\020\206\004\022\020\n\013SegmentInf"
"o\020\330\004\022\017\n\nSystemInfo\020\331\004\022\024\n\017GetRecoveryInfo"
"\020\332\004\022\024\n\017GetSegmentState\020\333\004\022\r\n\010TimeTick\020\260\t"
"\022\023\n\016QueryNodeStats\020\261\t\022\016\n\tLoadIndex\020\262\t\022\016\n"
"\tRequestID\020\263\t\022\017\n\nRequestTSO\020\264\t\022\024\n\017Alloca"
"teSegment\020\265\t\022\026\n\021SegmentStatistics\020\266\t\022\025\n\020"
"SegmentFlushDone\020\267\t\022\017\n\nDataNodeTt\020\270\t\022\025\n\020"
"CreateCredential\020\334\013\022\022\n\rGetCredential\020\335\013\022"
"\025\n\020DeleteCredential\020\336\013\022\025\n\020UpdateCredenti"
"al\020\337\013\022\026\n\021ListCredUsernames\020\340\013\022\017\n\nCreateR"
"ole\020\300\014\022\r\n\010DropRole\020\301\014\022\024\n\017OperateUserRole"
"\020\302\014\022\017\n\nSelectRole\020\303\014\022\017\n\nSelectUser\020\304\014\022\023\n"
"\016SelectResource\020\305\014\022\025\n\020OperatePrivilege\020\306"
"\014\022\020\n\013SelectGrant\020\307\014\022\033\n\026RefreshPolicyInfo"
"Cache\020\310\014\022\017\n\nListPolicy\020\311\014\022\030\n\023CreateResou"
"rceGroup\020\244\r\022\026\n\021DropResourceGroup\020\245\r\022\027\n\022L"
"istResourceGroups\020\246\r\022\032\n\025DescribeResource"
"Group\020\247\r\022\021\n\014TransferNode\020\250\r\022\024\n\017TransferR"
"eplica\020\251\r*\"\n\007DslType\022\007\n\003Dsl\020\000\022\016\n\nBoolExp"
"rV1\020\001*B\n\017CompactionState\022\021\n\rUndefiedStat"
"e\020\000\022\r\n\tExecuting\020\001\022\r\n\tCompleted\020\002*X\n\020Con"
"sistencyLevel\022\n\n\006Strong\020\000\022\013\n\007Session\020\001\022\013"
"\n\007Bounded\020\002\022\016\n\nEventually\020\003\022\016\n\nCustomize"
"d\020\004*\236\001\n\013ImportState\022\021\n\rImportPending\020\000\022\020"
"\n\014ImportFailed\020\001\022\021\n\rImportStarted\020\002\022\023\n\017I"
"mportPersisted\020\005\022\021\n\rImportFlushed\020\010\022\023\n\017I"
"mportCompleted\020\006\022\032\n\026ImportFailedAndClean"
"ed\020\007*2\n\nObjectType\022\016\n\nCollection\020\000\022\n\n\006Gl"
"obal\020\001\022\010\n\004User\020\002*\227\007\n\017ObjectPrivilege\022\020\n\014"
"PrivilegeAll\020\000\022\035\n\031PrivilegeCreateCollect"
"ion\020\001\022\033\n\027PrivilegeDropCollection\020\002\022\037\n\033Pr"
"ivilegeDescribeCollection\020\003\022\034\n\030Privilege"
"ShowCollections\020\004\022\021\n\rPrivilegeLoad\020\005\022\024\n\020"
"PrivilegeRelease\020\006\022\027\n\023PrivilegeCompactio"
"n\020\007\022\023\n\017PrivilegeInsert\020\010\022\023\n\017PrivilegeDel"
"ete\020\t\022\032\n\026PrivilegeGetStatistics\020\n\022\030\n\024Pri"
"vilegeCreateIndex\020\013\022\030\n\024PrivilegeIndexDet"
"ail\020\014\022\026\n\022PrivilegeDropIndex\020\r\022\023\n\017Privile"
"geSearch\020\016\022\022\n\016PrivilegeFlush\020\017\022\022\n\016Privil"
"egeQuery\020\020\022\030\n\024PrivilegeLoadBalance\020\021\022\023\n\017"
"PrivilegeImport\020\022\022\034\n\030PrivilegeCreateOwne"
"rship\020\023\022\027\n\023PrivilegeUpdateUser\020\024\022\032\n\026Priv"
"ilegeDropOwnership\020\025\022\034\n\030PrivilegeSelectO"
"wnership\020\026\022\034\n\030PrivilegeManageOwnership\020\027"
"\022\027\n\023PrivilegeSelectUser\020\030\022\023\n\017PrivilegeUp"
"sert\020\031\022 \n\034PrivilegeCreateResourceGroup\020\032"
"\022\036\n\032PrivilegeDropResourceGroup\020\033\022\"\n\036Priv"
"ilegeDescribeResourceGroup\020\034\022\037\n\033Privileg"
"eListResourceGroups\020\035\022\031\n\025PrivilegeTransf"
"erNode\020\036\022\034\n\030PrivilegeTransferReplica\020\037\022\037"
"\n\033PrivilegeGetLoadingProgress\020 \022\031\n\025Privi"
"legeGetLoadState\020!*S\n\tStateCode\022\020\n\014Initi"
"alizing\020\000\022\013\n\007Healthy\020\001\022\014\n\010Abnormal\020\002\022\013\n\007"
"StandBy\020\003\022\014\n\010Stopping\020\004*c\n\tLoadState\022\025\n\021"
"LoadStateNotExist\020\000\022\024\n\020LoadStateNotLoad\020"
"\001\022\024\n\020LoadStateLoading\020\002\022\023\n\017LoadStateLoad"
"ed\020\003:^\n\021privilege_ext_obj\022\037.google.proto"
"buf.MessageOptions\030\351\007 \001(\0132!.milvus.proto"
".common.PrivilegeExtBf\n\016io.milvus.grpcB\013"
"CommonProtoP\001Z1github.com/milvus-io/milv"
"us-proto/go-api/commonpb\240\001\001\252\002\016IO.Milvus."
"Grpcb\006proto3"
;
static const ::_pbi::DescriptorTable* const descriptor_table_common_2eproto_deps[1] = {
&::descriptor_table_google_2fprotobuf_2fdescriptor_2eproto,
};
static ::_pbi::once_flag descriptor_table_common_2eproto_once;
const ::_pbi::DescriptorTable descriptor_table_common_2eproto = {
false, false, 6160, descriptor_table_protodef_common_2eproto,
false, false, 6212, descriptor_table_protodef_common_2eproto,
"common.proto",
&descriptor_table_common_2eproto_once, descriptor_table_common_2eproto_deps, 1, 11,
&descriptor_table_common_2eproto_once, descriptor_table_common_2eproto_deps, 1, 12,
schemas, file_default_instances, TableStruct_common_2eproto::offsets,
file_level_metadata_common_2eproto, file_level_enum_descriptors_common_2eproto,
file_level_service_descriptors_common_2eproto,
@ -3472,6 +3498,217 @@ void PrivilegeExt::InternalSwap(PrivilegeExt* other) {
&descriptor_table_common_2eproto_getter, &descriptor_table_common_2eproto_once,
file_level_metadata_common_2eproto[10]);
}
// ===================================================================
class SegmentStats::_Internal {
public:
};
SegmentStats::SegmentStats(::PROTOBUF_NAMESPACE_ID::Arena* arena,
bool is_message_owned)
: ::PROTOBUF_NAMESPACE_ID::Message(arena, is_message_owned) {
SharedCtor(arena, is_message_owned);
// @@protoc_insertion_point(arena_constructor:milvus.proto.common.SegmentStats)
}
SegmentStats::SegmentStats(const SegmentStats& from)
: ::PROTOBUF_NAMESPACE_ID::Message() {
SegmentStats* const _this = this; (void)_this;
new (&_impl_) Impl_{
decltype(_impl_.segmentid_){}
, decltype(_impl_.numrows_){}
, /*decltype(_impl_._cached_size_)*/{}};
_internal_metadata_.MergeFrom<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(from._internal_metadata_);
::memcpy(&_impl_.segmentid_, &from._impl_.segmentid_,
static_cast<size_t>(reinterpret_cast<char*>(&_impl_.numrows_) -
reinterpret_cast<char*>(&_impl_.segmentid_)) + sizeof(_impl_.numrows_));
// @@protoc_insertion_point(copy_constructor:milvus.proto.common.SegmentStats)
}
inline void SegmentStats::SharedCtor(
::_pb::Arena* arena, bool is_message_owned) {
(void)arena;
(void)is_message_owned;
new (&_impl_) Impl_{
decltype(_impl_.segmentid_){int64_t{0}}
, decltype(_impl_.numrows_){int64_t{0}}
, /*decltype(_impl_._cached_size_)*/{}
};
}
SegmentStats::~SegmentStats() {
// @@protoc_insertion_point(destructor:milvus.proto.common.SegmentStats)
if (auto *arena = _internal_metadata_.DeleteReturnArena<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>()) {
(void)arena;
return;
}
SharedDtor();
}
inline void SegmentStats::SharedDtor() {
GOOGLE_DCHECK(GetArenaForAllocation() == nullptr);
}
void SegmentStats::SetCachedSize(int size) const {
_impl_._cached_size_.Set(size);
}
void SegmentStats::Clear() {
// @@protoc_insertion_point(message_clear_start:milvus.proto.common.SegmentStats)
uint32_t cached_has_bits = 0;
// Prevent compiler warnings about cached_has_bits being unused
(void) cached_has_bits;
::memset(&_impl_.segmentid_, 0, static_cast<size_t>(
reinterpret_cast<char*>(&_impl_.numrows_) -
reinterpret_cast<char*>(&_impl_.segmentid_)) + sizeof(_impl_.numrows_));
_internal_metadata_.Clear<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>();
}
const char* SegmentStats::_InternalParse(const char* ptr, ::_pbi::ParseContext* ctx) {
#define CHK_(x) if (PROTOBUF_PREDICT_FALSE(!(x))) goto failure
while (!ctx->Done(&ptr)) {
uint32_t tag;
ptr = ::_pbi::ReadTag(ptr, &tag);
switch (tag >> 3) {
// int64 SegmentID = 1;
case 1:
if (PROTOBUF_PREDICT_TRUE(static_cast<uint8_t>(tag) == 8)) {
_impl_.segmentid_ = ::PROTOBUF_NAMESPACE_ID::internal::ReadVarint64(&ptr);
CHK_(ptr);
} else
goto handle_unusual;
continue;
// int64 NumRows = 2;
case 2:
if (PROTOBUF_PREDICT_TRUE(static_cast<uint8_t>(tag) == 16)) {
_impl_.numrows_ = ::PROTOBUF_NAMESPACE_ID::internal::ReadVarint64(&ptr);
CHK_(ptr);
} else
goto handle_unusual;
continue;
default:
goto handle_unusual;
} // switch
handle_unusual:
if ((tag == 0) || ((tag & 7) == 4)) {
CHK_(ptr);
ctx->SetLastTag(tag);
goto message_done;
}
ptr = UnknownFieldParse(
tag,
_internal_metadata_.mutable_unknown_fields<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(),
ptr, ctx);
CHK_(ptr != nullptr);
} // while
message_done:
return ptr;
failure:
ptr = nullptr;
goto message_done;
#undef CHK_
}
uint8_t* SegmentStats::_InternalSerialize(
uint8_t* target, ::PROTOBUF_NAMESPACE_ID::io::EpsCopyOutputStream* stream) const {
// @@protoc_insertion_point(serialize_to_array_start:milvus.proto.common.SegmentStats)
uint32_t cached_has_bits = 0;
(void) cached_has_bits;
// int64 SegmentID = 1;
if (this->_internal_segmentid() != 0) {
target = stream->EnsureSpace(target);
target = ::_pbi::WireFormatLite::WriteInt64ToArray(1, this->_internal_segmentid(), target);
}
// int64 NumRows = 2;
if (this->_internal_numrows() != 0) {
target = stream->EnsureSpace(target);
target = ::_pbi::WireFormatLite::WriteInt64ToArray(2, this->_internal_numrows(), target);
}
if (PROTOBUF_PREDICT_FALSE(_internal_metadata_.have_unknown_fields())) {
target = ::_pbi::WireFormat::InternalSerializeUnknownFieldsToArray(
_internal_metadata_.unknown_fields<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(::PROTOBUF_NAMESPACE_ID::UnknownFieldSet::default_instance), target, stream);
}
// @@protoc_insertion_point(serialize_to_array_end:milvus.proto.common.SegmentStats)
return target;
}
size_t SegmentStats::ByteSizeLong() const {
// @@protoc_insertion_point(message_byte_size_start:milvus.proto.common.SegmentStats)
size_t total_size = 0;
uint32_t cached_has_bits = 0;
// Prevent compiler warnings about cached_has_bits being unused
(void) cached_has_bits;
// int64 SegmentID = 1;
if (this->_internal_segmentid() != 0) {
total_size += ::_pbi::WireFormatLite::Int64SizePlusOne(this->_internal_segmentid());
}
// int64 NumRows = 2;
if (this->_internal_numrows() != 0) {
total_size += ::_pbi::WireFormatLite::Int64SizePlusOne(this->_internal_numrows());
}
return MaybeComputeUnknownFieldsSize(total_size, &_impl_._cached_size_);
}
const ::PROTOBUF_NAMESPACE_ID::Message::ClassData SegmentStats::_class_data_ = {
::PROTOBUF_NAMESPACE_ID::Message::CopyWithSourceCheck,
SegmentStats::MergeImpl
};
const ::PROTOBUF_NAMESPACE_ID::Message::ClassData*SegmentStats::GetClassData() const { return &_class_data_; }
void SegmentStats::MergeImpl(::PROTOBUF_NAMESPACE_ID::Message& to_msg, const ::PROTOBUF_NAMESPACE_ID::Message& from_msg) {
auto* const _this = static_cast<SegmentStats*>(&to_msg);
auto& from = static_cast<const SegmentStats&>(from_msg);
// @@protoc_insertion_point(class_specific_merge_from_start:milvus.proto.common.SegmentStats)
GOOGLE_DCHECK_NE(&from, _this);
uint32_t cached_has_bits = 0;
(void) cached_has_bits;
if (from._internal_segmentid() != 0) {
_this->_internal_set_segmentid(from._internal_segmentid());
}
if (from._internal_numrows() != 0) {
_this->_internal_set_numrows(from._internal_numrows());
}
_this->_internal_metadata_.MergeFrom<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(from._internal_metadata_);
}
void SegmentStats::CopyFrom(const SegmentStats& from) {
// @@protoc_insertion_point(class_specific_copy_from_start:milvus.proto.common.SegmentStats)
if (&from == this) return;
Clear();
MergeFrom(from);
}
bool SegmentStats::IsInitialized() const {
return true;
}
void SegmentStats::InternalSwap(SegmentStats* other) {
using std::swap;
_internal_metadata_.InternalSwap(&other->_internal_metadata_);
::PROTOBUF_NAMESPACE_ID::internal::memswap<
PROTOBUF_FIELD_OFFSET(SegmentStats, _impl_.numrows_)
+ sizeof(SegmentStats::_impl_.numrows_)
- PROTOBUF_FIELD_OFFSET(SegmentStats, _impl_.segmentid_)>(
reinterpret_cast<char*>(&_impl_.segmentid_),
reinterpret_cast<char*>(&other->_impl_.segmentid_));
}
::PROTOBUF_NAMESPACE_ID::Metadata SegmentStats::GetMetadata() const {
return ::_pbi::AssignDescriptors(
&descriptor_table_common_2eproto_getter, &descriptor_table_common_2eproto_once,
file_level_metadata_common_2eproto[11]);
}
PROTOBUF_ATTRIBUTE_INIT_PRIORITY2 ::PROTOBUF_NAMESPACE_ID::internal::ExtensionIdentifier< ::PROTOBUF_NAMESPACE_ID::MessageOptions,
::PROTOBUF_NAMESPACE_ID::internal::MessageTypeTraits< ::milvus::proto::common::PrivilegeExt >, 11, false>
privilege_ext_obj(kPrivilegeExtObjFieldNumber, ::milvus::proto::common::PrivilegeExt::default_instance(), nullptr);
@ -3525,6 +3762,10 @@ template<> PROTOBUF_NOINLINE ::milvus::proto::common::PrivilegeExt*
Arena::CreateMaybeMessage< ::milvus::proto::common::PrivilegeExt >(Arena* arena) {
return Arena::CreateMessageInternal< ::milvus::proto::common::PrivilegeExt >(arena);
}
template<> PROTOBUF_NOINLINE ::milvus::proto::common::SegmentStats*
Arena::CreateMaybeMessage< ::milvus::proto::common::SegmentStats >(Arena* arena) {
return Arena::CreateMessageInternal< ::milvus::proto::common::SegmentStats >(arena);
}
PROTOBUF_NAMESPACE_CLOSE
// @@protoc_insertion_point(global_scope)

209
internal/core/src/pb/common.pb.h
View File

@ -79,6 +79,9 @@ extern PlaceholderValueDefaultTypeInternal _PlaceholderValue_default_instance_;
class PrivilegeExt;
struct PrivilegeExtDefaultTypeInternal;
extern PrivilegeExtDefaultTypeInternal _PrivilegeExt_default_instance_;
class SegmentStats;
struct SegmentStatsDefaultTypeInternal;
extern SegmentStatsDefaultTypeInternal _SegmentStats_default_instance_;
class Status;
struct StatusDefaultTypeInternal;
extern StatusDefaultTypeInternal _Status_default_instance_;
@ -96,6 +99,7 @@ template<> ::milvus::proto::common::MsgHeader* Arena::CreateMaybeMessage<::milvu
template<> ::milvus::proto::common::PlaceholderGroup* Arena::CreateMaybeMessage<::milvus::proto::common::PlaceholderGroup>(Arena*);
template<> ::milvus::proto::common::PlaceholderValue* Arena::CreateMaybeMessage<::milvus::proto::common::PlaceholderValue>(Arena*);
template<> ::milvus::proto::common::PrivilegeExt* Arena::CreateMaybeMessage<::milvus::proto::common::PrivilegeExt>(Arena*);
template<> ::milvus::proto::common::SegmentStats* Arena::CreateMaybeMessage<::milvus::proto::common::SegmentStats>(Arena*);
template<> ::milvus::proto::common::Status* Arena::CreateMaybeMessage<::milvus::proto::common::Status>(Arena*);
PROTOBUF_NAMESPACE_CLOSE
namespace milvus {
@ -2498,6 +2502,165 @@ class PrivilegeExt final :
union { Impl_ _impl_; };
friend struct ::TableStruct_common_2eproto;
};
// -------------------------------------------------------------------
class SegmentStats final :
public ::PROTOBUF_NAMESPACE_ID::Message /* @@protoc_insertion_point(class_definition:milvus.proto.common.SegmentStats) */ {
public:
inline SegmentStats() : SegmentStats(nullptr) {}
~SegmentStats() override;
explicit PROTOBUF_CONSTEXPR SegmentStats(::PROTOBUF_NAMESPACE_ID::internal::ConstantInitialized);
SegmentStats(const SegmentStats& from);
SegmentStats(SegmentStats&& from) noexcept
: SegmentStats() {
*this = ::std::move(from);
}
inline SegmentStats& operator=(const SegmentStats& from) {
CopyFrom(from);
return *this;
}
inline SegmentStats& operator=(SegmentStats&& from) noexcept {
if (this == &from) return *this;
if (GetOwningArena() == from.GetOwningArena()
#ifdef PROTOBUF_FORCE_COPY_IN_MOVE
&& GetOwningArena() != nullptr
#endif // !PROTOBUF_FORCE_COPY_IN_MOVE
) {
InternalSwap(&from);
} else {
CopyFrom(from);
}
return *this;
}
static const ::PROTOBUF_NAMESPACE_ID::Descriptor* descriptor() {
return GetDescriptor();
}
static const ::PROTOBUF_NAMESPACE_ID::Descriptor* GetDescriptor() {
return default_instance().GetMetadata().descriptor;
}
static const ::PROTOBUF_NAMESPACE_ID::Reflection* GetReflection() {
return default_instance().GetMetadata().reflection;
}
static const SegmentStats& default_instance() {
return *internal_default_instance();
}
static inline const SegmentStats* internal_default_instance() {
return reinterpret_cast<const SegmentStats*>(
&_SegmentStats_default_instance_);
}
static constexpr int kIndexInFileMessages =
11;
friend void swap(SegmentStats& a, SegmentStats& b) {
a.Swap(&b);
}
inline void Swap(SegmentStats* other) {
if (other == this) return;
#ifdef PROTOBUF_FORCE_COPY_IN_SWAP
if (GetOwningArena() != nullptr &&
GetOwningArena() == other->GetOwningArena()) {
#else // PROTOBUF_FORCE_COPY_IN_SWAP
if (GetOwningArena() == other->GetOwningArena()) {
#endif // !PROTOBUF_FORCE_COPY_IN_SWAP
InternalSwap(other);
} else {
::PROTOBUF_NAMESPACE_ID::internal::GenericSwap(this, other);
}
}
void UnsafeArenaSwap(SegmentStats* other) {
if (other == this) return;
GOOGLE_DCHECK(GetOwningArena() == other->GetOwningArena());
InternalSwap(other);
}
// implements Message ----------------------------------------------
SegmentStats* New(::PROTOBUF_NAMESPACE_ID::Arena* arena = nullptr) const final {
return CreateMaybeMessage<SegmentStats>(arena);
}
using ::PROTOBUF_NAMESPACE_ID::Message::CopyFrom;
void CopyFrom(const SegmentStats& from);
using ::PROTOBUF_NAMESPACE_ID::Message::MergeFrom;
void MergeFrom( const SegmentStats& from) {
SegmentStats::MergeImpl(*this, from);
}
private:
static void MergeImpl(::PROTOBUF_NAMESPACE_ID::Message& to_msg, const ::PROTOBUF_NAMESPACE_ID::Message& from_msg);
public:
PROTOBUF_ATTRIBUTE_REINITIALIZES void Clear() final;
bool IsInitialized() const final;
size_t ByteSizeLong() const final;
const char* _InternalParse(const char* ptr, ::PROTOBUF_NAMESPACE_ID::internal::ParseContext* ctx) final;
uint8_t* _InternalSerialize(
uint8_t* target, ::PROTOBUF_NAMESPACE_ID::io::EpsCopyOutputStream* stream) const final;
int GetCachedSize() const final { return _impl_._cached_size_.Get(); }
private:
void SharedCtor(::PROTOBUF_NAMESPACE_ID::Arena* arena, bool is_message_owned);
void SharedDtor();
void SetCachedSize(int size) const final;
void InternalSwap(SegmentStats* other);
private:
friend class ::PROTOBUF_NAMESPACE_ID::internal::AnyMetadata;
static ::PROTOBUF_NAMESPACE_ID::StringPiece FullMessageName() {
return "milvus.proto.common.SegmentStats";
}
protected:
explicit SegmentStats(::PROTOBUF_NAMESPACE_ID::Arena* arena,
bool is_message_owned = false);
public:
static const ClassData _class_data_;
const ::PROTOBUF_NAMESPACE_ID::Message::ClassData*GetClassData() const final;
::PROTOBUF_NAMESPACE_ID::Metadata GetMetadata() const final;
// nested types ----------------------------------------------------
// accessors -------------------------------------------------------
enum : int {
kSegmentIDFieldNumber = 1,
kNumRowsFieldNumber = 2,
};
// int64 SegmentID = 1;
void clear_segmentid();
int64_t segmentid() const;
void set_segmentid(int64_t value);
private:
int64_t _internal_segmentid() const;
void _internal_set_segmentid(int64_t value);
public:
// int64 NumRows = 2;
void clear_numrows();
int64_t numrows() const;
void set_numrows(int64_t value);
private:
int64_t _internal_numrows() const;
void _internal_set_numrows(int64_t value);
public:
// @@protoc_insertion_point(class_scope:milvus.proto.common.SegmentStats)
private:
class _Internal;
template <typename T> friend class ::PROTOBUF_NAMESPACE_ID::Arena::InternalHelper;
typedef void InternalArenaConstructable_;
typedef void DestructorSkippable_;
struct Impl_ {
int64_t segmentid_;
int64_t numrows_;
mutable ::PROTOBUF_NAMESPACE_ID::internal::CachedSize _cached_size_;
};
union { Impl_ _impl_; };
friend struct ::TableStruct_common_2eproto;
};
// ===================================================================
static const int kPrivilegeExtObjFieldNumber = 1001;
@ -3538,6 +3701,50 @@ inline void PrivilegeExt::set_object_name_indexs(int32_t value) {
// @@protoc_insertion_point(field_set:milvus.proto.common.PrivilegeExt.object_name_indexs)
}
// -------------------------------------------------------------------
// SegmentStats
// int64 SegmentID = 1;
inline void SegmentStats::clear_segmentid() {
_impl_.segmentid_ = int64_t{0};
}
inline int64_t SegmentStats::_internal_segmentid() const {
return _impl_.segmentid_;
}
inline int64_t SegmentStats::segmentid() const {
// @@protoc_insertion_point(field_get:milvus.proto.common.SegmentStats.SegmentID)
return _internal_segmentid();
}
inline void SegmentStats::_internal_set_segmentid(int64_t value) {
_impl_.segmentid_ = value;
}
inline void SegmentStats::set_segmentid(int64_t value) {
_internal_set_segmentid(value);
// @@protoc_insertion_point(field_set:milvus.proto.common.SegmentStats.SegmentID)
}
// int64 NumRows = 2;
inline void SegmentStats::clear_numrows() {
_impl_.numrows_ = int64_t{0};
}
inline int64_t SegmentStats::_internal_numrows() const {
return _impl_.numrows_;
}
inline int64_t SegmentStats::numrows() const {
// @@protoc_insertion_point(field_get:milvus.proto.common.SegmentStats.NumRows)
return _internal_numrows();
}
inline void SegmentStats::_internal_set_numrows(int64_t value) {
_impl_.numrows_ = value;
}
inline void SegmentStats::set_numrows(int64_t value) {
_internal_set_numrows(value);
// @@protoc_insertion_point(field_set:milvus.proto.common.SegmentStats.NumRows)
}
#ifdef __GNUC__
#pragma GCC diagnostic pop
#endif // __GNUC__
@ -3561,6 +3768,8 @@ inline void PrivilegeExt::set_object_name_indexs(int32_t value) {
// -------------------------------------------------------------------
// -------------------------------------------------------------------
// @@protoc_insertion_point(namespace_scope)

19
internal/datacoord/garbage_collector_test.go
View File

@ -27,28 +27,25 @@ import (
"time"
"github.com/cockroachdb/errors"
"github.com/samber/lo"
"github.com/minio/minio-go/v7"
minio "github.com/minio/minio-go/v7"
"github.com/minio/minio-go/v7/pkg/credentials"
"github.com/samber/lo"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/suite"
"github.com/milvus-io/milvus/internal/proto/indexpb"
"github.com/milvus-io/milvus/internal/proto/internalpb"
"github.com/milvus-io/milvus/internal/mocks"
"github.com/stretchr/testify/mock"
"github.com/stretchr/testify/require"
"github.com/stretchr/testify/suite"
"github.com/milvus-io/milvus-proto/go-api/commonpb"
"github.com/milvus-io/milvus-proto/go-api/msgpb"
"github.com/milvus-io/milvus-proto/go-api/schemapb"
kvmocks "github.com/milvus-io/milvus/internal/kv/mocks"
"github.com/milvus-io/milvus/internal/metastore"
"github.com/milvus-io/milvus/internal/metastore/kv/datacoord"
catalogmocks "github.com/milvus-io/milvus/internal/metastore/mocks"
"github.com/milvus-io/milvus/internal/metastore/model"
"github.com/milvus-io/milvus/internal/mocks"
"github.com/milvus-io/milvus/internal/proto/datapb"
"github.com/milvus-io/milvus/internal/proto/indexpb"
"github.com/milvus-io/milvus/internal/storage"
"github.com/milvus-io/milvus/internal/util/funcutil"
)
@ -921,7 +918,7 @@ func TestGarbageCollector_clearETCD(t *testing.T) {
m := &meta{
catalog: catalog,
channelCPs: map[string]*internalpb.MsgPosition{
channelCPs: map[string]*msgpb.MsgPosition{
"dmlChannel": {
Timestamp: 1000,
},
@ -1044,7 +1041,7 @@ func TestGarbageCollector_clearETCD(t *testing.T) {
MaxRowNum: 65535,
DroppedAt: 0,
CompactionFrom: nil,
DmlPosition: &internalpb.MsgPosition{
DmlPosition: &msgpb.MsgPosition{
Timestamp: 1200,
},
},

25
internal/datacoord/handler.go
View File

@ -19,14 +19,15 @@ package datacoord
import (
"context"
"go.uber.org/zap"
"github.com/milvus-io/milvus-proto/go-api/commonpb"
"github.com/milvus-io/milvus-proto/go-api/msgpb"
"github.com/milvus-io/milvus/internal/log"
"github.com/milvus-io/milvus/internal/proto/datapb"
"github.com/milvus-io/milvus/internal/proto/internalpb"
"github.com/milvus-io/milvus/internal/util/funcutil"
"github.com/milvus-io/milvus/internal/util/tsoutil"
"github.com/milvus-io/milvus/internal/util/typeutil"
"go.uber.org/zap"
)
// Handler handles some channel method for ChannelManager
@ -172,8 +173,8 @@ func (h *ServerHandler) GetQueryVChanPositions(channel *channel, partitionID Uni
// getEarliestSegmentDMLPos returns the earliest dml position of segments,
// this is mainly for COMPATIBILITY with old version <=2.1.x
func (h *ServerHandler) getEarliestSegmentDMLPos(channel *channel, partitionID UniqueID) *internalpb.MsgPosition {
var minPos *internalpb.MsgPosition
func (h *ServerHandler) getEarliestSegmentDMLPos(channel *channel, partitionID UniqueID) *msgpb.MsgPosition {
var minPos *msgpb.MsgPosition
var minPosSegID int64
var minPosTs uint64
segments := h.s.meta.SelectSegments(func(s *SegmentInfo) bool {
@ -192,7 +193,7 @@ func (h *ServerHandler) getEarliestSegmentDMLPos(channel *channel, partitionID U
continue
}
var segmentPosition *internalpb.MsgPosition
var segmentPosition *msgpb.MsgPosition
if s.GetDmlPosition() != nil {
segmentPosition = s.GetDmlPosition()
} else {
@ -214,9 +215,9 @@ func (h *ServerHandler) getEarliestSegmentDMLPos(channel *channel, partitionID U
}
// getCollectionStartPos returns collection start position.
func (h *ServerHandler) getCollectionStartPos(channel *channel) *internalpb.MsgPosition {
func (h *ServerHandler) getCollectionStartPos(channel *channel) *msgpb.MsgPosition {
// use collection start position when segment position is not found
var startPosition *internalpb.MsgPosition
var startPosition *msgpb.MsgPosition
if channel.StartPositions == nil {
collection, err := h.GetCollection(h.s.ctx, channel.CollectionID)
if collection != nil && err == nil {
@ -244,8 +245,8 @@ func (h *ServerHandler) getCollectionStartPos(channel *channel) *internalpb.MsgP
// 2. Segments earliest dml position;
// 3. Collection start position;
// And would return if any position is valid.
func (h *ServerHandler) GetChannelSeekPosition(channel *channel, partitionID UniqueID) *internalpb.MsgPosition {
var seekPosition *internalpb.MsgPosition
func (h *ServerHandler) GetChannelSeekPosition(channel *channel, partitionID UniqueID) *msgpb.MsgPosition {
var seekPosition *msgpb.MsgPosition
seekPosition = h.s.meta.GetChannelCheckpoint(channel.Name)
if seekPosition != nil {
log.Info("channel seek position set from channel checkpoint meta",
@ -278,16 +279,16 @@ func (h *ServerHandler) GetChannelSeekPosition(channel *channel, partitionID Uni
return nil
}
func getCollectionStartPosition(channel string, collectionInfo *collectionInfo) *internalpb.MsgPosition {
func getCollectionStartPosition(channel string, collectionInfo *collectionInfo) *msgpb.MsgPosition {
return toMsgPosition(channel, collectionInfo.StartPositions)
}
func toMsgPosition(channel string, startPositions []*commonpb.KeyDataPair) *internalpb.MsgPosition {
func toMsgPosition(channel string, startPositions []*commonpb.KeyDataPair) *msgpb.MsgPosition {
for _, sp := range startPositions {
if sp.GetKey() != funcutil.ToPhysicalChannel(channel) {
continue
}
return &internalpb.MsgPosition{
return &msgpb.MsgPosition{
ChannelName: channel,
MsgID: sp.GetData(),
}

18
internal/datacoord/meta.go
View File

@ -30,6 +30,7 @@ import (
"golang.org/x/exp/maps"
"github.com/milvus-io/milvus-proto/go-api/commonpb"
"github.com/milvus-io/milvus-proto/go-api/msgpb"
"github.com/milvus-io/milvus-proto/go-api/schemapb"
"github.com/milvus-io/milvus/internal/common"
"github.com/milvus-io/milvus/internal/log"
@ -37,7 +38,6 @@ import (
"github.com/milvus-io/milvus/internal/metastore/model"
"github.com/milvus-io/milvus/internal/metrics"
"github.com/milvus-io/milvus/internal/proto/datapb"
"github.com/milvus-io/milvus/internal/proto/internalpb"
"github.com/milvus-io/milvus/internal/storage"
"github.com/milvus-io/milvus/internal/util/metautil"
"github.com/milvus-io/milvus/internal/util/segmentutil"
@ -50,9 +50,9 @@ type meta struct {
sync.RWMutex
ctx context.Context
catalog metastore.DataCoordCatalog
collections map[UniqueID]*collectionInfo // collection id to collection info
segments *SegmentsInfo // segment id to segment info
channelCPs map[string]*internalpb.MsgPosition // vChannel -> channel checkpoint/see position
collections map[UniqueID]*collectionInfo // collection id to collection info
segments *SegmentsInfo // segment id to segment info
channelCPs map[string]*msgpb.MsgPosition // vChannel -> channel checkpoint/see position
chunkManager storage.ChunkManager
// collectionIndexes records which indexes are on the collection
@ -85,7 +85,7 @@ func newMeta(ctx context.Context, catalog metastore.DataCoordCatalog, chunkManag
catalog: catalog,
collections: make(map[UniqueID]*collectionInfo),
segments: NewSegmentsInfo(),
channelCPs: make(map[string]*internalpb.MsgPosition),
channelCPs: make(map[string]*msgpb.MsgPosition),
chunkManager: chunkManager,
indexes: make(map[UniqueID]map[UniqueID]*model.Index),
buildID2SegmentIndex: make(map[UniqueID]*model.SegmentIndex),
@ -979,7 +979,7 @@ func (m *meta) PrepareCompleteCompactionMutation(compactionLogs []*datapb.Compac
}
}
var startPosition, dmlPosition *internalpb.MsgPosition
var startPosition, dmlPosition *msgpb.MsgPosition
for _, s := range modSegments {
if dmlPosition == nil ||
s.GetDmlPosition() != nil && s.GetDmlPosition().GetTimestamp() < dmlPosition.GetTimestamp() {
@ -1254,7 +1254,7 @@ func (m *meta) GetCompactionTo(segmentID int64) *SegmentInfo {
}
// UpdateChannelCheckpoint updates and saves channel checkpoint.
func (m *meta) UpdateChannelCheckpoint(vChannel string, pos *internalpb.MsgPosition) error {
func (m *meta) UpdateChannelCheckpoint(vChannel string, pos *msgpb.MsgPosition) error {
if pos == nil {
return fmt.Errorf("channelCP is nil, vChannel=%s", vChannel)
}
@ -1275,13 +1275,13 @@ func (m *meta) UpdateChannelCheckpoint(vChannel string, pos *internalpb.MsgPosit
return nil
}
func (m *meta) GetChannelCheckpoint(vChannel string) *internalpb.MsgPosition {
func (m *meta) GetChannelCheckpoint(vChannel string) *msgpb.MsgPosition {
m.RLock()
defer m.RUnlock()
if m.channelCPs[vChannel] == nil {
return nil
}
return proto.Clone(m.channelCPs[vChannel]).(*internalpb.MsgPosition)
return proto.Clone(m.channelCPs[vChannel]).(*msgpb.MsgPosition)
}
func (m *meta) DropChannelCheckpoint(vChannel string) error {

28
internal/datacoord/meta_test.go
View File

@ -22,20 +22,20 @@ import (
"testing"
"github.com/cockroachdb/errors"
"github.com/golang/protobuf/proto"
"github.com/samber/lo"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/mock"
"github.com/stretchr/testify/require"
"github.com/milvus-io/milvus-proto/go-api/commonpb"
"github.com/milvus-io/milvus-proto/go-api/msgpb"
"github.com/milvus-io/milvus/internal/common"
"github.com/milvus-io/milvus/internal/kv"
"github.com/milvus-io/milvus/internal/metastore/kv/datacoord"
"github.com/milvus-io/milvus/internal/metastore/model"
"github.com/milvus-io/milvus/internal/mocks"
"github.com/milvus-io/milvus/internal/proto/datapb"
"github.com/milvus-io/milvus/internal/proto/internalpb"
"github.com/samber/lo"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/mock"
"github.com/stretchr/testify/require"
)
func TestMetaReloadFromKV(t *testing.T) {
@ -68,7 +68,7 @@ func TestMetaReloadFromKV(t *testing.T) {
).Return([]*datapb.SegmentInfo{}, nil)
catalog.On("ListChannelCheckpoint",
mock.Anything,
).Return(map[string]*internalpb.MsgPosition{}, nil)
).Return(map[string]*msgpb.MsgPosition{}, nil)
catalog.On("ListIndexes",
mock.Anything,
).Return(nil, errors.New("error"))
@ -83,7 +83,7 @@ func TestMetaReloadFromKV(t *testing.T) {
).Return([]*datapb.SegmentInfo{}, nil)
catalog.On("ListChannelCheckpoint",
mock.Anything,
).Return(map[string]*internalpb.MsgPosition{}, nil)
).Return(map[string]*msgpb.MsgPosition{}, nil)
catalog.On("ListIndexes",
mock.Anything,
).Return([]*model.Index{}, nil)
@ -109,7 +109,7 @@ func TestMetaReloadFromKV(t *testing.T) {
catalog.On("ListChannelCheckpoint",
mock.Anything,
).Return(map[string]*internalpb.MsgPosition{
).Return(map[string]*msgpb.MsgPosition{
"ch": {
ChannelName: "cn",
MsgID: []byte{},
@ -439,13 +439,13 @@ func TestUpdateFlushSegmentsInfo(t *testing.T) {
err = meta.UpdateFlushSegmentsInfo(1, true, false, true, []*datapb.FieldBinlog{getFieldBinlogPathsWithEntry(1, 10, getInsertLogPath("binlog1", 1))},
[]*datapb.FieldBinlog{getFieldBinlogPaths(1, getStatsLogPath("statslog1", 1))},
[]*datapb.FieldBinlog{{Binlogs: []*datapb.Binlog{{EntriesNum: 1, TimestampFrom: 100, TimestampTo: 200, LogSize: 1000, LogPath: getDeltaLogPath("deltalog1", 1)}}}},
[]*datapb.CheckPoint{{SegmentID: 1, NumOfRows: 10}}, []*datapb.SegmentStartPosition{{SegmentID: 1, StartPosition: &internalpb.MsgPosition{MsgID: []byte{1, 2, 3}}}})
[]*datapb.CheckPoint{{SegmentID: 1, NumOfRows: 10}}, []*datapb.SegmentStartPosition{{SegmentID: 1, StartPosition: &msgpb.MsgPosition{MsgID: []byte{1, 2, 3}}}})
assert.Nil(t, err)
updated := meta.GetHealthySegment(1)
expected := &SegmentInfo{SegmentInfo: &datapb.SegmentInfo{
ID: 1, State: commonpb.SegmentState_Flushing, NumOfRows: 10,
StartPosition: &internalpb.MsgPosition{MsgID: []byte{1, 2, 3}},
StartPosition: &msgpb.MsgPosition{MsgID: []byte{1, 2, 3}},
Binlogs: []*datapb.FieldBinlog{getFieldBinlogPaths(1, "binlog0", "binlog1")},
Statslogs: []*datapb.FieldBinlog{getFieldBinlogPaths(1, "statslog0", "statslog1")},
Deltalogs: []*datapb.FieldBinlog{{Binlogs: []*datapb.Binlog{{EntriesNum: 1, TimestampFrom: 100, TimestampTo: 200, LogSize: 1000}}}},
@ -483,7 +483,7 @@ func TestUpdateFlushSegmentsInfo(t *testing.T) {
err = meta.UpdateFlushSegmentsInfo(1, false, false, false, nil, nil, nil, []*datapb.CheckPoint{{SegmentID: 2, NumOfRows: 10}},
[]*datapb.SegmentStartPosition{{SegmentID: 2, StartPosition: &internalpb.MsgPosition{MsgID: []byte{1, 2, 3}}}})
[]*datapb.SegmentStartPosition{{SegmentID: 2, StartPosition: &msgpb.MsgPosition{MsgID: []byte{1, 2, 3}}}})
assert.Nil(t, err)
assert.Nil(t, meta.GetHealthySegment(2))
})
@ -507,7 +507,7 @@ func TestUpdateFlushSegmentsInfo(t *testing.T) {
err = meta.UpdateFlushSegmentsInfo(1, true, false, false, []*datapb.FieldBinlog{getFieldBinlogPaths(1, getInsertLogPath("binlog", 1))},
[]*datapb.FieldBinlog{getFieldBinlogPaths(1, getInsertLogPath("statslog", 1))},
[]*datapb.FieldBinlog{{Binlogs: []*datapb.Binlog{{EntriesNum: 1, TimestampFrom: 100, TimestampTo: 200, LogSize: 1000, LogPath: getDeltaLogPath("deltalog", 1)}}}},
[]*datapb.CheckPoint{{SegmentID: 1, NumOfRows: 10}}, []*datapb.SegmentStartPosition{{SegmentID: 1, StartPosition: &internalpb.MsgPosition{MsgID: []byte{1, 2, 3}}}})
[]*datapb.CheckPoint{{SegmentID: 1, NumOfRows: 10}}, []*datapb.SegmentStartPosition{{SegmentID: 1, StartPosition: &msgpb.MsgPosition{MsgID: []byte{1, 2, 3}}}})
assert.NotNil(t, err)
assert.Equal(t, "mocked fail", err.Error())
segmentInfo = meta.GetHealthySegment(1)
@ -899,7 +899,7 @@ func TestChannelCP(t *testing.T) {
mockVChannel := "fake-by-dev-rootcoord-dml-1-testchannelcp-v0"
mockPChannel := "fake-by-dev-rootcoord-dml-1"
pos := &internalpb.MsgPosition{
pos := &msgpb.MsgPosition{
ChannelName: mockPChannel,
MsgID: []byte{},
Timestamp: 1000,

16
internal/datacoord/segment_info.go
View File

@ -19,14 +19,14 @@ package datacoord
import (
"time"
"github.com/milvus-io/milvus/internal/log"
"github.com/milvus-io/milvus/internal/metastore/model"
"github.com/golang/protobuf/proto"
"go.uber.org/zap"
"github.com/golang/protobuf/proto"
"github.com/milvus-io/milvus-proto/go-api/commonpb"
"github.com/milvus-io/milvus-proto/go-api/msgpb"
"github.com/milvus-io/milvus/internal/log"
"github.com/milvus-io/milvus/internal/metastore/model"
"github.com/milvus-io/milvus/internal/proto/datapb"
"github.com/milvus-io/milvus/internal/proto/internalpb"
)
// SegmentsInfo wraps a map, which maintains ID to SegmentInfo relation
@ -148,7 +148,7 @@ func (s *SegmentsInfo) SetIsImporting(segmentID UniqueID, isImporting bool) {
// SetDmlPosition sets DmlPosition info (checkpoint for recovery) for SegmentInfo with provided segmentID
// if SegmentInfo not found, do nothing
func (s *SegmentsInfo) SetDmlPosition(segmentID UniqueID, pos *internalpb.MsgPosition) {
func (s *SegmentsInfo) SetDmlPosition(segmentID UniqueID, pos *msgpb.MsgPosition) {
if segment, ok := s.segments[segmentID]; ok {
s.segments[segmentID] = segment.Clone(SetDmlPosition(pos))
}
@ -156,7 +156,7 @@ func (s *SegmentsInfo) SetDmlPosition(segmentID UniqueID, pos *internalpb.MsgPos
// SetStartPosition sets StartPosition info (recovery info when no checkout point found) for SegmentInfo with provided segmentID
// if SegmentInfo not found, do nothing
func (s *SegmentsInfo) SetStartPosition(segmentID UniqueID, pos *internalpb.MsgPosition) {
func (s *SegmentsInfo) SetStartPosition(segmentID UniqueID, pos *msgpb.MsgPosition) {
if segment, ok := s.segments[segmentID]; ok {
s.segments[segmentID] = segment.Clone(SetStartPosition(pos))
}
@ -301,14 +301,14 @@ func SetIsImporting(isImporting bool) SegmentInfoOption {
}
// SetDmlPosition is the option to set dml position for segment info
func SetDmlPosition(pos *internalpb.MsgPosition) SegmentInfoOption {
func SetDmlPosition(pos *msgpb.MsgPosition) SegmentInfoOption {
return func(segment *SegmentInfo) {
segment.DmlPosition = pos
}
}
// SetStartPosition is the option to set start position for segment info
func SetStartPosition(pos *internalpb.MsgPosition) SegmentInfoOption {
func SetStartPosition(pos *msgpb.MsgPosition) SegmentInfoOption {
return func(segment *SegmentInfo) {
segment.StartPosition = pos
}

7
internal/datacoord/server.go
View File

@ -26,9 +26,8 @@ import (
"syscall"
"time"
semver "github.com/blang/semver/v4"
"github.com/cockroachdb/errors"
"github.com/blang/semver/v4"
clientv3 "go.etcd.io/etcd/client/v3"
"go.uber.org/zap"
@ -81,7 +80,9 @@ type (
)
type dataNodeCreatorFunc func(ctx context.Context, addr string) (types.DataNode, error)
type indexNodeCreatorFunc func(ctx context.Context, addr string) (types.IndexNode, error)
type rootCoordCreatorFunc func(ctx context.Context, metaRootPath string, etcdClient *clientv3.Client) (types.RootCoord, error)
// makes sure Server implements `DataCoord`
@ -679,7 +680,7 @@ func (s *Server) handleTimetickMessage(ctx context.Context, ttMsg *msgstream.Dat
return nil
}
func (s *Server) updateSegmentStatistics(stats []*datapb.SegmentStats) {
func (s *Server) updateSegmentStatistics(stats []*commonpb.SegmentStats) {
for _, stat := range stats {
segment := s.meta.GetSegment(stat.GetSegmentID())
if segment == nil {

102
internal/datacoord/server_test.go
View File

@ -31,7 +31,6 @@ import (
"time"
"github.com/cockroachdb/errors"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/mock"
"github.com/stretchr/testify/require"
@ -40,6 +39,7 @@ import (
"github.com/milvus-io/milvus-proto/go-api/commonpb"
"github.com/milvus-io/milvus-proto/go-api/milvuspb"
"github.com/milvus-io/milvus-proto/go-api/msgpb"
"github.com/milvus-io/milvus-proto/go-api/schemapb"
"github.com/milvus-io/milvus/internal/common"
"github.com/milvus-io/milvus/internal/indexnode"
@ -371,7 +371,7 @@ func TestGetSegmentStates(t *testing.T) {
InsertChannel: "c1",
NumOfRows: 0,
State: commonpb.SegmentState_Growing,
StartPosition: &internalpb.MsgPosition{
StartPosition: &msgpb.MsgPosition{
ChannelName: "c1",
MsgID: []byte{},
MsgGroup: "",
@ -657,7 +657,7 @@ func TestGetSegmentInfo(t *testing.T) {
mockVChannel := "fake-by-dev-rootcoord-dml-1-testgetsegmentinfo-v0"
mockPChannel := "fake-by-dev-rootcoord-dml-1"
pos := &internalpb.MsgPosition{
pos := &msgpb.MsgPosition{
ChannelName: mockPChannel,
MsgID: []byte{},
Timestamp: 1000,
@ -1299,7 +1299,7 @@ func TestSaveBinlogPaths(t *testing.T) {
CheckPoints: []*datapb.CheckPoint{
{
SegmentID: 1,
Position: &internalpb.MsgPosition{
Position: &msgpb.MsgPosition{
ChannelName: "ch1",
MsgID: []byte{1, 2, 3},
MsgGroup: "",
@ -1386,7 +1386,7 @@ func TestSaveBinlogPaths(t *testing.T) {
CheckPoints: []*datapb.CheckPoint{
{
SegmentID: 1,
Position: &internalpb.MsgPosition{
Position: &msgpb.MsgPosition{
ChannelName: "ch1",
MsgID: []byte{1, 2, 3},
MsgGroup: "",
@ -1464,7 +1464,7 @@ func TestSaveBinlogPaths(t *testing.T) {
CheckPoints: []*datapb.CheckPoint{
{
SegmentID: 1,
Position: &internalpb.MsgPosition{
Position: &msgpb.MsgPosition{
ChannelName: "ch1",
MsgID: []byte{1, 2, 3},
MsgGroup: "",
@ -1518,7 +1518,7 @@ func TestSaveBinlogPaths(t *testing.T) {
CheckPoints: []*datapb.CheckPoint{
{
SegmentID: 1,
Position: &internalpb.MsgPosition{
Position: &msgpb.MsgPosition{
ChannelName: "ch1",
MsgID: []byte{1, 2, 3},
MsgGroup: "",
@ -1706,13 +1706,13 @@ func TestDropVirtualChannel(t *testing.T) {
},
},
},
CheckPoint: &internalpb.MsgPosition{
CheckPoint: &msgpb.MsgPosition{
ChannelName: "ch1",
MsgID: []byte{1, 2, 3},
MsgGroup: "",
Timestamp: 0,
},
StartPosition: &internalpb.MsgPosition{
StartPosition: &msgpb.MsgPosition{
ChannelName: "ch1",
MsgID: []byte{1, 2, 3},
MsgGroup: "",
@ -1769,7 +1769,7 @@ func TestDataNodeTtChannel(t *testing.T) {
BaseMsg: msgstream.BaseMsg{
HashValues: []uint32{0},
},
DataNodeTtMsg: datapb.DataNodeTtMsg{
DataNodeTtMsg: msgpb.DataNodeTtMsg{
Base: &commonpb.MsgBase{
MsgType: msgType,
MsgID: 0,
@ -1836,7 +1836,7 @@ func TestDataNodeTtChannel(t *testing.T) {
msgPack := msgstream.MsgPack{}
msg := genMsg(commonpb.MsgType_DataNodeTt, "ch-1", assign.ExpireTime)
msg.SegmentsStats = append(msg.SegmentsStats, &datapb.SegmentStats{
msg.SegmentsStats = append(msg.SegmentsStats, &commonpb.SegmentStats{
SegmentID: assign.GetSegID(),
NumRows: 1,
})
@ -1913,7 +1913,7 @@ func TestDataNodeTtChannel(t *testing.T) {
msgPack := msgstream.MsgPack{}
msg := genMsg(commonpb.MsgType_DataNodeTt, "ch-1", assign.ExpireTime)
msg.SegmentsStats = append(msg.SegmentsStats, &datapb.SegmentStats{
msg.SegmentsStats = append(msg.SegmentsStats, &commonpb.SegmentStats{
SegmentID: assign.GetSegID(),
NumRows: 1,
})
@ -1999,29 +1999,29 @@ func TestGetChannelSeekPosition(t *testing.T) {
tests := []struct {
testName string
channelCP *internalpb.MsgPosition
segDMLPos []*internalpb.MsgPosition
channelCP *msgpb.MsgPosition
segDMLPos []*msgpb.MsgPosition
collStartPos []*commonpb.KeyDataPair
channelName string
expectedPos *internalpb.MsgPosition
expectedPos *msgpb.MsgPosition
}{
{"test-with-channelCP",
&internalpb.MsgPosition{ChannelName: "ch1", Timestamp: 100},
[]*internalpb.MsgPosition{{ChannelName: "ch1", Timestamp: 50}, {ChannelName: "ch1", Timestamp: 200}},
&msgpb.MsgPosition{ChannelName: "ch1", Timestamp: 100},
[]*msgpb.MsgPosition{{ChannelName: "ch1", Timestamp: 50}, {ChannelName: "ch1", Timestamp: 200}},
startPos1,
"ch1", &internalpb.MsgPosition{ChannelName: "ch1", Timestamp: 100}},
"ch1", &msgpb.MsgPosition{ChannelName: "ch1", Timestamp: 100}},
{"test-with-segmentDMLPos",
nil,
[]*internalpb.MsgPosition{{ChannelName: "ch1", Timestamp: 50}, {ChannelName: "ch1", Timestamp: 200}},
[]*msgpb.MsgPosition{{ChannelName: "ch1", Timestamp: 50}, {ChannelName: "ch1", Timestamp: 200}},
startPos1,
"ch1", &internalpb.MsgPosition{ChannelName: "ch1", Timestamp: 50}},
"ch1", &msgpb.MsgPosition{ChannelName: "ch1", Timestamp: 50}},
{"test-with-collStartPos",
nil,
nil,
startPos1,
"ch1", &internalpb.MsgPosition{ChannelName: "ch1", MsgID: startPos1[0].Data}},
"ch1", &msgpb.MsgPosition{ChannelName: "ch1", MsgID: startPos1[0].Data}},
{"test-non-exist-channel-1",
nil,
@ -2108,7 +2108,7 @@ func TestGetDataVChanPositions(t *testing.T) {
PartitionID: 0,
InsertChannel: "ch1",
State: commonpb.SegmentState_Flushed,
DmlPosition: &internalpb.MsgPosition{
DmlPosition: &msgpb.MsgPosition{
ChannelName: "ch1",
MsgID: []byte{1, 2, 3},
},
@ -2121,11 +2121,11 @@ func TestGetDataVChanPositions(t *testing.T) {
PartitionID: 0,
InsertChannel: "ch1",
State: commonpb.SegmentState_Growing,
StartPosition: &internalpb.MsgPosition{
StartPosition: &msgpb.MsgPosition{
ChannelName: "ch1",
MsgID: []byte{8, 9, 10},
},
DmlPosition: &internalpb.MsgPosition{
DmlPosition: &msgpb.MsgPosition{
ChannelName: "ch1",
MsgID: []byte{1, 2, 3},
Timestamp: 1,
@ -2139,11 +2139,11 @@ func TestGetDataVChanPositions(t *testing.T) {
PartitionID: 1,
InsertChannel: "ch1",
State: commonpb.SegmentState_Growing,
StartPosition: &internalpb.MsgPosition{
StartPosition: &msgpb.MsgPosition{
ChannelName: "ch1",
MsgID: []byte{8, 9, 10},
},
DmlPosition: &internalpb.MsgPosition{
DmlPosition: &msgpb.MsgPosition{
ChannelName: "ch1",
MsgID: []byte{11, 12, 13},
Timestamp: 2,
@ -2232,7 +2232,7 @@ func TestGetQueryVChanPositions(t *testing.T) {
PartitionID: 0,
InsertChannel: "ch1",
State: commonpb.SegmentState_Flushed,
DmlPosition: &internalpb.MsgPosition{
DmlPosition: &msgpb.MsgPosition{
ChannelName: "ch1",
MsgID: []byte{1, 2, 3},
MsgGroup: "",
@ -2258,12 +2258,12 @@ func TestGetQueryVChanPositions(t *testing.T) {
PartitionID: 0,
InsertChannel: "ch1",
State: commonpb.SegmentState_Growing,
StartPosition: &internalpb.MsgPosition{
StartPosition: &msgpb.MsgPosition{
ChannelName: "ch1",
MsgID: []byte{8, 9, 10},
MsgGroup: "",
},
DmlPosition: &internalpb.MsgPosition{
DmlPosition: &msgpb.MsgPosition{
ChannelName: "ch1",
MsgID: []byte{1, 2, 3},
MsgGroup: "",
@ -2278,12 +2278,12 @@ func TestGetQueryVChanPositions(t *testing.T) {
PartitionID: 1,
InsertChannel: "ch1",
State: commonpb.SegmentState_Growing,
StartPosition: &internalpb.MsgPosition{
StartPosition: &msgpb.MsgPosition{
ChannelName: "ch1",
MsgID: []byte{8, 9, 10},
MsgGroup: "",
},
DmlPosition: &internalpb.MsgPosition{
DmlPosition: &msgpb.MsgPosition{
ChannelName: "ch1",
MsgID: []byte{11, 12, 13},
MsgGroup: "",
@ -2381,13 +2381,13 @@ func TestShouldDropChannel(t *testing.T) {
PartitionID: 0,
InsertChannel: "ch1",
State: commonpb.SegmentState_Dropped,
StartPosition: &internalpb.MsgPosition{
StartPosition: &msgpb.MsgPosition{
ChannelName: "ch1",
MsgID: []byte{8, 9, 10},
MsgGroup: "",
Timestamp: 0,
},
DmlPosition: &internalpb.MsgPosition{
DmlPosition: &msgpb.MsgPosition{
ChannelName: "ch1",
MsgID: []byte{1, 2, 3},
MsgGroup: "",
@ -2401,12 +2401,12 @@ func TestShouldDropChannel(t *testing.T) {
InsertChannel: "ch1",
State: commonpb.SegmentState_Flushed,
CompactionFrom: []int64{4, 5},
StartPosition: &internalpb.MsgPosition{
StartPosition: &msgpb.MsgPosition{
ChannelName: "ch1",
MsgID: []byte{8, 9, 10},
MsgGroup: "",
},
DmlPosition: &internalpb.MsgPosition{
DmlPosition: &msgpb.MsgPosition{
ChannelName: "ch1",
MsgID: []byte{1, 2, 3},
MsgGroup: "",
@ -2419,12 +2419,12 @@ func TestShouldDropChannel(t *testing.T) {
PartitionID: 1,
InsertChannel: "ch1",
State: commonpb.SegmentState_Growing,
StartPosition: &internalpb.MsgPosition{
StartPosition: &msgpb.MsgPosition{
ChannelName: "ch1",
MsgID: []byte{8, 9, 10},
MsgGroup: "",
},
DmlPosition: &internalpb.MsgPosition{
DmlPosition: &msgpb.MsgPosition{
ChannelName: "ch1",
MsgID: []byte{11, 12, 13},
MsgGroup: "",
@ -2528,12 +2528,12 @@ func TestGetRecoveryInfo(t *testing.T) {
InsertChannel: channel,
NumOfRows: numOfRows,
State: state,
DmlPosition: &internalpb.MsgPosition{
DmlPosition: &msgpb.MsgPosition{
ChannelName: channel,
MsgID: []byte{},
Timestamp: posTs,
},
StartPosition: &internalpb.MsgPosition{
StartPosition: &msgpb.MsgPosition{
ChannelName: "",
MsgID: []byte{},
MsgGroup: "",
@ -2554,7 +2554,7 @@ func TestGetRecoveryInfo(t *testing.T) {
Schema: newTestSchema(),
})
err := svr.meta.UpdateChannelCheckpoint("vchan1", &internalpb.MsgPosition{
err := svr.meta.UpdateChannelCheckpoint("vchan1", &msgpb.MsgPosition{
ChannelName: "vchan1",
Timestamp: 10,
})
@ -2659,7 +2659,7 @@ func TestGetRecoveryInfo(t *testing.T) {
Schema: newTestSchema(),
})
err := svr.meta.UpdateChannelCheckpoint("vchan1", &internalpb.MsgPosition{
err := svr.meta.UpdateChannelCheckpoint("vchan1", &msgpb.MsgPosition{
ChannelName: "vchan1",
Timestamp: 0,
})
@ -2834,7 +2834,7 @@ func TestGetRecoveryInfo(t *testing.T) {
Schema: newTestSchema(),
})
err := svr.meta.UpdateChannelCheckpoint("vchan1", &internalpb.MsgPosition{
err := svr.meta.UpdateChannelCheckpoint("vchan1", &msgpb.MsgPosition{
ChannelName: "vchan1",
Timestamp: 0,
})
@ -2875,7 +2875,7 @@ func TestGetRecoveryInfo(t *testing.T) {
Schema: newTestSchema(),
})
err := svr.meta.UpdateChannelCheckpoint("vchan1", &internalpb.MsgPosition{
err := svr.meta.UpdateChannelCheckpoint("vchan1", &msgpb.MsgPosition{
ChannelName: "vchan1",
Timestamp: 0,
})
@ -2915,7 +2915,7 @@ func TestGetRecoveryInfo(t *testing.T) {
Schema: newTestSchema(),
})
err := svr.meta.UpdateChannelCheckpoint("vchan1", &internalpb.MsgPosition{
err := svr.meta.UpdateChannelCheckpoint("vchan1", &msgpb.MsgPosition{
ChannelName: "vchan1",
Timestamp: 0,
})
@ -3462,7 +3462,7 @@ func TestDataCoordServer_SetSegmentState(t *testing.T) {
InsertChannel: "c1",
NumOfRows: 0,
State: commonpb.SegmentState_Growing,
StartPosition: &internalpb.MsgPosition{
StartPosition: &msgpb.MsgPosition{
ChannelName: "c1",
MsgID: []byte{},
MsgGroup: "",
@ -3610,7 +3610,7 @@ func TestDataCoord_Import(t *testing.T) {
svr := newTestServer(t, nil)
status, err := svr.UpdateSegmentStatistics(context.TODO(), &datapb.UpdateSegmentStatisticsRequest{
Stats: []*datapb.SegmentStats{{
Stats: []*commonpb.SegmentStats{{
SegmentID: 100,
NumRows: int64(1),
}},
@ -3625,7 +3625,7 @@ func TestDataCoord_Import(t *testing.T) {
closeTestServer(t, svr)
status, err := svr.UpdateSegmentStatistics(context.TODO(), &datapb.UpdateSegmentStatisticsRequest{
Stats: []*datapb.SegmentStats{{
Stats: []*commonpb.SegmentStats{{
SegmentID: 100,
NumRows: int64(1),
}},
@ -3651,7 +3651,7 @@ func TestDataCoord_SegmentStatistics(t *testing.T) {
svr.meta.AddSegment(info)
status, err := svr.UpdateSegmentStatistics(context.TODO(), &datapb.UpdateSegmentStatisticsRequest{
Stats: []*datapb.SegmentStats{{
Stats: []*commonpb.SegmentStats{{
SegmentID: 100,
NumRows: int64(1),
}},
@ -3678,7 +3678,7 @@ func TestDataCoord_SegmentStatistics(t *testing.T) {
svr.meta.AddSegment(info)
status, err := svr.UpdateSegmentStatistics(context.TODO(), &datapb.UpdateSegmentStatisticsRequest{
Stats: []*datapb.SegmentStats{{
Stats: []*commonpb.SegmentStats{{
SegmentID: 100,
NumRows: int64(1),
}},
@ -3724,7 +3724,7 @@ func TestDataCoord_SaveImportSegment(t *testing.T) {
Importing: true,
StartPositions: []*datapb.SegmentStartPosition{
{
StartPosition: &internalpb.MsgPosition{
StartPosition: &msgpb.MsgPosition{
ChannelName: "ch1",
Timestamp: 1,
},
@ -3802,7 +3802,7 @@ func TestDataCoordServer_UpdateChannelCheckpoint(t *testing.T) {
SourceID: paramtable.GetNodeID(),
},
VChannel: mockVChannel,
Position: &internalpb.MsgPosition{
Position: &msgpb.MsgPosition{
ChannelName: mockPChannel,
Timestamp: 1000,
},

3
internal/datacoord/services.go
View File

@ -30,6 +30,7 @@ import (
"github.com/milvus-io/milvus-proto/go-api/commonpb"
"github.com/milvus-io/milvus-proto/go-api/milvuspb"
"github.com/milvus-io/milvus-proto/go-api/msgpb"
"github.com/milvus-io/milvus/internal/common"
"github.com/milvus-io/milvus/internal/log"
"github.com/milvus-io/milvus/internal/proto/datapb"
@ -350,7 +351,7 @@ func (s *Server) GetSegmentInfo(ctx context.Context, req *datapb.GetSegmentInfoR
return resp, nil
}
infos := make([]*datapb.SegmentInfo, 0, len(req.GetSegmentIDs()))
channelCPs := make(map[string]*internalpb.MsgPosition)
channelCPs := make(map[string]*msgpb.MsgPosition)
for _, id := range req.SegmentIDs {
var info *SegmentInfo
if req.IncludeUnHealthy {

17
internal/datanode/buffer.go
View File

@ -23,14 +23,13 @@ import (
"sync"
"github.com/cockroachdb/errors"
"go.uber.org/zap"
"github.com/milvus-io/milvus-proto/go-api/msgpb"
"github.com/milvus-io/milvus-proto/go-api/schemapb"
"github.com/milvus-io/milvus/internal/log"
"github.com/milvus-io/milvus/internal/metrics"
"github.com/milvus-io/milvus/internal/proto/datapb"
"github.com/milvus-io/milvus/internal/proto/internalpb"
"github.com/milvus-io/milvus/internal/storage"
"github.com/milvus-io/milvus/internal/util/paramtable"
"github.com/milvus-io/milvus/internal/util/typeutil"
@ -71,7 +70,7 @@ func (bm *DelBufferManager) Store(segID UniqueID, delDataBuf *DelDataBuf) {
}
func (bm *DelBufferManager) StoreNewDeletes(segID UniqueID, pks []primaryKey,
tss []Timestamp, tr TimeRange, startPos, endPos *internalpb.MsgPosition) {
tss []Timestamp, tr TimeRange, startPos, endPos *msgpb.MsgPosition) {
//1. load or create delDataBuf
var delDataBuf *DelDataBuf
buffer, loaded := bm.channel.getCurDeleteBuffer(segID)
@ -253,8 +252,8 @@ type BufferData struct {
limit int64
tsFrom Timestamp
tsTo Timestamp
startPos *internalpb.MsgPosition
endPos *internalpb.MsgPosition
startPos *msgpb.MsgPosition
endPos *msgpb.MsgPosition
}
func (bd *BufferData) effectiveCap() int64 {
@ -275,7 +274,7 @@ func (bd *BufferData) updateTimeRange(tr TimeRange) {
}
}
func (bd *BufferData) updateStartAndEndPosition(startPos *internalpb.MsgPosition, endPos *internalpb.MsgPosition) {
func (bd *BufferData) updateStartAndEndPosition(startPos *msgpb.MsgPosition, endPos *msgpb.MsgPosition) {
if bd.startPos == nil || startPos.Timestamp < bd.startPos.Timestamp {
bd.startPos = startPos
}
@ -298,8 +297,8 @@ type DelDataBuf struct {
datapb.Binlog
delData *DeleteData
item *Item
startPos *internalpb.MsgPosition
endPos *internalpb.MsgPosition
startPos *msgpb.MsgPosition
endPos *msgpb.MsgPosition
}
func (ddb *DelDataBuf) accumulateEntriesNum(entryNum int64) {
@ -327,7 +326,7 @@ func (ddb *DelDataBuf) mergeDelDataBuf(buf *DelDataBuf) {
ddb.item.memorySize += buf.item.memorySize
}
func (ddb *DelDataBuf) updateStartAndEndPosition(startPos *internalpb.MsgPosition, endPos *internalpb.MsgPosition) {
func (ddb *DelDataBuf) updateStartAndEndPosition(startPos *msgpb.MsgPosition, endPos *msgpb.MsgPosition) {
if ddb.startPos == nil || startPos.Timestamp < ddb.startPos.Timestamp {
ddb.startPos = startPos
}

10
internal/datanode/buffer_test.go
View File

@ -27,8 +27,8 @@ import (
"github.com/stretchr/testify/require"
"github.com/milvus-io/milvus-proto/go-api/commonpb"
"github.com/milvus-io/milvus-proto/go-api/msgpb"
"github.com/milvus-io/milvus-proto/go-api/schemapb"
"github.com/milvus-io/milvus/internal/proto/internalpb"
"github.com/milvus-io/milvus/internal/util/paramtable"
)
@ -166,12 +166,12 @@ func Test_CompactSegBuff(t *testing.T) {
//2. set up deleteDataBuf for seg1 and seg2
delDataBuf1 := newDelDataBuf()
delDataBuf1.EntriesNum++
delDataBuf1.updateStartAndEndPosition(nil, &internalpb.MsgPosition{Timestamp: 50})
delDataBuf1.updateStartAndEndPosition(nil, &msgpb.MsgPosition{Timestamp: 50})
delBufferManager.Store(segID1, delDataBuf1)
heap.Push(delBufferManager.delBufHeap, delDataBuf1.item)
delDataBuf2 := newDelDataBuf()
delDataBuf2.EntriesNum++
delDataBuf2.updateStartAndEndPosition(nil, &internalpb.MsgPosition{Timestamp: 50})
delDataBuf2.updateStartAndEndPosition(nil, &msgpb.MsgPosition{Timestamp: 50})
delBufferManager.Store(segID2, delDataBuf2)
heap.Push(delBufferManager.delBufHeap, delDataBuf2.item)
@ -191,10 +191,10 @@ func Test_CompactSegBuff(t *testing.T) {
delBufferManager.channel.rollDeleteBuffer(compactedToSegID)
_, segCompactedToExist := delBufferManager.Load(compactedToSegID)
assert.False(t, segCompactedToExist)
delBufferManager.channel.evictHistoryDeleteBuffer(compactedToSegID, &internalpb.MsgPosition{
delBufferManager.channel.evictHistoryDeleteBuffer(compactedToSegID, &msgpb.MsgPosition{
Timestamp: 100,
})
cp := delBufferManager.channel.getChannelCheckpoint(&internalpb.MsgPosition{
cp := delBufferManager.channel.getChannelCheckpoint(&msgpb.MsgPosition{
Timestamp: 200,
})
assert.Equal(t, Timestamp(200), cp.Timestamp) // evict all buffer, use ttPos as cp

30
internal/datanode/channel_meta.go
View File

@ -23,17 +23,19 @@ import (
"sync"
"time"
"github.com/samber/lo"
"go.uber.org/zap"
"github.com/milvus-io/milvus-proto/go-api/commonpb"
"github.com/milvus-io/milvus-proto/go-api/msgpb"
"github.com/milvus-io/milvus-proto/go-api/schemapb"
"github.com/milvus-io/milvus/internal/common"
"github.com/milvus-io/milvus/internal/log"
"github.com/milvus-io/milvus/internal/proto/datapb"
"github.com/milvus-io/milvus/internal/proto/internalpb"
"github.com/milvus-io/milvus/internal/storage"
"github.com/milvus-io/milvus/internal/types"
"github.com/milvus-io/milvus/internal/util/tsoutil"
"github.com/milvus-io/milvus/internal/util/typeutil"
"github.com/samber/lo"
"go.uber.org/zap"
)
type (
@ -73,20 +75,20 @@ type Channel interface {
updateSegmentMemorySize(segID UniqueID, memorySize int64)
InitPKstats(ctx context.Context, s *Segment, statsBinlogs []*datapb.FieldBinlog, ts Timestamp) error
RollPKstats(segID UniqueID, stats []*storage.PrimaryKeyStats)
getSegmentStatisticsUpdates(segID UniqueID) (*datapb.SegmentStats, error)
getSegmentStatisticsUpdates(segID UniqueID) (*commonpb.SegmentStats, error)
segmentFlushed(segID UniqueID)
getChannelCheckpoint(ttPos *internalpb.MsgPosition) *internalpb.MsgPosition
getChannelCheckpoint(ttPos *msgpb.MsgPosition) *msgpb.MsgPosition
getCurInsertBuffer(segmentID UniqueID) (*BufferData, bool)
setCurInsertBuffer(segmentID UniqueID, buf *BufferData)
rollInsertBuffer(segmentID UniqueID)
evictHistoryInsertBuffer(segmentID UniqueID, endPos *internalpb.MsgPosition)
evictHistoryInsertBuffer(segmentID UniqueID, endPos *msgpb.MsgPosition)
getCurDeleteBuffer(segmentID UniqueID) (*DelDataBuf, bool)
setCurDeleteBuffer(segmentID UniqueID, buf *DelDataBuf)
rollDeleteBuffer(segmentID UniqueID)
evictHistoryDeleteBuffer(segmentID UniqueID, endPos *internalpb.MsgPosition)
evictHistoryDeleteBuffer(segmentID UniqueID, endPos *msgpb.MsgPosition)
}
// ChannelMeta contains channel meta and the latest segments infos of the channel.
@ -109,7 +111,7 @@ type addSegmentReq struct {
segType datapb.SegmentType
segID, collID, partitionID UniqueID
numOfRows int64
startPos, endPos *internalpb.MsgPosition
startPos, endPos *msgpb.MsgPosition
statsBinLogs []*datapb.FieldBinlog
recoverTs Timestamp
importing bool
@ -506,12 +508,12 @@ func (c *ChannelMeta) updateSegmentMemorySize(segID UniqueID, memorySize int64)
}
// getSegmentStatisticsUpdates gives current segment's statistics updates.
func (c *ChannelMeta) getSegmentStatisticsUpdates(segID UniqueID) (*datapb.SegmentStats, error) {
func (c *ChannelMeta) getSegmentStatisticsUpdates(segID UniqueID) (*commonpb.SegmentStats, error) {
c.segMu.RLock()
defer c.segMu.RUnlock()
if seg, ok := c.segments[segID]; ok && seg.isValid() {
return &datapb.SegmentStats{SegmentID: segID, NumRows: seg.numRows}, nil
return &commonpb.SegmentStats{SegmentID: segID, NumRows: seg.numRows}, nil
}
return nil, fmt.Errorf("error, there's no segment %d", segID)
@ -672,10 +674,10 @@ func (c *ChannelMeta) listNotFlushedSegmentIDs() []UniqueID {
return segIDs
}
func (c *ChannelMeta) getChannelCheckpoint(ttPos *internalpb.MsgPosition) *internalpb.MsgPosition {
func (c *ChannelMeta) getChannelCheckpoint(ttPos *msgpb.MsgPosition) *msgpb.MsgPosition {
c.segMu.RLock()
defer c.segMu.RUnlock()
channelCP := &internalpb.MsgPosition{Timestamp: math.MaxUint64}
channelCP := &msgpb.MsgPosition{Timestamp: math.MaxUint64}
// 1. find the earliest startPos in current buffer and history buffer
for _, seg := range c.segments {
if seg.curInsertBuf != nil && seg.curInsertBuf.startPos != nil && seg.curInsertBuf.startPos.Timestamp < channelCP.Timestamp {
@ -742,7 +744,7 @@ func (c *ChannelMeta) rollInsertBuffer(segmentID UniqueID) {
log.Warn("cannot find segment when rollInsertBuffer", zap.Int64("segmentID", segmentID))
}
func (c *ChannelMeta) evictHistoryInsertBuffer(segmentID UniqueID, endPos *internalpb.MsgPosition) {
func (c *ChannelMeta) evictHistoryInsertBuffer(segmentID UniqueID, endPos *msgpb.MsgPosition) {
c.segMu.Lock()
defer c.segMu.Unlock()
@ -789,7 +791,7 @@ func (c *ChannelMeta) rollDeleteBuffer(segmentID UniqueID) {
log.Warn("cannot find segment when rollDeleteBuffer", zap.Int64("segmentID", segmentID))
}
func (c *ChannelMeta) evictHistoryDeleteBuffer(segmentID UniqueID, endPos *internalpb.MsgPosition) {
func (c *ChannelMeta) evictHistoryDeleteBuffer(segmentID UniqueID, endPos *msgpb.MsgPosition) {
c.segMu.Lock()
defer c.segMu.Unlock()

54
internal/datanode/channel_meta_test.go
View File

@ -23,18 +23,17 @@ import (
"math/rand"
"testing"
bloom "github.com/bits-and-blooms/bloom/v3"
"github.com/cockroachdb/errors"
"github.com/bits-and-blooms/bloom/v3"
"github.com/samber/lo"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/stretchr/testify/suite"
"github.com/milvus-io/milvus-proto/go-api/msgpb"
"github.com/milvus-io/milvus-proto/go-api/schemapb"
"github.com/milvus-io/milvus/internal/common"
"github.com/milvus-io/milvus/internal/proto/datapb"
"github.com/milvus-io/milvus/internal/proto/internalpb"
"github.com/milvus-io/milvus/internal/storage"
)
@ -122,8 +121,8 @@ func TestChannelMeta_InnerFunction(t *testing.T) {
var err error
startPos := &internalpb.MsgPosition{ChannelName: "insert-01", Timestamp: Timestamp(100)}
endPos := &internalpb.MsgPosition{ChannelName: "insert-01", Timestamp: Timestamp(200)}
startPos := &msgpb.MsgPosition{ChannelName: "insert-01", Timestamp: Timestamp(100)}
endPos := &msgpb.MsgPosition{ChannelName: "insert-01", Timestamp: Timestamp(200)}
err = channel.addSegment(
addSegmentReq{
segType: datapb.SegmentType_New,
@ -325,15 +324,15 @@ func TestChannelMeta_InterfaceMethod(t *testing.T) {
inCollID UniqueID
inSegID UniqueID
instartPos *internalpb.MsgPosition
instartPos *msgpb.MsgPosition
expectedSegType datapb.SegmentType
description string
}{
{isValidCase: false, channelCollID: 1, inCollID: 2, inSegID: 300, description: "input CollID 2 mismatch with channel collID"},
{true, 1, 1, 200, new(internalpb.MsgPosition), datapb.SegmentType_New, "nill address for startPos"},
{true, 1, 1, 200, &internalpb.MsgPosition{}, datapb.SegmentType_New, "empty struct for startPos"},
{true, 1, 1, 200, new(msgpb.MsgPosition), datapb.SegmentType_New, "nill address for startPos"},
{true, 1, 1, 200, &msgpb.MsgPosition{}, datapb.SegmentType_New, "empty struct for startPos"},
}
for _, test := range tests {
@ -347,7 +346,7 @@ func TestChannelMeta_InterfaceMethod(t *testing.T) {
collID: test.inCollID,
partitionID: 1,
startPos: test.instartPos,
endPos: &internalpb.MsgPosition{},
endPos: &msgpb.MsgPosition{},
})
if test.isValidCase {
assert.NoError(t, err)
@ -697,6 +696,7 @@ func TestChannelMeta_InterfaceMethod(t *testing.T) {
})
}
func TestChannelMeta_UpdatePKRange(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
@ -706,8 +706,8 @@ func TestChannelMeta_UpdatePKRange(t *testing.T) {
collID := UniqueID(1)
partID := UniqueID(2)
chanName := "insert-02"
startPos := &internalpb.MsgPosition{ChannelName: chanName, Timestamp: Timestamp(100)}
endPos := &internalpb.MsgPosition{ChannelName: chanName, Timestamp: Timestamp(200)}
startPos := &msgpb.MsgPosition{ChannelName: chanName, Timestamp: Timestamp(100)}
endPos := &msgpb.MsgPosition{ChannelName: chanName, Timestamp: Timestamp(200)}
cm := storage.NewLocalChunkManager(storage.RootPath(channelMetaNodeTestDir))
defer cm.RemoveWithPrefix(ctx, cm.RootPath())
@ -774,7 +774,7 @@ func TestChannelMeta_ChannelCP(t *testing.T) {
}()
t.Run("get and set", func(t *testing.T) {
pos := &internalpb.MsgPosition{
pos := &msgpb.MsgPosition{
ChannelName: mockPChannel,
Timestamp: 1000,
}
@ -787,9 +787,9 @@ func TestChannelMeta_ChannelCP(t *testing.T) {
})
t.Run("set insertBuffer&deleteBuffer then get", func(t *testing.T) {
run := func(curInsertPos, curDeletePos *internalpb.MsgPosition,
hisInsertPoss, hisDeletePoss []*internalpb.MsgPosition,
ttPos, expectedPos *internalpb.MsgPosition) {
run := func(curInsertPos, curDeletePos *msgpb.MsgPosition,
hisInsertPoss, hisDeletePoss []*msgpb.MsgPosition,
ttPos, expectedPos *msgpb.MsgPosition) {
segmentID := UniqueID(1)
channel := newChannel(mockVChannel, collID, nil, rc, cm)
channel.chunkManager = &mockDataCM{}
@ -836,28 +836,28 @@ func TestChannelMeta_ChannelCP(t *testing.T) {
assert.True(t, resPos.Timestamp == expectedPos.Timestamp)
}
run(&internalpb.MsgPosition{Timestamp: 50}, &internalpb.MsgPosition{Timestamp: 60},
[]*internalpb.MsgPosition{{Timestamp: 70}}, []*internalpb.MsgPosition{{Timestamp: 120}},
&internalpb.MsgPosition{Timestamp: 120}, &internalpb.MsgPosition{Timestamp: 50})
run(&msgpb.MsgPosition{Timestamp: 50}, &msgpb.MsgPosition{Timestamp: 60},
[]*msgpb.MsgPosition{{Timestamp: 70}}, []*msgpb.MsgPosition{{Timestamp: 120}},
&msgpb.MsgPosition{Timestamp: 120}, &msgpb.MsgPosition{Timestamp: 50})
run(&internalpb.MsgPosition{Timestamp: 50}, &internalpb.MsgPosition{Timestamp: 60},
[]*internalpb.MsgPosition{{Timestamp: 70}}, []*internalpb.MsgPosition{{Timestamp: 120}},
&internalpb.MsgPosition{Timestamp: 30}, &internalpb.MsgPosition{Timestamp: 50})
run(&msgpb.MsgPosition{Timestamp: 50}, &msgpb.MsgPosition{Timestamp: 60},
[]*msgpb.MsgPosition{{Timestamp: 70}}, []*msgpb.MsgPosition{{Timestamp: 120}},
&msgpb.MsgPosition{Timestamp: 30}, &msgpb.MsgPosition{Timestamp: 50})
// nil cur buffer
run(nil, nil,
[]*internalpb.MsgPosition{{Timestamp: 120}}, []*internalpb.MsgPosition{{Timestamp: 110}},
&internalpb.MsgPosition{Timestamp: 130}, &internalpb.MsgPosition{Timestamp: 110})
[]*msgpb.MsgPosition{{Timestamp: 120}}, []*msgpb.MsgPosition{{Timestamp: 110}},
&msgpb.MsgPosition{Timestamp: 130}, &msgpb.MsgPosition{Timestamp: 110})
// nil history buffer
run(&internalpb.MsgPosition{Timestamp: 50}, &internalpb.MsgPosition{Timestamp: 100},
run(&msgpb.MsgPosition{Timestamp: 50}, &msgpb.MsgPosition{Timestamp: 100},
nil, nil,
&internalpb.MsgPosition{Timestamp: 100}, &internalpb.MsgPosition{Timestamp: 50})
&msgpb.MsgPosition{Timestamp: 100}, &msgpb.MsgPosition{Timestamp: 50})
// nil buffer
run(nil, nil,
nil, nil,
&internalpb.MsgPosition{Timestamp: 100}, &internalpb.MsgPosition{Timestamp: 100})
&msgpb.MsgPosition{Timestamp: 100}, &msgpb.MsgPosition{Timestamp: 100})
})
}
@ -893,7 +893,7 @@ func (s *ChannelMetaSuite) SetupTest() {
segID: 1,
collID: s.collID,
partitionID: s.partID,
startPos: &internalpb.MsgPosition{},
startPos: &msgpb.MsgPosition{},
endPos: nil,
})
s.Require().NoError(err)

17
internal/datanode/compactor_test.go
View File

@ -20,24 +20,23 @@ import (
"context"
"fmt"
"math"
// "math"
"testing"
"time"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/mock"
"github.com/stretchr/testify/require"
"github.com/milvus-io/milvus-proto/go-api/commonpb"
"github.com/milvus-io/milvus-proto/go-api/milvuspb"
"github.com/milvus-io/milvus-proto/go-api/msgpb"
"github.com/milvus-io/milvus-proto/go-api/schemapb"
memkv "github.com/milvus-io/milvus/internal/kv/mem"
"github.com/milvus-io/milvus/internal/mocks"
"github.com/milvus-io/milvus/internal/proto/datapb"
"github.com/milvus-io/milvus/internal/proto/etcdpb"
"github.com/milvus-io/milvus/internal/proto/internalpb"
"github.com/milvus-io/milvus/internal/storage"
"github.com/milvus-io/milvus/internal/util/paramtable"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/mock"
"github.com/stretchr/testify/require"
)
var compactTestDir = "/tmp/milvus_test/compact"
@ -67,7 +66,7 @@ func TestCompactionTaskInnerMethods(t *testing.T) {
segID: 100,
collID: 1,
partitionID: 10,
startPos: new(internalpb.MsgPosition),
startPos: new(msgpb.MsgPosition),
endPos: nil,
})
require.NoError(t, err)
@ -856,14 +855,14 @@ type mockFlushManager struct {
var _ flushManager = (*mockFlushManager)(nil)
func (mfm *mockFlushManager) flushBufferData(data *BufferData, segmentID UniqueID, flushed bool, dropped bool, pos *internalpb.MsgPosition) ([]*Blob, error) {
func (mfm *mockFlushManager) flushBufferData(data *BufferData, segmentID UniqueID, flushed bool, dropped bool, pos *msgpb.MsgPosition) ([]*Blob, error) {
if mfm.returnError {
return nil, fmt.Errorf("mock error")
}
return nil, nil
}
func (mfm *mockFlushManager) flushDelData(data *DelDataBuf, segmentID UniqueID, pos *internalpb.MsgPosition) error {
func (mfm *mockFlushManager) flushDelData(data *DelDataBuf, segmentID UniqueID, pos *msgpb.MsgPosition) error {
if mfm.returnError {
return fmt.Errorf("mock error")
}

40
internal/datanode/data_sync_service_test.go
View File

@ -30,13 +30,13 @@ import (
"go.uber.org/zap"
"github.com/milvus-io/milvus-proto/go-api/commonpb"
"github.com/milvus-io/milvus-proto/go-api/msgpb"
"github.com/milvus-io/milvus-proto/go-api/schemapb"
"github.com/milvus-io/milvus/internal/common"
"github.com/milvus-io/milvus/internal/log"
"github.com/milvus-io/milvus/internal/mq/msgdispatcher"
"github.com/milvus-io/milvus/internal/mq/msgstream"
"github.com/milvus-io/milvus/internal/proto/datapb"
"github.com/milvus-io/milvus/internal/proto/internalpb"
"github.com/milvus-io/milvus/internal/storage"
"github.com/milvus-io/milvus/internal/util/dependency"
"github.com/milvus-io/milvus/internal/util/paramtable"
@ -60,7 +60,7 @@ func getVchanInfo(info *testInfo) *datapb.VchannelInfo {
InsertChannel: info.ufchanName,
ID: info.ufSegID,
NumOfRows: info.ufNor,
DmlPosition: &internalpb.MsgPosition{},
DmlPosition: &msgpb.MsgPosition{},
}}
fs = []*datapb.SegmentInfo{{
CollectionID: info.fCollID,
@ -68,7 +68,7 @@ func getVchanInfo(info *testInfo) *datapb.VchannelInfo {
InsertChannel: info.fchanName,
ID: info.fSegID,
NumOfRows: info.fNor,
DmlPosition: &internalpb.MsgPosition{},
DmlPosition: &msgpb.MsgPosition{},
}}
} else {
ufs = []*datapb.SegmentInfo{}
@ -85,7 +85,7 @@ func getVchanInfo(info *testInfo) *datapb.VchannelInfo {
vi := &datapb.VchannelInfo{
CollectionID: info.collID,
ChannelName: info.chanName,
SeekPosition: &internalpb.MsgPosition{},
SeekPosition: &msgpb.MsgPosition{},
UnflushedSegmentIds: ufsIds,
FlushedSegmentIds: fsIds,
}
@ -236,7 +236,7 @@ func TestDataSyncService_Start(t *testing.T) {
InsertChannel: insertChannelName,
ID: 0,
NumOfRows: 0,
DmlPosition: &internalpb.MsgPosition{},
DmlPosition: &msgpb.MsgPosition{},
}}
fs := []*datapb.SegmentInfo{{
CollectionID: collMeta.ID,
@ -244,7 +244,7 @@ func TestDataSyncService_Start(t *testing.T) {
InsertChannel: insertChannelName,
ID: 1,
NumOfRows: 0,
DmlPosition: &internalpb.MsgPosition{},
DmlPosition: &msgpb.MsgPosition{},
}}
var ufsIds []int64
var fsIds []int64
@ -299,10 +299,10 @@ func TestDataSyncService_Start(t *testing.T) {
BeginTs: timeRange.timestampMin,
EndTs: timeRange.timestampMax,
Msgs: insertMessages,
StartPositions: []*internalpb.MsgPosition{{
StartPositions: []*msgpb.MsgPosition{{
ChannelName: insertChannelName,
}},
EndPositions: []*internalpb.MsgPosition{{
EndPositions: []*msgpb.MsgPosition{{
ChannelName: insertChannelName,
}},
}
@ -316,7 +316,7 @@ func TestDataSyncService_Start(t *testing.T) {
EndTimestamp: Timestamp(0),
HashValues: []uint32{0},
},
TimeTickMsg: internalpb.TimeTickMsg{
TimeTickMsg: msgpb.TimeTickMsg{
Base: &commonpb.MsgBase{
MsgType: commonpb.MsgType_TimeTick,
MsgID: UniqueID(0),
@ -377,7 +377,7 @@ func TestDataSyncService_Close(t *testing.T) {
InsertChannel: insertChannelName,
ID: 1,
NumOfRows: 1,
DmlPosition: &internalpb.MsgPosition{},
DmlPosition: &msgpb.MsgPosition{},
}}
fs := []*datapb.SegmentInfo{{
CollectionID: collMeta.ID,
@ -385,7 +385,7 @@ func TestDataSyncService_Close(t *testing.T) {
InsertChannel: insertChannelName,
ID: 0,
NumOfRows: 1,
DmlPosition: &internalpb.MsgPosition{},
DmlPosition: &msgpb.MsgPosition{},
}}
var ufsIds []int64
var fsIds []int64
@ -449,10 +449,10 @@ func TestDataSyncService_Close(t *testing.T) {
BeginTs: ts,
EndTs: ts,
Msgs: insertMessages,
StartPositions: []*internalpb.MsgPosition{{
StartPositions: []*msgpb.MsgPosition{{
ChannelName: insertChannelName,
}},
EndPositions: []*internalpb.MsgPosition{{
EndPositions: []*msgpb.MsgPosition{{
ChannelName: insertChannelName,
}},
}
@ -467,10 +467,10 @@ func TestDataSyncService_Close(t *testing.T) {
BeginTs: ts + 1,
EndTs: ts + 1,
Msgs: inMsgs,
StartPositions: []*internalpb.MsgPosition{{
StartPositions: []*msgpb.MsgPosition{{
ChannelName: insertChannelName,
}},
EndPositions: []*internalpb.MsgPosition{{
EndPositions: []*msgpb.MsgPosition{{
ChannelName: insertChannelName,
}},
}
@ -482,7 +482,7 @@ func TestDataSyncService_Close(t *testing.T) {
EndTimestamp: ts + 2,
HashValues: []uint32{0},
},
TimeTickMsg: internalpb.TimeTickMsg{
TimeTickMsg: msgpb.TimeTickMsg{
Base: &commonpb.MsgBase{
MsgType: commonpb.MsgType_TimeTick,
MsgID: UniqueID(2),
@ -495,10 +495,10 @@ func TestDataSyncService_Close(t *testing.T) {
timeTickMsgPack := msgstream.MsgPack{
BeginTs: ts + 2,
EndTs: ts + 2,
StartPositions: []*internalpb.MsgPosition{{
StartPositions: []*msgpb.MsgPosition{{
ChannelName: insertChannelName,
}},
EndPositions: []*internalpb.MsgPosition{{
EndPositions: []*msgpb.MsgPosition{{
ChannelName: insertChannelName,
}},
}
@ -667,8 +667,8 @@ func TestClearGlobalFlushingCache(t *testing.T) {
segID: 1,
collID: 1,
partitionID: 1,
startPos: &internalpb.MsgPosition{},
endPos: &internalpb.MsgPosition{}})
startPos: &msgpb.MsgPosition{},
endPos: &msgpb.MsgPosition{}})
assert.NoError(t, err)
err = channel.addSegment(

13
internal/datanode/flow_graph_dd_node.go
View File

@ -22,25 +22,24 @@ import (
"reflect"
"sync/atomic"
"github.com/milvus-io/milvus/internal/util/timerecord"
"go.opentelemetry.io/otel/trace"
"github.com/golang/protobuf/proto"
"go.opentelemetry.io/otel/trace"
"go.uber.org/zap"
"github.com/milvus-io/milvus-proto/go-api/commonpb"
"github.com/milvus-io/milvus-proto/go-api/msgpb"
"github.com/milvus-io/milvus/internal/common"
"github.com/milvus-io/milvus/internal/log"
"github.com/milvus-io/milvus/internal/metrics"
"github.com/milvus-io/milvus/internal/mq/msgstream"
"github.com/milvus-io/milvus/internal/proto/datapb"
"github.com/milvus-io/milvus/internal/proto/internalpb"
"github.com/milvus-io/milvus/internal/util/commonpbutil"
"github.com/milvus-io/milvus/internal/util/flowgraph"
"github.com/milvus-io/milvus/internal/util/funcutil"
"github.com/milvus-io/milvus/internal/util/metricsinfo"
"github.com/milvus-io/milvus/internal/util/paramtable"
"github.com/milvus-io/milvus/internal/util/retry"
"github.com/milvus-io/milvus/internal/util/timerecord"
"github.com/milvus-io/milvus/internal/util/tsoutil"
)
@ -146,8 +145,8 @@ func (ddn *ddNode) Operate(in []Msg) []Msg {
timestampMin: msMsg.TimestampMin(),
timestampMax: msMsg.TimestampMax(),
},
startPositions: make([]*internalpb.MsgPosition, 0),
endPositions: make([]*internalpb.MsgPosition, 0),
startPositions: make([]*msgpb.MsgPosition, 0),
endPositions: make([]*msgpb.MsgPosition, 0),
dropCollection: false,
}
@ -333,7 +332,7 @@ func (ddn *ddNode) sendDeltaTimeTick(ts Timestamp) error {
EndTimestamp: ts,
HashValues: []uint32{0},
}
timeTickResult := internalpb.TimeTickMsg{
timeTickResult := msgpb.TimeTickMsg{
Base: commonpbutil.NewMsgBase(
commonpbutil.WithMsgType(commonpb.MsgType_TimeTick),
commonpbutil.WithMsgID(0),

17
internal/datanode/flow_graph_dd_node_test.go
View File

@ -25,10 +25,9 @@ import (
"github.com/stretchr/testify/require"
"github.com/milvus-io/milvus-proto/go-api/commonpb"
"github.com/milvus-io/milvus/internal/proto/datapb"
"github.com/milvus-io/milvus/internal/proto/internalpb"
"github.com/milvus-io/milvus-proto/go-api/msgpb"
"github.com/milvus-io/milvus/internal/mq/msgstream"
"github.com/milvus-io/milvus/internal/proto/datapb"
"github.com/milvus-io/milvus/internal/util/dependency"
"github.com/milvus-io/milvus/internal/util/flowgraph"
"github.com/milvus-io/milvus/internal/util/retry"
@ -144,7 +143,7 @@ func TestFlowGraph_DDNode_Operate(t *testing.T) {
}
var dropCollMsg msgstream.TsMsg = &msgstream.DropCollectionMsg{
DropCollectionRequest: internalpb.DropCollectionRequest{
DropCollectionRequest: msgpb.DropCollectionRequest{
Base: &commonpb.MsgBase{MsgType: commonpb.MsgType_DropCollection},
CollectionID: test.msgCollID,
},
@ -197,7 +196,7 @@ func TestFlowGraph_DDNode_Operate(t *testing.T) {
}
var dropPartMsg msgstream.TsMsg = &msgstream.DropPartitionMsg{
DropPartitionRequest: internalpb.DropPartitionRequest{
DropPartitionRequest: msgpb.DropPartitionRequest{
Base: &commonpb.MsgBase{MsgType: commonpb.MsgType_DropPartition},
CollectionID: test.msgCollID,
PartitionID: test.msgPartID,
@ -276,7 +275,7 @@ func TestFlowGraph_DDNode_Operate(t *testing.T) {
EndTimestamp: test.MsgEndTs,
HashValues: []uint32{0},
},
DeleteRequest: internalpb.DeleteRequest{
DeleteRequest: msgpb.DeleteRequest{
Base: &commonpb.MsgBase{MsgType: commonpb.MsgType_Delete},
ShardName: "by-dev-rootcoord-dml-mock-0",
CollectionID: test.inMsgCollID,
@ -310,7 +309,7 @@ func TestFlowGraph_DDNode_Operate(t *testing.T) {
EndTimestamp: 2000,
HashValues: []uint32{0},
},
DeleteRequest: internalpb.DeleteRequest{
DeleteRequest: msgpb.DeleteRequest{
Base: &commonpb.MsgBase{MsgType: commonpb.MsgType_Delete},
CollectionID: 1,
},
@ -590,7 +589,7 @@ func TestFlowGraph_DDNode_isDropped(t *testing.T) {
func getSegmentInfo(segmentID UniqueID, ts Timestamp) *datapb.SegmentInfo {
return &datapb.SegmentInfo{
ID: segmentID,
DmlPosition: &internalpb.MsgPosition{Timestamp: ts},
DmlPosition: &msgpb.MsgPosition{Timestamp: ts},
}
}
@ -601,7 +600,7 @@ func getInsertMsg(segmentID UniqueID, ts Timestamp) *msgstream.InsertMsg {
func getInsertMsgWithChannel(segmentID UniqueID, ts Timestamp, vChannelName string) *msgstream.InsertMsg {
return &msgstream.InsertMsg{
BaseMsg: msgstream.BaseMsg{EndTimestamp: ts},
InsertRequest: internalpb.InsertRequest{
InsertRequest: msgpb.InsertRequest{
Base: &commonpb.MsgBase{MsgType: commonpb.MsgType_Insert},
SegmentID: segmentID,
CollectionID: 1,

5
internal/datanode/flow_graph_delete_node.go
View File

@ -22,12 +22,11 @@ import (
"reflect"
"go.opentelemetry.io/otel/trace"
"go.uber.org/zap"
"github.com/milvus-io/milvus-proto/go-api/msgpb"
"github.com/milvus-io/milvus/internal/log"
"github.com/milvus-io/milvus/internal/mq/msgstream"
"github.com/milvus-io/milvus/internal/proto/internalpb"
"github.com/milvus-io/milvus/internal/storage"
"github.com/milvus-io/milvus/internal/util/retry"
"github.com/milvus-io/milvus/internal/util/tsoutil"
@ -176,7 +175,7 @@ func (dn *deleteNode) updateCompactedSegments() {
}
}
func (dn *deleteNode) bufferDeleteMsg(msg *msgstream.DeleteMsg, tr TimeRange, startPos, endPos *internalpb.MsgPosition) ([]UniqueID, error) {
func (dn *deleteNode) bufferDeleteMsg(msg *msgstream.DeleteMsg, tr TimeRange, startPos, endPos *msgpb.MsgPosition) ([]UniqueID, error) {
log.Debug("bufferDeleteMsg", zap.Any("primary keys", msg.PrimaryKeys), zap.String("vChannelName", dn.channelName))
primaryKeys := storage.ParseIDs2PrimaryKeys(msg.PrimaryKeys)

4
internal/datanode/flow_graph_dmstream_input_node.go
View File

@ -22,12 +22,12 @@ import (
"go.uber.org/zap"
"github.com/milvus-io/milvus-proto/go-api/msgpb"
"github.com/milvus-io/milvus/internal/log"
"github.com/milvus-io/milvus/internal/metrics"
"github.com/milvus-io/milvus/internal/mq/msgdispatcher"
"github.com/milvus-io/milvus/internal/mq/msgstream"
"github.com/milvus-io/milvus/internal/mq/msgstream/mqwrapper"
"github.com/milvus-io/milvus/internal/proto/internalpb"
"github.com/milvus-io/milvus/internal/util/flowgraph"
"github.com/milvus-io/milvus/internal/util/paramtable"
"github.com/milvus-io/milvus/internal/util/tsoutil"
@ -38,7 +38,7 @@ import (
//
// messages between two timeticks to the following flowgraph node. In DataNode, the following flow graph node is
// flowgraph ddNode.
func newDmInputNode(dispatcherClient msgdispatcher.Client, seekPos *internalpb.MsgPosition, dmNodeConfig *nodeConfig) (*flowgraph.InputNode, error) {
func newDmInputNode(dispatcherClient msgdispatcher.Client, seekPos *msgpb.MsgPosition, dmNodeConfig *nodeConfig) (*flowgraph.InputNode, error) {
log := log.With(zap.Int64("nodeID", paramtable.GetNodeID()),
zap.Int64("collection ID", dmNodeConfig.collectionID),
zap.String("vchannel", dmNodeConfig.vChannelName))

13
internal/datanode/flow_graph_dmstream_input_node_test.go
View File

@ -21,13 +21,12 @@ import (
"testing"
"github.com/cockroachdb/errors"
"github.com/stretchr/testify/assert"
"github.com/milvus-io/milvus-proto/go-api/msgpb"
"github.com/milvus-io/milvus/internal/mq/msgdispatcher"
"github.com/milvus-io/milvus/internal/mq/msgstream"
"github.com/milvus-io/milvus/internal/mq/msgstream/mqwrapper"
"github.com/milvus-io/milvus/internal/proto/internalpb"
"github.com/milvus-io/milvus/internal/util/paramtable"
"github.com/milvus-io/milvus/internal/util/typeutil"
)
@ -69,25 +68,31 @@ type mockTtMsgStream struct {
}
func (mtm *mockTtMsgStream) Close() {}
func (mtm *mockTtMsgStream) Chan() <-chan *msgstream.MsgPack {
return make(chan *msgstream.MsgPack, 100)
}
func (mtm *mockTtMsgStream) AsProducer(channels []string) {}
func (mtm *mockTtMsgStream) AsConsumer(channels []string, subName string, position mqwrapper.SubscriptionInitialPosition) {
}
func (mtm *mockTtMsgStream) SetRepackFunc(repackFunc msgstream.RepackFunc) {}
func (mtm *mockTtMsgStream) GetProduceChannels() []string {
return make([]string, 0)
}
func (mtm *mockTtMsgStream) Produce(*msgstream.MsgPack) error {
return nil
}
func (mtm *mockTtMsgStream) Broadcast(*msgstream.MsgPack) (map[string][]msgstream.MessageID, error) {
return nil, nil
}
func (mtm *mockTtMsgStream) Seek(offset []*internalpb.MsgPosition) error {
func (mtm *mockTtMsgStream) Seek(offset []*msgpb.MsgPosition) error {
return nil
}
@ -97,7 +102,7 @@ func (mtm *mockTtMsgStream) GetLatestMsgID(channel string) (msgstream.MessageID,
func TestNewDmInputNode(t *testing.T) {
client := msgdispatcher.NewClient(&mockMsgStreamFactory{}, typeutil.DataNodeRole, paramtable.GetNodeID())
_, err := newDmInputNode(client, new(internalpb.MsgPosition), &nodeConfig{
_, err := newDmInputNode(client, new(msgpb.MsgPosition), &nodeConfig{
msFactory: &mockMsgStreamFactory{},
vChannelName: "mock_vchannel_0",
})

20
internal/datanode/flow_graph_insert_buffer_node.go
View File

@ -29,11 +29,11 @@ import (
"go.uber.org/zap"
"github.com/milvus-io/milvus-proto/go-api/commonpb"
"github.com/milvus-io/milvus-proto/go-api/msgpb"
"github.com/milvus-io/milvus/internal/log"
"github.com/milvus-io/milvus/internal/metrics"
"github.com/milvus-io/milvus/internal/mq/msgstream"
"github.com/milvus-io/milvus/internal/proto/datapb"
"github.com/milvus-io/milvus/internal/proto/internalpb"
"github.com/milvus-io/milvus/internal/storage"
"github.com/milvus-io/milvus/internal/util/commonpbutil"
"github.com/milvus-io/milvus/internal/util/flowgraph"
@ -119,16 +119,16 @@ func (ibNode *insertBufferNode) Operate(in []Msg) []Msg {
fgMsg := in[0].(*flowGraphMsg)
// replace pchannel with vchannel
startPositions := make([]*internalpb.MsgPosition, 0, len(fgMsg.startPositions))
startPositions := make([]*msgpb.MsgPosition, 0, len(fgMsg.startPositions))
for idx := range fgMsg.startPositions {
pos := proto.Clone(fgMsg.startPositions[idx]).(*internalpb.MsgPosition)
pos := proto.Clone(fgMsg.startPositions[idx]).(*msgpb.MsgPosition)
pos.ChannelName = ibNode.channelName
startPositions = append(startPositions, pos)
}
fgMsg.startPositions = startPositions
endPositions := make([]*internalpb.MsgPosition, 0, len(fgMsg.endPositions))
endPositions := make([]*msgpb.MsgPosition, 0, len(fgMsg.endPositions))
for idx := range fgMsg.endPositions {
pos := proto.Clone(fgMsg.endPositions[idx]).(*internalpb.MsgPosition)
pos := proto.Clone(fgMsg.endPositions[idx]).(*msgpb.MsgPosition)
pos.ChannelName = ibNode.channelName
endPositions = append(endPositions, pos)
}
@ -449,7 +449,7 @@ func (ibNode *insertBufferNode) FillInSyncTasks(fgMsg *flowGraphMsg, seg2Upload
return syncTasks
}
func (ibNode *insertBufferNode) Sync(fgMsg *flowGraphMsg, seg2Upload []UniqueID, endPosition *internalpb.MsgPosition) []UniqueID {
func (ibNode *insertBufferNode) Sync(fgMsg *flowGraphMsg, seg2Upload []UniqueID, endPosition *msgpb.MsgPosition) []UniqueID {
syncTasks := ibNode.FillInSyncTasks(fgMsg, seg2Upload)
segmentsToSync := make([]UniqueID, 0, len(syncTasks))
@ -510,7 +510,7 @@ func (ibNode *insertBufferNode) Sync(fgMsg *flowGraphMsg, seg2Upload []UniqueID,
//
// If the segment doesn't exist, a new segment will be created.
// The segment number of rows will be updated in mem, waiting to be uploaded to DataCoord.
func (ibNode *insertBufferNode) addSegmentAndUpdateRowNum(insertMsgs []*msgstream.InsertMsg, startPos, endPos *internalpb.MsgPosition) (seg2Upload []UniqueID, err error) {
func (ibNode *insertBufferNode) addSegmentAndUpdateRowNum(insertMsgs []*msgstream.InsertMsg, startPos, endPos *msgpb.MsgPosition) (seg2Upload []UniqueID, err error) {
uniqueSeg := make(map[UniqueID]int64)
for _, msg := range insertMsgs {
@ -558,7 +558,7 @@ func (ibNode *insertBufferNode) addSegmentAndUpdateRowNum(insertMsgs []*msgstrea
// 1.2 Get buffer data and put data into each field buffer
// 1.3 Put back into buffer
// 1.4 Update related statistics
func (ibNode *insertBufferNode) bufferInsertMsg(msg *msgstream.InsertMsg, startPos, endPos *internalpb.MsgPosition) error {
func (ibNode *insertBufferNode) bufferInsertMsg(msg *msgstream.InsertMsg, startPos, endPos *msgpb.MsgPosition) error {
if err := msg.CheckAligned(); err != nil {
return err
}
@ -673,7 +673,7 @@ func newInsertBufferNode(ctx context.Context, collID UniqueID, delBufManager *De
var wTtMsgStream msgstream.MsgStream = wTt
mt := newMergedTimeTickerSender(func(ts Timestamp, segmentIDs []int64) error {
stats := make([]*datapb.SegmentStats, 0, len(segmentIDs))
stats := make([]*commonpb.SegmentStats, 0, len(segmentIDs))
for _, sid := range segmentIDs {
stat, err := config.channel.getSegmentStatisticsUpdates(sid)
if err != nil {
@ -689,7 +689,7 @@ func newInsertBufferNode(ctx context.Context, collID UniqueID, delBufManager *De
EndTimestamp: ts,
HashValues: []uint32{0},
},
DataNodeTtMsg: datapb.DataNodeTtMsg{
DataNodeTtMsg: msgpb.DataNodeTtMsg{
Base: commonpbutil.NewMsgBase(
commonpbutil.WithMsgType(commonpb.MsgType_DataNodeTt),
commonpbutil.WithMsgID(0),

66
internal/datanode/flow_graph_insert_buffer_node_test.go
View File

@ -25,14 +25,18 @@ import (
"time"
"github.com/cockroachdb/errors"
"github.com/samber/lo"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/stretchr/testify/suite"
"github.com/milvus-io/milvus-proto/go-api/commonpb"
"github.com/milvus-io/milvus-proto/go-api/milvuspb"
"github.com/milvus-io/milvus-proto/go-api/msgpb"
"github.com/milvus-io/milvus-proto/go-api/schemapb"
"github.com/milvus-io/milvus/internal/mq/msgstream"
"github.com/milvus-io/milvus/internal/proto/datapb"
"github.com/milvus-io/milvus/internal/proto/etcdpb"
"github.com/milvus-io/milvus/internal/proto/internalpb"
"github.com/milvus-io/milvus/internal/storage"
"github.com/milvus-io/milvus/internal/types"
"github.com/milvus-io/milvus/internal/util/dependency"
@ -41,10 +45,6 @@ import (
"github.com/milvus-io/milvus/internal/util/retry"
"github.com/milvus-io/milvus/internal/util/tsoutil"
"github.com/milvus-io/milvus/internal/util/typeutil"
"github.com/samber/lo"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/stretchr/testify/suite"
)
var insertNodeTestDir = "/tmp/milvus_test/insert_node"
@ -89,8 +89,8 @@ func TestFlowGraphInsertBufferNodeCreate(t *testing.T) {
segID: 1,
collID: collMeta.ID,
partitionID: 0,
startPos: &internalpb.MsgPosition{},
endPos: &internalpb.MsgPosition{},
startPos: &msgpb.MsgPosition{},
endPos: &msgpb.MsgPosition{},
})
require.NoError(t, err)
@ -193,8 +193,8 @@ func TestFlowGraphInsertBufferNode_Operate(t *testing.T) {
segID: 1,
collID: collMeta.ID,
partitionID: 0,
startPos: &internalpb.MsgPosition{},
endPos: &internalpb.MsgPosition{},
startPos: &msgpb.MsgPosition{},
endPos: &msgpb.MsgPosition{},
})
require.NoError(t, err)
@ -408,8 +408,8 @@ func TestFlowGraphInsertBufferNode_AutoFlush(t *testing.T) {
for i := range inMsg.insertMessages {
inMsg.insertMessages[i].SegmentID = int64(i%2) + 1
}
inMsg.startPositions = []*internalpb.MsgPosition{{Timestamp: 100}}
inMsg.endPositions = []*internalpb.MsgPosition{{Timestamp: 123}}
inMsg.startPositions = []*msgpb.MsgPosition{{Timestamp: 100}}
inMsg.endPositions = []*msgpb.MsgPosition{{Timestamp: 123}}
type Test struct {
expectedSegID UniqueID
@ -443,8 +443,8 @@ func TestFlowGraphInsertBufferNode_AutoFlush(t *testing.T) {
for i := range inMsg.insertMessages {
inMsg.insertMessages[i].SegmentID = int64(i%2) + 2
}
inMsg.startPositions = []*internalpb.MsgPosition{{Timestamp: 200}}
inMsg.endPositions = []*internalpb.MsgPosition{{Timestamp: 234}}
inMsg.startPositions = []*msgpb.MsgPosition{{Timestamp: 200}}
inMsg.endPositions = []*msgpb.MsgPosition{{Timestamp: 234}}
iMsg = &inMsg
// Triger auto flush
@ -507,8 +507,8 @@ func TestFlowGraphInsertBufferNode_AutoFlush(t *testing.T) {
for i := range inMsg.insertMessages {
inMsg.insertMessages[i].SegmentID = UniqueID(10 + i)
}
inMsg.startPositions = []*internalpb.MsgPosition{{Timestamp: 300}}
inMsg.endPositions = []*internalpb.MsgPosition{{Timestamp: 323}}
inMsg.startPositions = []*msgpb.MsgPosition{{Timestamp: 300}}
inMsg.endPositions = []*msgpb.MsgPosition{{Timestamp: 323}}
var iMsg flowgraph.Msg = &inMsg
type Test struct {
@ -540,8 +540,8 @@ func TestFlowGraphInsertBufferNode_AutoFlush(t *testing.T) {
assert.Equal(t, test.expectedStartPosTs, seg.startPos.GetTimestamp())
}
inMsg.startPositions = []*internalpb.MsgPosition{{Timestamp: 400}}
inMsg.endPositions = []*internalpb.MsgPosition{{Timestamp: 434}}
inMsg.startPositions = []*msgpb.MsgPosition{{Timestamp: 400}}
inMsg.endPositions = []*msgpb.MsgPosition{{Timestamp: 434}}
// trigger manual flush
flushChan <- flushMsg{segmentID: 10}
@ -644,8 +644,8 @@ func TestRollBF(t *testing.T) {
Params.DataNodeCfg.FlushInsertBufferSize = tmp
}()
inMsg.startPositions = []*internalpb.MsgPosition{{Timestamp: 100}}
inMsg.endPositions = []*internalpb.MsgPosition{{Timestamp: 123}}
inMsg.startPositions = []*msgpb.MsgPosition{{Timestamp: 100}}
inMsg.endPositions = []*msgpb.MsgPosition{{Timestamp: 123}}
type Test struct {
expectedSegID UniqueID
@ -671,8 +671,8 @@ func TestRollBF(t *testing.T) {
// because this is the origincal
assert.True(t, seg.currentStat.PkFilter.Cap() > uint(1000000))
inMsg.startPositions = []*internalpb.MsgPosition{{Timestamp: 200}}
inMsg.endPositions = []*internalpb.MsgPosition{{Timestamp: 234}}
inMsg.startPositions = []*msgpb.MsgPosition{{Timestamp: 200}}
inMsg.endPositions = []*msgpb.MsgPosition{{Timestamp: 234}}
iMsg = &inMsg
// Triger auto flush
@ -760,8 +760,8 @@ func (s *InsertBufferNodeSuite) SetupTest() {
segID: seg.segID,
collID: s.collID,
partitionID: s.partID,
startPos: new(internalpb.MsgPosition),
endPos: new(internalpb.MsgPosition),
startPos: new(msgpb.MsgPosition),
endPos: new(msgpb.MsgPosition),
})
s.Require().NoError(err)
}
@ -809,7 +809,7 @@ func (s *InsertBufferNodeSuite) TestFillInSyncTasks() {
}
node.channel.setCurInsertBuffer(UniqueID(1), &buffer)
syncTasks := node.FillInSyncTasks(&flowGraphMsg{endPositions: []*internalpb.MsgPosition{{Timestamp: 100}}}, segToFlush)
syncTasks := node.FillInSyncTasks(&flowGraphMsg{endPositions: []*msgpb.MsgPosition{{Timestamp: 100}}}, segToFlush)
s.Assert().NotEmpty(syncTasks)
s.Assert().Equal(1, len(syncTasks))
@ -822,7 +822,7 @@ func (s *InsertBufferNodeSuite) TestFillInSyncTasks() {
})
s.Run("drop partition", func() {
fgMsg := flowGraphMsg{dropPartitions: []UniqueID{s.partID}, endPositions: []*internalpb.MsgPosition{{Timestamp: 100}}}
fgMsg := flowGraphMsg{dropPartitions: []UniqueID{s.partID}, endPositions: []*msgpb.MsgPosition{{Timestamp: 100}}}
node := &insertBufferNode{
channelName: s.channel.channelName,
channel: s.channel,
@ -859,7 +859,7 @@ func (s *InsertBufferNodeSuite) TestFillInSyncTasks() {
flushCh <- msg
}
syncTasks := node.FillInSyncTasks(&flowGraphMsg{endPositions: []*internalpb.MsgPosition{{Timestamp: 100}}}, nil)
syncTasks := node.FillInSyncTasks(&flowGraphMsg{endPositions: []*msgpb.MsgPosition{{Timestamp: 100}}}, nil)
s.Assert().NotEmpty(syncTasks)
for _, task := range syncTasks {
@ -887,7 +887,7 @@ func (s *InsertBufferNodeSuite) TestFillInSyncTasks() {
flushCh <- msg
}
syncTasks := node.FillInSyncTasks(&flowGraphMsg{endPositions: []*internalpb.MsgPosition{{Timestamp: 100}}}, nil)
syncTasks := node.FillInSyncTasks(&flowGraphMsg{endPositions: []*msgpb.MsgPosition{{Timestamp: 100}}}, nil)
s.Assert().NotEmpty(syncTasks)
s.Assert().Equal(10, len(syncTasks)) // 10 is max batch
@ -982,8 +982,8 @@ func TestInsertBufferNode_bufferInsertMsg(t *testing.T) {
segID: 1,
collID: collMeta.ID,
partitionID: 0,
startPos: &internalpb.MsgPosition{},
endPos: &internalpb.MsgPosition{Timestamp: 101},
startPos: &msgpb.MsgPosition{},
endPos: &msgpb.MsgPosition{Timestamp: 101},
})
require.NoError(t, err)
@ -1010,14 +1010,14 @@ func TestInsertBufferNode_bufferInsertMsg(t *testing.T) {
inMsg := genFlowGraphInsertMsg(insertChannelName)
for _, msg := range inMsg.insertMessages {
msg.EndTimestamp = 101 // ts valid
err = iBNode.bufferInsertMsg(msg, &internalpb.MsgPosition{}, &internalpb.MsgPosition{})
err = iBNode.bufferInsertMsg(msg, &msgpb.MsgPosition{}, &msgpb.MsgPosition{})
assert.Nil(t, err)
}
for _, msg := range inMsg.insertMessages {
msg.EndTimestamp = 101 // ts valid
msg.RowIDs = []int64{} //misaligned data
err = iBNode.bufferInsertMsg(msg, &internalpb.MsgPosition{}, &internalpb.MsgPosition{})
err = iBNode.bufferInsertMsg(msg, &msgpb.MsgPosition{}, &msgpb.MsgPosition{})
assert.NotNil(t, err)
}
}
@ -1047,14 +1047,14 @@ func TestInsertBufferNode_updateSegmentStates(te *testing.T) {
im := []*msgstream.InsertMsg{
{
InsertRequest: internalpb.InsertRequest{
InsertRequest: msgpb.InsertRequest{
CollectionID: test.inCollID,
SegmentID: test.segID,
},
},
}
seg, err := ibNode.addSegmentAndUpdateRowNum(im, &internalpb.MsgPosition{}, &internalpb.MsgPosition{})
seg, err := ibNode.addSegmentAndUpdateRowNum(im, &msgpb.MsgPosition{}, &msgpb.MsgPosition{})
assert.Error(te, err)
assert.Empty(te, seg)

18
internal/datanode/flow_graph_manager_test.go
View File

@ -20,13 +20,13 @@ import (
"context"
"testing"
"github.com/milvus-io/milvus-proto/go-api/schemapb"
"github.com/milvus-io/milvus/internal/proto/datapb"
"github.com/milvus-io/milvus/internal/proto/internalpb"
"github.com/milvus-io/milvus/internal/util/etcd"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/milvus-io/milvus-proto/go-api/msgpb"
"github.com/milvus-io/milvus-proto/go-api/schemapb"
"github.com/milvus-io/milvus/internal/proto/datapb"
"github.com/milvus-io/milvus/internal/util/etcd"
)
func TestFlowGraphManager(t *testing.T) {
@ -107,8 +107,8 @@ func TestFlowGraphManager(t *testing.T) {
segID: 100,
collID: 1,
partitionID: 10,
startPos: &internalpb.MsgPosition{},
endPos: &internalpb.MsgPosition{},
startPos: &msgpb.MsgPosition{},
endPos: &msgpb.MsgPosition{},
})
require.NoError(t, err)
@ -156,8 +156,8 @@ func TestFlowGraphManager(t *testing.T) {
segID: 100,
collID: 1,
partitionID: 10,
startPos: &internalpb.MsgPosition{},
endPos: &internalpb.MsgPosition{},
startPos: &msgpb.MsgPosition{},
endPos: &msgpb.MsgPosition{},
})
require.NoError(t, err)

6
internal/datanode/flow_graph_message.go
View File

@ -17,8 +17,8 @@
package datanode
import (
"github.com/milvus-io/milvus-proto/go-api/msgpb"
"github.com/milvus-io/milvus/internal/mq/msgstream"
"github.com/milvus-io/milvus/internal/proto/internalpb"
"github.com/milvus-io/milvus/internal/storage"
"github.com/milvus-io/milvus/internal/util/flowgraph"
)
@ -47,8 +47,8 @@ type flowGraphMsg struct {
insertMessages []*msgstream.InsertMsg
deleteMessages []*msgstream.DeleteMsg
timeRange TimeRange
startPositions []*internalpb.MsgPosition
endPositions []*internalpb.MsgPosition
startPositions []*msgpb.MsgPosition
endPositions []*msgpb.MsgPosition
//segmentsToSync is the signal used by insertBufferNode to notify deleteNode to flush
segmentsToSync []UniqueID
dropCollection bool

4
internal/datanode/flow_graph_time_tick_node.go
View File

@ -24,9 +24,9 @@ import (
"go.uber.org/zap"
"github.com/milvus-io/milvus-proto/go-api/msgpb"
"github.com/milvus-io/milvus/internal/log"
"github.com/milvus-io/milvus/internal/proto/datapb"
"github.com/milvus-io/milvus/internal/proto/internalpb"
"github.com/milvus-io/milvus/internal/types"
"github.com/milvus-io/milvus/internal/util/commonpbutil"
"github.com/milvus-io/milvus/internal/util/flowgraph"
@ -84,7 +84,7 @@ func (ttn *ttNode) Operate(in []Msg) []Msg {
return []Msg{}
}
func (ttn *ttNode) updateChannelCP(ttPos *internalpb.MsgPosition) {
func (ttn *ttNode) updateChannelCP(ttPos *msgpb.MsgPosition) {
channelPos := ttn.channel.getChannelCheckpoint(ttPos)
if channelPos == nil || channelPos.MsgID == nil {
log.Warn("updateChannelCP failed, get nil check point", zap.String("vChannel", ttn.vChannelName))

29
internal/datanode/flush_manager.go
View File

@ -24,32 +24,31 @@ import (
"sync"
"github.com/cockroachdb/errors"
"github.com/samber/lo"
"go.uber.org/atomic"
"go.uber.org/zap"
"github.com/milvus-io/milvus-proto/go-api/commonpb"
"github.com/milvus-io/milvus-proto/go-api/msgpb"
"github.com/milvus-io/milvus/internal/common"
"github.com/milvus-io/milvus/internal/log"
"github.com/milvus-io/milvus/internal/metrics"
"github.com/milvus-io/milvus/internal/proto/datapb"
"github.com/milvus-io/milvus/internal/proto/etcdpb"
"github.com/milvus-io/milvus/internal/proto/internalpb"
"github.com/milvus-io/milvus/internal/storage"
"github.com/milvus-io/milvus/internal/util/commonpbutil"
"github.com/milvus-io/milvus/internal/util/metautil"
"github.com/milvus-io/milvus/internal/util/paramtable"
"github.com/milvus-io/milvus/internal/util/retry"
"github.com/milvus-io/milvus/internal/util/timerecord"
"github.com/samber/lo"
)
// flushManager defines a flush manager signature
type flushManager interface {
// notify flush manager insert buffer data
flushBufferData(data *BufferData, segmentID UniqueID, flushed bool, dropped bool, pos *internalpb.MsgPosition) ([]*Blob, error)
flushBufferData(data *BufferData, segmentID UniqueID, flushed bool, dropped bool, pos *msgpb.MsgPosition) ([]*Blob, error)
// notify flush manager del buffer data
flushDelData(data *DelDataBuf, segmentID UniqueID, pos *internalpb.MsgPosition) error
flushDelData(data *DelDataBuf, segmentID UniqueID, pos *msgpb.MsgPosition) error
// injectFlush injects compaction or other blocking task before flush sync
injectFlush(injection *taskInjection, segments ...UniqueID)
// startDropping changes flush manager into dropping mode
@ -66,7 +65,7 @@ type segmentFlushPack struct {
insertLogs map[UniqueID]*datapb.Binlog
statsLogs map[UniqueID]*datapb.Binlog
deltaLogs []*datapb.Binlog
pos *internalpb.MsgPosition
pos *msgpb.MsgPosition
flushed bool
dropped bool
err error // task execution error, if not nil, notify func should stop datanode
@ -124,7 +123,7 @@ func (q *orderFlushQueue) init() {
})
}
func (q *orderFlushQueue) getFlushTaskRunner(pos *internalpb.MsgPosition) *flushTaskRunner {
func (q *orderFlushQueue) getFlushTaskRunner(pos *msgpb.MsgPosition) *flushTaskRunner {
actual, loaded := q.working.LoadOrStore(getSyncTaskID(pos), newFlushTaskRunner(q.segmentID, q.injectCh))
t := actual.(*flushTaskRunner)
// not loaded means the task runner is new, do initializtion
@ -174,12 +173,12 @@ func (q *orderFlushQueue) postTask(pack *segmentFlushPack, postInjection postInj
}
// enqueueInsertBuffer put insert buffer data into queue
func (q *orderFlushQueue) enqueueInsertFlush(task flushInsertTask, binlogs, statslogs map[UniqueID]*datapb.Binlog, flushed bool, dropped bool, pos *internalpb.MsgPosition) {
func (q *orderFlushQueue) enqueueInsertFlush(task flushInsertTask, binlogs, statslogs map[UniqueID]*datapb.Binlog, flushed bool, dropped bool, pos *msgpb.MsgPosition) {
q.getFlushTaskRunner(pos).runFlushInsert(task, binlogs, statslogs, flushed, dropped, pos)
}
// enqueueDelBuffer put delete buffer data into queue
func (q *orderFlushQueue) enqueueDelFlush(task flushDeleteTask, deltaLogs *DelDataBuf, pos *internalpb.MsgPosition) {
func (q *orderFlushQueue) enqueueDelFlush(task flushDeleteTask, deltaLogs *DelDataBuf, pos *msgpb.MsgPosition) {
q.getFlushTaskRunner(pos).runFlushDel(task, deltaLogs)
}
@ -285,7 +284,7 @@ func (m *rendezvousFlushManager) getFlushQueue(segmentID UniqueID) *orderFlushQu
return queue
}
func (m *rendezvousFlushManager) handleInsertTask(segmentID UniqueID, task flushInsertTask, binlogs, statslogs map[UniqueID]*datapb.Binlog, flushed bool, dropped bool, pos *internalpb.MsgPosition) {
func (m *rendezvousFlushManager) handleInsertTask(segmentID UniqueID, task flushInsertTask, binlogs, statslogs map[UniqueID]*datapb.Binlog, flushed bool, dropped bool, pos *msgpb.MsgPosition) {
log.Info("handling insert task",
zap.Int64("segment ID", segmentID),
zap.Bool("flushed", flushed),
@ -312,7 +311,7 @@ func (m *rendezvousFlushManager) handleInsertTask(segmentID UniqueID, task flush
m.getFlushQueue(segmentID).enqueueInsertFlush(task, binlogs, statslogs, flushed, dropped, pos)
}
func (m *rendezvousFlushManager) handleDeleteTask(segmentID UniqueID, task flushDeleteTask, deltaLogs *DelDataBuf, pos *internalpb.MsgPosition) {
func (m *rendezvousFlushManager) handleDeleteTask(segmentID UniqueID, task flushDeleteTask, deltaLogs *DelDataBuf, pos *msgpb.MsgPosition) {
log.Info("handling delete task", zap.Int64("segment ID", segmentID))
// in dropping mode
if m.dropping.Load() {
@ -342,7 +341,7 @@ func (m *rendezvousFlushManager) handleDeleteTask(segmentID UniqueID, task flush
// flushBufferData notifies flush manager insert buffer data.
// This method will be retired on errors. Final errors will be propagated upstream and logged.
func (m *rendezvousFlushManager) flushBufferData(data *BufferData, segmentID UniqueID, flushed bool, dropped bool, pos *internalpb.MsgPosition) ([]*Blob, error) {
func (m *rendezvousFlushManager) flushBufferData(data *BufferData, segmentID UniqueID, flushed bool, dropped bool, pos *msgpb.MsgPosition) ([]*Blob, error) {
tr := timerecord.NewTimeRecorder("flushDuration")
// empty flush
if data == nil || data.buffer == nil {
@ -439,7 +438,7 @@ func (m *rendezvousFlushManager) flushBufferData(data *BufferData, segmentID Uni
// notify flush manager del buffer data
func (m *rendezvousFlushManager) flushDelData(data *DelDataBuf, segmentID UniqueID,
pos *internalpb.MsgPosition) error {
pos *msgpb.MsgPosition) error {
// del signal with empty data
if data == nil || data.delData == nil {
@ -487,7 +486,7 @@ func (m *rendezvousFlushManager) injectFlush(injection *taskInjection, segments
}
// fetch meta info for segment
func (m *rendezvousFlushManager) getSegmentMeta(segmentID UniqueID, pos *internalpb.MsgPosition) (UniqueID, UniqueID, *etcdpb.CollectionMeta, error) {
func (m *rendezvousFlushManager) getSegmentMeta(segmentID UniqueID, pos *msgpb.MsgPosition) (UniqueID, UniqueID, *etcdpb.CollectionMeta, error) {
if !m.hasSegment(segmentID, true) {
return -1, -1, nil, fmt.Errorf("no such segment %d in the channel", segmentID)
}
@ -552,7 +551,7 @@ func (m *rendezvousFlushManager) notifyAllFlushed() {
close(m.dropHandler.allFlushed)
}
func getSyncTaskID(pos *internalpb.MsgPosition) string {
func getSyncTaskID(pos *msgpb.MsgPosition) string {
// use msgID & timestamp to generate unique taskID, see also #20926
return fmt.Sprintf("%s%d", string(pos.GetMsgID()), pos.GetTimestamp())
}

81
internal/datanode/flush_manager_test.go
View File

@ -24,16 +24,16 @@ import (
"time"
"github.com/cockroachdb/errors"
"github.com/milvus-io/milvus-proto/go-api/commonpb"
"github.com/milvus-io/milvus-proto/go-api/schemapb"
"github.com/milvus-io/milvus/internal/proto/datapb"
"github.com/milvus-io/milvus/internal/proto/internalpb"
"github.com/milvus-io/milvus/internal/storage"
"github.com/milvus-io/milvus/internal/util/retry"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"go.uber.org/atomic"
"github.com/milvus-io/milvus-proto/go-api/commonpb"
"github.com/milvus-io/milvus-proto/go-api/msgpb"
"github.com/milvus-io/milvus-proto/go-api/schemapb"
"github.com/milvus-io/milvus/internal/proto/datapb"
"github.com/milvus-io/milvus/internal/storage"
"github.com/milvus-io/milvus/internal/util/retry"
)
var flushTestDir = "/tmp/milvus_test/flush"
@ -81,13 +81,13 @@ func TestOrderFlushQueue_Execute(t *testing.T) {
wg.Add(2 * size)
for i := 0; i < size; i++ {
go func(id []byte) {
q.enqueueDelFlush(&emptyFlushTask{}, &DelDataBuf{}, &internalpb.MsgPosition{
q.enqueueDelFlush(&emptyFlushTask{}, &DelDataBuf{}, &msgpb.MsgPosition{
MsgID: id,
})
wg.Done()
}(ids[i])
go func(id []byte) {
q.enqueueInsertFlush(&emptyFlushTask{}, map[UniqueID]*datapb.Binlog{}, map[UniqueID]*datapb.Binlog{}, false, false, &internalpb.MsgPosition{
q.enqueueInsertFlush(&emptyFlushTask{}, map[UniqueID]*datapb.Binlog{}, map[UniqueID]*datapb.Binlog{}, false, false, &msgpb.MsgPosition{
MsgID: id,
})
wg.Done()
@ -98,6 +98,7 @@ func TestOrderFlushQueue_Execute(t *testing.T) {
assert.EqualValues(t, size, counter.Load())
}
func TestOrderFlushQueue_Order(t *testing.T) {
counter := atomic.Int64{}
finish := sync.WaitGroup{}
@ -122,10 +123,10 @@ func TestOrderFlushQueue_Order(t *testing.T) {
wg := sync.WaitGroup{}
wg.Add(size)
for i := 0; i < size; i++ {
q.enqueueDelFlush(&emptyFlushTask{}, &DelDataBuf{}, &internalpb.MsgPosition{
q.enqueueDelFlush(&emptyFlushTask{}, &DelDataBuf{}, &msgpb.MsgPosition{
MsgID: ids[i],
})
q.enqueueInsertFlush(&emptyFlushTask{}, map[UniqueID]*datapb.Binlog{}, map[UniqueID]*datapb.Binlog{}, false, false, &internalpb.MsgPosition{
q.enqueueInsertFlush(&emptyFlushTask{}, map[UniqueID]*datapb.Binlog{}, map[UniqueID]*datapb.Binlog{}, false, false, &msgpb.MsgPosition{
MsgID: ids[i],
})
wg.Done()
@ -172,10 +173,10 @@ func TestRendezvousFlushManager(t *testing.T) {
wg := sync.WaitGroup{}
wg.Add(size)
for i := 0; i < size; i++ {
m.flushDelData(nil, 1, &internalpb.MsgPosition{
m.flushDelData(nil, 1, &msgpb.MsgPosition{
MsgID: ids[i],
})
m.flushBufferData(nil, 1, true, false, &internalpb.MsgPosition{
m.flushBufferData(nil, 1, true, false, &msgpb.MsgPosition{
MsgID: ids[i],
})
wg.Done()
@ -221,10 +222,10 @@ func TestRendezvousFlushManager_Inject(t *testing.T) {
wg := sync.WaitGroup{}
wg.Add(size)
for i := 0; i < size; i++ {
m.flushDelData(nil, 1, &internalpb.MsgPosition{
m.flushDelData(nil, 1, &msgpb.MsgPosition{
MsgID: ids[i],
})
m.flushBufferData(nil, 1, true, false, &internalpb.MsgPosition{
m.flushBufferData(nil, 1, true, false, &msgpb.MsgPosition{
MsgID: ids[i],
})
wg.Done()
@ -239,10 +240,10 @@ func TestRendezvousFlushManager_Inject(t *testing.T) {
rand.Read(id)
id2 := make([]byte, 10)
rand.Read(id2)
m.flushBufferData(nil, 2, true, false, &internalpb.MsgPosition{
m.flushBufferData(nil, 2, true, false, &msgpb.MsgPosition{
MsgID: id,
})
m.flushBufferData(nil, 3, true, false, &internalpb.MsgPosition{
m.flushBufferData(nil, 3, true, false, &msgpb.MsgPosition{
MsgID: id2,
})
@ -251,10 +252,10 @@ func TestRendezvousFlushManager_Inject(t *testing.T) {
})
m.injectFlush(ti, 2, 3)
m.flushDelData(nil, 2, &internalpb.MsgPosition{
m.flushDelData(nil, 2, &msgpb.MsgPosition{
MsgID: id,
})
m.flushDelData(nil, 3, &internalpb.MsgPosition{
m.flushDelData(nil, 3, &msgpb.MsgPosition{
MsgID: id2,
})
<-ti.Injected()
@ -267,7 +268,7 @@ func TestRendezvousFlushManager_Inject(t *testing.T) {
finish.Add(1)
rand.Read(id)
m.flushBufferData(nil, 2, false, false, &internalpb.MsgPosition{
m.flushBufferData(nil, 2, false, false, &msgpb.MsgPosition{
MsgID: id,
})
ti = newTaskInjection(1, func(pack *segmentFlushPack) {
@ -279,7 +280,7 @@ func TestRendezvousFlushManager_Inject(t *testing.T) {
}()
m.injectFlush(ti, 2)
m.flushDelData(nil, 2, &internalpb.MsgPosition{
m.flushDelData(nil, 2, &msgpb.MsgPosition{
MsgID: id,
})
finish.Wait()
@ -300,7 +301,7 @@ func TestRendezvousFlushManager_getSegmentMeta(t *testing.T) {
}, emptyFlushAndDropFunc)
// non exists segment
_, _, _, err := fm.getSegmentMeta(-1, &internalpb.MsgPosition{})
_, _, _, err := fm.getSegmentMeta(-1, &msgpb.MsgPosition{})
assert.Error(t, err)
seg0 := Segment{segmentID: -1}
@ -312,10 +313,10 @@ func TestRendezvousFlushManager_getSegmentMeta(t *testing.T) {
channel.segments[1] = &seg1
// // injected get part/coll id error
// _, _, _, err = fm.getSegmentMeta(-1, &internalpb.MsgPosition{})
// _, _, _, err = fm.getSegmentMeta(-1, &msgpb.MsgPosition{})
// assert.Error(t, err)
// // injected get schema error
// _, _, _, err = fm.getSegmentMeta(1, &internalpb.MsgPosition{})
// _, _, _, err = fm.getSegmentMeta(1, &msgpb.MsgPosition{})
// assert.Error(t, err)
}
@ -342,7 +343,7 @@ func TestRendezvousFlushManager_waitForAllFlushQueue(t *testing.T) {
}
for i := 0; i < size; i++ {
m.flushDelData(nil, 1, &internalpb.MsgPosition{
m.flushDelData(nil, 1, &msgpb.MsgPosition{
MsgID: ids[i],
})
}
@ -364,7 +365,7 @@ func TestRendezvousFlushManager_waitForAllFlushQueue(t *testing.T) {
mut.RUnlock()
for i := 0; i < size/2; i++ {
m.flushBufferData(nil, 1, true, false, &internalpb.MsgPosition{
m.flushBufferData(nil, 1, true, false, &msgpb.MsgPosition{
MsgID: ids[i],
})
}
@ -374,7 +375,7 @@ func TestRendezvousFlushManager_waitForAllFlushQueue(t *testing.T) {
mut.RUnlock()
for i := size / 2; i < size; i++ {
m.flushBufferData(nil, 1, true, false, &internalpb.MsgPosition{
m.flushBufferData(nil, 1, true, false, &msgpb.MsgPosition{
MsgID: ids[i],
})
}
@ -410,23 +411,23 @@ func TestRendezvousFlushManager_dropMode(t *testing.T) {
})
halfMsgID := []byte{1, 1, 1}
m.flushBufferData(nil, -1, true, false, &internalpb.MsgPosition{
m.flushBufferData(nil, -1, true, false, &msgpb.MsgPosition{
MsgID: halfMsgID,
})
m.startDropping()
// half normal, half drop mode, should not appear in final packs
m.flushDelData(nil, -1, &internalpb.MsgPosition{
m.flushDelData(nil, -1, &msgpb.MsgPosition{
MsgID: halfMsgID,
})
target := make(map[int64]struct{})
for i := 1; i < 11; i++ {
target[int64(i)] = struct{}{}
m.flushBufferData(nil, int64(i), true, false, &internalpb.MsgPosition{
m.flushBufferData(nil, int64(i), true, false, &msgpb.MsgPosition{
MsgID: []byte{byte(i)},
})
m.flushDelData(nil, int64(i), &internalpb.MsgPosition{
m.flushDelData(nil, int64(i), &msgpb.MsgPosition{
MsgID: []byte{byte(i)},
})
t.Log(i)
@ -464,7 +465,7 @@ func TestRendezvousFlushManager_dropMode(t *testing.T) {
})
halfMsgID := []byte{1, 1, 1}
m.flushBufferData(nil, -1, true, false, &internalpb.MsgPosition{
m.flushBufferData(nil, -1, true, false, &msgpb.MsgPosition{
MsgID: halfMsgID,
})
@ -480,15 +481,15 @@ func TestRendezvousFlushManager_dropMode(t *testing.T) {
m.startDropping()
// half normal, half drop mode, should not appear in final packs
m.flushDelData(nil, -1, &internalpb.MsgPosition{
m.flushDelData(nil, -1, &msgpb.MsgPosition{
MsgID: halfMsgID,
})
for i := 1; i < 11; i++ {
m.flushBufferData(nil, int64(i), true, false, &internalpb.MsgPosition{
m.flushBufferData(nil, int64(i), true, false, &msgpb.MsgPosition{
MsgID: []byte{byte(i)},
})
m.flushDelData(nil, int64(i), &internalpb.MsgPosition{
m.flushDelData(nil, int64(i), &msgpb.MsgPosition{
MsgID: []byte{byte(i)},
})
}
@ -532,10 +533,10 @@ func TestRendezvousFlushManager_close(t *testing.T) {
wg := sync.WaitGroup{}
wg.Add(size)
for i := 0; i < size; i++ {
m.flushDelData(nil, 1, &internalpb.MsgPosition{
m.flushDelData(nil, 1, &msgpb.MsgPosition{
MsgID: ids[i],
})
m.flushBufferData(nil, 1, true, false, &internalpb.MsgPosition{
m.flushBufferData(nil, 1, true, false, &msgpb.MsgPosition{
MsgID: ids[i],
})
wg.Done()
@ -648,7 +649,7 @@ func TestDropVirtualChannelFunc(t *testing.T) {
segID: 2,
collID: 1,
partitionID: 10,
startPos: &internalpb.MsgPosition{
startPos: &msgpb.MsgPosition{
ChannelName: vchanName,
MsgID: []byte{1, 2, 3},
Timestamp: 10,
@ -660,7 +661,7 @@ func TestDropVirtualChannelFunc(t *testing.T) {
insertLogs: map[UniqueID]*datapb.Binlog{1: {LogPath: "/dev/test/id"}},
statsLogs: map[UniqueID]*datapb.Binlog{1: {LogPath: "/dev/test/id-stats"}},
deltaLogs: []*datapb.Binlog{{LogPath: "/dev/test/del"}},
pos: &internalpb.MsgPosition{
pos: &msgpb.MsgPosition{
ChannelName: vchanName,
MsgID: []byte{1, 2, 3},
Timestamp: 10,
@ -671,7 +672,7 @@ func TestDropVirtualChannelFunc(t *testing.T) {
insertLogs: map[UniqueID]*datapb.Binlog{1: {LogPath: "/dev/test/idi_2"}},
statsLogs: map[UniqueID]*datapb.Binlog{1: {LogPath: "/dev/test/id-stats-2"}},
deltaLogs: []*datapb.Binlog{{LogPath: "/dev/test/del-2"}},
pos: &internalpb.MsgPosition{
pos: &msgpb.MsgPosition{
ChannelName: vchanName,
MsgID: []byte{1, 2, 3},
Timestamp: 30,

7
internal/datanode/flush_task.go
View File

@ -21,13 +21,12 @@ import (
"sync"
"github.com/cockroachdb/errors"
"go.uber.org/zap"
"github.com/milvus-io/milvus-proto/go-api/msgpb"
"github.com/milvus-io/milvus/internal/kv"
"github.com/milvus-io/milvus/internal/log"
"github.com/milvus-io/milvus/internal/proto/datapb"
"github.com/milvus-io/milvus/internal/proto/internalpb"
"github.com/milvus-io/milvus/internal/util/retry"
"github.com/milvus-io/milvus/internal/util/tsoutil"
)
@ -65,7 +64,7 @@ type flushTaskRunner struct {
insertLogs map[UniqueID]*datapb.Binlog
statsLogs map[UniqueID]*datapb.Binlog
deltaLogs []*datapb.Binlog //[]*DelDataBuf
pos *internalpb.MsgPosition
pos *msgpb.MsgPosition
flushed bool
dropped bool
@ -126,7 +125,7 @@ func (t *flushTaskRunner) init(f notifyMetaFunc, postFunc taskPostFunc, signal <
// runFlushInsert executes flush insert task with once and retry
func (t *flushTaskRunner) runFlushInsert(task flushInsertTask,
binlogs, statslogs map[UniqueID]*datapb.Binlog, flushed bool, dropped bool, pos *internalpb.MsgPosition, opts ...retry.Option) {
binlogs, statslogs map[UniqueID]*datapb.Binlog, flushed bool, dropped bool, pos *msgpb.MsgPosition, opts ...retry.Option) {
t.insertOnce.Do(func() {
t.insertLogs = binlogs
t.statsLogs = statslogs

20
internal/datanode/mock_test.go
View File

@ -27,11 +27,11 @@ import (
"time"
"github.com/cockroachdb/errors"
"go.uber.org/zap"
"github.com/milvus-io/milvus-proto/go-api/commonpb"
"github.com/milvus-io/milvus-proto/go-api/milvuspb"
"github.com/milvus-io/milvus-proto/go-api/msgpb"
"github.com/milvus-io/milvus-proto/go-api/schemapb"
"github.com/milvus-io/milvus/internal/common"
"github.com/milvus-io/milvus/internal/kv"
@ -41,7 +41,6 @@ import (
"github.com/milvus-io/milvus/internal/mq/msgstream"
"github.com/milvus-io/milvus/internal/proto/datapb"
"github.com/milvus-io/milvus/internal/proto/etcdpb"
"github.com/milvus-io/milvus/internal/proto/internalpb"
"github.com/milvus-io/milvus/internal/proto/rootcoordpb"
s "github.com/milvus-io/milvus/internal/storage"
"github.com/milvus-io/milvus/internal/types"
@ -55,6 +54,7 @@ import (
)
const ctxTimeInMillisecond = 5000
const debug = false
// As used in data_sync_service_test.go
@ -781,7 +781,7 @@ func (df *DataFactory) GenMsgStreamInsertMsg(idx int, chanName string) *msgstrea
BaseMsg: msgstream.BaseMsg{
HashValues: []uint32{uint32(idx)},
},
InsertRequest: internalpb.InsertRequest{
InsertRequest: msgpb.InsertRequest{
Base: &commonpb.MsgBase{
MsgType: commonpb.MsgType_Insert,
MsgID: 0,
@ -797,7 +797,7 @@ func (df *DataFactory) GenMsgStreamInsertMsg(idx int, chanName string) *msgstrea
RowIDs: []UniqueID{UniqueID(idx)},
// RowData: []*commonpb.Blob{{Value: df.rawData}},
FieldsData: df.columnData,
Version: internalpb.InsertDataVersion_ColumnBased,
Version: msgpb.InsertDataVersion_ColumnBased,
NumRows: 1,
},
}
@ -811,7 +811,7 @@ func (df *DataFactory) GenMsgStreamInsertMsgWithTs(idx int, chanName string, ts
BeginTimestamp: ts,
EndTimestamp: ts,
},
InsertRequest: internalpb.InsertRequest{
InsertRequest: msgpb.InsertRequest{
Base: &commonpb.MsgBase{
MsgType: commonpb.MsgType_Insert,
MsgID: 0,
@ -827,7 +827,7 @@ func (df *DataFactory) GenMsgStreamInsertMsgWithTs(idx int, chanName string, ts
RowIDs: []UniqueID{UniqueID(idx)},
// RowData: []*commonpb.Blob{{Value: df.rawData}},
FieldsData: df.columnData,
Version: internalpb.InsertDataVersion_ColumnBased,
Version: msgpb.InsertDataVersion_ColumnBased,
NumRows: 1,
},
}
@ -861,7 +861,7 @@ func (df *DataFactory) GenMsgStreamDeleteMsg(pks []primaryKey, chanName string)
BaseMsg: msgstream.BaseMsg{
HashValues: []uint32{uint32(idx)},
},
DeleteRequest: internalpb.DeleteRequest{
DeleteRequest: msgpb.DeleteRequest{
Base: &commonpb.MsgBase{
MsgType: commonpb.MsgType_Delete,
MsgID: 0,
@ -887,7 +887,7 @@ func (df *DataFactory) GenMsgStreamDeleteMsgWithTs(idx int, pks []primaryKey, ch
BeginTimestamp: ts,
EndTimestamp: ts,
},
DeleteRequest: internalpb.DeleteRequest{
DeleteRequest: msgpb.DeleteRequest{
Base: &commonpb.MsgBase{
MsgType: commonpb.MsgType_Delete,
MsgID: 1,
@ -913,7 +913,7 @@ func genFlowGraphInsertMsg(chanName string) flowGraphMsg {
timestampMax: math.MaxUint64,
}
startPos := []*internalpb.MsgPosition{
startPos := []*msgpb.MsgPosition{
{
ChannelName: chanName,
MsgID: make([]byte, 0),
@ -943,7 +943,7 @@ func genFlowGraphDeleteMsg(pks []primaryKey, chanName string) flowGraphMsg {
timestampMax: math.MaxUint64,
}
startPos := []*internalpb.MsgPosition{
startPos := []*msgpb.MsgPosition{
{
ChannelName: chanName,
MsgID: make([]byte, 0),

10
internal/datanode/segment.go
View File

@ -20,12 +20,12 @@ import (
"sync"
"sync/atomic"
"github.com/bits-and-blooms/bloom/v3"
bloom "github.com/bits-and-blooms/bloom/v3"
"go.uber.org/zap"
"github.com/milvus-io/milvus-proto/go-api/msgpb"
"github.com/milvus-io/milvus/internal/log"
"github.com/milvus-io/milvus/internal/proto/datapb"
"github.com/milvus-io/milvus/internal/proto/internalpb"
"github.com/milvus-io/milvus/internal/storage"
"github.com/milvus-io/milvus/internal/util/tsoutil"
)
@ -51,7 +51,7 @@ type Segment struct {
historyStats []*storage.PkStatistics
lastSyncTs Timestamp
startPos *internalpb.MsgPosition // TODO readonly
startPos *msgpb.MsgPosition // TODO readonly
}
func (s *Segment) isValid() bool {
@ -115,7 +115,7 @@ func (s *Segment) rollInsertBuffer() {
}
// evictHistoryInsertBuffer removes flushed buffer from historyInsertBuf after saveBinlogPath.
func (s *Segment) evictHistoryInsertBuffer(endPos *internalpb.MsgPosition) {
func (s *Segment) evictHistoryInsertBuffer(endPos *msgpb.MsgPosition) {
tmpBuffers := make([]*BufferData, 0)
for _, buf := range s.historyInsertBuf {
if buf.endPos.Timestamp > endPos.Timestamp {
@ -138,7 +138,7 @@ func (s *Segment) rollDeleteBuffer() {
}
// evictHistoryDeleteBuffer removes flushed buffer from historyDeleteBuf after saveBinlogPath.
func (s *Segment) evictHistoryDeleteBuffer(endPos *internalpb.MsgPosition) {
func (s *Segment) evictHistoryDeleteBuffer(endPos *msgpb.MsgPosition) {
tmpBuffers := make([]*DelDataBuf, 0)
for _, buf := range s.historyDeleteBuf {
if buf.endPos.Timestamp > endPos.Timestamp {

19
internal/datanode/services.go
View File

@ -26,20 +26,19 @@ import (
"strconv"
"github.com/cockroachdb/errors"
"go.uber.org/zap"
"github.com/milvus-io/milvus-proto/go-api/commonpb"
"github.com/milvus-io/milvus-proto/go-api/milvuspb"
"github.com/milvus-io/milvus-proto/go-api/msgpb"
"github.com/milvus-io/milvus-proto/go-api/schemapb"
"go.uber.org/zap"
"github.com/milvus-io/milvus/internal/common"
"github.com/milvus-io/milvus/internal/log"
"github.com/milvus-io/milvus/internal/metrics"
"github.com/milvus-io/milvus/internal/proto/datapb"
"github.com/milvus-io/milvus/internal/proto/etcdpb"
"github.com/milvus-io/milvus/internal/proto/internalpb"
"github.com/milvus-io/milvus/internal/proto/rootcoordpb"
"github.com/milvus-io/milvus/internal/common"
"github.com/milvus-io/milvus/internal/log"
"github.com/milvus-io/milvus/internal/metrics"
"github.com/milvus-io/milvus/internal/storage"
"github.com/milvus-io/milvus/internal/util/commonpbutil"
"github.com/milvus-io/milvus/internal/util/importutil"
@ -645,12 +644,12 @@ func (node *DataNode) AddImportSegment(ctx context.Context, req *datapb.AddImpor
partitionID: req.GetPartitionId(),
numOfRows: req.GetRowNum(),
statsBinLogs: req.GetStatsLog(),
startPos: &internalpb.MsgPosition{
startPos: &msgpb.MsgPosition{
ChannelName: req.GetChannelName(),
MsgID: posID,
Timestamp: req.GetBase().GetTimestamp(),
},
endPos: &internalpb.MsgPosition{
endPos: &msgpb.MsgPosition{
ChannelName: req.GetChannelName(),
MsgID: posID,
Timestamp: req.GetBase().GetTimestamp(),
@ -825,7 +824,7 @@ func saveSegmentFunc(node *DataNode, req *datapb.ImportTaskRequest, res *rootcoo
// Set start positions of a SaveBinlogPathRequest explicitly.
StartPositions: []*datapb.SegmentStartPosition{
{
StartPosition: &internalpb.MsgPosition{
StartPosition: &msgpb.MsgPosition{
ChannelName: targetChName,
Timestamp: ts,
},
@ -899,7 +898,7 @@ func createBinLogs(rowNum int, schema *schemapb.CollectionSchema, ts Timestamp,
}
if status, _ := node.dataCoord.UpdateSegmentStatistics(context.TODO(), &datapb.UpdateSegmentStatisticsRequest{
Stats: []*datapb.SegmentStats{{
Stats: []*commonpb.SegmentStats{{
SegmentID: segmentID,
NumRows: int64(rowNum),
}},

28
internal/datanode/services_test.go
View File

@ -24,10 +24,14 @@ import (
"sync"
"testing"
"github.com/stretchr/testify/suite"
clientv3 "go.etcd.io/etcd/client/v3"
"go.uber.org/zap"
"github.com/milvus-io/milvus-proto/go-api/commonpb"
"github.com/milvus-io/milvus-proto/go-api/milvuspb"
"github.com/milvus-io/milvus-proto/go-api/msgpb"
"github.com/milvus-io/milvus-proto/go-api/schemapb"
"github.com/milvus-io/milvus/internal/common"
"github.com/milvus-io/milvus/internal/log"
"github.com/milvus-io/milvus/internal/mq/msgstream"
@ -40,10 +44,6 @@ import (
"github.com/milvus-io/milvus/internal/util/metricsinfo"
"github.com/milvus-io/milvus/internal/util/paramtable"
"github.com/milvus-io/milvus/internal/util/sessionutil"
"github.com/stretchr/testify/suite"
clientv3 "go.etcd.io/etcd/client/v3"
"go.uber.org/zap"
)
type DataNodeServicesSuite struct {
@ -196,8 +196,8 @@ func (s *DataNodeServicesSuite) TestFlushSegments() {
segID: 0,
collID: 1,
partitionID: 1,
startPos: &internalpb.MsgPosition{},
endPos: &internalpb.MsgPosition{},
startPos: &msgpb.MsgPosition{},
endPos: &msgpb.MsgPosition{},
})
s.Require().NoError(err)
@ -231,7 +231,7 @@ func (s *DataNodeServicesSuite) TestFlushSegments() {
EndTimestamp: Timestamp(0),
HashValues: []uint32{0},
},
TimeTickMsg: internalpb.TimeTickMsg{
TimeTickMsg: msgpb.TimeTickMsg{
Base: &commonpb.MsgBase{
MsgType: commonpb.MsgType_TimeTick,
MsgID: UniqueID(0),
@ -707,8 +707,8 @@ func (s *DataNodeServicesSuite) TestResendSegmentStats() {
segID: 0,
collID: 1,
partitionID: 1,
startPos: &internalpb.MsgPosition{},
endPos: &internalpb.MsgPosition{},
startPos: &msgpb.MsgPosition{},
endPos: &msgpb.MsgPosition{},
})
s.Assert().Nil(err)
err = fgService.channel.addSegment(addSegmentReq{
@ -716,8 +716,8 @@ func (s *DataNodeServicesSuite) TestResendSegmentStats() {
segID: 1,
collID: 1,
partitionID: 2,
startPos: &internalpb.MsgPosition{},
endPos: &internalpb.MsgPosition{},
startPos: &msgpb.MsgPosition{},
endPos: &msgpb.MsgPosition{},
})
s.Assert().Nil(err)
err = fgService.channel.addSegment(addSegmentReq{
@ -725,8 +725,8 @@ func (s *DataNodeServicesSuite) TestResendSegmentStats() {
segID: 2,
collID: 1,
partitionID: 3,
startPos: &internalpb.MsgPosition{},
endPos: &internalpb.MsgPosition{},
startPos: &msgpb.MsgPosition{},
endPos: &msgpb.MsgPosition{},
})
s.Assert().Nil(err)

6
internal/metastore/catalog.go
View File

@ -4,9 +4,9 @@ import (
"context"
"github.com/milvus-io/milvus-proto/go-api/milvuspb"
"github.com/milvus-io/milvus-proto/go-api/msgpb"
"github.com/milvus-io/milvus/internal/metastore/model"
"github.com/milvus-io/milvus/internal/proto/datapb"
"github.com/milvus-io/milvus/internal/proto/internalpb"
"github.com/milvus-io/milvus/internal/proto/querypb"
"github.com/milvus-io/milvus/internal/util/typeutil"
)
@ -115,8 +115,8 @@ type DataCoordCatalog interface {
IsChannelDropped(ctx context.Context, channel string) bool
DropChannel(ctx context.Context, channel string) error
ListChannelCheckpoint(ctx context.Context) (map[string]*internalpb.MsgPosition, error)
SaveChannelCheckpoint(ctx context.Context, vChannel string, pos *internalpb.MsgPosition) error
ListChannelCheckpoint(ctx context.Context) (map[string]*msgpb.MsgPosition, error)
SaveChannelCheckpoint(ctx context.Context, vChannel string, pos *msgpb.MsgPosition) error
DropChannelCheckpoint(ctx context.Context, vChannel string) error
CreateIndex(ctx context.Context, index *model.Index) error

11
internal/metastore/kv/datacoord/kv_catalog.go
View File

@ -29,13 +29,13 @@ import (
"golang.org/x/sync/errgroup"
"github.com/milvus-io/milvus-proto/go-api/commonpb"
"github.com/milvus-io/milvus-proto/go-api/msgpb"
"github.com/milvus-io/milvus/internal/kv"
"github.com/milvus-io/milvus/internal/log"
"github.com/milvus-io/milvus/internal/metastore/model"
"github.com/milvus-io/milvus/internal/metrics"
"github.com/milvus-io/milvus/internal/proto/datapb"
"github.com/milvus-io/milvus/internal/proto/indexpb"
"github.com/milvus-io/milvus/internal/proto/internalpb"
"github.com/milvus-io/milvus/internal/storage"
"github.com/milvus-io/milvus/internal/util"
"github.com/milvus-io/milvus/internal/util/etcd"
@ -45,6 +45,7 @@ import (
)
var maxEtcdTxnNum = 128
var paginationSize = 2000
type Catalog struct {
@ -497,16 +498,16 @@ func (kc *Catalog) DropChannel(ctx context.Context, channel string) error {
return kc.MetaKv.Remove(key)
}
func (kc *Catalog) ListChannelCheckpoint(ctx context.Context) (map[string]*internalpb.MsgPosition, error) {
func (kc *Catalog) ListChannelCheckpoint(ctx context.Context) (map[string]*msgpb.MsgPosition, error) {
keys, values, err := kc.MetaKv.LoadWithPrefix(ChannelCheckpointPrefix)
if err != nil {
return nil, err
}
channelCPs := make(map[string]*internalpb.MsgPosition)
channelCPs := make(map[string]*msgpb.MsgPosition)
for i, key := range keys {
value := values[i]
channelCP := &internalpb.MsgPosition{}
channelCP := &msgpb.MsgPosition{}
err = proto.Unmarshal([]byte(value), channelCP)
if err != nil {
log.Error("unmarshal channelCP failed when ListChannelCheckpoint", zap.Error(err))
@ -520,7 +521,7 @@ func (kc *Catalog) ListChannelCheckpoint(ctx context.Context) (map[string]*inter
return channelCPs, nil
}
func (kc *Catalog) SaveChannelCheckpoint(ctx context.Context, vChannel string, pos *internalpb.MsgPosition) error {
func (kc *Catalog) SaveChannelCheckpoint(ctx context.Context, vChannel string, pos *msgpb.MsgPosition) error {
k := buildChannelCPKey(vChannel)
v, err := proto.Marshal(pos)
if err != nil {

10
internal/metastore/kv/datacoord/kv_catalog_test.go
View File

@ -27,21 +27,19 @@ import (
"time"
"github.com/cockroachdb/errors"
"github.com/milvus-io/milvus/internal/proto/indexpb"
"github.com/golang/protobuf/proto"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/mock"
"golang.org/x/exp/maps"
"github.com/milvus-io/milvus-proto/go-api/commonpb"
"github.com/milvus-io/milvus-proto/go-api/msgpb"
"github.com/milvus-io/milvus/internal/kv"
etcdkv "github.com/milvus-io/milvus/internal/kv/etcd"
"github.com/milvus-io/milvus/internal/kv/mocks"
"github.com/milvus-io/milvus/internal/metastore/model"
"github.com/milvus-io/milvus/internal/proto/datapb"
"github.com/milvus-io/milvus/internal/proto/internalpb"
"github.com/milvus-io/milvus/internal/proto/indexpb"
"github.com/milvus-io/milvus/internal/util/etcd"
"github.com/milvus-io/milvus/internal/util/metautil"
"github.com/milvus-io/milvus/internal/util/paramtable"
@ -669,7 +667,7 @@ func TestChannelCP(t *testing.T) {
mockVChannel := "fake-by-dev-rootcoord-dml-1-testchannelcp-v0"
mockPChannel := "fake-by-dev-rootcoord-dml-1"
pos := &internalpb.MsgPosition{
pos := &msgpb.MsgPosition{
ChannelName: mockPChannel,
MsgID: []byte{},
Timestamp: 1000,
@ -711,7 +709,7 @@ func TestChannelCP(t *testing.T) {
txn := mocks.NewMetaKv(t)
catalog := NewCatalog(txn, rootPath, "")
txn.EXPECT().Save(mock.Anything, mock.Anything).Return(errors.New("mock error"))
err = catalog.SaveChannelCheckpoint(context.TODO(), mockVChannel, &internalpb.MsgPosition{})
err = catalog.SaveChannelCheckpoint(context.TODO(), mockVChannel, &msgpb.MsgPosition{})
assert.Error(t, err)
})

23
internal/metastore/mocks/DataCoordCatalog.go
View File

@ -6,11 +6,12 @@ import (
context "context"
datapb "github.com/milvus-io/milvus/internal/proto/datapb"
internalpb "github.com/milvus-io/milvus/internal/proto/internalpb"
mock "github.com/stretchr/testify/mock"
model "github.com/milvus-io/milvus/internal/metastore/model"
msgpb "github.com/milvus-io/milvus-proto/go-api/msgpb"
)
// DataCoordCatalog is an autogenerated mock type for the DataCoordCatalog type
@ -680,15 +681,15 @@ func (_c *DataCoordCatalog_IsChannelDropped_Call) Return(_a0 bool) *DataCoordCat
}
// ListChannelCheckpoint provides a mock function with given fields: ctx
func (_m *DataCoordCatalog) ListChannelCheckpoint(ctx context.Context) (map[string]*internalpb.MsgPosition, error) {
func (_m *DataCoordCatalog) ListChannelCheckpoint(ctx context.Context) (map[string]*msgpb.MsgPosition, error) {
ret := _m.Called(ctx)
var r0 map[string]*internalpb.MsgPosition
if rf, ok := ret.Get(0).(func(context.Context) map[string]*internalpb.MsgPosition); ok {
var r0 map[string]*msgpb.MsgPosition
if rf, ok := ret.Get(0).(func(context.Context) map[string]*msgpb.MsgPosition); ok {
r0 = rf(ctx)
} else {
if ret.Get(0) != nil {
r0 = ret.Get(0).(map[string]*internalpb.MsgPosition)
r0 = ret.Get(0).(map[string]*msgpb.MsgPosition)
}
}
@ -720,7 +721,7 @@ func (_c *DataCoordCatalog_ListChannelCheckpoint_Call) Run(run func(ctx context.
return _c
}
func (_c *DataCoordCatalog_ListChannelCheckpoint_Call) Return(_a0 map[string]*internalpb.MsgPosition, _a1 error) *DataCoordCatalog_ListChannelCheckpoint_Call {
func (_c *DataCoordCatalog_ListChannelCheckpoint_Call) Return(_a0 map[string]*msgpb.MsgPosition, _a1 error) *DataCoordCatalog_ListChannelCheckpoint_Call {
_c.Call.Return(_a0, _a1)
return _c
}
@ -941,11 +942,11 @@ func (_c *DataCoordCatalog_RevertAlterSegmentsAndAddNewSegment_Call) Return(_a0
}
// SaveChannelCheckpoint provides a mock function with given fields: ctx, vChannel, pos
func (_m *DataCoordCatalog) SaveChannelCheckpoint(ctx context.Context, vChannel string, pos *internalpb.MsgPosition) error {
func (_m *DataCoordCatalog) SaveChannelCheckpoint(ctx context.Context, vChannel string, pos *msgpb.MsgPosition) error {
ret := _m.Called(ctx, vChannel, pos)
var r0 error
if rf, ok := ret.Get(0).(func(context.Context, string, *internalpb.MsgPosition) error); ok {
if rf, ok := ret.Get(0).(func(context.Context, string, *msgpb.MsgPosition) error); ok {
r0 = rf(ctx, vChannel, pos)
} else {
r0 = ret.Error(0)
@ -962,14 +963,14 @@ type DataCoordCatalog_SaveChannelCheckpoint_Call struct {
// SaveChannelCheckpoint is a helper method to define mock.On call
// - ctx context.Context
// - vChannel string
// - pos *internalpb.MsgPosition
// - pos *msgpb.MsgPosition
func (_e *DataCoordCatalog_Expecter) SaveChannelCheckpoint(ctx interface{}, vChannel interface{}, pos interface{}) *DataCoordCatalog_SaveChannelCheckpoint_Call {
return &DataCoordCatalog_SaveChannelCheckpoint_Call{Call: _e.mock.On("SaveChannelCheckpoint", ctx, vChannel, pos)}
}
func (_c *DataCoordCatalog_SaveChannelCheckpoint_Call) Run(run func(ctx context.Context, vChannel string, pos *internalpb.MsgPosition)) *DataCoordCatalog_SaveChannelCheckpoint_Call {
func (_c *DataCoordCatalog_SaveChannelCheckpoint_Call) Run(run func(ctx context.Context, vChannel string, pos *msgpb.MsgPosition)) *DataCoordCatalog_SaveChannelCheckpoint_Call {
_c.Call.Run(func(args mock.Arguments) {
run(args[0].(context.Context), args[1].(string), args[2].(*internalpb.MsgPosition))
run(args[0].(context.Context), args[1].(string), args[2].(*msgpb.MsgPosition))
})
return _c
}

22
internal/mocks/mock_datacoord_catalog.go
View File

@ -4,9 +4,9 @@ package mocks
import (
context "context"
"github.com/milvus-io/milvus-proto/go-api/msgpb"
datapb "github.com/milvus-io/milvus/internal/proto/datapb"
internalpb "github.com/milvus-io/milvus/internal/proto/internalpb"
mock "github.com/stretchr/testify/mock"
@ -680,15 +680,15 @@ func (_c *DataCoordCatalog_IsChannelDropped_Call) Return(_a0 bool) *DataCoordCat
}
// ListChannelCheckpoint provides a mock function with given fields: ctx
func (_m *DataCoordCatalog) ListChannelCheckpoint(ctx context.Context) (map[string]*internalpb.MsgPosition, error) {
func (_m *DataCoordCatalog) ListChannelCheckpoint(ctx context.Context) (map[string]*msgpb.MsgPosition, error) {
ret := _m.Called(ctx)
var r0 map[string]*internalpb.MsgPosition
if rf, ok := ret.Get(0).(func(context.Context) map[string]*internalpb.MsgPosition); ok {
var r0 map[string]*msgpb.MsgPosition
if rf, ok := ret.Get(0).(func(context.Context) map[string]*msgpb.MsgPosition); ok {
r0 = rf(ctx)
} else {
if ret.Get(0) != nil {
r0 = ret.Get(0).(map[string]*internalpb.MsgPosition)
r0 = ret.Get(0).(map[string]*msgpb.MsgPosition)
}
}
@ -720,7 +720,7 @@ func (_c *DataCoordCatalog_ListChannelCheckpoint_Call) Run(run func(ctx context.
return _c
}
func (_c *DataCoordCatalog_ListChannelCheckpoint_Call) Return(_a0 map[string]*internalpb.MsgPosition, _a1 error) *DataCoordCatalog_ListChannelCheckpoint_Call {
func (_c *DataCoordCatalog_ListChannelCheckpoint_Call) Return(_a0 map[string]*msgpb.MsgPosition, _a1 error) *DataCoordCatalog_ListChannelCheckpoint_Call {
_c.Call.Return(_a0, _a1)
return _c
}
@ -941,11 +941,11 @@ func (_c *DataCoordCatalog_RevertAlterSegmentsAndAddNewSegment_Call) Return(_a0
}
// SaveChannelCheckpoint provides a mock function with given fields: ctx, vChannel, pos
func (_m *DataCoordCatalog) SaveChannelCheckpoint(ctx context.Context, vChannel string, pos *internalpb.MsgPosition) error {
func (_m *DataCoordCatalog) SaveChannelCheckpoint(ctx context.Context, vChannel string, pos *msgpb.MsgPosition) error {
ret := _m.Called(ctx, vChannel, pos)
var r0 error
if rf, ok := ret.Get(0).(func(context.Context, string, *internalpb.MsgPosition) error); ok {
if rf, ok := ret.Get(0).(func(context.Context, string, *msgpb.MsgPosition) error); ok {
r0 = rf(ctx, vChannel, pos)
} else {
r0 = ret.Error(0)
@ -962,14 +962,14 @@ type DataCoordCatalog_SaveChannelCheckpoint_Call struct {
// SaveChannelCheckpoint is a helper method to define mock.On call
// - ctx context.Context
// - vChannel string
// - pos *internalpb.MsgPosition
// - pos *msgpb.MsgPosition
func (_e *DataCoordCatalog_Expecter) SaveChannelCheckpoint(ctx interface{}, vChannel interface{}, pos interface{}) *DataCoordCatalog_SaveChannelCheckpoint_Call {
return &DataCoordCatalog_SaveChannelCheckpoint_Call{Call: _e.mock.On("SaveChannelCheckpoint", ctx, vChannel, pos)}
}
func (_c *DataCoordCatalog_SaveChannelCheckpoint_Call) Run(run func(ctx context.Context, vChannel string, pos *internalpb.MsgPosition)) *DataCoordCatalog_SaveChannelCheckpoint_Call {
func (_c *DataCoordCatalog_SaveChannelCheckpoint_Call) Run(run func(ctx context.Context, vChannel string, pos *msgpb.MsgPosition)) *DataCoordCatalog_SaveChannelCheckpoint_Call {
_c.Call.Run(func(args mock.Arguments) {
run(args[0].(context.Context), args[1].(string), args[2].(*internalpb.MsgPosition))
run(args[0].(context.Context), args[1].(string), args[2].(*msgpb.MsgPosition))
})
return _c
}

4
internal/mq/msgdispatcher/client.go
View File

@ -21,15 +21,15 @@ import (
"go.uber.org/zap"
"github.com/milvus-io/milvus-proto/go-api/msgpb"
"github.com/milvus-io/milvus/internal/log"
"github.com/milvus-io/milvus/internal/mq/msgstream"
"github.com/milvus-io/milvus/internal/mq/msgstream/mqwrapper"
"github.com/milvus-io/milvus/internal/proto/internalpb"
"github.com/milvus-io/milvus/internal/util/funcutil"
)
type (
Pos = internalpb.MsgPosition
Pos = msgpb.MsgPosition
MsgPack = msgstream.MsgPack
SubPos = mqwrapper.SubscriptionInitialPosition
)

18
internal/mq/msgdispatcher/mock_test.go
View File

@ -23,10 +23,10 @@ import (
"time"
"github.com/milvus-io/milvus-proto/go-api/commonpb"
"github.com/milvus-io/milvus-proto/go-api/msgpb"
"github.com/milvus-io/milvus-proto/go-api/schemapb"
"github.com/milvus-io/milvus/internal/mq/msgstream"
"github.com/milvus-io/milvus/internal/mq/msgstream/mqwrapper"
"github.com/milvus-io/milvus/internal/proto/internalpb"
"github.com/milvus-io/milvus/internal/util/paramtable"
"github.com/milvus-io/milvus/internal/util/typeutil"
)
@ -100,7 +100,7 @@ func genInsertMsg(numRows int, vchannel string, msgID typeutil.UniqueID) *msgstr
}
return &msgstream.InsertMsg{
BaseMsg: msgstream.BaseMsg{HashValues: hashValues},
InsertRequest: internalpb.InsertRequest{
InsertRequest: msgpb.InsertRequest{
Base: &commonpb.MsgBase{MsgType: commonpb.MsgType_Insert, MsgID: msgID},
ShardName: vchannel,
Timestamps: genTimestamps(numRows),
@ -114,7 +114,7 @@ func genInsertMsg(numRows int, vchannel string, msgID typeutil.UniqueID) *msgstr
},
}},
NumRows: uint64(numRows),
Version: internalpb.InsertDataVersion_ColumnBased,
Version: msgpb.InsertDataVersion_ColumnBased,
},
}
}
@ -122,7 +122,7 @@ func genInsertMsg(numRows int, vchannel string, msgID typeutil.UniqueID) *msgstr
func genDeleteMsg(numRows int, vchannel string, msgID typeutil.UniqueID) *msgstream.DeleteMsg {
return &msgstream.DeleteMsg{
BaseMsg: msgstream.BaseMsg{HashValues: make([]uint32, numRows)},
DeleteRequest: internalpb.DeleteRequest{
DeleteRequest: msgpb.DeleteRequest{
Base: &commonpb.MsgBase{MsgType: commonpb.MsgType_Delete, MsgID: msgID},
ShardName: vchannel,
PrimaryKeys: &schemapb.IDs{
@ -143,28 +143,28 @@ func genDDLMsg(msgType commonpb.MsgType) msgstream.TsMsg {
case commonpb.MsgType_CreateCollection:
return &msgstream.CreateCollectionMsg{
BaseMsg: msgstream.BaseMsg{HashValues: []uint32{0}},
CreateCollectionRequest: internalpb.CreateCollectionRequest{
CreateCollectionRequest: msgpb.CreateCollectionRequest{
Base: &commonpb.MsgBase{MsgType: commonpb.MsgType_CreateCollection},
},
}
case commonpb.MsgType_DropCollection:
return &msgstream.DropCollectionMsg{
BaseMsg: msgstream.BaseMsg{HashValues: []uint32{0}},
DropCollectionRequest: internalpb.DropCollectionRequest{
DropCollectionRequest: msgpb.DropCollectionRequest{
Base: &commonpb.MsgBase{MsgType: commonpb.MsgType_DropCollection},
},
}
case commonpb.MsgType_CreatePartition:
return &msgstream.CreatePartitionMsg{
BaseMsg: msgstream.BaseMsg{HashValues: []uint32{0}},
CreatePartitionRequest: internalpb.CreatePartitionRequest{
CreatePartitionRequest: msgpb.CreatePartitionRequest{
Base: &commonpb.MsgBase{MsgType: commonpb.MsgType_CreatePartition},
},
}
case commonpb.MsgType_DropPartition:
return &msgstream.DropPartitionMsg{
BaseMsg: msgstream.BaseMsg{HashValues: []uint32{0}},
DropPartitionRequest: internalpb.DropPartitionRequest{
DropPartitionRequest: msgpb.DropPartitionRequest{
Base: &commonpb.MsgBase{MsgType: commonpb.MsgType_DropPartition},
},
}
@ -175,7 +175,7 @@ func genDDLMsg(msgType commonpb.MsgType) msgstream.TsMsg {
func genTimeTickMsg(ts typeutil.Timestamp) *msgstream.TimeTickMsg {
return &msgstream.TimeTickMsg{
BaseMsg: msgstream.BaseMsg{HashValues: []uint32{0}},
TimeTickMsg: internalpb.TimeTickMsg{
TimeTickMsg: msgpb.TimeTickMsg{
Base: &commonpb.MsgBase{
MsgType: commonpb.MsgType_TimeTick,
Timestamp: ts,

14
internal/mq/msgstream/mq_kafka_msgstream_test.go
View File

@ -24,15 +24,13 @@ import (
"testing"
"github.com/confluentinc/confluent-kafka-go/kafka"
"github.com/milvus-io/milvus/internal/mq/msgstream/mqwrapper"
kafkawrapper "github.com/milvus-io/milvus/internal/mq/msgstream/mqwrapper/kafka"
"github.com/stretchr/testify/assert"
"github.com/milvus-io/milvus-proto/go-api/commonpb"
"github.com/milvus-io/milvus/internal/proto/internalpb"
"github.com/milvus-io/milvus-proto/go-api/msgpb"
"github.com/milvus-io/milvus/internal/mq/msgstream/mqwrapper"
kafkawrapper "github.com/milvus-io/milvus/internal/mq/msgstream/mqwrapper/kafka"
"github.com/milvus-io/milvus/internal/util/funcutil"
"github.com/stretchr/testify/assert"
)
// Note: kafka does not support get all data when consuming from the earliest position again.
@ -130,7 +128,7 @@ func TestStream_KafkaMsgStream_SeekToLast(t *testing.T) {
assert.Nil(t, err)
// pick a seekPosition
var seekPosition *internalpb.MsgPosition
var seekPosition *msgpb.MsgPosition
outputStream := getKafkaOutputStream(ctx, kafkaAddress, consumerChannels, consumerSubName, mqwrapper.SubscriptionPositionEarliest)
for i := 0; i < 10; i++ {
result := consumer(ctx, outputStream)
@ -148,7 +146,7 @@ func TestStream_KafkaMsgStream_SeekToLast(t *testing.T) {
defer outputStream2.Close()
assert.Nil(t, err)
err = outputStream2.Seek([]*internalpb.MsgPosition{seekPosition})
err = outputStream2.Seek([]*msgpb.MsgPosition{seekPosition})
assert.Nil(t, err)
cnt := 0

29
internal/mq/msgstream/mq_msgstream.go
View File

@ -25,14 +25,13 @@ import (
"time"
"github.com/cockroachdb/errors"
"github.com/golang/protobuf/proto"
"go.uber.org/zap"
"github.com/milvus-io/milvus-proto/go-api/commonpb"
"github.com/milvus-io/milvus-proto/go-api/msgpb"
"github.com/milvus-io/milvus/internal/log"
"github.com/milvus-io/milvus/internal/mq/msgstream/mqwrapper"
"github.com/milvus-io/milvus/internal/proto/internalpb"
"github.com/milvus-io/milvus/internal/util/retry"
"github.com/milvus-io/milvus/internal/util/typeutil"
)
@ -393,8 +392,8 @@ func (ms *mqMsgStream) receiveMsg(consumer mqwrapper.Consumer) {
msgPack := MsgPack{
Msgs: []TsMsg{tsMsg},
StartPositions: []*internalpb.MsgPosition{tsMsg.Position()},
EndPositions: []*internalpb.MsgPosition{tsMsg.Position()},
StartPositions: []*msgpb.MsgPosition{tsMsg.Position()},
EndPositions: []*msgpb.MsgPosition{tsMsg.Position()},
}
select {
case ms.receiveBuf <- &msgPack:
@ -419,7 +418,7 @@ func (ms *mqMsgStream) Chan() <-chan *MsgPack {
// Seek reset the subscription associated with this consumer to a specific position, the seek position is exclusive
// User has to ensure mq_msgstream is not closed before seek, and the seek position is already written.
func (ms *mqMsgStream) Seek(msgPositions []*internalpb.MsgPosition) error {
func (ms *mqMsgStream) Seek(msgPositions []*msgpb.MsgPosition) error {
for _, mp := range msgPositions {
consumer, ok := ms.consumers[mp.ChannelName]
if !ok {
@ -447,7 +446,7 @@ var _ MsgStream = (*MqTtMsgStream)(nil)
type MqTtMsgStream struct {
mqMsgStream
chanMsgBuf map[mqwrapper.Consumer][]TsMsg
chanMsgPos map[mqwrapper.Consumer]*internalpb.MsgPosition
chanMsgPos map[mqwrapper.Consumer]*msgpb.MsgPosition
chanStopChan map[mqwrapper.Consumer]chan bool
chanTtMsgTime map[mqwrapper.Consumer]Timestamp
chanMsgBufMutex *sync.Mutex
@ -468,7 +467,7 @@ func NewMqTtMsgStream(ctx context.Context,
return nil, err
}
chanMsgBuf := make(map[mqwrapper.Consumer][]TsMsg)
chanMsgPos := make(map[mqwrapper.Consumer]*internalpb.MsgPosition)
chanMsgPos := make(map[mqwrapper.Consumer]*msgpb.MsgPosition)
chanStopChan := make(map[mqwrapper.Consumer]chan bool)
chanTtMsgTime := make(map[mqwrapper.Consumer]Timestamp)
syncConsumer := make(chan int, 1)
@ -493,7 +492,7 @@ func (ms *MqTtMsgStream) addConsumer(consumer mqwrapper.Consumer, channel string
ms.consumers[channel] = consumer
ms.consumerChannels = append(ms.consumerChannels, channel)
ms.chanMsgBuf[consumer] = make([]TsMsg, 0)
ms.chanMsgPos[consumer] = &internalpb.MsgPosition{
ms.chanMsgPos[consumer] = &msgpb.MsgPosition{
ChannelName: channel,
MsgID: make([]byte, 0),
Timestamp: ms.lastTimeStamp,
@ -584,8 +583,8 @@ func (ms *MqTtMsgStream) bufMsgPackToChannel() {
}
timeTickBuf := make([]TsMsg, 0)
startMsgPosition := make([]*internalpb.MsgPosition, 0)
endMsgPositions := make([]*internalpb.MsgPosition, 0)
startMsgPosition := make([]*msgpb.MsgPosition, 0)
endMsgPositions := make([]*msgpb.MsgPosition, 0)
ms.chanMsgBufMutex.Lock()
for consumer, msgs := range ms.chanMsgBuf {
if len(msgs) == 0 {
@ -607,11 +606,11 @@ func (ms *MqTtMsgStream) bufMsgPackToChannel() {
}
ms.chanMsgBuf[consumer] = tempBuffer
startMsgPosition = append(startMsgPosition, proto.Clone(ms.chanMsgPos[consumer]).(*internalpb.MsgPosition))
var newPos *internalpb.MsgPosition
startMsgPosition = append(startMsgPosition, proto.Clone(ms.chanMsgPos[consumer]).(*msgpb.MsgPosition))
var newPos *msgpb.MsgPosition
if len(tempBuffer) > 0 {
// if tempBuffer is not empty, use tempBuffer[0] to seek
newPos = &internalpb.MsgPosition{
newPos = &msgpb.MsgPosition{
ChannelName: tempBuffer[0].Position().ChannelName,
MsgID: tempBuffer[0].Position().MsgID,
Timestamp: currTs,
@ -620,7 +619,7 @@ func (ms *MqTtMsgStream) bufMsgPackToChannel() {
endMsgPositions = append(endMsgPositions, newPos)
} else if timeTickMsg != nil {
// if tempBuffer is empty, use timeTickMsg to seek
newPos = &internalpb.MsgPosition{
newPos = &msgpb.MsgPosition{
ChannelName: timeTickMsg.Position().ChannelName,
MsgID: timeTickMsg.Position().MsgID,
Timestamp: currTs,
@ -737,7 +736,7 @@ func (ms *MqTtMsgStream) allChanReachSameTtMsg(chanTtMsgSync map[mqwrapper.Consu
}
// Seek to the specified position
func (ms *MqTtMsgStream) Seek(msgPositions []*internalpb.MsgPosition) error {
func (ms *MqTtMsgStream) Seek(msgPositions []*msgpb.MsgPosition) error {
var consumer mqwrapper.Consumer
var mp *MsgPosition
var err error

55
internal/mq/msgstream/mq_msgstream_test.go
View File

@ -28,25 +28,22 @@ import (
"time"
"unsafe"
"github.com/cockroachdb/errors"
"github.com/confluentinc/confluent-kafka-go/kafka"
"github.com/milvus-io/milvus/internal/mq/mqimpl/rocksmq/server"
"go.uber.org/atomic"
"github.com/apache/pulsar-client-go/pulsar"
pulsarwrapper "github.com/milvus-io/milvus/internal/mq/msgstream/mqwrapper/pulsar"
"github.com/milvus-io/milvus/internal/mq/msgstream/mqwrapper/rmq"
"github.com/cockroachdb/errors"
"github.com/confluentinc/confluent-kafka-go/kafka"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"go.uber.org/atomic"
"github.com/milvus-io/milvus-proto/go-api/commonpb"
"github.com/milvus-io/milvus-proto/go-api/msgpb"
"github.com/milvus-io/milvus/internal/allocator"
"github.com/milvus-io/milvus/internal/common"
etcdkv "github.com/milvus-io/milvus/internal/kv/etcd"
"github.com/milvus-io/milvus/internal/mq/mqimpl/rocksmq/server"
"github.com/milvus-io/milvus/internal/mq/msgstream/mqwrapper"
"github.com/milvus-io/milvus/internal/proto/internalpb"
pulsarwrapper "github.com/milvus-io/milvus/internal/mq/msgstream/mqwrapper/pulsar"
"github.com/milvus-io/milvus/internal/mq/msgstream/mqwrapper/rmq"
"github.com/milvus-io/milvus/internal/util/etcd"
"github.com/milvus-io/milvus/internal/util/funcutil"
"github.com/milvus-io/milvus/internal/util/paramtable"
@ -209,7 +206,7 @@ func TestMqMsgStream_ComputeProduceChannelIndexes(t *testing.T) {
// not called AsProducer yet
insertMsg := &InsertMsg{
BaseMsg: generateBaseMsg(),
InsertRequest: internalpb.InsertRequest{
InsertRequest: msgpb.InsertRequest{
Base: &commonpb.MsgBase{
MsgType: commonpb.MsgType_Insert,
MsgID: 1,
@ -272,7 +269,7 @@ func TestMqMsgStream_Produce(t *testing.T) {
// Produce before called AsProducer
insertMsg := &InsertMsg{
BaseMsg: generateBaseMsg(),
InsertRequest: internalpb.InsertRequest{
InsertRequest: msgpb.InsertRequest{
Base: &commonpb.MsgBase{
MsgType: commonpb.MsgType_Insert,
MsgID: 1,
@ -389,7 +386,7 @@ func TestMqMsgStream_SeekNotSubscribed(t *testing.T) {
assert.Nil(t, err)
// seek in not subscribed channel
p := []*internalpb.MsgPosition{
p := []*msgpb.MsgPosition{
{
ChannelName: "b",
},
@ -524,7 +521,7 @@ func TestStream_PulsarMsgStream_InsertRepackFunc(t *testing.T) {
HashValues: []uint32{1, 3},
}
insertRequest := internalpb.InsertRequest{
insertRequest := msgpb.InsertRequest{
Base: &commonpb.MsgBase{
MsgType: commonpb.MsgType_Insert,
MsgID: 1,
@ -580,7 +577,7 @@ func TestStream_PulsarMsgStream_DeleteRepackFunc(t *testing.T) {
HashValues: []uint32{1},
}
deleteRequest := internalpb.DeleteRequest{
deleteRequest := msgpb.DeleteRequest{
Base: &commonpb.MsgBase{
MsgType: commonpb.MsgType_Delete,
MsgID: 1,
@ -776,7 +773,7 @@ func TestStream_PulsarMsgStream_SeekToLast(t *testing.T) {
assert.Nil(t, err)
// pick a seekPosition
var seekPosition *internalpb.MsgPosition
var seekPosition *msgpb.MsgPosition
for i := 0; i < 10; i++ {
result := consumer(ctx, outputStream)
assert.Equal(t, result.Msgs[0].ID(), int64(i))
@ -795,7 +792,7 @@ func TestStream_PulsarMsgStream_SeekToLast(t *testing.T) {
defer outputStream2.Close()
assert.Nil(t, err)
err = outputStream2.Seek([]*internalpb.MsgPosition{seekPosition})
err = outputStream2.Seek([]*msgpb.MsgPosition{seekPosition})
assert.Nil(t, err)
cnt := 0
@ -1184,7 +1181,7 @@ func TestStream_MqMsgStream_Seek(t *testing.T) {
err := inputStream.Produce(msgPack)
assert.Nil(t, err)
var seekPosition *internalpb.MsgPosition
var seekPosition *msgpb.MsgPosition
for i := 0; i < 10; i++ {
result := consumer(ctx, outputStream)
assert.Equal(t, result.Msgs[0].ID(), int64(i))
@ -1198,7 +1195,7 @@ func TestStream_MqMsgStream_Seek(t *testing.T) {
pulsarClient, _ := pulsarwrapper.NewClient(DefaultPulsarTenant, DefaultPulsarNamespace, pulsar.ClientOptions{URL: pulsarAddress})
outputStream2, _ := NewMqMsgStream(ctx, 100, 100, pulsarClient, factory.NewUnmarshalDispatcher())
outputStream2.AsConsumer(consumerChannels, consumerSubName, mqwrapper.SubscriptionPositionEarliest)
outputStream2.Seek([]*internalpb.MsgPosition{seekPosition})
outputStream2.Seek([]*msgpb.MsgPosition{seekPosition})
for i := 6; i < 10; i++ {
result := consumer(ctx, outputStream2)
@ -1229,7 +1226,7 @@ func TestStream_MqMsgStream_SeekInvalidMessage(t *testing.T) {
err := inputStream.Produce(msgPack)
assert.Nil(t, err)
var seekPosition *internalpb.MsgPosition
var seekPosition *msgpb.MsgPosition
for i := 0; i < 10; i++ {
result := consumer(ctx, outputStream)
assert.Equal(t, result.Msgs[0].ID(), int64(i))
@ -1245,7 +1242,7 @@ func TestStream_MqMsgStream_SeekInvalidMessage(t *testing.T) {
// try to seek to not written position
patchMessageID(&messageID, 13)
p := []*internalpb.MsgPosition{
p := []*msgpb.MsgPosition{
{
ChannelName: seekPosition.ChannelName,
Timestamp: seekPosition.Timestamp,
@ -1285,7 +1282,7 @@ func TestStream_RMqMsgStream_SeekInvalidMessage(t *testing.T) {
err := inputStream.Produce(msgPack)
assert.Nil(t, err)
var seekPosition *internalpb.MsgPosition
var seekPosition *msgpb.MsgPosition
for i := 0; i < 10; i++ {
result := consumer(ctx, outputStream)
assert.Equal(t, result.Msgs[0].ID(), int64(i))
@ -1301,7 +1298,7 @@ func TestStream_RMqMsgStream_SeekInvalidMessage(t *testing.T) {
id := common.Endian.Uint64(seekPosition.MsgID) + 10
bs := make([]byte, 8)
common.Endian.PutUint64(bs, id)
p := []*internalpb.MsgPosition{
p := []*msgpb.MsgPosition{
{
ChannelName: seekPosition.ChannelName,
Timestamp: seekPosition.Timestamp,
@ -1797,7 +1794,7 @@ func getTsMsg(msgType MsgType, reqID UniqueID) TsMsg {
}
switch msgType {
case commonpb.MsgType_Insert:
insertRequest := internalpb.InsertRequest{
insertRequest := msgpb.InsertRequest{
Base: &commonpb.MsgBase{
MsgType: commonpb.MsgType_Insert,
MsgID: reqID,
@ -1818,7 +1815,7 @@ func getTsMsg(msgType MsgType, reqID UniqueID) TsMsg {
}
return insertMsg
case commonpb.MsgType_Delete:
deleteRequest := internalpb.DeleteRequest{
deleteRequest := msgpb.DeleteRequest{
Base: &commonpb.MsgBase{
MsgType: commonpb.MsgType_Delete,
MsgID: reqID,
@ -1837,7 +1834,7 @@ func getTsMsg(msgType MsgType, reqID UniqueID) TsMsg {
}
return deleteMsg
case commonpb.MsgType_CreateCollection:
createCollectionRequest := internalpb.CreateCollectionRequest{
createCollectionRequest := msgpb.CreateCollectionRequest{
Base: &commonpb.MsgBase{
MsgType: commonpb.MsgType_CreateCollection,
MsgID: reqID,
@ -1860,7 +1857,7 @@ func getTsMsg(msgType MsgType, reqID UniqueID) TsMsg {
}
return createCollectionMsg
case commonpb.MsgType_TimeTick:
timeTickResult := internalpb.TimeTickMsg{
timeTickResult := msgpb.TimeTickMsg{
Base: &commonpb.MsgBase{
MsgType: commonpb.MsgType_TimeTick,
MsgID: reqID,
@ -1885,7 +1882,7 @@ func getTimeTickMsg(reqID UniqueID) TsMsg {
EndTimestamp: 0,
HashValues: []uint32{hashValue},
}
timeTickResult := internalpb.TimeTickMsg{
timeTickResult := msgpb.TimeTickMsg{
Base: &commonpb.MsgBase{
MsgType: commonpb.MsgType_TimeTick,
MsgID: reqID,
@ -1935,7 +1932,7 @@ func getInsertMsgUniqueID(ts UniqueID) TsMsg {
HashValues: []uint32{hashValue},
}
insertRequest := internalpb.InsertRequest{
insertRequest := msgpb.InsertRequest{
Base: &commonpb.MsgBase{
MsgType: commonpb.MsgType_Insert,
MsgID: idCounter.Inc(),

59
internal/mq/msgstream/msg.go
View File

@ -21,17 +21,14 @@ import (
"fmt"
"github.com/cockroachdb/errors"
"github.com/golang/protobuf/proto"
"github.com/milvus-io/milvus-proto/go-api/commonpb"
"github.com/milvus-io/milvus-proto/go-api/msgpb"
"github.com/milvus-io/milvus-proto/go-api/schemapb"
"github.com/milvus-io/milvus/internal/util/commonpbutil"
"github.com/milvus-io/milvus/internal/util/funcutil"
"github.com/milvus-io/milvus/internal/util/typeutil"
"github.com/golang/protobuf/proto"
"github.com/milvus-io/milvus-proto/go-api/commonpb"
"github.com/milvus-io/milvus/internal/proto/datapb"
"github.com/milvus-io/milvus/internal/proto/internalpb"
)
// MsgType is an alias of commonpb.MsgType
@ -114,7 +111,7 @@ func convertToByteArray(input interface{}) ([]byte, error) {
// InsertMsg is a message pack that contains insert request
type InsertMsg struct {
BaseMsg
internalpb.InsertRequest
msgpb.InsertRequest
}
// interface implementation validation
@ -148,7 +145,7 @@ func (it *InsertMsg) Marshal(input TsMsg) (MarshalType, error) {
// Unmarshal is used to deserialize a message pack from byte array
func (it *InsertMsg) Unmarshal(input MarshalType) (TsMsg, error) {
insertRequest := internalpb.InsertRequest{}
insertRequest := msgpb.InsertRequest{}
in, err := convertToByteArray(input)
if err != nil {
return nil, err
@ -176,11 +173,11 @@ func (it *InsertMsg) Unmarshal(input MarshalType) (TsMsg, error) {
}
func (it *InsertMsg) IsRowBased() bool {
return it.GetVersion() == internalpb.InsertDataVersion_RowBased
return it.GetVersion() == msgpb.InsertDataVersion_RowBased
}
func (it *InsertMsg) IsColumnBased() bool {
return it.GetVersion() == internalpb.InsertDataVersion_ColumnBased
return it.GetVersion() == msgpb.InsertDataVersion_ColumnBased
}
func (it *InsertMsg) NRows() uint64 {
@ -218,8 +215,8 @@ func (it *InsertMsg) CheckAligned() error {
return nil
}
func (it *InsertMsg) rowBasedIndexRequest(index int) internalpb.InsertRequest {
return internalpb.InsertRequest{
func (it *InsertMsg) rowBasedIndexRequest(index int) msgpb.InsertRequest {
return msgpb.InsertRequest{
Base: commonpbutil.NewMsgBase(
commonpbutil.WithMsgType(commonpb.MsgType_Insert),
commonpbutil.WithMsgID(it.Base.MsgID),
@ -236,15 +233,15 @@ func (it *InsertMsg) rowBasedIndexRequest(index int) internalpb.InsertRequest {
Timestamps: []uint64{it.Timestamps[index]},
RowIDs: []int64{it.RowIDs[index]},
RowData: []*commonpb.Blob{it.RowData[index]},
Version: internalpb.InsertDataVersion_RowBased,
Version: msgpb.InsertDataVersion_RowBased,
}
}
func (it *InsertMsg) columnBasedIndexRequest(index int) internalpb.InsertRequest {
func (it *InsertMsg) columnBasedIndexRequest(index int) msgpb.InsertRequest {
colNum := len(it.GetFieldsData())
fieldsData := make([]*schemapb.FieldData, colNum)
typeutil.AppendFieldData(fieldsData, it.GetFieldsData(), int64(index))
return internalpb.InsertRequest{
return msgpb.InsertRequest{
Base: commonpbutil.NewMsgBase(
commonpbutil.WithMsgType(commonpb.MsgType_Insert),
commonpbutil.WithMsgID(it.Base.MsgID),
@ -262,11 +259,11 @@ func (it *InsertMsg) columnBasedIndexRequest(index int) internalpb.InsertRequest
RowIDs: []int64{it.RowIDs[index]},
FieldsData: fieldsData,
NumRows: 1,
Version: internalpb.InsertDataVersion_ColumnBased,
Version: msgpb.InsertDataVersion_ColumnBased,
}
}
func (it *InsertMsg) IndexRequest(index int) internalpb.InsertRequest {
func (it *InsertMsg) IndexRequest(index int) msgpb.InsertRequest {
if it.IsRowBased() {
return it.rowBasedIndexRequest(index)
}
@ -291,7 +288,7 @@ func (it *InsertMsg) IndexMsg(index int) *InsertMsg {
// DeleteMsg is a message pack that contains delete request
type DeleteMsg struct {
BaseMsg
internalpb.DeleteRequest
msgpb.DeleteRequest
}
// interface implementation validation
@ -326,7 +323,7 @@ func (dt *DeleteMsg) Marshal(input TsMsg) (MarshalType, error) {
// Unmarshal is used to deserializing a message pack from byte array
func (dt *DeleteMsg) Unmarshal(input MarshalType) (TsMsg, error) {
deleteRequest := internalpb.DeleteRequest{}
deleteRequest := msgpb.DeleteRequest{}
in, err := convertToByteArray(input)
if err != nil {
return nil, err
@ -391,7 +388,7 @@ type UpsertMsg struct {
// TimeTickMsg is a message pack that contains time tick only
type TimeTickMsg struct {
BaseMsg
internalpb.TimeTickMsg
msgpb.TimeTickMsg
}
// interface implementation validation
@ -425,7 +422,7 @@ func (tst *TimeTickMsg) Marshal(input TsMsg) (MarshalType, error) {
// Unmarshal is used to deserializing a message pack from byte array
func (tst *TimeTickMsg) Unmarshal(input MarshalType) (TsMsg, error) {
timeTickMsg := internalpb.TimeTickMsg{}
timeTickMsg := msgpb.TimeTickMsg{}
in, err := convertToByteArray(input)
if err != nil {
return nil, err
@ -446,7 +443,7 @@ func (tst *TimeTickMsg) Unmarshal(input MarshalType) (TsMsg, error) {
// CreateCollectionMsg is a message pack that contains create collection request
type CreateCollectionMsg struct {
BaseMsg
internalpb.CreateCollectionRequest
msgpb.CreateCollectionRequest
}
// interface implementation validation
@ -480,7 +477,7 @@ func (cc *CreateCollectionMsg) Marshal(input TsMsg) (MarshalType, error) {
// Unmarshal is used to deserializing a message pack from byte array
func (cc *CreateCollectionMsg) Unmarshal(input MarshalType) (TsMsg, error) {
createCollectionRequest := internalpb.CreateCollectionRequest{}
createCollectionRequest := msgpb.CreateCollectionRequest{}
in, err := convertToByteArray(input)
if err != nil {
return nil, err
@ -501,7 +498,7 @@ func (cc *CreateCollectionMsg) Unmarshal(input MarshalType) (TsMsg, error) {
// DropCollectionMsg is a message pack that contains drop collection request
type DropCollectionMsg struct {
BaseMsg
internalpb.DropCollectionRequest
msgpb.DropCollectionRequest
}
// interface implementation validation
@ -535,7 +532,7 @@ func (dc *DropCollectionMsg) Marshal(input TsMsg) (MarshalType, error) {
// Unmarshal is used to deserializing a message pack from byte array
func (dc *DropCollectionMsg) Unmarshal(input MarshalType) (TsMsg, error) {
dropCollectionRequest := internalpb.DropCollectionRequest{}
dropCollectionRequest := msgpb.DropCollectionRequest{}
in, err := convertToByteArray(input)
if err != nil {
return nil, err
@ -556,7 +553,7 @@ func (dc *DropCollectionMsg) Unmarshal(input MarshalType) (TsMsg, error) {
// CreatePartitionMsg is a message pack that contains create partition request
type CreatePartitionMsg struct {
BaseMsg
internalpb.CreatePartitionRequest
msgpb.CreatePartitionRequest
}
// interface implementation validation
@ -590,7 +587,7 @@ func (cp *CreatePartitionMsg) Marshal(input TsMsg) (MarshalType, error) {
// Unmarshal is used to deserializing a message pack from byte array
func (cp *CreatePartitionMsg) Unmarshal(input MarshalType) (TsMsg, error) {
createPartitionRequest := internalpb.CreatePartitionRequest{}
createPartitionRequest := msgpb.CreatePartitionRequest{}
in, err := convertToByteArray(input)
if err != nil {
return nil, err
@ -611,7 +608,7 @@ func (cp *CreatePartitionMsg) Unmarshal(input MarshalType) (TsMsg, error) {
// DropPartitionMsg is a message pack that contains drop partition request
type DropPartitionMsg struct {
BaseMsg
internalpb.DropPartitionRequest
msgpb.DropPartitionRequest
}
// interface implementation validation
@ -645,7 +642,7 @@ func (dp *DropPartitionMsg) Marshal(input TsMsg) (MarshalType, error) {
// Unmarshal is used to deserializing a message pack from byte array
func (dp *DropPartitionMsg) Unmarshal(input MarshalType) (TsMsg, error) {
dropPartitionRequest := internalpb.DropPartitionRequest{}
dropPartitionRequest := msgpb.DropPartitionRequest{}
in, err := convertToByteArray(input)
if err != nil {
return nil, err
@ -666,7 +663,7 @@ func (dp *DropPartitionMsg) Unmarshal(input MarshalType) (TsMsg, error) {
// DataNodeTtMsg is a message pack that contains datanode time tick
type DataNodeTtMsg struct {
BaseMsg
datapb.DataNodeTtMsg
msgpb.DataNodeTtMsg
}
// interface implementation validation
@ -699,7 +696,7 @@ func (m *DataNodeTtMsg) Marshal(input TsMsg) (MarshalType, error) {
// Unmarshal is used to deserializing a message pack from byte array
func (m *DataNodeTtMsg) Unmarshal(input MarshalType) (TsMsg, error) {
msg := datapb.DataNodeTtMsg{}
msg := msgpb.DataNodeTtMsg{}
in, err := convertToByteArray(input)
if err != nil {
return nil, err

47
internal/mq/msgstream/msg_test.go
View File

@ -23,9 +23,8 @@ import (
"github.com/stretchr/testify/assert"
"github.com/milvus-io/milvus-proto/go-api/commonpb"
"github.com/milvus-io/milvus-proto/go-api/msgpb"
"github.com/milvus-io/milvus-proto/go-api/schemapb"
"github.com/milvus-io/milvus/internal/proto/datapb"
"github.com/milvus-io/milvus/internal/proto/internalpb"
)
func TestBaseMsg(t *testing.T) {
@ -84,7 +83,7 @@ func generateBaseMsg() BaseMsg {
func TestInsertMsg(t *testing.T) {
insertMsg := &InsertMsg{
BaseMsg: generateBaseMsg(),
InsertRequest: internalpb.InsertRequest{
InsertRequest: msgpb.InsertRequest{
Base: &commonpb.MsgBase{
MsgType: commonpb.MsgType_Insert,
MsgID: 1,
@ -137,8 +136,8 @@ func TestInsertMsg_Unmarshal_IllegalParameter(t *testing.T) {
func TestInsertMsg_RowBasedFormat(t *testing.T) {
msg := &InsertMsg{
InsertRequest: internalpb.InsertRequest{
Version: internalpb.InsertDataVersion_RowBased,
InsertRequest: msgpb.InsertRequest{
Version: msgpb.InsertDataVersion_RowBased,
},
}
assert.True(t, msg.IsRowBased())
@ -146,8 +145,8 @@ func TestInsertMsg_RowBasedFormat(t *testing.T) {
func TestInsertMsg_ColumnBasedFormat(t *testing.T) {
msg := &InsertMsg{
InsertRequest: internalpb.InsertRequest{
Version: internalpb.InsertDataVersion_ColumnBased,
InsertRequest: msgpb.InsertRequest{
Version: msgpb.InsertDataVersion_ColumnBased,
},
}
assert.True(t, msg.IsColumnBased())
@ -155,24 +154,24 @@ func TestInsertMsg_ColumnBasedFormat(t *testing.T) {
func TestInsertMsg_NRows(t *testing.T) {
msg1 := &InsertMsg{
InsertRequest: internalpb.InsertRequest{
InsertRequest: msgpb.InsertRequest{
RowData: []*commonpb.Blob{
{},
{},
},
FieldsData: nil,
Version: internalpb.InsertDataVersion_RowBased,
Version: msgpb.InsertDataVersion_RowBased,
},
}
assert.Equal(t, uint64(2), msg1.NRows())
msg2 := &InsertMsg{
InsertRequest: internalpb.InsertRequest{
InsertRequest: msgpb.InsertRequest{
RowData: nil,
FieldsData: []*schemapb.FieldData{
{},
},
NumRows: 2,
Version: internalpb.InsertDataVersion_ColumnBased,
Version: msgpb.InsertDataVersion_ColumnBased,
},
}
assert.Equal(t, uint64(2), msg2.NRows())
@ -180,14 +179,14 @@ func TestInsertMsg_NRows(t *testing.T) {
func TestInsertMsg_CheckAligned(t *testing.T) {
msg1 := &InsertMsg{
InsertRequest: internalpb.InsertRequest{
InsertRequest: msgpb.InsertRequest{
Timestamps: []uint64{1},
RowIDs: []int64{1},
RowData: []*commonpb.Blob{
{},
},
FieldsData: nil,
Version: internalpb.InsertDataVersion_RowBased,
Version: msgpb.InsertDataVersion_RowBased,
},
}
msg1.InsertRequest.NumRows = 1
@ -208,7 +207,7 @@ func TestInsertMsg_CheckAligned(t *testing.T) {
},
}
msg1.Version = internalpb.InsertDataVersion_ColumnBased
msg1.Version = msgpb.InsertDataVersion_ColumnBased
assert.NoError(t, msg1.CheckAligned())
}
@ -218,7 +217,7 @@ func TestInsertMsg_IndexMsg(t *testing.T) {
BeginTimestamp: 1,
EndTimestamp: 2,
},
InsertRequest: internalpb.InsertRequest{
InsertRequest: msgpb.InsertRequest{
Base: &commonpb.MsgBase{
MsgType: commonpb.MsgType_Insert,
MsgID: 3,
@ -239,7 +238,7 @@ func TestInsertMsg_IndexMsg(t *testing.T) {
Value: []byte{1},
},
},
Version: internalpb.InsertDataVersion_RowBased,
Version: msgpb.InsertDataVersion_RowBased,
},
}
indexMsg := msg.IndexMsg(0)
@ -247,7 +246,7 @@ func TestInsertMsg_IndexMsg(t *testing.T) {
assert.Equal(t, int64(11), indexMsg.GetRowIDs()[0])
assert.Equal(t, []byte{1}, indexMsg.GetRowData()[0].Value)
msg.Version = internalpb.InsertDataVersion_ColumnBased
msg.Version = msgpb.InsertDataVersion_ColumnBased
msg.FieldsData = []*schemapb.FieldData{
{
Type: schemapb.DataType_Int64,
@ -273,7 +272,7 @@ func TestInsertMsg_IndexMsg(t *testing.T) {
func TestDeleteMsg(t *testing.T) {
deleteMsg := &DeleteMsg{
BaseMsg: generateBaseMsg(),
DeleteRequest: internalpb.DeleteRequest{
DeleteRequest: msgpb.DeleteRequest{
Base: &commonpb.MsgBase{
MsgType: commonpb.MsgType_Delete,
MsgID: 1,
@ -322,7 +321,7 @@ func TestDeleteMsg_Unmarshal_IllegalParameter(t *testing.T) {
func TestTimeTickMsg(t *testing.T) {
timeTickMsg := &TimeTickMsg{
BaseMsg: generateBaseMsg(),
TimeTickMsg: internalpb.TimeTickMsg{
TimeTickMsg: msgpb.TimeTickMsg{
Base: &commonpb.MsgBase{
MsgType: commonpb.MsgType_TimeTick,
MsgID: 1,
@ -365,7 +364,7 @@ func TestTimeTickMsg_Unmarshal_IllegalParameter(t *testing.T) {
func TestCreateCollectionMsg(t *testing.T) {
createCollectionMsg := &CreateCollectionMsg{
BaseMsg: generateBaseMsg(),
CreateCollectionRequest: internalpb.CreateCollectionRequest{
CreateCollectionRequest: msgpb.CreateCollectionRequest{
Base: &commonpb.MsgBase{
MsgType: commonpb.MsgType_CreateCollection,
MsgID: 1,
@ -417,7 +416,7 @@ func TestCreateCollectionMsg_Unmarshal_IllegalParameter(t *testing.T) {
func TestDropCollectionMsg(t *testing.T) {
dropCollectionMsg := &DropCollectionMsg{
BaseMsg: generateBaseMsg(),
DropCollectionRequest: internalpb.DropCollectionRequest{
DropCollectionRequest: msgpb.DropCollectionRequest{
Base: &commonpb.MsgBase{
MsgType: commonpb.MsgType_DropCollection,
MsgID: 1,
@ -464,7 +463,7 @@ func TestDropCollectionMsg_Unmarshal_IllegalParameter(t *testing.T) {
func TestCreatePartitionMsg(t *testing.T) {
createPartitionMsg := &CreatePartitionMsg{
BaseMsg: generateBaseMsg(),
CreatePartitionRequest: internalpb.CreatePartitionRequest{
CreatePartitionRequest: msgpb.CreatePartitionRequest{
Base: &commonpb.MsgBase{
MsgType: commonpb.MsgType_CreatePartition,
MsgID: 1,
@ -513,7 +512,7 @@ func TestCreatePartitionMsg_Unmarshal_IllegalParameter(t *testing.T) {
func TestDropPartitionMsg(t *testing.T) {
dropPartitionMsg := &DropPartitionMsg{
BaseMsg: generateBaseMsg(),
DropPartitionRequest: internalpb.DropPartitionRequest{
DropPartitionRequest: msgpb.DropPartitionRequest{
Base: &commonpb.MsgBase{
MsgType: commonpb.MsgType_DropPartition,
MsgID: 1,
@ -562,7 +561,7 @@ func TestDropPartitionMsg_Unmarshal_IllegalParameter(t *testing.T) {
func TestDataNodeTtMsg(t *testing.T) {
dataNodeTtMsg := &DataNodeTtMsg{
BaseMsg: generateBaseMsg(),
DataNodeTtMsg: datapb.DataNodeTtMsg{
DataNodeTtMsg: msgpb.DataNodeTtMsg{
Base: &commonpb.MsgBase{
MsgType: commonpb.MsgType_DataNodeTt,
MsgID: 1,

4
internal/mq/msgstream/msgstream.go
View File

@ -19,8 +19,8 @@ package msgstream
import (
"context"
"github.com/milvus-io/milvus-proto/go-api/msgpb"
"github.com/milvus-io/milvus/internal/mq/msgstream/mqwrapper"
"github.com/milvus-io/milvus/internal/proto/internalpb"
"github.com/milvus-io/milvus/internal/util/typeutil"
)
@ -34,7 +34,7 @@ type Timestamp = typeutil.Timestamp
type IntPrimaryKey = typeutil.IntPrimaryKey
// MsgPosition is an alias for short
type MsgPosition = internalpb.MsgPosition
type MsgPosition = msgpb.MsgPosition
// MessageID is an alias for short
type MessageID = mqwrapper.MessageID

6
internal/mq/msgstream/unmarshal_test.go
View File

@ -24,11 +24,11 @@ import (
"github.com/stretchr/testify/assert"
"github.com/milvus-io/milvus-proto/go-api/commonpb"
"github.com/milvus-io/milvus/internal/proto/internalpb"
"github.com/milvus-io/milvus-proto/go-api/msgpb"
)
func newInsertMsgUnmarshal(input []byte) (TsMsg, error) {
insertRequest := internalpb.InsertRequest{}
insertRequest := msgpb.InsertRequest{}
err := proto.Unmarshal(input, &insertRequest)
insertMsg := &InsertMsg{InsertRequest: insertRequest}
fmt.Println("use func newInsertMsgUnmarshal unmarshal")
@ -47,7 +47,7 @@ func Test_ProtoUnmarshalDispatcher(t *testing.T) {
EndTimestamp: 0,
HashValues: []uint32{1},
},
InsertRequest: internalpb.InsertRequest{
InsertRequest: msgpb.InsertRequest{
Base: &commonpb.MsgBase{
MsgType: commonpb.MsgType_Insert,
MsgID: 1,

37
internal/proto/data_coord.proto
View File

@ -8,6 +8,7 @@ import "common.proto";
import "internal.proto";
import "milvus.proto";
import "schema.proto";
import "msg.proto";
import "index_coord.proto";
// TODO: import google/protobuf/empty.proto
@ -159,8 +160,8 @@ message GetSegmentStatesRequest {
message SegmentStateInfo {
int64 segmentID = 1;
common.SegmentState state = 2;
internal.MsgPosition start_position = 3;
internal.MsgPosition end_position = 4;
msg.MsgPosition start_position = 3;
msg.MsgPosition end_position = 4;
common.Status status = 5;
}
@ -178,7 +179,7 @@ message GetSegmentInfoRequest {
message GetSegmentInfoResponse {
common.Status status = 1;
repeated SegmentInfo infos = 2;
map<string, internal.MsgPosition> channel_checkpoint = 3;
map<string, msg.MsgPosition> channel_checkpoint = 3;
}
message GetInsertBinlogPathsRequest {
@ -221,7 +222,7 @@ message GetSegmentInfoChannelRequest {
message VchannelInfo {
int64 collectionID = 1;
string channelName = 2;
internal.MsgPosition seek_position = 3;
msg.MsgPosition seek_position = 3;
repeated SegmentInfo unflushedSegments = 4; // deprecated, keep it for compatibility
repeated SegmentInfo flushedSegments = 5; // deprecated, keep it for compatibility
repeated SegmentInfo dropped_segments = 6; // deprecated, keep it for compatibility
@ -258,8 +259,8 @@ message SegmentInfo {
common.SegmentState state = 6;
int64 max_row_num = 7;
uint64 last_expire_time = 8;
internal.MsgPosition start_position = 9;
internal.MsgPosition dml_position = 10;
msg.MsgPosition start_position = 9;
msg.MsgPosition dml_position = 10;
// binlogs consist of insert binlogs
repeated FieldBinlog binlogs = 11;
repeated FieldBinlog statslogs = 12;
@ -276,7 +277,7 @@ message SegmentInfo {
}
message SegmentStartPosition {
internal.MsgPosition start_position = 1;
msg.MsgPosition start_position = 1;
int64 segmentID = 2;
}
@ -296,7 +297,7 @@ message SaveBinlogPathsRequest {
message CheckPoint {
int64 segmentID = 1;
internal.MsgPosition position = 2;
msg.MsgPosition position = 2;
int64 num_of_rows = 3;
}
@ -308,18 +309,6 @@ message DeltaLogInfo {
int64 delta_log_size = 5;
}
message DataNodeTtMsg {
common.MsgBase base =1;
string channel_name = 2;
uint64 timestamp = 3;
repeated SegmentStats segments_stats = 4;
}
message SegmentStats {
int64 SegmentID = 1;
int64 NumRows = 2;
}
enum ChannelWatchState {
Uncomplete = 0; // deprecated, keep it for compatibility
Complete = 1; // deprecated, keep it for compatibility
@ -520,8 +509,8 @@ message DropVirtualChannelSegment {
repeated FieldBinlog field2BinlogPaths = 3;
repeated FieldBinlog field2StatslogPaths = 4;
repeated FieldBinlog deltalogs = 5;
internal.MsgPosition startPosition = 6;
internal.MsgPosition checkPoint = 7;
msg.MsgPosition startPosition = 6;
msg.MsgPosition checkPoint = 7;
int64 numOfRows = 8;
}
@ -579,13 +568,13 @@ message ImportTaskRequest {
message UpdateSegmentStatisticsRequest {
common.MsgBase base = 1;
repeated SegmentStats stats = 2;
repeated common.SegmentStats stats = 2;
}
message UpdateChannelCheckpointRequest {
common.MsgBase base = 1;
string vChannel = 2;
internal.MsgPosition position = 3;
msg.MsgPosition position = 3;
}
message ResendSegmentStatsRequest {

961
internal/proto/datapb/data_coord.pb.go
View File

File diff suppressed because it is too large Load Diff

3
internal/proto/etcdpb/etcd_meta.pb.go
View File

@ -5,11 +5,10 @@ package etcdpb
import (
fmt "fmt"
math "math"
proto "github.com/golang/protobuf/proto"
commonpb "github.com/milvus-io/milvus-proto/go-api/commonpb"
schemapb "github.com/milvus-io/milvus-proto/go-api/schemapb"
math "math"
)
// Reference imports to suppress errors if they are not otherwise used.

3
internal/proto/indexcgopb/index_cgo_msg.pb.go
View File

@ -5,10 +5,9 @@ package indexcgopb
import (
fmt "fmt"
math "math"
proto "github.com/golang/protobuf/proto"
commonpb "github.com/milvus-io/milvus-proto/go-api/commonpb"
math "math"
)
// Reference imports to suppress errors if they are not otherwise used.

3
internal/proto/indexpb/index_coord.pb.go
View File

@ -6,8 +6,6 @@ package indexpb
import (
context "context"
fmt "fmt"
math "math"
proto "github.com/golang/protobuf/proto"
commonpb "github.com/milvus-io/milvus-proto/go-api/commonpb"
milvuspb "github.com/milvus-io/milvus-proto/go-api/milvuspb"
@ -15,6 +13,7 @@ import (
grpc "google.golang.org/grpc"
codes "google.golang.org/grpc/codes"
status "google.golang.org/grpc/status"
math "math"
)
// Reference imports to suppress errors if they are not otherwise used.

93
internal/proto/internal.proto
View File

@ -29,10 +29,6 @@ message StringList {
common.Status status = 2;
}
message TimeTickMsg {
common.MsgBase base = 1;
}
message GetStatisticsRequest {
common.MsgBase base = 1;
// Not useful for now
@ -55,48 +51,6 @@ message GetStatisticsResponse {
repeated common.KeyValuePair stats = 3;
}
message CreateCollectionRequest {
common.MsgBase base = 1;
string db_name = 2;
string collectionName = 3;
string partitionName = 4;
// `schema` is the serialized `schema.CollectionSchema`
int64 dbID = 5;
int64 collectionID = 6;
int64 partitionID = 7;
bytes schema = 8;
repeated string virtualChannelNames = 9;
repeated string physicalChannelNames = 10;
}
message DropCollectionRequest {
common.MsgBase base = 1;
string db_name = 2;
string collectionName = 3;
int64 dbID = 4;
int64 collectionID = 5;
}
message CreatePartitionRequest {
common.MsgBase base = 1;
string db_name = 2;
string collection_name = 3;
string partition_name = 4;
int64 dbID = 5;
int64 collectionID = 6;
int64 partitionID = 7;
}
message DropPartitionRequest {
common.MsgBase base = 1;
string db_name = 2;
string collection_name = 3;
string partition_name = 4;
int64 dbID = 5;
int64 collectionID = 6;
int64 partitionID = 7;
}
message CreateAliasRequest {
common.MsgBase base = 1;
string db_name = 2;
@ -128,31 +82,6 @@ message CreateIndexRequest {
repeated common.KeyValuePair extra_params = 8;
}
enum InsertDataVersion {
// 0 must refer to row-based format, since it's the first version in Milvus.
RowBased = 0;
ColumnBased = 1;
}
message InsertRequest {
common.MsgBase base = 1;
string shardName = 2;
string db_name = 3;
string collection_name = 4;
string partition_name = 5;
int64 dbID = 6;
int64 collectionID = 7;
int64 partitionID = 8;
int64 segmentID = 9;
repeated uint64 timestamps = 10;
repeated int64 rowIDs = 11;
// row_data was reserved for compatibility
repeated common.Blob row_data = 12;
repeated schema.FieldData fields_data = 13;
uint64 num_rows = 14;
InsertDataVersion version = 15;
}
message SearchRequest {
common.MsgBase base = 1;
int64 reqID = 2;
@ -216,21 +145,6 @@ message RetrieveResults {
repeated int64 global_sealed_segmentIDs = 8;
}
message DeleteRequest {
common.MsgBase base = 1;
string shardName = 2;
string db_name = 3;
string collection_name = 4;
string partition_name = 5;
int64 dbID = 6;
int64 collectionID = 7;
int64 partitionID = 8;
repeated int64 int64_primary_keys = 9; // deprecated
repeated uint64 timestamps = 10;
int64 num_rows = 11;
schema.IDs primary_keys = 12;
}
message LoadIndex {
common.MsgBase base = 1;
int64 segmentID = 2;
@ -258,13 +172,6 @@ message SegmentStats {
bool recently_modified = 4;
}
message MsgPosition {
string channel_name = 1;
bytes msgID = 2;
string msgGroup = 3;
uint64 timestamp = 4;
}
message ChannelTimeTickMsg {
common.MsgBase base = 1;
repeated string channelNames = 2;

1074
internal/proto/internalpb/internal.pb.go
View File

File diff suppressed because it is too large Load Diff

3
internal/proto/planpb/plan.pb.go
View File

@ -5,10 +5,9 @@ package planpb
import (
fmt "fmt"
math "math"
proto "github.com/golang/protobuf/proto"
schemapb "github.com/milvus-io/milvus-proto/go-api/schemapb"
math "math"
)
// Reference imports to suppress errors if they are not otherwise used.

3
internal/proto/proxypb/proxy.pb.go
View File

@ -6,8 +6,6 @@ package proxypb
import (
context "context"
fmt "fmt"
math "math"
proto "github.com/golang/protobuf/proto"
commonpb "github.com/milvus-io/milvus-proto/go-api/commonpb"
milvuspb "github.com/milvus-io/milvus-proto/go-api/milvuspb"
@ -15,6 +13,7 @@ import (
grpc "google.golang.org/grpc"
codes "google.golang.org/grpc/codes"
status "google.golang.org/grpc/status"
math "math"
)
// Reference imports to suppress errors if they are not otherwise used.

9
internal/proto/query_coord.proto
View File

@ -8,6 +8,7 @@ import "common.proto";
import "milvus.proto";
import "internal.proto";
import "schema.proto";
import "msg.proto";
import "data_coord.proto";
service QueryCoord {
@ -233,7 +234,7 @@ message SegmentLoadInfo {
repeated FieldIndexInfo index_infos = 11;
int64 segment_size = 12;
string insert_channel = 13;
internal.MsgPosition start_position = 14;
msg.MsgPosition start_position = 14;
}
message FieldIndexInfo {
@ -259,7 +260,7 @@ message LoadSegmentsRequest {
int64 collectionID = 6;
LoadMetaInfo load_meta = 7;
int64 replicaID = 8;
repeated internal.MsgPosition delta_positions = 9;
repeated msg.MsgPosition delta_positions = 9;
int64 version = 10;
bool need_transfer = 11;
}
@ -368,7 +369,7 @@ message QueryChannelInfo {
string query_channel = 2;
string query_result_channel = 3;
repeated SegmentInfo global_sealed_segments = 4;
internal.MsgPosition seek_position = 5;
msg.MsgPosition seek_position = 5;
}
message PartitionStates {
@ -450,7 +451,7 @@ message LeaderView {
string channel = 2;
map<int64, SegmentDist> segment_dist = 3;
repeated int64 growing_segmentIDs = 4;
map<int64, internal.MsgPosition> growing_segments = 5;
map<int64, msg.MsgPosition> growing_segments = 5;
}
message SegmentDist {

613
internal/proto/querypb/query_coord.pb.go
View File

@ -6,17 +6,17 @@ package querypb
import (
context "context"
fmt "fmt"
math "math"
proto "github.com/golang/protobuf/proto"
commonpb "github.com/milvus-io/milvus-proto/go-api/commonpb"
milvuspb "github.com/milvus-io/milvus-proto/go-api/milvuspb"
msgpb "github.com/milvus-io/milvus-proto/go-api/msgpb"
schemapb "github.com/milvus-io/milvus-proto/go-api/schemapb"
datapb "github.com/milvus-io/milvus/internal/proto/datapb"
internalpb "github.com/milvus-io/milvus/internal/proto/internalpb"
grpc "google.golang.org/grpc"
codes "google.golang.org/grpc/codes"
status "google.golang.org/grpc/status"
math "math"
)
// Reference imports to suppress errors if they are not otherwise used.
@ -216,7 +216,7 @@ func (SyncType) EnumDescriptor() ([]byte, []int) {
return fileDescriptor_aab7cc9a69ed26e8, []int{5}
}
// --------------------QueryCoord grpc request and response proto------------------
//--------------------QueryCoord grpc request and response proto------------------
type ShowCollectionsRequest struct {
Base *commonpb.MsgBase `protobuf:"bytes,1,opt,name=base,proto3" json:"base,omitempty"`
// Not useful for now
@ -1222,7 +1222,7 @@ func (m *ShardLeadersList) GetNodeAddrs() []string {
return nil
}
// -----------------query node grpc request and response proto----------------
//-----------------query node grpc request and response proto----------------
type LoadMetaInfo struct {
LoadType LoadType `protobuf:"varint,1,opt,name=load_type,json=loadType,proto3,enum=milvus.proto.query.LoadType" json:"load_type,omitempty"`
CollectionID int64 `protobuf:"varint,2,opt,name=collectionID,proto3" json:"collectionID,omitempty"`
@ -1471,23 +1471,23 @@ func (m *UnsubDmChannelRequest) GetChannelName() string {
}
type SegmentLoadInfo struct {
SegmentID int64 `protobuf:"varint,1,opt,name=segmentID,proto3" json:"segmentID,omitempty"`
PartitionID int64 `protobuf:"varint,2,opt,name=partitionID,proto3" json:"partitionID,omitempty"`
CollectionID int64 `protobuf:"varint,3,opt,name=collectionID,proto3" json:"collectionID,omitempty"`
DbID int64 `protobuf:"varint,4,opt,name=dbID,proto3" json:"dbID,omitempty"`
FlushTime int64 `protobuf:"varint,5,opt,name=flush_time,json=flushTime,proto3" json:"flush_time,omitempty"`
BinlogPaths []*datapb.FieldBinlog `protobuf:"bytes,6,rep,name=binlog_paths,json=binlogPaths,proto3" json:"binlog_paths,omitempty"`
NumOfRows int64 `protobuf:"varint,7,opt,name=num_of_rows,json=numOfRows,proto3" json:"num_of_rows,omitempty"`
Statslogs []*datapb.FieldBinlog `protobuf:"bytes,8,rep,name=statslogs,proto3" json:"statslogs,omitempty"`
Deltalogs []*datapb.FieldBinlog `protobuf:"bytes,9,rep,name=deltalogs,proto3" json:"deltalogs,omitempty"`
CompactionFrom []int64 `protobuf:"varint,10,rep,packed,name=compactionFrom,proto3" json:"compactionFrom,omitempty"`
IndexInfos []*FieldIndexInfo `protobuf:"bytes,11,rep,name=index_infos,json=indexInfos,proto3" json:"index_infos,omitempty"`
SegmentSize int64 `protobuf:"varint,12,opt,name=segment_size,json=segmentSize,proto3" json:"segment_size,omitempty"`
InsertChannel string `protobuf:"bytes,13,opt,name=insert_channel,json=insertChannel,proto3" json:"insert_channel,omitempty"`
StartPosition *internalpb.MsgPosition `protobuf:"bytes,14,opt,name=start_position,json=startPosition,proto3" json:"start_position,omitempty"`
XXX_NoUnkeyedLiteral struct{} `json:"-"`
XXX_unrecognized []byte `json:"-"`
XXX_sizecache int32 `json:"-"`
SegmentID int64 `protobuf:"varint,1,opt,name=segmentID,proto3" json:"segmentID,omitempty"`
PartitionID int64 `protobuf:"varint,2,opt,name=partitionID,proto3" json:"partitionID,omitempty"`
CollectionID int64 `protobuf:"varint,3,opt,name=collectionID,proto3" json:"collectionID,omitempty"`
DbID int64 `protobuf:"varint,4,opt,name=dbID,proto3" json:"dbID,omitempty"`
FlushTime int64 `protobuf:"varint,5,opt,name=flush_time,json=flushTime,proto3" json:"flush_time,omitempty"`
BinlogPaths []*datapb.FieldBinlog `protobuf:"bytes,6,rep,name=binlog_paths,json=binlogPaths,proto3" json:"binlog_paths,omitempty"`
NumOfRows int64 `protobuf:"varint,7,opt,name=num_of_rows,json=numOfRows,proto3" json:"num_of_rows,omitempty"`
Statslogs []*datapb.FieldBinlog `protobuf:"bytes,8,rep,name=statslogs,proto3" json:"statslogs,omitempty"`
Deltalogs []*datapb.FieldBinlog `protobuf:"bytes,9,rep,name=deltalogs,proto3" json:"deltalogs,omitempty"`
CompactionFrom []int64 `protobuf:"varint,10,rep,packed,name=compactionFrom,proto3" json:"compactionFrom,omitempty"`
IndexInfos []*FieldIndexInfo `protobuf:"bytes,11,rep,name=index_infos,json=indexInfos,proto3" json:"index_infos,omitempty"`
SegmentSize int64 `protobuf:"varint,12,opt,name=segment_size,json=segmentSize,proto3" json:"segment_size,omitempty"`
InsertChannel string `protobuf:"bytes,13,opt,name=insert_channel,json=insertChannel,proto3" json:"insert_channel,omitempty"`
StartPosition *msgpb.MsgPosition `protobuf:"bytes,14,opt,name=start_position,json=startPosition,proto3" json:"start_position,omitempty"`
XXX_NoUnkeyedLiteral struct{} `json:"-"`
XXX_unrecognized []byte `json:"-"`
XXX_sizecache int32 `json:"-"`
}
func (m *SegmentLoadInfo) Reset() { *m = SegmentLoadInfo{} }
@ -1606,7 +1606,7 @@ func (m *SegmentLoadInfo) GetInsertChannel() string {
return ""
}
func (m *SegmentLoadInfo) GetStartPosition() *internalpb.MsgPosition {
func (m *SegmentLoadInfo) GetStartPosition() *msgpb.MsgPosition {
if m != nil {
return m.StartPosition
}
@ -1734,7 +1734,7 @@ type LoadSegmentsRequest struct {
CollectionID int64 `protobuf:"varint,6,opt,name=collectionID,proto3" json:"collectionID,omitempty"`
LoadMeta *LoadMetaInfo `protobuf:"bytes,7,opt,name=load_meta,json=loadMeta,proto3" json:"load_meta,omitempty"`
ReplicaID int64 `protobuf:"varint,8,opt,name=replicaID,proto3" json:"replicaID,omitempty"`
DeltaPositions []*internalpb.MsgPosition `protobuf:"bytes,9,rep,name=delta_positions,json=deltaPositions,proto3" json:"delta_positions,omitempty"`
DeltaPositions []*msgpb.MsgPosition `protobuf:"bytes,9,rep,name=delta_positions,json=deltaPositions,proto3" json:"delta_positions,omitempty"`
Version int64 `protobuf:"varint,10,opt,name=version,proto3" json:"version,omitempty"`
NeedTransfer bool `protobuf:"varint,11,opt,name=need_transfer,json=needTransfer,proto3" json:"need_transfer,omitempty"`
XXX_NoUnkeyedLiteral struct{} `json:"-"`
@ -1823,7 +1823,7 @@ func (m *LoadSegmentsRequest) GetReplicaID() int64 {
return 0
}
func (m *LoadSegmentsRequest) GetDeltaPositions() []*internalpb.MsgPosition {
func (m *LoadSegmentsRequest) GetDeltaPositions() []*msgpb.MsgPosition {
if m != nil {
return m.DeltaPositions
}
@ -2208,7 +2208,7 @@ func (m *ReplicaSegmentsInfo) GetVersions() []int64 {
return nil
}
// ----------------request auto triggered by QueryCoord-----------------
//----------------request auto triggered by QueryCoord-----------------
type HandoffSegmentsRequest struct {
Base *commonpb.MsgBase `protobuf:"bytes,1,opt,name=base,proto3" json:"base,omitempty"`
SegmentInfos []*SegmentInfo `protobuf:"bytes,2,rep,name=segmentInfos,proto3" json:"segmentInfos,omitempty"`
@ -2415,14 +2415,14 @@ func (m *DmChannelWatchInfo) GetNodeIds() []int64 {
}
type QueryChannelInfo struct {
CollectionID int64 `protobuf:"varint,1,opt,name=collectionID,proto3" json:"collectionID,omitempty"`
QueryChannel string `protobuf:"bytes,2,opt,name=query_channel,json=queryChannel,proto3" json:"query_channel,omitempty"`
QueryResultChannel string `protobuf:"bytes,3,opt,name=query_result_channel,json=queryResultChannel,proto3" json:"query_result_channel,omitempty"`
GlobalSealedSegments []*SegmentInfo `protobuf:"bytes,4,rep,name=global_sealed_segments,json=globalSealedSegments,proto3" json:"global_sealed_segments,omitempty"`
SeekPosition *internalpb.MsgPosition `protobuf:"bytes,5,opt,name=seek_position,json=seekPosition,proto3" json:"seek_position,omitempty"`
XXX_NoUnkeyedLiteral struct{} `json:"-"`
XXX_unrecognized []byte `json:"-"`
XXX_sizecache int32 `json:"-"`
CollectionID int64 `protobuf:"varint,1,opt,name=collectionID,proto3" json:"collectionID,omitempty"`
QueryChannel string `protobuf:"bytes,2,opt,name=query_channel,json=queryChannel,proto3" json:"query_channel,omitempty"`
QueryResultChannel string `protobuf:"bytes,3,opt,name=query_result_channel,json=queryResultChannel,proto3" json:"query_result_channel,omitempty"`
GlobalSealedSegments []*SegmentInfo `protobuf:"bytes,4,rep,name=global_sealed_segments,json=globalSealedSegments,proto3" json:"global_sealed_segments,omitempty"`
SeekPosition *msgpb.MsgPosition `protobuf:"bytes,5,opt,name=seek_position,json=seekPosition,proto3" json:"seek_position,omitempty"`
XXX_NoUnkeyedLiteral struct{} `json:"-"`
XXX_unrecognized []byte `json:"-"`
XXX_sizecache int32 `json:"-"`
}
func (m *QueryChannelInfo) Reset() { *m = QueryChannelInfo{} }
@ -2478,7 +2478,7 @@ func (m *QueryChannelInfo) GetGlobalSealedSegments() []*SegmentInfo {
return nil
}
func (m *QueryChannelInfo) GetSeekPosition() *internalpb.MsgPosition {
func (m *QueryChannelInfo) GetSeekPosition() *msgpb.MsgPosition {
if m != nil {
return m.SeekPosition
}
@ -2905,7 +2905,7 @@ func (m *UnsubscribeChannelInfo) GetCollectionChannels() []*UnsubscribeChannels
return nil
}
// ---- synchronize messages proto between QueryCoord and QueryNode -----
//---- synchronize messages proto between QueryCoord and QueryNode -----
type SegmentChangeInfo struct {
OnlineNodeID int64 `protobuf:"varint,1,opt,name=online_nodeID,json=onlineNodeID,proto3" json:"online_nodeID,omitempty"`
OnlineSegments []*SegmentInfo `protobuf:"bytes,2,rep,name=online_segments,json=onlineSegments,proto3" json:"online_segments,omitempty"`
@ -3127,14 +3127,14 @@ func (m *GetDataDistributionResponse) GetLeaderViews() []*LeaderView {
}
type LeaderView struct {
Collection int64 `protobuf:"varint,1,opt,name=collection,proto3" json:"collection,omitempty"`
Channel string `protobuf:"bytes,2,opt,name=channel,proto3" json:"channel,omitempty"`
SegmentDist map[int64]*SegmentDist `protobuf:"bytes,3,rep,name=segment_dist,json=segmentDist,proto3" json:"segment_dist,omitempty" protobuf_key:"varint,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"`
GrowingSegmentIDs []int64 `protobuf:"varint,4,rep,packed,name=growing_segmentIDs,json=growingSegmentIDs,proto3" json:"growing_segmentIDs,omitempty"`
GrowingSegments map[int64]*internalpb.MsgPosition `protobuf:"bytes,5,rep,name=growing_segments,json=growingSegments,proto3" json:"growing_segments,omitempty" protobuf_key:"varint,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"`
XXX_NoUnkeyedLiteral struct{} `json:"-"`
XXX_unrecognized []byte `json:"-"`
XXX_sizecache int32 `json:"-"`
Collection int64 `protobuf:"varint,1,opt,name=collection,proto3" json:"collection,omitempty"`
Channel string `protobuf:"bytes,2,opt,name=channel,proto3" json:"channel,omitempty"`
SegmentDist map[int64]*SegmentDist `protobuf:"bytes,3,rep,name=segment_dist,json=segmentDist,proto3" json:"segment_dist,omitempty" protobuf_key:"varint,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"`
GrowingSegmentIDs []int64 `protobuf:"varint,4,rep,packed,name=growing_segmentIDs,json=growingSegmentIDs,proto3" json:"growing_segmentIDs,omitempty"`
GrowingSegments map[int64]*msgpb.MsgPosition `protobuf:"bytes,5,rep,name=growing_segments,json=growingSegments,proto3" json:"growing_segments,omitempty" protobuf_key:"varint,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"`
XXX_NoUnkeyedLiteral struct{} `json:"-"`
XXX_unrecognized []byte `json:"-"`
XXX_sizecache int32 `json:"-"`
}
func (m *LeaderView) Reset() { *m = LeaderView{} }
@ -3190,7 +3190,7 @@ func (m *LeaderView) GetGrowingSegmentIDs() []int64 {
return nil
}
func (m *LeaderView) GetGrowingSegments() map[int64]*internalpb.MsgPosition {
func (m *LeaderView) GetGrowingSegments() map[int64]*msgpb.MsgPosition {
if m != nil {
return m.GrowingSegments
}
@ -4063,7 +4063,7 @@ func init() {
proto.RegisterType((*GetDataDistributionRequest)(nil), "milvus.proto.query.GetDataDistributionRequest")
proto.RegisterType((*GetDataDistributionResponse)(nil), "milvus.proto.query.GetDataDistributionResponse")
proto.RegisterType((*LeaderView)(nil), "milvus.proto.query.LeaderView")
proto.RegisterMapType((map[int64]*internalpb.MsgPosition)(nil), "milvus.proto.query.LeaderView.GrowingSegmentsEntry")
proto.RegisterMapType((map[int64]*msgpb.MsgPosition)(nil), "milvus.proto.query.LeaderView.GrowingSegmentsEntry")
proto.RegisterMapType((map[int64]*SegmentDist)(nil), "milvus.proto.query.LeaderView.SegmentDistEntry")
proto.RegisterType((*SegmentDist)(nil), "milvus.proto.query.SegmentDist")
proto.RegisterType((*SegmentVersionInfo)(nil), "milvus.proto.query.SegmentVersionInfo")
@ -4088,267 +4088,268 @@ func init() {
func init() { proto.RegisterFile("query_coord.proto", fileDescriptor_aab7cc9a69ed26e8) }
var fileDescriptor_aab7cc9a69ed26e8 = []byte{
// 4155 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xec, 0x3b, 0x49, 0x6c, 0x24, 0x59,
0x56, 0x15, 0xb9, 0xd8, 0x99, 0x2f, 0x17, 0xa7, 0xbf, 0x97, 0xca, 0xc9, 0xa9, 0xc5, 0x1d, 0xd5,
0xd5, 0x6d, 0xdc, 0xdd, 0x76, 0x8f, 0x6b, 0xa6, 0xa9, 0xd9, 0x34, 0x54, 0xd9, 0x53, 0x6e, 0xd3,
0x55, 0x6e, 0x13, 0xae, 0xaa, 0x41, 0xad, 0x66, 0x72, 0x22, 0x33, 0x7e, 0xa6, 0x43, 0x15, 0x19,
0x91, 0x15, 0x11, 0x69, 0xb7, 0x1b, 0x89, 0x13, 0x97, 0x41, 0x80, 0x04, 0x07, 0x4e, 0x88, 0x03,
0x02, 0x09, 0x24, 0x5a, 0xe2, 0x00, 0x37, 0x0e, 0x48, 0x48, 0x70, 0x02, 0x71, 0xe3, 0xc8, 0x15,
0x09, 0x24, 0x04, 0xd2, 0x68, 0x34, 0x37, 0xf4, 0xb7, 0xc8, 0xf8, 0x11, 0x3f, 0x9c, 0x61, 0xbb,
0xb7, 0x41, 0x73, 0x8b, 0x78, 0x7f, 0x79, 0xef, 0xbf, 0xfd, 0xfd, 0x05, 0x16, 0x5f, 0x4e, 0xb0,
0x7f, 0xd6, 0xed, 0x7b, 0x9e, 0x6f, 0x6d, 0x8e, 0x7d, 0x2f, 0xf4, 0x10, 0x1a, 0xd9, 0xce, 0xc9,
0x24, 0x60, 0x7f, 0x9b, 0xb4, 0xbd, 0x53, 0xef, 0x7b, 0xa3, 0x91, 0xe7, 0x32, 0x58, 0xa7, 0x1e,
0xef, 0xd1, 0x69, 0xda, 0x6e, 0x88, 0x7d, 0xd7, 0x74, 0x44, 0x6b, 0xd0, 0x3f, 0xc6, 0x23, 0x93,
0xff, 0xb5, 0x2c, 0x33, 0x34, 0xe3, 0xf3, 0xeb, 0xbf, 0xad, 0xc1, 0xea, 0xd1, 0xb1, 0x77, 0xba,
0xe3, 0x39, 0x0e, 0xee, 0x87, 0xb6, 0xe7, 0x06, 0x06, 0x7e, 0x39, 0xc1, 0x41, 0x88, 0xde, 0x86,
0x52, 0xcf, 0x0c, 0x70, 0x5b, 0x5b, 0xd3, 0xd6, 0x6b, 0xdb, 0x37, 0x36, 0x25, 0x4a, 0x38, 0x09,
0x4f, 0x82, 0xe1, 0x43, 0x33, 0xc0, 0x06, 0xed, 0x89, 0x10, 0x94, 0xac, 0xde, 0xfe, 0x6e, 0xbb,
0xb0, 0xa6, 0xad, 0x17, 0x0d, 0xfa, 0x8d, 0x5e, 0x85, 0x46, 0x3f, 0x9a, 0x7b, 0x7f, 0x37, 0x68,
0x17, 0xd7, 0x8a, 0xeb, 0x45, 0x43, 0x06, 0xea, 0xff, 0xae, 0xc1, 0xf5, 0x14, 0x19, 0xc1, 0xd8,
0x73, 0x03, 0x8c, 0xee, 0xc1, 0x5c, 0x10, 0x9a, 0xe1, 0x24, 0xe0, 0x94, 0x7c, 0x55, 0x49, 0xc9,
0x11, 0xed, 0x62, 0xf0, 0xae, 0x69, 0xb4, 0x05, 0x05, 0x5a, 0xf4, 0x35, 0x58, 0xb6, 0xdd, 0x27,
0x78, 0xe4, 0xf9, 0x67, 0xdd, 0x31, 0xf6, 0xfb, 0xd8, 0x0d, 0xcd, 0x21, 0x16, 0x34, 0x2e, 0x89,
0xb6, 0xc3, 0x69, 0x13, 0x7a, 0x07, 0xae, 0x33, 0x29, 0x05, 0xd8, 0x3f, 0xb1, 0xfb, 0xb8, 0x6b,
0x9e, 0x98, 0xb6, 0x63, 0xf6, 0x1c, 0xdc, 0x2e, 0xad, 0x15, 0xd7, 0x2b, 0xc6, 0x0a, 0x6d, 0x3e,
0x62, 0xad, 0x0f, 0x44, 0xa3, 0xfe, 0xe7, 0x1a, 0xac, 0x90, 0x15, 0x1e, 0x9a, 0x7e, 0x68, 0x7f,
0x06, 0x7c, 0xd6, 0xa1, 0x1e, 0x5f, 0x5b, 0xbb, 0x48, 0xdb, 0x24, 0x18, 0xe9, 0x33, 0x16, 0xe8,
0x09, 0x4f, 0x4a, 0x74, 0x99, 0x12, 0x4c, 0xff, 0x33, 0xae, 0x10, 0x71, 0x3a, 0xaf, 0x22, 0x88,
0x24, 0xce, 0x42, 0x1a, 0xe7, 0x25, 0xc4, 0xa0, 0xff, 0x73, 0x11, 0x56, 0x1e, 0x7b, 0xa6, 0x35,
0x55, 0x98, 0xcf, 0x9f, 0x9d, 0xdf, 0x85, 0x39, 0x66, 0x5d, 0xed, 0x12, 0xc5, 0x75, 0x57, 0xc6,
0xc5, 0x2d, 0x6f, 0x4a, 0xe1, 0x11, 0x05, 0x18, 0x7c, 0x10, 0xba, 0x0b, 0x4d, 0x1f, 0x8f, 0x1d,
0xbb, 0x6f, 0x76, 0xdd, 0xc9, 0xa8, 0x87, 0xfd, 0x76, 0x79, 0x4d, 0x5b, 0x2f, 0x1b, 0x0d, 0x0e,
0x3d, 0xa0, 0x40, 0xf4, 0x23, 0x68, 0x0c, 0x6c, 0xec, 0x58, 0x5d, 0xdb, 0xb5, 0xf0, 0x47, 0xfb,
0xbb, 0xed, 0xb9, 0xb5, 0xe2, 0x7a, 0x6d, 0xfb, 0xdb, 0x9b, 0x69, 0xcf, 0xb0, 0xa9, 0xe4, 0xc8,
0xe6, 0x23, 0x32, 0x7c, 0x9f, 0x8d, 0xfe, 0xbe, 0x1b, 0xfa, 0x67, 0x46, 0x7d, 0x10, 0x03, 0xa1,
0x36, 0xcc, 0xfb, 0x78, 0xe0, 0xe3, 0xe0, 0xb8, 0x3d, 0xbf, 0xa6, 0xad, 0x57, 0x0c, 0xf1, 0x8b,
0x5e, 0x87, 0x05, 0x1f, 0x07, 0xde, 0xc4, 0xef, 0xe3, 0xee, 0xd0, 0xf7, 0x26, 0xe3, 0xa0, 0x5d,
0x59, 0x2b, 0xae, 0x57, 0x8d, 0xa6, 0x00, 0xef, 0x51, 0x68, 0xe7, 0x7b, 0xb0, 0x98, 0xc2, 0x82,
0x5a, 0x50, 0x7c, 0x81, 0xcf, 0xa8, 0x20, 0x8a, 0x06, 0xf9, 0x44, 0xcb, 0x50, 0x3e, 0x31, 0x9d,
0x09, 0xe6, 0xac, 0x66, 0x3f, 0xdf, 0x2a, 0xdc, 0xd7, 0xf4, 0x3f, 0xd6, 0xa0, 0x6d, 0x60, 0x07,
0x9b, 0x01, 0xfe, 0x22, 0x45, 0xba, 0x0a, 0x73, 0xae, 0x67, 0xe1, 0xfd, 0x5d, 0x2a, 0xd2, 0xa2,
0xc1, 0xff, 0xf4, 0x9f, 0x69, 0xb0, 0xbc, 0x87, 0x43, 0xa2, 0xdb, 0x76, 0x10, 0xda, 0xfd, 0xc8,
0x78, 0xbf, 0x0b, 0x45, 0x1f, 0xbf, 0xe4, 0x94, 0xbd, 0x21, 0x53, 0x16, 0xb9, 0x62, 0xd5, 0x48,
0x83, 0x8c, 0x43, 0xaf, 0x40, 0xdd, 0x1a, 0x39, 0xdd, 0xfe, 0xb1, 0xe9, 0xba, 0xd8, 0x61, 0xd6,
0x51, 0x35, 0x6a, 0xd6, 0xc8, 0xd9, 0xe1, 0x20, 0x74, 0x0b, 0x20, 0xc0, 0xc3, 0x11, 0x76, 0xc3,
0xa9, 0xf7, 0x8c, 0x41, 0xd0, 0x06, 0x2c, 0x0e, 0x7c, 0x6f, 0xd4, 0x0d, 0x8e, 0x4d, 0xdf, 0xea,
0x3a, 0xd8, 0xb4, 0xb0, 0x4f, 0xa9, 0xaf, 0x18, 0x0b, 0xa4, 0xe1, 0x88, 0xc0, 0x1f, 0x53, 0x30,
0xba, 0x07, 0xe5, 0xa0, 0xef, 0x8d, 0x31, 0xd5, 0xb4, 0xe6, 0xf6, 0x4d, 0x95, 0x0e, 0xed, 0x9a,
0xa1, 0x79, 0x44, 0x3a, 0x19, 0xac, 0xaf, 0xfe, 0x3f, 0xdc, 0xd4, 0xbe, 0xe4, 0x9e, 0x2b, 0x66,
0x8e, 0xe5, 0x4f, 0xc7, 0x1c, 0xe7, 0x72, 0x99, 0xe3, 0xfc, 0xf9, 0xe6, 0x98, 0xe2, 0xda, 0x45,
0xcc, 0xb1, 0x32, 0xd3, 0x1c, 0xab, 0x9f, 0x8d, 0x39, 0xfe, 0xfd, 0xd4, 0x1c, 0xbf, 0xec, 0x62,
0x9f, 0x9a, 0x6c, 0x59, 0x32, 0xd9, 0xbf, 0xd4, 0xe0, 0x2b, 0x7b, 0x38, 0x8c, 0xc8, 0x27, 0x16,
0x88, 0xbf, 0xa4, 0x41, 0xf7, 0x13, 0x0d, 0x3a, 0x2a, 0x5a, 0xaf, 0x12, 0x78, 0x3f, 0x80, 0xd5,
0x08, 0x47, 0xd7, 0xc2, 0x41, 0xdf, 0xb7, 0xc7, 0x54, 0x8c, 0xd4, 0xc9, 0xd4, 0xb6, 0xef, 0xa8,
0x34, 0x36, 0x49, 0xc1, 0x4a, 0x34, 0xc5, 0x6e, 0x6c, 0x06, 0xfd, 0xf7, 0x34, 0x58, 0x21, 0x4e,
0x8d, 0x7b, 0x21, 0x77, 0xe0, 0x5d, 0x9e, 0xaf, 0xb2, 0x7f, 0x2b, 0xa4, 0xfc, 0x5b, 0x0e, 0x1e,
0xd3, 0x2c, 0x36, 0x49, 0xcf, 0x55, 0x78, 0xf7, 0x0d, 0x28, 0xdb, 0xee, 0xc0, 0x13, 0xac, 0xba,
0xad, 0x62, 0x55, 0x1c, 0x19, 0xeb, 0xad, 0xbb, 0x8c, 0x8a, 0xa9, 0xc3, 0xbd, 0x82, 0xba, 0x25,
0x97, 0x5d, 0x50, 0x2c, 0xfb, 0x77, 0x35, 0xb8, 0x9e, 0x42, 0x78, 0x95, 0x75, 0x7f, 0x07, 0xe6,
0x68, 0x18, 0x11, 0x0b, 0x7f, 0x55, 0xb9, 0xf0, 0x18, 0xba, 0xc7, 0x76, 0x10, 0x1a, 0x7c, 0x8c,
0xee, 0x41, 0x2b, 0xd9, 0x46, 0x02, 0x1c, 0x0f, 0x6e, 0x5d, 0xd7, 0x1c, 0x31, 0x06, 0x54, 0x8d,
0x1a, 0x87, 0x1d, 0x98, 0x23, 0x8c, 0xbe, 0x02, 0x15, 0x62, 0xb2, 0x5d, 0xdb, 0x12, 0xe2, 0x9f,
0xa7, 0x26, 0x6c, 0x05, 0xe8, 0x26, 0x00, 0x6d, 0x32, 0x2d, 0xcb, 0x67, 0xb1, 0xaf, 0x6a, 0x54,
0x09, 0xe4, 0x01, 0x01, 0xe8, 0x7f, 0xa0, 0x41, 0x9d, 0xf8, 0xd8, 0x27, 0x38, 0x34, 0x89, 0x1c,
0xd0, 0x37, 0xa1, 0xea, 0x78, 0xa6, 0xd5, 0x0d, 0xcf, 0xc6, 0x0c, 0x55, 0x33, 0xc9, 0xeb, 0xa9,
0x63, 0x7e, 0x7a, 0x36, 0xc6, 0x46, 0xc5, 0xe1, 0x5f, 0x79, 0xf8, 0x9d, 0x32, 0xe5, 0xa2, 0xc2,
0x94, 0xff, 0xb1, 0x0c, 0xab, 0x3f, 0x30, 0xc3, 0xfe, 0xf1, 0xee, 0x48, 0x84, 0xf0, 0xcb, 0x2b,
0xc1, 0xd4, 0xb7, 0x15, 0xe2, 0xbe, 0xed, 0x53, 0xf3, 0x9d, 0x91, 0x9e, 0x97, 0x55, 0x7a, 0x4e,
0x8a, 0xc5, 0xcd, 0xe7, 0x5c, 0x54, 0x31, 0x3d, 0x8f, 0x45, 0xda, 0xb9, 0xcb, 0x44, 0xda, 0x1d,
0x68, 0xe0, 0x8f, 0xfa, 0xce, 0x84, 0xc8, 0x9c, 0x62, 0x67, 0x21, 0xf4, 0x96, 0x02, 0x7b, 0xdc,
0xc8, 0xea, 0x7c, 0xd0, 0x3e, 0xa7, 0x81, 0x89, 0x7a, 0x84, 0x43, 0x93, 0xc6, 0xc9, 0xda, 0xf6,
0x5a, 0x96, 0xa8, 0x85, 0x7e, 0x30, 0x71, 0x93, 0x3f, 0x74, 0x03, 0xaa, 0x3c, 0xae, 0xef, 0xef,
0xb6, 0xab, 0x94, 0x7d, 0x53, 0x00, 0x32, 0xa1, 0xc1, 0x3d, 0x10, 0xa7, 0x10, 0x28, 0x85, 0xdf,
0x51, 0x21, 0x50, 0x0b, 0x3b, 0x4e, 0x79, 0xc0, 0xa3, 0x7c, 0x10, 0x03, 0x91, 0x02, 0xd5, 0x1b,
0x0c, 0x1c, 0xdb, 0xc5, 0x07, 0x4c, 0xc2, 0x35, 0x4a, 0x84, 0x0c, 0x24, 0xb9, 0xc0, 0x09, 0xf6,
0x03, 0xdb, 0x73, 0xdb, 0x75, 0xda, 0x2e, 0x7e, 0x3b, 0x5d, 0x58, 0x4c, 0xa1, 0x50, 0x84, 0xf8,
0xaf, 0xc7, 0x43, 0xfc, 0x6c, 0x1e, 0xc7, 0x52, 0x80, 0xbf, 0xd0, 0x60, 0xe5, 0x99, 0x1b, 0x4c,
0x7a, 0xd1, 0xda, 0xbe, 0x18, 0x3d, 0x4e, 0x7a, 0x90, 0x52, 0xca, 0x83, 0xe8, 0x3f, 0x2e, 0xc3,
0x02, 0x5f, 0x05, 0x11, 0x37, 0x75, 0x05, 0x37, 0xa0, 0x1a, 0x05, 0x11, 0xce, 0x90, 0x29, 0x00,
0xad, 0x41, 0x2d, 0x66, 0x08, 0x9c, 0xaa, 0x38, 0x28, 0x17, 0x69, 0x22, 0x25, 0x28, 0xc5, 0x52,
0x82, 0x9b, 0x00, 0x03, 0x67, 0x12, 0x1c, 0x77, 0x43, 0x7b, 0x84, 0x79, 0x4a, 0x52, 0xa5, 0x90,
0xa7, 0xf6, 0x08, 0xa3, 0x07, 0x50, 0xef, 0xd9, 0xae, 0xe3, 0x0d, 0xbb, 0x63, 0x33, 0x3c, 0x0e,
0x78, 0x31, 0xa7, 0x12, 0x0b, 0x4d, 0xe0, 0x1e, 0xd2, 0xbe, 0x46, 0x8d, 0x8d, 0x39, 0x24, 0x43,
0xd0, 0x2d, 0xa8, 0xb9, 0x93, 0x51, 0xd7, 0x1b, 0x74, 0x7d, 0xef, 0x34, 0xa0, 0x25, 0x5b, 0xd1,
0xa8, 0xba, 0x93, 0xd1, 0xfb, 0x03, 0xc3, 0x3b, 0x25, 0x4e, 0xbc, 0x4a, 0xdc, 0x79, 0xe0, 0x78,
0x43, 0x56, 0xae, 0xcd, 0x9e, 0x7f, 0x3a, 0x80, 0x8c, 0xb6, 0xb0, 0x13, 0x9a, 0x74, 0x74, 0x35,
0xdf, 0xe8, 0x68, 0x00, 0x7a, 0x0d, 0x9a, 0x7d, 0x6f, 0x34, 0x36, 0x29, 0x87, 0x1e, 0xf9, 0xde,
0x88, 0x5a, 0x4e, 0xd1, 0x48, 0x40, 0xd1, 0x0e, 0xd4, 0x68, 0xfe, 0xcc, 0xcd, 0xab, 0x46, 0xf1,
0xe8, 0x2a, 0xf3, 0x8a, 0xe5, 0xb1, 0x44, 0x41, 0xc1, 0x16, 0x9f, 0x01, 0xd1, 0x0c, 0x61, 0xa5,
0x81, 0xfd, 0x31, 0xe6, 0x16, 0x52, 0xe3, 0xb0, 0x23, 0xfb, 0x63, 0x4c, 0x92, 0x7a, 0xdb, 0x0d,
0xb0, 0x1f, 0x8a, 0x12, 0xab, 0xdd, 0xa0, 0xea, 0xd3, 0x60, 0x50, 0xae, 0xd8, 0x68, 0x1f, 0x9a,
0x41, 0x68, 0xfa, 0x61, 0x77, 0xec, 0x05, 0x54, 0x01, 0xda, 0x4d, 0xaa, 0xdb, 0x7a, 0x46, 0x41,
0xf7, 0x24, 0x18, 0x1e, 0xf2, 0x9e, 0x46, 0x83, 0x8e, 0x14, 0xbf, 0xfa, 0x7f, 0x17, 0xa0, 0x29,
0xd3, 0x4c, 0x8c, 0x98, 0x25, 0xf8, 0x42, 0x11, 0xc5, 0x2f, 0x59, 0x01, 0x76, 0xcd, 0x9e, 0x83,
0x59, 0x35, 0x41, 0xf5, 0xb0, 0x62, 0xd4, 0x18, 0x8c, 0x4e, 0x40, 0xf4, 0x89, 0x71, 0x8a, 0x2a,
0x7f, 0x91, 0x52, 0x5f, 0xa5, 0x10, 0x1a, 0x3c, 0xdb, 0x30, 0x2f, 0x0a, 0x11, 0xa6, 0x85, 0xe2,
0x97, 0xb4, 0xf4, 0x26, 0x36, 0xc5, 0xca, 0xb4, 0x50, 0xfc, 0xa2, 0x5d, 0xa8, 0xb3, 0x29, 0xc7,
0xa6, 0x6f, 0x8e, 0x84, 0x0e, 0xbe, 0xa2, 0xb4, 0xe3, 0xf7, 0xf0, 0xd9, 0x73, 0xe2, 0x12, 0x0e,
0x4d, 0xdb, 0x37, 0x98, 0xcc, 0x0e, 0xe9, 0x28, 0xb4, 0x0e, 0x2d, 0x36, 0xcb, 0xc0, 0x76, 0x30,
0xd7, 0xe6, 0x79, 0x56, 0x8d, 0x50, 0xf8, 0x23, 0xdb, 0xc1, 0x4c, 0x61, 0xa3, 0x25, 0x50, 0x29,
0x55, 0x98, 0xbe, 0x52, 0x08, 0x95, 0xd1, 0x1d, 0x68, 0xb0, 0x66, 0xe1, 0xe9, 0x98, 0x3b, 0x66,
0x34, 0x3e, 0x67, 0x30, 0x9a, 0x24, 0x4c, 0x46, 0x4c, 0xe3, 0x81, 0x2d, 0xc7, 0x9d, 0x8c, 0x88,
0xbe, 0xeb, 0x7f, 0x58, 0x82, 0x25, 0x62, 0xf6, 0xdc, 0x03, 0x5c, 0x21, 0xdc, 0xde, 0x04, 0xb0,
0x82, 0xb0, 0x2b, 0xb9, 0xaa, 0xaa, 0x15, 0x84, 0xdc, 0x19, 0x7f, 0x53, 0x44, 0xcb, 0x62, 0x76,
0x02, 0x9d, 0x70, 0x43, 0xe9, 0x88, 0x79, 0xa9, 0xad, 0xa2, 0x3b, 0xd0, 0xe0, 0x65, 0x9f, 0x54,
0xea, 0xd4, 0x19, 0xf0, 0x40, 0xed, 0x4c, 0xe7, 0x94, 0x5b, 0x56, 0xb1, 0xa8, 0x39, 0x7f, 0xb5,
0xa8, 0x59, 0x49, 0x46, 0xcd, 0xf7, 0x60, 0x81, 0x7a, 0x82, 0xc8, 0x8a, 0x84, 0x03, 0xc9, 0x63,
0x46, 0x4d, 0x3a, 0x54, 0xfc, 0x06, 0xf1, 0xc8, 0x07, 0x52, 0xe4, 0x23, 0xcc, 0x70, 0x31, 0xb6,
0xba, 0xa1, 0x6f, 0xba, 0xc1, 0x00, 0xfb, 0x34, 0x72, 0x56, 0x8c, 0x3a, 0x01, 0x3e, 0xe5, 0x30,
0xfd, 0x5f, 0x0a, 0xb0, 0xca, 0x0b, 0xd8, 0xab, 0xeb, 0x45, 0x56, 0xf8, 0x12, 0xfe, 0xbf, 0x78,
0x4e, 0x49, 0x58, 0xca, 0x91, 0x9a, 0x95, 0x15, 0xa9, 0x99, 0x5c, 0x16, 0xcd, 0xa5, 0xca, 0xa2,
0x68, 0x2b, 0x67, 0x3e, 0xff, 0x56, 0x0e, 0x29, 0xf8, 0x69, 0xae, 0x4e, 0x65, 0x57, 0x35, 0xd8,
0x4f, 0x3e, 0x86, 0xfe, 0xa7, 0x06, 0x8d, 0x23, 0x6c, 0xfa, 0xfd, 0x63, 0xc1, 0xc7, 0x77, 0xe2,
0x5b, 0x5f, 0xaf, 0x66, 0x88, 0x58, 0x1a, 0xf2, 0xf3, 0xb3, 0xe7, 0xf5, 0x5f, 0x1a, 0xd4, 0x7f,
0x8d, 0x34, 0x89, 0xc5, 0xde, 0x8f, 0x2f, 0xf6, 0xb5, 0x8c, 0xc5, 0x1a, 0x38, 0xf4, 0x6d, 0x7c,
0x82, 0x7f, 0xee, 0x96, 0xfb, 0x4f, 0x1a, 0x74, 0x8e, 0xce, 0xdc, 0xbe, 0xc1, 0x6c, 0xf9, 0xea,
0x16, 0x73, 0x07, 0x1a, 0x27, 0x52, 0xd6, 0x56, 0xa0, 0x0a, 0x57, 0x3f, 0x89, 0x17, 0x7e, 0x06,
0xb4, 0xc4, 0x8e, 0x1b, 0x5f, 0xac, 0x70, 0xad, 0xaf, 0xab, 0xa8, 0x4e, 0x10, 0x47, 0x5d, 0xd3,
0x82, 0x2f, 0x03, 0xf5, 0xdf, 0xd7, 0x60, 0x49, 0xd1, 0x11, 0x5d, 0x87, 0x79, 0x5e, 0x64, 0xf2,
0x18, 0xcc, 0x6c, 0xd8, 0x22, 0xe2, 0x99, 0x6e, 0x93, 0xd8, 0x56, 0x3a, 0x15, 0xb4, 0xd0, 0x6d,
0xa8, 0x45, 0xd5, 0x80, 0x95, 0x92, 0x8f, 0x15, 0xa0, 0x0e, 0x54, 0xb8, 0x73, 0x12, 0x65, 0x56,
0xf4, 0xaf, 0xff, 0x9d, 0x06, 0xab, 0xef, 0x9a, 0xae, 0xe5, 0x0d, 0x06, 0x57, 0x67, 0xeb, 0x0e,
0x48, 0x45, 0x44, 0xde, 0xed, 0x09, 0xb9, 0xf2, 0x78, 0x03, 0x16, 0x7d, 0xe6, 0x19, 0x2d, 0x99,
0xef, 0x45, 0xa3, 0x25, 0x1a, 0x22, 0x7e, 0xfe, 0x55, 0x01, 0x10, 0x09, 0x06, 0x0f, 0x4d, 0xc7,
0x74, 0xfb, 0xf8, 0xf2, 0xa4, 0xdf, 0x85, 0xa6, 0x14, 0xc2, 0xa2, 0x13, 0xb9, 0x78, 0x0c, 0x0b,
0xd0, 0x7b, 0xd0, 0xec, 0x31, 0x54, 0x5d, 0x1f, 0x9b, 0x81, 0xe7, 0x52, 0xe7, 0xda, 0x54, 0xef,
0x44, 0x3c, 0xf5, 0xed, 0xe1, 0x10, 0xfb, 0x3b, 0x9e, 0x6b, 0xf1, 0x5c, 0xac, 0x27, 0xc8, 0x24,
0x43, 0x89, 0xe0, 0xa6, 0xf1, 0x5c, 0x88, 0x06, 0xa2, 0x80, 0x4e, 0x59, 0x11, 0x60, 0xd3, 0x99,
0x32, 0x62, 0xea, 0x8d, 0x5b, 0xac, 0xe1, 0x28, 0x7b, 0x23, 0x4a, 0x11, 0x5f, 0xf5, 0xbf, 0xd1,
0x00, 0x45, 0xf5, 0x12, 0xad, 0x0c, 0xa9, 0xf6, 0x25, 0x87, 0x6a, 0x8a, 0xa0, 0x70, 0x03, 0xaa,
0x96, 0x18, 0xc9, 0xcd, 0x65, 0x0a, 0xa0, 0x3e, 0x9a, 0x12, 0xdd, 0x25, 0xc1, 0x18, 0x5b, 0xa2,
0x1e, 0x61, 0xc0, 0xc7, 0x14, 0x26, 0x87, 0xe7, 0x52, 0x32, 0x3c, 0xc7, 0xf7, 0x59, 0xca, 0xd2,
0x3e, 0x8b, 0xfe, 0x49, 0x01, 0x5a, 0xd4, 0xdd, 0xed, 0x4c, 0x8b, 0xfd, 0x5c, 0x44, 0xdf, 0x81,
0x06, 0x3f, 0xb3, 0x96, 0x08, 0xaf, 0xbf, 0x8c, 0x4d, 0x86, 0xde, 0x86, 0x65, 0xd6, 0xc9, 0xc7,
0xc1, 0xc4, 0x99, 0xa6, 0xe2, 0x2c, 0x99, 0x45, 0x2f, 0x99, 0x9f, 0x25, 0x4d, 0x62, 0xc4, 0x33,
0x58, 0x1d, 0x3a, 0x5e, 0xcf, 0x74, 0xba, 0xb2, 0x78, 0x98, 0x0c, 0x73, 0x68, 0xfc, 0x32, 0x1b,
0x7e, 0x14, 0x97, 0x61, 0x80, 0xf6, 0x48, 0x59, 0x8f, 0x5f, 0x4c, 0xb3, 0xfc, 0x72, 0xee, 0x2c,
0xbf, 0x4e, 0x06, 0x46, 0x49, 0xfe, 0x9f, 0x68, 0xb0, 0x90, 0xd8, 0x2a, 0x4d, 0x96, 0x94, 0x5a,
0xba, 0xa4, 0xbc, 0x0f, 0x65, 0x52, 0x67, 0x31, 0x67, 0xd8, 0x54, 0x97, 0x3b, 0xf2, 0xac, 0x06,
0x1b, 0x80, 0xb6, 0x60, 0x49, 0x71, 0x40, 0xca, 0x75, 0x00, 0xa5, 0xcf, 0x47, 0xf5, 0x9f, 0x94,
0xa0, 0x16, 0xe3, 0xc7, 0x8c, 0x6a, 0x38, 0xcf, 0xde, 0x57, 0x62, 0x79, 0xc5, 0xf4, 0xf2, 0x32,
0xce, 0xce, 0x88, 0xde, 0x8d, 0xf0, 0x88, 0x25, 0xff, 0xbc, 0x12, 0x19, 0xe1, 0x11, 0x4d, 0xfd,
0xe3, 0x59, 0xfd, 0x9c, 0x94, 0xd5, 0x27, 0xea, 0x9e, 0xf9, 0x73, 0xea, 0x9e, 0x8a, 0x5c, 0xf7,
0x48, 0x76, 0x54, 0x4d, 0xda, 0x51, 0xde, 0x02, 0xf5, 0x6d, 0x58, 0xea, 0xfb, 0xd8, 0x0c, 0xb1,
0xf5, 0xf0, 0x6c, 0x27, 0x6a, 0xe2, 0x99, 0x91, 0xaa, 0x09, 0x3d, 0x9a, 0xee, 0x19, 0x31, 0x29,
0xd7, 0xa9, 0x94, 0xd5, 0x65, 0x15, 0x97, 0x0d, 0x13, 0xb2, 0x70, 0xcf, 0xf4, 0x2f, 0x59, 0x1a,
0x37, 0x2e, 0x55, 0x1a, 0xdf, 0x86, 0x9a, 0x08, 0xad, 0xc4, 0xdc, 0x9b, 0xcc, 0xf3, 0x09, 0x5f,
0x60, 0x05, 0x92, 0x33, 0x58, 0x90, 0x37, 0x5d, 0x93, 0x45, 0x69, 0x2b, 0x5d, 0x94, 0x5e, 0x87,
0x79, 0x3b, 0xe8, 0x0e, 0xcc, 0x17, 0xb8, 0xbd, 0x48, 0x5b, 0xe7, 0xec, 0xe0, 0x91, 0xf9, 0x02,
0xeb, 0xff, 0x5a, 0x84, 0xe6, 0xb4, 0x8a, 0xc9, 0xed, 0x46, 0xf2, 0x5c, 0x12, 0x38, 0x80, 0xd6,
0x34, 0x50, 0x53, 0x0e, 0x9f, 0x5b, 0x88, 0x25, 0x4f, 0x32, 0x16, 0xc6, 0x09, 0x7b, 0x95, 0xf6,
0x8a, 0x4b, 0x17, 0xda, 0x2b, 0xbe, 0xe2, 0x49, 0xe3, 0x3d, 0x58, 0x89, 0x02, 0xb0, 0xb4, 0x6c,
0x96, 0xe5, 0x2f, 0x8b, 0xc6, 0xc3, 0xf8, 0xf2, 0x33, 0x5c, 0xc0, 0x7c, 0x96, 0x0b, 0x48, 0xaa,
0x40, 0x25, 0xa5, 0x02, 0xe9, 0x03, 0xcf, 0xaa, 0xe2, 0xc0, 0x53, 0x7f, 0x06, 0x4b, 0x74, 0x1b,
0x30, 0xe8, 0xfb, 0x76, 0x0f, 0x47, 0x39, 0x6b, 0x1e, 0xb1, 0x76, 0xa0, 0x92, 0x48, 0x7b, 0xa3,
0x7f, 0xfd, 0x77, 0x34, 0x58, 0x4d, 0xcf, 0x4b, 0x35, 0x66, 0xea, 0x48, 0x34, 0xc9, 0x91, 0xfc,
0x3a, 0x2c, 0x4d, 0xa7, 0x97, 0x13, 0xea, 0x8c, 0x94, 0x51, 0x41, 0xb8, 0x81, 0xa6, 0x73, 0x08,
0x98, 0xfe, 0x13, 0x2d, 0xda, 0x4d, 0x25, 0xb0, 0x21, 0xdd, 0x63, 0x26, 0xc1, 0xcd, 0x73, 0x1d,
0xdb, 0x8d, 0xaa, 0x6e, 0xbe, 0x46, 0x06, 0xe4, 0x55, 0xf7, 0xbb, 0xb0, 0xc0, 0x3b, 0x45, 0x31,
0x2a, 0x67, 0x56, 0xd6, 0x64, 0xe3, 0xa2, 0xe8, 0x74, 0x17, 0x9a, 0x7c, 0xf3, 0x57, 0xe0, 0x2b,
0xaa, 0xb6, 0x84, 0x7f, 0x15, 0x5a, 0xa2, 0xdb, 0x45, 0xa3, 0xe2, 0x02, 0x1f, 0x18, 0x65, 0x77,
0x3f, 0xd6, 0xa0, 0x2d, 0xc7, 0xc8, 0xd8, 0xf2, 0x2f, 0x9e, 0xe3, 0x7d, 0x5b, 0x3e, 0x36, 0xbb,
0x7b, 0x0e, 0x3d, 0x53, 0x3c, 0xe2, 0xf0, 0xec, 0x80, 0x1e, 0x81, 0x92, 0xd2, 0x64, 0xd7, 0x0e,
0x42, 0xdf, 0xee, 0x4d, 0xae, 0x74, 0x05, 0x44, 0xff, 0xdb, 0x02, 0x7c, 0x55, 0x39, 0xe1, 0x55,
0x0e, 0xc8, 0xb2, 0x76, 0x02, 0x1e, 0x42, 0x25, 0x51, 0xc2, 0xbc, 0x76, 0xce, 0xe2, 0xf9, 0xa6,
0x16, 0xdb, 0x5c, 0x11, 0xe3, 0xc8, 0x1c, 0x91, 0x4e, 0x97, 0xb2, 0xe7, 0xe0, 0x4a, 0x2b, 0xcd,
0x21, 0xc6, 0xa1, 0x07, 0x50, 0x67, 0xe5, 0x61, 0xf7, 0xc4, 0xc6, 0xa7, 0xe2, 0x5c, 0xe7, 0x96,
0xd2, 0xaf, 0xd1, 0x7e, 0xcf, 0x6d, 0x7c, 0x6a, 0xd4, 0x9c, 0xe8, 0x3b, 0xd0, 0xff, 0xb7, 0x08,
0x30, 0x6d, 0x23, 0xb5, 0xe9, 0xd4, 0x60, 0xb8, 0x05, 0xc4, 0x20, 0x24, 0x10, 0xcb, 0xb9, 0x9f,
0xf8, 0x45, 0xc6, 0x74, 0x7b, 0xd6, 0xb2, 0x83, 0x90, 0xf3, 0x65, 0xeb, 0x7c, 0x5a, 0x04, 0x8b,
0x88, 0xc8, 0xd8, 0xb1, 0x89, 0xa8, 0xbd, 0x08, 0x04, 0xbd, 0x05, 0x68, 0xe8, 0x7b, 0xa7, 0xb6,
0x3b, 0x8c, 0x67, 0xec, 0x2c, 0xb1, 0x5f, 0xe4, 0x2d, 0xb1, 0x94, 0xfd, 0x87, 0xd0, 0x4a, 0x74,
0x17, 0x2c, 0xb9, 0x37, 0x83, 0x8c, 0x3d, 0x69, 0x2e, 0x7e, 0x82, 0xb3, 0x20, 0x63, 0x08, 0x3a,
0x5d, 0x68, 0x25, 0xe9, 0x55, 0x9c, 0xc1, 0x7c, 0x43, 0x3e, 0x83, 0x39, 0xcf, 0x4c, 0xc9, 0x34,
0xb1, 0x43, 0x98, 0xce, 0x00, 0x96, 0x55, 0x94, 0x28, 0x90, 0xdc, 0x97, 0x91, 0xe4, 0xc9, 0x69,
0x63, 0x87, 0x3d, 0xdf, 0x8b, 0xd2, 0x45, 0xca, 0xe6, 0x2c, 0x0f, 0x1c, 0xdb, 0x94, 0x2b, 0x48,
0x9b, 0x72, 0xfa, 0x1f, 0x69, 0x80, 0xd2, 0xda, 0x8d, 0x9a, 0x50, 0x88, 0x26, 0x29, 0xec, 0xef,
0x26, 0xb4, 0xa9, 0x90, 0xd2, 0xa6, 0x1b, 0x50, 0x8d, 0x22, 0x22, 0x77, 0x7f, 0x53, 0x40, 0x5c,
0xd7, 0x4a, 0xb2, 0xae, 0xc5, 0x08, 0x2b, 0xcb, 0x84, 0x1d, 0x03, 0x4a, 0x5b, 0x4c, 0x7c, 0x26,
0x4d, 0x9e, 0x69, 0x16, 0x85, 0x31, 0x4c, 0x45, 0x19, 0xd3, 0x7f, 0x14, 0x00, 0x4d, 0x63, 0x7e,
0x74, 0x10, 0x95, 0x27, 0x50, 0x6e, 0xc1, 0x52, 0x3a, 0x23, 0x10, 0x69, 0x10, 0x4a, 0xe5, 0x03,
0xaa, 0xd8, 0x5d, 0x54, 0x5d, 0x56, 0x7a, 0x27, 0xf2, 0x71, 0x2c, 0xc1, 0xb9, 0x95, 0x95, 0xe0,
0x24, 0xdc, 0xdc, 0x6f, 0x24, 0x2f, 0x39, 0x31, 0xa3, 0xb9, 0xaf, 0xf4, 0x47, 0xa9, 0x25, 0xcf,
0xba, 0xe1, 0x74, 0xf5, 0xeb, 0x49, 0xff, 0x56, 0x80, 0xc5, 0x88, 0x1b, 0x17, 0xe2, 0xf4, 0xec,
0x83, 0xbf, 0xcf, 0x98, 0xb5, 0x1f, 0xaa, 0x59, 0xfb, 0xcb, 0xe7, 0xe6, 0xb0, 0x9f, 0x1f, 0x67,
0x3f, 0x86, 0x79, 0xbe, 0x7d, 0x96, 0xb2, 0xdd, 0x3c, 0x55, 0xe2, 0x32, 0x94, 0x89, 0xab, 0x10,
0xfb, 0x49, 0xec, 0x87, 0xb1, 0x34, 0x7e, 0x6f, 0x8d, 0x9b, 0x6f, 0x43, 0xba, 0xb6, 0xa6, 0xff,
0xb5, 0x06, 0x70, 0x74, 0xe6, 0xf6, 0x1f, 0x30, 0x4b, 0x7b, 0x1b, 0x4a, 0xb3, 0xee, 0x71, 0x90,
0xde, 0x34, 0x37, 0xa7, 0x3d, 0x73, 0x08, 0x57, 0xaa, 0x83, 0x8b, 0xc9, 0x3a, 0x38, 0xab, 0x82,
0xcd, 0xf6, 0x2e, 0xff, 0xa0, 0xc1, 0x75, 0x42, 0xc4, 0xa7, 0x92, 0xb2, 0xe4, 0xe2, 0x70, 0xcc,
0x73, 0x15, 0x65, 0xcf, 0x75, 0x1f, 0xe6, 0x59, 0x29, 0x2a, 0xd2, 0x87, 0x5b, 0x59, 0x2c, 0x63,
0x0c, 0x36, 0x44, 0x77, 0xfd, 0x19, 0x34, 0x8c, 0xb8, 0x24, 0x10, 0x82, 0x52, 0xec, 0xb6, 0x0e,
0xfd, 0xa6, 0xc9, 0xbc, 0x39, 0x36, 0xfb, 0x76, 0x78, 0x46, 0x09, 0x2b, 0x1b, 0xd1, 0xbf, 0x5a,
0xec, 0xfa, 0x4f, 0x35, 0x58, 0x15, 0xe7, 0x07, 0x5c, 0xa9, 0x2e, 0xcf, 0x9b, 0x6d, 0x58, 0xe1,
0x1a, 0x94, 0x50, 0x25, 0x96, 0x75, 0x2c, 0x31, 0x98, 0xbc, 0x8c, 0x6d, 0x58, 0x09, 0x4d, 0x7f,
0x88, 0xc3, 0xe4, 0x18, 0xc6, 0xb9, 0x25, 0xd6, 0x28, 0x8f, 0xc9, 0x73, 0x7e, 0x73, 0x9b, 0x9d,
0xc0, 0x73, 0x87, 0xc0, 0x75, 0x02, 0xdc, 0xc9, 0x88, 0xaf, 0x52, 0x3f, 0x85, 0x1b, 0xec, 0xbe,
0x5c, 0x4f, 0xa6, 0xe8, 0x4a, 0xdb, 0xa7, 0xca, 0x75, 0x27, 0x4c, 0xe8, 0x4f, 0x35, 0xb8, 0x99,
0x81, 0xf9, 0x2a, 0x69, 0xef, 0x63, 0x25, 0xf6, 0x8c, 0x0c, 0x5f, 0xc2, 0x4b, 0xf3, 0xd3, 0x04,
0x91, 0x3f, 0x2b, 0xc1, 0x62, 0xaa, 0xd3, 0x85, 0x75, 0xee, 0x4d, 0x40, 0x44, 0x08, 0xd1, 0xf3,
0x0b, 0x5a, 0x34, 0x71, 0x5f, 0xdd, 0x72, 0x27, 0xa3, 0xe8, 0xe9, 0x05, 0xa9, 0x9b, 0x90, 0xcd,
0x7a, 0xb3, 0xcd, 0xd3, 0x48, 0x72, 0xa5, 0xec, 0xbb, 0xbb, 0x29, 0x02, 0x37, 0x0f, 0x26, 0x23,
0xb6, 0xcf, 0xca, 0xa5, 0xcc, 0xfc, 0x2f, 0x41, 0x25, 0x81, 0xd1, 0x00, 0x16, 0xe9, 0xfd, 0x8c,
0x49, 0x38, 0xf4, 0x48, 0xe6, 0x49, 0xe9, 0x62, 0x5e, 0xfe, 0x5b, 0xb9, 0x31, 0xbd, 0xcf, 0x47,
0x13, 0xe2, 0x79, 0xf2, 0xe9, 0xca, 0x50, 0x81, 0xc7, 0x76, 0xfb, 0xde, 0x28, 0xc2, 0x33, 0x77,
0x41, 0x3c, 0xfb, 0x7c, 0xb4, 0x8c, 0x27, 0x0e, 0xed, 0xec, 0xc0, 0x8a, 0x72, 0xe9, 0xb3, 0xe2,
0x4a, 0x39, 0x9e, 0xc8, 0x3e, 0x84, 0x65, 0xd5, 0xaa, 0x2e, 0x31, 0x47, 0x8a, 0xe2, 0x8b, 0xcc,
0xb1, 0xf1, 0x2b, 0x50, 0x8d, 0x4e, 0xbf, 0x50, 0x0d, 0xe6, 0x9f, 0xb9, 0xef, 0xb9, 0xde, 0xa9,
0xdb, 0xba, 0x86, 0xe6, 0xa1, 0xf8, 0xc0, 0x71, 0x5a, 0x1a, 0x6a, 0x40, 0xf5, 0x28, 0xf4, 0xb1,
0x49, 0x90, 0xb4, 0x0a, 0xa8, 0x09, 0xf0, 0xae, 0x1d, 0x84, 0x9e, 0x6f, 0xf7, 0x4d, 0xa7, 0x55,
0xdc, 0xf8, 0x18, 0x9a, 0xf2, 0xde, 0x12, 0xaa, 0x43, 0xe5, 0xc0, 0x0b, 0xbf, 0xff, 0x91, 0x1d,
0x84, 0xad, 0x6b, 0xa4, 0xff, 0x81, 0x17, 0x1e, 0xfa, 0x38, 0xc0, 0x6e, 0xd8, 0xd2, 0x10, 0xc0,
0xdc, 0xfb, 0xee, 0xae, 0x1d, 0xbc, 0x68, 0x15, 0xd0, 0x12, 0xdf, 0x36, 0x36, 0x9d, 0x7d, 0xbe,
0x61, 0xd3, 0x2a, 0x92, 0xe1, 0xd1, 0x5f, 0x09, 0xb5, 0xa0, 0x1e, 0x75, 0xd9, 0x3b, 0x7c, 0xd6,
0x2a, 0xa3, 0x2a, 0x94, 0xd9, 0xe7, 0xdc, 0x86, 0x05, 0xad, 0xe4, 0x99, 0x07, 0x99, 0x93, 0x2d,
0x22, 0x02, 0xb5, 0xae, 0x91, 0x95, 0xf1, 0x43, 0xa7, 0x96, 0x86, 0x16, 0xa0, 0x16, 0x3b, 0xc2,
0x69, 0x15, 0x08, 0x60, 0xcf, 0x1f, 0xf7, 0xb9, 0x37, 0x62, 0x24, 0x10, 0x76, 0xee, 0x12, 0x4e,
0x94, 0x36, 0x1e, 0x42, 0x45, 0x6c, 0x7a, 0x91, 0xae, 0x9c, 0x45, 0xe4, 0xb7, 0x75, 0x0d, 0x2d,
0x42, 0x43, 0xba, 0xd6, 0xde, 0xd2, 0x10, 0x82, 0xa6, 0xfc, 0xf0, 0xa4, 0x55, 0xd8, 0xd8, 0x06,
0x98, 0x26, 0x3f, 0x84, 0x9c, 0x7d, 0xf7, 0xc4, 0x74, 0x6c, 0x8b, 0xd1, 0x46, 0x9a, 0x08, 0x77,
0x29, 0x77, 0x98, 0x66, 0xb5, 0x0a, 0x1b, 0xb7, 0xa1, 0x22, 0x02, 0x3a, 0x81, 0x1b, 0x78, 0xe4,
0x9d, 0x60, 0x26, 0x99, 0x23, 0x1c, 0xb6, 0xb4, 0xed, 0x9f, 0x22, 0x00, 0x76, 0x4c, 0xe1, 0x79,
0xbe, 0x85, 0x1c, 0x40, 0x7b, 0x38, 0xdc, 0xf1, 0x46, 0x63, 0xcf, 0x15, 0xdb, 0xa7, 0x01, 0xda,
0x94, 0x75, 0x9f, 0xff, 0xa4, 0x3b, 0xf2, 0xd5, 0x77, 0x5e, 0x55, 0xf6, 0x4f, 0x74, 0xd6, 0xaf,
0xa1, 0x11, 0xc5, 0xf6, 0xd4, 0x1e, 0xe1, 0xa7, 0x76, 0xff, 0x45, 0x74, 0xb6, 0x91, 0xfd, 0xe4,
0x23, 0xd1, 0x55, 0xe0, 0xbb, 0xa3, 0xc4, 0x77, 0x14, 0xfa, 0xb6, 0x3b, 0x14, 0x5e, 0x5a, 0xbf,
0x86, 0x5e, 0x26, 0x1e, 0x9c, 0x08, 0x84, 0xdb, 0x79, 0xde, 0x98, 0x5c, 0x0e, 0xa5, 0x03, 0x0b,
0x89, 0x37, 0x78, 0x68, 0x43, 0x7d, 0x01, 0x58, 0xf5, 0x5e, 0xb0, 0xf3, 0x46, 0xae, 0xbe, 0x11,
0x36, 0x1b, 0x9a, 0xf2, 0x3b, 0x33, 0xf4, 0x4b, 0x59, 0x13, 0xa4, 0x9e, 0x20, 0x74, 0x36, 0xf2,
0x74, 0x8d, 0x50, 0x7d, 0xc0, 0x14, 0x74, 0x16, 0x2a, 0xe5, 0x73, 0x8d, 0xce, 0x79, 0x01, 0x52,
0xbf, 0x86, 0x7e, 0x44, 0x62, 0x59, 0xe2, 0xa1, 0x04, 0x7a, 0x53, 0xed, 0x7f, 0xd5, 0xef, 0x29,
0x66, 0x61, 0xf8, 0x20, 0x69, 0x5e, 0xd9, 0xd4, 0xa7, 0x9e, 0x4e, 0xe5, 0xa7, 0x3e, 0x36, 0xfd,
0x79, 0xd4, 0x5f, 0x18, 0xc3, 0x84, 0x9a, 0x4d, 0xf2, 0xb0, 0xec, 0x2d, 0x15, 0x8a, 0xcc, 0xd7,
0x1a, 0x9d, 0xcd, 0xbc, 0xdd, 0xe3, 0xda, 0x25, 0x3f, 0x08, 0x50, 0x33, 0x4d, 0xf9, 0x88, 0x41,
0xad, 0x5d, 0xea, 0xf7, 0x05, 0xfa, 0x35, 0xf4, 0x54, 0x72, 0xaf, 0xe8, 0xb5, 0x2c, 0xe1, 0xc8,
0x47, 0xe8, 0xb3, 0xf8, 0xf6, 0x9b, 0x80, 0x98, 0xed, 0xb8, 0x03, 0x7b, 0x38, 0xf1, 0x4d, 0xa6,
0x58, 0x59, 0xee, 0x26, 0xdd, 0x55, 0xa0, 0xf9, 0xda, 0x05, 0x46, 0x44, 0x4b, 0xea, 0x02, 0xec,
0xe1, 0xf0, 0x09, 0x0e, 0x7d, 0xbb, 0x1f, 0x24, 0x57, 0x34, 0xf5, 0xa8, 0xbc, 0x83, 0x40, 0xf5,
0xfa, 0xcc, 0x7e, 0x11, 0x82, 0x1e, 0xd4, 0xf6, 0x48, 0xe6, 0x4d, 0xb3, 0x89, 0x00, 0x65, 0x8e,
0x14, 0x3d, 0x04, 0x8a, 0xf5, 0xd9, 0x1d, 0xe3, 0xee, 0x2c, 0xf1, 0x38, 0x02, 0x65, 0x0a, 0x36,
0xfd, 0x64, 0x43, 0xed, 0xce, 0x32, 0x5e, 0x5b, 0xb0, 0x15, 0xed, 0x1c, 0xe3, 0xfe, 0x8b, 0x77,
0xb1, 0xe9, 0x84, 0xc7, 0x19, 0x2b, 0x8a, 0xf5, 0x38, 0x7f, 0x45, 0x52, 0xc7, 0x08, 0x07, 0x86,
0xa5, 0x1d, 0x7a, 0xf2, 0x28, 0x97, 0x2c, 0x5b, 0xea, 0x29, 0xd2, 0x3d, 0x73, 0xaa, 0x9e, 0x09,
0x8b, 0xbb, 0xbe, 0x37, 0x96, 0x91, 0xbc, 0xa5, 0x44, 0x92, 0xea, 0x97, 0x13, 0xc5, 0x0f, 0xa0,
0x2e, 0x2a, 0x43, 0x9a, 0xcb, 0xaa, 0xb9, 0x10, 0xef, 0x92, 0x73, 0xe2, 0x0f, 0x61, 0x21, 0x51,
0x72, 0xaa, 0x85, 0xae, 0xae, 0x4b, 0x67, 0xcd, 0x7e, 0x0a, 0x88, 0xbe, 0x78, 0x91, 0x5e, 0xdb,
0x65, 0x64, 0x1c, 0xe9, 0x8e, 0x02, 0xc9, 0x56, 0xee, 0xfe, 0x91, 0xe4, 0x7f, 0x0b, 0x56, 0x94,
0x65, 0x5d, 0xd2, 0x21, 0xf0, 0xfb, 0x5d, 0xe7, 0xd4, 0x9e, 0x49, 0x87, 0x70, 0xee, 0x08, 0x81,
0x7f, 0xfb, 0x93, 0x26, 0x54, 0x69, 0xe6, 0x45, 0xa5, 0xf5, 0x8b, 0xc4, 0xeb, 0xd3, 0x4d, 0xbc,
0x3e, 0x84, 0x85, 0xc4, 0x2b, 0x12, 0xb5, 0xd2, 0xaa, 0x9f, 0x9a, 0xe4, 0xc8, 0x1f, 0xe4, 0x77,
0x1c, 0xea, 0x50, 0xa8, 0x7c, 0xeb, 0x31, 0x6b, 0xee, 0xe7, 0xec, 0x01, 0x56, 0x74, 0x86, 0xf9,
0x7a, 0xe6, 0x2e, 0xa8, 0x7c, 0xf7, 0xed, 0x8b, 0xcf, 0x4b, 0x3e, 0xfb, 0xbc, 0xed, 0x43, 0x58,
0x48, 0xdc, 0x40, 0x56, 0x4b, 0x55, 0x7d, 0x4d, 0x79, 0xd6, 0xec, 0x9f, 0x63, 0x82, 0x63, 0xc1,
0x92, 0xe2, 0x72, 0x28, 0xda, 0xcc, 0xda, 0x5e, 0x54, 0xdf, 0x22, 0x9d, 0xbd, 0xa0, 0x86, 0x64,
0x4a, 0xc9, 0x98, 0x30, 0x25, 0x32, 0xf9, 0x94, 0xbe, 0xf3, 0x66, 0xbe, 0x77, 0xf7, 0xd1, 0x82,
0x8e, 0x60, 0x8e, 0xdd, 0x4b, 0x46, 0xaf, 0xa8, 0xcf, 0xe2, 0x62, 0x77, 0x96, 0x3b, 0xb3, 0x6e,
0x36, 0x07, 0x13, 0x27, 0x0c, 0xe8, 0xa4, 0x65, 0xea, 0x21, 0x91, 0xf2, 0x42, 0x7d, 0xfc, 0x32,
0x71, 0x67, 0xf6, 0xfd, 0x61, 0x31, 0xe9, 0xff, 0xef, 0x2c, 0xf0, 0x23, 0x58, 0x52, 0x9c, 0xd0,
0xa3, 0xac, 0x6c, 0x3f, 0xe3, 0x6e, 0x40, 0x67, 0x2b, 0x77, 0xff, 0x08, 0xf3, 0x0f, 0xa1, 0x95,
0xdc, 0xb6, 0x47, 0x6f, 0x64, 0xe9, 0xb3, 0x0a, 0xe7, 0xf9, 0xca, 0xfc, 0xf0, 0xeb, 0x1f, 0x6c,
0x0f, 0xed, 0xf0, 0x78, 0xd2, 0x23, 0x2d, 0x5b, 0xac, 0xeb, 0x5b, 0xb6, 0xc7, 0xbf, 0xb6, 0x04,
0xff, 0xb7, 0xe8, 0xe8, 0x2d, 0x8a, 0x6a, 0xdc, 0xeb, 0xcd, 0xd1, 0xdf, 0x7b, 0xff, 0x17, 0x00,
0x00, 0xff, 0xff, 0xc2, 0xde, 0x60, 0xae, 0x08, 0x48, 0x00, 0x00,
// 4161 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xec, 0x3b, 0x49, 0x8c, 0x1c, 0x59,
0x56, 0x8e, 0x5c, 0xaa, 0x32, 0x5f, 0x2e, 0x95, 0xf5, 0xcb, 0x65, 0xe7, 0xe4, 0x78, 0xa9, 0x09,
0xb7, 0xbb, 0x0b, 0x77, 0x77, 0x55, 0x4f, 0x79, 0xa6, 0xf1, 0x6c, 0x1a, 0xec, 0xaa, 0xb1, 0xbb,
0x68, 0xb7, 0xdb, 0x44, 0xd9, 0x1e, 0xd4, 0x6a, 0x26, 0x27, 0x32, 0xe3, 0x67, 0x56, 0xc8, 0xb1,
0xa4, 0x23, 0x22, 0xcb, 0x5d, 0x8d, 0xc4, 0x69, 0x2e, 0x20, 0x06, 0x89, 0x0b, 0x27, 0xc4, 0x01,
0x81, 0x34, 0x48, 0x8c, 0xc4, 0x01, 0x6e, 0x1c, 0x90, 0x90, 0xe0, 0x04, 0xe2, 0xc6, 0x91, 0x2b,
0x12, 0x20, 0xc4, 0x61, 0x34, 0x9a, 0x1b, 0xfa, 0x5b, 0x44, 0xfc, 0x88, 0x1f, 0x95, 0x51, 0x55,
0xbd, 0x0d, 0xe2, 0x16, 0xf1, 0xfe, 0xf2, 0xde, 0x7f, 0xfb, 0xfb, 0x0b, 0xac, 0xbe, 0x98, 0xe3,
0xe0, 0x78, 0x38, 0xf6, 0xfd, 0xc0, 0xda, 0x9a, 0x05, 0x7e, 0xe4, 0x23, 0xe4, 0xda, 0xce, 0xd1,
0x3c, 0x64, 0x7f, 0x5b, 0xb4, 0x7d, 0xd0, 0x1e, 0xfb, 0xae, 0xeb, 0x7b, 0x0c, 0x36, 0x68, 0xa7,
0x7b, 0x0c, 0xba, 0xb6, 0x17, 0xe1, 0xc0, 0x33, 0x1d, 0xd1, 0x1a, 0x8e, 0x0f, 0xb1, 0x6b, 0xf2,
0xbf, 0xa6, 0x1b, 0x4e, 0xf9, 0x67, 0xcf, 0x32, 0x23, 0x33, 0x8d, 0x4a, 0xff, 0x91, 0x06, 0x97,
0x0e, 0x0e, 0xfd, 0x97, 0xbb, 0xbe, 0xe3, 0xe0, 0x71, 0x64, 0xfb, 0x5e, 0x68, 0xe0, 0x17, 0x73,
0x1c, 0x46, 0xe8, 0x2d, 0xa8, 0x8d, 0xcc, 0x10, 0xf7, 0xb5, 0x0d, 0x6d, 0xb3, 0xb5, 0x73, 0x65,
0x4b, 0x22, 0x8a, 0x53, 0xf3, 0x5e, 0x38, 0xbd, 0x67, 0x86, 0xd8, 0xa0, 0x3d, 0x11, 0x82, 0x9a,
0x35, 0xda, 0xdf, 0xeb, 0x57, 0x36, 0xb4, 0xcd, 0xaa, 0x41, 0xbf, 0xd1, 0x2b, 0xd0, 0x19, 0xc7,
0x73, 0xef, 0xef, 0x85, 0xfd, 0xea, 0x46, 0x75, 0xb3, 0x6a, 0xc8, 0x40, 0xfd, 0xdf, 0x34, 0xb8,
0x9c, 0x23, 0x23, 0x9c, 0xf9, 0x5e, 0x88, 0xd1, 0x6d, 0x58, 0x0a, 0x23, 0x33, 0x9a, 0x87, 0x9c,
0x92, 0x2f, 0x2b, 0x29, 0x39, 0xa0, 0x5d, 0x0c, 0xde, 0x35, 0x8f, 0xb6, 0xa2, 0x40, 0x8b, 0xbe,
0x0a, 0x17, 0x6d, 0xef, 0x3d, 0xec, 0xfa, 0xc1, 0xf1, 0x70, 0x86, 0x83, 0x31, 0xf6, 0x22, 0x73,
0x8a, 0x05, 0x8d, 0x6b, 0xa2, 0xed, 0x71, 0xd2, 0x84, 0xde, 0x86, 0xcb, 0x4c, 0x60, 0x21, 0x0e,
0x8e, 0xec, 0x31, 0x1e, 0x9a, 0x47, 0xa6, 0xed, 0x98, 0x23, 0x07, 0xf7, 0x6b, 0x1b, 0xd5, 0xcd,
0x86, 0xb1, 0x4e, 0x9b, 0x0f, 0x58, 0xeb, 0x5d, 0xd1, 0xa8, 0xff, 0xb9, 0x06, 0xeb, 0x64, 0x85,
0x8f, 0xcd, 0x20, 0xb2, 0x3f, 0x05, 0x3e, 0xeb, 0xd0, 0x4e, 0xaf, 0xad, 0x5f, 0xa5, 0x6d, 0x12,
0x8c, 0xf4, 0x99, 0x09, 0xf4, 0x84, 0x27, 0x35, 0xba, 0x4c, 0x09, 0xa6, 0xff, 0x19, 0x57, 0x88,
0x34, 0x9d, 0xe7, 0x11, 0x44, 0x16, 0x67, 0x25, 0x8f, 0xf3, 0x0c, 0x62, 0xd0, 0xff, 0xa9, 0x0a,
0xeb, 0x0f, 0x7d, 0xd3, 0x4a, 0x14, 0xe6, 0xb3, 0x67, 0xe7, 0x77, 0x60, 0x89, 0x19, 0x5a, 0xbf,
0x46, 0x71, 0xdd, 0x94, 0x71, 0x71, 0x23, 0x4c, 0x28, 0x3c, 0xa0, 0x00, 0x83, 0x0f, 0x42, 0x37,
0xa1, 0x1b, 0xe0, 0x99, 0x63, 0x8f, 0xcd, 0xa1, 0x37, 0x77, 0x47, 0x38, 0xe8, 0xd7, 0x37, 0xb4,
0xcd, 0xba, 0xd1, 0xe1, 0xd0, 0x47, 0x14, 0x88, 0x7e, 0x08, 0x9d, 0x89, 0x8d, 0x1d, 0x6b, 0x68,
0x7b, 0x16, 0xfe, 0x68, 0x7f, 0xaf, 0xbf, 0xb4, 0x51, 0xdd, 0x6c, 0xed, 0x7c, 0x6b, 0x2b, 0xef,
0x24, 0xb6, 0x94, 0x1c, 0xd9, 0xba, 0x4f, 0x86, 0xef, 0xb3, 0xd1, 0xdf, 0xf3, 0xa2, 0xe0, 0xd8,
0x68, 0x4f, 0x52, 0x20, 0xd4, 0x87, 0xe5, 0x00, 0x4f, 0x02, 0x1c, 0x1e, 0xf6, 0x97, 0x37, 0xb4,
0xcd, 0x86, 0x21, 0x7e, 0xd1, 0x6b, 0xb0, 0x12, 0xe0, 0xd0, 0x9f, 0x07, 0x63, 0x3c, 0x9c, 0x06,
0xfe, 0x7c, 0x16, 0xf6, 0x1b, 0x1b, 0xd5, 0xcd, 0xa6, 0xd1, 0x15, 0xe0, 0x07, 0x14, 0x3a, 0xf8,
0x2e, 0xac, 0xe6, 0xb0, 0xa0, 0x1e, 0x54, 0x9f, 0xe3, 0x63, 0x2a, 0x88, 0xaa, 0x41, 0x3e, 0xd1,
0x45, 0xa8, 0x1f, 0x99, 0xce, 0x1c, 0x73, 0x56, 0xb3, 0x9f, 0x6f, 0x56, 0xee, 0x68, 0xfa, 0x1f,
0x6b, 0xd0, 0x37, 0xb0, 0x83, 0xcd, 0x10, 0x7f, 0x9e, 0x22, 0xbd, 0x04, 0x4b, 0x9e, 0x6f, 0xe1,
0xfd, 0x3d, 0x2a, 0xd2, 0xaa, 0xc1, 0xff, 0xf4, 0x5f, 0x68, 0x70, 0xf1, 0x01, 0x8e, 0x88, 0x6e,
0xdb, 0x61, 0x64, 0x8f, 0x63, 0xe3, 0xfd, 0x0e, 0x54, 0x03, 0xfc, 0x82, 0x53, 0xf6, 0xba, 0x4c,
0x59, 0xec, 0x95, 0x55, 0x23, 0x0d, 0x32, 0x0e, 0x7d, 0x05, 0xda, 0x96, 0xeb, 0x0c, 0xc7, 0x87,
0xa6, 0xe7, 0x61, 0x87, 0x59, 0x47, 0xd3, 0x68, 0x59, 0xae, 0xb3, 0xcb, 0x41, 0xe8, 0x1a, 0x40,
0x88, 0xa7, 0x2e, 0xf6, 0xa2, 0xc4, 0x7b, 0xa6, 0x20, 0xe8, 0x16, 0xac, 0x4e, 0x02, 0xdf, 0x1d,
0x86, 0x87, 0x66, 0x60, 0x0d, 0x1d, 0x6c, 0x5a, 0x38, 0xa0, 0xd4, 0x37, 0x8c, 0x15, 0xd2, 0x70,
0x40, 0xe0, 0x0f, 0x29, 0x18, 0xdd, 0x86, 0x7a, 0x38, 0xf6, 0x67, 0x98, 0x6a, 0x5a, 0x77, 0xe7,
0xaa, 0x4a, 0x87, 0xf6, 0xcc, 0xc8, 0x3c, 0x20, 0x9d, 0x0c, 0xd6, 0x57, 0xff, 0x1f, 0x6e, 0x6a,
0x5f, 0x70, 0xcf, 0x95, 0x32, 0xc7, 0xfa, 0x27, 0x63, 0x8e, 0x4b, 0xa5, 0xcc, 0x71, 0xf9, 0x64,
0x73, 0xcc, 0x71, 0xed, 0x34, 0xe6, 0xd8, 0x58, 0x68, 0x8e, 0xcd, 0x4f, 0xc7, 0x1c, 0xff, 0x2e,
0x31, 0xc7, 0x2f, 0xba, 0xd8, 0x13, 0x93, 0xad, 0x4b, 0x26, 0xfb, 0x17, 0x1a, 0x7c, 0xe9, 0x01,
0x8e, 0x62, 0xf2, 0x89, 0x05, 0xe2, 0x2f, 0x68, 0xd0, 0xfd, 0xa9, 0x06, 0x03, 0x15, 0xad, 0xe7,
0x09, 0xbc, 0x1f, 0xc0, 0xa5, 0x18, 0xc7, 0xd0, 0xc2, 0xe1, 0x38, 0xb0, 0x67, 0x54, 0x8c, 0xd4,
0xc9, 0xb4, 0x76, 0x6e, 0xa8, 0x34, 0x36, 0x4b, 0xc1, 0x7a, 0x3c, 0xc5, 0x5e, 0x6a, 0x06, 0xfd,
0xc7, 0x1a, 0xac, 0x13, 0xa7, 0xc6, 0xbd, 0x90, 0x37, 0xf1, 0xcf, 0xce, 0x57, 0xd9, 0xbf, 0x55,
0x72, 0xfe, 0xad, 0x04, 0x8f, 0x69, 0x16, 0x9b, 0xa5, 0xe7, 0x3c, 0xbc, 0xfb, 0x3a, 0xd4, 0x6d,
0x6f, 0xe2, 0x0b, 0x56, 0x5d, 0x57, 0xb1, 0x2a, 0x8d, 0x8c, 0xf5, 0xd6, 0x3d, 0x46, 0x45, 0xe2,
0x70, 0xcf, 0xa1, 0x6e, 0xd9, 0x65, 0x57, 0x14, 0xcb, 0xfe, 0x7d, 0x0d, 0x2e, 0xe7, 0x10, 0x9e,
0x67, 0xdd, 0xdf, 0x86, 0x25, 0x1a, 0x46, 0xc4, 0xc2, 0x5f, 0x51, 0x2e, 0x3c, 0x85, 0xee, 0xa1,
0x1d, 0x46, 0x06, 0x1f, 0xa3, 0xfb, 0xd0, 0xcb, 0xb6, 0x91, 0x00, 0xc7, 0x83, 0xdb, 0xd0, 0x33,
0x5d, 0xc6, 0x80, 0xa6, 0xd1, 0xe2, 0xb0, 0x47, 0xa6, 0x8b, 0xd1, 0x97, 0xa0, 0x41, 0x4c, 0x76,
0x68, 0x5b, 0x42, 0xfc, 0xcb, 0xd4, 0x84, 0xad, 0x10, 0x5d, 0x05, 0xa0, 0x4d, 0xa6, 0x65, 0x05,
0x2c, 0xf6, 0x35, 0x8d, 0x26, 0x81, 0xdc, 0x25, 0x00, 0xfd, 0x0f, 0x35, 0x68, 0x13, 0x1f, 0xfb,
0x1e, 0x8e, 0x4c, 0x22, 0x07, 0xf4, 0x0d, 0x68, 0x3a, 0xbe, 0x69, 0x0d, 0xa3, 0xe3, 0x19, 0x43,
0xd5, 0xcd, 0xf2, 0x3a, 0x71, 0xcc, 0x4f, 0x8e, 0x67, 0xd8, 0x68, 0x38, 0xfc, 0xab, 0x0c, 0xbf,
0x73, 0xa6, 0x5c, 0x55, 0x98, 0xf2, 0x3f, 0xd4, 0xe1, 0xd2, 0xf7, 0xcd, 0x68, 0x7c, 0xb8, 0xe7,
0x8a, 0x10, 0x7e, 0x76, 0x25, 0x48, 0x7c, 0x5b, 0x25, 0xed, 0xdb, 0x3e, 0x31, 0xdf, 0x19, 0xeb,
0x79, 0x5d, 0xa5, 0xe7, 0xa4, 0x58, 0xdc, 0x7a, 0xc6, 0x45, 0x95, 0xd2, 0xf3, 0x54, 0xa4, 0x5d,
0x3a, 0x4b, 0xa4, 0xdd, 0x85, 0x0e, 0xfe, 0x68, 0xec, 0xcc, 0x89, 0xcc, 0x29, 0x76, 0x16, 0x42,
0xaf, 0x29, 0xb0, 0xa7, 0x8d, 0xac, 0xcd, 0x07, 0xed, 0x73, 0x1a, 0x98, 0xa8, 0x5d, 0x1c, 0x99,
0x34, 0x4e, 0xb6, 0x76, 0x36, 0x8a, 0x44, 0x2d, 0xf4, 0x83, 0x89, 0x9b, 0xfc, 0xa1, 0x2b, 0xd0,
0xe4, 0x71, 0x7d, 0x7f, 0xaf, 0xdf, 0xa4, 0xec, 0x4b, 0x00, 0xc8, 0x84, 0x0e, 0xf7, 0x40, 0x9c,
0x42, 0xa0, 0x14, 0x7e, 0x5b, 0x85, 0x40, 0x2d, 0xec, 0x34, 0xe5, 0x21, 0x8f, 0xf2, 0x61, 0x0a,
0x44, 0x0a, 0x54, 0x7f, 0x32, 0x71, 0x6c, 0x0f, 0x3f, 0x62, 0x12, 0x6e, 0x51, 0x22, 0x64, 0x20,
0xc9, 0x05, 0x8e, 0x70, 0x10, 0xda, 0xbe, 0xd7, 0x6f, 0xd3, 0x76, 0xf1, 0x3b, 0x18, 0xc2, 0x6a,
0x0e, 0x85, 0x22, 0xc4, 0x7f, 0x2d, 0x1d, 0xe2, 0x17, 0xf3, 0x38, 0x95, 0x02, 0xfc, 0x44, 0x83,
0xf5, 0xa7, 0x5e, 0x38, 0x1f, 0xc5, 0x6b, 0xfb, 0x7c, 0xf4, 0x38, 0xeb, 0x41, 0x6a, 0x39, 0x0f,
0xa2, 0xff, 0xa8, 0x0e, 0x2b, 0x7c, 0x15, 0x44, 0xdc, 0xd4, 0x15, 0x5c, 0x81, 0x66, 0x1c, 0x44,
0x38, 0x43, 0x12, 0x00, 0xda, 0x80, 0x56, 0xca, 0x10, 0x38, 0x55, 0x69, 0x50, 0x29, 0xd2, 0x44,
0x4a, 0x50, 0x4b, 0xa5, 0x04, 0x57, 0x01, 0x26, 0xce, 0x3c, 0x3c, 0x1c, 0x46, 0xb6, 0x8b, 0x79,
0x4a, 0xd2, 0xa4, 0x90, 0x27, 0xb6, 0x8b, 0xd1, 0x5d, 0x68, 0x8f, 0x6c, 0xcf, 0xf1, 0xa7, 0xc3,
0x99, 0x19, 0x1d, 0x86, 0xbc, 0x98, 0x53, 0x89, 0x85, 0x26, 0x70, 0xf7, 0x68, 0x5f, 0xa3, 0xc5,
0xc6, 0x3c, 0x26, 0x43, 0xd0, 0x35, 0x68, 0x79, 0x73, 0x77, 0xe8, 0x4f, 0x86, 0x81, 0xff, 0x32,
0xa4, 0x25, 0x5b, 0xd5, 0x68, 0x7a, 0x73, 0xf7, 0xfd, 0x89, 0xe1, 0xbf, 0x24, 0x4e, 0xbc, 0x49,
0xdc, 0x79, 0xe8, 0xf8, 0x53, 0x56, 0xae, 0x2d, 0x9e, 0x3f, 0x19, 0x40, 0x46, 0x5b, 0xd8, 0x89,
0x4c, 0x3a, 0xba, 0x59, 0x6e, 0x74, 0x3c, 0x00, 0xbd, 0x0a, 0xdd, 0xb1, 0xef, 0xce, 0x4c, 0xca,
0xa1, 0xfb, 0x81, 0xef, 0x52, 0xcb, 0xa9, 0x1a, 0x19, 0x28, 0xda, 0x85, 0x16, 0xcd, 0x9f, 0xb9,
0x79, 0xb5, 0x28, 0x1e, 0x5d, 0x65, 0x5e, 0xa9, 0x3c, 0x96, 0x28, 0x28, 0xd8, 0xe2, 0x33, 0x24,
0x9a, 0x21, 0xac, 0x34, 0xb4, 0x3f, 0xc6, 0xdc, 0x42, 0x5a, 0x1c, 0x76, 0x60, 0x7f, 0x8c, 0x49,
0x52, 0x6f, 0x7b, 0x21, 0x0e, 0x22, 0x51, 0x62, 0xf5, 0x3b, 0x54, 0x7d, 0x3a, 0x0c, 0xca, 0x15,
0x1b, 0xed, 0x41, 0x37, 0x8c, 0xcc, 0x20, 0x1a, 0xce, 0xfc, 0x90, 0x2a, 0x40, 0xbf, 0x4b, 0x75,
0x3b, 0x53, 0x20, 0xb9, 0xe1, 0x94, 0x28, 0xf6, 0x63, 0xde, 0xc9, 0xe8, 0xd0, 0x41, 0xe2, 0x57,
0xff, 0xef, 0x0a, 0x74, 0x65, 0x72, 0x89, 0xfd, 0xb2, 0xdc, 0x5e, 0xe8, 0xa0, 0xf8, 0x25, 0xc4,
0x63, 0xcf, 0x1c, 0x39, 0x98, 0x15, 0x12, 0x54, 0x05, 0x1b, 0x46, 0x8b, 0xc1, 0xe8, 0x04, 0x44,
0x95, 0x18, 0x93, 0xa8, 0xde, 0x57, 0x29, 0xe1, 0x4d, 0x0a, 0xa1, 0x71, 0xb3, 0x0f, 0xcb, 0xa2,
0x06, 0x61, 0x0a, 0x28, 0x7e, 0x49, 0xcb, 0x68, 0x6e, 0x53, 0xac, 0x4c, 0x01, 0xc5, 0x2f, 0xda,
0x83, 0x36, 0x9b, 0x72, 0x66, 0x06, 0xa6, 0x2b, 0xd4, 0xef, 0x2b, 0x4a, 0x13, 0x7e, 0x17, 0x1f,
0x3f, 0x23, 0xde, 0xe0, 0xb1, 0x69, 0x07, 0x06, 0x13, 0xd7, 0x63, 0x3a, 0x0a, 0x6d, 0x42, 0x8f,
0xcd, 0x32, 0xb1, 0x1d, 0xcc, 0x15, 0x79, 0x99, 0x15, 0x22, 0x14, 0x7e, 0xdf, 0x76, 0x30, 0xd3,
0xd5, 0x78, 0x09, 0x54, 0x40, 0x0d, 0xa6, 0xaa, 0x14, 0x42, 0xc5, 0x73, 0x03, 0x3a, 0xac, 0x59,
0x38, 0x39, 0xe6, 0x89, 0x19, 0x8d, 0xcf, 0x18, 0x8c, 0xe6, 0x07, 0x73, 0x97, 0x29, 0x3b, 0xb0,
0xe5, 0x78, 0x73, 0x97, 0xa8, 0xba, 0xfe, 0xe3, 0x1a, 0xac, 0x11, 0x8b, 0xe7, 0xc6, 0x7f, 0x8e,
0x48, 0x7b, 0x15, 0xc0, 0x0a, 0xa3, 0xa1, 0xe4, 0xa5, 0x9a, 0x56, 0x18, 0x71, 0x3f, 0xfc, 0x0d,
0x11, 0x28, 0xab, 0xc5, 0xb9, 0x73, 0xc6, 0x03, 0xe5, 0x83, 0xe5, 0x99, 0x76, 0x89, 0x6e, 0x40,
0x87, 0x57, 0x7c, 0x52, 0x95, 0xd3, 0x66, 0xc0, 0x47, 0x6a, 0x3f, 0xba, 0xa4, 0xdc, 0xad, 0x4a,
0x05, 0xcc, 0xe5, 0xf3, 0x05, 0xcc, 0x46, 0x36, 0x60, 0xde, 0x87, 0x15, 0xea, 0x04, 0x62, 0x03,
0x12, 0xbe, 0x63, 0x81, 0x05, 0x75, 0xe9, 0x28, 0xf1, 0x1b, 0xa6, 0xe3, 0x1d, 0x48, 0xf1, 0x8e,
0xf0, 0xc1, 0xc3, 0xd8, 0x1a, 0x46, 0x81, 0xe9, 0x85, 0x13, 0x1c, 0xd0, 0x78, 0xd9, 0x30, 0xda,
0x04, 0xf8, 0x84, 0xc3, 0xf4, 0x7f, 0xae, 0xc0, 0x25, 0x5e, 0xb6, 0x9e, 0x5f, 0x25, 0x8a, 0x82,
0x96, 0xf0, 0xfa, 0xd5, 0x13, 0x0a, 0xc1, 0x5a, 0x89, 0x84, 0xac, 0xae, 0x48, 0xc8, 0xe4, 0x62,
0x68, 0x29, 0x57, 0x0c, 0xc5, 0x1b, 0x38, 0xcb, 0xe5, 0x37, 0x70, 0x48, 0x99, 0x4f, 0x33, 0x74,
0x2a, 0xb6, 0xa6, 0xc1, 0x7e, 0xca, 0x31, 0xf4, 0x3f, 0x34, 0xe8, 0x1c, 0x60, 0x33, 0x18, 0x1f,
0x0a, 0x3e, 0xbe, 0x9d, 0xde, 0xf0, 0x7a, 0xa5, 0x60, 0xc3, 0x4b, 0x1a, 0xf2, 0xcb, 0xb3, 0xd3,
0xf5, 0x9f, 0x1a, 0xb4, 0x7f, 0x83, 0x34, 0x89, 0xc5, 0xde, 0x49, 0x2f, 0xf6, 0xd5, 0x82, 0xc5,
0x1a, 0x38, 0x0a, 0x6c, 0x7c, 0x84, 0x7f, 0xe9, 0x96, 0xfb, 0x8f, 0x1a, 0x0c, 0x0e, 0x8e, 0xbd,
0xb1, 0xc1, 0xcc, 0xf8, 0xfc, 0x16, 0x73, 0x03, 0x3a, 0x47, 0x52, 0xae, 0x56, 0xa1, 0x0a, 0xd7,
0x3e, 0x4a, 0x97, 0x7b, 0x06, 0xf4, 0xc4, 0x3e, 0x1b, 0x5f, 0xac, 0xf0, 0xaa, 0xaf, 0xa9, 0xa8,
0xce, 0x10, 0x47, 0xbd, 0xd2, 0x4a, 0x20, 0x03, 0xf5, 0x3f, 0xd0, 0x60, 0x4d, 0xd1, 0x11, 0x5d,
0x86, 0x65, 0x5e, 0x5a, 0xf2, 0xf0, 0xcb, 0x6c, 0xd8, 0x22, 0xe2, 0x49, 0x36, 0x47, 0x6c, 0x2b,
0x9f, 0x00, 0x5a, 0xe8, 0x3a, 0xb4, 0xe2, 0x1a, 0xc0, 0xca, 0xc9, 0xc7, 0x0a, 0xd1, 0x00, 0x1a,
0xdc, 0x39, 0x89, 0xe2, 0x2a, 0xfe, 0xd7, 0xff, 0x56, 0x83, 0x4b, 0xef, 0x98, 0x9e, 0xe5, 0x4f,
0x26, 0xe7, 0x67, 0xeb, 0x2e, 0x48, 0xa5, 0x43, 0xd9, 0x4d, 0x09, 0xb9, 0xde, 0x78, 0x1d, 0x56,
0x03, 0xe6, 0x19, 0x2d, 0x99, 0xef, 0x55, 0xa3, 0x27, 0x1a, 0x62, 0x7e, 0xfe, 0x65, 0x05, 0x10,
0x89, 0x03, 0xf7, 0x4c, 0xc7, 0xf4, 0xc6, 0xf8, 0xec, 0xa4, 0xdf, 0x84, 0xae, 0x14, 0xbd, 0xe2,
0x73, 0xb8, 0x74, 0xf8, 0x0a, 0xd1, 0xbb, 0xd0, 0x1d, 0x31, 0x54, 0xc3, 0x00, 0x9b, 0xa1, 0xef,
0x51, 0xe7, 0xda, 0x55, 0xef, 0x3f, 0x3c, 0x09, 0xec, 0xe9, 0x14, 0x07, 0xbb, 0xbe, 0x67, 0xf1,
0x34, 0x6c, 0x24, 0xc8, 0x24, 0x43, 0x89, 0xe0, 0x92, 0x50, 0x2e, 0x44, 0x03, 0x71, 0x2c, 0xa7,
0xac, 0x08, 0xb1, 0xe9, 0x24, 0x8c, 0x48, 0xbc, 0x71, 0x8f, 0x35, 0x1c, 0x14, 0x6f, 0x3f, 0x29,
0x42, 0xab, 0xfe, 0xd7, 0x1a, 0xa0, 0xb8, 0x4a, 0xa2, 0xf5, 0x20, 0xd5, 0xbe, 0xec, 0x50, 0x4d,
0x11, 0x14, 0xae, 0x40, 0xd3, 0x12, 0x23, 0xb9, 0xb9, 0x24, 0x00, 0xea, 0xa3, 0x29, 0xd1, 0x43,
0x12, 0x87, 0xb1, 0x25, 0xaa, 0x10, 0x06, 0x7c, 0x48, 0x61, 0x72, 0x64, 0xae, 0x65, 0x23, 0x73,
0x7a, 0x77, 0xa5, 0x2e, 0xed, 0xae, 0xe8, 0x3f, 0xa9, 0x40, 0x8f, 0xba, 0xbb, 0xdd, 0xa4, 0xc4,
0x2f, 0x45, 0xf4, 0x0d, 0xe8, 0xf0, 0x43, 0x6b, 0x89, 0xf0, 0xf6, 0x8b, 0xd4, 0x64, 0xe8, 0x2d,
0xb8, 0xc8, 0x3a, 0x05, 0x38, 0x9c, 0x3b, 0x49, 0x02, 0xce, 0xf2, 0x58, 0xf4, 0x82, 0xf9, 0x59,
0xd2, 0x24, 0x46, 0x3c, 0x85, 0x4b, 0x53, 0xc7, 0x1f, 0x99, 0xce, 0x50, 0x16, 0x0f, 0x93, 0x61,
0x09, 0x8d, 0xbf, 0xc8, 0x86, 0x1f, 0xa4, 0x65, 0x18, 0xa2, 0x7b, 0xa4, 0x98, 0xc7, 0xcf, 0x93,
0xdc, 0xbe, 0x5e, 0x26, 0xb7, 0x6f, 0x93, 0x31, 0x71, 0x6a, 0xff, 0x27, 0x1a, 0xac, 0x64, 0xf6,
0x46, 0xb3, 0x35, 0xa4, 0x96, 0xaf, 0x21, 0xef, 0x40, 0x9d, 0x14, 0x56, 0xcc, 0x0f, 0x76, 0xd5,
0xf5, 0x8d, 0x3c, 0xab, 0xc1, 0x06, 0xa0, 0x6d, 0x58, 0x53, 0x9c, 0x88, 0x72, 0xf1, 0xa3, 0xfc,
0x81, 0xa8, 0xfe, 0xb3, 0x1a, 0xb4, 0x52, 0xac, 0x58, 0x50, 0xfe, 0x96, 0xd9, 0xec, 0xca, 0x2c,
0xaf, 0x9a, 0x5f, 0x5e, 0xc1, 0x61, 0x19, 0x51, 0x39, 0x17, 0xbb, 0x2c, 0xe5, 0xe7, 0xf5, 0x87,
0x8b, 0x5d, 0x9a, 0xf0, 0xa7, 0x73, 0xf9, 0x25, 0x29, 0x97, 0xcf, 0x54, 0x3b, 0xcb, 0x27, 0x54,
0x3b, 0x0d, 0xb9, 0xda, 0x91, 0x4c, 0xa8, 0x99, 0x35, 0xa1, 0xb2, 0x15, 0xe9, 0x5b, 0xb0, 0x36,
0x0e, 0xb0, 0x19, 0x61, 0xeb, 0xde, 0xf1, 0x6e, 0xdc, 0xc4, 0x93, 0x22, 0x55, 0x13, 0xba, 0x9f,
0x6c, 0x12, 0x31, 0x29, 0xb7, 0xa9, 0x94, 0xd5, 0xc5, 0x14, 0x97, 0x0d, 0x13, 0xb2, 0xf0, 0xcc,
0xf4, 0x2f, 0x5b, 0x0b, 0x77, 0xce, 0x54, 0x0b, 0x5f, 0x87, 0x96, 0x88, 0xaa, 0xc4, 0xd2, 0xbb,
0xcc, 0xe9, 0x09, 0x37, 0x60, 0x85, 0x92, 0x1f, 0x58, 0x91, 0x77, 0x59, 0xb3, 0xa5, 0x68, 0x2f,
0x5f, 0x8a, 0x5e, 0x86, 0x65, 0x3b, 0x1c, 0x4e, 0xcc, 0xe7, 0xb8, 0xbf, 0x4a, 0x5b, 0x97, 0xec,
0xf0, 0xbe, 0xf9, 0x1c, 0xeb, 0xff, 0x52, 0x85, 0x6e, 0x52, 0xbb, 0x94, 0xf6, 0x20, 0x65, 0x6e,
0x05, 0x3c, 0x82, 0x5e, 0x12, 0xa3, 0x29, 0x87, 0x4f, 0x2c, 0xbf, 0xb2, 0x47, 0x17, 0x2b, 0xb3,
0x8c, 0xbd, 0x4a, 0x9b, 0xc3, 0xb5, 0x53, 0x6d, 0x0e, 0x9f, 0xf3, 0x68, 0xf1, 0x36, 0xac, 0xc7,
0xb1, 0x57, 0x5a, 0x36, 0x4b, 0xf0, 0x2f, 0x8a, 0xc6, 0xc7, 0xe9, 0xe5, 0x17, 0xb8, 0x80, 0xe5,
0x22, 0x17, 0x90, 0x55, 0x81, 0x46, 0x4e, 0x05, 0xf2, 0x27, 0x9c, 0x4d, 0xc5, 0x09, 0xa7, 0xfe,
0x14, 0xd6, 0xe8, 0xbe, 0x5f, 0x38, 0x0e, 0xec, 0x11, 0x8e, 0xd3, 0xd5, 0x32, 0x62, 0x1d, 0x40,
0x23, 0x93, 0xf1, 0xc6, 0xff, 0xfa, 0xef, 0x69, 0x70, 0x29, 0x3f, 0x2f, 0xd5, 0x98, 0xc4, 0x91,
0x68, 0x92, 0x23, 0xf9, 0x4d, 0x58, 0x4b, 0xa6, 0x97, 0x73, 0xe9, 0x82, 0x6c, 0x51, 0x41, 0xb8,
0x81, 0x92, 0x39, 0x04, 0x4c, 0xff, 0x99, 0x16, 0x6f, 0x9f, 0x12, 0xd8, 0x94, 0x6e, 0x2a, 0x93,
0xb8, 0xe6, 0x7b, 0x8e, 0xed, 0xc5, 0xb5, 0x36, 0x5f, 0x23, 0x03, 0xf2, 0x5a, 0xfb, 0x1d, 0x58,
0xe1, 0x9d, 0xe2, 0xf0, 0x54, 0x32, 0x21, 0xeb, 0xb2, 0x71, 0x71, 0x60, 0xba, 0x09, 0x5d, 0xbe,
0xdb, 0x2b, 0xf0, 0x55, 0x55, 0x7b, 0xc0, 0xbf, 0x0e, 0x3d, 0xd1, 0xed, 0xb4, 0x01, 0x71, 0x85,
0x0f, 0x8c, 0x13, 0xbb, 0xdf, 0xd5, 0xa0, 0x2f, 0x87, 0xc7, 0xd4, 0xf2, 0x4f, 0x9f, 0xde, 0x7d,
0x4b, 0x3e, 0x27, 0xbb, 0x79, 0x02, 0x3d, 0x09, 0x1e, 0x71, 0x5a, 0xf6, 0x88, 0x9e, 0x79, 0x92,
0xaa, 0x64, 0xcf, 0x0e, 0xa3, 0xc0, 0x1e, 0xcd, 0xcf, 0x75, 0xe7, 0x43, 0xff, 0x9b, 0x0a, 0x7c,
0x59, 0x39, 0xe1, 0x79, 0x4e, 0xc4, 0x8a, 0x36, 0x01, 0xee, 0x41, 0x23, 0x53, 0xbd, 0xbc, 0x7a,
0xc2, 0xe2, 0xf9, 0x56, 0x16, 0xdb, 0x52, 0x09, 0x93, 0xc4, 0x24, 0xb1, 0x96, 0x5a, 0xf1, 0x1c,
0x5c, 0x69, 0xa5, 0x39, 0xc4, 0x38, 0x74, 0x17, 0xda, 0xac, 0x32, 0x1c, 0x1e, 0xd9, 0xf8, 0xa5,
0x38, 0xc8, 0xb9, 0xa6, 0xf4, 0x6b, 0xb4, 0xdf, 0x33, 0x1b, 0xbf, 0x34, 0x5a, 0x4e, 0xfc, 0x1d,
0xea, 0xff, 0x55, 0x05, 0x48, 0xda, 0x48, 0x59, 0x9a, 0x18, 0x0c, 0xb7, 0x80, 0x14, 0x84, 0x04,
0x62, 0x39, 0xed, 0x13, 0xbf, 0xc8, 0x48, 0xf6, 0x63, 0x2d, 0x3b, 0x8c, 0x38, 0x5f, 0xb6, 0x4f,
0xa6, 0x45, 0xb0, 0x88, 0x88, 0x8c, 0x9d, 0x93, 0x88, 0xb2, 0x8b, 0x40, 0xd0, 0x9b, 0x80, 0xa6,
0x81, 0xff, 0xd2, 0xf6, 0xa6, 0xe9, 0x64, 0x9d, 0xe5, 0xf4, 0xab, 0xbc, 0x25, 0x95, 0xad, 0xff,
0x00, 0x7a, 0x99, 0xee, 0x82, 0x25, 0xb7, 0x17, 0x90, 0xf1, 0x40, 0x9a, 0x8b, 0x1f, 0xd9, 0xac,
0xc8, 0x18, 0xc2, 0xc1, 0x10, 0x7a, 0x59, 0x7a, 0x15, 0x87, 0x2e, 0x5f, 0x97, 0x0f, 0x5d, 0x4e,
0x32, 0x53, 0x32, 0x4d, 0xea, 0xd4, 0x65, 0x60, 0xc2, 0x45, 0x15, 0x25, 0x0a, 0x24, 0xb7, 0x65,
0x24, 0x0b, 0xd2, 0xd9, 0xd4, 0xc1, 0xce, 0x77, 0xe3, 0x4c, 0x91, 0x72, 0xb8, 0xc8, 0xf9, 0xa6,
0xb6, 0xe2, 0x2a, 0xd2, 0x56, 0x9c, 0xfe, 0x47, 0x1a, 0xa0, 0xbc, 0x62, 0xa3, 0x2e, 0x54, 0xe2,
0x49, 0x2a, 0xfb, 0x7b, 0x19, 0x45, 0xaa, 0xe4, 0x14, 0xe9, 0x0a, 0x34, 0xe3, 0x60, 0xc8, 0x3d,
0x5f, 0x02, 0x48, 0xab, 0x59, 0x4d, 0x56, 0xb3, 0x14, 0x61, 0x75, 0x99, 0xb0, 0x43, 0x40, 0x79,
0x63, 0x49, 0xcf, 0xa4, 0xc9, 0x33, 0x2d, 0xa2, 0x30, 0x85, 0xa9, 0x2a, 0x63, 0xfa, 0xf7, 0x0a,
0xa0, 0x24, 0xdc, 0xc7, 0x87, 0x4e, 0x65, 0x62, 0xe4, 0x36, 0xac, 0xe5, 0x93, 0x01, 0x91, 0x01,
0xa1, 0x5c, 0x2a, 0xa0, 0x0a, 0xdb, 0x55, 0xd5, 0xc5, 0xa4, 0xb7, 0x63, 0xf7, 0xc6, 0x72, 0x9b,
0x6b, 0x45, 0xb9, 0x4d, 0xc6, 0xc3, 0xfd, 0x56, 0xf6, 0x42, 0x13, 0xb3, 0x97, 0x3b, 0x4a, 0x57,
0x94, 0x5b, 0xf2, 0xa2, 0xdb, 0x4c, 0xe7, 0xbf, 0x8a, 0xf4, 0xaf, 0x15, 0x58, 0x8d, 0xb9, 0x71,
0x2a, 0x4e, 0x2f, 0x3e, 0xe4, 0xfb, 0x94, 0x59, 0xfb, 0xa1, 0x9a, 0xb5, 0xbf, 0x7a, 0x62, 0xfa,
0xfa, 0xd9, 0x71, 0xf6, 0x63, 0x58, 0xe6, 0x9b, 0x66, 0x39, 0xdb, 0x2d, 0x53, 0x20, 0x5e, 0x84,
0x3a, 0x71, 0x15, 0x62, 0x17, 0x89, 0xfd, 0x30, 0x96, 0xa6, 0xef, 0xa8, 0x71, 0xf3, 0xed, 0x48,
0x57, 0xd4, 0xf4, 0xbf, 0xd2, 0x00, 0x0e, 0x8e, 0xbd, 0xf1, 0x5d, 0x66, 0x69, 0x6f, 0x41, 0x6d,
0xd1, 0x9d, 0x0d, 0xd2, 0x9b, 0xa6, 0xe5, 0xb4, 0x67, 0x09, 0xe1, 0x4a, 0x25, 0x70, 0x35, 0x5b,
0x02, 0x17, 0x15, 0xaf, 0xc5, 0xde, 0xe5, 0xef, 0x35, 0xb8, 0x4c, 0x88, 0xf8, 0x44, 0xb2, 0x95,
0x52, 0x1c, 0x4e, 0x79, 0xae, 0xaa, 0xec, 0xb9, 0xee, 0xc0, 0x32, 0xab, 0x42, 0x45, 0xe6, 0x70,
0xad, 0x88, 0x65, 0x8c, 0xc1, 0x86, 0xe8, 0xae, 0x3f, 0x85, 0x8e, 0x91, 0x96, 0x04, 0x42, 0x50,
0x4b, 0xdd, 0xcc, 0xa1, 0xdf, 0x34, 0x8f, 0x37, 0x67, 0xe6, 0xd8, 0x8e, 0x8e, 0x29, 0x61, 0x75,
0x23, 0xfe, 0x57, 0x8b, 0x5d, 0xff, 0xb9, 0x06, 0x97, 0xc4, 0xa9, 0x01, 0x57, 0xaa, 0xb3, 0xf3,
0x66, 0x07, 0xd6, 0xb9, 0x06, 0x65, 0x54, 0x89, 0x25, 0x1c, 0x6b, 0x0c, 0x26, 0x2f, 0x63, 0x07,
0xd6, 0x23, 0x33, 0x98, 0xe2, 0x28, 0x3b, 0x86, 0x71, 0x6e, 0x8d, 0x35, 0xca, 0x63, 0xca, 0x9c,
0xda, 0x5c, 0x67, 0xa7, 0xed, 0xdc, 0x21, 0x70, 0x9d, 0x00, 0x6f, 0xee, 0xf2, 0x55, 0xea, 0x2f,
0xe1, 0x0a, 0xbb, 0x1b, 0x37, 0x92, 0x29, 0x3a, 0xd7, 0xa6, 0xa9, 0x72, 0xdd, 0x19, 0x13, 0xfa,
0x53, 0x0d, 0xae, 0x16, 0x60, 0x3e, 0x4f, 0xc6, 0xfb, 0x50, 0x89, 0xbd, 0x20, 0xb9, 0x97, 0xf0,
0xd2, 0xd4, 0x34, 0x43, 0xe4, 0x2f, 0x6a, 0xb0, 0x9a, 0xeb, 0x74, 0x6a, 0x9d, 0x7b, 0x03, 0x10,
0x11, 0x42, 0xfc, 0xd4, 0x82, 0xd6, 0x4b, 0xdc, 0x57, 0xf7, 0xbc, 0xb9, 0x1b, 0x3f, 0xb3, 0x20,
0x25, 0x13, 0xb2, 0x59, 0x6f, 0xb6, 0x65, 0x1a, 0x4b, 0xae, 0x56, 0x7c, 0x4f, 0x37, 0x47, 0xe0,
0xd6, 0xa3, 0xb9, 0xcb, 0x76, 0x57, 0xb9, 0x94, 0x99, 0xff, 0x25, 0xa8, 0x24, 0x30, 0x9a, 0xc0,
0x2a, 0xbd, 0x8b, 0x31, 0x8f, 0xa6, 0x3e, 0x49, 0x3a, 0x29, 0x5d, 0xcc, 0xcb, 0x7f, 0xb3, 0x34,
0xa6, 0xf7, 0xf9, 0x68, 0x42, 0x3c, 0xcf, 0x3b, 0x3d, 0x19, 0x2a, 0xf0, 0xd8, 0xde, 0xd8, 0x77,
0x63, 0x3c, 0x4b, 0xa7, 0xc4, 0xb3, 0xcf, 0x47, 0xcb, 0x78, 0xd2, 0xd0, 0xc1, 0x2e, 0xac, 0x2b,
0x97, 0xbe, 0x28, 0xae, 0xd4, 0xd3, 0x39, 0xec, 0x3d, 0xb8, 0xa8, 0x5a, 0xd5, 0x19, 0xe6, 0xc8,
0x51, 0x7c, 0x9a, 0x39, 0x6e, 0xfd, 0x1a, 0x34, 0xe3, 0x33, 0x2f, 0xd4, 0x82, 0xe5, 0xa7, 0xde,
0xbb, 0x9e, 0xff, 0xd2, 0xeb, 0x5d, 0x40, 0xcb, 0x50, 0xbd, 0xeb, 0x38, 0x3d, 0x0d, 0x75, 0xa0,
0x79, 0x10, 0x05, 0xd8, 0x24, 0x48, 0x7a, 0x15, 0xd4, 0x05, 0x78, 0xc7, 0x0e, 0x23, 0x3f, 0xb0,
0xc7, 0xa6, 0xd3, 0xab, 0xde, 0xfa, 0x18, 0xba, 0xf2, 0xb6, 0x12, 0x6a, 0x43, 0xe3, 0x91, 0x1f,
0x7d, 0xef, 0x23, 0x3b, 0x8c, 0x7a, 0x17, 0x48, 0xff, 0x47, 0x7e, 0xf4, 0x38, 0xc0, 0x21, 0xf6,
0xa2, 0x9e, 0x86, 0x00, 0x96, 0xde, 0xf7, 0xf6, 0xec, 0xf0, 0x79, 0xaf, 0x82, 0xd6, 0xf8, 0x8e,
0xb1, 0xe9, 0xec, 0xf3, 0xbd, 0x9a, 0x5e, 0x95, 0x0c, 0x8f, 0xff, 0x6a, 0xa8, 0x07, 0xed, 0xb8,
0xcb, 0x83, 0xc7, 0x4f, 0x7b, 0x75, 0xd4, 0x84, 0x3a, 0xfb, 0x5c, 0xba, 0x65, 0x41, 0x2f, 0x7b,
0xd2, 0x41, 0xe6, 0x64, 0x8b, 0x88, 0x41, 0xbd, 0x0b, 0x64, 0x65, 0xfc, 0xa8, 0xa9, 0xa7, 0xa1,
0x15, 0x68, 0xa5, 0x0e, 0x6e, 0x7a, 0x15, 0x02, 0x78, 0x10, 0xcc, 0xc6, 0xdc, 0x1b, 0x31, 0x12,
0x08, 0x3b, 0xf7, 0x08, 0x27, 0x6a, 0xb7, 0xee, 0x41, 0x43, 0xec, 0x77, 0x91, 0xae, 0x9c, 0x45,
0xe4, 0xb7, 0x77, 0x01, 0xad, 0x42, 0x47, 0xba, 0xc2, 0xde, 0xd3, 0x10, 0x82, 0xae, 0xfc, 0xc8,
0xa4, 0x57, 0xb9, 0xb5, 0x03, 0x90, 0x24, 0x3f, 0x84, 0x9c, 0x7d, 0xef, 0xc8, 0x74, 0x6c, 0x8b,
0xd1, 0x46, 0x9a, 0x08, 0x77, 0x29, 0x77, 0x98, 0x66, 0xf5, 0x2a, 0xb7, 0xae, 0x43, 0x43, 0x04,
0x74, 0x02, 0x37, 0xb0, 0xeb, 0x1f, 0x61, 0x26, 0x99, 0x03, 0x1c, 0xf5, 0xb4, 0x9d, 0x9f, 0x23,
0x00, 0x76, 0x38, 0xe1, 0xfb, 0x81, 0x85, 0x1c, 0x40, 0x0f, 0x70, 0xb4, 0xeb, 0xbb, 0x33, 0xdf,
0x13, 0x3b, 0xa7, 0x21, 0xda, 0xca, 0x14, 0x3d, 0xec, 0x27, 0xdf, 0x91, 0xaf, 0x7e, 0xf0, 0x8a,
0xb2, 0x7f, 0xa6, 0xb3, 0x7e, 0x01, 0xb9, 0x14, 0xdb, 0x13, 0xdb, 0xc5, 0x4f, 0xec, 0xf1, 0xf3,
0xf8, 0x44, 0xa3, 0xf8, 0x79, 0x47, 0xa6, 0xab, 0xc0, 0x77, 0x43, 0x89, 0xef, 0x20, 0x0a, 0x6c,
0x6f, 0x2a, 0xbc, 0xb4, 0x7e, 0x01, 0xbd, 0xc8, 0x3c, 0x2e, 0x11, 0x08, 0x77, 0xca, 0xbc, 0x27,
0x39, 0x1b, 0x4a, 0x07, 0x56, 0x32, 0xef, 0xed, 0xd0, 0x2d, 0xf5, 0x65, 0x5f, 0xd5, 0xdb, 0xc0,
0xc1, 0xeb, 0xa5, 0xfa, 0xc6, 0xd8, 0x6c, 0xe8, 0xca, 0x6f, 0xca, 0xd0, 0xaf, 0x14, 0x4d, 0x90,
0x7b, 0x6e, 0x30, 0xb8, 0x55, 0xa6, 0x6b, 0x8c, 0xea, 0x03, 0xa6, 0xa0, 0x8b, 0x50, 0x29, 0x9f,
0x66, 0x0c, 0x4e, 0x0a, 0x90, 0xfa, 0x05, 0xf4, 0x43, 0x12, 0xcb, 0x32, 0x8f, 0x22, 0xd0, 0x1b,
0x6a, 0xff, 0xab, 0x7e, 0x3b, 0xb1, 0x08, 0xc3, 0x07, 0x59, 0xf3, 0x2a, 0xa6, 0x3e, 0xf7, 0x4c,
0xaa, 0x3c, 0xf5, 0xa9, 0xe9, 0x4f, 0xa2, 0xfe, 0xd4, 0x18, 0xe6, 0xd4, 0x6c, 0xb2, 0xe7, 0x64,
0x6f, 0xaa, 0x50, 0x14, 0xbe, 0xcc, 0x18, 0x6c, 0x95, 0xed, 0x9e, 0xd6, 0x2e, 0xf9, 0xf2, 0xbf,
0x9a, 0x69, 0xca, 0x07, 0x0b, 0x6a, 0xed, 0x52, 0xbf, 0x25, 0xd0, 0x2f, 0xa0, 0x27, 0x92, 0x7b,
0x45, 0xaf, 0x16, 0x09, 0x47, 0x3e, 0x38, 0x5f, 0xc4, 0xb7, 0xdf, 0x06, 0xc4, 0x6c, 0xc7, 0x9b,
0xd8, 0xd3, 0x79, 0x60, 0x32, 0xc5, 0x2a, 0x72, 0x37, 0xf9, 0xae, 0x02, 0xcd, 0x57, 0x4f, 0x31,
0x22, 0x5e, 0xd2, 0x10, 0xe0, 0x01, 0x8e, 0xde, 0xc3, 0x51, 0x60, 0x8f, 0xc3, 0xec, 0x8a, 0x12,
0x8f, 0xca, 0x3b, 0x08, 0x54, 0xaf, 0x2d, 0xec, 0x17, 0x23, 0x18, 0x41, 0xeb, 0x01, 0xc9, 0xbc,
0x69, 0x36, 0x11, 0xa2, 0xc2, 0x91, 0xa2, 0x87, 0x40, 0xb1, 0xb9, 0xb8, 0x63, 0xda, 0x9d, 0x65,
0x1e, 0x42, 0xa0, 0x42, 0xc1, 0xe6, 0x9f, 0x67, 0xa8, 0xdd, 0x59, 0xc1, 0xcb, 0x0a, 0xb6, 0xa2,
0xdd, 0x43, 0x3c, 0x7e, 0xfe, 0x0e, 0x36, 0x9d, 0xe8, 0xb0, 0x60, 0x45, 0xa9, 0x1e, 0x27, 0xaf,
0x48, 0xea, 0x18, 0xe3, 0xc0, 0xb0, 0xb6, 0x4b, 0x0f, 0x1d, 0xe5, 0x92, 0x65, 0x5b, 0x3d, 0x45,
0xbe, 0x67, 0x49, 0xd5, 0x33, 0x61, 0x75, 0x2f, 0xf0, 0x67, 0x32, 0x92, 0x37, 0x95, 0x48, 0x72,
0xfd, 0x4a, 0xa2, 0xf8, 0x3e, 0xb4, 0x45, 0x65, 0x48, 0x73, 0x59, 0x35, 0x17, 0xd2, 0x5d, 0x4a,
0x4e, 0xfc, 0x21, 0xac, 0x64, 0x4a, 0x4e, 0xb5, 0xd0, 0xd5, 0x75, 0xe9, 0xa2, 0xd9, 0x5f, 0x02,
0xa2, 0xaf, 0x5b, 0xa4, 0x97, 0x75, 0x05, 0x19, 0x47, 0xbe, 0xa3, 0x40, 0xb2, 0x5d, 0xba, 0x7f,
0x2c, 0xf9, 0xdf, 0x81, 0x75, 0x65, 0x59, 0x97, 0x75, 0x08, 0xfc, 0x56, 0xd7, 0x09, 0xb5, 0x67,
0xd6, 0x21, 0x9c, 0x38, 0x42, 0xe0, 0xdf, 0xf9, 0x69, 0x17, 0x9a, 0x34, 0xf3, 0xa2, 0xd2, 0xfa,
0xff, 0xc4, 0xeb, 0x93, 0x4d, 0xbc, 0x3e, 0x84, 0x95, 0xcc, 0x8b, 0x11, 0xb5, 0xd2, 0xaa, 0x9f,
0x95, 0x94, 0xc8, 0x1f, 0xe4, 0x37, 0x1b, 0xea, 0x50, 0xa8, 0x7c, 0xd7, 0xb1, 0x68, 0xee, 0x67,
0xec, 0xb1, 0x55, 0x7c, 0x7c, 0xf9, 0x5a, 0xe1, 0x2e, 0xa8, 0x7c, 0xe3, 0xed, 0xf3, 0xcf, 0x4b,
0x3e, 0xfd, 0xbc, 0xed, 0x43, 0x58, 0xc9, 0xdc, 0x3b, 0x56, 0x4b, 0x55, 0x7d, 0x39, 0x79, 0xd1,
0xec, 0x9f, 0x61, 0x82, 0x63, 0xc1, 0x9a, 0xe2, 0x4a, 0x28, 0xda, 0x2a, 0xda, 0x5e, 0x54, 0xdf,
0x1d, 0x5d, 0xbc, 0xa0, 0x8e, 0x64, 0x4a, 0xd9, 0x98, 0x90, 0x10, 0x99, 0x7d, 0x36, 0x3f, 0x78,
0xa3, 0xdc, 0x1b, 0xfb, 0x78, 0x41, 0x07, 0xb0, 0xc4, 0x6e, 0x23, 0xa3, 0xaf, 0xa8, 0x8f, 0xe1,
0x52, 0x37, 0x95, 0x07, 0x8b, 0xee, 0x33, 0x87, 0x73, 0x27, 0x0a, 0xe9, 0xa4, 0x75, 0xea, 0x21,
0x91, 0xf2, 0x06, 0x7d, 0xfa, 0x0a, 0xf1, 0x60, 0xf1, 0xad, 0x61, 0x31, 0xe9, 0xff, 0xed, 0x2c,
0xf0, 0x23, 0x58, 0x53, 0x1c, 0xce, 0xa3, 0xa2, 0x6c, 0xbf, 0xe0, 0x5a, 0xc0, 0x60, 0xbb, 0x74,
0xff, 0x18, 0xf3, 0x0f, 0xa0, 0x97, 0xdd, 0xb6, 0x47, 0xaf, 0x17, 0xe9, 0xb3, 0x0a, 0xe7, 0xc9,
0xca, 0x7c, 0xef, 0x6b, 0x1f, 0xec, 0x4c, 0xed, 0xe8, 0x70, 0x3e, 0x22, 0x2d, 0xdb, 0xac, 0xeb,
0x9b, 0xb6, 0xcf, 0xbf, 0xb6, 0x05, 0xff, 0xb7, 0xe9, 0xe8, 0x6d, 0x8a, 0x6a, 0x36, 0x1a, 0x2d,
0xd1, 0xdf, 0xdb, 0xff, 0x1b, 0x00, 0x00, 0xff, 0xff, 0xbe, 0x39, 0xcb, 0x3c, 0xff, 0x47, 0x00,
0x00,
}
// Reference imports to suppress errors if they are not otherwise used.

3
internal/proto/rootcoordpb/root_coord.pb.go
View File

@ -6,8 +6,6 @@ package rootcoordpb
import (
context "context"
fmt "fmt"
math "math"
proto "github.com/golang/protobuf/proto"
commonpb "github.com/milvus-io/milvus-proto/go-api/commonpb"
milvuspb "github.com/milvus-io/milvus-proto/go-api/milvuspb"
@ -17,6 +15,7 @@ import (
grpc "google.golang.org/grpc"
codes "google.golang.org/grpc/codes"
status "google.golang.org/grpc/status"
math "math"
)
// Reference imports to suppress errors if they are not otherwise used.

3
internal/proto/segcorepb/segcore.pb.go
View File

@ -5,10 +5,9 @@ package segcorepb
import (
fmt "fmt"
math "math"
proto "github.com/golang/protobuf/proto"
schemapb "github.com/milvus-io/milvus-proto/go-api/schemapb"
math "math"
)
// Reference imports to suppress errors if they are not otherwise used.

8
internal/proxy/impl.go
View File

@ -24,7 +24,6 @@ import (
"sync"
"github.com/cockroachdb/errors"
"github.com/golang/protobuf/proto"
"go.opentelemetry.io/otel"
"go.uber.org/zap"
@ -32,6 +31,7 @@ import (
"github.com/milvus-io/milvus-proto/go-api/commonpb"
"github.com/milvus-io/milvus-proto/go-api/milvuspb"
"github.com/milvus-io/milvus-proto/go-api/msgpb"
"github.com/milvus-io/milvus-proto/go-api/schemapb"
"github.com/milvus-io/milvus/internal/common"
"github.com/milvus-io/milvus/internal/log"
@ -2042,7 +2042,7 @@ func (node *Proxy) Insert(ctx context.Context, request *milvuspb.InsertRequest)
BaseMsg: msgstream.BaseMsg{
HashValues: request.HashKeys,
},
InsertRequest: internalpb.InsertRequest{
InsertRequest: msgpb.InsertRequest{
Base: commonpbutil.NewMsgBase(
commonpbutil.WithMsgType(commonpb.MsgType_Insert),
commonpbutil.WithMsgID(0),
@ -2052,7 +2052,7 @@ func (node *Proxy) Insert(ctx context.Context, request *milvuspb.InsertRequest)
PartitionName: request.PartitionName,
FieldsData: request.FieldsData,
NumRows: uint64(request.NumRows),
Version: internalpb.InsertDataVersion_ColumnBased,
Version: msgpb.InsertDataVersion_ColumnBased,
// RowData: transfer column based request to this
},
},
@ -2171,7 +2171,7 @@ func (node *Proxy) Delete(ctx context.Context, request *milvuspb.DeleteRequest)
BaseMsg: msgstream.BaseMsg{
HashValues: request.HashKeys,
},
DeleteRequest: internalpb.DeleteRequest{
DeleteRequest: msgpb.DeleteRequest{
Base: commonpbutil.NewMsgBase(
commonpbutil.WithMsgType(commonpb.MsgType_Delete),
commonpbutil.WithMsgID(0),

9
internal/proxy/msg_pack.go
View File

@ -19,17 +19,18 @@ package proxy
import (
"context"
"go.uber.org/zap"
"github.com/milvus-io/milvus-proto/go-api/commonpb"
"github.com/milvus-io/milvus-proto/go-api/milvuspb"
"github.com/milvus-io/milvus-proto/go-api/msgpb"
"github.com/milvus-io/milvus-proto/go-api/schemapb"
"github.com/milvus-io/milvus/internal/allocator"
"github.com/milvus-io/milvus/internal/log"
"github.com/milvus-io/milvus/internal/mq/msgstream"
"github.com/milvus-io/milvus/internal/proto/internalpb"
"github.com/milvus-io/milvus/internal/util/commonpbutil"
"github.com/milvus-io/milvus/internal/util/retry"
"github.com/milvus-io/milvus/internal/util/typeutil"
"go.uber.org/zap"
)
func assignSegmentID(ctx context.Context, insertMsg *msgstream.InsertMsg, result *milvuspb.MutationResult, channelNames []string, idAllocator *allocator.IDAllocator, segIDAssigner *segIDAssigner) (*msgstream.MsgPack, error) {
@ -91,7 +92,7 @@ func assignSegmentID(ctx context.Context, insertMsg *msgstream.InsertMsg, result
// create empty insert message
createInsertMsg := func(segmentID UniqueID, channelName string, msgID int64) *msgstream.InsertMsg {
insertReq := internalpb.InsertRequest{
insertReq := msgpb.InsertRequest{
Base: commonpbutil.NewMsgBase(
commonpbutil.WithMsgType(commonpb.MsgType_Insert),
commonpbutil.WithMsgID(msgID),
@ -104,7 +105,7 @@ func assignSegmentID(ctx context.Context, insertMsg *msgstream.InsertMsg, result
PartitionName: insertMsg.PartitionName,
SegmentID: segmentID,
ShardName: channelName,
Version: internalpb.InsertDataVersion_ColumnBased,
Version: msgpb.InsertDataVersion_ColumnBased,
}
insertReq.FieldsData = make([]*schemapb.FieldData, len(insertMsg.GetFieldsData()))

4
internal/proxy/task_delete.go
View File

@ -10,12 +10,12 @@ import (
"github.com/milvus-io/milvus-proto/go-api/commonpb"
"github.com/milvus-io/milvus-proto/go-api/milvuspb"
"github.com/milvus-io/milvus-proto/go-api/msgpb"
"github.com/milvus-io/milvus-proto/go-api/schemapb"
"github.com/milvus-io/milvus/internal/common"
"github.com/milvus-io/milvus/internal/log"
"github.com/milvus-io/milvus/internal/metrics"
"github.com/milvus-io/milvus/internal/mq/msgstream"
"github.com/milvus-io/milvus/internal/proto/internalpb"
"github.com/milvus-io/milvus/internal/proto/planpb"
"github.com/milvus-io/milvus/internal/util/commonpbutil"
"github.com/milvus-io/milvus/internal/util/paramtable"
@ -272,7 +272,7 @@ func (dt *deleteTask) Execute(ctx context.Context) (err error) {
ts := dt.deleteMsg.Timestamps[index]
_, ok := result[key]
if !ok {
sliceRequest := internalpb.DeleteRequest{
sliceRequest := msgpb.DeleteRequest{
Base: commonpbutil.NewMsgBase(
commonpbutil.WithMsgType(commonpb.MsgType_Delete),
commonpbutil.WithMsgID(dt.deleteMsg.Base.MsgID),

13
internal/proxy/task_insert_test.go
View File

@ -3,10 +3,11 @@ package proxy
import (
"testing"
"github.com/milvus-io/milvus-proto/go-api/commonpb"
"github.com/milvus-io/milvus-proto/go-api/schemapb"
"github.com/milvus-io/milvus/internal/proto/internalpb"
"github.com/stretchr/testify/assert"
"github.com/milvus-io/milvus-proto/go-api/commonpb"
"github.com/milvus-io/milvus-proto/go-api/msgpb"
"github.com/milvus-io/milvus-proto/go-api/schemapb"
)
func TestInsertTask_CheckAligned(t *testing.T) {
@ -15,7 +16,7 @@ func TestInsertTask_CheckAligned(t *testing.T) {
// passed NumRows is less than 0
case1 := insertTask{
insertMsg: &BaseInsertTask{
InsertRequest: internalpb.InsertRequest{
InsertRequest: msgpb.InsertRequest{
Base: &commonpb.MsgBase{
MsgType: commonpb.MsgType_Insert,
},
@ -43,11 +44,11 @@ func TestInsertTask_CheckAligned(t *testing.T) {
dim := 128
case2 := insertTask{
insertMsg: &BaseInsertTask{
InsertRequest: internalpb.InsertRequest{
InsertRequest: msgpb.InsertRequest{
Base: &commonpb.MsgBase{
MsgType: commonpb.MsgType_Insert,
},
Version: internalpb.InsertDataVersion_ColumnBased,
Version: msgpb.InsertDataVersion_ColumnBased,
RowIDs: generateInt64Array(numRows),
Timestamps: generateUint64Array(numRows),
},

36
internal/proxy/task_test.go
View File

@ -27,27 +27,26 @@ import (
"time"
"github.com/cockroachdb/errors"
"github.com/milvus-io/milvus/internal/proto/indexpb"
"github.com/milvus-io/milvus/internal/querycoordv2/meta"
"github.com/golang/protobuf/proto"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/mock"
"github.com/milvus-io/milvus-proto/go-api/commonpb"
"github.com/milvus-io/milvus-proto/go-api/milvuspb"
"github.com/milvus-io/milvus-proto/go-api/msgpb"
"github.com/milvus-io/milvus-proto/go-api/schemapb"
"github.com/milvus-io/milvus/internal/allocator"
"github.com/milvus-io/milvus/internal/common"
"github.com/milvus-io/milvus/internal/mocks"
"github.com/milvus-io/milvus/internal/mq/msgstream"
"github.com/milvus-io/milvus/internal/proto/internalpb"
"github.com/milvus-io/milvus/internal/proto/indexpb"
"github.com/milvus-io/milvus/internal/proto/querypb"
"github.com/milvus-io/milvus/internal/querycoordv2/meta"
"github.com/milvus-io/milvus/internal/util/distance"
"github.com/milvus-io/milvus/internal/util/funcutil"
"github.com/milvus-io/milvus/internal/util/paramtable"
"github.com/milvus-io/milvus/internal/util/typeutil"
"github.com/milvus-io/milvus/internal/util/uniquegenerator"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/mock"
)
// TODO(dragondriver): add more test cases
@ -1280,6 +1279,7 @@ func TestShowPartitionsTask(t *testing.T) {
assert.NotNil(t, err)
}
func TestTask_Int64PrimaryKey(t *testing.T) {
var err error
@ -1389,7 +1389,7 @@ func TestTask_Int64PrimaryKey(t *testing.T) {
BaseMsg: msgstream.BaseMsg{
HashValues: hash,
},
InsertRequest: internalpb.InsertRequest{
InsertRequest: msgpb.InsertRequest{
Base: &commonpb.MsgBase{
MsgType: commonpb.MsgType_Insert,
MsgID: 0,
@ -1399,7 +1399,7 @@ func TestTask_Int64PrimaryKey(t *testing.T) {
CollectionName: collectionName,
PartitionName: partitionName,
NumRows: uint64(nb),
Version: internalpb.InsertDataVersion_ColumnBased,
Version: msgpb.InsertDataVersion_ColumnBased,
},
},
@ -1443,7 +1443,7 @@ func TestTask_Int64PrimaryKey(t *testing.T) {
Condition: NewTaskCondition(ctx),
deleteMsg: &msgstream.DeleteMsg{
BaseMsg: msgstream.BaseMsg{},
DeleteRequest: internalpb.DeleteRequest{
DeleteRequest: msgpb.DeleteRequest{
Base: &commonpb.MsgBase{
MsgType: commonpb.MsgType_Delete,
MsgID: 0,
@ -1497,7 +1497,7 @@ func TestTask_Int64PrimaryKey(t *testing.T) {
Condition: NewTaskCondition(ctx),
deleteMsg: &msgstream.DeleteMsg{
BaseMsg: msgstream.BaseMsg{},
DeleteRequest: internalpb.DeleteRequest{
DeleteRequest: msgpb.DeleteRequest{
Base: &commonpb.MsgBase{
MsgType: commonpb.MsgType_Delete,
MsgID: 0,
@ -1642,7 +1642,7 @@ func TestTask_VarCharPrimaryKey(t *testing.T) {
BaseMsg: msgstream.BaseMsg{
HashValues: hash,
},
InsertRequest: internalpb.InsertRequest{
InsertRequest: msgpb.InsertRequest{
Base: &commonpb.MsgBase{
MsgType: commonpb.MsgType_Insert,
MsgID: 0,
@ -1652,7 +1652,7 @@ func TestTask_VarCharPrimaryKey(t *testing.T) {
CollectionName: collectionName,
PartitionName: partitionName,
NumRows: uint64(nb),
Version: internalpb.InsertDataVersion_ColumnBased,
Version: msgpb.InsertDataVersion_ColumnBased,
},
},
@ -1701,7 +1701,7 @@ func TestTask_VarCharPrimaryKey(t *testing.T) {
BaseMsg: msgstream.BaseMsg{
HashValues: hash,
},
InsertRequest: internalpb.InsertRequest{
InsertRequest: msgpb.InsertRequest{
Base: &commonpb.MsgBase{
MsgType: commonpb.MsgType_Insert,
MsgID: 0,
@ -1711,14 +1711,14 @@ func TestTask_VarCharPrimaryKey(t *testing.T) {
CollectionName: collectionName,
PartitionName: partitionName,
NumRows: uint64(nb),
Version: internalpb.InsertDataVersion_ColumnBased,
Version: msgpb.InsertDataVersion_ColumnBased,
},
},
DeleteMsg: &msgstream.DeleteMsg{
BaseMsg: msgstream.BaseMsg{
HashValues: hash,
},
DeleteRequest: internalpb.DeleteRequest{
DeleteRequest: msgpb.DeleteRequest{
Base: &commonpb.MsgBase{
MsgType: commonpb.MsgType_Delete,
MsgID: 0,
@ -1786,7 +1786,7 @@ func TestTask_VarCharPrimaryKey(t *testing.T) {
Condition: NewTaskCondition(ctx),
deleteMsg: &msgstream.DeleteMsg{
BaseMsg: msgstream.BaseMsg{},
DeleteRequest: internalpb.DeleteRequest{
DeleteRequest: msgpb.DeleteRequest{
Base: &commonpb.MsgBase{
MsgType: commonpb.MsgType_Delete,
MsgID: 0,
@ -1840,7 +1840,7 @@ func TestTask_VarCharPrimaryKey(t *testing.T) {
Condition: NewTaskCondition(ctx),
deleteMsg: &msgstream.DeleteMsg{
BaseMsg: msgstream.BaseMsg{},
DeleteRequest: internalpb.DeleteRequest{
DeleteRequest: msgpb.DeleteRequest{
Base: &commonpb.MsgBase{
MsgType: commonpb.MsgType_Delete,
MsgID: 0,

15
internal/proxy/task_upsert.go
View File

@ -20,21 +20,22 @@ import (
"fmt"
"strconv"
"go.opentelemetry.io/otel"
"go.uber.org/zap"
"github.com/milvus-io/milvus-proto/go-api/commonpb"
"github.com/milvus-io/milvus-proto/go-api/milvuspb"
"github.com/milvus-io/milvus-proto/go-api/msgpb"
"github.com/milvus-io/milvus-proto/go-api/schemapb"
"github.com/milvus-io/milvus/internal/allocator"
"github.com/milvus-io/milvus/internal/common"
"github.com/milvus-io/milvus/internal/log"
"github.com/milvus-io/milvus/internal/metrics"
"github.com/milvus-io/milvus/internal/mq/msgstream"
"github.com/milvus-io/milvus/internal/proto/internalpb"
"github.com/milvus-io/milvus/internal/util/commonpbutil"
"github.com/milvus-io/milvus/internal/util/paramtable"
"github.com/milvus-io/milvus/internal/util/timerecord"
"github.com/milvus-io/milvus/internal/util/typeutil"
"go.opentelemetry.io/otel"
"go.uber.org/zap"
)
type upsertTask struct {
@ -260,7 +261,7 @@ func (it *upsertTask) PreExecute(ctx context.Context) error {
it.upsertMsg = &msgstream.UpsertMsg{
InsertMsg: &msgstream.InsertMsg{
InsertRequest: internalpb.InsertRequest{
InsertRequest: msgpb.InsertRequest{
Base: commonpbutil.NewMsgBase(
commonpbutil.WithMsgType(commonpb.MsgType_Insert),
commonpbutil.WithSourceID(paramtable.GetNodeID()),
@ -269,11 +270,11 @@ func (it *upsertTask) PreExecute(ctx context.Context) error {
PartitionName: it.req.PartitionName,
FieldsData: it.req.FieldsData,
NumRows: uint64(it.req.NumRows),
Version: internalpb.InsertDataVersion_ColumnBased,
Version: msgpb.InsertDataVersion_ColumnBased,
},
},
DeleteMsg: &msgstream.DeleteMsg{
DeleteRequest: internalpb.DeleteRequest{
DeleteRequest: msgpb.DeleteRequest{
Base: commonpbutil.NewMsgBase(
commonpbutil.WithMsgType(commonpb.MsgType_Delete),
commonpbutil.WithSourceID(paramtable.GetNodeID()),
@ -414,7 +415,7 @@ func (it *upsertTask) deleteExecute(ctx context.Context, msgPack *msgstream.MsgP
ts := it.upsertMsg.DeleteMsg.Timestamps[index]
_, ok := result[key]
if !ok {
sliceRequest := internalpb.DeleteRequest{
sliceRequest := msgpb.DeleteRequest{
Base: commonpbutil.NewMsgBase(
commonpbutil.WithMsgType(commonpb.MsgType_Delete),
commonpbutil.WithTimeStamp(ts),

17
internal/proxy/task_upsert_test.go
View File

@ -18,13 +18,14 @@ package proxy
import (
"testing"
"github.com/stretchr/testify/assert"
"github.com/milvus-io/milvus-proto/go-api/commonpb"
"github.com/milvus-io/milvus-proto/go-api/milvuspb"
"github.com/milvus-io/milvus-proto/go-api/msgpb"
"github.com/milvus-io/milvus-proto/go-api/schemapb"
"github.com/milvus-io/milvus/internal/mq/msgstream"
"github.com/milvus-io/milvus/internal/proto/internalpb"
"github.com/milvus-io/milvus/internal/util/commonpbutil"
"github.com/stretchr/testify/assert"
)
func TestUpsertTask_CheckAligned(t *testing.T) {
@ -37,11 +38,11 @@ func TestUpsertTask_CheckAligned(t *testing.T) {
},
upsertMsg: &msgstream.UpsertMsg{
InsertMsg: &msgstream.InsertMsg{
InsertRequest: internalpb.InsertRequest{},
InsertRequest: msgpb.InsertRequest{},
},
},
}
case1.upsertMsg.InsertMsg.InsertRequest = internalpb.InsertRequest{
case1.upsertMsg.InsertMsg.InsertRequest = msgpb.InsertRequest{
Base: commonpbutil.NewMsgBase(
commonpbutil.WithMsgType(commonpb.MsgType_Insert),
),
@ -49,7 +50,7 @@ func TestUpsertTask_CheckAligned(t *testing.T) {
PartitionName: case1.req.PartitionName,
FieldsData: case1.req.FieldsData,
NumRows: uint64(case1.req.NumRows),
Version: internalpb.InsertDataVersion_ColumnBased,
Version: msgpb.InsertDataVersion_ColumnBased,
}
err = case1.upsertMsg.InsertMsg.CheckAligned()
@ -96,7 +97,7 @@ func TestUpsertTask_CheckAligned(t *testing.T) {
},
upsertMsg: &msgstream.UpsertMsg{
InsertMsg: &msgstream.InsertMsg{
InsertRequest: internalpb.InsertRequest{},
InsertRequest: msgpb.InsertRequest{},
},
},
}
@ -114,7 +115,7 @@ func TestUpsertTask_CheckAligned(t *testing.T) {
newBinaryVectorFieldData("BinaryVector", numRows, dim),
newScalarFieldData(varCharFieldSchema, "VarChar", numRows),
}
case2.upsertMsg.InsertMsg.InsertRequest = internalpb.InsertRequest{
case2.upsertMsg.InsertMsg.InsertRequest = msgpb.InsertRequest{
Base: commonpbutil.NewMsgBase(
commonpbutil.WithMsgType(commonpb.MsgType_Insert),
),
@ -124,7 +125,7 @@ func TestUpsertTask_CheckAligned(t *testing.T) {
NumRows: uint64(case2.req.NumRows),
RowIDs: case2.rowIDs,
Timestamps: case2.timestamps,
Version: internalpb.InsertDataVersion_ColumnBased,
Version: msgpb.InsertDataVersion_ColumnBased,
}
err = case2.upsertMsg.InsertMsg.CheckAligned()
assert.NoError(t, err)

34
internal/proxy/util_test.go
View File

@ -25,13 +25,13 @@ import (
"time"
"github.com/cockroachdb/errors"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/mock"
"google.golang.org/grpc/metadata"
"github.com/milvus-io/milvus-proto/go-api/commonpb"
"github.com/milvus-io/milvus-proto/go-api/milvuspb"
"github.com/milvus-io/milvus-proto/go-api/msgpb"
"github.com/milvus-io/milvus-proto/go-api/schemapb"
"github.com/milvus-io/milvus/internal/proto/internalpb"
"github.com/milvus-io/milvus/internal/proto/querypb"
@ -1014,7 +1014,7 @@ func Test_InsertTaskCheckLengthOfFieldsData(t *testing.T) {
Fields: []*schemapb.FieldSchema{},
},
insertMsg: &BaseInsertTask{
InsertRequest: internalpb.InsertRequest{
InsertRequest: msgpb.InsertRequest{
Base: &commonpb.MsgBase{
MsgType: commonpb.MsgType_Insert,
},
@ -1046,7 +1046,7 @@ func Test_InsertTaskCheckLengthOfFieldsData(t *testing.T) {
},
},
insertMsg: &BaseInsertTask{
InsertRequest: internalpb.InsertRequest{
InsertRequest: msgpb.InsertRequest{
Base: &commonpb.MsgBase{
MsgType: commonpb.MsgType_Insert,
},
@ -1055,7 +1055,7 @@ func Test_InsertTaskCheckLengthOfFieldsData(t *testing.T) {
}
// passed fields is empty
// case2.BaseInsertTask = BaseInsertTask{
// InsertRequest: internalpb.insertRequest{
// InsertRequest: msgpb.InsertRequest{
// Base: &commonpb.MsgBase{
// MsgType: commonpb.MsgType_Insert,
// MsgID: 0,
@ -1103,7 +1103,7 @@ func Test_InsertTaskCheckLengthOfFieldsData(t *testing.T) {
},
},
insertMsg: &BaseInsertTask{
InsertRequest: internalpb.InsertRequest{
InsertRequest: msgpb.InsertRequest{
Base: &commonpb.MsgBase{
MsgType: commonpb.MsgType_Insert,
},
@ -1137,7 +1137,7 @@ func Test_InsertTaskCheckLengthOfFieldsData(t *testing.T) {
},
},
insertMsg: &BaseInsertTask{
InsertRequest: internalpb.InsertRequest{
InsertRequest: msgpb.InsertRequest{
Base: &commonpb.MsgBase{
MsgType: commonpb.MsgType_Insert,
},
@ -1162,7 +1162,7 @@ func Test_InsertTaskCheckPrimaryFieldData(t *testing.T) {
Fields: []*schemapb.FieldSchema{},
},
insertMsg: &BaseInsertTask{
InsertRequest: internalpb.InsertRequest{
InsertRequest: msgpb.InsertRequest{
Base: &commonpb.MsgBase{
MsgType: commonpb.MsgType_Insert,
},
@ -1199,7 +1199,7 @@ func Test_InsertTaskCheckPrimaryFieldData(t *testing.T) {
},
},
insertMsg: &BaseInsertTask{
InsertRequest: internalpb.InsertRequest{
InsertRequest: msgpb.InsertRequest{
Base: &commonpb.MsgBase{
MsgType: commonpb.MsgType_Insert,
},
@ -1212,7 +1212,7 @@ func Test_InsertTaskCheckPrimaryFieldData(t *testing.T) {
Type: schemapb.DataType_Int64,
},
},
Version: internalpb.InsertDataVersion_RowBased,
Version: msgpb.InsertDataVersion_RowBased,
},
},
result: &milvuspb.MutationResult{
@ -1243,7 +1243,7 @@ func Test_InsertTaskCheckPrimaryFieldData(t *testing.T) {
},
},
insertMsg: &BaseInsertTask{
InsertRequest: internalpb.InsertRequest{
InsertRequest: msgpb.InsertRequest{
Base: &commonpb.MsgBase{
MsgType: commonpb.MsgType_Insert,
},
@ -1287,7 +1287,7 @@ func Test_InsertTaskCheckPrimaryFieldData(t *testing.T) {
},
},
insertMsg: &BaseInsertTask{
InsertRequest: internalpb.InsertRequest{
InsertRequest: msgpb.InsertRequest{
Base: &commonpb.MsgBase{
MsgType: commonpb.MsgType_Insert,
},
@ -1335,7 +1335,7 @@ func Test_UpsertTaskCheckPrimaryFieldData(t *testing.T) {
Fields: []*schemapb.FieldSchema{},
},
insertMsg: &BaseInsertTask{
InsertRequest: internalpb.InsertRequest{
InsertRequest: msgpb.InsertRequest{
Base: &commonpb.MsgBase{
MsgType: commonpb.MsgType_Insert,
},
@ -1373,7 +1373,7 @@ func Test_UpsertTaskCheckPrimaryFieldData(t *testing.T) {
},
},
insertMsg: &BaseInsertTask{
InsertRequest: internalpb.InsertRequest{
InsertRequest: msgpb.InsertRequest{
Base: &commonpb.MsgBase{
MsgType: commonpb.MsgType_Insert,
},
@ -1417,7 +1417,7 @@ func Test_UpsertTaskCheckPrimaryFieldData(t *testing.T) {
},
},
insertMsg: &BaseInsertTask{
InsertRequest: internalpb.InsertRequest{
InsertRequest: msgpb.InsertRequest{
Base: &commonpb.MsgBase{
MsgType: commonpb.MsgType_Insert,
},
@ -1460,7 +1460,7 @@ func Test_UpsertTaskCheckPrimaryFieldData(t *testing.T) {
},
},
insertMsg: &BaseInsertTask{
InsertRequest: internalpb.InsertRequest{
InsertRequest: msgpb.InsertRequest{
Base: &commonpb.MsgBase{
MsgType: commonpb.MsgType_Insert,
},
@ -1508,7 +1508,7 @@ func Test_UpsertTaskCheckPrimaryFieldData(t *testing.T) {
},
},
insertMsg: &BaseInsertTask{
InsertRequest: internalpb.InsertRequest{
InsertRequest: msgpb.InsertRequest{
Base: &commonpb.MsgBase{
MsgType: commonpb.MsgType_Insert,
},
@ -1553,7 +1553,7 @@ func Test_UpsertTaskCheckPrimaryFieldData(t *testing.T) {
},
},
insertMsg: &BaseInsertTask{
InsertRequest: internalpb.InsertRequest{
InsertRequest: msgpb.InsertRequest{
Base: &commonpb.MsgBase{
MsgType: commonpb.MsgType_Insert,
},

10
internal/querycoordv2/checkers/segment_checker_test.go
View File

@ -24,9 +24,9 @@ import (
"github.com/stretchr/testify/mock"
"github.com/stretchr/testify/suite"
"github.com/milvus-io/milvus-proto/go-api/msgpb"
etcdkv "github.com/milvus-io/milvus/internal/kv/etcd"
"github.com/milvus-io/milvus/internal/proto/datapb"
"github.com/milvus-io/milvus/internal/proto/internalpb"
"github.com/milvus-io/milvus/internal/querycoordv2/balance"
"github.com/milvus-io/milvus/internal/querycoordv2/meta"
. "github.com/milvus-io/milvus/internal/querycoordv2/params"
@ -214,7 +214,7 @@ func (suite *SegmentCheckerTestSuite) TestReleaseGrowingSegments() {
{
CollectionID: 1,
ChannelName: "test-insert-channel",
SeekPosition: &internalpb.MsgPosition{Timestamp: 10},
SeekPosition: &msgpb.MsgPosition{Timestamp: 10},
},
}
suite.broker.EXPECT().GetRecoveryInfo(mock.Anything, int64(1), int64(1)).Return(
@ -224,11 +224,11 @@ func (suite *SegmentCheckerTestSuite) TestReleaseGrowingSegments() {
growingSegments := make(map[int64]*meta.Segment)
growingSegments[2] = utils.CreateTestSegment(1, 1, 2, 2, 0, "test-insert-channel")
growingSegments[2].SegmentInfo.StartPosition = &internalpb.MsgPosition{Timestamp: 2}
growingSegments[2].SegmentInfo.StartPosition = &msgpb.MsgPosition{Timestamp: 2}
growingSegments[3] = utils.CreateTestSegment(1, 1, 3, 2, 1, "test-insert-channel")
growingSegments[3].SegmentInfo.StartPosition = &internalpb.MsgPosition{Timestamp: 3}
growingSegments[3].SegmentInfo.StartPosition = &msgpb.MsgPosition{Timestamp: 3}
growingSegments[4] = utils.CreateTestSegment(1, 1, 4, 2, 1, "test-insert-channel")
growingSegments[4].SegmentInfo.StartPosition = &internalpb.MsgPosition{Timestamp: 11}
growingSegments[4].SegmentInfo.StartPosition = &msgpb.MsgPosition{Timestamp: 11}
dmChannel := utils.CreateTestChannel(1, 2, 1, "test-insert-channel")
dmChannel.UnflushedSegmentIds = []int64{2, 3}

6
internal/querycoordv2/task/merge_task.go
View File

@ -17,7 +17,7 @@
package task
import (
"github.com/milvus-io/milvus/internal/proto/internalpb"
"github.com/milvus-io/milvus-proto/go-api/msgpb"
"github.com/milvus-io/milvus/internal/proto/querypb"
)
@ -61,7 +61,7 @@ func (task *LoadSegmentsTask) Merge(other MergeableTask[segmentIndex, *querypb.L
task.tasks = append(task.tasks, otherTask.tasks...)
task.steps = append(task.steps, otherTask.steps...)
task.req.Infos = append(task.req.Infos, otherTask.req.GetInfos()...)
positions := make(map[string]*internalpb.MsgPosition)
positions := make(map[string]*msgpb.MsgPosition)
for _, position := range task.req.DeltaPositions {
positions[position.GetChannelName()] = position
}
@ -72,7 +72,7 @@ func (task *LoadSegmentsTask) Merge(other MergeableTask[segmentIndex, *querypb.L
}
positions[position.GetChannelName()] = merged
}
task.req.DeltaPositions = make([]*internalpb.MsgPosition, 0, len(positions))
task.req.DeltaPositions = make([]*msgpb.MsgPosition, 0, len(positions))
for _, position := range positions {
task.req.DeltaPositions = append(task.req.DeltaPositions, position)
}

11
internal/querycoordv2/task/merger_test.go
View File

@ -21,11 +21,12 @@ import (
"testing"
"time"
"github.com/milvus-io/milvus/internal/proto/internalpb"
"github.com/stretchr/testify/suite"
"github.com/milvus-io/milvus-proto/go-api/msgpb"
"github.com/milvus-io/milvus/internal/proto/querypb"
. "github.com/milvus-io/milvus/internal/querycoordv2/params"
"github.com/milvus-io/milvus/internal/util/paramtable"
"github.com/stretchr/testify/suite"
)
type MergerSuite struct {
@ -55,7 +56,7 @@ func (suite *MergerSuite) SetupSuite() {
InsertChannel: "dmc0",
},
},
DeltaPositions: []*internalpb.MsgPosition{
DeltaPositions: []*msgpb.MsgPosition{
{
ChannelName: "dmc0",
Timestamp: 2,
@ -75,7 +76,7 @@ func (suite *MergerSuite) SetupSuite() {
InsertChannel: "dmc0",
},
},
DeltaPositions: []*internalpb.MsgPosition{
DeltaPositions: []*msgpb.MsgPosition{
{
ChannelName: "dmc0",
Timestamp: 3,
@ -95,7 +96,7 @@ func (suite *MergerSuite) SetupSuite() {
InsertChannel: "dmc0",
},
},
DeltaPositions: []*internalpb.MsgPosition{
DeltaPositions: []*msgpb.MsgPosition{
{
ChannelName: "dmc0",
Timestamp: 1,

6
internal/querycoordv2/task/utils.go
View File

@ -23,10 +23,10 @@ import (
"go.uber.org/zap"
"github.com/milvus-io/milvus-proto/go-api/commonpb"
"github.com/milvus-io/milvus-proto/go-api/msgpb"
"github.com/milvus-io/milvus-proto/go-api/schemapb"
"github.com/milvus-io/milvus/internal/log"
"github.com/milvus-io/milvus/internal/proto/datapb"
"github.com/milvus-io/milvus/internal/proto/internalpb"
"github.com/milvus-io/milvus/internal/proto/querypb"
"github.com/milvus-io/milvus/internal/querycoordv2/meta"
"github.com/milvus-io/milvus/internal/util/commonpbutil"
@ -89,7 +89,7 @@ func packLoadSegmentRequest(
loadInfo *querypb.SegmentLoadInfo,
resp *datapb.GetSegmentInfoResponse,
) *querypb.LoadSegmentsRequest {
var deltaPosition *internalpb.MsgPosition
var deltaPosition *msgpb.MsgPosition
segment := resp.GetInfos()[0]
var posSrcStr string
@ -130,7 +130,7 @@ func packLoadSegmentRequest(
LoadMeta: loadMeta,
CollectionID: task.CollectionID(),
ReplicaID: task.ReplicaID(),
DeltaPositions: []*internalpb.MsgPosition{deltaPosition},
DeltaPositions: []*msgpb.MsgPosition{deltaPosition},
DstNodeID: action.Node(),
Version: time.Now().UnixNano(),
NeedTransfer: true,

8
internal/querycoordv2/task/utils_test.go
View File

@ -24,8 +24,8 @@ import (
"github.com/golang/protobuf/proto"
"github.com/stretchr/testify/assert"
"github.com/milvus-io/milvus-proto/go-api/msgpb"
"github.com/milvus-io/milvus/internal/proto/datapb"
"github.com/milvus-io/milvus/internal/proto/internalpb"
"github.com/milvus-io/milvus/internal/util/tsoutil"
)
@ -41,11 +41,11 @@ func Test_packLoadSegmentRequest(t *testing.T) {
segmentInfo := &datapb.SegmentInfo{
ID: 0,
InsertChannel: mockVChannel,
StartPosition: &internalpb.MsgPosition{
StartPosition: &msgpb.MsgPosition{
ChannelName: mockPChannel,
Timestamp: t1,
},
DmlPosition: &internalpb.MsgPosition{
DmlPosition: &msgpb.MsgPosition{
ChannelName: mockPChannel,
Timestamp: t2,
},
@ -60,7 +60,7 @@ func Test_packLoadSegmentRequest(t *testing.T) {
Infos: []*datapb.SegmentInfo{
proto.Clone(segmentInfo).(*datapb.SegmentInfo),
},
ChannelCheckpoint: map[string]*internalpb.MsgPosition{
ChannelCheckpoint: map[string]*msgpb.MsgPosition{
mockVChannel: {
ChannelName: mockPChannel,
Timestamp: t3,

4
internal/querynode/flow_graph_filter_dm_node_test.go
View File

@ -22,10 +22,10 @@ import (
"github.com/stretchr/testify/assert"
"github.com/milvus-io/milvus-proto/go-api/commonpb"
"github.com/milvus-io/milvus-proto/go-api/msgpb"
"github.com/milvus-io/milvus-proto/go-api/schemapb"
"github.com/milvus-io/milvus/internal/mq/msgstream"
"github.com/milvus-io/milvus/internal/proto/datapb"
"github.com/milvus-io/milvus/internal/proto/internalpb"
"github.com/milvus-io/milvus/internal/storage"
"github.com/milvus-io/milvus/internal/util/flowgraph"
)
@ -115,7 +115,7 @@ func TestFlowGraphFilterDmNode_filterInvalidInsertMessage(t *testing.T) {
ID: defaultSegmentID,
CollectionID: defaultCollectionID,
PartitionID: defaultPartitionID,
DmlPosition: &internalpb.MsgPosition{
DmlPosition: &msgpb.MsgPosition{
Timestamp: Timestamp(1000),
},
},

5
internal/querynode/flow_graph_insert_node.go
View File

@ -27,15 +27,14 @@ import (
"sync"
"github.com/cockroachdb/errors"
"go.opentelemetry.io/otel/trace"
"go.uber.org/zap"
"github.com/milvus-io/milvus-proto/go-api/msgpb"
"github.com/milvus-io/milvus-proto/go-api/schemapb"
"github.com/milvus-io/milvus/internal/common"
"github.com/milvus-io/milvus/internal/log"
"github.com/milvus-io/milvus/internal/mq/msgstream"
"github.com/milvus-io/milvus/internal/proto/internalpb"
"github.com/milvus-io/milvus/internal/proto/segcorepb"
"github.com/milvus-io/milvus/internal/storage"
"github.com/milvus-io/milvus/internal/util/flowgraph"
@ -150,7 +149,7 @@ func (iNode *insertNode) Operate(in []Msg) []Msg {
zap.Int64("segmentID", insertMsg.SegmentID),
zap.Uint64("startPosition", insertMsg.BeginTs()),
)
startPosition := &internalpb.MsgPosition{
startPosition := &msgpb.MsgPosition{
ChannelName: insertMsg.ShardName,
Timestamp: insertMsg.BeginTs(),
}

9
internal/querynode/flow_graph_insert_node_test.go
View File

@ -20,18 +20,17 @@ import (
"fmt"
"testing"
"github.com/milvus-io/milvus/internal/proto/internalpb"
"github.com/bits-and-blooms/bloom/v3"
bloom "github.com/bits-and-blooms/bloom/v3"
"github.com/stretchr/testify/assert"
"go.uber.org/atomic"
"github.com/milvus-io/milvus-proto/go-api/commonpb"
"github.com/milvus-io/milvus-proto/go-api/msgpb"
"github.com/milvus-io/milvus-proto/go-api/schemapb"
"github.com/milvus-io/milvus/internal/common"
"github.com/milvus-io/milvus/internal/mq/msgstream"
"github.com/milvus-io/milvus/internal/storage"
"github.com/milvus-io/milvus/internal/util/flowgraph"
"go.uber.org/atomic"
)
func getInsertNode() (*insertNode, error) {
@ -341,7 +340,7 @@ func TestFlowGraphInsertNode_operate(t *testing.T) {
}
iMsg := genInsertMsg()
iMsg.insertMessages[0].Version = internalpb.InsertDataVersion_RowBased
iMsg.insertMessages[0].Version = msgpb.InsertDataVersion_RowBased
msg := []flowgraph.Msg{iMsg}
assert.Panics(t, func() {
insertNode.Operate(msg)

6
internal/querynode/impl.go
View File

@ -25,7 +25,6 @@ import (
"time"
"github.com/cockroachdb/errors"
"github.com/golang/protobuf/proto"
"github.com/samber/lo"
"go.opentelemetry.io/otel"
@ -35,6 +34,7 @@ import (
"github.com/milvus-io/milvus-proto/go-api/commonpb"
"github.com/milvus-io/milvus-proto/go-api/milvuspb"
"github.com/milvus-io/milvus-proto/go-api/msgpb"
"github.com/milvus-io/milvus/internal/common"
"github.com/milvus-io/milvus/internal/log"
"github.com/milvus-io/milvus/internal/metrics"
@ -1293,10 +1293,10 @@ func (node *QueryNode) GetDataDistribution(ctx context.Context, req *querypb.Get
sealedSegments := node.metaReplica.getSealedSegments()
shardClusters := node.ShardClusterService.GetShardClusters()
channelGrowingsMap := make(map[string]map[int64]*internalpb.MsgPosition)
channelGrowingsMap := make(map[string]map[int64]*msgpb.MsgPosition)
for _, s := range growingSegments {
if _, ok := channelGrowingsMap[s.vChannelID]; !ok {
channelGrowingsMap[s.vChannelID] = make(map[int64]*internalpb.MsgPosition)
channelGrowingsMap[s.vChannelID] = make(map[int64]*msgpb.MsgPosition)
}
channelGrowingsMap[s.vChannelID][s.ID()] = s.startPosition

16
internal/querynode/load_segment_task_test.go
View File

@ -27,11 +27,11 @@ import (
"github.com/stretchr/testify/require"
"github.com/milvus-io/milvus-proto/go-api/commonpb"
"github.com/milvus-io/milvus-proto/go-api/msgpb"
"github.com/milvus-io/milvus-proto/go-api/schemapb"
"github.com/milvus-io/milvus/internal/mq/msgstream"
"github.com/milvus-io/milvus/internal/mq/msgstream/mqwrapper/rmq"
"github.com/milvus-io/milvus/internal/proto/datapb"
"github.com/milvus-io/milvus/internal/proto/internalpb"
"github.com/milvus-io/milvus/internal/proto/querypb"
"github.com/milvus-io/milvus/internal/util/funcutil"
"github.com/milvus-io/milvus/internal/util/hardware"
@ -213,7 +213,7 @@ func TestTask_loadSegmentsTask(t *testing.T) {
Base: genCommonMsgBase(commonpb.MsgType_LoadSegments, node.GetSession().ServerID),
Schema: schema,
Infos: []*querypb.SegmentLoadInfo{segmentLoadInfo},
DeltaPositions: []*internalpb.MsgPosition{
DeltaPositions: []*msgpb.MsgPosition{
{
ChannelName: testVChannel,
MsgID: rmq.SerializeRmqID(0),
@ -248,7 +248,7 @@ func TestTask_loadSegmentsTask(t *testing.T) {
BaseMsg: msgstream.BaseMsg{
HashValues: []uint32{1},
},
TimeTickMsg: internalpb.TimeTickMsg{
TimeTickMsg: msgpb.TimeTickMsg{
Base: &commonpb.MsgBase{
MsgType: commonpb.MsgType_TimeTick,
Timestamp: 100,
@ -260,7 +260,7 @@ func TestTask_loadSegmentsTask(t *testing.T) {
BaseMsg: msgstream.BaseMsg{
HashValues: []uint32{1, 1, 1},
},
DeleteRequest: internalpb.DeleteRequest{
DeleteRequest: msgpb.DeleteRequest{
Base: &commonpb.MsgBase{
MsgType: commonpb.MsgType_Delete,
Timestamp: 110,
@ -318,7 +318,7 @@ func TestTask_loadSegmentsTask(t *testing.T) {
InsertChannel: testVChannel,
},
},
DeltaPositions: []*internalpb.MsgPosition{
DeltaPositions: []*msgpb.MsgPosition{
{
ChannelName: testVChannel,
MsgID: msgIDs[0].Serialize(),
@ -382,7 +382,7 @@ func TestTask_loadSegmentsTask(t *testing.T) {
InsertChannel: testVChannel,
},
},
DeltaPositions: []*internalpb.MsgPosition{
DeltaPositions: []*msgpb.MsgPosition{
{
ChannelName: testVChannel,
MsgID: msgIDs[0].Serialize(),
@ -431,7 +431,7 @@ func TestTask_loadSegmentsTaskLoadDelta(t *testing.T) {
Base: genCommonMsgBase(commonpb.MsgType_LoadSegments, node.GetSession().ServerID),
Schema: schema,
Infos: []*querypb.SegmentLoadInfo{segmentLoadInfo},
DeltaPositions: []*internalpb.MsgPosition{
DeltaPositions: []*msgpb.MsgPosition{
{
ChannelName: vDmChannel,
MsgID: rmq.SerializeRmqID(0),
@ -467,7 +467,7 @@ func TestTask_loadSegmentsTaskLoadDelta(t *testing.T) {
CollectionID: defaultCollectionID,
},
},
DeltaPositions: []*internalpb.MsgPosition{
DeltaPositions: []*msgpb.MsgPosition{
{
ChannelName: vDmChannel,
MsgID: rmq.SerializeRmqID(0),

10
internal/querynode/meta_replica.go
View File

@ -23,13 +23,17 @@ package querynode
#include "segcore/segment_c.h"
*/
import "C"
import (
"fmt"
"strconv"
"sync"
"github.com/cockroachdb/errors"
"github.com/samber/lo"
"go.uber.org/zap"
"github.com/milvus-io/milvus-proto/go-api/msgpb"
"github.com/milvus-io/milvus-proto/go-api/schemapb"
"github.com/milvus-io/milvus/internal/common"
"github.com/milvus-io/milvus/internal/log"
@ -39,8 +43,6 @@ import (
"github.com/milvus-io/milvus/internal/proto/querypb"
"github.com/milvus-io/milvus/internal/util/paramtable"
"github.com/milvus-io/milvus/internal/util/typeutil"
"github.com/samber/lo"
"go.uber.org/zap"
)
var (
@ -104,7 +106,7 @@ type ReplicaInterface interface {
// segment
// addSegment add a new segment to collectionReplica
addSegment(segmentID UniqueID, partitionID UniqueID, collectionID UniqueID, vChannelID Channel, version UniqueID, seekPosition *internalpb.MsgPosition, segType segmentType) error
addSegment(segmentID UniqueID, partitionID UniqueID, collectionID UniqueID, vChannelID Channel, version UniqueID, seekPosition *msgpb.MsgPosition, segType segmentType) error
// setSegment adds a segment to collectionReplica
setSegment(segment *Segment) error
// removeSegment removes a segment from collectionReplica
@ -566,7 +568,7 @@ func (replica *metaReplica) getSegmentIDsPrivate(partitionID UniqueID, segType s
// ----------------------------------------------------------------------------------------------------- segment
// addSegment add a new segment to collectionReplica
func (replica *metaReplica) addSegment(segmentID UniqueID, partitionID UniqueID, collectionID UniqueID, vChannelID Channel, version UniqueID, seekPosition *internalpb.MsgPosition, segType segmentType) error {
func (replica *metaReplica) addSegment(segmentID UniqueID, partitionID UniqueID, collectionID UniqueID, vChannelID Channel, version UniqueID, seekPosition *msgpb.MsgPosition, segType segmentType) error {
replica.mu.Lock()
defer replica.mu.Unlock()

18
internal/querynode/mock_test.go
View File

@ -27,11 +27,12 @@ import (
"strconv"
"github.com/cockroachdb/errors"
"github.com/golang/protobuf/proto"
ants "github.com/panjf2000/ants/v2"
"go.uber.org/zap"
"github.com/milvus-io/milvus-proto/go-api/commonpb"
"github.com/milvus-io/milvus-proto/go-api/msgpb"
"github.com/milvus-io/milvus-proto/go-api/schemapb"
"github.com/milvus-io/milvus/internal/common"
etcdkv "github.com/milvus-io/milvus/internal/kv/etcd"
@ -53,12 +54,12 @@ import (
"github.com/milvus-io/milvus/internal/util/indexcgowrapper"
"github.com/milvus-io/milvus/internal/util/paramtable"
"github.com/milvus-io/milvus/internal/util/typeutil"
"github.com/panjf2000/ants/v2"
)
// ---------- unittest util functions ----------
// common definitions
const ctxTimeInMillisecond = 500
const debugUT = false
const (
@ -96,7 +97,7 @@ const (
defaultChannelName = "default-channel"
)
var defaultSegmentStartPosition = &internalpb.MsgPosition{
var defaultSegmentStartPosition = &msgpb.MsgPosition{
ChannelName: defaultChannelName,
MsgID: []byte{},
Timestamp: 0,
@ -952,7 +953,7 @@ func genSimpleInsertMsg(schema *schemapb.CollectionSchema, numRows int) (*msgstr
return &msgstream.InsertMsg{
BaseMsg: genMsgStreamBaseMsg(),
InsertRequest: internalpb.InsertRequest{
InsertRequest: msgpb.InsertRequest{
Base: genCommonMsgBase(commonpb.MsgType_Insert, 0),
CollectionName: defaultCollectionName,
PartitionName: defaultPartitionName,
@ -964,7 +965,7 @@ func genSimpleInsertMsg(schema *schemapb.CollectionSchema, numRows int) (*msgstr
RowIDs: genSimpleRowIDField(numRows),
FieldsData: fieldsData,
NumRows: uint64(numRows),
Version: internalpb.InsertDataVersion_ColumnBased,
Version: msgpb.InsertDataVersion_ColumnBased,
},
}, nil
}
@ -1190,7 +1191,7 @@ func genMsgStreamBaseMsg() msgstream.BaseMsg {
BeginTimestamp: 0,
EndTimestamp: 0,
HashValues: []uint32{0},
MsgPosition: &internalpb.MsgPosition{
MsgPosition: &msgpb.MsgPosition{
ChannelName: "",
MsgID: []byte{},
MsgGroup: "",
@ -1210,7 +1211,7 @@ func genCommonMsgBase(msgType commonpb.MsgType, targetID int64) *commonpb.MsgBas
func genDeleteMsg(collectionID int64, pkType schemapb.DataType, numRows int) *msgstream.DeleteMsg {
return &msgstream.DeleteMsg{
BaseMsg: genMsgStreamBaseMsg(),
DeleteRequest: internalpb.DeleteRequest{
DeleteRequest: msgpb.DeleteRequest{
Base: genCommonMsgBase(commonpb.MsgType_Delete, 0),
CollectionName: defaultCollectionName,
PartitionName: defaultPartitionName,
@ -1897,14 +1898,17 @@ func (mm *mockMsgStreamFactory) NewTtMsgStream(ctx context.Context) (msgstream.M
func (mm *mockMsgStreamFactory) NewQueryMsgStream(ctx context.Context) (msgstream.MsgStream, error) {
return nil, nil
}
func (mm *mockMsgStreamFactory) NewCacheStorageChunkManager(ctx context.Context) (storage.ChunkManager, error) {
return nil, nil
}
func (mm *mockMsgStreamFactory) NewPersistentStorageChunkManager(ctx context.Context) (storage.ChunkManager, error) {
return nil, nil
}
type readAtFunc func(path string, offset int64, length int64) ([]byte, error)
type readFunc func(path string) ([]byte, error)
type mockChunkManager struct {

1
internal/querynode/plan.go
View File

@ -24,6 +24,7 @@ package querynode
#include "segcore/plan_c.h"
*/
import "C"
import (
"fmt"
"unsafe"

22
internal/querynode/segment.go
View File

@ -24,6 +24,7 @@ package querynode
#include "segcore/reduce_c.h"
*/
import "C"
import (
"bytes"
"context"
@ -33,29 +34,26 @@ import (
"sync"
"unsafe"
bloom "github.com/bits-and-blooms/bloom/v3"
"github.com/cockroachdb/errors"
"github.com/milvus-io/milvus/internal/proto/internalpb"
"github.com/milvus-io/milvus/internal/util/funcutil"
"github.com/milvus-io/milvus/internal/util/paramtable"
"github.com/milvus-io/milvus/internal/util/typeutil"
"github.com/bits-and-blooms/bloom/v3"
"github.com/milvus-io/milvus/internal/metrics"
"github.com/milvus-io/milvus/internal/util/timerecord"
"github.com/golang/protobuf/proto"
"go.uber.org/atomic"
"go.uber.org/zap"
"github.com/milvus-io/milvus-proto/go-api/commonpb"
"github.com/milvus-io/milvus-proto/go-api/msgpb"
"github.com/milvus-io/milvus-proto/go-api/schemapb"
"github.com/milvus-io/milvus/internal/common"
"github.com/milvus-io/milvus/internal/log"
"github.com/milvus-io/milvus/internal/metrics"
"github.com/milvus-io/milvus/internal/proto/datapb"
"github.com/milvus-io/milvus/internal/proto/querypb"
"github.com/milvus-io/milvus/internal/proto/segcorepb"
"github.com/milvus-io/milvus/internal/storage"
"github.com/milvus-io/milvus/internal/util/funcutil"
"github.com/milvus-io/milvus/internal/util/paramtable"
"github.com/milvus-io/milvus/internal/util/timerecord"
"github.com/milvus-io/milvus/internal/util/typeutil"
)
type segmentType = commonpb.SegmentState
@ -84,7 +82,7 @@ type Segment struct {
partitionID UniqueID
collectionID UniqueID
version UniqueID
startPosition *internalpb.MsgPosition // for growing segment release
startPosition *msgpb.MsgPosition // for growing segment release
vChannelID Channel
lastMemSize int64
@ -162,7 +160,7 @@ func newSegment(collection *Collection,
vChannelID Channel,
segType segmentType,
version UniqueID,
startPosition *internalpb.MsgPosition) (*Segment, error) {
startPosition *msgpb.MsgPosition) (*Segment, error) {
/*
CSegmentInterface
NewSegment(CCollection collection, uint64_t segment_id, SegmentType seg_type);

13
internal/querynode/segment_loader.go
View File

@ -27,12 +27,13 @@ import (
"time"
"github.com/cockroachdb/errors"
ants "github.com/panjf2000/ants/v2"
"go.uber.org/multierr"
"go.uber.org/zap"
"golang.org/x/sync/errgroup"
"github.com/milvus-io/milvus-proto/go-api/commonpb"
"github.com/milvus-io/milvus-proto/go-api/msgpb"
"github.com/milvus-io/milvus-proto/go-api/schemapb"
"github.com/milvus-io/milvus/internal/common"
etcdkv "github.com/milvus-io/milvus/internal/kv/etcd"
@ -41,7 +42,6 @@ import (
"github.com/milvus-io/milvus/internal/mq/msgstream"
"github.com/milvus-io/milvus/internal/mq/msgstream/mqwrapper"
"github.com/milvus-io/milvus/internal/proto/datapb"
"github.com/milvus-io/milvus/internal/proto/internalpb"
"github.com/milvus-io/milvus/internal/proto/querypb"
"github.com/milvus-io/milvus/internal/storage"
"github.com/milvus-io/milvus/internal/types"
@ -53,7 +53,6 @@ import (
"github.com/milvus-io/milvus/internal/util/timerecord"
"github.com/milvus-io/milvus/internal/util/tsoutil"
"github.com/milvus-io/milvus/internal/util/typeutil"
"github.com/panjf2000/ants/v2"
)
const (
@ -594,13 +593,13 @@ func (loader *segmentLoader) loadGrowingSegments(segment *Segment,
return err
}
tmpInsertMsg := &msgstream.InsertMsg{
InsertRequest: internalpb.InsertRequest{
InsertRequest: msgpb.InsertRequest{
CollectionID: segment.collectionID,
Timestamps: timestamps,
RowIDs: ids,
NumRows: uint64(numRows),
FieldsData: insertRecord.FieldsData,
Version: internalpb.InsertDataVersion_ColumnBased,
Version: msgpb.InsertDataVersion_ColumnBased,
},
}
pks, err := getPrimaryKeys(tmpInsertMsg, loader.metaReplica)
@ -703,7 +702,7 @@ func (loader *segmentLoader) loadDeltaLogs(ctx context.Context, segment *Segment
return nil
}
func (loader *segmentLoader) FromDmlCPLoadDelete(ctx context.Context, collectionID int64, position *internalpb.MsgPosition,
func (loader *segmentLoader) FromDmlCPLoadDelete(ctx context.Context, collectionID int64, position *msgpb.MsgPosition,
segmentIDs []int64) error {
startTs := time.Now()
stream, err := loader.factory.NewTtMsgStream(ctx)
@ -743,7 +742,7 @@ func (loader *segmentLoader) FromDmlCPLoadDelete(ctx context.Context, collection
}
metrics.QueryNodeNumConsumers.WithLabelValues(fmt.Sprint(paramtable.GetNodeID())).Inc()
err = stream.Seek([]*internalpb.MsgPosition{position})
err = stream.Seek([]*msgpb.MsgPosition{position})
if err != nil {
return err
}

9
internal/querynode/segment_loader_test.go
View File

@ -24,20 +24,19 @@ import (
"time"
"github.com/cockroachdb/errors"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/mock"
"github.com/stretchr/testify/require"
"go.uber.org/atomic"
"github.com/milvus-io/milvus-proto/go-api/commonpb"
"github.com/milvus-io/milvus-proto/go-api/msgpb"
"github.com/milvus-io/milvus-proto/go-api/schemapb"
"github.com/milvus-io/milvus/internal/common"
"github.com/milvus-io/milvus/internal/log"
"github.com/milvus-io/milvus/internal/mocks"
"github.com/milvus-io/milvus/internal/mq/msgstream"
"github.com/milvus-io/milvus/internal/mq/msgstream/mqwrapper"
"github.com/milvus-io/milvus/internal/proto/internalpb"
"github.com/milvus-io/milvus/internal/proto/querypb"
"github.com/milvus-io/milvus/internal/storage"
"github.com/milvus-io/milvus/internal/util/funcutil"
@ -886,8 +885,8 @@ func testConsumingDeltaMsg(ctx context.Context, t *testing.T, position *msgstrea
deleteMsg1 := genDeleteMsg(defaultCollectionID+1, schemapb.DataType_Int64, defaultDelLength)
deleteMsg2 := genDeleteMsg(defaultCollectionID, schemapb.DataType_Int64, defaultDelLength)
msgChan <- &msgstream.MsgPack{Msgs: []msgstream.TsMsg{deleteMsg1, deleteMsg2},
StartPositions: []*internalpb.MsgPosition{genMsgStreamBaseMsg().MsgPosition},
EndPositions: []*internalpb.MsgPosition{genMsgStreamBaseMsg().MsgPosition}}
StartPositions: []*msgpb.MsgPosition{genMsgStreamBaseMsg().MsgPosition},
EndPositions: []*msgpb.MsgPosition{genMsgStreamBaseMsg().MsgPosition}}
}
if closedStream {
close(msgChan)
@ -949,7 +948,7 @@ func (ms *LoadDeleteMsgStream) Chan() <-chan *msgstream.MsgPack {
return args.Get(0).(chan *msgstream.MsgPack)
}
func (ms *LoadDeleteMsgStream) Seek(offset []*internalpb.MsgPosition) error {
func (ms *LoadDeleteMsgStream) Seek(offset []*msgpb.MsgPosition) error {
args := ms.Called()
if args.Get(0) == nil {
return nil

10
internal/querynode/task_test.go
View File

@ -24,9 +24,9 @@ import (
"github.com/stretchr/testify/require"
"github.com/milvus-io/milvus-proto/go-api/commonpb"
"github.com/milvus-io/milvus-proto/go-api/msgpb"
"github.com/milvus-io/milvus/internal/mq/msgstream"
"github.com/milvus-io/milvus/internal/proto/datapb"
"github.com/milvus-io/milvus/internal/proto/internalpb"
"github.com/milvus-io/milvus/internal/proto/querypb"
)
@ -184,7 +184,7 @@ func TestTask_watchDmChannelsTask(t *testing.T) {
UnflushedSegmentIds: []int64{100},
FlushedSegmentIds: []int64{101},
DroppedSegmentIds: []int64{102},
SeekPosition: &internalpb.MsgPosition{
SeekPosition: &msgpb.MsgPosition{
ChannelName: defaultDMLChannel,
MsgID: []byte{235, 50, 164, 248, 255, 255, 255, 255},
Timestamp: Timestamp(999),
@ -194,21 +194,21 @@ func TestTask_watchDmChannelsTask(t *testing.T) {
task.req.SegmentInfos = map[int64]*datapb.SegmentInfo{
100: {
ID: 100,
DmlPosition: &internalpb.MsgPosition{
DmlPosition: &msgpb.MsgPosition{
ChannelName: defaultDMLChannel,
Timestamp: Timestamp(1000),
},
},
101: {
ID: 101,
DmlPosition: &internalpb.MsgPosition{
DmlPosition: &msgpb.MsgPosition{
ChannelName: defaultDMLChannel,
Timestamp: Timestamp(1001),
},
},
102: {
ID: 102,
DmlPosition: &internalpb.MsgPosition{
DmlPosition: &msgpb.MsgPosition{
ChannelName: defaultDMLChannel,
Timestamp: Timestamp(1002),
},

7
internal/querynode/watch_dm_channels_task.go
View File

@ -22,13 +22,12 @@ import (
"math"
"github.com/cockroachdb/errors"
"go.uber.org/zap"
"github.com/milvus-io/milvus-proto/go-api/commonpb"
"github.com/milvus-io/milvus-proto/go-api/msgpb"
"github.com/milvus-io/milvus/internal/log"
"github.com/milvus-io/milvus/internal/proto/datapb"
"github.com/milvus-io/milvus/internal/proto/internalpb"
queryPb "github.com/milvus-io/milvus/internal/proto/querypb"
"github.com/milvus-io/milvus/internal/util/commonpbutil"
"github.com/milvus-io/milvus/internal/util/funcutil"
@ -253,7 +252,7 @@ func (w *watchDmChannelsTask) initFlowGraph(collectionID UniqueID, vChannels []C
consumeSubName := funcutil.GenChannelSubName(Params.CommonCfg.QueryNodeSubName.GetValue(), collectionID, paramtable.GetNodeID())
// group channels by to seeking
channel2SeekPosition := make(map[string]*internalpb.MsgPosition)
channel2SeekPosition := make(map[string]*msgpb.MsgPosition)
for _, info := range w.req.Infos {
if info.SeekPosition != nil && len(info.SeekPosition.MsgID) != 0 {
info.SeekPosition.MsgGroup = consumeSubName
@ -315,7 +314,7 @@ func (w *watchDmChannelsTask) initFlowGraph(collectionID UniqueID, vChannels []C
ID: droppedSegmentID,
CollectionID: collectionID,
InsertChannel: info.GetChannelName(),
DmlPosition: &internalpb.MsgPosition{
DmlPosition: &msgpb.MsgPosition{
ChannelName: info.GetChannelName(),
Timestamp: math.MaxUint64,
},

32
internal/rootcoord/create_collection_task.go
View File

@ -21,28 +21,20 @@ import (
"fmt"
"github.com/cockroachdb/errors"
ms "github.com/milvus-io/milvus/internal/mq/msgstream"
"github.com/milvus-io/milvus/internal/proto/internalpb"
"github.com/milvus-io/milvus/internal/util/commonpbutil"
"github.com/milvus-io/milvus/internal/util/funcutil"
"github.com/milvus-io/milvus-proto/go-api/commonpb"
pb "github.com/milvus-io/milvus/internal/proto/etcdpb"
"github.com/milvus-io/milvus/internal/metastore/model"
"github.com/milvus-io/milvus/internal/log"
"github.com/milvus-io/milvus/internal/util/typeutil"
"github.com/golang/protobuf/proto"
"go.uber.org/zap"
"github.com/golang/protobuf/proto"
"github.com/milvus-io/milvus-proto/go-api/schemapb"
"github.com/milvus-io/milvus-proto/go-api/commonpb"
"github.com/milvus-io/milvus-proto/go-api/milvuspb"
"github.com/milvus-io/milvus-proto/go-api/msgpb"
"github.com/milvus-io/milvus-proto/go-api/schemapb"
"github.com/milvus-io/milvus/internal/log"
"github.com/milvus-io/milvus/internal/metastore/model"
ms "github.com/milvus-io/milvus/internal/mq/msgstream"
pb "github.com/milvus-io/milvus/internal/proto/etcdpb"
"github.com/milvus-io/milvus/internal/util/commonpbutil"
"github.com/milvus-io/milvus/internal/util/funcutil"
"github.com/milvus-io/milvus/internal/util/typeutil"
)
type collectionChannels struct {
@ -216,7 +208,7 @@ func (t *createCollectionTask) genCreateCollectionMsg(ctx context.Context) *ms.M
}
msg := &ms.CreateCollectionMsg{
BaseMsg: baseMsg,
CreateCollectionRequest: internalpb.CreateCollectionRequest{
CreateCollectionRequest: msgpb.CreateCollectionRequest{
Base: commonpbutil.NewMsgBase(
commonpbutil.WithMsgType(commonpb.MsgType_CreateCollection),
commonpbutil.WithTimeStamp(ts),

11
internal/rootcoord/garbage_collector.go
View File

@ -20,11 +20,10 @@ import (
"context"
"github.com/milvus-io/milvus-proto/go-api/commonpb"
ms "github.com/milvus-io/milvus/internal/mq/msgstream"
"github.com/milvus-io/milvus/internal/proto/internalpb"
"github.com/milvus-io/milvus/internal/util/commonpbutil"
"github.com/milvus-io/milvus-proto/go-api/msgpb"
"github.com/milvus-io/milvus/internal/metastore/model"
ms "github.com/milvus-io/milvus/internal/mq/msgstream"
"github.com/milvus-io/milvus/internal/util/commonpbutil"
)
//go:generate mockery --name=GarbageCollector --outpkg=mockrootcoord
@ -163,7 +162,7 @@ func (c *bgGarbageCollector) notifyCollectionGc(ctx context.Context, coll *model
}
msg := &ms.DropCollectionMsg{
BaseMsg: baseMsg,
DropCollectionRequest: internalpb.DropCollectionRequest{
DropCollectionRequest: msgpb.DropCollectionRequest{
Base: commonpbutil.NewMsgBase(
commonpbutil.WithMsgType(commonpb.MsgType_DropCollection),
commonpbutil.WithTimeStamp(ts),
@ -196,7 +195,7 @@ func (c *bgGarbageCollector) notifyPartitionGc(ctx context.Context, pChannels []
}
msg := &ms.DropPartitionMsg{
BaseMsg: baseMsg,
DropPartitionRequest: internalpb.DropPartitionRequest{
DropPartitionRequest: msgpb.DropPartitionRequest{
Base: commonpbutil.NewMsgBase(
commonpbutil.WithMsgType(commonpb.MsgType_DropPartition),
commonpbutil.WithTimeStamp(ts),

6
internal/rootcoord/timeticksync.go
View File

@ -22,7 +22,10 @@ import (
"sync"
"time"
"go.uber.org/zap"
"github.com/milvus-io/milvus-proto/go-api/commonpb"
"github.com/milvus-io/milvus-proto/go-api/msgpb"
"github.com/milvus-io/milvus/internal/log"
"github.com/milvus-io/milvus/internal/metrics"
"github.com/milvus-io/milvus/internal/mq/msgstream"
@ -32,7 +35,6 @@ import (
"github.com/milvus-io/milvus/internal/util/timerecord"
"github.com/milvus-io/milvus/internal/util/tsoutil"
"github.com/milvus-io/milvus/internal/util/typeutil"
"go.uber.org/zap"
)
var (
@ -294,7 +296,7 @@ func (t *timetickSync) sendTimeTickToChannel(chanNames []string, ts typeutil.Tim
EndTimestamp: ts,
HashValues: []uint32{0},
}
timeTickResult := internalpb.TimeTickMsg{
timeTickResult := msgpb.TimeTickMsg{
Base: commonpbutil.NewMsgBase(
commonpbutil.WithMsgType(commonpb.MsgType_TimeTick),
commonpbutil.WithMsgID(0),

13
internal/storage/print_binlog.go
View File

@ -22,12 +22,11 @@ import (
"os"
"github.com/cockroachdb/errors"
"github.com/golang/protobuf/proto"
"golang.org/x/exp/mmap"
"github.com/golang/protobuf/proto"
"github.com/milvus-io/milvus-proto/go-api/msgpb"
"github.com/milvus-io/milvus-proto/go-api/schemapb"
"github.com/milvus-io/milvus/internal/proto/internalpb"
"github.com/milvus-io/milvus/internal/util/tsoutil"
)
@ -352,25 +351,25 @@ func printDDLPayloadValues(eventType EventTypeCode, colType schemapb.DataType, r
valBytes := []byte(val[i])
switch eventType {
case CreateCollectionEventType:
var req internalpb.CreateCollectionRequest
var req msgpb.CreateCollectionRequest
if err := proto.Unmarshal(valBytes, &req); err != nil {
return err
}
fmt.Printf("\t\t%d : create collection: %v\n", i, req)
case DropCollectionEventType:
var req internalpb.DropCollectionRequest
var req msgpb.DropCollectionRequest
if err := proto.Unmarshal(valBytes, &req); err != nil {
return err
}
fmt.Printf("\t\t%d : drop collection: %v\n", i, req)
case CreatePartitionEventType:
var req internalpb.CreatePartitionRequest
var req msgpb.CreatePartitionRequest
if err := proto.Unmarshal(valBytes, &req); err != nil {
return err
}
fmt.Printf("\t\t%d : create partition: %v\n", i, req)
case DropPartitionEventType:
var req internalpb.DropPartitionRequest
var req msgpb.DropPartitionRequest
if err := proto.Unmarshal(valBytes, &req); err != nil {
return err
}

18
internal/storage/print_binlog_test.go
View File

@ -23,16 +23,16 @@ import (
"testing"
"time"
"github.com/milvus-io/milvus/internal/util/funcutil"
"github.com/milvus-io/milvus/internal/util/uniquegenerator"
"github.com/golang/protobuf/proto"
"github.com/stretchr/testify/assert"
"github.com/milvus-io/milvus-proto/go-api/commonpb"
"github.com/milvus-io/milvus-proto/go-api/msgpb"
"github.com/milvus-io/milvus-proto/go-api/schemapb"
"github.com/milvus-io/milvus/internal/proto/etcdpb"
"github.com/milvus-io/milvus/internal/proto/internalpb"
"github.com/milvus-io/milvus/internal/util/funcutil"
"github.com/milvus-io/milvus/internal/util/tsoutil"
"github.com/stretchr/testify/assert"
"github.com/milvus-io/milvus/internal/util/uniquegenerator"
)
func TestPrintBinlogFilesInt64(t *testing.T) {
@ -316,7 +316,7 @@ func TestPrintDDFiles(t *testing.T) {
partitionID := int64(1)
collName := "test"
partitionName := "test"
createCollReq := internalpb.CreateCollectionRequest{
createCollReq := msgpb.CreateCollectionRequest{
Base: &commonpb.MsgBase{
MsgType: commonpb.MsgType_CreateCollection,
MsgID: 1,
@ -332,7 +332,7 @@ func TestPrintDDFiles(t *testing.T) {
createCollString, err := proto.Marshal(&createCollReq)
assert.Nil(t, err)
dropCollReq := internalpb.DropCollectionRequest{
dropCollReq := msgpb.DropCollectionRequest{
Base: &commonpb.MsgBase{
MsgType: commonpb.MsgType_DropCollection,
MsgID: 2,
@ -347,7 +347,7 @@ func TestPrintDDFiles(t *testing.T) {
dropCollString, err := proto.Marshal(&dropCollReq)
assert.Nil(t, err)
createPartitionReq := internalpb.CreatePartitionRequest{
createPartitionReq := msgpb.CreatePartitionRequest{
Base: &commonpb.MsgBase{
MsgType: commonpb.MsgType_CreatePartition,
MsgID: 3,
@ -364,7 +364,7 @@ func TestPrintDDFiles(t *testing.T) {
createPartitionString, err := proto.Marshal(&createPartitionReq)
assert.Nil(t, err)
dropPartitionReq := internalpb.DropPartitionRequest{
dropPartitionReq := msgpb.DropPartitionRequest{
Base: &commonpb.MsgBase{
MsgType: commonpb.MsgType_DropPartition,
MsgID: 4,

Some files were not shown because too many files have changed in this diff Show More