milvus-io
/
milvus
mirror of https://github.com/milvus-io/milvus.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Support to replicate the mq message (#27240) 

Signed-off-by: SimFG <bang.fu@zilliz.com>
pull/27834/head
SimFG 2023-10-20 14:26:09 +08:00 committed by GitHub
parent 5247ea3fd1
commit 9b0ecbdca7
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
62 changed files with 3439 additions and 469 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
configs/milvus.yaml
View File

@ -464,6 +464,7 @@ common:
rootCoordTimeTick: rootcoord-timetick
rootCoordStatistics: rootcoord-statistics
rootCoordDml: rootcoord-dml
replicateMsg: replicate-msg
rootCoordDelta: rootcoord-delta
search: search
searchResult: searchResult
@ -528,6 +529,7 @@ common:
threshold:
info: 500 # minimum milliseconds for printing durations in info level
warn: 1000 # minimum milliseconds for printing durations in warn level
ttMsgEnabled: true # Whether the instance disable sending ts messages
# QuotaConfig, configurations of Milvus quota and limits.
# By default, we enable:

5
go.mod
View File

@ -24,7 +24,8 @@ require (
github.com/grpc-ecosystem/go-grpc-middleware v1.3.0
github.com/klauspost/compress v1.16.5
github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d
github.com/milvus-io/milvus-proto/go-api/v2 v2.3.2-0.20231017024957-5e5a27fd4875
github.com/milvus-io/milvus-proto/go-api/v2 v2.3.2-0.20231019101159-a0a6f5e7eff8
github.com/milvus-io/milvus/pkg v0.0.1
github.com/minio/minio-go/v7 v7.0.56
github.com/prometheus/client_golang v1.14.0
github.com/prometheus/client_model v0.3.0
@ -58,8 +59,6 @@ require (
stathat.com/c/consistent v1.0.0
)
require github.com/milvus-io/milvus/pkg v0.0.0-20230607023836-1593278f9d9c
require (
cloud.google.com/go/compute v1.19.0 // indirect
cloud.google.com/go/compute/metadata v0.2.3 // indirect

4
go.sum
View File

@ -582,8 +582,8 @@ github.com/microcosm-cc/bluemonday v1.0.2/go.mod h1:iVP4YcDBq+n/5fb23BhYFvIMq/le
github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg=
github.com/milvus-io/gorocksdb v0.0.0-20220624081344-8c5f4212846b h1:TfeY0NxYxZzUfIfYe5qYDBzt4ZYRqzUjTR6CvUzjat8=
github.com/milvus-io/gorocksdb v0.0.0-20220624081344-8c5f4212846b/go.mod h1:iwW+9cWfIzzDseEBCCeDSN5SD16Tidvy8cwQ7ZY8Qj4=
github.com/milvus-io/milvus-proto/go-api/v2 v2.3.2-0.20231017024957-5e5a27fd4875 h1:7OPJn0sOeueXNnreWup0GR7ZlXEURpcKklzplXM9kDg=
github.com/milvus-io/milvus-proto/go-api/v2 v2.3.2-0.20231017024957-5e5a27fd4875/go.mod h1:1OIl0v5PQeNxIJhCvY+K55CBUOYDZevw9g9380u1Wek=
github.com/milvus-io/milvus-proto/go-api/v2 v2.3.2-0.20231019101159-a0a6f5e7eff8 h1:GoGErEOhdWjwSfQilXso3eINqb11yEBDLtoBMNdlve0=
github.com/milvus-io/milvus-proto/go-api/v2 v2.3.2-0.20231019101159-a0a6f5e7eff8/go.mod h1:1OIl0v5PQeNxIJhCvY+K55CBUOYDZevw9g9380u1Wek=
github.com/milvus-io/pulsar-client-go v0.6.10 h1:eqpJjU+/QX0iIhEo3nhOqMNXL+TyInAs1IAHZCrCM/A=
github.com/milvus-io/pulsar-client-go v0.6.10/go.mod h1:lQqCkgwDF8YFYjKA+zOheTk1tev2B+bKj5j7+nm8M1w=
github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8 h1:AMFGa4R4MiIpspGNG7Z948v4n35fFGB3RR3G/ry4FWs=

698
internal/core/src/pb/common.pb.cc
View File

@ -121,20 +121,9 @@ struct AddressDefaultTypeInternal {
};
};
PROTOBUF_ATTRIBUTE_NO_DESTROY PROTOBUF_CONSTINIT PROTOBUF_ATTRIBUTE_INIT_PRIORITY1 AddressDefaultTypeInternal _Address_default_instance_;
PROTOBUF_CONSTEXPR MsgBase_PropertiesEntry_DoNotUse::MsgBase_PropertiesEntry_DoNotUse(
::_pbi::ConstantInitialized) {}
struct MsgBase_PropertiesEntry_DoNotUseDefaultTypeInternal {
PROTOBUF_CONSTEXPR MsgBase_PropertiesEntry_DoNotUseDefaultTypeInternal()
: _instance(::_pbi::ConstantInitialized{}) {}
~MsgBase_PropertiesEntry_DoNotUseDefaultTypeInternal() {}
union {
MsgBase_PropertiesEntry_DoNotUse _instance;
};
};
PROTOBUF_ATTRIBUTE_NO_DESTROY PROTOBUF_CONSTINIT PROTOBUF_ATTRIBUTE_INIT_PRIORITY1 MsgBase_PropertiesEntry_DoNotUseDefaultTypeInternal _MsgBase_PropertiesEntry_DoNotUse_default_instance_;
PROTOBUF_CONSTEXPR MsgBase::MsgBase(
::_pbi::ConstantInitialized): _impl_{
/*decltype(_impl_.properties_)*/{::_pbi::ConstantInitialized()}
/*decltype(_impl_.replicate_)*/nullptr
, /*decltype(_impl_.msgid_)*/int64_t{0}
, /*decltype(_impl_.timestamp_)*/uint64_t{0u}
, /*decltype(_impl_.sourceid_)*/int64_t{0}
@ -150,6 +139,19 @@ struct MsgBaseDefaultTypeInternal {
};
};
PROTOBUF_ATTRIBUTE_NO_DESTROY PROTOBUF_CONSTINIT PROTOBUF_ATTRIBUTE_INIT_PRIORITY1 MsgBaseDefaultTypeInternal _MsgBase_default_instance_;
PROTOBUF_CONSTEXPR ReplicateEntity::ReplicateEntity(
::_pbi::ConstantInitialized): _impl_{
/*decltype(_impl_.isreplicate_)*/false
, /*decltype(_impl_._cached_size_)*/{}} {}
struct ReplicateEntityDefaultTypeInternal {
PROTOBUF_CONSTEXPR ReplicateEntityDefaultTypeInternal()
: _instance(::_pbi::ConstantInitialized{}) {}
~ReplicateEntityDefaultTypeInternal() {}
union {
ReplicateEntity _instance;
};
};
PROTOBUF_ATTRIBUTE_NO_DESTROY PROTOBUF_CONSTINIT PROTOBUF_ATTRIBUTE_INIT_PRIORITY1 ReplicateEntityDefaultTypeInternal _ReplicateEntity_default_instance_;
PROTOBUF_CONSTEXPR MsgHeader::MsgHeader(
::_pbi::ConstantInitialized): _impl_{
/*decltype(_impl_.base_)*/nullptr
@ -329,16 +331,6 @@ const uint32_t TableStruct_common_2eproto::offsets[] PROTOBUF_SECTION_VARIABLE(p
~0u, // no _inlined_string_donated_
PROTOBUF_FIELD_OFFSET(::milvus::proto::common::Address, _impl_.ip_),
PROTOBUF_FIELD_OFFSET(::milvus::proto::common::Address, _impl_.port_),
PROTOBUF_FIELD_OFFSET(::milvus::proto::common::MsgBase_PropertiesEntry_DoNotUse, _has_bits_),
PROTOBUF_FIELD_OFFSET(::milvus::proto::common::MsgBase_PropertiesEntry_DoNotUse, _internal_metadata_),
~0u, // no _extensions_
~0u, // no _oneof_case_
~0u, // no _weak_field_map_
~0u, // no _inlined_string_donated_
PROTOBUF_FIELD_OFFSET(::milvus::proto::common::MsgBase_PropertiesEntry_DoNotUse, key_),
PROTOBUF_FIELD_OFFSET(::milvus::proto::common::MsgBase_PropertiesEntry_DoNotUse, value_),
0,
1,
~0u, // no _has_bits_
PROTOBUF_FIELD_OFFSET(::milvus::proto::common::MsgBase, _internal_metadata_),
~0u, // no _extensions_
@ -350,7 +342,14 @@ const uint32_t TableStruct_common_2eproto::offsets[] PROTOBUF_SECTION_VARIABLE(p
PROTOBUF_FIELD_OFFSET(::milvus::proto::common::MsgBase, _impl_.timestamp_),
PROTOBUF_FIELD_OFFSET(::milvus::proto::common::MsgBase, _impl_.sourceid_),
PROTOBUF_FIELD_OFFSET(::milvus::proto::common::MsgBase, _impl_.targetid_),
PROTOBUF_FIELD_OFFSET(::milvus::proto::common::MsgBase, _impl_.properties_),
PROTOBUF_FIELD_OFFSET(::milvus::proto::common::MsgBase, _impl_.replicate_),
~0u, // no _has_bits_
PROTOBUF_FIELD_OFFSET(::milvus::proto::common::ReplicateEntity, _internal_metadata_),
~0u, // no _extensions_
~0u, // no _oneof_case_
~0u, // no _weak_field_map_
~0u, // no _inlined_string_donated_
PROTOBUF_FIELD_OFFSET(::milvus::proto::common::ReplicateEntity, _impl_.isreplicate_),
~0u, // no _has_bits_
PROTOBUF_FIELD_OFFSET(::milvus::proto::common::MsgHeader, _internal_metadata_),
~0u, // no _extensions_
@ -437,16 +436,16 @@ static const ::_pbi::MigrationSchema schemas[] PROTOBUF_SECTION_VARIABLE(protode
{ 32, -1, -1, sizeof(::milvus::proto::common::PlaceholderValue)},
{ 41, -1, -1, sizeof(::milvus::proto::common::PlaceholderGroup)},
{ 48, -1, -1, sizeof(::milvus::proto::common::Address)},
{ 56, 64, -1, sizeof(::milvus::proto::common::MsgBase_PropertiesEntry_DoNotUse)},
{ 66, -1, -1, sizeof(::milvus::proto::common::MsgBase)},
{ 78, -1, -1, sizeof(::milvus::proto::common::MsgHeader)},
{ 85, -1, -1, sizeof(::milvus::proto::common::DMLMsgHeader)},
{ 93, -1, -1, sizeof(::milvus::proto::common::PrivilegeExt)},
{ 103, -1, -1, sizeof(::milvus::proto::common::SegmentStats)},
{ 111, 119, -1, sizeof(::milvus::proto::common::ClientInfo_ReservedEntry_DoNotUse)},
{ 121, -1, -1, sizeof(::milvus::proto::common::ClientInfo)},
{ 133, 141, -1, sizeof(::milvus::proto::common::ServerInfo_ReservedEntry_DoNotUse)},
{ 143, -1, -1, sizeof(::milvus::proto::common::ServerInfo)},
{ 56, -1, -1, sizeof(::milvus::proto::common::MsgBase)},
{ 68, -1, -1, sizeof(::milvus::proto::common::ReplicateEntity)},
{ 75, -1, -1, sizeof(::milvus::proto::common::MsgHeader)},
{ 82, -1, -1, sizeof(::milvus::proto::common::DMLMsgHeader)},
{ 90, -1, -1, sizeof(::milvus::proto::common::PrivilegeExt)},
{ 100, -1, -1, sizeof(::milvus::proto::common::SegmentStats)},
{ 108, 116, -1, sizeof(::milvus::proto::common::ClientInfo_ReservedEntry_DoNotUse)},
{ 118, -1, -1, sizeof(::milvus::proto::common::ClientInfo)},
{ 130, 138, -1, sizeof(::milvus::proto::common::ServerInfo_ReservedEntry_DoNotUse)},
{ 140, -1, -1, sizeof(::milvus::proto::common::ServerInfo)},
};
static const ::_pb::Message* const file_default_instances[] = {
@ -457,8 +456,8 @@ static const ::_pb::Message* const file_default_instances[] = {
&::milvus::proto::common::_PlaceholderValue_default_instance_._instance,
&::milvus::proto::common::_PlaceholderGroup_default_instance_._instance,
&::milvus::proto::common::_Address_default_instance_._instance,
&::milvus::proto::common::_MsgBase_PropertiesEntry_DoNotUse_default_instance_._instance,
&::milvus::proto::common::_MsgBase_default_instance_._instance,
&::milvus::proto::common::_ReplicateEntity_default_instance_._instance,
&::milvus::proto::common::_MsgHeader_default_instance_._instance,
&::milvus::proto::common::_DMLMsgHeader_default_instance_._instance,
&::milvus::proto::common::_PrivilegeExt_default_instance_._instance,
@ -482,182 +481,182 @@ const char descriptor_table_protodef_common_2eproto[] PROTOBUF_SECTION_VARIABLE(
"alues\030\003 \003(\014\"O\n\020PlaceholderGroup\022;\n\014place"
"holders\030\001 \003(\0132%.milvus.proto.common.Plac"
"eholderValue\"#\n\007Address\022\n\n\002ip\030\001 \001(\t\022\014\n\004p"
"ort\030\002 \001(\003\"\364\001\n\007MsgBase\022.\n\010msg_type\030\001 \001(\0162"
"ort\030\002 \001(\003\"\270\001\n\007MsgBase\022.\n\010msg_type\030\001 \001(\0162"
"\034.milvus.proto.common.MsgType\022\r\n\005msgID\030\002"
" \001(\003\022\021\n\ttimestamp\030\003 \001(\004\022\020\n\010sourceID\030\004 \001("
"\003\022\020\n\010targetID\030\005 \001(\003\022@\n\nproperties\030\006 \003(\0132"
",.milvus.proto.common.MsgBase.Properties"
"Entry\0321\n\017PropertiesEntry\022\013\n\003key\030\001 \001(\t\022\r\n"
"\005value\030\002 \001(\t:\0028\001\"7\n\tMsgHeader\022*\n\004base\030\001 "
"\001(\0132\034.milvus.proto.common.MsgBase\"M\n\014DML"
"\003\022\020\n\010targetID\030\005 \001(\003\0227\n\treplicate\030\006 \001(\0132$"
".milvus.proto.common.ReplicateEntity\"&\n\017"
"ReplicateEntity\022\023\n\013isReplicate\030\001 \001(\010\"7\n\t"
"MsgHeader\022*\n\004base\030\001 \001(\0132\034.milvus.proto.c"
"ommon.MsgBase\022\021\n\tshardName\030\002 \001(\t\"\273\001\n\014Pri"
"vilegeExt\0224\n\013object_type\030\001 \001(\0162\037.milvus."
"proto.common.ObjectType\022>\n\020object_privil"
"ege\030\002 \001(\0162$.milvus.proto.common.ObjectPr"
"ivilege\022\031\n\021object_name_index\030\003 \001(\005\022\032\n\022ob"
"ject_name_indexs\030\004 \001(\005\"2\n\014SegmentStats\022\021"
"\n\tSegmentID\030\001 \001(\003\022\017\n\007NumRows\030\002 \001(\003\"\325\001\n\nC"
"lientInfo\022\020\n\010sdk_type\030\001 \001(\t\022\023\n\013sdk_versi"
"on\030\002 \001(\t\022\022\n\nlocal_time\030\003 \001(\t\022\014\n\004user\030\004 \001"
"(\t\022\014\n\004host\030\005 \001(\t\022\?\n\010reserved\030\006 \003(\0132-.mil"
"vus.proto.common.ClientInfo.ReservedEntr"
"y\032/\n\rReservedEntry\022\013\n\003key\030\001 \001(\t\022\r\n\005value"
"\030\002 \001(\t:\0028\001\"\343\001\n\nServerInfo\022\022\n\nbuild_tags\030"
"\001 \001(\t\022\022\n\nbuild_time\030\002 \001(\t\022\022\n\ngit_commit\030"
"\003 \001(\t\022\022\n\ngo_version\030\004 \001(\t\022\023\n\013deploy_mode"
"\030\005 \001(\t\022\?\n\010reserved\030\006 \003(\0132-.milvus.proto."
"common.ServerInfo.ReservedEntry\032/\n\rReser"
"vedEntry\022\013\n\003key\030\001 \001(\t\022\r\n\005value\030\002 \001(\t:\0028\001"
"*\303\n\n\tErrorCode\022\013\n\007Success\020\000\022\023\n\017Unexpecte"
"dError\020\001\022\021\n\rConnectFailed\020\002\022\024\n\020Permissio"
"nDenied\020\003\022\027\n\023CollectionNotExists\020\004\022\023\n\017Il"
"legalArgument\020\005\022\024\n\020IllegalDimension\020\007\022\024\n"
"\020IllegalIndexType\020\010\022\031\n\025IllegalCollection"
"Name\020\t\022\017\n\013IllegalTOPK\020\n\022\024\n\020IllegalRowRec"
"ord\020\013\022\023\n\017IllegalVectorID\020\014\022\027\n\023IllegalSea"
"rchResult\020\r\022\020\n\014FileNotFound\020\016\022\016\n\nMetaFai"
"led\020\017\022\017\n\013CacheFailed\020\020\022\026\n\022CannotCreateFo"
"lder\020\021\022\024\n\020CannotCreateFile\020\022\022\026\n\022CannotDe"
"leteFolder\020\023\022\024\n\020CannotDeleteFile\020\024\022\023\n\017Bu"
"ildIndexError\020\025\022\020\n\014IllegalNLIST\020\026\022\025\n\021Ill"
"egalMetricType\020\027\022\017\n\013OutOfMemory\020\030\022\021\n\rInd"
"exNotExist\020\031\022\023\n\017EmptyCollection\020\032\022\033\n\027Upd"
"ateImportTaskFailure\020\033\022\032\n\026CollectionName"
"NotFound\020\034\022\033\n\027CreateCredentialFailure\020\035\022"
"\033\n\027UpdateCredentialFailure\020\036\022\033\n\027DeleteCr"
"edentialFailure\020\037\022\030\n\024GetCredentialFailur"
"e\020 \022\030\n\024ListCredUsersFailure\020!\022\022\n\016GetUser"
"Failure\020\"\022\025\n\021CreateRoleFailure\020#\022\023\n\017Drop"
"RoleFailure\020$\022\032\n\026OperateUserRoleFailure\020"
"%\022\025\n\021SelectRoleFailure\020&\022\025\n\021SelectUserFa"
"ilure\020\'\022\031\n\025SelectResourceFailure\020(\022\033\n\027Op"
"eratePrivilegeFailure\020)\022\026\n\022SelectGrantFa"
"ilure\020*\022!\n\035RefreshPolicyInfoCacheFailure"
"\020+\022\025\n\021ListPolicyFailure\020,\022\022\n\016NotShardLea"
"der\020-\022\026\n\022NoReplicaAvailable\020.\022\023\n\017Segment"
"NotFound\020/\022\r\n\tForceDeny\0200\022\r\n\tRateLimit\0201"
"\022\022\n\016NodeIDNotMatch\0202\022\024\n\020UpsertAutoIDTrue"
"\0203\022\034\n\030InsufficientMemoryToLoad\0204\022\030\n\024Memo"
"ryQuotaExhausted\0205\022\026\n\022DiskQuotaExhausted"
"\0206\022\025\n\021TimeTickLongDelay\0207\022\021\n\rNotReadySer"
"ve\0208\022\033\n\027NotReadyCoordActivating\0209\022\017\n\013Dat"
"aCoordNA\020d\022\022\n\rDDRequestRace\020\350\007*c\n\nIndexS"
"tate\022\022\n\016IndexStateNone\020\000\022\014\n\010Unissued\020\001\022\016"
"\n\nInProgress\020\002\022\014\n\010Finished\020\003\022\n\n\006Failed\020\004"
"\022\t\n\005Retry\020\005*\202\001\n\014SegmentState\022\024\n\020SegmentS"
"tateNone\020\000\022\014\n\010NotExist\020\001\022\013\n\007Growing\020\002\022\n\n"
"\006Sealed\020\003\022\013\n\007Flushed\020\004\022\014\n\010Flushing\020\005\022\013\n\007"
"Dropped\020\006\022\r\n\tImporting\020\007*i\n\017PlaceholderT"
"ype\022\010\n\004None\020\000\022\020\n\014BinaryVector\020d\022\017\n\013Float"
"Vector\020e\022\021\n\rFloat16Vector\020f\022\t\n\005Int64\020\005\022\013"
"\n\007VarChar\020\025*\264\020\n\007MsgType\022\r\n\tUndefined\020\000\022\024"
"\n\020CreateCollection\020d\022\022\n\016DropCollection\020e"
"\022\021\n\rHasCollection\020f\022\026\n\022DescribeCollectio"
"n\020g\022\023\n\017ShowCollections\020h\022\024\n\020GetSystemCon"
"figs\020i\022\022\n\016LoadCollection\020j\022\025\n\021ReleaseCol"
"lection\020k\022\017\n\013CreateAlias\020l\022\r\n\tDropAlias\020"
"m\022\016\n\nAlterAlias\020n\022\023\n\017AlterCollection\020o\022\024"
"\n\020RenameCollection\020p\022\021\n\rDescribeAlias\020q\022"
"\017\n\013ListAliases\020r\022\024\n\017CreatePartition\020\310\001\022\022"
"\n\rDropPartition\020\311\001\022\021\n\014HasPartition\020\312\001\022\026\n"
"\021DescribePartition\020\313\001\022\023\n\016ShowPartitions\020"
"\314\001\022\023\n\016LoadPartitions\020\315\001\022\026\n\021ReleasePartit"
"ions\020\316\001\022\021\n\014ShowSegments\020\372\001\022\024\n\017DescribeSe"
"gment\020\373\001\022\021\n\014LoadSegments\020\374\001\022\024\n\017ReleaseSe"
"gments\020\375\001\022\024\n\017HandoffSegments\020\376\001\022\030\n\023LoadB"
"alanceSegments\020\377\001\022\025\n\020DescribeSegments\020\200\002"
"\022\034\n\027FederListIndexedSegment\020\201\002\022\"\n\035FederD"
"escribeSegmentIndexData\020\202\002\022\020\n\013CreateInde"
"x\020\254\002\022\022\n\rDescribeIndex\020\255\002\022\016\n\tDropIndex\020\256\002"
"\022\027\n\022GetIndexStatistics\020\257\002\022\013\n\006Insert\020\220\003\022\013"
"\n\006Delete\020\221\003\022\n\n\005Flush\020\222\003\022\027\n\022ResendSegment"
"Stats\020\223\003\022\013\n\006Upsert\020\224\003\022\013\n\006Search\020\364\003\022\021\n\014Se"
"archResult\020\365\003\022\022\n\rGetIndexState\020\366\003\022\032\n\025Get"
"IndexBuildProgress\020\367\003\022\034\n\027GetCollectionSt"
"atistics\020\370\003\022\033\n\026GetPartitionStatistics\020\371\003"
"\022\r\n\010Retrieve\020\372\003\022\023\n\016RetrieveResult\020\373\003\022\024\n\017"
"WatchDmChannels\020\374\003\022\025\n\020RemoveDmChannels\020\375"
"\003\022\027\n\022WatchQueryChannels\020\376\003\022\030\n\023RemoveQuer"
"yChannels\020\377\003\022\035\n\030SealedSegmentsChangeInfo"
"\020\200\004\022\027\n\022WatchDeltaChannels\020\201\004\022\024\n\017GetShard"
"Leaders\020\202\004\022\020\n\013GetReplicas\020\203\004\022\023\n\016UnsubDmC"
"hannel\020\204\004\022\024\n\017GetDistribution\020\205\004\022\025\n\020SyncD"
"istribution\020\206\004\022\020\n\013SegmentInfo\020\330\004\022\017\n\nSyst"
"emInfo\020\331\004\022\024\n\017GetRecoveryInfo\020\332\004\022\024\n\017GetSe"
"gmentState\020\333\004\022\r\n\010TimeTick\020\260\t\022\023\n\016QueryNod"
"eStats\020\261\t\022\016\n\tLoadIndex\020\262\t\022\016\n\tRequestID\020\263"
"\t\022\017\n\nRequestTSO\020\264\t\022\024\n\017AllocateSegment\020\265\t"
"\022\026\n\021SegmentStatistics\020\266\t\022\025\n\020SegmentFlush"
"Done\020\267\t\022\017\n\nDataNodeTt\020\270\t\022\014\n\007Connect\020\271\t\022\024"
"\n\017ListClientInfos\020\272\t\022\023\n\016AllocTimestamp\020\273"
"\t\022\025\n\020CreateCredential\020\334\013\022\022\n\rGetCredentia"
"l\020\335\013\022\025\n\020DeleteCredential\020\336\013\022\025\n\020UpdateCre"
"dential\020\337\013\022\026\n\021ListCredUsernames\020\340\013\022\017\n\nCr"
"eateRole\020\300\014\022\r\n\010DropRole\020\301\014\022\024\n\017OperateUse"
"rRole\020\302\014\022\017\n\nSelectRole\020\303\014\022\017\n\nSelectUser\020"
"\304\014\022\023\n\016SelectResource\020\305\014\022\025\n\020OperatePrivil"
"ege\020\306\014\022\020\n\013SelectGrant\020\307\014\022\033\n\026RefreshPolic"
"yInfoCache\020\310\014\022\017\n\nListPolicy\020\311\014\022\030\n\023Create"
"ResourceGroup\020\244\r\022\026\n\021DropResourceGroup\020\245\r"
"\022\027\n\022ListResourceGroups\020\246\r\022\032\n\025DescribeRes"
"ourceGroup\020\247\r\022\021\n\014TransferNode\020\250\r\022\024\n\017Tran"
"sferReplica\020\251\r\022\023\n\016CreateDatabase\020\211\016\022\021\n\014D"
"ropDatabase\020\212\016\022\022\n\rListDatabases\020\213\016*\"\n\007Ds"
"lType\022\007\n\003Dsl\020\000\022\016\n\nBoolExprV1\020\001*B\n\017Compac"
"tionState\022\021\n\rUndefiedState\020\000\022\r\n\tExecutin"
"g\020\001\022\r\n\tCompleted\020\002*X\n\020ConsistencyLevel\022\n"
"\n\006Strong\020\000\022\013\n\007Session\020\001\022\013\n\007Bounded\020\002\022\016\n\n"
"Eventually\020\003\022\016\n\nCustomized\020\004*\236\001\n\013ImportS"
"tate\022\021\n\rImportPending\020\000\022\020\n\014ImportFailed\020"
"\001\022\021\n\rImportStarted\020\002\022\023\n\017ImportPersisted\020"
"\005\022\021\n\rImportFlushed\020\010\022\023\n\017ImportCompleted\020"
"\006\022\032\n\026ImportFailedAndCleaned\020\007*2\n\nObjectT"
"ype\022\016\n\nCollection\020\000\022\n\n\006Global\020\001\022\010\n\004User\020"
"\002*\241\010\n\017ObjectPrivilege\022\020\n\014PrivilegeAll\020\000\022"
"\035\n\031PrivilegeCreateCollection\020\001\022\033\n\027Privil"
"egeDropCollection\020\002\022\037\n\033PrivilegeDescribe"
"Collection\020\003\022\034\n\030PrivilegeShowCollections"
"\020\004\022\021\n\rPrivilegeLoad\020\005\022\024\n\020PrivilegeReleas"
"e\020\006\022\027\n\023PrivilegeCompaction\020\007\022\023\n\017Privileg"
"eInsert\020\010\022\023\n\017PrivilegeDelete\020\t\022\032\n\026Privil"
"egeGetStatistics\020\n\022\030\n\024PrivilegeCreateInd"
"ex\020\013\022\030\n\024PrivilegeIndexDetail\020\014\022\026\n\022Privil"
"egeDropIndex\020\r\022\023\n\017PrivilegeSearch\020\016\022\022\n\016P"
"rivilegeFlush\020\017\022\022\n\016PrivilegeQuery\020\020\022\030\n\024P"
"rivilegeLoadBalance\020\021\022\023\n\017PrivilegeImport"
"\020\022\022\034\n\030PrivilegeCreateOwnership\020\023\022\027\n\023Priv"
"ilegeUpdateUser\020\024\022\032\n\026PrivilegeDropOwners"
"hip\020\025\022\034\n\030PrivilegeSelectOwnership\020\026\022\034\n\030P"
"rivilegeManageOwnership\020\027\022\027\n\023PrivilegeSe"
"lectUser\020\030\022\023\n\017PrivilegeUpsert\020\031\022 \n\034Privi"
"legeCreateResourceGroup\020\032\022\036\n\032PrivilegeDr"
"opResourceGroup\020\033\022\"\n\036PrivilegeDescribeRe"
"sourceGroup\020\034\022\037\n\033PrivilegeListResourceGr"
"oups\020\035\022\031\n\025PrivilegeTransferNode\020\036\022\034\n\030Pri"
"vilegeTransferReplica\020\037\022\037\n\033PrivilegeGetL"
"oadingProgress\020 \022\031\n\025PrivilegeGetLoadStat"
"e\020!\022\035\n\031PrivilegeRenameCollection\020\"\022\033\n\027Pr"
"ivilegeCreateDatabase\020#\022\031\n\025PrivilegeDrop"
"Database\020$\022\032\n\026PrivilegeListDatabases\020%\022\025"
"\n\021PrivilegeFlushAll\020&*S\n\tStateCode\022\020\n\014In"
"itializing\020\000\022\013\n\007Healthy\020\001\022\014\n\010Abnormal\020\002\022"
"\013\n\007StandBy\020\003\022\014\n\010Stopping\020\004*c\n\tLoadState\022"
"\025\n\021LoadStateNotExist\020\000\022\024\n\020LoadStateNotLo"
"ad\020\001\022\024\n\020LoadStateLoading\020\002\022\023\n\017LoadStateL"
"oaded\020\003:^\n\021privilege_ext_obj\022\037.google.pr"
"otobuf.MessageOptions\030\351\007 \001(\0132!.milvus.pr"
"oto.common.PrivilegeExtBm\n\016io.milvus.grp"
"cB\013CommonProtoP\001Z4github.com/milvus-io/m"
"ilvus-proto/go-api/v2/commonpb\240\001\001\252\002\022Milv"
"us.Client.Grpcb\006proto3"
"ommon.MsgBase\"M\n\014DMLMsgHeader\022*\n\004base\030\001 "
"\001(\0132\034.milvus.proto.common.MsgBase\022\021\n\tsha"
"rdName\030\002 \001(\t\"\273\001\n\014PrivilegeExt\0224\n\013object_"
"type\030\001 \001(\0162\037.milvus.proto.common.ObjectT"
"ype\022>\n\020object_privilege\030\002 \001(\0162$.milvus.p"
"roto.common.ObjectPrivilege\022\031\n\021object_na"
"me_index\030\003 \001(\005\022\032\n\022object_name_indexs\030\004 \001"
"(\005\"2\n\014SegmentStats\022\021\n\tSegmentID\030\001 \001(\003\022\017\n"
"\007NumRows\030\002 \001(\003\"\325\001\n\nClientInfo\022\020\n\010sdk_typ"
"e\030\001 \001(\t\022\023\n\013sdk_version\030\002 \001(\t\022\022\n\nlocal_ti"
"me\030\003 \001(\t\022\014\n\004user\030\004 \001(\t\022\014\n\004host\030\005 \001(\t\022\?\n\010"
"reserved\030\006 \003(\0132-.milvus.proto.common.Cli"
"entInfo.ReservedEntry\032/\n\rReservedEntry\022\013"
"\n\003key\030\001 \001(\t\022\r\n\005value\030\002 \001(\t:\0028\001\"\343\001\n\nServe"
"rInfo\022\022\n\nbuild_tags\030\001 \001(\t\022\022\n\nbuild_time\030"
"\002 \001(\t\022\022\n\ngit_commit\030\003 \001(\t\022\022\n\ngo_version\030"
"\004 \001(\t\022\023\n\013deploy_mode\030\005 \001(\t\022\?\n\010reserved\030\006"
" \003(\0132-.milvus.proto.common.ServerInfo.Re"
"servedEntry\032/\n\rReservedEntry\022\013\n\003key\030\001 \001("
"\t\022\r\n\005value\030\002 \001(\t:\0028\001*\303\n\n\tErrorCode\022\013\n\007Su"
"ccess\020\000\022\023\n\017UnexpectedError\020\001\022\021\n\rConnectF"
"ailed\020\002\022\024\n\020PermissionDenied\020\003\022\027\n\023Collect"
"ionNotExists\020\004\022\023\n\017IllegalArgument\020\005\022\024\n\020I"
"llegalDimension\020\007\022\024\n\020IllegalIndexType\020\010\022"
"\031\n\025IllegalCollectionName\020\t\022\017\n\013IllegalTOP"
"K\020\n\022\024\n\020IllegalRowRecord\020\013\022\023\n\017IllegalVect"
"orID\020\014\022\027\n\023IllegalSearchResult\020\r\022\020\n\014FileN"
"otFound\020\016\022\016\n\nMetaFailed\020\017\022\017\n\013CacheFailed"
"\020\020\022\026\n\022CannotCreateFolder\020\021\022\024\n\020CannotCrea"
"teFile\020\022\022\026\n\022CannotDeleteFolder\020\023\022\024\n\020Cann"
"otDeleteFile\020\024\022\023\n\017BuildIndexError\020\025\022\020\n\014I"
"llegalNLIST\020\026\022\025\n\021IllegalMetricType\020\027\022\017\n\013"
"OutOfMemory\020\030\022\021\n\rIndexNotExist\020\031\022\023\n\017Empt"
"yCollection\020\032\022\033\n\027UpdateImportTaskFailure"
"\020\033\022\032\n\026CollectionNameNotFound\020\034\022\033\n\027Create"
"CredentialFailure\020\035\022\033\n\027UpdateCredentialF"
"ailure\020\036\022\033\n\027DeleteCredentialFailure\020\037\022\030\n"
"\024GetCredentialFailure\020 \022\030\n\024ListCredUsers"
"Failure\020!\022\022\n\016GetUserFailure\020\"\022\025\n\021CreateR"
"oleFailure\020#\022\023\n\017DropRoleFailure\020$\022\032\n\026Ope"
"rateUserRoleFailure\020%\022\025\n\021SelectRoleFailu"
"re\020&\022\025\n\021SelectUserFailure\020\'\022\031\n\025SelectRes"
"ourceFailure\020(\022\033\n\027OperatePrivilegeFailur"
"e\020)\022\026\n\022SelectGrantFailure\020*\022!\n\035RefreshPo"
"licyInfoCacheFailure\020+\022\025\n\021ListPolicyFail"
"ure\020,\022\022\n\016NotShardLeader\020-\022\026\n\022NoReplicaAv"
"ailable\020.\022\023\n\017SegmentNotFound\020/\022\r\n\tForceD"
"eny\0200\022\r\n\tRateLimit\0201\022\022\n\016NodeIDNotMatch\0202"
"\022\024\n\020UpsertAutoIDTrue\0203\022\034\n\030InsufficientMe"
"moryToLoad\0204\022\030\n\024MemoryQuotaExhausted\0205\022\026"
"\n\022DiskQuotaExhausted\0206\022\025\n\021TimeTickLongDe"
"lay\0207\022\021\n\rNotReadyServe\0208\022\033\n\027NotReadyCoor"
"dActivating\0209\022\017\n\013DataCoordNA\020d\022\022\n\rDDRequ"
"estRace\020\350\007*c\n\nIndexState\022\022\n\016IndexStateNo"
"ne\020\000\022\014\n\010Unissued\020\001\022\016\n\nInProgress\020\002\022\014\n\010Fi"
"nished\020\003\022\n\n\006Failed\020\004\022\t\n\005Retry\020\005*\202\001\n\014Segm"
"entState\022\024\n\020SegmentStateNone\020\000\022\014\n\010NotExi"
"st\020\001\022\013\n\007Growing\020\002\022\n\n\006Sealed\020\003\022\013\n\007Flushed"
"\020\004\022\014\n\010Flushing\020\005\022\013\n\007Dropped\020\006\022\r\n\tImporti"
"ng\020\007*i\n\017PlaceholderType\022\010\n\004None\020\000\022\020\n\014Bin"
"aryVector\020d\022\017\n\013FloatVector\020e\022\021\n\rFloat16V"
"ector\020f\022\t\n\005Int64\020\005\022\013\n\007VarChar\020\025*\264\020\n\007MsgT"
"ype\022\r\n\tUndefined\020\000\022\024\n\020CreateCollection\020d"
"\022\022\n\016DropCollection\020e\022\021\n\rHasCollection\020f\022"
"\026\n\022DescribeCollection\020g\022\023\n\017ShowCollectio"
"ns\020h\022\024\n\020GetSystemConfigs\020i\022\022\n\016LoadCollec"
"tion\020j\022\025\n\021ReleaseCollection\020k\022\017\n\013CreateA"
"lias\020l\022\r\n\tDropAlias\020m\022\016\n\nAlterAlias\020n\022\023\n"
"\017AlterCollection\020o\022\024\n\020RenameCollection\020p"
"\022\021\n\rDescribeAlias\020q\022\017\n\013ListAliases\020r\022\024\n\017"
"CreatePartition\020\310\001\022\022\n\rDropPartition\020\311\001\022\021"
"\n\014HasPartition\020\312\001\022\026\n\021DescribePartition\020\313"
"\001\022\023\n\016ShowPartitions\020\314\001\022\023\n\016LoadPartitions"
"\020\315\001\022\026\n\021ReleasePartitions\020\316\001\022\021\n\014ShowSegme"
"nts\020\372\001\022\024\n\017DescribeSegment\020\373\001\022\021\n\014LoadSegm"
"ents\020\374\001\022\024\n\017ReleaseSegments\020\375\001\022\024\n\017Handoff"
"Segments\020\376\001\022\030\n\023LoadBalanceSegments\020\377\001\022\025\n"
"\020DescribeSegments\020\200\002\022\034\n\027FederListIndexed"
"Segment\020\201\002\022\"\n\035FederDescribeSegmentIndexD"
"ata\020\202\002\022\020\n\013CreateIndex\020\254\002\022\022\n\rDescribeInde"
"x\020\255\002\022\016\n\tDropIndex\020\256\002\022\027\n\022GetIndexStatisti"
"cs\020\257\002\022\013\n\006Insert\020\220\003\022\013\n\006Delete\020\221\003\022\n\n\005Flush"
"\020\222\003\022\027\n\022ResendSegmentStats\020\223\003\022\013\n\006Upsert\020\224"
"\003\022\013\n\006Search\020\364\003\022\021\n\014SearchResult\020\365\003\022\022\n\rGet"
"IndexState\020\366\003\022\032\n\025GetIndexBuildProgress\020\367"
"\003\022\034\n\027GetCollectionStatistics\020\370\003\022\033\n\026GetPa"
"rtitionStatistics\020\371\003\022\r\n\010Retrieve\020\372\003\022\023\n\016R"
"etrieveResult\020\373\003\022\024\n\017WatchDmChannels\020\374\003\022\025"
"\n\020RemoveDmChannels\020\375\003\022\027\n\022WatchQueryChann"
"els\020\376\003\022\030\n\023RemoveQueryChannels\020\377\003\022\035\n\030Seal"
"edSegmentsChangeInfo\020\200\004\022\027\n\022WatchDeltaCha"
"nnels\020\201\004\022\024\n\017GetShardLeaders\020\202\004\022\020\n\013GetRep"
"licas\020\203\004\022\023\n\016UnsubDmChannel\020\204\004\022\024\n\017GetDist"
"ribution\020\205\004\022\025\n\020SyncDistribution\020\206\004\022\020\n\013Se"
"gmentInfo\020\330\004\022\017\n\nSystemInfo\020\331\004\022\024\n\017GetReco"
"veryInfo\020\332\004\022\024\n\017GetSegmentState\020\333\004\022\r\n\010Tim"
"eTick\020\260\t\022\023\n\016QueryNodeStats\020\261\t\022\016\n\tLoadInd"
"ex\020\262\t\022\016\n\tRequestID\020\263\t\022\017\n\nRequestTSO\020\264\t\022\024"
"\n\017AllocateSegment\020\265\t\022\026\n\021SegmentStatistic"
"s\020\266\t\022\025\n\020SegmentFlushDone\020\267\t\022\017\n\nDataNodeT"
"t\020\270\t\022\014\n\007Connect\020\271\t\022\024\n\017ListClientInfos\020\272\t"
"\022\023\n\016AllocTimestamp\020\273\t\022\025\n\020CreateCredentia"
"l\020\334\013\022\022\n\rGetCredential\020\335\013\022\025\n\020DeleteCreden"
"tial\020\336\013\022\025\n\020UpdateCredential\020\337\013\022\026\n\021ListCr"
"edUsernames\020\340\013\022\017\n\nCreateRole\020\300\014\022\r\n\010DropR"
"ole\020\301\014\022\024\n\017OperateUserRole\020\302\014\022\017\n\nSelectRo"
"le\020\303\014\022\017\n\nSelectUser\020\304\014\022\023\n\016SelectResource"
"\020\305\014\022\025\n\020OperatePrivilege\020\306\014\022\020\n\013SelectGran"
"t\020\307\014\022\033\n\026RefreshPolicyInfoCache\020\310\014\022\017\n\nLis"
"tPolicy\020\311\014\022\030\n\023CreateResourceGroup\020\244\r\022\026\n\021"
"DropResourceGroup\020\245\r\022\027\n\022ListResourceGrou"
"ps\020\246\r\022\032\n\025DescribeResourceGroup\020\247\r\022\021\n\014Tra"
"nsferNode\020\250\r\022\024\n\017TransferReplica\020\251\r\022\023\n\016Cr"
"eateDatabase\020\211\016\022\021\n\014DropDatabase\020\212\016\022\022\n\rLi"
"stDatabases\020\213\016*\"\n\007DslType\022\007\n\003Dsl\020\000\022\016\n\nBo"
"olExprV1\020\001*B\n\017CompactionState\022\021\n\rUndefie"
"dState\020\000\022\r\n\tExecuting\020\001\022\r\n\tCompleted\020\002*X"
"\n\020ConsistencyLevel\022\n\n\006Strong\020\000\022\013\n\007Sessio"
"n\020\001\022\013\n\007Bounded\020\002\022\016\n\nEventually\020\003\022\016\n\nCust"
"omized\020\004*\236\001\n\013ImportState\022\021\n\rImportPendin"
"g\020\000\022\020\n\014ImportFailed\020\001\022\021\n\rImportStarted\020\002"
"\022\023\n\017ImportPersisted\020\005\022\021\n\rImportFlushed\020\010"
"\022\023\n\017ImportCompleted\020\006\022\032\n\026ImportFailedAnd"
"Cleaned\020\007*2\n\nObjectType\022\016\n\nCollection\020\000\022"
"\n\n\006Global\020\001\022\010\n\004User\020\002*\241\010\n\017ObjectPrivileg"
"e\022\020\n\014PrivilegeAll\020\000\022\035\n\031PrivilegeCreateCo"
"llection\020\001\022\033\n\027PrivilegeDropCollection\020\002\022"
"\037\n\033PrivilegeDescribeCollection\020\003\022\034\n\030Priv"
"ilegeShowCollections\020\004\022\021\n\rPrivilegeLoad\020"
"\005\022\024\n\020PrivilegeRelease\020\006\022\027\n\023PrivilegeComp"
"action\020\007\022\023\n\017PrivilegeInsert\020\010\022\023\n\017Privile"
"geDelete\020\t\022\032\n\026PrivilegeGetStatistics\020\n\022\030"
"\n\024PrivilegeCreateIndex\020\013\022\030\n\024PrivilegeInd"
"exDetail\020\014\022\026\n\022PrivilegeDropIndex\020\r\022\023\n\017Pr"
"ivilegeSearch\020\016\022\022\n\016PrivilegeFlush\020\017\022\022\n\016P"
"rivilegeQuery\020\020\022\030\n\024PrivilegeLoadBalance\020"
"\021\022\023\n\017PrivilegeImport\020\022\022\034\n\030PrivilegeCreat"
"eOwnership\020\023\022\027\n\023PrivilegeUpdateUser\020\024\022\032\n"
"\026PrivilegeDropOwnership\020\025\022\034\n\030PrivilegeSe"
"lectOwnership\020\026\022\034\n\030PrivilegeManageOwners"
"hip\020\027\022\027\n\023PrivilegeSelectUser\020\030\022\023\n\017Privil"
"egeUpsert\020\031\022 \n\034PrivilegeCreateResourceGr"
"oup\020\032\022\036\n\032PrivilegeDropResourceGroup\020\033\022\"\n"
"\036PrivilegeDescribeResourceGroup\020\034\022\037\n\033Pri"
"vilegeListResourceGroups\020\035\022\031\n\025PrivilegeT"
"ransferNode\020\036\022\034\n\030PrivilegeTransferReplic"
"a\020\037\022\037\n\033PrivilegeGetLoadingProgress\020 \022\031\n\025"
"PrivilegeGetLoadState\020!\022\035\n\031PrivilegeRena"
"meCollection\020\"\022\033\n\027PrivilegeCreateDatabas"
"e\020#\022\031\n\025PrivilegeDropDatabase\020$\022\032\n\026Privil"
"egeListDatabases\020%\022\025\n\021PrivilegeFlushAll\020"
"&*S\n\tStateCode\022\020\n\014Initializing\020\000\022\013\n\007Heal"
"thy\020\001\022\014\n\010Abnormal\020\002\022\013\n\007StandBy\020\003\022\014\n\010Stop"
"ping\020\004*c\n\tLoadState\022\025\n\021LoadStateNotExist"
"\020\000\022\024\n\020LoadStateNotLoad\020\001\022\024\n\020LoadStateLoa"
"ding\020\002\022\023\n\017LoadStateLoaded\020\003:^\n\021privilege"
"_ext_obj\022\037.google.protobuf.MessageOption"
"s\030\351\007 \001(\0132!.milvus.proto.common.Privilege"
"ExtBm\n\016io.milvus.grpcB\013CommonProtoP\001Z4gi"
"thub.com/milvus-io/milvus-proto/go-api/v"
"2/commonpb\240\001\001\252\002\022Milvus.Client.Grpcb\006prot"
"o3"
;
static const ::_pbi::DescriptorTable* const descriptor_table_common_2eproto_deps[1] = {
&::descriptor_table_google_2fprotobuf_2fdescriptor_2eproto,
};
static ::_pbi::once_flag descriptor_table_common_2eproto_once;
const ::_pbi::DescriptorTable descriptor_table_common_2eproto = {
false, false, 7222, descriptor_table_protodef_common_2eproto,
false, false, 7202, descriptor_table_protodef_common_2eproto,
"common.proto",
&descriptor_table_common_2eproto_once, descriptor_table_common_2eproto_deps, 1, 17,
schemas, file_default_instances, TableStruct_common_2eproto::offsets,
@ -2726,38 +2725,26 @@ void Address::InternalSwap(Address* other) {
// ===================================================================
MsgBase_PropertiesEntry_DoNotUse::MsgBase_PropertiesEntry_DoNotUse() {}
MsgBase_PropertiesEntry_DoNotUse::MsgBase_PropertiesEntry_DoNotUse(::PROTOBUF_NAMESPACE_ID::Arena* arena)
: SuperType(arena) {}
void MsgBase_PropertiesEntry_DoNotUse::MergeFrom(const MsgBase_PropertiesEntry_DoNotUse& other) {
MergeFromInternal(other);
}
::PROTOBUF_NAMESPACE_ID::Metadata MsgBase_PropertiesEntry_DoNotUse::GetMetadata() const {
return ::_pbi::AssignDescriptors(
&descriptor_table_common_2eproto_getter, &descriptor_table_common_2eproto_once,
file_level_metadata_common_2eproto[7]);
}
// ===================================================================
class MsgBase::_Internal {
public:
static const ::milvus::proto::common::ReplicateEntity& replicate(const MsgBase* msg);
};
const ::milvus::proto::common::ReplicateEntity&
MsgBase::_Internal::replicate(const MsgBase* msg) {
return *msg->_impl_.replicate_;
}
MsgBase::MsgBase(::PROTOBUF_NAMESPACE_ID::Arena* arena,
bool is_message_owned)
: ::PROTOBUF_NAMESPACE_ID::Message(arena, is_message_owned) {
SharedCtor(arena, is_message_owned);
if (arena != nullptr && !is_message_owned) {
arena->OwnCustomDestructor(this, &MsgBase::ArenaDtor);
}
// @@protoc_insertion_point(arena_constructor:milvus.proto.common.MsgBase)
}
MsgBase::MsgBase(const MsgBase& from)
: ::PROTOBUF_NAMESPACE_ID::Message() {
MsgBase* const _this = this; (void)_this;
new (&_impl_) Impl_{
/*decltype(_impl_.properties_)*/{}
decltype(_impl_.replicate_){nullptr}
, decltype(_impl_.msgid_){}
, decltype(_impl_.timestamp_){}
, decltype(_impl_.sourceid_){}
@ -2766,7 +2753,9 @@ MsgBase::MsgBase(const MsgBase& from)
, /*decltype(_impl_._cached_size_)*/{}};
_internal_metadata_.MergeFrom<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(from._internal_metadata_);
_this->_impl_.properties_.MergeFrom(from._impl_.properties_);
if (from._internal_has_replicate()) {
_this->_impl_.replicate_ = new ::milvus::proto::common::ReplicateEntity(*from._impl_.replicate_);
}
::memcpy(&_impl_.msgid_, &from._impl_.msgid_,
static_cast<size_t>(reinterpret_cast<char*>(&_impl_.msg_type_) -
reinterpret_cast<char*>(&_impl_.msgid_)) + sizeof(_impl_.msg_type_));
@ -2778,7 +2767,7 @@ inline void MsgBase::SharedCtor(
(void)arena;
(void)is_message_owned;
new (&_impl_) Impl_{
/*decltype(_impl_.properties_)*/{::_pbi::ArenaInitialized(), arena}
decltype(_impl_.replicate_){nullptr}
, decltype(_impl_.msgid_){int64_t{0}}
, decltype(_impl_.timestamp_){uint64_t{0u}}
, decltype(_impl_.sourceid_){int64_t{0}}
@ -2792,7 +2781,6 @@ MsgBase::~MsgBase() {
// @@protoc_insertion_point(destructor:milvus.proto.common.MsgBase)
if (auto *arena = _internal_metadata_.DeleteReturnArena<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>()) {
(void)arena;
ArenaDtor(this);
return;
}
SharedDtor();
@ -2800,14 +2788,9 @@ MsgBase::~MsgBase() {
inline void MsgBase::SharedDtor() {
GOOGLE_DCHECK(GetArenaForAllocation() == nullptr);
_impl_.properties_.Destruct();
_impl_.properties_.~MapField();
if (this != internal_default_instance()) delete _impl_.replicate_;
}
void MsgBase::ArenaDtor(void* object) {
MsgBase* _this = reinterpret_cast< MsgBase* >(object);
_this->_impl_.properties_.Destruct();
}
void MsgBase::SetCachedSize(int size) const {
_impl_._cached_size_.Set(size);
}
@ -2818,7 +2801,10 @@ void MsgBase::Clear() {
// Prevent compiler warnings about cached_has_bits being unused
(void) cached_has_bits;
_impl_.properties_.Clear();
if (GetArenaForAllocation() == nullptr && _impl_.replicate_ != nullptr) {
delete _impl_.replicate_;
}
_impl_.replicate_ = nullptr;
::memset(&_impl_.msgid_, 0, static_cast<size_t>(
reinterpret_cast<char*>(&_impl_.msg_type_) -
reinterpret_cast<char*>(&_impl_.msgid_)) + sizeof(_impl_.msg_type_));
@ -2872,16 +2858,11 @@ const char* MsgBase::_InternalParse(const char* ptr, ::_pbi::ParseContext* ctx)
} else
goto handle_unusual;
continue;
// map<string, string> properties = 6;
// .milvus.proto.common.ReplicateEntity replicate = 6;
case 6:
if (PROTOBUF_PREDICT_TRUE(static_cast<uint8_t>(tag) == 50)) {
ptr -= 1;
do {
ptr += 1;
ptr = ctx->ParseMessage(&_impl_.properties_, ptr);
CHK_(ptr);
if (!ctx->DataAvailable(ptr)) break;
} while (::PROTOBUF_NAMESPACE_ID::internal::ExpectTag<50>(ptr));
ptr = ctx->ParseMessage(_internal_mutable_replicate(), ptr);
CHK_(ptr);
} else
goto handle_unusual;
continue;
@ -2945,34 +2926,11 @@ uint8_t* MsgBase::_InternalSerialize(
target = ::_pbi::WireFormatLite::WriteInt64ToArray(5, this->_internal_targetid(), target);
}
// map<string, string> properties = 6;
if (!this->_internal_properties().empty()) {
using MapType = ::_pb::Map<std::string, std::string>;
using WireHelper = MsgBase_PropertiesEntry_DoNotUse::Funcs;
const auto& map_field = this->_internal_properties();
auto check_utf8 = [](const MapType::value_type& entry) {
(void)entry;
::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::VerifyUtf8String(
entry.first.data(), static_cast<int>(entry.first.length()),
::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::SERIALIZE,
"milvus.proto.common.MsgBase.PropertiesEntry.key");
::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::VerifyUtf8String(
entry.second.data(), static_cast<int>(entry.second.length()),
::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::SERIALIZE,
"milvus.proto.common.MsgBase.PropertiesEntry.value");
};
if (stream->IsSerializationDeterministic() && map_field.size() > 1) {
for (const auto& entry : ::_pbi::MapSorterPtr<MapType>(map_field)) {
target = WireHelper::InternalSerialize(6, entry.first, entry.second, target, stream);
check_utf8(entry);
}
} else {
for (const auto& entry : map_field) {
target = WireHelper::InternalSerialize(6, entry.first, entry.second, target, stream);
check_utf8(entry);
}
}
// .milvus.proto.common.ReplicateEntity replicate = 6;
if (this->_internal_has_replicate()) {
target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::
InternalWriteMessage(6, _Internal::replicate(this),
_Internal::replicate(this).GetCachedSize(), target, stream);
}
if (PROTOBUF_PREDICT_FALSE(_internal_metadata_.have_unknown_fields())) {
@ -2991,13 +2949,11 @@ size_t MsgBase::ByteSizeLong() const {
// Prevent compiler warnings about cached_has_bits being unused
(void) cached_has_bits;
// map<string, string> properties = 6;
total_size += 1 *
::PROTOBUF_NAMESPACE_ID::internal::FromIntSize(this->_internal_properties_size());
for (::PROTOBUF_NAMESPACE_ID::Map< std::string, std::string >::const_iterator
it = this->_internal_properties().begin();
it != this->_internal_properties().end(); ++it) {
total_size += MsgBase_PropertiesEntry_DoNotUse::Funcs::ByteSizeLong(it->first, it->second);
// .milvus.proto.common.ReplicateEntity replicate = 6;
if (this->_internal_has_replicate()) {
total_size += 1 +
::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::MessageSize(
*_impl_.replicate_);
}
// int64 msgID = 2;
@ -3044,7 +3000,10 @@ void MsgBase::MergeImpl(::PROTOBUF_NAMESPACE_ID::Message& to_msg, const ::PROTOB
uint32_t cached_has_bits = 0;
(void) cached_has_bits;
_this->_impl_.properties_.MergeFrom(from._impl_.properties_);
if (from._internal_has_replicate()) {
_this->_internal_mutable_replicate()->::milvus::proto::common::ReplicateEntity::MergeFrom(
from._internal_replicate());
}
if (from._internal_msgid() != 0) {
_this->_internal_set_msgid(from._internal_msgid());
}
@ -3077,16 +3036,193 @@ bool MsgBase::IsInitialized() const {
void MsgBase::InternalSwap(MsgBase* other) {
using std::swap;
_internal_metadata_.InternalSwap(&other->_internal_metadata_);
_impl_.properties_.InternalSwap(&other->_impl_.properties_);
::PROTOBUF_NAMESPACE_ID::internal::memswap<
PROTOBUF_FIELD_OFFSET(MsgBase, _impl_.msg_type_)
+ sizeof(MsgBase::_impl_.msg_type_)
- PROTOBUF_FIELD_OFFSET(MsgBase, _impl_.msgid_)>(
reinterpret_cast<char*>(&_impl_.msgid_),
reinterpret_cast<char*>(&other->_impl_.msgid_));
- PROTOBUF_FIELD_OFFSET(MsgBase, _impl_.replicate_)>(
reinterpret_cast<char*>(&_impl_.replicate_),
reinterpret_cast<char*>(&other->_impl_.replicate_));
}
::PROTOBUF_NAMESPACE_ID::Metadata MsgBase::GetMetadata() const {
return ::_pbi::AssignDescriptors(
&descriptor_table_common_2eproto_getter, &descriptor_table_common_2eproto_once,
file_level_metadata_common_2eproto[7]);
}
// ===================================================================
class ReplicateEntity::_Internal {
public:
};
ReplicateEntity::ReplicateEntity(::PROTOBUF_NAMESPACE_ID::Arena* arena,
bool is_message_owned)
: ::PROTOBUF_NAMESPACE_ID::Message(arena, is_message_owned) {
SharedCtor(arena, is_message_owned);
// @@protoc_insertion_point(arena_constructor:milvus.proto.common.ReplicateEntity)
}
ReplicateEntity::ReplicateEntity(const ReplicateEntity& from)
: ::PROTOBUF_NAMESPACE_ID::Message() {
ReplicateEntity* const _this = this; (void)_this;
new (&_impl_) Impl_{
decltype(_impl_.isreplicate_){}
, /*decltype(_impl_._cached_size_)*/{}};
_internal_metadata_.MergeFrom<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(from._internal_metadata_);
_this->_impl_.isreplicate_ = from._impl_.isreplicate_;
// @@protoc_insertion_point(copy_constructor:milvus.proto.common.ReplicateEntity)
}
inline void ReplicateEntity::SharedCtor(
::_pb::Arena* arena, bool is_message_owned) {
(void)arena;
(void)is_message_owned;
new (&_impl_) Impl_{
decltype(_impl_.isreplicate_){false}
, /*decltype(_impl_._cached_size_)*/{}
};
}
ReplicateEntity::~ReplicateEntity() {
// @@protoc_insertion_point(destructor:milvus.proto.common.ReplicateEntity)
if (auto *arena = _internal_metadata_.DeleteReturnArena<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>()) {
(void)arena;
return;
}
SharedDtor();
}
inline void ReplicateEntity::SharedDtor() {
GOOGLE_DCHECK(GetArenaForAllocation() == nullptr);
}
void ReplicateEntity::SetCachedSize(int size) const {
_impl_._cached_size_.Set(size);
}
void ReplicateEntity::Clear() {
// @@protoc_insertion_point(message_clear_start:milvus.proto.common.ReplicateEntity)
uint32_t cached_has_bits = 0;
// Prevent compiler warnings about cached_has_bits being unused
(void) cached_has_bits;
_impl_.isreplicate_ = false;
_internal_metadata_.Clear<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>();
}
const char* ReplicateEntity::_InternalParse(const char* ptr, ::_pbi::ParseContext* ctx) {
#define CHK_(x) if (PROTOBUF_PREDICT_FALSE(!(x))) goto failure
while (!ctx->Done(&ptr)) {
uint32_t tag;
ptr = ::_pbi::ReadTag(ptr, &tag);
switch (tag >> 3) {
// bool isReplicate = 1;
case 1:
if (PROTOBUF_PREDICT_TRUE(static_cast<uint8_t>(tag) == 8)) {
_impl_.isreplicate_ = ::PROTOBUF_NAMESPACE_ID::internal::ReadVarint64(&ptr);
CHK_(ptr);
} else
goto handle_unusual;
continue;
default:
goto handle_unusual;
} // switch
handle_unusual:
if ((tag == 0) || ((tag & 7) == 4)) {
CHK_(ptr);
ctx->SetLastTag(tag);
goto message_done;
}
ptr = UnknownFieldParse(
tag,
_internal_metadata_.mutable_unknown_fields<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(),
ptr, ctx);
CHK_(ptr != nullptr);
} // while
message_done:
return ptr;
failure:
ptr = nullptr;
goto message_done;
#undef CHK_
}
uint8_t* ReplicateEntity::_InternalSerialize(
uint8_t* target, ::PROTOBUF_NAMESPACE_ID::io::EpsCopyOutputStream* stream) const {
// @@protoc_insertion_point(serialize_to_array_start:milvus.proto.common.ReplicateEntity)
uint32_t cached_has_bits = 0;
(void) cached_has_bits;
// bool isReplicate = 1;
if (this->_internal_isreplicate() != 0) {
target = stream->EnsureSpace(target);
target = ::_pbi::WireFormatLite::WriteBoolToArray(1, this->_internal_isreplicate(), target);
}
if (PROTOBUF_PREDICT_FALSE(_internal_metadata_.have_unknown_fields())) {
target = ::_pbi::WireFormat::InternalSerializeUnknownFieldsToArray(
_internal_metadata_.unknown_fields<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(::PROTOBUF_NAMESPACE_ID::UnknownFieldSet::default_instance), target, stream);
}
// @@protoc_insertion_point(serialize_to_array_end:milvus.proto.common.ReplicateEntity)
return target;
}
size_t ReplicateEntity::ByteSizeLong() const {
// @@protoc_insertion_point(message_byte_size_start:milvus.proto.common.ReplicateEntity)
size_t total_size = 0;
uint32_t cached_has_bits = 0;
// Prevent compiler warnings about cached_has_bits being unused
(void) cached_has_bits;
// bool isReplicate = 1;
if (this->_internal_isreplicate() != 0) {
total_size += 1 + 1;
}
return MaybeComputeUnknownFieldsSize(total_size, &_impl_._cached_size_);
}
const ::PROTOBUF_NAMESPACE_ID::Message::ClassData ReplicateEntity::_class_data_ = {
::PROTOBUF_NAMESPACE_ID::Message::CopyWithSourceCheck,
ReplicateEntity::MergeImpl
};
const ::PROTOBUF_NAMESPACE_ID::Message::ClassData*ReplicateEntity::GetClassData() const { return &_class_data_; }
void ReplicateEntity::MergeImpl(::PROTOBUF_NAMESPACE_ID::Message& to_msg, const ::PROTOBUF_NAMESPACE_ID::Message& from_msg) {
auto* const _this = static_cast<ReplicateEntity*>(&to_msg);
auto& from = static_cast<const ReplicateEntity&>(from_msg);
// @@protoc_insertion_point(class_specific_merge_from_start:milvus.proto.common.ReplicateEntity)
GOOGLE_DCHECK_NE(&from, _this);
uint32_t cached_has_bits = 0;
(void) cached_has_bits;
if (from._internal_isreplicate() != 0) {
_this->_internal_set_isreplicate(from._internal_isreplicate());
}
_this->_internal_metadata_.MergeFrom<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(from._internal_metadata_);
}
void ReplicateEntity::CopyFrom(const ReplicateEntity& from) {
// @@protoc_insertion_point(class_specific_copy_from_start:milvus.proto.common.ReplicateEntity)
if (&from == this) return;
Clear();
MergeFrom(from);
}
bool ReplicateEntity::IsInitialized() const {
return true;
}
void ReplicateEntity::InternalSwap(ReplicateEntity* other) {
using std::swap;
_internal_metadata_.InternalSwap(&other->_internal_metadata_);
swap(_impl_.isreplicate_, other->_impl_.isreplicate_);
}
::PROTOBUF_NAMESPACE_ID::Metadata ReplicateEntity::GetMetadata() const {
return ::_pbi::AssignDescriptors(
&descriptor_table_common_2eproto_getter, &descriptor_table_common_2eproto_once,
file_level_metadata_common_2eproto[8]);
@ -5012,14 +5148,14 @@ template<> PROTOBUF_NOINLINE ::milvus::proto::common::Address*
Arena::CreateMaybeMessage< ::milvus::proto::common::Address >(Arena* arena) {
return Arena::CreateMessageInternal< ::milvus::proto::common::Address >(arena);
}
template<> PROTOBUF_NOINLINE ::milvus::proto::common::MsgBase_PropertiesEntry_DoNotUse*
Arena::CreateMaybeMessage< ::milvus::proto::common::MsgBase_PropertiesEntry_DoNotUse >(Arena* arena) {
return Arena::CreateMessageInternal< ::milvus::proto::common::MsgBase_PropertiesEntry_DoNotUse >(arena);
}
template<> PROTOBUF_NOINLINE ::milvus::proto::common::MsgBase*
Arena::CreateMaybeMessage< ::milvus::proto::common::MsgBase >(Arena* arena) {
return Arena::CreateMessageInternal< ::milvus::proto::common::MsgBase >(arena);
}
template<> PROTOBUF_NOINLINE ::milvus::proto::common::ReplicateEntity*
Arena::CreateMaybeMessage< ::milvus::proto::common::ReplicateEntity >(Arena* arena) {
return Arena::CreateMessageInternal< ::milvus::proto::common::ReplicateEntity >(arena);
}
template<> PROTOBUF_NOINLINE ::milvus::proto::common::MsgHeader*
Arena::CreateMaybeMessage< ::milvus::proto::common::MsgHeader >(Arena* arena) {
return Arena::CreateMessageInternal< ::milvus::proto::common::MsgHeader >(arena);

351
internal/core/src/pb/common.pb.h
View File

@ -76,9 +76,6 @@ extern KeyValuePairDefaultTypeInternal _KeyValuePair_default_instance_;
class MsgBase;
struct MsgBaseDefaultTypeInternal;
extern MsgBaseDefaultTypeInternal _MsgBase_default_instance_;
class MsgBase_PropertiesEntry_DoNotUse;
struct MsgBase_PropertiesEntry_DoNotUseDefaultTypeInternal;
extern MsgBase_PropertiesEntry_DoNotUseDefaultTypeInternal _MsgBase_PropertiesEntry_DoNotUse_default_instance_;
class MsgHeader;
struct MsgHeaderDefaultTypeInternal;
extern MsgHeaderDefaultTypeInternal _MsgHeader_default_instance_;
@ -91,6 +88,9 @@ extern PlaceholderValueDefaultTypeInternal _PlaceholderValue_default_instance_;
class PrivilegeExt;
struct PrivilegeExtDefaultTypeInternal;
extern PrivilegeExtDefaultTypeInternal _PrivilegeExt_default_instance_;
class ReplicateEntity;
struct ReplicateEntityDefaultTypeInternal;
extern ReplicateEntityDefaultTypeInternal _ReplicateEntity_default_instance_;
class SegmentStats;
struct SegmentStatsDefaultTypeInternal;
extern SegmentStatsDefaultTypeInternal _SegmentStats_default_instance_;
@ -115,11 +115,11 @@ template<> ::milvus::proto::common::DMLMsgHeader* Arena::CreateMaybeMessage<::mi
template<> ::milvus::proto::common::KeyDataPair* Arena::CreateMaybeMessage<::milvus::proto::common::KeyDataPair>(Arena*);
template<> ::milvus::proto::common::KeyValuePair* Arena::CreateMaybeMessage<::milvus::proto::common::KeyValuePair>(Arena*);
template<> ::milvus::proto::common::MsgBase* Arena::CreateMaybeMessage<::milvus::proto::common::MsgBase>(Arena*);
template<> ::milvus::proto::common::MsgBase_PropertiesEntry_DoNotUse* Arena::CreateMaybeMessage<::milvus::proto::common::MsgBase_PropertiesEntry_DoNotUse>(Arena*);
template<> ::milvus::proto::common::MsgHeader* Arena::CreateMaybeMessage<::milvus::proto::common::MsgHeader>(Arena*);
template<> ::milvus::proto::common::PlaceholderGroup* Arena::CreateMaybeMessage<::milvus::proto::common::PlaceholderGroup>(Arena*);
template<> ::milvus::proto::common::PlaceholderValue* Arena::CreateMaybeMessage<::milvus::proto::common::PlaceholderValue>(Arena*);
template<> ::milvus::proto::common::PrivilegeExt* Arena::CreateMaybeMessage<::milvus::proto::common::PrivilegeExt>(Arena*);
template<> ::milvus::proto::common::ReplicateEntity* Arena::CreateMaybeMessage<::milvus::proto::common::ReplicateEntity>(Arena*);
template<> ::milvus::proto::common::SegmentStats* Arena::CreateMaybeMessage<::milvus::proto::common::SegmentStats>(Arena*);
template<> ::milvus::proto::common::ServerInfo* Arena::CreateMaybeMessage<::milvus::proto::common::ServerInfo>(Arena*);
template<> ::milvus::proto::common::ServerInfo_ReservedEntry_DoNotUse* Arena::CreateMaybeMessage<::milvus::proto::common::ServerInfo_ReservedEntry_DoNotUse>(Arena*);
@ -1854,34 +1854,6 @@ class Address final :
};
// -------------------------------------------------------------------
class MsgBase_PropertiesEntry_DoNotUse : public ::PROTOBUF_NAMESPACE_ID::internal::MapEntry<MsgBase_PropertiesEntry_DoNotUse,
std::string, std::string,
::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::TYPE_STRING,
::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::TYPE_STRING> {
public:
typedef ::PROTOBUF_NAMESPACE_ID::internal::MapEntry<MsgBase_PropertiesEntry_DoNotUse,
std::string, std::string,
::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::TYPE_STRING,
::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::TYPE_STRING> SuperType;
MsgBase_PropertiesEntry_DoNotUse();
explicit PROTOBUF_CONSTEXPR MsgBase_PropertiesEntry_DoNotUse(
::PROTOBUF_NAMESPACE_ID::internal::ConstantInitialized);
explicit MsgBase_PropertiesEntry_DoNotUse(::PROTOBUF_NAMESPACE_ID::Arena* arena);
void MergeFrom(const MsgBase_PropertiesEntry_DoNotUse& other);
static const MsgBase_PropertiesEntry_DoNotUse* internal_default_instance() { return reinterpret_cast<const MsgBase_PropertiesEntry_DoNotUse*>(&_MsgBase_PropertiesEntry_DoNotUse_default_instance_); }
static bool ValidateKey(std::string* s) {
return ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::VerifyUtf8String(s->data(), static_cast<int>(s->size()), ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::PARSE, "milvus.proto.common.MsgBase.PropertiesEntry.key");
}
static bool ValidateValue(std::string* s) {
return ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::VerifyUtf8String(s->data(), static_cast<int>(s->size()), ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::PARSE, "milvus.proto.common.MsgBase.PropertiesEntry.value");
}
using ::PROTOBUF_NAMESPACE_ID::Message::MergeFrom;
::PROTOBUF_NAMESPACE_ID::Metadata GetMetadata() const final;
friend struct ::TableStruct_common_2eproto;
};
// -------------------------------------------------------------------
class MsgBase final :
public ::PROTOBUF_NAMESPACE_ID::Message /* @@protoc_insertion_point(class_definition:milvus.proto.common.MsgBase) */ {
public:
@ -1930,7 +1902,7 @@ class MsgBase final :
&_MsgBase_default_instance_);
}
static constexpr int kIndexInFileMessages =
8;
7;
friend void swap(MsgBase& a, MsgBase& b) {
a.Swap(&b);
@ -1991,8 +1963,6 @@ class MsgBase final :
protected:
explicit MsgBase(::PROTOBUF_NAMESPACE_ID::Arena* arena,
bool is_message_owned = false);
private:
static void ArenaDtor(void* object);
public:
static const ClassData _class_data_;
@ -2002,33 +1972,33 @@ class MsgBase final :
// nested types ----------------------------------------------------
// accessors -------------------------------------------------------
enum : int {
kPropertiesFieldNumber = 6,
kReplicateFieldNumber = 6,
kMsgIDFieldNumber = 2,
kTimestampFieldNumber = 3,
kSourceIDFieldNumber = 4,
kTargetIDFieldNumber = 5,
kMsgTypeFieldNumber = 1,
};
// map<string, string> properties = 6;
int properties_size() const;
// .milvus.proto.common.ReplicateEntity replicate = 6;
bool has_replicate() const;
private:
int _internal_properties_size() const;
bool _internal_has_replicate() const;
public:
void clear_properties();
void clear_replicate();
const ::milvus::proto::common::ReplicateEntity& replicate() const;
PROTOBUF_NODISCARD ::milvus::proto::common::ReplicateEntity* release_replicate();
::milvus::proto::common::ReplicateEntity* mutable_replicate();
void set_allocated_replicate(::milvus::proto::common::ReplicateEntity* replicate);
private:
const ::PROTOBUF_NAMESPACE_ID::Map< std::string, std::string >&
_internal_properties() const;
::PROTOBUF_NAMESPACE_ID::Map< std::string, std::string >*
_internal_mutable_properties();
const ::milvus::proto::common::ReplicateEntity& _internal_replicate() const;
::milvus::proto::common::ReplicateEntity* _internal_mutable_replicate();
public:
const ::PROTOBUF_NAMESPACE_ID::Map< std::string, std::string >&
properties() const;
::PROTOBUF_NAMESPACE_ID::Map< std::string, std::string >*
mutable_properties();
void unsafe_arena_set_allocated_replicate(
::milvus::proto::common::ReplicateEntity* replicate);
::milvus::proto::common::ReplicateEntity* unsafe_arena_release_replicate();
// int64 msgID = 2;
void clear_msgid();
@ -2083,11 +2053,7 @@ class MsgBase final :
typedef void InternalArenaConstructable_;
typedef void DestructorSkippable_;
struct Impl_ {
::PROTOBUF_NAMESPACE_ID::internal::MapField<
MsgBase_PropertiesEntry_DoNotUse,
std::string, std::string,
::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::TYPE_STRING,
::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::TYPE_STRING> properties_;
::milvus::proto::common::ReplicateEntity* replicate_;
int64_t msgid_;
uint64_t timestamp_;
int64_t sourceid_;
@ -2100,6 +2066,154 @@ class MsgBase final :
};
// -------------------------------------------------------------------
class ReplicateEntity final :
public ::PROTOBUF_NAMESPACE_ID::Message /* @@protoc_insertion_point(class_definition:milvus.proto.common.ReplicateEntity) */ {
public:
inline ReplicateEntity() : ReplicateEntity(nullptr) {}
~ReplicateEntity() override;
explicit PROTOBUF_CONSTEXPR ReplicateEntity(::PROTOBUF_NAMESPACE_ID::internal::ConstantInitialized);
ReplicateEntity(const ReplicateEntity& from);
ReplicateEntity(ReplicateEntity&& from) noexcept
: ReplicateEntity() {
*this = ::std::move(from);
}
inline ReplicateEntity& operator=(const ReplicateEntity& from) {
CopyFrom(from);
return *this;
}
inline ReplicateEntity& operator=(ReplicateEntity&& from) noexcept {
if (this == &from) return *this;
if (GetOwningArena() == from.GetOwningArena()
#ifdef PROTOBUF_FORCE_COPY_IN_MOVE
&& GetOwningArena() != nullptr
#endif // !PROTOBUF_FORCE_COPY_IN_MOVE
) {
InternalSwap(&from);
} else {
CopyFrom(from);
}
return *this;
}
static const ::PROTOBUF_NAMESPACE_ID::Descriptor* descriptor() {
return GetDescriptor();
}
static const ::PROTOBUF_NAMESPACE_ID::Descriptor* GetDescriptor() {
return default_instance().GetMetadata().descriptor;
}
static const ::PROTOBUF_NAMESPACE_ID::Reflection* GetReflection() {
return default_instance().GetMetadata().reflection;
}
static const ReplicateEntity& default_instance() {
return *internal_default_instance();
}
static inline const ReplicateEntity* internal_default_instance() {
return reinterpret_cast<const ReplicateEntity*>(
&_ReplicateEntity_default_instance_);
}
static constexpr int kIndexInFileMessages =
8;
friend void swap(ReplicateEntity& a, ReplicateEntity& b) {
a.Swap(&b);
}
inline void Swap(ReplicateEntity* other) {
if (other == this) return;
#ifdef PROTOBUF_FORCE_COPY_IN_SWAP
if (GetOwningArena() != nullptr &&
GetOwningArena() == other->GetOwningArena()) {
#else // PROTOBUF_FORCE_COPY_IN_SWAP
if (GetOwningArena() == other->GetOwningArena()) {
#endif // !PROTOBUF_FORCE_COPY_IN_SWAP
InternalSwap(other);
} else {
::PROTOBUF_NAMESPACE_ID::internal::GenericSwap(this, other);
}
}
void UnsafeArenaSwap(ReplicateEntity* other) {
if (other == this) return;
GOOGLE_DCHECK(GetOwningArena() == other->GetOwningArena());
InternalSwap(other);
}
// implements Message ----------------------------------------------
ReplicateEntity* New(::PROTOBUF_NAMESPACE_ID::Arena* arena = nullptr) const final {
return CreateMaybeMessage<ReplicateEntity>(arena);
}
using ::PROTOBUF_NAMESPACE_ID::Message::CopyFrom;
void CopyFrom(const ReplicateEntity& from);
using ::PROTOBUF_NAMESPACE_ID::Message::MergeFrom;
void MergeFrom( const ReplicateEntity& from) {
ReplicateEntity::MergeImpl(*this, from);
}
private:
static void MergeImpl(::PROTOBUF_NAMESPACE_ID::Message& to_msg, const ::PROTOBUF_NAMESPACE_ID::Message& from_msg);
public:
PROTOBUF_ATTRIBUTE_REINITIALIZES void Clear() final;
bool IsInitialized() const final;
size_t ByteSizeLong() const final;
const char* _InternalParse(const char* ptr, ::PROTOBUF_NAMESPACE_ID::internal::ParseContext* ctx) final;
uint8_t* _InternalSerialize(
uint8_t* target, ::PROTOBUF_NAMESPACE_ID::io::EpsCopyOutputStream* stream) const final;
int GetCachedSize() const final { return _impl_._cached_size_.Get(); }
private:
void SharedCtor(::PROTOBUF_NAMESPACE_ID::Arena* arena, bool is_message_owned);
void SharedDtor();
void SetCachedSize(int size) const final;
void InternalSwap(ReplicateEntity* other);
private:
friend class ::PROTOBUF_NAMESPACE_ID::internal::AnyMetadata;
static ::PROTOBUF_NAMESPACE_ID::StringPiece FullMessageName() {
return "milvus.proto.common.ReplicateEntity";
}
protected:
explicit ReplicateEntity(::PROTOBUF_NAMESPACE_ID::Arena* arena,
bool is_message_owned = false);
public:
static const ClassData _class_data_;
const ::PROTOBUF_NAMESPACE_ID::Message::ClassData*GetClassData() const final;
::PROTOBUF_NAMESPACE_ID::Metadata GetMetadata() const final;
// nested types ----------------------------------------------------
// accessors -------------------------------------------------------
enum : int {
kIsReplicateFieldNumber = 1,
};
// bool isReplicate = 1;
void clear_isreplicate();
bool isreplicate() const;
void set_isreplicate(bool value);
private:
bool _internal_isreplicate() const;
void _internal_set_isreplicate(bool value);
public:
// @@protoc_insertion_point(class_scope:milvus.proto.common.ReplicateEntity)
private:
class _Internal;
template <typename T> friend class ::PROTOBUF_NAMESPACE_ID::Arena::InternalHelper;
typedef void InternalArenaConstructable_;
typedef void DestructorSkippable_;
struct Impl_ {
bool isreplicate_;
mutable ::PROTOBUF_NAMESPACE_ID::internal::CachedSize _cached_size_;
};
union { Impl_ _impl_; };
friend struct ::TableStruct_common_2eproto;
};
// -------------------------------------------------------------------
class MsgHeader final :
public ::PROTOBUF_NAMESPACE_ID::Message /* @@protoc_insertion_point(class_definition:milvus.proto.common.MsgHeader) */ {
public:
@ -3946,8 +4060,6 @@ inline void Address::set_port(int64_t value) {
// -------------------------------------------------------------------
// -------------------------------------------------------------------
// MsgBase
// .milvus.proto.common.MsgType msg_type = 1;
@ -4050,33 +4162,118 @@ inline void MsgBase::set_targetid(int64_t value) {
// @@protoc_insertion_point(field_set:milvus.proto.common.MsgBase.targetID)
}
// map<string, string> properties = 6;
inline int MsgBase::_internal_properties_size() const {
return _impl_.properties_.size();
// .milvus.proto.common.ReplicateEntity replicate = 6;
inline bool MsgBase::_internal_has_replicate() const {
return this != internal_default_instance() && _impl_.replicate_ != nullptr;
}
inline int MsgBase::properties_size() const {
return _internal_properties_size();
inline bool MsgBase::has_replicate() const {
return _internal_has_replicate();
}
inline void MsgBase::clear_properties() {
_impl_.properties_.Clear();
inline void MsgBase::clear_replicate() {
if (GetArenaForAllocation() == nullptr && _impl_.replicate_ != nullptr) {
delete _impl_.replicate_;
}
_impl_.replicate_ = nullptr;
}
inline const ::PROTOBUF_NAMESPACE_ID::Map< std::string, std::string >&
MsgBase::_internal_properties() const {
return _impl_.properties_.GetMap();
inline const ::milvus::proto::common::ReplicateEntity& MsgBase::_internal_replicate() const {
const ::milvus::proto::common::ReplicateEntity* p = _impl_.replicate_;
return p != nullptr ? *p : reinterpret_cast<const ::milvus::proto::common::ReplicateEntity&>(
::milvus::proto::common::_ReplicateEntity_default_instance_);
}
inline const ::PROTOBUF_NAMESPACE_ID::Map< std::string, std::string >&
MsgBase::properties() const {
// @@protoc_insertion_point(field_map:milvus.proto.common.MsgBase.properties)
return _internal_properties();
inline const ::milvus::proto::common::ReplicateEntity& MsgBase::replicate() const {
// @@protoc_insertion_point(field_get:milvus.proto.common.MsgBase.replicate)
return _internal_replicate();
}
inline ::PROTOBUF_NAMESPACE_ID::Map< std::string, std::string >*
MsgBase::_internal_mutable_properties() {
return _impl_.properties_.MutableMap();
inline void MsgBase::unsafe_arena_set_allocated_replicate(
::milvus::proto::common::ReplicateEntity* replicate) {
if (GetArenaForAllocation() == nullptr) {
delete reinterpret_cast<::PROTOBUF_NAMESPACE_ID::MessageLite*>(_impl_.replicate_);
}
_impl_.replicate_ = replicate;
if (replicate) {
} else {
}
// @@protoc_insertion_point(field_unsafe_arena_set_allocated:milvus.proto.common.MsgBase.replicate)
}
inline ::PROTOBUF_NAMESPACE_ID::Map< std::string, std::string >*
MsgBase::mutable_properties() {
// @@protoc_insertion_point(field_mutable_map:milvus.proto.common.MsgBase.properties)
return _internal_mutable_properties();
inline ::milvus::proto::common::ReplicateEntity* MsgBase::release_replicate() {
::milvus::proto::common::ReplicateEntity* temp = _impl_.replicate_;
_impl_.replicate_ = nullptr;
#ifdef PROTOBUF_FORCE_COPY_IN_RELEASE
auto* old = reinterpret_cast<::PROTOBUF_NAMESPACE_ID::MessageLite*>(temp);
temp = ::PROTOBUF_NAMESPACE_ID::internal::DuplicateIfNonNull(temp);
if (GetArenaForAllocation() == nullptr) { delete old; }
#else // PROTOBUF_FORCE_COPY_IN_RELEASE
if (GetArenaForAllocation() != nullptr) {
temp = ::PROTOBUF_NAMESPACE_ID::internal::DuplicateIfNonNull(temp);
}
#endif // !PROTOBUF_FORCE_COPY_IN_RELEASE
return temp;
}
inline ::milvus::proto::common::ReplicateEntity* MsgBase::unsafe_arena_release_replicate() {
// @@protoc_insertion_point(field_release:milvus.proto.common.MsgBase.replicate)
::milvus::proto::common::ReplicateEntity* temp = _impl_.replicate_;
_impl_.replicate_ = nullptr;
return temp;
}
inline ::milvus::proto::common::ReplicateEntity* MsgBase::_internal_mutable_replicate() {
if (_impl_.replicate_ == nullptr) {
auto* p = CreateMaybeMessage<::milvus::proto::common::ReplicateEntity>(GetArenaForAllocation());
_impl_.replicate_ = p;
}
return _impl_.replicate_;
}
inline ::milvus::proto::common::ReplicateEntity* MsgBase::mutable_replicate() {
::milvus::proto::common::ReplicateEntity* _msg = _internal_mutable_replicate();
// @@protoc_insertion_point(field_mutable:milvus.proto.common.MsgBase.replicate)
return _msg;
}
inline void MsgBase::set_allocated_replicate(::milvus::proto::common::ReplicateEntity* replicate) {
::PROTOBUF_NAMESPACE_ID::Arena* message_arena = GetArenaForAllocation();
if (message_arena == nullptr) {
delete _impl_.replicate_;
}
if (replicate) {
::PROTOBUF_NAMESPACE_ID::Arena* submessage_arena =
::PROTOBUF_NAMESPACE_ID::Arena::InternalGetOwningArena(replicate);
if (message_arena != submessage_arena) {
replicate = ::PROTOBUF_NAMESPACE_ID::internal::GetOwnedMessage(
message_arena, replicate, submessage_arena);
}
} else {
}
_impl_.replicate_ = replicate;
// @@protoc_insertion_point(field_set_allocated:milvus.proto.common.MsgBase.replicate)
}
// -------------------------------------------------------------------
// ReplicateEntity
// bool isReplicate = 1;
inline void ReplicateEntity::clear_isreplicate() {
_impl_.isreplicate_ = false;
}
inline bool ReplicateEntity::_internal_isreplicate() const {
return _impl_.isreplicate_;
}
inline bool ReplicateEntity::isreplicate() const {
// @@protoc_insertion_point(field_get:milvus.proto.common.ReplicateEntity.isReplicate)
return _internal_isreplicate();
}
inline void ReplicateEntity::_internal_set_isreplicate(bool value) {
_impl_.isreplicate_ = value;
}
inline void ReplicateEntity::set_isreplicate(bool value) {
_internal_set_isreplicate(value);
// @@protoc_insertion_point(field_set:milvus.proto.common.ReplicateEntity.isReplicate)
}
// -------------------------------------------------------------------

4
internal/datacoord/server.go
View File

@ -1063,7 +1063,7 @@ func (s *Server) stopServerLoop() {
s.serverLoopWg.Wait()
}
//func (s *Server) validateAllocRequest(collID UniqueID, partID UniqueID, channelName string) error {
// func (s *Server) validateAllocRequest(collID UniqueID, partID UniqueID, channelName string) error {
// if !s.meta.HasCollection(collID) {
// return fmt.Errorf("can not find collection %d", collID)
// }
@ -1076,7 +1076,7 @@ func (s *Server) stopServerLoop() {
// }
// }
// return fmt.Errorf("can not find channel %s", channelName)
//}
// }
// loadCollectionFromRootCoord communicates with RootCoord and asks for collection information.
// collection information will be added to server meta info.

3
internal/datanode/flow_graph_dmstream_input_node_test.go
View File

@ -103,6 +103,9 @@ func (mtm *mockTtMsgStream) CheckTopicValid(channel string) error {
return nil
}
func (mtm *mockTtMsgStream) EnableProduce(can bool) {
}
func TestNewDmInputNode(t *testing.T) {
client := msgdispatcher.NewClient(&mockMsgStreamFactory{}, typeutil.DataNodeRole, paramtable.GetNodeID())
_, err := newDmInputNode(context.Background(), client, new(msgpb.MsgPosition), &nodeConfig{

4
internal/datanode/flow_graph_insert_buffer_node.go
View File

@ -464,6 +464,7 @@ func (ibNode *insertBufferNode) Sync(fgMsg *flowGraphMsg, seg2Upload []UniqueID,
).WithRateGroup("ibNode.sync", 1, 60)
// check if segment is syncing
segment := ibNode.channel.getSegment(task.segmentID)
if !task.dropped && !task.flushed && segment.isSyncing() {
log.RatedInfo(10, "segment is syncing, skip it")
continue
@ -739,7 +740,8 @@ func newInsertBufferNode(
wTt.AsProducer([]string{Params.CommonCfg.DataCoordTimeTick.GetValue()})
metrics.DataNodeNumProducers.WithLabelValues(fmt.Sprint(paramtable.GetNodeID())).Inc()
log.Info("datanode AsProducer", zap.String("TimeTickChannelName", Params.CommonCfg.DataCoordTimeTick.GetValue()))
var wTtMsgStream msgstream.MsgStream = wTt
wTtMsgStream := wTt
wTtMsgStream.EnableProduce(true)
mt := newMergedTimeTickerSender(func(ts Timestamp, segmentIDs []int64) error {
stats := make([]*commonpb.SegmentStats, 0, len(segmentIDs))

4
internal/datanode/flush_task.go
View File

@ -63,7 +63,7 @@ type flushTaskRunner struct {
segmentID UniqueID
insertLogs map[UniqueID]*datapb.Binlog
statsLogs map[UniqueID]*datapb.Binlog
deltaLogs []*datapb.Binlog //[]*DelDataBuf
deltaLogs []*datapb.Binlog // []*DelDataBuf
pos *msgpb.MsgPosition
flushed bool
dropped bool
@ -156,7 +156,7 @@ func (t *flushTaskRunner) runFlushInsert(task flushInsertTask,
func (t *flushTaskRunner) runFlushDel(task flushDeleteTask, deltaLogs *DelDataBuf, opts ...retry.Option) {
t.deleteOnce.Do(func() {
if deltaLogs == nil {
t.deltaLogs = nil //[]*DelDataBuf{}
t.deltaLogs = nil // []*DelDataBuf{}
} else {
t.deltaLogs = []*datapb.Binlog{
{

4
internal/distributed/proxy/service.go
View File

@ -1109,3 +1109,7 @@ func (s *Server) ListDatabases(ctx context.Context, request *milvuspb.ListDataba
func (s *Server) AllocTimestamp(ctx context.Context, req *milvuspb.AllocTimestampRequest) (*milvuspb.AllocTimestampResponse, error) {
return s.proxy.AllocTimestamp(ctx, req)
}
func (s *Server) ReplicateMessage(ctx context.Context, req *milvuspb.ReplicateMessageRequest) (*milvuspb.ReplicateMessageResponse, error) {
return s.proxy.ReplicateMessage(ctx, req)
}

6
internal/distributed/proxy/service_test.go
View File

@ -508,7 +508,11 @@ func (m *MockProxy) AllocTimestamp(ctx context.Context, req *milvuspb.AllocTimes
return nil, nil
}
///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
func (m *MockProxy) ReplicateMessage(ctx context.Context, req *milvuspb.ReplicateMessageRequest) (*milvuspb.ReplicateMessageResponse, error) {
return nil, nil
}
// /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
type WaitOption struct {
Duration time.Duration `json:"duration"`

55
internal/mocks/mock_proxy.go
View File

@ -4445,6 +4445,61 @@ func (_c *MockProxy_RenameCollection_Call) RunAndReturn(run func(context.Context
return _c
}
// ReplicateMessage provides a mock function with given fields: ctx, req
func (_m *MockProxy) ReplicateMessage(ctx context.Context, req *milvuspb.ReplicateMessageRequest) (*milvuspb.ReplicateMessageResponse, error) {
ret := _m.Called(ctx, req)
var r0 *milvuspb.ReplicateMessageResponse
var r1 error
if rf, ok := ret.Get(0).(func(context.Context, *milvuspb.ReplicateMessageRequest) (*milvuspb.ReplicateMessageResponse, error)); ok {
return rf(ctx, req)
}
if rf, ok := ret.Get(0).(func(context.Context, *milvuspb.ReplicateMessageRequest) *milvuspb.ReplicateMessageResponse); ok {
r0 = rf(ctx, req)
} else {
if ret.Get(0) != nil {
r0 = ret.Get(0).(*milvuspb.ReplicateMessageResponse)
}
}
if rf, ok := ret.Get(1).(func(context.Context, *milvuspb.ReplicateMessageRequest) error); ok {
r1 = rf(ctx, req)
} else {
r1 = ret.Error(1)
}
return r0, r1
}
// MockProxy_ReplicateMessage_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'ReplicateMessage'
type MockProxy_ReplicateMessage_Call struct {
*mock.Call
}
// ReplicateMessage is a helper method to define mock.On call
// - ctx context.Context
// - req *milvuspb.ReplicateMessageRequest
func (_e *MockProxy_Expecter) ReplicateMessage(ctx interface{}, req interface{}) *MockProxy_ReplicateMessage_Call {
return &MockProxy_ReplicateMessage_Call{Call: _e.mock.On("ReplicateMessage", ctx, req)}
}
func (_c *MockProxy_ReplicateMessage_Call) Run(run func(ctx context.Context, req *milvuspb.ReplicateMessageRequest)) *MockProxy_ReplicateMessage_Call {
_c.Call.Run(func(args mock.Arguments) {
run(args[0].(context.Context), args[1].(*milvuspb.ReplicateMessageRequest))
})
return _c
}
func (_c *MockProxy_ReplicateMessage_Call) Return(_a0 *milvuspb.ReplicateMessageResponse, _a1 error) *MockProxy_ReplicateMessage_Call {
_c.Call.Return(_a0, _a1)
return _c
}
func (_c *MockProxy_ReplicateMessage_Call) RunAndReturn(run func(context.Context, *milvuspb.ReplicateMessageRequest) (*milvuspb.ReplicateMessageResponse, error)) *MockProxy_ReplicateMessage_Call {
_c.Call.Return(run)
return _c
}
// Search provides a mock function with given fields: _a0, _a1
func (_m *MockProxy) Search(_a0 context.Context, _a1 *milvuspb.SearchRequest) (*milvuspb.SearchResults, error) {
ret := _m.Called(_a0, _a1)

3
internal/proto/planpb/plan.pb.go
View File

@ -237,6 +237,7 @@ func (BinaryExpr_BinaryOp) EnumDescriptor() ([]byte, []int) {
type GenericValue struct {
// Types that are valid to be assigned to Val:
//
// *GenericValue_BoolVal
// *GenericValue_Int64Val
// *GenericValue_FloatVal
@ -1297,6 +1298,7 @@ var xxx_messageInfo_AlwaysTrueExpr proto.InternalMessageInfo
type Expr struct {
// Types that are valid to be assigned to Expr:
//
// *Expr_TermExpr
// *Expr_UnaryExpr
// *Expr_BinaryExpr
@ -1668,6 +1670,7 @@ func (m *QueryPlanNode) GetLimit() int64 {
type PlanNode struct {
// Types that are valid to be assigned to Node:
//
// *PlanNode_VectorAnns
// *PlanNode_Predicates
// *PlanNode_Query

5
internal/proto/proxypb/proxy.pb.go
View File

@ -29,8 +29,9 @@ const _ = proto.ProtoPackageIsVersion3 // please upgrade the proto package
type InvalidateCollMetaCacheRequest struct {
// MsgType:
// DropCollection -> {meta cache, dml channels}
// Other -> {meta cache}
//
// DropCollection -> {meta cache, dml channels}
// Other -> {meta cache}
Base *commonpb.MsgBase `protobuf:"bytes,1,opt,name=base,proto3" json:"base,omitempty"`
DbName string `protobuf:"bytes,2,opt,name=db_name,json=dbName,proto3" json:"db_name,omitempty"`
CollectionName string `protobuf:"bytes,3,opt,name=collection_name,json=collectionName,proto3" json:"collection_name,omitempty"`

40
internal/proto/rootcoordpb/root_coord.pb.go
View File

@ -793,28 +793,28 @@ type RootCoordClient interface {
GetComponentStates(ctx context.Context, in *milvuspb.GetComponentStatesRequest, opts ...grpc.CallOption) (*milvuspb.ComponentStates, error)
GetTimeTickChannel(ctx context.Context, in *internalpb.GetTimeTickChannelRequest, opts ...grpc.CallOption) (*milvuspb.StringResponse, error)
GetStatisticsChannel(ctx context.Context, in *internalpb.GetStatisticsChannelRequest, opts ...grpc.CallOption) (*milvuspb.StringResponse, error)
//*
// *
// @brief This method is used to create collection
//
// @param CreateCollectionRequest, use to provide collection information to be created.
//
// @return Status
CreateCollection(ctx context.Context, in *milvuspb.CreateCollectionRequest, opts ...grpc.CallOption) (*commonpb.Status, error)
//*
// *
// @brief This method is used to delete collection.
//
// @param DropCollectionRequest, collection name is going to be deleted.
//
// @return Status
DropCollection(ctx context.Context, in *milvuspb.DropCollectionRequest, opts ...grpc.CallOption) (*commonpb.Status, error)
//*
// *
// @brief This method is used to test collection existence.
//
// @param HasCollectionRequest, collection name is going to be tested.
//
// @return BoolResponse
HasCollection(ctx context.Context, in *milvuspb.HasCollectionRequest, opts ...grpc.CallOption) (*milvuspb.BoolResponse, error)
//*
// *
// @brief This method is used to get collection schema.
//
// @param DescribeCollectionRequest, target collection name.
@ -825,28 +825,28 @@ type RootCoordClient interface {
CreateAlias(ctx context.Context, in *milvuspb.CreateAliasRequest, opts ...grpc.CallOption) (*commonpb.Status, error)
DropAlias(ctx context.Context, in *milvuspb.DropAliasRequest, opts ...grpc.CallOption) (*commonpb.Status, error)
AlterAlias(ctx context.Context, in *milvuspb.AlterAliasRequest, opts ...grpc.CallOption) (*commonpb.Status, error)
//*
// *
// @brief This method is used to list all collections.
//
// @return StringListResponse, collection name list
ShowCollections(ctx context.Context, in *milvuspb.ShowCollectionsRequest, opts ...grpc.CallOption) (*milvuspb.ShowCollectionsResponse, error)
AlterCollection(ctx context.Context, in *milvuspb.AlterCollectionRequest, opts ...grpc.CallOption) (*commonpb.Status, error)
//*
// *
// @brief This method is used to create partition
//
// @return Status
CreatePartition(ctx context.Context, in *milvuspb.CreatePartitionRequest, opts ...grpc.CallOption) (*commonpb.Status, error)
//*
// *
// @brief This method is used to drop partition
//
// @return Status
DropPartition(ctx context.Context, in *milvuspb.DropPartitionRequest, opts ...grpc.CallOption) (*commonpb.Status, error)
//*
// *
// @brief This method is used to test partition existence.
//
// @return BoolResponse
HasPartition(ctx context.Context, in *milvuspb.HasPartitionRequest, opts ...grpc.CallOption) (*milvuspb.BoolResponse, error)
//*
// *
// @brief This method is used to show partition information
//
// @param ShowPartitionRequest, target collection name.
@ -854,7 +854,7 @@ type RootCoordClient interface {
// @return StringListResponse
ShowPartitions(ctx context.Context, in *milvuspb.ShowPartitionsRequest, opts ...grpc.CallOption) (*milvuspb.ShowPartitionsResponse, error)
ShowPartitionsInternal(ctx context.Context, in *milvuspb.ShowPartitionsRequest, opts ...grpc.CallOption) (*milvuspb.ShowPartitionsResponse, error)
// rpc DescribeSegment(milvus.DescribeSegmentRequest) returns (milvus.DescribeSegmentResponse) {}
// rpc DescribeSegment(milvus.DescribeSegmentRequest) returns (milvus.DescribeSegmentResponse) {}
ShowSegments(ctx context.Context, in *milvuspb.ShowSegmentsRequest, opts ...grpc.CallOption) (*milvuspb.ShowSegmentsResponse, error)
AllocTimestamp(ctx context.Context, in *AllocTimestampRequest, opts ...grpc.CallOption) (*AllocTimestampResponse, error)
AllocID(ctx context.Context, in *AllocIDRequest, opts ...grpc.CallOption) (*AllocIDResponse, error)
@ -1327,28 +1327,28 @@ type RootCoordServer interface {
GetComponentStates(context.Context, *milvuspb.GetComponentStatesRequest) (*milvuspb.ComponentStates, error)
GetTimeTickChannel(context.Context, *internalpb.GetTimeTickChannelRequest) (*milvuspb.StringResponse, error)
GetStatisticsChannel(context.Context, *internalpb.GetStatisticsChannelRequest) (*milvuspb.StringResponse, error)
//*
// *
// @brief This method is used to create collection
//
// @param CreateCollectionRequest, use to provide collection information to be created.
//
// @return Status
CreateCollection(context.Context, *milvuspb.CreateCollectionRequest) (*commonpb.Status, error)
//*
// *
// @brief This method is used to delete collection.
//
// @param DropCollectionRequest, collection name is going to be deleted.
//
// @return Status
DropCollection(context.Context, *milvuspb.DropCollectionRequest) (*commonpb.Status, error)
//*
// *
// @brief This method is used to test collection existence.
//
// @param HasCollectionRequest, collection name is going to be tested.
//
// @return BoolResponse
HasCollection(context.Context, *milvuspb.HasCollectionRequest) (*milvuspb.BoolResponse, error)
//*
// *
// @brief This method is used to get collection schema.
//
// @param DescribeCollectionRequest, target collection name.
@ -1359,28 +1359,28 @@ type RootCoordServer interface {
CreateAlias(context.Context, *milvuspb.CreateAliasRequest) (*commonpb.Status, error)
DropAlias(context.Context, *milvuspb.DropAliasRequest) (*commonpb.Status, error)
AlterAlias(context.Context, *milvuspb.AlterAliasRequest) (*commonpb.Status, error)
//*
// *
// @brief This method is used to list all collections.
//
// @return StringListResponse, collection name list
ShowCollections(context.Context, *milvuspb.ShowCollectionsRequest) (*milvuspb.ShowCollectionsResponse, error)
AlterCollection(context.Context, *milvuspb.AlterCollectionRequest) (*commonpb.Status, error)
//*
// *
// @brief This method is used to create partition
//
// @return Status
CreatePartition(context.Context, *milvuspb.CreatePartitionRequest) (*commonpb.Status, error)
//*
// *
// @brief This method is used to drop partition
//
// @return Status
DropPartition(context.Context, *milvuspb.DropPartitionRequest) (*commonpb.Status, error)
//*
// *
// @brief This method is used to test partition existence.
//
// @return BoolResponse
HasPartition(context.Context, *milvuspb.HasPartitionRequest) (*milvuspb.BoolResponse, error)
//*
// *
// @brief This method is used to show partition information
//
// @param ShowPartitionRequest, target collection name.
@ -1388,7 +1388,7 @@ type RootCoordServer interface {
// @return StringListResponse
ShowPartitions(context.Context, *milvuspb.ShowPartitionsRequest) (*milvuspb.ShowPartitionsResponse, error)
ShowPartitionsInternal(context.Context, *milvuspb.ShowPartitionsRequest) (*milvuspb.ShowPartitionsResponse, error)
// rpc DescribeSegment(milvus.DescribeSegmentRequest) returns (milvus.DescribeSegmentResponse) {}
// rpc DescribeSegment(milvus.DescribeSegmentRequest) returns (milvus.DescribeSegmentResponse) {}
ShowSegments(context.Context, *milvuspb.ShowSegmentsRequest) (*milvuspb.ShowSegmentsResponse, error)
AllocTimestamp(context.Context, *AllocTimestampRequest) (*AllocTimestampResponse, error)
AllocID(context.Context, *AllocIDRequest) (*AllocIDResponse, error)

135
internal/proxy/impl.go
View File

@ -18,6 +18,7 @@ package proxy
import (
"context"
"encoding/base64"
"fmt"
"os"
"strconv"
@ -156,6 +157,7 @@ func (node *Proxy) CreateDatabase(ctx context.Context, request *milvuspb.CreateD
Condition: NewTaskCondition(ctx),
CreateDatabaseRequest: request,
rootCoord: node.rootCoord,
replicateMsgStream: node.replicateMsgStream,
}
log := log.With(
@ -216,6 +218,7 @@ func (node *Proxy) DropDatabase(ctx context.Context, request *milvuspb.DropDatab
Condition: NewTaskCondition(ctx),
DropDatabaseRequest: request,
rootCoord: node.rootCoord,
replicateMsgStream: node.replicateMsgStream,
}
log := log.With(
@ -580,6 +583,7 @@ func (node *Proxy) LoadCollection(ctx context.Context, request *milvuspb.LoadCol
LoadCollectionRequest: request,
queryCoord: node.queryCoord,
datacoord: node.dataCoord,
replicateMsgStream: node.replicateMsgStream,
}
log := log.Ctx(ctx).With(
@ -652,7 +656,7 @@ func (node *Proxy) ReleaseCollection(ctx context.Context, request *milvuspb.Rele
Condition: NewTaskCondition(ctx),
ReleaseCollectionRequest: request,
queryCoord: node.queryCoord,
chMgr: node.chMgr,
replicateMsgStream: node.replicateMsgStream,
}
log := log.Ctx(ctx).With(
@ -1746,11 +1750,12 @@ func (node *Proxy) CreateIndex(ctx context.Context, request *milvuspb.CreateInde
defer sp.End()
cit := &createIndexTask{
ctx: ctx,
Condition: NewTaskCondition(ctx),
req: request,
rootCoord: node.rootCoord,
datacoord: node.dataCoord,
ctx: ctx,
Condition: NewTaskCondition(ctx),
req: request,
rootCoord: node.rootCoord,
datacoord: node.dataCoord,
replicateMsgStream: node.replicateMsgStream,
}
method := "CreateIndex"
@ -1964,11 +1969,12 @@ func (node *Proxy) DropIndex(ctx context.Context, request *milvuspb.DropIndexReq
defer sp.End()
dit := &dropIndexTask{
ctx: ctx,
Condition: NewTaskCondition(ctx),
DropIndexRequest: request,
dataCoord: node.dataCoord,
queryCoord: node.queryCoord,
ctx: ctx,
Condition: NewTaskCondition(ctx),
DropIndexRequest: request,
dataCoord: node.dataCoord,
queryCoord: node.queryCoord,
replicateMsgStream: node.replicateMsgStream,
}
method := "DropIndex"
@ -2710,10 +2716,11 @@ func (node *Proxy) Flush(ctx context.Context, request *milvuspb.FlushRequest) (*
defer sp.End()
ft := &flushTask{
ctx: ctx,
Condition: NewTaskCondition(ctx),
FlushRequest: request,
dataCoord: node.dataCoord,
ctx: ctx,
Condition: NewTaskCondition(ctx),
FlushRequest: request,
dataCoord: node.dataCoord,
replicateMsgStream: node.replicateMsgStream,
}
method := "Flush"
@ -5023,6 +5030,104 @@ func (node *Proxy) Connect(ctx context.Context, request *milvuspb.ConnectRequest
}, nil
}
func (node *Proxy) ReplicateMessage(ctx context.Context, req *milvuspb.ReplicateMessageRequest) (*milvuspb.ReplicateMessageResponse, error) {
if err := merr.CheckHealthy(node.GetStateCode()); err != nil {
return &milvuspb.ReplicateMessageResponse{Status: merr.Status(err)}, nil
}
if paramtable.Get().CommonCfg.TTMsgEnabled.GetAsBool() {
return &milvuspb.ReplicateMessageResponse{
Status: merr.Status(merr.ErrDenyReplicateMessage),
}, nil
}
var err error
ctxLog := log.Ctx(ctx)
if req.GetChannelName() == "" {
ctxLog.Warn("channel name is empty")
return &milvuspb.ReplicateMessageResponse{
Status: merr.Status(merr.WrapErrParameterInvalidMsg("invalid channel name for the replicate message request")),
}, nil
}
// get the latest position of the replicate msg channel
replicateMsgChannel := Params.CommonCfg.ReplicateMsgChannel.GetValue()
if req.GetChannelName() == replicateMsgChannel {
msgID, err := msgstream.GetChannelLatestMsgID(ctx, node.factory, replicateMsgChannel)
if err != nil {
ctxLog.Warn("failed to get the latest message id of the replicate msg channel", zap.Error(err))
return &milvuspb.ReplicateMessageResponse{Status: merr.Status(err)}, nil
}
position := base64.StdEncoding.EncodeToString(msgID)
return &milvuspb.ReplicateMessageResponse{Status: merr.Status(nil), Position: position}, nil
}
msgPack := &msgstream.MsgPack{
BeginTs: req.BeginTs,
EndTs: req.EndTs,
Msgs: make([]msgstream.TsMsg, 0),
StartPositions: req.StartPositions,
EndPositions: req.EndPositions,
}
// getTsMsgFromConsumerMsg
for i, msgBytes := range req.Msgs {
header := commonpb.MsgHeader{}
err = proto.Unmarshal(msgBytes, &header)
if err != nil {
ctxLog.Warn("failed to unmarshal msg header", zap.Int("index", i), zap.Error(err))
return &milvuspb.ReplicateMessageResponse{Status: merr.Status(err)}, nil
}
if header.GetBase() == nil {
ctxLog.Warn("msg header base is nil", zap.Int("index", i))
return &milvuspb.ReplicateMessageResponse{Status: merr.Status(merr.ErrInvalidMsgBytes)}, nil
}
tsMsg, err := node.replicateStreamManager.GetMsgDispatcher().Unmarshal(msgBytes, header.GetBase().GetMsgType())
if err != nil {
ctxLog.Warn("failed to unmarshal msg", zap.Int("index", i), zap.Error(err))
return &milvuspb.ReplicateMessageResponse{Status: merr.Status(merr.ErrInvalidMsgBytes)}, nil
}
switch realMsg := tsMsg.(type) {
case *msgstream.InsertMsg:
assignedSegmentInfos, err := node.segAssigner.GetSegmentID(realMsg.GetCollectionID(), realMsg.GetPartitionID(),
realMsg.GetShardName(), uint32(realMsg.NumRows), req.EndTs)
if err != nil {
ctxLog.Warn("failed to get segment id", zap.Error(err))
return &milvuspb.ReplicateMessageResponse{Status: merr.Status(err)}, nil
}
if len(assignedSegmentInfos) == 0 {
ctxLog.Warn("no segment id assigned")
return &milvuspb.ReplicateMessageResponse{Status: merr.Status(merr.ErrNoAssignSegmentID)}, nil
}
for assignSegmentID := range assignedSegmentInfos {
realMsg.SegmentID = assignSegmentID
break
}
}
msgPack.Msgs = append(msgPack.Msgs, tsMsg)
}
msgStream, err := node.replicateStreamManager.GetReplicateMsgStream(ctx, req.ChannelName)
if err != nil {
ctxLog.Warn("failed to get msg stream from the replicate stream manager", zap.Error(err))
return &milvuspb.ReplicateMessageResponse{
Status: merr.Status(err),
}, nil
}
messageIDsMap, err := msgStream.Broadcast(msgPack)
if err != nil {
ctxLog.Warn("failed to produce msg", zap.Error(err))
return &milvuspb.ReplicateMessageResponse{Status: merr.Status(err)}, nil
}
var position string
if len(messageIDsMap[req.GetChannelName()]) == 0 {
ctxLog.Warn("no message id returned")
} else {
messageIDs := messageIDsMap[req.GetChannelName()]
position = base64.StdEncoding.EncodeToString(messageIDs[len(messageIDs)-1].Serialize())
}
return &milvuspb.ReplicateMessageResponse{Status: merr.Status(nil), Position: position}, nil
}
func (node *Proxy) ListClientInfos(ctx context.Context, req *proxypb.ListClientInfosRequest) (*proxypb.ListClientInfosResponse, error) {
if err := merr.CheckHealthy(node.GetStateCode()); err != nil {
return &proxypb.ListClientInfosResponse{Status: merr.Status(err)}, nil

285
internal/proxy/impl_test.go
View File

@ -18,16 +18,20 @@ package proxy
import (
"context"
"encoding/base64"
"testing"
"time"
"github.com/cockroachdb/errors"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/mock"
"go.uber.org/zap"
"google.golang.org/grpc"
"google.golang.org/grpc/metadata"
"github.com/milvus-io/milvus-proto/go-api/v2/commonpb"
"github.com/milvus-io/milvus-proto/go-api/v2/milvuspb"
"github.com/milvus-io/milvus-proto/go-api/v2/msgpb"
"github.com/milvus-io/milvus/internal/mocks"
"github.com/milvus-io/milvus/internal/proto/datapb"
"github.com/milvus-io/milvus/internal/proto/proxypb"
@ -35,8 +39,13 @@ import (
"github.com/milvus-io/milvus/internal/proto/rootcoordpb"
"github.com/milvus-io/milvus/internal/util/dependency"
"github.com/milvus-io/milvus/internal/util/sessionutil"
"github.com/milvus-io/milvus/pkg/log"
"github.com/milvus-io/milvus/pkg/mq/msgstream"
"github.com/milvus-io/milvus/pkg/mq/msgstream/mqwrapper"
"github.com/milvus-io/milvus/pkg/util/commonpbutil"
"github.com/milvus-io/milvus/pkg/util/merr"
"github.com/milvus-io/milvus/pkg/util/paramtable"
"github.com/milvus-io/milvus/pkg/util/resource"
)
func TestProxy_InvalidateCollectionMetaCache_remove_stream(t *testing.T) {
@ -402,6 +411,10 @@ func TestProxy_FlushAll_DbCollection(t *testing.T) {
node.tsoAllocator = &timestampAllocator{
tso: newMockTimestampAllocatorInterface(),
}
rpcRequestChannel := Params.CommonCfg.ReplicateMsgChannel.GetValue()
node.replicateMsgStream, err = node.factory.NewMsgStream(node.ctx)
assert.NoError(t, err)
node.replicateMsgStream.AsProducer([]string{rpcRequestChannel})
Params.Save(Params.ProxyCfg.MaxTaskNum.Key, "1000")
node.sched, err = newTaskScheduler(ctx, node.tsoAllocator, node.factory)
@ -441,6 +454,10 @@ func TestProxy_FlushAll(t *testing.T) {
node.tsoAllocator = &timestampAllocator{
tso: newMockTimestampAllocatorInterface(),
}
rpcRequestChannel := Params.CommonCfg.ReplicateMsgChannel.GetValue()
node.replicateMsgStream, err = node.factory.NewMsgStream(node.ctx)
assert.NoError(t, err)
node.replicateMsgStream.AsProducer([]string{rpcRequestChannel})
Params.Save(Params.ProxyCfg.MaxTaskNum.Key, "1000")
node.sched, err = newTaskScheduler(ctx, node.tsoAllocator, node.factory)
@ -909,6 +926,11 @@ func TestProxyCreateDatabase(t *testing.T) {
assert.NoError(t, err)
defer node.sched.Close()
rpcRequestChannel := Params.CommonCfg.ReplicateMsgChannel.GetValue()
node.replicateMsgStream, err = node.factory.NewMsgStream(node.ctx)
assert.NoError(t, err)
node.replicateMsgStream.AsProducer([]string{rpcRequestChannel})
t.Run("create database fail", func(t *testing.T) {
rc := mocks.NewMockRootCoordClient(t)
rc.On("CreateDatabase", mock.Anything, mock.Anything).
@ -963,6 +985,11 @@ func TestProxyDropDatabase(t *testing.T) {
assert.NoError(t, err)
defer node.sched.Close()
rpcRequestChannel := Params.CommonCfg.ReplicateMsgChannel.GetValue()
node.replicateMsgStream, err = node.factory.NewMsgStream(node.ctx)
assert.NoError(t, err)
node.replicateMsgStream.AsProducer([]string{rpcRequestChannel})
t.Run("drop database fail", func(t *testing.T) {
rc := mocks.NewMockRootCoordClient(t)
rc.On("DropDatabase", mock.Anything, mock.Anything).
@ -1092,3 +1119,261 @@ func TestProxy_AllocTimestamp(t *testing.T) {
assert.Equal(t, commonpb.ErrorCode_UnexpectedError, resp.GetStatus().GetErrorCode())
})
}
func TestProxy_ReplicateMessage(t *testing.T) {
paramtable.Init()
defer paramtable.Get().Save(paramtable.Get().CommonCfg.TTMsgEnabled.Key, "true")
t.Run("proxy unhealthy", func(t *testing.T) {
node := &Proxy{}
node.UpdateStateCode(commonpb.StateCode_Abnormal)
resp, err := node.ReplicateMessage(context.TODO(), nil)
assert.NoError(t, err)
assert.NotEqual(t, 0, resp.GetStatus().GetCode())
})
t.Run("not backup instance", func(t *testing.T) {
node := &Proxy{}
node.UpdateStateCode(commonpb.StateCode_Healthy)
resp, err := node.ReplicateMessage(context.TODO(), nil)
assert.NoError(t, err)
assert.NotEqual(t, 0, resp.GetStatus().GetCode())
})
t.Run("empty channel name", func(t *testing.T) {
node := &Proxy{}
node.UpdateStateCode(commonpb.StateCode_Healthy)
paramtable.Get().Save(paramtable.Get().CommonCfg.TTMsgEnabled.Key, "false")
resp, err := node.ReplicateMessage(context.TODO(), nil)
assert.NoError(t, err)
assert.NotEqual(t, 0, resp.GetStatus().GetCode())
})
t.Run("fail to get msg stream", func(t *testing.T) {
factory := newMockMsgStreamFactory()
factory.f = func(ctx context.Context) (msgstream.MsgStream, error) {
return nil, errors.New("mock error: get msg stream")
}
resourceManager := resource.NewManager(time.Second, 2*time.Second, nil)
manager := NewReplicateStreamManager(context.Background(), factory, resourceManager)
node := &Proxy{
replicateStreamManager: manager,
}
node.UpdateStateCode(commonpb.StateCode_Healthy)
paramtable.Get().Save(paramtable.Get().CommonCfg.TTMsgEnabled.Key, "false")
resp, err := node.ReplicateMessage(context.TODO(), &milvuspb.ReplicateMessageRequest{ChannelName: "unit_test_replicate_message"})
assert.NoError(t, err)
assert.NotEqual(t, 0, resp.GetStatus().GetCode())
})
t.Run("get latest position", func(t *testing.T) {
paramtable.Get().Save(paramtable.Get().CommonCfg.TTMsgEnabled.Key, "false")
defer paramtable.Get().Save(paramtable.Get().CommonCfg.TTMsgEnabled.Key, "true")
factory := dependency.NewMockFactory(t)
stream := msgstream.NewMockMsgStream(t)
mockMsgID := mqwrapper.NewMockMessageID(t)
factory.EXPECT().NewMsgStream(mock.Anything).Return(stream, nil).Once()
mockMsgID.EXPECT().Serialize().Return([]byte("mock")).Once()
stream.EXPECT().AsConsumer(mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(nil).Once()
stream.EXPECT().GetLatestMsgID(mock.Anything).Return(mockMsgID, nil).Once()
stream.EXPECT().Close().Return()
node := &Proxy{
factory: factory,
}
node.UpdateStateCode(commonpb.StateCode_Healthy)
resp, err := node.ReplicateMessage(context.TODO(), &milvuspb.ReplicateMessageRequest{
ChannelName: Params.CommonCfg.ReplicateMsgChannel.GetValue(),
})
assert.NoError(t, err)
assert.EqualValues(t, 0, resp.GetStatus().GetCode())
assert.Equal(t, base64.StdEncoding.EncodeToString([]byte("mock")), resp.GetPosition())
factory.EXPECT().NewMsgStream(mock.Anything).Return(nil, errors.New("mock")).Once()
resp, err = node.ReplicateMessage(context.TODO(), &milvuspb.ReplicateMessageRequest{
ChannelName: Params.CommonCfg.ReplicateMsgChannel.GetValue(),
})
assert.NoError(t, err)
assert.NotEqualValues(t, 0, resp.GetStatus().GetCode())
})
t.Run("invalid msg pack", func(t *testing.T) {
node := &Proxy{
replicateStreamManager: NewReplicateStreamManager(context.Background(), nil, nil),
}
node.UpdateStateCode(commonpb.StateCode_Healthy)
paramtable.Get().Save(paramtable.Get().CommonCfg.TTMsgEnabled.Key, "false")
{
resp, err := node.ReplicateMessage(context.TODO(), &milvuspb.ReplicateMessageRequest{
ChannelName: "unit_test_replicate_message",
Msgs: [][]byte{{1, 2, 3}},
})
assert.NoError(t, err)
assert.NotEqual(t, 0, resp.GetStatus().GetCode())
}
{
timeTickResult := msgpb.TimeTickMsg{}
timeTickMsg := &msgstream.TimeTickMsg{
BaseMsg: msgstream.BaseMsg{
BeginTimestamp: 1,
EndTimestamp: 10,
HashValues: []uint32{0},
},
TimeTickMsg: timeTickResult,
}
msgBytes, _ := timeTickMsg.Marshal(timeTickMsg)
resp, err := node.ReplicateMessage(context.TODO(), &milvuspb.ReplicateMessageRequest{
ChannelName: "unit_test_replicate_message",
Msgs: [][]byte{msgBytes.([]byte)},
})
assert.NoError(t, err)
log.Info("resp", zap.Any("resp", resp))
assert.NotEqual(t, 0, resp.GetStatus().GetCode())
}
{
timeTickResult := msgpb.TimeTickMsg{
Base: commonpbutil.NewMsgBase(
commonpbutil.WithMsgType(commonpb.MsgType(-1)),
commonpbutil.WithMsgID(0),
commonpbutil.WithTimeStamp(10),
commonpbutil.WithSourceID(-1),
),
}
timeTickMsg := &msgstream.TimeTickMsg{
BaseMsg: msgstream.BaseMsg{
BeginTimestamp: 1,
EndTimestamp: 10,
HashValues: []uint32{0},
},
TimeTickMsg: timeTickResult,
}
msgBytes, _ := timeTickMsg.Marshal(timeTickMsg)
resp, err := node.ReplicateMessage(context.TODO(), &milvuspb.ReplicateMessageRequest{
ChannelName: "unit_test_replicate_message",
Msgs: [][]byte{msgBytes.([]byte)},
})
assert.NoError(t, err)
log.Info("resp", zap.Any("resp", resp))
assert.NotEqual(t, 0, resp.GetStatus().GetCode())
}
})
t.Run("success", func(t *testing.T) {
paramtable.Init()
factory := newMockMsgStreamFactory()
msgStreamObj := msgstream.NewMockMsgStream(t)
msgStreamObj.EXPECT().SetRepackFunc(mock.Anything).Return()
msgStreamObj.EXPECT().AsProducer(mock.Anything).Return()
msgStreamObj.EXPECT().EnableProduce(mock.Anything).Return()
msgStreamObj.EXPECT().Close().Return()
mockMsgID1 := mqwrapper.NewMockMessageID(t)
mockMsgID2 := mqwrapper.NewMockMessageID(t)
mockMsgID2.EXPECT().Serialize().Return([]byte("mock message id 2"))
broadcastMock := msgStreamObj.EXPECT().Broadcast(mock.Anything).Return(map[string][]mqwrapper.MessageID{
"unit_test_replicate_message": {mockMsgID1, mockMsgID2},
}, nil)
factory.f = func(ctx context.Context) (msgstream.MsgStream, error) {
return msgStreamObj, nil
}
resourceManager := resource.NewManager(time.Second, 2*time.Second, nil)
manager := NewReplicateStreamManager(context.Background(), factory, resourceManager)
ctx := context.Background()
dataCoord := &mockDataCoord{}
dataCoord.expireTime = Timestamp(1000)
segAllocator, err := newSegIDAssigner(ctx, dataCoord, getLastTick1)
assert.NoError(t, err)
segAllocator.Start()
node := &Proxy{
replicateStreamManager: manager,
segAssigner: segAllocator,
}
node.UpdateStateCode(commonpb.StateCode_Healthy)
paramtable.Get().Save(paramtable.Get().CommonCfg.TTMsgEnabled.Key, "false")
insertMsg := &msgstream.InsertMsg{
BaseMsg: msgstream.BaseMsg{
BeginTimestamp: 4,
EndTimestamp: 10,
HashValues: []uint32{0},
MsgPosition: &msgstream.MsgPosition{
ChannelName: "unit_test_replicate_message",
MsgID: []byte("mock message id 2"),
},
},
InsertRequest: msgpb.InsertRequest{
Base: &commonpb.MsgBase{
MsgType: commonpb.MsgType_Insert,
MsgID: 10001,
Timestamp: 10,
SourceID: -1,
},
ShardName: "unit_test_replicate_message_v1",
DbName: "default",
CollectionName: "foo_collection",
PartitionName: "_default",
DbID: 1,
CollectionID: 11,
PartitionID: 22,
SegmentID: 33,
Timestamps: []uint64{10},
RowIDs: []int64{66},
NumRows: 1,
},
}
msgBytes, _ := insertMsg.Marshal(insertMsg)
replicateRequest := &milvuspb.ReplicateMessageRequest{
ChannelName: "unit_test_replicate_message",
BeginTs: 1,
EndTs: 10,
Msgs: [][]byte{msgBytes.([]byte)},
StartPositions: []*msgpb.MsgPosition{
{ChannelName: "unit_test_replicate_message", MsgID: []byte("mock message id 1")},
},
EndPositions: []*msgpb.MsgPosition{
{ChannelName: "unit_test_replicate_message", MsgID: []byte("mock message id 2")},
},
}
resp, err := node.ReplicateMessage(context.TODO(), replicateRequest)
assert.NoError(t, err)
assert.EqualValues(t, 0, resp.GetStatus().GetCode())
assert.Equal(t, base64.StdEncoding.EncodeToString([]byte("mock message id 2")), resp.GetPosition())
res := resourceManager.Delete(ReplicateMsgStreamTyp, replicateRequest.GetChannelName())
assert.NotNil(t, res)
time.Sleep(2 * time.Second)
{
broadcastMock.Unset()
broadcastMock = msgStreamObj.EXPECT().Broadcast(mock.Anything).Return(nil, errors.New("mock error: broadcast"))
resp, err := node.ReplicateMessage(context.TODO(), replicateRequest)
assert.NoError(t, err)
assert.NotEqualValues(t, 0, resp.GetStatus().GetCode())
resourceManager.Delete(ReplicateMsgStreamTyp, replicateRequest.GetChannelName())
time.Sleep(2 * time.Second)
}
{
broadcastMock.Unset()
broadcastMock = msgStreamObj.EXPECT().Broadcast(mock.Anything).Return(map[string][]mqwrapper.MessageID{
"unit_test_replicate_message": {},
}, nil)
resp, err := node.ReplicateMessage(context.TODO(), replicateRequest)
assert.NoError(t, err)
assert.EqualValues(t, 0, resp.GetStatus().GetCode())
assert.Empty(t, resp.GetPosition())
resourceManager.Delete(ReplicateMsgStreamTyp, replicateRequest.GetChannelName())
time.Sleep(2 * time.Second)
broadcastMock.Unset()
}
})
}

13
internal/proxy/mock_msgstream_test.go
View File

@ -10,9 +10,10 @@ import (
type mockMsgStream struct {
msgstream.MsgStream
asProducer func([]string)
setRepack func(repackFunc msgstream.RepackFunc)
close func()
asProducer func([]string)
setRepack func(repackFunc msgstream.RepackFunc)
close func()
enableProduce func(bool)
}
func (m *mockMsgStream) AsProducer(producers []string) {
@ -33,6 +34,12 @@ func (m *mockMsgStream) Close() {
}
}
func (m *mockMsgStream) EnableProduce(enabled bool) {
if m.enableProduce != nil {
m.enableProduce(enabled)
}
}
func newMockMsgStream() *mockMsgStream {
return &mockMsgStream{}
}

3
internal/proxy/mock_test.go
View File

@ -306,6 +306,9 @@ func (ms *simpleMockMsgStream) CheckTopicValid(topic string) error {
return nil
}
func (ms *simpleMockMsgStream) EnableProduce(enabled bool) {
}
func newSimpleMockMsgStream() *simpleMockMsgStream {
return &simpleMockMsgStream{
msgChan: make(chan *msgstream.MsgPack, 1024),

44
internal/proxy/proxy.go
View File

@ -40,11 +40,13 @@ import (
"github.com/milvus-io/milvus/internal/util/sessionutil"
"github.com/milvus-io/milvus/pkg/log"
"github.com/milvus-io/milvus/pkg/metrics"
"github.com/milvus-io/milvus/pkg/mq/msgstream"
"github.com/milvus-io/milvus/pkg/util/commonpbutil"
"github.com/milvus-io/milvus/pkg/util/logutil"
"github.com/milvus-io/milvus/pkg/util/metricsinfo"
"github.com/milvus-io/milvus/pkg/util/paramtable"
"github.com/milvus-io/milvus/pkg/util/ratelimitutil"
"github.com/milvus-io/milvus/pkg/util/resource"
"github.com/milvus-io/milvus/pkg/util/tsoutil"
"github.com/milvus-io/milvus/pkg/util/typeutil"
)
@ -88,6 +90,8 @@ type Proxy struct {
chMgr channelsMgr
replicateMsgStream msgstream.MsgStream
sched *taskScheduler
chTicker channelsTimeTicker
@ -111,6 +115,10 @@ type Proxy struct {
// for load balance in replicas
lbPolicy LBPolicy
// resource manager
resourceManager resource.Manager
replicateStreamManager *ReplicateStreamManager
}
// NewProxy returns a Proxy struct.
@ -121,14 +129,18 @@ func NewProxy(ctx context.Context, factory dependency.Factory) (*Proxy, error) {
mgr := newShardClientMgr()
lbPolicy := NewLBPolicyImpl(mgr)
lbPolicy.Start(ctx)
resourceManager := resource.NewManager(10*time.Second, 20*time.Second, make(map[string]time.Duration))
replicateStreamManager := NewReplicateStreamManager(ctx, factory, resourceManager)
node := &Proxy{
ctx: ctx1,
cancel: cancel,
factory: factory,
searchResultCh: make(chan *internalpb.SearchResults, n),
shardMgr: mgr,
multiRateLimiter: NewMultiRateLimiter(),
lbPolicy: lbPolicy,
ctx: ctx1,
cancel: cancel,
factory: factory,
searchResultCh: make(chan *internalpb.SearchResults, n),
shardMgr: mgr,
multiRateLimiter: NewMultiRateLimiter(),
lbPolicy: lbPolicy,
resourceManager: resourceManager,
replicateStreamManager: replicateStreamManager,
}
node.UpdateStateCode(commonpb.StateCode_Abnormal)
logutil.Logger(ctx).Debug("create a new Proxy instance", zap.Any("state", node.stateCode.Load()))
@ -250,6 +262,17 @@ func (node *Proxy) Init() error {
node.chMgr = chMgr
log.Debug("create channels manager done", zap.String("role", typeutil.ProxyRole))
replicateMsgChannel := Params.CommonCfg.ReplicateMsgChannel.GetValue()
node.replicateMsgStream, err = node.factory.NewMsgStream(node.ctx)
if err != nil {
log.Warn("failed to create replicate msg stream",
zap.String("role", typeutil.ProxyRole), zap.Int64("ProxyID", paramtable.GetNodeID()),
zap.Error(err))
return err
}
node.replicateMsgStream.EnableProduce(true)
node.replicateMsgStream.AsProducer([]string{replicateMsgChannel})
node.sched, err = newTaskScheduler(node.ctx, node.tsoAllocator, node.factory)
if err != nil {
log.Warn("failed to create task scheduler", zap.String("role", typeutil.ProxyRole), zap.Error(err))
@ -287,6 +310,9 @@ func (node *Proxy) sendChannelsTimeTickLoop() {
log.Info("send channels time tick loop exit")
return
case <-ticker.C:
if !Params.CommonCfg.TTMsgEnabled.GetAsBool() {
continue
}
stats, ts, err := node.chTicker.getMinTsStatistics()
if err != nil {
log.Warn("sendChannelsTimeTickLoop.getMinTsStatistics", zap.Error(err))
@ -442,6 +468,10 @@ func (node *Proxy) Stop() error {
node.lbPolicy.Close()
}
if node.resourceManager != nil {
node.resourceManager.Close()
}
// https://github.com/milvus-io/milvus/issues/12282
node.UpdateStateCode(commonpb.StateCode_Abnormal)

11
internal/proxy/repack_func.go
View File

@ -81,3 +81,14 @@ func defaultInsertRepackFunc(
}
return pack, nil
}
func replicatePackFunc(
tsMsgs []msgstream.TsMsg,
hashKeys [][]int32,
) (map[int32]*msgstream.MsgPack, error) {
return map[int32]*msgstream.MsgPack{
0: {
Msgs: tsMsgs,
},
}, nil
}

72
internal/proxy/replicate_stream_manager.go Normal file
View File

@ -0,0 +1,72 @@
package proxy
import (
"context"
"time"
"go.uber.org/zap"
"github.com/milvus-io/milvus/pkg/log"
"github.com/milvus-io/milvus/pkg/mq/msgstream"
"github.com/milvus-io/milvus/pkg/util/merr"
"github.com/milvus-io/milvus/pkg/util/resource"
)
const (
ReplicateMsgStreamTyp = "replicate_msg_stream"
ReplicateMsgStreamExpireTime = 30 * time.Second
)
type ReplicateStreamManager struct {
ctx context.Context
factory msgstream.Factory
dispatcher msgstream.UnmarshalDispatcher
resourceManager resource.Manager
}
func NewReplicateStreamManager(ctx context.Context, factory msgstream.Factory, resourceManager resource.Manager) *ReplicateStreamManager {
manager := &ReplicateStreamManager{
ctx: ctx,
factory: factory,
dispatcher: (&msgstream.ProtoUDFactory{}).NewUnmarshalDispatcher(),
resourceManager: resourceManager,
}
return manager
}
func (m *ReplicateStreamManager) newMsgStreamResource(channel string) resource.NewResourceFunc {
return func() (resource.Resource, error) {
msgStream, err := m.factory.NewMsgStream(m.ctx)
if err != nil {
log.Ctx(m.ctx).Warn("failed to create msg stream", zap.String("channel", channel), zap.Error(err))
return nil, err
}
msgStream.SetRepackFunc(replicatePackFunc)
msgStream.AsProducer([]string{channel})
msgStream.EnableProduce(true)
res := resource.NewSimpleResource(msgStream, ReplicateMsgStreamTyp, channel, ReplicateMsgStreamExpireTime, func() {
msgStream.Close()
})
return res, nil
}
}
func (m *ReplicateStreamManager) GetReplicateMsgStream(ctx context.Context, channel string) (msgstream.MsgStream, error) {
ctxLog := log.Ctx(ctx).With(zap.String("proxy_channel", channel))
res, err := m.resourceManager.Get(ReplicateMsgStreamTyp, channel, m.newMsgStreamResource(channel))
if err != nil {
ctxLog.Warn("failed to get replicate msg stream", zap.String("channel", channel), zap.Error(err))
return nil, err
}
if obj, ok := res.Get().(msgstream.MsgStream); ok && obj != nil {
return obj, nil
}
ctxLog.Warn("invalid resource object", zap.Any("obj", res.Get()))
return nil, merr.ErrInvalidStreamObj
}
func (m *ReplicateStreamManager) GetMsgDispatcher() msgstream.UnmarshalDispatcher {
return m.dispatcher
}

79
internal/proxy/replicate_stream_manager_test.go Normal file
View File

@ -0,0 +1,79 @@
package proxy
import (
"context"
"testing"
"time"
"github.com/cockroachdb/errors"
"github.com/stretchr/testify/assert"
"github.com/milvus-io/milvus/pkg/mq/msgstream"
"github.com/milvus-io/milvus/pkg/util/merr"
"github.com/milvus-io/milvus/pkg/util/resource"
)
func TestReplicateManager(t *testing.T) {
factory := newMockMsgStreamFactory()
resourceManager := resource.NewManager(time.Second, 2*time.Second, nil)
manager := NewReplicateStreamManager(context.Background(), factory, resourceManager)
{
factory.f = func(ctx context.Context) (msgstream.MsgStream, error) {
return nil, errors.New("mock msgstream fail")
}
_, err := manager.GetReplicateMsgStream(context.Background(), "test")
assert.Error(t, err)
}
{
mockMsgStream := newMockMsgStream()
i := 0
mockMsgStream.setRepack = func(repackFunc msgstream.RepackFunc) {
i++
}
mockMsgStream.asProducer = func(producers []string) {
i++
}
mockMsgStream.enableProduce = func(b bool) {
i++
}
mockMsgStream.close = func() {
i++
}
factory.f = func(ctx context.Context) (msgstream.MsgStream, error) {
return mockMsgStream, nil
}
_, err := manager.GetReplicateMsgStream(context.Background(), "test")
assert.NoError(t, err)
assert.Equal(t, 3, i)
time.Sleep(time.Second)
_, err = manager.GetReplicateMsgStream(context.Background(), "test")
assert.NoError(t, err)
assert.Equal(t, 3, i)
res := resourceManager.Delete(ReplicateMsgStreamTyp, "test")
assert.NotNil(t, res)
time.Sleep(2 * time.Second)
_, err = manager.GetReplicateMsgStream(context.Background(), "test")
assert.NoError(t, err)
assert.Equal(t, 7, i)
}
{
res := resourceManager.Delete(ReplicateMsgStreamTyp, "test")
assert.NotNil(t, res)
time.Sleep(2 * time.Second)
res, err := resourceManager.Get(ReplicateMsgStreamTyp, "test", func() (resource.Resource, error) {
return resource.NewResource(resource.WithObj("str")), nil
})
assert.NoError(t, err)
assert.Equal(t, "str", res.Get())
_, err = manager.GetReplicateMsgStream(context.Background(), "test")
assert.ErrorIs(t, err, merr.ErrInvalidStreamObj)
}
{
assert.NotNil(t, manager.GetMsgDispatcher())
}
}

84
internal/proxy/task.go
View File

@ -156,7 +156,9 @@ func (t *createCollectionTask) SetTs(ts Timestamp) {
}
func (t *createCollectionTask) OnEnqueue() error {
t.Base = commonpbutil.NewMsgBase()
if t.Base == nil {
t.Base = commonpbutil.NewMsgBase()
}
t.Base.MsgType = commonpb.MsgType_CreateCollection
t.Base.SourceID = paramtable.GetNodeID()
return nil
@ -354,7 +356,9 @@ func (t *dropCollectionTask) SetTs(ts Timestamp) {
}
func (t *dropCollectionTask) OnEnqueue() error {
t.Base = commonpbutil.NewMsgBase()
if t.Base == nil {
t.Base = commonpbutil.NewMsgBase()
}
return nil
}
@ -786,7 +790,9 @@ func (t *alterCollectionTask) SetTs(ts Timestamp) {
}
func (t *alterCollectionTask) OnEnqueue() error {
t.Base = commonpbutil.NewMsgBase()
if t.Base == nil {
t.Base = commonpbutil.NewMsgBase()
}
return nil
}
@ -848,7 +854,9 @@ func (t *createPartitionTask) SetTs(ts Timestamp) {
}
func (t *createPartitionTask) OnEnqueue() error {
t.Base = commonpbutil.NewMsgBase()
if t.Base == nil {
t.Base = commonpbutil.NewMsgBase()
}
return nil
}
@ -934,7 +942,9 @@ func (t *dropPartitionTask) SetTs(ts Timestamp) {
}
func (t *dropPartitionTask) OnEnqueue() error {
t.Base = commonpbutil.NewMsgBase()
if t.Base == nil {
t.Base = commonpbutil.NewMsgBase()
}
return nil
}
@ -1248,6 +1258,8 @@ type flushTask struct {
ctx context.Context
dataCoord types.DataCoordClient
result *milvuspb.FlushResponse
replicateMsgStream msgstream.MsgStream
}
func (t *flushTask) TraceCtx() context.Context {
@ -1283,7 +1295,9 @@ func (t *flushTask) SetTs(ts Timestamp) {
}
func (t *flushTask) OnEnqueue() error {
t.Base = commonpbutil.NewMsgBase()
if t.Base == nil {
t.Base = commonpbutil.NewMsgBase()
}
return nil
}
@ -1323,6 +1337,7 @@ func (t *flushTask) Execute(ctx context.Context) error {
coll2SealTimes[collName] = resp.GetTimeOfSeal()
coll2FlushTs[collName] = resp.GetFlushTs()
}
SendReplicateMessagePack(ctx, t.replicateMsgStream, t.FlushRequest)
t.result = &milvuspb.FlushResponse{
Status: merr.Success(),
DbName: t.GetDbName(),
@ -1346,7 +1361,8 @@ type loadCollectionTask struct {
datacoord types.DataCoordClient
result *commonpb.Status
collectionID UniqueID
collectionID UniqueID
replicateMsgStream msgstream.MsgStream
}
func (t *loadCollectionTask) TraceCtx() context.Context {
@ -1382,7 +1398,9 @@ func (t *loadCollectionTask) SetTs(ts Timestamp) {
}
func (t *loadCollectionTask) OnEnqueue() error {
t.Base = commonpbutil.NewMsgBase()
if t.Base == nil {
t.Base = commonpbutil.NewMsgBase()
}
return nil
}
@ -1472,6 +1490,7 @@ func (t *loadCollectionTask) Execute(ctx context.Context) (err error) {
if err != nil {
return fmt.Errorf("call query coordinator LoadCollection: %s", err)
}
SendReplicateMessagePack(ctx, t.replicateMsgStream, t.LoadCollectionRequest)
return nil
}
@ -1492,9 +1511,9 @@ type releaseCollectionTask struct {
ctx context.Context
queryCoord types.QueryCoordClient
result *commonpb.Status
chMgr channelsMgr
collectionID UniqueID
collectionID UniqueID
replicateMsgStream msgstream.MsgStream
}
func (t *releaseCollectionTask) TraceCtx() context.Context {
@ -1530,7 +1549,9 @@ func (t *releaseCollectionTask) SetTs(ts Timestamp) {
}
func (t *releaseCollectionTask) OnEnqueue() error {
t.Base = commonpbutil.NewMsgBase()
if t.Base == nil {
t.Base = commonpbutil.NewMsgBase()
}
return nil
}
@ -1565,7 +1586,10 @@ func (t *releaseCollectionTask) Execute(ctx context.Context) (err error) {
t.result, err = t.queryCoord.ReleaseCollection(ctx, request)
globalMetaCache.RemoveCollection(ctx, t.GetDbName(), t.CollectionName)
if err != nil {
return err
}
SendReplicateMessagePack(ctx, t.replicateMsgStream, t.ReleaseCollectionRequest)
return err
}
@ -1618,7 +1642,9 @@ func (t *loadPartitionsTask) SetTs(ts Timestamp) {
}
func (t *loadPartitionsTask) OnEnqueue() error {
t.Base = commonpbutil.NewMsgBase()
if t.Base == nil {
t.Base = commonpbutil.NewMsgBase()
}
return nil
}
@ -1759,7 +1785,9 @@ func (t *releasePartitionsTask) SetTs(ts Timestamp) {
}
func (t *releasePartitionsTask) OnEnqueue() error {
t.Base = commonpbutil.NewMsgBase()
if t.Base == nil {
t.Base = commonpbutil.NewMsgBase()
}
return nil
}
@ -1867,7 +1895,9 @@ func (t *CreateAliasTask) SetTs(ts Timestamp) {
// OnEnqueue defines the behavior task enqueued
func (t *CreateAliasTask) OnEnqueue() error {
t.Base = commonpbutil.NewMsgBase()
if t.Base == nil {
t.Base = commonpbutil.NewMsgBase()
}
return nil
}
@ -1947,7 +1977,9 @@ func (t *DropAliasTask) SetTs(ts Timestamp) {
}
func (t *DropAliasTask) OnEnqueue() error {
t.Base = commonpbutil.NewMsgBase()
if t.Base == nil {
t.Base = commonpbutil.NewMsgBase()
}
return nil
}
@ -2013,7 +2045,9 @@ func (t *AlterAliasTask) SetTs(ts Timestamp) {
}
func (t *AlterAliasTask) OnEnqueue() error {
t.Base = commonpbutil.NewMsgBase()
if t.Base == nil {
t.Base = commonpbutil.NewMsgBase()
}
return nil
}
@ -2086,7 +2120,9 @@ func (t *CreateResourceGroupTask) SetTs(ts Timestamp) {
}
func (t *CreateResourceGroupTask) OnEnqueue() error {
t.Base = commonpbutil.NewMsgBase()
if t.Base == nil {
t.Base = commonpbutil.NewMsgBase()
}
return nil
}
@ -2148,7 +2184,9 @@ func (t *DropResourceGroupTask) SetTs(ts Timestamp) {
}
func (t *DropResourceGroupTask) OnEnqueue() error {
t.Base = commonpbutil.NewMsgBase()
if t.Base == nil {
t.Base = commonpbutil.NewMsgBase()
}
return nil
}
@ -2331,7 +2369,9 @@ func (t *TransferNodeTask) SetTs(ts Timestamp) {
}
func (t *TransferNodeTask) OnEnqueue() error {
t.Base = commonpbutil.NewMsgBase()
if t.Base == nil {
t.Base = commonpbutil.NewMsgBase()
}
return nil
}
@ -2393,7 +2433,9 @@ func (t *TransferReplicaTask) SetTs(ts Timestamp) {
}
func (t *TransferReplicaTask) OnEnqueue() error {
t.Base = commonpbutil.NewMsgBase()
if t.Base == nil {
t.Base = commonpbutil.NewMsgBase()
}
return nil
}

17
internal/proxy/task_database.go
View File

@ -6,6 +6,7 @@ import (
"github.com/milvus-io/milvus-proto/go-api/v2/commonpb"
"github.com/milvus-io/milvus-proto/go-api/v2/milvuspb"
"github.com/milvus-io/milvus/internal/types"
"github.com/milvus-io/milvus/pkg/mq/msgstream"
"github.com/milvus-io/milvus/pkg/util/commonpbutil"
"github.com/milvus-io/milvus/pkg/util/paramtable"
)
@ -16,6 +17,8 @@ type createDatabaseTask struct {
ctx context.Context
rootCoord types.RootCoordClient
result *commonpb.Status
replicateMsgStream msgstream.MsgStream
}
func (cdt *createDatabaseTask) TraceCtx() context.Context {
@ -51,7 +54,9 @@ func (cdt *createDatabaseTask) SetTs(ts Timestamp) {
}
func (cdt *createDatabaseTask) OnEnqueue() error {
cdt.Base = commonpbutil.NewMsgBase()
if cdt.Base == nil {
cdt.Base = commonpbutil.NewMsgBase()
}
cdt.Base.MsgType = commonpb.MsgType_CreateDatabase
cdt.Base.SourceID = paramtable.GetNodeID()
return nil
@ -64,6 +69,9 @@ func (cdt *createDatabaseTask) PreExecute(ctx context.Context) error {
func (cdt *createDatabaseTask) Execute(ctx context.Context) error {
var err error
cdt.result, err = cdt.rootCoord.CreateDatabase(ctx, cdt.CreateDatabaseRequest)
if cdt.result != nil && cdt.result.ErrorCode == commonpb.ErrorCode_Success {
SendReplicateMessagePack(ctx, cdt.replicateMsgStream, cdt.CreateDatabaseRequest)
}
return err
}
@ -77,6 +85,8 @@ type dropDatabaseTask struct {
ctx context.Context
rootCoord types.RootCoordClient
result *commonpb.Status
replicateMsgStream msgstream.MsgStream
}
func (ddt *dropDatabaseTask) TraceCtx() context.Context {
@ -112,7 +122,9 @@ func (ddt *dropDatabaseTask) SetTs(ts Timestamp) {
}
func (ddt *dropDatabaseTask) OnEnqueue() error {
ddt.Base = commonpbutil.NewMsgBase()
if ddt.Base == nil {
ddt.Base = commonpbutil.NewMsgBase()
}
ddt.Base.MsgType = commonpb.MsgType_DropDatabase
ddt.Base.SourceID = paramtable.GetNodeID()
return nil
@ -128,6 +140,7 @@ func (ddt *dropDatabaseTask) Execute(ctx context.Context) error {
if ddt.result != nil && ddt.result.ErrorCode == commonpb.ErrorCode_Success {
globalMetaCache.RemoveDatabase(ctx, ddt.DbName)
SendReplicateMessagePack(ctx, ddt.replicateMsgStream, ddt.DropDatabaseRequest)
}
return err
}

2
internal/proxy/task_database_test.go
View File

@ -45,6 +45,7 @@ func TestCreateDatabaseTask(t *testing.T) {
err = task.Execute(ctx)
assert.NoError(t, err)
task.Base = nil
err = task.OnEnqueue()
assert.NoError(t, err)
assert.Equal(t, paramtable.GetNodeID(), task.GetBase().GetSourceID())
@ -98,6 +99,7 @@ func TestDropDatabaseTask(t *testing.T) {
err = task.Execute(ctx)
assert.NoError(t, err)
task.Base = nil
err = task.OnEnqueue()
assert.NoError(t, err)
assert.Equal(t, paramtable.GetNodeID(), task.GetBase().GetSourceID())

30
internal/proxy/task_index.go
View File

@ -31,6 +31,7 @@ import (
"github.com/milvus-io/milvus/internal/types"
"github.com/milvus-io/milvus/pkg/common"
"github.com/milvus-io/milvus/pkg/log"
"github.com/milvus-io/milvus/pkg/mq/msgstream"
"github.com/milvus-io/milvus/pkg/util/commonpbutil"
"github.com/milvus-io/milvus/pkg/util/funcutil"
"github.com/milvus-io/milvus/pkg/util/indexparamcheck"
@ -59,6 +60,8 @@ type createIndexTask struct {
datacoord types.DataCoordClient
result *commonpb.Status
replicateMsgStream msgstream.MsgStream
isAutoIndex bool
newIndexParams []*commonpb.KeyValuePair
newTypeParams []*commonpb.KeyValuePair
@ -101,7 +104,9 @@ func (cit *createIndexTask) SetTs(ts Timestamp) {
}
func (cit *createIndexTask) OnEnqueue() error {
cit.req.Base = commonpbutil.NewMsgBase()
if cit.req.Base == nil {
cit.req.Base = commonpbutil.NewMsgBase()
}
return nil
}
@ -414,7 +419,8 @@ func (cit *createIndexTask) Execute(ctx context.Context) error {
if cit.result.ErrorCode != commonpb.ErrorCode_Success {
return errors.New(cit.result.Reason)
}
return err
SendReplicateMessagePack(ctx, cit.replicateMsgStream, cit.req)
return nil
}
func (cit *createIndexTask) PostExecute(ctx context.Context) error {
@ -669,6 +675,8 @@ type dropIndexTask struct {
result *commonpb.Status
collectionID UniqueID
replicateMsgStream msgstream.MsgStream
}
func (dit *dropIndexTask) TraceCtx() context.Context {
@ -704,7 +712,9 @@ func (dit *dropIndexTask) SetTs(ts Timestamp) {
}
func (dit *dropIndexTask) OnEnqueue() error {
dit.Base = commonpbutil.NewMsgBase()
if dit.Base == nil {
dit.Base = commonpbutil.NewMsgBase()
}
return nil
}
@ -743,6 +753,13 @@ func (dit *dropIndexTask) PreExecute(ctx context.Context) error {
}
func (dit *dropIndexTask) Execute(ctx context.Context) error {
ctxLog := log.Ctx(ctx)
ctxLog.Info("proxy drop index", zap.Int64("collID", dit.collectionID),
zap.String("field_name", dit.FieldName),
zap.String("index_name", dit.IndexName),
zap.String("db_name", dit.DbName),
)
var err error
dit.result, err = dit.dataCoord.DropIndex(ctx, &indexpb.DropIndexRequest{
CollectionID: dit.collectionID,
@ -750,13 +767,18 @@ func (dit *dropIndexTask) Execute(ctx context.Context) error {
IndexName: dit.IndexName,
DropAll: false,
})
if err != nil {
ctxLog.Warn("drop index failed", zap.Error(err))
return err
}
if dit.result == nil {
return errors.New("drop index resp is nil")
}
if dit.result.ErrorCode != commonpb.ErrorCode_Success {
return errors.New(dit.result.Reason)
}
return err
SendReplicateMessagePack(ctx, dit.replicateMsgStream, dit.DropIndexRequest)
return nil
}
func (dit *dropIndexTask) PostExecute(ctx context.Context) error {

73
internal/proxy/util.go
View File

@ -1504,3 +1504,76 @@ func checkDynamicFieldData(schema *schemapb.CollectionSchema, insertMsg *msgstre
insertMsg.FieldsData = append(insertMsg.FieldsData, dynamicData)
return nil
}
func SendReplicateMessagePack(ctx context.Context, replicateMsgStream msgstream.MsgStream, request interface{ GetBase() *commonpb.MsgBase }) {
if replicateMsgStream == nil || request == nil {
log.Warn("replicate msg stream or request is nil", zap.Any("request", request))
return
}
msgBase := request.GetBase()
ts := msgBase.GetTimestamp()
if msgBase.GetReplicateInfo().GetIsReplicate() {
ts = msgBase.GetReplicateInfo().GetMsgTimestamp()
}
getBaseMsg := func(ctx context.Context, ts uint64) msgstream.BaseMsg {
return msgstream.BaseMsg{
Ctx: ctx,
HashValues: []uint32{0},
BeginTimestamp: ts,
EndTimestamp: ts,
}
}
var tsMsg msgstream.TsMsg
switch r := request.(type) {
case *milvuspb.CreateDatabaseRequest:
tsMsg = &msgstream.CreateDatabaseMsg{
BaseMsg: getBaseMsg(ctx, ts),
CreateDatabaseRequest: *r,
}
case *milvuspb.DropDatabaseRequest:
tsMsg = &msgstream.DropDatabaseMsg{
BaseMsg: getBaseMsg(ctx, ts),
DropDatabaseRequest: *r,
}
case *milvuspb.FlushRequest:
tsMsg = &msgstream.FlushMsg{
BaseMsg: getBaseMsg(ctx, ts),
FlushRequest: *r,
}
case *milvuspb.LoadCollectionRequest:
tsMsg = &msgstream.LoadCollectionMsg{
BaseMsg: getBaseMsg(ctx, ts),
LoadCollectionRequest: *r,
}
case *milvuspb.ReleaseCollectionRequest:
tsMsg = &msgstream.ReleaseCollectionMsg{
BaseMsg: getBaseMsg(ctx, ts),
ReleaseCollectionRequest: *r,
}
case *milvuspb.CreateIndexRequest:
tsMsg = &msgstream.CreateIndexMsg{
BaseMsg: getBaseMsg(ctx, ts),
CreateIndexRequest: *r,
}
case *milvuspb.DropIndexRequest:
tsMsg = &msgstream.DropIndexMsg{
BaseMsg: getBaseMsg(ctx, ts),
DropIndexRequest: *r,
}
default:
log.Warn("unknown request", zap.Any("request", request))
return
}
msgPack := &msgstream.MsgPack{
BeginTs: ts,
EndTs: ts,
Msgs: []msgstream.TsMsg{tsMsg},
}
msgErr := replicateMsgStream.Produce(msgPack)
// ignore the error if the msg stream failed to produce the msg,
// because it can be manually fixed in this error
if msgErr != nil {
log.Warn("send replicate msg failed", zap.Any("pack", msgPack), zap.Error(msgErr))
}
}

35
internal/proxy/util_test.go
View File

@ -2052,3 +2052,38 @@ func Test_validateMaxCapacityPerRow(t *testing.T) {
assert.Error(t, err)
})
}
func TestSendReplicateMessagePack(t *testing.T) {
ctx := context.Background()
mockStream := msgstream.NewMockMsgStream(t)
t.Run("empty case", func(t *testing.T) {
SendReplicateMessagePack(ctx, nil, nil)
})
t.Run("produce fail", func(t *testing.T) {
mockStream.EXPECT().Produce(mock.Anything).Return(errors.New("produce error")).Once()
SendReplicateMessagePack(ctx, mockStream, &milvuspb.CreateDatabaseRequest{
Base: &commonpb.MsgBase{ReplicateInfo: &commonpb.ReplicateInfo{
IsReplicate: true,
MsgTimestamp: 100,
}},
})
})
t.Run("unknown request", func(t *testing.T) {
SendReplicateMessagePack(ctx, mockStream, &milvuspb.ListDatabasesRequest{})
})
t.Run("normal case", func(t *testing.T) {
mockStream.EXPECT().Produce(mock.Anything).Return(nil)
SendReplicateMessagePack(ctx, mockStream, &milvuspb.CreateDatabaseRequest{})
SendReplicateMessagePack(ctx, mockStream, &milvuspb.DropDatabaseRequest{})
SendReplicateMessagePack(ctx, mockStream, &milvuspb.FlushRequest{})
SendReplicateMessagePack(ctx, mockStream, &milvuspb.LoadCollectionRequest{})
SendReplicateMessagePack(ctx, mockStream, &milvuspb.ReleaseCollectionRequest{})
SendReplicateMessagePack(ctx, mockStream, &milvuspb.CreateIndexRequest{})
SendReplicateMessagePack(ctx, mockStream, &milvuspb.DropIndexRequest{})
})
}

29
internal/rootcoord/create_collection_task.go
View File

@ -349,8 +349,7 @@ func (t *createCollectionTask) Prepare(ctx context.Context) error {
return t.assignChannels()
}
func (t *createCollectionTask) genCreateCollectionMsg(ctx context.Context) *ms.MsgPack {
ts := t.GetTs()
func (t *createCollectionTask) genCreateCollectionMsg(ctx context.Context, ts uint64) *ms.MsgPack {
collectionID := t.collID
partitionIDs := t.partIDs
// error won't happen here.
@ -382,21 +381,36 @@ func (t *createCollectionTask) genCreateCollectionMsg(ctx context.Context) *ms.M
return &msgPack
}
func (t *createCollectionTask) addChannelsAndGetStartPositions(ctx context.Context) (map[string][]byte, error) {
func (t *createCollectionTask) addChannelsAndGetStartPositions(ctx context.Context, ts uint64) (map[string][]byte, error) {
t.core.chanTimeTick.addDmlChannels(t.channels.physicalChannels...)
msg := t.genCreateCollectionMsg(ctx)
msg := t.genCreateCollectionMsg(ctx, ts)
return t.core.chanTimeTick.broadcastMarkDmlChannels(t.channels.physicalChannels, msg)
}
func (t *createCollectionTask) getCreateTs() (uint64, error) {
replicateInfo := t.Req.GetBase().GetReplicateInfo()
if !replicateInfo.GetIsReplicate() {
return t.GetTs(), nil
}
if replicateInfo.GetMsgTimestamp() == 0 {
log.Warn("the cdc timestamp is not set in the request for the backup instance")
return 0, merr.WrapErrParameterInvalidMsg("the cdc timestamp is not set in the request for the backup instance")
}
return replicateInfo.GetMsgTimestamp(), nil
}
func (t *createCollectionTask) Execute(ctx context.Context) error {
collID := t.collID
partIDs := t.partIDs
ts := t.GetTs()
ts, err := t.getCreateTs()
if err != nil {
return err
}
vchanNames := t.channels.virtualChannels
chanNames := t.channels.physicalChannels
startPositions, err := t.addChannelsAndGetStartPositions(ctx)
startPositions, err := t.addChannelsAndGetStartPositions(ctx, ts)
if err != nil {
// ugly here, since we must get start positions first.
t.core.chanTimeTick.removeDmlChannels(t.channels.physicalChannels...)
@ -445,7 +459,7 @@ func (t *createCollectionTask) Execute(ctx context.Context) error {
return fmt.Errorf("create duplicate collection with different parameters, collection: %s", t.Req.GetCollectionName())
}
// make creating collection idempotent.
log.Warn("add duplicate collection", zap.String("collection", t.Req.GetCollectionName()), zap.Uint64("ts", t.GetTs()))
log.Warn("add duplicate collection", zap.String("collection", t.Req.GetCollectionName()), zap.Uint64("ts", ts))
return nil
}
@ -475,6 +489,7 @@ func (t *createCollectionTask) Execute(ctx context.Context) error {
baseStep: baseStep{core: t.core},
collectionID: collID,
channels: t.channels,
isSkip: !Params.CommonCfg.TTMsgEnabled.GetAsBool(),
})
undoTask.AddStep(&watchChannelsStep{
baseStep: baseStep{core: t.core},

34
internal/rootcoord/create_collection_task_test.go
View File

@ -51,6 +51,40 @@ func Test_createCollectionTask_validate(t *testing.T) {
assert.Error(t, err)
})
t.Run("create ts", func(t *testing.T) {
task := createCollectionTask{
Req: nil,
}
{
task.SetTs(1000)
ts, err := task.getCreateTs()
assert.NoError(t, err)
assert.EqualValues(t, 1000, ts)
}
task.Req = &milvuspb.CreateCollectionRequest{
Base: &commonpb.MsgBase{
MsgType: commonpb.MsgType_CreateCollection,
ReplicateInfo: &commonpb.ReplicateInfo{
IsReplicate: true,
},
},
}
{
task.SetTs(1000)
_, err := task.getCreateTs()
assert.Error(t, err)
err = task.Execute(context.Background())
assert.Error(t, err)
}
{
task.Req.Base.ReplicateInfo.MsgTimestamp = 2000
ts, err := task.getCreateTs()
assert.NoError(t, err)
assert.EqualValues(t, 2000, ts)
}
})
t.Run("invalid msg type", func(t *testing.T) {
task := createCollectionTask{
Req: &milvuspb.CreateCollectionRequest{

1
internal/rootcoord/drop_collection_task.go
View File

@ -100,6 +100,7 @@ func (t *dropCollectionTask) Execute(ctx context.Context) error {
redoTask.AddAsyncStep(&deleteCollectionDataStep{
baseStep: baseStep{core: t.core},
coll: collMeta,
isSkip: t.Req.GetBase().GetReplicateInfo().GetIsReplicate(),
})
redoTask.AddAsyncStep(&removeDmlChannelsStep{
baseStep: baseStep{core: t.core},

1
internal/rootcoord/drop_partition_task.go
View File

@ -91,6 +91,7 @@ func (t *dropPartitionTask) Execute(ctx context.Context) error {
PartitionName: t.Req.GetPartitionName(),
CollectionID: t.collMeta.CollectionID,
},
isSkip: t.Req.GetBase().GetReplicateInfo().GetIsReplicate(),
})
redoTask.AddAsyncStep(newConfirmGCStep(t.core, t.collMeta.CollectionID, partID))
redoTask.AddAsyncStep(&removePartitionMetaStep{

3
internal/rootcoord/garbage_collector.go
View File

@ -61,6 +61,7 @@ func (c *bgGarbageCollector) ReDropCollection(collMeta *model.Collection, ts Tim
redo.AddAsyncStep(&deleteCollectionDataStep{
baseStep: baseStep{core: c.s},
coll: collMeta,
isSkip: !Params.CommonCfg.TTMsgEnabled.GetAsBool(),
})
redo.AddAsyncStep(&removeDmlChannelsStep{
baseStep: baseStep{core: c.s},
@ -93,6 +94,7 @@ func (c *bgGarbageCollector) RemoveCreatingCollection(collMeta *model.Collection
virtualChannels: collMeta.VirtualChannelNames,
physicalChannels: collMeta.PhysicalChannelNames,
},
isSkip: !Params.CommonCfg.TTMsgEnabled.GetAsBool(),
})
redo.AddAsyncStep(&removeDmlChannelsStep{
baseStep: baseStep{core: c.s},
@ -117,6 +119,7 @@ func (c *bgGarbageCollector) ReDropPartition(dbID int64, pChannels []string, par
baseStep: baseStep{core: c.s},
pchans: pChannels,
partition: partition,
isSkip: !Params.CommonCfg.TTMsgEnabled.GetAsBool(),
})
redo.AddAsyncStep(&removeDmlChannelsStep{
baseStep: baseStep{core: c.s},

3
internal/rootcoord/root_coord.go
View File

@ -179,6 +179,9 @@ func (c *Core) sendTimeTick(t Timestamp, reason string) error {
}
func (c *Core) sendMinDdlTsAsTt() {
if !paramtable.Get().CommonCfg.TTMsgEnabled.GetAsBool() {
return
}
code := c.GetStateCode()
if code != commonpb.StateCode_Healthy {
log.Warn("rootCoord is not healthy, skip send timetick")

5
internal/rootcoord/root_coord_test.go
View File

@ -1649,6 +1649,11 @@ func TestCore_sendMinDdlTsAsTt(t *testing.T) {
c.UpdateStateCode(commonpb.StateCode_Healthy)
c.session.ServerID = TestRootCoordID
_ = paramtable.Get().Save(paramtable.Get().CommonCfg.TTMsgEnabled.Key, "false")
c.sendMinDdlTsAsTt() // disable ts msg
_ = paramtable.Get().Save(paramtable.Get().CommonCfg.TTMsgEnabled.Key, "true")
c.sendMinDdlTsAsTt() // no session.
ticker.addSession(&sessionutil.Session{SessionRaw: sessionutil.SessionRaw{ServerID: TestRootCoordID}})
c.sendMinDdlTsAsTt()

13
internal/rootcoord/step.go
View File

@ -125,12 +125,15 @@ type unwatchChannelsStep struct {
baseStep
collectionID UniqueID
channels collectionChannels
isSkip bool
}
func (s *unwatchChannelsStep) Execute(ctx context.Context) ([]nestedStep, error) {
unwatchByDropMsg := &deleteCollectionDataStep{
baseStep: baseStep{core: s.core},
coll: &model.Collection{CollectionID: s.collectionID, PhysicalChannelNames: s.channels.physicalChannels},
isSkip: s.isSkip,
}
return unwatchByDropMsg.Execute(ctx)
}
@ -183,9 +186,14 @@ func (s *expireCacheStep) Desc() string {
type deleteCollectionDataStep struct {
baseStep
coll *model.Collection
isSkip bool
}
func (s *deleteCollectionDataStep) Execute(ctx context.Context) ([]nestedStep, error) {
if s.isSkip {
return nil, nil
}
ddlTs, err := s.core.garbageCollector.GcCollectionData(ctx, s.coll)
if err != nil {
return nil, err
@ -239,9 +247,14 @@ type deletePartitionDataStep struct {
baseStep
pchans []string
partition *model.Partition
isSkip bool
}
func (s *deletePartitionDataStep) Execute(ctx context.Context) ([]nestedStep, error) {
if s.isSkip {
return nil, nil
}
_, err := s.core.garbageCollector.GcPartitionData(ctx, s.pchans, s.partition)
return nil, err
}

20
internal/rootcoord/step_test.go
View File

@ -95,3 +95,23 @@ func Test_confirmGCStep_Execute(t *testing.T) {
assert.NoError(t, err)
})
}
func TestSkip(t *testing.T) {
{
s := &unwatchChannelsStep{isSkip: true}
_, err := s.Execute(context.Background())
assert.NoError(t, err)
}
{
s := &deleteCollectionDataStep{isSkip: true}
_, err := s.Execute(context.Background())
assert.NoError(t, err)
}
{
s := &deletePartitionDataStep{isSkip: true}
_, err := s.Execute(context.Background())
assert.NoError(t, err)
}
}

3
internal/util/sessionutil/session_util_test.go
View File

@ -636,6 +636,7 @@ func TestSessionProcessActiveStandBy(t *testing.T) {
})
wg.Wait()
s1.LivenessCheck(ctx1, func() {
log.Debug("Session 1 livenessCheck callback")
flag = true
close(signal)
s1.cancelKeepAlive()
@ -675,8 +676,10 @@ func TestSessionProcessActiveStandBy(t *testing.T) {
t.FailNow()
}
assert.True(t, flag)
log.Debug("session s1 stop")
wg.Wait()
log.Debug("session s2 wait done")
assert.False(t, s2.isStandby.Load().(bool))
s2.Stop()
}

1
pkg/Makefile
View File

@ -15,3 +15,4 @@ generate-mockery: getdeps
$(INSTALL_PATH)/mockery --name=MsgStream --dir=$(PWD)/mq/msgstream --output=$(PWD)/mq/msgstream --filename=mock_msgstream.go --with-expecter --structname=MockMsgStream --outpkg=msgstream --inpackage
$(INSTALL_PATH)/mockery --name=Client --dir=$(PWD)/mq/msgdispatcher --output=$(PWD)/mq/msgsdispatcher --filename=mock_client.go --with-expecter --structname=MockClient --outpkg=msgdispatcher --inpackage
$(INSTALL_PATH)/mockery --name=Logger --dir=$(PWD)/eventlog --output=$(PWD)/eventlog --filename=mock_logger.go --with-expecter --structname=MockLogger --outpkg=eventlog --inpackage
$(INSTALL_PATH)/mockery --name=MessageID --dir=$(PWD)/mq/msgstream/mqwrapper --output=$(PWD)/mq/msgstream/mqwrapper --filename=mock_id.go --with-expecter --structname=MockMessageID --outpkg=mqwrapper --inpackage

2
pkg/go.mod
View File

@ -13,7 +13,7 @@ require (
github.com/grpc-ecosystem/go-grpc-middleware v1.3.0
github.com/klauspost/compress v1.16.5
github.com/lingdor/stackerror v0.0.0-20191119040541-976d8885ed76
github.com/milvus-io/milvus-proto/go-api/v2 v2.3.0
github.com/milvus-io/milvus-proto/go-api/v2 v2.3.2-0.20231008032233-5d64d443769d
github.com/nats-io/nats-server/v2 v2.9.17
github.com/nats-io/nats.go v1.24.0
github.com/panjf2000/ants/v2 v2.7.2

4
pkg/go.sum
View File

@ -477,8 +477,8 @@ github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfr
github.com/mediocregopher/radix/v3 v3.4.2/go.mod h1:8FL3F6UQRXHXIBSPUs5h0RybMF8i4n7wVopoX3x7Bv8=
github.com/microcosm-cc/bluemonday v1.0.2/go.mod h1:iVP4YcDBq+n/5fb23BhYFvIMq/leAFZyRl6bYmGDlGc=
github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg=
github.com/milvus-io/milvus-proto/go-api/v2 v2.3.0 h1:t5CKm7+FXuD2rDLv/H8tpN9iY8F2dZvHF87xWBx8muU=
github.com/milvus-io/milvus-proto/go-api/v2 v2.3.0/go.mod h1:1OIl0v5PQeNxIJhCvY+K55CBUOYDZevw9g9380u1Wek=
github.com/milvus-io/milvus-proto/go-api/v2 v2.3.2-0.20231008032233-5d64d443769d h1:K8yyzz8BCBm+wirhRgySyB8wN+sw33eB3VsLz6Slu5s=
github.com/milvus-io/milvus-proto/go-api/v2 v2.3.2-0.20231008032233-5d64d443769d/go.mod h1:1OIl0v5PQeNxIJhCvY+K55CBUOYDZevw9g9380u1Wek=
github.com/milvus-io/pulsar-client-go v0.6.10 h1:eqpJjU+/QX0iIhEo3nhOqMNXL+TyInAs1IAHZCrCM/A=
github.com/milvus-io/pulsar-client-go v0.6.10/go.mod h1:lQqCkgwDF8YFYjKA+zOheTk1tev2B+bKj5j7+nm8M1w=
github.com/minio/highwayhash v1.0.2 h1:Aak5U0nElisjDCfPSG79Tgzkn2gl66NxOMspRrKnA/g=

33
pkg/mq/msgstream/mock_msgstream.go
View File

@ -273,6 +273,39 @@ func (_c *MockMsgStream_Close_Call) RunAndReturn(run func()) *MockMsgStream_Clos
return _c
}
// EnableProduce provides a mock function with given fields: can
func (_m *MockMsgStream) EnableProduce(can bool) {
_m.Called(can)
}
// MockMsgStream_EnableProduce_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'EnableProduce'
type MockMsgStream_EnableProduce_Call struct {
*mock.Call
}
// EnableProduce is a helper method to define mock.On call
// - can bool
func (_e *MockMsgStream_Expecter) EnableProduce(can interface{}) *MockMsgStream_EnableProduce_Call {
return &MockMsgStream_EnableProduce_Call{Call: _e.mock.On("EnableProduce", can)}
}
func (_c *MockMsgStream_EnableProduce_Call) Run(run func(can bool)) *MockMsgStream_EnableProduce_Call {
_c.Call.Run(func(args mock.Arguments) {
run(args[0].(bool))
})
return _c
}
func (_c *MockMsgStream_EnableProduce_Call) Return() *MockMsgStream_EnableProduce_Call {
_c.Call.Return()
return _c
}
func (_c *MockMsgStream_EnableProduce_Call) RunAndReturn(run func(bool)) *MockMsgStream_EnableProduce_Call {
_c.Call.Return(run)
return _c
}
// GetLatestMsgID provides a mock function with given fields: channel
func (_m *MockMsgStream) GetLatestMsgID(channel string) (mqwrapper.MessageID, error) {
ret := _m.Called(channel)

56
pkg/mq/msgstream/mq_msgstream.go
View File

@ -20,6 +20,7 @@ import (
"context"
"fmt"
"path/filepath"
"strconv"
"sync"
"sync/atomic"
"time"
@ -31,6 +32,7 @@ import (
"github.com/milvus-io/milvus-proto/go-api/v2/commonpb"
"github.com/milvus-io/milvus-proto/go-api/v2/msgpb"
"github.com/milvus-io/milvus/pkg/config"
"github.com/milvus-io/milvus/pkg/log"
"github.com/milvus-io/milvus/pkg/mq/msgstream/mqwrapper"
"github.com/milvus-io/milvus/pkg/util/merr"
@ -50,16 +52,17 @@ type mqMsgStream struct {
consumers map[string]mqwrapper.Consumer
consumerChannels []string
repackFunc RepackFunc
unmarshal UnmarshalDispatcher
receiveBuf chan *MsgPack
closeRWMutex *sync.RWMutex
streamCancel func()
bufSize int64
producerLock *sync.RWMutex
consumerLock *sync.Mutex
closed int32
onceChan sync.Once
repackFunc RepackFunc
unmarshal UnmarshalDispatcher
receiveBuf chan *MsgPack
closeRWMutex *sync.RWMutex
streamCancel func()
bufSize int64
producerLock *sync.RWMutex
consumerLock *sync.Mutex
closed int32
onceChan sync.Once
enableProduce atomic.Value
}
// NewMqMsgStream is used to generate a new mqMsgStream object
@ -93,6 +96,18 @@ func NewMqMsgStream(ctx context.Context,
closeRWMutex: &sync.RWMutex{},
closed: 0,
}
ctxLog := log.Ctx(ctx)
stream.enableProduce.Store(paramtable.Get().CommonCfg.TTMsgEnabled.GetAsBool())
paramtable.Get().Watch(paramtable.Get().CommonCfg.TTMsgEnabled.Key, config.NewHandler("enable send tt msg", func(event *config.Event) {
value, err := strconv.ParseBool(event.Value)
if err != nil {
ctxLog.Warn("Failed to parse bool value", zap.String("v", event.Value), zap.Error(err))
return
}
stream.enableProduce.Store(value)
ctxLog.Info("Msg Stream state updated", zap.Bool("can_produce", stream.isEnabledProduce()))
}))
ctxLog.Info("Msg Stream state", zap.Bool("can_produce", stream.isEnabledProduce()))
return stream, nil
}
@ -241,7 +256,19 @@ func (ms *mqMsgStream) GetProduceChannels() []string {
return ms.producerChannels
}
func (ms *mqMsgStream) EnableProduce(can bool) {
ms.enableProduce.Store(can)
}
func (ms *mqMsgStream) isEnabledProduce() bool {
return ms.enableProduce.Load().(bool)
}
func (ms *mqMsgStream) Produce(msgPack *MsgPack) error {
if !ms.isEnabledProduce() {
log.Warn("can't produce the msg in the backup instance", zap.Stack("stack"))
return merr.ErrDenyProduceMsg
}
if msgPack == nil || len(msgPack.Msgs) <= 0 {
log.Debug("Warning: Receive empty msgPack")
return nil
@ -307,6 +334,14 @@ func (ms *mqMsgStream) Broadcast(msgPack *MsgPack) (map[string][]MessageID, erro
if msgPack == nil || len(msgPack.Msgs) <= 0 {
return ids, errors.New("empty msgs")
}
// Only allow to create collection msg in backup instance
// However, there may be a problem of ts disorder here, but because the start position of the collection only uses offsets, not time, there is no problem for the time being
isCreateCollectionMsg := len(msgPack.Msgs) == 1 && msgPack.Msgs[0].Type() == commonpb.MsgType_CreateCollection
if !ms.isEnabledProduce() && !isCreateCollectionMsg {
log.Warn("can't broadcast the msg in the backup instance", zap.Stack("stack"))
return ids, merr.ErrDenyProduceMsg
}
for _, v := range msgPack.Msgs {
spanCtx, sp := MsgSpanFromCtx(v.TraceCtx(), v)
@ -357,7 +392,6 @@ func (ms *mqMsgStream) getTsMsgFromConsumerMsg(msg mqwrapper.Message) (TsMsg, er
return nil, fmt.Errorf("failed to unmarshal tsMsg, err %s", err.Error())
}
// set msg info to tsMsg
tsMsg.SetPosition(&MsgPosition{
ChannelName: filepath.Base(msg.Topic()),
MsgID: msg.ID().Serialize(),

14
pkg/mq/msgstream/mq_msgstream_test.go
View File

@ -111,6 +111,13 @@ func TestStream_PulsarMsgStream_Insert(t *testing.T) {
inputStream := getPulsarInputStream(ctx, pulsarAddress, producerChannels)
outputStream := getPulsarOutputStream(ctx, pulsarAddress, consumerChannels, consumerSubName)
{
inputStream.EnableProduce(false)
err := inputStream.Produce(&msgPack)
require.Error(t, err)
}
inputStream.EnableProduce(true)
err := inputStream.Produce(&msgPack)
require.NoErrorf(t, err, fmt.Sprintf("produce error = %v", err))
@ -177,6 +184,13 @@ func TestStream_PulsarMsgStream_BroadCast(t *testing.T) {
inputStream := getPulsarInputStream(ctx, pulsarAddress, producerChannels)
outputStream := getPulsarOutputStream(ctx, pulsarAddress, consumerChannels, consumerSubName)
{
inputStream.EnableProduce(false)
_, err := inputStream.Broadcast(&msgPack)
require.Error(t, err)
}
inputStream.EnableProduce(true)
_, err := inputStream.Broadcast(&msgPack)
require.NoErrorf(t, err, fmt.Sprintf("broadcast error = %v", err))

2
pkg/mq/msgstream/mqwrapper/consumer.go
View File

@ -59,7 +59,7 @@ type Consumer interface {
// Get Message channel, once you chan you can not seek again
Chan() <-chan Message
// Seek to the uniqueID position
// Seek to the uniqueID position, the second bool param indicates whether the message is included in the position
Seek(MessageID, bool) error //nolint:govet
// Ack make sure that msg is received

220
pkg/mq/msgstream/mqwrapper/mock_id.go Normal file
View File

@ -0,0 +1,220 @@
// Code generated by mockery v2.32.4. DO NOT EDIT.
package mqwrapper
import mock "github.com/stretchr/testify/mock"
// MockMessageID is an autogenerated mock type for the MessageID type
type MockMessageID struct {
mock.Mock
}
type MockMessageID_Expecter struct {
mock *mock.Mock
}
func (_m *MockMessageID) EXPECT() *MockMessageID_Expecter {
return &MockMessageID_Expecter{mock: &_m.Mock}
}
// AtEarliestPosition provides a mock function with given fields:
func (_m *MockMessageID) AtEarliestPosition() bool {
ret := _m.Called()
var r0 bool
if rf, ok := ret.Get(0).(func() bool); ok {
r0 = rf()
} else {
r0 = ret.Get(0).(bool)
}
return r0
}
// MockMessageID_AtEarliestPosition_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'AtEarliestPosition'
type MockMessageID_AtEarliestPosition_Call struct {
*mock.Call
}
// AtEarliestPosition is a helper method to define mock.On call
func (_e *MockMessageID_Expecter) AtEarliestPosition() *MockMessageID_AtEarliestPosition_Call {
return &MockMessageID_AtEarliestPosition_Call{Call: _e.mock.On("AtEarliestPosition")}
}
func (_c *MockMessageID_AtEarliestPosition_Call) Run(run func()) *MockMessageID_AtEarliestPosition_Call {
_c.Call.Run(func(args mock.Arguments) {
run()
})
return _c
}
func (_c *MockMessageID_AtEarliestPosition_Call) Return(_a0 bool) *MockMessageID_AtEarliestPosition_Call {
_c.Call.Return(_a0)
return _c
}
func (_c *MockMessageID_AtEarliestPosition_Call) RunAndReturn(run func() bool) *MockMessageID_AtEarliestPosition_Call {
_c.Call.Return(run)
return _c
}
// Equal provides a mock function with given fields: msgID
func (_m *MockMessageID) Equal(msgID []byte) (bool, error) {
ret := _m.Called(msgID)
var r0 bool
var r1 error
if rf, ok := ret.Get(0).(func([]byte) (bool, error)); ok {
return rf(msgID)
}
if rf, ok := ret.Get(0).(func([]byte) bool); ok {
r0 = rf(msgID)
} else {
r0 = ret.Get(0).(bool)
}
if rf, ok := ret.Get(1).(func([]byte) error); ok {
r1 = rf(msgID)
} else {
r1 = ret.Error(1)
}
return r0, r1
}
// MockMessageID_Equal_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Equal'
type MockMessageID_Equal_Call struct {
*mock.Call
}
// Equal is a helper method to define mock.On call
// - msgID []byte
func (_e *MockMessageID_Expecter) Equal(msgID interface{}) *MockMessageID_Equal_Call {
return &MockMessageID_Equal_Call{Call: _e.mock.On("Equal", msgID)}
}
func (_c *MockMessageID_Equal_Call) Run(run func(msgID []byte)) *MockMessageID_Equal_Call {
_c.Call.Run(func(args mock.Arguments) {
run(args[0].([]byte))
})
return _c
}
func (_c *MockMessageID_Equal_Call) Return(_a0 bool, _a1 error) *MockMessageID_Equal_Call {
_c.Call.Return(_a0, _a1)
return _c
}
func (_c *MockMessageID_Equal_Call) RunAndReturn(run func([]byte) (bool, error)) *MockMessageID_Equal_Call {
_c.Call.Return(run)
return _c
}
// LessOrEqualThan provides a mock function with given fields: msgID
func (_m *MockMessageID) LessOrEqualThan(msgID []byte) (bool, error) {
ret := _m.Called(msgID)
var r0 bool
var r1 error
if rf, ok := ret.Get(0).(func([]byte) (bool, error)); ok {
return rf(msgID)
}
if rf, ok := ret.Get(0).(func([]byte) bool); ok {
r0 = rf(msgID)
} else {
r0 = ret.Get(0).(bool)
}
if rf, ok := ret.Get(1).(func([]byte) error); ok {
r1 = rf(msgID)
} else {
r1 = ret.Error(1)
}
return r0, r1
}
// MockMessageID_LessOrEqualThan_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'LessOrEqualThan'
type MockMessageID_LessOrEqualThan_Call struct {
*mock.Call
}
// LessOrEqualThan is a helper method to define mock.On call
// - msgID []byte
func (_e *MockMessageID_Expecter) LessOrEqualThan(msgID interface{}) *MockMessageID_LessOrEqualThan_Call {
return &MockMessageID_LessOrEqualThan_Call{Call: _e.mock.On("LessOrEqualThan", msgID)}
}
func (_c *MockMessageID_LessOrEqualThan_Call) Run(run func(msgID []byte)) *MockMessageID_LessOrEqualThan_Call {
_c.Call.Run(func(args mock.Arguments) {
run(args[0].([]byte))
})
return _c
}
func (_c *MockMessageID_LessOrEqualThan_Call) Return(_a0 bool, _a1 error) *MockMessageID_LessOrEqualThan_Call {
_c.Call.Return(_a0, _a1)
return _c
}
func (_c *MockMessageID_LessOrEqualThan_Call) RunAndReturn(run func([]byte) (bool, error)) *MockMessageID_LessOrEqualThan_Call {
_c.Call.Return(run)
return _c
}
// Serialize provides a mock function with given fields:
func (_m *MockMessageID) Serialize() []byte {
ret := _m.Called()
var r0 []byte
if rf, ok := ret.Get(0).(func() []byte); ok {
r0 = rf()
} else {
if ret.Get(0) != nil {
r0 = ret.Get(0).([]byte)
}
}
return r0
}
// MockMessageID_Serialize_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Serialize'
type MockMessageID_Serialize_Call struct {
*mock.Call
}
// Serialize is a helper method to define mock.On call
func (_e *MockMessageID_Expecter) Serialize() *MockMessageID_Serialize_Call {
return &MockMessageID_Serialize_Call{Call: _e.mock.On("Serialize")}
}
func (_c *MockMessageID_Serialize_Call) Run(run func()) *MockMessageID_Serialize_Call {
_c.Call.Run(func(args mock.Arguments) {
run()
})
return _c
}
func (_c *MockMessageID_Serialize_Call) Return(_a0 []byte) *MockMessageID_Serialize_Call {
_c.Call.Return(_a0)
return _c
}
func (_c *MockMessageID_Serialize_Call) RunAndReturn(run func() []byte) *MockMessageID_Serialize_Call {
_c.Call.Return(run)
return _c
}
// NewMockMessageID creates a new instance of MockMessageID. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations.
// The first argument is typically a *testing.T value.
func NewMockMessageID(t interface {
mock.TestingT
Cleanup(func())
}) *MockMessageID {
mock := &MockMessageID{}
mock.Mock.Test(t)
t.Cleanup(func() { mock.AssertExpectations(t) })
return mock
}

190
pkg/mq/msgstream/msg_for_collection.go Normal file
View File

@ -0,0 +1,190 @@
/*
* Licensed to the LF AI & Data foundation under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package msgstream
import (
"github.com/golang/protobuf/proto"
"github.com/milvus-io/milvus-proto/go-api/v2/milvuspb"
)
// LoadCollectionMsg is a message pack that contains load collection request
type LoadCollectionMsg struct {
BaseMsg
milvuspb.LoadCollectionRequest
}
// interface implementation validation
var _ TsMsg = &LoadCollectionMsg{}
func (l *LoadCollectionMsg) ID() UniqueID {
return l.Base.MsgID
}
func (l *LoadCollectionMsg) SetID(id UniqueID) {
l.Base.MsgID = id
}
func (l *LoadCollectionMsg) Type() MsgType {
return l.Base.MsgType
}
func (l *LoadCollectionMsg) SourceID() int64 {
return l.Base.SourceID
}
func (l *LoadCollectionMsg) Marshal(input TsMsg) (MarshalType, error) {
loadCollectionMsg := input.(*LoadCollectionMsg)
createIndexRequest := &loadCollectionMsg.LoadCollectionRequest
mb, err := proto.Marshal(createIndexRequest)
if err != nil {
return nil, err
}
return mb, nil
}
func (l *LoadCollectionMsg) Unmarshal(input MarshalType) (TsMsg, error) {
loadCollectionRequest := milvuspb.LoadCollectionRequest{}
in, err := convertToByteArray(input)
if err != nil {
return nil, err
}
err = proto.Unmarshal(in, &loadCollectionRequest)
if err != nil {
return nil, err
}
loadCollectionMsg := &LoadCollectionMsg{LoadCollectionRequest: loadCollectionRequest}
loadCollectionMsg.BeginTimestamp = loadCollectionMsg.GetBase().GetTimestamp()
loadCollectionMsg.EndTimestamp = loadCollectionMsg.GetBase().GetTimestamp()
return loadCollectionMsg, nil
}
func (l *LoadCollectionMsg) Size() int {
return proto.Size(&l.LoadCollectionRequest)
}
// ReleaseCollectionMsg is a message pack that contains release collection request
type ReleaseCollectionMsg struct {
BaseMsg
milvuspb.ReleaseCollectionRequest
}
var _ TsMsg = &ReleaseCollectionMsg{}
func (r *ReleaseCollectionMsg) ID() UniqueID {
return r.Base.MsgID
}
func (r *ReleaseCollectionMsg) SetID(id UniqueID) {
r.Base.MsgID = id
}
func (r *ReleaseCollectionMsg) Type() MsgType {
return r.Base.MsgType
}
func (r *ReleaseCollectionMsg) SourceID() int64 {
return r.Base.SourceID
}
func (r *ReleaseCollectionMsg) Marshal(input TsMsg) (MarshalType, error) {
releaseCollectionMsg := input.(*ReleaseCollectionMsg)
releaseCollectionRequest := &releaseCollectionMsg.ReleaseCollectionRequest
mb, err := proto.Marshal(releaseCollectionRequest)
if err != nil {
return nil, err
}
return mb, nil
}
func (r *ReleaseCollectionMsg) Unmarshal(input MarshalType) (TsMsg, error) {
releaseCollectionRequest := milvuspb.ReleaseCollectionRequest{}
in, err := convertToByteArray(input)
if err != nil {
return nil, err
}
err = proto.Unmarshal(in, &releaseCollectionRequest)
if err != nil {
return nil, err
}
releaseCollectionMsg := &ReleaseCollectionMsg{ReleaseCollectionRequest: releaseCollectionRequest}
releaseCollectionMsg.BeginTimestamp = releaseCollectionMsg.GetBase().GetTimestamp()
releaseCollectionMsg.EndTimestamp = releaseCollectionMsg.GetBase().GetTimestamp()
return releaseCollectionMsg, nil
}
func (r *ReleaseCollectionMsg) Size() int {
return proto.Size(&r.ReleaseCollectionRequest)
}
type FlushMsg struct {
BaseMsg
milvuspb.FlushRequest
}
var _ TsMsg = &FlushMsg{}
func (f *FlushMsg) ID() UniqueID {
return f.Base.MsgID
}
func (f *FlushMsg) SetID(id UniqueID) {
f.Base.MsgID = id
}
func (f *FlushMsg) Type() MsgType {
return f.Base.MsgType
}
func (f *FlushMsg) SourceID() int64 {
return f.Base.SourceID
}
func (f *FlushMsg) Marshal(input TsMsg) (MarshalType, error) {
flushMsg := input.(*FlushMsg)
flushRequest := &flushMsg.FlushRequest
mb, err := proto.Marshal(flushRequest)
if err != nil {
return nil, err
}
return mb, nil
}
func (f *FlushMsg) Unmarshal(input MarshalType) (TsMsg, error) {
flushRequest := milvuspb.FlushRequest{}
in, err := convertToByteArray(input)
if err != nil {
return nil, err
}
err = proto.Unmarshal(in, &flushRequest)
if err != nil {
return nil, err
}
flushMsg := &FlushMsg{FlushRequest: flushRequest}
flushMsg.BeginTimestamp = flushMsg.GetBase().GetTimestamp()
flushMsg.EndTimestamp = flushMsg.GetBase().GetTimestamp()
return flushMsg, nil
}
func (f *FlushMsg) Size() int {
return proto.Size(&f.FlushRequest)
}

139
pkg/mq/msgstream/msg_for_collection_test.go Normal file
View File

@ -0,0 +1,139 @@
/*
* Licensed to the LF AI & Data foundation under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package msgstream
import (
"testing"
"github.com/stretchr/testify/assert"
"github.com/milvus-io/milvus-proto/go-api/v2/commonpb"
"github.com/milvus-io/milvus-proto/go-api/v2/milvuspb"
)
func TestFlushMsg(t *testing.T) {
var msg TsMsg = &FlushMsg{
FlushRequest: milvuspb.FlushRequest{
Base: &commonpb.MsgBase{
MsgType: commonpb.MsgType_Flush,
MsgID: 100,
Timestamp: 1000,
SourceID: 10000,
TargetID: 100000,
ReplicateInfo: nil,
},
DbName: "unit_db",
CollectionNames: []string{"col1", "col2"},
},
}
assert.EqualValues(t, 100, msg.ID())
msg.SetID(200)
assert.EqualValues(t, 200, msg.ID())
assert.Equal(t, commonpb.MsgType_Flush, msg.Type())
assert.EqualValues(t, 10000, msg.SourceID())
msgBytes, err := msg.Marshal(msg)
assert.NoError(t, err)
var newMsg TsMsg = &FlushMsg{}
_, err = newMsg.Unmarshal("1")
assert.Error(t, err)
newMsg, err = newMsg.Unmarshal(msgBytes)
assert.NoError(t, err)
assert.EqualValues(t, 200, newMsg.ID())
assert.EqualValues(t, 1000, newMsg.BeginTs())
assert.EqualValues(t, 1000, newMsg.EndTs())
assert.True(t, msg.Size() > 0)
}
func TestLoadCollection(t *testing.T) {
var msg TsMsg = &LoadCollectionMsg{
LoadCollectionRequest: milvuspb.LoadCollectionRequest{
Base: &commonpb.MsgBase{
MsgType: commonpb.MsgType_LoadCollection,
MsgID: 100,
Timestamp: 1000,
SourceID: 10000,
TargetID: 100000,
ReplicateInfo: nil,
},
DbName: "unit_db",
CollectionName: "col1",
},
}
assert.EqualValues(t, 100, msg.ID())
msg.SetID(200)
assert.EqualValues(t, 200, msg.ID())
assert.Equal(t, commonpb.MsgType_LoadCollection, msg.Type())
assert.EqualValues(t, 10000, msg.SourceID())
msgBytes, err := msg.Marshal(msg)
assert.NoError(t, err)
var newMsg TsMsg = &LoadCollectionMsg{}
_, err = newMsg.Unmarshal("1")
assert.Error(t, err)
newMsg, err = newMsg.Unmarshal(msgBytes)
assert.NoError(t, err)
assert.EqualValues(t, 200, newMsg.ID())
assert.EqualValues(t, 1000, newMsg.BeginTs())
assert.EqualValues(t, 1000, newMsg.EndTs())
assert.True(t, msg.Size() > 0)
}
func TestReleaseCollection(t *testing.T) {
var msg TsMsg = &ReleaseCollectionMsg{
ReleaseCollectionRequest: milvuspb.ReleaseCollectionRequest{
Base: &commonpb.MsgBase{
MsgType: commonpb.MsgType_ReleaseCollection,
MsgID: 100,
Timestamp: 1000,
SourceID: 10000,
TargetID: 100000,
ReplicateInfo: nil,
},
DbName: "unit_db",
CollectionName: "col1",
},
}
assert.EqualValues(t, 100, msg.ID())
msg.SetID(200)
assert.EqualValues(t, 200, msg.ID())
assert.Equal(t, commonpb.MsgType_ReleaseCollection, msg.Type())
assert.EqualValues(t, 10000, msg.SourceID())
msgBytes, err := msg.Marshal(msg)
assert.NoError(t, err)
var newMsg TsMsg = &ReleaseCollectionMsg{}
_, err = newMsg.Unmarshal("1")
assert.Error(t, err)
newMsg, err = newMsg.Unmarshal(msgBytes)
assert.NoError(t, err)
assert.EqualValues(t, 200, newMsg.ID())
assert.EqualValues(t, 1000, newMsg.BeginTs())
assert.EqualValues(t, 1000, newMsg.EndTs())
assert.True(t, msg.Size() > 0)
}

133
pkg/mq/msgstream/msg_for_database.go Normal file
View File

@ -0,0 +1,133 @@
/*
* Licensed to the LF AI & Data foundation under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package msgstream
import (
"github.com/golang/protobuf/proto"
"github.com/milvus-io/milvus-proto/go-api/v2/milvuspb"
)
type CreateDatabaseMsg struct {
BaseMsg
milvuspb.CreateDatabaseRequest
}
var _ TsMsg = &CreateDatabaseMsg{}
func (c *CreateDatabaseMsg) ID() UniqueID {
return c.Base.MsgID
}
func (c *CreateDatabaseMsg) SetID(id UniqueID) {
c.Base.MsgID = id
}
func (c *CreateDatabaseMsg) Type() MsgType {
return c.Base.MsgType
}
func (c *CreateDatabaseMsg) SourceID() int64 {
return c.Base.SourceID
}
func (c *CreateDatabaseMsg) Marshal(input TsMsg) (MarshalType, error) {
createDataBaseMsg := input.(*CreateDatabaseMsg)
createDatabaseRequest := &createDataBaseMsg.CreateDatabaseRequest
mb, err := proto.Marshal(createDatabaseRequest)
if err != nil {
return nil, err
}
return mb, nil
}
func (c *CreateDatabaseMsg) Unmarshal(input MarshalType) (TsMsg, error) {
createDatabaseRequest := milvuspb.CreateDatabaseRequest{}
in, err := convertToByteArray(input)
if err != nil {
return nil, err
}
err = proto.Unmarshal(in, &createDatabaseRequest)
if err != nil {
return nil, err
}
createDatabaseMsg := &CreateDatabaseMsg{CreateDatabaseRequest: createDatabaseRequest}
createDatabaseMsg.BeginTimestamp = createDatabaseMsg.GetBase().GetTimestamp()
createDatabaseMsg.EndTimestamp = createDatabaseMsg.GetBase().GetTimestamp()
return createDatabaseMsg, nil
}
func (c *CreateDatabaseMsg) Size() int {
return proto.Size(&c.CreateDatabaseRequest)
}
type DropDatabaseMsg struct {
BaseMsg
milvuspb.DropDatabaseRequest
}
var _ TsMsg = &DropDatabaseMsg{}
func (d *DropDatabaseMsg) ID() UniqueID {
return d.Base.MsgID
}
func (d *DropDatabaseMsg) SetID(id UniqueID) {
d.Base.MsgID = id
}
func (d *DropDatabaseMsg) Type() MsgType {
return d.Base.MsgType
}
func (d *DropDatabaseMsg) SourceID() int64 {
return d.Base.SourceID
}
func (d *DropDatabaseMsg) Marshal(input TsMsg) (MarshalType, error) {
dropDataBaseMsg := input.(*DropDatabaseMsg)
dropDatabaseRequest := &dropDataBaseMsg.DropDatabaseRequest
mb, err := proto.Marshal(dropDatabaseRequest)
if err != nil {
return nil, err
}
return mb, nil
}
func (d *DropDatabaseMsg) Unmarshal(input MarshalType) (TsMsg, error) {
dropDatabaseRequest := milvuspb.DropDatabaseRequest{}
in, err := convertToByteArray(input)
if err != nil {
return nil, err
}
err = proto.Unmarshal(in, &dropDatabaseRequest)
if err != nil {
return nil, err
}
dropDatabaseMsg := &DropDatabaseMsg{DropDatabaseRequest: dropDatabaseRequest}
dropDatabaseMsg.BeginTimestamp = dropDatabaseMsg.GetBase().GetTimestamp()
dropDatabaseMsg.EndTimestamp = dropDatabaseMsg.GetBase().GetTimestamp()
return dropDatabaseMsg, nil
}
func (d *DropDatabaseMsg) Size() int {
return proto.Size(&d.DropDatabaseRequest)
}

100
pkg/mq/msgstream/msg_for_database_test.go Normal file
View File

@ -0,0 +1,100 @@
/*
* Licensed to the LF AI & Data foundation under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package msgstream
import (
"testing"
"github.com/stretchr/testify/assert"
"github.com/milvus-io/milvus-proto/go-api/v2/commonpb"
"github.com/milvus-io/milvus-proto/go-api/v2/milvuspb"
)
func TestCreateDatabase(t *testing.T) {
var msg TsMsg = &CreateDatabaseMsg{
CreateDatabaseRequest: milvuspb.CreateDatabaseRequest{
Base: &commonpb.MsgBase{
MsgType: commonpb.MsgType_CreateDatabase,
MsgID: 100,
Timestamp: 1000,
SourceID: 10000,
TargetID: 100000,
ReplicateInfo: nil,
},
DbName: "unit_db",
},
}
assert.EqualValues(t, 100, msg.ID())
msg.SetID(200)
assert.EqualValues(t, 200, msg.ID())
assert.Equal(t, commonpb.MsgType_CreateDatabase, msg.Type())
assert.EqualValues(t, 10000, msg.SourceID())
msgBytes, err := msg.Marshal(msg)
assert.NoError(t, err)
var newMsg TsMsg = &ReleaseCollectionMsg{}
_, err = newMsg.Unmarshal("1")
assert.Error(t, err)
newMsg, err = newMsg.Unmarshal(msgBytes)
assert.NoError(t, err)
assert.EqualValues(t, 200, newMsg.ID())
assert.EqualValues(t, 1000, newMsg.BeginTs())
assert.EqualValues(t, 1000, newMsg.EndTs())
assert.True(t, msg.Size() > 0)
}
func TestDropDatabase(t *testing.T) {
var msg TsMsg = &DropDatabaseMsg{
DropDatabaseRequest: milvuspb.DropDatabaseRequest{
Base: &commonpb.MsgBase{
MsgType: commonpb.MsgType_DropDatabase,
MsgID: 100,
Timestamp: 1000,
SourceID: 10000,
TargetID: 100000,
ReplicateInfo: nil,
},
DbName: "unit_db",
},
}
assert.EqualValues(t, 100, msg.ID())
msg.SetID(200)
assert.EqualValues(t, 200, msg.ID())
assert.Equal(t, commonpb.MsgType_DropDatabase, msg.Type())
assert.EqualValues(t, 10000, msg.SourceID())
msgBytes, err := msg.Marshal(msg)
assert.NoError(t, err)
var newMsg TsMsg = &DropDatabaseMsg{}
_, err = newMsg.Unmarshal("1")
assert.Error(t, err)
newMsg, err = newMsg.Unmarshal(msgBytes)
assert.NoError(t, err)
assert.EqualValues(t, 200, newMsg.ID())
assert.EqualValues(t, 1000, newMsg.BeginTs())
assert.EqualValues(t, 1000, newMsg.EndTs())
assert.True(t, msg.Size() > 0)
}

142
pkg/mq/msgstream/msg_for_index.go Normal file
View File

@ -0,0 +1,142 @@
/*
* Licensed to the LF AI & Data foundation under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package msgstream
import (
"github.com/golang/protobuf/proto"
"github.com/milvus-io/milvus-proto/go-api/v2/milvuspb"
)
// CreateIndexMsg is a message pack that contains create index request
type CreateIndexMsg struct {
BaseMsg
milvuspb.CreateIndexRequest
}
// interface implementation validation
var _ TsMsg = &CreateIndexMsg{}
// ID returns the ID of this message pack
func (it *CreateIndexMsg) ID() UniqueID {
return it.Base.MsgID
}
// SetID set the ID of this message pack
func (it *CreateIndexMsg) SetID(id UniqueID) {
it.Base.MsgID = id
}
// Type returns the type of this message pack
func (it *CreateIndexMsg) Type() MsgType {
return it.Base.MsgType
}
// SourceID indicates which component generated this message
func (it *CreateIndexMsg) SourceID() int64 {
return it.Base.SourceID
}
// Marshal is used to serialize a message pack to byte array
func (it *CreateIndexMsg) Marshal(input TsMsg) (MarshalType, error) {
createIndexMsg := input.(*CreateIndexMsg)
createIndexRequest := &createIndexMsg.CreateIndexRequest
mb, err := proto.Marshal(createIndexRequest)
if err != nil {
return nil, err
}
return mb, nil
}
// Unmarshal is used to deserialize a message pack from byte array
func (it *CreateIndexMsg) Unmarshal(input MarshalType) (TsMsg, error) {
createIndexRequest := milvuspb.CreateIndexRequest{}
in, err := convertToByteArray(input)
if err != nil {
return nil, err
}
err = proto.Unmarshal(in, &createIndexRequest)
if err != nil {
return nil, err
}
createIndexMsg := &CreateIndexMsg{CreateIndexRequest: createIndexRequest}
createIndexMsg.BeginTimestamp = createIndexMsg.GetBase().GetTimestamp()
createIndexMsg.EndTimestamp = createIndexMsg.GetBase().GetTimestamp()
return createIndexMsg, nil
}
func (it *CreateIndexMsg) Size() int {
return proto.Size(&it.CreateIndexRequest)
}
// DropIndexMsg is a message pack that contains drop index request
type DropIndexMsg struct {
BaseMsg
milvuspb.DropIndexRequest
}
var _ TsMsg = &DropIndexMsg{}
func (d *DropIndexMsg) ID() UniqueID {
return d.Base.MsgID
}
func (d *DropIndexMsg) SetID(id UniqueID) {
d.Base.MsgID = id
}
func (d *DropIndexMsg) Type() MsgType {
return d.Base.MsgType
}
func (d *DropIndexMsg) SourceID() int64 {
return d.Base.SourceID
}
func (d *DropIndexMsg) Marshal(input TsMsg) (MarshalType, error) {
dropIndexMsg := input.(*DropIndexMsg)
dropIndexRequest := &dropIndexMsg.DropIndexRequest
mb, err := proto.Marshal(dropIndexRequest)
if err != nil {
return nil, err
}
return mb, nil
}
func (d *DropIndexMsg) Unmarshal(input MarshalType) (TsMsg, error) {
dropIndexRequest := milvuspb.DropIndexRequest{}
in, err := convertToByteArray(input)
if err != nil {
return nil, err
}
err = proto.Unmarshal(in, &dropIndexRequest)
if err != nil {
return nil, err
}
dropIndexMsg := &DropIndexMsg{DropIndexRequest: dropIndexRequest}
dropIndexMsg.BeginTimestamp = dropIndexMsg.GetBase().GetTimestamp()
dropIndexMsg.EndTimestamp = dropIndexMsg.GetBase().GetTimestamp()
return dropIndexMsg, nil
}
func (d *DropIndexMsg) Size() int {
return proto.Size(&d.DropIndexRequest)
}

100
pkg/mq/msgstream/msg_for_index_test.go Normal file
View File

@ -0,0 +1,100 @@
/*
* Licensed to the LF AI & Data foundation under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package msgstream
import (
"testing"
"github.com/stretchr/testify/assert"
"github.com/milvus-io/milvus-proto/go-api/v2/commonpb"
"github.com/milvus-io/milvus-proto/go-api/v2/milvuspb"
)
func TestCreateIndex(t *testing.T) {
var msg TsMsg = &CreateIndexMsg{
CreateIndexRequest: milvuspb.CreateIndexRequest{
Base: &commonpb.MsgBase{
MsgType: commonpb.MsgType_CreateIndex,
MsgID: 100,
Timestamp: 1000,
SourceID: 10000,
TargetID: 100000,
ReplicateInfo: nil,
},
DbName: "unit_db",
},
}
assert.EqualValues(t, 100, msg.ID())
msg.SetID(200)
assert.EqualValues(t, 200, msg.ID())
assert.Equal(t, commonpb.MsgType_CreateIndex, msg.Type())
assert.EqualValues(t, 10000, msg.SourceID())
msgBytes, err := msg.Marshal(msg)
assert.NoError(t, err)
var newMsg TsMsg = &ReleaseCollectionMsg{}
_, err = newMsg.Unmarshal("1")
assert.Error(t, err)
newMsg, err = newMsg.Unmarshal(msgBytes)
assert.NoError(t, err)
assert.EqualValues(t, 200, newMsg.ID())
assert.EqualValues(t, 1000, newMsg.BeginTs())
assert.EqualValues(t, 1000, newMsg.EndTs())
assert.True(t, msg.Size() > 0)
}
func TestDropIndex(t *testing.T) {
var msg TsMsg = &DropIndexMsg{
DropIndexRequest: milvuspb.DropIndexRequest{
Base: &commonpb.MsgBase{
MsgType: commonpb.MsgType_DropIndex,
MsgID: 100,
Timestamp: 1000,
SourceID: 10000,
TargetID: 100000,
ReplicateInfo: nil,
},
DbName: "unit_db",
},
}
assert.EqualValues(t, 100, msg.ID())
msg.SetID(200)
assert.EqualValues(t, 200, msg.ID())
assert.Equal(t, commonpb.MsgType_DropIndex, msg.Type())
assert.EqualValues(t, 10000, msg.SourceID())
msgBytes, err := msg.Marshal(msg)
assert.NoError(t, err)
var newMsg TsMsg = &ReleaseCollectionMsg{}
_, err = newMsg.Unmarshal("1")
assert.Error(t, err)
newMsg, err = newMsg.Unmarshal(msgBytes)
assert.NoError(t, err)
assert.EqualValues(t, 200, newMsg.ID())
assert.EqualValues(t, 1000, newMsg.BeginTs())
assert.EqualValues(t, 1000, newMsg.EndTs())
assert.True(t, msg.Size() > 0)
}

2
pkg/mq/msgstream/msgstream.go
View File

@ -67,6 +67,8 @@ type MsgStream interface {
GetLatestMsgID(channel string) (MessageID, error)
CheckTopicValid(channel string) error
EnableProduce(can bool)
}
type Factory interface {

25
pkg/mq/msgstream/msgstream_util.go
View File

@ -18,10 +18,13 @@ package msgstream
import (
"context"
"fmt"
"math/rand"
"go.uber.org/zap"
"github.com/milvus-io/milvus/pkg/log"
"github.com/milvus-io/milvus/pkg/mq/msgstream/mqwrapper"
)
// unsubscribeChannels create consumer first, and unsubscribe channel through msgStream.close()
@ -34,3 +37,25 @@ func UnsubscribeChannels(ctx context.Context, factory Factory, subName string, c
panic(err)
}
}
func GetChannelLatestMsgID(ctx context.Context, factory Factory, channelName string) ([]byte, error) {
dmlStream, err := factory.NewMsgStream(ctx)
if err != nil {
log.Warn("fail to NewMsgStream", zap.String("channelName", channelName), zap.Error(err))
return nil, err
}
defer dmlStream.Close()
subName := fmt.Sprintf("get-latest_msg_id-%s-%d", channelName, rand.Int())
err = dmlStream.AsConsumer(ctx, []string{channelName}, subName, mqwrapper.SubscriptionPositionUnknown)
if err != nil {
log.Warn("fail to AsConsumer", zap.String("channelName", channelName), zap.Error(err))
return nil, err
}
id, err := dmlStream.GetLatestMsgID(channelName)
if err != nil {
log.Error("fail to GetLatestMsgID", zap.String("channelName", channelName), zap.Error(err))
return nil, err
}
return id.Serialize(), nil
}

44
pkg/mq/msgstream/msgstream_util_test.go
View File

@ -20,7 +20,11 @@ import (
"context"
"testing"
"github.com/cockroachdb/errors"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/mock"
"github.com/milvus-io/milvus/pkg/mq/msgstream/mqwrapper"
)
func TestPulsarMsgUtil(t *testing.T) {
@ -36,3 +40,43 @@ func TestPulsarMsgUtil(t *testing.T) {
UnsubscribeChannels(ctx, pmsFactory, "sub", []string{"test"})
}
func TestGetLatestMsgID(t *testing.T) {
factory := NewMockMqFactory()
ctx := context.Background()
{
factory.NewMsgStreamFunc = func(ctx context.Context) (MsgStream, error) {
return nil, errors.New("mock")
}
_, err := GetChannelLatestMsgID(ctx, factory, "test")
assert.Error(t, err)
}
stream := NewMockMsgStream(t)
factory.NewMsgStreamFunc = func(ctx context.Context) (MsgStream, error) {
return stream, nil
}
stream.EXPECT().Close().Return()
{
stream.EXPECT().AsConsumer(mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(errors.New("mock")).Once()
_, err := GetChannelLatestMsgID(ctx, factory, "test")
assert.Error(t, err)
}
{
stream.EXPECT().AsConsumer(mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(nil).Once()
stream.EXPECT().GetLatestMsgID(mock.Anything).Return(nil, errors.New("mock")).Once()
_, err := GetChannelLatestMsgID(ctx, factory, "test")
assert.Error(t, err)
}
{
mockMsgID := mqwrapper.NewMockMessageID(t)
mockMsgID.EXPECT().Serialize().Return([]byte("mock")).Once()
stream.EXPECT().AsConsumer(mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(nil).Once()
stream.EXPECT().GetLatestMsgID(mock.Anything).Return(mockMsgID, nil).Once()
id, err := GetChannelLatestMsgID(ctx, factory, "test")
assert.NoError(t, err)
assert.Equal(t, []byte("mock"), id)
}
}

17
pkg/mq/msgstream/unmarshal.go
View File

@ -63,6 +63,16 @@ func (pudf *ProtoUDFactory) NewUnmarshalDispatcher() *ProtoUnmarshalDispatcher {
dropPartitionMsg := DropPartitionMsg{}
dataNodeTtMsg := DataNodeTtMsg{}
createIndexMsg := CreateIndexMsg{}
dropIndexMsg := DropIndexMsg{}
loadCollectionMsg := LoadCollectionMsg{}
releaseCollectionMsg := ReleaseCollectionMsg{}
flushMsg := FlushMsg{}
createDatabaseMsg := CreateDatabaseMsg{}
dropDatabaseMsg := DropDatabaseMsg{}
p := &ProtoUnmarshalDispatcher{}
p.TempMap = make(map[commonpb.MsgType]UnmarshalFunc)
p.TempMap[commonpb.MsgType_Insert] = insertMsg.Unmarshal
@ -73,6 +83,13 @@ func (pudf *ProtoUDFactory) NewUnmarshalDispatcher() *ProtoUnmarshalDispatcher {
p.TempMap[commonpb.MsgType_CreatePartition] = createPartitionMsg.Unmarshal
p.TempMap[commonpb.MsgType_DropPartition] = dropPartitionMsg.Unmarshal
p.TempMap[commonpb.MsgType_DataNodeTt] = dataNodeTtMsg.Unmarshal
p.TempMap[commonpb.MsgType_CreateIndex] = createIndexMsg.Unmarshal
p.TempMap[commonpb.MsgType_DropIndex] = dropIndexMsg.Unmarshal
p.TempMap[commonpb.MsgType_LoadCollection] = loadCollectionMsg.Unmarshal
p.TempMap[commonpb.MsgType_ReleaseCollection] = releaseCollectionMsg.Unmarshal
p.TempMap[commonpb.MsgType_Flush] = flushMsg.Unmarshal
p.TempMap[commonpb.MsgType_CreateDatabase] = createDatabaseMsg.Unmarshal
p.TempMap[commonpb.MsgType_DropDatabase] = dropDatabaseMsg.Unmarshal
return p
}

7
pkg/util/merr/errors.go
View File

@ -105,6 +105,7 @@ var (
ErrMqTopicNotFound = newMilvusError("topic not found", 1300, false)
ErrMqTopicNotEmpty = newMilvusError("topic not empty", 1301, false)
ErrMqInternal = newMilvusError("message queue internal error", 1302, false)
ErrDenyProduceMsg = newMilvusError("deny to write the message to mq", 1303, false)
// Privilege related
// this operation is denied because the user not authorized, user need to login in first
@ -130,6 +131,12 @@ var (
ErrInvalidSearchResult = newMilvusError("fail to parse search result", 1805, false)
ErrCheckPrimaryKey = newMilvusError("please check the primary key and its' type can only in [int, string]", 1806, false)
// replicate related
ErrDenyReplicateMessage = newMilvusError("deny to use the replicate message in the normal instance", 1900, false)
ErrInvalidMsgBytes = newMilvusError("invalid replicate msg bytes", 1901, false)
ErrNoAssignSegmentID = newMilvusError("no assign segment id", 1902, false)
ErrInvalidStreamObj = newMilvusError("invalid stream object", 1903, false)
// Segcore related
ErrSegcore = newMilvusError("segcore error", 2000, false)

29
pkg/util/paramtable/component_param.go
View File

@ -162,6 +162,7 @@ type commonConfig struct {
RootCoordTimeTick ParamItem `refreshable:"true"`
RootCoordStatistics ParamItem `refreshable:"true"`
RootCoordDml ParamItem `refreshable:"false"`
ReplicateMsgChannel ParamItem `refreshable:"false"`
QueryCoordTimeTick ParamItem `refreshable:"true"`
@ -216,6 +217,8 @@ type commonConfig struct {
EnableLockMetrics ParamItem `refreshable:"false"`
LockSlowLogInfoThreshold ParamItem `refreshable:"true"`
LockSlowLogWarnThreshold ParamItem `refreshable:"true"`
TTMsgEnabled ParamItem `refreshable:"true"`
}
func (p *commonConfig) init(base *BaseTable) {
@ -266,6 +269,16 @@ func (p *commonConfig) init(base *BaseTable) {
}
p.RootCoordDml.Init(base.mgr)
p.ReplicateMsgChannel = ParamItem{
Key: "msgChannel.chanNamePrefix.replicateMsg",
Version: "2.3.2",
FallbackKeys: []string{"common.chanNamePrefix.replicateMsg"},
PanicIfEmpty: true,
Formatter: chanNamePrefix,
Export: true,
}
p.ReplicateMsgChannel.Init(base.mgr)
p.QueryCoordTimeTick = ParamItem{
Key: "msgChannel.chanNamePrefix.queryTimeTick",
Version: "2.1.0",
@ -612,6 +625,14 @@ like the old password verification when updating the credential`,
Export: true,
}
p.LockSlowLogWarnThreshold.Init(base.mgr)
p.TTMsgEnabled = ParamItem{
Key: "common.ttMsgEnabled",
Version: "2.3.2",
DefaultValue: "true",
Doc: "Whether the instance disable sending ts messages",
}
p.TTMsgEnabled.Init(base.mgr)
}
type traceConfig struct {
@ -1143,11 +1164,11 @@ type queryCoordConfig struct {
TaskMergeCap ParamItem `refreshable:"false"`
TaskExecutionCap ParamItem `refreshable:"true"`
//---- Handoff ---
//Deprecated: Since 2.2.2
// ---- Handoff ---
// Deprecated: Since 2.2.2
AutoHandoff ParamItem `refreshable:"true"`
//---- Balance ---
// ---- Balance ---
AutoBalance ParamItem `refreshable:"true"`
Balancer ParamItem `refreshable:"true"`
GlobalRowCountFactor ParamItem `refreshable:"true"`
@ -1186,7 +1207,7 @@ type queryCoordConfig struct {
}
func (p *queryCoordConfig) init(base *BaseTable) {
//---- Task ---
// ---- Task ---
p.RetryNum = ParamItem{
Key: "queryCoord.task.retrynum",
Version: "2.2.0",

4
pkg/util/paramtable/hook_config.go
View File

@ -1,7 +1,10 @@
package paramtable
import (
"go.uber.org/zap"
"github.com/milvus-io/milvus/pkg/config"
"github.com/milvus-io/milvus/pkg/log"
)
const hookYamlFile = "hook.yaml"
@ -15,6 +18,7 @@ type hookConfig struct {
func (h *hookConfig) init(base *BaseTable) {
h.hookBase = base
log.Info("hook config", zap.Any("hook", base.FileConfigs()))
h.SoPath = ParamItem{
Key: "soPath",

301
pkg/util/resource/resource_manager.go Normal file
View File

@ -0,0 +1,301 @@
package resource
import (
"sync"
"time"
)
const (
NoExpiration time.Duration = -1
DefaultCheckInterval = 2 * time.Second
DefaultExpiration = 4 * time.Second
)
type Resource interface {
Type() string
Name() string
Get() any
Close()
// KeepAliveTime returns the time duration of the resource keep alive if the resource isn't used.
KeepAliveTime() time.Duration
}
type wrapper struct {
res Resource
obj any
typ string
name string
closeFunc func()
keepAliveTime time.Duration
}
func (w *wrapper) Type() string {
if w.typ != "" {
return w.typ
}
if w.res == nil {
return ""
}
return w.res.Type()
}
func (w *wrapper) Name() string {
if w.name != "" {
return w.name
}
if w.res == nil {
return ""
}
return w.res.Name()
}
func (w *wrapper) Get() any {
if w.obj != nil {
return w.obj
}
if w.res == nil {
return nil
}
return w.res.Get()
}
func (w *wrapper) Close() {
if w.res != nil {
w.res.Close()
}
if w.closeFunc != nil {
w.closeFunc()
}
}
func (w *wrapper) KeepAliveTime() time.Duration {
if w.keepAliveTime != 0 {
return w.keepAliveTime
}
if w.res == nil {
return 0
}
return w.res.KeepAliveTime()
}
type Option func(res *wrapper)
func WithResource(res Resource) Option {
return func(w *wrapper) {
w.res = res
}
}
func WithType(typ string) Option {
return func(res *wrapper) {
res.typ = typ
}
}
func WithName(name string) Option {
return func(res *wrapper) {
res.name = name
}
}
func WithObj(obj any) Option {
return func(res *wrapper) {
res.obj = obj
}
}
func WithCloseFunc(closeFunc func()) Option {
return func(res *wrapper) {
res.closeFunc = closeFunc
}
}
func WithKeepAliveTime(keepAliveTime time.Duration) Option {
return func(res *wrapper) {
res.keepAliveTime = keepAliveTime
}
}
func NewResource(opts ...Option) Resource {
w := &wrapper{}
for _, opt := range opts {
opt(w)
}
return w
}
func NewSimpleResource(obj any, typ, name string, keepAliveTime time.Duration, closeFunc func()) Resource {
return NewResource(WithObj(obj), WithType(typ), WithName(name), WithKeepAliveTime(keepAliveTime), WithCloseFunc(closeFunc))
}
type Manager interface {
Get(typ, name string, newResourceFunc NewResourceFunc) (Resource, error)
Delete(typ, name string) Resource
Close()
}
type item struct {
res Resource
updateTimeChan chan int64
deleteMark chan struct{}
expiration int64
}
type manager struct {
resources map[string]map[string]*item // key: resource type, value: resource name -> resource
checkInterval time.Duration
defaultExpiration time.Duration
defaultTypeExpirations map[string]time.Duration // key: resource type, value: expiration
mu sync.RWMutex
wg sync.WaitGroup
stop chan struct{}
stopOnce sync.Once
}
func NewManager(checkInterval, defaultExpiration time.Duration, defaultTypeExpirations map[string]time.Duration) Manager {
if checkInterval <= 0 {
checkInterval = DefaultCheckInterval
}
if defaultExpiration <= 0 {
defaultExpiration = DefaultExpiration
}
if defaultTypeExpirations == nil {
defaultTypeExpirations = make(map[string]time.Duration)
}
m := &manager{
resources: make(map[string]map[string]*item),
checkInterval: checkInterval,
defaultExpiration: defaultExpiration,
defaultTypeExpirations: defaultTypeExpirations,
stop: make(chan struct{}),
}
m.wg.Add(1)
go m.backgroundGC()
return m
}
func (m *manager) backgroundGC() {
ticker := time.NewTicker(m.checkInterval)
defer m.wg.Done()
defer ticker.Stop()
for {
select {
case <-ticker.C:
m.gc()
case <-m.stop:
m.mu.Lock()
for _, typMap := range m.resources {
for _, item := range typMap {
item.res.Close()
}
}
m.resources = nil
m.mu.Unlock()
return
}
}
}
func (m *manager) gc() {
m.mu.Lock()
defer m.mu.Unlock()
now := time.Now().UnixNano()
for typ, typMap := range m.resources {
for resName, item := range typMap {
select {
case lastTime := <-item.updateTimeChan:
if item.expiration >= 0 {
item.expiration = lastTime
}
case <-item.deleteMark:
item.res.Close()
delete(typMap, resName)
default:
if item.expiration >= 0 && item.expiration <= now {
item.res.Close()
delete(typMap, resName)
}
}
}
if len(typMap) == 0 {
delete(m.resources, typ)
}
}
}
func (m *manager) updateExpire(item *item) {
select {
case item.updateTimeChan <- time.Now().UnixNano() + item.res.KeepAliveTime().Nanoseconds():
default:
}
}
type NewResourceFunc func() (Resource, error)
func (m *manager) Get(typ, name string, newResourceFunc NewResourceFunc) (Resource, error) {
m.mu.RLock()
typMap, ok := m.resources[typ]
if ok {
item := typMap[name]
if item != nil {
m.mu.RUnlock()
m.updateExpire(item)
return item.res, nil
}
}
m.mu.RUnlock()
m.mu.Lock()
defer m.mu.Unlock()
typMap, ok = m.resources[typ]
if !ok {
typMap = make(map[string]*item)
m.resources[typ] = typMap
}
ite, ok := typMap[name]
if !ok {
res, err := newResourceFunc()
if err != nil {
return nil, err
}
if res.KeepAliveTime() == 0 {
defaultExpiration := m.defaultTypeExpirations[typ]
if defaultExpiration == 0 {
defaultExpiration = m.defaultExpiration
}
res = NewResource(WithResource(res), WithKeepAliveTime(defaultExpiration))
}
ite = &item{
res: res,
updateTimeChan: make(chan int64, 1),
deleteMark: make(chan struct{}, 1),
}
typMap[name] = ite
}
m.updateExpire(ite)
return ite.res, nil
}
func (m *manager) Delete(typ, name string) Resource {
m.mu.Lock()
defer m.mu.Unlock()
typMap, ok := m.resources[typ]
if !ok {
return nil
}
ite, ok := typMap[name]
if !ok {
return nil
}
select {
case ite.deleteMark <- struct{}{}:
default:
}
return ite.res
}
func (m *manager) Close() {
m.stopOnce.Do(func() {
close(m.stop)
m.wg.Wait()
})
}

160
pkg/util/resource/resource_manager_test.go Normal file
View File

@ -0,0 +1,160 @@
package resource
import (
"testing"
"time"
"github.com/cockroachdb/errors"
"github.com/stretchr/testify/assert"
)
func TestResourceManager(t *testing.T) {
{
manager := NewManager(0, 0, nil)
manager.Close()
}
manager := NewManager(500*time.Millisecond, 2*time.Second, map[string]time.Duration{
"test": time.Second,
})
defer manager.Close()
{
assert.Nil(t, manager.Delete("test", "test"))
res, err := manager.Get("stream", "foo", func() (Resource, error) {
return NewSimpleResource("stream-foo", "stream", "foo", 0, nil), nil
})
assert.NoError(t, err)
assert.Equal(t, 2*time.Second, res.KeepAliveTime())
assert.Equal(t, "stream-foo", res.Get())
}
{
_, err := manager.Get("err", "foo", func() (Resource, error) {
return nil, errors.New("mock test error")
})
assert.Error(t, err)
}
{
res, err := manager.Get("test", "foo", func() (Resource, error) {
return NewSimpleResource("foo", "test", "foo", 0, nil), nil
})
assert.NoError(t, err)
assert.Equal(t, "foo", res.Get())
assert.Nil(t, manager.Delete("test", "test"))
}
{
time.Sleep(500 * time.Millisecond)
res, err := manager.Get("test", "foo", func() (Resource, error) {
return NewSimpleResource("foox", "test", "foo", 0, nil), nil
})
assert.NoError(t, err)
assert.Equal(t, "foo", res.Get())
}
{
time.Sleep(3 * time.Second)
res, err := manager.Get("test", "foo", func() (Resource, error) {
return NewSimpleResource("foo2", "test", "foo", 0, nil), nil
})
assert.NoError(t, err)
assert.Equal(t, "foo2", res.Get(), res.KeepAliveTime())
}
{
res := manager.Delete("test", "foo")
assert.Equal(t, "foo2", res.Get())
res = manager.Delete("test", "foo")
assert.Equal(t, "foo2", res.Get())
time.Sleep(time.Second)
res, err := manager.Get("test", "foo", func() (Resource, error) {
return NewSimpleResource("foo3", "test", "foo", 0, nil), nil
})
assert.NoError(t, err)
assert.Equal(t, "foo3", res.Get())
}
{
time.Sleep(2 * time.Second)
res, err := manager.Get("stream", "foo", func() (Resource, error) {
return NewSimpleResource("stream-foox", "stream", "foo", 0, nil), nil
})
assert.NoError(t, err)
assert.Equal(t, "stream-foox", res.Get())
}
{
var res Resource
var err error
res, err = manager.Get("ever", "foo", func() (Resource, error) {
return NewSimpleResource("ever-foo", "ever", "foo", NoExpiration, nil), nil
})
assert.NoError(t, err)
assert.Equal(t, "ever-foo", res.Get())
res, err = manager.Get("ever", "foo", func() (Resource, error) {
return NewSimpleResource("ever-foo2", "ever", "foo", NoExpiration, nil), nil
})
assert.NoError(t, err)
assert.Equal(t, "ever-foo", res.Get())
manager.Delete("ever", "foo")
time.Sleep(time.Second)
res, err = manager.Get("ever", "foo", func() (Resource, error) {
return NewSimpleResource("ever-foo3", "ever", "foo", NoExpiration, nil), nil
})
assert.NoError(t, err)
assert.Equal(t, "ever-foo3", res.Get())
}
}
func TestResource(t *testing.T) {
{
isClose := false
res := NewSimpleResource("obj", "test", "foo", 0, func() {
isClose = true
})
assert.Equal(t, "test", res.Type())
assert.Equal(t, "foo", res.Name())
assert.Equal(t, "obj", res.Get())
assert.EqualValues(t, 0, res.KeepAliveTime())
res.Close()
assert.True(t, isClose)
}
{
res := NewResource()
assert.Empty(t, res.Type())
assert.Empty(t, res.Name())
assert.Empty(t, res.Get())
assert.EqualValues(t, 0, res.KeepAliveTime())
}
{
isClose := false
res := NewSimpleResource("obj", "test", "foo", 0, func() {
isClose = true
})
isClose2 := false
wrapper := NewResource(WithResource(res), WithType("test2"), WithName("foo2"), WithObj("obj2"), WithKeepAliveTime(time.Second), WithCloseFunc(func() {
isClose2 = true
}))
wrapper.Close()
assert.Equal(t, "test2", wrapper.Type())
assert.Equal(t, "foo2", wrapper.Name())
assert.Equal(t, "obj2", wrapper.Get())
assert.Equal(t, time.Second, wrapper.KeepAliveTime())
assert.True(t, isClose)
assert.True(t, isClose2)
}
{
isClose := false
res := NewSimpleResource("obj", "test", "foo", 0, func() {
isClose = true
})
wrapper := NewResource(WithResource(res))
assert.Equal(t, "test", wrapper.Type())
assert.Equal(t, "foo", wrapper.Name())
assert.Equal(t, "obj", wrapper.Get())
assert.EqualValues(t, 0, wrapper.KeepAliveTime())
wrapper.Close()
assert.True(t, isClose)
}
}