From 69252f812debcd0e26b3636a037f6efabb6f423a Mon Sep 17 00:00:00 2001 From: congqixia Date: Wed, 20 Apr 2022 16:15:41 +0800 Subject: [PATCH] Implement memory replica in Proxy, QueryNode and QueryCoord (#16470) Related to #16298 #16291 #16154 Co-authored-by: sunby Co-authored-by: yangxuan Co-authored-by: yah01 Co-authored-by: Letian Jiang Signed-off-by: Congqi Xia --- internal/core/src/pb/common.pb.cc | 48 +- internal/core/src/pb/common.pb.h | 1 + internal/core/src/pb/milvus.pb.cc | 961 ++++--- internal/core/src/pb/milvus.pb.h | 85 + internal/datacoord/services.go | 13 +- .../querynode/client/client_test.go | 3 +- internal/proto/common.proto | 3 +- internal/proto/commonpb/common.pb.go | 218 +- internal/proto/data_coord.proto | 2 + internal/proto/datapb/data_coord.pb.go | 445 +-- internal/proto/milvus.proto | 6 +- internal/proto/milvuspb/milvus.pb.go | 553 ++-- internal/proto/query_coord.proto | 6 +- internal/proto/querypb/query_coord.pb.go | 385 +-- internal/proxy/impl.go | 63 +- internal/proxy/meta_cache.go | 43 + internal/proxy/meta_cache_test.go | 56 +- internal/proxy/proxy.go | 16 +- internal/proxy/proxy_test.go | 436 ++- internal/proxy/query_coord_mock_test.go | 38 + internal/proxy/query_node_mock_test.go | 114 + internal/proxy/result_receiver.go | 24 - internal/proxy/task.go | 16 +- internal/proxy/task_policies.go | 81 + internal/proxy/task_query.go | 689 ++--- internal/proxy/task_query_test.go | 211 +- internal/proxy/task_scheduler.go | 284 -- internal/proxy/task_search.go | 713 ++--- internal/proxy/task_search_test.go | 2402 ++++++++--------- internal/querycoord/channel_allocator.go | 20 +- internal/querycoord/channel_allocator_test.go | 13 +- internal/querycoord/channel_unsubscribe.go | 3 +- internal/querycoord/cluster.go | 89 +- internal/querycoord/cluster_test.go | 9 +- internal/querycoord/global_meta_broker.go | 17 +- internal/querycoord/group_balance.go | 57 + internal/querycoord/impl.go | 132 +- internal/querycoord/impl_test.go | 443 ++- internal/querycoord/index_checker_test.go | 41 +- internal/querycoord/meta.go | 195 +- internal/querycoord/meta_test.go | 27 +- .../querycoord/mock_3rd_component_test.go | 29 +- internal/querycoord/query_coord.go | 278 +- internal/querycoord/query_coord_test.go | 9 +- internal/querycoord/querynode_test.go | 15 +- internal/querycoord/replica.go | 51 +- internal/querycoord/segment_allocator.go | 17 +- internal/querycoord/segment_allocator_test.go | 20 +- internal/querycoord/segments_info.go | 12 +- internal/querycoord/task.go | 663 +++-- internal/querycoord/task_scheduler.go | 30 +- internal/querycoord/task_scheduler_test.go | 2 + internal/querycoord/task_test.go | 100 +- internal/querynode/historical.go | 23 + internal/querynode/impl.go | 114 +- internal/querynode/impl_test.go | 24 +- internal/querynode/mock_test.go | 32 +- internal/querynode/query_channel.go | 139 + internal/querynode/query_channel_test.go | 174 ++ internal/querynode/query_node.go | 103 +- internal/querynode/query_node_test.go | 72 +- internal/querynode/query_service_test.go | 5 +- internal/querynode/query_shard.go | 886 +++++- internal/querynode/query_shard_service.go | 103 +- .../querynode/query_shard_service_test.go | 8 +- internal/querynode/query_shard_test.go | 254 +- internal/querynode/shard_cluster.go | 60 +- internal/querynode/shard_cluster_service.go | 109 + .../querynode/shard_cluster_service_test.go | 34 + internal/querynode/shard_cluster_test.go | 3 +- internal/querynode/shard_node_detector.go | 4 + .../querynode/shard_node_detector_test.go | 34 +- internal/querynode/shard_segment_detector.go | 209 ++ .../querynode/shard_segment_detector_test.go | 298 ++ internal/querynode/streaming.go | 11 +- internal/querynode/task.go | 125 +- internal/querynode/task_test.go | 95 +- 77 files changed, 8487 insertions(+), 4619 deletions(-) create mode 100644 internal/proxy/query_node_mock_test.go delete mode 100644 internal/proxy/result_receiver.go create mode 100644 internal/proxy/task_policies.go create mode 100644 internal/querycoord/group_balance.go create mode 100644 internal/querynode/query_channel.go create mode 100644 internal/querynode/query_channel_test.go create mode 100644 internal/querynode/shard_cluster_service.go create mode 100644 internal/querynode/shard_cluster_service_test.go create mode 100644 internal/querynode/shard_segment_detector.go create mode 100644 internal/querynode/shard_segment_detector_test.go diff --git a/internal/core/src/pb/common.pb.cc b/internal/core/src/pb/common.pb.cc index 0322e8d2dc..5ac82c67a3 100644 --- a/internal/core/src/pb/common.pb.cc +++ b/internal/core/src/pb/common.pb.cc @@ -290,7 +290,7 @@ const char descriptor_table_protodef_common_2eproto[] PROTOBUF_SECTION_VARIABLE( "led\020\004*\202\001\n\014SegmentState\022\024\n\020SegmentStateNo" "ne\020\000\022\014\n\010NotExist\020\001\022\013\n\007Growing\020\002\022\n\n\006Seale" "d\020\003\022\013\n\007Flushed\020\004\022\014\n\010Flushing\020\005\022\013\n\007Droppe" - "d\020\006\022\r\n\tImporting\020\007*\261\n\n\007MsgType\022\r\n\tUndefi" + "d\020\006\022\r\n\tImporting\020\007*\307\n\n\007MsgType\022\r\n\tUndefi" "ned\020\000\022\024\n\020CreateCollection\020d\022\022\n\016DropColle" "ction\020e\022\021\n\rHasCollection\020f\022\026\n\022DescribeCo" "llection\020g\022\023\n\017ShowCollections\020h\022\024\n\020GetSy" @@ -314,27 +314,28 @@ const char descriptor_table_protodef_common_2eproto[] PROTOBUF_SECTION_VARIABLE( "atchDmChannels\020\374\003\022\025\n\020RemoveDmChannels\020\375\003" "\022\027\n\022WatchQueryChannels\020\376\003\022\030\n\023RemoveQuery" "Channels\020\377\003\022\035\n\030SealedSegmentsChangeInfo\020" - "\200\004\022\027\n\022WatchDeltaChannels\020\201\004\022\020\n\013SegmentIn" - "fo\020\330\004\022\017\n\nSystemInfo\020\331\004\022\024\n\017GetRecoveryInf" - "o\020\332\004\022\024\n\017GetSegmentState\020\333\004\022\r\n\010TimeTick\020\260" - "\t\022\023\n\016QueryNodeStats\020\261\t\022\016\n\tLoadIndex\020\262\t\022\016" - "\n\tRequestID\020\263\t\022\017\n\nRequestTSO\020\264\t\022\024\n\017Alloc" - "ateSegment\020\265\t\022\026\n\021SegmentStatistics\020\266\t\022\025\n" - "\020SegmentFlushDone\020\267\t\022\017\n\nDataNodeTt\020\270\t\022\025\n" - "\020CreateCredential\020\334\013\022\022\n\rGetCredential\020\335\013" - "\022\025\n\020DeleteCredential\020\336\013\022\025\n\020UpdateCredent" - "ial\020\337\013\022\026\n\021ListCredUsernames\020\340\013*\"\n\007DslTyp" - "e\022\007\n\003Dsl\020\000\022\016\n\nBoolExprV1\020\001*B\n\017Compaction" - "State\022\021\n\rUndefiedState\020\000\022\r\n\tExecuting\020\001\022" - "\r\n\tCompleted\020\002*X\n\020ConsistencyLevel\022\n\n\006St" - "rong\020\000\022\013\n\007Session\020\001\022\013\n\007Bounded\020\002\022\016\n\nEven" - "tually\020\003\022\016\n\nCustomized\020\004*\227\001\n\013ImportState" - "\022\021\n\rImportPending\020\000\022\020\n\014ImportFailed\020\001\022\021\n" - "\rImportStarted\020\002\022\024\n\020ImportDownloaded\020\003\022\020" - "\n\014ImportParsed\020\004\022\023\n\017ImportPersisted\020\005\022\023\n" - "\017ImportCompleted\020\006BW\n\016io.milvus.grpcB\013Co" - "mmonProtoP\001Z3github.com/milvus-io/milvus" - "/internal/proto/commonpb\240\001\001b\006proto3" + "\200\004\022\027\n\022WatchDeltaChannels\020\201\004\022\024\n\017GetShardL" + "eaders\020\202\004\022\020\n\013SegmentInfo\020\330\004\022\017\n\nSystemInf" + "o\020\331\004\022\024\n\017GetRecoveryInfo\020\332\004\022\024\n\017GetSegment" + "State\020\333\004\022\r\n\010TimeTick\020\260\t\022\023\n\016QueryNodeStat" + "s\020\261\t\022\016\n\tLoadIndex\020\262\t\022\016\n\tRequestID\020\263\t\022\017\n\n" + "RequestTSO\020\264\t\022\024\n\017AllocateSegment\020\265\t\022\026\n\021S" + "egmentStatistics\020\266\t\022\025\n\020SegmentFlushDone\020" + "\267\t\022\017\n\nDataNodeTt\020\270\t\022\025\n\020CreateCredential\020" + "\334\013\022\022\n\rGetCredential\020\335\013\022\025\n\020DeleteCredenti" + "al\020\336\013\022\025\n\020UpdateCredential\020\337\013\022\026\n\021ListCred" + "Usernames\020\340\013*\"\n\007DslType\022\007\n\003Dsl\020\000\022\016\n\nBool" + "ExprV1\020\001*B\n\017CompactionState\022\021\n\rUndefiedS" + "tate\020\000\022\r\n\tExecuting\020\001\022\r\n\tCompleted\020\002*X\n\020" + "ConsistencyLevel\022\n\n\006Strong\020\000\022\013\n\007Session\020" + "\001\022\013\n\007Bounded\020\002\022\016\n\nEventually\020\003\022\016\n\nCustom" + "ized\020\004*\227\001\n\013ImportState\022\021\n\rImportPending\020" + "\000\022\020\n\014ImportFailed\020\001\022\021\n\rImportStarted\020\002\022\024" + "\n\020ImportDownloaded\020\003\022\020\n\014ImportParsed\020\004\022\023" + "\n\017ImportPersisted\020\005\022\023\n\017ImportCompleted\020\006" + "BW\n\016io.milvus.grpcB\013CommonProtoP\001Z3githu" + "b.com/milvus-io/milvus/internal/proto/co" + "mmonpb\240\001\001b\006proto3" ; static const ::PROTOBUF_NAMESPACE_ID::internal::DescriptorTable*const descriptor_table_common_2eproto_deps[1] = { }; @@ -351,7 +352,7 @@ static ::PROTOBUF_NAMESPACE_ID::internal::SCCInfoBase*const descriptor_table_com static ::PROTOBUF_NAMESPACE_ID::internal::once_flag descriptor_table_common_2eproto_once; static bool descriptor_table_common_2eproto_initialized = false; const ::PROTOBUF_NAMESPACE_ID::internal::DescriptorTable descriptor_table_common_2eproto = { - &descriptor_table_common_2eproto_initialized, descriptor_table_protodef_common_2eproto, "common.proto", 3275, + &descriptor_table_common_2eproto_initialized, descriptor_table_protodef_common_2eproto, "common.proto", 3297, &descriptor_table_common_2eproto_once, descriptor_table_common_2eproto_sccs, descriptor_table_common_2eproto_deps, 8, 0, schemas, file_default_instances, TableStruct_common_2eproto::offsets, file_level_metadata_common_2eproto, 8, file_level_enum_descriptors_common_2eproto, file_level_service_descriptors_common_2eproto, @@ -497,6 +498,7 @@ bool MsgType_IsValid(int value) { case 511: case 512: case 513: + case 514: case 600: case 601: case 602: diff --git a/internal/core/src/pb/common.pb.h b/internal/core/src/pb/common.pb.h index 69117672a0..6efa0c73c2 100644 --- a/internal/core/src/pb/common.pb.h +++ b/internal/core/src/pb/common.pb.h @@ -262,6 +262,7 @@ enum MsgType : int { RemoveQueryChannels = 511, SealedSegmentsChangeInfo = 512, WatchDeltaChannels = 513, + GetShardLeaders = 514, SegmentInfo = 600, SystemInfo = 601, GetRecoveryInfo = 602, diff --git a/internal/core/src/pb/milvus.pb.cc b/internal/core/src/pb/milvus.pb.cc index f73e7f651f..9b3c6d0bdc 100644 --- a/internal/core/src/pb/milvus.pb.cc +++ b/internal/core/src/pb/milvus.pb.cc @@ -1955,6 +1955,7 @@ const ::PROTOBUF_NAMESPACE_ID::uint32 TableStruct_milvus_2eproto::offsets[] PROT PROTOBUF_FIELD_OFFSET(::milvus::proto::milvus::LoadPartitionsRequest, db_name_), PROTOBUF_FIELD_OFFSET(::milvus::proto::milvus::LoadPartitionsRequest, collection_name_), PROTOBUF_FIELD_OFFSET(::milvus::proto::milvus::LoadPartitionsRequest, partition_names_), + PROTOBUF_FIELD_OFFSET(::milvus::proto::milvus::LoadPartitionsRequest, replica_number_), ~0u, // no _has_bits_ PROTOBUF_FIELD_OFFSET(::milvus::proto::milvus::ReleasePartitionsRequest, _internal_metadata_), ~0u, // no _extensions_ @@ -2384,6 +2385,7 @@ const ::PROTOBUF_NAMESPACE_ID::uint32 TableStruct_milvus_2eproto::offsets[] PROT PROTOBUF_FIELD_OFFSET(::milvus::proto::milvus::LoadBalanceRequest, src_nodeid_), PROTOBUF_FIELD_OFFSET(::milvus::proto::milvus::LoadBalanceRequest, dst_nodeids_), PROTOBUF_FIELD_OFFSET(::milvus::proto::milvus::LoadBalanceRequest, sealed_segmentids_), + PROTOBUF_FIELD_OFFSET(::milvus::proto::milvus::LoadBalanceRequest, collectionname_), ~0u, // no _has_bits_ PROTOBUF_FIELD_OFFSET(::milvus::proto::milvus::ManualCompactionRequest, _internal_metadata_), ~0u, // no _extensions_ @@ -2579,76 +2581,76 @@ static const ::PROTOBUF_NAMESPACE_ID::internal::MigrationSchema schemas[] PROTOB { 157, -1, sizeof(::milvus::proto::milvus::DropPartitionRequest)}, { 166, -1, sizeof(::milvus::proto::milvus::HasPartitionRequest)}, { 175, -1, sizeof(::milvus::proto::milvus::LoadPartitionsRequest)}, - { 184, -1, sizeof(::milvus::proto::milvus::ReleasePartitionsRequest)}, - { 193, -1, sizeof(::milvus::proto::milvus::GetPartitionStatisticsRequest)}, - { 202, -1, sizeof(::milvus::proto::milvus::GetPartitionStatisticsResponse)}, - { 209, -1, sizeof(::milvus::proto::milvus::ShowPartitionsRequest)}, - { 220, -1, sizeof(::milvus::proto::milvus::ShowPartitionsResponse)}, - { 231, -1, sizeof(::milvus::proto::milvus::DescribeSegmentRequest)}, - { 239, -1, sizeof(::milvus::proto::milvus::DescribeSegmentResponse)}, - { 249, -1, sizeof(::milvus::proto::milvus::ShowSegmentsRequest)}, - { 257, -1, sizeof(::milvus::proto::milvus::ShowSegmentsResponse)}, - { 264, -1, sizeof(::milvus::proto::milvus::CreateIndexRequest)}, - { 275, -1, sizeof(::milvus::proto::milvus::DescribeIndexRequest)}, - { 285, -1, sizeof(::milvus::proto::milvus::IndexDescription)}, - { 294, -1, sizeof(::milvus::proto::milvus::DescribeIndexResponse)}, - { 301, -1, sizeof(::milvus::proto::milvus::GetIndexBuildProgressRequest)}, - { 311, -1, sizeof(::milvus::proto::milvus::GetIndexBuildProgressResponse)}, - { 319, -1, sizeof(::milvus::proto::milvus::GetIndexStateRequest)}, - { 329, -1, sizeof(::milvus::proto::milvus::GetIndexStateResponse)}, - { 337, -1, sizeof(::milvus::proto::milvus::DropIndexRequest)}, - { 347, -1, sizeof(::milvus::proto::milvus::InsertRequest)}, - { 359, -1, sizeof(::milvus::proto::milvus::MutationResult)}, - { 373, -1, sizeof(::milvus::proto::milvus::DeleteRequest)}, - { 384, -1, sizeof(::milvus::proto::milvus::PlaceholderValue)}, - { 392, -1, sizeof(::milvus::proto::milvus::PlaceholderGroup)}, - { 398, -1, sizeof(::milvus::proto::milvus::SearchRequest)}, - { 414, -1, sizeof(::milvus::proto::milvus::Hits)}, - { 422, -1, sizeof(::milvus::proto::milvus::SearchResults)}, - { 430, -1, sizeof(::milvus::proto::milvus::FlushRequest)}, - { 438, 445, sizeof(::milvus::proto::milvus::FlushResponse_CollSegIDsEntry_DoNotUse)}, - { 447, -1, sizeof(::milvus::proto::milvus::FlushResponse)}, - { 455, -1, sizeof(::milvus::proto::milvus::QueryRequest)}, - { 468, -1, sizeof(::milvus::proto::milvus::QueryResults)}, - { 476, -1, sizeof(::milvus::proto::milvus::VectorIDs)}, - { 485, -1, sizeof(::milvus::proto::milvus::VectorsArray)}, - { 493, -1, sizeof(::milvus::proto::milvus::CalcDistanceRequest)}, - { 502, -1, sizeof(::milvus::proto::milvus::CalcDistanceResults)}, - { 511, -1, sizeof(::milvus::proto::milvus::PersistentSegmentInfo)}, - { 521, -1, sizeof(::milvus::proto::milvus::GetPersistentSegmentInfoRequest)}, - { 529, -1, sizeof(::milvus::proto::milvus::GetPersistentSegmentInfoResponse)}, - { 536, -1, sizeof(::milvus::proto::milvus::QuerySegmentInfo)}, - { 550, -1, sizeof(::milvus::proto::milvus::GetQuerySegmentInfoRequest)}, - { 558, -1, sizeof(::milvus::proto::milvus::GetQuerySegmentInfoResponse)}, - { 565, -1, sizeof(::milvus::proto::milvus::DummyRequest)}, - { 571, -1, sizeof(::milvus::proto::milvus::DummyResponse)}, - { 577, -1, sizeof(::milvus::proto::milvus::RegisterLinkRequest)}, - { 582, -1, sizeof(::milvus::proto::milvus::RegisterLinkResponse)}, - { 589, -1, sizeof(::milvus::proto::milvus::GetMetricsRequest)}, - { 596, -1, sizeof(::milvus::proto::milvus::GetMetricsResponse)}, - { 604, -1, sizeof(::milvus::proto::milvus::LoadBalanceRequest)}, - { 613, -1, sizeof(::milvus::proto::milvus::ManualCompactionRequest)}, - { 620, -1, sizeof(::milvus::proto::milvus::ManualCompactionResponse)}, - { 627, -1, sizeof(::milvus::proto::milvus::GetCompactionStateRequest)}, - { 633, -1, sizeof(::milvus::proto::milvus::GetCompactionStateResponse)}, - { 643, -1, sizeof(::milvus::proto::milvus::GetCompactionPlansRequest)}, - { 649, -1, sizeof(::milvus::proto::milvus::GetCompactionPlansResponse)}, - { 657, -1, sizeof(::milvus::proto::milvus::CompactionMergeInfo)}, - { 664, -1, sizeof(::milvus::proto::milvus::GetFlushStateRequest)}, - { 670, -1, sizeof(::milvus::proto::milvus::GetFlushStateResponse)}, - { 677, -1, sizeof(::milvus::proto::milvus::ImportRequest)}, - { 688, -1, sizeof(::milvus::proto::milvus::ImportResponse)}, - { 695, -1, sizeof(::milvus::proto::milvus::GetImportStateRequest)}, - { 701, -1, sizeof(::milvus::proto::milvus::GetImportStateResponse)}, - { 711, -1, sizeof(::milvus::proto::milvus::GetReplicasRequest)}, - { 719, -1, sizeof(::milvus::proto::milvus::GetReplicasResponse)}, - { 726, -1, sizeof(::milvus::proto::milvus::ReplicaInfo)}, - { 736, -1, sizeof(::milvus::proto::milvus::ShardReplica)}, - { 745, -1, sizeof(::milvus::proto::milvus::CreateCredentialRequest)}, - { 755, -1, sizeof(::milvus::proto::milvus::UpdateCredentialRequest)}, - { 766, -1, sizeof(::milvus::proto::milvus::DeleteCredentialRequest)}, - { 773, -1, sizeof(::milvus::proto::milvus::ListCredUsersResponse)}, - { 780, -1, sizeof(::milvus::proto::milvus::ListCredUsersRequest)}, + { 185, -1, sizeof(::milvus::proto::milvus::ReleasePartitionsRequest)}, + { 194, -1, sizeof(::milvus::proto::milvus::GetPartitionStatisticsRequest)}, + { 203, -1, sizeof(::milvus::proto::milvus::GetPartitionStatisticsResponse)}, + { 210, -1, sizeof(::milvus::proto::milvus::ShowPartitionsRequest)}, + { 221, -1, sizeof(::milvus::proto::milvus::ShowPartitionsResponse)}, + { 232, -1, sizeof(::milvus::proto::milvus::DescribeSegmentRequest)}, + { 240, -1, sizeof(::milvus::proto::milvus::DescribeSegmentResponse)}, + { 250, -1, sizeof(::milvus::proto::milvus::ShowSegmentsRequest)}, + { 258, -1, sizeof(::milvus::proto::milvus::ShowSegmentsResponse)}, + { 265, -1, sizeof(::milvus::proto::milvus::CreateIndexRequest)}, + { 276, -1, sizeof(::milvus::proto::milvus::DescribeIndexRequest)}, + { 286, -1, sizeof(::milvus::proto::milvus::IndexDescription)}, + { 295, -1, sizeof(::milvus::proto::milvus::DescribeIndexResponse)}, + { 302, -1, sizeof(::milvus::proto::milvus::GetIndexBuildProgressRequest)}, + { 312, -1, sizeof(::milvus::proto::milvus::GetIndexBuildProgressResponse)}, + { 320, -1, sizeof(::milvus::proto::milvus::GetIndexStateRequest)}, + { 330, -1, sizeof(::milvus::proto::milvus::GetIndexStateResponse)}, + { 338, -1, sizeof(::milvus::proto::milvus::DropIndexRequest)}, + { 348, -1, sizeof(::milvus::proto::milvus::InsertRequest)}, + { 360, -1, sizeof(::milvus::proto::milvus::MutationResult)}, + { 374, -1, sizeof(::milvus::proto::milvus::DeleteRequest)}, + { 385, -1, sizeof(::milvus::proto::milvus::PlaceholderValue)}, + { 393, -1, sizeof(::milvus::proto::milvus::PlaceholderGroup)}, + { 399, -1, sizeof(::milvus::proto::milvus::SearchRequest)}, + { 415, -1, sizeof(::milvus::proto::milvus::Hits)}, + { 423, -1, sizeof(::milvus::proto::milvus::SearchResults)}, + { 431, -1, sizeof(::milvus::proto::milvus::FlushRequest)}, + { 439, 446, sizeof(::milvus::proto::milvus::FlushResponse_CollSegIDsEntry_DoNotUse)}, + { 448, -1, sizeof(::milvus::proto::milvus::FlushResponse)}, + { 456, -1, sizeof(::milvus::proto::milvus::QueryRequest)}, + { 469, -1, sizeof(::milvus::proto::milvus::QueryResults)}, + { 477, -1, sizeof(::milvus::proto::milvus::VectorIDs)}, + { 486, -1, sizeof(::milvus::proto::milvus::VectorsArray)}, + { 494, -1, sizeof(::milvus::proto::milvus::CalcDistanceRequest)}, + { 503, -1, sizeof(::milvus::proto::milvus::CalcDistanceResults)}, + { 512, -1, sizeof(::milvus::proto::milvus::PersistentSegmentInfo)}, + { 522, -1, sizeof(::milvus::proto::milvus::GetPersistentSegmentInfoRequest)}, + { 530, -1, sizeof(::milvus::proto::milvus::GetPersistentSegmentInfoResponse)}, + { 537, -1, sizeof(::milvus::proto::milvus::QuerySegmentInfo)}, + { 551, -1, sizeof(::milvus::proto::milvus::GetQuerySegmentInfoRequest)}, + { 559, -1, sizeof(::milvus::proto::milvus::GetQuerySegmentInfoResponse)}, + { 566, -1, sizeof(::milvus::proto::milvus::DummyRequest)}, + { 572, -1, sizeof(::milvus::proto::milvus::DummyResponse)}, + { 578, -1, sizeof(::milvus::proto::milvus::RegisterLinkRequest)}, + { 583, -1, sizeof(::milvus::proto::milvus::RegisterLinkResponse)}, + { 590, -1, sizeof(::milvus::proto::milvus::GetMetricsRequest)}, + { 597, -1, sizeof(::milvus::proto::milvus::GetMetricsResponse)}, + { 605, -1, sizeof(::milvus::proto::milvus::LoadBalanceRequest)}, + { 615, -1, sizeof(::milvus::proto::milvus::ManualCompactionRequest)}, + { 622, -1, sizeof(::milvus::proto::milvus::ManualCompactionResponse)}, + { 629, -1, sizeof(::milvus::proto::milvus::GetCompactionStateRequest)}, + { 635, -1, sizeof(::milvus::proto::milvus::GetCompactionStateResponse)}, + { 645, -1, sizeof(::milvus::proto::milvus::GetCompactionPlansRequest)}, + { 651, -1, sizeof(::milvus::proto::milvus::GetCompactionPlansResponse)}, + { 659, -1, sizeof(::milvus::proto::milvus::CompactionMergeInfo)}, + { 666, -1, sizeof(::milvus::proto::milvus::GetFlushStateRequest)}, + { 672, -1, sizeof(::milvus::proto::milvus::GetFlushStateResponse)}, + { 679, -1, sizeof(::milvus::proto::milvus::ImportRequest)}, + { 690, -1, sizeof(::milvus::proto::milvus::ImportResponse)}, + { 697, -1, sizeof(::milvus::proto::milvus::GetImportStateRequest)}, + { 703, -1, sizeof(::milvus::proto::milvus::GetImportStateResponse)}, + { 713, -1, sizeof(::milvus::proto::milvus::GetReplicasRequest)}, + { 721, -1, sizeof(::milvus::proto::milvus::GetReplicasResponse)}, + { 728, -1, sizeof(::milvus::proto::milvus::ReplicaInfo)}, + { 738, -1, sizeof(::milvus::proto::milvus::ShardReplica)}, + { 747, -1, sizeof(::milvus::proto::milvus::CreateCredentialRequest)}, + { 757, -1, sizeof(::milvus::proto::milvus::UpdateCredentialRequest)}, + { 768, -1, sizeof(::milvus::proto::milvus::DeleteCredentialRequest)}, + { 775, -1, sizeof(::milvus::proto::milvus::ListCredUsersResponse)}, + { 782, -1, sizeof(::milvus::proto::milvus::ListCredUsersRequest)}, }; static ::PROTOBUF_NAMESPACE_ID::Message const * const file_default_instances[] = { @@ -2818,365 +2820,366 @@ const char descriptor_table_protodef_milvus_2eproto[] PROTOBUF_SECTION_VARIABLE( "\"\203\001\n\023HasPartitionRequest\022*\n\004base\030\001 \001(\0132\034" ".milvus.proto.common.MsgBase\022\017\n\007db_name\030" "\002 \001(\t\022\027\n\017collection_name\030\003 \001(\t\022\026\n\016partit" - "ion_name\030\004 \001(\t\"\206\001\n\025LoadPartitionsRequest" + "ion_name\030\004 \001(\t\"\236\001\n\025LoadPartitionsRequest" "\022*\n\004base\030\001 \001(\0132\034.milvus.proto.common.Msg" "Base\022\017\n\007db_name\030\002 \001(\t\022\027\n\017collection_name" - "\030\003 \001(\t\022\027\n\017partition_names\030\004 \003(\t\"\211\001\n\030Rele" - "asePartitionsRequest\022*\n\004base\030\001 \001(\0132\034.mil" - "vus.proto.common.MsgBase\022\017\n\007db_name\030\002 \001(" - "\t\022\027\n\017collection_name\030\003 \001(\t\022\027\n\017partition_" - "names\030\004 \003(\t\"\215\001\n\035GetPartitionStatisticsRe" - "quest\022*\n\004base\030\001 \001(\0132\034.milvus.proto.commo" - "n.MsgBase\022\017\n\007db_name\030\002 \001(\t\022\027\n\017collection" - "_name\030\003 \001(\t\022\026\n\016partition_name\030\004 \001(\t\"\177\n\036G" - "etPartitionStatisticsResponse\022+\n\006status\030" - "\001 \001(\0132\033.milvus.proto.common.Status\0220\n\005st" - "ats\030\002 \003(\0132!.milvus.proto.common.KeyValue" - "Pair\"\311\001\n\025ShowPartitionsRequest\022*\n\004base\030\001" - " \001(\0132\034.milvus.proto.common.MsgBase\022\017\n\007db" - "_name\030\002 \001(\t\022\027\n\017collection_name\030\003 \001(\t\022\024\n\014" - "collectionID\030\004 \001(\003\022\027\n\017partition_names\030\005 " - "\003(\t\022+\n\004type\030\006 \001(\0162\035.milvus.proto.milvus." - "ShowType\"\316\001\n\026ShowPartitionsResponse\022+\n\006s" - "tatus\030\001 \001(\0132\033.milvus.proto.common.Status" - "\022\027\n\017partition_names\030\002 \003(\t\022\024\n\014partitionID" - "s\030\003 \003(\003\022\032\n\022created_timestamps\030\004 \003(\004\022\036\n\026c" - "reated_utc_timestamps\030\005 \003(\004\022\034\n\024inMemory_" - "percentages\030\006 \003(\003\"m\n\026DescribeSegmentRequ" - "est\022*\n\004base\030\001 \001(\0132\034.milvus.proto.common." - "MsgBase\022\024\n\014collectionID\030\002 \001(\003\022\021\n\tsegment" - "ID\030\003 \001(\003\"\217\001\n\027DescribeSegmentResponse\022+\n\006" - "status\030\001 \001(\0132\033.milvus.proto.common.Statu" - "s\022\017\n\007indexID\030\002 \001(\003\022\017\n\007buildID\030\003 \001(\003\022\024\n\014e" - "nable_index\030\004 \001(\010\022\017\n\007fieldID\030\005 \001(\003\"l\n\023Sh" - "owSegmentsRequest\022*\n\004base\030\001 \001(\0132\034.milvus" - ".proto.common.MsgBase\022\024\n\014collectionID\030\002 " - "\001(\003\022\023\n\013partitionID\030\003 \001(\003\"W\n\024ShowSegments" + "\030\003 \001(\t\022\027\n\017partition_names\030\004 \003(\t\022\026\n\016repli" + "ca_number\030\005 \001(\005\"\211\001\n\030ReleasePartitionsReq" + "uest\022*\n\004base\030\001 \001(\0132\034.milvus.proto.common" + ".MsgBase\022\017\n\007db_name\030\002 \001(\t\022\027\n\017collection_" + "name\030\003 \001(\t\022\027\n\017partition_names\030\004 \003(\t\"\215\001\n\035" + "GetPartitionStatisticsRequest\022*\n\004base\030\001 " + "\001(\0132\034.milvus.proto.common.MsgBase\022\017\n\007db_" + "name\030\002 \001(\t\022\027\n\017collection_name\030\003 \001(\t\022\026\n\016p" + "artition_name\030\004 \001(\t\"\177\n\036GetPartitionStati" + "sticsResponse\022+\n\006status\030\001 \001(\0132\033.milvus.p" + "roto.common.Status\0220\n\005stats\030\002 \003(\0132!.milv" + "us.proto.common.KeyValuePair\"\311\001\n\025ShowPar" + "titionsRequest\022*\n\004base\030\001 \001(\0132\034.milvus.pr" + "oto.common.MsgBase\022\017\n\007db_name\030\002 \001(\t\022\027\n\017c" + "ollection_name\030\003 \001(\t\022\024\n\014collectionID\030\004 \001" + "(\003\022\027\n\017partition_names\030\005 \003(\t\022+\n\004type\030\006 \001(" + "\0162\035.milvus.proto.milvus.ShowType\"\316\001\n\026Sho" + "wPartitionsResponse\022+\n\006status\030\001 \001(\0132\033.mi" + "lvus.proto.common.Status\022\027\n\017partition_na" + "mes\030\002 \003(\t\022\024\n\014partitionIDs\030\003 \003(\003\022\032\n\022creat" + "ed_timestamps\030\004 \003(\004\022\036\n\026created_utc_times" + "tamps\030\005 \003(\004\022\034\n\024inMemory_percentages\030\006 \003(" + "\003\"m\n\026DescribeSegmentRequest\022*\n\004base\030\001 \001(" + "\0132\034.milvus.proto.common.MsgBase\022\024\n\014colle" + "ctionID\030\002 \001(\003\022\021\n\tsegmentID\030\003 \001(\003\"\217\001\n\027Des" + "cribeSegmentResponse\022+\n\006status\030\001 \001(\0132\033.m" + "ilvus.proto.common.Status\022\017\n\007indexID\030\002 \001" + "(\003\022\017\n\007buildID\030\003 \001(\003\022\024\n\014enable_index\030\004 \001(" + "\010\022\017\n\007fieldID\030\005 \001(\003\"l\n\023ShowSegmentsReques" + "t\022*\n\004base\030\001 \001(\0132\034.milvus.proto.common.Ms" + "gBase\022\024\n\014collectionID\030\002 \001(\003\022\023\n\013partition" + "ID\030\003 \001(\003\"W\n\024ShowSegmentsResponse\022+\n\006stat" + "us\030\001 \001(\0132\033.milvus.proto.common.Status\022\022\n" + "\nsegmentIDs\030\002 \003(\003\"\313\001\n\022CreateIndexRequest" + "\022*\n\004base\030\001 \001(\0132\034.milvus.proto.common.Msg" + "Base\022\017\n\007db_name\030\002 \001(\t\022\027\n\017collection_name" + "\030\003 \001(\t\022\022\n\nfield_name\030\004 \001(\t\0227\n\014extra_para" + "ms\030\005 \003(\0132!.milvus.proto.common.KeyValueP" + "air\022\022\n\nindex_name\030\006 \001(\t\"\224\001\n\024DescribeInde" + "xRequest\022*\n\004base\030\001 \001(\0132\034.milvus.proto.co" + "mmon.MsgBase\022\017\n\007db_name\030\002 \001(\t\022\027\n\017collect" + "ion_name\030\003 \001(\t\022\022\n\nfield_name\030\004 \001(\t\022\022\n\nin" + "dex_name\030\005 \001(\t\"~\n\020IndexDescription\022\022\n\nin" + "dex_name\030\001 \001(\t\022\017\n\007indexID\030\002 \001(\003\0221\n\006param" + "s\030\003 \003(\0132!.milvus.proto.common.KeyValuePa" + "ir\022\022\n\nfield_name\030\004 \001(\t\"\207\001\n\025DescribeIndex" "Response\022+\n\006status\030\001 \001(\0132\033.milvus.proto." - "common.Status\022\022\n\nsegmentIDs\030\002 \003(\003\"\313\001\n\022Cr" - "eateIndexRequest\022*\n\004base\030\001 \001(\0132\034.milvus." - "proto.common.MsgBase\022\017\n\007db_name\030\002 \001(\t\022\027\n" - "\017collection_name\030\003 \001(\t\022\022\n\nfield_name\030\004 \001" - "(\t\0227\n\014extra_params\030\005 \003(\0132!.milvus.proto." - "common.KeyValuePair\022\022\n\nindex_name\030\006 \001(\t\"" - "\224\001\n\024DescribeIndexRequest\022*\n\004base\030\001 \001(\0132\034" + "common.Status\022A\n\022index_descriptions\030\002 \003(" + "\0132%.milvus.proto.milvus.IndexDescription" + "\"\234\001\n\034GetIndexBuildProgressRequest\022*\n\004bas" + "e\030\001 \001(\0132\034.milvus.proto.common.MsgBase\022\017\n" + "\007db_name\030\002 \001(\t\022\027\n\017collection_name\030\003 \001(\t\022" + "\022\n\nfield_name\030\004 \001(\t\022\022\n\nindex_name\030\005 \001(\t\"" + "v\n\035GetIndexBuildProgressResponse\022+\n\006stat" + "us\030\001 \001(\0132\033.milvus.proto.common.Status\022\024\n" + "\014indexed_rows\030\002 \001(\003\022\022\n\ntotal_rows\030\003 \001(\003\"" + "\224\001\n\024GetIndexStateRequest\022*\n\004base\030\001 \001(\0132\034" ".milvus.proto.common.MsgBase\022\017\n\007db_name\030" "\002 \001(\t\022\027\n\017collection_name\030\003 \001(\t\022\022\n\nfield_" - "name\030\004 \001(\t\022\022\n\nindex_name\030\005 \001(\t\"~\n\020IndexD" - "escription\022\022\n\nindex_name\030\001 \001(\t\022\017\n\007indexI" - "D\030\002 \001(\003\0221\n\006params\030\003 \003(\0132!.milvus.proto.c" - "ommon.KeyValuePair\022\022\n\nfield_name\030\004 \001(\t\"\207" - "\001\n\025DescribeIndexResponse\022+\n\006status\030\001 \001(\013" - "2\033.milvus.proto.common.Status\022A\n\022index_d" - "escriptions\030\002 \003(\0132%.milvus.proto.milvus." - "IndexDescription\"\234\001\n\034GetIndexBuildProgre" - "ssRequest\022*\n\004base\030\001 \001(\0132\034.milvus.proto.c" - "ommon.MsgBase\022\017\n\007db_name\030\002 \001(\t\022\027\n\017collec" - "tion_name\030\003 \001(\t\022\022\n\nfield_name\030\004 \001(\t\022\022\n\ni" - "ndex_name\030\005 \001(\t\"v\n\035GetIndexBuildProgress" - "Response\022+\n\006status\030\001 \001(\0132\033.milvus.proto." - "common.Status\022\024\n\014indexed_rows\030\002 \001(\003\022\022\n\nt" - "otal_rows\030\003 \001(\003\"\224\001\n\024GetIndexStateRequest" - "\022*\n\004base\030\001 \001(\0132\034.milvus.proto.common.Msg" - "Base\022\017\n\007db_name\030\002 \001(\t\022\027\n\017collection_name" - "\030\003 \001(\t\022\022\n\nfield_name\030\004 \001(\t\022\022\n\nindex_name" - "\030\005 \001(\t\"\211\001\n\025GetIndexStateResponse\022+\n\006stat" - "us\030\001 \001(\0132\033.milvus.proto.common.Status\022.\n" - "\005state\030\002 \001(\0162\037.milvus.proto.common.Index" - "State\022\023\n\013fail_reason\030\003 \001(\t\"\220\001\n\020DropIndex" - "Request\022*\n\004base\030\001 \001(\0132\034.milvus.proto.com" - "mon.MsgBase\022\017\n\007db_name\030\002 \001(\t\022\027\n\017collecti" - "on_name\030\003 \001(\t\022\022\n\nfield_name\030\004 \001(\t\022\022\n\nind" - "ex_name\030\005 \001(\t\"\327\001\n\rInsertRequest\022*\n\004base\030" + "name\030\004 \001(\t\022\022\n\nindex_name\030\005 \001(\t\"\211\001\n\025GetIn" + "dexStateResponse\022+\n\006status\030\001 \001(\0132\033.milvu" + "s.proto.common.Status\022.\n\005state\030\002 \001(\0162\037.m" + "ilvus.proto.common.IndexState\022\023\n\013fail_re" + "ason\030\003 \001(\t\"\220\001\n\020DropIndexRequest\022*\n\004base\030" "\001 \001(\0132\034.milvus.proto.common.MsgBase\022\017\n\007d" - "b_name\030\002 \001(\t\022\027\n\017collection_name\030\003 \001(\t\022\026\n" - "\016partition_name\030\004 \001(\t\0223\n\013fields_data\030\005 \003" - "(\0132\036.milvus.proto.schema.FieldData\022\021\n\tha" - "sh_keys\030\006 \003(\r\022\020\n\010num_rows\030\007 \001(\r\"\360\001\n\016Muta" - "tionResult\022+\n\006status\030\001 \001(\0132\033.milvus.prot" - "o.common.Status\022%\n\003IDs\030\002 \001(\0132\030.milvus.pr" - "oto.schema.IDs\022\022\n\nsucc_index\030\003 \003(\r\022\021\n\ter" - "r_index\030\004 \003(\r\022\024\n\014acknowledged\030\005 \001(\010\022\022\n\ni" - "nsert_cnt\030\006 \001(\003\022\022\n\ndelete_cnt\030\007 \001(\003\022\022\n\nu" - "psert_cnt\030\010 \001(\003\022\021\n\ttimestamp\030\t \001(\004\"\236\001\n\rD" - "eleteRequest\022*\n\004base\030\001 \001(\0132\034.milvus.prot" - "o.common.MsgBase\022\017\n\007db_name\030\002 \001(\t\022\027\n\017col" - "lection_name\030\003 \001(\t\022\026\n\016partition_name\030\004 \001" - "(\t\022\014\n\004expr\030\005 \001(\t\022\021\n\thash_keys\030\006 \003(\r\"c\n\020P" - "laceholderValue\022\013\n\003tag\030\001 \001(\t\0222\n\004type\030\002 \001" - "(\0162$.milvus.proto.milvus.PlaceholderType" - "\022\016\n\006values\030\003 \003(\014\"O\n\020PlaceholderGroup\022;\n\014" - "placeholders\030\001 \003(\0132%.milvus.proto.milvus" - ".PlaceholderValue\"\336\002\n\rSearchRequest\022*\n\004b" - "ase\030\001 \001(\0132\034.milvus.proto.common.MsgBase\022" - "\017\n\007db_name\030\002 \001(\t\022\027\n\017collection_name\030\003 \001(" - "\t\022\027\n\017partition_names\030\004 \003(\t\022\013\n\003dsl\030\005 \001(\t\022" - "\031\n\021placeholder_group\030\006 \001(\014\022.\n\010dsl_type\030\007" - " \001(\0162\034.milvus.proto.common.DslType\022\025\n\rou" - "tput_fields\030\010 \003(\t\0228\n\rsearch_params\030\t \003(\013" - "2!.milvus.proto.common.KeyValuePair\022\030\n\020t" - "ravel_timestamp\030\n \001(\004\022\033\n\023guarantee_times" - "tamp\030\013 \001(\004\"5\n\004Hits\022\013\n\003IDs\030\001 \003(\003\022\020\n\010row_d" - "ata\030\002 \003(\014\022\016\n\006scores\030\003 \003(\002\"\215\001\n\rSearchResu" - "lts\022+\n\006status\030\001 \001(\0132\033.milvus.proto.commo" - "n.Status\0226\n\007results\030\002 \001(\0132%.milvus.proto" - ".schema.SearchResultData\022\027\n\017collection_n" - "ame\030\003 \001(\t\"e\n\014FlushRequest\022*\n\004base\030\001 \001(\0132" - "\034.milvus.proto.common.MsgBase\022\017\n\007db_name" - "\030\002 \001(\t\022\030\n\020collection_names\030\003 \003(\t\"\351\001\n\rFlu" - "shResponse\022+\n\006status\030\001 \001(\0132\033.milvus.prot" - "o.common.Status\022\017\n\007db_name\030\002 \001(\t\022G\n\013coll" - "_segIDs\030\003 \003(\01322.milvus.proto.milvus.Flus" - "hResponse.CollSegIDsEntry\032Q\n\017CollSegIDsE" - "ntry\022\013\n\003key\030\001 \001(\t\022-\n\005value\030\002 \001(\0132\036.milvu" - "s.proto.schema.LongArray:\0028\001\"\331\001\n\014QueryRe" - "quest\022*\n\004base\030\001 \001(\0132\034.milvus.proto.commo" - "n.MsgBase\022\017\n\007db_name\030\002 \001(\t\022\027\n\017collection" - "_name\030\003 \001(\t\022\014\n\004expr\030\004 \001(\t\022\025\n\routput_fiel" - "ds\030\005 \003(\t\022\027\n\017partition_names\030\006 \003(\t\022\030\n\020tra" - "vel_timestamp\030\007 \001(\004\022\033\n\023guarantee_timesta" - "mp\030\010 \001(\004\"\211\001\n\014QueryResults\022+\n\006status\030\001 \001(" - "\0132\033.milvus.proto.common.Status\0223\n\013fields" - "_data\030\002 \003(\0132\036.milvus.proto.schema.FieldD" - "ata\022\027\n\017collection_name\030\003 \001(\t\"}\n\tVectorID" - "s\022\027\n\017collection_name\030\001 \001(\t\022\022\n\nfield_name" - "\030\002 \001(\t\022*\n\010id_array\030\003 \001(\0132\030.milvus.proto." - "schema.IDs\022\027\n\017partition_names\030\004 \003(\t\"\203\001\n\014" - "VectorsArray\0222\n\010id_array\030\001 \001(\0132\036.milvus." - "proto.milvus.VectorIDsH\000\0226\n\ndata_array\030\002" - " \001(\0132 .milvus.proto.schema.VectorFieldH\000" - "B\007\n\005array\"\335\001\n\023CalcDistanceRequest\022*\n\004bas" - "e\030\001 \001(\0132\034.milvus.proto.common.MsgBase\0222\n" - "\007op_left\030\002 \001(\0132!.milvus.proto.milvus.Vec" - "torsArray\0223\n\010op_right\030\003 \001(\0132!.milvus.pro" - "to.milvus.VectorsArray\0221\n\006params\030\004 \003(\0132!" - ".milvus.proto.common.KeyValuePair\"\265\001\n\023Ca" - "lcDistanceResults\022+\n\006status\030\001 \001(\0132\033.milv" - "us.proto.common.Status\0221\n\010int_dist\030\002 \001(\013" - "2\035.milvus.proto.schema.IntArrayH\000\0225\n\nflo" - "at_dist\030\003 \001(\0132\037.milvus.proto.schema.Floa" - "tArrayH\000B\007\n\005array\"\231\001\n\025PersistentSegmentI" - "nfo\022\021\n\tsegmentID\030\001 \001(\003\022\024\n\014collectionID\030\002" - " \001(\003\022\023\n\013partitionID\030\003 \001(\003\022\020\n\010num_rows\030\004 " - "\001(\003\0220\n\005state\030\005 \001(\0162!.milvus.proto.common" - ".SegmentState\"u\n\037GetPersistentSegmentInf" - "oRequest\022*\n\004base\030\001 \001(\0132\034.milvus.proto.co" - "mmon.MsgBase\022\016\n\006dbName\030\002 \001(\t\022\026\n\016collecti" - "onName\030\003 \001(\t\"\212\001\n GetPersistentSegmentInf" - "oResponse\022+\n\006status\030\001 \001(\0132\033.milvus.proto" - ".common.Status\0229\n\005infos\030\002 \003(\0132*.milvus.p" - "roto.milvus.PersistentSegmentInfo\"\333\001\n\020Qu" - "erySegmentInfo\022\021\n\tsegmentID\030\001 \001(\003\022\024\n\014col" - "lectionID\030\002 \001(\003\022\023\n\013partitionID\030\003 \001(\003\022\020\n\010" - "mem_size\030\004 \001(\003\022\020\n\010num_rows\030\005 \001(\003\022\022\n\ninde" - "x_name\030\006 \001(\t\022\017\n\007indexID\030\007 \001(\003\022\016\n\006nodeID\030" - "\010 \001(\003\0220\n\005state\030\t \001(\0162!.milvus.proto.comm" - "on.SegmentState\"p\n\032GetQuerySegmentInfoRe" - "quest\022*\n\004base\030\001 \001(\0132\034.milvus.proto.commo" - "n.MsgBase\022\016\n\006dbName\030\002 \001(\t\022\026\n\016collectionN" - "ame\030\003 \001(\t\"\200\001\n\033GetQuerySegmentInfoRespons" - "e\022+\n\006status\030\001 \001(\0132\033.milvus.proto.common." - "Status\0224\n\005infos\030\002 \003(\0132%.milvus.proto.mil" - "vus.QuerySegmentInfo\"$\n\014DummyRequest\022\024\n\014" - "request_type\030\001 \001(\t\"!\n\rDummyResponse\022\020\n\010r" - "esponse\030\001 \001(\t\"\025\n\023RegisterLinkRequest\"r\n\024" - "RegisterLinkResponse\022-\n\007address\030\001 \001(\0132\034." - "milvus.proto.common.Address\022+\n\006status\030\002 " - "\001(\0132\033.milvus.proto.common.Status\"P\n\021GetM" - "etricsRequest\022*\n\004base\030\001 \001(\0132\034.milvus.pro" - "to.common.MsgBase\022\017\n\007request\030\002 \001(\t\"k\n\022Ge" - "tMetricsResponse\022+\n\006status\030\001 \001(\0132\033.milvu" - "s.proto.common.Status\022\020\n\010response\030\002 \001(\t\022" - "\026\n\016component_name\030\003 \001(\t\"\204\001\n\022LoadBalanceR" - "equest\022*\n\004base\030\001 \001(\0132\034.milvus.proto.comm" - "on.MsgBase\022\022\n\nsrc_nodeID\030\002 \001(\003\022\023\n\013dst_no" - "deIDs\030\003 \003(\003\022\031\n\021sealed_segmentIDs\030\004 \003(\003\"C" - "\n\027ManualCompactionRequest\022\024\n\014collectionI" - "D\030\001 \001(\003\022\022\n\ntimetravel\030\002 \001(\004\"]\n\030ManualCom" - "pactionResponse\022+\n\006status\030\001 \001(\0132\033.milvus" - ".proto.common.Status\022\024\n\014compactionID\030\002 \001" - "(\003\"1\n\031GetCompactionStateRequest\022\024\n\014compa" - "ctionID\030\001 \001(\003\"\307\001\n\032GetCompactionStateResp" - "onse\022+\n\006status\030\001 \001(\0132\033.milvus.proto.comm" - "on.Status\0223\n\005state\030\002 \001(\0162$.milvus.proto." - "common.CompactionState\022\027\n\017executingPlanN" - "o\030\003 \001(\003\022\025\n\rtimeoutPlanNo\030\004 \001(\003\022\027\n\017comple" - "tedPlanNo\030\005 \001(\003\"1\n\031GetCompactionPlansReq" - "uest\022\024\n\014compactionID\030\001 \001(\003\"\274\001\n\032GetCompac" - "tionPlansResponse\022+\n\006status\030\001 \001(\0132\033.milv" - "us.proto.common.Status\0223\n\005state\030\002 \001(\0162$." - "milvus.proto.common.CompactionState\022<\n\nm" - "ergeInfos\030\003 \003(\0132(.milvus.proto.milvus.Co" - "mpactionMergeInfo\"6\n\023CompactionMergeInfo" - "\022\017\n\007sources\030\001 \003(\003\022\016\n\006target\030\002 \001(\003\"*\n\024Get" - "FlushStateRequest\022\022\n\nsegmentIDs\030\001 \003(\003\"U\n" - "\025GetFlushStateResponse\022+\n\006status\030\001 \001(\0132\033" - ".milvus.proto.common.Status\022\017\n\007flushed\030\002" - " \001(\010\"\255\001\n\rImportRequest\022\027\n\017collection_nam" - "e\030\001 \001(\t\022\026\n\016partition_name\030\002 \001(\t\022\025\n\rchann" - "el_names\030\003 \003(\t\022\021\n\trow_based\030\004 \001(\010\022\r\n\005fil" - "es\030\005 \003(\t\0222\n\007options\030\006 \003(\0132!.milvus.proto" - ".common.KeyValuePair\"L\n\016ImportResponse\022+" - "\n\006status\030\001 \001(\0132\033.milvus.proto.common.Sta" - "tus\022\r\n\005tasks\030\002 \003(\003\"%\n\025GetImportStateRequ" - "est\022\014\n\004task\030\001 \001(\003\"\314\001\n\026GetImportStateResp" - "onse\022+\n\006status\030\001 \001(\0132\033.milvus.proto.comm" - "on.Status\022/\n\005state\030\002 \001(\0162 .milvus.proto." - "common.ImportState\022\021\n\trow_count\030\003 \001(\003\022\017\n" - "\007id_list\030\004 \003(\003\0220\n\005infos\030\005 \003(\0132!.milvus.p" - "roto.common.KeyValuePair\"p\n\022GetReplicasR" - "equest\022*\n\004base\030\001 \001(\0132\034.milvus.proto.comm" - "on.MsgBase\022\024\n\014collectionID\030\002 \001(\003\022\030\n\020with" - "_shard_nodes\030\003 \001(\010\"v\n\023GetReplicasRespons" - "e\022+\n\006status\030\001 \001(\0132\033.milvus.proto.common." - "Status\0222\n\010replicas\030\002 \003(\0132 .milvus.proto." - "milvus.ReplicaInfo\"\232\001\n\013ReplicaInfo\022\021\n\tre" - "plicaID\030\001 \001(\003\022\024\n\014collectionID\030\002 \001(\003\022\025\n\rp" - "artition_ids\030\003 \003(\003\0229\n\016shard_replicas\030\004 \003" - "(\0132!.milvus.proto.milvus.ShardReplica\022\020\n" - "\010node_ids\030\005 \003(\003\"`\n\014ShardReplica\022\020\n\010leade" - "rID\030\001 \001(\003\022\023\n\013leader_addr\030\002 \001(\t\022\027\n\017dm_cha" - "nnel_name\030\003 \001(\t\022\020\n\010node_ids\030\004 \003(\003\"\252\001\n\027Cr" - "eateCredentialRequest\022*\n\004base\030\001 \001(\0132\034.mi" - "lvus.proto.common.MsgBase\022\020\n\010username\030\002 " - "\001(\t\022\020\n\010password\030\003 \001(\t\022\036\n\026created_utc_tim" - "estamps\030\004 \001(\004\022\037\n\027modified_utc_timestamps" - "\030\005 \001(\004\"\302\001\n\027UpdateCredentialRequest\022*\n\004ba" - "se\030\001 \001(\0132\034.milvus.proto.common.MsgBase\022\020" - "\n\010username\030\002 \001(\t\022\023\n\013oldPassword\030\003 \001(\t\022\023\n" - "\013newPassword\030\004 \001(\t\022\036\n\026created_utc_timest" - "amps\030\005 \001(\004\022\037\n\027modified_utc_timestamps\030\006 " - "\001(\004\"W\n\027DeleteCredentialRequest\022*\n\004base\030\001" - " \001(\0132\034.milvus.proto.common.MsgBase\022\020\n\010us" - "ername\030\002 \001(\t\"W\n\025ListCredUsersResponse\022+\n" - "\006status\030\001 \001(\0132\033.milvus.proto.common.Stat" - "us\022\021\n\tusernames\030\002 \003(\t\"B\n\024ListCredUsersRe" - "quest\022*\n\004base\030\001 \001(\0132\034.milvus.proto.commo" - "n.MsgBase*!\n\010ShowType\022\007\n\003All\020\000\022\014\n\010InMemo" - "ry\020\001*>\n\017PlaceholderType\022\010\n\004None\020\000\022\020\n\014Bin" - "aryVector\020d\022\017\n\013FloatVector\020e2\270$\n\rMilvusS" - "ervice\022_\n\020CreateCollection\022,.milvus.prot" - "o.milvus.CreateCollectionRequest\032\033.milvu" - "s.proto.common.Status\"\000\022[\n\016DropCollectio" - "n\022*.milvus.proto.milvus.DropCollectionRe" - "quest\032\033.milvus.proto.common.Status\"\000\022_\n\r" - "HasCollection\022).milvus.proto.milvus.HasC" - "ollectionRequest\032!.milvus.proto.milvus.B" - "oolResponse\"\000\022[\n\016LoadCollection\022*.milvus" - ".proto.milvus.LoadCollectionRequest\032\033.mi" - "lvus.proto.common.Status\"\000\022a\n\021ReleaseCol" - "lection\022-.milvus.proto.milvus.ReleaseCol" - "lectionRequest\032\033.milvus.proto.common.Sta" - "tus\"\000\022w\n\022DescribeCollection\022..milvus.pro" - "to.milvus.DescribeCollectionRequest\032/.mi" - "lvus.proto.milvus.DescribeCollectionResp" - "onse\"\000\022\206\001\n\027GetCollectionStatistics\0223.mil" - "vus.proto.milvus.GetCollectionStatistics" - "Request\0324.milvus.proto.milvus.GetCollect" - "ionStatisticsResponse\"\000\022n\n\017ShowCollectio" - "ns\022+.milvus.proto.milvus.ShowCollections" - "Request\032,.milvus.proto.milvus.ShowCollec" - "tionsResponse\"\000\022]\n\017CreatePartition\022+.mil" - "vus.proto.milvus.CreatePartitionRequest\032" - "\033.milvus.proto.common.Status\"\000\022Y\n\rDropPa" - "rtition\022).milvus.proto.milvus.DropPartit" - "ionRequest\032\033.milvus.proto.common.Status\"" - "\000\022]\n\014HasPartition\022(.milvus.proto.milvus." - "HasPartitionRequest\032!.milvus.proto.milvu" - "s.BoolResponse\"\000\022[\n\016LoadPartitions\022*.mil" - "vus.proto.milvus.LoadPartitionsRequest\032\033" - ".milvus.proto.common.Status\"\000\022a\n\021Release" - "Partitions\022-.milvus.proto.milvus.Release" - "PartitionsRequest\032\033.milvus.proto.common." - "Status\"\000\022\203\001\n\026GetPartitionStatistics\0222.mi" - "lvus.proto.milvus.GetPartitionStatistics" - "Request\0323.milvus.proto.milvus.GetPartiti" - "onStatisticsResponse\"\000\022k\n\016ShowPartitions" - "\022*.milvus.proto.milvus.ShowPartitionsReq" - "uest\032+.milvus.proto.milvus.ShowPartition" - "sResponse\"\000\022U\n\013CreateAlias\022\'.milvus.prot" - "o.milvus.CreateAliasRequest\032\033.milvus.pro" - "to.common.Status\"\000\022Q\n\tDropAlias\022%.milvus" - ".proto.milvus.DropAliasRequest\032\033.milvus." - "proto.common.Status\"\000\022S\n\nAlterAlias\022&.mi" - "lvus.proto.milvus.AlterAliasRequest\032\033.mi" - "lvus.proto.common.Status\"\000\022U\n\013CreateInde" - "x\022\'.milvus.proto.milvus.CreateIndexReque" - "st\032\033.milvus.proto.common.Status\"\000\022h\n\rDes" - "cribeIndex\022).milvus.proto.milvus.Describ" - "eIndexRequest\032*.milvus.proto.milvus.Desc" - "ribeIndexResponse\"\000\022h\n\rGetIndexState\022).m" - "ilvus.proto.milvus.GetIndexStateRequest\032" - "*.milvus.proto.milvus.GetIndexStateRespo" - "nse\"\000\022\200\001\n\025GetIndexBuildProgress\0221.milvus" - ".proto.milvus.GetIndexBuildProgressReque" - "st\0322.milvus.proto.milvus.GetIndexBuildPr" - "ogressResponse\"\000\022Q\n\tDropIndex\022%.milvus.p" - "roto.milvus.DropIndexRequest\032\033.milvus.pr" - "oto.common.Status\"\000\022S\n\006Insert\022\".milvus.p" - "roto.milvus.InsertRequest\032#.milvus.proto" - ".milvus.MutationResult\"\000\022S\n\006Delete\022\".mil" - "vus.proto.milvus.DeleteRequest\032#.milvus." - "proto.milvus.MutationResult\"\000\022R\n\006Search\022" - "\".milvus.proto.milvus.SearchRequest\032\".mi" - "lvus.proto.milvus.SearchResults\"\000\022P\n\005Flu" - "sh\022!.milvus.proto.milvus.FlushRequest\032\"." - "milvus.proto.milvus.FlushResponse\"\000\022O\n\005Q" - "uery\022!.milvus.proto.milvus.QueryRequest\032" - "!.milvus.proto.milvus.QueryResults\"\000\022d\n\014" - "CalcDistance\022(.milvus.proto.milvus.CalcD" - "istanceRequest\032(.milvus.proto.milvus.Cal" - "cDistanceResults\"\000\022h\n\rGetFlushState\022).mi" - "lvus.proto.milvus.GetFlushStateRequest\032*" - ".milvus.proto.milvus.GetFlushStateRespon" - "se\"\000\022\211\001\n\030GetPersistentSegmentInfo\0224.milv" - "us.proto.milvus.GetPersistentSegmentInfo" - "Request\0325.milvus.proto.milvus.GetPersist" - "entSegmentInfoResponse\"\000\022z\n\023GetQuerySegm" - "entInfo\022/.milvus.proto.milvus.GetQuerySe" - "gmentInfoRequest\0320.milvus.proto.milvus.G" - "etQuerySegmentInfoResponse\"\000\022b\n\013GetRepli" - "cas\022\'.milvus.proto.milvus.GetReplicasReq" - "uest\032(.milvus.proto.milvus.GetReplicasRe" - "sponse\"\000\022P\n\005Dummy\022!.milvus.proto.milvus." - "DummyRequest\032\".milvus.proto.milvus.Dummy" - "Response\"\000\022e\n\014RegisterLink\022(.milvus.prot" - "o.milvus.RegisterLinkRequest\032).milvus.pr" - "oto.milvus.RegisterLinkResponse\"\000\022_\n\nGet" - "Metrics\022&.milvus.proto.milvus.GetMetrics" - "Request\032\'.milvus.proto.milvus.GetMetrics" - "Response\"\000\022U\n\013LoadBalance\022\'.milvus.proto" - ".milvus.LoadBalanceRequest\032\033.milvus.prot" - "o.common.Status\"\000\022w\n\022GetCompactionState\022" - "..milvus.proto.milvus.GetCompactionState" - "Request\032/.milvus.proto.milvus.GetCompact" - "ionStateResponse\"\000\022q\n\020ManualCompaction\022," - ".milvus.proto.milvus.ManualCompactionReq" - "uest\032-.milvus.proto.milvus.ManualCompact" - "ionResponse\"\000\022\200\001\n\033GetCompactionStateWith" - "Plans\022..milvus.proto.milvus.GetCompactio" - "nPlansRequest\032/.milvus.proto.milvus.GetC" - "ompactionPlansResponse\"\000\022S\n\006Import\022\".mil" - "vus.proto.milvus.ImportRequest\032#.milvus." - "proto.milvus.ImportResponse\"\000\022k\n\016GetImpo" - "rtState\022*.milvus.proto.milvus.GetImportS" - "tateRequest\032+.milvus.proto.milvus.GetImp" - "ortStateResponse\"\000\022_\n\020CreateCredential\022," - ".milvus.proto.milvus.CreateCredentialReq" - "uest\032\033.milvus.proto.common.Status\"\000\022_\n\020U" - "pdateCredential\022,.milvus.proto.milvus.Up" - "dateCredentialRequest\032\033.milvus.proto.com" - "mon.Status\"\000\022_\n\020DeleteCredential\022,.milvu" - "s.proto.milvus.DeleteCredentialRequest\032\033" - ".milvus.proto.common.Status\"\000\022h\n\rListCre" - "dUsers\022).milvus.proto.milvus.ListCredUse" - "rsRequest\032*.milvus.proto.milvus.ListCred" - "UsersResponse\"\0002u\n\014ProxyService\022e\n\014Regis" - "terLink\022(.milvus.proto.milvus.RegisterLi" - "nkRequest\032).milvus.proto.milvus.Register" - "LinkResponse\"\000BW\n\016io.milvus.grpcB\013Milvus" - "ProtoP\001Z3github.com/milvus-io/milvus/int" - "ernal/proto/milvuspb\240\001\001b\006proto3" + "b_name\030\002 \001(\t\022\027\n\017collection_name\030\003 \001(\t\022\022\n" + "\nfield_name\030\004 \001(\t\022\022\n\nindex_name\030\005 \001(\t\"\327\001" + "\n\rInsertRequest\022*\n\004base\030\001 \001(\0132\034.milvus.p" + "roto.common.MsgBase\022\017\n\007db_name\030\002 \001(\t\022\027\n\017" + "collection_name\030\003 \001(\t\022\026\n\016partition_name\030" + "\004 \001(\t\0223\n\013fields_data\030\005 \003(\0132\036.milvus.prot" + "o.schema.FieldData\022\021\n\thash_keys\030\006 \003(\r\022\020\n" + "\010num_rows\030\007 \001(\r\"\360\001\n\016MutationResult\022+\n\006st" + "atus\030\001 \001(\0132\033.milvus.proto.common.Status\022" + "%\n\003IDs\030\002 \001(\0132\030.milvus.proto.schema.IDs\022\022" + "\n\nsucc_index\030\003 \003(\r\022\021\n\terr_index\030\004 \003(\r\022\024\n" + "\014acknowledged\030\005 \001(\010\022\022\n\ninsert_cnt\030\006 \001(\003\022" + "\022\n\ndelete_cnt\030\007 \001(\003\022\022\n\nupsert_cnt\030\010 \001(\003\022" + "\021\n\ttimestamp\030\t \001(\004\"\236\001\n\rDeleteRequest\022*\n\004" + "base\030\001 \001(\0132\034.milvus.proto.common.MsgBase" + "\022\017\n\007db_name\030\002 \001(\t\022\027\n\017collection_name\030\003 \001" + "(\t\022\026\n\016partition_name\030\004 \001(\t\022\014\n\004expr\030\005 \001(\t" + "\022\021\n\thash_keys\030\006 \003(\r\"c\n\020PlaceholderValue\022" + "\013\n\003tag\030\001 \001(\t\0222\n\004type\030\002 \001(\0162$.milvus.prot" + "o.milvus.PlaceholderType\022\016\n\006values\030\003 \003(\014" + "\"O\n\020PlaceholderGroup\022;\n\014placeholders\030\001 \003" + "(\0132%.milvus.proto.milvus.PlaceholderValu" + "e\"\336\002\n\rSearchRequest\022*\n\004base\030\001 \001(\0132\034.milv" + "us.proto.common.MsgBase\022\017\n\007db_name\030\002 \001(\t" + "\022\027\n\017collection_name\030\003 \001(\t\022\027\n\017partition_n" + "ames\030\004 \003(\t\022\013\n\003dsl\030\005 \001(\t\022\031\n\021placeholder_g" + "roup\030\006 \001(\014\022.\n\010dsl_type\030\007 \001(\0162\034.milvus.pr" + "oto.common.DslType\022\025\n\routput_fields\030\010 \003(" + "\t\0228\n\rsearch_params\030\t \003(\0132!.milvus.proto." + "common.KeyValuePair\022\030\n\020travel_timestamp\030" + "\n \001(\004\022\033\n\023guarantee_timestamp\030\013 \001(\004\"5\n\004Hi" + "ts\022\013\n\003IDs\030\001 \003(\003\022\020\n\010row_data\030\002 \003(\014\022\016\n\006sco" + "res\030\003 \003(\002\"\215\001\n\rSearchResults\022+\n\006status\030\001 " + "\001(\0132\033.milvus.proto.common.Status\0226\n\007resu" + "lts\030\002 \001(\0132%.milvus.proto.schema.SearchRe" + "sultData\022\027\n\017collection_name\030\003 \001(\t\"e\n\014Flu" + "shRequest\022*\n\004base\030\001 \001(\0132\034.milvus.proto.c" + "ommon.MsgBase\022\017\n\007db_name\030\002 \001(\t\022\030\n\020collec" + "tion_names\030\003 \003(\t\"\351\001\n\rFlushResponse\022+\n\006st" + "atus\030\001 \001(\0132\033.milvus.proto.common.Status\022" + "\017\n\007db_name\030\002 \001(\t\022G\n\013coll_segIDs\030\003 \003(\01322." + "milvus.proto.milvus.FlushResponse.CollSe" + "gIDsEntry\032Q\n\017CollSegIDsEntry\022\013\n\003key\030\001 \001(" + "\t\022-\n\005value\030\002 \001(\0132\036.milvus.proto.schema.L" + "ongArray:\0028\001\"\331\001\n\014QueryRequest\022*\n\004base\030\001 " + "\001(\0132\034.milvus.proto.common.MsgBase\022\017\n\007db_" + "name\030\002 \001(\t\022\027\n\017collection_name\030\003 \001(\t\022\014\n\004e" + "xpr\030\004 \001(\t\022\025\n\routput_fields\030\005 \003(\t\022\027\n\017part" + "ition_names\030\006 \003(\t\022\030\n\020travel_timestamp\030\007 " + "\001(\004\022\033\n\023guarantee_timestamp\030\010 \001(\004\"\211\001\n\014Que" + "ryResults\022+\n\006status\030\001 \001(\0132\033.milvus.proto" + ".common.Status\0223\n\013fields_data\030\002 \003(\0132\036.mi" + "lvus.proto.schema.FieldData\022\027\n\017collectio" + "n_name\030\003 \001(\t\"}\n\tVectorIDs\022\027\n\017collection_" + "name\030\001 \001(\t\022\022\n\nfield_name\030\002 \001(\t\022*\n\010id_arr" + "ay\030\003 \001(\0132\030.milvus.proto.schema.IDs\022\027\n\017pa" + "rtition_names\030\004 \003(\t\"\203\001\n\014VectorsArray\0222\n\010" + "id_array\030\001 \001(\0132\036.milvus.proto.milvus.Vec" + "torIDsH\000\0226\n\ndata_array\030\002 \001(\0132 .milvus.pr" + "oto.schema.VectorFieldH\000B\007\n\005array\"\335\001\n\023Ca" + "lcDistanceRequest\022*\n\004base\030\001 \001(\0132\034.milvus" + ".proto.common.MsgBase\0222\n\007op_left\030\002 \001(\0132!" + ".milvus.proto.milvus.VectorsArray\0223\n\010op_" + "right\030\003 \001(\0132!.milvus.proto.milvus.Vector" + "sArray\0221\n\006params\030\004 \003(\0132!.milvus.proto.co" + "mmon.KeyValuePair\"\265\001\n\023CalcDistanceResult" + "s\022+\n\006status\030\001 \001(\0132\033.milvus.proto.common." + "Status\0221\n\010int_dist\030\002 \001(\0132\035.milvus.proto." + "schema.IntArrayH\000\0225\n\nfloat_dist\030\003 \001(\0132\037." + "milvus.proto.schema.FloatArrayH\000B\007\n\005arra" + "y\"\231\001\n\025PersistentSegmentInfo\022\021\n\tsegmentID" + "\030\001 \001(\003\022\024\n\014collectionID\030\002 \001(\003\022\023\n\013partitio" + "nID\030\003 \001(\003\022\020\n\010num_rows\030\004 \001(\003\0220\n\005state\030\005 \001" + "(\0162!.milvus.proto.common.SegmentState\"u\n" + "\037GetPersistentSegmentInfoRequest\022*\n\004base" + "\030\001 \001(\0132\034.milvus.proto.common.MsgBase\022\016\n\006" + "dbName\030\002 \001(\t\022\026\n\016collectionName\030\003 \001(\t\"\212\001\n" + " GetPersistentSegmentInfoResponse\022+\n\006sta" + "tus\030\001 \001(\0132\033.milvus.proto.common.Status\0229" + "\n\005infos\030\002 \003(\0132*.milvus.proto.milvus.Pers" + "istentSegmentInfo\"\333\001\n\020QuerySegmentInfo\022\021" + "\n\tsegmentID\030\001 \001(\003\022\024\n\014collectionID\030\002 \001(\003\022" + "\023\n\013partitionID\030\003 \001(\003\022\020\n\010mem_size\030\004 \001(\003\022\020" + "\n\010num_rows\030\005 \001(\003\022\022\n\nindex_name\030\006 \001(\t\022\017\n\007" + "indexID\030\007 \001(\003\022\016\n\006nodeID\030\010 \001(\003\0220\n\005state\030\t" + " \001(\0162!.milvus.proto.common.SegmentState\"" + "p\n\032GetQuerySegmentInfoRequest\022*\n\004base\030\001 " + "\001(\0132\034.milvus.proto.common.MsgBase\022\016\n\006dbN" + "ame\030\002 \001(\t\022\026\n\016collectionName\030\003 \001(\t\"\200\001\n\033Ge" + "tQuerySegmentInfoResponse\022+\n\006status\030\001 \001(" + "\0132\033.milvus.proto.common.Status\0224\n\005infos\030" + "\002 \003(\0132%.milvus.proto.milvus.QuerySegment" + "Info\"$\n\014DummyRequest\022\024\n\014request_type\030\001 \001" + "(\t\"!\n\rDummyResponse\022\020\n\010response\030\001 \001(\t\"\025\n" + "\023RegisterLinkRequest\"r\n\024RegisterLinkResp" + "onse\022-\n\007address\030\001 \001(\0132\034.milvus.proto.com" + "mon.Address\022+\n\006status\030\002 \001(\0132\033.milvus.pro" + "to.common.Status\"P\n\021GetMetricsRequest\022*\n" + "\004base\030\001 \001(\0132\034.milvus.proto.common.MsgBas" + "e\022\017\n\007request\030\002 \001(\t\"k\n\022GetMetricsResponse" + "\022+\n\006status\030\001 \001(\0132\033.milvus.proto.common.S" + "tatus\022\020\n\010response\030\002 \001(\t\022\026\n\016component_nam" + "e\030\003 \001(\t\"\234\001\n\022LoadBalanceRequest\022*\n\004base\030\001" + " \001(\0132\034.milvus.proto.common.MsgBase\022\022\n\nsr" + "c_nodeID\030\002 \001(\003\022\023\n\013dst_nodeIDs\030\003 \003(\003\022\031\n\021s" + "ealed_segmentIDs\030\004 \003(\003\022\026\n\016collectionName" + "\030\005 \001(\t\"C\n\027ManualCompactionRequest\022\024\n\014col" + "lectionID\030\001 \001(\003\022\022\n\ntimetravel\030\002 \001(\004\"]\n\030M" + "anualCompactionResponse\022+\n\006status\030\001 \001(\0132" + "\033.milvus.proto.common.Status\022\024\n\014compacti" + "onID\030\002 \001(\003\"1\n\031GetCompactionStateRequest\022" + "\024\n\014compactionID\030\001 \001(\003\"\307\001\n\032GetCompactionS" + "tateResponse\022+\n\006status\030\001 \001(\0132\033.milvus.pr" + "oto.common.Status\0223\n\005state\030\002 \001(\0162$.milvu" + "s.proto.common.CompactionState\022\027\n\017execut" + "ingPlanNo\030\003 \001(\003\022\025\n\rtimeoutPlanNo\030\004 \001(\003\022\027" + "\n\017completedPlanNo\030\005 \001(\003\"1\n\031GetCompaction" + "PlansRequest\022\024\n\014compactionID\030\001 \001(\003\"\274\001\n\032G" + "etCompactionPlansResponse\022+\n\006status\030\001 \001(" + "\0132\033.milvus.proto.common.Status\0223\n\005state\030" + "\002 \001(\0162$.milvus.proto.common.CompactionSt" + "ate\022<\n\nmergeInfos\030\003 \003(\0132(.milvus.proto.m" + "ilvus.CompactionMergeInfo\"6\n\023CompactionM" + "ergeInfo\022\017\n\007sources\030\001 \003(\003\022\016\n\006target\030\002 \001(" + "\003\"*\n\024GetFlushStateRequest\022\022\n\nsegmentIDs\030" + "\001 \003(\003\"U\n\025GetFlushStateResponse\022+\n\006status" + "\030\001 \001(\0132\033.milvus.proto.common.Status\022\017\n\007f" + "lushed\030\002 \001(\010\"\255\001\n\rImportRequest\022\027\n\017collec" + "tion_name\030\001 \001(\t\022\026\n\016partition_name\030\002 \001(\t\022" + "\025\n\rchannel_names\030\003 \003(\t\022\021\n\trow_based\030\004 \001(" + "\010\022\r\n\005files\030\005 \003(\t\0222\n\007options\030\006 \003(\0132!.milv" + "us.proto.common.KeyValuePair\"L\n\016ImportRe" + "sponse\022+\n\006status\030\001 \001(\0132\033.milvus.proto.co" + "mmon.Status\022\r\n\005tasks\030\002 \003(\003\"%\n\025GetImportS" + "tateRequest\022\014\n\004task\030\001 \001(\003\"\314\001\n\026GetImportS" + "tateResponse\022+\n\006status\030\001 \001(\0132\033.milvus.pr" + "oto.common.Status\022/\n\005state\030\002 \001(\0162 .milvu" + "s.proto.common.ImportState\022\021\n\trow_count\030" + "\003 \001(\003\022\017\n\007id_list\030\004 \003(\003\0220\n\005infos\030\005 \003(\0132!." + "milvus.proto.common.KeyValuePair\"p\n\022GetR" + "eplicasRequest\022*\n\004base\030\001 \001(\0132\034.milvus.pr" + "oto.common.MsgBase\022\024\n\014collectionID\030\002 \001(\003" + "\022\030\n\020with_shard_nodes\030\003 \001(\010\"v\n\023GetReplica" + "sResponse\022+\n\006status\030\001 \001(\0132\033.milvus.proto" + ".common.Status\0222\n\010replicas\030\002 \003(\0132 .milvu" + "s.proto.milvus.ReplicaInfo\"\232\001\n\013ReplicaIn" + "fo\022\021\n\treplicaID\030\001 \001(\003\022\024\n\014collectionID\030\002 " + "\001(\003\022\025\n\rpartition_ids\030\003 \003(\003\0229\n\016shard_repl" + "icas\030\004 \003(\0132!.milvus.proto.milvus.ShardRe" + "plica\022\020\n\010node_ids\030\005 \003(\003\"`\n\014ShardReplica\022" + "\020\n\010leaderID\030\001 \001(\003\022\023\n\013leader_addr\030\002 \001(\t\022\027" + "\n\017dm_channel_name\030\003 \001(\t\022\020\n\010node_ids\030\004 \003(" + "\003\"\252\001\n\027CreateCredentialRequest\022*\n\004base\030\001 " + "\001(\0132\034.milvus.proto.common.MsgBase\022\020\n\010use" + "rname\030\002 \001(\t\022\020\n\010password\030\003 \001(\t\022\036\n\026created" + "_utc_timestamps\030\004 \001(\004\022\037\n\027modified_utc_ti" + "mestamps\030\005 \001(\004\"\302\001\n\027UpdateCredentialReque" + "st\022*\n\004base\030\001 \001(\0132\034.milvus.proto.common.M" + "sgBase\022\020\n\010username\030\002 \001(\t\022\023\n\013oldPassword\030" + "\003 \001(\t\022\023\n\013newPassword\030\004 \001(\t\022\036\n\026created_ut" + "c_timestamps\030\005 \001(\004\022\037\n\027modified_utc_times" + "tamps\030\006 \001(\004\"W\n\027DeleteCredentialRequest\022*" + "\n\004base\030\001 \001(\0132\034.milvus.proto.common.MsgBa" + "se\022\020\n\010username\030\002 \001(\t\"W\n\025ListCredUsersRes" + "ponse\022+\n\006status\030\001 \001(\0132\033.milvus.proto.com" + "mon.Status\022\021\n\tusernames\030\002 \003(\t\"B\n\024ListCre" + "dUsersRequest\022*\n\004base\030\001 \001(\0132\034.milvus.pro" + "to.common.MsgBase*!\n\010ShowType\022\007\n\003All\020\000\022\014" + "\n\010InMemory\020\001*>\n\017PlaceholderType\022\010\n\004None\020" + "\000\022\020\n\014BinaryVector\020d\022\017\n\013FloatVector\020e2\270$\n" + "\rMilvusService\022_\n\020CreateCollection\022,.mil" + "vus.proto.milvus.CreateCollectionRequest" + "\032\033.milvus.proto.common.Status\"\000\022[\n\016DropC" + "ollection\022*.milvus.proto.milvus.DropColl" + "ectionRequest\032\033.milvus.proto.common.Stat" + "us\"\000\022_\n\rHasCollection\022).milvus.proto.mil" + "vus.HasCollectionRequest\032!.milvus.proto." + "milvus.BoolResponse\"\000\022[\n\016LoadCollection\022" + "*.milvus.proto.milvus.LoadCollectionRequ" + "est\032\033.milvus.proto.common.Status\"\000\022a\n\021Re" + "leaseCollection\022-.milvus.proto.milvus.Re" + "leaseCollectionRequest\032\033.milvus.proto.co" + "mmon.Status\"\000\022w\n\022DescribeCollection\022..mi" + "lvus.proto.milvus.DescribeCollectionRequ" + "est\032/.milvus.proto.milvus.DescribeCollec" + "tionResponse\"\000\022\206\001\n\027GetCollectionStatisti" + "cs\0223.milvus.proto.milvus.GetCollectionSt" + "atisticsRequest\0324.milvus.proto.milvus.Ge" + "tCollectionStatisticsResponse\"\000\022n\n\017ShowC" + "ollections\022+.milvus.proto.milvus.ShowCol" + "lectionsRequest\032,.milvus.proto.milvus.Sh" + "owCollectionsResponse\"\000\022]\n\017CreatePartiti" + "on\022+.milvus.proto.milvus.CreatePartition" + "Request\032\033.milvus.proto.common.Status\"\000\022Y" + "\n\rDropPartition\022).milvus.proto.milvus.Dr" + "opPartitionRequest\032\033.milvus.proto.common" + ".Status\"\000\022]\n\014HasPartition\022(.milvus.proto" + ".milvus.HasPartitionRequest\032!.milvus.pro" + "to.milvus.BoolResponse\"\000\022[\n\016LoadPartitio" + "ns\022*.milvus.proto.milvus.LoadPartitionsR" + "equest\032\033.milvus.proto.common.Status\"\000\022a\n" + "\021ReleasePartitions\022-.milvus.proto.milvus" + ".ReleasePartitionsRequest\032\033.milvus.proto" + ".common.Status\"\000\022\203\001\n\026GetPartitionStatist" + "ics\0222.milvus.proto.milvus.GetPartitionSt" + "atisticsRequest\0323.milvus.proto.milvus.Ge" + "tPartitionStatisticsResponse\"\000\022k\n\016ShowPa" + "rtitions\022*.milvus.proto.milvus.ShowParti" + "tionsRequest\032+.milvus.proto.milvus.ShowP" + "artitionsResponse\"\000\022U\n\013CreateAlias\022\'.mil" + "vus.proto.milvus.CreateAliasRequest\032\033.mi" + "lvus.proto.common.Status\"\000\022Q\n\tDropAlias\022" + "%.milvus.proto.milvus.DropAliasRequest\032\033" + ".milvus.proto.common.Status\"\000\022S\n\nAlterAl" + "ias\022&.milvus.proto.milvus.AlterAliasRequ" + "est\032\033.milvus.proto.common.Status\"\000\022U\n\013Cr" + "eateIndex\022\'.milvus.proto.milvus.CreateIn" + "dexRequest\032\033.milvus.proto.common.Status\"" + "\000\022h\n\rDescribeIndex\022).milvus.proto.milvus" + ".DescribeIndexRequest\032*.milvus.proto.mil" + "vus.DescribeIndexResponse\"\000\022h\n\rGetIndexS" + "tate\022).milvus.proto.milvus.GetIndexState" + "Request\032*.milvus.proto.milvus.GetIndexSt" + "ateResponse\"\000\022\200\001\n\025GetIndexBuildProgress\022" + "1.milvus.proto.milvus.GetIndexBuildProgr" + "essRequest\0322.milvus.proto.milvus.GetInde" + "xBuildProgressResponse\"\000\022Q\n\tDropIndex\022%." + "milvus.proto.milvus.DropIndexRequest\032\033.m" + "ilvus.proto.common.Status\"\000\022S\n\006Insert\022\"." + "milvus.proto.milvus.InsertRequest\032#.milv" + "us.proto.milvus.MutationResult\"\000\022S\n\006Dele" + "te\022\".milvus.proto.milvus.DeleteRequest\032#" + ".milvus.proto.milvus.MutationResult\"\000\022R\n" + "\006Search\022\".milvus.proto.milvus.SearchRequ" + "est\032\".milvus.proto.milvus.SearchResults\"" + "\000\022P\n\005Flush\022!.milvus.proto.milvus.FlushRe" + "quest\032\".milvus.proto.milvus.FlushRespons" + "e\"\000\022O\n\005Query\022!.milvus.proto.milvus.Query" + "Request\032!.milvus.proto.milvus.QueryResul" + "ts\"\000\022d\n\014CalcDistance\022(.milvus.proto.milv" + "us.CalcDistanceRequest\032(.milvus.proto.mi" + "lvus.CalcDistanceResults\"\000\022h\n\rGetFlushSt" + "ate\022).milvus.proto.milvus.GetFlushStateR" + "equest\032*.milvus.proto.milvus.GetFlushSta" + "teResponse\"\000\022\211\001\n\030GetPersistentSegmentInf" + "o\0224.milvus.proto.milvus.GetPersistentSeg" + "mentInfoRequest\0325.milvus.proto.milvus.Ge" + "tPersistentSegmentInfoResponse\"\000\022z\n\023GetQ" + "uerySegmentInfo\022/.milvus.proto.milvus.Ge" + "tQuerySegmentInfoRequest\0320.milvus.proto." + "milvus.GetQuerySegmentInfoResponse\"\000\022b\n\013" + "GetReplicas\022\'.milvus.proto.milvus.GetRep" + "licasRequest\032(.milvus.proto.milvus.GetRe" + "plicasResponse\"\000\022P\n\005Dummy\022!.milvus.proto" + ".milvus.DummyRequest\032\".milvus.proto.milv" + "us.DummyResponse\"\000\022e\n\014RegisterLink\022(.mil" + "vus.proto.milvus.RegisterLinkRequest\032).m" + "ilvus.proto.milvus.RegisterLinkResponse\"" + "\000\022_\n\nGetMetrics\022&.milvus.proto.milvus.Ge" + "tMetricsRequest\032\'.milvus.proto.milvus.Ge" + "tMetricsResponse\"\000\022U\n\013LoadBalance\022\'.milv" + "us.proto.milvus.LoadBalanceRequest\032\033.mil" + "vus.proto.common.Status\"\000\022w\n\022GetCompacti" + "onState\022..milvus.proto.milvus.GetCompact" + "ionStateRequest\032/.milvus.proto.milvus.Ge" + "tCompactionStateResponse\"\000\022q\n\020ManualComp" + "action\022,.milvus.proto.milvus.ManualCompa" + "ctionRequest\032-.milvus.proto.milvus.Manua" + "lCompactionResponse\"\000\022\200\001\n\033GetCompactionS" + "tateWithPlans\022..milvus.proto.milvus.GetC" + "ompactionPlansRequest\032/.milvus.proto.mil" + "vus.GetCompactionPlansResponse\"\000\022S\n\006Impo" + "rt\022\".milvus.proto.milvus.ImportRequest\032#" + ".milvus.proto.milvus.ImportResponse\"\000\022k\n" + "\016GetImportState\022*.milvus.proto.milvus.Ge" + "tImportStateRequest\032+.milvus.proto.milvu" + "s.GetImportStateResponse\"\000\022_\n\020CreateCred" + "ential\022,.milvus.proto.milvus.CreateCrede" + "ntialRequest\032\033.milvus.proto.common.Statu" + "s\"\000\022_\n\020UpdateCredential\022,.milvus.proto.m" + "ilvus.UpdateCredentialRequest\032\033.milvus.p" + "roto.common.Status\"\000\022_\n\020DeleteCredential" + "\022,.milvus.proto.milvus.DeleteCredentialR" + "equest\032\033.milvus.proto.common.Status\"\000\022h\n" + "\rListCredUsers\022).milvus.proto.milvus.Lis" + "tCredUsersRequest\032*.milvus.proto.milvus." + "ListCredUsersResponse\"\0002u\n\014ProxyService\022" + "e\n\014RegisterLink\022(.milvus.proto.milvus.Re" + "gisterLinkRequest\032).milvus.proto.milvus." + "RegisterLinkResponse\"\000BW\n\016io.milvus.grpc" + "B\013MilvusProtoP\001Z3github.com/milvus-io/mi" + "lvus/internal/proto/milvuspb\240\001\001b\006proto3" ; static const ::PROTOBUF_NAMESPACE_ID::internal::DescriptorTable*const descriptor_table_milvus_2eproto_deps[2] = { &::descriptor_table_common_2eproto, @@ -3277,7 +3280,7 @@ static ::PROTOBUF_NAMESPACE_ID::internal::SCCInfoBase*const descriptor_table_mil static ::PROTOBUF_NAMESPACE_ID::internal::once_flag descriptor_table_milvus_2eproto_once; static bool descriptor_table_milvus_2eproto_initialized = false; const ::PROTOBUF_NAMESPACE_ID::internal::DescriptorTable descriptor_table_milvus_2eproto = { - &descriptor_table_milvus_2eproto_initialized, descriptor_table_protodef_milvus_2eproto, "milvus.proto", 17271, + &descriptor_table_milvus_2eproto_initialized, descriptor_table_protodef_milvus_2eproto, "milvus.proto", 17319, &descriptor_table_milvus_2eproto_once, descriptor_table_milvus_2eproto_sccs, descriptor_table_milvus_2eproto_deps, 90, 2, schemas, file_default_instances, TableStruct_milvus_2eproto::offsets, file_level_metadata_milvus_2eproto, 90, file_level_enum_descriptors_milvus_2eproto, file_level_service_descriptors_milvus_2eproto, @@ -12105,6 +12108,7 @@ LoadPartitionsRequest::LoadPartitionsRequest(const LoadPartitionsRequest& from) } else { base_ = nullptr; } + replica_number_ = from.replica_number_; // @@protoc_insertion_point(copy_constructor:milvus.proto.milvus.LoadPartitionsRequest) } @@ -12112,7 +12116,9 @@ void LoadPartitionsRequest::SharedCtor() { ::PROTOBUF_NAMESPACE_ID::internal::InitSCC(&scc_info_LoadPartitionsRequest_milvus_2eproto.base); db_name_.UnsafeSetDefault(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited()); collection_name_.UnsafeSetDefault(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited()); - base_ = nullptr; + ::memset(&base_, 0, static_cast( + reinterpret_cast(&replica_number_) - + reinterpret_cast(&base_)) + sizeof(replica_number_)); } LoadPartitionsRequest::~LoadPartitionsRequest() { @@ -12148,6 +12154,7 @@ void LoadPartitionsRequest::Clear() { delete base_; } base_ = nullptr; + replica_number_ = 0; _internal_metadata_.Clear(); } @@ -12192,6 +12199,13 @@ const char* LoadPartitionsRequest::_InternalParse(const char* ptr, ::PROTOBUF_NA } while (::PROTOBUF_NAMESPACE_ID::internal::UnalignedLoad<::PROTOBUF_NAMESPACE_ID::uint8>(ptr) == 34); } else goto handle_unusual; continue; + // int32 replica_number = 5; + case 5: + if (PROTOBUF_PREDICT_TRUE(static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 40)) { + replica_number_ = ::PROTOBUF_NAMESPACE_ID::internal::ReadVarint(&ptr); + CHK_(ptr); + } else goto handle_unusual; + continue; default: { handle_unusual: if ((tag & 7) == 4 || tag == 0) { @@ -12279,6 +12293,19 @@ bool LoadPartitionsRequest::MergePartialFromCodedStream( break; } + // int32 replica_number = 5; + case 5: { + if (static_cast< ::PROTOBUF_NAMESPACE_ID::uint8>(tag) == (40 & 0xFF)) { + + DO_((::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::ReadPrimitive< + ::PROTOBUF_NAMESPACE_ID::int32, ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::TYPE_INT32>( + input, &replica_number_))); + } else { + goto handle_unusual; + } + break; + } + default: { handle_unusual: if (tag == 0) { @@ -12342,6 +12369,11 @@ void LoadPartitionsRequest::SerializeWithCachedSizes( 4, this->partition_names(i), output); } + // int32 replica_number = 5; + if (this->replica_number() != 0) { + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteInt32(5, this->replica_number(), output); + } + if (_internal_metadata_.have_unknown_fields()) { ::PROTOBUF_NAMESPACE_ID::internal::WireFormat::SerializeUnknownFields( _internal_metadata_.unknown_fields(), output); @@ -12394,6 +12426,11 @@ void LoadPartitionsRequest::SerializeWithCachedSizes( WriteStringToArray(4, this->partition_names(i), target); } + // int32 replica_number = 5; + if (this->replica_number() != 0) { + target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteInt32ToArray(5, this->replica_number(), target); + } + if (_internal_metadata_.have_unknown_fields()) { target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormat::SerializeUnknownFieldsToArray( _internal_metadata_.unknown_fields(), target); @@ -12444,6 +12481,13 @@ size_t LoadPartitionsRequest::ByteSizeLong() const { *base_); } + // int32 replica_number = 5; + if (this->replica_number() != 0) { + total_size += 1 + + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::Int32Size( + this->replica_number()); + } + int cached_size = ::PROTOBUF_NAMESPACE_ID::internal::ToCachedSize(total_size); SetCachedSize(cached_size); return total_size; @@ -12483,6 +12527,9 @@ void LoadPartitionsRequest::MergeFrom(const LoadPartitionsRequest& from) { if (from.has_base()) { mutable_base()->::milvus::proto::common::MsgBase::MergeFrom(from.base()); } + if (from.replica_number() != 0) { + set_replica_number(from.replica_number()); + } } void LoadPartitionsRequest::CopyFrom(const ::PROTOBUF_NAMESPACE_ID::Message& from) { @@ -12512,6 +12559,7 @@ void LoadPartitionsRequest::InternalSwap(LoadPartitionsRequest* other) { collection_name_.Swap(&other->collection_name_, &::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), GetArenaNoVirtual()); swap(base_, other->base_); + swap(replica_number_, other->replica_number_); } ::PROTOBUF_NAMESPACE_ID::Metadata LoadPartitionsRequest::GetMetadata() const { @@ -32790,6 +32838,10 @@ LoadBalanceRequest::LoadBalanceRequest(const LoadBalanceRequest& from) dst_nodeids_(from.dst_nodeids_), sealed_segmentids_(from.sealed_segmentids_) { _internal_metadata_.MergeFrom(from._internal_metadata_); + collectionname_.UnsafeSetDefault(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited()); + if (!from.collectionname().empty()) { + collectionname_.AssignWithDefault(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), from.collectionname_); + } if (from.has_base()) { base_ = new ::milvus::proto::common::MsgBase(*from.base_); } else { @@ -32801,6 +32853,7 @@ LoadBalanceRequest::LoadBalanceRequest(const LoadBalanceRequest& from) void LoadBalanceRequest::SharedCtor() { ::PROTOBUF_NAMESPACE_ID::internal::InitSCC(&scc_info_LoadBalanceRequest_milvus_2eproto.base); + collectionname_.UnsafeSetDefault(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited()); ::memset(&base_, 0, static_cast( reinterpret_cast(&src_nodeid_) - reinterpret_cast(&base_)) + sizeof(src_nodeid_)); @@ -32812,6 +32865,7 @@ LoadBalanceRequest::~LoadBalanceRequest() { } void LoadBalanceRequest::SharedDtor() { + collectionname_.DestroyNoArena(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited()); if (this != internal_default_instance()) delete base_; } @@ -32832,6 +32886,7 @@ void LoadBalanceRequest::Clear() { dst_nodeids_.Clear(); sealed_segmentids_.Clear(); + collectionname_.ClearToEmptyNoArena(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited()); if (GetArenaNoVirtual() == nullptr && base_ != nullptr) { delete base_; } @@ -32882,6 +32937,13 @@ const char* LoadBalanceRequest::_InternalParse(const char* ptr, ::PROTOBUF_NAMES CHK_(ptr); } else goto handle_unusual; continue; + // string collectionName = 5; + case 5: + if (PROTOBUF_PREDICT_TRUE(static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 42)) { + ptr = ::PROTOBUF_NAMESPACE_ID::internal::InlineGreedyStringParserUTF8(mutable_collectionname(), ptr, ctx, "milvus.proto.milvus.LoadBalanceRequest.collectionName"); + CHK_(ptr); + } else goto handle_unusual; + continue; default: { handle_unusual: if ((tag & 7) == 4 || tag == 0) { @@ -32968,6 +33030,21 @@ bool LoadBalanceRequest::MergePartialFromCodedStream( break; } + // string collectionName = 5; + case 5: { + if (static_cast< ::PROTOBUF_NAMESPACE_ID::uint8>(tag) == (42 & 0xFF)) { + DO_(::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::ReadString( + input, this->mutable_collectionname())); + DO_(::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::VerifyUtf8String( + this->collectionname().data(), static_cast(this->collectionname().length()), + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::PARSE, + "milvus.proto.milvus.LoadBalanceRequest.collectionName")); + } else { + goto handle_unusual; + } + break; + } + default: { handle_unusual: if (tag == 0) { @@ -33028,6 +33105,16 @@ void LoadBalanceRequest::SerializeWithCachedSizes( this->sealed_segmentids(i), output); } + // string collectionName = 5; + if (this->collectionname().size() > 0) { + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::VerifyUtf8String( + this->collectionname().data(), static_cast(this->collectionname().length()), + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::SERIALIZE, + "milvus.proto.milvus.LoadBalanceRequest.collectionName"); + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteStringMaybeAliased( + 5, this->collectionname(), output); + } + if (_internal_metadata_.have_unknown_fields()) { ::PROTOBUF_NAMESPACE_ID::internal::WireFormat::SerializeUnknownFields( _internal_metadata_.unknown_fields(), output); @@ -33079,6 +33166,17 @@ void LoadBalanceRequest::SerializeWithCachedSizes( WriteInt64NoTagToArray(this->sealed_segmentids_, target); } + // string collectionName = 5; + if (this->collectionname().size() > 0) { + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::VerifyUtf8String( + this->collectionname().data(), static_cast(this->collectionname().length()), + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::SERIALIZE, + "milvus.proto.milvus.LoadBalanceRequest.collectionName"); + target = + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteStringToArray( + 5, this->collectionname(), target); + } + if (_internal_metadata_.have_unknown_fields()) { target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormat::SerializeUnknownFieldsToArray( _internal_metadata_.unknown_fields(), target); @@ -33130,6 +33228,13 @@ size_t LoadBalanceRequest::ByteSizeLong() const { total_size += data_size; } + // string collectionName = 5; + if (this->collectionname().size() > 0) { + total_size += 1 + + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::StringSize( + this->collectionname()); + } + // .milvus.proto.common.MsgBase base = 1; if (this->has_base()) { total_size += 1 + @@ -33173,6 +33278,10 @@ void LoadBalanceRequest::MergeFrom(const LoadBalanceRequest& from) { dst_nodeids_.MergeFrom(from.dst_nodeids_); sealed_segmentids_.MergeFrom(from.sealed_segmentids_); + if (from.collectionname().size() > 0) { + + collectionname_.AssignWithDefault(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), from.collectionname_); + } if (from.has_base()) { mutable_base()->::milvus::proto::common::MsgBase::MergeFrom(from.base()); } @@ -33204,6 +33313,8 @@ void LoadBalanceRequest::InternalSwap(LoadBalanceRequest* other) { _internal_metadata_.Swap(&other->_internal_metadata_); dst_nodeids_.InternalSwap(&other->dst_nodeids_); sealed_segmentids_.InternalSwap(&other->sealed_segmentids_); + collectionname_.Swap(&other->collectionname_, &::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), + GetArenaNoVirtual()); swap(base_, other->base_); swap(src_nodeid_, other->src_nodeid_); } diff --git a/internal/core/src/pb/milvus.pb.h b/internal/core/src/pb/milvus.pb.h index d00b7c6018..6c4efb34c3 100644 --- a/internal/core/src/pb/milvus.pb.h +++ b/internal/core/src/pb/milvus.pb.h @@ -3888,6 +3888,7 @@ class LoadPartitionsRequest : kDbNameFieldNumber = 2, kCollectionNameFieldNumber = 3, kBaseFieldNumber = 1, + kReplicaNumberFieldNumber = 5, }; // repeated string partition_names = 4; int partition_names_size() const; @@ -3936,6 +3937,11 @@ class LoadPartitionsRequest : ::milvus::proto::common::MsgBase* mutable_base(); void set_allocated_base(::milvus::proto::common::MsgBase* base); + // int32 replica_number = 5; + void clear_replica_number(); + ::PROTOBUF_NAMESPACE_ID::int32 replica_number() const; + void set_replica_number(::PROTOBUF_NAMESPACE_ID::int32 value); + // @@protoc_insertion_point(class_scope:milvus.proto.milvus.LoadPartitionsRequest) private: class _Internal; @@ -3945,6 +3951,7 @@ class LoadPartitionsRequest : ::PROTOBUF_NAMESPACE_ID::internal::ArenaStringPtr db_name_; ::PROTOBUF_NAMESPACE_ID::internal::ArenaStringPtr collection_name_; ::milvus::proto::common::MsgBase* base_; + ::PROTOBUF_NAMESPACE_ID::int32 replica_number_; mutable ::PROTOBUF_NAMESPACE_ID::internal::CachedSize _cached_size_; friend struct ::TableStruct_milvus_2eproto; }; @@ -11873,6 +11880,7 @@ class LoadBalanceRequest : enum : int { kDstNodeIDsFieldNumber = 3, kSealedSegmentIDsFieldNumber = 4, + kCollectionNameFieldNumber = 5, kBaseFieldNumber = 1, kSrcNodeIDFieldNumber = 2, }; @@ -11898,6 +11906,17 @@ class LoadBalanceRequest : ::PROTOBUF_NAMESPACE_ID::RepeatedField< ::PROTOBUF_NAMESPACE_ID::int64 >* mutable_sealed_segmentids(); + // string collectionName = 5; + void clear_collectionname(); + const std::string& collectionname() const; + void set_collectionname(const std::string& value); + void set_collectionname(std::string&& value); + void set_collectionname(const char* value); + void set_collectionname(const char* value, size_t size); + std::string* mutable_collectionname(); + std::string* release_collectionname(); + void set_allocated_collectionname(std::string* collectionname); + // .milvus.proto.common.MsgBase base = 1; bool has_base() const; void clear_base(); @@ -11920,6 +11939,7 @@ class LoadBalanceRequest : mutable std::atomic _dst_nodeids_cached_byte_size_; ::PROTOBUF_NAMESPACE_ID::RepeatedField< ::PROTOBUF_NAMESPACE_ID::int64 > sealed_segmentids_; mutable std::atomic _sealed_segmentids_cached_byte_size_; + ::PROTOBUF_NAMESPACE_ID::internal::ArenaStringPtr collectionname_; ::milvus::proto::common::MsgBase* base_; ::PROTOBUF_NAMESPACE_ID::int64 src_nodeid_; mutable ::PROTOBUF_NAMESPACE_ID::internal::CachedSize _cached_size_; @@ -18987,6 +19007,20 @@ LoadPartitionsRequest::mutable_partition_names() { return &partition_names_; } +// int32 replica_number = 5; +inline void LoadPartitionsRequest::clear_replica_number() { + replica_number_ = 0; +} +inline ::PROTOBUF_NAMESPACE_ID::int32 LoadPartitionsRequest::replica_number() const { + // @@protoc_insertion_point(field_get:milvus.proto.milvus.LoadPartitionsRequest.replica_number) + return replica_number_; +} +inline void LoadPartitionsRequest::set_replica_number(::PROTOBUF_NAMESPACE_ID::int32 value) { + + replica_number_ = value; + // @@protoc_insertion_point(field_set:milvus.proto.milvus.LoadPartitionsRequest.replica_number) +} + // ------------------------------------------------------------------- // ReleasePartitionsRequest @@ -26343,6 +26377,57 @@ LoadBalanceRequest::mutable_sealed_segmentids() { return &sealed_segmentids_; } +// string collectionName = 5; +inline void LoadBalanceRequest::clear_collectionname() { + collectionname_.ClearToEmptyNoArena(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited()); +} +inline const std::string& LoadBalanceRequest::collectionname() const { + // @@protoc_insertion_point(field_get:milvus.proto.milvus.LoadBalanceRequest.collectionName) + return collectionname_.GetNoArena(); +} +inline void LoadBalanceRequest::set_collectionname(const std::string& value) { + + collectionname_.SetNoArena(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), value); + // @@protoc_insertion_point(field_set:milvus.proto.milvus.LoadBalanceRequest.collectionName) +} +inline void LoadBalanceRequest::set_collectionname(std::string&& value) { + + collectionname_.SetNoArena( + &::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), ::std::move(value)); + // @@protoc_insertion_point(field_set_rvalue:milvus.proto.milvus.LoadBalanceRequest.collectionName) +} +inline void LoadBalanceRequest::set_collectionname(const char* value) { + GOOGLE_DCHECK(value != nullptr); + + collectionname_.SetNoArena(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), ::std::string(value)); + // @@protoc_insertion_point(field_set_char:milvus.proto.milvus.LoadBalanceRequest.collectionName) +} +inline void LoadBalanceRequest::set_collectionname(const char* value, size_t size) { + + collectionname_.SetNoArena(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), + ::std::string(reinterpret_cast(value), size)); + // @@protoc_insertion_point(field_set_pointer:milvus.proto.milvus.LoadBalanceRequest.collectionName) +} +inline std::string* LoadBalanceRequest::mutable_collectionname() { + + // @@protoc_insertion_point(field_mutable:milvus.proto.milvus.LoadBalanceRequest.collectionName) + return collectionname_.MutableNoArena(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited()); +} +inline std::string* LoadBalanceRequest::release_collectionname() { + // @@protoc_insertion_point(field_release:milvus.proto.milvus.LoadBalanceRequest.collectionName) + + return collectionname_.ReleaseNoArena(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited()); +} +inline void LoadBalanceRequest::set_allocated_collectionname(std::string* collectionname) { + if (collectionname != nullptr) { + + } else { + + } + collectionname_.SetAllocatedNoArena(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), collectionname); + // @@protoc_insertion_point(field_set_allocated:milvus.proto.milvus.LoadBalanceRequest.collectionName) +} + // ------------------------------------------------------------------- // ManualCompactionRequest diff --git a/internal/datacoord/services.go b/internal/datacoord/services.go index b7ebf34e39..38f666c733 100644 --- a/internal/datacoord/services.go +++ b/internal/datacoord/services.go @@ -528,6 +528,7 @@ func (s *Server) GetRecoveryInfo(ctx context.Context, req *datapb.GetRecoveryInf segment2Binlogs := make(map[UniqueID][]*datapb.FieldBinlog) segment2StatsBinlogs := make(map[UniqueID][]*datapb.FieldBinlog) segment2DeltaBinlogs := make(map[UniqueID][]*datapb.FieldBinlog) + segment2InsertChannel := make(map[UniqueID]string) segmentsNumOfRows := make(map[UniqueID]int64) flushedIDs := make(map[int64]struct{}) @@ -542,6 +543,7 @@ func (s *Server) GetRecoveryInfo(ctx context.Context, req *datapb.GetRecoveryInf if segment.State != commonpb.SegmentState_Flushed && segment.State != commonpb.SegmentState_Flushing { continue } + segment2InsertChannel[segment.ID] = segment.InsertChannel binlogs := segment.GetBinlogs() if len(binlogs) == 0 { @@ -590,11 +592,12 @@ func (s *Server) GetRecoveryInfo(ctx context.Context, req *datapb.GetRecoveryInf binlogs := make([]*datapb.SegmentBinlogs, 0, len(segment2Binlogs)) for segmentID := range flushedIDs { sbl := &datapb.SegmentBinlogs{ - SegmentID: segmentID, - NumOfRows: segmentsNumOfRows[segmentID], - FieldBinlogs: segment2Binlogs[segmentID], - Statslogs: segment2StatsBinlogs[segmentID], - Deltalogs: segment2DeltaBinlogs[segmentID], + SegmentID: segmentID, + NumOfRows: segmentsNumOfRows[segmentID], + FieldBinlogs: segment2Binlogs[segmentID], + Statslogs: segment2StatsBinlogs[segmentID], + Deltalogs: segment2DeltaBinlogs[segmentID], + InsertChannel: segment2InsertChannel[segmentID], } binlogs = append(binlogs, sbl) } diff --git a/internal/distributed/querynode/client/client_test.go b/internal/distributed/querynode/client/client_test.go index 57a09cd88d..f73a7397a6 100644 --- a/internal/distributed/querynode/client/client_test.go +++ b/internal/distributed/querynode/client/client_test.go @@ -21,7 +21,6 @@ import ( "errors" "testing" - "github.com/milvus-io/milvus/internal/proxy" "github.com/milvus-io/milvus/internal/util/mock" "github.com/milvus-io/milvus/internal/util/typeutil" "github.com/stretchr/testify/assert" @@ -29,7 +28,7 @@ import ( ) func Test_NewClient(t *testing.T) { - proxy.Params.InitOnce() + ClientParams.InitOnce(typeutil.QueryNodeRole) ctx := context.Background() client, err := NewClient(ctx, "") diff --git a/internal/proto/common.proto b/internal/proto/common.proto index a572fdba29..cd3fdce8cb 100644 --- a/internal/proto/common.proto +++ b/internal/proto/common.proto @@ -149,6 +149,7 @@ enum MsgType { RemoveQueryChannels = 511; SealedSegmentsChangeInfo = 512; WatchDeltaChannels = 513; + GetShardLeaders = 514; /* DATA SERVICE */ SegmentInfo = 600; @@ -221,4 +222,4 @@ enum ImportState { ImportParsed = 4; ImportPersisted = 5; ImportCompleted = 6; -} \ No newline at end of file +} diff --git a/internal/proto/commonpb/common.pb.go b/internal/proto/commonpb/common.pb.go index a2821708a7..6a57a3721e 100644 --- a/internal/proto/commonpb/common.pb.go +++ b/internal/proto/commonpb/common.pb.go @@ -274,6 +274,7 @@ const ( MsgType_RemoveQueryChannels MsgType = 511 MsgType_SealedSegmentsChangeInfo MsgType = 512 MsgType_WatchDeltaChannels MsgType = 513 + MsgType_GetShardLeaders MsgType = 514 // DATA SERVICE MsgType_SegmentInfo MsgType = 600 MsgType_SystemInfo MsgType = 601 @@ -344,6 +345,7 @@ var MsgType_name = map[int32]string{ 511: "RemoveQueryChannels", 512: "SealedSegmentsChangeInfo", 513: "WatchDeltaChannels", + 514: "GetShardLeaders", 600: "SegmentInfo", 601: "SystemInfo", 602: "GetRecoveryInfo", @@ -411,6 +413,7 @@ var MsgType_value = map[string]int32{ "RemoveQueryChannels": 511, "SealedSegmentsChangeInfo": 512, "WatchDeltaChannels": 513, + "GetShardLeaders": 514, "SegmentInfo": 600, "SystemInfo": 601, "GetRecoveryInfo": 602, @@ -966,113 +969,114 @@ func init() { func init() { proto.RegisterFile("common.proto", fileDescriptor_555bd8c177793206) } var fileDescriptor_555bd8c177793206 = []byte{ - // 1727 bytes of a gzipped FileDescriptorProto + // 1736 bytes of a gzipped FileDescriptorProto 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xa4, 0x56, 0x4b, 0x73, 0x1c, 0x49, - 0xf1, 0x57, 0xcf, 0x8c, 0x34, 0x9e, 0x1c, 0x3d, 0xca, 0x25, 0x59, 0xd6, 0xda, 0xda, 0xfd, 0xfb, - 0xaf, 0x93, 0x43, 0x11, 0x6b, 0x03, 0x8e, 0x80, 0xd3, 0x1e, 0xa4, 0x69, 0x49, 0x9e, 0xb0, 0x24, - 0x8b, 0x19, 0xc9, 0xbb, 0xc1, 0x01, 0x47, 0xa9, 0x3b, 0x35, 0x2a, 0xdc, 0x5d, 0x35, 0x54, 0x55, - 0xcb, 0x1a, 0x4e, 0x0b, 0x9f, 0x00, 0xf6, 0xc2, 0x95, 0x0f, 0x00, 0x04, 0xcb, 0xfb, 0x23, 0xf0, - 0x3e, 0xf3, 0x86, 0x23, 0x1f, 0x80, 0xe7, 0x3e, 0xbc, 0x44, 0x56, 0xf7, 0x74, 0xb7, 0xed, 0xdd, - 0x13, 0xb7, 0xca, 0x5f, 0x66, 0xfe, 0x2a, 0x2b, 0x33, 0x2b, 0xab, 0x60, 0x3e, 0xd2, 0x69, 0xaa, - 0xd5, 0x9d, 0xb1, 0xd1, 0x4e, 0xf3, 0xe5, 0x54, 0x26, 0x17, 0x99, 0xcd, 0xa5, 0x3b, 0xb9, 0x6a, - 0xe3, 0x31, 0xcc, 0x0d, 0x9d, 0x70, 0x99, 0xe5, 0x6f, 0x00, 0xa0, 0x31, 0xda, 0x3c, 0x8e, 0x74, - 0x8c, 0x6b, 0xc1, 0xad, 0xe0, 0xf6, 0xe2, 0x67, 0x5e, 0xbb, 0xf3, 0x31, 0x3e, 0x77, 0x76, 0xc8, - 0xac, 0xa7, 0x63, 0x1c, 0x74, 0x70, 0xba, 0xe4, 0xab, 0x30, 0x67, 0x50, 0x58, 0xad, 0xd6, 0x1a, - 0xb7, 0x82, 0xdb, 0x9d, 0x41, 0x21, 0x6d, 0x7c, 0x16, 0xe6, 0x1f, 0xe0, 0xe4, 0x91, 0x48, 0x32, - 0x3c, 0x12, 0xd2, 0x70, 0x06, 0xcd, 0x27, 0x38, 0xf1, 0xfc, 0x9d, 0x01, 0x2d, 0xf9, 0x0a, 0xcc, - 0x5e, 0x90, 0xba, 0x70, 0xcc, 0x85, 0x8d, 0x7b, 0xd0, 0x7d, 0x80, 0x93, 0x50, 0x38, 0xf1, 0x09, - 0x6e, 0x1c, 0x5a, 0xb1, 0x70, 0xc2, 0x7b, 0xcd, 0x0f, 0xfc, 0x7a, 0x63, 0x1d, 0x5a, 0xdb, 0x89, - 0x3e, 0xad, 0x28, 0x03, 0xaf, 0x2c, 0x28, 0x5f, 0x87, 0xf6, 0x56, 0x1c, 0x1b, 0xb4, 0x96, 0x2f, - 0x42, 0x43, 0x8e, 0x0b, 0xb6, 0x86, 0x1c, 0x13, 0xd9, 0x58, 0x1b, 0xe7, 0xc9, 0x9a, 0x03, 0xbf, - 0xde, 0x78, 0x27, 0x80, 0xf6, 0x81, 0x1d, 0x6d, 0x0b, 0x8b, 0xfc, 0x73, 0x70, 0x25, 0xb5, 0xa3, - 0xc7, 0x6e, 0x32, 0x9e, 0xa6, 0x66, 0xfd, 0x63, 0x53, 0x73, 0x60, 0x47, 0xc7, 0x93, 0x31, 0x0e, - 0xda, 0x69, 0xbe, 0xa0, 0x48, 0x52, 0x3b, 0xea, 0x87, 0x05, 0x73, 0x2e, 0xf0, 0x75, 0xe8, 0x38, - 0x99, 0xa2, 0x75, 0x22, 0x1d, 0xaf, 0x35, 0x6f, 0x05, 0xb7, 0x5b, 0x83, 0x0a, 0xe0, 0x37, 0xe0, - 0x8a, 0xd5, 0x99, 0x89, 0xb0, 0x1f, 0xae, 0xb5, 0xbc, 0x5b, 0x29, 0x6f, 0xbc, 0x01, 0x9d, 0x03, - 0x3b, 0xba, 0x8f, 0x22, 0x46, 0xc3, 0x3f, 0x05, 0xad, 0x53, 0x61, 0xf3, 0x88, 0xba, 0x9f, 0x1c, - 0x11, 0x9d, 0x60, 0xe0, 0x2d, 0x37, 0xbe, 0x08, 0xf3, 0xe1, 0xc1, 0xfe, 0xff, 0xc0, 0x40, 0xa1, - 0xdb, 0x73, 0x61, 0xe2, 0x43, 0x91, 0x4e, 0x2b, 0x56, 0x01, 0x9b, 0xcf, 0x66, 0xa1, 0x53, 0xb6, - 0x07, 0xef, 0x42, 0x7b, 0x98, 0x45, 0x11, 0x5a, 0xcb, 0x66, 0xf8, 0x32, 0x2c, 0x9d, 0x28, 0xbc, - 0x1c, 0x63, 0xe4, 0x30, 0xf6, 0x36, 0x2c, 0xe0, 0x57, 0x61, 0xa1, 0xa7, 0x95, 0xc2, 0xc8, 0xed, - 0x0a, 0x99, 0x60, 0xcc, 0x1a, 0x7c, 0x05, 0xd8, 0x11, 0x9a, 0x54, 0x5a, 0x2b, 0xb5, 0x0a, 0x51, - 0x49, 0x8c, 0x59, 0x93, 0x5f, 0x87, 0xe5, 0x9e, 0x4e, 0x12, 0x8c, 0x9c, 0xd4, 0xea, 0x50, 0xbb, - 0x9d, 0x4b, 0x69, 0x9d, 0x65, 0x2d, 0xa2, 0xed, 0x27, 0x09, 0x8e, 0x44, 0xb2, 0x65, 0x46, 0x59, - 0x8a, 0xca, 0xb1, 0x59, 0xe2, 0x28, 0xc0, 0x50, 0xa6, 0xa8, 0x88, 0x89, 0xb5, 0x6b, 0x68, 0x5f, - 0xc5, 0x78, 0x49, 0xf5, 0x61, 0x57, 0xf8, 0x2b, 0x70, 0xad, 0x40, 0x6b, 0x1b, 0x88, 0x14, 0x59, - 0x87, 0x2f, 0x41, 0xb7, 0x50, 0x1d, 0x3f, 0x3c, 0x7a, 0xc0, 0xa0, 0xc6, 0x30, 0xd0, 0x4f, 0x07, - 0x18, 0x69, 0x13, 0xb3, 0x6e, 0x2d, 0x84, 0x47, 0x18, 0x39, 0x6d, 0xfa, 0x21, 0x9b, 0xa7, 0x80, - 0x0b, 0x70, 0x88, 0xc2, 0x44, 0xe7, 0x03, 0xb4, 0x59, 0xe2, 0xd8, 0x02, 0x67, 0x30, 0xbf, 0x2b, - 0x13, 0x3c, 0xd4, 0x6e, 0x57, 0x67, 0x2a, 0x66, 0x8b, 0x7c, 0x11, 0xe0, 0x00, 0x9d, 0x28, 0x32, - 0xb0, 0x44, 0xdb, 0xf6, 0x44, 0x74, 0x8e, 0x05, 0xc0, 0xf8, 0x2a, 0xf0, 0x9e, 0x50, 0x4a, 0xbb, - 0x9e, 0x41, 0xe1, 0x70, 0x57, 0x27, 0x31, 0x1a, 0x76, 0x95, 0xc2, 0x79, 0x0e, 0x97, 0x09, 0x32, - 0x5e, 0x59, 0x87, 0x98, 0x60, 0x69, 0xbd, 0x5c, 0x59, 0x17, 0x38, 0x59, 0xaf, 0x50, 0xf0, 0xdb, - 0x99, 0x4c, 0x62, 0x9f, 0x92, 0xbc, 0x2c, 0xd7, 0x28, 0xc6, 0x22, 0xf8, 0xc3, 0xfd, 0xfe, 0xf0, - 0x98, 0xad, 0xf2, 0x6b, 0x70, 0xb5, 0x40, 0x0e, 0xd0, 0x19, 0x19, 0xf9, 0xe4, 0x5d, 0xa7, 0x50, - 0x1f, 0x66, 0xee, 0xe1, 0xd9, 0x01, 0xa6, 0xda, 0x4c, 0xd8, 0x1a, 0x15, 0xd4, 0x33, 0x4d, 0x4b, - 0xc4, 0x5e, 0xa1, 0x1d, 0x76, 0xd2, 0xb1, 0x9b, 0x54, 0xe9, 0x65, 0x37, 0xf8, 0x4d, 0xb8, 0x7e, - 0x32, 0x8e, 0x85, 0xc3, 0x7e, 0x4a, 0x97, 0xed, 0x58, 0xd8, 0x27, 0x74, 0xdc, 0xcc, 0x20, 0xbb, - 0xc9, 0x6f, 0xc0, 0xea, 0xf3, 0xb5, 0x28, 0x93, 0xb5, 0x4e, 0x8e, 0xf9, 0x69, 0x7b, 0x06, 0x63, - 0x54, 0x4e, 0x8a, 0x64, 0xea, 0xf8, 0x6a, 0xc5, 0xfa, 0xb2, 0xf2, 0x35, 0x52, 0xe6, 0x27, 0x7f, - 0x59, 0xf9, 0x7f, 0x7c, 0x0d, 0x56, 0xf6, 0xd0, 0xbd, 0xac, 0xb9, 0x45, 0x9a, 0x7d, 0x69, 0xbd, - 0xea, 0xc4, 0xa2, 0xb1, 0x53, 0xcd, 0xff, 0x73, 0x0e, 0x0b, 0x61, 0x38, 0xc0, 0x2f, 0x67, 0x68, - 0xdd, 0x40, 0x44, 0xc8, 0xfe, 0xd6, 0xde, 0x7c, 0x0b, 0xc0, 0x9f, 0x9f, 0x86, 0x2a, 0x72, 0x0e, - 0x8b, 0x95, 0x74, 0xa8, 0x15, 0xb2, 0x19, 0x3e, 0x0f, 0x57, 0x4e, 0x94, 0xb4, 0x36, 0xc3, 0x98, - 0x05, 0x54, 0xfb, 0xbe, 0x3a, 0x32, 0x7a, 0x44, 0x63, 0x89, 0x35, 0x48, 0xbb, 0x2b, 0x95, 0xb4, - 0xe7, 0xbe, 0xeb, 0x01, 0xe6, 0x8a, 0x26, 0x68, 0x6d, 0x7e, 0x2d, 0x80, 0xf9, 0x21, 0x8e, 0xa8, - 0xc3, 0x73, 0xf2, 0x15, 0x60, 0x75, 0xb9, 0xa2, 0x2f, 0x73, 0x1f, 0xd0, 0x0d, 0xdc, 0x33, 0xfa, - 0xa9, 0x54, 0x23, 0xd6, 0x20, 0xb6, 0x21, 0x8a, 0xc4, 0x33, 0x77, 0xa1, 0xbd, 0x9b, 0x64, 0x7e, - 0x9b, 0x96, 0xdf, 0x94, 0x04, 0x32, 0x9b, 0x25, 0x55, 0x68, 0xf4, 0x78, 0x8c, 0x31, 0x9b, 0xe3, - 0x0b, 0xd0, 0xc9, 0x2b, 0x44, 0xba, 0xf6, 0xe6, 0xbb, 0xe0, 0x67, 0xa2, 0x1f, 0x6d, 0x0b, 0xd0, - 0x39, 0x51, 0x31, 0x9e, 0x49, 0x85, 0x31, 0x9b, 0xf1, 0xed, 0x95, 0x17, 0xa6, 0xaa, 0x73, 0x4c, - 0x19, 0x20, 0xb2, 0x1a, 0x86, 0xd4, 0x23, 0xf7, 0x85, 0xad, 0x41, 0x67, 0xd4, 0xb3, 0x21, 0xda, - 0xc8, 0xc8, 0xd3, 0xba, 0xfb, 0x88, 0x7a, 0x67, 0x78, 0xae, 0x9f, 0x56, 0x98, 0x65, 0xe7, 0xb4, - 0xd3, 0x1e, 0xba, 0xe1, 0xc4, 0x3a, 0x4c, 0x7b, 0x5a, 0x9d, 0xc9, 0x91, 0x65, 0x92, 0x76, 0xda, - 0xd7, 0x22, 0xae, 0xb9, 0x7f, 0x89, 0xba, 0x76, 0x80, 0x09, 0x0a, 0x5b, 0x67, 0x7d, 0xe2, 0x2f, - 0x98, 0x0f, 0x75, 0x2b, 0x91, 0xc2, 0xb2, 0x84, 0x8e, 0x42, 0x51, 0xe6, 0x62, 0x4a, 0x45, 0xd9, + 0x11, 0x56, 0xcf, 0x8c, 0x34, 0x9e, 0x1c, 0x3d, 0xca, 0x25, 0x59, 0xd6, 0x7a, 0xb5, 0x8b, 0xd1, + 0xc9, 0xa1, 0x88, 0xb5, 0x01, 0x47, 0xc0, 0x69, 0x0f, 0xd2, 0xb4, 0x24, 0x4f, 0x58, 0x92, 0xc5, + 0x8c, 0xe4, 0xdd, 0xe0, 0x80, 0xa3, 0xd4, 0x9d, 0x1a, 0x15, 0xee, 0xae, 0x1a, 0xaa, 0xaa, 0x65, + 0x0d, 0xa7, 0x65, 0x7f, 0x01, 0xec, 0x85, 0x2b, 0x3f, 0x00, 0x08, 0xde, 0xf0, 0x0f, 0x78, 0x73, + 0xe6, 0x0d, 0x47, 0x7e, 0x00, 0xcf, 0x7d, 0x78, 0x89, 0xac, 0xee, 0xe9, 0x6e, 0xdb, 0xbb, 0x27, + 0x6e, 0x95, 0x5f, 0x66, 0x7d, 0x95, 0x95, 0x99, 0x95, 0x59, 0x30, 0x1f, 0xe9, 0x34, 0xd5, 0xea, + 0xf6, 0xd8, 0x68, 0xa7, 0xf9, 0x72, 0x2a, 0x93, 0x8b, 0xcc, 0xe6, 0xd2, 0xed, 0x5c, 0xb5, 0xf1, + 0x08, 0xe6, 0x86, 0x4e, 0xb8, 0xcc, 0xf2, 0xd7, 0x01, 0xd0, 0x18, 0x6d, 0x1e, 0x45, 0x3a, 0xc6, + 0xb5, 0xe0, 0x66, 0x70, 0x6b, 0xf1, 0x33, 0xaf, 0xde, 0xfe, 0x88, 0x3d, 0xb7, 0x77, 0xc8, 0xac, + 0xa7, 0x63, 0x1c, 0x74, 0x70, 0xba, 0xe4, 0xab, 0x30, 0x67, 0x50, 0x58, 0xad, 0xd6, 0x1a, 0x37, + 0x83, 0x5b, 0x9d, 0x41, 0x21, 0x6d, 0x7c, 0x16, 0xe6, 0xef, 0xe3, 0xe4, 0xa1, 0x48, 0x32, 0x3c, + 0x12, 0xd2, 0x70, 0x06, 0xcd, 0xc7, 0x38, 0xf1, 0xfc, 0x9d, 0x01, 0x2d, 0xf9, 0x0a, 0xcc, 0x5e, + 0x90, 0xba, 0xd8, 0x98, 0x0b, 0x1b, 0x77, 0xa1, 0x7b, 0x1f, 0x27, 0xa1, 0x70, 0xe2, 0x63, 0xb6, + 0x71, 0x68, 0xc5, 0xc2, 0x09, 0xbf, 0x6b, 0x7e, 0xe0, 0xd7, 0x1b, 0xeb, 0xd0, 0xda, 0x4e, 0xf4, + 0x69, 0x45, 0x19, 0x78, 0x65, 0x41, 0xf9, 0x1a, 0xb4, 0xb7, 0xe2, 0xd8, 0xa0, 0xb5, 0x7c, 0x11, + 0x1a, 0x72, 0x5c, 0xb0, 0x35, 0xe4, 0x98, 0xc8, 0xc6, 0xda, 0x38, 0x4f, 0xd6, 0x1c, 0xf8, 0xf5, + 0xc6, 0x3b, 0x01, 0xb4, 0x0f, 0xec, 0x68, 0x5b, 0x58, 0xe4, 0x9f, 0x83, 0x2b, 0xa9, 0x1d, 0x3d, + 0x72, 0x93, 0xf1, 0x34, 0x34, 0xeb, 0x1f, 0x19, 0x9a, 0x03, 0x3b, 0x3a, 0x9e, 0x8c, 0x71, 0xd0, + 0x4e, 0xf3, 0x05, 0x79, 0x92, 0xda, 0x51, 0x3f, 0x2c, 0x98, 0x73, 0x81, 0xaf, 0x43, 0xc7, 0xc9, + 0x14, 0xad, 0x13, 0xe9, 0x78, 0xad, 0x79, 0x33, 0xb8, 0xd5, 0x1a, 0x54, 0x00, 0xbf, 0x01, 0x57, + 0xac, 0xce, 0x4c, 0x84, 0xfd, 0x70, 0xad, 0xe5, 0xb7, 0x95, 0xf2, 0xc6, 0xeb, 0xd0, 0x39, 0xb0, + 0xa3, 0x7b, 0x28, 0x62, 0x34, 0xfc, 0x53, 0xd0, 0x3a, 0x15, 0x36, 0xf7, 0xa8, 0xfb, 0xf1, 0x1e, + 0xd1, 0x0d, 0x06, 0xde, 0x72, 0xe3, 0x8b, 0x30, 0x1f, 0x1e, 0xec, 0xff, 0x1f, 0x0c, 0xe4, 0xba, + 0x3d, 0x17, 0x26, 0x3e, 0x14, 0xe9, 0x34, 0x63, 0x15, 0xb0, 0xf9, 0x74, 0x16, 0x3a, 0x65, 0x79, + 0xf0, 0x2e, 0xb4, 0x87, 0x59, 0x14, 0xa1, 0xb5, 0x6c, 0x86, 0x2f, 0xc3, 0xd2, 0x89, 0xc2, 0xcb, + 0x31, 0x46, 0x0e, 0x63, 0x6f, 0xc3, 0x02, 0x7e, 0x15, 0x16, 0x7a, 0x5a, 0x29, 0x8c, 0xdc, 0xae, + 0x90, 0x09, 0xc6, 0xac, 0xc1, 0x57, 0x80, 0x1d, 0xa1, 0x49, 0xa5, 0xb5, 0x52, 0xab, 0x10, 0x95, + 0xc4, 0x98, 0x35, 0xf9, 0x75, 0x58, 0xee, 0xe9, 0x24, 0xc1, 0xc8, 0x49, 0xad, 0x0e, 0xb5, 0xdb, + 0xb9, 0x94, 0xd6, 0x59, 0xd6, 0x22, 0xda, 0x7e, 0x92, 0xe0, 0x48, 0x24, 0x5b, 0x66, 0x94, 0xa5, + 0xa8, 0x1c, 0x9b, 0x25, 0x8e, 0x02, 0x0c, 0x65, 0x8a, 0x8a, 0x98, 0x58, 0xbb, 0x86, 0xf6, 0x55, + 0x8c, 0x97, 0x94, 0x1f, 0x76, 0x85, 0xbf, 0x04, 0xd7, 0x0a, 0xb4, 0x76, 0x80, 0x48, 0x91, 0x75, + 0xf8, 0x12, 0x74, 0x0b, 0xd5, 0xf1, 0x83, 0xa3, 0xfb, 0x0c, 0x6a, 0x0c, 0x03, 0xfd, 0x64, 0x80, + 0x91, 0x36, 0x31, 0xeb, 0xd6, 0x5c, 0x78, 0x88, 0x91, 0xd3, 0xa6, 0x1f, 0xb2, 0x79, 0x72, 0xb8, + 0x00, 0x87, 0x28, 0x4c, 0x74, 0x3e, 0x40, 0x9b, 0x25, 0x8e, 0x2d, 0x70, 0x06, 0xf3, 0xbb, 0x32, + 0xc1, 0x43, 0xed, 0x76, 0x75, 0xa6, 0x62, 0xb6, 0xc8, 0x17, 0x01, 0x0e, 0xd0, 0x89, 0x22, 0x02, + 0x4b, 0x74, 0x6c, 0x4f, 0x44, 0xe7, 0x58, 0x00, 0x8c, 0xaf, 0x02, 0xef, 0x09, 0xa5, 0xb4, 0xeb, + 0x19, 0x14, 0x0e, 0x77, 0x75, 0x12, 0xa3, 0x61, 0x57, 0xc9, 0x9d, 0x67, 0x70, 0x99, 0x20, 0xe3, + 0x95, 0x75, 0x88, 0x09, 0x96, 0xd6, 0xcb, 0x95, 0x75, 0x81, 0x93, 0xf5, 0x0a, 0x39, 0xbf, 0x9d, + 0xc9, 0x24, 0xf6, 0x21, 0xc9, 0xd3, 0x72, 0x8d, 0x7c, 0x2c, 0x9c, 0x3f, 0xdc, 0xef, 0x0f, 0x8f, + 0xd9, 0x2a, 0xbf, 0x06, 0x57, 0x0b, 0xe4, 0x00, 0x9d, 0x91, 0x91, 0x0f, 0xde, 0x75, 0x72, 0xf5, + 0x41, 0xe6, 0x1e, 0x9c, 0x1d, 0x60, 0xaa, 0xcd, 0x84, 0xad, 0x51, 0x42, 0x3d, 0xd3, 0x34, 0x45, + 0xec, 0x25, 0x3a, 0x61, 0x27, 0x1d, 0xbb, 0x49, 0x15, 0x5e, 0x76, 0x83, 0xbf, 0x0c, 0xd7, 0x4f, + 0xc6, 0xb1, 0x70, 0xd8, 0x4f, 0xe9, 0xb1, 0x1d, 0x0b, 0xfb, 0x98, 0xae, 0x9b, 0x19, 0x64, 0x2f, + 0xf3, 0x1b, 0xb0, 0xfa, 0x6c, 0x2e, 0xca, 0x60, 0xad, 0xd3, 0xc6, 0xfc, 0xb6, 0x3d, 0x83, 0x31, + 0x2a, 0x27, 0x45, 0x32, 0xdd, 0xf8, 0x4a, 0xc5, 0xfa, 0xa2, 0xf2, 0x55, 0x52, 0xe6, 0x37, 0x7f, + 0x51, 0xf9, 0x09, 0xbe, 0x06, 0x2b, 0x7b, 0xe8, 0x5e, 0xd4, 0xdc, 0x24, 0xcd, 0xbe, 0xb4, 0x5e, + 0x75, 0x62, 0xd1, 0xd8, 0xa9, 0xe6, 0x93, 0x9c, 0xc3, 0x42, 0x18, 0x0e, 0xf0, 0xcb, 0x19, 0x5a, + 0x37, 0x10, 0x11, 0xb2, 0xbf, 0xb7, 0x37, 0xdf, 0x04, 0xf0, 0xf7, 0xa7, 0xa6, 0x8a, 0x9c, 0xc3, + 0x62, 0x25, 0x1d, 0x6a, 0x85, 0x6c, 0x86, 0xcf, 0xc3, 0x95, 0x13, 0x25, 0xad, 0xcd, 0x30, 0x66, + 0x01, 0xe5, 0xbe, 0xaf, 0x8e, 0x8c, 0x1e, 0x51, 0x5b, 0x62, 0x0d, 0xd2, 0xee, 0x4a, 0x25, 0xed, + 0xb9, 0xaf, 0x7a, 0x80, 0xb9, 0xa2, 0x08, 0x5a, 0x9b, 0x6f, 0x07, 0x30, 0x3f, 0xc4, 0x11, 0x55, + 0x78, 0x4e, 0xbe, 0x02, 0xac, 0x2e, 0x57, 0xf4, 0x65, 0xec, 0x03, 0x7a, 0x81, 0x7b, 0x46, 0x3f, + 0x91, 0x6a, 0xc4, 0x1a, 0xc4, 0x36, 0x44, 0x91, 0x78, 0xe6, 0x2e, 0xb4, 0x77, 0x93, 0xcc, 0x1f, + 0xd3, 0xf2, 0x87, 0x92, 0x40, 0x66, 0xb3, 0xa4, 0x0a, 0x8d, 0x1e, 0x8f, 0x31, 0x66, 0x73, 0x7c, + 0x01, 0x3a, 0x79, 0x86, 0x48, 0xd7, 0xde, 0xfc, 0x19, 0xf8, 0x9e, 0xe8, 0x5b, 0xdb, 0x02, 0x74, + 0x4e, 0x54, 0x8c, 0x67, 0x52, 0x61, 0xcc, 0x66, 0x7c, 0x79, 0xe5, 0x89, 0xa9, 0xf2, 0x1c, 0x53, + 0x04, 0x88, 0xac, 0x86, 0x21, 0xd5, 0xc8, 0x3d, 0x61, 0x6b, 0xd0, 0x19, 0xd5, 0x6c, 0x88, 0x36, + 0x32, 0xf2, 0xb4, 0xbe, 0x7d, 0x44, 0xb5, 0x33, 0x3c, 0xd7, 0x4f, 0x2a, 0xcc, 0xb2, 0x73, 0x3a, + 0x69, 0x0f, 0xdd, 0x70, 0x62, 0x1d, 0xa6, 0x3d, 0xad, 0xce, 0xe4, 0xc8, 0x32, 0x49, 0x27, 0xed, + 0x6b, 0x11, 0xd7, 0xb6, 0x7f, 0x89, 0xaa, 0x76, 0x80, 0x09, 0x0a, 0x5b, 0x67, 0x7d, 0xec, 0x1f, + 0x98, 0x77, 0x75, 0x2b, 0x91, 0xc2, 0xb2, 0x84, 0xae, 0x42, 0x5e, 0xe6, 0x62, 0x4a, 0x49, 0xd9, 0x4a, 0x1c, 0x9a, 0x5c, 0x56, 0x7c, 0x05, 0x96, 0x72, 0xfb, 0x23, 0x61, 0x9c, 0xf4, 0x24, 0x3f, - 0x0b, 0x7c, 0xf9, 0x8d, 0x1e, 0x57, 0xd8, 0xcf, 0x69, 0x9e, 0xcd, 0xdf, 0x17, 0xb6, 0x82, 0x7e, - 0x11, 0xf0, 0x55, 0xb8, 0x3a, 0x3d, 0x5a, 0x85, 0xff, 0x32, 0xe0, 0xcb, 0xb0, 0x48, 0x47, 0x2b, - 0x31, 0xcb, 0x7e, 0xe5, 0x41, 0x3a, 0x44, 0x0d, 0xfc, 0xb5, 0x67, 0x28, 0x4e, 0x51, 0xc3, 0x7f, - 0xe3, 0x37, 0x23, 0x86, 0xa2, 0x09, 0x2c, 0x7b, 0x2f, 0xa0, 0x48, 0xa7, 0x9b, 0x15, 0x30, 0x7b, - 0xdf, 0x1b, 0x12, 0x6b, 0x69, 0xf8, 0x81, 0x37, 0x2c, 0x38, 0x4b, 0xf4, 0x43, 0x8f, 0xde, 0x17, - 0x2a, 0xd6, 0x67, 0x67, 0x25, 0xfa, 0x2c, 0xe0, 0x6b, 0xb0, 0x4c, 0xee, 0xdb, 0x22, 0x11, 0x2a, - 0xaa, 0xec, 0x3f, 0x0a, 0xf8, 0x35, 0x60, 0x2f, 0x6c, 0x67, 0xd9, 0xdb, 0x0d, 0xce, 0xa6, 0xf9, - 0xf5, 0xcd, 0xcf, 0xbe, 0xdd, 0xf0, 0xb9, 0x2a, 0x0c, 0x73, 0xec, 0x3b, 0x0d, 0xbe, 0x98, 0x27, - 0x3d, 0x97, 0xbf, 0xdb, 0xe0, 0x5d, 0x98, 0xeb, 0x2b, 0x8b, 0xc6, 0xb1, 0xaf, 0x53, 0x7f, 0xce, - 0xe5, 0x97, 0x95, 0x7d, 0x83, 0xae, 0xc1, 0xac, 0xef, 0x4f, 0xf6, 0x8e, 0x57, 0xe4, 0x03, 0x95, - 0xfd, 0xbd, 0xe9, 0x33, 0x50, 0x9f, 0xae, 0xff, 0x68, 0xd2, 0x4e, 0x7b, 0xe8, 0xaa, 0x5b, 0xc7, - 0xfe, 0xd9, 0xe4, 0x37, 0xe0, 0xda, 0x14, 0xf3, 0xb3, 0xae, 0xbc, 0x6f, 0xff, 0x6a, 0xf2, 0x75, - 0xb8, 0x4e, 0x17, 0xbf, 0x6c, 0x0f, 0x72, 0x92, 0xd6, 0xc9, 0xc8, 0xb2, 0x7f, 0x37, 0xf9, 0x4d, - 0x58, 0xdd, 0x43, 0x57, 0xa6, 0xbd, 0xa6, 0xfc, 0x4f, 0x93, 0x2f, 0xc0, 0x95, 0x01, 0x0d, 0x43, - 0xbc, 0x40, 0xf6, 0x5e, 0x93, 0x6a, 0x37, 0x15, 0x8b, 0x70, 0xde, 0x6f, 0x52, 0x46, 0xdf, 0x14, - 0x2e, 0x3a, 0x0f, 0xd3, 0xde, 0xb9, 0x50, 0x0a, 0x13, 0xcb, 0x3e, 0x68, 0x52, 0xde, 0x06, 0x98, - 0xea, 0x0b, 0xac, 0xc1, 0x1f, 0xd2, 0x23, 0xc7, 0xbd, 0xf1, 0xe7, 0x33, 0x34, 0x93, 0x52, 0xf1, - 0xac, 0x49, 0x15, 0xc8, 0xed, 0x9f, 0xd7, 0x7c, 0xd4, 0xe4, 0xaf, 0xc2, 0x5a, 0x7e, 0xa7, 0xa7, - 0xf9, 0x27, 0xe5, 0x08, 0xfb, 0xea, 0x4c, 0xb3, 0xb7, 0x5b, 0x25, 0x63, 0x88, 0x89, 0x13, 0xa5, - 0xdf, 0x57, 0x5b, 0x54, 0xa2, 0xc2, 0xc3, 0x9b, 0xfe, 0xb6, 0xc5, 0x97, 0x00, 0xf2, 0x2b, 0xe5, - 0x81, 0xdf, 0xb5, 0x28, 0xf4, 0x3d, 0x74, 0xf4, 0xca, 0x5d, 0xa0, 0x99, 0x78, 0xf4, 0xf7, 0x53, - 0xb4, 0x3e, 0x78, 0xd8, 0x1f, 0x5a, 0x94, 0x8a, 0x63, 0x99, 0xe2, 0xb1, 0x8c, 0x9e, 0xb0, 0xef, - 0x75, 0x28, 0x15, 0x3e, 0xd2, 0x43, 0x1d, 0x23, 0xd9, 0x58, 0xf6, 0x6e, 0x87, 0xea, 0x4d, 0x6d, - 0x94, 0xd7, 0xfb, 0xfb, 0x5e, 0x2e, 0x86, 0x67, 0x3f, 0x64, 0x3f, 0xa0, 0xd7, 0x16, 0x0a, 0xf9, - 0x78, 0xf8, 0x90, 0xfd, 0xb0, 0x43, 0x5b, 0x6d, 0x25, 0x89, 0x8e, 0x84, 0x2b, 0x9b, 0xf9, 0x47, - 0x1d, 0xba, 0x0d, 0xb5, 0xdd, 0x8b, 0x6a, 0xfc, 0xb8, 0x43, 0x39, 0x2d, 0x70, 0xdf, 0x2b, 0x21, - 0x8d, 0xc3, 0x9f, 0x78, 0x56, 0xfa, 0x44, 0x52, 0x24, 0xc7, 0x8e, 0xfd, 0xd4, 0xdb, 0xbd, 0xf8, - 0x80, 0xb0, 0x3f, 0x76, 0x8b, 0xbe, 0xa9, 0x61, 0x7f, 0xea, 0xe6, 0xed, 0xfd, 0xfc, 0x8b, 0xc1, - 0xfe, 0xec, 0xe1, 0x17, 0x5f, 0x19, 0xf6, 0x97, 0x2e, 0x05, 0x56, 0x7f, 0x28, 0x94, 0x48, 0xd1, - 0xb2, 0xbf, 0x76, 0x37, 0x37, 0xa0, 0x1d, 0xda, 0xc4, 0x8f, 0xcc, 0x36, 0x34, 0x43, 0x9b, 0xb0, - 0x19, 0x9a, 0x30, 0xdb, 0x5a, 0x27, 0x3b, 0x97, 0x63, 0xf3, 0xe8, 0xd3, 0x2c, 0xd8, 0xdc, 0x86, - 0xa5, 0x9e, 0x4e, 0xc7, 0xa2, 0x6c, 0x41, 0x3f, 0x25, 0xf3, 0xf1, 0x8a, 0x71, 0x9e, 0xe6, 0x19, - 0x1a, 0x53, 0x3b, 0x97, 0x18, 0x65, 0x7e, 0x18, 0x07, 0x24, 0x92, 0x13, 0x05, 0x18, 0xb3, 0xc6, - 0xe6, 0x5b, 0xc0, 0x7a, 0x5a, 0x59, 0x69, 0x1d, 0xaa, 0x68, 0xb2, 0x8f, 0x17, 0x98, 0xf8, 0x91, - 0xef, 0x8c, 0x56, 0x23, 0x36, 0xe3, 0x7f, 0x63, 0xe8, 0x7f, 0x55, 0xf9, 0xc3, 0xb0, 0x4d, 0x2f, - 0xaa, 0xff, 0x72, 0x2d, 0x02, 0xec, 0x5c, 0xa0, 0x72, 0x99, 0x48, 0x92, 0x09, 0x6b, 0x92, 0xdc, - 0xcb, 0xac, 0xd3, 0xa9, 0xfc, 0x8a, 0x7f, 0x7a, 0xbe, 0x19, 0x40, 0x37, 0x7f, 0x05, 0xca, 0xd0, - 0x72, 0xf1, 0x08, 0x55, 0x2c, 0x3d, 0x39, 0xfd, 0x18, 0x3c, 0x54, 0xbc, 0x57, 0x41, 0x65, 0x34, - 0x74, 0xc2, 0xb8, 0xe9, 0xd7, 0x2e, 0x87, 0x42, 0xfd, 0x54, 0x25, 0x5a, 0xc4, 0xfe, 0x29, 0x2a, - 0x5d, 0x8f, 0x84, 0xb1, 0xfe, 0x3d, 0xa2, 0x0f, 0x55, 0xc1, 0x6f, 0xfc, 0x79, 0x62, 0x36, 0x5b, - 0x81, 0xd5, 0x99, 0xe7, 0xb6, 0xdf, 0x84, 0x45, 0xa9, 0xa7, 0xbf, 0xd6, 0x91, 0x19, 0x47, 0xdb, - 0xdd, 0x9e, 0xff, 0xb5, 0x1e, 0xd1, 0x0f, 0xf6, 0x28, 0xf8, 0xc2, 0xbd, 0x91, 0x74, 0xe7, 0xd9, - 0x29, 0xfd, 0x65, 0xef, 0xe6, 0x66, 0xaf, 0x4b, 0x5d, 0xac, 0xee, 0x4a, 0xe5, 0xa8, 0x4e, 0xc9, - 0x5d, 0xff, 0xdf, 0xbd, 0x9b, 0xff, 0x77, 0xc7, 0xa7, 0xdf, 0x0a, 0x82, 0xd3, 0x39, 0x0f, 0xdd, - 0xfb, 0x6f, 0x00, 0x00, 0x00, 0xff, 0xff, 0xdc, 0x25, 0x53, 0x30, 0x43, 0x0d, 0x00, 0x00, + 0x0f, 0x7c, 0xfa, 0x8d, 0x1e, 0x57, 0xd8, 0x2f, 0xa8, 0x9f, 0xcd, 0xdf, 0x13, 0xb6, 0x82, 0x7e, + 0x19, 0xf0, 0x55, 0xb8, 0x3a, 0xbd, 0x5a, 0x85, 0xff, 0x2a, 0xe0, 0xcb, 0xb0, 0x48, 0x57, 0x2b, + 0x31, 0xcb, 0x7e, 0xed, 0x41, 0xba, 0x44, 0x0d, 0xfc, 0x8d, 0x67, 0x28, 0x6e, 0x51, 0xc3, 0x7f, + 0xeb, 0x0f, 0x23, 0x86, 0xa2, 0x08, 0x2c, 0x7b, 0x37, 0x20, 0x4f, 0xa7, 0x87, 0x15, 0x30, 0x7b, + 0xcf, 0x1b, 0x12, 0x6b, 0x69, 0xf8, 0xbe, 0x37, 0x2c, 0x38, 0x4b, 0xf4, 0x03, 0x8f, 0xde, 0x13, + 0x2a, 0xd6, 0x67, 0x67, 0x25, 0xfa, 0x34, 0xe0, 0x6b, 0xb0, 0x4c, 0xdb, 0xb7, 0x45, 0x22, 0x54, + 0x54, 0xd9, 0x7f, 0x18, 0xf0, 0x6b, 0xc0, 0x9e, 0x3b, 0xce, 0xb2, 0xb7, 0x1a, 0x9c, 0x4d, 0xe3, + 0xeb, 0x8b, 0x9f, 0x7d, 0xab, 0xe1, 0x63, 0x55, 0x18, 0xe6, 0xd8, 0xb7, 0x1b, 0x7c, 0x31, 0x0f, + 0x7a, 0x2e, 0x7f, 0xa7, 0xc1, 0xbb, 0x30, 0xd7, 0x57, 0x16, 0x8d, 0x63, 0x5f, 0xa3, 0xfa, 0x9c, + 0xcb, 0x1f, 0x2b, 0xfb, 0x3a, 0x3d, 0x83, 0x59, 0x5f, 0x9f, 0xec, 0x1d, 0xaf, 0xc8, 0x1b, 0x2a, + 0xfb, 0x47, 0xd3, 0x47, 0xa0, 0xde, 0x5d, 0xff, 0xd9, 0xa4, 0x93, 0xf6, 0xd0, 0x55, 0xaf, 0x8e, + 0xfd, 0xab, 0xc9, 0x6f, 0xc0, 0xb5, 0x29, 0xe6, 0x7b, 0x5d, 0xf9, 0xde, 0xfe, 0xdd, 0xe4, 0xeb, + 0x70, 0x9d, 0x1e, 0x7e, 0x59, 0x1e, 0xb4, 0x49, 0x5a, 0x27, 0x23, 0xcb, 0xfe, 0xd3, 0xe4, 0x2f, + 0xc3, 0xea, 0x1e, 0xba, 0x32, 0xec, 0x35, 0xe5, 0x7f, 0x9b, 0x7c, 0x01, 0xae, 0x0c, 0xa8, 0x19, + 0xe2, 0x05, 0xb2, 0x77, 0x9b, 0x94, 0xbb, 0xa9, 0x58, 0xb8, 0xf3, 0x5e, 0x93, 0x22, 0xfa, 0x86, + 0x70, 0xd1, 0x79, 0x98, 0xf6, 0xce, 0x85, 0x52, 0x98, 0x58, 0xf6, 0x7e, 0x93, 0xe2, 0x36, 0xc0, + 0x54, 0x5f, 0x60, 0x0d, 0xfe, 0x80, 0x86, 0x1c, 0xf7, 0xc6, 0x9f, 0xcf, 0xd0, 0x4c, 0x4a, 0xc5, + 0xd3, 0x26, 0x65, 0x20, 0xb7, 0x7f, 0x56, 0xf3, 0x61, 0x93, 0xbf, 0x02, 0x6b, 0xf9, 0x9b, 0x9e, + 0xc6, 0x9f, 0x94, 0x23, 0xec, 0xab, 0x33, 0xcd, 0xde, 0x6a, 0x95, 0x8c, 0x21, 0x26, 0x4e, 0x94, + 0xfb, 0xbe, 0xda, 0x22, 0xbf, 0xe8, 0x0d, 0xd1, 0xe0, 0xde, 0xf7, 0x5f, 0x01, 0xcb, 0xde, 0x6e, + 0x51, 0xe2, 0x0a, 0x1e, 0x4f, 0xf0, 0xbb, 0x16, 0x5f, 0x02, 0xc8, 0x1f, 0x9a, 0x07, 0x7e, 0x3f, + 0xdd, 0x48, 0xb3, 0xef, 0x02, 0xcd, 0xc4, 0xa3, 0x7f, 0x28, 0xe9, 0x6a, 0xed, 0x88, 0xfd, 0xb1, + 0x45, 0x01, 0x3a, 0x96, 0x29, 0x1e, 0xcb, 0xe8, 0x31, 0xfb, 0x6e, 0x87, 0x02, 0xe4, 0xfd, 0x3f, + 0xd4, 0x31, 0x92, 0x8d, 0x65, 0xdf, 0xeb, 0x50, 0x15, 0x50, 0x71, 0xe5, 0x55, 0xf0, 0x7d, 0x2f, + 0x17, 0x2d, 0xb5, 0x1f, 0xb2, 0x1f, 0xd0, 0x0c, 0x86, 0x42, 0x3e, 0x1e, 0x3e, 0x60, 0x3f, 0xec, + 0xd0, 0x51, 0x5b, 0x49, 0xa2, 0x23, 0xe1, 0xca, 0x12, 0xff, 0x51, 0x87, 0xde, 0x48, 0xed, 0xf4, + 0x22, 0x47, 0x3f, 0xee, 0x50, 0xa4, 0x0b, 0xdc, 0x57, 0x50, 0x48, 0x4d, 0xf2, 0x27, 0x9e, 0x95, + 0xbe, 0x96, 0xe4, 0xc9, 0xb1, 0x63, 0x3f, 0xf5, 0x76, 0xcf, 0x8f, 0x15, 0xf6, 0xa7, 0x6e, 0x51, + 0x4d, 0x35, 0xec, 0xcf, 0xdd, 0xbc, 0xe8, 0x9f, 0x9d, 0x23, 0xec, 0x2f, 0x1e, 0x7e, 0x7e, 0xf6, + 0xb0, 0xbf, 0x76, 0xc9, 0xb1, 0xfa, 0xf8, 0x50, 0x22, 0x45, 0xcb, 0xfe, 0xd6, 0xdd, 0xdc, 0x80, + 0x76, 0x68, 0x13, 0xdf, 0x48, 0xdb, 0xd0, 0x0c, 0x6d, 0xc2, 0x66, 0xa8, 0xef, 0x6c, 0x6b, 0x9d, + 0xec, 0x5c, 0x8e, 0xcd, 0xc3, 0x4f, 0xb3, 0x60, 0x73, 0x1b, 0x96, 0x7a, 0x3a, 0x1d, 0x8b, 0xb2, + 0x30, 0x7d, 0xef, 0xcc, 0x9b, 0x2e, 0xc6, 0x79, 0x98, 0x67, 0xa8, 0x79, 0xed, 0x5c, 0x62, 0x94, + 0xf9, 0x16, 0x1d, 0x90, 0x48, 0x9b, 0xc8, 0xc1, 0x98, 0x35, 0x36, 0xdf, 0x04, 0xd6, 0xd3, 0xca, + 0x4a, 0xeb, 0x50, 0x45, 0x93, 0x7d, 0xbc, 0xc0, 0xc4, 0x0f, 0x02, 0x67, 0xb4, 0x1a, 0xb1, 0x19, + 0xff, 0x47, 0x43, 0xff, 0xd7, 0xca, 0xc7, 0xc5, 0x36, 0xcd, 0x59, 0xff, 0x11, 0x5b, 0x04, 0xd8, + 0xb9, 0x40, 0xe5, 0x32, 0x91, 0x24, 0x13, 0xd6, 0x24, 0xb9, 0x97, 0x59, 0xa7, 0x53, 0xf9, 0x15, + 0x3f, 0x90, 0xbe, 0x11, 0x40, 0x37, 0x9f, 0x0d, 0xa5, 0x6b, 0xb9, 0x78, 0x84, 0x2a, 0x96, 0x9e, + 0x9c, 0xfe, 0x11, 0x1e, 0x2a, 0xa6, 0x58, 0x50, 0x19, 0x0d, 0x9d, 0x30, 0x6e, 0xfa, 0xe1, 0xcb, + 0xa1, 0x50, 0x3f, 0x51, 0x89, 0x16, 0xb1, 0x1f, 0x50, 0xe5, 0xd6, 0x23, 0x61, 0xac, 0x9f, 0x52, + 0xf4, 0xcd, 0x2a, 0xf8, 0x8d, 0xbf, 0x4f, 0xcc, 0x66, 0x2b, 0xb0, 0xba, 0xf3, 0xdc, 0xf6, 0x1b, + 0xb0, 0x28, 0xf5, 0xf4, 0x2f, 0x3b, 0x32, 0xe3, 0x68, 0xbb, 0xdb, 0xf3, 0x7f, 0xd9, 0x23, 0xfa, + 0xd7, 0x1e, 0x05, 0x5f, 0xb8, 0x3b, 0x92, 0xee, 0x3c, 0x3b, 0xa5, 0x1f, 0xee, 0x9d, 0xdc, 0xec, + 0x35, 0xa9, 0x8b, 0xd5, 0x1d, 0xa9, 0x1c, 0xe5, 0x29, 0xb9, 0xe3, 0x7f, 0xc1, 0x77, 0xf2, 0x5f, + 0xf0, 0xf8, 0xf4, 0x9b, 0x41, 0x70, 0x3a, 0xe7, 0xa1, 0xbb, 0xff, 0x0b, 0x00, 0x00, 0xff, 0xff, + 0x04, 0x66, 0xf7, 0xea, 0x59, 0x0d, 0x00, 0x00, } diff --git a/internal/proto/data_coord.proto b/internal/proto/data_coord.proto index 93a904aa2f..77b7108a48 100644 --- a/internal/proto/data_coord.proto +++ b/internal/proto/data_coord.proto @@ -303,6 +303,7 @@ message SegmentBinlogs { int64 num_of_rows = 3; repeated FieldBinlog statslogs = 4; repeated FieldBinlog deltalogs = 5; + string insert_channel = 6; } message FieldBinlog{ @@ -367,6 +368,7 @@ message CompactionSegmentBinlogs { repeated FieldBinlog fieldBinlogs = 2; repeated FieldBinlog field2StatslogPaths = 3; repeated FieldBinlog deltalogs = 4; + string insert_channel = 5; } message CompactionPlan { diff --git a/internal/proto/datapb/data_coord.pb.go b/internal/proto/datapb/data_coord.pb.go index 8044ebe4ff..86409a7899 100644 --- a/internal/proto/datapb/data_coord.pb.go +++ b/internal/proto/datapb/data_coord.pb.go @@ -2094,6 +2094,7 @@ type SegmentBinlogs struct { NumOfRows int64 `protobuf:"varint,3,opt,name=num_of_rows,json=numOfRows,proto3" json:"num_of_rows,omitempty"` Statslogs []*FieldBinlog `protobuf:"bytes,4,rep,name=statslogs,proto3" json:"statslogs,omitempty"` Deltalogs []*FieldBinlog `protobuf:"bytes,5,rep,name=deltalogs,proto3" json:"deltalogs,omitempty"` + InsertChannel string `protobuf:"bytes,6,opt,name=insert_channel,json=insertChannel,proto3" json:"insert_channel,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -2159,6 +2160,13 @@ func (m *SegmentBinlogs) GetDeltalogs() []*FieldBinlog { return nil } +func (m *SegmentBinlogs) GetInsertChannel() string { + if m != nil { + return m.InsertChannel + } + return "" +} + type FieldBinlog struct { FieldID int64 `protobuf:"varint,1,opt,name=fieldID,proto3" json:"fieldID,omitempty"` Binlogs []*Binlog `protobuf:"bytes,2,rep,name=binlogs,proto3" json:"binlogs,omitempty"` @@ -2606,6 +2614,7 @@ type CompactionSegmentBinlogs struct { FieldBinlogs []*FieldBinlog `protobuf:"bytes,2,rep,name=fieldBinlogs,proto3" json:"fieldBinlogs,omitempty"` Field2StatslogPaths []*FieldBinlog `protobuf:"bytes,3,rep,name=field2StatslogPaths,proto3" json:"field2StatslogPaths,omitempty"` Deltalogs []*FieldBinlog `protobuf:"bytes,4,rep,name=deltalogs,proto3" json:"deltalogs,omitempty"` + InsertChannel string `protobuf:"bytes,5,opt,name=insert_channel,json=insertChannel,proto3" json:"insert_channel,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -2664,6 +2673,13 @@ func (m *CompactionSegmentBinlogs) GetDeltalogs() []*FieldBinlog { return nil } +func (m *CompactionSegmentBinlogs) GetInsertChannel() string { + if m != nil { + return m.InsertChannel + } + return "" +} + type CompactionPlan struct { PlanID int64 `protobuf:"varint,1,opt,name=planID,proto3" json:"planID,omitempty"` SegmentBinlogs []*CompactionSegmentBinlogs `protobuf:"bytes,2,rep,name=segmentBinlogs,proto3" json:"segmentBinlogs,omitempty"` @@ -3748,220 +3764,221 @@ func init() { func init() { proto.RegisterFile("data_coord.proto", fileDescriptor_82cd95f524594f49) } var fileDescriptor_82cd95f524594f49 = []byte{ - // 3401 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xc4, 0x1b, 0x4b, 0x6f, 0x1b, 0xc7, - 0xd9, 0xcb, 0x97, 0xc8, 0x8f, 0x0f, 0x51, 0x63, 0x47, 0xa6, 0x69, 0x5b, 0x96, 0xd7, 0xb1, 0xa3, - 0x38, 0x8e, 0x9d, 0xc8, 0x0d, 0x12, 0x34, 0x2f, 0x44, 0x92, 0xa5, 0x10, 0x95, 0x54, 0x65, 0xa5, - 0xc4, 0x45, 0x53, 0x94, 0x58, 0x71, 0x47, 0xd4, 0x46, 0xdc, 0x5d, 0x66, 0x67, 0x69, 0x59, 0xb9, - 0xc4, 0x68, 0x80, 0x02, 0x29, 0x82, 0xb4, 0x45, 0x4f, 0x05, 0x7a, 0x28, 0x0a, 0x14, 0xe8, 0xe3, - 0x52, 0xa0, 0xa7, 0xb6, 0xe8, 0xa5, 0xa7, 0xa0, 0x3d, 0xf4, 0x67, 0xf4, 0xd6, 0xfe, 0x85, 0x62, - 0x1e, 0x3b, 0xfb, 0xe0, 0x92, 0x5c, 0x49, 0x7e, 0xdc, 0x34, 0xdf, 0x7e, 0xdf, 0x37, 0xdf, 0x7c, - 0xf3, 0xbd, 0x39, 0x82, 0xba, 0xa1, 0x7b, 0x7a, 0xbb, 0xe3, 0x38, 0xae, 0x71, 0xbb, 0xef, 0x3a, - 0x9e, 0x83, 0x66, 0x2c, 0xb3, 0xf7, 0x60, 0x40, 0xf8, 0xea, 0x36, 0xfd, 0xdc, 0xac, 0x74, 0x1c, - 0xcb, 0x72, 0x6c, 0x0e, 0x6a, 0xd6, 0x4c, 0xdb, 0xc3, 0xae, 0xad, 0xf7, 0xc4, 0xba, 0x12, 0x26, - 0x68, 0x56, 0x48, 0x67, 0x1f, 0x5b, 0x3a, 0x5f, 0xa9, 0x53, 0x90, 0xbf, 0x67, 0xf5, 0xbd, 0x23, - 0xf5, 0x21, 0x54, 0x56, 0x7b, 0x03, 0xb2, 0xaf, 0xe1, 0x4f, 0x07, 0x98, 0x78, 0xe8, 0x15, 0xc8, - 0xed, 0xea, 0x04, 0x37, 0x94, 0x79, 0x65, 0xa1, 0xbc, 0x78, 0xe9, 0x76, 0x64, 0x53, 0xb1, 0xdd, - 0x06, 0xe9, 0x2e, 0xe9, 0x04, 0x6b, 0x0c, 0x13, 0x21, 0xc8, 0x19, 0xbb, 0xad, 0x95, 0x46, 0x66, - 0x5e, 0x59, 0xc8, 0x6a, 0xec, 0x6f, 0xa4, 0x42, 0xa5, 0xe3, 0xf4, 0x7a, 0xb8, 0xe3, 0x99, 0x8e, - 0xdd, 0x5a, 0x69, 0xe4, 0xd8, 0xb7, 0x08, 0x4c, 0xfd, 0x95, 0x02, 0x55, 0xb1, 0x35, 0xe9, 0x3b, - 0x36, 0xc1, 0xe8, 0x2e, 0x14, 0x88, 0xa7, 0x7b, 0x03, 0x22, 0x76, 0xbf, 0x98, 0xb8, 0xfb, 0x36, - 0x43, 0xd1, 0x04, 0x6a, 0xaa, 0xed, 0xb3, 0xc3, 0xdb, 0xa3, 0x39, 0x00, 0x82, 0xbb, 0x16, 0xb6, - 0xbd, 0xd6, 0x0a, 0x69, 0xe4, 0xe6, 0xb3, 0x0b, 0x59, 0x2d, 0x04, 0x51, 0x7f, 0xae, 0x40, 0x7d, - 0xdb, 0x5f, 0xfa, 0xda, 0x39, 0x07, 0xf9, 0x8e, 0x33, 0xb0, 0x3d, 0x26, 0x60, 0x55, 0xe3, 0x0b, - 0x74, 0x15, 0x2a, 0x9d, 0x7d, 0xdd, 0xb6, 0x71, 0xaf, 0x6d, 0xeb, 0x16, 0x66, 0xa2, 0x94, 0xb4, - 0xb2, 0x80, 0x6d, 0xea, 0x16, 0x4e, 0x25, 0xd1, 0x3c, 0x94, 0xfb, 0xba, 0xeb, 0x99, 0x11, 0x9d, - 0x85, 0x41, 0xea, 0xaf, 0x15, 0x98, 0x7d, 0x8f, 0x10, 0xb3, 0x6b, 0x0f, 0x49, 0x36, 0x0b, 0x05, - 0xdb, 0x31, 0x70, 0x6b, 0x85, 0x89, 0x96, 0xd5, 0xc4, 0x0a, 0x5d, 0x84, 0x52, 0x1f, 0x63, 0xb7, - 0xed, 0x3a, 0x3d, 0x5f, 0xb0, 0x22, 0x05, 0x68, 0x4e, 0x0f, 0xa3, 0x0f, 0x60, 0x86, 0xc4, 0x18, - 0x91, 0x46, 0x76, 0x3e, 0xbb, 0x50, 0x5e, 0xbc, 0x76, 0x7b, 0xc8, 0xdc, 0x6e, 0xc7, 0x37, 0xd5, - 0x86, 0xa9, 0xd5, 0x47, 0x19, 0x38, 0x2b, 0xf1, 0xb8, 0xac, 0xf4, 0x6f, 0xaa, 0x39, 0x82, 0xbb, - 0x52, 0x3c, 0xbe, 0x48, 0xa3, 0x39, 0xa9, 0xf2, 0x6c, 0x58, 0xe5, 0x29, 0x0c, 0x2c, 0xae, 0xcf, - 0xfc, 0x90, 0x3e, 0xd1, 0x15, 0x28, 0xe3, 0x87, 0x7d, 0xd3, 0xc5, 0x6d, 0xcf, 0xb4, 0x70, 0xa3, - 0x30, 0xaf, 0x2c, 0xe4, 0x34, 0xe0, 0xa0, 0x1d, 0xd3, 0x0a, 0x5b, 0xe4, 0x54, 0x6a, 0x8b, 0x54, - 0x7f, 0xa3, 0xc0, 0xf9, 0xa1, 0x5b, 0x12, 0x26, 0xae, 0x41, 0x9d, 0x9d, 0x3c, 0xd0, 0x0c, 0x35, - 0x76, 0xaa, 0xf0, 0x1b, 0xe3, 0x14, 0x1e, 0xa0, 0x6b, 0x43, 0xf4, 0x21, 0x21, 0x33, 0xe9, 0x85, - 0x3c, 0x80, 0xf3, 0x6b, 0xd8, 0x13, 0x1b, 0xd0, 0x6f, 0x98, 0x9c, 0x3c, 0x04, 0x44, 0x7d, 0x29, - 0x33, 0xe4, 0x4b, 0x7f, 0xca, 0x48, 0x5f, 0x62, 0x5b, 0xb5, 0xec, 0x3d, 0x07, 0x5d, 0x82, 0x92, - 0x44, 0x11, 0x56, 0x11, 0x00, 0xd0, 0xeb, 0x90, 0xa7, 0x92, 0x72, 0x93, 0xa8, 0x2d, 0x5e, 0x4d, - 0x3e, 0x53, 0x88, 0xa7, 0xc6, 0xf1, 0x51, 0x0b, 0x6a, 0xc4, 0xd3, 0x5d, 0xaf, 0xdd, 0x77, 0x08, - 0xbb, 0x67, 0x66, 0x38, 0xe5, 0x45, 0x35, 0xca, 0x41, 0xc6, 0xca, 0x0d, 0xd2, 0xdd, 0x12, 0x98, - 0x5a, 0x95, 0x51, 0xfa, 0x4b, 0x74, 0x0f, 0x2a, 0xd8, 0x36, 0x02, 0x46, 0xb9, 0xd4, 0x8c, 0xca, - 0xd8, 0x36, 0x24, 0x9b, 0xe0, 0x7e, 0xf2, 0xe9, 0xef, 0xe7, 0x2b, 0x05, 0x1a, 0xc3, 0x17, 0x74, - 0x9a, 0x40, 0xf9, 0x26, 0x27, 0xc2, 0xfc, 0x82, 0xc6, 0x7a, 0xb8, 0xbc, 0x24, 0x4d, 0x90, 0xa8, - 0x26, 0x3c, 0x17, 0x48, 0xc3, 0xbe, 0x3c, 0x31, 0x63, 0xf9, 0x42, 0x81, 0xd9, 0xf8, 0x5e, 0xa7, - 0x39, 0xf7, 0xb7, 0x20, 0x6f, 0xda, 0x7b, 0x8e, 0x7f, 0xec, 0xb9, 0x31, 0x7e, 0x46, 0xf7, 0xe2, - 0xc8, 0xaa, 0x05, 0x17, 0xd7, 0xb0, 0xd7, 0xb2, 0x09, 0x76, 0xbd, 0x25, 0xd3, 0xee, 0x39, 0xdd, - 0x2d, 0xdd, 0xdb, 0x3f, 0x85, 0x8f, 0x44, 0xcc, 0x3d, 0x13, 0x33, 0x77, 0xf5, 0x77, 0x0a, 0x5c, - 0x4a, 0xde, 0x4f, 0x1c, 0xbd, 0x09, 0xc5, 0x3d, 0x13, 0xf7, 0x0c, 0xaa, 0x33, 0x85, 0xe9, 0x4c, - 0xae, 0xa9, 0xaf, 0xf4, 0x29, 0xb2, 0x38, 0xe1, 0xd5, 0x11, 0x06, 0xba, 0xed, 0xb9, 0xa6, 0xdd, - 0x5d, 0x37, 0x89, 0xa7, 0x71, 0xfc, 0x90, 0x3e, 0xb3, 0xe9, 0x2d, 0xf3, 0x27, 0x0a, 0xcc, 0xad, - 0x61, 0x6f, 0x59, 0x86, 0x5a, 0xfa, 0xdd, 0x24, 0x9e, 0xd9, 0x21, 0x4f, 0xb6, 0x88, 0x48, 0xc8, - 0x99, 0xea, 0x4f, 0x15, 0xb8, 0x32, 0x52, 0x18, 0xa1, 0x3a, 0x11, 0x4a, 0xfc, 0x40, 0x9b, 0x1c, - 0x4a, 0xbe, 0x83, 0x8f, 0x3e, 0xd2, 0x7b, 0x03, 0xbc, 0xa5, 0x9b, 0x2e, 0x0f, 0x25, 0x27, 0x0c, - 0xac, 0x7f, 0x54, 0xe0, 0xf2, 0x1a, 0xf6, 0xb6, 0xfc, 0x34, 0xf3, 0x0c, 0xb5, 0x93, 0xa2, 0xa2, - 0xf8, 0x9a, 0x5f, 0x66, 0xa2, 0xb4, 0xcf, 0x44, 0x7d, 0x73, 0xcc, 0x0f, 0x42, 0x0e, 0xb9, 0xcc, - 0x6b, 0x01, 0xa1, 0x3c, 0xf5, 0x51, 0x16, 0x2a, 0x1f, 0x89, 0xfa, 0x80, 0xa5, 0x91, 0xb8, 0x1e, - 0x94, 0x64, 0x3d, 0x84, 0x4a, 0x8a, 0xa4, 0x2a, 0x63, 0x0d, 0xaa, 0x04, 0xe3, 0x83, 0x93, 0x24, - 0x8d, 0x0a, 0x25, 0x94, 0xc1, 0x7e, 0x1d, 0x66, 0x06, 0xf6, 0x1e, 0x2d, 0x6b, 0xb1, 0x21, 0x4e, - 0xc1, 0xab, 0xcb, 0xc9, 0x91, 0x67, 0x98, 0x10, 0xbd, 0x0f, 0xd3, 0x71, 0x5e, 0xf9, 0x54, 0xbc, - 0xe2, 0x64, 0xa8, 0x05, 0x75, 0xc3, 0x75, 0xfa, 0x7d, 0x6c, 0xb4, 0x89, 0xcf, 0xaa, 0x90, 0x8e, - 0x95, 0xa0, 0xf3, 0x59, 0xa9, 0x5f, 0x2a, 0x30, 0x7b, 0x5f, 0xf7, 0x3a, 0xfb, 0x2b, 0x96, 0xb8, - 0x9c, 0x53, 0x98, 0xf6, 0xdb, 0x50, 0x7a, 0x20, 0x2e, 0xc2, 0x8f, 0x5f, 0x57, 0x12, 0x04, 0x0a, - 0x5f, 0xb9, 0x16, 0x50, 0xa8, 0xdf, 0x28, 0x70, 0x8e, 0x35, 0x11, 0xbe, 0x74, 0x4f, 0xdf, 0xc9, - 0x26, 0x34, 0x12, 0xe8, 0x06, 0xd4, 0x2c, 0xdd, 0x3d, 0xd8, 0x0e, 0x70, 0xf2, 0x0c, 0x27, 0x06, - 0x55, 0x1f, 0x02, 0x88, 0xd5, 0x06, 0xe9, 0x9e, 0x40, 0xfe, 0x37, 0x60, 0x4a, 0xec, 0x2a, 0xfc, - 0x6d, 0xd2, 0xc5, 0xfa, 0xe8, 0xea, 0x3f, 0x15, 0xa8, 0x05, 0x11, 0x94, 0x79, 0x55, 0x0d, 0x32, - 0xd2, 0x97, 0x32, 0xad, 0x15, 0xf4, 0x36, 0x14, 0x78, 0xff, 0x28, 0x78, 0x5f, 0x8f, 0xf2, 0x16, - 0xbd, 0x65, 0x28, 0x0c, 0x33, 0x80, 0x26, 0x88, 0xa8, 0x8e, 0x64, 0xd4, 0xe1, 0x1d, 0x46, 0x56, - 0x0b, 0x41, 0x50, 0x0b, 0xa6, 0xa3, 0x45, 0x9b, 0xef, 0x33, 0xf3, 0xa3, 0xa2, 0xcd, 0x8a, 0xee, - 0xe9, 0x2c, 0xd8, 0xd4, 0x22, 0x35, 0x1b, 0x51, 0xff, 0x9b, 0x87, 0x72, 0xe8, 0x94, 0x43, 0x27, - 0x89, 0x5f, 0x69, 0x66, 0x72, 0xdc, 0xcc, 0x0e, 0x77, 0x0e, 0xd7, 0xa1, 0x66, 0xb2, 0x5c, 0xdd, - 0x16, 0xa6, 0xc8, 0x82, 0x6b, 0x49, 0xab, 0x72, 0xa8, 0xf0, 0x0b, 0x34, 0x07, 0x65, 0x7b, 0x60, - 0xb5, 0x9d, 0xbd, 0xb6, 0xeb, 0x1c, 0x12, 0xd1, 0x82, 0x94, 0xec, 0x81, 0xf5, 0xdd, 0x3d, 0xcd, - 0x39, 0x24, 0x41, 0x95, 0x5b, 0x38, 0x66, 0x95, 0x3b, 0x07, 0x65, 0x4b, 0x7f, 0x48, 0xb9, 0xb6, - 0xed, 0x81, 0xc5, 0xba, 0x93, 0xac, 0x56, 0xb2, 0xf4, 0x87, 0x9a, 0x73, 0xb8, 0x39, 0xb0, 0xd0, - 0x02, 0xd4, 0x7b, 0x3a, 0xf1, 0xda, 0xe1, 0xf6, 0xa6, 0xc8, 0xda, 0x9b, 0x1a, 0x85, 0xdf, 0x0b, - 0x5a, 0x9c, 0xe1, 0x7a, 0xb9, 0x74, 0x8a, 0x7a, 0xd9, 0xb0, 0x7a, 0x01, 0x23, 0x48, 0x5f, 0x2f, - 0x1b, 0x56, 0x4f, 0xb2, 0x79, 0x03, 0xa6, 0x76, 0x59, 0x05, 0x44, 0x1a, 0xe5, 0x91, 0x11, 0x6a, - 0x95, 0x16, 0x3f, 0xbc, 0x50, 0xd2, 0x7c, 0x74, 0xf4, 0x16, 0x94, 0x58, 0xea, 0x61, 0xb4, 0x95, - 0x54, 0xb4, 0x01, 0x01, 0xa5, 0x36, 0x70, 0xcf, 0xd3, 0x19, 0x75, 0x35, 0x1d, 0xb5, 0x24, 0x40, - 0xaf, 0xc0, 0xd9, 0x8e, 0x8b, 0x75, 0x0f, 0x1b, 0x4b, 0x47, 0xcb, 0x8e, 0xd5, 0xd7, 0x99, 0x31, - 0x35, 0x6a, 0xf3, 0xca, 0x42, 0x51, 0x4b, 0xfa, 0x44, 0x03, 0x43, 0x47, 0xae, 0x56, 0x5d, 0xc7, - 0x6a, 0x4c, 0xf3, 0xc0, 0x10, 0x85, 0xa2, 0xcb, 0x00, 0x7e, 0xe8, 0xd6, 0xbd, 0x46, 0x9d, 0xdd, - 0x62, 0x49, 0x40, 0xde, 0xf3, 0xd4, 0xcf, 0xe1, 0x5c, 0x60, 0x21, 0xa1, 0xdb, 0x18, 0xbe, 0x58, - 0xe5, 0xa4, 0x17, 0x3b, 0xbe, 0x76, 0xfd, 0x77, 0x0e, 0x66, 0xb7, 0xf5, 0x07, 0xf8, 0xc9, 0x97, - 0xc9, 0xa9, 0xe2, 0xf1, 0x3a, 0xcc, 0xb0, 0xca, 0x78, 0x31, 0x24, 0xcf, 0x98, 0x0c, 0x1c, 0xbe, - 0xce, 0x61, 0x42, 0xf4, 0x2e, 0x2d, 0x1d, 0x70, 0xe7, 0x60, 0xcb, 0x31, 0x83, 0xec, 0x7b, 0x39, - 0x81, 0xcf, 0xb2, 0xc4, 0xd2, 0xc2, 0x14, 0x68, 0x6b, 0x38, 0xb4, 0xf1, 0xbc, 0xfb, 0xc2, 0xd8, - 0xfe, 0x2b, 0xd0, 0x7e, 0x3c, 0xc2, 0xa1, 0x06, 0x4c, 0x89, 0xec, 0xce, 0xfc, 0xbe, 0xa8, 0xf9, - 0x4b, 0xb4, 0x05, 0x67, 0xf9, 0x09, 0xb6, 0x85, 0x51, 0xf3, 0xc3, 0x17, 0x53, 0x1d, 0x3e, 0x89, - 0x34, 0xea, 0x13, 0xa5, 0xe3, 0xfa, 0x44, 0x03, 0xa6, 0x84, 0x9d, 0xb2, 0x58, 0x50, 0xd4, 0xfc, - 0x25, 0xbd, 0x66, 0xd3, 0xea, 0x3b, 0xae, 0x67, 0xda, 0xdd, 0x46, 0x99, 0x7d, 0x0b, 0x00, 0xb4, - 0xc5, 0x80, 0x40, 0x9f, 0x13, 0x26, 0x05, 0xef, 0x40, 0x51, 0x5a, 0x78, 0x26, 0xb5, 0x85, 0x4b, - 0x9a, 0x78, 0x8c, 0xce, 0xc6, 0x62, 0xb4, 0xfa, 0x2f, 0x05, 0x2a, 0x2b, 0xf4, 0x48, 0xeb, 0x4e, - 0x97, 0x65, 0x94, 0xeb, 0x50, 0x73, 0x71, 0xc7, 0x71, 0x8d, 0x36, 0xb6, 0x3d, 0xd7, 0xc4, 0xbc, - 0x1b, 0xcd, 0x69, 0x55, 0x0e, 0xbd, 0xc7, 0x81, 0x14, 0x8d, 0x86, 0x5d, 0xe2, 0xe9, 0x56, 0xbf, - 0xbd, 0x47, 0xdd, 0x3b, 0xc3, 0xd1, 0x24, 0x94, 0x79, 0xf7, 0x55, 0xa8, 0x04, 0x68, 0x9e, 0xc3, - 0xf6, 0xcf, 0x69, 0x65, 0x09, 0xdb, 0x71, 0xd0, 0xf3, 0x50, 0x63, 0x3a, 0x6d, 0xf7, 0x9c, 0x6e, - 0x9b, 0x76, 0x6e, 0x22, 0xd9, 0x54, 0x0c, 0x21, 0x16, 0xbd, 0xab, 0x28, 0x16, 0x31, 0x3f, 0xc3, - 0x22, 0xdd, 0x48, 0xac, 0x6d, 0xf3, 0x33, 0x4c, 0x73, 0x7d, 0x95, 0xe6, 0xce, 0x4d, 0xc7, 0xc0, - 0x3b, 0x27, 0xac, 0x34, 0x52, 0x4c, 0xed, 0x2e, 0x41, 0x49, 0x9e, 0x40, 0x1c, 0x29, 0x00, 0xa0, - 0x55, 0xa8, 0xf9, 0x45, 0x68, 0x9b, 0xf7, 0x16, 0xb9, 0x91, 0x95, 0x5f, 0x28, 0xfb, 0x11, 0xad, - 0xea, 0x93, 0xb1, 0xa5, 0xba, 0x0a, 0x95, 0xf0, 0x67, 0xba, 0xeb, 0x76, 0xdc, 0x50, 0x24, 0x80, - 0x5a, 0xe3, 0xe6, 0xc0, 0xa2, 0x77, 0x2a, 0x02, 0x8b, 0xbf, 0x54, 0xbf, 0x50, 0xa0, 0x2a, 0x52, - 0xf6, 0xb6, 0x9c, 0x2a, 0xb3, 0xa3, 0x29, 0xec, 0x68, 0xec, 0x6f, 0xf4, 0xed, 0xe8, 0x48, 0xea, - 0xf9, 0xc4, 0x20, 0xc0, 0x98, 0xb0, 0xea, 0x38, 0x92, 0xaf, 0xd3, 0xf4, 0xb2, 0x8f, 0xa8, 0xa1, - 0x89, 0xab, 0x61, 0x86, 0xd6, 0x80, 0x29, 0xdd, 0x30, 0x5c, 0x4c, 0x88, 0x90, 0xc3, 0x5f, 0xd2, - 0x2f, 0x0f, 0xb0, 0x4b, 0x7c, 0x93, 0xcf, 0x6a, 0xfe, 0x12, 0xbd, 0x05, 0x45, 0x59, 0x4e, 0x67, - 0x93, 0x4a, 0xa8, 0xb0, 0x9c, 0xa2, 0xf7, 0x92, 0x14, 0xea, 0xd7, 0x19, 0xa8, 0x09, 0x85, 0x2d, - 0x89, 0x9c, 0x3a, 0xde, 0xf9, 0x96, 0xa0, 0xb2, 0x17, 0xf8, 0xfe, 0xb8, 0x19, 0x4b, 0x38, 0x44, - 0x44, 0x68, 0x26, 0x39, 0x60, 0x34, 0xab, 0xe7, 0x4e, 0x95, 0xd5, 0xf3, 0xc7, 0x8c, 0x60, 0xea, - 0x0f, 0xa0, 0x1c, 0xfa, 0xc2, 0x42, 0x2f, 0x9f, 0xba, 0x08, 0x55, 0xf8, 0x4b, 0x74, 0x37, 0x28, - 0x5a, 0xb8, 0x0e, 0x2e, 0x24, 0x6c, 0x12, 0xab, 0x57, 0xd4, 0xdf, 0x2b, 0x50, 0x10, 0x9c, 0xaf, - 0x40, 0x59, 0x44, 0x13, 0x56, 0xd0, 0x71, 0xee, 0x20, 0x40, 0xb4, 0xa2, 0x7b, 0x7c, 0xe1, 0xe4, - 0x02, 0x14, 0x63, 0x81, 0x64, 0x4a, 0xc4, 0x7b, 0xff, 0x53, 0x28, 0x7a, 0xd0, 0x4f, 0x2c, 0x70, - 0x7c, 0xa3, 0xb0, 0x89, 0xb1, 0x86, 0x3b, 0xce, 0x03, 0xec, 0x1e, 0x9d, 0x7e, 0x2e, 0xf7, 0x66, - 0xc8, 0x52, 0x53, 0x36, 0x7e, 0x92, 0x00, 0xbd, 0x19, 0xa8, 0x3b, 0x9b, 0x34, 0x96, 0x08, 0x87, - 0x0e, 0x61, 0x67, 0x81, 0xda, 0x7f, 0xc6, 0x27, 0x8c, 0xd1, 0xa3, 0x9c, 0xb4, 0x60, 0x79, 0x2c, - 0xfd, 0x84, 0xfa, 0x0b, 0x05, 0x2e, 0xac, 0x61, 0x6f, 0x35, 0xda, 0xb5, 0x3f, 0x6b, 0xa9, 0x2c, - 0x68, 0x26, 0x09, 0x75, 0x9a, 0x5b, 0x6f, 0x42, 0x51, 0xce, 0x1f, 0xf8, 0xec, 0x57, 0xae, 0xd5, - 0x1f, 0x2b, 0xd0, 0x10, 0xbb, 0xb0, 0x3d, 0x69, 0xad, 0xdc, 0xc3, 0x1e, 0x36, 0x9e, 0x76, 0x43, - 0xfc, 0x77, 0x05, 0xea, 0xe1, 0x50, 0xce, 0xa2, 0xf1, 0x6b, 0x90, 0x67, 0x73, 0x07, 0x21, 0xc1, - 0x44, 0x63, 0xe5, 0xd8, 0x34, 0x64, 0xb0, 0xfa, 0x6d, 0x47, 0x66, 0x1d, 0xb1, 0x0c, 0xf2, 0x49, - 0xf6, 0xf8, 0xf9, 0x44, 0xe4, 0x57, 0x67, 0x40, 0xf9, 0xf2, 0xb9, 0x5e, 0x00, 0x50, 0xbf, 0xca, - 0x40, 0x23, 0x68, 0x34, 0x9e, 0x7a, 0x40, 0x1f, 0x51, 0x86, 0x66, 0x1f, 0x53, 0x19, 0x9a, 0x3b, - 0x6e, 0x10, 0xff, 0x5b, 0x06, 0x6a, 0x81, 0x3a, 0xb6, 0x7a, 0xba, 0x8d, 0x66, 0xa1, 0xd0, 0xef, - 0xe9, 0xc1, 0xbc, 0x50, 0xac, 0xd0, 0xb6, 0xac, 0x4c, 0xa2, 0x0a, 0x78, 0x29, 0xe9, 0x72, 0x46, - 0x68, 0x58, 0x8b, 0xb1, 0xa0, 0x0d, 0x1c, 0x6f, 0x01, 0x58, 0x1b, 0x2e, 0xaa, 0x21, 0x6e, 0x05, - 0xb4, 0x03, 0xbf, 0x05, 0x48, 0x5c, 0x5d, 0xdb, 0xb4, 0xdb, 0x04, 0x77, 0x1c, 0xdb, 0xe0, 0x97, - 0x9a, 0xd7, 0xea, 0xe2, 0x4b, 0xcb, 0xde, 0xe6, 0x70, 0xf4, 0x1a, 0xe4, 0xbc, 0xa3, 0x3e, 0x0f, - 0xcf, 0xb5, 0xc4, 0xb0, 0x17, 0xc8, 0xb5, 0x73, 0xd4, 0xc7, 0x1a, 0x43, 0x47, 0x73, 0x00, 0x94, - 0x95, 0xe7, 0xea, 0x0f, 0x70, 0xcf, 0xff, 0xa5, 0x33, 0x80, 0x50, 0x33, 0xf5, 0x27, 0x19, 0x53, - 0x3c, 0x27, 0x88, 0xa5, 0xfa, 0x97, 0x0c, 0xd4, 0x03, 0x96, 0x1a, 0x26, 0x83, 0x9e, 0x37, 0x52, - 0x7f, 0xe3, 0xdb, 0xb7, 0x49, 0x99, 0xfe, 0x5d, 0x28, 0x8b, 0xa9, 0xca, 0x31, 0x2e, 0x1a, 0x38, - 0xc9, 0xfa, 0x18, 0xcb, 0xcb, 0x3f, 0x26, 0xcb, 0x2b, 0x1c, 0xd7, 0xf2, 0xb6, 0x61, 0xd6, 0x0f, - 0x68, 0x01, 0xc2, 0x06, 0xf6, 0xf4, 0x31, 0x95, 0xc4, 0x15, 0x28, 0xf3, 0x44, 0xc5, 0x33, 0x34, - 0x2f, 0xae, 0x61, 0x57, 0xf6, 0xa4, 0xea, 0x0f, 0xe1, 0x1c, 0x0b, 0x08, 0xf1, 0xe1, 0x6b, 0x9a, - 0x49, 0xb8, 0x2a, 0x4b, 0x77, 0x5a, 0xa6, 0x73, 0xeb, 0x2e, 0x69, 0x11, 0x98, 0xba, 0x0e, 0xcf, - 0xc5, 0xf8, 0x9f, 0x22, 0xe0, 0xd3, 0x1a, 0x67, 0x76, 0x3b, 0xfa, 0x43, 0xe6, 0xc9, 0xd3, 0xda, - 0x65, 0x39, 0x6b, 0x6d, 0x9b, 0x46, 0xdc, 0xbe, 0x0c, 0xf4, 0x0e, 0x94, 0x6c, 0x7c, 0xd8, 0x0e, - 0x47, 0xd5, 0x14, 0x23, 0xb5, 0xa2, 0x8d, 0x0f, 0xd9, 0x5f, 0xea, 0x26, 0x9c, 0x1f, 0x12, 0xf5, - 0x34, 0x67, 0xff, 0xab, 0x02, 0x17, 0x56, 0x5c, 0xa7, 0xff, 0x91, 0xe9, 0x7a, 0x03, 0xbd, 0x17, - 0xfd, 0x29, 0xe3, 0xc9, 0x34, 0x5e, 0xef, 0x87, 0xf2, 0x2b, 0x0f, 0xb8, 0xb7, 0x12, 0xcc, 0x75, - 0x58, 0x28, 0x71, 0xe8, 0x50, 0x36, 0xfe, 0x4f, 0x36, 0x49, 0x78, 0x81, 0x37, 0x21, 0x8b, 0xa4, - 0x29, 0x3f, 0x12, 0xe7, 0x34, 0xd9, 0x93, 0xce, 0x69, 0x46, 0x78, 0x7e, 0xee, 0x31, 0x79, 0xfe, - 0x71, 0x1b, 0x07, 0xf4, 0x3e, 0x44, 0x67, 0x68, 0x2c, 0xe4, 0x9e, 0x68, 0xf8, 0xb6, 0x04, 0x10, - 0xcc, 0x93, 0xc4, 0x3b, 0x94, 0x34, 0x6c, 0x42, 0x54, 0xf4, 0xb6, 0x64, 0x94, 0x65, 0x73, 0xe0, - 0xc8, 0x84, 0xe3, 0x03, 0x68, 0x26, 0x59, 0xe9, 0x69, 0x2c, 0xff, 0xcf, 0x19, 0x80, 0x16, 0x9b, - 0xe7, 0xec, 0xe8, 0xe4, 0xe0, 0x64, 0xa5, 0xe2, 0x35, 0xa8, 0x06, 0x06, 0x13, 0xf8, 0x7b, 0xd8, - 0x8a, 0x0c, 0xea, 0x12, 0xb2, 0x62, 0xa5, 0x38, 0x43, 0x55, 0xac, 0xc1, 0xf8, 0x84, 0xbc, 0x86, - 0x1b, 0x45, 0x2c, 0xe8, 0xa1, 0x8b, 0x50, 0x72, 0x9d, 0xc3, 0x36, 0x75, 0x33, 0x83, 0xe5, 0xd6, - 0xa2, 0x56, 0x74, 0x9d, 0x43, 0xea, 0x7c, 0x06, 0x3a, 0x0f, 0x53, 0x9e, 0x4e, 0x0e, 0x28, 0xff, - 0x02, 0x4f, 0x77, 0x74, 0xd9, 0x32, 0xd0, 0x39, 0xc8, 0xef, 0x99, 0x3d, 0x4c, 0x1a, 0x53, 0x8c, - 0x25, 0x5f, 0xa0, 0xd7, 0xfd, 0x17, 0x07, 0xc5, 0xd4, 0xbf, 0x98, 0xf2, 0x47, 0x07, 0xdf, 0x28, - 0x30, 0x1d, 0x68, 0x8d, 0x05, 0x20, 0x1a, 0xd3, 0x58, 0x3c, 0x5b, 0x76, 0x0c, 0x1e, 0x2a, 0x6a, - 0x23, 0x7e, 0x14, 0xe1, 0x84, 0x3c, 0x6a, 0x05, 0x24, 0xe3, 0x0a, 0x6e, 0x7a, 0x2e, 0x7a, 0x68, - 0xd3, 0xf0, 0x7f, 0x93, 0x29, 0xb8, 0xce, 0x61, 0xcb, 0x90, 0xda, 0xe0, 0x0f, 0xaf, 0x78, 0x79, - 0x49, 0xb5, 0xb1, 0xcc, 0xde, 0x5e, 0x5d, 0x83, 0x2a, 0x76, 0x5d, 0xc7, 0x6d, 0x5b, 0x98, 0x10, - 0xbd, 0xcb, 0x4b, 0x91, 0x92, 0x56, 0x61, 0xc0, 0x0d, 0x0e, 0x53, 0xff, 0x97, 0x81, 0x5a, 0x70, - 0x14, 0xff, 0x97, 0x18, 0xd3, 0xf0, 0x7f, 0x89, 0x31, 0x0d, 0x1a, 0xcc, 0x5d, 0x1e, 0x0a, 0x43, - 0xc1, 0x5c, 0x40, 0x5a, 0x06, 0xcd, 0x83, 0xd4, 0xc1, 0x6c, 0xc7, 0xc0, 0xc1, 0xc5, 0x82, 0x0f, - 0x12, 0xf7, 0x1a, 0xb1, 0x8f, 0x5c, 0x0a, 0xfb, 0xc8, 0xa7, 0xb0, 0x8f, 0x42, 0x82, 0x7d, 0xcc, - 0x42, 0x61, 0x77, 0xd0, 0x39, 0xc0, 0x9e, 0x28, 0x8f, 0xc4, 0x2a, 0x6a, 0x37, 0xc5, 0x98, 0xdd, - 0x48, 0xf3, 0x28, 0x85, 0xcd, 0xe3, 0x22, 0x94, 0xf8, 0xcf, 0x01, 0x6d, 0x8f, 0xb0, 0xb9, 0x68, - 0x56, 0x2b, 0x72, 0xc0, 0x0e, 0x41, 0x6f, 0xf8, 0x4d, 0x41, 0x39, 0xc9, 0xd1, 0x59, 0xc4, 0x89, - 0x59, 0x88, 0x68, 0x09, 0xd4, 0x4f, 0x00, 0x05, 0x5f, 0x4e, 0xd7, 0xa4, 0xc5, 0x54, 0x9f, 0x89, - 0xab, 0x5e, 0xfd, 0x83, 0x02, 0x33, 0xe1, 0xcd, 0x4e, 0x9a, 0xd0, 0xde, 0x81, 0x32, 0x9f, 0xfa, - 0xb6, 0xa9, 0x43, 0x89, 0x36, 0xed, 0xf2, 0xd8, 0x33, 0x6b, 0x60, 0x06, 0x71, 0xe5, 0x1a, 0x54, - 0x0f, 0x1d, 0xf7, 0xc0, 0xb4, 0xbb, 0x6d, 0x2a, 0x99, 0x6f, 0xc6, 0x15, 0x01, 0xdc, 0xa4, 0x30, - 0xf5, 0x4b, 0x05, 0xe6, 0x3e, 0xec, 0x1b, 0xba, 0x87, 0x43, 0x99, 0xfd, 0xb4, 0x4f, 0x32, 0x5e, - 0xf3, 0x5f, 0x45, 0x64, 0xd2, 0x4d, 0x2e, 0x39, 0xf6, 0xcd, 0x5f, 0x2a, 0x30, 0x33, 0xd4, 0xd4, - 0xa1, 0x1a, 0xc0, 0x87, 0x76, 0x47, 0x74, 0xbb, 0xf5, 0x33, 0xa8, 0x02, 0x45, 0xbf, 0xf7, 0xad, - 0x2b, 0xa8, 0x0c, 0x53, 0x3b, 0x0e, 0xc3, 0xae, 0x67, 0x50, 0x1d, 0x2a, 0x9c, 0x70, 0xd0, 0xe9, - 0x60, 0x42, 0xea, 0x59, 0x09, 0x59, 0xd5, 0xcd, 0xde, 0xc0, 0xc5, 0xf5, 0x1c, 0xaa, 0x42, 0x69, - 0xc7, 0xd1, 0x70, 0x0f, 0xeb, 0x04, 0xd7, 0xf3, 0x08, 0x41, 0x4d, 0x2c, 0x7c, 0xa2, 0x42, 0x08, - 0xe6, 0x93, 0x4d, 0xdd, 0xdc, 0x0b, 0x77, 0x49, 0xb4, 0x75, 0x40, 0xe7, 0xe1, 0xec, 0x87, 0xb6, - 0x81, 0xf7, 0x4c, 0x1b, 0x1b, 0xc1, 0xa7, 0xfa, 0x19, 0x74, 0x16, 0xa6, 0x5b, 0xb6, 0x8d, 0xdd, - 0x10, 0x50, 0xa1, 0xc0, 0x0d, 0xec, 0x76, 0x71, 0x08, 0x98, 0x41, 0x33, 0x50, 0xdd, 0x30, 0x1f, - 0x86, 0x40, 0xd9, 0xc5, 0x7f, 0x3c, 0x07, 0xa5, 0x15, 0xdd, 0xd3, 0x97, 0x1d, 0xc7, 0x35, 0x50, - 0x1f, 0x10, 0x7b, 0xc0, 0x63, 0xf5, 0x1d, 0x5b, 0xbe, 0x74, 0x43, 0xaf, 0x8c, 0x48, 0x70, 0xc3, - 0xa8, 0xe2, 0x0a, 0x9b, 0x37, 0x46, 0x50, 0xc4, 0xd0, 0xd5, 0x33, 0xc8, 0x62, 0x3b, 0xd2, 0xd6, - 0x6b, 0xc7, 0xec, 0x1c, 0xf8, 0x3f, 0xd5, 0x8e, 0xd9, 0x31, 0x86, 0xea, 0xef, 0x18, 0x7b, 0x40, - 0x27, 0x16, 0xfc, 0x95, 0x95, 0xef, 0x7f, 0xea, 0x19, 0xf4, 0x29, 0x9c, 0x5b, 0xc3, 0x21, 0x9b, - 0xf3, 0x37, 0x5c, 0x1c, 0xbd, 0xe1, 0x10, 0xf2, 0x31, 0xb7, 0x5c, 0x87, 0x3c, 0x1b, 0xa0, 0xa0, - 0x24, 0xb3, 0x0c, 0x3f, 0xf7, 0x6e, 0xce, 0x8f, 0x46, 0x90, 0xdc, 0x3e, 0x81, 0xe9, 0xd8, 0x73, - 0x56, 0xf4, 0x62, 0x02, 0x59, 0xf2, 0xc3, 0xe4, 0xe6, 0xcd, 0x34, 0xa8, 0x72, 0xaf, 0x2e, 0xd4, - 0xa2, 0xcf, 0x7f, 0xd0, 0x42, 0x02, 0x7d, 0xe2, 0x53, 0xc4, 0xe6, 0x8b, 0x29, 0x30, 0xe5, 0x46, - 0x16, 0xd4, 0xe3, 0xcf, 0x2b, 0xd1, 0xcd, 0xb1, 0x0c, 0xa2, 0xe6, 0xf6, 0x52, 0x2a, 0x5c, 0xb9, - 0xdd, 0x11, 0x33, 0x82, 0xa1, 0xe7, 0x7d, 0xe8, 0x76, 0x32, 0x9b, 0x51, 0xef, 0x0e, 0x9b, 0x77, - 0x52, 0xe3, 0xcb, 0xad, 0x7f, 0xc4, 0x07, 0xb7, 0x49, 0x4f, 0xe4, 0xd0, 0xab, 0xc9, 0xec, 0xc6, - 0xbc, 0xed, 0x6b, 0x2e, 0x1e, 0x87, 0x44, 0x0a, 0xf1, 0x39, 0x9b, 0xb8, 0x26, 0x3c, 0x33, 0x8b, - 0xfb, 0x9d, 0xcf, 0x6f, 0xf4, 0xfb, 0xb9, 0xe6, 0xab, 0xc7, 0xa0, 0x90, 0x02, 0x38, 0xf1, 0x07, - 0xac, 0xbe, 0x1b, 0xde, 0x99, 0x68, 0x35, 0x27, 0xf3, 0xc1, 0x8f, 0x61, 0x3a, 0xf6, 0xa3, 0x78, - 0xa2, 0xd7, 0x24, 0xff, 0x70, 0xde, 0x1c, 0x97, 0xa6, 0xb9, 0x4b, 0xc6, 0x06, 0xd8, 0x68, 0x84, - 0xf5, 0x27, 0x0c, 0xb9, 0x9b, 0x37, 0xd3, 0xa0, 0xca, 0x83, 0x10, 0x16, 0x2e, 0x63, 0x43, 0x60, - 0x74, 0x2b, 0x99, 0x47, 0xf2, 0x00, 0xbb, 0xf9, 0x72, 0x4a, 0x6c, 0xb9, 0x69, 0x1b, 0x60, 0x0d, - 0x7b, 0x1b, 0xd8, 0x73, 0xa9, 0x8d, 0xdc, 0x48, 0x54, 0x79, 0x80, 0xe0, 0x6f, 0xf3, 0xc2, 0x44, - 0x3c, 0xb9, 0xc1, 0xf7, 0x00, 0xf9, 0x29, 0x36, 0xf4, 0x24, 0xe3, 0xda, 0xd8, 0x71, 0x1a, 0x9f, - 0x7d, 0x4d, 0xba, 0x9b, 0x4f, 0xa1, 0xbe, 0xa1, 0xdb, 0xb4, 0x91, 0x0a, 0xf8, 0xde, 0x4a, 0x14, - 0x2c, 0x8e, 0x36, 0x42, 0x5b, 0x23, 0xb1, 0xe5, 0x61, 0x0e, 0x65, 0x0e, 0xd5, 0xa5, 0x0b, 0xe2, - 0x78, 0x6c, 0x09, 0xb4, 0x11, 0x43, 0x1c, 0x11, 0x5b, 0xc6, 0xe0, 0xcb, 0x8d, 0x1f, 0x29, 0xec, - 0x99, 0x74, 0x0c, 0xe1, 0xbe, 0xe9, 0xed, 0x6f, 0xf5, 0x74, 0x9b, 0xa4, 0x11, 0x81, 0x21, 0x1e, - 0x43, 0x04, 0x81, 0x2f, 0x45, 0x30, 0xa0, 0x1a, 0x99, 0x56, 0xa1, 0xa4, 0x77, 0x15, 0x49, 0xf3, - 0xb2, 0xe6, 0xc2, 0x64, 0x44, 0xb9, 0xcb, 0x3e, 0x54, 0x7d, 0x7b, 0xe5, 0xca, 0x7d, 0x71, 0x94, - 0xa4, 0x01, 0xce, 0x08, 0x77, 0x4b, 0x46, 0x0d, 0xbb, 0xdb, 0x70, 0x33, 0x8e, 0xd2, 0x0d, 0x71, - 0xc6, 0xb9, 0xdb, 0xe8, 0x0e, 0x9f, 0xc7, 0x93, 0xd8, 0xe0, 0x2b, 0x39, 0x58, 0x25, 0xce, 0xf1, - 0x12, 0xe3, 0xc9, 0x88, 0x39, 0x9a, 0x7a, 0x06, 0xdd, 0x87, 0x02, 0xaf, 0xe6, 0xd1, 0xf3, 0xe3, - 0x0b, 0x7d, 0xc1, 0xfd, 0xfa, 0x04, 0x2c, 0xc9, 0xf8, 0x00, 0xce, 0x8f, 0x28, 0xf3, 0x13, 0xf3, - 0xdc, 0xf8, 0x96, 0x60, 0x82, 0x97, 0x2f, 0xfe, 0x36, 0x0f, 0x45, 0xff, 0xc7, 0xfa, 0x67, 0x50, - 0xc3, 0x3e, 0x83, 0xa2, 0xf2, 0x63, 0x98, 0x8e, 0xbd, 0xfa, 0x4d, 0xb4, 0x91, 0xe4, 0x97, 0xc1, - 0x93, 0x82, 0xe6, 0x7d, 0xf1, 0xbf, 0x80, 0x32, 0xbf, 0xbc, 0x30, 0xaa, 0x30, 0x8d, 0xa7, 0x96, - 0x09, 0x8c, 0x9f, 0x78, 0x22, 0xd9, 0x04, 0x08, 0x05, 0xfa, 0xf1, 0xbf, 0xc7, 0xd0, 0xd8, 0x35, - 0x49, 0xe0, 0x8d, 0x63, 0xba, 0xc7, 0x78, 0x76, 0x4b, 0x77, 0xbf, 0xff, 0x6a, 0xd7, 0xf4, 0xf6, - 0x07, 0xbb, 0xf4, 0xcb, 0x1d, 0x8e, 0xfa, 0xb2, 0xe9, 0x88, 0xbf, 0xee, 0xf8, 0x06, 0x72, 0x87, - 0x51, 0xdf, 0xa1, 0x7b, 0xf4, 0x77, 0x77, 0x0b, 0x6c, 0x75, 0xf7, 0xff, 0x01, 0x00, 0x00, 0xff, - 0xff, 0xa1, 0x07, 0xba, 0xab, 0x85, 0x3a, 0x00, 0x00, + // 3412 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xc4, 0x3b, 0x4b, 0x6f, 0x1b, 0xd7, + 0xd5, 0x1e, 0xbe, 0x44, 0x1e, 0x3e, 0x44, 0x5f, 0x3b, 0x32, 0x4d, 0xdb, 0xb2, 0x3c, 0x8e, 0x1d, + 0xc5, 0x71, 0xec, 0x44, 0xfe, 0x82, 0x04, 0x5f, 0x5e, 0x88, 0x24, 0x4b, 0x21, 0x3e, 0x49, 0x9f, + 0x32, 0x52, 0xe2, 0xa2, 0x29, 0x4a, 0x8c, 0x38, 0x57, 0xd4, 0x44, 0x9c, 0x19, 0x66, 0xee, 0xd0, + 0xb2, 0xb2, 0x89, 0xd1, 0x00, 0x05, 0x52, 0x14, 0x7d, 0xa0, 0xab, 0x02, 0x5d, 0x14, 0x05, 0x0a, + 0xf4, 0xb1, 0x29, 0x10, 0x74, 0xd1, 0x16, 0xdd, 0x74, 0x15, 0xb4, 0x8b, 0xfe, 0x8c, 0xee, 0xda, + 0xbf, 0x50, 0xdc, 0xc7, 0xdc, 0x79, 0x70, 0x48, 0x8e, 0x44, 0x3b, 0xde, 0xe9, 0x9e, 0x39, 0xe7, + 0xdc, 0x73, 0xcf, 0x3d, 0x6f, 0x5e, 0x41, 0xdd, 0xd0, 0x3d, 0xbd, 0xdd, 0x71, 0x1c, 0xd7, 0xb8, + 0xd3, 0x77, 0x1d, 0xcf, 0x41, 0x67, 0x2d, 0xb3, 0xf7, 0x70, 0x40, 0xf8, 0xea, 0x0e, 0xfd, 0xdc, + 0xac, 0x74, 0x1c, 0xcb, 0x72, 0x6c, 0x0e, 0x6a, 0xd6, 0x4c, 0xdb, 0xc3, 0xae, 0xad, 0xf7, 0xc4, + 0xba, 0x12, 0x26, 0x68, 0x56, 0x48, 0xe7, 0x00, 0x5b, 0x3a, 0x5f, 0xa9, 0x33, 0x90, 0xbf, 0x6f, + 0xf5, 0xbd, 0x63, 0xf5, 0x11, 0x54, 0xd6, 0x7a, 0x03, 0x72, 0xa0, 0xe1, 0x4f, 0x07, 0x98, 0x78, + 0xe8, 0x15, 0xc8, 0xed, 0xe9, 0x04, 0x37, 0x94, 0x05, 0x65, 0xb1, 0xbc, 0x74, 0xf9, 0x4e, 0x64, + 0x53, 0xb1, 0xdd, 0x26, 0xe9, 0x2e, 0xeb, 0x04, 0x6b, 0x0c, 0x13, 0x21, 0xc8, 0x19, 0x7b, 0xad, + 0xd5, 0x46, 0x66, 0x41, 0x59, 0xcc, 0x6a, 0xec, 0x6f, 0xa4, 0x42, 0xa5, 0xe3, 0xf4, 0x7a, 0xb8, + 0xe3, 0x99, 0x8e, 0xdd, 0x5a, 0x6d, 0xe4, 0xd8, 0xb7, 0x08, 0x4c, 0xfd, 0x85, 0x02, 0x55, 0xb1, + 0x35, 0xe9, 0x3b, 0x36, 0xc1, 0xe8, 0x1e, 0x14, 0x88, 0xa7, 0x7b, 0x03, 0x22, 0x76, 0xbf, 0x94, + 0xb8, 0xfb, 0x0e, 0x43, 0xd1, 0x04, 0x6a, 0xaa, 0xed, 0xb3, 0xc3, 0xdb, 0xa3, 0x79, 0x00, 0x82, + 0xbb, 0x16, 0xb6, 0xbd, 0xd6, 0x2a, 0x69, 0xe4, 0x16, 0xb2, 0x8b, 0x59, 0x2d, 0x04, 0x51, 0x7f, + 0xaa, 0x40, 0x7d, 0xc7, 0x5f, 0xfa, 0xda, 0x39, 0x0f, 0xf9, 0x8e, 0x33, 0xb0, 0x3d, 0x26, 0x60, + 0x55, 0xe3, 0x0b, 0x74, 0x0d, 0x2a, 0x9d, 0x03, 0xdd, 0xb6, 0x71, 0xaf, 0x6d, 0xeb, 0x16, 0x66, + 0xa2, 0x94, 0xb4, 0xb2, 0x80, 0x6d, 0xe9, 0x16, 0x4e, 0x25, 0xd1, 0x02, 0x94, 0xfb, 0xba, 0xeb, + 0x99, 0x11, 0x9d, 0x85, 0x41, 0xea, 0x2f, 0x15, 0x98, 0x7b, 0x8f, 0x10, 0xb3, 0x6b, 0x0f, 0x49, + 0x36, 0x07, 0x05, 0xdb, 0x31, 0x70, 0x6b, 0x95, 0x89, 0x96, 0xd5, 0xc4, 0x0a, 0x5d, 0x82, 0x52, + 0x1f, 0x63, 0xb7, 0xed, 0x3a, 0x3d, 0x5f, 0xb0, 0x22, 0x05, 0x68, 0x4e, 0x0f, 0xa3, 0x0f, 0xe0, + 0x2c, 0x89, 0x31, 0x22, 0x8d, 0xec, 0x42, 0x76, 0xb1, 0xbc, 0x74, 0xfd, 0xce, 0x90, 0xb9, 0xdd, + 0x89, 0x6f, 0xaa, 0x0d, 0x53, 0xab, 0x8f, 0x33, 0x70, 0x4e, 0xe2, 0x71, 0x59, 0xe9, 0xdf, 0x54, + 0x73, 0x04, 0x77, 0xa5, 0x78, 0x7c, 0x91, 0x46, 0x73, 0x52, 0xe5, 0xd9, 0xb0, 0xca, 0x53, 0x18, + 0x58, 0x5c, 0x9f, 0xf9, 0x21, 0x7d, 0xa2, 0xab, 0x50, 0xc6, 0x8f, 0xfa, 0xa6, 0x8b, 0xdb, 0x9e, + 0x69, 0xe1, 0x46, 0x61, 0x41, 0x59, 0xcc, 0x69, 0xc0, 0x41, 0xbb, 0xa6, 0x15, 0xb6, 0xc8, 0x99, + 0xd4, 0x16, 0xa9, 0xfe, 0x4a, 0x81, 0x0b, 0x43, 0xb7, 0x24, 0x4c, 0x5c, 0x83, 0x3a, 0x3b, 0x79, + 0xa0, 0x19, 0x6a, 0xec, 0x54, 0xe1, 0x37, 0xc7, 0x29, 0x3c, 0x40, 0xd7, 0x86, 0xe8, 0x43, 0x42, + 0x66, 0xd2, 0x0b, 0x79, 0x08, 0x17, 0xd6, 0xb1, 0x27, 0x36, 0xa0, 0xdf, 0x30, 0x39, 0x7d, 0x08, + 0x88, 0xfa, 0x52, 0x66, 0xc8, 0x97, 0xfe, 0x90, 0x91, 0xbe, 0xc4, 0xb6, 0x6a, 0xd9, 0xfb, 0x0e, + 0xba, 0x0c, 0x25, 0x89, 0x22, 0xac, 0x22, 0x00, 0xa0, 0xd7, 0x21, 0x4f, 0x25, 0xe5, 0x26, 0x51, + 0x5b, 0xba, 0x96, 0x7c, 0xa6, 0x10, 0x4f, 0x8d, 0xe3, 0xa3, 0x16, 0xd4, 0x88, 0xa7, 0xbb, 0x5e, + 0xbb, 0xef, 0x10, 0x76, 0xcf, 0xcc, 0x70, 0xca, 0x4b, 0x6a, 0x94, 0x83, 0x8c, 0x95, 0x9b, 0xa4, + 0xbb, 0x2d, 0x30, 0xb5, 0x2a, 0xa3, 0xf4, 0x97, 0xe8, 0x3e, 0x54, 0xb0, 0x6d, 0x04, 0x8c, 0x72, + 0xa9, 0x19, 0x95, 0xb1, 0x6d, 0x48, 0x36, 0xc1, 0xfd, 0xe4, 0xd3, 0xdf, 0xcf, 0x0f, 0x15, 0x68, + 0x0c, 0x5f, 0xd0, 0x34, 0x81, 0xf2, 0x4d, 0x4e, 0x84, 0xf9, 0x05, 0x8d, 0xf5, 0x70, 0x79, 0x49, + 0x9a, 0x20, 0x51, 0x4d, 0x78, 0x2e, 0x90, 0x86, 0x7d, 0x79, 0x6a, 0xc6, 0xf2, 0x85, 0x02, 0x73, + 0xf1, 0xbd, 0xa6, 0x39, 0xf7, 0xff, 0x40, 0xde, 0xb4, 0xf7, 0x1d, 0xff, 0xd8, 0xf3, 0x63, 0xfc, + 0x8c, 0xee, 0xc5, 0x91, 0x55, 0x0b, 0x2e, 0xad, 0x63, 0xaf, 0x65, 0x13, 0xec, 0x7a, 0xcb, 0xa6, + 0xdd, 0x73, 0xba, 0xdb, 0xba, 0x77, 0x30, 0x85, 0x8f, 0x44, 0xcc, 0x3d, 0x13, 0x33, 0x77, 0xf5, + 0x37, 0x0a, 0x5c, 0x4e, 0xde, 0x4f, 0x1c, 0xbd, 0x09, 0xc5, 0x7d, 0x13, 0xf7, 0x0c, 0xaa, 0x33, + 0x85, 0xe9, 0x4c, 0xae, 0xa9, 0xaf, 0xf4, 0x29, 0xb2, 0x38, 0xe1, 0xb5, 0x11, 0x06, 0xba, 0xe3, + 0xb9, 0xa6, 0xdd, 0xdd, 0x30, 0x89, 0xa7, 0x71, 0xfc, 0x90, 0x3e, 0xb3, 0xe9, 0x2d, 0xf3, 0x07, + 0x0a, 0xcc, 0xaf, 0x63, 0x6f, 0x45, 0x86, 0x5a, 0xfa, 0xdd, 0x24, 0x9e, 0xd9, 0x21, 0x4f, 0xb7, + 0x88, 0x48, 0xc8, 0x99, 0xea, 0x8f, 0x15, 0xb8, 0x3a, 0x52, 0x18, 0xa1, 0x3a, 0x11, 0x4a, 0xfc, + 0x40, 0x9b, 0x1c, 0x4a, 0xfe, 0x0f, 0x1f, 0x7f, 0xa4, 0xf7, 0x06, 0x78, 0x5b, 0x37, 0x5d, 0x1e, + 0x4a, 0x4e, 0x19, 0x58, 0x7f, 0xaf, 0xc0, 0x95, 0x75, 0xec, 0x6d, 0xfb, 0x69, 0xe6, 0x19, 0x6a, + 0x27, 0x45, 0x45, 0xf1, 0x23, 0x7e, 0x99, 0x89, 0xd2, 0x3e, 0x13, 0xf5, 0xcd, 0x33, 0x3f, 0x08, + 0x39, 0xe4, 0x0a, 0xaf, 0x05, 0x84, 0xf2, 0xd4, 0xc7, 0x59, 0xa8, 0x7c, 0x24, 0xea, 0x03, 0x96, + 0x46, 0xe2, 0x7a, 0x50, 0x92, 0xf5, 0x10, 0x2a, 0x29, 0x92, 0xaa, 0x8c, 0x75, 0xa8, 0x12, 0x8c, + 0x0f, 0x4f, 0x93, 0x34, 0x2a, 0x94, 0x50, 0x06, 0xfb, 0x0d, 0x38, 0x3b, 0xb0, 0xf7, 0x69, 0x59, + 0x8b, 0x0d, 0x71, 0x0a, 0x5e, 0x5d, 0x4e, 0x8e, 0x3c, 0xc3, 0x84, 0xe8, 0x7d, 0x98, 0x8d, 0xf3, + 0xca, 0xa7, 0xe2, 0x15, 0x27, 0x43, 0x2d, 0xa8, 0x1b, 0xae, 0xd3, 0xef, 0x63, 0xa3, 0x4d, 0x7c, + 0x56, 0x85, 0x74, 0xac, 0x04, 0x9d, 0xcf, 0x4a, 0xfd, 0x52, 0x81, 0xb9, 0x07, 0xba, 0xd7, 0x39, + 0x58, 0xb5, 0xc4, 0xe5, 0x4c, 0x61, 0xda, 0x6f, 0x43, 0xe9, 0xa1, 0xb8, 0x08, 0x3f, 0x7e, 0x5d, + 0x4d, 0x10, 0x28, 0x7c, 0xe5, 0x5a, 0x40, 0xa1, 0x7e, 0xad, 0xc0, 0x79, 0xd6, 0x44, 0xf8, 0xd2, + 0x7d, 0xf3, 0x4e, 0x36, 0xa1, 0x91, 0x40, 0x37, 0xa1, 0x66, 0xe9, 0xee, 0xe1, 0x4e, 0x80, 0x93, + 0x67, 0x38, 0x31, 0xa8, 0xfa, 0x08, 0x40, 0xac, 0x36, 0x49, 0xf7, 0x14, 0xf2, 0xbf, 0x01, 0x33, + 0x62, 0x57, 0xe1, 0x6f, 0x93, 0x2e, 0xd6, 0x47, 0x57, 0xff, 0xae, 0x40, 0x2d, 0x88, 0xa0, 0xcc, + 0xab, 0x6a, 0x90, 0x91, 0xbe, 0x94, 0x69, 0xad, 0xa2, 0xb7, 0xa1, 0xc0, 0xfb, 0x47, 0xc1, 0xfb, + 0x46, 0x94, 0xb7, 0xe8, 0x2d, 0x43, 0x61, 0x98, 0x01, 0x34, 0x41, 0x44, 0x75, 0x24, 0xa3, 0x0e, + 0xef, 0x30, 0xb2, 0x5a, 0x08, 0x82, 0x5a, 0x30, 0x1b, 0x2d, 0xda, 0x7c, 0x9f, 0x59, 0x18, 0x15, + 0x6d, 0x56, 0x75, 0x4f, 0x67, 0xc1, 0xa6, 0x16, 0xa9, 0xd9, 0x88, 0xfa, 0xef, 0x3c, 0x94, 0x43, + 0xa7, 0x1c, 0x3a, 0x49, 0xfc, 0x4a, 0x33, 0x93, 0xe3, 0x66, 0x76, 0xb8, 0x73, 0xb8, 0x01, 0x35, + 0x93, 0xe5, 0xea, 0xb6, 0x30, 0x45, 0x16, 0x5c, 0x4b, 0x5a, 0x95, 0x43, 0x85, 0x5f, 0xa0, 0x79, + 0x28, 0xdb, 0x03, 0xab, 0xed, 0xec, 0xb7, 0x5d, 0xe7, 0x88, 0x88, 0x16, 0xa4, 0x64, 0x0f, 0xac, + 0xff, 0xdf, 0xd7, 0x9c, 0x23, 0x12, 0x54, 0xb9, 0x85, 0x13, 0x56, 0xb9, 0xf3, 0x50, 0xb6, 0xf4, + 0x47, 0x94, 0x6b, 0xdb, 0x1e, 0x58, 0xac, 0x3b, 0xc9, 0x6a, 0x25, 0x4b, 0x7f, 0xa4, 0x39, 0x47, + 0x5b, 0x03, 0x0b, 0x2d, 0x42, 0xbd, 0xa7, 0x13, 0xaf, 0x1d, 0x6e, 0x6f, 0x8a, 0xac, 0xbd, 0xa9, + 0x51, 0xf8, 0xfd, 0xa0, 0xc5, 0x19, 0xae, 0x97, 0x4b, 0x53, 0xd4, 0xcb, 0x86, 0xd5, 0x0b, 0x18, + 0x41, 0xfa, 0x7a, 0xd9, 0xb0, 0x7a, 0x92, 0xcd, 0x1b, 0x30, 0xb3, 0xc7, 0x2a, 0x20, 0xd2, 0x28, + 0x8f, 0x8c, 0x50, 0x6b, 0xb4, 0xf8, 0xe1, 0x85, 0x92, 0xe6, 0xa3, 0xa3, 0xb7, 0xa0, 0xc4, 0x52, + 0x0f, 0xa3, 0xad, 0xa4, 0xa2, 0x0d, 0x08, 0x28, 0xb5, 0x81, 0x7b, 0x9e, 0xce, 0xa8, 0xab, 0xe9, + 0xa8, 0x25, 0x01, 0x7a, 0x05, 0xce, 0x75, 0x5c, 0xac, 0x7b, 0xd8, 0x58, 0x3e, 0x5e, 0x71, 0xac, + 0xbe, 0xce, 0x8c, 0xa9, 0x51, 0x5b, 0x50, 0x16, 0x8b, 0x5a, 0xd2, 0x27, 0x1a, 0x18, 0x3a, 0x72, + 0xb5, 0xe6, 0x3a, 0x56, 0x63, 0x96, 0x07, 0x86, 0x28, 0x14, 0x5d, 0x01, 0xf0, 0x43, 0xb7, 0xee, + 0x35, 0xea, 0xec, 0x16, 0x4b, 0x02, 0xf2, 0x9e, 0xa7, 0x7e, 0x0e, 0xe7, 0x03, 0x0b, 0x09, 0xdd, + 0xc6, 0xf0, 0xc5, 0x2a, 0xa7, 0xbd, 0xd8, 0xf1, 0xb5, 0xeb, 0x3f, 0x73, 0x30, 0xb7, 0xa3, 0x3f, + 0xc4, 0x4f, 0xbf, 0x4c, 0x4e, 0x15, 0x8f, 0x37, 0xe0, 0x2c, 0xab, 0x8c, 0x97, 0x42, 0xf2, 0x8c, + 0xc9, 0xc0, 0xe1, 0xeb, 0x1c, 0x26, 0x44, 0xef, 0xd2, 0xd2, 0x01, 0x77, 0x0e, 0xb7, 0x1d, 0x33, + 0xc8, 0xbe, 0x57, 0x12, 0xf8, 0xac, 0x48, 0x2c, 0x2d, 0x4c, 0x81, 0xb6, 0x87, 0x43, 0x1b, 0xcf, + 0xbb, 0x2f, 0x8c, 0xed, 0xbf, 0x02, 0xed, 0xc7, 0x23, 0x1c, 0x6a, 0xc0, 0x8c, 0xc8, 0xee, 0xcc, + 0xef, 0x8b, 0x9a, 0xbf, 0x44, 0xdb, 0x70, 0x8e, 0x9f, 0x60, 0x47, 0x18, 0x35, 0x3f, 0x7c, 0x31, + 0xd5, 0xe1, 0x93, 0x48, 0xa3, 0x3e, 0x51, 0x3a, 0xa9, 0x4f, 0x34, 0x60, 0x46, 0xd8, 0x29, 0x8b, + 0x05, 0x45, 0xcd, 0x5f, 0xd2, 0x6b, 0x36, 0xad, 0xbe, 0xe3, 0x7a, 0xa6, 0xdd, 0x6d, 0x94, 0xd9, + 0xb7, 0x00, 0x40, 0x5b, 0x0c, 0x08, 0xf4, 0x39, 0x61, 0x52, 0xf0, 0x0e, 0x14, 0xa5, 0x85, 0x67, + 0x52, 0x5b, 0xb8, 0xa4, 0x89, 0xc7, 0xe8, 0x6c, 0x2c, 0x46, 0xab, 0xff, 0x50, 0xa0, 0xb2, 0x4a, + 0x8f, 0xb4, 0xe1, 0x74, 0x59, 0x46, 0xb9, 0x01, 0x35, 0x17, 0x77, 0x1c, 0xd7, 0x68, 0x63, 0xdb, + 0x73, 0x4d, 0xcc, 0xbb, 0xd1, 0x9c, 0x56, 0xe5, 0xd0, 0xfb, 0x1c, 0x48, 0xd1, 0x68, 0xd8, 0x25, + 0x9e, 0x6e, 0xf5, 0xdb, 0xfb, 0xd4, 0xbd, 0x33, 0x1c, 0x4d, 0x42, 0x99, 0x77, 0x5f, 0x83, 0x4a, + 0x80, 0xe6, 0x39, 0x6c, 0xff, 0x9c, 0x56, 0x96, 0xb0, 0x5d, 0x07, 0x3d, 0x0f, 0x35, 0xa6, 0xd3, + 0x76, 0xcf, 0xe9, 0xb6, 0x69, 0xe7, 0x26, 0x92, 0x4d, 0xc5, 0x10, 0x62, 0xd1, 0xbb, 0x8a, 0x62, + 0x11, 0xf3, 0x33, 0x2c, 0xd2, 0x8d, 0xc4, 0xda, 0x31, 0x3f, 0xc3, 0x34, 0xd7, 0x57, 0x69, 0xee, + 0xdc, 0x72, 0x0c, 0xbc, 0x7b, 0xca, 0x4a, 0x23, 0xc5, 0xd4, 0xee, 0x32, 0x94, 0xe4, 0x09, 0xc4, + 0x91, 0x02, 0x00, 0x5a, 0x83, 0x9a, 0x5f, 0x84, 0xb6, 0x79, 0x6f, 0x91, 0x1b, 0x59, 0xf9, 0x85, + 0xb2, 0x1f, 0xd1, 0xaa, 0x3e, 0x19, 0x5b, 0xaa, 0x6b, 0x50, 0x09, 0x7f, 0xa6, 0xbb, 0xee, 0xc4, + 0x0d, 0x45, 0x02, 0xa8, 0x35, 0x6e, 0x0d, 0x2c, 0x7a, 0xa7, 0x22, 0xb0, 0xf8, 0x4b, 0xf5, 0x0b, + 0x05, 0xaa, 0x22, 0x65, 0xef, 0xc8, 0xa9, 0x32, 0x3b, 0x9a, 0xc2, 0x8e, 0xc6, 0xfe, 0x46, 0xff, + 0x1b, 0x1d, 0x49, 0x3d, 0x9f, 0x18, 0x04, 0x18, 0x13, 0x56, 0x1d, 0x47, 0xf2, 0x75, 0x9a, 0x5e, + 0xf6, 0x31, 0x35, 0x34, 0x71, 0x35, 0xcc, 0xd0, 0x1a, 0x30, 0xa3, 0x1b, 0x86, 0x8b, 0x09, 0x11, + 0x72, 0xf8, 0x4b, 0xfa, 0xe5, 0x21, 0x76, 0x89, 0x6f, 0xf2, 0x59, 0xcd, 0x5f, 0xa2, 0xb7, 0xa0, + 0x28, 0xcb, 0xe9, 0x6c, 0x52, 0x09, 0x15, 0x96, 0x53, 0xf4, 0x5e, 0x92, 0x42, 0xfd, 0x63, 0x06, + 0x6a, 0x42, 0x61, 0xcb, 0x22, 0xa7, 0x8e, 0x77, 0xbe, 0x65, 0xa8, 0xec, 0x07, 0xbe, 0x3f, 0x6e, + 0xc6, 0x12, 0x0e, 0x11, 0x11, 0x9a, 0x49, 0x0e, 0x18, 0xcd, 0xea, 0xb9, 0xa9, 0xb2, 0x7a, 0xfe, + 0xa4, 0x11, 0x6c, 0xb8, 0xce, 0x2b, 0x24, 0xd4, 0x79, 0xea, 0x77, 0xa0, 0x1c, 0x62, 0xc0, 0x22, + 0x34, 0x1f, 0xce, 0x08, 0x8d, 0xf9, 0x4b, 0x74, 0x2f, 0xa8, 0x6d, 0xb8, 0xaa, 0x2e, 0x26, 0xc8, + 0x12, 0x2b, 0x6b, 0xd4, 0xdf, 0x2a, 0x50, 0x10, 0x9c, 0xaf, 0x42, 0x59, 0x04, 0x1d, 0x56, 0xf7, + 0x71, 0xee, 0x20, 0x40, 0xb4, 0xf0, 0x7b, 0x72, 0x51, 0xe7, 0x22, 0x14, 0x63, 0xf1, 0x66, 0x46, + 0xa4, 0x05, 0xff, 0x53, 0x28, 0xc8, 0xd0, 0x4f, 0x2c, 0xbe, 0x7c, 0xad, 0xb0, 0xc1, 0xb2, 0x86, + 0x3b, 0xce, 0x43, 0xec, 0x1e, 0x4f, 0x3f, 0xbe, 0x7b, 0x33, 0x64, 0xd0, 0x29, 0xfb, 0x43, 0x49, + 0x80, 0xde, 0x0c, 0xd4, 0x9d, 0x4d, 0x9a, 0x5e, 0x84, 0x23, 0x8c, 0x30, 0xc7, 0x40, 0xed, 0x3f, + 0xe1, 0x83, 0xc8, 0xe8, 0x51, 0x4e, 0x5b, 0xd7, 0x3c, 0x91, 0xb6, 0x43, 0xfd, 0x99, 0x02, 0x17, + 0xd7, 0xb1, 0xb7, 0x16, 0x6d, 0xee, 0x9f, 0xb5, 0x54, 0x16, 0x34, 0x93, 0x84, 0x9a, 0xe6, 0xd6, + 0x9b, 0x50, 0x94, 0x63, 0x0a, 0x3e, 0x22, 0x96, 0x6b, 0xf5, 0xfb, 0x0a, 0x34, 0xc4, 0x2e, 0x6c, + 0x4f, 0x5a, 0x52, 0xf7, 0xb0, 0x87, 0x8d, 0x6f, 0xba, 0x6f, 0xfe, 0xab, 0x02, 0xf5, 0x70, 0xc4, + 0x67, 0x41, 0xfb, 0x35, 0xc8, 0xb3, 0xf1, 0x84, 0x90, 0x60, 0xa2, 0xb1, 0x72, 0x6c, 0x1a, 0x32, + 0x58, 0x99, 0xb7, 0x2b, 0x93, 0x93, 0x58, 0x06, 0x69, 0x27, 0x7b, 0xf2, 0xb4, 0x23, 0xd2, 0xb0, + 0x33, 0xa0, 0x7c, 0xf9, 0xf8, 0x2f, 0x00, 0xa8, 0x5f, 0x65, 0xa0, 0x11, 0xf4, 0x23, 0xdf, 0x78, + 0xdc, 0x1f, 0x51, 0xad, 0x66, 0x9f, 0x50, 0xb5, 0x9a, 0x9b, 0x3e, 0xd6, 0xe7, 0x93, 0x62, 0xfd, + 0x5f, 0x32, 0x50, 0x0b, 0xb4, 0xb6, 0xdd, 0xd3, 0x6d, 0x34, 0x07, 0x85, 0x7e, 0x4f, 0x0f, 0xa6, + 0x8f, 0x62, 0x85, 0x76, 0x64, 0x9d, 0x13, 0xd5, 0xd3, 0x4b, 0x49, 0x77, 0x38, 0xe2, 0x22, 0xb4, + 0x18, 0x0b, 0xda, 0x0e, 0xf2, 0x86, 0x82, 0x35, 0xf5, 0xa2, 0xb6, 0xe2, 0xc6, 0x42, 0xfb, 0xf9, + 0xdb, 0x80, 0xc4, 0x0d, 0xb7, 0x4d, 0xbb, 0x4d, 0x70, 0xc7, 0xb1, 0x0d, 0x7e, 0xf7, 0x79, 0xad, + 0x2e, 0xbe, 0xb4, 0xec, 0x1d, 0x0e, 0x47, 0xaf, 0x41, 0xce, 0x3b, 0xee, 0xf3, 0x28, 0x5e, 0x4b, + 0x8c, 0x8e, 0x81, 0x5c, 0xbb, 0xc7, 0x7d, 0xac, 0x31, 0x74, 0x34, 0x0f, 0x40, 0x59, 0x79, 0xae, + 0xfe, 0x50, 0xa4, 0xc4, 0x9c, 0x16, 0x82, 0x50, 0x6b, 0xf6, 0x75, 0x38, 0xc3, 0x53, 0x87, 0x58, + 0xaa, 0x7f, 0xca, 0x40, 0x3d, 0x60, 0xa9, 0x61, 0x32, 0xe8, 0x79, 0x23, 0xf5, 0x37, 0xbe, 0x19, + 0x9c, 0x54, 0x37, 0xbc, 0x0b, 0x65, 0x71, 0x9f, 0x27, 0xb0, 0x07, 0xe0, 0x24, 0x1b, 0x63, 0x0c, + 0x34, 0xff, 0x84, 0x0c, 0xb4, 0x70, 0x42, 0x03, 0x55, 0x77, 0x60, 0xce, 0x8f, 0x7b, 0x01, 0xc2, + 0x26, 0xf6, 0xf4, 0x31, 0x05, 0xc7, 0x55, 0x28, 0xf3, 0x7c, 0xc6, 0x13, 0x39, 0x2f, 0xd5, 0x61, + 0x4f, 0x76, 0xb8, 0xea, 0x77, 0xe1, 0x3c, 0x8b, 0x1b, 0xf1, 0x51, 0x6e, 0x9a, 0xb9, 0xba, 0x2a, + 0x1b, 0x01, 0x5a, 0xf4, 0x73, 0xeb, 0x2e, 0x69, 0x11, 0x98, 0xba, 0x01, 0xcf, 0xc5, 0xf8, 0x4f, + 0x91, 0x17, 0x68, 0x29, 0x34, 0xb7, 0x13, 0xfd, 0x59, 0xf4, 0xf4, 0xd9, 0xef, 0x8a, 0x9c, 0xdc, + 0xb6, 0x4d, 0x23, 0x6e, 0x5f, 0x06, 0x7a, 0x07, 0x4a, 0x36, 0x3e, 0x6a, 0x87, 0x83, 0x6f, 0x8a, + 0x01, 0x5d, 0xd1, 0xc6, 0x47, 0xec, 0x2f, 0x75, 0x0b, 0x2e, 0x0c, 0x89, 0x3a, 0xcd, 0xd9, 0xff, + 0xac, 0xc0, 0xc5, 0x55, 0xd7, 0xe9, 0x7f, 0x64, 0xba, 0xde, 0x40, 0xef, 0x45, 0x7f, 0x18, 0x79, + 0x3a, 0x6d, 0xdc, 0xfb, 0xa1, 0x34, 0xcc, 0xe3, 0xf2, 0xed, 0x04, 0x73, 0x1d, 0x16, 0x4a, 0x1c, + 0x3a, 0x94, 0xb4, 0xff, 0x95, 0x4d, 0x12, 0x5e, 0xe0, 0x4d, 0x48, 0x36, 0x69, 0xaa, 0x94, 0xc4, + 0xa9, 0x4f, 0xf6, 0xb4, 0x53, 0x9f, 0x11, 0x9e, 0x9f, 0x7b, 0x42, 0x9e, 0x7f, 0xe2, 0x36, 0xe4, + 0x7d, 0x88, 0x4e, 0xe4, 0x58, 0xc8, 0x3d, 0xd5, 0x28, 0x6f, 0x19, 0x20, 0x98, 0x4e, 0x89, 0x57, + 0x2d, 0x69, 0xd8, 0x84, 0xa8, 0xe8, 0x6d, 0xc9, 0x28, 0xcb, 0xa6, 0xca, 0x91, 0x79, 0xc9, 0x07, + 0xd0, 0x4c, 0xb2, 0xd2, 0x69, 0x2c, 0xff, 0xab, 0x0c, 0x40, 0x8b, 0x4d, 0x87, 0x76, 0x75, 0x72, + 0x78, 0xba, 0x8a, 0xf2, 0x3a, 0x54, 0x03, 0x83, 0x09, 0xfc, 0x3d, 0x6c, 0x45, 0x06, 0x75, 0x09, + 0x59, 0xd8, 0x52, 0x9c, 0xa1, 0x62, 0xd7, 0x60, 0x7c, 0x42, 0x5e, 0xc3, 0x8d, 0x22, 0x16, 0xf4, + 0xd0, 0x25, 0x28, 0xb9, 0xce, 0x51, 0x9b, 0xba, 0x99, 0xc1, 0x72, 0x6b, 0x51, 0x2b, 0xba, 0xce, + 0x11, 0x75, 0x3e, 0x03, 0x5d, 0x80, 0x19, 0x4f, 0x27, 0x87, 0x94, 0x7f, 0x81, 0xa7, 0x3b, 0xba, + 0x6c, 0x19, 0xe8, 0x3c, 0xe4, 0xf7, 0xcd, 0x1e, 0x26, 0x8d, 0x19, 0xc6, 0x92, 0x2f, 0xd0, 0xeb, + 0xfe, 0xfb, 0x85, 0x62, 0xea, 0xdf, 0x5f, 0xf9, 0x13, 0x86, 0xaf, 0x15, 0x98, 0x0d, 0xb4, 0xc6, + 0x02, 0x10, 0x8d, 0x69, 0x2c, 0x9e, 0xad, 0x38, 0x06, 0x0f, 0x15, 0xb5, 0x11, 0x3f, 0xb1, 0x70, + 0x42, 0x1e, 0xb5, 0x02, 0x92, 0x71, 0x75, 0x39, 0x3d, 0x17, 0x3d, 0xb4, 0x69, 0xf8, 0xbf, 0xf0, + 0x14, 0x5c, 0xe7, 0xa8, 0x65, 0x48, 0x6d, 0xf0, 0x67, 0x5c, 0xbc, 0x0a, 0xa5, 0xda, 0x58, 0x61, + 0x2f, 0xb9, 0xae, 0x43, 0x15, 0xbb, 0xae, 0xe3, 0xb6, 0x2d, 0x4c, 0x88, 0xde, 0xc5, 0xa2, 0xe8, + 0xaa, 0x30, 0xe0, 0x26, 0x87, 0xa9, 0xff, 0xc9, 0x40, 0x2d, 0x38, 0x8a, 0xff, 0xbb, 0x8e, 0x69, + 0xf8, 0xbf, 0xeb, 0x98, 0x06, 0x0d, 0xe6, 0x2e, 0x0f, 0x85, 0xa1, 0x60, 0x2e, 0x20, 0x2d, 0x83, + 0xe6, 0x41, 0xea, 0x60, 0xb6, 0x63, 0xe0, 0xe0, 0x62, 0xc1, 0x07, 0x89, 0x7b, 0x8d, 0xd8, 0x47, + 0x2e, 0x85, 0x7d, 0xe4, 0x53, 0xd8, 0x47, 0x21, 0xc1, 0x3e, 0xe6, 0xa0, 0xb0, 0x37, 0xe8, 0x1c, + 0x62, 0x4f, 0x94, 0x47, 0x62, 0x15, 0xb5, 0x9b, 0x62, 0xcc, 0x6e, 0xa4, 0x79, 0x94, 0xc2, 0xe6, + 0x71, 0x09, 0x4a, 0xfc, 0xc7, 0x85, 0xb6, 0x47, 0xd8, 0x94, 0x35, 0xab, 0x15, 0x39, 0x60, 0x97, + 0xa0, 0x37, 0xfc, 0xde, 0xa1, 0x9c, 0xe4, 0xe8, 0x2c, 0xe2, 0xc4, 0x2c, 0x44, 0x74, 0x0e, 0xea, + 0x27, 0x80, 0x82, 0x2f, 0xd3, 0xf5, 0x72, 0x31, 0xd5, 0x67, 0xe2, 0xaa, 0x57, 0x7f, 0xa7, 0xc0, + 0xd9, 0xf0, 0x66, 0xa7, 0x4d, 0x68, 0xef, 0x40, 0x99, 0xcf, 0x90, 0xdb, 0xd4, 0xa1, 0x44, 0x37, + 0x77, 0x65, 0xec, 0x99, 0x35, 0x30, 0x83, 0xb8, 0x72, 0x1d, 0xaa, 0x47, 0x8e, 0x7b, 0x68, 0xda, + 0xdd, 0x36, 0x95, 0xcc, 0x37, 0xe3, 0x8a, 0x00, 0x6e, 0x51, 0x98, 0xfa, 0xa5, 0x02, 0xf3, 0x1f, + 0xf6, 0x0d, 0xdd, 0xc3, 0xa1, 0xcc, 0x3e, 0xed, 0x03, 0x8f, 0xd7, 0xfc, 0x37, 0x16, 0x99, 0x74, + 0x73, 0x50, 0x8e, 0x7d, 0xeb, 0xe7, 0x0a, 0x9c, 0x1d, 0xea, 0xfd, 0x50, 0x0d, 0xe0, 0x43, 0xbb, + 0x23, 0x9a, 0xe2, 0xfa, 0x19, 0x54, 0x81, 0xa2, 0xdf, 0x22, 0xd7, 0x15, 0x54, 0x86, 0x99, 0x5d, + 0x87, 0x61, 0xd7, 0x33, 0xa8, 0x0e, 0x15, 0x4e, 0x38, 0xe8, 0x74, 0x30, 0x21, 0xf5, 0xac, 0x84, + 0xac, 0xe9, 0x66, 0x6f, 0xe0, 0xe2, 0x7a, 0x0e, 0x55, 0xa1, 0xb4, 0xeb, 0x68, 0xb8, 0x87, 0x75, + 0x82, 0xeb, 0x79, 0x84, 0xa0, 0x26, 0x16, 0x3e, 0x51, 0x21, 0x04, 0xf3, 0xc9, 0x66, 0x6e, 0xed, + 0x87, 0xbb, 0x24, 0xda, 0x3a, 0xa0, 0x0b, 0x70, 0xee, 0x43, 0xdb, 0xc0, 0xfb, 0xa6, 0x8d, 0x8d, + 0xe0, 0x53, 0xfd, 0x0c, 0x3a, 0x07, 0xb3, 0x2d, 0xdb, 0xc6, 0x6e, 0x08, 0xa8, 0x50, 0xe0, 0x26, + 0x76, 0xbb, 0x38, 0x04, 0xcc, 0xa0, 0xb3, 0x50, 0xdd, 0x34, 0x1f, 0x85, 0x40, 0xd9, 0xa5, 0xbf, + 0x3d, 0x07, 0xa5, 0x55, 0xdd, 0xd3, 0x57, 0x1c, 0xc7, 0x35, 0x50, 0x1f, 0x10, 0x7b, 0x0e, 0x64, + 0xf5, 0x1d, 0x5b, 0xbe, 0x9b, 0x43, 0xaf, 0x8c, 0x48, 0x70, 0xc3, 0xa8, 0xe2, 0x0a, 0x9b, 0x37, + 0x47, 0x50, 0xc4, 0xd0, 0xd5, 0x33, 0xc8, 0x62, 0x3b, 0xd2, 0xd6, 0x6b, 0xd7, 0xec, 0x1c, 0xfa, + 0x3f, 0xfc, 0x8e, 0xd9, 0x31, 0x86, 0xea, 0xef, 0x18, 0x7b, 0x8e, 0x27, 0x16, 0xfc, 0xcd, 0x96, + 0xef, 0x7f, 0xea, 0x19, 0xf4, 0x29, 0x9c, 0x5f, 0xc7, 0x21, 0x9b, 0xf3, 0x37, 0x5c, 0x1a, 0xbd, + 0xe1, 0x10, 0xf2, 0x09, 0xb7, 0xdc, 0x80, 0x3c, 0x9b, 0xb3, 0xa0, 0x24, 0xb3, 0x0c, 0x3f, 0x1e, + 0x6f, 0x2e, 0x8c, 0x46, 0x90, 0xdc, 0x3e, 0x81, 0xd9, 0xd8, 0xe3, 0x58, 0xf4, 0x62, 0x02, 0x59, + 0xf2, 0x33, 0xe7, 0xe6, 0xad, 0x34, 0xa8, 0x72, 0xaf, 0x2e, 0xd4, 0xa2, 0x8f, 0x89, 0xd0, 0x62, + 0x02, 0x7d, 0xe2, 0xc3, 0xc6, 0xe6, 0x8b, 0x29, 0x30, 0xe5, 0x46, 0x16, 0xd4, 0xe3, 0x8f, 0x35, + 0xd1, 0xad, 0xb1, 0x0c, 0xa2, 0xe6, 0xf6, 0x52, 0x2a, 0x5c, 0xb9, 0xdd, 0x31, 0x33, 0x82, 0xa1, + 0xc7, 0x82, 0xe8, 0x4e, 0x32, 0x9b, 0x51, 0xaf, 0x18, 0x9b, 0x77, 0x53, 0xe3, 0xcb, 0xad, 0xbf, + 0xc7, 0xe7, 0xbb, 0x49, 0x0f, 0xee, 0xd0, 0xab, 0xc9, 0xec, 0xc6, 0xbc, 0x14, 0x6c, 0x2e, 0x9d, + 0x84, 0x44, 0x0a, 0xf1, 0x39, 0x1b, 0xcc, 0x26, 0x3c, 0x5a, 0x8b, 0xfb, 0x9d, 0xcf, 0x6f, 0xf4, + 0x6b, 0xbc, 0xe6, 0xab, 0x27, 0xa0, 0x90, 0x02, 0x38, 0xf1, 0xe7, 0xb0, 0xbe, 0x1b, 0xde, 0x9d, + 0x68, 0x35, 0xa7, 0xf3, 0xc1, 0x8f, 0x61, 0x36, 0xf6, 0x13, 0x7b, 0xa2, 0xd7, 0x24, 0xff, 0x0c, + 0xdf, 0x1c, 0x97, 0xa6, 0xb9, 0x4b, 0xc6, 0xe6, 0xdc, 0x68, 0x84, 0xf5, 0x27, 0xcc, 0xc2, 0x9b, + 0xb7, 0xd2, 0xa0, 0xca, 0x83, 0x10, 0x16, 0x2e, 0x63, 0xb3, 0x62, 0x74, 0x3b, 0x99, 0x47, 0xf2, + 0x9c, 0xbb, 0xf9, 0x72, 0x4a, 0x6c, 0xb9, 0x69, 0x1b, 0x60, 0x1d, 0x7b, 0x9b, 0xd8, 0x73, 0xa9, + 0x8d, 0xdc, 0x4c, 0x54, 0x79, 0x80, 0xe0, 0x6f, 0xf3, 0xc2, 0x44, 0x3c, 0xb9, 0xc1, 0xb7, 0x00, + 0xf9, 0x29, 0x36, 0xf4, 0xc0, 0xe3, 0xfa, 0xd8, 0x71, 0x1a, 0x9f, 0x7d, 0x4d, 0xba, 0x9b, 0x4f, + 0xa1, 0xbe, 0xa9, 0xdb, 0xb4, 0x91, 0x0a, 0xf8, 0xde, 0x4e, 0x14, 0x2c, 0x8e, 0x36, 0x42, 0x5b, + 0x23, 0xb1, 0xe5, 0x61, 0x8e, 0x64, 0x0e, 0xd5, 0xa5, 0x0b, 0xe2, 0x78, 0x6c, 0x09, 0xb4, 0x11, + 0x43, 0x1c, 0x11, 0x5b, 0xc6, 0xe0, 0xcb, 0x8d, 0x1f, 0x2b, 0xec, 0xd1, 0x75, 0x0c, 0xe1, 0x81, + 0xe9, 0x1d, 0x6c, 0xf7, 0x74, 0x9b, 0xa4, 0x11, 0x81, 0x21, 0x9e, 0x40, 0x04, 0x81, 0x2f, 0x45, + 0x30, 0xa0, 0x1a, 0x99, 0x56, 0xa1, 0xa4, 0x57, 0x1a, 0x49, 0xf3, 0xb2, 0xe6, 0xe2, 0x64, 0x44, + 0xb9, 0xcb, 0x01, 0x54, 0x7d, 0x7b, 0xe5, 0xca, 0x7d, 0x71, 0x94, 0xa4, 0x01, 0xce, 0x08, 0x77, + 0x4b, 0x46, 0x0d, 0xbb, 0xdb, 0x70, 0x33, 0x8e, 0xd2, 0x0d, 0x71, 0xc6, 0xb9, 0xdb, 0xe8, 0x0e, + 0x9f, 0xc7, 0x93, 0xd8, 0xe0, 0x2b, 0x39, 0x58, 0x25, 0xce, 0xf1, 0x12, 0xe3, 0xc9, 0x88, 0x39, + 0x9a, 0x7a, 0x06, 0x3d, 0x80, 0x02, 0xaf, 0xe6, 0xd1, 0xf3, 0xe3, 0x0b, 0x7d, 0xc1, 0xfd, 0xc6, + 0x04, 0x2c, 0xc9, 0xf8, 0x10, 0x2e, 0x8c, 0x28, 0xf3, 0x13, 0xf3, 0xdc, 0xf8, 0x96, 0x60, 0x82, + 0x97, 0x2f, 0xfd, 0x3a, 0x0f, 0x45, 0xff, 0xa7, 0xff, 0x67, 0x50, 0xc3, 0x3e, 0x83, 0xa2, 0xf2, + 0x63, 0x98, 0x8d, 0xbd, 0x21, 0x4e, 0xb4, 0x91, 0xe4, 0x77, 0xc6, 0x93, 0x82, 0xe6, 0x03, 0xf1, + 0x9f, 0x85, 0x32, 0xbf, 0xbc, 0x30, 0xaa, 0x30, 0x8d, 0xa7, 0x96, 0x09, 0x8c, 0x9f, 0x7a, 0x22, + 0xd9, 0x02, 0x08, 0x05, 0xfa, 0xf1, 0xbf, 0xc7, 0xd0, 0xd8, 0x35, 0x49, 0xe0, 0xcd, 0x13, 0xba, + 0xc7, 0x78, 0x76, 0xcb, 0xf7, 0xbe, 0xfd, 0x6a, 0xd7, 0xf4, 0x0e, 0x06, 0x7b, 0xf4, 0xcb, 0x5d, + 0x8e, 0xfa, 0xb2, 0xe9, 0x88, 0xbf, 0xee, 0xfa, 0x06, 0x72, 0x97, 0x51, 0xdf, 0xa5, 0x7b, 0xf4, + 0xf7, 0xf6, 0x0a, 0x6c, 0x75, 0xef, 0xbf, 0x01, 0x00, 0x00, 0xff, 0xff, 0xd1, 0xa4, 0x8b, 0x98, + 0xd3, 0x3a, 0x00, 0x00, } // Reference imports to suppress errors if they are not otherwise used. diff --git a/internal/proto/milvus.proto b/internal/proto/milvus.proto index 2b164db90c..20ecc0d596 100644 --- a/internal/proto/milvus.proto +++ b/internal/proto/milvus.proto @@ -343,6 +343,8 @@ message LoadPartitionsRequest { string collection_name = 3; // The partition names you want to load repeated string partition_names = 4; + // The replicas number you would load, 1 by default + int32 replica_number = 5; } /* @@ -755,6 +757,7 @@ message LoadBalanceRequest { int64 src_nodeID = 2; repeated int64 dst_nodeIDs = 3; repeated int64 sealed_segmentIDs = 4; + string collectionName = 5; } message ManualCompactionRequest { @@ -857,8 +860,6 @@ message ShardReplica { repeated int64 node_ids = 4; } - - service ProxyService { rpc RegisterLink(RegisterLinkRequest) returns (RegisterLinkResponse) {} } @@ -911,3 +912,4 @@ message ListCredUsersRequest { // Not useful for now common.MsgBase base = 1; } + diff --git a/internal/proto/milvuspb/milvus.pb.go b/internal/proto/milvuspb/milvus.pb.go index d386bf2bd8..ea0f094937 100644 --- a/internal/proto/milvuspb/milvus.pb.go +++ b/internal/proto/milvuspb/milvus.pb.go @@ -1417,7 +1417,9 @@ type LoadPartitionsRequest struct { // The collection name in milvus CollectionName string `protobuf:"bytes,3,opt,name=collection_name,json=collectionName,proto3" json:"collection_name,omitempty"` // The partition names you want to load - PartitionNames []string `protobuf:"bytes,4,rep,name=partition_names,json=partitionNames,proto3" json:"partition_names,omitempty"` + PartitionNames []string `protobuf:"bytes,4,rep,name=partition_names,json=partitionNames,proto3" json:"partition_names,omitempty"` + // The replicas number you would load, 1 by default + ReplicaNumber int32 `protobuf:"varint,5,opt,name=replica_number,json=replicaNumber,proto3" json:"replica_number,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -1476,6 +1478,13 @@ func (m *LoadPartitionsRequest) GetPartitionNames() []string { return nil } +func (m *LoadPartitionsRequest) GetReplicaNumber() int32 { + if m != nil { + return m.ReplicaNumber + } + return 0 +} + // // Release specific partitions data of one collection from query nodes. // Then you can not get these data as result when you do vector search on this collection. @@ -4474,6 +4483,7 @@ type LoadBalanceRequest struct { SrcNodeID int64 `protobuf:"varint,2,opt,name=src_nodeID,json=srcNodeID,proto3" json:"src_nodeID,omitempty"` DstNodeIDs []int64 `protobuf:"varint,3,rep,packed,name=dst_nodeIDs,json=dstNodeIDs,proto3" json:"dst_nodeIDs,omitempty"` SealedSegmentIDs []int64 `protobuf:"varint,4,rep,packed,name=sealed_segmentIDs,json=sealedSegmentIDs,proto3" json:"sealed_segmentIDs,omitempty"` + CollectionName string `protobuf:"bytes,5,opt,name=collectionName,proto3" json:"collectionName,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -4532,6 +4542,13 @@ func (m *LoadBalanceRequest) GetSealedSegmentIDs() []int64 { return nil } +func (m *LoadBalanceRequest) GetCollectionName() string { + if m != nil { + return m.CollectionName + } + return "" +} + type ManualCompactionRequest struct { CollectionID int64 `protobuf:"varint,1,opt,name=collectionID,proto3" json:"collectionID,omitempty"` Timetravel uint64 `protobuf:"varint,2,opt,name=timetravel,proto3" json:"timetravel,omitempty"` @@ -5834,273 +5851,273 @@ func init() { func init() { proto.RegisterFile("milvus.proto", fileDescriptor_02345ba45cc0e303) } var fileDescriptor_02345ba45cc0e303 = []byte{ - // 4247 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xd4, 0x7c, 0x4b, 0x8f, 0x1c, 0xc9, - 0x71, 0xf0, 0x54, 0xf7, 0xf4, 0x2b, 0xba, 0x7b, 0xa6, 0x99, 0xf3, 0xea, 0x6d, 0x92, 0xcb, 0x61, - 0xed, 0x6b, 0x96, 0xd4, 0x92, 0xda, 0xe1, 0x6a, 0x57, 0xdf, 0xae, 0x3e, 0xaf, 0x48, 0x8e, 0x97, - 0x1c, 0x2c, 0x49, 0x8f, 0x6a, 0x76, 0x25, 0xc8, 0x0b, 0xa2, 0x91, 0x53, 0x95, 0xd3, 0x53, 0x66, - 0x75, 0x55, 0xab, 0x32, 0x9b, 0xc3, 0xd9, 0x93, 0x00, 0x19, 0x7e, 0x40, 0xeb, 0x15, 0x0c, 0x0b, - 0xb6, 0x75, 0xb0, 0x61, 0xf8, 0x71, 0xf0, 0xc1, 0x86, 0x25, 0x19, 0xb6, 0xe0, 0x8b, 0x7d, 0xf0, - 0xc1, 0x07, 0x03, 0x7e, 0x5c, 0x0c, 0xc3, 0x17, 0xff, 0x01, 0x1f, 0x0c, 0xf8, 0xe8, 0x83, 0x91, - 0x8f, 0xaa, 0xae, 0xaa, 0xce, 0xea, 0xe9, 0x61, 0x8b, 0x9a, 0xe1, 0xad, 0x2b, 0x32, 0x22, 0x33, - 0x32, 0x32, 0x32, 0x32, 0x32, 0x22, 0xb2, 0xa1, 0xd1, 0x77, 0xbd, 0xc7, 0x43, 0x7a, 0x6d, 0x10, - 0x06, 0x2c, 0x40, 0x4b, 0xc9, 0xaf, 0x6b, 0xf2, 0xa3, 0xd3, 0xb0, 0x83, 0x7e, 0x3f, 0xf0, 0x25, - 0xb0, 0xd3, 0xa0, 0xf6, 0x01, 0xe9, 0x63, 0xf9, 0x65, 0xfe, 0xbe, 0x01, 0xe8, 0x76, 0x48, 0x30, - 0x23, 0x37, 0x3d, 0x17, 0x53, 0x8b, 0x7c, 0x6b, 0x48, 0x28, 0x43, 0x5f, 0x84, 0xf9, 0x3d, 0x4c, - 0x49, 0xdb, 0x58, 0x37, 0x36, 0xea, 0x9b, 0x17, 0xae, 0xa5, 0xba, 0x55, 0xdd, 0xdd, 0xa7, 0xbd, - 0x5b, 0x98, 0x12, 0x4b, 0x60, 0xa2, 0x35, 0xa8, 0x38, 0x7b, 0x5d, 0x1f, 0xf7, 0x49, 0xbb, 0xb0, - 0x6e, 0x6c, 0xd4, 0xac, 0xb2, 0xb3, 0xf7, 0x00, 0xf7, 0x09, 0x7a, 0x0d, 0x16, 0xed, 0xc0, 0xf3, - 0x88, 0xcd, 0xdc, 0xc0, 0x97, 0x08, 0x45, 0x81, 0xb0, 0x30, 0x02, 0x0b, 0xc4, 0x65, 0x28, 0x61, - 0xce, 0x43, 0x7b, 0x5e, 0x34, 0xcb, 0x0f, 0x93, 0x42, 0x6b, 0x2b, 0x0c, 0x06, 0xcf, 0x8a, 0xbb, - 0x78, 0xd0, 0x62, 0x72, 0xd0, 0xdf, 0x33, 0xe0, 0xdc, 0x4d, 0x8f, 0x91, 0xf0, 0x8c, 0x0a, 0xe5, - 0x77, 0x0b, 0xb0, 0x26, 0x57, 0xed, 0x76, 0x8c, 0x7e, 0x9a, 0x5c, 0xae, 0x42, 0x59, 0x6a, 0x95, - 0x60, 0xb3, 0x61, 0xa9, 0x2f, 0x74, 0x11, 0x80, 0x1e, 0xe0, 0xd0, 0xa1, 0x5d, 0x7f, 0xd8, 0x6f, - 0x97, 0xd6, 0x8d, 0x8d, 0x92, 0x55, 0x93, 0x90, 0x07, 0xc3, 0x3e, 0xb2, 0xe0, 0x9c, 0x1d, 0xf8, - 0xd4, 0xa5, 0x8c, 0xf8, 0xf6, 0x51, 0xd7, 0x23, 0x8f, 0x89, 0xd7, 0x2e, 0xaf, 0x1b, 0x1b, 0x0b, - 0x9b, 0xaf, 0x68, 0xf9, 0xbe, 0x3d, 0xc2, 0xbe, 0xc7, 0x91, 0xad, 0x96, 0x9d, 0x81, 0x98, 0xdf, - 0x35, 0x60, 0x85, 0x2b, 0xcc, 0x99, 0x10, 0x8c, 0xf9, 0xa7, 0x06, 0x2c, 0xdf, 0xc5, 0xf4, 0x6c, - 0xac, 0xd2, 0x45, 0x00, 0xe6, 0xf6, 0x49, 0x97, 0x32, 0xdc, 0x1f, 0x88, 0x95, 0x9a, 0xb7, 0x6a, - 0x1c, 0xb2, 0xcb, 0x01, 0xe6, 0x37, 0xa1, 0x71, 0x2b, 0x08, 0x3c, 0x8b, 0xd0, 0x41, 0xe0, 0x53, - 0x82, 0x6e, 0x40, 0x99, 0x32, 0xcc, 0x86, 0x54, 0x31, 0x79, 0x5e, 0xcb, 0xe4, 0xae, 0x40, 0xb1, - 0x14, 0x2a, 0xd7, 0xd7, 0xc7, 0xd8, 0x1b, 0x4a, 0x1e, 0xab, 0x96, 0xfc, 0x30, 0x3f, 0x81, 0x85, - 0x5d, 0x16, 0xba, 0x7e, 0xef, 0xa7, 0xd8, 0x79, 0x2d, 0xea, 0xfc, 0x5f, 0x0d, 0x78, 0x61, 0x8b, - 0x50, 0x3b, 0x74, 0xf7, 0xce, 0xc8, 0x76, 0x30, 0xa1, 0x31, 0x82, 0x6c, 0x6f, 0x09, 0x51, 0x17, - 0xad, 0x14, 0x2c, 0xb3, 0x18, 0xa5, 0xec, 0x62, 0x7c, 0xbb, 0x04, 0x1d, 0xdd, 0xa4, 0x66, 0x11, - 0xdf, 0xff, 0x8f, 0x77, 0x69, 0x41, 0x10, 0x65, 0xf6, 0x98, 0x3a, 0x17, 0x46, 0xa3, 0xed, 0x0a, - 0x40, 0xbc, 0x99, 0xb3, 0xb3, 0x2a, 0x6a, 0x66, 0xb5, 0x09, 0x2b, 0x8f, 0xdd, 0x90, 0x0d, 0xb1, - 0xd7, 0xb5, 0x0f, 0xb0, 0xef, 0x13, 0x4f, 0xc8, 0x89, 0x9b, 0xaf, 0xe2, 0x46, 0xcd, 0x5a, 0x52, - 0x8d, 0xb7, 0x65, 0x1b, 0x17, 0x16, 0x45, 0x6f, 0xc1, 0xea, 0xe0, 0xe0, 0x88, 0xba, 0xf6, 0x18, - 0x51, 0x49, 0x10, 0x2d, 0x47, 0xad, 0x29, 0xaa, 0xab, 0x70, 0xce, 0x16, 0x16, 0xd0, 0xe9, 0x72, - 0xa9, 0x49, 0x31, 0x96, 0x85, 0x18, 0x5b, 0xaa, 0xe1, 0xa3, 0x08, 0xce, 0xd9, 0x8a, 0x90, 0x87, - 0xcc, 0x4e, 0x10, 0x54, 0x04, 0xc1, 0x92, 0x6a, 0xfc, 0x98, 0xd9, 0x23, 0x9a, 0xb4, 0xed, 0xaa, - 0x66, 0x6d, 0x57, 0x1b, 0x2a, 0xc2, 0x16, 0x13, 0xda, 0xae, 0x09, 0x36, 0xa3, 0x4f, 0xb4, 0x0d, - 0x8b, 0x94, 0xe1, 0x90, 0x75, 0x07, 0x01, 0x75, 0xb9, 0x5c, 0x68, 0x1b, 0xd6, 0x8b, 0x1b, 0xf5, - 0xcd, 0x75, 0xed, 0x22, 0x7d, 0x48, 0x8e, 0xb6, 0x30, 0xc3, 0x3b, 0xd8, 0x0d, 0xad, 0x05, 0x41, - 0xb8, 0x13, 0xd1, 0xe9, 0x0d, 0x64, 0x7d, 0x26, 0x03, 0xa9, 0xd3, 0xe2, 0x86, 0xd6, 0x76, 0xfd, - 0xc8, 0x80, 0x95, 0x7b, 0x01, 0x76, 0xce, 0xc6, 0x9e, 0x7a, 0x05, 0x16, 0x42, 0x32, 0xf0, 0x5c, - 0x1b, 0xf3, 0xf5, 0xd8, 0x23, 0xa1, 0xd8, 0x55, 0x25, 0xab, 0xa9, 0xa0, 0x0f, 0x04, 0xd0, 0xfc, - 0xdc, 0x80, 0xb6, 0x45, 0x3c, 0x82, 0xe9, 0xd9, 0xb0, 0x05, 0xe6, 0xf7, 0x0d, 0x78, 0xf1, 0x0e, - 0x61, 0x89, 0x5d, 0xc5, 0x30, 0x73, 0x29, 0x73, 0xed, 0xd3, 0xf4, 0x2b, 0xcc, 0xef, 0x19, 0x70, - 0x29, 0x97, 0xad, 0x59, 0x8c, 0xcc, 0x3b, 0x50, 0xe2, 0xbf, 0x68, 0xbb, 0x20, 0x74, 0xfe, 0x72, - 0x9e, 0xce, 0x7f, 0x9d, 0xdb, 0x6e, 0xa1, 0xf4, 0x12, 0xdf, 0xfc, 0x4f, 0x03, 0x56, 0x77, 0x0f, - 0x82, 0xc3, 0x11, 0x4b, 0xcf, 0x42, 0x40, 0x69, 0xb3, 0x5b, 0xcc, 0x98, 0x5d, 0xf4, 0x26, 0xcc, - 0xb3, 0xa3, 0x01, 0x11, 0xba, 0xb5, 0xb0, 0x79, 0xf1, 0x9a, 0xc6, 0x9d, 0xbe, 0xc6, 0x99, 0xfc, - 0xe8, 0x68, 0x40, 0x2c, 0x81, 0x8a, 0x5e, 0x87, 0x56, 0x46, 0xe4, 0x91, 0xe1, 0x5a, 0x4c, 0xcb, - 0x9c, 0x9a, 0x3f, 0x29, 0xc0, 0xda, 0xd8, 0x14, 0x67, 0x11, 0xb6, 0x6e, 0xec, 0x82, 0x76, 0x6c, - 0xbe, 0x7f, 0x12, 0xa8, 0xae, 0xc3, 0x3d, 0xde, 0xe2, 0x46, 0xd1, 0x6a, 0x26, 0xec, 0xb7, 0x43, - 0xd1, 0x1b, 0x80, 0xc6, 0xcc, 0xaa, 0xb4, 0xde, 0xf3, 0xd6, 0xb9, 0xac, 0x5d, 0x15, 0xb6, 0x5b, - 0x6b, 0x58, 0xa5, 0x08, 0xe6, 0xad, 0x65, 0x8d, 0x65, 0xa5, 0xe8, 0x4d, 0x58, 0x76, 0xfd, 0xfb, - 0xa4, 0x1f, 0x84, 0x47, 0xdd, 0x01, 0x09, 0x6d, 0xe2, 0x33, 0xdc, 0x23, 0xb4, 0x5d, 0x16, 0x1c, - 0x2d, 0x45, 0x6d, 0x3b, 0xa3, 0x26, 0xf3, 0xc7, 0x06, 0xac, 0x4a, 0x8f, 0x77, 0x07, 0x87, 0xcc, - 0x3d, 0x03, 0xd6, 0x68, 0x10, 0xf1, 0x21, 0xf1, 0xa4, 0x7f, 0xde, 0x8c, 0xa1, 0x62, 0x97, 0xfd, - 0xd0, 0x80, 0x65, 0xee, 0x8c, 0x3e, 0x4f, 0x3c, 0xff, 0x85, 0x01, 0x4b, 0x77, 0x31, 0x7d, 0x9e, - 0x58, 0xfe, 0x4b, 0x75, 0x52, 0xc5, 0x3c, 0x9f, 0xea, 0x95, 0xed, 0x35, 0x58, 0x4c, 0x33, 0x1d, - 0x79, 0x3f, 0x0b, 0x29, 0xae, 0xa9, 0xf9, 0xd7, 0xa3, 0xb3, 0xea, 0x39, 0xe3, 0xfc, 0x6f, 0x0c, - 0xb8, 0x78, 0x87, 0xb0, 0x98, 0xeb, 0x33, 0x71, 0xa6, 0x4d, 0xab, 0x2d, 0x9f, 0xcb, 0x13, 0x59, - 0xcb, 0xfc, 0xa9, 0x9c, 0x7c, 0xdf, 0x2d, 0xc0, 0x0a, 0x3f, 0x16, 0xce, 0x86, 0x12, 0x4c, 0x73, - 0x79, 0xd1, 0x28, 0x4a, 0x49, 0xa7, 0x28, 0xf1, 0x79, 0x5a, 0x9e, 0xfa, 0x3c, 0x35, 0x7f, 0x54, - 0x90, 0x7e, 0x40, 0x52, 0x1a, 0xb3, 0x2c, 0x8b, 0x86, 0xd7, 0x82, 0x96, 0x57, 0x13, 0x1a, 0x31, - 0x64, 0x7b, 0x2b, 0x3a, 0x1f, 0x53, 0xb0, 0x33, 0x7b, 0x3c, 0x7e, 0x66, 0xc0, 0x6a, 0x74, 0x5d, - 0xdc, 0x25, 0xbd, 0x3e, 0xf1, 0xd9, 0xd3, 0xeb, 0x50, 0x56, 0x03, 0x0a, 0x1a, 0x0d, 0xb8, 0x00, - 0x35, 0x2a, 0xc7, 0x89, 0x6f, 0x82, 0x23, 0x80, 0xf9, 0xb7, 0x06, 0xac, 0x8d, 0xb1, 0x33, 0xcb, - 0x22, 0xb6, 0xa1, 0xe2, 0xfa, 0x0e, 0x79, 0x12, 0x73, 0x13, 0x7d, 0xf2, 0x96, 0xbd, 0xa1, 0xeb, - 0x39, 0x31, 0x1b, 0xd1, 0x27, 0xba, 0x0c, 0x0d, 0xe2, 0xe3, 0x3d, 0x8f, 0x74, 0x05, 0xae, 0x50, - 0xe4, 0xaa, 0x55, 0x97, 0xb0, 0x6d, 0x0e, 0xe2, 0xc4, 0xfb, 0x2e, 0x11, 0xc4, 0x25, 0x49, 0xac, - 0x3e, 0xcd, 0xdf, 0x30, 0x60, 0x89, 0x6b, 0xa1, 0xe2, 0x9e, 0x3e, 0x5b, 0x69, 0xae, 0x43, 0x3d, - 0xa1, 0x66, 0x6a, 0x22, 0x49, 0x90, 0xf9, 0x08, 0x96, 0xd3, 0xec, 0xcc, 0x22, 0xcd, 0x17, 0x01, - 0xe2, 0xb5, 0x92, 0xbb, 0xa1, 0x68, 0x25, 0x20, 0xe6, 0x67, 0x85, 0x28, 0x28, 0x2c, 0xc4, 0x74, - 0xca, 0x31, 0x2b, 0xb1, 0x24, 0x49, 0x7b, 0x5e, 0x13, 0x10, 0xd1, 0xbc, 0x05, 0x0d, 0xf2, 0x84, - 0x85, 0xb8, 0x3b, 0xc0, 0x21, 0xee, 0xcb, 0x6d, 0x35, 0x95, 0xe9, 0xad, 0x0b, 0xb2, 0x1d, 0x41, - 0xc5, 0x07, 0x11, 0x2a, 0x22, 0x07, 0x29, 0xcb, 0x41, 0x04, 0x44, 0x1c, 0x18, 0xff, 0xc0, 0xbd, - 0x38, 0xa5, 0xcd, 0x67, 0x5d, 0x20, 0xe9, 0xa9, 0x94, 0xb2, 0x53, 0xf9, 0x13, 0x03, 0x5a, 0x62, - 0x0a, 0x72, 0x3e, 0x03, 0xde, 0x6d, 0x86, 0xc6, 0xc8, 0xd0, 0x4c, 0xd8, 0x7b, 0xff, 0x0f, 0xca, - 0x4a, 0xee, 0xc5, 0x69, 0xe5, 0xae, 0x08, 0x8e, 0x99, 0x86, 0xf9, 0x87, 0x06, 0xac, 0x64, 0x44, - 0x3e, 0x8b, 0xc2, 0x7f, 0x04, 0x48, 0xce, 0xd0, 0x19, 0x4d, 0x3b, 0x3a, 0xa7, 0x5f, 0xd1, 0x1e, - 0x4a, 0x59, 0x21, 0x59, 0xe7, 0xdc, 0x0c, 0x84, 0x9a, 0xff, 0x6c, 0xc0, 0x85, 0x3b, 0x84, 0x09, - 0xd4, 0x5b, 0xdc, 0xe8, 0xec, 0x84, 0x41, 0x2f, 0x24, 0x94, 0x3e, 0xbf, 0xfa, 0xf1, 0xdb, 0xd2, - 0xb1, 0xd3, 0x4d, 0x69, 0x16, 0xf9, 0x5f, 0x86, 0x86, 0x18, 0x83, 0x38, 0xdd, 0x30, 0x38, 0xa4, - 0x4a, 0x8f, 0xea, 0x0a, 0x66, 0x05, 0x87, 0x42, 0x21, 0x58, 0xc0, 0xb0, 0x27, 0x11, 0xd4, 0x89, - 0x22, 0x20, 0xbc, 0x59, 0xec, 0xc1, 0x88, 0x31, 0xde, 0x39, 0x79, 0x7e, 0x65, 0xfc, 0xc7, 0x06, - 0xac, 0x64, 0xa6, 0x32, 0x8b, 0x6c, 0xbf, 0x24, 0xdd, 0x4e, 0x39, 0x99, 0x85, 0xcd, 0x4b, 0x5a, - 0x9a, 0xc4, 0x60, 0x12, 0x1b, 0x5d, 0x82, 0xfa, 0x3e, 0x76, 0xbd, 0x6e, 0x48, 0x30, 0x0d, 0x7c, - 0x35, 0x51, 0xe0, 0x20, 0x4b, 0x40, 0xcc, 0xbf, 0x37, 0x64, 0xe6, 0xed, 0x39, 0xb7, 0x78, 0x7f, - 0x54, 0x80, 0xe6, 0xb6, 0x4f, 0x49, 0xc8, 0xce, 0xfe, 0xd5, 0x04, 0xbd, 0x0f, 0x75, 0x31, 0x31, - 0xda, 0x75, 0x30, 0xc3, 0xea, 0x34, 0x7b, 0x51, 0x1b, 0xa6, 0xff, 0x80, 0xe3, 0x6d, 0x61, 0x86, - 0x2d, 0x29, 0x1d, 0xca, 0x7f, 0xa3, 0xf3, 0x50, 0x3b, 0xc0, 0xf4, 0xa0, 0xfb, 0x88, 0x1c, 0x49, - 0x7f, 0xb1, 0x69, 0x55, 0x39, 0xe0, 0x43, 0x72, 0x44, 0xd1, 0x0b, 0x50, 0xf5, 0x87, 0x7d, 0xb9, - 0xc1, 0x2a, 0xeb, 0xc6, 0x46, 0xd3, 0xaa, 0xf8, 0xc3, 0xbe, 0xd8, 0x5e, 0xff, 0x58, 0x80, 0x85, - 0xfb, 0x43, 0x7e, 0x11, 0x12, 0x49, 0x86, 0xa1, 0xc7, 0x9e, 0x4e, 0x19, 0xaf, 0x40, 0x51, 0xba, - 0x14, 0x9c, 0xa2, 0xad, 0x65, 0x7c, 0x7b, 0x8b, 0x5a, 0x1c, 0x49, 0x04, 0xd8, 0x87, 0xb6, 0xad, - 0xbc, 0xb3, 0xa2, 0x60, 0xb6, 0xc6, 0x21, 0xd2, 0x37, 0x3b, 0x0f, 0x35, 0x12, 0x86, 0xb1, 0xef, - 0x26, 0xa6, 0x42, 0xc2, 0x50, 0x36, 0x9a, 0xd0, 0xc0, 0xf6, 0x23, 0x3f, 0x38, 0xf4, 0x88, 0xd3, - 0x23, 0x8e, 0x58, 0xf6, 0xaa, 0x95, 0x82, 0x49, 0xc5, 0xe0, 0x0b, 0xdf, 0xb5, 0x7d, 0x26, 0x4e, - 0xf5, 0x22, 0x57, 0x0c, 0x0e, 0xb9, 0xed, 0x33, 0xde, 0xec, 0x10, 0x8f, 0x30, 0x22, 0x9a, 0x2b, - 0xb2, 0x59, 0x42, 0x54, 0xf3, 0x70, 0x10, 0x53, 0x57, 0x65, 0xb3, 0x84, 0xf0, 0xe6, 0x0b, 0x50, - 0x1b, 0x65, 0x11, 0x6a, 0xa3, 0x30, 0xa2, 0x00, 0x98, 0xff, 0x61, 0x40, 0x73, 0x4b, 0x74, 0xf5, - 0x1c, 0x28, 0x1d, 0x82, 0x79, 0xf2, 0x64, 0x10, 0xaa, 0xad, 0x23, 0x7e, 0x4f, 0xd4, 0x23, 0xf3, - 0x31, 0xb4, 0x76, 0x3c, 0x6c, 0x93, 0x83, 0xc0, 0x73, 0x48, 0x28, 0xce, 0x76, 0xd4, 0x82, 0x22, - 0xc3, 0x3d, 0xe5, 0x3c, 0xf0, 0x9f, 0xe8, 0xcb, 0xea, 0xea, 0x27, 0xcd, 0xd2, 0xcb, 0xda, 0x53, - 0x36, 0xd1, 0x4d, 0x22, 0xa2, 0xba, 0x0a, 0x65, 0x91, 0xd9, 0x93, 0x6e, 0x45, 0xc3, 0x52, 0x5f, - 0xe6, 0xc3, 0xd4, 0xb8, 0x77, 0xc2, 0x60, 0x38, 0x40, 0xdb, 0xd0, 0x18, 0x8c, 0x60, 0x5c, 0x57, - 0xf3, 0xcf, 0xf4, 0x2c, 0xd3, 0x56, 0x8a, 0xd4, 0xfc, 0xaf, 0x22, 0x34, 0x77, 0x09, 0x0e, 0xed, - 0x83, 0xe7, 0x21, 0x06, 0xc3, 0x25, 0xee, 0x50, 0x4f, 0xad, 0x1a, 0xff, 0x89, 0xae, 0xc2, 0xb9, - 0xc4, 0x84, 0xba, 0x3d, 0x2e, 0x20, 0xa1, 0xf7, 0x0d, 0xab, 0x35, 0xc8, 0x0a, 0xee, 0x1d, 0xa8, - 0x3a, 0xd4, 0xeb, 0x8a, 0x25, 0xaa, 0x88, 0x25, 0xd2, 0xcf, 0x6f, 0x8b, 0x7a, 0x62, 0x69, 0x2a, - 0x8e, 0xfc, 0x81, 0x5e, 0x82, 0x66, 0x30, 0x64, 0x83, 0x21, 0xeb, 0x4a, 0xbb, 0xd3, 0xae, 0x0a, - 0xf6, 0x1a, 0x12, 0x28, 0xcc, 0x12, 0x45, 0x1f, 0x40, 0x93, 0x0a, 0x51, 0x46, 0x8e, 0x79, 0x6d, - 0x5a, 0x07, 0xb1, 0x21, 0xe9, 0x94, 0x67, 0xfe, 0x3a, 0xb4, 0x58, 0x88, 0x1f, 0x13, 0x2f, 0x91, - 0xb3, 0x03, 0xb1, 0xdb, 0x16, 0x25, 0x7c, 0x94, 0xaf, 0xbb, 0x0e, 0x4b, 0xbd, 0x21, 0x0e, 0xb1, - 0xcf, 0x08, 0x49, 0x60, 0xd7, 0x05, 0x36, 0x8a, 0x9b, 0x62, 0x02, 0xf3, 0x43, 0x98, 0xbf, 0xeb, - 0x32, 0x21, 0x48, 0x6e, 0xb3, 0x0c, 0x71, 0x0d, 0x12, 0x96, 0xe9, 0x05, 0xa8, 0x86, 0xc1, 0xa1, - 0xb4, 0xc1, 0x05, 0xa1, 0x82, 0x95, 0x30, 0x38, 0x14, 0x06, 0x56, 0x54, 0x3a, 0x04, 0xa1, 0xd2, - 0xcd, 0x82, 0xa5, 0xbe, 0xcc, 0x3f, 0x37, 0x46, 0xca, 0xc3, 0xcd, 0x27, 0x7d, 0x3a, 0xfb, 0xf9, - 0x3e, 0x54, 0x42, 0x49, 0x3f, 0x31, 0x47, 0x9b, 0x1c, 0x49, 0x9c, 0x01, 0x11, 0xd5, 0xf4, 0x09, - 0xa0, 0x5f, 0x36, 0xa0, 0xf1, 0x81, 0x37, 0xa4, 0xcf, 0x42, 0xd9, 0x75, 0x69, 0x89, 0xa2, 0x3e, - 0x25, 0xf2, 0x9b, 0x05, 0x68, 0x2a, 0x36, 0x66, 0x71, 0x82, 0x72, 0x59, 0xd9, 0x85, 0x3a, 0x1f, - 0xb2, 0x4b, 0x49, 0x2f, 0x8a, 0xe9, 0xd4, 0x37, 0x37, 0xb5, 0xe6, 0x21, 0xc5, 0x86, 0x48, 0x83, - 0xef, 0x0a, 0xa2, 0x9f, 0xf7, 0x59, 0x78, 0x64, 0x81, 0x1d, 0x03, 0x3a, 0x0f, 0x61, 0x31, 0xd3, - 0xcc, 0x95, 0xe8, 0x11, 0x39, 0x8a, 0xec, 0xdf, 0x23, 0x72, 0x84, 0xde, 0x4a, 0x16, 0x2b, 0xe4, - 0x9d, 0xe2, 0xf7, 0x02, 0xbf, 0x77, 0x33, 0x0c, 0xf1, 0x91, 0x2a, 0x66, 0x78, 0xb7, 0xf0, 0x65, - 0xc3, 0xfc, 0xbb, 0x02, 0x34, 0xbe, 0x36, 0x24, 0xe1, 0xd1, 0x69, 0xda, 0xa1, 0xe8, 0x54, 0x98, - 0x4f, 0x9c, 0x0a, 0x63, 0x5b, 0xbf, 0xa4, 0xd9, 0xfa, 0x1a, 0x03, 0x56, 0xd6, 0x1a, 0x30, 0xdd, - 0xde, 0xae, 0x9c, 0x68, 0x6f, 0x57, 0x73, 0xf7, 0xf6, 0x9f, 0x19, 0xb1, 0x08, 0x67, 0xda, 0x8d, - 0x29, 0x77, 0xac, 0x70, 0x62, 0x77, 0x6c, 0xea, 0xdd, 0xf8, 0x43, 0x03, 0x6a, 0x5f, 0x27, 0x36, - 0x0b, 0x42, 0x6e, 0x7f, 0x34, 0x64, 0xc6, 0x14, 0xae, 0x71, 0x21, 0xeb, 0x1a, 0xdf, 0x80, 0xaa, - 0xeb, 0x74, 0x31, 0xd7, 0x2f, 0x31, 0xee, 0x24, 0x97, 0xac, 0xe2, 0x3a, 0x42, 0x11, 0xa7, 0x4f, - 0x02, 0xfc, 0x8e, 0x01, 0x0d, 0xc9, 0x33, 0x95, 0x94, 0xef, 0x25, 0x86, 0x33, 0x74, 0x4a, 0xaf, - 0x3e, 0xe2, 0x89, 0xde, 0x9d, 0x1b, 0x0d, 0x7b, 0x13, 0x80, 0x0b, 0x59, 0x91, 0xcb, 0x3d, 0xb3, - 0xae, 0xe5, 0x56, 0x92, 0x0b, 0x81, 0xdf, 0x9d, 0xb3, 0x6a, 0x9c, 0x4a, 0x74, 0x71, 0xab, 0x02, - 0x25, 0x41, 0x6d, 0xfe, 0xaf, 0x01, 0x4b, 0xb7, 0xb1, 0x67, 0x6f, 0xb9, 0x94, 0x61, 0xdf, 0x9e, - 0xc1, 0x09, 0x7b, 0x17, 0x2a, 0xc1, 0xa0, 0xeb, 0x91, 0x7d, 0xa6, 0x58, 0xba, 0x3c, 0x61, 0x46, - 0x52, 0x0c, 0x56, 0x39, 0x18, 0xdc, 0x23, 0xfb, 0x0c, 0x7d, 0x05, 0xaa, 0xc1, 0xa0, 0x1b, 0xba, - 0xbd, 0x03, 0xa6, 0xa4, 0x3f, 0x05, 0x71, 0x25, 0x18, 0x58, 0x9c, 0x22, 0x11, 0x5b, 0x99, 0x3f, - 0x61, 0x6c, 0xc5, 0xfc, 0x97, 0xb1, 0xe9, 0xcf, 0xb0, 0x07, 0xde, 0x85, 0xaa, 0xeb, 0xb3, 0xae, - 0xe3, 0xd2, 0x48, 0x04, 0x17, 0xf5, 0x3a, 0xe4, 0x33, 0x31, 0x03, 0xb1, 0xa6, 0x3e, 0xe3, 0x63, - 0xa3, 0xaf, 0x02, 0xec, 0x7b, 0x01, 0x56, 0xd4, 0x52, 0x06, 0x97, 0xf4, 0xdb, 0x87, 0xa3, 0x45, - 0xf4, 0x35, 0x41, 0xc4, 0x7b, 0x18, 0x2d, 0xe9, 0x3f, 0x19, 0xb0, 0xb2, 0x43, 0x42, 0x59, 0xca, - 0xc2, 0x54, 0x18, 0x74, 0xdb, 0xdf, 0x0f, 0xd2, 0x91, 0x68, 0x23, 0x13, 0x89, 0xfe, 0xe9, 0x44, - 0x5f, 0x53, 0x37, 0x27, 0x99, 0x0f, 0x89, 0x6e, 0x4e, 0x51, 0xd6, 0x47, 0xde, 0x3c, 0x17, 0x72, - 0x96, 0x49, 0xf1, 0x9b, 0xbc, 0x80, 0x9b, 0xbf, 0x25, 0x2b, 0x30, 0xb4, 0x93, 0x7a, 0x7a, 0x85, - 0x5d, 0x05, 0x65, 0xe9, 0x33, 0x76, 0xff, 0x55, 0xc8, 0xd8, 0x8e, 0x1c, 0x43, 0xf4, 0x03, 0x03, - 0xd6, 0xf3, 0xb9, 0x9a, 0xe5, 0x88, 0xfe, 0x2a, 0x94, 0x5c, 0x7f, 0x3f, 0x88, 0xc2, 0x6e, 0x57, - 0xf4, 0x2e, 0xba, 0x76, 0x5c, 0x49, 0x68, 0xfe, 0x55, 0x01, 0x5a, 0xc2, 0xa8, 0x9f, 0xc2, 0xf2, - 0xf7, 0x49, 0xbf, 0x4b, 0xdd, 0x4f, 0x49, 0xb4, 0xfc, 0x7d, 0xd2, 0xdf, 0x75, 0x3f, 0x25, 0x29, - 0xcd, 0x28, 0xa5, 0x35, 0x63, 0x72, 0x54, 0x39, 0x19, 0x56, 0xad, 0xa4, 0xc3, 0xaa, 0xab, 0x50, - 0xf6, 0x03, 0x87, 0x6c, 0x6f, 0xa9, 0x6b, 0xa7, 0xfa, 0x1a, 0xa9, 0x5a, 0xed, 0x84, 0xaa, 0xf6, - 0xb9, 0x01, 0x9d, 0x3b, 0x84, 0x65, 0x65, 0x77, 0x7a, 0x5a, 0xf6, 0x3d, 0x03, 0xce, 0x6b, 0x19, - 0x9a, 0x45, 0xc1, 0xde, 0x4b, 0x2b, 0x98, 0xfe, 0x0e, 0x38, 0x36, 0xa4, 0xd2, 0xad, 0x37, 0xa1, - 0xb1, 0x35, 0xec, 0xf7, 0x63, 0x97, 0xeb, 0x32, 0x34, 0x42, 0xf9, 0x53, 0x5e, 0x91, 0xe4, 0xf9, - 0x5b, 0x57, 0x30, 0x7e, 0x11, 0x32, 0xaf, 0x42, 0x53, 0x91, 0x28, 0xae, 0x3b, 0x50, 0x0d, 0xd5, - 0x6f, 0x85, 0x1f, 0x7f, 0x9b, 0x2b, 0xb0, 0x64, 0x91, 0x1e, 0x57, 0xed, 0xf0, 0x9e, 0xeb, 0x3f, - 0x52, 0xc3, 0x98, 0xdf, 0x31, 0x60, 0x39, 0x0d, 0x57, 0x7d, 0xbd, 0x0d, 0x15, 0xec, 0x38, 0x21, - 0xa1, 0x74, 0xe2, 0xb2, 0xdc, 0x94, 0x38, 0x56, 0x84, 0x9c, 0x90, 0x5c, 0x61, 0x6a, 0xc9, 0x99, - 0x5d, 0x38, 0x77, 0x87, 0xb0, 0xfb, 0x84, 0x85, 0x33, 0x65, 0xf0, 0xdb, 0xfc, 0xf2, 0x22, 0x88, - 0x95, 0x5a, 0x44, 0x9f, 0xe6, 0x67, 0x06, 0xa0, 0xe4, 0x08, 0xb3, 0x2c, 0x73, 0x52, 0xca, 0x85, - 0xb4, 0x94, 0x65, 0x91, 0x53, 0x7f, 0x10, 0xf8, 0xc4, 0x67, 0x49, 0x77, 0xab, 0x19, 0x43, 0x85, - 0xfa, 0xfd, 0xd8, 0x00, 0x74, 0x2f, 0xc0, 0xce, 0x2d, 0xec, 0xcd, 0xe6, 0x1e, 0x5c, 0x04, 0xa0, - 0xa1, 0xdd, 0x55, 0xbb, 0xb5, 0xa0, 0xac, 0x4f, 0x68, 0x3f, 0x90, 0x1b, 0xf6, 0x12, 0xd4, 0x1d, - 0xca, 0x54, 0x73, 0x94, 0x50, 0x06, 0x87, 0x32, 0xd9, 0x2e, 0x8a, 0x58, 0x29, 0xc1, 0x1e, 0x71, - 0xba, 0x89, 0x7c, 0xdc, 0xbc, 0x40, 0x6b, 0xc9, 0x86, 0xdd, 0x51, 0x56, 0xee, 0x21, 0xac, 0xdd, - 0xc7, 0xfe, 0x10, 0x7b, 0xb7, 0x83, 0xfe, 0x00, 0xa7, 0x0a, 0x1b, 0xb3, 0x66, 0xce, 0xd0, 0x98, - 0xb9, 0x17, 0x65, 0xe5, 0x9b, 0x74, 0xad, 0x05, 0xaf, 0xf3, 0x56, 0x02, 0x62, 0x52, 0x68, 0x8f, - 0x77, 0x3f, 0xcb, 0x42, 0x09, 0xa6, 0xa2, 0xae, 0x92, 0xb6, 0x77, 0x04, 0x33, 0xdf, 0x87, 0x17, - 0x44, 0x15, 0x62, 0x04, 0x4a, 0x85, 0xf6, 0xb3, 0x1d, 0x18, 0x9a, 0x0e, 0x7e, 0xb5, 0x20, 0x4c, - 0xdb, 0x58, 0x0f, 0xb3, 0x30, 0xfe, 0x6e, 0x3a, 0xa2, 0xfe, 0x72, 0x4e, 0xa5, 0x6d, 0x7a, 0x44, - 0x15, 0x56, 0xdf, 0x80, 0x45, 0xf2, 0x84, 0xd8, 0x43, 0xe6, 0xfa, 0xbd, 0x1d, 0x0f, 0xfb, 0x0f, - 0x02, 0x75, 0xa0, 0x64, 0xc1, 0xe8, 0x65, 0x68, 0x72, 0xe9, 0x07, 0x43, 0xa6, 0xf0, 0xe4, 0xc9, - 0x92, 0x06, 0xf2, 0xfe, 0xf8, 0x7c, 0x3d, 0xc2, 0x88, 0xa3, 0xf0, 0xe4, 0x31, 0x93, 0x05, 0x8f, - 0x89, 0x92, 0x83, 0xe9, 0x49, 0x44, 0xf9, 0x6f, 0x46, 0x46, 0x94, 0xaa, 0x87, 0xd3, 0x12, 0xe5, - 0x5d, 0x80, 0x3e, 0x09, 0x7b, 0x64, 0x5b, 0x18, 0x75, 0x79, 0x73, 0xdf, 0xd0, 0x1a, 0xf5, 0x51, - 0x07, 0xf7, 0x23, 0x02, 0x2b, 0x41, 0x6b, 0xde, 0x81, 0x25, 0x0d, 0x0a, 0xb7, 0x57, 0x34, 0x18, - 0x86, 0x36, 0x89, 0x82, 0x3f, 0xd1, 0x27, 0x3f, 0xdf, 0x18, 0x0e, 0x7b, 0x84, 0x29, 0xa5, 0x55, - 0x5f, 0xe6, 0xdb, 0x22, 0x09, 0x25, 0x02, 0x05, 0x29, 0x4d, 0x4d, 0x27, 0xd4, 0x8d, 0xb1, 0x84, - 0xfa, 0xbe, 0xc8, 0xf8, 0x24, 0xe9, 0x66, 0x2c, 0x86, 0xd8, 0xe7, 0x5d, 0x11, 0x47, 0xbd, 0xb2, - 0x88, 0x3e, 0xcd, 0xff, 0x31, 0xa0, 0xb9, 0xdd, 0x1f, 0x04, 0xa3, 0x64, 0xc7, 0xd4, 0x57, 0xc9, - 0xf1, 0x60, 0x71, 0x41, 0x17, 0x2c, 0x7e, 0x09, 0x9a, 0xe9, 0x1a, 0x7d, 0x19, 0xd7, 0x69, 0xd8, - 0xc9, 0xda, 0xfc, 0xf3, 0x50, 0x0b, 0x83, 0xc3, 0x2e, 0x37, 0x91, 0x8e, 0x2a, 0xbb, 0xa8, 0x86, - 0xc1, 0x21, 0x37, 0x9c, 0x0e, 0x5a, 0x86, 0xd2, 0xbe, 0xeb, 0xc5, 0x15, 0x43, 0xf2, 0x03, 0xbd, - 0xc7, 0x2f, 0x5a, 0x32, 0x2d, 0x5b, 0x9e, 0xf6, 0xbe, 0x13, 0x51, 0x98, 0x9f, 0xc0, 0x42, 0x34, - 0xeb, 0x19, 0x9f, 0x97, 0x30, 0x4c, 0x1f, 0x45, 0x15, 0x11, 0xf2, 0xc3, 0xbc, 0x2a, 0xb3, 0x75, - 0xa2, 0xff, 0xd4, 0xa2, 0x23, 0x98, 0xe7, 0x18, 0x6a, 0x2f, 0x89, 0xdf, 0x7c, 0x01, 0x56, 0xb3, - 0xd8, 0xb3, 0xb0, 0xf4, 0x76, 0x7a, 0xff, 0xe8, 0x5f, 0x10, 0x24, 0x47, 0x53, 0x7b, 0x47, 0xad, - 0x80, 0x1d, 0x0c, 0x7d, 0xa6, 0x0c, 0x10, 0x5f, 0x81, 0xdb, 0xfc, 0x1b, 0xad, 0x41, 0xc5, 0x75, - 0xba, 0x1e, 0xbf, 0x93, 0xc9, 0xb3, 0xa6, 0xec, 0x3a, 0xf7, 0xf8, 0x7d, 0xed, 0x9d, 0xc8, 0x83, - 0x9a, 0xba, 0x8c, 0x42, 0x79, 0x4f, 0xdf, 0x97, 0xe7, 0xbb, 0x25, 0x4b, 0xf1, 0x9f, 0x71, 0xb1, - 0xcc, 0x06, 0xb4, 0x0e, 0x5d, 0x76, 0xd0, 0x15, 0x6f, 0x31, 0xc4, 0xe1, 0x2a, 0xf3, 0xc5, 0x55, - 0x6b, 0x81, 0xc3, 0x77, 0x39, 0x98, 0x1f, 0xb0, 0xd4, 0xfc, 0x35, 0x03, 0x96, 0x52, 0x6c, 0xcd, - 0xb2, 0x14, 0x5f, 0xe1, 0x7e, 0x87, 0xec, 0x48, 0x79, 0x98, 0xeb, 0x5a, 0x63, 0xa4, 0x46, 0x13, - 0x46, 0x28, 0xa6, 0x30, 0xff, 0xdd, 0x80, 0x7a, 0xa2, 0x85, 0x5f, 0x5b, 0x54, 0xdb, 0xe8, 0xda, - 0x12, 0x03, 0xa6, 0x12, 0xc3, 0x4b, 0x30, 0xda, 0x9a, 0x89, 0x7a, 0xee, 0x44, 0xbd, 0x9a, 0x43, - 0xd1, 0x5d, 0x58, 0x90, 0x62, 0x8a, 0x59, 0xd7, 0x46, 0x13, 0xe2, 0x4a, 0x3c, 0x1c, 0x3a, 0x8a, - 0x4b, 0xab, 0x49, 0x13, 0x5f, 0x32, 0x79, 0x18, 0x38, 0x44, 0x8c, 0x54, 0x92, 0xd6, 0x52, 0xf8, - 0x35, 0x0e, 0xe5, 0xd7, 0x8b, 0x46, 0x92, 0x94, 0xbb, 0x68, 0x1e, 0xc1, 0x0e, 0x09, 0xe3, 0xb9, - 0xc5, 0xdf, 0xdc, 0x27, 0x92, 0xbf, 0xbb, 0xdc, 0x65, 0x55, 0x46, 0x06, 0x24, 0x88, 0x7b, 0xb3, - 0xe8, 0x55, 0x58, 0x74, 0xfa, 0xa9, 0x87, 0x40, 0x91, 0x13, 0xe7, 0xf4, 0x13, 0x2f, 0x80, 0x52, - 0x0c, 0xcd, 0xa7, 0x19, 0xfa, 0x6f, 0x23, 0x7e, 0x1e, 0x19, 0x12, 0x87, 0xf8, 0xcc, 0xc5, 0xde, - 0xd3, 0xeb, 0x64, 0x07, 0xaa, 0x43, 0x4a, 0xc2, 0x84, 0x4d, 0x8c, 0xbf, 0x79, 0xdb, 0x00, 0x53, - 0x7a, 0x18, 0x84, 0x8e, 0xe2, 0x32, 0xfe, 0x9e, 0x50, 0xfc, 0x27, 0x9f, 0xde, 0xe9, 0x8b, 0xff, - 0xde, 0x86, 0xb5, 0x7e, 0xe0, 0xb8, 0xfb, 0xae, 0xae, 0x66, 0x90, 0x93, 0xad, 0x44, 0xcd, 0x29, - 0x3a, 0xf3, 0x07, 0x05, 0x58, 0xfb, 0x78, 0xe0, 0xfc, 0x0c, 0xe6, 0xbc, 0x0e, 0xf5, 0xc0, 0x73, - 0x76, 0xd2, 0xd3, 0x4e, 0x82, 0x38, 0x86, 0x4f, 0x0e, 0x63, 0x0c, 0x19, 0x42, 0x4e, 0x82, 0x26, - 0x16, 0x46, 0x3e, 0x95, 0x6c, 0xca, 0x93, 0x64, 0xd3, 0x83, 0x35, 0x99, 0x8d, 0x7d, 0xc6, 0xa2, - 0x31, 0x7f, 0x09, 0x56, 0xb8, 0x21, 0xe5, 0xc3, 0x7c, 0x4c, 0x49, 0x38, 0xa3, 0xc5, 0xb9, 0x00, - 0xb5, 0xa8, 0xe7, 0xa8, 0x66, 0x75, 0x04, 0x30, 0xef, 0xc2, 0x72, 0x66, 0xac, 0xa7, 0x9c, 0xd1, - 0x95, 0xcb, 0x50, 0x8d, 0x6a, 0x70, 0x51, 0x05, 0x8a, 0x37, 0x3d, 0xaf, 0x35, 0x87, 0x1a, 0x50, - 0xdd, 0x56, 0x85, 0xa6, 0x2d, 0xe3, 0xca, 0xcf, 0xc1, 0x62, 0x26, 0x57, 0x8b, 0xaa, 0x30, 0xff, - 0x20, 0xf0, 0x49, 0x6b, 0x0e, 0xb5, 0xa0, 0x71, 0xcb, 0xf5, 0x71, 0x78, 0x24, 0x23, 0x99, 0x2d, - 0x07, 0x2d, 0x42, 0x5d, 0x44, 0xf4, 0x14, 0x80, 0x6c, 0xfe, 0xe4, 0x65, 0x68, 0xde, 0x17, 0x8c, - 0xec, 0x92, 0xf0, 0xb1, 0x6b, 0x13, 0xd4, 0x85, 0x56, 0xf6, 0x05, 0x33, 0xfa, 0x82, 0xde, 0xbb, - 0xd3, 0x3f, 0x74, 0xee, 0x4c, 0x92, 0xa1, 0x39, 0x87, 0x3e, 0x81, 0x85, 0xf4, 0x3b, 0x60, 0xa4, - 0x0f, 0x39, 0x69, 0x1f, 0x0b, 0x1f, 0xd7, 0x79, 0x17, 0x9a, 0xa9, 0x67, 0xbd, 0xe8, 0x75, 0x6d, - 0xdf, 0xba, 0xa7, 0xbf, 0x1d, 0xbd, 0xed, 0x4d, 0x3e, 0xbd, 0x95, 0xdc, 0xa7, 0xdf, 0xde, 0xe5, - 0x70, 0xaf, 0x7d, 0xa0, 0x77, 0x1c, 0xf7, 0x18, 0xce, 0x8d, 0xbd, 0x91, 0x43, 0x6f, 0xe4, 0x9c, - 0x66, 0xfa, 0xb7, 0x74, 0xc7, 0x0d, 0x71, 0x08, 0x68, 0xfc, 0xf9, 0x2a, 0xba, 0xa6, 0x5f, 0x81, - 0xbc, 0xc7, 0xbb, 0x9d, 0xeb, 0x53, 0xe3, 0xc7, 0x82, 0xfb, 0x15, 0x03, 0xd6, 0x72, 0x1e, 0xb6, - 0xa1, 0x1b, 0xda, 0xee, 0x26, 0xbf, 0xce, 0xeb, 0xbc, 0x75, 0x32, 0xa2, 0x98, 0x11, 0x1f, 0x16, - 0x33, 0x6f, 0xbd, 0xd0, 0xd5, 0xdc, 0xfa, 0xf7, 0xf1, 0x47, 0x6f, 0x9d, 0x2f, 0x4c, 0x87, 0x1c, - 0x8f, 0xf7, 0x10, 0x16, 0x33, 0x0f, 0xa4, 0x72, 0xc6, 0xd3, 0x3f, 0xa3, 0x3a, 0x6e, 0x41, 0xbf, - 0x09, 0xcd, 0xd4, 0x4b, 0xa6, 0x1c, 0x8d, 0xd7, 0xbd, 0x76, 0x3a, 0xae, 0xeb, 0x87, 0xd0, 0x48, - 0x3e, 0x38, 0x42, 0x1b, 0x79, 0x7b, 0x69, 0xac, 0xe3, 0x93, 0x6c, 0xa5, 0xd1, 0x7b, 0x82, 0x09, - 0x5b, 0x69, 0xec, 0x09, 0xc6, 0xf4, 0x5b, 0x29, 0xd1, 0xff, 0xc4, 0xad, 0x74, 0xe2, 0x21, 0xbe, - 0x23, 0xef, 0x14, 0x9a, 0xf7, 0x2a, 0x68, 0x33, 0x4f, 0x37, 0xf3, 0x5f, 0xe6, 0x74, 0x6e, 0x9c, - 0x88, 0x26, 0x96, 0xe2, 0x23, 0x58, 0x48, 0xbf, 0xca, 0xc8, 0x91, 0xa2, 0xf6, 0x21, 0x4b, 0xe7, - 0xea, 0x54, 0xb8, 0xf1, 0x60, 0x1f, 0x43, 0x3d, 0xf1, 0xa7, 0x24, 0xe8, 0xb5, 0x09, 0x7a, 0x9c, - 0xfc, 0x87, 0x8e, 0xe3, 0x24, 0xf9, 0x35, 0xa8, 0xc5, 0xff, 0x25, 0x82, 0x5e, 0xc9, 0xd5, 0xdf, - 0x93, 0x74, 0xb9, 0x0b, 0x30, 0xfa, 0xa3, 0x10, 0xf4, 0xaa, 0xb6, 0xcf, 0xb1, 0x7f, 0x12, 0x39, - 0xae, 0xd3, 0x78, 0xfa, 0xb2, 0xd8, 0x6d, 0xd2, 0xf4, 0x93, 0xd5, 0x99, 0xc7, 0x75, 0x7b, 0x00, - 0xcd, 0x54, 0x4d, 0x75, 0xde, 0x16, 0xd6, 0x94, 0xba, 0x77, 0xae, 0x4c, 0x83, 0x1a, 0xaf, 0xdf, - 0x01, 0x34, 0x53, 0x15, 0xae, 0x39, 0x23, 0xe9, 0x0a, 0x7a, 0x73, 0x46, 0xd2, 0x16, 0xcc, 0x9a, - 0x73, 0xe8, 0xdb, 0x89, 0x62, 0xda, 0x54, 0xc1, 0x32, 0x7a, 0x73, 0x62, 0x3f, 0xba, 0x7a, 0xed, - 0xce, 0xe6, 0x49, 0x48, 0x62, 0x16, 0x94, 0x56, 0x49, 0x91, 0xe6, 0x6b, 0xd5, 0x49, 0x56, 0x6a, - 0x17, 0xca, 0xb2, 0x66, 0x15, 0x99, 0x39, 0xd5, 0xe9, 0x89, 0x82, 0xd6, 0xce, 0x4b, 0x5a, 0x9c, - 0x74, 0x39, 0xa7, 0xec, 0x54, 0x7a, 0xc1, 0x39, 0x9d, 0xa6, 0x0a, 0x16, 0xa7, 0xed, 0xd4, 0x82, - 0xb2, 0x2c, 0x46, 0xca, 0xe9, 0x34, 0x55, 0x50, 0xd7, 0x99, 0x8c, 0x23, 0xb2, 0xd4, 0xe6, 0x1c, - 0xda, 0x81, 0x92, 0x08, 0x95, 0xa1, 0xcb, 0x93, 0xea, 0x74, 0x26, 0xf5, 0x98, 0x2a, 0xe5, 0x31, - 0xe7, 0xd0, 0x2f, 0x40, 0x49, 0x24, 0x7e, 0x72, 0x7a, 0x4c, 0x16, 0xdb, 0x74, 0x26, 0xa2, 0x44, - 0x2c, 0x3a, 0xd0, 0x48, 0x66, 0xd8, 0x73, 0x8e, 0x2c, 0x4d, 0x0d, 0x42, 0x67, 0x1a, 0xcc, 0x68, - 0x14, 0xb9, 0x8d, 0x46, 0x61, 0xc3, 0xfc, 0x6d, 0x34, 0x16, 0x92, 0xcc, 0xdf, 0x46, 0xe3, 0x51, - 0x48, 0x73, 0x0e, 0xfd, 0xba, 0x01, 0xed, 0xbc, 0xb4, 0x2f, 0xca, 0xf5, 0x80, 0x26, 0xe5, 0xae, - 0x3b, 0x5f, 0x3a, 0x21, 0x55, 0xcc, 0xcb, 0xa7, 0x22, 0x68, 0x33, 0x96, 0xe8, 0xbd, 0x9e, 0xd7, - 0x5f, 0x4e, 0x5a, 0xb3, 0xf3, 0xc5, 0xe9, 0x09, 0xe2, 0xb1, 0xf7, 0xa0, 0x9e, 0x08, 0x18, 0xe5, - 0x58, 0xde, 0xf1, 0x48, 0x57, 0xce, 0xaa, 0x6a, 0x62, 0x4f, 0x52, 0xbd, 0x45, 0xde, 0x30, 0x47, - 0x19, 0x93, 0x69, 0xc8, 0x1c, 0xf5, 0x4e, 0xa5, 0x1d, 0xcd, 0x39, 0x44, 0xa0, 0x91, 0x4c, 0x22, - 0xe6, 0x68, 0xa3, 0x26, 0xff, 0xd8, 0x79, 0x7d, 0x0a, 0xcc, 0x78, 0x98, 0x2e, 0xc0, 0x28, 0x89, - 0x97, 0x73, 0xd6, 0x8d, 0xe5, 0x11, 0x3b, 0xaf, 0x1d, 0x8b, 0x97, 0x3c, 0xf6, 0x13, 0x69, 0xb9, - 0x1c, 0xe9, 0x8f, 0x27, 0xee, 0xa6, 0xb8, 0x8b, 0x8c, 0xa7, 0x88, 0x72, 0xee, 0x22, 0xb9, 0xd9, - 0xa8, 0xce, 0xf5, 0xa9, 0xf1, 0xe3, 0xf9, 0x7c, 0x0b, 0x5a, 0xd9, 0x94, 0x5a, 0xce, 0x1d, 0x37, - 0x27, 0xb1, 0xd7, 0x79, 0x63, 0x4a, 0xec, 0xe4, 0x79, 0x78, 0x7e, 0x9c, 0xa7, 0x6f, 0xb8, 0xec, - 0x40, 0x64, 0x73, 0xa6, 0x99, 0x75, 0x32, 0x71, 0x34, 0xcd, 0xac, 0x53, 0x69, 0x22, 0x75, 0x78, - 0x89, 0x88, 0x74, 0xde, 0xe1, 0x95, 0x4c, 0x50, 0xe4, 0x9c, 0x33, 0xe9, 0x70, 0xbe, 0x74, 0x3f, - 0xd3, 0x71, 0x75, 0x94, 0xef, 0x27, 0x8c, 0x85, 0xea, 0x73, 0xdc, 0x4f, 0x7d, 0xa0, 0x5e, 0x28, - 0x7a, 0x2b, 0x1b, 0x3e, 0x9c, 0x1c, 0x9b, 0xc8, 0x86, 0x95, 0x8e, 0x0f, 0x1f, 0xb4, 0xb2, 0xb1, - 0xba, 0x9c, 0x01, 0x72, 0x42, 0x7a, 0x53, 0x0c, 0x90, 0x8d, 0x78, 0xe5, 0x0c, 0x90, 0x13, 0x18, - 0x9b, 0xc2, 0x97, 0x4c, 0x45, 0x9f, 0x72, 0x8e, 0x26, 0x5d, 0x84, 0x2a, 0xe7, 0x68, 0xd2, 0x06, - 0xce, 0xcc, 0xb9, 0xcd, 0x21, 0x34, 0x76, 0xc2, 0xe0, 0xc9, 0x51, 0x14, 0x38, 0xfa, 0xd9, 0x18, - 0xbb, 0x5b, 0xdf, 0x80, 0x05, 0x37, 0xc6, 0xe9, 0x85, 0x03, 0xfb, 0x56, 0x5d, 0x06, 0xb0, 0x76, - 0x38, 0xf1, 0x8e, 0xf1, 0x8b, 0x37, 0x7a, 0x2e, 0x3b, 0x18, 0xee, 0x71, 0xc9, 0x5c, 0x97, 0x68, - 0x6f, 0xb8, 0x81, 0xfa, 0x75, 0xdd, 0xf5, 0x19, 0x09, 0x7d, 0xec, 0x5d, 0x17, 0x43, 0x29, 0xe8, - 0x60, 0xef, 0x0f, 0x0c, 0x63, 0xaf, 0x2c, 0x40, 0x37, 0xfe, 0x2f, 0x00, 0x00, 0xff, 0xff, 0xc5, - 0x6f, 0xa3, 0xd8, 0xb9, 0x51, 0x00, 0x00, + // 4255 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xd4, 0x3c, 0x4d, 0x6f, 0x1c, 0xc9, + 0x75, 0xec, 0x19, 0xce, 0xd7, 0x9b, 0x19, 0x72, 0x54, 0xfc, 0x9a, 0x1d, 0x49, 0x2b, 0xaa, 0xf7, + 0x8b, 0x2b, 0x79, 0x25, 0x2f, 0xb5, 0xde, 0x75, 0x76, 0x9d, 0xac, 0x25, 0x31, 0x2b, 0x11, 0x2b, + 0x29, 0x74, 0x73, 0xd7, 0x86, 0xb3, 0x10, 0x06, 0xc5, 0xee, 0xe2, 0xb0, 0xa3, 0x9e, 0xee, 0x71, + 0x57, 0x8d, 0x28, 0xee, 0xc9, 0x80, 0x83, 0x7c, 0xc0, 0x9b, 0x35, 0x82, 0x18, 0x49, 0x7c, 0x48, + 0x10, 0xe4, 0xe3, 0x90, 0x43, 0x82, 0xd8, 0x01, 0x12, 0x23, 0x97, 0xe4, 0x90, 0x43, 0x0e, 0x01, + 0xf2, 0x71, 0x09, 0x02, 0x5f, 0xf2, 0x07, 0x72, 0x08, 0x90, 0x63, 0x0e, 0x41, 0x7d, 0x74, 0x4f, + 0x77, 0x4f, 0xf5, 0x70, 0xa8, 0xb1, 0x4c, 0xea, 0x36, 0xfd, 0xea, 0xbd, 0xaa, 0x57, 0xaf, 0x5e, + 0xbd, 0x7a, 0xf5, 0xde, 0xab, 0x81, 0x46, 0xdf, 0xf5, 0x1e, 0x0f, 0xe9, 0xb5, 0x41, 0x18, 0xb0, + 0x00, 0x2d, 0x25, 0xbf, 0xae, 0xc9, 0x8f, 0x4e, 0xc3, 0x0e, 0xfa, 0xfd, 0xc0, 0x97, 0xc0, 0x4e, + 0x83, 0xda, 0x07, 0xa4, 0x8f, 0xe5, 0x97, 0xf9, 0x87, 0x06, 0xa0, 0xdb, 0x21, 0xc1, 0x8c, 0xdc, + 0xf4, 0x5c, 0x4c, 0x2d, 0xf2, 0xad, 0x21, 0xa1, 0x0c, 0x7d, 0x11, 0xe6, 0xf7, 0x30, 0x25, 0x6d, + 0x63, 0xdd, 0xd8, 0xa8, 0x6f, 0x5e, 0xb8, 0x96, 0xea, 0x56, 0x75, 0x77, 0x9f, 0xf6, 0x6e, 0x61, + 0x4a, 0x2c, 0x81, 0x89, 0xd6, 0xa0, 0xe2, 0xec, 0x75, 0x7d, 0xdc, 0x27, 0xed, 0xc2, 0xba, 0xb1, + 0x51, 0xb3, 0xca, 0xce, 0xde, 0x03, 0xdc, 0x27, 0xe8, 0x35, 0x58, 0xb4, 0x03, 0xcf, 0x23, 0x36, + 0x73, 0x03, 0x5f, 0x22, 0x14, 0x05, 0xc2, 0xc2, 0x08, 0x2c, 0x10, 0x97, 0xa1, 0x84, 0x39, 0x0f, + 0xed, 0x79, 0xd1, 0x2c, 0x3f, 0x4c, 0x0a, 0xad, 0xad, 0x30, 0x18, 0x3c, 0x2b, 0xee, 0xe2, 0x41, + 0x8b, 0xc9, 0x41, 0xff, 0xc0, 0x80, 0x73, 0x37, 0x3d, 0x46, 0xc2, 0x33, 0x2a, 0x94, 0xdf, 0x2f, + 0xc0, 0x9a, 0x5c, 0xb5, 0xdb, 0x31, 0xfa, 0x69, 0x72, 0xb9, 0x0a, 0x65, 0xa9, 0x55, 0x82, 0xcd, + 0x86, 0xa5, 0xbe, 0xd0, 0x45, 0x00, 0x7a, 0x80, 0x43, 0x87, 0x76, 0xfd, 0x61, 0xbf, 0x5d, 0x5a, + 0x37, 0x36, 0x4a, 0x56, 0x4d, 0x42, 0x1e, 0x0c, 0xfb, 0xc8, 0x82, 0x73, 0x76, 0xe0, 0x53, 0x97, + 0x32, 0xe2, 0xdb, 0x47, 0x5d, 0x8f, 0x3c, 0x26, 0x5e, 0xbb, 0xbc, 0x6e, 0x6c, 0x2c, 0x6c, 0xbe, + 0xa2, 0xe5, 0xfb, 0xf6, 0x08, 0xfb, 0x1e, 0x47, 0xb6, 0x5a, 0x76, 0x06, 0x62, 0x7e, 0xd7, 0x80, + 0x15, 0xae, 0x30, 0x67, 0x42, 0x30, 0xe6, 0x9f, 0x1b, 0xb0, 0x7c, 0x17, 0xd3, 0xb3, 0xb1, 0x4a, + 0x17, 0x01, 0x98, 0xdb, 0x27, 0x5d, 0xca, 0x70, 0x7f, 0x20, 0x56, 0x6a, 0xde, 0xaa, 0x71, 0xc8, + 0x2e, 0x07, 0x98, 0xdf, 0x84, 0xc6, 0xad, 0x20, 0xf0, 0x2c, 0x42, 0x07, 0x81, 0x4f, 0x09, 0xba, + 0x01, 0x65, 0xca, 0x30, 0x1b, 0x52, 0xc5, 0xe4, 0x79, 0x2d, 0x93, 0xbb, 0x02, 0xc5, 0x52, 0xa8, + 0x5c, 0x5f, 0x1f, 0x63, 0x6f, 0x28, 0x79, 0xac, 0x5a, 0xf2, 0xc3, 0xfc, 0x04, 0x16, 0x76, 0x59, + 0xe8, 0xfa, 0xbd, 0x9f, 0x62, 0xe7, 0xb5, 0xa8, 0xf3, 0x7f, 0x37, 0xe0, 0x85, 0x2d, 0x42, 0xed, + 0xd0, 0xdd, 0x3b, 0x23, 0xdb, 0xc1, 0x84, 0xc6, 0x08, 0xb2, 0xbd, 0x25, 0x44, 0x5d, 0xb4, 0x52, + 0xb0, 0xcc, 0x62, 0x94, 0xb2, 0x8b, 0xf1, 0xed, 0x12, 0x74, 0x74, 0x93, 0x9a, 0x45, 0x7c, 0x3f, + 0x1f, 0xef, 0xd2, 0x82, 0x20, 0xca, 0xec, 0x31, 0x75, 0x2e, 0x8c, 0x46, 0xdb, 0x15, 0x80, 0x78, + 0x33, 0x67, 0x67, 0x55, 0xd4, 0xcc, 0x6a, 0x13, 0x56, 0x1e, 0xbb, 0x21, 0x1b, 0x62, 0xaf, 0x6b, + 0x1f, 0x60, 0xdf, 0x27, 0x9e, 0x90, 0x13, 0x37, 0x5f, 0xc5, 0x8d, 0x9a, 0xb5, 0xa4, 0x1a, 0x6f, + 0xcb, 0x36, 0x2e, 0x2c, 0x8a, 0xde, 0x82, 0xd5, 0xc1, 0xc1, 0x11, 0x75, 0xed, 0x31, 0xa2, 0x92, + 0x20, 0x5a, 0x8e, 0x5a, 0x53, 0x54, 0x57, 0xe1, 0x9c, 0x2d, 0x2c, 0xa0, 0xd3, 0xe5, 0x52, 0x93, + 0x62, 0x2c, 0x0b, 0x31, 0xb6, 0x54, 0xc3, 0x47, 0x11, 0x9c, 0xb3, 0x15, 0x21, 0x0f, 0x99, 0x9d, + 0x20, 0xa8, 0x08, 0x82, 0x25, 0xd5, 0xf8, 0x31, 0xb3, 0x47, 0x34, 0x69, 0xdb, 0x55, 0xcd, 0xda, + 0xae, 0x36, 0x54, 0x84, 0x2d, 0x26, 0xb4, 0x5d, 0x13, 0x6c, 0x46, 0x9f, 0x68, 0x1b, 0x16, 0x29, + 0xc3, 0x21, 0xeb, 0x0e, 0x02, 0xea, 0x72, 0xb9, 0xd0, 0x36, 0xac, 0x17, 0x37, 0xea, 0x9b, 0xeb, + 0xda, 0x45, 0xfa, 0x90, 0x1c, 0x6d, 0x61, 0x86, 0x77, 0xb0, 0x1b, 0x5a, 0x0b, 0x82, 0x70, 0x27, + 0xa2, 0xd3, 0x1b, 0xc8, 0xfa, 0x4c, 0x06, 0x52, 0xa7, 0xc5, 0x0d, 0xad, 0xed, 0xfa, 0x91, 0x01, + 0x2b, 0xf7, 0x02, 0xec, 0x9c, 0x8d, 0x3d, 0xf5, 0x0a, 0x2c, 0x84, 0x64, 0xe0, 0xb9, 0x36, 0xe6, + 0xeb, 0xb1, 0x47, 0x42, 0xb1, 0xab, 0x4a, 0x56, 0x53, 0x41, 0x1f, 0x08, 0xa0, 0xf9, 0xb9, 0x01, + 0x6d, 0x8b, 0x78, 0x04, 0xd3, 0xb3, 0x61, 0x0b, 0xcc, 0xef, 0x1b, 0xf0, 0xe2, 0x1d, 0xc2, 0x12, + 0xbb, 0x8a, 0x61, 0xe6, 0x52, 0xe6, 0xda, 0xa7, 0xe9, 0x57, 0x98, 0xdf, 0x33, 0xe0, 0x52, 0x2e, + 0x5b, 0xb3, 0x18, 0x99, 0x77, 0xa0, 0xc4, 0x7f, 0xd1, 0x76, 0x41, 0xe8, 0xfc, 0xe5, 0x3c, 0x9d, + 0xff, 0x3a, 0xb7, 0xdd, 0x42, 0xe9, 0x25, 0xbe, 0xf9, 0x5f, 0x06, 0xac, 0xee, 0x1e, 0x04, 0x87, + 0x23, 0x96, 0x9e, 0x85, 0x80, 0xd2, 0x66, 0xb7, 0x98, 0x31, 0xbb, 0xe8, 0x4d, 0x98, 0x67, 0x47, + 0x03, 0x22, 0x74, 0x6b, 0x61, 0xf3, 0xe2, 0x35, 0x8d, 0x3b, 0x7d, 0x8d, 0x33, 0xf9, 0xd1, 0xd1, + 0x80, 0x58, 0x02, 0x15, 0xbd, 0x0e, 0xad, 0x8c, 0xc8, 0x23, 0xc3, 0xb5, 0x98, 0x96, 0x39, 0x35, + 0x7f, 0x5c, 0x80, 0xb5, 0xb1, 0x29, 0xce, 0x22, 0x6c, 0xdd, 0xd8, 0x05, 0xed, 0xd8, 0x7c, 0xff, + 0x24, 0x50, 0x5d, 0x87, 0x7b, 0xbc, 0xc5, 0x8d, 0xa2, 0xd5, 0x4c, 0xd8, 0x6f, 0x87, 0xa2, 0x37, + 0x00, 0x8d, 0x99, 0x55, 0x69, 0xbd, 0xe7, 0xad, 0x73, 0x59, 0xbb, 0x2a, 0x6c, 0xb7, 0xd6, 0xb0, + 0x4a, 0x11, 0xcc, 0x5b, 0xcb, 0x1a, 0xcb, 0x4a, 0xd1, 0x9b, 0xb0, 0xec, 0xfa, 0xf7, 0x49, 0x3f, + 0x08, 0x8f, 0xba, 0x03, 0x12, 0xda, 0xc4, 0x67, 0xb8, 0x47, 0x68, 0xbb, 0x2c, 0x38, 0x5a, 0x8a, + 0xda, 0x76, 0x46, 0x4d, 0xe6, 0x5f, 0x1b, 0xb0, 0x2a, 0x3d, 0xde, 0x1d, 0x1c, 0x32, 0xf7, 0x0c, + 0x58, 0xa3, 0x41, 0xc4, 0x87, 0xc4, 0x93, 0xfe, 0x79, 0x33, 0x86, 0x8a, 0x5d, 0xf6, 0x43, 0x03, + 0x96, 0xb9, 0x33, 0xfa, 0x3c, 0xf1, 0xfc, 0x57, 0x06, 0x2c, 0xdd, 0xc5, 0xf4, 0x79, 0x62, 0xf9, + 0x27, 0xea, 0xa4, 0x8a, 0x79, 0x3e, 0xd5, 0x2b, 0xdb, 0x6b, 0xb0, 0x98, 0x66, 0x3a, 0xf2, 0x7e, + 0x16, 0x52, 0x5c, 0x53, 0xcd, 0x91, 0x56, 0xd2, 0x1d, 0x69, 0x7f, 0x3b, 0x3a, 0xd2, 0x9e, 0xaf, + 0x09, 0x9a, 0x7f, 0x67, 0xc0, 0xc5, 0x3b, 0x84, 0xc5, 0x5c, 0x9f, 0x89, 0xa3, 0x6f, 0x5a, 0xa5, + 0xfa, 0x5c, 0x1e, 0xdc, 0x5a, 0xe6, 0x4f, 0xe5, 0x80, 0xfc, 0x6e, 0x01, 0x56, 0xf8, 0xe9, 0x71, + 0x36, 0x94, 0x60, 0x9a, 0x3b, 0x8e, 0x46, 0x51, 0x4a, 0xda, 0x9d, 0x10, 0x1d, 0xbb, 0xe5, 0xa9, + 0x8f, 0x5d, 0xf3, 0x47, 0x05, 0xe9, 0x2e, 0x24, 0xa5, 0x31, 0xcb, 0xb2, 0x68, 0x78, 0x2d, 0x68, + 0x79, 0x35, 0xa1, 0x11, 0x43, 0xb6, 0xb7, 0xa2, 0x63, 0x34, 0x05, 0x3b, 0xb3, 0xa7, 0xe8, 0x67, + 0x06, 0xac, 0x46, 0xb7, 0xca, 0x5d, 0xd2, 0xeb, 0x13, 0x9f, 0x3d, 0xbd, 0x0e, 0x65, 0x35, 0xa0, + 0xa0, 0xd1, 0x80, 0x0b, 0x50, 0xa3, 0x72, 0x9c, 0xf8, 0xc2, 0x38, 0x02, 0x98, 0x7f, 0x6f, 0xc0, + 0xda, 0x18, 0x3b, 0xb3, 0x2c, 0x62, 0x1b, 0x2a, 0xae, 0xef, 0x90, 0x27, 0x31, 0x37, 0xd1, 0x27, + 0x6f, 0xd9, 0x1b, 0xba, 0x9e, 0x13, 0xb3, 0x11, 0x7d, 0xa2, 0xcb, 0xd0, 0x20, 0x3e, 0xde, 0xf3, + 0x48, 0x57, 0xe0, 0x0a, 0x45, 0xae, 0x5a, 0x75, 0x09, 0xdb, 0xe6, 0x20, 0x4e, 0xbc, 0xef, 0x12, + 0x41, 0x5c, 0x92, 0xc4, 0xea, 0xd3, 0xfc, 0x2d, 0x03, 0x96, 0xb8, 0x16, 0x2a, 0xee, 0xe9, 0xb3, + 0x95, 0xe6, 0x3a, 0xd4, 0x13, 0x6a, 0xa6, 0x26, 0x92, 0x04, 0x99, 0x8f, 0x60, 0x39, 0xcd, 0xce, + 0x2c, 0xd2, 0x7c, 0x11, 0x20, 0x5e, 0x2b, 0xb9, 0x1b, 0x8a, 0x56, 0x02, 0x62, 0x7e, 0x56, 0x88, + 0x62, 0xc7, 0x42, 0x4c, 0xa7, 0x1c, 0xda, 0x12, 0x4b, 0x92, 0xb4, 0xe7, 0x35, 0x01, 0x11, 0xcd, + 0x5b, 0xd0, 0x20, 0x4f, 0x58, 0x88, 0xbb, 0x03, 0x1c, 0xe2, 0xbe, 0xdc, 0x56, 0x53, 0x99, 0xde, + 0xba, 0x20, 0xdb, 0x11, 0x54, 0x7c, 0x10, 0xa1, 0x22, 0x72, 0x90, 0xb2, 0x1c, 0x44, 0x40, 0xc4, + 0x81, 0xf1, 0x4f, 0xdc, 0xd9, 0x53, 0xda, 0x7c, 0xd6, 0x05, 0x92, 0x9e, 0x4a, 0x29, 0x3b, 0x95, + 0x3f, 0x33, 0xa0, 0x25, 0xa6, 0x20, 0xe7, 0x33, 0xe0, 0xdd, 0x66, 0x68, 0x8c, 0x0c, 0xcd, 0x84, + 0xbd, 0xf7, 0x73, 0x50, 0x56, 0x72, 0x2f, 0x4e, 0x2b, 0x77, 0x45, 0x70, 0xcc, 0x34, 0xcc, 0x3f, + 0x36, 0x60, 0x25, 0x23, 0xf2, 0x59, 0x14, 0xfe, 0x23, 0x40, 0x72, 0x86, 0xce, 0x68, 0xda, 0xd1, + 0x39, 0xfd, 0x8a, 0xf6, 0x50, 0xca, 0x0a, 0xc9, 0x3a, 0xe7, 0x66, 0x20, 0xd4, 0xfc, 0x57, 0x03, + 0x2e, 0xdc, 0x21, 0x4c, 0xa0, 0xde, 0xe2, 0x46, 0x67, 0x27, 0x0c, 0x7a, 0x21, 0xa1, 0xf4, 0xf9, + 0xd5, 0x8f, 0xdf, 0x95, 0x8e, 0x9d, 0x6e, 0x4a, 0xb3, 0xc8, 0xff, 0x32, 0x34, 0xc4, 0x18, 0xc4, + 0xe9, 0x86, 0xc1, 0x21, 0x55, 0x7a, 0x54, 0x57, 0x30, 0x2b, 0x38, 0x14, 0x0a, 0xc1, 0x02, 0x86, + 0x3d, 0x89, 0xa0, 0x4e, 0x14, 0x01, 0xe1, 0xcd, 0x62, 0x0f, 0x46, 0x8c, 0xf1, 0xce, 0xc9, 0xf3, + 0x2b, 0xe3, 0x3f, 0x35, 0x60, 0x25, 0x33, 0x95, 0x59, 0x64, 0xfb, 0x25, 0xe9, 0x76, 0xca, 0xc9, + 0x2c, 0x6c, 0x5e, 0xd2, 0xd2, 0x24, 0x06, 0x93, 0xd8, 0xe8, 0x12, 0xd4, 0xf7, 0xb1, 0xeb, 0x75, + 0x43, 0x82, 0x69, 0xe0, 0xab, 0x89, 0x02, 0x07, 0x59, 0x02, 0x62, 0xfe, 0xa3, 0x21, 0x13, 0x74, + 0xcf, 0xb9, 0xc5, 0xfb, 0x93, 0x02, 0x34, 0xb7, 0x7d, 0x4a, 0x42, 0x76, 0xf6, 0xaf, 0x26, 0xe8, + 0x7d, 0xa8, 0x8b, 0x89, 0xd1, 0xae, 0x83, 0x19, 0x56, 0xa7, 0xd9, 0x8b, 0xda, 0x68, 0xfe, 0x07, + 0x1c, 0x6f, 0x0b, 0x33, 0x6c, 0x49, 0xe9, 0x50, 0xfe, 0x1b, 0x9d, 0x87, 0xda, 0x01, 0xa6, 0x07, + 0xdd, 0x47, 0xe4, 0x48, 0xfa, 0x8b, 0x4d, 0xab, 0xca, 0x01, 0x1f, 0x92, 0x23, 0x8a, 0x5e, 0x80, + 0xaa, 0x3f, 0xec, 0xcb, 0x0d, 0x56, 0x59, 0x37, 0x36, 0x9a, 0x56, 0xc5, 0x1f, 0xf6, 0xc5, 0xf6, + 0xfa, 0xe7, 0x02, 0x2c, 0xdc, 0x1f, 0xf2, 0x8b, 0x90, 0xc8, 0x45, 0x0c, 0x3d, 0xf6, 0x74, 0xca, + 0x78, 0x05, 0x8a, 0xd2, 0xa5, 0xe0, 0x14, 0x6d, 0x2d, 0xe3, 0xdb, 0x5b, 0xd4, 0xe2, 0x48, 0x22, + 0x0e, 0x3f, 0xb4, 0x6d, 0xe5, 0x9d, 0x15, 0x05, 0xb3, 0x35, 0x0e, 0x91, 0xbe, 0xd9, 0x79, 0xa8, + 0x91, 0x30, 0x8c, 0x7d, 0x37, 0x31, 0x15, 0x12, 0x86, 0xb2, 0xd1, 0x84, 0x06, 0xb6, 0x1f, 0xf9, + 0xc1, 0xa1, 0x47, 0x9c, 0x1e, 0x71, 0xc4, 0xb2, 0x57, 0xad, 0x14, 0x4c, 0x2a, 0x06, 0x5f, 0xf8, + 0xae, 0xed, 0x33, 0x71, 0xaa, 0x17, 0xb9, 0x62, 0x70, 0xc8, 0x6d, 0x9f, 0xf1, 0x66, 0x87, 0x78, + 0x84, 0x11, 0xd1, 0x5c, 0x91, 0xcd, 0x12, 0xa2, 0x9a, 0x87, 0x83, 0x98, 0xba, 0x2a, 0x9b, 0x25, + 0x84, 0x37, 0x5f, 0x80, 0xda, 0x28, 0xd9, 0x50, 0x1b, 0x45, 0x1b, 0x05, 0xc0, 0xfc, 0x89, 0x01, + 0xcd, 0x2d, 0xd1, 0xd5, 0x73, 0xa0, 0x74, 0x08, 0xe6, 0xc9, 0x93, 0x41, 0xa8, 0xb6, 0x8e, 0xf8, + 0x3d, 0x51, 0x8f, 0xcc, 0xc7, 0xd0, 0xda, 0xf1, 0xb0, 0x4d, 0x0e, 0x02, 0xcf, 0x21, 0xa1, 0x38, + 0xdb, 0x51, 0x0b, 0x8a, 0x0c, 0xf7, 0x94, 0xf3, 0xc0, 0x7f, 0xa2, 0x2f, 0xab, 0xab, 0x9f, 0x34, + 0x4b, 0x2f, 0x6b, 0x4f, 0xd9, 0x44, 0x37, 0x89, 0xc0, 0xeb, 0x2a, 0x94, 0x45, 0x02, 0x50, 0xba, + 0x15, 0x0d, 0x4b, 0x7d, 0x99, 0x0f, 0x53, 0xe3, 0xde, 0x09, 0x83, 0xe1, 0x00, 0x6d, 0x43, 0x63, + 0x30, 0x82, 0x71, 0x5d, 0xcd, 0x3f, 0xd3, 0xb3, 0x4c, 0x5b, 0x29, 0x52, 0xf3, 0xbf, 0x8b, 0xd0, + 0xdc, 0x25, 0x38, 0xb4, 0x0f, 0x9e, 0x8b, 0x20, 0x53, 0x0b, 0x8a, 0x0e, 0xf5, 0xd4, 0xaa, 0xf1, + 0x9f, 0xe8, 0x2a, 0x9c, 0x4b, 0x4c, 0xa8, 0xdb, 0xe3, 0x02, 0x12, 0x7a, 0xdf, 0xb0, 0x5a, 0x83, + 0xac, 0xe0, 0xde, 0x81, 0xaa, 0x43, 0xbd, 0xae, 0x58, 0xa2, 0x8a, 0x58, 0x22, 0xfd, 0xfc, 0xb6, + 0xa8, 0x27, 0x96, 0xa6, 0xe2, 0xc8, 0x1f, 0xe8, 0x25, 0x68, 0x06, 0x43, 0x36, 0x18, 0xb2, 0xae, + 0xb4, 0x3b, 0xed, 0xaa, 0x60, 0xaf, 0x21, 0x81, 0xc2, 0x2c, 0x51, 0xf4, 0x01, 0x34, 0xa9, 0x10, + 0x65, 0xe4, 0x98, 0xd7, 0xa6, 0x75, 0x10, 0x1b, 0x92, 0x4e, 0x79, 0xe6, 0xaf, 0x43, 0x8b, 0x85, + 0xf8, 0x31, 0xf1, 0x12, 0xa9, 0x3d, 0x10, 0xbb, 0x6d, 0x51, 0xc2, 0x47, 0x69, 0xbd, 0xeb, 0xb0, + 0xd4, 0x1b, 0xe2, 0x10, 0xfb, 0x8c, 0x90, 0x04, 0x76, 0x5d, 0x60, 0xa3, 0xb8, 0x29, 0x26, 0x30, + 0x3f, 0x84, 0xf9, 0xbb, 0x2e, 0x13, 0x82, 0xe4, 0x36, 0xcb, 0x10, 0xd7, 0x20, 0x61, 0x99, 0x5e, + 0x80, 0x6a, 0x18, 0x1c, 0x4a, 0x1b, 0x5c, 0x10, 0x2a, 0x58, 0x09, 0x83, 0x43, 0x61, 0x60, 0x45, + 0x41, 0x44, 0x10, 0x2a, 0xdd, 0x2c, 0x58, 0xea, 0xcb, 0xfc, 0x4b, 0x63, 0xa4, 0x3c, 0xdc, 0x7c, + 0xd2, 0xa7, 0xb3, 0x9f, 0xef, 0x43, 0x25, 0x94, 0xf4, 0x13, 0x53, 0xb9, 0xc9, 0x91, 0xc4, 0x19, + 0x10, 0x51, 0x4d, 0x9f, 0x27, 0xfa, 0x55, 0x03, 0x1a, 0x1f, 0x78, 0x43, 0xfa, 0x2c, 0x94, 0x5d, + 0x97, 0xbd, 0x28, 0xea, 0x33, 0x27, 0xbf, 0x5d, 0x80, 0xa6, 0x62, 0x63, 0x16, 0x27, 0x28, 0x97, + 0x95, 0x5d, 0xa8, 0xf3, 0x21, 0xbb, 0x94, 0xf4, 0xa2, 0x98, 0x4e, 0x7d, 0x73, 0x53, 0x6b, 0x1e, + 0x52, 0x6c, 0x88, 0x6c, 0xf9, 0xae, 0x20, 0xfa, 0x45, 0x9f, 0x85, 0x47, 0x16, 0xd8, 0x31, 0xa0, + 0xf3, 0x10, 0x16, 0x33, 0xcd, 0x5c, 0x89, 0x1e, 0x91, 0xa3, 0xc8, 0xfe, 0x3d, 0x22, 0x47, 0xe8, + 0xad, 0x64, 0x4d, 0x43, 0xde, 0x29, 0x7e, 0x2f, 0xf0, 0x7b, 0x37, 0xc3, 0x10, 0x1f, 0xa9, 0x9a, + 0x87, 0x77, 0x0b, 0x5f, 0x36, 0xcc, 0x7f, 0x28, 0x40, 0xe3, 0x6b, 0x43, 0x12, 0x1e, 0x9d, 0xa6, + 0x1d, 0x8a, 0x4e, 0x85, 0xf9, 0xc4, 0xa9, 0x30, 0xb6, 0xf5, 0x4b, 0x9a, 0xad, 0xaf, 0x31, 0x60, + 0x65, 0xad, 0x01, 0xd3, 0xed, 0xed, 0xca, 0x89, 0xf6, 0x76, 0x35, 0x77, 0x6f, 0xff, 0x85, 0x11, + 0x8b, 0x70, 0xa6, 0xdd, 0x98, 0x72, 0xc7, 0x0a, 0x27, 0x76, 0xc7, 0xa6, 0xde, 0x8d, 0x3f, 0x34, + 0xa0, 0xf6, 0x75, 0x62, 0xb3, 0x20, 0xe4, 0xf6, 0x47, 0x43, 0x66, 0x4c, 0xe1, 0x1a, 0x17, 0xb2, + 0xae, 0xf1, 0x0d, 0xa8, 0xba, 0x4e, 0x17, 0x73, 0xfd, 0x12, 0xe3, 0x4e, 0x72, 0xc9, 0x2a, 0xae, + 0x23, 0x14, 0x71, 0xfa, 0x24, 0xc0, 0xef, 0x19, 0xd0, 0x90, 0x3c, 0x53, 0x49, 0xf9, 0x5e, 0x62, + 0x38, 0x43, 0xa7, 0xf4, 0xea, 0x23, 0x9e, 0xe8, 0xdd, 0xb9, 0xd1, 0xb0, 0x37, 0x01, 0xb8, 0x90, + 0x15, 0xb9, 0xdc, 0x33, 0xeb, 0x5a, 0x6e, 0x25, 0xb9, 0x10, 0xf8, 0xdd, 0x39, 0xab, 0xc6, 0xa9, + 0x44, 0x17, 0xb7, 0x2a, 0x50, 0x12, 0xd4, 0xe6, 0xff, 0x19, 0xb0, 0x74, 0x1b, 0x7b, 0xf6, 0x96, + 0x4b, 0x19, 0xf6, 0xed, 0x19, 0x9c, 0xb0, 0x77, 0xa1, 0x12, 0x0c, 0xba, 0x1e, 0xd9, 0x67, 0x8a, + 0xa5, 0xcb, 0x13, 0x66, 0x24, 0xc5, 0x60, 0x95, 0x83, 0xc1, 0x3d, 0xb2, 0xcf, 0xd0, 0x57, 0xa0, + 0x1a, 0x0c, 0xba, 0xa1, 0xdb, 0x3b, 0x60, 0x4a, 0xfa, 0x53, 0x10, 0x57, 0x82, 0x81, 0xc5, 0x29, + 0x12, 0xb1, 0x95, 0xf9, 0x13, 0xc6, 0x56, 0xcc, 0x7f, 0x1b, 0x9b, 0xfe, 0x0c, 0x7b, 0xe0, 0x5d, + 0xa8, 0xba, 0x3e, 0xeb, 0x3a, 0x2e, 0x8d, 0x44, 0x70, 0x51, 0xaf, 0x43, 0x3e, 0x13, 0x33, 0x10, + 0x6b, 0xea, 0x33, 0x3e, 0x36, 0xfa, 0x2a, 0xc0, 0xbe, 0x17, 0x60, 0x45, 0x2d, 0x65, 0x70, 0x49, + 0xbf, 0x7d, 0x38, 0x5a, 0x44, 0x5f, 0x13, 0x44, 0xbc, 0x87, 0xd1, 0x92, 0xfe, 0x8b, 0x01, 0x2b, + 0x3b, 0x24, 0x94, 0x15, 0x2f, 0x4c, 0x85, 0x41, 0xb7, 0xfd, 0xfd, 0x20, 0x1d, 0x89, 0x36, 0x32, + 0x91, 0xe8, 0x9f, 0x4e, 0xf4, 0x35, 0x75, 0x73, 0x92, 0xf9, 0x90, 0xe8, 0xe6, 0x14, 0x65, 0x7d, + 0xe4, 0xcd, 0x73, 0x21, 0x67, 0x99, 0x14, 0xbf, 0xc9, 0x0b, 0xb8, 0xf9, 0x3b, 0xb2, 0x50, 0x43, + 0x3b, 0xa9, 0xa7, 0x57, 0xd8, 0x55, 0x50, 0x96, 0x3e, 0x63, 0xf7, 0x5f, 0x85, 0x8c, 0xed, 0xc8, + 0x31, 0x44, 0x3f, 0x30, 0x60, 0x3d, 0x9f, 0xab, 0x59, 0x8e, 0xe8, 0xaf, 0x42, 0xc9, 0xf5, 0xf7, + 0x83, 0x28, 0xec, 0x76, 0x45, 0xef, 0xa2, 0x6b, 0xc7, 0x95, 0x84, 0xe6, 0xdf, 0x14, 0xa0, 0x25, + 0x8c, 0xfa, 0x29, 0x2c, 0x7f, 0x9f, 0xf4, 0xbb, 0xd4, 0xfd, 0x94, 0x44, 0xcb, 0xdf, 0x27, 0xfd, + 0x5d, 0xf7, 0x53, 0x92, 0xd2, 0x8c, 0x52, 0x5a, 0x33, 0x26, 0x47, 0x95, 0x93, 0x61, 0xd5, 0x4a, + 0x3a, 0xac, 0xba, 0x0a, 0x65, 0x3f, 0x70, 0xc8, 0xf6, 0x96, 0xba, 0x76, 0xaa, 0xaf, 0x91, 0xaa, + 0xd5, 0x4e, 0xa8, 0x6a, 0x9f, 0x1b, 0xd0, 0xb9, 0x43, 0x58, 0x56, 0x76, 0xa7, 0xa7, 0x65, 0xdf, + 0x33, 0xe0, 0xbc, 0x96, 0xa1, 0x59, 0x14, 0xec, 0xbd, 0xb4, 0x82, 0xe9, 0xef, 0x80, 0x63, 0x43, + 0x2a, 0xdd, 0x7a, 0x13, 0x1a, 0x5b, 0xc3, 0x7e, 0x3f, 0x76, 0xb9, 0x2e, 0x43, 0x23, 0x94, 0x3f, + 0xe5, 0x15, 0x49, 0x9e, 0xbf, 0x75, 0x05, 0xe3, 0x17, 0x21, 0xf3, 0x2a, 0x34, 0x15, 0x89, 0xe2, + 0xba, 0x03, 0xd5, 0x50, 0xfd, 0x56, 0xf8, 0xf1, 0xb7, 0xb9, 0x02, 0x4b, 0x16, 0xe9, 0x71, 0xd5, + 0x0e, 0xef, 0xb9, 0xfe, 0x23, 0x35, 0x8c, 0xf9, 0x1d, 0x03, 0x96, 0xd3, 0x70, 0xd5, 0xd7, 0xdb, + 0x50, 0xc1, 0x8e, 0x13, 0x12, 0x4a, 0x27, 0x2e, 0xcb, 0x4d, 0x89, 0x63, 0x45, 0xc8, 0x09, 0xc9, + 0x15, 0xa6, 0x96, 0x9c, 0xd9, 0x85, 0x73, 0x77, 0x08, 0xbb, 0x4f, 0x58, 0x38, 0x53, 0x06, 0xbf, + 0xcd, 0x2f, 0x2f, 0x82, 0x58, 0xa9, 0x45, 0xf4, 0x69, 0x7e, 0x66, 0x00, 0x4a, 0x8e, 0x30, 0xcb, + 0x32, 0x27, 0xa5, 0x5c, 0x48, 0x4b, 0x59, 0xd6, 0x42, 0xf5, 0x07, 0x81, 0x4f, 0x7c, 0x96, 0x74, + 0xb7, 0x9a, 0x31, 0x34, 0x2a, 0x2b, 0x41, 0xf7, 0x02, 0xec, 0xdc, 0xc2, 0xde, 0x6c, 0xee, 0xc1, + 0x45, 0x00, 0x1a, 0xda, 0x5d, 0xb5, 0x5b, 0x0b, 0xca, 0xfa, 0x84, 0xf6, 0x03, 0xb9, 0x61, 0x2f, + 0x41, 0xdd, 0xa1, 0x4c, 0x35, 0x47, 0x09, 0x65, 0x70, 0x28, 0x93, 0xed, 0xa2, 0xd6, 0x95, 0x12, + 0xec, 0x11, 0xa7, 0x9b, 0xc8, 0xc7, 0xcd, 0x0b, 0xb4, 0x96, 0x6c, 0xd8, 0x8d, 0xe1, 0x9a, 0xcd, + 0x55, 0xd2, 0x6e, 0xae, 0x87, 0xb0, 0x76, 0x1f, 0xfb, 0x43, 0xec, 0xdd, 0x0e, 0xfa, 0x03, 0x9c, + 0xaa, 0x93, 0xcc, 0x9a, 0x43, 0x43, 0x63, 0x0e, 0x5f, 0x94, 0x85, 0x74, 0xd2, 0x05, 0x17, 0x73, + 0x9a, 0xb7, 0x12, 0x10, 0x93, 0x42, 0x7b, 0xbc, 0xfb, 0x59, 0x16, 0x54, 0x30, 0x15, 0x75, 0x95, + 0xb4, 0xd1, 0x23, 0x98, 0xf9, 0x3e, 0xbc, 0x20, 0x8a, 0x1a, 0x23, 0x50, 0x2a, 0x05, 0x90, 0xed, + 0xc0, 0xd0, 0x74, 0xf0, 0xeb, 0x05, 0x61, 0x02, 0xc7, 0x7a, 0x98, 0x85, 0xf1, 0x77, 0xd3, 0x91, + 0xf7, 0x97, 0x73, 0x0a, 0x77, 0xd3, 0x23, 0xaa, 0xf0, 0xfb, 0x06, 0x2c, 0x92, 0x27, 0xc4, 0x1e, + 0x32, 0xd7, 0xef, 0xed, 0x78, 0xd8, 0x7f, 0x10, 0xa8, 0x83, 0x27, 0x0b, 0x46, 0x2f, 0x43, 0x93, + 0x4b, 0x3f, 0x18, 0x32, 0x85, 0x27, 0x4f, 0xa0, 0x34, 0x90, 0xf7, 0xc7, 0xe7, 0xeb, 0x11, 0x46, + 0x1c, 0x85, 0x27, 0x8f, 0xa3, 0x2c, 0x78, 0x4c, 0x94, 0x1c, 0x4c, 0x4f, 0x22, 0xca, 0xff, 0x30, + 0x32, 0xa2, 0x54, 0x3d, 0x9c, 0x96, 0x28, 0xef, 0x02, 0xf4, 0x49, 0xd8, 0x23, 0xdb, 0xc2, 0xf8, + 0xcb, 0x1b, 0xfe, 0x86, 0xd6, 0xf8, 0x8f, 0x3a, 0xb8, 0x1f, 0x11, 0x58, 0x09, 0x5a, 0xf3, 0x0e, + 0x2c, 0x69, 0x50, 0xb8, 0x5d, 0xa3, 0xc1, 0x30, 0xb4, 0x49, 0x14, 0x24, 0x8a, 0x3e, 0xf9, 0x39, + 0xc8, 0x70, 0xd8, 0x23, 0x4c, 0x29, 0xad, 0xfa, 0x32, 0xdf, 0x16, 0xc9, 0x2a, 0x11, 0x50, 0x48, + 0x69, 0x6a, 0x3a, 0xf1, 0x6e, 0x8c, 0x25, 0xde, 0xf7, 0x45, 0x66, 0x28, 0x49, 0x37, 0x63, 0xd1, + 0xc4, 0x3e, 0xef, 0x8a, 0x38, 0xea, 0xd1, 0x46, 0xf4, 0x69, 0xfe, 0xaf, 0x01, 0xcd, 0xed, 0xfe, + 0x20, 0x18, 0x25, 0x45, 0xa6, 0xbe, 0x72, 0x8e, 0x07, 0x95, 0x0b, 0xba, 0xa0, 0xf2, 0x4b, 0xd0, + 0x4c, 0x97, 0xfc, 0xcb, 0xf8, 0x4f, 0xc3, 0x4e, 0x96, 0xfa, 0x9f, 0x87, 0x5a, 0x18, 0x1c, 0x76, + 0xb9, 0x29, 0x75, 0x54, 0x79, 0x46, 0x35, 0x0c, 0x0e, 0xb9, 0x81, 0x75, 0xd0, 0x32, 0x94, 0xf6, + 0x5d, 0x2f, 0xae, 0x2c, 0x92, 0x1f, 0xe8, 0x3d, 0x7e, 0x21, 0x93, 0xe9, 0xdb, 0xf2, 0xb4, 0xf7, + 0xa2, 0x88, 0xc2, 0xfc, 0x04, 0x16, 0xa2, 0x59, 0xcf, 0xf8, 0x5a, 0x85, 0x61, 0xfa, 0x28, 0xaa, + 0x9c, 0x90, 0x1f, 0xe6, 0x55, 0x99, 0xd5, 0x13, 0xfd, 0xa7, 0x16, 0x1d, 0xc1, 0x3c, 0xc7, 0x50, + 0x7b, 0x49, 0xfc, 0xe6, 0x0b, 0xb0, 0x9a, 0xc5, 0x9e, 0x85, 0xa5, 0xb7, 0xd3, 0xfb, 0x47, 0xff, + 0x20, 0x21, 0x39, 0x9a, 0xda, 0x3b, 0x6a, 0x05, 0xec, 0x60, 0xe8, 0x33, 0x65, 0x80, 0xf8, 0x0a, + 0xdc, 0xe6, 0xdf, 0x68, 0x0d, 0x2a, 0xae, 0xd3, 0xf5, 0xf8, 0xdd, 0x4d, 0x9e, 0x49, 0x65, 0xd7, + 0xb9, 0xc7, 0xef, 0x75, 0xef, 0x44, 0x9e, 0xd6, 0xd4, 0xe5, 0x16, 0xca, 0xcb, 0xfa, 0xbe, 0xf4, + 0x03, 0x2c, 0x59, 0x06, 0xf9, 0x8c, 0x8b, 0x6a, 0x36, 0xa0, 0x75, 0xe8, 0xb2, 0x83, 0xae, 0x78, + 0xda, 0x21, 0x0e, 0x61, 0x99, 0x57, 0xae, 0x5a, 0x0b, 0x1c, 0xbe, 0xcb, 0xc1, 0xfc, 0x20, 0xa6, + 0xe6, 0x6f, 0x18, 0xb0, 0x94, 0x62, 0x6b, 0x96, 0xa5, 0xf8, 0x0a, 0xf7, 0x4f, 0x64, 0x47, 0xca, + 0x13, 0x5d, 0xd7, 0x1a, 0x23, 0x35, 0x9a, 0x30, 0x42, 0x31, 0x85, 0xf9, 0x9f, 0x06, 0xd4, 0x13, + 0x2d, 0xfc, 0x7a, 0xa3, 0xda, 0x46, 0xd7, 0x9b, 0x18, 0x30, 0x95, 0x18, 0x5e, 0x82, 0xd1, 0xd6, + 0x4c, 0x94, 0x87, 0x27, 0xea, 0xda, 0x1c, 0x8a, 0xee, 0xc2, 0x82, 0x14, 0x53, 0xcc, 0xba, 0x36, + 0xea, 0x10, 0x57, 0xec, 0xe1, 0xd0, 0x51, 0x5c, 0x5a, 0x4d, 0x9a, 0xf8, 0x92, 0x49, 0xc6, 0xc0, + 0x21, 0x62, 0xa4, 0x92, 0xb4, 0x96, 0xc2, 0xff, 0x71, 0x28, 0xbf, 0x86, 0x34, 0x92, 0xa4, 0xdc, + 0x95, 0xf3, 0x08, 0x76, 0x48, 0x18, 0xcf, 0x2d, 0xfe, 0xe6, 0xbe, 0x93, 0xfc, 0xdd, 0xe5, 0xae, + 0xad, 0x32, 0x32, 0x20, 0x41, 0xdc, 0xeb, 0x45, 0xaf, 0xc2, 0xa2, 0xd3, 0x4f, 0xbd, 0x2b, 0x8a, + 0x9c, 0x3d, 0xa7, 0x9f, 0x78, 0x50, 0x94, 0x62, 0x68, 0x3e, 0xcd, 0xd0, 0xff, 0x18, 0xf1, 0x6b, + 0xcb, 0x90, 0x38, 0xc4, 0x67, 0x2e, 0xf6, 0x9e, 0x5e, 0x27, 0x3b, 0x50, 0x1d, 0x52, 0x12, 0x26, + 0x6c, 0x62, 0xfc, 0xcd, 0xdb, 0x06, 0x98, 0xd2, 0xc3, 0x20, 0x74, 0x14, 0x97, 0xf1, 0xf7, 0x84, + 0x22, 0x41, 0xf9, 0x92, 0x4f, 0x5f, 0x24, 0xf8, 0x36, 0xac, 0xf5, 0x03, 0xc7, 0xdd, 0x77, 0x75, + 0xb5, 0x85, 0x9c, 0x6c, 0x25, 0x6a, 0x4e, 0xd1, 0x99, 0x3f, 0x28, 0xc0, 0xda, 0xc7, 0x03, 0xe7, + 0x67, 0x30, 0xe7, 0x75, 0xa8, 0x07, 0x9e, 0xb3, 0x93, 0x9e, 0x76, 0x12, 0xc4, 0x31, 0x7c, 0x72, + 0x18, 0x63, 0xc8, 0x50, 0x73, 0x12, 0x34, 0xb1, 0x80, 0xf2, 0xa9, 0x64, 0x53, 0x9e, 0x24, 0x9b, + 0x1e, 0xac, 0xc9, 0xac, 0xed, 0x33, 0x16, 0x8d, 0xf9, 0x2b, 0xb0, 0xc2, 0x0d, 0x29, 0x1f, 0xe6, + 0x63, 0x4a, 0xc2, 0x19, 0x2d, 0xce, 0x05, 0xa8, 0x45, 0x3d, 0x47, 0xb5, 0xad, 0x23, 0x80, 0x79, + 0x17, 0x96, 0x33, 0x63, 0x3d, 0xe5, 0x8c, 0xae, 0x5c, 0x86, 0x6a, 0x54, 0xab, 0x8b, 0x2a, 0x50, + 0xbc, 0xe9, 0x79, 0xad, 0x39, 0xd4, 0x80, 0xea, 0xb6, 0x2a, 0x48, 0x6d, 0x19, 0x57, 0x7e, 0x01, + 0x16, 0x33, 0x39, 0x5d, 0x54, 0x85, 0xf9, 0x07, 0x81, 0x4f, 0x5a, 0x73, 0xa8, 0x05, 0x8d, 0x5b, + 0xae, 0x8f, 0xc3, 0x23, 0x19, 0xf1, 0x6c, 0x39, 0x68, 0x11, 0xea, 0x22, 0xf2, 0xa7, 0x00, 0x64, + 0xf3, 0xc7, 0x2f, 0x43, 0xf3, 0xbe, 0x60, 0x64, 0x97, 0x84, 0x8f, 0x5d, 0x9b, 0xa0, 0x2e, 0xb4, + 0xb2, 0x0f, 0xa2, 0xd1, 0x17, 0xf4, 0xde, 0x9d, 0xfe, 0xdd, 0x74, 0x67, 0x92, 0x0c, 0xcd, 0x39, + 0xf4, 0x09, 0x2c, 0xa4, 0x9f, 0x15, 0x23, 0x7d, 0x68, 0x4a, 0xfb, 0xf6, 0xf8, 0xb8, 0xce, 0xbb, + 0xd0, 0x4c, 0xbd, 0x12, 0x46, 0xaf, 0x6b, 0xfb, 0xd6, 0xbd, 0x24, 0xee, 0xe8, 0x6d, 0x6f, 0xf2, + 0x25, 0xaf, 0xe4, 0x3e, 0xfd, 0x94, 0x2f, 0x87, 0x7b, 0xed, 0x7b, 0xbf, 0xe3, 0xb8, 0xc7, 0x70, + 0x6e, 0xec, 0xc9, 0x1d, 0x7a, 0x23, 0xe7, 0x34, 0xd3, 0x3f, 0xcd, 0x3b, 0x6e, 0x88, 0x43, 0x40, + 0xe3, 0xaf, 0x61, 0xd1, 0x35, 0xfd, 0x0a, 0xe4, 0xbd, 0x05, 0xee, 0x5c, 0x9f, 0x1a, 0x3f, 0x16, + 0xdc, 0xaf, 0x19, 0xb0, 0x96, 0xf3, 0x4e, 0x0e, 0xdd, 0xd0, 0x76, 0x37, 0xf9, 0xb1, 0x5f, 0xe7, + 0xad, 0x93, 0x11, 0xc5, 0x8c, 0xf8, 0xb0, 0x98, 0x79, 0x3a, 0x86, 0xae, 0xe6, 0xd6, 0xc9, 0x8f, + 0xbf, 0xa1, 0xeb, 0x7c, 0x61, 0x3a, 0xe4, 0x78, 0xbc, 0x87, 0xb0, 0x98, 0x79, 0x6f, 0x95, 0x33, + 0x9e, 0xfe, 0x55, 0xd6, 0x71, 0x0b, 0xfa, 0x4d, 0x68, 0xa6, 0x1e, 0x46, 0xe5, 0x68, 0xbc, 0xee, + 0xf1, 0xd4, 0x71, 0x5d, 0x3f, 0x84, 0x46, 0xf2, 0xfd, 0x12, 0xda, 0xc8, 0xdb, 0x4b, 0x63, 0x1d, + 0x9f, 0x64, 0x2b, 0x8d, 0xde, 0x1d, 0x4c, 0xd8, 0x4a, 0x63, 0x4f, 0x35, 0xa6, 0xdf, 0x4a, 0x89, + 0xfe, 0x27, 0x6e, 0xa5, 0x13, 0x0f, 0xf1, 0x1d, 0x79, 0xa7, 0xd0, 0xbc, 0x6b, 0x41, 0x9b, 0x79, + 0xba, 0x99, 0xff, 0x82, 0xa7, 0x73, 0xe3, 0x44, 0x34, 0xb1, 0x14, 0x1f, 0xc1, 0x42, 0xfa, 0xf5, + 0x46, 0x8e, 0x14, 0xb5, 0x0f, 0x5e, 0x3a, 0x57, 0xa7, 0xc2, 0x8d, 0x07, 0xfb, 0x18, 0xea, 0x89, + 0xff, 0x38, 0x41, 0xaf, 0x4d, 0xd0, 0xe3, 0xe4, 0x1f, 0x7e, 0x1c, 0x27, 0xc9, 0xaf, 0x41, 0x2d, + 0xfe, 0x6b, 0x12, 0xf4, 0x4a, 0xae, 0xfe, 0x9e, 0xa4, 0xcb, 0x5d, 0x80, 0xd1, 0xff, 0x8e, 0xa0, + 0x57, 0xb5, 0x7d, 0x8e, 0xfd, 0x31, 0xc9, 0x71, 0x9d, 0xc6, 0xd3, 0x97, 0x45, 0x71, 0x93, 0xa6, + 0x9f, 0xac, 0xe2, 0x3c, 0xae, 0xdb, 0x03, 0x68, 0xa6, 0x6a, 0xaf, 0xf3, 0xb6, 0xb0, 0xa6, 0x24, + 0xbe, 0x73, 0x65, 0x1a, 0xd4, 0x78, 0xfd, 0x0e, 0xa0, 0x99, 0xaa, 0x84, 0xcd, 0x19, 0x49, 0x57, + 0xf8, 0x9b, 0x33, 0x92, 0xb6, 0xb0, 0xd6, 0x9c, 0x43, 0xdf, 0x4e, 0x14, 0xdd, 0xa6, 0x0a, 0x9b, + 0xd1, 0x9b, 0x13, 0xfb, 0xd1, 0xd5, 0x75, 0x77, 0x36, 0x4f, 0x42, 0x12, 0xb3, 0xa0, 0xb4, 0x4a, + 0x8a, 0x34, 0x5f, 0xab, 0x4e, 0xb2, 0x52, 0xbb, 0x50, 0x96, 0xb5, 0xad, 0xc8, 0xcc, 0xa9, 0x62, + 0x4f, 0x14, 0xbe, 0x76, 0x5e, 0xd2, 0xe2, 0xa4, 0xcb, 0x3e, 0x65, 0xa7, 0xd2, 0x0b, 0xce, 0xe9, + 0x34, 0x55, 0xd8, 0x38, 0x6d, 0xa7, 0x16, 0x94, 0x65, 0xd1, 0x52, 0x4e, 0xa7, 0xa9, 0xc2, 0xbb, + 0xce, 0x64, 0x1c, 0x91, 0xcd, 0x36, 0xe7, 0xd0, 0x0e, 0x94, 0x44, 0xa8, 0x0c, 0x5d, 0x9e, 0x54, + 0xcf, 0x33, 0xa9, 0xc7, 0x54, 0xc9, 0x8f, 0x39, 0x87, 0x7e, 0x09, 0x4a, 0x22, 0x41, 0x94, 0xd3, + 0x63, 0xb2, 0x28, 0xa7, 0x33, 0x11, 0x25, 0x62, 0xd1, 0x81, 0x46, 0x32, 0x13, 0x9f, 0x73, 0x64, + 0x69, 0x6a, 0x15, 0x3a, 0xd3, 0x60, 0x46, 0xa3, 0xc8, 0x6d, 0x34, 0x0a, 0x1b, 0xe6, 0x6f, 0xa3, + 0xb1, 0x90, 0x64, 0xfe, 0x36, 0x1a, 0x8f, 0x42, 0x9a, 0x73, 0xe8, 0x37, 0x0d, 0x68, 0xe7, 0xa5, + 0x87, 0x51, 0xae, 0x07, 0x34, 0x29, 0xc7, 0xdd, 0xf9, 0xd2, 0x09, 0xa9, 0x62, 0x5e, 0x3e, 0x15, + 0x41, 0x9b, 0xb1, 0x84, 0xf0, 0xf5, 0xbc, 0xfe, 0x72, 0xd2, 0x9f, 0x9d, 0x2f, 0x4e, 0x4f, 0x10, + 0x8f, 0xbd, 0x07, 0xf5, 0x44, 0xc0, 0x28, 0xc7, 0xf2, 0x8e, 0x47, 0xba, 0x72, 0x56, 0x55, 0x13, + 0x7b, 0x92, 0xea, 0x2d, 0xf2, 0x8b, 0x39, 0xca, 0x98, 0x4c, 0x57, 0xe6, 0xa8, 0x77, 0x2a, 0x3d, + 0x69, 0xce, 0x21, 0x02, 0x8d, 0x64, 0xb2, 0x31, 0x47, 0x1b, 0x35, 0x79, 0xca, 0xce, 0xeb, 0x53, + 0x60, 0xc6, 0xc3, 0x74, 0x01, 0x46, 0xc9, 0xbe, 0x9c, 0xb3, 0x6e, 0x2c, 0xdf, 0xd8, 0x79, 0xed, + 0x58, 0xbc, 0xe4, 0xb1, 0x9f, 0x48, 0xdf, 0xe5, 0x48, 0x7f, 0x3c, 0xc1, 0x37, 0xc5, 0x5d, 0x64, + 0x3c, 0x45, 0x94, 0x73, 0x17, 0xc9, 0xcd, 0x46, 0x75, 0xae, 0x4f, 0x8d, 0x1f, 0xcf, 0xe7, 0x5b, + 0xd0, 0xca, 0xa6, 0xd4, 0x72, 0xee, 0xb8, 0x39, 0x89, 0xbd, 0xce, 0x1b, 0x53, 0x62, 0x27, 0xcf, + 0xc3, 0xf3, 0xe3, 0x3c, 0x7d, 0xc3, 0x65, 0x07, 0x22, 0x9b, 0x33, 0xcd, 0xac, 0x93, 0x89, 0xa3, + 0x69, 0x66, 0x9d, 0x4a, 0x13, 0xa9, 0xc3, 0x4b, 0x44, 0xa4, 0xf3, 0x0e, 0xaf, 0x64, 0x82, 0x22, + 0xe7, 0x9c, 0x49, 0x87, 0xf3, 0xa5, 0xfb, 0x99, 0x8e, 0xab, 0xa3, 0x7c, 0x3f, 0x61, 0x2c, 0x54, + 0x9f, 0xe3, 0x7e, 0xea, 0x03, 0xf5, 0x42, 0xd1, 0x5b, 0xd9, 0xf0, 0xe1, 0xe4, 0xd8, 0x44, 0x36, + 0xac, 0x74, 0x7c, 0xf8, 0xa0, 0x95, 0x8d, 0xd5, 0xe5, 0x0c, 0x90, 0x13, 0xd2, 0x9b, 0x62, 0x80, + 0x6c, 0xc4, 0x2b, 0x67, 0x80, 0x9c, 0xc0, 0xd8, 0x14, 0xbe, 0x64, 0x2a, 0xfa, 0x94, 0x73, 0x34, + 0xe9, 0x22, 0x54, 0x39, 0x47, 0x93, 0x36, 0x70, 0x66, 0xce, 0x6d, 0x0e, 0xa1, 0xb1, 0x13, 0x06, + 0x4f, 0x8e, 0xa2, 0xc0, 0xd1, 0xcf, 0xc6, 0xd8, 0xdd, 0xfa, 0x06, 0x2c, 0xb8, 0x31, 0x4e, 0x2f, + 0x1c, 0xd8, 0xb7, 0xea, 0x32, 0x80, 0xb5, 0xc3, 0x89, 0x77, 0x8c, 0x5f, 0xbe, 0xd1, 0x73, 0xd9, + 0xc1, 0x70, 0x8f, 0x4b, 0xe6, 0xba, 0x44, 0x7b, 0xc3, 0x0d, 0xd4, 0xaf, 0xeb, 0xae, 0xcf, 0x48, + 0xe8, 0x63, 0xef, 0xba, 0x18, 0x4a, 0x41, 0x07, 0x7b, 0x7f, 0x64, 0x18, 0x7b, 0x65, 0x01, 0xba, + 0xf1, 0xff, 0x01, 0x00, 0x00, 0xff, 0xff, 0x77, 0xf8, 0xe0, 0x92, 0x08, 0x52, 0x00, 0x00, } // Reference imports to suppress errors if they are not otherwise used. diff --git a/internal/proto/query_coord.proto b/internal/proto/query_coord.proto index 691612b98b..b714d3c03c 100644 --- a/internal/proto/query_coord.proto +++ b/internal/proto/query_coord.proto @@ -201,6 +201,7 @@ message WatchDmChannelsRequest { schema.CollectionSchema schema = 6; repeated data.SegmentInfo exclude_infos = 7; LoadMetaInfo load_meta = 8; + int64 replicaID = 9; } message WatchDeltaChannelsRequest { @@ -224,6 +225,7 @@ message SegmentLoadInfo { repeated int64 compactionFrom = 10; // segmentIDs compacted from repeated FieldIndexInfo index_infos = 11; int64 segment_size = 12; + string insert_channel = 13; } message FieldIndexInfo { @@ -245,6 +247,7 @@ message LoadSegmentsRequest { int64 source_nodeID = 5; int64 collectionID = 6; LoadMetaInfo load_meta = 7; + int64 replicaID = 8; } message ReleaseSegmentsRequest { @@ -281,6 +284,7 @@ message LoadBalanceRequest { TriggerCondition balance_reason = 3; repeated int64 dst_nodeIDs = 4; repeated int64 sealed_segmentIDs = 5; + int64 collectionID = 6; } //-------------------- internal meta proto------------------ @@ -312,6 +316,7 @@ message DmChannelWatchInfo { int64 collectionID = 1; string dmChannel = 2; int64 nodeID_loaded = 3; + int64 replicaID = 4; } message QueryChannelInfo { @@ -380,4 +385,3 @@ message SealedSegmentsChangeInfo { common.MsgBase base = 1; repeated SegmentChangeInfo infos = 2; } - diff --git a/internal/proto/querypb/query_coord.pb.go b/internal/proto/querypb/query_coord.pb.go index dc90118846..cf8d1edc13 100644 --- a/internal/proto/querypb/query_coord.pb.go +++ b/internal/proto/querypb/query_coord.pb.go @@ -1332,6 +1332,7 @@ type WatchDmChannelsRequest struct { Schema *schemapb.CollectionSchema `protobuf:"bytes,6,opt,name=schema,proto3" json:"schema,omitempty"` ExcludeInfos []*datapb.SegmentInfo `protobuf:"bytes,7,rep,name=exclude_infos,json=excludeInfos,proto3" json:"exclude_infos,omitempty"` LoadMeta *LoadMetaInfo `protobuf:"bytes,8,opt,name=load_meta,json=loadMeta,proto3" json:"load_meta,omitempty"` + ReplicaID int64 `protobuf:"varint,9,opt,name=replicaID,proto3" json:"replicaID,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -1418,6 +1419,13 @@ func (m *WatchDmChannelsRequest) GetLoadMeta() *LoadMetaInfo { return nil } +func (m *WatchDmChannelsRequest) GetReplicaID() int64 { + if m != nil { + return m.ReplicaID + } + return 0 +} + type WatchDeltaChannelsRequest struct { Base *commonpb.MsgBase `protobuf:"bytes,1,opt,name=base,proto3" json:"base,omitempty"` NodeID int64 `protobuf:"varint,2,opt,name=nodeID,proto3" json:"nodeID,omitempty"` @@ -1502,6 +1510,7 @@ type SegmentLoadInfo struct { CompactionFrom []int64 `protobuf:"varint,10,rep,packed,name=compactionFrom,proto3" json:"compactionFrom,omitempty"` IndexInfos []*FieldIndexInfo `protobuf:"bytes,11,rep,name=index_infos,json=indexInfos,proto3" json:"index_infos,omitempty"` SegmentSize int64 `protobuf:"varint,12,opt,name=segment_size,json=segmentSize,proto3" json:"segment_size,omitempty"` + InsertChannel string `protobuf:"bytes,13,opt,name=insert_channel,json=insertChannel,proto3" json:"insert_channel,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -1616,6 +1625,13 @@ func (m *SegmentLoadInfo) GetSegmentSize() int64 { return 0 } +func (m *SegmentLoadInfo) GetInsertChannel() string { + if m != nil { + return m.InsertChannel + } + return "" +} + type FieldIndexInfo struct { FieldID int64 `protobuf:"varint,1,opt,name=fieldID,proto3" json:"fieldID,omitempty"` EnableIndex bool `protobuf:"varint,2,opt,name=enable_index,json=enableIndex,proto3" json:"enable_index,omitempty"` @@ -1719,6 +1735,7 @@ type LoadSegmentsRequest struct { SourceNodeID int64 `protobuf:"varint,5,opt,name=source_nodeID,json=sourceNodeID,proto3" json:"source_nodeID,omitempty"` CollectionID int64 `protobuf:"varint,6,opt,name=collectionID,proto3" json:"collectionID,omitempty"` LoadMeta *LoadMetaInfo `protobuf:"bytes,7,opt,name=load_meta,json=loadMeta,proto3" json:"load_meta,omitempty"` + ReplicaID int64 `protobuf:"varint,8,opt,name=replicaID,proto3" json:"replicaID,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -1798,6 +1815,13 @@ func (m *LoadSegmentsRequest) GetLoadMeta() *LoadMetaInfo { return nil } +func (m *LoadSegmentsRequest) GetReplicaID() int64 { + if m != nil { + return m.ReplicaID + } + return 0 +} + type ReleaseSegmentsRequest struct { Base *commonpb.MsgBase `protobuf:"bytes,1,opt,name=base,proto3" json:"base,omitempty"` NodeID int64 `protobuf:"varint,2,opt,name=nodeID,proto3" json:"nodeID,omitempty"` @@ -2042,6 +2066,7 @@ type LoadBalanceRequest struct { BalanceReason TriggerCondition `protobuf:"varint,3,opt,name=balance_reason,json=balanceReason,proto3,enum=milvus.proto.query.TriggerCondition" json:"balance_reason,omitempty"` DstNodeIDs []int64 `protobuf:"varint,4,rep,packed,name=dst_nodeIDs,json=dstNodeIDs,proto3" json:"dst_nodeIDs,omitempty"` SealedSegmentIDs []int64 `protobuf:"varint,5,rep,packed,name=sealed_segmentIDs,json=sealedSegmentIDs,proto3" json:"sealed_segmentIDs,omitempty"` + CollectionID int64 `protobuf:"varint,6,opt,name=collectionID,proto3" json:"collectionID,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -2107,10 +2132,18 @@ func (m *LoadBalanceRequest) GetSealedSegmentIDs() []int64 { return nil } +func (m *LoadBalanceRequest) GetCollectionID() int64 { + if m != nil { + return m.CollectionID + } + return 0 +} + type DmChannelWatchInfo struct { CollectionID int64 `protobuf:"varint,1,opt,name=collectionID,proto3" json:"collectionID,omitempty"` DmChannel string `protobuf:"bytes,2,opt,name=dmChannel,proto3" json:"dmChannel,omitempty"` NodeIDLoaded int64 `protobuf:"varint,3,opt,name=nodeID_loaded,json=nodeIDLoaded,proto3" json:"nodeID_loaded,omitempty"` + ReplicaID int64 `protobuf:"varint,4,opt,name=replicaID,proto3" json:"replicaID,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -2162,6 +2195,13 @@ func (m *DmChannelWatchInfo) GetNodeIDLoaded() int64 { return 0 } +func (m *DmChannelWatchInfo) GetReplicaID() int64 { + if m != nil { + return m.ReplicaID + } + return 0 +} + type QueryChannelInfo struct { CollectionID int64 `protobuf:"varint,1,opt,name=collectionID,proto3" json:"collectionID,omitempty"` QueryChannel string `protobuf:"bytes,2,opt,name=query_channel,json=queryChannel,proto3" json:"query_channel,omitempty"` @@ -2795,177 +2835,180 @@ func init() { func init() { proto.RegisterFile("query_coord.proto", fileDescriptor_aab7cc9a69ed26e8) } var fileDescriptor_aab7cc9a69ed26e8 = []byte{ - // 2707 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xec, 0x5a, 0x4b, 0x73, 0x1c, 0x49, - 0x11, 0x56, 0xcf, 0x4b, 0x33, 0x39, 0x0f, 0x8d, 0x4b, 0xb6, 0x76, 0x34, 0xec, 0xae, 0xe5, 0xf6, - 0x4b, 0x78, 0x59, 0xd9, 0xc8, 0x40, 0xec, 0x06, 0x70, 0xb0, 0x24, 0xac, 0x15, 0xb6, 0xb5, 0xda, - 0x96, 0x6c, 0xc0, 0xe1, 0x88, 0xa1, 0x67, 0xba, 0x34, 0xea, 0x70, 0x3f, 0xc6, 0x5d, 0x3d, 0xb6, - 0x65, 0xae, 0x1c, 0x76, 0x79, 0x04, 0xc1, 0x69, 0x2f, 0x04, 0x27, 0x08, 0xe0, 0xb0, 0xc1, 0x99, - 0x1b, 0x3f, 0x81, 0x08, 0xee, 0x04, 0x37, 0xf8, 0x03, 0x1c, 0x09, 0x88, 0x7a, 0x74, 0x4f, 0x3f, - 0xaa, 0x35, 0x2d, 0x69, 0xbd, 0x76, 0x10, 0xdc, 0xba, 0xb2, 0xaa, 0x32, 0xb3, 0x2a, 0xb3, 0x32, - 0xbf, 0xaa, 0x6c, 0x38, 0xf3, 0x64, 0x8c, 0xbd, 0xc3, 0xde, 0xc0, 0x75, 0x3d, 0x63, 0x65, 0xe4, - 0xb9, 0xbe, 0x8b, 0x90, 0x6d, 0x5a, 0x4f, 0xc7, 0x84, 0xb7, 0x56, 0x58, 0x7f, 0xb7, 0x31, 0x70, - 0x6d, 0xdb, 0x75, 0x38, 0xad, 0xdb, 0x88, 0x8e, 0xe8, 0xb6, 0x4c, 0xc7, 0xc7, 0x9e, 0xa3, 0x5b, - 0x41, 0x2f, 0x19, 0x1c, 0x60, 0x5b, 0x17, 0xad, 0xb6, 0xa1, 0xfb, 0x7a, 0x94, 0xbf, 0xfa, 0x63, - 0x05, 0x16, 0x76, 0x0f, 0xdc, 0x67, 0xeb, 0xae, 0x65, 0xe1, 0x81, 0x6f, 0xba, 0x0e, 0xd1, 0xf0, - 0x93, 0x31, 0x26, 0x3e, 0xba, 0x01, 0xa5, 0xbe, 0x4e, 0x70, 0x47, 0x59, 0x52, 0x96, 0xeb, 0xab, - 0x6f, 0xae, 0xc4, 0x34, 0x11, 0x2a, 0xdc, 0x23, 0xc3, 0x35, 0x9d, 0x60, 0x8d, 0x8d, 0x44, 0x08, - 0x4a, 0x46, 0x7f, 0x6b, 0xa3, 0x53, 0x58, 0x52, 0x96, 0x8b, 0x1a, 0xfb, 0x46, 0x97, 0xa0, 0x39, - 0x08, 0x79, 0x6f, 0x6d, 0x90, 0x4e, 0x71, 0xa9, 0xb8, 0x5c, 0xd4, 0xe2, 0x44, 0xf5, 0x77, 0x0a, - 0xbc, 0x91, 0x52, 0x83, 0x8c, 0x5c, 0x87, 0x60, 0x74, 0x13, 0x2a, 0xc4, 0xd7, 0xfd, 0x31, 0x11, - 0x9a, 0x7c, 0x49, 0xaa, 0xc9, 0x2e, 0x1b, 0xa2, 0x89, 0xa1, 0x69, 0xb1, 0x05, 0x89, 0x58, 0xf4, - 0x55, 0x38, 0x6b, 0x3a, 0xf7, 0xb0, 0xed, 0x7a, 0x87, 0xbd, 0x11, 0xf6, 0x06, 0xd8, 0xf1, 0xf5, - 0x21, 0x0e, 0x74, 0x9c, 0x0f, 0xfa, 0x76, 0x26, 0x5d, 0xea, 0x6f, 0x15, 0x38, 0x47, 0x35, 0xdd, - 0xd1, 0x3d, 0xdf, 0x7c, 0x09, 0xfb, 0xa5, 0x42, 0x23, 0xaa, 0x63, 0xa7, 0xc8, 0xfa, 0x62, 0x34, - 0x3a, 0x66, 0x14, 0x88, 0xa7, 0x6b, 0x2b, 0x31, 0x75, 0x63, 0x34, 0xf5, 0x37, 0xc2, 0xb0, 0x51, - 0x3d, 0x4f, 0xb3, 0xa1, 0x49, 0x99, 0x85, 0xb4, 0xcc, 0x93, 0x6c, 0xe7, 0x3f, 0x14, 0x38, 0x77, - 0xd7, 0xd5, 0x8d, 0x89, 0xe1, 0xbf, 0xf8, 0xed, 0xfc, 0x36, 0x54, 0xf8, 0x29, 0xe9, 0x94, 0x98, - 0xac, 0xcb, 0x71, 0x59, 0xe2, 0x04, 0x4d, 0x34, 0xdc, 0x65, 0x04, 0x4d, 0x4c, 0x42, 0x97, 0xa1, - 0xe5, 0xe1, 0x91, 0x65, 0x0e, 0xf4, 0x9e, 0x33, 0xb6, 0xfb, 0xd8, 0xeb, 0x94, 0x97, 0x94, 0xe5, - 0xb2, 0xd6, 0x14, 0xd4, 0x6d, 0x46, 0x54, 0x7f, 0xa5, 0x40, 0x47, 0xc3, 0x16, 0xd6, 0x09, 0x7e, - 0x95, 0x8b, 0x5d, 0x80, 0x8a, 0xe3, 0x1a, 0x78, 0x6b, 0x83, 0x2d, 0xb6, 0xa8, 0x89, 0x96, 0xfa, - 0xd3, 0x02, 0x37, 0xc4, 0x6b, 0xee, 0xd7, 0x11, 0x63, 0x95, 0x3f, 0x1f, 0x63, 0x55, 0x64, 0xc6, - 0xfa, 0xf3, 0xc4, 0x58, 0xaf, 0xfb, 0x86, 0x4c, 0x0c, 0x5a, 0x8e, 0x19, 0xf4, 0x07, 0xb0, 0xb8, - 0xee, 0x61, 0xdd, 0xc7, 0x1f, 0xd1, 0xa4, 0xb1, 0x7e, 0xa0, 0x3b, 0x0e, 0xb6, 0x82, 0x25, 0x24, - 0x85, 0x2b, 0x12, 0xe1, 0x1d, 0x98, 0x1d, 0x79, 0xee, 0xf3, 0xc3, 0x50, 0xef, 0xa0, 0xa9, 0xfe, - 0x5e, 0x81, 0xae, 0x8c, 0xf7, 0x69, 0xe2, 0xcb, 0x45, 0x68, 0x8a, 0xec, 0xc7, 0xb9, 0x31, 0x99, - 0x35, 0xad, 0xf1, 0x24, 0x22, 0x01, 0xdd, 0x80, 0xb3, 0x7c, 0x90, 0x87, 0xc9, 0xd8, 0xf2, 0xc3, - 0xb1, 0x45, 0x36, 0x16, 0xb1, 0x3e, 0x8d, 0x75, 0x89, 0x19, 0xea, 0x1f, 0x14, 0x58, 0xdc, 0xc4, - 0x7e, 0x68, 0x44, 0x2a, 0x15, 0xbf, 0xa6, 0x21, 0xfb, 0x33, 0x05, 0xba, 0x32, 0x5d, 0x4f, 0xb3, - 0xad, 0x0f, 0x61, 0x21, 0x94, 0xd1, 0x33, 0x30, 0x19, 0x78, 0xe6, 0x88, 0x39, 0x33, 0x0b, 0xe0, - 0xf5, 0xd5, 0x8b, 0x2b, 0x69, 0x80, 0xb1, 0x92, 0xd4, 0xe0, 0x5c, 0xc8, 0x62, 0x23, 0xc2, 0x41, - 0xfd, 0xb9, 0x02, 0xe7, 0x36, 0xb1, 0xbf, 0x8b, 0x87, 0x36, 0x76, 0xfc, 0x2d, 0x67, 0xdf, 0x3d, - 0xf9, 0xbe, 0xbe, 0x0d, 0x40, 0x04, 0x9f, 0x30, 0xb9, 0x44, 0x28, 0x79, 0xf6, 0x98, 0x61, 0x99, - 0xa4, 0x3e, 0xa7, 0xd9, 0xbb, 0xaf, 0x43, 0xd9, 0x74, 0xf6, 0xdd, 0x60, 0xab, 0xce, 0xcb, 0xb6, - 0x2a, 0x2a, 0x8c, 0x8f, 0x56, 0x1d, 0xae, 0xc5, 0x81, 0xee, 0x19, 0x77, 0xb1, 0x6e, 0x60, 0xef, - 0x14, 0xee, 0x96, 0x5c, 0x76, 0x41, 0xb2, 0xec, 0x9f, 0x29, 0xf0, 0x46, 0x4a, 0xe0, 0x69, 0xd6, - 0xfd, 0x2d, 0xa8, 0x10, 0xca, 0x2c, 0x58, 0xf8, 0x25, 0xe9, 0xc2, 0x23, 0xe2, 0xee, 0x9a, 0xc4, - 0xd7, 0xc4, 0x1c, 0xd5, 0x85, 0x76, 0xb2, 0x0f, 0x5d, 0x80, 0x86, 0x38, 0xaa, 0x3d, 0x47, 0xb7, - 0xf9, 0x06, 0xd4, 0xb4, 0xba, 0xa0, 0x6d, 0xeb, 0x36, 0x46, 0x8b, 0x50, 0xa5, 0x81, 0xab, 0x67, - 0x1a, 0x81, 0xf9, 0x67, 0x59, 0x20, 0x33, 0x08, 0x7a, 0x0b, 0x80, 0x75, 0xe9, 0x86, 0xe1, 0x71, - 0x30, 0x51, 0xd3, 0x6a, 0x94, 0x72, 0x8b, 0x12, 0xd4, 0x7f, 0x17, 0x60, 0xe1, 0x96, 0x61, 0xc8, - 0xc2, 0xdc, 0xf1, 0x37, 0x7c, 0x12, 0x4d, 0x0b, 0xd1, 0x68, 0x9a, 0xeb, 0x8c, 0xa7, 0x42, 0x58, - 0xe9, 0x18, 0x21, 0xac, 0x9c, 0x15, 0xc2, 0xd0, 0x26, 0x34, 0x09, 0xc6, 0x8f, 0x7b, 0x23, 0x97, - 0xb0, 0x33, 0xc8, 0x32, 0x56, 0x7d, 0x55, 0x8d, 0xaf, 0x26, 0xc4, 0xfd, 0xf7, 0xc8, 0x70, 0x47, - 0x8c, 0xd4, 0x1a, 0x74, 0x62, 0xd0, 0x42, 0xf7, 0x61, 0x61, 0x68, 0xb9, 0x7d, 0xdd, 0xea, 0x11, - 0xac, 0x5b, 0xd8, 0xe8, 0x89, 0xf3, 0x45, 0x3a, 0xb3, 0xf9, 0x1c, 0xfc, 0x2c, 0x9f, 0xbe, 0xcb, - 0x66, 0x8b, 0x0e, 0xa2, 0xfe, 0x5d, 0x81, 0x45, 0x0d, 0xdb, 0xee, 0x53, 0xfc, 0xbf, 0x6a, 0x02, - 0xf5, 0x97, 0x0a, 0x34, 0x28, 0x38, 0xba, 0x87, 0x7d, 0x9d, 0xee, 0x04, 0x7a, 0x1f, 0x6a, 0x96, - 0xab, 0x1b, 0x3d, 0xff, 0x70, 0xc4, 0x97, 0xd6, 0x4a, 0x2e, 0x8d, 0xef, 0x1e, 0x9d, 0xb4, 0x77, - 0x38, 0xc2, 0x5a, 0xd5, 0x12, 0x5f, 0x79, 0x8e, 0x74, 0x2a, 0x5b, 0x14, 0x65, 0x00, 0xbf, 0x08, - 0x0b, 0xdf, 0xd3, 0xfd, 0xc1, 0xc1, 0x86, 0x2d, 0xd4, 0x24, 0xaf, 0x66, 0xcf, 0xf3, 0x80, 0x94, - 0x30, 0x94, 0x96, 0x65, 0x9e, 0x46, 0x6f, 0xa5, 0x2b, 0x0f, 0x84, 0x19, 0x22, 0xa1, 0x34, 0x02, - 0xf6, 0x2a, 0x27, 0x01, 0x7b, 0xeb, 0xd0, 0xc4, 0xcf, 0x07, 0xd6, 0x98, 0x86, 0x15, 0x26, 0x9d, - 0xfb, 0xf9, 0xdb, 0x12, 0xe9, 0x51, 0x37, 0x6f, 0x88, 0x49, 0x5b, 0x42, 0x07, 0x6e, 0x6a, 0x1b, - 0xfb, 0x7a, 0xa7, 0xca, 0xd4, 0x58, 0xca, 0x32, 0x75, 0xe0, 0x1f, 0xdc, 0xdc, 0xb4, 0xa5, 0xfe, - 0x47, 0x81, 0x45, 0x6e, 0x26, 0x6c, 0xf9, 0xfa, 0xab, 0xb5, 0x54, 0x68, 0x85, 0xd2, 0x31, 0xad, - 0x10, 0xd9, 0x81, 0xda, 0xb1, 0x77, 0xe0, 0xd3, 0x12, 0xcc, 0x89, 0xed, 0xa5, 0x23, 0xd8, 0xf9, - 0x79, 0x13, 0x6a, 0x61, 0x72, 0x17, 0xe0, 0x73, 0x42, 0x40, 0x4b, 0x50, 0x8f, 0x78, 0x8f, 0x58, - 0x68, 0x94, 0x94, 0x6b, 0xb5, 0x01, 0x54, 0x2b, 0x45, 0xa0, 0xda, 0x5b, 0x00, 0xfb, 0xd6, 0x98, - 0x1c, 0xf4, 0x7c, 0xd3, 0xc6, 0x02, 0x30, 0xd7, 0x18, 0x65, 0xcf, 0xb4, 0x31, 0xba, 0x05, 0x8d, - 0xbe, 0xe9, 0x58, 0xee, 0xb0, 0x37, 0xd2, 0xfd, 0x03, 0xd2, 0xa9, 0x64, 0xfa, 0xcb, 0x6d, 0x13, - 0x5b, 0xc6, 0x1a, 0x1b, 0xab, 0xd5, 0xf9, 0x9c, 0x1d, 0x3a, 0x05, 0xbd, 0x0d, 0x75, 0x67, 0x6c, - 0xf7, 0xdc, 0xfd, 0x9e, 0xe7, 0x3e, 0xa3, 0x1e, 0xc7, 0x44, 0x38, 0x63, 0xfb, 0xc3, 0x7d, 0xcd, - 0x7d, 0x46, 0x93, 0x6b, 0x8d, 0xa6, 0x59, 0x62, 0xb9, 0x43, 0xd2, 0xa9, 0xe6, 0xe2, 0x3f, 0x99, - 0x40, 0x67, 0x1b, 0xd4, 0x8f, 0xd8, 0xec, 0x5a, 0xbe, 0xd9, 0xe1, 0x04, 0x74, 0x05, 0x5a, 0x03, - 0xd7, 0x1e, 0xe9, 0x6c, 0x87, 0x6e, 0x7b, 0xae, 0xdd, 0x01, 0x76, 0x56, 0x13, 0x54, 0xb4, 0x0e, - 0x75, 0xd3, 0x31, 0xf0, 0x73, 0x71, 0x6a, 0xea, 0x4c, 0x8e, 0x2a, 0x33, 0x39, 0x13, 0xb4, 0x45, - 0xc7, 0x32, 0xa3, 0x83, 0x19, 0x7c, 0x12, 0x9a, 0xf3, 0x85, 0x45, 0x7b, 0xc4, 0x7c, 0x81, 0x3b, - 0x0d, 0x6e, 0x45, 0x41, 0xdb, 0x35, 0x5f, 0x60, 0xf5, 0x8f, 0x05, 0x68, 0xc5, 0x39, 0xd0, 0x4b, - 0xc7, 0x3e, 0xa3, 0x04, 0x6e, 0x11, 0x34, 0x29, 0x3f, 0xec, 0xe8, 0x7d, 0x8b, 0x9e, 0x65, 0x03, - 0x3f, 0x67, 0x5e, 0x51, 0xd5, 0xea, 0x9c, 0xc6, 0x18, 0x50, 0xeb, 0x72, 0xbd, 0x19, 0xc8, 0xe0, - 0x97, 0x82, 0x1a, 0xa3, 0x30, 0x88, 0xd1, 0x81, 0x59, 0xae, 0x5f, 0xe0, 0x13, 0x41, 0x93, 0xf6, - 0xf4, 0xc7, 0x26, 0x93, 0xca, 0x7d, 0x22, 0x68, 0xa2, 0x0d, 0x68, 0x70, 0x96, 0x23, 0xdd, 0xd3, - 0xed, 0xc0, 0x23, 0x2e, 0x48, 0x0f, 0xea, 0x1d, 0x7c, 0xf8, 0x40, 0xb7, 0xc6, 0x78, 0x47, 0x37, - 0x3d, 0x8d, 0xef, 0xe0, 0x0e, 0x9b, 0x85, 0x96, 0xa1, 0xcd, 0xb9, 0xec, 0x9b, 0x16, 0x16, 0xbe, - 0x35, 0xcb, 0x70, 0x4c, 0x8b, 0xd1, 0x6f, 0x9b, 0x16, 0xe6, 0xee, 0x13, 0x2e, 0x81, 0xed, 0x59, - 0x95, 0x7b, 0x0f, 0xa3, 0xb0, 0x1d, 0xfb, 0x67, 0x01, 0xe6, 0xe9, 0x21, 0x0a, 0x92, 0xef, 0xc9, - 0xe3, 0xc8, 0x5b, 0x00, 0x06, 0xf1, 0x7b, 0xb1, 0x58, 0x52, 0x33, 0x88, 0xbf, 0xcd, 0xc3, 0xc9, - 0xfb, 0x41, 0xa8, 0x28, 0x66, 0x5f, 0x13, 0x12, 0x87, 0x3a, 0x1d, 0xb4, 0x4f, 0xf4, 0x9c, 0x72, - 0x11, 0x9a, 0xc4, 0x1d, 0x7b, 0x03, 0xdc, 0x8b, 0x5d, 0x6b, 0x1b, 0x9c, 0xb8, 0x2d, 0x8f, 0x76, - 0x15, 0xe9, 0xb3, 0x4e, 0x24, 0x6c, 0xcd, 0x1e, 0x3b, 0x6c, 0xfd, 0x4d, 0x81, 0x05, 0xf1, 0x04, - 0x70, 0xfa, 0xdd, 0xce, 0x8a, 0xda, 0x41, 0x8c, 0x2a, 0x1e, 0x71, 0x9d, 0x2c, 0xe5, 0xc8, 0xb9, - 0x65, 0x49, 0xce, 0x8d, 0x5f, 0xa9, 0x2a, 0xc9, 0x2b, 0x95, 0xfa, 0xb1, 0x02, 0xcd, 0x5d, 0xac, - 0x7b, 0x83, 0x83, 0x60, 0x5d, 0xdf, 0x80, 0xa2, 0x87, 0x9f, 0x88, 0x65, 0x5d, 0xca, 0xc0, 0x97, - 0xb1, 0x29, 0x1a, 0x9d, 0x80, 0xce, 0x43, 0xdd, 0xb0, 0xad, 0xc4, 0xcd, 0x1d, 0x0c, 0xdb, 0x0a, - 0x10, 0x57, 0x5c, 0x95, 0x62, 0x4a, 0x95, 0x4f, 0x14, 0x68, 0x7c, 0xc4, 0x61, 0x17, 0xd7, 0xe4, - 0xbd, 0xa8, 0x26, 0x57, 0x32, 0x34, 0xd1, 0xb0, 0xef, 0x99, 0xf8, 0x29, 0xfe, 0x7c, 0x75, 0xf9, - 0x85, 0x02, 0x0b, 0x1f, 0xe8, 0x8e, 0xe1, 0xee, 0xef, 0x9f, 0xde, 0xee, 0xeb, 0x61, 0x10, 0xdc, - 0x3a, 0xce, 0x4d, 0x32, 0x36, 0x49, 0xfd, 0xb8, 0x00, 0x88, 0x3a, 0xe9, 0x9a, 0x6e, 0xe9, 0xce, - 0x00, 0x9f, 0x5c, 0x9b, 0xcb, 0xd0, 0x8a, 0x1d, 0xad, 0xf0, 0x55, 0x3c, 0x7a, 0xb6, 0x08, 0xba, - 0x03, 0xad, 0x3e, 0x17, 0xd5, 0xf3, 0xb0, 0x4e, 0x5c, 0x87, 0xb9, 0x67, 0x4b, 0x7e, 0x0f, 0xdc, - 0xf3, 0xcc, 0xe1, 0x10, 0x7b, 0xeb, 0xae, 0x63, 0xf0, 0x3b, 0x47, 0xb3, 0x1f, 0xa8, 0x49, 0xa7, - 0x32, 0x7b, 0x84, 0x71, 0x26, 0x00, 0x87, 0x10, 0x06, 0x1a, 0x82, 0xde, 0x81, 0x33, 0xf1, 0xeb, - 0xc8, 0xc4, 0x9f, 0xdb, 0x24, 0x7a, 0xd3, 0xa0, 0xc6, 0xf9, 0x11, 0xa0, 0x10, 0xee, 0x32, 0x54, - 0xc5, 0x92, 0x46, 0x9e, 0xd7, 0xac, 0x37, 0xa1, 0x66, 0x04, 0x33, 0x85, 0x57, 0x4c, 0x08, 0x34, - 0xe8, 0x70, 0x0d, 0x7b, 0xf4, 0xfc, 0x63, 0x23, 0x00, 0x14, 0x9c, 0x78, 0x97, 0xd1, 0xd4, 0xcf, - 0x0a, 0xd0, 0x8e, 0x5e, 0x71, 0x72, 0xcb, 0x7e, 0x39, 0x6f, 0x5b, 0x47, 0xdc, 0xe7, 0x4a, 0xa7, - 0xb8, 0xcf, 0xa5, 0xef, 0x9b, 0xe5, 0x93, 0xdd, 0x37, 0xd5, 0x5f, 0x2b, 0x30, 0x97, 0x78, 0x4a, - 0x4a, 0x42, 0x3b, 0x25, 0x0d, 0xed, 0xde, 0x83, 0x32, 0xc5, 0x3b, 0x98, 0x6d, 0x52, 0x4b, 0x0e, - 0x3b, 0xe2, 0x5c, 0x35, 0x3e, 0x01, 0x5d, 0x87, 0x79, 0x49, 0xf9, 0x41, 0x98, 0x12, 0xa5, 0xab, - 0x0f, 0xea, 0x9f, 0x4a, 0x50, 0x8f, 0xec, 0xc7, 0x14, 0x54, 0x9a, 0xe7, 0xe2, 0x96, 0x58, 0x5e, - 0x31, 0xbd, 0xbc, 0x8c, 0xf7, 0x77, 0xb4, 0x08, 0x55, 0x1b, 0xdb, 0x3c, 0xed, 0x0b, 0x0c, 0x62, - 0x63, 0x9b, 0x26, 0x7d, 0xf6, 0x34, 0x32, 0xb6, 0x39, 0x9e, 0xe4, 0x89, 0x6e, 0xd6, 0x19, 0xdb, - 0x0c, 0x4d, 0xc6, 0x11, 0xcf, 0xec, 0x11, 0x88, 0xa7, 0x1a, 0x47, 0x3c, 0xb1, 0xe3, 0x50, 0x4b, - 0x1e, 0x87, 0xbc, 0x40, 0xf1, 0x06, 0xcc, 0x0f, 0xd8, 0x3b, 0xb0, 0xb1, 0x76, 0xb8, 0x1e, 0x76, - 0x75, 0xea, 0x0c, 0x9a, 0xc9, 0xba, 0xd0, 0x6d, 0xea, 0x5c, 0x02, 0x15, 0x32, 0x2b, 0x37, 0x98, - 0x95, 0xe5, 0x80, 0x4a, 0xd8, 0x86, 0x1b, 0x39, 0x88, 0x89, 0xac, 0x95, 0x84, 0xa8, 0xcd, 0x13, - 0x41, 0xd4, 0xf3, 0x50, 0x0f, 0x8a, 0x01, 0xa6, 0x41, 0x3a, 0x2d, 0x1e, 0x9b, 0x04, 0x69, 0xcb, - 0x20, 0xb1, 0x47, 0xa9, 0xb9, 0xd8, 0xa3, 0x94, 0xfa, 0x97, 0x22, 0xb4, 0x26, 0x18, 0x26, 0x77, - 0x28, 0xc8, 0x53, 0x46, 0xdb, 0x86, 0xf6, 0xe4, 0xcd, 0x96, 0xed, 0xd2, 0x91, 0x30, 0x2c, 0xf9, - 0x5a, 0x3b, 0x37, 0x4a, 0x9c, 0xb9, 0xd8, 0x63, 0x45, 0xe9, 0x58, 0x8f, 0x15, 0xa7, 0xac, 0xb6, - 0xdc, 0x84, 0x73, 0x1e, 0x87, 0x50, 0x46, 0x2f, 0xb6, 0x6c, 0x8e, 0x46, 0xce, 0x06, 0x9d, 0x3b, - 0xd1, 0xe5, 0x67, 0x1c, 0xe3, 0xd9, 0xac, 0x63, 0x9c, 0x34, 0x63, 0x35, 0x65, 0xc6, 0x74, 0xd1, - 0xa7, 0x26, 0x2b, 0xfa, 0xdc, 0x87, 0xf9, 0xfb, 0x0e, 0x19, 0xf7, 0xc9, 0xc0, 0x33, 0xfb, 0x38, - 0xb8, 0xaa, 0xe7, 0x32, 0x6b, 0x17, 0xaa, 0x22, 0x5e, 0x73, 0x93, 0xd6, 0xb4, 0xb0, 0xad, 0xfe, - 0x44, 0x81, 0x85, 0x34, 0x5f, 0xe6, 0x31, 0x93, 0x60, 0xa0, 0xc4, 0x82, 0xc1, 0xf7, 0x61, 0x7e, - 0xc2, 0xbe, 0x17, 0xe3, 0x5c, 0x5f, 0xbd, 0x2a, 0xb3, 0x9d, 0x44, 0x71, 0x0d, 0x4d, 0x78, 0x04, - 0x34, 0xf5, 0x5f, 0x0a, 0x9c, 0x11, 0xc7, 0x8a, 0xd2, 0x86, 0xec, 0x91, 0x83, 0x26, 0x28, 0xd7, - 0xb1, 0x4c, 0x27, 0xc4, 0xdc, 0x62, 0x8d, 0x9c, 0x28, 0x30, 0xf7, 0x07, 0x30, 0x27, 0x06, 0x85, - 0x79, 0x26, 0x27, 0x9c, 0x69, 0xf1, 0x79, 0x61, 0x86, 0xb9, 0x0c, 0x2d, 0x77, 0x7f, 0x3f, 0x2a, - 0x8f, 0x07, 0xca, 0xa6, 0xa0, 0x0a, 0x81, 0xdf, 0x85, 0x76, 0x30, 0xec, 0xb8, 0x99, 0x6d, 0x4e, - 0x4c, 0x0c, 0x1f, 0x29, 0x3f, 0x51, 0xa0, 0x13, 0xcf, 0x73, 0x91, 0xe5, 0x1f, 0x1f, 0x49, 0x7d, - 0x33, 0x5e, 0x1a, 0xb8, 0x7c, 0x84, 0x3e, 0x13, 0x39, 0xe2, 0x82, 0x74, 0xed, 0x05, 0xb4, 0xe2, - 0x67, 0x16, 0x35, 0xa0, 0xba, 0xed, 0xfa, 0xdf, 0x79, 0x6e, 0x12, 0xbf, 0x3d, 0x83, 0x5a, 0x00, - 0xdb, 0xae, 0xbf, 0xe3, 0x61, 0x82, 0x1d, 0xbf, 0xad, 0x20, 0x80, 0xca, 0x87, 0xce, 0x86, 0x49, - 0x1e, 0xb7, 0x0b, 0x68, 0x5e, 0xa4, 0x54, 0xdd, 0xda, 0x12, 0x07, 0xa1, 0x5d, 0xa4, 0xd3, 0xc3, - 0x56, 0x09, 0xb5, 0xa1, 0x11, 0x0e, 0xd9, 0xdc, 0xb9, 0xdf, 0x2e, 0xa3, 0x1a, 0x94, 0xf9, 0x67, - 0xe5, 0x9a, 0x01, 0xed, 0x24, 0x62, 0xa3, 0x3c, 0xef, 0x3b, 0x77, 0x1c, 0xf7, 0x59, 0x48, 0x6a, - 0xcf, 0xa0, 0x3a, 0xcc, 0x0a, 0x14, 0xdc, 0x56, 0xd0, 0x1c, 0xd4, 0x23, 0x00, 0xb4, 0x5d, 0xa0, - 0x84, 0x4d, 0x6f, 0x34, 0x10, 0x50, 0x94, 0xab, 0x40, 0xad, 0xb6, 0xe1, 0x3e, 0x73, 0xda, 0xa5, - 0x6b, 0x6b, 0x50, 0x0d, 0x82, 0x09, 0x1d, 0xca, 0xb9, 0x3b, 0xb4, 0xd9, 0x9e, 0x41, 0x67, 0xa0, - 0x19, 0x2b, 0x34, 0xb7, 0x15, 0x84, 0xa0, 0x15, 0xff, 0x09, 0xa0, 0x5d, 0x58, 0xfd, 0xb4, 0x09, - 0xc0, 0xd1, 0x96, 0xeb, 0x7a, 0x06, 0x1a, 0x01, 0xda, 0xc4, 0x3e, 0xcd, 0x24, 0xae, 0x13, 0x64, - 0x01, 0x82, 0x6e, 0x64, 0x80, 0x92, 0xf4, 0x50, 0xa1, 0x6a, 0x37, 0xeb, 0x32, 0x91, 0x18, 0xae, - 0xce, 0x20, 0x9b, 0x49, 0xdc, 0x33, 0x6d, 0xbc, 0x67, 0x0e, 0x1e, 0x87, 0x30, 0x2d, 0x5b, 0x62, - 0x62, 0x68, 0x20, 0x31, 0x11, 0xb4, 0x45, 0x63, 0xd7, 0xf7, 0x4c, 0x67, 0x18, 0x14, 0x6a, 0xd4, - 0x19, 0xf4, 0x04, 0xce, 0x6e, 0x62, 0x26, 0xdd, 0x24, 0xbe, 0x39, 0x20, 0x81, 0xc0, 0xd5, 0x6c, - 0x81, 0xa9, 0xc1, 0xc7, 0x14, 0x69, 0xc1, 0x5c, 0xe2, 0xa7, 0x1b, 0x74, 0x4d, 0x5e, 0xeb, 0x91, - 0xfd, 0x20, 0xd4, 0x7d, 0x27, 0xd7, 0xd8, 0x50, 0x9a, 0x09, 0xad, 0xf8, 0x0f, 0x29, 0xe8, 0xcb, - 0x59, 0x0c, 0x52, 0x35, 0xf7, 0xee, 0xb5, 0x3c, 0x43, 0x43, 0x51, 0x0f, 0xb9, 0x3f, 0x4d, 0x13, - 0x25, 0xfd, 0xdf, 0xa1, 0x7b, 0x54, 0x8d, 0x4c, 0x9d, 0x41, 0x3f, 0x84, 0x33, 0xa9, 0x3f, 0x03, - 0xd0, 0x57, 0x64, 0xec, 0xb3, 0x7e, 0x20, 0x98, 0x26, 0xe1, 0x61, 0xf2, 0x34, 0x64, 0x6b, 0x9f, - 0xfa, 0x93, 0x24, 0xbf, 0xf6, 0x11, 0xf6, 0x47, 0x69, 0x7f, 0x6c, 0x09, 0x63, 0x40, 0xe9, 0x7f, - 0x03, 0xd0, 0xbb, 0x32, 0x11, 0x99, 0xff, 0x27, 0x74, 0x57, 0xf2, 0x0e, 0x0f, 0x4d, 0x3e, 0x66, - 0xa7, 0x35, 0x79, 0xdd, 0x90, 0x8a, 0xcd, 0xfc, 0x1f, 0x40, 0x2e, 0x36, 0xbb, 0x24, 0xcf, 0x9d, - 0x3a, 0x5e, 0x72, 0x96, 0xdb, 0x4a, 0x5a, 0x26, 0x97, 0x3b, 0xb5, 0xbc, 0x82, 0xad, 0xce, 0xa0, - 0xbd, 0x58, 0x10, 0x46, 0x57, 0xb2, 0x7c, 0x22, 0xfe, 0x4c, 0x30, 0xcd, 0x5c, 0x3d, 0x80, 0x4d, - 0xec, 0xdf, 0xc3, 0xbe, 0x67, 0x0e, 0x48, 0x92, 0xa9, 0x68, 0x4c, 0x06, 0x04, 0x4c, 0xaf, 0x4e, - 0x1d, 0x17, 0xaa, 0xdd, 0x87, 0xfa, 0x26, 0xf6, 0x35, 0x8e, 0xb4, 0x08, 0xca, 0x9c, 0x19, 0x8c, - 0x08, 0x44, 0x2c, 0x4f, 0x1f, 0x18, 0x0d, 0x64, 0x89, 0x0a, 0x38, 0xca, 0xdc, 0xdb, 0x74, 0x5d, - 0x5e, 0x1e, 0xc8, 0x32, 0x4a, 0xea, 0xea, 0xcc, 0xea, 0x5f, 0xeb, 0x50, 0x63, 0x5e, 0x48, 0x33, - 0xde, 0xff, 0x13, 0xd3, 0x4b, 0x48, 0x4c, 0x8f, 0x60, 0x2e, 0x51, 0xd1, 0x97, 0xdb, 0x53, 0x5e, - 0xf6, 0x9f, 0xe6, 0xf2, 0x7d, 0x40, 0xe9, 0x7a, 0xb5, 0x3c, 0x54, 0x64, 0xd6, 0xb5, 0xa7, 0xc9, - 0x78, 0x04, 0x73, 0x89, 0xe2, 0xac, 0x7c, 0x05, 0xf2, 0x0a, 0x6e, 0x8e, 0x15, 0xa4, 0x6b, 0x8a, - 0xf2, 0x15, 0x64, 0xd6, 0x1e, 0xa7, 0xc9, 0x78, 0xc0, 0x4b, 0xde, 0x21, 0x68, 0xbf, 0x9a, 0x15, - 0x6f, 0x12, 0xaf, 0xa4, 0xaf, 0x3e, 0x03, 0xbd, 0xfc, 0x0c, 0xfd, 0x08, 0xe6, 0x12, 0xa5, 0x01, - 0xb9, 0x75, 0xe5, 0xf5, 0x83, 0x69, 0xdc, 0xbf, 0xc0, 0x9c, 0xb2, 0x0b, 0x15, 0xfe, 0x9e, 0x8f, - 0x2e, 0xc8, 0xaf, 0x30, 0x91, 0xb7, 0xfe, 0xee, 0xb4, 0x8a, 0x00, 0x19, 0x5b, 0x3e, 0x61, 0x4c, - 0xcb, 0xec, 0xc4, 0x20, 0x69, 0xb9, 0x25, 0xfa, 0xce, 0xdf, 0x9d, 0xfe, 0xb4, 0x1f, 0x30, 0x7d, - 0xd9, 0x79, 0x6a, 0xed, 0x6b, 0x0f, 0x57, 0x87, 0xa6, 0x7f, 0x30, 0xee, 0x53, 0x7b, 0x5c, 0xe7, - 0x23, 0xdf, 0x35, 0x5d, 0xf1, 0x75, 0x3d, 0x50, 0xed, 0x3a, 0xe3, 0x74, 0x9d, 0xad, 0x65, 0xd4, - 0xef, 0x57, 0x58, 0xf3, 0xe6, 0x7f, 0x03, 0x00, 0x00, 0xff, 0xff, 0xe5, 0x45, 0x22, 0x53, 0xbb, - 0x2f, 0x00, 0x00, + // 2755 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xec, 0x5a, 0x5b, 0x6f, 0x1c, 0x49, + 0x15, 0x76, 0xcf, 0xcd, 0x33, 0x67, 0x2e, 0x9e, 0x94, 0x13, 0xef, 0x78, 0xc8, 0xc5, 0xe9, 0xdc, + 0x4c, 0x96, 0x75, 0x82, 0x03, 0x68, 0x57, 0xc0, 0x43, 0x6c, 0x13, 0xaf, 0x49, 0xe2, 0xf5, 0xb6, + 0x9d, 0x00, 0x51, 0xa4, 0xa1, 0x67, 0xba, 0x3c, 0x6e, 0xa5, 0x2f, 0x93, 0xae, 0x9e, 0x24, 0xce, + 0x33, 0x42, 0x2c, 0x17, 0x21, 0x9e, 0x10, 0x12, 0xe2, 0x09, 0x04, 0x48, 0xac, 0x78, 0xe6, 0x05, + 0x21, 0x7e, 0x01, 0x12, 0xef, 0x88, 0x37, 0x7e, 0x01, 0x8f, 0x08, 0x54, 0x97, 0xee, 0xe9, 0x4b, + 0xb5, 0x67, 0xec, 0xd9, 0x6c, 0x22, 0xb4, 0x6f, 0x5d, 0xa7, 0x4f, 0xd5, 0x39, 0xa7, 0xce, 0xa9, + 0x53, 0x5f, 0xd5, 0x29, 0x38, 0xf5, 0x74, 0x88, 0xbd, 0xc3, 0x4e, 0xcf, 0x75, 0x3d, 0x63, 0x65, + 0xe0, 0xb9, 0xbe, 0x8b, 0x90, 0x6d, 0x5a, 0xcf, 0x86, 0x84, 0xb7, 0x56, 0xd8, 0xff, 0x76, 0xad, + 0xe7, 0xda, 0xb6, 0xeb, 0x70, 0x5a, 0xbb, 0x16, 0xe5, 0x68, 0x37, 0x4c, 0xc7, 0xc7, 0x9e, 0xa3, + 0x5b, 0xc1, 0x5f, 0xd2, 0x3b, 0xc0, 0xb6, 0x2e, 0x5a, 0x4d, 0x43, 0xf7, 0xf5, 0xe8, 0xf8, 0xea, + 0xf7, 0x14, 0x58, 0xd8, 0x3d, 0x70, 0x9f, 0xaf, 0xbb, 0x96, 0x85, 0x7b, 0xbe, 0xe9, 0x3a, 0x44, + 0xc3, 0x4f, 0x87, 0x98, 0xf8, 0xe8, 0x26, 0x14, 0xba, 0x3a, 0xc1, 0x2d, 0x65, 0x49, 0x59, 0xae, + 0xae, 0x9e, 0x5d, 0x89, 0x69, 0x22, 0x54, 0xb8, 0x4f, 0xfa, 0x6b, 0x3a, 0xc1, 0x1a, 0xe3, 0x44, + 0x08, 0x0a, 0x46, 0x77, 0x6b, 0xa3, 0x95, 0x5b, 0x52, 0x96, 0xf3, 0x1a, 0xfb, 0x46, 0x97, 0xa1, + 0xde, 0x0b, 0xc7, 0xde, 0xda, 0x20, 0xad, 0xfc, 0x52, 0x7e, 0x39, 0xaf, 0xc5, 0x89, 0xea, 0x6f, + 0x15, 0x78, 0x2b, 0xa5, 0x06, 0x19, 0xb8, 0x0e, 0xc1, 0xe8, 0x16, 0x94, 0x88, 0xaf, 0xfb, 0x43, + 0x22, 0x34, 0xf9, 0x9c, 0x54, 0x93, 0x5d, 0xc6, 0xa2, 0x09, 0xd6, 0xb4, 0xd8, 0x9c, 0x44, 0x2c, + 0xfa, 0x22, 0x9c, 0x36, 0x9d, 0xfb, 0xd8, 0x76, 0xbd, 0xc3, 0xce, 0x00, 0x7b, 0x3d, 0xec, 0xf8, + 0x7a, 0x1f, 0x07, 0x3a, 0xce, 0x07, 0xff, 0x76, 0x46, 0xbf, 0xd4, 0xdf, 0x28, 0x70, 0x86, 0x6a, + 0xba, 0xa3, 0x7b, 0xbe, 0xf9, 0x0a, 0xe6, 0x4b, 0x85, 0x5a, 0x54, 0xc7, 0x56, 0x9e, 0xfd, 0x8b, + 0xd1, 0x28, 0xcf, 0x20, 0x10, 0x4f, 0x6d, 0x2b, 0x30, 0x75, 0x63, 0x34, 0xf5, 0xd7, 0xc2, 0xb1, + 0x51, 0x3d, 0xa7, 0x99, 0xd0, 0xa4, 0xcc, 0x5c, 0x5a, 0xe6, 0x49, 0xa6, 0xf3, 0x5f, 0x0a, 0x9c, + 0xb9, 0xe7, 0xea, 0xc6, 0xc8, 0xf1, 0x9f, 0xfe, 0x74, 0x7e, 0x1d, 0x4a, 0x7c, 0x95, 0xb4, 0x0a, + 0x4c, 0xd6, 0x95, 0xb8, 0x2c, 0xb1, 0x82, 0x46, 0x1a, 0xee, 0x32, 0x82, 0x26, 0x3a, 0xa1, 0x2b, + 0xd0, 0xf0, 0xf0, 0xc0, 0x32, 0x7b, 0x7a, 0xc7, 0x19, 0xda, 0x5d, 0xec, 0xb5, 0x8a, 0x4b, 0xca, + 0x72, 0x51, 0xab, 0x0b, 0xea, 0x36, 0x23, 0xaa, 0xbf, 0x54, 0xa0, 0xa5, 0x61, 0x0b, 0xeb, 0x04, + 0xbf, 0x4e, 0x63, 0x17, 0xa0, 0xe4, 0xb8, 0x06, 0xde, 0xda, 0x60, 0xc6, 0xe6, 0x35, 0xd1, 0x52, + 0x7f, 0x94, 0xe3, 0x8e, 0x78, 0xc3, 0xe3, 0x3a, 0xe2, 0xac, 0xe2, 0x27, 0xe3, 0xac, 0x92, 0xcc, + 0x59, 0x7f, 0x19, 0x39, 0xeb, 0x4d, 0x9f, 0x90, 0x91, 0x43, 0x8b, 0x31, 0x87, 0x7e, 0x07, 0x16, + 0xd7, 0x3d, 0xac, 0xfb, 0xf8, 0x43, 0xba, 0x69, 0xac, 0x1f, 0xe8, 0x8e, 0x83, 0xad, 0xc0, 0x84, + 0xa4, 0x70, 0x45, 0x22, 0xbc, 0x05, 0xb3, 0x03, 0xcf, 0x7d, 0x71, 0x18, 0xea, 0x1d, 0x34, 0xd5, + 0xdf, 0x29, 0xd0, 0x96, 0x8d, 0x3d, 0x4d, 0x7e, 0xb9, 0x04, 0x75, 0xb1, 0xfb, 0xf1, 0xd1, 0x98, + 0xcc, 0x8a, 0x56, 0x7b, 0x1a, 0x91, 0x80, 0x6e, 0xc2, 0x69, 0xce, 0xe4, 0x61, 0x32, 0xb4, 0xfc, + 0x90, 0x37, 0xcf, 0x78, 0x11, 0xfb, 0xa7, 0xb1, 0x5f, 0xa2, 0x87, 0xfa, 0x7b, 0x05, 0x16, 0x37, + 0xb1, 0x1f, 0x3a, 0x91, 0x4a, 0xc5, 0x6f, 0x68, 0xca, 0xfe, 0x58, 0x81, 0xb6, 0x4c, 0xd7, 0x69, + 0xa6, 0xf5, 0x11, 0x2c, 0x84, 0x32, 0x3a, 0x06, 0x26, 0x3d, 0xcf, 0x1c, 0xb0, 0x60, 0x66, 0x09, + 0xbc, 0xba, 0x7a, 0x69, 0x25, 0x0d, 0x30, 0x56, 0x92, 0x1a, 0x9c, 0x09, 0x87, 0xd8, 0x88, 0x8c, + 0xa0, 0xfe, 0x44, 0x81, 0x33, 0x9b, 0xd8, 0xdf, 0xc5, 0x7d, 0x1b, 0x3b, 0xfe, 0x96, 0xb3, 0xef, + 0x9e, 0x7c, 0x5e, 0xcf, 0x03, 0x10, 0x31, 0x4e, 0xb8, 0xb9, 0x44, 0x28, 0x93, 0xcc, 0x31, 0xc3, + 0x32, 0x49, 0x7d, 0xa6, 0x99, 0xbb, 0x2f, 0x43, 0xd1, 0x74, 0xf6, 0xdd, 0x60, 0xaa, 0x2e, 0xc8, + 0xa6, 0x2a, 0x2a, 0x8c, 0x73, 0xab, 0x0e, 0xd7, 0xe2, 0x40, 0xf7, 0x8c, 0x7b, 0x58, 0x37, 0xb0, + 0x37, 0x45, 0xb8, 0x25, 0xcd, 0xce, 0x49, 0xcc, 0xfe, 0xb1, 0x02, 0x6f, 0xa5, 0x04, 0x4e, 0x63, + 0xf7, 0xd7, 0xa0, 0x44, 0xe8, 0x60, 0x81, 0xe1, 0x97, 0xa5, 0x86, 0x47, 0xc4, 0xdd, 0x33, 0x89, + 0xaf, 0x89, 0x3e, 0xaa, 0x0b, 0xcd, 0xe4, 0x3f, 0x74, 0x11, 0x6a, 0x62, 0xa9, 0x76, 0x1c, 0xdd, + 0xe6, 0x13, 0x50, 0xd1, 0xaa, 0x82, 0xb6, 0xad, 0xdb, 0x18, 0x2d, 0x42, 0x99, 0x26, 0xae, 0x8e, + 0x69, 0x04, 0xee, 0x9f, 0x65, 0x89, 0xcc, 0x20, 0xe8, 0x1c, 0x00, 0xfb, 0xa5, 0x1b, 0x86, 0xc7, + 0xc1, 0x44, 0x45, 0xab, 0x50, 0xca, 0x6d, 0x4a, 0x50, 0xff, 0x93, 0x83, 0x85, 0xdb, 0x86, 0x21, + 0x4b, 0x73, 0xc7, 0x9f, 0xf0, 0x51, 0x36, 0xcd, 0x45, 0xb3, 0xe9, 0x44, 0x6b, 0x3c, 0x95, 0xc2, + 0x0a, 0xc7, 0x48, 0x61, 0xc5, 0xac, 0x14, 0x86, 0x36, 0xa1, 0x4e, 0x30, 0x7e, 0xd2, 0x19, 0xb8, + 0x84, 0xad, 0x41, 0xb6, 0x63, 0x55, 0x57, 0xd5, 0xb8, 0x35, 0x21, 0xee, 0xbf, 0x4f, 0xfa, 0x3b, + 0x82, 0x53, 0xab, 0xd1, 0x8e, 0x41, 0x0b, 0x3d, 0x80, 0x85, 0xbe, 0xe5, 0x76, 0x75, 0xab, 0x43, + 0xb0, 0x6e, 0x61, 0xa3, 0x23, 0xd6, 0x17, 0x69, 0xcd, 0x4e, 0x16, 0xe0, 0xa7, 0x79, 0xf7, 0x5d, + 0xd6, 0x5b, 0xfc, 0x20, 0xea, 0x3f, 0x15, 0x58, 0xd4, 0xb0, 0xed, 0x3e, 0xc3, 0xff, 0xaf, 0x2e, + 0x50, 0x7f, 0xa6, 0x40, 0x8d, 0x82, 0xa3, 0xfb, 0xd8, 0xd7, 0xe9, 0x4c, 0xa0, 0xf7, 0xa0, 0x62, + 0xb9, 0xba, 0xd1, 0xf1, 0x0f, 0x07, 0xdc, 0xb4, 0x46, 0xd2, 0x34, 0x3e, 0x7b, 0xb4, 0xd3, 0xde, + 0xe1, 0x00, 0x6b, 0x65, 0x4b, 0x7c, 0x4d, 0xb2, 0xa4, 0x53, 0xbb, 0x45, 0x5e, 0xb2, 0x5b, 0xfc, + 0x35, 0x0f, 0x0b, 0xdf, 0xd2, 0xfd, 0xde, 0xc1, 0x86, 0x2d, 0xd4, 0x24, 0xaf, 0x67, 0xce, 0x27, + 0x01, 0x29, 0x61, 0x2a, 0x2d, 0xca, 0x22, 0x8d, 0x9e, 0x4a, 0x57, 0x1e, 0x0a, 0x37, 0x44, 0x52, + 0x69, 0x04, 0xec, 0x95, 0x4e, 0x02, 0xf6, 0xd6, 0xa1, 0x8e, 0x5f, 0xf4, 0xac, 0x21, 0x4d, 0x2b, + 0x4c, 0x3a, 0x8f, 0xf3, 0xf3, 0x12, 0xe9, 0xd1, 0x30, 0xaf, 0x89, 0x4e, 0x5b, 0x42, 0x07, 0xee, + 0x6a, 0x1b, 0xfb, 0x7a, 0xab, 0xcc, 0xd4, 0x58, 0xca, 0x72, 0x75, 0x10, 0x1f, 0xdc, 0xdd, 0xb4, + 0x85, 0xce, 0x42, 0x45, 0x40, 0xcb, 0xad, 0x8d, 0x56, 0x85, 0x4d, 0xdf, 0x88, 0xa0, 0xfe, 0x57, + 0x81, 0x45, 0xee, 0x44, 0x6c, 0xf9, 0xfa, 0xeb, 0xf5, 0x63, 0xe8, 0xa3, 0xc2, 0x31, 0x7d, 0x14, + 0x99, 0x9f, 0xca, 0x71, 0xe7, 0x47, 0xfd, 0x73, 0x01, 0xe6, 0xc4, 0xe4, 0x53, 0x0e, 0xb6, 0xba, + 0xce, 0x42, 0x25, 0xdc, 0xfa, 0x05, 0x34, 0x1d, 0x11, 0xd0, 0x12, 0x54, 0x23, 0xb1, 0x25, 0x0c, + 0x8d, 0x92, 0x26, 0xb2, 0x36, 0x00, 0x72, 0x85, 0x08, 0x90, 0x3b, 0x07, 0xb0, 0x6f, 0x0d, 0xc9, + 0x41, 0xc7, 0x37, 0x6d, 0x2c, 0xe0, 0x74, 0x85, 0x51, 0xf6, 0x4c, 0x1b, 0xa3, 0xdb, 0x50, 0xeb, + 0x9a, 0x8e, 0xe5, 0xf6, 0x3b, 0x03, 0xdd, 0x3f, 0x20, 0xad, 0x52, 0x66, 0x34, 0xdd, 0x31, 0xb1, + 0x65, 0xac, 0x31, 0x5e, 0xad, 0xca, 0xfb, 0xec, 0xd0, 0x2e, 0xe8, 0x3c, 0x54, 0x9d, 0xa1, 0xdd, + 0x71, 0xf7, 0x3b, 0x9e, 0xfb, 0x9c, 0xc6, 0x23, 0x13, 0xe1, 0x0c, 0xed, 0x0f, 0xf6, 0x35, 0xf7, + 0x39, 0xdd, 0x7a, 0x2b, 0x74, 0x13, 0x26, 0x96, 0xdb, 0x27, 0xad, 0xf2, 0x44, 0xe3, 0x8f, 0x3a, + 0xd0, 0xde, 0x06, 0x8d, 0x23, 0xd6, 0xbb, 0x32, 0x59, 0xef, 0xb0, 0x03, 0xba, 0x0a, 0x8d, 0x9e, + 0x6b, 0x0f, 0x74, 0x36, 0x43, 0x77, 0x3c, 0xd7, 0x6e, 0x01, 0x5b, 0xc9, 0x09, 0x2a, 0x5a, 0x87, + 0xaa, 0xe9, 0x18, 0xf8, 0x85, 0x58, 0x53, 0x55, 0x26, 0x47, 0x95, 0xb9, 0x9c, 0x09, 0xda, 0xa2, + 0xbc, 0xcc, 0xe9, 0x60, 0x06, 0x9f, 0x84, 0x22, 0x02, 0xe1, 0xd1, 0x0e, 0x31, 0x5f, 0xe2, 0x56, + 0x8d, 0x7b, 0x51, 0xd0, 0x76, 0xcd, 0x97, 0x98, 0x1e, 0xd5, 0x4c, 0x87, 0x60, 0x6f, 0x94, 0xa0, + 0xeb, 0x2c, 0x41, 0xd7, 0x39, 0x35, 0xc8, 0xcd, 0x7f, 0xcc, 0x41, 0x23, 0x2e, 0x88, 0x9e, 0x5c, + 0xf6, 0x19, 0x25, 0x88, 0x9e, 0xa0, 0x49, 0xc5, 0x62, 0x47, 0xef, 0x5a, 0x34, 0x21, 0x18, 0xf8, + 0x05, 0x0b, 0x9e, 0xb2, 0x56, 0xe5, 0x34, 0x36, 0x00, 0x0d, 0x02, 0x6e, 0x1e, 0x43, 0x2a, 0xfc, + 0x64, 0x51, 0x61, 0x14, 0x86, 0x53, 0x5a, 0x30, 0xcb, 0xcd, 0x08, 0x42, 0x27, 0x68, 0xd2, 0x3f, + 0xdd, 0xa1, 0xc9, 0xa4, 0xf2, 0xd0, 0x09, 0x9a, 0x68, 0x03, 0x6a, 0x7c, 0xc8, 0x81, 0xee, 0xe9, + 0x76, 0x10, 0x38, 0x17, 0xa5, 0xeb, 0xf9, 0x2e, 0x3e, 0x7c, 0xa8, 0x5b, 0x43, 0xbc, 0xa3, 0x9b, + 0x9e, 0xc6, 0x27, 0x7a, 0x87, 0xf5, 0x42, 0xcb, 0xd0, 0xe4, 0xa3, 0xec, 0x9b, 0x16, 0x16, 0x21, + 0x38, 0xcb, 0xc0, 0x50, 0x83, 0xd1, 0xef, 0x98, 0x16, 0xe6, 0x51, 0x16, 0x9a, 0xc0, 0xa6, 0xb6, + 0xcc, 0x83, 0x8c, 0x51, 0xe8, 0xc4, 0xaa, 0xdf, 0xcf, 0xc3, 0x3c, 0x5d, 0x6b, 0xc1, 0x0e, 0x7e, + 0xf2, 0x74, 0x73, 0x0e, 0xc0, 0x20, 0x7e, 0x27, 0x96, 0x72, 0x2a, 0x06, 0xf1, 0xb7, 0x79, 0xd6, + 0x79, 0x2f, 0xc8, 0x28, 0xf9, 0xec, 0xb3, 0x46, 0x62, 0xed, 0xa7, 0x33, 0xff, 0x89, 0xee, 0x64, + 0x2e, 0x41, 0x9d, 0xb8, 0x43, 0xaf, 0x87, 0x3b, 0xb1, 0xb3, 0x71, 0x8d, 0x13, 0xb7, 0xe5, 0x49, + 0xb1, 0x24, 0xbd, 0x1b, 0x8a, 0x64, 0xb7, 0xd9, 0xe9, 0xb2, 0x7f, 0x39, 0x99, 0xfd, 0xff, 0xa1, + 0xc0, 0x82, 0xb8, 0x65, 0x98, 0xde, 0x17, 0x59, 0xa9, 0x3f, 0x48, 0x74, 0xf9, 0x23, 0x4e, 0xac, + 0x85, 0x09, 0xb6, 0xf5, 0xa2, 0x64, 0x5b, 0x8f, 0x9f, 0xda, 0x4a, 0xc9, 0x53, 0x9b, 0xfa, 0x03, + 0x05, 0xea, 0xbb, 0x58, 0xf7, 0x7a, 0x07, 0x81, 0x5d, 0x5f, 0x81, 0xbc, 0x87, 0x9f, 0x0a, 0xb3, + 0x2e, 0x67, 0x40, 0xd8, 0x58, 0x17, 0x8d, 0x76, 0x40, 0x17, 0xa0, 0x6a, 0xd8, 0x56, 0xe2, 0x72, + 0x00, 0x0c, 0xdb, 0x0a, 0x40, 0x5d, 0x5c, 0x95, 0x7c, 0x4a, 0x95, 0x8f, 0x14, 0xa8, 0x7d, 0xc8, + 0x91, 0x1d, 0xd7, 0xe4, 0xdd, 0xa8, 0x26, 0x57, 0x33, 0x34, 0xd1, 0xb0, 0xef, 0x99, 0xf8, 0x19, + 0xfe, 0x64, 0x75, 0xf9, 0xa9, 0x02, 0x0b, 0xef, 0xeb, 0x8e, 0xe1, 0xee, 0xef, 0x4f, 0xef, 0xf7, + 0xf5, 0x30, 0x93, 0x6e, 0x1d, 0xe7, 0xb0, 0x1a, 0xeb, 0xa4, 0xfe, 0x21, 0x07, 0x88, 0x86, 0xf0, + 0x9a, 0x6e, 0xe9, 0x4e, 0x0f, 0x9f, 0x5c, 0x9b, 0x2b, 0xd0, 0x88, 0x2d, 0xbc, 0xf0, 0xe2, 0x3d, + 0xba, 0xf2, 0x08, 0xba, 0x0b, 0x8d, 0x2e, 0x17, 0xd5, 0xf1, 0xb0, 0x4e, 0x5c, 0x87, 0x85, 0x67, + 0x43, 0x7e, 0xd4, 0xdc, 0xf3, 0xcc, 0x7e, 0x1f, 0x7b, 0xeb, 0xae, 0x63, 0xf0, 0x63, 0x4d, 0xbd, + 0x1b, 0xa8, 0x49, 0xbb, 0x32, 0x7f, 0x84, 0x59, 0x28, 0xc0, 0x9f, 0x10, 0xa6, 0x21, 0x82, 0xde, + 0x86, 0x53, 0xf1, 0x13, 0xcf, 0x28, 0x9e, 0x9b, 0x24, 0x7a, 0x98, 0x91, 0xdd, 0x34, 0x48, 0xb2, + 0x82, 0xfa, 0x0b, 0x05, 0x50, 0x08, 0xbb, 0x19, 0x7e, 0x63, 0xfb, 0xce, 0x24, 0xb7, 0x6a, 0x67, + 0xa1, 0x62, 0x04, 0x3d, 0x45, 0xe8, 0x8c, 0x08, 0x34, 0x6f, 0x71, 0x33, 0x3a, 0x34, 0x85, 0x60, + 0x23, 0x80, 0x2e, 0x9c, 0x78, 0x8f, 0xd1, 0xe2, 0x49, 0xa5, 0x90, 0x4c, 0x2a, 0x1f, 0xe7, 0xa0, + 0x19, 0x3d, 0x88, 0x4d, 0xac, 0xd9, 0xab, 0xb9, 0x81, 0x3b, 0xe2, 0xd4, 0x59, 0x98, 0xe2, 0xd4, + 0x99, 0x3e, 0x15, 0x17, 0x4f, 0x76, 0x2a, 0x56, 0x7f, 0xa5, 0xc0, 0x5c, 0xe2, 0xc2, 0x2b, 0x09, + 0x31, 0x95, 0x34, 0xc4, 0x7c, 0x17, 0x8a, 0x14, 0x77, 0x61, 0x36, 0x49, 0x0d, 0x39, 0xfc, 0x89, + 0x8f, 0xaa, 0xf1, 0x0e, 0xe8, 0x06, 0xcc, 0x4b, 0x8a, 0x24, 0xc2, 0xd1, 0x28, 0x5d, 0x23, 0x51, + 0xff, 0x54, 0x80, 0x6a, 0x64, 0x3e, 0xc6, 0xa0, 0xe3, 0x49, 0x8e, 0x97, 0x09, 0xf3, 0xf2, 0x69, + 0xf3, 0x32, 0xaa, 0x04, 0x68, 0x11, 0xca, 0x36, 0xb6, 0x39, 0xae, 0x10, 0x20, 0xc7, 0xc6, 0x36, + 0x83, 0x6b, 0x8b, 0x50, 0xa6, 0xd0, 0x96, 0xe1, 0x5a, 0xbe, 0x66, 0x66, 0x9d, 0xa1, 0xcd, 0x50, + 0x6d, 0x1c, 0x52, 0xcd, 0x1e, 0x01, 0xa9, 0xca, 0x71, 0x48, 0x15, 0x5b, 0x2c, 0x95, 0xe4, 0x62, + 0x99, 0x14, 0xb0, 0xde, 0x84, 0xf9, 0x1e, 0xbb, 0xad, 0x36, 0xd6, 0x0e, 0xd7, 0xc3, 0x5f, 0xad, + 0x2a, 0xc3, 0x7e, 0xb2, 0x5f, 0xe8, 0x0e, 0x0d, 0x2e, 0x81, 0x4e, 0x99, 0x97, 0x6b, 0xcc, 0xcb, + 0x72, 0xc4, 0x26, 0x7c, 0xc3, 0x9d, 0x1c, 0xa4, 0x55, 0xd6, 0x4a, 0x42, 0xe5, 0xfa, 0x89, 0xa0, + 0xf2, 0x05, 0xa8, 0x06, 0x25, 0x0b, 0xd3, 0x20, 0xad, 0x06, 0x4f, 0x6f, 0xc1, 0x82, 0x37, 0x48, + 0xec, 0xea, 0x6c, 0x2e, 0x76, 0x75, 0xa6, 0xfe, 0x2d, 0x0f, 0x8d, 0x11, 0x48, 0x9a, 0x38, 0x15, + 0x4c, 0x52, 0xec, 0xdb, 0x86, 0xe6, 0xe8, 0x66, 0x99, 0xcd, 0xd2, 0x91, 0x38, 0x2f, 0x79, 0xa7, + 0x3c, 0x37, 0x48, 0xac, 0xb9, 0xd8, 0x95, 0x4a, 0xe1, 0x58, 0x57, 0x2a, 0x53, 0xd6, 0x84, 0x6e, + 0xc1, 0x19, 0x8f, 0xa3, 0x30, 0xa3, 0x13, 0x33, 0x9b, 0x03, 0x9a, 0xd3, 0xc1, 0xcf, 0x9d, 0xa8, + 0xf9, 0x19, 0xcb, 0x78, 0x36, 0x6b, 0x19, 0x27, 0xdd, 0x58, 0x4e, 0xb9, 0x31, 0x5d, 0x9a, 0xaa, + 0xc8, 0x4a, 0x53, 0x0f, 0x60, 0xfe, 0x81, 0x43, 0x86, 0x5d, 0xd2, 0xf3, 0xcc, 0x2e, 0x0e, 0xae, + 0x0c, 0x26, 0x72, 0x6b, 0x1b, 0xca, 0x22, 0x5f, 0x73, 0x97, 0x56, 0xb4, 0xb0, 0xad, 0xfe, 0x50, + 0x81, 0x85, 0xf4, 0xb8, 0x2c, 0x62, 0x46, 0xc9, 0x40, 0x89, 0x25, 0x83, 0x6f, 0xc3, 0xfc, 0x68, + 0xf8, 0x4e, 0x6c, 0xe4, 0xea, 0xea, 0x35, 0x99, 0xef, 0x24, 0x8a, 0x6b, 0x68, 0x34, 0x46, 0x40, + 0x53, 0xff, 0xad, 0xc0, 0x29, 0xb1, 0xac, 0x28, 0xad, 0xcf, 0xae, 0x62, 0xe8, 0x06, 0xe5, 0x3a, + 0x96, 0xe9, 0x84, 0xa0, 0x5e, 0xd8, 0xc8, 0x89, 0x02, 0xd4, 0xbf, 0x0f, 0x73, 0x82, 0x29, 0xdc, + 0x67, 0x26, 0x44, 0x44, 0x0d, 0xde, 0x2f, 0xdc, 0x61, 0xae, 0x40, 0xc3, 0xdd, 0xdf, 0x8f, 0xca, + 0xe3, 0x89, 0xb2, 0x2e, 0xa8, 0x42, 0xe0, 0x37, 0xa1, 0x19, 0xb0, 0x1d, 0x77, 0x67, 0x9b, 0x13, + 0x1d, 0xc3, 0xab, 0xd4, 0x8f, 0x14, 0x68, 0xc5, 0xf7, 0xb9, 0x88, 0xf9, 0xc7, 0x07, 0x63, 0x5f, + 0x8d, 0x17, 0x30, 0xae, 0x1c, 0xa1, 0xcf, 0x48, 0x8e, 0x38, 0x81, 0x5d, 0x7f, 0x09, 0x8d, 0xf8, + 0x9a, 0x45, 0x35, 0x28, 0x6f, 0xbb, 0xfe, 0x37, 0x5e, 0x98, 0xc4, 0x6f, 0xce, 0xa0, 0x06, 0xc0, + 0xb6, 0xeb, 0xef, 0x78, 0x98, 0x60, 0xc7, 0x6f, 0x2a, 0x08, 0xa0, 0xf4, 0x81, 0xb3, 0x61, 0x92, + 0x27, 0xcd, 0x1c, 0x9a, 0x17, 0x5b, 0xaa, 0x6e, 0x6d, 0x89, 0x85, 0xd0, 0xcc, 0xd3, 0xee, 0x61, + 0xab, 0x80, 0x9a, 0x50, 0x0b, 0x59, 0x36, 0x77, 0x1e, 0x34, 0x8b, 0xa8, 0x02, 0x45, 0xfe, 0x59, + 0xba, 0x6e, 0x40, 0x33, 0x09, 0xfa, 0xe8, 0x98, 0x0f, 0x9c, 0xbb, 0x8e, 0xfb, 0x3c, 0x24, 0x35, + 0x67, 0x50, 0x15, 0x66, 0x05, 0x90, 0x6e, 0x2a, 0x68, 0x0e, 0xaa, 0x11, 0x0c, 0xdb, 0xcc, 0x51, + 0xc2, 0xa6, 0x37, 0xe8, 0x09, 0x34, 0xcb, 0x55, 0xa0, 0x5e, 0xdb, 0x70, 0x9f, 0x3b, 0xcd, 0xc2, + 0xf5, 0x35, 0x28, 0x07, 0xc9, 0x84, 0xb2, 0xf2, 0xd1, 0x1d, 0xda, 0x6c, 0xce, 0xa0, 0x53, 0x50, + 0x8f, 0x95, 0xc3, 0x9b, 0x0a, 0x42, 0xd0, 0x88, 0x3f, 0x55, 0x68, 0xe6, 0x56, 0x7f, 0x5e, 0x07, + 0xe0, 0x68, 0xcb, 0x75, 0x3d, 0x03, 0x0d, 0x00, 0x6d, 0x62, 0x9f, 0xee, 0x24, 0xae, 0x13, 0xec, + 0x02, 0x04, 0xdd, 0xcc, 0x00, 0x25, 0x69, 0x56, 0xa1, 0x6a, 0x3b, 0xeb, 0x3c, 0x92, 0x60, 0x57, + 0x67, 0x90, 0xcd, 0x24, 0xee, 0x99, 0x36, 0xde, 0x33, 0x7b, 0x4f, 0x42, 0x98, 0x96, 0x2d, 0x31, + 0xc1, 0x1a, 0x48, 0x4c, 0x24, 0x6d, 0xd1, 0xd8, 0xf5, 0x3d, 0xd3, 0xe9, 0x07, 0xe5, 0x24, 0x75, + 0x06, 0x3d, 0x85, 0xd3, 0x9b, 0x98, 0x49, 0x37, 0x89, 0x6f, 0xf6, 0x48, 0x20, 0x70, 0x35, 0x5b, + 0x60, 0x8a, 0xf9, 0x98, 0x22, 0x2d, 0x98, 0x4b, 0x3c, 0x0d, 0x42, 0xd7, 0xe5, 0x15, 0x29, 0xd9, + 0x33, 0xa6, 0xf6, 0xdb, 0x13, 0xf1, 0x86, 0xd2, 0x4c, 0x68, 0xc4, 0x9f, 0xcd, 0xa0, 0xcf, 0x67, + 0x0d, 0x90, 0x7a, 0x19, 0xd0, 0xbe, 0x3e, 0x09, 0x6b, 0x28, 0xea, 0x11, 0x8f, 0xa7, 0x71, 0xa2, + 0xa4, 0xaf, 0x32, 0xda, 0x47, 0x55, 0xf2, 0xd4, 0x19, 0xf4, 0x5d, 0x38, 0x95, 0x7a, 0xbf, 0x80, + 0xbe, 0x20, 0x1b, 0x3e, 0xeb, 0x99, 0xc3, 0x38, 0x09, 0x8f, 0x92, 0xab, 0x21, 0x5b, 0xfb, 0xd4, + 0x7b, 0x97, 0xc9, 0xb5, 0x8f, 0x0c, 0x7f, 0x94, 0xf6, 0xc7, 0x96, 0x30, 0x04, 0x94, 0x7e, 0xc1, + 0x80, 0xde, 0x91, 0x89, 0xc8, 0x7c, 0x45, 0xd1, 0x5e, 0x99, 0x94, 0x3d, 0x74, 0xf9, 0x90, 0xad, + 0xd6, 0xe4, 0x71, 0x43, 0x2a, 0x36, 0xf3, 0xd5, 0x82, 0x5c, 0x6c, 0xf6, 0xc3, 0x01, 0x1e, 0xd4, + 0xf1, 0xc2, 0xb8, 0xdc, 0x57, 0xd2, 0x62, 0xbe, 0x3c, 0xa8, 0xe5, 0x75, 0x76, 0x75, 0x06, 0xed, + 0xc5, 0x92, 0x30, 0xba, 0x9a, 0x15, 0x13, 0xf1, 0x9b, 0x86, 0x71, 0xee, 0xea, 0x00, 0x6c, 0x62, + 0xff, 0x3e, 0xf6, 0x3d, 0xb3, 0x47, 0x92, 0x83, 0x8a, 0xc6, 0x88, 0x21, 0x18, 0xf4, 0xda, 0x58, + 0xbe, 0x50, 0xed, 0x2e, 0x54, 0x37, 0xb1, 0xaf, 0x71, 0xa4, 0x45, 0x50, 0x66, 0xcf, 0x80, 0x23, + 0x10, 0xb1, 0x3c, 0x9e, 0x31, 0x9a, 0xc8, 0x12, 0x75, 0x7a, 0x94, 0x39, 0xb7, 0xe9, 0xd7, 0x03, + 0xf2, 0x44, 0x96, 0x51, 0xf8, 0x57, 0x67, 0x56, 0xff, 0x5e, 0x85, 0x0a, 0x8b, 0x42, 0xba, 0xe3, + 0x7d, 0xb6, 0x31, 0xbd, 0x82, 0x8d, 0xe9, 0x31, 0xcc, 0x25, 0xde, 0x1d, 0xc8, 0xfd, 0x29, 0x7f, + 0x9c, 0x30, 0x2e, 0xe4, 0xbb, 0x80, 0xd2, 0x55, 0x75, 0x79, 0xaa, 0xc8, 0xac, 0xbe, 0x8f, 0x93, + 0xf1, 0x18, 0xe6, 0x12, 0x25, 0x64, 0xb9, 0x05, 0xf2, 0x3a, 0xf3, 0x04, 0x16, 0xa4, 0x6b, 0x9b, + 0x72, 0x0b, 0x32, 0x6b, 0xa0, 0xe3, 0x64, 0x3c, 0xe4, 0x85, 0xf9, 0x10, 0xb4, 0x5f, 0xcb, 0xca, + 0x37, 0x89, 0x8b, 0xd6, 0xd7, 0xbf, 0x03, 0xbd, 0xfa, 0x1d, 0xfa, 0x31, 0xcc, 0x25, 0xaa, 0x0b, + 0x72, 0xef, 0xca, 0x4b, 0x10, 0xe3, 0x46, 0xff, 0x14, 0xf7, 0x94, 0x5d, 0x28, 0xf1, 0x92, 0x00, + 0xba, 0x28, 0x3f, 0xc2, 0x44, 0xca, 0x05, 0xed, 0x71, 0x45, 0x05, 0x32, 0xb4, 0x7c, 0xc2, 0x06, + 0x2d, 0xb2, 0x15, 0x83, 0xa4, 0xf5, 0x9c, 0x68, 0xa9, 0xa0, 0x3d, 0xbe, 0x3a, 0x10, 0x0c, 0xfa, + 0xaa, 0xf7, 0xa9, 0xb5, 0x2f, 0x3d, 0x5a, 0xed, 0x9b, 0xfe, 0xc1, 0xb0, 0x4b, 0xfd, 0x71, 0x83, + 0x73, 0xbe, 0x63, 0xba, 0xe2, 0xeb, 0x46, 0xa0, 0xda, 0x0d, 0x36, 0xd2, 0x0d, 0x66, 0xcb, 0xa0, + 0xdb, 0x2d, 0xb1, 0xe6, 0xad, 0xff, 0x05, 0x00, 0x00, 0xff, 0xff, 0xaa, 0xf8, 0xae, 0x2f, 0x61, + 0x30, 0x00, 0x00, } // Reference imports to suppress errors if they are not otherwise used. diff --git a/internal/proxy/impl.go b/internal/proxy/impl.go index f5a1ffe579..7e4a74f0d5 100644 --- a/internal/proxy/impl.go +++ b/internal/proxy/impl.go @@ -31,6 +31,7 @@ import ( "github.com/milvus-io/milvus/internal/log" "github.com/milvus-io/milvus/internal/metrics" "github.com/milvus-io/milvus/internal/mq/msgstream" + "github.com/milvus-io/milvus/internal/proto/commonpb" "github.com/milvus-io/milvus/internal/proto/datapb" "github.com/milvus-io/milvus/internal/proto/internalpb" @@ -144,8 +145,8 @@ func (node *Proxy) ReleaseDQLMessageStream(ctx context.Context, request *proxypb }, nil } -// TODO(dragondriver): add more detailed ut for ConsistencyLevel, should we support multiple consistency level in Proxy? // CreateCollection create a collection by the schema. +// TODO(dragondriver): add more detailed ut for ConsistencyLevel, should we support multiple consistency level in Proxy? func (node *Proxy) CreateCollection(ctx context.Context, request *milvuspb.CreateCollectionRequest) (*commonpb.Status, error) { if !node.checkHealthy() { return unhealthyStatus(), nil @@ -2399,11 +2400,10 @@ func (node *Proxy) Search(ctx context.Context, request *milvuspb.SearchRequest) }, ResultChannelID: strconv.FormatInt(Params.ProxyCfg.ProxyID, 10), }, - resultBuf: make(chan []*internalpb.SearchResults, 1), - query: request, - chMgr: node.chMgr, - qc: node.queryCoord, - tr: timerecord.NewTimeRecorder("search"), + request: request, + qc: node.queryCoord, + tr: timerecord.NewTimeRecorder("search"), + getQueryNodePolicy: defaultGetQueryNodePolicy, } travelTs := request.TravelTimestamp @@ -2516,11 +2516,11 @@ func (node *Proxy) Search(ctx context.Context, request *milvuspb.SearchRequest) metrics.ProxySearchCount.WithLabelValues(strconv.FormatInt(Params.ProxyCfg.ProxyID, 10), metrics.SearchLabel, metrics.SuccessLabel).Inc() metrics.ProxySearchVectors.WithLabelValues(strconv.FormatInt(Params.ProxyCfg.ProxyID, 10), - metrics.SearchLabel).Set(float64(qt.result.Results.NumQueries)) + metrics.SearchLabel).Set(float64(qt.result.GetResults().GetNumQueries())) searchDur := tr.ElapseSpan().Milliseconds() metrics.ProxySearchLatency.WithLabelValues(strconv.FormatInt(Params.ProxyCfg.ProxyID, 10), metrics.SearchLabel).Observe(float64(searchDur)) - metrics.ProxySearchLatencyPerNQ.WithLabelValues(strconv.FormatInt(Params.ProxyCfg.ProxyID, 10)).Observe(float64(searchDur) / float64(qt.result.Results.NumQueries)) + metrics.ProxySearchLatencyPerNQ.WithLabelValues(strconv.FormatInt(Params.ProxyCfg.ProxyID, 10)).Observe(float64(searchDur) / float64(qt.result.GetResults().GetNumQueries())) return qt.result, nil } @@ -2641,10 +2641,10 @@ func (node *Proxy) Query(ctx context.Context, request *milvuspb.QueryRequest) (* }, ResultChannelID: strconv.FormatInt(Params.ProxyCfg.ProxyID, 10), }, - resultBuf: make(chan []*internalpb.RetrieveResults), - query: request, - chMgr: node.chMgr, - qc: node.queryCoord, + request: request, + qc: node.queryCoord, + getQueryNodePolicy: defaultGetQueryNodePolicy, + queryShardPolicy: roundRobinPolicy, } method := "Query" @@ -3058,11 +3058,12 @@ func (node *Proxy) CalcDistance(ctx context.Context, request *milvuspb.CalcDista }, ResultChannelID: strconv.FormatInt(Params.ProxyCfg.ProxyID, 10), }, - resultBuf: make(chan []*internalpb.RetrieveResults), - query: queryRequest, - chMgr: node.chMgr, - qc: node.queryCoord, - ids: ids.IdArray, + request: queryRequest, + qc: node.queryCoord, + ids: ids.IdArray, + + getQueryNodePolicy: defaultGetQueryNodePolicy, + queryShardPolicy: roundRobinPolicy, } err := node.sched.dqQueue.Enqueue(qt) @@ -3715,6 +3716,7 @@ func (node *Proxy) RegisterLink(ctx context.Context, req *milvuspb.RegisterLinkR }, nil } +// GetMetrics gets the metrics of proxy // TODO(dragondriver): cache the Metrics and set a retention to the cache func (node *Proxy) GetMetrics(ctx context.Context, req *milvuspb.GetMetricsRequest) (*milvuspb.GetMetricsResponse, error) { log.Debug("Proxy.GetMetrics", @@ -3817,6 +3819,13 @@ func (node *Proxy) LoadBalance(ctx context.Context, req *milvuspb.LoadBalanceReq status := &commonpb.Status{ ErrorCode: commonpb.ErrorCode_UnexpectedError, } + + collectionID, err := globalMetaCache.GetCollectionID(ctx, req.GetCollectionName()) + if err != nil { + log.Error("failed to get collection id", zap.String("collection name", req.GetCollectionName()), zap.Error(err)) + status.Reason = err.Error() + return status, nil + } infoResp, err := node.queryCoord.LoadBalance(ctx, &querypb.LoadBalanceRequest{ Base: &commonpb.MsgBase{ MsgType: commonpb.MsgType_LoadBalanceSegments, @@ -3828,6 +3837,7 @@ func (node *Proxy) LoadBalance(ctx context.Context, req *milvuspb.LoadBalanceReq DstNodeIDs: req.DstNodeIDs, BalanceReason: querypb.TriggerCondition_GrpcRequest, SealedSegmentIDs: req.SealedSegmentIDs, + CollectionID: collectionID, }) if err != nil { log.Error("Failed to LoadBalance from Query Coordinator", @@ -3873,6 +3883,7 @@ func (node *Proxy) ManualCompaction(ctx context.Context, req *milvuspb.ManualCom return resp, err } +// GetCompactionStateWithPlans returns the compactions states with the given plan ID func (node *Proxy) GetCompactionStateWithPlans(ctx context.Context, req *milvuspb.GetCompactionPlansRequest) (*milvuspb.GetCompactionPlansResponse, error) { log.Info("received GetCompactionStateWithPlans request", zap.Int64("compactionID", req.GetCompactionID())) resp := &milvuspb.GetCompactionPlansResponse{} @@ -3979,7 +3990,7 @@ func (node *Proxy) GetReplicas(ctx context.Context, req *milvuspb.GetReplicasReq return resp, err } -// Check import task state from datanode +// GetImportState checks import task state from datanode func (node *Proxy) GetImportState(ctx context.Context, req *milvuspb.GetImportStateRequest) (*milvuspb.GetImportStateResponse, error) { log.Info("received get import state request", zap.Int64("taskID", req.GetTask())) resp := &milvuspb.GetImportStateResponse{} @@ -4206,3 +4217,19 @@ func (node *Proxy) ListCredUsers(ctx context.Context, req *milvuspb.ListCredUser Usernames: usernames, }, nil } + +// SendSearchResult needs to be removed TODO +func (node *Proxy) SendSearchResult(ctx context.Context, req *internalpb.SearchResults) (*commonpb.Status, error) { + return &commonpb.Status{ + ErrorCode: commonpb.ErrorCode_UnexpectedError, + Reason: "Not implemented", + }, nil +} + +// SendRetrieveResult needs to be removed TODO +func (node *Proxy) SendRetrieveResult(ctx context.Context, req *internalpb.RetrieveResults) (*commonpb.Status, error) { + return &commonpb.Status{ + ErrorCode: commonpb.ErrorCode_UnexpectedError, + Reason: "Not implemented", + }, nil +} diff --git a/internal/proxy/meta_cache.go b/internal/proxy/meta_cache.go index 8faec4a37c..1b0c47af20 100644 --- a/internal/proxy/meta_cache.go +++ b/internal/proxy/meta_cache.go @@ -31,6 +31,7 @@ import ( "github.com/milvus-io/milvus/internal/proto/commonpb" "github.com/milvus-io/milvus/internal/proto/internalpb" "github.com/milvus-io/milvus/internal/proto/milvuspb" + "github.com/milvus-io/milvus/internal/proto/querypb" "github.com/milvus-io/milvus/internal/proto/rootcoordpb" "github.com/milvus-io/milvus/internal/proto/schemapb" "github.com/milvus-io/milvus/internal/types" @@ -52,6 +53,7 @@ type Cache interface { GetPartitionInfo(ctx context.Context, collectionName string, partitionName string) (*partitionInfo, error) // GetCollectionSchema get collection's schema. GetCollectionSchema(ctx context.Context, collectionName string) (*schemapb.CollectionSchema, error) + GetShards(ctx context.Context, withCache bool, collectionName string, qc types.QueryCoord) ([]*querypb.ShardLeadersList, error) RemoveCollection(ctx context.Context, collectionName string) RemovePartition(ctx context.Context, collectionName string, partitionName string) @@ -67,6 +69,7 @@ type collectionInfo struct { collID typeutil.UniqueID schema *schemapb.CollectionSchema partInfo map[string]*partitionInfo + shardLeaders []*querypb.ShardLeadersList createdTimestamp uint64 createdUtcTimestamp uint64 } @@ -160,6 +163,7 @@ func (m *MetaCache) GetCollectionInfo(ctx context.Context, collectionName string collInfo = m.collInfo[collectionName] metrics.ProxyUpdateCacheLatency.WithLabelValues(strconv.FormatInt(Params.ProxyCfg.ProxyID, 10)).Observe(float64(tr.ElapseSpan().Milliseconds())) } + metrics.ProxyCacheHitCounter.WithLabelValues(strconv.FormatInt(Params.ProxyCfg.ProxyID, 10), "GetCollectionInfo", metrics.CacheHitLabel).Inc() return &collectionInfo{ collID: collInfo.collID, @@ -167,6 +171,7 @@ func (m *MetaCache) GetCollectionInfo(ctx context.Context, collectionName string partInfo: collInfo.partInfo, createdTimestamp: collInfo.createdTimestamp, createdUtcTimestamp: collInfo.createdUtcTimestamp, + shardLeaders: collInfo.shardLeaders, }, nil } @@ -520,3 +525,41 @@ func (m *MetaCache) GetCredUsernames(ctx context.Context) ([]string, error) { return usernames, nil } + +// GetShards update cache if withCache == false +func (m *MetaCache) GetShards(ctx context.Context, withCache bool, collectionName string, qc types.QueryCoord) ([]*querypb.ShardLeadersList, error) { + info, err := m.GetCollectionInfo(ctx, collectionName) + if err != nil { + return nil, err + } + + if withCache { + if len(info.shardLeaders) > 0 { + return info.shardLeaders, nil + } + log.Info("no shard cache for collection, try to get shard leaders from QueryCoord", + zap.String("collectionName", collectionName)) + } + + m.mu.Lock() + defer m.mu.Unlock() + req := &querypb.GetShardLeadersRequest{ + Base: &commonpb.MsgBase{ + MsgType: commonpb.MsgType_GetShardLeaders, + SourceID: Params.ProxyCfg.ProxyID, + }, + CollectionID: info.collID, + } + resp, err := qc.GetShardLeaders(ctx, req) + if err != nil { + return nil, err + } + if resp.Status.ErrorCode != commonpb.ErrorCode_Success { + return nil, fmt.Errorf("fail to get shard leaders from QueryCoord: %s", resp.Status.Reason) + } + + shards := resp.GetShards() + + m.collInfo[collectionName].shardLeaders = shards + return shards, nil +} diff --git a/internal/proxy/meta_cache_test.go b/internal/proxy/meta_cache_test.go index fc8c5604b0..718aad0c68 100644 --- a/internal/proxy/meta_cache_test.go +++ b/internal/proxy/meta_cache_test.go @@ -22,17 +22,17 @@ import ( "fmt" "testing" - "github.com/milvus-io/milvus/internal/util/crypto" - - "github.com/milvus-io/milvus/internal/proto/rootcoordpb" - "github.com/milvus-io/milvus/internal/log" "github.com/milvus-io/milvus/internal/proto/commonpb" "github.com/milvus-io/milvus/internal/proto/milvuspb" + "github.com/milvus-io/milvus/internal/proto/rootcoordpb" "github.com/milvus-io/milvus/internal/proto/schemapb" "github.com/milvus-io/milvus/internal/types" + "github.com/milvus-io/milvus/internal/util/crypto" "github.com/milvus-io/milvus/internal/util/typeutil" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) type MockRootCoordClientInterface struct { @@ -310,3 +310,51 @@ func TestMetaCache_GetPartitionError(t *testing.T) { log.Debug(err.Error()) assert.Equal(t, id, typeutil.UniqueID(0)) } + +func TestMetaCache_GetShards(t *testing.T) { + client := &MockRootCoordClientInterface{} + err := InitMetaCache(client) + require.Nil(t, err) + + var ( + ctx = context.TODO() + collectionName = "collection1" + qc = NewQueryCoordMock() + ) + qc.Init() + qc.Start() + defer qc.Stop() + + t.Run("No collection in meta cache", func(t *testing.T) { + shards, err := globalMetaCache.GetShards(ctx, true, "non-exists", qc) + assert.Error(t, err) + assert.Empty(t, shards) + }) + + t.Run("without shardLeaders in collection info invalid shardLeaders", func(t *testing.T) { + qc.validShardLeaders = false + shards, err := globalMetaCache.GetShards(ctx, false, collectionName, qc) + assert.Error(t, err) + assert.Empty(t, shards) + }) + + t.Run("without shardLeaders in collection info", func(t *testing.T) { + qc.validShardLeaders = true + shards, err := globalMetaCache.GetShards(ctx, true, collectionName, qc) + assert.NoError(t, err) + assert.NotEmpty(t, shards) + assert.Equal(t, 1, len(shards)) + assert.Equal(t, 3, len(shards[0].GetNodeAddrs())) + assert.Equal(t, 3, len(shards[0].GetNodeIds())) + + // get from cache + qc.validShardLeaders = false + shards, err = globalMetaCache.GetShards(ctx, true, collectionName, qc) + assert.NoError(t, err) + assert.NotEmpty(t, shards) + assert.Equal(t, 1, len(shards)) + assert.Equal(t, 3, len(shards[0].GetNodeAddrs())) + assert.Equal(t, 3, len(shards[0].GetNodeIds())) + }) + +} diff --git a/internal/proxy/proxy.go b/internal/proxy/proxy.go index d96d58c0ca..12b13012a3 100644 --- a/internal/proxy/proxy.go +++ b/internal/proxy/proxy.go @@ -93,8 +93,7 @@ type Proxy struct { factory dependency.Factory - searchResultCh chan *internalpb.SearchResults - retrieveResultCh chan *internalpb.RetrieveResults + searchResultCh chan *internalpb.SearchResults // Add callback functions at different stages startCallbacks []func() @@ -107,11 +106,10 @@ func NewProxy(ctx context.Context, factory dependency.Factory) (*Proxy, error) { ctx1, cancel := context.WithCancel(ctx) n := 1024 // better to be configurable node := &Proxy{ - ctx: ctx1, - cancel: cancel, - factory: factory, - searchResultCh: make(chan *internalpb.SearchResults, n), - retrieveResultCh: make(chan *internalpb.RetrieveResults, n), + ctx: ctx1, + cancel: cancel, + factory: factory, + searchResultCh: make(chan *internalpb.SearchResults, n), } node.UpdateStateCode(internalpb.StateCode_Abnormal) logutil.Logger(ctx).Debug("create a new Proxy instance", zap.Any("state", node.stateCode.Load())) @@ -228,9 +226,7 @@ func (node *Proxy) Init() error { log.Debug("create channels manager done", zap.String("role", typeutil.ProxyRole)) log.Debug("create task scheduler", zap.String("role", typeutil.ProxyRole)) - node.sched, err = newTaskScheduler(node.ctx, node.idAllocator, node.tsoAllocator, node.factory, - schedOptWithSearchResultCh(node.searchResultCh), - schedOptWithRetrieveResultCh(node.retrieveResultCh)) + node.sched, err = newTaskScheduler(node.ctx, node.idAllocator, node.tsoAllocator, node.factory) if err != nil { log.Warn("failed to create task scheduler", zap.Error(err), zap.String("role", typeutil.ProxyRole)) return err diff --git a/internal/proxy/proxy_test.go b/internal/proxy/proxy_test.go index 17842a96ee..5ec6058f5c 100644 --- a/internal/proxy/proxy_test.go +++ b/internal/proxy/proxy_test.go @@ -17,12 +17,7 @@ package proxy import ( - "bytes" "context" - "encoding/binary" - "encoding/json" - "fmt" - "math/rand" "net" "os" "strconv" @@ -30,81 +25,56 @@ import ( "testing" "time" + "github.com/golang/protobuf/proto" ot "github.com/grpc-ecosystem/go-grpc-middleware/tracing/opentracing" + "github.com/prometheus/client_golang/prometheus" + "github.com/stretchr/testify/assert" + "go.uber.org/zap" "google.golang.org/grpc" "google.golang.org/grpc/keepalive" + "github.com/milvus-io/milvus/internal/common" + "github.com/milvus-io/milvus/internal/log" + "github.com/milvus-io/milvus/internal/metrics" + "github.com/milvus-io/milvus/internal/rootcoord" + "github.com/milvus-io/milvus/internal/util/crypto" "github.com/milvus-io/milvus/internal/util/dependency" - "github.com/milvus-io/milvus/internal/util/trace" - - "github.com/prometheus/client_golang/prometheus" - - "github.com/milvus-io/milvus/internal/util/paramtable" - - "github.com/milvus-io/milvus/internal/util/etcd" - "github.com/milvus-io/milvus/internal/util/sessionutil" - "github.com/milvus-io/milvus/internal/util/tsoutil" - - "go.uber.org/zap" - - "github.com/milvus-io/milvus/internal/common" - - "github.com/milvus-io/milvus/internal/util/metricsinfo" - "github.com/milvus-io/milvus/internal/util/distance" - - "github.com/milvus-io/milvus/internal/proto/proxypb" - "github.com/milvus-io/milvus/internal/proto/rootcoordpb" - - "github.com/golang/protobuf/proto" - - "github.com/milvus-io/milvus/internal/proto/schemapb" - - "github.com/milvus-io/milvus/internal/proto/milvuspb" - - "github.com/milvus-io/milvus/internal/proto/commonpb" - - "github.com/milvus-io/milvus/internal/proto/internalpb" - - grpcindexcoordclient "github.com/milvus-io/milvus/internal/distributed/indexcoord/client" - - grpcquerycoordclient "github.com/milvus-io/milvus/internal/distributed/querycoord/client" - - grpcdatacoordclient2 "github.com/milvus-io/milvus/internal/distributed/datacoord/client" - + "github.com/milvus-io/milvus/internal/util/etcd" "github.com/milvus-io/milvus/internal/util/funcutil" + "github.com/milvus-io/milvus/internal/util/logutil" + "github.com/milvus-io/milvus/internal/util/metricsinfo" + "github.com/milvus-io/milvus/internal/util/paramtable" + "github.com/milvus-io/milvus/internal/util/sessionutil" + "github.com/milvus-io/milvus/internal/util/trace" "github.com/milvus-io/milvus/internal/util/typeutil" - rcc "github.com/milvus-io/milvus/internal/distributed/rootcoord/client" - - grpcindexnode "github.com/milvus-io/milvus/internal/distributed/indexnode" - - grpcindexcoord "github.com/milvus-io/milvus/internal/distributed/indexcoord" + "github.com/milvus-io/milvus/internal/proto/commonpb" + "github.com/milvus-io/milvus/internal/proto/internalpb" + "github.com/milvus-io/milvus/internal/proto/milvuspb" + "github.com/milvus-io/milvus/internal/proto/proxypb" + "github.com/milvus-io/milvus/internal/proto/rootcoordpb" + "github.com/milvus-io/milvus/internal/proto/schemapb" grpcdatacoordclient "github.com/milvus-io/milvus/internal/distributed/datacoord" + grpcdatacoordclient2 "github.com/milvus-io/milvus/internal/distributed/datacoord/client" grpcdatanode "github.com/milvus-io/milvus/internal/distributed/datanode" - - grpcquerynode "github.com/milvus-io/milvus/internal/distributed/querynode" - + grpcindexcoord "github.com/milvus-io/milvus/internal/distributed/indexcoord" + grpcindexcoordclient "github.com/milvus-io/milvus/internal/distributed/indexcoord/client" + grpcindexnode "github.com/milvus-io/milvus/internal/distributed/indexnode" grpcquerycoord "github.com/milvus-io/milvus/internal/distributed/querycoord" - + grpcquerycoordclient "github.com/milvus-io/milvus/internal/distributed/querycoord/client" + grpcquerynode "github.com/milvus-io/milvus/internal/distributed/querynode" grpcrootcoord "github.com/milvus-io/milvus/internal/distributed/rootcoord" + rcc "github.com/milvus-io/milvus/internal/distributed/rootcoord/client" "github.com/milvus-io/milvus/internal/datacoord" "github.com/milvus-io/milvus/internal/datanode" "github.com/milvus-io/milvus/internal/indexcoord" "github.com/milvus-io/milvus/internal/indexnode" - "github.com/milvus-io/milvus/internal/querynode" - - "github.com/stretchr/testify/assert" - "github.com/milvus-io/milvus/internal/querycoord" - - "github.com/milvus-io/milvus/internal/log" - "github.com/milvus-io/milvus/internal/metrics" - "github.com/milvus-io/milvus/internal/rootcoord" - "github.com/milvus-io/milvus/internal/util/logutil" + "github.com/milvus-io/milvus/internal/querynode" ) const ( @@ -620,12 +590,12 @@ func TestProxy(t *testing.T) { rowNum := 3000 indexName := "_default" nlist := 10 - nprobe := 10 - topk := 10 + // nprobe := 10 + // topk := 10 // add a test parameter - roundDecimal := 6 + // roundDecimal := 6 nq := 10 - expr := fmt.Sprintf("%s > 0", int64Field) + // expr := fmt.Sprintf("%s > 0", int64Field) var segmentIDs []int64 // an int64 field (pk) & a float vector field @@ -721,76 +691,6 @@ func TestProxy(t *testing.T) { } } - constructPlaceholderGroup := func() *milvuspb.PlaceholderGroup { - values := make([][]byte, 0, nq) - for i := 0; i < nq; i++ { - bs := make([]byte, 0, dim*4) - for j := 0; j < dim; j++ { - var buffer bytes.Buffer - f := rand.Float32() - err := binary.Write(&buffer, common.Endian, f) - assert.NoError(t, err) - bs = append(bs, buffer.Bytes()...) - } - values = append(values, bs) - } - - return &milvuspb.PlaceholderGroup{ - Placeholders: []*milvuspb.PlaceholderValue{ - { - Tag: "$0", - Type: milvuspb.PlaceholderType_FloatVector, - Values: values, - }, - }, - } - } - - constructSearchRequest := func() *milvuspb.SearchRequest { - params := make(map[string]string) - params["nprobe"] = strconv.Itoa(nprobe) - b, err := json.Marshal(params) - assert.NoError(t, err) - plg := constructPlaceholderGroup() - plgBs, err := proto.Marshal(plg) - assert.NoError(t, err) - - return &milvuspb.SearchRequest{ - Base: nil, - DbName: dbName, - CollectionName: collectionName, - PartitionNames: nil, - Dsl: expr, - PlaceholderGroup: plgBs, - DslType: commonpb.DslType_BoolExprV1, - OutputFields: nil, - SearchParams: []*commonpb.KeyValuePair{ - { - Key: MetricTypeKey, - Value: distance.L2, - }, - { - Key: SearchParamsKey, - Value: string(b), - }, - { - Key: AnnsFieldKey, - Value: floatVecField, - }, - { - Key: TopKKey, - Value: strconv.Itoa(topk), - }, - { - Key: RoundDecimalKey, - Value: strconv.Itoa(roundDecimal), - }, - }, - TravelTimestamp: 0, - GuaranteeTimestamp: 0, - } - } - wg.Add(1) t.Run("create collection", func(t *testing.T) { defer wg.Done() @@ -1368,103 +1268,178 @@ func TestProxy(t *testing.T) { assert.NotEqual(t, commonpb.ErrorCode_Success, resp.Status.ErrorCode) }) - if loaded { - wg.Add(1) - t.Run("search", func(t *testing.T) { - defer wg.Done() - req := constructSearchRequest() + // nprobe := 10 + // topk := 10 + // roundDecimal := 6 + // expr := fmt.Sprintf("%s > 0", int64Field) + // constructPlaceholderGroup := func() *milvuspb.PlaceholderGroup { + // values := make([][]byte, 0, nq) + // for i := 0; i < nq; i++ { + // bs := make([]byte, 0, dim*4) + // for j := 0; j < dim; j++ { + // var buffer bytes.Buffer + // f := rand.Float32() + // err := binary.Write(&buffer, common.Endian, f) + // assert.NoError(t, err) + // bs = append(bs, buffer.Bytes()...) + // } + // values = append(values, bs) + // } + // + // return &milvuspb.PlaceholderGroup{ + // Placeholders: []*milvuspb.PlaceholderValue{ + // { + // Tag: "$0", + // Type: milvuspb.PlaceholderType_FloatVector, + // Values: values, + // }, + // }, + // } + // } + // + // constructSearchRequest := func() *milvuspb.SearchRequest { + // params := make(map[string]string) + // params["nprobe"] = strconv.Itoa(nprobe) + // b, err := json.Marshal(params) + // assert.NoError(t, err) + // plg := constructPlaceholderGroup() + // plgBs, err := proto.Marshal(plg) + // assert.NoError(t, err) + // + // return &milvuspb.SearchRequest{ + // Base: nil, + // DbName: dbName, + // CollectionName: collectionName, + // PartitionNames: nil, + // Dsl: expr, + // PlaceholderGroup: plgBs, + // DslType: commonpb.DslType_BoolExprV1, + // OutputFields: nil, + // SearchParams: []*commonpb.KeyValuePair{ + // { + // Key: MetricTypeKey, + // Value: distance.L2, + // }, + // { + // Key: SearchParamsKey, + // Value: string(b), + // }, + // { + // Key: AnnsFieldKey, + // Value: floatVecField, + // }, + // { + // Key: TopKKey, + // Value: strconv.Itoa(topk), + // }, + // { + // Key: RoundDecimalKey, + // Value: strconv.Itoa(roundDecimal), + // }, + // }, + // TravelTimestamp: 0, + // GuaranteeTimestamp: 0, + // } + // } - resp, err := proxy.Search(ctx, req) - assert.NoError(t, err) - assert.Equal(t, commonpb.ErrorCode_Success, resp.Status.ErrorCode) - }) - - wg.Add(1) - t.Run("search_travel", func(t *testing.T) { - defer wg.Done() - past := time.Now().Add(time.Duration(-1*Params.CommonCfg.RetentionDuration-100) * time.Second) - travelTs := tsoutil.ComposeTSByTime(past, 0) - req := constructSearchRequest() - req.TravelTimestamp = travelTs - //resp, err := proxy.Search(ctx, req) - res, err := proxy.Search(ctx, req) - assert.NoError(t, err) - assert.NotEqual(t, commonpb.ErrorCode_Success, res.Status.ErrorCode) - }) - - wg.Add(1) - t.Run("search_travel_succ", func(t *testing.T) { - defer wg.Done() - past := time.Now().Add(time.Duration(-1*Params.CommonCfg.RetentionDuration+100) * time.Second) - travelTs := tsoutil.ComposeTSByTime(past, 0) - req := constructSearchRequest() - req.TravelTimestamp = travelTs - //resp, err := proxy.Search(ctx, req) - res, err := proxy.Search(ctx, req) - assert.NoError(t, err) - assert.Equal(t, commonpb.ErrorCode_Success, res.Status.ErrorCode) - }) - - wg.Add(1) - t.Run("query", func(t *testing.T) { - defer wg.Done() - //resp, err := proxy.Query(ctx, &milvuspb.QueryRequest{ - _, err := proxy.Query(ctx, &milvuspb.QueryRequest{ - Base: nil, - DbName: dbName, - CollectionName: collectionName, - Expr: expr, - OutputFields: nil, - PartitionNames: nil, - TravelTimestamp: 0, - GuaranteeTimestamp: 0, - }) - assert.NoError(t, err) - // FIXME(dragondriver) - // assert.Equal(t, commonpb.ErrorCode_Success, resp.Status.ErrorCode) - // TODO(dragondriver): compare query result - }) - - wg.Add(1) - t.Run("query_travel", func(t *testing.T) { - defer wg.Done() - past := time.Now().Add(time.Duration(-1*Params.CommonCfg.RetentionDuration-100) * time.Second) - travelTs := tsoutil.ComposeTSByTime(past, 0) - queryReq := &milvuspb.QueryRequest{ - Base: nil, - DbName: dbName, - CollectionName: collectionName, - Expr: expr, - OutputFields: nil, - PartitionNames: nil, - TravelTimestamp: travelTs, - GuaranteeTimestamp: 0, - } - res, err := proxy.Query(ctx, queryReq) - assert.NoError(t, err) - assert.NotEqual(t, commonpb.ErrorCode_Success, res.Status.ErrorCode) - }) - - wg.Add(1) - t.Run("query_travel_succ", func(t *testing.T) { - defer wg.Done() - past := time.Now().Add(time.Duration(-1*Params.CommonCfg.RetentionDuration+100) * time.Second) - travelTs := tsoutil.ComposeTSByTime(past, 0) - queryReq := &milvuspb.QueryRequest{ - Base: nil, - DbName: dbName, - CollectionName: collectionName, - Expr: expr, - OutputFields: nil, - PartitionNames: nil, - TravelTimestamp: travelTs, - GuaranteeTimestamp: 0, - } - res, err := proxy.Query(ctx, queryReq) - assert.NoError(t, err) - assert.Equal(t, commonpb.ErrorCode_EmptyCollection, res.Status.ErrorCode) - }) - } + // TODO(Goose): reopen after joint-tests + // if loaded { + // wg.Add(1) + // t.Run("search", func(t *testing.T) { + // defer wg.Done() + // req := constructSearchRequest() + // + // resp, err := proxy.Search(ctx, req) + // assert.NoError(t, err) + // assert.Equal(t, commonpb.ErrorCode_Success, resp.Status.ErrorCode) + // }) + // + // wg.Add(1) + // t.Run("search_travel", func(t *testing.T) { + // defer wg.Done() + // past := time.Now().Add(time.Duration(-1*Params.CommonCfg.RetentionDuration-100) * time.Second) + // travelTs := tsoutil.ComposeTSByTime(past, 0) + // req := constructSearchRequest() + // req.TravelTimestamp = travelTs + // //resp, err := proxy.Search(ctx, req) + // res, err := proxy.Search(ctx, req) + // assert.NoError(t, err) + // assert.NotEqual(t, commonpb.ErrorCode_Success, res.Status.ErrorCode) + // }) + // + // wg.Add(1) + // t.Run("search_travel_succ", func(t *testing.T) { + // defer wg.Done() + // past := time.Now().Add(time.Duration(-1*Params.CommonCfg.RetentionDuration+100) * time.Second) + // travelTs := tsoutil.ComposeTSByTime(past, 0) + // req := constructSearchRequest() + // req.TravelTimestamp = travelTs + // //resp, err := proxy.Search(ctx, req) + // res, err := proxy.Search(ctx, req) + // assert.NoError(t, err) + // assert.Equal(t, commonpb.ErrorCode_Success, res.Status.ErrorCode) + // }) + // + // wg.Add(1) + // t.Run("query", func(t *testing.T) { + // defer wg.Done() + // //resp, err := proxy.Query(ctx, &milvuspb.QueryRequest{ + // _, err := proxy.Query(ctx, &milvuspb.QueryRequest{ + // Base: nil, + // DbName: dbName, + // CollectionName: collectionName, + // Expr: expr, + // OutputFields: nil, + // PartitionNames: nil, + // TravelTimestamp: 0, + // GuaranteeTimestamp: 0, + // }) + // assert.NoError(t, err) + // // FIXME(dragondriver) + // // assert.Equal(t, commonpb.ErrorCode_Success, resp.Status.ErrorCode) + // // TODO(dragondriver): compare query result + // }) + // + // wg.Add(1) + // t.Run("query_travel", func(t *testing.T) { + // defer wg.Done() + // past := time.Now().Add(time.Duration(-1*Params.CommonCfg.RetentionDuration-100) * time.Second) + // travelTs := tsoutil.ComposeTSByTime(past, 0) + // queryReq := &milvuspb.QueryRequest{ + // Base: nil, + // DbName: dbName, + // CollectionName: collectionName, + // Expr: expr, + // OutputFields: nil, + // PartitionNames: nil, + // TravelTimestamp: travelTs, + // GuaranteeTimestamp: 0, + // } + // res, err := proxy.Query(ctx, queryReq) + // assert.NoError(t, err) + // assert.NotEqual(t, commonpb.ErrorCode_Success, res.Status.ErrorCode) + // }) + // + // wg.Add(1) + // t.Run("query_travel_succ", func(t *testing.T) { + // defer wg.Done() + // past := time.Now().Add(time.Duration(-1*Params.CommonCfg.RetentionDuration+100) * time.Second) + // travelTs := tsoutil.ComposeTSByTime(past, 0) + // queryReq := &milvuspb.QueryRequest{ + // Base: nil, + // DbName: dbName, + // CollectionName: collectionName, + // Expr: expr, + // OutputFields: nil, + // PartitionNames: nil, + // TravelTimestamp: travelTs, + // GuaranteeTimestamp: 0, + // } + // res, err := proxy.Query(ctx, queryReq) + // assert.NoError(t, err) + // assert.Equal(t, commonpb.ErrorCode_EmptyCollection, res.Status.ErrorCode) + // }) + // } wg.Add(1) t.Run("calculate distance", func(t *testing.T) { @@ -1683,6 +1658,7 @@ func TestProxy(t *testing.T) { DbName: dbName, CollectionName: collectionName, PartitionNames: []string{partitionName}, + ReplicaNumber: 1, }) assert.NoError(t, err) assert.Equal(t, commonpb.ErrorCode_Success, resp.ErrorCode) @@ -1693,6 +1669,7 @@ func TestProxy(t *testing.T) { DbName: dbName, CollectionName: collectionName, PartitionNames: []string{otherPartitionName}, + ReplicaNumber: 1, }) assert.NoError(t, err) assert.NotEqual(t, commonpb.ErrorCode_Success, resp.ErrorCode) @@ -1703,6 +1680,7 @@ func TestProxy(t *testing.T) { DbName: dbName, CollectionName: otherCollectionName, PartitionNames: []string{partitionName}, + ReplicaNumber: 1, }) assert.NoError(t, err) assert.NotEqual(t, commonpb.ErrorCode_Success, resp.ErrorCode) diff --git a/internal/proxy/query_coord_mock_test.go b/internal/proxy/query_coord_mock_test.go index 4cd22b3ec6..98122a1bdd 100644 --- a/internal/proxy/query_coord_mock_test.go +++ b/internal/proxy/query_coord_mock_test.go @@ -36,12 +36,20 @@ type QueryCoordMockOption func(mock *QueryCoordMock) type queryCoordShowCollectionsFuncType func(ctx context.Context, request *querypb.ShowCollectionsRequest) (*querypb.ShowCollectionsResponse, error) +type queryCoordShowPartitionsFuncType func(ctx context.Context, request *querypb.ShowPartitionsRequest) (*querypb.ShowPartitionsResponse, error) + func SetQueryCoordShowCollectionsFunc(f queryCoordShowCollectionsFuncType) QueryCoordMockOption { return func(mock *QueryCoordMock) { mock.showCollectionsFunc = f } } +func withValidShardLeaders() QueryCoordMockOption { + return func(mock *QueryCoordMock) { + mock.validShardLeaders = true + } +} + type QueryCoordMock struct { nodeID typeutil.UniqueID address string @@ -54,9 +62,12 @@ type QueryCoordMock struct { showCollectionsFunc queryCoordShowCollectionsFuncType getMetricsFunc getMetricsFuncType + showPartitionsFunc queryCoordShowPartitionsFuncType statisticsChannel string timeTickChannel string + + validShardLeaders bool } func (coord *QueryCoordMock) updateState(state internalpb.StateCode) { @@ -223,6 +234,14 @@ func (coord *QueryCoordMock) ReleaseCollection(ctx context.Context, req *querypb }, nil } +func (coord *QueryCoordMock) SetShowPartitionsFunc(f queryCoordShowPartitionsFuncType) { + coord.showPartitionsFunc = f +} + +func (coord *QueryCoordMock) ResetShowPartitionsFunc() { + coord.showPartitionsFunc = nil +} + func (coord *QueryCoordMock) ShowPartitions(ctx context.Context, req *querypb.ShowPartitionsRequest) (*querypb.ShowPartitionsResponse, error) { if !coord.healthy() { return &querypb.ShowPartitionsResponse{ @@ -233,6 +252,10 @@ func (coord *QueryCoordMock) ShowPartitions(ctx context.Context, req *querypb.Sh }, nil } + if coord.showPartitionsFunc != nil { + return coord.showPartitionsFunc(ctx, req) + } + panic("implement me") } @@ -360,6 +383,21 @@ func (coord *QueryCoordMock) GetShardLeaders(ctx context.Context, req *querypb.G }, nil } + if coord.validShardLeaders { + return &querypb.GetShardLeadersResponse{ + Status: &commonpb.Status{ + ErrorCode: commonpb.ErrorCode_Success, + }, + Shards: []*querypb.ShardLeadersList{ + { + ChannelName: "channel-1", + NodeIds: []int64{1, 2, 3}, + NodeAddrs: []string{"localhost:9000", "localhost:9001", "localhost:9002"}, + }, + }, + }, nil + } + return &querypb.GetShardLeadersResponse{ Status: &commonpb.Status{ ErrorCode: commonpb.ErrorCode_UnexpectedError, diff --git a/internal/proxy/query_node_mock_test.go b/internal/proxy/query_node_mock_test.go new file mode 100644 index 0000000000..b3a2bdc2b2 --- /dev/null +++ b/internal/proxy/query_node_mock_test.go @@ -0,0 +1,114 @@ +// Licensed to the LF AI & Data foundation under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package proxy + +import ( + "context" + "sync/atomic" + + "github.com/milvus-io/milvus/internal/types" + + "github.com/milvus-io/milvus/internal/proto/commonpb" + "github.com/milvus-io/milvus/internal/proto/internalpb" + "github.com/milvus-io/milvus/internal/proto/milvuspb" + "github.com/milvus-io/milvus/internal/proto/querypb" + "github.com/milvus-io/milvus/internal/util/typeutil" +) + +var _ types.QueryNode = &QueryNodeMock{} + +type QueryNodeMock struct { + nodeID typeutil.UniqueID + address string + + state atomic.Value // internal.StateCode + + withSearchResult *internalpb.SearchResults + withQueryResult *internalpb.RetrieveResults +} + +func (m *QueryNodeMock) Search(ctx context.Context, req *querypb.SearchRequest) (*internalpb.SearchResults, error) { + return m.withSearchResult, nil +} + +func (m *QueryNodeMock) Query(ctx context.Context, req *querypb.QueryRequest) (*internalpb.RetrieveResults, error) { + return m.withQueryResult, nil +} + +// TODO +func (m *QueryNodeMock) AddQueryChannel(ctx context.Context, req *querypb.AddQueryChannelRequest) (*commonpb.Status, error) { + return nil, nil +} + +// TODO +func (m *QueryNodeMock) RemoveQueryChannel(ctx context.Context, req *querypb.RemoveQueryChannelRequest) (*commonpb.Status, error) { + return nil, nil +} + +// TODO +func (m *QueryNodeMock) WatchDmChannels(ctx context.Context, req *querypb.WatchDmChannelsRequest) (*commonpb.Status, error) { + return nil, nil +} + +// TODO +func (m *QueryNodeMock) WatchDeltaChannels(ctx context.Context, req *querypb.WatchDeltaChannelsRequest) (*commonpb.Status, error) { + return nil, nil +} + +// TODO +func (m *QueryNodeMock) LoadSegments(ctx context.Context, req *querypb.LoadSegmentsRequest) (*commonpb.Status, error) { + return nil, nil +} + +// TODO +func (m *QueryNodeMock) ReleaseCollection(ctx context.Context, req *querypb.ReleaseCollectionRequest) (*commonpb.Status, error) { + return nil, nil +} + +// TODO +func (m *QueryNodeMock) ReleasePartitions(ctx context.Context, req *querypb.ReleasePartitionsRequest) (*commonpb.Status, error) { + return nil, nil +} + +// TODO +func (m *QueryNodeMock) ReleaseSegments(ctx context.Context, req *querypb.ReleaseSegmentsRequest) (*commonpb.Status, error) { + return nil, nil +} + +// TODO +func (m *QueryNodeMock) GetSegmentInfo(ctx context.Context, req *querypb.GetSegmentInfoRequest) (*querypb.GetSegmentInfoResponse, error) { + return nil, nil +} + +// TODO +func (m *QueryNodeMock) GetMetrics(ctx context.Context, req *milvuspb.GetMetricsRequest) (*milvuspb.GetMetricsResponse, error) { + return nil, nil +} + +func (m *QueryNodeMock) Init() error { return nil } +func (m *QueryNodeMock) Start() error { return nil } +func (m *QueryNodeMock) Stop() error { return nil } +func (m *QueryNodeMock) Register() error { return nil } +func (m *QueryNodeMock) GetComponentStates(ctx context.Context) (*internalpb.ComponentStates, error) { + return nil, nil +} +func (m *QueryNodeMock) GetStatisticsChannel(ctx context.Context) (*milvuspb.StringResponse, error) { + return nil, nil +} +func (m *QueryNodeMock) GetTimeTickChannel(ctx context.Context) (*milvuspb.StringResponse, error) { + return nil, nil +} diff --git a/internal/proxy/result_receiver.go b/internal/proxy/result_receiver.go deleted file mode 100644 index 69fe7b0a7d..0000000000 --- a/internal/proxy/result_receiver.go +++ /dev/null @@ -1,24 +0,0 @@ -package proxy - -import ( - "context" - - "github.com/milvus-io/milvus/internal/proto/commonpb" - "github.com/milvus-io/milvus/internal/proto/internalpb" -) - -func (node *Proxy) SendSearchResult(ctx context.Context, req *internalpb.SearchResults) (*commonpb.Status, error) { - node.searchResultCh <- req - return &commonpb.Status{ - ErrorCode: commonpb.ErrorCode_Success, - Reason: "", - }, nil -} - -func (node *Proxy) SendRetrieveResult(ctx context.Context, req *internalpb.RetrieveResults) (*commonpb.Status, error) { - node.retrieveResultCh <- req - return &commonpb.Status{ - ErrorCode: commonpb.ErrorCode_Success, - Reason: "", - }, nil -} diff --git a/internal/proxy/task.go b/internal/proxy/task.go index 8f60e1279a..bcebd6921f 100644 --- a/internal/proxy/task.go +++ b/internal/proxy/task.go @@ -2675,9 +2675,10 @@ func (lct *loadCollectionTask) Execute(ctx context.Context) (err error) { Timestamp: lct.Base.Timestamp, SourceID: lct.Base.SourceID, }, - DbID: 0, - CollectionID: collID, - Schema: collSchema, + DbID: 0, + CollectionID: collID, + Schema: collSchema, + ReplicaNumber: lct.ReplicaNumber, } log.Debug("send LoadCollectionRequest to query coordinator", zap.String("role", typeutil.ProxyRole), zap.Int64("msgID", request.Base.MsgID), zap.Int64("collectionID", request.CollectionID), @@ -2869,10 +2870,11 @@ func (lpt *loadPartitionsTask) Execute(ctx context.Context) error { Timestamp: lpt.Base.Timestamp, SourceID: lpt.Base.SourceID, }, - DbID: 0, - CollectionID: collID, - PartitionIDs: partitionIDs, - Schema: collSchema, + DbID: 0, + CollectionID: collID, + PartitionIDs: partitionIDs, + Schema: collSchema, + ReplicaNumber: lpt.ReplicaNumber, } lpt.result, err = lpt.queryCoord.LoadPartitions(ctx, request) return err diff --git a/internal/proxy/task_policies.go b/internal/proxy/task_policies.go new file mode 100644 index 0000000000..cddd92750b --- /dev/null +++ b/internal/proxy/task_policies.go @@ -0,0 +1,81 @@ +package proxy + +import ( + "context" + "errors" + "fmt" + + qnClient "github.com/milvus-io/milvus/internal/distributed/querynode/client" + + "github.com/milvus-io/milvus/internal/log" + "github.com/milvus-io/milvus/internal/proto/querypb" + "github.com/milvus-io/milvus/internal/types" + + "go.uber.org/zap" +) + +type getQueryNodePolicy func(context.Context, string) (types.QueryNode, error) + +type pickShardPolicy func(ctx context.Context, policy getQueryNodePolicy, query func(UniqueID, types.QueryNode) error, leaders *querypb.ShardLeadersList) error + +// TODO add another policy to enbale the use of cache +// defaultGetQueryNodePolicy creates QueryNode client for every address everytime +func defaultGetQueryNodePolicy(ctx context.Context, address string) (types.QueryNode, error) { + qn, err := qnClient.NewClient(ctx, address) + if err != nil { + return nil, err + } + + if err := qn.Init(); err != nil { + return nil, err + } + + if err := qn.Start(); err != nil { + return nil, err + } + return qn, nil +} + +var ( + errBegin = errors.New("begin error") + errInvalidShardLeaders = errors.New("Invalid shard leader") +) + +func roundRobinPolicy(ctx context.Context, getQueryNodePolicy getQueryNodePolicy, query func(UniqueID, types.QueryNode) error, leaders *querypb.ShardLeadersList) error { + var ( + err = errBegin + current = 0 + qn types.QueryNode + ) + replicaNum := len(leaders.GetNodeIds()) + + for err != nil && current < replicaNum { + currentID := leaders.GetNodeIds()[current] + if err != errBegin { + log.Warn("retry with another QueryNode", zap.String("leader", leaders.GetChannelName()), zap.Int64("nodeID", currentID)) + } + + qn, err = getQueryNodePolicy(ctx, leaders.GetNodeAddrs()[current]) + if err != nil { + log.Warn("fail to get valid QueryNode", zap.Int64("nodeID", currentID), + zap.Error(err)) + current++ + continue + } + + defer qn.Stop() + err = query(currentID, qn) + if err != nil { + log.Warn("fail to Query with shard leader", + zap.String("leader", leaders.GetChannelName()), + zap.Int64("nodeID", currentID), + zap.Error(err)) + } + current++ + } + + if current == replicaNum && err != nil { + return fmt.Errorf("no shard leaders available for channel: %s, leaders: %v, err: %s", leaders.GetChannelName(), leaders.GetNodeIds(), err.Error()) + } + return nil +} diff --git a/internal/proxy/task_query.go b/internal/proxy/task_query.go index 9c3ed14499..9aedfa1b18 100644 --- a/internal/proxy/task_query.go +++ b/internal/proxy/task_query.go @@ -6,219 +6,121 @@ import ( "fmt" "regexp" "strings" + "sync" "time" "github.com/golang/protobuf/proto" "go.uber.org/zap" + "golang.org/x/sync/errgroup" "github.com/milvus-io/milvus/internal/log" - "github.com/milvus-io/milvus/internal/mq/msgstream" "github.com/milvus-io/milvus/internal/types" + "github.com/milvus-io/milvus/internal/util/timerecord" + "github.com/milvus-io/milvus/internal/util/tsoutil" + "github.com/milvus-io/milvus/internal/util/typeutil" "github.com/milvus-io/milvus/internal/proto/commonpb" "github.com/milvus-io/milvus/internal/proto/internalpb" "github.com/milvus-io/milvus/internal/proto/milvuspb" "github.com/milvus-io/milvus/internal/proto/querypb" "github.com/milvus-io/milvus/internal/proto/schemapb" +) - "github.com/milvus-io/milvus/internal/util/timerecord" - "github.com/milvus-io/milvus/internal/util/tsoutil" - "github.com/milvus-io/milvus/internal/util/typeutil" +const ( + WithCache = true + WithoutCache = false ) type queryTask struct { Condition *internalpb.RetrieveRequest + ctx context.Context - resultBuf chan []*internalpb.RetrieveResults result *milvuspb.QueryResults - query *milvuspb.QueryRequest - chMgr channelsMgr + request *milvuspb.QueryRequest qc types.QueryCoord ids *schemapb.IDs collectionName string - collectionID UniqueID + + resultBuf chan *internalpb.RetrieveResults + toReduceResults []*internalpb.RetrieveResults + runningGroup *errgroup.Group + runningGroupCtx context.Context + + getQueryNodePolicy getQueryNodePolicy + queryShardPolicy pickShardPolicy } -func (qt *queryTask) PreExecute(ctx context.Context) error { - qt.Base.MsgType = commonpb.MsgType_Retrieve - qt.Base.SourceID = Params.ProxyCfg.ProxyID +func (t *queryTask) PreExecute(ctx context.Context) error { + if t.getQueryNodePolicy == nil { + t.getQueryNodePolicy = defaultGetQueryNodePolicy + } - collectionName := qt.query.CollectionName + if t.queryShardPolicy == nil { + t.queryShardPolicy = roundRobinPolicy + } - if err := validateCollectionName(qt.query.CollectionName); err != nil { + t.Base.MsgType = commonpb.MsgType_Retrieve + t.Base.SourceID = Params.ProxyCfg.ProxyID + + collectionName := t.request.CollectionName + t.collectionName = collectionName + if err := validateCollectionName(collectionName); err != nil { log.Warn("Invalid collection name.", zap.String("collectionName", collectionName), - zap.Int64("requestID", qt.Base.MsgID), zap.String("requestType", "query")) + zap.Int64("requestID", t.Base.MsgID), zap.String("requestType", "query")) return err } - log.Info("Validate collection name.", zap.Any("collectionName", collectionName), - zap.Any("requestID", qt.Base.MsgID), zap.Any("requestType", "query")) - info, err := globalMetaCache.GetCollectionInfo(ctx, collectionName) + log.Info("Validate collection name.", zap.Any("collectionName", collectionName), + zap.Any("requestID", t.Base.MsgID), zap.Any("requestType", "query")) + + collID, err := globalMetaCache.GetCollectionID(ctx, collectionName) if err != nil { log.Debug("Failed to get collection id.", zap.Any("collectionName", collectionName), - zap.Any("requestID", qt.Base.MsgID), zap.Any("requestType", "query")) + zap.Any("requestID", t.Base.MsgID), zap.Any("requestType", "query")) return err } - qt.collectionName = info.schema.Name - log.Info("Get collection id by name.", zap.Any("collectionName", collectionName), - zap.Any("requestID", qt.Base.MsgID), zap.Any("requestType", "query")) - for _, tag := range qt.query.PartitionNames { + t.CollectionID = collID + log.Info("Get collection ID by name", + zap.Int64("collectionID", t.CollectionID), zap.String("collection name", collectionName), + zap.Any("requestID", t.Base.MsgID), zap.Any("requestType", "query")) + + for _, tag := range t.request.PartitionNames { if err := validatePartitionTag(tag, false); err != nil { - log.Debug("Invalid partition name.", zap.Any("partitionName", tag), - zap.Any("requestID", qt.Base.MsgID), zap.Any("requestType", "query")) + log.Warn("invalid partition name", zap.String("partition name", tag), + zap.Any("requestID", t.Base.MsgID), zap.Any("requestType", "query")) return err } } - log.Info("Validate partition names.", - zap.Any("requestID", qt.Base.MsgID), zap.Any("requestType", "query")) - - // check if collection was already loaded into query node - showResp, err := qt.qc.ShowCollections(qt.ctx, &querypb.ShowCollectionsRequest{ - Base: &commonpb.MsgBase{ - MsgType: commonpb.MsgType_ShowCollections, - MsgID: qt.Base.MsgID, - Timestamp: qt.Base.Timestamp, - SourceID: Params.ProxyCfg.ProxyID, - }, - DbID: 0, // TODO(dragondriver) - }) - if err != nil { - return err - } - if showResp.Status.ErrorCode != commonpb.ErrorCode_Success { - return errors.New(showResp.Status.Reason) - } - log.Debug("QueryCoord show collections", - zap.Any("collections", showResp.CollectionIDs), - zap.Any("collID", info.collID)) - - collectionLoaded := false - for _, collID := range showResp.CollectionIDs { - if info.collID == collID { - collectionLoaded = true - break - } - } - if !collectionLoaded { - return fmt.Errorf("collection %v was not loaded into memory", collectionName) - } - - schema, _ := globalMetaCache.GetCollectionSchema(ctx, qt.query.CollectionName) - - if qt.ids != nil { - pkField := "" - for _, field := range schema.Fields { - if field.IsPrimaryKey { - pkField = field.Name - } - } - qt.query.Expr = IDs2Expr(pkField, qt.ids.GetIntId().Data) - } - - if qt.query.Expr == "" { - errMsg := "Query expression is empty" - return fmt.Errorf(errMsg) - } - - plan, err := createExprPlan(schema, qt.query.Expr) - if err != nil { - return err - } - qt.query.OutputFields, err = translateOutputFields(qt.query.OutputFields, schema, true) - if err != nil { - return err - } - log.Debug("translate output fields", zap.Any("OutputFields", qt.query.OutputFields)) - if len(qt.query.OutputFields) == 0 { - for _, field := range schema.Fields { - if field.FieldID >= 100 && field.DataType != schemapb.DataType_FloatVector && field.DataType != schemapb.DataType_BinaryVector { - qt.OutputFieldsId = append(qt.OutputFieldsId, field.FieldID) - } - } - } else { - addPrimaryKey := false - for _, reqField := range qt.query.OutputFields { - findField := false - for _, field := range schema.Fields { - if reqField == field.Name { - if field.IsPrimaryKey { - addPrimaryKey = true - } - findField = true - qt.OutputFieldsId = append(qt.OutputFieldsId, field.FieldID) - plan.OutputFieldIds = append(plan.OutputFieldIds, field.FieldID) - } else { - if field.IsPrimaryKey && !addPrimaryKey { - qt.OutputFieldsId = append(qt.OutputFieldsId, field.FieldID) - plan.OutputFieldIds = append(plan.OutputFieldIds, field.FieldID) - addPrimaryKey = true - } - } - } - if !findField { - errMsg := "Field " + reqField + " not exist" - return errors.New(errMsg) - } - } - } - log.Debug("translate output fields to field ids", zap.Any("OutputFieldsID", qt.OutputFieldsId)) - - qt.RetrieveRequest.SerializedExprPlan, err = proto.Marshal(plan) - if err != nil { - return err - } - travelTimestamp := qt.query.TravelTimestamp - if travelTimestamp == 0 { - travelTimestamp = qt.BeginTs() - } else { - durationSeconds := tsoutil.CalculateDuration(qt.BeginTs(), travelTimestamp) / 1000 - if durationSeconds > Params.CommonCfg.RetentionDuration { - duration := time.Second * time.Duration(durationSeconds) - return fmt.Errorf("only support to travel back to %s so far", duration.String()) - } - } - guaranteeTimestamp := qt.query.GuaranteeTimestamp - if guaranteeTimestamp == 0 { - guaranteeTimestamp = qt.BeginTs() - } - qt.TravelTimestamp = travelTimestamp - qt.GuaranteeTimestamp = guaranteeTimestamp - deadline, ok := qt.TraceCtx().Deadline() - if ok { - qt.RetrieveRequest.TimeoutTimestamp = tsoutil.ComposeTSByTime(deadline, 0) - } - - qt.ResultChannelID = Params.ProxyCfg.RetrieveResultChannelNames[0] - qt.DbID = 0 // todo(yukun) - - qt.CollectionID = info.collID - qt.PartitionIDs = make([]UniqueID, 0) + log.Debug("Validate partition names.", + zap.Any("requestID", t.Base.MsgID), zap.Any("requestType", "query")) + t.PartitionIDs = make([]UniqueID, 0) partitionsMap, err := globalMetaCache.GetPartitions(ctx, collectionName) if err != nil { - log.Debug("Failed to get partitions in collection.", zap.Any("collectionName", collectionName), - zap.Any("requestID", qt.Base.MsgID), zap.Any("requestType", "query")) + log.Warn("failed to get partitions in collection.", zap.String("collection name", collectionName), + zap.Any("requestID", t.Base.MsgID), zap.Any("requestType", "query")) return err } - log.Info("Get partitions in collection.", zap.Any("collectionName", collectionName), - zap.Any("requestID", qt.Base.MsgID), zap.Any("requestType", "query")) + log.Debug("Get partitions in collection.", zap.Any("collectionName", collectionName), + zap.Any("requestID", t.Base.MsgID), zap.Any("requestType", "query")) + // Check if partitions are valid partitions in collection partitionsRecord := make(map[UniqueID]bool) - for _, partitionName := range qt.query.PartitionNames { + for _, partitionName := range t.request.PartitionNames { pattern := fmt.Sprintf("^%s$", partitionName) re, err := regexp.Compile(pattern) if err != nil { - log.Debug("Failed to compile partition name regex expression.", zap.Any("partitionName", partitionName), - zap.Any("requestID", qt.Base.MsgID), zap.Any("requestType", "query")) + log.Debug("failed to compile partition name regex expression.", zap.Any("partition name", partitionName), + zap.Any("requestID", t.Base.MsgID), zap.Any("requestType", "query")) return errors.New("invalid partition names") } found := false for name, pID := range partitionsMap { if re.MatchString(name) { if _, exist := partitionsRecord[pID]; !exist { - qt.PartitionIDs = append(qt.PartitionIDs, pID) + t.PartitionIDs = append(t.PartitionIDs, pID) partitionsRecord[pID] = true } found = true @@ -226,176 +128,355 @@ func (qt *queryTask) PreExecute(ctx context.Context) error { } if !found { // FIXME(wxyu): undefined behavior - errMsg := fmt.Sprintf("PartitonName: %s not found", partitionName) + errMsg := fmt.Sprintf("partition name: %s not found", partitionName) return errors.New(errMsg) } } + if !t.checkIfLoaded(collID, t.PartitionIDs) { + return fmt.Errorf("collection:%v or partition:%v not loaded into memory", collectionName, t.request.GetPartitionNames()) + } + + schema, _ := globalMetaCache.GetCollectionSchema(ctx, collectionName) + + if t.ids != nil { + pkField := "" + for _, field := range schema.Fields { + if field.IsPrimaryKey { + pkField = field.Name + } + } + t.request.Expr = IDs2Expr(pkField, t.ids.GetIntId().Data) + } + + if t.request.Expr == "" { + return fmt.Errorf("query expression is empty") + } + + plan, err := createExprPlan(schema, t.request.Expr) + if err != nil { + return err + } + t.request.OutputFields, err = translateOutputFields(t.request.OutputFields, schema, true) + if err != nil { + return err + } + log.Debug("translate output fields", zap.Any("OutputFields", t.request.OutputFields), + zap.Any("requestID", t.Base.MsgID), zap.Any("requestType", "query")) + + if len(t.request.OutputFields) == 0 { + for _, field := range schema.Fields { + if field.FieldID >= 100 && field.DataType != schemapb.DataType_FloatVector && field.DataType != schemapb.DataType_BinaryVector { + t.OutputFieldsId = append(t.OutputFieldsId, field.FieldID) + } + } + } else { + addPrimaryKey := false + for _, reqField := range t.request.OutputFields { + findField := false + for _, field := range schema.Fields { + if reqField == field.Name { + if field.IsPrimaryKey { + addPrimaryKey = true + } + findField = true + t.OutputFieldsId = append(t.OutputFieldsId, field.FieldID) + plan.OutputFieldIds = append(plan.OutputFieldIds, field.FieldID) + } else { + if field.IsPrimaryKey && !addPrimaryKey { + t.OutputFieldsId = append(t.OutputFieldsId, field.FieldID) + plan.OutputFieldIds = append(plan.OutputFieldIds, field.FieldID) + addPrimaryKey = true + } + } + } + if !findField { + return fmt.Errorf("field %s not exist", reqField) + } + } + } + log.Debug("translate output fields to field ids", zap.Any("OutputFieldsID", t.OutputFieldsId), + zap.Any("requestID", t.Base.MsgID), zap.Any("requestType", "query")) + + t.RetrieveRequest.SerializedExprPlan, err = proto.Marshal(plan) + if err != nil { + return err + } + + if t.request.TravelTimestamp == 0 { + t.TravelTimestamp = t.BeginTs() + } else { + durationSeconds := tsoutil.CalculateDuration(t.BeginTs(), t.request.TravelTimestamp) / 1000 + if durationSeconds > Params.CommonCfg.RetentionDuration { + duration := time.Second * time.Duration(durationSeconds) + return fmt.Errorf("only support to travel back to %s so far", duration.String()) + } + t.TravelTimestamp = t.request.TravelTimestamp + } + + if t.request.GuaranteeTimestamp == 0 { + t.GuaranteeTimestamp = t.BeginTs() + } else { + t.GuaranteeTimestamp = t.request.GuaranteeTimestamp + } + + deadline, ok := t.TraceCtx().Deadline() + if ok { + t.TimeoutTimestamp = tsoutil.ComposeTSByTime(deadline, 0) + } + + t.DbID = 0 // TODO log.Info("Query PreExecute done.", - zap.Any("requestID", qt.Base.MsgID), zap.Any("requestType", "query")) + zap.Any("requestID", t.Base.MsgID), zap.Any("requestType", "query")) return nil } -func (qt *queryTask) Execute(ctx context.Context) error { - tr := timerecord.NewTimeRecorder(fmt.Sprintf("proxy execute query %d", qt.ID())) +func (t *queryTask) Execute(ctx context.Context) error { + tr := timerecord.NewTimeRecorder(fmt.Sprintf("proxy execute query %d", t.ID())) defer tr.Elapse("done") - var tsMsg msgstream.TsMsg = &msgstream.RetrieveMsg{ - RetrieveRequest: *qt.RetrieveRequest, - BaseMsg: msgstream.BaseMsg{ - Ctx: ctx, - HashValues: []uint32{uint32(Params.ProxyCfg.ProxyID)}, - BeginTimestamp: qt.Base.Timestamp, - EndTimestamp: qt.Base.Timestamp, - }, - } - msgPack := msgstream.MsgPack{ - BeginTs: qt.Base.Timestamp, - EndTs: qt.Base.Timestamp, - Msgs: make([]msgstream.TsMsg, 1), - } - msgPack.Msgs[0] = tsMsg - - stream, err := qt.chMgr.getDQLStream(qt.CollectionID) - if err != nil { - err = qt.chMgr.createDQLStream(qt.CollectionID) + executeQuery := func(withCache bool) error { + shards, err := globalMetaCache.GetShards(ctx, withCache, t.collectionName, t.qc) if err != nil { - qt.result.Status.ErrorCode = commonpb.ErrorCode_UnexpectedError - qt.result.Status.Reason = err.Error() return err } - stream, err = qt.chMgr.getDQLStream(qt.CollectionID) - if err != nil { - qt.result.Status.ErrorCode = commonpb.ErrorCode_UnexpectedError - qt.result.Status.Reason = err.Error() - return err - } - } - tr.Record("get used message stream") - err = stream.Produce(&msgPack) - if err != nil { - log.Debug("Failed to send retrieve request.", - zap.Any("requestID", qt.Base.MsgID), zap.Any("requestType", "query")) + t.resultBuf = make(chan *internalpb.RetrieveResults, len(shards)) + t.toReduceResults = make([]*internalpb.RetrieveResults, 0, len(shards)) + t.runningGroup, t.runningGroupCtx = errgroup.WithContext(ctx) + for _, shard := range shards { + s := shard + t.runningGroup.Go(func() error { + log.Debug("proxy starting to query one shard", + zap.Int64("collectionID", t.CollectionID), + zap.String("collection name", t.collectionName), + zap.String("shard channel", s.GetChannelName()), + zap.Uint64("timeoutTs", t.TimeoutTimestamp)) + + err := t.queryShard(t.runningGroupCtx, s) + if err != nil { + return err + } + return nil + }) + } + + err = t.runningGroup.Wait() + return err + } + + err := executeQuery(WithCache) + if err == errInvalidShardLeaders { + log.Warn("invalid shard leaders cache, updating shardleader caches and retry search") + return executeQuery(WithoutCache) + } + if err != nil { + return err } - log.Debug("proxy sent one retrieveMsg", - zap.Int64("collectionID", qt.CollectionID), - zap.Int64("msgID", tsMsg.ID()), - zap.Int("length of search msg", len(msgPack.Msgs)), - zap.Uint64("timeoutTs", qt.RetrieveRequest.TimeoutTimestamp)) - tr.Record("send retrieve request to message stream") log.Info("Query Execute done.", - zap.Any("requestID", qt.Base.MsgID), zap.Any("requestType", "query")) - return err + zap.Any("requestID", t.Base.MsgID), zap.Any("requestType", "query")) + return nil } -func (qt *queryTask) PostExecute(ctx context.Context) error { +func (t *queryTask) PostExecute(ctx context.Context) error { tr := timerecord.NewTimeRecorder("queryTask PostExecute") defer func() { tr.Elapse("done") }() - select { - case <-qt.TraceCtx().Done(): - log.Debug("proxy", zap.Int64("Query: wait to finish failed, timeout!, taskID:", qt.ID())) - return fmt.Errorf("queryTask:wait to finish failed, timeout : %d", qt.ID()) - case retrieveResults := <-qt.resultBuf: - filterRetrieveResults := make([]*internalpb.RetrieveResults, 0) - var reason string - for _, partialRetrieveResult := range retrieveResults { - if partialRetrieveResult.Status.ErrorCode == commonpb.ErrorCode_Success { - filterRetrieveResults = append(filterRetrieveResults, partialRetrieveResult) - } else { - reason += partialRetrieveResult.Status.Reason + "\n" - } - } - if len(filterRetrieveResults) == 0 { - qt.result = &milvuspb.QueryResults{ - Status: &commonpb.Status{ - ErrorCode: commonpb.ErrorCode_UnexpectedError, - Reason: reason, - }, - CollectionName: qt.collectionName, - } - log.Debug("Query failed on all querynodes.", - zap.Any("requestID", qt.Base.MsgID), zap.Any("requestType", "query")) - return errors.New(reason) - } - - var err error - qt.result, err = mergeRetrieveResults(filterRetrieveResults) - if err != nil { - return err - } - qt.result.CollectionName = qt.collectionName - - if len(qt.result.FieldsData) > 0 { - qt.result.Status = &commonpb.Status{ - ErrorCode: commonpb.ErrorCode_Success, - } - } else { - log.Info("Query result is nil", zap.Any("requestID", qt.Base.MsgID), zap.Any("requestType", "query")) - qt.result.Status = &commonpb.Status{ - ErrorCode: commonpb.ErrorCode_EmptyCollection, - Reason: reason, - } - return nil - } - - schema, err := globalMetaCache.GetCollectionSchema(ctx, qt.query.CollectionName) - if err != nil { - return err - } - for i := 0; i < len(qt.result.FieldsData); i++ { - for _, field := range schema.Fields { - if field.FieldID == qt.OutputFieldsId[i] { - qt.result.FieldsData[i].FieldName = field.Name - qt.result.FieldsData[i].FieldId = field.FieldID - qt.result.FieldsData[i].Type = field.DataType + var err error + wg := sync.WaitGroup{} + wg.Add(1) + go func() { + for { + select { + case <-t.TraceCtx().Done(): + log.Warn("proxy", zap.Int64("Query: wait to finish failed, timeout!, taskID:", t.ID())) + return + case <-t.runningGroupCtx.Done(): + log.Debug("all queries are finished or canceled", zap.Any("taskID", t.ID())) + close(t.resultBuf) + for res := range t.resultBuf { + t.toReduceResults = append(t.toReduceResults, res) + log.Debug("proxy receives one query result", zap.Int64("sourceID", res.GetBase().GetSourceID()), zap.Any("taskID", t.ID())) } + wg.Done() + return + } + } + }() + + wg.Wait() + t.result, err = mergeRetrieveResults(t.toReduceResults) + if err != nil { + return err + } + t.result.CollectionName = t.collectionName + + if len(t.result.FieldsData) > 0 { + t.result.Status = &commonpb.Status{ + ErrorCode: commonpb.ErrorCode_Success, + } + } else { + log.Info("Query result is nil", zap.Any("requestID", t.Base.MsgID), zap.Any("requestType", "query")) + t.result.Status = &commonpb.Status{ + ErrorCode: commonpb.ErrorCode_EmptyCollection, + Reason: "emptly collection", // TODO + } + return nil + } + + schema, err := globalMetaCache.GetCollectionSchema(ctx, t.request.CollectionName) + if err != nil { + return err + } + for i := 0; i < len(t.result.FieldsData); i++ { + for _, field := range schema.Fields { + if field.FieldID == t.OutputFieldsId[i] { + t.result.FieldsData[i].FieldName = field.Name + t.result.FieldsData[i].FieldId = field.FieldID + t.result.FieldsData[i].Type = field.DataType } } } - - log.Info("Query PostExecute done", zap.Any("requestID", qt.Base.MsgID), zap.Any("requestType", "query")) + log.Info("Query PostExecute done", zap.Any("requestID", t.Base.MsgID), zap.String("requestType", "query")) return nil } -func (qt *queryTask) getChannels() ([]pChan, error) { - collID, err := globalMetaCache.GetCollectionID(qt.ctx, qt.query.CollectionName) - if err != nil { - return nil, err - } - var channels []pChan - channels, err = qt.chMgr.getChannels(collID) - if err != nil { - err := qt.chMgr.createDMLMsgStream(collID) - if err != nil { - return nil, err + +func (t *queryTask) queryShard(ctx context.Context, leaders *querypb.ShardLeadersList) error { + query := func(nodeID UniqueID, qn types.QueryNode) error { + req := &querypb.QueryRequest{ + Req: t.RetrieveRequest, + DmlChannel: leaders.GetChannelName(), } - return qt.chMgr.getChannels(collID) + + result, err := qn.Query(ctx, req) + if err != nil { + log.Warn("QueryNode query returns error", zap.Int64("nodeID", nodeID), + zap.Error(err)) + return errInvalidShardLeaders + } + if result.GetStatus().GetErrorCode() != commonpb.ErrorCode_Success { + log.Warn("QueryNode query result error", zap.Int64("nodeID", nodeID), + zap.String("reason", result.GetStatus().GetReason())) + return fmt.Errorf("fail to Query, QueryNode ID = %d, reason=%s", nodeID, result.GetStatus().GetReason()) + } + + log.Debug("get query result", zap.Int64("nodeID", nodeID), zap.String("channelID", leaders.GetChannelName())) + t.resultBuf <- result + return nil } - return channels, nil + err := t.queryShardPolicy(t.TraceCtx(), t.getQueryNodePolicy, query, leaders) + if err != nil { + log.Warn("fail to Query to all shard leaders", zap.Int64("taskID", t.ID()), zap.Any("shard leaders", leaders.GetNodeIds())) + return err + } + + return nil } -func (qt *queryTask) getVChannels() ([]vChan, error) { - collID, err := globalMetaCache.GetCollectionID(qt.ctx, qt.query.CollectionName) - if err != nil { - return nil, err - } - - var channels []vChan - channels, err = qt.chMgr.getVChannels(collID) - if err != nil { - err := qt.chMgr.createDMLMsgStream(collID) +func (t *queryTask) checkIfLoaded(collectionID UniqueID, searchPartitionIDs []UniqueID) bool { + // If request to search partitions + if len(searchPartitionIDs) > 0 { + resp, err := t.qc.ShowPartitions(t.ctx, &querypb.ShowPartitionsRequest{ + Base: &commonpb.MsgBase{ + MsgType: commonpb.MsgType_ShowCollections, + MsgID: t.Base.MsgID, + Timestamp: t.Base.Timestamp, + SourceID: Params.ProxyCfg.ProxyID, + }, + CollectionID: collectionID, + PartitionIDs: searchPartitionIDs, + }) if err != nil { - return nil, err + log.Warn("fail to show partitions by QueryCoord", + zap.Int64("requestID", t.Base.MsgID), + zap.Int64("collectionID", collectionID), + zap.Int64s("partitionIDs", searchPartitionIDs), + zap.String("requestType", "search"), + zap.Error(err)) + return false } - return qt.chMgr.getVChannels(collID) + + if resp.Status.ErrorCode != commonpb.ErrorCode_Success { + log.Warn("fail to show partitions by QueryCoord", + zap.Int64("collectionID", collectionID), + zap.Int64s("partitionIDs", searchPartitionIDs), + zap.Int64("requestID", t.Base.MsgID), zap.String("requestType", "search"), + zap.String("reason", resp.GetStatus().GetReason())) + return false + } + // Current logic: show partitions won't return error if the given partitions are all loaded + return true } - return channels, nil + // If request to search collection + resp, err := t.qc.ShowCollections(t.ctx, &querypb.ShowCollectionsRequest{ + Base: &commonpb.MsgBase{ + MsgType: commonpb.MsgType_ShowCollections, + MsgID: t.Base.MsgID, + Timestamp: t.Base.Timestamp, + SourceID: Params.ProxyCfg.ProxyID, + }, + }) + if err != nil { + log.Warn("fail to show collections by QueryCoord", + zap.Int64("requestID", t.Base.MsgID), zap.String("requestType", "search"), + zap.Error(err)) + return false + } + + if resp.Status.ErrorCode != commonpb.ErrorCode_Success { + log.Warn("fail to show collections by QueryCoord", + zap.Int64("requestID", t.Base.MsgID), zap.String("requestType", "search"), + zap.String("reason", resp.GetStatus().GetReason())) + return false + } + + loaded := false + for index, collID := range resp.CollectionIDs { + if collID == collectionID && resp.GetInMemoryPercentages()[index] >= int64(100) { + loaded = true + break + } + } + + if !loaded { + resp, err := t.qc.ShowPartitions(t.ctx, &querypb.ShowPartitionsRequest{ + Base: &commonpb.MsgBase{ + MsgType: commonpb.MsgType_ShowCollections, + MsgID: t.Base.MsgID, + Timestamp: t.Base.Timestamp, + SourceID: Params.ProxyCfg.ProxyID, + }, + CollectionID: collectionID, + }) + if err != nil { + log.Warn("fail to show partitions by QueryCoord", + zap.Int64("requestID", t.Base.MsgID), + zap.Int64("collectionID", collectionID), + zap.String("requestType", "search"), + zap.Error(err)) + return false + } + + if resp.Status.ErrorCode != commonpb.ErrorCode_Success { + log.Warn("fail to show partitions by QueryCoord", + zap.Int64("collectionID", collectionID), + zap.Int64("requestID", t.Base.MsgID), zap.String("requestType", "search"), + zap.String("reason", resp.GetStatus().GetReason())) + return false + } + + if len(resp.GetPartitionIDs()) > 0 { + log.Warn("collection not fully loaded, search on these partitions", zap.Int64s("partitionIDs", resp.GetPartitionIDs())) + return true + } + } + + return loaded } // IDs2Expr converts ids slices to bool expresion with specified field name @@ -447,39 +528,39 @@ func mergeRetrieveResults(retrieveResults []*internalpb.RetrieveResults) (*milvu return ret, nil } -func (qt *queryTask) TraceCtx() context.Context { - return qt.ctx +func (t *queryTask) TraceCtx() context.Context { + return t.ctx } -func (qt *queryTask) ID() UniqueID { - return qt.Base.MsgID +func (t *queryTask) ID() UniqueID { + return t.Base.MsgID } -func (qt *queryTask) SetID(uid UniqueID) { - qt.Base.MsgID = uid +func (t *queryTask) SetID(uid UniqueID) { + t.Base.MsgID = uid } -func (qt *queryTask) Name() string { +func (t *queryTask) Name() string { return RetrieveTaskName } -func (qt *queryTask) Type() commonpb.MsgType { - return qt.Base.MsgType +func (t *queryTask) Type() commonpb.MsgType { + return t.Base.MsgType } -func (qt *queryTask) BeginTs() Timestamp { - return qt.Base.Timestamp +func (t *queryTask) BeginTs() Timestamp { + return t.Base.Timestamp } -func (qt *queryTask) EndTs() Timestamp { - return qt.Base.Timestamp +func (t *queryTask) EndTs() Timestamp { + return t.Base.Timestamp } -func (qt *queryTask) SetTs(ts Timestamp) { - qt.Base.Timestamp = ts +func (t *queryTask) SetTs(ts Timestamp) { + t.Base.Timestamp = ts } -func (qt *queryTask) OnEnqueue() error { - qt.Base.MsgType = commonpb.MsgType_Retrieve +func (t *queryTask) OnEnqueue() error { + t.Base.MsgType = commonpb.MsgType_Retrieve return nil } diff --git a/internal/proxy/task_query_test.go b/internal/proxy/task_query_test.go index 775b545120..658cd7e15a 100644 --- a/internal/proxy/task_query_test.go +++ b/internal/proxy/task_query_test.go @@ -3,16 +3,15 @@ package proxy import ( "context" "fmt" - "strconv" - "sync" "testing" "time" "github.com/golang/protobuf/proto" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" "github.com/milvus-io/milvus/internal/common" - "github.com/milvus-io/milvus/internal/mq/msgstream" + "github.com/milvus-io/milvus/internal/types" "github.com/milvus-io/milvus/internal/proto/commonpb" "github.com/milvus-io/milvus/internal/proto/internalpb" @@ -25,25 +24,35 @@ import ( ) func TestQueryTask_all(t *testing.T) { - var err error - Params.Init() - Params.ProxyCfg.RetrieveResultChannelNames = []string{funcutil.GenRandomStr()} - rc := NewRootCoordMock() + var ( + err error + ctx = context.TODO() + + rc = NewRootCoordMock() + qc = NewQueryCoordMock(withValidShardLeaders()) + qn = &QueryNodeMock{} + + shardsNum = int32(2) + collectionName = t.Name() + funcutil.GenRandomStr() + + expr = fmt.Sprintf("%s > 0", testInt64Field) + hitNum = 10 + ) + + mockGetQueryNodePolicy := func(ctx context.Context, address string) (types.QueryNode, error) { + return qn, nil + } + rc.Start() defer rc.Stop() - - ctx := context.Background() + qc.Start() + defer qc.Stop() err = InitMetaCache(rc) assert.NoError(t, err) - shardsNum := int32(2) - prefix := "TestQueryTask_all" - dbName := "" - collectionName := prefix + funcutil.GenRandomStr() - fieldName2Types := map[string]schemapb.DataType{ testBoolField: schemapb.DataType_Bool, testInt32Field: schemapb.DataType_Int32, @@ -56,9 +65,6 @@ func TestQueryTask_all(t *testing.T) { fieldName2Types[testBinaryVecField] = schemapb.DataType_BinaryVector } - expr := fmt.Sprintf("%s > 0", testInt64Field) - hitNum := 10 - schema := constructCollectionSchemaByDataType(collectionName, fieldName2Types, testInt64Field, false) marshaledSchema, err := proto.Marshal(schema) assert.NoError(t, err) @@ -66,165 +72,66 @@ func TestQueryTask_all(t *testing.T) { createColT := &createCollectionTask{ Condition: NewTaskCondition(ctx), CreateCollectionRequest: &milvuspb.CreateCollectionRequest{ - Base: nil, - DbName: dbName, CollectionName: collectionName, Schema: marshaledSchema, ShardsNum: shardsNum, }, ctx: ctx, rootCoord: rc, - result: nil, - schema: nil, } - assert.NoError(t, createColT.OnEnqueue()) - assert.NoError(t, createColT.PreExecute(ctx)) - assert.NoError(t, createColT.Execute(ctx)) - assert.NoError(t, createColT.PostExecute(ctx)) - - dmlChannelsFunc := getDmlChannelsFunc(ctx, rc) - query := newMockGetChannelsService() - factory := newSimpleMockMsgStreamFactory() - chMgr := newChannelsMgrImpl(dmlChannelsFunc, nil, query.GetChannels, nil, factory) - defer chMgr.removeAllDMLStream() - defer chMgr.removeAllDQLStream() + require.NoError(t, createColT.OnEnqueue()) + require.NoError(t, createColT.PreExecute(ctx)) + require.NoError(t, createColT.Execute(ctx)) + require.NoError(t, createColT.PostExecute(ctx)) collectionID, err := globalMetaCache.GetCollectionID(ctx, collectionName) assert.NoError(t, err) - qc := NewQueryCoordMock() - qc.Start() - defer qc.Stop() status, err := qc.LoadCollection(ctx, &querypb.LoadCollectionRequest{ Base: &commonpb.MsgBase{ - MsgType: commonpb.MsgType_LoadCollection, - MsgID: 0, - Timestamp: 0, - SourceID: Params.ProxyCfg.ProxyID, + MsgType: commonpb.MsgType_LoadCollection, + SourceID: Params.ProxyCfg.ProxyID, }, - DbID: 0, CollectionID: collectionID, - Schema: nil, }) - assert.NoError(t, err) - assert.Equal(t, commonpb.ErrorCode_Success, status.ErrorCode) + require.NoError(t, err) + require.Equal(t, commonpb.ErrorCode_Success, status.ErrorCode) + // test begins task := &queryTask{ Condition: NewTaskCondition(ctx), RetrieveRequest: &internalpb.RetrieveRequest{ Base: &commonpb.MsgBase{ - MsgType: commonpb.MsgType_Retrieve, - MsgID: 0, - Timestamp: 0, - SourceID: Params.ProxyCfg.ProxyID, + MsgType: commonpb.MsgType_Retrieve, + SourceID: Params.ProxyCfg.ProxyID, }, - ResultChannelID: strconv.Itoa(int(Params.ProxyCfg.ProxyID)), - DbID: 0, - CollectionID: collectionID, - PartitionIDs: nil, - SerializedExprPlan: nil, - OutputFieldsId: make([]int64, len(fieldName2Types)), - TravelTimestamp: 0, - GuaranteeTimestamp: 0, + CollectionID: collectionID, + OutputFieldsId: make([]int64, len(fieldName2Types)), }, - ctx: ctx, - resultBuf: make(chan []*internalpb.RetrieveResults), + ctx: ctx, result: &milvuspb.QueryResults{ Status: &commonpb.Status{ ErrorCode: commonpb.ErrorCode_Success, }, - FieldsData: nil, }, - query: &milvuspb.QueryRequest{ + request: &milvuspb.QueryRequest{ Base: &commonpb.MsgBase{ - MsgType: commonpb.MsgType_Retrieve, - MsgID: 0, - Timestamp: 0, - SourceID: Params.ProxyCfg.ProxyID, + MsgType: commonpb.MsgType_Retrieve, + SourceID: Params.ProxyCfg.ProxyID, }, - DbName: dbName, - CollectionName: collectionName, - Expr: expr, - OutputFields: nil, - PartitionNames: nil, - TravelTimestamp: 0, - GuaranteeTimestamp: 0, + CollectionName: collectionName, + Expr: expr, }, - chMgr: chMgr, - qc: qc, - ids: nil, + qc: qc, + + getQueryNodePolicy: mockGetQueryNodePolicy, + queryShardPolicy: roundRobinPolicy, } for i := 0; i < len(fieldName2Types); i++ { task.RetrieveRequest.OutputFieldsId[i] = int64(common.StartOfUserFieldID + i) } - // simple mock for query node - // TODO(dragondriver): should we replace this mock using RocksMq or MemMsgStream? - - err = chMgr.createDQLStream(collectionID) - assert.NoError(t, err) - stream, err := chMgr.getDQLStream(collectionID) - assert.NoError(t, err) - - var wg sync.WaitGroup - wg.Add(1) - consumeCtx, cancel := context.WithCancel(ctx) - go func() { - defer wg.Done() - for { - select { - case <-consumeCtx.Done(): - return - case pack, ok := <-stream.Chan(): - assert.True(t, ok) - - if pack == nil { - continue - } - - for _, msg := range pack.Msgs { - _, ok := msg.(*msgstream.RetrieveMsg) - assert.True(t, ok) - // TODO(dragondriver): construct result according to the request - - result1 := &internalpb.RetrieveResults{ - Base: &commonpb.MsgBase{ - MsgType: commonpb.MsgType_RetrieveResult, - MsgID: 0, - Timestamp: 0, - SourceID: 0, - }, - Status: &commonpb.Status{ - ErrorCode: commonpb.ErrorCode_Success, - Reason: "", - }, - ResultChannelID: strconv.Itoa(int(Params.ProxyCfg.ProxyID)), - Ids: &schemapb.IDs{ - IdField: &schemapb.IDs_IntId{ - IntId: &schemapb.LongArray{ - Data: generateInt64Array(hitNum), - }, - }, - }, - SealedSegmentIDsRetrieved: nil, - ChannelIDsRetrieved: nil, - GlobalSealedSegmentIDs: nil, - } - - fieldID := common.StartOfUserFieldID - for fieldName, dataType := range fieldName2Types { - result1.FieldsData = append(result1.FieldsData, generateFieldData(dataType, fieldName, int64(fieldID), hitNum)) - fieldID++ - } - - // send search result - task.resultBuf <- []*internalpb.RetrieveResults{result1} - } - } - } - }() - assert.NoError(t, task.OnEnqueue()) // test query task with timeout @@ -236,11 +143,29 @@ func TestQueryTask_all(t *testing.T) { assert.NoError(t, task.PreExecute(ctx)) // after preExecute assert.Greater(t, task.TimeoutTimestamp, typeutil.ZeroTimestamp) + + result1 := &internalpb.RetrieveResults{ + Base: &commonpb.MsgBase{MsgType: commonpb.MsgType_RetrieveResult}, + Status: &commonpb.Status{ + ErrorCode: commonpb.ErrorCode_Success, + }, + Ids: &schemapb.IDs{ + IdField: &schemapb.IDs_IntId{ + IntId: &schemapb.LongArray{Data: generateInt64Array(hitNum)}, + }, + }, + } + + fieldID := common.StartOfUserFieldID + for fieldName, dataType := range fieldName2Types { + result1.FieldsData = append(result1.FieldsData, generateFieldData(dataType, fieldName, int64(fieldID), hitNum)) + fieldID++ + } + + qn.withQueryResult = result1 + task.ctx = ctx - assert.NoError(t, task.Execute(ctx)) - assert.NoError(t, task.PostExecute(ctx)) - cancel() - wg.Wait() + assert.NoError(t, task.PostExecute(ctx)) } diff --git a/internal/proxy/task_scheduler.go b/internal/proxy/task_scheduler.go index fa4e00f963..5c5bdeb945 100644 --- a/internal/proxy/task_scheduler.go +++ b/internal/proxy/task_scheduler.go @@ -23,16 +23,10 @@ import ( "fmt" "sync" - "github.com/milvus-io/milvus/internal/util/typeutil" - - "github.com/milvus-io/milvus/internal/util/funcutil" - "go.uber.org/zap" "github.com/milvus-io/milvus/internal/log" "github.com/milvus-io/milvus/internal/mq/msgstream" - "github.com/milvus-io/milvus/internal/proto/commonpb" - "github.com/milvus-io/milvus/internal/proto/internalpb" "github.com/milvus-io/milvus/internal/util/trace" "github.com/opentracing/opentracing-go" oplog "github.com/opentracing/opentracing-go/log" @@ -385,25 +379,10 @@ type taskScheduler struct { cancel context.CancelFunc msFactory msgstream.Factory - - searchResultCh chan *internalpb.SearchResults - retrieveResultCh chan *internalpb.RetrieveResults } type schedOpt func(*taskScheduler) -func schedOptWithSearchResultCh(ch chan *internalpb.SearchResults) schedOpt { - return func(sched *taskScheduler) { - sched.searchResultCh = ch - } -} - -func schedOptWithRetrieveResultCh(ch chan *internalpb.RetrieveResults) schedOpt { - return func(sched *taskScheduler) { - sched.retrieveResultCh = ch - } -} - func newTaskScheduler(ctx context.Context, idAllocatorIns idAllocatorInterface, tsoAllocatorIns tsoAllocator, @@ -551,265 +530,6 @@ func (sched *taskScheduler) queryLoop() { } } -type resultBufHeader struct { - msgID UniqueID - usedVChans map[interface{}]struct{} // set of vChan - receivedVChansSet map[interface{}]struct{} // set of vChan - receivedSealedSegmentIDsSet map[interface{}]struct{} // set of UniqueID - receivedGlobalSegmentIDsSet map[interface{}]struct{} // set of UniqueID - haveError bool -} - -type searchResultBuf struct { - resultBufHeader - resultBuf []*internalpb.SearchResults -} - -type queryResultBuf struct { - resultBufHeader - resultBuf []*internalpb.RetrieveResults -} - -func newSearchResultBuf(msgID UniqueID) *searchResultBuf { - return &searchResultBuf{ - resultBufHeader: resultBufHeader{ - usedVChans: make(map[interface{}]struct{}), - receivedVChansSet: make(map[interface{}]struct{}), - receivedSealedSegmentIDsSet: make(map[interface{}]struct{}), - receivedGlobalSegmentIDsSet: make(map[interface{}]struct{}), - haveError: false, - msgID: msgID, - }, - resultBuf: make([]*internalpb.SearchResults, 0), - } -} - -func newQueryResultBuf(msgID UniqueID) *queryResultBuf { - return &queryResultBuf{ - resultBufHeader: resultBufHeader{ - usedVChans: make(map[interface{}]struct{}), - receivedVChansSet: make(map[interface{}]struct{}), - receivedSealedSegmentIDsSet: make(map[interface{}]struct{}), - receivedGlobalSegmentIDsSet: make(map[interface{}]struct{}), - haveError: false, - msgID: msgID, - }, - resultBuf: make([]*internalpb.RetrieveResults, 0), - } -} - -func (sr *resultBufHeader) readyToReduce() bool { - if sr.haveError { - log.Debug("Proxy searchResultBuf readyToReduce", zap.Any("haveError", true)) - return true - } - - log.Debug("check if result buf is ready to reduce", - zap.String("role", typeutil.ProxyRole), - zap.Int64("MsgID", sr.msgID), - zap.Any("receivedVChansSet", funcutil.SetToSlice(sr.receivedVChansSet)), - zap.Any("usedVChans", funcutil.SetToSlice(sr.usedVChans)), - zap.Any("receivedSealedSegmentIDsSet", funcutil.SetToSlice(sr.receivedSealedSegmentIDsSet)), - zap.Any("receivedGlobalSegmentIDsSet", funcutil.SetToSlice(sr.receivedGlobalSegmentIDsSet))) - - ret1 := funcutil.SetContain(sr.receivedVChansSet, sr.usedVChans) - if !ret1 { - return false - } - - return funcutil.SetContain(sr.receivedSealedSegmentIDsSet, sr.receivedGlobalSegmentIDsSet) -} - -func (sr *resultBufHeader) addPartialResult(vchans []vChan, searchSegIDs, globalSegIDs []UniqueID) { - - for _, vchan := range vchans { - sr.receivedVChansSet[vchan] = struct{}{} - } - - for _, sealedSegment := range searchSegIDs { - sr.receivedSealedSegmentIDsSet[sealedSegment] = struct{}{} - } - - for _, globalSegment := range globalSegIDs { - sr.receivedGlobalSegmentIDsSet[globalSegment] = struct{}{} - } -} - -func (sr *searchResultBuf) addPartialResult(result *internalpb.SearchResults) { - sr.resultBuf = append(sr.resultBuf, result) - if result.Status.ErrorCode != commonpb.ErrorCode_Success { - sr.haveError = true - return - } - sr.resultBufHeader.addPartialResult(result.ChannelIDsSearched, result.SealedSegmentIDsSearched, - result.GlobalSealedSegmentIDs) -} - -func (qr *queryResultBuf) addPartialResult(result *internalpb.RetrieveResults) { - qr.resultBuf = append(qr.resultBuf, result) - if result.Status.ErrorCode != commonpb.ErrorCode_Success { - qr.haveError = true - return - } - qr.resultBufHeader.addPartialResult(result.ChannelIDsRetrieved, result.SealedSegmentIDsRetrieved, - result.GlobalSealedSegmentIDs) -} - -func (sched *taskScheduler) collectionResultLoopV2() { - defer sched.wg.Done() - - searchResultBufs := make(map[UniqueID]*searchResultBuf) - searchResultBufFlags := newIDCache(Params.ProxyCfg.BufFlagExpireTime, Params.ProxyCfg.BufFlagCleanupInterval) // if value is true, we can ignore searchResult - queryResultBufs := make(map[UniqueID]*queryResultBuf) - queryResultBufFlags := newIDCache(Params.ProxyCfg.BufFlagExpireTime, Params.ProxyCfg.BufFlagCleanupInterval) // if value is true, we can ignore queryResult - - processSearchResult := func(results *internalpb.SearchResults) error { - reqID := results.Base.MsgID - - ignoreThisResult, ok := searchResultBufFlags.Get(reqID) - if !ok { - searchResultBufFlags.Set(reqID, false) - ignoreThisResult = false - } - if ignoreThisResult { - log.Debug("got a search result, but we should ignore", zap.String("role", typeutil.ProxyRole), zap.Int64("ReqID", reqID)) - return nil - } - - log.Debug("got a search result", zap.String("role", typeutil.ProxyRole), zap.Int64("ReqID", reqID)) - - t := sched.getTaskByReqID(reqID) - - if t == nil { - log.Debug("got a search result, but not in task scheduler", zap.String("role", typeutil.ProxyRole), zap.Int64("ReqID", reqID)) - delete(searchResultBufs, reqID) - searchResultBufFlags.Set(reqID, true) - } - - st, ok := t.(*searchTask) - if !ok { - log.Debug("got a search result, but the related task is not of search task", zap.String("role", typeutil.ProxyRole), zap.Int64("ReqID", reqID)) - delete(searchResultBufs, reqID) - searchResultBufFlags.Set(reqID, true) - return nil - } - - resultBuf, ok := searchResultBufs[reqID] - if !ok { - log.Debug("first receive search result of this task", zap.String("role", typeutil.ProxyRole), zap.Int64("reqID", reqID)) - resultBuf = newSearchResultBuf(reqID) - vchans, err := st.getVChannels() - if err != nil { - delete(searchResultBufs, reqID) - log.Warn("failed to get virtual channels", zap.String("role", typeutil.ProxyRole), zap.Error(err), zap.Int64("reqID", reqID)) - return err - } - for _, vchan := range vchans { - resultBuf.usedVChans[vchan] = struct{}{} - } - searchResultBufs[reqID] = resultBuf - } - resultBuf.addPartialResult(results) - - colName := t.(*searchTask).query.CollectionName - log.Debug("process search result", zap.String("role", typeutil.ProxyRole), zap.String("collection", colName), zap.Int64("reqID", reqID), zap.Int("answer cnt", len(searchResultBufs[reqID].resultBuf))) - - if resultBuf.readyToReduce() { - log.Debug("process search result, ready to reduce", zap.String("role", typeutil.ProxyRole), zap.Int64("reqID", reqID)) - searchResultBufFlags.Set(reqID, true) - st.resultBuf <- resultBuf.resultBuf - delete(searchResultBufs, reqID) - } - - return nil - } - - processRetrieveResult := func(results *internalpb.RetrieveResults) error { - reqID := results.Base.MsgID - - ignoreThisResult, ok := queryResultBufFlags.Get(reqID) - if !ok { - queryResultBufFlags.Set(reqID, false) - ignoreThisResult = false - } - if ignoreThisResult { - log.Debug("got a retrieve result, but we should ignore", zap.String("role", typeutil.ProxyRole), zap.Int64("ReqID", reqID)) - return nil - } - - log.Debug("got a retrieve result", zap.String("role", typeutil.ProxyRole), zap.Int64("ReqID", reqID)) - - t := sched.getTaskByReqID(reqID) - - if t == nil { - log.Debug("got a retrieve result, but not in task scheduler", zap.String("role", typeutil.ProxyRole), zap.Int64("ReqID", reqID)) - delete(queryResultBufs, reqID) - queryResultBufFlags.Set(reqID, true) - } - - st, ok := t.(*queryTask) - if !ok { - log.Debug("got a retrieve result, but the related task is not of retrieve task", zap.String("role", typeutil.ProxyRole), zap.Int64("ReqID", reqID)) - delete(queryResultBufs, reqID) - queryResultBufFlags.Set(reqID, true) - return nil - } - - resultBuf, ok := queryResultBufs[reqID] - if !ok { - log.Debug("first receive retrieve result of this task", zap.String("role", typeutil.ProxyRole), zap.Int64("reqID", reqID)) - resultBuf = newQueryResultBuf(reqID) - vchans, err := st.getVChannels() - if err != nil { - delete(queryResultBufs, reqID) - log.Warn("failed to get virtual channels", zap.String("role", typeutil.ProxyRole), zap.Error(err), zap.Int64("reqID", reqID)) - return err - } - for _, vchan := range vchans { - resultBuf.usedVChans[vchan] = struct{}{} - } - queryResultBufs[reqID] = resultBuf - } - resultBuf.addPartialResult(results) - - colName := t.(*queryTask).query.CollectionName - log.Debug("process retrieve result", zap.String("role", typeutil.ProxyRole), zap.String("collection", colName), zap.Int64("reqID", reqID), zap.Int("answer cnt", len(queryResultBufs[reqID].resultBuf))) - - if resultBuf.readyToReduce() { - log.Debug("process retrieve result, ready to reduce", zap.String("role", typeutil.ProxyRole), zap.Int64("reqID", reqID)) - queryResultBufFlags.Set(reqID, true) - st.resultBuf <- resultBuf.resultBuf - delete(queryResultBufs, reqID) - } - - return nil - } - - for { - select { - case <-sched.ctx.Done(): - log.Info("task scheduler's result loop of Proxy exit", zap.String("reason", "context done")) - return - case sr, ok := <-sched.searchResultCh: - if !ok { - log.Info("task scheduler's result loop of Proxy exit", zap.String("reason", "search result channel closed")) - return - } - if err := processSearchResult(sr); err != nil { - log.Warn("failed to process search result", zap.Error(err)) - } - case rr, ok := <-sched.retrieveResultCh: - if !ok { - log.Info("task scheduler's result loop of Proxy exit", zap.String("reason", "retrieve result channel closed")) - return - } - if err := processRetrieveResult(rr); err != nil { - log.Warn("failed to process retrieve result", zap.Error(err)) - } - } - } -} - func (sched *taskScheduler) Start() error { sched.wg.Add(1) go sched.definitionLoop() @@ -820,10 +540,6 @@ func (sched *taskScheduler) Start() error { sched.wg.Add(1) go sched.queryLoop() - sched.wg.Add(1) - // go sched.collectResultLoop() - go sched.collectionResultLoopV2() - return nil } diff --git a/internal/proxy/task_search.go b/internal/proxy/task_search.go index ed0803fe02..3683c9a9f0 100644 --- a/internal/proxy/task_search.go +++ b/internal/proxy/task_search.go @@ -6,14 +6,15 @@ import ( "fmt" "regexp" "strconv" + "sync" "time" "github.com/golang/protobuf/proto" "go.uber.org/zap" + "golang.org/x/sync/errgroup" "github.com/milvus-io/milvus/internal/log" "github.com/milvus-io/milvus/internal/metrics" - "github.com/milvus-io/milvus/internal/mq/msgstream" "github.com/milvus-io/milvus/internal/types" "github.com/milvus-io/milvus/internal/util/distance" @@ -34,94 +35,108 @@ import ( type searchTask struct { Condition *internalpb.SearchRequest - ctx context.Context - resultBuf chan []*internalpb.SearchResults + ctx context.Context + result *milvuspb.SearchResults - query *milvuspb.SearchRequest - chMgr channelsMgr + request *milvuspb.SearchRequest qc types.QueryCoord + tr *timerecord.TimeRecorder collectionName string - tr *timerecord.TimeRecorder - collectionID UniqueID + resultBuf chan *internalpb.SearchResults + toReduceResults []*internalpb.SearchResults + runningGroup *errgroup.Group + runningGroupCtx context.Context + + getQueryNodePolicy getQueryNodePolicy + searchShardPolicy pickShardPolicy } -func (st *searchTask) PreExecute(ctx context.Context) error { - sp, ctx := trace.StartSpanFromContextWithOperationName(st.TraceCtx(), "Proxy-Search-PreExecute") - defer sp.Finish() - st.Base.MsgType = commonpb.MsgType_Search - st.Base.SourceID = Params.ProxyCfg.ProxyID +func (t *searchTask) PreExecute(ctx context.Context) error { + sp, ctx := trace.StartSpanFromContextWithOperationName(t.TraceCtx(), "Proxy-Search-PreExecute") + + if t.getQueryNodePolicy == nil { + t.getQueryNodePolicy = defaultGetQueryNodePolicy + } + + if t.searchShardPolicy == nil { + t.searchShardPolicy = roundRobinPolicy + } + + defer sp.Finish() + t.Base.MsgType = commonpb.MsgType_Search + t.Base.SourceID = Params.ProxyCfg.ProxyID + + collectionName := t.request.CollectionName + if err := validateCollectionName(collectionName); err != nil { + return err + } - collectionName := st.query.CollectionName collID, err := globalMetaCache.GetCollectionID(ctx, collectionName) if err != nil { // err is not nil if collection not exists return err } - st.collectionID = collID + t.CollectionID = collID + t.collectionName = collectionName + t.PartitionIDs = []UniqueID{} - if err := validateCollectionName(st.query.CollectionName); err != nil { - return err - } - - for _, tag := range st.query.PartitionNames { + for _, tag := range t.request.PartitionNames { if err := validatePartitionTag(tag, false); err != nil { return err } } - // check if collection was already loaded into query node - showResp, err := st.qc.ShowCollections(st.ctx, &querypb.ShowCollectionsRequest{ - Base: &commonpb.MsgBase{ - MsgType: commonpb.MsgType_ShowCollections, - MsgID: st.Base.MsgID, - Timestamp: st.Base.Timestamp, - SourceID: Params.ProxyCfg.ProxyID, - }, - DbID: 0, // TODO(dragondriver) - }) + partitionsMap, err := globalMetaCache.GetPartitions(ctx, collectionName) if err != nil { return err } - if showResp.Status.ErrorCode != commonpb.ErrorCode_Success { - return errors.New(showResp.Status.Reason) - } - log.Debug("successfully get collections from QueryCoord", - zap.String("target collection name", collectionName), - zap.Int64("target collection ID", collID), - zap.Any("collections", showResp.CollectionIDs), - ) - collectionLoaded := false - for _, collectionID := range showResp.CollectionIDs { - if collectionID == collID { - collectionLoaded = true - break + partitionsRecord := make(map[UniqueID]bool) + for _, partitionName := range t.request.PartitionNames { + pattern := fmt.Sprintf("^%s$", partitionName) + re, err := regexp.Compile(pattern) + if err != nil { + return errors.New("invalid partition names") + } + found := false + for name, pID := range partitionsMap { + if re.MatchString(name) { + if _, exist := partitionsRecord[pID]; !exist { + t.PartitionIDs = append(t.PartitionIDs, pID) + partitionsRecord[pID] = true + } + found = true + } + } + if !found { + return fmt.Errorf("partition name %s not found", partitionName) } } - if !collectionLoaded { - return fmt.Errorf("collection %v was not loaded into memory", collectionName) + + // check if collection/partitions are loaded into query node + if !t.checkIfLoaded(collID, t.PartitionIDs) { + return fmt.Errorf("collection:%v or partitions:%v not loaded into memory", collectionName, t.request.GetPartitionNames()) } // TODO(dragondriver): necessary to check if partition was loaded into query node? - - st.Base.MsgType = commonpb.MsgType_Search + t.Base.MsgType = commonpb.MsgType_Search schema, _ := globalMetaCache.GetCollectionSchema(ctx, collectionName) - outputFields, err := translateOutputFields(st.query.OutputFields, schema, false) + outputFields, err := translateOutputFields(t.request.OutputFields, schema, false) if err != nil { return err } log.Debug("translate output fields", zap.Any("OutputFields", outputFields)) - st.query.OutputFields = outputFields + t.request.OutputFields = outputFields - if st.query.GetDslType() == commonpb.DslType_BoolExprV1 { - annsField, err := funcutil.GetAttrByKeyFromRepeatedKV(AnnsFieldKey, st.query.SearchParams) + if t.request.GetDslType() == commonpb.DslType_BoolExprV1 { + annsField, err := funcutil.GetAttrByKeyFromRepeatedKV(AnnsFieldKey, t.request.SearchParams) if err != nil { return errors.New(AnnsFieldKey + " not found in search_params") } - topKStr, err := funcutil.GetAttrByKeyFromRepeatedKV(TopKKey, st.query.SearchParams) + topKStr, err := funcutil.GetAttrByKeyFromRepeatedKV(TopKKey, t.request.SearchParams) if err != nil { return errors.New(TopKKey + " not found in search_params") } @@ -130,16 +145,16 @@ func (st *searchTask) PreExecute(ctx context.Context) error { return errors.New(TopKKey + " " + topKStr + " is not invalid") } - metricType, err := funcutil.GetAttrByKeyFromRepeatedKV(MetricTypeKey, st.query.SearchParams) + metricType, err := funcutil.GetAttrByKeyFromRepeatedKV(MetricTypeKey, t.request.SearchParams) if err != nil { return errors.New(MetricTypeKey + " not found in search_params") } - searchParams, err := funcutil.GetAttrByKeyFromRepeatedKV(SearchParamsKey, st.query.SearchParams) + searchParams, err := funcutil.GetAttrByKeyFromRepeatedKV(SearchParamsKey, t.request.SearchParams) if err != nil { return errors.New(SearchParamsKey + " not found in search_params") } - roundDecimalStr, err := funcutil.GetAttrByKeyFromRepeatedKV(RoundDecimalKey, st.query.SearchParams) + roundDecimalStr, err := funcutil.GetAttrByKeyFromRepeatedKV(RoundDecimalKey, t.request.SearchParams) if err != nil { roundDecimalStr = "-1" } @@ -161,22 +176,22 @@ func (st *searchTask) PreExecute(ctx context.Context) error { log.Debug("create query plan", //zap.Any("schema", schema), - zap.String("dsl", st.query.Dsl), + zap.String("dsl", t.request.Dsl), zap.String("anns field", annsField), zap.Any("query info", queryInfo)) - plan, err := createQueryPlan(schema, st.query.Dsl, annsField, queryInfo) + plan, err := createQueryPlan(schema, t.request.Dsl, annsField, queryInfo) if err != nil { log.Debug("failed to create query plan", zap.Error(err), //zap.Any("schema", schema), - zap.String("dsl", st.query.Dsl), + zap.String("dsl", t.request.Dsl), zap.String("anns field", annsField), zap.Any("query info", queryInfo)) return fmt.Errorf("failed to create query plan: %v", err) } - for _, name := range st.query.OutputFields { + for _, name := range t.request.OutputFields { hitField := false for _, field := range schema.Fields { if field.Name == name { @@ -184,7 +199,7 @@ func (st *searchTask) PreExecute(ctx context.Context) error { return errors.New("search doesn't support vector field as output_fields") } - st.SearchRequest.OutputFieldsId = append(st.SearchRequest.OutputFieldsId, field.FieldID) + t.SearchRequest.OutputFieldsId = append(t.SearchRequest.OutputFieldsId, field.FieldID) plan.OutputFieldIds = append(plan.OutputFieldIds, field.FieldID) hitField = true break @@ -196,238 +211,315 @@ func (st *searchTask) PreExecute(ctx context.Context) error { } } - st.SearchRequest.DslType = commonpb.DslType_BoolExprV1 - st.SearchRequest.SerializedExprPlan, err = proto.Marshal(plan) + t.SearchRequest.DslType = commonpb.DslType_BoolExprV1 + t.SearchRequest.SerializedExprPlan, err = proto.Marshal(plan) if err != nil { return err } log.Debug("Proxy::searchTask::PreExecute", zap.Any("plan.OutputFieldIds", plan.OutputFieldIds), zap.Any("plan", plan.String())) } - travelTimestamp := st.query.TravelTimestamp + travelTimestamp := t.request.TravelTimestamp if travelTimestamp == 0 { - travelTimestamp = st.BeginTs() + travelTimestamp = t.BeginTs() } else { - durationSeconds := tsoutil.CalculateDuration(st.BeginTs(), travelTimestamp) / 1000 + durationSeconds := tsoutil.CalculateDuration(t.BeginTs(), travelTimestamp) / 1000 if durationSeconds > Params.CommonCfg.RetentionDuration { duration := time.Second * time.Duration(durationSeconds) return fmt.Errorf("only support to travel back to %s so far", duration.String()) } } - guaranteeTimestamp := st.query.GuaranteeTimestamp + guaranteeTimestamp := t.request.GuaranteeTimestamp if guaranteeTimestamp == 0 { - guaranteeTimestamp = st.BeginTs() + guaranteeTimestamp = t.BeginTs() } - st.SearchRequest.TravelTimestamp = travelTimestamp - st.SearchRequest.GuaranteeTimestamp = guaranteeTimestamp - deadline, ok := st.TraceCtx().Deadline() + t.TravelTimestamp = travelTimestamp + t.GuaranteeTimestamp = guaranteeTimestamp + deadline, ok := t.TraceCtx().Deadline() if ok { - st.SearchRequest.TimeoutTimestamp = tsoutil.ComposeTSByTime(deadline, 0) + t.SearchRequest.TimeoutTimestamp = tsoutil.ComposeTSByTime(deadline, 0) } - st.SearchRequest.ResultChannelID = Params.ProxyCfg.SearchResultChannelNames[0] - st.SearchRequest.DbID = 0 // todo - st.SearchRequest.CollectionID = collID - st.SearchRequest.PartitionIDs = make([]UniqueID, 0) - - partitionsMap, err := globalMetaCache.GetPartitions(ctx, collectionName) - if err != nil { - return err - } - - partitionsRecord := make(map[UniqueID]bool) - for _, partitionName := range st.query.PartitionNames { - pattern := fmt.Sprintf("^%s$", partitionName) - re, err := regexp.Compile(pattern) - if err != nil { - return errors.New("invalid partition names") - } - found := false - for name, pID := range partitionsMap { - if re.MatchString(name) { - if _, exist := partitionsRecord[pID]; !exist { - st.PartitionIDs = append(st.PartitionIDs, pID) - partitionsRecord[pID] = true - } - found = true - } - } - if !found { - errMsg := fmt.Sprintf("PartitonName: %s not found", partitionName) - return errors.New(errMsg) - } - } - - st.SearchRequest.Dsl = st.query.Dsl - st.SearchRequest.PlaceholderGroup = st.query.PlaceholderGroup + t.DbID = 0 // todo + t.SearchRequest.Dsl = t.request.Dsl + t.SearchRequest.PlaceholderGroup = t.request.PlaceholderGroup + log.Info("search PreExecute done.", + zap.Any("requestID", t.Base.MsgID), zap.Any("requestType", "search")) return nil } -func (st *searchTask) Execute(ctx context.Context) error { - sp, ctx := trace.StartSpanFromContextWithOperationName(st.TraceCtx(), "Proxy-Search-Execute") +func (t *searchTask) Execute(ctx context.Context) error { + sp, ctx := trace.StartSpanFromContextWithOperationName(t.TraceCtx(), "Proxy-Search-Execute") defer sp.Finish() - tr := timerecord.NewTimeRecorder(fmt.Sprintf("proxy execute search %d", st.ID())) + tr := timerecord.NewTimeRecorder(fmt.Sprintf("proxy execute search %d", t.ID())) defer tr.Elapse("done") - var tsMsg msgstream.TsMsg = &msgstream.SearchMsg{ - SearchRequest: *st.SearchRequest, - BaseMsg: msgstream.BaseMsg{ - Ctx: ctx, - HashValues: []uint32{uint32(Params.ProxyCfg.ProxyID)}, - BeginTimestamp: st.Base.Timestamp, - EndTimestamp: st.Base.Timestamp, - }, - } - msgPack := msgstream.MsgPack{ - BeginTs: st.Base.Timestamp, - EndTs: st.Base.Timestamp, - Msgs: make([]msgstream.TsMsg, 1), - } - msgPack.Msgs[0] = tsMsg + executeSearch := func(withCache bool) error { + shards, err := globalMetaCache.GetShards(ctx, withCache, t.collectionName, t.qc) + if err != nil { + return err + } - collectionName := st.query.CollectionName - info, err := globalMetaCache.GetCollectionInfo(ctx, collectionName) - if err != nil { // err is not nil if collection not exists + t.resultBuf = make(chan *internalpb.SearchResults, len(shards)) + t.toReduceResults = make([]*internalpb.SearchResults, 0, len(shards)) + t.runningGroup, t.runningGroupCtx = errgroup.WithContext(ctx) + + // TODO: try to merge rpc send to different shard leaders. + // If two shard leader is on the same querynode maybe we should merge request to save rpc + for _, shard := range shards { + s := shard + t.runningGroup.Go(func() error { + log.Debug("proxy starting to query one shard", + zap.Int64("collectionID", t.CollectionID), + zap.String("collection name", t.collectionName), + zap.String("shard channel", s.GetChannelName()), + zap.Uint64("timeoutTs", t.TimeoutTimestamp)) + + err := t.searchShard(t.runningGroupCtx, s) + if err != nil { + return err + } + return nil + }) + } + + err = t.runningGroup.Wait() return err } - st.collectionName = info.schema.Name - stream, err := st.chMgr.getDQLStream(info.collID) - if err != nil { - err = st.chMgr.createDQLStream(info.collID) - if err != nil { - st.result = &milvuspb.SearchResults{ - Status: &commonpb.Status{ - ErrorCode: commonpb.ErrorCode_UnexpectedError, - Reason: err.Error(), - }, - } - return err - } - stream, err = st.chMgr.getDQLStream(info.collID) - if err != nil { - st.result = &milvuspb.SearchResults{ - Status: &commonpb.Status{ - ErrorCode: commonpb.ErrorCode_UnexpectedError, - Reason: err.Error(), - }, - } - return err - } + err := executeSearch(WithCache) + if err == errInvalidShardLeaders { + log.Warn("invalid shard leaders from cache, updating shardleader caches and retry search") + return executeSearch(WithoutCache) } - tr.Record("get used message stream") - err = stream.Produce(&msgPack) if err != nil { - log.Debug("proxy", zap.String("send search request failed", err.Error())) + return err } - st.tr.Record("send message done") - log.Debug("proxy sent one searchMsg", - zap.Int64("collectionID", st.CollectionID), - zap.Int64("msgID", tsMsg.ID()), - zap.Int("length of search msg", len(msgPack.Msgs)), - zap.Uint64("timeoutTs", st.SearchRequest.TimeoutTimestamp)) - sendMsgDur := tr.Record("send search msg to message stream") - metrics.ProxySendMessageLatency.WithLabelValues(strconv.FormatInt(Params.ProxyCfg.ProxyID, 10), - metrics.SearchLabel).Observe(float64(sendMsgDur.Milliseconds())) - return err + log.Info("Search Execute done.", + zap.Any("requestID", t.Base.MsgID), zap.Any("requestType", "search")) + return nil } -func (st *searchTask) PostExecute(ctx context.Context) error { - sp, ctx := trace.StartSpanFromContextWithOperationName(st.TraceCtx(), "Proxy-Search-PostExecute") +func (t *searchTask) PostExecute(ctx context.Context) error { + sp, ctx := trace.StartSpanFromContextWithOperationName(t.TraceCtx(), "Proxy-Search-PostExecute") defer sp.Finish() tr := timerecord.NewTimeRecorder("searchTask PostExecute") defer func() { tr.Elapse("done") }() - for { - select { - case <-st.TraceCtx().Done(): - log.Debug("Proxy searchTask PostExecute Loop exit caused by ctx.Done", zap.Int64("taskID", st.ID())) - return fmt.Errorf("searchTask:wait to finish failed, timeout: %d", st.ID()) - case searchResults := <-st.resultBuf: - // fmt.Println("searchResults: ", searchResults) - filterSearchResults := make([]*internalpb.SearchResults, 0) - var filterReason string - errNum := 0 - for _, partialSearchResult := range searchResults { - if partialSearchResult.Status.ErrorCode == commonpb.ErrorCode_Success { - filterSearchResults = append(filterSearchResults, partialSearchResult) - // For debugging, please don't delete. - // printSearchResult(partialSearchResult) - } else { - errNum++ - filterReason += partialSearchResult.Status.Reason + "\n" + + wg := sync.WaitGroup{} + wg.Add(1) + go func() { + for { + select { + case <-t.TraceCtx().Done(): + log.Debug("wait to finish timeout!", zap.Int64("taskID", t.ID())) + return + case <-t.runningGroupCtx.Done(): + log.Debug("all searches are finished or canceled", zap.Any("taskID", t.ID())) + close(t.resultBuf) + for res := range t.resultBuf { + t.toReduceResults = append(t.toReduceResults, res) + log.Debug("proxy receives one query result", zap.Int64("sourceID", res.GetBase().GetSourceID()), zap.Any("taskID", t.ID())) + } + wg.Done() + return + } + } + }() + + wg.Wait() + tr.Record("decodeResultStart") + validSearchResults, err := decodeSearchResults(t.toReduceResults) + if err != nil { + return err + } + metrics.ProxyDecodeSearchResultLatency.WithLabelValues(strconv.FormatInt(Params.ProxyCfg.ProxyID, 10), metrics.SearchLabel).Observe(float64(tr.RecordSpan().Milliseconds())) + log.Debug("proxy search post execute stage 2", zap.Any("len(validSearchResults)", len(validSearchResults))) + if len(validSearchResults) <= 0 { + log.Warn("search result is empty", zap.Any("requestID", t.Base.MsgID), zap.String("requestType", "search")) + + t.result = &milvuspb.SearchResults{ + Status: &commonpb.Status{ + ErrorCode: commonpb.ErrorCode_Success, + Reason: "search result is empty", + }, + CollectionName: t.collectionName, + } + // add information if any + if len(t.toReduceResults) > 0 { + t.result.Results = &schemapb.SearchResultData{ + NumQueries: t.toReduceResults[0].NumQueries, + Topks: make([]int64, t.toReduceResults[0].NumQueries), + } + } + return nil + } + + tr.Record("reduceResultStart") + t.result, err = reduceSearchResultData(validSearchResults, t.toReduceResults[0].NumQueries, t.toReduceResults[0].TopK, t.toReduceResults[0].MetricType) + if err != nil { + return err + } + metrics.ProxyReduceSearchResultLatency.WithLabelValues(strconv.FormatInt(Params.ProxyCfg.ProxyID, 10), metrics.SuccessLabel).Observe(float64(tr.RecordSpan().Milliseconds())) + t.result.CollectionName = t.collectionName + + schema, err := globalMetaCache.GetCollectionSchema(ctx, t.request.CollectionName) + if err != nil { + return err + } + if len(t.request.OutputFields) != 0 && len(t.result.Results.FieldsData) != 0 { + for k, fieldName := range t.request.OutputFields { + for _, field := range schema.Fields { + if t.result.Results.FieldsData[k] != nil && field.Name == fieldName { + t.result.Results.FieldsData[k].FieldName = field.Name + t.result.Results.FieldsData[k].FieldId = field.FieldID + t.result.Results.FieldsData[k].Type = field.DataType } } - - log.Debug("Proxy Search PostExecute stage1", - zap.Any("len(filterSearchResults)", len(filterSearchResults))) - metrics.ProxyWaitForSearchResultLatency.WithLabelValues(strconv.FormatInt(Params.ProxyCfg.ProxyID, 10), metrics.SearchLabel).Observe(float64(st.tr.RecordSpan().Milliseconds())) - tr.Record("Proxy Search PostExecute stage1 done") - if len(filterSearchResults) <= 0 || errNum > 0 { - st.result = &milvuspb.SearchResults{ - Status: &commonpb.Status{ - ErrorCode: commonpb.ErrorCode_UnexpectedError, - Reason: filterReason, - }, - CollectionName: st.collectionName, - } - return fmt.Errorf("QueryNode search fail, reason %s: id %d", filterReason, st.ID()) - } - tr.Record("decodeResultStart") - validSearchResults, err := decodeSearchResults(filterSearchResults) - if err != nil { - return err - } - metrics.ProxyDecodeSearchResultLatency.WithLabelValues(strconv.FormatInt(Params.ProxyCfg.ProxyID, 10), metrics.SearchLabel).Observe(float64(tr.RecordSpan().Milliseconds())) - log.Debug("Proxy Search PostExecute stage2", zap.Any("len(validSearchResults)", len(validSearchResults))) - if len(validSearchResults) <= 0 { - filterReason += "empty search result\n" - log.Debug("Proxy Search PostExecute stage2 failed", zap.Any("filterReason", filterReason)) - - st.result = &milvuspb.SearchResults{ - Status: &commonpb.Status{ - ErrorCode: commonpb.ErrorCode_Success, - Reason: filterReason, - }, - Results: &schemapb.SearchResultData{ - NumQueries: searchResults[0].NumQueries, - Topks: make([]int64, searchResults[0].NumQueries), - }, - CollectionName: st.collectionName, - } - return nil - } - - tr.Record("reduceResultStart") - st.result, err = reduceSearchResultData(validSearchResults, searchResults[0].NumQueries, searchResults[0].TopK, searchResults[0].MetricType) - if err != nil { - return err - } - metrics.ProxyReduceSearchResultLatency.WithLabelValues(strconv.FormatInt(Params.ProxyCfg.ProxyID, 10), metrics.SuccessLabel).Observe(float64(tr.RecordSpan().Milliseconds())) - st.result.CollectionName = st.collectionName - - schema, err := globalMetaCache.GetCollectionSchema(ctx, st.query.CollectionName) - if err != nil { - return err - } - if len(st.query.OutputFields) != 0 && len(st.result.Results.FieldsData) != 0 { - for k, fieldName := range st.query.OutputFields { - for _, field := range schema.Fields { - if st.result.Results.FieldsData[k] != nil && field.Name == fieldName { - st.result.Results.FieldsData[k].FieldName = field.Name - st.result.Results.FieldsData[k].FieldId = field.FieldID - st.result.Results.FieldsData[k].Type = field.DataType - } - } - } - } - return nil } } + log.Info("Search post execute done", zap.Any("requestID", t.Base.MsgID), zap.String("requestType", "search")) + return nil +} + +func (t *searchTask) searchShard(ctx context.Context, leaders *querypb.ShardLeadersList) error { + + search := func(nodeID UniqueID, qn types.QueryNode) error { + req := &querypb.SearchRequest{ + Req: t.SearchRequest, + DmlChannel: leaders.GetChannelName(), + } + + result, err := qn.Search(ctx, req) + if err != nil { + log.Warn("QueryNode search returns error", zap.Int64("nodeID", nodeID), + zap.Error(err)) + return errInvalidShardLeaders + } + if result.GetStatus().GetErrorCode() != commonpb.ErrorCode_Success { + log.Warn("QueryNode search result error", zap.Int64("nodeID", nodeID), + zap.String("reason", result.GetStatus().GetReason())) + return fmt.Errorf("fail to Search, QueryNode ID=%d, reason=%s", nodeID, result.GetStatus().GetReason()) + } + + t.resultBuf <- result + return nil + } + + err := t.searchShardPolicy(t.TraceCtx(), t.getQueryNodePolicy, search, leaders) + if err != nil { + log.Warn("fail to search to all shard leaders", zap.Any("shard leaders", leaders.GetNodeIds())) + return err + } + + return nil +} + +func (t *searchTask) checkIfLoaded(collectionID UniqueID, searchPartitionIDs []UniqueID) bool { + // If request to search partitions + if len(searchPartitionIDs) > 0 { + resp, err := t.qc.ShowPartitions(t.ctx, &querypb.ShowPartitionsRequest{ + Base: &commonpb.MsgBase{ + MsgType: commonpb.MsgType_ShowCollections, + MsgID: t.Base.MsgID, + Timestamp: t.Base.Timestamp, + SourceID: Params.ProxyCfg.ProxyID, + }, + CollectionID: collectionID, + PartitionIDs: searchPartitionIDs, + }) + if err != nil { + log.Warn("fail to show partitions by QueryCoord", + zap.Int64("requestID", t.Base.MsgID), + zap.Int64("collectionID", collectionID), + zap.Int64s("partitionIDs", searchPartitionIDs), + zap.String("requestType", "search"), + zap.Error(err)) + return false + } + + if resp.Status.ErrorCode != commonpb.ErrorCode_Success { + log.Warn("fail to show partitions by QueryCoord", + zap.Int64("collectionID", collectionID), + zap.Int64s("partitionIDs", searchPartitionIDs), + zap.Int64("requestID", t.Base.MsgID), zap.String("requestType", "search"), + zap.String("reason", resp.GetStatus().GetReason())) + return false + } + // Current logic: show partitions won't return error if the given partitions are all loaded + return true + } + + // If request to search collection + resp, err := t.qc.ShowCollections(t.ctx, &querypb.ShowCollectionsRequest{ + Base: &commonpb.MsgBase{ + MsgType: commonpb.MsgType_ShowCollections, + MsgID: t.Base.MsgID, + Timestamp: t.Base.Timestamp, + SourceID: Params.ProxyCfg.ProxyID, + }, + }) + if err != nil { + log.Warn("fail to show collections by QueryCoord", + zap.Int64("requestID", t.Base.MsgID), zap.String("requestType", "search"), + zap.Error(err)) + return false + } + + if resp.Status.ErrorCode != commonpb.ErrorCode_Success { + log.Warn("fail to show collections by QueryCoord", + zap.Int64("requestID", t.Base.MsgID), zap.String("requestType", "search"), + zap.String("reason", resp.GetStatus().GetReason())) + return false + } + + loaded := false + for index, collID := range resp.CollectionIDs { + if collID == collectionID && resp.GetInMemoryPercentages()[index] >= int64(100) { + loaded = true + break + } + } + + if !loaded { + resp, err := t.qc.ShowPartitions(t.ctx, &querypb.ShowPartitionsRequest{ + Base: &commonpb.MsgBase{ + MsgType: commonpb.MsgType_ShowCollections, + MsgID: t.Base.MsgID, + Timestamp: t.Base.Timestamp, + SourceID: Params.ProxyCfg.ProxyID, + }, + CollectionID: collectionID, + }) + if err != nil { + log.Warn("fail to show partitions by QueryCoord", + zap.Int64("requestID", t.Base.MsgID), + zap.Int64("collectionID", collectionID), + zap.String("requestType", "search"), + zap.Error(err)) + return false + } + + if resp.Status.ErrorCode != commonpb.ErrorCode_Success { + log.Warn("fail to show partitions by QueryCoord", + zap.Int64("collectionID", collectionID), + zap.Int64("requestID", t.Base.MsgID), zap.String("requestType", "search"), + zap.String("reason", resp.GetStatus().GetReason())) + return false + } + + if len(resp.GetPartitionIDs()) > 0 { + log.Warn("collection not fully loaded, search on these partitions", zap.Int64s("partitionIDs", resp.GetPartitionIDs())) + return true + } + } + + return loaded } func decodeSearchResults(searchResults []*internalpb.SearchResults) ([]*schemapb.SearchResultData, error) { @@ -457,18 +549,22 @@ func checkSearchResultData(data *schemapb.SearchResultData, nq int64, topk int64 if data.TopK != topk { return fmt.Errorf("search result's topk(%d) mis-match with %d", data.TopK, topk) } - if len(data.Ids.GetIntId().Data) != (int)(nq*topk) { - return fmt.Errorf("search result's id length %d invalid", len(data.Ids.GetIntId().Data)) + + expectedLength := (int)(nq * topk) + if len(data.Ids.GetIntId().Data) != expectedLength { + return fmt.Errorf("search result's ID length invalid, ID length=%d, expectd length=%d", + len(data.Ids.GetIntId().Data), expectedLength) } - if len(data.Scores) != (int)(nq*topk) { - return fmt.Errorf("search result's score length %d invalid", len(data.Scores)) + if len(data.Scores) != expectedLength { + return fmt.Errorf("search result's score length invalid, score length=%d, expectedLength=%d", + len(data.Scores), expectedLength) } return nil } func selectSearchResultData(dataArray []*schemapb.SearchResultData, offsets []int64, topk int64, qi int64) int { sel := -1 - maxDistance := minFloat32 + maxDistance := minFloat32 // distance here means score :) for i, offset := range offsets { // query num, the number of ways to merge if offset >= topk { continue @@ -518,11 +614,12 @@ func reduceSearchResultData(searchResultData []*schemapb.SearchResultData, nq in for i, sData := range searchResultData { log.Debug("reduceSearchResultData", - zap.Int("i", i), + zap.Int("result No.", i), zap.Int64("nq", sData.NumQueries), zap.Int64("topk", sData.TopK), zap.Any("len(FieldsData)", len(sData.FieldsData))) if err := checkSearchResultData(sData, nq, topk); err != nil { + log.Warn("invalid search results", zap.Error(err)) return ret, err } //printSearchResultData(sData, strconv.FormatInt(int64(i), 10)) @@ -577,7 +674,7 @@ func reduceSearchResultData(searchResultData []*schemapb.SearchResultData, nq in ret.Results.Scores[k] *= -1 } } - + // printSearchResultData(ret.Results, "proxy reduce result") return ret, nil } @@ -605,79 +702,79 @@ func reduceSearchResultData(searchResultData []*schemapb.SearchResultData, nq in // } // } -func (st *searchTask) TraceCtx() context.Context { - return st.ctx +func (t *searchTask) TraceCtx() context.Context { + return t.ctx } -func (st *searchTask) ID() UniqueID { - return st.Base.MsgID +func (t *searchTask) ID() UniqueID { + return t.Base.MsgID } -func (st *searchTask) SetID(uid UniqueID) { - st.Base.MsgID = uid +func (t *searchTask) SetID(uid UniqueID) { + t.Base.MsgID = uid } -func (st *searchTask) Name() string { +func (t *searchTask) Name() string { return SearchTaskName } -func (st *searchTask) Type() commonpb.MsgType { - return st.Base.MsgType +func (t *searchTask) Type() commonpb.MsgType { + return t.Base.MsgType } -func (st *searchTask) BeginTs() Timestamp { - return st.Base.Timestamp +func (t *searchTask) BeginTs() Timestamp { + return t.Base.Timestamp } -func (st *searchTask) EndTs() Timestamp { - return st.Base.Timestamp +func (t *searchTask) EndTs() Timestamp { + return t.Base.Timestamp } -func (st *searchTask) SetTs(ts Timestamp) { - st.Base.Timestamp = ts +func (t *searchTask) SetTs(ts Timestamp) { + t.Base.Timestamp = ts } -func (st *searchTask) OnEnqueue() error { - st.Base = &commonpb.MsgBase{} - st.Base.MsgType = commonpb.MsgType_Search - st.Base.SourceID = Params.ProxyCfg.ProxyID +func (t *searchTask) OnEnqueue() error { + t.Base = &commonpb.MsgBase{} + t.Base.MsgType = commonpb.MsgType_Search + t.Base.SourceID = Params.ProxyCfg.ProxyID return nil } -func (st *searchTask) getChannels() ([]pChan, error) { - collID, err := globalMetaCache.GetCollectionID(st.ctx, st.query.CollectionName) - if err != nil { - return nil, err - } +// func (t *searchTaskV2) getChannels() ([]pChan, error) { +// collID, err := globalMetaCache.GetCollectionID(t.ctx, t.request.CollectionName) +// if err != nil { +// return nil, err +// } +// +// var channels []pChan +// channels, err = t.chMgr.getChannels(collID) +// if err != nil { +// err := t.chMgr.createDMLMsgStream(collID) +// if err != nil { +// return nil, err +// } +// return t.chMgr.getChannels(collID) +// } +// +// return channels, nil +// } - var channels []pChan - channels, err = st.chMgr.getChannels(collID) - if err != nil { - err := st.chMgr.createDMLMsgStream(collID) - if err != nil { - return nil, err - } - return st.chMgr.getChannels(collID) - } - - return channels, nil -} - -func (st *searchTask) getVChannels() ([]vChan, error) { - collID, err := globalMetaCache.GetCollectionID(st.ctx, st.query.CollectionName) - if err != nil { - return nil, err - } - - var channels []vChan - channels, err = st.chMgr.getVChannels(collID) - if err != nil { - err := st.chMgr.createDMLMsgStream(collID) - if err != nil { - return nil, err - } - return st.chMgr.getVChannels(collID) - } - - return channels, nil -} +// func (t *searchTaskV2) getVChannels() ([]vChan, error) { +// collID, err := globalMetaCache.GetCollectionID(t.ctx, t.request.CollectionName) +// if err != nil { +// return nil, err +// } +// +// var channels []vChan +// channels, err = t.chMgr.getVChannels(collID) +// if err != nil { +// err := t.chMgr.createDMLMsgStream(collID) +// if err != nil { +// return nil, err +// } +// return t.chMgr.getVChannels(collID) +// } +// +// return channels, nil +// } diff --git a/internal/proxy/task_search_test.go b/internal/proxy/task_search_test.go index a5f5688a31..cd93c33226 100644 --- a/internal/proxy/task_search_test.go +++ b/internal/proxy/task_search_test.go @@ -3,18 +3,14 @@ package proxy import ( "context" "errors" - "fmt" - "strconv" - "sync" "testing" "time" "github.com/golang/protobuf/proto" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" - "github.com/milvus-io/milvus/internal/common" - "github.com/milvus-io/milvus/internal/log" - "github.com/milvus-io/milvus/internal/mq/msgstream" + "github.com/milvus-io/milvus/internal/types" "github.com/milvus-io/milvus/internal/proto/commonpb" "github.com/milvus-io/milvus/internal/proto/internalpb" @@ -26,597 +22,416 @@ import ( "github.com/milvus-io/milvus/internal/util/funcutil" "github.com/milvus-io/milvus/internal/util/timerecord" "github.com/milvus-io/milvus/internal/util/typeutil" - "github.com/milvus-io/milvus/internal/util/uniquegenerator" ) -func TestSearchTask(t *testing.T) { - ctx := context.Background() - ctxCancel, cancel := context.WithCancel(ctx) - qt := &searchTask{ - ctx: ctxCancel, - Condition: NewTaskCondition(context.TODO()), - SearchRequest: &internalpb.SearchRequest{ - Base: &commonpb.MsgBase{ - MsgType: commonpb.MsgType_Search, - SourceID: Params.ProxyCfg.ProxyID, +const ( + testShardsNum = int32(2) +) + +func TestSearchTask_PostExecute(t *testing.T) { + t.Run("Test empty result", func(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + qt := &searchTask{ + ctx: ctx, + Condition: NewTaskCondition(context.TODO()), + SearchRequest: &internalpb.SearchRequest{ + Base: &commonpb.MsgBase{ + MsgType: commonpb.MsgType_Search, + SourceID: Params.ProxyCfg.ProxyID, + }, }, - ResultChannelID: strconv.FormatInt(Params.ProxyCfg.ProxyID, 10), - }, - resultBuf: make(chan []*internalpb.SearchResults), - query: nil, - chMgr: nil, - qc: nil, - tr: timerecord.NewTimeRecorder("search"), - } + request: nil, + qc: nil, + tr: timerecord.NewTimeRecorder("search"), - // no result - go func() { - qt.resultBuf <- []*internalpb.SearchResults{} - }() - err := qt.PostExecute(context.TODO()) - assert.NotNil(t, err) - - // test trace context done - cancel() - err = qt.PostExecute(context.TODO()) - assert.NotNil(t, err) - - // error result - ctx = context.Background() - qt = &searchTask{ - ctx: ctx, - Condition: NewTaskCondition(context.TODO()), - SearchRequest: &internalpb.SearchRequest{ - Base: &commonpb.MsgBase{ - MsgType: commonpb.MsgType_Search, - SourceID: Params.ProxyCfg.ProxyID, - }, - ResultChannelID: strconv.FormatInt(Params.ProxyCfg.ProxyID, 10), - }, - resultBuf: make(chan []*internalpb.SearchResults), - query: nil, - chMgr: nil, - qc: nil, - tr: timerecord.NewTimeRecorder("search"), - } - - // no result - go func() { - result := internalpb.SearchResults{ - Status: &commonpb.Status{ - ErrorCode: commonpb.ErrorCode_UnexpectedError, - Reason: "test", - }, + resultBuf: make(chan *internalpb.SearchResults, 10), + toReduceResults: make([]*internalpb.SearchResults, 0), } - results := make([]*internalpb.SearchResults, 1) - results[0] = &result - qt.resultBuf <- results - }() - err = qt.PostExecute(context.TODO()) - assert.NotNil(t, err) + // no result + qt.resultBuf <- &internalpb.SearchResults{} - log.Debug("PostExecute failed" + err.Error()) - // check result SlicedBlob + mockctx, mockcancel := context.WithCancel(ctx) + qt.runningGroupCtx = mockctx + mockcancel() - ctx = context.Background() - qt = &searchTask{ - ctx: ctx, - Condition: NewTaskCondition(context.TODO()), - SearchRequest: &internalpb.SearchRequest{ - Base: &commonpb.MsgBase{ - MsgType: commonpb.MsgType_Search, - SourceID: Params.ProxyCfg.ProxyID, - }, - ResultChannelID: strconv.FormatInt(Params.ProxyCfg.ProxyID, 10), - }, - resultBuf: make(chan []*internalpb.SearchResults), - query: nil, - chMgr: nil, - qc: nil, - tr: timerecord.NewTimeRecorder("search"), - } - - // no result - go func() { - result := internalpb.SearchResults{ - Status: &commonpb.Status{ - ErrorCode: commonpb.ErrorCode_Success, - Reason: "test", - }, - SlicedBlob: nil, - } - results := make([]*internalpb.SearchResults, 1) - results[0] = &result - qt.resultBuf <- results - }() - err = qt.PostExecute(context.TODO()) - assert.Nil(t, err) - - assert.Equal(t, qt.result.Status.ErrorCode, commonpb.ErrorCode_Success) - - // TODO, add decode result, reduce result test + err := qt.PostExecute(context.TODO()) + assert.NoError(t, err) + assert.Equal(t, qt.result.Status.ErrorCode, commonpb.ErrorCode_Success) + }) } -func TestSearchTask_Channels(t *testing.T) { - var err error - - Params.Init() - - rc := NewRootCoordMock() - rc.Start() - defer rc.Stop() - - ctx := context.Background() - - err = InitMetaCache(rc) - assert.NoError(t, err) - - dmlChannelsFunc := getDmlChannelsFunc(ctx, rc) - query := newMockGetChannelsService() - factory := newSimpleMockMsgStreamFactory() - chMgr := newChannelsMgrImpl(dmlChannelsFunc, nil, query.GetChannels, nil, factory) - defer chMgr.removeAllDMLStream() - defer chMgr.removeAllDQLStream() - - prefix := "TestSearchTask_Channels" - collectionName := prefix + funcutil.GenRandomStr() - shardsNum := int32(2) - dbName := "" - int64Field := "int64" - floatVecField := "fvec" - dim := 128 - - task := &searchTask{ - ctx: ctx, - query: &milvuspb.SearchRequest{ - CollectionName: collectionName, - }, - chMgr: chMgr, - tr: timerecord.NewTimeRecorder("search"), - } - - // collection not exist - _, err = task.getVChannels() - assert.Error(t, err) - _, err = task.getVChannels() - assert.Error(t, err) - - schema := constructCollectionSchema(int64Field, floatVecField, dim, collectionName) +func createColl(t *testing.T, name string, rc types.RootCoord) { + schema := constructCollectionSchema(testInt64Field, testFloatVecField, testVecDim, name) marshaledSchema, err := proto.Marshal(schema) - assert.NoError(t, err) + require.NoError(t, err) + ctx := context.TODO() createColT := &createCollectionTask{ - Condition: NewTaskCondition(ctx), + Condition: NewTaskCondition(context.TODO()), CreateCollectionRequest: &milvuspb.CreateCollectionRequest{ - Base: nil, - DbName: dbName, - CollectionName: collectionName, + CollectionName: name, Schema: marshaledSchema, - ShardsNum: shardsNum, + ShardsNum: testShardsNum, }, ctx: ctx, rootCoord: rc, - result: nil, - schema: nil, } - assert.NoError(t, createColT.OnEnqueue()) - assert.NoError(t, createColT.PreExecute(ctx)) - assert.NoError(t, createColT.Execute(ctx)) - assert.NoError(t, createColT.PostExecute(ctx)) + require.NoError(t, createColT.OnEnqueue()) + require.NoError(t, createColT.PreExecute(ctx)) + require.NoError(t, createColT.Execute(ctx)) + require.NoError(t, createColT.PostExecute(ctx)) +} - _, err = task.getChannels() - assert.NoError(t, err) - _, err = task.getVChannels() - assert.NoError(t, err) - - _ = chMgr.removeAllDMLStream() - chMgr.dmlChannelsMgr.getChannelsFunc = func(collectionID UniqueID) (map[vChan]pChan, error) { - return nil, errors.New("mock") - } - _, err = task.getChannels() - assert.Error(t, err) - _, err = task.getVChannels() - assert.Error(t, err) +func getValidSearchParams() []*commonpb.KeyValuePair { + return []*commonpb.KeyValuePair{ + { + Key: AnnsFieldKey, + Value: testFloatVecField, + }, + { + Key: TopKKey, + Value: "10", + }, + { + Key: MetricTypeKey, + Value: distance.L2, + }, + { + Key: SearchParamsKey, + Value: `{"nprobe": 10}`, + }, + { + Key: RoundDecimalKey, + Value: "-1", + }} } func TestSearchTask_PreExecute(t *testing.T) { var err error Params.Init() - Params.ProxyCfg.SearchResultChannelNames = []string{funcutil.GenRandomStr()} + var ( + rc = NewRootCoordMock() + qc = NewQueryCoordMock() + ctx = context.TODO() - rc := NewRootCoordMock() - rc.Start() + collectionName = t.Name() + funcutil.GenRandomStr() + ) + + err = rc.Start() defer rc.Stop() - - qc := NewQueryCoordMock() - qc.Start() - defer qc.Stop() - - ctx := context.Background() - + require.NoError(t, err) err = InitMetaCache(rc) - assert.NoError(t, err) + require.NoError(t, err) - dmlChannelsFunc := getDmlChannelsFunc(ctx, rc) - query := newMockGetChannelsService() - factory := newSimpleMockMsgStreamFactory() - chMgr := newChannelsMgrImpl(dmlChannelsFunc, nil, query.GetChannels, nil, factory) - defer chMgr.removeAllDMLStream() - defer chMgr.removeAllDQLStream() + err = qc.Start() + defer qc.Stop() + require.NoError(t, err) - prefix := "TestSearchTask_PreExecute" - collectionName := prefix + funcutil.GenRandomStr() - shardsNum := int32(2) - dbName := "" - int64Field := "int64" - floatVecField := "fvec" - dim := 128 - - task := &searchTask{ - ctx: ctx, - SearchRequest: &internalpb.SearchRequest{}, - query: &milvuspb.SearchRequest{ - CollectionName: collectionName, - }, - chMgr: chMgr, - qc: qc, - tr: timerecord.NewTimeRecorder("search"), - } - assert.NoError(t, task.OnEnqueue()) - - // collection not exist - assert.Error(t, task.PreExecute(ctx)) - - schema := constructCollectionSchema(int64Field, floatVecField, dim, collectionName) - marshaledSchema, err := proto.Marshal(schema) - assert.NoError(t, err) - - createColT := &createCollectionTask{ - Condition: NewTaskCondition(ctx), - CreateCollectionRequest: &milvuspb.CreateCollectionRequest{ - Base: nil, - DbName: dbName, - CollectionName: collectionName, - Schema: marshaledSchema, - ShardsNum: shardsNum, - }, - ctx: ctx, - rootCoord: rc, - result: nil, - schema: nil, - } - - assert.NoError(t, createColT.OnEnqueue()) - assert.NoError(t, createColT.PreExecute(ctx)) - assert.NoError(t, createColT.Execute(ctx)) - assert.NoError(t, createColT.PostExecute(ctx)) - - collectionID, _ := globalMetaCache.GetCollectionID(ctx, collectionName) - - // validateCollectionName - task.query.CollectionName = "$" - assert.Error(t, task.PreExecute(ctx)) - task.query.CollectionName = collectionName - - // Validate Partition - task.query.PartitionNames = []string{"$"} - assert.Error(t, task.PreExecute(ctx)) - task.query.PartitionNames = nil - - // mock show collections of QueryCoord - qc.SetShowCollectionsFunc(func(ctx context.Context, request *querypb.ShowCollectionsRequest) (*querypb.ShowCollectionsResponse, error) { - return nil, errors.New("mock") - }) - assert.Error(t, task.PreExecute(ctx)) - qc.SetShowCollectionsFunc(func(ctx context.Context, request *querypb.ShowCollectionsRequest) (*querypb.ShowCollectionsResponse, error) { - return &querypb.ShowCollectionsResponse{ - Status: &commonpb.Status{ - ErrorCode: commonpb.ErrorCode_UnexpectedError, - Reason: "mock", + getSearchTask := func(t *testing.T, collName string) *searchTask { + task := &searchTask{ + ctx: ctx, + SearchRequest: &internalpb.SearchRequest{}, + request: &milvuspb.SearchRequest{ + CollectionName: collName, }, - }, nil - }) - assert.Error(t, task.PreExecute(ctx)) - qc.ResetShowCollectionsFunc() + qc: qc, + tr: timerecord.NewTimeRecorder("test-search"), + } + require.NoError(t, task.OnEnqueue()) + return task + } - // collection not loaded - assert.Error(t, task.PreExecute(ctx)) - _, _ = qc.LoadCollection(ctx, &querypb.LoadCollectionRequest{ - Base: &commonpb.MsgBase{ - MsgType: commonpb.MsgType_LoadCollection, - MsgID: 0, - Timestamp: 0, - SourceID: 0, - }, - DbID: 0, - CollectionID: collectionID, - Schema: nil, + t.Run("collection not exist", func(t *testing.T) { + task := getSearchTask(t, collectionName) + err = task.PreExecute(ctx) + assert.Error(t, err) }) - // no anns field - task.query.DslType = commonpb.DslType_BoolExprV1 - assert.Error(t, task.PreExecute(ctx)) - task.query.SearchParams = []*commonpb.KeyValuePair{ - { - Key: AnnsFieldKey, - Value: floatVecField, - }, - } + t.Run("invalid collection name", func(t *testing.T) { + task := getSearchTask(t, collectionName) + createColl(t, collectionName, rc) - // no topk - assert.Error(t, task.PreExecute(ctx)) - task.query.SearchParams = []*commonpb.KeyValuePair{ - { + invalidCollNameTests := []struct { + inCollName string + description string + }{ + {"$", "invalid collection name $"}, + {"0", "invalid collection name 0"}, + } + + for _, test := range invalidCollNameTests { + t.Run(test.description, func(t *testing.T) { + task.request.CollectionName = test.inCollName + assert.Error(t, task.PreExecute(context.TODO())) + }) + } + }) + + t.Run("invalid partition names", func(t *testing.T) { + task := getSearchTask(t, collectionName) + createColl(t, collectionName, rc) + + invalidCollNameTests := []struct { + inPartNames []string + description string + }{ + {[]string{"$"}, "invalid partition name $"}, + {[]string{"0"}, "invalid collection name 0"}, + {[]string{"default", "$"}, "invalid empty partition name"}, + } + + for _, test := range invalidCollNameTests { + t.Run(test.description, func(t *testing.T) { + task.request.PartitionNames = test.inPartNames + assert.Error(t, task.PreExecute(context.TODO())) + }) + } + }) + + t.Run("test checkIfLoaded error", func(t *testing.T) { + collName := "test_checkIfLoaded_error" + funcutil.GenRandomStr() + createColl(t, collName, rc) + collID, err := globalMetaCache.GetCollectionID(context.TODO(), collName) + require.NoError(t, err) + task := getSearchTask(t, collName) + + t.Run("show collection err", func(t *testing.T) { + qc.SetShowCollectionsFunc(func(ctx context.Context, request *querypb.ShowCollectionsRequest) (*querypb.ShowCollectionsResponse, error) { + return nil, errors.New("mock") + }) + + assert.False(t, task.checkIfLoaded(collID, []UniqueID{})) + }) + + t.Run("show collection status unexpected error", func(t *testing.T) { + qc.SetShowCollectionsFunc(func(ctx context.Context, request *querypb.ShowCollectionsRequest) (*querypb.ShowCollectionsResponse, error) { + return &querypb.ShowCollectionsResponse{ + Status: &commonpb.Status{ + ErrorCode: commonpb.ErrorCode_UnexpectedError, + Reason: "mock", + }, + }, nil + }) + + assert.False(t, task.checkIfLoaded(collID, []UniqueID{})) + assert.Error(t, task.PreExecute(ctx)) + qc.ResetShowCollectionsFunc() + }) + + t.Run("show partition error", func(t *testing.T) { + qc.SetShowPartitionsFunc(func(ctx context.Context, req *querypb.ShowPartitionsRequest) (*querypb.ShowPartitionsResponse, error) { + return &querypb.ShowPartitionsResponse{ + Status: &commonpb.Status{ + ErrorCode: commonpb.ErrorCode_UnexpectedError, + Reason: "mock", + }, + }, nil + }) + assert.False(t, task.checkIfLoaded(collID, []UniqueID{1})) + }) + + t.Run("show partition status unexpected error", func(t *testing.T) { + qc.SetShowPartitionsFunc(func(ctx context.Context, req *querypb.ShowPartitionsRequest) (*querypb.ShowPartitionsResponse, error) { + return nil, errors.New("mock error") + }) + assert.False(t, task.checkIfLoaded(collID, []UniqueID{1})) + }) + + t.Run("show partitions success", func(t *testing.T) { + qc.SetShowPartitionsFunc(func(ctx context.Context, req *querypb.ShowPartitionsRequest) (*querypb.ShowPartitionsResponse, error) { + return &querypb.ShowPartitionsResponse{ + Status: &commonpb.Status{ + ErrorCode: commonpb.ErrorCode_Success, + }, + }, nil + }) + assert.True(t, task.checkIfLoaded(collID, []UniqueID{1})) + qc.ResetShowPartitionsFunc() + }) + + t.Run("show collection success but not loaded", func(t *testing.T) { + qc.SetShowCollectionsFunc(func(ctx context.Context, request *querypb.ShowCollectionsRequest) (*querypb.ShowCollectionsResponse, error) { + return &querypb.ShowCollectionsResponse{ + Status: &commonpb.Status{ + ErrorCode: commonpb.ErrorCode_Success, + }, + CollectionIDs: []UniqueID{collID}, + InMemoryPercentages: []int64{0}, + }, nil + }) + + qc.SetShowPartitionsFunc(func(ctx context.Context, req *querypb.ShowPartitionsRequest) (*querypb.ShowPartitionsResponse, error) { + return nil, errors.New("mock error") + }) + assert.False(t, task.checkIfLoaded(collID, []UniqueID{})) + + qc.SetShowPartitionsFunc(func(ctx context.Context, req *querypb.ShowPartitionsRequest) (*querypb.ShowPartitionsResponse, error) { + return nil, errors.New("mock error") + }) + assert.False(t, task.checkIfLoaded(collID, []UniqueID{})) + + qc.SetShowPartitionsFunc(func(ctx context.Context, req *querypb.ShowPartitionsRequest) (*querypb.ShowPartitionsResponse, error) { + return &querypb.ShowPartitionsResponse{ + Status: &commonpb.Status{ + ErrorCode: commonpb.ErrorCode_Success, + }, + PartitionIDs: []UniqueID{1}, + }, nil + }) + assert.True(t, task.checkIfLoaded(collID, []UniqueID{})) + }) + + qc.ResetShowCollectionsFunc() + qc.ResetShowPartitionsFunc() + }) + + t.Run("invalid key value pairs", func(t *testing.T) { + spNoTopk := []*commonpb.KeyValuePair{{ Key: AnnsFieldKey, - Value: floatVecField, - }, - { + Value: testFloatVecField}} + + spInvalidTopk := append(spNoTopk, &commonpb.KeyValuePair{ Key: TopKKey, Value: "invalid", - }, - } + }) - // invalid topk - assert.Error(t, task.PreExecute(ctx)) - task.query.SearchParams = []*commonpb.KeyValuePair{ - { - Key: AnnsFieldKey, - Value: floatVecField, - }, - { + spNoMetricType := append(spNoTopk, &commonpb.KeyValuePair{ Key: TopKKey, Value: "10", - }, - } + }) - // no metric type - assert.Error(t, task.PreExecute(ctx)) - task.query.SearchParams = []*commonpb.KeyValuePair{ - { - Key: AnnsFieldKey, - Value: floatVecField, - }, - { - Key: TopKKey, - Value: "10", - }, - { + spNoSearchParams := append(spNoMetricType, &commonpb.KeyValuePair{ Key: MetricTypeKey, Value: distance.L2, - }, - } + }) - // no search params - assert.Error(t, task.PreExecute(ctx)) - task.query.SearchParams = []*commonpb.KeyValuePair{ - { - Key: AnnsFieldKey, - Value: int64Field, - }, - { - Key: TopKKey, - Value: "10", - }, - { - Key: MetricTypeKey, - Value: distance.L2, - }, - { + spNoRoundDecimal := append(spNoSearchParams, &commonpb.KeyValuePair{ Key: SearchParamsKey, Value: `{"nprobe": 10}`, - }, - } + }) - // invalid round_decimal - assert.Error(t, task.PreExecute(ctx)) - task.query.SearchParams = []*commonpb.KeyValuePair{ - { - Key: AnnsFieldKey, - Value: int64Field, - }, - { - Key: TopKKey, - Value: "10", - }, - { - Key: MetricTypeKey, - Value: distance.L2, - }, - { - Key: SearchParamsKey, - Value: `{"nprobe": 10}`, - }, - { + spInvalidRoundDecimal := append(spNoRoundDecimal, &commonpb.KeyValuePair{ Key: RoundDecimalKey, Value: "invalid", - }, - } + }) - // invalid round_decimal - assert.Error(t, task.PreExecute(ctx)) - task.query.SearchParams = []*commonpb.KeyValuePair{ - { - Key: AnnsFieldKey, - Value: floatVecField, - }, - { - Key: TopKKey, - Value: "10", - }, - { - Key: MetricTypeKey, - Value: distance.L2, - }, - { - Key: RoundDecimalKey, - Value: "-1", - }, - } + tests := []struct { + description string + invalidParams []*commonpb.KeyValuePair + }{ + {"No_topk", spNoTopk}, + {"Invalid_topk", spInvalidTopk}, + {"No_Metric_type", spNoMetricType}, + {"No_search_params", spNoSearchParams}, + {"no_round_decimal", spNoRoundDecimal}, + {"Invalid_round_decimal", spInvalidRoundDecimal}, + } - // failed to create query plan - assert.Error(t, task.PreExecute(ctx)) - task.query.SearchParams = []*commonpb.KeyValuePair{ - { - Key: AnnsFieldKey, - Value: floatVecField, - }, - { - Key: TopKKey, - Value: "10", - }, - { - Key: MetricTypeKey, - Value: distance.L2, - }, - { - Key: SearchParamsKey, - Value: `{"nprobe": 10}`, - }, - { - Key: RoundDecimalKey, - Value: "-1", - }, - } + for _, test := range tests { + t.Run(test.description, func(t *testing.T) { + collName := "collection_" + test.description + createColl(t, collName, rc) + collID, err := globalMetaCache.GetCollectionID(context.TODO(), collName) + require.NoError(t, err) + task := getSearchTask(t, collName) + task.request.DslType = commonpb.DslType_BoolExprV1 - // search task with timeout - ctx1, cancel := context.WithTimeout(ctx, time.Second) - defer cancel() - // before preExecute - assert.Equal(t, typeutil.ZeroTimestamp, task.TimeoutTimestamp) - task.ctx = ctx1 - assert.NoError(t, task.PreExecute(ctx)) - // after preExecute - assert.Greater(t, task.TimeoutTimestamp, typeutil.ZeroTimestamp) + status, err := qc.LoadCollection(ctx, &querypb.LoadCollectionRequest{ + Base: &commonpb.MsgBase{ + MsgType: commonpb.MsgType_LoadCollection, + }, + CollectionID: collID, + }) + require.NoError(t, err) + require.Equal(t, commonpb.ErrorCode_Success, status.GetErrorCode()) + assert.Error(t, task.PreExecute(ctx)) + }) + } + }) - // field not exist - task.query.OutputFields = []string{int64Field + funcutil.GenRandomStr()} - assert.Error(t, task.PreExecute(ctx)) - // contain vector field - task.query.OutputFields = []string{floatVecField} - assert.Error(t, task.PreExecute(ctx)) - task.query.OutputFields = []string{int64Field} + t.Run("search with timeout", func(t *testing.T) { + collName := "search_with_timeout" + funcutil.GenRandomStr() + createColl(t, collName, rc) + collID, err := globalMetaCache.GetCollectionID(context.TODO(), collName) + require.NoError(t, err) + status, err := qc.LoadCollection(ctx, &querypb.LoadCollectionRequest{ + Base: &commonpb.MsgBase{ + MsgType: commonpb.MsgType_LoadCollection, + }, + CollectionID: collID, + }) + require.NoError(t, err) + require.Equal(t, commonpb.ErrorCode_Success, status.GetErrorCode()) - // partition - rc.showPartitionsFunc = func(ctx context.Context, request *milvuspb.ShowPartitionsRequest) (*milvuspb.ShowPartitionsResponse, error) { - return nil, errors.New("mock") - } - assert.Error(t, task.PreExecute(ctx)) - rc.showPartitionsFunc = nil + task := getSearchTask(t, collName) + task.request.SearchParams = getValidSearchParams() + task.request.DslType = commonpb.DslType_BoolExprV1 - // TODO(dragondriver): test partition-related error + ctxTimeout, cancel := context.WithTimeout(ctx, time.Second) + defer cancel() + require.Equal(t, typeutil.ZeroTimestamp, task.TimeoutTimestamp) + + task.ctx = ctxTimeout + assert.NoError(t, task.PreExecute(ctx)) + assert.Greater(t, task.TimeoutTimestamp, typeutil.ZeroTimestamp) + + // field not exist + task.ctx = context.TODO() + task.request.OutputFields = []string{testInt64Field + funcutil.GenRandomStr()} + assert.Error(t, task.PreExecute(ctx)) + + // contain vector field + task.request.OutputFields = []string{testFloatVecField} + assert.Error(t, task.PreExecute(ctx)) + }) } -func TestSearchTask_Ts(t *testing.T) { +func TestSearchTaskV2_Execute(t *testing.T) { Params.Init() - task := &searchTask{ - SearchRequest: &internalpb.SearchRequest{ - Base: nil, - }, - tr: timerecord.NewTimeRecorder("search"), - } - assert.NoError(t, task.OnEnqueue()) + var ( + err error - ts := Timestamp(time.Now().Nanosecond()) - task.SetTs(ts) - assert.Equal(t, ts, task.BeginTs()) - assert.Equal(t, ts, task.EndTs()) -} + rc = NewRootCoordMock() + qc = NewQueryCoordMock() + ctx = context.TODO() -func TestSearchTask_Execute(t *testing.T) { - var err error + collectionName = t.Name() + funcutil.GenRandomStr() + ) - Params.Init() - Params.ProxyCfg.SearchResultChannelNames = []string{funcutil.GenRandomStr()} - - rc := NewRootCoordMock() - rc.Start() + err = rc.Start() + require.NoError(t, err) defer rc.Stop() - - qc := NewQueryCoordMock() - qc.Start() - defer qc.Stop() - - ctx := context.Background() - err = InitMetaCache(rc) - assert.NoError(t, err) + require.NoError(t, err) - dmlChannelsFunc := getDmlChannelsFunc(ctx, rc) - query := newMockGetChannelsService() - factory := newSimpleMockMsgStreamFactory() - chMgr := newChannelsMgrImpl(dmlChannelsFunc, nil, query.GetChannels, nil, factory) - defer chMgr.removeAllDMLStream() - defer chMgr.removeAllDQLStream() - - prefix := "TestSearchTask_Execute" - collectionName := prefix + funcutil.GenRandomStr() - shardsNum := int32(2) - dbName := "" - int64Field := "int64" - floatVecField := "fvec" - dim := 128 + err = qc.Start() + require.NoError(t, err) + defer qc.Stop() task := &searchTask{ ctx: ctx, SearchRequest: &internalpb.SearchRequest{ Base: &commonpb.MsgBase{ MsgType: commonpb.MsgType_Search, - MsgID: 0, Timestamp: uint64(time.Now().UnixNano()), - SourceID: 0, }, }, - query: &milvuspb.SearchRequest{ + request: &milvuspb.SearchRequest{ CollectionName: collectionName, }, result: &milvuspb.SearchResults{ - Status: &commonpb.Status{}, - Results: nil, + Status: &commonpb.Status{}, }, - chMgr: chMgr, - qc: qc, - tr: timerecord.NewTimeRecorder("search"), + qc: qc, + tr: timerecord.NewTimeRecorder("search"), } - assert.NoError(t, task.OnEnqueue()) - - // collection not exist - assert.Error(t, task.PreExecute(ctx)) - - schema := constructCollectionSchema(int64Field, floatVecField, dim, collectionName) - marshaledSchema, err := proto.Marshal(schema) - assert.NoError(t, err) - - createColT := &createCollectionTask{ - Condition: NewTaskCondition(ctx), - CreateCollectionRequest: &milvuspb.CreateCollectionRequest{ - Base: nil, - DbName: dbName, - CollectionName: collectionName, - Schema: marshaledSchema, - ShardsNum: shardsNum, - }, - ctx: ctx, - rootCoord: rc, - result: nil, - schema: nil, - } - - assert.NoError(t, createColT.OnEnqueue()) - assert.NoError(t, createColT.PreExecute(ctx)) - assert.NoError(t, createColT.Execute(ctx)) - assert.NoError(t, createColT.PostExecute(ctx)) - - assert.NoError(t, task.Execute(ctx)) - - _ = chMgr.removeAllDQLStream() - query.f = func(collectionID UniqueID) (map[vChan]pChan, error) { - return nil, errors.New("mock") - } - assert.Error(t, task.Execute(ctx)) - // TODO(dragondriver): cover getDQLStream + require.NoError(t, task.OnEnqueue()) + createColl(t, collectionName, rc) } func genSearchResultData(nq int64, topk int64, ids []int64, scores []float32) *schemapb.SearchResultData { @@ -636,794 +451,781 @@ func genSearchResultData(nq int64, topk int64, ids []int64, scores []float32) *s } } +func TestSearchTask_Ts(t *testing.T) { + Params.Init() + task := &searchTask{ + SearchRequest: &internalpb.SearchRequest{}, + + tr: timerecord.NewTimeRecorder("test-search"), + } + require.NoError(t, task.OnEnqueue()) + + ts := Timestamp(time.Now().Nanosecond()) + task.SetTs(ts) + assert.Equal(t, ts, task.BeginTs()) + assert.Equal(t, ts, task.EndTs()) +} + func TestSearchTask_Reduce(t *testing.T) { - const ( - nq = 1 - topk = 4 - metricType = "L2" - ) - t.Run("case1", func(t *testing.T) { - ids := []int64{1, 2, 3, 4} - scores := []float32{-1.0, -2.0, -3.0, -4.0} - data1 := genSearchResultData(nq, topk, ids, scores) - data2 := genSearchResultData(nq, topk, ids, scores) - dataArray := make([]*schemapb.SearchResultData, 0) - dataArray = append(dataArray, data1) - dataArray = append(dataArray, data2) - res, err := reduceSearchResultData(dataArray, nq, topk, metricType) - assert.Nil(t, err) - assert.Equal(t, ids, res.Results.Ids.GetIntId().Data) - assert.Equal(t, []float32{1.0, 2.0, 3.0, 4.0}, res.Results.Scores) - }) - t.Run("case2", func(t *testing.T) { - ids1 := []int64{1, 2, 3, 4} - scores1 := []float32{-1.0, -2.0, -3.0, -4.0} - ids2 := []int64{5, 1, 3, 4} - scores2 := []float32{-1.0, -1.0, -3.0, -4.0} - data1 := genSearchResultData(nq, topk, ids1, scores1) - data2 := genSearchResultData(nq, topk, ids2, scores2) - dataArray := make([]*schemapb.SearchResultData, 0) - dataArray = append(dataArray, data1) - dataArray = append(dataArray, data2) - res, err := reduceSearchResultData(dataArray, nq, topk, metricType) - assert.Nil(t, err) - assert.ElementsMatch(t, []int64{1, 5, 2, 3}, res.Results.Ids.GetIntId().Data) - }) + // const ( + // nq = 1 + // topk = 4 + // metricType = "L2" + // ) + // t.Run("case1", func(t *testing.T) { + // ids := []int64{1, 2, 3, 4} + // scores := []float32{-1.0, -2.0, -3.0, -4.0} + // data1 := genSearchResultData(nq, topk, ids, scores) + // data2 := genSearchResultData(nq, topk, ids, scores) + // dataArray := make([]*schemapb.SearchResultData, 0) + // dataArray = append(dataArray, data1) + // dataArray = append(dataArray, data2) + // res, err := reduceSearchResultData(dataArray, nq, topk, metricType) + // assert.Nil(t, err) + // assert.Equal(t, ids, res.Results.Ids.GetIntId().Data) + // assert.Equal(t, []float32{1.0, 2.0, 3.0, 4.0}, res.Results.Scores) + // }) + // t.Run("case2", func(t *testing.T) { + // ids1 := []int64{1, 2, 3, 4} + // scores1 := []float32{-1.0, -2.0, -3.0, -4.0} + // ids2 := []int64{5, 1, 3, 4} + // scores2 := []float32{-1.0, -1.0, -3.0, -4.0} + // data1 := genSearchResultData(nq, topk, ids1, scores1) + // data2 := genSearchResultData(nq, topk, ids2, scores2) + // dataArray := make([]*schemapb.SearchResultData, 0) + // dataArray = append(dataArray, data1) + // dataArray = append(dataArray, data2) + // res, err := reduceSearchResultData(dataArray, nq, topk, metricType) + // assert.Nil(t, err) + // assert.ElementsMatch(t, []int64{1, 5, 2, 3}, res.Results.Ids.GetIntId().Data) + // }) } func TestSearchTaskWithInvalidRoundDecimal(t *testing.T) { - var err error - - Params.Init() - Params.ProxyCfg.SearchResultChannelNames = []string{funcutil.GenRandomStr()} - - rc := NewRootCoordMock() - rc.Start() - defer rc.Stop() - - ctx := context.Background() - - err = InitMetaCache(rc) - assert.NoError(t, err) - - shardsNum := int32(2) - prefix := "TestSearchTask_all" - dbName := "" - collectionName := prefix + funcutil.GenRandomStr() - - dim := 128 - expr := fmt.Sprintf("%s > 0", testInt64Field) - nq := 10 - topk := 10 - roundDecimal := 7 - nprobe := 10 - - fieldName2Types := map[string]schemapb.DataType{ - testBoolField: schemapb.DataType_Bool, - testInt32Field: schemapb.DataType_Int32, - testInt64Field: schemapb.DataType_Int64, - testFloatField: schemapb.DataType_Float, - testDoubleField: schemapb.DataType_Double, - testFloatVecField: schemapb.DataType_FloatVector, - } - if enableMultipleVectorFields { - fieldName2Types[testBinaryVecField] = schemapb.DataType_BinaryVector - } - schema := constructCollectionSchemaByDataType(collectionName, fieldName2Types, testInt64Field, false) - marshaledSchema, err := proto.Marshal(schema) - assert.NoError(t, err) - - createColT := &createCollectionTask{ - Condition: NewTaskCondition(ctx), - CreateCollectionRequest: &milvuspb.CreateCollectionRequest{ - Base: nil, - DbName: dbName, - CollectionName: collectionName, - Schema: marshaledSchema, - ShardsNum: shardsNum, - }, - ctx: ctx, - rootCoord: rc, - result: nil, - schema: nil, - } - - assert.NoError(t, createColT.OnEnqueue()) - assert.NoError(t, createColT.PreExecute(ctx)) - assert.NoError(t, createColT.Execute(ctx)) - assert.NoError(t, createColT.PostExecute(ctx)) - - dmlChannelsFunc := getDmlChannelsFunc(ctx, rc) - query := newMockGetChannelsService() - factory := newSimpleMockMsgStreamFactory() - chMgr := newChannelsMgrImpl(dmlChannelsFunc, nil, query.GetChannels, nil, factory) - defer chMgr.removeAllDMLStream() - defer chMgr.removeAllDQLStream() - - collectionID, err := globalMetaCache.GetCollectionID(ctx, collectionName) - assert.NoError(t, err) - - qc := NewQueryCoordMock() - qc.Start() - defer qc.Stop() - status, err := qc.LoadCollection(ctx, &querypb.LoadCollectionRequest{ - Base: &commonpb.MsgBase{ - MsgType: commonpb.MsgType_LoadCollection, - MsgID: 0, - Timestamp: 0, - SourceID: Params.ProxyCfg.ProxyID, - }, - DbID: 0, - CollectionID: collectionID, - Schema: nil, - }) - assert.NoError(t, err) - assert.Equal(t, commonpb.ErrorCode_Success, status.ErrorCode) - - req := constructSearchRequest(dbName, collectionName, - expr, - testFloatVecField, - nq, dim, nprobe, topk, roundDecimal) - - task := &searchTask{ - Condition: NewTaskCondition(ctx), - SearchRequest: &internalpb.SearchRequest{ - Base: &commonpb.MsgBase{ - MsgType: commonpb.MsgType_Search, - MsgID: 0, - Timestamp: 0, - SourceID: Params.ProxyCfg.ProxyID, - }, - ResultChannelID: strconv.FormatInt(Params.ProxyCfg.ProxyID, 10), - DbID: 0, - CollectionID: 0, - PartitionIDs: nil, - Dsl: "", - PlaceholderGroup: nil, - DslType: 0, - SerializedExprPlan: nil, - OutputFieldsId: nil, - TravelTimestamp: 0, - GuaranteeTimestamp: 0, - }, - ctx: ctx, - resultBuf: make(chan []*internalpb.SearchResults), - result: nil, - query: req, - chMgr: chMgr, - qc: qc, - tr: timerecord.NewTimeRecorder("search"), - } - - // simple mock for query node - // TODO(dragondriver): should we replace this mock using RocksMq or MemMsgStream? - - err = chMgr.createDQLStream(collectionID) - assert.NoError(t, err) - stream, err := chMgr.getDQLStream(collectionID) - assert.NoError(t, err) - - var wg sync.WaitGroup - wg.Add(1) - consumeCtx, cancel := context.WithCancel(ctx) - go func() { - defer wg.Done() - for { - select { - case <-consumeCtx.Done(): - return - case pack, ok := <-stream.Chan(): - assert.True(t, ok) - if pack == nil { - continue - } - - for _, msg := range pack.Msgs { - _, ok := msg.(*msgstream.SearchMsg) - assert.True(t, ok) - // TODO(dragondriver): construct result according to the request - - constructSearchResulstData := func() *schemapb.SearchResultData { - resultData := &schemapb.SearchResultData{ - NumQueries: int64(nq), - TopK: int64(topk), - Scores: make([]float32, nq*topk), - Ids: &schemapb.IDs{ - IdField: &schemapb.IDs_IntId{ - IntId: &schemapb.LongArray{ - Data: make([]int64, nq*topk), - }, - }, - }, - Topks: make([]int64, nq), - } - - fieldID := common.StartOfUserFieldID - for fieldName, dataType := range fieldName2Types { - resultData.FieldsData = append(resultData.FieldsData, generateFieldData(dataType, fieldName, int64(fieldID), nq*topk)) - fieldID++ - } - - for i := 0; i < nq; i++ { - for j := 0; j < topk; j++ { - offset := i*topk + j - score := float32(uniquegenerator.GetUniqueIntGeneratorIns().GetInt()) // increasingly - id := int64(uniquegenerator.GetUniqueIntGeneratorIns().GetInt()) - resultData.Scores[offset] = score - resultData.Ids.IdField.(*schemapb.IDs_IntId).IntId.Data[offset] = id - } - resultData.Topks[i] = int64(topk) - } - - return resultData - } - - result1 := &internalpb.SearchResults{ - Base: &commonpb.MsgBase{ - MsgType: commonpb.MsgType_SearchResult, - MsgID: 0, - Timestamp: 0, - SourceID: 0, - }, - Status: &commonpb.Status{ - ErrorCode: commonpb.ErrorCode_Success, - Reason: "", - }, - ResultChannelID: "", - MetricType: distance.L2, - NumQueries: int64(nq), - TopK: int64(topk), - SealedSegmentIDsSearched: nil, - ChannelIDsSearched: nil, - GlobalSealedSegmentIDs: nil, - SlicedBlob: nil, - SlicedNumCount: 1, - SlicedOffset: 0, - } - resultData := constructSearchResulstData() - sliceBlob, err := proto.Marshal(resultData) - assert.NoError(t, err) - result1.SlicedBlob = sliceBlob - - // result2.SliceBlob = nil, will be skipped in decode stage - result2 := &internalpb.SearchResults{ - Base: &commonpb.MsgBase{ - MsgType: commonpb.MsgType_SearchResult, - MsgID: 0, - Timestamp: 0, - SourceID: 0, - }, - Status: &commonpb.Status{ - ErrorCode: commonpb.ErrorCode_Success, - Reason: "", - }, - ResultChannelID: "", - MetricType: distance.L2, - NumQueries: int64(nq), - TopK: int64(topk), - SealedSegmentIDsSearched: nil, - ChannelIDsSearched: nil, - GlobalSealedSegmentIDs: nil, - SlicedBlob: nil, - SlicedNumCount: 1, - SlicedOffset: 0, - } - - // send search result - task.resultBuf <- []*internalpb.SearchResults{result1, result2} - } - } - } - }() - - assert.NoError(t, task.OnEnqueue()) - assert.Error(t, task.PreExecute(ctx)) - - cancel() - wg.Wait() -} -func TestSearchTask_all(t *testing.T) { - var err error - - Params.Init() - Params.ProxyCfg.SearchResultChannelNames = []string{funcutil.GenRandomStr()} - - rc := NewRootCoordMock() - rc.Start() - defer rc.Stop() - - ctx := context.Background() - - err = InitMetaCache(rc) - assert.NoError(t, err) - - shardsNum := int32(2) - prefix := "TestSearchTask_all" - dbName := "" - collectionName := prefix + funcutil.GenRandomStr() - - dim := 128 - expr := fmt.Sprintf("%s > 0", testInt64Field) - nq := 10 - topk := 10 - roundDecimal := 3 - nprobe := 10 - - fieldName2Types := map[string]schemapb.DataType{ - testBoolField: schemapb.DataType_Bool, - testInt32Field: schemapb.DataType_Int32, - testInt64Field: schemapb.DataType_Int64, - testFloatField: schemapb.DataType_Float, - testDoubleField: schemapb.DataType_Double, - testFloatVecField: schemapb.DataType_FloatVector, - } - if enableMultipleVectorFields { - fieldName2Types[testBinaryVecField] = schemapb.DataType_BinaryVector - } - - schema := constructCollectionSchemaByDataType(collectionName, fieldName2Types, testInt64Field, false) - marshaledSchema, err := proto.Marshal(schema) - assert.NoError(t, err) - - createColT := &createCollectionTask{ - Condition: NewTaskCondition(ctx), - CreateCollectionRequest: &milvuspb.CreateCollectionRequest{ - Base: nil, - DbName: dbName, - CollectionName: collectionName, - Schema: marshaledSchema, - ShardsNum: shardsNum, - }, - ctx: ctx, - rootCoord: rc, - result: nil, - schema: nil, - } - - assert.NoError(t, createColT.OnEnqueue()) - assert.NoError(t, createColT.PreExecute(ctx)) - assert.NoError(t, createColT.Execute(ctx)) - assert.NoError(t, createColT.PostExecute(ctx)) - - dmlChannelsFunc := getDmlChannelsFunc(ctx, rc) - query := newMockGetChannelsService() - factory := newSimpleMockMsgStreamFactory() - chMgr := newChannelsMgrImpl(dmlChannelsFunc, nil, query.GetChannels, nil, factory) - defer chMgr.removeAllDMLStream() - defer chMgr.removeAllDQLStream() - - collectionID, err := globalMetaCache.GetCollectionID(ctx, collectionName) - assert.NoError(t, err) - - qc := NewQueryCoordMock() - qc.Start() - defer qc.Stop() - status, err := qc.LoadCollection(ctx, &querypb.LoadCollectionRequest{ - Base: &commonpb.MsgBase{ - MsgType: commonpb.MsgType_LoadCollection, - MsgID: 0, - Timestamp: 0, - SourceID: Params.ProxyCfg.ProxyID, - }, - DbID: 0, - CollectionID: collectionID, - Schema: nil, - }) - assert.NoError(t, err) - assert.Equal(t, commonpb.ErrorCode_Success, status.ErrorCode) - - req := constructSearchRequest(dbName, collectionName, - expr, - testFloatVecField, - nq, dim, nprobe, topk, roundDecimal) - - task := &searchTask{ - Condition: NewTaskCondition(ctx), - SearchRequest: &internalpb.SearchRequest{ - Base: &commonpb.MsgBase{ - MsgType: commonpb.MsgType_Search, - MsgID: 0, - Timestamp: 0, - SourceID: Params.ProxyCfg.ProxyID, - }, - ResultChannelID: strconv.FormatInt(Params.ProxyCfg.ProxyID, 10), - DbID: 0, - CollectionID: 0, - PartitionIDs: nil, - Dsl: "", - PlaceholderGroup: nil, - DslType: 0, - SerializedExprPlan: nil, - OutputFieldsId: nil, - TravelTimestamp: 0, - GuaranteeTimestamp: 0, - }, - ctx: ctx, - resultBuf: make(chan []*internalpb.SearchResults), - result: nil, - query: req, - chMgr: chMgr, - qc: qc, - tr: timerecord.NewTimeRecorder("search"), - } - - // simple mock for query node - // TODO(dragondriver): should we replace this mock using RocksMq or MemMsgStream? - - err = chMgr.createDQLStream(collectionID) - assert.NoError(t, err) - stream, err := chMgr.getDQLStream(collectionID) - assert.NoError(t, err) - - var wg sync.WaitGroup - wg.Add(1) - consumeCtx, cancel := context.WithCancel(ctx) - go func() { - defer wg.Done() - for { - select { - case <-consumeCtx.Done(): - return - case pack, ok := <-stream.Chan(): - assert.True(t, ok) - if pack == nil { - continue - } - - for _, msg := range pack.Msgs { - _, ok := msg.(*msgstream.SearchMsg) - assert.True(t, ok) - // TODO(dragondriver): construct result according to the request - - constructSearchResulstData := func() *schemapb.SearchResultData { - resultData := &schemapb.SearchResultData{ - NumQueries: int64(nq), - TopK: int64(topk), - Scores: make([]float32, nq*topk), - Ids: &schemapb.IDs{ - IdField: &schemapb.IDs_IntId{ - IntId: &schemapb.LongArray{ - Data: make([]int64, nq*topk), - }, - }, - }, - Topks: make([]int64, nq), - } - - fieldID := common.StartOfUserFieldID - for fieldName, dataType := range fieldName2Types { - resultData.FieldsData = append(resultData.FieldsData, generateFieldData(dataType, fieldName, int64(fieldID), nq*topk)) - fieldID++ - } - - for i := 0; i < nq; i++ { - for j := 0; j < topk; j++ { - offset := i*topk + j - score := float32(uniquegenerator.GetUniqueIntGeneratorIns().GetInt()) // increasingly - id := int64(uniquegenerator.GetUniqueIntGeneratorIns().GetInt()) - resultData.Scores[offset] = score - resultData.Ids.IdField.(*schemapb.IDs_IntId).IntId.Data[offset] = id - } - resultData.Topks[i] = int64(topk) - } - - return resultData - } - - result1 := &internalpb.SearchResults{ - Base: &commonpb.MsgBase{ - MsgType: commonpb.MsgType_SearchResult, - MsgID: 0, - Timestamp: 0, - SourceID: 0, - }, - Status: &commonpb.Status{ - ErrorCode: commonpb.ErrorCode_Success, - Reason: "", - }, - ResultChannelID: "", - MetricType: distance.L2, - NumQueries: int64(nq), - TopK: int64(topk), - SealedSegmentIDsSearched: nil, - ChannelIDsSearched: nil, - GlobalSealedSegmentIDs: nil, - SlicedBlob: nil, - SlicedNumCount: 1, - SlicedOffset: 0, - } - resultData := constructSearchResulstData() - sliceBlob, err := proto.Marshal(resultData) - assert.NoError(t, err) - result1.SlicedBlob = sliceBlob - - // result2.SliceBlob = nil, will be skipped in decode stage - result2 := &internalpb.SearchResults{ - Base: &commonpb.MsgBase{ - MsgType: commonpb.MsgType_SearchResult, - MsgID: 0, - Timestamp: 0, - SourceID: 0, - }, - Status: &commonpb.Status{ - ErrorCode: commonpb.ErrorCode_Success, - Reason: "", - }, - ResultChannelID: "", - MetricType: distance.L2, - NumQueries: int64(nq), - TopK: int64(topk), - SealedSegmentIDsSearched: nil, - ChannelIDsSearched: nil, - GlobalSealedSegmentIDs: nil, - SlicedBlob: nil, - SlicedNumCount: 1, - SlicedOffset: 0, - } - - // send search result - task.resultBuf <- []*internalpb.SearchResults{result1, result2} - } - } - } - }() - - assert.NoError(t, task.OnEnqueue()) - assert.NoError(t, task.PreExecute(ctx)) - assert.NoError(t, task.Execute(ctx)) - assert.NoError(t, task.PostExecute(ctx)) - - cancel() - wg.Wait() + // var err error + // + // Params.Init() + // Params.ProxyCfg.SearchResultChannelNames = []string{funcutil.GenRandomStr()} + // + // rc := NewRootCoordMock() + // rc.Start() + // defer rc.Stop() + // + // ctx := context.Background() + // + // err = InitMetaCache(rc) + // assert.NoError(t, err) + // + // shardsNum := int32(2) + // prefix := "TestSearchTaskV2_all" + // collectionName := prefix + funcutil.GenRandomStr() + // + // dim := 128 + // expr := fmt.Sprintf("%s > 0", testInt64Field) + // nq := 10 + // topk := 10 + // roundDecimal := 7 + // nprobe := 10 + // + // fieldName2Types := map[string]schemapb.DataType{ + // testBoolField: schemapb.DataType_Bool, + // testInt32Field: schemapb.DataType_Int32, + // testInt64Field: schemapb.DataType_Int64, + // testFloatField: schemapb.DataType_Float, + // testDoubleField: schemapb.DataType_Double, + // testFloatVecField: schemapb.DataType_FloatVector, + // } + // if enableMultipleVectorFields { + // fieldName2Types[testBinaryVecField] = schemapb.DataType_BinaryVector + // } + // schema := constructCollectionSchemaByDataType(collectionName, fieldName2Types, testInt64Field, false) + // marshaledSchema, err := proto.Marshal(schema) + // assert.NoError(t, err) + // + // createColT := &createCollectionTask{ + // Condition: NewTaskCondition(ctx), + // CreateCollectionRequest: &milvuspb.CreateCollectionRequest{ + // Base: nil, + // CollectionName: collectionName, + // Schema: marshaledSchema, + // ShardsNum: shardsNum, + // }, + // ctx: ctx, + // rootCoord: rc, + // result: nil, + // schema: nil, + // } + // + // assert.NoError(t, createColT.OnEnqueue()) + // assert.NoError(t, createColT.PreExecute(ctx)) + // assert.NoError(t, createColT.Execute(ctx)) + // assert.NoError(t, createColT.PostExecute(ctx)) + // + // dmlChannelsFunc := getDmlChannelsFunc(ctx, rc) + // query := newMockGetChannelsService() + // factory := newSimpleMockMsgStreamFactory() + // + // collectionID, err := globalMetaCache.GetCollectionID(ctx, collectionName) + // assert.NoError(t, err) + // + // qc := NewQueryCoordMock() + // qc.Start() + // defer qc.Stop() + // status, err := qc.LoadCollection(ctx, &querypb.LoadCollectionRequest{ + // Base: &commonpb.MsgBase{ + // MsgType: commonpb.MsgType_LoadCollection, + // MsgID: 0, + // Timestamp: 0, + // SourceID: Params.ProxyCfg.ProxyID, + // }, + // DbID: 0, + // CollectionID: collectionID, + // Schema: nil, + // }) + // assert.NoError(t, err) + // assert.Equal(t, commonpb.ErrorCode_Success, status.ErrorCode) + // + // req := constructSearchRequest("", collectionName, + // expr, + // testFloatVecField, + // nq, dim, nprobe, topk, roundDecimal) + // + // task := &searchTaskV2{ + // Condition: NewTaskCondition(ctx), + // SearchRequest: &internalpb.SearchRequest{ + // Base: &commonpb.MsgBase{ + // MsgType: commonpb.MsgType_Search, + // MsgID: 0, + // Timestamp: 0, + // SourceID: Params.ProxyCfg.ProxyID, + // }, + // ResultChannelID: strconv.FormatInt(Params.ProxyCfg.ProxyID, 10), + // DbID: 0, + // CollectionID: 0, + // PartitionIDs: nil, + // Dsl: "", + // PlaceholderGroup: nil, + // DslType: 0, + // SerializedExprPlan: nil, + // OutputFieldsId: nil, + // TravelTimestamp: 0, + // GuaranteeTimestamp: 0, + // }, + // ctx: ctx, + // resultBuf: make(chan *internalpb.SearchResults, 10), + // result: nil, + // request: req, + // qc: qc, + // tr: timerecord.NewTimeRecorder("search"), + // } + // + // // simple mock for query node + // // TODO(dragondriver): should we replace this mock using RocksMq or MemMsgStream? + // + // + // var wg sync.WaitGroup + // wg.Add(1) + // consumeCtx, cancel := context.WithCancel(ctx) + // go func() { + // defer wg.Done() + // for { + // select { + // case <-consumeCtx.Done(): + // return + // case pack, ok := <-stream.Chan(): + // assert.True(t, ok) + // if pack == nil { + // continue + // } + // + // for _, msg := range pack.Msgs { + // _, ok := msg.(*msgstream.SearchMsg) + // assert.True(t, ok) + // // TODO(dragondriver): construct result according to the request + // + // constructSearchResulstData := func() *schemapb.SearchResultData { + // resultData := &schemapb.SearchResultData{ + // NumQueries: int64(nq), + // TopK: int64(topk), + // Scores: make([]float32, nq*topk), + // Ids: &schemapb.IDs{ + // IdField: &schemapb.IDs_IntId{ + // IntId: &schemapb.LongArray{ + // Data: make([]int64, nq*topk), + // }, + // }, + // }, + // Topks: make([]int64, nq), + // } + // + // fieldID := common.StartOfUserFieldID + // for fieldName, dataType := range fieldName2Types { + // resultData.FieldsData = append(resultData.FieldsData, generateFieldData(dataType, fieldName, int64(fieldID), nq*topk)) + // fieldID++ + // } + // + // for i := 0; i < nq; i++ { + // for j := 0; j < topk; j++ { + // offset := i*topk + j + // score := float32(uniquegenerator.GetUniqueIntGeneratorIns().GetInt()) // increasingly + // id := int64(uniquegenerator.GetUniqueIntGeneratorIns().GetInt()) + // resultData.Scores[offset] = score + // resultData.Ids.IdField.(*schemapb.IDs_IntId).IntId.Data[offset] = id + // } + // resultData.Topks[i] = int64(topk) + // } + // + // return resultData + // } + // + // result1 := &internalpb.SearchResults{ + // Base: &commonpb.MsgBase{ + // MsgType: commonpb.MsgType_SearchResult, + // MsgID: 0, + // Timestamp: 0, + // SourceID: 0, + // }, + // Status: &commonpb.Status{ + // ErrorCode: commonpb.ErrorCode_Success, + // Reason: "", + // }, + // ResultChannelID: "", + // MetricType: distance.L2, + // NumQueries: int64(nq), + // TopK: int64(topk), + // SealedSegmentIDsSearched: nil, + // ChannelIDsSearched: nil, + // GlobalSealedSegmentIDs: nil, + // SlicedBlob: nil, + // SlicedNumCount: 1, + // SlicedOffset: 0, + // } + // resultData := constructSearchResulstData() + // sliceBlob, err := proto.Marshal(resultData) + // assert.NoError(t, err) + // result1.SlicedBlob = sliceBlob + // + // // result2.SliceBlob = nil, will be skipped in decode stage + // result2 := &internalpb.SearchResults{ + // Base: &commonpb.MsgBase{ + // MsgType: commonpb.MsgType_SearchResult, + // MsgID: 0, + // Timestamp: 0, + // SourceID: 0, + // }, + // Status: &commonpb.Status{ + // ErrorCode: commonpb.ErrorCode_Success, + // Reason: "", + // }, + // ResultChannelID: "", + // MetricType: distance.L2, + // NumQueries: int64(nq), + // TopK: int64(topk), + // SealedSegmentIDsSearched: nil, + // ChannelIDsSearched: nil, + // GlobalSealedSegmentIDs: nil, + // SlicedBlob: nil, + // SlicedNumCount: 1, + // SlicedOffset: 0, + // } + // + // // send search result + // task.resultBuf <- result1 + // task.resultBuf <- result2 + // } + // } + // } + // }() + // + // assert.NoError(t, task.OnEnqueue()) + // assert.Error(t, task.PreExecute(ctx)) + // + // cancel() + // wg.Wait() } -func TestSearchTask_7803_reduce(t *testing.T) { - var err error - - Params.Init() - Params.ProxyCfg.SearchResultChannelNames = []string{funcutil.GenRandomStr()} - - rc := NewRootCoordMock() - rc.Start() - defer rc.Stop() - - ctx := context.Background() - - err = InitMetaCache(rc) - assert.NoError(t, err) - - shardsNum := int32(2) - prefix := "TestSearchTask_7803_reduce" - dbName := "" - collectionName := prefix + funcutil.GenRandomStr() - int64Field := "int64" - floatVecField := "fvec" - dim := 128 - expr := fmt.Sprintf("%s > 0", int64Field) - nq := 10 - topk := 10 - roundDecimal := 3 - nprobe := 10 - - schema := constructCollectionSchema( - int64Field, - floatVecField, - dim, - collectionName) - marshaledSchema, err := proto.Marshal(schema) - assert.NoError(t, err) - - createColT := &createCollectionTask{ - Condition: NewTaskCondition(ctx), - CreateCollectionRequest: &milvuspb.CreateCollectionRequest{ - Base: nil, - DbName: dbName, - CollectionName: collectionName, - Schema: marshaledSchema, - ShardsNum: shardsNum, - }, - ctx: ctx, - rootCoord: rc, - result: nil, - schema: nil, - } - - assert.NoError(t, createColT.OnEnqueue()) - assert.NoError(t, createColT.PreExecute(ctx)) - assert.NoError(t, createColT.Execute(ctx)) - assert.NoError(t, createColT.PostExecute(ctx)) - - dmlChannelsFunc := getDmlChannelsFunc(ctx, rc) - query := newMockGetChannelsService() - factory := newSimpleMockMsgStreamFactory() - chMgr := newChannelsMgrImpl(dmlChannelsFunc, nil, query.GetChannels, nil, factory) - defer chMgr.removeAllDMLStream() - defer chMgr.removeAllDQLStream() - - collectionID, err := globalMetaCache.GetCollectionID(ctx, collectionName) - assert.NoError(t, err) - - qc := NewQueryCoordMock() - qc.Start() - defer qc.Stop() - status, err := qc.LoadCollection(ctx, &querypb.LoadCollectionRequest{ - Base: &commonpb.MsgBase{ - MsgType: commonpb.MsgType_LoadCollection, - MsgID: 0, - Timestamp: 0, - SourceID: Params.ProxyCfg.ProxyID, - }, - DbID: 0, - CollectionID: collectionID, - Schema: nil, - }) - assert.NoError(t, err) - assert.Equal(t, commonpb.ErrorCode_Success, status.ErrorCode) - - req := constructSearchRequest(dbName, collectionName, - expr, - floatVecField, - nq, dim, nprobe, topk, roundDecimal) - - task := &searchTask{ - Condition: NewTaskCondition(ctx), - SearchRequest: &internalpb.SearchRequest{ - Base: &commonpb.MsgBase{ - MsgType: commonpb.MsgType_Search, - MsgID: 0, - Timestamp: 0, - SourceID: Params.ProxyCfg.ProxyID, - }, - ResultChannelID: strconv.FormatInt(Params.ProxyCfg.ProxyID, 10), - DbID: 0, - CollectionID: 0, - PartitionIDs: nil, - Dsl: "", - PlaceholderGroup: nil, - DslType: 0, - SerializedExprPlan: nil, - OutputFieldsId: nil, - TravelTimestamp: 0, - GuaranteeTimestamp: 0, - }, - ctx: ctx, - resultBuf: make(chan []*internalpb.SearchResults), - result: nil, - query: req, - chMgr: chMgr, - qc: qc, - tr: timerecord.NewTimeRecorder("search"), - } - - // simple mock for query node - // TODO(dragondriver): should we replace this mock using RocksMq or MemMsgStream? - - err = chMgr.createDQLStream(collectionID) - assert.NoError(t, err) - stream, err := chMgr.getDQLStream(collectionID) - assert.NoError(t, err) - - var wg sync.WaitGroup - wg.Add(1) - consumeCtx, cancel := context.WithCancel(ctx) - go func() { - defer wg.Done() - for { - select { - case <-consumeCtx.Done(): - return - case pack, ok := <-stream.Chan(): - assert.True(t, ok) - if pack == nil { - continue - } - - for _, msg := range pack.Msgs { - _, ok := msg.(*msgstream.SearchMsg) - assert.True(t, ok) - // TODO(dragondriver): construct result according to the request - - constructSearchResulstData := func(invalidNum int) *schemapb.SearchResultData { - resultData := &schemapb.SearchResultData{ - NumQueries: int64(nq), - TopK: int64(topk), - FieldsData: nil, - Scores: make([]float32, nq*topk), - Ids: &schemapb.IDs{ - IdField: &schemapb.IDs_IntId{ - IntId: &schemapb.LongArray{ - Data: make([]int64, nq*topk), - }, - }, - }, - Topks: make([]int64, nq), - } - - for i := 0; i < nq; i++ { - for j := 0; j < topk; j++ { - offset := i*topk + j - if j >= invalidNum { - resultData.Scores[offset] = minFloat32 - resultData.Ids.IdField.(*schemapb.IDs_IntId).IntId.Data[offset] = -1 - } else { - score := float32(uniquegenerator.GetUniqueIntGeneratorIns().GetInt()) // increasingly - id := int64(uniquegenerator.GetUniqueIntGeneratorIns().GetInt()) - resultData.Scores[offset] = score - resultData.Ids.IdField.(*schemapb.IDs_IntId).IntId.Data[offset] = id - } - } - resultData.Topks[i] = int64(topk) - } - - return resultData - } - - result1 := &internalpb.SearchResults{ - Base: &commonpb.MsgBase{ - MsgType: commonpb.MsgType_SearchResult, - MsgID: 0, - Timestamp: 0, - SourceID: 0, - }, - Status: &commonpb.Status{ - ErrorCode: commonpb.ErrorCode_Success, - Reason: "", - }, - ResultChannelID: "", - MetricType: distance.L2, - NumQueries: int64(nq), - TopK: int64(topk), - SealedSegmentIDsSearched: nil, - ChannelIDsSearched: nil, - GlobalSealedSegmentIDs: nil, - SlicedBlob: nil, - SlicedNumCount: 1, - SlicedOffset: 0, - } - resultData := constructSearchResulstData(topk / 2) - sliceBlob, err := proto.Marshal(resultData) - assert.NoError(t, err) - result1.SlicedBlob = sliceBlob - - result2 := &internalpb.SearchResults{ - Base: &commonpb.MsgBase{ - MsgType: commonpb.MsgType_SearchResult, - MsgID: 0, - Timestamp: 0, - SourceID: 0, - }, - Status: &commonpb.Status{ - ErrorCode: commonpb.ErrorCode_Success, - Reason: "", - }, - ResultChannelID: "", - MetricType: distance.L2, - NumQueries: int64(nq), - TopK: int64(topk), - SealedSegmentIDsSearched: nil, - ChannelIDsSearched: nil, - GlobalSealedSegmentIDs: nil, - SlicedBlob: nil, - SlicedNumCount: 1, - SlicedOffset: 0, - } - resultData2 := constructSearchResulstData(topk - topk/2) - sliceBlob2, err := proto.Marshal(resultData2) - assert.NoError(t, err) - result2.SlicedBlob = sliceBlob2 - - // send search result - task.resultBuf <- []*internalpb.SearchResults{result1, result2} - } - } - } - }() - - assert.NoError(t, task.OnEnqueue()) - assert.NoError(t, task.PreExecute(ctx)) - assert.NoError(t, task.Execute(ctx)) - assert.NoError(t, task.PostExecute(ctx)) - - cancel() - wg.Wait() +func TestSearchTaskV2_all(t *testing.T) { + // var err error + // + // Params.Init() + // Params.ProxyCfg.SearchResultChannelNames = []string{funcutil.GenRandomStr()} + // + // rc := NewRootCoordMock() + // rc.Start() + // defer rc.Stop() + // + // ctx := context.Background() + // + // err = InitMetaCache(rc) + // assert.NoError(t, err) + // + // shardsNum := int32(2) + // prefix := "TestSearchTaskV2_all" + // collectionName := prefix + funcutil.GenRandomStr() + // + // dim := 128 + // expr := fmt.Sprintf("%s > 0", testInt64Field) + // nq := 10 + // topk := 10 + // roundDecimal := 3 + // nprobe := 10 + // + // fieldName2Types := map[string]schemapb.DataType{ + // testBoolField: schemapb.DataType_Bool, + // testInt32Field: schemapb.DataType_Int32, + // testInt64Field: schemapb.DataType_Int64, + // testFloatField: schemapb.DataType_Float, + // testDoubleField: schemapb.DataType_Double, + // testFloatVecField: schemapb.DataType_FloatVector, + // } + // if enableMultipleVectorFields { + // fieldName2Types[testBinaryVecField] = schemapb.DataType_BinaryVector + // } + // + // schema := constructCollectionSchemaByDataType(collectionName, fieldName2Types, testInt64Field, false) + // marshaledSchema, err := proto.Marshal(schema) + // assert.NoError(t, err) + // + // createColT := &createCollectionTask{ + // Condition: NewTaskCondition(ctx), + // CreateCollectionRequest: &milvuspb.CreateCollectionRequest{ + // Base: nil, + // CollectionName: collectionName, + // Schema: marshaledSchema, + // ShardsNum: shardsNum, + // }, + // ctx: ctx, + // rootCoord: rc, + // result: nil, + // schema: nil, + // } + // + // assert.NoError(t, createColT.OnEnqueue()) + // assert.NoError(t, createColT.PreExecute(ctx)) + // assert.NoError(t, createColT.Execute(ctx)) + // assert.NoError(t, createColT.PostExecute(ctx)) + // + // dmlChannelsFunc := getDmlChannelsFunc(ctx, rc) + // query := newMockGetChannelsService() + // factory := newSimpleMockMsgStreamFactory() + // + // collectionID, err := globalMetaCache.GetCollectionID(ctx, collectionName) + // assert.NoError(t, err) + // + // qc := NewQueryCoordMock() + // qc.Start() + // defer qc.Stop() + // status, err := qc.LoadCollection(ctx, &querypb.LoadCollectionRequest{ + // Base: &commonpb.MsgBase{ + // MsgType: commonpb.MsgType_LoadCollection, + // MsgID: 0, + // Timestamp: 0, + // SourceID: Params.ProxyCfg.ProxyID, + // }, + // DbID: 0, + // CollectionID: collectionID, + // Schema: nil, + // }) + // assert.NoError(t, err) + // assert.Equal(t, commonpb.ErrorCode_Success, status.ErrorCode) + // + // req := constructSearchRequest("", collectionName, + // expr, + // testFloatVecField, + // nq, dim, nprobe, topk, roundDecimal) + // + // task := &searchTaskV2{ + // Condition: NewTaskCondition(ctx), + // SearchRequest: &internalpb.SearchRequest{ + // Base: &commonpb.MsgBase{ + // MsgType: commonpb.MsgType_Search, + // MsgID: 0, + // Timestamp: 0, + // SourceID: Params.ProxyCfg.ProxyID, + // }, + // ResultChannelID: strconv.FormatInt(Params.ProxyCfg.ProxyID, 10), + // DbID: 0, + // CollectionID: 0, + // PartitionIDs: nil, + // Dsl: "", + // PlaceholderGroup: nil, + // DslType: 0, + // SerializedExprPlan: nil, + // OutputFieldsId: nil, + // TravelTimestamp: 0, + // GuaranteeTimestamp: 0, + // }, + // ctx: ctx, + // resultBuf: make(chan *internalpb.SearchResults, 10), + // result: nil, + // request: req, + // qc: qc, + // tr: timerecord.NewTimeRecorder("search"), + // } + // + // // simple mock for query node + // // TODO(dragondriver): should we replace this mock using RocksMq or MemMsgStream? + // + // var wg sync.WaitGroup + // wg.Add(1) + // consumeCtx, cancel := context.WithCancel(ctx) + // go func() { + // defer wg.Done() + // for { + // select { + // case <-consumeCtx.Done(): + // return + // case pack, ok := <-stream.Chan(): + // assert.True(t, ok) + // if pack == nil { + // continue + // } + // + // for _, msg := range pack.Msgs { + // _, ok := msg.(*msgstream.SearchMsg) + // assert.True(t, ok) + // // TODO(dragondriver): construct result according to the request + // + // constructSearchResulstData := func() *schemapb.SearchResultData { + // resultData := &schemapb.SearchResultData{ + // NumQueries: int64(nq), + // TopK: int64(topk), + // Scores: make([]float32, nq*topk), + // Ids: &schemapb.IDs{ + // IdField: &schemapb.IDs_IntId{ + // IntId: &schemapb.LongArray{ + // Data: make([]int64, nq*topk), + // }, + // }, + // }, + // Topks: make([]int64, nq), + // } + // + // fieldID := common.StartOfUserFieldID + // for fieldName, dataType := range fieldName2Types { + // resultData.FieldsData = append(resultData.FieldsData, generateFieldData(dataType, fieldName, int64(fieldID), nq*topk)) + // fieldID++ + // } + // + // for i := 0; i < nq; i++ { + // for j := 0; j < topk; j++ { + // offset := i*topk + j + // score := float32(uniquegenerator.GetUniqueIntGeneratorIns().GetInt()) // increasingly + // id := int64(uniquegenerator.GetUniqueIntGeneratorIns().GetInt()) + // resultData.Scores[offset] = score + // resultData.Ids.IdField.(*schemapb.IDs_IntId).IntId.Data[offset] = id + // } + // resultData.Topks[i] = int64(topk) + // } + // + // return resultData + // } + // + // result1 := &internalpb.SearchResults{ + // Base: &commonpb.MsgBase{ + // MsgType: commonpb.MsgType_SearchResult, + // MsgID: 0, + // Timestamp: 0, + // SourceID: 0, + // }, + // Status: &commonpb.Status{ + // ErrorCode: commonpb.ErrorCode_Success, + // Reason: "", + // }, + // ResultChannelID: "", + // MetricType: distance.L2, + // NumQueries: int64(nq), + // TopK: int64(topk), + // SealedSegmentIDsSearched: nil, + // ChannelIDsSearched: nil, + // GlobalSealedSegmentIDs: nil, + // SlicedBlob: nil, + // SlicedNumCount: 1, + // SlicedOffset: 0, + // } + // resultData := constructSearchResulstData() + // sliceBlob, err := proto.Marshal(resultData) + // assert.NoError(t, err) + // result1.SlicedBlob = sliceBlob + // + // // result2.SliceBlob = nil, will be skipped in decode stage + // result2 := &internalpb.SearchResults{ + // Base: &commonpb.MsgBase{ + // MsgType: commonpb.MsgType_SearchResult, + // MsgID: 0, + // Timestamp: 0, + // SourceID: 0, + // }, + // Status: &commonpb.Status{ + // ErrorCode: commonpb.ErrorCode_Success, + // Reason: "", + // }, + // ResultChannelID: "", + // MetricType: distance.L2, + // NumQueries: int64(nq), + // TopK: int64(topk), + // SealedSegmentIDsSearched: nil, + // ChannelIDsSearched: nil, + // GlobalSealedSegmentIDs: nil, + // SlicedBlob: nil, + // SlicedNumCount: 1, + // SlicedOffset: 0, + // } + // + // // send search result + // task.resultBuf <- result1 + // task.resultBuf <- result2 + // } + // } + // } + // }() + // + // assert.NoError(t, task.OnEnqueue()) + // assert.NoError(t, task.PreExecute(ctx)) + // assert.NoError(t, task.Execute(ctx)) + // assert.NoError(t, task.PostExecute(ctx)) + // + // cancel() + // wg.Wait() +} + +func TestSearchTaskV2_7803_reduce(t *testing.T) { + // var err error + // + // Params.Init() + // Params.ProxyCfg.SearchResultChannelNames = []string{funcutil.GenRandomStr()} + // + // rc := NewRootCoordMock() + // rc.Start() + // defer rc.Stop() + // + // ctx := context.Background() + // + // err = InitMetaCache(rc) + // assert.NoError(t, err) + // + // shardsNum := int32(2) + // prefix := "TestSearchTaskV2_7803_reduce" + // collectionName := prefix + funcutil.GenRandomStr() + // int64Field := "int64" + // floatVecField := "fvec" + // dim := 128 + // expr := fmt.Sprintf("%s > 0", int64Field) + // nq := 10 + // topk := 10 + // roundDecimal := 3 + // nprobe := 10 + // + // schema := constructCollectionSchema( + // int64Field, + // floatVecField, + // dim, + // collectionName) + // marshaledSchema, err := proto.Marshal(schema) + // assert.NoError(t, err) + // + // createColT := &createCollectionTask{ + // Condition: NewTaskCondition(ctx), + // CreateCollectionRequest: &milvuspb.CreateCollectionRequest{ + // Base: nil, + // CollectionName: collectionName, + // Schema: marshaledSchema, + // ShardsNum: shardsNum, + // }, + // ctx: ctx, + // rootCoord: rc, + // result: nil, + // schema: nil, + // } + // + // assert.NoError(t, createColT.OnEnqueue()) + // assert.NoError(t, createColT.PreExecute(ctx)) + // assert.NoError(t, createColT.Execute(ctx)) + // assert.NoError(t, createColT.PostExecute(ctx)) + // + // dmlChannelsFunc := getDmlChannelsFunc(ctx, rc) + // query := newMockGetChannelsService() + // factory := newSimpleMockMsgStreamFactory() + // + // collectionID, err := globalMetaCache.GetCollectionID(ctx, collectionName) + // assert.NoError(t, err) + // + // qc := NewQueryCoordMock() + // qc.Start() + // defer qc.Stop() + // status, err := qc.LoadCollection(ctx, &querypb.LoadCollectionRequest{ + // Base: &commonpb.MsgBase{ + // MsgType: commonpb.MsgType_LoadCollection, + // MsgID: 0, + // Timestamp: 0, + // SourceID: Params.ProxyCfg.ProxyID, + // }, + // DbID: 0, + // CollectionID: collectionID, + // Schema: nil, + // }) + // assert.NoError(t, err) + // assert.Equal(t, commonpb.ErrorCode_Success, status.ErrorCode) + // + // req := constructSearchRequest("", collectionName, + // expr, + // floatVecField, + // nq, dim, nprobe, topk, roundDecimal) + // + // task := &searchTaskV2{ + // Condition: NewTaskCondition(ctx), + // SearchRequest: &internalpb.SearchRequest{ + // Base: &commonpb.MsgBase{ + // MsgType: commonpb.MsgType_Search, + // MsgID: 0, + // Timestamp: 0, + // SourceID: Params.ProxyCfg.ProxyID, + // }, + // ResultChannelID: strconv.FormatInt(Params.ProxyCfg.ProxyID, 10), + // DbID: 0, + // CollectionID: 0, + // PartitionIDs: nil, + // Dsl: "", + // PlaceholderGroup: nil, + // DslType: 0, + // SerializedExprPlan: nil, + // OutputFieldsId: nil, + // TravelTimestamp: 0, + // GuaranteeTimestamp: 0, + // }, + // ctx: ctx, + // resultBuf: make(chan *internalpb.SearchResults, 10), + // result: nil, + // request: req, + // qc: qc, + // tr: timerecord.NewTimeRecorder("search"), + // } + // + // // simple mock for query node + // // TODO(dragondriver): should we replace this mock using RocksMq or MemMsgStream? + // + // var wg sync.WaitGroup + // wg.Add(1) + // consumeCtx, cancel := context.WithCancel(ctx) + // go func() { + // defer wg.Done() + // for { + // select { + // case <-consumeCtx.Done(): + // return + // case pack, ok := <-stream.Chan(): + // assert.True(t, ok) + // if pack == nil { + // continue + // } + // + // for _, msg := range pack.Msgs { + // _, ok := msg.(*msgstream.SearchMsg) + // assert.True(t, ok) + // // TODO(dragondriver): construct result according to the request + // + // constructSearchResulstData := func(invalidNum int) *schemapb.SearchResultData { + // resultData := &schemapb.SearchResultData{ + // NumQueries: int64(nq), + // TopK: int64(topk), + // FieldsData: nil, + // Scores: make([]float32, nq*topk), + // Ids: &schemapb.IDs{ + // IdField: &schemapb.IDs_IntId{ + // IntId: &schemapb.LongArray{ + // Data: make([]int64, nq*topk), + // }, + // }, + // }, + // Topks: make([]int64, nq), + // } + // + // for i := 0; i < nq; i++ { + // for j := 0; j < topk; j++ { + // offset := i*topk + j + // if j >= invalidNum { + // resultData.Scores[offset] = minFloat32 + // resultData.Ids.IdField.(*schemapb.IDs_IntId).IntId.Data[offset] = -1 + // } else { + // score := float32(uniquegenerator.GetUniqueIntGeneratorIns().GetInt()) // increasingly + // id := int64(uniquegenerator.GetUniqueIntGeneratorIns().GetInt()) + // resultData.Scores[offset] = score + // resultData.Ids.IdField.(*schemapb.IDs_IntId).IntId.Data[offset] = id + // } + // } + // resultData.Topks[i] = int64(topk) + // } + // + // return resultData + // } + // + // result1 := &internalpb.SearchResults{ + // Base: &commonpb.MsgBase{ + // MsgType: commonpb.MsgType_SearchResult, + // MsgID: 0, + // Timestamp: 0, + // SourceID: 0, + // }, + // Status: &commonpb.Status{ + // ErrorCode: commonpb.ErrorCode_Success, + // Reason: "", + // }, + // ResultChannelID: "", + // MetricType: distance.L2, + // NumQueries: int64(nq), + // TopK: int64(topk), + // SealedSegmentIDsSearched: nil, + // ChannelIDsSearched: nil, + // GlobalSealedSegmentIDs: nil, + // SlicedBlob: nil, + // SlicedNumCount: 1, + // SlicedOffset: 0, + // } + // resultData := constructSearchResulstData(topk / 2) + // sliceBlob, err := proto.Marshal(resultData) + // assert.NoError(t, err) + // result1.SlicedBlob = sliceBlob + // + // result2 := &internalpb.SearchResults{ + // Base: &commonpb.MsgBase{ + // MsgType: commonpb.MsgType_SearchResult, + // MsgID: 0, + // Timestamp: 0, + // SourceID: 0, + // }, + // Status: &commonpb.Status{ + // ErrorCode: commonpb.ErrorCode_Success, + // Reason: "", + // }, + // ResultChannelID: "", + // MetricType: distance.L2, + // NumQueries: int64(nq), + // TopK: int64(topk), + // SealedSegmentIDsSearched: nil, + // ChannelIDsSearched: nil, + // GlobalSealedSegmentIDs: nil, + // SlicedBlob: nil, + // SlicedNumCount: 1, + // SlicedOffset: 0, + // } + // resultData2 := constructSearchResulstData(topk - topk/2) + // sliceBlob2, err := proto.Marshal(resultData2) + // assert.NoError(t, err) + // result2.SlicedBlob = sliceBlob2 + // + // // send search result + // task.resultBuf <- result1 + // task.resultBuf <- result2 + // } + // } + // } + // }() + // + // assert.NoError(t, task.OnEnqueue()) + // assert.NoError(t, task.PreExecute(ctx)) + // assert.NoError(t, task.Execute(ctx)) + // assert.NoError(t, task.PostExecute(ctx)) + // + // cancel() + // wg.Wait() } diff --git a/internal/querycoord/channel_allocator.go b/internal/querycoord/channel_allocator.go index db821bc3a1..5b32e4604f 100644 --- a/internal/querycoord/channel_allocator.go +++ b/internal/querycoord/channel_allocator.go @@ -33,14 +33,23 @@ func defaultChannelAllocatePolicy() ChannelAllocatePolicy { } // ChannelAllocatePolicy helper function definition to allocate dmChannel to queryNode -type ChannelAllocatePolicy func(ctx context.Context, reqs []*querypb.WatchDmChannelsRequest, cluster Cluster, metaCache Meta, wait bool, excludeNodeIDs []int64) error +type ChannelAllocatePolicy func(ctx context.Context, reqs []*querypb.WatchDmChannelsRequest, cluster Cluster, metaCache Meta, wait bool, excludeNodeIDs []int64, includeNodeIDs []int64, replicaID int64) error -func shuffleChannelsToQueryNode(ctx context.Context, reqs []*querypb.WatchDmChannelsRequest, cluster Cluster, metaCache Meta, wait bool, excludeNodeIDs []int64) error { +func shuffleChannelsToQueryNode(ctx context.Context, reqs []*querypb.WatchDmChannelsRequest, cluster Cluster, metaCache Meta, wait bool, excludeNodeIDs []int64, includeNodeIDs []int64, replicaID int64) error { if len(reqs) == 0 { return nil } + var onlineNodeIDs []int64 for { - onlineNodeIDs := cluster.onlineNodeIDs() + if replicaID == -1 { + onlineNodeIDs = cluster.onlineNodeIDs() + } else { + replica, err := metaCache.getReplicaByID(replicaID) + if err != nil { + return err + } + onlineNodeIDs = replica.GetNodeIds() + } if len(onlineNodeIDs) == 0 { err := errors.New("no online QueryNode to allocate") log.Error("shuffleChannelsToQueryNode failed", zap.Error(err)) @@ -54,6 +63,11 @@ func shuffleChannelsToQueryNode(ctx context.Context, reqs []*querypb.WatchDmChan var availableNodeIDs []int64 nodeID2NumChannels := make(map[int64]int) for _, nodeID := range onlineNodeIDs { + // nodeID not in includeNodeIDs + if len(includeNodeIDs) > 0 && !nodeIncluded(nodeID, includeNodeIDs) { + continue + } + // nodeID in excludeNodeIDs if nodeIncluded(nodeID, excludeNodeIDs) { continue diff --git a/internal/querycoord/channel_allocator_test.go b/internal/querycoord/channel_allocator_test.go index 36b0a2024a..9477c5f742 100644 --- a/internal/querycoord/channel_allocator_test.go +++ b/internal/querycoord/channel_allocator_test.go @@ -18,6 +18,8 @@ package querycoord import ( "context" + "math/rand" + "sync/atomic" "testing" "github.com/stretchr/testify/assert" @@ -41,7 +43,12 @@ func TestShuffleChannelsToQueryNode(t *testing.T) { clusterSession := sessionutil.NewSession(context.Background(), Params.EtcdCfg.MetaRootPath, etcdCli) clusterSession.Init(typeutil.QueryCoordRole, Params.QueryCoordCfg.Address, true, false) clusterSession.Register() - meta, err := newMeta(baseCtx, kv, nil, nil) + id := UniqueID(rand.Int31()) + idAllocator := func() (UniqueID, error) { + newID := atomic.AddInt64(&id, 1) + return newID, nil + } + meta, err := newMeta(baseCtx, kv, nil, idAllocator) assert.Nil(t, err) cluster := &queryNodeCluster{ ctx: baseCtx, @@ -73,7 +80,7 @@ func TestShuffleChannelsToQueryNode(t *testing.T) { } reqs := []*querypb.WatchDmChannelsRequest{firstReq, secondReq} - err = shuffleChannelsToQueryNode(baseCtx, reqs, cluster, meta, false, nil) + err = shuffleChannelsToQueryNode(baseCtx, reqs, cluster, meta, false, nil, nil, -1) assert.NotNil(t, err) node, err := startQueryNodeServer(baseCtx) @@ -83,7 +90,7 @@ func TestShuffleChannelsToQueryNode(t *testing.T) { cluster.registerNode(baseCtx, nodeSession, nodeID, disConnect) waitQueryNodeOnline(cluster, nodeID) - err = shuffleChannelsToQueryNode(baseCtx, reqs, cluster, meta, false, nil) + err = shuffleChannelsToQueryNode(baseCtx, reqs, cluster, meta, false, nil, nil, -1) assert.Nil(t, err) assert.Equal(t, nodeID, firstReq.NodeID) diff --git a/internal/querycoord/channel_unsubscribe.go b/internal/querycoord/channel_unsubscribe.go index 5b2284a809..eb47dee421 100644 --- a/internal/querycoord/channel_unsubscribe.go +++ b/internal/querycoord/channel_unsubscribe.go @@ -30,6 +30,7 @@ import ( "github.com/milvus-io/milvus/internal/log" "github.com/milvus-io/milvus/internal/mq/msgstream" "github.com/milvus-io/milvus/internal/proto/querypb" + "github.com/milvus-io/milvus/internal/util/dependency" "github.com/milvus-io/milvus/internal/util/funcutil" ) @@ -51,7 +52,7 @@ type channelUnsubscribeHandler struct { } // newChannelUnsubscribeHandler create a new handler service to unsubscribe channels -func newChannelUnsubscribeHandler(ctx context.Context, kv *etcdkv.EtcdKV, factory msgstream.Factory) (*channelUnsubscribeHandler, error) { +func newChannelUnsubscribeHandler(ctx context.Context, kv *etcdkv.EtcdKV, factory dependency.Factory) (*channelUnsubscribeHandler, error) { childCtx, cancel := context.WithCancel(ctx) handler := &channelUnsubscribeHandler{ ctx: childCtx, diff --git a/internal/querycoord/cluster.go b/internal/querycoord/cluster.go index 0ecde750f5..489546c670 100644 --- a/internal/querycoord/cluster.go +++ b/internal/querycoord/cluster.go @@ -21,8 +21,9 @@ import ( "encoding/json" "errors" "fmt" - "math/rand" + "math" "path/filepath" + "sort" "strconv" "sync" @@ -73,10 +74,10 @@ type Cluster interface { offlineNodeIDs() []int64 hasNode(nodeID int64) bool - allocateSegmentsToQueryNode(ctx context.Context, reqs []*querypb.LoadSegmentsRequest, wait bool, excludeNodeIDs []int64, includeNodeIDs []int64) error - allocateChannelsToQueryNode(ctx context.Context, reqs []*querypb.WatchDmChannelsRequest, wait bool, excludeNodeIDs []int64) error + allocateSegmentsToQueryNode(ctx context.Context, reqs []*querypb.LoadSegmentsRequest, wait bool, excludeNodeIDs []int64, includeNodeIDs []int64, replicaID int64) error + allocateChannelsToQueryNode(ctx context.Context, reqs []*querypb.WatchDmChannelsRequest, wait bool, excludeNodeIDs []int64, includeNodeIDs []int64, replicaID int64) error - assignNodesToReplicas(ctx context.Context, replicas []*milvuspb.ReplicaInfo) error + assignNodesToReplicas(ctx context.Context, replicas []*milvuspb.ReplicaInfo, collectionSize uint64) error getSessionVersion() int64 @@ -267,6 +268,7 @@ func (c *queryNodeCluster) watchDmChannels(ctx context.Context, nodeID int64, in CollectionID: info.CollectionID, DmChannel: info.ChannelName, NodeIDLoaded: nodeID, + ReplicaID: in.ReplicaID, } } @@ -696,24 +698,81 @@ func (c *queryNodeCluster) isOnline(nodeID int64) (bool, error) { // } //} -func (c *queryNodeCluster) allocateSegmentsToQueryNode(ctx context.Context, reqs []*querypb.LoadSegmentsRequest, wait bool, excludeNodeIDs []int64, includeNodeIDs []int64) error { - return c.segmentAllocator(ctx, reqs, c, c.clusterMeta, wait, excludeNodeIDs, includeNodeIDs) +func (c *queryNodeCluster) allocateSegmentsToQueryNode(ctx context.Context, reqs []*querypb.LoadSegmentsRequest, wait bool, excludeNodeIDs []int64, includeNodeIDs []int64, replicaID int64) error { + return c.segmentAllocator(ctx, reqs, c, c.clusterMeta, wait, excludeNodeIDs, includeNodeIDs, replicaID) } -func (c *queryNodeCluster) allocateChannelsToQueryNode(ctx context.Context, reqs []*querypb.WatchDmChannelsRequest, wait bool, excludeNodeIDs []int64) error { - return c.channelAllocator(ctx, reqs, c, c.clusterMeta, wait, excludeNodeIDs) +func (c *queryNodeCluster) allocateChannelsToQueryNode(ctx context.Context, reqs []*querypb.WatchDmChannelsRequest, wait bool, excludeNodeIDs []int64, includeNodeIDs []int64, replicaID int64) error { + return c.channelAllocator(ctx, reqs, c, c.clusterMeta, wait, excludeNodeIDs, includeNodeIDs, replicaID) } -func (c *queryNodeCluster) assignNodesToReplicas(ctx context.Context, replicas []*milvuspb.ReplicaInfo) error { - nodes := c.onlineNodeIDs() - if len(nodes) < len(replicas) { - return errors.New("no enough nodes to create replicas") +// Return error if no enough nodes/resources to create replicas +func (c *queryNodeCluster) assignNodesToReplicas(ctx context.Context, replicas []*milvuspb.ReplicaInfo, collectionSize uint64) error { + nodeIds := c.onlineNodeIDs() + if len(nodeIds) < len(replicas) { + return fmt.Errorf("no enough nodes to create replicas, node_num=%d replica_num=%d", len(nodeIds), len(replicas)) } - for _, node := range nodes { - idx := rand.Int() % len(replicas) - replicas[idx].NodeIds = append(replicas[idx].NodeIds, node) + nodeInfos, err := getNodeInfos(c, nodeIds) + if err != nil { + return err + } + if len(nodeInfos) < len(replicas) { + return fmt.Errorf("no enough nodes to create replicas, node_num=%d replica_num=%d", len(nodeInfos), len(replicas)) + } + + sort.Slice(nodeInfos, func(i, j int) bool { + return nodeInfos[i].totalMem-nodeInfos[i].memUsage > nodeInfos[j].totalMem-nodeInfos[j].memUsage + }) + + memCapCount := make([]uint64, len(replicas)) + for _, info := range nodeInfos { + i := 0 + minMemCap := uint64(math.MaxUint64) + for j, memCap := range memCapCount { + if memCap < minMemCap { + minMemCap = memCap + i = j + } + } + + replicas[i].NodeIds = append(replicas[i].NodeIds, info.id) + memCapCount[i] += info.totalMem - info.memUsage + } + + for _, memCap := range memCapCount { + if memCap < collectionSize { + return fmt.Errorf("no enough memory to load collection/partitions, collectionSize=%v, replicasNum=%v", collectionSize, len(replicas)) + } } return nil } + +// It's a helper method to concurrently get nodes' info +// Remove nodes that it can't connect to +func getNodeInfos(cluster *queryNodeCluster, nodeIds []UniqueID) ([]*queryNode, error) { + nodeCh := make(chan *queryNode, len(nodeIds)) + + wg := sync.WaitGroup{} + for _, id := range nodeIds { + wg.Add(1) + go func(id UniqueID) { + defer wg.Done() + info, err := cluster.getNodeInfoByID(id) + if err != nil { + return + } + nodeCh <- info.(*queryNode) + }(id) + } + wg.Wait() + close(nodeCh) + + nodes := make([]*queryNode, 0, len(nodeCh)) + for node := range nodeCh { + nodes = append(nodes, node) + } + + return nodes, nil +} diff --git a/internal/querycoord/cluster_test.go b/internal/querycoord/cluster_test.go index d781bc92c8..5e995aee19 100644 --- a/internal/querycoord/cluster_test.go +++ b/internal/querycoord/cluster_test.go @@ -21,8 +21,10 @@ import ( "encoding/json" "errors" "fmt" + "math/rand" "path" "strconv" + "sync/atomic" "testing" "github.com/milvus-io/milvus/internal/util/dependency" @@ -412,7 +414,12 @@ func TestReloadClusterFromKV(t *testing.T) { factory := dependency.NewDefaultFactory(true) handler, err := newChannelUnsubscribeHandler(ctx, kv, factory) assert.Nil(t, err) - meta, err := newMeta(ctx, kv, factory, nil) + id := UniqueID(rand.Int31()) + idAllocator := func() (UniqueID, error) { + newID := atomic.AddInt64(&id, 1) + return newID, nil + } + meta, err := newMeta(ctx, kv, factory, idAllocator) assert.Nil(t, err) cluster := &queryNodeCluster{ diff --git a/internal/querycoord/global_meta_broker.go b/internal/querycoord/global_meta_broker.go index 696665395b..b8b4fbb6c6 100644 --- a/internal/querycoord/global_meta_broker.go +++ b/internal/querycoord/global_meta_broker.go @@ -56,6 +56,8 @@ func (broker *globalMetaBroker) releaseDQLMessageStream(ctx context.Context, col }, CollectionID: collectionID, } + + // TODO(yah01): check whether RootCoord returns error if QueryChannel not exists res, err := broker.rootCoord.ReleaseDQLMessageStream(ctx2, releaseDQLMessageStreamReq) if err != nil { log.Error("releaseDQLMessageStream occur error", zap.Int64("collectionID", collectionID), zap.Error(err)) @@ -421,13 +423,14 @@ func (broker *globalMetaBroker) generateSegmentLoadInfo(ctx context.Context, schema *schemapb.CollectionSchema) *querypb.SegmentLoadInfo { segmentID := segmentBinlog.SegmentID segmentLoadInfo := &querypb.SegmentLoadInfo{ - SegmentID: segmentID, - PartitionID: partitionID, - CollectionID: collectionID, - BinlogPaths: segmentBinlog.FieldBinlogs, - NumOfRows: segmentBinlog.NumOfRows, - Statslogs: segmentBinlog.Statslogs, - Deltalogs: segmentBinlog.Deltalogs, + SegmentID: segmentID, + PartitionID: partitionID, + CollectionID: collectionID, + BinlogPaths: segmentBinlog.FieldBinlogs, + NumOfRows: segmentBinlog.NumOfRows, + Statslogs: segmentBinlog.Statslogs, + Deltalogs: segmentBinlog.Deltalogs, + InsertChannel: segmentBinlog.InsertChannel, } if setIndex { // if index not exist, load binlog to query node diff --git a/internal/querycoord/group_balance.go b/internal/querycoord/group_balance.go new file mode 100644 index 0000000000..938149a867 --- /dev/null +++ b/internal/querycoord/group_balance.go @@ -0,0 +1,57 @@ +package querycoord + +import "sort" + +type balancer interface { + addNode(nodeID int64) ([]*balancePlan, error) + removeNode(nodeID int64) []*balancePlan + rebalance() []*balancePlan +} + +type balancePlan struct { + nodeID int64 + sourceReplica int64 + targetReplica int64 +} + +type replicaBalancer struct { + meta Meta +} + +func newReplicaBalancer(meta Meta) *replicaBalancer { + return &replicaBalancer{meta} +} + +func (b *replicaBalancer) addNode(nodeID int64) ([]*balancePlan, error) { + // allocate this node to all collections replicas + var ret []*balancePlan + collections := b.meta.showCollections() + for _, c := range collections { + replicas, err := b.meta.getReplicasByCollectionID(c.GetCollectionID()) + if err != nil { + return nil, err + } + if len(replicas) == 0 { + continue + } + sort.Slice(replicas, func(i, j int) bool { + return len(replicas[i].GetNodeIds()) < len(replicas[j].GetNodeIds()) + }) + + ret = append(ret, &balancePlan{ + nodeID: nodeID, + sourceReplica: -1, + targetReplica: replicas[0].GetReplicaID(), + }) + } + return ret, nil +} + +func (b *replicaBalancer) removeNode(nodeID int64) []*balancePlan { + // for this version, querynode does not support move from a replica to another + return nil +} + +func (b *replicaBalancer) rebalance() []*balancePlan { + return nil +} diff --git a/internal/querycoord/impl.go b/internal/querycoord/impl.go index 9d3b5d5547..fdbcf2aee4 100644 --- a/internal/querycoord/impl.go +++ b/internal/querycoord/impl.go @@ -884,6 +884,7 @@ func (qc *QueryCoord) LoadBalance(ctx context.Context, req *querypb.LoadBalanceR zap.Int64s("source nodeIDs", req.SourceNodeIDs), zap.Int64s("dst nodeIDs", req.DstNodeIDs), zap.Int64s("balanced segments", req.SealedSegmentIDs), + zap.Int64("collectionID", req.CollectionID), zap.Int64("msgID", req.Base.MsgID)) status := &commonpb.Status{ @@ -930,6 +931,7 @@ func (qc *QueryCoord) LoadBalance(ctx context.Context, req *querypb.LoadBalanceR zap.Int64s("source nodeIDs", req.SourceNodeIDs), zap.Int64s("dst nodeIDs", req.DstNodeIDs), zap.Int64s("balanced segments", req.SealedSegmentIDs), + zap.Int64("collectionID", req.CollectionID), zap.Int64("msgID", req.Base.MsgID)) return status, nil @@ -1018,22 +1020,132 @@ func (qc *QueryCoord) GetMetrics(ctx context.Context, req *milvuspb.GetMetricsRe // GetReplicas gets replicas of a certain collection func (qc *QueryCoord) GetReplicas(ctx context.Context, req *milvuspb.GetReplicasRequest) (*milvuspb.GetReplicasResponse, error) { - // TODO: to impl + log.Debug("GetReplicas received", + zap.String("role", typeutil.QueryCoordRole), + zap.Int64("collectionID", req.CollectionID), + zap.Int64("msgID", req.Base.MsgID)) + + status := &commonpb.Status{ + ErrorCode: commonpb.ErrorCode_Success, + } + + if qc.stateCode.Load() != internalpb.StateCode_Healthy { + status.ErrorCode = commonpb.ErrorCode_UnexpectedError + err := errors.New("QueryCoord is not healthy") + status.Reason = err.Error() + log.Error("GetReplicasResponse failed", zap.String("role", typeutil.QueryCoordRole), zap.Int64("msgID", req.Base.MsgID), zap.Error(err)) + return &milvuspb.GetReplicasResponse{ + Status: status, + }, nil + } + + replicas, err := qc.meta.getReplicasByCollectionID(req.CollectionID) + if err != nil { + status.ErrorCode = commonpb.ErrorCode_MetaFailed + status.Reason = err.Error() + log.Error("GetReplicasResponse failed to get replicas", + zap.String("role", typeutil.QueryCoordRole), + zap.Int64("collectionID", req.CollectionID), + zap.Int64("msgID", req.Base.MsgID), + zap.Error(err)) + + return &milvuspb.GetReplicasResponse{ + Status: status, + }, nil + } + + if req.WithShardNodes { + shardNodes := make(map[string]map[UniqueID]struct{}) + segments := qc.meta.showSegmentInfos(req.CollectionID, nil) + for _, segment := range segments { + nodes, ok := shardNodes[segment.DmChannel] + if !ok { + nodes = make(map[UniqueID]struct{}) + } + + for _, nodeID := range segment.NodeIds { + nodes[nodeID] = struct{}{} + } + + shardNodes[segment.DmChannel] = nodes + } + + for _, replica := range replicas { + for _, shard := range replica.ShardReplicas { + for nodeID := range shardNodes[shard.DmChannelName] { + shard.NodeIds = append(shard.NodeIds, nodeID) + } + } + } + } + return &milvuspb.GetReplicasResponse{ - Status: &commonpb.Status{ - ErrorCode: commonpb.ErrorCode_UnexpectedError, - Reason: "Not implemented", - }, + Status: status, + Replicas: replicas, }, nil } // GetShardLeaders gets shard leaders of a certain collection func (qc *QueryCoord) GetShardLeaders(ctx context.Context, req *querypb.GetShardLeadersRequest) (*querypb.GetShardLeadersResponse, error) { - // TODO: to impl + log.Debug("GetShardLeaders received", + zap.String("role", typeutil.QueryCoordRole), + zap.Int64("collectionID", req.CollectionID), + zap.Int64("msgID", req.Base.MsgID)) + + status := &commonpb.Status{ + ErrorCode: commonpb.ErrorCode_Success, + } + + if qc.stateCode.Load() != internalpb.StateCode_Healthy { + status.ErrorCode = commonpb.ErrorCode_UnexpectedError + err := errors.New("QueryCoord is not healthy") + status.Reason = err.Error() + log.Error("GetShardLeadersResponse failed", zap.String("role", typeutil.QueryCoordRole), zap.Int64("msgID", req.Base.MsgID), zap.Error(err)) + return &querypb.GetShardLeadersResponse{ + Status: status, + }, nil + } + + replicas, err := qc.meta.getReplicasByCollectionID(req.CollectionID) + if err != nil { + status.ErrorCode = commonpb.ErrorCode_MetaFailed + status.Reason = err.Error() + log.Error("GetShardLeadersResponse failed to get replicas", + zap.String("role", typeutil.QueryCoordRole), + zap.Int64("collectionID", req.CollectionID), + zap.Int64("msgID", req.Base.MsgID), + zap.Error(err)) + + return &querypb.GetShardLeadersResponse{ + Status: status, + }, nil + } + + shards := make(map[string]*querypb.ShardLeadersList) + for _, replica := range replicas { + for _, shard := range replica.ShardReplicas { + list, ok := shards[shard.DmChannelName] + if !ok { + list = &querypb.ShardLeadersList{ + ChannelName: shard.DmChannelName, + NodeIds: make([]int64, 0), + NodeAddrs: make([]string, 0), + } + } + + list.NodeIds = append(list.NodeIds, shard.LeaderID) + list.NodeAddrs = append(list.NodeAddrs, shard.LeaderAddr) + shards[shard.DmChannelName] = list + } + } + + shardLeaderLists := make([]*querypb.ShardLeadersList, 0, len(shards)) + for _, shard := range shards { + shardLeaderLists = append(shardLeaderLists, shard) + } + return &querypb.GetShardLeadersResponse{ - Status: &commonpb.Status{ - ErrorCode: commonpb.ErrorCode_UnexpectedError, - Reason: "Not implemented", - }, + Status: status, + Shards: shardLeaderLists, }, nil } diff --git a/internal/querycoord/impl_test.go b/internal/querycoord/impl_test.go index c07c169e0e..6aaad7c76a 100644 --- a/internal/querycoord/impl_test.go +++ b/internal/querycoord/impl_test.go @@ -24,7 +24,9 @@ import ( "time" "github.com/milvus-io/milvus/internal/common" + "github.com/milvus-io/milvus/internal/log" "github.com/milvus-io/milvus/internal/util/sessionutil" + "go.uber.org/zap" "github.com/stretchr/testify/assert" @@ -120,8 +122,9 @@ func TestGrpcTask(t *testing.T) { Base: &commonpb.MsgBase{ MsgType: commonpb.MsgType_LoadPartitions, }, - CollectionID: defaultCollectionID, - Schema: genDefaultCollectionSchema(false), + CollectionID: defaultCollectionID, + Schema: genDefaultCollectionSchema(false), + ReplicaNumber: 1, }) assert.Equal(t, commonpb.ErrorCode_UnexpectedError, status.ErrorCode) assert.Nil(t, err) @@ -132,9 +135,10 @@ func TestGrpcTask(t *testing.T) { Base: &commonpb.MsgBase{ MsgType: commonpb.MsgType_LoadPartitions, }, - CollectionID: defaultCollectionID, - PartitionIDs: []UniqueID{defaultPartitionID}, - Schema: genDefaultCollectionSchema(false), + CollectionID: defaultCollectionID, + PartitionIDs: []UniqueID{defaultPartitionID}, + Schema: genDefaultCollectionSchema(false), + ReplicaNumber: 1, }) assert.Equal(t, commonpb.ErrorCode_Success, status.ErrorCode) assert.Nil(t, err) @@ -227,8 +231,9 @@ func TestGrpcTask(t *testing.T) { Base: &commonpb.MsgBase{ MsgType: commonpb.MsgType_LoadCollection, }, - CollectionID: defaultCollectionID, - Schema: genDefaultCollectionSchema(false), + CollectionID: defaultCollectionID, + Schema: genDefaultCollectionSchema(false), + ReplicaNumber: 1, }) assert.Equal(t, commonpb.ErrorCode_Success, status.ErrorCode) assert.Nil(t, err) @@ -431,9 +436,10 @@ func TestGrpcTaskEnqueueFail(t *testing.T) { Base: &commonpb.MsgBase{ MsgType: commonpb.MsgType_LoadPartitions, }, - CollectionID: defaultCollectionID, - PartitionIDs: []UniqueID{defaultPartitionID}, - Schema: genDefaultCollectionSchema(false), + CollectionID: defaultCollectionID, + PartitionIDs: []UniqueID{defaultPartitionID}, + Schema: genDefaultCollectionSchema(false), + ReplicaNumber: 1, }) assert.Equal(t, commonpb.ErrorCode_UnexpectedError, status.ErrorCode) assert.Nil(t, err) @@ -444,8 +450,9 @@ func TestGrpcTaskEnqueueFail(t *testing.T) { Base: &commonpb.MsgBase{ MsgType: commonpb.MsgType_LoadCollection, }, - CollectionID: defaultCollectionID, - Schema: genDefaultCollectionSchema(false), + CollectionID: defaultCollectionID, + Schema: genDefaultCollectionSchema(false), + ReplicaNumber: 1, }) assert.Equal(t, commonpb.ErrorCode_UnexpectedError, status.ErrorCode) assert.Nil(t, err) @@ -456,8 +463,9 @@ func TestGrpcTaskEnqueueFail(t *testing.T) { Base: &commonpb.MsgBase{ MsgType: commonpb.MsgType_LoadCollection, }, - CollectionID: defaultCollectionID, - Schema: genDefaultCollectionSchema(false), + CollectionID: defaultCollectionID, + Schema: genDefaultCollectionSchema(false), + ReplicaNumber: 1, }) assert.Equal(t, commonpb.ErrorCode_Success, status.ErrorCode) assert.Nil(t, err) @@ -489,9 +497,10 @@ func TestGrpcTaskEnqueueFail(t *testing.T) { Base: &commonpb.MsgBase{ MsgType: commonpb.MsgType_LoadPartitions, }, - CollectionID: defaultCollectionID, - PartitionIDs: []UniqueID{defaultPartitionID}, - Schema: genDefaultCollectionSchema(false), + CollectionID: defaultCollectionID, + PartitionIDs: []UniqueID{defaultPartitionID}, + Schema: genDefaultCollectionSchema(false), + ReplicaNumber: 1, }) assert.Equal(t, commonpb.ErrorCode_Success, status.ErrorCode) @@ -543,8 +552,9 @@ func TestLoadBalanceTask(t *testing.T) { Base: &commonpb.MsgBase{ MsgType: commonpb.MsgType_LoadCollection, }, - CollectionID: defaultCollectionID, - Schema: genDefaultCollectionSchema(false), + CollectionID: defaultCollectionID, + Schema: genDefaultCollectionSchema(false), + ReplicaNumber: 1, }) assert.Nil(t, err) assert.Equal(t, commonpb.ErrorCode_Success, res.ErrorCode) @@ -606,9 +616,10 @@ func TestGrpcTaskBeforeHealthy(t *testing.T) { Base: &commonpb.MsgBase{ MsgType: commonpb.MsgType_LoadPartitions, }, - CollectionID: defaultCollectionID, - PartitionIDs: []UniqueID{defaultPartitionID}, - Schema: genDefaultCollectionSchema(false), + CollectionID: defaultCollectionID, + PartitionIDs: []UniqueID{defaultPartitionID}, + Schema: genDefaultCollectionSchema(false), + ReplicaNumber: 1, }) assert.Equal(t, commonpb.ErrorCode_UnexpectedError, status.ErrorCode) assert.Nil(t, err) @@ -654,8 +665,9 @@ func TestGrpcTaskBeforeHealthy(t *testing.T) { Base: &commonpb.MsgBase{ MsgType: commonpb.MsgType_LoadCollection, }, - CollectionID: defaultCollectionID, - Schema: genDefaultCollectionSchema(false), + CollectionID: defaultCollectionID, + Schema: genDefaultCollectionSchema(false), + ReplicaNumber: 1, }) assert.Equal(t, commonpb.ErrorCode_UnexpectedError, status.ErrorCode) assert.Nil(t, err) @@ -756,6 +768,24 @@ func TestGrpcTaskBeforeHealthy(t *testing.T) { assert.Equal(t, commonpb.ErrorCode_UnexpectedError, res.Status.ErrorCode) }) + t.Run("Test GetReplicas", func(t *testing.T) { + resp, err := unHealthyCoord.GetReplicas(ctx, &milvuspb.GetReplicasRequest{ + Base: &commonpb.MsgBase{}, + CollectionID: defaultCollectionID, + }) + assert.NoError(t, err) + assert.Equal(t, commonpb.ErrorCode_UnexpectedError, resp.Status.ErrorCode) + }) + + t.Run("Test GetShardLeaders", func(t *testing.T) { + resp, err := unHealthyCoord.GetShardLeaders(ctx, &querypb.GetShardLeadersRequest{ + Base: &commonpb.MsgBase{}, + CollectionID: defaultCollectionID, + }) + assert.NoError(t, err) + assert.Equal(t, commonpb.ErrorCode_UnexpectedError, resp.Status.ErrorCode) + }) + unHealthyCoord.Stop() err = removeAllSession() assert.Nil(t, err) @@ -789,8 +819,9 @@ func Test_RepeatedLoadSameCollection(t *testing.T) { Base: &commonpb.MsgBase{ MsgType: commonpb.MsgType_LoadCollection, }, - CollectionID: defaultCollectionID, - Schema: genDefaultCollectionSchema(false), + CollectionID: defaultCollectionID, + Schema: genDefaultCollectionSchema(false), + ReplicaNumber: 1, } //first load defaultCollectionID @@ -824,17 +855,19 @@ func Test_LoadCollectionAndLoadPartitions(t *testing.T) { Base: &commonpb.MsgBase{ MsgType: commonpb.MsgType_LoadCollection, }, - CollectionID: defaultCollectionID, - Schema: genDefaultCollectionSchema(false), + CollectionID: defaultCollectionID, + Schema: genDefaultCollectionSchema(false), + ReplicaNumber: 1, } loadPartitionReq := &querypb.LoadPartitionsRequest{ Base: &commonpb.MsgBase{ MsgType: commonpb.MsgType_LoadPartitions, }, - CollectionID: defaultCollectionID, - PartitionIDs: []UniqueID{defaultPartitionID}, - Schema: genDefaultCollectionSchema(false), + CollectionID: defaultCollectionID, + PartitionIDs: []UniqueID{defaultPartitionID}, + Schema: genDefaultCollectionSchema(false), + ReplicaNumber: 1, } //first load defaultCollectionID @@ -854,6 +887,131 @@ func Test_LoadCollectionAndLoadPartitions(t *testing.T) { assert.Nil(t, err) } +func TestLoadCollectionWithReplicas(t *testing.T) { + refreshParams() + ctx := context.Background() + queryCoord, err := startQueryCoord(ctx) + assert.Nil(t, err) + + node1, err := startQueryNodeServer(ctx) + assert.Nil(t, err) + node2, err := startQueryNodeServer(ctx) + assert.Nil(t, err) + + waitQueryNodeOnline(queryCoord.cluster, node1.queryNodeID) + waitQueryNodeOnline(queryCoord.cluster, node2.queryNodeID) + loadCollectionReq := &querypb.LoadCollectionRequest{ + Base: &commonpb.MsgBase{ + MsgType: commonpb.MsgType_LoadCollection, + }, + CollectionID: defaultCollectionID, + Schema: genDefaultCollectionSchema(false), + ReplicaNumber: 3, + } + + // load collection with 3 replicas, but no enough querynodes + assert.Equal(t, 2, len(queryCoord.cluster.onlineNodeIDs())) + status, err := queryCoord.LoadCollection(ctx, loadCollectionReq) + assert.NoError(t, err) + assert.Equal(t, commonpb.ErrorCode_UnexpectedError, status.ErrorCode) + + // Now it should can load collection with 3 replicas + node3, err := startQueryNodeServer(ctx) + assert.Nil(t, err) + waitQueryNodeOnline(queryCoord.cluster, node3.queryNodeID) + + status, err = queryCoord.LoadCollection(ctx, loadCollectionReq) + assert.NoError(t, err) + assert.Equal(t, commonpb.ErrorCode_Success, status.ErrorCode) + waitLoadCollectionDone(ctx, queryCoord, defaultCollectionID) + replicas, err := queryCoord.meta.getReplicasByCollectionID(loadCollectionReq.CollectionID) + assert.NoError(t, err) + for i := range replicas { + log.Info("replicas", + zap.Int64("collectionID", replicas[i].CollectionID), + zap.Int64("id", replicas[i].ReplicaID), + zap.Int64s("nodeIds", replicas[i].NodeIds)) + } + assert.Equal(t, 3, len(replicas)) + for i := range replicas { + assert.Equal(t, loadCollectionReq.CollectionID, replicas[i].CollectionID) + } + + status, err = queryCoord.ReleaseCollection(ctx, &querypb.ReleaseCollectionRequest{ + Base: &commonpb.MsgBase{ + MsgType: commonpb.MsgType_ReleaseCollection, + }, + CollectionID: loadCollectionReq.CollectionID, + }) + assert.NoError(t, err) + assert.Equal(t, commonpb.ErrorCode_Success, status.ErrorCode) + + node1.stop() + node2.stop() + node3.stop() + queryCoord.Stop() + err = removeAllSession() + assert.Nil(t, err) +} + +func Test_LoadPartitionsWithReplicas(t *testing.T) { + refreshParams() + ctx := context.Background() + queryCoord, err := startQueryCoord(ctx) + assert.Nil(t, err) + + node1, err := startQueryNodeServer(ctx) + assert.Nil(t, err) + node2, err := startQueryNodeServer(ctx) + assert.Nil(t, err) + + waitQueryNodeOnline(queryCoord.cluster, node1.queryNodeID) + waitQueryNodeOnline(queryCoord.cluster, node2.queryNodeID) + loadPartitionsReq := &querypb.LoadPartitionsRequest{ + Base: &commonpb.MsgBase{ + MsgType: commonpb.MsgType_LoadPartitions, + }, + CollectionID: defaultCollectionID, + PartitionIDs: []UniqueID{defaultPartitionID}, + Schema: genDefaultCollectionSchema(false), + ReplicaNumber: 3, + } + + // load collection with 3 replicas, but no enough querynodes + assert.Equal(t, 2, len(queryCoord.cluster.onlineNodeIDs())) + status, err := queryCoord.LoadPartitions(ctx, loadPartitionsReq) + assert.NoError(t, err) + assert.Equal(t, commonpb.ErrorCode_UnexpectedError, status.ErrorCode) + + // Now it should can load collection with 3 replicas + node3, err := startQueryNodeServer(ctx) + assert.Nil(t, err) + waitQueryNodeOnline(queryCoord.cluster, node3.queryNodeID) + + status, err = queryCoord.LoadPartitions(ctx, loadPartitionsReq) + assert.NoError(t, err) + assert.Equal(t, commonpb.ErrorCode_Success, status.ErrorCode) + waitLoadPartitionDone(ctx, queryCoord, + loadPartitionsReq.CollectionID, loadPartitionsReq.PartitionIDs) + + status, err = queryCoord.ReleasePartitions(ctx, &querypb.ReleasePartitionsRequest{ + Base: &commonpb.MsgBase{ + MsgType: commonpb.MsgType_ReleaseCollection, + }, + CollectionID: loadPartitionsReq.CollectionID, + PartitionIDs: loadPartitionsReq.PartitionIDs, + }) + assert.NoError(t, err) + assert.Equal(t, commonpb.ErrorCode_Success, status.ErrorCode) + + node1.stop() + node2.stop() + node3.stop() + queryCoord.Stop() + err = removeAllSession() + assert.Nil(t, err) +} + func Test_RepeatedLoadSamePartitions(t *testing.T) { refreshParams() ctx := context.Background() @@ -868,9 +1026,10 @@ func Test_RepeatedLoadSamePartitions(t *testing.T) { Base: &commonpb.MsgBase{ MsgType: commonpb.MsgType_LoadPartitions, }, - CollectionID: defaultCollectionID, - PartitionIDs: []UniqueID{defaultPartitionID}, - Schema: genDefaultCollectionSchema(false), + CollectionID: defaultCollectionID, + PartitionIDs: []UniqueID{defaultPartitionID}, + Schema: genDefaultCollectionSchema(false), + ReplicaNumber: 1, } //first load defaultPartitionID @@ -904,9 +1063,10 @@ func Test_RepeatedLoadDifferentPartitions(t *testing.T) { Base: &commonpb.MsgBase{ MsgType: commonpb.MsgType_LoadPartitions, }, - CollectionID: defaultCollectionID, - PartitionIDs: []UniqueID{defaultPartitionID}, - Schema: genDefaultCollectionSchema(false), + CollectionID: defaultCollectionID, + PartitionIDs: []UniqueID{defaultPartitionID}, + Schema: genDefaultCollectionSchema(false), + ReplicaNumber: 1, } //first load defaultPartitionID @@ -920,9 +1080,10 @@ func Test_RepeatedLoadDifferentPartitions(t *testing.T) { Base: &commonpb.MsgBase{ MsgType: commonpb.MsgType_LoadPartitions, }, - CollectionID: defaultCollectionID, - PartitionIDs: []UniqueID{defaultPartitionID + 1}, - Schema: genDefaultCollectionSchema(false), + CollectionID: defaultCollectionID, + PartitionIDs: []UniqueID{defaultPartitionID + 1}, + Schema: genDefaultCollectionSchema(false), + ReplicaNumber: 1, } status, err = queryCoord.LoadPartitions(ctx, failLoadRequest) assert.Equal(t, commonpb.ErrorCode_UnexpectedError, status.ErrorCode) @@ -948,17 +1109,19 @@ func Test_LoadPartitionsAndLoadCollection(t *testing.T) { Base: &commonpb.MsgBase{ MsgType: commonpb.MsgType_LoadCollection, }, - CollectionID: defaultCollectionID, - Schema: genDefaultCollectionSchema(false), + CollectionID: defaultCollectionID, + Schema: genDefaultCollectionSchema(false), + ReplicaNumber: 1, } loadPartitionReq := &querypb.LoadPartitionsRequest{ Base: &commonpb.MsgBase{ MsgType: commonpb.MsgType_LoadPartitions, }, - CollectionID: defaultCollectionID, - PartitionIDs: []UniqueID{defaultPartitionID}, - Schema: genDefaultCollectionSchema(false), + CollectionID: defaultCollectionID, + PartitionIDs: []UniqueID{defaultPartitionID}, + Schema: genDefaultCollectionSchema(false), + ReplicaNumber: 1, } //first load defaultPartitionID @@ -992,8 +1155,9 @@ func Test_LoadAndReleaseCollection(t *testing.T) { Base: &commonpb.MsgBase{ MsgType: commonpb.MsgType_LoadCollection, }, - CollectionID: defaultCollectionID, - Schema: genDefaultCollectionSchema(false), + CollectionID: defaultCollectionID, + Schema: genDefaultCollectionSchema(false), + ReplicaNumber: 1, } releaseCollectionReq := &querypb.ReleaseCollectionRequest{ @@ -1034,9 +1198,10 @@ func Test_LoadAndReleasePartitions(t *testing.T) { Base: &commonpb.MsgBase{ MsgType: commonpb.MsgType_LoadPartitions, }, - CollectionID: defaultCollectionID, - PartitionIDs: []UniqueID{defaultPartitionID}, - Schema: genDefaultCollectionSchema(false), + CollectionID: defaultCollectionID, + PartitionIDs: []UniqueID{defaultPartitionID}, + Schema: genDefaultCollectionSchema(false), + ReplicaNumber: 1, } releasePartitionReq := &querypb.ReleasePartitionsRequest{ @@ -1078,8 +1243,9 @@ func Test_LoadCollectionAndReleasePartitions(t *testing.T) { Base: &commonpb.MsgBase{ MsgType: commonpb.MsgType_LoadCollection, }, - CollectionID: defaultCollectionID, - Schema: genDefaultCollectionSchema(false), + CollectionID: defaultCollectionID, + Schema: genDefaultCollectionSchema(false), + ReplicaNumber: 1, } releasePartitionReq := &querypb.ReleasePartitionsRequest{ @@ -1121,9 +1287,10 @@ func Test_LoadPartitionsAndReleaseCollection(t *testing.T) { Base: &commonpb.MsgBase{ MsgType: commonpb.MsgType_LoadPartitions, }, - CollectionID: defaultCollectionID, - PartitionIDs: []UniqueID{defaultPartitionID}, - Schema: genDefaultCollectionSchema(false), + CollectionID: defaultCollectionID, + PartitionIDs: []UniqueID{defaultPartitionID}, + Schema: genDefaultCollectionSchema(false), + ReplicaNumber: 1, } releaseCollectionReq := &querypb.ReleaseCollectionRequest{ Base: &commonpb.MsgBase{ @@ -1163,8 +1330,9 @@ func Test_RepeatedReleaseCollection(t *testing.T) { Base: &commonpb.MsgBase{ MsgType: commonpb.MsgType_LoadCollection, }, - CollectionID: defaultCollectionID, - Schema: genDefaultCollectionSchema(false), + CollectionID: defaultCollectionID, + Schema: genDefaultCollectionSchema(false), + ReplicaNumber: 1, } releaseCollectionReq := &querypb.ReleaseCollectionRequest{ @@ -1210,9 +1378,10 @@ func Test_RepeatedReleaseSamePartitions(t *testing.T) { Base: &commonpb.MsgBase{ MsgType: commonpb.MsgType_LoadPartitions, }, - CollectionID: defaultCollectionID, - PartitionIDs: []UniqueID{defaultPartitionID}, - Schema: genDefaultCollectionSchema(false), + CollectionID: defaultCollectionID, + PartitionIDs: []UniqueID{defaultPartitionID}, + Schema: genDefaultCollectionSchema(false), + ReplicaNumber: 1, } releasePartitionReq := &querypb.ReleasePartitionsRequest{ @@ -1259,9 +1428,10 @@ func Test_RepeatedReleaseDifferentPartitions(t *testing.T) { Base: &commonpb.MsgBase{ MsgType: commonpb.MsgType_LoadPartitions, }, - CollectionID: defaultCollectionID, - PartitionIDs: []UniqueID{defaultPartitionID, defaultPartitionID + 1}, - Schema: genDefaultCollectionSchema(false), + CollectionID: defaultCollectionID, + PartitionIDs: []UniqueID{defaultPartitionID, defaultPartitionID + 1}, + Schema: genDefaultCollectionSchema(false), + ReplicaNumber: 1, } releasePartitionReq := &querypb.ReleasePartitionsRequest{ @@ -1294,3 +1464,150 @@ func Test_RepeatedReleaseDifferentPartitions(t *testing.T) { err = removeAllSession() assert.Nil(t, err) } + +func TestGetReplicas(t *testing.T) { + refreshParams() + ctx := context.Background() + queryCoord, err := startQueryCoord(ctx) + assert.Nil(t, err) + + node1, err := startQueryNodeServer(ctx) + assert.Nil(t, err) + node2, err := startQueryNodeServer(ctx) + assert.Nil(t, err) + node3, err := startQueryNodeServer(ctx) + assert.Nil(t, err) + waitQueryNodeOnline(queryCoord.cluster, node1.queryNodeID) + waitQueryNodeOnline(queryCoord.cluster, node2.queryNodeID) + waitQueryNodeOnline(queryCoord.cluster, node3.queryNodeID) + + // First, load collection with replicas + loadCollectionReq := &querypb.LoadCollectionRequest{ + Base: &commonpb.MsgBase{ + MsgType: commonpb.MsgType_LoadCollection, + }, + CollectionID: defaultCollectionID, + Schema: genDefaultCollectionSchema(false), + ReplicaNumber: 3, + } + status, err := queryCoord.LoadCollection(ctx, loadCollectionReq) + assert.NoError(t, err) + assert.Equal(t, commonpb.ErrorCode_Success, status.ErrorCode) + waitLoadCollectionDone(ctx, queryCoord, defaultCollectionID) + + getReplicasReq := &milvuspb.GetReplicasRequest{ + Base: &commonpb.MsgBase{}, + CollectionID: loadCollectionReq.CollectionID, + } + resp, err := queryCoord.GetReplicas(ctx, getReplicasReq) + assert.NoError(t, err) + assert.Equal(t, commonpb.ErrorCode_Success, resp.Status.ErrorCode) + assert.Equal(t, 3, len(resp.Replicas)) + for i := range resp.Replicas { + assert.Equal(t, 1, len(resp.Replicas[i].NodeIds)) + for j := 0; j < i; j++ { + assert.NotEqual(t, + resp.Replicas[i].NodeIds[0], + resp.Replicas[j].NodeIds[0]) + } + } + + getReplicasReq.WithShardNodes = true + resp, err = queryCoord.GetReplicas(ctx, getReplicasReq) + assert.NoError(t, err) + assert.Equal(t, commonpb.ErrorCode_Success, resp.Status.ErrorCode) + assert.Equal(t, 3, len(resp.Replicas)) + for i := range resp.Replicas { + assert.Equal(t, 1, len(resp.Replicas[i].NodeIds)) + for j := range resp.Replicas[i].ShardReplicas { + assert.Equal(t, + resp.Replicas[i].NodeIds[0], + resp.Replicas[i].ShardReplicas[j].LeaderID) + } + + for j := 0; j < i; j++ { + assert.NotEqual(t, + resp.Replicas[i].NodeIds[0], + resp.Replicas[j].NodeIds[0]) + } + } + + // GetReplicas after release collection, it should return meta failed + status, err = queryCoord.ReleaseCollection(ctx, &querypb.ReleaseCollectionRequest{ + Base: &commonpb.MsgBase{}, + CollectionID: loadCollectionReq.CollectionID, + }) + assert.NoError(t, err) + assert.Equal(t, commonpb.ErrorCode_Success, status.ErrorCode) + + resp, err = queryCoord.GetReplicas(ctx, getReplicasReq) + assert.NoError(t, err) + assert.Equal(t, commonpb.ErrorCode_MetaFailed, resp.Status.ErrorCode) + + node1.stop() + node2.stop() + node3.stop() + queryCoord.Stop() +} + +func TestGetShardLeaders(t *testing.T) { + refreshParams() + ctx := context.Background() + queryCoord, err := startQueryCoord(ctx) + assert.Nil(t, err) + + node1, err := startQueryNodeServer(ctx) + assert.Nil(t, err) + node2, err := startQueryNodeServer(ctx) + assert.Nil(t, err) + node3, err := startQueryNodeServer(ctx) + assert.Nil(t, err) + waitQueryNodeOnline(queryCoord.cluster, node1.queryNodeID) + waitQueryNodeOnline(queryCoord.cluster, node2.queryNodeID) + waitQueryNodeOnline(queryCoord.cluster, node3.queryNodeID) + + // First, load collection with replicas + loadCollectionReq := &querypb.LoadCollectionRequest{ + Base: &commonpb.MsgBase{ + MsgType: commonpb.MsgType_LoadCollection, + }, + CollectionID: defaultCollectionID, + Schema: genDefaultCollectionSchema(false), + ReplicaNumber: 3, + } + status, err := queryCoord.LoadCollection(ctx, loadCollectionReq) + assert.NoError(t, err) + assert.Equal(t, commonpb.ErrorCode_Success, status.ErrorCode) + waitLoadCollectionDone(ctx, queryCoord, defaultCollectionID) + + getShardLeadersReq := &querypb.GetShardLeadersRequest{ + Base: &commonpb.MsgBase{}, + CollectionID: loadCollectionReq.CollectionID, + } + resp, err := queryCoord.GetShardLeaders(ctx, getShardLeadersReq) + assert.NoError(t, err) + assert.Equal(t, commonpb.ErrorCode_Success, resp.Status.ErrorCode) + + totalLeaders := 0 + for i := 0; i < len(resp.Shards); i++ { + totalLeaders += len(resp.Shards[i].NodeIds) + } + assert.Equal(t, 0, totalLeaders%3) + + // GetShardLeaders after release collection, it should return meta failed + status, err = queryCoord.ReleaseCollection(ctx, &querypb.ReleaseCollectionRequest{ + Base: &commonpb.MsgBase{}, + CollectionID: loadCollectionReq.CollectionID, + }) + assert.NoError(t, err) + assert.Equal(t, commonpb.ErrorCode_Success, status.ErrorCode) + + resp, err = queryCoord.GetShardLeaders(ctx, getShardLeadersReq) + assert.NoError(t, err) + assert.Equal(t, commonpb.ErrorCode_MetaFailed, resp.Status.ErrorCode) + + node1.stop() + node2.stop() + node3.stop() + queryCoord.Stop() +} diff --git a/internal/querycoord/index_checker_test.go b/internal/querycoord/index_checker_test.go index 8b1cde8a9d..015d131ebe 100644 --- a/internal/querycoord/index_checker_test.go +++ b/internal/querycoord/index_checker_test.go @@ -19,18 +19,18 @@ package querycoord import ( "context" "fmt" + "math/rand" + "sync/atomic" "testing" "github.com/golang/protobuf/proto" "github.com/milvus-io/milvus/internal/storage" "github.com/stretchr/testify/assert" - "github.com/milvus-io/milvus/internal/allocator" etcdkv "github.com/milvus-io/milvus/internal/kv/etcd" "github.com/milvus-io/milvus/internal/proto/commonpb" "github.com/milvus-io/milvus/internal/proto/querypb" "github.com/milvus-io/milvus/internal/util/etcd" - "github.com/milvus-io/milvus/internal/util/tsoutil" ) var indexCheckerTestDir = "/tmp/milvus_test/index_checker" @@ -42,7 +42,12 @@ func TestReloadFromKV(t *testing.T) { defer etcdCli.Close() assert.Nil(t, err) kv := etcdkv.NewEtcdKV(etcdCli, Params.EtcdCfg.MetaRootPath) - meta, err := newMeta(baseCtx, kv, nil, nil) + id := UniqueID(rand.Int31()) + idAllocator := func() (UniqueID, error) { + newID := atomic.AddInt64(&id, 1) + return newID, nil + } + meta, err := newMeta(baseCtx, kv, nil, idAllocator) assert.Nil(t, err) segmentInfo := &querypb.SegmentInfo{ @@ -98,7 +103,13 @@ func TestCheckIndexLoop(t *testing.T) { defer etcdCli.Close() assert.Nil(t, err) kv := etcdkv.NewEtcdKV(etcdCli, Params.EtcdCfg.MetaRootPath) - meta, err := newMeta(ctx, kv, nil, nil) + id := UniqueID(rand.Int31()) + idAllocator := func() (UniqueID, error) { + newID := atomic.AddInt64(&id, 1) + return newID, nil + } + + meta, err := newMeta(ctx, kv, nil, idAllocator) assert.Nil(t, err) rootCoord := newRootCoordMock(ctx) @@ -168,7 +179,12 @@ func TestHandoffNotExistSegment(t *testing.T) { defer etcdCli.Close() assert.Nil(t, err) kv := etcdkv.NewEtcdKV(etcdCli, Params.EtcdCfg.MetaRootPath) - meta, err := newMeta(ctx, kv, nil, nil) + id := UniqueID(rand.Int31()) + idAllocator := func() (UniqueID, error) { + newID := atomic.AddInt64(&id, 1) + return newID, nil + } + meta, err := newMeta(ctx, kv, nil, idAllocator) assert.Nil(t, err) rootCoord := newRootCoordMock(ctx) @@ -222,20 +238,19 @@ func TestProcessHandoffAfterIndexDone(t *testing.T) { defer etcdCli.Close() kv := etcdkv.NewEtcdKV(etcdCli, Params.EtcdCfg.MetaRootPath) - meta, err := newMeta(ctx, kv, nil, nil) + id := UniqueID(rand.Int31()) + idAllocator := func() (UniqueID, error) { + newID := atomic.AddInt64(&id, 1) + return newID, nil + } + meta, err := newMeta(ctx, kv, nil, idAllocator) assert.Nil(t, err) taskScheduler := &TaskScheduler{ ctx: ctx, cancel: cancel, client: kv, triggerTaskQueue: newTaskQueue(), - } - idAllocatorKV := tsoutil.NewTSOKVBase(etcdCli, Params.EtcdCfg.KvRootPath, "queryCoordTaskID") - idAllocator := allocator.NewGlobalIDAllocator("idTimestamp", idAllocatorKV) - err = idAllocator.Initialize() - assert.Nil(t, err) - taskScheduler.taskIDAllocator = func() (UniqueID, error) { - return idAllocator.AllocOne() + taskIDAllocator: idAllocator, } indexChecker, err := newIndexChecker(ctx, kv, meta, nil, taskScheduler, nil) assert.Nil(t, err) diff --git a/internal/querycoord/meta.go b/internal/querycoord/meta.go index 2d42a18e40..3d6d5b7f68 100644 --- a/internal/querycoord/meta.go +++ b/internal/querycoord/meta.go @@ -20,7 +20,9 @@ import ( "context" "errors" "fmt" + "math/rand" "path/filepath" + "runtime" "strconv" "strings" "sync" @@ -39,6 +41,7 @@ import ( "github.com/milvus-io/milvus/internal/proto/querypb" "github.com/milvus-io/milvus/internal/proto/schemapb" "github.com/milvus-io/milvus/internal/util" + "github.com/milvus-io/milvus/internal/util/dependency" "github.com/milvus-io/milvus/internal/util/funcutil" ) @@ -72,6 +75,7 @@ type Meta interface { showSegmentInfos(collectionID UniqueID, partitionIDs []UniqueID) []*querypb.SegmentInfo getSegmentInfoByID(segmentID UniqueID) (*querypb.SegmentInfo, error) getSegmentInfosByNode(nodeID int64) []*querypb.SegmentInfo + getSegmentInfosByNodeAndCollection(nodeID, collectionID int64) []*querypb.SegmentInfo getPartitionStatesByID(collectionID UniqueID, partitionID UniqueID) (*querypb.PartitionStates, error) @@ -106,7 +110,7 @@ type MetaReplica struct { ctx context.Context cancel context.CancelFunc client kv.MetaKv // client of a reliable kv service, i.e. etcd client - msFactory msgstream.Factory + factory dependency.Factory idAllocator func() (UniqueID, error) //sync.RWMutex @@ -126,7 +130,7 @@ type MetaReplica struct { replicas *ReplicaInfos } -func newMeta(ctx context.Context, kv kv.MetaKv, factory msgstream.Factory, idAllocator func() (UniqueID, error)) (Meta, error) { +func newMeta(ctx context.Context, kv kv.MetaKv, factory dependency.Factory, idAllocator func() (UniqueID, error)) (Meta, error) { childCtx, cancel := context.WithCancel(ctx) collectionInfos := make(map[UniqueID]*querypb.CollectionInfo) queryChannelInfos := make(map[UniqueID]*querypb.QueryChannelInfo) @@ -138,7 +142,7 @@ func newMeta(ctx context.Context, kv kv.MetaKv, factory msgstream.Factory, idAll ctx: childCtx, cancel: cancel, client: kv, - msFactory: factory, + factory: factory, idAllocator: idAllocator, collectionInfos: collectionInfos, @@ -161,6 +165,8 @@ func newMeta(ctx context.Context, kv kv.MetaKv, factory msgstream.Factory, idAll func (m *MetaReplica) reloadFromKV() error { log.Debug("start reload from kv") + + log.Info("recovery collections...") collectionKeys, collectionValues, err := m.client.LoadWithPrefix(collectionMetaPrefix) if err != nil { return err @@ -176,6 +182,9 @@ func (m *MetaReplica) reloadFromKV() error { return err } m.collectionInfos[collectionID] = collectionInfo + + log.Debug("recovery collection", + zap.Int64("collectionID", collectionID)) } metrics.QueryCoordNumCollections.WithLabelValues().Set(float64(len(m.collectionInfos))) @@ -215,12 +224,120 @@ func (m *MetaReplica) reloadFromKV() error { m.dmChannelInfos[dmChannel] = dmChannelWatchInfo } + // Compatibility for old meta format + // For collections that don't have replica(s), create 1 replica for them + // Add replica into meta storage and rewrite collection + dmChannels := make(map[UniqueID][]*querypb.DmChannelWatchInfo) // CollectionID -> []*DmChannelWatchInfo + for _, dmc := range m.dmChannelInfos { + dmChannels[dmc.CollectionID] = append(dmChannels[dmc.CollectionID], dmc) + } + for _, collectionInfo := range m.collectionInfos { + if len(collectionInfo.ReplicaIds) == 0 { + replica, err := m.generateReplica(collectionInfo.CollectionID, collectionInfo.PartitionIDs) + if err != nil { + return err + } + + segments := m.showSegmentInfos(collectionInfo.CollectionID, collectionInfo.PartitionIDs) + // remove duplicates + nodes := make(map[UniqueID]struct{}) + for _, segment := range segments { + for _, nodeID := range segment.NodeIds { + nodes[nodeID] = struct{}{} + } + } + for nodeID := range nodes { + replica.NodeIds = append(replica.NodeIds, nodeID) + } + + shardReplicas := make([]*milvuspb.ShardReplica, 0, len(dmChannels[collectionInfo.CollectionID])) + for _, dmc := range dmChannels[collectionInfo.CollectionID] { + shardReplicas = append(shardReplicas, &milvuspb.ShardReplica{ + LeaderID: dmc.NodeIDLoaded, + // LeaderAddr: Will set it after the cluster is reloaded + DmChannelName: dmc.DmChannel, + }) + } + + err = m.addReplica(replica) + if err != nil { + log.Error("failed to add replica for old collection info format without replicas info", + zap.Int64("collectionID", replica.CollectionID), + zap.Error(err)) + return err + } + + // DO NOT insert the replica into m.replicas + // it will be recovered below + } + } + + replicaKeys, replicaValues, err := m.client.LoadWithPrefix(ReplicaMetaPrefix) + if err != nil { + return err + } + for i := range replicaKeys { + replicaInfo := &milvuspb.ReplicaInfo{} + err = proto.Unmarshal([]byte(replicaValues[i]), replicaInfo) + if err != nil { + return err + } + m.replicas.Insert(replicaInfo) + } + //TODO::update partition states log.Debug("reload from kv finished") return nil } +// Compatibility for old meta format, this retrieves node address from cluster. +// The leader address is not always valid +func reloadShardLeaderAddress(meta Meta, cluster Cluster) error { + collections := meta.showCollections() + + reloadShardLeaderAddressFunc := func(idx int) error { + collection := collections[idx] + replicas, err := meta.getReplicasByCollectionID(collection.CollectionID) + if err != nil { + return err + } + + for _, replica := range replicas { + isModified := false + for _, shard := range replica.ShardReplicas { + if len(shard.LeaderAddr) == 0 { + nodeInfo, err := cluster.getNodeInfoByID(shard.LeaderID) + if err != nil { + log.Warn("failed to retrieve the node's address", + zap.Int64("nodeID", shard.LeaderID), + zap.Error(err)) + continue + } + shard.LeaderAddr = nodeInfo.(*queryNode).address + isModified = true + } + } + + if isModified { + err := meta.setReplicaInfo(replica) + if err != nil { + return err + } + } + } + + return nil + } + + concurrencyLevel := len(collections) + if concurrencyLevel > runtime.NumCPU() { + concurrencyLevel = runtime.NumCPU() + } + return funcutil.ProcessFuncParallel(len(collections), concurrencyLevel, + reloadShardLeaderAddressFunc, "reloadShardLeaderAddressFunc") +} + func (m *MetaReplica) setKvClient(kv kv.MetaKv) { m.client = kv } @@ -304,7 +421,8 @@ func (m *MetaReplica) addCollection(collectionID UniqueID, loadType querypb.Load PartitionStates: partitionStates, LoadType: loadType, Schema: schema, - // ReplicaIDs: replicas, + ReplicaIds: make([]int64, 0), + ReplicaNumber: 0, } err := saveGlobalCollectionInfo(collectionID, newCollection, m.client) if err != nil { @@ -369,7 +487,14 @@ func (m *MetaReplica) addPartitions(collectionID UniqueID, partitionIDs []Unique } func (m *MetaReplica) releaseCollection(collectionID UniqueID) error { - err := removeCollectionMeta(collectionID, m.client) + collection, err := m.getCollectionInfoByID(collectionID) + if err != nil { + log.Warn("the collection has been released", + zap.Int64("collectionID", collectionID)) + return nil + } + + err = removeCollectionMeta(collectionID, collection.ReplicaIds, m.client) if err != nil { log.Warn("remove collectionInfo from etcd failed", zap.Int64("collectionID", collectionID), zap.Any("error", err.Error())) return err @@ -733,6 +858,16 @@ func (m *MetaReplica) getSegmentInfosByNode(nodeID int64) []*querypb.SegmentInfo } return res } +func (m *MetaReplica) getSegmentInfosByNodeAndCollection(nodeID, collectionID int64) []*querypb.SegmentInfo { + var res []*querypb.SegmentInfo + segments := m.segmentsInfo.getSegments() + for _, segment := range segments { + if segment.GetNodeID() == nodeID && segment.GetCollectionID() == collectionID { + res = append(res, segment) + } + } + return res +} func (m *MetaReplica) getCollectionInfoByID(collectionID UniqueID) (*querypb.CollectionInfo, error) { m.collectionMu.RLock() @@ -742,7 +877,7 @@ func (m *MetaReplica) getCollectionInfoByID(collectionID UniqueID) (*querypb.Col return proto.Clone(info).(*querypb.CollectionInfo), nil } - return nil, errors.New("getCollectionInfoByID: can't find collectionID in collectionInfo") + return nil, fmt.Errorf("getCollectionInfoByID: can't find collectionID=%v in collectionInfo", collectionID) } func (m *MetaReplica) getPartitionStatesByID(collectionID UniqueID, partitionID UniqueID) (*querypb.PartitionStates, error) { @@ -872,7 +1007,7 @@ func (m *MetaReplica) getQueryStreamByID(collectionID UniqueID, queryChannel str if stream, ok := m.queryStreams[collectionID]; ok { queryStream = stream } else { - queryStream, err = m.msFactory.NewMsgStream(m.ctx) + queryStream, err = m.factory.NewMsgStream(m.ctx) if err != nil { log.Error("updateGlobalSealedSegmentInfos: create msgStream failed", zap.Error(err)) return nil, err @@ -932,6 +1067,7 @@ func (m *MetaReplica) setLoadPercentage(collectionID UniqueID, partitionID Uniqu } } else { findPartition := false + info.InMemoryPercentage = 0 for _, partitionState := range info.PartitionStates { if partitionState.PartitionID == partitionID { findPartition = true @@ -941,16 +1077,19 @@ func (m *MetaReplica) setLoadPercentage(collectionID UniqueID, partitionID Uniqu partitionState.State = querypb.PartitionState_PartialInMemory } partitionState.InMemoryPercentage = percentage - err := saveGlobalCollectionInfo(collectionID, info, m.client) - if err != nil { - log.Error("save collectionInfo error", zap.Any("error", err.Error()), zap.Int64("collectionID", collectionID)) - return err - } } + info.InMemoryPercentage += partitionState.InMemoryPercentage } if !findPartition { return errors.New("setLoadPercentage: can't find partitionID in collectionInfos") } + + info.InMemoryPercentage /= int64(len(info.PartitionIDs)) + err := saveGlobalCollectionInfo(collectionID, info, m.client) + if err != nil { + log.Error("save collectionInfo error", zap.Any("error", err.Error()), zap.Int64("collectionID", collectionID)) + return err + } } m.collectionInfos[collectionID] = info @@ -1042,7 +1181,24 @@ func (m *MetaReplica) generateReplica(collectionID int64, partitionIds []int64) } func (m *MetaReplica) addReplica(replica *milvuspb.ReplicaInfo) error { - err := saveReplicaInfo(replica, m.client) + collectionInfo, err := m.getCollectionInfoByID(replica.CollectionID) + if err != nil { + return err + } + + collectionInfo.ReplicaIds = append(collectionInfo.ReplicaIds, replica.ReplicaID) + collectionInfo.ReplicaNumber++ + + err = saveGlobalCollectionInfo(collectionInfo.CollectionID, collectionInfo, m.client) + if err != nil { + return err + } + + m.collectionMu.Lock() + m.collectionInfos[collectionInfo.CollectionID] = collectionInfo + m.collectionMu.Unlock() + + err = saveReplicaInfo(replica, m.client) if err != nil { return err } @@ -1091,6 +1247,10 @@ func (m *MetaReplica) getReplicasByCollectionID(collectionID int64) ([]*milvuspb replicas = append(replicas, replica) } + rand.Shuffle(len(replicas), func(i, j int) { + replicas[i], replicas[j] = replicas[j], replicas[i] + }) + return replicas, nil } @@ -1158,14 +1318,21 @@ func saveReplicaInfo(info *milvuspb.ReplicaInfo, kv kv.MetaKv) error { return kv.Save(key, string(infoBytes)) } -func removeCollectionMeta(collectionID UniqueID, kv kv.MetaKv) error { +func removeCollectionMeta(collectionID UniqueID, replicas []UniqueID, kv kv.MetaKv) error { var prefixes []string collectionInfosPrefix := fmt.Sprintf("%s/%d", collectionMetaPrefix, collectionID) prefixes = append(prefixes, collectionInfosPrefix) + dmChannelInfosPrefix := fmt.Sprintf("%s/%d", dmChannelMetaPrefix, collectionID) prefixes = append(prefixes, dmChannelInfosPrefix) + deltaChannelInfosPrefix := fmt.Sprintf("%s/%d", deltaChannelMetaPrefix, collectionID) prefixes = append(prefixes, deltaChannelInfosPrefix) + for _, replicaID := range replicas { + replicaPrefix := fmt.Sprintf("%s/%d", ReplicaMetaPrefix, replicaID) + prefixes = append(prefixes, replicaPrefix) + } + return kv.MultiRemoveWithPrefix(prefixes) } diff --git a/internal/querycoord/meta_test.go b/internal/querycoord/meta_test.go index 9df9566744..fedbb800dc 100644 --- a/internal/querycoord/meta_test.go +++ b/internal/querycoord/meta_test.go @@ -20,6 +20,8 @@ import ( "context" "errors" "fmt" + "math/rand" + "sync/atomic" "testing" "github.com/golang/protobuf/proto" @@ -68,7 +70,12 @@ func TestReplica_Release(t *testing.T) { assert.Nil(t, err) defer etcdCli.Close() etcdKV := etcdkv.NewEtcdKV(etcdCli, Params.EtcdCfg.MetaRootPath) - meta, err := newMeta(context.Background(), etcdKV, nil, nil) + id := UniqueID(rand.Int31()) + idAllocator := func() (UniqueID, error) { + newID := atomic.AddInt64(&id, 1) + return newID, nil + } + meta, err := newMeta(context.Background(), etcdKV, nil, idAllocator) assert.Nil(t, err) err = meta.addCollection(1, querypb.LoadType_LoadCollection, nil) require.NoError(t, err) @@ -294,13 +301,20 @@ func TestReloadMetaFromKV(t *testing.T) { assert.Nil(t, err) defer etcdCli.Close() kv := etcdkv.NewEtcdKV(etcdCli, Params.EtcdCfg.MetaRootPath) + id := UniqueID(rand.Int31()) + idAllocator := func() (UniqueID, error) { + newID := atomic.AddInt64(&id, 1) + return newID, nil + } meta := &MetaReplica{ client: kv, + idAllocator: idAllocator, collectionInfos: map[UniqueID]*querypb.CollectionInfo{}, queryChannelInfos: map[UniqueID]*querypb.QueryChannelInfo{}, dmChannelInfos: map[string]*querypb.DmChannelWatchInfo{}, deltaChannelInfos: map[UniqueID][]*datapb.VchannelInfo{}, segmentsInfo: newSegmentsInfo(kv), + replicas: NewReplicaInfos(), } kvs := make(map[string]string) @@ -349,10 +363,17 @@ func TestReloadMetaFromKV(t *testing.T) { assert.Equal(t, 1, len(meta.collectionInfos)) assert.Equal(t, 1, len(meta.segmentsInfo.getSegments())) - _, ok := meta.collectionInfos[defaultCollectionID] - assert.Equal(t, true, ok) + collectionInfo, err = meta.getCollectionInfoByID(collectionInfo.CollectionID) + assert.NoError(t, err) + assert.Equal(t, 1, len(collectionInfo.ReplicaIds)) + assert.Equal(t, int32(1), collectionInfo.ReplicaNumber) segment := meta.segmentsInfo.getSegment(defaultSegmentID) assert.NotNil(t, segment) + + replicas, err := meta.getReplicasByCollectionID(collectionInfo.CollectionID) + assert.NoError(t, err) + assert.Equal(t, 1, len(replicas)) + assert.Equal(t, collectionInfo.CollectionID, replicas[0].CollectionID) } func TestCreateQueryChannel(t *testing.T) { diff --git a/internal/querycoord/mock_3rd_component_test.go b/internal/querycoord/mock_3rd_component_test.go index 57b76d099c..c0f2cfba45 100644 --- a/internal/querycoord/mock_3rd_component_test.go +++ b/internal/querycoord/mock_3rd_component_test.go @@ -344,20 +344,6 @@ func (data *dataCoordMock) GetRecoveryInfo(ctx context.Context, req *datapb.GetR }, nil } - if _, ok := data.partitionID2Segment[partitionID]; !ok { - segmentIDs := make([]UniqueID, 0) - for i := 0; i < data.channelNumPerCol; i++ { - segmentID := data.baseSegmentID - if _, ok := data.Segment2Binlog[segmentID]; !ok { - segmentBinlog := generateInsertBinLog(segmentID) - data.Segment2Binlog[segmentID] = segmentBinlog - } - segmentIDs = append(segmentIDs, segmentID) - data.baseSegmentID++ - } - data.partitionID2Segment[partitionID] = segmentIDs - } - if _, ok := data.col2DmChannels[collectionID]; !ok { channelInfos := make([]*datapb.VchannelInfo, 0) data.collections = append(data.collections, collectionID) @@ -376,6 +362,21 @@ func (data *dataCoordMock) GetRecoveryInfo(ctx context.Context, req *datapb.GetR data.col2DmChannels[collectionID] = channelInfos } + if _, ok := data.partitionID2Segment[partitionID]; !ok { + segmentIDs := make([]UniqueID, 0) + for i := 0; i < data.channelNumPerCol; i++ { + segmentID := data.baseSegmentID + if _, ok := data.Segment2Binlog[segmentID]; !ok { + segmentBinlog := generateInsertBinLog(segmentID) + segmentBinlog.InsertChannel = data.col2DmChannels[collectionID][i].ChannelName + data.Segment2Binlog[segmentID] = segmentBinlog + } + segmentIDs = append(segmentIDs, segmentID) + data.baseSegmentID++ + } + data.partitionID2Segment[partitionID] = segmentIDs + } + binlogs := make([]*datapb.SegmentBinlogs, 0) for _, segmentID := range data.partitionID2Segment[partitionID] { if _, ok := data.Segment2Binlog[segmentID]; ok { diff --git a/internal/querycoord/query_coord.go b/internal/querycoord/query_coord.go index 8967a69abc..aa6c6d6796 100644 --- a/internal/querycoord/query_coord.go +++ b/internal/querycoord/query_coord.go @@ -39,6 +39,7 @@ import ( "github.com/milvus-io/milvus/internal/log" "github.com/milvus-io/milvus/internal/proto/commonpb" "github.com/milvus-io/milvus/internal/proto/internalpb" + "github.com/milvus-io/milvus/internal/proto/milvuspb" "github.com/milvus-io/milvus/internal/proto/querypb" "github.com/milvus-io/milvus/internal/storage" "github.com/milvus-io/milvus/internal/types" @@ -99,8 +100,9 @@ type QueryCoord struct { stateCode atomic.Value - factory dependency.Factory - chunkManager storage.ChunkManager + factory dependency.Factory + chunkManager storage.ChunkManager + groupBalancer balancer } // Register register query service at etcd @@ -168,6 +170,7 @@ func (qc *QueryCoord) Init() error { log.Error("query coordinator init meta failed", zap.Error(initError)) return } + qc.groupBalancer = newReplicaBalancer(qc.meta) // init channelUnsubscribeHandler qc.handler, initError = newChannelUnsubscribeHandler(qc.loopCtx, qc.kvClient, qc.factory) @@ -183,7 +186,17 @@ func (qc *QueryCoord) Init() error { return } - qc.chunkManager, initError = qc.factory.NewVectorStorageChunkManager(qc.loopCtx) + // NOTE: ignore the returned error + // we only try best to reload the leader addresses + reloadShardLeaderAddress(qc.meta, qc.cluster) + + qc.chunkManager, initError = storage.NewMinioChunkManager(qc.loopCtx, + storage.Address(Params.MinioCfg.Address), + storage.AccessKeyID(Params.MinioCfg.AccessKeyID), + storage.SecretAccessKeyID(Params.MinioCfg.SecretAccessKey), + storage.UseSSL(Params.MinioCfg.UseSSL), + storage.BucketName(Params.MinioCfg.BucketName), + storage.CreateBucket(true)) if initError != nil { log.Error("query coordinator init cluster failed", zap.Error(initError)) @@ -337,6 +350,13 @@ func (qc *QueryCoord) watchNodeLoop() { defer qc.loopWg.Done() log.Debug("QueryCoord start watch node loop") + unallocatedNodes := qc.getUnallocatedNodes() + for _, n := range unallocatedNodes { + if err := qc.allocateNode(n); err != nil { + log.Warn("unable to allcoate node", zap.Int64("nodeID", n), zap.Error(err)) + } + } + offlineNodeIDs := qc.cluster.offlineNodeIDs() if len(offlineNodeIDs) != 0 { loadBalanceSegment := &querypb.LoadBalanceRequest{ @@ -366,6 +386,70 @@ func (qc *QueryCoord) watchNodeLoop() { qc.handleNodeEvent(ctx) } +func (qc *QueryCoord) allocateNode(nodeID int64) error { + plans, err := qc.groupBalancer.addNode(nodeID) + if err != nil { + return err + } + for _, p := range plans { + if err := qc.applyBalancePlan(p); err != nil { + log.Warn("failed to apply balance plan", zap.Error(err), zap.Any("plan", p)) + } + } + return nil +} +func (qc *QueryCoord) getUnallocatedNodes() []int64 { + onlines := qc.cluster.onlineNodeIDs() + var ret []int64 + for _, n := range onlines { + replica, err := qc.meta.getReplicasByNodeID(n) + if err != nil { + log.Warn("failed to get replica", zap.Int64("nodeID", n), zap.Error(err)) + continue + } + if replica == nil { + ret = append(ret, n) + } + } + return ret +} + +func (qc *QueryCoord) applyBalancePlan(p *balancePlan) error { + if p.sourceReplica != -1 { + replica, err := qc.meta.getReplicaByID(p.sourceReplica) + if err != nil { + return err + } + replica = removeNodeFromReplica(replica, p.nodeID) + if err := qc.meta.setReplicaInfo(replica); err != nil { + return err + } + } + if p.targetReplica != -1 { + replica, err := qc.meta.getReplicaByID(p.targetReplica) + if err != nil { + return err + } + + replica.NodeIds = append(replica.NodeIds, p.nodeID) + if err := qc.meta.setReplicaInfo(replica); err != nil { + return err + } + } + return nil +} + +func removeNodeFromReplica(replica *milvuspb.ReplicaInfo, nodeID int64) *milvuspb.ReplicaInfo { + for i := 0; i < len(replica.NodeIds); i++ { + if replica.NodeIds[i] != nodeID { + continue + } + replica.NodeIds = append(replica.NodeIds[:i], replica.NodeIds[i+1:]...) + return replica + } + return replica +} + func (qc *QueryCoord) handleNodeEvent(ctx context.Context) { for { select { @@ -390,6 +474,10 @@ func (qc *QueryCoord) handleNodeEvent(ctx context.Context) { err := qc.cluster.registerNode(ctx, event.Session, serverID, disConnect) if err != nil { log.Error("QueryCoord failed to register a QueryNode", zap.Int64("nodeID", serverID), zap.String("error info", err.Error())) + continue + } + if err := qc.allocateNode(serverID); err != nil { + log.Error("unable to allcoate node", zap.Int64("nodeID", serverID), zap.Error(err)) } qc.metricsCacheManager.InvalidateSystemInfoMetrics() case sessionutil.SessionDelEvent: @@ -480,81 +568,104 @@ func (qc *QueryCoord) loadBalanceSegmentLoop() { timer := time.NewTicker(time.Duration(Params.QueryCoordCfg.BalanceIntervalSeconds) * time.Second) + var collectionInfos []*querypb.CollectionInfo + pos := 0 + for { select { case <-ctx.Done(): return case <-timer.C: - onlineNodeIDs := qc.cluster.onlineNodeIDs() - if len(onlineNodeIDs) == 0 { - log.Error("[auto balance]there is no online QueryNode to do balance") - continue + if pos == len(collectionInfos) { + pos = 0 + collectionInfos = qc.meta.showCollections() } // get mem info of online nodes from cluster nodeID2MemUsageRate := make(map[int64]float64) nodeID2MemUsage := make(map[int64]uint64) nodeID2TotalMem := make(map[int64]uint64) - nodeID2SegmentInfos := make(map[int64]map[UniqueID]*querypb.SegmentInfo) - var availableNodeIDs []int64 - for _, nodeID := range onlineNodeIDs { - nodeInfo, err := qc.cluster.getNodeInfoByID(nodeID) + loadBalanceTasks := make([]*loadBalanceTask, 0) + // balance at most 20 collections in a round + for i := 0; pos < len(collectionInfos) && i < 20; i, pos = i+1, pos+1 { + info := collectionInfos[pos] + replicas, err := qc.meta.getReplicasByCollectionID(info.GetCollectionID()) if err != nil { - log.Warn("loadBalanceSegmentLoop: get node info from QueryNode failed", zap.Int64("nodeID", nodeID), zap.Error(err)) + log.Warn("unable to get replicas of collection", zap.Int64("collectionID", info.GetCollectionID())) continue } - - updateSegmentInfoDone := true - leastSegmentInfos := make(map[UniqueID]*querypb.SegmentInfo) - segmentInfos := qc.meta.getSegmentInfosByNode(nodeID) - for _, segmentInfo := range segmentInfos { - leastInfo, err := qc.cluster.getSegmentInfoByID(ctx, segmentInfo.SegmentID) - if err != nil { - log.Warn("[auto balance] failed to get segment information from QueryNode", zap.Int64("nodeID", nodeID), zap.Error(err)) - updateSegmentInfoDone = false - break + for _, replica := range replicas { + // auto balance is executed on replica level + onlineNodeIDs := replica.GetNodeIds() + if len(onlineNodeIDs) == 0 { + log.Error("loadBalanceSegmentLoop: there are no online QueryNode to balance") + continue } - leastSegmentInfos[segmentInfo.SegmentID] = leastInfo - } - if updateSegmentInfoDone { - nodeID2MemUsageRate[nodeID] = nodeInfo.(*queryNode).memUsageRate - nodeID2MemUsage[nodeID] = nodeInfo.(*queryNode).memUsage - nodeID2TotalMem[nodeID] = nodeInfo.(*queryNode).totalMem - availableNodeIDs = append(availableNodeIDs, nodeID) - nodeID2SegmentInfos[nodeID] = leastSegmentInfos - } - } - log.Debug("loadBalanceSegmentLoop: memory usage rate of all online QueryNode", zap.Any("mem rate", nodeID2MemUsageRate)) - if len(availableNodeIDs) <= 1 { - log.Warn("loadBalanceSegmentLoop: there are too few available query nodes to balance", zap.Int64s("onlineNodeIDs", onlineNodeIDs), zap.Int64s("availableNodeIDs", availableNodeIDs)) - continue - } + var availableNodeIDs []int64 + nodeID2SegmentInfos := make(map[int64]map[UniqueID]*querypb.SegmentInfo) + for _, nodeID := range onlineNodeIDs { + if _, ok := nodeID2MemUsage[nodeID]; !ok { + nodeInfo, err := qc.cluster.getNodeInfoByID(nodeID) + if err != nil { + log.Warn("loadBalanceSegmentLoop: get node info from QueryNode failed", zap.Int64("nodeID", nodeID), zap.Error(err)) + continue + } + nodeID2MemUsageRate[nodeID] = nodeInfo.(*queryNode).memUsageRate + nodeID2MemUsage[nodeID] = nodeInfo.(*queryNode).memUsage + nodeID2TotalMem[nodeID] = nodeInfo.(*queryNode).totalMem + } - // check which nodes need balance and determine which segments on these nodes need to be migrated to other nodes - memoryInsufficient := false - loadBalanceTasks := make([]*loadBalanceTask, 0) - for { - sort.Slice(availableNodeIDs, func(i, j int) bool { - return nodeID2MemUsageRate[availableNodeIDs[i]] > nodeID2MemUsageRate[availableNodeIDs[j]] - }) - - // the memoryUsageRate of the sourceNode is higher than other query node - sourceNodeID := availableNodeIDs[0] - dstNodeID := availableNodeIDs[len(availableNodeIDs)-1] - memUsageRateDiff := nodeID2MemUsageRate[sourceNodeID] - nodeID2MemUsageRate[dstNodeID] - // if memoryUsageRate of source node is greater than 90%, and the max memUsageDiff is greater than 30% - // then migrate the segments on source node to other query nodes - if nodeID2MemUsageRate[sourceNodeID] > Params.QueryCoordCfg.OverloadedMemoryThresholdPercentage || - memUsageRateDiff > Params.QueryCoordCfg.MemoryUsageMaxDifferencePercentage { - segmentInfos := nodeID2SegmentInfos[sourceNodeID] - // select the segment that needs balance on the source node - selectedSegmentInfo, err := chooseSegmentToBalance(sourceNodeID, dstNodeID, segmentInfos, nodeID2MemUsage, nodeID2TotalMem, nodeID2MemUsageRate) - if err != nil { - // no enough memory on query nodes to balance, then notify proxy to stop insert - memoryInsufficient = true - break + updateSegmentInfoDone := true + leastSegmentInfos := make(map[UniqueID]*querypb.SegmentInfo) + segmentInfos := qc.meta.getSegmentInfosByNodeAndCollection(nodeID, replica.GetCollectionID()) + for _, segmentInfo := range segmentInfos { + leastInfo, err := qc.cluster.getSegmentInfoByID(ctx, segmentInfo.SegmentID) + if err != nil { + log.Warn("loadBalanceSegmentLoop: get segment info from QueryNode failed", zap.Int64("nodeID", nodeID), zap.Error(err)) + updateSegmentInfoDone = false + break + } + leastSegmentInfos[segmentInfo.SegmentID] = leastInfo + } + if updateSegmentInfoDone { + availableNodeIDs = append(availableNodeIDs, nodeID) + nodeID2SegmentInfos[nodeID] = leastSegmentInfos + } } - // select a segment to balance successfully, then recursive traversal whether there are other segments that can balance - if selectedSegmentInfo != nil { + log.Debug("loadBalanceSegmentLoop: memory usage rate of all online QueryNode", zap.Any("mem rate", nodeID2MemUsageRate)) + if len(availableNodeIDs) <= 1 { + log.Warn("loadBalanceSegmentLoop: there are too few available query nodes to balance", zap.Int64s("onlineNodeIDs", onlineNodeIDs), zap.Int64s("availableNodeIDs", availableNodeIDs)) + continue + } + + // check which nodes need balance and determine which segments on these nodes need to be migrated to other nodes + memoryInsufficient := false + for { + sort.Slice(availableNodeIDs, func(i, j int) bool { + return nodeID2MemUsageRate[availableNodeIDs[i]] > nodeID2MemUsageRate[availableNodeIDs[j]] + }) + + // the memoryUsageRate of the sourceNode is higher than other query node + sourceNodeID := availableNodeIDs[0] + dstNodeID := availableNodeIDs[len(availableNodeIDs)-1] + memUsageRateDiff := nodeID2MemUsageRate[sourceNodeID] - nodeID2MemUsageRate[dstNodeID] + if nodeID2MemUsageRate[sourceNodeID] <= Params.QueryCoordCfg.OverloadedMemoryThresholdPercentage && + memUsageRateDiff <= Params.QueryCoordCfg.MemoryUsageMaxDifferencePercentage { + break + } + // if memoryUsageRate of source node is greater than 90%, and the max memUsageDiff is greater than 30% + // then migrate the segments on source node to other query nodes + segmentInfos := nodeID2SegmentInfos[sourceNodeID] + // select the segment that needs balance on the source node + selectedSegmentInfo, err := chooseSegmentToBalance(sourceNodeID, dstNodeID, segmentInfos, nodeID2MemUsage, nodeID2TotalMem, nodeID2MemUsageRate) + if err != nil { + // no enough memory on query nodes to balance, then notify proxy to stop insert + memoryInsufficient = true + break + } + if selectedSegmentInfo == nil { + break + } + // select a segment to balance successfully, then recursive traversal whether there are other segments that can balance req := &querypb.LoadBalanceRequest{ Base: &commonpb.MsgBase{ MsgType: commonpb.MsgType_LoadBalanceSegments, @@ -580,35 +691,28 @@ func (qc *QueryCoord) loadBalanceSegmentLoop() { delete(nodeID2SegmentInfos[sourceNodeID], selectedSegmentInfo.SegmentID) nodeID2SegmentInfos[dstNodeID][selectedSegmentInfo.SegmentID] = selectedSegmentInfo continue - } else { - // moving any segment will not improve the balance status - break } + if memoryInsufficient { + // no enough memory on query nodes to balance, then notify proxy to stop insert + //TODO:: xige-16 + log.Warn("loadBalanceSegmentLoop: QueryNode has insufficient memory, stop inserting data") + } + } + } + for _, t := range loadBalanceTasks { + qc.scheduler.Enqueue(t) + log.Debug("loadBalanceSegmentLoop: enqueue a loadBalance task", zap.Any("task", t)) + err := t.waitToFinish() + if err != nil { + // if failed, wait for next balance loop + // it may be that the collection/partition of the balanced segment has been released + // it also may be other abnormal errors + log.Error("loadBalanceSegmentLoop: balance task execute failed", zap.Any("task", t)) } else { - // all query node's memoryUsageRate is less than 90%, and the max memUsageDiff is less than 30% - break + log.Debug("loadBalanceSegmentLoop: balance task execute success", zap.Any("task", t)) } } - if !memoryInsufficient { - for _, t := range loadBalanceTasks { - qc.scheduler.Enqueue(t) - log.Debug("loadBalanceSegmentLoop: enqueue a loadBalance task", zap.Any("task", t)) - err := t.waitToFinish() - if err != nil { - // if failed, wait for next balance loop - // it may be that the collection/partition of the balanced segment has been released - // it also may be other abnormal errors - log.Error("loadBalanceSegmentLoop: balance task execute failed", zap.Any("task", t)) - } else { - log.Debug("loadBalanceSegmentLoop: balance task execute success", zap.Any("task", t)) - } - } - log.Debug("loadBalanceSegmentLoop: load balance Done in this loop", zap.Any("tasks", loadBalanceTasks)) - } else { - // no enough memory on query nodes to balance, then notify proxy to stop insert - //TODO:: xige-16 - log.Error("loadBalanceSegmentLoop: QueryNode has insufficient memory, stop inserting data") - } + log.Debug("loadBalanceSegmentLoop: load balance Done in this loop", zap.Any("tasks", loadBalanceTasks)) } } } diff --git a/internal/querycoord/query_coord_test.go b/internal/querycoord/query_coord_test.go index 807f3b296b..e0b9ded682 100644 --- a/internal/querycoord/query_coord_test.go +++ b/internal/querycoord/query_coord_test.go @@ -177,7 +177,7 @@ func TestWatchNodeLoop(t *testing.T) { } collectionBlobs, err := proto.Marshal(collectionInfo) assert.Nil(t, err) - nodeKey := fmt.Sprintf("%s/%d", collectionMetaPrefix, 100) + nodeKey := fmt.Sprintf("%s/%d", collectionMetaPrefix, defaultCollectionID) kvs[nodeKey] = string(collectionBlobs) err = kv.MultiSave(kvs) @@ -582,9 +582,10 @@ func TestLoadBalanceSegmentLoop(t *testing.T) { Base: &commonpb.MsgBase{ MsgType: commonpb.MsgType_LoadPartitions, }, - CollectionID: defaultCollectionID, - PartitionIDs: []UniqueID{partitionID}, - Schema: genDefaultCollectionSchema(false), + CollectionID: defaultCollectionID, + PartitionIDs: []UniqueID{partitionID}, + Schema: genDefaultCollectionSchema(false), + ReplicaNumber: 1, } baseTask := newBaseTask(baseCtx, querypb.TriggerCondition_GrpcRequest) loadPartitionTask := &loadPartitionTask{ diff --git a/internal/querycoord/querynode_test.go b/internal/querycoord/querynode_test.go index 47c0f656cc..56ca2891b0 100644 --- a/internal/querycoord/querynode_test.go +++ b/internal/querycoord/querynode_test.go @@ -118,8 +118,9 @@ func TestQueryNode_MultiNode_stop(t *testing.T) { Base: &commonpb.MsgBase{ MsgType: commonpb.MsgType_LoadCollection, }, - CollectionID: defaultCollectionID, - Schema: genDefaultCollectionSchema(false), + CollectionID: defaultCollectionID, + Schema: genDefaultCollectionSchema(false), + ReplicaNumber: 1, }) _, err = queryCoord.ReleaseCollection(baseCtx, &querypb.ReleaseCollectionRequest{ Base: &commonpb.MsgBase{ @@ -157,8 +158,9 @@ func TestQueryNode_MultiNode_reStart(t *testing.T) { Base: &commonpb.MsgBase{ MsgType: commonpb.MsgType_LoadCollection, }, - CollectionID: defaultCollectionID, - Schema: genDefaultCollectionSchema(false), + CollectionID: defaultCollectionID, + Schema: genDefaultCollectionSchema(false), + ReplicaNumber: 1, }) queryNode1.stop() err = removeNodeSession(queryNode1.queryNodeID) @@ -256,8 +258,9 @@ func TestSealedSegmentChangeAfterQueryNodeStop(t *testing.T) { Base: &commonpb.MsgBase{ MsgType: commonpb.MsgType_LoadCollection, }, - CollectionID: defaultCollectionID, - Schema: genDefaultCollectionSchema(false), + CollectionID: defaultCollectionID, + Schema: genDefaultCollectionSchema(false), + ReplicaNumber: 1, }) queryNode2, err := startQueryNodeServer(baseCtx) diff --git a/internal/querycoord/replica.go b/internal/querycoord/replica.go index 941f0ece7f..237339c494 100644 --- a/internal/querycoord/replica.go +++ b/internal/querycoord/replica.go @@ -19,6 +19,7 @@ package querycoord import ( "sync" + "github.com/golang/protobuf/proto" "github.com/milvus-io/milvus/internal/proto/milvuspb" ) @@ -28,17 +29,17 @@ type ReplicaInfos struct { globalGuard sync.RWMutex // We have to make sure atomically update replicas and index // Persistent Info - replicas map[UniqueID]*milvuspb.ReplicaInfo // replica_id -> *ReplicaInfo + replicas map[UniqueID]*milvuspb.ReplicaInfo // replicaID -> *ReplicaInfo // Non-persistent info - nodeIndex map[UniqueID][]*milvuspb.ReplicaInfo // node_id -> []*ReplicaInfo + nodeIndex map[UniqueID]map[UniqueID]*milvuspb.ReplicaInfo // nodeID, replicaID -> []*ReplicaInfo } func NewReplicaInfos() *ReplicaInfos { return &ReplicaInfos{ globalGuard: sync.RWMutex{}, replicas: make(map[int64]*milvuspb.ReplicaInfo), - nodeIndex: make(map[int64][]*milvuspb.ReplicaInfo), + nodeIndex: make(map[int64]map[int64]*milvuspb.ReplicaInfo), } } @@ -47,7 +48,8 @@ func (rep *ReplicaInfos) Get(replicaID UniqueID) (*milvuspb.ReplicaInfo, bool) { defer rep.globalGuard.RUnlock() info, ok := rep.replicas[replicaID] - return info, ok + clone := proto.Clone(info).(*milvuspb.ReplicaInfo) + return clone, ok } // Make sure atomically update replica and index @@ -56,23 +58,25 @@ func (rep *ReplicaInfos) Insert(info *milvuspb.ReplicaInfo) { defer rep.globalGuard.Unlock() old, ok := rep.replicas[info.ReplicaID] - // This updates ReplicaInfo, not inserts a new one - // No need to update nodeIndex - if ok { - *old = *info - return - } + info = proto.Clone(info).(*milvuspb.ReplicaInfo) rep.replicas[info.ReplicaID] = info + // This updates ReplicaInfo, not inserts a new one + if ok { + for _, nodeID := range old.NodeIds { + nodeReplicas := rep.nodeIndex[nodeID] + delete(nodeReplicas, old.ReplicaID) + } + } + for _, nodeID := range info.NodeIds { replicas, ok := rep.nodeIndex[nodeID] if !ok { - replicas = make([]*milvuspb.ReplicaInfo, 0) - rep.nodeIndex[nodeID] = replicas + replicas = make(map[UniqueID]*milvuspb.ReplicaInfo) } - replicas = append(replicas, info) + replicas[info.ReplicaID] = info rep.nodeIndex[nodeID] = replicas } } @@ -87,5 +91,24 @@ func (rep *ReplicaInfos) GetReplicasByNodeID(nodeID UniqueID) []*milvuspb.Replic return nil } - return replicas + clones := make([]*milvuspb.ReplicaInfo, 0, len(replicas)) + for _, replica := range replicas { + clones = append(clones, proto.Clone(replica).(*milvuspb.ReplicaInfo)) + } + + return clones +} + +func (rep *ReplicaInfos) Remove(replicaIds ...UniqueID) { + rep.globalGuard.Lock() + defer rep.globalGuard.Unlock() + + for _, replicaID := range replicaIds { + delete(rep.replicas, replicaID) + } + for _, replicaIndex := range rep.nodeIndex { + for _, replicaID := range replicaIds { + delete(replicaIndex, replicaID) + } + } } diff --git a/internal/querycoord/segment_allocator.go b/internal/querycoord/segment_allocator.go index 9b5e3548b4..d9c1ba5502 100644 --- a/internal/querycoord/segment_allocator.go +++ b/internal/querycoord/segment_allocator.go @@ -34,12 +34,12 @@ func defaultSegAllocatePolicy() SegmentAllocatePolicy { const shuffleWaitInterval = 1 * time.Second // SegmentAllocatePolicy helper function definition to allocate Segment to queryNode -type SegmentAllocatePolicy func(ctx context.Context, reqs []*querypb.LoadSegmentsRequest, cluster Cluster, metaCache Meta, wait bool, excludeNodeIDs []int64, includeNodeIDs []int64) error +type SegmentAllocatePolicy func(ctx context.Context, reqs []*querypb.LoadSegmentsRequest, cluster Cluster, metaCache Meta, wait bool, excludeNodeIDs []int64, includeNodeIDs []int64, replicaID int64) error // shuffleSegmentsToQueryNode shuffle segments to online nodes // returned are noded id for each segment, which satisfies: // len(returnedNodeIds) == len(segmentIDs) && segmentIDs[i] is assigned to returnedNodeIds[i] -func shuffleSegmentsToQueryNode(ctx context.Context, reqs []*querypb.LoadSegmentsRequest, cluster Cluster, metaCache Meta, wait bool, excludeNodeIDs []int64, includeNodeIDs []int64) error { +func shuffleSegmentsToQueryNode(ctx context.Context, reqs []*querypb.LoadSegmentsRequest, cluster Cluster, metaCache Meta, wait bool, excludeNodeIDs []int64, includeNodeIDs []int64, replicaID int64) error { if len(reqs) == 0 { return nil } @@ -95,7 +95,7 @@ func shuffleSegmentsToQueryNode(ctx context.Context, reqs []*querypb.LoadSegment } } -func shuffleSegmentsToQueryNodeV2(ctx context.Context, reqs []*querypb.LoadSegmentsRequest, cluster Cluster, metaCache Meta, wait bool, excludeNodeIDs []int64, includeNodeIDs []int64) error { +func shuffleSegmentsToQueryNodeV2(ctx context.Context, reqs []*querypb.LoadSegmentsRequest, cluster Cluster, metaCache Meta, wait bool, excludeNodeIDs []int64, includeNodeIDs []int64, replicaID int64) error { // key = offset, value = segmentSize if len(reqs) == 0 { return nil @@ -115,7 +115,16 @@ func shuffleSegmentsToQueryNodeV2(ctx context.Context, reqs []*querypb.LoadSegme totalMem := make(map[int64]uint64) memUsage := make(map[int64]uint64) memUsageRate := make(map[int64]float64) - onlineNodeIDs := cluster.onlineNodeIDs() + var onlineNodeIDs []int64 + if replicaID == -1 { + onlineNodeIDs = cluster.onlineNodeIDs() + } else { + replica, err := metaCache.getReplicaByID(replicaID) + if err != nil { + return err + } + onlineNodeIDs = replica.GetNodeIds() + } if len(onlineNodeIDs) == 0 && !wait { err := errors.New("no online queryNode to allocate") log.Error("shuffleSegmentsToQueryNode failed", zap.Error(err)) diff --git a/internal/querycoord/segment_allocator_test.go b/internal/querycoord/segment_allocator_test.go index b92173b815..42ce453e71 100644 --- a/internal/querycoord/segment_allocator_test.go +++ b/internal/querycoord/segment_allocator_test.go @@ -18,6 +18,8 @@ package querycoord import ( "context" + "math/rand" + "sync/atomic" "testing" "github.com/stretchr/testify/assert" @@ -39,8 +41,14 @@ func TestShuffleSegmentsToQueryNode(t *testing.T) { kv := etcdkv.NewEtcdKV(etcdCli, Params.EtcdCfg.MetaRootPath) clusterSession := sessionutil.NewSession(context.Background(), Params.EtcdCfg.MetaRootPath, etcdCli) clusterSession.Init(typeutil.QueryCoordRole, Params.QueryCoordCfg.Address, true, false) - factory := dependency.NewDefaultFactory(true) - meta, err := newMeta(baseCtx, kv, factory, nil) + factory := dependency.NewDefaultFactory(true) //msgstream.NewPmsFactory() + + id := UniqueID(rand.Int31()) + idAllocator := func() (UniqueID, error) { + newID := atomic.AddInt64(&id, 1) + return newID, nil + } + meta, err := newMeta(baseCtx, kv, factory, idAllocator) assert.Nil(t, err) handler, err := newChannelUnsubscribeHandler(baseCtx, kv, factory) assert.Nil(t, err) @@ -83,7 +91,7 @@ func TestShuffleSegmentsToQueryNode(t *testing.T) { reqs := []*querypb.LoadSegmentsRequest{firstReq, secondReq} t.Run("Test shuffleSegmentsWithoutQueryNode", func(t *testing.T) { - err = shuffleSegmentsToQueryNode(baseCtx, reqs, cluster, meta, false, nil, nil) + err = shuffleSegmentsToQueryNode(baseCtx, reqs, cluster, meta, false, nil, nil, -1) assert.NotNil(t, err) }) @@ -95,7 +103,7 @@ func TestShuffleSegmentsToQueryNode(t *testing.T) { waitQueryNodeOnline(cluster, node1ID) t.Run("Test shuffleSegmentsToQueryNode", func(t *testing.T) { - err = shuffleSegmentsToQueryNode(baseCtx, reqs, cluster, meta, false, nil, nil) + err = shuffleSegmentsToQueryNode(baseCtx, reqs, cluster, meta, false, nil, nil, -1) assert.Nil(t, err) assert.Equal(t, node1ID, firstReq.DstNodeID) @@ -111,13 +119,13 @@ func TestShuffleSegmentsToQueryNode(t *testing.T) { cluster.stopNode(node1ID) t.Run("Test shuffleSegmentsToQueryNodeV2", func(t *testing.T) { - err = shuffleSegmentsToQueryNodeV2(baseCtx, reqs, cluster, meta, false, nil, nil) + err = shuffleSegmentsToQueryNodeV2(baseCtx, reqs, cluster, meta, false, nil, nil, -1) assert.Nil(t, err) assert.Equal(t, node2ID, firstReq.DstNodeID) assert.Equal(t, node2ID, secondReq.DstNodeID) - err = shuffleSegmentsToQueryNodeV2(baseCtx, reqs, cluster, meta, true, nil, nil) + err = shuffleSegmentsToQueryNodeV2(baseCtx, reqs, cluster, meta, true, nil, nil, -1) assert.Nil(t, err) assert.Equal(t, node2ID, firstReq.DstNodeID) diff --git a/internal/querycoord/segments_info.go b/internal/querycoord/segments_info.go index f693264272..fc16196dcb 100644 --- a/internal/querycoord/segments_info.go +++ b/internal/querycoord/segments_info.go @@ -45,8 +45,6 @@ func newSegmentsInfo(kv kv.TxnKV) *segmentsInfo { func (s *segmentsInfo) loadSegments() error { var err error s.loadOnce.Do(func() { - s.mu.Lock() - defer s.mu.Unlock() var values []string _, values, err = s.kv.LoadWithPrefix(util.SegmentMetaPrefix) if err != nil { @@ -60,6 +58,16 @@ func (s *segmentsInfo) loadSegments() error { } s.segmentIDMap[segment.GetSegmentID()] = segment numRowsCnt += float64(segment.NumRows) + + // Compatibility for old meta format + if len(segment.NodeIds) == 0 { + segment.NodeIds = append(segment.NodeIds, segment.NodeID) + } + // rewrite segment info + err = s.saveSegment(segment) + if err != nil { + return + } } metrics.QueryCoordNumEntities.WithLabelValues().Add(numRowsCnt) diff --git a/internal/querycoord/task.go b/internal/querycoord/task.go index 53c0597e63..7bd3080d37 100644 --- a/internal/querycoord/task.go +++ b/internal/querycoord/task.go @@ -31,6 +31,7 @@ import ( "github.com/milvus-io/milvus/internal/metrics" "github.com/milvus-io/milvus/internal/proto/commonpb" "github.com/milvus-io/milvus/internal/proto/datapb" + "github.com/milvus-io/milvus/internal/proto/milvuspb" "github.com/milvus-io/milvus/internal/proto/querypb" "github.com/milvus-io/milvus/internal/util/funcutil" "github.com/milvus-io/milvus/internal/util/timerecord" @@ -353,13 +354,20 @@ func (lct *loadCollectionTask) updateTaskProcess() { } func (lct *loadCollectionTask) preExecute(ctx context.Context) error { + if lct.ReplicaNumber < 1 { + log.Warn("replicaNumber is less than 1 for load collection request, will set it to 1", + zap.Int32("replicaNumber", lct.ReplicaNumber)) + lct.ReplicaNumber = 1 + } + collectionID := lct.CollectionID schema := lct.Schema lct.setResultInfo(nil) log.Debug("start do loadCollectionTask", zap.Int64("msgID", lct.getTaskID()), zap.Int64("collectionID", collectionID), - zap.Stringer("schema", schema)) + zap.Stringer("schema", schema), + zap.Int32("replicaNumber", lct.ReplicaNumber)) return nil } @@ -367,22 +375,24 @@ func (lct *loadCollectionTask) execute(ctx context.Context) error { defer lct.reduceRetryCount() collectionID := lct.CollectionID - toLoadPartitionIDs, err := lct.broker.showPartitionIDs(ctx, collectionID) + partitionIds, err := lct.broker.showPartitionIDs(ctx, collectionID) if err != nil { log.Error("loadCollectionTask: showPartition failed", zap.Int64("collectionID", collectionID), zap.Int64("msgID", lct.Base.MsgID), zap.Error(err)) lct.setResultInfo(err) return err } - log.Debug("loadCollectionTask: get collection's all partitionIDs", zap.Int64("collectionID", collectionID), zap.Int64s("partitionIDs", toLoadPartitionIDs), zap.Int64("msgID", lct.Base.MsgID)) + log.Debug("loadCollectionTask: get collection's all partitionIDs", zap.Int64("collectionID", collectionID), zap.Int64s("partitionIDs", partitionIds), zap.Int64("msgID", lct.Base.MsgID)) var ( - loadSegmentReqs = []*querypb.LoadSegmentsRequest{} - watchDmChannelReqs = []*querypb.WatchDmChannelsRequest{} - deltaChannelInfos = []*datapb.VchannelInfo{} - dmChannelInfos = []*datapb.VchannelInfo{} + replicas = make([]*milvuspb.ReplicaInfo, lct.ReplicaNumber) + replicaIds = make([]int64, lct.ReplicaNumber) + segmentLoadInfos = make([]*querypb.SegmentLoadInfo, 0) + deltaChannelInfos = make([]*datapb.VchannelInfo, 0) + dmChannelInfos = make([]*datapb.VchannelInfo, 0) + collectionSize uint64 ) - for _, partitionID := range toLoadPartitionIDs { + for _, partitionID := range partitionIds { vChannelInfos, binlogs, err := lct.broker.getRecoveryInfo(lct.ctx, collectionID, partitionID) if err != nil { log.Error("loadCollectionTask: getRecoveryInfo failed", zap.Int64("collectionID", collectionID), zap.Int64("partitionID", partitionID), zap.Int64("msgID", lct.Base.MsgID), zap.Error(err)) @@ -392,21 +402,8 @@ func (lct *loadCollectionTask) execute(ctx context.Context) error { for _, segmentBinlog := range binlogs { segmentLoadInfo := lct.broker.generateSegmentLoadInfo(ctx, collectionID, partitionID, segmentBinlog, true, lct.Schema) - msgBase := proto.Clone(lct.Base).(*commonpb.MsgBase) - msgBase.MsgType = commonpb.MsgType_LoadSegments - loadSegmentReq := &querypb.LoadSegmentsRequest{ - Base: msgBase, - Infos: []*querypb.SegmentLoadInfo{segmentLoadInfo}, - Schema: lct.Schema, - CollectionID: collectionID, - LoadMeta: &querypb.LoadMetaInfo{ - LoadType: querypb.LoadType_LoadCollection, - CollectionID: collectionID, - PartitionIDs: toLoadPartitionIDs, - }, - } - - loadSegmentReqs = append(loadSegmentReqs, loadSegmentReq) + collectionSize += uint64(segmentLoadInfo.SegmentSize) + segmentLoadInfos = append(segmentLoadInfos, segmentLoadInfo) } for _, info := range vChannelInfos { @@ -420,6 +417,7 @@ func (lct *loadCollectionTask) execute(ctx context.Context) error { dmChannelInfos = append(dmChannelInfos, info) } } + mergedDeltaChannels := mergeWatchDeltaChannelInfo(deltaChannelInfos) // If meta is not updated here, deltaChannel meta will not be available when loadSegment reschedule err = lct.meta.setDeltaChannel(collectionID, mergedDeltaChannels) @@ -429,40 +427,108 @@ func (lct *loadCollectionTask) execute(ctx context.Context) error { return err } - //TODO:: queryNode receive dm message according partitionID cache - //TODO:: queryNode add partitionID to cache if receive create partition message from dmChannel mergedDmChannel := mergeDmChannelInfo(dmChannelInfos) - for _, info := range mergedDmChannel { - msgBase := proto.Clone(lct.Base).(*commonpb.MsgBase) - msgBase.MsgType = commonpb.MsgType_WatchDmChannels - watchRequest := &querypb.WatchDmChannelsRequest{ - Base: msgBase, - CollectionID: collectionID, - //PartitionIDs: toLoadPartitionIDs, - Infos: []*datapb.VchannelInfo{info}, - Schema: lct.Schema, - LoadMeta: &querypb.LoadMetaInfo{ - LoadType: querypb.LoadType_LoadCollection, - CollectionID: collectionID, - PartitionIDs: toLoadPartitionIDs, - }, + + for i := range replicas { + replica, err := lct.meta.generateReplica(lct.CollectionID, partitionIds) + if err != nil { + lct.setResultInfo(err) + return err } - watchDmChannelReqs = append(watchDmChannelReqs, watchRequest) + replicas[i] = replica + replicaIds[i] = replica.ReplicaID } - internalTasks, err := assignInternalTask(ctx, lct, lct.meta, lct.cluster, loadSegmentReqs, watchDmChannelReqs, false, nil, nil) + err = lct.cluster.assignNodesToReplicas(ctx, replicas, collectionSize) if err != nil { - log.Error("loadCollectionTask: assign child task failed", zap.Int64("collectionID", collectionID), zap.Int64("msgID", lct.Base.MsgID), zap.Error(err)) + log.Error("failed to assign nodes to replicas", + zap.Int64("collectionID", collectionID), + zap.Int64s("partitionIDs", partitionIds), + zap.Int64("msgID", lct.Base.MsgID), + zap.Int32("replicaNumber", lct.ReplicaNumber), + zap.Error(err)) lct.setResultInfo(err) return err } - for _, internalTask := range internalTasks { - lct.addChildTask(internalTask) - log.Debug("loadCollectionTask: add a childTask", zap.Int64("collectionID", collectionID), zap.Int32("task type", int32(internalTask.msgType())), zap.Int64("msgID", lct.Base.MsgID)) + + for _, replica := range replicas { + var ( + loadSegmentReqs = []*querypb.LoadSegmentsRequest{} + watchDmChannelReqs = []*querypb.WatchDmChannelsRequest{} + ) + + for _, segmentLoadInfo := range segmentLoadInfos { + msgBase := proto.Clone(lct.Base).(*commonpb.MsgBase) + msgBase.MsgType = commonpb.MsgType_LoadSegments + loadSegmentReq := &querypb.LoadSegmentsRequest{ + Base: msgBase, + Infos: []*querypb.SegmentLoadInfo{segmentLoadInfo}, + Schema: lct.Schema, + CollectionID: collectionID, + LoadMeta: &querypb.LoadMetaInfo{ + LoadType: querypb.LoadType_LoadCollection, + CollectionID: collectionID, + PartitionIDs: partitionIds, + }, + ReplicaID: replica.ReplicaID, + } + + loadSegmentReqs = append(loadSegmentReqs, loadSegmentReq) + } + + //TODO:: queryNode receive dm message according partitionID cache + //TODO:: queryNode add partitionID to cache if receive create partition message from dmChannel + for _, info := range mergedDmChannel { + msgBase := proto.Clone(lct.Base).(*commonpb.MsgBase) + msgBase.MsgType = commonpb.MsgType_WatchDmChannels + watchRequest := &querypb.WatchDmChannelsRequest{ + Base: msgBase, + CollectionID: collectionID, + //PartitionIDs: toLoadPartitionIDs, + Infos: []*datapb.VchannelInfo{info}, + Schema: lct.Schema, + LoadMeta: &querypb.LoadMetaInfo{ + LoadType: querypb.LoadType_LoadCollection, + CollectionID: collectionID, + PartitionIDs: partitionIds, + }, + ReplicaID: replica.GetReplicaID(), + } + + watchDmChannelReqs = append(watchDmChannelReqs, watchRequest) + } + + internalTasks, err := assignInternalTask(ctx, lct, lct.meta, lct.cluster, loadSegmentReqs, watchDmChannelReqs, false, nil, replica.GetNodeIds(), -1) + if err != nil { + log.Error("loadCollectionTask: assign child task failed", zap.Int64("collectionID", collectionID), zap.Int64("msgID", lct.Base.MsgID), zap.Error(err)) + lct.setResultInfo(err) + return err + } + for _, internalTask := range internalTasks { + lct.addChildTask(internalTask) + if task, ok := internalTask.(*watchDmChannelTask); ok { + nodeInfo, err := lct.cluster.getNodeInfoByID(task.NodeID) + if err != nil { + log.Error("loadCollectionTask: get shard leader node info failed", + zap.Int64("collectionID", collectionID), + zap.Int64("msgID", lct.Base.MsgID), + zap.Int64("nodeID", task.NodeID), + zap.Error(err)) + lct.setResultInfo(err) + return err + } + replica.ShardReplicas = append(replica.ShardReplicas, &milvuspb.ShardReplica{ + LeaderID: task.NodeID, + LeaderAddr: nodeInfo.(*queryNode).address, + DmChannelName: task.WatchDmChannelsRequest.Infos[0].ChannelName, + }) + } + log.Debug("loadCollectionTask: add a childTask", zap.Int64("collectionID", collectionID), zap.Int32("task type", int32(internalTask.msgType())), zap.Int64("msgID", lct.Base.MsgID)) + } + metrics.QueryCoordNumChildTasks.WithLabelValues().Add(float64(len(internalTasks))) + log.Debug("loadCollectionTask: assign child task done", zap.Int64("collectionID", collectionID), zap.Int64("msgID", lct.Base.MsgID)) } - metrics.QueryCoordNumChildTasks.WithLabelValues().Add(float64(len(internalTasks))) - log.Debug("loadCollectionTask: assign child task done", zap.Int64("collectionID", collectionID), zap.Int64("msgID", lct.Base.MsgID)) err = lct.meta.addCollection(collectionID, querypb.LoadType_LoadCollection, lct.Schema) if err != nil { @@ -470,13 +536,23 @@ func (lct *loadCollectionTask) execute(ctx context.Context) error { lct.setResultInfo(err) return err } - err = lct.meta.addPartitions(collectionID, toLoadPartitionIDs) + + err = lct.meta.addPartitions(collectionID, partitionIds) if err != nil { - log.Error("loadCollectionTask: add partitions to meta failed", zap.Int64("collectionID", collectionID), zap.Int64s("partitionIDs", toLoadPartitionIDs), zap.Int64("msgID", lct.Base.MsgID), zap.Error(err)) + log.Error("loadCollectionTask: add partitions to meta failed", zap.Int64("collectionID", collectionID), zap.Int64s("partitionIDs", partitionIds), zap.Int64("msgID", lct.Base.MsgID), zap.Error(err)) lct.setResultInfo(err) return err } + for _, replica := range replicas { + err = lct.meta.addReplica(replica) + if err != nil { + log.Error("failed to add replica", zap.Int64("collectionID", collectionID), zap.Int64s("partitionIDs", partitionIds), zap.Int64("msgID", lct.Base.MsgID), zap.Int32("replicaNumber", lct.ReplicaNumber)) + lct.setResultInfo(err) + return err + } + } + log.Debug("LoadCollection execute done", zap.Int64("msgID", lct.getTaskID()), zap.Int64("collectionID", collectionID)) @@ -596,6 +672,7 @@ func (rct *releaseCollectionTask) execute(ctx context.Context) error { return err } + // TODO(yah01): broadcast to all nodes? Or only nodes serve the collection onlineNodeIDs := rct.cluster.onlineNodeIDs() for _, nodeID := range onlineNodeIDs { req := proto.Clone(rct.ReleaseCollectionRequest).(*querypb.ReleaseCollectionRequest) @@ -720,6 +797,12 @@ func (lpt *loadPartitionTask) updateTaskProcess() { } func (lpt *loadPartitionTask) preExecute(context.Context) error { + if lpt.ReplicaNumber < 1 { + log.Warn("replicaNumber is less than 1 for load partitions request, will set it to 1", + zap.Int32("replicaNumber", lpt.ReplicaNumber)) + lpt.ReplicaNumber = 1 + } + collectionID := lpt.CollectionID lpt.setResultInfo(nil) log.Debug("start do loadPartitionTask", @@ -733,10 +816,15 @@ func (lpt *loadPartitionTask) execute(ctx context.Context) error { collectionID := lpt.CollectionID partitionIDs := lpt.PartitionIDs - var loadSegmentReqs []*querypb.LoadSegmentsRequest - var watchDmChannelReqs []*querypb.WatchDmChannelsRequest - var deltaChannelInfos []*datapb.VchannelInfo - var dmChannelInfos []*datapb.VchannelInfo + var ( + replicas = make([]*milvuspb.ReplicaInfo, lpt.ReplicaNumber) + replicaIds = make([]int64, lpt.ReplicaNumber) + segmentLoadInfos = make([]*querypb.SegmentLoadInfo, 0) + deltaChannelInfos = make([]*datapb.VchannelInfo, 0) + dmChannelInfos = make([]*datapb.VchannelInfo, 0) + collectionSize uint64 + ) + for _, partitionID := range partitionIDs { vChannelInfos, binlogs, err := lpt.broker.getRecoveryInfo(lpt.ctx, collectionID, partitionID) if err != nil { @@ -747,20 +835,8 @@ func (lpt *loadPartitionTask) execute(ctx context.Context) error { for _, segmentBingLog := range binlogs { segmentLoadInfo := lpt.broker.generateSegmentLoadInfo(ctx, collectionID, partitionID, segmentBingLog, true, lpt.Schema) - msgBase := proto.Clone(lpt.Base).(*commonpb.MsgBase) - msgBase.MsgType = commonpb.MsgType_LoadSegments - loadSegmentReq := &querypb.LoadSegmentsRequest{ - Base: msgBase, - Infos: []*querypb.SegmentLoadInfo{segmentLoadInfo}, - Schema: lpt.Schema, - CollectionID: collectionID, - LoadMeta: &querypb.LoadMetaInfo{ - LoadType: querypb.LoadType_LoadPartition, - CollectionID: collectionID, - PartitionIDs: partitionIDs, - }, - } - loadSegmentReqs = append(loadSegmentReqs, loadSegmentReq) + segmentLoadInfos = append(segmentLoadInfos, segmentLoadInfo) + collectionSize += uint64(segmentLoadInfo.SegmentSize) } for _, info := range vChannelInfos { @@ -784,37 +860,105 @@ func (lpt *loadPartitionTask) execute(ctx context.Context) error { } mergedDmChannel := mergeDmChannelInfo(dmChannelInfos) - for _, info := range mergedDmChannel { - msgBase := proto.Clone(lpt.Base).(*commonpb.MsgBase) - msgBase.MsgType = commonpb.MsgType_WatchDmChannels - watchRequest := &querypb.WatchDmChannelsRequest{ - Base: msgBase, - CollectionID: collectionID, - PartitionIDs: partitionIDs, - Infos: []*datapb.VchannelInfo{info}, - Schema: lpt.Schema, - LoadMeta: &querypb.LoadMetaInfo{ - LoadType: querypb.LoadType_LoadPartition, - CollectionID: collectionID, - PartitionIDs: partitionIDs, - }, + + for i := range replicas { + replica, err := lpt.meta.generateReplica(lpt.CollectionID, partitionIDs) + if err != nil { + lpt.setResultInfo(err) + return err } - watchDmChannelReqs = append(watchDmChannelReqs, watchRequest) + replicas[i] = replica + replicaIds[i] = replica.ReplicaID } - internalTasks, err := assignInternalTask(ctx, lpt, lpt.meta, lpt.cluster, loadSegmentReqs, watchDmChannelReqs, false, nil, nil) + err = lpt.cluster.assignNodesToReplicas(ctx, replicas, collectionSize) if err != nil { - log.Error("loadPartitionTask: assign child task failed", zap.Int64("collectionID", collectionID), zap.Int64s("partitionIDs", partitionIDs), zap.Int64("msgID", lpt.Base.MsgID), zap.Error(err)) + log.Error("failed to assign nodes to replicas", + zap.Int64("collectionID", collectionID), + zap.Int64s("partitionIDs", partitionIDs), + zap.Int64("msgID", lpt.Base.MsgID), + zap.Int32("replicaNumber", lpt.ReplicaNumber), + zap.Error(err)) lpt.setResultInfo(err) return err } - for _, internalTask := range internalTasks { - lpt.addChildTask(internalTask) - log.Debug("loadPartitionTask: add a childTask", zap.Int64("collectionID", collectionID), zap.Int32("task type", int32(internalTask.msgType()))) + + for _, replica := range replicas { + var ( + loadSegmentReqs = []*querypb.LoadSegmentsRequest{} + watchDmChannelReqs = []*querypb.WatchDmChannelsRequest{} + ) + + for _, segmentLoadInfo := range segmentLoadInfos { + msgBase := proto.Clone(lpt.Base).(*commonpb.MsgBase) + msgBase.MsgType = commonpb.MsgType_LoadSegments + loadSegmentReq := &querypb.LoadSegmentsRequest{ + Base: msgBase, + Infos: []*querypb.SegmentLoadInfo{segmentLoadInfo}, + Schema: lpt.Schema, + CollectionID: collectionID, + LoadMeta: &querypb.LoadMetaInfo{ + LoadType: querypb.LoadType_LoadPartition, + CollectionID: collectionID, + PartitionIDs: partitionIDs, + }, + ReplicaID: replica.ReplicaID, + } + loadSegmentReqs = append(loadSegmentReqs, loadSegmentReq) + } + + for _, info := range mergedDmChannel { + msgBase := proto.Clone(lpt.Base).(*commonpb.MsgBase) + msgBase.MsgType = commonpb.MsgType_WatchDmChannels + watchRequest := &querypb.WatchDmChannelsRequest{ + Base: msgBase, + CollectionID: collectionID, + PartitionIDs: partitionIDs, + Infos: []*datapb.VchannelInfo{info}, + Schema: lpt.Schema, + LoadMeta: &querypb.LoadMetaInfo{ + LoadType: querypb.LoadType_LoadPartition, + CollectionID: collectionID, + PartitionIDs: partitionIDs, + }, + ReplicaID: replica.GetReplicaID(), + } + + watchDmChannelReqs = append(watchDmChannelReqs, watchRequest) + } + + internalTasks, err := assignInternalTask(ctx, lpt, lpt.meta, lpt.cluster, loadSegmentReqs, watchDmChannelReqs, false, nil, replica.GetNodeIds(), -1) + if err != nil { + log.Error("loadPartitionTask: assign child task failed", zap.Int64("collectionID", collectionID), zap.Int64s("partitionIDs", partitionIDs), zap.Int64("msgID", lpt.Base.MsgID), zap.Error(err)) + lpt.setResultInfo(err) + return err + } + for _, internalTask := range internalTasks { + lpt.addChildTask(internalTask) + if task, ok := internalTask.(*watchDmChannelTask); ok { + nodeInfo, err := lpt.cluster.getNodeInfoByID(task.NodeID) + if err != nil { + log.Error("loadCollectionTask: get shard leader node info failed", + zap.Int64("collectionID", collectionID), + zap.Int64("msgID", lpt.Base.MsgID), + zap.Int64("nodeID", task.NodeID), + zap.Error(err)) + lpt.setResultInfo(err) + return err + } + + replica.ShardReplicas = append(replica.ShardReplicas, &milvuspb.ShardReplica{ + LeaderID: task.NodeID, + LeaderAddr: nodeInfo.(*queryNode).address, + DmChannelName: task.WatchDmChannelsRequest.Infos[0].ChannelName, + }) + } + log.Debug("loadPartitionTask: add a childTask", zap.Int64("collectionID", collectionID), zap.Int32("task type", int32(internalTask.msgType()))) + } + metrics.QueryCoordNumChildTasks.WithLabelValues().Add(float64(len(internalTasks))) + log.Debug("loadPartitionTask: assign child task done", zap.Int64("collectionID", collectionID), zap.Int64s("partitionIDs", partitionIDs), zap.Int64("msgID", lpt.Base.MsgID)) } - metrics.QueryCoordNumChildTasks.WithLabelValues().Add(float64(len(internalTasks))) - log.Debug("loadPartitionTask: assign child task done", zap.Int64("collectionID", collectionID), zap.Int64s("partitionIDs", partitionIDs), zap.Int64("msgID", lpt.Base.MsgID)) err = lpt.meta.addCollection(collectionID, querypb.LoadType_LoadPartition, lpt.Schema) if err != nil { @@ -830,6 +974,15 @@ func (lpt *loadPartitionTask) execute(ctx context.Context) error { return err } + for _, replica := range replicas { + err = lpt.meta.addReplica(replica) + if err != nil { + log.Error("failed to add replica", zap.Int64("collectionID", collectionID), zap.Int64s("partitionIDs", partitionIDs), zap.Int64("msgID", lpt.Base.MsgID), zap.Int32("replicaNumber", lpt.ReplicaNumber)) + lpt.setResultInfo(err) + return err + } + } + log.Debug("loadPartitionTask Execute done", zap.Int64("msgID", lpt.getTaskID()), zap.Int64("collectionID", collectionID), @@ -1098,6 +1251,7 @@ func (lst *loadSegmentTask) reschedule(ctx context.Context) ([]task, error) { CollectionID: lst.GetCollectionID(), PartitionIDs: lst.GetLoadMeta().GetPartitionIDs(), }, + ReplicaID: lst.ReplicaID, } loadSegmentReqs = append(loadSegmentReqs, req) } @@ -1110,7 +1264,7 @@ func (lst *loadSegmentTask) reschedule(ctx context.Context) ([]task, error) { if lst.getParentTask().getTriggerCondition() == querypb.TriggerCondition_NodeDown { wait2AssignTaskSuccess = true } - reScheduledTasks, err := assignInternalTask(ctx, lst.getParentTask(), lst.meta, lst.cluster, loadSegmentReqs, nil, wait2AssignTaskSuccess, lst.excludeNodeIDs, nil) + reScheduledTasks, err := assignInternalTask(ctx, lst.getParentTask(), lst.meta, lst.cluster, loadSegmentReqs, nil, wait2AssignTaskSuccess, lst.excludeNodeIDs, nil, lst.ReplicaID) if err != nil { log.Error("loadSegment reschedule failed", zap.Int64s("excludeNodes", lst.excludeNodeIDs), zap.Int64("taskID", lst.getTaskID()), zap.Error(err)) return nil, err @@ -1277,6 +1431,7 @@ func (wdt *watchDmChannelTask) reschedule(ctx context.Context) ([]task, error) { CollectionID: collectionID, PartitionIDs: wdt.GetLoadMeta().GetPartitionIDs(), }, + ReplicaID: wdt.GetReplicaID(), } watchDmChannelReqs = append(watchDmChannelReqs, req) } @@ -1289,7 +1444,7 @@ func (wdt *watchDmChannelTask) reschedule(ctx context.Context) ([]task, error) { if wdt.getParentTask().getTriggerCondition() == querypb.TriggerCondition_NodeDown { wait2AssignTaskSuccess = true } - reScheduledTasks, err := assignInternalTask(ctx, wdt.parentTask, wdt.meta, wdt.cluster, nil, watchDmChannelReqs, wait2AssignTaskSuccess, wdt.excludeNodeIDs, nil) + reScheduledTasks, err := assignInternalTask(ctx, wdt.parentTask, wdt.meta, wdt.cluster, nil, watchDmChannelReqs, wait2AssignTaskSuccess, wdt.excludeNodeIDs, nil, wdt.ReplicaID) if err != nil { log.Error("watchDmChannel reschedule failed", zap.Int64("taskID", wdt.getTaskID()), zap.Int64s("excludeNodes", wdt.excludeNodeIDs), zap.Error(err)) return nil, err @@ -1578,12 +1733,30 @@ func (ht *handoffTask) execute(ctx context.Context) error { ht.setResultInfo(err) return err } - internalTasks, err := assignInternalTask(ctx, ht, ht.meta, ht.cluster, []*querypb.LoadSegmentsRequest{loadSegmentReq}, nil, true, nil, nil) + replicas, err := ht.meta.getReplicasByCollectionID(collectionID) if err != nil { - log.Error("handoffTask: assign child task failed", zap.Int64("collectionID", collectionID), zap.Int64("segmentID", segmentID), zap.Error(err)) ht.setResultInfo(err) return err } + var internalTasks []task + for _, replica := range replicas { + if len(replica.NodeIds) == 0 { + log.Warn("handoffTask: find empty replica", zap.Int64("collectionID", collectionID), zap.Int64("segmentID", segmentID), zap.Int64("replicaID", replica.GetReplicaID())) + err := fmt.Errorf("replica %d of collection %d is empty", replica.GetReplicaID(), collectionID) + ht.setResultInfo(err) + return err + } + // we should copy a request because assignInternalTask will change DstNodeID of LoadSegmentRequest + clonedReq := proto.Clone(loadSegmentReq).(*querypb.LoadSegmentsRequest) + clonedReq.ReplicaID = replica.ReplicaID + tasks, err := assignInternalTask(ctx, ht, ht.meta, ht.cluster, []*querypb.LoadSegmentsRequest{clonedReq}, nil, true, nil, nil, replica.GetReplicaID()) + if err != nil { + log.Error("handoffTask: assign child task failed", zap.Int64("collectionID", collectionID), zap.Int64("segmentID", segmentID), zap.Error(err)) + ht.setResultInfo(err) + return err + } + internalTasks = append(internalTasks, tasks...) + } for _, internalTask := range internalTasks { ht.addChildTask(internalTask) log.Debug("handoffTask: add a childTask", zap.Int32("task type", int32(internalTask.msgType())), zap.Int64("segmentID", segmentID)) @@ -1629,9 +1802,10 @@ func (ht *handoffTask) rollBack(ctx context.Context) []task { type loadBalanceTask struct { *baseTask *querypb.LoadBalanceRequest - broker *globalMetaBroker - cluster Cluster - meta Meta + broker *globalMetaBroker + cluster Cluster + meta Meta + replicaID int64 } func (lbt *loadBalanceTask) msgBase() *commonpb.MsgBase { @@ -1660,16 +1834,76 @@ func (lbt *loadBalanceTask) preExecute(context.Context) error { return nil } +func (lbt *loadBalanceTask) checkForManualLoadBalance() error { + // check segments belong to the same collection + collectionID := lbt.GetCollectionID() + for _, sid := range lbt.SealedSegmentIDs { + segment, err := lbt.meta.getSegmentInfoByID(sid) + if err != nil { + return err + } + if collectionID == 0 { + collectionID = segment.GetCollectionID() + } else if collectionID != segment.GetCollectionID() { + err := errors.New("segments of a load balance task do not belong to the same collection") + return err + } + } + + if collectionID == 0 { + err := errors.New("a load balance task has to specify a collectionID or pass segments of a collection") + return err + } + + // check source and dst nodes belong to the same replica + var replicaID int64 = -1 + for _, nodeID := range lbt.SourceNodeIDs { + replica, err := lbt.getReplica(nodeID, collectionID) + if err != nil { + return err + } + if replicaID == -1 { + replicaID = replica.GetReplicaID() + } else if replicaID != replica.GetReplicaID() { + err := errors.New("source nodes and destination nodes must be in the same replica group") + return err + } + } + + if replicaID == -1 { + return errors.New("source nodes is empty") + } + + for _, nodeID := range lbt.DstNodeIDs { + replica, err := lbt.getReplica(nodeID, collectionID) + if err != nil { + return err + } + if replicaID != replica.GetReplicaID() { + err := errors.New("source nodes and destination nodes must be in the same replica group") + return err + } + } + + lbt.replicaID = replicaID + + log.Debug("start do loadBalanceTask", + zap.Int32("trigger type", int32(lbt.triggerCondition)), + zap.Int64s("sourceNodeIDs", lbt.SourceNodeIDs), + zap.Any("balanceReason", lbt.BalanceReason), + zap.Int64("taskID", lbt.getTaskID())) + return nil +} + func (lbt *loadBalanceTask) execute(ctx context.Context) error { defer lbt.reduceRetryCount() if lbt.triggerCondition == querypb.TriggerCondition_NodeDown { - segmentID2Info := make(map[UniqueID]*querypb.SegmentInfo) - dmChannel2WatchInfo := make(map[string]*querypb.DmChannelWatchInfo) - loadSegmentReqs := make([]*querypb.LoadSegmentsRequest, 0) - watchDmChannelReqs := make([]*querypb.WatchDmChannelsRequest, 0) - recoveredCollectionIDs := make(map[UniqueID]struct{}) + var internalTasks []task for _, nodeID := range lbt.SourceNodeIDs { + segmentID2Info := make(map[UniqueID]*querypb.SegmentInfo) + dmChannel2WatchInfo := make(map[string]*querypb.DmChannelWatchInfo) + recoveredCollectionIDs := make(map[UniqueID]struct{}) segmentInfos := lbt.meta.getSegmentInfosByNode(nodeID) for _, segmentInfo := range segmentInfos { segmentID2Info[segmentInfo.SegmentID] = segmentInfo @@ -1680,113 +1914,127 @@ func (lbt *loadBalanceTask) execute(ctx context.Context) error { dmChannel2WatchInfo[watchInfo.DmChannel] = watchInfo recoveredCollectionIDs[watchInfo.CollectionID] = struct{}{} } - } - for collectionID := range recoveredCollectionIDs { - collectionInfo, err := lbt.meta.getCollectionInfoByID(collectionID) - if err != nil { - log.Error("loadBalanceTask: get collectionInfo from meta failed", zap.Int64("collectionID", collectionID), zap.Error(err)) - lbt.setResultInfo(err) - return err - } - schema := collectionInfo.Schema - var deltaChannelInfos []*datapb.VchannelInfo - var dmChannelInfos []*datapb.VchannelInfo - - var toRecoverPartitionIDs []UniqueID - if collectionInfo.LoadType == querypb.LoadType_LoadCollection { - toRecoverPartitionIDs, err = lbt.broker.showPartitionIDs(ctx, collectionID) + for collectionID := range recoveredCollectionIDs { + loadSegmentReqs := make([]*querypb.LoadSegmentsRequest, 0) + watchDmChannelReqs := make([]*querypb.WatchDmChannelsRequest, 0) + collectionInfo, err := lbt.meta.getCollectionInfoByID(collectionID) if err != nil { - log.Error("loadBalanceTask: show collection's partitionIDs failed", zap.Int64("collectionID", collectionID), zap.Error(err)) + log.Error("loadBalanceTask: get collectionInfo from meta failed", zap.Int64("collectionID", collectionID), zap.Error(err)) lbt.setResultInfo(err) - panic(err) + return err } - } else { - toRecoverPartitionIDs = collectionInfo.PartitionIDs - } - log.Debug("loadBalanceTask: get collection's all partitionIDs", zap.Int64("collectionID", collectionID), zap.Int64s("partitionIDs", toRecoverPartitionIDs)) + schema := collectionInfo.Schema + var deltaChannelInfos []*datapb.VchannelInfo + var dmChannelInfos []*datapb.VchannelInfo - for _, partitionID := range toRecoverPartitionIDs { - vChannelInfos, binlogs, err := lbt.broker.getRecoveryInfo(lbt.ctx, collectionID, partitionID) + var toRecoverPartitionIDs []UniqueID + if collectionInfo.LoadType == querypb.LoadType_LoadCollection { + toRecoverPartitionIDs, err = lbt.broker.showPartitionIDs(ctx, collectionID) + if err != nil { + log.Error("loadBalanceTask: show collection's partitionIDs failed", zap.Int64("collectionID", collectionID), zap.Error(err)) + lbt.setResultInfo(err) + panic(err) + } + } else { + toRecoverPartitionIDs = collectionInfo.PartitionIDs + } + log.Debug("loadBalanceTask: get collection's all partitionIDs", zap.Int64("collectionID", collectionID), zap.Int64s("partitionIDs", toRecoverPartitionIDs)) + replica, err := lbt.getReplica(nodeID, collectionID) if err != nil { - log.Error("loadBalanceTask: getRecoveryInfo failed", zap.Int64("collectionID", collectionID), zap.Int64("partitionID", partitionID), zap.Error(err)) + lbt.setResultInfo(err) + return err + } + + for _, partitionID := range toRecoverPartitionIDs { + vChannelInfos, binlogs, err := lbt.broker.getRecoveryInfo(lbt.ctx, collectionID, partitionID) + if err != nil { + log.Error("loadBalanceTask: getRecoveryInfo failed", zap.Int64("collectionID", collectionID), zap.Int64("partitionID", partitionID), zap.Error(err)) + lbt.setResultInfo(err) + panic(err) + } + + for _, segmentBingLog := range binlogs { + segmentID := segmentBingLog.SegmentID + if info, ok := segmentID2Info[segmentID]; ok { + segmentLoadInfo := lbt.broker.generateSegmentLoadInfo(ctx, collectionID, partitionID, segmentBingLog, true, schema) + msgBase := proto.Clone(lbt.Base).(*commonpb.MsgBase) + msgBase.MsgType = commonpb.MsgType_LoadSegments + for _, replica := range info.ReplicaIds { + loadSegmentReq := &querypb.LoadSegmentsRequest{ + Base: msgBase, + Infos: []*querypb.SegmentLoadInfo{segmentLoadInfo}, + Schema: schema, + CollectionID: collectionID, + + LoadMeta: &querypb.LoadMetaInfo{ + LoadType: collectionInfo.LoadType, + CollectionID: collectionID, + PartitionIDs: toRecoverPartitionIDs, + }, + ReplicaID: replica, + } + + loadSegmentReqs = append(loadSegmentReqs, loadSegmentReq) + } + } + } + + for _, info := range vChannelInfos { + deltaChannel, err := generateWatchDeltaChannelInfo(info) + if err != nil { + log.Error("loadBalanceTask: generateWatchDeltaChannelInfo failed", zap.Int64("collectionID", collectionID), zap.String("channelName", info.ChannelName), zap.Error(err)) + lbt.setResultInfo(err) + panic(err) + } + deltaChannelInfos = append(deltaChannelInfos, deltaChannel) + dmChannelInfos = append(dmChannelInfos, info) + } + } + + mergedDeltaChannel := mergeWatchDeltaChannelInfo(deltaChannelInfos) + // If meta is not updated here, deltaChannel meta will not be available when loadSegment reschedule + err = lbt.meta.setDeltaChannel(collectionID, mergedDeltaChannel) + if err != nil { + log.Error("loadBalanceTask: set delta channel info meta failed", zap.Int64("collectionID", collectionID), zap.Error(err)) lbt.setResultInfo(err) panic(err) } - for _, segmentBingLog := range binlogs { - segmentID := segmentBingLog.SegmentID - if _, ok := segmentID2Info[segmentID]; ok { - segmentLoadInfo := lbt.broker.generateSegmentLoadInfo(ctx, collectionID, partitionID, segmentBingLog, true, schema) + mergedDmChannel := mergeDmChannelInfo(dmChannelInfos) + for channelName, vChannelInfo := range mergedDmChannel { + if info, ok := dmChannel2WatchInfo[channelName]; ok { msgBase := proto.Clone(lbt.Base).(*commonpb.MsgBase) - msgBase.MsgType = commonpb.MsgType_LoadSegments - loadSegmentReq := &querypb.LoadSegmentsRequest{ + msgBase.MsgType = commonpb.MsgType_WatchDmChannels + watchRequest := &querypb.WatchDmChannelsRequest{ Base: msgBase, - Infos: []*querypb.SegmentLoadInfo{segmentLoadInfo}, - Schema: schema, CollectionID: collectionID, + Infos: []*datapb.VchannelInfo{vChannelInfo}, + Schema: schema, LoadMeta: &querypb.LoadMetaInfo{ LoadType: collectionInfo.LoadType, CollectionID: collectionID, PartitionIDs: toRecoverPartitionIDs, }, + ReplicaID: info.ReplicaID, } - loadSegmentReqs = append(loadSegmentReqs, loadSegmentReq) + if collectionInfo.LoadType == querypb.LoadType_LoadPartition { + watchRequest.PartitionIDs = toRecoverPartitionIDs + } + + watchDmChannelReqs = append(watchDmChannelReqs, watchRequest) } } - for _, info := range vChannelInfos { - deltaChannel, err := generateWatchDeltaChannelInfo(info) - if err != nil { - log.Error("loadBalanceTask: generateWatchDeltaChannelInfo failed", zap.Int64("collectionID", collectionID), zap.String("channelName", info.ChannelName), zap.Error(err)) - lbt.setResultInfo(err) - panic(err) - } - deltaChannelInfos = append(deltaChannelInfos, deltaChannel) - dmChannelInfos = append(dmChannelInfos, info) + tasks, err := assignInternalTask(ctx, lbt, lbt.meta, lbt.cluster, loadSegmentReqs, watchDmChannelReqs, true, lbt.SourceNodeIDs, lbt.DstNodeIDs, replica.GetReplicaID()) + if err != nil { + log.Error("loadBalanceTask: assign child task failed", zap.Int64("sourceNodeID", nodeID)) + lbt.setResultInfo(err) + panic(err) } + internalTasks = append(internalTasks, tasks...) } - - mergedDeltaChannel := mergeWatchDeltaChannelInfo(deltaChannelInfos) - // If meta is not updated here, deltaChannel meta will not be available when loadSegment reschedule - err = lbt.meta.setDeltaChannel(collectionID, mergedDeltaChannel) - if err != nil { - log.Error("loadBalanceTask: set delta channel info meta failed", zap.Int64("collectionID", collectionID), zap.Error(err)) - lbt.setResultInfo(err) - panic(err) - } - - mergedDmChannel := mergeDmChannelInfo(dmChannelInfos) - for channelName, vChannelInfo := range mergedDmChannel { - if _, ok := dmChannel2WatchInfo[channelName]; ok { - msgBase := proto.Clone(lbt.Base).(*commonpb.MsgBase) - msgBase.MsgType = commonpb.MsgType_WatchDmChannels - watchRequest := &querypb.WatchDmChannelsRequest{ - Base: msgBase, - CollectionID: collectionID, - Infos: []*datapb.VchannelInfo{vChannelInfo}, - Schema: schema, - LoadMeta: &querypb.LoadMetaInfo{ - LoadType: collectionInfo.LoadType, - CollectionID: collectionID, - PartitionIDs: toRecoverPartitionIDs, - }, - } - - if collectionInfo.LoadType == querypb.LoadType_LoadPartition { - watchRequest.PartitionIDs = toRecoverPartitionIDs - } - - watchDmChannelReqs = append(watchDmChannelReqs, watchRequest) - } - } - } - internalTasks, err := assignInternalTask(ctx, lbt, lbt.meta, lbt.cluster, loadSegmentReqs, watchDmChannelReqs, true, lbt.SourceNodeIDs, lbt.DstNodeIDs) - if err != nil { - log.Error("loadBalanceTask: assign child task failed", zap.Int64s("sourceNodeIDs", lbt.SourceNodeIDs)) - lbt.setResultInfo(err) - panic(err) } for _, internalTask := range internalTasks { lbt.addChildTask(internalTask) @@ -1796,6 +2044,10 @@ func (lbt *loadBalanceTask) execute(ctx context.Context) error { } if lbt.triggerCondition == querypb.TriggerCondition_LoadBalance { + if err := lbt.checkForManualLoadBalance(); err != nil { + lbt.setResultInfo(err) + return err + } if len(lbt.SourceNodeIDs) == 0 { err := errors.New("loadBalanceTask: empty source Node list to balance") log.Error(err.Error()) @@ -1879,17 +2131,21 @@ func (lbt *loadBalanceTask) execute(ctx context.Context) error { log.Warn("loadBalanceTask: can't find binlog of segment to balance, may be has been compacted", zap.Int64("segmentID", segmentID)) continue } - segmentBingLog := segmentID2Binlog[segmentID] - segmentLoadInfo := lbt.broker.generateSegmentLoadInfo(ctx, collectionID, partitionID, segmentBingLog, true, collectionInfo.Schema) - msgBase := proto.Clone(lbt.Base).(*commonpb.MsgBase) - msgBase.MsgType = commonpb.MsgType_LoadSegments - loadSegmentReq := &querypb.LoadSegmentsRequest{ - Base: msgBase, - Infos: []*querypb.SegmentLoadInfo{segmentLoadInfo}, - Schema: collectionInfo.Schema, - CollectionID: collectionID, + + for _, replica := range segmentInfo.ReplicaIds { + segmentBingLog := segmentID2Binlog[segmentID] + segmentLoadInfo := lbt.broker.generateSegmentLoadInfo(ctx, collectionID, partitionID, segmentBingLog, true, collectionInfo.Schema) + msgBase := proto.Clone(lbt.Base).(*commonpb.MsgBase) + msgBase.MsgType = commonpb.MsgType_LoadSegments + loadSegmentReq := &querypb.LoadSegmentsRequest{ + Base: msgBase, + Infos: []*querypb.SegmentLoadInfo{segmentLoadInfo}, + Schema: collectionInfo.Schema, + CollectionID: collectionID, + ReplicaID: replica, + } + loadSegmentReqs = append(loadSegmentReqs, loadSegmentReq) } - loadSegmentReqs = append(loadSegmentReqs, loadSegmentReq) } for _, info := range dmChannelInfos { @@ -1909,7 +2165,7 @@ func (lbt *loadBalanceTask) execute(ctx context.Context) error { return err } } - internalTasks, err := assignInternalTask(ctx, lbt, lbt.meta, lbt.cluster, loadSegmentReqs, nil, false, lbt.SourceNodeIDs, lbt.DstNodeIDs) + internalTasks, err := assignInternalTask(ctx, lbt, lbt.meta, lbt.cluster, loadSegmentReqs, nil, false, lbt.SourceNodeIDs, lbt.DstNodeIDs, lbt.replicaID) if err != nil { log.Error("loadBalanceTask: assign child task failed", zap.Any("balance request", lbt.LoadBalanceRequest)) lbt.setResultInfo(err) @@ -1930,6 +2186,19 @@ func (lbt *loadBalanceTask) execute(ctx context.Context) error { return nil } +func (lbt *loadBalanceTask) getReplica(nodeID, collectionID int64) (*milvuspb.ReplicaInfo, error) { + replicas, err := lbt.meta.getReplicasByNodeID(nodeID) + if err != nil { + return nil, err + } + for _, replica := range replicas { + if replica.GetCollectionID() == collectionID { + return replica, nil + } + } + return nil, fmt.Errorf("unable to find replicas of collection %d and node %d", collectionID, nodeID) +} + func (lbt *loadBalanceTask) postExecute(context.Context) error { if lbt.getResultInfo().ErrorCode != commonpb.ErrorCode_Success { lbt.clearChildTasks() @@ -1960,17 +2229,17 @@ func assignInternalTask(ctx context.Context, parentTask task, meta Meta, cluster Cluster, loadSegmentRequests []*querypb.LoadSegmentsRequest, watchDmChannelRequests []*querypb.WatchDmChannelsRequest, - wait bool, excludeNodeIDs []int64, includeNodeIDs []int64) ([]task, error) { + wait bool, excludeNodeIDs []int64, includeNodeIDs []int64, replicaID int64) ([]task, error) { log.Debug("assignInternalTask: start assign task to query node") internalTasks := make([]task, 0) - err := cluster.allocateSegmentsToQueryNode(ctx, loadSegmentRequests, wait, excludeNodeIDs, includeNodeIDs) + err := cluster.allocateSegmentsToQueryNode(ctx, loadSegmentRequests, wait, excludeNodeIDs, includeNodeIDs, replicaID) if err != nil { log.Error("assignInternalTask: assign segment to node failed", zap.Any("load segments requests", loadSegmentRequests)) return nil, err } log.Debug("assignInternalTask: assign segment to node success") - err = cluster.allocateChannelsToQueryNode(ctx, watchDmChannelRequests, wait, excludeNodeIDs) + err = cluster.allocateChannelsToQueryNode(ctx, watchDmChannelRequests, wait, excludeNodeIDs, includeNodeIDs, replicaID) if err != nil { log.Error("assignInternalTask: assign dmChannel to node failed", zap.Any("watch dmChannel requests", watchDmChannelRequests)) return nil, err diff --git a/internal/querycoord/task_scheduler.go b/internal/querycoord/task_scheduler.go index a47dc9a835..71356a07ce 100644 --- a/internal/querycoord/task_scheduler.go +++ b/internal/querycoord/task_scheduler.go @@ -905,6 +905,8 @@ func updateSegmentInfoFromTask(ctx context.Context, triggerTask task, meta Meta) sealedSegmentChangeInfos, err = meta.removeGlobalSealedSegInfos(collectionID, req.PartitionIDs) default: // save new segmentInfo when load segment + segments := make(map[UniqueID]*querypb.SegmentInfo) + for _, childTask := range triggerTask.getChildTask() { if childTask.msgType() == commonpb.MsgType_LoadSegments { req := childTask.(*loadSegmentTask).LoadSegmentsRequest @@ -912,18 +914,30 @@ func updateSegmentInfoFromTask(ctx context.Context, triggerTask task, meta Meta) for _, loadInfo := range req.Infos { collectionID := loadInfo.CollectionID segmentID := loadInfo.SegmentID - segmentInfo := &querypb.SegmentInfo{ - SegmentID: segmentID, - CollectionID: loadInfo.CollectionID, - PartitionID: loadInfo.PartitionID, - NodeID: dstNodeID, - SegmentState: commonpb.SegmentState_Sealed, - CompactionFrom: loadInfo.CompactionFrom, + + segment, ok := segments[segmentID] + if !ok { + segment = &querypb.SegmentInfo{ + SegmentID: segmentID, + CollectionID: loadInfo.CollectionID, + PartitionID: loadInfo.PartitionID, + NodeID: dstNodeID, + DmChannel: loadInfo.InsertChannel, + SegmentState: commonpb.SegmentState_Sealed, + CompactionFrom: loadInfo.CompactionFrom, + ReplicaIds: []UniqueID{req.ReplicaID}, + NodeIds: []UniqueID{dstNodeID}, + } + segments[segmentID] = segment + } else { + segment.ReplicaIds = append(segment.ReplicaIds, req.ReplicaID) + segment.NodeIds = append(segment.NodeIds, dstNodeID) } + if _, ok := segmentInfosToSave[collectionID]; !ok { segmentInfosToSave[collectionID] = make([]*querypb.SegmentInfo, 0) } - segmentInfosToSave[collectionID] = append(segmentInfosToSave[collectionID], segmentInfo) + segmentInfosToSave[collectionID] = append(segmentInfosToSave[collectionID], segment) } } } diff --git a/internal/querycoord/task_scheduler_test.go b/internal/querycoord/task_scheduler_test.go index d685d27402..6c854ddc6a 100644 --- a/internal/querycoord/task_scheduler_test.go +++ b/internal/querycoord/task_scheduler_test.go @@ -226,6 +226,7 @@ func TestUnMarshalTask(t *testing.T) { Base: &commonpb.MsgBase{ MsgType: commonpb.MsgType_LoadCollection, }, + ReplicaNumber: 1, }, } blobs, err := loadTask.marshal() @@ -478,6 +479,7 @@ func TestReloadTaskFromKV(t *testing.T) { Timestamp: 1, MsgType: commonpb.MsgType_LoadCollection, }, + ReplicaNumber: 1, }, } triggerBlobs, err := triggerTask.marshal() diff --git a/internal/querycoord/task_test.go b/internal/querycoord/task_test.go index b7a8023b28..67c9091de3 100644 --- a/internal/querycoord/task_test.go +++ b/internal/querycoord/task_test.go @@ -35,8 +35,9 @@ func genLoadCollectionTask(ctx context.Context, queryCoord *QueryCoord) *loadCol Base: &commonpb.MsgBase{ MsgType: commonpb.MsgType_LoadCollection, }, - CollectionID: defaultCollectionID, - Schema: genDefaultCollectionSchema(false), + CollectionID: defaultCollectionID, + Schema: genDefaultCollectionSchema(false), + ReplicaNumber: 1, } baseTask := newBaseTask(ctx, querypb.TriggerCondition_GrpcRequest) loadCollectionTask := &loadCollectionTask{ @@ -55,9 +56,10 @@ func genLoadPartitionTask(ctx context.Context, queryCoord *QueryCoord) *loadPart Base: &commonpb.MsgBase{ MsgType: commonpb.MsgType_LoadPartitions, }, - CollectionID: defaultCollectionID, - PartitionIDs: []UniqueID{defaultPartitionID}, - Schema: genDefaultCollectionSchema(false), + CollectionID: defaultCollectionID, + PartitionIDs: []UniqueID{defaultPartitionID}, + Schema: genDefaultCollectionSchema(false), + ReplicaNumber: 1, } baseTask := newBaseTask(ctx, querypb.TriggerCondition_GrpcRequest) loadPartitionTask := &loadPartitionTask{ @@ -162,8 +164,9 @@ func genWatchDmChannelTask(ctx context.Context, queryCoord *QueryCoord, nodeID i Base: &commonpb.MsgBase{ MsgType: commonpb.MsgType_LoadCollection, }, - CollectionID: defaultCollectionID, - Schema: genDefaultCollectionSchema(false), + CollectionID: defaultCollectionID, + Schema: genDefaultCollectionSchema(false), + ReplicaNumber: 1, } baseParentTask := newBaseTask(ctx, querypb.TriggerCondition_GrpcRequest) baseParentTask.taskID = 10 @@ -219,8 +222,9 @@ func genLoadSegmentTask(ctx context.Context, queryCoord *QueryCoord, nodeID int6 Base: &commonpb.MsgBase{ MsgType: commonpb.MsgType_LoadCollection, }, - CollectionID: defaultCollectionID, - Schema: genDefaultCollectionSchema(false), + CollectionID: defaultCollectionID, + Schema: genDefaultCollectionSchema(false), + ReplicaNumber: 1, } baseParentTask := newBaseTask(ctx, querypb.TriggerCondition_GrpcRequest) baseParentTask.taskID = 10 @@ -297,9 +301,15 @@ func TestTriggerTask(t *testing.T) { queryCoord, err := startQueryCoord(ctx) assert.Nil(t, err) - node, err := startQueryNodeServer(ctx) + node1, err := startQueryNodeServer(ctx) assert.Nil(t, err) - waitQueryNodeOnline(queryCoord.cluster, node.queryNodeID) + node2, err := startQueryNodeServer(ctx) + assert.Nil(t, err) + node3, err := startQueryNodeServer(ctx) + assert.Nil(t, err) + waitQueryNodeOnline(queryCoord.cluster, node1.queryNodeID) + waitQueryNodeOnline(queryCoord.cluster, node2.queryNodeID) + waitQueryNodeOnline(queryCoord.cluster, node3.queryNodeID) t.Run("Test LoadCollection", func(t *testing.T) { loadCollectionTask := genLoadCollectionTask(ctx, queryCoord) @@ -328,10 +338,38 @@ func TestTriggerTask(t *testing.T) { assert.Nil(t, err) }) - err = node.stop() - queryCoord.Stop() - err = removeAllSession() - assert.Nil(t, err) + t.Run("Test LoadCollection With Replicas", func(t *testing.T) { + loadCollectionTask := genLoadCollectionTask(ctx, queryCoord) + loadCollectionTask.ReplicaNumber = 3 + err = queryCoord.scheduler.processTask(loadCollectionTask) + assert.Nil(t, err) + }) + + t.Run("Test ReleaseCollection With Replicas", func(t *testing.T) { + releaseCollectionTask := genReleaseCollectionTask(ctx, queryCoord) + err = queryCoord.scheduler.processTask(releaseCollectionTask) + assert.Nil(t, err) + }) + + t.Run("Test LoadPartition With Replicas", func(t *testing.T) { + loadPartitionTask := genLoadPartitionTask(ctx, queryCoord) + loadPartitionTask.ReplicaNumber = 3 + err = queryCoord.scheduler.processTask(loadPartitionTask) + assert.Nil(t, err) + }) + + t.Run("Test ReleasePartition With Replicas", func(t *testing.T) { + releasePartitionTask := genReleaseCollectionTask(ctx, queryCoord) + + err = queryCoord.scheduler.processTask(releasePartitionTask) + assert.Nil(t, err) + }) + + assert.NoError(t, node1.stop()) + assert.NoError(t, node2.stop()) + assert.NoError(t, node3.stop()) + assert.NoError(t, queryCoord.Stop()) + assert.NoError(t, removeAllSession()) } func Test_LoadCollectionAfterLoadPartition(t *testing.T) { @@ -439,6 +477,30 @@ func Test_LoadCollectionExecuteFail(t *testing.T) { assert.Nil(t, err) } +func TestLoadCollectionNoEnoughNodeFail(t *testing.T) { + refreshParams() + ctx := context.Background() + queryCoord, err := startQueryCoord(ctx) + assert.Nil(t, err) + + node1, err := startQueryNodeServer(ctx) + assert.Nil(t, err) + node2, err := startQueryNodeServer(ctx) + assert.Nil(t, err) + waitQueryNodeOnline(queryCoord.cluster, node1.queryNodeID) + waitQueryNodeOnline(queryCoord.cluster, node2.queryNodeID) + + loadCollectionTask := genLoadCollectionTask(ctx, queryCoord) + loadCollectionTask.ReplicaNumber = 3 + err = queryCoord.scheduler.processTask(loadCollectionTask) + assert.Error(t, err) + + assert.NoError(t, node1.stop()) + assert.NoError(t, node2.stop()) + assert.NoError(t, queryCoord.Stop()) + assert.NoError(t, removeAllSession()) +} + func Test_LoadPartitionAssignTaskFail(t *testing.T) { refreshParams() ctx := context.Background() @@ -577,10 +639,11 @@ func Test_RescheduleDmChannel(t *testing.T) { loadCollectionTask := watchDmChannelTask.parentTask queryCoord.scheduler.triggerTaskQueue.addTask(loadCollectionTask) - waitTaskFinalState(loadCollectionTask, taskExpired) + waitTaskFinalState(loadCollectionTask, taskFailed) queryCoord.Stop() err = removeAllSession() + assert.Nil(t, err) } @@ -604,7 +667,7 @@ func Test_RescheduleSegment(t *testing.T) { loadCollectionTask := loadSegmentTask.parentTask queryCoord.scheduler.triggerTaskQueue.addTask(loadCollectionTask) - waitTaskFinalState(loadCollectionTask, taskExpired) + waitTaskFinalState(loadCollectionTask, taskFailed) queryCoord.Stop() err = removeAllSession() @@ -703,7 +766,7 @@ func Test_AssignInternalTask(t *testing.T) { loadSegmentRequests = append(loadSegmentRequests, req) } - internalTasks, err := assignInternalTask(queryCoord.loopCtx, loadCollectionTask, queryCoord.meta, queryCoord.cluster, loadSegmentRequests, nil, false, nil, nil) + internalTasks, err := assignInternalTask(queryCoord.loopCtx, loadCollectionTask, queryCoord.meta, queryCoord.cluster, loadSegmentRequests, nil, false, nil, nil, -1) assert.Nil(t, err) assert.NotEqual(t, 1, len(internalTasks)) @@ -870,6 +933,7 @@ func TestLoadBalanceSegmentsTask(t *testing.T) { MsgType: commonpb.MsgType_LoadBalanceSegments, }, SourceNodeIDs: []int64{node1.queryNodeID}, + CollectionID: defaultCollectionID, }, broker: queryCoord.broker, cluster: queryCoord.cluster, diff --git a/internal/querynode/historical.go b/internal/querynode/historical.go index e6f950b841..bfb3d902e3 100644 --- a/internal/querynode/historical.go +++ b/internal/querynode/historical.go @@ -95,6 +95,29 @@ func (h *historical) retrieve(collID UniqueID, partIDs []UniqueID, vcm storage.C return retrieveResults, retrieveSegmentIDs, retrievePartIDs, nil } +// retrieveBySegmentIDs retrieves records from segments specified by their IDs +func (h *historical) retrieveBySegmentIDs(collID UniqueID, segmentIDs []UniqueID, vcm storage.ChunkManager, plan *RetrievePlan) ( + retrieveResults []*segcorepb.RetrieveResults, err error) { + + for _, segID := range segmentIDs { + seg, err := h.replica.getSegmentByID(segID) + if err != nil { + return nil, err + } + result, err := seg.retrieve(plan) + if err != nil { + return nil, err + } + err = seg.fillIndexedFieldsData(collID, vcm, result) + if err != nil { + return nil, err + } + retrieveResults = append(retrieveResults, result) + } + + return retrieveResults, nil +} + // search will search all the target segments in historical func (h *historical) search(searchReqs []*searchRequest, collID UniqueID, partIDs []UniqueID, plan *SearchPlan, searchTs Timestamp) (searchResults []*SearchResult, searchSegmentIDs []UniqueID, searchPartIDs []UniqueID, err error) { diff --git a/internal/querynode/impl.go b/internal/querynode/impl.go index 3a3e9c043a..28fae9ebbc 100644 --- a/internal/querynode/impl.go +++ b/internal/querynode/impl.go @@ -18,7 +18,6 @@ package querynode import ( "context" - "errors" "fmt" "go.uber.org/zap" @@ -227,7 +226,7 @@ func (node *QueryNode) WatchDmChannels(ctx context.Context, in *queryPb.WatchDmC log.Error(err.Error()) return status, nil } - log.Debug("watchDmChannelsTask Enqueue done", zap.Int64("collectionID", in.CollectionID), zap.Int64("nodeID", Params.QueryNodeCfg.QueryNodeID)) + log.Debug("watchDmChannelsTask Enqueue done", zap.Int64("collectionID", in.CollectionID), zap.Int64("nodeID", Params.QueryNodeCfg.QueryNodeID), zap.Int64("replicaID", in.GetReplicaID())) waitFunc := func() (*commonpb.Status, error) { err = dct.WaitToFinish() @@ -557,12 +556,119 @@ func (node *QueryNode) isHealthy() bool { // Search performs replica search tasks. func (node *QueryNode) Search(ctx context.Context, req *queryPb.SearchRequest) (*internalpb.SearchResults, error) { - return nil, errors.New("not implemented") + if !node.isHealthy() { + return &internalpb.SearchResults{ + Status: &commonpb.Status{ + ErrorCode: commonpb.ErrorCode_UnexpectedError, + Reason: msgQueryNodeIsUnhealthy(Params.QueryNodeCfg.QueryNodeID), + }, + }, nil + } + + log.Debug("Received SearchRequest", zap.String("vchannel", req.GetDmlChannel()), zap.Int64s("segmentIDs", req.GetSegmentIDs())) + + if node.queryShardService == nil { + return &internalpb.SearchResults{ + Status: &commonpb.Status{ + ErrorCode: commonpb.ErrorCode_UnexpectedError, + Reason: "queryShardService is nil", + }, + }, nil + } + + if !node.queryShardService.hasQueryShard(req.GetDmlChannel()) { + err := node.queryShardService.addQueryShard(req.Req.CollectionID, req.GetDmlChannel(), 0) // TODO: add replicaID in request or remove it in query shard + if err != nil { + return &internalpb.SearchResults{ + Status: &commonpb.Status{ + ErrorCode: commonpb.ErrorCode_UnexpectedError, + Reason: err.Error(), + }, + }, nil + } + } + + qs, err := node.queryShardService.getQueryShard(req.GetDmlChannel()) + if err != nil { + return &internalpb.SearchResults{ + Status: &commonpb.Status{ + ErrorCode: commonpb.ErrorCode_UnexpectedError, + Reason: err.Error(), + }, + }, nil + } + + results, err := qs.search(ctx, req) + if err != nil { + log.Warn("QueryService failed to search", zap.String("vchannel", req.GetDmlChannel()), zap.Int64s("segmentIDs", req.GetSegmentIDs()), zap.Error(err)) + return &internalpb.SearchResults{ + Status: &commonpb.Status{ + ErrorCode: commonpb.ErrorCode_UnexpectedError, + Reason: err.Error(), + }, + }, nil + } + log.Debug("Search Shard Done", zap.String("vchannel", req.GetDmlChannel()), zap.Int64s("segmentIDs", req.GetSegmentIDs())) + + return results, err } // Query performs replica query tasks. func (node *QueryNode) Query(ctx context.Context, req *queryPb.QueryRequest) (*internalpb.RetrieveResults, error) { - return nil, errors.New("not implemented") + if !node.isHealthy() { + return &internalpb.RetrieveResults{ + Status: &commonpb.Status{ + ErrorCode: commonpb.ErrorCode_UnexpectedError, + Reason: msgQueryNodeIsUnhealthy(Params.QueryNodeCfg.QueryNodeID), + }, + }, nil + } + log.Debug("Received QueryRequest", zap.String("vchannel", req.GetDmlChannel()), zap.Int64s("segmentIDs", req.GetSegmentIDs())) + + if node.queryShardService == nil { + return &internalpb.RetrieveResults{ + Status: &commonpb.Status{ + ErrorCode: commonpb.ErrorCode_UnexpectedError, + Reason: "queryShardService is nil", + }, + }, nil + } + + if !node.queryShardService.hasQueryShard(req.GetDmlChannel()) { + err := node.queryShardService.addQueryShard(req.Req.CollectionID, req.GetDmlChannel(), 0) // TODO: add replicaID in request or remove it in query shard + if err != nil { + return &internalpb.RetrieveResults{ + Status: &commonpb.Status{ + ErrorCode: commonpb.ErrorCode_UnexpectedError, + Reason: err.Error(), + }, + }, nil + } + } + + qs, err := node.queryShardService.getQueryShard(req.GetDmlChannel()) + if err != nil { + return &internalpb.RetrieveResults{ + Status: &commonpb.Status{ + ErrorCode: commonpb.ErrorCode_UnexpectedError, + Reason: err.Error(), + }, + }, nil + } + + results, err := qs.query(ctx, req) + if err != nil { + log.Warn("QueryService failed to query", zap.String("vchannel", req.GetDmlChannel()), zap.Int64s("segmentIDs", req.GetSegmentIDs()), zap.Error(err)) + return &internalpb.RetrieveResults{ + Status: &commonpb.Status{ + ErrorCode: commonpb.ErrorCode_UnexpectedError, + Reason: err.Error(), + }, + }, nil + } + log.Debug("Query Shard Done", zap.String("vchannel", req.GetDmlChannel()), zap.Int64s("segmentIDs", req.GetSegmentIDs())) + + return results, nil } // GetMetrics return system infos of the query node, such as total memory, memory usage, cpu usage ... diff --git a/internal/querynode/impl_test.go b/internal/querynode/impl_test.go index 23c84dfecb..116f2c63a4 100644 --- a/internal/querynode/impl_test.go +++ b/internal/querynode/impl_test.go @@ -584,8 +584,16 @@ func TestImpl_Search(t *testing.T) { node, err := genSimpleQueryNode(ctx) require.NoError(t, err) - _, err = node.Search(ctx, nil) - assert.Error(t, err) + req, err := genSimpleSearchRequest(IndexFaissIDMap) + require.NoError(t, err) + + node.queryShardService.addQueryShard(defaultCollectionID, defaultDMLChannel, defaultReplicaID) + + _, err = node.Search(ctx, &queryPb.SearchRequest{ + Req: req, + DmlChannel: defaultDMLChannel, + }) + assert.NoError(t, err) } func TestImpl_Query(t *testing.T) { @@ -595,6 +603,14 @@ func TestImpl_Query(t *testing.T) { node, err := genSimpleQueryNode(ctx) require.NoError(t, err) - _, err = node.Query(ctx, nil) - assert.Error(t, err) + req, err := genSimpleRetrieveRequest() + require.NoError(t, err) + + node.queryShardService.addQueryShard(defaultCollectionID, defaultDMLChannel, defaultReplicaID) + + _, err = node.Query(ctx, &queryPb.QueryRequest{ + Req: req, + DmlChannel: defaultDMLChannel, + }) + assert.NoError(t, err) } diff --git a/internal/querynode/mock_test.go b/internal/querynode/mock_test.go index d30eb5f9e0..df11f7f792 100644 --- a/internal/querynode/mock_test.go +++ b/internal/querynode/mock_test.go @@ -79,6 +79,7 @@ const ( defaultCollectionID = UniqueID(0) defaultPartitionID = UniqueID(1) defaultSegmentID = UniqueID(2) + defaultReplicaID = UniqueID(10) defaultCollectionName = "query-node-unittest-default-collection" defaultPartitionName = "query-node-unittest-default-partition" @@ -1748,18 +1749,27 @@ func genSimpleQueryNodeWithMQFactory(ctx context.Context, fac dependency.Factory // start task scheduler go node.scheduler.Start() + /* + vectorStorage, err := node.factory.NewVectorStorageChunkManager(ctx) + if err != nil { + return nil, err + } + cacheStorage, err := node.factory.NewCacheStorageChunkManager(ctx) + if err != nil { + return nil, err + }*/ + /* + qs := newQueryService(ctx, node.historical, node.streaming, vectorStorage, cacheStorage, fac) + defer qs.close() + node.queryService = qs + qs := newQueryService(ctx, node.historical, node.streaming, node.msFactory) + defer qs.close() + node.queryService = qs*/ - vectorStorage, err := node.factory.NewVectorStorageChunkManager(ctx) - if err != nil { - return nil, err - } - cacheStorage, err := node.factory.NewCacheStorageChunkManager(ctx) - if err != nil { - return nil, err - } - qs := newQueryService(ctx, node.historical, node.streaming, vectorStorage, cacheStorage, fac) - defer qs.close() - node.queryService = qs + // init shard cluster service + node.ShardClusterService = newShardClusterService(node.etcdCli, node.session, node) + + node.queryShardService = newQueryShardService(node.queryNodeLoopCtx, node.historical, node.streaming, node.ShardClusterService, node.factory) node.UpdateStateCode(internalpb.StateCode_Healthy) diff --git a/internal/querynode/query_channel.go b/internal/querynode/query_channel.go new file mode 100644 index 0000000000..eeed849bb7 --- /dev/null +++ b/internal/querynode/query_channel.go @@ -0,0 +1,139 @@ +package querynode + +import ( + "fmt" + "sync" + + "github.com/milvus-io/milvus/internal/log" + "github.com/milvus-io/milvus/internal/metrics" + "github.com/milvus-io/milvus/internal/mq/msgstream" + "github.com/milvus-io/milvus/internal/proto/datapb" + "github.com/milvus-io/milvus/internal/proto/internalpb" + "github.com/milvus-io/milvus/internal/util/typeutil" + "go.uber.org/zap" +) + +// queryChannel simple query channel wrapper in query shard service +type queryChannel struct { + closeCh chan struct{} + collectionID int64 + + streaming *streaming + queryMsgStream msgstream.MsgStream + asConsumeOnce sync.Once + closeOnce sync.Once +} + +// AsConsumer do AsConsumer for query msgstream and seek if position is not nil +func (qc *queryChannel) AsConsumer(channelName string, subName string, position *internalpb.MsgPosition) error { + var err error + qc.asConsumeOnce.Do(func() { + qc.queryMsgStream.AsConsumer([]string{channelName}, subName) + metrics.QueryNodeNumConsumers.WithLabelValues(fmt.Sprint(Params.QueryNodeCfg.QueryNodeID)).Inc() + if position == nil || len(position.MsgID) == 0 { + log.Debug("QueryNode AsConsumer", zap.String("channel", channelName), zap.String("sub name", subName)) + } else { + err = qc.queryMsgStream.Seek([]*internalpb.MsgPosition{position}) + if err == nil { + log.Debug("querynode seek query channel: ", zap.Any("consumeChannel", channelName), + zap.String("seek position", string(position.MsgID))) + } + } + }) + + return err +} + +// Start start a goroutine for consume msg +func (qc *queryChannel) Start() { + go qc.queryMsgStream.Start() + + go qc.consumeQuery() +} + +// Stop all workers and msgstream +func (qc *queryChannel) Stop() { + qc.closeOnce.Do(func() { + qc.queryMsgStream.Close() + close(qc.closeCh) + }) +} + +func (qc *queryChannel) consumeQuery() { + for { + select { + case <-qc.closeCh: + log.Info("query channel worker quit", zap.Int64("collection id", qc.collectionID)) + return + case msgPack, ok := <-qc.queryMsgStream.Chan(): + if !ok { + log.Warn("Receive Query Msg from chan failed", zap.Int64("collectionID", qc.collectionID)) + return + } + if !ok || msgPack == nil || len(msgPack.Msgs) == 0 { + continue + } + + for _, msg := range msgPack.Msgs { + switch sm := msg.(type) { + case *msgstream.SealedSegmentsChangeInfoMsg: + qc.adjustByChangeInfo(sm) + default: + log.Warn("ignore msgs other than SegmentChangeInfo", zap.Any("msgType", msg.Type().String())) + } + } + } + } +} + +func (qc *queryChannel) adjustByChangeInfo(msg *msgstream.SealedSegmentsChangeInfoMsg) { + for _, info := range msg.Infos { + // precheck collection id, if not the same collection, skip + for _, segment := range info.OnlineSegments { + if segment.CollectionID != qc.collectionID { + return + } + } + + for _, segment := range info.OfflineSegments { + if segment.CollectionID != qc.collectionID { + return + } + } + + // should handle segment change in shardCluster + // for OnlineSegments: + for _, segment := range info.OnlineSegments { + /* + // 1. update global sealed segments + q.globalSegmentManager.addGlobalSegmentInfo(segment) + // 2. update excluded segment, cluster have been loaded sealed segments, + // so we need to avoid getting growing segment from flow graph.*/ + qc.streaming.replica.addExcludedSegments(segment.CollectionID, []*datapb.SegmentInfo{ + { + ID: segment.SegmentID, + CollectionID: segment.CollectionID, + PartitionID: segment.PartitionID, + InsertChannel: segment.DmChannel, + NumOfRows: segment.NumRows, + // TODO: add status, remove query pb segment status, use common pb segment status? + DmlPosition: &internalpb.MsgPosition{ + // use max timestamp to filter out dm messages + Timestamp: typeutil.MaxTimestamp, + }, + }, + }) + } + /* + // for OfflineSegments: + for _, segment := range info.OfflineSegments { + // 1. update global sealed segments + q.globalSegmentManager.removeGlobalSealedSegmentInfo(segment.SegmentID) + }*/ + + log.Info("Successfully changed global sealed segment info ", + zap.Int64("collection ", qc.collectionID), + zap.Any("online segments ", info.OnlineSegments), + zap.Any("offline segments ", info.OfflineSegments)) + } +} diff --git a/internal/querynode/query_channel_test.go b/internal/querynode/query_channel_test.go new file mode 100644 index 0000000000..51e4ac39b6 --- /dev/null +++ b/internal/querynode/query_channel_test.go @@ -0,0 +1,174 @@ +// Licensed to the LF AI & Data foundation under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package querynode + +import ( + "math/rand" + "testing" + + "github.com/milvus-io/milvus/internal/mq/msgstream" + "github.com/milvus-io/milvus/internal/mq/msgstream/mqwrapper" + "github.com/milvus-io/milvus/internal/proto/internalpb" + "github.com/stretchr/testify/mock" +) + +type mockQueryMsgStream struct { + mock.Mock +} + +func (m *mockQueryMsgStream) Start() { + m.Called() +} + +func (m *mockQueryMsgStream) Close() { + m.Called() +} + +func (m *mockQueryMsgStream) AsProducer(channels []string) { + panic("not implemented") // TODO: Implement +} + +func (m *mockQueryMsgStream) Produce(_ *msgstream.MsgPack) error { + panic("not implemented") // TODO: Implement +} + +func (m *mockQueryMsgStream) SetRepackFunc(repackFunc msgstream.RepackFunc) { + panic("not implemented") // TODO: Implement +} + +func (m *mockQueryMsgStream) ComputeProduceChannelIndexes(tsMsgs []msgstream.TsMsg) [][]int32 { + panic("not implemented") // TODO: Implement +} + +func (m *mockQueryMsgStream) GetProduceChannels() []string { + panic("not implemented") // TODO: Implement +} + +func (m *mockQueryMsgStream) ProduceMark(_ *msgstream.MsgPack) (map[string][]msgstream.MessageID, error) { + panic("not implemented") // TODO: Implement +} + +func (m *mockQueryMsgStream) Broadcast(_ *msgstream.MsgPack) error { + panic("not implemented") // TODO: Implement +} + +func (m *mockQueryMsgStream) BroadcastMark(_ *msgstream.MsgPack) (map[string][]msgstream.MessageID, error) { + panic("not implemented") // TODO: Implement +} + +func (m *mockQueryMsgStream) AsConsumer(channels []string, subName string) { + m.Called(channels, subName) +} + +func (m *mockQueryMsgStream) AsConsumerWithPosition(channels []string, subName string, position mqwrapper.SubscriptionInitialPosition) { + panic("not implemented") // TODO: Implement +} + +func (m *mockQueryMsgStream) Chan() <-chan *msgstream.MsgPack { + args := m.Called() + return args.Get(0).(<-chan *msgstream.MsgPack) +} + +func (m *mockQueryMsgStream) Seek(offset []*msgstream.MsgPosition) error { + args := m.Called(offset) + return args.Error(0) +} + +func (m *mockQueryMsgStream) GetLatestMsgID(channel string) (msgstream.MessageID, error) { + panic("not implemented") // TODO: Implement +} + +func TestQueryChannel_AsConsumer(t *testing.T) { + t.Run("AsConsumer with no seek", func(t *testing.T) { + mqs := &mockQueryMsgStream{} + mqs.On("Close").Return() + + qc := &queryChannel{ + closeCh: make(chan struct{}), + collectionID: defaultCollectionID, + + streaming: nil, + queryMsgStream: mqs, + } + + mqs.On("AsConsumer", []string{defaultDMLChannel}, defaultSubName).Return() + + qc.AsConsumer(defaultDMLChannel, defaultSubName, nil) + qc.Stop() + + mqs.AssertCalled(t, "AsConsumer", []string{defaultDMLChannel}, defaultSubName) + mqs.AssertNotCalled(t, "Seek") + + mqs.AssertExpectations(t) + + qc.Stop() + }) + + t.Run("AsConsumer with bad position", func(t *testing.T) { + mqs := &mockQueryMsgStream{} + mqs.On("Close").Return() + + qc := &queryChannel{ + closeCh: make(chan struct{}), + collectionID: defaultCollectionID, + + streaming: nil, + queryMsgStream: mqs, + } + + mqs.On("AsConsumer", []string{defaultDMLChannel}, defaultSubName).Return() + + qc.AsConsumer(defaultDMLChannel, defaultSubName, &internalpb.MsgPosition{}) + qc.Stop() + + mqs.AssertCalled(t, "AsConsumer", []string{defaultDMLChannel}, defaultSubName) + mqs.AssertNotCalled(t, "Seek") + + mqs.AssertExpectations(t) + + }) + + t.Run("AsConsumer with position", func(t *testing.T) { + mqs := &mockQueryMsgStream{} + mqs.On("Close").Return() + + qc := &queryChannel{ + closeCh: make(chan struct{}), + collectionID: defaultCollectionID, + + streaming: nil, + queryMsgStream: mqs, + } + msgID := make([]byte, 8) + rand.Read(msgID) + pos := &internalpb.MsgPosition{MsgID: msgID} + + mqs.On("AsConsumer", []string{defaultDMLChannel}, defaultSubName).Return() + mqs.On("Seek", []*internalpb.MsgPosition{pos}).Return(nil) + + qc.AsConsumer(defaultDMLChannel, defaultSubName, pos) + + qc.Stop() + + mqs.AssertCalled(t, "AsConsumer", []string{defaultDMLChannel}, defaultSubName) + mqs.AssertCalled(t, "Seek", []*internalpb.MsgPosition{pos}) + + mqs.AssertExpectations(t) + + }) + +} diff --git a/internal/querynode/query_node.go b/internal/querynode/query_node.go index b91c1bbbfa..aae4dc97c5 100644 --- a/internal/querynode/query_node.go +++ b/internal/querynode/query_node.go @@ -100,7 +100,7 @@ type QueryNode struct { dataSyncService *dataSyncService // internal services - queryService *queryService + //queryService *queryService statsService *statsService // segment loader @@ -119,6 +119,11 @@ type QueryNode struct { vectorStorage storage.ChunkManager cacheStorage storage.ChunkManager etcdKV *etcdkv.EtcdKV + + // shard cluster service, handle shard leader functions + ShardClusterService *ShardClusterService + //shard query service, handles shard-level query & search + queryShardService *queryShardService } // NewQueryNode will return a QueryNode with abnormal state. @@ -127,7 +132,6 @@ func NewQueryNode(ctx context.Context, factory dependency.Factory) *QueryNode { node := &QueryNode{ queryNodeLoopCtx: ctx1, queryNodeLoopCancel: cancel, - queryService: nil, factory: factory, } @@ -319,7 +323,7 @@ func (node *QueryNode) Init() error { node.vectorStorage, node.factory) - //node.statsService = newStatsService(node.queryNodeLoopCtx, node.historical.replica, node.factory) + // node.statsService = newStatsService(node.queryNodeLoopCtx, node.historical.replica, node.factory) node.dataSyncService = newDataSyncService(node.queryNodeLoopCtx, streamingReplica, historicalReplica, node.tSafeReplica, node.factory) node.InitSegcore() @@ -329,13 +333,13 @@ func (node *QueryNode) Init() error { // init services and manager // TODO: pass node.streaming.replica to search service - node.queryService = newQueryService(node.queryNodeLoopCtx, - node.historical, - node.streaming, - node.vectorStorage, - node.cacheStorage, - node.factory, - qsOptWithSessionManager(node.sessionManager)) + // node.queryService = newQueryService(node.queryNodeLoopCtx, + // node.historical, + // node.streaming, + // node.vectorStorage, + // node.cacheStorage, + // node.factory, + // qsOptWithSessionManager(node.sessionManager)) log.Debug("query node init successfully", zap.Any("queryNodeID", Params.QueryNodeCfg.QueryNodeID), @@ -364,6 +368,11 @@ func (node *QueryNode) Start() error { node.wg.Add(1) go node.watchService(node.queryNodeLoopCtx) + // create shardClusterService for shardLeader functions. + node.ShardClusterService = newShardClusterService(node.etcdCli, node.session, node) + // create shard-level query service + node.queryShardService = newQueryShardService(node.queryNodeLoopCtx, node.historical, node.streaming, node.ShardClusterService, node.factory) + Params.QueryNodeCfg.CreatedTime = time.Now() Params.QueryNodeCfg.UpdatedTime = time.Now() @@ -391,8 +400,13 @@ func (node *QueryNode) Stop() error { if node.streaming != nil { node.streaming.close() } - if node.queryService != nil { - node.queryService.close() + /* + if node.queryService != nil { + node.queryService.close() + }*/ + + if node.queryShardService != nil { + node.queryShardService.close() } //if node.statsService != nil { // node.statsService.close() @@ -454,43 +468,44 @@ func (node *QueryNode) watchChangeInfo() { func (node *QueryNode) waitChangeInfo(segmentChangeInfos *querypb.SealedSegmentsChangeInfo) error { fn := func() error { - for _, info := range segmentChangeInfos.Infos { - canDoLoadBalance := true - // make sure all query channel already received segment location changes - // Check online segments: - for _, segmentInfo := range info.OnlineSegments { - if node.queryService.hasQueryCollection(segmentInfo.CollectionID) { - qc, err := node.queryService.getQueryCollection(segmentInfo.CollectionID) - if err != nil { - canDoLoadBalance = false - break - } - if info.OnlineNodeID == Params.QueryNodeCfg.QueryNodeID && !qc.globalSegmentManager.hasGlobalSealedSegment(segmentInfo.SegmentID) { - canDoLoadBalance = false - break + /* + for _, info := range segmentChangeInfos.Infos { + canDoLoadBalance := true + // make sure all query channel already received segment location changes + // Check online segments: + for _, segmentInfo := range info.OnlineSegments { + if node.queryService.hasQueryCollection(segmentInfo.CollectionID) { + qc, err := node.queryService.getQueryCollection(segmentInfo.CollectionID) + if err != nil { + canDoLoadBalance = false + break + } + if info.OnlineNodeID == Params.QueryNodeCfg.QueryNodeID && !qc.globalSegmentManager.hasGlobalSealedSegment(segmentInfo.SegmentID) { + canDoLoadBalance = false + break + } } } - } - // Check offline segments: - for _, segmentInfo := range info.OfflineSegments { - if node.queryService.hasQueryCollection(segmentInfo.CollectionID) { - qc, err := node.queryService.getQueryCollection(segmentInfo.CollectionID) - if err != nil { - canDoLoadBalance = false - break - } - if info.OfflineNodeID == Params.QueryNodeCfg.QueryNodeID && qc.globalSegmentManager.hasGlobalSealedSegment(segmentInfo.SegmentID) { - canDoLoadBalance = false - break + // Check offline segments: + for _, segmentInfo := range info.OfflineSegments { + if node.queryService.hasQueryCollection(segmentInfo.CollectionID) { + qc, err := node.queryService.getQueryCollection(segmentInfo.CollectionID) + if err != nil { + canDoLoadBalance = false + break + } + if info.OfflineNodeID == Params.QueryNodeCfg.QueryNodeID && qc.globalSegmentManager.hasGlobalSealedSegment(segmentInfo.SegmentID) { + canDoLoadBalance = false + break + } } } + if canDoLoadBalance { + return nil + } + return errors.New(fmt.Sprintln("waitChangeInfo failed, infoID = ", segmentChangeInfos.Base.GetMsgID())) } - if canDoLoadBalance { - return nil - } - return errors.New(fmt.Sprintln("waitChangeInfo failed, infoID = ", segmentChangeInfos.Base.GetMsgID())) - } - + */ return nil } diff --git a/internal/querynode/query_node_test.go b/internal/querynode/query_node_test.go index 1b9c89a9c8..910ebe4dac 100644 --- a/internal/querynode/query_node_test.go +++ b/internal/querynode/query_node_test.go @@ -18,7 +18,9 @@ package querynode import ( "context" + "io/ioutil" "math/rand" + "net/url" "os" "os/signal" "strconv" @@ -29,6 +31,7 @@ import ( "github.com/golang/protobuf/proto" "github.com/stretchr/testify/assert" + "go.etcd.io/etcd/server/v3/embed" "github.com/milvus-io/milvus/internal/util/dependency" @@ -42,6 +45,8 @@ import ( "github.com/milvus-io/milvus/internal/util/sessionutil" ) +var embedetcdServer *embed.Etcd + // mock of query coordinator client type queryCoordMock struct { types.QueryCoord @@ -224,9 +229,41 @@ func newMessageStreamFactory() dependency.Factory { return dependency.NewDefaultFactory(true) } +func startEmbedEtcdServer() (*embed.Etcd, error) { + dir, err := ioutil.TempDir(os.TempDir(), "milvus_ut") + if err != nil { + return nil, err + } + defer os.RemoveAll(dir) + config := embed.NewConfig() + + config.Dir = os.TempDir() + config.LogLevel = "warn" + config.LogOutputs = []string{"default"} + u, err := url.Parse("http://localhost:2389") + if err != nil { + return nil, err + } + config.LCUrls = []url.URL{*u} + u, err = url.Parse("http://localhost:2390") + if err != nil { + return nil, err + } + config.LPUrls = []url.URL{*u} + + return embed.StartEtcd(config) +} + func TestMain(m *testing.M) { setup() Params.CommonCfg.QueryNodeStats = Params.CommonCfg.QueryNodeStats + strconv.Itoa(rand.Int()) + // init embed etcd + var err error + embedetcdServer, err = startEmbedEtcdServer() + if err != nil { + os.Exit(1) + } + defer embedetcdServer.Close() exitCode := m.Run() os.Exit(exitCode) } @@ -278,16 +315,17 @@ func genSimpleQueryNodeToTestWatchChangeInfo(ctx context.Context) (*QueryNode, e return nil, err } - err = node.queryService.addQueryCollection(defaultCollectionID) - if err != nil { - return nil, err - } + /* + err = node.queryService.addQueryCollection(defaultCollectionID) + if err != nil { + return nil, err + } - qc, err := node.queryService.getQueryCollection(defaultCollectionID) - if err != nil { - return nil, err - } - qc.globalSegmentManager.addGlobalSegmentInfo(genSimpleSegmentInfo()) + qc, err := node.queryService.getQueryCollection(defaultCollectionID) + if err != nil { + return nil, err + }*/ + //qc.globalSegmentManager.addGlobalSegmentInfo(genSimpleSegmentInfo()) return node, nil } @@ -330,10 +368,11 @@ func TestQueryNode_adjustByChangeInfo(t *testing.T) { segmentChangeInfos.Infos[0].OnlineSegments = nil segmentChangeInfos.Infos[0].OfflineNodeID = Params.QueryNodeCfg.QueryNodeID - qc, err := node.queryService.getQueryCollection(defaultCollectionID) - assert.NoError(t, err) - qc.globalSegmentManager.removeGlobalSealedSegmentInfo(defaultSegmentID) - + /* + qc, err := node.queryService.getQueryCollection(defaultCollectionID) + assert.NoError(t, err) + qc.globalSegmentManager.removeGlobalSealedSegmentInfo(defaultSegmentID) + */ err = node.removeSegments(segmentChangeInfos) assert.Error(t, err) }) @@ -402,9 +441,10 @@ func TestQueryNode_watchChangeInfo(t *testing.T) { segmentChangeInfos.Infos[0].OnlineSegments = nil segmentChangeInfos.Infos[0].OfflineNodeID = Params.QueryNodeCfg.QueryNodeID - qc, err := node.queryService.getQueryCollection(defaultCollectionID) - assert.NoError(t, err) - qc.globalSegmentManager.removeGlobalSealedSegmentInfo(defaultSegmentID) + /* + qc, err := node.queryService.getQueryCollection(defaultCollectionID) + assert.NoError(t, err) + qc.globalSegmentManager.removeGlobalSealedSegmentInfo(defaultSegmentID)*/ go node.watchChangeInfo() diff --git a/internal/querynode/query_service_test.go b/internal/querynode/query_service_test.go index 63a77fc2cf..e7a6565230 100644 --- a/internal/querynode/query_service_test.go +++ b/internal/querynode/query_service_test.go @@ -20,11 +20,8 @@ import ( "context" "math" "math/rand" - "testing" - "time" "github.com/golang/protobuf/proto" - "github.com/stretchr/testify/assert" "github.com/milvus-io/milvus/internal/common" "github.com/milvus-io/milvus/internal/mq/msgstream" @@ -129,6 +126,7 @@ func sendSearchRequest(ctx context.Context, DIM int) error { return err } +/* func TestSearch_Search(t *testing.T) { const N = 10000 const DIM = 16 @@ -258,3 +256,4 @@ func TestQueryService_addQueryCollection(t *testing.T) { qs.close() assert.Len(t, qs.queryCollections, 0) } +*/ diff --git a/internal/querynode/query_shard.go b/internal/querynode/query_shard.go index 9801ffb3dc..41583c7ead 100644 --- a/internal/querynode/query_shard.go +++ b/internal/querynode/query_shard.go @@ -19,9 +19,26 @@ package querynode import ( "context" "errors" + "fmt" + "math" + "sync" + "time" - "github.com/milvus-io/milvus/internal/proto/milvuspb" + "github.com/golang/protobuf/proto" + "go.uber.org/atomic" + "go.uber.org/zap" + + "github.com/milvus-io/milvus/internal/log" + "github.com/milvus-io/milvus/internal/proto/commonpb" + "github.com/milvus-io/milvus/internal/proto/etcdpb" + "github.com/milvus-io/milvus/internal/proto/internalpb" "github.com/milvus-io/milvus/internal/proto/querypb" + "github.com/milvus-io/milvus/internal/proto/schemapb" + "github.com/milvus-io/milvus/internal/proto/segcorepb" + "github.com/milvus-io/milvus/internal/storage" + "github.com/milvus-io/milvus/internal/util/funcutil" + "github.com/milvus-io/milvus/internal/util/tsoutil" + "github.com/milvus-io/milvus/internal/util/typeutil" ) type queryShard struct { @@ -30,27 +47,878 @@ type queryShard struct { collectionID UniqueID channel Channel + deltaChannel Channel + replicaID int64 + + clusterService *ShardClusterService + historical *historical + streaming *streaming + + dmTSafeWatcher *tSafeWatcher + deltaTSafeWatcher *tSafeWatcher + watcherCond *sync.Cond + serviceDmTs atomic.Uint64 + serviceDeltaTs atomic.Uint64 + startTickerOnce sync.Once + ticker *time.Ticker // timed ticker for trigger timeout check + + localChunkManager storage.ChunkManager + remoteChunkManager storage.ChunkManager + vectorChunkManager *storage.VectorChunkManager + localCacheEnabled bool + localCacheSize int64 } func newQueryShard( ctx context.Context, collectionID UniqueID, channel Channel, + replicaID int64, + clusterService *ShardClusterService, + historical *historical, + streaming *streaming, + localChunkManager storage.ChunkManager, + remoteChunkManager storage.ChunkManager, + localCacheEnabled bool, ) *queryShard { ctx, cancel := context.WithCancel(ctx) qs := &queryShard{ - ctx: ctx, - cancel: cancel, - collectionID: collectionID, - channel: channel, + ctx: ctx, + cancel: cancel, + collectionID: collectionID, + channel: channel, + replicaID: replicaID, + clusterService: clusterService, + historical: historical, + streaming: streaming, + localChunkManager: localChunkManager, + remoteChunkManager: remoteChunkManager, + localCacheEnabled: localCacheEnabled, + localCacheSize: Params.QueryNodeCfg.CacheMemoryLimit, + + watcherCond: sync.NewCond(&sync.Mutex{}), } + deltaChannel, err := funcutil.ConvertChannelName(channel, Params.CommonCfg.RootCoordDml, Params.CommonCfg.RootCoordDelta) + if err != nil { + log.Warn("failed to convert dm channel to delta", zap.String("channel", channel), zap.Error(err)) + } + qs.deltaChannel = deltaChannel + return qs } -func (q *queryShard) search(ctx context.Context, req *querypb.SearchRequest) (*milvuspb.SearchResults, error) { - return nil, errors.New("not implemented") +// Close cleans query shard +func (q *queryShard) Close() { + q.cancel() } -func (q *queryShard) query(ctx context.Context, req *querypb.QueryRequest) (*milvuspb.QueryResults, error) { - return nil, errors.New("not implemented") +func (q *queryShard) watchDMLTSafe() error { + q.dmTSafeWatcher = newTSafeWatcher() + err := q.streaming.tSafeReplica.registerTSafeWatcher(q.channel, q.dmTSafeWatcher) + if err != nil { + log.Warn("failed to register dml tsafe watcher", zap.String("channel", q.channel), zap.Error(err)) + return err + } + go q.watchTs(q.dmTSafeWatcher.watcherChan(), q.dmTSafeWatcher.closeCh, tsTypeDML) + + q.startTsTicker() + return nil } + +func (q *queryShard) watchDeltaTSafe() error { + q.deltaTSafeWatcher = newTSafeWatcher() + err := q.streaming.tSafeReplica.registerTSafeWatcher(q.deltaChannel, q.deltaTSafeWatcher) + if err != nil { + log.Warn("failed to register delta tsafe watcher", zap.String("channel", q.deltaChannel), zap.Error(err)) + return err + } + + go q.watchTs(q.deltaTSafeWatcher.watcherChan(), q.deltaTSafeWatcher.closeCh, tsTypeDelta) + q.startTsTicker() + + return nil +} + +func (q *queryShard) startTsTicker() { + q.startTickerOnce.Do(func() { + go func() { + q.ticker = time.NewTicker(time.Millisecond * 50) // check timeout every 50 milliseconds, need not to be to frequent + defer q.ticker.Stop() + for { + select { + case <-q.ticker.C: + q.watcherCond.L.Lock() + q.watcherCond.Broadcast() + q.watcherCond.L.Unlock() + case <-q.ctx.Done(): + return + } + } + }() + }) +} + +type tsType int32 + +const ( + tsTypeDML tsType = 1 + tsTypeDelta tsType = 2 +) + +func (tp tsType) String() string { + switch tp { + case tsTypeDML: + return "DML tSafe" + case tsTypeDelta: + return "Delta tSafe" + } + return "" +} + +func (q *queryShard) watchTs(channel <-chan bool, closeCh <-chan struct{}, tp tsType) { + for { + select { + case <-q.ctx.Done(): + log.Debug("stop queryShard watcher due to ctx done", zap.Int64("collectionID", q.collectionID), zap.String("vChannel", q.channel)) + return + case <-closeCh: + log.Debug("stop queryShard watcher due to watcher closed", zap.Int64("collectionID", q.collectionID), zap.String("vChannel", q.channel)) + return + case _, ok := <-channel: + if !ok { + log.Warn("tsafe watcher channel closed", zap.Int64("collectionID", q.collectionID), zap.String("vChannel", q.channel)) + return + } + + ts, err := q.getNewTSafe(tp) + if err == nil { + q.watcherCond.L.Lock() + q.setServiceableTime(ts, tp) + q.watcherCond.Broadcast() + q.watcherCond.L.Unlock() + } + } + } +} + +func (q *queryShard) getNewTSafe(tp tsType) (Timestamp, error) { + var channel string + switch tp { + case tsTypeDML: + channel = q.channel + case tsTypeDelta: + channel = q.deltaChannel + default: + return 0, errors.New("invalid ts type") + } + t := Timestamp(math.MaxInt64) + ts, err := q.streaming.tSafeReplica.getTSafe(channel) + if err != nil { + return 0, err + } + if ts <= t { + t = ts + } + return t, nil +} + +func (q *queryShard) waitUntilServiceable(ctx context.Context, guaranteeTs Timestamp, tp tsType) { + q.watcherCond.L.Lock() + defer q.watcherCond.L.Unlock() + st := q.getServiceableTime(tp) + for guaranteeTs > st { + log.Debug("serviceable ts before guarantee ts", zap.Uint64("serviceable ts", st), zap.Uint64("guarantee ts", guaranteeTs), zap.String("channel", q.channel)) + q.watcherCond.Wait() + if err := ctx.Err(); err != nil { + log.Warn("waitUntialServiceable timeout", zap.Uint64("serviceable ts", st), zap.Uint64("guarantee ts", guaranteeTs), zap.String("channel", q.channel)) + return + } + st = q.getServiceableTime(tp) + } + log.Debug("wait serviceable ts done", zap.String("tsType", tp.String()), zap.Uint64("guarantee ts", guaranteeTs), zap.Uint64("serviceable ts", st), zap.String("channel", q.channel)) +} + +func (q *queryShard) getServiceableTime(tp tsType) Timestamp { + gracefulTimeInMilliSecond := Params.QueryNodeCfg.GracefulTime + gracefulTime := typeutil.ZeroTimestamp + if gracefulTimeInMilliSecond > 0 { + gracefulTime = tsoutil.ComposeTS(gracefulTimeInMilliSecond, 0) + } + var serviceTs Timestamp + switch tp { + case tsTypeDML: // use min value of dml & delta + serviceTs = q.serviceDmTs.Load() + case tsTypeDelta: // check delta ts only + serviceTs = q.serviceDeltaTs.Load() + } + return serviceTs + gracefulTime +} + +func (q *queryShard) setServiceableTime(t Timestamp, tp tsType) { + switch tp { + case tsTypeDML: + ts := q.serviceDmTs.Load() + if t < ts { + return + } + for !q.serviceDmTs.CAS(ts, t) { + ts = q.serviceDmTs.Load() + if t < ts { + return + } + } + case tsTypeDelta: + ts := q.serviceDeltaTs.Load() + if t < ts { + return + } + for !q.serviceDeltaTs.CAS(ts, t) { + ts = q.serviceDeltaTs.Load() + if t < ts { + return + } + } + } +} + +func (q *queryShard) search(ctx context.Context, req *querypb.SearchRequest) (*internalpb.SearchResults, error) { + collectionID := req.Req.CollectionID + segmentIDs := req.SegmentIDs + timestamp := req.Req.TravelTimestamp + + // check ctx timeout + if !funcutil.CheckCtxValid(ctx) { + return nil, errors.New("search context timeout") + } + + // check if collection has been released + collection, err := q.historical.replica.getCollectionByID(collectionID) + if err != nil { + return nil, err + } + if req.GetReq().GetGuaranteeTimestamp() >= collection.getReleaseTime() { + log.Warn("collection release before search", zap.Int64("collectionID", collectionID)) + return nil, fmt.Errorf("retrieve failed, collection has been released, collectionID = %d", collectionID) + } + + // deserialize query plan + + var plan *SearchPlan + if req.Req.GetDslType() == commonpb.DslType_BoolExprV1 { + expr := req.Req.SerializedExprPlan + plan, err = createSearchPlanByExpr(collection, expr) + if err != nil { + return nil, err + } + } else { + dsl := req.Req.Dsl + plan, err = createSearchPlan(collection, dsl) + if err != nil { + return nil, err + } + } + defer plan.delete() + + schemaHelper, err := typeutil.CreateSchemaHelper(collection.schema) + if err != nil { + return nil, err + } + + // validate top-k + topK := plan.getTopK() + if topK <= 0 || topK >= 16385 { + return nil, fmt.Errorf("limit should be in range [1, 16385], but got %d", topK) + } + + // parse plan to search request + searchReq, err := parseSearchRequest(plan, req.Req.PlaceholderGroup) + if err != nil { + return nil, err + } + defer searchReq.delete() + queryNum := searchReq.getNumOfQuery() + searchRequests := []*searchRequest{searchReq} + + if len(segmentIDs) == 0 { + // segmentIDs not specified, searching as shard leader + return q.searchLeader(ctx, req, searchRequests, collectionID, schemaHelper, plan, topK, queryNum, timestamp) + } + + // segmentIDs specified search as shard follower + return q.searchFollower(ctx, req, searchRequests, collectionID, schemaHelper, plan, topK, queryNum, timestamp) +} + +func (q *queryShard) searchLeader(ctx context.Context, req *querypb.SearchRequest, searchRequests []*searchRequest, collectionID UniqueID, + schemaHelper *typeutil.SchemaHelper, plan *SearchPlan, topK int64, queryNum int64, timestamp Timestamp) (*internalpb.SearchResults, error) { + q.streaming.replica.queryRLock() + defer q.streaming.replica.queryRUnlock() + cluster, ok := q.clusterService.getShardCluster(req.GetDmlChannel()) + if !ok { + return nil, fmt.Errorf("channel %s leader is not here", req.GetDmlChannel()) + } + + searchCtx, cancel := context.WithCancel(ctx) + defer cancel() + + var results []*internalpb.SearchResults + var streamingResults []*SearchResult + var err error + var mut sync.Mutex + var wg sync.WaitGroup + + wg.Add(2) // search cluster and search streaming + + go func() { + defer wg.Done() + // shard leader dispatches request to its shard cluster + cResults, cErr := cluster.Search(searchCtx, req) + mut.Lock() + defer mut.Unlock() + if cErr != nil { + log.Warn("search cluster failed", zap.Int64("collectionID", q.collectionID), zap.Error(cErr)) + err = cErr + cancel() + return + } + + results = cResults + }() + + go func() { + defer wg.Done() + // hold request until guarantee timestamp >= service timestamp + guaranteeTs := req.GetReq().GetGuaranteeTimestamp() + q.waitUntilServiceable(ctx, guaranteeTs, tsTypeDML) + // shard leader queries its own streaming data + // TODO add context + sResults, _, _, sErr := q.streaming.search(searchRequests, collectionID, req.Req.PartitionIDs, req.DmlChannel, plan, timestamp) + mut.Lock() + defer mut.Unlock() + if sErr != nil { + log.Warn("failed to search streaming data", zap.Int64("collectionID", q.collectionID), zap.Error(sErr)) + err = sErr + cancel() + return + } + streamingResults = sResults + }() + + wg.Wait() + if err != nil { + return nil, err + } + + defer deleteSearchResults(streamingResults) + + results = append(results, &internalpb.SearchResults{ + Status: &commonpb.Status{ErrorCode: commonpb.ErrorCode_Success}, + MetricType: plan.getMetricType(), + NumQueries: queryNum, + TopK: topK, + SlicedBlob: nil, + SlicedOffset: 1, + SlicedNumCount: 1, + }) + + if len(streamingResults) > 0 { + // reduce search results + numSegment := int64(len(streamingResults)) + err = reduceSearchResultsAndFillData(plan, streamingResults, numSegment) + if err != nil { + return nil, err + } + + nq := searchRequests[0].getNumOfQuery() + nqOfReqs := []int64{nq} + nqPerSlice := nq + reqSlices, err := getReqSlices(nqOfReqs, nqPerSlice) + if err != nil { + log.Warn("getReqSlices for streaming results error", zap.Error(err)) + return nil, err + } + + blobs, err := marshal(collectionID, 0, streamingResults, int(numSegment), reqSlices) + defer deleteSearchResultDataBlobs(blobs) + if err != nil { + log.Warn("marshal for streaming results error", zap.Error(err)) + return nil, err + } + + // assume only one blob will be sent back + blob, err := getSearchResultDataBlob(blobs, 0) + if err != nil { + log.Warn("getSearchResultDataBlob for streaming results error", zap.Error(err)) + } + + results[len(results)-1].SlicedBlob = blob + } + + // reduce shard search results: unmarshal -> reduce -> marshal + log.Debug("shard leader get search results", zap.Int("numbers", len(results))) + searchResultData, err := decodeSearchResults(results) + if err != nil { + log.Warn("shard leader decode search results errors", zap.Error(err)) + return nil, err + } + log.Debug("shard leader get valid search results", zap.Int("numbers", len(searchResultData))) + + for i, sData := range searchResultData { + log.Debug("reduceSearchResultData", + zap.Int("result No.", i), + zap.Int64("nq", sData.NumQueries), + zap.Int64("topk", sData.TopK), + zap.String("ids", sData.Ids.String()), + zap.Any("len(FieldsData)", len(sData.FieldsData))) + } + + reducedResultData, err := reduceSearchResultData(searchResultData, queryNum, plan.getTopK(), plan.getMetricType()) + if err != nil { + log.Warn("shard leader reduce errors", zap.Error(err)) + return nil, err + } + searchResults, err := encodeSearchResultData(reducedResultData, queryNum, plan.getTopK(), plan.getMetricType()) + if err != nil { + log.Warn("shard leader encode search result errors", zap.Error(err)) + return nil, err + } + if searchResults.SlicedBlob == nil { + log.Debug("shard leader send nil results to proxy", + zap.String("shard", q.channel)) + } else { + log.Debug("shard leader send non-nil results to proxy", + zap.String("shard", q.channel), + zap.String("ids", reducedResultData.Ids.String())) + // printSearchResultData(reducedResultData, q.channel) + } + return searchResults, nil +} + +func (q *queryShard) searchFollower(ctx context.Context, req *querypb.SearchRequest, searchRequests []*searchRequest, collectionID UniqueID, + schemaHelper *typeutil.SchemaHelper, plan *SearchPlan, topK int64, queryNum int64, timestamp Timestamp) (*internalpb.SearchResults, error) { + q.historical.replica.queryRLock() + defer q.historical.replica.queryRUnlock() + segmentIDs := req.GetSegmentIDs() + // hold request until guarantee timestamp >= service timestamp + guaranteeTs := req.GetReq().GetGuaranteeTimestamp() + q.waitUntilServiceable(ctx, guaranteeTs, tsTypeDelta) + // search each segments by segment IDs in request + historicalResults, _, err := q.historical.searchSegments(segmentIDs, searchRequests, plan, timestamp) + if err != nil { + return nil, err + } + defer deleteSearchResults(historicalResults) + + // reduce search results + numSegment := int64(len(historicalResults)) + err = reduceSearchResultsAndFillData(plan, historicalResults, numSegment) + if err != nil { + return nil, err + } + + nq := searchRequests[0].getNumOfQuery() + nqOfReqs := []int64{nq} + nqPerSlice := nq + reqSlices, err := getReqSlices(nqOfReqs, nqPerSlice) + if err != nil { + log.Warn("getReqSlices for historical results error", zap.Error(err)) + return nil, err + } + + blobs, err := marshal(collectionID, 0, historicalResults, int(numSegment), reqSlices) + defer deleteSearchResultDataBlobs(blobs) + if err != nil { + log.Warn("marshal for historical results error", zap.Error(err)) + return nil, err + } + + // assume only one blob will be sent back + blob, err := getSearchResultDataBlob(blobs, 0) + if err != nil { + log.Warn("getSearchResultDataBlob for historical results error", zap.Error(err)) + } + bs := make([]byte, len(blob)) + copy(bs, blob) + + resp := &internalpb.SearchResults{ + Status: &commonpb.Status{ErrorCode: commonpb.ErrorCode_Success}, + MetricType: plan.getMetricType(), + NumQueries: queryNum, + TopK: topK, + SlicedBlob: bs, + SlicedOffset: 1, + SlicedNumCount: 1, + } + log.Debug("shard follower send search result to leader") + return resp, nil +} + +func reduceSearchResultData(searchResultData []*schemapb.SearchResultData, nq int64, topk int64, metricType string) (*schemapb.SearchResultData, error) { + if len(searchResultData) == 0 { + return &schemapb.SearchResultData{ + NumQueries: nq, + TopK: topk, + FieldsData: make([]*schemapb.FieldData, 0), + Scores: make([]float32, 0), + Ids: &schemapb.IDs{ + IdField: &schemapb.IDs_IntId{ + IntId: &schemapb.LongArray{ + Data: make([]int64, 0), + }, + }, + }, + Topks: make([]int64, 0), + }, nil + } + ret := &schemapb.SearchResultData{ + NumQueries: nq, + TopK: topk, + FieldsData: make([]*schemapb.FieldData, len(searchResultData[0].FieldsData)), + Scores: make([]float32, 0), + Ids: &schemapb.IDs{ + IdField: &schemapb.IDs_IntId{ + IntId: &schemapb.LongArray{ + Data: make([]int64, 0), + }, + }, + }, + Topks: make([]int64, 0), + } + + var skipDupCnt int64 + var dummyCnt int64 + // var realTopK int64 = -1 + for i := int64(0); i < nq; i++ { + offsets := make([]int64, len(searchResultData)) + + var idSet = make(map[int64]struct{}) + var j int64 + for j = 0; j < topk; { + sel := selectSearchResultData(searchResultData, offsets, topk, i) + if sel == -1 { + break + } + idx := i*topk + offsets[sel] + + id := searchResultData[sel].Ids.GetIntId().Data[idx] + score := searchResultData[sel].Scores[idx] + // ignore invalid search result + if id == -1 { + continue + } + + // remove duplicates + if _, ok := idSet[id]; !ok { + typeutil.AppendFieldData(ret.FieldsData, searchResultData[sel].FieldsData, idx) + ret.Ids.GetIntId().Data = append(ret.Ids.GetIntId().Data, id) + ret.Scores = append(ret.Scores, score) + idSet[id] = struct{}{} + j++ + } else { + // skip entity with same id + skipDupCnt++ + } + offsets[sel]++ + } + // add empty data + for j < topk { + typeutil.AppendFieldData(ret.FieldsData, searchResultData[0].FieldsData, 0) + ret.Ids.GetIntId().Data = append(ret.Ids.GetIntId().Data, -1) + ret.Scores = append(ret.Scores, -1*float32(math.MaxFloat32)) + j++ + dummyCnt++ + } + + // if realTopK != -1 && realTopK != j { + // log.Warn("Proxy Reduce Search Result", zap.Error(errors.New("the length (topk) between all result of query is different"))) + // // return nil, errors.New("the length (topk) between all result of query is different") + // } + // realTopK = j + // ret.Topks = append(ret.Topks, realTopK) + } + log.Debug("skip duplicated search result", zap.Int64("count", skipDupCnt)) + log.Debug("add dummy data in search result", zap.Int64("count", dummyCnt)) + // ret.TopK = realTopK + + // if !distance.PositivelyRelated(metricType) { + // for k := range ret.Scores { + // ret.Scores[k] *= -1 + // } + // } + + return ret, nil +} + +func selectSearchResultData(dataArray []*schemapb.SearchResultData, offsets []int64, topk int64, qi int64) int { + sel := -1 + maxDistance := -1 * float32(math.MaxFloat32) + for i, offset := range offsets { // query num, the number of ways to merge + if offset >= topk { + continue + } + idx := qi*topk + offset + id := dataArray[i].Ids.GetIntId().Data[idx] + if id != -1 { + distance := dataArray[i].Scores[idx] + if distance > maxDistance { + sel = i + maxDistance = distance + } + } + } + return sel +} + +func decodeSearchResults(searchResults []*internalpb.SearchResults) ([]*schemapb.SearchResultData, error) { + results := make([]*schemapb.SearchResultData, 0) + for _, partialSearchResult := range searchResults { + if partialSearchResult.SlicedBlob == nil { + continue + } + + var partialResultData schemapb.SearchResultData + err := proto.Unmarshal(partialSearchResult.SlicedBlob, &partialResultData) + if err != nil { + return nil, err + } + + results = append(results, &partialResultData) + } + return results, nil +} + +func encodeSearchResultData(searchResultData *schemapb.SearchResultData, nq int64, topk int64, metricType string) (searchResults *internalpb.SearchResults, err error) { + searchResults = &internalpb.SearchResults{ + Status: &commonpb.Status{ + ErrorCode: commonpb.ErrorCode_Success, + }, + NumQueries: nq, + TopK: topk, + MetricType: metricType, + SlicedBlob: nil, + } + slicedBlob, err := proto.Marshal(searchResultData) + if err != nil { + return nil, err + } + if searchResultData != nil && searchResultData.Ids != nil && len(searchResultData.Ids.GetIntId().Data) != 0 { + searchResults.SlicedBlob = slicedBlob + } + return +} + +func (q *queryShard) query(ctx context.Context, req *querypb.QueryRequest) (*internalpb.RetrieveResults, error) { + collectionID := req.Req.CollectionID + segmentIDs := req.SegmentIDs + partitionIDs := req.Req.PartitionIDs + expr := req.Req.SerializedExprPlan + timestamp := req.Req.TravelTimestamp + + // check ctx timeout + if !funcutil.CheckCtxValid(ctx) { + return nil, errors.New("search context timeout") + } + + // check if collection has been released + collection, err := q.streaming.replica.getCollectionByID(collectionID) + if err != nil { + return nil, err + } + + if req.GetReq().GetGuaranteeTimestamp() >= collection.getReleaseTime() { + log.Warn("collection release before query", zap.Int64("collectionID", collectionID)) + return nil, fmt.Errorf("retrieve failed, collection has been released, collectionID = %d", collectionID) + } + // deserialize query plan + plan, err := createRetrievePlanByExpr(collection, expr, timestamp) + if err != nil { + return nil, err + } + defer plan.delete() + + // TODO: init vector chunk manager at most once + if q.vectorChunkManager == nil { + if q.localChunkManager == nil { + return nil, fmt.Errorf("can not create vector chunk manager for local chunk manager is nil") + } + if q.remoteChunkManager == nil { + return nil, fmt.Errorf("can not create vector chunk manager for remote chunk manager is nil") + } + q.vectorChunkManager, err = storage.NewVectorChunkManager(q.localChunkManager, q.remoteChunkManager, + &etcdpb.CollectionMeta{ + ID: collection.id, + Schema: collection.schema, + }, q.localCacheSize, q.localCacheEnabled) + if err != nil { + return nil, err + } + } + + // check if shard leader b.c only leader receives request with no segment specified + if len(req.GetSegmentIDs()) == 0 { + q.streaming.replica.queryRLock() + defer q.streaming.replica.queryRUnlock() + cluster, ok := q.clusterService.getShardCluster(req.GetDmlChannel()) + if !ok { + return nil, fmt.Errorf("channel %s leader is not here", req.GetDmlChannel()) + } + + // add cancel when error occurs + queryCtx, cancel := context.WithCancel(ctx) + defer cancel() + + var results []*internalpb.RetrieveResults + var streamingResults []*segcorepb.RetrieveResults + var err error + var mut sync.Mutex + var wg sync.WaitGroup + + wg.Add(2) + + go func() { + defer wg.Done() + // shard leader dispatches request to its shard cluster + cResults, cErr := cluster.Query(queryCtx, req) + mut.Lock() + defer mut.Unlock() + if cErr != nil { + err = cErr + log.Warn("failed to query cluster", zap.Int64("collectionID", q.collectionID), zap.Error(cErr)) + cancel() + return + } + results = cResults + }() + + go func() { + defer wg.Done() + // hold request until guarantee timestamp >= service timestamp + guaranteeTs := req.GetReq().GetGuaranteeTimestamp() + q.waitUntilServiceable(ctx, guaranteeTs, tsTypeDML) + // shard leader queries its own streaming data + // TODO add context + sResults, _, _, sErr := q.streaming.retrieve(collectionID, partitionIDs, plan, func(segment *Segment) bool { return segment.vChannelID == q.channel }) + mut.Lock() + defer mut.Unlock() + if sErr != nil { + err = sErr + log.Warn("failed to query streaming", zap.Int64("collectionID", q.collectionID), zap.Error(err)) + cancel() + return + } + streamingResults = sResults + }() + + wg.Wait() + if err != nil { + return nil, err + } + + streamingResult, err := mergeRetrieveResults(streamingResults) + if err != nil { + return nil, err + } + + // complete results with merged streaming result + results = append(results, &internalpb.RetrieveResults{ + Status: &commonpb.Status{ErrorCode: commonpb.ErrorCode_Success}, + Ids: streamingResult.Ids, + FieldsData: streamingResult.FieldsData, + }) + // merge shard query results + mergedResults, err := mergeInternalRetrieveResults(results) + if err != nil { + return nil, err + } + log.Debug("leader retrieve result", zap.String("channel", req.DmlChannel), zap.String("ids", mergedResults.Ids.String())) + return mergedResults, nil + } + q.historical.replica.queryRLock() + defer q.historical.replica.queryRUnlock() + // hold request until guarantee timestamp >= service timestamp + guaranteeTs := req.GetReq().GetGuaranteeTimestamp() + q.waitUntilServiceable(ctx, guaranteeTs, tsTypeDelta) + // shard follower considers solely historical segments + retrieveResults, err := q.historical.retrieveBySegmentIDs(collectionID, segmentIDs, q.vectorChunkManager, plan) + if err != nil { + return nil, err + } + mergedResult, err := mergeRetrieveResults(retrieveResults) + if err != nil { + return nil, err + } + + log.Debug("follower retrieve result", zap.String("ids", mergedResult.Ids.String())) + RetrieveResults := &internalpb.RetrieveResults{ + Status: &commonpb.Status{ErrorCode: commonpb.ErrorCode_Success}, + Ids: mergedResult.Ids, + FieldsData: mergedResult.FieldsData, + } + return RetrieveResults, nil +} + +// TODO: largely based on function mergeRetrieveResults, need rewriting +func mergeInternalRetrieveResults(retrieveResults []*internalpb.RetrieveResults) (*internalpb.RetrieveResults, error) { + var ret *internalpb.RetrieveResults + var skipDupCnt int64 + var idSet = make(map[int64]struct{}) + + // merge results and remove duplicates + for _, rr := range retrieveResults { + // skip if fields data is empty + if len(rr.FieldsData) == 0 { + continue + } + + if ret == nil { + ret = &internalpb.RetrieveResults{ + Ids: &schemapb.IDs{ + IdField: &schemapb.IDs_IntId{ + IntId: &schemapb.LongArray{ + Data: []int64{}, + }, + }, + }, + FieldsData: make([]*schemapb.FieldData, len(rr.FieldsData)), + } + } + + if len(ret.FieldsData) != len(rr.FieldsData) { + log.Warn("mismatch FieldData in RetrieveResults") + return nil, fmt.Errorf("mismatch FieldData in RetrieveResults") + } + + dstIds := ret.Ids.GetIntId() + for i, id := range rr.Ids.GetIntId().GetData() { + if _, ok := idSet[id]; !ok { + dstIds.Data = append(dstIds.Data, id) + typeutil.AppendFieldData(ret.FieldsData, rr.FieldsData, int64(i)) + idSet[id] = struct{}{} + } else { + // primary keys duplicate + skipDupCnt++ + } + } + } + + // not found, return default values indicating not result found + if ret == nil { + ret = &internalpb.RetrieveResults{ + Ids: &schemapb.IDs{}, + FieldsData: []*schemapb.FieldData{}, + } + } + + return ret, nil +} + +// func printSearchResultData(data *schemapb.SearchResultData, header string) { +// size := len(data.Ids.GetIntId().Data) +// if size != len(data.Scores) { +// log.Error("SearchResultData length mis-match") +// } +// log.Debug("==== SearchResultData ====", +// zap.String("header", header), zap.Int64("nq", data.NumQueries), zap.Int64("topk", data.TopK)) +// for i := 0; i < size; i++ { +// log.Debug("", zap.Int("i", i), zap.Int64("id", data.Ids.GetIntId().Data[i]), zap.Float32("score", data.Scores[i])) +// } +// } diff --git a/internal/querynode/query_shard_service.go b/internal/querynode/query_shard_service.go index 82352213b4..c022b06af5 100644 --- a/internal/querynode/query_shard_service.go +++ b/internal/querynode/query_shard_service.go @@ -20,7 +20,11 @@ import ( "context" "errors" "fmt" + "strconv" "sync" + + "github.com/milvus-io/milvus/internal/storage" + "github.com/milvus-io/milvus/internal/util/dependency" ) type queryShardService struct { @@ -29,19 +33,54 @@ type queryShardService struct { queryShardsMu sync.Mutex // guards queryShards queryShards map[Channel]*queryShard // Virtual Channel -> *queryShard + + queryChannelMu sync.Mutex // guards queryChannels + queryChannels map[int64]*queryChannel // Collection ID -> query channel + + factory dependency.Factory + + historical *historical + streaming *streaming + + shardClusterService *ShardClusterService + localChunkManager storage.ChunkManager + remoteChunkManager storage.ChunkManager + localCacheEnabled bool } -func newQueryShardService(ctx context.Context) *queryShardService { +func newQueryShardService(ctx context.Context, historical *historical, streaming *streaming, clusterService *ShardClusterService, factory dependency.Factory) *queryShardService { queryShardServiceCtx, queryShardServiceCancel := context.WithCancel(ctx) + + path := Params.LoadWithDefault("localStorage.Path", "/tmp/milvus/data") + enabled, _ := Params.Load("localStorage.enabled") + localCacheEnabled, _ := strconv.ParseBool(enabled) + localChunkManager := storage.NewLocalChunkManager(storage.RootPath(path)) + remoteChunkManager, _ := storage.NewMinioChunkManager( + ctx, + storage.Address(Params.MinioCfg.Address), + storage.AccessKeyID(Params.MinioCfg.AccessKeyID), + storage.SecretAccessKeyID(Params.MinioCfg.SecretAccessKey), + storage.UseSSL(Params.MinioCfg.UseSSL), + storage.BucketName(Params.MinioCfg.BucketName), + storage.CreateBucket(true)) + qss := &queryShardService{ - ctx: queryShardServiceCtx, - cancel: queryShardServiceCancel, - queryShards: make(map[Channel]*queryShard), + ctx: queryShardServiceCtx, + cancel: queryShardServiceCancel, + queryShards: make(map[Channel]*queryShard), + queryChannels: make(map[int64]*queryChannel), + historical: historical, + streaming: streaming, + shardClusterService: clusterService, + localChunkManager: localChunkManager, + remoteChunkManager: remoteChunkManager, + localCacheEnabled: localCacheEnabled, + factory: factory, } return qss } -func (q *queryShardService) addQueryShard(collectionID UniqueID, channel Channel) error { +func (q *queryShardService) addQueryShard(collectionID UniqueID, channel Channel, replicaID int64) error { q.queryShardsMu.Lock() defer q.queryShardsMu.Unlock() if _, ok := q.queryShards[channel]; ok { @@ -51,6 +90,13 @@ func (q *queryShardService) addQueryShard(collectionID UniqueID, channel Channel q.ctx, collectionID, channel, + replicaID, + q.shardClusterService, + q.historical, + q.streaming, + q.localChunkManager, + q.remoteChunkManager, + q.localCacheEnabled, ) q.queryShards[channel] = qs return nil @@ -81,3 +127,50 @@ func (q *queryShardService) getQueryShard(channel Channel) (*queryShard, error) } return q.queryShards[channel], nil } + +func (q *queryShardService) close() { + q.cancel() + q.queryShardsMu.Lock() + defer q.queryShardsMu.Unlock() + + for _, queryShard := range q.queryShards { + queryShard.Close() + } +} + +func (q *queryShardService) getQueryChannel(collectionID int64) *queryChannel { + q.queryChannelMu.Lock() + defer q.queryChannelMu.Unlock() + + qc, ok := q.queryChannels[collectionID] + if !ok { + queryStream, _ := q.factory.NewQueryMsgStream(q.ctx) + qc = &queryChannel{ + closeCh: make(chan struct{}), + collectionID: collectionID, + queryMsgStream: queryStream, + streaming: q.streaming, + } + q.queryChannels[collectionID] = qc + } + + return qc +} + +func (q *queryShardService) releaseCollection(collectionID int64) { + q.queryChannelMu.Lock() + qc, ok := q.queryChannels[collectionID] + if ok && qc != nil { + qc.Stop() + } + q.queryChannelMu.Unlock() + + q.queryShardsMu.Lock() + for _, queryShard := range q.queryShards { + if queryShard.collectionID == collectionID { + queryShard.Close() + delete(q.queryShards, queryShard.channel) + } + } + q.queryShardsMu.Unlock() +} diff --git a/internal/querynode/query_shard_service_test.go b/internal/querynode/query_shard_service_test.go index 39ab084352..76ecc3bc42 100644 --- a/internal/querynode/query_shard_service_test.go +++ b/internal/querynode/query_shard_service_test.go @@ -21,11 +21,15 @@ import ( "testing" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) func TestQueryShardService(t *testing.T) { - qss := newQueryShardService(context.Background()) - err := qss.addQueryShard(0, "vchan1") + qn, err := genSimpleQueryNode(context.Background()) + require.NoError(t, err) + + qss := newQueryShardService(context.Background(), qn.historical, qn.streaming, qn.ShardClusterService, qn.factory) + err = qss.addQueryShard(0, "vchan1", 0) assert.NoError(t, err) found1 := qss.hasQueryShard("vchan1") assert.Equal(t, true, found1) diff --git a/internal/querynode/query_shard_test.go b/internal/querynode/query_shard_test.go index 727d07bd4d..76b2c5087c 100644 --- a/internal/querynode/query_shard_test.go +++ b/internal/querynode/query_shard_test.go @@ -18,21 +18,263 @@ package querynode import ( "context" + "sync" "testing" "github.com/stretchr/testify/assert" + "github.com/milvus-io/milvus/internal/common" + "github.com/milvus-io/milvus/internal/proto/internalpb" "github.com/milvus-io/milvus/internal/proto/querypb" + "github.com/milvus-io/milvus/internal/proto/schemapb" ) +func genSimpleQueryShard(ctx context.Context) (*queryShard, error) { + tSafe := newTSafeReplica() + historical, err := genSimpleHistorical(ctx, tSafe) + if err != nil { + return nil, err + } + + streaming, err := genSimpleStreaming(ctx, tSafe) + if err != nil { + return nil, err + } + + localCM, err := genLocalChunkManager() + if err != nil { + return nil, err + } + + remoteCM, err := genRemoteChunkManager(ctx) + if err != nil { + return nil, err + } + + shardCluster := NewShardCluster(defaultCollectionID, defaultReplicaID, defaultDMLChannel, + &mockNodeDetector{}, &mockSegmentDetector{}, buildMockQueryNode) + shardClusterService := &ShardClusterService{ + clusters: sync.Map{}, + } + shardClusterService.clusters.Store(defaultDMLChannel, shardCluster) + + qs := newQueryShard(ctx, defaultCollectionID, defaultDMLChannel, defaultReplicaID, shardClusterService, + historical, streaming, localCM, remoteCM, false) + qs.deltaChannel = defaultDeltaChannel + + err = qs.watchDMLTSafe() + if err != nil { + return nil, err + } + err = qs.watchDeltaTSafe() + if err != nil { + return nil, err + } + + return qs, nil +} + +func updateQueryShardTSafe(qs *queryShard, timestamp Timestamp) error { + err := qs.streaming.tSafeReplica.setTSafe(defaultDMLChannel, timestamp) + if err != nil { + return err + } + return qs.historical.tSafeReplica.setTSafe(defaultDeltaChannel, timestamp) +} + +func TestQueryShard_Close(t *testing.T) { + qs, err := genSimpleQueryShard(context.Background()) + assert.NoError(t, err) + + qs.Close() +} + func TestQueryShard_Search(t *testing.T) { - qs := newQueryShard(context.Background(), 0, "vchan1") - _, err := qs.search(context.Background(), &querypb.SearchRequest{}) - assert.Error(t, err) + qs, err := genSimpleQueryShard(context.Background()) + assert.NoError(t, err) + + req, err := genSimpleSearchRequest(IndexFaissIDMap) + assert.NoError(t, err) + + t.Run("search follower", func(t *testing.T) { + request := &querypb.SearchRequest{ + Req: req, + DmlChannel: "", + SegmentIDs: []int64{defaultSegmentID}, + } + + _, err = qs.search(context.Background(), request) + assert.NoError(t, err) + }) + + t.Run("search leader", func(t *testing.T) { + request := &querypb.SearchRequest{ + Req: req, + DmlChannel: defaultDMLChannel, + SegmentIDs: []int64{}, + } + + _, err = qs.search(context.Background(), request) + assert.NoError(t, err) + }) } func TestQueryShard_Query(t *testing.T) { - qs := newQueryShard(context.Background(), 0, "vchan1") - _, err := qs.query(context.Background(), &querypb.QueryRequest{}) - assert.Error(t, err) + qs, err := genSimpleQueryShard(context.Background()) + assert.NoError(t, err) + + req, err := genSimpleRetrieveRequest() + assert.NoError(t, err) + + t.Run("query follower", func(t *testing.T) { + request := &querypb.QueryRequest{ + Req: req, + DmlChannel: "", + SegmentIDs: []int64{defaultSegmentID}, + } + + resp, err := qs.query(context.Background(), request) + assert.NoError(t, err) + assert.ElementsMatch(t, resp.Ids.GetIntId().Data, []int64{1, 2, 3}) + }) + + t.Run("query leader", func(t *testing.T) { + request := &querypb.QueryRequest{ + Req: req, + DmlChannel: defaultDMLChannel, + SegmentIDs: []int64{}, + } + + _, err := qs.query(context.Background(), request) + assert.NoError(t, err) + }) +} + +func TestQueryShard_waitNewTSafe(t *testing.T) { + qs, err := genSimpleQueryShard(context.Background()) + assert.NoError(t, err) + + timestamp := Timestamp(1000) + err = updateQueryShardTSafe(qs, timestamp) + assert.NoError(t, err) + + dmlTimestamp, err := qs.getNewTSafe(tsTypeDML) + assert.NoError(t, err) + assert.Equal(t, timestamp, dmlTimestamp) + + deltaTimestamp, err := qs.getNewTSafe(tsTypeDelta) + assert.NoError(t, err) + assert.Equal(t, timestamp, deltaTimestamp) +} + +func TestQueryShard_WaitUntilServiceable(t *testing.T) { + qs, err := genSimpleQueryShard(context.Background()) + assert.NoError(t, err) + + err = updateQueryShardTSafe(qs, 1000) + assert.NoError(t, err) + qs.waitUntilServiceable(context.Background(), 1000, tsTypeDML) +} + +func genSearchResultData(nq int64, topk int64, ids []int64, scores []float32) *schemapb.SearchResultData { + return &schemapb.SearchResultData{ + NumQueries: nq, + TopK: topk, + FieldsData: nil, + Scores: scores, + Ids: &schemapb.IDs{ + IdField: &schemapb.IDs_IntId{ + IntId: &schemapb.LongArray{ + Data: ids, + }, + }, + }, + Topks: make([]int64, nq), + } +} + +func TestReduceSearchResultData(t *testing.T) { + const ( + nq = 1 + topk = 4 + metricType = "L2" + ) + t.Run("case1", func(t *testing.T) { + ids := []int64{1, 2, 3, 4} + scores := []float32{-1.0, -2.0, -3.0, -4.0} + data1 := genSearchResultData(nq, topk, ids, scores) + data2 := genSearchResultData(nq, topk, ids, scores) + dataArray := make([]*schemapb.SearchResultData, 0) + dataArray = append(dataArray, data1) + dataArray = append(dataArray, data2) + res, err := reduceSearchResultData(dataArray, nq, topk, metricType) + assert.Nil(t, err) + assert.Equal(t, ids, res.Ids.GetIntId().Data) + assert.Equal(t, scores, res.Scores) + }) + t.Run("case2", func(t *testing.T) { + ids1 := []int64{1, 2, 3, 4} + scores1 := []float32{-1.0, -2.0, -3.0, -4.0} + ids2 := []int64{5, 1, 3, 4} + scores2 := []float32{-1.0, -1.0, -3.0, -4.0} + data1 := genSearchResultData(nq, topk, ids1, scores1) + data2 := genSearchResultData(nq, topk, ids2, scores2) + dataArray := make([]*schemapb.SearchResultData, 0) + dataArray = append(dataArray, data1) + dataArray = append(dataArray, data2) + res, err := reduceSearchResultData(dataArray, nq, topk, metricType) + assert.Nil(t, err) + assert.ElementsMatch(t, []int64{1, 5, 2, 3}, res.Ids.GetIntId().Data) + }) +} + +func TestMergeInternalRetrieveResults(t *testing.T) { + const ( + Dim = 8 + Int64FieldName = "Int64Field" + FloatVectorFieldName = "FloatVectorField" + Int64FieldID = common.StartOfUserFieldID + 1 + FloatVectorFieldID = common.StartOfUserFieldID + 2 + ) + Int64Array := []int64{11, 22} + FloatVector := []float32{1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 11.0, 22.0, 33.0, 44.0, 55.0, 66.0, 77.0, 88.0} + + var fieldDataArray1 []*schemapb.FieldData + fieldDataArray1 = append(fieldDataArray1, genFieldData(Int64FieldName, Int64FieldID, schemapb.DataType_Int64, Int64Array[0:2], 1)) + fieldDataArray1 = append(fieldDataArray1, genFieldData(FloatVectorFieldName, FloatVectorFieldID, schemapb.DataType_FloatVector, FloatVector[0:16], Dim)) + + var fieldDataArray2 []*schemapb.FieldData + fieldDataArray2 = append(fieldDataArray2, genFieldData(Int64FieldName, Int64FieldID, schemapb.DataType_Int64, Int64Array[0:2], 1)) + fieldDataArray2 = append(fieldDataArray2, genFieldData(FloatVectorFieldName, FloatVectorFieldID, schemapb.DataType_FloatVector, FloatVector[0:16], Dim)) + + result1 := &internalpb.RetrieveResults{ + Ids: &schemapb.IDs{ + IdField: &schemapb.IDs_IntId{ + IntId: &schemapb.LongArray{ + Data: []int64{0, 1}, + }, + }, + }, + // Offset: []int64{0, 1}, + FieldsData: fieldDataArray1, + } + result2 := &internalpb.RetrieveResults{ + Ids: &schemapb.IDs{ + IdField: &schemapb.IDs_IntId{ + IntId: &schemapb.LongArray{ + Data: []int64{0, 1}, + }, + }, + }, + // Offset: []int64{0, 1}, + FieldsData: fieldDataArray2, + } + + result, err := mergeInternalRetrieveResults([]*internalpb.RetrieveResults{result1, result2}) + assert.NoError(t, err) + assert.Equal(t, 2, len(result.FieldsData[0].GetScalars().GetLongData().Data)) + assert.Equal(t, 2*Dim, len(result.FieldsData[1].GetVectors().GetFloatVector().Data)) + + _, err = mergeInternalRetrieveResults(nil) + assert.NoError(t, err) } diff --git a/internal/querynode/shard_cluster.go b/internal/querynode/shard_cluster.go index 9c27ffcd79..d23147b8a8 100644 --- a/internal/querynode/shard_cluster.go +++ b/internal/querynode/shard_cluster.go @@ -68,16 +68,17 @@ type nodeEvent struct { } type segmentEvent struct { - eventType segmentEventType - segmentID int64 - nodeID int64 - state segmentState + eventType segmentEventType + segmentID int64 + partitionID int64 + nodeID int64 + state segmentState } type shardQueryNode interface { Search(context.Context, *querypb.SearchRequest) (*internalpb.SearchResults, error) Query(context.Context, *querypb.QueryRequest) (*internalpb.RetrieveResults, error) - Stop() + Stop() error } type shardNode struct { @@ -87,9 +88,10 @@ type shardNode struct { } type shardSegmentInfo struct { - segmentID int64 - nodeID int64 - state segmentState + segmentID int64 + partitionID int64 + nodeID int64 + state segmentState } // ShardNodeDetector provides method to detect node events @@ -159,6 +161,7 @@ func (sc *ShardCluster) Close() { // addNode add a node into cluster func (sc *ShardCluster) addNode(evt nodeEvent) { + log.Debug("ShardCluster add node", zap.Int64("nodeID", evt.nodeID)) sc.mut.Lock() defer sc.mut.Unlock() @@ -202,15 +205,18 @@ func (sc *ShardCluster) removeNode(evt nodeEvent) { // updateSegment apply segment change to shard cluster func (sc *ShardCluster) updateSegment(evt segmentEvent) { + + log.Debug("ShardCluster update segment", zap.Int64("nodeID", evt.nodeID), zap.Int64("segmentID", evt.segmentID), zap.Int32("state", int32(evt.state))) sc.mut.Lock() defer sc.mut.Unlock() old, ok := sc.segments[evt.segmentID] if !ok { // newly add sc.segments[evt.segmentID] = &shardSegmentInfo{ - nodeID: evt.nodeID, - segmentID: evt.segmentID, - state: evt.state, + nodeID: evt.nodeID, + partitionID: evt.partitionID, + segmentID: evt.segmentID, + state: evt.state, } return } @@ -302,6 +308,7 @@ func (sc *ShardCluster) watchNodes(evtCh <-chan nodeEvent) { for { select { case evt, ok := <-evtCh: + log.Debug("node event", zap.Any("evt", evt)) if !ok { log.Warn("ShardCluster node channel closed", zap.Int64("collectionID", sc.collectionID), zap.Int64("replicaID", sc.replicaID)) return @@ -324,6 +331,7 @@ func (sc *ShardCluster) watchSegments(evtCh <-chan segmentEvent) { for { select { case evt, ok := <-evtCh: + log.Debug("segment event", zap.Any("evt", evt)) if !ok { log.Warn("ShardCluster segment channel closed", zap.Int64("collectionID", sc.collectionID), zap.Int64("replicaID", sc.replicaID)) return @@ -365,19 +373,23 @@ func (sc *ShardCluster) getSegment(segmentID int64) (*shardSegmentInfo, bool) { return nil, false } return &shardSegmentInfo{ - segmentID: segment.segmentID, - nodeID: segment.nodeID, - state: segment.state, + segmentID: segment.segmentID, + nodeID: segment.nodeID, + partitionID: segment.partitionID, + state: segment.state, }, true } // segmentAllocations returns node to segments mappings. -func (sc *ShardCluster) segmentAllocations() map[int64][]int64 { +func (sc *ShardCluster) segmentAllocations(partitionIDs []int64) map[int64][]int64 { result := make(map[int64][]int64) // nodeID => segmentIDs sc.mut.RLock() defer sc.mut.RUnlock() for _, segment := range sc.segments { + if len(partitionIDs) > 0 && !inList(partitionIDs, segment.partitionID) { + continue + } result[segment.nodeID] = append(result[segment.nodeID], segment.segmentID) } return result @@ -386,16 +398,22 @@ func (sc *ShardCluster) segmentAllocations() map[int64][]int64 { // Search preforms search operation on shard cluster. func (sc *ShardCluster) Search(ctx context.Context, req *querypb.SearchRequest) ([]*internalpb.SearchResults, error) { if sc.state.Load() != int32(available) { - return nil, fmt.Errorf("SharcCluster for %s replicaID %d is no available", sc.vchannelName, sc.replicaID) + return nil, fmt.Errorf("ShardCluster for %s replicaID %d is no available", sc.vchannelName, sc.replicaID) } if sc.vchannelName != req.GetDmlChannel() { return nil, fmt.Errorf("ShardCluster for %s does not match to request channel :%s", sc.vchannelName, req.GetDmlChannel()) } - // get node allocation - segAllocs := sc.segmentAllocations() + //req.GetReq().GetPartitionIDs() + // get node allocation + segAllocs := sc.segmentAllocations(req.GetReq().GetPartitionIDs()) + + log.Debug("cluster segment distribution", zap.Int("len", len(segAllocs))) + for nodeID, segmentIDs := range segAllocs { + log.Debug("segments distribution", zap.Int64("nodeID", nodeID), zap.Int64s("segments", segmentIDs)) + } // TODO dispatch to local queryShardService query dml channel growing segments // concurrent visiting nodes @@ -409,7 +427,6 @@ func (sc *ShardCluster) Search(ctx context.Context, req *querypb.SearchRequest) for nodeID, segments := range segAllocs { nodeReq := proto.Clone(req).(*querypb.SearchRequest) - nodeReq.DmlChannel = "" nodeReq.SegmentIDs = segments node, ok := sc.getNode(nodeID) if !ok { // meta dismatch, report error @@ -441,7 +458,7 @@ func (sc *ShardCluster) Search(ctx context.Context, req *querypb.SearchRequest) // Query performs query operation on shard cluster. func (sc *ShardCluster) Query(ctx context.Context, req *querypb.QueryRequest) ([]*internalpb.RetrieveResults, error) { if sc.state.Load() != int32(available) { - return nil, fmt.Errorf("SharcCluster for %s replicaID %d is no available", sc.vchannelName, sc.replicaID) + return nil, fmt.Errorf("ShardCluster for %s replicaID %d is no available", sc.vchannelName, sc.replicaID) } // handles only the dml channel part, segment ids is dispatch by cluster itself @@ -450,7 +467,7 @@ func (sc *ShardCluster) Query(ctx context.Context, req *querypb.QueryRequest) ([ } // get node allocation - segAllocs := sc.segmentAllocations() + segAllocs := sc.segmentAllocations(req.GetReq().GetPartitionIDs()) // TODO dispatch to local queryShardService query dml channel growing segments @@ -465,7 +482,6 @@ func (sc *ShardCluster) Query(ctx context.Context, req *querypb.QueryRequest) ([ for nodeID, segments := range segAllocs { nodeReq := proto.Clone(req).(*querypb.QueryRequest) - nodeReq.DmlChannel = "" nodeReq.SegmentIDs = segments node, ok := sc.getNode(nodeID) if !ok { // meta dismatch, report error diff --git a/internal/querynode/shard_cluster_service.go b/internal/querynode/shard_cluster_service.go new file mode 100644 index 0000000000..1a1aabd596 --- /dev/null +++ b/internal/querynode/shard_cluster_service.go @@ -0,0 +1,109 @@ +package querynode + +import ( + "context" + "fmt" + "path" + "strconv" + "sync" + + grpcquerynodeclient "github.com/milvus-io/milvus/internal/distributed/querynode/client" + "github.com/milvus-io/milvus/internal/util" + "github.com/milvus-io/milvus/internal/util/sessionutil" + "github.com/milvus-io/milvus/internal/util/typeutil" + clientv3 "go.etcd.io/etcd/client/v3" +) + +const ( + ReplicaMetaPrefix = "queryCoord-ReplicaMeta" +) + +// shardQueryNodeWrapper wraps a querynode to shardQueryNode and preventing it been closed +type shardQueryNodeWrapper struct { + *QueryNode +} + +// Stop overrides default close method +func (w *shardQueryNodeWrapper) Stop() error { return nil } + +// ShardClusterService maintains the online ShardCluster(leader) in this querynode. +type ShardClusterService struct { + client *clientv3.Client // etcd client for detectors + session *sessionutil.Session + node *QueryNode + + clusters sync.Map // channel name => *shardCluster +} + +// newShardClusterService returns a new shardClusterService +func newShardClusterService(client *clientv3.Client, session *sessionutil.Session, node *QueryNode) *ShardClusterService { + return &ShardClusterService{ + node: node, + session: session, + client: client, + clusters: sync.Map{}, + } +} + +// addShardCluster adds shardCluster into service. +func (s *ShardClusterService) addShardCluster(collectionID, replicaID int64, vchannelName string) { + nodeDetector := NewEtcdShardNodeDetector(s.client, path.Join(Params.EtcdCfg.MetaRootPath, ReplicaMetaPrefix), + func() (map[int64]string, error) { + result := make(map[int64]string) + sessions, _, err := s.session.GetSessions(typeutil.QueryNodeRole) + if err != nil { + return nil, err + } + for _, session := range sessions { + result[session.ServerID] = session.Address + } + return result, nil + }) + + segmentDetector := NewEtcdShardSegmentDetector(s.client, path.Join(Params.EtcdCfg.MetaRootPath, util.SegmentMetaPrefix, strconv.FormatInt(collectionID, 10))) + + cs := NewShardCluster(collectionID, replicaID, vchannelName, nodeDetector, segmentDetector, + func(nodeID int64, addr string) shardQueryNode { + if nodeID == s.session.ServerID { + // wrap node itself + return &shardQueryNodeWrapper{QueryNode: s.node} + } + ctx := context.Background() + qn, _ := grpcquerynodeclient.NewClient(ctx, addr) + return qn + }) + + s.clusters.Store(vchannelName, cs) +} + +// getShardCluster gets shardCluster of specified vchannel if exists. +func (s *ShardClusterService) getShardCluster(vchannelName string) (*ShardCluster, bool) { + raw, ok := s.clusters.Load(vchannelName) + if !ok { + return nil, false + } + return raw.(*ShardCluster), true +} + +// releaseShardCluster removes shardCluster from service and stops it. +func (s *ShardClusterService) releaseShardCluster(vchannelName string) error { + raw, ok := s.clusters.LoadAndDelete(vchannelName) + if !ok { + return fmt.Errorf("ShardCluster of channel: %s does not exists", vchannelName) + } + + cs := raw.(*ShardCluster) + cs.Close() + return nil +} + +// releaseCollection removes all shardCluster matching specified collectionID +func (s *ShardClusterService) releaseCollection(collectionID int64) { + s.clusters.Range(func(k, v interface{}) bool { + cs := v.(*ShardCluster) + if cs.collectionID == collectionID { + s.releaseShardCluster(k.(string)) + } + return true + }) +} diff --git a/internal/querynode/shard_cluster_service_test.go b/internal/querynode/shard_cluster_service_test.go new file mode 100644 index 0000000000..8acb441ee8 --- /dev/null +++ b/internal/querynode/shard_cluster_service_test.go @@ -0,0 +1,34 @@ +package querynode + +import ( + "context" + "testing" + + "github.com/milvus-io/milvus/internal/util/sessionutil" + "github.com/stretchr/testify/assert" + "go.etcd.io/etcd/server/v3/etcdserver/api/v3client" +) + +func TestShardClusterService(t *testing.T) { + client := v3client.New(embedetcdServer.Server) + defer client.Close() + session := sessionutil.NewSession(context.Background(), "/by-dev/sessions/unittest/querynode/", client) + clusterService := newShardClusterService(client, session, nil) + + assert.NotPanics(t, func() { + clusterService.addShardCluster(defaultCollectionID, defaultReplicaID, defaultDMLChannel) + }) + + shardCluster, ok := clusterService.getShardCluster(defaultDMLChannel) + assert.True(t, ok) + assert.NotNil(t, shardCluster) + + _, ok = clusterService.getShardCluster("non-exist-channel") + assert.False(t, ok) + + err := clusterService.releaseShardCluster(defaultDMLChannel) + assert.NoError(t, err) + + err = clusterService.releaseShardCluster("non-exist-channel") + assert.Error(t, err) +} diff --git a/internal/querynode/shard_cluster_test.go b/internal/querynode/shard_cluster_test.go index 1bf47d4ab6..abd5208115 100644 --- a/internal/querynode/shard_cluster_test.go +++ b/internal/querynode/shard_cluster_test.go @@ -61,7 +61,8 @@ func (m *mockShardQueryNode) Query(_ context.Context, _ *querypb.QueryRequest) ( return m.queryResult, m.queryErr } -func (m *mockShardQueryNode) Stop() { +func (m *mockShardQueryNode) Stop() error { + return nil } func buildMockQueryNode(nodeID int64, addr string) shardQueryNode { diff --git a/internal/querynode/shard_node_detector.go b/internal/querynode/shard_node_detector.go index 1bb8760c3b..cb5a439015 100644 --- a/internal/querynode/shard_node_detector.go +++ b/internal/querynode/shard_node_detector.go @@ -65,6 +65,7 @@ func (nd *etcdShardNodeDetector) Close() { // watchNodes lists current online nodes and returns a channel for incoming events. func (nd *etcdShardNodeDetector) watchNodes(collectionID int64, replicaID int64, vchannelName string) ([]nodeEvent, <-chan nodeEvent) { + log.Debug("nodeDetector watch", zap.Int64("collectionID", collectionID), zap.Int64("replicaID", replicaID), zap.String("vchannelName", vchannelName)) resp, err := nd.client.Get(context.Background(), nd.path, clientv3.WithPrefix()) if err != nil { log.Warn("Etcd NodeDetector get replica info failed", zap.Error(err)) @@ -264,5 +265,8 @@ func (nd *etcdShardNodeDetector) handleDelEvent(e *clientv3.Event, collectionID, func (nd *etcdShardNodeDetector) parseReplicaInfo(bs []byte) (*milvuspb.ReplicaInfo, error) { info := &milvuspb.ReplicaInfo{} err := proto.Unmarshal(bs, info) + if err == nil { + log.Debug("ReplicaInfo", zap.Any("info", info)) + } return info, err } diff --git a/internal/querynode/shard_node_detector_test.go b/internal/querynode/shard_node_detector_test.go index 293a9957aa..592039ac69 100644 --- a/internal/querynode/shard_node_detector_test.go +++ b/internal/querynode/shard_node_detector_test.go @@ -19,9 +19,6 @@ package querynode import ( "context" "fmt" - "io/ioutil" - "net/url" - "os" "path" "strconv" "testing" @@ -33,41 +30,12 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "go.etcd.io/etcd/server/v3/embed" "go.etcd.io/etcd/server/v3/etcdserver/api/v3client" ) -func startEmbedEtcdServer() (*embed.Etcd, error) { - dir, err := ioutil.TempDir(os.TempDir(), "milvus_ut") - if err != nil { - return nil, err - } - defer os.RemoveAll(dir) - config := embed.NewConfig() - - config.Dir = os.TempDir() - config.LogLevel = "warn" - config.LogOutputs = []string{"default"} - u, err := url.Parse("http://localhost:2389") - if err != nil { - return nil, err - } - config.LCUrls = []url.URL{*u} - u, err = url.Parse("http://localhost:2390") - if err != nil { - return nil, err - } - config.LPUrls = []url.URL{*u} - - return embed.StartEtcd(config) -} - func TestEtcdShardNodeDetector_watch(t *testing.T) { - etcdServer, err := startEmbedEtcdServer() - require.NoError(t, err) - defer etcdServer.Close() - client := v3client.New(etcdServer.Server) + client := v3client.New(embedetcdServer.Server) defer client.Close() type testCase struct { diff --git a/internal/querynode/shard_segment_detector.go b/internal/querynode/shard_segment_detector.go new file mode 100644 index 0000000000..a40a50e929 --- /dev/null +++ b/internal/querynode/shard_segment_detector.go @@ -0,0 +1,209 @@ +// Licensed to the LF AI & Data foundation under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package querynode + +import ( + "context" + "sync" + + "github.com/golang/protobuf/proto" + "github.com/milvus-io/milvus/internal/log" + "github.com/milvus-io/milvus/internal/proto/querypb" + "go.etcd.io/etcd/api/v3/mvccpb" + v3rpc "go.etcd.io/etcd/api/v3/v3rpc/rpctypes" + clientv3 "go.etcd.io/etcd/client/v3" + "go.uber.org/zap" +) + +// etcdShardSegmentDetector watch etcd prefix for segment event. +type etcdShardSegmentDetector struct { + client *clientv3.Client + path string + evtCh chan segmentEvent + + wg sync.WaitGroup + closeCh chan struct{} + closeOnce sync.Once +} + +// NewEtcdShardSegmentDetector returns a segmentDetector with provided etcd client and root path. +func NewEtcdShardSegmentDetector(client *clientv3.Client, rootPath string) *etcdShardSegmentDetector { + return &etcdShardSegmentDetector{ + client: client, + path: rootPath, + evtCh: make(chan segmentEvent, 32), + closeCh: make(chan struct{}), + } +} + +// Close perform closing procedure and notify all watcher to quit. +func (sd *etcdShardSegmentDetector) Close() { + sd.closeOnce.Do(func() { + close(sd.closeCh) + sd.wg.Wait() + close(sd.evtCh) + }) +} + +func (sd *etcdShardSegmentDetector) afterClose(fn func()) { + <-sd.closeCh + fn() +} + +func (sd *etcdShardSegmentDetector) getCtx() context.Context { + ctx, cancel := context.WithCancel(context.Background()) + go sd.afterClose(cancel) + return ctx +} + +func (sd *etcdShardSegmentDetector) watchSegments(collectionID int64, replicaID int64, vchannelName string) ([]segmentEvent, <-chan segmentEvent) { + log.Debug("segmentDetector start watch", zap.Int64("collectionID", collectionID), + zap.Int64("replicaID", replicaID), + zap.String("vchannelName", vchannelName), + zap.String("rootPath", sd.path)) + resp, err := sd.client.Get(context.Background(), sd.path, clientv3.WithPrefix()) + if err != nil { + log.Warn("Etcd SegmentDetector get replica info failed", zap.Error(err)) + panic(err) + } + + var events []segmentEvent + for _, kv := range resp.Kvs { + info, err := sd.parseSegmentInfo(kv.Value) + if err != nil { + log.Warn("SegmentDetector failed to parse segmentInfo", zap.Error(err)) + continue + } + if info.CollectionID != collectionID || info.GetDmChannel() != vchannelName { + continue + } + + if inList(info.GetReplicaIds(), replicaID) { + events = append(events, segmentEvent{ + eventType: segmentAdd, + segmentID: info.GetSegmentID(), + partitionID: info.GetPartitionID(), + nodeID: info.GetNodeID(), + state: segmentStateLoaded, + }) + } + } + + sd.wg.Add(1) + watchCh := sd.client.Watch(sd.getCtx(), sd.path, clientv3.WithRev(resp.Header.GetRevision()+1), clientv3.WithPrefix(), clientv3.WithPrevKV()) + go sd.watch(watchCh, collectionID, replicaID, vchannelName) + + return events, sd.evtCh +} + +func (sd *etcdShardSegmentDetector) watch(ch clientv3.WatchChan, collectionID int64, replicaID int64, vchannel string) { + defer sd.wg.Done() + log.Debug("etcdSegmentDetector start watch") + for { + select { + case <-sd.closeCh: + log.Warn("Closed SegmentDetector watch loop quit", zap.Int64("collectionID", collectionID), zap.Int64("replicaID", replicaID)) + return + case evt, ok := <-ch: + if !ok { + log.Warn("SegmentDetector event channel closed") + return + } + if err := evt.Err(); err != nil { + if err == v3rpc.ErrCompacted { + sd.wg.Add(1) + watchCh := sd.client.Watch(sd.getCtx(), sd.path, clientv3.WithPrefix()) + go sd.watch(watchCh, collectionID, replicaID, vchannel) + return + } + } + for _, e := range evt.Events { + log.Debug("segment evt", zap.Any("evt", evt)) + switch e.Type { + case mvccpb.PUT: + sd.handlePutEvent(e, collectionID, replicaID, vchannel) + case mvccpb.DELETE: + sd.handleDelEvent(e, collectionID, replicaID, vchannel) + } + } + } + } +} + +func (sd *etcdShardSegmentDetector) handlePutEvent(e *clientv3.Event, collectionID int64, replicaID int64, vchannel string) { + info, err := sd.parseSegmentInfo(e.Kv.Value) + if err != nil { + log.Warn("Segment detector failed to parse event", zap.Any("event", e), zap.Error(err)) + return + } + + if info.GetCollectionID() != collectionID || vchannel != info.GetDmChannel() || !inList(info.GetReplicaIds(), replicaID) { + // ignore not match events + return + } + + sd.evtCh <- segmentEvent{ + eventType: segmentAdd, + segmentID: info.GetSegmentID(), + partitionID: info.GetPartitionID(), + nodeID: info.GetNodeID(), + state: segmentStateLoaded, + } +} + +func (sd *etcdShardSegmentDetector) handleDelEvent(e *clientv3.Event, collectionID int64, replicaID int64, vchannel string) { + if e.PrevKv == nil { + return + } + info, err := sd.parseSegmentInfo(e.PrevKv.Value) + if err != nil { + log.Warn("SegmentDetector failed to parse delete event", zap.Any("event", e), zap.Error(err)) + return + } + + if info.GetCollectionID() != collectionID || vchannel != info.GetDmChannel() || !inList(info.GetReplicaIds(), replicaID) { + // ignore not match events + return + } + + sd.evtCh <- segmentEvent{ + eventType: segmentDel, + segmentID: info.GetSegmentID(), + partitionID: info.GetPartitionID(), + nodeID: info.GetNodeID(), + state: segmentStateOffline, + } +} + +// TODO maybe should use other proto +func (sd *etcdShardSegmentDetector) parseSegmentInfo(bs []byte) (*querypb.SegmentInfo, error) { + info := &querypb.SegmentInfo{} + err := proto.Unmarshal(bs, info) + if err == nil { + log.Debug("segment info", zap.Any("segmentInfo", info)) + } + return info, err +} + +func inList(list []int64, target int64) bool { + for _, i := range list { + if i == target { + return true + } + } + return false +} diff --git a/internal/querynode/shard_segment_detector_test.go b/internal/querynode/shard_segment_detector_test.go new file mode 100644 index 0000000000..0372b8d786 --- /dev/null +++ b/internal/querynode/shard_segment_detector_test.go @@ -0,0 +1,298 @@ +package querynode + +import ( + "context" + "fmt" + "path" + "testing" + "time" + + "github.com/golang/protobuf/proto" + "github.com/milvus-io/milvus/internal/proto/querypb" + "github.com/milvus-io/milvus/internal/util/funcutil" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.etcd.io/etcd/server/v3/etcdserver/api/v3client" +) + +func TestEtcdShardSegmentDetector_watch(t *testing.T) { + + client := v3client.New(embedetcdServer.Server) + defer client.Close() + + type testCase struct { + name string + oldRecords map[string]*querypb.SegmentInfo + updateRecords map[string]*querypb.SegmentInfo + delRecords []string + expectInitEvents []segmentEvent + expectUpdateEvents []segmentEvent + oldGarbage map[string]string + newGarbage map[string]string + + collectionID int64 + replicaID int64 + channel string + } + cases := []testCase{ + { + name: "normal init", + collectionID: 1, + replicaID: 1, + channel: "dml_1_1_v0", + oldRecords: map[string]*querypb.SegmentInfo{ + "segment_1": { + CollectionID: 1, + SegmentID: 1, + NodeID: 1, + DmChannel: "dml_1_1_v0", + ReplicaIds: []int64{1, 2}, + }, + }, + oldGarbage: map[string]string{ + "noice1": string([]byte{23, 21, 20}), + }, + expectInitEvents: []segmentEvent{ + { + eventType: segmentAdd, + segmentID: 1, + nodeID: 1, + state: segmentStateLoaded, + }, + }, + }, + { + name: "normal init with other segments", + collectionID: 1, + replicaID: 1, + channel: "dml_1_1_v0", + oldRecords: map[string]*querypb.SegmentInfo{ + "segment_1": { + CollectionID: 1, + SegmentID: 1, + NodeID: 1, + DmChannel: "dml_1_1_v0", + ReplicaIds: []int64{1, 2}, + }, + "segment_2": { + CollectionID: 1, + SegmentID: 2, + NodeID: 1, + DmChannel: "dml_1_1_v1", + ReplicaIds: []int64{1, 2}, + }, + "segment_3": { + CollectionID: 2, + SegmentID: 3, + NodeID: 2, + DmChannel: "dml_3_2_v0", + ReplicaIds: []int64{1, 2}, + }, + "segment_4": { // may not happen + CollectionID: 1, + SegmentID: 4, + NodeID: 1, + DmChannel: "dml_1_1_v0", + ReplicaIds: []int64{2}, + }, + }, + expectInitEvents: []segmentEvent{ + { + eventType: segmentAdd, + segmentID: 1, + nodeID: 1, + state: segmentStateLoaded, + }, + }, + }, + { + name: "normal add segment", + collectionID: 1, + replicaID: 1, + channel: "dml_1_1_v0", + updateRecords: map[string]*querypb.SegmentInfo{ + "segment_1": { + CollectionID: 1, + SegmentID: 1, + NodeID: 1, + DmChannel: "dml_1_1_v0", + ReplicaIds: []int64{1, 2}, + }, + }, + oldGarbage: map[string]string{ + "noice1": string([]byte{23, 21, 20}), + }, + expectUpdateEvents: []segmentEvent{ + { + eventType: segmentAdd, + segmentID: 1, + nodeID: 1, + state: segmentStateLoaded, + }, + }, + }, + { + name: "normal add segment with other replica", + collectionID: 1, + replicaID: 1, + channel: "dml_1_1_v0", + updateRecords: map[string]*querypb.SegmentInfo{ + "segment_1": { + CollectionID: 1, + SegmentID: 1, + NodeID: 1, + DmChannel: "dml_1_1_v0", + ReplicaIds: []int64{1, 2}, + }, + "segment_2": { + CollectionID: 1, + SegmentID: 2, + NodeID: 2, + DmChannel: "dml_2_1_v1", + ReplicaIds: []int64{2}, + }, + }, + newGarbage: map[string]string{ + "noice1": string([]byte{23, 21, 20}), + }, + expectUpdateEvents: []segmentEvent{ + { + eventType: segmentAdd, + segmentID: 1, + nodeID: 1, + state: segmentStateLoaded, + }, + }, + }, + { + name: "normal remove segment", + collectionID: 1, + replicaID: 1, + channel: "dml_1_1_v0", + oldRecords: map[string]*querypb.SegmentInfo{ + "segment_1": { + CollectionID: 1, + SegmentID: 1, + NodeID: 1, + DmChannel: "dml_1_1_v0", + ReplicaIds: []int64{1, 2}, + }, + }, + oldGarbage: map[string]string{ + "noice1": string([]byte{23, 21, 20}), + }, + expectInitEvents: []segmentEvent{ + { + eventType: segmentAdd, + segmentID: 1, + nodeID: 1, + state: segmentStateLoaded, + }, + }, + delRecords: []string{ + "segment_1", "noice1", + }, + expectUpdateEvents: []segmentEvent{ + { + eventType: segmentDel, + segmentID: 1, + nodeID: 1, + state: segmentStateOffline, + }, + }, + }, + { + name: "normal remove segment with other replica", + collectionID: 1, + replicaID: 1, + channel: "dml_1_1_v0", + oldRecords: map[string]*querypb.SegmentInfo{ + "segment_1": { + CollectionID: 1, + SegmentID: 1, + NodeID: 1, + DmChannel: "dml_1_1_v0", + ReplicaIds: []int64{1, 2}, + }, + "segment_2": { + CollectionID: 1, + SegmentID: 2, + NodeID: 2, + DmChannel: "dml_2_1_v1", + ReplicaIds: []int64{2}, + }, + }, + expectInitEvents: []segmentEvent{ + { + eventType: segmentAdd, + segmentID: 1, + nodeID: 1, + state: segmentStateLoaded, + }, + }, + delRecords: []string{ + "segment_1", "segment_2", + }, + expectUpdateEvents: []segmentEvent{ + { + eventType: segmentDel, + segmentID: 1, + nodeID: 1, + state: segmentStateOffline, + }, + }, + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + suffix := funcutil.RandomString(6) + rootPath := fmt.Sprintf("qn_shard_segment_detector_watch_%s", suffix) + + ctx := context.Background() + // put existing records + for key, info := range tc.oldRecords { + bs, err := proto.Marshal(info) + require.NoError(t, err) + _, err = client.Put(ctx, path.Join(rootPath, key), string(bs)) + require.NoError(t, err) + } + // put garbage data + for key, value := range tc.oldGarbage { + _, err := client.Put(ctx, path.Join(rootPath, key), value) + require.NoError(t, err) + } + + sd := NewEtcdShardSegmentDetector(client, rootPath) + segments, ch := sd.watchSegments(tc.collectionID, tc.replicaID, tc.channel) + assert.ElementsMatch(t, tc.expectInitEvents, segments) + + // update etcd kvs to generate events + go func() { + for key, info := range tc.updateRecords { + bs, err := proto.Marshal(info) + require.NoError(t, err) + _, err = client.Put(ctx, path.Join(rootPath, key), string(bs)) + require.NoError(t, err) + } + for _, k := range tc.delRecords { + _, err := client.Delete(ctx, path.Join(rootPath, k)) + require.NoError(t, err) + } + for key, value := range tc.newGarbage { + _, err := client.Put(ctx, path.Join(rootPath, key), value) + require.NoError(t, err) + } + // need a way to detect event processed + time.Sleep(time.Millisecond * 100) + + sd.Close() + }() + + var newEvents []segmentEvent + for evt := range ch { + newEvents = append(newEvents, evt) + } + assert.ElementsMatch(t, tc.expectUpdateEvents, newEvents) + }) + } +} diff --git a/internal/querynode/streaming.go b/internal/querynode/streaming.go index 79ce6c1ac6..1e5c2109ac 100644 --- a/internal/querynode/streaming.go +++ b/internal/querynode/streaming.go @@ -61,7 +61,7 @@ func (s *streaming) close() { s.replica.freeAll() } -func (s *streaming) retrieve(collID UniqueID, partIDs []UniqueID, plan *RetrievePlan) ([]*segcorepb.RetrieveResults, []UniqueID, []UniqueID, error) { +func (s *streaming) retrieve(collID UniqueID, partIDs []UniqueID, plan *RetrievePlan, filters ...func(segment *Segment) bool) ([]*segcorepb.RetrieveResults, []UniqueID, []UniqueID, error) { retrieveResults := make([]*segcorepb.RetrieveResults, 0) retrieveSegmentIDs := make([]UniqueID, 0) @@ -91,6 +91,15 @@ func (s *streaming) retrieve(collID UniqueID, partIDs []UniqueID, plan *Retrieve if err != nil { return retrieveResults, retrieveSegmentIDs, retrievePartIDs, err } + filtered := false + for _, filter := range filters { + if !filter(seg) { + filtered = true + } + } + if filtered { + continue + } result, err := seg.retrieve(plan) if err != nil { return retrieveResults, retrieveSegmentIDs, retrievePartIDs, err diff --git a/internal/querynode/task.go b/internal/querynode/task.go index ae1f8e7b19..6e9776d258 100644 --- a/internal/querynode/task.go +++ b/internal/querynode/task.go @@ -27,7 +27,6 @@ import ( "go.uber.org/zap" "github.com/milvus-io/milvus/internal/log" - "github.com/milvus-io/milvus/internal/metrics" "github.com/milvus-io/milvus/internal/proto/commonpb" "github.com/milvus-io/milvus/internal/proto/datapb" "github.com/milvus-io/milvus/internal/proto/internalpb" @@ -133,61 +132,29 @@ func (r *addQueryChannelTask) Execute(ctx context.Context) error { zap.Any("collectionID", r.req.CollectionID)) collectionID := r.req.CollectionID - if r.node.queryService == nil { - errMsg := "null query service, collectionID = " + fmt.Sprintln(collectionID) - return errors.New(errMsg) + if r.node.queryShardService == nil { + return fmt.Errorf("null query shard service, collectionID %d", collectionID) } - if r.node.queryService.hasQueryCollection(collectionID) { - log.Debug("queryCollection has been existed when addQueryChannel", - zap.Any("collectionID", collectionID), - ) - return nil - } + qc := r.node.queryShardService.getQueryChannel(collectionID) + log.Debug("add query channel for collection", zap.Int64("collectionID", collectionID)) - // add search collection - err := r.node.queryService.addQueryCollection(collectionID) - if err != nil { - return err - } - log.Debug("add query collection", zap.Any("collectionID", collectionID)) - - // add request channel - sc, err := r.node.queryService.getQueryCollection(collectionID) - if err != nil { - return err - } - consumeChannels := []string{r.req.QueryChannel} consumeSubName := funcutil.GenChannelSubName(Params.CommonCfg.QueryNodeSubName, collectionID, Params.QueryNodeCfg.QueryNodeID) - sc.queryMsgStream.AsConsumer(consumeChannels, consumeSubName) - metrics.QueryNodeNumConsumers.WithLabelValues(fmt.Sprint(Params.QueryNodeCfg.QueryNodeID)).Inc() - if r.req.SeekPosition == nil || len(r.req.SeekPosition.MsgID) == 0 { - // as consumer - log.Debug("QueryNode AsConsumer", zap.Strings("channels", consumeChannels), zap.String("sub name", consumeSubName)) - } else { - // seek query channel - err = sc.queryMsgStream.Seek([]*internalpb.MsgPosition{r.req.SeekPosition}) - if err != nil { - return err - } - log.Debug("QueryNode seek query channel: ", zap.Any("consumeChannels", consumeChannels), - zap.String("seek position", string(r.req.SeekPosition.MsgID))) + err := qc.AsConsumer(r.req.QueryChannel, consumeSubName, r.req.SeekPosition) + if err != nil { + log.Warn("query channel as consumer failed", zap.Int64("collectionID", collectionID), zap.String("channel", r.req.QueryChannel), zap.Error(err)) + return err } - // add result channel - // producerChannels := []string{r.req.QueryResultChannel} - // sc.queryResultMsgStream.AsProducer(producerChannels) - // log.Debug("QueryNode AsProducer", zap.Strings("channels", producerChannels)) - // init global sealed segments - for _, segment := range r.req.GlobalSealedSegments { - sc.globalSegmentManager.addGlobalSegmentInfo(segment) - } + /* + for _, segment := range r.req.GlobalSealedSegments { + sc.globalSegmentManager.addGlobalSegmentInfo(segment) + }*/ - // start queryCollection, message stream need to asConsumer before start - sc.start() - log.Debug("start query collection", zap.Any("collectionID", collectionID)) + qc.Start() + log.Debug("start query channel", zap.Int64("collectionID", collectionID)) log.Debug("addQueryChannelTask done", zap.Any("collectionID", r.req.CollectionID), @@ -254,6 +221,7 @@ func (w *watchDmChannelsTask) Execute(ctx context.Context) error { log.Debug("Starting WatchDmChannels ...", zap.String("collectionName", w.req.Schema.Name), zap.Int64("collectionID", collectionID), + zap.Int64("replicaID", w.req.GetReplicaID()), zap.Any("load type", lType), zap.Strings("vChannels", vChannels), zap.Strings("pChannels", pChannels), @@ -263,6 +231,11 @@ func (w *watchDmChannelsTask) Execute(ctx context.Context) error { sCol := w.node.streaming.replica.addCollection(collectionID, w.req.Schema) hCol := w.node.historical.replica.addCollection(collectionID, w.req.Schema) + //add shard cluster + for _, vchannel := range vChannels { + w.node.ShardClusterService.addShardCluster(w.req.GetCollectionID(), w.req.GetReplicaID(), vchannel) + } + // load growing segments unFlushedSegments := make([]*queryPb.SegmentLoadInfo, 0) unFlushedSegmentIDs := make([]UniqueID, 0) @@ -453,15 +426,21 @@ func (w *watchDmChannelsTask) Execute(ctx context.Context) error { w.node.tSafeReplica.addTSafe(channel) } - // add tSafe watcher if queryCollection exists - qc, err := w.node.queryService.getQueryCollection(collectionID) - if err == nil { - for _, channel := range vChannels { - err = qc.addTSafeWatcher(channel) - if err != nil { - // tSafe have been exist, not error - log.Warn(err.Error()) - } + // add tsafe watch in query shard if exists + for _, dmlChannel := range vChannels { + if !w.node.queryShardService.hasQueryShard(dmlChannel) { + //TODO add replica id in req + w.node.queryShardService.addQueryShard(collectionID, dmlChannel, 0) + } + + qs, err := w.node.queryShardService.getQueryShard(dmlChannel) + if err != nil { + log.Warn("failed to get query shard", zap.String("dmlChannel", dmlChannel), zap.Error(err)) + continue + } + err = qs.watchDMLTSafe() + if err != nil { + log.Warn("failed to start query shard watch dml tsafe", zap.Error(err)) } } @@ -591,15 +570,26 @@ func (w *watchDeltaChannelsTask) Execute(ctx context.Context) error { w.node.tSafeReplica.addTSafe(channel) } - // add tSafe watcher if queryCollection exists - qc, err := w.node.queryService.getQueryCollection(collectionID) - if err == nil { - for _, channel := range vDeltaChannels { - err = qc.addTSafeWatcher(channel) - if err != nil { - // tSafe have been existed, not error - log.Warn(err.Error()) - } + // add tsafe watch in query shard if exists + for _, channel := range vDeltaChannels { + dmlChannel, err := funcutil.ConvertChannelName(channel, Params.CommonCfg.RootCoordDelta, Params.CommonCfg.RootCoordDml) + if err != nil { + log.Warn("failed to convert delta channel to dml", zap.String("channel", channel), zap.Error(err)) + continue + } + if !w.node.queryShardService.hasQueryShard(dmlChannel) { + //TODO add replica id in req + w.node.queryShardService.addQueryShard(collectionID, dmlChannel, 0) + } + + qs, err := w.node.queryShardService.getQueryShard(dmlChannel) + if err != nil { + log.Warn("failed to get query shard", zap.String("dmlChannel", dmlChannel), zap.Error(err)) + continue + } + err = qs.watchDeltaTSafe() + if err != nil { + log.Warn("failed to start query shard watch delta tsafe", zap.Error(err)) } } @@ -712,10 +702,7 @@ func (r *releaseCollectionTask) Execute(ctx context.Context) error { zap.Any("collectionID", r.req.CollectionID), ) - // remove query collection - // queryCollection and Collection would be deleted in releaseCollection, - // so we don't need to remove the tSafeWatcher or channel manually. - r.node.queryService.stopQueryCollection(r.req.CollectionID) + r.node.queryShardService.releaseCollection(r.req.CollectionID) err := r.releaseReplica(r.node.streaming.replica, replicaStreaming) if err != nil { diff --git a/internal/querynode/task_test.go b/internal/querynode/task_test.go index cd2aebf640..1364156f27 100644 --- a/internal/querynode/task_test.go +++ b/internal/querynode/task_test.go @@ -82,14 +82,10 @@ func TestTask_AddQueryChannel(t *testing.T) { err = task.Execute(ctx) assert.NoError(t, err) }) - t.Run("test execute has queryCollection", func(t *testing.T) { node, err := genSimpleQueryNode(ctx) assert.NoError(t, err) - err = node.queryService.addQueryCollection(defaultCollectionID) - assert.NoError(t, err) - task := addQueryChannelTask{ req: genAddQueryChanelRequest(), node: node, @@ -98,12 +94,11 @@ func TestTask_AddQueryChannel(t *testing.T) { err = task.Execute(ctx) assert.NoError(t, err) }) - t.Run("test execute nil query service", func(t *testing.T) { node, err := genSimpleQueryNode(ctx) assert.NoError(t, err) - node.queryService = nil + node.queryShardService = nil task := addQueryChannelTask{ req: genAddQueryChanelRequest(), @@ -114,23 +109,24 @@ func TestTask_AddQueryChannel(t *testing.T) { assert.Error(t, err) }) - t.Run("test execute add query collection failed", func(t *testing.T) { - node, err := genSimpleQueryNode(ctx) - assert.NoError(t, err) + /* + t.Run("test execute add query collection failed", func(t *testing.T) { + node, err := genSimpleQueryNode(ctx) + assert.NoError(t, err) - err = node.streaming.replica.removeCollection(defaultCollectionID) - assert.NoError(t, err) - err = node.historical.replica.removeCollection(defaultCollectionID) - assert.NoError(t, err) + err = node.streaming.replica.removeCollection(defaultCollectionID) + assert.NoError(t, err) + err = node.historical.replica.removeCollection(defaultCollectionID) + assert.NoError(t, err) - task := addQueryChannelTask{ - req: genAddQueryChanelRequest(), - node: node, - } + task := addQueryChannelTask{ + req: genAddQueryChanelRequest(), + node: node, + } - err = task.Execute(ctx) - assert.Error(t, err) - }) + err = task.Execute(ctx) + assert.Error(t, err) + })*/ t.Run("test execute init global sealed segments", func(t *testing.T) { node, err := genSimpleQueryNode(ctx) @@ -169,28 +165,31 @@ func TestTask_AddQueryChannel(t *testing.T) { err = task.Execute(ctx) assert.NoError(t, err) }) + /* + t.Run("test execute seek error", func(t *testing.T) { + node, err := genSimpleQueryNode(ctx) + assert.NoError(t, err) - t.Run("test execute seek error", func(t *testing.T) { - node, err := genSimpleQueryNode(ctx) - assert.NoError(t, err) + t.Run("test execute seek error", func(t *testing.T) { + node, err := genSimpleQueryNode(ctx) + assert.NoError(t, err) - position := &internalpb.MsgPosition{ - ChannelName: genQueryChannel(), - MsgID: []byte{1, 2, 3, 4, 5, 6, 7, 8}, - MsgGroup: defaultSubName, - Timestamp: 0, - } + position := &internalpb.MsgPosition{ + ChannelName: genQueryChannel(), + MsgID: []byte{1, 2, 3, 4, 5, 6, 7, 8}, + MsgGroup: defaultSubName, + Timestamp: 0, + } + task := addQueryChannelTask{ + req: genAddQueryChanelRequest(), + node: node, + } - task := addQueryChannelTask{ - req: genAddQueryChanelRequest(), - node: node, - } + task.req.SeekPosition = position - task.req.SeekPosition = position - - err = task.Execute(ctx) - assert.Error(t, err) - }) + err = task.Execute(ctx) + assert.Error(t, err) + })*/ } func TestTask_watchDmChannelsTask(t *testing.T) { @@ -687,8 +686,9 @@ func TestTask_releaseCollectionTask(t *testing.T) { node, err := genSimpleQueryNode(ctx) assert.NoError(t, err) - err = node.queryService.addQueryCollection(defaultCollectionID) - assert.NoError(t, err) + /* + err = node.queryService.addQueryCollection(defaultCollectionID) + assert.NoError(t, err)*/ task := releaseCollectionTask{ req: genReleaseCollectionRequest(), @@ -719,8 +719,9 @@ func TestTask_releaseCollectionTask(t *testing.T) { node, err := genSimpleQueryNode(ctx) assert.NoError(t, err) - err = node.queryService.addQueryCollection(defaultCollectionID) - assert.NoError(t, err) + /* + err = node.queryService.addQueryCollection(defaultCollectionID) + assert.NoError(t, err)*/ col, err := node.historical.replica.getCollectionByID(defaultCollectionID) assert.NoError(t, err) @@ -776,8 +777,9 @@ func TestTask_releasePartitionTask(t *testing.T) { node, err := genSimpleQueryNode(ctx) assert.NoError(t, err) - err = node.queryService.addQueryCollection(defaultCollectionID) - assert.NoError(t, err) + /* + err = node.queryService.addQueryCollection(defaultCollectionID) + assert.NoError(t, err)*/ task := releasePartitionsTask{ req: genReleasePartitionsRequest(), @@ -820,8 +822,9 @@ func TestTask_releasePartitionTask(t *testing.T) { col.addVDeltaChannels([]Channel{defaultDeltaChannel}) col.setLoadType(loadTypePartition) - err = node.queryService.addQueryCollection(defaultCollectionID) - assert.NoError(t, err) + /* + err = node.queryService.addQueryCollection(defaultCollectionID) + assert.NoError(t, err)*/ task := releasePartitionsTask{ req: genReleasePartitionsRequest(),