milvus-io
/
milvus
mirror of https://github.com/milvus-io/milvus.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Warp cpp reduce and repack searchResultMsg to specified pulsar channel 

Signed-off-by: xige-16 <xi.ge@zilliz.com>
pull/4973/head^2
xige-16 2020-11-26 11:12:33 +08:00 committed by yefu.chen
parent ae26a81550
commit ae78aa1ac1
34 changed files with 1678 additions and 1407 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
configs/advanced/master.yaml
View File

@ -20,4 +20,7 @@ master:
minIDAssignCnt: 1024
maxIDAssignCnt: 16384
# old name: segmentExpireDuration: 2000
IDAssignExpiration: 2000 # ms
IDAssignExpiration: 2000 # ms
maxPartitionNum: 4096
defaultPartitionTag: _default

1
configs/advanced/proxy.yaml
View File

@ -28,3 +28,4 @@ proxy:
bufSize: 512
maxNameLength: 255
maxFieldNum: 64

582
internal/core/src/pb/service_msg.pb.cc
View File

@ -19,10 +19,8 @@ extern PROTOBUF_INTERNAL_EXPORT_common_2eproto ::PROTOBUF_NAMESPACE_ID::internal
extern PROTOBUF_INTERNAL_EXPORT_common_2eproto ::PROTOBUF_NAMESPACE_ID::internal::SCCInfo<0> scc_info_KeyValuePair_common_2eproto;
extern PROTOBUF_INTERNAL_EXPORT_common_2eproto ::PROTOBUF_NAMESPACE_ID::internal::SCCInfo<0> scc_info_Status_common_2eproto;
extern PROTOBUF_INTERNAL_EXPORT_schema_2eproto ::PROTOBUF_NAMESPACE_ID::internal::SCCInfo<1> scc_info_CollectionSchema_schema_2eproto;
extern PROTOBUF_INTERNAL_EXPORT_service_5fmsg_2eproto ::PROTOBUF_NAMESPACE_ID::internal::SCCInfo<2> scc_info_Hits_service_5fmsg_2eproto;
extern PROTOBUF_INTERNAL_EXPORT_service_5fmsg_2eproto ::PROTOBUF_NAMESPACE_ID::internal::SCCInfo<0> scc_info_PartitionName_service_5fmsg_2eproto;
extern PROTOBUF_INTERNAL_EXPORT_service_5fmsg_2eproto ::PROTOBUF_NAMESPACE_ID::internal::SCCInfo<0> scc_info_PlaceholderValue_service_5fmsg_2eproto;
extern PROTOBUF_INTERNAL_EXPORT_service_5fmsg_2eproto ::PROTOBUF_NAMESPACE_ID::internal::SCCInfo<0> scc_info_Score_service_5fmsg_2eproto;
namespace milvus {
namespace proto {
namespace service {
@ -78,10 +76,6 @@ class PartitionDescriptionDefaultTypeInternal {
public:
::PROTOBUF_NAMESPACE_ID::internal::ExplicitlyConstructed<PartitionDescription> _instance;
} _PartitionDescription_default_instance_;
class ScoreDefaultTypeInternal {
public:
::PROTOBUF_NAMESPACE_ID::internal::ExplicitlyConstructed<Score> _instance;
} _Score_default_instance_;
class HitsDefaultTypeInternal {
public:
::PROTOBUF_NAMESPACE_ID::internal::ExplicitlyConstructed<Hits> _instance;
@ -150,10 +144,8 @@ static void InitDefaultsscc_info_Hits_service_5fmsg_2eproto() {
::milvus::proto::service::Hits::InitAsDefaultInstance();
}
::PROTOBUF_NAMESPACE_ID::internal::SCCInfo<2> scc_info_Hits_service_5fmsg_2eproto =
{{ATOMIC_VAR_INIT(::PROTOBUF_NAMESPACE_ID::internal::SCCInfoBase::kUninitialized), 2, InitDefaultsscc_info_Hits_service_5fmsg_2eproto}, {
&scc_info_Blob_common_2eproto.base,
&scc_info_Score_service_5fmsg_2eproto.base,}};
::PROTOBUF_NAMESPACE_ID::internal::SCCInfo<0> scc_info_Hits_service_5fmsg_2eproto =
{{ATOMIC_VAR_INIT(::PROTOBUF_NAMESPACE_ID::internal::SCCInfoBase::kUninitialized), 0, InitDefaultsscc_info_Hits_service_5fmsg_2eproto}, {}};
static void InitDefaultsscc_info_IntegerListResponse_service_5fmsg_2eproto() {
GOOGLE_PROTOBUF_VERIFY_VERSION;
@ -270,10 +262,9 @@ static void InitDefaultsscc_info_QueryResult_service_5fmsg_2eproto() {
::milvus::proto::service::QueryResult::InitAsDefaultInstance();
}
::PROTOBUF_NAMESPACE_ID::internal::SCCInfo<2> scc_info_QueryResult_service_5fmsg_2eproto =
{{ATOMIC_VAR_INIT(::PROTOBUF_NAMESPACE_ID::internal::SCCInfoBase::kUninitialized), 2, InitDefaultsscc_info_QueryResult_service_5fmsg_2eproto}, {
&scc_info_Status_common_2eproto.base,
&scc_info_Hits_service_5fmsg_2eproto.base,}};
::PROTOBUF_NAMESPACE_ID::internal::SCCInfo<1> scc_info_QueryResult_service_5fmsg_2eproto =
{{ATOMIC_VAR_INIT(::PROTOBUF_NAMESPACE_ID::internal::SCCInfoBase::kUninitialized), 1, InitDefaultsscc_info_QueryResult_service_5fmsg_2eproto}, {
&scc_info_Status_common_2eproto.base,}};
static void InitDefaultsscc_info_RowBatch_service_5fmsg_2eproto() {
GOOGLE_PROTOBUF_VERIFY_VERSION;
@ -290,20 +281,6 @@ static void InitDefaultsscc_info_RowBatch_service_5fmsg_2eproto() {
{{ATOMIC_VAR_INIT(::PROTOBUF_NAMESPACE_ID::internal::SCCInfoBase::kUninitialized), 1, InitDefaultsscc_info_RowBatch_service_5fmsg_2eproto}, {
&scc_info_Blob_common_2eproto.base,}};
static void InitDefaultsscc_info_Score_service_5fmsg_2eproto() {
GOOGLE_PROTOBUF_VERIFY_VERSION;
{
void* ptr = &::milvus::proto::service::_Score_default_instance_;
new (ptr) ::milvus::proto::service::Score();
::PROTOBUF_NAMESPACE_ID::internal::OnShutdownDestroyMessage(ptr);
}
::milvus::proto::service::Score::InitAsDefaultInstance();
}
::PROTOBUF_NAMESPACE_ID::internal::SCCInfo<0> scc_info_Score_service_5fmsg_2eproto =
{{ATOMIC_VAR_INIT(::PROTOBUF_NAMESPACE_ID::internal::SCCInfoBase::kUninitialized), 0, InitDefaultsscc_info_Score_service_5fmsg_2eproto}, {}};
static void InitDefaultsscc_info_StringListResponse_service_5fmsg_2eproto() {
GOOGLE_PROTOBUF_VERIFY_VERSION;
@ -334,7 +311,7 @@ static void InitDefaultsscc_info_StringResponse_service_5fmsg_2eproto() {
{{ATOMIC_VAR_INIT(::PROTOBUF_NAMESPACE_ID::internal::SCCInfoBase::kUninitialized), 1, InitDefaultsscc_info_StringResponse_service_5fmsg_2eproto}, {
&scc_info_Status_common_2eproto.base,}};
static ::PROTOBUF_NAMESPACE_ID::Metadata file_level_metadata_service_5fmsg_2eproto[16];
static ::PROTOBUF_NAMESPACE_ID::Metadata file_level_metadata_service_5fmsg_2eproto[15];
static const ::PROTOBUF_NAMESPACE_ID::EnumDescriptor* file_level_enum_descriptors_service_5fmsg_2eproto[1];
static constexpr ::PROTOBUF_NAMESPACE_ID::ServiceDescriptor const** file_level_service_descriptors_service_5fmsg_2eproto = nullptr;
@ -437,13 +414,6 @@ const ::PROTOBUF_NAMESPACE_ID::uint32 TableStruct_service_5fmsg_2eproto::offsets
PROTOBUF_FIELD_OFFSET(::milvus::proto::service::PartitionDescription, name_),
PROTOBUF_FIELD_OFFSET(::milvus::proto::service::PartitionDescription, statistics_),
~0u, // no _has_bits_
PROTOBUF_FIELD_OFFSET(::milvus::proto::service::Score, _internal_metadata_),
~0u, // no _extensions_
~0u, // no _oneof_case_
~0u, // no _weak_field_map_
PROTOBUF_FIELD_OFFSET(::milvus::proto::service::Score, tag_),
PROTOBUF_FIELD_OFFSET(::milvus::proto::service::Score, values_),
~0u, // no _has_bits_
PROTOBUF_FIELD_OFFSET(::milvus::proto::service::Hits, _internal_metadata_),
~0u, // no _extensions_
~0u, // no _oneof_case_
@ -473,9 +443,8 @@ static const ::PROTOBUF_NAMESPACE_ID::internal::MigrationSchema schemas[] PROTOB
{ 73, -1, sizeof(::milvus::proto::service::IntegerRangeResponse)},
{ 81, -1, sizeof(::milvus::proto::service::CollectionDescription)},
{ 89, -1, sizeof(::milvus::proto::service::PartitionDescription)},
{ 97, -1, sizeof(::milvus::proto::service::Score)},
{ 104, -1, sizeof(::milvus::proto::service::Hits)},
{ 112, -1, sizeof(::milvus::proto::service::QueryResult)},
{ 97, -1, sizeof(::milvus::proto::service::Hits)},
{ 105, -1, sizeof(::milvus::proto::service::QueryResult)},
};
static ::PROTOBUF_NAMESPACE_ID::Message const * const file_default_instances[] = {
@ -492,7 +461,6 @@ static ::PROTOBUF_NAMESPACE_ID::Message const * const file_default_instances[] =
reinterpret_cast<const ::PROTOBUF_NAMESPACE_ID::Message*>(&::milvus::proto::service::_IntegerRangeResponse_default_instance_),
reinterpret_cast<const ::PROTOBUF_NAMESPACE_ID::Message*>(&::milvus::proto::service::_CollectionDescription_default_instance_),
reinterpret_cast<const ::PROTOBUF_NAMESPACE_ID::Message*>(&::milvus::proto::service::_PartitionDescription_default_instance_),
reinterpret_cast<const ::PROTOBUF_NAMESPACE_ID::Message*>(&::milvus::proto::service::_Score_default_instance_),
reinterpret_cast<const ::PROTOBUF_NAMESPACE_ID::Message*>(&::milvus::proto::service::_Hits_default_instance_),
reinterpret_cast<const ::PROTOBUF_NAMESPACE_ID::Message*>(&::milvus::proto::service::_QueryResult_default_instance_),
};
@ -531,23 +499,20 @@ const char descriptor_table_protodef_service_5fmsg_2eproto[] PROTOBUF_SECTION_VA
"\006status\030\001 \001(\0132\033.milvus.proto.common.Stat"
"us\0221\n\004name\030\002 \001(\0132#.milvus.proto.service."
"PartitionName\0225\n\nstatistics\030\003 \003(\0132!.milv"
"us.proto.common.KeyValuePair\"$\n\005Score\022\013\n"
"\003tag\030\001 \001(\t\022\016\n\006values\030\002 \003(\002\"m\n\004Hits\022\013\n\003ID"
"s\030\001 \003(\003\022+\n\010row_data\030\002 \003(\0132\031.milvus.proto"
".common.Blob\022+\n\006scores\030\003 \003(\0132\033.milvus.pr"
"oto.service.Score\"d\n\013QueryResult\022+\n\006stat"
"us\030\001 \001(\0132\033.milvus.proto.common.Status\022(\n"
"\004hits\030\002 \003(\0132\032.milvus.proto.service.Hits*"
"@\n\017PlaceholderType\022\010\n\004NONE\020\000\022\021\n\rVECTOR_B"
"INARY\020d\022\020\n\014VECTOR_FLOAT\020eBCZAgithub.com/"
"zilliztech/milvus-distributed/internal/p"
"roto/servicepbb\006proto3"
"us.proto.common.KeyValuePair\"5\n\004Hits\022\013\n\003"
"IDs\030\001 \003(\003\022\020\n\010row_data\030\002 \003(\014\022\016\n\006scores\030\003 "
"\003(\002\"H\n\013QueryResult\022+\n\006status\030\001 \001(\0132\033.mil"
"vus.proto.common.Status\022\014\n\004hits\030\002 \003(\014*@\n"
"\017PlaceholderType\022\010\n\004NONE\020\000\022\021\n\rVECTOR_BIN"
"ARY\020d\022\020\n\014VECTOR_FLOAT\020eBCZAgithub.com/zi"
"lliztech/milvus-distributed/internal/pro"
"to/servicepbb\006proto3"
;
static const ::PROTOBUF_NAMESPACE_ID::internal::DescriptorTable*const descriptor_table_service_5fmsg_2eproto_deps[2] = {
&::descriptor_table_common_2eproto,
&::descriptor_table_schema_2eproto,
};
static ::PROTOBUF_NAMESPACE_ID::internal::SCCInfoBase*const descriptor_table_service_5fmsg_2eproto_sccs[16] = {
static ::PROTOBUF_NAMESPACE_ID::internal::SCCInfoBase*const descriptor_table_service_5fmsg_2eproto_sccs[15] = {
&scc_info_BoolResponse_service_5fmsg_2eproto.base,
&scc_info_CollectionDescription_service_5fmsg_2eproto.base,
&scc_info_CollectionName_service_5fmsg_2eproto.base,
@ -561,17 +526,16 @@ static ::PROTOBUF_NAMESPACE_ID::internal::SCCInfoBase*const descriptor_table_ser
&scc_info_Query_service_5fmsg_2eproto.base,
&scc_info_QueryResult_service_5fmsg_2eproto.base,
&scc_info_RowBatch_service_5fmsg_2eproto.base,
&scc_info_Score_service_5fmsg_2eproto.base,
&scc_info_StringListResponse_service_5fmsg_2eproto.base,
&scc_info_StringResponse_service_5fmsg_2eproto.base,
};
static ::PROTOBUF_NAMESPACE_ID::internal::once_flag descriptor_table_service_5fmsg_2eproto_once;
static bool descriptor_table_service_5fmsg_2eproto_initialized = false;
const ::PROTOBUF_NAMESPACE_ID::internal::DescriptorTable descriptor_table_service_5fmsg_2eproto = {
&descriptor_table_service_5fmsg_2eproto_initialized, descriptor_table_protodef_service_5fmsg_2eproto, "service_msg.proto", 1742,
&descriptor_table_service_5fmsg_2eproto_once, descriptor_table_service_5fmsg_2eproto_sccs, descriptor_table_service_5fmsg_2eproto_deps, 16, 2,
&descriptor_table_service_5fmsg_2eproto_initialized, descriptor_table_protodef_service_5fmsg_2eproto, "service_msg.proto", 1620,
&descriptor_table_service_5fmsg_2eproto_once, descriptor_table_service_5fmsg_2eproto_sccs, descriptor_table_service_5fmsg_2eproto_deps, 15, 2,
schemas, file_default_instances, TableStruct_service_5fmsg_2eproto::offsets,
file_level_metadata_service_5fmsg_2eproto, 16, file_level_enum_descriptors_service_5fmsg_2eproto, file_level_service_descriptors_service_5fmsg_2eproto,
file_level_metadata_service_5fmsg_2eproto, 15, file_level_enum_descriptors_service_5fmsg_2eproto, file_level_service_descriptors_service_5fmsg_2eproto,
};
// Force running AddDescriptors() at dynamic initialization time.
@ -5195,342 +5159,6 @@ void PartitionDescription::InternalSwap(PartitionDescription* other) {
}
// ===================================================================
void Score::InitAsDefaultInstance() {
}
class Score::_Internal {
public:
};
Score::Score()
: ::PROTOBUF_NAMESPACE_ID::Message(), _internal_metadata_(nullptr) {
SharedCtor();
// @@protoc_insertion_point(constructor:milvus.proto.service.Score)
}
Score::Score(const Score& from)
: ::PROTOBUF_NAMESPACE_ID::Message(),
_internal_metadata_(nullptr),
values_(from.values_) {
_internal_metadata_.MergeFrom(from._internal_metadata_);
tag_.UnsafeSetDefault(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited());
if (!from.tag().empty()) {
tag_.AssignWithDefault(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), from.tag_);
}
// @@protoc_insertion_point(copy_constructor:milvus.proto.service.Score)
}
void Score::SharedCtor() {
::PROTOBUF_NAMESPACE_ID::internal::InitSCC(&scc_info_Score_service_5fmsg_2eproto.base);
tag_.UnsafeSetDefault(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited());
}
Score::~Score() {
// @@protoc_insertion_point(destructor:milvus.proto.service.Score)
SharedDtor();
}
void Score::SharedDtor() {
tag_.DestroyNoArena(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited());
}
void Score::SetCachedSize(int size) const {
_cached_size_.Set(size);
}
const Score& Score::default_instance() {
::PROTOBUF_NAMESPACE_ID::internal::InitSCC(&::scc_info_Score_service_5fmsg_2eproto.base);
return *internal_default_instance();
}
void Score::Clear() {
// @@protoc_insertion_point(message_clear_start:milvus.proto.service.Score)
::PROTOBUF_NAMESPACE_ID::uint32 cached_has_bits = 0;
// Prevent compiler warnings about cached_has_bits being unused
(void) cached_has_bits;
values_.Clear();
tag_.ClearToEmptyNoArena(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited());
_internal_metadata_.Clear();
}
#if GOOGLE_PROTOBUF_ENABLE_EXPERIMENTAL_PARSER
const char* Score::_InternalParse(const char* ptr, ::PROTOBUF_NAMESPACE_ID::internal::ParseContext* ctx) {
#define CHK_(x) if (PROTOBUF_PREDICT_FALSE(!(x))) goto failure
while (!ctx->Done(&ptr)) {
::PROTOBUF_NAMESPACE_ID::uint32 tag;
ptr = ::PROTOBUF_NAMESPACE_ID::internal::ReadTag(ptr, &tag);
CHK_(ptr);
switch (tag >> 3) {
// string tag = 1;
case 1:
if (PROTOBUF_PREDICT_TRUE(static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 10)) {
ptr = ::PROTOBUF_NAMESPACE_ID::internal::InlineGreedyStringParserUTF8(mutable_tag(), ptr, ctx, "milvus.proto.service.Score.tag");
CHK_(ptr);
} else goto handle_unusual;
continue;
// repeated float values = 2;
case 2:
if (PROTOBUF_PREDICT_TRUE(static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 18)) {
ptr = ::PROTOBUF_NAMESPACE_ID::internal::PackedFloatParser(mutable_values(), ptr, ctx);
CHK_(ptr);
} else if (static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 21) {
add_values(::PROTOBUF_NAMESPACE_ID::internal::UnalignedLoad<float>(ptr));
ptr += sizeof(float);
} else goto handle_unusual;
continue;
default: {
handle_unusual:
if ((tag & 7) == 4 || tag == 0) {
ctx->SetLastTag(tag);
goto success;
}
ptr = UnknownFieldParse(tag, &_internal_metadata_, ptr, ctx);
CHK_(ptr != nullptr);
continue;
}
} // switch
} // while
success:
return ptr;
failure:
ptr = nullptr;
goto success;
#undef CHK_
}
#else // GOOGLE_PROTOBUF_ENABLE_EXPERIMENTAL_PARSER
bool Score::MergePartialFromCodedStream(
::PROTOBUF_NAMESPACE_ID::io::CodedInputStream* input) {
#define DO_(EXPRESSION) if (!PROTOBUF_PREDICT_TRUE(EXPRESSION)) goto failure
::PROTOBUF_NAMESPACE_ID::uint32 tag;
// @@protoc_insertion_point(parse_start:milvus.proto.service.Score)
for (;;) {
::std::pair<::PROTOBUF_NAMESPACE_ID::uint32, bool> p = input->ReadTagWithCutoffNoLastTag(127u);
tag = p.first;
if (!p.second) goto handle_unusual;
switch (::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::GetTagFieldNumber(tag)) {
// string tag = 1;
case 1: {
if (static_cast< ::PROTOBUF_NAMESPACE_ID::uint8>(tag) == (10 & 0xFF)) {
DO_(::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::ReadString(
input, this->mutable_tag()));
DO_(::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::VerifyUtf8String(
this->tag().data(), static_cast<int>(this->tag().length()),
::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::PARSE,
"milvus.proto.service.Score.tag"));
} else {
goto handle_unusual;
}
break;
}
// repeated float values = 2;
case 2: {
if (static_cast< ::PROTOBUF_NAMESPACE_ID::uint8>(tag) == (18 & 0xFF)) {
DO_((::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::ReadPackedPrimitive<
float, ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::TYPE_FLOAT>(
input, this->mutable_values())));
} else if (static_cast< ::PROTOBUF_NAMESPACE_ID::uint8>(tag) == (21 & 0xFF)) {
DO_((::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::ReadRepeatedPrimitiveNoInline<
float, ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::TYPE_FLOAT>(
1, 18u, input, this->mutable_values())));
} else {
goto handle_unusual;
}
break;
}
default: {
handle_unusual:
if (tag == 0) {
goto success;
}
DO_(::PROTOBUF_NAMESPACE_ID::internal::WireFormat::SkipField(
input, tag, _internal_metadata_.mutable_unknown_fields()));
break;
}
}
}
success:
// @@protoc_insertion_point(parse_success:milvus.proto.service.Score)
return true;
failure:
// @@protoc_insertion_point(parse_failure:milvus.proto.service.Score)
return false;
#undef DO_
}
#endif // GOOGLE_PROTOBUF_ENABLE_EXPERIMENTAL_PARSER
void Score::SerializeWithCachedSizes(
::PROTOBUF_NAMESPACE_ID::io::CodedOutputStream* output) const {
// @@protoc_insertion_point(serialize_start:milvus.proto.service.Score)
::PROTOBUF_NAMESPACE_ID::uint32 cached_has_bits = 0;
(void) cached_has_bits;
// string tag = 1;
if (this->tag().size() > 0) {
::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::VerifyUtf8String(
this->tag().data(), static_cast<int>(this->tag().length()),
::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::SERIALIZE,
"milvus.proto.service.Score.tag");
::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteStringMaybeAliased(
1, this->tag(), output);
}
// repeated float values = 2;
if (this->values_size() > 0) {
::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteTag(2, ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WIRETYPE_LENGTH_DELIMITED, output);
output->WriteVarint32(_values_cached_byte_size_.load(
std::memory_order_relaxed));
::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteFloatArray(
this->values().data(), this->values_size(), output);
}
if (_internal_metadata_.have_unknown_fields()) {
::PROTOBUF_NAMESPACE_ID::internal::WireFormat::SerializeUnknownFields(
_internal_metadata_.unknown_fields(), output);
}
// @@protoc_insertion_point(serialize_end:milvus.proto.service.Score)
}
::PROTOBUF_NAMESPACE_ID::uint8* Score::InternalSerializeWithCachedSizesToArray(
::PROTOBUF_NAMESPACE_ID::uint8* target) const {
// @@protoc_insertion_point(serialize_to_array_start:milvus.proto.service.Score)
::PROTOBUF_NAMESPACE_ID::uint32 cached_has_bits = 0;
(void) cached_has_bits;
// string tag = 1;
if (this->tag().size() > 0) {
::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::VerifyUtf8String(
this->tag().data(), static_cast<int>(this->tag().length()),
::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::SERIALIZE,
"milvus.proto.service.Score.tag");
target =
::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteStringToArray(
1, this->tag(), target);
}
// repeated float values = 2;
if (this->values_size() > 0) {
target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteTagToArray(
2,
::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WIRETYPE_LENGTH_DELIMITED,
target);
target = ::PROTOBUF_NAMESPACE_ID::io::CodedOutputStream::WriteVarint32ToArray(
_values_cached_byte_size_.load(std::memory_order_relaxed),
target);
target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::
WriteFloatNoTagToArray(this->values_, target);
}
if (_internal_metadata_.have_unknown_fields()) {
target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormat::SerializeUnknownFieldsToArray(
_internal_metadata_.unknown_fields(), target);
}
// @@protoc_insertion_point(serialize_to_array_end:milvus.proto.service.Score)
return target;
}
size_t Score::ByteSizeLong() const {
// @@protoc_insertion_point(message_byte_size_start:milvus.proto.service.Score)
size_t total_size = 0;
if (_internal_metadata_.have_unknown_fields()) {
total_size +=
::PROTOBUF_NAMESPACE_ID::internal::WireFormat::ComputeUnknownFieldsSize(
_internal_metadata_.unknown_fields());
}
::PROTOBUF_NAMESPACE_ID::uint32 cached_has_bits = 0;
// Prevent compiler warnings about cached_has_bits being unused
(void) cached_has_bits;
// repeated float values = 2;
{
unsigned int count = static_cast<unsigned int>(this->values_size());
size_t data_size = 4UL * count;
if (data_size > 0) {
total_size += 1 +
::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::Int32Size(
static_cast<::PROTOBUF_NAMESPACE_ID::int32>(data_size));
}
int cached_size = ::PROTOBUF_NAMESPACE_ID::internal::ToCachedSize(data_size);
_values_cached_byte_size_.store(cached_size,
std::memory_order_relaxed);
total_size += data_size;
}
// string tag = 1;
if (this->tag().size() > 0) {
total_size += 1 +
::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::StringSize(
this->tag());
}
int cached_size = ::PROTOBUF_NAMESPACE_ID::internal::ToCachedSize(total_size);
SetCachedSize(cached_size);
return total_size;
}
void Score::MergeFrom(const ::PROTOBUF_NAMESPACE_ID::Message& from) {
// @@protoc_insertion_point(generalized_merge_from_start:milvus.proto.service.Score)
GOOGLE_DCHECK_NE(&from, this);
const Score* source =
::PROTOBUF_NAMESPACE_ID::DynamicCastToGenerated<Score>(
&from);
if (source == nullptr) {
// @@protoc_insertion_point(generalized_merge_from_cast_fail:milvus.proto.service.Score)
::PROTOBUF_NAMESPACE_ID::internal::ReflectionOps::Merge(from, this);
} else {
// @@protoc_insertion_point(generalized_merge_from_cast_success:milvus.proto.service.Score)
MergeFrom(*source);
}
}
void Score::MergeFrom(const Score& from) {
// @@protoc_insertion_point(class_specific_merge_from_start:milvus.proto.service.Score)
GOOGLE_DCHECK_NE(&from, this);
_internal_metadata_.MergeFrom(from._internal_metadata_);
::PROTOBUF_NAMESPACE_ID::uint32 cached_has_bits = 0;
(void) cached_has_bits;
values_.MergeFrom(from.values_);
if (from.tag().size() > 0) {
tag_.AssignWithDefault(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), from.tag_);
}
}
void Score::CopyFrom(const ::PROTOBUF_NAMESPACE_ID::Message& from) {
// @@protoc_insertion_point(generalized_copy_from_start:milvus.proto.service.Score)
if (&from == this) return;
Clear();
MergeFrom(from);
}
void Score::CopyFrom(const Score& from) {
// @@protoc_insertion_point(class_specific_copy_from_start:milvus.proto.service.Score)
if (&from == this) return;
Clear();
MergeFrom(from);
}
bool Score::IsInitialized() const {
return true;
}
void Score::InternalSwap(Score* other) {
using std::swap;
_internal_metadata_.Swap(&other->_internal_metadata_);
values_.InternalSwap(&other->values_);
tag_.Swap(&other->tag_, &::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(),
GetArenaNoVirtual());
}
::PROTOBUF_NAMESPACE_ID::Metadata Score::GetMetadata() const {
return GetMetadataStatic();
}
// ===================================================================
void Hits::InitAsDefaultInstance() {
@ -5539,9 +5167,6 @@ class Hits::_Internal {
public:
};
void Hits::clear_row_data() {
row_data_.Clear();
}
Hits::Hits()
: ::PROTOBUF_NAMESPACE_ID::Message(), _internal_metadata_(nullptr) {
SharedCtor();
@ -5608,28 +5233,26 @@ const char* Hits::_InternalParse(const char* ptr, ::PROTOBUF_NAMESPACE_ID::inter
CHK_(ptr);
} else goto handle_unusual;
continue;
// repeated .milvus.proto.common.Blob row_data = 2;
// repeated bytes row_data = 2;
case 2:
if (PROTOBUF_PREDICT_TRUE(static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 18)) {
ptr -= 1;
do {
ptr += 1;
ptr = ctx->ParseMessage(add_row_data(), ptr);
ptr = ::PROTOBUF_NAMESPACE_ID::internal::InlineGreedyStringParser(add_row_data(), ptr, ctx);
CHK_(ptr);
if (!ctx->DataAvailable(ptr)) break;
} while (::PROTOBUF_NAMESPACE_ID::internal::UnalignedLoad<::PROTOBUF_NAMESPACE_ID::uint8>(ptr) == 18);
} else goto handle_unusual;
continue;
// repeated .milvus.proto.service.Score scores = 3;
// repeated float scores = 3;
case 3:
if (PROTOBUF_PREDICT_TRUE(static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 26)) {
ptr -= 1;
do {
ptr += 1;
ptr = ctx->ParseMessage(add_scores(), ptr);
CHK_(ptr);
if (!ctx->DataAvailable(ptr)) break;
} while (::PROTOBUF_NAMESPACE_ID::internal::UnalignedLoad<::PROTOBUF_NAMESPACE_ID::uint8>(ptr) == 26);
ptr = ::PROTOBUF_NAMESPACE_ID::internal::PackedFloatParser(mutable_scores(), ptr, ctx);
CHK_(ptr);
} else if (static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 29) {
add_scores(::PROTOBUF_NAMESPACE_ID::internal::UnalignedLoad<float>(ptr));
ptr += sizeof(float);
} else goto handle_unusual;
continue;
default: {
@ -5678,22 +5301,27 @@ bool Hits::MergePartialFromCodedStream(
break;
}
// repeated .milvus.proto.common.Blob row_data = 2;
// repeated bytes row_data = 2;
case 2: {
if (static_cast< ::PROTOBUF_NAMESPACE_ID::uint8>(tag) == (18 & 0xFF)) {
DO_(::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::ReadMessage(
input, add_row_data()));
DO_(::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::ReadBytes(
input, this->add_row_data()));
} else {
goto handle_unusual;
}
break;
}
// repeated .milvus.proto.service.Score scores = 3;
// repeated float scores = 3;
case 3: {
if (static_cast< ::PROTOBUF_NAMESPACE_ID::uint8>(tag) == (26 & 0xFF)) {
DO_(::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::ReadMessage(
input, add_scores()));
DO_((::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::ReadPackedPrimitive<
float, ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::TYPE_FLOAT>(
input, this->mutable_scores())));
} else if (static_cast< ::PROTOBUF_NAMESPACE_ID::uint8>(tag) == (29 & 0xFF)) {
DO_((::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::ReadRepeatedPrimitiveNoInline<
float, ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::TYPE_FLOAT>(
1, 26u, input, this->mutable_scores())));
} else {
goto handle_unusual;
}
@ -5738,22 +5366,19 @@ void Hits::SerializeWithCachedSizes(
this->ids(i), output);
}
// repeated .milvus.proto.common.Blob row_data = 2;
for (unsigned int i = 0,
n = static_cast<unsigned int>(this->row_data_size()); i < n; i++) {
::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteMessageMaybeToArray(
2,
this->row_data(static_cast<int>(i)),
output);
// repeated bytes row_data = 2;
for (int i = 0, n = this->row_data_size(); i < n; i++) {
::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteBytes(
2, this->row_data(i), output);
}
// repeated .milvus.proto.service.Score scores = 3;
for (unsigned int i = 0,
n = static_cast<unsigned int>(this->scores_size()); i < n; i++) {
::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteMessageMaybeToArray(
3,
this->scores(static_cast<int>(i)),
output);
// repeated float scores = 3;
if (this->scores_size() > 0) {
::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteTag(3, ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WIRETYPE_LENGTH_DELIMITED, output);
output->WriteVarint32(_scores_cached_byte_size_.load(
std::memory_order_relaxed));
::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteFloatArray(
this->scores().data(), this->scores_size(), output);
}
if (_internal_metadata_.have_unknown_fields()) {
@ -5782,20 +5407,23 @@ void Hits::SerializeWithCachedSizes(
WriteInt64NoTagToArray(this->ids_, target);
}
// repeated .milvus.proto.common.Blob row_data = 2;
for (unsigned int i = 0,
n = static_cast<unsigned int>(this->row_data_size()); i < n; i++) {
// repeated bytes row_data = 2;
for (int i = 0, n = this->row_data_size(); i < n; i++) {
target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::
InternalWriteMessageToArray(
2, this->row_data(static_cast<int>(i)), target);
WriteBytesToArray(2, this->row_data(i), target);
}
// repeated .milvus.proto.service.Score scores = 3;
for (unsigned int i = 0,
n = static_cast<unsigned int>(this->scores_size()); i < n; i++) {
// repeated float scores = 3;
if (this->scores_size() > 0) {
target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteTagToArray(
3,
::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WIRETYPE_LENGTH_DELIMITED,
target);
target = ::PROTOBUF_NAMESPACE_ID::io::CodedOutputStream::WriteVarint32ToArray(
_scores_cached_byte_size_.load(std::memory_order_relaxed),
target);
target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::
InternalWriteMessageToArray(
3, this->scores(static_cast<int>(i)), target);
WriteFloatNoTagToArray(this->scores_, target);
}
if (_internal_metadata_.have_unknown_fields()) {
@ -5834,26 +5462,27 @@ size_t Hits::ByteSizeLong() const {
total_size += data_size;
}
// repeated .milvus.proto.common.Blob row_data = 2;
{
unsigned int count = static_cast<unsigned int>(this->row_data_size());
total_size += 1UL * count;
for (unsigned int i = 0; i < count; i++) {
total_size +=
::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::MessageSize(
this->row_data(static_cast<int>(i)));
}
// repeated bytes row_data = 2;
total_size += 1 *
::PROTOBUF_NAMESPACE_ID::internal::FromIntSize(this->row_data_size());
for (int i = 0, n = this->row_data_size(); i < n; i++) {
total_size += ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::BytesSize(
this->row_data(i));
}
// repeated .milvus.proto.service.Score scores = 3;
// repeated float scores = 3;
{
unsigned int count = static_cast<unsigned int>(this->scores_size());
total_size += 1UL * count;
for (unsigned int i = 0; i < count; i++) {
total_size +=
::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::MessageSize(
this->scores(static_cast<int>(i)));
size_t data_size = 4UL * count;
if (data_size > 0) {
total_size += 1 +
::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::Int32Size(
static_cast<::PROTOBUF_NAMESPACE_ID::int32>(data_size));
}
int cached_size = ::PROTOBUF_NAMESPACE_ID::internal::ToCachedSize(data_size);
_scores_cached_byte_size_.store(cached_size,
std::memory_order_relaxed);
total_size += data_size;
}
int cached_size = ::PROTOBUF_NAMESPACE_ID::internal::ToCachedSize(total_size);
@ -5910,8 +5539,8 @@ void Hits::InternalSwap(Hits* other) {
using std::swap;
_internal_metadata_.Swap(&other->_internal_metadata_);
ids_.InternalSwap(&other->ids_);
CastToBase(&row_data_)->InternalSwap(CastToBase(&other->row_data_));
CastToBase(&scores_)->InternalSwap(CastToBase(&other->scores_));
row_data_.InternalSwap(CastToBase(&other->row_data_));
scores_.InternalSwap(&other->scores_);
}
::PROTOBUF_NAMESPACE_ID::Metadata Hits::GetMetadata() const {
@ -6010,13 +5639,13 @@ const char* QueryResult::_InternalParse(const char* ptr, ::PROTOBUF_NAMESPACE_ID
CHK_(ptr);
} else goto handle_unusual;
continue;
// repeated .milvus.proto.service.Hits hits = 2;
// repeated bytes hits = 2;
case 2:
if (PROTOBUF_PREDICT_TRUE(static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 18)) {
ptr -= 1;
do {
ptr += 1;
ptr = ctx->ParseMessage(add_hits(), ptr);
ptr = ::PROTOBUF_NAMESPACE_ID::internal::InlineGreedyStringParser(add_hits(), ptr, ctx);
CHK_(ptr);
if (!ctx->DataAvailable(ptr)) break;
} while (::PROTOBUF_NAMESPACE_ID::internal::UnalignedLoad<::PROTOBUF_NAMESPACE_ID::uint8>(ptr) == 18);
@ -6063,11 +5692,11 @@ bool QueryResult::MergePartialFromCodedStream(
break;
}
// repeated .milvus.proto.service.Hits hits = 2;
// repeated bytes hits = 2;
case 2: {
if (static_cast< ::PROTOBUF_NAMESPACE_ID::uint8>(tag) == (18 & 0xFF)) {
DO_(::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::ReadMessage(
input, add_hits()));
DO_(::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::ReadBytes(
input, this->add_hits()));
} else {
goto handle_unusual;
}
@ -6107,13 +5736,10 @@ void QueryResult::SerializeWithCachedSizes(
1, _Internal::status(this), output);
}
// repeated .milvus.proto.service.Hits hits = 2;
for (unsigned int i = 0,
n = static_cast<unsigned int>(this->hits_size()); i < n; i++) {
::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteMessageMaybeToArray(
2,
this->hits(static_cast<int>(i)),
output);
// repeated bytes hits = 2;
for (int i = 0, n = this->hits_size(); i < n; i++) {
::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteBytes(
2, this->hits(i), output);
}
if (_internal_metadata_.have_unknown_fields()) {
@ -6136,12 +5762,10 @@ void QueryResult::SerializeWithCachedSizes(
1, _Internal::status(this), target);
}
// repeated .milvus.proto.service.Hits hits = 2;
for (unsigned int i = 0,
n = static_cast<unsigned int>(this->hits_size()); i < n; i++) {
// repeated bytes hits = 2;
for (int i = 0, n = this->hits_size(); i < n; i++) {
target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::
InternalWriteMessageToArray(
2, this->hits(static_cast<int>(i)), target);
WriteBytesToArray(2, this->hits(i), target);
}
if (_internal_metadata_.have_unknown_fields()) {
@ -6165,15 +5789,12 @@ size_t QueryResult::ByteSizeLong() const {
// Prevent compiler warnings about cached_has_bits being unused
(void) cached_has_bits;
// repeated .milvus.proto.service.Hits hits = 2;
{
unsigned int count = static_cast<unsigned int>(this->hits_size());
total_size += 1UL * count;
for (unsigned int i = 0; i < count; i++) {
total_size +=
::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::MessageSize(
this->hits(static_cast<int>(i)));
}
// repeated bytes hits = 2;
total_size += 1 *
::PROTOBUF_NAMESPACE_ID::internal::FromIntSize(this->hits_size());
for (int i = 0, n = this->hits_size(); i < n; i++) {
total_size += ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::BytesSize(
this->hits(i));
}
// .milvus.proto.common.Status status = 1;
@ -6237,7 +5858,7 @@ bool QueryResult::IsInitialized() const {
void QueryResult::InternalSwap(QueryResult* other) {
using std::swap;
_internal_metadata_.Swap(&other->_internal_metadata_);
CastToBase(&hits_)->InternalSwap(CastToBase(&other->hits_));
hits_.InternalSwap(CastToBase(&other->hits_));
swap(status_, other->status_);
}
@ -6290,9 +5911,6 @@ template<> PROTOBUF_NOINLINE ::milvus::proto::service::CollectionDescription* Ar
template<> PROTOBUF_NOINLINE ::milvus::proto::service::PartitionDescription* Arena::CreateMaybeMessage< ::milvus::proto::service::PartitionDescription >(Arena* arena) {
return Arena::CreateInternal< ::milvus::proto::service::PartitionDescription >(arena);
}
template<> PROTOBUF_NOINLINE ::milvus::proto::service::Score* Arena::CreateMaybeMessage< ::milvus::proto::service::Score >(Arena* arena) {
return Arena::CreateInternal< ::milvus::proto::service::Score >(arena);
}
template<> PROTOBUF_NOINLINE ::milvus::proto::service::Hits* Arena::CreateMaybeMessage< ::milvus::proto::service::Hits >(Arena* arena) {
return Arena::CreateInternal< ::milvus::proto::service::Hits >(arena);
}

470
internal/core/src/pb/service_msg.pb.h
View File

@ -50,7 +50,7 @@ struct TableStruct_service_5fmsg_2eproto {
PROTOBUF_SECTION_VARIABLE(protodesc_cold);
static const ::PROTOBUF_NAMESPACE_ID::internal::AuxillaryParseTableField aux[]
PROTOBUF_SECTION_VARIABLE(protodesc_cold);
static const ::PROTOBUF_NAMESPACE_ID::internal::ParseTable schema[16]
static const ::PROTOBUF_NAMESPACE_ID::internal::ParseTable schema[15]
PROTOBUF_SECTION_VARIABLE(protodesc_cold);
static const ::PROTOBUF_NAMESPACE_ID::internal::FieldMetadata field_metadata[];
static const ::PROTOBUF_NAMESPACE_ID::internal::SerializationTable serialization_table[];
@ -99,9 +99,6 @@ extern QueryResultDefaultTypeInternal _QueryResult_default_instance_;
class RowBatch;
class RowBatchDefaultTypeInternal;
extern RowBatchDefaultTypeInternal _RowBatch_default_instance_;
class Score;
class ScoreDefaultTypeInternal;
extern ScoreDefaultTypeInternal _Score_default_instance_;
class StringListResponse;
class StringListResponseDefaultTypeInternal;
extern StringListResponseDefaultTypeInternal _StringListResponse_default_instance_;
@ -125,7 +122,6 @@ template<> ::milvus::proto::service::PlaceholderValue* Arena::CreateMaybeMessage
template<> ::milvus::proto::service::Query* Arena::CreateMaybeMessage<::milvus::proto::service::Query>(Arena*);
template<> ::milvus::proto::service::QueryResult* Arena::CreateMaybeMessage<::milvus::proto::service::QueryResult>(Arena*);
template<> ::milvus::proto::service::RowBatch* Arena::CreateMaybeMessage<::milvus::proto::service::RowBatch>(Arena*);
template<> ::milvus::proto::service::Score* Arena::CreateMaybeMessage<::milvus::proto::service::Score>(Arena*);
template<> ::milvus::proto::service::StringListResponse* Arena::CreateMaybeMessage<::milvus::proto::service::StringListResponse>(Arena*);
template<> ::milvus::proto::service::StringResponse* Arena::CreateMaybeMessage<::milvus::proto::service::StringResponse>(Arena*);
PROTOBUF_NAMESPACE_CLOSE
@ -2158,157 +2154,6 @@ class PartitionDescription :
};
// -------------------------------------------------------------------
class Score :
public ::PROTOBUF_NAMESPACE_ID::Message /* @@protoc_insertion_point(class_definition:milvus.proto.service.Score) */ {
public:
Score();
virtual ~Score();
Score(const Score& from);
Score(Score&& from) noexcept
: Score() {
*this = ::std::move(from);
}
inline Score& operator=(const Score& from) {
CopyFrom(from);
return *this;
}
inline Score& operator=(Score&& from) noexcept {
if (GetArenaNoVirtual() == from.GetArenaNoVirtual()) {
if (this != &from) InternalSwap(&from);
} else {
CopyFrom(from);
}
return *this;
}
static const ::PROTOBUF_NAMESPACE_ID::Descriptor* descriptor() {
return GetDescriptor();
}
static const ::PROTOBUF_NAMESPACE_ID::Descriptor* GetDescriptor() {
return GetMetadataStatic().descriptor;
}
static const ::PROTOBUF_NAMESPACE_ID::Reflection* GetReflection() {
return GetMetadataStatic().reflection;
}
static const Score& default_instance();
static void InitAsDefaultInstance(); // FOR INTERNAL USE ONLY
static inline const Score* internal_default_instance() {
return reinterpret_cast<const Score*>(
&_Score_default_instance_);
}
static constexpr int kIndexInFileMessages =
13;
friend void swap(Score& a, Score& b) {
a.Swap(&b);
}
inline void Swap(Score* other) {
if (other == this) return;
InternalSwap(other);
}
// implements Message ----------------------------------------------
inline Score* New() const final {
return CreateMaybeMessage<Score>(nullptr);
}
Score* New(::PROTOBUF_NAMESPACE_ID::Arena* arena) const final {
return CreateMaybeMessage<Score>(arena);
}
void CopyFrom(const ::PROTOBUF_NAMESPACE_ID::Message& from) final;
void MergeFrom(const ::PROTOBUF_NAMESPACE_ID::Message& from) final;
void CopyFrom(const Score& from);
void MergeFrom(const Score& from);
PROTOBUF_ATTRIBUTE_REINITIALIZES void Clear() final;
bool IsInitialized() const final;
size_t ByteSizeLong() const final;
#if GOOGLE_PROTOBUF_ENABLE_EXPERIMENTAL_PARSER
const char* _InternalParse(const char* ptr, ::PROTOBUF_NAMESPACE_ID::internal::ParseContext* ctx) final;
#else
bool MergePartialFromCodedStream(
::PROTOBUF_NAMESPACE_ID::io::CodedInputStream* input) final;
#endif // GOOGLE_PROTOBUF_ENABLE_EXPERIMENTAL_PARSER
void SerializeWithCachedSizes(
::PROTOBUF_NAMESPACE_ID::io::CodedOutputStream* output) const final;
::PROTOBUF_NAMESPACE_ID::uint8* InternalSerializeWithCachedSizesToArray(
::PROTOBUF_NAMESPACE_ID::uint8* target) const final;
int GetCachedSize() const final { return _cached_size_.Get(); }
private:
inline void SharedCtor();
inline void SharedDtor();
void SetCachedSize(int size) const final;
void InternalSwap(Score* other);
friend class ::PROTOBUF_NAMESPACE_ID::internal::AnyMetadata;
static ::PROTOBUF_NAMESPACE_ID::StringPiece FullMessageName() {
return "milvus.proto.service.Score";
}
private:
inline ::PROTOBUF_NAMESPACE_ID::Arena* GetArenaNoVirtual() const {
return nullptr;
}
inline void* MaybeArenaPtr() const {
return nullptr;
}
public:
::PROTOBUF_NAMESPACE_ID::Metadata GetMetadata() const final;
private:
static ::PROTOBUF_NAMESPACE_ID::Metadata GetMetadataStatic() {
::PROTOBUF_NAMESPACE_ID::internal::AssignDescriptors(&::descriptor_table_service_5fmsg_2eproto);
return ::descriptor_table_service_5fmsg_2eproto.file_level_metadata[kIndexInFileMessages];
}
public:
// nested types ----------------------------------------------------
// accessors -------------------------------------------------------
enum : int {
kValuesFieldNumber = 2,
kTagFieldNumber = 1,
};
// repeated float values = 2;
int values_size() const;
void clear_values();
float values(int index) const;
void set_values(int index, float value);
void add_values(float value);
const ::PROTOBUF_NAMESPACE_ID::RepeatedField< float >&
values() const;
::PROTOBUF_NAMESPACE_ID::RepeatedField< float >*
mutable_values();
// string tag = 1;
void clear_tag();
const std::string& tag() const;
void set_tag(const std::string& value);
void set_tag(std::string&& value);
void set_tag(const char* value);
void set_tag(const char* value, size_t size);
std::string* mutable_tag();
std::string* release_tag();
void set_allocated_tag(std::string* tag);
// @@protoc_insertion_point(class_scope:milvus.proto.service.Score)
private:
class _Internal;
::PROTOBUF_NAMESPACE_ID::internal::InternalMetadataWithArena _internal_metadata_;
::PROTOBUF_NAMESPACE_ID::RepeatedField< float > values_;
mutable std::atomic<int> _values_cached_byte_size_;
::PROTOBUF_NAMESPACE_ID::internal::ArenaStringPtr tag_;
mutable ::PROTOBUF_NAMESPACE_ID::internal::CachedSize _cached_size_;
friend struct ::TableStruct_service_5fmsg_2eproto;
};
// -------------------------------------------------------------------
class Hits :
public ::PROTOBUF_NAMESPACE_ID::Message /* @@protoc_insertion_point(class_definition:milvus.proto.service.Hits) */ {
public:
@ -2351,7 +2196,7 @@ class Hits :
&_Hits_default_instance_);
}
static constexpr int kIndexInFileMessages =
14;
13;
friend void swap(Hits& a, Hits& b) {
a.Swap(&b);
@ -2437,27 +2282,33 @@ class Hits :
::PROTOBUF_NAMESPACE_ID::RepeatedField< ::PROTOBUF_NAMESPACE_ID::int64 >*
mutable_ids();
// repeated .milvus.proto.common.Blob row_data = 2;
// repeated bytes row_data = 2;
int row_data_size() const;
void clear_row_data();
::milvus::proto::common::Blob* mutable_row_data(int index);
::PROTOBUF_NAMESPACE_ID::RepeatedPtrField< ::milvus::proto::common::Blob >*
mutable_row_data();
const ::milvus::proto::common::Blob& row_data(int index) const;
::milvus::proto::common::Blob* add_row_data();
const ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField< ::milvus::proto::common::Blob >&
row_data() const;
const std::string& row_data(int index) const;
std::string* mutable_row_data(int index);
void set_row_data(int index, const std::string& value);
void set_row_data(int index, std::string&& value);
void set_row_data(int index, const char* value);
void set_row_data(int index, const void* value, size_t size);
std::string* add_row_data();
void add_row_data(const std::string& value);
void add_row_data(std::string&& value);
void add_row_data(const char* value);
void add_row_data(const void* value, size_t size);
const ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField<std::string>& row_data() const;
::PROTOBUF_NAMESPACE_ID::RepeatedPtrField<std::string>* mutable_row_data();
// repeated .milvus.proto.service.Score scores = 3;
// repeated float scores = 3;
int scores_size() const;
void clear_scores();
::milvus::proto::service::Score* mutable_scores(int index);
::PROTOBUF_NAMESPACE_ID::RepeatedPtrField< ::milvus::proto::service::Score >*
mutable_scores();
const ::milvus::proto::service::Score& scores(int index) const;
::milvus::proto::service::Score* add_scores();
const ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField< ::milvus::proto::service::Score >&
float scores(int index) const;
void set_scores(int index, float value);
void add_scores(float value);
const ::PROTOBUF_NAMESPACE_ID::RepeatedField< float >&
scores() const;
::PROTOBUF_NAMESPACE_ID::RepeatedField< float >*
mutable_scores();
// @@protoc_insertion_point(class_scope:milvus.proto.service.Hits)
private:
@ -2466,8 +2317,9 @@ class Hits :
::PROTOBUF_NAMESPACE_ID::internal::InternalMetadataWithArena _internal_metadata_;
::PROTOBUF_NAMESPACE_ID::RepeatedField< ::PROTOBUF_NAMESPACE_ID::int64 > ids_;
mutable std::atomic<int> _ids_cached_byte_size_;
::PROTOBUF_NAMESPACE_ID::RepeatedPtrField< ::milvus::proto::common::Blob > row_data_;
::PROTOBUF_NAMESPACE_ID::RepeatedPtrField< ::milvus::proto::service::Score > scores_;
::PROTOBUF_NAMESPACE_ID::RepeatedPtrField<std::string> row_data_;
::PROTOBUF_NAMESPACE_ID::RepeatedField< float > scores_;
mutable std::atomic<int> _scores_cached_byte_size_;
mutable ::PROTOBUF_NAMESPACE_ID::internal::CachedSize _cached_size_;
friend struct ::TableStruct_service_5fmsg_2eproto;
};
@ -2515,7 +2367,7 @@ class QueryResult :
&_QueryResult_default_instance_);
}
static constexpr int kIndexInFileMessages =
15;
14;
friend void swap(QueryResult& a, QueryResult& b) {
a.Swap(&b);
@ -2589,16 +2441,22 @@ class QueryResult :
kHitsFieldNumber = 2,
kStatusFieldNumber = 1,
};
// repeated .milvus.proto.service.Hits hits = 2;
// repeated bytes hits = 2;
int hits_size() const;
void clear_hits();
::milvus::proto::service::Hits* mutable_hits(int index);
::PROTOBUF_NAMESPACE_ID::RepeatedPtrField< ::milvus::proto::service::Hits >*
mutable_hits();
const ::milvus::proto::service::Hits& hits(int index) const;
::milvus::proto::service::Hits* add_hits();
const ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField< ::milvus::proto::service::Hits >&
hits() const;
const std::string& hits(int index) const;
std::string* mutable_hits(int index);
void set_hits(int index, const std::string& value);
void set_hits(int index, std::string&& value);
void set_hits(int index, const char* value);
void set_hits(int index, const void* value, size_t size);
std::string* add_hits();
void add_hits(const std::string& value);
void add_hits(std::string&& value);
void add_hits(const char* value);
void add_hits(const void* value, size_t size);
const ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField<std::string>& hits() const;
::PROTOBUF_NAMESPACE_ID::RepeatedPtrField<std::string>* mutable_hits();
// .milvus.proto.common.Status status = 1;
bool has_status() const;
@ -2613,7 +2471,7 @@ class QueryResult :
class _Internal;
::PROTOBUF_NAMESPACE_ID::internal::InternalMetadataWithArena _internal_metadata_;
::PROTOBUF_NAMESPACE_ID::RepeatedPtrField< ::milvus::proto::service::Hits > hits_;
::PROTOBUF_NAMESPACE_ID::RepeatedPtrField<std::string> hits_;
::milvus::proto::common::Status* status_;
mutable ::PROTOBUF_NAMESPACE_ID::internal::CachedSize _cached_size_;
friend struct ::TableStruct_service_5fmsg_2eproto;
@ -4022,91 +3880,6 @@ PartitionDescription::statistics() const {
// -------------------------------------------------------------------
// Score
// string tag = 1;
inline void Score::clear_tag() {
tag_.ClearToEmptyNoArena(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited());
}
inline const std::string& Score::tag() const {
// @@protoc_insertion_point(field_get:milvus.proto.service.Score.tag)
return tag_.GetNoArena();
}
inline void Score::set_tag(const std::string& value) {
tag_.SetNoArena(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), value);
// @@protoc_insertion_point(field_set:milvus.proto.service.Score.tag)
}
inline void Score::set_tag(std::string&& value) {
tag_.SetNoArena(
&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), ::std::move(value));
// @@protoc_insertion_point(field_set_rvalue:milvus.proto.service.Score.tag)
}
inline void Score::set_tag(const char* value) {
GOOGLE_DCHECK(value != nullptr);
tag_.SetNoArena(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), ::std::string(value));
// @@protoc_insertion_point(field_set_char:milvus.proto.service.Score.tag)
}
inline void Score::set_tag(const char* value, size_t size) {
tag_.SetNoArena(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(),
::std::string(reinterpret_cast<const char*>(value), size));
// @@protoc_insertion_point(field_set_pointer:milvus.proto.service.Score.tag)
}
inline std::string* Score::mutable_tag() {
// @@protoc_insertion_point(field_mutable:milvus.proto.service.Score.tag)
return tag_.MutableNoArena(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited());
}
inline std::string* Score::release_tag() {
// @@protoc_insertion_point(field_release:milvus.proto.service.Score.tag)
return tag_.ReleaseNoArena(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited());
}
inline void Score::set_allocated_tag(std::string* tag) {
if (tag != nullptr) {
} else {
}
tag_.SetAllocatedNoArena(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), tag);
// @@protoc_insertion_point(field_set_allocated:milvus.proto.service.Score.tag)
}
// repeated float values = 2;
inline int Score::values_size() const {
return values_.size();
}
inline void Score::clear_values() {
values_.Clear();
}
inline float Score::values(int index) const {
// @@protoc_insertion_point(field_get:milvus.proto.service.Score.values)
return values_.Get(index);
}
inline void Score::set_values(int index, float value) {
values_.Set(index, value);
// @@protoc_insertion_point(field_set:milvus.proto.service.Score.values)
}
inline void Score::add_values(float value) {
values_.Add(value);
// @@protoc_insertion_point(field_add:milvus.proto.service.Score.values)
}
inline const ::PROTOBUF_NAMESPACE_ID::RepeatedField< float >&
Score::values() const {
// @@protoc_insertion_point(field_list:milvus.proto.service.Score.values)
return values_;
}
inline ::PROTOBUF_NAMESPACE_ID::RepeatedField< float >*
Score::mutable_values() {
// @@protoc_insertion_point(field_mutable_list:milvus.proto.service.Score.values)
return &values_;
}
// -------------------------------------------------------------------
// Hits
// repeated int64 IDs = 1;
@ -4139,62 +3912,100 @@ Hits::mutable_ids() {
return &ids_;
}
// repeated .milvus.proto.common.Blob row_data = 2;
// repeated bytes row_data = 2;
inline int Hits::row_data_size() const {
return row_data_.size();
}
inline ::milvus::proto::common::Blob* Hits::mutable_row_data(int index) {
// @@protoc_insertion_point(field_mutable:milvus.proto.service.Hits.row_data)
return row_data_.Mutable(index);
inline void Hits::clear_row_data() {
row_data_.Clear();
}
inline ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField< ::milvus::proto::common::Blob >*
Hits::mutable_row_data() {
// @@protoc_insertion_point(field_mutable_list:milvus.proto.service.Hits.row_data)
return &row_data_;
}
inline const ::milvus::proto::common::Blob& Hits::row_data(int index) const {
inline const std::string& Hits::row_data(int index) const {
// @@protoc_insertion_point(field_get:milvus.proto.service.Hits.row_data)
return row_data_.Get(index);
}
inline ::milvus::proto::common::Blob* Hits::add_row_data() {
// @@protoc_insertion_point(field_add:milvus.proto.service.Hits.row_data)
inline std::string* Hits::mutable_row_data(int index) {
// @@protoc_insertion_point(field_mutable:milvus.proto.service.Hits.row_data)
return row_data_.Mutable(index);
}
inline void Hits::set_row_data(int index, const std::string& value) {
// @@protoc_insertion_point(field_set:milvus.proto.service.Hits.row_data)
row_data_.Mutable(index)->assign(value);
}
inline void Hits::set_row_data(int index, std::string&& value) {
// @@protoc_insertion_point(field_set:milvus.proto.service.Hits.row_data)
row_data_.Mutable(index)->assign(std::move(value));
}
inline void Hits::set_row_data(int index, const char* value) {
GOOGLE_DCHECK(value != nullptr);
row_data_.Mutable(index)->assign(value);
// @@protoc_insertion_point(field_set_char:milvus.proto.service.Hits.row_data)
}
inline void Hits::set_row_data(int index, const void* value, size_t size) {
row_data_.Mutable(index)->assign(
reinterpret_cast<const char*>(value), size);
// @@protoc_insertion_point(field_set_pointer:milvus.proto.service.Hits.row_data)
}
inline std::string* Hits::add_row_data() {
// @@protoc_insertion_point(field_add_mutable:milvus.proto.service.Hits.row_data)
return row_data_.Add();
}
inline const ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField< ::milvus::proto::common::Blob >&
inline void Hits::add_row_data(const std::string& value) {
row_data_.Add()->assign(value);
// @@protoc_insertion_point(field_add:milvus.proto.service.Hits.row_data)
}
inline void Hits::add_row_data(std::string&& value) {
row_data_.Add(std::move(value));
// @@protoc_insertion_point(field_add:milvus.proto.service.Hits.row_data)
}
inline void Hits::add_row_data(const char* value) {
GOOGLE_DCHECK(value != nullptr);
row_data_.Add()->assign(value);
// @@protoc_insertion_point(field_add_char:milvus.proto.service.Hits.row_data)
}
inline void Hits::add_row_data(const void* value, size_t size) {
row_data_.Add()->assign(reinterpret_cast<const char*>(value), size);
// @@protoc_insertion_point(field_add_pointer:milvus.proto.service.Hits.row_data)
}
inline const ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField<std::string>&
Hits::row_data() const {
// @@protoc_insertion_point(field_list:milvus.proto.service.Hits.row_data)
return row_data_;
}
inline ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField<std::string>*
Hits::mutable_row_data() {
// @@protoc_insertion_point(field_mutable_list:milvus.proto.service.Hits.row_data)
return &row_data_;
}
// repeated .milvus.proto.service.Score scores = 3;
// repeated float scores = 3;
inline int Hits::scores_size() const {
return scores_.size();
}
inline void Hits::clear_scores() {
scores_.Clear();
}
inline ::milvus::proto::service::Score* Hits::mutable_scores(int index) {
// @@protoc_insertion_point(field_mutable:milvus.proto.service.Hits.scores)
return scores_.Mutable(index);
}
inline ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField< ::milvus::proto::service::Score >*
Hits::mutable_scores() {
// @@protoc_insertion_point(field_mutable_list:milvus.proto.service.Hits.scores)
return &scores_;
}
inline const ::milvus::proto::service::Score& Hits::scores(int index) const {
inline float Hits::scores(int index) const {
// @@protoc_insertion_point(field_get:milvus.proto.service.Hits.scores)
return scores_.Get(index);
}
inline ::milvus::proto::service::Score* Hits::add_scores() {
// @@protoc_insertion_point(field_add:milvus.proto.service.Hits.scores)
return scores_.Add();
inline void Hits::set_scores(int index, float value) {
scores_.Set(index, value);
// @@protoc_insertion_point(field_set:milvus.proto.service.Hits.scores)
}
inline const ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField< ::milvus::proto::service::Score >&
inline void Hits::add_scores(float value) {
scores_.Add(value);
// @@protoc_insertion_point(field_add:milvus.proto.service.Hits.scores)
}
inline const ::PROTOBUF_NAMESPACE_ID::RepeatedField< float >&
Hits::scores() const {
// @@protoc_insertion_point(field_list:milvus.proto.service.Hits.scores)
return scores_;
}
inline ::PROTOBUF_NAMESPACE_ID::RepeatedField< float >*
Hits::mutable_scores() {
// @@protoc_insertion_point(field_mutable_list:milvus.proto.service.Hits.scores)
return &scores_;
}
// -------------------------------------------------------------------
@ -4245,35 +4056,70 @@ inline void QueryResult::set_allocated_status(::milvus::proto::common::Status* s
// @@protoc_insertion_point(field_set_allocated:milvus.proto.service.QueryResult.status)
}
// repeated .milvus.proto.service.Hits hits = 2;
// repeated bytes hits = 2;
inline int QueryResult::hits_size() const {
return hits_.size();
}
inline void QueryResult::clear_hits() {
hits_.Clear();
}
inline ::milvus::proto::service::Hits* QueryResult::mutable_hits(int index) {
// @@protoc_insertion_point(field_mutable:milvus.proto.service.QueryResult.hits)
return hits_.Mutable(index);
}
inline ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField< ::milvus::proto::service::Hits >*
QueryResult::mutable_hits() {
// @@protoc_insertion_point(field_mutable_list:milvus.proto.service.QueryResult.hits)
return &hits_;
}
inline const ::milvus::proto::service::Hits& QueryResult::hits(int index) const {
inline const std::string& QueryResult::hits(int index) const {
// @@protoc_insertion_point(field_get:milvus.proto.service.QueryResult.hits)
return hits_.Get(index);
}
inline ::milvus::proto::service::Hits* QueryResult::add_hits() {
// @@protoc_insertion_point(field_add:milvus.proto.service.QueryResult.hits)
inline std::string* QueryResult::mutable_hits(int index) {
// @@protoc_insertion_point(field_mutable:milvus.proto.service.QueryResult.hits)
return hits_.Mutable(index);
}
inline void QueryResult::set_hits(int index, const std::string& value) {
// @@protoc_insertion_point(field_set:milvus.proto.service.QueryResult.hits)
hits_.Mutable(index)->assign(value);
}
inline void QueryResult::set_hits(int index, std::string&& value) {
// @@protoc_insertion_point(field_set:milvus.proto.service.QueryResult.hits)
hits_.Mutable(index)->assign(std::move(value));
}
inline void QueryResult::set_hits(int index, const char* value) {
GOOGLE_DCHECK(value != nullptr);
hits_.Mutable(index)->assign(value);
// @@protoc_insertion_point(field_set_char:milvus.proto.service.QueryResult.hits)
}
inline void QueryResult::set_hits(int index, const void* value, size_t size) {
hits_.Mutable(index)->assign(
reinterpret_cast<const char*>(value), size);
// @@protoc_insertion_point(field_set_pointer:milvus.proto.service.QueryResult.hits)
}
inline std::string* QueryResult::add_hits() {
// @@protoc_insertion_point(field_add_mutable:milvus.proto.service.QueryResult.hits)
return hits_.Add();
}
inline const ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField< ::milvus::proto::service::Hits >&
inline void QueryResult::add_hits(const std::string& value) {
hits_.Add()->assign(value);
// @@protoc_insertion_point(field_add:milvus.proto.service.QueryResult.hits)
}
inline void QueryResult::add_hits(std::string&& value) {
hits_.Add(std::move(value));
// @@protoc_insertion_point(field_add:milvus.proto.service.QueryResult.hits)
}
inline void QueryResult::add_hits(const char* value) {
GOOGLE_DCHECK(value != nullptr);
hits_.Add()->assign(value);
// @@protoc_insertion_point(field_add_char:milvus.proto.service.QueryResult.hits)
}
inline void QueryResult::add_hits(const void* value, size_t size) {
hits_.Add()->assign(reinterpret_cast<const char*>(value), size);
// @@protoc_insertion_point(field_add_pointer:milvus.proto.service.QueryResult.hits)
}
inline const ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField<std::string>&
QueryResult::hits() const {
// @@protoc_insertion_point(field_list:milvus.proto.service.QueryResult.hits)
return hits_;
}
inline ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField<std::string>*
QueryResult::mutable_hits() {
// @@protoc_insertion_point(field_mutable_list:milvus.proto.service.QueryResult.hits)
return &hits_;
}
#ifdef __GNUC__
#pragma GCC diagnostic pop
@ -4306,8 +4152,6 @@ QueryResult::hits() const {
// -------------------------------------------------------------------
// -------------------------------------------------------------------
// @@protoc_insertion_point(namespace_scope)

2
internal/core/src/segcore/plan_c.h
View File

@ -13,9 +13,9 @@
extern "C" {
#endif
#include "segcore/collection_c.h"
#include <stdbool.h>
#include <stdint.h>
#include "segcore/collection_c.h"
typedef void* CPlan;
typedef void* CPlaceholderGroup;

172
internal/core/src/segcore/reduce_c.cpp
View File

@ -9,11 +9,183 @@
// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
// or implied. See the License for the specific language governing permissions and limitations under the License
#include <vector>
#include <utils/EasyAssert.h>
#include "segcore/reduce_c.h"
#include "segcore/Reduce.h"
#include "utils/Types.h"
#include "pb/service_msg.pb.h"
using SearchResult = milvus::engine::QueryResult;
int
MergeInto(int64_t num_queries, int64_t topk, float* distances, int64_t* uids, float* new_distances, int64_t* new_uids) {
auto status = milvus::segcore::merge_into(num_queries, topk, distances, uids, new_distances, new_uids);
return status.code();
}
struct MarshaledHitsPeerGroup {
std::vector<std::string> hits_;
std::vector<int64_t> blob_length_;
};
struct MarshaledHits {
explicit MarshaledHits(int64_t num_group) {
marshaled_hits_.resize(num_group);
}
int
get_num_group() {
return marshaled_hits_.size();
}
std::vector<MarshaledHitsPeerGroup> marshaled_hits_;
};
void
DeleteMarshaledHits(CMarshaledHits c_marshaled_hits) {
auto hits = (MarshaledHits*)c_marshaled_hits;
delete hits;
}
struct SearchResultPair {
uint64_t id_;
float distance_;
int64_t segment_id_;
SearchResultPair(uint64_t id, float distance, int64_t segment_id)
: id_(id), distance_(distance), segment_id_(segment_id) {
}
bool
operator<(const SearchResultPair& pair) const {
return (distance_ < pair.distance_);
}
};
void
GetResultData(std::vector<SearchResult*>& search_results,
SearchResult& final_result,
int64_t query_offset,
int64_t topk) {
auto num_segments = search_results.size();
std::map<int, int> iter_loc_peer_result;
std::vector<SearchResultPair> result_pairs;
for (int j = 0; j < num_segments; ++j) {
auto id = search_results[j]->result_ids_[query_offset];
auto distance = search_results[j]->result_distances_[query_offset];
result_pairs.push_back(SearchResultPair(id, distance, j));
iter_loc_peer_result[j] = query_offset;
}
std::sort(result_pairs.begin(), result_pairs.end());
final_result.result_ids_.push_back(result_pairs[0].id_);
final_result.result_distances_.push_back(result_pairs[0].distance_);
for (int i = 1; i < topk; ++i) {
auto segment_id = result_pairs[0].segment_id_;
auto query_offset = ++(iter_loc_peer_result[segment_id]);
auto id = search_results[segment_id]->result_ids_[query_offset];
auto distance = search_results[segment_id]->result_distances_[query_offset];
result_pairs[0] = SearchResultPair(id, distance, segment_id);
std::sort(result_pairs.begin(), result_pairs.end());
final_result.result_ids_.push_back(result_pairs[0].id_);
final_result.result_distances_.push_back(result_pairs[0].distance_);
}
}
CQueryResult
ReduceQueryResults(CQueryResult* query_results, int64_t num_segments) {
std::vector<SearchResult*> search_results;
for (int i = 0; i < num_segments; ++i) {
search_results.push_back((SearchResult*)query_results[i]);
}
auto topk = search_results[0]->topK_;
auto num_queries = search_results[0]->num_queries_;
auto final_result = std::make_unique<SearchResult>();
int64_t query_offset = 0;
for (int j = 0; j < num_queries; ++j) {
GetResultData(search_results, *final_result, query_offset, topk);
query_offset += topk;
}
return (CQueryResult)final_result.release();
}
CMarshaledHits
ReorganizeQueryResults(CQueryResult c_query_result,
CPlan c_plan,
CPlaceholderGroup* c_placeholder_groups,
int64_t num_groups) {
auto marshaledHits = std::make_unique<MarshaledHits>(num_groups);
auto search_result = (milvus::engine::QueryResult*)c_query_result;
auto& result_ids = search_result->result_ids_;
auto& result_distances = search_result->result_distances_;
auto topk = GetTopK(c_plan);
int64_t queries_offset = 0;
for (int i = 0; i < num_groups; i++) {
auto num_queries = GetNumOfQueries(c_placeholder_groups[i]);
MarshaledHitsPeerGroup& hits_peer_group = (*marshaledHits).marshaled_hits_[i];
for (int j = 0; j < num_queries; j++) {
auto index = topk * queries_offset++;
milvus::proto::service::Hits hits;
for (int k = index; k < index + topk; k++) {
hits.add_ids(result_ids[k]);
hits.add_scores(result_distances[k]);
}
auto blob = hits.SerializeAsString();
hits_peer_group.hits_.push_back(blob);
hits_peer_group.blob_length_.push_back(blob.size());
}
}
return (CMarshaledHits)marshaledHits.release();
}
int64_t
GetHitsBlobSize(CMarshaledHits c_marshaled_hits) {
int64_t total_size = 0;
auto marshaled_hits = (MarshaledHits*)c_marshaled_hits;
auto num_group = marshaled_hits->get_num_group();
for (int i = 0; i < num_group; i++) {
auto& length_vector = marshaled_hits->marshaled_hits_[i].blob_length_;
for (int j = 0; j < length_vector.size(); j++) {
total_size += length_vector[j];
}
}
return total_size;
}
void
GetHitsBlob(CMarshaledHits c_marshaled_hits, const void* hits) {
auto byte_hits = (char*)hits;
auto marshaled_hits = (MarshaledHits*)c_marshaled_hits;
auto num_group = marshaled_hits->get_num_group();
int offset = 0;
for (int i = 0; i < num_group; i++) {
auto& hits = marshaled_hits->marshaled_hits_[i];
auto num_queries = hits.hits_.size();
for (int j = 0; j < num_queries; j++) {
auto blob_size = hits.blob_length_[j];
memcpy(byte_hits + offset, hits.hits_[j].data(), blob_size);
offset += blob_size;
}
}
}
int64_t
GetNumQueriesPeerGroup(CMarshaledHits c_marshaled_hits, int64_t group_index) {
auto marshaled_hits = (MarshaledHits*)c_marshaled_hits;
auto& hits = marshaled_hits->marshaled_hits_[group_index].hits_;
return hits.size();
}
void
GetHitSizePeerQueries(CMarshaledHits c_marshaled_hits, int64_t group_index, int64_t* hit_size_peer_query) {
auto marshaled_hits = (MarshaledHits*)c_marshaled_hits;
auto& blob_lens = marshaled_hits->marshaled_hits_[group_index].blob_length_;
for (int i = 0; i < blob_lens.size(); i++) {
hit_size_peer_query[i] = blob_lens[i];
}
}

27
internal/core/src/segcore/reduce_c.h
View File

@ -15,10 +15,37 @@ extern "C" {
#include <stdbool.h>
#include <stdint.h>
#include "segcore/segment_c.h"
typedef void* CMarshaledHits;
void
DeleteMarshaledHits(CMarshaledHits c_marshaled_hits);
int
MergeInto(int64_t num_queries, int64_t topk, float* distances, int64_t* uids, float* new_distances, int64_t* new_uids);
CQueryResult
ReduceQueryResults(CQueryResult* query_results, int64_t num_segments);
CMarshaledHits
ReorganizeQueryResults(CQueryResult query_result,
CPlan c_plan,
CPlaceholderGroup* c_placeholder_groups,
int64_t num_groups);
int64_t
GetHitsBlobSize(CMarshaledHits c_marshaled_hits);
void
GetHitsBlob(CMarshaledHits c_marshaled_hits, const void* hits);
int64_t
GetNumQueriesPeerGroup(CMarshaledHits c_marshaled_hits, int64_t group_index);
void
GetHitSizePeerQueries(CMarshaledHits c_marshaled_hits, int64_t group_index, int64_t* hit_size_peer_query);
#ifdef __cplusplus
}
#endif

19
internal/core/src/segcore/segment_c.cpp
View File

@ -40,6 +40,12 @@ DeleteSegment(CSegmentBase segment) {
delete s;
}
void
DeleteQueryResult(CQueryResult query_result) {
auto res = (milvus::segcore::QueryResult*)query_result;
delete res;
}
//////////////////////////////////////////////////////////////////
CStatus
@ -120,30 +126,31 @@ Search(CSegmentBase c_segment,
CPlaceholderGroup* c_placeholder_groups,
uint64_t* timestamps,
int num_groups,
int64_t* result_ids,
float* result_distances) {
CQueryResult* result) {
auto segment = (milvus::segcore::SegmentBase*)c_segment;
auto plan = (milvus::query::Plan*)c_plan;
std::vector<const milvus::query::PlaceholderGroup*> placeholder_groups;
for (int i = 0; i < num_groups; ++i) {
placeholder_groups.push_back((const milvus::query::PlaceholderGroup*)c_placeholder_groups[i]);
}
milvus::segcore::QueryResult query_result;
auto query_result = std::make_unique<milvus::segcore::QueryResult>();
auto status = CStatus();
try {
auto res = segment->Search(plan, placeholder_groups.data(), timestamps, num_groups, query_result);
auto res = segment->Search(plan, placeholder_groups.data(), timestamps, num_groups, *query_result);
status.error_code = Success;
status.error_msg = "";
} catch (std::runtime_error& e) {
status.error_code = UnexpectedException;
status.error_msg = strdup(e.what());
}
*result = query_result.release();
// result_ids and result_distances have been allocated memory in goLang,
// so we don't need to malloc here.
memcpy(result_ids, query_result.result_ids_.data(), query_result.get_row_count() * sizeof(int64_t));
memcpy(result_distances, query_result.result_distances_.data(), query_result.get_row_count() * sizeof(float));
// memcpy(result_ids, query_result.result_ids_.data(), query_result.get_row_count() * sizeof(long int));
// memcpy(result_distances, query_result.result_distances_.data(), query_result.get_row_count() * sizeof(float));
return status;
}

7
internal/core/src/segcore/segment_c.h
View File

@ -21,6 +21,7 @@ extern "C" {
#include "segcore/plan_c.h"
typedef void* CSegmentBase;
typedef void* CQueryResult;
enum ErrorCode {
Success = 0,
@ -38,6 +39,9 @@ NewSegment(CCollection collection, uint64_t segment_id);
void
DeleteSegment(CSegmentBase segment);
void
DeleteQueryResult(CQueryResult query_result);
//////////////////////////////////////////////////////////////////
CStatus
@ -66,8 +70,7 @@ Search(CSegmentBase c_segment,
CPlaceholderGroup* placeholder_groups,
uint64_t* timestamps,
int num_groups,
int64_t* result_ids,
float* result_distances);
CQueryResult* result);
//////////////////////////////////////////////////////////////////

286
internal/core/unittest/test_c_api.cpp
View File

@ -15,7 +15,6 @@
#include <gtest/gtest.h>
#include "segcore/collection_c.h"
#include "segcore/segment_c.h"
#include "pb/service_msg.pb.h"
#include "segcore/reduce_c.h"
@ -159,105 +158,100 @@ TEST(CApiTest, SearchTest) {
timestamps.clear();
timestamps.push_back(1);
long result_ids[100];
float result_distances[100];
auto sea_res = Search(segment, plan, placeholderGroups.data(), timestamps.data(), 1, result_ids, result_distances);
assert(sea_res.error_code == Success);
CQueryResult search_result;
auto res = Search(segment, plan, placeholderGroups.data(), timestamps.data(), 1, &search_result);
assert(res.error_code == Success);
DeletePlan(plan);
DeletePlaceholderGroup(placeholderGroup);
DeleteQueryResult(search_result);
DeleteCollection(collection);
DeleteSegment(segment);
}
TEST(CApiTest, BuildIndexTest) {
auto schema_tmp_conf = "";
auto collection = NewCollection(schema_tmp_conf);
auto segment = NewSegment(collection, 0);
std::vector<char> raw_data;
std::vector<uint64_t> timestamps;
std::vector<int64_t> uids;
int N = 10000;
std::default_random_engine e(67);
for (int i = 0; i < N; ++i) {
uids.push_back(100000 + i);
timestamps.push_back(0);
// append vec
float vec[16];
for (auto& x : vec) {
x = e() % 2000 * 0.001 - 1.0;
}
raw_data.insert(raw_data.end(), (const char*)std::begin(vec), (const char*)std::end(vec));
int age = e() % 100;
raw_data.insert(raw_data.end(), (const char*)&age, ((const char*)&age) + sizeof(age));
}
auto line_sizeof = (sizeof(int) + sizeof(float) * 16);
auto offset = PreInsert(segment, N);
auto ins_res = Insert(segment, offset, N, uids.data(), timestamps.data(), raw_data.data(), (int)line_sizeof, N);
assert(ins_res.error_code == Success);
// TODO: add index ptr
Close(segment);
// BuildIndex(collection, segment);
const char* dsl_string = R"(
{
"bool": {
"vector": {
"fakevec": {
"metric_type": "L2",
"params": {
"nprobe": 10
},
"query": "$0",
"topk": 10
}
}
}
})";
namespace ser = milvus::proto::service;
int num_queries = 10;
int dim = 16;
std::normal_distribution<double> dis(0, 1);
ser::PlaceholderGroup raw_group;
auto value = raw_group.add_placeholders();
value->set_tag("$0");
value->set_type(ser::PlaceholderType::VECTOR_FLOAT);
for (int i = 0; i < num_queries; ++i) {
std::vector<float> vec;
for (int d = 0; d < dim; ++d) {
vec.push_back(dis(e));
}
// std::string line((char*)vec.data(), (char*)vec.data() + vec.size() * sizeof(float));
value->add_values(vec.data(), vec.size() * sizeof(float));
}
auto blob = raw_group.SerializeAsString();
auto plan = CreatePlan(collection, dsl_string);
auto placeholderGroup = ParsePlaceholderGroup(plan, blob.data(), blob.length());
std::vector<CPlaceholderGroup> placeholderGroups;
placeholderGroups.push_back(placeholderGroup);
timestamps.clear();
timestamps.push_back(1);
long result_ids[100];
float result_distances[100];
auto sea_res = Search(segment, plan, placeholderGroups.data(), timestamps.data(), 1, result_ids, result_distances);
assert(sea_res.error_code == Success);
DeletePlan(plan);
DeletePlaceholderGroup(placeholderGroup);
DeleteCollection(collection);
DeleteSegment(segment);
}
// TEST(CApiTest, BuildIndexTest) {
// auto schema_tmp_conf = "";
// auto collection = NewCollection(schema_tmp_conf);
// auto segment = NewSegment(collection, 0);
//
// std::vector<char> raw_data;
// std::vector<uint64_t> timestamps;
// std::vector<int64_t> uids;
// int N = 10000;
// std::default_random_engine e(67);
// for (int i = 0; i < N; ++i) {
// uids.push_back(100000 + i);
// timestamps.push_back(0);
// // append vec
// float vec[16];
// for (auto& x : vec) {
// x = e() % 2000 * 0.001 - 1.0;
// }
// raw_data.insert(raw_data.end(), (const char*)std::begin(vec), (const char*)std::end(vec));
// int age = e() % 100;
// raw_data.insert(raw_data.end(), (const char*)&age, ((const char*)&age) + sizeof(age));
// }
//
// auto line_sizeof = (sizeof(int) + sizeof(float) * 16);
//
// auto offset = PreInsert(segment, N);
//
// auto ins_res = Insert(segment, offset, N, uids.data(), timestamps.data(), raw_data.data(), (int)line_sizeof, N);
// assert(ins_res == 0);
//
// // TODO: add index ptr
// Close(segment);
// BuildIndex(collection, segment);
//
// const char* dsl_string = R"(
// {
// "bool": {
// "vector": {
// "fakevec": {
// "metric_type": "L2",
// "params": {
// "nprobe": 10
// },
// "query": "$0",
// "topk": 10
// }
// }
// }
// })";
//
// namespace ser = milvus::proto::service;
// int num_queries = 10;
// int dim = 16;
// std::normal_distribution<double> dis(0, 1);
// ser::PlaceholderGroup raw_group;
// auto value = raw_group.add_placeholders();
// value->set_tag("$0");
// value->set_type(ser::PlaceholderType::VECTOR_FLOAT);
// for (int i = 0; i < num_queries; ++i) {
// std::vector<float> vec;
// for (int d = 0; d < dim; ++d) {
// vec.push_back(dis(e));
// }
// // std::string line((char*)vec.data(), (char*)vec.data() + vec.size() * sizeof(float));
// value->add_values(vec.data(), vec.size() * sizeof(float));
// }
// auto blob = raw_group.SerializeAsString();
//
// auto plan = CreatePlan(collection, dsl_string);
// auto placeholderGroup = ParsePlaceholderGroup(plan, blob.data(), blob.length());
// std::vector<CPlaceholderGroup> placeholderGroups;
// placeholderGroups.push_back(placeholderGroup);
// timestamps.clear();
// timestamps.push_back(1);
//
// auto search_res = Search(segment, plan, placeholderGroups.data(), timestamps.data(), 1);
//
// DeletePlan(plan);
// DeletePlaceholderGroup(placeholderGroup);
// DeleteQueryResult(search_res);
// DeleteCollection(collection);
// DeleteSegment(segment);
//}
TEST(CApiTest, IsOpenedTest) {
auto schema_tmp_conf = "";
@ -552,3 +546,109 @@ TEST(CApiTest, MergeInto) {
ASSERT_EQ(uids[1], 1);
ASSERT_EQ(distance[1], 5);
}
TEST(CApiTest, Reduce) {
auto schema_tmp_conf = "";
auto collection = NewCollection(schema_tmp_conf);
auto segment = NewSegment(collection, 0);
std::vector<char> raw_data;
std::vector<uint64_t> timestamps;
std::vector<int64_t> uids;
int N = 10000;
std::default_random_engine e(67);
for (int i = 0; i < N; ++i) {
uids.push_back(100000 + i);
timestamps.push_back(0);
// append vec
float vec[16];
for (auto& x : vec) {
x = e() % 2000 * 0.001 - 1.0;
}
raw_data.insert(raw_data.end(), (const char*)std::begin(vec), (const char*)std::end(vec));
int age = e() % 100;
raw_data.insert(raw_data.end(), (const char*)&age, ((const char*)&age) + sizeof(age));
}
auto line_sizeof = (sizeof(int) + sizeof(float) * 16);
auto offset = PreInsert(segment, N);
auto ins_res = Insert(segment, offset, N, uids.data(), timestamps.data(), raw_data.data(), (int)line_sizeof, N);
assert(ins_res.error_code == Success);
const char* dsl_string = R"(
{
"bool": {
"vector": {
"fakevec": {
"metric_type": "L2",
"params": {
"nprobe": 10
},
"query": "$0",
"topk": 10
}
}
}
})";
namespace ser = milvus::proto::service;
int num_queries = 10;
int dim = 16;
std::normal_distribution<double> dis(0, 1);
ser::PlaceholderGroup raw_group;
auto value = raw_group.add_placeholders();
value->set_tag("$0");
value->set_type(ser::PlaceholderType::VECTOR_FLOAT);
for (int i = 0; i < num_queries; ++i) {
std::vector<float> vec;
for (int d = 0; d < dim; ++d) {
vec.push_back(dis(e));
}
// std::string line((char*)vec.data(), (char*)vec.data() + vec.size() * sizeof(float));
value->add_values(vec.data(), vec.size() * sizeof(float));
}
auto blob = raw_group.SerializeAsString();
auto plan = CreatePlan(collection, dsl_string);
auto placeholderGroup = ParsePlaceholderGroup(plan, blob.data(), blob.length());
std::vector<CPlaceholderGroup> placeholderGroups;
placeholderGroups.push_back(placeholderGroup);
timestamps.clear();
timestamps.push_back(1);
std::vector<CQueryResult> results;
CQueryResult res1;
CQueryResult res2;
auto res = Search(segment, plan, placeholderGroups.data(), timestamps.data(), 1, &res1);
assert(res.error_code == Success);
res = Search(segment, plan, placeholderGroups.data(), timestamps.data(), 1, &res2);
assert(res.error_code == Success);
results.push_back(res1);
results.push_back(res2);
auto reduced_search_result = ReduceQueryResults(results.data(), 2);
auto reorganize_search_result = ReorganizeQueryResults(reduced_search_result, plan, placeholderGroups.data(), 1);
auto hits_blob_size = GetHitsBlobSize(reorganize_search_result);
assert(hits_blob_size > 0);
std::vector<char> hits_blob;
hits_blob.resize(hits_blob_size);
GetHitsBlob(reorganize_search_result, hits_blob.data());
assert(hits_blob.data() != nullptr);
auto num_queries_group = GetNumQueriesPeerGroup(reorganize_search_result, 0);
assert(num_queries_group == 10);
std::vector<int64_t> hit_size_peer_query;
hit_size_peer_query.resize(num_queries_group);
GetHitSizePeerQueries(reorganize_search_result, 0, hit_size_peer_query.data());
assert(hit_size_peer_query[0] > 0);
DeletePlan(plan);
DeletePlaceholderGroup(placeholderGroup);
DeleteQueryResult(res1);
DeleteQueryResult(res2);
DeleteQueryResult(reduced_search_result);
DeleteMarshaledHits(reorganize_search_result);
DeleteCollection(collection);
DeleteSegment(segment);
}

18
internal/master/meta_table.go
View File

@ -207,7 +207,7 @@ func (mt *metaTable) AddCollection(coll *pb.CollectionMeta) error {
}
if len(coll.PartitionTags) == 0 {
coll.PartitionTags = append(coll.PartitionTags, "default")
coll.PartitionTags = append(coll.PartitionTags, Params.DefaultPartitionTag)
}
_, ok := mt.collName2ID[coll.Schema.Name]
if ok {
@ -279,6 +279,10 @@ func (mt *metaTable) AddPartition(collID UniqueID, tag string) error {
return errors.Errorf("can't find collection. id = " + strconv.FormatInt(collID, 10))
}
// number of partition tags (except _default) should be limited to 4096 by default
if int64(len(coll.PartitionTags)) > Params.MaxPartitionNum {
return errors.New("maximum partition's number should be limit to " + strconv.FormatInt(Params.MaxPartitionNum, 10))
}
for _, t := range coll.PartitionTags {
if t == tag {
return errors.Errorf("partition already exists.")
@ -313,17 +317,29 @@ func (mt *metaTable) DeletePartition(collID UniqueID, tag string) error {
mt.ddLock.Lock()
defer mt.ddLock.Unlock()
if tag == Params.DefaultPartitionTag {
return errors.New("default partition cannot be deleted")
}
collMeta, ok := mt.collID2Meta[collID]
if !ok {
return errors.Errorf("can't find collection. id = " + strconv.FormatInt(collID, 10))
}
// check tag exists
exist := false
pt := make([]string, 0, len(collMeta.PartitionTags))
for _, t := range collMeta.PartitionTags {
if t != tag {
pt = append(pt, t)
} else {
exist = true
}
}
if !exist {
return errors.New("partition " + tag + " does not exist")
}
if len(pt) == len(collMeta.PartitionTags) {
return nil
}

41
internal/master/meta_table_test.go
View File

@ -3,6 +3,7 @@ package master
import (
"context"
"reflect"
"strconv"
"testing"
"github.com/stretchr/testify/assert"
@ -238,6 +239,10 @@ func TestMetaTable_DeletePartition(t *testing.T) {
assert.Equal(t, 1, len(meta.collName2ID))
assert.Equal(t, 1, len(meta.collID2Meta))
assert.Equal(t, 1, len(meta.segID2Meta))
// delete not exist
err = meta.DeletePartition(100, "not_exist")
assert.NotNil(t, err)
}
func TestMetaTable_Segment(t *testing.T) {
@ -366,3 +371,39 @@ func TestMetaTable_UpdateSegment(t *testing.T) {
assert.Nil(t, err)
assert.Equal(t, seg.NumRows, int64(210))
}
func TestMetaTable_AddPartition_Limit(t *testing.T) {
Init()
Params.MaxPartitionNum = 256 // adding 4096 partitions is too slow
etcdAddr := Params.EtcdAddress
cli, err := clientv3.New(clientv3.Config{Endpoints: []string{etcdAddr}})
assert.Nil(t, err)
etcdKV := kv.NewEtcdKV(cli, "/etcd/test/root")
_, err = cli.Delete(context.TODO(), "/etcd/test/root", clientv3.WithPrefix())
assert.Nil(t, err)
meta, err := NewMetaTable(etcdKV)
assert.Nil(t, err)
defer meta.client.Close()
colMeta := pb.CollectionMeta{
ID: 100,
Schema: &schemapb.CollectionSchema{
Name: "coll1",
},
CreateTime: 0,
SegmentIDs: []UniqueID{},
PartitionTags: []string{},
}
err = meta.AddCollection(&colMeta)
assert.Nil(t, err)
for i := 0; i < int(Params.MaxPartitionNum); i++ {
err := meta.AddPartition(100, "partition_"+strconv.Itoa(i))
assert.Nil(t, err)
}
err = meta.AddPartition(100, "partition_limit")
assert.NotNil(t, err)
}

26
internal/master/param_table.go
View File

@ -43,6 +43,9 @@ type ParamTable struct {
K2SChannelNames []string
QueryNodeStatsChannelName string
MsgChannelSubName string
MaxPartitionNum int64
DefaultPartitionTag string
}
var Params ParamTable
@ -91,6 +94,8 @@ func (p *ParamTable) Init() {
p.initK2SChannelNames()
p.initQueryNodeStatsChannelName()
p.initMsgChannelSubName()
p.initMaxPartitionNum()
p.initDefaultPartitionTag()
}
func (p *ParamTable) initAddress() {
@ -411,3 +416,24 @@ func (p *ParamTable) initK2SChannelNames() {
}
p.K2SChannelNames = channels
}
func (p *ParamTable) initMaxPartitionNum() {
str, err := p.Load("master.maxPartitionNum")
if err != nil {
panic(err)
}
maxPartitionNum, err := strconv.ParseInt(str, 10, 64)
if err != nil {
panic(err)
}
p.MaxPartitionNum = maxPartitionNum
}
func (p *ParamTable) initDefaultPartitionTag() {
defaultTag, err := p.Load("master.defaultPartitionTag")
if err != nil {
panic(err)
}
p.DefaultPartitionTag = defaultTag
}

8
internal/master/partition_task.go
View File

@ -191,10 +191,12 @@ func (t *showPartitionTask) Execute() error {
return errors.New("null request")
}
partitions := make([]string, 0)
for _, collection := range t.mt.collID2Meta {
partitions = append(partitions, collection.PartitionTags...)
collMeta, err := t.mt.GetCollectionByName(t.req.CollectionName.CollectionName)
if err != nil {
return err
}
partitions := make([]string, 0)
partitions = append(partitions, collMeta.PartitionTags...)
stringListResponse := servicepb.StringListResponse{
Status: &commonpb.Status{

7
internal/master/partition_task_test.go
View File

@ -60,6 +60,9 @@ func TestMaster_Partition(t *testing.T) {
K2SChannelNames: []string{"k2s0", "k2s1"},
QueryNodeStatsChannelName: "statistic",
MsgChannelSubName: Params.MsgChannelSubName,
MaxPartitionNum: int64(4096),
DefaultPartitionTag: "_default",
}
port := 10000 + rand.Intn(1000)
@ -212,7 +215,7 @@ func TestMaster_Partition(t *testing.T) {
//assert.Equal(t, collMeta.PartitionTags[0], "partition1")
//assert.Equal(t, collMeta.PartitionTags[1], "partition2")
assert.ElementsMatch(t, []string{"default", "partition1", "partition2"}, collMeta.PartitionTags)
assert.ElementsMatch(t, []string{"_default", "partition1", "partition2"}, collMeta.PartitionTags)
showPartitionReq := internalpb.ShowPartitionRequest{
MsgType: internalpb.MsgType_kShowPartitions,
@ -224,7 +227,7 @@ func TestMaster_Partition(t *testing.T) {
stringList, err := cli.ShowPartitions(ctx, &showPartitionReq)
assert.Nil(t, err)
assert.ElementsMatch(t, []string{"default", "partition1", "partition2"}, stringList.Values)
assert.ElementsMatch(t, []string{"_default", "partition1", "partition2"}, stringList.Values)
showPartitionReq = internalpb.ShowPartitionRequest{
MsgType: internalpb.MsgType_kShowPartitions,

3
internal/master/segment_manager_test.go
View File

@ -261,6 +261,9 @@ func startupMaster() {
K2SChannelNames: []string{"k2s0", "k2s1"},
QueryNodeStatsChannelName: "statistic",
MsgChannelSubName: Params.MsgChannelSubName,
MaxPartitionNum: int64(4096),
DefaultPartitionTag: "_default",
}
master, err = CreateServer(ctx)

9
internal/msgstream/msgstream.go
View File

@ -141,6 +141,15 @@ func (ms *PulsarMsgStream) Produce(msgPack *MsgPack) error {
hashValues := tsMsg.HashKeys()
bucketValues := make([]int32, len(hashValues))
for index, hashValue := range hashValues {
if tsMsg.Type() == internalPb.MsgType_kSearchResult {
searchResult := tsMsg.(*SearchResultMsg)
channelID := int32(searchResult.ResultChannelID)
if channelID >= int32(len(ms.producers)) {
return errors.New("Failed to produce pulsar msg to unKnow channel")
}
bucketValues[index] = channelID
continue
}
bucketValues[index] = hashValue % int32(len(ms.producers))
}
reBucketValues[channelID] = bucketValues

6
internal/msgstream/msgstream_test.go
View File

@ -50,7 +50,7 @@ func getTsMsg(msgType MsgType, reqID UniqueID, hashValue int32) TsMsg {
CollectionName: "Collection",
PartitionTag: "Partition",
SegmentID: 1,
ChannelID: 1,
ChannelID: 0,
ProxyID: 1,
Timestamps: []Timestamp{1},
RowIDs: []int64{1},
@ -82,7 +82,7 @@ func getTsMsg(msgType MsgType, reqID UniqueID, hashValue int32) TsMsg {
ReqID: reqID,
ProxyID: 1,
Timestamp: 1,
ResultChannelID: 1,
ResultChannelID: 0,
}
searchMsg := &SearchMsg{
BaseMsg: baseMsg,
@ -97,7 +97,7 @@ func getTsMsg(msgType MsgType, reqID UniqueID, hashValue int32) TsMsg {
ProxyID: 1,
QueryNodeID: 1,
Timestamp: 1,
ResultChannelID: 1,
ResultChannelID: 0,
}
searchResultMsg := &SearchResultMsg{
BaseMsg: baseMsg,

2
internal/proto/internal_msg.proto
View File

@ -231,7 +231,7 @@ message SearchResult {
int64 query_nodeID = 5;
uint64 timestamp = 6;
int64 result_channelID = 7;
repeated service.Hits hits = 8;
repeated bytes hits = 8;
}
message TimeTickMsg {

205
internal/proto/internalpb/internal_msg.pb.go
View File

@ -1580,17 +1580,17 @@ func (m *SearchRequest) GetQuery() *commonpb.Blob {
}
type SearchResult struct {
MsgType MsgType `protobuf:"varint,1,opt,name=msg_type,json=msgType,proto3,enum=milvus.proto.internal.MsgType" json:"msg_type,omitempty"`
Status *commonpb.Status `protobuf:"bytes,2,opt,name=status,proto3" json:"status,omitempty"`
ReqID int64 `protobuf:"varint,3,opt,name=reqID,proto3" json:"reqID,omitempty"`
ProxyID int64 `protobuf:"varint,4,opt,name=proxyID,proto3" json:"proxyID,omitempty"`
QueryNodeID int64 `protobuf:"varint,5,opt,name=query_nodeID,json=queryNodeID,proto3" json:"query_nodeID,omitempty"`
Timestamp uint64 `protobuf:"varint,6,opt,name=timestamp,proto3" json:"timestamp,omitempty"`
ResultChannelID int64 `protobuf:"varint,7,opt,name=result_channelID,json=resultChannelID,proto3" json:"result_channelID,omitempty"`
Hits []*servicepb.Hits `protobuf:"bytes,8,rep,name=hits,proto3" json:"hits,omitempty"`
XXX_NoUnkeyedLiteral struct{} `json:"-"`
XXX_unrecognized []byte `json:"-"`
XXX_sizecache int32 `json:"-"`
MsgType MsgType `protobuf:"varint,1,opt,name=msg_type,json=msgType,proto3,enum=milvus.proto.internal.MsgType" json:"msg_type,omitempty"`
Status *commonpb.Status `protobuf:"bytes,2,opt,name=status,proto3" json:"status,omitempty"`
ReqID int64 `protobuf:"varint,3,opt,name=reqID,proto3" json:"reqID,omitempty"`
ProxyID int64 `protobuf:"varint,4,opt,name=proxyID,proto3" json:"proxyID,omitempty"`
QueryNodeID int64 `protobuf:"varint,5,opt,name=query_nodeID,json=queryNodeID,proto3" json:"query_nodeID,omitempty"`
Timestamp uint64 `protobuf:"varint,6,opt,name=timestamp,proto3" json:"timestamp,omitempty"`
ResultChannelID int64 `protobuf:"varint,7,opt,name=result_channelID,json=resultChannelID,proto3" json:"result_channelID,omitempty"`
Hits [][]byte `protobuf:"bytes,8,rep,name=hits,proto3" json:"hits,omitempty"`
XXX_NoUnkeyedLiteral struct{} `json:"-"`
XXX_unrecognized []byte `json:"-"`
XXX_sizecache int32 `json:"-"`
}
func (m *SearchResult) Reset() { *m = SearchResult{} }
@ -1667,7 +1667,7 @@ func (m *SearchResult) GetResultChannelID() int64 {
return 0
}
func (m *SearchResult) GetHits() []*servicepb.Hits {
func (m *SearchResult) GetHits() [][]byte {
if m != nil {
return m.Hits
}
@ -2000,95 +2000,94 @@ func init() {
func init() { proto.RegisterFile("internal_msg.proto", fileDescriptor_7eb37f6b80b23116) }
var fileDescriptor_7eb37f6b80b23116 = []byte{
// 1427 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe4, 0x58, 0x4d, 0x6f, 0x1c, 0x45,
0x13, 0x4e, 0xef, 0xac, 0xf7, 0xa3, 0xd6, 0x5e, 0x8f, 0xdb, 0x76, 0xb2, 0x49, 0x5e, 0x25, 0xce,
0xe4, 0x15, 0x31, 0x41, 0xd8, 0x62, 0xc3, 0x81, 0xdc, 0x20, 0xde, 0x43, 0x96, 0xc8, 0x51, 0x18,
0x5b, 0x20, 0xa1, 0x48, 0xa3, 0xf1, 0x6e, 0x65, 0x77, 0x34, 0x5f, 0xeb, 0xee, 0x59, 0x3b, 0xeb,
0x03, 0xa7, 0xfc, 0x00, 0x38, 0x70, 0xe0, 0x80, 0xc4, 0x91, 0x13, 0x82, 0x7f, 0x01, 0x81, 0x2b,
0x07, 0xfe, 0x02, 0x08, 0x22, 0x41, 0xb8, 0xa3, 0xee, 0x9e, 0x8f, 0x1d, 0x7b, 0x6d, 0x47, 0x4a,
0x0c, 0x96, 0x7c, 0xdb, 0xaa, 0xad, 0xe9, 0xaa, 0x7a, 0x9e, 0xee, 0xea, 0xaa, 0x06, 0xea, 0x04,
0x11, 0xb2, 0xc0, 0xf6, 0x2c, 0x9f, 0xf7, 0x56, 0x06, 0x2c, 0x8c, 0x42, 0xba, 0xe8, 0x3b, 0xde,
0xce, 0x90, 0x2b, 0x69, 0x25, 0x31, 0xb8, 0x34, 0xdd, 0x09, 0x7d, 0x3f, 0x0c, 0x94, 0xfa, 0xd2,
0x1c, 0x47, 0xb6, 0xe3, 0x74, 0x30, 0xfb, 0xce, 0x08, 0xa0, 0xda, 0x6e, 0x99, 0xb8, 0x3d, 0x44,
0x1e, 0xd1, 0xf3, 0x50, 0x1a, 0x20, 0xb2, 0x76, 0xab, 0x41, 0x96, 0xc8, 0xb2, 0x66, 0xc6, 0x12,
0xbd, 0x05, 0x45, 0x16, 0x7a, 0xd8, 0x28, 0x2c, 0x91, 0xe5, 0x7a, 0xf3, 0xea, 0xca, 0x44, 0x5f,
0x2b, 0x0f, 0x10, 0x99, 0x19, 0x7a, 0x68, 0x4a, 0x63, 0xba, 0x00, 0x53, 0x9d, 0x70, 0x18, 0x44,
0x0d, 0x6d, 0x89, 0x2c, 0xcf, 0x98, 0x4a, 0x30, 0x7a, 0x00, 0xc2, 0x1f, 0x1f, 0x84, 0x01, 0x47,
0x7a, 0x0b, 0x4a, 0x3c, 0xb2, 0xa3, 0x21, 0x97, 0x0e, 0x6b, 0xcd, 0xcb, 0xf9, 0xa5, 0xe3, 0xe0,
0x37, 0xa4, 0x89, 0x19, 0x9b, 0xd2, 0x3a, 0x14, 0xda, 0x2d, 0x19, 0x8b, 0x66, 0x16, 0xda, 0xad,
0x43, 0x1c, 0x85, 0x00, 0x9b, 0x3c, 0xfc, 0x17, 0x33, 0xdb, 0x81, 0x9a, 0x74, 0xf8, 0x32, 0xa9,
0xfd, 0x0f, 0xaa, 0x91, 0xe3, 0x23, 0x8f, 0x6c, 0x7f, 0x20, 0x63, 0x2a, 0x9a, 0x99, 0xe2, 0x10,
0xbf, 0x4f, 0x08, 0x4c, 0x6f, 0x60, 0x2f, 0x63, 0x31, 0x35, 0x23, 0x63, 0x66, 0x62, 0xe9, 0x4e,
0xdf, 0x0e, 0x02, 0xf4, 0x62, 0xf0, 0xa6, 0xcc, 0x4c, 0x41, 0x2f, 0x43, 0xb5, 0x13, 0x7a, 0x9e,
0x15, 0xd8, 0x3e, 0xca, 0xe5, 0xab, 0x66, 0x45, 0x28, 0xee, 0xdb, 0x3e, 0xd2, 0xeb, 0x30, 0x33,
0xb0, 0x59, 0xe4, 0x44, 0x4e, 0x18, 0x58, 0x91, 0xdd, 0x6b, 0x14, 0xa5, 0xc1, 0x74, 0xaa, 0xdc,
0xb4, 0x7b, 0xc6, 0x37, 0x04, 0xe8, 0x7b, 0x9c, 0x3b, 0xbd, 0x20, 0x17, 0xcc, 0x2b, 0x05, 0xfe,
0x1e, 0xcc, 0x0e, 0x90, 0x59, 0x71, 0xd8, 0x16, 0xc3, 0xed, 0x86, 0xb6, 0xa4, 0x2d, 0xd7, 0x9a,
0xd7, 0x0f, 0xf9, 0x7e, 0x3c, 0x14, 0x73, 0x66, 0x80, 0x6c, 0x4d, 0x7d, 0x6a, 0xe2, 0xb6, 0xf1,
0x25, 0x81, 0x59, 0xf9, 0xbf, 0x8a, 0xda, 0xc7, 0x40, 0x42, 0xc7, 0x85, 0x2a, 0x0e, 0x56, 0x09,
0xc7, 0x40, 0x37, 0x91, 0x95, 0x3c, 0xa0, 0xc5, 0xe3, 0x00, 0x9d, 0x9a, 0x00, 0xe8, 0x33, 0x02,
0xf3, 0x39, 0x40, 0x4f, 0x6e, 0x63, 0xdd, 0x80, 0x59, 0x7c, 0x3c, 0x70, 0x18, 0x5a, 0xdd, 0x21,
0xb3, 0x45, 0x00, 0x32, 0x99, 0xa2, 0x59, 0x57, 0xea, 0x56, 0xac, 0xa5, 0x0f, 0xe1, 0xfc, 0x38,
0x01, 0x76, 0x8a, 0x5c, 0xa3, 0x28, 0x79, 0x78, 0xed, 0x28, 0x1e, 0x32, 0x9c, 0xcd, 0x85, 0x8c,
0x8a, 0x4c, 0x6b, 0xfc, 0x4c, 0xe0, 0xc2, 0x1a, 0x43, 0x3b, 0xc2, 0xb5, 0xd0, 0xf3, 0xb0, 0x23,
0x5c, 0x26, 0xfb, 0xe8, 0x36, 0x54, 0x7c, 0xde, 0xb3, 0xa2, 0xd1, 0x00, 0x65, 0xde, 0xf5, 0xe6,
0x95, 0x43, 0x7c, 0xad, 0xf3, 0xde, 0xe6, 0x68, 0x80, 0x66, 0xd9, 0x57, 0x3f, 0x04, 0x41, 0x0c,
0xb7, 0xd3, 0x92, 0xa1, 0x84, 0x3c, 0x22, 0xda, 0x7e, 0x44, 0x1a, 0x50, 0x1e, 0xb0, 0xf0, 0xf1,
0xa8, 0xdd, 0x92, 0xe4, 0x69, 0x66, 0x22, 0xd2, 0xb7, 0xa0, 0xc4, 0x3b, 0x7d, 0xf4, 0x6d, 0x49,
0x5a, 0xad, 0x79, 0x71, 0x22, 0xfc, 0x77, 0xbc, 0x70, 0xcb, 0x8c, 0x0d, 0x05, 0x93, 0x8b, 0x2d,
0x16, 0x0e, 0x4e, 0x71, 0x56, 0xeb, 0x30, 0xdb, 0x49, 0xa3, 0x53, 0x9b, 0x56, 0xa5, 0xf7, 0xff,
0x7c, 0x3c, 0xf1, 0x05, 0xb2, 0x92, 0xa5, 0x22, 0x36, 0xb4, 0x59, 0xef, 0xe4, 0x64, 0xe3, 0x0f,
0x02, 0x0b, 0x77, 0x6d, 0x7e, 0x76, 0x12, 0xfe, 0x8b, 0xc0, 0xc5, 0x16, 0xf2, 0x0e, 0x73, 0xb6,
0xf0, 0xec, 0x64, 0xfd, 0x15, 0x81, 0xc5, 0x8d, 0x7e, 0xb8, 0x7b, 0x7a, 0x33, 0x36, 0x7e, 0x27,
0x70, 0x5e, 0xd5, 0x94, 0x07, 0x49, 0x71, 0x3d, 0x75, 0xac, 0xbc, 0x0f, 0xf5, 0xec, 0x3a, 0x18,
0x23, 0xe5, 0xfa, 0x64, 0x52, 0xd2, 0x44, 0x24, 0x27, 0xd9, 0x4d, 0x22, 0x29, 0xf9, 0x8d, 0xc0,
0x82, 0xa8, 0x35, 0x67, 0x23, 0xdb, 0x5f, 0x09, 0xcc, 0xdf, 0xb5, 0xf9, 0xd9, 0x48, 0xf6, 0x19,
0x81, 0x46, 0x52, 0x63, 0xce, 0x46, 0xc6, 0xe2, 0x1a, 0x11, 0xf5, 0xe5, 0xf4, 0x66, 0xfb, 0x8a,
0x0b, 0xea, 0x9f, 0x05, 0x98, 0x69, 0x07, 0x1c, 0x59, 0x74, 0x62, 0x99, 0xde, 0x38, 0x18, 0xb1,
0xea, 0xf7, 0xf7, 0xc5, 0xf2, 0x42, 0x5d, 0xbf, 0xc0, 0x8d, 0x63, 0x4f, 0x74, 0x6f, 0xed, 0x96,
0xcc, 0x5c, 0x33, 0x33, 0x45, 0xbe, 0x71, 0x2e, 0xa9, 0x7f, 0xb3, 0xc6, 0x79, 0x0c, 0xd5, 0x72,
0x1e, 0xd5, 0x2b, 0x00, 0x29, 0xf8, 0xbc, 0x51, 0x59, 0xd2, 0x96, 0x8b, 0xe6, 0x98, 0x46, 0x0c,
0x15, 0x2c, 0xdc, 0x6d, 0xb7, 0x78, 0xa3, 0xba, 0xa4, 0x89, 0xa1, 0x42, 0x49, 0xf4, 0x6d, 0xa8,
0xb0, 0x70, 0xd7, 0xea, 0xda, 0x91, 0xdd, 0x00, 0xd9, 0x90, 0x1e, 0xd1, 0x9d, 0x95, 0x59, 0xb8,
0xdb, 0xb2, 0x23, 0xdb, 0x78, 0x52, 0x80, 0x99, 0x16, 0x7a, 0x18, 0xe1, 0x7f, 0x0f, 0x7a, 0x0e,
0xb1, 0xe2, 0x11, 0x88, 0x4d, 0x1d, 0x85, 0x58, 0xe9, 0x00, 0x62, 0xd7, 0x60, 0x7a, 0xc0, 0x1c,
0xdf, 0x66, 0x23, 0xcb, 0xc5, 0x11, 0x6f, 0x94, 0x25, 0x6e, 0xb5, 0x58, 0x77, 0x0f, 0x47, 0xdc,
0x78, 0x4e, 0x60, 0x66, 0x03, 0x6d, 0xd6, 0xe9, 0x9f, 0x18, 0x0c, 0x63, 0xf1, 0x6b, 0xf9, 0xf8,
0x73, 0xe7, 0xaf, 0xb8, 0xff, 0xfc, 0xbd, 0x0e, 0x3a, 0x43, 0x3e, 0xf4, 0x22, 0x2b, 0x03, 0x47,
0x01, 0x30, 0xab, 0xf4, 0x6b, 0x29, 0x44, 0xab, 0x30, 0xb5, 0x3d, 0x44, 0x36, 0x92, 0xdb, 0xed,
0x48, 0xfe, 0x95, 0x9d, 0xf1, 0xb4, 0x20, 0xc6, 0x67, 0x95, 0xb6, 0x58, 0xea, 0x65, 0xb2, 0xce,
0x46, 0xb3, 0xc2, 0x8b, 0x8f, 0x66, 0x29, 0x54, 0xda, 0x21, 0x50, 0xed, 0x2b, 0x39, 0xd7, 0x60,
0x5a, 0x46, 0x6e, 0x05, 0x61, 0x17, 0x53, 0x20, 0x6a, 0x52, 0x77, 0x5f, 0xaa, 0xf2, 0x68, 0x96,
0x5e, 0x04, 0xcd, 0xf2, 0x64, 0x34, 0x57, 0xa0, 0xd8, 0x77, 0x22, 0x75, 0x04, 0x6b, 0xcd, 0x4b,
0x93, 0x6b, 0xda, 0x5d, 0x27, 0xe2, 0xa6, 0xb4, 0x33, 0x3e, 0x81, 0xda, 0xa6, 0xe3, 0xe3, 0xa6,
0xd3, 0x71, 0xd7, 0x79, 0xef, 0x65, 0xa0, 0xcc, 0xde, 0x0d, 0x0a, 0xb9, 0x77, 0x83, 0x23, 0x0b,
0xb5, 0xf1, 0x05, 0x81, 0xf2, 0x3d, 0x1c, 0x35, 0x37, 0xb0, 0x27, 0x71, 0x15, 0x65, 0x21, 0x99,
0xe5, 0xa5, 0x40, 0xaf, 0x42, 0x6d, 0xec, 0x20, 0xc4, 0x8b, 0x43, 0x76, 0x0e, 0x8e, 0xb9, 0x09,
0x2e, 0x42, 0xc5, 0xe1, 0xd6, 0x8e, 0xed, 0x39, 0x5d, 0xc9, 0x4b, 0xc5, 0x2c, 0x3b, 0xfc, 0x43,
0x21, 0x8a, 0x23, 0x98, 0x56, 0x3e, 0xde, 0x98, 0x92, 0x07, 0x6c, 0x4c, 0x63, 0x3c, 0x04, 0x88,
0x43, 0x13, 0xd0, 0xa4, 0xac, 0x93, 0x71, 0xd6, 0xdf, 0x81, 0xb2, 0x8b, 0xa3, 0x26, 0xc7, 0x5e,
0xa3, 0x20, 0x21, 0x3f, 0x0c, 0xaf, 0x78, 0x25, 0x33, 0x31, 0x37, 0x3e, 0x57, 0xaf, 0x40, 0xc2,
0x99, 0xd8, 0x5f, 0x3c, 0x5f, 0x99, 0xc9, 0xfe, 0xca, 0x7c, 0x15, 0x6a, 0x3e, 0xfa, 0x21, 0x1b,
0x59, 0xdc, 0xd9, 0xc3, 0x04, 0x06, 0xa5, 0xda, 0x70, 0xf6, 0x50, 0x24, 0x1a, 0x0c, 0x7d, 0x8b,
0x85, 0xbb, 0x3c, 0x39, 0xab, 0xc1, 0xd0, 0x37, 0xc3, 0x5d, 0x4e, 0xdf, 0x80, 0x39, 0x86, 0x1d,
0x0c, 0x22, 0x6f, 0x64, 0xf9, 0x61, 0xd7, 0x79, 0xe4, 0x60, 0x02, 0x86, 0x9e, 0xfc, 0xb1, 0x1e,
0xeb, 0x8d, 0xaf, 0x09, 0xcc, 0x7d, 0x90, 0x6c, 0xcd, 0x0d, 0xec, 0xa9, 0xe0, 0x4e, 0x60, 0x63,
0xbc, 0x2b, 0xf3, 0xb5, 0xc4, 0xa1, 0xe2, 0xc7, 0xbf, 0x0a, 0xa5, 0x38, 0x99, 0x15, 0x1e, 0x07,
0x75, 0xf3, 0x97, 0x02, 0x94, 0x63, 0x77, 0xb4, 0x0a, 0x53, 0xee, 0xfd, 0x30, 0x40, 0xfd, 0x1c,
0x5d, 0x84, 0x39, 0x77, 0xff, 0xab, 0x84, 0xde, 0xa5, 0xf3, 0x30, 0xeb, 0xe6, 0x87, 0x7a, 0x1d,
0x29, 0x85, 0xba, 0x9b, 0x9b, 0x7b, 0xf5, 0x47, 0xf4, 0x02, 0xcc, 0xbb, 0x07, 0x47, 0x43, 0x5d,
0x6c, 0x01, 0xdd, 0xcd, 0x4f, 0x4f, 0x5c, 0xef, 0xd3, 0x45, 0xd0, 0xdd, 0x7d, 0x03, 0x8b, 0xfe,
0x3d, 0xa1, 0xf3, 0x50, 0x77, 0x73, 0x7d, 0xbd, 0xfe, 0x03, 0xa1, 0x14, 0x66, 0xdc, 0xf1, 0xf6,
0x57, 0x7f, 0x4a, 0xe8, 0x05, 0xa0, 0xee, 0x81, 0x2e, 0x51, 0xff, 0x91, 0xd0, 0x05, 0x98, 0x75,
0x73, 0xcd, 0x14, 0xd7, 0x7f, 0x22, 0x74, 0x1a, 0xca, 0xae, 0xea, 0x38, 0xf4, 0x4f, 0x35, 0x29,
0xa9, 0xab, 0x50, 0xff, 0x4c, 0x49, 0xaa, 0x34, 0xea, 0xcf, 0x35, 0xe9, 0x6c, 0xbc, 0x50, 0xea,
0x7f, 0x6b, 0xb4, 0x0e, 0x55, 0x37, 0x39, 0xf0, 0xfa, 0xb7, 0x55, 0xe9, 0xfc, 0x00, 0xdb, 0xfa,
0x77, 0xd5, 0x9b, 0xb7, 0xa1, 0x92, 0xbc, 0xe5, 0x51, 0x80, 0xd2, 0xba, 0xcd, 0x23, 0x64, 0xfa,
0x39, 0xf1, 0xdb, 0x44, 0xbb, 0x8b, 0x4c, 0x27, 0xe2, 0xf7, 0x47, 0xcc, 0x11, 0xfa, 0x82, 0xc0,
0xff, 0x81, 0xa8, 0x77, 0xba, 0x76, 0xa7, 0xf5, 0xf1, 0x9d, 0x9e, 0x13, 0xf5, 0x87, 0x5b, 0xa2,
0x7e, 0xae, 0xee, 0x39, 0x9e, 0xe7, 0xec, 0x45, 0xd8, 0xe9, 0xaf, 0x2a, 0x72, 0xdf, 0xec, 0x3a,
0x3c, 0x62, 0xce, 0xd6, 0x30, 0xc2, 0xee, 0x6a, 0x42, 0xf1, 0xaa, 0x64, 0x3c, 0x15, 0x07, 0x5b,
0x5b, 0x25, 0xa9, 0xb9, 0xf5, 0x4f, 0x00, 0x00, 0x00, 0xff, 0xff, 0xdf, 0x3a, 0xa9, 0xbc, 0x3c,
0x17, 0x00, 0x00,
// 1416 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe4, 0x58, 0x4b, 0x6f, 0x1c, 0xc5,
0x13, 0x4f, 0xef, 0xac, 0xf7, 0x51, 0x6b, 0xaf, 0xc7, 0x6d, 0x3b, 0xd9, 0x24, 0x7f, 0x25, 0xce,
0xe4, 0x2f, 0x62, 0x82, 0xb0, 0x85, 0xc3, 0x81, 0xdc, 0x20, 0xde, 0x43, 0x96, 0xc8, 0x51, 0x18,
0x5b, 0x20, 0xa1, 0x48, 0xa3, 0xf1, 0x6e, 0x65, 0x77, 0x34, 0x4f, 0x77, 0xcf, 0xda, 0x59, 0x1f,
0x38, 0xe5, 0x03, 0xc0, 0x81, 0x03, 0x07, 0x24, 0x8e, 0x9c, 0x22, 0xf8, 0x16, 0xbc, 0xae, 0x1c,
0xf8, 0x0a, 0x20, 0x88, 0x04, 0xe1, 0x8e, 0xba, 0x7b, 0x1e, 0x3b, 0x7e, 0x46, 0x4a, 0x0c, 0x96,
0x7c, 0x9b, 0xaa, 0xe9, 0xe9, 0xaa, 0xfa, 0xfd, 0xaa, 0x6b, 0xaa, 0x1a, 0xa8, 0x13, 0xc4, 0xc8,
0x02, 0xdb, 0xb3, 0x7c, 0xde, 0x5f, 0x8a, 0x58, 0x18, 0x87, 0x74, 0xde, 0x77, 0xbc, 0xed, 0x21,
0x57, 0xd2, 0x52, 0xba, 0xe0, 0xd2, 0x64, 0x37, 0xf4, 0xfd, 0x30, 0x50, 0xea, 0x4b, 0x33, 0x1c,
0xd9, 0xb6, 0xd3, 0xc5, 0xfc, 0x3b, 0x23, 0x80, 0x7a, 0xa7, 0x6d, 0xe2, 0xd6, 0x10, 0x79, 0x4c,
0xcf, 0x43, 0x25, 0x42, 0x64, 0x9d, 0x76, 0x8b, 0x2c, 0x90, 0x45, 0xcd, 0x4c, 0x24, 0x7a, 0x0b,
0xca, 0x2c, 0xf4, 0xb0, 0x55, 0x5a, 0x20, 0x8b, 0xcd, 0x95, 0xab, 0x4b, 0x07, 0xda, 0x5a, 0x7a,
0x80, 0xc8, 0xcc, 0xd0, 0x43, 0x53, 0x2e, 0xa6, 0x73, 0x30, 0xd1, 0x0d, 0x87, 0x41, 0xdc, 0xd2,
0x16, 0xc8, 0xe2, 0x94, 0xa9, 0x04, 0xa3, 0x0f, 0x20, 0xec, 0xf1, 0x28, 0x0c, 0x38, 0xd2, 0x5b,
0x50, 0xe1, 0xb1, 0x1d, 0x0f, 0xb9, 0x34, 0xd8, 0x58, 0xb9, 0x5c, 0xdc, 0x3a, 0x71, 0x7e, 0x5d,
0x2e, 0x31, 0x93, 0xa5, 0xb4, 0x09, 0xa5, 0x4e, 0x5b, 0xfa, 0xa2, 0x99, 0xa5, 0x4e, 0xfb, 0x10,
0x43, 0x21, 0xc0, 0x06, 0x0f, 0xff, 0xc5, 0xc8, 0xb6, 0xa1, 0x21, 0x0d, 0xbe, 0x4c, 0x68, 0xff,
0x83, 0x7a, 0xec, 0xf8, 0xc8, 0x63, 0xdb, 0x8f, 0xa4, 0x4f, 0x65, 0x33, 0x57, 0x1c, 0x62, 0xf7,
0x09, 0x81, 0xc9, 0x75, 0xec, 0xe7, 0x2c, 0x66, 0xcb, 0xc8, 0xd8, 0x32, 0xb1, 0x75, 0x77, 0x60,
0x07, 0x01, 0x7a, 0x09, 0x78, 0x13, 0x66, 0xae, 0xa0, 0x97, 0xa1, 0xde, 0x0d, 0x3d, 0xcf, 0x0a,
0x6c, 0x1f, 0xe5, 0xf6, 0x75, 0xb3, 0x26, 0x14, 0xf7, 0x6d, 0x1f, 0xe9, 0x75, 0x98, 0x8a, 0x6c,
0x16, 0x3b, 0xb1, 0x13, 0x06, 0x56, 0x6c, 0xf7, 0x5b, 0x65, 0xb9, 0x60, 0x32, 0x53, 0x6e, 0xd8,
0x7d, 0xe3, 0x29, 0x01, 0xfa, 0x1e, 0xe7, 0x4e, 0x3f, 0x28, 0x38, 0xf3, 0x4a, 0x81, 0xbf, 0x07,
0xd3, 0x11, 0x32, 0x2b, 0x71, 0xdb, 0x62, 0xb8, 0xd5, 0xd2, 0x16, 0xb4, 0xc5, 0xc6, 0xca, 0xf5,
0x43, 0xbe, 0x1f, 0x77, 0xc5, 0x9c, 0x8a, 0x90, 0xad, 0xaa, 0x4f, 0x4d, 0xdc, 0x32, 0xbe, 0x24,
0x30, 0x2d, 0xdf, 0x2b, 0xaf, 0x7d, 0x0c, 0x24, 0x74, 0x5c, 0xa8, 0x12, 0x67, 0x95, 0x70, 0x0c,
0x74, 0x07, 0xb2, 0x52, 0x04, 0xb4, 0x7c, 0x1c, 0xa0, 0x13, 0x07, 0x00, 0xfa, 0x8c, 0xc0, 0x6c,
0x01, 0xd0, 0x93, 0x4b, 0xac, 0x1b, 0x30, 0x8d, 0x8f, 0x23, 0x87, 0xa1, 0xd5, 0x1b, 0x32, 0x5b,
0x38, 0x20, 0x83, 0x29, 0x9b, 0x4d, 0xa5, 0x6e, 0x27, 0x5a, 0xfa, 0x10, 0xce, 0x8f, 0x13, 0x60,
0x67, 0xc8, 0xb5, 0xca, 0x92, 0x87, 0xd7, 0x8e, 0xe2, 0x21, 0xc7, 0xd9, 0x9c, 0xcb, 0xa9, 0xc8,
0xb5, 0xc6, 0xcf, 0x04, 0x2e, 0xac, 0x32, 0xb4, 0x63, 0x5c, 0x0d, 0x3d, 0x0f, 0xbb, 0xc2, 0x64,
0x9a, 0x47, 0xb7, 0xa1, 0xe6, 0xf3, 0xbe, 0x15, 0x8f, 0x22, 0x94, 0x71, 0x37, 0x57, 0xae, 0x1c,
0x62, 0x6b, 0x8d, 0xf7, 0x37, 0x46, 0x11, 0x9a, 0x55, 0x5f, 0x3d, 0x08, 0x82, 0x18, 0x6e, 0x65,
0x25, 0x43, 0x09, 0x45, 0x44, 0xb4, 0xbd, 0x88, 0xb4, 0xa0, 0x1a, 0xb1, 0xf0, 0xf1, 0xa8, 0xd3,
0x96, 0xe4, 0x69, 0x66, 0x2a, 0xd2, 0xb7, 0xa0, 0xc2, 0xbb, 0x03, 0xf4, 0x6d, 0x49, 0x5a, 0x63,
0xe5, 0xe2, 0x81, 0xf0, 0xdf, 0xf1, 0xc2, 0x4d, 0x33, 0x59, 0x28, 0x98, 0x9c, 0x6f, 0xb3, 0x30,
0x3a, 0xc5, 0x51, 0xad, 0xc1, 0x74, 0x37, 0xf3, 0x4e, 0x25, 0xad, 0x0a, 0xef, 0xff, 0x45, 0x7f,
0x92, 0x1f, 0xc8, 0x52, 0x1e, 0x8a, 0x48, 0x68, 0xb3, 0xd9, 0x2d, 0xc8, 0xc6, 0x1f, 0x04, 0xe6,
0xee, 0xda, 0xfc, 0xec, 0x04, 0xfc, 0x17, 0x81, 0x8b, 0x6d, 0xe4, 0x5d, 0xe6, 0x6c, 0xe2, 0xd9,
0x89, 0xfa, 0x2b, 0x02, 0xf3, 0xeb, 0x83, 0x70, 0xe7, 0xf4, 0x46, 0x6c, 0xfc, 0x4e, 0xe0, 0xbc,
0xaa, 0x29, 0x0f, 0xd2, 0xe2, 0x7a, 0xea, 0x58, 0x79, 0x1f, 0x9a, 0xf9, 0xef, 0x60, 0x8c, 0x94,
0xeb, 0x07, 0x93, 0x92, 0x05, 0x22, 0x39, 0xc9, 0xff, 0x24, 0x92, 0x92, 0xdf, 0x08, 0xcc, 0x89,
0x5a, 0x73, 0x36, 0xa2, 0xfd, 0x95, 0xc0, 0xec, 0x5d, 0x9b, 0x9f, 0x8d, 0x60, 0x9f, 0x11, 0x68,
0xa5, 0x35, 0xe6, 0x6c, 0x44, 0x2c, 0x7e, 0x23, 0xa2, 0xbe, 0x9c, 0xde, 0x68, 0x5f, 0x71, 0x41,
0xfd, 0xb3, 0x04, 0x53, 0x9d, 0x80, 0x23, 0x8b, 0x4f, 0x2c, 0xd2, 0x1b, 0xfb, 0x3d, 0x56, 0xfd,
0xfe, 0x1e, 0x5f, 0x5e, 0xa8, 0xeb, 0x17, 0xb8, 0x71, 0xec, 0x8b, 0xee, 0xad, 0xd3, 0x96, 0x91,
0x6b, 0x66, 0xae, 0x28, 0x36, 0xce, 0x15, 0xf5, 0x36, 0x6f, 0x9c, 0xc7, 0x50, 0xad, 0x16, 0x51,
0xbd, 0x02, 0x90, 0x81, 0xcf, 0x5b, 0xb5, 0x05, 0x6d, 0xb1, 0x6c, 0x8e, 0x69, 0xc4, 0x50, 0xc1,
0xc2, 0x9d, 0x4e, 0x9b, 0xb7, 0xea, 0x0b, 0x9a, 0x18, 0x2a, 0x94, 0x44, 0xdf, 0x86, 0x1a, 0x0b,
0x77, 0xac, 0x9e, 0x1d, 0xdb, 0x2d, 0x90, 0x0d, 0xe9, 0x11, 0xdd, 0x59, 0x95, 0x85, 0x3b, 0x6d,
0x3b, 0xb6, 0x8d, 0x27, 0x25, 0x98, 0x6a, 0xa3, 0x87, 0x31, 0xfe, 0xf7, 0xa0, 0x17, 0x10, 0x2b,
0x1f, 0x81, 0xd8, 0xc4, 0x51, 0x88, 0x55, 0xf6, 0x21, 0x76, 0x0d, 0x26, 0x23, 0xe6, 0xf8, 0x36,
0x1b, 0x59, 0x2e, 0x8e, 0x78, 0xab, 0x2a, 0x71, 0x6b, 0x24, 0xba, 0x7b, 0x38, 0xe2, 0xc6, 0x73,
0x02, 0x53, 0xeb, 0x68, 0xb3, 0xee, 0xe0, 0xc4, 0x60, 0x18, 0xf3, 0x5f, 0x2b, 0xfa, 0x5f, 0x38,
0x7f, 0xe5, 0xbd, 0xe7, 0xef, 0x75, 0xd0, 0x19, 0xf2, 0xa1, 0x17, 0x5b, 0x39, 0x38, 0x0a, 0x80,
0x69, 0xa5, 0x5f, 0xcd, 0x20, 0x5a, 0x86, 0x89, 0xad, 0x21, 0xb2, 0x91, 0x4c, 0xb7, 0x23, 0xf9,
0x57, 0xeb, 0x8c, 0xa7, 0x25, 0x31, 0x3e, 0xab, 0xb0, 0xc5, 0x56, 0x2f, 0x13, 0x75, 0x3e, 0x9a,
0x95, 0x5e, 0x7c, 0x34, 0xcb, 0xa0, 0xd2, 0x0e, 0x81, 0x6a, 0x4f, 0xc9, 0xb9, 0x06, 0x93, 0xd2,
0x73, 0x2b, 0x08, 0x7b, 0x98, 0x01, 0xd1, 0x90, 0xba, 0xfb, 0x52, 0x55, 0x44, 0xb3, 0xf2, 0x22,
0x68, 0x56, 0x0f, 0x46, 0x93, 0x42, 0x79, 0xe0, 0xc4, 0xea, 0x08, 0x4e, 0x9a, 0xf2, 0xd9, 0xf8,
0x04, 0x1a, 0x1b, 0x8e, 0x8f, 0x1b, 0x4e, 0xd7, 0x5d, 0xe3, 0xfd, 0x97, 0x81, 0x2b, 0xbf, 0x1b,
0x28, 0x15, 0xee, 0x06, 0x8e, 0x2c, 0xc6, 0xc6, 0x17, 0x04, 0xaa, 0xf7, 0x70, 0xb4, 0xb2, 0x8e,
0x7d, 0x89, 0x9d, 0x38, 0xfa, 0xe9, 0xbc, 0x2e, 0x05, 0x7a, 0x15, 0x1a, 0x63, 0xc9, 0x9e, 0x6c,
0x0e, 0x79, 0xae, 0x1f, 0x53, 0xed, 0x2f, 0x42, 0xcd, 0xe1, 0xd6, 0xb6, 0xed, 0x39, 0x3d, 0x89,
0x7d, 0xcd, 0xac, 0x3a, 0xfc, 0x43, 0x21, 0x8a, 0x63, 0x96, 0x55, 0x37, 0xde, 0x9a, 0x90, 0x87,
0x68, 0x4c, 0x63, 0x3c, 0x04, 0x48, 0x5c, 0x13, 0xd0, 0x64, 0xcc, 0x92, 0x71, 0x66, 0xdf, 0x81,
0xaa, 0x8b, 0xa3, 0x15, 0x8e, 0xfd, 0x56, 0x49, 0xd6, 0xa8, 0xc3, 0xf0, 0x4a, 0x76, 0x32, 0xd3,
0xe5, 0xc6, 0xe7, 0xea, 0xa6, 0x47, 0x18, 0x13, 0x39, 0xc4, 0x8b, 0xd5, 0x97, 0xec, 0xad, 0xbe,
0x57, 0xa1, 0xe1, 0xa3, 0x1f, 0xb2, 0x91, 0xc5, 0x9d, 0x5d, 0x4c, 0x61, 0x50, 0xaa, 0x75, 0x67,
0x17, 0x45, 0xa0, 0xc1, 0xd0, 0xb7, 0x58, 0xb8, 0xc3, 0xd3, 0xf3, 0x18, 0x0c, 0x7d, 0x33, 0xdc,
0xe1, 0xf4, 0x0d, 0x98, 0x61, 0xd8, 0xc5, 0x20, 0xf6, 0x46, 0x96, 0x1f, 0xf6, 0x9c, 0x47, 0x0e,
0xa6, 0x60, 0xe8, 0xe9, 0x8b, 0xb5, 0x44, 0x6f, 0x7c, 0x4d, 0x60, 0xe6, 0x83, 0x34, 0xfd, 0xd6,
0xb1, 0xaf, 0x9c, 0x3b, 0x81, 0xc4, 0x78, 0x57, 0xc6, 0x6b, 0x89, 0x83, 0xc3, 0x8f, 0xbf, 0xf9,
0xc9, 0x70, 0x32, 0x6b, 0x3c, 0x71, 0xea, 0xe6, 0x2f, 0x25, 0xa8, 0x26, 0xe6, 0x68, 0x1d, 0x26,
0xdc, 0xfb, 0x61, 0x80, 0xfa, 0x39, 0x3a, 0x0f, 0x33, 0xee, 0xde, 0x9b, 0x07, 0xbd, 0x47, 0x67,
0x61, 0xda, 0x2d, 0x0e, 0xee, 0x3a, 0x52, 0x0a, 0x4d, 0xb7, 0x30, 0xdb, 0xea, 0x8f, 0xe8, 0x05,
0x98, 0x75, 0xf7, 0x8f, 0x7f, 0xba, 0x48, 0x01, 0xdd, 0x2d, 0x4e, 0x48, 0x5c, 0x1f, 0xd0, 0x79,
0xd0, 0xdd, 0x3d, 0x43, 0x89, 0xfe, 0x1d, 0xa1, 0xb3, 0xd0, 0x74, 0x0b, 0xbd, 0xbb, 0xfe, 0x3d,
0xa1, 0x14, 0xa6, 0xdc, 0xf1, 0x16, 0x57, 0xff, 0x81, 0xd0, 0x0b, 0x40, 0xdd, 0x7d, 0x9d, 0xa0,
0xfe, 0x23, 0xa1, 0x73, 0x30, 0xed, 0x16, 0x1a, 0x26, 0xae, 0xff, 0x44, 0xe8, 0x24, 0x54, 0x5d,
0xd5, 0x55, 0xe8, 0x9f, 0x6a, 0x52, 0x52, 0xbf, 0x3b, 0xfd, 0x33, 0x25, 0xa9, 0xf2, 0xa7, 0x3f,
0xd7, 0xa4, 0xb1, 0xf1, 0x62, 0xa8, 0xff, 0xad, 0xd1, 0x26, 0xd4, 0xdd, 0xf4, 0xc0, 0xeb, 0xdf,
0xd4, 0xa5, 0xf1, 0x7d, 0x6c, 0xeb, 0xdf, 0xd6, 0x6f, 0xde, 0x86, 0x5a, 0x7a, 0x5f, 0x47, 0x01,
0x2a, 0x6b, 0x36, 0x8f, 0x91, 0xe9, 0xe7, 0xc4, 0xb3, 0x89, 0x76, 0x0f, 0x99, 0x4e, 0xc4, 0xf3,
0x47, 0xcc, 0x11, 0xfa, 0x92, 0xc0, 0xff, 0x81, 0xa8, 0x69, 0xba, 0x76, 0xa7, 0xfd, 0xf1, 0x9d,
0xbe, 0x13, 0x0f, 0x86, 0x9b, 0xa2, 0x46, 0x2e, 0xef, 0x3a, 0x9e, 0xe7, 0xec, 0xc6, 0xd8, 0x1d,
0x2c, 0x2b, 0x72, 0xdf, 0xec, 0x39, 0x3c, 0x66, 0xce, 0xe6, 0x30, 0xc6, 0xde, 0x72, 0x4a, 0xf1,
0xb2, 0x64, 0x3c, 0x13, 0xa3, 0xcd, 0xcd, 0x8a, 0xd4, 0xdc, 0xfa, 0x27, 0x00, 0x00, 0xff, 0xff,
0x56, 0x57, 0x32, 0x28, 0x20, 0x17, 0x00, 0x00,
}

17
internal/proto/service_msg.proto
View File

@ -136,24 +136,13 @@ message PartitionDescription {
}
/**
* @brief Scores of a query.
* The default value of tag is "root".
* It corresponds to the final score of each hit.
*/
message Score {
string tag = 1;
repeated float values = 2;
}
/**
* @brief Entities hit by query
*/
message Hits {
repeated int64 IDs = 1;
repeated common.Blob row_data = 2;
repeated Score scores = 3;
repeated bytes row_data = 2;
repeated float scores = 3;
}
@ -162,6 +151,6 @@ message Hits {
*/
message QueryResult {
common.Status status = 1;
repeated Hits hits = 2;
repeated bytes hits = 2;
}

171
internal/proto/servicepb/service_msg.pb.go
View File

@ -738,72 +738,21 @@ func (m *PartitionDescription) GetStatistics() []*commonpb.KeyValuePair {
}
//*
// @brief Scores of a query.
// The default value of tag is "root".
// It corresponds to the final score of each hit.
type Score struct {
Tag string `protobuf:"bytes,1,opt,name=tag,proto3" json:"tag,omitempty"`
Values []float32 `protobuf:"fixed32,2,rep,packed,name=values,proto3" json:"values,omitempty"`
// @brief Entities hit by query
type Hits struct {
IDs []int64 `protobuf:"varint,1,rep,packed,name=IDs,proto3" json:"IDs,omitempty"`
RowData [][]byte `protobuf:"bytes,2,rep,name=row_data,json=rowData,proto3" json:"row_data,omitempty"`
Scores []float32 `protobuf:"fixed32,3,rep,packed,name=scores,proto3" json:"scores,omitempty"`
XXX_NoUnkeyedLiteral struct{} `json:"-"`
XXX_unrecognized []byte `json:"-"`
XXX_sizecache int32 `json:"-"`
}
func (m *Score) Reset() { *m = Score{} }
func (m *Score) String() string { return proto.CompactTextString(m) }
func (*Score) ProtoMessage() {}
func (*Score) Descriptor() ([]byte, []int) {
return fileDescriptor_b4b40b84dd2f74cb, []int{13}
}
func (m *Score) XXX_Unmarshal(b []byte) error {
return xxx_messageInfo_Score.Unmarshal(m, b)
}
func (m *Score) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
return xxx_messageInfo_Score.Marshal(b, m, deterministic)
}
func (m *Score) XXX_Merge(src proto.Message) {
xxx_messageInfo_Score.Merge(m, src)
}
func (m *Score) XXX_Size() int {
return xxx_messageInfo_Score.Size(m)
}
func (m *Score) XXX_DiscardUnknown() {
xxx_messageInfo_Score.DiscardUnknown(m)
}
var xxx_messageInfo_Score proto.InternalMessageInfo
func (m *Score) GetTag() string {
if m != nil {
return m.Tag
}
return ""
}
func (m *Score) GetValues() []float32 {
if m != nil {
return m.Values
}
return nil
}
//*
// @brief Entities hit by query
type Hits struct {
IDs []int64 `protobuf:"varint,1,rep,packed,name=IDs,proto3" json:"IDs,omitempty"`
RowData []*commonpb.Blob `protobuf:"bytes,2,rep,name=row_data,json=rowData,proto3" json:"row_data,omitempty"`
Scores []*Score `protobuf:"bytes,3,rep,name=scores,proto3" json:"scores,omitempty"`
XXX_NoUnkeyedLiteral struct{} `json:"-"`
XXX_unrecognized []byte `json:"-"`
XXX_sizecache int32 `json:"-"`
}
func (m *Hits) Reset() { *m = Hits{} }
func (m *Hits) String() string { return proto.CompactTextString(m) }
func (*Hits) ProtoMessage() {}
func (*Hits) Descriptor() ([]byte, []int) {
return fileDescriptor_b4b40b84dd2f74cb, []int{14}
return fileDescriptor_b4b40b84dd2f74cb, []int{13}
}
func (m *Hits) XXX_Unmarshal(b []byte) error {
@ -831,14 +780,14 @@ func (m *Hits) GetIDs() []int64 {
return nil
}
func (m *Hits) GetRowData() []*commonpb.Blob {
func (m *Hits) GetRowData() [][]byte {
if m != nil {
return m.RowData
}
return nil
}
func (m *Hits) GetScores() []*Score {
func (m *Hits) GetScores() []float32 {
if m != nil {
return m.Scores
}
@ -849,7 +798,7 @@ func (m *Hits) GetScores() []*Score {
// @brief Query result
type QueryResult struct {
Status *commonpb.Status `protobuf:"bytes,1,opt,name=status,proto3" json:"status,omitempty"`
Hits []*Hits `protobuf:"bytes,2,rep,name=hits,proto3" json:"hits,omitempty"`
Hits [][]byte `protobuf:"bytes,2,rep,name=hits,proto3" json:"hits,omitempty"`
XXX_NoUnkeyedLiteral struct{} `json:"-"`
XXX_unrecognized []byte `json:"-"`
XXX_sizecache int32 `json:"-"`
@ -859,7 +808,7 @@ func (m *QueryResult) Reset() { *m = QueryResult{} }
func (m *QueryResult) String() string { return proto.CompactTextString(m) }
func (*QueryResult) ProtoMessage() {}
func (*QueryResult) Descriptor() ([]byte, []int) {
return fileDescriptor_b4b40b84dd2f74cb, []int{15}
return fileDescriptor_b4b40b84dd2f74cb, []int{14}
}
func (m *QueryResult) XXX_Unmarshal(b []byte) error {
@ -887,7 +836,7 @@ func (m *QueryResult) GetStatus() *commonpb.Status {
return nil
}
func (m *QueryResult) GetHits() []*Hits {
func (m *QueryResult) GetHits() [][]byte {
if m != nil {
return m.Hits
}
@ -909,7 +858,6 @@ func init() {
proto.RegisterType((*IntegerRangeResponse)(nil), "milvus.proto.service.IntegerRangeResponse")
proto.RegisterType((*CollectionDescription)(nil), "milvus.proto.service.CollectionDescription")
proto.RegisterType((*PartitionDescription)(nil), "milvus.proto.service.PartitionDescription")
proto.RegisterType((*Score)(nil), "milvus.proto.service.Score")
proto.RegisterType((*Hits)(nil), "milvus.proto.service.Hits")
proto.RegisterType((*QueryResult)(nil), "milvus.proto.service.QueryResult")
}
@ -917,53 +865,52 @@ func init() {
func init() { proto.RegisterFile("service_msg.proto", fileDescriptor_b4b40b84dd2f74cb) }
var fileDescriptor_b4b40b84dd2f74cb = []byte{
// 763 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x54, 0x5d, 0x6f, 0xf3, 0x34,
0x14, 0x26, 0x4d, 0x5b, 0xba, 0xd3, 0xb4, 0x6f, 0x67, 0x0a, 0x0a, 0xdb, 0x4d, 0xc9, 0xab, 0x41,
0x05, 0xa2, 0x15, 0x1b, 0x12, 0xda, 0x05, 0x12, 0xed, 0x36, 0x60, 0x1f, 0xea, 0x86, 0x5b, 0x4d,
0x1a, 0x48, 0x54, 0x6e, 0x62, 0x25, 0x16, 0x69, 0x1c, 0xd9, 0xce, 0xaa, 0xee, 0x96, 0xff, 0xc0,
0x9f, 0xe0, 0x87, 0x70, 0xc7, 0x6f, 0x42, 0x71, 0xd2, 0xaf, 0x51, 0xc4, 0xde, 0x6d, 0x77, 0xf6,
0xb1, 0xcf, 0x79, 0x9e, 0xf3, 0xf8, 0xf8, 0x81, 0x5d, 0x49, 0xc5, 0x3d, 0x73, 0xe9, 0x78, 0x2a,
0xfd, 0x4e, 0x2c, 0xb8, 0xe2, 0xa8, 0x39, 0x65, 0xe1, 0x7d, 0x22, 0xb3, 0x5d, 0x27, 0x3f, 0xdf,
0xb3, 0x5c, 0x3e, 0x9d, 0xf2, 0x28, 0x8b, 0xee, 0x59, 0xd2, 0x0d, 0xe8, 0x94, 0x64, 0x3b, 0xe7,
0x18, 0xea, 0x27, 0x3c, 0x0c, 0xa9, 0xab, 0x18, 0x8f, 0x06, 0x64, 0x4a, 0xd1, 0x67, 0xf0, 0xc6,
0x5d, 0x46, 0xc6, 0x11, 0x99, 0x52, 0xdb, 0x68, 0x19, 0xed, 0x1d, 0x5c, 0x77, 0x37, 0x2e, 0x3a,
0x17, 0x50, 0xbb, 0x21, 0x42, 0xb1, 0x77, 0xce, 0x44, 0x0d, 0x30, 0x15, 0xf1, 0xed, 0x82, 0x3e,
0x4c, 0x97, 0xce, 0x9f, 0x06, 0x54, 0x30, 0x9f, 0xf5, 0x89, 0x72, 0x83, 0xa7, 0xd7, 0x79, 0x0b,
0xb5, 0x78, 0xc1, 0x60, 0xbc, 0xaa, 0x68, 0x2d, 0x83, 0x23, 0xe2, 0xa3, 0xaf, 0xa1, 0x22, 0xf8,
0x6c, 0xec, 0x11, 0x45, 0x6c, 0xb3, 0x65, 0xb6, 0xab, 0x87, 0x1f, 0x77, 0x36, 0x64, 0xca, 0xd5,
0xe9, 0x87, 0x7c, 0x82, 0xdf, 0x17, 0x7c, 0x76, 0x4a, 0x14, 0x41, 0xfb, 0xb0, 0x13, 0x10, 0x19,
0x8c, 0x7f, 0xa3, 0x73, 0x69, 0x17, 0x5b, 0x66, 0xbb, 0x84, 0x2b, 0x69, 0xe0, 0x92, 0xce, 0xa5,
0x33, 0x83, 0xc6, 0x4d, 0x48, 0x5c, 0x1a, 0xf0, 0xd0, 0xa3, 0xe2, 0x96, 0x84, 0xc9, 0xb2, 0x27,
0x63, 0xd9, 0x13, 0x3a, 0x86, 0xa2, 0x9a, 0xc7, 0x54, 0x93, 0xaa, 0x1f, 0x1e, 0x74, 0xb6, 0xbd,
0x4d, 0x67, 0xad, 0xce, 0x68, 0x1e, 0x53, 0xac, 0x53, 0xd0, 0x47, 0x50, 0xbe, 0x4f, 0xab, 0x4a,
0xcd, 0xd8, 0xc2, 0xf9, 0xce, 0xf9, 0x75, 0x03, 0xf8, 0x07, 0xc1, 0x93, 0x18, 0x5d, 0x80, 0x15,
0xaf, 0x62, 0xd2, 0x36, 0x74, 0x8f, 0x9f, 0xfe, 0x2f, 0x9c, 0xa6, 0x8d, 0x37, 0x72, 0x9d, 0x3f,
0x0c, 0x28, 0xfd, 0x94, 0x50, 0x31, 0x7f, 0xfa, 0x1b, 0x1c, 0x40, 0x7d, 0xe3, 0x0d, 0xa4, 0x5d,
0x68, 0x99, 0xed, 0x1d, 0x5c, 0x5b, 0x7f, 0x04, 0x99, 0xca, 0xe3, 0xc9, 0xd0, 0x36, 0x33, 0x79,
0x3c, 0x19, 0xa2, 0x2f, 0x60, 0x77, 0x0d, 0x7b, 0xec, 0xa7, 0xcd, 0xd8, 0xc5, 0x96, 0xd1, 0xb6,
0x70, 0x23, 0x7e, 0xd4, 0xa4, 0xf3, 0x0b, 0xd4, 0x87, 0x4a, 0xb0, 0xc8, 0xc7, 0x54, 0xc6, 0x3c,
0x92, 0x14, 0x1d, 0x41, 0x59, 0x2a, 0xa2, 0x12, 0xa9, 0x79, 0x55, 0x0f, 0xf7, 0xb7, 0x3e, 0xea,
0x50, 0x5f, 0xc1, 0xf9, 0x55, 0xd4, 0x84, 0x92, 0x56, 0x32, 0x1f, 0x94, 0x6c, 0xe3, 0xdc, 0x81,
0xd5, 0xe7, 0x3c, 0x7c, 0xc5, 0xd2, 0x95, 0x45, 0x69, 0x02, 0x28, 0xe3, 0x7d, 0xc5, 0xa4, 0x7a,
0x19, 0xc0, 0x6a, 0x26, 0x32, 0x81, 0x17, 0x33, 0x31, 0x81, 0x0f, 0xce, 0x23, 0x45, 0x7d, 0x2a,
0x5e, 0x1b, 0xc3, 0x5c, 0x62, 0x48, 0x68, 0xe6, 0x18, 0x98, 0x44, 0x3e, 0x7d, 0xb1, 0x52, 0x13,
0xea, 0xb3, 0x48, 0x2b, 0x65, 0xe2, 0x6c, 0x93, 0x0e, 0x08, 0x8d, 0x3c, 0x3d, 0x20, 0x26, 0x4e,
0x97, 0xce, 0xdf, 0x06, 0x7c, 0xb8, 0xf2, 0xa6, 0x53, 0x2a, 0x5d, 0xc1, 0xe2, 0x74, 0xf9, 0x3c,
0xd8, 0x6f, 0xa1, 0x9c, 0x39, 0x9f, 0xc6, 0xad, 0xfe, 0xeb, 0x43, 0x66, 0xae, 0xb8, 0x02, 0x1c,
0xea, 0x00, 0xce, 0x93, 0x50, 0x0f, 0x20, 0x2d, 0xc4, 0xa4, 0x62, 0xae, 0xcc, 0x8d, 0xe4, 0x93,
0xad, 0xb8, 0x97, 0x74, 0xae, 0xff, 0xd6, 0x0d, 0x61, 0x02, 0xaf, 0x25, 0x39, 0x7f, 0x19, 0xd0,
0x5c, 0x3a, 0xe6, 0x8b, 0xfb, 0xf9, 0x06, 0x8a, 0xfa, 0x5b, 0x66, 0xdd, 0xbc, 0xfd, 0x8f, 0xff,
0xbe, 0x6e, 0xd0, 0x58, 0x27, 0xbc, 0x46, 0x27, 0x5f, 0x41, 0x69, 0xe8, 0x72, 0xb1, 0xcd, 0xf5,
0x36, 0x47, 0xa8, 0xb0, 0x1c, 0xa1, 0xdf, 0x0d, 0x28, 0xfe, 0xc8, 0x94, 0x76, 0x82, 0xf3, 0xd3,
0xcc, 0xa6, 0x4c, 0x9c, 0x2e, 0x37, 0x1c, 0xba, 0xf0, 0x64, 0x87, 0x4e, 0x45, 0x4b, 0x39, 0x2c,
0x5a, 0xd8, 0xdf, 0xae, 0x80, 0xe6, 0x89, 0xf3, 0xab, 0x8e, 0x80, 0xaa, 0xf6, 0x37, 0x4c, 0x65,
0x12, 0xaa, 0xe7, 0x09, 0xdf, 0x81, 0x62, 0xc0, 0x94, 0xcc, 0xa9, 0xee, 0x6d, 0x87, 0x4d, 0x5b,
0xc5, 0xfa, 0xde, 0xe7, 0xdf, 0xc1, 0x9b, 0x47, 0x2e, 0x8f, 0x2a, 0x50, 0x1c, 0x5c, 0x0f, 0xce,
0x1a, 0xef, 0xa1, 0x5d, 0xa8, 0xdd, 0x9e, 0x9d, 0x8c, 0xae, 0xf1, 0xb8, 0x7f, 0x3e, 0xe8, 0xe1,
0xbb, 0x86, 0x87, 0x1a, 0x60, 0xe5, 0xa1, 0xef, 0xaf, 0xae, 0x7b, 0xa3, 0x06, 0xed, 0x9f, 0xfc,
0xdc, 0xf3, 0x99, 0x0a, 0x92, 0x49, 0xca, 0xa8, 0xfb, 0xc0, 0xc2, 0x90, 0x3d, 0x28, 0xea, 0x06,
0xdd, 0x0c, 0xfa, 0x4b, 0x8f, 0x49, 0x25, 0xd8, 0x24, 0x51, 0xd4, 0xeb, 0xb2, 0x48, 0x51, 0x11,
0x91, 0xb0, 0xab, 0xf9, 0x74, 0x73, 0x3e, 0xf1, 0x64, 0x52, 0xd6, 0x81, 0xa3, 0x7f, 0x02, 0x00,
0x00, 0xff, 0xff, 0x12, 0x37, 0x33, 0x02, 0x37, 0x08, 0x00, 0x00,
// 739 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x54, 0xdd, 0x4e, 0xdb, 0x4a,
0x10, 0x3e, 0x8e, 0x43, 0x4e, 0x98, 0x38, 0x21, 0xec, 0xc9, 0x41, 0x06, 0x6e, 0x72, 0x8c, 0x38,
0x8d, 0x5a, 0x35, 0x91, 0xa0, 0x52, 0xc5, 0x45, 0xa5, 0x26, 0x40, 0x5b, 0x7e, 0x14, 0xe8, 0x12,
0x21, 0xd1, 0x4a, 0x8d, 0x36, 0xf6, 0xca, 0x5e, 0xd5, 0xf1, 0x5a, 0xde, 0x35, 0x51, 0x78, 0x90,
0xbe, 0x44, 0x1f, 0xa4, 0x77, 0x7d, 0xa6, 0xca, 0x6b, 0x93, 0x1f, 0x4a, 0x55, 0x0a, 0xdc, 0xcd,
0xcc, 0xee, 0xcc, 0x37, 0xbf, 0x1f, 0x2c, 0x0b, 0x1a, 0x5d, 0x32, 0x9b, 0xf6, 0x87, 0xc2, 0x6d,
0x86, 0x11, 0x97, 0x1c, 0xd5, 0x86, 0xcc, 0xbf, 0x8c, 0x45, 0xaa, 0x35, 0xb3, 0xf7, 0x35, 0xc3,
0xe6, 0xc3, 0x21, 0x0f, 0x52, 0xeb, 0x9a, 0x21, 0x6c, 0x8f, 0x0e, 0x49, 0xaa, 0x59, 0x3b, 0x50,
0xd9, 0xe5, 0xbe, 0x4f, 0x6d, 0xc9, 0x78, 0xd0, 0x25, 0x43, 0x8a, 0x9e, 0xc0, 0x92, 0x3d, 0xb1,
0xf4, 0x03, 0x32, 0xa4, 0xa6, 0x56, 0xd7, 0x1a, 0x8b, 0xb8, 0x62, 0xcf, 0x7d, 0xb4, 0x0e, 0xa1,
0x7c, 0x4a, 0x22, 0xc9, 0xfe, 0xd8, 0x13, 0x55, 0x41, 0x97, 0xc4, 0x35, 0x73, 0xea, 0x31, 0x11,
0xad, 0xaf, 0x1a, 0x14, 0x31, 0x1f, 0x75, 0x88, 0xb4, 0xbd, 0xbb, 0xc7, 0xd9, 0x80, 0x72, 0x78,
0x9d, 0x41, 0x7f, 0x1a, 0xd1, 0x98, 0x18, 0x7b, 0xc4, 0x45, 0x2f, 0xa0, 0x18, 0xf1, 0x51, 0xdf,
0x21, 0x92, 0x98, 0x7a, 0x5d, 0x6f, 0x94, 0xb6, 0x56, 0x9b, 0x73, 0x6d, 0xca, 0xba, 0xd3, 0xf1,
0xf9, 0x00, 0xff, 0x1d, 0xf1, 0xd1, 0x1e, 0x91, 0x04, 0xad, 0xc3, 0xa2, 0x47, 0x84, 0xd7, 0xff,
0x4c, 0xc7, 0xc2, 0xcc, 0xd7, 0xf5, 0xc6, 0x02, 0x2e, 0x26, 0x86, 0x23, 0x3a, 0x16, 0xd6, 0x08,
0xaa, 0xa7, 0x3e, 0xb1, 0xa9, 0xc7, 0x7d, 0x87, 0x46, 0xe7, 0xc4, 0x8f, 0x27, 0x35, 0x69, 0x93,
0x9a, 0xd0, 0x0e, 0xe4, 0xe5, 0x38, 0xa4, 0x2a, 0xa9, 0xca, 0xd6, 0x66, 0xf3, 0xb6, 0xd9, 0x34,
0x67, 0xe2, 0xf4, 0xc6, 0x21, 0xc5, 0xca, 0x05, 0xad, 0x40, 0xe1, 0x32, 0x89, 0x2a, 0x54, 0xc6,
0x06, 0xce, 0x34, 0xeb, 0xd3, 0x1c, 0xf0, 0xdb, 0x88, 0xc7, 0x21, 0x3a, 0x04, 0x23, 0x9c, 0xda,
0x84, 0xa9, 0xa9, 0x1a, 0xff, 0xff, 0x2d, 0x9c, 0x4a, 0x1b, 0xcf, 0xf9, 0x5a, 0x5f, 0x34, 0x58,
0x78, 0x1f, 0xd3, 0x68, 0x7c, 0xf7, 0x19, 0x6c, 0x42, 0x65, 0x6e, 0x06, 0xc2, 0xcc, 0xd5, 0xf5,
0xc6, 0x22, 0x2e, 0xcf, 0x0e, 0x41, 0x24, 0xed, 0x71, 0x84, 0x6f, 0xea, 0x69, 0x7b, 0x1c, 0xe1,
0xa3, 0x67, 0xb0, 0x3c, 0x83, 0xdd, 0x77, 0x93, 0x62, 0xcc, 0x7c, 0x5d, 0x6b, 0x18, 0xb8, 0x1a,
0xde, 0x28, 0xd2, 0xfa, 0x08, 0x95, 0x33, 0x19, 0xb1, 0xc0, 0xc5, 0x54, 0x84, 0x3c, 0x10, 0x14,
0x6d, 0x43, 0x41, 0x48, 0x22, 0x63, 0xa1, 0xf2, 0x2a, 0x6d, 0xad, 0xdf, 0x3a, 0xd4, 0x33, 0xf5,
0x05, 0x67, 0x5f, 0x51, 0x0d, 0x16, 0x54, 0x27, 0xb3, 0x45, 0x49, 0x15, 0xeb, 0x02, 0x8c, 0x0e,
0xe7, 0xfe, 0x23, 0x86, 0x2e, 0x5e, 0x87, 0x26, 0x80, 0xd2, 0xbc, 0x8f, 0x99, 0x90, 0x0f, 0x03,
0x98, 0xee, 0x44, 0xda, 0xe0, 0xeb, 0x9d, 0x18, 0xc0, 0x3f, 0x07, 0x81, 0xa4, 0x2e, 0x8d, 0x1e,
0x1b, 0x43, 0x9f, 0x60, 0x08, 0xa8, 0x65, 0x18, 0x98, 0x04, 0x2e, 0x7d, 0x70, 0xa7, 0x06, 0xd4,
0x65, 0x81, 0xea, 0x94, 0x8e, 0x53, 0x25, 0x59, 0x10, 0x1a, 0x38, 0x6a, 0x41, 0x74, 0x9c, 0x88,
0xd6, 0x77, 0x0d, 0xfe, 0x9d, 0x72, 0xd3, 0x1e, 0x15, 0x76, 0xc4, 0xc2, 0x44, 0xbc, 0x1f, 0xec,
0x2b, 0x28, 0xa4, 0xcc, 0xa7, 0x70, 0x4b, 0x3f, 0x1d, 0x64, 0xca, 0x8a, 0x53, 0xc0, 0x33, 0x65,
0xc0, 0x99, 0x13, 0x6a, 0x03, 0x24, 0x81, 0x98, 0x90, 0xcc, 0x16, 0x19, 0x91, 0xfc, 0x77, 0x2b,
0xee, 0x11, 0x1d, 0xab, 0xdb, 0x3a, 0x25, 0x2c, 0xc2, 0x33, 0x4e, 0xd6, 0x37, 0x0d, 0x6a, 0x13,
0xc6, 0x7c, 0x70, 0x3d, 0x2f, 0x21, 0xaf, 0xce, 0x32, 0xad, 0x66, 0xe3, 0x17, 0xf7, 0x3e, 0x4b,
0xd0, 0x58, 0x39, 0x3c, 0x46, 0x25, 0x47, 0x90, 0x7f, 0xc7, 0xa4, 0xba, 0xea, 0x83, 0xbd, 0x94,
0x72, 0x74, 0x9c, 0x88, 0x68, 0x75, 0x86, 0x6d, 0x73, 0x8a, 0xbb, 0x26, 0x94, 0xba, 0x92, 0x0c,
0x80, 0x47, 0x19, 0xa9, 0xe5, 0x70, 0xa6, 0x59, 0xe7, 0x50, 0x52, 0x9c, 0x83, 0xa9, 0x88, 0x7d,
0x79, 0xbf, 0x66, 0x20, 0xc8, 0x7b, 0x4c, 0x8a, 0x0c, 0x52, 0xc9, 0x4f, 0x5f, 0xc3, 0xd2, 0x0d,
0x76, 0x45, 0x45, 0xc8, 0x77, 0x4f, 0xba, 0xfb, 0xd5, 0xbf, 0xd0, 0x32, 0x94, 0xcf, 0xf7, 0x77,
0x7b, 0x27, 0xb8, 0xdf, 0x39, 0xe8, 0xb6, 0xf1, 0x45, 0xd5, 0x41, 0x55, 0x30, 0x32, 0xd3, 0x9b,
0xe3, 0x93, 0x76, 0xaf, 0x4a, 0x3b, 0xbb, 0x1f, 0xda, 0x2e, 0x93, 0x5e, 0x3c, 0x48, 0x50, 0x5b,
0x57, 0xcc, 0xf7, 0xd9, 0x95, 0xa4, 0xb6, 0xd7, 0x4a, 0x33, 0x7a, 0xee, 0x30, 0x21, 0x23, 0x36,
0x88, 0x25, 0x75, 0x5a, 0x2c, 0x90, 0x34, 0x0a, 0x88, 0xdf, 0x52, 0x69, 0xb6, 0xb2, 0x01, 0x84,
0x83, 0x41, 0x41, 0x19, 0xb6, 0x7f, 0x04, 0x00, 0x00, 0xff, 0xff, 0x33, 0xc8, 0x08, 0xe2, 0xaf,
0x07, 0x00, 0x00,
}

12
internal/proxy/paramtable.go
View File

@ -463,3 +463,15 @@ func (pt *ParamTable) MaxNameLength() int64 {
}
return maxNameLength
}
func (pt *ParamTable) MaxFieldNum() int64 {
str, err := pt.Load("proxy.maxFieldNum")
if err != nil {
panic(err)
}
maxFieldNum, err := strconv.ParseInt(str, 10, 64)
if err != nil {
panic(err)
}
return maxFieldNum
}

44
internal/proxy/task.go
View File

@ -4,8 +4,11 @@ import (
"context"
"errors"
"log"
"math"
"strconv"
"github.com/golang/protobuf/proto"
"github.com/zilliztech/milvus-distributed/internal/allocator"
"github.com/zilliztech/milvus-distributed/internal/msgstream"
"github.com/zilliztech/milvus-distributed/internal/proto/commonpb"
@ -164,6 +167,10 @@ func (cct *CreateCollectionTask) SetTs(ts Timestamp) {
}
func (cct *CreateCollectionTask) PreExecute() error {
if int64(len(cct.schema.Fields)) > Params.MaxFieldNum() {
return errors.New("maximum field's number should be limited to " + strconv.FormatInt(Params.MaxFieldNum(), 10))
}
// validate collection name
if err := ValidateCollectionName(cct.schema.Name); err != nil {
return err
@ -338,7 +345,17 @@ func (qt *QueryTask) PostExecute() error {
qt.result = &servicepb.QueryResult{}
return nil
}
k := len(searchResults[0].Hits[0].IDs) // k
var hits [][]*servicepb.Hits = make([][]*servicepb.Hits, rlen)
for i, sr := range searchResults {
hits[i] = make([]*servicepb.Hits, n)
for j, bs := range sr.Hits {
err := proto.Unmarshal(bs, hits[i][j])
if err != nil {
return err
}
}
}
k := len(hits[0][0].IDs)
queryResult := &servicepb.QueryResult{
Status: &commonpb.Status{
ErrorCode: 0,
@ -351,26 +368,27 @@ func (qt *QueryTask) PostExecute() error {
// len(queryResult.Hits[i].Ids) == k for i in range(n)
for i := 0; i < n; n++ { // n
locs := make([]int, rlen)
hits := &servicepb.Hits{}
reducedHits := &servicepb.Hits{}
for j := 0; j < k; j++ { // k
choice, maxScore := 0, float32(0)
choice, minDistance := 0, float32(math.MaxFloat32)
for q, loc := range locs { // query num, the number of ways to merge
score := func(score *servicepb.Score) float32 {
// TODO: get score of root
return 0.0
}(searchResults[q].Hits[i].Scores[loc])
if score > maxScore {
distance := hits[q][i].Scores[loc]
if distance < minDistance {
choice = q
maxScore = score
minDistance = distance
}
}
choiceOffset := locs[choice]
hits.IDs = append(hits.IDs, searchResults[choice].Hits[i].IDs[choiceOffset])
hits.RowData = append(hits.RowData, searchResults[choice].Hits[i].RowData[choiceOffset])
hits.Scores = append(hits.Scores, searchResults[choice].Hits[i].Scores[choiceOffset])
reducedHits.IDs = append(reducedHits.IDs, hits[choice][i].IDs[choiceOffset])
reducedHits.RowData = append(reducedHits.RowData, hits[choice][i].RowData[choiceOffset])
reducedHits.Scores = append(reducedHits.Scores, hits[choice][i].Scores[choiceOffset])
locs[choice]++
}
queryResult.Hits = append(queryResult.Hits, hits)
reducedHitsBs, err := proto.Marshal(reducedHits)
if err != nil {
return err
}
queryResult.Hits = append(queryResult.Hits, reducedHitsBs)
}
qt.result = queryResult
}

2
internal/proxy/validate_util.go
View File

@ -68,7 +68,7 @@ func ValidatePartitionTag(partitionTag string, strictCheck bool) error {
if strictCheck {
firstChar := partitionTag[0]
if firstChar != '_' && !isAlpha(firstChar) {
if firstChar != '_' && !isAlpha(firstChar) && !isNumber(firstChar) {
msg := invalidMsg + "The first character of a partition tag must be an underscore or letter."
return errors.New(msg)
}

18
internal/querynode/param_table.go
View File

@ -20,6 +20,12 @@ func (p *ParamTable) Init() {
if err != nil {
panic(err)
}
err = p.LoadYaml("milvus.yaml")
if err != nil {
panic(err)
}
err = p.LoadYaml("advanced/channel.yaml")
if err != nil {
panic(err)
@ -201,6 +207,18 @@ func (p *ParamTable) etcdRootPath() string {
return etcdRootPath
}
func (p *ParamTable) gracefulTime() int64 {
gracefulTime, err := p.Load("queryNode.gracefulTime")
if err != nil {
panic(err)
}
time, err := strconv.Atoi(gracefulTime)
if err != nil {
panic(err)
}
return int64(time)
}
func (p *ParamTable) insertChannelNames() []string {
ch, err := p.Load("msgChannel.chanNamePrefix.insert")
if err != nil {

13
internal/querynode/plan.go
View File

@ -3,6 +3,7 @@ package querynode
/*
#cgo CFLAGS: -I${SRCDIR}/../core/output/include
#cgo LDFLAGS: -L${SRCDIR}/../core/output/lib -lmilvus_segcore -Wl,-rpath=${SRCDIR}/../core/output/lib
#include "segcore/collection_c.h"
#include "segcore/segment_c.h"
#include "segcore/plan_c.h"
@ -16,19 +17,19 @@ type Plan struct {
cPlan C.CPlan
}
func CreatePlan(col Collection, dsl string) *Plan {
func createPlan(col Collection, dsl string) *Plan {
cDsl := C.CString(dsl)
cPlan := C.CreatePlan(col.collectionPtr, cDsl)
var newPlan = &Plan{cPlan: cPlan}
return newPlan
}
func (plan *Plan) GetTopK() int64 {
func (plan *Plan) getTopK() int64 {
topK := C.GetTopK(plan.cPlan)
return int64(topK)
}
func (plan *Plan) Delete() {
func (plan *Plan) delete() {
C.DeletePlan(plan.cPlan)
}
@ -36,7 +37,7 @@ type PlaceholderGroup struct {
cPlaceholderGroup C.CPlaceholderGroup
}
func ParserPlaceholderGroup(plan *Plan, placeHolderBlob []byte) *PlaceholderGroup {
func parserPlaceholderGroup(plan *Plan, placeHolderBlob []byte) *PlaceholderGroup {
var blobPtr = unsafe.Pointer(&placeHolderBlob[0])
blobSize := C.long(len(placeHolderBlob))
cPlaceholderGroup := C.ParsePlaceholderGroup(plan.cPlan, blobPtr, blobSize)
@ -44,11 +45,11 @@ func ParserPlaceholderGroup(plan *Plan, placeHolderBlob []byte) *PlaceholderGrou
return newPlaceholderGroup
}
func (pg *PlaceholderGroup) GetNumOfQuery() int64 {
func (pg *PlaceholderGroup) getNumOfQuery() int64 {
numQueries := C.GetNumOfQueries(pg.cPlaceholderGroup)
return int64(numQueries)
}
func (pg *PlaceholderGroup) Delete() {
func (pg *PlaceholderGroup) delete() {
C.DeletePlaceholderGroup(pg.cPlaceholderGroup)
}

156
internal/querynode/plan_test.go Normal file
View File

@ -0,0 +1,156 @@
package querynode
import (
"encoding/binary"
"math"
"testing"
"github.com/golang/protobuf/proto"
"github.com/stretchr/testify/assert"
"github.com/zilliztech/milvus-distributed/internal/proto/commonpb"
"github.com/zilliztech/milvus-distributed/internal/proto/etcdpb"
"github.com/zilliztech/milvus-distributed/internal/proto/schemapb"
"github.com/zilliztech/milvus-distributed/internal/proto/servicepb"
)
func TestPlan_Plan(t *testing.T) {
fieldVec := schemapb.FieldSchema{
Name: "vec",
DataType: schemapb.DataType_VECTOR_FLOAT,
TypeParams: []*commonpb.KeyValuePair{
{
Key: "dim",
Value: "16",
},
},
}
fieldInt := schemapb.FieldSchema{
Name: "age",
DataType: schemapb.DataType_INT32,
TypeParams: []*commonpb.KeyValuePair{
{
Key: "dim",
Value: "1",
},
},
}
schema := schemapb.CollectionSchema{
Name: "collection0",
Fields: []*schemapb.FieldSchema{
&fieldVec, &fieldInt,
},
}
collectionMeta := etcdpb.CollectionMeta{
ID: UniqueID(0),
Schema: &schema,
CreateTime: Timestamp(0),
SegmentIDs: []UniqueID{0},
PartitionTags: []string{"default"},
}
collectionMetaBlob := proto.MarshalTextString(&collectionMeta)
assert.NotEqual(t, "", collectionMetaBlob)
collection := newCollection(&collectionMeta, collectionMetaBlob)
assert.Equal(t, collection.meta.Schema.Name, "collection0")
assert.Equal(t, collection.meta.ID, UniqueID(0))
dslString := "{\"bool\": { \n\"vector\": {\n \"vec\": {\n \"metric_type\": \"L2\", \n \"params\": {\n \"nprobe\": 10 \n},\n \"query\": \"$0\",\"topk\": 10 \n } \n } \n } \n }"
plan := createPlan(*collection, dslString)
assert.NotEqual(t, plan, nil)
topk := plan.getTopK()
assert.Equal(t, int(topk), 10)
plan.delete()
deleteCollection(collection)
}
func TestPlan_PlaceholderGroup(t *testing.T) {
fieldVec := schemapb.FieldSchema{
Name: "vec",
DataType: schemapb.DataType_VECTOR_FLOAT,
TypeParams: []*commonpb.KeyValuePair{
{
Key: "dim",
Value: "16",
},
},
}
fieldInt := schemapb.FieldSchema{
Name: "age",
DataType: schemapb.DataType_INT32,
TypeParams: []*commonpb.KeyValuePair{
{
Key: "dim",
Value: "1",
},
},
}
schema := schemapb.CollectionSchema{
Name: "collection0",
Fields: []*schemapb.FieldSchema{
&fieldVec, &fieldInt,
},
}
collectionMeta := etcdpb.CollectionMeta{
ID: UniqueID(0),
Schema: &schema,
CreateTime: Timestamp(0),
SegmentIDs: []UniqueID{0},
PartitionTags: []string{"default"},
}
collectionMetaBlob := proto.MarshalTextString(&collectionMeta)
assert.NotEqual(t, "", collectionMetaBlob)
collection := newCollection(&collectionMeta, collectionMetaBlob)
assert.Equal(t, collection.meta.Schema.Name, "collection0")
assert.Equal(t, collection.meta.ID, UniqueID(0))
dslString := "{\"bool\": { \n\"vector\": {\n \"vec\": {\n \"metric_type\": \"L2\", \n \"params\": {\n \"nprobe\": 10 \n},\n \"query\": \"$0\",\"topk\": 10 \n } \n } \n } \n }"
plan := createPlan(*collection, dslString)
assert.NotNil(t, plan)
var searchRawData1 []byte
var searchRawData2 []byte
const DIM = 16
var vec = [DIM]float32{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16}
for i, ele := range vec {
buf := make([]byte, 4)
binary.LittleEndian.PutUint32(buf, math.Float32bits(ele+float32(i*2)))
searchRawData1 = append(searchRawData1, buf...)
}
for i, ele := range vec {
buf := make([]byte, 4)
binary.LittleEndian.PutUint32(buf, math.Float32bits(ele+float32(i*4)))
searchRawData2 = append(searchRawData2, buf...)
}
placeholderValue := servicepb.PlaceholderValue{
Tag: "$0",
Type: servicepb.PlaceholderType_VECTOR_FLOAT,
Values: [][]byte{searchRawData1, searchRawData2},
}
placeholderGroup := servicepb.PlaceholderGroup{
Placeholders: []*servicepb.PlaceholderValue{&placeholderValue},
}
placeGroupByte, err := proto.Marshal(&placeholderGroup)
assert.Nil(t, err)
holder := parserPlaceholderGroup(plan, placeGroupByte)
assert.NotNil(t, holder)
numQueries := holder.getNumOfQuery()
assert.Equal(t, int(numQueries), 2)
plan.delete()
holder.delete()
deleteCollection(collection)
}

76
internal/querynode/reduce.go Normal file
View File

@ -0,0 +1,76 @@
package querynode
/*
#cgo CFLAGS: -I${SRCDIR}/../core/output/include
#cgo LDFLAGS: -L${SRCDIR}/../core/output/lib -lmilvus_segcore -Wl,-rpath=${SRCDIR}/../core/output/lib
#include "segcore/plan_c.h"
#include "segcore/reduce_c.h"
*/
import "C"
import (
"unsafe"
)
type SearchResult struct {
cQueryResult C.CQueryResult
}
type MarshaledHits struct {
cMarshaledHits C.CMarshaledHits
}
func reduceSearchResults(searchResults []*SearchResult, numSegments int64) *SearchResult {
cSearchResults := make([]C.CQueryResult, 0)
for _, res := range searchResults {
cSearchResults = append(cSearchResults, res.cQueryResult)
}
cSearchResultPtr := (*C.CQueryResult)(&cSearchResults[0])
cNumSegments := C.long(numSegments)
res := C.ReduceQueryResults(cSearchResultPtr, cNumSegments)
return &SearchResult{cQueryResult: res}
}
func (sr *SearchResult) reorganizeQueryResults(plan *Plan, placeholderGroups []*PlaceholderGroup) *MarshaledHits {
cPlaceholderGroups := make([]C.CPlaceholderGroup, 0)
for _, pg := range placeholderGroups {
cPlaceholderGroups = append(cPlaceholderGroups, (*pg).cPlaceholderGroup)
}
cNumGroup := (C.long)(len(placeholderGroups))
var cPlaceHolder = (*C.CPlaceholderGroup)(&cPlaceholderGroups[0])
res := C.ReorganizeQueryResults(sr.cQueryResult, plan.cPlan, cPlaceHolder, cNumGroup)
return &MarshaledHits{cMarshaledHits: res}
}
func (mh *MarshaledHits) getHitsBlobSize() int64 {
res := C.GetHitsBlobSize(mh.cMarshaledHits)
return int64(res)
}
func (mh *MarshaledHits) getHitsBlob() ([]byte, error) {
byteSize := mh.getHitsBlobSize()
result := make([]byte, byteSize)
cResultPtr := unsafe.Pointer(&result[0])
C.GetHitsBlob(mh.cMarshaledHits, cResultPtr)
return result, nil
}
func (mh *MarshaledHits) hitBlobSizeInGroup(groupOffset int64) ([]int64, error) {
cGroupOffset := (C.long)(groupOffset)
numQueries := C.GetNumQueriesPeerGroup(mh.cMarshaledHits, cGroupOffset)
result := make([]int64, int64(numQueries))
cResult := (*C.long)(&result[0])
C.GetHitSizePeerQueries(mh.cMarshaledHits, cGroupOffset, cResult)
return result, nil
}
func deleteMarshaledHits(hits *MarshaledHits) {
C.DeleteMarshaledHits(hits.cMarshaledHits)
}
func deleteSearchResults(results []*SearchResult) {
for _, result := range results {
C.DeleteQueryResult(result.cQueryResult)
}
}

139
internal/querynode/reduce_test.go Normal file
View File

@ -0,0 +1,139 @@
package querynode
import (
"encoding/binary"
"log"
"math"
"testing"
"github.com/golang/protobuf/proto"
"github.com/stretchr/testify/assert"
"github.com/zilliztech/milvus-distributed/internal/proto/commonpb"
"github.com/zilliztech/milvus-distributed/internal/proto/etcdpb"
"github.com/zilliztech/milvus-distributed/internal/proto/schemapb"
"github.com/zilliztech/milvus-distributed/internal/proto/servicepb"
)
func TestReduce_AllFunc(t *testing.T) {
fieldVec := schemapb.FieldSchema{
Name: "vec",
DataType: schemapb.DataType_VECTOR_FLOAT,
TypeParams: []*commonpb.KeyValuePair{
{
Key: "dim",
Value: "16",
},
},
}
fieldInt := schemapb.FieldSchema{
Name: "age",
DataType: schemapb.DataType_INT32,
TypeParams: []*commonpb.KeyValuePair{
{
Key: "dim",
Value: "1",
},
},
}
schema := schemapb.CollectionSchema{
Name: "collection0",
Fields: []*schemapb.FieldSchema{
&fieldVec, &fieldInt,
},
}
collectionMeta := etcdpb.CollectionMeta{
ID: UniqueID(0),
Schema: &schema,
CreateTime: Timestamp(0),
SegmentIDs: []UniqueID{0},
PartitionTags: []string{"default"},
}
collectionMetaBlob := proto.MarshalTextString(&collectionMeta)
assert.NotEqual(t, "", collectionMetaBlob)
collection := newCollection(&collectionMeta, collectionMetaBlob)
assert.Equal(t, collection.meta.Schema.Name, "collection0")
assert.Equal(t, collection.meta.ID, UniqueID(0))
segmentID := UniqueID(0)
segment := newSegment(collection, segmentID)
assert.Equal(t, segmentID, segment.segmentID)
const DIM = 16
var vec = [DIM]float32{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16}
// start search service
dslString := "{\"bool\": { \n\"vector\": {\n \"vec\": {\n \"metric_type\": \"L2\", \n \"params\": {\n \"nprobe\": 10 \n},\n \"query\": \"$0\",\"topk\": 10 \n } \n } \n } \n }"
var searchRawData1 []byte
var searchRawData2 []byte
for i, ele := range vec {
buf := make([]byte, 4)
binary.LittleEndian.PutUint32(buf, math.Float32bits(ele+float32(i*2)))
searchRawData1 = append(searchRawData1, buf...)
}
for i, ele := range vec {
buf := make([]byte, 4)
binary.LittleEndian.PutUint32(buf, math.Float32bits(ele+float32(i*4)))
searchRawData2 = append(searchRawData2, buf...)
}
placeholderValue := servicepb.PlaceholderValue{
Tag: "$0",
Type: servicepb.PlaceholderType_VECTOR_FLOAT,
Values: [][]byte{searchRawData1, searchRawData2},
}
placeholderGroup := servicepb.PlaceholderGroup{
Placeholders: []*servicepb.PlaceholderValue{&placeholderValue},
}
placeGroupByte, err := proto.Marshal(&placeholderGroup)
if err != nil {
log.Print("marshal placeholderGroup failed")
}
plan := createPlan(*collection, dslString)
holder := parserPlaceholderGroup(plan, placeGroupByte)
placeholderGroups := make([]*PlaceholderGroup, 0)
placeholderGroups = append(placeholderGroups, holder)
searchResults := make([]*SearchResult, 0)
searchResult, err := segment.segmentSearch(plan, placeholderGroups, []Timestamp{0})
assert.Nil(t, err)
searchResults = append(searchResults, searchResult)
reducedSearchResults := reduceSearchResults(searchResults, 1)
assert.NotNil(t, reducedSearchResults)
marshaledHits := reducedSearchResults.reorganizeQueryResults(plan, placeholderGroups)
assert.NotNil(t, marshaledHits)
hitsBlob, err := marshaledHits.getHitsBlob()
assert.Nil(t, err)
var offset int64 = 0
for index := range placeholderGroups {
hitBolbSizePeerQuery, err := marshaledHits.hitBlobSizeInGroup(int64(index))
assert.Nil(t, err)
for _, len := range hitBolbSizePeerQuery {
marshaledHit := hitsBlob[offset : offset+len]
unMarshaledHit := servicepb.Hits{}
err = proto.Unmarshal(marshaledHit, &unMarshaledHit)
assert.Nil(t, err)
log.Println("hits msg = ", unMarshaledHit)
offset += len
}
}
plan.delete()
holder.delete()
deleteSearchResults(searchResults)
deleteSearchResults([]*SearchResult{reducedSearchResults})
deleteMarshaledHits(marshaledHits)
deleteSegment(segment)
deleteCollection(collection)
}

290
internal/querynode/search_service.go
View File

@ -4,9 +4,7 @@ import "C"
import (
"context"
"errors"
"fmt"
"log"
"math"
"sync"
"github.com/golang/protobuf/proto"
@ -18,15 +16,18 @@ import (
)
type searchService struct {
ctx context.Context
wait sync.WaitGroup
cancel context.CancelFunc
msgBuffer chan msgstream.TsMsg
unsolvedMsg []msgstream.TsMsg
ctx context.Context
wait sync.WaitGroup
cancel context.CancelFunc
replica *collectionReplica
tSafeWatcher *tSafeWatcher
serviceableTime Timestamp
serviceableTimeMutex sync.Mutex
msgBuffer chan msgstream.TsMsg
unsolvedMsg []msgstream.TsMsg
searchMsgStream *msgstream.MsgStream
searchResultMsgStream *msgstream.MsgStream
}
@ -60,10 +61,11 @@ func newSearchService(ctx context.Context, replica *collectionReplica) *searchSe
msgBuffer := make(chan msgstream.TsMsg, receiveBufSize)
unsolvedMsg := make([]msgstream.TsMsg, 0)
return &searchService{
ctx: searchServiceCtx,
cancel: searchServiceCancel,
msgBuffer: msgBuffer,
unsolvedMsg: unsolvedMsg,
ctx: searchServiceCtx,
cancel: searchServiceCancel,
serviceableTime: Timestamp(0),
msgBuffer: msgBuffer,
unsolvedMsg: unsolvedMsg,
replica: replica,
tSafeWatcher: newTSafeWatcher(),
@ -76,9 +78,10 @@ func newSearchService(ctx context.Context, replica *collectionReplica) *searchSe
func (ss *searchService) start() {
(*ss.searchMsgStream).Start()
(*ss.searchResultMsgStream).Start()
ss.register()
ss.wait.Add(2)
go ss.receiveSearchMsg()
go ss.startSearchService()
go ss.doUnsolvedMsgSearch()
ss.wait.Wait()
}
@ -100,6 +103,19 @@ func (ss *searchService) waitNewTSafe() Timestamp {
return timestamp
}
func (ss *searchService) getServiceableTime() Timestamp {
ss.serviceableTimeMutex.Lock()
defer ss.serviceableTimeMutex.Unlock()
return ss.serviceableTime
}
func (ss *searchService) setServiceableTime(t Timestamp) {
ss.serviceableTimeMutex.Lock()
// TODO:: add gracefulTime
ss.serviceableTime = t
ss.serviceableTimeMutex.Unlock()
}
func (ss *searchService) receiveSearchMsg() {
defer ss.wait.Done()
for {
@ -111,28 +127,45 @@ func (ss *searchService) receiveSearchMsg() {
if msgPack == nil || len(msgPack.Msgs) <= 0 {
continue
}
searchMsg := make([]msgstream.TsMsg, 0)
serverTime := ss.getServiceableTime()
for i := range msgPack.Msgs {
ss.msgBuffer <- msgPack.Msgs[i]
//fmt.Println("receive a search msg")
if msgPack.Msgs[i].BeginTs() > serverTime {
ss.msgBuffer <- msgPack.Msgs[i]
continue
}
searchMsg = append(searchMsg, msgPack.Msgs[i])
}
for _, msg := range searchMsg {
err := ss.search(msg)
if err != nil {
log.Println("search Failed, error msg type: ", msg.Type())
}
err = ss.publishFailedSearchResult(msg)
if err != nil {
log.Println("publish FailedSearchResult failed, error message: ", err)
}
}
log.Println("Do search done, num of searchMsg = ", len(searchMsg))
}
}
}
func (ss *searchService) startSearchService() {
func (ss *searchService) doUnsolvedMsgSearch() {
defer ss.wait.Done()
for {
select {
case <-ss.ctx.Done():
return
default:
serviceTimestamp := (*(*ss.replica).getTSafe()).get()
serviceTime := ss.waitNewTSafe()
ss.setServiceableTime(serviceTime)
searchMsg := make([]msgstream.TsMsg, 0)
tempMsg := make([]msgstream.TsMsg, 0)
tempMsg = append(tempMsg, ss.unsolvedMsg...)
ss.unsolvedMsg = ss.unsolvedMsg[:0]
for _, msg := range tempMsg {
if msg.BeginTs() > serviceTimestamp {
if msg.EndTs() <= serviceTime {
searchMsg = append(searchMsg, msg)
continue
}
@ -142,7 +175,7 @@ func (ss *searchService) startSearchService() {
msgBufferLength := len(ss.msgBuffer)
for i := 0; i < msgBufferLength; i++ {
msg := <-ss.msgBuffer
if msg.BeginTs() > serviceTimestamp {
if msg.EndTs() <= serviceTime {
searchMsg = append(searchMsg, msg)
continue
}
@ -151,97 +184,92 @@ func (ss *searchService) startSearchService() {
if len(searchMsg) <= 0 {
continue
}
err := ss.search(searchMsg)
if err != nil {
fmt.Println("search Failed")
ss.publishFailedSearchResult()
for _, msg := range searchMsg {
err := ss.search(msg)
if err != nil {
log.Println("search Failed, error msg type: ", msg.Type())
}
err = ss.publishFailedSearchResult(msg)
if err != nil {
log.Println("publish FailedSearchResult failed, error message: ", err)
}
}
fmt.Println("Do search done")
log.Println("Do search done, num of searchMsg = ", len(searchMsg))
}
}
}
func (ss *searchService) search(searchMessages []msgstream.TsMsg) error {
// TODO:: cache map[dsl]plan
// TODO: reBatched search requests
for _, msg := range searchMessages {
searchMsg, ok := msg.(*msgstream.SearchMsg)
if !ok {
return errors.New("invalid request type = " + string(msg.Type()))
}
// TODO:: cache map[dsl]plan
// TODO: reBatched search requests
func (ss *searchService) search(msg msgstream.TsMsg) error {
searchMsg, ok := msg.(*msgstream.SearchMsg)
if !ok {
return errors.New("invalid request type = " + string(msg.Type()))
}
searchTimestamp := searchMsg.Timestamp
var queryBlob = searchMsg.Query.Value
query := servicepb.Query{}
err := proto.Unmarshal(queryBlob, &query)
if err != nil {
return errors.New("unmarshal query failed")
}
collectionName := query.CollectionName
partitionTags := query.PartitionTags
collection, err := (*ss.replica).getCollectionByName(collectionName)
searchTimestamp := searchMsg.Timestamp
var queryBlob = searchMsg.Query.Value
query := servicepb.Query{}
err := proto.Unmarshal(queryBlob, &query)
if err != nil {
return errors.New("unmarshal query failed")
}
collectionName := query.CollectionName
partitionTags := query.PartitionTags
collection, err := (*ss.replica).getCollectionByName(collectionName)
if err != nil {
return err
}
collectionID := collection.ID()
dsl := query.Dsl
plan := createPlan(*collection, dsl)
placeHolderGroupBlob := query.PlaceholderGroup
placeholderGroup := parserPlaceholderGroup(plan, placeHolderGroupBlob)
placeholderGroups := make([]*PlaceholderGroup, 0)
placeholderGroups = append(placeholderGroups, placeholderGroup)
searchResults := make([]*SearchResult, 0)
for _, partitionTag := range partitionTags {
partition, err := (*ss.replica).getPartitionByTag(collectionID, partitionTag)
if err != nil {
return err
}
collectionID := collection.ID()
dsl := query.Dsl
plan := CreatePlan(*collection, dsl)
topK := plan.GetTopK()
placeHolderGroupBlob := query.PlaceholderGroup
group := servicepb.PlaceholderGroup{}
err = proto.Unmarshal(placeHolderGroupBlob, &group)
if err != nil {
return err
}
placeholderGroup := ParserPlaceholderGroup(plan, placeHolderGroupBlob)
placeholderGroups := make([]*PlaceholderGroup, 0)
placeholderGroups = append(placeholderGroups, placeholderGroup)
// 2d slice for receiving multiple queries's results
var numQueries int64 = 0
for _, pg := range placeholderGroups {
numQueries += pg.GetNumOfQuery()
}
resultIds := make([]IntPrimaryKey, topK*numQueries)
resultDistances := make([]float32, topK*numQueries)
for i := range resultDistances {
resultDistances[i] = math.MaxFloat32
}
// 3. Do search in all segments
for _, partitionTag := range partitionTags {
partition, err := (*ss.replica).getPartitionByTag(collectionID, partitionTag)
for _, segment := range partition.segments {
searchResult, err := segment.segmentSearch(plan, placeholderGroups, []Timestamp{searchTimestamp})
if err != nil {
return err
}
for _, segment := range partition.segments {
err := segment.segmentSearch(plan,
placeholderGroups,
[]Timestamp{searchTimestamp},
resultIds,
resultDistances,
numQueries,
topK)
if err != nil {
return err
}
}
searchResults = append(searchResults, searchResult)
}
}
// 4. return results
hits := make([]*servicepb.Hits, 0)
for i := int64(0); i < numQueries; i++ {
hit := servicepb.Hits{}
score := servicepb.Score{}
for j := i * topK; j < (i+1)*topK; j++ {
hit.IDs = append(hit.IDs, resultIds[j])
score.Values = append(score.Values, resultDistances[j])
}
hit.Scores = append(hit.Scores, &score)
hits = append(hits, &hit)
reducedSearchResult := reduceSearchResults(searchResults, int64(len(searchResults)))
marshaledHits := reducedSearchResult.reorganizeQueryResults(plan, placeholderGroups)
hitsBlob, err := marshaledHits.getHitsBlob()
if err != nil {
return err
}
var offset int64 = 0
for index := range placeholderGroups {
hitBolbSizePeerQuery, err := marshaledHits.hitBlobSizeInGroup(int64(index))
if err != nil {
return err
}
hits := make([][]byte, 0)
for _, len := range hitBolbSizePeerQuery {
hits = append(hits, hitsBlob[offset:offset+len])
//test code to checkout marshaled hits
//marshaledHit := hitsBlob[offset:offset+len]
//unMarshaledHit := servicepb.Hits{}
//err = proto.Unmarshal(marshaledHit, &unMarshaledHit)
//if err != nil {
// return err
//}
//fmt.Println("hits msg = ", unMarshaledHit)
offset += len
}
var results = internalpb.SearchResult{
MsgType: internalpb.MsgType_kSearchResult,
Status: &commonpb.Status{ErrorCode: commonpb.ErrorCode_SUCCESS},
@ -252,30 +280,60 @@ func (ss *searchService) search(searchMessages []msgstream.TsMsg) error {
ResultChannelID: searchMsg.ResultChannelID,
Hits: hits,
}
searchResultMsg := &msgstream.SearchResultMsg{
BaseMsg: msgstream.BaseMsg{HashValues: []int32{0}},
SearchResult: results,
}
err = ss.publishSearchResult(searchResultMsg)
if err != nil {
return err
}
}
var tsMsg msgstream.TsMsg = &msgstream.SearchResultMsg{SearchResult: results}
ss.publishSearchResult(tsMsg)
plan.Delete()
placeholderGroup.Delete()
deleteSearchResults(searchResults)
deleteSearchResults([]*SearchResult{reducedSearchResult})
deleteMarshaledHits(marshaledHits)
plan.delete()
placeholderGroup.delete()
return nil
}
func (ss *searchService) publishSearchResult(msg msgstream.TsMsg) error {
msgPack := msgstream.MsgPack{}
msgPack.Msgs = append(msgPack.Msgs, msg)
err := (*ss.searchResultMsgStream).Produce(&msgPack)
if err != nil {
return err
}
return nil
}
func (ss *searchService) publishFailedSearchResult(msg msgstream.TsMsg) error {
msgPack := msgstream.MsgPack{}
searchMsg, ok := msg.(*msgstream.SearchMsg)
if !ok {
return errors.New("invalid request type = " + string(msg.Type()))
}
var results = internalpb.SearchResult{
MsgType: internalpb.MsgType_kSearchResult,
Status: &commonpb.Status{ErrorCode: commonpb.ErrorCode_UNEXPECTED_ERROR},
ReqID: searchMsg.ReqID,
ProxyID: searchMsg.ProxyID,
QueryNodeID: searchMsg.ProxyID,
Timestamp: searchMsg.Timestamp,
ResultChannelID: searchMsg.ResultChannelID,
Hits: [][]byte{},
}
tsMsg := &msgstream.SearchResultMsg{
BaseMsg: msgstream.BaseMsg{HashValues: []int32{0}},
SearchResult: results,
}
msgPack.Msgs = append(msgPack.Msgs, tsMsg)
err := (*ss.searchResultMsgStream).Produce(&msgPack)
if err != nil {
return err
}
return nil
}
func (ss *searchService) publishSearchResult(res msgstream.TsMsg) {
msgPack := msgstream.MsgPack{}
msgPack.Msgs = append(msgPack.Msgs, res)
(*ss.searchResultMsgStream).Produce(&msgPack)
}
func (ss *searchService) publishFailedSearchResult() {
var errorResults = internalpb.SearchResult{
MsgType: internalpb.MsgType_kSearchResult,
Status: &commonpb.Status{ErrorCode: commonpb.ErrorCode_UNEXPECTED_ERROR},
}
var tsMsg msgstream.TsMsg = &msgstream.SearchResultMsg{SearchResult: errorResults}
msgPack := msgstream.MsgPack{}
msgPack.Msgs = append(msgPack.Msgs, tsMsg)
(*ss.searchResultMsgStream).Produce(&msgPack)
}

200
internal/querynode/search_service_test.go
View File

@ -3,7 +3,6 @@ package querynode
import (
"context"
"encoding/binary"
"log"
"math"
"testing"
@ -88,103 +87,30 @@ func TestSearch_Search(t *testing.T) {
// test data generate
const msgLength = 10
const DIM = 16
const N = 10
var vec = [DIM]float32{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16}
var rawData []byte
for _, ele := range vec {
buf := make([]byte, 4)
binary.LittleEndian.PutUint32(buf, math.Float32bits(ele))
rawData = append(rawData, buf...)
}
bs := make([]byte, 4)
binary.LittleEndian.PutUint32(bs, 1)
rawData = append(rawData, bs...)
var records []*commonpb.Blob
for i := 0; i < N; i++ {
blob := &commonpb.Blob{
Value: rawData,
}
records = append(records, blob)
}
timeRange := TimeRange{
timestampMin: 0,
timestampMax: math.MaxUint64,
}
// messages generate
insertMessages := make([]msgstream.TsMsg, 0)
for i := 0; i < msgLength; i++ {
var msg msgstream.TsMsg = &msgstream.InsertMsg{
BaseMsg: msgstream.BaseMsg{
HashValues: []int32{
int32(i), int32(i),
},
},
InsertRequest: internalpb.InsertRequest{
MsgType: internalpb.MsgType_kInsert,
ReqID: int64(i),
CollectionName: "collection0",
PartitionTag: "default",
SegmentID: int64(0),
ChannelID: int64(0),
ProxyID: int64(0),
Timestamps: []uint64{uint64(i + 1000), uint64(i + 1000)},
RowIDs: []int64{int64(i * 2), int64(i*2 + 1)},
RowData: []*commonpb.Blob{
{Value: rawData},
{Value: rawData},
},
},
}
insertMessages = append(insertMessages, msg)
}
msgPack := msgstream.MsgPack{
BeginTs: timeRange.timestampMin,
EndTs: timeRange.timestampMax,
Msgs: insertMessages,
}
// pulsar produce
const receiveBufSize = 1024
insertProducerChannels := []string{"insert"}
insertStream := msgstream.NewPulsarMsgStream(ctx, receiveBufSize)
insertStream.SetPulsarClient(pulsarURL)
insertStream.CreatePulsarProducers(insertProducerChannels)
var insertMsgStream msgstream.MsgStream = insertStream
insertMsgStream.Start()
err = insertMsgStream.Produce(&msgPack)
assert.NoError(t, err)
// dataSync
node.dataSyncService = newDataSyncService(node.ctx, node.replica)
go node.dataSyncService.start()
time.Sleep(2 * time.Second)
const DIM = 16
insertProducerChannels := Params.insertChannelNames()
searchProducerChannels := Params.searchChannelNames()
var vec = [DIM]float32{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16}
// start search service
dslString := "{\"bool\": { \n\"vector\": {\n \"vec\": {\n \"metric_type\": \"L2\", \n \"params\": {\n \"nprobe\": 10 \n},\n \"query\": \"$0\",\"topk\": 10 \n } \n } \n } \n }"
searchProducerChannels := []string{"search"}
searchStream := msgstream.NewPulsarMsgStream(ctx, receiveBufSize)
searchStream.SetPulsarClient(pulsarURL)
searchStream.CreatePulsarProducers(searchProducerChannels)
var vecSearch = [DIM]float32{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 17}
var searchRawData []byte
for _, ele := range vecSearch {
var searchRawData1 []byte
var searchRawData2 []byte
for i, ele := range vec {
buf := make([]byte, 4)
binary.LittleEndian.PutUint32(buf, math.Float32bits(ele))
searchRawData = append(searchRawData, buf...)
binary.LittleEndian.PutUint32(buf, math.Float32bits(ele+float32(i*2)))
searchRawData1 = append(searchRawData1, buf...)
}
for i, ele := range vec {
buf := make([]byte, 4)
binary.LittleEndian.PutUint32(buf, math.Float32bits(ele+float32(i*4)))
searchRawData2 = append(searchRawData2, buf...)
}
placeholderValue := servicepb.PlaceholderValue{
Tag: "$0",
Type: servicepb.PlaceholderType_VECTOR_FLOAT,
Values: [][]byte{searchRawData},
Values: [][]byte{searchRawData1, searchRawData2},
}
placeholderGroup := servicepb.PlaceholderGroup{
@ -220,8 +146,8 @@ func TestSearch_Search(t *testing.T) {
MsgType: internalpb.MsgType_kSearch,
ReqID: int64(1),
ProxyID: int64(1),
Timestamp: uint64(20 + 1000),
ResultChannelID: int64(1),
Timestamp: uint64(10 + 1000),
ResultChannelID: int64(0),
Query: &blob,
},
}
@ -229,15 +155,97 @@ func TestSearch_Search(t *testing.T) {
msgPackSearch := msgstream.MsgPack{}
msgPackSearch.Msgs = append(msgPackSearch.Msgs, searchMsg)
var searchMsgStream msgstream.MsgStream = searchStream
searchMsgStream.Start()
err = searchMsgStream.Produce(&msgPackSearch)
searchStream := msgstream.NewPulsarMsgStream(ctx, receiveBufSize)
searchStream.SetPulsarClient(pulsarURL)
searchStream.CreatePulsarProducers(searchProducerChannels)
searchStream.Start()
err = searchStream.Produce(&msgPackSearch)
assert.NoError(t, err)
node.searchService = newSearchService(node.ctx, node.replica)
go node.searchService.start()
time.Sleep(2 * time.Second)
// start insert
timeRange := TimeRange{
timestampMin: 0,
timestampMax: math.MaxUint64,
}
insertMessages := make([]msgstream.TsMsg, 0)
for i := 0; i < msgLength; i++ {
var rawData []byte
for _, ele := range vec {
buf := make([]byte, 4)
binary.LittleEndian.PutUint32(buf, math.Float32bits(ele+float32(i*2)))
rawData = append(rawData, buf...)
}
bs := make([]byte, 4)
binary.LittleEndian.PutUint32(bs, 1)
rawData = append(rawData, bs...)
var msg msgstream.TsMsg = &msgstream.InsertMsg{
BaseMsg: msgstream.BaseMsg{
HashValues: []int32{
int32(i),
},
},
InsertRequest: internalpb.InsertRequest{
MsgType: internalpb.MsgType_kInsert,
ReqID: int64(i),
CollectionName: "collection0",
PartitionTag: "default",
SegmentID: int64(0),
ChannelID: int64(0),
ProxyID: int64(0),
Timestamps: []uint64{uint64(i + 1000)},
RowIDs: []int64{int64(i)},
RowData: []*commonpb.Blob{
{Value: rawData},
},
},
}
insertMessages = append(insertMessages, msg)
}
msgPack := msgstream.MsgPack{
BeginTs: timeRange.timestampMin,
EndTs: timeRange.timestampMax,
Msgs: insertMessages,
}
// generate timeTick
timeTickMsgPack := msgstream.MsgPack{}
baseMsg := msgstream.BaseMsg{
BeginTimestamp: 0,
EndTimestamp: 0,
HashValues: []int32{0},
}
timeTickResult := internalpb.TimeTickMsg{
MsgType: internalpb.MsgType_kTimeTick,
PeerID: UniqueID(0),
Timestamp: math.MaxUint64,
}
timeTickMsg := &msgstream.TimeTickMsg{
BaseMsg: baseMsg,
TimeTickMsg: timeTickResult,
}
timeTickMsgPack.Msgs = append(timeTickMsgPack.Msgs, timeTickMsg)
// pulsar produce
insertStream := msgstream.NewPulsarMsgStream(ctx, receiveBufSize)
insertStream.SetPulsarClient(pulsarURL)
insertStream.CreatePulsarProducers(insertProducerChannels)
insertStream.Start()
err = insertStream.Produce(&msgPack)
assert.NoError(t, err)
err = insertStream.Broadcast(&timeTickMsgPack)
assert.NoError(t, err)
// dataSync
node.dataSyncService = newDataSyncService(node.ctx, node.replica)
go node.dataSyncService.start()
time.Sleep(1 * time.Second)
cancel()
node.Close()

28
internal/querynode/segment.go
View File

@ -7,7 +7,6 @@ package querynode
#cgo LDFLAGS: -L${SRCDIR}/../core/output/lib -lmilvus_segcore -Wl,-rpath=${SRCDIR}/../core/output/lib
#include "segcore/collection_c.h"
#include "segcore/segment_c.h"
#include "segcore/plan_c.h"
#include "segcore/reduce_c.h"
@ -189,11 +188,7 @@ func (s *Segment) segmentDelete(offset int64, entityIDs *[]UniqueID, timestamps
func (s *Segment) segmentSearch(plan *Plan,
placeHolderGroups []*PlaceholderGroup,
timestamp []Timestamp,
resultIds []IntPrimaryKey,
resultDistances []float32,
numQueries int64,
topK int64) error {
timestamp []Timestamp) (*SearchResult, error) {
/*
CStatus
Search(void* plan,
@ -204,36 +199,25 @@ func (s *Segment) segmentSearch(plan *Plan,
float* result_distances);
*/
newResultIds := make([]IntPrimaryKey, topK*numQueries)
NewResultDistances := make([]float32, topK*numQueries)
cPlaceholderGroups := make([]C.CPlaceholderGroup, 0)
for _, pg := range placeHolderGroups {
cPlaceholderGroups = append(cPlaceholderGroups, (*pg).cPlaceholderGroup)
}
var searchResult SearchResult
var cTimestamp = (*C.ulong)(&timestamp[0])
var cResultIds = (*C.long)(&resultIds[0])
var cResultDistances = (*C.float)(&resultDistances[0])
var cNewResultIds = (*C.long)(&newResultIds[0])
var cNewResultDistances = (*C.float)(&NewResultDistances[0])
var cPlaceHolder = (*C.CPlaceholderGroup)(&cPlaceholderGroups[0])
var cNumGroups = C.int(len(placeHolderGroups))
cQueryResult := (*C.CQueryResult)(&searchResult.cQueryResult)
var status = C.Search(s.segmentPtr, plan.cPlan, cPlaceHolder, cTimestamp, cNumGroups, cNewResultIds, cNewResultDistances)
var status = C.Search(s.segmentPtr, plan.cPlan, cPlaceHolder, cTimestamp, cNumGroups, cQueryResult)
errorCode := status.error_code
if errorCode != 0 {
errorMsg := C.GoString(status.error_msg)
defer C.free(unsafe.Pointer(status.error_msg))
return errors.New("Search failed, C runtime error detected, error code = " + strconv.Itoa(int(errorCode)) + ", error msg = " + errorMsg)
return nil, errors.New("Search failed, C runtime error detected, error code = " + strconv.Itoa(int(errorCode)) + ", error msg = " + errorMsg)
}
cNumQueries := C.long(numQueries)
cTopK := C.long(topK)
// reduce search result
mergeStatus := C.MergeInto(cNumQueries, cTopK, cResultDistances, cResultIds, cNewResultDistances, cNewResultIds)
if mergeStatus != 0 {
return errors.New("merge search result failed, error code = " + strconv.Itoa(int(mergeStatus)))
}
return nil
return &searchResult, nil
}

23
internal/querynode/segment_test.go
View File

@ -666,25 +666,16 @@ func TestSegment_segmentSearch(t *testing.T) {
}
searchTimestamp := Timestamp(1020)
cPlan := CreatePlan(*collection, dslString)
topK := cPlan.GetTopK()
cPlaceholderGroup := ParserPlaceholderGroup(cPlan, placeHolderGroupBlob)
plan := createPlan(*collection, dslString)
holder := parserPlaceholderGroup(plan, placeHolderGroupBlob)
placeholderGroups := make([]*PlaceholderGroup, 0)
placeholderGroups = append(placeholderGroups, cPlaceholderGroup)
placeholderGroups = append(placeholderGroups, holder)
var numQueries int64 = 0
for _, pg := range placeholderGroups {
numQueries += pg.GetNumOfQuery()
}
resultIds := make([]IntPrimaryKey, topK*numQueries)
resultDistances := make([]float32, topK*numQueries)
for i := range resultDistances {
resultDistances[i] = math.MaxFloat32
}
err = segment.segmentSearch(cPlan, placeholderGroups, []Timestamp{searchTimestamp}, resultIds, resultDistances, numQueries, topK)
assert.NoError(t, err)
_, err = segment.segmentSearch(plan, placeholderGroups, []Timestamp{searchTimestamp})
assert.Nil(t, err)
plan.delete()
holder.delete()
deleteSegment(segment)
deleteCollection(collection)
}