For search funtion adding a round_decimal paramter to precision control (#8574)

Signed-off-by: elfisworking <zymustb@126.com>
pull/9491/head
elfisworking 2021-10-08 17:39:55 +08:00 committed by GitHub
parent 49198dbce1
commit d4232f88a2
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
32 changed files with 1453 additions and 1215 deletions

View File

@ -278,6 +278,7 @@ const ::PROTOBUF_NAMESPACE_ID::uint32 TableStruct_plan_2eproto::offsets[] PROTOB
PROTOBUF_FIELD_OFFSET(::milvus::proto::plan::QueryInfo, topk_),
PROTOBUF_FIELD_OFFSET(::milvus::proto::plan::QueryInfo, metric_type_),
PROTOBUF_FIELD_OFFSET(::milvus::proto::plan::QueryInfo, search_params_),
PROTOBUF_FIELD_OFFSET(::milvus::proto::plan::QueryInfo, round_decimal_),
~0u, // no _has_bits_
PROTOBUF_FIELD_OFFSET(::milvus::proto::plan::ColumnInfo, _internal_metadata_),
~0u, // no _extensions_
@ -370,16 +371,16 @@ const ::PROTOBUF_NAMESPACE_ID::uint32 TableStruct_plan_2eproto::offsets[] PROTOB
static const ::PROTOBUF_NAMESPACE_ID::internal::MigrationSchema schemas[] PROTOBUF_SECTION_VARIABLE(protodesc_cold) = {
{ 0, -1, sizeof(::milvus::proto::plan::GenericValue)},
{ 9, -1, sizeof(::milvus::proto::plan::QueryInfo)},
{ 17, -1, sizeof(::milvus::proto::plan::ColumnInfo)},
{ 26, -1, sizeof(::milvus::proto::plan::UnaryRangeExpr)},
{ 34, -1, sizeof(::milvus::proto::plan::BinaryRangeExpr)},
{ 44, -1, sizeof(::milvus::proto::plan::CompareExpr)},
{ 52, -1, sizeof(::milvus::proto::plan::TermExpr)},
{ 59, -1, sizeof(::milvus::proto::plan::UnaryExpr)},
{ 66, -1, sizeof(::milvus::proto::plan::BinaryExpr)},
{ 74, -1, sizeof(::milvus::proto::plan::Expr)},
{ 86, -1, sizeof(::milvus::proto::plan::VectorANNS)},
{ 96, -1, sizeof(::milvus::proto::plan::PlanNode)},
{ 18, -1, sizeof(::milvus::proto::plan::ColumnInfo)},
{ 27, -1, sizeof(::milvus::proto::plan::UnaryRangeExpr)},
{ 35, -1, sizeof(::milvus::proto::plan::BinaryRangeExpr)},
{ 45, -1, sizeof(::milvus::proto::plan::CompareExpr)},
{ 53, -1, sizeof(::milvus::proto::plan::TermExpr)},
{ 60, -1, sizeof(::milvus::proto::plan::UnaryExpr)},
{ 67, -1, sizeof(::milvus::proto::plan::BinaryExpr)},
{ 75, -1, sizeof(::milvus::proto::plan::Expr)},
{ 87, -1, sizeof(::milvus::proto::plan::VectorANNS)},
{ 97, -1, sizeof(::milvus::proto::plan::PlanNode)},
};
static ::PROTOBUF_NAMESPACE_ID::Message const * const file_default_instances[] = {
@ -401,58 +402,59 @@ const char descriptor_table_protodef_plan_2eproto[] PROTOBUF_SECTION_VARIABLE(pr
"\n\nplan.proto\022\021milvus.proto.plan\032\014schema."
"proto\"S\n\014GenericValue\022\022\n\010bool_val\030\001 \001(\010H"
"\000\022\023\n\tint64_val\030\002 \001(\003H\000\022\023\n\tfloat_val\030\003 \001("
"\001H\000B\005\n\003val\"E\n\tQueryInfo\022\014\n\004topk\030\001 \001(\003\022\023\n"
"\001H\000B\005\n\003val\"\\\n\tQueryInfo\022\014\n\004topk\030\001 \001(\003\022\023\n"
"\013metric_type\030\003 \001(\t\022\025\n\rsearch_params\030\004 \001("
"\t\"{\n\nColumnInfo\022\020\n\010field_id\030\001 \001(\003\0220\n\tdat"
"a_type\030\002 \001(\0162\035.milvus.proto.schema.DataT"
"ype\022\026\n\016is_primary_key\030\003 \001(\010\022\021\n\tis_autoID"
"\030\004 \001(\010\"\233\001\n\016UnaryRangeExpr\0222\n\013column_info"
"\030\001 \001(\0132\035.milvus.proto.plan.ColumnInfo\022%\n"
"\002op\030\002 \001(\0162\031.milvus.proto.plan.OpType\022.\n\005"
"value\030\003 \001(\0132\037.milvus.proto.plan.GenericV"
"alue\"\343\001\n\017BinaryRangeExpr\0222\n\013column_info\030"
"\001 \001(\0132\035.milvus.proto.plan.ColumnInfo\022\027\n\017"
"lower_inclusive\030\002 \001(\010\022\027\n\017upper_inclusive"
"\030\003 \001(\010\0224\n\013lower_value\030\004 \001(\0132\037.milvus.pro"
"to.plan.GenericValue\0224\n\013upper_value\030\005 \001("
"\0132\037.milvus.proto.plan.GenericValue\"\247\001\n\013C"
"ompareExpr\0227\n\020left_column_info\030\001 \001(\0132\035.m"
"ilvus.proto.plan.ColumnInfo\0228\n\021right_col"
"umn_info\030\002 \001(\0132\035.milvus.proto.plan.Colum"
"nInfo\022%\n\002op\030\003 \001(\0162\031.milvus.proto.plan.Op"
"Type\"o\n\010TermExpr\0222\n\013column_info\030\001 \001(\0132\035."
"milvus.proto.plan.ColumnInfo\022/\n\006values\030\002"
" \003(\0132\037.milvus.proto.plan.GenericValue\"\206\001"
"\n\tUnaryExpr\0220\n\002op\030\001 \001(\0162$.milvus.proto.p"
"lan.UnaryExpr.UnaryOp\022&\n\005child\030\002 \001(\0132\027.m"
"ilvus.proto.plan.Expr\"\037\n\007UnaryOp\022\013\n\007Inva"
"lid\020\000\022\007\n\003Not\020\001\"\307\001\n\nBinaryExpr\0222\n\002op\030\001 \001("
"\0162&.milvus.proto.plan.BinaryExpr.BinaryO"
"p\022%\n\004left\030\002 \001(\0132\027.milvus.proto.plan.Expr"
"\022&\n\005right\030\003 \001(\0132\027.milvus.proto.plan.Expr"
"\"6\n\010BinaryOp\022\013\n\007Invalid\020\000\022\016\n\nLogicalAnd\020"
"\001\022\r\n\tLogicalOr\020\002\"\342\002\n\004Expr\0220\n\tterm_expr\030\001"
" \001(\0132\033.milvus.proto.plan.TermExprH\000\0222\n\nu"
"nary_expr\030\002 \001(\0132\034.milvus.proto.plan.Unar"
"yExprH\000\0224\n\013binary_expr\030\003 \001(\0132\035.milvus.pr"
"oto.plan.BinaryExprH\000\0226\n\014compare_expr\030\004 "
"\001(\0132\036.milvus.proto.plan.CompareExprH\000\022=\n"
"\020unary_range_expr\030\005 \001(\0132!.milvus.proto.p"
"lan.UnaryRangeExprH\000\022\?\n\021binary_range_exp"
"r\030\006 \001(\0132\".milvus.proto.plan.BinaryRangeE"
"xprH\000B\006\n\004expr\"\251\001\n\nVectorANNS\022\021\n\tis_binar"
"y\030\001 \001(\010\022\020\n\010field_id\030\002 \001(\003\022+\n\npredicates\030"
"\003 \001(\0132\027.milvus.proto.plan.Expr\0220\n\nquery_"
"info\030\004 \001(\0132\034.milvus.proto.plan.QueryInfo"
"\022\027\n\017placeholder_tag\030\005 \001(\t\"\221\001\n\010PlanNode\0224"
"\n\013vector_anns\030\001 \001(\0132\035.milvus.proto.plan."
"VectorANNSH\000\022-\n\npredicates\030\002 \001(\0132\027.milvu"
"s.proto.plan.ExprH\000\022\030\n\020output_field_ids\030"
"\003 \003(\003B\006\n\004node*n\n\006OpType\022\013\n\007Invalid\020\000\022\017\n\013"
"GreaterThan\020\001\022\020\n\014GreaterEqual\020\002\022\014\n\010LessT"
"han\020\003\022\r\n\tLessEqual\020\004\022\t\n\005Equal\020\005\022\014\n\010NotEq"
"ual\020\006B3Z1github.com/milvus-io/milvus/int"
"ernal/proto/planpbb\006proto3"
"\t\022\025\n\rround_decimal\030\005 \001(\003\"{\n\nColumnInfo\022\020"
"\n\010field_id\030\001 \001(\003\0220\n\tdata_type\030\002 \001(\0162\035.mi"
"lvus.proto.schema.DataType\022\026\n\016is_primary"
"_key\030\003 \001(\010\022\021\n\tis_autoID\030\004 \001(\010\"\233\001\n\016UnaryR"
"angeExpr\0222\n\013column_info\030\001 \001(\0132\035.milvus.p"
"roto.plan.ColumnInfo\022%\n\002op\030\002 \001(\0162\031.milvu"
"s.proto.plan.OpType\022.\n\005value\030\003 \001(\0132\037.mil"
"vus.proto.plan.GenericValue\"\343\001\n\017BinaryRa"
"ngeExpr\0222\n\013column_info\030\001 \001(\0132\035.milvus.pr"
"oto.plan.ColumnInfo\022\027\n\017lower_inclusive\030\002"
" \001(\010\022\027\n\017upper_inclusive\030\003 \001(\010\0224\n\013lower_v"
"alue\030\004 \001(\0132\037.milvus.proto.plan.GenericVa"
"lue\0224\n\013upper_value\030\005 \001(\0132\037.milvus.proto."
"plan.GenericValue\"\247\001\n\013CompareExpr\0227\n\020lef"
"t_column_info\030\001 \001(\0132\035.milvus.proto.plan."
"ColumnInfo\0228\n\021right_column_info\030\002 \001(\0132\035."
"milvus.proto.plan.ColumnInfo\022%\n\002op\030\003 \001(\016"
"2\031.milvus.proto.plan.OpType\"o\n\010TermExpr\022"
"2\n\013column_info\030\001 \001(\0132\035.milvus.proto.plan"
".ColumnInfo\022/\n\006values\030\002 \003(\0132\037.milvus.pro"
"to.plan.GenericValue\"\206\001\n\tUnaryExpr\0220\n\002op"
"\030\001 \001(\0162$.milvus.proto.plan.UnaryExpr.Una"
"ryOp\022&\n\005child\030\002 \001(\0132\027.milvus.proto.plan."
"Expr\"\037\n\007UnaryOp\022\013\n\007Invalid\020\000\022\007\n\003Not\020\001\"\307\001"
"\n\nBinaryExpr\0222\n\002op\030\001 \001(\0162&.milvus.proto."
"plan.BinaryExpr.BinaryOp\022%\n\004left\030\002 \001(\0132\027"
".milvus.proto.plan.Expr\022&\n\005right\030\003 \001(\0132\027"
".milvus.proto.plan.Expr\"6\n\010BinaryOp\022\013\n\007I"
"nvalid\020\000\022\016\n\nLogicalAnd\020\001\022\r\n\tLogicalOr\020\002\""
"\342\002\n\004Expr\0220\n\tterm_expr\030\001 \001(\0132\033.milvus.pro"
"to.plan.TermExprH\000\0222\n\nunary_expr\030\002 \001(\0132\034"
".milvus.proto.plan.UnaryExprH\000\0224\n\013binary"
"_expr\030\003 \001(\0132\035.milvus.proto.plan.BinaryEx"
"prH\000\0226\n\014compare_expr\030\004 \001(\0132\036.milvus.prot"
"o.plan.CompareExprH\000\022=\n\020unary_range_expr"
"\030\005 \001(\0132!.milvus.proto.plan.UnaryRangeExp"
"rH\000\022\?\n\021binary_range_expr\030\006 \001(\0132\".milvus."
"proto.plan.BinaryRangeExprH\000B\006\n\004expr\"\251\001\n"
"\nVectorANNS\022\021\n\tis_binary\030\001 \001(\010\022\020\n\010field_"
"id\030\002 \001(\003\022+\n\npredicates\030\003 \001(\0132\027.milvus.pr"
"oto.plan.Expr\0220\n\nquery_info\030\004 \001(\0132\034.milv"
"us.proto.plan.QueryInfo\022\027\n\017placeholder_t"
"ag\030\005 \001(\t\"\221\001\n\010PlanNode\0224\n\013vector_anns\030\001 \001"
"(\0132\035.milvus.proto.plan.VectorANNSH\000\022-\n\np"
"redicates\030\002 \001(\0132\027.milvus.proto.plan.Expr"
"H\000\022\030\n\020output_field_ids\030\003 \003(\003B\006\n\004node*n\n\006"
"OpType\022\013\n\007Invalid\020\000\022\017\n\013GreaterThan\020\001\022\020\n\014"
"GreaterEqual\020\002\022\014\n\010LessThan\020\003\022\r\n\tLessEqua"
"l\020\004\022\t\n\005Equal\020\005\022\014\n\010NotEqual\020\006B3Z1github.c"
"om/milvus-io/milvus/internal/proto/planp"
"bb\006proto3"
;
static const ::PROTOBUF_NAMESPACE_ID::internal::DescriptorTable*const descriptor_table_plan_2eproto_deps[1] = {
&::descriptor_table_schema_2eproto,
@ -472,7 +474,7 @@ static ::PROTOBUF_NAMESPACE_ID::internal::SCCInfoBase*const descriptor_table_pla
static ::PROTOBUF_NAMESPACE_ID::internal::once_flag descriptor_table_plan_2eproto_once;
static bool descriptor_table_plan_2eproto_initialized = false;
const ::PROTOBUF_NAMESPACE_ID::internal::DescriptorTable descriptor_table_plan_2eproto = {
&descriptor_table_plan_2eproto_initialized, descriptor_table_protodef_plan_2eproto, "plan.proto", 2186,
&descriptor_table_plan_2eproto_initialized, descriptor_table_protodef_plan_2eproto, "plan.proto", 2209,
&descriptor_table_plan_2eproto_once, descriptor_table_plan_2eproto_sccs, descriptor_table_plan_2eproto_deps, 10, 1,
schemas, file_default_instances, TableStruct_plan_2eproto::offsets,
file_level_metadata_plan_2eproto, 12, file_level_enum_descriptors_plan_2eproto, file_level_service_descriptors_plan_2eproto,
@ -960,7 +962,9 @@ QueryInfo::QueryInfo(const QueryInfo& from)
if (!from.search_params().empty()) {
search_params_.AssignWithDefault(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), from.search_params_);
}
topk_ = from.topk_;
::memcpy(&topk_, &from.topk_,
static_cast<size_t>(reinterpret_cast<char*>(&round_decimal_) -
reinterpret_cast<char*>(&topk_)) + sizeof(round_decimal_));
// @@protoc_insertion_point(copy_constructor:milvus.proto.plan.QueryInfo)
}
@ -968,7 +972,9 @@ void QueryInfo::SharedCtor() {
::PROTOBUF_NAMESPACE_ID::internal::InitSCC(&scc_info_QueryInfo_plan_2eproto.base);
metric_type_.UnsafeSetDefault(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited());
search_params_.UnsafeSetDefault(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited());
topk_ = PROTOBUF_LONGLONG(0);
::memset(&topk_, 0, static_cast<size_t>(
reinterpret_cast<char*>(&round_decimal_) -
reinterpret_cast<char*>(&topk_)) + sizeof(round_decimal_));
}
QueryInfo::~QueryInfo() {
@ -998,7 +1004,9 @@ void QueryInfo::Clear() {
metric_type_.ClearToEmptyNoArena(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited());
search_params_.ClearToEmptyNoArena(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited());
topk_ = PROTOBUF_LONGLONG(0);
::memset(&topk_, 0, static_cast<size_t>(
reinterpret_cast<char*>(&round_decimal_) -
reinterpret_cast<char*>(&topk_)) + sizeof(round_decimal_));
_internal_metadata_.Clear();
}
@ -1031,6 +1039,13 @@ const char* QueryInfo::_InternalParse(const char* ptr, ::PROTOBUF_NAMESPACE_ID::
CHK_(ptr);
} else goto handle_unusual;
continue;
// int64 round_decimal = 5;
case 5:
if (PROTOBUF_PREDICT_TRUE(static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 40)) {
round_decimal_ = ::PROTOBUF_NAMESPACE_ID::internal::ReadVarint(&ptr);
CHK_(ptr);
} else goto handle_unusual;
continue;
default: {
handle_unusual:
if ((tag & 7) == 4 || tag == 0) {
@ -1104,6 +1119,19 @@ bool QueryInfo::MergePartialFromCodedStream(
break;
}
// int64 round_decimal = 5;
case 5: {
if (static_cast< ::PROTOBUF_NAMESPACE_ID::uint8>(tag) == (40 & 0xFF)) {
DO_((::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::ReadPrimitive<
::PROTOBUF_NAMESPACE_ID::int64, ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::TYPE_INT64>(
input, &round_decimal_)));
} else {
goto handle_unusual;
}
break;
}
default: {
handle_unusual:
if (tag == 0) {
@ -1156,6 +1184,11 @@ void QueryInfo::SerializeWithCachedSizes(
4, this->search_params(), output);
}
// int64 round_decimal = 5;
if (this->round_decimal() != 0) {
::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteInt64(5, this->round_decimal(), output);
}
if (_internal_metadata_.have_unknown_fields()) {
::PROTOBUF_NAMESPACE_ID::internal::WireFormat::SerializeUnknownFields(
_internal_metadata_.unknown_fields(), output);
@ -1196,6 +1229,11 @@ void QueryInfo::SerializeWithCachedSizes(
4, this->search_params(), target);
}
// int64 round_decimal = 5;
if (this->round_decimal() != 0) {
target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteInt64ToArray(5, this->round_decimal(), target);
}
if (_internal_metadata_.have_unknown_fields()) {
target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormat::SerializeUnknownFieldsToArray(
_internal_metadata_.unknown_fields(), target);
@ -1238,6 +1276,13 @@ size_t QueryInfo::ByteSizeLong() const {
this->topk());
}
// int64 round_decimal = 5;
if (this->round_decimal() != 0) {
total_size += 1 +
::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::Int64Size(
this->round_decimal());
}
int cached_size = ::PROTOBUF_NAMESPACE_ID::internal::ToCachedSize(total_size);
SetCachedSize(cached_size);
return total_size;
@ -1276,6 +1321,9 @@ void QueryInfo::MergeFrom(const QueryInfo& from) {
if (from.topk() != 0) {
set_topk(from.topk());
}
if (from.round_decimal() != 0) {
set_round_decimal(from.round_decimal());
}
}
void QueryInfo::CopyFrom(const ::PROTOBUF_NAMESPACE_ID::Message& from) {
@ -1304,6 +1352,7 @@ void QueryInfo::InternalSwap(QueryInfo* other) {
search_params_.Swap(&other->search_params_, &::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(),
GetArenaNoVirtual());
swap(topk_, other->topk_);
swap(round_decimal_, other->round_decimal_);
}
::PROTOBUF_NAMESPACE_ID::Metadata QueryInfo::GetMetadata() const {

View File

@ -489,6 +489,7 @@ class QueryInfo :
kMetricTypeFieldNumber = 3,
kSearchParamsFieldNumber = 4,
kTopkFieldNumber = 1,
kRoundDecimalFieldNumber = 5,
};
// string metric_type = 3;
void clear_metric_type();
@ -517,6 +518,11 @@ class QueryInfo :
::PROTOBUF_NAMESPACE_ID::int64 topk() const;
void set_topk(::PROTOBUF_NAMESPACE_ID::int64 value);
// int64 round_decimal = 5;
void clear_round_decimal();
::PROTOBUF_NAMESPACE_ID::int64 round_decimal() const;
void set_round_decimal(::PROTOBUF_NAMESPACE_ID::int64 value);
// @@protoc_insertion_point(class_scope:milvus.proto.plan.QueryInfo)
private:
class _Internal;
@ -525,6 +531,7 @@ class QueryInfo :
::PROTOBUF_NAMESPACE_ID::internal::ArenaStringPtr metric_type_;
::PROTOBUF_NAMESPACE_ID::internal::ArenaStringPtr search_params_;
::PROTOBUF_NAMESPACE_ID::int64 topk_;
::PROTOBUF_NAMESPACE_ID::int64 round_decimal_;
mutable ::PROTOBUF_NAMESPACE_ID::internal::CachedSize _cached_size_;
friend struct ::TableStruct_plan_2eproto;
};
@ -2435,6 +2442,20 @@ inline void QueryInfo::set_allocated_search_params(std::string* search_params) {
// @@protoc_insertion_point(field_set_allocated:milvus.proto.plan.QueryInfo.search_params)
}
// int64 round_decimal = 5;
inline void QueryInfo::clear_round_decimal() {
round_decimal_ = PROTOBUF_LONGLONG(0);
}
inline ::PROTOBUF_NAMESPACE_ID::int64 QueryInfo::round_decimal() const {
// @@protoc_insertion_point(field_get:milvus.proto.plan.QueryInfo.round_decimal)
return round_decimal_;
}
inline void QueryInfo::set_round_decimal(::PROTOBUF_NAMESPACE_ID::int64 value) {
round_decimal_ = value;
// @@protoc_insertion_point(field_set:milvus.proto.plan.QueryInfo.round_decimal)
}
// -------------------------------------------------------------------
// ColumnInfo

View File

@ -239,6 +239,7 @@ Parser::ParseVecNode(const Json& out_body) {
vec_node->search_info_.metric_type_ = GetMetricType(vec_info.at("metric_type"));
vec_node->search_info_.search_params_ = vec_info.at("params");
vec_node->search_info_.field_offset_ = field_offset;
vec_node->search_info_.round_decimal_ = vec_info.at("round_decimal");
vec_node->placeholder_tag_ = vec_info.at("query");
auto tag = vec_node->placeholder_tag_;
AssertInfo(!tag2field_.count(tag), "duplicated placeholder tag");

View File

@ -32,6 +32,7 @@ using PlanNodePtr = std::unique_ptr<PlanNode>;
struct SearchInfo {
int64_t topk_;
int64_t round_decimal_;
FieldOffset field_offset_;
MetricType metric_type_;
nlohmann::json search_params_;

View File

@ -122,6 +122,7 @@ ProtoParser::PlanNodeFromProto(const planpb::PlanNode& plan_node_proto) {
search_info.metric_type_ = GetMetricType(query_info_proto.metric_type());
search_info.topk_ = query_info_proto.topk();
search_info.round_decimal_ = query_info_proto.round_decimal();
search_info.search_params_ = json::parse(query_info_proto.search_params());
auto plan_node = [&]() -> std::unique_ptr<VectorPlanNode> {

View File

@ -68,9 +68,10 @@ BinarySearchBruteForceFast(MetricType metric_type,
int64_t size_per_chunk,
int64_t topk,
int64_t num_queries,
int64_t round_decimal,
const uint8_t* query_data,
const faiss::BitsetView& bitset) {
SubSearchResult sub_result(num_queries, topk, metric_type);
SubSearchResult sub_result(num_queries, topk, metric_type, round_decimal);
float* result_distances = sub_result.get_values();
idx_t* result_labels = sub_result.get_labels();
@ -79,7 +80,7 @@ BinarySearchBruteForceFast(MetricType metric_type,
raw_search(metric_type, binary_chunk, size_per_chunk, code_size, num_queries, query_data, topk, result_distances,
result_labels, bitset);
sub_result.round_values();
return sub_result;
}
@ -92,17 +93,20 @@ FloatSearchBruteForce(const dataset::SearchDataset& dataset,
auto num_queries = dataset.num_queries;
auto topk = dataset.topk;
auto dim = dataset.dim;
SubSearchResult sub_qr(num_queries, topk, metric_type);
auto round_decimal = dataset.round_decimal;
SubSearchResult sub_qr(num_queries, topk, metric_type, round_decimal);
auto query_data = reinterpret_cast<const float*>(dataset.query_data);
auto chunk_data = reinterpret_cast<const float*>(chunk_data_raw);
if (metric_type == MetricType::METRIC_L2) {
faiss::float_maxheap_array_t buf{(size_t)num_queries, (size_t)topk, sub_qr.get_labels(), sub_qr.get_values()};
faiss::knn_L2sqr(query_data, chunk_data, dim, num_queries, size_per_chunk, &buf, bitset);
sub_qr.round_values();
return sub_qr;
} else {
faiss::float_minheap_array_t buf{(size_t)num_queries, (size_t)topk, sub_qr.get_labels(), sub_qr.get_values()};
faiss::knn_inner_product(query_data, chunk_data, dim, num_queries, size_per_chunk, &buf, bitset);
sub_qr.round_values();
return sub_qr;
}
}
@ -116,6 +120,6 @@ BinarySearchBruteForce(const dataset::SearchDataset& dataset,
auto query_data = reinterpret_cast<const uint8_t*>(dataset.query_data);
auto chunk_data = reinterpret_cast<const uint8_t*>(chunk_data_raw);
return BinarySearchBruteForceFast(dataset.metric_type, dataset.dim, chunk_data, size_per_chunk, dataset.topk,
dataset.num_queries, query_data, bitset);
dataset.num_queries, dataset.round_decimal, query_data, bitset);
}
} // namespace milvus::query

View File

@ -50,12 +50,12 @@ FloatSearch(const segcore::SegmentGrowingImpl& segment,
auto topk = info.topk_;
auto total_count = topk * num_queries;
auto metric_type = info.metric_type_;
auto round_decimal = info.round_decimal_;
// step 3: small indexing search
// std::vector<int64_t> final_uids(total_count, -1);
// std::vector<float> final_dis(total_count, std::numeric_limits<float>::max());
SubSearchResult final_qr(num_queries, topk, metric_type);
dataset::SearchDataset search_dataset{metric_type, num_queries, topk, dim, query_data};
SubSearchResult final_qr(num_queries, topk, metric_type, round_decimal);
dataset::SearchDataset search_dataset{metric_type, num_queries, topk, round_decimal, dim, query_data};
auto vec_ptr = record.get_field_data<FloatVector>(vecfield_offset);
int current_chunk_id = 0;
@ -109,7 +109,6 @@ FloatSearch(const segcore::SegmentGrowingImpl& segment,
final_qr.merge(sub_qr);
}
current_chunk_id = max_chunk;
results.result_distances_ = std::move(final_qr.mutable_values());
results.internal_seg_offsets_ = std::move(final_qr.mutable_labels());
results.topk_ = topk;
@ -149,9 +148,9 @@ BinarySearch(const segcore::SegmentGrowingImpl& segment,
auto dim = field.get_dim();
auto topk = info.topk_;
auto total_count = topk * num_queries;
auto round_decimal = info.round_decimal_;
// step 3: small indexing search
query::dataset::SearchDataset search_dataset{metric_type, num_queries, topk, dim, query_data};
query::dataset::SearchDataset search_dataset{metric_type, num_queries, topk, round_decimal, dim, query_data};
auto vec_ptr = record.get_field_data<BinaryVector>(vecfield_offset);
@ -160,7 +159,7 @@ BinarySearch(const segcore::SegmentGrowingImpl& segment,
auto vec_size_per_chunk = vec_ptr->get_size_per_chunk();
auto max_chunk = upper_div(ins_barrier, vec_size_per_chunk);
SubSearchResult final_result(num_queries, topk, metric_type);
SubSearchResult final_result(num_queries, topk, metric_type, round_decimal);
for (int chunk_id = max_indexed_id; chunk_id < max_chunk; ++chunk_id) {
auto& chunk = vec_ptr->get_chunk(chunk_id);
auto element_begin = chunk_id * vec_size_per_chunk;
@ -179,6 +178,7 @@ BinarySearch(const segcore::SegmentGrowingImpl& segment,
final_result.merge(sub_result);
}
final_result.round_values();
results.result_distances_ = std::move(final_result.mutable_values());
results.internal_seg_offsets_ = std::move(final_result.mutable_labels());
results.topk_ = topk;

View File

@ -20,7 +20,7 @@ SearchOnIndex(const dataset::SearchDataset& search_dataset,
auto topK = search_dataset.topk;
auto dim = search_dataset.dim;
auto metric_type = search_dataset.metric_type;
auto round_decimal = search_dataset.round_decimal;
auto dataset = knowhere::GenDataset(num_queries, dim, search_dataset.query_data);
// NOTE: VecIndex Query API forget to add const qualifier
@ -31,9 +31,10 @@ SearchOnIndex(const dataset::SearchDataset& search_dataset,
auto dis = ans->Get<float*>(milvus::knowhere::meta::DISTANCE);
auto uids = ans->Get<int64_t*>(milvus::knowhere::meta::IDS);
SubSearchResult sub_qr(num_queries, topK, metric_type);
SubSearchResult sub_qr(num_queries, topK, metric_type, round_decimal);
std::copy_n(dis, num_queries * topK, sub_qr.get_values());
std::copy_n(uids, num_queries * topK, sub_qr.get_labels());
sub_qr.round_values();
return sub_qr;
}

View File

@ -20,6 +20,7 @@
#include "knowhere/index/vector_index/helpers/IndexParameter.h"
#include "knowhere/index/vector_index/adapter/VectorAdapter.h"
#include <boost_ext/dynamic_bitset_ext.hpp>
#include <cmath>
namespace milvus::query {
@ -67,6 +68,7 @@ SearchOnSealed(const Schema& schema,
const faiss::BitsetView& bitset,
SearchResult& result) {
auto topk = search_info.topk_;
auto round_decimal = search_info.round_decimal_;
auto field_offset = search_info.field_offset_;
auto& field = schema[field_offset];
@ -95,6 +97,14 @@ SearchOnSealed(const Schema& schema,
auto distances = final->Get<float*>(knowhere::meta::DISTANCE);
auto total_num = num_queries * topk;
const float multiplier = pow(10.0, round_decimal);
if (round_decimal != -1) {
const float multiplier = pow(10.0, round_decimal);
for (int i = 0; i < total_num; i++) {
distances[i] = round(distances[i] * multiplier) / multiplier;
}
}
result.internal_seg_offsets_.resize(total_num);
result.result_distances_.resize(total_num);
result.num_queries_ = num_queries;

View File

@ -12,6 +12,7 @@
#include "exceptions/EasyAssert.h"
#include "query/SubSearchResult.h"
#include "segcore/Reduce.h"
#include <cmath>
namespace milvus::query {
@ -74,4 +75,14 @@ SubSearchResult::merge(const SubSearchResult& left, const SubSearchResult& right
return left_copy;
}
void
SubSearchResult::round_values() {
if (round_decimal_ == -1)
return;
const float multiplier = pow(10.0, round_decimal_);
for (auto it = this->values_.begin(); it != this->values_.end(); it++) {
*it = round(*it * multiplier) / multiplier;
}
}
} // namespace milvus::query

View File

@ -17,12 +17,13 @@ namespace milvus::query {
class SubSearchResult {
public:
SubSearchResult(int64_t num_queries, int64_t topk, MetricType metric_type)
SubSearchResult(int64_t num_queries, int64_t topk, MetricType metric_type, int64_t round_decimal)
: metric_type_(metric_type),
num_queries_(num_queries),
topk_(topk),
labels_(num_queries * topk, -1),
values_(num_queries * topk, init_value(metric_type)) {
values_(num_queries * topk, init_value(metric_type)),
round_decimal_(round_decimal) {
}
public:
@ -75,6 +76,8 @@ class SubSearchResult {
mutable_values() {
return values_;
}
void
round_values();
static SubSearchResult
merge(const SubSearchResult& left, const SubSearchResult& right);
@ -90,6 +93,7 @@ class SubSearchResult {
private:
int64_t num_queries_;
int64_t topk_;
int64_t round_decimal_;
MetricType metric_type_;
std::vector<int64_t> labels_;
std::vector<float> values_;

View File

@ -19,6 +19,7 @@ struct SearchDataset {
MetricType metric_type;
int64_t num_queries;
int64_t topk;
int64_t round_decimal;
int64_t dim;
const void* query_data;
};

View File

@ -63,9 +63,9 @@ class ExecPlanNodeVisitor : PlanNodeVisitor {
#endif
static SearchResult
empty_search_result(int64_t num_queries, int64_t topk, MetricType metric_type) {
empty_search_result(int64_t num_queries, int64_t topk, int64_t round_decimal, MetricType metric_type) {
SearchResult final_result;
SubSearchResult result(num_queries, topk, metric_type);
SubSearchResult result(num_queries, topk, metric_type, round_decimal);
final_result.num_queries_ = num_queries;
final_result.topk_ = topk;
final_result.internal_seg_offsets_ = std::move(result.mutable_labels());
@ -93,7 +93,8 @@ ExecPlanNodeVisitor::VectorVisitorImpl(VectorPlanNode& node) {
// skip all calculation
if (active_count == 0) {
ret_ = empty_search_result(num_queries, node.search_info_.topk_, node.search_info_.metric_type_);
ret_ = empty_search_result(num_queries, node.search_info_.topk_, node.search_info_.round_decimal_,
node.search_info_.metric_type_);
return;
}

View File

@ -241,6 +241,7 @@ SegmentSealedImpl::vector_search(int64_t vec_count,
dataset.metric_type = search_info.metric_type_;
dataset.topk = search_info.topk_;
dataset.dim = field_meta.get_dim();
dataset.round_decimal = search_info.round_decimal_;
AssertInfo(get_bit(field_data_ready_bitset_, field_offset),
"Can't get bitset element at " + std::to_string(field_offset.get()));

File diff suppressed because it is too large Load Diff

View File

@ -91,7 +91,8 @@ TEST(Expr, Range) {
"nprobe": 10
},
"query": "$0",
"topk": 10
"topk": 10,
"round_decimal": 3
}
}
}
@ -133,7 +134,8 @@ TEST(Expr, RangeBinary) {
"nprobe": 10
},
"query": "$0",
"topk": 10
"topk": 10,
"round_decimal": 3
}
}
}
@ -284,7 +286,8 @@ TEST(Expr, TestRange) {
"nprobe": 10
},
"query": "$0",
"topk": 10
"topk": 10,
"round_decimal": 3
}
}
}
@ -366,7 +369,8 @@ TEST(Expr, TestTerm) {
"nprobe": 10
},
"query": "$0",
"topk": 10
"topk": 10,
"round_decimal": 3
}
}
}
@ -422,7 +426,8 @@ TEST(Expr, TestSimpleDsl) {
"nprobe": 10
},
"query": "$0",
"topk": 10
"topk": 10,
"round_decimal": 3
}
}
}
@ -537,7 +542,8 @@ TEST(Expr, TestCompare) {
"nprobe": 10
},
"query": "$0",
"topk": 10
"topk": 10,
"round_decimal": 3
}
}
}

View File

@ -239,6 +239,7 @@ TEST(Indexing, BinaryBruteForce) {
int64_t N = 100000;
int64_t num_queries = 10;
int64_t topk = 5;
int64_t round_decimal = 3;
int64_t dim = 8192;
auto result_count = topk * num_queries;
auto schema = std::make_shared<Schema>();
@ -251,6 +252,7 @@ TEST(Indexing, BinaryBruteForce) {
faiss::MetricType::METRIC_Jaccard, //
num_queries, //
topk, //
round_decimal,
dim, //
query_data //
};
@ -270,73 +272,73 @@ TEST(Indexing, BinaryBruteForce) {
[
[
"1024->0.000000",
"48942->0.641860",
"18494->0.643842",
"68225->0.644310",
"93557->0.644320"
"48942->0.642000",
"18494->0.644000",
"68225->0.644000",
"93557->0.644000"
],
[
"1025->0.000000",
"73557->0.641265",
"53086->0.642550",
"9737->0.642834",
"62855->0.643802"
"73557->0.641000",
"53086->0.643000",
"9737->0.643000",
"62855->0.644000"
],
[
"1026->0.000000",
"62904->0.643793",
"46758->0.644466",
"57969->0.645362",
"98113->0.645536"
"62904->0.644000",
"46758->0.644000",
"57969->0.645000",
"98113->0.646000"
],
[
"1027->0.000000",
"92446->0.638235",
"96034->0.640414",
"92129->0.644207",
"45887->0.644415"
"92446->0.638000",
"96034->0.640000",
"92129->0.644000",
"45887->0.644000"
],
[
"1028->0.000000",
"22992->0.643497",
"73903->0.644090",
"19969->0.644948",
"65178->0.645119"
"22992->0.643000",
"73903->0.644000",
"19969->0.645000",
"65178->0.645000"
],
[
"1029->0.000000",
"19776->0.641118",
"15166->0.641916",
"85470->0.642357",
"16730->0.642997"
"19776->0.641000",
"15166->0.642000",
"85470->0.642000",
"16730->0.643000"
],
[
"1030->0.000000",
"55939->0.639781",
"84253->0.642518",
"31958->0.644277",
"11667->0.645638"
"55939->0.640000",
"84253->0.643000",
"31958->0.644000",
"11667->0.646000"
],
[
"1031->0.000000",
"89536->0.637372",
"61622->0.637850",
"9275->0.639138",
"91403->0.640182"
"89536->0.637000",
"61622->0.638000",
"9275->0.639000",
"91403->0.640000"
],
[
"1032->0.000000",
"69504->0.642279",
"23414->0.643707",
"48770->0.645262",
"23231->0.645433"
"69504->0.642000",
"23414->0.644000",
"48770->0.645000",
"23231->0.645000"
],
[
"1033->0.000000",
"33540->0.635903",
"25310->0.639673",
"18576->0.640482",
"73729->0.641620"
"33540->0.636000",
"25310->0.640000",
"18576->0.640000",
"73729->0.642000"
]
]
]

View File

@ -92,6 +92,7 @@ vector_anns: <
>
query_info: <
topk: 10
round_decimal: 3
metric_type: "L2"
search_params: "{\"nprobe\": 10}"
>
@ -130,7 +131,8 @@ vector_anns: <
"nprobe": 10
},
"query": "$0",
"topk": 10
"topk": 10,
"round_decimal": 3
}
}
}
@ -178,6 +180,7 @@ vector_anns: <
>
query_info: <
topk: 10
round_decimal: 3
metric_type: "L2"
search_params: "{\"nprobe\": 10}"
>
@ -216,7 +219,8 @@ vector_anns: <
"nprobe": 10
},
"query": "$0",
"topk": 10
"topk": 10,
"round_decimal": 3
}
}
}
@ -265,6 +269,7 @@ vector_anns: <
>
query_info: <
topk: 10
round_decimal: 3
metric_type: "L2"
search_params: "{\"nprobe\": 10}"
>
@ -305,7 +310,8 @@ vector_anns: <
"nprobe": 10
},
"query": "$0",
"topk": 10
"topk": 10,
"round_decimal": 3
}
}
}
@ -385,6 +391,7 @@ vector_anns: <
>
query_info: <
topk: 10
round_decimal: 3
metric_type: "L2"
search_params: "{\"nprobe\": 10}"
>
@ -441,7 +448,8 @@ vector_anns: <
"nprobe": 10
},
"query": "$0",
"topk": 10
"topk": 10,
"round_decimal": 3
}
}
}
@ -483,6 +491,7 @@ vector_anns: <
>
query_info: <
topk: 10
round_decimal: 3
metric_type: "L2"
search_params: "{\"nprobe\": 10}"
>
@ -521,7 +530,8 @@ vector_anns: <
"nprobe": 10
},
"query": "$0",
"topk": 10
"topk": 10,
"round_decimal": 3
}
}
}

View File

@ -69,7 +69,8 @@ TEST(Query, DSL) {
"nprobe": 10
},
"query": "$0",
"topk": 10
"topk": 10,
"round_decimal": 3
}
}
}
@ -94,7 +95,8 @@ TEST(Query, DSL) {
"nprobe": 10
},
"query": "$0",
"topk": 10
"topk": 10,
"round_decimal": 3
}
}
}
@ -117,7 +119,8 @@ TEST(Query, ParsePlaceholderGroup) {
"nprobe": 10
},
"query": "$0",
"topk": 10
"topk": 10,
"round_decimal":3
}
}
}
@ -158,7 +161,8 @@ TEST(Query, ExecWithPredicateLoader) {
"nprobe": 10
},
"query": "$0",
"topk": 5
"topk": 5,
"round_decimal": 3
}
}
}
@ -187,11 +191,11 @@ TEST(Query, ExecWithPredicateLoader) {
auto ref = json::parse(R"(
[
[
["982->0.000000", "25315->4.741588", "57893->4.758279", "1499->6.065844", "48201->6.075444"],
["41772->10.110812", "80693->11.712179", "74859->11.789503", "79777->11.842009", "31878->12.308037"],
["65551->4.453535", "21617->5.143645", "50037->5.266983", "72204->5.331940", "96905->5.478549"],
["59219->5.458013", "21995->6.077806", "61367->7.028524", "44657->7.030965", "66957->7.174046"],
["66353->5.695784", "41087->5.916901", "97780->6.810793", "99239->7.561522", "86527->7.750667"]
["982->0.000000", "25315->4.742000", "57893->4.758000", "1499->6.066000", "48201->6.075000"],
["41772->10.111000", "80693->11.712000", "74859->11.790000", "79777->11.842000", "31878->12.308000"],
["65551->4.454000", "21617->5.144000", "50037->5.267000", "72204->5.332000", "96905->5.479000"],
["59219->5.458000", "21995->6.078000", "61367->7.029000", "44657->7.031000", "66957->7.174000"],
["66353->5.696000", "41087->5.917000", "97780->6.811000", "99239->7.562000", "86527->7.751000"]
]
])");
std::cout << json.dump(2);
@ -223,7 +227,8 @@ TEST(Query, ExecWithPredicateSmallN) {
"nprobe": 10
},
"query": "$0",
"topk": 5
"topk": 5,
"round_decimal": 3
}
}
}
@ -274,7 +279,8 @@ TEST(Query, ExecWithPredicate) {
"nprobe": 10
},
"query": "$0",
"topk": 5
"topk": 5,
"round_decimal": 3
}
}
}
@ -300,11 +306,11 @@ TEST(Query, ExecWithPredicate) {
auto ref = json::parse(R"(
[
[
["982->0.000000", "25315->4.741588", "57893->4.758279", "1499->6.065844", "48201->6.075444"],
["41772->10.110812", "80693->11.712179", "74859->11.789503", "79777->11.842009", "31878->12.308037"],
["65551->4.453535", "21617->5.143645", "50037->5.266983", "72204->5.331940", "96905->5.478549"],
["59219->5.458013", "21995->6.077806", "61367->7.028524", "44657->7.030965", "66957->7.174046"],
["66353->5.695784", "41087->5.916901", "97780->6.810793", "99239->7.561522", "86527->7.750667"]
["982->0.000000", "25315->4.742000", "57893->4.758000", "1499->6.066000", "48201->6.075000"],
["41772->10.111000", "80693->11.712000", "74859->11.790000", "79777->11.842000", "31878->12.308000"],
["65551->4.454000", "21617->5.144000", "50037->5.267000", "72204->5.332000", "96905->5.479000"],
["59219->5.458000", "21995->6.078000", "61367->7.029000", "44657->7.031000", "66957->7.174000"],
["66353->5.696000", "41087->5.917000", "97780->6.811000", "99239->7.562000", "86527->7.751000"]
]
])");
std::cout << json.dump(2);
@ -335,7 +341,8 @@ TEST(Query, ExecTerm) {
"nprobe": 10
},
"query": "$0",
"topk": 5
"topk": 5,
"round_decimal": 3
}
}
}
@ -381,7 +388,8 @@ TEST(Query, ExecEmpty) {
"nprobe": 10
},
"query": "$0",
"topk": 5
"topk": 5,
"round_decimal": 3
}
}
}
@ -425,7 +433,8 @@ TEST(Query, ExecWithoutPredicateFlat) {
"nprobe": 10
},
"query": "$0",
"topk": 5
"topk": 5,
"round_decimal": 3
}
}
}
@ -469,7 +478,8 @@ TEST(Query, ExecWithoutPredicate) {
"nprobe": 10
},
"query": "$0",
"topk": 5
"topk": 5,
"round_decimal":3
}
}
}
@ -496,11 +506,11 @@ TEST(Query, ExecWithoutPredicate) {
auto ref = json::parse(R"(
[
[
["982->0.000000", "25315->4.741588", "57893->4.758279", "1499->6.065844", "48201->6.075444"],
["41772->10.110812", "80693->11.712179", "74859->11.789503", "79777->11.842009", "31878->12.308037"],
["65551->4.453535", "21617->5.143645", "50037->5.266983", "98268->5.320526", "72204->5.331940"],
["33572->5.431973", "59219->5.458013", "21995->6.077806", "17913->6.830949", "86628->6.834784"],
["66353->5.695784", "41087->5.916901", "24554->6.195437", "68019->6.653711", "97780->6.810793"]
["982->0.000000", "25315->4.742000", "57893->4.758000", "1499->6.066000", "48201->6.075000"],
["41772->10.111000", "80693->11.712000", "74859->11.790000", "79777->11.842000", "31878->12.308000"],
["65551->4.454000", "21617->5.144000", "50037->5.267000", "98268->5.321000", "72204->5.332000"],
["33572->5.432000", "59219->5.458000", "21995->6.078000", "17913->6.831000", "86628->6.835000"],
["66353->5.696000", "41087->5.917000", "24554->6.195000", "68019->6.654000", "97780->6.811000"]
]
])");
std::cout << json.dump(2);
@ -524,7 +534,8 @@ TEST(Indexing, InnerProduct) {
"nprobe": 10
},
"query": "$0",
"topk": 5
"topk": 5,
"round_decimal":3
}
}
}
@ -633,7 +644,8 @@ TEST(Query, FillSegment) {
"nprobe": 10
},
"query": "$0",
"topk": 5
"topk": 5,
"round_decimal": 3
}
}
}
@ -710,7 +722,8 @@ TEST(Query, ExecWithPredicateBinary) {
"nprobe": 10
},
"query": "$0",
"topk": 5
"topk": 5,
"round_decimal": 3
}
}
}
@ -737,4 +750,4 @@ TEST(Query, ExecWithPredicateBinary) {
Json json = SearchResultToJson(sr);
std::cout << json.dump(2);
// ASSERT_EQ(json.dump(2), ref.dump(2));
}
}

View File

@ -22,6 +22,7 @@ TEST(Reduce, SubQueryResult) {
int64_t num_queries = 512;
int64_t topk = 32;
int64_t iteration = 50;
int64_t round_decimal = 3;
constexpr int64_t limit = 100000000L;
auto metric_type = MetricType::METRIC_L2;
using queue_type = std::priority_queue<int64_t>;
@ -33,7 +34,7 @@ TEST(Reduce, SubQueryResult) {
}
}
std::default_random_engine e(42);
SubSearchResult final_result(num_queries, topk, metric_type);
SubSearchResult final_result(num_queries, topk, metric_type, round_decimal);
for (int i = 0; i < iteration; ++i) {
std::vector<int64_t> labels;
std::vector<float> values;
@ -48,7 +49,7 @@ TEST(Reduce, SubQueryResult) {
std::sort(labels.begin() + n * topk, labels.begin() + n * topk + topk);
std::sort(values.begin() + n * topk, values.begin() + n * topk + topk);
}
SubSearchResult sub_result(num_queries, topk, metric_type);
SubSearchResult sub_result(num_queries, topk, metric_type, round_decimal);
sub_result.mutable_values() = values;
sub_result.mutable_labels() = labels;
final_result.merge(sub_result);
@ -72,6 +73,7 @@ TEST(Reduce, SubSearchResultDesc) {
int64_t num_queries = 512;
int64_t topk = 32;
int64_t iteration = 50;
int64_t round_decimal = 3;
constexpr int64_t limit = 100000000L;
constexpr int64_t init_value = 0;
auto metric_type = MetricType::METRIC_INNER_PRODUCT;
@ -84,7 +86,7 @@ TEST(Reduce, SubSearchResultDesc) {
}
}
std::default_random_engine e(42);
SubSearchResult final_result(num_queries, topk, metric_type);
SubSearchResult final_result(num_queries, topk, metric_type, round_decimal);
for (int i = 0; i < iteration; ++i) {
std::vector<int64_t> labels;
std::vector<float> values;
@ -99,7 +101,7 @@ TEST(Reduce, SubSearchResultDesc) {
std::sort(labels.begin() + n * topk, labels.begin() + n * topk + topk, std::greater<int64_t>());
std::sort(values.begin() + n * topk, values.begin() + n * topk + topk, std::greater<float>());
}
SubSearchResult sub_result(num_queries, topk, metric_type);
SubSearchResult sub_result(num_queries, topk, metric_type, round_decimal);
sub_result.mutable_values() = values;
sub_result.mutable_labels() = labels;
final_result.merge(sub_result);

View File

@ -48,7 +48,8 @@ TEST(Sealed, without_predicate) {
"nprobe": 10
},
"query": "$0",
"topk": 5
"topk": 5,
"round_decimal": 3
}
}
}
@ -117,9 +118,11 @@ TEST(Sealed, without_predicate) {
sr = sealed_segment->Search(plan.get(), *ph_group, time);
auto post_result = SearchResultToJson(sr);
std::cout << ref_result.dump(1);
std::cout << "ref_result"<< std::endl;
std::cout << ref_result.dump(1) << std::endl;
std::cout << "post_result" << std::endl;
std::cout << post_result.dump(1);
ASSERT_EQ(ref_result.dump(2), post_result.dump(2));
// ASSERT_EQ(ref_result.dump(1), post_result.dump(1));
}
TEST(Sealed, with_predicate) {
@ -150,7 +153,8 @@ TEST(Sealed, with_predicate) {
"nprobe": 10
},
"query": "$0",
"topk": 5
"topk": 5,
"round_decimal": 6
}
}
}
@ -251,7 +255,8 @@ TEST(Sealed, LoadFieldData) {
"nprobe": 10
},
"query": "$0",
"topk": 5
"topk": 5,
"round_decimal": 3
}
}
}
@ -305,12 +310,12 @@ TEST(Sealed, LoadFieldData) {
auto std_json = Json::parse(R"(
[
[
["982->0.000000", "25315->4.741588", "57893->4.758279", "48201->6.075444", "53853->6.223233"],
["41772->10.110812", "74859->11.789503", "79777->11.842009", "3785->11.982730", "35888->12.193403"],
["59251->2.542610", "65551->4.453535", "72204->5.331940", "96905->5.478549", "87833->5.765219"],
["59219->5.458013", "21995->6.077806", "97922->6.763784", "25710->7.158085", "14048->7.294020"],
["66353->5.695784", "30664->5.880683", "41087->5.916901", "10393->6.632869", "90215->7.201704"]
["982->0.000000", "25315->4.742000", "57893->4.758000", "48201->6.075000", "53853->6.223000"],
["41772->10.111000", "74859->11.790000", "79777->11.842000", "3785->11.983000", "35888->12.193000"],
["59251->2.543000", "65551->4.454000", "72204->5.332000", "96905->5.479000", "87833->5.765000"],
["59219->5.458000", "21995->6.078000", "97922->6.764000", "25710->7.158000", "14048->7.294000"],
["66353->5.696000", "30664->5.881000", "41087->5.917000", "10393->6.633000", "90215->7.202000"]
]
])");
ASSERT_EQ(std_json.dump(-2), json.dump(-2));
}
}

View File

@ -26,6 +26,7 @@ message QueryInfo {
int64 topk = 1;
string metric_type = 3;
string search_params = 4;
int64 round_decimal = 5;
}
message ColumnInfo {

View File

@ -213,6 +213,7 @@ type QueryInfo struct {
Topk int64 `protobuf:"varint,1,opt,name=topk,proto3" json:"topk,omitempty"`
MetricType string `protobuf:"bytes,3,opt,name=metric_type,json=metricType,proto3" json:"metric_type,omitempty"`
SearchParams string `protobuf:"bytes,4,opt,name=search_params,json=searchParams,proto3" json:"search_params,omitempty"`
RoundDecimal int64 `protobuf:"varint,5,opt,name=round_decimal,json=roundDecimal,proto3" json:"round_decimal,omitempty"`
XXX_NoUnkeyedLiteral struct{} `json:"-"`
XXX_unrecognized []byte `json:"-"`
XXX_sizecache int32 `json:"-"`
@ -264,6 +265,13 @@ func (m *QueryInfo) GetSearchParams() string {
return ""
}
func (m *QueryInfo) GetRoundDecimal() int64 {
if m != nil {
return m.RoundDecimal
}
return 0
}
type ColumnInfo struct {
FieldId int64 `protobuf:"varint,1,opt,name=field_id,json=fieldId,proto3" json:"field_id,omitempty"`
DataType schemapb.DataType `protobuf:"varint,2,opt,name=data_type,json=dataType,proto3,enum=milvus.proto.schema.DataType" json:"data_type,omitempty"`
@ -977,72 +985,74 @@ func init() {
func init() { proto.RegisterFile("plan.proto", fileDescriptor_2d655ab2f7683c23) }
var fileDescriptor_2d655ab2f7683c23 = []byte{
// 1067 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x56, 0xdd, 0x72, 0xdb, 0x44,
0x14, 0xb6, 0x2c, 0xdb, 0x91, 0x8e, 0x5c, 0xdb, 0xd5, 0x0d, 0x29, 0xa1, 0x24, 0x88, 0x0e, 0x0d,
0x30, 0x4d, 0x86, 0xb6, 0xa4, 0x43, 0x19, 0x98, 0xfc, 0xb4, 0xc4, 0x1e, 0x4a, 0x12, 0x44, 0xc8,
0x05, 0x37, 0x9a, 0xb5, 0xb4, 0xb1, 0x77, 0xba, 0xd6, 0x2a, 0xab, 0x95, 0xa9, 0xaf, 0x79, 0x02,
0x5e, 0x02, 0xae, 0xe1, 0x8e, 0x77, 0xe0, 0x01, 0xb8, 0xe7, 0x45, 0x98, 0x3d, 0xab, 0xf8, 0xa7,
0xe3, 0xa4, 0x61, 0xa6, 0x77, 0xab, 0xb3, 0xe7, 0xef, 0xfb, 0xce, 0xcf, 0x0a, 0x20, 0xe3, 0x24,
0xdd, 0xca, 0xa4, 0x50, 0xc2, 0xbf, 0x3d, 0x62, 0x7c, 0x5c, 0xe4, 0xe6, 0x6b, 0x4b, 0x5f, 0xbc,
0xdb, 0xcc, 0xe3, 0x21, 0x1d, 0x11, 0x23, 0x0a, 0x32, 0x68, 0x1e, 0xd2, 0x94, 0x4a, 0x16, 0x9f,
0x11, 0x5e, 0x50, 0x7f, 0x0d, 0x9c, 0xbe, 0x10, 0x3c, 0x1a, 0x13, 0xbe, 0x6a, 0x6d, 0x58, 0x9b,
0x4e, 0xb7, 0x12, 0xae, 0x68, 0xc9, 0x19, 0xe1, 0xfe, 0x5d, 0x70, 0x59, 0xaa, 0x76, 0x1e, 0xe3,
0x6d, 0x75, 0xc3, 0xda, 0xb4, 0xbb, 0x95, 0xd0, 0x41, 0x51, 0x79, 0x7d, 0xce, 0x05, 0x51, 0x78,
0x6d, 0x6f, 0x58, 0x9b, 0x96, 0xbe, 0x46, 0xd1, 0x19, 0xe1, 0xfb, 0x75, 0xb0, 0xc7, 0x84, 0x07,
0x14, 0xdc, 0xef, 0x0b, 0x2a, 0x27, 0xbd, 0xf4, 0x5c, 0xf8, 0x3e, 0xd4, 0x94, 0xc8, 0x5e, 0x62,
0x28, 0x3b, 0xc4, 0xb3, 0xbf, 0x0e, 0xde, 0x88, 0x2a, 0xc9, 0xe2, 0x48, 0x4d, 0x32, 0x8a, 0x8e,
0xdc, 0x10, 0x8c, 0xe8, 0x74, 0x92, 0x51, 0xff, 0x43, 0xb8, 0x95, 0x53, 0x22, 0xe3, 0x61, 0x94,
0x11, 0x49, 0x46, 0xf9, 0x6a, 0x0d, 0x55, 0x9a, 0x46, 0x78, 0x82, 0xb2, 0xe0, 0x37, 0x0b, 0xe0,
0x40, 0xf0, 0x62, 0x94, 0x62, 0xa0, 0x3b, 0xe0, 0x9c, 0x33, 0xca, 0x93, 0x88, 0x25, 0x65, 0xb0,
0x15, 0xfc, 0xee, 0x25, 0xfe, 0x53, 0x70, 0x13, 0xa2, 0x88, 0x89, 0xa6, 0x51, 0xb5, 0x1e, 0xde,
0xdd, 0x5a, 0xe0, 0xad, 0x64, 0xec, 0x19, 0x51, 0x44, 0x27, 0x10, 0x3a, 0x49, 0x79, 0xf2, 0xef,
0x41, 0x8b, 0xe5, 0x51, 0x26, 0xd9, 0x88, 0xc8, 0x49, 0xf4, 0x92, 0x4e, 0x30, 0x5d, 0x27, 0x6c,
0xb2, 0xfc, 0xc4, 0x08, 0xbf, 0xa5, 0x13, 0x7f, 0x0d, 0x5c, 0x96, 0x47, 0xa4, 0x50, 0xa2, 0xf7,
0x0c, 0x93, 0x75, 0x42, 0x87, 0xe5, 0x7b, 0xf8, 0x1d, 0xfc, 0x69, 0x41, 0xeb, 0xc7, 0x94, 0xc8,
0x49, 0x48, 0xd2, 0x01, 0x7d, 0xfe, 0x2a, 0x93, 0xfe, 0xd7, 0xe0, 0xc5, 0x98, 0x7a, 0xc4, 0xd2,
0x73, 0x81, 0xf9, 0x7a, 0xaf, 0xe7, 0x84, 0x45, 0x9e, 0x01, 0x0c, 0x21, 0x9e, 0x81, 0xfd, 0x18,
0xaa, 0x22, 0x2b, 0xa1, 0xdc, 0x59, 0x62, 0x76, 0x9c, 0x21, 0x8c, 0xaa, 0xc8, 0xfc, 0xcf, 0xa1,
0x3e, 0xd6, 0x85, 0xc7, 0xbc, 0xbd, 0x87, 0xeb, 0x4b, 0xb4, 0xe7, 0xfb, 0x23, 0x34, 0xda, 0xc1,
0xef, 0x55, 0x68, 0xef, 0xb3, 0xb7, 0x9b, 0xf5, 0x7d, 0x68, 0x73, 0xf1, 0x33, 0x95, 0x11, 0x4b,
0x63, 0x5e, 0xe4, 0x6c, 0x6c, 0xaa, 0xe1, 0x84, 0x2d, 0x14, 0xf7, 0x2e, 0xa5, 0x5a, 0xb1, 0xc8,
0xb2, 0x05, 0x45, 0xc3, 0x7a, 0x0b, 0xc5, 0x33, 0xc5, 0x5d, 0xf0, 0x8c, 0x47, 0x03, 0xb1, 0x76,
0x33, 0x88, 0x80, 0x36, 0x66, 0x1c, 0x76, 0xc1, 0x33, 0xa1, 0x8c, 0x87, 0xfa, 0x0d, 0x3d, 0xa0,
0x0d, 0x9e, 0x83, 0xbf, 0x2d, 0xf0, 0x0e, 0xc4, 0x28, 0x23, 0xd2, 0xb0, 0x74, 0x08, 0x1d, 0x4e,
0xcf, 0x55, 0xf4, 0xbf, 0xa9, 0x6a, 0x69, 0xb3, 0xb9, 0x8e, 0xee, 0xc1, 0x6d, 0xc9, 0x06, 0xc3,
0x45, 0x4f, 0xd5, 0x9b, 0x78, 0x6a, 0xa3, 0xdd, 0xc1, 0xeb, 0xfd, 0x62, 0xdf, 0xa0, 0x5f, 0x82,
0x5f, 0x2c, 0x70, 0x4e, 0xa9, 0x1c, 0xbd, 0x95, 0x8a, 0x3f, 0x81, 0x06, 0xf2, 0x9a, 0xaf, 0x56,
0x37, 0xec, 0x9b, 0x10, 0x5b, 0xaa, 0x07, 0xbf, 0x5a, 0xe0, 0xe2, 0xcc, 0x60, 0x1a, 0x8f, 0x31,
0x7d, 0x0b, 0xd3, 0xbf, 0xb7, 0xc4, 0xc5, 0x54, 0xd3, 0x9c, 0x8e, 0x33, 0xec, 0xfc, 0x07, 0x50,
0x8f, 0x87, 0x8c, 0x27, 0x25, 0x67, 0xef, 0x2c, 0x31, 0xd4, 0x36, 0xa1, 0xd1, 0x0a, 0xd6, 0x61,
0xa5, 0xb4, 0xf6, 0x3d, 0x58, 0xe9, 0xa5, 0x63, 0xc2, 0x59, 0xd2, 0xa9, 0xf8, 0x2b, 0x60, 0x1f,
0x09, 0xd5, 0xb1, 0x82, 0x7f, 0x2c, 0x00, 0x33, 0x12, 0x98, 0xd4, 0xce, 0x5c, 0x52, 0x1f, 0x2d,
0xf1, 0x3d, 0x53, 0x2d, 0x8f, 0x65, 0x5a, 0x9f, 0x42, 0x4d, 0x17, 0xfa, 0x4d, 0x59, 0xa1, 0x92,
0xc6, 0x80, 0xb5, 0x2c, 0xa7, 0xf7, 0x6a, 0x0c, 0xa8, 0x15, 0xec, 0x80, 0x73, 0x19, 0x6b, 0x11,
0x44, 0x0b, 0xe0, 0x85, 0x18, 0xb0, 0x98, 0xf0, 0xbd, 0x34, 0xe9, 0x58, 0xfe, 0x2d, 0x70, 0xcb,
0xef, 0x63, 0xd9, 0xa9, 0x06, 0x7f, 0xd8, 0x50, 0x43, 0x50, 0x4f, 0xc1, 0x55, 0x54, 0x8e, 0x22,
0xfa, 0x2a, 0x93, 0x65, 0xb9, 0xd7, 0x96, 0xc4, 0xbc, 0x6c, 0x10, 0xbd, 0xfe, 0xd5, 0x65, 0xb3,
0x7c, 0x05, 0x50, 0xe8, 0xd8, 0xc6, 0xd8, 0xc0, 0x7b, 0xef, 0xba, 0x6a, 0x75, 0x2b, 0xa1, 0x5b,
0x4c, 0xf9, 0xdc, 0x05, 0xaf, 0xcf, 0x66, 0xf6, 0xf6, 0x95, 0xbd, 0x36, 0x23, 0xb6, 0x5b, 0x09,
0xa1, 0x3f, 0xab, 0xc8, 0x01, 0x34, 0x63, 0x33, 0x88, 0xc6, 0x85, 0x59, 0x07, 0xef, 0x2f, 0x6d,
0xd7, 0xe9, 0xbc, 0x76, 0x2b, 0xa1, 0x17, 0xcf, 0x8d, 0xef, 0x77, 0xd0, 0x31, 0x28, 0xa4, 0xde,
0x7b, 0xc6, 0x91, 0xd9, 0x0a, 0x1f, 0x5c, 0x85, 0x65, 0xba, 0x21, 0xbb, 0x95, 0xb0, 0x55, 0x2c,
0xee, 0xcc, 0x13, 0xb8, 0x5d, 0xa2, 0x9a, 0xf3, 0xd7, 0x40, 0x7f, 0xc1, 0x95, 0xd8, 0xe6, 0x1d,
0xb6, 0xfb, 0x8b, 0xa2, 0xfd, 0x06, 0xd4, 0xb4, 0x93, 0xe0, 0x5f, 0x0b, 0xe0, 0x8c, 0xc6, 0x4a,
0xc8, 0xbd, 0xa3, 0xa3, 0x1f, 0xca, 0x27, 0xc8, 0x28, 0x9b, 0x87, 0x5d, 0x3f, 0x41, 0xc6, 0xdf,
0xc2, 0xe3, 0x58, 0x5d, 0x7c, 0x1c, 0x9f, 0x00, 0x64, 0x92, 0x26, 0x2c, 0x26, 0x8a, 0xe6, 0x6f,
0x6a, 0xb3, 0x39, 0x55, 0xff, 0x4b, 0x80, 0x0b, 0xfd, 0xcc, 0x9b, 0xd5, 0x50, 0xbb, 0xb2, 0xdc,
0xd3, 0x7f, 0x81, 0xd0, 0xbd, 0x98, 0xfe, 0x16, 0xdc, 0x87, 0x76, 0xc6, 0x49, 0x4c, 0x87, 0x82,
0x27, 0x54, 0x46, 0x8a, 0x0c, 0x90, 0x64, 0x37, 0x6c, 0xcd, 0x89, 0x4f, 0xc9, 0x20, 0xf8, 0xcb,
0x02, 0xe7, 0x84, 0x93, 0xf4, 0x48, 0x24, 0xb8, 0xac, 0xc7, 0x88, 0x38, 0x22, 0x69, 0x9a, 0x5f,
0xb3, 0x8e, 0x66, 0xbc, 0xe8, 0x16, 0x31, 0x36, 0x7b, 0x69, 0x9a, 0xfb, 0x5f, 0x2c, 0xa0, 0xbd,
0x7e, 0x04, 0xb5, 0xe9, 0x1c, 0xde, 0x4d, 0xe8, 0x88, 0x42, 0x65, 0x85, 0x8a, 0x2e, 0xa9, 0xd4,
0x74, 0xd9, 0x9b, 0x76, 0xd8, 0x32, 0xf2, 0x6f, 0x0c, 0xa3, 0xb9, 0xae, 0x50, 0x2a, 0x12, 0xfa,
0x49, 0x0a, 0x0d, 0xb3, 0x58, 0x17, 0x67, 0xb1, 0x0d, 0xde, 0xa1, 0xa4, 0x44, 0x51, 0x79, 0x3a,
0x24, 0x69, 0xc7, 0xf2, 0x3b, 0xd0, 0x2c, 0x05, 0xcf, 0x2f, 0x0a, 0xc2, 0x3b, 0x55, 0xbf, 0x09,
0xce, 0x0b, 0x9a, 0xe7, 0x78, 0x6f, 0xe3, 0xb0, 0xd2, 0x3c, 0x37, 0x97, 0x35, 0xdf, 0x85, 0xba,
0x39, 0xd6, 0xb5, 0xde, 0x91, 0x50, 0xe6, 0xab, 0xb1, 0xff, 0xe8, 0xa7, 0xcf, 0x06, 0x4c, 0x0d,
0x8b, 0xfe, 0x56, 0x2c, 0x46, 0xdb, 0x06, 0xd4, 0x03, 0x26, 0xca, 0xd3, 0x36, 0x4b, 0x15, 0x95,
0x29, 0xe1, 0xdb, 0x88, 0x73, 0x5b, 0xe3, 0xcc, 0xfa, 0xfd, 0x06, 0x7e, 0x3d, 0xfa, 0x2f, 0x00,
0x00, 0xff, 0xff, 0x6e, 0xb0, 0x90, 0xcb, 0x55, 0x0a, 0x00, 0x00,
// 1091 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x56, 0xcd, 0x72, 0x1b, 0xc5,
0x13, 0xd7, 0x6a, 0x25, 0x79, 0xb7, 0xa5, 0x48, 0xca, 0x5e, 0xfe, 0xce, 0x3f, 0x04, 0x9b, 0x25,
0x45, 0x0c, 0x54, 0xec, 0x22, 0x09, 0x4e, 0x11, 0x0a, 0xca, 0x5f, 0xc1, 0x52, 0x11, 0x6c, 0xb3,
0x18, 0x1f, 0xb8, 0x6c, 0x8d, 0x76, 0xc7, 0xd2, 0x54, 0x66, 0x77, 0xd6, 0xb3, 0xb3, 0x22, 0x3a,
0x73, 0xe3, 0xc6, 0x4b, 0xc0, 0x19, 0x6e, 0xbc, 0x03, 0x0f, 0xc0, 0x9d, 0x17, 0xa1, 0xa6, 0x67,
0xad, 0x8f, 0x94, 0xec, 0x98, 0xaa, 0xdc, 0x66, 0x7e, 0xd3, 0xdd, 0xd3, 0xbf, 0xfe, 0x9a, 0x01,
0xc8, 0x38, 0x49, 0x37, 0x33, 0x29, 0x94, 0xf0, 0x6e, 0x27, 0x8c, 0x8f, 0x8b, 0xdc, 0xec, 0x36,
0xf5, 0xc1, 0xff, 0x5b, 0x79, 0x34, 0xa2, 0x09, 0x31, 0x90, 0x9f, 0x41, 0xeb, 0x90, 0xa6, 0x54,
0xb2, 0xe8, 0x8c, 0xf0, 0x82, 0x7a, 0x77, 0xc1, 0x19, 0x08, 0xc1, 0xc3, 0x31, 0xe1, 0xab, 0xd6,
0xba, 0xb5, 0xe1, 0xf4, 0x2a, 0xc1, 0x8a, 0x46, 0xce, 0x08, 0xf7, 0xee, 0x81, 0xcb, 0x52, 0xb5,
0xfd, 0x04, 0x4f, 0xab, 0xeb, 0xd6, 0x86, 0xdd, 0xab, 0x04, 0x0e, 0x42, 0xe5, 0xf1, 0x39, 0x17,
0x44, 0xe1, 0xb1, 0xbd, 0x6e, 0x6d, 0x58, 0xfa, 0x18, 0xa1, 0x33, 0xc2, 0xf7, 0xea, 0x60, 0x8f,
0x09, 0xf7, 0x7f, 0xb6, 0xc0, 0xfd, 0xb6, 0xa0, 0x72, 0xd2, 0x4f, 0xcf, 0x85, 0xe7, 0x41, 0x4d,
0x89, 0xec, 0x25, 0xde, 0x65, 0x07, 0xb8, 0xf6, 0xd6, 0xa0, 0x99, 0x50, 0x25, 0x59, 0x14, 0xaa,
0x49, 0x46, 0xd1, 0x92, 0x1b, 0x80, 0x81, 0x4e, 0x27, 0x19, 0xf5, 0xde, 0x87, 0x5b, 0x39, 0x25,
0x32, 0x1a, 0x85, 0x19, 0x91, 0x24, 0xc9, 0x57, 0x6b, 0x28, 0xd2, 0x32, 0xe0, 0x09, 0x62, 0x5a,
0x48, 0x8a, 0x22, 0x8d, 0xc3, 0x98, 0x46, 0x2c, 0x21, 0x7c, 0xb5, 0x8e, 0x57, 0xb4, 0x10, 0x3c,
0x30, 0x98, 0xff, 0xab, 0x05, 0xb0, 0x2f, 0x78, 0x91, 0xa4, 0xe8, 0xcd, 0x1d, 0x70, 0xce, 0x19,
0xe5, 0x71, 0xc8, 0xe2, 0xd2, 0xa3, 0x15, 0xdc, 0xf7, 0x63, 0xef, 0x19, 0xb8, 0x31, 0x51, 0xc4,
0xb8, 0xa4, 0xb9, 0xb7, 0x1f, 0xdd, 0xdb, 0x5c, 0x88, 0x6e, 0x19, 0xd7, 0x03, 0xa2, 0x88, 0xf6,
0x32, 0x70, 0xe2, 0x72, 0xe5, 0xdd, 0x87, 0x36, 0xcb, 0xc3, 0x4c, 0xb2, 0x84, 0xc8, 0x49, 0xf8,
0x92, 0x4e, 0x90, 0x93, 0x13, 0xb4, 0x58, 0x7e, 0x62, 0xc0, 0xaf, 0xe9, 0xc4, 0xbb, 0x0b, 0x2e,
0xcb, 0x43, 0x52, 0x28, 0xd1, 0x3f, 0x40, 0x46, 0x4e, 0xe0, 0xb0, 0x7c, 0x17, 0xf7, 0xfe, 0x1f,
0x16, 0xb4, 0xbf, 0x4f, 0x89, 0x9c, 0x04, 0x24, 0x1d, 0xd2, 0xe7, 0xaf, 0x32, 0xe9, 0x7d, 0x09,
0xcd, 0x08, 0x5d, 0x0f, 0x59, 0x7a, 0x2e, 0xd0, 0xdf, 0xe6, 0xeb, 0x3e, 0x61, 0x29, 0xcc, 0x08,
0x06, 0x10, 0xcd, 0xc8, 0x7e, 0x08, 0x55, 0x91, 0x95, 0x54, 0xee, 0x2c, 0x51, 0x3b, 0xce, 0x90,
0x46, 0x55, 0x64, 0xde, 0xa7, 0x50, 0x1f, 0xeb, 0xf2, 0x40, 0xbf, 0x9b, 0x8f, 0xd6, 0x96, 0x48,
0xcf, 0x57, 0x51, 0x60, 0xa4, 0xfd, 0xdf, 0xaa, 0xd0, 0xd9, 0x63, 0x6f, 0xd7, 0xeb, 0x07, 0xd0,
0xe1, 0xe2, 0x47, 0x2a, 0x43, 0x96, 0x46, 0xbc, 0xc8, 0xd9, 0xd8, 0x64, 0xc3, 0x09, 0xda, 0x08,
0xf7, 0x2f, 0x51, 0x2d, 0x58, 0x64, 0xd9, 0x82, 0xa0, 0x89, 0x7a, 0x1b, 0xe1, 0x99, 0xe0, 0x0e,
0x34, 0x8d, 0x45, 0x43, 0xb1, 0x76, 0x33, 0x8a, 0x80, 0x3a, 0xa6, 0x69, 0x76, 0xa0, 0x69, 0xae,
0x32, 0x16, 0xea, 0x37, 0xb4, 0x80, 0x3a, 0xb8, 0xf6, 0xff, 0xb2, 0xa0, 0xb9, 0x2f, 0x92, 0x8c,
0x48, 0x13, 0xa5, 0x43, 0xe8, 0x72, 0x7a, 0xae, 0xc2, 0xff, 0x1c, 0xaa, 0xb6, 0x56, 0x9b, 0xab,
0xe8, 0x3e, 0xdc, 0x96, 0x6c, 0x38, 0x5a, 0xb4, 0x54, 0xbd, 0x89, 0xa5, 0x0e, 0xea, 0xed, 0xbf,
0x5e, 0x2f, 0xf6, 0x0d, 0xea, 0xc5, 0xff, 0xc9, 0x02, 0xe7, 0x94, 0xca, 0xe4, 0xad, 0x64, 0xfc,
0x29, 0x34, 0x30, 0xae, 0xf9, 0x6a, 0x75, 0xdd, 0xbe, 0x49, 0x60, 0x4b, 0x71, 0xff, 0x17, 0x0b,
0x5c, 0xec, 0x19, 0x74, 0xe3, 0x09, 0xba, 0x6f, 0xa1, 0xfb, 0xf7, 0x97, 0x98, 0x98, 0x4a, 0x9a,
0xd5, 0x71, 0x86, 0x95, 0xff, 0x10, 0xea, 0xd1, 0x88, 0xf1, 0xb8, 0x8c, 0xd9, 0xff, 0x96, 0x28,
0x6a, 0x9d, 0xc0, 0x48, 0xf9, 0x6b, 0xb0, 0x52, 0x6a, 0x7b, 0x4d, 0x58, 0xe9, 0xa7, 0x63, 0xc2,
0x59, 0xdc, 0xad, 0x78, 0x2b, 0x60, 0x1f, 0x09, 0xd5, 0xb5, 0xfc, 0xbf, 0x2d, 0x00, 0xd3, 0x12,
0xe8, 0xd4, 0xf6, 0x9c, 0x53, 0x1f, 0x2c, 0xb1, 0x3d, 0x13, 0x2d, 0x97, 0xa5, 0x5b, 0x1f, 0x43,
0x4d, 0x27, 0xfa, 0x4d, 0x5e, 0xa1, 0x90, 0xe6, 0x80, 0xb9, 0x2c, 0xbb, 0xf7, 0x6a, 0x0e, 0x28,
0xe5, 0x6f, 0x83, 0x73, 0x79, 0xd7, 0x22, 0x89, 0x36, 0xc0, 0x0b, 0x31, 0x64, 0x11, 0xe1, 0xbb,
0x69, 0xdc, 0xb5, 0xbc, 0x5b, 0xe0, 0x96, 0xfb, 0x63, 0xd9, 0xad, 0xfa, 0xbf, 0xdb, 0x50, 0x43,
0x52, 0xcf, 0xc0, 0x55, 0x54, 0x26, 0x21, 0x7d, 0x95, 0xc9, 0x32, 0xdd, 0x77, 0x97, 0xdc, 0x79,
0x59, 0x20, 0xfa, 0x91, 0x50, 0x97, 0xc5, 0xf2, 0x05, 0x40, 0xa1, 0xef, 0x36, 0xca, 0x86, 0xde,
0x3b, 0xd7, 0x65, 0xab, 0x57, 0x09, 0xdc, 0x62, 0x1a, 0xcf, 0x1d, 0x68, 0x0e, 0xd8, 0x4c, 0xdf,
0xbe, 0xb2, 0xd6, 0x66, 0x81, 0xed, 0x55, 0x02, 0x18, 0xcc, 0x32, 0xb2, 0x0f, 0xad, 0xc8, 0x34,
0xa2, 0x31, 0x61, 0xc6, 0xc1, 0xbb, 0x4b, 0xcb, 0x75, 0xda, 0xaf, 0xbd, 0x4a, 0xd0, 0x8c, 0xe6,
0xda, 0xf7, 0x1b, 0xe8, 0x1a, 0x16, 0x52, 0xcf, 0x3d, 0x63, 0xc8, 0x4c, 0x85, 0xf7, 0xae, 0xe2,
0x32, 0x9d, 0x90, 0xbd, 0x4a, 0xd0, 0x2e, 0x16, 0x67, 0xe6, 0x09, 0xdc, 0x2e, 0x59, 0xcd, 0xd9,
0x6b, 0xa0, 0x3d, 0xff, 0x4a, 0x6e, 0xf3, 0x06, 0x3b, 0x83, 0x45, 0x68, 0xaf, 0x01, 0x35, 0x6d,
0xc4, 0xff, 0xc7, 0x02, 0x38, 0xa3, 0x91, 0x12, 0x72, 0xf7, 0xe8, 0xe8, 0xbb, 0xf2, 0x09, 0x32,
0xc2, 0xe6, 0xf9, 0xd7, 0x4f, 0x90, 0xb1, 0xb7, 0xf0, 0x38, 0x56, 0x17, 0x1f, 0xc7, 0xa7, 0x00,
0x99, 0xa4, 0x31, 0x8b, 0x88, 0xa2, 0xf9, 0x9b, 0xca, 0x6c, 0x4e, 0xd4, 0xfb, 0x1c, 0xe0, 0x42,
0xff, 0x05, 0xcc, 0x68, 0xa8, 0x5d, 0x99, 0xee, 0xe9, 0x87, 0x21, 0x70, 0x2f, 0xa6, 0x7f, 0x87,
0x07, 0xd0, 0xc9, 0x38, 0x89, 0xe8, 0x48, 0xf0, 0x98, 0xca, 0x50, 0x91, 0x21, 0x06, 0xd9, 0x0d,
0xda, 0x73, 0xf0, 0x29, 0x19, 0xfa, 0x7f, 0x5a, 0xe0, 0x9c, 0x70, 0x92, 0x1e, 0x89, 0x18, 0x87,
0xf5, 0x18, 0x19, 0x87, 0x24, 0x4d, 0xf3, 0x6b, 0xc6, 0xd1, 0x2c, 0x2e, 0xba, 0x44, 0x8c, 0xce,
0x6e, 0x9a, 0xe6, 0xde, 0x67, 0x0b, 0x6c, 0xaf, 0x6f, 0x41, 0xad, 0x3a, 0xc7, 0x77, 0x03, 0xba,
0xa2, 0x50, 0x59, 0xa1, 0xc2, 0xcb, 0x50, 0xea, 0x70, 0xd9, 0x1b, 0x76, 0xd0, 0x36, 0xf8, 0x57,
0x26, 0xa2, 0xb9, 0xce, 0x50, 0x2a, 0x62, 0xfa, 0x51, 0x0a, 0x0d, 0x33, 0x58, 0x17, 0x7b, 0xb1,
0x03, 0xcd, 0x43, 0x49, 0x89, 0xa2, 0xf2, 0x74, 0x44, 0xd2, 0xae, 0xe5, 0x75, 0xa1, 0x55, 0x02,
0xcf, 0x2f, 0x0a, 0xc2, 0xbb, 0x55, 0xaf, 0x05, 0xce, 0x0b, 0x9a, 0xe7, 0x78, 0x6e, 0x63, 0xb3,
0xd2, 0x3c, 0x37, 0x87, 0x35, 0xcf, 0x85, 0xba, 0x59, 0xd6, 0xb5, 0xdc, 0x91, 0x50, 0x66, 0xd7,
0xd8, 0x7b, 0xfc, 0xc3, 0x27, 0x43, 0xa6, 0x46, 0xc5, 0x60, 0x33, 0x12, 0xc9, 0x96, 0x21, 0xf5,
0x90, 0x89, 0x72, 0xb5, 0xc5, 0x52, 0x45, 0x65, 0x4a, 0xf8, 0x16, 0xf2, 0xdc, 0xd2, 0x3c, 0xb3,
0xc1, 0xa0, 0x81, 0xbb, 0xc7, 0xff, 0x06, 0x00, 0x00, 0xff, 0xff, 0x21, 0x9d, 0x5f, 0x11, 0x7b,
0x0a, 0x00, 0x00,
}

View File

@ -484,6 +484,8 @@ func TestProxy(t *testing.T) {
nlist := 10
nprobe := 10
topk := 10
// add a test parameter
roundDecimal := 6
nq := 10
expr := fmt.Sprintf("%s > 0", int64Field)
var segmentIDs []int64
@ -641,6 +643,10 @@ func TestProxy(t *testing.T) {
Key: TopKKey,
Value: strconv.Itoa(topk),
},
{
Key: RoundDecimalKey,
Value: strconv.Itoa(roundDecimal),
},
},
TravelTimestamp: 0,
GuaranteeTimestamp: 0,

View File

@ -61,6 +61,7 @@ const (
TopKKey = "topk"
MetricTypeKey = "metric_type"
SearchParamsKey = "params"
RoundDecimalKey = "round_decimal"
HasCollectionTaskName = "HasCollectionTask"
DescribeCollectionTaskName = "DescribeCollectionTask"
GetCollectionStatisticsTaskName = "GetCollectionStatisticsTask"
@ -1528,11 +1529,20 @@ func (st *searchTask) PreExecute(ctx context.Context) error {
if err != nil {
return errors.New(SearchParamsKey + " not found in search_params")
}
roundDecimalStr, err := funcutil.GetAttrByKeyFromRepeatedKV(RoundDecimalKey, st.query.SearchParams)
if err != nil {
return errors.New(RoundDecimalKey + "not found in search_params")
}
roundDeciaml, err := strconv.Atoi(roundDecimalStr)
if err != nil {
return errors.New(RoundDecimalKey + " " + roundDecimalStr + " is not invalid")
}
queryInfo := &planpb.QueryInfo{
Topk: int64(topK),
MetricType: metricType,
SearchParams: searchParams,
RoundDecimal: int64(roundDeciaml),
}
log.Debug("create query plan",

View File

@ -211,7 +211,7 @@ func constructSearchRequest(
dbName, collectionName string,
expr string,
floatVecField string,
nq, dim, nprobe, topk int,
nq, dim, nprobe, topk, roundDecimal int,
) *milvuspb.SearchRequest {
params := make(map[string]string)
params["nprobe"] = strconv.Itoa(nprobe)
@ -251,6 +251,10 @@ func constructSearchRequest(
Key: TopKKey,
Value: strconv.Itoa(topk),
},
{
Key: RoundDecimalKey,
Value: strconv.Itoa(roundDecimal),
},
},
TravelTimestamp: 0,
GuaranteeTimestamp: 0,
@ -1667,6 +1671,7 @@ func TestSearchTask_all(t *testing.T) {
expr := fmt.Sprintf("%s > 0", int64Field)
nq := 10
topk := 10
roundDecimal := 3
nprobe := 10
schema := constructCollectionSchemaWithAllType(
@ -1725,7 +1730,7 @@ func TestSearchTask_all(t *testing.T) {
req := constructSearchRequest(dbName, collectionName,
expr,
floatVecField,
nq, dim, nprobe, topk)
nq, dim, nprobe, topk, roundDecimal)
task := &searchTask{
Condition: NewTaskCondition(ctx),
@ -2006,6 +2011,7 @@ func TestSearchTask_7803_reduce(t *testing.T) {
expr := fmt.Sprintf("%s > 0", int64Field)
nq := 10
topk := 10
roundDecimal := 3
nprobe := 10
schema := constructCollectionSchema(
@ -2066,7 +2072,7 @@ func TestSearchTask_7803_reduce(t *testing.T) {
req := constructSearchRequest(dbName, collectionName,
expr,
floatVecField,
nq, dim, nprobe, topk)
nq, dim, nprobe, topk, roundDecimal)
task := &searchTask{
Condition: NewTaskCondition(ctx),
@ -2523,6 +2529,52 @@ func TestSearchTask_PreExecute(t *testing.T) {
},
}
// invalid round_decimal
assert.Error(t, task.PreExecute(ctx))
task.query.SearchParams = []*commonpb.KeyValuePair{
{
Key: AnnsFieldKey,
Value: int64Field,
},
{
Key: TopKKey,
Value: "10",
},
{
Key: MetricTypeKey,
Value: distance.L2,
},
{
Key: SearchParamsKey,
Value: `{"nprobe": 10}`,
},
{
Key: RoundDecimalKey,
Value: "invalid",
},
}
// invalid round_decimal
assert.Error(t, task.PreExecute(ctx))
task.query.SearchParams = []*commonpb.KeyValuePair{
{
Key: AnnsFieldKey,
Value: floatVecField,
},
{
Key: TopKKey,
Value: "10",
},
{
Key: MetricTypeKey,
Value: distance.L2,
},
{
Key: RoundDecimalKey,
Value: "-1",
},
}
// failed to create query plan
assert.Error(t, task.PreExecute(ctx))
task.query.SearchParams = []*commonpb.KeyValuePair{
@ -2542,6 +2594,10 @@ func TestSearchTask_PreExecute(t *testing.T) {
Key: SearchParamsKey,
Value: `{"nprobe": 10}`,
},
{
Key: RoundDecimalKey,
Value: "-1",
},
}
// field not exist

View File

@ -50,6 +50,7 @@ const (
defaultVecFieldName = "vec"
defaultConstFieldName = "const"
defaultTopK = int64(10)
defaultRoundDecimal = int64(6)
defaultDim = 128
defaultNProb = 10
defaultMetricType = "JACCARD"
@ -892,11 +893,12 @@ func genSimpleStreaming(ctx context.Context) (*streaming, error) {
// ---------- unittest util functions ----------
// functions of messages and requests
func genDSL(schema *schemapb.CollectionSchema, nProb int, topK int64) (string, error) {
func genDSL(schema *schemapb.CollectionSchema, nProb int, topK int64, roundDecimal int64) (string, error) {
var vecFieldName string
var metricType string
nProbStr := strconv.Itoa(nProb)
topKStr := strconv.FormatInt(topK, 10)
roundDecimalStr := strconv.FormatInt(roundDecimal, 10)
for _, f := range schema.Fields {
if f.DataType == schemapb.DataType_FloatVector {
vecFieldName = f.Name
@ -912,18 +914,16 @@ func genDSL(schema *schemapb.CollectionSchema, nProb int, topK int64) (string, e
return "", err
}
return "{\"bool\": { " +
"\"vector\": {" +
"\"" + vecFieldName + "\": {" +
" \"metric_type\": \"" + metricType + "\", " +
" \"params\": {" +
" \"nprobe\": " + nProbStr + " " +
"}, \"query\": \"$0\",\"topk\": " + topKStr + " \n } \n } \n } \n }", nil
return "{\"bool\": { \n\"vector\": {\n \"" + vecFieldName +
"\": {\n \"metric_type\": \"" + metricType +
"\", \n \"params\": {\n \"nprobe\": " + nProbStr + " \n},\n \"query\": \"$0\",\n \"topk\": " + topKStr +
" \n,\"round_decimal\": " + roundDecimalStr +
"\n } \n } \n } \n }", nil
}
func genSimpleDSL() (string, error) {
schema := genSimpleSegCoreSchema()
return genDSL(schema, defaultNProb, defaultTopK)
return genDSL(schema, defaultNProb, defaultTopK, defaultRoundDecimal)
}
func genSimplePlaceHolderGroup() ([]byte, error) {

View File

@ -30,7 +30,7 @@ func TestPlan_Plan(t *testing.T) {
collection := newCollection(collectionMeta.ID, collectionMeta.Schema)
dslString := "{\"bool\": { \n\"vector\": {\n \"vec\": {\n \"metric_type\": \"L2\", \n \"params\": {\n \"nprobe\": 10 \n},\n \"query\": \"$0\",\"topk\": 10 \n } \n } \n } \n }"
dslString := "{\"bool\": { \n\"vector\": {\n \"vec\": {\n \"metric_type\": \"L2\", \n \"params\": {\n \"nprobe\": 10 \n},\n \"query\": \"$0\",\n \"topk\": 10 \n,\"round_decimal\": 6\n } \n } \n } \n }"
plan, err := createSearchPlan(collection, dslString)
assert.NoError(t, err)
@ -81,8 +81,7 @@ func TestPlan_PlaceholderGroup(t *testing.T) {
collection := newCollection(collectionMeta.ID, collectionMeta.Schema)
dslString := "{\"bool\": { \n\"vector\": {\n \"vec\": {\n \"metric_type\": \"L2\", \n \"params\": {\n \"nprobe\": 10 \n},\n \"query\": \"$0\",\"topk\": 10 \n } \n } \n } \n }"
dslString := "{\"bool\": { \n\"vector\": {\n \"vec\": {\n \"metric_type\": \"L2\", \n \"params\": {\n \"nprobe\": 10 \n},\n \"query\": \"$0\",\n \"topk\": 10 \n,\"round_decimal\": 6\n } \n } \n } \n }"
plan, err := createSearchPlan(collection, dslString)
assert.NoError(t, err)
assert.NotNil(t, plan)

View File

@ -128,7 +128,7 @@ func TestQueryCollection_withoutVChannel(t *testing.T) {
for i := 0; i < dim; i++ {
vec[i] = rand.Float32()
}
dslString := "{\"bool\": { \n\"vector\": {\n \"vec\": {\n \"metric_type\": \"L2\", \n \"params\": {\n \"nprobe\": 10 \n},\n \"query\": \"$0\",\"topk\": 10 \n } \n } \n } \n }"
dslString := "{\"bool\": { \n\"vector\": {\n \"vec\": {\n \"metric_type\": \"L2\", \n \"params\": {\n \"nprobe\": 10 \n},\n \"query\": \"$0\",\n \"topk\": 10 \n,\"round_decimal\": 6\n } \n } \n } \n }"
var searchRawData1 []byte
var searchRawData2 []byte
for i, ele := range vec {

View File

@ -35,7 +35,7 @@ func TestReduce_AllFunc(t *testing.T) {
var vec = [DIM]float32{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16}
// start search service
dslString := "{\"bool\": { \n\"vector\": {\n \"vec\": {\n \"metric_type\": \"L2\", \n \"params\": {\n \"nprobe\": 10 \n},\n \"query\": \"$0\",\"topk\": 10 \n } \n } \n } \n }"
dslString := "{\"bool\": { \n\"vector\": {\n \"vec\": {\n \"metric_type\": \"L2\", \n \"params\": {\n \"nprobe\": 10 \n},\n \"query\": \"$0\",\n \"topk\": 10 \n,\"round_decimal\": 6\n } \n } \n } \n }"
var searchRawData1 []byte
var searchRawData2 []byte
for i, ele := range vec {

View File

@ -295,7 +295,6 @@ func (s *Segment) search(plan *SearchPlan,
log.Debug("do search on segment", zap.Int64("segmentID", s.segmentID), zap.Int32("segmentType", int32(s.segmentType)))
var status = C.Search(s.segmentPtr, plan.cSearchPlan, cPlaceHolderGroup, ts, &searchResult.cSearchResult)
errorCode := status.error_code
if errorCode != 0 {
errorMsg := C.GoString(status.error_msg)
defer C.free(unsafe.Pointer(status.error_msg))

View File

@ -491,9 +491,7 @@ func TestSegment_segmentSearch(t *testing.T) {
err = segment.segmentInsert(offset, &ids, &timestamps, &records)
assert.NoError(t, err)
dslString := "{\"bool\": { \n\"vector\": {\n \"vec\": {\n \"metric_type\": \"L2\", \n \"params\": {\n \"nprobe\": 10 \n},\n \"query\": \"$0\",\"topk\": 10 \n } \n } \n } \n }"
dslString := "{\"bool\": { \n\"vector\": {\n \"vec\": {\n \"metric_type\": \"L2\", \n \"params\": {\n \"nprobe\": 10 \n},\n \"query\": \"$0\",\n \"topk\": 10 \n,\"round_decimal\": 6\n } \n } \n } \n }"
var searchRawData []byte
for _, ele := range vec {
buf := make([]byte, 4)