milvus-io
/
milvus
mirror of https://github.com/milvus-io/milvus.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

feat: Support knowhere trace using OpenTelemetry (#30750) 

Issue: #21508

Signed-off-by: Yudong Cai <yudong.cai@zilliz.com>
pull/30887/head
Cai Yudong 2024-02-28 12:29:00 +08:00 committed by GitHub
parent a4f3e01a3a
commit 8a219e0102
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
33 changed files with 464 additions and 617 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
internal/core/src/common/QueryInfo.h
View File

@ -18,8 +18,10 @@
#include <memory>
#include "common/Tracer.h"
#include "common/Types.h"
#include "knowhere/config.h"
namespace milvus {
struct SearchInfo {
@ -29,6 +31,7 @@ struct SearchInfo {
MetricType metric_type_;
knowhere::Json search_params_;
std::optional<FieldId> group_by_field_id_;
tracer::TraceContext trace_ctx_;
};
using SearchInfoPtr = std::shared_ptr<SearchInfo>;

76
internal/core/src/common/Tracer.cpp
View File

@ -8,20 +8,22 @@
// Unless required by applicable law or agreed to in writing, software distributed under the License
// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
// or implied. See the License for the specific language governing permissions and limitations under the License
#include "log/Log.h"
#include "Tracer.h"
#include "log/Log.h"
#include <utility>
#include "opentelemetry/exporters/ostream/span_exporter_factory.h"
#include "opentelemetry/exporters/jaeger/jaeger_exporter_factory.h"
#include "opentelemetry/exporters/ostream/span_exporter_factory.h"
#include "opentelemetry/exporters/otlp/otlp_grpc_exporter_factory.h"
#include "opentelemetry/sdk/trace/samplers/always_on.h"
#include "opentelemetry/sdk/trace/batch_span_processor_factory.h"
#include "opentelemetry/sdk/trace/tracer_provider_factory.h"
#include "opentelemetry/sdk/trace/sampler.h"
#include "opentelemetry/sdk/trace/samplers/parent.h"
#include "opentelemetry/sdk/resource/resource.h"
#include "opentelemetry/sdk/trace/batch_span_processor_factory.h"
#include "opentelemetry/sdk/trace/sampler.h"
#include "opentelemetry/sdk/trace/samplers/always_on.h"
#include "opentelemetry/sdk/trace/samplers/parent.h"
#include "opentelemetry/sdk/trace/tracer_provider_factory.h"
#include "opentelemetry/sdk/version/version.h"
#include "opentelemetry/trace/span_context.h"
#include "opentelemetry/trace/span_metadata.h"
@ -41,20 +43,20 @@ static std::shared_ptr<trace::TracerProvider> noop_trace_provider =
std::make_shared<opentelemetry::trace::NoopTracerProvider>();
void
initTelementry(TraceConfig* config) {
initTelemetry(const TraceConfig& cfg) {
std::unique_ptr<opentelemetry::sdk::trace::SpanExporter> exporter;
if (config->exporter == "stdout") {
if (cfg.exporter == "stdout") {
exporter = ostream::OStreamSpanExporterFactory::Create();
} else if (config->exporter == "jaeger") {
} else if (cfg.exporter == "jaeger") {
auto opts = jaeger::JaegerExporterOptions{};
opts.transport_format = jaeger::TransportFormat::kThriftHttp;
opts.endpoint = config->jaegerURL;
opts.endpoint = cfg.jaegerURL;
exporter = jaeger::JaegerExporterFactory::Create(opts);
LOG_INFO("init jaeger exporter, endpoint:", opts.endpoint);
} else if (config->exporter == "otlp") {
} else if (cfg.exporter == "otlp") {
auto opts = otlp::OtlpGrpcExporterOptions{};
opts.endpoint = config->otlpEndpoint;
opts.use_ssl_credentials = config->oltpSecure;
opts.endpoint = cfg.otlpEndpoint;
opts.use_ssl_credentials = cfg.oltpSecure;
exporter = otlp::OtlpGrpcExporterFactory::Create(opts);
LOG_INFO("init otlp exporter, endpoint:", opts.endpoint);
} else {
@ -64,8 +66,8 @@ initTelementry(TraceConfig* config) {
if (enable_trace) {
auto processor = trace_sdk::BatchSpanProcessorFactory::Create(
std::move(exporter), {});
resource::ResourceAttributes attributes = {{"service.name", "segcore"},
{"NodeID", config->nodeID}};
resource::ResourceAttributes attributes = {
{"service.name", TRACE_SERVICE_SEGCORE}, {"NodeID", cfg.nodeID}};
auto resource = resource::Resource::Create(attributes);
auto sampler = std::make_unique<trace_sdk::ParentBasedSampler>(
std::make_shared<trace_sdk::AlwaysOnSampler>());
@ -81,7 +83,8 @@ initTelementry(TraceConfig* config) {
std::shared_ptr<trace::Tracer>
GetTracer() {
auto provider = trace::Provider::GetTracerProvider();
return provider->GetTracer("segcore", OPENTELEMETRY_SDK_VERSION);
return provider->GetTracer(TRACE_SERVICE_SEGCORE,
OPENTELEMETRY_SDK_VERSION);
}
std::shared_ptr<trace::Span>
@ -89,14 +92,13 @@ StartSpan(const std::string& name, TraceContext* parentCtx) {
trace::StartSpanOptions opts;
if (enable_trace && parentCtx != nullptr && parentCtx->traceID != nullptr &&
parentCtx->spanID != nullptr) {
if (isEmptyID(parentCtx->traceID, trace::TraceId::kSize) ||
isEmptyID(parentCtx->spanID, trace::SpanId::kSize)) {
if (EmptyTraceID(parentCtx) || EmptySpanID(parentCtx)) {
return noop_trace_provider->GetTracer("noop")->StartSpan("noop");
}
opts.parent = trace::SpanContext(
trace::TraceId({parentCtx->traceID, trace::TraceId::kSize}),
trace::SpanId({parentCtx->spanID, trace::SpanId::kSize}),
trace::TraceFlags(parentCtx->flag),
trace::TraceFlags(parentCtx->traceFlags),
true);
}
return GetTracer()->StartSpan(name, opts);
@ -118,7 +120,7 @@ CloseRootSpan() {
}
void
AddEvent(std::string event_label) {
AddEvent(const std::string& event_label) {
if (enable_trace && local_span != nullptr) {
local_span->AddEvent(event_label);
}
@ -126,12 +128,44 @@ AddEvent(std::string event_label) {
bool
isEmptyID(const uint8_t* id, int length) {
if (id != nullptr) {
for (int i = 0; i < length; i++) {
if (id[i] != 0) {
return false;
}
}
}
return true;
}
bool
EmptyTraceID(const TraceContext* ctx) {
return isEmptyID(ctx->traceID, trace::TraceId::kSize);
}
bool
EmptySpanID(const TraceContext* ctx) {
return isEmptyID(ctx->spanID, trace::SpanId::kSize);
}
std::string
GetTraceIDAsStr(const TraceContext* ctx) {
if (ctx != nullptr && !EmptyTraceID(ctx)) {
return std::string((char*)ctx->traceID,
opentelemetry::trace::TraceId::kSize);
} else {
return std::string();
}
}
std::string
GetSpanIDAsStr(const TraceContext* ctx) {
if (ctx != nullptr && !EmptySpanID(ctx)) {
return std::string((char*)ctx->spanID,
opentelemetry::trace::SpanId::kSize);
} else {
return std::string();
}
}
} // namespace milvus::tracer

24
internal/core/src/common/Tracer.h
View File

@ -14,9 +14,10 @@
#include <memory>
#include <string>
#include "opentelemetry/sdk/version/version.h"
#include "opentelemetry/trace/provider.h"
#define TRACE_SERVICE_SEGCORE "segcore"
namespace milvus::tracer {
struct TraceConfig {
@ -30,14 +31,14 @@ struct TraceConfig {
};
struct TraceContext {
const uint8_t* traceID;
const uint8_t* spanID;
uint8_t flag;
const uint8_t* traceID = nullptr;
const uint8_t* spanID = nullptr;
uint8_t traceFlags = 0;
};
namespace trace = opentelemetry::trace;
void
initTelementry(TraceConfig* config);
initTelemetry(const TraceConfig& cfg);
std::shared_ptr<trace::Tracer>
GetTracer();
@ -52,9 +53,18 @@ void
CloseRootSpan();
void
AddEvent(std::string event_label);
AddEvent(const std::string& event_label);
bool
isEmptyID(const uint8_t* id, const int length);
EmptyTraceID(const TraceContext* ctx);
bool
EmptySpanID(const TraceContext* ctx);
std::string
GetTraceIDAsStr(const TraceContext* ctx);
std::string
GetSpanIDAsStr(const TraceContext* ctx);
} // namespace milvus::tracer

6
internal/core/src/common/init_c.cpp
View File

@ -88,8 +88,8 @@ InitTrace(CTraceConfig* config) {
config->nodeID};
std::call_once(
traceFlag,
[](milvus::tracer::TraceConfig* c) {
milvus::tracer::initTelementry(c);
[](const milvus::tracer::TraceConfig& c) {
milvus::tracer::initTelemetry(c);
},
&traceConfig);
traceConfig);
}

2
internal/core/src/common/type_c.h
View File

@ -104,7 +104,7 @@ typedef struct CTraceConfig {
typedef struct CTraceContext {
const uint8_t* traceID;
const uint8_t* spanID;
uint8_t flag;
uint8_t traceFlags;
} CTraceContext;
typedef struct CNewSegmentResult {

6
internal/core/src/expr/ITypeExpr.h
View File

@ -16,14 +16,14 @@
#pragma once
#include <fmt/core.h>
#include <memory>
#include <string>
#include <vector>
#include <fmt/core.h>
#include "common/Schema.h"
#include "common/Types.h"
#include "common/Utils.h"
#include "pb/plan.pb.h"
namespace milvus {
@ -73,7 +73,7 @@ struct ColumnInfo {
return fmt::format("[FieldId:{}, data_type:{}, nested_path:{}]",
std::to_string(field_id_.get()),
data_type_,
milvus::Join(nested_path_, ","));
milvus::Join<std::string>(nested_path_, ","));
}
};

5
internal/core/src/index/VectorDiskIndex.cpp
View File

@ -324,10 +324,7 @@ VectorDiskAnnIndex<T>::Query(const DatasetPtr dataset,
auto num_queries = dataset->GetRows();
auto topk = search_info.topk_;
knowhere::Json search_config = search_info.search_params_;
search_config[knowhere::meta::TOPK] = topk;
search_config[knowhere::meta::METRIC_TYPE] = GetMetricType();
knowhere::Json search_config = PrepareSearchParams(search_info);
if (GetIndexType() == knowhere::IndexEnum::INDEX_DISKANN) {
// set search list size

21
internal/core/src/index/VectorIndex.h
View File

@ -112,6 +112,27 @@ class VectorIndex : public IndexBase {
err_msg);
}
knowhere::Json
PrepareSearchParams(const SearchInfo& search_info) const {
knowhere::Json search_cfg = search_info.search_params_;
search_cfg[knowhere::meta::METRIC_TYPE] = search_info.metric_type_;
search_cfg[knowhere::meta::TOPK] = search_info.topk_;
// save trace context into search conf
if (search_info.trace_ctx_.traceID != nullptr &&
search_info.trace_ctx_.spanID != nullptr) {
search_cfg[knowhere::meta::TRACE_ID] =
tracer::GetTraceIDAsStr(&search_info.trace_ctx_);
search_cfg[knowhere::meta::SPAN_ID] =
tracer::GetSpanIDAsStr(&search_info.trace_ctx_);
search_cfg[knowhere::meta::TRACE_FLAGS] =
search_info.trace_ctx_.traceFlags;
}
return search_cfg;
}
private:
MetricType metric_type_;
int64_t dim_;

5
internal/core/src/index/VectorMemIndex.cpp
View File

@ -26,6 +26,7 @@
#include <unordered_map>
#include <unordered_set>
#include "common/Tracer.h"
#include "common/Types.h"
#include "common/type_c.h"
#include "fmt/format.h"
@ -540,12 +541,10 @@ VectorMemIndex<T>::Query(const DatasetPtr dataset,
// "Metric type of field index isn't the same with search info");
auto num_queries = dataset->GetRows();
knowhere::Json search_conf = search_info.search_params_;
knowhere::Json search_conf = PrepareSearchParams(search_info);
auto topk = search_info.topk_;
// TODO :: check dim of search data
auto final = [&] {
search_conf[knowhere::meta::TOPK] = topk;
search_conf[knowhere::meta::METRIC_TYPE] = GetMetricType();
auto index_type = GetIndexType();
if (CheckKeyInConfig(search_conf, RADIUS)) {
if (CheckKeyInConfig(search_conf, RANGE_FILTER)) {

74
internal/core/src/query/SearchBruteForce.cpp
View File

@ -12,17 +12,18 @@
#include <string>
#include <vector>
#include "SearchBruteForce.h"
#include "SubSearchResult.h"
#include "common/Consts.h"
#include "common/EasyAssert.h"
#include "common/RangeSearchHelper.h"
#include "common/Utils.h"
#include "common/Tracer.h"
#include "SearchBruteForce.h"
#include "SubSearchResult.h"
#include "knowhere/comp/brute_force.h"
#include "knowhere/comp/index_param.h"
#include "knowhere/index_node.h"
#include "log/Log.h"
namespace milvus::query {
void
@ -41,11 +42,32 @@ CheckBruteForceSearchParam(const FieldMeta& field,
"[BruteForceSearch] Data type and metric type miss-match");
}
knowhere::Json
PrepareBFSearchParams(const SearchInfo& search_info) {
knowhere::Json search_cfg = search_info.search_params_;
search_cfg[knowhere::meta::METRIC_TYPE] = search_info.metric_type_;
search_cfg[knowhere::meta::TOPK] = search_info.topk_;
// save trace context into search conf
if (search_info.trace_ctx_.traceID != nullptr &&
search_info.trace_ctx_.spanID != nullptr) {
search_cfg[knowhere::meta::TRACE_ID] =
tracer::GetTraceIDAsStr(&search_info.trace_ctx_);
search_cfg[knowhere::meta::SPAN_ID] =
tracer::GetSpanIDAsStr(&search_info.trace_ctx_);
search_cfg[knowhere::meta::TRACE_FLAGS] =
search_info.trace_ctx_.traceFlags;
}
return search_cfg;
}
SubSearchResult
BruteForceSearch(const dataset::SearchDataset& dataset,
const void* chunk_data_raw,
int64_t chunk_rows,
const knowhere::Json& conf,
const SearchInfo& search_info,
const BitsetView& bitset,
DataType data_type) {
SubSearchResult sub_result(dataset.num_queries,
@ -58,40 +80,35 @@ BruteForceSearch(const dataset::SearchDataset& dataset,
auto base_dataset = knowhere::GenDataSet(chunk_rows, dim, chunk_data_raw);
auto query_dataset = knowhere::GenDataSet(nq, dim, dataset.query_data);
auto config = knowhere::Json{
{knowhere::meta::METRIC_TYPE, dataset.metric_type},
{knowhere::meta::DIM, dim},
{knowhere::meta::TOPK, topk},
};
auto search_cfg = PrepareBFSearchParams(search_info);
sub_result.mutable_seg_offsets().resize(nq * topk);
sub_result.mutable_distances().resize(nq * topk);
if (conf.contains(RADIUS)) {
config[RADIUS] = conf[RADIUS].get<float>();
if (conf.contains(RANGE_FILTER)) {
config[RANGE_FILTER] = conf[RANGE_FILTER].get<float>();
CheckRangeSearchParam(
config[RADIUS], config[RANGE_FILTER], dataset.metric_type);
if (search_cfg.contains(RADIUS)) {
if (search_cfg.contains(RANGE_FILTER)) {
CheckRangeSearchParam(search_cfg[RADIUS],
search_cfg[RANGE_FILTER],
search_info.metric_type_);
}
knowhere::expected<knowhere::DataSetPtr> res;
if (data_type == DataType::VECTOR_FLOAT) {
res = knowhere::BruteForce::RangeSearch<float>(
base_dataset, query_dataset, config, bitset);
base_dataset, query_dataset, search_cfg, bitset);
} else if (data_type == DataType::VECTOR_FLOAT16) {
res = knowhere::BruteForce::RangeSearch<float16>(
base_dataset, query_dataset, config, bitset);
base_dataset, query_dataset, search_cfg, bitset);
} else if (data_type == DataType::VECTOR_BFLOAT16) {
res = knowhere::BruteForce::RangeSearch<bfloat16>(
base_dataset, query_dataset, config, bitset);
base_dataset, query_dataset, search_cfg, bitset);
} else if (data_type == DataType::VECTOR_BINARY) {
res = knowhere::BruteForce::RangeSearch<uint8_t>(
base_dataset, query_dataset, config, bitset);
base_dataset, query_dataset, search_cfg, bitset);
}
milvus::tracer::AddEvent("knowhere_finish_BruteForce_RangeSearch");
if (!res.has_value()) {
PanicInfo(KnowhereError,
"failed to range search: {}: {}",
"Brute force range search fail: {}, {}",
KnowhereStatusString(res.error()),
res.what());
}
@ -110,7 +127,7 @@ BruteForceSearch(const dataset::SearchDataset& dataset,
query_dataset,
sub_result.mutable_seg_offsets().data(),
sub_result.mutable_distances().data(),
config,
search_cfg,
bitset);
} else if (data_type == DataType::VECTOR_FLOAT16) {
stat = knowhere::BruteForce::SearchWithBuf<float16>(
@ -118,7 +135,7 @@ BruteForceSearch(const dataset::SearchDataset& dataset,
query_dataset,
sub_result.mutable_seg_offsets().data(),
sub_result.mutable_distances().data(),
config,
search_cfg,
bitset);
} else if (data_type == DataType::VECTOR_BFLOAT16) {
stat = knowhere::BruteForce::SearchWithBuf<bfloat16>(
@ -126,7 +143,7 @@ BruteForceSearch(const dataset::SearchDataset& dataset,
query_dataset,
sub_result.mutable_seg_offsets().data(),
sub_result.mutable_distances().data(),
config,
search_cfg,
bitset);
} else if (data_type == DataType::VECTOR_BINARY) {
stat = knowhere::BruteForce::SearchWithBuf<uint8_t>(
@ -134,14 +151,14 @@ BruteForceSearch(const dataset::SearchDataset& dataset,
query_dataset,
sub_result.mutable_seg_offsets().data(),
sub_result.mutable_distances().data(),
config,
search_cfg,
bitset);
}
milvus::tracer::AddEvent("knowhere_finish_BruteForce_SearchWithBuf");
if (stat != knowhere::Status::success) {
throw SegcoreError(
KnowhereError,
"invalid metric type, " + KnowhereStatusString(stat));
"Brute force search fail: " + KnowhereStatusString(stat));
}
}
sub_result.round_values();
@ -152,13 +169,14 @@ SubSearchResult
BruteForceSearchIterators(const dataset::SearchDataset& dataset,
const void* chunk_data_raw,
int64_t chunk_rows,
const knowhere::Json& conf,
const SearchInfo& search_info,
const BitsetView& bitset,
DataType data_type) {
auto nq = dataset.num_queries;
auto dim = dataset.dim;
auto base_dataset = knowhere::GenDataSet(chunk_rows, dim, chunk_data_raw);
auto query_dataset = knowhere::GenDataSet(nq, dim, dataset.query_data);
auto search_cfg = PrepareBFSearchParams(search_info);
knowhere::expected<
std::vector<std::shared_ptr<knowhere::IndexNode::iterator>>>
@ -166,15 +184,15 @@ BruteForceSearchIterators(const dataset::SearchDataset& dataset,
switch (data_type) {
case DataType::VECTOR_FLOAT:
iterators_val = knowhere::BruteForce::AnnIterator<float>(
base_dataset, query_dataset, conf, bitset);
base_dataset, query_dataset, search_cfg, bitset);
break;
case DataType::VECTOR_FLOAT16:
iterators_val = knowhere::BruteForce::AnnIterator<float16>(
base_dataset, query_dataset, conf, bitset);
base_dataset, query_dataset, search_cfg, bitset);
break;
case DataType::VECTOR_BFLOAT16:
iterators_val = knowhere::BruteForce::AnnIterator<bfloat16>(
base_dataset, query_dataset, conf, bitset);
base_dataset, query_dataset, search_cfg, bitset);
break;
default:
PanicInfo(ErrorCode::Unsupported,

4
internal/core/src/query/SearchBruteForce.h
View File

@ -27,7 +27,7 @@ SubSearchResult
BruteForceSearch(const dataset::SearchDataset& dataset,
const void* chunk_data_raw,
int64_t chunk_rows,
const knowhere::Json& conf,
const SearchInfo& search_info,
const BitsetView& bitset,
DataType data_type);
@ -35,7 +35,7 @@ SubSearchResult
BruteForceSearchIterators(const dataset::SearchDataset& dataset,
const void* chunk_data_raw,
int64_t chunk_rows,
const knowhere::Json& conf,
const SearchInfo& search_info,
const BitsetView& bitset,
DataType data_type);

4
internal/core/src/query/SearchOnGrowing.cpp
View File

@ -111,7 +111,7 @@ SearchOnGrowing(const segcore::SegmentGrowingImpl& segment,
auto sub_qr = BruteForceSearchIterators(search_dataset,
chunk_data,
size_per_chunk,
info.search_params_,
info,
sub_view,
data_type);
final_qr.merge(sub_qr);
@ -119,7 +119,7 @@ SearchOnGrowing(const segcore::SegmentGrowingImpl& segment,
auto sub_qr = BruteForceSearch(search_dataset,
chunk_data,
size_per_chunk,
info.search_params_,
info,
sub_view,
data_type);

17
internal/core/src/query/SearchOnSealed.cpp
View File

@ -90,22 +90,13 @@ SearchOnSealed(const Schema& schema,
auto data_type = field.get_data_type();
CheckBruteForceSearchParam(field, search_info);
if (search_info.group_by_field_id_.has_value()) {
auto sub_qr = BruteForceSearchIterators(dataset,
vec_data,
row_count,
search_info.search_params_,
bitset,
data_type);
auto sub_qr = BruteForceSearchIterators(
dataset, vec_data, row_count, search_info, bitset, data_type);
result.AssembleChunkVectorIterators(
num_queries, 1, -1, sub_qr.chunk_iterators());
} else {
auto sub_qr = BruteForceSearch(dataset,
vec_data,
row_count,
search_info.search_params_,
bitset,
data_type);
auto sub_qr = BruteForceSearch(
dataset, vec_data, row_count, search_info, bitset, data_type);
result.distances_ = std::move(sub_qr.mutable_distances());
result.seg_offsets_ = std::move(sub_qr.mutable_seg_offsets());
}

2
internal/core/src/segcore/load_index_c.cpp
View File

@ -226,7 +226,7 @@ AppendIndexV2(CTraceContext c_trace, CLoadIndexInfo c_load_index_info) {
index_info.index_engine_version = engine_version;
auto ctx = milvus::tracer::TraceContext{
c_trace.traceID, c_trace.spanID, c_trace.flag};
c_trace.traceID, c_trace.spanID, c_trace.traceFlags};
auto span = milvus::tracer::StartSpan("SegCoreLoadIndex", &ctx);
milvus::tracer::SetRootSpan(span);

6
internal/core/src/segcore/plan_c.h
View File

@ -9,6 +9,8 @@
// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
// or implied. See the License for the specific language governing permissions and limitations under the License
#pragma once
#ifdef __cplusplus
extern "C" {
#endif
@ -25,13 +27,13 @@ typedef void* CRetrievePlan;
// Note: serialized_expr_plan is of binary format
CStatus
CreateSearchPlanByExpr(CCollection col,
CreateSearchPlanByExpr(CCollection c_col,
const void* serialized_expr_plan,
const int64_t size,
CSearchPlan* res_plan);
CStatus
ParsePlaceholderGroup(CSearchPlan plan,
ParsePlaceholderGroup(CSearchPlan c_plan,
const void* placeholder_group_blob,
const int64_t blob_size,
CPlaceholderGroup* res_placeholder_group);

27
internal/core/src/segcore/segment_c.cpp
View File

@ -82,10 +82,10 @@ DeleteSearchResult(CSearchResult search_result) {
}
CStatus
Search(CSegmentInterface c_segment,
Search(CTraceContext c_trace,
CSegmentInterface c_segment,
CSearchPlan c_plan,
CPlaceholderGroup c_placeholder_group,
CTraceContext c_trace,
uint64_t timestamp,
CSearchResult* result) {
try {
@ -93,10 +93,16 @@ Search(CSegmentInterface c_segment,
auto plan = (milvus::query::Plan*)c_plan;
auto phg_ptr = reinterpret_cast<const milvus::query::PlaceholderGroup*>(
c_placeholder_group);
auto ctx = milvus::tracer::TraceContext{
c_trace.traceID, c_trace.spanID, c_trace.flag};
auto span = milvus::tracer::StartSpan("SegCoreSearch", &ctx);
// save trace context into search_info
auto& trace_ctx = plan->plan_node_->search_info_.trace_ctx_;
trace_ctx.traceID = c_trace.traceID;
trace_ctx.spanID = c_trace.spanID;
trace_ctx.traceFlags = c_trace.traceFlags;
auto span = milvus::tracer::StartSpan("SegCoreSearch", &trace_ctx);
milvus::tracer::SetRootSpan(span);
auto search_result = segment->Search(plan, phg_ptr, timestamp);
if (!milvus::PositivelyRelated(
plan->plan_node_->search_info_.metric_type_)) {
@ -119,9 +125,9 @@ DeleteRetrieveResult(CRetrieveResult* retrieve_result) {
}
CStatus
Retrieve(CSegmentInterface c_segment,
Retrieve(CTraceContext c_trace,
CSegmentInterface c_segment,
CRetrievePlan c_plan,
CTraceContext c_trace,
uint64_t timestamp,
CRetrieveResult* result,
int64_t limit_size) {
@ -130,9 +136,10 @@ Retrieve(CSegmentInterface c_segment,
static_cast<milvus::segcore::SegmentInterface*>(c_segment);
auto plan = static_cast<const milvus::query::RetrievePlan*>(c_plan);
auto ctx = milvus::tracer::TraceContext{
c_trace.traceID, c_trace.spanID, c_trace.flag};
auto span = milvus::tracer::StartSpan("SegCoreRetrieve", &ctx);
auto trace_ctx = milvus::tracer::TraceContext{
c_trace.traceID, c_trace.spanID, c_trace.traceFlags};
auto span = milvus::tracer::StartSpan("SegCoreRetrieve", &trace_ctx);
milvus::tracer::SetRootSpan(span);
auto retrieve_result = segment->Retrieve(plan, timestamp, limit_size);

8
internal/core/src/segcore/segment_c.h
View File

@ -44,10 +44,10 @@ void
DeleteSearchResult(CSearchResult search_result);
CStatus
Search(CSegmentInterface c_segment,
Search(CTraceContext c_trace,
CSegmentInterface c_segment,
CSearchPlan c_plan,
CPlaceholderGroup c_placeholder_group,
CTraceContext c_trace,
uint64_t timestamp,
CSearchResult* result);
@ -55,9 +55,9 @@ void
DeleteRetrieveResult(CRetrieveResult* retrieve_result);
CStatus
Retrieve(CSegmentInterface c_segment,
Retrieve(CTraceContext c_trace,
CSegmentInterface c_segment,
CRetrievePlan c_plan,
CTraceContext c_trace,
uint64_t timestamp,
CRetrieveResult* result,
int64_t limit_size);

36
internal/core/unittest/test_array_expr.cpp
View File

@ -17,7 +17,10 @@
#include <chrono>
#include "common/Types.h"
#include "expr/ITypeExpr.h"
#include "index/IndexFactory.h"
#include "pb/plan.pb.h"
#include "plan/PlanNode.h"
#include "query/Expr.h"
#include "query/ExprImpl.h"
#include "query/Plan.h"
@ -26,14 +29,12 @@
#include "segcore/SegmentGrowingImpl.h"
#include "simdjson/padded_string.h"
#include "test_utils/DataGen.h"
#include "index/IndexFactory.h"
#include "expr/ITypeExpr.h"
#include "plan/PlanNode.h"
TEST(Expr, TestArrayRange) {
using namespace milvus;
using namespace milvus::query;
using namespace milvus::segcore;
TEST(Expr, TestArrayRange) {
std::vector<std::tuple<std::string,
std::string,
std::function<bool(milvus::Array & array)>>>
@ -622,9 +623,6 @@ TEST(Expr, TestArrayRange) {
}
TEST(Expr, TestArrayEqual) {
using namespace milvus;
using namespace milvus::query;
using namespace milvus::segcore;
std::vector<
std::tuple<std::string, std::function<bool(std::vector<int64_t>)>>>
testcases = {
@ -746,10 +744,6 @@ TEST(Expr, TestArrayEqual) {
}
TEST(Expr, PraseArrayContainsExpr) {
using namespace milvus;
using namespace milvus::query;
using namespace milvus::segcore;
std::vector<const char*> raw_plans{
R"(vector_anns:<
field_id:100
@ -835,10 +829,6 @@ struct ArrayTestcase {
};
TEST(Expr, TestArrayContains) {
using namespace milvus;
using namespace milvus::query;
using namespace milvus::segcore;
auto schema = std::make_shared<Schema>();
auto i64_fid = schema->AddDebugField("id", DataType::INT64);
auto int_array_fid =
@ -1198,10 +1188,6 @@ TEST(Expr, TestArrayContains) {
}
TEST(Expr, TestArrayBinaryArith) {
using namespace milvus;
using namespace milvus::query;
using namespace milvus::segcore;
auto schema = std::make_shared<Schema>();
auto i64_fid = schema->AddDebugField("id", DataType::INT64);
auto int_array_fid =
@ -1765,10 +1751,6 @@ struct UnaryRangeTestcase {
};
TEST(Expr, TestArrayStringMatch) {
using namespace milvus;
using namespace milvus::query;
using namespace milvus::segcore;
auto schema = std::make_shared<Schema>();
auto i64_fid = schema->AddDebugField("id", DataType::INT64);
auto string_array_fid = schema->AddDebugField(
@ -1850,10 +1832,6 @@ TEST(Expr, TestArrayStringMatch) {
}
TEST(Expr, TestArrayInTerm) {
using namespace milvus;
using namespace milvus::query;
using namespace milvus::segcore;
auto schema = std::make_shared<Schema>();
auto i64_fid = schema->AddDebugField("id", DataType::INT64);
auto long_array_fid =
@ -2057,10 +2035,6 @@ TEST(Expr, TestArrayInTerm) {
}
TEST(Expr, TestTermInArray) {
using namespace milvus;
using namespace milvus::query;
using namespace milvus::segcore;
auto schema = std::make_shared<Schema>();
auto i64_fid = schema->AddDebugField("id", DataType::INT64);
auto long_array_fid =

5
internal/core/unittest/test_bf.cpp
View File

@ -130,10 +130,13 @@ class TestFloatSearchBruteForce : public ::testing::Test {
// ASSERT_ANY_THROW(BruteForceSearch(dataset, base.data(), nb, bitset_view));
return;
}
SearchInfo search_info;
search_info.topk_ = topk;
search_info.metric_type_ = metric_type;
auto result = BruteForceSearch(dataset,
base.data(),
nb,
knowhere::Json(),
search_info,
bitset_view,
DataType::VECTOR_FLOAT);
for (int i = 0; i < nq; i++) {

10
internal/core/unittest/test_binlog_index.cpp
View File

@ -13,18 +13,18 @@
#include <boost/format.hpp>
#include <regex>
#include "index/IndexFactory.h"
#include "knowhere/comp/brute_force.h"
#include "pb/plan.pb.h"
#include "pb/schema.pb.h"
#include "query/Plan.h"
#include "segcore/segcore_init_c.h"
#include "segcore/SegmentSealed.h"
#include "segcore/SegmentSealedImpl.h"
#include "pb/schema.pb.h"
#include "test_utils/DataGen.h"
#include "index/IndexFactory.h"
#include "query/Plan.h"
#include "knowhere/comp/brute_force.h"
using namespace milvus::segcore;
using namespace milvus;
using namespace milvus::segcore;
namespace pb = milvus::proto;
std::shared_ptr<float[]>

181
internal/core/unittest/test_c_api.cpp
View File

@ -46,8 +46,9 @@
namespace chrono = std::chrono;
using namespace milvus;
using namespace milvus::segcore;
using namespace milvus::index;
using namespace milvus::segcore;
using namespace milvus::tracer;
using namespace knowhere;
using milvus::index::VectorIndex;
using milvus::segcore::LoadIndexInfo;
@ -57,14 +58,23 @@ namespace {
const int64_t ROW_COUNT = 10 * 1000;
const int64_t BIAS = 4200;
CStatus
CSearch(CSegmentInterface c_segment,
CSearchPlan c_plan,
CPlaceholderGroup c_placeholder_group,
uint64_t timestamp,
CSearchResult* result) {
return Search(
{}, c_segment, c_plan, c_placeholder_group, timestamp, result);
}
CStatus
CRetrieve(CSegmentInterface c_segment,
CRetrievePlan c_plan,
CTraceContext c_trace,
uint64_t timestamp,
CRetrieveResult* result) {
return Retrieve(
c_segment, c_plan, c_trace, timestamp, result, DEFAULT_MAX_OUTPUT_SIZE);
{}, c_segment, c_plan, timestamp, result, DEFAULT_MAX_OUTPUT_SIZE);
}
const char*
@ -675,7 +685,7 @@ TEST(CApiTest, MultiDeleteGrowingSegment) {
auto max_ts = dataset.timestamps_[N - 1] + 10;
CRetrieveResult retrieve_result;
res = CRetrieve(segment, plan.get(), {}, max_ts, &retrieve_result);
res = CRetrieve(segment, plan.get(), max_ts, &retrieve_result);
ASSERT_EQ(res.error_code, Success);
auto query_result = std::make_unique<proto::segcore::RetrieveResults>();
auto suc = query_result->ParseFromArray(retrieve_result.proto_blob,
@ -696,7 +706,7 @@ TEST(CApiTest, MultiDeleteGrowingSegment) {
retrive_pks);
plan->plan_node_->filter_plannode_ =
std::make_shared<plan::FilterBitsNode>(DEFAULT_PLANNODE_ID, term_expr);
res = CRetrieve(segment, plan.get(), {}, max_ts, &retrieve_result);
res = CRetrieve(segment, plan.get(), max_ts, &retrieve_result);
ASSERT_EQ(res.error_code, Success);
suc = query_result->ParseFromArray(retrieve_result.proto_blob,
retrieve_result.proto_size);
@ -721,7 +731,7 @@ TEST(CApiTest, MultiDeleteGrowingSegment) {
ASSERT_EQ(del_res.error_code, Success);
// retrieve pks in {2}
res = CRetrieve(segment, plan.get(), {}, max_ts, &retrieve_result);
res = CRetrieve(segment, plan.get(), max_ts, &retrieve_result);
ASSERT_EQ(res.error_code, Success);
suc = query_result->ParseFromArray(retrieve_result.proto_blob,
retrieve_result.proto_size);
@ -787,7 +797,7 @@ TEST(CApiTest, MultiDeleteSealedSegment) {
auto max_ts = dataset.timestamps_[N - 1] + 10;
CRetrieveResult retrieve_result;
auto res = CRetrieve(segment, plan.get(), {}, max_ts, &retrieve_result);
auto res = CRetrieve(segment, plan.get(), max_ts, &retrieve_result);
ASSERT_EQ(res.error_code, Success);
auto query_result = std::make_unique<proto::segcore::RetrieveResults>();
auto suc = query_result->ParseFromArray(retrieve_result.proto_blob,
@ -808,7 +818,7 @@ TEST(CApiTest, MultiDeleteSealedSegment) {
retrive_pks);
plan->plan_node_->filter_plannode_ =
std::make_shared<plan::FilterBitsNode>(DEFAULT_PLANNODE_ID, term_expr);
res = CRetrieve(segment, plan.get(), {}, max_ts, &retrieve_result);
res = CRetrieve(segment, plan.get(), max_ts, &retrieve_result);
ASSERT_EQ(res.error_code, Success);
suc = query_result->ParseFromArray(retrieve_result.proto_blob,
retrieve_result.proto_size);
@ -833,7 +843,7 @@ TEST(CApiTest, MultiDeleteSealedSegment) {
ASSERT_EQ(del_res.error_code, Success);
// retrieve pks in {2}
res = CRetrieve(segment, plan.get(), {}, max_ts, &retrieve_result);
res = CRetrieve(segment, plan.get(), max_ts, &retrieve_result);
ASSERT_EQ(res.error_code, Success);
suc = query_result->ParseFromArray(retrieve_result.proto_blob,
retrieve_result.proto_size);
@ -906,7 +916,7 @@ TEST(CApiTest, DeleteRepeatedPksFromGrowingSegment) {
CRetrieveResult retrieve_result;
res = CRetrieve(
segment, plan.get(), {}, dataset.timestamps_[N - 1], &retrieve_result);
segment, plan.get(), dataset.timestamps_[N - 1], &retrieve_result);
ASSERT_EQ(res.error_code, Success);
auto query_result = std::make_unique<proto::segcore::RetrieveResults>();
auto suc = query_result->ParseFromArray(retrieve_result.proto_blob,
@ -934,7 +944,7 @@ TEST(CApiTest, DeleteRepeatedPksFromGrowingSegment) {
// retrieve pks in {1, 2, 3}
res = CRetrieve(
segment, plan.get(), {}, dataset.timestamps_[N - 1], &retrieve_result);
segment, plan.get(), dataset.timestamps_[N - 1], &retrieve_result);
ASSERT_EQ(res.error_code, Success);
query_result = std::make_unique<proto::segcore::RetrieveResults>();
@ -987,7 +997,7 @@ TEST(CApiTest, DeleteRepeatedPksFromSealedSegment) {
CRetrieveResult retrieve_result;
auto res = CRetrieve(
segment, plan.get(), {}, dataset.timestamps_[N - 1], &retrieve_result);
segment, plan.get(), dataset.timestamps_[N - 1], &retrieve_result);
ASSERT_EQ(res.error_code, Success);
auto query_result = std::make_unique<proto::segcore::RetrieveResults>();
auto suc = query_result->ParseFromArray(retrieve_result.proto_blob,
@ -1016,7 +1026,7 @@ TEST(CApiTest, DeleteRepeatedPksFromSealedSegment) {
// retrieve pks in {1, 2, 3}
res = CRetrieve(
segment, plan.get(), {}, dataset.timestamps_[N - 1], &retrieve_result);
segment, plan.get(), dataset.timestamps_[N - 1], &retrieve_result);
ASSERT_EQ(res.error_code, Success);
query_result = std::make_unique<proto::segcore::RetrieveResults>();
@ -1097,7 +1107,7 @@ TEST(CApiTest, InsertSamePkAfterDeleteOnGrowingSegment) {
CRetrieveResult retrieve_result;
res = CRetrieve(
segment, plan.get(), {}, dataset.timestamps_[N - 1], &retrieve_result);
segment, plan.get(), dataset.timestamps_[N - 1], &retrieve_result);
ASSERT_EQ(res.error_code, Success);
auto query_result = std::make_unique<proto::segcore::RetrieveResults>();
auto suc = query_result->ParseFromArray(retrieve_result.proto_blob,
@ -1122,7 +1132,7 @@ TEST(CApiTest, InsertSamePkAfterDeleteOnGrowingSegment) {
// retrieve pks in {1, 2, 3}, timestamp = 19
res = CRetrieve(
segment, plan.get(), {}, dataset.timestamps_[N - 1], &retrieve_result);
segment, plan.get(), dataset.timestamps_[N - 1], &retrieve_result);
ASSERT_EQ(res.error_code, Success);
query_result = std::make_unique<proto::segcore::RetrieveResults>();
@ -1194,7 +1204,7 @@ TEST(CApiTest, InsertSamePkAfterDeleteOnSealedSegment) {
CRetrieveResult retrieve_result;
auto res = CRetrieve(
segment, plan.get(), {}, dataset.timestamps_[N - 1], &retrieve_result);
segment, plan.get(), dataset.timestamps_[N - 1], &retrieve_result);
ASSERT_EQ(res.error_code, Success);
auto query_result = std::make_unique<proto::segcore::RetrieveResults>();
auto suc = query_result->ParseFromArray(retrieve_result.proto_blob,
@ -1263,12 +1273,12 @@ TEST(CApiTest, SearchTest) {
CSearchResult search_result;
auto res =
Search(segment, plan, placeholderGroup, {}, ts_offset, &search_result);
CSearch(segment, plan, placeholderGroup, ts_offset, &search_result);
ASSERT_EQ(res.error_code, Success);
CSearchResult search_result2;
auto res2 =
Search(segment, plan, placeholderGroup, {}, ts_offset, &search_result2);
CSearch(segment, plan, placeholderGroup, ts_offset, &search_result2);
ASSERT_EQ(res2.error_code, Success);
DeleteSearchPlan(plan);
@ -1332,10 +1342,9 @@ TEST(CApiTest, SearchTestWithExpr) {
dataset.timestamps_.push_back(1);
CSearchResult search_result;
auto res = Search(segment,
auto res = CSearch(segment,
plan,
placeholderGroup,
{},
dataset.timestamps_[0],
&search_result);
ASSERT_EQ(res.error_code, Success);
@ -1392,7 +1401,7 @@ TEST(CApiTest, RetrieveTestWithExpr) {
CRetrieveResult retrieve_result;
auto res = CRetrieve(
segment, plan.get(), {}, dataset.timestamps_[0], &retrieve_result);
segment, plan.get(), dataset.timestamps_[0], &retrieve_result);
ASSERT_EQ(res.error_code, Success);
DeleteRetrievePlan(plan.release());
@ -1621,7 +1630,7 @@ TEST(CApiTest, ReduceNullResult) {
auto slice_topKs = std::vector<int64_t>{1};
std::vector<CSearchResult> results;
CSearchResult res;
status = Search(segment, plan, placeholderGroup, {}, 1L << 63, &res);
status = CSearch(segment, plan, placeholderGroup, 1L << 63, &res);
ASSERT_EQ(status.error_code, Success);
results.push_back(res);
CSearchResultDataBlobs cSearchResultData;
@ -1708,11 +1717,11 @@ TEST(CApiTest, ReduceRemoveDuplicates) {
auto slice_topKs = std::vector<int64_t>{topK / 2, topK};
std::vector<CSearchResult> results;
CSearchResult res1, res2;
status = Search(
segment, plan, placeholderGroup, {}, dataset.timestamps_[0], &res1);
status = CSearch(
segment, plan, placeholderGroup, dataset.timestamps_[0], &res1);
ASSERT_EQ(status.error_code, Success);
status = Search(
segment, plan, placeholderGroup, {}, dataset.timestamps_[0], &res2);
status = CSearch(
segment, plan, placeholderGroup, dataset.timestamps_[0], &res2);
ASSERT_EQ(status.error_code, Success);
results.push_back(res1);
results.push_back(res2);
@ -1741,14 +1750,14 @@ TEST(CApiTest, ReduceRemoveDuplicates) {
auto slice_topKs = std::vector<int64_t>{topK / 2, topK, topK};
std::vector<CSearchResult> results;
CSearchResult res1, res2, res3;
status = Search(
segment, plan, placeholderGroup, {}, dataset.timestamps_[0], &res1);
status = CSearch(
segment, plan, placeholderGroup, dataset.timestamps_[0], &res1);
ASSERT_EQ(status.error_code, Success);
status = Search(
segment, plan, placeholderGroup, {}, dataset.timestamps_[0], &res2);
status = CSearch(
segment, plan, placeholderGroup, dataset.timestamps_[0], &res2);
ASSERT_EQ(status.error_code, Success);
status = Search(
segment, plan, placeholderGroup, {}, dataset.timestamps_[0], &res3);
status = CSearch(
segment, plan, placeholderGroup, dataset.timestamps_[0], &res3);
ASSERT_EQ(status.error_code, Success);
results.push_back(res1);
results.push_back(res2);
@ -1865,11 +1874,11 @@ testReduceSearchWithExpr(int N,
std::vector<CSearchResult> results;
CSearchResult res1;
CSearchResult res2;
auto res = Search(
segment, plan, placeholderGroup, {}, dataset.timestamps_[N - 1], &res1);
auto res = CSearch(
segment, plan, placeholderGroup, dataset.timestamps_[N - 1], &res1);
ASSERT_EQ(res.error_code, Success);
res = Search(
segment, plan, placeholderGroup, {}, dataset.timestamps_[N - 1], &res2);
res = CSearch(
segment, plan, placeholderGroup, dataset.timestamps_[N - 1], &res2);
ASSERT_EQ(res.error_code, Success);
results.push_back(res1);
results.push_back(res2);
@ -2104,10 +2113,9 @@ TEST(CApiTest, Indexing_Without_Predicate) {
Timestamp timestmap = 10000000;
CSearchResult c_search_result_on_smallIndex;
auto res_before_load_index = Search(segment,
auto res_before_load_index = CSearch(segment,
plan,
placeholderGroup,
{},
timestmap,
&c_search_result_on_smallIndex);
ASSERT_EQ(res_before_load_index.error_code, Success);
@ -2165,10 +2173,9 @@ TEST(CApiTest, Indexing_Without_Predicate) {
sealed_segment->LoadIndex(*(LoadIndexInfo*)c_load_index_info);
CSearchResult c_search_result_on_bigIndex;
auto res_after_load_index = Search(sealed_segment.get(),
auto res_after_load_index = CSearch(sealed_segment.get(),
plan,
placeholderGroup,
{},
timestmap,
&c_search_result_on_bigIndex);
ASSERT_EQ(res_after_load_index.error_code, Success);
@ -2255,10 +2262,9 @@ TEST(CApiTest, Indexing_Expr_Without_Predicate) {
Timestamp timestamp = 10000000;
CSearchResult c_search_result_on_smallIndex;
auto res_before_load_index = Search(segment,
auto res_before_load_index = CSearch(segment,
plan,
placeholderGroup,
{},
timestamp,
&c_search_result_on_smallIndex);
ASSERT_EQ(res_before_load_index.error_code, Success);
@ -2317,10 +2323,9 @@ TEST(CApiTest, Indexing_Expr_Without_Predicate) {
sealed_segment->LoadIndex(*(LoadIndexInfo*)c_load_index_info);
CSearchResult c_search_result_on_bigIndex;
auto res_after_load_index = Search(sealed_segment.get(),
auto res_after_load_index = CSearch(sealed_segment.get(),
plan,
placeholderGroup,
{},
timestamp,
&c_search_result_on_bigIndex);
ASSERT_EQ(res_after_load_index.error_code, Success);
@ -2435,10 +2440,9 @@ TEST(CApiTest, Indexing_With_float_Predicate_Range) {
Timestamp timestamp = 10000000;
CSearchResult c_search_result_on_smallIndex;
auto res_before_load_index = Search(segment,
auto res_before_load_index = CSearch(segment,
plan,
placeholderGroup,
{},
timestamp,
&c_search_result_on_smallIndex);
ASSERT_EQ(res_before_load_index.error_code, Success);
@ -2497,10 +2501,9 @@ TEST(CApiTest, Indexing_With_float_Predicate_Range) {
sealed_segment->LoadIndex(*(LoadIndexInfo*)c_load_index_info);
CSearchResult c_search_result_on_bigIndex;
auto res_after_load_index = Search(sealed_segment.get(),
auto res_after_load_index = CSearch(sealed_segment.get(),
plan,
placeholderGroup,
{},
timestamp,
&c_search_result_on_bigIndex);
ASSERT_EQ(res_after_load_index.error_code, Success);
@ -2617,10 +2620,9 @@ TEST(CApiTest, Indexing_Expr_With_float_Predicate_Range) {
Timestamp timestamp = 10000000;
CSearchResult c_search_result_on_smallIndex;
auto res_before_load_index = Search(segment,
auto res_before_load_index = CSearch(segment,
plan,
placeholderGroup,
{},
timestamp,
&c_search_result_on_smallIndex);
ASSERT_EQ(res_before_load_index.error_code, Success);
@ -2679,10 +2681,9 @@ TEST(CApiTest, Indexing_Expr_With_float_Predicate_Range) {
sealed_segment->LoadIndex(*(LoadIndexInfo*)c_load_index_info);
CSearchResult c_search_result_on_bigIndex;
auto res_after_load_index = Search(sealed_segment.get(),
auto res_after_load_index = CSearch(sealed_segment.get(),
plan,
placeholderGroup,
{},
timestamp,
&c_search_result_on_bigIndex);
ASSERT_EQ(res_after_load_index.error_code, Success);
@ -2791,10 +2792,9 @@ TEST(CApiTest, Indexing_With_float_Predicate_Term) {
Timestamp timestamp = 10000000;
CSearchResult c_search_result_on_smallIndex;
auto res_before_load_index = Search(segment,
auto res_before_load_index = CSearch(segment,
plan,
placeholderGroup,
{},
timestamp,
&c_search_result_on_smallIndex);
ASSERT_EQ(res_before_load_index.error_code, Success);
@ -2853,10 +2853,9 @@ TEST(CApiTest, Indexing_With_float_Predicate_Term) {
sealed_segment->LoadIndex(*(LoadIndexInfo*)c_load_index_info);
CSearchResult c_search_result_on_bigIndex;
auto res_after_load_index = Search(sealed_segment.get(),
auto res_after_load_index = CSearch(sealed_segment.get(),
plan,
placeholderGroup,
{},
timestamp,
&c_search_result_on_bigIndex);
ASSERT_EQ(res_after_load_index.error_code, Success);
@ -2966,10 +2965,9 @@ TEST(CApiTest, Indexing_Expr_With_float_Predicate_Term) {
Timestamp timestamp = 10000000;
CSearchResult c_search_result_on_smallIndex;
auto res_before_load_index = Search(segment,
auto res_before_load_index = CSearch(segment,
plan,
placeholderGroup,
{},
timestamp,
&c_search_result_on_smallIndex);
ASSERT_EQ(res_before_load_index.error_code, Success);
@ -3028,10 +3026,9 @@ TEST(CApiTest, Indexing_Expr_With_float_Predicate_Term) {
sealed_segment->LoadIndex(*(LoadIndexInfo*)c_load_index_info);
CSearchResult c_search_result_on_bigIndex;
auto res_after_load_index = Search(sealed_segment.get(),
auto res_after_load_index = CSearch(sealed_segment.get(),
plan,
placeholderGroup,
{},
timestamp,
&c_search_result_on_bigIndex);
ASSERT_EQ(res_after_load_index.error_code, Success);
@ -3147,10 +3144,9 @@ TEST(CApiTest, Indexing_With_binary_Predicate_Range) {
Timestamp timestamp = 10000000;
CSearchResult c_search_result_on_smallIndex;
auto res_before_load_index = Search(segment,
auto res_before_load_index = CSearch(segment,
plan,
placeholderGroup,
{},
timestamp,
&c_search_result_on_smallIndex);
ASSERT_EQ(res_before_load_index.error_code, Success);
@ -3210,10 +3206,9 @@ TEST(CApiTest, Indexing_With_binary_Predicate_Range) {
sealed_segment->LoadIndex(*(LoadIndexInfo*)c_load_index_info);
CSearchResult c_search_result_on_bigIndex;
auto res_after_load_index = Search(sealed_segment.get(),
auto res_after_load_index = CSearch(sealed_segment.get(),
plan,
placeholderGroup,
{},
timestamp,
&c_search_result_on_bigIndex);
ASSERT_EQ(res_after_load_index.error_code, Success);
@ -3329,10 +3324,9 @@ TEST(CApiTest, Indexing_Expr_With_binary_Predicate_Range) {
Timestamp timestamp = 10000000;
CSearchResult c_search_result_on_smallIndex;
auto res_before_load_index = Search(segment,
auto res_before_load_index = CSearch(segment,
plan,
placeholderGroup,
{},
timestamp,
&c_search_result_on_smallIndex);
ASSERT_TRUE(res_before_load_index.error_code == Success)
@ -3392,10 +3386,9 @@ TEST(CApiTest, Indexing_Expr_With_binary_Predicate_Range) {
sealed_segment->LoadIndex(*(LoadIndexInfo*)c_load_index_info);
CSearchResult c_search_result_on_bigIndex;
auto res_after_load_index = Search(sealed_segment.get(),
auto res_after_load_index = CSearch(sealed_segment.get(),
plan,
placeholderGroup,
{},
timestamp,
&c_search_result_on_bigIndex);
ASSERT_EQ(res_after_load_index.error_code, Success);
@ -3506,10 +3499,9 @@ TEST(CApiTest, Indexing_With_binary_Predicate_Term) {
Timestamp timestamp = 10000000;
CSearchResult c_search_result_on_smallIndex;
auto res_before_load_index = Search(segment,
auto res_before_load_index = CSearch(segment,
plan,
placeholderGroup,
{},
timestamp,
&c_search_result_on_smallIndex);
ASSERT_EQ(res_before_load_index.error_code, Success);
@ -3568,10 +3560,9 @@ TEST(CApiTest, Indexing_With_binary_Predicate_Term) {
sealed_segment->LoadIndex(*(LoadIndexInfo*)c_load_index_info);
CSearchResult c_search_result_on_bigIndex;
auto res_after_load_index = Search(sealed_segment.get(),
auto res_after_load_index = CSearch(sealed_segment.get(),
plan,
placeholderGroup,
{},
timestamp,
&c_search_result_on_bigIndex);
ASSERT_EQ(res_after_load_index.error_code, Success);
@ -3704,10 +3695,9 @@ TEST(CApiTest, Indexing_Expr_With_binary_Predicate_Term) {
Timestamp timestamp = 10000000;
CSearchResult c_search_result_on_smallIndex;
auto res_before_load_index = Search(segment,
auto res_before_load_index = CSearch(segment,
plan,
placeholderGroup,
{},
timestamp,
&c_search_result_on_smallIndex);
ASSERT_EQ(res_before_load_index.error_code, Success);
@ -3766,10 +3756,9 @@ TEST(CApiTest, Indexing_Expr_With_binary_Predicate_Term) {
sealed_segment->LoadIndex(*(LoadIndexInfo*)c_load_index_info);
CSearchResult c_search_result_on_bigIndex;
auto res_after_load_index = Search(sealed_segment.get(),
auto res_after_load_index = CSearch(sealed_segment.get(),
plan,
placeholderGroup,
{},
timestamp,
&c_search_result_on_bigIndex);
ASSERT_EQ(res_after_load_index.error_code, Success);
@ -3964,10 +3953,9 @@ TEST(CApiTest, SealedSegment_search_float_Predicate_Range) {
sealed_segment->LoadIndex(*(LoadIndexInfo*)c_load_index_info);
CSearchResult c_search_result_on_bigIndex;
auto res_after_load_index = Search(sealed_segment.get(),
auto res_after_load_index = CSearch(sealed_segment.get(),
plan,
placeholderGroup,
{},
timestamp,
&c_search_result_on_bigIndex);
ASSERT_EQ(res_after_load_index.error_code, Success);
@ -4047,14 +4035,14 @@ TEST(CApiTest, SealedSegment_search_without_predicates) {
std::vector<CPlaceholderGroup> placeholderGroups;
placeholderGroups.push_back(placeholderGroup);
CSearchResult search_result;
auto res = Search(
segment, plan, placeholderGroup, {}, N + ts_offset, &search_result);
auto res = CSearch(
segment, plan, placeholderGroup, N + ts_offset, &search_result);
std::cout << res.error_msg << std::endl;
ASSERT_EQ(res.error_code, Success);
CSearchResult search_result2;
auto res2 = Search(
segment, plan, placeholderGroup, {}, N + ts_offset, &search_result2);
auto res2 = CSearch(
segment, plan, placeholderGroup, N + ts_offset, &search_result2);
ASSERT_EQ(res2.error_code, Success);
DeleteSearchPlan(plan);
@ -4203,10 +4191,9 @@ TEST(CApiTest, SealedSegment_search_float_With_Expr_Predicate_Range) {
}
CSearchResult c_search_result_on_bigIndex;
auto res_after_load_index = Search(segment,
auto res_after_load_index = CSearch(segment,
plan,
placeholderGroup,
{},
timestamp,
&c_search_result_on_bigIndex);
ASSERT_EQ(res_after_load_index.error_code, Success);
@ -4403,7 +4390,7 @@ TEST(CApiTest, RetriveScalarFieldFromSealedSegmentWithIndex) {
CRetrieveResult retrieve_result;
res = CRetrieve(
segment, plan.get(), {}, raw_data.timestamps_[N - 1], &retrieve_result);
segment, plan.get(), raw_data.timestamps_[N - 1], &retrieve_result);
ASSERT_EQ(res.error_code, Success);
auto query_result = std::make_unique<proto::segcore::RetrieveResults>();
auto suc = query_result->ParseFromArray(retrieve_result.proto_blob,
@ -4505,7 +4492,7 @@ TEST(CApiTest, RANGE_SEARCH_WITH_RADIUS_WHEN_IP) {
CSearchResult search_result;
auto res =
Search(segment, plan, placeholderGroup, {}, ts_offset, &search_result);
CSearch(segment, plan, placeholderGroup, ts_offset, &search_result);
ASSERT_EQ(res.error_code, Success);
DeleteSearchPlan(plan);
@ -4570,7 +4557,7 @@ TEST(CApiTest, RANGE_SEARCH_WITH_RADIUS_AND_RANGE_FILTER_WHEN_IP) {
CSearchResult search_result;
auto res =
Search(segment, plan, placeholderGroup, {}, ts_offset, &search_result);
CSearch(segment, plan, placeholderGroup, ts_offset, &search_result);
ASSERT_EQ(res.error_code, Success);
DeleteSearchPlan(plan);
@ -4634,7 +4621,7 @@ TEST(CApiTest, RANGE_SEARCH_WITH_RADIUS_WHEN_L2) {
CSearchResult search_result;
auto res =
Search(segment, plan, placeholderGroup, {}, ts_offset, &search_result);
CSearch(segment, plan, placeholderGroup, ts_offset, &search_result);
ASSERT_EQ(res.error_code, Success);
DeleteSearchPlan(plan);
@ -4698,7 +4685,7 @@ TEST(CApiTest, RANGE_SEARCH_WITH_RADIUS_AND_RANGE_FILTER_WHEN_L2) {
CSearchResult search_result;
auto res =
Search(segment, plan, placeholderGroup, {}, ts_offset, &search_result);
CSearch(segment, plan, placeholderGroup, ts_offset, &search_result);
ASSERT_EQ(res.error_code, Success);
DeleteSearchPlan(plan);
@ -4922,10 +4909,9 @@ TEST(CApiTest, Indexing_Without_Predicate_float16) {
Timestamp timestmap = 10000000;
CSearchResult c_search_result_on_smallIndex;
auto res_before_load_index = Search(segment,
auto res_before_load_index = CSearch(segment,
plan,
placeholderGroup,
{},
timestmap,
&c_search_result_on_smallIndex);
ASSERT_EQ(res_before_load_index.error_code, Success);
@ -4983,10 +4969,9 @@ TEST(CApiTest, Indexing_Without_Predicate_float16) {
sealed_segment->DropFieldData(FieldId(100));
sealed_segment->LoadIndex(*(LoadIndexInfo*)c_load_index_info);
CSearchResult c_search_result_on_bigIndex;
auto res_after_load_index = Search(sealed_segment.get(),
auto res_after_load_index = CSearch(sealed_segment.get(),
plan,
placeholderGroup,
{},
timestmap,
&c_search_result_on_bigIndex);
ASSERT_EQ(res_after_load_index.error_code, Success);
@ -5074,10 +5059,9 @@ TEST(CApiTest, Indexing_Without_Predicate_bfloat16) {
Timestamp timestmap = 10000000;
CSearchResult c_search_result_on_smallIndex;
auto res_before_load_index = Search(segment,
auto res_before_load_index = CSearch(segment,
plan,
placeholderGroup,
{},
timestmap,
&c_search_result_on_smallIndex);
ASSERT_EQ(res_before_load_index.error_code, Success);
@ -5135,10 +5119,9 @@ TEST(CApiTest, Indexing_Without_Predicate_bfloat16) {
sealed_segment->DropFieldData(FieldId(100));
sealed_segment->LoadIndex(*(LoadIndexInfo*)c_load_index_info);
CSearchResult c_search_result_on_bigIndex;
auto res_after_load_index = Search(sealed_segment.get(),
auto res_after_load_index = CSearch(sealed_segment.get(),
plan,
placeholderGroup,
{},
timestmap,
&c_search_result_on_bigIndex);
ASSERT_EQ(res_after_load_index.error_code, Success);
@ -5215,7 +5198,7 @@ TEST(CApiTest, RANGE_SEARCH_WITH_RADIUS_AND_RANGE_FILTER_WHEN_IP_FLOAT16) {
CSearchResult search_result;
auto res =
Search(segment, plan, placeholderGroup, {}, ts_offset, &search_result);
CSearch(segment, plan, placeholderGroup, ts_offset, &search_result);
ASSERT_EQ(res.error_code, Success);
DeleteSearchPlan(plan);
@ -5280,7 +5263,7 @@ TEST(CApiTest, RANGE_SEARCH_WITH_RADIUS_AND_RANGE_FILTER_WHEN_IP_BFLOAT16) {
CSearchResult search_result;
auto res =
Search(segment, plan, placeholderGroup, {}, ts_offset, &search_result);
CSearch(segment, plan, placeholderGroup, ts_offset, &search_result);
ASSERT_EQ(res.error_code, Success);
DeleteSearchPlan(plan);

140
internal/core/unittest/test_expr.cpp
View File

@ -35,11 +35,12 @@
#include "exec/expression/Expr.h"
#include "exec/Task.h"
TEST(Expr, Range) {
SUCCEED();
using namespace milvus;
using namespace milvus::query;
using namespace milvus::segcore;
TEST(Expr, Range) {
SUCCEED();
// std::string dsl_string = R"({
// "bool": {
// "must": [
@ -121,9 +122,6 @@ TEST(Expr, Range) {
TEST(Expr, RangeBinary) {
SUCCEED();
using namespace milvus;
using namespace milvus::query;
using namespace milvus::segcore;
// std::string dsl_string = R"({
// "bool": {
// "must": [
@ -204,9 +202,6 @@ TEST(Expr, RangeBinary) {
TEST(Expr, InvalidRange) {
SUCCEED();
using namespace milvus;
using namespace milvus::query;
using namespace milvus::segcore;
// std::string dsl_string = R"(
// {
// "bool": {
@ -283,9 +278,6 @@ TEST(Expr, InvalidRange) {
}
TEST(Expr, ShowExecutor) {
using namespace milvus;
using namespace milvus::query;
using namespace milvus::segcore;
auto node = std::make_unique<FloatVectorANNS>();
auto schema = std::make_shared<Schema>();
auto metric_type = knowhere::metric::L2;
@ -308,9 +300,6 @@ TEST(Expr, ShowExecutor) {
}
TEST(Expr, TestRange) {
using namespace milvus;
using namespace milvus::query;
using namespace milvus::segcore;
std::vector<std::tuple<std::string, std::function<bool(int)>>> testcases = {
{R"(binary_range_expr: <
column_info: <
@ -529,10 +518,6 @@ TEST(Expr, TestRange) {
}
TEST(Expr, TestBinaryRangeJSON) {
using namespace milvus;
using namespace milvus::query;
using namespace milvus::segcore;
struct Testcase {
bool lower_inclusive;
bool upper_inclusive;
@ -632,10 +617,6 @@ TEST(Expr, TestBinaryRangeJSON) {
}
TEST(Expr, TestExistsJson) {
using namespace milvus;
using namespace milvus::query;
using namespace milvus::segcore;
struct Testcase {
std::vector<std::string> nested_path;
};
@ -727,10 +708,6 @@ GetValueFromProto(const milvus::proto::plan::GenericValue& value_proto) {
};
TEST(Expr, TestUnaryRangeJson) {
using namespace milvus;
using namespace milvus::query;
using namespace milvus::segcore;
struct Testcase {
int64_t val;
std::vector<std::string> nested_path;
@ -900,10 +877,6 @@ TEST(Expr, TestUnaryRangeJson) {
}
TEST(Expr, TestTermJson) {
using namespace milvus;
using namespace milvus::query;
using namespace milvus::segcore;
struct Testcase {
std::vector<int64_t> term;
std::vector<std::string> nested_path;
@ -975,9 +948,6 @@ TEST(Expr, TestTermJson) {
}
TEST(Expr, TestTerm) {
using namespace milvus;
using namespace milvus::query;
using namespace milvus::segcore;
auto vec_2k_3k = [] {
std::string buf;
for (int i = 2000; i < 3000; ++i) {
@ -1102,9 +1072,6 @@ TEST(Expr, TestTerm) {
}
TEST(Expr, TestCompare) {
using namespace milvus;
using namespace milvus::query;
using namespace milvus::segcore;
std::vector<std::tuple<std::string, std::function<bool(int, int64_t)>>>
testcases = {
{R"(LessThan)", [](int a, int64_t b) { return a < b; }},
@ -1222,9 +1189,6 @@ TEST(Expr, TestCompare) {
}
TEST(Expr, TestCompareWithScalarIndex) {
using namespace milvus;
using namespace milvus::query;
using namespace milvus::segcore;
std::vector<std::tuple<std::string, std::function<bool(int, int64_t)>>>
testcases = {
{R"(LessThan)", [](int a, int64_t b) { return a < b; }},
@ -1321,9 +1285,6 @@ TEST(Expr, TestCompareWithScalarIndex) {
}
TEST(Expr, TestCompareExpr) {
using namespace milvus;
using namespace milvus::query;
using namespace milvus::segcore;
auto schema = std::make_shared<Schema>();
auto vec_fid = schema->AddDebugField(
"fakevec", DataType::VECTOR_FLOAT, 16, knowhere::metric::L2);
@ -1473,9 +1434,6 @@ TEST(Expr, TestCompareExpr) {
}
TEST(Expr, TestMultiLogicalExprsOptimization) {
using namespace milvus;
using namespace milvus::query;
using namespace milvus::segcore;
auto schema = std::make_shared<Schema>();
auto vec_fid = schema->AddDebugField(
"fakevec", DataType::VECTOR_FLOAT, 16, knowhere::metric::L2);
@ -1562,9 +1520,6 @@ TEST(Expr, TestMultiLogicalExprsOptimization) {
}
TEST(Expr, TestExprs) {
using namespace milvus;
using namespace milvus::query;
using namespace milvus::segcore;
auto schema = std::make_shared<Schema>();
auto vec_fid = schema->AddDebugField(
"fakevec", DataType::VECTOR_FLOAT, 16, knowhere::metric::L2);
@ -1737,9 +1692,6 @@ TEST(Expr, TestExprs) {
}
TEST(Expr, test_term_pk) {
using namespace milvus;
using namespace milvus::query;
using namespace milvus::segcore;
auto schema = std::make_shared<Schema>();
schema->AddField(FieldName("Timestamp"), FieldId(1), DataType::INT64);
auto vec_fid = schema->AddDebugField(
@ -1804,9 +1756,6 @@ TEST(Expr, test_term_pk) {
}
TEST(Expr, TestSealedSegmentGetBatchSize) {
using namespace milvus;
using namespace milvus::query;
using namespace milvus::segcore;
auto schema = std::make_shared<Schema>();
auto vec_fid = schema->AddDebugField(
"fakevec", DataType::VECTOR_FLOAT, 16, knowhere::metric::L2);
@ -1869,9 +1818,6 @@ TEST(Expr, TestSealedSegmentGetBatchSize) {
}
TEST(Expr, TestGrowingSegmentGetBatchSize) {
using namespace milvus;
using namespace milvus::query;
using namespace milvus::segcore;
auto schema = std::make_shared<Schema>();
auto vec_fid = schema->AddDebugField(
"fakevec", DataType::VECTOR_FLOAT, 16, knowhere::metric::L2);
@ -1928,9 +1874,6 @@ TEST(Expr, TestGrowingSegmentGetBatchSize) {
}
TEST(Expr, TestConjuctExpr) {
using namespace milvus;
using namespace milvus::query;
using namespace milvus::segcore;
auto schema = std::make_shared<Schema>();
auto vec_fid = schema->AddDebugField(
"fakevec", DataType::VECTOR_FLOAT, 16, knowhere::metric::L2);
@ -1999,9 +1942,6 @@ TEST(Expr, TestConjuctExpr) {
}
TEST(Expr, TestUnaryBenchTest) {
using namespace milvus;
using namespace milvus::query;
using namespace milvus::segcore;
auto schema = std::make_shared<Schema>();
auto vec_fid = schema->AddDebugField(
"fakevec", DataType::VECTOR_FLOAT, 16, knowhere::metric::L2);
@ -2074,9 +2014,6 @@ TEST(Expr, TestUnaryBenchTest) {
}
TEST(Expr, TestBinaryRangeBenchTest) {
using namespace milvus;
using namespace milvus::query;
using namespace milvus::segcore;
auto schema = std::make_shared<Schema>();
auto vec_fid = schema->AddDebugField(
"fakevec", DataType::VECTOR_FLOAT, 16, knowhere::metric::L2);
@ -2158,9 +2095,6 @@ TEST(Expr, TestBinaryRangeBenchTest) {
}
TEST(Expr, TestLogicalUnaryBenchTest) {
using namespace milvus;
using namespace milvus::query;
using namespace milvus::segcore;
auto schema = std::make_shared<Schema>();
auto vec_fid = schema->AddDebugField(
"fakevec", DataType::VECTOR_FLOAT, 16, knowhere::metric::L2);
@ -2236,9 +2170,6 @@ TEST(Expr, TestLogicalUnaryBenchTest) {
}
TEST(Expr, TestBinaryLogicalBenchTest) {
using namespace milvus;
using namespace milvus::query;
using namespace milvus::segcore;
auto schema = std::make_shared<Schema>();
auto vec_fid = schema->AddDebugField(
"fakevec", DataType::VECTOR_FLOAT, 16, knowhere::metric::L2);
@ -2324,9 +2255,6 @@ TEST(Expr, TestBinaryLogicalBenchTest) {
}
TEST(Expr, TestBinaryArithOpEvalRangeBenchExpr) {
using namespace milvus;
using namespace milvus::query;
using namespace milvus::segcore;
auto schema = std::make_shared<Schema>();
auto vec_fid = schema->AddDebugField(
"fakevec", DataType::VECTOR_FLOAT, 16, knowhere::metric::L2);
@ -2408,9 +2336,6 @@ TEST(Expr, TestBinaryArithOpEvalRangeBenchExpr) {
}
TEST(Expr, TestCompareExprBenchTest) {
using namespace milvus;
using namespace milvus::query;
using namespace milvus::segcore;
auto schema = std::make_shared<Schema>();
auto vec_fid = schema->AddDebugField(
"fakevec", DataType::VECTOR_FLOAT, 16, knowhere::metric::L2);
@ -2485,9 +2410,6 @@ TEST(Expr, TestCompareExprBenchTest) {
}
TEST(Expr, TestRefactorExprs) {
using namespace milvus;
using namespace milvus::query;
using namespace milvus::segcore;
auto schema = std::make_shared<Schema>();
auto vec_fid = schema->AddDebugField(
"fakevec", DataType::VECTOR_FLOAT, 16, knowhere::metric::L2);
@ -2658,9 +2580,6 @@ TEST(Expr, TestRefactorExprs) {
}
TEST(Expr, TestCompareWithScalarIndexMaris) {
using namespace milvus;
using namespace milvus::query;
using namespace milvus::segcore;
std::vector<
std::tuple<std::string, std::function<bool(std::string, std::string)>>>
testcases = {
@ -2760,9 +2679,6 @@ TEST(Expr, TestCompareWithScalarIndexMaris) {
}
TEST(Expr, TestBinaryArithOpEvalRange) {
using namespace milvus;
using namespace milvus::query;
using namespace milvus::segcore;
std::vector<std::tuple<std::string, std::function<bool(int)>, DataType>>
testcases = {
// Add test cases for BinaryArithOpEvalRangeExpr EQ of various data types
@ -3136,10 +3052,6 @@ TEST(Expr, TestBinaryArithOpEvalRange) {
}
TEST(Expr, TestBinaryArithOpEvalRangeJSON) {
using namespace milvus;
using namespace milvus::query;
using namespace milvus::segcore;
struct Testcase {
int64_t right_operand;
int64_t value;
@ -3229,10 +3141,6 @@ TEST(Expr, TestBinaryArithOpEvalRangeJSON) {
}
TEST(Expr, TestBinaryArithOpEvalRangeJSONFloat) {
using namespace milvus;
using namespace milvus::query;
using namespace milvus::segcore;
struct Testcase {
double right_operand;
double value;
@ -3359,9 +3267,6 @@ TEST(Expr, TestBinaryArithOpEvalRangeJSONFloat) {
}
TEST(Expr, TestBinaryArithOpEvalRangeWithScalarSortIndex) {
using namespace milvus;
using namespace milvus::query;
using namespace milvus::segcore;
std::vector<std::tuple<std::string, std::function<bool(int)>, DataType>>
testcases = {
// Add test cases for BinaryArithOpEvalRangeExpr EQ of various data types
@ -3678,9 +3583,6 @@ TEST(Expr, TestBinaryArithOpEvalRangeWithScalarSortIndex) {
}
TEST(Expr, TestUnaryRangeWithJSON) {
using namespace milvus;
using namespace milvus::query;
using namespace milvus::segcore;
std::vector<
std::tuple<std::string,
std::function<bool(
@ -3881,9 +3783,6 @@ TEST(Expr, TestUnaryRangeWithJSON) {
}
TEST(Expr, TestTermWithJSON) {
using namespace milvus;
using namespace milvus::query;
using namespace milvus::segcore;
std::vector<
std::tuple<std::string,
std::function<bool(
@ -4062,9 +3961,6 @@ TEST(Expr, TestTermWithJSON) {
}
TEST(Expr, TestExistsWithJSON) {
using namespace milvus;
using namespace milvus::query;
using namespace milvus::segcore;
std::vector<std::tuple<std::string, std::function<bool(bool)>, DataType>>
testcases = {
{R"()", [](bool v) { return v; }, DataType::BOOL},
@ -4225,10 +4121,6 @@ struct Testcase {
};
TEST(Expr, TestTermInFieldJson) {
using namespace milvus;
using namespace milvus::query;
using namespace milvus::segcore;
auto schema = std::make_shared<Schema>();
auto i64_fid = schema->AddDebugField("id", DataType::INT64);
auto json_fid = schema->AddDebugField("json", DataType::JSON);
@ -4445,10 +4337,6 @@ TEST(Expr, TestTermInFieldJson) {
}
TEST(Expr, PraseJsonContainsExpr) {
using namespace milvus;
using namespace milvus::query;
using namespace milvus::segcore;
std::vector<const char*> raw_plans{
R"(vector_anns:<
field_id:100
@ -4592,10 +4480,6 @@ TEST(Expr, PraseJsonContainsExpr) {
}
TEST(Expr, TestJsonContainsAny) {
using namespace milvus;
using namespace milvus::query;
using namespace milvus::segcore;
auto schema = std::make_shared<Schema>();
auto i64_fid = schema->AddDebugField("id", DataType::INT64);
auto json_fid = schema->AddDebugField("json", DataType::JSON);
@ -4816,10 +4700,6 @@ TEST(Expr, TestJsonContainsAny) {
}
TEST(Expr, TestJsonContainsAll) {
using namespace milvus;
using namespace milvus::query;
using namespace milvus::segcore;
auto schema = std::make_shared<Schema>();
auto i64_fid = schema->AddDebugField("id", DataType::INT64);
auto json_fid = schema->AddDebugField("json", DataType::JSON);
@ -5064,10 +4944,6 @@ TEST(Expr, TestJsonContainsAll) {
}
TEST(Expr, TestJsonContainsArray) {
using namespace milvus;
using namespace milvus::query;
using namespace milvus::segcore;
auto schema = std::make_shared<Schema>();
auto i64_fid = schema->AddDebugField("id", DataType::INT64);
auto json_fid = schema->AddDebugField("json", DataType::JSON);
@ -5371,8 +5247,6 @@ generatedArrayWithFourDiffType(int64_t int_val,
double float_val,
bool bool_val,
std::string string_val) {
using namespace milvus;
proto::plan::GenericValue value;
proto::plan::Array diff_type_array;
diff_type_array.set_same_type(false);
@ -5397,10 +5271,6 @@ generatedArrayWithFourDiffType(int64_t int_val,
}
TEST(Expr, TestJsonContainsDiffTypeArray) {
using namespace milvus;
using namespace milvus::query;
using namespace milvus::segcore;
auto schema = std::make_shared<Schema>();
auto i64_fid = schema->AddDebugField("id", DataType::INT64);
auto json_fid = schema->AddDebugField("json", DataType::JSON);
@ -5503,10 +5373,6 @@ TEST(Expr, TestJsonContainsDiffTypeArray) {
}
TEST(Expr, TestJsonContainsDiffType) {
using namespace milvus;
using namespace milvus::query;
using namespace milvus::segcore;
auto schema = std::make_shared<Schema>();
auto i64_fid = schema->AddDebugField("id", DataType::INT64);
auto json_fid = schema->AddDebugField("json", DataType::JSON);

26
internal/core/unittest/test_float16.cpp
View File

@ -43,19 +43,18 @@
#include "test_utils/AssertUtils.h"
#include "test_utils/DataGen.h"
using namespace milvus::segcore;
using namespace milvus;
using namespace milvus::index;
using namespace milvus::query;
using namespace milvus::segcore;
using namespace knowhere;
using milvus::index::VectorIndex;
using milvus::segcore::LoadIndexInfo;
const int64_t ROW_COUNT = 100 * 1000;
// TEST(Float16, Insert) {
// using namespace milvus;
// using namespace milvus::query;
// using namespace milvus::segcore;
// int64_t N = ROW_COUNT;
// constexpr int64_t size_per_chunk = 32 * 1024;
// auto schema = std::make_shared<Schema>();
@ -93,9 +92,6 @@ const int64_t ROW_COUNT = 100 * 1000;
// }
TEST(Float16, ShowExecutor) {
using namespace milvus::query;
using namespace milvus::segcore;
using namespace milvus;
auto metric_type = knowhere::metric::L2;
auto node = std::make_unique<Float16VectorANNS>();
auto schema = std::make_shared<Schema>();
@ -116,9 +112,6 @@ TEST(Float16, ShowExecutor) {
}
TEST(Float16, ExecWithoutPredicateFlat) {
using namespace milvus::query;
using namespace milvus::segcore;
using namespace milvus;
auto schema = std::make_shared<Schema>();
auto vec_fid = schema->AddDebugField(
"fakevec", DataType::VECTOR_FLOAT16, 32, knowhere::metric::L2);
@ -260,8 +253,6 @@ TEST(Float16, RetrieveEmpty) {
}
TEST(Float16, ExecWithPredicate) {
using namespace milvus::query;
using namespace milvus::segcore;
auto schema = std::make_shared<Schema>();
schema->AddDebugField(
"fakevec", DataType::VECTOR_FLOAT16, 16, knowhere::metric::L2);
@ -320,9 +311,6 @@ TEST(Float16, ExecWithPredicate) {
}
// TEST(BFloat16, Insert) {
// using namespace milvus;
// using namespace milvus::query;
// using namespace milvus::segcore;
// int64_t N = ROW_COUNT;
// constexpr int64_t size_per_chunk = 32 * 1024;
// auto schema = std::make_shared<Schema>();
@ -360,9 +348,6 @@ TEST(Float16, ExecWithPredicate) {
// }
TEST(BFloat16, ShowExecutor) {
using namespace milvus::query;
using namespace milvus::segcore;
using namespace milvus;
auto metric_type = knowhere::metric::L2;
auto node = std::make_unique<BFloat16VectorANNS>();
auto schema = std::make_shared<Schema>();
@ -383,9 +368,6 @@ TEST(BFloat16, ShowExecutor) {
}
TEST(BFloat16, ExecWithoutPredicateFlat) {
using namespace milvus::query;
using namespace milvus::segcore;
using namespace milvus;
auto schema = std::make_shared<Schema>();
auto vec_fid = schema->AddDebugField(
"fakevec", DataType::VECTOR_BFLOAT16, 32, knowhere::metric::L2);
@ -524,8 +506,6 @@ TEST(BFloat16, RetrieveEmpty) {
}
TEST(BFloat16, ExecWithPredicate) {
using namespace milvus::query;
using namespace milvus::segcore;
auto schema = std::make_shared<Schema>();
schema->AddDebugField(
"fakevec", DataType::VECTOR_BFLOAT16, 16, knowhere::metric::L2);

15
internal/core/unittest/test_group_by.cpp
View File

@ -11,18 +11,19 @@
#include <gtest/gtest.h>
#include "common/Schema.h"
#include "segcore/SegmentSealedImpl.h"
#include "test_utils/DataGen.h"
#include "query/Plan.h"
#include "segcore/segment_c.h"
#include "segcore/SegmentSealedImpl.h"
#include "segcore/reduce_c.h"
#include "test_utils/c_api_test_utils.h"
#include "segcore/plan_c.h"
#include "segcore/segment_c.h"
#include "test_utils/DataGen.h"
#include "test_utils/c_api_test_utils.h"
using namespace milvus;
using namespace milvus::segcore;
using namespace milvus::query;
using namespace milvus::segcore;
using namespace milvus::storage;
using namespace milvus::tracer;
const char* METRICS_TYPE = "metric_type";
@ -604,10 +605,10 @@ TEST(GroupBY, Reduce) {
CSearchResult c_search_res_1;
CSearchResult c_search_res_2;
auto status =
Search(c_segment_1, c_plan, c_ph_group, {}, 1L << 63, &c_search_res_1);
Search({}, c_segment_1, c_plan, c_ph_group, 1L << 63, &c_search_res_1);
ASSERT_EQ(status.error_code, Success);
status =
Search(c_segment_2, c_plan, c_ph_group, {}, 1L << 63, &c_search_res_2);
Search({}, c_segment_2, c_plan, c_ph_group, 1L << 63, &c_search_res_2);
ASSERT_EQ(status.error_code, Success);
std::vector<CSearchResult> results;
results.push_back(c_search_res_1);

6
internal/core/unittest/test_growing_index.cpp
View File

@ -12,14 +12,14 @@
#include <gtest/gtest.h>
#include "pb/plan.pb.h"
#include "pb/schema.pb.h"
#include "query/Plan.h"
#include "segcore/SegmentGrowing.h"
#include "segcore/SegmentGrowingImpl.h"
#include "pb/schema.pb.h"
#include "test_utils/DataGen.h"
#include "query/Plan.h"
using namespace milvus::segcore;
using namespace milvus;
using namespace milvus::segcore;
namespace pb = milvus::proto;
TEST(GrowingIndex, Correctness) {

6
internal/core/unittest/test_indexing.cpp
View File

@ -173,10 +173,14 @@ TEST(Indexing, BinaryBruteForce) {
query_data //
};
SearchInfo search_info;
search_info.topk_ = topk;
search_info.round_decimal_ = round_decimal;
search_info.metric_type_ = metric_type;
auto sub_result = query::BruteForceSearch(search_dataset,
bin_vec.data(),
N,
knowhere::Json(),
search_info,
nullptr,
DataType::VECTOR_BINARY);

6
internal/core/unittest/test_integer_overflow.cpp
View File

@ -9,13 +9,11 @@
// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
// or implied. See the License for the specific language governing permissions and limitations under the License
#include <boost/format.hpp>
#include <gtest/gtest.h>
#include <cstdint>
#include <memory>
#include <regex>
#include <vector>
#include <chrono>
#include "common/Types.h"
#include "query/Expr.h"
@ -25,10 +23,10 @@
#include "test_utils/DataGen.h"
using namespace milvus;
TEST(Expr, IntegerOverflow) {
using namespace milvus::query;
using namespace milvus::segcore;
TEST(Expr, IntegerOverflow) {
std::vector<std::tuple<std::string, std::function<bool(int8_t)>>> testcases = {
/////////////////////////////////////////////////////////// term
{

16
internal/core/unittest/test_query.cpp
View File

@ -77,8 +77,6 @@ TEST(Query, ParsePlaceholderGroup) {
}
TEST(Query, ExecWithPredicateLoader) {
using namespace milvus::query;
using namespace milvus::segcore;
auto schema = std::make_shared<Schema>();
schema->AddDebugField(
"fakevec", DataType::VECTOR_FLOAT, 16, knowhere::metric::L2);
@ -161,8 +159,6 @@ TEST(Query, ExecWithPredicateLoader) {
}
TEST(Query, ExecWithPredicateSmallN) {
using namespace milvus::query;
using namespace milvus::segcore;
auto schema = std::make_shared<Schema>();
schema->AddDebugField(
"fakevec", DataType::VECTOR_FLOAT, 7, knowhere::metric::L2);
@ -222,8 +218,6 @@ TEST(Query, ExecWithPredicateSmallN) {
}
TEST(Query, ExecWithPredicate) {
using namespace milvus::query;
using namespace milvus::segcore;
auto schema = std::make_shared<Schema>();
schema->AddDebugField(
"fakevec", DataType::VECTOR_FLOAT, 16, knowhere::metric::L2);
@ -306,8 +300,6 @@ TEST(Query, ExecWithPredicate) {
}
TEST(Query, ExecTerm) {
using namespace milvus::query;
using namespace milvus::segcore;
auto schema = std::make_shared<Schema>();
schema->AddDebugField(
"fakevec", DataType::VECTOR_FLOAT, 16, knowhere::metric::L2);
@ -365,8 +357,6 @@ TEST(Query, ExecTerm) {
}
TEST(Query, ExecEmpty) {
using namespace milvus::query;
using namespace milvus::segcore;
auto schema = std::make_shared<Schema>();
schema->AddDebugField("age", DataType::FLOAT);
schema->AddDebugField(
@ -406,8 +396,6 @@ TEST(Query, ExecEmpty) {
}
TEST(Query, ExecWithoutPredicateFlat) {
using namespace milvus::query;
using namespace milvus::segcore;
auto schema = std::make_shared<Schema>();
schema->AddDebugField("fakevec", DataType::VECTOR_FLOAT, 16, std::nullopt);
schema->AddDebugField("age", DataType::FLOAT);
@ -448,8 +436,6 @@ TEST(Query, ExecWithoutPredicateFlat) {
}
TEST(Query, ExecWithoutPredicate) {
using namespace milvus::query;
using namespace milvus::segcore;
auto schema = std::make_shared<Schema>();
schema->AddDebugField(
"fakevec", DataType::VECTOR_FLOAT, 16, knowhere::metric::L2);
@ -704,8 +690,6 @@ TEST(Query, FillSegment) {
}
TEST(Query, ExecWithPredicateBinary) {
using namespace milvus::query;
using namespace milvus::segcore;
auto schema = std::make_shared<Schema>();
auto vec_fid = schema->AddDebugField(
"fakevec", DataType::VECTOR_BINARY, 512, knowhere::metric::JACCARD);

20
internal/core/unittest/test_sealed.cpp
View File

@ -9,32 +9,32 @@
// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
// or implied. See the License for the specific language governing permissions and limitations under the License
#include <gtest/gtest.h>
#include <boost/format.hpp>
#include <gtest/gtest.h>
#include "common/Types.h"
#include "segcore/SegmentSealedImpl.h"
#include "test_utils/DataGen.h"
#include "test_utils/storage_test_utils.h"
#include "common/Tracer.h"
#include "index/IndexFactory.h"
#include "storage/Util.h"
#include "knowhere/version.h"
#include "segcore/SegmentSealedImpl.h"
#include "storage/ChunkCacheSingleton.h"
#include "storage/RemoteChunkManagerSingleton.h"
#include "storage/MinioChunkManager.h"
#include "storage/RemoteChunkManagerSingleton.h"
#include "storage/Util.h"
#include "test_utils/DataGen.h"
#include "test_utils/indexbuilder_test_utils.h"
#include "test_utils/storage_test_utils.h"
using namespace milvus;
using namespace milvus::query;
using namespace milvus::segcore;
using milvus::segcore::LoadIndexInfo;
const int64_t ROW_COUNT = 10 * 1000;
const int64_t BIAS = 4200;
TEST(Sealed, without_predicate) {
using namespace milvus::query;
using namespace milvus::segcore;
auto schema = std::make_shared<Schema>();
auto dim = 16;
auto topK = 5;
@ -143,8 +143,6 @@ TEST(Sealed, without_predicate) {
}
TEST(Sealed, with_predicate) {
using namespace milvus::query;
using namespace milvus::segcore;
auto schema = std::make_shared<Schema>();
auto dim = 16;
auto topK = 5;
@ -259,8 +257,6 @@ TEST(Sealed, with_predicate) {
}
TEST(Sealed, with_predicate_filter_all) {
using namespace milvus::query;
using namespace milvus::segcore;
auto schema = std::make_shared<Schema>();
auto dim = 16;
auto topK = 5;

41
internal/core/unittest/test_string_expr.cpp
View File

@ -9,23 +9,26 @@
// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
// or implied. See the License for the specific language governing permissions and limitations under the License
#include <boost/format.hpp>
#include <gtest/gtest.h>
#include <memory>
#include <boost/format.hpp>
#include <regex>
#include "common/Tracer.h"
#include "pb/plan.pb.h"
#include "query/Expr.h"
#include "query/generated/PlanNodeVisitor.h"
#include "query/PlanProto.h"
#include "query/SearchBruteForce.h"
#include "query/Utils.h"
#include "query/generated/ExecExprVisitor.h"
#include "query/generated/PlanNodeVisitor.h"
#include "segcore/SegmentGrowingImpl.h"
#include "test_utils/DataGen.h"
#include "test_utils/GenExprProto.h"
#include "query/PlanProto.h"
#include "query/Utils.h"
#include "query/SearchBruteForce.h"
using namespace milvus;
using namespace milvus::query;
using namespace milvus::segcore;
namespace {
auto
@ -237,9 +240,6 @@ GenStrPKSchema() {
} // namespace
TEST(StringExpr, Term) {
using namespace milvus::query;
using namespace milvus::segcore;
auto schema = GenTestSchema();
const auto& fvec_meta = schema->operator[](FieldName("fvec"));
const auto& str_meta = schema->operator[](FieldName("str"));
@ -301,9 +301,6 @@ TEST(StringExpr, Term) {
}
TEST(StringExpr, Compare) {
using namespace milvus::query;
using namespace milvus::segcore;
auto schema = GenTestSchema();
const auto& fvec_meta = schema->operator[](FieldName("fvec"));
const auto& str_meta = schema->operator[](FieldName("str"));
@ -419,9 +416,6 @@ TEST(StringExpr, Compare) {
}
TEST(StringExpr, UnaryRange) {
using namespace milvus::query;
using namespace milvus::segcore;
auto schema = GenTestSchema();
const auto& fvec_meta = schema->operator[](FieldName("fvec"));
const auto& str_meta = schema->operator[](FieldName("str"));
@ -518,9 +512,6 @@ TEST(StringExpr, UnaryRange) {
}
TEST(StringExpr, BinaryRange) {
using namespace milvus::query;
using namespace milvus::segcore;
auto schema = GenTestSchema();
const auto& fvec_meta = schema->operator[](FieldName("fvec"));
const auto& str_meta = schema->operator[](FieldName("str"));
@ -636,9 +627,6 @@ TEST(StringExpr, BinaryRange) {
}
TEST(AlwaysTrueStringPlan, SearchWithOutputFields) {
using namespace milvus::query;
using namespace milvus::segcore;
auto schema = GenStrPKSchema();
const auto& fvec_meta = schema->operator[](FieldName("fvec"));
const auto& str_meta = schema->operator[](FieldName("str"));
@ -671,18 +659,24 @@ TEST(AlwaysTrueStringPlan, SearchWithOutputFields) {
std::vector<const PlaceholderGroup*> ph_group_arr = {ph_group.get()};
MetricType metric_type = knowhere::metric::L2;
query::dataset::SearchDataset search_dataset{
knowhere::metric::L2, //
metric_type, //
num_queries, //
topk, //
round_decimal,
dim, //
query_ptr //
};
SearchInfo search_info;
search_info.topk_ = topk;
search_info.round_decimal_ = round_decimal;
search_info.metric_type_ = metric_type;
auto sub_result = BruteForceSearch(search_dataset,
vec_col.data(),
N,
knowhere::Json(),
search_info,
nullptr,
DataType::VECTOR_FLOAT);
@ -708,9 +702,6 @@ TEST(AlwaysTrueStringPlan, SearchWithOutputFields) {
}
TEST(AlwaysTrueStringPlan, QueryWithOutputFields) {
using namespace milvus::query;
using namespace milvus::segcore;
auto schema = GenStrPKSchema();
const auto& fvec_meta = schema->operator[](FieldName("fvec"));
const auto& str_meta = schema->operator[](FieldName("str"));

29
internal/core/unittest/test_tracer.cpp
View File

@ -25,7 +25,7 @@ TEST(Tracer, Init) {
auto config = std::make_shared<TraceConfig>();
config->exporter = "stdout";
config->nodeID = 1;
initTelementry(config.get());
initTelemetry(*config);
auto span = StartSpan("test");
Assert(span->IsRecording());
@ -33,7 +33,7 @@ TEST(Tracer, Init) {
config->exporter = "jaeger";
config->jaegerURL = "http://localhost:14268/api/traces";
config->nodeID = 1;
initTelementry(config.get());
initTelemetry(*config);
span = StartSpan("test");
Assert(span->IsRecording());
}
@ -42,28 +42,13 @@ TEST(Tracer, Span) {
auto config = std::make_shared<TraceConfig>();
config->exporter = "stdout";
config->nodeID = 1;
initTelementry(config.get());
initTelemetry(*config);
auto ctx = std::make_shared<TraceContext>();
ctx->traceID = new uint8_t[16]{0x01,
0x23,
0x45,
0x67,
0x89,
0xab,
0xcd,
0xef,
0xfe,
0xdc,
0xba,
0x98,
0x76,
0x54,
0x32,
0x10};
ctx->spanID =
new uint8_t[8]{0x01, 0x23, 0x45, 0x67, 0x89, 0xab, 0xcd, 0xef};
ctx->flag = 1;
ctx->traceID = new uint8_t[16]{0x01, 0x23, 0x45, 0x67, 0x89, 0xab, 0xcd, 0xef,
0xfe, 0xdc, 0xba, 0x98, 0x76, 0x54, 0x32, 0x10};
ctx->spanID = new uint8_t[8]{0x01, 0x23, 0x45, 0x67, 0x89, 0xab, 0xcd, 0xef};
ctx->traceFlags = 1;
auto span = StartSpan("test", ctx.get());
Assert(span->GetContext().trace_id() == trace::TraceId({ctx->traceID, 16}));

2
internal/querynodev2/segments/load_index_info.go
View File

@ -216,7 +216,7 @@ func (li *LoadIndexInfo) appendIndexData(ctx context.Context, indexKeys []string
traceCtx := C.CTraceContext{
traceID: (*C.uint8_t)(unsafe.Pointer(&traceID[0])),
spanID: (*C.uint8_t)(unsafe.Pointer(&spanID[0])),
flag: C.uchar(span.SpanContext().TraceFlags()),
traceFlags: (C.uint8_t)(span.SpanContext().TraceFlags()),
}
status = C.AppendIndexV2(traceCtx, li.cLoadIndexInfo)

12
internal/querynodev2/segments/segment.go
View File

@ -478,7 +478,7 @@ func (s *LocalSegment) Search(ctx context.Context, searchReq *SearchRequest) (*S
traceCtx := C.CTraceContext{
traceID: (*C.uint8_t)(unsafe.Pointer(&traceID[0])),
spanID: (*C.uint8_t)(unsafe.Pointer(&spanID[0])),
flag: C.uchar(span.SpanContext().TraceFlags()),
traceFlags: (C.uint8_t)(span.SpanContext().TraceFlags()),
}
hasIndex := s.ExistIndex(searchReq.searchFieldID)
@ -489,10 +489,10 @@ func (s *LocalSegment) Search(ctx context.Context, searchReq *SearchRequest) (*S
var status C.CStatus
GetSQPool().Submit(func() (any, error) {
tr := timerecord.NewTimeRecorder("cgoSearch")
status = C.Search(s.ptr,
status = C.Search(traceCtx,
s.ptr,
searchReq.plan.cSearchPlan,
searchReq.cPlaceholderGroup,
traceCtx,
C.uint64_t(searchReq.mvccTimestamp),
&searchResult.cSearchResult,
)
@ -532,7 +532,7 @@ func (s *LocalSegment) Retrieve(ctx context.Context, plan *RetrievePlan) (*segco
traceCtx := C.CTraceContext{
traceID: (*C.uint8_t)(unsafe.Pointer(&traceID[0])),
spanID: (*C.uint8_t)(unsafe.Pointer(&spanID[0])),
flag: C.uchar(span.SpanContext().TraceFlags()),
traceFlags: (C.uint8_t)(span.SpanContext().TraceFlags()),
}
maxLimitSize := paramtable.Get().QuotaConfig.MaxOutputSize.GetAsInt64()
@ -541,9 +541,9 @@ func (s *LocalSegment) Retrieve(ctx context.Context, plan *RetrievePlan) (*segco
GetSQPool().Submit(func() (any, error) {
ts := C.uint64_t(plan.Timestamp)
tr := timerecord.NewTimeRecorder("cgoRetrieve")
status = C.Retrieve(s.ptr,
status = C.Retrieve(traceCtx,
s.ptr,
plan.cRetrievePlan,
traceCtx,
ts,
&retrieveResult.cRetrieveResult,
C.int64_t(maxLimitSize))