add more event for segcore search(#26277) (#26688)

Signed-off-by: MrPresent-Han <chun.han@zilliz.com>

configs/milvus.yaml

@@ -625,7 +625,8 @@ trace:
   # Fractions >= 1 will always sample. Fractions < 0 are treated as zero.
   sampleFraction: 0
   jaeger:
-    url: # when exporter is jaeger should set the jaeger's URL
+    url: # "http://127.0.0.1:14268/api/traces"
+    # when exporter is jaeger should set the jaeger's URL
 
 autoIndex:
   params:

internal/core/src/common/Tracer.cpp

@@ -47,10 +47,12 @@ initTelementry(TraceConfig* config) {
         opts.transport_format = jaeger::TransportFormat::kThriftHttp;
         opts.endpoint = config->jaegerURL;
         exporter = jaeger::JaegerExporterFactory::Create(opts);
+        LOG_SEGCORE_INFO_ << "init jaeger exporter, endpoint:" << opts.endpoint;
     } else if (config->exporter == "otlp") {
         auto opts = otlp::OtlpGrpcExporterOptions{};
         opts.endpoint = config->otlpEndpoint;
         exporter = otlp::OtlpGrpcExporterFactory::Create(opts);
+        LOG_SEGCORE_INFO_ << "init otlp exporter, endpoint:" << opts.endpoint;
     } else {
         LOG_SEGCORE_INFO_ << "Empty Trace";
         enable_trace = false;
@@ -111,6 +113,13 @@ GetRootSpan() {
     return nullptr;
 }
 
+void
+AddEvent(std::string event_label) {
+    if (enable_trace && local_span != nullptr) {
+        local_span->AddEvent(event_label);
+    }
+}
+
 void
 logTraceContext(const std::string& extended_info,
                 const std::shared_ptr<trace::Span> span) {
@@ -119,7 +128,7 @@ logTraceContext(const std::string& extended_info,
         span->GetContext().trace_id().ToLowerBase16(
             nostd::span<char, 2 * opentelemetry::trace::TraceId::kSize>{
                 &traceID[0], trace_id_size});
-        LOG_SEGCORE_INFO_ << extended_info << ", traceID:" << traceID;
+        LOG_SEGCORE_DEBUG_ << extended_info << ", traceID:" << traceID;
     }
 }
 

internal/core/src/common/Tracer.h

@@ -55,4 +55,7 @@ CloseRootSpan();
 std::shared_ptr<trace::Span>
 GetRootSpan();
 
+void
+AddEvent(std::string event_label);
+
 }  // namespace milvus::tracer

internal/core/src/index/VectorMemIndex.cpp

@@ -44,6 +44,7 @@
 #include "storage/ThreadPools.h"
 #include "storage/Util.h"
 #include "utils/File.h"
+#include "common/Tracer.h"
 
 namespace milvus::index {
 
@@ -304,17 +305,23 @@ VectorMemIndex::Query(const DatasetPtr dataset,
                                       search_conf[RANGE_FILTER],
                                       GetMetricType());
             }
+            milvus::tracer::AddEvent("start_knowhere_index_range_search");
             auto res = index_.RangeSearch(*dataset, search_conf, bitset);
+            milvus::tracer::AddEvent("finish_knowhere_index_range_search");
             if (!res.has_value()) {
                 PanicCodeInfo(ErrorCodeEnum::UnexpectedError,
                               fmt::format("failed to range search: {}: {}",
                                           KnowhereStatusString(res.error()),
                                           res.what()));
             }
-            return ReGenRangeSearchResult(
+            auto result = ReGenRangeSearchResult(
                 res.value(), topk, num_queries, GetMetricType());
+            milvus::tracer::AddEvent("finish_ReGenRangeSearchResult");
+            return result;
         } else {
+            milvus::tracer::AddEvent("start_knowhere_index_search");
             auto res = index_.Search(*dataset, search_conf, bitset);
+            milvus::tracer::AddEvent("finish_knowhere_index_search");
             if (!res.has_value()) {
                 PanicCodeInfo(ErrorCodeEnum::UnexpectedError,
                               fmt::format("failed to search: {}: {}",

internal/core/src/index/VectorMemNMIndex.cpp

@@ -23,6 +23,7 @@
 #include "knowhere/factory.h"
 #include "knowhere/comp/time_recorder.h"
 #define RAW_DATA "RAW_DATA"
+#include "common/Tracer.h"
 
 namespace milvus::index {
 
@@ -87,7 +88,6 @@ VectorMemNMIndex::Query(const DatasetPtr dataset,
     // load -> query, raw data has been loaded
     // build -> query, this case just for test, should load raw data before query
     std::call_once(raw_data_loaded_, load_raw_data_closure);
-
     return VectorMemIndex::Query(dataset, search_info, bitset);
 }
 
@@ -127,6 +127,7 @@ VectorMemNMIndex::LoadRawData() {
         PanicCodeInfo(
             ErrorCodeEnum::UnexpectedError,
             "failed to Deserialize index, " + KnowhereStatusString(stat));
+    milvus::tracer::AddEvent("VectorMemNMIndex_Loaded_RawData");
 }
 
 }  // namespace milvus::index

internal/core/src/query/SearchBruteForce.cpp

@@ -15,6 +15,7 @@
 #include "common/Consts.h"
 #include "common/RangeSearchHelper.h"
 #include "common/Utils.h"
+#include "common/Tracer.h"
 #include "SearchBruteForce.h"
 #include "SubSearchResult.h"
 #include "knowhere/comp/brute_force.h"
@@ -71,7 +72,7 @@ BruteForceSearch(const dataset::SearchDataset& dataset,
             }
             auto res = knowhere::BruteForce::RangeSearch(
                 base_dataset, query_dataset, config, bitset);
-
+            milvus::tracer::AddEvent("knowhere_finish_BruteForce_RangeSearch");
             if (!res.has_value()) {
                 PanicCodeInfo(ErrorCodeEnum::UnexpectedError,
                               fmt::format("failed to range search: {}: {}",
@@ -80,6 +81,7 @@ BruteForceSearch(const dataset::SearchDataset& dataset,
             }
             auto result = ReGenRangeSearchResult(
                 res.value(), topk, nq, dataset.metric_type);
+            milvus::tracer::AddEvent("ReGenRangeSearchResult");
             std::copy_n(
                 GetDatasetIDs(result), nq * topk, sub_result.get_seg_offsets());
             std::copy_n(GetDatasetDistance(result),
@@ -93,7 +95,8 @@ BruteForceSearch(const dataset::SearchDataset& dataset,
                 sub_result.mutable_distances().data(),
                 config,
                 bitset);
-
+            milvus::tracer::AddEvent(
+                "knowhere_finish_BruteForce_SearchWithBuf");
             if (stat != knowhere::Status::success) {
                 throw std::invalid_argument("invalid metric type, " +
                                             KnowhereStatusString(stat));

internal/core/src/query/SearchOnGrowing.cpp

@@ -12,6 +12,7 @@
 #include <cstddef>
 #include "common/BitsetView.h"
 #include "common/QueryInfo.h"
+#include "common/Tracer.h"
 #include "SearchOnGrowing.h"
 #include "query/SearchBruteForce.h"
 #include "query/SearchOnIndex.h"
@@ -123,7 +124,6 @@ SearchOnGrowing(const segcore::SegmentGrowingImpl& segment,
                                            size_per_chunk,
                                            info.search_params_,
                                            sub_view);
-
             // convert chunk uid to segment uid
             for (auto& x : sub_qr.mutable_seg_offsets()) {
                 if (x != -1) {

internal/core/src/segcore/SegmentInterface.cpp

@@ -15,6 +15,7 @@
 
 #include "Utils.h"
 #include "common/SystemProperty.h"
+#include "common/Tracer.h"
 #include "common/Types.h"
 #include "query/generated/ExecPlanNodeVisitor.h"
 
@@ -67,6 +68,7 @@ SegmentInternalInterface::Search(
     const query::Plan* plan,
     const query::PlaceholderGroup* placeholder_group) const {
     std::shared_lock lck(mutex_);
+    milvus::tracer::AddEvent("obtained_segment_lock_mutex");
     check_search(plan);
     query::ExecPlanNodeVisitor visitor(*this, 1L << 63, placeholder_group);
     auto results = std::make_unique<SearchResult>();

internal/core/src/segcore/SegmentSealedImpl.cpp

@@ -37,6 +37,7 @@
 #include "storage/Util.h"
 #include "storage/ThreadPools.h"
 #include "utils/File.h"
+#include "common/Tracer.h"
 
 namespace milvus::segcore {
 
@@ -572,6 +573,7 @@ SegmentSealedImpl::vector_search(SearchInfo& search_info,
                                    query_count,
                                    bitset,
                                    output);
+        milvus::tracer::AddEvent("finish_searching_vector_index");
     } else {
         AssertInfo(
             get_bit(field_data_ready_bitset_, field_id),
@@ -587,6 +589,7 @@ SegmentSealedImpl::vector_search(SearchInfo& search_info,
                               row_count,
                               bitset,
                               output);
+        milvus::tracer::AddEvent("finish_searching_vector_data");
     }
 }
 

internal/core/src/segcore/segment_c.cpp

@@ -79,10 +79,6 @@ Search(CSegmentInterface c_segment,
         auto ctx = milvus::tracer::TraceContext{
             c_trace.traceID, c_trace.spanID, c_trace.flag};
         auto span = milvus::tracer::StartSpan("SegCoreSearch", &ctx);
-        milvus::tracer::logTraceContext(
-            "SegCore_SegmentSearch_SegmentID:" +
-                std::to_string(segment->get_segment_id()),
-            span);
         milvus::tracer::SetRootSpan(span);
         auto search_result = segment->Search(plan, phg_ptr);
         if (!milvus::PositivelyRelated(