Format and and extra parameters for checking

Signed-off-by: FluorineDog <guilin.gou@zilliz.com>

cmd/master/main.go

@@ -32,7 +32,7 @@ func main() {
 	etcdPort, _ := gparams.GParams.Load("etcd.port")
 	etcdAddr := etcdAddress + ":" + etcdPort
 	etcdRootPath, _ := gparams.GParams.Load("etcd.rootpath")
-	svr, err := master.CreateServer(ctx, etcdRootPath, etcdRootPath, []string{etcdAddr})
+	svr, err := master.CreateServer(ctx, etcdRootPath, etcdRootPath, etcdRootPath, []string{etcdAddr})
 	if err != nil {
 		log.Print("create server failed", zap.Error(err))
 	}

internal/core/src/common/FieldMeta.h

@@ -1,25 +1,15 @@
 #pragma once
-
-#include <vector>
-#include <assert.h>
-#include <stdexcept>
-
 #include "utils/Types.h"
-// #include "knowhere/index/Index.h"
 #include "utils/Status.h"
 #include "utils/EasyAssert.h"
 
-namespace milvus::segcore {
+#include <stdexcept>
+namespace milvus {
+
 using Timestamp = uint64_t;  // TODO: use TiKV-like timestamp
 using engine::DataType;
 using engine::FieldElementType;
 
-struct RowBasedRawData {
-    void* raw_data;      // schema
-    int sizeof_per_row;  // alignment
-    int64_t count;
-};
-
 inline int
 field_sizeof(DataType data_type, int dim = 1) {
     switch (data_type) {
@@ -126,100 +116,4 @@ struct FieldMeta {
     DataType type_ = DataType::NONE;
     int dim_ = 1;
 };
-
-class Schema {
- public:
-    void
-    AddField(std::string_view field_name, DataType data_type, int dim = 1) {
-        auto field_meta = FieldMeta(field_name, data_type, dim);
-        this->AddField(std::move(field_meta));
-    }
-
-    void
-    AddField(FieldMeta field_meta) {
-        auto offset = fields_.size();
-        fields_.emplace_back(field_meta);
-        offsets_.emplace(field_meta.get_name(), offset);
-        auto field_sizeof = field_meta.get_sizeof();
-        sizeof_infos_.push_back(field_sizeof);
-        total_sizeof_ += field_sizeof;
-    }
-
-    auto
-    begin() {
-        return fields_.begin();
-    }
-
-    auto
-    end() {
-        return fields_.end();
-    }
-    auto
-    begin() const {
-        return fields_.begin();
-    }
-
-    auto
-    end() const {
-        return fields_.end();
-    }
-
-    int
-    size() const {
-        return fields_.size();
-    }
-
-    const FieldMeta&
-    operator[](int field_index) const {
-        Assert(field_index >= 0);
-        Assert(field_index < fields_.size());
-        return fields_[field_index];
-    }
-
-    auto
-    get_total_sizeof() const {
-        return total_sizeof_;
-    }
-
-    const std::vector<int>&
-    get_sizeof_infos() {
-        return sizeof_infos_;
-    }
-
-    std::optional<int>
-    get_offset(const std::string& field_name) {
-        if (!offsets_.count(field_name)) {
-            return std::nullopt;
-        } else {
-            return offsets_[field_name];
-        }
-    }
-
-    const std::vector<FieldMeta>&
-    get_fields() {
-        return fields_;
-    }
-
-    const FieldMeta&
-    operator[](const std::string& field_name) const {
-        auto offset_iter = offsets_.find(field_name);
-        AssertInfo(offset_iter != offsets_.end(), "Cannot found field_name: " + field_name);
-        auto offset = offset_iter->second;
-        return (*this)[offset];
-    }
-
- private:
-    // this is where data holds
-    std::vector<FieldMeta> fields_;
-
- private:
-    // a mapping for random access
-    std::unordered_map<std::string, int> offsets_;
-    std::vector<int> sizeof_infos_;
-    int total_sizeof_ = 0;
-};
-
-using SchemaPtr = std::shared_ptr<Schema>;
-using idx_t = int64_t;
-
-}  // namespace milvus::segcore
+}  // namespace milvus

internal/core/src/common/Schema.h

@@ -0,0 +1,102 @@
+#pragma once
+#include "FieldMeta.h"
+#include <vector>
+
+namespace milvus {
+
+class Schema {
+ public:
+    void
+    AddField(std::string_view field_name, DataType data_type, int dim = 1) {
+        auto field_meta = FieldMeta(field_name, data_type, dim);
+        this->AddField(std::move(field_meta));
+    }
+
+    void
+    AddField(FieldMeta field_meta) {
+        auto offset = fields_.size();
+        fields_.emplace_back(field_meta);
+        offsets_.emplace(field_meta.get_name(), offset);
+        auto field_sizeof = field_meta.get_sizeof();
+        sizeof_infos_.push_back(field_sizeof);
+        total_sizeof_ += field_sizeof;
+    }
+
+    auto
+    begin() {
+        return fields_.begin();
+    }
+
+    auto
+    end() {
+        return fields_.end();
+    }
+    auto
+    begin() const {
+        return fields_.begin();
+    }
+
+    auto
+    end() const {
+        return fields_.end();
+    }
+
+    int
+    size() const {
+        return fields_.size();
+    }
+
+    const FieldMeta&
+    operator[](int field_index) const {
+        Assert(field_index >= 0);
+        Assert(field_index < fields_.size());
+        return fields_[field_index];
+    }
+
+    auto
+    get_total_sizeof() const {
+        return total_sizeof_;
+    }
+
+    const std::vector<int>&
+    get_sizeof_infos() {
+        return sizeof_infos_;
+    }
+
+    std::optional<int>
+    get_offset(const std::string& field_name) {
+        if (!offsets_.count(field_name)) {
+            return std::nullopt;
+        } else {
+            return offsets_[field_name];
+        }
+    }
+
+    const std::vector<FieldMeta>&
+    get_fields() {
+        return fields_;
+    }
+
+    const FieldMeta&
+    operator[](const std::string& field_name) const {
+        auto offset_iter = offsets_.find(field_name);
+        AssertInfo(offset_iter != offsets_.end(), "Cannot found field_name: " + field_name);
+        auto offset = offset_iter->second;
+        return (*this)[offset];
+    }
+
+ private:
+    // this is where data holds
+    std::vector<FieldMeta> fields_;
+
+ private:
+    // a mapping for random access
+    std::unordered_map<std::string, int> offsets_;
+    std::vector<int> sizeof_infos_;
+    int total_sizeof_ = 0;
+};
+
+using SchemaPtr = std::shared_ptr<Schema>;
+using idx_t = int64_t;
+
+}  // namespace milvus

internal/core/src/index/knowhere/knowhere/common/Config.h

@@ -16,7 +16,7 @@
 namespace milvus {
 namespace knowhere {
 
-using Config = milvus::json;
+using Config = milvus::Json;
 
 }  // namespace knowhere
 }  // namespace milvus

internal/core/src/query/CMakeLists.txt

@@ -6,7 +6,6 @@ set(MILVUS_QUERY_SRCS
         visitors/ShowPlanNodeVisitor.cpp
         visitors/ExecPlanNodeVisitor.cpp
         visitors/ShowExprVisitor.cpp
-        Parser.cpp
         Plan.cpp
         )
 add_library(milvus_query ${MILVUS_QUERY_SRCS})

internal/core/src/query/Expr.h

@@ -4,7 +4,7 @@
 #include <any>
 #include <string>
 #include <optional>
-#include "segcore/SegmentDefs.h"
+#include "common/Schema.h"
 
 namespace milvus::query {
 class ExprVisitor;
@@ -60,7 +60,7 @@ using FieldId = std::string;
 
 struct TermExpr : Expr {
     FieldId field_id_;
-    segcore::DataType data_type_;
+    DataType data_type_;
     // std::vector<std::any> terms_;
 
  protected:
@@ -74,7 +74,7 @@ struct TermExpr : Expr {
 
 struct RangeExpr : Expr {
     FieldId field_id_;
-    segcore::DataType data_type_;
+    DataType data_type_;
     // std::vector<std::tuple<OpType, std::any>> conditions_;
  protected:
     // prevent accidential instantiation

internal/core/src/query/Plan.cpp

@@ -9,7 +9,7 @@
 namespace milvus::query {
 
 static std::unique_ptr<VectorPlanNode>
-CreateVec(const std::string& field_name, const json& vec_info) {
+CreateVecNode(const std::string& field_name, const Json& vec_info) {
     // TODO add binary info
     auto vec_node = std::make_unique<FloatVectorANNS>();
     auto topK = vec_info["topk"];
@@ -35,7 +35,7 @@ CreatePlanImplNaive(const std::string& dsl_str) {
                 auto iter = pack["vector"].begin();
                 auto key = iter.key();
                 auto& body = iter.value();
-                plan->plan_node_ = CreateVec(key, body);
+                plan->plan_node_ = CreateVecNode(key, body);
                 return plan;
             }
         }
@@ -44,13 +44,18 @@ CreatePlanImplNaive(const std::string& dsl_str) {
         auto iter = bool_dsl["vector"].begin();
         auto key = iter.key();
         auto& body = iter.value();
-        plan->plan_node_ = CreateVec(key, body);
+        plan->plan_node_ = CreateVecNode(key, body);
         return plan;
     } else {
         PanicInfo("Unsupported DSL: vector node not detected");
     }
 }
 
+static std::unique_ptr<Plan>
+CreateRangeNode(const Json& range_group) {
+    return nullptr;
+}
+
 void
 CheckNull(const Json& json) {
     Assert(!json.is_null());
@@ -82,13 +87,15 @@ class PlanParser {
 };
 
 std::unique_ptr<Plan>
-CreatePlan(const std::string& dsl_str) {
+CreatePlan(const Schema& schema, const std::string& dsl_str) {
+    (void)schema;
     auto plan = CreatePlanImplNaive(dsl_str);
     return plan;
 }
 
 std::unique_ptr<PlaceholderGroup>
-ParsePlaceholderGroup(const std::string& blob) {
+ParsePlaceholderGroup(const Plan* plan, const std::string& blob) {
+    (void)plan;
     namespace ser = milvus::proto::service;
     auto result = std::make_unique<PlaceholderGroup>();
     ser::PlaceholderGroup ph_group;

internal/core/src/query/Plan.h

@@ -1,6 +1,7 @@
 #pragma once
 #include <memory>
 #include <string_view>
+#include "common/Schema.h"
 
 namespace milvus::query {
 // NOTE: APIs for C wrapper
@@ -10,10 +11,10 @@ struct Plan;
 struct PlaceholderGroup;
 
 std::unique_ptr<Plan>
-CreatePlan(const std::string& dsl);
+CreatePlan(const Schema& schema, const std::string& dsl);
 
 std::unique_ptr<PlaceholderGroup>
-ParsePlaceholderGroup(const std::string& placeholder_group_blob);
+ParsePlaceholderGroup(const Plan* plan, const std::string& placeholder_group_blob);
 
 int64_t
 GetNumOfQueries(const PlaceholderGroup*);

internal/core/src/query/deprecated/GeneralQuery.h

@@ -49,7 +49,7 @@ struct QueryColumn {
 };
 
 struct TermQuery {
-    milvus::json json_obj;
+    milvus::Json json_obj;
     //    std::string field_name;
     //    std::vector<uint8_t> field_value;
     //    float boost;
@@ -62,7 +62,7 @@ struct CompareExpr {
 };
 
 struct RangeQuery {
-    milvus::json json_obj;
+    milvus::Json json_obj;
     //    std::string field_name;
     //    std::vector<CompareExpr> compare_expr;
     //    float boost;
@@ -77,7 +77,7 @@ struct VectorRecord {
 
 struct VectorQuery {
     std::string field_name;
-    milvus::json extra_params = {};
+    milvus::Json extra_params = {};
     int64_t topk;
     int64_t nq;
     std::string metric_type = "";

internal/core/src/query/deprecated/Parser.cpp

@@ -5,7 +5,7 @@
 namespace milvus::wtf {
 using google::protobuf::RepeatedField;
 using google::protobuf::RepeatedPtrField;
-#if 0
+#if 1
 #if 0
 void
 CopyRowRecords(const RepeatedPtrField<proto::service::PlaceholderValue>& grpc_records,
@@ -51,27 +51,27 @@ CopyRowRecords(const RepeatedPtrField<proto::service::PlaceholderValue>& grpc_re
 #endif
 
 Status
-ProcessLeafQueryJson(const milvus::json& query_json, query_old::BooleanQueryPtr& query, std::string& field_name) {
-#if 0
+ProcessLeafQueryJson(const milvus::Json& query_json, query_old::BooleanQueryPtr& query, std::string& field_name) {
+#if 1
     if (query_json.contains("term")) {
         auto leaf_query = std::make_shared<query_old::LeafQuery>();
         auto term_query = std::make_shared<query_old::TermQuery>();
-        milvus::json json_obj = query_json["term"];
+        milvus::Json json_obj = query_json["term"];
         JSON_NULL_CHECK(json_obj);
         JSON_OBJECT_CHECK(json_obj);
         term_query->json_obj = json_obj;
-        milvus::json::iterator json_it = json_obj.begin();
+        milvus::Json::iterator json_it = json_obj.begin();
         field_name = json_it.key();
         leaf_query->term_query = term_query;
         query->AddLeafQuery(leaf_query);
     } else if (query_json.contains("range")) {
         auto leaf_query = std::make_shared<query_old::LeafQuery>();
         auto range_query = std::make_shared<query_old::RangeQuery>();
-        milvus::json json_obj = query_json["range"];
+        milvus::Json json_obj = query_json["range"];
         JSON_NULL_CHECK(json_obj);
         JSON_OBJECT_CHECK(json_obj);
         range_query->json_obj = json_obj;
-        milvus::json::iterator json_it = json_obj.begin();
+        milvus::Json::iterator json_it = json_obj.begin();
         field_name = json_it.key();
 
         leaf_query->range_query = range_query;
@@ -91,7 +91,7 @@ ProcessLeafQueryJson(const milvus::json& query_json, query_old::BooleanQueryPtr&
 }
 
 Status
-ProcessBooleanQueryJson(const milvus::json& query_json,
+ProcessBooleanQueryJson(const milvus::Json& query_json,
                         query_old::BooleanQueryPtr& boolean_query,
                         query_old::QueryPtr& query_ptr) {
 #if 1
@@ -173,12 +173,12 @@ ProcessBooleanQueryJson(const milvus::json& query_json,
 
 Status
 DeserializeJsonToBoolQuery(const google::protobuf::RepeatedPtrField<::milvus::grpc::VectorParam>& vector_params,
-     const std::string& dsl_string,
-     query_old::BooleanQueryPtr& boolean_query,
-     query_old::QueryPtr& query_ptr) {
+                           const std::string& dsl_string,
+                           query_old::BooleanQueryPtr& boolean_query,
+                           query_old::QueryPtr& query_ptr) {
 #if 1
     try {
-        milvus::json dsl_json = json::parse(dsl_string);
+        milvus::Json dsl_json = Json::parse(dsl_string);
 
         if (dsl_json.empty()) {
             return Status{SERVER_INVALID_ARGUMENT, "Query dsl is null"};
@@ -189,16 +189,16 @@ DeserializeJsonToBoolQuery(const google::protobuf::RepeatedPtrField<::milvus::gr
         }
         for (const auto& vector_param : vector_params) {
             const std::string& vector_string = vector_param.json();
-            milvus::json vector_json = json::parse(vector_string);
-            milvus::json::iterator it = vector_json.begin();
+            milvus::Json vector_json = Json::parse(vector_string);
+            milvus::Json::iterator it = vector_json.begin();
             std::string placeholder = it.key();
 
             auto vector_query = std::make_shared<query_old::VectorQuery>();
-            milvus::json::iterator vector_param_it = it.value().begin();
+            milvus::Json::iterator vector_param_it = it.value().begin();
             if (vector_param_it != it.value().end()) {
                 const std::string& field_name = vector_param_it.key();
                 vector_query->field_name = field_name;
-                milvus::json param_json = vector_param_it.value();
+                milvus::Json param_json = vector_param_it.value();
                 int64_t topk = param_json["topk"];
                 // STATUS_CHECK(server::ValidateSearchTopk(topk));
                 vector_query->topk = topk;

internal/core/src/query/generated/ExecPlanNodeVisitor.h

@@ -17,9 +17,7 @@ class ExecPlanNodeVisitor : PlanNodeVisitor {
 
  public:
     using RetType = segcore::QueryResult;
-    ExecPlanNodeVisitor(segcore::SegmentBase& segment,
-                        segcore::Timestamp timestamp,
-                        const PlaceholderGroup& placeholder_group)
+    ExecPlanNodeVisitor(segcore::SegmentBase& segment, Timestamp timestamp, const PlaceholderGroup& placeholder_group)
         : segment_(segment), timestamp_(timestamp), placeholder_group_(placeholder_group) {
     }
     // using RetType = nlohmann::json;
@@ -37,7 +35,7 @@ class ExecPlanNodeVisitor : PlanNodeVisitor {
  private:
     // std::optional<RetType> ret_;
     segcore::SegmentBase& segment_;
-    segcore::Timestamp timestamp_;
+    Timestamp timestamp_;
     const PlaceholderGroup& placeholder_group_;
 
     std::optional<RetType> ret_;

internal/core/src/query/visitors/ExecPlanNodeVisitor.cpp

@@ -13,9 +13,7 @@ namespace impl {
 class ExecPlanNodeVisitor : PlanNodeVisitor {
  public:
     using RetType = segcore::QueryResult;
-    ExecPlanNodeVisitor(segcore::SegmentBase& segment,
-                        segcore::Timestamp timestamp,
-                        const PlaceholderGroup& placeholder_group)
+    ExecPlanNodeVisitor(segcore::SegmentBase& segment, Timestamp timestamp, const PlaceholderGroup& placeholder_group)
         : segment_(segment), timestamp_(timestamp), placeholder_group_(placeholder_group) {
     }
     // using RetType = nlohmann::json;
@@ -33,7 +31,7 @@ class ExecPlanNodeVisitor : PlanNodeVisitor {
  private:
     // std::optional<RetType> ret_;
     segcore::SegmentBase& segment_;
-    segcore::Timestamp timestamp_;
+    Timestamp timestamp_;
     const PlaceholderGroup& placeholder_group_;
 
     std::optional<RetType> ret_;

internal/core/src/query/visitors/ShowExprVisitor.cpp

@@ -100,8 +100,7 @@ TermExtract(const TermExpr& expr_raw) {
 void
 ShowExprVisitor::visit(TermExpr& expr) {
     Assert(!ret_.has_value());
-    Assert(segcore::field_is_vector(expr.data_type_) == false);
-    using segcore::DataType;
+    Assert(field_is_vector(expr.data_type_) == false);
     auto terms = [&] {
         switch (expr.data_type_) {
             case DataType::INT8:
@@ -125,7 +124,7 @@ ShowExprVisitor::visit(TermExpr& expr) {
 
     Json res{{"expr_type", "Term"},
              {"field_id", expr.field_id_},
-             {"data_type", segcore::datatype_name(expr.data_type_)},
+             {"data_type", datatype_name(expr.data_type_)},
              {"terms", std::move(terms)}};
 
     ret_ = res;
@@ -142,8 +141,7 @@ CondtionExtract(const RangeExpr& expr_raw) {
 void
 ShowExprVisitor::visit(RangeExpr& expr) {
     Assert(!ret_.has_value());
-    Assert(segcore::field_is_vector(expr.data_type_) == false);
-    using segcore::DataType;
+    Assert(field_is_vector(expr.data_type_) == false);
     auto conditions = [&] {
         switch (expr.data_type_) {
             case DataType::BOOL:
@@ -167,7 +165,7 @@ ShowExprVisitor::visit(RangeExpr& expr) {
 
     Json res{{"expr_type", "Range"},
              {"field_id", expr.field_id_},
-             {"data_type", segcore::datatype_name(expr.data_type_)},
+             {"data_type", datatype_name(expr.data_type_)},
              {"conditions", std::move(conditions)}};
 }
 }  // namespace milvus::query

internal/core/src/segcore/Collection.h

@@ -1,6 +1,6 @@
 #pragma once
 
-#include "SegmentDefs.h"
+#include "common/Schema.h"
 #include "IndexMeta.h"
 
 namespace milvus::segcore {

internal/core/src/segcore/DeletedRecord.h

@@ -1,8 +1,8 @@
 #pragma once
 
 #include "AckResponder.h"
-#include "SegmentDefs.h"
-#include "knowhere//index/vector_index/IndexIVF.h"
+#include "common/Schema.h"
+#include "knowhere/index/vector_index/IndexIVF.h"
 #include <memory>
 
 namespace milvus::segcore {

internal/core/src/segcore/IndexMeta.h

@@ -2,9 +2,9 @@
 //
 //#include <shared_mutex>
 //
-//#include "SegmentDefs.h"
+//#include "common/Schema.h"
 // #include "segcore/SegmentBase.h"
-#include "segcore/SegmentDefs.h"
+#include "common/Schema.h"
 #include "knowhere/index/IndexType.h"
 #include "knowhere/common/Config.h"
 #include <map>

internal/core/src/segcore/IndexingEntry.h

@@ -1,7 +1,7 @@
 #pragma once
 #include "AckResponder.h"
 #include <tbb/concurrent_vector.h>
-#include "SegmentDefs.h"
+#include "common/Schema.h"
 #include <optional>
 #include "InsertRecord.h"
 #include <knowhere/index/vector_index/IndexIVF.h>

internal/core/src/segcore/InsertRecord.h

@@ -1,5 +1,5 @@
 #pragma once
-#include "SegmentDefs.h"
+#include "common/Schema.h"
 #include "ConcurrentVector.h"
 #include "AckResponder.h"
 

internal/core/src/segcore/SegmentBase.h

@@ -3,9 +3,8 @@
 
 #include "IndexMeta.h"
 #include "utils/Types.h"
-#include "segcore/SegmentDefs.h"
-// #include "knowhere/index/Index.h"
-// #include "knowhere/index/IndexType.h"
+#include "common/Schema.h"
+
 #include "query/deprecated/GeneralQuery.h"
 #include "query/Plan.h"
 
@@ -14,6 +13,11 @@ namespace segcore {
 // using engine::DataChunk;
 // using engine::DataChunkPtr;
 using engine::QueryResult;
+struct RowBasedRawData {
+    void* raw_data;      // schema
+    int sizeof_per_row;  // alignment
+    int64_t count;
+};
 
 int
 TestABI();

internal/core/src/utils/Json.h

@@ -15,7 +15,7 @@
 
 namespace milvus {
 
-using json = nlohmann::json;
+using Json = nlohmann::json;
 
 #define JSON_NULL_CHECK(json)                                       \
     do {                                                            \

internal/core/unittest/test_expr.cpp

@@ -1,11 +1,12 @@
 #include <gtest/gtest.h>
-#include "query/Parser.h"
+#include "query/deprecated/Parser.h"
 #include "query/Expr.h"
 #include "query/PlanNode.h"
 #include "query/generated/ExprVisitor.h"
 #include "query/generated/PlanNodeVisitor.h"
 #include "test_utils/DataGen.h"
 #include "query/generated/ShowPlanNodeVisitor.h"
+using namespace milvus;
 
 TEST(Expr, Naive) {
     SUCCEED();

internal/core/unittest/test_query.cpp

@@ -1,5 +1,5 @@
 #include <gtest/gtest.h>
-#include "query/Parser.h"
+#include "query/deprecated/Parser.h"
 #include "query/Expr.h"
 #include "query/PlanNode.h"
 #include "query/generated/ExprVisitor.h"
@@ -9,6 +9,9 @@
 #include "query/generated/ExecPlanNodeVisitor.h"
 #include "query/PlanImpl.h"
 
+using namespace milvus;
+using namespace milvus::query;
+using namespace milvus::segcore;
 TEST(Query, Naive) {
     SUCCEED();
     using namespace milvus::wtf;
@@ -53,10 +56,11 @@ TEST(Query, Naive) {
 TEST(Query, ShowExecutor) {
     using namespace milvus::query;
     using namespace milvus::segcore;
+    using namespace milvus;
     auto node = std::make_unique<FloatVectorANNS>();
     auto schema = std::make_shared<Schema>();
-    int64_t num_queries = 100L;
     schema->AddField("fakevec", DataType::VECTOR_FLOAT, 16);
+    int64_t num_queries = 100L;
     auto raw_data = DataGen(schema, num_queries);
     auto& info = node->query_info_;
     info.metric_type_ = "L2";
@@ -94,7 +98,11 @@ TEST(Query, DSL) {
         ]
     }
 })";
-    auto plan = CreatePlan(dsl_string);
+
+    auto schema = std::make_shared<Schema>();
+    schema->AddField("fakevec", DataType::VECTOR_FLOAT, 16);
+
+    auto plan = CreatePlan(*schema, dsl_string);
     auto res = shower.call_child(*plan->plan_node_);
     std::cout << res.dump(4) << std::endl;
 
@@ -113,16 +121,33 @@ TEST(Query, DSL) {
         }
     }
 })";
-    auto plan2 = CreatePlan(dsl_string2);
+    auto plan2 = CreatePlan(*schema, dsl_string2);
     auto res2 = shower.call_child(*plan2->plan_node_);
     std::cout << res2.dump(4) << std::endl;
     ASSERT_EQ(res, res2);
 }
 
 TEST(Query, ParsePlaceholderGroup) {
-    using namespace milvus::query;
-    using namespace milvus::segcore;
     namespace ser = milvus::proto::service;
+    std::string dsl_string = R"(
+{
+    "bool": {
+        "vector": {
+            "Vec": {
+                "metric_type": "L2",
+                "params": {
+                    "nprobe": 10
+                },
+                "query": "$0",
+                "topk": 10
+            }
+        }
+    }
+})";
+
+    auto schema = std::make_shared<Schema>();
+    schema->AddField("fakevec", DataType::VECTOR_FLOAT, 16);
+    auto plan = CreatePlan(*schema, dsl_string);
     int num_queries = 10;
     int dim = 16;
     std::default_random_engine e;
@@ -131,22 +156,21 @@ TEST(Query, ParsePlaceholderGroup) {
     auto value = raw_group.add_placeholders();
     value->set_tag("$0");
     value->set_type(ser::PlaceholderType::VECTOR_FLOAT);
-    for(int i = 0; i < num_queries; ++i) {
+    for (int i = 0; i < num_queries; ++i) {
         std::vector<float> vec;
-        for(int d = 0; d < dim; ++d) {
+        for (int d = 0; d < dim; ++d) {
             vec.push_back(dis(e));
         }
         // std::string line((char*)vec.data(), (char*)vec.data() + vec.size() * sizeof(float));
         value->add_values(vec.data(), vec.size() * sizeof(float));
     }
     auto blob = raw_group.SerializeAsString();
-    //ser::PlaceholderGroup new_group;
-    //new_group.ParseFromString()
-    auto fuck = ParsePlaceholderGroup(blob);
-    int x = 1+1;
+    // ser::PlaceholderGroup new_group;
+    // new_group.ParseFromString()
+    auto placeholder = ParsePlaceholderGroup(plan.get(), blob);
+    int x = 1 + 1;
 }
 
-
 TEST(Query, Exec) {
     using namespace milvus::query;
     using namespace milvus::segcore;

internal/core/unittest/test_segcore.cpp

@@ -24,6 +24,7 @@
 using std::cin;
 using std::cout;
 using std::endl;
+using namespace milvus;
 
 namespace {
 auto

internal/core/unittest/test_utils/DataGen.h

@@ -1,5 +1,5 @@
 #pragma once
-#include "segcore/SegmentDefs.h"
+#include "common/Schema.h"
 #include <random>
 #include <memory>
 #include <cstring>

internal/master/collection_task_test.go

@@ -36,7 +36,7 @@ func TestMaster_CollectionTask(t *testing.T) {
 	_, err = etcdCli.Delete(ctx, "/test/root", clientv3.WithPrefix())
 	assert.Nil(t, err)
 
-	svr, err := CreateServer(ctx, "/test/root/kv", "/test/root/meta", []string{etcdAddr})
+	svr, err := CreateServer(ctx, "/test/root/kv", "/test/root/meta", "/test/root/meta/tso", []string{etcdAddr})
 	assert.Nil(t, err)
 	err = svr.Run(10002)
 	assert.Nil(t, err)

internal/master/grpc_service_test.go

@@ -34,7 +34,7 @@ func TestMaster_CreateCollection(t *testing.T) {
 	_, err = etcdCli.Delete(ctx, "/test/root", clientv3.WithPrefix())
 	assert.Nil(t, err)
 
-	svr, err := CreateServer(ctx, "/test/root/kv", "/test/root/meta", []string{etcdAddr})
+	svr, err := CreateServer(ctx, "/test/root/kv", "/test/root/meta", "/test/root/meta/tso", []string{etcdAddr})
 	assert.Nil(t, err)
 	err = svr.Run(10001)
 	assert.Nil(t, err)

internal/master/id/id.go

@@ -16,8 +16,8 @@ type GlobalIDAllocator struct {
 
 var allocator *GlobalIDAllocator
 
-func Init(etcdAddr []string, rootPath string) {
-	InitGlobalIDAllocator("idTimestamp", tsoutil.NewTSOKVBase(etcdAddr, rootPath, "gid"))
+func Init() {
+	InitGlobalIDAllocator("idTimestamp", tsoutil.NewTSOKVBase("gid"))
 }
 
 func InitGlobalIDAllocator(key string, base kvutil.Base) {

internal/master/id/id_test.go

@@ -17,14 +17,7 @@ func TestMain(m *testing.M) {
 	if err != nil {
 		panic(err)
 	}
-
-	etcdPort, err := gparams.GParams.Load("etcd.port")
-	if err != nil {
-		panic(err)
-	}
-	etcdAddr := "127.0.0.1:" + etcdPort
-
-	GIdAllocator = NewGlobalIDAllocator("idTimestamp", tsoutil.NewTSOKVBase([]string{etcdAddr}, "/test/root/kv", "gid"))
+	GIdAllocator = NewGlobalIDAllocator("idTimestamp", tsoutil.NewTSOKVBase("gid"))
 	exitCode := m.Run()
 	os.Exit(exitCode)
 }

internal/master/master.go

@@ -72,15 +72,15 @@ func newKVBase(kvRoot string, etcdAddr []string) *kv.EtcdKV {
 	return kvBase
 }
 
-func Init(etcdAddr []string, rootPath string) {
+func Init() {
 	rand.Seed(time.Now().UnixNano())
-	id.Init(etcdAddr, rootPath)
-	tso.Init(etcdAddr, rootPath)
+	id.Init()
+	tso.Init()
 }
 
 // CreateServer creates the UNINITIALIZED pd server with given configuration.
-func CreateServer(ctx context.Context, kvRootPath, metaRootPath string, etcdAddr []string) (*Master, error) {
-	Init(etcdAddr, kvRootPath)
+func CreateServer(ctx context.Context, kvRootPath string, metaRootPath, tsoRootPath string, etcdAddr []string) (*Master, error) {
+	Init()
 
 	etcdClient, err := clientv3.New(clientv3.Config{Endpoints: etcdAddr})
 	if err != nil {

internal/master/partition_task_test.go

@@ -38,7 +38,7 @@ func TestMaster_Partition(t *testing.T) {
 	assert.Nil(t, err)
 
 	port := 10000 + rand.Intn(1000)
-	svr, err := CreateServer(ctx, "/test/root/kv", "/test/root/meta", []string{etcdAddr})
+	svr, err := CreateServer(ctx, "/test/root/kv", "/test/root/meta", "/test/root/meta/tso", []string{etcdAddr})
 	assert.Nil(t, err)
 	err = svr.Run(int64(port))
 	assert.Nil(t, err)

internal/master/scheduler.go

@@ -1,8 +1,6 @@
 package master
 
-import (
-	"github.com/zilliztech/milvus-distributed/internal/master/id"
-)
+import "math/rand"
 
 type ddRequestScheduler struct {
 	reqQueue          chan task
@@ -23,6 +21,7 @@ func (rs *ddRequestScheduler) Enqueue(task task) error {
 	return nil
 }
 
+//TODO, allocGlobalID
 func allocGlobalID() (UniqueID, error) {
-	return id.AllocOne()
+	return rand.Int63(), nil
 }

internal/master/tso/global_allocator.go

@@ -37,8 +37,8 @@ type GlobalTSOAllocator struct {
 
 var allocator *GlobalTSOAllocator
 
-func Init(etcdAddr []string, rootPath string) {
-	InitGlobalTsoAllocator("timestamp", tsoutil.NewTSOKVBase(etcdAddr, rootPath, "tso"))
+func Init() {
+	InitGlobalTsoAllocator("timestamp", tsoutil.NewTSOKVBase("tso"))
 }
 
 func InitGlobalTsoAllocator(key string, base kvutil.Base) {

internal/master/tso/global_allocator_test.go

@@ -18,13 +18,7 @@ func TestMain(m *testing.M) {
 	if err != nil {
 		panic(err)
 	}
-	etcdPort, err := gparams.GParams.Load("etcd.port")
-	if err != nil {
-		panic(err)
-	}
-	etcdAddr := "127.0.0.1:" + etcdPort
-
-	GTsoAllocator = NewGlobalTSOAllocator("timestamp", tsoutil.NewTSOKVBase([]string{etcdAddr}, "/test/root/kv", "tso"))
+	GTsoAllocator = NewGlobalTSOAllocator("timestamp", tsoutil.NewTSOKVBase("tso"))
 
 	exitCode := m.Run()
 	os.Exit(exitCode)

internal/msgstream/msgstream_test.go

@@ -106,16 +106,6 @@ func getTsMsg(msgType MsgType, reqID UniqueID, hashValue int32) *TsMsg {
 			TimeTickMsg: timeTickResult,
 		}
 		tsMsg = timeTickMsg
-	case internalPb.MsgType_kQueryNodeSegStats:
-		queryNodeSegStats := internalPb.QueryNodeSegStats{
-			MsgType: internalPb.MsgType_kQueryNodeSegStats,
-			PeerID:  reqID,
-		}
-		queryNodeSegStatsMsg := &QueryNodeSegStatsMsg{
-			BaseMsg:           baseMsg,
-			QueryNodeSegStats: queryNodeSegStats,
-		}
-		tsMsg = queryNodeSegStatsMsg
 	}
 	return &tsMsg
 }
@@ -462,11 +452,24 @@ func TestStream_PulsarMsgStream_DefaultRepackFunc(t *testing.T) {
 	consumerChannels := []string{"insert1", "insert2"}
 	consumerSubName := "subInsert"
 
+	baseMsg := BaseMsg{
+		BeginTimestamp: 0,
+		EndTimestamp:   0,
+		HashValues:     []int32{1},
+	}
+
+	timeTickRequest := internalPb.TimeTickMsg{
+		MsgType:   internalPb.MsgType_kTimeTick,
+		PeerID:    int64(1),
+		Timestamp: uint64(1),
+	}
+	timeTick := &TimeTickMsg{
+		BaseMsg:     baseMsg,
+		TimeTickMsg: timeTickRequest,
+	}
+	var tsMsg TsMsg = timeTick
 	msgPack := MsgPack{}
-	msgPack.Msgs = append(msgPack.Msgs, getTsMsg(internalPb.MsgType_kTimeTick, 1, 1))
-	msgPack.Msgs = append(msgPack.Msgs, getTsMsg(internalPb.MsgType_kSearch, 2, 2))
-	msgPack.Msgs = append(msgPack.Msgs, getTsMsg(internalPb.MsgType_kSearchResult, 3, 3))
-	msgPack.Msgs = append(msgPack.Msgs, getTsMsg(internalPb.MsgType_kQueryNodeSegStats, 4, 4))
+	msgPack.Msgs = append(msgPack.Msgs, &tsMsg)
 
 	inputStream := NewPulsarMsgStream(context.Background(), 100)
 	inputStream.SetPulsarCient(pulsarAddress)

internal/msgstream/task.go

@@ -57,24 +57,24 @@ func (it *InsertMsg) Marshal(input *TsMsg) ([]byte, error) {
 func (it *InsertMsg) Unmarshal(input []byte) (*TsMsg, error) {
 	insertRequest := internalPb.InsertRequest{}
 	err := proto.Unmarshal(input, &insertRequest)
+	insertMsg := &InsertMsg{InsertRequest: insertRequest}
+
 	if err != nil {
 		return nil, err
 	}
-	insertMsg := &InsertMsg{InsertRequest: insertRequest}
 	for _, timestamp := range insertMsg.Timestamps {
-		insertMsg.BeginTimestamp = timestamp
-		insertMsg.EndTimestamp = timestamp
+		it.BeginTimestamp = timestamp
+		it.EndTimestamp = timestamp
 		break
 	}
 	for _, timestamp := range insertMsg.Timestamps {
-		if timestamp > insertMsg.EndTimestamp {
-			insertMsg.EndTimestamp = timestamp
+		if timestamp > it.EndTimestamp {
+			it.EndTimestamp = timestamp
 		}
-		if timestamp < insertMsg.BeginTimestamp {
-			insertMsg.BeginTimestamp = timestamp
+		if timestamp < it.BeginTimestamp {
+			it.BeginTimestamp = timestamp
 		}
 	}
-
 	var tsMsg TsMsg = insertMsg
 	return &tsMsg, nil
 }
@@ -102,24 +102,24 @@ func (dt *DeleteMsg) Marshal(input *TsMsg) ([]byte, error) {
 func (dt *DeleteMsg) Unmarshal(input []byte) (*TsMsg, error) {
 	deleteRequest := internalPb.DeleteRequest{}
 	err := proto.Unmarshal(input, &deleteRequest)
+	deleteMsg := &DeleteMsg{DeleteRequest: deleteRequest}
+
 	if err != nil {
 		return nil, err
 	}
-	deleteMsg := &DeleteMsg{DeleteRequest: deleteRequest}
 	for _, timestamp := range deleteMsg.Timestamps {
-		deleteMsg.BeginTimestamp = timestamp
-		deleteMsg.EndTimestamp = timestamp
+		dt.BeginTimestamp = timestamp
+		dt.EndTimestamp = timestamp
 		break
 	}
 	for _, timestamp := range deleteMsg.Timestamps {
-		if timestamp > deleteMsg.EndTimestamp {
-			deleteMsg.EndTimestamp = timestamp
+		if timestamp > dt.EndTimestamp {
+			dt.EndTimestamp = timestamp
 		}
-		if timestamp < deleteMsg.BeginTimestamp {
-			deleteMsg.BeginTimestamp = timestamp
+		if timestamp < dt.BeginTimestamp {
+			dt.BeginTimestamp = timestamp
 		}
 	}
-
 	var tsMsg TsMsg = deleteMsg
 	return &tsMsg, nil
 }
@@ -147,13 +147,13 @@ func (st *SearchMsg) Marshal(input *TsMsg) ([]byte, error) {
 func (st *SearchMsg) Unmarshal(input []byte) (*TsMsg, error) {
 	searchRequest := internalPb.SearchRequest{}
 	err := proto.Unmarshal(input, &searchRequest)
+	searchMsg := &SearchMsg{SearchRequest: searchRequest}
+
 	if err != nil {
 		return nil, err
 	}
-	searchMsg := &SearchMsg{SearchRequest: searchRequest}
-	searchMsg.BeginTimestamp = searchMsg.Timestamp
-	searchMsg.EndTimestamp = searchMsg.Timestamp
-
+	st.BeginTimestamp = searchMsg.Timestamp
+	st.EndTimestamp = searchMsg.Timestamp
 	var tsMsg TsMsg = searchMsg
 	return &tsMsg, nil
 }
@@ -181,13 +181,13 @@ func (srt *SearchResultMsg) Marshal(input *TsMsg) ([]byte, error) {
 func (srt *SearchResultMsg) Unmarshal(input []byte) (*TsMsg, error) {
 	searchResultRequest := internalPb.SearchResult{}
 	err := proto.Unmarshal(input, &searchResultRequest)
+	searchResultMsg := &SearchResultMsg{SearchResult: searchResultRequest}
+
 	if err != nil {
 		return nil, err
 	}
-	searchResultMsg := &SearchResultMsg{SearchResult: searchResultRequest}
-	searchResultMsg.BeginTimestamp = searchResultMsg.Timestamp
-	searchResultMsg.EndTimestamp = searchResultMsg.Timestamp
-
+	srt.BeginTimestamp = searchResultMsg.Timestamp
+	srt.EndTimestamp = searchResultMsg.Timestamp
 	var tsMsg TsMsg = searchResultMsg
 	return &tsMsg, nil
 }
@@ -215,49 +215,17 @@ func (tst *TimeTickMsg) Marshal(input *TsMsg) ([]byte, error) {
 func (tst *TimeTickMsg) Unmarshal(input []byte) (*TsMsg, error) {
 	timeTickMsg := internalPb.TimeTickMsg{}
 	err := proto.Unmarshal(input, &timeTickMsg)
-	if err != nil {
-		return nil, err
-	}
 	timeTick := &TimeTickMsg{TimeTickMsg: timeTickMsg}
-	timeTick.BeginTimestamp = timeTick.Timestamp
-	timeTick.EndTimestamp = timeTick.Timestamp
 
+	if err != nil {
+		return nil, err
+	}
+	tst.BeginTimestamp = timeTick.Timestamp
+	tst.EndTimestamp = timeTick.Timestamp
 	var tsMsg TsMsg = timeTick
 	return &tsMsg, nil
 }
 
-/////////////////////////////////////////QueryNodeSegStats//////////////////////////////////////////
-type QueryNodeSegStatsMsg struct {
-	BaseMsg
-	internalPb.QueryNodeSegStats
-}
-
-func (qs *QueryNodeSegStatsMsg) Type() MsgType {
-	return qs.MsgType
-}
-
-func (qs *QueryNodeSegStatsMsg) Marshal(input *TsMsg) ([]byte, error) {
-	queryNodeSegStatsTask := (*input).(*QueryNodeSegStatsMsg)
-	queryNodeSegStats := &queryNodeSegStatsTask.QueryNodeSegStats
-	mb, err := proto.Marshal(queryNodeSegStats)
-	if err != nil {
-		return nil, err
-	}
-	return mb, nil
-}
-
-func (qs *QueryNodeSegStatsMsg) Unmarshal(input []byte) (*TsMsg, error) {
-	queryNodeSegStats := internalPb.QueryNodeSegStats{}
-	err := proto.Unmarshal(input, &queryNodeSegStats)
-	if err != nil {
-		return nil, err
-	}
-	queryNodeSegStatsMsg := &QueryNodeSegStatsMsg{QueryNodeSegStats: queryNodeSegStats}
-
-	var tsMsg TsMsg = queryNodeSegStatsMsg
-	return &tsMsg, nil
-}
-
 ///////////////////////////////////////////Key2Seg//////////////////////////////////////////
 //type Key2SegMsg struct {
 //	BaseMsg

internal/msgstream/unmarshal.go

@@ -30,14 +30,12 @@ func (dispatcher *UnmarshalDispatcher) addDefaultMsgTemplates() {
 	searchMsg := SearchMsg{}
 	searchResultMsg := SearchResultMsg{}
 	timeTickMsg := TimeTickMsg{}
-	queryNodeSegStatsMsg := QueryNodeSegStatsMsg{}
 	dispatcher.tempMap = make(map[internalPb.MsgType]UnmarshalFunc)
 	dispatcher.tempMap[internalPb.MsgType_kInsert] = insertMsg.Unmarshal
 	dispatcher.tempMap[internalPb.MsgType_kDelete] = deleteMsg.Unmarshal
 	dispatcher.tempMap[internalPb.MsgType_kSearch] = searchMsg.Unmarshal
 	dispatcher.tempMap[internalPb.MsgType_kSearchResult] = searchResultMsg.Unmarshal
 	dispatcher.tempMap[internalPb.MsgType_kTimeTick] = timeTickMsg.Unmarshal
-	dispatcher.tempMap[internalPb.MsgType_kQueryNodeSegStats] = queryNodeSegStatsMsg.Unmarshal
 }
 
 func NewUnmarshalDispatcher() *UnmarshalDispatcher {

internal/proxy/proxy_test.go

@@ -41,8 +41,9 @@ func startMaster(ctx context.Context) {
 	rootPath := conf.Config.Etcd.Rootpath
 	kvRootPath := path.Join(rootPath, "kv")
 	metaRootPath := path.Join(rootPath, "meta")
+	tsoRootPath := path.Join(rootPath, "timestamp")
 
-	svr, err := master.CreateServer(ctx, kvRootPath, metaRootPath, []string{etcdAddr})
+	svr, err := master.CreateServer(ctx, kvRootPath, metaRootPath, tsoRootPath, []string{etcdAddr})
 	masterServer = svr
 	if err != nil {
 		log.Print("create server failed", zap.Error(err))

internal/reader/col_seg_container.go

@@ -206,7 +206,6 @@ func (container *colSegContainer) getSegmentStatistics() *internalpb.QueryNodeSe
 		}
 
 		statisticData = append(statisticData, &stat)
-		segment.recentlyModified = false
 	}
 
 	return &internalpb.QueryNodeSegStats{

internal/reader/flow_graph_insert_node.go

@@ -56,20 +56,6 @@ func (iNode *insertNode) Operate(in []*Msg) []*Msg {
 		insertData.insertIDs[task.SegmentID] = append(insertData.insertIDs[task.SegmentID], task.RowIDs...)
 		insertData.insertTimestamps[task.SegmentID] = append(insertData.insertTimestamps[task.SegmentID], task.Timestamps...)
 		insertData.insertRecords[task.SegmentID] = append(insertData.insertRecords[task.SegmentID], task.RowData...)
-
-		// check if segment exists, if not, create this segment
-		if !(*iNode.container).hasSegment(task.SegmentID) {
-			collection, err := (*iNode.container).getCollectionByName(task.CollectionName)
-			if err != nil {
-				log.Println(err)
-				continue
-			}
-			err = (*iNode.container).addSegment(task.SegmentID, task.PartitionTag, collection.ID())
-			if err != nil {
-				log.Println(err)
-				continue
-			}
-		}
 	}
 
 	// 2. do preInsert

internal/reader/flow_graph_service_time_node.go

@@ -28,7 +28,7 @@ func (stNode *serviceTimeNode) Operate(in []*Msg) []*Msg {
 	}
 
 	// update service time
-	stNode.node.tSafe.setTSafe(serviceTimeMsg.timeRange.timestampMax)
+	stNode.node.tSafe = serviceTimeMsg.timeRange.timestampMax
 	return nil
 }
 

internal/reader/query_node.go

@@ -14,7 +14,6 @@ import "C"
 
 import (
 	"context"
-	"sync"
 )
 
 type QueryNode struct {
@@ -23,7 +22,7 @@ type QueryNode struct {
 	QueryNodeID uint64
 	pulsarURL   string
 
-	tSafe tSafe
+	tSafe Timestamp
 
 	container *container
 
@@ -33,16 +32,6 @@ type QueryNode struct {
 	statsService    *statsService
 }
 
-type tSafe interface {
-	getTSafe() Timestamp
-	setTSafe(t Timestamp)
-}
-
-type serviceTime struct {
-	tSafeMu sync.Mutex
-	time    Timestamp
-}
-
 func NewQueryNode(ctx context.Context, queryNodeID uint64, pulsarURL string) *QueryNode {
 	segmentsMap := make(map[int64]*Segment)
 	collections := make([]*Collection, 0)
@@ -52,15 +41,13 @@ func NewQueryNode(ctx context.Context, queryNodeID uint64, pulsarURL string) *Qu
 		segments:    segmentsMap,
 	}
 
-	var tSafe tSafe = &serviceTime{}
-
 	return &QueryNode{
 		ctx: ctx,
 
 		QueryNodeID: queryNodeID,
 		pulsarURL:   pulsarURL,
 
-		tSafe: tSafe,
+		tSafe: 0,
 
 		container: &container,
 
@@ -86,15 +73,3 @@ func (node *QueryNode) Start() {
 func (node *QueryNode) Close() {
 	// TODO: close services
 }
-
-func (st *serviceTime) getTSafe() Timestamp {
-	st.tSafeMu.Lock()
-	defer st.tSafeMu.Unlock()
-	return st.time
-}
-
-func (st *serviceTime) setTSafe(t Timestamp) {
-	st.tSafeMu.Lock()
-	st.time = t
-	st.tSafeMu.Unlock()
-}

internal/reader/segment.go

@@ -151,7 +151,6 @@ func (s *Segment) segmentInsert(offset int64, entityIDs *[]UniqueID, timestamps
 		return errors.New("Insert failed, error code = " + strconv.Itoa(int(status)))
 	}
 
-	s.recentlyModified = true
 	return nil
 }
 

internal/reader/stats_service.go

@@ -3,7 +3,6 @@ package reader
 import (
 	"context"
 	"fmt"
-	"log"
 	"strconv"
 	"time"
 
@@ -14,55 +13,35 @@ import (
 
 type statsService struct {
 	ctx       context.Context
-	pulsarURL string
-
-	msgStream *msgstream.MsgStream
-
+	msgStream *msgstream.PulsarMsgStream
 	container *container
 }
 
-func newStatsService(ctx context.Context, container *container, pulsarURL string) *statsService {
+func newStatsService(ctx context.Context, container *container, pulsarAddress string) *statsService {
+	// TODO: add pulsar message stream init
 
 	return &statsService{
 		ctx:       ctx,
-		pulsarURL: pulsarURL,
-		msgStream: nil,
 		container: container,
 	}
 }
 
 func (sService *statsService) start() {
-	const (
-		receiveBufSize       = 1024
-		sleepMillisecondTime = 1000
-	)
-
-	// start pulsar
-	producerChannels := []string{"statistic"}
-
-	statsStream := msgstream.NewPulsarMsgStream(sService.ctx, receiveBufSize)
-	statsStream.SetPulsarCient(sService.pulsarURL)
-	statsStream.CreatePulsarProducers(producerChannels)
-
-	var statsMsgStream msgstream.MsgStream = statsStream
-
-	sService.msgStream = &statsMsgStream
-	(*sService.msgStream).Start()
-
-	// start service
+	sleepMillisecondTime := 1000
 	fmt.Println("do segments statistic in ", strconv.Itoa(sleepMillisecondTime), "ms")
 	for {
 		select {
 		case <-sService.ctx.Done():
 			return
-		case <-time.After(sleepMillisecondTime * time.Millisecond):
+		default:
+			time.Sleep(time.Duration(sleepMillisecondTime) * time.Millisecond)
 			sService.sendSegmentStatistic()
 		}
 	}
 }
 
 func (sService *statsService) sendSegmentStatistic() {
-	statisticData := (*sService.container).getSegmentStatistics()
+	var statisticData = (*sService.container).getSegmentStatistics()
 
 	// fmt.Println("Publish segment statistic")
 	// fmt.Println(statisticData)
@@ -70,15 +49,5 @@ func (sService *statsService) sendSegmentStatistic() {
 }
 
 func (sService *statsService) publicStatistic(statistic *internalpb.QueryNodeSegStats) {
-	var msg msgstream.TsMsg = &msgstream.QueryNodeSegStatsMsg{
-		QueryNodeSegStats: *statistic,
-	}
-
-	var msgPack = msgstream.MsgPack{
-		Msgs: []*msgstream.TsMsg{&msg},
-	}
-	err := (*sService.msgStream).Produce(&msgPack)
-	if err != nil {
-		log.Println(err)
-	}
+	// TODO: publish statistic
 }

internal/util/tsoutil/tso.go

@@ -1,10 +1,12 @@
 package tsoutil
 
 import (
+	"fmt"
 	"path"
 	"time"
 
 	"github.com/zilliztech/milvus-distributed/internal/kv"
+	gparams "github.com/zilliztech/milvus-distributed/internal/util/paramtableutil"
 	"go.etcd.io/etcd/clientv3"
 )
 
@@ -25,10 +27,25 @@ func ParseTS(ts uint64) (time.Time, uint64) {
 	return physicalTime, logical
 }
 
-func NewTSOKVBase(etcdAddr []string, tsoRoot, subPath string) *kv.EtcdKV {
+func NewTSOKVBase(subPath string) *kv.EtcdKV {
+	etcdAddr, err := gparams.GParams.Load("etcd.address")
+	if err != nil {
+		panic(err)
+	}
+	etcdPort, err := gparams.GParams.Load("etcd.port")
+	if err != nil {
+		panic(err)
+	}
+	etcdAddr = etcdAddr + ":" + etcdPort
+	fmt.Println("etcdAddr ::: ", etcdAddr)
 	client, _ := clientv3.New(clientv3.Config{
-		Endpoints:   etcdAddr,
+		Endpoints:   []string{etcdAddr},
 		DialTimeout: 5 * time.Second,
 	})
-	return kv.NewEtcdKV(client, path.Join(tsoRoot, subPath))
+
+	etcdRootPath, err := gparams.GParams.Load("etcd.rootpath")
+	if err != nil {
+		panic(err)
+	}
+	return kv.NewEtcdKV(client, path.Join(etcdRootPath, subPath))
 }