milvus-io
/
milvus
mirror of https://github.com/milvus-io/milvus.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Change name and tag to ID 

Signed-off-by: XuanYang-cn <xuan.yang@zilliz.com>
pull/4973/head^2
XuanYang-cn 2021-01-22 19:36:09 +08:00 committed by yefu.chen
parent bff208d78c
commit 50e2369000
45 changed files with 906 additions and 994 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
internal/core/src/pb/common.pb.cc
View File

@ -249,7 +249,7 @@ const char descriptor_table_protodef_common_2eproto[] PROTOBUF_SECTION_VARIABLE(
"PE\020\027\022\021\n\rOUT_OF_MEMORY\020\030\022\024\n\017DD_REQUEST_RA"
"CE\020\350\007*N\n\nIndexState\022\010\n\004NONE\020\000\022\014\n\010UNISSUE"
"D\020\001\022\016\n\nINPROGRESS\020\002\022\014\n\010FINISHED\020\003\022\n\n\006FAI"
"LED\020\004*\377\004\n\007MsgType\022\t\n\005kNone\020\000\022\025\n\021kCreateC"
"LED\020\004*\227\005\n\007MsgType\022\t\n\005kNone\020\000\022\025\n\021kCreateC"
"ollection\020d\022\023\n\017kDropCollection\020e\022\022\n\016kHas"
"Collection\020f\022\027\n\023kDescribeCollection\020g\022\024\n"
"\020kShowCollections\020h\022\022\n\016kGetSysConfigs\020i\022"
@ -265,9 +265,9 @@ const char descriptor_table_protodef_common_2eproto[] PROTOBUF_SECTION_VARIABLE(
"ck\020\260\t\022\024\n\017kQueryNodeStats\020\261\t\022\017\n\nkLoadInde"
"x\020\262\t\022\017\n\nkRequestID\020\263\t\022\020\n\013kRequestTSO\020\264\t\022"
"\025\n\020kAllocateSegment\020\265\t\022\027\n\022kSegmentStatis"
"tics\020\266\tBBZ@github.com/zilliztech/milvus-"
"distributed/internal/proto/commonpbb\006pro"
"to3"
"tics\020\266\t\022\026\n\021kSegmentFlushDone\020\267\tBBZ@githu"
"b.com/zilliztech/milvus-distributed/inte"
"rnal/proto/commonpbb\006proto3"
;
static const ::PROTOBUF_NAMESPACE_ID::internal::DescriptorTable*const descriptor_table_common_2eproto_deps[1] = {
};
@ -283,7 +283,7 @@ static ::PROTOBUF_NAMESPACE_ID::internal::SCCInfoBase*const descriptor_table_com
static ::PROTOBUF_NAMESPACE_ID::internal::once_flag descriptor_table_common_2eproto_once;
static bool descriptor_table_common_2eproto_initialized = false;
const ::PROTOBUF_NAMESPACE_ID::internal::DescriptorTable descriptor_table_common_2eproto = {
&descriptor_table_common_2eproto_initialized, descriptor_table_protodef_common_2eproto, "common.proto", 1763,
&descriptor_table_common_2eproto_initialized, descriptor_table_protodef_common_2eproto, "common.proto", 1787,
&descriptor_table_common_2eproto_once, descriptor_table_common_2eproto_sccs, descriptor_table_common_2eproto_deps, 7, 0,
schemas, file_default_instances, TableStruct_common_2eproto::offsets,
file_level_metadata_common_2eproto, 7, file_level_enum_descriptors_common_2eproto, file_level_service_descriptors_common_2eproto,
@ -385,6 +385,7 @@ bool MsgType_IsValid(int value) {
case 1204:
case 1205:
case 1206:
case 1207:
return true;
default:
return false;

3
internal/core/src/pb/common.pb.h
View File

@ -203,12 +203,13 @@ enum MsgType : int {
kRequestTSO = 1204,
kAllocateSegment = 1205,
kSegmentStatistics = 1206,
kSegmentFlushDone = 1207,
MsgType_INT_MIN_SENTINEL_DO_NOT_USE_ = std::numeric_limits<::PROTOBUF_NAMESPACE_ID::int32>::min(),
MsgType_INT_MAX_SENTINEL_DO_NOT_USE_ = std::numeric_limits<::PROTOBUF_NAMESPACE_ID::int32>::max()
};
bool MsgType_IsValid(int value);
constexpr MsgType MsgType_MIN = kNone;
constexpr MsgType MsgType_MAX = kSegmentStatistics;
constexpr MsgType MsgType_MAX = kSegmentFlushDone;
constexpr int MsgType_ARRAYSIZE = MsgType_MAX + 1;
const ::PROTOBUF_NAMESPACE_ID::EnumDescriptor* MsgType_descriptor();

8
internal/core/src/query/Plan.cpp
View File

@ -319,14 +319,14 @@ Parser::ParseItemList(const Json& body) {
std::vector<ExprPtr> results;
if (body.is_object()) {
// only one item;
auto new_expr = ParseAnyNode(body);
results.emplace_back(std::move(new_expr));
auto new_entry = ParseAnyNode(body);
results.emplace_back(std::move(new_entry));
} else {
// item array
Assert(body.is_array());
for (auto& item : body) {
auto new_expr = ParseAnyNode(item);
results.emplace_back(std::move(new_expr));
auto new_entry = ParseAnyNode(item);
results.emplace_back(std::move(new_entry));
}
}
auto old_size = results.size();

22
internal/core/src/query/SearchBruteForce.cpp
View File

@ -24,7 +24,7 @@ SubQueryResult
BinarySearchBruteForceFast(MetricType metric_type,
int64_t dim,
const uint8_t* binary_chunk,
int64_t size_per_chunk,
int64_t chunk_size,
int64_t topk,
int64_t num_queries,
const uint8_t* query_data,
@ -34,7 +34,7 @@ BinarySearchBruteForceFast(MetricType metric_type,
idx_t* result_labels = sub_result.get_labels();
int64_t code_size = dim / 8;
const idx_t block_size = size_per_chunk;
const idx_t block_size = chunk_size;
bool use_heap = true;
if (metric_type == faiss::METRIC_Jaccard || metric_type == faiss::METRIC_Tanimoto) {
@ -50,7 +50,7 @@ BinarySearchBruteForceFast(MetricType metric_type,
result_labels + query_base_index * topk, D + query_base_index * topk};
binary_distence_knn_hc(metric_type, &res, query_data + query_base_index * code_size, binary_chunk,
size_per_chunk, code_size,
chunk_size, code_size,
/* ordered = */ true, bitset);
}
if (metric_type == faiss::METRIC_Tanimoto) {
@ -67,7 +67,7 @@ BinarySearchBruteForceFast(MetricType metric_type,
}
// only match ids will be chosed, not to use heap
binary_distence_knn_mc(metric_type, query_data + s * code_size, binary_chunk, nn, size_per_chunk, topk,
binary_distence_knn_mc(metric_type, query_data + s * code_size, binary_chunk, nn, chunk_size, topk,
code_size, D + s * topk, result_labels + s * topk, bitset);
}
} else if (metric_type == faiss::METRIC_Hamming) {
@ -82,10 +82,10 @@ BinarySearchBruteForceFast(MetricType metric_type,
faiss::int_maxheap_array_t res = {size_t(nn), size_t(topk), result_labels + s * topk,
int_distances.data() + s * topk};
hammings_knn_hc(&res, query_data + s * code_size, binary_chunk, size_per_chunk, code_size,
hammings_knn_hc(&res, query_data + s * code_size, binary_chunk, chunk_size, code_size,
/* ordered = */ true, bitset);
} else {
hammings_knn_mc(query_data + s * code_size, binary_chunk, nn, size_per_chunk, topk, code_size,
hammings_knn_mc(query_data + s * code_size, binary_chunk, nn, chunk_size, topk, code_size,
int_distances.data() + s * topk, result_labels + s * topk, bitset);
}
}
@ -101,7 +101,7 @@ BinarySearchBruteForceFast(MetricType metric_type,
SubQueryResult
FloatSearchBruteForce(const dataset::FloatQueryDataset& query_dataset,
const float* chunk_data,
int64_t size_per_chunk,
int64_t chunk_size,
const faiss::BitsetView& bitset) {
auto metric_type = query_dataset.metric_type;
auto num_queries = query_dataset.num_queries;
@ -111,11 +111,11 @@ FloatSearchBruteForce(const dataset::FloatQueryDataset& query_dataset,
if (metric_type == MetricType::METRIC_L2) {
faiss::float_maxheap_array_t buf{(size_t)num_queries, (size_t)topk, sub_qr.get_labels(), sub_qr.get_values()};
faiss::knn_L2sqr(query_dataset.query_data, chunk_data, dim, num_queries, size_per_chunk, &buf, bitset);
faiss::knn_L2sqr(query_dataset.query_data, chunk_data, dim, num_queries, chunk_size, &buf, bitset);
return sub_qr;
} else {
faiss::float_minheap_array_t buf{(size_t)num_queries, (size_t)topk, sub_qr.get_labels(), sub_qr.get_values()};
faiss::knn_inner_product(query_dataset.query_data, chunk_data, dim, num_queries, size_per_chunk, &buf, bitset);
faiss::knn_inner_product(query_dataset.query_data, chunk_data, dim, num_queries, chunk_size, &buf, bitset);
return sub_qr;
}
}
@ -123,10 +123,10 @@ FloatSearchBruteForce(const dataset::FloatQueryDataset& query_dataset,
SubQueryResult
BinarySearchBruteForce(const dataset::BinaryQueryDataset& query_dataset,
const uint8_t* binary_chunk,
int64_t size_per_chunk,
int64_t chunk_size,
const faiss::BitsetView& bitset) {
// TODO: refactor the internal function
return BinarySearchBruteForceFast(query_dataset.metric_type, query_dataset.dim, binary_chunk, size_per_chunk,
return BinarySearchBruteForceFast(query_dataset.metric_type, query_dataset.dim, binary_chunk, chunk_size,
query_dataset.topk, query_dataset.num_queries, query_dataset.query_data, bitset);
}
} // namespace milvus::query

4
internal/core/src/query/SearchBruteForce.h
View File

@ -21,13 +21,13 @@ namespace milvus::query {
SubQueryResult
BinarySearchBruteForce(const dataset::BinaryQueryDataset& query_dataset,
const uint8_t* binary_chunk,
int64_t size_per_chunk,
int64_t chunk_size,
const faiss::BitsetView& bitset);
SubQueryResult
FloatSearchBruteForce(const dataset::FloatQueryDataset& query_dataset,
const float* chunk_data,
int64_t size_per_chunk,
int64_t chunk_size,
const faiss::BitsetView& bitset);
} // namespace milvus::query

44
internal/core/src/query/SearchOnGrowing.cpp
View File

@ -72,46 +72,46 @@ FloatSearch(const segcore::SegmentGrowingImpl& segment,
dataset::FloatQueryDataset query_dataset{metric_type, num_queries, topK, dim, query_data};
auto max_indexed_id = indexing_record.get_finished_ack();
const auto& field_indexing = indexing_record.get_vec_field_indexing(vecfield_offset);
auto search_conf = field_indexing.get_search_conf(topK);
const auto& indexing_entry = indexing_record.get_vec_entry(vecfield_offset);
auto search_conf = indexing_entry.get_search_conf(topK);
for (int chunk_id = 0; chunk_id < max_indexed_id; ++chunk_id) {
auto size_per_chunk = field_indexing.get_size_per_chunk();
auto indexing = field_indexing.get_chunk_indexing(chunk_id);
auto chunk_size = indexing_entry.get_chunk_size();
auto indexing = indexing_entry.get_indexing(chunk_id);
auto sub_view = BitsetSubView(bitset, chunk_id * size_per_chunk, size_per_chunk);
auto sub_view = BitsetSubView(bitset, chunk_id * chunk_size, chunk_size);
auto sub_qr = SearchOnIndex(query_dataset, *indexing, search_conf, sub_view);
// convert chunk uid to segment uid
for (auto& x : sub_qr.mutable_labels()) {
if (x != -1) {
x += chunk_id * size_per_chunk;
x += chunk_id * chunk_size;
}
}
final_qr.merge(sub_qr);
}
auto vec_ptr = record.get_field_data<FloatVector>(vecfield_offset);
auto vec_ptr = record.get_entity<FloatVector>(vecfield_offset);
// step 4: brute force search where small indexing is unavailable
auto vec_size_per_chunk = vec_ptr->get_size_per_chunk();
Assert(vec_size_per_chunk == field_indexing.get_size_per_chunk());
auto max_chunk = upper_div(ins_barrier, vec_size_per_chunk);
auto vec_chunk_size = vec_ptr->get_chunk_size();
Assert(vec_chunk_size == indexing_entry.get_chunk_size());
auto max_chunk = upper_div(ins_barrier, vec_chunk_size);
for (int chunk_id = max_indexed_id; chunk_id < max_chunk; ++chunk_id) {
auto& chunk = vec_ptr->get_chunk(chunk_id);
auto element_begin = chunk_id * vec_size_per_chunk;
auto element_end = std::min(ins_barrier, (chunk_id + 1) * vec_size_per_chunk);
auto size_per_chunk = element_end - element_begin;
auto element_begin = chunk_id * vec_chunk_size;
auto element_end = std::min(ins_barrier, (chunk_id + 1) * vec_chunk_size);
auto chunk_size = element_end - element_begin;
auto sub_view = BitsetSubView(bitset, element_begin, size_per_chunk);
auto sub_qr = FloatSearchBruteForce(query_dataset, chunk.data(), size_per_chunk, sub_view);
auto sub_view = BitsetSubView(bitset, element_begin, chunk_size);
auto sub_qr = FloatSearchBruteForce(query_dataset, chunk.data(), chunk_size, sub_view);
// convert chunk uid to segment uid
for (auto& x : sub_qr.mutable_labels()) {
if (x != -1) {
x += chunk_id * vec_size_per_chunk;
x += chunk_id * vec_chunk_size;
}
}
final_qr.merge(sub_qr);
@ -160,18 +160,18 @@ BinarySearch(const segcore::SegmentGrowingImpl& segment,
// step 3: small indexing search
query::dataset::BinaryQueryDataset query_dataset{metric_type, num_queries, topK, dim, query_data};
auto vec_ptr = record.get_field_data<BinaryVector>(vecfield_offset);
auto vec_ptr = record.get_entity<BinaryVector>(vecfield_offset);
auto max_indexed_id = 0;
// step 4: brute force search where small indexing is unavailable
auto vec_size_per_chunk = vec_ptr->get_size_per_chunk();
auto max_chunk = upper_div(ins_barrier, vec_size_per_chunk);
auto vec_chunk_size = vec_ptr->get_chunk_size();
auto max_chunk = upper_div(ins_barrier, vec_chunk_size);
SubQueryResult final_result(num_queries, topK, metric_type);
for (int chunk_id = max_indexed_id; chunk_id < max_chunk; ++chunk_id) {
auto& chunk = vec_ptr->get_chunk(chunk_id);
auto element_begin = chunk_id * vec_size_per_chunk;
auto element_end = std::min(ins_barrier, (chunk_id + 1) * vec_size_per_chunk);
auto element_begin = chunk_id * vec_chunk_size;
auto element_end = std::min(ins_barrier, (chunk_id + 1) * vec_chunk_size);
auto nsize = element_end - element_begin;
auto sub_view = BitsetSubView(bitset, element_begin, nsize);
@ -180,7 +180,7 @@ BinarySearch(const segcore::SegmentGrowingImpl& segment,
// convert chunk uid to segment uid
for (auto& x : sub_result.mutable_labels()) {
if (x != -1) {
x += chunk_id * vec_size_per_chunk;
x += chunk_id * vec_chunk_size;
}
}
final_result.merge(sub_result);

8
internal/core/src/query/SearchOnSealed.cpp
View File

@ -62,16 +62,16 @@ SearchOnSealed(const Schema& schema,
auto dim = field.get_dim();
Assert(record.is_ready(field_offset));
auto field_indexing = record.get_field_indexing(field_offset);
Assert(field_indexing->metric_type_ == query_info.metric_type_);
auto indexing_entry = record.get_entry(field_offset);
Assert(indexing_entry->metric_type_ == query_info.metric_type_);
auto final = [&] {
auto ds = knowhere::GenDataset(num_queries, dim, query_data);
auto conf = query_info.search_params_;
conf[milvus::knowhere::meta::TOPK] = query_info.topK_;
conf[milvus::knowhere::Metric::TYPE] = MetricTypeToName(field_indexing->metric_type_);
return field_indexing->indexing_->Query(ds, conf, bitset);
conf[milvus::knowhere::Metric::TYPE] = MetricTypeToName(indexing_entry->metric_type_);
return indexing_entry->indexing_->Query(ds, conf, bitset);
}();
auto ids = final->Get<idx_t*>(knowhere::meta::IDS);

40
internal/core/src/query/visitors/ExecExprVisitor.cpp
View File

@ -120,33 +120,41 @@ template <typename T, typename IndexFunc, typename ElementFunc>
auto
ExecExprVisitor::ExecRangeVisitorImpl(RangeExprImpl<T>& expr, IndexFunc index_func, ElementFunc element_func)
-> RetType {
auto data_type = expr.data_type_;
auto& schema = segment_.get_schema();
auto field_offset = expr.field_offset_;
auto& field_meta = schema[field_offset];
auto indexing_barrier = segment_.num_chunk_index(field_offset);
auto size_per_chunk = segment_.size_per_chunk();
auto num_chunk = upper_div(row_count_, size_per_chunk);
// auto vec_ptr = records.get_entity<T>(field_offset);
// auto& vec = *vec_ptr;
// const segcore::ScalarIndexingEntry<T>& entry = indexing_record.get_scalar_entry<T>(field_offset);
// RetType results(vec.num_chunk());
auto indexing_barrier = segment_.num_chunk_index_safe(field_offset);
auto chunk_size = segment_.size_per_chunk();
auto num_chunk = upper_div(row_count_, chunk_size);
RetType results;
using Index = knowhere::scalar::StructuredIndex<T>;
for (auto chunk_id = 0; chunk_id < indexing_barrier; ++chunk_id) {
// auto& result = results[chunk_id];
const Index& indexing = segment_.chunk_scalar_index<T>(field_offset, chunk_id);
// NOTE: knowhere is not const-ready
// This is a dirty workaround
auto data = index_func(const_cast<Index*>(&indexing));
Assert(data->size() == size_per_chunk);
Assert(data->size() == chunk_size);
results.emplace_back(std::move(*data));
}
for (auto chunk_id = indexing_barrier; chunk_id < num_chunk; ++chunk_id) {
boost::dynamic_bitset<> result(size_per_chunk);
result.resize(size_per_chunk);
boost::dynamic_bitset<> result(chunk_size);
// auto& result = results[chunk_id];
result.resize(chunk_size);
auto chunk = segment_.chunk_data<T>(field_offset, chunk_id);
const T* data = chunk.data();
for (int index = 0; index < size_per_chunk; ++index) {
for (int index = 0; index < chunk_size; ++index) {
result[index] = element_func(data[index]);
}
Assert(result.size() == size_per_chunk);
Assert(result.size() == chunk_size);
results.emplace_back(std::move(result));
}
return results;
@ -274,19 +282,27 @@ template <typename T>
auto
ExecExprVisitor::ExecTermVisitorImpl(TermExpr& expr_raw) -> RetType {
auto& expr = static_cast<TermExprImpl<T>&>(expr_raw);
// auto& records = segment_.get_insert_record();
auto data_type = expr.data_type_;
auto& schema = segment_.get_schema();
auto field_offset = expr_raw.field_offset_;
auto& field_meta = schema[field_offset];
auto size_per_chunk = segment_.size_per_chunk();
auto num_chunk = upper_div(row_count_, size_per_chunk);
// auto vec_ptr = records.get_entity<T>(field_offset);
// auto& vec = *vec_ptr;
auto chunk_size = segment_.size_per_chunk();
auto num_chunk = upper_div(row_count_, chunk_size);
RetType bitsets;
// auto N = records.ack_responder_.GetAck();
// TODO: enable index for term
for (int64_t chunk_id = 0; chunk_id < num_chunk; ++chunk_id) {
Span<T> chunk = segment_.chunk_data<T>(field_offset, chunk_id);
auto size = chunk_id == num_chunk - 1 ? row_count_ - chunk_id * size_per_chunk : size_per_chunk;
auto size = chunk_id == num_chunk - 1 ? row_count_ - chunk_id * chunk_size : chunk_size;
boost::dynamic_bitset<> bitset(size_per_chunk);
boost::dynamic_bitset<> bitset(chunk_size);
for (int i = 0; i < size; ++i) {
auto value = chunk.data()[i];
bool is_in = std::binary_search(expr.terms_.begin(), expr.terms_.end(), value);

2
internal/core/src/segcore/CMakeLists.txt
View File

@ -6,7 +6,7 @@ set(SEGCORE_FILES
SegmentGrowing.cpp
SegmentGrowingImpl.cpp
SegmentSealedImpl.cpp
FieldIndexing.cpp
IndexingEntry.cpp
InsertRecord.cpp
Reduce.cpp
plan_c.cpp

57
internal/core/src/segcore/ConcurrentVector.h
View File

@ -72,7 +72,7 @@ class ThreadSafeVector {
class VectorBase {
public:
explicit VectorBase(int64_t size_per_chunk) : size_per_chunk_(size_per_chunk) {
explicit VectorBase(int64_t chunk_size) : chunk_size_(chunk_size) {
}
virtual ~VectorBase() = default;
@ -86,12 +86,12 @@ class VectorBase {
get_span_base(int64_t chunk_id) const = 0;
int64_t
get_size_per_chunk() const {
return size_per_chunk_;
get_chunk_size() const {
return chunk_size_;
}
protected:
const int64_t size_per_chunk_;
const int64_t chunk_size_;
};
template <typename Type, bool is_scalar = false>
@ -111,28 +111,27 @@ class ConcurrentVectorImpl : public VectorBase {
std::conditional_t<is_scalar, Type, std::conditional_t<std::is_same_v<Type, float>, FloatVector, BinaryVector>>;
public:
explicit ConcurrentVectorImpl(ssize_t dim, int64_t size_per_chunk)
: VectorBase(size_per_chunk), Dim(is_scalar ? 1 : dim) {
explicit ConcurrentVectorImpl(ssize_t dim, int64_t chunk_size) : VectorBase(chunk_size), Dim(is_scalar ? 1 : dim) {
Assert(is_scalar ? dim == 1 : dim != 1);
}
void
grow_to_at_least(int64_t element_count) override {
auto chunk_count = upper_div(element_count, size_per_chunk_);
chunks_.emplace_to_at_least(chunk_count, Dim * size_per_chunk_);
auto chunk_count = upper_div(element_count, chunk_size_);
chunks_.emplace_to_at_least(chunk_count, Dim * chunk_size_);
}
Span<TraitType>
get_span(int64_t chunk_id) const {
auto& chunk = get_chunk(chunk_id);
if constexpr (is_scalar) {
return Span<TraitType>(chunk.data(), size_per_chunk_);
return Span<TraitType>(chunk.data(), chunk_size_);
} else if constexpr (std::is_same_v<Type, int64_t> || std::is_same_v<Type, int>) {
// only for testing
PanicInfo("unimplemented");
} else {
static_assert(std::is_same_v<typename TraitType::embedded_type, Type>);
return Span<TraitType>(chunk.data(), size_per_chunk_, Dim);
return Span<TraitType>(chunk.data(), chunk_size_, Dim);
}
}
@ -152,28 +151,28 @@ class ConcurrentVectorImpl : public VectorBase {
return;
}
this->grow_to_at_least(element_offset + element_count);
auto chunk_id = element_offset / size_per_chunk_;
auto chunk_offset = element_offset % size_per_chunk_;
auto chunk_id = element_offset / chunk_size_;
auto chunk_offset = element_offset % chunk_size_;
ssize_t source_offset = 0;
// first partition:
if (chunk_offset + element_count <= size_per_chunk_) {
if (chunk_offset + element_count <= chunk_size_) {
// only first
fill_chunk(chunk_id, chunk_offset, element_count, source, source_offset);
return;
}
auto first_size = size_per_chunk_ - chunk_offset;
auto first_size = chunk_size_ - chunk_offset;
fill_chunk(chunk_id, chunk_offset, first_size, source, source_offset);
source_offset += size_per_chunk_ - chunk_offset;
source_offset += chunk_size_ - chunk_offset;
element_count -= first_size;
++chunk_id;
// the middle
while (element_count >= size_per_chunk_) {
fill_chunk(chunk_id, 0, size_per_chunk_, source, source_offset);
source_offset += size_per_chunk_;
element_count -= size_per_chunk_;
while (element_count >= chunk_size_) {
fill_chunk(chunk_id, 0, chunk_size_, source, source_offset);
source_offset += chunk_size_;
element_count -= chunk_size_;
++chunk_id;
}
@ -191,16 +190,16 @@ class ConcurrentVectorImpl : public VectorBase {
// just for fun, don't use it directly
const Type*
get_element(ssize_t element_index) const {
auto chunk_id = element_index / size_per_chunk_;
auto chunk_offset = element_index % size_per_chunk_;
auto chunk_id = element_index / chunk_size_;
auto chunk_offset = element_index % chunk_size_;
return get_chunk(chunk_id).data() + chunk_offset * Dim;
}
const Type&
operator[](ssize_t element_index) const {
Assert(Dim == 1);
auto chunk_id = element_index / size_per_chunk_;
auto chunk_offset = element_index % size_per_chunk_;
auto chunk_id = element_index / chunk_size_;
auto chunk_offset = element_index % chunk_size_;
return get_chunk(chunk_id)[chunk_offset];
}
@ -233,24 +232,24 @@ template <typename Type>
class ConcurrentVector : public ConcurrentVectorImpl<Type, true> {
public:
static_assert(std::is_fundamental_v<Type>);
explicit ConcurrentVector(int64_t size_per_chunk)
: ConcurrentVectorImpl<Type, true>::ConcurrentVectorImpl(1, size_per_chunk) {
explicit ConcurrentVector(int64_t chunk_size)
: ConcurrentVectorImpl<Type, true>::ConcurrentVectorImpl(1, chunk_size) {
}
};
template <>
class ConcurrentVector<FloatVector> : public ConcurrentVectorImpl<float, false> {
public:
ConcurrentVector(int64_t dim, int64_t size_per_chunk)
: ConcurrentVectorImpl<float, false>::ConcurrentVectorImpl(dim, size_per_chunk) {
ConcurrentVector(int64_t dim, int64_t chunk_size)
: ConcurrentVectorImpl<float, false>::ConcurrentVectorImpl(dim, chunk_size) {
}
};
template <>
class ConcurrentVector<BinaryVector> : public ConcurrentVectorImpl<uint8_t, false> {
public:
explicit ConcurrentVector(int64_t dim, int64_t size_per_chunk)
: binary_dim_(dim), ConcurrentVectorImpl(dim / 8, size_per_chunk) {
explicit ConcurrentVector(int64_t dim, int64_t chunk_size)
: binary_dim_(dim), ConcurrentVectorImpl(dim / 8, chunk_size) {
Assert(dim % 8 == 0);
}

6
internal/core/src/segcore/DeletedRecord.h
View File

@ -29,11 +29,9 @@ struct DeletedRecord {
std::shared_ptr<TmpBitmap>
clone(int64_t capacity);
};
static constexpr int64_t deprecated_size_per_chunk = 32 * 1024;
static constexpr int64_t deprecated_chunk_size = 32 * 1024;
DeletedRecord()
: lru_(std::make_shared<TmpBitmap>()),
timestamps_(deprecated_size_per_chunk),
uids_(deprecated_size_per_chunk) {
: lru_(std::make_shared<TmpBitmap>()), timestamps_(deprecated_chunk_size), uids_(deprecated_chunk_size) {
lru_->bitmap_ptr = std::make_shared<faiss::ConcurrentBitset>(0);
}

38
internal/core/src/segcore/FieldIndexing.cpp → internal/core/src/segcore/IndexingEntry.cpp
View File

@ -9,14 +9,14 @@
// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
// or implied. See the License for the specific language governing permissions and limitations under the License
#include "segcore/FieldIndexing.h"
#include "segcore/IndexingEntry.h"
#include <thread>
#include <knowhere/index/vector_index/IndexIVF.h>
#include <knowhere/index/vector_index/adapter/VectorAdapter.h>
namespace milvus::segcore {
void
VectorFieldIndexing::BuildIndexRange(int64_t ack_beg, int64_t ack_end, const VectorBase* vec_base) {
VecIndexingEntry::BuildIndexRange(int64_t ack_beg, int64_t ack_end, const VectorBase* vec_base) {
assert(field_meta_.get_data_type() == DataType::VECTOR_FLOAT);
auto dim = field_meta_.get_dim();
@ -30,7 +30,7 @@ VectorFieldIndexing::BuildIndexRange(int64_t ack_beg, int64_t ack_end, const Vec
const auto& chunk = source->get_chunk(chunk_id);
// build index for chunk
auto indexing = std::make_unique<knowhere::IVF>();
auto dataset = knowhere::GenDataset(source->get_size_per_chunk(), dim, chunk.data());
auto dataset = knowhere::GenDataset(source->get_chunk_size(), dim, chunk.data());
indexing->Train(dataset, conf);
indexing->AddWithoutIds(dataset, conf);
data_[chunk_id] = std::move(indexing);
@ -38,7 +38,7 @@ VectorFieldIndexing::BuildIndexRange(int64_t ack_beg, int64_t ack_end, const Vec
}
knowhere::Config
VectorFieldIndexing::get_build_conf() const {
VecIndexingEntry::get_build_conf() const {
return knowhere::Config{{knowhere::meta::DIM, field_meta_.get_dim()},
{knowhere::IndexParams::nlist, 100},
{knowhere::IndexParams::nprobe, 4},
@ -47,7 +47,7 @@ VectorFieldIndexing::get_build_conf() const {
}
knowhere::Config
VectorFieldIndexing::get_search_conf(int top_K) const {
VecIndexingEntry::get_search_conf(int top_K) const {
return knowhere::Config{{knowhere::meta::DIM, field_meta_.get_dim()},
{knowhere::meta::TOPK, top_K},
{knowhere::IndexParams::nlist, 100},
@ -71,8 +71,8 @@ IndexingRecord::UpdateResourceAck(int64_t chunk_ack, const InsertRecord& record)
lck.unlock();
// std::thread([this, old_ack, chunk_ack, &record] {
for (auto& [field_offset, entry] : field_indexings_) {
auto vec_base = record.get_field_data_base(field_offset);
for (auto& [field_offset, entry] : entries_) {
auto vec_base = record.get_base_entity(field_offset);
entry->BuildIndexRange(old_ack, chunk_ack, vec_base);
}
finished_ack_.AddSegment(old_ack, chunk_ack);
@ -81,7 +81,7 @@ IndexingRecord::UpdateResourceAck(int64_t chunk_ack, const InsertRecord& record)
template <typename T>
void
ScalarFieldIndexing<T>::BuildIndexRange(int64_t ack_beg, int64_t ack_end, const VectorBase* vec_base) {
ScalarIndexingEntry<T>::BuildIndexRange(int64_t ack_beg, int64_t ack_end, const VectorBase* vec_base) {
auto source = dynamic_cast<const ConcurrentVector<T>*>(vec_base);
Assert(source);
auto num_chunk = source->num_chunk();
@ -92,16 +92,16 @@ ScalarFieldIndexing<T>::BuildIndexRange(int64_t ack_beg, int64_t ack_end, const
// build index for chunk
// TODO
auto indexing = std::make_unique<knowhere::scalar::StructuredIndexSort<T>>();
indexing->Build(vec_base->get_size_per_chunk(), chunk.data());
indexing->Build(vec_base->get_chunk_size(), chunk.data());
data_[chunk_id] = std::move(indexing);
}
}
std::unique_ptr<FieldIndexing>
CreateIndex(const FieldMeta& field_meta, int64_t size_per_chunk) {
std::unique_ptr<IndexingEntry>
CreateIndex(const FieldMeta& field_meta, int64_t chunk_size) {
if (field_meta.is_vector()) {
if (field_meta.get_data_type() == DataType::VECTOR_FLOAT) {
return std::make_unique<VectorFieldIndexing>(field_meta, size_per_chunk);
return std::make_unique<VecIndexingEntry>(field_meta, chunk_size);
} else {
// TODO
PanicInfo("unsupported");
@ -109,19 +109,19 @@ CreateIndex(const FieldMeta& field_meta, int64_t size_per_chunk) {
}
switch (field_meta.get_data_type()) {
case DataType::BOOL:
return std::make_unique<ScalarFieldIndexing<bool>>(field_meta, size_per_chunk);
return std::make_unique<ScalarIndexingEntry<bool>>(field_meta, chunk_size);
case DataType::INT8:
return std::make_unique<ScalarFieldIndexing<int8_t>>(field_meta, size_per_chunk);
return std::make_unique<ScalarIndexingEntry<int8_t>>(field_meta, chunk_size);
case DataType::INT16:
return std::make_unique<ScalarFieldIndexing<int16_t>>(field_meta, size_per_chunk);
return std::make_unique<ScalarIndexingEntry<int16_t>>(field_meta, chunk_size);
case DataType::INT32:
return std::make_unique<ScalarFieldIndexing<int32_t>>(field_meta, size_per_chunk);
return std::make_unique<ScalarIndexingEntry<int32_t>>(field_meta, chunk_size);
case DataType::INT64:
return std::make_unique<ScalarFieldIndexing<int64_t>>(field_meta, size_per_chunk);
return std::make_unique<ScalarIndexingEntry<int64_t>>(field_meta, chunk_size);
case DataType::FLOAT:
return std::make_unique<ScalarFieldIndexing<float>>(field_meta, size_per_chunk);
return std::make_unique<ScalarIndexingEntry<float>>(field_meta, chunk_size);
case DataType::DOUBLE:
return std::make_unique<ScalarFieldIndexing<double>>(field_meta, size_per_chunk);
return std::make_unique<ScalarIndexingEntry<double>>(field_meta, chunk_size);
default:
PanicInfo("unsupported");
}

67
internal/core/src/segcore/FieldIndexing.h → internal/core/src/segcore/IndexingEntry.h
View File

@ -24,14 +24,14 @@ namespace milvus::segcore {
// this should be concurrent
// All concurrent
class FieldIndexing {
class IndexingEntry {
public:
explicit FieldIndexing(const FieldMeta& field_meta, int64_t size_per_chunk)
: field_meta_(field_meta), size_per_chunk_(size_per_chunk) {
explicit IndexingEntry(const FieldMeta& field_meta, int64_t chunk_size)
: field_meta_(field_meta), chunk_size_(chunk_size) {
}
FieldIndexing(const FieldIndexing&) = delete;
FieldIndexing&
operator=(const FieldIndexing&) = delete;
IndexingEntry(const IndexingEntry&) = delete;
IndexingEntry&
operator=(const IndexingEntry&) = delete;
// Do this in parallel
virtual void
@ -43,29 +43,29 @@ class FieldIndexing {
}
int64_t
get_size_per_chunk() const {
return size_per_chunk_;
get_chunk_size() const {
return chunk_size_;
}
virtual knowhere::Index*
get_chunk_indexing(int64_t chunk_id) const = 0;
get_indexing(int64_t chunk_id) const = 0;
protected:
// additional info
const FieldMeta& field_meta_;
const int64_t size_per_chunk_;
const int64_t chunk_size_;
};
template <typename T>
class ScalarFieldIndexing : public FieldIndexing {
class ScalarIndexingEntry : public IndexingEntry {
public:
using FieldIndexing::FieldIndexing;
using IndexingEntry::IndexingEntry;
void
BuildIndexRange(int64_t ack_beg, int64_t ack_end, const VectorBase* vec_base) override;
// concurrent
knowhere::scalar::StructuredIndex<T>*
get_chunk_indexing(int64_t chunk_id) const override {
get_indexing(int64_t chunk_id) const override {
Assert(!field_meta_.is_vector());
return data_.at(chunk_id).get();
}
@ -74,16 +74,16 @@ class ScalarFieldIndexing : public FieldIndexing {
tbb::concurrent_vector<std::unique_ptr<knowhere::scalar::StructuredIndex<T>>> data_;
};
class VectorFieldIndexing : public FieldIndexing {
class VecIndexingEntry : public IndexingEntry {
public:
using FieldIndexing::FieldIndexing;
using IndexingEntry::IndexingEntry;
void
BuildIndexRange(int64_t ack_beg, int64_t ack_end, const VectorBase* vec_base) override;
// concurrent
knowhere::VecIndex*
get_chunk_indexing(int64_t chunk_id) const override {
get_indexing(int64_t chunk_id) const override {
Assert(field_meta_.is_vector());
return data_.at(chunk_id).get();
}
@ -97,13 +97,12 @@ class VectorFieldIndexing : public FieldIndexing {
tbb::concurrent_vector<std::unique_ptr<knowhere::VecIndex>> data_;
};
std::unique_ptr<FieldIndexing>
CreateIndex(const FieldMeta& field_meta, int64_t size_per_chunk);
std::unique_ptr<IndexingEntry>
CreateIndex(const FieldMeta& field_meta, int64_t chunk_size);
class IndexingRecord {
public:
explicit IndexingRecord(const Schema& schema, int64_t size_per_chunk)
: schema_(schema), size_per_chunk_(size_per_chunk) {
explicit IndexingRecord(const Schema& schema, int64_t chunk_size) : schema_(schema), chunk_size_(chunk_size) {
Initialize();
}
@ -112,7 +111,7 @@ class IndexingRecord {
int offset = 0;
for (auto& field : schema_) {
if (field.get_data_type() != DataType::VECTOR_BINARY) {
field_indexings_.try_emplace(FieldOffset(offset), CreateIndex(field, size_per_chunk_));
entries_.try_emplace(FieldOffset(offset), CreateIndex(field, chunk_size_));
}
++offset;
}
@ -129,24 +128,24 @@ class IndexingRecord {
return finished_ack_.GetAck();
}
const FieldIndexing&
get_field_indexing(FieldOffset field_offset) const {
assert(field_indexings_.count(field_offset));
return *field_indexings_.at(field_offset);
const IndexingEntry&
get_entry(FieldOffset field_offset) const {
assert(entries_.count(field_offset));
return *entries_.at(field_offset);
}
const VectorFieldIndexing&
get_vec_field_indexing(FieldOffset field_offset) const {
auto& field_indexing = get_field_indexing(field_offset);
auto ptr = dynamic_cast<const VectorFieldIndexing*>(&field_indexing);
const VecIndexingEntry&
get_vec_entry(FieldOffset field_offset) const {
auto& entry = get_entry(field_offset);
auto ptr = dynamic_cast<const VecIndexingEntry*>(&entry);
AssertInfo(ptr, "invalid indexing");
return *ptr;
}
template <typename T>
auto
get_scalar_field_indexing(FieldOffset field_offset) const -> const ScalarFieldIndexing<T>& {
auto& entry = get_field_indexing(field_offset);
auto ptr = dynamic_cast<const ScalarFieldIndexing<T>*>(&entry);
get_scalar_entry(FieldOffset field_offset) const -> const ScalarIndexingEntry<T>& {
auto& entry = get_entry(field_offset);
auto ptr = dynamic_cast<const ScalarIndexingEntry<T>*>(&entry);
AssertInfo(ptr, "invalid indexing");
return *ptr;
}
@ -160,11 +159,11 @@ class IndexingRecord {
// std::atomic<int64_t> finished_ack_ = 0;
AckResponder finished_ack_;
std::mutex mutex_;
int64_t size_per_chunk_;
int64_t chunk_size_;
private:
// field_offset => indexing
std::map<FieldOffset, std::unique_ptr<FieldIndexing>> field_indexings_;
std::map<FieldOffset, std::unique_ptr<IndexingEntry>> entries_;
};
} // namespace milvus::segcore

20
internal/core/src/segcore/InsertRecord.cpp
View File

@ -13,14 +13,14 @@
namespace milvus::segcore {
InsertRecord::InsertRecord(const Schema& schema, int64_t size_per_chunk) : uids_(1), timestamps_(1) {
InsertRecord::InsertRecord(const Schema& schema, int64_t chunk_size) : uids_(1), timestamps_(1) {
for (auto& field : schema) {
if (field.is_vector()) {
if (field.get_data_type() == DataType::VECTOR_FLOAT) {
this->append_field_data<FloatVector>(field.get_dim(), size_per_chunk);
this->insert_entity<FloatVector>(field.get_dim(), chunk_size);
continue;
} else if (field.get_data_type() == DataType::VECTOR_BINARY) {
this->append_field_data<BinaryVector>(field.get_dim(), size_per_chunk);
this->insert_entity<BinaryVector>(field.get_dim(), chunk_size);
continue;
} else {
PanicInfo("unsupported");
@ -28,34 +28,34 @@ InsertRecord::InsertRecord(const Schema& schema, int64_t size_per_chunk) : uids_
}
switch (field.get_data_type()) {
case DataType::BOOL: {
this->append_field_data<bool>(size_per_chunk);
this->insert_entity<bool>(chunk_size);
break;
}
case DataType::INT8: {
this->append_field_data<int8_t>(size_per_chunk);
this->insert_entity<int8_t>(chunk_size);
break;
}
case DataType::INT16: {
this->append_field_data<int16_t>(size_per_chunk);
this->insert_entity<int16_t>(chunk_size);
break;
}
case DataType::INT32: {
this->append_field_data<int32_t>(size_per_chunk);
this->insert_entity<int32_t>(chunk_size);
break;
}
case DataType::INT64: {
this->append_field_data<int64_t>(size_per_chunk);
this->insert_entity<int64_t>(chunk_size);
break;
}
case DataType::FLOAT: {
this->append_field_data<float>(size_per_chunk);
this->insert_entity<float>(chunk_size);
break;
}
case DataType::DOUBLE: {
this->append_field_data<double>(size_per_chunk);
this->insert_entity<double>(chunk_size);
break;
}
default: {

30
internal/core/src/segcore/InsertRecord.h
View File

@ -24,53 +24,47 @@ struct InsertRecord {
ConcurrentVector<Timestamp> timestamps_;
ConcurrentVector<idx_t> uids_;
explicit InsertRecord(const Schema& schema, int64_t size_per_chunk);
explicit InsertRecord(const Schema& schema, int64_t chunk_size);
// get field data without knowing the type
// return VectorBase type
auto
get_field_data_base(FieldOffset field_offset) const {
auto ptr = field_datas_[field_offset.get()].get();
get_base_entity(FieldOffset field_offset) const {
auto ptr = entity_vec_[field_offset.get()].get();
return ptr;
}
// get field data in given type, const version
template <typename Type>
auto
get_field_data(FieldOffset field_offset) const {
auto base_ptr = get_field_data_base(field_offset);
get_entity(FieldOffset field_offset) const {
auto base_ptr = get_base_entity(field_offset);
auto ptr = dynamic_cast<const ConcurrentVector<Type>*>(base_ptr);
Assert(ptr);
return ptr;
}
// get field data in given type, nonconst version
template <typename Type>
auto
get_field_data(FieldOffset field_offset) {
auto base_ptr = get_field_data_base(field_offset);
get_entity(FieldOffset field_offset) {
auto base_ptr = get_base_entity(field_offset);
auto ptr = dynamic_cast<ConcurrentVector<Type>*>(base_ptr);
Assert(ptr);
return ptr;
}
// append a column of scalar type
template <typename Type>
void
append_field_data(int64_t size_per_chunk) {
insert_entity(int64_t chunk_size) {
static_assert(std::is_fundamental_v<Type>);
field_datas_.emplace_back(std::make_unique<ConcurrentVector<Type>>(size_per_chunk));
entity_vec_.emplace_back(std::make_unique<ConcurrentVector<Type>>(chunk_size));
}
// append a column of vector type
template <typename VectorType>
void
append_field_data(int64_t dim, int64_t size_per_chunk) {
insert_entity(int64_t dim, int64_t chunk_size) {
static_assert(std::is_base_of_v<VectorTrait, VectorType>);
field_datas_.emplace_back(std::make_unique<ConcurrentVector<VectorType>>(dim, size_per_chunk));
entity_vec_.emplace_back(std::make_unique<ConcurrentVector<VectorType>>(dim, chunk_size));
}
private:
std::vector<std::unique_ptr<VectorBase>> field_datas_;
std::vector<std::unique_ptr<VectorBase>> entity_vec_;
};
} // namespace milvus::segcore

14
internal/core/src/segcore/SealedIndexingRecord.h
View File

@ -31,30 +31,30 @@ using SealedIndexingEntryPtr = std::unique_ptr<SealedIndexingEntry>;
struct SealedIndexingRecord {
void
append_field_indexing(FieldOffset field_offset, MetricType metric_type, knowhere::VecIndexPtr indexing) {
add_entry(FieldOffset field_offset, MetricType metric_type, knowhere::VecIndexPtr indexing) {
auto ptr = std::make_unique<SealedIndexingEntry>();
ptr->indexing_ = indexing;
ptr->metric_type_ = metric_type;
std::unique_lock lck(mutex_);
field_indexings_[field_offset] = std::move(ptr);
entries_[field_offset] = std::move(ptr);
}
const SealedIndexingEntry*
get_field_indexing(FieldOffset field_offset) const {
get_entry(FieldOffset field_offset) const {
std::shared_lock lck(mutex_);
AssertInfo(field_indexings_.count(field_offset), "field_offset not found");
return field_indexings_.at(field_offset).get();
AssertInfo(entries_.count(field_offset), "field_offset not found");
return entries_.at(field_offset).get();
}
bool
is_ready(FieldOffset field_offset) const {
std::shared_lock lck(mutex_);
return field_indexings_.count(field_offset);
return entries_.count(field_offset);
}
private:
// field_offset -> SealedIndexingEntry
std::map<FieldOffset, SealedIndexingEntryPtr> field_indexings_;
std::map<FieldOffset, SealedIndexingEntryPtr> entries_;
mutable std::shared_mutex mutex_;
};
} // namespace milvus::segcore

4
internal/core/src/segcore/SegmentGrowing.cpp
View File

@ -20,8 +20,8 @@ TestABI() {
}
std::unique_ptr<SegmentGrowing>
CreateGrowingSegment(SchemaPtr schema, int64_t size_per_chunk) {
auto segment = std::make_unique<SegmentGrowingImpl>(schema, size_per_chunk);
CreateGrowingSegment(SchemaPtr schema, int64_t chunk_size) {
auto segment = std::make_unique<SegmentGrowingImpl>(schema, chunk_size);
return segment;
}

2
internal/core/src/segcore/SegmentGrowing.h
View File

@ -80,7 +80,7 @@ class SegmentGrowing : public SegmentInternalInterface {
using SegmentGrowingPtr = std::unique_ptr<SegmentGrowing>;
SegmentGrowingPtr
CreateGrowingSegment(SchemaPtr schema, int64_t size_per_chunk = 32 * 1024);
CreateGrowingSegment(SchemaPtr schema, int64_t chunk_size = 32 * 1024);
} // namespace segcore
} // namespace milvus

16
internal/core/src/segcore/SegmentGrowingImpl.cpp
View File

@ -170,7 +170,7 @@ SegmentGrowingImpl::Insert(int64_t reserved_begin,
record_.uids_.set_data(reserved_begin, uids.data(), size);
for (int fid = 0; fid < schema_->size(); ++fid) {
auto field_offset = FieldOffset(fid);
record_.get_field_data_base(field_offset)->set_data_raw(reserved_begin, entities[fid].data(), size);
record_.get_base_entity(field_offset)->set_data_raw(reserved_begin, entities[fid].data(), size);
}
for (int i = 0; i < uids.size(); ++i) {
@ -180,7 +180,7 @@ SegmentGrowingImpl::Insert(int64_t reserved_begin,
}
record_.ack_responder_.AddSegment(reserved_begin, reserved_begin + size);
indexing_record_.UpdateResourceAck(record_.ack_responder_.GetAck() / size_per_chunk_, record_);
indexing_record_.UpdateResourceAck(record_.ack_responder_.GetAck() / chunk_size_, record_);
return Status::OK();
}
@ -231,9 +231,9 @@ SegmentGrowingImpl::Close() {
int64_t
SegmentGrowingImpl::GetMemoryUsageInBytes() const {
int64_t total_bytes = 0;
int64_t ins_n = upper_align(record_.reserved, size_per_chunk_);
int64_t ins_n = upper_align(record_.reserved, chunk_size_);
total_bytes += ins_n * (schema_->get_total_sizeof() + 16 + 1);
int64_t del_n = upper_align(deleted_record_.reserved, size_per_chunk_);
int64_t del_n = upper_align(deleted_record_.reserved, chunk_size_);
total_bytes += del_n * (16 * 2);
return total_bytes;
}
@ -245,20 +245,20 @@ SegmentGrowingImpl::LoadIndexing(const LoadIndexInfo& info) {
Assert(info.index_params.count("metric_type"));
auto metric_type_str = info.index_params.at("metric_type");
sealed_indexing_record_.append_field_indexing(field_offset, GetMetricType(metric_type_str), info.index);
sealed_indexing_record_.add_entry(field_offset, GetMetricType(metric_type_str), info.index);
return Status::OK();
}
SpanBase
SegmentGrowingImpl::chunk_data_impl(FieldOffset field_offset, int64_t chunk_id) const {
auto vec = get_insert_record().get_field_data_base(field_offset);
auto vec = get_insert_record().get_base_entity(field_offset);
return vec->get_span_base(chunk_id);
}
int64_t
SegmentGrowingImpl::num_chunk() const {
SegmentGrowingImpl::num_chunk_data() const {
auto size = get_insert_record().ack_responder_.GetAck();
return upper_div(size, size_per_chunk_);
return upper_div(size, chunk_size_);
}
void
SegmentGrowingImpl::vector_search(int64_t vec_count,

24
internal/core/src/segcore/SegmentGrowingImpl.h
View File

@ -27,7 +27,7 @@
#include "utils/Status.h"
#include "segcore/DeletedRecord.h"
#include "utils/EasyAssert.h"
#include "FieldIndexing.h"
#include "IndexingEntry.h"
#include "InsertRecord.h"
#include <utility>
#include <memory>
@ -89,18 +89,18 @@ class SegmentGrowingImpl : public SegmentGrowing {
// return count of index that has index, i.e., [0, num_chunk_index) have built index
int64_t
num_chunk_index(FieldOffset field_offset) const final {
num_chunk_index_safe(FieldOffset field_offset) const final {
return indexing_record_.get_finished_ack();
}
const knowhere::Index*
chunk_index_impl(FieldOffset field_offset, int64_t chunk_id) const final {
return indexing_record_.get_field_indexing(field_offset).get_chunk_indexing(chunk_id);
return indexing_record_.get_entry(field_offset).get_indexing(chunk_id);
}
int64_t
size_per_chunk() const final {
return size_per_chunk_;
return chunk_size_;
}
public:
@ -152,27 +152,27 @@ class SegmentGrowingImpl : public SegmentGrowing {
void
bulk_subscript(FieldOffset field_offset, const int64_t* seg_offsets, int64_t count, void* output) const override {
// TODO: support more types
auto vec_ptr = record_.get_field_data_base(field_offset);
auto vec_ptr = record_.get_base_entity(field_offset);
auto data_type = schema_->operator[](field_offset).get_data_type();
Assert(data_type == DataType::INT64);
bulk_subscript_impl<int64_t>(*vec_ptr, seg_offsets, count, output);
}
int64_t
num_chunk() const override;
num_chunk_data() const override;
Status
LoadIndexing(const LoadIndexInfo& info) override;
public:
friend std::unique_ptr<SegmentGrowing>
CreateGrowingSegment(SchemaPtr schema, int64_t size_per_chunk);
CreateGrowingSegment(SchemaPtr schema, int64_t chunk_size);
explicit SegmentGrowingImpl(SchemaPtr schema, int64_t size_per_chunk)
: size_per_chunk_(size_per_chunk),
explicit SegmentGrowingImpl(SchemaPtr schema, int64_t chunk_size)
: chunk_size_(chunk_size),
schema_(std::move(schema)),
record_(*schema_, size_per_chunk),
indexing_record_(*schema_, size_per_chunk) {
record_(*schema_, chunk_size),
indexing_record_(*schema_, chunk_size) {
}
void
@ -192,7 +192,7 @@ class SegmentGrowingImpl : public SegmentGrowing {
chunk_data_impl(FieldOffset field_offset, int64_t chunk_id) const override;
private:
int64_t size_per_chunk_;
int64_t chunk_size_;
SchemaPtr schema_;
std::atomic<SegmentState> state_ = SegmentState::Open;

19
internal/core/src/segcore/SegmentInterface.h
View File

@ -14,18 +14,15 @@
#include "common/Schema.h"
#include "query/Plan.h"
#include "common/Span.h"
#include "FieldIndexing.h"
#include "IndexingEntry.h"
#include <knowhere/index/vector_index/VecIndex.h>
#include "common/SystemProperty.h"
#include "query/PlanNode.h"
namespace milvus::segcore {
// common interface of SegmentSealed and SegmentGrowing
// used by C API
class SegmentInterface {
public:
// fill results according to target_entries in plan
void
FillTargetEntry(const query::Plan* plan, QueryResult& results) const;
@ -47,17 +44,14 @@ class SegmentInterface {
virtual ~SegmentInterface() = default;
protected:
// calculate output[i] = Vec[seg_offsets[i]}, where Vec binds to system_type
virtual void
bulk_subscript(SystemFieldType system_type, const int64_t* seg_offsets, int64_t count, void* output) const = 0;
// calculate output[i] = Vec[seg_offsets[i]}, where Vec binds to field_offset
virtual void
bulk_subscript(FieldOffset field_offset, const int64_t* seg_offsets, int64_t count, void* output) const = 0;
};
// internal API for DSL calculation
// only for implementation
class SegmentInternalInterface : public SegmentInterface {
public:
template <typename T>
@ -86,24 +80,21 @@ class SegmentInternalInterface : public SegmentInterface {
const BitsetView& bitset,
QueryResult& output) const = 0;
// count of chunk that has index available
virtual int64_t
num_chunk_index(FieldOffset field_offset) const = 0;
num_chunk_index_safe(FieldOffset field_offset) const = 0;
// count of chunks
virtual int64_t
num_chunk() const = 0;
num_chunk_data() const = 0;
// element size in each chunk
// return chunk_size for each chunk, renaming against confusion
virtual int64_t
size_per_chunk() const = 0;
protected:
// internal API: return chunk_data in span
// blob and row_count
virtual SpanBase
chunk_data_impl(FieldOffset field_offset, int64_t chunk_id) const = 0;
// internal API: return chunk_index in span, support scalar index only
virtual const knowhere::Index*
chunk_index_impl(FieldOffset field_offset, int64_t chunk_id) const = 0;
};

2
internal/core/src/segcore/SegmentSealed.h
View File

@ -27,6 +27,6 @@ class SegmentSealed : public SegmentInternalInterface {
using SegmentSealedPtr = std::unique_ptr<SegmentSealed>;
SegmentSealedPtr
CreateSealedSegment(SchemaPtr schema, int64_t size_per_chunk = 32 * 1024);
CreateSealedSegment(SchemaPtr schema, int64_t chunk_size = 32 * 1024);
} // namespace milvus::segcore

8
internal/core/src/segcore/SegmentSealedImpl.cpp
View File

@ -29,7 +29,7 @@ SegmentSealedImpl::LoadIndex(const LoadIndexInfo& info) {
row_count_opt_ = row_count;
}
Assert(!vec_indexings_.is_ready(field_offset));
vec_indexings_.append_field_indexing(field_offset, GetMetricType(metric_type_str), info.index);
vec_indexings_.add_entry(field_offset, GetMetricType(metric_type_str), info.index);
++ready_count_;
}
@ -77,13 +77,13 @@ SegmentSealedImpl::LoadFieldData(const LoadFieldDataInfo& info) {
}
int64_t
SegmentSealedImpl::num_chunk_index(FieldOffset field_offset) const {
SegmentSealedImpl::num_chunk_index_safe(FieldOffset field_offset) const {
// TODO: support scalar index
return 0;
}
int64_t
SegmentSealedImpl::num_chunk() const {
SegmentSealedImpl::num_chunk_data() const {
return 1;
}
@ -142,7 +142,7 @@ SegmentSealedImpl::vector_search(int64_t vec_count,
}
SegmentSealedPtr
CreateSealedSegment(SchemaPtr schema, int64_t size_per_chunk) {
CreateSealedSegment(SchemaPtr schema, int64_t chunk_size) {
return std::make_unique<SegmentSealedImpl>(schema);
}

6
internal/core/src/segcore/SegmentSealedImpl.h
View File

@ -37,12 +37,12 @@ class SegmentSealedImpl : public SegmentSealed {
public:
int64_t
num_chunk_index(FieldOffset field_offset) const override;
num_chunk_index_safe(FieldOffset field_offset) const override;
int64_t
num_chunk() const override;
num_chunk_data() const override;
// return size_per_chunk for each chunk, renaming against confusion
// return chunk_size for each chunk, renaming against confusion
int64_t
size_per_chunk() const override;

2
internal/core/unittest/test_sealed.cpp
View File

@ -277,7 +277,7 @@ TEST(Sealed, LoadFieldData) {
vec_info.index_params["metric_type"] = milvus::knowhere::Metric::L2;
segment->LoadIndex(vec_info);
}
ASSERT_EQ(segment->num_chunk(), 1);
ASSERT_EQ(segment->num_chunk_data(), 1);
auto chunk_span1 = segment->chunk_data<int64_t>(FieldOffset(1), 0);
auto chunk_span2 = segment->chunk_data<double>(FieldOffset(2), 0);
auto ref1 = dataset.get_col<int64_t>(1);

20
internal/core/unittest/test_span.cpp
View File

@ -19,38 +19,38 @@ TEST(Span, Naive) {
using namespace milvus::query;
using namespace milvus::segcore;
int64_t N = 1000 * 1000;
constexpr int64_t size_per_chunk = 32 * 1024;
constexpr int64_t chunk_size = 32 * 1024;
auto schema = std::make_shared<Schema>();
schema->AddDebugField("binaryvec", DataType::VECTOR_BINARY, 512, MetricType::METRIC_Jaccard);
schema->AddDebugField("age", DataType::FLOAT);
schema->AddDebugField("floatvec", DataType::VECTOR_FLOAT, 32, MetricType::METRIC_L2);
auto dataset = DataGen(schema, N);
auto segment = CreateGrowingSegment(schema, size_per_chunk);
auto segment = CreateGrowingSegment(schema, chunk_size);
segment->PreInsert(N);
segment->Insert(0, N, dataset.row_ids_.data(), dataset.timestamps_.data(), dataset.raw_);
auto vec_ptr = dataset.get_col<uint8_t>(0);
auto age_ptr = dataset.get_col<float>(1);
auto float_ptr = dataset.get_col<float>(2);
SegmentInternalInterface& interface = *segment;
auto num_chunk = interface.num_chunk();
ASSERT_EQ(num_chunk, upper_div(N, size_per_chunk));
auto num_chunk = interface.num_chunk_data();
ASSERT_EQ(num_chunk, upper_div(N, chunk_size));
auto row_count = interface.get_row_count();
ASSERT_EQ(N, row_count);
for (auto chunk_id = 0; chunk_id < num_chunk; ++chunk_id) {
auto vec_span = interface.chunk_data<BinaryVector>(FieldOffset(0), chunk_id);
auto age_span = interface.chunk_data<float>(FieldOffset(1), chunk_id);
auto float_span = interface.chunk_data<FloatVector>(FieldOffset(2), chunk_id);
auto begin = chunk_id * size_per_chunk;
auto end = std::min((chunk_id + 1) * size_per_chunk, N);
auto size_per_chunk = end - begin;
for (int i = 0; i < size_per_chunk * 512 / 8; ++i) {
auto begin = chunk_id * chunk_size;
auto end = std::min((chunk_id + 1) * chunk_size, N);
auto chunk_size = end - begin;
for (int i = 0; i < chunk_size * 512 / 8; ++i) {
ASSERT_EQ(vec_span.data()[i], vec_ptr[i + begin * 512 / 8]);
}
for (int i = 0; i < size_per_chunk; ++i) {
for (int i = 0; i < chunk_size; ++i) {
ASSERT_EQ(age_span.data()[i], age_ptr[i + begin]);
}
for (int i = 0; i < size_per_chunk; ++i) {
for (int i = 0; i < chunk_size; ++i) {
ASSERT_EQ(float_span.data()[i], float_ptr[i + begin * 32]);
}
}

25
internal/datanode/collection_replica.go
View File

@ -20,7 +20,7 @@ type collectionReplica interface {
hasCollection(collectionID UniqueID) bool
// segment
addSegment(segmentID UniqueID, collName string, partitionName string) error
addSegment(segmentID UniqueID, collID UniqueID, partitionID UniqueID) error
removeSegment(segmentID UniqueID) error
hasSegment(segmentID UniqueID) bool
updateSegmentRowNums(segmentID UniqueID, numRows int64) error
@ -30,11 +30,11 @@ type collectionReplica interface {
type (
Segment struct {
segmentID UniqueID
collectionID UniqueID
partitionName string
numRows int64
memorySize int64
segmentID UniqueID
collectionID UniqueID
partitionID UniqueID
numRows int64
memorySize int64
}
collectionReplicaImpl struct {
@ -47,7 +47,6 @@ type (
//----------------------------------------------------------------------------------------------------- collection
func (colReplica *collectionReplicaImpl) getSegmentByID(segmentID UniqueID) (*Segment, error) {
// GOOSE TODO: read write lock
colReplica.mu.RLock()
defer colReplica.mu.RUnlock()
@ -59,19 +58,15 @@ func (colReplica *collectionReplicaImpl) getSegmentByID(segmentID UniqueID) (*Se
return nil, errors.Errorf("cannot find segment, id = %v", segmentID)
}
func (colReplica *collectionReplicaImpl) addSegment(segmentID UniqueID, collName string, partitionName string) error {
func (colReplica *collectionReplicaImpl) addSegment(segmentID UniqueID, collID UniqueID, partitionID UniqueID) error {
colReplica.mu.Lock()
defer colReplica.mu.Unlock()
log.Println("Add Segment", segmentID)
collID, err := colReplica.getCollectionIDByName(collName)
if err != nil {
return err
}
seg := &Segment{
segmentID: segmentID,
collectionID: collID,
partitionName: partitionName,
segmentID: segmentID,
collectionID: collID,
partitionID: partitionID,
}
colReplica.segments = append(colReplica.segments, seg)
return nil

54
internal/datanode/collection_replica_test.go
View File

@ -4,9 +4,8 @@ import (
"testing"
"github.com/golang/protobuf/proto"
"github.com/zilliztech/milvus-distributed/internal/proto/commonpb"
"github.com/zilliztech/milvus-distributed/internal/proto/etcdpb"
"github.com/zilliztech/milvus-distributed/internal/proto/schemapb"
"github.com/zilliztech/milvus-distributed/internal/datanode/factory"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
@ -21,54 +20,9 @@ func newReplica() collectionReplica {
return replica
}
func genTestCollectionMeta(collectionName string, collectionID UniqueID) *etcdpb.CollectionMeta {
fieldVec := schemapb.FieldSchema{
FieldID: UniqueID(100),
Name: "vec",
IsPrimaryKey: false,
DataType: schemapb.DataType_VECTOR_FLOAT,
TypeParams: []*commonpb.KeyValuePair{
{
Key: "dim",
Value: "16",
},
},
IndexParams: []*commonpb.KeyValuePair{
{
Key: "metric_type",
Value: "L2",
},
},
}
fieldInt := schemapb.FieldSchema{
FieldID: UniqueID(101),
Name: "age",
IsPrimaryKey: false,
DataType: schemapb.DataType_INT32,
}
schema := schemapb.CollectionSchema{
Name: collectionName,
AutoID: true,
Fields: []*schemapb.FieldSchema{
&fieldVec, &fieldInt,
},
}
collectionMeta := etcdpb.CollectionMeta{
ID: collectionID,
Schema: &schema,
CreateTime: Timestamp(0),
SegmentIDs: []UniqueID{0},
PartitionTags: []string{"default"},
}
return &collectionMeta
}
func initTestReplicaMeta(t *testing.T, replica collectionReplica, collectionName string, collectionID UniqueID, segmentID UniqueID) {
collectionMeta := genTestCollectionMeta(collectionName, collectionID)
Factory := &factory.MetaFactory{}
collectionMeta := Factory.CollectionMetaFactory(collectionID, collectionName)
schemaBlob := proto.MarshalTextString(collectionMeta.Schema)
require.NotEqual(t, "", schemaBlob)

12
internal/datanode/collection_test.go
View File

@ -5,12 +5,15 @@ import (
"github.com/golang/protobuf/proto"
"github.com/stretchr/testify/assert"
"github.com/zilliztech/milvus-distributed/internal/datanode/factory"
)
func TestCollection_newCollection(t *testing.T) {
collectionName := "collection0"
collectionID := UniqueID(0)
collectionMeta := genTestCollectionMeta(collectionName, collectionID)
collectionID := UniqueID(1)
Factory := &factory.MetaFactory{}
collectionMeta := Factory.CollectionMetaFactory(collectionID, collectionName)
schemaBlob := proto.MarshalTextString(collectionMeta.Schema)
assert.NotEqual(t, "", schemaBlob)
@ -22,8 +25,9 @@ func TestCollection_newCollection(t *testing.T) {
func TestCollection_deleteCollection(t *testing.T) {
collectionName := "collection0"
collectionID := UniqueID(0)
collectionMeta := genTestCollectionMeta(collectionName, collectionID)
collectionID := UniqueID(1)
Factory := &factory.MetaFactory{}
collectionMeta := Factory.CollectionMetaFactory(collectionID, collectionName)
schemaBlob := proto.MarshalTextString(collectionMeta.Schema)
assert.NotEqual(t, "", schemaBlob)

117
internal/datanode/data_node_test.go
View File

@ -10,15 +10,11 @@ import (
"testing"
"time"
"github.com/golang/protobuf/proto"
"github.com/stretchr/testify/require"
"go.etcd.io/etcd/clientv3"
"go.uber.org/zap"
etcdkv "github.com/zilliztech/milvus-distributed/internal/kv/etcd"
"github.com/zilliztech/milvus-distributed/internal/master"
"github.com/zilliztech/milvus-distributed/internal/proto/commonpb"
"github.com/zilliztech/milvus-distributed/internal/proto/etcdpb"
"github.com/zilliztech/milvus-distributed/internal/proto/schemapb"
)
func makeNewChannelNames(names []string, suffix string) []string {
@ -65,6 +61,7 @@ func startMaster(ctx context.Context) {
func TestMain(m *testing.M) {
Params.Init()
refreshChannelNames()
const ctxTimeInMillisecond = 2000
const closeWithDeadline = true
@ -104,93 +101,43 @@ func newDataNode() *DataNode {
svr := NewDataNode(ctx, 0)
return svr
}
func genTestDataNodeCollectionMeta(collectionName string, collectionID UniqueID, isBinary bool) *etcdpb.CollectionMeta {
var fieldVec schemapb.FieldSchema
if isBinary {
fieldVec = schemapb.FieldSchema{
FieldID: UniqueID(100),
Name: "vec",
IsPrimaryKey: false,
DataType: schemapb.DataType_VECTOR_BINARY,
TypeParams: []*commonpb.KeyValuePair{
{
Key: "dim",
Value: "128",
},
},
IndexParams: []*commonpb.KeyValuePair{
{
Key: "metric_type",
Value: "JACCARD",
},
},
}
} else {
fieldVec = schemapb.FieldSchema{
FieldID: UniqueID(100),
Name: "vec",
IsPrimaryKey: false,
DataType: schemapb.DataType_VECTOR_FLOAT,
TypeParams: []*commonpb.KeyValuePair{
{
Key: "dim",
Value: "16",
},
},
IndexParams: []*commonpb.KeyValuePair{
{
Key: "metric_type",
Value: "L2",
},
},
}
}
func newMetaTable() *metaTable {
etcdClient, _ := clientv3.New(clientv3.Config{Endpoints: []string{Params.EtcdAddress}})
fieldInt := schemapb.FieldSchema{
FieldID: UniqueID(101),
Name: "age",
IsPrimaryKey: false,
DataType: schemapb.DataType_INT32,
}
schema := schemapb.CollectionSchema{
Name: collectionName,
AutoID: true,
Fields: []*schemapb.FieldSchema{
&fieldVec, &fieldInt,
},
}
collectionMeta := etcdpb.CollectionMeta{
ID: collectionID,
Schema: &schema,
CreateTime: Timestamp(0),
SegmentIDs: []UniqueID{0},
PartitionTags: []string{"default"},
}
return &collectionMeta
etcdKV := etcdkv.NewEtcdKV(etcdClient, Params.MetaRootPath)
mt, _ := NewMetaTable(etcdKV)
return mt
}
func initTestMeta(t *testing.T, node *DataNode, collectionName string, collectionID UniqueID, segmentID UniqueID, optional ...bool) {
isBinary := false
if len(optional) > 0 {
isBinary = optional[0]
func clearEtcd(rootPath string) error {
etcdAddr := Params.EtcdAddress
etcdClient, err := clientv3.New(clientv3.Config{Endpoints: []string{etcdAddr}})
if err != nil {
return err
}
collectionMeta := genTestDataNodeCollectionMeta(collectionName, collectionID, isBinary)
etcdKV := etcdkv.NewEtcdKV(etcdClient, rootPath)
schemaBlob := proto.MarshalTextString(collectionMeta.Schema)
require.NotEqual(t, "", schemaBlob)
err = etcdKV.RemoveWithPrefix("writer/segment")
if err != nil {
return err
}
_, _, err = etcdKV.LoadWithPrefix("writer/segment")
if err != nil {
return err
}
log.Println("Clear ETCD with prefix writer/segment ")
var err = node.replica.addCollection(collectionMeta.ID, schemaBlob)
require.NoError(t, err)
err = etcdKV.RemoveWithPrefix("writer/ddl")
if err != nil {
return err
}
_, _, err = etcdKV.LoadWithPrefix("writer/ddl")
if err != nil {
return err
}
log.Println("Clear ETCD with prefix writer/ddl")
return nil
collection, err := node.replica.getCollectionByName(collectionName)
require.NoError(t, err)
require.Equal(t, collection.Name(), collectionName)
require.Equal(t, collection.ID(), collectionID)
require.Equal(t, node.replica.getCollectionNum(), 1)
}

182
internal/datanode/data_sync_service_test.go
View File

@ -4,21 +4,17 @@ import (
"context"
"encoding/binary"
"math"
"strconv"
"testing"
"time"
"github.com/golang/protobuf/proto"
"github.com/stretchr/testify/assert"
"go.etcd.io/etcd/clientv3"
etcdkv "github.com/zilliztech/milvus-distributed/internal/kv/etcd"
"github.com/zilliztech/milvus-distributed/internal/datanode/factory"
"github.com/zilliztech/milvus-distributed/internal/msgstream"
"github.com/zilliztech/milvus-distributed/internal/msgstream/pulsarms"
"github.com/zilliztech/milvus-distributed/internal/proto/commonpb"
"github.com/zilliztech/milvus-distributed/internal/proto/etcdpb"
"github.com/zilliztech/milvus-distributed/internal/proto/internalpb2"
"github.com/zilliztech/milvus-distributed/internal/proto/schemapb"
)
// NOTE: start pulsar before test
@ -38,7 +34,9 @@ func TestDataSyncService_Start(t *testing.T) {
// init data node
pulsarURL := Params.PulsarAddress
collMeta := newMeta()
Factory := &factory.MetaFactory{}
collMeta := Factory.CollectionMetaFactory(UniqueID(0), "coll1")
node := NewDataNode(ctx, 0)
node.replica.addCollection(collMeta.ID, proto.MarshalTextString(collMeta.Schema))
@ -213,175 +211,3 @@ func TestDataSyncService_Start(t *testing.T) {
<-ctx.Done()
}
func newMeta() *etcdpb.CollectionMeta {
ETCDAddr := Params.EtcdAddress
MetaRootPath := Params.MetaRootPath
cli, _ := clientv3.New(clientv3.Config{
Endpoints: []string{ETCDAddr},
DialTimeout: 5 * time.Second,
})
kvClient := etcdkv.NewEtcdKV(cli, MetaRootPath)
defer kvClient.Close()
sch := schemapb.CollectionSchema{
Name: "col1",
Description: "test collection",
AutoID: false,
Fields: []*schemapb.FieldSchema{
{
FieldID: 1,
Name: "Timestamp",
Description: "test collection filed 1",
DataType: schemapb.DataType_INT64,
TypeParams: []*commonpb.KeyValuePair{
{
Key: "col1_f1_tk2",
Value: "col1_f1_tv2",
},
},
},
{
FieldID: 0,
Name: "RowID",
Description: "test collection filed 1",
DataType: schemapb.DataType_INT64,
TypeParams: []*commonpb.KeyValuePair{
{
Key: "col1_f1_tk2",
Value: "col1_f1_tv2",
},
},
},
{
FieldID: 100,
Name: "col1_f1",
Description: "test collection filed 1",
DataType: schemapb.DataType_VECTOR_FLOAT,
TypeParams: []*commonpb.KeyValuePair{
{
Key: "dim",
Value: "2",
},
{
Key: "col1_f1_tk2",
Value: "col1_f1_tv2",
},
},
IndexParams: []*commonpb.KeyValuePair{
{
Key: "col1_f1_ik1",
Value: "col1_f1_iv1",
},
{
Key: "col1_f1_ik2",
Value: "col1_f1_iv2",
},
},
},
{
FieldID: 101,
Name: "col1_f2",
Description: "test collection filed 2",
DataType: schemapb.DataType_VECTOR_BINARY,
TypeParams: []*commonpb.KeyValuePair{
{
Key: "dim",
Value: "32",
},
{
Key: "col1_f2_tk2",
Value: "col1_f2_tv2",
},
},
IndexParams: []*commonpb.KeyValuePair{
{
Key: "col1_f2_ik1",
Value: "col1_f2_iv1",
},
{
Key: "col1_f2_ik2",
Value: "col1_f2_iv2",
},
},
},
{
FieldID: 102,
Name: "col1_f3",
Description: "test collection filed 3",
DataType: schemapb.DataType_BOOL,
TypeParams: []*commonpb.KeyValuePair{},
IndexParams: []*commonpb.KeyValuePair{},
},
{
FieldID: 103,
Name: "col1_f4",
Description: "test collection filed 3",
DataType: schemapb.DataType_INT8,
TypeParams: []*commonpb.KeyValuePair{},
IndexParams: []*commonpb.KeyValuePair{},
},
{
FieldID: 104,
Name: "col1_f5",
Description: "test collection filed 3",
DataType: schemapb.DataType_INT16,
TypeParams: []*commonpb.KeyValuePair{},
IndexParams: []*commonpb.KeyValuePair{},
},
{
FieldID: 105,
Name: "col1_f6",
Description: "test collection filed 3",
DataType: schemapb.DataType_INT32,
TypeParams: []*commonpb.KeyValuePair{},
IndexParams: []*commonpb.KeyValuePair{},
},
{
FieldID: 106,
Name: "col1_f7",
Description: "test collection filed 3",
DataType: schemapb.DataType_INT64,
TypeParams: []*commonpb.KeyValuePair{},
IndexParams: []*commonpb.KeyValuePair{},
},
{
FieldID: 107,
Name: "col1_f8",
Description: "test collection filed 3",
DataType: schemapb.DataType_FLOAT,
TypeParams: []*commonpb.KeyValuePair{},
IndexParams: []*commonpb.KeyValuePair{},
},
{
FieldID: 108,
Name: "col1_f9",
Description: "test collection filed 3",
DataType: schemapb.DataType_DOUBLE,
TypeParams: []*commonpb.KeyValuePair{},
IndexParams: []*commonpb.KeyValuePair{},
},
},
}
collection := etcdpb.CollectionMeta{
ID: UniqueID(1),
Schema: &sch,
CreateTime: Timestamp(1),
SegmentIDs: make([]UniqueID, 0),
PartitionTags: make([]string, 0),
}
collBytes := proto.MarshalTextString(&collection)
kvClient.Save("/collection/"+strconv.FormatInt(collection.ID, 10), collBytes)
segSch := etcdpb.SegmentMeta{
SegmentID: UniqueID(1),
CollectionID: UniqueID(1),
}
segBytes := proto.MarshalTextString(&segSch)
kvClient.Save("/segment/"+strconv.FormatInt(segSch.SegmentID, 10), segBytes)
return &collection
}

154
internal/datanode/factory/factory.go Normal file
View File

@ -0,0 +1,154 @@
package factory
import (
"github.com/zilliztech/milvus-distributed/internal/proto/commonpb"
"github.com/zilliztech/milvus-distributed/internal/proto/etcdpb"
"github.com/zilliztech/milvus-distributed/internal/proto/schemapb"
"github.com/zilliztech/milvus-distributed/internal/util/typeutil"
)
type (
UniqueID = typeutil.UniqueID
Timestamp = typeutil.Timestamp
Factory interface {
}
MetaFactory struct {
}
)
func (mf *MetaFactory) CollectionMetaFactory(collectionID UniqueID, collectionName string) *etcdpb.CollectionMeta {
sch := schemapb.CollectionSchema{
Name: collectionName,
Description: "test collection by meta factory",
AutoID: true,
Fields: []*schemapb.FieldSchema{
{
FieldID: 0,
Name: "RowID",
Description: "RowID field",
DataType: schemapb.DataType_INT64,
TypeParams: []*commonpb.KeyValuePair{
{
Key: "f0_tk1",
Value: "f0_tv1",
},
},
},
{
FieldID: 1,
Name: "Timestamp",
Description: "Timestamp field",
DataType: schemapb.DataType_INT64,
TypeParams: []*commonpb.KeyValuePair{
{
Key: "f1_tk1",
Value: "f1_tv1",
},
},
},
{
FieldID: 100,
Name: "float_vector_field",
Description: "field 100",
DataType: schemapb.DataType_VECTOR_FLOAT,
TypeParams: []*commonpb.KeyValuePair{
{
Key: "dim",
Value: "2",
},
},
IndexParams: []*commonpb.KeyValuePair{
{
Key: "indexkey",
Value: "indexvalue",
},
},
},
{
FieldID: 101,
Name: "binary_vector_field",
Description: "field 101",
DataType: schemapb.DataType_VECTOR_BINARY,
TypeParams: []*commonpb.KeyValuePair{
{
Key: "dim",
Value: "32",
},
},
IndexParams: []*commonpb.KeyValuePair{
{
Key: "indexkey",
Value: "indexvalue",
},
},
},
{
FieldID: 102,
Name: "bool_field",
Description: "field 102",
DataType: schemapb.DataType_BOOL,
TypeParams: []*commonpb.KeyValuePair{},
IndexParams: []*commonpb.KeyValuePair{},
},
{
FieldID: 103,
Name: "int8_field",
Description: "field 103",
DataType: schemapb.DataType_INT8,
TypeParams: []*commonpb.KeyValuePair{},
IndexParams: []*commonpb.KeyValuePair{},
},
{
FieldID: 104,
Name: "int16_field",
Description: "field 104",
DataType: schemapb.DataType_INT16,
TypeParams: []*commonpb.KeyValuePair{},
IndexParams: []*commonpb.KeyValuePair{},
},
{
FieldID: 105,
Name: "int32_field",
Description: "field 105",
DataType: schemapb.DataType_INT32,
TypeParams: []*commonpb.KeyValuePair{},
IndexParams: []*commonpb.KeyValuePair{},
},
{
FieldID: 106,
Name: "int64_field",
Description: "field 106",
DataType: schemapb.DataType_INT64,
TypeParams: []*commonpb.KeyValuePair{},
IndexParams: []*commonpb.KeyValuePair{},
},
{
FieldID: 107,
Name: "float32_field",
Description: "field 107",
DataType: schemapb.DataType_FLOAT,
TypeParams: []*commonpb.KeyValuePair{},
IndexParams: []*commonpb.KeyValuePair{},
},
{
FieldID: 108,
Name: "float64_field",
Description: "field 108",
DataType: schemapb.DataType_DOUBLE,
TypeParams: []*commonpb.KeyValuePair{},
IndexParams: []*commonpb.KeyValuePair{},
},
},
}
collection := etcdpb.CollectionMeta{
ID: collectionID,
Schema: &sch,
CreateTime: Timestamp(1),
SegmentIDs: make([]UniqueID, 0),
PartitionTags: make([]string, 0),
}
return &collection
}

14
internal/datanode/flow_graph_dd_node.go
View File

@ -120,8 +120,18 @@ func (ddNode *ddNode) Operate(in []*Msg) []*Msg {
select {
case fmsg := <-ddNode.inFlushCh:
log.Println(". receive flush message, flushing ...")
ddNode.flush()
ddNode.ddMsg.flushMessages = append(ddNode.ddMsg.flushMessages, fmsg)
localSegs := make([]UniqueID, 0)
for _, segID := range fmsg.segmentIDs {
if ddNode.replica.hasSegment(segID) {
localSegs = append(localSegs, segID)
}
}
if len(localSegs) > 0 {
ddNode.flush()
fmsg.segmentIDs = localSegs
ddNode.ddMsg.flushMessages = append(ddNode.ddMsg.flushMessages, fmsg)
}
default:
log.Println("..........default do nothing")
}

49
internal/datanode/flow_graph_dd_node_test.go
View File

@ -2,30 +2,18 @@ package datanode
import (
"context"
"log"
"testing"
"time"
"github.com/stretchr/testify/require"
"go.etcd.io/etcd/clientv3"
etcdkv "github.com/zilliztech/milvus-distributed/internal/kv/etcd"
"github.com/zilliztech/milvus-distributed/internal/msgstream"
"github.com/zilliztech/milvus-distributed/internal/proto/commonpb"
"github.com/zilliztech/milvus-distributed/internal/proto/internalpb2"
"github.com/zilliztech/milvus-distributed/internal/util/flowgraph"
)
func newMetaTable() *metaTable {
etcdClient, _ := clientv3.New(clientv3.Config{Endpoints: []string{Params.EtcdAddress}})
etcdKV := etcdkv.NewEtcdKV(etcdClient, Params.MetaRootPath)
mt, _ := NewMetaTable(etcdKV)
return mt
}
func TestFlowGraphDDNode_Operate(t *testing.T) {
newMeta()
const ctxTimeInMillisecond = 2000
const closeWithDeadline = false
var ctx context.Context
@ -94,7 +82,7 @@ func TestFlowGraphDDNode_Operate(t *testing.T) {
}
partitionID := UniqueID(100)
partitionTag := "partition-test-0"
partitionName := "partition-test-0"
// create partition
createPartitionReq := internalpb2.CreatePartitionRequest{
Base: &commonpb.MsgBase{
@ -106,7 +94,7 @@ func TestFlowGraphDDNode_Operate(t *testing.T) {
CollectionID: colID,
PartitionID: partitionID,
CollectionName: colName,
PartitionName: partitionTag,
PartitionName: partitionName,
}
createPartitionMsg := msgstream.CreatePartitionMsg{
BaseMsg: msgstream.BaseMsg{
@ -128,7 +116,7 @@ func TestFlowGraphDDNode_Operate(t *testing.T) {
CollectionID: colID,
PartitionID: partitionID,
CollectionName: colName,
PartitionName: partitionTag,
PartitionName: partitionName,
}
dropPartitionMsg := msgstream.DropPartitionMsg{
BaseMsg: msgstream.BaseMsg{
@ -155,34 +143,3 @@ func TestFlowGraphDDNode_Operate(t *testing.T) {
var inMsg Msg = msgStream
ddNode.Operate([]*Msg{&inMsg})
}
func clearEtcd(rootPath string) error {
etcdAddr := Params.EtcdAddress
etcdClient, err := clientv3.New(clientv3.Config{Endpoints: []string{etcdAddr}})
if err != nil {
return err
}
etcdKV := etcdkv.NewEtcdKV(etcdClient, rootPath)
err = etcdKV.RemoveWithPrefix("writer/segment")
if err != nil {
return err
}
_, _, err = etcdKV.LoadWithPrefix("writer/segment")
if err != nil {
return err
}
log.Println("Clear ETCD with prefix writer/segment ")
err = etcdKV.RemoveWithPrefix("writer/ddl")
if err != nil {
return err
}
_, _, err = etcdKV.LoadWithPrefix("writer/ddl")
if err != nil {
return err
}
log.Println("Clear ETCD with prefix writer/ddl")
return nil
}

99
internal/datanode/flow_graph_insert_buffer_node.go
View File

@ -25,7 +25,6 @@ import (
"github.com/zilliztech/milvus-distributed/internal/proto/internalpb2"
"github.com/zilliztech/milvus-distributed/internal/proto/schemapb"
"github.com/zilliztech/milvus-distributed/internal/storage"
"github.com/zilliztech/milvus-distributed/internal/util/typeutil"
)
const (
@ -50,6 +49,7 @@ type (
timeTickStream msgstream.MsgStream
segmentStatisticsStream msgstream.MsgStream
completeFlushStream msgstream.MsgStream
}
insertBuffer struct {
@ -109,14 +109,23 @@ func (ibNode *insertBufferNode) Operate(in []*Msg) []*Msg {
uniqueSeg := make(map[UniqueID]bool)
for _, msg := range iMsg.insertMessages {
currentSegID := msg.GetSegmentID()
collName := msg.GetCollectionName()
partitionName := msg.GetPartitionName()
collID := msg.GetCollectionID()
partitionID := msg.GetPartitionID()
if !ibNode.replica.hasSegment(currentSegID) {
err := ibNode.replica.addSegment(currentSegID, collName, partitionName)
err := ibNode.replica.addSegment(currentSegID, collID, partitionID)
if err != nil {
log.Println("Error: add segment error")
log.Println("Error: add segment error", err)
}
}
if !ibNode.flushMeta.hasSegmentFlush(currentSegID) {
err := ibNode.flushMeta.addSegmentFlush(currentSegID)
if err != nil {
log.Println("Error: add segment flush meta error", err)
}
}
err := ibNode.replica.updateSegmentRowNums(currentSegID, int64(len(msg.RowIDs)))
if err != nil {
log.Println("Error: update Segment Row number wrong, ", err)
@ -153,7 +162,7 @@ func (ibNode *insertBufferNode) Operate(in []*Msg) []*Msg {
continue
}
currentSegID := msg.GetSegmentID()
collectionName := msg.GetCollectionName()
collectionID := msg.GetCollectionID()
span.LogFields(oplog.Int("segment id", int(currentSegID)))
idata, ok := ibNode.insertBuffer.insertData[currentSegID]
@ -163,15 +172,14 @@ func (ibNode *insertBufferNode) Operate(in []*Msg) []*Msg {
}
}
// 1.1 Get CollectionMeta from etcd
collection, err := ibNode.replica.getCollectionByName(collectionName)
// 1.1 Get CollectionMeta
collection, err := ibNode.replica.getCollectionByID(collectionID)
if err != nil {
// GOOSE TODO add error handler
log.Println("bbb, Get meta wrong:", err)
continue
}
// collectionID := collection.ID() GOOSE TODO remove
collSchema := collection.schema
// 1.2 Get Fields
var pos int = 0 // Record position of blob
@ -418,7 +426,7 @@ func (ibNode *insertBufferNode) Operate(in []*Msg) []*Msg {
if ibNode.insertBuffer.full(currentSegID) {
log.Printf(". Insert Buffer full, auto flushing (%v) rows of data...", ibNode.insertBuffer.size(currentSegID))
err = ibNode.flushSegment(currentSegID, msg.GetPartitionName(), collection.ID())
err = ibNode.flushSegment(currentSegID, msg.GetPartitionID(), collection.ID())
if err != nil {
log.Printf("flush segment (%v) fail: %v", currentSegID, err)
}
@ -430,6 +438,7 @@ func (ibNode *insertBufferNode) Operate(in []*Msg) []*Msg {
var stopSign int = 0
for k := range ibNode.insertBuffer.insertData {
if stopSign >= 10 {
log.Printf("......")
break
}
log.Printf("seg(%v) buffer size = (%v)", k, ibNode.insertBuffer.size(k))
@ -440,7 +449,6 @@ func (ibNode *insertBufferNode) Operate(in []*Msg) []*Msg {
// iMsg is Flush() msg from dataservice
// 1. insertBuffer(not empty) -> binLogs -> minIO/S3
for _, msg := range iMsg.flushMessages {
// flushTs := msg.Timestamp
for _, currentSegID := range msg.segmentIDs {
log.Printf(". Receiving flush message segID(%v)...", currentSegID)
if ibNode.insertBuffer.size(currentSegID) > 0 {
@ -451,12 +459,11 @@ func (ibNode *insertBufferNode) Operate(in []*Msg) []*Msg {
continue
}
err = ibNode.flushSegment(currentSegID, seg.partitionName, seg.collectionID)
err = ibNode.flushSegment(currentSegID, seg.partitionID, seg.collectionID)
if err != nil {
log.Printf("flush segment (%v) fail: %v", currentSegID, err)
continue
}
}
}
}
@ -473,20 +480,15 @@ func (ibNode *insertBufferNode) Operate(in []*Msg) []*Msg {
return []*Msg{&res}
}
func (ibNode *insertBufferNode) flushSegment(segID UniqueID, partitionName string, collID UniqueID) error {
// partitionName -> partitionID GOOSE TODO: remove this
partitionID, err := typeutil.Hash32String(partitionName)
if err != nil {
return errors.Errorf("partitionName to partitionID wrong, %v", err)
}
func (ibNode *insertBufferNode) flushSegment(segID UniqueID, partitionID UniqueID, collID UniqueID) error {
coll, err := ibNode.replica.getCollectionByID(collID)
collSch, err := ibNode.getCollectionSchemaByID(collID)
if err != nil {
return errors.Errorf("Get collection by ID wrong, %v", err)
}
collMeta := &etcdpb.CollectionMeta{
Schema: coll.schema,
Schema: collSch,
ID: collID,
}
@ -505,7 +507,7 @@ func (ibNode *insertBufferNode) flushSegment(segID UniqueID, partitionName strin
log.Println(".. Clearing buffer")
// 1.5.2 binLogs -> minIO/S3
collIDStr := strconv.FormatInt(coll.ID(), 10)
collIDStr := strconv.FormatInt(collID, 10)
partitionIDStr := strconv.FormatInt(partitionID, 10)
segIDStr := strconv.FormatInt(segID, 10)
keyPrefix := path.Join(ibNode.minioPrefix, collIDStr, partitionIDStr, segIDStr)
@ -534,6 +536,29 @@ func (ibNode *insertBufferNode) flushSegment(segID UniqueID, partitionName strin
return nil
}
func (ibNode *insertBufferNode) completeFlush(segID UniqueID) error {
msgPack := msgstream.MsgPack{}
completeFlushMsg := internalpb2.SegmentFlushCompletedMsg{
Base: &commonpb.MsgBase{
MsgType: commonpb.MsgType_kSegmentFlushDone,
MsgID: 0, // GOOSE TODO
Timestamp: 0, // GOOSE TODO
SourceID: Params.DataNodeID,
},
SegmentID: segID,
}
var msg msgstream.TsMsg = &msgstream.FlushCompletedMsg{
BaseMsg: msgstream.BaseMsg{
HashValues: []uint32{0},
},
SegmentFlushCompletedMsg: completeFlushMsg,
}
msgPack.Msgs = append(msgPack.Msgs, msg)
return ibNode.timeTickStream.Produce(&msgPack)
}
func (ibNode *insertBufferNode) writeHardTimeTick(ts Timestamp) error {
msgPack := msgstream.MsgPack{}
timeTickMsg := msgstream.TimeTickMsg{
@ -545,8 +570,8 @@ func (ibNode *insertBufferNode) writeHardTimeTick(ts Timestamp) error {
TimeTickMsg: internalpb2.TimeTickMsg{
Base: &commonpb.MsgBase{
MsgType: commonpb.MsgType_kTimeTick,
MsgID: 0,
Timestamp: ts,
MsgID: 0, // GOOSE TODO
Timestamp: ts, // GOOSE TODO
SourceID: Params.DataNodeID,
},
},
@ -587,11 +612,7 @@ func (ibNode *insertBufferNode) updateSegStatistics(segIDs []UniqueID) error {
var msgPack = msgstream.MsgPack{
Msgs: []msgstream.TsMsg{msg},
}
err := ibNode.segmentStatisticsStream.Produce(&msgPack)
if err != nil {
return err
}
return nil
return ibNode.segmentStatisticsStream.Produce(&msgPack)
}
func (ibNode *insertBufferNode) getCollectionSchemaByID(collectionID UniqueID) (*schemapb.CollectionSchema, error) {
@ -602,14 +623,6 @@ func (ibNode *insertBufferNode) getCollectionSchemaByID(collectionID UniqueID) (
return ret.schema, nil
}
func (ibNode *insertBufferNode) getCollectionSchemaByName(collectionName string) (*schemapb.CollectionSchema, error) {
ret, err := ibNode.replica.getCollectionByName(collectionName)
if err != nil {
return nil, err
}
return ret.schema, nil
}
func newInsertBufferNode(ctx context.Context,
flushMeta *metaTable, replica collectionReplica) *insertBufferNode {
maxQueueLength := Params.FlowGraphMaxQueueLength
@ -655,14 +668,21 @@ func newInsertBufferNode(ctx context.Context,
wTt.SetPulsarClient(Params.PulsarAddress)
wTt.CreatePulsarProducers([]string{Params.TimeTickChannelName})
var wTtMsgStream msgstream.MsgStream = wTt
wTtMsgStream.Start()
wTtMsgStream.Start() // GOOSE TODO remove
// update statistics channel
segS := pulsarms.NewPulsarMsgStream(ctx, Params.SegmentStatisticsBufSize)
segS.SetPulsarClient(Params.PulsarAddress)
segS.CreatePulsarProducers([]string{Params.SegmentStatisticsChannelName})
var segStatisticsMsgStream msgstream.MsgStream = segS
segStatisticsMsgStream.Start()
segStatisticsMsgStream.Start() // GOOSE TODO remove
// segment flush completed channel
cf := pulsarms.NewPulsarMsgStream(ctx, 1024)
cf.SetPulsarClient(Params.PulsarAddress)
cf.CreatePulsarProducers([]string{Params.CompleteFlushChannelName})
var completeFlushStream msgstream.MsgStream = cf
completeFlushStream.Start() // GOOSE TODO remove
return &insertBufferNode{
BaseNode: baseNode,
@ -672,6 +692,7 @@ func newInsertBufferNode(ctx context.Context,
idAllocator: idAllocator,
timeTickStream: wTtMsgStream,
segmentStatisticsStream: segStatisticsMsgStream,
completeFlushStream: completeFlushStream,
replica: replica,
flushMeta: flushMeta,
}

7
internal/datanode/flow_graph_insert_buffer_node_test.go
View File

@ -12,6 +12,7 @@ import (
"github.com/golang/protobuf/proto"
"github.com/stretchr/testify/require"
"github.com/zilliztech/milvus-distributed/internal/datanode/factory"
"github.com/zilliztech/milvus-distributed/internal/msgstream"
"github.com/zilliztech/milvus-distributed/internal/proto/commonpb"
"github.com/zilliztech/milvus-distributed/internal/proto/internalpb2"
@ -37,7 +38,8 @@ func TestFlowGraphInsertBufferNode_Operate(t *testing.T) {
require.NoError(t, err)
Params.MetaRootPath = testPath
collMeta := newMeta()
Factory := &factory.MetaFactory{}
collMeta := Factory.CollectionMetaFactory(UniqueID(0), "coll1")
schemaBlob := proto.MarshalTextString(collMeta.Schema)
require.NotEqual(t, "", schemaBlob)
@ -54,7 +56,6 @@ func TestFlowGraphInsertBufferNode_Operate(t *testing.T) {
func genInsertMsg() insertMsg {
// test data generate
// GOOSE TODO orgnize
const DIM = 2
const N = 1
var rawData []byte
@ -164,6 +165,8 @@ func genInsertMsg() insertMsg {
},
CollectionName: "col1",
PartitionName: "default",
CollectionID: 0,
PartitionID: 1,
SegmentID: UniqueID(1),
ChannelID: "0",
Timestamps: []Timestamp{

32
internal/datanode/meta_table.go
View File

@ -14,7 +14,7 @@ import (
type metaTable struct {
client kv.TxnBase //
segID2FlushMeta map[UniqueID]*datapb.SegmentFlushMeta
collID2DdlMeta map[UniqueID]*datapb.DDLFlushMeta
collID2DdlMeta map[UniqueID]*datapb.DDLFlushMeta // GOOSE TODO: addDDLFlush and has DDLFlush
lock sync.RWMutex
}
@ -158,10 +158,7 @@ func (mt *metaTable) reloadSegMetaFromKV() error {
func (mt *metaTable) addSegmentFlush(segmentID UniqueID) error {
mt.lock.Lock()
defer mt.lock.Unlock()
_, ok := mt.segID2FlushMeta[segmentID]
if ok {
return errors.Errorf("segment already exists with ID = " + strconv.FormatInt(segmentID, 10))
}
meta := &datapb.SegmentFlushMeta{
IsFlushed: false,
SegmentID: segmentID,
@ -169,25 +166,12 @@ func (mt *metaTable) addSegmentFlush(segmentID UniqueID) error {
return mt.saveSegFlushMeta(meta)
}
// func (mt *metaTable) getFlushCloseTime(segmentID UniqueID) (Timestamp, error) {
// mt.lock.RLock()
// defer mt.lock.RUnlock()
// meta, ok := mt.segID2FlushMeta[segmentID]
// if !ok {
// return typeutil.ZeroTimestamp, errors.Errorf("segment not exists with ID = " + strconv.FormatInt(segmentID, 10))
// }
// return meta.CloseTime, nil
// }
// func (mt *metaTable) getFlushOpenTime(segmentID UniqueID) (Timestamp, error) {
// mt.lock.RLock()
// defer mt.lock.RUnlock()
// meta, ok := mt.segID2FlushMeta[segmentID]
// if !ok {
// return typeutil.ZeroTimestamp, errors.Errorf("segment not exists with ID = " + strconv.FormatInt(segmentID, 10))
// }
// return meta.OpenTime, nil
// }
func (mt *metaTable) hasSegmentFlush(segmentID UniqueID) bool {
mt.lock.RLock()
defer mt.lock.RUnlock()
_, ok := mt.segID2FlushMeta[segmentID]
return ok
}
func (mt *metaTable) checkFlushComplete(segmentID UniqueID) (bool, error) {
mt.lock.RLock()

13
internal/datanode/param_table.go
View File

@ -50,6 +50,9 @@ type ParamTable struct {
// - timetick channel -
TimeTickChannelName string
// - complete flush channel -
CompleteFlushChannelName string
// - channel subname -
MsgChannelSubName string
@ -113,6 +116,9 @@ func (p *ParamTable) Init() {
// - timetick channel -
p.initTimeTickChannelName()
// - flush completed channel -
p.initCompleteFlushChannelName()
// - channel subname -
p.initMsgChannelSubName()
@ -313,6 +319,13 @@ func (p *ParamTable) initSegmentStatisticsUpdateInterval() {
p.SegmentStatisticsUpdateInterval = p.ParseInt("dataNode.msgStream.segStatistics.updateInterval")
}
// - flush completed channel -
func (p *ParamTable) initCompleteFlushChannelName() {
// GOOSE TODO
p.CompleteFlushChannelName = "flush-completed"
}
// - Timetick channel -
func (p *ParamTable) initTimeTickChannelName() {
channels, err := p.Load("msgChannel.chanNamePrefix.dataNodeTimeTick")

41
internal/msgstream/msg.go
View File

@ -104,6 +104,47 @@ func (it *InsertMsg) Unmarshal(input []byte) (TsMsg, error) {
return insertMsg, nil
}
/////////////////////////////////////////FlushCompletedMsg//////////////////////////////////////////
// GOOSE TODO remove this
type FlushCompletedMsg struct {
BaseMsg
internalpb2.SegmentFlushCompletedMsg
}
func (fl *FlushCompletedMsg) Type() MsgType {
return fl.Base.MsgType
}
func (fl *FlushCompletedMsg) GetMsgContext() context.Context {
return fl.MsgCtx
}
func (fl *FlushCompletedMsg) SetMsgContext(ctx context.Context) {
fl.MsgCtx = ctx
}
func (fl *FlushCompletedMsg) Marshal(input TsMsg) ([]byte, error) {
flushCompletedMsgTask := input.(*FlushCompletedMsg)
flushCompletedMsg := &flushCompletedMsgTask.SegmentFlushCompletedMsg
mb, err := proto.Marshal(flushCompletedMsg)
if err != nil {
return nil, err
}
return mb, nil
}
func (fl *FlushCompletedMsg) Unmarshal(input []byte) (TsMsg, error) {
flushCompletedMsg := internalpb2.SegmentFlushCompletedMsg{}
err := proto.Unmarshal(input, &flushCompletedMsg)
if err != nil {
return nil, err
}
flushCompletedMsgTask := &FlushCompletedMsg{SegmentFlushCompletedMsg: flushCompletedMsg}
flushCompletedMsgTask.BeginTimestamp = flushCompletedMsgTask.Base.Timestamp
flushCompletedMsgTask.EndTimestamp = flushCompletedMsgTask.Base.Timestamp
return flushCompletedMsgTask, nil
}
/////////////////////////////////////////Flush//////////////////////////////////////////
// GOOSE TODO remove this
type FlushMsg struct {

1
internal/proto/common.proto
View File

@ -111,6 +111,7 @@ enum MsgType {
kRequestTSO = 1204;
kAllocateSegment = 1205;
kSegmentStatistics = 1206;
kSegmentFlushDone = 1207;
}
message MsgBase {

144
internal/proto/commonpb/common.pb.go
View File

@ -190,6 +190,7 @@ const (
MsgType_kRequestTSO MsgType = 1204
MsgType_kAllocateSegment MsgType = 1205
MsgType_kSegmentStatistics MsgType = 1206
MsgType_kSegmentFlushDone MsgType = 1207
)
var MsgType_name = map[int32]string{
@ -224,6 +225,7 @@ var MsgType_name = map[int32]string{
1204: "kRequestTSO",
1205: "kAllocateSegment",
1206: "kSegmentStatistics",
1207: "kSegmentFlushDone",
}
var MsgType_value = map[string]int32{
@ -258,6 +260,7 @@ var MsgType_value = map[string]int32{
"kRequestTSO": 1204,
"kAllocateSegment": 1205,
"kSegmentStatistics": 1206,
"kSegmentFlushDone": 1207,
}
func (x MsgType) String() string {
@ -598,74 +601,75 @@ func init() {
func init() { proto.RegisterFile("common.proto", fileDescriptor_555bd8c177793206) }
var fileDescriptor_555bd8c177793206 = []byte{
// 1101 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x74, 0x54, 0xdf, 0x6e, 0xdb, 0xb6,
0x1b, 0xad, 0x62, 0x27, 0x8e, 0x3e, 0xbb, 0x0e, 0xc3, 0x24, 0x8d, 0x7f, 0xbf, 0x65, 0x43, 0xe1,
0xab, 0xa2, 0x40, 0x93, 0x61, 0x03, 0xb6, 0xab, 0x02, 0x53, 0x24, 0x3a, 0x11, 0x2a, 0x4b, 0x2e,
0x25, 0x77, 0xed, 0x6e, 0x04, 0xd9, 0x66, 0x6d, 0x41, 0xb2, 0xe5, 0x89, 0x74, 0x37, 0xf7, 0x29,
0xb6, 0x3e, 0xc7, 0x06, 0xec, 0x3f, 0xf6, 0x08, 0xfb, 0xf7, 0x20, 0x7b, 0x80, 0xfd, 0xc3, 0x76,
0xb1, 0x81, 0x94, 0x65, 0x1b, 0x43, 0x77, 0xc7, 0xef, 0x1c, 0x7e, 0x47, 0xe7, 0x7c, 0x14, 0x09,
0x8d, 0x61, 0x36, 0x9d, 0x66, 0xb3, 0xf3, 0x79, 0x9e, 0x89, 0x0c, 0x1f, 0x4d, 0xe3, 0xf4, 0xd9,
0x82, 0x17, 0xd5, 0x79, 0x41, 0xb5, 0x6b, 0xb0, 0x4b, 0xa6, 0x73, 0xb1, 0x6c, 0x87, 0xb0, 0xe7,
0x8b, 0x48, 0x2c, 0x38, 0xbe, 0x0f, 0xc0, 0xf2, 0x3c, 0xcb, 0xc3, 0x61, 0x36, 0x62, 0x2d, 0xed,
0xb6, 0x76, 0xa7, 0xf9, 0xc6, 0x6b, 0xe7, 0x2f, 0x69, 0x3e, 0x27, 0x72, 0x9b, 0x99, 0x8d, 0x18,
0xd5, 0x59, 0xb9, 0xc4, 0xb7, 0x60, 0x2f, 0x67, 0x11, 0xcf, 0x66, 0xad, 0x9d, 0xdb, 0xda, 0x1d,
0x9d, 0xae, 0xaa, 0xf6, 0x5b, 0xd0, 0x78, 0xc0, 0x96, 0x8f, 0xa2, 0x74, 0xc1, 0x7a, 0x51, 0x9c,
0x63, 0x04, 0x95, 0x84, 0x2d, 0x95, 0xbe, 0x4e, 0xe5, 0x12, 0x1f, 0xc3, 0xee, 0x33, 0x49, 0xaf,
0x1a, 0x8b, 0xa2, 0x7d, 0x06, 0xd5, 0xcb, 0x34, 0x1b, 0x6c, 0x58, 0xd9, 0xd1, 0x28, 0xd9, 0x7b,
0x50, 0x33, 0x46, 0xa3, 0x9c, 0x71, 0x8e, 0x9b, 0xb0, 0x13, 0xcf, 0x57, 0x7a, 0x3b, 0xf1, 0x1c,
0x63, 0xa8, 0xce, 0xb3, 0x5c, 0x28, 0xb5, 0x0a, 0x55, 0xeb, 0xf6, 0x0b, 0x0d, 0x6a, 0x5d, 0x3e,
0xbe, 0x8c, 0x38, 0xc3, 0x6f, 0xc3, 0xfe, 0x94, 0x8f, 0x43, 0xb1, 0x9c, 0x97, 0x29, 0xcf, 0x5e,
0x9a, 0xb2, 0xcb, 0xc7, 0xc1, 0x72, 0xce, 0x68, 0x6d, 0x5a, 0x2c, 0xa4, 0x93, 0x29, 0x1f, 0xdb,
0xd6, 0x4a, 0xb9, 0x28, 0xf0, 0x19, 0xe8, 0x22, 0x9e, 0x32, 0x2e, 0xa2, 0xe9, 0xbc, 0x55, 0xb9,
0xad, 0xdd, 0xa9, 0xd2, 0x0d, 0x80, 0xff, 0x0f, 0xfb, 0x3c, 0x5b, 0xe4, 0x43, 0x66, 0x5b, 0xad,
0xaa, 0x6a, 0x5b, 0xd7, 0xed, 0xfb, 0xa0, 0x77, 0xf9, 0xf8, 0x9a, 0x45, 0x23, 0x96, 0xe3, 0xd7,
0xa1, 0x3a, 0x88, 0x78, 0xe1, 0xa8, 0xfe, 0xdf, 0x8e, 0x64, 0x02, 0xaa, 0x76, 0xde, 0xfd, 0xb6,
0x0a, 0xfa, 0xfa, 0x24, 0x70, 0x1d, 0x6a, 0x7e, 0xdf, 0x34, 0x89, 0xef, 0xa3, 0x1b, 0xf8, 0x18,
0x50, 0xdf, 0x25, 0x8f, 0x7b, 0xc4, 0x0c, 0x88, 0x15, 0x12, 0x4a, 0x3d, 0x8a, 0x34, 0x8c, 0xa1,
0x69, 0x7a, 0xae, 0x4b, 0xcc, 0x20, 0xec, 0x18, 0xb6, 0x43, 0x2c, 0xb4, 0x83, 0x4f, 0xe0, 0xb0,
0x47, 0x68, 0xd7, 0xf6, 0x7d, 0xdb, 0x73, 0x43, 0x8b, 0xb8, 0x36, 0xb1, 0x50, 0x05, 0xff, 0x0f,
0x4e, 0x4c, 0xcf, 0x71, 0x88, 0x19, 0x48, 0xd8, 0xf5, 0x82, 0x90, 0x3c, 0xb6, 0xfd, 0xc0, 0x47,
0x55, 0xa9, 0x6d, 0x3b, 0x0e, 0xb9, 0x32, 0x9c, 0xd0, 0xa0, 0x57, 0xfd, 0x2e, 0x71, 0x03, 0xb4,
0x2b, 0x75, 0x4a, 0xd4, 0xb2, 0xbb, 0xc4, 0x95, 0x72, 0xa8, 0x86, 0x6f, 0x01, 0x2e, 0x61, 0xdb,
0xb5, 0xc8, 0xe3, 0x30, 0x78, 0xd2, 0x23, 0x68, 0x1f, 0xbf, 0x02, 0xa7, 0x25, 0xbe, 0xfd, 0x1d,
0xa3, 0x4b, 0x90, 0x8e, 0x11, 0x34, 0x4a, 0x32, 0xf0, 0x7a, 0x0f, 0x10, 0x6c, 0xab, 0x53, 0xef,
0x5d, 0x4a, 0x4c, 0x8f, 0x5a, 0xa8, 0xbe, 0x0d, 0x3f, 0x22, 0x66, 0xe0, 0xd1, 0xd0, 0xb6, 0x50,
0x43, 0x9a, 0x2f, 0x61, 0x9f, 0x18, 0xd4, 0xbc, 0x0e, 0x29, 0xf1, 0xfb, 0x4e, 0x80, 0x6e, 0xca,
0x11, 0x74, 0x6c, 0x87, 0xa8, 0x44, 0x1d, 0xaf, 0xef, 0x5a, 0xa8, 0x89, 0x0f, 0xa0, 0xde, 0x25,
0x81, 0x51, 0xce, 0xe4, 0x40, 0x7e, 0xdf, 0x34, 0xcc, 0x6b, 0x52, 0x22, 0x08, 0xb7, 0xe0, 0xd8,
0x34, 0x5c, 0xd9, 0x64, 0x52, 0x62, 0x04, 0x24, 0xec, 0x78, 0x8e, 0x45, 0x28, 0x3a, 0x94, 0x01,
0xff, 0xc5, 0xd8, 0x0e, 0x41, 0x78, 0xab, 0xc3, 0x22, 0x0e, 0xd9, 0x74, 0x1c, 0x6d, 0x75, 0x94,
0x8c, 0xec, 0x38, 0x96, 0x61, 0x2e, 0xfb, 0xb6, 0x63, 0xad, 0x06, 0x55, 0x1c, 0xda, 0x09, 0x3e,
0x84, 0x9b, 0x65, 0x18, 0xd7, 0xb1, 0xfd, 0x00, 0xdd, 0xc2, 0xa7, 0x70, 0x54, 0x42, 0x5d, 0x12,
0x50, 0xdb, 0x2c, 0xa6, 0x7a, 0x2a, 0xf7, 0x7a, 0xfd, 0x20, 0xf4, 0x3a, 0x61, 0x97, 0x74, 0x3d,
0xfa, 0x04, 0xb5, 0xf0, 0x31, 0x1c, 0x58, 0x56, 0x48, 0xc9, 0xc3, 0x3e, 0xf1, 0x83, 0x90, 0x1a,
0x26, 0x41, 0x3f, 0xd7, 0xee, 0xba, 0x00, 0xf6, 0x6c, 0xc4, 0x3e, 0x94, 0x37, 0x9f, 0xe1, 0x7d,
0xa8, 0xba, 0x9e, 0x4b, 0xd0, 0x0d, 0xdc, 0x80, 0xfd, 0xbe, 0x6b, 0xfb, 0x7e, 0x9f, 0x58, 0x48,
0xc3, 0x4d, 0x00, 0xdb, 0xed, 0x51, 0xef, 0x8a, 0xca, 0xbf, 0x6a, 0x47, 0xb2, 0x1d, 0xdb, 0xb5,
0xfd, 0x6b, 0xf5, 0x8b, 0x00, 0xec, 0xad, 0xe6, 0x53, 0xbd, 0xfb, 0x77, 0x55, 0x5d, 0x2f, 0x75,
0x4b, 0x74, 0xd8, 0x4d, 0xdc, 0x6c, 0xc6, 0xd0, 0x0d, 0x19, 0x29, 0x31, 0x73, 0x16, 0x09, 0x66,
0x66, 0x69, 0xca, 0x86, 0x22, 0xce, 0x66, 0x68, 0x84, 0x8f, 0xe0, 0x20, 0xb1, 0xf2, 0x6c, 0xbe,
0x05, 0x32, 0x79, 0x32, 0xc9, 0x75, 0xc4, 0xb7, 0xb0, 0xa7, 0x32, 0x68, 0x62, 0x31, 0x3e, 0xcc,
0xe3, 0xc1, 0xb6, 0xc2, 0x58, 0xfe, 0x83, 0x89, 0x3f, 0xc9, 0x3e, 0xd8, 0x80, 0x1c, 0x4d, 0x94,
0xc4, 0x15, 0x13, 0xfe, 0x92, 0x9b, 0xd9, 0xec, 0x69, 0x3c, 0xe6, 0x28, 0xc6, 0x27, 0x80, 0x56,
0x16, 0x7a, 0x51, 0x2e, 0x62, 0xd5, 0xff, 0x9d, 0x86, 0x8f, 0xa0, 0xa9, 0x2c, 0x6c, 0xc0, 0xef,
0xe5, 0xfd, 0xb8, 0x29, 0x2d, 0x6c, 0xb0, 0x1f, 0x34, 0x7c, 0x0a, 0x78, 0x6d, 0x61, 0x43, 0xfc,
0xa8, 0xc9, 0xc1, 0x2a, 0x0b, 0x6b, 0x90, 0xa3, 0x9f, 0x34, 0x7c, 0x08, 0x0d, 0x85, 0xfa, 0x6c,
0x3c, 0x65, 0x33, 0x81, 0xfe, 0xd4, 0x94, 0x83, 0x52, 0xa1, 0x84, 0xff, 0x2a, 0x76, 0x16, 0xc6,
0xd4, 0x49, 0xa0, 0x4f, 0x76, 0x0a, 0x53, 0xab, 0x9d, 0x05, 0xf8, 0xa9, 0x1c, 0x7a, 0x2d, 0xb1,
0x67, 0x9c, 0xe5, 0x02, 0x7d, 0x54, 0x51, 0x95, 0xc5, 0x52, 0x26, 0x18, 0xfa, 0xb8, 0x82, 0xeb,
0xb0, 0x97, 0x74, 0xd2, 0x05, 0x9f, 0xa0, 0x17, 0x05, 0xe5, 0xb3, 0x28, 0x1f, 0x4e, 0xd0, 0x2f,
0x15, 0x95, 0xa5, 0xa8, 0x28, 0xe3, 0x8b, 0x54, 0xa0, 0x5f, 0x2b, 0x4a, 0xff, 0x8a, 0x89, 0xcd,
0xc9, 0xa3, 0xdf, 0x2a, 0xf8, 0x55, 0x68, 0x49, 0x70, 0x33, 0x49, 0xc9, 0xc4, 0x5c, 0xc4, 0x43,
0x8e, 0x7e, 0xaf, 0xe0, 0x33, 0x38, 0x95, 0xf4, 0x3a, 0xe5, 0x16, 0xfb, 0x47, 0x05, 0x37, 0x41,
0x4f, 0x82, 0x78, 0xca, 0x82, 0x78, 0x98, 0xa0, 0xcf, 0x74, 0x35, 0x94, 0x87, 0x0b, 0x96, 0x2f,
0xdd, 0x6c, 0xc4, 0xe4, 0x56, 0x8e, 0x3e, 0xd7, 0xf1, 0x01, 0x40, 0xe2, 0x64, 0xd1, 0xa8, 0xc8,
0xf4, 0x45, 0x01, 0x50, 0xf6, 0xfe, 0x82, 0x71, 0x61, 0x5b, 0xe8, 0x4b, 0x79, 0xe3, 0xeb, 0x25,
0x10, 0xf8, 0x1e, 0xfa, 0x4a, 0x57, 0x53, 0x33, 0xd2, 0x34, 0x1b, 0x46, 0x62, 0x3d, 0xb5, 0xaf,
0x75, 0x75, 0x1c, 0xab, 0x72, 0xcb, 0xc9, 0x37, 0xfa, 0xe5, 0xe5, 0x7b, 0xef, 0x8c, 0x63, 0x31,
0x59, 0x0c, 0xe4, 0x5b, 0x79, 0xf1, 0x3c, 0x4e, 0xd3, 0xf8, 0xb9, 0x60, 0xc3, 0xc9, 0x45, 0xf1,
0x8e, 0xde, 0x1b, 0xc5, 0x5c, 0xe4, 0xf1, 0x60, 0x21, 0xd8, 0xe8, 0x22, 0x9e, 0x09, 0x96, 0xcf,
0xa2, 0xf4, 0x42, 0x3d, 0xae, 0x17, 0xc5, 0xe3, 0x3a, 0x1f, 0x0c, 0xf6, 0x54, 0xfd, 0xe6, 0x3f,
0x01, 0x00, 0x00, 0xff, 0xff, 0x62, 0x56, 0xa9, 0xb4, 0x3f, 0x07, 0x00, 0x00,
// 1111 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x74, 0x54, 0xdb, 0x8e, 0xdb, 0x44,
0x18, 0xae, 0x37, 0xd9, 0xcd, 0xfa, 0xdf, 0x34, 0x3b, 0x3b, 0x7b, 0x0a, 0xb0, 0xa0, 0x2a, 0x57,
0x55, 0xa5, 0xee, 0x22, 0x90, 0xe0, 0xaa, 0x12, 0x5e, 0x7b, 0xb2, 0x6b, 0xd5, 0xb1, 0xd3, 0xb1,
0x53, 0x5a, 0x6e, 0x2c, 0x27, 0x99, 0x26, 0x96, 0x9d, 0x38, 0x78, 0x26, 0x85, 0xf4, 0x29, 0xa0,
0x37, 0xbc, 0x04, 0x48, 0x9c, 0xe1, 0x11, 0x38, 0x3d, 0x08, 0x0f, 0xc0, 0x49, 0x70, 0x83, 0x66,
0x1c, 0x27, 0x11, 0x2a, 0x77, 0xfe, 0xbf, 0x6f, 0xfe, 0xdf, 0xdf, 0xf7, 0xcd, 0x01, 0xea, 0x83,
0x6c, 0x32, 0xc9, 0xa6, 0xe7, 0xb3, 0x3c, 0x13, 0x19, 0x3e, 0x9c, 0xc4, 0xe9, 0xd3, 0x39, 0x2f,
0xaa, 0xf3, 0x82, 0x6a, 0xd5, 0x60, 0x9b, 0x4c, 0x66, 0x62, 0xd1, 0x0a, 0x61, 0xc7, 0x17, 0x91,
0x98, 0x73, 0x7c, 0x0f, 0x80, 0xe5, 0x79, 0x96, 0x87, 0x83, 0x6c, 0xc8, 0x9a, 0xda, 0x2d, 0xed,
0x76, 0xe3, 0x8d, 0xd7, 0xce, 0x5f, 0xd0, 0x7c, 0x4e, 0xe4, 0x32, 0x33, 0x1b, 0x32, 0xaa, 0xb3,
0xf2, 0x13, 0x9f, 0xc0, 0x4e, 0xce, 0x22, 0x9e, 0x4d, 0x9b, 0x5b, 0xb7, 0xb4, 0xdb, 0x3a, 0x5d,
0x56, 0xad, 0xb7, 0xa0, 0x7e, 0x9f, 0x2d, 0x1e, 0x46, 0xe9, 0x9c, 0x75, 0xa3, 0x38, 0xc7, 0x08,
0x2a, 0x09, 0x5b, 0xa8, 0xf9, 0x3a, 0x95, 0x9f, 0xf8, 0x08, 0xb6, 0x9f, 0x4a, 0x7a, 0xd9, 0x58,
0x14, 0xad, 0x33, 0xa8, 0x5e, 0xa6, 0x59, 0x7f, 0xcd, 0xca, 0x8e, 0x7a, 0xc9, 0xde, 0x85, 0x9a,
0x31, 0x1c, 0xe6, 0x8c, 0x73, 0xdc, 0x80, 0xad, 0x78, 0xb6, 0x9c, 0xb7, 0x15, 0xcf, 0x30, 0x86,
0xea, 0x2c, 0xcb, 0x85, 0x9a, 0x56, 0xa1, 0xea, 0xbb, 0xf5, 0x5c, 0x83, 0x5a, 0x87, 0x8f, 0x2e,
0x23, 0xce, 0xf0, 0xdb, 0xb0, 0x3b, 0xe1, 0xa3, 0x50, 0x2c, 0x66, 0xa5, 0xcb, 0xb3, 0x17, 0xba,
0xec, 0xf0, 0x51, 0xb0, 0x98, 0x31, 0x5a, 0x9b, 0x14, 0x1f, 0x52, 0xc9, 0x84, 0x8f, 0x6c, 0x6b,
0x39, 0xb9, 0x28, 0xf0, 0x19, 0xe8, 0x22, 0x9e, 0x30, 0x2e, 0xa2, 0xc9, 0xac, 0x59, 0xb9, 0xa5,
0xdd, 0xae, 0xd2, 0x35, 0x80, 0x5f, 0x86, 0x5d, 0x9e, 0xcd, 0xf3, 0x01, 0xb3, 0xad, 0x66, 0x55,
0xb5, 0xad, 0xea, 0xd6, 0x3d, 0xd0, 0x3b, 0x7c, 0x74, 0xcd, 0xa2, 0x21, 0xcb, 0xf1, 0xeb, 0x50,
0xed, 0x47, 0xbc, 0x50, 0xb4, 0xf7, 0xff, 0x8a, 0xa4, 0x03, 0xaa, 0x56, 0xde, 0xf9, 0xbe, 0x0a,
0xfa, 0x6a, 0x27, 0xf0, 0x1e, 0xd4, 0xfc, 0x9e, 0x69, 0x12, 0xdf, 0x47, 0x37, 0xf0, 0x11, 0xa0,
0x9e, 0x4b, 0x1e, 0x75, 0x89, 0x19, 0x10, 0x2b, 0x24, 0x94, 0x7a, 0x14, 0x69, 0x18, 0x43, 0xc3,
0xf4, 0x5c, 0x97, 0x98, 0x41, 0xd8, 0x36, 0x6c, 0x87, 0x58, 0x68, 0x0b, 0x1f, 0xc3, 0x41, 0x97,
0xd0, 0x8e, 0xed, 0xfb, 0xb6, 0xe7, 0x86, 0x16, 0x71, 0x6d, 0x62, 0xa1, 0x0a, 0x7e, 0x09, 0x8e,
0x4d, 0xcf, 0x71, 0x88, 0x19, 0x48, 0xd8, 0xf5, 0x82, 0x90, 0x3c, 0xb2, 0xfd, 0xc0, 0x47, 0x55,
0x39, 0xdb, 0x76, 0x1c, 0x72, 0x65, 0x38, 0xa1, 0x41, 0xaf, 0x7a, 0x1d, 0xe2, 0x06, 0x68, 0x5b,
0xce, 0x29, 0x51, 0xcb, 0xee, 0x10, 0x57, 0x8e, 0x43, 0x35, 0x7c, 0x02, 0xb8, 0x84, 0x6d, 0xd7,
0x22, 0x8f, 0xc2, 0xe0, 0x71, 0x97, 0xa0, 0x5d, 0xfc, 0x0a, 0x9c, 0x96, 0xf8, 0xe6, 0x7f, 0x8c,
0x0e, 0x41, 0x3a, 0x46, 0x50, 0x2f, 0xc9, 0xc0, 0xeb, 0xde, 0x47, 0xb0, 0x39, 0x9d, 0x7a, 0xef,
0x52, 0x62, 0x7a, 0xd4, 0x42, 0x7b, 0x9b, 0xf0, 0x43, 0x62, 0x06, 0x1e, 0x0d, 0x6d, 0x0b, 0xd5,
0xa5, 0xf8, 0x12, 0xf6, 0x89, 0x41, 0xcd, 0xeb, 0x90, 0x12, 0xbf, 0xe7, 0x04, 0xe8, 0xa6, 0x8c,
0xa0, 0x6d, 0x3b, 0x44, 0x39, 0x6a, 0x7b, 0x3d, 0xd7, 0x42, 0x0d, 0xbc, 0x0f, 0x7b, 0x1d, 0x12,
0x18, 0x65, 0x26, 0xfb, 0xf2, 0xff, 0xa6, 0x61, 0x5e, 0x93, 0x12, 0x41, 0xb8, 0x09, 0x47, 0xa6,
0xe1, 0xca, 0x26, 0x93, 0x12, 0x23, 0x20, 0x61, 0xdb, 0x73, 0x2c, 0x42, 0xd1, 0x81, 0x34, 0xf8,
0x1f, 0xc6, 0x76, 0x08, 0xc2, 0x1b, 0x1d, 0x16, 0x71, 0xc8, 0xba, 0xe3, 0x70, 0xa3, 0xa3, 0x64,
0x64, 0xc7, 0x91, 0x34, 0x73, 0xd9, 0xb3, 0x1d, 0x6b, 0x19, 0x54, 0xb1, 0x69, 0xc7, 0xf8, 0x00,
0x6e, 0x96, 0x66, 0x5c, 0xc7, 0xf6, 0x03, 0x74, 0x82, 0x4f, 0xe1, 0xb0, 0x84, 0x3a, 0x24, 0xa0,
0xb6, 0x59, 0xa4, 0x7a, 0x2a, 0xd7, 0x7a, 0xbd, 0x20, 0xf4, 0xda, 0x61, 0x87, 0x74, 0x3c, 0xfa,
0x18, 0x35, 0xf1, 0x11, 0xec, 0x5b, 0x56, 0x48, 0xc9, 0x83, 0x1e, 0xf1, 0x83, 0x90, 0x1a, 0x26,
0x41, 0xbf, 0xd6, 0xee, 0xb8, 0x00, 0xf6, 0x74, 0xc8, 0x3e, 0x94, 0x37, 0x9f, 0xe1, 0x5d, 0xa8,
0xba, 0x9e, 0x4b, 0xd0, 0x0d, 0x5c, 0x87, 0xdd, 0x9e, 0x6b, 0xfb, 0x7e, 0x8f, 0x58, 0x48, 0xc3,
0x0d, 0x00, 0xdb, 0xed, 0x52, 0xef, 0x8a, 0xca, 0x53, 0xb5, 0x25, 0xd9, 0xb6, 0xed, 0xda, 0xfe,
0xb5, 0x3a, 0x22, 0x00, 0x3b, 0xcb, 0x7c, 0xaa, 0x77, 0x3e, 0xd9, 0x56, 0xd7, 0x4b, 0xdd, 0x12,
0x1d, 0xb6, 0x13, 0x37, 0x9b, 0x32, 0x74, 0x43, 0x5a, 0x4a, 0xcc, 0x9c, 0x45, 0x82, 0x99, 0x59,
0x9a, 0xb2, 0x81, 0x88, 0xb3, 0x29, 0x1a, 0xe2, 0x43, 0xd8, 0x4f, 0xac, 0x3c, 0x9b, 0x6d, 0x80,
0x4c, 0xee, 0x4c, 0x72, 0x1d, 0xf1, 0x0d, 0xec, 0x89, 0x34, 0x9a, 0x58, 0x8c, 0x0f, 0xf2, 0xb8,
0xbf, 0x39, 0x61, 0x24, 0xcf, 0x60, 0xe2, 0x8f, 0xb3, 0x0f, 0xd6, 0x20, 0x47, 0x63, 0x35, 0xe2,
0x8a, 0x09, 0x7f, 0xc1, 0xcd, 0x6c, 0xfa, 0x24, 0x1e, 0x71, 0x14, 0xe3, 0x63, 0x40, 0x4b, 0x09,
0xdd, 0x28, 0x17, 0xb1, 0xea, 0xff, 0x41, 0xc3, 0x87, 0xd0, 0x50, 0x12, 0xd6, 0xe0, 0x8f, 0xf2,
0x7e, 0xdc, 0x94, 0x12, 0xd6, 0xd8, 0x4f, 0x1a, 0x3e, 0x05, 0xbc, 0x92, 0xb0, 0x26, 0x7e, 0xd6,
0x64, 0xb0, 0x4a, 0xc2, 0x0a, 0xe4, 0xe8, 0x17, 0x0d, 0x1f, 0x40, 0x5d, 0xa1, 0x3e, 0x1b, 0x4d,
0xd8, 0x54, 0xa0, 0xbf, 0x35, 0xa5, 0xa0, 0x9c, 0x50, 0xc2, 0xff, 0x14, 0x2b, 0x0b, 0x61, 0x6a,
0x27, 0xd0, 0xa7, 0x5b, 0x85, 0xa8, 0xe5, 0xca, 0x02, 0xfc, 0x4c, 0x86, 0x5e, 0x4b, 0xec, 0x29,
0x67, 0xb9, 0x40, 0x1f, 0x55, 0x54, 0x65, 0xb1, 0x94, 0x09, 0x86, 0x3e, 0xae, 0xe0, 0x3d, 0xd8,
0x49, 0xda, 0xe9, 0x9c, 0x8f, 0xd1, 0xf3, 0x82, 0xf2, 0x59, 0x94, 0x0f, 0xc6, 0xe8, 0xb7, 0x8a,
0xf2, 0x52, 0x54, 0x94, 0xf1, 0x79, 0x2a, 0xd0, 0xef, 0x15, 0x35, 0xff, 0x8a, 0x89, 0xf5, 0xce,
0xa3, 0x3f, 0x2a, 0xf8, 0x55, 0x68, 0x4a, 0x70, 0x9d, 0xa4, 0x64, 0x62, 0x2e, 0xe2, 0x01, 0x47,
0x7f, 0x56, 0xf0, 0x19, 0x9c, 0x4a, 0x7a, 0xe5, 0x72, 0x83, 0xfd, 0xab, 0x82, 0x1b, 0xa0, 0x27,
0x41, 0x3c, 0x61, 0x41, 0x3c, 0x48, 0xd0, 0xe7, 0xba, 0x0a, 0xe5, 0xc1, 0x9c, 0xe5, 0x0b, 0x37,
0x1b, 0x32, 0xb9, 0x94, 0xa3, 0x2f, 0x74, 0xbc, 0x0f, 0x90, 0x38, 0x59, 0x34, 0x2c, 0x3c, 0x7d,
0x59, 0x00, 0x94, 0xbd, 0x3f, 0x67, 0x5c, 0xd8, 0x16, 0xfa, 0x4a, 0xde, 0xf8, 0xbd, 0x12, 0x08,
0x7c, 0x0f, 0x7d, 0xad, 0xab, 0xd4, 0x8c, 0x34, 0xcd, 0x06, 0x91, 0x58, 0xa5, 0xf6, 0x8d, 0xae,
0xb6, 0x63, 0x59, 0x6e, 0x28, 0xf9, 0x56, 0xc7, 0x27, 0x70, 0x50, 0x12, 0x2a, 0x11, 0x4b, 0x9e,
0xc0, 0xef, 0xf4, 0xcb, 0xcb, 0xf7, 0xde, 0x19, 0xc5, 0x62, 0x3c, 0xef, 0xcb, 0x37, 0xf4, 0xe2,
0x59, 0x9c, 0xa6, 0xf1, 0x33, 0xc1, 0x06, 0xe3, 0x8b, 0xe2, 0x7d, 0xbd, 0x3b, 0x8c, 0xb9, 0xc8,
0xe3, 0xfe, 0x5c, 0xb0, 0xe1, 0x45, 0x3c, 0x15, 0x2c, 0x9f, 0x46, 0xe9, 0x85, 0x7a, 0x74, 0x2f,
0x8a, 0x47, 0x77, 0xd6, 0xef, 0xef, 0xa8, 0xfa, 0xcd, 0x7f, 0x03, 0x00, 0x00, 0xff, 0xff, 0xb5,
0x6e, 0xd8, 0x40, 0x57, 0x07, 0x00, 0x00,
}

5
internal/proto/data_service.proto
View File

@ -183,11 +183,6 @@ message DDLFlushMeta {
repeated string binlog_paths = 2;
}
message SegmentFlushCompletedMsg {
common.MsgBase base = 1;
int64 segmentID = 2;
}
service DataService {
rpc RegisterNode(RegisterNodeRequest) returns (RegisterNodeResponse) {}

239
internal/proto/datapb/data_service.pb.go
View File

@ -1516,53 +1516,6 @@ func (m *DDLFlushMeta) GetBinlogPaths() []string {
return nil
}
type SegmentFlushCompletedMsg struct {
Base *commonpb.MsgBase `protobuf:"bytes,1,opt,name=base,proto3" json:"base,omitempty"`
SegmentID int64 `protobuf:"varint,2,opt,name=segmentID,proto3" json:"segmentID,omitempty"`
XXX_NoUnkeyedLiteral struct{} `json:"-"`
XXX_unrecognized []byte `json:"-"`
XXX_sizecache int32 `json:"-"`
}
func (m *SegmentFlushCompletedMsg) Reset() { *m = SegmentFlushCompletedMsg{} }
func (m *SegmentFlushCompletedMsg) String() string { return proto.CompactTextString(m) }
func (*SegmentFlushCompletedMsg) ProtoMessage() {}
func (*SegmentFlushCompletedMsg) Descriptor() ([]byte, []int) {
return fileDescriptor_3385cd32ad6cfe64, []int{25}
}
func (m *SegmentFlushCompletedMsg) XXX_Unmarshal(b []byte) error {
return xxx_messageInfo_SegmentFlushCompletedMsg.Unmarshal(m, b)
}
func (m *SegmentFlushCompletedMsg) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
return xxx_messageInfo_SegmentFlushCompletedMsg.Marshal(b, m, deterministic)
}
func (m *SegmentFlushCompletedMsg) XXX_Merge(src proto.Message) {
xxx_messageInfo_SegmentFlushCompletedMsg.Merge(m, src)
}
func (m *SegmentFlushCompletedMsg) XXX_Size() int {
return xxx_messageInfo_SegmentFlushCompletedMsg.Size(m)
}
func (m *SegmentFlushCompletedMsg) XXX_DiscardUnknown() {
xxx_messageInfo_SegmentFlushCompletedMsg.DiscardUnknown(m)
}
var xxx_messageInfo_SegmentFlushCompletedMsg proto.InternalMessageInfo
func (m *SegmentFlushCompletedMsg) GetBase() *commonpb.MsgBase {
if m != nil {
return m.Base
}
return nil
}
func (m *SegmentFlushCompletedMsg) GetSegmentID() int64 {
if m != nil {
return m.SegmentID
}
return 0
}
func init() {
proto.RegisterEnum("milvus.proto.data.SegmentState", SegmentState_name, SegmentState_value)
proto.RegisterType((*RegisterNodeRequest)(nil), "milvus.proto.data.RegisterNodeRequest")
@ -1590,108 +1543,106 @@ func init() {
proto.RegisterType((*FieldFlushMeta)(nil), "milvus.proto.data.FieldFlushMeta")
proto.RegisterType((*SegmentFlushMeta)(nil), "milvus.proto.data.SegmentFlushMeta")
proto.RegisterType((*DDLFlushMeta)(nil), "milvus.proto.data.DDLFlushMeta")
proto.RegisterType((*SegmentFlushCompletedMsg)(nil), "milvus.proto.data.SegmentFlushCompletedMsg")
}
func init() { proto.RegisterFile("data_service.proto", fileDescriptor_3385cd32ad6cfe64) }
var fileDescriptor_3385cd32ad6cfe64 = []byte{
// 1512 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xc4, 0x58, 0xdb, 0x6e, 0x1b, 0x45,
0x18, 0xce, 0x7a, 0x7d, 0x88, 0x7f, 0x3b, 0x8e, 0x3b, 0x49, 0x13, 0xd7, 0x2d, 0x6d, 0xb2, 0xa8,
0x4d, 0x5a, 0x41, 0x82, 0x5a, 0x41, 0xe1, 0x06, 0xd1, 0xd4, 0x6d, 0x64, 0xb5, 0x89, 0xa2, 0x71,
0xa1, 0xa2, 0x37, 0xd6, 0xda, 0x3b, 0x71, 0xa6, 0xec, 0xc1, 0xec, 0x8c, 0x9b, 0x34, 0x37, 0x70,
0x05, 0x12, 0x42, 0x82, 0x6b, 0xb8, 0xe6, 0x05, 0xe0, 0x01, 0x78, 0x05, 0x1e, 0x83, 0xa7, 0x40,
0x68, 0x67, 0xc6, 0x7b, 0xb0, 0x37, 0xb1, 0x71, 0x5b, 0x7a, 0xe7, 0xf9, 0xf7, 0x9b, 0xff, 0x7c,
0x1a, 0x03, 0xb2, 0x4c, 0x6e, 0xb6, 0x19, 0xf1, 0x5f, 0xd0, 0x2e, 0xd9, 0xea, 0xfb, 0x1e, 0xf7,
0xd0, 0x05, 0x87, 0xda, 0x2f, 0x06, 0x4c, 0x9e, 0xb6, 0x02, 0x40, 0xbd, 0xdc, 0xf5, 0x1c, 0xc7,
0x73, 0x25, 0xa9, 0x5e, 0xa1, 0x2e, 0x27, 0xbe, 0x6b, 0xda, 0xea, 0x5c, 0x8e, 0x5f, 0x30, 0xbe,
0x81, 0x25, 0x4c, 0x7a, 0x94, 0x71, 0xe2, 0xef, 0x7b, 0x16, 0xc1, 0xe4, 0xeb, 0x01, 0x61, 0x1c,
0x7d, 0x00, 0xd9, 0x8e, 0xc9, 0x48, 0x4d, 0x5b, 0xd3, 0x36, 0x4b, 0xb7, 0xaf, 0x6c, 0x25, 0x84,
0x28, 0xf6, 0x7b, 0xac, 0xb7, 0x63, 0x32, 0x82, 0x05, 0x12, 0x7d, 0x04, 0x05, 0xd3, 0xb2, 0x7c,
0xc2, 0x58, 0x2d, 0x73, 0xce, 0xa5, 0x7b, 0x12, 0x83, 0x87, 0x60, 0xe3, 0x27, 0x0d, 0x96, 0x93,
0x1a, 0xb0, 0xbe, 0xe7, 0x32, 0x82, 0x76, 0xa0, 0x44, 0x5d, 0xca, 0xdb, 0x7d, 0xd3, 0x37, 0x1d,
0xa6, 0x34, 0x59, 0x4f, 0x32, 0x0d, 0x4d, 0x6b, 0xba, 0x94, 0x1f, 0x08, 0x20, 0x06, 0x1a, 0xfe,
0x46, 0x77, 0x20, 0xcf, 0xb8, 0xc9, 0x07, 0x43, 0x9d, 0x2e, 0xa7, 0xea, 0xd4, 0x12, 0x10, 0xac,
0xa0, 0xc6, 0x5f, 0x1a, 0x94, 0x5b, 0xa4, 0xd7, 0x6c, 0x0c, 0x9d, 0xb1, 0x0c, 0xb9, 0xae, 0x37,
0x70, 0xb9, 0xd0, 0x61, 0x01, 0xcb, 0x03, 0x5a, 0x83, 0x52, 0xf7, 0xc8, 0x74, 0x5d, 0x62, 0xef,
0x9b, 0x0e, 0x11, 0x02, 0x8a, 0x38, 0x4e, 0x42, 0x06, 0x94, 0xbb, 0x9e, 0x6d, 0x93, 0x2e, 0xa7,
0x9e, 0xdb, 0x6c, 0xd4, 0xf4, 0x35, 0x6d, 0x53, 0xc7, 0x09, 0x5a, 0xc0, 0xa5, 0x6f, 0xfa, 0x9c,
0x2a, 0x48, 0x56, 0x40, 0xe2, 0x24, 0x74, 0x19, 0x8a, 0xc1, 0x8d, 0xb6, 0x1b, 0x48, 0xc9, 0x09,
0x29, 0xf3, 0x01, 0x41, 0x88, 0xb8, 0x0e, 0x95, 0x10, 0x2b, 0x11, 0x79, 0x81, 0x58, 0x08, 0xa9,
0x01, 0xcc, 0xf8, 0x59, 0x03, 0x74, 0x8f, 0x31, 0xda, 0x73, 0x13, 0x86, 0xad, 0x40, 0xde, 0xf5,
0x2c, 0xd2, 0x6c, 0x08, 0xcb, 0x74, 0xac, 0x4e, 0x81, 0xc8, 0x3e, 0x21, 0x7e, 0xdb, 0xf7, 0xec,
0xa1, 0x61, 0xf3, 0x01, 0x01, 0x7b, 0x36, 0x41, 0x0f, 0x60, 0x81, 0xc5, 0x98, 0xb0, 0x9a, 0xbe,
0xa6, 0x6f, 0x96, 0x6e, 0x5f, 0xdb, 0x1a, 0x4b, 0xc4, 0xad, 0xb8, 0x30, 0x9c, 0xbc, 0x65, 0xfc,
0x99, 0x81, 0x45, 0xf1, 0x5d, 0xea, 0xe5, 0x10, 0x57, 0x38, 0x5a, 0x80, 0x94, 0x3a, 0xf2, 0x30,
0x85, 0xa3, 0xc3, 0x00, 0xe9, 0xf1, 0x00, 0x8d, 0xba, 0x3f, 0x3b, 0xd9, 0xfd, 0xb9, 0x71, 0xf7,
0x5f, 0x83, 0x12, 0x39, 0xe9, 0x53, 0x9f, 0xb4, 0x39, 0x55, 0xee, 0xcd, 0x62, 0x90, 0xa4, 0x27,
0xd4, 0x21, 0xb1, 0x1c, 0x2b, 0x4c, 0x9d, 0x63, 0xc9, 0xa0, 0xce, 0x4f, 0x0c, 0x6a, 0x31, 0x2d,
0xa8, 0xbf, 0x68, 0xb0, 0x94, 0x08, 0xaa, 0x2a, 0x9c, 0x7d, 0xa8, 0xb2, 0xa4, 0x63, 0x83, 0xea,
0x09, 0x62, 0x64, 0x9c, 0x15, 0xa3, 0x08, 0x8a, 0xc7, 0xee, 0xce, 0x56, 0x44, 0x27, 0x50, 0x7e,
0x68, 0x0f, 0xd8, 0xd1, 0xec, 0x0d, 0x05, 0x41, 0xd6, 0xea, 0x34, 0x1b, 0x42, 0xa8, 0x8e, 0xc5,
0xef, 0x69, 0x42, 0x6a, 0xfc, 0xa8, 0x01, 0x6a, 0x1d, 0x79, 0xc7, 0x2d, 0xd2, 0x13, 0x06, 0xcd,
0xac, 0xc0, 0xa8, 0xb0, 0xcc, 0xe4, 0xfc, 0xd1, 0xc7, 0xf2, 0xc7, 0x78, 0x0e, 0x4b, 0x09, 0x6d,
0x54, 0x90, 0xae, 0x02, 0x30, 0x49, 0x6a, 0x36, 0x64, 0x78, 0x74, 0x1c, 0xa3, 0xcc, 0xe6, 0xf4,
0x43, 0x58, 0x56, 0x72, 0x82, 0x0f, 0x84, 0xcd, 0x6e, 0xfb, 0x15, 0x28, 0x86, 0xca, 0x28, 0xc3,
0x23, 0x82, 0xf1, 0x4f, 0x06, 0x2e, 0x8e, 0x08, 0x52, 0x66, 0x7d, 0x08, 0xb9, 0x40, 0x17, 0x29,
0xaa, 0x72, 0x56, 0x53, 0x08, 0x2f, 0x62, 0x89, 0x0e, 0x8a, 0xac, 0xeb, 0x13, 0x93, 0xab, 0x22,
0xcb, 0xc8, 0x22, 0x93, 0x24, 0x51, 0x64, 0xd7, 0xa0, 0xc4, 0x88, 0x69, 0x13, 0x4b, 0x02, 0x74,
0x09, 0x90, 0x24, 0x01, 0x58, 0x87, 0xf2, 0x61, 0x90, 0x6f, 0x43, 0x44, 0x56, 0x20, 0x4a, 0x8a,
0x26, 0x20, 0x8f, 0x60, 0x91, 0x71, 0xd3, 0xe7, 0xed, 0xbe, 0xc7, 0x44, 0x74, 0x58, 0x2d, 0x97,
0x56, 0x16, 0xe1, 0x50, 0xd9, 0x63, 0xbd, 0x03, 0x05, 0xc5, 0x15, 0x71, 0x75, 0x78, 0x64, 0x68,
0x17, 0x16, 0x88, 0x6b, 0xc5, 0x58, 0xe5, 0xa7, 0x66, 0x55, 0x26, 0xae, 0x15, 0x31, 0x9a, 0xa5,
0x7d, 0x18, 0x14, 0x56, 0x9b, 0x2e, 0x23, 0x3e, 0xdf, 0xa1, 0xae, 0xed, 0xf5, 0x0e, 0x4c, 0x7e,
0xf4, 0xa6, 0x62, 0xfd, 0x9b, 0x06, 0x97, 0x46, 0x65, 0x45, 0xf1, 0xae, 0xc3, 0xfc, 0x21, 0x25,
0xb6, 0x15, 0x25, 0x71, 0x78, 0x46, 0x77, 0x21, 0xd7, 0x0f, 0xc0, 0xb5, 0x8c, 0x70, 0xcd, 0x59,
0xa3, 0xbb, 0xc5, 0x7d, 0xea, 0xf6, 0x1e, 0x53, 0xc6, 0xb1, 0xc4, 0xc7, 0x5c, 0xa2, 0x4f, 0xef,
0x92, 0x6f, 0x35, 0x58, 0x96, 0x7a, 0xde, 0x97, 0x93, 0xe1, 0xcd, 0x76, 0x9e, 0x94, 0x59, 0x6e,
0x38, 0x70, 0xf1, 0xa9, 0xc9, 0xbb, 0x47, 0x0d, 0xe7, 0x95, 0x55, 0x08, 0xc4, 0x45, 0x03, 0x4e,
0xba, 0xb0, 0x88, 0x13, 0x34, 0xe3, 0x57, 0x0d, 0x16, 0x45, 0x8f, 0x6d, 0x91, 0xde, 0xff, 0x6e,
0xec, 0x48, 0x03, 0xcb, 0x8e, 0x36, 0x30, 0xe3, 0xef, 0x0c, 0x94, 0x54, 0xa9, 0x37, 0xdd, 0x43,
0x2f, 0x99, 0x65, 0xda, 0x48, 0x96, 0xbd, 0x9e, 0x5e, 0x8b, 0x36, 0x60, 0x91, 0x8a, 0x14, 0x68,
0x2b, 0x47, 0x49, 0xc5, 0x8a, 0xb8, 0x42, 0xe3, 0x99, 0x21, 0xc6, 0xaf, 0xd7, 0x27, 0xae, 0x6c,
0x15, 0x39, 0xd1, 0x2a, 0xe6, 0x03, 0x42, 0x5a, 0xaf, 0xc9, 0x4f, 0xec, 0x35, 0x85, 0xf1, 0x5e,
0x73, 0x09, 0xe6, 0xdd, 0x81, 0xd3, 0xf6, 0xbd, 0x63, 0x26, 0xc6, 0xbb, 0x8e, 0x0b, 0xee, 0xc0,
0xc1, 0xde, 0x31, 0x0b, 0x3e, 0x39, 0xc4, 0x69, 0x33, 0x7a, 0x2a, 0xe7, 0xba, 0x8e, 0x0b, 0x0e,
0x71, 0x5a, 0xf4, 0x34, 0xd6, 0x3d, 0xe1, 0xbf, 0x74, 0x4f, 0xe3, 0x04, 0x40, 0x91, 0xf7, 0x58,
0x6f, 0x86, 0x14, 0xf8, 0x18, 0x0a, 0x2a, 0x12, 0x6a, 0xd8, 0x5c, 0x3d, 0x5b, 0x70, 0x10, 0x4b,
0x3c, 0x84, 0x07, 0xb3, 0x76, 0xe5, 0x7e, 0x18, 0xa3, 0x40, 0xa9, 0x57, 0x98, 0x39, 0xab, 0x50,
0xb0, 0x3a, 0x72, 0xdf, 0x91, 0x3b, 0x5e, 0xde, 0xea, 0x88, 0x7d, 0x68, 0x03, 0x16, 0xa3, 0x44,
0x90, 0x00, 0x5d, 0x00, 0x2a, 0x11, 0x59, 0x6c, 0x44, 0xdf, 0x6b, 0xb0, 0x3a, 0xa6, 0x8e, 0xea,
0x54, 0x77, 0xa5, 0x6f, 0x87, 0xab, 0xd0, 0x7a, 0xaa, 0x42, 0x8f, 0xc8, 0xcb, 0x2f, 0x4c, 0x7b,
0x40, 0x0e, 0x4c, 0xea, 0x4b, 0xef, 0xce, 0x38, 0x89, 0x7f, 0xd7, 0xe0, 0xe2, 0xc1, 0x30, 0x33,
0xdf, 0xb6, 0x5f, 0x52, 0x16, 0xca, 0x6c, 0xda, 0x42, 0xf9, 0x9d, 0x06, 0x2b, 0xa3, 0x4a, 0xbf,
0x15, 0xef, 0xed, 0x41, 0xe5, 0x61, 0x30, 0x45, 0x44, 0x77, 0xdb, 0x23, 0xdc, 0x44, 0x35, 0x28,
0xa8, 0xb9, 0xa2, 0x7a, 0xc7, 0xf0, 0x18, 0x14, 0x63, 0x47, 0x0c, 0xa6, 0x76, 0x34, 0x6c, 0x8a,
0xb8, 0xd4, 0x89, 0x86, 0x95, 0xf1, 0x83, 0x06, 0x55, 0x95, 0xbe, 0x11, 0xc7, 0xf3, 0xfb, 0xd1,
0x3b, 0x00, 0x94, 0xb5, 0x55, 0x45, 0x0b, 0xd5, 0xe7, 0x71, 0x91, 0xb2, 0x87, 0x92, 0x80, 0x3e,
0x81, 0xbc, 0x90, 0x3f, 0xdc, 0x20, 0xd6, 0x53, 0x0a, 0x26, 0x69, 0x01, 0x56, 0x17, 0x8c, 0xcf,
0xa1, 0xdc, 0x68, 0x3c, 0x8e, 0xf4, 0x18, 0xed, 0x7c, 0x5a, 0x4a, 0xe7, 0x9b, 0xc2, 0xc6, 0xe7,
0x50, 0x8b, 0x9b, 0x78, 0xdf, 0x73, 0xfa, 0x36, 0xe1, 0xc4, 0x9a, 0xad, 0x23, 0x9c, 0xbb, 0x12,
0xdc, 0xf2, 0xc4, 0xfb, 0x38, 0x6c, 0x43, 0x68, 0x31, 0xec, 0xf4, 0xfb, 0x9e, 0x4b, 0xaa, 0x73,
0x68, 0x49, 0x3c, 0xed, 0x24, 0x81, 0x3f, 0x38, 0xa1, 0x8c, 0x57, 0x35, 0x84, 0xa0, 0xa2, 0x88,
0xbb, 0xbe, 0x77, 0x4c, 0xdd, 0x5e, 0x35, 0x83, 0x2e, 0xc0, 0xc2, 0x90, 0x93, 0x68, 0xaf, 0x55,
0x3d, 0x06, 0x53, 0xce, 0xae, 0x66, 0x6f, 0xff, 0x51, 0x84, 0x52, 0xc3, 0xe4, 0x66, 0x4b, 0xfe,
0xf3, 0x81, 0x4c, 0x28, 0xc7, 0xff, 0x32, 0x40, 0x37, 0x52, 0xdc, 0x9f, 0xf2, 0xaf, 0x46, 0x7d,
0x63, 0x22, 0x4e, 0xa6, 0xbb, 0x31, 0x87, 0x76, 0x21, 0x27, 0xe4, 0xa3, 0xb4, 0x26, 0x1c, 0x7f,
0xd9, 0xd4, 0xcf, 0xcb, 0x68, 0x63, 0x0e, 0x75, 0x60, 0x31, 0x7c, 0xa4, 0xa9, 0xe4, 0xba, 0x9e,
0xc2, 0x72, 0xfc, 0x75, 0x5e, 0xbf, 0x31, 0x09, 0x16, 0x2a, 0xdb, 0x86, 0x72, 0xec, 0x8d, 0xc1,
0x52, 0x05, 0x8c, 0x3f, 0x89, 0x52, 0x05, 0xa4, 0xbc, 0x55, 0x8c, 0x39, 0xd4, 0x83, 0xea, 0x2e,
0xe1, 0x89, 0x95, 0x1f, 0x6d, 0x4c, 0x98, 0x4e, 0xc3, 0x8e, 0x57, 0xdf, 0x9c, 0x0c, 0x0c, 0x05,
0xf9, 0xb0, 0xbc, 0x4b, 0xf8, 0xd8, 0xbe, 0x89, 0x6e, 0xa5, 0xf0, 0x38, 0x63, 0x03, 0xae, 0xbf,
0x37, 0x05, 0x36, 0x2e, 0xd3, 0x84, 0x0b, 0xa1, 0xcc, 0x70, 0x43, 0xd8, 0x38, 0x93, 0x49, 0x72,
0xb7, 0xab, 0x4f, 0x5e, 0x6b, 0x85, 0x59, 0xab, 0xbb, 0x84, 0x27, 0x47, 0x13, 0x65, 0x9c, 0x76,
0x19, 0xba, 0x99, 0x22, 0x28, 0x7d, 0xa4, 0xd6, 0x6f, 0x4d, 0x03, 0x0d, 0xcd, 0xf2, 0x60, 0x65,
0x97, 0xf0, 0x44, 0x3f, 0x57, 0x22, 0xd3, 0x02, 0x92, 0x3a, 0xac, 0xea, 0x37, 0xa7, 0x40, 0x86,
0x02, 0x9f, 0x01, 0x12, 0x46, 0x3a, 0x7d, 0xcf, 0x8d, 0xd2, 0xa4, 0x9e, 0x5a, 0x1e, 0x0f, 0x9c,
0x3e, 0x7f, 0x39, 0x9a, 0x80, 0xa1, 0xef, 0x46, 0x78, 0x18, 0x73, 0xe8, 0xa9, 0xe0, 0x1d, 0xac,
0x56, 0x4f, 0x68, 0xf7, 0x2b, 0x15, 0x82, 0x73, 0x79, 0xbf, 0x9b, 0xfc, 0xa6, 0x0e, 0x32, 0x2a,
0x31, 0xa5, 0xbf, 0x14, 0x09, 0x17, 0x39, 0xe7, 0xf5, 0xb1, 0xde, 0xf9, 0xec, 0xd9, 0xa7, 0x3d,
0xca, 0x8f, 0x06, 0x9d, 0xe0, 0xf6, 0xf6, 0x29, 0xb5, 0x6d, 0x7a, 0xca, 0x49, 0xf7, 0x68, 0x5b,
0x5e, 0x78, 0xdf, 0xa2, 0x8c, 0xfb, 0xb4, 0x33, 0xe0, 0xc4, 0xda, 0x1e, 0x9a, 0xbe, 0x2d, 0x58,
0x6e, 0x07, 0x8e, 0xee, 0x77, 0x3a, 0x79, 0x71, 0xba, 0xf3, 0x6f, 0x00, 0x00, 0x00, 0xff, 0xff,
0x65, 0x67, 0x5c, 0x8f, 0xf8, 0x15, 0x00, 0x00,
// 1496 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xc4, 0x58, 0xdd, 0x6e, 0x1b, 0xc5,
0x1e, 0xcf, 0x7a, 0xfd, 0x11, 0xff, 0xed, 0x38, 0xee, 0x24, 0x4d, 0x52, 0xb7, 0xa7, 0x4d, 0xf6,
0xa8, 0x4d, 0x5a, 0x9d, 0x93, 0x1c, 0xb5, 0x3a, 0x14, 0x6e, 0x10, 0x4d, 0xdd, 0x46, 0x56, 0x9b,
0x28, 0x1a, 0x17, 0x2a, 0x7a, 0x63, 0xad, 0xbd, 0x13, 0x67, 0x60, 0x3f, 0xcc, 0xce, 0xb8, 0x49,
0x73, 0x03, 0x57, 0x20, 0x21, 0x24, 0xb8, 0x86, 0x6b, 0x5e, 0x00, 0x1e, 0x80, 0x57, 0xe0, 0x31,
0x78, 0x0a, 0x84, 0x76, 0x66, 0xf6, 0xcb, 0xde, 0xc4, 0xc6, 0x6d, 0xe9, 0x9d, 0xe7, 0xbf, 0xbf,
0xf9, 0x7f, 0x7f, 0x8d, 0x01, 0x59, 0x26, 0x37, 0x3b, 0x8c, 0xf8, 0x2f, 0x69, 0x8f, 0x6c, 0x0f,
0x7c, 0x8f, 0x7b, 0xe8, 0x92, 0x43, 0xed, 0x97, 0x43, 0x26, 0x4f, 0xdb, 0x01, 0xa0, 0x51, 0xed,
0x79, 0x8e, 0xe3, 0xb9, 0x92, 0xd4, 0xa8, 0x51, 0x97, 0x13, 0xdf, 0x35, 0x6d, 0x75, 0xae, 0x26,
0x2f, 0x18, 0x5f, 0xc2, 0x12, 0x26, 0x7d, 0xca, 0x38, 0xf1, 0x0f, 0x3c, 0x8b, 0x60, 0xf2, 0xc5,
0x90, 0x30, 0x8e, 0xfe, 0x07, 0xf9, 0xae, 0xc9, 0xc8, 0x9a, 0xb6, 0xae, 0x6d, 0x55, 0xee, 0x5e,
0xdb, 0x4e, 0x09, 0x51, 0xec, 0xf7, 0x59, 0x7f, 0xd7, 0x64, 0x04, 0x0b, 0x24, 0x7a, 0x0f, 0x4a,
0xa6, 0x65, 0xf9, 0x84, 0xb1, 0xb5, 0xdc, 0x05, 0x97, 0x1e, 0x48, 0x0c, 0x0e, 0xc1, 0xc6, 0xf7,
0x1a, 0x2c, 0xa7, 0x35, 0x60, 0x03, 0xcf, 0x65, 0x04, 0xed, 0x42, 0x85, 0xba, 0x94, 0x77, 0x06,
0xa6, 0x6f, 0x3a, 0x4c, 0x69, 0xb2, 0x91, 0x66, 0x1a, 0x99, 0xd6, 0x72, 0x29, 0x3f, 0x14, 0x40,
0x0c, 0x34, 0xfa, 0x8d, 0xee, 0x41, 0x91, 0x71, 0x93, 0x0f, 0x43, 0x9d, 0xae, 0x66, 0xea, 0xd4,
0x16, 0x10, 0xac, 0xa0, 0xc6, 0xef, 0x1a, 0x54, 0xdb, 0xa4, 0xdf, 0x6a, 0x86, 0xce, 0x58, 0x86,
0x42, 0xcf, 0x1b, 0xba, 0x5c, 0xe8, 0xb0, 0x80, 0xe5, 0x01, 0xad, 0x43, 0xa5, 0x77, 0x6c, 0xba,
0x2e, 0xb1, 0x0f, 0x4c, 0x87, 0x08, 0x01, 0x65, 0x9c, 0x24, 0x21, 0x03, 0xaa, 0x3d, 0xcf, 0xb6,
0x49, 0x8f, 0x53, 0xcf, 0x6d, 0x35, 0xd7, 0xf4, 0x75, 0x6d, 0x4b, 0xc7, 0x29, 0x5a, 0xc0, 0x65,
0x60, 0xfa, 0x9c, 0x2a, 0x48, 0x5e, 0x40, 0x92, 0x24, 0x74, 0x15, 0xca, 0xc1, 0x8d, 0x8e, 0x1b,
0x48, 0x29, 0x08, 0x29, 0xf3, 0x01, 0x41, 0x88, 0xb8, 0x09, 0xb5, 0x08, 0x2b, 0x11, 0x45, 0x81,
0x58, 0x88, 0xa8, 0x01, 0xcc, 0xf8, 0x41, 0x03, 0xf4, 0x80, 0x31, 0xda, 0x77, 0x53, 0x86, 0xad,
0x40, 0xd1, 0xf5, 0x2c, 0xd2, 0x6a, 0x0a, 0xcb, 0x74, 0xac, 0x4e, 0x81, 0xc8, 0x01, 0x21, 0x7e,
0xc7, 0xf7, 0xec, 0xd0, 0xb0, 0xf9, 0x80, 0x80, 0x3d, 0x9b, 0xa0, 0x47, 0xb0, 0xc0, 0x12, 0x4c,
0xd8, 0x9a, 0xbe, 0xae, 0x6f, 0x55, 0xee, 0xde, 0xd8, 0x1e, 0x4b, 0xc4, 0xed, 0xa4, 0x30, 0x9c,
0xbe, 0x65, 0xfc, 0x96, 0x83, 0x45, 0xf1, 0x5d, 0xea, 0xe5, 0x10, 0x57, 0x38, 0x5a, 0x80, 0x94,
0x3a, 0xf2, 0x30, 0x85, 0xa3, 0xa3, 0x00, 0xe9, 0xc9, 0x00, 0x8d, 0xba, 0x3f, 0x3f, 0xd9, 0xfd,
0x85, 0x71, 0xf7, 0xdf, 0x80, 0x0a, 0x39, 0x1d, 0x50, 0x9f, 0x74, 0x38, 0x55, 0xee, 0xcd, 0x63,
0x90, 0xa4, 0x67, 0xd4, 0x21, 0x89, 0x1c, 0x2b, 0x4d, 0x9d, 0x63, 0xe9, 0xa0, 0xce, 0x4f, 0x0c,
0x6a, 0x39, 0x2b, 0xa8, 0x3f, 0x6a, 0xb0, 0x94, 0x0a, 0xaa, 0x2a, 0x9c, 0x03, 0xa8, 0xb3, 0xb4,
0x63, 0x83, 0xea, 0x09, 0x62, 0x64, 0x9c, 0x17, 0xa3, 0x18, 0x8a, 0xc7, 0xee, 0xce, 0x56, 0x44,
0xa7, 0x50, 0x7d, 0x6c, 0x0f, 0xd9, 0xf1, 0xec, 0x0d, 0x05, 0x41, 0xde, 0xea, 0xb6, 0x9a, 0x42,
0xa8, 0x8e, 0xc5, 0xef, 0x69, 0x42, 0x6a, 0x7c, 0xa7, 0x01, 0x6a, 0x1f, 0x7b, 0x27, 0x6d, 0xd2,
0x17, 0x06, 0xcd, 0xac, 0xc0, 0xa8, 0xb0, 0xdc, 0xe4, 0xfc, 0xd1, 0xc7, 0xf2, 0xc7, 0xf8, 0x0c,
0x96, 0x52, 0xda, 0xa8, 0x20, 0x5d, 0x07, 0x60, 0x92, 0xd4, 0x6a, 0xca, 0xf0, 0xe8, 0x38, 0x41,
0x99, 0xcd, 0xe9, 0x47, 0xb0, 0xac, 0xe4, 0x04, 0x1f, 0x08, 0x9b, 0xdd, 0xf6, 0x6b, 0x50, 0x8e,
0x94, 0x51, 0x86, 0xc7, 0x04, 0xe3, 0xcf, 0x1c, 0x5c, 0x1e, 0x11, 0xa4, 0xcc, 0xfa, 0x3f, 0x14,
0x02, 0x5d, 0xa4, 0xa8, 0xda, 0x79, 0x4d, 0x21, 0xba, 0x88, 0x25, 0x3a, 0x28, 0xb2, 0x9e, 0x4f,
0x4c, 0xae, 0x8a, 0x2c, 0x27, 0x8b, 0x4c, 0x92, 0x44, 0x91, 0xdd, 0x80, 0x0a, 0x23, 0xa6, 0x4d,
0x2c, 0x09, 0xd0, 0x25, 0x40, 0x92, 0x04, 0x60, 0x03, 0xaa, 0x47, 0x41, 0xbe, 0x85, 0x88, 0xbc,
0x40, 0x54, 0x14, 0x4d, 0x40, 0x9e, 0xc0, 0x22, 0xe3, 0xa6, 0xcf, 0x3b, 0x03, 0x8f, 0x89, 0xe8,
0xb0, 0xb5, 0x42, 0x56, 0x59, 0x44, 0x43, 0x65, 0x9f, 0xf5, 0x0f, 0x15, 0x14, 0xd7, 0xc4, 0xd5,
0xf0, 0xc8, 0xd0, 0x1e, 0x2c, 0x10, 0xd7, 0x4a, 0xb0, 0x2a, 0x4e, 0xcd, 0xaa, 0x4a, 0x5c, 0x2b,
0x66, 0x34, 0x4b, 0xfb, 0x30, 0x28, 0xac, 0xb6, 0x5c, 0x46, 0x7c, 0xbe, 0x4b, 0x5d, 0xdb, 0xeb,
0x1f, 0x9a, 0xfc, 0xf8, 0x6d, 0xc5, 0xfa, 0x67, 0x0d, 0xae, 0x8c, 0xca, 0x8a, 0xe3, 0xdd, 0x80,
0xf9, 0x23, 0x4a, 0x6c, 0x2b, 0x4e, 0xe2, 0xe8, 0x8c, 0xee, 0x43, 0x61, 0x10, 0x80, 0xd7, 0x72,
0xc2, 0x35, 0xe7, 0x8d, 0xee, 0x36, 0xf7, 0xa9, 0xdb, 0x7f, 0x4a, 0x19, 0xc7, 0x12, 0x9f, 0x70,
0x89, 0x3e, 0xbd, 0x4b, 0xbe, 0xd2, 0x60, 0x59, 0xea, 0xf9, 0x50, 0x4e, 0x86, 0xb7, 0xdb, 0x79,
0x32, 0x66, 0xb9, 0xe1, 0xc0, 0xe5, 0xe7, 0x26, 0xef, 0x1d, 0x37, 0x9d, 0xd7, 0x56, 0x21, 0x10,
0x17, 0x0f, 0x38, 0xe9, 0xc2, 0x32, 0x4e, 0xd1, 0x8c, 0x9f, 0x34, 0x58, 0x14, 0x3d, 0xb6, 0x4d,
0xfa, 0xff, 0xb8, 0xb1, 0x23, 0x0d, 0x2c, 0x3f, 0xda, 0xc0, 0x8c, 0x3f, 0x72, 0x50, 0x51, 0xa5,
0xde, 0x72, 0x8f, 0xbc, 0x74, 0x96, 0x69, 0x23, 0x59, 0xf6, 0x66, 0x7a, 0x2d, 0xda, 0x84, 0x45,
0x2a, 0x52, 0xa0, 0xa3, 0x1c, 0x25, 0x15, 0x2b, 0xe3, 0x1a, 0x4d, 0x66, 0x86, 0x18, 0xbf, 0xde,
0x80, 0xb8, 0xb2, 0x55, 0x14, 0x44, 0xab, 0x98, 0x0f, 0x08, 0x59, 0xbd, 0xa6, 0x38, 0xb1, 0xd7,
0x94, 0xc6, 0x7b, 0xcd, 0x15, 0x98, 0x77, 0x87, 0x4e, 0xc7, 0xf7, 0x4e, 0x98, 0x18, 0xef, 0x3a,
0x2e, 0xb9, 0x43, 0x07, 0x7b, 0x27, 0x2c, 0xf8, 0xe4, 0x10, 0xa7, 0xc3, 0xe8, 0x99, 0x9c, 0xeb,
0x3a, 0x2e, 0x39, 0xc4, 0x69, 0xd3, 0xb3, 0x44, 0xf7, 0x84, 0xbf, 0xd3, 0x3d, 0x8d, 0x53, 0x00,
0x45, 0xde, 0x67, 0xfd, 0x19, 0x52, 0xe0, 0x7d, 0x28, 0xa9, 0x48, 0xa8, 0x61, 0x73, 0xfd, 0x7c,
0xc1, 0x41, 0x2c, 0x71, 0x08, 0x0f, 0x66, 0xed, 0xca, 0xc3, 0x28, 0x46, 0x81, 0x52, 0xaf, 0x31,
0x73, 0x56, 0xa1, 0x64, 0x75, 0xe5, 0xbe, 0x23, 0x77, 0xbc, 0xa2, 0xd5, 0x15, 0xfb, 0xd0, 0x26,
0x2c, 0xc6, 0x89, 0x20, 0x01, 0xba, 0x00, 0xd4, 0x62, 0xb2, 0xd8, 0x88, 0xbe, 0xd1, 0x60, 0x75,
0x4c, 0x1d, 0xd5, 0xa9, 0xee, 0x4b, 0xdf, 0x86, 0xab, 0xd0, 0x46, 0xa6, 0x42, 0x4f, 0xc8, 0xab,
0x4f, 0x4c, 0x7b, 0x48, 0x0e, 0x4d, 0xea, 0x4b, 0xef, 0xce, 0x38, 0x89, 0x7f, 0xd1, 0xe0, 0xf2,
0x61, 0x98, 0x99, 0xef, 0xda, 0x2f, 0x19, 0x0b, 0x65, 0x3e, 0x6b, 0xa1, 0xfc, 0x5a, 0x83, 0x95,
0x51, 0xa5, 0xdf, 0x89, 0xf7, 0xf6, 0xa1, 0xf6, 0x38, 0x98, 0x22, 0xa2, 0xbb, 0xed, 0x13, 0x6e,
0xa2, 0x35, 0x28, 0xa9, 0xb9, 0xa2, 0x7a, 0x47, 0x78, 0x0c, 0x8a, 0xb1, 0x2b, 0x06, 0x53, 0x27,
0x1e, 0x36, 0x65, 0x5c, 0xe9, 0xc6, 0xc3, 0xca, 0xf8, 0x56, 0x83, 0xba, 0x4a, 0xdf, 0x98, 0xe3,
0xc5, 0xfd, 0xe8, 0x5f, 0x00, 0x94, 0x75, 0x54, 0x45, 0x0b, 0xd5, 0xe7, 0x71, 0x99, 0xb2, 0xc7,
0x92, 0x80, 0x3e, 0x80, 0xa2, 0x90, 0x1f, 0x6e, 0x10, 0x1b, 0x19, 0x05, 0x93, 0xb6, 0x00, 0xab,
0x0b, 0xc6, 0xc7, 0x50, 0x6d, 0x36, 0x9f, 0xc6, 0x7a, 0x8c, 0x76, 0x3e, 0x2d, 0xa3, 0xf3, 0x4d,
0xb6, 0xf1, 0x8e, 0x27, 0xde, 0xac, 0x51, 0x6b, 0x40, 0x8b, 0x51, 0xf7, 0x3d, 0xf0, 0x5c, 0x52,
0x9f, 0x43, 0x4b, 0xe2, 0xb9, 0x25, 0x09, 0xfc, 0xd1, 0x29, 0x65, 0xbc, 0xae, 0x21, 0x04, 0x35,
0x45, 0xdc, 0xf3, 0xbd, 0x13, 0xea, 0xf6, 0xeb, 0x39, 0x74, 0x09, 0x16, 0x42, 0x4e, 0xa2, 0xe5,
0xd5, 0xf5, 0x04, 0x4c, 0x39, 0xa0, 0x9e, 0xbf, 0xfb, 0x6b, 0x19, 0x2a, 0x4d, 0x93, 0x9b, 0x6d,
0xf9, 0x6f, 0x04, 0x32, 0xa1, 0x9a, 0x7c, 0xc6, 0xa3, 0x5b, 0x19, 0x2e, 0xc9, 0xf8, 0xa7, 0xa1,
0xb1, 0x39, 0x11, 0x27, 0x53, 0xd0, 0x98, 0x43, 0x7b, 0x50, 0x10, 0xf2, 0x51, 0x56, 0x63, 0x4c,
0xbe, 0x36, 0x1a, 0x17, 0x65, 0x99, 0x31, 0x87, 0xba, 0xb0, 0x18, 0x3d, 0x9c, 0x54, 0xc0, 0x6f,
0x66, 0xb0, 0x1c, 0x7f, 0x31, 0x37, 0x6e, 0x4d, 0x82, 0x45, 0xca, 0x76, 0xa0, 0x9a, 0xd8, 0xfb,
0x59, 0xa6, 0x80, 0xf1, 0x67, 0x4a, 0xa6, 0x80, 0x8c, 0xf7, 0x83, 0x31, 0x87, 0xfa, 0x50, 0xdf,
0x23, 0x3c, 0xb5, 0x86, 0xa3, 0xcd, 0x09, 0x13, 0x23, 0xec, 0x42, 0x8d, 0xad, 0xc9, 0xc0, 0x48,
0x90, 0x0f, 0xcb, 0x7b, 0x84, 0x8f, 0xed, 0x80, 0xe8, 0x4e, 0x06, 0x8f, 0x73, 0xb6, 0xd2, 0xc6,
0x7f, 0xa6, 0xc0, 0x26, 0x65, 0x9a, 0x70, 0x29, 0x92, 0x19, 0x4d, 0xed, 0xcd, 0x73, 0x99, 0xa4,
0xf7, 0xad, 0xc6, 0xe4, 0x55, 0x53, 0x98, 0xb5, 0xba, 0x47, 0x78, 0x7a, 0x5c, 0x50, 0xc6, 0x69,
0x8f, 0xa1, 0xdb, 0x19, 0x82, 0xb2, 0xc7, 0x5c, 0xe3, 0xce, 0x34, 0xd0, 0xc8, 0x2c, 0x0f, 0x56,
0xf6, 0x08, 0x4f, 0xf5, 0x58, 0x25, 0x32, 0x2b, 0x20, 0x99, 0x03, 0xa4, 0x71, 0x7b, 0x0a, 0x64,
0x24, 0xf0, 0x05, 0x20, 0x61, 0xa4, 0x33, 0xf0, 0xdc, 0x38, 0x4d, 0x1a, 0x99, 0xe5, 0xf1, 0xc8,
0x19, 0xf0, 0x57, 0xa3, 0x09, 0x18, 0xf9, 0x6e, 0x84, 0x87, 0x31, 0x87, 0x9e, 0x0b, 0xde, 0xc1,
0xba, 0xf3, 0x8c, 0xf6, 0x3e, 0x57, 0x21, 0xb8, 0x90, 0xf7, 0xbf, 0xd3, 0xdf, 0xd4, 0x41, 0x46,
0x25, 0xa1, 0xf4, 0xa7, 0x22, 0xe1, 0x62, 0xe7, 0xbc, 0x39, 0xd6, 0xbb, 0x1f, 0xbd, 0xf8, 0xb0,
0x4f, 0xf9, 0xf1, 0xb0, 0x1b, 0xdc, 0xde, 0x39, 0xa3, 0xb6, 0x4d, 0xcf, 0x38, 0xe9, 0x1d, 0xef,
0xc8, 0x0b, 0xff, 0xb5, 0x28, 0xe3, 0x3e, 0xed, 0x0e, 0x39, 0xb1, 0x76, 0x42, 0xd3, 0x77, 0x04,
0xcb, 0x9d, 0xc0, 0xd1, 0x83, 0x6e, 0xb7, 0x28, 0x4e, 0xf7, 0xfe, 0x0a, 0x00, 0x00, 0xff, 0xff,
0x6d, 0x7e, 0x0b, 0xe1, 0x8c, 0x15, 0x00, 0x00,
}
// Reference imports to suppress errors if they are not otherwise used.

4
internal/proto/internal.proto
View File

@ -178,6 +178,10 @@ message SegmentStatistics {
repeated SegmentStatisticsUpdates SegStats = 2;
}
message SegmentFlushCompletedMsg {
common.MsgBase base = 1;
int64 segmentID = 2;
}
message IndexStats {
repeated common.KeyValuePair index_params = 1;

231
internal/proto/internalpb2/internal.pb.go
View File

@ -1335,6 +1335,53 @@ func (m *SegmentStatistics) GetSegStats() []*SegmentStatisticsUpdates {
return nil
}
type SegmentFlushCompletedMsg struct {
Base *commonpb.MsgBase `protobuf:"bytes,1,opt,name=base,proto3" json:"base,omitempty"`
SegmentID int64 `protobuf:"varint,2,opt,name=segmentID,proto3" json:"segmentID,omitempty"`
XXX_NoUnkeyedLiteral struct{} `json:"-"`
XXX_unrecognized []byte `json:"-"`
XXX_sizecache int32 `json:"-"`
}
func (m *SegmentFlushCompletedMsg) Reset() { *m = SegmentFlushCompletedMsg{} }
func (m *SegmentFlushCompletedMsg) String() string { return proto.CompactTextString(m) }
func (*SegmentFlushCompletedMsg) ProtoMessage() {}
func (*SegmentFlushCompletedMsg) Descriptor() ([]byte, []int) {
return fileDescriptor_41f4a519b878ee3b, []int{19}
}
func (m *SegmentFlushCompletedMsg) XXX_Unmarshal(b []byte) error {
return xxx_messageInfo_SegmentFlushCompletedMsg.Unmarshal(m, b)
}
func (m *SegmentFlushCompletedMsg) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
return xxx_messageInfo_SegmentFlushCompletedMsg.Marshal(b, m, deterministic)
}
func (m *SegmentFlushCompletedMsg) XXX_Merge(src proto.Message) {
xxx_messageInfo_SegmentFlushCompletedMsg.Merge(m, src)
}
func (m *SegmentFlushCompletedMsg) XXX_Size() int {
return xxx_messageInfo_SegmentFlushCompletedMsg.Size(m)
}
func (m *SegmentFlushCompletedMsg) XXX_DiscardUnknown() {
xxx_messageInfo_SegmentFlushCompletedMsg.DiscardUnknown(m)
}
var xxx_messageInfo_SegmentFlushCompletedMsg proto.InternalMessageInfo
func (m *SegmentFlushCompletedMsg) GetBase() *commonpb.MsgBase {
if m != nil {
return m.Base
}
return nil
}
func (m *SegmentFlushCompletedMsg) GetSegmentID() int64 {
if m != nil {
return m.SegmentID
}
return 0
}
type IndexStats struct {
IndexParams []*commonpb.KeyValuePair `protobuf:"bytes,1,rep,name=index_params,json=indexParams,proto3" json:"index_params,omitempty"`
NumRelatedSegments int64 `protobuf:"varint,2,opt,name=num_related_segments,json=numRelatedSegments,proto3" json:"num_related_segments,omitempty"`
@ -1347,7 +1394,7 @@ func (m *IndexStats) Reset() { *m = IndexStats{} }
func (m *IndexStats) String() string { return proto.CompactTextString(m) }
func (*IndexStats) ProtoMessage() {}
func (*IndexStats) Descriptor() ([]byte, []int) {
return fileDescriptor_41f4a519b878ee3b, []int{19}
return fileDescriptor_41f4a519b878ee3b, []int{20}
}
func (m *IndexStats) XXX_Unmarshal(b []byte) error {
@ -1395,7 +1442,7 @@ func (m *FieldStats) Reset() { *m = FieldStats{} }
func (m *FieldStats) String() string { return proto.CompactTextString(m) }
func (*FieldStats) ProtoMessage() {}
func (*FieldStats) Descriptor() ([]byte, []int) {
return fileDescriptor_41f4a519b878ee3b, []int{20}
return fileDescriptor_41f4a519b878ee3b, []int{21}
}
func (m *FieldStats) XXX_Unmarshal(b []byte) error {
@ -1451,7 +1498,7 @@ func (m *SegmentStats) Reset() { *m = SegmentStats{} }
func (m *SegmentStats) String() string { return proto.CompactTextString(m) }
func (*SegmentStats) ProtoMessage() {}
func (*SegmentStats) Descriptor() ([]byte, []int) {
return fileDescriptor_41f4a519b878ee3b, []int{21}
return fileDescriptor_41f4a519b878ee3b, []int{22}
}
func (m *SegmentStats) XXX_Unmarshal(b []byte) error {
@ -1513,7 +1560,7 @@ func (m *QueryNodeStats) Reset() { *m = QueryNodeStats{} }
func (m *QueryNodeStats) String() string { return proto.CompactTextString(m) }
func (*QueryNodeStats) ProtoMessage() {}
func (*QueryNodeStats) Descriptor() ([]byte, []int) {
return fileDescriptor_41f4a519b878ee3b, []int{22}
return fileDescriptor_41f4a519b878ee3b, []int{23}
}
func (m *QueryNodeStats) XXX_Unmarshal(b []byte) error {
@ -1568,7 +1615,7 @@ func (m *MsgPosition) Reset() { *m = MsgPosition{} }
func (m *MsgPosition) String() string { return proto.CompactTextString(m) }
func (*MsgPosition) ProtoMessage() {}
func (*MsgPosition) Descriptor() ([]byte, []int) {
return fileDescriptor_41f4a519b878ee3b, []int{23}
return fileDescriptor_41f4a519b878ee3b, []int{24}
}
func (m *MsgPosition) XXX_Unmarshal(b []byte) error {
@ -1631,6 +1678,7 @@ func init() {
proto.RegisterType((*LoadIndex)(nil), "milvus.proto.internal.LoadIndex")
proto.RegisterType((*SegmentStatisticsUpdates)(nil), "milvus.proto.internal.SegmentStatisticsUpdates")
proto.RegisterType((*SegmentStatistics)(nil), "milvus.proto.internal.SegmentStatistics")
proto.RegisterType((*SegmentFlushCompletedMsg)(nil), "milvus.proto.internal.SegmentFlushCompletedMsg")
proto.RegisterType((*IndexStats)(nil), "milvus.proto.internal.IndexStats")
proto.RegisterType((*FieldStats)(nil), "milvus.proto.internal.FieldStats")
proto.RegisterType((*SegmentStats)(nil), "milvus.proto.internal.SegmentStats")
@ -1641,90 +1689,91 @@ func init() {
func init() { proto.RegisterFile("internal.proto", fileDescriptor_41f4a519b878ee3b) }
var fileDescriptor_41f4a519b878ee3b = []byte{
// 1360 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xec, 0x58, 0x4d, 0x6f, 0x1b, 0x45,
0x18, 0x66, 0x6d, 0xc7, 0xb1, 0x5f, 0x3b, 0x69, 0xba, 0xf4, 0x63, 0x0b, 0x81, 0xba, 0xcb, 0x97,
0x01, 0x91, 0x54, 0x29, 0x42, 0x88, 0x4b, 0x9b, 0xc4, 0x2d, 0x5d, 0x35, 0x09, 0x61, 0x9d, 0x56,
0x6a, 0x2f, 0xab, 0xf1, 0xee, 0xc4, 0x1e, 0xba, 0x1f, 0xee, 0xcc, 0x6c, 0x53, 0xf7, 0xcc, 0x0d,
0xc1, 0x01, 0x89, 0x23, 0x17, 0x7e, 0x00, 0x3f, 0x01, 0x24, 0x4e, 0x48, 0xdc, 0x91, 0x90, 0xf8,
0x25, 0x9c, 0xd0, 0x7c, 0xec, 0xda, 0x4e, 0x36, 0x21, 0x35, 0x20, 0x84, 0xe0, 0xe6, 0x79, 0xe6,
0xdd, 0x77, 0xdf, 0xe7, 0x79, 0x9f, 0x79, 0x77, 0xd7, 0xb0, 0x48, 0x62, 0x8e, 0x69, 0x8c, 0xc2,
0x95, 0x21, 0x4d, 0x78, 0x62, 0x9e, 0x8f, 0x48, 0xf8, 0x38, 0x65, 0x6a, 0xb5, 0x92, 0x6d, 0xbe,
0xd0, 0xf4, 0x93, 0x28, 0x4a, 0x62, 0x05, 0xdb, 0xdf, 0x19, 0xb0, 0xb0, 0x99, 0x44, 0xc3, 0x24,
0xc6, 0x31, 0x77, 0xe2, 0xfd, 0xc4, 0xbc, 0x00, 0xd5, 0x38, 0x09, 0xb0, 0xd3, 0xb1, 0x8c, 0x96,
0xd1, 0x2e, 0xbb, 0x7a, 0x65, 0x9a, 0x50, 0xa1, 0x49, 0x88, 0xad, 0x52, 0xcb, 0x68, 0xd7, 0x5d,
0xf9, 0xdb, 0xbc, 0x0e, 0xc0, 0x38, 0xe2, 0xd8, 0xf3, 0x93, 0x00, 0x5b, 0xe5, 0x96, 0xd1, 0x5e,
0x5c, 0x6b, 0xad, 0x14, 0xde, 0x77, 0xa5, 0x2b, 0x02, 0x37, 0x93, 0x00, 0xbb, 0x75, 0x96, 0xfd,
0x34, 0x6f, 0x00, 0xe0, 0x27, 0x9c, 0x22, 0x8f, 0xc4, 0xfb, 0x89, 0x55, 0x69, 0x95, 0xdb, 0x8d,
0xb5, 0x2b, 0xd3, 0x09, 0x74, 0xb9, 0x77, 0xf0, 0xe8, 0x1e, 0x0a, 0x53, 0xbc, 0x8b, 0x08, 0x75,
0xeb, 0xf2, 0x22, 0x51, 0xae, 0xfd, 0x8b, 0x01, 0x67, 0x72, 0x02, 0xf2, 0x1e, 0xcc, 0xfc, 0x00,
0xe6, 0xe4, 0x2d, 0x24, 0x83, 0xc6, 0xda, 0xab, 0xc7, 0x54, 0x34, 0xc5, 0xdb, 0x55, 0x97, 0x98,
0x77, 0xe1, 0x79, 0x96, 0xf6, 0xfc, 0x6c, 0xcb, 0x93, 0x28, 0xb3, 0x4a, 0xb2, 0xb4, 0xd3, 0x65,
0x32, 0x27, 0x13, 0xe8, 0x92, 0xae, 0x41, 0x55, 0x64, 0x4a, 0x99, 0x54, 0xa9, 0xb1, 0xf6, 0x62,
0x21, 0xc9, 0xae, 0x0c, 0x71, 0x75, 0xa8, 0x7d, 0x0f, 0x6a, 0x3b, 0x42, 0x7c, 0xd1, 0x96, 0xf7,
0x60, 0x1e, 0x05, 0x01, 0xc5, 0x8c, 0x69, 0x56, 0xcb, 0x85, 0x19, 0xd6, 0x55, 0x8c, 0x9b, 0x05,
0x17, 0xb5, 0xcd, 0xfe, 0x04, 0xc0, 0x89, 0x09, 0xdf, 0x45, 0x14, 0x45, 0xec, 0xd8, 0x86, 0x77,
0xa0, 0xc9, 0x38, 0xa2, 0xdc, 0x1b, 0xca, 0x38, 0x2d, 0xc1, 0x29, 0xba, 0xd3, 0x90, 0x97, 0xa9,
0xec, 0xf6, 0x7d, 0x80, 0x2e, 0xa7, 0x24, 0xee, 0x6f, 0x11, 0xc6, 0xc5, 0xbd, 0x1e, 0x8b, 0x38,
0x41, 0xa2, 0xdc, 0xae, 0xbb, 0x7a, 0x35, 0x21, 0x4f, 0xe9, 0xf4, 0xf2, 0x5c, 0x87, 0xc6, 0x1e,
0x89, 0xf0, 0x1e, 0xf1, 0x1f, 0x6e, 0xb3, 0xbe, 0x79, 0x15, 0x2a, 0x3d, 0xc4, 0xf0, 0x89, 0xf2,
0x6c, 0xb3, 0xfe, 0x06, 0x62, 0xd8, 0x95, 0x91, 0xf6, 0xaf, 0x06, 0x5c, 0xdc, 0xa4, 0x58, 0x9a,
0x31, 0x0c, 0xb1, 0xcf, 0x49, 0x12, 0xbb, 0xf8, 0x51, 0x8a, 0x19, 0x7f, 0xf6, 0x6c, 0xe6, 0x45,
0x98, 0x0f, 0x7a, 0x5e, 0x8c, 0xa2, 0x4c, 0xec, 0x6a, 0xd0, 0xdb, 0x41, 0x11, 0x36, 0x5f, 0x87,
0x45, 0x3f, 0xcf, 0x2f, 0x10, 0xe9, 0x81, 0xba, 0x7b, 0x08, 0x15, 0xad, 0x0a, 0x7a, 0x4e, 0xc7,
0xaa, 0xc8, 0x36, 0xc8, 0xdf, 0xa6, 0x0d, 0xcd, 0x71, 0x94, 0xd3, 0xb1, 0xe6, 0xe4, 0xde, 0x14,
0x26, 0x44, 0x65, 0xfe, 0x00, 0x47, 0xc8, 0xaa, 0xb6, 0x8c, 0x76, 0xd3, 0xd5, 0x2b, 0xfb, 0x07,
0x03, 0xce, 0x77, 0x68, 0x32, 0xfc, 0x37, 0x93, 0xb3, 0x3f, 0x2f, 0xc1, 0x05, 0xd5, 0xa3, 0x5d,
0x44, 0x39, 0xf9, 0x9b, 0x58, 0xbc, 0x01, 0x67, 0xc6, 0x77, 0x55, 0x01, 0xc5, 0x34, 0x5e, 0x83,
0xc5, 0x61, 0x56, 0x87, 0x8a, 0xab, 0xc8, 0xb8, 0x85, 0x1c, 0x9d, 0x62, 0x3b, 0x77, 0x02, 0xdb,
0x6a, 0x41, 0x2b, 0x5b, 0xd0, 0xc8, 0x13, 0x39, 0x1d, 0x6b, 0x5e, 0x86, 0x4c, 0x42, 0xf6, 0x67,
0x25, 0x38, 0x27, 0x9a, 0xfa, 0xbf, 0x1a, 0x42, 0x8d, 0xef, 0x4b, 0x60, 0x2a, 0x77, 0x38, 0x71,
0x80, 0x9f, 0xfc, 0x93, 0x5a, 0xbc, 0x04, 0xb0, 0x4f, 0x70, 0x18, 0x4c, 0xea, 0x50, 0x97, 0xc8,
0x9f, 0xd2, 0xc0, 0x82, 0x79, 0x99, 0x24, 0xe7, 0x9f, 0x2d, 0xc5, 0x7c, 0x56, 0xcf, 0x4e, 0x3d,
0x9f, 0x6b, 0xa7, 0x9e, 0xcf, 0xf2, 0x32, 0x3d, 0x9f, 0xbf, 0x2d, 0xc3, 0x82, 0x13, 0x33, 0x4c,
0xf9, 0x7f, 0xd9, 0x48, 0xe6, 0x32, 0xd4, 0x19, 0xee, 0x47, 0xe2, 0x11, 0xde, 0xb1, 0x6a, 0x72,
0x7f, 0x0c, 0x88, 0x5d, 0x7f, 0x80, 0xe2, 0x18, 0x87, 0x4e, 0xc7, 0xaa, 0xab, 0xd6, 0xe6, 0x80,
0xf9, 0x32, 0x00, 0x27, 0x11, 0x66, 0x1c, 0x45, 0x43, 0x66, 0x41, 0xab, 0xdc, 0xae, 0xb8, 0x13,
0x88, 0x98, 0xcf, 0x34, 0x39, 0x70, 0x3a, 0xcc, 0x6a, 0xb4, 0xca, 0xe2, 0x01, 0xab, 0x56, 0xe6,
0xbb, 0x50, 0xa3, 0xc9, 0x81, 0x17, 0x20, 0x8e, 0xac, 0xa6, 0x6c, 0xde, 0xa5, 0x42, 0xb1, 0x37,
0xc2, 0xa4, 0xe7, 0xce, 0xd3, 0xe4, 0xa0, 0x83, 0x38, 0xb2, 0xbf, 0x36, 0x60, 0xa1, 0x8b, 0x11,
0xf5, 0x07, 0xb3, 0x37, 0xec, 0x4d, 0x58, 0xa2, 0x98, 0xa5, 0x21, 0xf7, 0xc6, 0xb4, 0x54, 0xe7,
0xce, 0x28, 0x7c, 0x33, 0x27, 0xb7, 0x0a, 0x73, 0x8f, 0x52, 0x4c, 0x47, 0xfa, 0xbd, 0xe5, 0x84,
0x0a, 0x55, 0x9c, 0xfd, 0xf3, 0x44, 0x7d, 0x22, 0x15, 0x9b, 0xa1, 0xbe, 0x59, 0x5e, 0x07, 0x0a,
0x49, 0x95, 0x8b, 0x49, 0x5d, 0x86, 0x46, 0x84, 0x39, 0x25, 0xbe, 0xc7, 0x47, 0xc3, 0xcc, 0x6b,
0xa0, 0xa0, 0xbd, 0xd1, 0x50, 0x1a, 0x6d, 0x40, 0x38, 0xb3, 0xe6, 0x5a, 0xe5, 0x76, 0xd3, 0x95,
0xbf, 0xed, 0x9f, 0x0c, 0x58, 0xe8, 0xe0, 0x10, 0x73, 0x3c, 0xbb, 0xf0, 0x05, 0x07, 0xa2, 0x54,
0x78, 0x20, 0xa6, 0x1c, 0x57, 0x3e, 0xd9, 0x71, 0x95, 0x23, 0x8e, 0xbb, 0x02, 0xcd, 0x21, 0x25,
0x11, 0xa2, 0x23, 0xef, 0x21, 0x1e, 0x29, 0x1a, 0xc2, 0xf0, 0x0a, 0xbb, 0x83, 0x47, 0xcc, 0xfe,
0xc6, 0x80, 0xda, 0xad, 0x30, 0x65, 0x83, 0x99, 0x5e, 0x9d, 0xa6, 0xcf, 0x4b, 0xe9, 0xf0, 0x79,
0x39, 0x7c, 0x26, 0xcb, 0x05, 0x67, 0xd2, 0x86, 0x66, 0x7e, 0x00, 0xf7, 0x50, 0x5f, 0x37, 0x61,
0x0a, 0xb3, 0x7f, 0x33, 0xa0, 0xbe, 0x95, 0xa0, 0x40, 0x0e, 0xf7, 0xbf, 0xbc, 0xca, 0x65, 0x18,
0xcf, 0xe7, 0x4c, 0xe3, 0xf1, 0xc0, 0x9e, 0x18, 0xbc, 0x95, 0xe9, 0xc1, 0x7b, 0x19, 0x1a, 0x44,
0x14, 0xe4, 0x0d, 0x11, 0x1f, 0x28, 0x71, 0xeb, 0x2e, 0x48, 0x68, 0x57, 0x20, 0x62, 0x32, 0x67,
0x01, 0x72, 0x32, 0x57, 0x4f, 0x3d, 0x99, 0x75, 0x12, 0x39, 0x99, 0x29, 0x58, 0x5d, 0x55, 0xab,
0x30, 0x3a, 0x61, 0x9c, 0xf8, 0xec, 0xee, 0x30, 0x90, 0x9f, 0x13, 0xcb, 0x50, 0xef, 0xe6, 0xc4,
0xd4, 0x6b, 0xfb, 0x18, 0x10, 0xf6, 0xd8, 0xc6, 0x51, 0x42, 0x47, 0x5d, 0xf2, 0x14, 0x6b, 0xde,
0x13, 0x88, 0xa0, 0xb6, 0x93, 0x46, 0x6e, 0x72, 0xc0, 0x74, 0x67, 0xb2, 0xa5, 0xfd, 0xa5, 0x01,
0x67, 0x8f, 0xdc, 0x74, 0x06, 0xe1, 0xef, 0x40, 0xad, 0x8b, 0xfb, 0x22, 0x45, 0xf6, 0xdd, 0xb0,
0x7a, 0xdc, 0x67, 0xe1, 0x31, 0x14, 0xdd, 0x3c, 0x81, 0xfd, 0xa9, 0x21, 0xbe, 0x57, 0x02, 0xfc,
0x44, 0x2e, 0x8f, 0xa8, 0x6b, 0xcc, 0xa2, 0xae, 0x79, 0x15, 0xce, 0xc5, 0x69, 0xe4, 0x51, 0x1c,
0x22, 0x8e, 0x03, 0x4f, 0xbb, 0x82, 0x69, 0xb5, 0xcc, 0x38, 0x8d, 0x5c, 0xb5, 0xa5, 0x0b, 0x64,
0xf6, 0x17, 0x06, 0xc0, 0x2d, 0x61, 0x01, 0x55, 0xc6, 0x61, 0x8f, 0x1b, 0x27, 0x3f, 0xbc, 0x4b,
0xd3, 0x1e, 0xda, 0xc8, 0x3c, 0xc4, 0xa4, 0x46, 0xe5, 0x22, 0x0e, 0xb9, 0x46, 0x63, 0xf2, 0xda,
0x66, 0x4a, 0x97, 0xaf, 0x0c, 0x68, 0x4e, 0xc8, 0xc7, 0xa6, 0xed, 0x6e, 0x1c, 0xb6, 0xbb, 0x1c,
0x7a, 0xc2, 0x03, 0x1e, 0x9b, 0xb0, 0x45, 0x34, 0xb6, 0xc5, 0x25, 0xa8, 0x49, 0x49, 0x26, 0x7c,
0x11, 0x2b, 0x5f, 0x98, 0x6f, 0xc3, 0x59, 0x8a, 0x7d, 0x1c, 0xf3, 0x70, 0xe4, 0x45, 0x49, 0x40,
0xf6, 0x09, 0x0e, 0xe4, 0xb1, 0xa8, 0xb9, 0x4b, 0xd9, 0xc6, 0xb6, 0xc6, 0xed, 0x1f, 0x0d, 0x58,
0xfc, 0x58, 0x3c, 0x0b, 0xc4, 0xc7, 0xab, 0xaa, 0xec, 0xd9, 0x1d, 0x74, 0x43, 0x72, 0xd1, 0xf2,
0x28, 0x0b, 0xbd, 0xf2, 0xc7, 0x16, 0x62, 0x6e, 0x8d, 0x69, 0xdb, 0x08, 0x89, 0xd5, 0x0b, 0xd9,
0x69, 0x24, 0x1e, 0x37, 0xd6, 0x55, 0xaf, 0x71, 0x4a, 0xe2, 0x00, 0x1a, 0xdb, 0xac, 0xbf, 0x9b,
0x30, 0x39, 0x92, 0xc4, 0x5c, 0xd5, 0x43, 0x58, 0xcd, 0x6e, 0x43, 0x0e, 0x8d, 0x86, 0xc6, 0xe4,
0xd8, 0x38, 0x07, 0x73, 0x11, 0xeb, 0xe7, 0xcf, 0x53, 0xb5, 0x10, 0x9d, 0xc9, 0xc7, 0xb3, 0xd4,
0xb6, 0xe2, 0x8e, 0x81, 0xb7, 0xde, 0x87, 0x7a, 0xfe, 0xef, 0x88, 0xb9, 0x04, 0x4d, 0x67, 0xc7,
0xd9, 0x73, 0xd6, 0xb7, 0x9c, 0x07, 0xce, 0xce, 0x87, 0x4b, 0xcf, 0x99, 0x0d, 0x98, 0xbf, 0x7d,
0x73, 0x7d, 0x6b, 0xef, 0xf6, 0xfd, 0x25, 0xc3, 0x6c, 0x42, 0x6d, 0x7d, 0x63, 0xe7, 0x23, 0x77,
0x7b, 0x7d, 0x6b, 0xa9, 0xb4, 0x71, 0xf3, 0xc1, 0x66, 0x9f, 0xf0, 0x41, 0xda, 0x13, 0x22, 0xae,
0x3e, 0x25, 0x61, 0x48, 0x9e, 0x72, 0xec, 0x0f, 0x56, 0x15, 0xcb, 0x77, 0x02, 0xc2, 0x38, 0x25,
0xbd, 0x94, 0xe3, 0x60, 0x35, 0xe3, 0xba, 0x2a, 0xa9, 0xe7, 0xcb, 0x61, 0x6f, 0xad, 0x57, 0x95,
0xd0, 0xb5, 0xdf, 0x03, 0x00, 0x00, 0xff, 0xff, 0x7e, 0xa8, 0x64, 0x49, 0x43, 0x12, 0x00, 0x00,
// 1376 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xec, 0x58, 0xcd, 0x6f, 0x1b, 0x45,
0x14, 0x67, 0x6d, 0xc7, 0xb1, 0x9f, 0x9d, 0x34, 0x5d, 0xfa, 0xb1, 0x85, 0x40, 0xdd, 0xe5, 0xcb,
0x80, 0x48, 0xaa, 0x14, 0x21, 0xc4, 0xa5, 0x4d, 0xe2, 0x96, 0xae, 0x9a, 0x84, 0xb0, 0x4e, 0x2b,
0xb5, 0x97, 0xd5, 0x78, 0x77, 0x62, 0x4f, 0xbb, 0x1f, 0xee, 0xcc, 0x6c, 0x53, 0xf7, 0xcc, 0x0d,
0xc1, 0x01, 0x89, 0x23, 0x17, 0xfe, 0x00, 0xfe, 0x04, 0x90, 0x38, 0x21, 0x71, 0x47, 0x42, 0xe2,
0x2f, 0xe1, 0x84, 0xe6, 0x63, 0xd7, 0x76, 0xba, 0x09, 0xa9, 0xa1, 0x42, 0x08, 0x6e, 0x9e, 0x37,
0x6f, 0xdf, 0xbc, 0xdf, 0xef, 0xfd, 0xe6, 0xcd, 0x8c, 0x61, 0x91, 0xc4, 0x1c, 0xd3, 0x18, 0x85,
0x2b, 0x43, 0x9a, 0xf0, 0xc4, 0x3c, 0x1b, 0x91, 0xf0, 0x51, 0xca, 0xd4, 0x68, 0x25, 0x9b, 0x7c,
0xa9, 0xe9, 0x27, 0x51, 0x94, 0xc4, 0xca, 0x6c, 0x7f, 0x6f, 0xc0, 0xc2, 0x66, 0x12, 0x0d, 0x93,
0x18, 0xc7, 0xdc, 0x89, 0xf7, 0x13, 0xf3, 0x1c, 0x54, 0xe3, 0x24, 0xc0, 0x4e, 0xc7, 0x32, 0x5a,
0x46, 0xbb, 0xec, 0xea, 0x91, 0x69, 0x42, 0x85, 0x26, 0x21, 0xb6, 0x4a, 0x2d, 0xa3, 0x5d, 0x77,
0xe5, 0x6f, 0xf3, 0x2a, 0x00, 0xe3, 0x88, 0x63, 0xcf, 0x4f, 0x02, 0x6c, 0x95, 0x5b, 0x46, 0x7b,
0x71, 0xad, 0xb5, 0x52, 0xb8, 0xee, 0x4a, 0x57, 0x38, 0x6e, 0x26, 0x01, 0x76, 0xeb, 0x2c, 0xfb,
0x69, 0x5e, 0x03, 0xc0, 0x8f, 0x39, 0x45, 0x1e, 0x89, 0xf7, 0x13, 0xab, 0xd2, 0x2a, 0xb7, 0x1b,
0x6b, 0x97, 0xa6, 0x03, 0xe8, 0x74, 0x6f, 0xe1, 0xd1, 0x1d, 0x14, 0xa6, 0x78, 0x17, 0x11, 0xea,
0xd6, 0xe5, 0x47, 0x22, 0x5d, 0xfb, 0x57, 0x03, 0x4e, 0xe5, 0x00, 0xe4, 0x1a, 0xcc, 0xfc, 0x08,
0xe6, 0xe4, 0x12, 0x12, 0x41, 0x63, 0xed, 0xf5, 0x23, 0x32, 0x9a, 0xc2, 0xed, 0xaa, 0x4f, 0xcc,
0xdb, 0xf0, 0x22, 0x4b, 0x7b, 0x7e, 0x36, 0xe5, 0x49, 0x2b, 0xb3, 0x4a, 0x32, 0xb5, 0x93, 0x45,
0x32, 0x27, 0x03, 0xe8, 0x94, 0xae, 0x40, 0x55, 0x44, 0x4a, 0x99, 0x64, 0xa9, 0xb1, 0xf6, 0x72,
0x21, 0xc8, 0xae, 0x74, 0x71, 0xb5, 0xab, 0x7d, 0x07, 0x6a, 0x3b, 0x82, 0x7c, 0x51, 0x96, 0x0f,
0x60, 0x1e, 0x05, 0x01, 0xc5, 0x8c, 0x69, 0x54, 0xcb, 0x85, 0x11, 0xd6, 0x95, 0x8f, 0x9b, 0x39,
0x17, 0x95, 0xcd, 0xbe, 0x0f, 0xe0, 0xc4, 0x84, 0xef, 0x22, 0x8a, 0x22, 0x76, 0x64, 0xc1, 0x3b,
0xd0, 0x64, 0x1c, 0x51, 0xee, 0x0d, 0xa5, 0x9f, 0xa6, 0xe0, 0x04, 0xd5, 0x69, 0xc8, 0xcf, 0x54,
0x74, 0xfb, 0x2e, 0x40, 0x97, 0x53, 0x12, 0xf7, 0xb7, 0x08, 0xe3, 0x62, 0xad, 0x47, 0xc2, 0x4f,
0x80, 0x28, 0xb7, 0xeb, 0xae, 0x1e, 0x4d, 0xd0, 0x53, 0x3a, 0x39, 0x3d, 0x57, 0xa1, 0xb1, 0x47,
0x22, 0xbc, 0x47, 0xfc, 0x07, 0xdb, 0xac, 0x6f, 0x5e, 0x86, 0x4a, 0x0f, 0x31, 0x7c, 0x2c, 0x3d,
0xdb, 0xac, 0xbf, 0x81, 0x18, 0x76, 0xa5, 0xa7, 0xfd, 0x9b, 0x01, 0xe7, 0x37, 0x29, 0x96, 0x62,
0x0c, 0x43, 0xec, 0x73, 0x92, 0xc4, 0x2e, 0x7e, 0x98, 0x62, 0xc6, 0x9f, 0x3d, 0x9a, 0x79, 0x1e,
0xe6, 0x83, 0x9e, 0x17, 0xa3, 0x28, 0x23, 0xbb, 0x1a, 0xf4, 0x76, 0x50, 0x84, 0xcd, 0x37, 0x61,
0xd1, 0xcf, 0xe3, 0x0b, 0x8b, 0xd4, 0x40, 0xdd, 0x3d, 0x64, 0x15, 0xa5, 0x0a, 0x7a, 0x4e, 0xc7,
0xaa, 0xc8, 0x32, 0xc8, 0xdf, 0xa6, 0x0d, 0xcd, 0xb1, 0x97, 0xd3, 0xb1, 0xe6, 0xe4, 0xdc, 0x94,
0x4d, 0x90, 0xca, 0xfc, 0x01, 0x8e, 0x90, 0x55, 0x6d, 0x19, 0xed, 0xa6, 0xab, 0x47, 0xf6, 0x8f,
0x06, 0x9c, 0xed, 0xd0, 0x64, 0xf8, 0x6f, 0x06, 0x67, 0x7f, 0x51, 0x82, 0x73, 0xaa, 0x46, 0xbb,
0x88, 0x72, 0xf2, 0x9c, 0x50, 0xbc, 0x05, 0xa7, 0xc6, 0xab, 0x2a, 0x87, 0x62, 0x18, 0x6f, 0xc0,
0xe2, 0x30, 0xcb, 0x43, 0xf9, 0x55, 0xa4, 0xdf, 0x42, 0x6e, 0x9d, 0x42, 0x3b, 0x77, 0x0c, 0xda,
0x6a, 0x41, 0x29, 0x5b, 0xd0, 0xc8, 0x03, 0x39, 0x1d, 0x6b, 0x5e, 0xba, 0x4c, 0x9a, 0xec, 0xcf,
0x4b, 0x70, 0x46, 0x14, 0xf5, 0x7f, 0x36, 0x04, 0x1b, 0x3f, 0x94, 0xc0, 0x54, 0xea, 0x70, 0xe2,
0x00, 0x3f, 0xfe, 0x27, 0xb9, 0x78, 0x05, 0x60, 0x9f, 0xe0, 0x30, 0x98, 0xe4, 0xa1, 0x2e, 0x2d,
0x7f, 0x89, 0x03, 0x0b, 0xe6, 0x65, 0x90, 0x1c, 0x7f, 0x36, 0x14, 0xfd, 0x59, 0x9d, 0x9d, 0xba,
0x3f, 0xd7, 0x4e, 0xdc, 0x9f, 0xe5, 0x67, 0xba, 0x3f, 0x7f, 0x57, 0x86, 0x05, 0x27, 0x66, 0x98,
0xf2, 0xff, 0xb2, 0x90, 0xcc, 0x65, 0xa8, 0x33, 0xdc, 0x8f, 0xc4, 0x11, 0xde, 0xb1, 0x6a, 0x72,
0x7e, 0x6c, 0x10, 0xb3, 0xfe, 0x00, 0xc5, 0x31, 0x0e, 0x9d, 0x8e, 0x55, 0x57, 0xa5, 0xcd, 0x0d,
0xe6, 0xab, 0x00, 0x9c, 0x44, 0x98, 0x71, 0x14, 0x0d, 0x99, 0x05, 0xad, 0x72, 0xbb, 0xe2, 0x4e,
0x58, 0x44, 0x7f, 0xa6, 0xc9, 0x81, 0xd3, 0x61, 0x56, 0xa3, 0x55, 0x16, 0x07, 0xac, 0x1a, 0x99,
0xef, 0x43, 0x8d, 0x26, 0x07, 0x5e, 0x80, 0x38, 0xb2, 0x9a, 0xb2, 0x78, 0x17, 0x0a, 0xc9, 0xde,
0x08, 0x93, 0x9e, 0x3b, 0x4f, 0x93, 0x83, 0x0e, 0xe2, 0xc8, 0xfe, 0xc6, 0x80, 0x85, 0x2e, 0x46,
0xd4, 0x1f, 0xcc, 0x5e, 0xb0, 0xb7, 0x61, 0x89, 0x62, 0x96, 0x86, 0xdc, 0x1b, 0xc3, 0x52, 0x95,
0x3b, 0xa5, 0xec, 0x9b, 0x39, 0xb8, 0x55, 0x98, 0x7b, 0x98, 0x62, 0x3a, 0xd2, 0xf7, 0x96, 0x63,
0x32, 0x54, 0x7e, 0xf6, 0x2f, 0x13, 0xf9, 0x89, 0x50, 0x6c, 0x86, 0xfc, 0x66, 0xb9, 0x0e, 0x14,
0x82, 0x2a, 0x17, 0x83, 0xba, 0x08, 0x8d, 0x08, 0x73, 0x4a, 0x7c, 0x8f, 0x8f, 0x86, 0x99, 0xd6,
0x40, 0x99, 0xf6, 0x46, 0x43, 0x29, 0xb4, 0x01, 0xe1, 0xcc, 0x9a, 0x6b, 0x95, 0xdb, 0x4d, 0x57,
0xfe, 0xb6, 0x7f, 0x36, 0x60, 0xa1, 0x83, 0x43, 0xcc, 0xf1, 0xec, 0xc4, 0x17, 0x6c, 0x88, 0x52,
0xe1, 0x86, 0x98, 0x52, 0x5c, 0xf9, 0x78, 0xc5, 0x55, 0x9e, 0x52, 0xdc, 0x25, 0x68, 0x0e, 0x29,
0x89, 0x10, 0x1d, 0x79, 0x0f, 0xf0, 0x48, 0xc1, 0x10, 0x82, 0x57, 0xb6, 0x5b, 0x78, 0xc4, 0xec,
0x6f, 0x0d, 0xa8, 0xdd, 0x08, 0x53, 0x36, 0x98, 0xe9, 0xea, 0x34, 0xbd, 0x5f, 0x4a, 0x87, 0xf7,
0xcb, 0xe1, 0x3d, 0x59, 0x2e, 0xd8, 0x93, 0x36, 0x34, 0xf3, 0x0d, 0xb8, 0x87, 0xfa, 0xba, 0x08,
0x53, 0x36, 0xfb, 0x77, 0x03, 0xea, 0x5b, 0x09, 0x0a, 0x64, 0x73, 0xff, 0xdb, 0xb3, 0x5c, 0x86,
0x71, 0x7f, 0xce, 0x38, 0x1e, 0x37, 0xec, 0x89, 0xc6, 0x5b, 0x99, 0x6e, 0xbc, 0x17, 0xa1, 0x41,
0x44, 0x42, 0xde, 0x10, 0xf1, 0x81, 0x22, 0xb7, 0xee, 0x82, 0x34, 0xed, 0x0a, 0x8b, 0xe8, 0xcc,
0x99, 0x83, 0xec, 0xcc, 0xd5, 0x13, 0x77, 0x66, 0x1d, 0x44, 0x76, 0x66, 0x0a, 0x56, 0x57, 0xe5,
0x2a, 0x84, 0x4e, 0x18, 0x27, 0x3e, 0xbb, 0x3d, 0x0c, 0xe4, 0x73, 0x62, 0x19, 0xea, 0xdd, 0x1c,
0x98, 0xba, 0xb6, 0x8f, 0x0d, 0x42, 0x1e, 0xdb, 0x38, 0x4a, 0xe8, 0xa8, 0x4b, 0x9e, 0x60, 0x8d,
0x7b, 0xc2, 0x22, 0xa0, 0xed, 0xa4, 0x91, 0x9b, 0x1c, 0x30, 0x5d, 0x99, 0x6c, 0x68, 0x7f, 0x65,
0xc0, 0xe9, 0xa7, 0x16, 0x9d, 0x81, 0xf8, 0x5b, 0x50, 0xeb, 0xe2, 0xbe, 0x08, 0x91, 0xbd, 0x1b,
0x56, 0x8f, 0x7a, 0x16, 0x1e, 0x01, 0xd1, 0xcd, 0x03, 0xd8, 0xf7, 0x73, 0x22, 0xa4, 0x60, 0xc5,
0x63, 0x4b, 0xec, 0xc2, 0xe0, 0x39, 0x28, 0xd7, 0xfe, 0xcc, 0x10, 0x6f, 0xa3, 0x00, 0x3f, 0x96,
0x4b, 0x3f, 0x55, 0x49, 0x63, 0x96, 0x4a, 0x9a, 0x97, 0xe1, 0x4c, 0x9c, 0x46, 0x1e, 0xc5, 0x21,
0xe2, 0x38, 0xf0, 0xf4, 0x6a, 0x4c, 0xaf, 0x6e, 0xc6, 0x69, 0xe4, 0xaa, 0x29, 0x0d, 0x93, 0xd9,
0x5f, 0x1a, 0x00, 0x37, 0x84, 0xdc, 0x54, 0x1a, 0x87, 0xf7, 0x93, 0x71, 0xfc, 0x45, 0xa1, 0x34,
0xad, 0xd7, 0x8d, 0x4c, 0xaf, 0x4c, 0xd6, 0xa3, 0x5c, 0x84, 0x21, 0xaf, 0xc7, 0x18, 0xbc, 0x96,
0xb4, 0xaa, 0xc1, 0xd7, 0x06, 0x34, 0x27, 0x4a, 0xc5, 0xa6, 0x69, 0x34, 0x0e, 0x6f, 0x2d, 0xd9,
0x60, 0x85, 0xde, 0x3c, 0x36, 0x21, 0xc1, 0x68, 0x2c, 0xc1, 0x0b, 0x50, 0x93, 0x94, 0x4c, 0x68,
0x30, 0x56, 0x1a, 0x34, 0xdf, 0x85, 0xd3, 0x14, 0xfb, 0x38, 0xe6, 0xe1, 0xc8, 0x8b, 0x92, 0x80,
0xec, 0x13, 0x1c, 0xc8, 0x2d, 0x58, 0x73, 0x97, 0xb2, 0x89, 0x6d, 0x6d, 0xb7, 0x7f, 0x32, 0x60,
0xf1, 0x53, 0x71, 0xee, 0x88, 0x87, 0xb2, 0xca, 0xec, 0xd9, 0x25, 0x71, 0x4d, 0x62, 0xd1, 0xf4,
0x28, 0xb9, 0xbe, 0xf6, 0xe7, 0x72, 0x65, 0x6e, 0x8d, 0x69, 0x89, 0x0a, 0x8a, 0xd5, 0xe5, 0xef,
0x24, 0x14, 0x8f, 0x0b, 0xeb, 0xaa, 0x2b, 0xa3, 0xa2, 0x38, 0x80, 0xc6, 0x36, 0xeb, 0xef, 0x26,
0x4c, 0xb6, 0x3f, 0xd1, 0xc3, 0x75, 0xc3, 0x57, 0xe7, 0x84, 0x21, 0x1b, 0x54, 0x43, 0xdb, 0x64,
0x8b, 0x3a, 0x03, 0x73, 0x11, 0xeb, 0xe7, 0x67, 0xb7, 0x1a, 0x88, 0xca, 0xe4, 0x47, 0x81, 0xe4,
0xb6, 0xe2, 0x8e, 0x0d, 0xef, 0x7c, 0x08, 0xf5, 0xfc, 0x9f, 0x18, 0x73, 0x09, 0x9a, 0xce, 0x8e,
0xb3, 0xe7, 0xac, 0x6f, 0x39, 0xf7, 0x9c, 0x9d, 0x8f, 0x97, 0x5e, 0x30, 0x1b, 0x30, 0x7f, 0xf3,
0xfa, 0xfa, 0xd6, 0xde, 0xcd, 0xbb, 0x4b, 0x86, 0xd9, 0x84, 0xda, 0xfa, 0xc6, 0xce, 0x27, 0xee,
0xf6, 0xfa, 0xd6, 0x52, 0x69, 0xe3, 0xfa, 0xbd, 0xcd, 0x3e, 0xe1, 0x83, 0xb4, 0x27, 0x48, 0x5c,
0x7d, 0x42, 0xc2, 0x90, 0x3c, 0xe1, 0xd8, 0x1f, 0xac, 0x2a, 0x94, 0xef, 0x05, 0x84, 0x71, 0x4a,
0x7a, 0x29, 0xc7, 0xc1, 0x6a, 0x86, 0x75, 0x55, 0x42, 0xcf, 0x87, 0xc3, 0xde, 0x5a, 0xaf, 0x2a,
0x4d, 0x57, 0xfe, 0x08, 0x00, 0x00, 0xff, 0xff, 0x86, 0x5e, 0x7e, 0x94, 0xaf, 0x12, 0x00, 0x00,
}