Add index meta to meta table

Signed-off-by: sunby <bingyi.sun@zilliz.com>
pull/4973/head^2
sunby 2020-12-22 20:16:32 +08:00 committed by yefu.chen
parent adfb5187cf
commit 2aa483c922
12 changed files with 7136 additions and 80 deletions

View File

@ -1,7 +1,7 @@
get_property(PROTOC_EXCUTABLE GLOBAL PROPERTY PROTOC_EXCUTABLE )
set(proto_file_names common.proto etcd_meta.proto schema.proto service_msg.proto index_cgo_msg.proto)
set(proto_file_names common.proto etcd_meta.proto schema.proto service_msg.proto index_cgo_msg.proto index_builder.proto)
set( PROTO_PATH "${MILVUS_SOURCE_DIR}/../proto/" )

File diff suppressed because it is too large Load Diff

View File

@ -32,6 +32,7 @@
#include <google/protobuf/repeated_field.h> // IWYU pragma: export
#include <google/protobuf/extension_set.h> // IWYU pragma: export
#include <google/protobuf/unknown_field_set.h>
#include "index_builder.pb.h"
#include "common.pb.h"
#include "schema.pb.h"
// @@protoc_insertion_point(includes)
@ -49,7 +50,7 @@ struct TableStruct_etcd_5fmeta_2eproto {
PROTOBUF_SECTION_VARIABLE(protodesc_cold);
static const ::PROTOBUF_NAMESPACE_ID::internal::AuxillaryParseTableField aux[]
PROTOBUF_SECTION_VARIABLE(protodesc_cold);
static const ::PROTOBUF_NAMESPACE_ID::internal::ParseTable schema[4]
static const ::PROTOBUF_NAMESPACE_ID::internal::ParseTable schema[6]
PROTOBUF_SECTION_VARIABLE(protodesc_cold);
static const ::PROTOBUF_NAMESPACE_ID::internal::FieldMetadata field_metadata[];
static const ::PROTOBUF_NAMESPACE_ID::internal::SerializationTable serialization_table[];
@ -62,6 +63,12 @@ namespace etcd {
class CollectionMeta;
class CollectionMetaDefaultTypeInternal;
extern CollectionMetaDefaultTypeInternal _CollectionMeta_default_instance_;
class FieldBinlogFiles;
class FieldBinlogFilesDefaultTypeInternal;
extern FieldBinlogFilesDefaultTypeInternal _FieldBinlogFiles_default_instance_;
class FieldIndexMeta;
class FieldIndexMetaDefaultTypeInternal;
extern FieldIndexMetaDefaultTypeInternal _FieldIndexMeta_default_instance_;
class ProxyMeta;
class ProxyMetaDefaultTypeInternal;
extern ProxyMetaDefaultTypeInternal _ProxyMeta_default_instance_;
@ -76,6 +83,8 @@ extern TenantMetaDefaultTypeInternal _TenantMeta_default_instance_;
} // namespace milvus
PROTOBUF_NAMESPACE_OPEN
template<> ::milvus::proto::etcd::CollectionMeta* Arena::CreateMaybeMessage<::milvus::proto::etcd::CollectionMeta>(Arena*);
template<> ::milvus::proto::etcd::FieldBinlogFiles* Arena::CreateMaybeMessage<::milvus::proto::etcd::FieldBinlogFiles>(Arena*);
template<> ::milvus::proto::etcd::FieldIndexMeta* Arena::CreateMaybeMessage<::milvus::proto::etcd::FieldIndexMeta>(Arena*);
template<> ::milvus::proto::etcd::ProxyMeta* Arena::CreateMaybeMessage<::milvus::proto::etcd::ProxyMeta>(Arena*);
template<> ::milvus::proto::etcd::SegmentMeta* Arena::CreateMaybeMessage<::milvus::proto::etcd::SegmentMeta>(Arena*);
template<> ::milvus::proto::etcd::TenantMeta* Arena::CreateMaybeMessage<::milvus::proto::etcd::TenantMeta>(Arena*);
@ -597,6 +606,156 @@ class CollectionMeta :
};
// -------------------------------------------------------------------
class FieldBinlogFiles :
public ::PROTOBUF_NAMESPACE_ID::Message /* @@protoc_insertion_point(class_definition:milvus.proto.etcd.FieldBinlogFiles) */ {
public:
FieldBinlogFiles();
virtual ~FieldBinlogFiles();
FieldBinlogFiles(const FieldBinlogFiles& from);
FieldBinlogFiles(FieldBinlogFiles&& from) noexcept
: FieldBinlogFiles() {
*this = ::std::move(from);
}
inline FieldBinlogFiles& operator=(const FieldBinlogFiles& from) {
CopyFrom(from);
return *this;
}
inline FieldBinlogFiles& operator=(FieldBinlogFiles&& from) noexcept {
if (GetArenaNoVirtual() == from.GetArenaNoVirtual()) {
if (this != &from) InternalSwap(&from);
} else {
CopyFrom(from);
}
return *this;
}
static const ::PROTOBUF_NAMESPACE_ID::Descriptor* descriptor() {
return GetDescriptor();
}
static const ::PROTOBUF_NAMESPACE_ID::Descriptor* GetDescriptor() {
return GetMetadataStatic().descriptor;
}
static const ::PROTOBUF_NAMESPACE_ID::Reflection* GetReflection() {
return GetMetadataStatic().reflection;
}
static const FieldBinlogFiles& default_instance();
static void InitAsDefaultInstance(); // FOR INTERNAL USE ONLY
static inline const FieldBinlogFiles* internal_default_instance() {
return reinterpret_cast<const FieldBinlogFiles*>(
&_FieldBinlogFiles_default_instance_);
}
static constexpr int kIndexInFileMessages =
3;
friend void swap(FieldBinlogFiles& a, FieldBinlogFiles& b) {
a.Swap(&b);
}
inline void Swap(FieldBinlogFiles* other) {
if (other == this) return;
InternalSwap(other);
}
// implements Message ----------------------------------------------
inline FieldBinlogFiles* New() const final {
return CreateMaybeMessage<FieldBinlogFiles>(nullptr);
}
FieldBinlogFiles* New(::PROTOBUF_NAMESPACE_ID::Arena* arena) const final {
return CreateMaybeMessage<FieldBinlogFiles>(arena);
}
void CopyFrom(const ::PROTOBUF_NAMESPACE_ID::Message& from) final;
void MergeFrom(const ::PROTOBUF_NAMESPACE_ID::Message& from) final;
void CopyFrom(const FieldBinlogFiles& from);
void MergeFrom(const FieldBinlogFiles& from);
PROTOBUF_ATTRIBUTE_REINITIALIZES void Clear() final;
bool IsInitialized() const final;
size_t ByteSizeLong() const final;
#if GOOGLE_PROTOBUF_ENABLE_EXPERIMENTAL_PARSER
const char* _InternalParse(const char* ptr, ::PROTOBUF_NAMESPACE_ID::internal::ParseContext* ctx) final;
#else
bool MergePartialFromCodedStream(
::PROTOBUF_NAMESPACE_ID::io::CodedInputStream* input) final;
#endif // GOOGLE_PROTOBUF_ENABLE_EXPERIMENTAL_PARSER
void SerializeWithCachedSizes(
::PROTOBUF_NAMESPACE_ID::io::CodedOutputStream* output) const final;
::PROTOBUF_NAMESPACE_ID::uint8* InternalSerializeWithCachedSizesToArray(
::PROTOBUF_NAMESPACE_ID::uint8* target) const final;
int GetCachedSize() const final { return _cached_size_.Get(); }
private:
inline void SharedCtor();
inline void SharedDtor();
void SetCachedSize(int size) const final;
void InternalSwap(FieldBinlogFiles* other);
friend class ::PROTOBUF_NAMESPACE_ID::internal::AnyMetadata;
static ::PROTOBUF_NAMESPACE_ID::StringPiece FullMessageName() {
return "milvus.proto.etcd.FieldBinlogFiles";
}
private:
inline ::PROTOBUF_NAMESPACE_ID::Arena* GetArenaNoVirtual() const {
return nullptr;
}
inline void* MaybeArenaPtr() const {
return nullptr;
}
public:
::PROTOBUF_NAMESPACE_ID::Metadata GetMetadata() const final;
private:
static ::PROTOBUF_NAMESPACE_ID::Metadata GetMetadataStatic() {
::PROTOBUF_NAMESPACE_ID::internal::AssignDescriptors(&::descriptor_table_etcd_5fmeta_2eproto);
return ::descriptor_table_etcd_5fmeta_2eproto.file_level_metadata[kIndexInFileMessages];
}
public:
// nested types ----------------------------------------------------
// accessors -------------------------------------------------------
enum : int {
kBinlogFilesFieldNumber = 2,
kFieldIDFieldNumber = 1,
};
// repeated string binlog_files = 2;
int binlog_files_size() const;
void clear_binlog_files();
const std::string& binlog_files(int index) const;
std::string* mutable_binlog_files(int index);
void set_binlog_files(int index, const std::string& value);
void set_binlog_files(int index, std::string&& value);
void set_binlog_files(int index, const char* value);
void set_binlog_files(int index, const char* value, size_t size);
std::string* add_binlog_files();
void add_binlog_files(const std::string& value);
void add_binlog_files(std::string&& value);
void add_binlog_files(const char* value);
void add_binlog_files(const char* value, size_t size);
const ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField<std::string>& binlog_files() const;
::PROTOBUF_NAMESPACE_ID::RepeatedPtrField<std::string>* mutable_binlog_files();
// int64 fieldID = 1;
void clear_fieldid();
::PROTOBUF_NAMESPACE_ID::int64 fieldid() const;
void set_fieldid(::PROTOBUF_NAMESPACE_ID::int64 value);
// @@protoc_insertion_point(class_scope:milvus.proto.etcd.FieldBinlogFiles)
private:
class _Internal;
::PROTOBUF_NAMESPACE_ID::internal::InternalMetadataWithArena _internal_metadata_;
::PROTOBUF_NAMESPACE_ID::RepeatedPtrField<std::string> binlog_files_;
::PROTOBUF_NAMESPACE_ID::int64 fieldid_;
mutable ::PROTOBUF_NAMESPACE_ID::internal::CachedSize _cached_size_;
friend struct ::TableStruct_etcd_5fmeta_2eproto;
};
// -------------------------------------------------------------------
class SegmentMeta :
public ::PROTOBUF_NAMESPACE_ID::Message /* @@protoc_insertion_point(class_definition:milvus.proto.etcd.SegmentMeta) */ {
public:
@ -639,7 +798,7 @@ class SegmentMeta :
&_SegmentMeta_default_instance_);
}
static constexpr int kIndexInFileMessages =
3;
4;
friend void swap(SegmentMeta& a, SegmentMeta& b) {
a.Swap(&b);
@ -710,6 +869,7 @@ class SegmentMeta :
// accessors -------------------------------------------------------
enum : int {
kBinlogFilePathsFieldNumber = 10,
kPartitionTagFieldNumber = 3,
kSegmentIDFieldNumber = 1,
kCollectionIDFieldNumber = 2,
@ -720,6 +880,17 @@ class SegmentMeta :
kNumRowsFieldNumber = 8,
kMemSizeFieldNumber = 9,
};
// repeated .milvus.proto.etcd.FieldBinlogFiles binlog_file_paths = 10;
int binlog_file_paths_size() const;
void clear_binlog_file_paths();
::milvus::proto::etcd::FieldBinlogFiles* mutable_binlog_file_paths(int index);
::PROTOBUF_NAMESPACE_ID::RepeatedPtrField< ::milvus::proto::etcd::FieldBinlogFiles >*
mutable_binlog_file_paths();
const ::milvus::proto::etcd::FieldBinlogFiles& binlog_file_paths(int index) const;
::milvus::proto::etcd::FieldBinlogFiles* add_binlog_file_paths();
const ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField< ::milvus::proto::etcd::FieldBinlogFiles >&
binlog_file_paths() const;
// string partition_tag = 3;
void clear_partition_tag();
const std::string& partition_tag() const;
@ -776,6 +947,7 @@ class SegmentMeta :
class _Internal;
::PROTOBUF_NAMESPACE_ID::internal::InternalMetadataWithArena _internal_metadata_;
::PROTOBUF_NAMESPACE_ID::RepeatedPtrField< ::milvus::proto::etcd::FieldBinlogFiles > binlog_file_paths_;
::PROTOBUF_NAMESPACE_ID::internal::ArenaStringPtr partition_tag_;
::PROTOBUF_NAMESPACE_ID::int64 segmentid_;
::PROTOBUF_NAMESPACE_ID::int64 collectionid_;
@ -788,6 +960,203 @@ class SegmentMeta :
mutable ::PROTOBUF_NAMESPACE_ID::internal::CachedSize _cached_size_;
friend struct ::TableStruct_etcd_5fmeta_2eproto;
};
// -------------------------------------------------------------------
class FieldIndexMeta :
public ::PROTOBUF_NAMESPACE_ID::Message /* @@protoc_insertion_point(class_definition:milvus.proto.etcd.FieldIndexMeta) */ {
public:
FieldIndexMeta();
virtual ~FieldIndexMeta();
FieldIndexMeta(const FieldIndexMeta& from);
FieldIndexMeta(FieldIndexMeta&& from) noexcept
: FieldIndexMeta() {
*this = ::std::move(from);
}
inline FieldIndexMeta& operator=(const FieldIndexMeta& from) {
CopyFrom(from);
return *this;
}
inline FieldIndexMeta& operator=(FieldIndexMeta&& from) noexcept {
if (GetArenaNoVirtual() == from.GetArenaNoVirtual()) {
if (this != &from) InternalSwap(&from);
} else {
CopyFrom(from);
}
return *this;
}
static const ::PROTOBUF_NAMESPACE_ID::Descriptor* descriptor() {
return GetDescriptor();
}
static const ::PROTOBUF_NAMESPACE_ID::Descriptor* GetDescriptor() {
return GetMetadataStatic().descriptor;
}
static const ::PROTOBUF_NAMESPACE_ID::Reflection* GetReflection() {
return GetMetadataStatic().reflection;
}
static const FieldIndexMeta& default_instance();
static void InitAsDefaultInstance(); // FOR INTERNAL USE ONLY
static inline const FieldIndexMeta* internal_default_instance() {
return reinterpret_cast<const FieldIndexMeta*>(
&_FieldIndexMeta_default_instance_);
}
static constexpr int kIndexInFileMessages =
5;
friend void swap(FieldIndexMeta& a, FieldIndexMeta& b) {
a.Swap(&b);
}
inline void Swap(FieldIndexMeta* other) {
if (other == this) return;
InternalSwap(other);
}
// implements Message ----------------------------------------------
inline FieldIndexMeta* New() const final {
return CreateMaybeMessage<FieldIndexMeta>(nullptr);
}
FieldIndexMeta* New(::PROTOBUF_NAMESPACE_ID::Arena* arena) const final {
return CreateMaybeMessage<FieldIndexMeta>(arena);
}
void CopyFrom(const ::PROTOBUF_NAMESPACE_ID::Message& from) final;
void MergeFrom(const ::PROTOBUF_NAMESPACE_ID::Message& from) final;
void CopyFrom(const FieldIndexMeta& from);
void MergeFrom(const FieldIndexMeta& from);
PROTOBUF_ATTRIBUTE_REINITIALIZES void Clear() final;
bool IsInitialized() const final;
size_t ByteSizeLong() const final;
#if GOOGLE_PROTOBUF_ENABLE_EXPERIMENTAL_PARSER
const char* _InternalParse(const char* ptr, ::PROTOBUF_NAMESPACE_ID::internal::ParseContext* ctx) final;
#else
bool MergePartialFromCodedStream(
::PROTOBUF_NAMESPACE_ID::io::CodedInputStream* input) final;
#endif // GOOGLE_PROTOBUF_ENABLE_EXPERIMENTAL_PARSER
void SerializeWithCachedSizes(
::PROTOBUF_NAMESPACE_ID::io::CodedOutputStream* output) const final;
::PROTOBUF_NAMESPACE_ID::uint8* InternalSerializeWithCachedSizesToArray(
::PROTOBUF_NAMESPACE_ID::uint8* target) const final;
int GetCachedSize() const final { return _cached_size_.Get(); }
private:
inline void SharedCtor();
inline void SharedDtor();
void SetCachedSize(int size) const final;
void InternalSwap(FieldIndexMeta* other);
friend class ::PROTOBUF_NAMESPACE_ID::internal::AnyMetadata;
static ::PROTOBUF_NAMESPACE_ID::StringPiece FullMessageName() {
return "milvus.proto.etcd.FieldIndexMeta";
}
private:
inline ::PROTOBUF_NAMESPACE_ID::Arena* GetArenaNoVirtual() const {
return nullptr;
}
inline void* MaybeArenaPtr() const {
return nullptr;
}
public:
::PROTOBUF_NAMESPACE_ID::Metadata GetMetadata() const final;
private:
static ::PROTOBUF_NAMESPACE_ID::Metadata GetMetadataStatic() {
::PROTOBUF_NAMESPACE_ID::internal::AssignDescriptors(&::descriptor_table_etcd_5fmeta_2eproto);
return ::descriptor_table_etcd_5fmeta_2eproto.file_level_metadata[kIndexInFileMessages];
}
public:
// nested types ----------------------------------------------------
// accessors -------------------------------------------------------
enum : int {
kIndexParamsFieldNumber = 5,
kIndexFilePathsFieldNumber = 7,
kIndexTypeFieldNumber = 3,
kSegmentIDFieldNumber = 1,
kFieldIDFieldNumber = 2,
kIndexIDFieldNumber = 4,
kStatusFieldNumber = 6,
};
// repeated .milvus.proto.common.KeyValuePair index_params = 5;
int index_params_size() const;
void clear_index_params();
::milvus::proto::common::KeyValuePair* mutable_index_params(int index);
::PROTOBUF_NAMESPACE_ID::RepeatedPtrField< ::milvus::proto::common::KeyValuePair >*
mutable_index_params();
const ::milvus::proto::common::KeyValuePair& index_params(int index) const;
::milvus::proto::common::KeyValuePair* add_index_params();
const ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField< ::milvus::proto::common::KeyValuePair >&
index_params() const;
// repeated string index_file_paths = 7;
int index_file_paths_size() const;
void clear_index_file_paths();
const std::string& index_file_paths(int index) const;
std::string* mutable_index_file_paths(int index);
void set_index_file_paths(int index, const std::string& value);
void set_index_file_paths(int index, std::string&& value);
void set_index_file_paths(int index, const char* value);
void set_index_file_paths(int index, const char* value, size_t size);
std::string* add_index_file_paths();
void add_index_file_paths(const std::string& value);
void add_index_file_paths(std::string&& value);
void add_index_file_paths(const char* value);
void add_index_file_paths(const char* value, size_t size);
const ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField<std::string>& index_file_paths() const;
::PROTOBUF_NAMESPACE_ID::RepeatedPtrField<std::string>* mutable_index_file_paths();
// string index_type = 3;
void clear_index_type();
const std::string& index_type() const;
void set_index_type(const std::string& value);
void set_index_type(std::string&& value);
void set_index_type(const char* value);
void set_index_type(const char* value, size_t size);
std::string* mutable_index_type();
std::string* release_index_type();
void set_allocated_index_type(std::string* index_type);
// int64 segmentID = 1;
void clear_segmentid();
::PROTOBUF_NAMESPACE_ID::int64 segmentid() const;
void set_segmentid(::PROTOBUF_NAMESPACE_ID::int64 value);
// int64 fieldID = 2;
void clear_fieldid();
::PROTOBUF_NAMESPACE_ID::int64 fieldid() const;
void set_fieldid(::PROTOBUF_NAMESPACE_ID::int64 value);
// int64 indexID = 4;
void clear_indexid();
::PROTOBUF_NAMESPACE_ID::int64 indexid() const;
void set_indexid(::PROTOBUF_NAMESPACE_ID::int64 value);
// .milvus.proto.service.IndexStatus status = 6;
void clear_status();
::milvus::proto::service::IndexStatus status() const;
void set_status(::milvus::proto::service::IndexStatus value);
// @@protoc_insertion_point(class_scope:milvus.proto.etcd.FieldIndexMeta)
private:
class _Internal;
::PROTOBUF_NAMESPACE_ID::internal::InternalMetadataWithArena _internal_metadata_;
::PROTOBUF_NAMESPACE_ID::RepeatedPtrField< ::milvus::proto::common::KeyValuePair > index_params_;
::PROTOBUF_NAMESPACE_ID::RepeatedPtrField<std::string> index_file_paths_;
::PROTOBUF_NAMESPACE_ID::internal::ArenaStringPtr index_type_;
::PROTOBUF_NAMESPACE_ID::int64 segmentid_;
::PROTOBUF_NAMESPACE_ID::int64 fieldid_;
::PROTOBUF_NAMESPACE_ID::int64 indexid_;
int status_;
mutable ::PROTOBUF_NAMESPACE_ID::internal::CachedSize _cached_size_;
friend struct ::TableStruct_etcd_5fmeta_2eproto;
};
// ===================================================================
@ -1245,6 +1614,89 @@ CollectionMeta::mutable_partition_tags() {
// -------------------------------------------------------------------
// FieldBinlogFiles
// int64 fieldID = 1;
inline void FieldBinlogFiles::clear_fieldid() {
fieldid_ = PROTOBUF_LONGLONG(0);
}
inline ::PROTOBUF_NAMESPACE_ID::int64 FieldBinlogFiles::fieldid() const {
// @@protoc_insertion_point(field_get:milvus.proto.etcd.FieldBinlogFiles.fieldID)
return fieldid_;
}
inline void FieldBinlogFiles::set_fieldid(::PROTOBUF_NAMESPACE_ID::int64 value) {
fieldid_ = value;
// @@protoc_insertion_point(field_set:milvus.proto.etcd.FieldBinlogFiles.fieldID)
}
// repeated string binlog_files = 2;
inline int FieldBinlogFiles::binlog_files_size() const {
return binlog_files_.size();
}
inline void FieldBinlogFiles::clear_binlog_files() {
binlog_files_.Clear();
}
inline const std::string& FieldBinlogFiles::binlog_files(int index) const {
// @@protoc_insertion_point(field_get:milvus.proto.etcd.FieldBinlogFiles.binlog_files)
return binlog_files_.Get(index);
}
inline std::string* FieldBinlogFiles::mutable_binlog_files(int index) {
// @@protoc_insertion_point(field_mutable:milvus.proto.etcd.FieldBinlogFiles.binlog_files)
return binlog_files_.Mutable(index);
}
inline void FieldBinlogFiles::set_binlog_files(int index, const std::string& value) {
// @@protoc_insertion_point(field_set:milvus.proto.etcd.FieldBinlogFiles.binlog_files)
binlog_files_.Mutable(index)->assign(value);
}
inline void FieldBinlogFiles::set_binlog_files(int index, std::string&& value) {
// @@protoc_insertion_point(field_set:milvus.proto.etcd.FieldBinlogFiles.binlog_files)
binlog_files_.Mutable(index)->assign(std::move(value));
}
inline void FieldBinlogFiles::set_binlog_files(int index, const char* value) {
GOOGLE_DCHECK(value != nullptr);
binlog_files_.Mutable(index)->assign(value);
// @@protoc_insertion_point(field_set_char:milvus.proto.etcd.FieldBinlogFiles.binlog_files)
}
inline void FieldBinlogFiles::set_binlog_files(int index, const char* value, size_t size) {
binlog_files_.Mutable(index)->assign(
reinterpret_cast<const char*>(value), size);
// @@protoc_insertion_point(field_set_pointer:milvus.proto.etcd.FieldBinlogFiles.binlog_files)
}
inline std::string* FieldBinlogFiles::add_binlog_files() {
// @@protoc_insertion_point(field_add_mutable:milvus.proto.etcd.FieldBinlogFiles.binlog_files)
return binlog_files_.Add();
}
inline void FieldBinlogFiles::add_binlog_files(const std::string& value) {
binlog_files_.Add()->assign(value);
// @@protoc_insertion_point(field_add:milvus.proto.etcd.FieldBinlogFiles.binlog_files)
}
inline void FieldBinlogFiles::add_binlog_files(std::string&& value) {
binlog_files_.Add(std::move(value));
// @@protoc_insertion_point(field_add:milvus.proto.etcd.FieldBinlogFiles.binlog_files)
}
inline void FieldBinlogFiles::add_binlog_files(const char* value) {
GOOGLE_DCHECK(value != nullptr);
binlog_files_.Add()->assign(value);
// @@protoc_insertion_point(field_add_char:milvus.proto.etcd.FieldBinlogFiles.binlog_files)
}
inline void FieldBinlogFiles::add_binlog_files(const char* value, size_t size) {
binlog_files_.Add()->assign(reinterpret_cast<const char*>(value), size);
// @@protoc_insertion_point(field_add_pointer:milvus.proto.etcd.FieldBinlogFiles.binlog_files)
}
inline const ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField<std::string>&
FieldBinlogFiles::binlog_files() const {
// @@protoc_insertion_point(field_list:milvus.proto.etcd.FieldBinlogFiles.binlog_files)
return binlog_files_;
}
inline ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField<std::string>*
FieldBinlogFiles::mutable_binlog_files() {
// @@protoc_insertion_point(field_mutable_list:milvus.proto.etcd.FieldBinlogFiles.binlog_files)
return &binlog_files_;
}
// -------------------------------------------------------------------
// SegmentMeta
// int64 segmentID = 1;
@ -1410,6 +1862,239 @@ inline void SegmentMeta::set_mem_size(::PROTOBUF_NAMESPACE_ID::int64 value) {
// @@protoc_insertion_point(field_set:milvus.proto.etcd.SegmentMeta.mem_size)
}
// repeated .milvus.proto.etcd.FieldBinlogFiles binlog_file_paths = 10;
inline int SegmentMeta::binlog_file_paths_size() const {
return binlog_file_paths_.size();
}
inline void SegmentMeta::clear_binlog_file_paths() {
binlog_file_paths_.Clear();
}
inline ::milvus::proto::etcd::FieldBinlogFiles* SegmentMeta::mutable_binlog_file_paths(int index) {
// @@protoc_insertion_point(field_mutable:milvus.proto.etcd.SegmentMeta.binlog_file_paths)
return binlog_file_paths_.Mutable(index);
}
inline ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField< ::milvus::proto::etcd::FieldBinlogFiles >*
SegmentMeta::mutable_binlog_file_paths() {
// @@protoc_insertion_point(field_mutable_list:milvus.proto.etcd.SegmentMeta.binlog_file_paths)
return &binlog_file_paths_;
}
inline const ::milvus::proto::etcd::FieldBinlogFiles& SegmentMeta::binlog_file_paths(int index) const {
// @@protoc_insertion_point(field_get:milvus.proto.etcd.SegmentMeta.binlog_file_paths)
return binlog_file_paths_.Get(index);
}
inline ::milvus::proto::etcd::FieldBinlogFiles* SegmentMeta::add_binlog_file_paths() {
// @@protoc_insertion_point(field_add:milvus.proto.etcd.SegmentMeta.binlog_file_paths)
return binlog_file_paths_.Add();
}
inline const ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField< ::milvus::proto::etcd::FieldBinlogFiles >&
SegmentMeta::binlog_file_paths() const {
// @@protoc_insertion_point(field_list:milvus.proto.etcd.SegmentMeta.binlog_file_paths)
return binlog_file_paths_;
}
// -------------------------------------------------------------------
// FieldIndexMeta
// int64 segmentID = 1;
inline void FieldIndexMeta::clear_segmentid() {
segmentid_ = PROTOBUF_LONGLONG(0);
}
inline ::PROTOBUF_NAMESPACE_ID::int64 FieldIndexMeta::segmentid() const {
// @@protoc_insertion_point(field_get:milvus.proto.etcd.FieldIndexMeta.segmentID)
return segmentid_;
}
inline void FieldIndexMeta::set_segmentid(::PROTOBUF_NAMESPACE_ID::int64 value) {
segmentid_ = value;
// @@protoc_insertion_point(field_set:milvus.proto.etcd.FieldIndexMeta.segmentID)
}
// int64 fieldID = 2;
inline void FieldIndexMeta::clear_fieldid() {
fieldid_ = PROTOBUF_LONGLONG(0);
}
inline ::PROTOBUF_NAMESPACE_ID::int64 FieldIndexMeta::fieldid() const {
// @@protoc_insertion_point(field_get:milvus.proto.etcd.FieldIndexMeta.fieldID)
return fieldid_;
}
inline void FieldIndexMeta::set_fieldid(::PROTOBUF_NAMESPACE_ID::int64 value) {
fieldid_ = value;
// @@protoc_insertion_point(field_set:milvus.proto.etcd.FieldIndexMeta.fieldID)
}
// string index_type = 3;
inline void FieldIndexMeta::clear_index_type() {
index_type_.ClearToEmptyNoArena(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited());
}
inline const std::string& FieldIndexMeta::index_type() const {
// @@protoc_insertion_point(field_get:milvus.proto.etcd.FieldIndexMeta.index_type)
return index_type_.GetNoArena();
}
inline void FieldIndexMeta::set_index_type(const std::string& value) {
index_type_.SetNoArena(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), value);
// @@protoc_insertion_point(field_set:milvus.proto.etcd.FieldIndexMeta.index_type)
}
inline void FieldIndexMeta::set_index_type(std::string&& value) {
index_type_.SetNoArena(
&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), ::std::move(value));
// @@protoc_insertion_point(field_set_rvalue:milvus.proto.etcd.FieldIndexMeta.index_type)
}
inline void FieldIndexMeta::set_index_type(const char* value) {
GOOGLE_DCHECK(value != nullptr);
index_type_.SetNoArena(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), ::std::string(value));
// @@protoc_insertion_point(field_set_char:milvus.proto.etcd.FieldIndexMeta.index_type)
}
inline void FieldIndexMeta::set_index_type(const char* value, size_t size) {
index_type_.SetNoArena(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(),
::std::string(reinterpret_cast<const char*>(value), size));
// @@protoc_insertion_point(field_set_pointer:milvus.proto.etcd.FieldIndexMeta.index_type)
}
inline std::string* FieldIndexMeta::mutable_index_type() {
// @@protoc_insertion_point(field_mutable:milvus.proto.etcd.FieldIndexMeta.index_type)
return index_type_.MutableNoArena(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited());
}
inline std::string* FieldIndexMeta::release_index_type() {
// @@protoc_insertion_point(field_release:milvus.proto.etcd.FieldIndexMeta.index_type)
return index_type_.ReleaseNoArena(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited());
}
inline void FieldIndexMeta::set_allocated_index_type(std::string* index_type) {
if (index_type != nullptr) {
} else {
}
index_type_.SetAllocatedNoArena(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), index_type);
// @@protoc_insertion_point(field_set_allocated:milvus.proto.etcd.FieldIndexMeta.index_type)
}
// int64 indexID = 4;
inline void FieldIndexMeta::clear_indexid() {
indexid_ = PROTOBUF_LONGLONG(0);
}
inline ::PROTOBUF_NAMESPACE_ID::int64 FieldIndexMeta::indexid() const {
// @@protoc_insertion_point(field_get:milvus.proto.etcd.FieldIndexMeta.indexID)
return indexid_;
}
inline void FieldIndexMeta::set_indexid(::PROTOBUF_NAMESPACE_ID::int64 value) {
indexid_ = value;
// @@protoc_insertion_point(field_set:milvus.proto.etcd.FieldIndexMeta.indexID)
}
// repeated .milvus.proto.common.KeyValuePair index_params = 5;
inline int FieldIndexMeta::index_params_size() const {
return index_params_.size();
}
inline ::milvus::proto::common::KeyValuePair* FieldIndexMeta::mutable_index_params(int index) {
// @@protoc_insertion_point(field_mutable:milvus.proto.etcd.FieldIndexMeta.index_params)
return index_params_.Mutable(index);
}
inline ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField< ::milvus::proto::common::KeyValuePair >*
FieldIndexMeta::mutable_index_params() {
// @@protoc_insertion_point(field_mutable_list:milvus.proto.etcd.FieldIndexMeta.index_params)
return &index_params_;
}
inline const ::milvus::proto::common::KeyValuePair& FieldIndexMeta::index_params(int index) const {
// @@protoc_insertion_point(field_get:milvus.proto.etcd.FieldIndexMeta.index_params)
return index_params_.Get(index);
}
inline ::milvus::proto::common::KeyValuePair* FieldIndexMeta::add_index_params() {
// @@protoc_insertion_point(field_add:milvus.proto.etcd.FieldIndexMeta.index_params)
return index_params_.Add();
}
inline const ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField< ::milvus::proto::common::KeyValuePair >&
FieldIndexMeta::index_params() const {
// @@protoc_insertion_point(field_list:milvus.proto.etcd.FieldIndexMeta.index_params)
return index_params_;
}
// .milvus.proto.service.IndexStatus status = 6;
inline void FieldIndexMeta::clear_status() {
status_ = 0;
}
inline ::milvus::proto::service::IndexStatus FieldIndexMeta::status() const {
// @@protoc_insertion_point(field_get:milvus.proto.etcd.FieldIndexMeta.status)
return static_cast< ::milvus::proto::service::IndexStatus >(status_);
}
inline void FieldIndexMeta::set_status(::milvus::proto::service::IndexStatus value) {
status_ = value;
// @@protoc_insertion_point(field_set:milvus.proto.etcd.FieldIndexMeta.status)
}
// repeated string index_file_paths = 7;
inline int FieldIndexMeta::index_file_paths_size() const {
return index_file_paths_.size();
}
inline void FieldIndexMeta::clear_index_file_paths() {
index_file_paths_.Clear();
}
inline const std::string& FieldIndexMeta::index_file_paths(int index) const {
// @@protoc_insertion_point(field_get:milvus.proto.etcd.FieldIndexMeta.index_file_paths)
return index_file_paths_.Get(index);
}
inline std::string* FieldIndexMeta::mutable_index_file_paths(int index) {
// @@protoc_insertion_point(field_mutable:milvus.proto.etcd.FieldIndexMeta.index_file_paths)
return index_file_paths_.Mutable(index);
}
inline void FieldIndexMeta::set_index_file_paths(int index, const std::string& value) {
// @@protoc_insertion_point(field_set:milvus.proto.etcd.FieldIndexMeta.index_file_paths)
index_file_paths_.Mutable(index)->assign(value);
}
inline void FieldIndexMeta::set_index_file_paths(int index, std::string&& value) {
// @@protoc_insertion_point(field_set:milvus.proto.etcd.FieldIndexMeta.index_file_paths)
index_file_paths_.Mutable(index)->assign(std::move(value));
}
inline void FieldIndexMeta::set_index_file_paths(int index, const char* value) {
GOOGLE_DCHECK(value != nullptr);
index_file_paths_.Mutable(index)->assign(value);
// @@protoc_insertion_point(field_set_char:milvus.proto.etcd.FieldIndexMeta.index_file_paths)
}
inline void FieldIndexMeta::set_index_file_paths(int index, const char* value, size_t size) {
index_file_paths_.Mutable(index)->assign(
reinterpret_cast<const char*>(value), size);
// @@protoc_insertion_point(field_set_pointer:milvus.proto.etcd.FieldIndexMeta.index_file_paths)
}
inline std::string* FieldIndexMeta::add_index_file_paths() {
// @@protoc_insertion_point(field_add_mutable:milvus.proto.etcd.FieldIndexMeta.index_file_paths)
return index_file_paths_.Add();
}
inline void FieldIndexMeta::add_index_file_paths(const std::string& value) {
index_file_paths_.Add()->assign(value);
// @@protoc_insertion_point(field_add:milvus.proto.etcd.FieldIndexMeta.index_file_paths)
}
inline void FieldIndexMeta::add_index_file_paths(std::string&& value) {
index_file_paths_.Add(std::move(value));
// @@protoc_insertion_point(field_add:milvus.proto.etcd.FieldIndexMeta.index_file_paths)
}
inline void FieldIndexMeta::add_index_file_paths(const char* value) {
GOOGLE_DCHECK(value != nullptr);
index_file_paths_.Add()->assign(value);
// @@protoc_insertion_point(field_add_char:milvus.proto.etcd.FieldIndexMeta.index_file_paths)
}
inline void FieldIndexMeta::add_index_file_paths(const char* value, size_t size) {
index_file_paths_.Add()->assign(reinterpret_cast<const char*>(value), size);
// @@protoc_insertion_point(field_add_pointer:milvus.proto.etcd.FieldIndexMeta.index_file_paths)
}
inline const ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField<std::string>&
FieldIndexMeta::index_file_paths() const {
// @@protoc_insertion_point(field_list:milvus.proto.etcd.FieldIndexMeta.index_file_paths)
return index_file_paths_;
}
inline ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField<std::string>*
FieldIndexMeta::mutable_index_file_paths() {
// @@protoc_insertion_point(field_mutable_list:milvus.proto.etcd.FieldIndexMeta.index_file_paths)
return &index_file_paths_;
}
#ifdef __GNUC__
#pragma GCC diagnostic pop
#endif // __GNUC__
@ -1419,6 +2104,10 @@ inline void SegmentMeta::set_mem_size(::PROTOBUF_NAMESPACE_ID::int64 value) {
// -------------------------------------------------------------------
// -------------------------------------------------------------------
// -------------------------------------------------------------------
// @@protoc_insertion_point(namespace_scope)

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

View File

@ -1,9 +1,14 @@
package master
import (
"fmt"
"strconv"
"sync"
"github.com/zilliztech/milvus-distributed/internal/proto/commonpb"
"github.com/zilliztech/milvus-distributed/internal/util/typeutil"
"github.com/golang/protobuf/proto"
"github.com/zilliztech/milvus-distributed/internal/errors"
"github.com/zilliztech/milvus-distributed/internal/kv"
@ -11,16 +16,18 @@ import (
)
type metaTable struct {
client kv.TxnBase // client of a reliable kv service, i.e. etcd client
tenantID2Meta map[UniqueID]pb.TenantMeta // tenant id to tenant meta
proxyID2Meta map[UniqueID]pb.ProxyMeta // proxy id to proxy meta
collID2Meta map[UniqueID]pb.CollectionMeta // collection id to collection meta
collName2ID map[string]UniqueID // collection name to collection id
segID2Meta map[UniqueID]pb.SegmentMeta // segment id to segment meta
client kv.TxnBase // client of a reliable kv service, i.e. etcd client
tenantID2Meta map[UniqueID]pb.TenantMeta // tenant id to tenant meta
proxyID2Meta map[UniqueID]pb.ProxyMeta // proxy id to proxy meta
collID2Meta map[UniqueID]pb.CollectionMeta // collection id to collection meta
collName2ID map[string]UniqueID // collection name to collection id
segID2Meta map[UniqueID]pb.SegmentMeta // segment id to segment meta
segID2IndexMetas map[UniqueID][]pb.FieldIndexMeta // segment id to array of field index meta
tenantLock sync.RWMutex
proxyLock sync.RWMutex
ddLock sync.RWMutex
indexLock sync.RWMutex
}
func NewMetaTable(kv kv.TxnBase) (*metaTable, error) {
@ -44,6 +51,7 @@ func (mt *metaTable) reloadFromKV() error {
mt.collID2Meta = make(map[UniqueID]pb.CollectionMeta)
mt.collName2ID = make(map[string]UniqueID)
mt.segID2Meta = make(map[UniqueID]pb.SegmentMeta)
mt.segID2IndexMetas = make(map[UniqueID][]pb.FieldIndexMeta)
_, values, err := mt.client.LoadWithPrefix("tenant")
if err != nil {
@ -102,6 +110,19 @@ func (mt *metaTable) reloadFromKV() error {
mt.segID2Meta[segmentMeta.SegmentID] = segmentMeta
}
_, values, err = mt.client.LoadWithPrefix("indexmeta")
if err != nil {
return err
}
for _, v := range values {
indexMeta := pb.FieldIndexMeta{}
err = proto.UnmarshalText(v, &indexMeta)
if err != nil {
return err
}
mt.segID2IndexMetas[indexMeta.SegmentID] = append(mt.segID2IndexMetas[indexMeta.SegmentID], indexMeta)
}
return nil
}
@ -235,6 +256,13 @@ func (mt *metaTable) DeleteCollection(collID UniqueID) error {
_ = mt.reloadFromKV()
return err
}
// remove index meta
for _, v := range collMeta.SegmentIDs {
if err := mt.removeSegmentIndexMeta(v); err != nil {
return err
}
}
return nil
}
@ -446,8 +474,8 @@ func (mt *metaTable) DeleteSegment(segID UniqueID) error {
_ = mt.reloadFromKV()
return err
}
return nil
return mt.removeSegmentIndexMeta(segID)
}
func (mt *metaTable) CloseSegment(segID UniqueID, closeTs Timestamp) error {
mt.ddLock.Lock()
@ -467,3 +495,127 @@ func (mt *metaTable) CloseSegment(segID UniqueID, closeTs Timestamp) error {
}
return nil
}
func (mt *metaTable) AddFieldIndexMeta(meta *pb.FieldIndexMeta) error {
mt.indexLock.Lock()
defer mt.indexLock.Unlock()
segID := meta.SegmentID
if _, ok := mt.segID2IndexMetas[segID]; !ok {
mt.segID2IndexMetas[segID] = make([]pb.FieldIndexMeta, 0)
}
for _, v := range mt.segID2IndexMetas[segID] {
if v.FieldID == meta.FieldID && v.IndexType == meta.IndexType && typeutil.CompareIndexParams(v.IndexParams, meta.IndexParams) {
return fmt.Errorf("segment %d field id %d's index meta already exist", segID, meta.FieldID)
}
}
mt.segID2IndexMetas[segID] = append(mt.segID2IndexMetas[segID], *meta)
err := mt.saveFieldIndexMetaToEtcd(meta)
if err != nil {
_ = mt.reloadFromKV()
return err
}
return nil
}
func (mt *metaTable) saveFieldIndexMetaToEtcd(meta *pb.FieldIndexMeta) error {
key := "/indexmeta/" + strconv.FormatInt(meta.SegmentID, 10) + strconv.FormatInt(meta.FieldID, 10) + strconv.FormatInt(meta.IndexID, 10)
marshaledMeta := proto.MarshalTextString(meta)
return mt.client.Save(key, marshaledMeta)
}
func (mt *metaTable) DeleteFieldIndexMeta(segID UniqueID, fieldID UniqueID, indexType string, indexParams []*commonpb.KeyValuePair) error {
mt.indexLock.Lock()
defer mt.indexLock.Unlock()
if _, ok := mt.segID2IndexMetas[segID]; !ok {
return fmt.Errorf("can not find index meta of segment %d", segID)
}
for i, v := range mt.segID2IndexMetas[segID] {
if v.FieldID == fieldID && v.IndexType == indexType && typeutil.CompareIndexParams(v.IndexParams, indexParams) {
mt.segID2IndexMetas[segID] = append(mt.segID2IndexMetas[segID][:i], mt.segID2IndexMetas[segID][i+1:]...)
err := mt.deleteFieldIndexMetaToEtcd(segID, fieldID, v.IndexID)
if err != nil {
_ = mt.reloadFromKV()
return err
}
return nil
}
}
return fmt.Errorf("can not find index meta of field %d", fieldID)
}
func (mt *metaTable) deleteFieldIndexMetaToEtcd(segID UniqueID, fieldID UniqueID, indexID UniqueID) error {
key := "/indexmeta/" + strconv.FormatInt(segID, 10) + strconv.FormatInt(fieldID, 10) + strconv.FormatInt(indexID, 10)
return mt.client.Remove(key)
}
func (mt *metaTable) HasFieldIndexMeta(segID UniqueID, fieldID UniqueID, indexType string, indexParams []*commonpb.KeyValuePair) (bool, error) {
mt.indexLock.RLock()
defer mt.indexLock.RUnlock()
if _, ok := mt.segID2IndexMetas[segID]; !ok {
return false, nil
}
for _, v := range mt.segID2IndexMetas[segID] {
if v.FieldID == fieldID && v.IndexType == indexType && typeutil.CompareIndexParams(v.IndexParams, indexParams) {
return true, nil
}
}
return false, nil
}
func (mt *metaTable) UpdateFieldIndexMeta(meta *pb.FieldIndexMeta) error {
mt.indexLock.Lock()
defer mt.indexLock.Unlock()
segID := meta.SegmentID
if _, ok := mt.segID2IndexMetas[segID]; !ok {
mt.segID2IndexMetas[segID] = make([]pb.FieldIndexMeta, 0)
}
for i, v := range mt.segID2IndexMetas[segID] {
if v.FieldID == meta.FieldID && v.IndexType == meta.IndexType && typeutil.CompareIndexParams(v.IndexParams, meta.IndexParams) {
mt.segID2IndexMetas[segID][i] = *meta
err := mt.deleteFieldIndexMetaToEtcd(segID, v.FieldID, v.IndexID)
if err != nil {
_ = mt.reloadFromKV()
return err
}
err = mt.saveFieldIndexMetaToEtcd(meta)
if err != nil {
_ = mt.reloadFromKV()
return err
}
return nil
}
}
mt.segID2IndexMetas[segID] = append(mt.segID2IndexMetas[segID], *meta)
err := mt.saveFieldIndexMetaToEtcd(meta)
if err != nil {
_ = mt.reloadFromKV()
return err
}
return nil
}
func (mt *metaTable) removeSegmentIndexMeta(segID UniqueID) error {
mt.indexLock.Lock()
defer mt.indexLock.Unlock()
delete(mt.segID2IndexMetas, segID)
keys, _, err := mt.client.LoadWithPrefix("indexmeta/" + strconv.FormatInt(segID, 10))
if err != nil {
_ = mt.reloadFromKV()
return err
}
if err = mt.client.MultiRemove(keys); err != nil {
_ = mt.reloadFromKV()
return err
}
return nil
}

View File

@ -6,6 +6,11 @@ import (
"strconv"
"testing"
"github.com/golang/protobuf/proto"
"github.com/zilliztech/milvus-distributed/internal/proto/commonpb"
"github.com/zilliztech/milvus-distributed/internal/proto/indexbuilderpb"
"github.com/stretchr/testify/assert"
etcdkv "github.com/zilliztech/milvus-distributed/internal/kv/etcd"
pb "github.com/zilliztech/milvus-distributed/internal/proto/etcdpb"
@ -411,3 +416,94 @@ func TestMetaTable_AddPartition_Limit(t *testing.T) {
err = meta.AddPartition(100, "partition_limit")
assert.NotNil(t, err)
}
func TestMetaTable_LoadIndexMetaFromKv(t *testing.T) {
Init()
etcdAddr := Params.EtcdAddress
cli, err := clientv3.New(clientv3.Config{Endpoints: []string{etcdAddr}})
assert.Nil(t, err)
kv := etcdkv.NewEtcdKV(cli, "/etcd/test/root")
_, err = cli.Delete(context.TODO(), "/etcd/test/root", clientv3.WithPrefix())
assert.Nil(t, err)
meta := pb.FieldIndexMeta{
SegmentID: 1,
FieldID: 100,
IndexType: "type1",
IndexID: 1000,
IndexParams: []*commonpb.KeyValuePair{{Key: "k1", Value: "v1"}},
Status: indexbuilderpb.IndexStatus_FINISHED,
IndexFilePaths: []string{"path1"},
}
marshalRes := proto.MarshalTextString(&meta)
err = kv.Save("/indexmeta/"+strconv.FormatInt(meta.SegmentID, 10)+strconv.FormatInt(meta.FieldID, 10)+strconv.FormatInt(meta.IndexID, 10), marshalRes)
assert.Nil(t, err)
metaTable, err := NewMetaTable(kv)
assert.Nil(t, err)
res, err := metaTable.HasFieldIndexMeta(1, 100, "type1", []*commonpb.KeyValuePair{{Key: "k1", Value: "v1"}})
assert.Nil(t, err)
assert.True(t, res)
}
func TestMetaTable_IndexMeta(t *testing.T) {
Init()
etcdAddr := Params.EtcdAddress
cli, err := clientv3.New(clientv3.Config{Endpoints: []string{etcdAddr}})
assert.Nil(t, err)
etcdKV := etcdkv.NewEtcdKV(cli, "/etcd/test/root")
_, err = cli.Delete(context.TODO(), "/etcd/test/root", clientv3.WithPrefix())
assert.Nil(t, err)
meta, err := NewMetaTable(etcdKV)
assert.Nil(t, err)
err = meta.AddFieldIndexMeta(&pb.FieldIndexMeta{
SegmentID: 1,
FieldID: 100,
IndexType: "type1",
IndexID: 1000,
IndexParams: []*commonpb.KeyValuePair{{Key: "k1", Value: "v1"}},
Status: indexbuilderpb.IndexStatus_INPROGRESS,
IndexFilePaths: []string{},
})
assert.Nil(t, err)
err = meta.AddFieldIndexMeta(&pb.FieldIndexMeta{
SegmentID: 1,
FieldID: 100,
IndexType: "type1",
IndexID: 1000,
IndexParams: []*commonpb.KeyValuePair{{Key: "k1", Value: "v1"}},
Status: indexbuilderpb.IndexStatus_INPROGRESS,
IndexFilePaths: []string{},
})
assert.NotNil(t, err)
res, err := meta.HasFieldIndexMeta(1, 100, "type1", []*commonpb.KeyValuePair{{Key: "k1", Value: "v1"}})
assert.Nil(t, err)
assert.True(t, res)
res, err = meta.HasFieldIndexMeta(1, 100, "type2", []*commonpb.KeyValuePair{{Key: "k1", Value: "v1"}})
assert.Nil(t, err)
assert.False(t, res)
err = meta.UpdateFieldIndexMeta(&pb.FieldIndexMeta{
SegmentID: 1,
FieldID: 100,
IndexType: "type1",
IndexID: 1000,
IndexParams: []*commonpb.KeyValuePair{{Key: "k1", Value: "v1"}},
Status: indexbuilderpb.IndexStatus_FINISHED,
IndexFilePaths: []string{},
})
assert.Nil(t, err)
assert.EqualValues(t, indexbuilderpb.IndexStatus_FINISHED, meta.segID2IndexMetas[1][0].Status)
err = meta.DeleteFieldIndexMeta(1, 100, "type1", []*commonpb.KeyValuePair{{Key: "k1", Value: "v1"}})
assert.Nil(t, err)
res, err = meta.HasFieldIndexMeta(1, 100, "type1", []*commonpb.KeyValuePair{{Key: "k1", Value: "v1"}})
assert.Nil(t, err)
assert.False(t, res)
}

View File

@ -2,6 +2,7 @@ package master
import (
"github.com/zilliztech/milvus-distributed/internal/proto/commonpb"
"github.com/zilliztech/milvus-distributed/internal/util/typeutil"
)
type RuntimeStats struct {
@ -20,7 +21,7 @@ func (rs *RuntimeStats) UpdateFieldStat(collID UniqueID, fieldID UniqueID, stats
collRuntimeStats := rs.collStats[collID]
fieldStats := collRuntimeStats.fieldStats[fieldID]
for i, v := range fieldStats {
if v.peerID == peerID { // todo compare index params
if v.peerID == peerID && typeutil.CompareIndexParams(v.indexParams, stats.indexParams) {
fieldStats[i] = stats
return nil
}

View File

@ -2,6 +2,7 @@ syntax = "proto3";
package milvus.proto.etcd;
option go_package="github.com/zilliztech/milvus-distributed/internal/proto/etcdpb";
import "index_builder.proto";
import "common.proto";
import "schema.proto";
@ -28,6 +29,10 @@ message CollectionMeta {
repeated string partition_tags=5;
}
message FieldBinlogFiles {
int64 fieldID = 1;
repeated string binlog_files = 2;
}
message SegmentMeta {
int64 segmentID=1;
@ -39,4 +44,15 @@ message SegmentMeta {
uint64 close_time=7;
int64 num_rows=8;
int64 mem_size=9;
repeated FieldBinlogFiles binlog_file_paths = 10;
}
message FieldIndexMeta {
int64 segmentID = 1;
int64 fieldID = 2;
string index_type = 3;
int64 indexID = 4;
repeated common.KeyValuePair index_params = 5;
service.IndexStatus status = 6;
repeated string index_file_paths = 7;
}

View File

@ -7,6 +7,7 @@ import (
fmt "fmt"
proto "github.com/golang/protobuf/proto"
commonpb "github.com/zilliztech/milvus-distributed/internal/proto/commonpb"
indexbuilderpb "github.com/zilliztech/milvus-distributed/internal/proto/indexbuilderpb"
schemapb "github.com/zilliztech/milvus-distributed/internal/proto/schemapb"
math "math"
)
@ -211,26 +212,74 @@ func (m *CollectionMeta) GetPartitionTags() []string {
return nil
}
type SegmentMeta struct {
SegmentID int64 `protobuf:"varint,1,opt,name=segmentID,proto3" json:"segmentID,omitempty"`
CollectionID int64 `protobuf:"varint,2,opt,name=collectionID,proto3" json:"collectionID,omitempty"`
PartitionTag string `protobuf:"bytes,3,opt,name=partition_tag,json=partitionTag,proto3" json:"partition_tag,omitempty"`
ChannelStart int32 `protobuf:"varint,4,opt,name=channel_start,json=channelStart,proto3" json:"channel_start,omitempty"`
ChannelEnd int32 `protobuf:"varint,5,opt,name=channel_end,json=channelEnd,proto3" json:"channel_end,omitempty"`
OpenTime uint64 `protobuf:"varint,6,opt,name=open_time,json=openTime,proto3" json:"open_time,omitempty"`
CloseTime uint64 `protobuf:"varint,7,opt,name=close_time,json=closeTime,proto3" json:"close_time,omitempty"`
NumRows int64 `protobuf:"varint,8,opt,name=num_rows,json=numRows,proto3" json:"num_rows,omitempty"`
MemSize int64 `protobuf:"varint,9,opt,name=mem_size,json=memSize,proto3" json:"mem_size,omitempty"`
type FieldBinlogFiles struct {
FieldID int64 `protobuf:"varint,1,opt,name=fieldID,proto3" json:"fieldID,omitempty"`
BinlogFiles []string `protobuf:"bytes,2,rep,name=binlog_files,json=binlogFiles,proto3" json:"binlog_files,omitempty"`
XXX_NoUnkeyedLiteral struct{} `json:"-"`
XXX_unrecognized []byte `json:"-"`
XXX_sizecache int32 `json:"-"`
}
func (m *FieldBinlogFiles) Reset() { *m = FieldBinlogFiles{} }
func (m *FieldBinlogFiles) String() string { return proto.CompactTextString(m) }
func (*FieldBinlogFiles) ProtoMessage() {}
func (*FieldBinlogFiles) Descriptor() ([]byte, []int) {
return fileDescriptor_975d306d62b73e88, []int{3}
}
func (m *FieldBinlogFiles) XXX_Unmarshal(b []byte) error {
return xxx_messageInfo_FieldBinlogFiles.Unmarshal(m, b)
}
func (m *FieldBinlogFiles) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
return xxx_messageInfo_FieldBinlogFiles.Marshal(b, m, deterministic)
}
func (m *FieldBinlogFiles) XXX_Merge(src proto.Message) {
xxx_messageInfo_FieldBinlogFiles.Merge(m, src)
}
func (m *FieldBinlogFiles) XXX_Size() int {
return xxx_messageInfo_FieldBinlogFiles.Size(m)
}
func (m *FieldBinlogFiles) XXX_DiscardUnknown() {
xxx_messageInfo_FieldBinlogFiles.DiscardUnknown(m)
}
var xxx_messageInfo_FieldBinlogFiles proto.InternalMessageInfo
func (m *FieldBinlogFiles) GetFieldID() int64 {
if m != nil {
return m.FieldID
}
return 0
}
func (m *FieldBinlogFiles) GetBinlogFiles() []string {
if m != nil {
return m.BinlogFiles
}
return nil
}
type SegmentMeta struct {
SegmentID int64 `protobuf:"varint,1,opt,name=segmentID,proto3" json:"segmentID,omitempty"`
CollectionID int64 `protobuf:"varint,2,opt,name=collectionID,proto3" json:"collectionID,omitempty"`
PartitionTag string `protobuf:"bytes,3,opt,name=partition_tag,json=partitionTag,proto3" json:"partition_tag,omitempty"`
ChannelStart int32 `protobuf:"varint,4,opt,name=channel_start,json=channelStart,proto3" json:"channel_start,omitempty"`
ChannelEnd int32 `protobuf:"varint,5,opt,name=channel_end,json=channelEnd,proto3" json:"channel_end,omitempty"`
OpenTime uint64 `protobuf:"varint,6,opt,name=open_time,json=openTime,proto3" json:"open_time,omitempty"`
CloseTime uint64 `protobuf:"varint,7,opt,name=close_time,json=closeTime,proto3" json:"close_time,omitempty"`
NumRows int64 `protobuf:"varint,8,opt,name=num_rows,json=numRows,proto3" json:"num_rows,omitempty"`
MemSize int64 `protobuf:"varint,9,opt,name=mem_size,json=memSize,proto3" json:"mem_size,omitempty"`
BinlogFilePaths []*FieldBinlogFiles `protobuf:"bytes,10,rep,name=binlog_file_paths,json=binlogFilePaths,proto3" json:"binlog_file_paths,omitempty"`
XXX_NoUnkeyedLiteral struct{} `json:"-"`
XXX_unrecognized []byte `json:"-"`
XXX_sizecache int32 `json:"-"`
}
func (m *SegmentMeta) Reset() { *m = SegmentMeta{} }
func (m *SegmentMeta) String() string { return proto.CompactTextString(m) }
func (*SegmentMeta) ProtoMessage() {}
func (*SegmentMeta) Descriptor() ([]byte, []int) {
return fileDescriptor_975d306d62b73e88, []int{3}
return fileDescriptor_975d306d62b73e88, []int{4}
}
func (m *SegmentMeta) XXX_Unmarshal(b []byte) error {
@ -314,48 +363,157 @@ func (m *SegmentMeta) GetMemSize() int64 {
return 0
}
func (m *SegmentMeta) GetBinlogFilePaths() []*FieldBinlogFiles {
if m != nil {
return m.BinlogFilePaths
}
return nil
}
type FieldIndexMeta struct {
SegmentID int64 `protobuf:"varint,1,opt,name=segmentID,proto3" json:"segmentID,omitempty"`
FieldID int64 `protobuf:"varint,2,opt,name=fieldID,proto3" json:"fieldID,omitempty"`
IndexType string `protobuf:"bytes,3,opt,name=index_type,json=indexType,proto3" json:"index_type,omitempty"`
IndexID int64 `protobuf:"varint,4,opt,name=indexID,proto3" json:"indexID,omitempty"`
IndexParams []*commonpb.KeyValuePair `protobuf:"bytes,5,rep,name=index_params,json=indexParams,proto3" json:"index_params,omitempty"`
Status indexbuilderpb.IndexStatus `protobuf:"varint,6,opt,name=status,proto3,enum=milvus.proto.service.IndexStatus" json:"status,omitempty"`
IndexFilePaths []string `protobuf:"bytes,7,rep,name=index_file_paths,json=indexFilePaths,proto3" json:"index_file_paths,omitempty"`
XXX_NoUnkeyedLiteral struct{} `json:"-"`
XXX_unrecognized []byte `json:"-"`
XXX_sizecache int32 `json:"-"`
}
func (m *FieldIndexMeta) Reset() { *m = FieldIndexMeta{} }
func (m *FieldIndexMeta) String() string { return proto.CompactTextString(m) }
func (*FieldIndexMeta) ProtoMessage() {}
func (*FieldIndexMeta) Descriptor() ([]byte, []int) {
return fileDescriptor_975d306d62b73e88, []int{5}
}
func (m *FieldIndexMeta) XXX_Unmarshal(b []byte) error {
return xxx_messageInfo_FieldIndexMeta.Unmarshal(m, b)
}
func (m *FieldIndexMeta) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
return xxx_messageInfo_FieldIndexMeta.Marshal(b, m, deterministic)
}
func (m *FieldIndexMeta) XXX_Merge(src proto.Message) {
xxx_messageInfo_FieldIndexMeta.Merge(m, src)
}
func (m *FieldIndexMeta) XXX_Size() int {
return xxx_messageInfo_FieldIndexMeta.Size(m)
}
func (m *FieldIndexMeta) XXX_DiscardUnknown() {
xxx_messageInfo_FieldIndexMeta.DiscardUnknown(m)
}
var xxx_messageInfo_FieldIndexMeta proto.InternalMessageInfo
func (m *FieldIndexMeta) GetSegmentID() int64 {
if m != nil {
return m.SegmentID
}
return 0
}
func (m *FieldIndexMeta) GetFieldID() int64 {
if m != nil {
return m.FieldID
}
return 0
}
func (m *FieldIndexMeta) GetIndexType() string {
if m != nil {
return m.IndexType
}
return ""
}
func (m *FieldIndexMeta) GetIndexID() int64 {
if m != nil {
return m.IndexID
}
return 0
}
func (m *FieldIndexMeta) GetIndexParams() []*commonpb.KeyValuePair {
if m != nil {
return m.IndexParams
}
return nil
}
func (m *FieldIndexMeta) GetStatus() indexbuilderpb.IndexStatus {
if m != nil {
return m.Status
}
return indexbuilderpb.IndexStatus_NONE
}
func (m *FieldIndexMeta) GetIndexFilePaths() []string {
if m != nil {
return m.IndexFilePaths
}
return nil
}
func init() {
proto.RegisterType((*TenantMeta)(nil), "milvus.proto.etcd.TenantMeta")
proto.RegisterType((*ProxyMeta)(nil), "milvus.proto.etcd.ProxyMeta")
proto.RegisterType((*CollectionMeta)(nil), "milvus.proto.etcd.CollectionMeta")
proto.RegisterType((*FieldBinlogFiles)(nil), "milvus.proto.etcd.FieldBinlogFiles")
proto.RegisterType((*SegmentMeta)(nil), "milvus.proto.etcd.SegmentMeta")
proto.RegisterType((*FieldIndexMeta)(nil), "milvus.proto.etcd.FieldIndexMeta")
}
func init() { proto.RegisterFile("etcd_meta.proto", fileDescriptor_975d306d62b73e88) }
var fileDescriptor_975d306d62b73e88 = []byte{
// 521 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x6c, 0x93, 0xcd, 0x8e, 0xd3, 0x30,
0x14, 0x85, 0x95, 0x66, 0xfa, 0x93, 0xdb, 0x3f, 0x26, 0xab, 0x30, 0x0c, 0x50, 0x15, 0x0d, 0x54,
0x42, 0xb4, 0x12, 0x48, 0xec, 0x40, 0xc0, 0x94, 0x45, 0x17, 0x20, 0x48, 0xbb, 0x62, 0x13, 0xb9,
0xc9, 0x55, 0x6b, 0x29, 0xb6, 0x8b, 0xed, 0x30, 0x4c, 0x57, 0x3c, 0x07, 0xcf, 0xc0, 0x7b, 0xf0,
0x5a, 0xc8, 0x3f, 0xa4, 0x53, 0xa9, 0xcb, 0x7c, 0xe7, 0xc4, 0x39, 0xf7, 0x5c, 0x07, 0x86, 0xa8,
0xf3, 0x22, 0x63, 0xa8, 0xc9, 0x74, 0x27, 0x85, 0x16, 0xf1, 0x39, 0xa3, 0xe5, 0x8f, 0x4a, 0xb9,
0xa7, 0xa9, 0x51, 0x2f, 0x7a, 0xb9, 0x60, 0x4c, 0x70, 0x87, 0x2e, 0x7a, 0x2a, 0xdf, 0x22, 0xf3,
0xf6, 0xf1, 0xef, 0x00, 0x60, 0x85, 0x9c, 0x70, 0xfd, 0x09, 0x35, 0x89, 0x07, 0xd0, 0x58, 0xcc,
0x93, 0x60, 0x14, 0x4c, 0xc2, 0xb4, 0xb1, 0x98, 0xc7, 0x4f, 0x61, 0xc8, 0x2b, 0x96, 0x7d, 0xaf,
0x50, 0xde, 0x66, 0x5c, 0x14, 0xa8, 0x92, 0x86, 0x15, 0xfb, 0xbc, 0x62, 0x5f, 0x0d, 0xfd, 0x6c,
0x60, 0xfc, 0x1c, 0xce, 0x29, 0x57, 0x28, 0x75, 0x96, 0x6f, 0x09, 0xe7, 0x58, 0x2e, 0xe6, 0x2a,
0x09, 0x47, 0xe1, 0x24, 0x4a, 0xef, 0x39, 0xe1, 0xba, 0xe6, 0xf1, 0x33, 0x18, 0xba, 0x03, 0x6b,
0x6f, 0x72, 0x36, 0x0a, 0x26, 0x51, 0x3a, 0xb0, 0xb8, 0x76, 0x8e, 0x7f, 0x05, 0x10, 0x7d, 0x91,
0xe2, 0xe7, 0xed, 0xc9, 0x6c, 0xaf, 0xa1, 0x4d, 0x8a, 0x42, 0xa2, 0x72, 0x99, 0xba, 0x2f, 0x2f,
0xa7, 0x47, 0xb3, 0xfb, 0xa9, 0xdf, 0x3b, 0x4f, 0xfa, 0xdf, 0x6c, 0xb2, 0x4a, 0x54, 0x55, 0x79,
0x2a, 0xab, 0x13, 0x0e, 0x59, 0xc7, 0x7f, 0x03, 0x18, 0x5c, 0x8b, 0xb2, 0xc4, 0x5c, 0x53, 0xc1,
0x4f, 0xe6, 0x78, 0x03, 0x2d, 0x57, 0xa9, 0x8f, 0x71, 0x75, 0x1c, 0xc3, 0xd7, 0x7d, 0x38, 0x64,
0x69, 0x41, 0xea, 0x5f, 0x8a, 0x1f, 0x43, 0x37, 0x97, 0x48, 0x34, 0x66, 0x9a, 0x32, 0x4c, 0xc2,
0x51, 0x30, 0x39, 0x4b, 0xc1, 0xa1, 0x15, 0x65, 0x18, 0x3f, 0x02, 0x50, 0xb8, 0x61, 0xc8, 0xb5,
0x09, 0x7a, 0x36, 0x0a, 0x27, 0x61, 0x7a, 0x87, 0xc4, 0x57, 0x30, 0xd8, 0x11, 0xa9, 0xa9, 0x39,
0x3b, 0xd3, 0x64, 0xa3, 0x92, 0xa6, 0x1d, 0xa6, 0x5f, 0xd3, 0x15, 0xd9, 0xa8, 0xf1, 0x9f, 0x06,
0x74, 0x97, 0xee, 0x2d, 0x3b, 0xc6, 0x25, 0x44, 0xf5, 0x21, 0x7e, 0x9a, 0x03, 0x88, 0xc7, 0xd0,
0xcb, 0xeb, 0xc4, 0x8b, 0xb9, 0xdf, 0xfa, 0x11, 0x8b, 0x9f, 0x40, 0xff, 0xe8, 0xc3, 0x36, 0x7b,
0x94, 0xf6, 0xee, 0x7e, 0xd7, 0x98, 0x7c, 0xcd, 0x99, 0xd2, 0x44, 0x6a, 0xbb, 0xea, 0x66, 0xda,
0xf3, 0x70, 0x69, 0x98, 0xed, 0xc0, 0x9b, 0x90, 0x17, 0x49, 0xd3, 0x5a, 0xc0, 0xa3, 0x8f, 0xbc,
0x88, 0x1f, 0x40, 0x24, 0x76, 0xc8, 0x5d, 0x45, 0x2d, 0x5b, 0x51, 0xc7, 0x00, 0x5b, 0xd0, 0x43,
0x80, 0xbc, 0x14, 0xca, 0x17, 0xd8, 0xb6, 0x6a, 0x64, 0x89, 0x95, 0xef, 0x43, 0xc7, 0xdc, 0x61,
0x29, 0x6e, 0x54, 0xd2, 0xb1, 0x63, 0xb4, 0x79, 0xc5, 0x52, 0x71, 0xa3, 0x8c, 0xc4, 0x90, 0x65,
0x8a, 0xee, 0x31, 0x89, 0x9c, 0xc4, 0x90, 0x2d, 0xe9, 0x1e, 0x3f, 0xbc, 0xfb, 0xf6, 0x76, 0x43,
0xf5, 0xb6, 0x5a, 0x9b, 0x7b, 0x34, 0xdb, 0xd3, 0xb2, 0xa4, 0x7b, 0x8d, 0xf9, 0x76, 0xe6, 0x96,
0xfb, 0xa2, 0xa0, 0x4a, 0x4b, 0xba, 0xae, 0x34, 0x16, 0x33, 0xca, 0x35, 0x4a, 0x4e, 0xca, 0x99,
0xdd, 0xf8, 0xcc, 0xfc, 0x74, 0xbb, 0xf5, 0xba, 0x65, 0x9f, 0x5e, 0xfd, 0x0b, 0x00, 0x00, 0xff,
0xff, 0x59, 0xd6, 0xe1, 0xfb, 0xa3, 0x03, 0x00, 0x00,
// 725 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x8c, 0x54, 0xd1, 0x6e, 0xeb, 0x44,
0x10, 0x55, 0xe2, 0x36, 0xa9, 0x27, 0x69, 0xd2, 0x2e, 0x2f, 0xe6, 0x72, 0x2f, 0xe4, 0xa6, 0x2a,
0x44, 0x42, 0x24, 0x52, 0x91, 0x90, 0x78, 0x00, 0x41, 0x1b, 0x2a, 0x45, 0x08, 0x1a, 0x9c, 0x88,
0x07, 0x5e, 0xac, 0x8d, 0x3d, 0x4d, 0x56, 0xf2, 0xae, 0xcd, 0xee, 0xba, 0x6d, 0xf2, 0xc4, 0x77,
0xf0, 0x0d, 0xfc, 0x07, 0x5f, 0x85, 0x84, 0x76, 0xd7, 0x71, 0x92, 0x92, 0x07, 0x1e, 0xe7, 0xcc,
0x99, 0xf1, 0x99, 0x33, 0x3b, 0x86, 0x2e, 0xea, 0x38, 0x89, 0x38, 0x6a, 0x3a, 0xcc, 0x65, 0xa6,
0x33, 0x72, 0xc9, 0x59, 0xfa, 0x54, 0x28, 0x17, 0x0d, 0x4d, 0xf6, 0xcd, 0x07, 0x4c, 0x24, 0xf8,
0x12, 0x2d, 0x0a, 0x96, 0x26, 0x28, 0x5d, 0xe6, 0x4d, 0x3b, 0xce, 0x38, 0xcf, 0xc4, 0x36, 0x52,
0xf1, 0x0a, 0x79, 0xd9, 0xa3, 0xff, 0x67, 0x0d, 0x60, 0x8e, 0x82, 0x0a, 0xfd, 0x13, 0x6a, 0x4a,
0x3a, 0x50, 0x9f, 0x8c, 0x83, 0x5a, 0xaf, 0x36, 0xf0, 0xc2, 0xfa, 0x64, 0x4c, 0x3e, 0x85, 0xae,
0x28, 0x78, 0xf4, 0x7b, 0x81, 0x72, 0x1d, 0x89, 0x2c, 0x41, 0x15, 0xd4, 0x6d, 0xf2, 0x5c, 0x14,
0xfc, 0x17, 0x83, 0xfe, 0x6c, 0x40, 0xf2, 0x39, 0x5c, 0x32, 0xa1, 0x50, 0xea, 0x28, 0x5e, 0x51,
0x21, 0x30, 0x9d, 0x8c, 0x55, 0xe0, 0xf5, 0xbc, 0x81, 0x1f, 0x5e, 0xb8, 0xc4, 0x5d, 0x85, 0x93,
0xcf, 0xa0, 0xeb, 0x1a, 0x56, 0xdc, 0xe0, 0xa4, 0x57, 0x1b, 0xf8, 0x61, 0xc7, 0xc2, 0x15, 0xb3,
0xff, 0x47, 0x0d, 0xfc, 0xa9, 0xcc, 0x5e, 0xd6, 0x47, 0xb5, 0x7d, 0x05, 0x4d, 0x9a, 0x24, 0x12,
0x95, 0xd3, 0xd4, 0xba, 0x79, 0x3b, 0x3c, 0x30, 0xa4, 0x9c, 0xfa, 0x7b, 0xc7, 0x09, 0xb7, 0x64,
0xa3, 0x55, 0xa2, 0x2a, 0xd2, 0x63, 0x5a, 0x5d, 0x62, 0xa7, 0xb5, 0xff, 0x77, 0x0d, 0x3a, 0x77,
0x59, 0x9a, 0x62, 0xac, 0x59, 0x26, 0x8e, 0xea, 0xf8, 0x06, 0x1a, 0xce, 0xd2, 0x52, 0xc6, 0xf5,
0xa1, 0x8c, 0xd2, 0xee, 0x5d, 0x93, 0x99, 0x05, 0xc2, 0xb2, 0x88, 0x7c, 0x02, 0xad, 0x58, 0x22,
0xd5, 0x18, 0x69, 0xc6, 0x31, 0xf0, 0x7a, 0xb5, 0xc1, 0x49, 0x08, 0x0e, 0x9a, 0x33, 0x8e, 0xe4,
0x63, 0x00, 0x85, 0x4b, 0x8e, 0x42, 0x1b, 0xa1, 0x27, 0x3d, 0x6f, 0xe0, 0x85, 0x7b, 0x08, 0xb9,
0x86, 0x4e, 0x4e, 0xa5, 0x66, 0xa6, 0x77, 0xa4, 0xe9, 0x52, 0x05, 0xa7, 0x76, 0x98, 0xf3, 0x0a,
0x9d, 0xd3, 0xa5, 0xea, 0x3f, 0xc0, 0xc5, 0x3d, 0xc3, 0x34, 0xb9, 0x65, 0x22, 0xcd, 0x96, 0xf7,
0x2c, 0x45, 0x45, 0x02, 0x68, 0x3e, 0x1a, 0xac, 0x9a, 0x67, 0x1b, 0x92, 0xf7, 0xd0, 0x5e, 0x58,
0x62, 0xf4, 0x68, 0x98, 0x41, 0xdd, 0xb6, 0x6c, 0x2d, 0x76, 0xc5, 0xfd, 0x7f, 0xea, 0xd0, 0x9a,
0x39, 0x19, 0xd6, 0x97, 0xb7, 0xe0, 0x57, 0xaa, 0xca, 0x76, 0x3b, 0x80, 0xf4, 0xa1, 0x1d, 0x57,
0x16, 0x4c, 0xc6, 0xe5, 0x33, 0x3a, 0xc0, 0xc8, 0x15, 0x9c, 0x1f, 0x4c, 0x62, 0xcd, 0xf0, 0xc3,
0xf6, 0xfe, 0x20, 0x86, 0x54, 0xee, 0x2d, 0x52, 0x9a, 0x4a, 0x6d, 0xdf, 0xce, 0x69, 0xd8, 0x2e,
0xc1, 0x99, 0xc1, 0xac, 0xa9, 0x25, 0x09, 0x45, 0x12, 0x9c, 0x5a, 0x0a, 0x94, 0xd0, 0x0f, 0x22,
0x21, 0x1f, 0x81, 0x9f, 0xe5, 0x28, 0x9c, 0xe7, 0x0d, 0xeb, 0xf9, 0x99, 0x01, 0xac, 0xe3, 0xef,
0x00, 0xe2, 0x34, 0x53, 0xe5, 0x46, 0x9a, 0x36, 0xeb, 0x5b, 0xc4, 0xa6, 0x3f, 0x84, 0x33, 0x73,
0x14, 0x32, 0x7b, 0x56, 0xc1, 0x99, 0xb3, 0x4d, 0x14, 0x3c, 0xcc, 0x9e, 0x95, 0x49, 0x71, 0xe4,
0x91, 0x62, 0x1b, 0x0c, 0x7c, 0x97, 0xe2, 0xc8, 0x67, 0x6c, 0x83, 0xe4, 0x01, 0x2e, 0xf7, 0x1c,
0x8d, 0x72, 0xaa, 0x57, 0x2a, 0x80, 0x9e, 0x37, 0x68, 0xdd, 0x5c, 0x0d, 0xff, 0x73, 0xc9, 0xc3,
0xd7, 0xbb, 0x0a, 0xbb, 0x3b, 0xef, 0xa7, 0xa6, 0xb6, 0xff, 0x57, 0x1d, 0x3a, 0x96, 0x35, 0x31,
0x37, 0xff, 0x3f, 0x56, 0xb0, 0xb7, 0xed, 0xfa, 0xe1, 0xb6, 0xdf, 0x01, 0xb8, 0x1f, 0x87, 0x5e,
0xe7, 0x58, 0xba, 0xee, 0x5b, 0x64, 0xbe, 0xce, 0xd1, 0x14, 0xda, 0xa0, 0x3c, 0x54, 0x2f, 0xdc,
0x86, 0x64, 0x0c, 0x6d, 0x57, 0x98, 0x53, 0x49, 0xb9, 0x7b, 0x79, 0xad, 0x9b, 0xf7, 0x47, 0x0f,
0xf1, 0x47, 0x5c, 0xff, 0x4a, 0xd3, 0x02, 0xa7, 0x94, 0xc9, 0xb0, 0x65, 0xcb, 0xa6, 0xb6, 0x8a,
0x7c, 0x0d, 0x0d, 0xa5, 0xa9, 0x2e, 0x94, 0xdd, 0x44, 0xe7, 0x75, 0xbd, 0x42, 0xf9, 0xc4, 0x62,
0x1c, 0xda, 0x39, 0x67, 0x96, 0x18, 0x96, 0x05, 0x64, 0x00, 0x17, 0x4e, 0xc0, 0x9e, 0xa9, 0x4d,
0xfb, 0x56, 0x3b, 0x16, 0xaf, 0xec, 0xba, 0xfd, 0xee, 0xb7, 0x6f, 0x97, 0x4c, 0xaf, 0x8a, 0x85,
0xd1, 0x33, 0xda, 0xb0, 0x34, 0x65, 0x1b, 0x8d, 0xf1, 0x6a, 0xe4, 0xbe, 0xf5, 0x45, 0xc2, 0x94,
0x96, 0x6c, 0x51, 0x68, 0x4c, 0x46, 0x4c, 0x68, 0x94, 0x82, 0xa6, 0x23, 0x2b, 0x60, 0x64, 0x16,
0x92, 0x2f, 0x16, 0x0d, 0x1b, 0x7d, 0xf9, 0x6f, 0x00, 0x00, 0x00, 0xff, 0xff, 0x1c, 0xb6, 0x88,
0xc7, 0x89, 0x05, 0x00, 0x00,
}

View File

@ -0,0 +1,35 @@
package typeutil
import "github.com/zilliztech/milvus-distributed/internal/proto/commonpb"
func CompareIndexParams(indexParam1 []*commonpb.KeyValuePair, indexParam2 []*commonpb.KeyValuePair) bool {
if indexParam1 == nil && indexParam2 == nil {
return true
}
if indexParam1 == nil || indexParam2 == nil {
return false
}
if len(indexParam1) != len(indexParam2) {
return false
}
paramMap1 := make(map[string]string)
paramMap2 := make(map[string]string)
for _, kv := range indexParam1 {
paramMap1[kv.Key] = kv.Value
}
for _, kv := range indexParam2 {
paramMap2[kv.Key] = kv.Value
}
for k, v := range paramMap1 {
if _, ok := paramMap2[k]; !ok {
return false
}
if v != paramMap2[k] {
return false
}
}
return true
}

View File

@ -0,0 +1,26 @@
package typeutil
import (
"testing"
"github.com/stretchr/testify/assert"
"github.com/zilliztech/milvus-distributed/internal/proto/commonpb"
)
func TestCompareIndexParams(t *testing.T) {
cases := []*struct {
param1 []*commonpb.KeyValuePair
param2 []*commonpb.KeyValuePair
result bool
}{
{param1: nil, param2: nil, result: true},
{param1: nil, param2: []*commonpb.KeyValuePair{}, result: false},
{param1: []*commonpb.KeyValuePair{}, param2: []*commonpb.KeyValuePair{}, result: true},
{param1: []*commonpb.KeyValuePair{{Key: "k1", Value: "v1"}}, param2: []*commonpb.KeyValuePair{}, result: false},
{param1: []*commonpb.KeyValuePair{{Key: "k1", Value: "v1"}}, param2: []*commonpb.KeyValuePair{{Key: "k1", Value: "v1"}}, result: true},
}
for _, testcase := range cases {
assert.EqualValues(t, testcase.result, CompareIndexParams(testcase.param1, testcase.param2))
}
}