diff --git a/cpp/CHANGELOG.md b/cpp/CHANGELOG.md index e64ef8f9a0..d599b5256c 100644 --- a/cpp/CHANGELOG.md +++ b/cpp/CHANGELOG.md @@ -85,6 +85,7 @@ Please mark all change in change log and use the ticket from JIRA. - MS-460 - Put transport speed as weight when choosing neighbour to execute task - MS-459 - Add cache for pick function in tasktable - MS-482 - Change search stream transport to unary in grpc +- MS-487 - Define metric type in CreateTable - MS-488 - Improve code format in scheduler ## New Feature diff --git a/cpp/src/db/DBImpl.cpp b/cpp/src/db/DBImpl.cpp index 797ab13bd2..d3b9e1fc1a 100644 --- a/cpp/src/db/DBImpl.cpp +++ b/cpp/src/db/DBImpl.cpp @@ -250,7 +250,9 @@ Status DBImpl::CreateIndex(const std::string& table_id, const TableIndex& index) } //step 2: update index info - if(!utils::IsSameIndex(old_index, index)) { + TableIndex new_index = index; + new_index.metric_type_ = old_index.metric_type_;//dont change metric type, it was defined by CreateTable + if(!utils::IsSameIndex(old_index, new_index)) { DropIndex(table_id); status = meta_ptr_->UpdateTableIndexParam(table_id, index); diff --git a/cpp/src/db/engine/ExecutionEngineImpl.cpp b/cpp/src/db/engine/ExecutionEngineImpl.cpp index 098823c482..344d26f156 100644 --- a/cpp/src/db/engine/ExecutionEngineImpl.cpp +++ b/cpp/src/db/engine/ExecutionEngineImpl.cpp @@ -301,7 +301,8 @@ Status ExecutionEngineImpl::Search(long n, } ENGINE_LOG_DEBUG << "Search Params: [k] " << k << " [nprobe] " << nprobe; - auto ec = index_->Search(n, data, distances, labels, Config::object{{"k", k}, {"nprobe", nprobe}}); + auto cfg = Config::object{{"k", k}, {"nprobe", nprobe}}; + auto ec = index_->Search(n, data, distances, labels, cfg); if (ec != server::KNOWHERE_SUCCESS) { ENGINE_LOG_ERROR << "Search error"; return Status::Error("Search: Search Error"); diff --git a/cpp/src/db/meta/SqliteMetaImpl.cpp b/cpp/src/db/meta/SqliteMetaImpl.cpp index 4e6c1457e6..e85ab5122c 100644 --- a/cpp/src/db/meta/SqliteMetaImpl.cpp +++ b/cpp/src/db/meta/SqliteMetaImpl.cpp @@ -707,7 +707,7 @@ Status SqliteMetaImpl::FilesToSearch(const std::string &table_id, files[table_file.date_].push_back(table_file); } if(files.empty()) { - std::cout << "ERROR" << std::endl; + ENGINE_LOG_ERROR << "No file to search for table: " << table_id; } } catch (std::exception &e) { return HandleException("Encounter exception when iterate index files", e); diff --git a/cpp/src/grpc/gen-milvus/milvus.pb.cc b/cpp/src/grpc/gen-milvus/milvus.pb.cc index 2396c4509e..0a1ab2014c 100644 --- a/cpp/src/grpc/gen-milvus/milvus.pb.cc +++ b/cpp/src/grpc/gen-milvus/milvus.pb.cc @@ -387,6 +387,7 @@ const ::PROTOBUF_NAMESPACE_ID::uint32 TableStruct_milvus_2eproto::offsets[] PROT PROTOBUF_FIELD_OFFSET(::milvus::grpc::TableSchema, table_name_), PROTOBUF_FIELD_OFFSET(::milvus::grpc::TableSchema, dimension_), PROTOBUF_FIELD_OFFSET(::milvus::grpc::TableSchema, index_file_size_), + PROTOBUF_FIELD_OFFSET(::milvus::grpc::TableSchema, metric_type_), ~0u, // no _has_bits_ PROTOBUF_FIELD_OFFSET(::milvus::grpc::Range, _internal_metadata_), ~0u, // no _extensions_ @@ -486,7 +487,6 @@ const ::PROTOBUF_NAMESPACE_ID::uint32 TableStruct_milvus_2eproto::offsets[] PROT ~0u, // no _weak_field_map_ PROTOBUF_FIELD_OFFSET(::milvus::grpc::Index, index_type_), PROTOBUF_FIELD_OFFSET(::milvus::grpc::Index, nlist_), - PROTOBUF_FIELD_OFFSET(::milvus::grpc::Index, metric_type_), ~0u, // no _has_bits_ PROTOBUF_FIELD_OFFSET(::milvus::grpc::IndexParam, _internal_metadata_), ~0u, // no _extensions_ @@ -505,20 +505,20 @@ const ::PROTOBUF_NAMESPACE_ID::uint32 TableStruct_milvus_2eproto::offsets[] PROT static const ::PROTOBUF_NAMESPACE_ID::internal::MigrationSchema schemas[] PROTOBUF_SECTION_VARIABLE(protodesc_cold) = { { 0, -1, sizeof(::milvus::grpc::TableName)}, { 7, -1, sizeof(::milvus::grpc::TableSchema)}, - { 15, -1, sizeof(::milvus::grpc::Range)}, - { 22, -1, sizeof(::milvus::grpc::RowRecord)}, - { 28, -1, sizeof(::milvus::grpc::InsertParam)}, - { 36, -1, sizeof(::milvus::grpc::VectorIds)}, - { 43, -1, sizeof(::milvus::grpc::SearchParam)}, - { 53, -1, sizeof(::milvus::grpc::SearchInFilesParam)}, - { 60, -1, sizeof(::milvus::grpc::QueryResult)}, - { 67, -1, sizeof(::milvus::grpc::TopKQueryResult)}, - { 73, -1, sizeof(::milvus::grpc::TopKQueryResultList)}, - { 80, -1, sizeof(::milvus::grpc::StringReply)}, - { 87, -1, sizeof(::milvus::grpc::BoolReply)}, - { 94, -1, sizeof(::milvus::grpc::TableRowCount)}, - { 101, -1, sizeof(::milvus::grpc::Command)}, - { 107, -1, sizeof(::milvus::grpc::Index)}, + { 16, -1, sizeof(::milvus::grpc::Range)}, + { 23, -1, sizeof(::milvus::grpc::RowRecord)}, + { 29, -1, sizeof(::milvus::grpc::InsertParam)}, + { 37, -1, sizeof(::milvus::grpc::VectorIds)}, + { 44, -1, sizeof(::milvus::grpc::SearchParam)}, + { 54, -1, sizeof(::milvus::grpc::SearchInFilesParam)}, + { 61, -1, sizeof(::milvus::grpc::QueryResult)}, + { 68, -1, sizeof(::milvus::grpc::TopKQueryResult)}, + { 74, -1, sizeof(::milvus::grpc::TopKQueryResultList)}, + { 81, -1, sizeof(::milvus::grpc::StringReply)}, + { 88, -1, sizeof(::milvus::grpc::BoolReply)}, + { 95, -1, sizeof(::milvus::grpc::TableRowCount)}, + { 102, -1, sizeof(::milvus::grpc::Command)}, + { 108, -1, sizeof(::milvus::grpc::Index)}, { 115, -1, sizeof(::milvus::grpc::IndexParam)}, { 122, -1, sizeof(::milvus::grpc::DeleteByRangeParam)}, }; @@ -547,37 +547,37 @@ static ::PROTOBUF_NAMESPACE_ID::Message const * const file_default_instances[] = const char descriptor_table_protodef_milvus_2eproto[] PROTOBUF_SECTION_VARIABLE(protodesc_cold) = "\n\014milvus.proto\022\013milvus.grpc\032\014status.prot" "o\"D\n\tTableName\022#\n\006status\030\001 \001(\0132\023.milvus." - "grpc.Status\022\022\n\ntable_name\030\002 \001(\t\"e\n\013Table" + "grpc.Status\022\022\n\ntable_name\030\002 \001(\t\"z\n\013Table" "Schema\022*\n\ntable_name\030\001 \001(\0132\026.milvus.grpc" ".TableName\022\021\n\tdimension\030\002 \001(\003\022\027\n\017index_f" - "ile_size\030\003 \001(\003\"/\n\005Range\022\023\n\013start_value\030\001" - " \001(\t\022\021\n\tend_value\030\002 \001(\t\" \n\tRowRecord\022\023\n\013" - "vector_data\030\001 \003(\002\"i\n\013InsertParam\022\022\n\ntabl" - "e_name\030\001 \001(\t\0220\n\020row_record_array\030\002 \003(\0132\026" - ".milvus.grpc.RowRecord\022\024\n\014row_id_array\030\003" - " \003(\003\"I\n\tVectorIds\022#\n\006status\030\001 \001(\0132\023.milv" - "us.grpc.Status\022\027\n\017vector_id_array\030\002 \003(\003\"" - "\242\001\n\013SearchParam\022\022\n\ntable_name\030\001 \001(\t\0222\n\022q" - "uery_record_array\030\002 \003(\0132\026.milvus.grpc.Ro" - "wRecord\022-\n\021query_range_array\030\003 \003(\0132\022.mil" - "vus.grpc.Range\022\014\n\004topk\030\004 \001(\003\022\016\n\006nprobe\030\005" - " \001(\003\"[\n\022SearchInFilesParam\022\025\n\rfile_id_ar" - "ray\030\001 \003(\t\022.\n\014search_param\030\002 \001(\0132\030.milvus" - ".grpc.SearchParam\"+\n\013QueryResult\022\n\n\002id\030\001" - " \001(\003\022\020\n\010distance\030\002 \001(\001\"H\n\017TopKQueryResul" - "t\0225\n\023query_result_arrays\030\001 \003(\0132\030.milvus." - "grpc.QueryResult\"s\n\023TopKQueryResultList\022" - "#\n\006status\030\001 \001(\0132\023.milvus.grpc.Status\0227\n\021" - "topk_query_result\030\002 \003(\0132\034.milvus.grpc.To" - "pKQueryResult\"H\n\013StringReply\022#\n\006status\030\001" - " \001(\0132\023.milvus.grpc.Status\022\024\n\014string_repl" - "y\030\002 \001(\t\"D\n\tBoolReply\022#\n\006status\030\001 \001(\0132\023.m" - "ilvus.grpc.Status\022\022\n\nbool_reply\030\002 \001(\010\"M\n" - "\rTableRowCount\022#\n\006status\030\001 \001(\0132\023.milvus." - "grpc.Status\022\027\n\017table_row_count\030\002 \001(\003\"\026\n\007" - "Command\022\013\n\003cmd\030\001 \001(\t\"\?\n\005Index\022\022\n\nindex_t" - "ype\030\001 \001(\005\022\r\n\005nlist\030\002 \001(\005\022\023\n\013metric_type\030" - "\003 \001(\005\"[\n\nIndexParam\022*\n\ntable_name\030\001 \001(\0132" + "ile_size\030\003 \001(\003\022\023\n\013metric_type\030\004 \001(\005\"/\n\005R" + "ange\022\023\n\013start_value\030\001 \001(\t\022\021\n\tend_value\030\002" + " \001(\t\" \n\tRowRecord\022\023\n\013vector_data\030\001 \003(\002\"i" + "\n\013InsertParam\022\022\n\ntable_name\030\001 \001(\t\0220\n\020row" + "_record_array\030\002 \003(\0132\026.milvus.grpc.RowRec" + "ord\022\024\n\014row_id_array\030\003 \003(\003\"I\n\tVectorIds\022#" + "\n\006status\030\001 \001(\0132\023.milvus.grpc.Status\022\027\n\017v" + "ector_id_array\030\002 \003(\003\"\242\001\n\013SearchParam\022\022\n\n" + "table_name\030\001 \001(\t\0222\n\022query_record_array\030\002" + " \003(\0132\026.milvus.grpc.RowRecord\022-\n\021query_ra" + "nge_array\030\003 \003(\0132\022.milvus.grpc.Range\022\014\n\004t" + "opk\030\004 \001(\003\022\016\n\006nprobe\030\005 \001(\003\"[\n\022SearchInFil" + "esParam\022\025\n\rfile_id_array\030\001 \003(\t\022.\n\014search" + "_param\030\002 \001(\0132\030.milvus.grpc.SearchParam\"+" + "\n\013QueryResult\022\n\n\002id\030\001 \001(\003\022\020\n\010distance\030\002 " + "\001(\001\"H\n\017TopKQueryResult\0225\n\023query_result_a" + "rrays\030\001 \003(\0132\030.milvus.grpc.QueryResult\"s\n" + "\023TopKQueryResultList\022#\n\006status\030\001 \001(\0132\023.m" + "ilvus.grpc.Status\0227\n\021topk_query_result\030\002" + " \003(\0132\034.milvus.grpc.TopKQueryResult\"H\n\013St" + "ringReply\022#\n\006status\030\001 \001(\0132\023.milvus.grpc." + "Status\022\024\n\014string_reply\030\002 \001(\t\"D\n\tBoolRepl" + "y\022#\n\006status\030\001 \001(\0132\023.milvus.grpc.Status\022\022" + "\n\nbool_reply\030\002 \001(\010\"M\n\rTableRowCount\022#\n\006s" + "tatus\030\001 \001(\0132\023.milvus.grpc.Status\022\027\n\017tabl" + "e_row_count\030\002 \001(\003\"\026\n\007Command\022\013\n\003cmd\030\001 \001(" + "\t\"*\n\005Index\022\022\n\nindex_type\030\001 \001(\005\022\r\n\005nlist\030" + "\002 \001(\005\"[\n\nIndexParam\022*\n\ntable_name\030\001 \001(\0132" "\026.milvus.grpc.TableName\022!\n\005index\030\002 \001(\0132\022" ".milvus.grpc.Index\"K\n\022DeleteByRangeParam" "\022!\n\005range\030\001 \001(\0132\022.milvus.grpc.Range\022\022\n\nt" @@ -1010,16 +1010,16 @@ TableSchema::TableSchema(const TableSchema& from) table_name_ = nullptr; } ::memcpy(&dimension_, &from.dimension_, - static_cast(reinterpret_cast(&index_file_size_) - - reinterpret_cast(&dimension_)) + sizeof(index_file_size_)); + static_cast(reinterpret_cast(&metric_type_) - + reinterpret_cast(&dimension_)) + sizeof(metric_type_)); // @@protoc_insertion_point(copy_constructor:milvus.grpc.TableSchema) } void TableSchema::SharedCtor() { ::PROTOBUF_NAMESPACE_ID::internal::InitSCC(&scc_info_TableSchema_milvus_2eproto.base); ::memset(&table_name_, 0, static_cast( - reinterpret_cast(&index_file_size_) - - reinterpret_cast(&table_name_)) + sizeof(index_file_size_)); + reinterpret_cast(&metric_type_) - + reinterpret_cast(&table_name_)) + sizeof(metric_type_)); } TableSchema::~TableSchema() { @@ -1051,8 +1051,8 @@ void TableSchema::Clear() { } table_name_ = nullptr; ::memset(&dimension_, 0, static_cast( - reinterpret_cast(&index_file_size_) - - reinterpret_cast(&dimension_)) + sizeof(index_file_size_)); + reinterpret_cast(&metric_type_) - + reinterpret_cast(&dimension_)) + sizeof(metric_type_)); _internal_metadata_.Clear(); } @@ -1085,6 +1085,13 @@ const char* TableSchema::_InternalParse(const char* ptr, ::PROTOBUF_NAMESPACE_ID CHK_(ptr); } else goto handle_unusual; continue; + // int32 metric_type = 4; + case 4: + if (PROTOBUF_PREDICT_TRUE(static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 32)) { + metric_type_ = ::PROTOBUF_NAMESPACE_ID::internal::ReadVarint(&ptr); + CHK_(ptr); + } else goto handle_unusual; + continue; default: { handle_unusual: if ((tag & 7) == 4 || tag == 0) { @@ -1152,6 +1159,19 @@ bool TableSchema::MergePartialFromCodedStream( break; } + // int32 metric_type = 4; + case 4: { + if (static_cast< ::PROTOBUF_NAMESPACE_ID::uint8>(tag) == (32 & 0xFF)) { + + DO_((::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::ReadPrimitive< + ::PROTOBUF_NAMESPACE_ID::int32, ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::TYPE_INT32>( + input, &metric_type_))); + } else { + goto handle_unusual; + } + break; + } + default: { handle_unusual: if (tag == 0) { @@ -1195,6 +1215,11 @@ void TableSchema::SerializeWithCachedSizes( ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteInt64(3, this->index_file_size(), output); } + // int32 metric_type = 4; + if (this->metric_type() != 0) { + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteInt32(4, this->metric_type(), output); + } + if (_internal_metadata_.have_unknown_fields()) { ::PROTOBUF_NAMESPACE_ID::internal::WireFormat::SerializeUnknownFields( _internal_metadata_.unknown_fields(), output); @@ -1225,6 +1250,11 @@ void TableSchema::SerializeWithCachedSizes( target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteInt64ToArray(3, this->index_file_size(), target); } + // int32 metric_type = 4; + if (this->metric_type() != 0) { + target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteInt32ToArray(4, this->metric_type(), target); + } + if (_internal_metadata_.have_unknown_fields()) { target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormat::SerializeUnknownFieldsToArray( _internal_metadata_.unknown_fields(), target); @@ -1267,6 +1297,13 @@ size_t TableSchema::ByteSizeLong() const { this->index_file_size()); } + // int32 metric_type = 4; + if (this->metric_type() != 0) { + total_size += 1 + + ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::Int32Size( + this->metric_type()); + } + int cached_size = ::PROTOBUF_NAMESPACE_ID::internal::ToCachedSize(total_size); SetCachedSize(cached_size); return total_size; @@ -1303,6 +1340,9 @@ void TableSchema::MergeFrom(const TableSchema& from) { if (from.index_file_size() != 0) { set_index_file_size(from.index_file_size()); } + if (from.metric_type() != 0) { + set_metric_type(from.metric_type()); + } } void TableSchema::CopyFrom(const ::PROTOBUF_NAMESPACE_ID::Message& from) { @@ -1329,6 +1369,7 @@ void TableSchema::InternalSwap(TableSchema* other) { swap(table_name_, other->table_name_); swap(dimension_, other->dimension_); swap(index_file_size_, other->index_file_size_); + swap(metric_type_, other->metric_type_); } ::PROTOBUF_NAMESPACE_ID::Metadata TableSchema::GetMetadata() const { @@ -5606,15 +5647,15 @@ Index::Index(const Index& from) _internal_metadata_(nullptr) { _internal_metadata_.MergeFrom(from._internal_metadata_); ::memcpy(&index_type_, &from.index_type_, - static_cast(reinterpret_cast(&metric_type_) - - reinterpret_cast(&index_type_)) + sizeof(metric_type_)); + static_cast(reinterpret_cast(&nlist_) - + reinterpret_cast(&index_type_)) + sizeof(nlist_)); // @@protoc_insertion_point(copy_constructor:milvus.grpc.Index) } void Index::SharedCtor() { ::memset(&index_type_, 0, static_cast( - reinterpret_cast(&metric_type_) - - reinterpret_cast(&index_type_)) + sizeof(metric_type_)); + reinterpret_cast(&nlist_) - + reinterpret_cast(&index_type_)) + sizeof(nlist_)); } Index::~Index() { @@ -5641,8 +5682,8 @@ void Index::Clear() { (void) cached_has_bits; ::memset(&index_type_, 0, static_cast( - reinterpret_cast(&metric_type_) - - reinterpret_cast(&index_type_)) + sizeof(metric_type_)); + reinterpret_cast(&nlist_) - + reinterpret_cast(&index_type_)) + sizeof(nlist_)); _internal_metadata_.Clear(); } @@ -5668,13 +5709,6 @@ const char* Index::_InternalParse(const char* ptr, ::PROTOBUF_NAMESPACE_ID::inte CHK_(ptr); } else goto handle_unusual; continue; - // int32 metric_type = 3; - case 3: - if (PROTOBUF_PREDICT_TRUE(static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 24)) { - metric_type_ = ::PROTOBUF_NAMESPACE_ID::internal::ReadVarint(&ptr); - CHK_(ptr); - } else goto handle_unusual; - continue; default: { handle_unusual: if ((tag & 7) == 4 || tag == 0) { @@ -5731,19 +5765,6 @@ bool Index::MergePartialFromCodedStream( break; } - // int32 metric_type = 3; - case 3: { - if (static_cast< ::PROTOBUF_NAMESPACE_ID::uint8>(tag) == (24 & 0xFF)) { - - DO_((::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::ReadPrimitive< - ::PROTOBUF_NAMESPACE_ID::int32, ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::TYPE_INT32>( - input, &metric_type_))); - } else { - goto handle_unusual; - } - break; - } - default: { handle_unusual: if (tag == 0) { @@ -5781,11 +5802,6 @@ void Index::SerializeWithCachedSizes( ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteInt32(2, this->nlist(), output); } - // int32 metric_type = 3; - if (this->metric_type() != 0) { - ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteInt32(3, this->metric_type(), output); - } - if (_internal_metadata_.have_unknown_fields()) { ::PROTOBUF_NAMESPACE_ID::internal::WireFormat::SerializeUnknownFields( _internal_metadata_.unknown_fields(), output); @@ -5809,11 +5825,6 @@ void Index::SerializeWithCachedSizes( target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteInt32ToArray(2, this->nlist(), target); } - // int32 metric_type = 3; - if (this->metric_type() != 0) { - target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteInt32ToArray(3, this->metric_type(), target); - } - if (_internal_metadata_.have_unknown_fields()) { target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormat::SerializeUnknownFieldsToArray( _internal_metadata_.unknown_fields(), target); @@ -5849,13 +5860,6 @@ size_t Index::ByteSizeLong() const { this->nlist()); } - // int32 metric_type = 3; - if (this->metric_type() != 0) { - total_size += 1 + - ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::Int32Size( - this->metric_type()); - } - int cached_size = ::PROTOBUF_NAMESPACE_ID::internal::ToCachedSize(total_size); SetCachedSize(cached_size); return total_size; @@ -5889,9 +5893,6 @@ void Index::MergeFrom(const Index& from) { if (from.nlist() != 0) { set_nlist(from.nlist()); } - if (from.metric_type() != 0) { - set_metric_type(from.metric_type()); - } } void Index::CopyFrom(const ::PROTOBUF_NAMESPACE_ID::Message& from) { @@ -5917,7 +5918,6 @@ void Index::InternalSwap(Index* other) { _internal_metadata_.Swap(&other->_internal_metadata_); swap(index_type_, other->index_type_); swap(nlist_, other->nlist_); - swap(metric_type_, other->metric_type_); } ::PROTOBUF_NAMESPACE_ID::Metadata Index::GetMetadata() const { diff --git a/cpp/src/grpc/gen-milvus/milvus.pb.h b/cpp/src/grpc/gen-milvus/milvus.pb.h index c9f95d79f1..543e5b98c9 100644 --- a/cpp/src/grpc/gen-milvus/milvus.pb.h +++ b/cpp/src/grpc/gen-milvus/milvus.pb.h @@ -401,6 +401,7 @@ class TableSchema : kTableNameFieldNumber = 1, kDimensionFieldNumber = 2, kIndexFileSizeFieldNumber = 3, + kMetricTypeFieldNumber = 4, }; // .milvus.grpc.TableName table_name = 1; bool has_table_name() const; @@ -420,6 +421,11 @@ class TableSchema : ::PROTOBUF_NAMESPACE_ID::int64 index_file_size() const; void set_index_file_size(::PROTOBUF_NAMESPACE_ID::int64 value); + // int32 metric_type = 4; + void clear_metric_type(); + ::PROTOBUF_NAMESPACE_ID::int32 metric_type() const; + void set_metric_type(::PROTOBUF_NAMESPACE_ID::int32 value); + // @@protoc_insertion_point(class_scope:milvus.grpc.TableSchema) private: class _Internal; @@ -428,6 +434,7 @@ class TableSchema : ::milvus::grpc::TableName* table_name_; ::PROTOBUF_NAMESPACE_ID::int64 dimension_; ::PROTOBUF_NAMESPACE_ID::int64 index_file_size_; + ::PROTOBUF_NAMESPACE_ID::int32 metric_type_; mutable ::PROTOBUF_NAMESPACE_ID::internal::CachedSize _cached_size_; friend struct ::TableStruct_milvus_2eproto; }; @@ -2466,7 +2473,6 @@ class Index : enum : int { kIndexTypeFieldNumber = 1, kNlistFieldNumber = 2, - kMetricTypeFieldNumber = 3, }; // int32 index_type = 1; void clear_index_type(); @@ -2478,11 +2484,6 @@ class Index : ::PROTOBUF_NAMESPACE_ID::int32 nlist() const; void set_nlist(::PROTOBUF_NAMESPACE_ID::int32 value); - // int32 metric_type = 3; - void clear_metric_type(); - ::PROTOBUF_NAMESPACE_ID::int32 metric_type() const; - void set_metric_type(::PROTOBUF_NAMESPACE_ID::int32 value); - // @@protoc_insertion_point(class_scope:milvus.grpc.Index) private: class _Internal; @@ -2490,7 +2491,6 @@ class Index : ::PROTOBUF_NAMESPACE_ID::internal::InternalMetadataWithArena _internal_metadata_; ::PROTOBUF_NAMESPACE_ID::int32 index_type_; ::PROTOBUF_NAMESPACE_ID::int32 nlist_; - ::PROTOBUF_NAMESPACE_ID::int32 metric_type_; mutable ::PROTOBUF_NAMESPACE_ID::internal::CachedSize _cached_size_; friend struct ::TableStruct_milvus_2eproto; }; @@ -2975,6 +2975,20 @@ inline void TableSchema::set_index_file_size(::PROTOBUF_NAMESPACE_ID::int64 valu // @@protoc_insertion_point(field_set:milvus.grpc.TableSchema.index_file_size) } +// int32 metric_type = 4; +inline void TableSchema::clear_metric_type() { + metric_type_ = 0; +} +inline ::PROTOBUF_NAMESPACE_ID::int32 TableSchema::metric_type() const { + // @@protoc_insertion_point(field_get:milvus.grpc.TableSchema.metric_type) + return metric_type_; +} +inline void TableSchema::set_metric_type(::PROTOBUF_NAMESPACE_ID::int32 value) { + + metric_type_ = value; + // @@protoc_insertion_point(field_set:milvus.grpc.TableSchema.metric_type) +} + // ------------------------------------------------------------------- // Range @@ -4030,20 +4044,6 @@ inline void Index::set_nlist(::PROTOBUF_NAMESPACE_ID::int32 value) { // @@protoc_insertion_point(field_set:milvus.grpc.Index.nlist) } -// int32 metric_type = 3; -inline void Index::clear_metric_type() { - metric_type_ = 0; -} -inline ::PROTOBUF_NAMESPACE_ID::int32 Index::metric_type() const { - // @@protoc_insertion_point(field_get:milvus.grpc.Index.metric_type) - return metric_type_; -} -inline void Index::set_metric_type(::PROTOBUF_NAMESPACE_ID::int32 value) { - - metric_type_ = value; - // @@protoc_insertion_point(field_set:milvus.grpc.Index.metric_type) -} - // ------------------------------------------------------------------- // IndexParam diff --git a/cpp/src/grpc/milvus.proto b/cpp/src/grpc/milvus.proto index d2b7a0229a..fa3709e4e5 100644 --- a/cpp/src/grpc/milvus.proto +++ b/cpp/src/grpc/milvus.proto @@ -19,6 +19,7 @@ message TableSchema { TableName table_name = 1; int64 dimension = 2; int64 index_file_size = 3; + int32 metric_type = 4; } /** @@ -134,7 +135,6 @@ message Command { message Index { int32 index_type = 1; int32 nlist = 2; - int32 metric_type = 3; } /** diff --git a/cpp/src/sdk/examples/grpcsimple/src/ClientTest.cpp b/cpp/src/sdk/examples/grpcsimple/src/ClientTest.cpp index 8198d5a232..84f40bd40c 100644 --- a/cpp/src/sdk/examples/grpcsimple/src/ClientTest.cpp +++ b/cpp/src/sdk/examples/grpcsimple/src/ClientTest.cpp @@ -96,6 +96,7 @@ TableSchema BuildTableSchema() { tb_schema.table_name = TABLE_NAME; tb_schema.dimension = TABLE_DIMENSION; tb_schema.index_file_size = TABLE_INDEX_FILE_SIZE; + tb_schema.metric_type = MetricType::L2; return tb_schema; } @@ -291,7 +292,6 @@ ClientTest::Test(const std::string& address, const std::string& port) { index.table_name = TABLE_NAME; index.index_type = IndexType::gpu_ivfflat; index.nlist = 16384; - index.metric_type = 1; Status stat = conn->CreateIndex(index); std::cout << "CreateIndex function call status: " << stat.ToString() << std::endl; diff --git a/cpp/src/sdk/grpc/ClientProxy.cpp b/cpp/src/sdk/grpc/ClientProxy.cpp index 113c0b63c8..2163e78c66 100644 --- a/cpp/src/sdk/grpc/ClientProxy.cpp +++ b/cpp/src/sdk/grpc/ClientProxy.cpp @@ -84,6 +84,7 @@ ClientProxy::CreateTable(const TableSchema ¶m) { schema.mutable_table_name()->set_table_name(param.table_name); schema.set_dimension(param.dimension); schema.set_index_file_size(param.index_file_size); + schema.set_metric_type((int32_t)param.metric_type); return client_ptr_->CreateTable(schema); } catch (std::exception &ex) { @@ -116,11 +117,9 @@ ClientProxy::CreateIndex(const IndexParam &index_param) { try { //TODO:add index params ::milvus::grpc::IndexParam grpc_index_param; - grpc_index_param.mutable_table_name()->set_table_name( - index_param.table_name); + grpc_index_param.mutable_table_name()->set_table_name(index_param.table_name); grpc_index_param.mutable_index()->set_index_type((int32_t)index_param.index_type); grpc_index_param.mutable_index()->set_nlist(index_param.nlist); - grpc_index_param.mutable_index()->set_metric_type(index_param.metric_type); return client_ptr_->CreateIndex(grpc_index_param); } catch (std::exception &ex) { @@ -273,6 +272,7 @@ ClientProxy::DescribeTable(const std::string &table_name, TableSchema &table_sch table_schema.table_name = grpc_schema.table_name().table_name(); table_schema.dimension = grpc_schema.dimension(); table_schema.index_file_size = grpc_schema.index_file_size(); + table_schema.metric_type = (MetricType)grpc_schema.metric_type(); return status; } catch (std::exception &ex) { @@ -378,7 +378,6 @@ ClientProxy::DescribeIndex(const std::string &table_name, IndexParam &index_para Status status = client_ptr_->DescribeIndex(grpc_table_name, grpc_index_param); index_param.index_type = (IndexType)(grpc_index_param.mutable_index()->index_type()); index_param.nlist = grpc_index_param.mutable_index()->nlist(); - index_param.metric_type = grpc_index_param.mutable_index()->metric_type(); return status; diff --git a/cpp/src/sdk/include/MilvusApi.h b/cpp/src/sdk/include/MilvusApi.h index 94b3d96cb6..01147d36fb 100644 --- a/cpp/src/sdk/include/MilvusApi.h +++ b/cpp/src/sdk/include/MilvusApi.h @@ -22,6 +22,11 @@ enum class IndexType { mix_nsg, }; +enum class MetricType { + L2 = 1, + IP = 2, +}; + /** * @brief Connect API parameter */ @@ -37,6 +42,7 @@ struct TableSchema { std::string table_name; ///< Table name int64_t dimension = 0; ///< Vector dimension, must be a positive value int64_t index_file_size = 0; ///< Index file size, must be a positive value + MetricType metric_type = MetricType::L2; ///< Index metric type }; /** @@ -77,7 +83,6 @@ struct IndexParam { std::string table_name; IndexType index_type; int32_t nlist; - int32_t metric_type; }; /** diff --git a/cpp/src/server/grpc_impl/GrpcRequestTask.cpp b/cpp/src/server/grpc_impl/GrpcRequestTask.cpp index 602270eb19..e0529c1999 100644 --- a/cpp/src/server/grpc_impl/GrpcRequestTask.cpp +++ b/cpp/src/server/grpc_impl/GrpcRequestTask.cpp @@ -144,11 +144,17 @@ CreateTableTask::OnExecute() { return SetError(res, "Invalid index file size: " + std::to_string(schema_->index_file_size())); } + res = ValidationUtil::ValidateTableIndexMetricType(schema_->metric_type()); + if(res != SERVER_SUCCESS) { + return SetError(res, "Invalid index metric type: " + std::to_string(schema_->metric_type())); + } + //step 2: construct table schema engine::meta::TableSchema table_info; table_info.table_id_ = schema_->table_name().table_name(); table_info.dimension_ = (uint16_t) schema_->dimension(); table_info.index_file_size_ = schema_->index_file_size(); + table_info.metric_type_ = schema_->metric_type(); //step 3: create table engine::Status stat = DBWrapper::DB()->CreateTable(table_info); @@ -203,6 +209,7 @@ DescribeTableTask::OnExecute() { schema_->mutable_table_name()->set_table_name(table_info.table_id_); schema_->set_dimension(table_info.dimension_); schema_->set_index_file_size(table_info.index_file_size_); + schema_->set_metric_type(table_info.metric_type_); } catch (std::exception &ex) { return SetError(SERVER_UNEXPECTED_ERROR, ex.what()); @@ -261,16 +268,10 @@ CreateIndexTask::OnExecute() { return SetError(res, "Invalid index nlist: " + std::to_string(grpc_index.nlist())); } - res = ValidationUtil::ValidateTableIndexMetricType(grpc_index.metric_type()); - if(res != SERVER_SUCCESS) { - return SetError(res, "Invalid index metric type: " + std::to_string(grpc_index.metric_type())); - } - //step 2: check table existence engine::TableIndex index; index.engine_type_ = grpc_index.index_type(); index.nlist_ = grpc_index.nlist(); - index.metric_type_ = grpc_index.metric_type(); stat = DBWrapper::DB()->CreateIndex(table_name_, index); if (!stat.ok()) { return SetError(SERVER_BUILD_INDEX_ERROR, stat.ToString()); @@ -922,7 +923,6 @@ DescribeIndexTask::OnExecute() { index_param_->mutable_table_name()->set_table_name(table_name_); index_param_->mutable_index()->set_index_type(index.engine_type_); index_param_->mutable_index()->set_nlist(index.nlist_); - index_param_->mutable_index()->set_metric_type(index.metric_type_); rc.ElapseFromBegin("totally cost"); } catch (std::exception &ex) { diff --git a/cpp/unittest/db/mem_test.cpp b/cpp/unittest/db/mem_test.cpp index 3974a15520..9344d2ccc3 100644 --- a/cpp/unittest/db/mem_test.cpp +++ b/cpp/unittest/db/mem_test.cpp @@ -216,7 +216,7 @@ TEST_F(MemManagerTest, MEM_TABLE_TEST) { ASSERT_TRUE(status.ok()); } -TEST_F(MemManagerTest, SERIAL_INSERT_SEARCH_TEST) { +TEST_F(MemManagerTest2, SERIAL_INSERT_SEARCH_TEST) { engine::meta::TableSchema table_info = BuildTableSchema(); engine::Status stat = db_->CreateTable(table_info); @@ -262,7 +262,7 @@ TEST_F(MemManagerTest, SERIAL_INSERT_SEARCH_TEST) { } } -TEST_F(MemManagerTest, INSERT_TEST) { +TEST_F(MemManagerTest2, INSERT_TEST) { engine::meta::TableSchema table_info = BuildTableSchema(); engine::Status stat = db_->CreateTable(table_info); @@ -288,7 +288,7 @@ TEST_F(MemManagerTest, INSERT_TEST) { LOG(DEBUG) << "total_time spent in INSERT_TEST (ms) : " << total_time; } -TEST_F(MemManagerTest, CONCURRENT_INSERT_SEARCH_TEST) { +TEST_F(MemManagerTest2, CONCURRENT_INSERT_SEARCH_TEST) { engine::meta::TableSchema table_info = BuildTableSchema(); engine::Status stat = db_->CreateTable(table_info); @@ -359,7 +359,7 @@ TEST_F(MemManagerTest, CONCURRENT_INSERT_SEARCH_TEST) { search.join(); }; -TEST_F(MemManagerTest, VECTOR_IDS_TEST) { +TEST_F(MemManagerTest2, VECTOR_IDS_TEST) { engine::meta::TableSchema table_info = BuildTableSchema(); engine::Status stat = db_->CreateTable(table_info); diff --git a/cpp/unittest/db/utils.cpp b/cpp/unittest/db/utils.cpp index 724ea1c1d9..6a13ba526a 100644 --- a/cpp/unittest/db/utils.cpp +++ b/cpp/unittest/db/utils.cpp @@ -51,6 +51,10 @@ void BaseTest::InitLog() { el::Loggers::reconfigureLogger("default", defaultConf); } +void BaseTest::SetUp() { + InitLog(); +} + engine::Options BaseTest::GetOptions() { auto options = engine::OptionsFactory::Build(); options.meta.path = "/tmp/milvus_test"; @@ -60,7 +64,7 @@ engine::Options BaseTest::GetOptions() { ///////////////////////////////////////////////////////////////////////////////////////////////////////////////// void DBTest::SetUp() { - InitLog(); + BaseTest::SetUp(); server::ConfigNode& config = server::ServerConfig::GetInstance().GetConfig(server::CONFIG_CACHE); config.AddSequenceItem(server::CONFIG_GPU_IDS, "0"); @@ -104,7 +108,8 @@ engine::Options DBTest2::GetOptions() { ///////////////////////////////////////////////////////////////////////////////////////////////////////////////// void MetaTest::SetUp() { - InitLog(); + BaseTest::SetUp(); + impl_ = engine::DBMetaImplFactory::Build(); } @@ -127,7 +132,7 @@ engine::Options MySqlDBTest::GetOptions() { ///////////////////////////////////////////////////////////////////////////////////////////////////////////////// void MySqlMetaTest::SetUp() { - InitLog(); + BaseTest::SetUp(); engine::DBMetaOptions options = GetOptions().meta; int mode = engine::Options::MODE::SINGLE; diff --git a/cpp/unittest/db/utils.h b/cpp/unittest/db/utils.h index 09fa4eb122..15c8a3fa54 100644 --- a/cpp/unittest/db/utils.h +++ b/cpp/unittest/db/utils.h @@ -37,6 +37,8 @@ void ASSERT_STATS(zilliz::milvus::engine::Status &stat); class BaseTest : public ::testing::Test { protected: void InitLog(); + + virtual void SetUp() override; virtual zilliz::milvus::engine::Options GetOptions(); }; @@ -82,5 +84,9 @@ class MySqlMetaTest : public BaseTest { ///////////////////////////////////////////////////////////////////////////////////////////////////////////////// -class MemManagerTest : public DBTest { +class MemManagerTest : public BaseTest { +}; + +///////////////////////////////////////////////////////////////////////////////////////////////////////////////// +class MemManagerTest2 : public DBTest { };