milvus-io
/
milvus
mirror of https://github.com/milvus-io/milvus.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'branch-0.4.0' into 'branch-0.4.0' 

MS-487 Define metric type in CreateTable

See merge request megasearch/milvus!489

Former-commit-id: 79e50ffd1ba6cb43e9dbbbe0f68cbcdf81eb2ab9
pull/191/head
peng.xu 2019-09-05 20:34:44 +08:00
parent 6963ff5f40 fd84fdcf66
commit 29474af273
14 changed files with 162 additions and 143 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
cpp/CHANGELOG.md
View File

@ -85,6 +85,7 @@ Please mark all change in change log and use the ticket from JIRA.
- MS-460 - Put transport speed as weight when choosing neighbour to execute task
- MS-459 - Add cache for pick function in tasktable
- MS-482 - Change search stream transport to unary in grpc
- MS-487 - Define metric type in CreateTable
- MS-488 - Improve code format in scheduler
## New Feature

4
cpp/src/db/DBImpl.cpp
View File

@ -250,7 +250,9 @@ Status DBImpl::CreateIndex(const std::string& table_id, const TableIndex& index)
}
//step 2: update index info
if(!utils::IsSameIndex(old_index, index)) {
TableIndex new_index = index;
new_index.metric_type_ = old_index.metric_type_;//dont change metric type, it was defined by CreateTable
if(!utils::IsSameIndex(old_index, new_index)) {
DropIndex(table_id);
status = meta_ptr_->UpdateTableIndexParam(table_id, index);

3
cpp/src/db/engine/ExecutionEngineImpl.cpp
View File

@ -301,7 +301,8 @@ Status ExecutionEngineImpl::Search(long n,
}
ENGINE_LOG_DEBUG << "Search Params: [k] " << k << " [nprobe] " << nprobe;
auto ec = index_->Search(n, data, distances, labels, Config::object{{"k", k}, {"nprobe", nprobe}});
auto cfg = Config::object{{"k", k}, {"nprobe", nprobe}};
auto ec = index_->Search(n, data, distances, labels, cfg);
if (ec != server::KNOWHERE_SUCCESS) {
ENGINE_LOG_ERROR << "Search error";
return Status::Error("Search: Search Error");

2
cpp/src/db/meta/SqliteMetaImpl.cpp
View File

@ -707,7 +707,7 @@ Status SqliteMetaImpl::FilesToSearch(const std::string &table_id,
files[table_file.date_].push_back(table_file);
}
if(files.empty()) {
std::cout << "ERROR" << std::endl;
ENGINE_LOG_ERROR << "No file to search for table: " << table_id;
}
} catch (std::exception &e) {
return HandleException("Encounter exception when iterate index files", e);

194
cpp/src/grpc/gen-milvus/milvus.pb.cc
View File

@ -387,6 +387,7 @@ const ::PROTOBUF_NAMESPACE_ID::uint32 TableStruct_milvus_2eproto::offsets[] PROT
PROTOBUF_FIELD_OFFSET(::milvus::grpc::TableSchema, table_name_),
PROTOBUF_FIELD_OFFSET(::milvus::grpc::TableSchema, dimension_),
PROTOBUF_FIELD_OFFSET(::milvus::grpc::TableSchema, index_file_size_),
PROTOBUF_FIELD_OFFSET(::milvus::grpc::TableSchema, metric_type_),
~0u, // no _has_bits_
PROTOBUF_FIELD_OFFSET(::milvus::grpc::Range, _internal_metadata_),
~0u, // no _extensions_
@ -486,7 +487,6 @@ const ::PROTOBUF_NAMESPACE_ID::uint32 TableStruct_milvus_2eproto::offsets[] PROT
~0u, // no _weak_field_map_
PROTOBUF_FIELD_OFFSET(::milvus::grpc::Index, index_type_),
PROTOBUF_FIELD_OFFSET(::milvus::grpc::Index, nlist_),
PROTOBUF_FIELD_OFFSET(::milvus::grpc::Index, metric_type_),
~0u, // no _has_bits_
PROTOBUF_FIELD_OFFSET(::milvus::grpc::IndexParam, _internal_metadata_),
~0u, // no _extensions_
@ -505,20 +505,20 @@ const ::PROTOBUF_NAMESPACE_ID::uint32 TableStruct_milvus_2eproto::offsets[] PROT
static const ::PROTOBUF_NAMESPACE_ID::internal::MigrationSchema schemas[] PROTOBUF_SECTION_VARIABLE(protodesc_cold) = {
{ 0, -1, sizeof(::milvus::grpc::TableName)},
{ 7, -1, sizeof(::milvus::grpc::TableSchema)},
{ 15, -1, sizeof(::milvus::grpc::Range)},
{ 22, -1, sizeof(::milvus::grpc::RowRecord)},
{ 28, -1, sizeof(::milvus::grpc::InsertParam)},
{ 36, -1, sizeof(::milvus::grpc::VectorIds)},
{ 43, -1, sizeof(::milvus::grpc::SearchParam)},
{ 53, -1, sizeof(::milvus::grpc::SearchInFilesParam)},
{ 60, -1, sizeof(::milvus::grpc::QueryResult)},
{ 67, -1, sizeof(::milvus::grpc::TopKQueryResult)},
{ 73, -1, sizeof(::milvus::grpc::TopKQueryResultList)},
{ 80, -1, sizeof(::milvus::grpc::StringReply)},
{ 87, -1, sizeof(::milvus::grpc::BoolReply)},
{ 94, -1, sizeof(::milvus::grpc::TableRowCount)},
{ 101, -1, sizeof(::milvus::grpc::Command)},
{ 107, -1, sizeof(::milvus::grpc::Index)},
{ 16, -1, sizeof(::milvus::grpc::Range)},
{ 23, -1, sizeof(::milvus::grpc::RowRecord)},
{ 29, -1, sizeof(::milvus::grpc::InsertParam)},
{ 37, -1, sizeof(::milvus::grpc::VectorIds)},
{ 44, -1, sizeof(::milvus::grpc::SearchParam)},
{ 54, -1, sizeof(::milvus::grpc::SearchInFilesParam)},
{ 61, -1, sizeof(::milvus::grpc::QueryResult)},
{ 68, -1, sizeof(::milvus::grpc::TopKQueryResult)},
{ 74, -1, sizeof(::milvus::grpc::TopKQueryResultList)},
{ 81, -1, sizeof(::milvus::grpc::StringReply)},
{ 88, -1, sizeof(::milvus::grpc::BoolReply)},
{ 95, -1, sizeof(::milvus::grpc::TableRowCount)},
{ 102, -1, sizeof(::milvus::grpc::Command)},
{ 108, -1, sizeof(::milvus::grpc::Index)},
{ 115, -1, sizeof(::milvus::grpc::IndexParam)},
{ 122, -1, sizeof(::milvus::grpc::DeleteByRangeParam)},
};
@ -547,37 +547,37 @@ static ::PROTOBUF_NAMESPACE_ID::Message const * const file_default_instances[] =
const char descriptor_table_protodef_milvus_2eproto[] PROTOBUF_SECTION_VARIABLE(protodesc_cold) =
"\n\014milvus.proto\022\013milvus.grpc\032\014status.prot"
"o\"D\n\tTableName\022#\n\006status\030\001 \001(\0132\023.milvus."
"grpc.Status\022\022\n\ntable_name\030\002 \001(\t\"e\n\013Table"
"grpc.Status\022\022\n\ntable_name\030\002 \001(\t\"z\n\013Table"
"Schema\022*\n\ntable_name\030\001 \001(\0132\026.milvus.grpc"
".TableName\022\021\n\tdimension\030\002 \001(\003\022\027\n\017index_f"
"ile_size\030\003 \001(\003\"/\n\005Range\022\023\n\013start_value\030\001"
" \001(\t\022\021\n\tend_value\030\002 \001(\t\" \n\tRowRecord\022\023\n\013"
"vector_data\030\001 \003(\002\"i\n\013InsertParam\022\022\n\ntabl"
"e_name\030\001 \001(\t\0220\n\020row_record_array\030\002 \003(\0132\026"
".milvus.grpc.RowRecord\022\024\n\014row_id_array\030\003"
" \003(\003\"I\n\tVectorIds\022#\n\006status\030\001 \001(\0132\023.milv"
"us.grpc.Status\022\027\n\017vector_id_array\030\002 \003(\003\""
"\242\001\n\013SearchParam\022\022\n\ntable_name\030\001 \001(\t\0222\n\022q"
"uery_record_array\030\002 \003(\0132\026.milvus.grpc.Ro"
"wRecord\022-\n\021query_range_array\030\003 \003(\0132\022.mil"
"vus.grpc.Range\022\014\n\004topk\030\004 \001(\003\022\016\n\006nprobe\030\005"
" \001(\003\"[\n\022SearchInFilesParam\022\025\n\rfile_id_ar"
"ray\030\001 \003(\t\022.\n\014search_param\030\002 \001(\0132\030.milvus"
".grpc.SearchParam\"+\n\013QueryResult\022\n\n\002id\030\001"
" \001(\003\022\020\n\010distance\030\002 \001(\001\"H\n\017TopKQueryResul"
"t\0225\n\023query_result_arrays\030\001 \003(\0132\030.milvus."
"grpc.QueryResult\"s\n\023TopKQueryResultList\022"
"#\n\006status\030\001 \001(\0132\023.milvus.grpc.Status\0227\n\021"
"topk_query_result\030\002 \003(\0132\034.milvus.grpc.To"
"pKQueryResult\"H\n\013StringReply\022#\n\006status\030\001"
" \001(\0132\023.milvus.grpc.Status\022\024\n\014string_repl"
"y\030\002 \001(\t\"D\n\tBoolReply\022#\n\006status\030\001 \001(\0132\023.m"
"ilvus.grpc.Status\022\022\n\nbool_reply\030\002 \001(\010\"M\n"
"\rTableRowCount\022#\n\006status\030\001 \001(\0132\023.milvus."
"grpc.Status\022\027\n\017table_row_count\030\002 \001(\003\"\026\n\007"
"Command\022\013\n\003cmd\030\001 \001(\t\"\?\n\005Index\022\022\n\nindex_t"
"ype\030\001 \001(\005\022\r\n\005nlist\030\002 \001(\005\022\023\n\013metric_type\030"
"\003 \001(\005\"[\n\nIndexParam\022*\n\ntable_name\030\001 \001(\0132"
"ile_size\030\003 \001(\003\022\023\n\013metric_type\030\004 \001(\005\"/\n\005R"
"ange\022\023\n\013start_value\030\001 \001(\t\022\021\n\tend_value\030\002"
" \001(\t\" \n\tRowRecord\022\023\n\013vector_data\030\001 \003(\002\"i"
"\n\013InsertParam\022\022\n\ntable_name\030\001 \001(\t\0220\n\020row"
"_record_array\030\002 \003(\0132\026.milvus.grpc.RowRec"
"ord\022\024\n\014row_id_array\030\003 \003(\003\"I\n\tVectorIds\022#"
"\n\006status\030\001 \001(\0132\023.milvus.grpc.Status\022\027\n\017v"
"ector_id_array\030\002 \003(\003\"\242\001\n\013SearchParam\022\022\n\n"
"table_name\030\001 \001(\t\0222\n\022query_record_array\030\002"
" \003(\0132\026.milvus.grpc.RowRecord\022-\n\021query_ra"
"nge_array\030\003 \003(\0132\022.milvus.grpc.Range\022\014\n\004t"
"opk\030\004 \001(\003\022\016\n\006nprobe\030\005 \001(\003\"[\n\022SearchInFil"
"esParam\022\025\n\rfile_id_array\030\001 \003(\t\022.\n\014search"
"_param\030\002 \001(\0132\030.milvus.grpc.SearchParam\"+"
"\n\013QueryResult\022\n\n\002id\030\001 \001(\003\022\020\n\010distance\030\002 "
"\001(\001\"H\n\017TopKQueryResult\0225\n\023query_result_a"
"rrays\030\001 \003(\0132\030.milvus.grpc.QueryResult\"s\n"
"\023TopKQueryResultList\022#\n\006status\030\001 \001(\0132\023.m"
"ilvus.grpc.Status\0227\n\021topk_query_result\030\002"
" \003(\0132\034.milvus.grpc.TopKQueryResult\"H\n\013St"
"ringReply\022#\n\006status\030\001 \001(\0132\023.milvus.grpc."
"Status\022\024\n\014string_reply\030\002 \001(\t\"D\n\tBoolRepl"
"y\022#\n\006status\030\001 \001(\0132\023.milvus.grpc.Status\022\022"
"\n\nbool_reply\030\002 \001(\010\"M\n\rTableRowCount\022#\n\006s"
"tatus\030\001 \001(\0132\023.milvus.grpc.Status\022\027\n\017tabl"
"e_row_count\030\002 \001(\003\"\026\n\007Command\022\013\n\003cmd\030\001 \001("
"\t\"*\n\005Index\022\022\n\nindex_type\030\001 \001(\005\022\r\n\005nlist\030"
"\002 \001(\005\"[\n\nIndexParam\022*\n\ntable_name\030\001 \001(\0132"
"\026.milvus.grpc.TableName\022!\n\005index\030\002 \001(\0132\022"
".milvus.grpc.Index\"K\n\022DeleteByRangeParam"
"\022!\n\005range\030\001 \001(\0132\022.milvus.grpc.Range\022\022\n\nt"
@ -1010,16 +1010,16 @@ TableSchema::TableSchema(const TableSchema& from)
table_name_ = nullptr;
}
::memcpy(&dimension_, &from.dimension_,
static_cast<size_t>(reinterpret_cast<char*>(&index_file_size_) -
reinterpret_cast<char*>(&dimension_)) + sizeof(index_file_size_));
static_cast<size_t>(reinterpret_cast<char*>(&metric_type_) -
reinterpret_cast<char*>(&dimension_)) + sizeof(metric_type_));
// @@protoc_insertion_point(copy_constructor:milvus.grpc.TableSchema)
}
void TableSchema::SharedCtor() {
::PROTOBUF_NAMESPACE_ID::internal::InitSCC(&scc_info_TableSchema_milvus_2eproto.base);
::memset(&table_name_, 0, static_cast<size_t>(
reinterpret_cast<char*>(&index_file_size_) -
reinterpret_cast<char*>(&table_name_)) + sizeof(index_file_size_));
reinterpret_cast<char*>(&metric_type_) -
reinterpret_cast<char*>(&table_name_)) + sizeof(metric_type_));
}
TableSchema::~TableSchema() {
@ -1051,8 +1051,8 @@ void TableSchema::Clear() {
}
table_name_ = nullptr;
::memset(&dimension_, 0, static_cast<size_t>(
reinterpret_cast<char*>(&index_file_size_) -
reinterpret_cast<char*>(&dimension_)) + sizeof(index_file_size_));
reinterpret_cast<char*>(&metric_type_) -
reinterpret_cast<char*>(&dimension_)) + sizeof(metric_type_));
_internal_metadata_.Clear();
}
@ -1085,6 +1085,13 @@ const char* TableSchema::_InternalParse(const char* ptr, ::PROTOBUF_NAMESPACE_ID
CHK_(ptr);
} else goto handle_unusual;
continue;
// int32 metric_type = 4;
case 4:
if (PROTOBUF_PREDICT_TRUE(static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 32)) {
metric_type_ = ::PROTOBUF_NAMESPACE_ID::internal::ReadVarint(&ptr);
CHK_(ptr);
} else goto handle_unusual;
continue;
default: {
handle_unusual:
if ((tag & 7) == 4 || tag == 0) {
@ -1152,6 +1159,19 @@ bool TableSchema::MergePartialFromCodedStream(
break;
}
// int32 metric_type = 4;
case 4: {
if (static_cast< ::PROTOBUF_NAMESPACE_ID::uint8>(tag) == (32 & 0xFF)) {
DO_((::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::ReadPrimitive<
::PROTOBUF_NAMESPACE_ID::int32, ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::TYPE_INT32>(
input, &metric_type_)));
} else {
goto handle_unusual;
}
break;
}
default: {
handle_unusual:
if (tag == 0) {
@ -1195,6 +1215,11 @@ void TableSchema::SerializeWithCachedSizes(
::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteInt64(3, this->index_file_size(), output);
}
// int32 metric_type = 4;
if (this->metric_type() != 0) {
::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteInt32(4, this->metric_type(), output);
}
if (_internal_metadata_.have_unknown_fields()) {
::PROTOBUF_NAMESPACE_ID::internal::WireFormat::SerializeUnknownFields(
_internal_metadata_.unknown_fields(), output);
@ -1225,6 +1250,11 @@ void TableSchema::SerializeWithCachedSizes(
target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteInt64ToArray(3, this->index_file_size(), target);
}
// int32 metric_type = 4;
if (this->metric_type() != 0) {
target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteInt32ToArray(4, this->metric_type(), target);
}
if (_internal_metadata_.have_unknown_fields()) {
target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormat::SerializeUnknownFieldsToArray(
_internal_metadata_.unknown_fields(), target);
@ -1267,6 +1297,13 @@ size_t TableSchema::ByteSizeLong() const {
this->index_file_size());
}
// int32 metric_type = 4;
if (this->metric_type() != 0) {
total_size += 1 +
::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::Int32Size(
this->metric_type());
}
int cached_size = ::PROTOBUF_NAMESPACE_ID::internal::ToCachedSize(total_size);
SetCachedSize(cached_size);
return total_size;
@ -1303,6 +1340,9 @@ void TableSchema::MergeFrom(const TableSchema& from) {
if (from.index_file_size() != 0) {
set_index_file_size(from.index_file_size());
}
if (from.metric_type() != 0) {
set_metric_type(from.metric_type());
}
}
void TableSchema::CopyFrom(const ::PROTOBUF_NAMESPACE_ID::Message& from) {
@ -1329,6 +1369,7 @@ void TableSchema::InternalSwap(TableSchema* other) {
swap(table_name_, other->table_name_);
swap(dimension_, other->dimension_);
swap(index_file_size_, other->index_file_size_);
swap(metric_type_, other->metric_type_);
}
::PROTOBUF_NAMESPACE_ID::Metadata TableSchema::GetMetadata() const {
@ -5606,15 +5647,15 @@ Index::Index(const Index& from)
_internal_metadata_(nullptr) {
_internal_metadata_.MergeFrom(from._internal_metadata_);
::memcpy(&index_type_, &from.index_type_,
static_cast<size_t>(reinterpret_cast<char*>(&metric_type_) -
reinterpret_cast<char*>(&index_type_)) + sizeof(metric_type_));
static_cast<size_t>(reinterpret_cast<char*>(&nlist_) -
reinterpret_cast<char*>(&index_type_)) + sizeof(nlist_));
// @@protoc_insertion_point(copy_constructor:milvus.grpc.Index)
}
void Index::SharedCtor() {
::memset(&index_type_, 0, static_cast<size_t>(
reinterpret_cast<char*>(&metric_type_) -
reinterpret_cast<char*>(&index_type_)) + sizeof(metric_type_));
reinterpret_cast<char*>(&nlist_) -
reinterpret_cast<char*>(&index_type_)) + sizeof(nlist_));
}
Index::~Index() {
@ -5641,8 +5682,8 @@ void Index::Clear() {
(void) cached_has_bits;
::memset(&index_type_, 0, static_cast<size_t>(
reinterpret_cast<char*>(&metric_type_) -
reinterpret_cast<char*>(&index_type_)) + sizeof(metric_type_));
reinterpret_cast<char*>(&nlist_) -
reinterpret_cast<char*>(&index_type_)) + sizeof(nlist_));
_internal_metadata_.Clear();
}
@ -5668,13 +5709,6 @@ const char* Index::_InternalParse(const char* ptr, ::PROTOBUF_NAMESPACE_ID::inte
CHK_(ptr);
} else goto handle_unusual;
continue;
// int32 metric_type = 3;
case 3:
if (PROTOBUF_PREDICT_TRUE(static_cast<::PROTOBUF_NAMESPACE_ID::uint8>(tag) == 24)) {
metric_type_ = ::PROTOBUF_NAMESPACE_ID::internal::ReadVarint(&ptr);
CHK_(ptr);
} else goto handle_unusual;
continue;
default: {
handle_unusual:
if ((tag & 7) == 4 || tag == 0) {
@ -5731,19 +5765,6 @@ bool Index::MergePartialFromCodedStream(
break;
}
// int32 metric_type = 3;
case 3: {
if (static_cast< ::PROTOBUF_NAMESPACE_ID::uint8>(tag) == (24 & 0xFF)) {
DO_((::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::ReadPrimitive<
::PROTOBUF_NAMESPACE_ID::int32, ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::TYPE_INT32>(
input, &metric_type_)));
} else {
goto handle_unusual;
}
break;
}
default: {
handle_unusual:
if (tag == 0) {
@ -5781,11 +5802,6 @@ void Index::SerializeWithCachedSizes(
::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteInt32(2, this->nlist(), output);
}
// int32 metric_type = 3;
if (this->metric_type() != 0) {
::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteInt32(3, this->metric_type(), output);
}
if (_internal_metadata_.have_unknown_fields()) {
::PROTOBUF_NAMESPACE_ID::internal::WireFormat::SerializeUnknownFields(
_internal_metadata_.unknown_fields(), output);
@ -5809,11 +5825,6 @@ void Index::SerializeWithCachedSizes(
target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteInt32ToArray(2, this->nlist(), target);
}
// int32 metric_type = 3;
if (this->metric_type() != 0) {
target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::WriteInt32ToArray(3, this->metric_type(), target);
}
if (_internal_metadata_.have_unknown_fields()) {
target = ::PROTOBUF_NAMESPACE_ID::internal::WireFormat::SerializeUnknownFieldsToArray(
_internal_metadata_.unknown_fields(), target);
@ -5849,13 +5860,6 @@ size_t Index::ByteSizeLong() const {
this->nlist());
}
// int32 metric_type = 3;
if (this->metric_type() != 0) {
total_size += 1 +
::PROTOBUF_NAMESPACE_ID::internal::WireFormatLite::Int32Size(
this->metric_type());
}
int cached_size = ::PROTOBUF_NAMESPACE_ID::internal::ToCachedSize(total_size);
SetCachedSize(cached_size);
return total_size;
@ -5889,9 +5893,6 @@ void Index::MergeFrom(const Index& from) {
if (from.nlist() != 0) {
set_nlist(from.nlist());
}
if (from.metric_type() != 0) {
set_metric_type(from.metric_type());
}
}
void Index::CopyFrom(const ::PROTOBUF_NAMESPACE_ID::Message& from) {
@ -5917,7 +5918,6 @@ void Index::InternalSwap(Index* other) {
_internal_metadata_.Swap(&other->_internal_metadata_);
swap(index_type_, other->index_type_);
swap(nlist_, other->nlist_);
swap(metric_type_, other->metric_type_);
}
::PROTOBUF_NAMESPACE_ID::Metadata Index::GetMetadata() const {

42
cpp/src/grpc/gen-milvus/milvus.pb.h
View File

@ -401,6 +401,7 @@ class TableSchema :
kTableNameFieldNumber = 1,
kDimensionFieldNumber = 2,
kIndexFileSizeFieldNumber = 3,
kMetricTypeFieldNumber = 4,
};
// .milvus.grpc.TableName table_name = 1;
bool has_table_name() const;
@ -420,6 +421,11 @@ class TableSchema :
::PROTOBUF_NAMESPACE_ID::int64 index_file_size() const;
void set_index_file_size(::PROTOBUF_NAMESPACE_ID::int64 value);
// int32 metric_type = 4;
void clear_metric_type();
::PROTOBUF_NAMESPACE_ID::int32 metric_type() const;
void set_metric_type(::PROTOBUF_NAMESPACE_ID::int32 value);
// @@protoc_insertion_point(class_scope:milvus.grpc.TableSchema)
private:
class _Internal;
@ -428,6 +434,7 @@ class TableSchema :
::milvus::grpc::TableName* table_name_;
::PROTOBUF_NAMESPACE_ID::int64 dimension_;
::PROTOBUF_NAMESPACE_ID::int64 index_file_size_;
::PROTOBUF_NAMESPACE_ID::int32 metric_type_;
mutable ::PROTOBUF_NAMESPACE_ID::internal::CachedSize _cached_size_;
friend struct ::TableStruct_milvus_2eproto;
};
@ -2466,7 +2473,6 @@ class Index :
enum : int {
kIndexTypeFieldNumber = 1,
kNlistFieldNumber = 2,
kMetricTypeFieldNumber = 3,
};
// int32 index_type = 1;
void clear_index_type();
@ -2478,11 +2484,6 @@ class Index :
::PROTOBUF_NAMESPACE_ID::int32 nlist() const;
void set_nlist(::PROTOBUF_NAMESPACE_ID::int32 value);
// int32 metric_type = 3;
void clear_metric_type();
::PROTOBUF_NAMESPACE_ID::int32 metric_type() const;
void set_metric_type(::PROTOBUF_NAMESPACE_ID::int32 value);
// @@protoc_insertion_point(class_scope:milvus.grpc.Index)
private:
class _Internal;
@ -2490,7 +2491,6 @@ class Index :
::PROTOBUF_NAMESPACE_ID::internal::InternalMetadataWithArena _internal_metadata_;
::PROTOBUF_NAMESPACE_ID::int32 index_type_;
::PROTOBUF_NAMESPACE_ID::int32 nlist_;
::PROTOBUF_NAMESPACE_ID::int32 metric_type_;
mutable ::PROTOBUF_NAMESPACE_ID::internal::CachedSize _cached_size_;
friend struct ::TableStruct_milvus_2eproto;
};
@ -2975,6 +2975,20 @@ inline void TableSchema::set_index_file_size(::PROTOBUF_NAMESPACE_ID::int64 valu
// @@protoc_insertion_point(field_set:milvus.grpc.TableSchema.index_file_size)
}
// int32 metric_type = 4;
inline void TableSchema::clear_metric_type() {
metric_type_ = 0;
}
inline ::PROTOBUF_NAMESPACE_ID::int32 TableSchema::metric_type() const {
// @@protoc_insertion_point(field_get:milvus.grpc.TableSchema.metric_type)
return metric_type_;
}
inline void TableSchema::set_metric_type(::PROTOBUF_NAMESPACE_ID::int32 value) {
metric_type_ = value;
// @@protoc_insertion_point(field_set:milvus.grpc.TableSchema.metric_type)
}
// -------------------------------------------------------------------
// Range
@ -4030,20 +4044,6 @@ inline void Index::set_nlist(::PROTOBUF_NAMESPACE_ID::int32 value) {
// @@protoc_insertion_point(field_set:milvus.grpc.Index.nlist)
}
// int32 metric_type = 3;
inline void Index::clear_metric_type() {
metric_type_ = 0;
}
inline ::PROTOBUF_NAMESPACE_ID::int32 Index::metric_type() const {
// @@protoc_insertion_point(field_get:milvus.grpc.Index.metric_type)
return metric_type_;
}
inline void Index::set_metric_type(::PROTOBUF_NAMESPACE_ID::int32 value) {
metric_type_ = value;
// @@protoc_insertion_point(field_set:milvus.grpc.Index.metric_type)
}
// -------------------------------------------------------------------
// IndexParam

2
cpp/src/grpc/milvus.proto
View File

@ -19,6 +19,7 @@ message TableSchema {
TableName table_name = 1;
int64 dimension = 2;
int64 index_file_size = 3;
int32 metric_type = 4;
}
/**
@ -134,7 +135,6 @@ message Command {
message Index {
int32 index_type = 1;
int32 nlist = 2;
int32 metric_type = 3;
}
/**

2
cpp/src/sdk/examples/grpcsimple/src/ClientTest.cpp
View File

@ -96,6 +96,7 @@ TableSchema BuildTableSchema() {
tb_schema.table_name = TABLE_NAME;
tb_schema.dimension = TABLE_DIMENSION;
tb_schema.index_file_size = TABLE_INDEX_FILE_SIZE;
tb_schema.metric_type = MetricType::L2;
return tb_schema;
}
@ -291,7 +292,6 @@ ClientTest::Test(const std::string& address, const std::string& port) {
index.table_name = TABLE_NAME;
index.index_type = IndexType::gpu_ivfflat;
index.nlist = 16384;
index.metric_type = 1;
Status stat = conn->CreateIndex(index);
std::cout << "CreateIndex function call status: " << stat.ToString() << std::endl;

7
cpp/src/sdk/grpc/ClientProxy.cpp
View File

@ -84,6 +84,7 @@ ClientProxy::CreateTable(const TableSchema &param) {
schema.mutable_table_name()->set_table_name(param.table_name);
schema.set_dimension(param.dimension);
schema.set_index_file_size(param.index_file_size);
schema.set_metric_type((int32_t)param.metric_type);
return client_ptr_->CreateTable(schema);
} catch (std::exception &ex) {
@ -116,11 +117,9 @@ ClientProxy::CreateIndex(const IndexParam &index_param) {
try {
//TODO:add index params
::milvus::grpc::IndexParam grpc_index_param;
grpc_index_param.mutable_table_name()->set_table_name(
index_param.table_name);
grpc_index_param.mutable_table_name()->set_table_name(index_param.table_name);
grpc_index_param.mutable_index()->set_index_type((int32_t)index_param.index_type);
grpc_index_param.mutable_index()->set_nlist(index_param.nlist);
grpc_index_param.mutable_index()->set_metric_type(index_param.metric_type);
return client_ptr_->CreateIndex(grpc_index_param);
} catch (std::exception &ex) {
@ -273,6 +272,7 @@ ClientProxy::DescribeTable(const std::string &table_name, TableSchema &table_sch
table_schema.table_name = grpc_schema.table_name().table_name();
table_schema.dimension = grpc_schema.dimension();
table_schema.index_file_size = grpc_schema.index_file_size();
table_schema.metric_type = (MetricType)grpc_schema.metric_type();
return status;
} catch (std::exception &ex) {
@ -378,7 +378,6 @@ ClientProxy::DescribeIndex(const std::string &table_name, IndexParam &index_para
Status status = client_ptr_->DescribeIndex(grpc_table_name, grpc_index_param);
index_param.index_type = (IndexType)(grpc_index_param.mutable_index()->index_type());
index_param.nlist = grpc_index_param.mutable_index()->nlist();
index_param.metric_type = grpc_index_param.mutable_index()->metric_type();
return status;

7
cpp/src/sdk/include/MilvusApi.h
View File

@ -22,6 +22,11 @@ enum class IndexType {
mix_nsg,
};
enum class MetricType {
L2 = 1,
IP = 2,
};
/**
* @brief Connect API parameter
*/
@ -37,6 +42,7 @@ struct TableSchema {
std::string table_name; ///< Table name
int64_t dimension = 0; ///< Vector dimension, must be a positive value
int64_t index_file_size = 0; ///< Index file size, must be a positive value
MetricType metric_type = MetricType::L2; ///< Index metric type
};
/**
@ -77,7 +83,6 @@ struct IndexParam {
std::string table_name;
IndexType index_type;
int32_t nlist;
int32_t metric_type;
};
/**

14
cpp/src/server/grpc_impl/GrpcRequestTask.cpp
View File

@ -144,11 +144,17 @@ CreateTableTask::OnExecute() {
return SetError(res, "Invalid index file size: " + std::to_string(schema_->index_file_size()));
}
res = ValidationUtil::ValidateTableIndexMetricType(schema_->metric_type());
if(res != SERVER_SUCCESS) {
return SetError(res, "Invalid index metric type: " + std::to_string(schema_->metric_type()));
}
//step 2: construct table schema
engine::meta::TableSchema table_info;
table_info.table_id_ = schema_->table_name().table_name();
table_info.dimension_ = (uint16_t) schema_->dimension();
table_info.index_file_size_ = schema_->index_file_size();
table_info.metric_type_ = schema_->metric_type();
//step 3: create table
engine::Status stat = DBWrapper::DB()->CreateTable(table_info);
@ -203,6 +209,7 @@ DescribeTableTask::OnExecute() {
schema_->mutable_table_name()->set_table_name(table_info.table_id_);
schema_->set_dimension(table_info.dimension_);
schema_->set_index_file_size(table_info.index_file_size_);
schema_->set_metric_type(table_info.metric_type_);
} catch (std::exception &ex) {
return SetError(SERVER_UNEXPECTED_ERROR, ex.what());
@ -261,16 +268,10 @@ CreateIndexTask::OnExecute() {
return SetError(res, "Invalid index nlist: " + std::to_string(grpc_index.nlist()));
}
res = ValidationUtil::ValidateTableIndexMetricType(grpc_index.metric_type());
if(res != SERVER_SUCCESS) {
return SetError(res, "Invalid index metric type: " + std::to_string(grpc_index.metric_type()));
}
//step 2: check table existence
engine::TableIndex index;
index.engine_type_ = grpc_index.index_type();
index.nlist_ = grpc_index.nlist();
index.metric_type_ = grpc_index.metric_type();
stat = DBWrapper::DB()->CreateIndex(table_name_, index);
if (!stat.ok()) {
return SetError(SERVER_BUILD_INDEX_ERROR, stat.ToString());
@ -922,7 +923,6 @@ DescribeIndexTask::OnExecute() {
index_param_->mutable_table_name()->set_table_name(table_name_);
index_param_->mutable_index()->set_index_type(index.engine_type_);
index_param_->mutable_index()->set_nlist(index.nlist_);
index_param_->mutable_index()->set_metric_type(index.metric_type_);
rc.ElapseFromBegin("totally cost");
} catch (std::exception &ex) {

8
cpp/unittest/db/mem_test.cpp
View File

@ -216,7 +216,7 @@ TEST_F(MemManagerTest, MEM_TABLE_TEST) {
ASSERT_TRUE(status.ok());
}
TEST_F(MemManagerTest, SERIAL_INSERT_SEARCH_TEST) {
TEST_F(MemManagerTest2, SERIAL_INSERT_SEARCH_TEST) {
engine::meta::TableSchema table_info = BuildTableSchema();
engine::Status stat = db_->CreateTable(table_info);
@ -262,7 +262,7 @@ TEST_F(MemManagerTest, SERIAL_INSERT_SEARCH_TEST) {
}
}
TEST_F(MemManagerTest, INSERT_TEST) {
TEST_F(MemManagerTest2, INSERT_TEST) {
engine::meta::TableSchema table_info = BuildTableSchema();
engine::Status stat = db_->CreateTable(table_info);
@ -288,7 +288,7 @@ TEST_F(MemManagerTest, INSERT_TEST) {
LOG(DEBUG) << "total_time spent in INSERT_TEST (ms) : " << total_time;
}
TEST_F(MemManagerTest, CONCURRENT_INSERT_SEARCH_TEST) {
TEST_F(MemManagerTest2, CONCURRENT_INSERT_SEARCH_TEST) {
engine::meta::TableSchema table_info = BuildTableSchema();
engine::Status stat = db_->CreateTable(table_info);
@ -359,7 +359,7 @@ TEST_F(MemManagerTest, CONCURRENT_INSERT_SEARCH_TEST) {
search.join();
};
TEST_F(MemManagerTest, VECTOR_IDS_TEST) {
TEST_F(MemManagerTest2, VECTOR_IDS_TEST) {
engine::meta::TableSchema table_info = BuildTableSchema();
engine::Status stat = db_->CreateTable(table_info);

11
cpp/unittest/db/utils.cpp
View File

@ -51,6 +51,10 @@ void BaseTest::InitLog() {
el::Loggers::reconfigureLogger("default", defaultConf);
}
void BaseTest::SetUp() {
InitLog();
}
engine::Options BaseTest::GetOptions() {
auto options = engine::OptionsFactory::Build();
options.meta.path = "/tmp/milvus_test";
@ -60,7 +64,7 @@ engine::Options BaseTest::GetOptions() {
/////////////////////////////////////////////////////////////////////////////////////////////////////////////////
void DBTest::SetUp() {
InitLog();
BaseTest::SetUp();
server::ConfigNode& config = server::ServerConfig::GetInstance().GetConfig(server::CONFIG_CACHE);
config.AddSequenceItem(server::CONFIG_GPU_IDS, "0");
@ -104,7 +108,8 @@ engine::Options DBTest2::GetOptions() {
/////////////////////////////////////////////////////////////////////////////////////////////////////////////////
void MetaTest::SetUp() {
InitLog();
BaseTest::SetUp();
impl_ = engine::DBMetaImplFactory::Build();
}
@ -127,7 +132,7 @@ engine::Options MySqlDBTest::GetOptions() {
/////////////////////////////////////////////////////////////////////////////////////////////////////////////////
void MySqlMetaTest::SetUp() {
InitLog();
BaseTest::SetUp();
engine::DBMetaOptions options = GetOptions().meta;
int mode = engine::Options::MODE::SINGLE;

8
cpp/unittest/db/utils.h
View File

@ -37,6 +37,8 @@ void ASSERT_STATS(zilliz::milvus::engine::Status &stat);
class BaseTest : public ::testing::Test {
protected:
void InitLog();
virtual void SetUp() override;
virtual zilliz::milvus::engine::Options GetOptions();
};
@ -82,5 +84,9 @@ class MySqlMetaTest : public BaseTest {
/////////////////////////////////////////////////////////////////////////////////////////////////////////////////
class MemManagerTest : public DBTest {
class MemManagerTest : public BaseTest {
};
/////////////////////////////////////////////////////////////////////////////////////////////////////////////////
class MemManagerTest2 : public DBTest {
};