milvus-io
/
milvus
mirror of https://github.com/milvus-io/milvus.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #1562 from yhmo/master 

rename C++ sdk
pull/1566/head
Jin Hai 2020-03-08 16:59:15 +08:00 committed by GitHub
parent 929a976b11 0c09eb9e1a
commit 8034123a0a
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
11 changed files with 1213 additions and 1228 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

107
sdk/examples/binary_vector/src/ClientTest.cpp
View File

@ -22,52 +22,53 @@
namespace { namespace {
const char* TABLE_NAME = milvus_sdk::Utils::GenTableName().c_str(); const char* COLLECTION_NAME = milvus_sdk::Utils::GenCollectionName().c_str();
constexpr int64_t TABLE_DIMENSION = 512; constexpr int64_t COLLECTION_DIMENSION = 512;
constexpr int64_t TABLE_INDEX_FILE_SIZE = 128; constexpr int64_t COLLECTION_INDEX_FILE_SIZE = 128;
constexpr milvus::MetricType TABLE_METRIC_TYPE = milvus::MetricType::TANIMOTO; constexpr milvus::MetricType COLLECTION_METRIC_TYPE = milvus::MetricType::TANIMOTO;
constexpr int64_t BATCH_ROW_COUNT = 100000; constexpr int64_t BATCH_ENTITY_COUNT = 100000;
constexpr int64_t NQ = 5; constexpr int64_t NQ = 5;
constexpr int64_t TOP_K = 10; constexpr int64_t TOP_K = 10;
constexpr int64_t NPROBE = 32; constexpr int64_t NPROBE = 32;
constexpr int64_t SEARCH_TARGET = 5000; // change this value, result is different, ensure less than BATCH_ROW_COUNT constexpr int64_t SEARCH_TARGET = 5000; // change this value, result is different, ensure less than BATCH_ENTITY_COUNT
constexpr int64_t ADD_VECTOR_LOOP = 20; constexpr int64_t ADD_ENTITY_LOOP = 20;
constexpr milvus::IndexType INDEX_TYPE = milvus::IndexType::IVFFLAT; constexpr milvus::IndexType INDEX_TYPE = milvus::IndexType::IVFFLAT;
milvus::TableSchema milvus::CollectionParam
BuildTableSchema() { BuildCollectionParam() {
milvus::TableSchema tb_schema = {TABLE_NAME, TABLE_DIMENSION, TABLE_INDEX_FILE_SIZE, TABLE_METRIC_TYPE}; milvus::CollectionParam
return tb_schema; collection_param = {COLLECTION_NAME, COLLECTION_DIMENSION, COLLECTION_INDEX_FILE_SIZE, COLLECTION_METRIC_TYPE};
return collection_param;
} }
milvus::IndexParam milvus::IndexParam
BuildIndexParam() { BuildIndexParam() {
JSON json_params = {{"nlist", 1024}}; JSON json_params = {{"nlist", 1024}};
milvus::IndexParam index_param = {TABLE_NAME, INDEX_TYPE, json_params.dump()}; milvus::IndexParam index_param = {COLLECTION_NAME, INDEX_TYPE, json_params.dump()};
return index_param; return index_param;
} }
void void
BuildBinaryVectors(int64_t from, int64_t to, std::vector<milvus::RowRecord>& vector_record_array, BuildBinaryVectors(int64_t from, int64_t to, std::vector<milvus::Entity>& entity_array,
std::vector<int64_t>& record_ids, int64_t dimension) { std::vector<int64_t>& entity_ids, int64_t dimension) {
if (to <= from) { if (to <= from) {
return; return;
} }
vector_record_array.clear(); entity_array.clear();
record_ids.clear(); entity_ids.clear();
int64_t dim_byte = dimension/8; int64_t dim_byte = dimension/8;
for (int64_t k = from; k < to; k++) { for (int64_t k = from; k < to; k++) {
milvus::RowRecord record; milvus::Entity entity;
record.binary_data.resize(dim_byte); entity.binary_data.resize(dim_byte);
for (int64_t i = 0; i < dim_byte; i++) { for (int64_t i = 0; i < dim_byte; i++) {
record.binary_data[i] = (uint8_t)lrand48(); entity.binary_data[i] = (uint8_t)lrand48();
} }
vector_record_array.emplace_back(record); entity_array.emplace_back(entity);
record_ids.push_back(k); entity_ids.push_back(k);
} }
} }
@ -84,54 +85,54 @@ ClientTest::Test(const std::string& address, const std::string& port) {
std::cout << "Connect function call status: " << stat.message() << std::endl; std::cout << "Connect function call status: " << stat.message() << std::endl;
} }
{ // create table { // create collection
milvus::TableSchema tb_schema = BuildTableSchema(); milvus::CollectionParam collection_param = BuildCollectionParam();
stat = conn->CreateTable(tb_schema); stat = conn->CreateCollection(collection_param);
std::cout << "CreateTable function call status: " << stat.message() << std::endl; std::cout << "CreateCollection function call status: " << stat.message() << std::endl;
milvus_sdk::Utils::PrintTableSchema(tb_schema); milvus_sdk::Utils::PrintCollectionParam(collection_param);
bool has_table = conn->HasTable(tb_schema.table_name); bool has_collection = conn->HasCollection(collection_param.collection_name);
if (has_table) { if (has_collection) {
std::cout << "Table is created" << std::endl; std::cout << "Collection is created" << std::endl;
} }
} }
std::vector<std::pair<int64_t, milvus::RowRecord>> search_record_array; std::vector<std::pair<int64_t, milvus::Entity>> search_entity_array;
{ // insert vectors { // insert vectors
for (int i = 0; i < ADD_VECTOR_LOOP; i++) { for (int i = 0; i < ADD_ENTITY_LOOP; i++) {
std::vector<milvus::RowRecord> record_array; std::vector<milvus::Entity> entity_array;
std::vector<int64_t> record_ids; std::vector<int64_t> entity_ids;
int64_t begin_index = i * BATCH_ROW_COUNT; int64_t begin_index = i * BATCH_ENTITY_COUNT;
{ // generate vectors { // generate vectors
milvus_sdk::TimeRecorder rc("Build vectors No." + std::to_string(i)); milvus_sdk::TimeRecorder rc("Build entities No." + std::to_string(i));
BuildBinaryVectors(begin_index, BuildBinaryVectors(begin_index,
begin_index + BATCH_ROW_COUNT, begin_index + BATCH_ENTITY_COUNT,
record_array, entity_array,
record_ids, entity_ids,
TABLE_DIMENSION); COLLECTION_DIMENSION);
} }
if (search_record_array.size() < NQ) { if (search_entity_array.size() < NQ) {
search_record_array.push_back(std::make_pair(record_ids[SEARCH_TARGET], record_array[SEARCH_TARGET])); search_entity_array.push_back(std::make_pair(entity_ids[SEARCH_TARGET], entity_array[SEARCH_TARGET]));
} }
std::string title = "Insert " + std::to_string(record_array.size()) + " vectors No." + std::to_string(i); std::string title = "Insert " + std::to_string(entity_array.size()) + " entities No." + std::to_string(i);
milvus_sdk::TimeRecorder rc(title); milvus_sdk::TimeRecorder rc(title);
stat = conn->Insert(TABLE_NAME, "", record_array, record_ids); stat = conn->Insert(COLLECTION_NAME, "", entity_array, entity_ids);
std::cout << "InsertVector function call status: " << stat.message() << std::endl; std::cout << "Insert function call status: " << stat.message() << std::endl;
std::cout << "Returned id array count: " << record_ids.size() << std::endl; std::cout << "Returned id array count: " << entity_ids.size() << std::endl;
} }
} }
{ // flush buffer { // flush buffer
stat = conn->FlushTable(TABLE_NAME); stat = conn->FlushCollection(COLLECTION_NAME);
std::cout << "FlushTable function call status: " << stat.message() << std::endl; std::cout << "FlushCollection function call status: " << stat.message() << std::endl;
} }
{ // search vectors { // search vectors
std::vector<std::string> partition_tags; std::vector<std::string> partition_tags;
milvus::TopKQueryResult topk_query_result; milvus::TopKQueryResult topk_query_result;
milvus_sdk::Utils::DoSearch(conn, TABLE_NAME, partition_tags, TOP_K, NPROBE, search_record_array, milvus_sdk::Utils::DoSearch(conn, COLLECTION_NAME, partition_tags, TOP_K, NPROBE, search_entity_array,
topk_query_result); topk_query_result);
} }
@ -144,7 +145,7 @@ ClientTest::Test(const std::string& address, const std::string& port) {
std::cout << "CreateIndex function call status: " << stat.message() << std::endl; std::cout << "CreateIndex function call status: " << stat.message() << std::endl;
milvus::IndexParam index2; milvus::IndexParam index2;
stat = conn->DescribeIndex(TABLE_NAME, index2); stat = conn->DescribeIndex(COLLECTION_NAME, index2);
std::cout << "DescribeIndex function call status: " << stat.message() << std::endl; std::cout << "DescribeIndex function call status: " << stat.message() << std::endl;
milvus_sdk::Utils::PrintIndexParam(index2); milvus_sdk::Utils::PrintIndexParam(index2);
} }
@ -152,13 +153,13 @@ ClientTest::Test(const std::string& address, const std::string& port) {
{ // search vectors { // search vectors
std::vector<std::string> partition_tags; std::vector<std::string> partition_tags;
milvus::TopKQueryResult topk_query_result; milvus::TopKQueryResult topk_query_result;
milvus_sdk::Utils::DoSearch(conn, TABLE_NAME, partition_tags, TOP_K, NPROBE, search_record_array, milvus_sdk::Utils::DoSearch(conn, COLLECTION_NAME, partition_tags, TOP_K, NPROBE, search_entity_array,
topk_query_result); topk_query_result);
} }
{ // drop table { // drop collection
stat = conn->DropTable(TABLE_NAME); stat = conn->DropCollection(COLLECTION_NAME);
std::cout << "DropTable function call status: " << stat.message() << std::endl; std::cout << "DropCollection function call status: " << stat.message() << std::endl;
} }
milvus::Connection::Destroy(conn); milvus::Connection::Destroy(conn);

153
sdk/examples/partition/src/ClientTest.cpp
View File

@ -22,12 +22,12 @@
namespace { namespace {
const char* TABLE_NAME = milvus_sdk::Utils::GenTableName().c_str(); const char* COLLECTION_NAME = milvus_sdk::Utils::GenCollectionName().c_str();
constexpr int64_t TABLE_DIMENSION = 512; constexpr int64_t COLLECTION_DIMENSION = 512;
constexpr int64_t TABLE_INDEX_FILE_SIZE = 1024; constexpr int64_t COLLECTION_INDEX_FILE_SIZE = 1024;
constexpr milvus::MetricType TABLE_METRIC_TYPE = milvus::MetricType::L2; constexpr milvus::MetricType COLLECTION_METRIC_TYPE = milvus::MetricType::L2;
constexpr int64_t BATCH_ROW_COUNT = 10000; constexpr int64_t BATCH_ENTITY_COUNT = 10000;
constexpr int64_t NQ = 5; constexpr int64_t NQ = 5;
constexpr int64_t TOP_K = 10; constexpr int64_t TOP_K = 10;
constexpr int64_t NPROBE = 32; constexpr int64_t NPROBE = 32;
@ -36,27 +36,45 @@ constexpr milvus::IndexType INDEX_TYPE = milvus::IndexType::IVFSQ8;
constexpr int32_t PARTITION_COUNT = 5; constexpr int32_t PARTITION_COUNT = 5;
constexpr int32_t TARGET_PARTITION = 3; constexpr int32_t TARGET_PARTITION = 3;
milvus::TableSchema milvus::CollectionParam
BuildTableSchema() { BuildCollectionParam() {
milvus::TableSchema tb_schema = {TABLE_NAME, TABLE_DIMENSION, TABLE_INDEX_FILE_SIZE, TABLE_METRIC_TYPE}; milvus::CollectionParam
return tb_schema; collection_param = {COLLECTION_NAME, COLLECTION_DIMENSION, COLLECTION_INDEX_FILE_SIZE, COLLECTION_METRIC_TYPE};
return collection_param;
} }
milvus::PartitionParam milvus::PartitionParam
BuildPartitionParam(int32_t index) { BuildPartitionParam(int32_t index) {
std::string tag = std::to_string(index); std::string tag = std::to_string(index);
std::string partition_name = std::string(TABLE_NAME) + "_" + tag; std::string partition_name = std::string(COLLECTION_NAME) + "_" + tag;
milvus::PartitionParam partition_param = {TABLE_NAME, tag}; milvus::PartitionParam partition_param = {COLLECTION_NAME, tag};
return partition_param; return partition_param;
} }
milvus::IndexParam milvus::IndexParam
BuildIndexParam() { BuildIndexParam() {
JSON json_params = {{"nlist", 16384}}; JSON json_params = {{"nlist", 16384}};
milvus::IndexParam index_param = {TABLE_NAME, INDEX_TYPE, json_params.dump()}; milvus::IndexParam index_param = {COLLECTION_NAME, INDEX_TYPE, json_params.dump()};
return index_param; return index_param;
} }
void
CountCollection(std::shared_ptr<milvus::Connection>& conn) {
int64_t entity_count = 0;
auto stat = conn->CountCollection(COLLECTION_NAME, entity_count);
std::cout << COLLECTION_NAME << "(" << entity_count << " entities)" << std::endl;
}
void
ShowCollectionInfo(std::shared_ptr<milvus::Connection>& conn) {
CountCollection(conn);
milvus::CollectionInfo collection_info;
auto stat = conn->ShowCollectionInfo(COLLECTION_NAME, collection_info);
milvus_sdk::Utils::PrintCollectionInfo(collection_info);
std::cout << "ShowCollectionInfo function call status: " << stat.message() << std::endl;
}
} // namespace } // namespace
void void
@ -70,11 +88,11 @@ ClientTest::Test(const std::string& address, const std::string& port) {
std::cout << "Connect function call status: " << stat.message() << std::endl; std::cout << "Connect function call status: " << stat.message() << std::endl;
} }
{ // create table { // create collection
milvus::TableSchema tb_schema = BuildTableSchema(); milvus::CollectionParam tb_schema = BuildCollectionParam();
stat = conn->CreateTable(tb_schema); stat = conn->CreateCollection(tb_schema);
std::cout << "CreateTable function call status: " << stat.message() << std::endl; std::cout << "CreateCollection function call status: " << stat.message() << std::endl;
milvus_sdk::Utils::PrintTableSchema(tb_schema); milvus_sdk::Utils::PrintCollectionParam(tb_schema);
} }
{ // create partition { // create partition
@ -87,7 +105,7 @@ ClientTest::Test(const std::string& address, const std::string& port) {
// show partitions // show partitions
milvus::PartitionTagList partition_array; milvus::PartitionTagList partition_array;
stat = conn->ShowPartitions(TABLE_NAME, partition_array); stat = conn->ShowPartitions(COLLECTION_NAME, partition_array);
std::cout << partition_array.size() << " partitions created:" << std::endl; std::cout << partition_array.size() << " partitions created:" << std::endl;
for (auto& partition_tag : partition_array) { for (auto& partition_tag : partition_array) {
@ -96,64 +114,56 @@ ClientTest::Test(const std::string& address, const std::string& port) {
} }
{ // insert vectors { // insert vectors
milvus_sdk::TimeRecorder rc("All vectors"); milvus_sdk::TimeRecorder rc("All entities");
for (int i = 0; i < PARTITION_COUNT * 5; i++) { for (int i = 0; i < PARTITION_COUNT * 5; i++) {
std::vector<milvus::RowRecord> record_array; std::vector<milvus::Entity> entity_array;
std::vector<int64_t> record_ids; std::vector<int64_t> entity_ids;
int64_t begin_index = i * BATCH_ROW_COUNT; int64_t begin_index = i * BATCH_ENTITY_COUNT;
{ // generate vectors { // generate vectors
milvus_sdk::TimeRecorder rc("Build vectors No." + std::to_string(i)); milvus_sdk::TimeRecorder rc("Build entities No." + std::to_string(i));
milvus_sdk::Utils::BuildVectors(begin_index, begin_index + BATCH_ROW_COUNT, record_array, record_ids, milvus_sdk::Utils::BuildEntities(begin_index,
TABLE_DIMENSION); begin_index + BATCH_ENTITY_COUNT,
entity_array,
entity_ids,
COLLECTION_DIMENSION);
} }
std::string title = "Insert " + std::to_string(record_array.size()) + " vectors No." + std::to_string(i); std::string title = "Insert " + std::to_string(entity_array.size()) + " entities No." + std::to_string(i);
milvus_sdk::TimeRecorder rc(title); milvus_sdk::TimeRecorder rc(title);
stat = conn->Insert(TABLE_NAME, std::to_string(i % PARTITION_COUNT), record_array, record_ids); stat = conn->Insert(COLLECTION_NAME, std::to_string(i % PARTITION_COUNT), entity_array, entity_ids);
} }
} }
{ // flush buffer { // flush buffer
stat = conn->FlushTable(TABLE_NAME); stat = conn->FlushCollection(COLLECTION_NAME);
std::cout << "FlushTable function call status: " << stat.message() << std::endl; std::cout << "FlushCollection function call status: " << stat.message() << std::endl;
} }
{ // table row count ShowCollectionInfo(conn);
int64_t row_count = 0;
stat = conn->CountTable(TABLE_NAME, row_count);
std::cout << TABLE_NAME << "(" << row_count << " rows)" << std::endl;
}
{ // get table information std::vector<std::pair<int64_t, milvus::Entity>> search_entity_array;
milvus::TableInfo table_info;
stat = conn->ShowTableInfo(TABLE_NAME, table_info);
milvus_sdk::Utils::PrintTableInfo(table_info);
std::cout << "ShowTableInfo function call status: " << stat.message() << std::endl;
}
std::vector<std::pair<int64_t, milvus::RowRecord>> search_record_array;
{ // build search vectors { // build search vectors
std::vector<milvus::RowRecord> record_array; std::vector<milvus::Entity> entity_array;
std::vector<int64_t> record_ids; std::vector<int64_t> entity_ids;
int64_t index = TARGET_PARTITION * BATCH_ROW_COUNT + SEARCH_TARGET; int64_t index = TARGET_PARTITION * BATCH_ENTITY_COUNT + SEARCH_TARGET;
milvus_sdk::Utils::BuildVectors(index, index + 1, record_array, record_ids, TABLE_DIMENSION); milvus_sdk::Utils::BuildEntities(index, index + 1, entity_array, entity_ids, COLLECTION_DIMENSION);
search_record_array.push_back(std::make_pair(record_ids[0], record_array[0])); search_entity_array.push_back(std::make_pair(entity_ids[0], entity_array[0]));
} }
{ // search vectors { // search vectors
std::cout << "Search in correct partition" << std::endl; std::cout << "Search in correct partition" << std::endl;
std::vector<std::string> partition_tags = {std::to_string(TARGET_PARTITION)}; std::vector<std::string> partition_tags = {std::to_string(TARGET_PARTITION)};
milvus::TopKQueryResult topk_query_result; milvus::TopKQueryResult topk_query_result;
milvus_sdk::Utils::DoSearch(conn, TABLE_NAME, partition_tags, TOP_K, NPROBE, search_record_array, milvus_sdk::Utils::DoSearch(conn, COLLECTION_NAME, partition_tags, TOP_K, NPROBE, search_entity_array,
topk_query_result); topk_query_result);
std::cout << "Search in wrong partition" << std::endl; std::cout << "Search in wrong partition" << std::endl;
partition_tags = {"0"}; partition_tags = {"0"};
milvus_sdk::Utils::DoSearch(conn, TABLE_NAME, partition_tags, TOP_K, NPROBE, search_record_array, milvus_sdk::Utils::DoSearch(conn, COLLECTION_NAME, partition_tags, TOP_K, NPROBE, search_entity_array,
topk_query_result); topk_query_result);
std::cout << "Search by regex matched partition tag" << std::endl; std::cout << "Search by regex matched partition tag" << std::endl;
partition_tags = {"\\d"}; partition_tags = {"\\d"};
milvus_sdk::Utils::DoSearch(conn, TABLE_NAME, partition_tags, TOP_K, NPROBE, search_record_array, milvus_sdk::Utils::DoSearch(conn, COLLECTION_NAME, partition_tags, TOP_K, NPROBE, search_entity_array,
topk_query_result); topk_query_result);
} }
@ -166,57 +176,42 @@ ClientTest::Test(const std::string& address, const std::string& port) {
std::cout << "CreateIndex function call status: " << stat.message() << std::endl; std::cout << "CreateIndex function call status: " << stat.message() << std::endl;
milvus::IndexParam index2; milvus::IndexParam index2;
stat = conn->DescribeIndex(TABLE_NAME, index2); stat = conn->DescribeIndex(COLLECTION_NAME, index2);
std::cout << "DescribeIndex function call status: " << stat.message() << std::endl; std::cout << "DescribeIndex function call status: " << stat.message() << std::endl;
milvus_sdk::Utils::PrintIndexParam(index2); milvus_sdk::Utils::PrintIndexParam(index2);
} }
{ // table row count ShowCollectionInfo(conn);
int64_t row_count = 0;
stat = conn->CountTable(TABLE_NAME, row_count);
std::cout << TABLE_NAME << "(" << row_count << " rows)" << std::endl;
}
{ // get table information
milvus::TableInfo table_info;
stat = conn->ShowTableInfo(TABLE_NAME, table_info);
milvus_sdk::Utils::PrintTableInfo(table_info);
std::cout << "ShowTableInfo function call status: " << stat.message() << std::endl;
}
{ // drop partition { // drop partition
milvus::PartitionParam param1 = {TABLE_NAME, std::to_string(TARGET_PARTITION)}; milvus::PartitionParam param1 = {COLLECTION_NAME, std::to_string(TARGET_PARTITION)};
milvus_sdk::Utils::PrintPartitionParam(param1); milvus_sdk::Utils::PrintPartitionParam(param1);
stat = conn->DropPartition(param1); stat = conn->DropPartition(param1);
std::cout << "DropPartition function call status: " << stat.message() << std::endl; std::cout << "DropPartition function call status: " << stat.message() << std::endl;
} }
{ // table row count CountCollection(conn);
int64_t row_count = 0;
stat = conn->CountTable(TABLE_NAME, row_count);
std::cout << TABLE_NAME << "(" << row_count << " rows)" << std::endl;
}
{ // search vectors { // search vectors, will get search error since we delete a partition
std::cout << "Search in whole table" << std::endl; std::cout << "Search in whole collection after delete one partition" << std::endl;
std::vector<std::string> partition_tags; std::vector<std::string> partition_tags;
milvus::TopKQueryResult topk_query_result; milvus::TopKQueryResult topk_query_result;
milvus_sdk::Utils::DoSearch(conn, TABLE_NAME, partition_tags, TOP_K, NPROBE, search_record_array, milvus_sdk::Utils::DoSearch(conn, COLLECTION_NAME, partition_tags, TOP_K, NPROBE, search_entity_array,
topk_query_result); topk_query_result);
} }
{ // drop index { // drop index
stat = conn->DropIndex(TABLE_NAME); stat = conn->DropIndex(COLLECTION_NAME);
std::cout << "DropIndex function call status: " << stat.message() << std::endl; std::cout << "DropIndex function call status: " << stat.message() << std::endl;
int64_t row_count = 0; int64_t entity_count = 0;
stat = conn->CountTable(TABLE_NAME, row_count); stat = conn->CountCollection(COLLECTION_NAME, entity_count);
std::cout << TABLE_NAME << "(" << row_count << " rows)" << std::endl; std::cout << COLLECTION_NAME << "(" << entity_count << " entities)" << std::endl;
} }
{ // drop table { // drop collection
stat = conn->DropTable(TABLE_NAME); stat = conn->DropCollection(COLLECTION_NAME);
std::cout << "DropTable function call status: " << stat.message() << std::endl; std::cout << "DropCollection function call status: " << stat.message() << std::endl;
} }
milvus::Connection::Destroy(conn); milvus::Connection::Destroy(conn);

199
sdk/examples/simple/src/ClientTest.cpp
View File

@ -19,20 +19,19 @@
#include <utility> #include <utility>
#include <vector> #include <vector>
namespace { namespace {
const char* TABLE_NAME = milvus_sdk::Utils::GenTableName().c_str(); const char* COLLECTION_NAME = milvus_sdk::Utils::GenCollectionName().c_str();
constexpr int64_t TABLE_DIMENSION = 512; constexpr int64_t COLLECTION_DIMENSION = 512;
constexpr int64_t TABLE_INDEX_FILE_SIZE = 1024; constexpr int64_t COLLECTION_INDEX_FILE_SIZE = 1024;
constexpr milvus::MetricType TABLE_METRIC_TYPE = milvus::MetricType::L2; constexpr milvus::MetricType COLLECTION_METRIC_TYPE = milvus::MetricType::L2;
constexpr int64_t BATCH_ROW_COUNT = 100000; constexpr int64_t BATCH_ENTITY_COUNT = 100000;
constexpr int64_t NQ = 5; constexpr int64_t NQ = 5;
constexpr int64_t TOP_K = 10; constexpr int64_t TOP_K = 10;
constexpr int64_t NPROBE = 32; constexpr int64_t NPROBE = 32;
constexpr int64_t SEARCH_TARGET = 5000; // change this value, result is different constexpr int64_t SEARCH_TARGET = 5000; // change this value, result is different
constexpr int64_t ADD_VECTOR_LOOP = 5; constexpr int64_t ADD_ENTITY_LOOP = 5;
constexpr milvus::IndexType INDEX_TYPE = milvus::IndexType::IVFSQ8; constexpr milvus::IndexType INDEX_TYPE = milvus::IndexType::IVFSQ8;
constexpr int32_t NLIST = 16384; constexpr int32_t NLIST = 16384;
@ -63,191 +62,201 @@ ClientTest::ShowSdkVersion() {
} }
void void
ClientTest::ShowTables(std::vector<std::string>& tables) { ClientTest::ShowCollections(std::vector<std::string>& collection_array) {
milvus::Status stat = conn_->ShowTables(tables); milvus::Status stat = conn_->ShowCollections(collection_array);
std::cout << "ShowTables function call status: " << stat.message() << std::endl; std::cout << "ShowCollections function call status: " << stat.message() << std::endl;
std::cout << "All tables: " << std::endl; std::cout << "All collections: " << std::endl;
for (auto& table : tables) { for (auto& collection : collection_array) {
int64_t row_count = 0; int64_t entity_count = 0;
stat = conn_->CountTable(table, row_count); stat = conn_->CountCollection(collection, entity_count);
std::cout << "\t" << table << "(" << row_count << " rows)" << std::endl; std::cout << "\t" << collection << "(" << entity_count << " entities)" << std::endl;
} }
} }
void void
ClientTest::CreateTable(const std::string& table_name, int64_t dim, milvus::MetricType type) { ClientTest::CreateCollection(const std::string& collection_name, int64_t dim, milvus::MetricType type) {
milvus::TableSchema tb_schema = {table_name, dim, TABLE_INDEX_FILE_SIZE, type}; milvus::CollectionParam collection_param = {collection_name, dim, COLLECTION_INDEX_FILE_SIZE, type};
milvus::Status stat = conn_->CreateTable(tb_schema); milvus::Status stat = conn_->CreateCollection(collection_param);
std::cout << "CreateTable function call status: " << stat.message() << std::endl; std::cout << "CreateCollection function call status: " << stat.message() << std::endl;
milvus_sdk::Utils::PrintTableSchema(tb_schema); milvus_sdk::Utils::PrintCollectionParam(collection_param);
bool has_table = conn_->HasTable(tb_schema.table_name); bool has_table = conn_->HasCollection(collection_param.collection_name);
if (has_table) { if (has_table) {
std::cout << "Table is created" << std::endl; std::cout << "Collection is created" << std::endl;
} }
} }
void void
ClientTest::DescribeTable(const std::string& table_name) { ClientTest::DescribeCollection(const std::string& collection_name) {
milvus::TableSchema tb_schema; milvus::CollectionParam collection_param;
milvus::Status stat = conn_->DescribeTable(table_name, tb_schema); milvus::Status stat = conn_->DescribeCollection(collection_name, collection_param);
std::cout << "DescribeTable function call status: " << stat.message() << std::endl; std::cout << "DescribeCollection function call status: " << stat.message() << std::endl;
milvus_sdk::Utils::PrintTableSchema(tb_schema); milvus_sdk::Utils::PrintCollectionParam(collection_param);
} }
void void
ClientTest::InsertVectors(const std::string& table_name, int64_t dim) { ClientTest::InsertEntities(const std::string& collection_name, int64_t dim) {
for (int i = 0; i < ADD_VECTOR_LOOP; i++) { for (int i = 0; i < ADD_ENTITY_LOOP; i++) {
std::vector<milvus::RowRecord> record_array; std::vector<milvus::Entity> entity_array;
std::vector<int64_t> record_ids; std::vector<int64_t> record_ids;
int64_t begin_index = i * BATCH_ROW_COUNT; int64_t begin_index = i * BATCH_ENTITY_COUNT;
{ // generate vectors { // generate vectors
milvus_sdk::TimeRecorder rc("Build vectors No." + std::to_string(i)); milvus_sdk::TimeRecorder rc("Build entities No." + std::to_string(i));
milvus_sdk::Utils::BuildVectors(begin_index, begin_index + BATCH_ROW_COUNT, record_array, record_ids, dim); milvus_sdk::Utils::BuildEntities(begin_index,
begin_index + BATCH_ENTITY_COUNT,
entity_array,
record_ids,
dim);
} }
std::string title = "Insert " + std::to_string(record_array.size()) + " vectors No." + std::to_string(i); std::string title = "Insert " + std::to_string(entity_array.size()) + " entities No." + std::to_string(i);
milvus_sdk::TimeRecorder rc(title); milvus_sdk::TimeRecorder rc(title);
milvus::Status stat = conn_->Insert(table_name, "", record_array, record_ids); milvus::Status stat = conn_->Insert(collection_name, "", entity_array, record_ids);
std::cout << "InsertVector function call status: " << stat.message() << std::endl; std::cout << "InsertEntities function call status: " << stat.message() << std::endl;
std::cout << "Returned id array count: " << record_ids.size() << std::endl; std::cout << "Returned id array count: " << record_ids.size() << std::endl;
} }
} }
void void
ClientTest::BuildSearchVectors(int64_t nq, int64_t dim) { ClientTest::BuildSearchEntities(int64_t nq, int64_t dim) {
search_record_array_.clear(); search_entity_array_.clear();
search_id_array_.clear(); search_id_array_.clear();
for (int64_t i = 0; i < nq; i++) { for (int64_t i = 0; i < nq; i++) {
std::vector<milvus::RowRecord> record_array; std::vector<milvus::Entity> entity_array;
std::vector<int64_t> record_ids; std::vector<int64_t> record_ids;
int64_t index = i * BATCH_ROW_COUNT + SEARCH_TARGET; int64_t index = i * BATCH_ENTITY_COUNT + SEARCH_TARGET;
milvus_sdk::Utils::BuildVectors(index, index + 1, record_array, record_ids, dim); milvus_sdk::Utils::BuildEntities(index, index + 1, entity_array, record_ids, dim);
search_record_array_.push_back(std::make_pair(record_ids[0], record_array[0])); search_entity_array_.push_back(std::make_pair(record_ids[0], entity_array[0]));
search_id_array_.push_back(record_ids[0]); search_id_array_.push_back(record_ids[0]);
} }
} }
void void
ClientTest::Flush(const std::string& table_name) { ClientTest::Flush(const std::string& collection_name) {
milvus_sdk::TimeRecorder rc("Flush"); milvus_sdk::TimeRecorder rc("Flush");
milvus::Status stat = conn_->FlushTable(table_name); milvus::Status stat = conn_->FlushCollection(collection_name);
std::cout << "FlushTable function call status: " << stat.message() << std::endl; std::cout << "FlushCollection function call status: " << stat.message() << std::endl;
} }
void void
ClientTest::ShowTableInfo(const std::string& table_name) { ClientTest::ShowCollectionInfo(const std::string& collection_name) {
milvus::TableInfo table_info; milvus::CollectionInfo collection_info;
milvus::Status stat = conn_->ShowTableInfo(table_name, table_info); milvus::Status stat = conn_->ShowCollectionInfo(collection_name, collection_info);
milvus_sdk::Utils::PrintTableInfo(table_info); milvus_sdk::Utils::PrintCollectionInfo(collection_info);
std::cout << "ShowTableInfo function call status: " << stat.message() << std::endl; std::cout << "ShowCollectionInfo function call status: " << stat.message() << std::endl;
} }
void void
ClientTest::GetVectorById(const std::string& table_name, int64_t id) { ClientTest::GetEntityById(const std::string& collection_name, int64_t id) {
milvus::RowRecord vector_data; milvus::Entity entity;
milvus::Status stat = conn_->GetVectorByID(table_name, id, vector_data); milvus::Status stat = conn_->GetEntityByID(collection_name, id, entity);
std::cout << "The vector " << id << " has " << vector_data.float_data.size() << " float elements" << std::endl; std::cout << "The entity " << id << " has " << entity.float_data.size() << " float elements" << std::endl;
std::cout << "GetVectorByID function call status: " << stat.message() << std::endl; std::cout << "GetEntityById function call status: " << stat.message() << std::endl;
} }
void void
ClientTest::SearchVectors(const std::string& table_name, int64_t topk, int64_t nprobe) { ClientTest::SearchEntities(const std::string& collection_name, int64_t topk, int64_t nprobe) {
std::vector<std::string> partition_tags; std::vector<std::string> partition_tags;
milvus::TopKQueryResult topk_query_result; milvus::TopKQueryResult topk_query_result;
milvus_sdk::Utils::DoSearch(conn_, table_name, partition_tags, topk, nprobe, search_record_array_, milvus_sdk::Utils::DoSearch(conn_, collection_name, partition_tags, topk, nprobe, search_entity_array_,
topk_query_result); topk_query_result);
} }
void void
ClientTest::CreateIndex(const std::string& table_name, milvus::IndexType type, int64_t nlist) { ClientTest::CreateIndex(const std::string& collection_name, milvus::IndexType type, int64_t nlist) {
milvus_sdk::TimeRecorder rc("Create index"); milvus_sdk::TimeRecorder rc("Create index");
std::cout << "Wait until create all index done" << std::endl; std::cout << "Wait until create all index done" << std::endl;
JSON json_params = {{"nlist", nlist}}; JSON json_params = {{"nlist", nlist}};
milvus::IndexParam index1 = {table_name, type, json_params.dump()}; milvus::IndexParam index1 = {collection_name, type, json_params.dump()};
milvus_sdk::Utils::PrintIndexParam(index1); milvus_sdk::Utils::PrintIndexParam(index1);
milvus::Status stat = conn_->CreateIndex(index1); milvus::Status stat = conn_->CreateIndex(index1);
std::cout << "CreateIndex function call status: " << stat.message() << std::endl; std::cout << "CreateIndex function call status: " << stat.message() << std::endl;
milvus::IndexParam index2; milvus::IndexParam index2;
stat = conn_->DescribeIndex(table_name, index2); stat = conn_->DescribeIndex(collection_name, index2);
std::cout << "DescribeIndex function call status: " << stat.message() << std::endl; std::cout << "DescribeIndex function call status: " << stat.message() << std::endl;
milvus_sdk::Utils::PrintIndexParam(index2); milvus_sdk::Utils::PrintIndexParam(index2);
} }
void void
ClientTest::PreloadTable(const std::string& table_name) { ClientTest::PreloadCollection(const std::string& collection_name) {
milvus::Status stat = conn_->PreloadTable(table_name); milvus::Status stat = conn_->PreloadCollection(collection_name);
std::cout << "PreloadTable function call status: " << stat.message() << std::endl; std::cout << "PreloadCollection function call status: " << stat.message() << std::endl;
} }
void void
ClientTest::DeleteByIds(const std::string& table_name, const std::vector<int64_t>& id_array) { ClientTest::DeleteByIds(const std::string& collection_name, const std::vector<int64_t>& id_array) {
milvus::Status stat = conn_->DeleteByID(table_name, id_array); std::cout << "Delete entity: ";
for (auto id : id_array) {
std::cout << "\t" << id;
}
std::cout << std::endl;
milvus::Status stat = conn_->DeleteByID(collection_name, id_array);
std::cout << "DeleteByID function call status: " << stat.message() << std::endl; std::cout << "DeleteByID function call status: " << stat.message() << std::endl;
{ {
milvus_sdk::TimeRecorder rc("Flush"); milvus_sdk::TimeRecorder rc("Flush");
stat = conn_->FlushTable(table_name); stat = conn_->FlushCollection(collection_name);
std::cout << "FlushTable function call status: " << stat.message() << std::endl; std::cout << "FlushCollection function call status: " << stat.message() << std::endl;
} }
{ {
// compact table // compact table
milvus_sdk::TimeRecorder rc1("Compact"); milvus_sdk::TimeRecorder rc1("Compact");
stat = conn_->CompactTable(table_name); stat = conn_->CompactCollection(collection_name);
std::cout << "CompactTable function call status: " << stat.message() << std::endl; std::cout << "CompactCollection function call status: " << stat.message() << std::endl;
} }
} }
void void
ClientTest::DropIndex(const std::string& table_name) { ClientTest::DropIndex(const std::string& collection_name) {
milvus::Status stat = conn_->DropIndex(table_name); milvus::Status stat = conn_->DropIndex(collection_name);
std::cout << "DropIndex function call status: " << stat.message() << std::endl; std::cout << "DropIndex function call status: " << stat.message() << std::endl;
int64_t row_count = 0; int64_t row_count = 0;
stat = conn_->CountTable(table_name, row_count); stat = conn_->CountCollection(collection_name, row_count);
std::cout << table_name << "(" << row_count << " rows)" << std::endl; std::cout << collection_name << "(" << row_count << " rows)" << std::endl;
} }
void void
ClientTest::DropTable(const std::string& table_name) { ClientTest::DropCollection(const std::string& collection_name) {
milvus::Status stat = conn_->DropTable(table_name); milvus::Status stat = conn_->DropCollection(collection_name);
std::cout << "DropTable function call status: " << stat.message() << std::endl; std::cout << "DropCollection function call status: " << stat.message() << std::endl;
} }
void void
ClientTest::Test() { ClientTest::Test() {
std::string table_name = TABLE_NAME; std::string collection_name = COLLECTION_NAME;
int64_t dim = TABLE_DIMENSION; int64_t dim = COLLECTION_DIMENSION;
milvus::MetricType metric_type = TABLE_METRIC_TYPE; milvus::MetricType metric_type = COLLECTION_METRIC_TYPE;
ShowServerVersion(); ShowServerVersion();
ShowSdkVersion(); ShowSdkVersion();
std::vector<std::string> table_array; std::vector<std::string> table_array;
ShowTables(table_array); ShowCollections(table_array);
CreateTable(table_name, dim, metric_type); CreateCollection(collection_name, dim, metric_type);
DescribeTable(table_name); DescribeCollection(collection_name);
InsertVectors(table_name, dim); InsertEntities(collection_name, dim);
BuildSearchVectors(NQ, dim); BuildSearchEntities(NQ, dim);
Flush(table_name); Flush(collection_name);
ShowTableInfo(table_name); ShowCollectionInfo(collection_name);
GetVectorById(table_name, search_id_array_[0]); GetEntityById(collection_name, search_id_array_[0]);
SearchVectors(table_name, TOP_K, NPROBE); SearchEntities(collection_name, TOP_K, NPROBE);
CreateIndex(table_name, INDEX_TYPE, NLIST); CreateIndex(collection_name, INDEX_TYPE, NLIST);
ShowTableInfo(table_name); ShowCollectionInfo(collection_name);
PreloadTable(table_name); PreloadCollection(collection_name);
std::vector<int64_t> delete_ids = {search_id_array_[0], search_id_array_[1]}; std::vector<int64_t> delete_ids = {search_id_array_[0], search_id_array_[1]};
DeleteByIds(table_name, delete_ids); DeleteByIds(collection_name, delete_ids);
SearchVectors(table_name, TOP_K, NPROBE); SearchEntities(collection_name, TOP_K, NPROBE); // this line get two search error since we delete two entities
DropIndex(table_name); DropIndex(collection_name);
DropTable(table_name); DropCollection(collection_name);
} }

24
sdk/examples/simple/src/ClientTest.h
View File

@ -34,49 +34,49 @@ class ClientTest {
ShowSdkVersion(); ShowSdkVersion();
void void
ShowTables(std::vector<std::string>&); ShowCollections(std::vector<std::string>&);
void void
CreateTable(const std::string&, int64_t, milvus::MetricType); CreateCollection(const std::string&, int64_t, milvus::MetricType);
void void
DescribeTable(const std::string&); DescribeCollection(const std::string&);
void void
InsertVectors(const std::string&, int64_t); InsertEntities(const std::string&, int64_t);
void void
BuildSearchVectors(int64_t, int64_t); BuildSearchEntities(int64_t, int64_t);
void void
Flush(const std::string&); Flush(const std::string&);
void void
ShowTableInfo(const std::string&); ShowCollectionInfo(const std::string&);
void void
GetVectorById(const std::string&, int64_t); GetEntityById(const std::string&, int64_t);
void void
SearchVectors(const std::string&, int64_t, int64_t); SearchEntities(const std::string&, int64_t, int64_t);
void void
CreateIndex(const std::string&, milvus::IndexType, int64_t); CreateIndex(const std::string&, milvus::IndexType, int64_t);
void void
PreloadTable(const std::string&); PreloadCollection(const std::string&);
void void
DeleteByIds(const std::string&, const std::vector<int64_t>&); DeleteByIds(const std::string&, const std::vector<int64_t>& id_array);
void void
DropIndex(const std::string&); DropIndex(const std::string&);
void void
DropTable(const std::string&); DropCollection(const std::string&);
private: private:
std::shared_ptr<milvus::Connection> conn_; std::shared_ptr<milvus::Connection> conn_;
std::vector<std::pair<int64_t, milvus::RowRecord>> search_record_array_; std::vector<std::pair<int64_t, milvus::Entity>> search_entity_array_;
std::vector<int64_t> search_id_array_; std::vector<int64_t> search_id_array_;
}; };

79
sdk/examples/utils/Utils.cpp
View File

@ -64,8 +64,8 @@ Utils::Sleep(int seconds) {
} }
const std::string& const std::string&
Utils::GenTableName() { Utils::GenCollectionName() {
static std::string s_id("tbl_" + CurrentTime()); static std::string s_id("C_" + CurrentTime());
return s_id; return s_id;
} }
@ -97,19 +97,19 @@ Utils::IndexTypeName(const milvus::IndexType& index_type) {
} }
void void
Utils::PrintTableSchema(const milvus::TableSchema& tb_schema) { Utils::PrintCollectionParam(const milvus::CollectionParam& collection_param) {
BLOCK_SPLITER BLOCK_SPLITER
std::cout << "Table name: " << tb_schema.table_name << std::endl; std::cout << "Collection name: " << collection_param.collection_name << std::endl;
std::cout << "Table dimension: " << tb_schema.dimension << std::endl; std::cout << "Collection dimension: " << collection_param.dimension << std::endl;
std::cout << "Table index file size: " << tb_schema.index_file_size << std::endl; std::cout << "Collection index file size: " << collection_param.index_file_size << std::endl;
std::cout << "Table metric type: " << MetricTypeName(tb_schema.metric_type) << std::endl; std::cout << "Collection metric type: " << MetricTypeName(collection_param.metric_type) << std::endl;
BLOCK_SPLITER BLOCK_SPLITER
} }
void void
Utils::PrintPartitionParam(const milvus::PartitionParam& partition_param) { Utils::PrintPartitionParam(const milvus::PartitionParam& partition_param) {
BLOCK_SPLITER BLOCK_SPLITER
std::cout << "Table name: " << partition_param.table_name << std::endl; std::cout << "Collection name: " << partition_param.collection_name << std::endl;
std::cout << "Partition tag: " << partition_param.partition_tag << std::endl; std::cout << "Partition tag: " << partition_param.partition_tag << std::endl;
BLOCK_SPLITER BLOCK_SPLITER
} }
@ -117,40 +117,40 @@ Utils::PrintPartitionParam(const milvus::PartitionParam& partition_param) {
void void
Utils::PrintIndexParam(const milvus::IndexParam& index_param) { Utils::PrintIndexParam(const milvus::IndexParam& index_param) {
BLOCK_SPLITER BLOCK_SPLITER
std::cout << "Index table name: " << index_param.table_name << std::endl; std::cout << "Index collection name: " << index_param.collection_name << std::endl;
std::cout << "Index type: " << IndexTypeName(index_param.index_type) << std::endl; std::cout << "Index type: " << IndexTypeName(index_param.index_type) << std::endl;
std::cout << "Index extra_params: " << index_param.extra_params << std::endl; std::cout << "Index extra_params: " << index_param.extra_params << std::endl;
BLOCK_SPLITER BLOCK_SPLITER
} }
void void
Utils::BuildVectors(int64_t from, int64_t to, std::vector<milvus::RowRecord>& vector_record_array, Utils::BuildEntities(int64_t from, int64_t to, std::vector<milvus::Entity>& entity_array,
std::vector<int64_t>& record_ids, int64_t dimension) { std::vector<int64_t>& entity_ids, int64_t dimension) {
if (to <= from) { if (to <= from) {
return; return;
} }
vector_record_array.clear(); entity_array.clear();
record_ids.clear(); entity_ids.clear();
for (int64_t k = from; k < to; k++) { for (int64_t k = from; k < to; k++) {
milvus::RowRecord record; milvus::Entity entity;
record.float_data.resize(dimension); entity.float_data.resize(dimension);
for (int64_t i = 0; i < dimension; i++) { for (int64_t i = 0; i < dimension; i++) {
record.float_data[i] = (float)(k % (i + 1)); entity.float_data[i] = (float)(k % (i + 1));
} }
vector_record_array.emplace_back(record); entity_array.emplace_back(entity);
record_ids.push_back(k); entity_ids.push_back(k);
} }
} }
void void
Utils::PrintSearchResult(const std::vector<std::pair<int64_t, milvus::RowRecord>>& search_record_array, Utils::PrintSearchResult(const std::vector<std::pair<int64_t, milvus::Entity>>& entity_array,
const milvus::TopKQueryResult& topk_query_result) { const milvus::TopKQueryResult& topk_query_result) {
BLOCK_SPLITER BLOCK_SPLITER
std::cout << "Returned result count: " << topk_query_result.size() << std::endl; std::cout << "Returned result count: " << topk_query_result.size() << std::endl;
if (topk_query_result.size() != search_record_array.size()) { if (topk_query_result.size() != entity_array.size()) {
std::cout << "ERROR: Returned result count not equal nq" << std::endl; std::cout << "ERROR: Returned result count not equal nq" << std::endl;
return; return;
} }
@ -158,8 +158,8 @@ Utils::PrintSearchResult(const std::vector<std::pair<int64_t, milvus::RowRecord>
for (size_t i = 0; i < topk_query_result.size(); i++) { for (size_t i = 0; i < topk_query_result.size(); i++) {
const milvus::QueryResult& one_result = topk_query_result[i]; const milvus::QueryResult& one_result = topk_query_result[i];
size_t topk = one_result.ids.size(); size_t topk = one_result.ids.size();
auto search_id = search_record_array[i].first; auto search_id = entity_array[i].first;
std::cout << "No." << i << " vector " << search_id << " top " << topk << " search result:" << std::endl; std::cout << "No." << i << " entity " << search_id << " top " << topk << " search result:" << std::endl;
for (size_t j = 0; j < topk; j++) { for (size_t j = 0; j < topk; j++) {
std::cout << "\t" << one_result.ids[j] << "\t" << one_result.distances[j] << std::endl; std::cout << "\t" << one_result.ids[j] << "\t" << one_result.distances[j] << std::endl;
} }
@ -168,13 +168,13 @@ Utils::PrintSearchResult(const std::vector<std::pair<int64_t, milvus::RowRecord>
} }
void void
Utils::CheckSearchResult(const std::vector<std::pair<int64_t, milvus::RowRecord>>& search_record_array, Utils::CheckSearchResult(const std::vector<std::pair<int64_t, milvus::Entity>>& entity_array,
const milvus::TopKQueryResult& topk_query_result) { const milvus::TopKQueryResult& topk_query_result) {
BLOCK_SPLITER BLOCK_SPLITER
size_t nq = topk_query_result.size(); size_t nq = topk_query_result.size();
for (size_t i = 0; i < nq; i++) { for (size_t i = 0; i < nq; i++) {
const milvus::QueryResult& one_result = topk_query_result[i]; const milvus::QueryResult& one_result = topk_query_result[i];
auto search_id = search_record_array[i].first; auto search_id = entity_array[i].first;
uint64_t match_index = one_result.ids.size(); uint64_t match_index = one_result.ids.size();
for (uint64_t index = 0; index < one_result.ids.size(); index++) { for (uint64_t index = 0; index < one_result.ids.size(); index++) {
@ -195,15 +195,15 @@ Utils::CheckSearchResult(const std::vector<std::pair<int64_t, milvus::RowRecord>
} }
void void
Utils::DoSearch(std::shared_ptr<milvus::Connection> conn, const std::string& table_name, Utils::DoSearch(std::shared_ptr<milvus::Connection> conn, const std::string& collection_name,
const std::vector<std::string>& partition_tags, int64_t top_k, int64_t nprobe, const std::vector<std::string>& partition_tags, int64_t top_k, int64_t nprobe,
const std::vector<std::pair<int64_t, milvus::RowRecord>>& search_record_array, const std::vector<std::pair<int64_t, milvus::Entity>>& entity_array,
milvus::TopKQueryResult& topk_query_result) { milvus::TopKQueryResult& topk_query_result) {
topk_query_result.clear(); topk_query_result.clear();
std::vector<milvus::RowRecord> record_array; std::vector<milvus::Entity> temp_entity_array;
for (auto& pair : search_record_array) { for (auto& pair : entity_array) {
record_array.push_back(pair.second); temp_entity_array.push_back(pair.second);
} }
{ {
@ -211,28 +211,33 @@ Utils::DoSearch(std::shared_ptr<milvus::Connection> conn, const std::string& tab
JSON json_params = {{"nprobe", nprobe}}; JSON json_params = {{"nprobe", nprobe}};
milvus_sdk::TimeRecorder rc("search"); milvus_sdk::TimeRecorder rc("search");
milvus::Status stat = milvus::Status stat =
conn->Search(table_name, partition_tags, record_array, top_k, json_params.dump(), topk_query_result); conn->Search(collection_name,
std::cout << "SearchVector function call status: " << stat.message() << std::endl; partition_tags,
temp_entity_array,
top_k,
json_params.dump(),
topk_query_result);
std::cout << "Search function call status: " << stat.message() << std::endl;
BLOCK_SPLITER BLOCK_SPLITER
} }
PrintSearchResult(search_record_array, topk_query_result); PrintSearchResult(entity_array, topk_query_result);
CheckSearchResult(search_record_array, topk_query_result); CheckSearchResult(entity_array, topk_query_result);
} }
void void
PrintPartitionStat(const milvus::PartitionStat& partition_stat) { PrintPartitionStat(const milvus::PartitionStat& partition_stat) {
std::cout << "\tPartition " << partition_stat.tag << " row count: " << partition_stat.row_count << std::endl; std::cout << "\tPartition " << partition_stat.tag << " entity count: " << partition_stat.row_count << std::endl;
for (auto& seg_stat : partition_stat.segments_stat) { for (auto& seg_stat : partition_stat.segments_stat) {
std::cout << "\t\tsegment " << seg_stat.segment_name << " row count: " << seg_stat.row_count std::cout << "\t\tsegment " << seg_stat.segment_name << " entity count: " << seg_stat.row_count
<< " index: " << seg_stat.index_name << " data size: " << seg_stat.data_size << std::endl; << " index: " << seg_stat.index_name << " data size: " << seg_stat.data_size << std::endl;
} }
} }
void void
Utils::PrintTableInfo(const milvus::TableInfo& info) { Utils::PrintCollectionInfo(const milvus::CollectionInfo& info) {
BLOCK_SPLITER BLOCK_SPLITER
std::cout << "Table " << " total row count: " << info.total_row_count << std::endl; std::cout << "Collection " << " total entity count: " << info.total_row_count << std::endl;
for (const milvus::PartitionStat& partition_stat : info.partitions_stat) { for (const milvus::PartitionStat& partition_stat : info.partitions_stat) {
PrintPartitionStat(partition_stat); PrintPartitionStat(partition_stat);
} }

18
sdk/examples/utils/Utils.h
View File

@ -32,7 +32,7 @@ class Utils {
CurrentTmDate(int64_t offset_day = 0); CurrentTmDate(int64_t offset_day = 0);
static const std::string& static const std::string&
GenTableName(); GenCollectionName();
static void static void
Sleep(int seconds); Sleep(int seconds);
@ -44,7 +44,7 @@ class Utils {
IndexTypeName(const milvus::IndexType& index_type); IndexTypeName(const milvus::IndexType& index_type);
static void static void
PrintTableSchema(const milvus::TableSchema& tb_schema); PrintCollectionParam(const milvus::CollectionParam& collection_param);
static void static void
PrintPartitionParam(const milvus::PartitionParam& partition_param); PrintPartitionParam(const milvus::PartitionParam& partition_param);
@ -53,25 +53,25 @@ class Utils {
PrintIndexParam(const milvus::IndexParam& index_param); PrintIndexParam(const milvus::IndexParam& index_param);
static void static void
BuildVectors(int64_t from, int64_t to, std::vector<milvus::RowRecord>& vector_record_array, BuildEntities(int64_t from, int64_t to, std::vector<milvus::Entity>& entity_array,
std::vector<int64_t>& record_ids, int64_t dimension); std::vector<int64_t>& entity_ids, int64_t dimension);
static void static void
PrintSearchResult(const std::vector<std::pair<int64_t, milvus::RowRecord>>& search_record_array, PrintSearchResult(const std::vector<std::pair<int64_t, milvus::Entity>>& entity_array,
const milvus::TopKQueryResult& topk_query_result); const milvus::TopKQueryResult& topk_query_result);
static void static void
CheckSearchResult(const std::vector<std::pair<int64_t, milvus::RowRecord>>& search_record_array, CheckSearchResult(const std::vector<std::pair<int64_t, milvus::Entity>>& entity_array,
const milvus::TopKQueryResult& topk_query_result); const milvus::TopKQueryResult& topk_query_result);
static void static void
DoSearch(std::shared_ptr<milvus::Connection> conn, const std::string& table_name, DoSearch(std::shared_ptr<milvus::Connection> conn, const std::string& collection_name,
const std::vector<std::string>& partition_tags, int64_t top_k, int64_t nprobe, const std::vector<std::string>& partition_tags, int64_t top_k, int64_t nprobe,
const std::vector<std::pair<int64_t, milvus::RowRecord>>& search_record_array, const std::vector<std::pair<int64_t, milvus::Entity>>& entity_array,
milvus::TopKQueryResult& topk_query_result); milvus::TopKQueryResult& topk_query_result);
static void static void
PrintTableInfo(const milvus::TableInfo& info); PrintCollectionInfo(const milvus::CollectionInfo& collection_info);
}; };
} // namespace milvus_sdk } // namespace milvus_sdk

766
sdk/grpc/ClientProxy.cpp
View File

@ -31,12 +31,12 @@ UriCheck(const std::string& uri) {
template<typename T> template<typename T>
void void
ConstructSearchParam(const std::string& table_name, ConstructSearchParam(const std::string& collection_name,
const std::vector<std::string>& partition_tag_array, const std::vector<std::string>& partition_tag_array,
int64_t topk, int64_t topk,
const std::string& extra_params, const std::string& extra_params,
T& search_param) { T& search_param) {
search_param.set_table_name(table_name); search_param.set_table_name(collection_name);
search_param.set_topk(topk); search_param.set_topk(topk);
milvus::grpc::KeyValuePair* kv = search_param.add_extra_params(); milvus::grpc::KeyValuePair* kv = search_param.add_extra_params();
kv->set_key(EXTRA_PARAM_KEY); kv->set_key(EXTRA_PARAM_KEY);
@ -48,7 +48,7 @@ ConstructSearchParam(const std::string& table_name,
} }
void void
CopyRowRecord(::milvus::grpc::RowRecord* target, const RowRecord& src) { CopyRowRecord(::milvus::grpc::RowRecord* target, const Entity& src) {
if (!src.float_data.empty()) { if (!src.float_data.empty()) {
auto vector_data = target->mutable_float_data(); auto vector_data = target->mutable_float_data();
vector_data->Resize(static_cast<int>(src.float_data.size()), 0.0); vector_data->Resize(static_cast<int>(src.float_data.size()), 0.0);
@ -86,7 +86,7 @@ ClientProxy::Connect(const ConnectParam& param) {
return Status::OK(); return Status::OK();
} }
std::string reason = "connect Failed!"; std::string reason = "Connect failed!";
connected_ = false; connected_ = false;
return Status(StatusCode::NotConnected, reason); return Status(StatusCode::NotConnected, reason);
} }
@ -111,7 +111,7 @@ ClientProxy::Connected() const {
std::string info; std::string info;
return client_ptr_->Cmd("", info); return client_ptr_->Cmd("", info);
} catch (std::exception& ex) { } catch (std::exception& ex) {
return Status(StatusCode::NotConnected, "connection lost: " + std::string(ex.what())); return Status(StatusCode::NotConnected, "Connection lost: " + std::string(ex.what()));
} }
} }
@ -132,262 +132,6 @@ ClientProxy::ClientVersion() const {
return MILVUS_SDK_VERSION; return MILVUS_SDK_VERSION;
} }
Status
ClientProxy::CreateTable(const TableSchema& param) {
try {
::milvus::grpc::TableSchema schema;
schema.set_table_name(param.table_name);
schema.set_dimension(param.dimension);
schema.set_index_file_size(param.index_file_size);
schema.set_metric_type(static_cast<int32_t>(param.metric_type));
return client_ptr_->CreateTable(schema);
} catch (std::exception& ex) {
return Status(StatusCode::UnknownError, "Failed to create table: " + std::string(ex.what()));
}
}
bool
ClientProxy::HasTable(const std::string& table_name) {
Status status = Status::OK();
::milvus::grpc::TableName grpc_table_name;
grpc_table_name.set_table_name(table_name);
bool result = client_ptr_->HasTable(grpc_table_name, status);
return result;
}
Status
ClientProxy::DropTable(const std::string& table_name) {
try {
::milvus::grpc::TableName grpc_table_name;
grpc_table_name.set_table_name(table_name);
return client_ptr_->DropTable(grpc_table_name);
} catch (std::exception& ex) {
return Status(StatusCode::UnknownError, "Failed to drop table: " + std::string(ex.what()));
}
}
Status
ClientProxy::CreateIndex(const IndexParam& index_param) {
try {
::milvus::grpc::IndexParam grpc_index_param;
grpc_index_param.set_table_name(index_param.table_name);
grpc_index_param.set_index_type(static_cast<int32_t>(index_param.index_type));
milvus::grpc::KeyValuePair* kv = grpc_index_param.add_extra_params();
kv->set_key(EXTRA_PARAM_KEY);
kv->set_value(index_param.extra_params);
return client_ptr_->CreateIndex(grpc_index_param);
} catch (std::exception& ex) {
return Status(StatusCode::UnknownError, "Failed to build index: " + std::string(ex.what()));
}
}
Status
ClientProxy::Insert(const std::string& table_name, const std::string& partition_tag,
const std::vector<RowRecord>& record_array, std::vector<int64_t>& id_array) {
Status status = Status::OK();
try {
::milvus::grpc::InsertParam insert_param;
insert_param.set_table_name(table_name);
insert_param.set_partition_tag(partition_tag);
for (auto& record : record_array) {
::milvus::grpc::RowRecord* grpc_record = insert_param.add_row_record_array();
CopyRowRecord(grpc_record, record);
}
// Single thread
::milvus::grpc::VectorIds vector_ids;
if (!id_array.empty()) {
/* set user's ids */
auto row_ids = insert_param.mutable_row_id_array();
row_ids->Resize(static_cast<int>(id_array.size()), -1);
memcpy(row_ids->mutable_data(), id_array.data(), id_array.size() * sizeof(int64_t));
status = client_ptr_->Insert(insert_param, vector_ids);
} else {
status = client_ptr_->Insert(insert_param, vector_ids);
/* return Milvus generated ids back to user */
id_array.insert(id_array.end(), vector_ids.vector_id_array().begin(), vector_ids.vector_id_array().end());
}
} catch (std::exception& ex) {
return Status(StatusCode::UnknownError, "Failed to add vector: " + std::string(ex.what()));
}
return status;
}
Status
ClientProxy::GetVectorByID(const std::string& table_name, int64_t vector_id, RowRecord& vector_data) {
try {
::milvus::grpc::VectorIdentity vector_identity;
vector_identity.set_table_name(table_name);
vector_identity.set_id(vector_id);
::milvus::grpc::VectorData grpc_data;
Status status = client_ptr_->GetVectorByID(vector_identity, grpc_data);
if (!status.ok()) {
return status;
}
int float_size = grpc_data.vector_data().float_data_size();
if (float_size > 0) {
vector_data.float_data.resize(float_size);
memcpy(vector_data.float_data.data(), grpc_data.vector_data().float_data().data(),
float_size * sizeof(float));
}
auto byte_size = grpc_data.vector_data().binary_data().length();
if (byte_size > 0) {
vector_data.binary_data.resize(byte_size);
memcpy(vector_data.binary_data.data(), grpc_data.vector_data().binary_data().data(), byte_size);
}
return status;
} catch (std::exception& ex) {
return Status(StatusCode::UnknownError, "Failed to get vector by id: " + std::string(ex.what()));
}
}
Status
ClientProxy::GetIDsInSegment(const std::string& table_name, const std::string& segment_name,
std::vector<int64_t>& id_array) {
try {
::milvus::grpc::GetVectorIDsParam param;
param.set_table_name(table_name);
param.set_segment_name(segment_name);
::milvus::grpc::VectorIds vector_ids;
Status status = client_ptr_->GetIDsInSegment(param, vector_ids);
if (!status.ok()) {
return status;
}
id_array.insert(id_array.end(), vector_ids.vector_id_array().begin(), vector_ids.vector_id_array().end());
return status;
} catch (std::exception& ex) {
return Status(StatusCode::UnknownError, "Failed to get vector by id: " + std::string(ex.what()));
}
}
Status
ClientProxy::Search(const std::string& table_name, const std::vector<std::string>& partition_tag_array,
const std::vector<RowRecord>& query_record_array, int64_t topk, const std::string& extra_params,
TopKQueryResult& topk_query_result) {
try {
// step 1: convert vectors data
::milvus::grpc::SearchParam search_param;
ConstructSearchParam(table_name,
partition_tag_array,
topk,
extra_params,
search_param);
for (auto& record : query_record_array) {
::milvus::grpc::RowRecord* row_record = search_param.add_query_record_array();
CopyRowRecord(row_record, record);
}
// step 2: search vectors
::milvus::grpc::TopKQueryResult result;
Status status = client_ptr_->Search(search_param, result);
if (result.row_num() == 0) {
return status;
}
// step 3: convert result array
topk_query_result.reserve(result.row_num());
int64_t nq = result.row_num();
int64_t topk = result.ids().size() / nq;
for (int64_t i = 0; i < result.row_num(); i++) {
milvus::QueryResult one_result;
one_result.ids.resize(topk);
one_result.distances.resize(topk);
memcpy(one_result.ids.data(), result.ids().data() + topk * i, topk * sizeof(int64_t));
memcpy(one_result.distances.data(), result.distances().data() + topk * i, topk * sizeof(float));
topk_query_result.emplace_back(one_result);
}
return status;
} catch (std::exception& ex) {
return Status(StatusCode::UnknownError, "Failed to search vectors: " + std::string(ex.what()));
}
}
Status
ClientProxy::DescribeTable(const std::string& table_name, TableSchema& table_schema) {
try {
::milvus::grpc::TableSchema grpc_schema;
Status status = client_ptr_->DescribeTable(table_name, grpc_schema);
table_schema.table_name = grpc_schema.table_name();
table_schema.dimension = grpc_schema.dimension();
table_schema.index_file_size = grpc_schema.index_file_size();
table_schema.metric_type = static_cast<MetricType>(grpc_schema.metric_type());
return status;
} catch (std::exception& ex) {
return Status(StatusCode::UnknownError, "Failed to describe table: " + std::string(ex.what()));
}
}
Status
ClientProxy::CountTable(const std::string& table_name, int64_t& row_count) {
try {
Status status;
::milvus::grpc::TableName grpc_table_name;
grpc_table_name.set_table_name(table_name);
row_count = client_ptr_->CountTable(grpc_table_name, status);
return status;
} catch (std::exception& ex) {
return Status(StatusCode::UnknownError, "Failed to show tables: " + std::string(ex.what()));
}
}
Status
ClientProxy::ShowTables(std::vector<std::string>& table_array) {
try {
Status status;
milvus::grpc::TableNameList table_name_list;
status = client_ptr_->ShowTables(table_name_list);
table_array.resize(table_name_list.table_names_size());
for (uint64_t i = 0; i < table_name_list.table_names_size(); ++i) {
table_array[i] = table_name_list.table_names(i);
}
return status;
} catch (std::exception& ex) {
return Status(StatusCode::UnknownError, "Failed to show tables: " + std::string(ex.what()));
}
}
Status
ClientProxy::ShowTableInfo(const std::string& table_name, TableInfo& table_info) {
try {
Status status;
::milvus::grpc::TableName grpc_table_name;
grpc_table_name.set_table_name(table_name);
milvus::grpc::TableInfo grpc_table_info;
status = client_ptr_->ShowTableInfo(grpc_table_name, grpc_table_info);
// get native info
table_info.total_row_count = grpc_table_info.total_row_count();
// get partitions info
for (int i = 0; i < grpc_table_info.partitions_stat_size(); i++) {
auto& grpc_partition_stat = grpc_table_info.partitions_stat(i);
PartitionStat partition_stat;
ConstructPartitionStat(grpc_partition_stat, partition_stat);
table_info.partitions_stat.emplace_back(partition_stat);
}
return status;
} catch (std::exception& ex) {
return Status(StatusCode::UnknownError, "Failed to show table info: " + std::string(ex.what()));
}
}
std::string std::string
ClientProxy::ServerVersion() const { ClientProxy::ServerVersion() const {
Status status = Status::OK(); Status status = Status::OK();
@ -415,127 +159,6 @@ ClientProxy::ServerStatus() const {
} }
} }
std::string
ClientProxy::DumpTaskTables() const {
if (channel_ == nullptr) {
return "not connected to server";
}
try {
std::string dummy;
Status status = client_ptr_->Cmd("tasktable", dummy);
return dummy;
} catch (std::exception& ex) {
return "connection lost";
}
}
Status
ClientProxy::DeleteByID(const std::string& table_name, const std::vector<int64_t>& id_array) {
try {
::milvus::grpc::DeleteByIDParam delete_by_id_param;
delete_by_id_param.set_table_name(table_name);
for (auto id : id_array) {
delete_by_id_param.add_id_array(id);
}
return client_ptr_->DeleteByID(delete_by_id_param);
} catch (std::exception& ex) {
return Status(StatusCode::UnknownError, "Failed to delete by range: " + std::string(ex.what()));
}
}
Status
ClientProxy::PreloadTable(const std::string& table_name) const {
try {
::milvus::grpc::TableName grpc_table_name;
grpc_table_name.set_table_name(table_name);
Status status = client_ptr_->PreloadTable(grpc_table_name);
return status;
} catch (std::exception& ex) {
return Status(StatusCode::UnknownError, "Failed to preload tables: " + std::string(ex.what()));
}
}
Status
ClientProxy::DescribeIndex(const std::string& table_name, IndexParam& index_param) const {
try {
::milvus::grpc::TableName grpc_table_name;
grpc_table_name.set_table_name(table_name);
::milvus::grpc::IndexParam grpc_index_param;
Status status = client_ptr_->DescribeIndex(grpc_table_name, grpc_index_param);
index_param.index_type = static_cast<IndexType>(grpc_index_param.index_type());
for (int i = 0; i < grpc_index_param.extra_params_size(); i++) {
const milvus::grpc::KeyValuePair& kv = grpc_index_param.extra_params(i);
if (kv.key() == EXTRA_PARAM_KEY) {
index_param.extra_params = kv.value();
}
}
return status;
} catch (std::exception& ex) {
return Status(StatusCode::UnknownError, "Failed to describe index: " + std::string(ex.what()));
}
}
Status
ClientProxy::DropIndex(const std::string& table_name) const {
try {
::milvus::grpc::TableName grpc_table_name;
grpc_table_name.set_table_name(table_name);
Status status = client_ptr_->DropIndex(grpc_table_name);
return status;
} catch (std::exception& ex) {
return Status(StatusCode::UnknownError, "Failed to drop index: " + std::string(ex.what()));
}
}
Status
ClientProxy::CreatePartition(const PartitionParam& partition_param) {
try {
::milvus::grpc::PartitionParam grpc_partition_param;
grpc_partition_param.set_table_name(partition_param.table_name);
grpc_partition_param.set_tag(partition_param.partition_tag);
Status status = client_ptr_->CreatePartition(grpc_partition_param);
return status;
} catch (std::exception& ex) {
return Status(StatusCode::UnknownError, "Failed to create partition: " + std::string(ex.what()));
}
}
Status
ClientProxy::ShowPartitions(const std::string& table_name, PartitionTagList& partition_tag_array) const {
try {
::milvus::grpc::TableName grpc_table_name;
grpc_table_name.set_table_name(table_name);
::milvus::grpc::PartitionList grpc_partition_list;
Status status = client_ptr_->ShowPartitions(grpc_table_name, grpc_partition_list);
partition_tag_array.resize(grpc_partition_list.partition_tag_array_size());
for (uint64_t i = 0; i < grpc_partition_list.partition_tag_array_size(); ++i) {
partition_tag_array[i] = grpc_partition_list.partition_tag_array(i);
}
return status;
} catch (std::exception& ex) {
return Status(StatusCode::UnknownError, "Failed to show partitions: " + std::string(ex.what()));
}
}
Status
ClientProxy::DropPartition(const PartitionParam& partition_param) {
try {
::milvus::grpc::PartitionParam grpc_partition_param;
grpc_partition_param.set_table_name(partition_param.table_name);
grpc_partition_param.set_tag(partition_param.partition_tag);
Status status = client_ptr_->DropPartition(grpc_partition_param);
return status;
} catch (std::exception& ex) {
return Status(StatusCode::UnknownError, "Failed to drop partition: " + std::string(ex.what()));
}
}
Status Status
ClientProxy::GetConfig(const std::string& node_name, std::string& value) const { ClientProxy::GetConfig(const std::string& node_name, std::string& value) const {
try { try {
@ -556,12 +179,373 @@ ClientProxy::SetConfig(const std::string& node_name, const std::string& value) c
} }
Status Status
ClientProxy::FlushTable(const std::string& table_name) { ClientProxy::CreateCollection(const CollectionParam& param) {
try {
::milvus::grpc::TableSchema schema;
schema.set_table_name(param.collection_name);
schema.set_dimension(param.dimension);
schema.set_index_file_size(param.index_file_size);
schema.set_metric_type(static_cast<int32_t>(param.metric_type));
return client_ptr_->CreateTable(schema);
} catch (std::exception& ex) {
return Status(StatusCode::UnknownError, "Failed to create collection: " + std::string(ex.what()));
}
}
bool
ClientProxy::HasCollection(const std::string& collection_name) {
Status status = Status::OK();
::milvus::grpc::TableName grpc_collection_name;
grpc_collection_name.set_table_name(collection_name);
bool result = client_ptr_->HasTable(grpc_collection_name, status);
return result;
}
Status
ClientProxy::DropCollection(const std::string& collection_name) {
try {
::milvus::grpc::TableName grpc_collection_name;
grpc_collection_name.set_table_name(collection_name);
return client_ptr_->DropTable(grpc_collection_name);
} catch (std::exception& ex) {
return Status(StatusCode::UnknownError, "Failed to drop collection: " + std::string(ex.what()));
}
}
Status
ClientProxy::CreateIndex(const IndexParam& index_param) {
try {
::milvus::grpc::IndexParam grpc_index_param;
grpc_index_param.set_table_name(index_param.collection_name);
grpc_index_param.set_index_type(static_cast<int32_t>(index_param.index_type));
milvus::grpc::KeyValuePair* kv = grpc_index_param.add_extra_params();
kv->set_key(EXTRA_PARAM_KEY);
kv->set_value(index_param.extra_params);
return client_ptr_->CreateIndex(grpc_index_param);
} catch (std::exception& ex) {
return Status(StatusCode::UnknownError, "Failed to build index: " + std::string(ex.what()));
}
}
Status
ClientProxy::Insert(const std::string& collection_name, const std::string& partition_tag,
const std::vector<Entity>& entity_array, std::vector<int64_t>& id_array) {
Status status = Status::OK();
try {
::milvus::grpc::InsertParam insert_param;
insert_param.set_table_name(collection_name);
insert_param.set_partition_tag(partition_tag);
for (auto& entity : entity_array) {
::milvus::grpc::RowRecord* grpc_record = insert_param.add_row_record_array();
CopyRowRecord(grpc_record, entity);
}
// Single thread
::milvus::grpc::VectorIds vector_ids;
if (!id_array.empty()) {
/* set user's ids */
auto row_ids = insert_param.mutable_row_id_array();
row_ids->Resize(static_cast<int>(id_array.size()), -1);
memcpy(row_ids->mutable_data(), id_array.data(), id_array.size() * sizeof(int64_t));
status = client_ptr_->Insert(insert_param, vector_ids);
} else {
status = client_ptr_->Insert(insert_param, vector_ids);
/* return Milvus generated ids back to user */
id_array.insert(id_array.end(), vector_ids.vector_id_array().begin(), vector_ids.vector_id_array().end());
}
} catch (std::exception& ex) {
return Status(StatusCode::UnknownError, "Failed to add entities: " + std::string(ex.what()));
}
return status;
}
Status
ClientProxy::GetEntityByID(const std::string& collection_name, int64_t entity_id, Entity& entity_data) {
try {
::milvus::grpc::VectorIdentity vector_identity;
vector_identity.set_table_name(collection_name);
vector_identity.set_id(entity_id);
::milvus::grpc::VectorData grpc_data;
Status status = client_ptr_->GetVectorByID(vector_identity, grpc_data);
if (!status.ok()) {
return status;
}
int float_size = grpc_data.vector_data().float_data_size();
if (float_size > 0) {
entity_data.float_data.resize(float_size);
memcpy(entity_data.float_data.data(), grpc_data.vector_data().float_data().data(),
float_size * sizeof(float));
}
auto byte_size = grpc_data.vector_data().binary_data().length();
if (byte_size > 0) {
entity_data.binary_data.resize(byte_size);
memcpy(entity_data.binary_data.data(), grpc_data.vector_data().binary_data().data(), byte_size);
}
return status;
} catch (std::exception& ex) {
return Status(StatusCode::UnknownError, "Failed to get entity by id: " + std::string(ex.what()));
}
}
Status
ClientProxy::GetIDsInSegment(const std::string& collection_name, const std::string& segment_name,
std::vector<int64_t>& id_array) {
try {
::milvus::grpc::GetVectorIDsParam param;
param.set_table_name(collection_name);
param.set_segment_name(segment_name);
::milvus::grpc::VectorIds vector_ids;
Status status = client_ptr_->GetIDsInSegment(param, vector_ids);
if (!status.ok()) {
return status;
}
id_array.insert(id_array.end(), vector_ids.vector_id_array().begin(), vector_ids.vector_id_array().end());
return status;
} catch (std::exception& ex) {
return Status(StatusCode::UnknownError, "Failed to get ids from segment: " + std::string(ex.what()));
}
}
Status
ClientProxy::Search(const std::string& collection_name, const std::vector<std::string>& partition_tag_array,
const std::vector<Entity>& entity_array, int64_t topk, const std::string& extra_params,
TopKQueryResult& topk_query_result) {
try {
// step 1: convert vectors data
::milvus::grpc::SearchParam search_param;
ConstructSearchParam(collection_name,
partition_tag_array,
topk,
extra_params,
search_param);
for (auto& entity : entity_array) {
::milvus::grpc::RowRecord* row_record = search_param.add_query_record_array();
CopyRowRecord(row_record, entity);
}
// step 2: search vectors
::milvus::grpc::TopKQueryResult result;
Status status = client_ptr_->Search(search_param, result);
if (result.row_num() == 0) {
return status;
}
// step 3: convert result array
topk_query_result.reserve(result.row_num());
int64_t nq = result.row_num();
int64_t topk = result.ids().size() / nq;
for (int64_t i = 0; i < result.row_num(); i++) {
milvus::QueryResult one_result;
one_result.ids.resize(topk);
one_result.distances.resize(topk);
memcpy(one_result.ids.data(), result.ids().data() + topk * i, topk * sizeof(int64_t));
memcpy(one_result.distances.data(), result.distances().data() + topk * i, topk * sizeof(float));
topk_query_result.emplace_back(one_result);
}
return status;
} catch (std::exception& ex) {
return Status(StatusCode::UnknownError, "Failed to search entities: " + std::string(ex.what()));
}
}
Status
ClientProxy::DescribeCollection(const std::string& collection_name, CollectionParam& collection_param) {
try {
::milvus::grpc::TableSchema grpc_schema;
Status status = client_ptr_->DescribeTable(collection_name, grpc_schema);
collection_param.collection_name = grpc_schema.table_name();
collection_param.dimension = grpc_schema.dimension();
collection_param.index_file_size = grpc_schema.index_file_size();
collection_param.metric_type = static_cast<MetricType>(grpc_schema.metric_type());
return status;
} catch (std::exception& ex) {
return Status(StatusCode::UnknownError, "Failed to describe collection: " + std::string(ex.what()));
}
}
Status
ClientProxy::CountCollection(const std::string& collection_name, int64_t& row_count) {
try {
Status status;
::milvus::grpc::TableName grpc_collection_name;
grpc_collection_name.set_table_name(collection_name);
row_count = client_ptr_->CountTable(grpc_collection_name, status);
return status;
} catch (std::exception& ex) {
return Status(StatusCode::UnknownError, "Failed to count collection: " + std::string(ex.what()));
}
}
Status
ClientProxy::ShowCollections(std::vector<std::string>& collection_array) {
try {
Status status;
milvus::grpc::TableNameList collection_name_list;
status = client_ptr_->ShowTables(collection_name_list);
collection_array.resize(collection_name_list.table_names_size());
for (uint64_t i = 0; i < collection_name_list.table_names_size(); ++i) {
collection_array[i] = collection_name_list.table_names(i);
}
return status;
} catch (std::exception& ex) {
return Status(StatusCode::UnknownError, "Failed to show collections: " + std::string(ex.what()));
}
}
Status
ClientProxy::ShowCollectionInfo(const std::string& collection_name, CollectionInfo& collection_info) {
try {
Status status;
::milvus::grpc::TableName grpc_collection_name;
grpc_collection_name.set_table_name(collection_name);
milvus::grpc::TableInfo grpc_collection_info;
status = client_ptr_->ShowTableInfo(grpc_collection_name, grpc_collection_info);
// get native info
collection_info.total_row_count = grpc_collection_info.total_row_count();
// get partitions info
for (int i = 0; i < grpc_collection_info.partitions_stat_size(); i++) {
auto& grpc_partition_stat = grpc_collection_info.partitions_stat(i);
PartitionStat partition_stat;
ConstructPartitionStat(grpc_partition_stat, partition_stat);
collection_info.partitions_stat.emplace_back(partition_stat);
}
return status;
} catch (std::exception& ex) {
return Status(StatusCode::UnknownError, "Failed to show collection info: " + std::string(ex.what()));
}
}
Status
ClientProxy::DeleteByID(const std::string& collection_name, const std::vector<int64_t>& id_array) {
try {
::milvus::grpc::DeleteByIDParam delete_by_id_param;
delete_by_id_param.set_table_name(collection_name);
for (auto id : id_array) {
delete_by_id_param.add_id_array(id);
}
return client_ptr_->DeleteByID(delete_by_id_param);
} catch (std::exception& ex) {
return Status(StatusCode::UnknownError, "Failed to delete entity id: " + std::string(ex.what()));
}
}
Status
ClientProxy::PreloadCollection(const std::string& collection_name) const {
try {
::milvus::grpc::TableName grpc_collection_name;
grpc_collection_name.set_table_name(collection_name);
Status status = client_ptr_->PreloadTable(grpc_collection_name);
return status;
} catch (std::exception& ex) {
return Status(StatusCode::UnknownError, "Failed to preload collection: " + std::string(ex.what()));
}
}
Status
ClientProxy::DescribeIndex(const std::string& collection_name, IndexParam& index_param) const {
try {
::milvus::grpc::TableName grpc_collection_name;
grpc_collection_name.set_table_name(collection_name);
::milvus::grpc::IndexParam grpc_index_param;
Status status = client_ptr_->DescribeIndex(grpc_collection_name, grpc_index_param);
index_param.index_type = static_cast<IndexType>(grpc_index_param.index_type());
for (int i = 0; i < grpc_index_param.extra_params_size(); i++) {
const milvus::grpc::KeyValuePair& kv = grpc_index_param.extra_params(i);
if (kv.key() == EXTRA_PARAM_KEY) {
index_param.extra_params = kv.value();
}
}
return status;
} catch (std::exception& ex) {
return Status(StatusCode::UnknownError, "Failed to describe index: " + std::string(ex.what()));
}
}
Status
ClientProxy::DropIndex(const std::string& collection_name) const {
try {
::milvus::grpc::TableName grpc_collection_name;
grpc_collection_name.set_table_name(collection_name);
Status status = client_ptr_->DropIndex(grpc_collection_name);
return status;
} catch (std::exception& ex) {
return Status(StatusCode::UnknownError, "Failed to drop index: " + std::string(ex.what()));
}
}
Status
ClientProxy::CreatePartition(const PartitionParam& partition_param) {
try {
::milvus::grpc::PartitionParam grpc_partition_param;
grpc_partition_param.set_table_name(partition_param.collection_name);
grpc_partition_param.set_tag(partition_param.partition_tag);
Status status = client_ptr_->CreatePartition(grpc_partition_param);
return status;
} catch (std::exception& ex) {
return Status(StatusCode::UnknownError, "Failed to create partition: " + std::string(ex.what()));
}
}
Status
ClientProxy::ShowPartitions(const std::string& collection_name, PartitionTagList& partition_tag_array) const {
try {
::milvus::grpc::TableName grpc_collection_name;
grpc_collection_name.set_table_name(collection_name);
::milvus::grpc::PartitionList grpc_partition_list;
Status status = client_ptr_->ShowPartitions(grpc_collection_name, grpc_partition_list);
partition_tag_array.resize(grpc_partition_list.partition_tag_array_size());
for (uint64_t i = 0; i < grpc_partition_list.partition_tag_array_size(); ++i) {
partition_tag_array[i] = grpc_partition_list.partition_tag_array(i);
}
return status;
} catch (std::exception& ex) {
return Status(StatusCode::UnknownError, "Failed to show partitions: " + std::string(ex.what()));
}
}
Status
ClientProxy::DropPartition(const PartitionParam& partition_param) {
try {
::milvus::grpc::PartitionParam grpc_partition_param;
grpc_partition_param.set_table_name(partition_param.collection_name);
grpc_partition_param.set_tag(partition_param.partition_tag);
Status status = client_ptr_->DropPartition(grpc_partition_param);
return status;
} catch (std::exception& ex) {
return Status(StatusCode::UnknownError, "Failed to drop partition: " + std::string(ex.what()));
}
}
Status
ClientProxy::FlushCollection(const std::string& collection_name) {
try { try {
std::string dummy; std::string dummy;
return client_ptr_->Flush(table_name); return client_ptr_->Flush(collection_name);
} catch (std::exception& ex) { } catch (std::exception& ex) {
return Status(StatusCode::UnknownError, "Failed to flush"); return Status(StatusCode::UnknownError, "Failed to flush collection");
} }
} }
@ -571,19 +555,19 @@ ClientProxy::Flush() {
std::string dummy; std::string dummy;
return client_ptr_->Flush(""); return client_ptr_->Flush("");
} catch (std::exception& ex) { } catch (std::exception& ex) {
return Status(StatusCode::UnknownError, "Failed to flush"); return Status(StatusCode::UnknownError, "Failed to flush collections");
} }
} }
Status Status
ClientProxy::CompactTable(const std::string& table_name) { ClientProxy::CompactCollection(const std::string& collection_name) {
try { try {
::milvus::grpc::TableName grpc_table_name; ::milvus::grpc::TableName grpc_collection_name;
grpc_table_name.set_table_name(table_name); grpc_collection_name.set_table_name(collection_name);
Status status = client_ptr_->Compact(grpc_table_name); Status status = client_ptr_->Compact(grpc_collection_name);
return status; return status;
} catch (std::exception& ex) { } catch (std::exception& ex) {
return Status(StatusCode::UnknownError, "Failed to compact table: " + std::string(ex.what())); return Status(StatusCode::UnknownError, "Failed to compact collection: " + std::string(ex.what()));
} }
} }

133
sdk/grpc/ClientProxy.h
View File

@ -24,7 +24,7 @@ class ClientProxy : public Connection {
public: public:
// Implementations of the Connection interface // Implementations of the Connection interface
Status Status
Connect(const ConnectParam& param) override; Connect(const ConnectParam& connect_param) override;
Status Status
Connect(const std::string& uri) override; Connect(const std::string& uri) override;
@ -35,46 +35,6 @@ class ClientProxy : public Connection {
Status Status
Disconnect() override; Disconnect() override;
Status
CreateTable(const TableSchema& param) override;
bool
HasTable(const std::string& table_name) override;
Status
DropTable(const std::string& table_name) override;
Status
CreateIndex(const IndexParam& index_param) override;
Status
Insert(const std::string& table_name, const std::string& partition_tag, const std::vector<RowRecord>& record_array,
std::vector<int64_t>& id_array) override;
Status
GetVectorByID(const std::string& table_name, int64_t vector_id, RowRecord& vector_data) override;
Status
GetIDsInSegment(const std::string& table_name, const std::string& segment_name,
std::vector<int64_t>& id_array) override;
Status
Search(const std::string& table_name, const std::vector<std::string>& partition_tag_array,
const std::vector<RowRecord>& query_record_array, int64_t topk, const std::string& extra_params,
TopKQueryResult& topk_query_result) override;
Status
DescribeTable(const std::string& table_name, TableSchema& table_schema) override;
Status
CountTable(const std::string& table_name, int64_t& row_count) override;
Status
ShowTables(std::vector<std::string>& table_array) override;
Status
ShowTableInfo(const std::string& table_name, TableInfo& table_info) override;
std::string std::string
ClientVersion() const override; ClientVersion() const override;
@ -84,30 +44,6 @@ class ClientProxy : public Connection {
std::string std::string
ServerStatus() const override; ServerStatus() const override;
std::string
DumpTaskTables() const override;
Status
DeleteByID(const std::string& table_name, const std::vector<int64_t>& id_array) override;
Status
PreloadTable(const std::string& table_name) const override;
Status
DescribeIndex(const std::string& table_name, IndexParam& index_param) const override;
Status
DropIndex(const std::string& table_name) const override;
Status
CreatePartition(const PartitionParam& partition_param) override;
Status
ShowPartitions(const std::string& table_name, PartitionTagList& partition_tag_array) const override;
Status
DropPartition(const PartitionParam& partition_param) override;
Status Status
GetConfig(const std::string& node_name, std::string& value) const override; GetConfig(const std::string& node_name, std::string& value) const override;
@ -115,13 +51,76 @@ class ClientProxy : public Connection {
SetConfig(const std::string& node_name, const std::string& value) const override; SetConfig(const std::string& node_name, const std::string& value) const override;
Status Status
FlushTable(const std::string& table_name) override; CreateCollection(const CollectionParam& param) override;
bool
HasCollection(const std::string& collection_name) override;
Status
DropCollection(const std::string& collection_name) override;
Status
CreateIndex(const IndexParam& index_param) override;
Status
Insert(const std::string& collection_name,
const std::string& partition_tag,
const std::vector<Entity>& entity_array,
std::vector<int64_t>& id_array) override;
Status
GetEntityByID(const std::string& collection_name, int64_t entity_id, Entity& entity_data) override;
Status
GetIDsInSegment(const std::string& collection_name, const std::string& segment_name,
std::vector<int64_t>& id_array) override;
Status
Search(const std::string& collection_name, const std::vector<std::string>& partition_tag_array,
const std::vector<Entity>& entity_array, int64_t topk,
const std::string& extra_params, TopKQueryResult& topk_query_result) override;
Status
DescribeCollection(const std::string& collection_name, CollectionParam& collection_schema) override;
Status
CountCollection(const std::string& collection_name, int64_t& entity_count) override;
Status
ShowCollections(std::vector<std::string>& collection_array) override;
Status
ShowCollectionInfo(const std::string& collection_name, CollectionInfo& collection_info) override;
Status
DeleteByID(const std::string& collection_name, const std::vector<int64_t>& id_array) override;
Status
PreloadCollection(const std::string& collection_name) const override;
Status
DescribeIndex(const std::string& collection_name, IndexParam& index_param) const override;
Status
DropIndex(const std::string& collection_name) const override;
Status
CreatePartition(const PartitionParam& partition_param) override;
Status
ShowPartitions(const std::string& collection_name, PartitionTagList& partition_tag_array) const override;
Status
DropPartition(const PartitionParam& partition_param) override;
Status
FlushCollection(const std::string& collection_name) override;
Status Status
Flush() override; Flush() override;
Status Status
CompactTable(const std::string& table_name) override; CompactCollection(const std::string& collection_name) override;
private: private:
std::shared_ptr<::grpc::Channel> channel_; std::shared_ptr<::grpc::Channel> channel_;

618
sdk/include/MilvusApi.h
View File

@ -54,20 +54,19 @@ struct ConnectParam {
}; };
/** /**
* @brief Table Schema * @brief Collection parameters
*/ */
struct TableSchema { struct CollectionParam {
std::string table_name; ///< Table name std::string collection_name; ///< Collection_name name
int64_t dimension = 0; ///< Vector dimension, must be a positive value int64_t dimension = 0; ///< Vector dimension, must be a positive value
int64_t index_file_size = 1024; ///< Index file size, must be a positive value, unit: MB int64_t index_file_size = 1024; ///< Index file size, must be a positive value, unit: MB
MetricType metric_type = MetricType::L2; ///< Index metric type MetricType metric_type = MetricType::L2; ///< Index metric type
}; };
/** /**
* @brief Record inserted * @brief Entity inserted, currently each entity represent a vector
*/ */
struct RowRecord { struct Entity {
std::vector<float> float_data; ///< Vector raw float data std::vector<float> float_data; ///< Vector raw float data
std::vector<uint8_t> binary_data; ///< Vector raw binary data std::vector<uint8_t> binary_data; ///< Vector raw binary data
}; };
@ -76,7 +75,7 @@ struct RowRecord {
* @brief TopK query result * @brief TopK query result
*/ */
struct QueryResult { struct QueryResult {
std::vector<int64_t> ids; ///< Query ids result std::vector<int64_t> ids; ///< Query entity ids result
std::vector<float> distances; ///< Query distances result std::vector<float> distances; ///< Query distances result
}; };
using TopKQueryResult = std::vector<QueryResult>; ///< Topk query result using TopKQueryResult = std::vector<QueryResult>; ///< Topk query result
@ -86,12 +85,21 @@ using TopKQueryResult = std::vector<QueryResult>; ///< Topk query result
* Note: extra_params is extra parameters list, it must be json format * Note: extra_params is extra parameters list, it must be json format
* For different index type, parameter list is different accordingly, for example: * For different index type, parameter list is different accordingly, for example:
* FLAT/IVFLAT/SQ8: "{nlist: '16384'}" * FLAT/IVFLAT/SQ8: "{nlist: '16384'}"
* IVFPQ: "{nlist: '16384', nbits: "16"}" * ///< nlist range:[1, 999999]
* NSG: "{search_length: '100', out_degree:'40', pool_size:'66'}" * IVFPQ: "{nlist: '16384', m: "12"}"
* HNSW "{M: '16', ef_construct:'500'}" * ///< nlist range:[1, 999999]
* ///< m is decided by dim and have a couple of results.
* NSG: "{search_length: '45', out_degree:'50', candidate_pool_size:'300', "knng":'100'}"
* ///< search_length range:[10, 300]
* ///< out_degree range:[5, 300]
* ///< candidate_pool_size range:[50, 1000]
* ///< knng range:[5, 300]
* HNSW "{M: '16', efConstruction:'500'}"
* ///< M range:[5, 48]
* ///< efConstruction range:[topk, 4096]
*/ */
struct IndexParam { struct IndexParam {
std::string table_name; ///< Table name for create index std::string collection_name; ///< Collection name for create index
IndexType index_type; ///< Index type IndexType index_type; ///< Index type
std::string extra_params; ///< Extra parameters according to different index type, must be json format std::string extra_params; ///< Extra parameters according to different index type, must be json format
}; };
@ -100,7 +108,7 @@ struct IndexParam {
* @brief partition parameters * @brief partition parameters
*/ */
struct PartitionParam { struct PartitionParam {
std::string table_name; std::string collection_name;
std::string partition_tag; std::string partition_tag;
}; };
@ -126,11 +134,11 @@ struct PartitionStat {
}; };
/** /**
* @brief table info * @brief collection info
*/ */
struct TableInfo { struct CollectionInfo {
int64_t total_row_count; ///< Table total row count int64_t total_row_count; ///< Collection total entity count
std::vector<PartitionStat> partitions_stat; ///< Table's partitions statistics std::vector<PartitionStat> partitions_stat; ///< Collection's partitions statistics
}; };
/** /**
@ -139,18 +147,18 @@ struct TableInfo {
class Connection { class Connection {
public: public:
/** /**
* @brief CreateConnection * @brief Create connection
* *
* Create a connection instance and return it's shared pointer * Create a connection instance and return it's shared pointer
* *
* @return Connection instance pointer * @return connection instance pointer
*/ */
static std::shared_ptr<Connection> static std::shared_ptr<Connection>
Create(); Create();
/** /**
* @brief DestroyConnection * @brief Destroy connection
* *
* Destroy the connection instance * Destroy the connection instance
* *
@ -174,7 +182,7 @@ class Connection {
*/ */
virtual Status virtual Status
Connect(const ConnectParam& param) = 0; Connect(const ConnectParam& connect_param) = 0;
/** /**
* @brief Connect * @brief Connect
@ -190,7 +198,7 @@ class Connection {
Connect(const std::string& uri) = 0; Connect(const std::string& uri) = 0;
/** /**
* @brief connected * @brief Connected
* *
* This method is used to test whether server is connected. * This method is used to test whether server is connected.
* *
@ -210,183 +218,7 @@ class Connection {
Disconnect() = 0; Disconnect() = 0;
/** /**
* @brief Create table method * @brief Get the client version
*
* This method is used to create table.
*
* @param param, use to provide table information to be created.
*
* @return Indicate if table is created successfully
*/
virtual Status
CreateTable(const TableSchema& param) = 0;
/**
* @brief Test table existence method
*
* This method is used to create table.
*
* @param table_name, target table's name.
*
* @return Indicate if table is cexist
*/
virtual bool
HasTable(const std::string& table_name) = 0;
/**
* @brief Drop table method
*
* This method is used to drop table(and its partitions).
*
* @param table_name, target table's name.
*
* @return Indicate if table is drop successfully.
*/
virtual Status
DropTable(const std::string& table_name) = 0;
/**
* @brief Create index method
*
* This method is used to create index for whole table(and its partitions).
*
* @param IndexParam
* table_name, table name is going to be create index.
* index type,
* nlist,
* index file size
*
* @return Indicate if build index successfully.
*/
virtual Status
CreateIndex(const IndexParam& index_param) = 0;
/**
* @brief Insert vector to table
*
* This method is used to insert vector array to table.
*
* @param table_name, target table's name.
* @param partition_tag, target partition's tag, keep empty if no partition specified.
* @param record_array, vector array is inserted.
* @param id_array,
* specify id for each vector,
* if this array is empty, milvus will generate unique id for each vector,
* and return all ids by this parameter.
*
* @return Indicate if vector array are inserted successfully
*/
virtual Status
Insert(const std::string& table_name, const std::string& partition_tag, const std::vector<RowRecord>& record_array,
std::vector<int64_t>& id_array) = 0;
/**
* @brief Get vector data by id
*
* This method is used to get vector data by id from a table.
* Return the first found vector if there are vectors with duplicated id
*
* @param table_name, target table's name.
* @param vector_id, target vector id.
* @param vector_data, returned vector data.
*
* @return Indicate if the operation is succeed.
*/
virtual Status
GetVectorByID(const std::string& table_name, int64_t vector_id, RowRecord& vector_data) = 0;
/**
* @brief Get vector ids from a segment
*
* This method is used to get vector ids from a segment
* Return all vector(not deleted) ids
*
* @param table_name, target table's name.
* @param segment_name, target segment name.
* @param id_array, returned vector id array.
*
* @return Indicate if the operation is succeed.
*/
virtual Status
GetIDsInSegment(const std::string& table_name, const std::string& segment_name, std::vector<int64_t>& id_array) = 0;
/**
* @brief Search vector
*
* This method is used to query vector in table.
*
* @param table_name, target table's name.
* @param partition_tag_array, target partitions, keep empty if no partition specified.
* @param query_record_array, vectors to be queried.
* @param topk, how many similarity vectors will be returned.
* @param extra_params, extra search parameters according to different index type, must be json format.
* Note: extra_params is extra parameters list, it must be json format, for example:
* For different index type, parameter list is different accordingly
* FLAT/IVFLAT/SQ8/IVFPQ: "{nprobe: '32'}"
* NSG: "{search_length:'100'}
* HNSW "{ef: '64'}
* @param topk_query_result, result array.
*
* @return Indicate if query is successful.
*/
virtual Status
Search(const std::string& table_name, const PartitionTagList& partition_tag_array,
const std::vector<RowRecord>& query_record_array, int64_t topk,
const std::string& extra_params, TopKQueryResult& topk_query_result) = 0;
/**
* @brief Show table description
*
* This method is used to show table information.
*
* @param table_name, target table's name.
* @param table_schema, table_schema is given when operation is successful.
*
* @return Indicate if this operation is successful.
*/
virtual Status
DescribeTable(const std::string& table_name, TableSchema& table_schema) = 0;
/**
* @brief Get table row count
*
* This method is used to get table row count.
*
* @param table_name, target table's name.
* @param row_count, table total row count(including partitions).
*
* @return Indicate if this operation is successful.
*/
virtual Status
CountTable(const std::string& table_name, int64_t& row_count) = 0;
/**
* @brief Show all tables in database
*
* This method is used to list all tables.
*
* @param table_array, all tables in database.
*
* @return Indicate if this operation is successful.
*/
virtual Status
ShowTables(std::vector<std::string>& table_array) = 0;
/**
* @brief Show table information
*
* This method is used to get detail information of a table.
*
* @param table_name, target table's name.
* @param table_info, target table's information
*
* @return Indicate if this operation is successful.
*/
virtual Status
ShowTableInfo(const std::string& table_name, TableInfo& table_info) = 0;
/**
* @brief Give the client version
* *
* This method is used to give the client version. * This method is used to give the client version.
* *
@ -396,7 +228,7 @@ class Connection {
ClientVersion() const = 0; ClientVersion() const = 0;
/** /**
* @brief Give the server version * @brief Get the server version
* *
* This method is used to give the server version. * This method is used to give the server version.
* *
@ -406,7 +238,7 @@ class Connection {
ServerVersion() const = 0; ServerVersion() const = 0;
/** /**
* @brief Give the server status * @brief Get the server status
* *
* This method is used to give the server status. * This method is used to give the server status.
* *
@ -415,106 +247,6 @@ class Connection {
virtual std::string virtual std::string
ServerStatus() const = 0; ServerStatus() const = 0;
/**
* @brief dump server tasks information
*
* This method is internal used.
*
* @return Task information in tasktables.
*/
virtual std::string
DumpTaskTables() const = 0;
/**
* [deprecated]
* @brief delete tables by vector id
*
* This method is used to delete table data by date range.
*
* @param table_name, target table's name.
* @param id_array, vector ids to deleted.
*
* @return Indicate if this operation is successful.
*/
virtual Status
DeleteByID(const std::string& table_name, const std::vector<int64_t>& id_array) = 0;
/**
* @brief preload table
*
* This method is used to preload table
*
* @param table_name
*
* @return Indicate if this operation is successful.
*/
virtual Status
PreloadTable(const std::string& table_name) const = 0;
/**
* @brief describe index
*
* This method is used to describe index
*
* @param table_name, target table's name.
* @param index_param, returned index information.
*
* @return Indicate if this operation is successful.
*/
virtual Status
DescribeIndex(const std::string& table_name, IndexParam& index_param) const = 0;
/**
* @brief drop index
*
* This method is used to drop index of table(and its partitions)
*
* @param table_name, target table's name.
*
* @return Indicate if this operation is successful.
*/
virtual Status
DropIndex(const std::string& table_name) const = 0;
/**
* @brief Create partition method
*
* This method is used to create table partition
*
* @param param, use to provide partition information to be created.
*
* @return Indicate if partition is created successfully
*/
virtual Status
CreatePartition(const PartitionParam& param) = 0;
/**
* @brief Test table existence method
*
* This method is used to create table
*
* @param table_name, table name is going to be tested.
* @param partition_tag_array, partition tag array of the table.
*
* @return Indicate if this operation is successful
*/
virtual Status
ShowPartitions(const std::string& table_name, PartitionTagList& partition_tag_array) const = 0;
/**
* @brief Delete partition method
*
* This method is used to delete table partition.
*
* @param param, target partition to be deleted.
* NOTE: if param.table_name is empty, you must specify param.partition_name,
* else you can specify param.table_name and param.tag and let the param.partition_name be empty
*
* @return Indicate if partition is delete successfully.
*/
virtual Status
DropPartition(const PartitionParam& param) = 0;
/** /**
* @brief Get config method * @brief Get config method
* *
@ -542,21 +274,287 @@ class Connection {
SetConfig(const std::string& node_name, const std::string& value) const = 0; SetConfig(const std::string& node_name, const std::string& value) const = 0;
/** /**
* @brief flush table buffer into storage * @brief Create collection method
* *
* This method is used to flush table buffer into storage * This method is used to create collection.
* *
* @param table_name, target table's name. * @param param, use to provide collection information to be created.
*
* @return Indicate if collection is created successfully
*/
virtual Status
CreateCollection(const CollectionParam& param) = 0;
/**
* @brief Test collection existence method
*
* This method is used to test collection existence.
*
* @param collection_name, target collection's name.
*
* @return Indicate if collection is cexist
*/
virtual bool
HasCollection(const std::string& collection_name) = 0;
/**
* @brief Drop collection method
*
* This method is used to drop collection(and its partitions).
*
* @param collection_name, target collection's name.
*
* @return Indicate if collection is drop successfully.
*/
virtual Status
DropCollection(const std::string& collection_name) = 0;
/**
* @brief Create index method
*
* This method is used to create index for whole collection(and its partitions).
*
* @param index_param, use to provide index information to be created.
*
* @return Indicate if create index successfully.
*/
virtual Status
CreateIndex(const IndexParam& index_param) = 0;
/**
* @brief Insert entity to collection
*
* This method is used to insert vector array to collection.
*
* @param collection_name, target collection's name.
* @param partition_tag, target partition's tag, keep empty if no partition specified.
* @param entity_array, entity array is inserted, each entitu represent a vector.
* @param id_array,
* specify id for each entity,
* if this array is empty, milvus will generate unique id for each entity,
* and return all ids by this parameter.
*
* @return Indicate if entity array are inserted successfully
*/
virtual Status
Insert(const std::string& collection_name,
const std::string& partition_tag,
const std::vector<Entity>& entity_array,
std::vector<int64_t>& id_array) = 0;
/**
* @brief Get entity data by id
*
* This method is used to get entity data by id from a collection.
* Return the first found entity if there are entities with duplicated id
*
* @param collection_name, target collection's name.
* @param entity_id, target entity id.
* @param entity_data, returned entity data.
*
* @return Indicate if the operation is succeed.
*/
virtual Status
GetEntityByID(const std::string& collection_name, int64_t entity_id, Entity& entity_data) = 0;
/**
* @brief Get entity ids from a segment
*
* This method is used to get entity ids from a segment
* Return all entity(not deleted) ids
*
* @param collection_name, target collection's name.
* @param segment_name, target segment name.
* @param id_array, returned entity id array.
*
* @return Indicate if the operation is succeed.
*/
virtual Status
GetIDsInSegment(const std::string& collection_name,
const std::string& segment_name,
std::vector<int64_t>& id_array) = 0;
/**
* @brief Search entities in a collection
*
* This method is used to query entity in collection.
*
* @param collection_name, target collection's name.
* @param partition_tag_array, target partitions, keep empty if no partition specified.
* @param query_entity_array, vectors to be queried.
* @param topk, how many similarity entities will be returned.
* @param extra_params, extra search parameters according to different index type, must be json format.
* Note: extra_params is extra parameters list, it must be json format, for example:
* For different index type, parameter list is different accordingly
* FLAT/IVFLAT/SQ8/IVFPQ: "{nprobe: '32'}"
* ///< nprobe range:[1,999999]
* NSG: "{search_length:'100'}
* ///< search_length range:[10, 300]
* HNSW "{ef: '64'}
* ///< ef range:[k, 4096]
* @param topk_query_result, result array.
*
* @return Indicate if query is successful.
*/
virtual Status
Search(const std::string& collection_name, const PartitionTagList& partition_tag_array,
const std::vector<Entity>& entity_array, int64_t topk,
const std::string& extra_params, TopKQueryResult& topk_query_result) = 0;
/**
* @brief Show collection description
*
* This method is used to show collection information.
*
* @param collection_name, target collection's name.
* @param collection_param, collection_param is given when operation is successful.
* *
* @return Indicate if this operation is successful. * @return Indicate if this operation is successful.
*/ */
virtual Status virtual Status
FlushTable(const std::string& table_name) = 0; DescribeCollection(const std::string& collection_name, CollectionParam& collection_param) = 0;
/** /**
* @brief flush all buffer into storage * @brief Get collection entity count
* *
* This method is used to all table buffer into storage * This method is used to get collection entity count.
*
* @param collection_name, target collection's name.
* @param entity_count, collection total entity count(including partitions).
*
* @return Indicate if this operation is successful.
*/
virtual Status
CountCollection(const std::string& collection_name, int64_t& entity_count) = 0;
/**
* @brief Show all collections in database
*
* This method is used to list all collections.
*
* @param collection_array, all collections in database.
*
* @return Indicate if this operation is successful.
*/
virtual Status
ShowCollections(std::vector<std::string>& collection_array) = 0;
/**
* @brief Show collection information
*
* This method is used to get detail information of a collection.
*
* @param collection_name, target collection's name.
* @param collection_info, target collection's information
*
* @return Indicate if this operation is successful.
*/
virtual Status
ShowCollectionInfo(const std::string& collection_name, CollectionInfo& collection_info) = 0;
/**
* @brief Delete entity by id
*
* This method is used to delete entity by id.
*
* @param collection_name, target collection's name.
* @param id_array, entity id array to be deleted.
*
* @return Indicate if this operation is successful.
*/
virtual Status
DeleteByID(const std::string& collection_name, const std::vector<int64_t>& id_array) = 0;
/**
* @brief Preload collection
*
* This method is used to preload collection data into memory
*
* @param collection_name, target collection's name.
*
* @return Indicate if this operation is successful.
*/
virtual Status
PreloadCollection(const std::string& collection_name) const = 0;
/**
* @brief Describe index
*
* This method is used to describe index
*
* @param collection_name, target collection's name.
* @param index_param, returned index information.
*
* @return Indicate if this operation is successful.
*/
virtual Status
DescribeIndex(const std::string& collection_name, IndexParam& index_param) const = 0;
/**
* @brief Drop index
*
* This method is used to drop index of collection(and its partitions)
*
* @param collection_name, target collection's name.
*
* @return Indicate if this operation is successful.
*/
virtual Status
DropIndex(const std::string& collection_name) const = 0;
/**
* @brief Create partition method
*
* This method is used to create collection's partition
*
* @param partition_param, use to provide partition information to be created.
*
* @return Indicate if partition is created successfully
*/
virtual Status
CreatePartition(const PartitionParam& partition_param) = 0;
/**
* @brief Show all partitions method
*
* This method is used to show all partitions(return their tags)
*
* @param collection_name, target collection's name.
* @param partition_tag_array, partition tag array of the collection.
*
* @return Indicate if this operation is successful
*/
virtual Status
ShowPartitions(const std::string& collection_name, PartitionTagList& partition_tag_array) const = 0;
/**
* @brief Delete partition method
*
* This method is used to delete collection's partition.
*
* @param partition_param, target partition to be deleted.
*
* @return Indicate if partition is delete successfully.
*/
virtual Status
DropPartition(const PartitionParam& partition_param) = 0;
/**
* @brief Flush collection buffer into storage
*
* This method is used to flush collection buffer into storage
*
* @param collection_name, target collection's name.
*
* @return Indicate if this operation is successful.
*/
virtual Status
FlushCollection(const std::string& collection_name) = 0;
/**
* @brief Flush all buffer into storage
*
* This method is used to all collection buffer into storage
* *
* @return Indicate if this operation is successful. * @return Indicate if this operation is successful.
*/ */
@ -564,16 +562,16 @@ class Connection {
Flush() = 0; Flush() = 0;
/** /**
* @brief compact table, remove deleted vectors * @brief Compact collection, permanently remove deleted vectors
* *
* This method is used to compact table * This method is used to compact collection
* *
* @param table_name, target table's name. * @param collection_name, target collection's name.
* *
* @return Indicate if this operation is successful. * @return Indicate if this operation is successful.
*/ */
virtual Status virtual Status
CompactTable(const std::string& table_name) = 0; CompactCollection(const std::string& collection_name) = 0;
}; };
} // namespace milvus } // namespace milvus

211
sdk/interface/ConnectionImpl.cpp
View File

@ -56,70 +56,6 @@ ConnectionImpl::ClientVersion() const {
return client_proxy_->ClientVersion(); return client_proxy_->ClientVersion();
} }
Status
ConnectionImpl::CreateTable(const TableSchema& param) {
return client_proxy_->CreateTable(param);
}
bool
ConnectionImpl::HasTable(const std::string& table_name) {
return client_proxy_->HasTable(table_name);
}
Status
ConnectionImpl::DropTable(const std::string& table_name) {
return client_proxy_->DropTable(table_name);
}
Status
ConnectionImpl::CreateIndex(const IndexParam& index_param) {
return client_proxy_->CreateIndex(index_param);
}
Status
ConnectionImpl::Insert(const std::string& table_name, const std::string& partition_tag,
const std::vector<RowRecord>& record_array, std::vector<int64_t>& id_array) {
return client_proxy_->Insert(table_name, partition_tag, record_array, id_array);
}
Status
ConnectionImpl::GetVectorByID(const std::string& table_name, int64_t vector_id, RowRecord& vector_data) {
return client_proxy_->GetVectorByID(table_name, vector_id, vector_data);
}
Status
ConnectionImpl::GetIDsInSegment(const std::string& table_name, const std::string& segment_name,
std::vector<int64_t>& id_array) {
return client_proxy_->GetIDsInSegment(table_name, segment_name, id_array);
}
Status
ConnectionImpl::Search(const std::string& table_name, const std::vector<std::string>& partition_tags,
const std::vector<RowRecord>& query_record_array, int64_t topk, const std::string& extra_params,
TopKQueryResult& topk_query_result) {
return client_proxy_->Search(table_name, partition_tags, query_record_array, topk, extra_params, topk_query_result);
}
Status
ConnectionImpl::DescribeTable(const std::string& table_name, TableSchema& table_schema) {
return client_proxy_->DescribeTable(table_name, table_schema);
}
Status
ConnectionImpl::CountTable(const std::string& table_name, int64_t& row_count) {
return client_proxy_->CountTable(table_name, row_count);
}
Status
ConnectionImpl::ShowTables(std::vector<std::string>& table_array) {
return client_proxy_->ShowTables(table_array);
}
Status
ConnectionImpl::ShowTableInfo(const std::string& table_name, TableInfo& table_info) {
return client_proxy_->ShowTableInfo(table_name, table_info);
}
std::string std::string
ConnectionImpl::ServerVersion() const { ConnectionImpl::ServerVersion() const {
return client_proxy_->ServerVersion(); return client_proxy_->ServerVersion();
@ -130,46 +66,6 @@ ConnectionImpl::ServerStatus() const {
return client_proxy_->ServerStatus(); return client_proxy_->ServerStatus();
} }
std::string
ConnectionImpl::DumpTaskTables() const {
return client_proxy_->DumpTaskTables();
}
Status
ConnectionImpl::DeleteByID(const std::string& table_name, const std::vector<int64_t>& id_array) {
return client_proxy_->DeleteByID(table_name, id_array);
}
Status
ConnectionImpl::PreloadTable(const std::string& table_name) const {
return client_proxy_->PreloadTable(table_name);
}
Status
ConnectionImpl::DescribeIndex(const std::string& table_name, IndexParam& index_param) const {
return client_proxy_->DescribeIndex(table_name, index_param);
}
Status
ConnectionImpl::DropIndex(const std::string& table_name) const {
return client_proxy_->DropIndex(table_name);
}
Status
ConnectionImpl::CreatePartition(const PartitionParam& param) {
return client_proxy_->CreatePartition(param);
}
Status
ConnectionImpl::ShowPartitions(const std::string& table_name, PartitionTagList& partition_array) const {
return client_proxy_->ShowPartitions(table_name, partition_array);
}
Status
ConnectionImpl::DropPartition(const PartitionParam& param) {
return client_proxy_->DropPartition(param);
}
Status Status
ConnectionImpl::GetConfig(const std::string& node_name, std::string& value) const { ConnectionImpl::GetConfig(const std::string& node_name, std::string& value) const {
return client_proxy_->GetConfig(node_name, value); return client_proxy_->GetConfig(node_name, value);
@ -181,8 +77,107 @@ ConnectionImpl::SetConfig(const std::string& node_name, const std::string& value
} }
Status Status
ConnectionImpl::FlushTable(const std::string& Status) { ConnectionImpl::CreateCollection(const CollectionParam& param) {
return client_proxy_->FlushTable(Status); return client_proxy_->CreateCollection(param);
}
bool
ConnectionImpl::HasCollection(const std::string& collection_name) {
return client_proxy_->HasCollection(collection_name);
}
Status
ConnectionImpl::DropCollection(const std::string& collection_name) {
return client_proxy_->DropCollection(collection_name);
}
Status
ConnectionImpl::CreateIndex(const IndexParam& index_param) {
return client_proxy_->CreateIndex(index_param);
}
Status
ConnectionImpl::Insert(const std::string& collection_name, const std::string& partition_tag,
const std::vector<Entity>& entity_array, std::vector<int64_t>& id_array) {
return client_proxy_->Insert(collection_name, partition_tag, entity_array, id_array);
}
Status
ConnectionImpl::GetEntityByID(const std::string& collection_name, int64_t entity_id, Entity& entity_data) {
return client_proxy_->GetEntityByID(collection_name, entity_id, entity_data);
}
Status
ConnectionImpl::GetIDsInSegment(const std::string& collection_name, const std::string& segment_name,
std::vector<int64_t>& id_array) {
return client_proxy_->GetIDsInSegment(collection_name, segment_name, id_array);
}
Status
ConnectionImpl::Search(const std::string& collection_name, const std::vector<std::string>& partition_tags,
const std::vector<Entity>& entity_array, int64_t topk, const std::string& extra_params,
TopKQueryResult& topk_query_result) {
return client_proxy_->Search(collection_name, partition_tags, entity_array, topk, extra_params, topk_query_result);
}
Status
ConnectionImpl::DescribeCollection(const std::string& collection_name, CollectionParam& collection_schema) {
return client_proxy_->DescribeCollection(collection_name, collection_schema);
}
Status
ConnectionImpl::CountCollection(const std::string& collection_name, int64_t& row_count) {
return client_proxy_->CountCollection(collection_name, row_count);
}
Status
ConnectionImpl::ShowCollections(std::vector<std::string>& collection_array) {
return client_proxy_->ShowCollections(collection_array);
}
Status
ConnectionImpl::ShowCollectionInfo(const std::string& collection_name, CollectionInfo& collection_info) {
return client_proxy_->ShowCollectionInfo(collection_name, collection_info);
}
Status
ConnectionImpl::DeleteByID(const std::string& collection_name, const std::vector<int64_t>& id_array) {
return client_proxy_->DeleteByID(collection_name, id_array);
}
Status
ConnectionImpl::PreloadCollection(const std::string& collection_name) const {
return client_proxy_->PreloadCollection(collection_name);
}
Status
ConnectionImpl::DescribeIndex(const std::string& collection_name, IndexParam& index_param) const {
return client_proxy_->DescribeIndex(collection_name, index_param);
}
Status
ConnectionImpl::DropIndex(const std::string& collection_name) const {
return client_proxy_->DropIndex(collection_name);
}
Status
ConnectionImpl::CreatePartition(const PartitionParam& partition_param) {
return client_proxy_->CreatePartition(partition_param);
}
Status
ConnectionImpl::ShowPartitions(const std::string& collection_name, PartitionTagList& partition_array) const {
return client_proxy_->ShowPartitions(collection_name, partition_array);
}
Status
ConnectionImpl::DropPartition(const PartitionParam& partition_param) {
return client_proxy_->DropPartition(partition_param);
}
Status
ConnectionImpl::FlushCollection(const std::string& Status) {
return client_proxy_->FlushCollection(Status);
} }
Status Status
@ -191,8 +186,8 @@ ConnectionImpl::Flush() {
} }
Status Status
ConnectionImpl::CompactTable(const std::string& table_name) { ConnectionImpl::CompactCollection(const std::string& collection_name) {
return client_proxy_->CompactTable(table_name); return client_proxy_->CompactCollection(collection_name);
} }
} // namespace milvus } // namespace milvus

133
sdk/interface/ConnectionImpl.h
View File

@ -26,7 +26,7 @@ class ConnectionImpl : public Connection {
// Implementations of the Connection interface // Implementations of the Connection interface
Status Status
Connect(const ConnectParam& param) override; Connect(const ConnectParam& connect_param) override;
Status Status
Connect(const std::string& uri) override; Connect(const std::string& uri) override;
@ -37,46 +37,6 @@ class ConnectionImpl : public Connection {
Status Status
Disconnect() override; Disconnect() override;
Status
CreateTable(const TableSchema& param) override;
bool
HasTable(const std::string& table_name) override;
Status
DropTable(const std::string& table_name) override;
Status
CreateIndex(const IndexParam& index_param) override;
Status
Insert(const std::string& table_name, const std::string& partition_tag, const std::vector<RowRecord>& record_array,
std::vector<int64_t>& id_array) override;
Status
GetVectorByID(const std::string& table_name, int64_t vector_id, RowRecord& vector_data) override;
Status
GetIDsInSegment(const std::string& table_name, const std::string& segment_name,
std::vector<int64_t>& id_array) override;
Status
Search(const std::string& table_name, const std::vector<std::string>& partition_tag_array,
const std::vector<RowRecord>& query_record_array, int64_t topk,
const std::string& extra_params, TopKQueryResult& topk_query_result) override;
Status
DescribeTable(const std::string& table_name, TableSchema& table_schema) override;
Status
CountTable(const std::string& table_name, int64_t& row_count) override;
Status
ShowTables(std::vector<std::string>& table_array) override;
Status
ShowTableInfo(const std::string& table_name, TableInfo& table_info) override;
std::string std::string
ClientVersion() const override; ClientVersion() const override;
@ -86,30 +46,6 @@ class ConnectionImpl : public Connection {
std::string std::string
ServerStatus() const override; ServerStatus() const override;
std::string
DumpTaskTables() const override;
Status
DeleteByID(const std::string& table_name, const std::vector<int64_t>& id_array) override;
Status
PreloadTable(const std::string& table_name) const override;
Status
DescribeIndex(const std::string& table_name, IndexParam& index_param) const override;
Status
DropIndex(const std::string& table_name) const override;
Status
CreatePartition(const PartitionParam& param) override;
Status
ShowPartitions(const std::string& table_name, PartitionTagList& partition_tag_array) const override;
Status
DropPartition(const PartitionParam& param) override;
Status Status
GetConfig(const std::string& node_name, std::string& value) const override; GetConfig(const std::string& node_name, std::string& value) const override;
@ -117,13 +53,76 @@ class ConnectionImpl : public Connection {
SetConfig(const std::string& node_name, const std::string& value) const override; SetConfig(const std::string& node_name, const std::string& value) const override;
Status Status
FlushTable(const std::string& table_name) override; CreateCollection(const CollectionParam& param) override;
bool
HasCollection(const std::string& collection_name) override;
Status
DropCollection(const std::string& collection_name) override;
Status
CreateIndex(const IndexParam& index_param) override;
Status
Insert(const std::string& collection_name,
const std::string& partition_tag,
const std::vector<Entity>& entity_array,
std::vector<int64_t>& id_array) override;
Status
GetEntityByID(const std::string& collection_name, int64_t entity_id, Entity& entity_data) override;
Status
GetIDsInSegment(const std::string& collection_name, const std::string& segment_name,
std::vector<int64_t>& id_array) override;
Status
Search(const std::string& collection_name, const std::vector<std::string>& partition_tag_array,
const std::vector<Entity>& entity_array, int64_t topk,
const std::string& extra_params, TopKQueryResult& topk_query_result) override;
Status
DescribeCollection(const std::string& collection_name, CollectionParam& collection_schema) override;
Status
CountCollection(const std::string& collection_name, int64_t& entity_count) override;
Status
ShowCollections(std::vector<std::string>& collection_array) override;
Status
ShowCollectionInfo(const std::string& collection_name, CollectionInfo& collection_info) override;
Status
DeleteByID(const std::string& collection_name, const std::vector<int64_t>& id_array) override;
Status
PreloadCollection(const std::string& collection_name) const override;
Status
DescribeIndex(const std::string& collection_name, IndexParam& index_param) const override;
Status
DropIndex(const std::string& collection_name) const override;
Status
CreatePartition(const PartitionParam& partition_param) override;
Status
ShowPartitions(const std::string& collection_name, PartitionTagList& partition_tag_array) const override;
Status
DropPartition(const PartitionParam& partition_param) override;
Status
FlushCollection(const std::string& collection_name) override;
Status Status
Flush() override; Flush() override;
Status Status
CompactTable(const std::string& table_name) override; CompactCollection(const std::string& collection_name) override;
private: private:
std::shared_ptr<ClientProxy> client_proxy_; std::shared_ptr<ClientProxy> client_proxy_;