From fcb5656819b74069d26c0138f08e47813d271c8e Mon Sep 17 00:00:00 2001 From: groot Date: Sun, 8 Mar 2020 12:13:06 +0800 Subject: [PATCH 1/2] refine c++sdk Signed-off-by: groot --- sdk/grpc/ClientProxy.cpp | 15 -- sdk/grpc/ClientProxy.h | 3 - sdk/include/MilvusApi.h | 396 +++++++++++++++---------------- sdk/interface/ConnectionImpl.cpp | 5 - sdk/interface/ConnectionImpl.h | 3 - 5 files changed, 193 insertions(+), 229 deletions(-) diff --git a/sdk/grpc/ClientProxy.cpp b/sdk/grpc/ClientProxy.cpp index ad23f9b954..6986355b4e 100644 --- a/sdk/grpc/ClientProxy.cpp +++ b/sdk/grpc/ClientProxy.cpp @@ -415,21 +415,6 @@ ClientProxy::ServerStatus() const { } } -std::string -ClientProxy::DumpTaskTables() const { - if (channel_ == nullptr) { - return "not connected to server"; - } - - try { - std::string dummy; - Status status = client_ptr_->Cmd("tasktable", dummy); - return dummy; - } catch (std::exception& ex) { - return "connection lost"; - } -} - Status ClientProxy::DeleteByID(const std::string& table_name, const std::vector& id_array) { try { diff --git a/sdk/grpc/ClientProxy.h b/sdk/grpc/ClientProxy.h index 0fd493dcd2..2c4a195072 100644 --- a/sdk/grpc/ClientProxy.h +++ b/sdk/grpc/ClientProxy.h @@ -84,9 +84,6 @@ class ClientProxy : public Connection { std::string ServerStatus() const override; - std::string - DumpTaskTables() const override; - Status DeleteByID(const std::string& table_name, const std::vector& id_array) override; diff --git a/sdk/include/MilvusApi.h b/sdk/include/MilvusApi.h index 111cc79dc3..e899210c91 100644 --- a/sdk/include/MilvusApi.h +++ b/sdk/include/MilvusApi.h @@ -54,20 +54,19 @@ struct ConnectParam { }; /** - * @brief Table Schema + * @brief Collection parameters */ -struct TableSchema { - std::string table_name; ///< Table name +struct CollectionParam { + std::string collection_name; ///< Collection_name name int64_t dimension = 0; ///< Vector dimension, must be a positive value int64_t index_file_size = 1024; ///< Index file size, must be a positive value, unit: MB MetricType metric_type = MetricType::L2; ///< Index metric type }; - /** - * @brief Record inserted + * @brief Entity inserted, currently each entity represent a vector */ -struct RowRecord { +struct Entity { std::vector float_data; ///< Vector raw float data std::vector binary_data; ///< Vector raw binary data }; @@ -76,7 +75,7 @@ struct RowRecord { * @brief TopK query result */ struct QueryResult { - std::vector ids; ///< Query ids result + std::vector ids; ///< Query entity ids result std::vector distances; ///< Query distances result }; using TopKQueryResult = std::vector; ///< Topk query result @@ -85,13 +84,14 @@ using TopKQueryResult = std::vector; ///< Topk query result * @brief Index parameters * Note: extra_params is extra parameters list, it must be json format * For different index type, parameter list is different accordingly, for example: - * FLAT/IVFLAT/SQ8: "{nlist: '16384'}" - * IVFPQ: "{nlist: '16384', nbits: "16"}" - * NSG: "{search_length: '100', out_degree:'40', pool_size:'66'}" - * HNSW "{M: '16', ef_construct:'500'}" + * FLAT/IVFLAT/SQ8: "{nlist: '16384'}" nlist range:[1, 999999] + * IVFPQ: "{nlist: '16384', m: "12"}" nlist range:[1, 999999] m is decided by dim and have a couple of results. + * NSG: "{search_length: '45', out_degree:'50', candidate_pool_size:'300', "knng":'100'}" + * search_length range:[10, 300] out_degree range:[5, 300] candidate_pool_size range:[50, 1000] knng range:[5, 300] + * HNSW "{M: '16', efConstruction:'500'}" M range:[5, 48] efConstruction range:[topk, 4096] */ struct IndexParam { - std::string table_name; ///< Table name for create index + std::string collection_name; ///< Collection name for create index IndexType index_type; ///< Index type std::string extra_params; ///< Extra parameters according to different index type, must be json format }; @@ -100,7 +100,7 @@ struct IndexParam { * @brief partition parameters */ struct PartitionParam { - std::string table_name; + std::string collection_name; std::string partition_tag; }; @@ -126,11 +126,11 @@ struct PartitionStat { }; /** - * @brief table info + * @brief collection info */ -struct TableInfo { - int64_t total_row_count; ///< Table total row count - std::vector partitions_stat; ///< Table's partitions statistics +struct CollectionInfo { + int64_t total_row_count; ///< Collection total entity count + std::vector partitions_stat; ///< Collection's partitions statistics }; /** @@ -209,182 +209,6 @@ class Connection { virtual Status Disconnect() = 0; - /** - * @brief Create table method - * - * This method is used to create table. - * - * @param param, use to provide table information to be created. - * - * @return Indicate if table is created successfully - */ - virtual Status - CreateTable(const TableSchema& param) = 0; - - /** - * @brief Test table existence method - * - * This method is used to create table. - * - * @param table_name, target table's name. - * - * @return Indicate if table is cexist - */ - virtual bool - HasTable(const std::string& table_name) = 0; - - /** - * @brief Drop table method - * - * This method is used to drop table(and its partitions). - * - * @param table_name, target table's name. - * - * @return Indicate if table is drop successfully. - */ - virtual Status - DropTable(const std::string& table_name) = 0; - - /** - * @brief Create index method - * - * This method is used to create index for whole table(and its partitions). - * - * @param IndexParam - * table_name, table name is going to be create index. - * index type, - * nlist, - * index file size - * - * @return Indicate if build index successfully. - */ - virtual Status - CreateIndex(const IndexParam& index_param) = 0; - - /** - * @brief Insert vector to table - * - * This method is used to insert vector array to table. - * - * @param table_name, target table's name. - * @param partition_tag, target partition's tag, keep empty if no partition specified. - * @param record_array, vector array is inserted. - * @param id_array, - * specify id for each vector, - * if this array is empty, milvus will generate unique id for each vector, - * and return all ids by this parameter. - * - * @return Indicate if vector array are inserted successfully - */ - virtual Status - Insert(const std::string& table_name, const std::string& partition_tag, const std::vector& record_array, - std::vector& id_array) = 0; - - /** - * @brief Get vector data by id - * - * This method is used to get vector data by id from a table. - * Return the first found vector if there are vectors with duplicated id - * - * @param table_name, target table's name. - * @param vector_id, target vector id. - * @param vector_data, returned vector data. - * - * @return Indicate if the operation is succeed. - */ - virtual Status - GetVectorByID(const std::string& table_name, int64_t vector_id, RowRecord& vector_data) = 0; - - /** - * @brief Get vector ids from a segment - * - * This method is used to get vector ids from a segment - * Return all vector(not deleted) ids - * - * @param table_name, target table's name. - * @param segment_name, target segment name. - * @param id_array, returned vector id array. - * - * @return Indicate if the operation is succeed. - */ - virtual Status - GetIDsInSegment(const std::string& table_name, const std::string& segment_name, std::vector& id_array) = 0; - - /** - * @brief Search vector - * - * This method is used to query vector in table. - * - * @param table_name, target table's name. - * @param partition_tag_array, target partitions, keep empty if no partition specified. - * @param query_record_array, vectors to be queried. - * @param topk, how many similarity vectors will be returned. - * @param extra_params, extra search parameters according to different index type, must be json format. - * Note: extra_params is extra parameters list, it must be json format, for example: - * For different index type, parameter list is different accordingly - * FLAT/IVFLAT/SQ8/IVFPQ: "{nprobe: '32'}" - * NSG: "{search_length:'100'} - * HNSW "{ef: '64'} - * @param topk_query_result, result array. - * - * @return Indicate if query is successful. - */ - virtual Status - Search(const std::string& table_name, const PartitionTagList& partition_tag_array, - const std::vector& query_record_array, int64_t topk, - const std::string& extra_params, TopKQueryResult& topk_query_result) = 0; - - /** - * @brief Show table description - * - * This method is used to show table information. - * - * @param table_name, target table's name. - * @param table_schema, table_schema is given when operation is successful. - * - * @return Indicate if this operation is successful. - */ - virtual Status - DescribeTable(const std::string& table_name, TableSchema& table_schema) = 0; - - /** - * @brief Get table row count - * - * This method is used to get table row count. - * - * @param table_name, target table's name. - * @param row_count, table total row count(including partitions). - * - * @return Indicate if this operation is successful. - */ - virtual Status - CountTable(const std::string& table_name, int64_t& row_count) = 0; - - /** - * @brief Show all tables in database - * - * This method is used to list all tables. - * - * @param table_array, all tables in database. - * - * @return Indicate if this operation is successful. - */ - virtual Status - ShowTables(std::vector& table_array) = 0; - - /** - * @brief Show table information - * - * This method is used to get detail information of a table. - * - * @param table_name, target table's name. - * @param table_info, target table's information - * - * @return Indicate if this operation is successful. - */ - virtual Status - ShowTableInfo(const std::string& table_name, TableInfo& table_info) = 0; - /** * @brief Give the client version * @@ -405,6 +229,182 @@ class Connection { virtual std::string ServerVersion() const = 0; + /** + * @brief Create collection method + * + * This method is used to create collection. + * + * @param param, use to provide collection information to be created. + * + * @return Indicate if collection is created successfully + */ + virtual Status + CreateCollection(const CollectionParam& param) = 0; + + /** + * @brief Test collection existence method + * + * This method is used to test collection existence. + * + * @param collection_name, target collection's name. + * + * @return Indicate if collection is cexist + */ + virtual bool + HasCollection(const std::string& collection_name) = 0; + + /** + * @brief Drop collection method + * + * This method is used to drop collection(and its partitions). + * + * @param collection_name, target collection's name. + * + * @return Indicate if collection is drop successfully. + */ + virtual Status + DropCollection(const std::string& collection_name) = 0; + + /** + * @brief Create index method + * + * This method is used to create index for whole collection(and its partitions). + * + * @param index_param, use to provide index information to be created. + * + * @return Indicate if create index successfully. + */ + virtual Status + CreateIndex(const IndexParam& index_param) = 0; + + /** + * @brief Insert entity to table + * + * This method is used to insert vector array to table. + * + * @param collection_name, target collection's name. + * @param partition_tag, target partition's tag, keep empty if no partition specified. + * @param entity_array, entity array is inserted, each entitu represent a vector. + * @param id_array, + * specify id for each entity, + * if this array is empty, milvus will generate unique id for each entity, + * and return all ids by this parameter. + * + * @return Indicate if entity array are inserted successfully + */ + virtual Status + Insert(const std::string& collection_name, + const std::string& partition_tag, + const std::vector& entity_array, + std::vector& id_array) = 0; + + /** + * @brief Get entity data by id + * + * This method is used to get entity data by id from a collection. + * Return the first found entity if there are entities with duplicated id + * + * @param collection_name, target collection's name. + * @param entity_id, target entity id. + * @param entity_data, returned entity data. + * + * @return Indicate if the operation is succeed. + */ + virtual Status + GetEntityByID(const std::string& collection_name, int64_t entity_id, Entity& entity_data) = 0; + + /** + * @brief Get entity ids from a segment + * + * This method is used to get entity ids from a segment + * Return all entity(not deleted) ids + * + * @param collection_name, target collection's name. + * @param segment_name, target segment name. + * @param id_array, returned entity id array. + * + * @return Indicate if the operation is succeed. + */ + virtual Status + GetIDsInSegment(const std::string& collection_name, + const std::string& segment_name, + std::vector& id_array) = 0; + + /** + * @brief Search entities in a collection + * + * This method is used to query entity in collection. + * + * @param collection_name, target collection's name. + * @param partition_tag_array, target partitions, keep empty if no partition specified. + * @param query_record_array, vectors to be queried. + * @param topk, how many similarity vectors will be returned. + * @param extra_params, extra search parameters according to different index type, must be json format. + * Note: extra_params is extra parameters list, it must be json format, for example: + * For different index type, parameter list is different accordingly + * FLAT/IVFLAT/SQ8/IVFPQ: "{nprobe: '32'}" + * NSG: "{search_length:'100'} + * HNSW "{ef: '64'} + * @param topk_query_result, result array. + * + * @return Indicate if query is successful. + */ + virtual Status + Search(const std::string& table_name, const PartitionTagList& partition_tag_array, + const std::vector& query_record_array, int64_t topk, + const std::string& extra_params, TopKQueryResult& topk_query_result) = 0; + + /** + * @brief Show collection description + * + * This method is used to show collection information. + * + * @param collection_name, target collection's name. + * @param collection_schema, collection_schema is given when operation is successful. + * + * @return Indicate if this operation is successful. + */ + virtual Status + DescribeCollection(const std::string& collection_name, TableSchema& collection_schema) = 0; + + /** + * @brief Get collection row count + * + * This method is used to get collection row count. + * + * @param collection_name, target collection's name. + * @param entity_count, collection total entity count(including partitions). + * + * @return Indicate if this operation is successful. + */ + virtual Status + CountCollection(const std::string& collection_name, int64_t& entity_count) = 0; + + /** + * @brief Show all collections in database + * + * This method is used to list all collections. + * + * @param collection_array, all collections in database. + * + * @return Indicate if this operation is successful. + */ + virtual Status + ShowCollections(std::vector& collection_array) = 0; + + /** + * @brief Show collections information + * + * This method is used to get detail information of a collections. + * + * @param collections_name, target collections's name. + * @param collections_info, target collections's information + * + * @return Indicate if this operation is successful. + */ + virtual Status + ShowCollectionInfo(const std::string& collections_name, TableInfo& collections_info) = 0; + /** * @brief Give the server status * @@ -415,16 +415,6 @@ class Connection { virtual std::string ServerStatus() const = 0; - /** - * @brief dump server tasks information - * - * This method is internal used. - * - * @return Task information in tasktables. - */ - virtual std::string - DumpTaskTables() const = 0; - /** * [deprecated] * @brief delete tables by vector id diff --git a/sdk/interface/ConnectionImpl.cpp b/sdk/interface/ConnectionImpl.cpp index 4acbcec7b8..3a3fa52516 100644 --- a/sdk/interface/ConnectionImpl.cpp +++ b/sdk/interface/ConnectionImpl.cpp @@ -130,11 +130,6 @@ ConnectionImpl::ServerStatus() const { return client_proxy_->ServerStatus(); } -std::string -ConnectionImpl::DumpTaskTables() const { - return client_proxy_->DumpTaskTables(); -} - Status ConnectionImpl::DeleteByID(const std::string& table_name, const std::vector& id_array) { return client_proxy_->DeleteByID(table_name, id_array); diff --git a/sdk/interface/ConnectionImpl.h b/sdk/interface/ConnectionImpl.h index ec0f423857..469adf1a27 100644 --- a/sdk/interface/ConnectionImpl.h +++ b/sdk/interface/ConnectionImpl.h @@ -86,9 +86,6 @@ class ConnectionImpl : public Connection { std::string ServerStatus() const override; - std::string - DumpTaskTables() const override; - Status DeleteByID(const std::string& table_name, const std::vector& id_array) override; From 9203bdf84db282223c765dcb27e605cde4c82494 Mon Sep 17 00:00:00 2001 From: groot Date: Sun, 8 Mar 2020 14:27:16 +0800 Subject: [PATCH 2/2] rename c++sdk Signed-off-by: groot --- sdk/examples/binary_vector/src/ClientTest.cpp | 107 +-- sdk/examples/partition/src/ClientTest.cpp | 153 ++-- sdk/examples/simple/src/ClientTest.cpp | 199 ++--- sdk/examples/simple/src/ClientTest.h | 24 +- sdk/examples/utils/Utils.cpp | 79 +- sdk/examples/utils/Utils.h | 18 +- sdk/grpc/ClientProxy.cpp | 751 +++++++++--------- sdk/grpc/ClientProxy.h | 130 +-- sdk/include/MilvusApi.h | 212 ++--- sdk/interface/ConnectionImpl.cpp | 206 ++--- sdk/interface/ConnectionImpl.h | 130 +-- 11 files changed, 1015 insertions(+), 994 deletions(-) diff --git a/sdk/examples/binary_vector/src/ClientTest.cpp b/sdk/examples/binary_vector/src/ClientTest.cpp index f6dbd4b8d2..db16575f61 100644 --- a/sdk/examples/binary_vector/src/ClientTest.cpp +++ b/sdk/examples/binary_vector/src/ClientTest.cpp @@ -22,52 +22,53 @@ namespace { -const char* TABLE_NAME = milvus_sdk::Utils::GenTableName().c_str(); +const char* COLLECTION_NAME = milvus_sdk::Utils::GenCollectionName().c_str(); -constexpr int64_t TABLE_DIMENSION = 512; -constexpr int64_t TABLE_INDEX_FILE_SIZE = 128; -constexpr milvus::MetricType TABLE_METRIC_TYPE = milvus::MetricType::TANIMOTO; -constexpr int64_t BATCH_ROW_COUNT = 100000; +constexpr int64_t COLLECTION_DIMENSION = 512; +constexpr int64_t COLLECTION_INDEX_FILE_SIZE = 128; +constexpr milvus::MetricType COLLECTION_METRIC_TYPE = milvus::MetricType::TANIMOTO; +constexpr int64_t BATCH_ENTITY_COUNT = 100000; constexpr int64_t NQ = 5; constexpr int64_t TOP_K = 10; constexpr int64_t NPROBE = 32; -constexpr int64_t SEARCH_TARGET = 5000; // change this value, result is different, ensure less than BATCH_ROW_COUNT -constexpr int64_t ADD_VECTOR_LOOP = 20; +constexpr int64_t SEARCH_TARGET = 5000; // change this value, result is different, ensure less than BATCH_ENTITY_COUNT +constexpr int64_t ADD_ENTITY_LOOP = 20; constexpr milvus::IndexType INDEX_TYPE = milvus::IndexType::IVFFLAT; -milvus::TableSchema -BuildTableSchema() { - milvus::TableSchema tb_schema = {TABLE_NAME, TABLE_DIMENSION, TABLE_INDEX_FILE_SIZE, TABLE_METRIC_TYPE}; - return tb_schema; +milvus::CollectionParam +BuildCollectionParam() { + milvus::CollectionParam + collection_param = {COLLECTION_NAME, COLLECTION_DIMENSION, COLLECTION_INDEX_FILE_SIZE, COLLECTION_METRIC_TYPE}; + return collection_param; } milvus::IndexParam BuildIndexParam() { JSON json_params = {{"nlist", 1024}}; - milvus::IndexParam index_param = {TABLE_NAME, INDEX_TYPE, json_params.dump()}; + milvus::IndexParam index_param = {COLLECTION_NAME, INDEX_TYPE, json_params.dump()}; return index_param; } void -BuildBinaryVectors(int64_t from, int64_t to, std::vector& vector_record_array, - std::vector& record_ids, int64_t dimension) { +BuildBinaryVectors(int64_t from, int64_t to, std::vector& entity_array, + std::vector& entity_ids, int64_t dimension) { if (to <= from) { return; } - vector_record_array.clear(); - record_ids.clear(); + entity_array.clear(); + entity_ids.clear(); int64_t dim_byte = dimension/8; for (int64_t k = from; k < to; k++) { - milvus::RowRecord record; - record.binary_data.resize(dim_byte); + milvus::Entity entity; + entity.binary_data.resize(dim_byte); for (int64_t i = 0; i < dim_byte; i++) { - record.binary_data[i] = (uint8_t)lrand48(); + entity.binary_data[i] = (uint8_t)lrand48(); } - vector_record_array.emplace_back(record); - record_ids.push_back(k); + entity_array.emplace_back(entity); + entity_ids.push_back(k); } } @@ -84,54 +85,54 @@ ClientTest::Test(const std::string& address, const std::string& port) { std::cout << "Connect function call status: " << stat.message() << std::endl; } - { // create table - milvus::TableSchema tb_schema = BuildTableSchema(); - stat = conn->CreateTable(tb_schema); - std::cout << "CreateTable function call status: " << stat.message() << std::endl; - milvus_sdk::Utils::PrintTableSchema(tb_schema); + { // create collection + milvus::CollectionParam collection_param = BuildCollectionParam(); + stat = conn->CreateCollection(collection_param); + std::cout << "CreateCollection function call status: " << stat.message() << std::endl; + milvus_sdk::Utils::PrintCollectionParam(collection_param); - bool has_table = conn->HasTable(tb_schema.table_name); - if (has_table) { - std::cout << "Table is created" << std::endl; + bool has_collection = conn->HasCollection(collection_param.collection_name); + if (has_collection) { + std::cout << "Collection is created" << std::endl; } } - std::vector> search_record_array; + std::vector> search_entity_array; { // insert vectors - for (int i = 0; i < ADD_VECTOR_LOOP; i++) { - std::vector record_array; - std::vector record_ids; - int64_t begin_index = i * BATCH_ROW_COUNT; + for (int i = 0; i < ADD_ENTITY_LOOP; i++) { + std::vector entity_array; + std::vector entity_ids; + int64_t begin_index = i * BATCH_ENTITY_COUNT; { // generate vectors - milvus_sdk::TimeRecorder rc("Build vectors No." + std::to_string(i)); + milvus_sdk::TimeRecorder rc("Build entities No." + std::to_string(i)); BuildBinaryVectors(begin_index, - begin_index + BATCH_ROW_COUNT, - record_array, - record_ids, - TABLE_DIMENSION); + begin_index + BATCH_ENTITY_COUNT, + entity_array, + entity_ids, + COLLECTION_DIMENSION); } - if (search_record_array.size() < NQ) { - search_record_array.push_back(std::make_pair(record_ids[SEARCH_TARGET], record_array[SEARCH_TARGET])); + if (search_entity_array.size() < NQ) { + search_entity_array.push_back(std::make_pair(entity_ids[SEARCH_TARGET], entity_array[SEARCH_TARGET])); } - std::string title = "Insert " + std::to_string(record_array.size()) + " vectors No." + std::to_string(i); + std::string title = "Insert " + std::to_string(entity_array.size()) + " entities No." + std::to_string(i); milvus_sdk::TimeRecorder rc(title); - stat = conn->Insert(TABLE_NAME, "", record_array, record_ids); - std::cout << "InsertVector function call status: " << stat.message() << std::endl; - std::cout << "Returned id array count: " << record_ids.size() << std::endl; + stat = conn->Insert(COLLECTION_NAME, "", entity_array, entity_ids); + std::cout << "Insert function call status: " << stat.message() << std::endl; + std::cout << "Returned id array count: " << entity_ids.size() << std::endl; } } { // flush buffer - stat = conn->FlushTable(TABLE_NAME); - std::cout << "FlushTable function call status: " << stat.message() << std::endl; + stat = conn->FlushCollection(COLLECTION_NAME); + std::cout << "FlushCollection function call status: " << stat.message() << std::endl; } { // search vectors std::vector partition_tags; milvus::TopKQueryResult topk_query_result; - milvus_sdk::Utils::DoSearch(conn, TABLE_NAME, partition_tags, TOP_K, NPROBE, search_record_array, + milvus_sdk::Utils::DoSearch(conn, COLLECTION_NAME, partition_tags, TOP_K, NPROBE, search_entity_array, topk_query_result); } @@ -144,7 +145,7 @@ ClientTest::Test(const std::string& address, const std::string& port) { std::cout << "CreateIndex function call status: " << stat.message() << std::endl; milvus::IndexParam index2; - stat = conn->DescribeIndex(TABLE_NAME, index2); + stat = conn->DescribeIndex(COLLECTION_NAME, index2); std::cout << "DescribeIndex function call status: " << stat.message() << std::endl; milvus_sdk::Utils::PrintIndexParam(index2); } @@ -152,13 +153,13 @@ ClientTest::Test(const std::string& address, const std::string& port) { { // search vectors std::vector partition_tags; milvus::TopKQueryResult topk_query_result; - milvus_sdk::Utils::DoSearch(conn, TABLE_NAME, partition_tags, TOP_K, NPROBE, search_record_array, + milvus_sdk::Utils::DoSearch(conn, COLLECTION_NAME, partition_tags, TOP_K, NPROBE, search_entity_array, topk_query_result); } - { // drop table - stat = conn->DropTable(TABLE_NAME); - std::cout << "DropTable function call status: " << stat.message() << std::endl; + { // drop collection + stat = conn->DropCollection(COLLECTION_NAME); + std::cout << "DropCollection function call status: " << stat.message() << std::endl; } milvus::Connection::Destroy(conn); diff --git a/sdk/examples/partition/src/ClientTest.cpp b/sdk/examples/partition/src/ClientTest.cpp index aca815c6ed..e5841fed70 100644 --- a/sdk/examples/partition/src/ClientTest.cpp +++ b/sdk/examples/partition/src/ClientTest.cpp @@ -22,12 +22,12 @@ namespace { -const char* TABLE_NAME = milvus_sdk::Utils::GenTableName().c_str(); +const char* COLLECTION_NAME = milvus_sdk::Utils::GenCollectionName().c_str(); -constexpr int64_t TABLE_DIMENSION = 512; -constexpr int64_t TABLE_INDEX_FILE_SIZE = 1024; -constexpr milvus::MetricType TABLE_METRIC_TYPE = milvus::MetricType::L2; -constexpr int64_t BATCH_ROW_COUNT = 10000; +constexpr int64_t COLLECTION_DIMENSION = 512; +constexpr int64_t COLLECTION_INDEX_FILE_SIZE = 1024; +constexpr milvus::MetricType COLLECTION_METRIC_TYPE = milvus::MetricType::L2; +constexpr int64_t BATCH_ENTITY_COUNT = 10000; constexpr int64_t NQ = 5; constexpr int64_t TOP_K = 10; constexpr int64_t NPROBE = 32; @@ -36,27 +36,45 @@ constexpr milvus::IndexType INDEX_TYPE = milvus::IndexType::IVFSQ8; constexpr int32_t PARTITION_COUNT = 5; constexpr int32_t TARGET_PARTITION = 3; -milvus::TableSchema -BuildTableSchema() { - milvus::TableSchema tb_schema = {TABLE_NAME, TABLE_DIMENSION, TABLE_INDEX_FILE_SIZE, TABLE_METRIC_TYPE}; - return tb_schema; +milvus::CollectionParam +BuildCollectionParam() { + milvus::CollectionParam + collection_param = {COLLECTION_NAME, COLLECTION_DIMENSION, COLLECTION_INDEX_FILE_SIZE, COLLECTION_METRIC_TYPE}; + return collection_param; } milvus::PartitionParam BuildPartitionParam(int32_t index) { std::string tag = std::to_string(index); - std::string partition_name = std::string(TABLE_NAME) + "_" + tag; - milvus::PartitionParam partition_param = {TABLE_NAME, tag}; + std::string partition_name = std::string(COLLECTION_NAME) + "_" + tag; + milvus::PartitionParam partition_param = {COLLECTION_NAME, tag}; return partition_param; } milvus::IndexParam BuildIndexParam() { JSON json_params = {{"nlist", 16384}}; - milvus::IndexParam index_param = {TABLE_NAME, INDEX_TYPE, json_params.dump()}; + milvus::IndexParam index_param = {COLLECTION_NAME, INDEX_TYPE, json_params.dump()}; return index_param; } +void +CountCollection(std::shared_ptr& conn) { + int64_t entity_count = 0; + auto stat = conn->CountCollection(COLLECTION_NAME, entity_count); + std::cout << COLLECTION_NAME << "(" << entity_count << " entities)" << std::endl; +} + +void +ShowCollectionInfo(std::shared_ptr& conn) { + CountCollection(conn); + + milvus::CollectionInfo collection_info; + auto stat = conn->ShowCollectionInfo(COLLECTION_NAME, collection_info); + milvus_sdk::Utils::PrintCollectionInfo(collection_info); + std::cout << "ShowCollectionInfo function call status: " << stat.message() << std::endl; +} + } // namespace void @@ -70,11 +88,11 @@ ClientTest::Test(const std::string& address, const std::string& port) { std::cout << "Connect function call status: " << stat.message() << std::endl; } - { // create table - milvus::TableSchema tb_schema = BuildTableSchema(); - stat = conn->CreateTable(tb_schema); - std::cout << "CreateTable function call status: " << stat.message() << std::endl; - milvus_sdk::Utils::PrintTableSchema(tb_schema); + { // create collection + milvus::CollectionParam tb_schema = BuildCollectionParam(); + stat = conn->CreateCollection(tb_schema); + std::cout << "CreateCollection function call status: " << stat.message() << std::endl; + milvus_sdk::Utils::PrintCollectionParam(tb_schema); } { // create partition @@ -87,7 +105,7 @@ ClientTest::Test(const std::string& address, const std::string& port) { // show partitions milvus::PartitionTagList partition_array; - stat = conn->ShowPartitions(TABLE_NAME, partition_array); + stat = conn->ShowPartitions(COLLECTION_NAME, partition_array); std::cout << partition_array.size() << " partitions created:" << std::endl; for (auto& partition_tag : partition_array) { @@ -96,64 +114,56 @@ ClientTest::Test(const std::string& address, const std::string& port) { } { // insert vectors - milvus_sdk::TimeRecorder rc("All vectors"); + milvus_sdk::TimeRecorder rc("All entities"); for (int i = 0; i < PARTITION_COUNT * 5; i++) { - std::vector record_array; - std::vector record_ids; - int64_t begin_index = i * BATCH_ROW_COUNT; + std::vector entity_array; + std::vector entity_ids; + int64_t begin_index = i * BATCH_ENTITY_COUNT; { // generate vectors - milvus_sdk::TimeRecorder rc("Build vectors No." + std::to_string(i)); - milvus_sdk::Utils::BuildVectors(begin_index, begin_index + BATCH_ROW_COUNT, record_array, record_ids, - TABLE_DIMENSION); + milvus_sdk::TimeRecorder rc("Build entities No." + std::to_string(i)); + milvus_sdk::Utils::BuildEntities(begin_index, + begin_index + BATCH_ENTITY_COUNT, + entity_array, + entity_ids, + COLLECTION_DIMENSION); } - std::string title = "Insert " + std::to_string(record_array.size()) + " vectors No." + std::to_string(i); + std::string title = "Insert " + std::to_string(entity_array.size()) + " entities No." + std::to_string(i); milvus_sdk::TimeRecorder rc(title); - stat = conn->Insert(TABLE_NAME, std::to_string(i % PARTITION_COUNT), record_array, record_ids); + stat = conn->Insert(COLLECTION_NAME, std::to_string(i % PARTITION_COUNT), entity_array, entity_ids); } } { // flush buffer - stat = conn->FlushTable(TABLE_NAME); - std::cout << "FlushTable function call status: " << stat.message() << std::endl; + stat = conn->FlushCollection(COLLECTION_NAME); + std::cout << "FlushCollection function call status: " << stat.message() << std::endl; } - { // table row count - int64_t row_count = 0; - stat = conn->CountTable(TABLE_NAME, row_count); - std::cout << TABLE_NAME << "(" << row_count << " rows)" << std::endl; - } + ShowCollectionInfo(conn); - { // get table information - milvus::TableInfo table_info; - stat = conn->ShowTableInfo(TABLE_NAME, table_info); - milvus_sdk::Utils::PrintTableInfo(table_info); - std::cout << "ShowTableInfo function call status: " << stat.message() << std::endl; - } - - std::vector> search_record_array; + std::vector> search_entity_array; { // build search vectors - std::vector record_array; - std::vector record_ids; - int64_t index = TARGET_PARTITION * BATCH_ROW_COUNT + SEARCH_TARGET; - milvus_sdk::Utils::BuildVectors(index, index + 1, record_array, record_ids, TABLE_DIMENSION); - search_record_array.push_back(std::make_pair(record_ids[0], record_array[0])); + std::vector entity_array; + std::vector entity_ids; + int64_t index = TARGET_PARTITION * BATCH_ENTITY_COUNT + SEARCH_TARGET; + milvus_sdk::Utils::BuildEntities(index, index + 1, entity_array, entity_ids, COLLECTION_DIMENSION); + search_entity_array.push_back(std::make_pair(entity_ids[0], entity_array[0])); } { // search vectors std::cout << "Search in correct partition" << std::endl; std::vector partition_tags = {std::to_string(TARGET_PARTITION)}; milvus::TopKQueryResult topk_query_result; - milvus_sdk::Utils::DoSearch(conn, TABLE_NAME, partition_tags, TOP_K, NPROBE, search_record_array, + milvus_sdk::Utils::DoSearch(conn, COLLECTION_NAME, partition_tags, TOP_K, NPROBE, search_entity_array, topk_query_result); std::cout << "Search in wrong partition" << std::endl; partition_tags = {"0"}; - milvus_sdk::Utils::DoSearch(conn, TABLE_NAME, partition_tags, TOP_K, NPROBE, search_record_array, + milvus_sdk::Utils::DoSearch(conn, COLLECTION_NAME, partition_tags, TOP_K, NPROBE, search_entity_array, topk_query_result); std::cout << "Search by regex matched partition tag" << std::endl; partition_tags = {"\\d"}; - milvus_sdk::Utils::DoSearch(conn, TABLE_NAME, partition_tags, TOP_K, NPROBE, search_record_array, + milvus_sdk::Utils::DoSearch(conn, COLLECTION_NAME, partition_tags, TOP_K, NPROBE, search_entity_array, topk_query_result); } @@ -166,57 +176,42 @@ ClientTest::Test(const std::string& address, const std::string& port) { std::cout << "CreateIndex function call status: " << stat.message() << std::endl; milvus::IndexParam index2; - stat = conn->DescribeIndex(TABLE_NAME, index2); + stat = conn->DescribeIndex(COLLECTION_NAME, index2); std::cout << "DescribeIndex function call status: " << stat.message() << std::endl; milvus_sdk::Utils::PrintIndexParam(index2); } - { // table row count - int64_t row_count = 0; - stat = conn->CountTable(TABLE_NAME, row_count); - std::cout << TABLE_NAME << "(" << row_count << " rows)" << std::endl; - } - - { // get table information - milvus::TableInfo table_info; - stat = conn->ShowTableInfo(TABLE_NAME, table_info); - milvus_sdk::Utils::PrintTableInfo(table_info); - std::cout << "ShowTableInfo function call status: " << stat.message() << std::endl; - } + ShowCollectionInfo(conn); { // drop partition - milvus::PartitionParam param1 = {TABLE_NAME, std::to_string(TARGET_PARTITION)}; + milvus::PartitionParam param1 = {COLLECTION_NAME, std::to_string(TARGET_PARTITION)}; milvus_sdk::Utils::PrintPartitionParam(param1); stat = conn->DropPartition(param1); std::cout << "DropPartition function call status: " << stat.message() << std::endl; } - { // table row count - int64_t row_count = 0; - stat = conn->CountTable(TABLE_NAME, row_count); - std::cout << TABLE_NAME << "(" << row_count << " rows)" << std::endl; - } + CountCollection(conn); - { // search vectors - std::cout << "Search in whole table" << std::endl; + { // search vectors, will get search error since we delete a partition + std::cout << "Search in whole collection after delete one partition" << std::endl; std::vector partition_tags; milvus::TopKQueryResult topk_query_result; - milvus_sdk::Utils::DoSearch(conn, TABLE_NAME, partition_tags, TOP_K, NPROBE, search_record_array, + milvus_sdk::Utils::DoSearch(conn, COLLECTION_NAME, partition_tags, TOP_K, NPROBE, search_entity_array, topk_query_result); } { // drop index - stat = conn->DropIndex(TABLE_NAME); + stat = conn->DropIndex(COLLECTION_NAME); std::cout << "DropIndex function call status: " << stat.message() << std::endl; - int64_t row_count = 0; - stat = conn->CountTable(TABLE_NAME, row_count); - std::cout << TABLE_NAME << "(" << row_count << " rows)" << std::endl; + int64_t entity_count = 0; + stat = conn->CountCollection(COLLECTION_NAME, entity_count); + std::cout << COLLECTION_NAME << "(" << entity_count << " entities)" << std::endl; } - { // drop table - stat = conn->DropTable(TABLE_NAME); - std::cout << "DropTable function call status: " << stat.message() << std::endl; + { // drop collection + stat = conn->DropCollection(COLLECTION_NAME); + std::cout << "DropCollection function call status: " << stat.message() << std::endl; } milvus::Connection::Destroy(conn); diff --git a/sdk/examples/simple/src/ClientTest.cpp b/sdk/examples/simple/src/ClientTest.cpp index 635dabdc16..fb2e2b138d 100644 --- a/sdk/examples/simple/src/ClientTest.cpp +++ b/sdk/examples/simple/src/ClientTest.cpp @@ -19,20 +19,19 @@ #include #include - namespace { -const char* TABLE_NAME = milvus_sdk::Utils::GenTableName().c_str(); +const char* COLLECTION_NAME = milvus_sdk::Utils::GenCollectionName().c_str(); -constexpr int64_t TABLE_DIMENSION = 512; -constexpr int64_t TABLE_INDEX_FILE_SIZE = 1024; -constexpr milvus::MetricType TABLE_METRIC_TYPE = milvus::MetricType::L2; -constexpr int64_t BATCH_ROW_COUNT = 100000; +constexpr int64_t COLLECTION_DIMENSION = 512; +constexpr int64_t COLLECTION_INDEX_FILE_SIZE = 1024; +constexpr milvus::MetricType COLLECTION_METRIC_TYPE = milvus::MetricType::L2; +constexpr int64_t BATCH_ENTITY_COUNT = 100000; constexpr int64_t NQ = 5; constexpr int64_t TOP_K = 10; constexpr int64_t NPROBE = 32; constexpr int64_t SEARCH_TARGET = 5000; // change this value, result is different -constexpr int64_t ADD_VECTOR_LOOP = 5; +constexpr int64_t ADD_ENTITY_LOOP = 5; constexpr milvus::IndexType INDEX_TYPE = milvus::IndexType::IVFSQ8; constexpr int32_t NLIST = 16384; @@ -63,191 +62,201 @@ ClientTest::ShowSdkVersion() { } void -ClientTest::ShowTables(std::vector& tables) { - milvus::Status stat = conn_->ShowTables(tables); - std::cout << "ShowTables function call status: " << stat.message() << std::endl; - std::cout << "All tables: " << std::endl; - for (auto& table : tables) { - int64_t row_count = 0; - stat = conn_->CountTable(table, row_count); - std::cout << "\t" << table << "(" << row_count << " rows)" << std::endl; +ClientTest::ShowCollections(std::vector& collection_array) { + milvus::Status stat = conn_->ShowCollections(collection_array); + std::cout << "ShowCollections function call status: " << stat.message() << std::endl; + std::cout << "All collections: " << std::endl; + for (auto& collection : collection_array) { + int64_t entity_count = 0; + stat = conn_->CountCollection(collection, entity_count); + std::cout << "\t" << collection << "(" << entity_count << " entities)" << std::endl; } } void -ClientTest::CreateTable(const std::string& table_name, int64_t dim, milvus::MetricType type) { - milvus::TableSchema tb_schema = {table_name, dim, TABLE_INDEX_FILE_SIZE, type}; - milvus::Status stat = conn_->CreateTable(tb_schema); - std::cout << "CreateTable function call status: " << stat.message() << std::endl; - milvus_sdk::Utils::PrintTableSchema(tb_schema); +ClientTest::CreateCollection(const std::string& collection_name, int64_t dim, milvus::MetricType type) { + milvus::CollectionParam collection_param = {collection_name, dim, COLLECTION_INDEX_FILE_SIZE, type}; + milvus::Status stat = conn_->CreateCollection(collection_param); + std::cout << "CreateCollection function call status: " << stat.message() << std::endl; + milvus_sdk::Utils::PrintCollectionParam(collection_param); - bool has_table = conn_->HasTable(tb_schema.table_name); + bool has_table = conn_->HasCollection(collection_param.collection_name); if (has_table) { - std::cout << "Table is created" << std::endl; + std::cout << "Collection is created" << std::endl; } } void -ClientTest::DescribeTable(const std::string& table_name) { - milvus::TableSchema tb_schema; - milvus::Status stat = conn_->DescribeTable(table_name, tb_schema); - std::cout << "DescribeTable function call status: " << stat.message() << std::endl; - milvus_sdk::Utils::PrintTableSchema(tb_schema); +ClientTest::DescribeCollection(const std::string& collection_name) { + milvus::CollectionParam collection_param; + milvus::Status stat = conn_->DescribeCollection(collection_name, collection_param); + std::cout << "DescribeCollection function call status: " << stat.message() << std::endl; + milvus_sdk::Utils::PrintCollectionParam(collection_param); } void -ClientTest::InsertVectors(const std::string& table_name, int64_t dim) { - for (int i = 0; i < ADD_VECTOR_LOOP; i++) { - std::vector record_array; +ClientTest::InsertEntities(const std::string& collection_name, int64_t dim) { + for (int i = 0; i < ADD_ENTITY_LOOP; i++) { + std::vector entity_array; std::vector record_ids; - int64_t begin_index = i * BATCH_ROW_COUNT; + int64_t begin_index = i * BATCH_ENTITY_COUNT; { // generate vectors - milvus_sdk::TimeRecorder rc("Build vectors No." + std::to_string(i)); - milvus_sdk::Utils::BuildVectors(begin_index, begin_index + BATCH_ROW_COUNT, record_array, record_ids, dim); + milvus_sdk::TimeRecorder rc("Build entities No." + std::to_string(i)); + milvus_sdk::Utils::BuildEntities(begin_index, + begin_index + BATCH_ENTITY_COUNT, + entity_array, + record_ids, + dim); } - std::string title = "Insert " + std::to_string(record_array.size()) + " vectors No." + std::to_string(i); + std::string title = "Insert " + std::to_string(entity_array.size()) + " entities No." + std::to_string(i); milvus_sdk::TimeRecorder rc(title); - milvus::Status stat = conn_->Insert(table_name, "", record_array, record_ids); - std::cout << "InsertVector function call status: " << stat.message() << std::endl; + milvus::Status stat = conn_->Insert(collection_name, "", entity_array, record_ids); + std::cout << "InsertEntities function call status: " << stat.message() << std::endl; std::cout << "Returned id array count: " << record_ids.size() << std::endl; } } void -ClientTest::BuildSearchVectors(int64_t nq, int64_t dim) { - search_record_array_.clear(); +ClientTest::BuildSearchEntities(int64_t nq, int64_t dim) { + search_entity_array_.clear(); search_id_array_.clear(); for (int64_t i = 0; i < nq; i++) { - std::vector record_array; + std::vector entity_array; std::vector record_ids; - int64_t index = i * BATCH_ROW_COUNT + SEARCH_TARGET; - milvus_sdk::Utils::BuildVectors(index, index + 1, record_array, record_ids, dim); - search_record_array_.push_back(std::make_pair(record_ids[0], record_array[0])); + int64_t index = i * BATCH_ENTITY_COUNT + SEARCH_TARGET; + milvus_sdk::Utils::BuildEntities(index, index + 1, entity_array, record_ids, dim); + search_entity_array_.push_back(std::make_pair(record_ids[0], entity_array[0])); search_id_array_.push_back(record_ids[0]); } } void -ClientTest::Flush(const std::string& table_name) { +ClientTest::Flush(const std::string& collection_name) { milvus_sdk::TimeRecorder rc("Flush"); - milvus::Status stat = conn_->FlushTable(table_name); - std::cout << "FlushTable function call status: " << stat.message() << std::endl; + milvus::Status stat = conn_->FlushCollection(collection_name); + std::cout << "FlushCollection function call status: " << stat.message() << std::endl; } void -ClientTest::ShowTableInfo(const std::string& table_name) { - milvus::TableInfo table_info; - milvus::Status stat = conn_->ShowTableInfo(table_name, table_info); - milvus_sdk::Utils::PrintTableInfo(table_info); - std::cout << "ShowTableInfo function call status: " << stat.message() << std::endl; +ClientTest::ShowCollectionInfo(const std::string& collection_name) { + milvus::CollectionInfo collection_info; + milvus::Status stat = conn_->ShowCollectionInfo(collection_name, collection_info); + milvus_sdk::Utils::PrintCollectionInfo(collection_info); + std::cout << "ShowCollectionInfo function call status: " << stat.message() << std::endl; } void -ClientTest::GetVectorById(const std::string& table_name, int64_t id) { - milvus::RowRecord vector_data; - milvus::Status stat = conn_->GetVectorByID(table_name, id, vector_data); - std::cout << "The vector " << id << " has " << vector_data.float_data.size() << " float elements" << std::endl; - std::cout << "GetVectorByID function call status: " << stat.message() << std::endl; +ClientTest::GetEntityById(const std::string& collection_name, int64_t id) { + milvus::Entity entity; + milvus::Status stat = conn_->GetEntityByID(collection_name, id, entity); + std::cout << "The entity " << id << " has " << entity.float_data.size() << " float elements" << std::endl; + std::cout << "GetEntityById function call status: " << stat.message() << std::endl; } void -ClientTest::SearchVectors(const std::string& table_name, int64_t topk, int64_t nprobe) { +ClientTest::SearchEntities(const std::string& collection_name, int64_t topk, int64_t nprobe) { std::vector partition_tags; milvus::TopKQueryResult topk_query_result; - milvus_sdk::Utils::DoSearch(conn_, table_name, partition_tags, topk, nprobe, search_record_array_, + milvus_sdk::Utils::DoSearch(conn_, collection_name, partition_tags, topk, nprobe, search_entity_array_, topk_query_result); } void -ClientTest::CreateIndex(const std::string& table_name, milvus::IndexType type, int64_t nlist) { +ClientTest::CreateIndex(const std::string& collection_name, milvus::IndexType type, int64_t nlist) { milvus_sdk::TimeRecorder rc("Create index"); std::cout << "Wait until create all index done" << std::endl; JSON json_params = {{"nlist", nlist}}; - milvus::IndexParam index1 = {table_name, type, json_params.dump()}; + milvus::IndexParam index1 = {collection_name, type, json_params.dump()}; milvus_sdk::Utils::PrintIndexParam(index1); milvus::Status stat = conn_->CreateIndex(index1); std::cout << "CreateIndex function call status: " << stat.message() << std::endl; milvus::IndexParam index2; - stat = conn_->DescribeIndex(table_name, index2); + stat = conn_->DescribeIndex(collection_name, index2); std::cout << "DescribeIndex function call status: " << stat.message() << std::endl; milvus_sdk::Utils::PrintIndexParam(index2); } void -ClientTest::PreloadTable(const std::string& table_name) { - milvus::Status stat = conn_->PreloadTable(table_name); - std::cout << "PreloadTable function call status: " << stat.message() << std::endl; +ClientTest::PreloadCollection(const std::string& collection_name) { + milvus::Status stat = conn_->PreloadCollection(collection_name); + std::cout << "PreloadCollection function call status: " << stat.message() << std::endl; } void -ClientTest::DeleteByIds(const std::string& table_name, const std::vector& id_array) { - milvus::Status stat = conn_->DeleteByID(table_name, id_array); +ClientTest::DeleteByIds(const std::string& collection_name, const std::vector& id_array) { + std::cout << "Delete entity: "; + for (auto id : id_array) { + std::cout << "\t" << id; + } + std::cout << std::endl; + + milvus::Status stat = conn_->DeleteByID(collection_name, id_array); std::cout << "DeleteByID function call status: " << stat.message() << std::endl; { milvus_sdk::TimeRecorder rc("Flush"); - stat = conn_->FlushTable(table_name); - std::cout << "FlushTable function call status: " << stat.message() << std::endl; + stat = conn_->FlushCollection(collection_name); + std::cout << "FlushCollection function call status: " << stat.message() << std::endl; } { // compact table milvus_sdk::TimeRecorder rc1("Compact"); - stat = conn_->CompactTable(table_name); - std::cout << "CompactTable function call status: " << stat.message() << std::endl; + stat = conn_->CompactCollection(collection_name); + std::cout << "CompactCollection function call status: " << stat.message() << std::endl; } } void -ClientTest::DropIndex(const std::string& table_name) { - milvus::Status stat = conn_->DropIndex(table_name); +ClientTest::DropIndex(const std::string& collection_name) { + milvus::Status stat = conn_->DropIndex(collection_name); std::cout << "DropIndex function call status: " << stat.message() << std::endl; int64_t row_count = 0; - stat = conn_->CountTable(table_name, row_count); - std::cout << table_name << "(" << row_count << " rows)" << std::endl; + stat = conn_->CountCollection(collection_name, row_count); + std::cout << collection_name << "(" << row_count << " rows)" << std::endl; } void -ClientTest::DropTable(const std::string& table_name) { - milvus::Status stat = conn_->DropTable(table_name); - std::cout << "DropTable function call status: " << stat.message() << std::endl; +ClientTest::DropCollection(const std::string& collection_name) { + milvus::Status stat = conn_->DropCollection(collection_name); + std::cout << "DropCollection function call status: " << stat.message() << std::endl; } void ClientTest::Test() { - std::string table_name = TABLE_NAME; - int64_t dim = TABLE_DIMENSION; - milvus::MetricType metric_type = TABLE_METRIC_TYPE; + std::string collection_name = COLLECTION_NAME; + int64_t dim = COLLECTION_DIMENSION; + milvus::MetricType metric_type = COLLECTION_METRIC_TYPE; ShowServerVersion(); ShowSdkVersion(); std::vector table_array; - ShowTables(table_array); + ShowCollections(table_array); - CreateTable(table_name, dim, metric_type); - DescribeTable(table_name); + CreateCollection(collection_name, dim, metric_type); + DescribeCollection(collection_name); - InsertVectors(table_name, dim); - BuildSearchVectors(NQ, dim); - Flush(table_name); - ShowTableInfo(table_name); + InsertEntities(collection_name, dim); + BuildSearchEntities(NQ, dim); + Flush(collection_name); + ShowCollectionInfo(collection_name); - GetVectorById(table_name, search_id_array_[0]); - SearchVectors(table_name, TOP_K, NPROBE); + GetEntityById(collection_name, search_id_array_[0]); + SearchEntities(collection_name, TOP_K, NPROBE); - CreateIndex(table_name, INDEX_TYPE, NLIST); - ShowTableInfo(table_name); + CreateIndex(collection_name, INDEX_TYPE, NLIST); + ShowCollectionInfo(collection_name); - PreloadTable(table_name); + PreloadCollection(collection_name); std::vector delete_ids = {search_id_array_[0], search_id_array_[1]}; - DeleteByIds(table_name, delete_ids); - SearchVectors(table_name, TOP_K, NPROBE); + DeleteByIds(collection_name, delete_ids); + SearchEntities(collection_name, TOP_K, NPROBE); // this line get two search error since we delete two entities - DropIndex(table_name); - DropTable(table_name); + DropIndex(collection_name); + DropCollection(collection_name); } diff --git a/sdk/examples/simple/src/ClientTest.h b/sdk/examples/simple/src/ClientTest.h index 0c62b29729..2e676353eb 100644 --- a/sdk/examples/simple/src/ClientTest.h +++ b/sdk/examples/simple/src/ClientTest.h @@ -34,49 +34,49 @@ class ClientTest { ShowSdkVersion(); void - ShowTables(std::vector&); + ShowCollections(std::vector&); void - CreateTable(const std::string&, int64_t, milvus::MetricType); + CreateCollection(const std::string&, int64_t, milvus::MetricType); void - DescribeTable(const std::string&); + DescribeCollection(const std::string&); void - InsertVectors(const std::string&, int64_t); + InsertEntities(const std::string&, int64_t); void - BuildSearchVectors(int64_t, int64_t); + BuildSearchEntities(int64_t, int64_t); void Flush(const std::string&); void - ShowTableInfo(const std::string&); + ShowCollectionInfo(const std::string&); void - GetVectorById(const std::string&, int64_t); + GetEntityById(const std::string&, int64_t); void - SearchVectors(const std::string&, int64_t, int64_t); + SearchEntities(const std::string&, int64_t, int64_t); void CreateIndex(const std::string&, milvus::IndexType, int64_t); void - PreloadTable(const std::string&); + PreloadCollection(const std::string&); void - DeleteByIds(const std::string&, const std::vector&); + DeleteByIds(const std::string&, const std::vector& id_array); void DropIndex(const std::string&); void - DropTable(const std::string&); + DropCollection(const std::string&); private: std::shared_ptr conn_; - std::vector> search_record_array_; + std::vector> search_entity_array_; std::vector search_id_array_; }; diff --git a/sdk/examples/utils/Utils.cpp b/sdk/examples/utils/Utils.cpp index c9294cb3aa..ed709f4ffe 100644 --- a/sdk/examples/utils/Utils.cpp +++ b/sdk/examples/utils/Utils.cpp @@ -64,8 +64,8 @@ Utils::Sleep(int seconds) { } const std::string& -Utils::GenTableName() { - static std::string s_id("tbl_" + CurrentTime()); +Utils::GenCollectionName() { + static std::string s_id("C_" + CurrentTime()); return s_id; } @@ -97,19 +97,19 @@ Utils::IndexTypeName(const milvus::IndexType& index_type) { } void -Utils::PrintTableSchema(const milvus::TableSchema& tb_schema) { +Utils::PrintCollectionParam(const milvus::CollectionParam& collection_param) { BLOCK_SPLITER - std::cout << "Table name: " << tb_schema.table_name << std::endl; - std::cout << "Table dimension: " << tb_schema.dimension << std::endl; - std::cout << "Table index file size: " << tb_schema.index_file_size << std::endl; - std::cout << "Table metric type: " << MetricTypeName(tb_schema.metric_type) << std::endl; + std::cout << "Collection name: " << collection_param.collection_name << std::endl; + std::cout << "Collection dimension: " << collection_param.dimension << std::endl; + std::cout << "Collection index file size: " << collection_param.index_file_size << std::endl; + std::cout << "Collection metric type: " << MetricTypeName(collection_param.metric_type) << std::endl; BLOCK_SPLITER } void Utils::PrintPartitionParam(const milvus::PartitionParam& partition_param) { BLOCK_SPLITER - std::cout << "Table name: " << partition_param.table_name << std::endl; + std::cout << "Collection name: " << partition_param.collection_name << std::endl; std::cout << "Partition tag: " << partition_param.partition_tag << std::endl; BLOCK_SPLITER } @@ -117,40 +117,40 @@ Utils::PrintPartitionParam(const milvus::PartitionParam& partition_param) { void Utils::PrintIndexParam(const milvus::IndexParam& index_param) { BLOCK_SPLITER - std::cout << "Index table name: " << index_param.table_name << std::endl; + std::cout << "Index collection name: " << index_param.collection_name << std::endl; std::cout << "Index type: " << IndexTypeName(index_param.index_type) << std::endl; std::cout << "Index extra_params: " << index_param.extra_params << std::endl; BLOCK_SPLITER } void -Utils::BuildVectors(int64_t from, int64_t to, std::vector& vector_record_array, - std::vector& record_ids, int64_t dimension) { +Utils::BuildEntities(int64_t from, int64_t to, std::vector& entity_array, + std::vector& entity_ids, int64_t dimension) { if (to <= from) { return; } - vector_record_array.clear(); - record_ids.clear(); + entity_array.clear(); + entity_ids.clear(); for (int64_t k = from; k < to; k++) { - milvus::RowRecord record; - record.float_data.resize(dimension); + milvus::Entity entity; + entity.float_data.resize(dimension); for (int64_t i = 0; i < dimension; i++) { - record.float_data[i] = (float)(k % (i + 1)); + entity.float_data[i] = (float)(k % (i + 1)); } - vector_record_array.emplace_back(record); - record_ids.push_back(k); + entity_array.emplace_back(entity); + entity_ids.push_back(k); } } void -Utils::PrintSearchResult(const std::vector>& search_record_array, +Utils::PrintSearchResult(const std::vector>& entity_array, const milvus::TopKQueryResult& topk_query_result) { BLOCK_SPLITER std::cout << "Returned result count: " << topk_query_result.size() << std::endl; - if (topk_query_result.size() != search_record_array.size()) { + if (topk_query_result.size() != entity_array.size()) { std::cout << "ERROR: Returned result count not equal nq" << std::endl; return; } @@ -158,8 +158,8 @@ Utils::PrintSearchResult(const std::vector for (size_t i = 0; i < topk_query_result.size(); i++) { const milvus::QueryResult& one_result = topk_query_result[i]; size_t topk = one_result.ids.size(); - auto search_id = search_record_array[i].first; - std::cout << "No." << i << " vector " << search_id << " top " << topk << " search result:" << std::endl; + auto search_id = entity_array[i].first; + std::cout << "No." << i << " entity " << search_id << " top " << topk << " search result:" << std::endl; for (size_t j = 0; j < topk; j++) { std::cout << "\t" << one_result.ids[j] << "\t" << one_result.distances[j] << std::endl; } @@ -168,13 +168,13 @@ Utils::PrintSearchResult(const std::vector } void -Utils::CheckSearchResult(const std::vector>& search_record_array, +Utils::CheckSearchResult(const std::vector>& entity_array, const milvus::TopKQueryResult& topk_query_result) { BLOCK_SPLITER size_t nq = topk_query_result.size(); for (size_t i = 0; i < nq; i++) { const milvus::QueryResult& one_result = topk_query_result[i]; - auto search_id = search_record_array[i].first; + auto search_id = entity_array[i].first; uint64_t match_index = one_result.ids.size(); for (uint64_t index = 0; index < one_result.ids.size(); index++) { @@ -195,15 +195,15 @@ Utils::CheckSearchResult(const std::vector } void -Utils::DoSearch(std::shared_ptr conn, const std::string& table_name, +Utils::DoSearch(std::shared_ptr conn, const std::string& collection_name, const std::vector& partition_tags, int64_t top_k, int64_t nprobe, - const std::vector>& search_record_array, + const std::vector>& entity_array, milvus::TopKQueryResult& topk_query_result) { topk_query_result.clear(); - std::vector record_array; - for (auto& pair : search_record_array) { - record_array.push_back(pair.second); + std::vector temp_entity_array; + for (auto& pair : entity_array) { + temp_entity_array.push_back(pair.second); } { @@ -211,28 +211,33 @@ Utils::DoSearch(std::shared_ptr conn, const std::string& tab JSON json_params = {{"nprobe", nprobe}}; milvus_sdk::TimeRecorder rc("search"); milvus::Status stat = - conn->Search(table_name, partition_tags, record_array, top_k, json_params.dump(), topk_query_result); - std::cout << "SearchVector function call status: " << stat.message() << std::endl; + conn->Search(collection_name, + partition_tags, + temp_entity_array, + top_k, + json_params.dump(), + topk_query_result); + std::cout << "Search function call status: " << stat.message() << std::endl; BLOCK_SPLITER } - PrintSearchResult(search_record_array, topk_query_result); - CheckSearchResult(search_record_array, topk_query_result); + PrintSearchResult(entity_array, topk_query_result); + CheckSearchResult(entity_array, topk_query_result); } void PrintPartitionStat(const milvus::PartitionStat& partition_stat) { - std::cout << "\tPartition " << partition_stat.tag << " row count: " << partition_stat.row_count << std::endl; + std::cout << "\tPartition " << partition_stat.tag << " entity count: " << partition_stat.row_count << std::endl; for (auto& seg_stat : partition_stat.segments_stat) { - std::cout << "\t\tsegment " << seg_stat.segment_name << " row count: " << seg_stat.row_count + std::cout << "\t\tsegment " << seg_stat.segment_name << " entity count: " << seg_stat.row_count << " index: " << seg_stat.index_name << " data size: " << seg_stat.data_size << std::endl; } } void -Utils::PrintTableInfo(const milvus::TableInfo& info) { +Utils::PrintCollectionInfo(const milvus::CollectionInfo& info) { BLOCK_SPLITER - std::cout << "Table " << " total row count: " << info.total_row_count << std::endl; + std::cout << "Collection " << " total entity count: " << info.total_row_count << std::endl; for (const milvus::PartitionStat& partition_stat : info.partitions_stat) { PrintPartitionStat(partition_stat); } diff --git a/sdk/examples/utils/Utils.h b/sdk/examples/utils/Utils.h index 4f33dafc33..f83ee069b7 100644 --- a/sdk/examples/utils/Utils.h +++ b/sdk/examples/utils/Utils.h @@ -32,7 +32,7 @@ class Utils { CurrentTmDate(int64_t offset_day = 0); static const std::string& - GenTableName(); + GenCollectionName(); static void Sleep(int seconds); @@ -44,7 +44,7 @@ class Utils { IndexTypeName(const milvus::IndexType& index_type); static void - PrintTableSchema(const milvus::TableSchema& tb_schema); + PrintCollectionParam(const milvus::CollectionParam& collection_param); static void PrintPartitionParam(const milvus::PartitionParam& partition_param); @@ -53,25 +53,25 @@ class Utils { PrintIndexParam(const milvus::IndexParam& index_param); static void - BuildVectors(int64_t from, int64_t to, std::vector& vector_record_array, - std::vector& record_ids, int64_t dimension); + BuildEntities(int64_t from, int64_t to, std::vector& entity_array, + std::vector& entity_ids, int64_t dimension); static void - PrintSearchResult(const std::vector>& search_record_array, + PrintSearchResult(const std::vector>& entity_array, const milvus::TopKQueryResult& topk_query_result); static void - CheckSearchResult(const std::vector>& search_record_array, + CheckSearchResult(const std::vector>& entity_array, const milvus::TopKQueryResult& topk_query_result); static void - DoSearch(std::shared_ptr conn, const std::string& table_name, + DoSearch(std::shared_ptr conn, const std::string& collection_name, const std::vector& partition_tags, int64_t top_k, int64_t nprobe, - const std::vector>& search_record_array, + const std::vector>& entity_array, milvus::TopKQueryResult& topk_query_result); static void - PrintTableInfo(const milvus::TableInfo& info); + PrintCollectionInfo(const milvus::CollectionInfo& collection_info); }; } // namespace milvus_sdk diff --git a/sdk/grpc/ClientProxy.cpp b/sdk/grpc/ClientProxy.cpp index 6986355b4e..219f78e4ad 100644 --- a/sdk/grpc/ClientProxy.cpp +++ b/sdk/grpc/ClientProxy.cpp @@ -31,12 +31,12 @@ UriCheck(const std::string& uri) { template void -ConstructSearchParam(const std::string& table_name, +ConstructSearchParam(const std::string& collection_name, const std::vector& partition_tag_array, int64_t topk, const std::string& extra_params, T& search_param) { - search_param.set_table_name(table_name); + search_param.set_table_name(collection_name); search_param.set_topk(topk); milvus::grpc::KeyValuePair* kv = search_param.add_extra_params(); kv->set_key(EXTRA_PARAM_KEY); @@ -48,7 +48,7 @@ ConstructSearchParam(const std::string& table_name, } void -CopyRowRecord(::milvus::grpc::RowRecord* target, const RowRecord& src) { +CopyRowRecord(::milvus::grpc::RowRecord* target, const Entity& src) { if (!src.float_data.empty()) { auto vector_data = target->mutable_float_data(); vector_data->Resize(static_cast(src.float_data.size()), 0.0); @@ -86,7 +86,7 @@ ClientProxy::Connect(const ConnectParam& param) { return Status::OK(); } - std::string reason = "connect Failed!"; + std::string reason = "Connect failed!"; connected_ = false; return Status(StatusCode::NotConnected, reason); } @@ -111,7 +111,7 @@ ClientProxy::Connected() const { std::string info; return client_ptr_->Cmd("", info); } catch (std::exception& ex) { - return Status(StatusCode::NotConnected, "connection lost: " + std::string(ex.what())); + return Status(StatusCode::NotConnected, "Connection lost: " + std::string(ex.what())); } } @@ -132,262 +132,6 @@ ClientProxy::ClientVersion() const { return MILVUS_SDK_VERSION; } -Status -ClientProxy::CreateTable(const TableSchema& param) { - try { - ::milvus::grpc::TableSchema schema; - schema.set_table_name(param.table_name); - schema.set_dimension(param.dimension); - schema.set_index_file_size(param.index_file_size); - schema.set_metric_type(static_cast(param.metric_type)); - - return client_ptr_->CreateTable(schema); - } catch (std::exception& ex) { - return Status(StatusCode::UnknownError, "Failed to create table: " + std::string(ex.what())); - } -} - -bool -ClientProxy::HasTable(const std::string& table_name) { - Status status = Status::OK(); - ::milvus::grpc::TableName grpc_table_name; - grpc_table_name.set_table_name(table_name); - bool result = client_ptr_->HasTable(grpc_table_name, status); - return result; -} - -Status -ClientProxy::DropTable(const std::string& table_name) { - try { - ::milvus::grpc::TableName grpc_table_name; - grpc_table_name.set_table_name(table_name); - return client_ptr_->DropTable(grpc_table_name); - } catch (std::exception& ex) { - return Status(StatusCode::UnknownError, "Failed to drop table: " + std::string(ex.what())); - } -} - -Status -ClientProxy::CreateIndex(const IndexParam& index_param) { - try { - ::milvus::grpc::IndexParam grpc_index_param; - grpc_index_param.set_table_name(index_param.table_name); - grpc_index_param.set_index_type(static_cast(index_param.index_type)); - milvus::grpc::KeyValuePair* kv = grpc_index_param.add_extra_params(); - kv->set_key(EXTRA_PARAM_KEY); - kv->set_value(index_param.extra_params); - return client_ptr_->CreateIndex(grpc_index_param); - } catch (std::exception& ex) { - return Status(StatusCode::UnknownError, "Failed to build index: " + std::string(ex.what())); - } -} - -Status -ClientProxy::Insert(const std::string& table_name, const std::string& partition_tag, - const std::vector& record_array, std::vector& id_array) { - Status status = Status::OK(); - try { - ::milvus::grpc::InsertParam insert_param; - insert_param.set_table_name(table_name); - insert_param.set_partition_tag(partition_tag); - - for (auto& record : record_array) { - ::milvus::grpc::RowRecord* grpc_record = insert_param.add_row_record_array(); - CopyRowRecord(grpc_record, record); - } - - // Single thread - ::milvus::grpc::VectorIds vector_ids; - if (!id_array.empty()) { - /* set user's ids */ - auto row_ids = insert_param.mutable_row_id_array(); - row_ids->Resize(static_cast(id_array.size()), -1); - memcpy(row_ids->mutable_data(), id_array.data(), id_array.size() * sizeof(int64_t)); - status = client_ptr_->Insert(insert_param, vector_ids); - } else { - status = client_ptr_->Insert(insert_param, vector_ids); - /* return Milvus generated ids back to user */ - id_array.insert(id_array.end(), vector_ids.vector_id_array().begin(), vector_ids.vector_id_array().end()); - } - } catch (std::exception& ex) { - return Status(StatusCode::UnknownError, "Failed to add vector: " + std::string(ex.what())); - } - - return status; -} - -Status -ClientProxy::GetVectorByID(const std::string& table_name, int64_t vector_id, RowRecord& vector_data) { - try { - ::milvus::grpc::VectorIdentity vector_identity; - vector_identity.set_table_name(table_name); - vector_identity.set_id(vector_id); - - ::milvus::grpc::VectorData grpc_data; - Status status = client_ptr_->GetVectorByID(vector_identity, grpc_data); - if (!status.ok()) { - return status; - } - - int float_size = grpc_data.vector_data().float_data_size(); - if (float_size > 0) { - vector_data.float_data.resize(float_size); - memcpy(vector_data.float_data.data(), grpc_data.vector_data().float_data().data(), - float_size * sizeof(float)); - } - - auto byte_size = grpc_data.vector_data().binary_data().length(); - if (byte_size > 0) { - vector_data.binary_data.resize(byte_size); - memcpy(vector_data.binary_data.data(), grpc_data.vector_data().binary_data().data(), byte_size); - } - - return status; - } catch (std::exception& ex) { - return Status(StatusCode::UnknownError, "Failed to get vector by id: " + std::string(ex.what())); - } -} - -Status -ClientProxy::GetIDsInSegment(const std::string& table_name, const std::string& segment_name, - std::vector& id_array) { - try { - ::milvus::grpc::GetVectorIDsParam param; - param.set_table_name(table_name); - param.set_segment_name(segment_name); - - ::milvus::grpc::VectorIds vector_ids; - Status status = client_ptr_->GetIDsInSegment(param, vector_ids); - if (!status.ok()) { - return status; - } - - id_array.insert(id_array.end(), vector_ids.vector_id_array().begin(), vector_ids.vector_id_array().end()); - - return status; - } catch (std::exception& ex) { - return Status(StatusCode::UnknownError, "Failed to get vector by id: " + std::string(ex.what())); - } -} - -Status -ClientProxy::Search(const std::string& table_name, const std::vector& partition_tag_array, - const std::vector& query_record_array, int64_t topk, const std::string& extra_params, - TopKQueryResult& topk_query_result) { - try { - // step 1: convert vectors data - ::milvus::grpc::SearchParam search_param; - ConstructSearchParam(table_name, - partition_tag_array, - topk, - extra_params, - search_param); - - for (auto& record : query_record_array) { - ::milvus::grpc::RowRecord* row_record = search_param.add_query_record_array(); - CopyRowRecord(row_record, record); - } - - // step 2: search vectors - ::milvus::grpc::TopKQueryResult result; - Status status = client_ptr_->Search(search_param, result); - if (result.row_num() == 0) { - return status; - } - - // step 3: convert result array - topk_query_result.reserve(result.row_num()); - int64_t nq = result.row_num(); - int64_t topk = result.ids().size() / nq; - for (int64_t i = 0; i < result.row_num(); i++) { - milvus::QueryResult one_result; - one_result.ids.resize(topk); - one_result.distances.resize(topk); - memcpy(one_result.ids.data(), result.ids().data() + topk * i, topk * sizeof(int64_t)); - memcpy(one_result.distances.data(), result.distances().data() + topk * i, topk * sizeof(float)); - topk_query_result.emplace_back(one_result); - } - - return status; - } catch (std::exception& ex) { - return Status(StatusCode::UnknownError, "Failed to search vectors: " + std::string(ex.what())); - } -} - -Status -ClientProxy::DescribeTable(const std::string& table_name, TableSchema& table_schema) { - try { - ::milvus::grpc::TableSchema grpc_schema; - - Status status = client_ptr_->DescribeTable(table_name, grpc_schema); - - table_schema.table_name = grpc_schema.table_name(); - table_schema.dimension = grpc_schema.dimension(); - table_schema.index_file_size = grpc_schema.index_file_size(); - table_schema.metric_type = static_cast(grpc_schema.metric_type()); - - return status; - } catch (std::exception& ex) { - return Status(StatusCode::UnknownError, "Failed to describe table: " + std::string(ex.what())); - } -} - -Status -ClientProxy::CountTable(const std::string& table_name, int64_t& row_count) { - try { - Status status; - ::milvus::grpc::TableName grpc_table_name; - grpc_table_name.set_table_name(table_name); - row_count = client_ptr_->CountTable(grpc_table_name, status); - return status; - } catch (std::exception& ex) { - return Status(StatusCode::UnknownError, "Failed to show tables: " + std::string(ex.what())); - } -} - -Status -ClientProxy::ShowTables(std::vector& table_array) { - try { - Status status; - milvus::grpc::TableNameList table_name_list; - status = client_ptr_->ShowTables(table_name_list); - - table_array.resize(table_name_list.table_names_size()); - for (uint64_t i = 0; i < table_name_list.table_names_size(); ++i) { - table_array[i] = table_name_list.table_names(i); - } - return status; - } catch (std::exception& ex) { - return Status(StatusCode::UnknownError, "Failed to show tables: " + std::string(ex.what())); - } -} - -Status -ClientProxy::ShowTableInfo(const std::string& table_name, TableInfo& table_info) { - try { - Status status; - ::milvus::grpc::TableName grpc_table_name; - grpc_table_name.set_table_name(table_name); - milvus::grpc::TableInfo grpc_table_info; - status = client_ptr_->ShowTableInfo(grpc_table_name, grpc_table_info); - - // get native info - table_info.total_row_count = grpc_table_info.total_row_count(); - - // get partitions info - for (int i = 0; i < grpc_table_info.partitions_stat_size(); i++) { - auto& grpc_partition_stat = grpc_table_info.partitions_stat(i); - PartitionStat partition_stat; - ConstructPartitionStat(grpc_partition_stat, partition_stat); - table_info.partitions_stat.emplace_back(partition_stat); - } - - return status; - } catch (std::exception& ex) { - return Status(StatusCode::UnknownError, "Failed to show table info: " + std::string(ex.what())); - } -} - std::string ClientProxy::ServerVersion() const { Status status = Status::OK(); @@ -415,112 +159,6 @@ ClientProxy::ServerStatus() const { } } -Status -ClientProxy::DeleteByID(const std::string& table_name, const std::vector& id_array) { - try { - ::milvus::grpc::DeleteByIDParam delete_by_id_param; - delete_by_id_param.set_table_name(table_name); - - for (auto id : id_array) { - delete_by_id_param.add_id_array(id); - } - - return client_ptr_->DeleteByID(delete_by_id_param); - } catch (std::exception& ex) { - return Status(StatusCode::UnknownError, "Failed to delete by range: " + std::string(ex.what())); - } -} - -Status -ClientProxy::PreloadTable(const std::string& table_name) const { - try { - ::milvus::grpc::TableName grpc_table_name; - grpc_table_name.set_table_name(table_name); - Status status = client_ptr_->PreloadTable(grpc_table_name); - return status; - } catch (std::exception& ex) { - return Status(StatusCode::UnknownError, "Failed to preload tables: " + std::string(ex.what())); - } -} - -Status -ClientProxy::DescribeIndex(const std::string& table_name, IndexParam& index_param) const { - try { - ::milvus::grpc::TableName grpc_table_name; - grpc_table_name.set_table_name(table_name); - - ::milvus::grpc::IndexParam grpc_index_param; - Status status = client_ptr_->DescribeIndex(grpc_table_name, grpc_index_param); - index_param.index_type = static_cast(grpc_index_param.index_type()); - - for (int i = 0; i < grpc_index_param.extra_params_size(); i++) { - const milvus::grpc::KeyValuePair& kv = grpc_index_param.extra_params(i); - if (kv.key() == EXTRA_PARAM_KEY) { - index_param.extra_params = kv.value(); - } - } - - return status; - } catch (std::exception& ex) { - return Status(StatusCode::UnknownError, "Failed to describe index: " + std::string(ex.what())); - } -} - -Status -ClientProxy::DropIndex(const std::string& table_name) const { - try { - ::milvus::grpc::TableName grpc_table_name; - grpc_table_name.set_table_name(table_name); - Status status = client_ptr_->DropIndex(grpc_table_name); - return status; - } catch (std::exception& ex) { - return Status(StatusCode::UnknownError, "Failed to drop index: " + std::string(ex.what())); - } -} - -Status -ClientProxy::CreatePartition(const PartitionParam& partition_param) { - try { - ::milvus::grpc::PartitionParam grpc_partition_param; - grpc_partition_param.set_table_name(partition_param.table_name); - grpc_partition_param.set_tag(partition_param.partition_tag); - Status status = client_ptr_->CreatePartition(grpc_partition_param); - return status; - } catch (std::exception& ex) { - return Status(StatusCode::UnknownError, "Failed to create partition: " + std::string(ex.what())); - } -} - -Status -ClientProxy::ShowPartitions(const std::string& table_name, PartitionTagList& partition_tag_array) const { - try { - ::milvus::grpc::TableName grpc_table_name; - grpc_table_name.set_table_name(table_name); - ::milvus::grpc::PartitionList grpc_partition_list; - Status status = client_ptr_->ShowPartitions(grpc_table_name, grpc_partition_list); - partition_tag_array.resize(grpc_partition_list.partition_tag_array_size()); - for (uint64_t i = 0; i < grpc_partition_list.partition_tag_array_size(); ++i) { - partition_tag_array[i] = grpc_partition_list.partition_tag_array(i); - } - return status; - } catch (std::exception& ex) { - return Status(StatusCode::UnknownError, "Failed to show partitions: " + std::string(ex.what())); - } -} - -Status -ClientProxy::DropPartition(const PartitionParam& partition_param) { - try { - ::milvus::grpc::PartitionParam grpc_partition_param; - grpc_partition_param.set_table_name(partition_param.table_name); - grpc_partition_param.set_tag(partition_param.partition_tag); - Status status = client_ptr_->DropPartition(grpc_partition_param); - return status; - } catch (std::exception& ex) { - return Status(StatusCode::UnknownError, "Failed to drop partition: " + std::string(ex.what())); - } -} - Status ClientProxy::GetConfig(const std::string& node_name, std::string& value) const { try { @@ -541,12 +179,373 @@ ClientProxy::SetConfig(const std::string& node_name, const std::string& value) c } Status -ClientProxy::FlushTable(const std::string& table_name) { +ClientProxy::CreateCollection(const CollectionParam& param) { + try { + ::milvus::grpc::TableSchema schema; + schema.set_table_name(param.collection_name); + schema.set_dimension(param.dimension); + schema.set_index_file_size(param.index_file_size); + schema.set_metric_type(static_cast(param.metric_type)); + + return client_ptr_->CreateTable(schema); + } catch (std::exception& ex) { + return Status(StatusCode::UnknownError, "Failed to create collection: " + std::string(ex.what())); + } +} + +bool +ClientProxy::HasCollection(const std::string& collection_name) { + Status status = Status::OK(); + ::milvus::grpc::TableName grpc_collection_name; + grpc_collection_name.set_table_name(collection_name); + bool result = client_ptr_->HasTable(grpc_collection_name, status); + return result; +} + +Status +ClientProxy::DropCollection(const std::string& collection_name) { + try { + ::milvus::grpc::TableName grpc_collection_name; + grpc_collection_name.set_table_name(collection_name); + return client_ptr_->DropTable(grpc_collection_name); + } catch (std::exception& ex) { + return Status(StatusCode::UnknownError, "Failed to drop collection: " + std::string(ex.what())); + } +} + +Status +ClientProxy::CreateIndex(const IndexParam& index_param) { + try { + ::milvus::grpc::IndexParam grpc_index_param; + grpc_index_param.set_table_name(index_param.collection_name); + grpc_index_param.set_index_type(static_cast(index_param.index_type)); + milvus::grpc::KeyValuePair* kv = grpc_index_param.add_extra_params(); + kv->set_key(EXTRA_PARAM_KEY); + kv->set_value(index_param.extra_params); + return client_ptr_->CreateIndex(grpc_index_param); + } catch (std::exception& ex) { + return Status(StatusCode::UnknownError, "Failed to build index: " + std::string(ex.what())); + } +} + +Status +ClientProxy::Insert(const std::string& collection_name, const std::string& partition_tag, + const std::vector& entity_array, std::vector& id_array) { + Status status = Status::OK(); + try { + ::milvus::grpc::InsertParam insert_param; + insert_param.set_table_name(collection_name); + insert_param.set_partition_tag(partition_tag); + + for (auto& entity : entity_array) { + ::milvus::grpc::RowRecord* grpc_record = insert_param.add_row_record_array(); + CopyRowRecord(grpc_record, entity); + } + + // Single thread + ::milvus::grpc::VectorIds vector_ids; + if (!id_array.empty()) { + /* set user's ids */ + auto row_ids = insert_param.mutable_row_id_array(); + row_ids->Resize(static_cast(id_array.size()), -1); + memcpy(row_ids->mutable_data(), id_array.data(), id_array.size() * sizeof(int64_t)); + status = client_ptr_->Insert(insert_param, vector_ids); + } else { + status = client_ptr_->Insert(insert_param, vector_ids); + /* return Milvus generated ids back to user */ + id_array.insert(id_array.end(), vector_ids.vector_id_array().begin(), vector_ids.vector_id_array().end()); + } + } catch (std::exception& ex) { + return Status(StatusCode::UnknownError, "Failed to add entities: " + std::string(ex.what())); + } + + return status; +} + +Status +ClientProxy::GetEntityByID(const std::string& collection_name, int64_t entity_id, Entity& entity_data) { + try { + ::milvus::grpc::VectorIdentity vector_identity; + vector_identity.set_table_name(collection_name); + vector_identity.set_id(entity_id); + + ::milvus::grpc::VectorData grpc_data; + Status status = client_ptr_->GetVectorByID(vector_identity, grpc_data); + if (!status.ok()) { + return status; + } + + int float_size = grpc_data.vector_data().float_data_size(); + if (float_size > 0) { + entity_data.float_data.resize(float_size); + memcpy(entity_data.float_data.data(), grpc_data.vector_data().float_data().data(), + float_size * sizeof(float)); + } + + auto byte_size = grpc_data.vector_data().binary_data().length(); + if (byte_size > 0) { + entity_data.binary_data.resize(byte_size); + memcpy(entity_data.binary_data.data(), grpc_data.vector_data().binary_data().data(), byte_size); + } + + return status; + } catch (std::exception& ex) { + return Status(StatusCode::UnknownError, "Failed to get entity by id: " + std::string(ex.what())); + } +} + +Status +ClientProxy::GetIDsInSegment(const std::string& collection_name, const std::string& segment_name, + std::vector& id_array) { + try { + ::milvus::grpc::GetVectorIDsParam param; + param.set_table_name(collection_name); + param.set_segment_name(segment_name); + + ::milvus::grpc::VectorIds vector_ids; + Status status = client_ptr_->GetIDsInSegment(param, vector_ids); + if (!status.ok()) { + return status; + } + + id_array.insert(id_array.end(), vector_ids.vector_id_array().begin(), vector_ids.vector_id_array().end()); + + return status; + } catch (std::exception& ex) { + return Status(StatusCode::UnknownError, "Failed to get ids from segment: " + std::string(ex.what())); + } +} + +Status +ClientProxy::Search(const std::string& collection_name, const std::vector& partition_tag_array, + const std::vector& entity_array, int64_t topk, const std::string& extra_params, + TopKQueryResult& topk_query_result) { + try { + // step 1: convert vectors data + ::milvus::grpc::SearchParam search_param; + ConstructSearchParam(collection_name, + partition_tag_array, + topk, + extra_params, + search_param); + + for (auto& entity : entity_array) { + ::milvus::grpc::RowRecord* row_record = search_param.add_query_record_array(); + CopyRowRecord(row_record, entity); + } + + // step 2: search vectors + ::milvus::grpc::TopKQueryResult result; + Status status = client_ptr_->Search(search_param, result); + if (result.row_num() == 0) { + return status; + } + + // step 3: convert result array + topk_query_result.reserve(result.row_num()); + int64_t nq = result.row_num(); + int64_t topk = result.ids().size() / nq; + for (int64_t i = 0; i < result.row_num(); i++) { + milvus::QueryResult one_result; + one_result.ids.resize(topk); + one_result.distances.resize(topk); + memcpy(one_result.ids.data(), result.ids().data() + topk * i, topk * sizeof(int64_t)); + memcpy(one_result.distances.data(), result.distances().data() + topk * i, topk * sizeof(float)); + topk_query_result.emplace_back(one_result); + } + + return status; + } catch (std::exception& ex) { + return Status(StatusCode::UnknownError, "Failed to search entities: " + std::string(ex.what())); + } +} + +Status +ClientProxy::DescribeCollection(const std::string& collection_name, CollectionParam& collection_param) { + try { + ::milvus::grpc::TableSchema grpc_schema; + + Status status = client_ptr_->DescribeTable(collection_name, grpc_schema); + + collection_param.collection_name = grpc_schema.table_name(); + collection_param.dimension = grpc_schema.dimension(); + collection_param.index_file_size = grpc_schema.index_file_size(); + collection_param.metric_type = static_cast(grpc_schema.metric_type()); + + return status; + } catch (std::exception& ex) { + return Status(StatusCode::UnknownError, "Failed to describe collection: " + std::string(ex.what())); + } +} + +Status +ClientProxy::CountCollection(const std::string& collection_name, int64_t& row_count) { + try { + Status status; + ::milvus::grpc::TableName grpc_collection_name; + grpc_collection_name.set_table_name(collection_name); + row_count = client_ptr_->CountTable(grpc_collection_name, status); + return status; + } catch (std::exception& ex) { + return Status(StatusCode::UnknownError, "Failed to count collection: " + std::string(ex.what())); + } +} + +Status +ClientProxy::ShowCollections(std::vector& collection_array) { + try { + Status status; + milvus::grpc::TableNameList collection_name_list; + status = client_ptr_->ShowTables(collection_name_list); + + collection_array.resize(collection_name_list.table_names_size()); + for (uint64_t i = 0; i < collection_name_list.table_names_size(); ++i) { + collection_array[i] = collection_name_list.table_names(i); + } + return status; + } catch (std::exception& ex) { + return Status(StatusCode::UnknownError, "Failed to show collections: " + std::string(ex.what())); + } +} + +Status +ClientProxy::ShowCollectionInfo(const std::string& collection_name, CollectionInfo& collection_info) { + try { + Status status; + ::milvus::grpc::TableName grpc_collection_name; + grpc_collection_name.set_table_name(collection_name); + milvus::grpc::TableInfo grpc_collection_info; + status = client_ptr_->ShowTableInfo(grpc_collection_name, grpc_collection_info); + + // get native info + collection_info.total_row_count = grpc_collection_info.total_row_count(); + + // get partitions info + for (int i = 0; i < grpc_collection_info.partitions_stat_size(); i++) { + auto& grpc_partition_stat = grpc_collection_info.partitions_stat(i); + PartitionStat partition_stat; + ConstructPartitionStat(grpc_partition_stat, partition_stat); + collection_info.partitions_stat.emplace_back(partition_stat); + } + + return status; + } catch (std::exception& ex) { + return Status(StatusCode::UnknownError, "Failed to show collection info: " + std::string(ex.what())); + } +} + +Status +ClientProxy::DeleteByID(const std::string& collection_name, const std::vector& id_array) { + try { + ::milvus::grpc::DeleteByIDParam delete_by_id_param; + delete_by_id_param.set_table_name(collection_name); + for (auto id : id_array) { + delete_by_id_param.add_id_array(id); + } + + return client_ptr_->DeleteByID(delete_by_id_param); + } catch (std::exception& ex) { + return Status(StatusCode::UnknownError, "Failed to delete entity id: " + std::string(ex.what())); + } +} + +Status +ClientProxy::PreloadCollection(const std::string& collection_name) const { + try { + ::milvus::grpc::TableName grpc_collection_name; + grpc_collection_name.set_table_name(collection_name); + Status status = client_ptr_->PreloadTable(grpc_collection_name); + return status; + } catch (std::exception& ex) { + return Status(StatusCode::UnknownError, "Failed to preload collection: " + std::string(ex.what())); + } +} + +Status +ClientProxy::DescribeIndex(const std::string& collection_name, IndexParam& index_param) const { + try { + ::milvus::grpc::TableName grpc_collection_name; + grpc_collection_name.set_table_name(collection_name); + + ::milvus::grpc::IndexParam grpc_index_param; + Status status = client_ptr_->DescribeIndex(grpc_collection_name, grpc_index_param); + index_param.index_type = static_cast(grpc_index_param.index_type()); + + for (int i = 0; i < grpc_index_param.extra_params_size(); i++) { + const milvus::grpc::KeyValuePair& kv = grpc_index_param.extra_params(i); + if (kv.key() == EXTRA_PARAM_KEY) { + index_param.extra_params = kv.value(); + } + } + + return status; + } catch (std::exception& ex) { + return Status(StatusCode::UnknownError, "Failed to describe index: " + std::string(ex.what())); + } +} + +Status +ClientProxy::DropIndex(const std::string& collection_name) const { + try { + ::milvus::grpc::TableName grpc_collection_name; + grpc_collection_name.set_table_name(collection_name); + Status status = client_ptr_->DropIndex(grpc_collection_name); + return status; + } catch (std::exception& ex) { + return Status(StatusCode::UnknownError, "Failed to drop index: " + std::string(ex.what())); + } +} + +Status +ClientProxy::CreatePartition(const PartitionParam& partition_param) { + try { + ::milvus::grpc::PartitionParam grpc_partition_param; + grpc_partition_param.set_table_name(partition_param.collection_name); + grpc_partition_param.set_tag(partition_param.partition_tag); + Status status = client_ptr_->CreatePartition(grpc_partition_param); + return status; + } catch (std::exception& ex) { + return Status(StatusCode::UnknownError, "Failed to create partition: " + std::string(ex.what())); + } +} + +Status +ClientProxy::ShowPartitions(const std::string& collection_name, PartitionTagList& partition_tag_array) const { + try { + ::milvus::grpc::TableName grpc_collection_name; + grpc_collection_name.set_table_name(collection_name); + ::milvus::grpc::PartitionList grpc_partition_list; + Status status = client_ptr_->ShowPartitions(grpc_collection_name, grpc_partition_list); + partition_tag_array.resize(grpc_partition_list.partition_tag_array_size()); + for (uint64_t i = 0; i < grpc_partition_list.partition_tag_array_size(); ++i) { + partition_tag_array[i] = grpc_partition_list.partition_tag_array(i); + } + return status; + } catch (std::exception& ex) { + return Status(StatusCode::UnknownError, "Failed to show partitions: " + std::string(ex.what())); + } +} + +Status +ClientProxy::DropPartition(const PartitionParam& partition_param) { + try { + ::milvus::grpc::PartitionParam grpc_partition_param; + grpc_partition_param.set_table_name(partition_param.collection_name); + grpc_partition_param.set_tag(partition_param.partition_tag); + Status status = client_ptr_->DropPartition(grpc_partition_param); + return status; + } catch (std::exception& ex) { + return Status(StatusCode::UnknownError, "Failed to drop partition: " + std::string(ex.what())); + } +} + +Status +ClientProxy::FlushCollection(const std::string& collection_name) { try { std::string dummy; - return client_ptr_->Flush(table_name); + return client_ptr_->Flush(collection_name); } catch (std::exception& ex) { - return Status(StatusCode::UnknownError, "Failed to flush"); + return Status(StatusCode::UnknownError, "Failed to flush collection"); } } @@ -556,19 +555,19 @@ ClientProxy::Flush() { std::string dummy; return client_ptr_->Flush(""); } catch (std::exception& ex) { - return Status(StatusCode::UnknownError, "Failed to flush"); + return Status(StatusCode::UnknownError, "Failed to flush collections"); } } Status -ClientProxy::CompactTable(const std::string& table_name) { +ClientProxy::CompactCollection(const std::string& collection_name) { try { - ::milvus::grpc::TableName grpc_table_name; - grpc_table_name.set_table_name(table_name); - Status status = client_ptr_->Compact(grpc_table_name); + ::milvus::grpc::TableName grpc_collection_name; + grpc_collection_name.set_table_name(collection_name); + Status status = client_ptr_->Compact(grpc_collection_name); return status; } catch (std::exception& ex) { - return Status(StatusCode::UnknownError, "Failed to compact table: " + std::string(ex.what())); + return Status(StatusCode::UnknownError, "Failed to compact collection: " + std::string(ex.what())); } } diff --git a/sdk/grpc/ClientProxy.h b/sdk/grpc/ClientProxy.h index 2c4a195072..f395ce32ca 100644 --- a/sdk/grpc/ClientProxy.h +++ b/sdk/grpc/ClientProxy.h @@ -24,7 +24,7 @@ class ClientProxy : public Connection { public: // Implementations of the Connection interface Status - Connect(const ConnectParam& param) override; + Connect(const ConnectParam& connect_param) override; Status Connect(const std::string& uri) override; @@ -35,46 +35,6 @@ class ClientProxy : public Connection { Status Disconnect() override; - Status - CreateTable(const TableSchema& param) override; - - bool - HasTable(const std::string& table_name) override; - - Status - DropTable(const std::string& table_name) override; - - Status - CreateIndex(const IndexParam& index_param) override; - - Status - Insert(const std::string& table_name, const std::string& partition_tag, const std::vector& record_array, - std::vector& id_array) override; - - Status - GetVectorByID(const std::string& table_name, int64_t vector_id, RowRecord& vector_data) override; - - Status - GetIDsInSegment(const std::string& table_name, const std::string& segment_name, - std::vector& id_array) override; - - Status - Search(const std::string& table_name, const std::vector& partition_tag_array, - const std::vector& query_record_array, int64_t topk, const std::string& extra_params, - TopKQueryResult& topk_query_result) override; - - Status - DescribeTable(const std::string& table_name, TableSchema& table_schema) override; - - Status - CountTable(const std::string& table_name, int64_t& row_count) override; - - Status - ShowTables(std::vector& table_array) override; - - Status - ShowTableInfo(const std::string& table_name, TableInfo& table_info) override; - std::string ClientVersion() const override; @@ -84,27 +44,6 @@ class ClientProxy : public Connection { std::string ServerStatus() const override; - Status - DeleteByID(const std::string& table_name, const std::vector& id_array) override; - - Status - PreloadTable(const std::string& table_name) const override; - - Status - DescribeIndex(const std::string& table_name, IndexParam& index_param) const override; - - Status - DropIndex(const std::string& table_name) const override; - - Status - CreatePartition(const PartitionParam& partition_param) override; - - Status - ShowPartitions(const std::string& table_name, PartitionTagList& partition_tag_array) const override; - - Status - DropPartition(const PartitionParam& partition_param) override; - Status GetConfig(const std::string& node_name, std::string& value) const override; @@ -112,13 +51,76 @@ class ClientProxy : public Connection { SetConfig(const std::string& node_name, const std::string& value) const override; Status - FlushTable(const std::string& table_name) override; + CreateCollection(const CollectionParam& param) override; + + bool + HasCollection(const std::string& collection_name) override; + + Status + DropCollection(const std::string& collection_name) override; + + Status + CreateIndex(const IndexParam& index_param) override; + + Status + Insert(const std::string& collection_name, + const std::string& partition_tag, + const std::vector& entity_array, + std::vector& id_array) override; + + Status + GetEntityByID(const std::string& collection_name, int64_t entity_id, Entity& entity_data) override; + + Status + GetIDsInSegment(const std::string& collection_name, const std::string& segment_name, + std::vector& id_array) override; + + Status + Search(const std::string& collection_name, const std::vector& partition_tag_array, + const std::vector& entity_array, int64_t topk, + const std::string& extra_params, TopKQueryResult& topk_query_result) override; + + Status + DescribeCollection(const std::string& collection_name, CollectionParam& collection_schema) override; + + Status + CountCollection(const std::string& collection_name, int64_t& entity_count) override; + + Status + ShowCollections(std::vector& collection_array) override; + + Status + ShowCollectionInfo(const std::string& collection_name, CollectionInfo& collection_info) override; + + Status + DeleteByID(const std::string& collection_name, const std::vector& id_array) override; + + Status + PreloadCollection(const std::string& collection_name) const override; + + Status + DescribeIndex(const std::string& collection_name, IndexParam& index_param) const override; + + Status + DropIndex(const std::string& collection_name) const override; + + Status + CreatePartition(const PartitionParam& partition_param) override; + + Status + ShowPartitions(const std::string& collection_name, PartitionTagList& partition_tag_array) const override; + + Status + DropPartition(const PartitionParam& partition_param) override; + + Status + FlushCollection(const std::string& collection_name) override; Status Flush() override; Status - CompactTable(const std::string& table_name) override; + CompactCollection(const std::string& collection_name) override; private: std::shared_ptr<::grpc::Channel> channel_; diff --git a/sdk/include/MilvusApi.h b/sdk/include/MilvusApi.h index e899210c91..743248744c 100644 --- a/sdk/include/MilvusApi.h +++ b/sdk/include/MilvusApi.h @@ -84,11 +84,19 @@ using TopKQueryResult = std::vector; ///< Topk query result * @brief Index parameters * Note: extra_params is extra parameters list, it must be json format * For different index type, parameter list is different accordingly, for example: - * FLAT/IVFLAT/SQ8: "{nlist: '16384'}" nlist range:[1, 999999] - * IVFPQ: "{nlist: '16384', m: "12"}" nlist range:[1, 999999] m is decided by dim and have a couple of results. + * FLAT/IVFLAT/SQ8: "{nlist: '16384'}" + * ///< nlist range:[1, 999999] + * IVFPQ: "{nlist: '16384', m: "12"}" + * ///< nlist range:[1, 999999] + * ///< m is decided by dim and have a couple of results. * NSG: "{search_length: '45', out_degree:'50', candidate_pool_size:'300', "knng":'100'}" - * search_length range:[10, 300] out_degree range:[5, 300] candidate_pool_size range:[50, 1000] knng range:[5, 300] - * HNSW "{M: '16', efConstruction:'500'}" M range:[5, 48] efConstruction range:[topk, 4096] + * ///< search_length range:[10, 300] + * ///< out_degree range:[5, 300] + * ///< candidate_pool_size range:[50, 1000] + * ///< knng range:[5, 300] + * HNSW "{M: '16', efConstruction:'500'}" + * ///< M range:[5, 48] + * ///< efConstruction range:[topk, 4096] */ struct IndexParam { std::string collection_name; ///< Collection name for create index @@ -139,18 +147,18 @@ struct CollectionInfo { class Connection { public: /** - * @brief CreateConnection + * @brief Create connection * * Create a connection instance and return it's shared pointer * - * @return Connection instance pointer + * @return connection instance pointer */ static std::shared_ptr Create(); /** - * @brief DestroyConnection + * @brief Destroy connection * * Destroy the connection instance * @@ -174,7 +182,7 @@ class Connection { */ virtual Status - Connect(const ConnectParam& param) = 0; + Connect(const ConnectParam& connect_param) = 0; /** * @brief Connect @@ -190,7 +198,7 @@ class Connection { Connect(const std::string& uri) = 0; /** - * @brief connected + * @brief Connected * * This method is used to test whether server is connected. * @@ -210,7 +218,7 @@ class Connection { Disconnect() = 0; /** - * @brief Give the client version + * @brief Get the client version * * This method is used to give the client version. * @@ -220,7 +228,7 @@ class Connection { ClientVersion() const = 0; /** - * @brief Give the server version + * @brief Get the server version * * This method is used to give the server version. * @@ -229,6 +237,42 @@ class Connection { virtual std::string ServerVersion() const = 0; + /** + * @brief Get the server status + * + * This method is used to give the server status. + * + * @return Server status. + */ + virtual std::string + ServerStatus() const = 0; + + /** + * @brief Get config method + * + * This method is used to set config. + * + * @param node_name, config node name. + * @param value, config value. + * + * @return Indicate if this operation is successful. + */ + virtual Status + GetConfig(const std::string& node_name, std::string& value) const = 0; + + /** + * @brief Set config method + * + * This method is used to set config. + * + * @param node_name, config node name. + * @param value, config value. + * + * @return Indicate if this operation is successful. + */ + virtual Status + SetConfig(const std::string& node_name, const std::string& value) const = 0; + /** * @brief Create collection method * @@ -278,9 +322,9 @@ class Connection { CreateIndex(const IndexParam& index_param) = 0; /** - * @brief Insert entity to table + * @brief Insert entity to collection * - * This method is used to insert vector array to table. + * This method is used to insert vector array to collection. * * @param collection_name, target collection's name. * @param partition_tag, target partition's tag, keep empty if no partition specified. @@ -337,21 +381,24 @@ class Connection { * * @param collection_name, target collection's name. * @param partition_tag_array, target partitions, keep empty if no partition specified. - * @param query_record_array, vectors to be queried. - * @param topk, how many similarity vectors will be returned. + * @param query_entity_array, vectors to be queried. + * @param topk, how many similarity entities will be returned. * @param extra_params, extra search parameters according to different index type, must be json format. * Note: extra_params is extra parameters list, it must be json format, for example: * For different index type, parameter list is different accordingly * FLAT/IVFLAT/SQ8/IVFPQ: "{nprobe: '32'}" + * ///< nprobe range:[1,999999] * NSG: "{search_length:'100'} + * ///< search_length range:[10, 300] * HNSW "{ef: '64'} + * ///< ef range:[k, 4096] * @param topk_query_result, result array. * * @return Indicate if query is successful. */ virtual Status - Search(const std::string& table_name, const PartitionTagList& partition_tag_array, - const std::vector& query_record_array, int64_t topk, + Search(const std::string& collection_name, const PartitionTagList& partition_tag_array, + const std::vector& entity_array, int64_t topk, const std::string& extra_params, TopKQueryResult& topk_query_result) = 0; /** @@ -360,17 +407,17 @@ class Connection { * This method is used to show collection information. * * @param collection_name, target collection's name. - * @param collection_schema, collection_schema is given when operation is successful. + * @param collection_param, collection_param is given when operation is successful. * * @return Indicate if this operation is successful. */ virtual Status - DescribeCollection(const std::string& collection_name, TableSchema& collection_schema) = 0; + DescribeCollection(const std::string& collection_name, CollectionParam& collection_param) = 0; /** - * @brief Get collection row count + * @brief Get collection entity count * - * This method is used to get collection row count. + * This method is used to get collection entity count. * * @param collection_name, target collection's name. * @param entity_count, collection total entity count(including partitions). @@ -393,160 +440,121 @@ class Connection { ShowCollections(std::vector& collection_array) = 0; /** - * @brief Show collections information + * @brief Show collection information * - * This method is used to get detail information of a collections. + * This method is used to get detail information of a collection. * - * @param collections_name, target collections's name. - * @param collections_info, target collections's information + * @param collection_name, target collection's name. + * @param collection_info, target collection's information * * @return Indicate if this operation is successful. */ virtual Status - ShowCollectionInfo(const std::string& collections_name, TableInfo& collections_info) = 0; + ShowCollectionInfo(const std::string& collection_name, CollectionInfo& collection_info) = 0; /** - * @brief Give the server status + * @brief Delete entity by id * - * This method is used to give the server status. + * This method is used to delete entity by id. * - * @return Server status. - */ - virtual std::string - ServerStatus() const = 0; - - /** - * [deprecated] - * @brief delete tables by vector id - * - * This method is used to delete table data by date range. - * - * @param table_name, target table's name. - * @param id_array, vector ids to deleted. + * @param collection_name, target collection's name. + * @param id_array, entity id array to be deleted. * * @return Indicate if this operation is successful. */ virtual Status - DeleteByID(const std::string& table_name, const std::vector& id_array) = 0; + DeleteByID(const std::string& collection_name, const std::vector& id_array) = 0; /** - * @brief preload table + * @brief Preload collection * - * This method is used to preload table + * This method is used to preload collection data into memory * - * @param table_name + * @param collection_name, target collection's name. * * @return Indicate if this operation is successful. */ virtual Status - PreloadTable(const std::string& table_name) const = 0; + PreloadCollection(const std::string& collection_name) const = 0; /** - * @brief describe index + * @brief Describe index * * This method is used to describe index * - * @param table_name, target table's name. + * @param collection_name, target collection's name. * @param index_param, returned index information. * * @return Indicate if this operation is successful. */ virtual Status - DescribeIndex(const std::string& table_name, IndexParam& index_param) const = 0; + DescribeIndex(const std::string& collection_name, IndexParam& index_param) const = 0; /** - * @brief drop index + * @brief Drop index * - * This method is used to drop index of table(and its partitions) + * This method is used to drop index of collection(and its partitions) * - * @param table_name, target table's name. + * @param collection_name, target collection's name. * * @return Indicate if this operation is successful. */ virtual Status - DropIndex(const std::string& table_name) const = 0; + DropIndex(const std::string& collection_name) const = 0; /** * @brief Create partition method * - * This method is used to create table partition + * This method is used to create collection's partition * - * @param param, use to provide partition information to be created. + * @param partition_param, use to provide partition information to be created. * * @return Indicate if partition is created successfully */ virtual Status - CreatePartition(const PartitionParam& param) = 0; + CreatePartition(const PartitionParam& partition_param) = 0; /** - * @brief Test table existence method + * @brief Show all partitions method * - * This method is used to create table + * This method is used to show all partitions(return their tags) * - * @param table_name, table name is going to be tested. - * @param partition_tag_array, partition tag array of the table. + * @param collection_name, target collection's name. + * @param partition_tag_array, partition tag array of the collection. * * @return Indicate if this operation is successful */ virtual Status - ShowPartitions(const std::string& table_name, PartitionTagList& partition_tag_array) const = 0; + ShowPartitions(const std::string& collection_name, PartitionTagList& partition_tag_array) const = 0; /** * @brief Delete partition method * - * This method is used to delete table partition. + * This method is used to delete collection's partition. * - * @param param, target partition to be deleted. - * NOTE: if param.table_name is empty, you must specify param.partition_name, - * else you can specify param.table_name and param.tag and let the param.partition_name be empty + * @param partition_param, target partition to be deleted. * * @return Indicate if partition is delete successfully. */ virtual Status - DropPartition(const PartitionParam& param) = 0; + DropPartition(const PartitionParam& partition_param) = 0; /** - * @brief Get config method + * @brief Flush collection buffer into storage * - * This method is used to set config. + * This method is used to flush collection buffer into storage * - * @param node_name, config node name. - * @param value, config value. + * @param collection_name, target collection's name. * * @return Indicate if this operation is successful. */ virtual Status - GetConfig(const std::string& node_name, std::string& value) const = 0; + FlushCollection(const std::string& collection_name) = 0; /** - * @brief Set config method + * @brief Flush all buffer into storage * - * This method is used to set config. - * - * @param node_name, config node name. - * @param value, config value. - * - * @return Indicate if this operation is successful. - */ - virtual Status - SetConfig(const std::string& node_name, const std::string& value) const = 0; - - /** - * @brief flush table buffer into storage - * - * This method is used to flush table buffer into storage - * - * @param table_name, target table's name. - * - * @return Indicate if this operation is successful. - */ - virtual Status - FlushTable(const std::string& table_name) = 0; - - /** - * @brief flush all buffer into storage - * - * This method is used to all table buffer into storage + * This method is used to all collection buffer into storage * * @return Indicate if this operation is successful. */ @@ -554,16 +562,16 @@ class Connection { Flush() = 0; /** - * @brief compact table, remove deleted vectors + * @brief Compact collection, permanently remove deleted vectors * - * This method is used to compact table + * This method is used to compact collection * - * @param table_name, target table's name. + * @param collection_name, target collection's name. * * @return Indicate if this operation is successful. */ virtual Status - CompactTable(const std::string& table_name) = 0; + CompactCollection(const std::string& collection_name) = 0; }; } // namespace milvus diff --git a/sdk/interface/ConnectionImpl.cpp b/sdk/interface/ConnectionImpl.cpp index 3a3fa52516..2a51a11d50 100644 --- a/sdk/interface/ConnectionImpl.cpp +++ b/sdk/interface/ConnectionImpl.cpp @@ -56,70 +56,6 @@ ConnectionImpl::ClientVersion() const { return client_proxy_->ClientVersion(); } -Status -ConnectionImpl::CreateTable(const TableSchema& param) { - return client_proxy_->CreateTable(param); -} - -bool -ConnectionImpl::HasTable(const std::string& table_name) { - return client_proxy_->HasTable(table_name); -} - -Status -ConnectionImpl::DropTable(const std::string& table_name) { - return client_proxy_->DropTable(table_name); -} - -Status -ConnectionImpl::CreateIndex(const IndexParam& index_param) { - return client_proxy_->CreateIndex(index_param); -} - -Status -ConnectionImpl::Insert(const std::string& table_name, const std::string& partition_tag, - const std::vector& record_array, std::vector& id_array) { - return client_proxy_->Insert(table_name, partition_tag, record_array, id_array); -} - -Status -ConnectionImpl::GetVectorByID(const std::string& table_name, int64_t vector_id, RowRecord& vector_data) { - return client_proxy_->GetVectorByID(table_name, vector_id, vector_data); -} - -Status -ConnectionImpl::GetIDsInSegment(const std::string& table_name, const std::string& segment_name, - std::vector& id_array) { - return client_proxy_->GetIDsInSegment(table_name, segment_name, id_array); -} - -Status -ConnectionImpl::Search(const std::string& table_name, const std::vector& partition_tags, - const std::vector& query_record_array, int64_t topk, const std::string& extra_params, - TopKQueryResult& topk_query_result) { - return client_proxy_->Search(table_name, partition_tags, query_record_array, topk, extra_params, topk_query_result); -} - -Status -ConnectionImpl::DescribeTable(const std::string& table_name, TableSchema& table_schema) { - return client_proxy_->DescribeTable(table_name, table_schema); -} - -Status -ConnectionImpl::CountTable(const std::string& table_name, int64_t& row_count) { - return client_proxy_->CountTable(table_name, row_count); -} - -Status -ConnectionImpl::ShowTables(std::vector& table_array) { - return client_proxy_->ShowTables(table_array); -} - -Status -ConnectionImpl::ShowTableInfo(const std::string& table_name, TableInfo& table_info) { - return client_proxy_->ShowTableInfo(table_name, table_info); -} - std::string ConnectionImpl::ServerVersion() const { return client_proxy_->ServerVersion(); @@ -130,41 +66,6 @@ ConnectionImpl::ServerStatus() const { return client_proxy_->ServerStatus(); } -Status -ConnectionImpl::DeleteByID(const std::string& table_name, const std::vector& id_array) { - return client_proxy_->DeleteByID(table_name, id_array); -} - -Status -ConnectionImpl::PreloadTable(const std::string& table_name) const { - return client_proxy_->PreloadTable(table_name); -} - -Status -ConnectionImpl::DescribeIndex(const std::string& table_name, IndexParam& index_param) const { - return client_proxy_->DescribeIndex(table_name, index_param); -} - -Status -ConnectionImpl::DropIndex(const std::string& table_name) const { - return client_proxy_->DropIndex(table_name); -} - -Status -ConnectionImpl::CreatePartition(const PartitionParam& param) { - return client_proxy_->CreatePartition(param); -} - -Status -ConnectionImpl::ShowPartitions(const std::string& table_name, PartitionTagList& partition_array) const { - return client_proxy_->ShowPartitions(table_name, partition_array); -} - -Status -ConnectionImpl::DropPartition(const PartitionParam& param) { - return client_proxy_->DropPartition(param); -} - Status ConnectionImpl::GetConfig(const std::string& node_name, std::string& value) const { return client_proxy_->GetConfig(node_name, value); @@ -176,8 +77,107 @@ ConnectionImpl::SetConfig(const std::string& node_name, const std::string& value } Status -ConnectionImpl::FlushTable(const std::string& Status) { - return client_proxy_->FlushTable(Status); +ConnectionImpl::CreateCollection(const CollectionParam& param) { + return client_proxy_->CreateCollection(param); +} + +bool +ConnectionImpl::HasCollection(const std::string& collection_name) { + return client_proxy_->HasCollection(collection_name); +} + +Status +ConnectionImpl::DropCollection(const std::string& collection_name) { + return client_proxy_->DropCollection(collection_name); +} + +Status +ConnectionImpl::CreateIndex(const IndexParam& index_param) { + return client_proxy_->CreateIndex(index_param); +} + +Status +ConnectionImpl::Insert(const std::string& collection_name, const std::string& partition_tag, + const std::vector& entity_array, std::vector& id_array) { + return client_proxy_->Insert(collection_name, partition_tag, entity_array, id_array); +} + +Status +ConnectionImpl::GetEntityByID(const std::string& collection_name, int64_t entity_id, Entity& entity_data) { + return client_proxy_->GetEntityByID(collection_name, entity_id, entity_data); +} + +Status +ConnectionImpl::GetIDsInSegment(const std::string& collection_name, const std::string& segment_name, + std::vector& id_array) { + return client_proxy_->GetIDsInSegment(collection_name, segment_name, id_array); +} + +Status +ConnectionImpl::Search(const std::string& collection_name, const std::vector& partition_tags, + const std::vector& entity_array, int64_t topk, const std::string& extra_params, + TopKQueryResult& topk_query_result) { + return client_proxy_->Search(collection_name, partition_tags, entity_array, topk, extra_params, topk_query_result); +} + +Status +ConnectionImpl::DescribeCollection(const std::string& collection_name, CollectionParam& collection_schema) { + return client_proxy_->DescribeCollection(collection_name, collection_schema); +} + +Status +ConnectionImpl::CountCollection(const std::string& collection_name, int64_t& row_count) { + return client_proxy_->CountCollection(collection_name, row_count); +} + +Status +ConnectionImpl::ShowCollections(std::vector& collection_array) { + return client_proxy_->ShowCollections(collection_array); +} + +Status +ConnectionImpl::ShowCollectionInfo(const std::string& collection_name, CollectionInfo& collection_info) { + return client_proxy_->ShowCollectionInfo(collection_name, collection_info); +} + +Status +ConnectionImpl::DeleteByID(const std::string& collection_name, const std::vector& id_array) { + return client_proxy_->DeleteByID(collection_name, id_array); +} + +Status +ConnectionImpl::PreloadCollection(const std::string& collection_name) const { + return client_proxy_->PreloadCollection(collection_name); +} + +Status +ConnectionImpl::DescribeIndex(const std::string& collection_name, IndexParam& index_param) const { + return client_proxy_->DescribeIndex(collection_name, index_param); +} + +Status +ConnectionImpl::DropIndex(const std::string& collection_name) const { + return client_proxy_->DropIndex(collection_name); +} + +Status +ConnectionImpl::CreatePartition(const PartitionParam& partition_param) { + return client_proxy_->CreatePartition(partition_param); +} + +Status +ConnectionImpl::ShowPartitions(const std::string& collection_name, PartitionTagList& partition_array) const { + return client_proxy_->ShowPartitions(collection_name, partition_array); +} + +Status +ConnectionImpl::DropPartition(const PartitionParam& partition_param) { + return client_proxy_->DropPartition(partition_param); +} + +Status +ConnectionImpl::FlushCollection(const std::string& Status) { + return client_proxy_->FlushCollection(Status); } Status @@ -186,8 +186,8 @@ ConnectionImpl::Flush() { } Status -ConnectionImpl::CompactTable(const std::string& table_name) { - return client_proxy_->CompactTable(table_name); +ConnectionImpl::CompactCollection(const std::string& collection_name) { + return client_proxy_->CompactCollection(collection_name); } } // namespace milvus diff --git a/sdk/interface/ConnectionImpl.h b/sdk/interface/ConnectionImpl.h index 469adf1a27..15533b9de6 100644 --- a/sdk/interface/ConnectionImpl.h +++ b/sdk/interface/ConnectionImpl.h @@ -26,7 +26,7 @@ class ConnectionImpl : public Connection { // Implementations of the Connection interface Status - Connect(const ConnectParam& param) override; + Connect(const ConnectParam& connect_param) override; Status Connect(const std::string& uri) override; @@ -37,46 +37,6 @@ class ConnectionImpl : public Connection { Status Disconnect() override; - Status - CreateTable(const TableSchema& param) override; - - bool - HasTable(const std::string& table_name) override; - - Status - DropTable(const std::string& table_name) override; - - Status - CreateIndex(const IndexParam& index_param) override; - - Status - Insert(const std::string& table_name, const std::string& partition_tag, const std::vector& record_array, - std::vector& id_array) override; - - Status - GetVectorByID(const std::string& table_name, int64_t vector_id, RowRecord& vector_data) override; - - Status - GetIDsInSegment(const std::string& table_name, const std::string& segment_name, - std::vector& id_array) override; - - Status - Search(const std::string& table_name, const std::vector& partition_tag_array, - const std::vector& query_record_array, int64_t topk, - const std::string& extra_params, TopKQueryResult& topk_query_result) override; - - Status - DescribeTable(const std::string& table_name, TableSchema& table_schema) override; - - Status - CountTable(const std::string& table_name, int64_t& row_count) override; - - Status - ShowTables(std::vector& table_array) override; - - Status - ShowTableInfo(const std::string& table_name, TableInfo& table_info) override; - std::string ClientVersion() const override; @@ -86,27 +46,6 @@ class ConnectionImpl : public Connection { std::string ServerStatus() const override; - Status - DeleteByID(const std::string& table_name, const std::vector& id_array) override; - - Status - PreloadTable(const std::string& table_name) const override; - - Status - DescribeIndex(const std::string& table_name, IndexParam& index_param) const override; - - Status - DropIndex(const std::string& table_name) const override; - - Status - CreatePartition(const PartitionParam& param) override; - - Status - ShowPartitions(const std::string& table_name, PartitionTagList& partition_tag_array) const override; - - Status - DropPartition(const PartitionParam& param) override; - Status GetConfig(const std::string& node_name, std::string& value) const override; @@ -114,13 +53,76 @@ class ConnectionImpl : public Connection { SetConfig(const std::string& node_name, const std::string& value) const override; Status - FlushTable(const std::string& table_name) override; + CreateCollection(const CollectionParam& param) override; + + bool + HasCollection(const std::string& collection_name) override; + + Status + DropCollection(const std::string& collection_name) override; + + Status + CreateIndex(const IndexParam& index_param) override; + + Status + Insert(const std::string& collection_name, + const std::string& partition_tag, + const std::vector& entity_array, + std::vector& id_array) override; + + Status + GetEntityByID(const std::string& collection_name, int64_t entity_id, Entity& entity_data) override; + + Status + GetIDsInSegment(const std::string& collection_name, const std::string& segment_name, + std::vector& id_array) override; + + Status + Search(const std::string& collection_name, const std::vector& partition_tag_array, + const std::vector& entity_array, int64_t topk, + const std::string& extra_params, TopKQueryResult& topk_query_result) override; + + Status + DescribeCollection(const std::string& collection_name, CollectionParam& collection_schema) override; + + Status + CountCollection(const std::string& collection_name, int64_t& entity_count) override; + + Status + ShowCollections(std::vector& collection_array) override; + + Status + ShowCollectionInfo(const std::string& collection_name, CollectionInfo& collection_info) override; + + Status + DeleteByID(const std::string& collection_name, const std::vector& id_array) override; + + Status + PreloadCollection(const std::string& collection_name) const override; + + Status + DescribeIndex(const std::string& collection_name, IndexParam& index_param) const override; + + Status + DropIndex(const std::string& collection_name) const override; + + Status + CreatePartition(const PartitionParam& partition_param) override; + + Status + ShowPartitions(const std::string& collection_name, PartitionTagList& partition_tag_array) const override; + + Status + DropPartition(const PartitionParam& partition_param) override; + + Status + FlushCollection(const std::string& collection_name) override; Status Flush() override; Status - CompactTable(const std::string& table_name) override; + CompactCollection(const std::string& collection_name) override; private: std::shared_ptr client_proxy_;