milvus-io
/
milvus
mirror of https://github.com/milvus-io/milvus.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

#649 Typo partiton should be partition

pull/653/head
groot 2019-12-02 15:39:16 +08:00
parent 51e8be0130
commit 3df535e11e
20 changed files with 66 additions and 65 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
CHANGELOG.md
View File

@ -45,6 +45,7 @@ Please mark all change in change log and use the ticket from JIRA.
- \#602 - Optimizer specify wrong gpu_id
- \#606 - No log generated during building index with CPU
- \#631 - FAISS isn't compiled with O3 option
- \#649 - Typo "partiton" should be "partition"
## Feature
- \#12 - Pure CPU version for Milvus

2
core/src/db/DB.h
View File

@ -80,7 +80,7 @@ class DB {
DropPartitionByTag(const std::string& table_id, const std::string& partition_tag) = 0;
virtual Status
ShowPartitions(const std::string& table_id, std::vector<meta::TableSchema>& partiton_schema_array) = 0;
ShowPartitions(const std::string& table_id, std::vector<meta::TableSchema>& partition_schema_array) = 0;
virtual Status
InsertVectors(const std::string& table_id, const std::string& partition_tag, uint64_t n, const float* vectors,

52
core/src/db/DBImpl.cpp
View File

@ -190,9 +190,9 @@ DBImpl::PreloadTable(const std::string& table_id) {
}
// step 2: get files from partition tables
std::vector<meta::TableSchema> partiton_array;
status = meta_ptr_->ShowPartitions(table_id, partiton_array);
for (auto& schema : partiton_array) {
std::vector<meta::TableSchema> partition_array;
status = meta_ptr_->ShowPartitions(table_id, partition_array);
for (auto& schema : partition_array) {
status = GetFilesToSearch(schema.table_id_, ids, dates, files_array);
}
@ -296,12 +296,12 @@ DBImpl::DropPartitionByTag(const std::string& table_id, const std::string& parti
}
Status
DBImpl::ShowPartitions(const std::string& table_id, std::vector<meta::TableSchema>& partiton_schema_array) {
DBImpl::ShowPartitions(const std::string& table_id, std::vector<meta::TableSchema>& partition_schema_array) {
if (shutting_down_.load(std::memory_order_acquire)) {
return SHUTDOWN_ERROR;
}
return meta_ptr_->ShowPartitions(table_id, partiton_schema_array);
return meta_ptr_->ShowPartitions(table_id, partition_schema_array);
}
Status
@ -427,9 +427,9 @@ DBImpl::Query(const std::string& table_id, const std::vector<std::string>& parti
return status;
}
std::vector<meta::TableSchema> partiton_array;
status = meta_ptr_->ShowPartitions(table_id, partiton_array);
for (auto& schema : partiton_array) {
std::vector<meta::TableSchema> partition_array;
status = meta_ptr_->ShowPartitions(table_id, partition_array);
for (auto& schema : partition_array) {
status = GetFilesToSearch(schema.table_id_, ids, dates, files_array);
}
} else {
@ -917,15 +917,15 @@ DBImpl::GetFilesToSearch(const std::string& table_id, const std::vector<size_t>&
Status
DBImpl::GetPartitionsByTags(const std::string& table_id, const std::vector<std::string>& partition_tags,
std::set<std::string>& partition_name_array) {
std::vector<meta::TableSchema> partiton_array;
auto status = meta_ptr_->ShowPartitions(table_id, partiton_array);
std::vector<meta::TableSchema> partition_array;
auto status = meta_ptr_->ShowPartitions(table_id, partition_array);
for (auto& tag : partition_tags) {
// trim side-blank of tag, only compare valid characters
// for example: " ab cd " is treated as "ab cd"
std::string valid_tag = tag;
server::StringHelpFunctions::TrimStringBlank(valid_tag);
for (auto& schema : partiton_array) {
for (auto& schema : partition_array) {
if (server::StringHelpFunctions::IsRegexMatch(schema.partition_tag_, valid_tag)) {
partition_name_array.insert(schema.table_id_);
}
@ -955,9 +955,9 @@ DBImpl::DropTableRecursively(const std::string& table_id, const meta::DatesT& da
status = meta_ptr_->DropDataByDate(table_id, dates);
}
std::vector<meta::TableSchema> partiton_array;
status = meta_ptr_->ShowPartitions(table_id, partiton_array);
for (auto& schema : partiton_array) {
std::vector<meta::TableSchema> partition_array;
status = meta_ptr_->ShowPartitions(table_id, partition_array);
for (auto& schema : partition_array) {
status = DropTableRecursively(schema.table_id_, dates);
if (!status.ok()) {
return status;
@ -977,9 +977,9 @@ DBImpl::UpdateTableIndexRecursively(const std::string& table_id, const TableInde
return status;
}
std::vector<meta::TableSchema> partiton_array;
status = meta_ptr_->ShowPartitions(table_id, partiton_array);
for (auto& schema : partiton_array) {
std::vector<meta::TableSchema> partition_array;
status = meta_ptr_->ShowPartitions(table_id, partition_array);
for (auto& schema : partition_array) {
status = UpdateTableIndexRecursively(schema.table_id_, index);
if (!status.ok()) {
return status;
@ -1028,9 +1028,9 @@ DBImpl::BuildTableIndexRecursively(const std::string& table_id, const TableIndex
}
// build index for partition
std::vector<meta::TableSchema> partiton_array;
status = meta_ptr_->ShowPartitions(table_id, partiton_array);
for (auto& schema : partiton_array) {
std::vector<meta::TableSchema> partition_array;
status = meta_ptr_->ShowPartitions(table_id, partition_array);
for (auto& schema : partition_array) {
status = BuildTableIndexRecursively(schema.table_id_, index);
if (!status.ok()) {
return status;
@ -1060,9 +1060,9 @@ DBImpl::DropTableIndexRecursively(const std::string& table_id) {
}
// drop partition index
std::vector<meta::TableSchema> partiton_array;
status = meta_ptr_->ShowPartitions(table_id, partiton_array);
for (auto& schema : partiton_array) {
std::vector<meta::TableSchema> partition_array;
status = meta_ptr_->ShowPartitions(table_id, partition_array);
for (auto& schema : partition_array) {
status = DropTableIndexRecursively(schema.table_id_);
if (!status.ok()) {
return status;
@ -1081,9 +1081,9 @@ DBImpl::GetTableRowCountRecursively(const std::string& table_id, uint64_t& row_c
}
// get partition row count
std::vector<meta::TableSchema> partiton_array;
status = meta_ptr_->ShowPartitions(table_id, partiton_array);
for (auto& schema : partiton_array) {
std::vector<meta::TableSchema> partition_array;
status = meta_ptr_->ShowPartitions(table_id, partition_array);
for (auto& schema : partition_array) {
uint64_t partition_row_count = 0;
status = GetTableRowCountRecursively(schema.table_id_, partition_row_count);
if (!status.ok()) {

2
core/src/db/DBImpl.h
View File

@ -89,7 +89,7 @@ class DBImpl : public DB {
DropPartitionByTag(const std::string& table_id, const std::string& partition_tag) override;
Status
ShowPartitions(const std::string& table_id, std::vector<meta::TableSchema>& partiton_schema_array) override;
ShowPartitions(const std::string& table_id, std::vector<meta::TableSchema>& partition_schema_array) override;
Status
InsertVectors(const std::string& table_id, const std::string& partition_tag, uint64_t n, const float* vectors,

2
core/src/db/meta/Meta.h
View File

@ -100,7 +100,7 @@ class Meta {
DropPartition(const std::string& partition_name) = 0;
virtual Status
ShowPartitions(const std::string& table_name, std::vector<meta::TableSchema>& partiton_schema_array) = 0;
ShowPartitions(const std::string& table_name, std::vector<meta::TableSchema>& partition_schema_array) = 0;
virtual Status
GetPartitionName(const std::string& table_name, const std::string& tag, std::string& partition_name) = 0;

4
core/src/db/meta/MySQLMetaImpl.cpp
View File

@ -1212,7 +1212,7 @@ MySQLMetaImpl::DropPartition(const std::string& partition_name) {
}
Status
MySQLMetaImpl::ShowPartitions(const std::string& table_id, std::vector<meta::TableSchema>& partiton_schema_array) {
MySQLMetaImpl::ShowPartitions(const std::string& table_id, std::vector<meta::TableSchema>& partition_schema_array) {
try {
server::MetricCollector metric;
mysqlpp::StoreQueryResult res;
@ -1236,7 +1236,7 @@ MySQLMetaImpl::ShowPartitions(const std::string& table_id, std::vector<meta::Tab
meta::TableSchema partition_schema;
resRow["table_id"].to_string(partition_schema.table_id_);
DescribeTable(partition_schema);
partiton_schema_array.emplace_back(partition_schema);
partition_schema_array.emplace_back(partition_schema);
}
} catch (std::exception& e) {
return HandleException("GENERAL ERROR WHEN SHOW PARTITIONS", e.what());

2
core/src/db/meta/MySQLMetaImpl.h
View File

@ -91,7 +91,7 @@ class MySQLMetaImpl : public Meta {
DropPartition(const std::string& partition_name) override;
Status
ShowPartitions(const std::string& table_id, std::vector<meta::TableSchema>& partiton_schema_array) override;
ShowPartitions(const std::string& table_id, std::vector<meta::TableSchema>& partition_schema_array) override;
Status
GetPartitionName(const std::string& table_id, const std::string& tag, std::string& partition_name) override;

4
core/src/db/meta/SqliteMetaImpl.cpp
View File

@ -804,7 +804,7 @@ SqliteMetaImpl::DropPartition(const std::string& partition_name) {
}
Status
SqliteMetaImpl::ShowPartitions(const std::string& table_id, std::vector<meta::TableSchema>& partiton_schema_array) {
SqliteMetaImpl::ShowPartitions(const std::string& table_id, std::vector<meta::TableSchema>& partition_schema_array) {
try {
server::MetricCollector metric;
@ -816,7 +816,7 @@ SqliteMetaImpl::ShowPartitions(const std::string& table_id, std::vector<meta::Ta
meta::TableSchema partition_schema;
partition_schema.table_id_ = partition_name;
DescribeTable(partition_schema);
partiton_schema_array.emplace_back(partition_schema);
partition_schema_array.emplace_back(partition_schema);
}
} catch (std::exception& e) {
return HandleException("Encounter exception when show partitions", e.what());

2
core/src/db/meta/SqliteMetaImpl.h
View File

@ -91,7 +91,7 @@ class SqliteMetaImpl : public Meta {
DropPartition(const std::string& partition_name) override;
Status
ShowPartitions(const std::string& table_id, std::vector<meta::TableSchema>& partiton_schema_array) override;
ShowPartitions(const std::string& table_id, std::vector<meta::TableSchema>& partition_schema_array) override;
Status
GetPartitionName(const std::string& table_id, const std::string& tag, std::string& partition_name) override;

16
core/src/sdk/examples/partition/src/ClientTest.cpp
View File

@ -141,18 +141,18 @@ ClientTest::Test(const std::string& address, const std::string& port) {
{ // search vectors
std::cout << "Search in correct partition" << std::endl;
std::vector<std::string> partiton_tags = {std::to_string(TARGET_PARTITION)};
std::vector<std::string> partition_tags = {std::to_string(TARGET_PARTITION)};
milvus::TopKQueryResult topk_query_result;
milvus_sdk::Utils::DoSearch(conn, TABLE_NAME, partiton_tags, TOP_K, NPROBE, search_record_array,
milvus_sdk::Utils::DoSearch(conn, TABLE_NAME, partition_tags, TOP_K, NPROBE, search_record_array,
topk_query_result);
std::cout << "Search in wrong partition" << std::endl;
partiton_tags = {"0"};
milvus_sdk::Utils::DoSearch(conn, TABLE_NAME, partiton_tags, TOP_K, NPROBE, search_record_array,
partition_tags = {"0"};
milvus_sdk::Utils::DoSearch(conn, TABLE_NAME, partition_tags, TOP_K, NPROBE, search_record_array,
topk_query_result);
std::cout << "Search by regex matched partition tag" << std::endl;
partiton_tags = {"\\d"};
milvus_sdk::Utils::DoSearch(conn, TABLE_NAME, partiton_tags, TOP_K, NPROBE, search_record_array,
partition_tags = {"\\d"};
milvus_sdk::Utils::DoSearch(conn, TABLE_NAME, partition_tags, TOP_K, NPROBE, search_record_array,
topk_query_result);
}
@ -191,9 +191,9 @@ ClientTest::Test(const std::string& address, const std::string& port) {
{ // search vectors
std::cout << "Search in whole table" << std::endl;
std::vector<std::string> partiton_tags;
std::vector<std::string> partition_tags;
milvus::TopKQueryResult topk_query_result;
milvus_sdk::Utils::DoSearch(conn, TABLE_NAME, partiton_tags, TOP_K, NPROBE, search_record_array,
milvus_sdk::Utils::DoSearch(conn, TABLE_NAME, partition_tags, TOP_K, NPROBE, search_record_array,
topk_query_result);
}

8
core/src/sdk/examples/simple/src/ClientTest.cpp
View File

@ -143,9 +143,9 @@ ClientTest::Test(const std::string& address, const std::string& port) {
milvus_sdk::Utils::Sleep(3);
{ // search vectors
std::vector<std::string> partiton_tags;
std::vector<std::string> partition_tags;
milvus::TopKQueryResult topk_query_result;
milvus_sdk::Utils::DoSearch(conn, TABLE_NAME, partiton_tags, TOP_K, NPROBE, search_record_array,
milvus_sdk::Utils::DoSearch(conn, TABLE_NAME, partition_tags, TOP_K, NPROBE, search_record_array,
topk_query_result);
}
@ -169,9 +169,9 @@ ClientTest::Test(const std::string& address, const std::string& port) {
}
{ // search vectors
std::vector<std::string> partiton_tags;
std::vector<std::string> partition_tags;
milvus::TopKQueryResult topk_query_result;
milvus_sdk::Utils::DoSearch(conn, TABLE_NAME, partiton_tags, TOP_K, NPROBE, search_record_array,
milvus_sdk::Utils::DoSearch(conn, TABLE_NAME, partition_tags, TOP_K, NPROBE, search_record_array,
topk_query_result);
}

4
core/src/sdk/examples/utils/Utils.cpp
View File

@ -202,7 +202,7 @@ Utils::CheckSearchResult(const std::vector<std::pair<int64_t, milvus::RowRecord>
void
Utils::DoSearch(std::shared_ptr<milvus::Connection> conn, const std::string& table_name,
const std::vector<std::string>& partiton_tags, int64_t top_k, int64_t nprobe,
const std::vector<std::string>& partition_tags, int64_t top_k, int64_t nprobe,
const std::vector<std::pair<int64_t, milvus::RowRecord>>& search_record_array,
milvus::TopKQueryResult& topk_query_result) {
topk_query_result.clear();
@ -222,7 +222,7 @@ Utils::DoSearch(std::shared_ptr<milvus::Connection> conn, const std::string& tab
BLOCK_SPLITER
milvus_sdk::TimeRecorder rc("search");
milvus::Status stat =
conn->Search(table_name, partiton_tags, record_array, query_range_array, top_k, nprobe, topk_query_result);
conn->Search(table_name, partition_tags, record_array, query_range_array, top_k, nprobe, topk_query_result);
std::cout << "SearchVector function call status: " << stat.message() << std::endl;
BLOCK_SPLITER
}

2
core/src/sdk/examples/utils/Utils.h
View File

@ -69,7 +69,7 @@ class Utils {
static void
DoSearch(std::shared_ptr<milvus::Connection> conn, const std::string& table_name,
const std::vector<std::string>& partiton_tags, int64_t top_k, int64_t nprobe,
const std::vector<std::string>& partition_tags, int64_t top_k, int64_t nprobe,
const std::vector<std::pair<int64_t, milvus::RowRecord>>& search_record_array,
milvus::TopKQueryResult& topk_query_result);
};

4
core/src/sdk/grpc/ClientProxy.cpp
View File

@ -221,7 +221,7 @@ ClientProxy::Insert(const std::string& table_name, const std::string& partition_
}
Status
ClientProxy::Search(const std::string& table_name, const std::vector<std::string>& partiton_tags,
ClientProxy::Search(const std::string& table_name, const std::vector<std::string>& partition_tags,
const std::vector<RowRecord>& query_record_array, const std::vector<Range>& query_range_array,
int64_t topk, int64_t nprobe, TopKQueryResult& topk_query_result) {
try {
@ -230,7 +230,7 @@ ClientProxy::Search(const std::string& table_name, const std::vector<std::string
search_param.set_table_name(table_name);
search_param.set_topk(topk);
search_param.set_nprobe(nprobe);
for (auto& tag : partiton_tags) {
for (auto& tag : partition_tags) {
search_param.add_partition_tag_array(tag);
}
for (auto& record : query_record_array) {

2
core/src/sdk/grpc/ClientProxy.h
View File

@ -58,7 +58,7 @@ class ClientProxy : public Connection {
std::vector<int64_t>& id_array) override;
Status
Search(const std::string& table_name, const std::vector<std::string>& partiton_tags,
Search(const std::string& table_name, const std::vector<std::string>& partition_tags,
const std::vector<RowRecord>& query_record_array, const std::vector<Range>& query_range_array, int64_t topk,
int64_t nprobe, TopKQueryResult& topk_query_result) override;

2
core/src/sdk/include/MilvusApi.h
View File

@ -273,7 +273,7 @@ class Connection {
* @return Indicate if query is successful.
*/
virtual Status
Search(const std::string& table_name, const std::vector<std::string>& partiton_tags,
Search(const std::string& table_name, const std::vector<std::string>& partition_tags,
const std::vector<RowRecord>& query_record_array, const std::vector<Range>& query_range_array, int64_t topk,
int64_t nprobe, TopKQueryResult& topk_query_result) = 0;

4
core/src/sdk/interface/ConnectionImpl.cpp
View File

@ -89,10 +89,10 @@ ConnectionImpl::Insert(const std::string& table_name, const std::string& partiti
}
Status
ConnectionImpl::Search(const std::string& table_name, const std::vector<std::string>& partiton_tags,
ConnectionImpl::Search(const std::string& table_name, const std::vector<std::string>& partition_tags,
const std::vector<RowRecord>& query_record_array, const std::vector<Range>& query_range_array,
int64_t topk, int64_t nprobe, TopKQueryResult& topk_query_result) {
return client_proxy_->Search(table_name, partiton_tags, query_record_array, query_range_array, topk, nprobe,
return client_proxy_->Search(table_name, partition_tags, query_record_array, query_range_array, topk, nprobe,
topk_query_result);
}

2
core/src/sdk/interface/ConnectionImpl.h
View File

@ -60,7 +60,7 @@ class ConnectionImpl : public Connection {
std::vector<int64_t>& id_array) override;
Status
Search(const std::string& table_name, const std::vector<std::string>& partiton_tags,
Search(const std::string& table_name, const std::vector<std::string>& partition_tags,
const std::vector<RowRecord>& query_record_array, const std::vector<Range>& query_range_array, int64_t topk,
int64_t nprobe, TopKQueryResult& topk_query_result) override;

8
core/unittest/db/test_db.cpp
View File

@ -536,12 +536,12 @@ TEST_F(DBTest, PARTITION_TEST) {
stat = db_->CreatePartition(table_name, "", "0");
ASSERT_FALSE(stat.ok());
std::vector<milvus::engine::meta::TableSchema> partiton_schema_array;
stat = db_->ShowPartitions(table_name, partiton_schema_array);
std::vector<milvus::engine::meta::TableSchema> partition_schema_array;
stat = db_->ShowPartitions(table_name, partition_schema_array);
ASSERT_TRUE(stat.ok());
ASSERT_EQ(partiton_schema_array.size(), PARTITION_COUNT);
ASSERT_EQ(partition_schema_array.size(), PARTITION_COUNT);
for (int64_t i = 0; i < PARTITION_COUNT; i++) {
ASSERT_EQ(partiton_schema_array[i].table_id_, table_name + "_" + std::to_string(i));
ASSERT_EQ(partition_schema_array[i].table_id_, table_name + "_" + std::to_string(i));
}
{ // build index

8
core/unittest/db/test_db_mysql.cpp
View File

@ -323,12 +323,12 @@ TEST_F(MySqlDBTest, PARTITION_TEST) {
stat = db_->CreatePartition(table_name, "", "0");
ASSERT_FALSE(stat.ok());
std::vector<milvus::engine::meta::TableSchema> partiton_schema_array;
stat = db_->ShowPartitions(table_name, partiton_schema_array);
std::vector<milvus::engine::meta::TableSchema> partition_schema_array;
stat = db_->ShowPartitions(table_name, partition_schema_array);
ASSERT_TRUE(stat.ok());
ASSERT_EQ(partiton_schema_array.size(), PARTITION_COUNT);
ASSERT_EQ(partition_schema_array.size(), PARTITION_COUNT);
for (int64_t i = 0; i < PARTITION_COUNT; i++) {
ASSERT_EQ(partiton_schema_array[i].table_id_, table_name + "_" + std::to_string(i));
ASSERT_EQ(partition_schema_array[i].table_id_, table_name + "_" + std::to_string(i));
}
{ // build index