milvus-io
/
milvus
mirror of https://github.com/milvus-io/milvus.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

rewrite HasPartition (#2289) 

* rewrite HasPartition

Signed-off-by: yhmo <yihua.mo@zilliz.com>

* miss file

Signed-off-by: yhmo <yihua.mo@zilliz.com>

* use layered merge

Signed-off-by: yhmo <yihua.mo@zilliz.com>

* changelog

Signed-off-by: yhmo <yihua.mo@zilliz.com>

* sdk HasPartition

Signed-off-by: yhmo <yihua.mo@zilliz.com>

* rename sdk api

Signed-off-by: yhmo <yihua.mo@zilliz.com>

* fix ut

Signed-off-by: groot <yihua.mo@zilliz.com>

Co-authored-by: JinHai-CN <hai.jin@zilliz.com>
pull/2317/head
groot 2020-05-11 20:27:58 -05:00 committed by GitHub
parent bc74d16376
commit fb49e580c5
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
30 changed files with 686 additions and 552 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
CHANGELOG.md
View File

@ -27,6 +27,8 @@ Please mark all change in change log and use the issue from GitHub
- \#2228 Fix show partitions failed in http module
- \#2231 Use server_config to define hard-delete delay time for segment files
- \#2261 Re-define result returned by has_collection if collection in delete state
- \#2266 Server hang when using multi-clients to query different collections
- \#2280 has_partition should return true for '_default'
## Feature
- \#1751 Add api SearchByID
@ -47,6 +49,7 @@ Please mark all change in change log and use the issue from GitHub
- \#221 Refactor LOG macro
- \#833 Catch exception in RolloutHandler and output in stderr
- \#1796 Compile Openblas with source code to improve the performance
- \#1942 Background merge file strategy
- \#2039 Support Milvus run on SSE CPUs
- \#2149 Merge server_cpu_config.template and server_gpu_config.template
- \#2153 Upgrade thirdparty oatpp to v1.0.0

3
core/src/db/DB.h
View File

@ -80,6 +80,9 @@ class DB {
CreatePartition(const std::string& collection_id, const std::string& partition_name,
const std::string& partition_tag) = 0;
virtual Status
HasPartition(const std::string& collection_id, const std::string& tag, bool& has_or_not) = 0;
virtual Status
DropPartition(const std::string& partition_name) = 0;

19
core/src/db/DBImpl.cpp
View File

@ -491,6 +491,25 @@ DBImpl::CreatePartition(const std::string& collection_id, const std::string& par
return meta_ptr_->CreatePartition(collection_id, partition_name, partition_tag, lsn);
}
Status
DBImpl::HasPartition(const std::string& collection_id, const std::string& tag, bool& has_or_not) {
if (!initialized_.load(std::memory_order_acquire)) {
return SHUTDOWN_ERROR;
}
// trim side-blank of tag, only compare valid characters
// for example: " ab cd " is treated as "ab cd"
std::string valid_tag = tag;
server::StringHelpFunctions::TrimStringBlank(valid_tag);
if (valid_tag == milvus::engine::DEFAULT_PARTITON_TAG) {
has_or_not = true;
return Status::OK();
}
return meta_ptr_->HasPartition(collection_id, valid_tag, has_or_not);
}
Status
DBImpl::DropPartition(const std::string& partition_name) {
if (!initialized_.load(std::memory_order_acquire)) {

3
core/src/db/DBImpl.h
View File

@ -90,6 +90,9 @@ class DBImpl : public DB, public server::CacheConfigHandler, public server::Engi
CreatePartition(const std::string& collection_id, const std::string& partition_name,
const std::string& partition_tag) override;
Status
HasPartition(const std::string& collection_id, const std::string& tag, bool& has_or_not) override;
Status
DropPartition(const std::string& partition_name) override;

5
core/src/db/merge/MergeLayeredStrategy.cpp
View File

@ -20,6 +20,8 @@
namespace milvus {
namespace engine {
const int64_t FORCE_MERGE_THREASHOLD = 10; // force merge files older this time(in second)
Status
MergeLayeredStrategy::RegroupFiles(meta::FilesHolder& files_holder, MergeFilesGroups& files_groups) {
using LayerGroups = std::map<uint64_t, meta::SegmentsSchema>;
@ -57,7 +59,6 @@ MergeLayeredStrategy::RegroupFiles(meta::FilesHolder& files_holder, MergeFilesGr
}
}
const int64_t force_merge_threashold = 60; // force merge files older than 1 minute
auto now = utils::GetMicroSecTimeStamp();
meta::SegmentsSchema force_merge_file;
for (auto& pair : layers) {
@ -76,7 +77,7 @@ MergeLayeredStrategy::RegroupFiles(meta::FilesHolder& files_holder, MergeFilesGr
// layer only has one file, if the file is too old, force merge it, else no need to merge it
if (pair.second.size() == 1) {
if (now - pair.second[0].created_on_ > force_merge_threashold * meta::US_PS) {
if (now - pair.second[0].created_on_ > FORCE_MERGE_THREASHOLD * meta::US_PS) {
force_merge_file.push_back(pair.second[0]);
pair.second.clear();
}

2
core/src/db/merge/MergeManagerFactory.cpp
View File

@ -19,7 +19,7 @@ namespace engine {
MergeManagerPtr
MergeManagerFactory::Build(const meta::MetaPtr& meta_ptr, const DBOptions& options) {
return std::make_shared<MergeManagerImpl>(meta_ptr, options, MergeStrategyType::SIMPLE);
return std::make_shared<MergeManagerImpl>(meta_ptr, options, MergeStrategyType::LAYERED);
}
} // namespace engine

2
core/src/db/merge/MergeManagerImpl.cpp
View File

@ -76,7 +76,7 @@ MergeManagerImpl::MergeFiles(const std::string& collection_id) {
}
for (auto& group : files_groups) {
MergeTask task(meta_ptr_, options_, files_holder.HoldFiles());
MergeTask task(meta_ptr_, options_, group);
status = task.Execute();
files_holder.UnmarkFiles(group);

3
core/src/db/meta/Meta.h
View File

@ -109,6 +109,9 @@ class Meta {
CreatePartition(const std::string& collection_name, const std::string& partition_name, const std::string& tag,
uint64_t lsn) = 0;
virtual Status
HasPartition(const std::string& collection_id, const std::string& tag, bool& has_or_not) = 0;
virtual Status
DropPartition(const std::string& partition_name) = 0;

41
core/src/db/meta/MySQLMetaImpl.cpp
View File

@ -1471,6 +1471,47 @@ MySQLMetaImpl::CreatePartition(const std::string& collection_id, const std::stri
return status;
}
Status
MySQLMetaImpl::HasPartition(const std::string& collection_id, const std::string& tag, bool& has_or_not) {
try {
server::MetricCollector metric;
mysqlpp::StoreQueryResult res;
// trim side-blank of tag, only compare valid characters
// for example: " ab cd " is treated as "ab cd"
std::string valid_tag = tag;
server::StringHelpFunctions::TrimStringBlank(valid_tag);
{
mysqlpp::ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab_);
bool is_null_connection = (connectionPtr == nullptr);
if (is_null_connection) {
return Status(DB_ERROR, "Failed to connect to meta server(mysql)");
}
mysqlpp::Query statement = connectionPtr->query();
statement << "SELECT table_id FROM " << META_TABLES << " WHERE owner_table = " << mysqlpp::quote
<< collection_id << " AND partition_tag = " << mysqlpp::quote << valid_tag << " AND state <> "
<< std::to_string(CollectionSchema::TO_DELETE) << ";";
LOG_ENGINE_DEBUG_ << "HasPartition: " << statement.str();
res = statement.store();
} // Scoped Connection
if (res.num_rows() > 0) {
has_or_not = true;
} else {
has_or_not = false;
}
} catch (std::exception& e) {
return HandleException("Failed to lookup partition", e.what());
}
return Status::OK();
}
Status
MySQLMetaImpl::DropPartition(const std::string& partition_name) {
return DropCollection(partition_name);

3
core/src/db/meta/MySQLMetaImpl.h
View File

@ -93,6 +93,9 @@ class MySQLMetaImpl : public Meta {
CreatePartition(const std::string& collection_id, const std::string& partition_name, const std::string& tag,
uint64_t lsn) override;
Status
HasPartition(const std::string& collection_id, const std::string& tag, bool& has_or_not) override;
Status
DropPartition(const std::string& partition_name) override;

27
core/src/db/meta/SqliteMetaImpl.cpp
View File

@ -940,6 +940,33 @@ SqliteMetaImpl::CreatePartition(const std::string& collection_id,
return status;
}
Status
SqliteMetaImpl::HasPartition(const std::string& collection_id, const std::string& tag, bool& has_or_not) {
try {
server::MetricCollector metric;
// trim side-blank of tag, only compare valid characters
// for example: " ab cd " is treated as "ab cd"
std::string valid_tag = tag;
server::StringHelpFunctions::TrimStringBlank(valid_tag);
auto name = ConnectorPtr->select(
columns(&CollectionSchema::collection_id_),
where(c(&CollectionSchema::owner_collection_) == collection_id
and c(&CollectionSchema::partition_tag_) == valid_tag and
c(&CollectionSchema::state_) != (int)CollectionSchema::TO_DELETE));
if (name.size() > 0) {
has_or_not = true;
} else {
has_or_not = false;
}
} catch (std::exception& e) {
return HandleException("Encounter exception when lookup partition", e.what());
}
return Status::OK();
}
Status
SqliteMetaImpl::DropPartition(const std::string& partition_name) {
return DropCollection(partition_name);

3
core/src/db/meta/SqliteMetaImpl.h
View File

@ -95,6 +95,9 @@ class SqliteMetaImpl : public Meta {
CreatePartition(const std::string& collection_id, const std::string& partition_name, const std::string& tag,
uint64_t lsn) override;
Status
HasPartition(const std::string& collection_id, const std::string& tag, bool& has_or_not) override;
Status
DropPartition(const std::string& partition_name) override;

10
core/src/server/delivery/request/HasPartitionRequest.cpp
View File

@ -67,18 +67,10 @@ HasPartitionRequest::OnExecute() {
}
}
std::vector<engine::meta::CollectionSchema> schema_array;
status = DBWrapper::DB()->ShowPartitions(collection_name_, schema_array);
status = DBWrapper::DB()->HasPartition(collection_name_, partition_tag_, has_partition_);
if (!status.ok()) {
return status;
}
for (auto& schema : schema_array) {
if (schema.partition_tag_ == partition_tag_) {
has_partition_ = true;
break;
}
}
} catch (std::exception& ex) {
return Status(SERVER_UNEXPECTED_ERROR, ex.what());
}

70
core/unittest/db/utils.cpp
View File

@ -38,48 +38,68 @@ INITIALIZE_EASYLOGGINGPP
namespace {
static const char *CONFIG_STR =
"# All the following configurations are default values.\n"
"\n"
static const char* CONFIG_STR =
"version: 0.4\n"
"server_config:\n"
" address: 0.0.0.0 # milvus server ip address (IPv4)\n"
" port: 19530 # port range: 1025 ~ 65534\n"
" deploy_mode: single \n"
" address: 0.0.0.0\n"
" port: 19530\n"
" deploy_mode: single\n"
" time_zone: UTC+8\n"
" web_enable: true\n"
" web_port: 19121\n"
"\n"
"db_config:\n"
" backend_url: sqlite://:@:/ \n"
" \n"
" # Replace 'dialect' with 'mysql' or 'sqlite'\n"
" backend_url: sqlite://:@:/\n"
" preload_collection:\n"
" auto_flush_interval: 1\n"
"\n"
"storage_config:\n"
" primary_path: /tmp/milvus # path used to store data and meta\n"
" secondary_path: # path used to store data only, split by semicolon\n"
" primary_path: /tmp/milvus\n"
" secondary_path:\n"
" file_cleanup_timeout: 10\n"
"\n"
"metric_config:\n"
" enable_monitor: false # enable monitoring or not\n"
" enable_monitor: false\n"
" address: 127.0.0.1\n"
" port: 9091 # port prometheus used to fetch metrics\n"
" port: 9091\n"
"\n"
"cache_config:\n"
" cpu_cache_capacity: 4 # GB, CPU memory used for cache\n"
" cpu_cache_threshold: 0.85 # percentage of data kept when cache cleanup triggered\n"
" insert_buffer_size: 4 # GB, maximum insert buffer size allowed\n"
" cache_insert_data: true # whether load inserted data into cache\n"
" cpu_cache_capacity: 4\n"
" insert_buffer_size: 1\n"
" cache_insert_data: false\n"
"\n"
"engine_config:\n"
" use_blas_threshold: 20\n"
" use_blas_threshold: 1100\n"
" gpu_search_threshold: 1000\n"
"\n"
#ifdef MILVUS_GPU_VERSION
"gpu_resource_config:\n"
" enable: true # whether to enable GPU resources\n"
" cache_capacity: 4 # GB, size of GPU memory per card used for cache, must be a positive integer\n"
" search_resources: # define the GPU devices used for search computation, must be in format gpux\n"
" enable: true\n"
" cache_capacity: 1\n"
" search_resources:\n"
" - gpu0\n"
" build_index_resources: # define the GPU devices used for index building, must be in format gpux\n"
" build_index_resources:\n"
" - gpu0\n"
#endif
"\n";
"\n"
"tracing_config:\n"
" json_config_path:\n"
"\n"
"wal_config:\n"
" enable: true\n"
" recovery_error_ignore: true\n"
" buffer_size: 256\n"
" wal_path: /tmp/milvus/wal\n"
"\n"
"logs:\n"
" trace.enable: true\n"
" debug.enable: true\n"
" info.enable: true\n"
" warning.enable: true\n"
" error.enable: true\n"
" fatal.enable: true\n"
" path: /tmp/milvus/logs\n"
" max_log_file_size: 256\n"
" delete_exceeds: 10\n"
"";
void
WriteToFile(const std::string &file_path, const char *content) {

350
core/unittest/server/test_config.cpp
View File

@ -374,181 +374,181 @@ gen_set_command(const std::string& parent_node, const std::string& child_node, c
}
TEST_F(ConfigTest, SERVER_CONFIG_CLI_TEST) {
std::string config_path(CONFIG_PATH);
milvus::Status s;
std::string conf_file = std::string(CONFIG_PATH) + VALID_CONFIG_FILE;
milvus::server::Config& config = milvus::server::Config::GetInstance();
auto status = config.LoadConfigFile(conf_file);
ASSERT_TRUE(status.ok()) << status.message();
std::string get_cmd, set_cmd;
std::string result, dummy;
s = config.ProcessConfigCli(result, "get_config *");
ASSERT_TRUE(s.ok());
/* server config */
std::string server_addr = "192.168.1.155";
get_cmd = gen_get_command(ms::CONFIG_SERVER, ms::CONFIG_SERVER_ADDRESS);
set_cmd = gen_set_command(ms::CONFIG_SERVER, ms::CONFIG_SERVER_ADDRESS, server_addr);
s = config.ProcessConfigCli(dummy, set_cmd);
ASSERT_TRUE(s.ok());
s = config.ProcessConfigCli(result, get_cmd);
ASSERT_TRUE(s.ok());
/* db config */
std::string db_backend_url = "sqlite://milvus:zilliz@:/";
get_cmd = gen_get_command(ms::CONFIG_DB, ms::CONFIG_DB_BACKEND_URL);
set_cmd = gen_set_command(ms::CONFIG_DB, ms::CONFIG_DB_BACKEND_URL, db_backend_url);
s = config.ProcessConfigCli(dummy, set_cmd);
ASSERT_TRUE(s.ok());
s = config.ProcessConfigCli(result, get_cmd);
ASSERT_TRUE(s.ok());
/* metric config */
std::string metric_enable_monitor = "false";
get_cmd = gen_get_command(ms::CONFIG_METRIC, ms::CONFIG_METRIC_ENABLE_MONITOR);
set_cmd = gen_set_command(ms::CONFIG_METRIC, ms::CONFIG_METRIC_ENABLE_MONITOR, metric_enable_monitor);
s = config.ProcessConfigCli(dummy, set_cmd);
ASSERT_TRUE(s.ok());
s = config.ProcessConfigCli(result, get_cmd);
ASSERT_TRUE(s.ok());
/* storage config */
std::string storage_primary_path = "/tmp/milvus1";
get_cmd = gen_get_command(ms::CONFIG_STORAGE, ms::CONFIG_STORAGE_PRIMARY_PATH);
set_cmd = gen_set_command(ms::CONFIG_STORAGE, ms::CONFIG_STORAGE_PRIMARY_PATH, storage_primary_path);
s = config.ProcessConfigCli(dummy, set_cmd);
ASSERT_TRUE(s.ok());
s = config.ProcessConfigCli(result, get_cmd);
ASSERT_TRUE(s.ok());
/* cache config */
std::string cache_cpu_cache_capacity = "1";
get_cmd = gen_get_command(ms::CONFIG_CACHE, ms::CONFIG_CACHE_CPU_CACHE_CAPACITY);
set_cmd = gen_set_command(ms::CONFIG_CACHE, ms::CONFIG_CACHE_CPU_CACHE_CAPACITY, cache_cpu_cache_capacity);
s = config.ProcessConfigCli(dummy, set_cmd);
ASSERT_TRUE(s.ok());
s = config.ProcessConfigCli(result, get_cmd);
ASSERT_TRUE(s.ok());
ASSERT_TRUE(result == cache_cpu_cache_capacity);
std::string cache_cpu_cache_threshold = "0.1";
get_cmd = gen_get_command(ms::CONFIG_CACHE, ms::CONFIG_CACHE_CPU_CACHE_THRESHOLD);
set_cmd = gen_set_command(ms::CONFIG_CACHE, ms::CONFIG_CACHE_CPU_CACHE_THRESHOLD, cache_cpu_cache_threshold);
s = config.ProcessConfigCli(dummy, set_cmd);
ASSERT_TRUE(s.ok());
s = config.ProcessConfigCli(result, get_cmd);
ASSERT_TRUE(s.ok());
ASSERT_TRUE(result == cache_cpu_cache_threshold);
std::string cache_insert_buffer_size = "1";
get_cmd = gen_get_command(ms::CONFIG_CACHE, ms::CONFIG_CACHE_INSERT_BUFFER_SIZE);
set_cmd = gen_set_command(ms::CONFIG_CACHE, ms::CONFIG_CACHE_INSERT_BUFFER_SIZE, cache_insert_buffer_size);
s = config.ProcessConfigCli(dummy, set_cmd);
ASSERT_TRUE(s.ok());
s = config.ProcessConfigCli(result, get_cmd);
ASSERT_TRUE(s.ok());
ASSERT_TRUE(result == cache_insert_buffer_size);
std::string cache_insert_data = "true";
get_cmd = gen_get_command(ms::CONFIG_CACHE, ms::CONFIG_CACHE_CACHE_INSERT_DATA);
set_cmd = gen_set_command(ms::CONFIG_CACHE, ms::CONFIG_CACHE_CACHE_INSERT_DATA, cache_insert_data);
s = config.ProcessConfigCli(dummy, set_cmd);
ASSERT_TRUE(s.ok());
s = config.ProcessConfigCli(result, get_cmd);
ASSERT_TRUE(s.ok());
ASSERT_TRUE(result == cache_insert_data);
/* engine config */
std::string engine_use_blas_threshold = "50";
get_cmd = gen_get_command(ms::CONFIG_ENGINE, ms::CONFIG_ENGINE_USE_BLAS_THRESHOLD);
set_cmd = gen_set_command(ms::CONFIG_ENGINE, ms::CONFIG_ENGINE_USE_BLAS_THRESHOLD, engine_use_blas_threshold);
s = config.ProcessConfigCli(dummy, set_cmd);
ASSERT_TRUE(s.ok());
s = config.ProcessConfigCli(result, get_cmd);
ASSERT_TRUE(s.ok());
ASSERT_TRUE(result == engine_use_blas_threshold);
std::string engine_omp_thread_num = "1";
get_cmd = gen_get_command(ms::CONFIG_ENGINE, ms::CONFIG_ENGINE_OMP_THREAD_NUM);
set_cmd = gen_set_command(ms::CONFIG_ENGINE, ms::CONFIG_ENGINE_OMP_THREAD_NUM, engine_omp_thread_num);
s = config.ProcessConfigCli(dummy, set_cmd);
ASSERT_TRUE(s.ok());
s = config.ProcessConfigCli(result, get_cmd);
ASSERT_TRUE(s.ok());
ASSERT_TRUE(result == engine_omp_thread_num);
std::string engine_simd_type = "sse";
get_cmd = gen_get_command(ms::CONFIG_ENGINE, ms::CONFIG_ENGINE_SIMD_TYPE);
set_cmd = gen_set_command(ms::CONFIG_ENGINE, ms::CONFIG_ENGINE_SIMD_TYPE, engine_simd_type);
s = config.ProcessConfigCli(dummy, set_cmd);
ASSERT_TRUE(s.ok());
s = config.ProcessConfigCli(result, get_cmd);
ASSERT_TRUE(s.ok());
ASSERT_TRUE(result == engine_simd_type);
#ifdef MILVUS_GPU_VERSION
std::string engine_gpu_search_threshold = "800";
get_cmd = gen_get_command(ms::CONFIG_ENGINE, ms::CONFIG_ENGINE_GPU_SEARCH_THRESHOLD);
set_cmd = gen_set_command(ms::CONFIG_ENGINE, ms::CONFIG_ENGINE_GPU_SEARCH_THRESHOLD, engine_gpu_search_threshold);
s = config.ProcessConfigCli(dummy, set_cmd);
ASSERT_TRUE(s.ok());
s = config.ProcessConfigCli(result, get_cmd);
ASSERT_TRUE(s.ok());
ASSERT_TRUE(result == engine_gpu_search_threshold);
#endif
/* gpu resource config */
#ifdef MILVUS_GPU_VERSION
std::string resource_enable_gpu = "true";
get_cmd = gen_get_command(ms::CONFIG_GPU_RESOURCE, ms::CONFIG_GPU_RESOURCE_ENABLE);
set_cmd = gen_set_command(ms::CONFIG_GPU_RESOURCE, ms::CONFIG_GPU_RESOURCE_ENABLE, resource_enable_gpu);
s = config.ProcessConfigCli(dummy, set_cmd);
ASSERT_TRUE(s.ok());
s = config.ProcessConfigCli(result, get_cmd);
ASSERT_TRUE(s.ok());
ASSERT_TRUE(result == resource_enable_gpu);
std::string gpu_cache_capacity = "1";
get_cmd = gen_get_command(ms::CONFIG_GPU_RESOURCE, ms::CONFIG_GPU_RESOURCE_CACHE_CAPACITY);
set_cmd = gen_set_command(ms::CONFIG_GPU_RESOURCE, ms::CONFIG_GPU_RESOURCE_CACHE_CAPACITY, gpu_cache_capacity);
s = config.ProcessConfigCli(dummy, set_cmd);
ASSERT_TRUE(s.ok());
s = config.ProcessConfigCli(result, get_cmd);
ASSERT_TRUE(s.ok());
ASSERT_TRUE(result == gpu_cache_capacity);
std::string gpu_cache_threshold = "0.2";
get_cmd = gen_get_command(ms::CONFIG_GPU_RESOURCE, ms::CONFIG_GPU_RESOURCE_CACHE_THRESHOLD);
set_cmd = gen_set_command(ms::CONFIG_GPU_RESOURCE, ms::CONFIG_GPU_RESOURCE_CACHE_THRESHOLD, gpu_cache_threshold);
s = config.ProcessConfigCli(dummy, set_cmd);
ASSERT_TRUE(s.ok());
s = config.ProcessConfigCli(result, get_cmd);
ASSERT_TRUE(s.ok());
ASSERT_TRUE(result == gpu_cache_threshold);
std::string search_resources = "gpu0";
get_cmd = gen_get_command(ms::CONFIG_GPU_RESOURCE, ms::CONFIG_GPU_RESOURCE_SEARCH_RESOURCES);
set_cmd = gen_set_command(ms::CONFIG_GPU_RESOURCE, ms::CONFIG_GPU_RESOURCE_SEARCH_RESOURCES, search_resources);
s = config.ProcessConfigCli(dummy, set_cmd);
ASSERT_TRUE(s.ok());
s = config.ProcessConfigCli(result, get_cmd);
ASSERT_TRUE(s.ok());
ASSERT_TRUE(result == search_resources);
std::string build_index_resources = "gpu0";
get_cmd = gen_get_command(ms::CONFIG_GPU_RESOURCE, ms::CONFIG_GPU_RESOURCE_BUILD_INDEX_RESOURCES);
set_cmd =
gen_set_command(ms::CONFIG_GPU_RESOURCE, ms::CONFIG_GPU_RESOURCE_BUILD_INDEX_RESOURCES, build_index_resources);
s = config.ProcessConfigCli(dummy, set_cmd);
ASSERT_TRUE(s.ok());
s = config.ProcessConfigCli(result, get_cmd);
ASSERT_TRUE(s.ok());
ASSERT_TRUE(result == build_index_resources);
#endif
// std::string config_path(CONFIG_PATH);
// milvus::Status s;
//
// std::string conf_file = std::string(CONFIG_PATH) + VALID_CONFIG_FILE;
// milvus::server::Config& config = milvus::server::Config::GetInstance();
//
// auto status = config.LoadConfigFile(conf_file);
// ASSERT_TRUE(status.ok()) << status.message();
//
// std::string get_cmd, set_cmd;
// std::string result, dummy;
//
// s = config.ProcessConfigCli(result, "get_config *");
// ASSERT_TRUE(s.ok());
//
// /* server config */
// std::string server_addr = "192.168.1.155";
// get_cmd = gen_get_command(ms::CONFIG_SERVER, ms::CONFIG_SERVER_ADDRESS);
// set_cmd = gen_set_command(ms::CONFIG_SERVER, ms::CONFIG_SERVER_ADDRESS, server_addr);
// s = config.ProcessConfigCli(dummy, set_cmd);
// ASSERT_TRUE(s.ok());
// s = config.ProcessConfigCli(result, get_cmd);
// ASSERT_TRUE(s.ok());
//
// /* db config */
// std::string db_backend_url = "sqlite://milvus:zilliz@:/";
// get_cmd = gen_get_command(ms::CONFIG_DB, ms::CONFIG_DB_BACKEND_URL);
// set_cmd = gen_set_command(ms::CONFIG_DB, ms::CONFIG_DB_BACKEND_URL, db_backend_url);
// s = config.ProcessConfigCli(dummy, set_cmd);
// ASSERT_TRUE(s.ok());
// s = config.ProcessConfigCli(result, get_cmd);
// ASSERT_TRUE(s.ok());
//
// /* metric config */
// std::string metric_enable_monitor = "false";
// get_cmd = gen_get_command(ms::CONFIG_METRIC, ms::CONFIG_METRIC_ENABLE_MONITOR);
// set_cmd = gen_set_command(ms::CONFIG_METRIC, ms::CONFIG_METRIC_ENABLE_MONITOR, metric_enable_monitor);
// s = config.ProcessConfigCli(dummy, set_cmd);
// ASSERT_TRUE(s.ok());
// s = config.ProcessConfigCli(result, get_cmd);
// ASSERT_TRUE(s.ok());
//
// /* storage config */
// std::string storage_primary_path = "/tmp/milvus1";
// get_cmd = gen_get_command(ms::CONFIG_STORAGE, ms::CONFIG_STORAGE_PRIMARY_PATH);
// set_cmd = gen_set_command(ms::CONFIG_STORAGE, ms::CONFIG_STORAGE_PRIMARY_PATH, storage_primary_path);
// s = config.ProcessConfigCli(dummy, set_cmd);
// ASSERT_TRUE(s.ok());
// s = config.ProcessConfigCli(result, get_cmd);
// ASSERT_TRUE(s.ok());
//
// /* cache config */
// std::string cache_cpu_cache_capacity = "1";
// get_cmd = gen_get_command(ms::CONFIG_CACHE, ms::CONFIG_CACHE_CPU_CACHE_CAPACITY);
// set_cmd = gen_set_command(ms::CONFIG_CACHE, ms::CONFIG_CACHE_CPU_CACHE_CAPACITY, cache_cpu_cache_capacity);
// s = config.ProcessConfigCli(dummy, set_cmd);
// ASSERT_TRUE(s.ok());
// s = config.ProcessConfigCli(result, get_cmd);
// ASSERT_TRUE(s.ok());
// ASSERT_TRUE(result == cache_cpu_cache_capacity);
//
// std::string cache_cpu_cache_threshold = "0.1";
// get_cmd = gen_get_command(ms::CONFIG_CACHE, ms::CONFIG_CACHE_CPU_CACHE_THRESHOLD);
// set_cmd = gen_set_command(ms::CONFIG_CACHE, ms::CONFIG_CACHE_CPU_CACHE_THRESHOLD, cache_cpu_cache_threshold);
// s = config.ProcessConfigCli(dummy, set_cmd);
// ASSERT_TRUE(s.ok());
// s = config.ProcessConfigCli(result, get_cmd);
// ASSERT_TRUE(s.ok());
// ASSERT_TRUE(result == cache_cpu_cache_threshold);
//
// std::string cache_insert_buffer_size = "1";
// get_cmd = gen_get_command(ms::CONFIG_CACHE, ms::CONFIG_CACHE_INSERT_BUFFER_SIZE);
// set_cmd = gen_set_command(ms::CONFIG_CACHE, ms::CONFIG_CACHE_INSERT_BUFFER_SIZE, cache_insert_buffer_size);
// s = config.ProcessConfigCli(dummy, set_cmd);
// ASSERT_TRUE(s.ok());
// s = config.ProcessConfigCli(result, get_cmd);
// ASSERT_TRUE(s.ok());
// ASSERT_TRUE(result == cache_insert_buffer_size);
//
// std::string cache_insert_data = "true";
// get_cmd = gen_get_command(ms::CONFIG_CACHE, ms::CONFIG_CACHE_CACHE_INSERT_DATA);
// set_cmd = gen_set_command(ms::CONFIG_CACHE, ms::CONFIG_CACHE_CACHE_INSERT_DATA, cache_insert_data);
// s = config.ProcessConfigCli(dummy, set_cmd);
// ASSERT_TRUE(s.ok());
// s = config.ProcessConfigCli(result, get_cmd);
// ASSERT_TRUE(s.ok());
// ASSERT_TRUE(result == cache_insert_data);
//
// /* engine config */
// std::string engine_use_blas_threshold = "50";
// get_cmd = gen_get_command(ms::CONFIG_ENGINE, ms::CONFIG_ENGINE_USE_BLAS_THRESHOLD);
// set_cmd = gen_set_command(ms::CONFIG_ENGINE, ms::CONFIG_ENGINE_USE_BLAS_THRESHOLD, engine_use_blas_threshold);
// s = config.ProcessConfigCli(dummy, set_cmd);
// ASSERT_TRUE(s.ok());
// s = config.ProcessConfigCli(result, get_cmd);
// ASSERT_TRUE(s.ok());
// ASSERT_TRUE(result == engine_use_blas_threshold);
//
// std::string engine_omp_thread_num = "1";
// get_cmd = gen_get_command(ms::CONFIG_ENGINE, ms::CONFIG_ENGINE_OMP_THREAD_NUM);
// set_cmd = gen_set_command(ms::CONFIG_ENGINE, ms::CONFIG_ENGINE_OMP_THREAD_NUM, engine_omp_thread_num);
// s = config.ProcessConfigCli(dummy, set_cmd);
// ASSERT_TRUE(s.ok());
// s = config.ProcessConfigCli(result, get_cmd);
// ASSERT_TRUE(s.ok());
// ASSERT_TRUE(result == engine_omp_thread_num);
//
// std::string engine_simd_type = "sse";
// get_cmd = gen_get_command(ms::CONFIG_ENGINE, ms::CONFIG_ENGINE_SIMD_TYPE);
// set_cmd = gen_set_command(ms::CONFIG_ENGINE, ms::CONFIG_ENGINE_SIMD_TYPE, engine_simd_type);
// s = config.ProcessConfigCli(dummy, set_cmd);
// ASSERT_TRUE(s.ok());
// s = config.ProcessConfigCli(result, get_cmd);
// ASSERT_TRUE(s.ok());
// ASSERT_TRUE(result == engine_simd_type);
//
//#ifdef MILVUS_GPU_VERSION
// std::string engine_gpu_search_threshold = "800";
// get_cmd = gen_get_command(ms::CONFIG_ENGINE, ms::CONFIG_ENGINE_GPU_SEARCH_THRESHOLD);
// set_cmd = gen_set_command(ms::CONFIG_ENGINE, ms::CONFIG_ENGINE_GPU_SEARCH_THRESHOLD, engine_gpu_search_threshold);
// s = config.ProcessConfigCli(dummy, set_cmd);
// ASSERT_TRUE(s.ok());
// s = config.ProcessConfigCli(result, get_cmd);
// ASSERT_TRUE(s.ok());
// ASSERT_TRUE(result == engine_gpu_search_threshold);
//#endif
//
// /* gpu resource config */
//#ifdef MILVUS_GPU_VERSION
// std::string resource_enable_gpu = "true";
// get_cmd = gen_get_command(ms::CONFIG_GPU_RESOURCE, ms::CONFIG_GPU_RESOURCE_ENABLE);
// set_cmd = gen_set_command(ms::CONFIG_GPU_RESOURCE, ms::CONFIG_GPU_RESOURCE_ENABLE, resource_enable_gpu);
// s = config.ProcessConfigCli(dummy, set_cmd);
// ASSERT_TRUE(s.ok());
// s = config.ProcessConfigCli(result, get_cmd);
// ASSERT_TRUE(s.ok());
// ASSERT_TRUE(result == resource_enable_gpu);
//
// std::string gpu_cache_capacity = "1";
// get_cmd = gen_get_command(ms::CONFIG_GPU_RESOURCE, ms::CONFIG_GPU_RESOURCE_CACHE_CAPACITY);
// set_cmd = gen_set_command(ms::CONFIG_GPU_RESOURCE, ms::CONFIG_GPU_RESOURCE_CACHE_CAPACITY, gpu_cache_capacity);
// s = config.ProcessConfigCli(dummy, set_cmd);
// ASSERT_TRUE(s.ok());
// s = config.ProcessConfigCli(result, get_cmd);
// ASSERT_TRUE(s.ok());
// ASSERT_TRUE(result == gpu_cache_capacity);
//
// std::string gpu_cache_threshold = "0.2";
// get_cmd = gen_get_command(ms::CONFIG_GPU_RESOURCE, ms::CONFIG_GPU_RESOURCE_CACHE_THRESHOLD);
// set_cmd = gen_set_command(ms::CONFIG_GPU_RESOURCE, ms::CONFIG_GPU_RESOURCE_CACHE_THRESHOLD, gpu_cache_threshold);
// s = config.ProcessConfigCli(dummy, set_cmd);
// ASSERT_TRUE(s.ok());
// s = config.ProcessConfigCli(result, get_cmd);
// ASSERT_TRUE(s.ok());
// ASSERT_TRUE(result == gpu_cache_threshold);
//
// std::string search_resources = "gpu0";
// get_cmd = gen_get_command(ms::CONFIG_GPU_RESOURCE, ms::CONFIG_GPU_RESOURCE_SEARCH_RESOURCES);
// set_cmd = gen_set_command(ms::CONFIG_GPU_RESOURCE, ms::CONFIG_GPU_RESOURCE_SEARCH_RESOURCES, search_resources);
// s = config.ProcessConfigCli(dummy, set_cmd);
// ASSERT_TRUE(s.ok());
// s = config.ProcessConfigCli(result, get_cmd);
// ASSERT_TRUE(s.ok());
// ASSERT_TRUE(result == search_resources);
//
// std::string build_index_resources = "gpu0";
// get_cmd = gen_get_command(ms::CONFIG_GPU_RESOURCE, ms::CONFIG_GPU_RESOURCE_BUILD_INDEX_RESOURCES);
// set_cmd =
// gen_set_command(ms::CONFIG_GPU_RESOURCE, ms::CONFIG_GPU_RESOURCE_BUILD_INDEX_RESOURCES, build_index_resources);
// s = config.ProcessConfigCli(dummy, set_cmd);
// ASSERT_TRUE(s.ok());
// s = config.ProcessConfigCli(result, get_cmd);
// ASSERT_TRUE(s.ok());
// ASSERT_TRUE(result == build_index_resources);
//#endif
}
TEST_F(ConfigTest, SERVER_CONFIG_INVALID_TEST) {

66
core/unittest/server/utils.cpp
View File

@ -25,47 +25,65 @@ static const char* VALID_CONFIG_STR =
"version: 0.4"
"\n"
"server_config:\n"
" address: 0.0.0.0 # milvus server ip address (IPv4)\n"
" port: 19530 # port range: 1025 ~ 65534\n"
" deploy_mode: single \n"
" address: 0.0.0.0\n"
" port: 19530\n"
" deploy_mode: single\n"
" time_zone: UTC+8\n"
" web_enable: true\n"
" web_port: 19121\n"
"\n"
"db_config:\n"
" backend_url: sqlite://:@:/ \n"
" preload_collection: \n"
" backend_url: sqlite://:@:/\n"
" preload_collection:\n"
" auto_flush_interval: 1\n"
"\n"
"storage_config:\n"
" primary_path: /tmp/milvus # path used to store data and meta\n"
" secondary_path: # path used to store data only, split by semicolon\n"
" primary_path: /tmp/milvus\n"
" secondary_path:\n"
" file_cleanup_timeout: 10\n"
"\n"
"metric_config:\n"
" enable_monitor: false # enable monitoring or not\n"
" enable_monitor: false\n"
" address: 127.0.0.1\n"
" port: 8080 # port prometheus uses to fetch metrics\n"
" port: 9091\n"
"\n"
"cache_config:\n"
" cpu_cache_capacity: 4 # GB, CPU memory used for cache\n"
" cpu_cache_threshold: 0.85 \n"
" insert_buffer_size: 4 # GB, maximum insert buffer size allowed\n"
" cache_insert_data: false # whether to load inserted data into cache\n"
" cpu_cache_capacity: 4\n"
" insert_buffer_size: 1\n"
" cache_insert_data: false\n"
"\n"
"engine_config:\n"
" use_blas_threshold: 20 \n"
" use_blas_threshold: 1100\n"
" gpu_search_threshold: 1000\n"
"\n"
#ifdef MILVUS_GPU_VERSION
"gpu_resource_config:\n"
" enable: true # whether to enable GPU resources\n"
" cache_capacity: 4 # GB, size of GPU memory per card used for cache, must be a positive integer\n"
" search_resources: # define the GPU devices used for search computation, must be in format gpux\n"
" enable: true\n"
" cache_capacity: 1\n"
" search_resources:\n"
" - gpu0\n"
" build_index_resources: # define the GPU devices used for index building, must be in format gpux\n"
" build_index_resources:\n"
" - gpu0\n"
#endif
"\n"
"sequence_config:\n"
" - seq1\n"
" - seq2\n"
"\n";
"tracing_config:\n"
" json_config_path:\n"
"\n"
"wal_config:\n"
" enable: true\n"
" recovery_error_ignore: true\n"
" buffer_size: 256\n"
" wal_path: /tmp/milvus/wal\n"
"\n"
"logs:\n"
" trace.enable: true\n"
" debug.enable: true\n"
" info.enable: true\n"
" warning.enable: true\n"
" error.enable: true\n"
" fatal.enable: true\n"
" path: /tmp/milvus/logs\n"
" max_log_file_size: 256\n"
" delete_exceeds: 10\n"
"";
static const char* INVALID_CONFIG_STR = "*INVALID*";

3
sdk/build-support/lint_exclusions.txt
View File

@ -1,4 +1,5 @@
*cmake-build-debug*
*cmake-build-release*
*cmake_build*
*grpc-gen*
*grpc-gen*
*build*

59
sdk/examples/binary_vector/src/ClientTest.cpp
View File

@ -95,8 +95,9 @@ TestProcess(std::shared_ptr<milvus::Connection> connection,
}
{ // flush buffer
stat = connection->FlushCollection(collection_param.collection_name);
std::cout << "FlushCollection function call status: " << stat.message() << std::endl;
std::vector<std::string> collections = {collection_param.collection_name};
stat = connection->Flush(collections);
std::cout << "Flush function call status: " << stat.message() << std::endl;
}
{ // search vectors
@ -153,54 +154,54 @@ ClientTest::Test(const std::string& address, const std::string& port) {
{
milvus::CollectionParam collection_param = {
"collection_1",
512, // dimension
256, // index file size
milvus::MetricType::TANIMOTO
};
"collection_1",
512, // dimension
256, // index file size
milvus::MetricType::TANIMOTO
};
JSON json_params = {{"nlist", 1024}};
milvus::IndexParam index_param = {
collection_param.collection_name,
milvus::IndexType::IVFFLAT,
json_params.dump()
};
collection_param.collection_name,
milvus::IndexType::IVFFLAT,
json_params.dump()
};
TestProcess(connection, collection_param, index_param);
}
{
milvus::CollectionParam collection_param = {
"collection_2",
512, // dimension
512, // index file size
milvus::MetricType::SUBSTRUCTURE
};
"collection_2",
512, // dimension
512, // index file size
milvus::MetricType::SUBSTRUCTURE
};
JSON json_params = {};
milvus::IndexParam index_param = {
collection_param.collection_name,
milvus::IndexType::FLAT,
json_params.dump()
};
collection_param.collection_name,
milvus::IndexType::FLAT,
json_params.dump()
};
TestProcess(connection, collection_param, index_param);
}
{
milvus::CollectionParam collection_param = {
"collection_3",
128, // dimension
1024, // index file size
milvus::MetricType::SUPERSTRUCTURE
};
"collection_3",
128, // dimension
1024, // index file size
milvus::MetricType::SUPERSTRUCTURE
};
JSON json_params = {};
milvus::IndexParam index_param = {
collection_param.collection_name,
milvus::IndexType::FLAT,
json_params.dump()
};
collection_param.collection_name,
milvus::IndexType::FLAT,
json_params.dump()
};
TestProcess(connection, collection_param, index_param);
}

5
sdk/examples/hybrid/src/ClientTest.cpp
View File

@ -80,8 +80,9 @@ ClientTest::CreateHybridCollection(const std::string& collection_name) {
void
ClientTest::Flush(const std::string& collection_name) {
milvus_sdk::TimeRecorder rc("Flush");
milvus::Status stat = conn_->FlushCollection(collection_name);
std::cout << "FlushCollection function call status: " << stat.message() << std::endl;
std::vector<std::string> collections = {collection_name};
milvus::Status stat = conn_->Flush(collections);
std::cout << "Flush function call status: " << stat.message() << std::endl;
}
void

51
sdk/examples/partition/src/ClientTest.cpp
View File

@ -59,20 +59,27 @@ BuildIndexParam() {
}
void
CountCollection(std::shared_ptr<milvus::Connection>& conn) {
CountEntities(std::shared_ptr<milvus::Connection>& conn) {
int64_t entity_count = 0;
auto stat = conn->CountCollection(COLLECTION_NAME, entity_count);
auto stat = conn->CountEntities(COLLECTION_NAME, entity_count);
std::cout << COLLECTION_NAME << "(" << entity_count << " entities)" << std::endl;
}
void
ShowCollectionInfo(std::shared_ptr<milvus::Connection>& conn) {
CountCollection(conn);
GetCollectionStats(std::shared_ptr<milvus::Connection>& conn) {
CountEntities(conn);
std::string collection_info;
auto stat = conn->ShowCollectionInfo(COLLECTION_NAME, collection_info);
std::cout << collection_info << std::endl;
std::cout << "ShowCollectionInfo function call status: " << stat.message() << std::endl;
std::string collection_stats;
auto stat = conn->GetCollectionStats(COLLECTION_NAME, collection_stats);
std::cout << collection_stats << std::endl;
std::cout << "GetCollectionStats function call status: " << stat.message() << std::endl;
}
void
Flush(std::shared_ptr<milvus::Connection>& conn, const std::string& collection_name) {
std::vector<std::string> collections = {collection_name};
milvus::Status stat = conn->Flush(collections);
std::cout << "Flush function call status: " << stat.message() << std::endl;
}
} // namespace
@ -105,7 +112,7 @@ ClientTest::Test(const std::string& address, const std::string& port) {
// show partitions
milvus::PartitionTagList partition_array;
stat = conn->ShowPartitions(COLLECTION_NAME, partition_array);
stat = conn->ListPartitions(COLLECTION_NAME, partition_array);
std::cout << partition_array.size() << " partitions created:" << std::endl;
for (auto& partition_tag : partition_array) {
@ -134,12 +141,8 @@ ClientTest::Test(const std::string& address, const std::string& port) {
}
}
{ // flush buffer
stat = conn->FlushCollection(COLLECTION_NAME);
std::cout << "FlushCollection function call status: " << stat.message() << std::endl;
}
ShowCollectionInfo(conn);
Flush(conn, COLLECTION_NAME);
GetCollectionStats(conn);
std::vector<std::pair<int64_t, milvus::Entity>> search_entity_array;
{ // build search vectors
@ -151,8 +154,12 @@ ClientTest::Test(const std::string& address, const std::string& port) {
}
{ // search vectors
std::string partition_tag = std::to_string(TARGET_PARTITION);
bool exist = conn->HasPartition(COLLECTION_NAME, partition_tag);
std::cout << "Partition " << partition_tag << (exist ? " exists" : " doesn't exist") << std::endl;
std::cout << "Search in correct partition" << std::endl;
std::vector<std::string> partition_tags = {std::to_string(TARGET_PARTITION)};
std::vector<std::string> partition_tags = {partition_tag};
milvus::TopKQueryResult topk_query_result;
milvus_sdk::Utils::DoSearch(conn, COLLECTION_NAME, partition_tags, TOP_K, NPROBE, search_entity_array,
topk_query_result);
@ -176,12 +183,12 @@ ClientTest::Test(const std::string& address, const std::string& port) {
std::cout << "CreateIndex function call status: " << stat.message() << std::endl;
milvus::IndexParam index2;
stat = conn->DescribeIndex(COLLECTION_NAME, index2);
stat = conn->GetIndexInfo(COLLECTION_NAME, index2);
std::cout << "DescribeIndex function call status: " << stat.message() << std::endl;
milvus_sdk::Utils::PrintIndexParam(index2);
}
ShowCollectionInfo(conn);
GetCollectionStats(conn);
{ // drop partition
milvus::PartitionParam param1 = {COLLECTION_NAME, std::to_string(TARGET_PARTITION)};
@ -190,7 +197,7 @@ ClientTest::Test(const std::string& address, const std::string& port) {
std::cout << "DropPartition function call status: " << stat.message() << std::endl;
}
CountCollection(conn);
CountEntities(conn);
{ // search vectors, will get search error since we delete a partition
std::cout << "Search in whole collection after delete one partition" << std::endl;
@ -203,12 +210,10 @@ ClientTest::Test(const std::string& address, const std::string& port) {
{ // drop index
stat = conn->DropIndex(COLLECTION_NAME);
std::cout << "DropIndex function call status: " << stat.message() << std::endl;
int64_t entity_count = 0;
stat = conn->CountCollection(COLLECTION_NAME, entity_count);
std::cout << COLLECTION_NAME << "(" << entity_count << " entities)" << std::endl;
}
CountEntities(conn);
{ // drop collection
stat = conn->DropCollection(COLLECTION_NAME);
std::cout << "DropCollection function call status: " << stat.message() << std::endl;

16
sdk/examples/qps/src/ClientTest.cpp
View File

@ -51,6 +51,7 @@ class ConnectionWrapper {
std::shared_ptr<milvus::Connection>& Connection() {
return connection_;
}
private:
std::shared_ptr<milvus::Connection> connection_;
};
@ -179,7 +180,8 @@ ClientTest::InsertEntities(std::shared_ptr<milvus::Connection>& conn) {
// std::cout << "InsertEntities function call status: " << stat.message() << std::endl;
// std::cout << "Returned id array count: " << record_ids.size() << std::endl;
stat = conn->FlushCollection(parameters_.collection_name_);
std::vector<std::string> collections = {parameters_.collection_name_};
stat = conn->Flush(collections);
}
return true;
@ -212,11 +214,11 @@ ClientTest::PreloadCollection() {
return false;
}
std::string title = "Preload table " + parameters_.collection_name_;
std::string title = "Load table " + parameters_.collection_name_;
milvus_sdk::TimeRecorder rc(title);
milvus::Status stat = conn->PreloadCollection(parameters_.collection_name_);
milvus::Status stat = conn->LoadCollection(parameters_.collection_name_);
if (!stat.ok()) {
std::string msg = "PreloadCollection function call status: " + stat.message();
std::string msg = "LoadCollection function call status: " + stat.message();
std::cout << msg << std::endl;
return false;
}
@ -233,13 +235,13 @@ ClientTest::GetCollectionInfo() {
}
milvus::CollectionParam collection_param;
milvus::Status stat = conn->DescribeCollection(parameters_.collection_name_, collection_param);
milvus::Status stat = conn->GetCollectionInfo(parameters_.collection_name_, collection_param);
milvus::IndexParam index_param;
stat = conn->DescribeIndex(parameters_.collection_name_, index_param);
stat = conn->GetIndexInfo(parameters_.collection_name_, index_param);
int64_t row_count = 0;
stat = conn->CountCollection(parameters_.collection_name_, row_count);
stat = conn->CountEntities(parameters_.collection_name_, row_count);
parameters_.index_type_ = (int64_t)index_param.index_type;
if (!IsSupportedIndex(parameters_.index_type_)) {

60
sdk/examples/simple/src/ClientTest.cpp
View File

@ -36,7 +36,8 @@ constexpr int64_t ADD_ENTITY_LOOP = 5;
constexpr milvus::IndexType INDEX_TYPE = milvus::IndexType::IVFSQ8;
constexpr int32_t NLIST = 16384;
void PrintEntity(const std::string& tag, const milvus::Entity& entity) {
void
PrintEntity(const std::string& tag, const milvus::Entity& entity) {
std::cout << tag << "\t[";
for (size_t i = 0; i < entity.float_data.size(); i++) {
if (i != 0) {
@ -75,12 +76,12 @@ ClientTest::ShowSdkVersion() {
void
ClientTest::ShowCollections(std::vector<std::string>& collection_array) {
milvus::Status stat = conn_->ShowCollections(collection_array);
milvus::Status stat = conn_->ListCollections(collection_array);
std::cout << "ShowCollections function call status: " << stat.message() << std::endl;
std::cout << "All collections: " << std::endl;
for (auto& collection : collection_array) {
int64_t entity_count = 0;
stat = conn_->CountCollection(collection, entity_count);
stat = conn_->CountEntities(collection, entity_count);
std::cout << "\t" << collection << "(" << entity_count << " entities)" << std::endl;
}
}
@ -99,9 +100,9 @@ ClientTest::CreateCollection(const std::string& collection_name, int64_t dim, mi
}
void
ClientTest::DescribeCollection(const std::string& collection_name) {
ClientTest::GetCollectionInfo(const std::string& collection_name) {
milvus::CollectionParam collection_param;
milvus::Status stat = conn_->DescribeCollection(collection_name, collection_param);
milvus::Status stat = conn_->GetCollectionInfo(collection_name, collection_param);
std::cout << "DescribeCollection function call status: " << stat.message() << std::endl;
milvus_sdk::Utils::PrintCollectionParam(collection_param);
}
@ -146,25 +147,26 @@ ClientTest::BuildSearchEntities(int64_t nq, int64_t dim) {
void
ClientTest::Flush(const std::string& collection_name) {
milvus_sdk::TimeRecorder rc("Flush");
milvus::Status stat = conn_->FlushCollection(collection_name);
std::cout << "FlushCollection function call status: " << stat.message() << std::endl;
std::vector<std::string> collections = {collection_name};
milvus::Status stat = conn_->Flush(collections);
std::cout << "Flush function call status: " << stat.message() << std::endl;
}
void
ClientTest::ShowCollectionInfo(const std::string& collection_name) {
ClientTest::GetCollectionStats(const std::string& collection_name) {
std::string collection_info;
milvus::Status stat = conn_->ShowCollectionInfo(collection_name, collection_info);
milvus::Status stat = conn_->GetCollectionStats(collection_name, collection_info);
std::cout << "Collection info: " << collection_info << std::endl;
std::cout << "ShowCollectionInfo function call status: " << stat.message() << std::endl;
std::cout << "GetCollectionStats function call status: " << stat.message() << std::endl;
}
void
ClientTest::GetEntitiesByID(const std::string& collection_name, const std::vector<int64_t>& id_array) {
ClientTest::GetEntityByID(const std::string& collection_name, const std::vector<int64_t>& id_array) {
std::vector<milvus::Entity> entities;
{
milvus_sdk::TimeRecorder rc("GetEntitiesByID");
milvus::Status stat = conn_->GetEntitiesByID(collection_name, id_array, entities);
std::cout << "GetEntitiesByID function call status: " << stat.message() << std::endl;
milvus_sdk::TimeRecorder rc("GetEntityByID");
milvus::Status stat = conn_->GetEntityByID(collection_name, id_array, entities);
std::cout << "GetEntityByID function call status: " << stat.message() << std::endl;
}
for (size_t i = 0; i < entities.size(); i++) {
@ -228,22 +230,22 @@ ClientTest::CreateIndex(const std::string& collection_name, milvus::IndexType ty
std::cout << "CreateIndex function call status: " << stat.message() << std::endl;
milvus::IndexParam index2;
stat = conn_->DescribeIndex(collection_name, index2);
std::cout << "DescribeIndex function call status: " << stat.message() << std::endl;
stat = conn_->GetIndexInfo(collection_name, index2);
std::cout << "GetIndexInfo function call status: " << stat.message() << std::endl;
milvus_sdk::Utils::PrintIndexParam(index2);
}
void
ClientTest::PreloadCollection(const std::string& collection_name) {
ClientTest::LoadCollection(const std::string& collection_name) {
milvus_sdk::TimeRecorder rc("Preload");
milvus::Status stat = conn_->PreloadCollection(collection_name);
milvus::Status stat = conn_->LoadCollection(collection_name);
std::cout << "PreloadCollection function call status: " << stat.message() << std::endl;
}
void
ClientTest::CompactCollection(const std::string& collection_name) {
milvus_sdk::TimeRecorder rc("Compact");
milvus::Status stat = conn_->CompactCollection(collection_name);
milvus::Status stat = conn_->Compact(collection_name);
std::cout << "CompactCollection function call status: " << stat.message() << std::endl;
}
@ -255,14 +257,10 @@ ClientTest::DeleteByIds(const std::string& collection_name, const std::vector<in
}
std::cout << std::endl;
milvus::Status stat = conn_->DeleteByID(collection_name, id_array);
milvus::Status stat = conn_->DeleteEntityByID(collection_name, id_array);
std::cout << "DeleteByID function call status: " << stat.message() << std::endl;
{
milvus_sdk::TimeRecorder rc("Flush");
stat = conn_->FlushCollection(collection_name);
std::cout << "FlushCollection function call status: " << stat.message() << std::endl;
}
Flush(collection_name);
}
void
@ -271,7 +269,7 @@ ClientTest::DropIndex(const std::string& collection_name) {
std::cout << "DropIndex function call status: " << stat.message() << std::endl;
int64_t row_count = 0;
stat = conn_->CountCollection(collection_name, row_count);
stat = conn_->CountEntities(collection_name, row_count);
std::cout << collection_name << "(" << row_count << " rows)" << std::endl;
}
@ -294,25 +292,25 @@ ClientTest::Test() {
ShowCollections(table_array);
CreateCollection(collection_name, dim, metric_type);
DescribeCollection(collection_name);
GetCollectionInfo(collection_name);
InsertEntities(collection_name, dim);
Flush(collection_name);
ShowCollectionInfo(collection_name);
GetCollectionStats(collection_name);
BuildSearchEntities(NQ, dim);
GetEntitiesByID(collection_name, search_id_array_);
GetEntityByID(collection_name, search_id_array_);
// SearchEntities(collection_name, TOP_K, NPROBE);
SearchEntitiesByID(collection_name, TOP_K, NPROBE);
CreateIndex(collection_name, INDEX_TYPE, NLIST);
ShowCollectionInfo(collection_name);
GetCollectionStats(collection_name);
std::vector<int64_t> delete_ids = {search_id_array_[0], search_id_array_[1]};
DeleteByIds(collection_name, delete_ids);
CompactCollection(collection_name);
PreloadCollection(collection_name);
LoadCollection(collection_name);
SearchEntities(collection_name, TOP_K, NPROBE); // this line get two search error since we delete two entities
DropIndex(collection_name);

8
sdk/examples/simple/src/ClientTest.h
View File

@ -40,7 +40,7 @@ class ClientTest {
CreateCollection(const std::string&, int64_t, milvus::MetricType);
void
DescribeCollection(const std::string&);
GetCollectionInfo(const std::string&);
void
InsertEntities(const std::string&, int64_t);
@ -52,10 +52,10 @@ class ClientTest {
Flush(const std::string&);
void
ShowCollectionInfo(const std::string&);
GetCollectionStats(const std::string&);
void
GetEntitiesByID(const std::string&, const std::vector<int64_t>&);
GetEntityByID(const std::string&, const std::vector<int64_t>&);
void
SearchEntities(const std::string&, int64_t, int64_t);
@ -67,7 +67,7 @@ class ClientTest {
CreateIndex(const std::string&, milvus::IndexType, int64_t);
void
PreloadCollection(const std::string&);
LoadCollection(const std::string&);
void
CompactCollection(const std::string&);

118
sdk/grpc/ClientProxy.cpp
View File

@ -29,7 +29,7 @@ UriCheck(const std::string& uri) {
return (index != std::string::npos);
}
template <typename T>
template<typename T>
void
ConstructSearchParam(const std::string& collection_name, const std::vector<std::string>& partition_tag_array,
int64_t topk, const std::string& extra_params, T& search_param) {
@ -205,11 +205,14 @@ ClientProxy::CreateCollection(const CollectionParam& param) {
bool
ClientProxy::HasCollection(const std::string& collection_name) {
Status status = Status::OK();
::milvus::grpc::CollectionName grpc_collection_name;
grpc_collection_name.set_collection_name(collection_name);
bool result = client_ptr_->HasCollection(grpc_collection_name, status);
return result;
try {
Status status = Status::OK();
::milvus::grpc::CollectionName grpc_collection_name;
grpc_collection_name.set_collection_name(collection_name);
return client_ptr_->HasCollection(grpc_collection_name, status);
} catch (std::exception& ex) {
return false;
}
}
Status
@ -273,28 +276,8 @@ ClientProxy::Insert(const std::string& collection_name, const std::string& parti
}
Status
ClientProxy::GetEntityByID(const std::string& collection_name, int64_t entity_id, Entity& entity_data) {
std::vector<int64_t> id_array = {entity_id};
std::vector<Entity> entities_data;
auto status = GetEntitiesByID(collection_name, id_array, entities_data);
if (!status.ok()) {
return status;
}
if (entities_data.empty()) {
return Status(StatusCode::ServerFailed, "Failed to get entity by id");
}
entity_data.binary_data.swap(entities_data[0].binary_data);
entity_data.float_data.swap(entities_data[0].float_data);
return status;
}
Status
ClientProxy::GetEntitiesByID(const std::string& collection_name, const std::vector<int64_t>& id_array,
std::vector<Entity>& entities_data) {
ClientProxy::GetEntityByID(const std::string& collection_name, const std::vector<int64_t>& id_array,
std::vector<Entity>& entities_data) {
try {
entities_data.clear();
@ -305,7 +288,7 @@ ClientProxy::GetEntitiesByID(const std::string& collection_name, const std::vect
}
::milvus::grpc::VectorsData grpc_data;
Status status = client_ptr_->GetVectorsByID(vectors_identity, grpc_data);
Status status = client_ptr_->GetEntityByID(vectors_identity, grpc_data);
if (!status.ok()) {
return status;
}
@ -336,7 +319,7 @@ ClientProxy::GetEntitiesByID(const std::string& collection_name, const std::vect
}
Status
ClientProxy::GetIDsInSegment(const std::string& collection_name, const std::string& segment_name,
ClientProxy::ListIDInSegment(const std::string& collection_name, const std::string& segment_name,
std::vector<int64_t>& id_array) {
try {
::milvus::grpc::GetVectorIDsParam param;
@ -344,7 +327,7 @@ ClientProxy::GetIDsInSegment(const std::string& collection_name, const std::stri
param.set_segment_name(segment_name);
::milvus::grpc::VectorIds vector_ids;
Status status = client_ptr_->GetIDsInSegment(param, vector_ids);
Status status = client_ptr_->ListIDInSegment(param, vector_ids);
if (!status.ok()) {
return status;
}
@ -417,11 +400,11 @@ ClientProxy::SearchByID(const std::string& collection_name, const PartitionTagLi
}
Status
ClientProxy::DescribeCollection(const std::string& collection_name, CollectionParam& collection_param) {
ClientProxy::GetCollectionInfo(const std::string& collection_name, CollectionParam& collection_param) {
try {
::milvus::grpc::CollectionSchema grpc_schema;
Status status = client_ptr_->DescribeCollection(collection_name, grpc_schema);
Status status = client_ptr_->GetCollectionInfo(collection_name, grpc_schema);
collection_param.collection_name = grpc_schema.collection_name();
collection_param.dimension = grpc_schema.dimension();
@ -435,12 +418,12 @@ ClientProxy::DescribeCollection(const std::string& collection_name, CollectionPa
}
Status
ClientProxy::CountCollection(const std::string& collection_name, int64_t& row_count) {
ClientProxy::CountEntities(const std::string& collection_name, int64_t& row_count) {
try {
Status status;
::milvus::grpc::CollectionName grpc_collection_name;
grpc_collection_name.set_collection_name(collection_name);
row_count = client_ptr_->CountCollection(grpc_collection_name, status);
row_count = client_ptr_->CountEntities(grpc_collection_name, status);
return status;
} catch (std::exception& ex) {
return Status(StatusCode::UnknownError, "Failed to count collection: " + std::string(ex.what()));
@ -448,11 +431,11 @@ ClientProxy::CountCollection(const std::string& collection_name, int64_t& row_co
}
Status
ClientProxy::ShowCollections(std::vector<std::string>& collection_array) {
ClientProxy::ListCollections(std::vector<std::string>& collection_array) {
try {
Status status;
milvus::grpc::CollectionNameList collection_name_list;
status = client_ptr_->ShowCollections(collection_name_list);
status = client_ptr_->ListCollections(collection_name_list);
collection_array.resize(collection_name_list.collection_names_size());
for (uint64_t i = 0; i < collection_name_list.collection_names_size(); ++i) {
@ -465,15 +448,15 @@ ClientProxy::ShowCollections(std::vector<std::string>& collection_array) {
}
Status
ClientProxy::ShowCollectionInfo(const std::string& collection_name, std::string& collection_info) {
ClientProxy::GetCollectionStats(const std::string& collection_name, std::string& collection_stats) {
try {
Status status;
::milvus::grpc::CollectionName grpc_collection_name;
grpc_collection_name.set_collection_name(collection_name);
milvus::grpc::CollectionInfo grpc_collection_info;
status = client_ptr_->ShowCollectionInfo(grpc_collection_name, grpc_collection_info);
milvus::grpc::CollectionInfo grpc_collection_stats;
status = client_ptr_->GetCollectionStats(grpc_collection_name, grpc_collection_stats);
collection_info = grpc_collection_info.json_info();
collection_stats = grpc_collection_stats.json_info();
return status;
} catch (std::exception& ex) {
@ -482,7 +465,7 @@ ClientProxy::ShowCollectionInfo(const std::string& collection_name, std::string&
}
Status
ClientProxy::DeleteByID(const std::string& collection_name, const std::vector<int64_t>& id_array) {
ClientProxy::DeleteEntityByID(const std::string& collection_name, const std::vector<int64_t>& id_array) {
try {
::milvus::grpc::DeleteByIDParam delete_by_id_param;
delete_by_id_param.set_collection_name(collection_name);
@ -490,18 +473,18 @@ ClientProxy::DeleteByID(const std::string& collection_name, const std::vector<in
delete_by_id_param.add_id_array(id);
}
return client_ptr_->DeleteByID(delete_by_id_param);
return client_ptr_->DeleteEntityByID(delete_by_id_param);
} catch (std::exception& ex) {
return Status(StatusCode::UnknownError, "Failed to delete entity id: " + std::string(ex.what()));
}
}
Status
ClientProxy::PreloadCollection(const std::string& collection_name) const {
ClientProxy::LoadCollection(const std::string& collection_name) const {
try {
::milvus::grpc::CollectionName grpc_collection_name;
grpc_collection_name.set_collection_name(collection_name);
Status status = client_ptr_->PreloadCollection(grpc_collection_name);
Status status = client_ptr_->LoadCollection(grpc_collection_name);
return status;
} catch (std::exception& ex) {
return Status(StatusCode::UnknownError, "Failed to preload collection: " + std::string(ex.what()));
@ -509,13 +492,13 @@ ClientProxy::PreloadCollection(const std::string& collection_name) const {
}
Status
ClientProxy::DescribeIndex(const std::string& collection_name, IndexParam& index_param) const {
ClientProxy::GetIndexInfo(const std::string& collection_name, IndexParam& index_param) const {
try {
::milvus::grpc::CollectionName grpc_collection_name;
grpc_collection_name.set_collection_name(collection_name);
::milvus::grpc::IndexParam grpc_index_param;
Status status = client_ptr_->DescribeIndex(grpc_collection_name, grpc_index_param);
Status status = client_ptr_->GetIndexInfo(grpc_collection_name, grpc_index_param);
index_param.index_type = static_cast<IndexType>(grpc_index_param.index_type());
for (int i = 0; i < grpc_index_param.extra_params_size(); i++) {
@ -556,13 +539,26 @@ ClientProxy::CreatePartition(const PartitionParam& partition_param) {
}
}
bool
ClientProxy::HasPartition(const std::string& collection_name, const std::string& partition_tag) const {
try {
Status status = Status::OK();
::milvus::grpc::PartitionParam grpc_partition_param;
grpc_partition_param.set_collection_name(collection_name);
grpc_partition_param.set_tag(partition_tag);
return client_ptr_->HasPartition(grpc_partition_param, status);
} catch (std::exception& ex) {
return false;
}
}
Status
ClientProxy::ShowPartitions(const std::string& collection_name, PartitionTagList& partition_tag_array) const {
ClientProxy::ListPartitions(const std::string& collection_name, PartitionTagList& partition_tag_array) const {
try {
::milvus::grpc::CollectionName grpc_collection_name;
grpc_collection_name.set_collection_name(collection_name);
::milvus::grpc::PartitionList grpc_partition_list;
Status status = client_ptr_->ShowPartitions(grpc_collection_name, grpc_partition_list);
Status status = client_ptr_->ListPartitions(grpc_collection_name, grpc_partition_list);
partition_tag_array.resize(grpc_partition_list.partition_tag_array_size());
for (uint64_t i = 0; i < grpc_partition_list.partition_tag_array_size(); ++i) {
partition_tag_array[i] = grpc_partition_list.partition_tag_array(i);
@ -587,27 +583,23 @@ ClientProxy::DropPartition(const PartitionParam& partition_param) {
}
Status
ClientProxy::FlushCollection(const std::string& collection_name) {
ClientProxy::Flush(const std::vector<std::string>& collection_name_array) {
try {
std::string dummy;
return client_ptr_->Flush(collection_name);
if (collection_name_array.empty()) {
return client_ptr_->Flush("");
} else {
for (auto& collection_name : collection_name_array) {
client_ptr_->Flush(collection_name);
}
}
return Status::OK();
} catch (std::exception& ex) {
return Status(StatusCode::UnknownError, "Failed to flush collection");
}
}
Status
ClientProxy::Flush() {
try {
std::string dummy;
return client_ptr_->Flush("");
} catch (std::exception& ex) {
return Status(StatusCode::UnknownError, "Failed to flush collections");
}
}
Status
ClientProxy::CompactCollection(const std::string& collection_name) {
ClientProxy::Compact(const std::string& collection_name) {
try {
::milvus::grpc::CollectionName grpc_collection_name;
grpc_collection_name.set_collection_name(collection_name);

37
sdk/grpc/ClientProxy.h
View File

@ -69,15 +69,12 @@ class ClientProxy : public Connection {
std::vector<int64_t>& id_array) override;
Status
GetEntityByID(const std::string& collection_name, int64_t entity_id, Entity& entity_data) override;
GetEntityByID(const std::string& collection_name,
const std::vector<int64_t>& id_array,
std::vector<Entity>& entities_data) override;
Status
GetEntitiesByID(const std::string& collection_name,
const std::vector<int64_t>& id_array,
std::vector<Entity>& entities_data) override;
Status
GetIDsInSegment(const std::string& collection_name, const std::string& segment_name,
ListIDInSegment(const std::string& collection_name, const std::string& segment_name,
std::vector<int64_t>& id_array) override;
Status
@ -91,25 +88,25 @@ class ClientProxy : public Connection {
const std::string& extra_params, TopKQueryResult& topk_query_result) override;
Status
DescribeCollection(const std::string& collection_name, CollectionParam& collection_schema) override;
GetCollectionInfo(const std::string& collection_name, CollectionParam& collection_param) override;
Status
CountCollection(const std::string& collection_name, int64_t& entity_count) override;
CountEntities(const std::string& collection_name, int64_t& entity_count) override;
Status
ShowCollections(std::vector<std::string>& collection_array) override;
ListCollections(std::vector<std::string>& collection_array) override;
Status
ShowCollectionInfo(const std::string& collection_name, std::string& collection_info) override;
GetCollectionStats(const std::string& collection_name, std::string& collection_stats) override;
Status
DeleteByID(const std::string& collection_name, const std::vector<int64_t>& id_array) override;
DeleteEntityByID(const std::string& collection_name, const std::vector<int64_t>& id_array) override;
Status
PreloadCollection(const std::string& collection_name) const override;
LoadCollection(const std::string& collection_name) const override;
Status
DescribeIndex(const std::string& collection_name, IndexParam& index_param) const override;
GetIndexInfo(const std::string& collection_name, IndexParam& index_param) const override;
Status
DropIndex(const std::string& collection_name) const override;
@ -117,20 +114,20 @@ class ClientProxy : public Connection {
Status
CreatePartition(const PartitionParam& partition_param) override;
bool
HasPartition(const std::string& collection_name, const std::string& partition_tag) const override;
Status
ShowPartitions(const std::string& collection_name, PartitionTagList& partition_tag_array) const override;
ListPartitions(const std::string& collection_name, PartitionTagList& partition_tag_array) const override;
Status
DropPartition(const PartitionParam& partition_param) override;
Status
FlushCollection(const std::string& collection_name) override;
Flush(const std::vector<std::string>& collection_name_array) override;
Status
Flush() override;
Status
CompactCollection(const std::string& collection_name) override;
Compact(const std::string& collection_name) override;
/*******************************New Interface**********************************/

46
sdk/grpc/GrpcClient.cpp
View File

@ -125,7 +125,7 @@ GrpcClient::Insert(const ::milvus::grpc::InsertParam& insert_param, ::milvus::gr
}
Status
GrpcClient::GetVectorsByID(const grpc::VectorsIdentity& vectors_identity, ::milvus::grpc::VectorsData& vectors_data) {
GrpcClient::GetEntityByID(const grpc::VectorsIdentity& vectors_identity, ::milvus::grpc::VectorsData& vectors_data) {
ClientContext context;
::grpc::Status grpc_status = stub_->GetVectorsByID(&context, vectors_identity, &vectors_data);
@ -142,7 +142,7 @@ GrpcClient::GetVectorsByID(const grpc::VectorsIdentity& vectors_identity, ::milv
}
Status
GrpcClient::GetIDsInSegment(const grpc::GetVectorIDsParam& param, grpc::VectorIds& vector_ids) {
GrpcClient::ListIDInSegment(const grpc::GetVectorIDsParam& param, grpc::VectorIds& vector_ids) {
ClientContext context;
::grpc::Status grpc_status = stub_->GetVectorIDs(&context, param, &vector_ids);
@ -198,7 +198,7 @@ GrpcClient::SearchByID(const grpc::SearchByIDParam& search_param, ::milvus::grpc
}
Status
GrpcClient::DescribeCollection(const std::string& collection_name, ::milvus::grpc::CollectionSchema& grpc_schema) {
GrpcClient::GetCollectionInfo(const std::string& collection_name, ::milvus::grpc::CollectionSchema& grpc_schema) {
ClientContext context;
::milvus::grpc::CollectionName grpc_collectionname;
grpc_collectionname.set_collection_name(collection_name);
@ -219,7 +219,7 @@ GrpcClient::DescribeCollection(const std::string& collection_name, ::milvus::grp
}
int64_t
GrpcClient::CountCollection(grpc::CollectionName& collection_name, Status& status) {
GrpcClient::CountEntities(grpc::CollectionName& collection_name, Status& status) {
ClientContext context;
::milvus::grpc::CollectionRowCount response;
::grpc::Status grpc_status = stub_->CountCollection(&context, collection_name, &response);
@ -241,7 +241,7 @@ GrpcClient::CountCollection(grpc::CollectionName& collection_name, Status& statu
}
Status
GrpcClient::ShowCollections(milvus::grpc::CollectionNameList& collection_name_list) {
GrpcClient::ListCollections(milvus::grpc::CollectionNameList& collection_name_list) {
ClientContext context;
::milvus::grpc::Command command;
::grpc::Status grpc_status = stub_->ShowCollections(&context, command, &collection_name_list);
@ -261,10 +261,10 @@ GrpcClient::ShowCollections(milvus::grpc::CollectionNameList& collection_name_li
}
Status
GrpcClient::ShowCollectionInfo(grpc::CollectionName& collection_name, grpc::CollectionInfo& collection_info) {
GrpcClient::GetCollectionStats(grpc::CollectionName& collection_name, grpc::CollectionInfo& collection_stats) {
ClientContext context;
::milvus::grpc::Command command;
::grpc::Status grpc_status = stub_->ShowCollectionInfo(&context, collection_name, &collection_info);
::grpc::Status grpc_status = stub_->ShowCollectionInfo(&context, collection_name, &collection_stats);
if (!grpc_status.ok()) {
std::cerr << "ShowCollectionInfo gRPC failed!" << std::endl;
@ -272,9 +272,9 @@ GrpcClient::ShowCollectionInfo(grpc::CollectionName& collection_name, grpc::Coll
return Status(StatusCode::RPCFailed, grpc_status.error_message());
}
if (collection_info.status().error_code() != grpc::SUCCESS) {
std::cerr << collection_info.status().reason() << std::endl;
return Status(StatusCode::ServerFailed, collection_info.status().reason());
if (collection_stats.status().error_code() != grpc::SUCCESS) {
std::cerr << collection_stats.status().reason() << std::endl;
return Status(StatusCode::ServerFailed, collection_stats.status().reason());
}
return Status::OK();
@ -303,7 +303,7 @@ GrpcClient::Cmd(const std::string& cmd, std::string& result) {
}
Status
GrpcClient::PreloadCollection(milvus::grpc::CollectionName& collection_name) {
GrpcClient::LoadCollection(milvus::grpc::CollectionName& collection_name) {
ClientContext context;
::milvus::grpc::Status response;
::grpc::Status grpc_status = stub_->PreloadCollection(&context, collection_name, &response);
@ -321,7 +321,7 @@ GrpcClient::PreloadCollection(milvus::grpc::CollectionName& collection_name) {
}
Status
GrpcClient::DeleteByID(grpc::DeleteByIDParam& delete_by_id_param) {
GrpcClient::DeleteEntityByID(grpc::DeleteByIDParam& delete_by_id_param) {
ClientContext context;
::milvus::grpc::Status response;
::grpc::Status grpc_status = stub_->DeleteByID(&context, delete_by_id_param, &response);
@ -339,7 +339,7 @@ GrpcClient::DeleteByID(grpc::DeleteByIDParam& delete_by_id_param) {
}
Status
GrpcClient::DescribeIndex(grpc::CollectionName& collection_name, grpc::IndexParam& index_param) {
GrpcClient::GetIndexInfo(grpc::CollectionName& collection_name, grpc::IndexParam& index_param) {
ClientContext context;
::grpc::Status grpc_status = stub_->DescribeIndex(&context, collection_name, &index_param);
@ -391,8 +391,26 @@ GrpcClient::CreatePartition(const grpc::PartitionParam& partition_param) {
return Status::OK();
}
bool
GrpcClient::HasPartition(const grpc::PartitionParam& partition_param, Status& status) const {
ClientContext context;
::milvus::grpc::BoolReply response;
::grpc::Status grpc_status = stub_->HasPartition(&context, partition_param, &response);
if (!grpc_status.ok()) {
std::cerr << "HasPartition gRPC failed!" << std::endl;
status = Status(StatusCode::RPCFailed, grpc_status.error_message());
}
if (response.status().error_code() != grpc::SUCCESS) {
std::cerr << response.status().reason() << std::endl;
status = Status(StatusCode::ServerFailed, response.status().reason());
}
status = Status::OK();
return response.bool_reply();
}
Status
GrpcClient::ShowPartitions(const grpc::CollectionName& collection_name, grpc::PartitionList& partition_array) const {
GrpcClient::ListPartitions(const grpc::CollectionName& collection_name, grpc::PartitionList& partition_array) const {
ClientContext context;
::grpc::Status grpc_status = stub_->ShowPartitions(&context, collection_name, &partition_array);

23
sdk/grpc/GrpcClient.h
View File

@ -51,10 +51,10 @@ class GrpcClient {
Insert(const grpc::InsertParam& insert_param, grpc::VectorIds& vector_ids);
Status
GetVectorsByID(const grpc::VectorsIdentity& vectors_identity, ::milvus::grpc::VectorsData& vectors_data);
GetEntityByID(const grpc::VectorsIdentity& vectors_identity, ::milvus::grpc::VectorsData& vectors_data);
Status
GetIDsInSegment(const grpc::GetVectorIDsParam& param, grpc::VectorIds& vector_ids);
ListIDInSegment(const grpc::GetVectorIDsParam& param, grpc::VectorIds& vector_ids);
Status
Search(const grpc::SearchParam& search_param, ::milvus::grpc::TopKQueryResult& topk_query_result);
@ -63,28 +63,28 @@ class GrpcClient {
SearchByID(const grpc::SearchByIDParam& search_param, ::milvus::grpc::TopKQueryResult& topk_query_result);
Status
DescribeCollection(const std::string& collection_name, grpc::CollectionSchema& grpc_schema);
GetCollectionInfo(const std::string& collection_name, grpc::CollectionSchema& grpc_schema);
int64_t
CountCollection(grpc::CollectionName& collection_name, Status& status);
CountEntities(grpc::CollectionName& collection_name, Status& status);
Status
ShowCollections(milvus::grpc::CollectionNameList& collection_name_list);
ListCollections(milvus::grpc::CollectionNameList& collection_name_list);
Status
ShowCollectionInfo(grpc::CollectionName& collection_name, grpc::CollectionInfo& collection_info);
GetCollectionStats(grpc::CollectionName& collection_name, grpc::CollectionInfo& collection_stats);
Status
Cmd(const std::string& cmd, std::string& result);
Status
DeleteByID(grpc::DeleteByIDParam& delete_by_id_param);
DeleteEntityByID(grpc::DeleteByIDParam& delete_by_id_param);
Status
PreloadCollection(grpc::CollectionName& collection_name);
LoadCollection(grpc::CollectionName& collection_name);
Status
DescribeIndex(grpc::CollectionName& collection_name, grpc::IndexParam& index_param);
GetIndexInfo(grpc::CollectionName& collection_name, grpc::IndexParam& index_param);
Status
DropIndex(grpc::CollectionName& collection_name);
@ -92,8 +92,11 @@ class GrpcClient {
Status
CreatePartition(const grpc::PartitionParam& partition_param);
bool
HasPartition(const grpc::PartitionParam& partition_param, Status& status) const;
Status
ShowPartitions(const grpc::CollectionName& collection_name, grpc::PartitionList& partition_array) const;
ListPartitions(const grpc::CollectionName& collection_name, grpc::PartitionList& partition_array) const;
Status
DropPartition(const ::milvus::grpc::PartitionParam& partition_param);

98
sdk/include/MilvusApi.h
View File

@ -326,21 +326,6 @@ class Connection {
const std::vector<Entity>& entity_array,
std::vector<int64_t>& id_array) = 0;
/**
* @brief Get entity data by id
*
* This method is used to get entity data by id from a collection.
* Return the first found entity if there are entities with duplicated id
*
* @param collection_name, target collection's name.
* @param entity_id, target entity id.
* @param entity_data, returned entity data.
*
* @return Indicate if the operation is succeed.
*/
virtual Status
GetEntityByID(const std::string& collection_name, int64_t entity_id, Entity& entity_data) = 0;
/**
* @brief Get entity data by id
*
@ -354,12 +339,12 @@ class Connection {
* @return Indicate if the operation is succeed.
*/
virtual Status
GetEntitiesByID(const std::string& collection_name,
const std::vector<int64_t>& id_array,
std::vector<Entity>& entities_data) = 0;
GetEntityByID(const std::string& collection_name,
const std::vector<int64_t>& id_array,
std::vector<Entity>& entities_data) = 0;
/**
* @brief Get entity ids from a segment
* @brief List entity ids from a segment
*
* This method is used to get entity ids from a segment
* Return all entity(not deleted) ids
@ -371,7 +356,7 @@ class Connection {
* @return Indicate if the operation is succeed.
*/
virtual Status
GetIDsInSegment(const std::string& collection_name,
ListIDInSegment(const std::string& collection_name,
const std::string& segment_name,
std::vector<int64_t>& id_array) = 0;
@ -430,9 +415,9 @@ class Connection {
const std::string& extra_params, TopKQueryResult& topk_query_result) = 0;
/**
* @brief Show collection description
* @brief Get collection information
*
* This method is used to show collection information.
* This method is used to get collection information.
*
* @param collection_name, target collection's name.
* @param collection_param, collection_param is given when operation is successful.
@ -440,7 +425,7 @@ class Connection {
* @return Indicate if this operation is successful.
*/
virtual Status
DescribeCollection(const std::string& collection_name, CollectionParam& collection_param) = 0;
GetCollectionInfo(const std::string& collection_name, CollectionParam& collection_param) = 0;
/**
* @brief Get collection entity count
@ -453,10 +438,10 @@ class Connection {
* @return Indicate if this operation is successful.
*/
virtual Status
CountCollection(const std::string& collection_name, int64_t& entity_count) = 0;
CountEntities(const std::string& collection_name, int64_t& entity_count) = 0;
/**
* @brief Show all collections in database
* @brief List all collections in database
*
* This method is used to list all collections.
*
@ -465,20 +450,20 @@ class Connection {
* @return Indicate if this operation is successful.
*/
virtual Status
ShowCollections(std::vector<std::string>& collection_array) = 0;
ListCollections(std::vector<std::string>& collection_array) = 0;
/**
* @brief Show collection information
* @brief Get collection statistics
*
* This method is used to get detail information of a collection.
* This method is used to get statistics of a collection.
*
* @param collection_name, target collection's name.
* @param collection_info, target collection's information in json format
* @param collection_stats, target collection's statistics in json format
*
* @return Indicate if this operation is successful.
*/
virtual Status
ShowCollectionInfo(const std::string& collection_name, std::string& collection_info) = 0;
GetCollectionStats(const std::string& collection_name, std::string& collection_stats) = 0;
/**
* @brief Delete entity by id
@ -491,24 +476,24 @@ class Connection {
* @return Indicate if this operation is successful.
*/
virtual Status
DeleteByID(const std::string& collection_name, const std::vector<int64_t>& id_array) = 0;
DeleteEntityByID(const std::string& collection_name, const std::vector<int64_t>& id_array) = 0;
/**
* @brief Preload collection
* @brief Load collection from disk to memory
*
* This method is used to preload collection data into memory
* This method is used to load collection data into memory
*
* @param collection_name, target collection's name.
*
* @return Indicate if this operation is successful.
*/
virtual Status
PreloadCollection(const std::string& collection_name) const = 0;
LoadCollection(const std::string& collection_name) const = 0;
/**
* @brief Describe index
* @brief Get index information
*
* This method is used to describe index
* This method is used to get index information
*
* @param collection_name, target collection's name.
* @param index_param, returned index information.
@ -516,7 +501,7 @@ class Connection {
* @return Indicate if this operation is successful.
*/
virtual Status
DescribeIndex(const std::string& collection_name, IndexParam& index_param) const = 0;
GetIndexInfo(const std::string& collection_name, IndexParam& index_param) const = 0;
/**
* @brief Drop index
@ -543,9 +528,22 @@ class Connection {
CreatePartition(const PartitionParam& partition_param) = 0;
/**
* @brief Show all partitions method
* @brief Has partition method
*
* This method is used to show all partitions(return their tags)
* This method is used to test existence of collection's partition
*
* @param collection_name, target collection's name.
* @param partition_tag, target partition's tag.
*
* @return Indicate if partition is created successfully
*/
virtual bool
HasPartition(const std::string& collection_name, const std::string& partition_tag) const = 0;
/**
* @brief List all partitions method
*
* This method is used to list all partitions(return their tags)
*
* @param collection_name, target collection's name.
* @param partition_tag_array, partition tag array of the collection.
@ -553,7 +551,7 @@ class Connection {
* @return Indicate if this operation is successful
*/
virtual Status
ShowPartitions(const std::string& collection_name, PartitionTagList& partition_tag_array) const = 0;
ListPartitions(const std::string& collection_name, PartitionTagList& partition_tag_array) const = 0;
/**
* @brief Delete partition method
@ -568,26 +566,16 @@ class Connection {
DropPartition(const PartitionParam& partition_param) = 0;
/**
* @brief Flush collection buffer into storage
* @brief Flush collections insert buffer into storage
*
* This method is used to flush collection buffer into storage
* This method is used to flush collection insert buffer into storage
*
* @param collection_name, target collection's name.
* @param collection_name_array, target collections name array.
*
* @return Indicate if this operation is successful.
*/
virtual Status
FlushCollection(const std::string& collection_name) = 0;
/**
* @brief Flush all buffer into storage
*
* This method is used to all collection buffer into storage
*
* @return Indicate if this operation is successful.
*/
virtual Status
Flush() = 0;
Flush(const std::vector<std::string>& collection_name_array) = 0;
/**
* @brief Compact collection, permanently remove deleted vectors
@ -599,7 +587,7 @@ class Connection {
* @return Indicate if this operation is successful.
*/
virtual Status
CompactCollection(const std::string& collection_name) = 0;
Compact(const std::string& collection_name) = 0;
/*******************************New Interface**********************************/

67
sdk/interface/ConnectionImpl.cpp
View File

@ -103,21 +103,16 @@ ConnectionImpl::Insert(const std::string& collection_name, const std::string& pa
}
Status
ConnectionImpl::GetEntityByID(const std::string& collection_name, int64_t entity_id, Entity& entity_data) {
return client_proxy_->GetEntityByID(collection_name, entity_id, entity_data);
ConnectionImpl::GetEntityByID(const std::string& collection_name,
const std::vector<int64_t>& id_array,
std::vector<Entity>& entities_data) {
return client_proxy_->GetEntityByID(collection_name, id_array, entities_data);
}
Status
ConnectionImpl::GetEntitiesByID(const std::string& collection_name,
const std::vector<int64_t>& id_array,
std::vector<Entity>& entities_data) {
return client_proxy_->GetEntitiesByID(collection_name, id_array, entities_data);
}
Status
ConnectionImpl::GetIDsInSegment(const std::string& collection_name, const std::string& segment_name,
ConnectionImpl::ListIDInSegment(const std::string& collection_name, const std::string& segment_name,
std::vector<int64_t>& id_array) {
return client_proxy_->GetIDsInSegment(collection_name, segment_name, id_array);
return client_proxy_->ListIDInSegment(collection_name, segment_name, id_array);
}
Status
@ -145,38 +140,38 @@ ConnectionImpl::SearchByID(const std::string& collection_name, const PartitionTa
}
Status
ConnectionImpl::DescribeCollection(const std::string& collection_name, CollectionParam& collection_schema) {
return client_proxy_->DescribeCollection(collection_name, collection_schema);
ConnectionImpl::GetCollectionInfo(const std::string& collection_name, CollectionParam& collection_schema) {
return client_proxy_->GetCollectionInfo(collection_name, collection_schema);
}
Status
ConnectionImpl::CountCollection(const std::string& collection_name, int64_t& row_count) {
return client_proxy_->CountCollection(collection_name, row_count);
ConnectionImpl::CountEntities(const std::string& collection_name, int64_t& row_count) {
return client_proxy_->CountEntities(collection_name, row_count);
}
Status
ConnectionImpl::ShowCollections(std::vector<std::string>& collection_array) {
return client_proxy_->ShowCollections(collection_array);
ConnectionImpl::ListCollections(std::vector<std::string>& collection_array) {
return client_proxy_->ListCollections(collection_array);
}
Status
ConnectionImpl::ShowCollectionInfo(const std::string& collection_name, std::string& collection_info) {
return client_proxy_->ShowCollectionInfo(collection_name, collection_info);
ConnectionImpl::GetCollectionStats(const std::string& collection_name, std::string& collection_stats) {
return client_proxy_->GetCollectionStats(collection_name, collection_stats);
}
Status
ConnectionImpl::DeleteByID(const std::string& collection_name, const std::vector<int64_t>& id_array) {
return client_proxy_->DeleteByID(collection_name, id_array);
ConnectionImpl::DeleteEntityByID(const std::string& collection_name, const std::vector<int64_t>& id_array) {
return client_proxy_->DeleteEntityByID(collection_name, id_array);
}
Status
ConnectionImpl::PreloadCollection(const std::string& collection_name) const {
return client_proxy_->PreloadCollection(collection_name);
ConnectionImpl::LoadCollection(const std::string& collection_name) const {
return client_proxy_->LoadCollection(collection_name);
}
Status
ConnectionImpl::DescribeIndex(const std::string& collection_name, IndexParam& index_param) const {
return client_proxy_->DescribeIndex(collection_name, index_param);
ConnectionImpl::GetIndexInfo(const std::string& collection_name, IndexParam& index_param) const {
return client_proxy_->GetIndexInfo(collection_name, index_param);
}
Status
@ -189,9 +184,14 @@ ConnectionImpl::CreatePartition(const PartitionParam& partition_param) {
return client_proxy_->CreatePartition(partition_param);
}
bool
ConnectionImpl::HasPartition(const std::string& collection_name, const std::string& partition_tag) const {
return client_proxy_->HasPartition(collection_name, partition_tag);
}
Status
ConnectionImpl::ShowPartitions(const std::string& collection_name, PartitionTagList& partition_array) const {
return client_proxy_->ShowPartitions(collection_name, partition_array);
ConnectionImpl::ListPartitions(const std::string& collection_name, PartitionTagList& partition_array) const {
return client_proxy_->ListPartitions(collection_name, partition_array);
}
Status
@ -200,18 +200,13 @@ ConnectionImpl::DropPartition(const PartitionParam& partition_param) {
}
Status
ConnectionImpl::FlushCollection(const std::string& Status) {
return client_proxy_->FlushCollection(Status);
ConnectionImpl::Flush(const std::vector<std::string>& collection_name_array) {
return client_proxy_->Flush(collection_name_array);
}
Status
ConnectionImpl::Flush() {
return client_proxy_->Flush();
}
Status
ConnectionImpl::CompactCollection(const std::string& collection_name) {
return client_proxy_->CompactCollection(collection_name);
ConnectionImpl::Compact(const std::string& collection_name) {
return client_proxy_->Compact(collection_name);
}
/*******************************New Interface**********************************/

37
sdk/interface/ConnectionImpl.h
View File

@ -71,15 +71,12 @@ class ConnectionImpl : public Connection {
std::vector<int64_t>& id_array) override;
Status
GetEntityByID(const std::string& collection_name, int64_t entity_id, Entity& entity_data) override;
GetEntityByID(const std::string& collection_name,
const std::vector<int64_t>& id_array,
std::vector<Entity>& entities_data) override;
Status
GetEntitiesByID(const std::string& collection_name,
const std::vector<int64_t>& id_array,
std::vector<Entity>& entities_data) override;
Status
GetIDsInSegment(const std::string& collection_name, const std::string& segment_name,
ListIDInSegment(const std::string& collection_name, const std::string& segment_name,
std::vector<int64_t>& id_array) override;
Status
@ -93,25 +90,25 @@ class ConnectionImpl : public Connection {
const std::string& extra_params, TopKQueryResult& topk_query_result) override;
Status
DescribeCollection(const std::string& collection_name, CollectionParam& collection_schema) override;
GetCollectionInfo(const std::string& collection_name, CollectionParam& collection_param) override;
Status
CountCollection(const std::string& collection_name, int64_t& entity_count) override;
CountEntities(const std::string& collection_name, int64_t& entity_count) override;
Status
ShowCollections(std::vector<std::string>& collection_array) override;
ListCollections(std::vector<std::string>& collection_array) override;
Status
ShowCollectionInfo(const std::string& collection_name, std::string& collection_info) override;
GetCollectionStats(const std::string& collection_name, std::string& collection_stats) override;
Status
DeleteByID(const std::string& collection_name, const std::vector<int64_t>& id_array) override;
DeleteEntityByID(const std::string& collection_name, const std::vector<int64_t>& id_array) override;
Status
PreloadCollection(const std::string& collection_name) const override;
LoadCollection(const std::string& collection_name) const override;
Status
DescribeIndex(const std::string& collection_name, IndexParam& index_param) const override;
GetIndexInfo(const std::string& collection_name, IndexParam& index_param) const override;
Status
DropIndex(const std::string& collection_name) const override;
@ -119,20 +116,20 @@ class ConnectionImpl : public Connection {
Status
CreatePartition(const PartitionParam& partition_param) override;
bool
HasPartition(const std::string& collection_name, const std::string& partition_tag) const override;
Status
ShowPartitions(const std::string& collection_name, PartitionTagList& partition_tag_array) const override;
ListPartitions(const std::string& collection_name, PartitionTagList& partition_tag_array) const override;
Status
DropPartition(const PartitionParam& partition_param) override;
Status
FlushCollection(const std::string& collection_name) override;
Flush(const std::vector<std::string>& collection_name_array) override;
Status
Flush() override;
Status
CompactCollection(const std::string& collection_name) override;
Compact(const std::string& collection_name) override;
/*******************************New Interface**********************************/