milvus-io
/
milvus
mirror of https://github.com/milvus-io/milvus.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

#4082 Expose MAX_PARTITION_LIMIT to config (#4138) 

Signed-off-by: yhmo <yihua.mo@zilliz.com>
pull/4175/head
groot 2020-10-31 16:39:26 +08:00 committed by GitHub
parent 3ab73968c1
commit 001c8513fe
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 84 additions and 39 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

41
core/src/config/Config.cpp
View File

@ -117,6 +117,8 @@ const char* CONFIG_ENGINE_SIMD_TYPE = "simd_type";
const char* CONFIG_ENGINE_SIMD_TYPE_DEFAULT = "auto";
const char* CONFIG_ENGINE_SEARCH_COMBINE_MAX_NQ = "search_combine_nq";
const char* CONFIG_ENGINE_SEARCH_COMBINE_MAX_NQ_DEFAULT = "64";
const char* CONFIG_ENGINE_MAX_PARTITION_NUM = "max_partition_num";
const char* CONFIG_ENGINE_MAX_PARTITION_NUM_DEFAULT = "4096";
/* fpga resource config */
const char* CONFIG_FPGA_RESOURCE = "fpga";
const char* CONFIG_FPGA_RESOURCE_ENABLE = "enable";
@ -225,6 +227,9 @@ Config::Config() {
std::string node_search_combine = std::string(CONFIG_ENGINE) + "." + CONFIG_ENGINE_SEARCH_COMBINE_MAX_NQ;
config_callback_[node_search_combine] = empty_map;
std::string node_max_partition = std::string(CONFIG_ENGINE) + "." + CONFIG_ENGINE_MAX_PARTITION_NUM;
config_callback_[node_max_partition] = empty_map;
// gpu resources config
std::string node_gpu_enable = std::string(CONFIG_GPU_RESOURCE) + "." + CONFIG_GPU_RESOURCE_ENABLE;
config_callback_[node_gpu_enable] = empty_map;
@ -386,6 +391,9 @@ Config::ValidateConfig() {
std::string engine_simd_type;
STATUS_CHECK(GetEngineConfigSimdType(engine_simd_type));
int64_t max_partition_num;
STATUS_CHECK(GetEngineConfigMaxPartitionNum(max_partition_num));
/* gpu resource config */
#ifdef MILVUS_GPU_VERSION
bool gpu_resource_enable;
@ -506,6 +514,7 @@ Config::ResetDefaultConfig() {
STATUS_CHECK(SetEngineConfigOmpThreadNum(CONFIG_ENGINE_OMP_THREAD_NUM_DEFAULT));
STATUS_CHECK(SetEngineConfigSimdType(CONFIG_ENGINE_SIMD_TYPE_DEFAULT));
STATUS_CHECK(SetEngineSearchCombineMaxNq(CONFIG_ENGINE_SEARCH_COMBINE_MAX_NQ_DEFAULT));
STATUS_CHECK(SetEngineConfigMaxPartitionNum(CONFIG_ENGINE_MAX_PARTITION_NUM_DEFAULT));
/* gpu resource config */
#ifdef MILVUS_GPU_VERSION
@ -645,6 +654,8 @@ Config::SetConfigCli(const std::string& parent_key, const std::string& child_key
status = SetEngineConfigSimdType(value);
} else if (child_key == CONFIG_ENGINE_SEARCH_COMBINE_MAX_NQ) {
status = SetEngineSearchCombineMaxNq(value);
} else if (child_key == CONFIG_ENGINE_MAX_PARTITION_NUM) {
status = SetEngineConfigMaxPartitionNum(value);
} else {
status = Status(SERVER_UNEXPECTED_ERROR, invalid_node_str);
}
@ -1339,6 +1350,7 @@ Config::CheckMetricConfigPort(const std::string& value) {
}
return Status::OK();
}
#ifdef MILVUS_FPGA_VERSION
Status
Config::CheckFpgaResourceConfigEnable(const std::string& value) {
@ -1352,6 +1364,7 @@ Config::CheckFpgaResourceConfigEnable(const std::string& value) {
}
#endif
/* cache config */
Status
Config::CheckCacheConfigCpuCacheCapacity(const std::string& value) {
@ -1619,6 +1632,16 @@ Config::CheckEngineSearchCombineMaxNq(const std::string& value) {
return Status::OK();
}
Status
Config::CheckEngineConfigMaxPartitionNum(const std::string& value) {
if (!ValidationUtil::ValidateStringIsNumber(value).ok()) {
std::string msg = "Invalid max partition number: " + value +
". Possible reason: engine_config.max_partition_num is not a positive integer.";
return Status(SERVER_INVALID_ARGUMENT, msg);
}
return Status::OK();
}
#ifdef MILVUS_GPU_VERSION
/* gpu resource config */
@ -2078,6 +2101,7 @@ Config::GetGeneralConfigMetaURI(std::string& value) {
value = GetConfigStr(CONFIG_GENERAL, CONFIG_GENERAL_METAURI, CONFIG_GENERAL_METAURI_DEFAULT);
return CheckGeneralConfigMetaURI(value);
}
#ifdef MILVUS_FPGA_VERSION
Status
Config::GetFpgaResourceConfigEnable(bool& value) {
@ -2143,6 +2167,7 @@ Config::CheckFpgaResourceConfigCacheThreshold(const std::string& value) {
return Status::OK();
}
#endif
/* network config */
Status
Config::GetNetworkConfigBindAddress(std::string& value) {
@ -2397,6 +2422,15 @@ Config::GetEngineSearchCombineMaxNq(int64_t& value) {
return Status::OK();
}
Status
Config::GetEngineConfigMaxPartitionNum(int64_t& value) {
std::string str =
GetConfigStr(CONFIG_ENGINE, CONFIG_ENGINE_MAX_PARTITION_NUM, CONFIG_ENGINE_MAX_PARTITION_NUM_DEFAULT);
STATUS_CHECK(CheckEngineConfigMaxPartitionNum(str));
value = std::stoll(str);
return Status::OK();
}
/* gpu resource config */
#ifdef MILVUS_GPU_VERSION
@ -2852,8 +2886,15 @@ Config::SetEngineSearchCombineMaxNq(const std::string& value) {
return ExecCallBacks(CONFIG_ENGINE, CONFIG_ENGINE_SEARCH_COMBINE_MAX_NQ, value);
}
Status
Config::SetEngineConfigMaxPartitionNum(const std::string& value) {
STATUS_CHECK(CheckEngineConfigMaxPartitionNum(value));
return SetConfigValueInMem(CONFIG_ENGINE, CONFIG_ENGINE_MAX_PARTITION_NUM, value);
}
/* gpu resource config */
#ifdef MILVUS_GPU_VERSION
Status
Config::SetGpuResourceConfigEnable(const std::string& value) {
STATUS_CHECK(CheckGpuResourceConfigEnable(value));

8
core/src/config/Config.h
View File

@ -104,6 +104,8 @@ extern const char* CONFIG_ENGINE_SIMD_TYPE;
extern const char* CONFIG_ENGINE_SIMD_TYPE_DEFAULT;
extern const char* CONFIG_ENGINE_SEARCH_COMBINE_MAX_NQ;
extern const char* CONFIG_ENGINE_SEARCH_COMBINE_MAX_NQ_DEFAULT;
extern const char* CONFIG_ENGINE_MAX_PARTITION_NUM;
extern const char* CONFIG_ENGINE_MAX_PARTITION_NUM_DEFAULT;
/* fpga resource config*/
extern const char* CONFIG_FPGA_RESOURCE;
extern const char* CONFIG_FPGA_RESOURCE_ENABLE;
@ -286,6 +288,8 @@ class Config {
CheckEngineConfigSimdType(const std::string& value);
Status
CheckEngineSearchCombineMaxNq(const std::string& value);
Status
CheckEngineConfigMaxPartitionNum(const std::string& value);
#ifdef MILVUS_FPGA_VERSION
Status
GetFpgaResourceConfigCacheThreshold(float& value);
@ -419,6 +423,8 @@ class Config {
GetEngineConfigSimdType(std::string& value);
Status
GetEngineSearchCombineMaxNq(int64_t& value);
Status
GetEngineConfigMaxPartitionNum(int64_t& value);
#ifdef MILVUS_FPGA_VERSION
Status
@ -541,6 +547,8 @@ class Config {
SetEngineConfigSimdType(const std::string& value);
Status
SetEngineSearchCombineMaxNq(const std::string& value);
Status
SetEngineConfigMaxPartitionNum(const std::string& value);
#ifdef MILVUS_GPU_VERSION
/* gpu resource config */

2
core/src/db/meta/MySQLMetaImpl.cpp
View File

@ -1394,7 +1394,7 @@ MySQLMetaImpl::CreatePartition(const std::string& collection_id, const std::stri
// not allow create partition under partition
if (!collection_schema.owner_collection_.empty()) {
return Status(DB_ERROR, "Nested partition is not allowed");
return Status(DB_NOT_FOUND, "Nested partition is not allowed");
}
// trim side-blank of tag, only compare valid characters

39
core/src/db/meta/SqliteMetaImpl.cpp
View File

@ -2252,16 +2252,22 @@ SqliteMetaImpl::CleanUpFilesWithTTL(uint64_t seconds /*, CleanUpFilter* filter*/
// delete file from meta
std::vector<std::string> statements;
if (!delete_ids.empty()) {
std::stringstream idsToDeleteSS;
for (auto& id : delete_ids) {
idsToDeleteSS << "id = " << id << " OR ";
// distribute id array to batches
// sqlite could not parse long sql statement
std::vector<std::vector<std::string>> id_groups;
DistributeBatch(delete_ids, id_groups);
for (auto& group : id_groups) {
std::stringstream idsToDeleteSS;
for (auto& id : group) {
idsToDeleteSS << "id = " << id << " OR ";
}
std::string idsToDeleteStr = idsToDeleteSS.str();
idsToDeleteStr = idsToDeleteStr.substr(0, idsToDeleteStr.size() - 4); // remove the last " OR "
statement = "DELETE FROM " + std::string(META_TABLEFILES) + " WHERE " + idsToDeleteStr + ";";
statements.emplace_back(statement);
}
std::string idsToDeleteStr = idsToDeleteSS.str();
idsToDeleteStr = idsToDeleteStr.substr(0, idsToDeleteStr.size() - 4); // remove the last " OR "
statement = "DELETE FROM " + std::string(META_TABLEFILES) + " WHERE " + idsToDeleteStr + ";";
statements.emplace_back(statement);
}
auto status = SqlTransaction(statements);
@ -2289,23 +2295,18 @@ SqliteMetaImpl::CleanUpFilesWithTTL(uint64_t seconds /*, CleanUpFilter* filter*/
int64_t remove_collections = 0;
if (!res.empty()) {
std::stringstream idsToDeleteSS;
for (auto& resRow : res) {
size_t id = std::stoul(resRow["id"]);
std::string collection_id;
collection_id = resRow["table_id"];
utils::DeleteCollectionPath(options_, collection_id, false); // only delete empty folder
++remove_collections;
idsToDeleteSS << "id = " << std::to_string(id) << " OR ";
}
std::string idsToDeleteStr = idsToDeleteSS.str();
idsToDeleteStr = idsToDeleteStr.substr(0, idsToDeleteStr.size() - 4); // remove the last " OR "
statement = "DELETE FROM " + std::string(META_TABLES) + " WHERE " + idsToDeleteStr + ";";
status = SqlTransaction({statement});
if (!status.ok()) {
return HandleException("Failed to clean up with ttl", status.message().c_str());
statement = "DELETE FROM " + std::string(META_TABLES) + " WHERE id = " + resRow["id"] + ";";
status = SqlTransaction({statement});
if (!status.ok()) {
return HandleException("Failed to clean up with ttl", status.message().c_str());
}
}
}

28
core/src/server/delivery/request/CreatePartitionRequest.cpp
View File

@ -10,6 +10,7 @@
// or implied. See the License for the specific language governing permissions and limitations under the License.
#include "server/delivery/request/CreatePartitionRequest.h"
#include "config/Config.h"
#include "server/DBWrapper.h"
#include "utils/Log.h"
#include "utils/TimeRecorder.h"
@ -23,8 +24,6 @@
namespace milvus {
namespace server {
constexpr uint64_t MAX_PARTITION_LIMIT = 4096;
CreatePartitionRequest::CreatePartitionRequest(const std::shared_ptr<milvus::server::Context>& context,
const std::string& collection_name, const std::string& tag)
: BaseRequest(context, BaseRequest::kCreatePartition), collection_name_(collection_name), tag_(tag) {
@ -61,28 +60,16 @@ CreatePartitionRequest::OnExecute() {
return status;
}
// only process root collection, ignore partition collection
engine::meta::CollectionSchema collection_schema;
collection_schema.collection_id_ = collection_name_;
status = DBWrapper::DB()->DescribeCollection(collection_schema);
fiu_do_on("CreatePartitionRequest.OnExecute.invalid_partition_tags",
status = Status(milvus::SERVER_UNEXPECTED_ERROR, ""));
// check partition total count
int64_t max_partition_num = 4096;
status = Config::GetInstance().GetEngineConfigMaxPartitionNum(max_partition_num);
if (!status.ok()) {
if (status.code() == DB_NOT_FOUND) {
return Status(SERVER_COLLECTION_NOT_EXIST, CollectionNotExistMsg(collection_name_));
} else {
return status;
}
} else {
if (!collection_schema.owner_collection_.empty()) {
return Status(SERVER_INVALID_COLLECTION_NAME, CollectionNotExistMsg(collection_name_));
}
return status;
}
// check partition total count
int64_t partition_count = 0;
status = DBWrapper::DB()->CountPartitions(collection_name_, partition_count);
if (partition_count >= MAX_PARTITION_LIMIT) {
if (partition_count >= max_partition_num) {
return Status(SERVER_UNSUPPORTED_ERROR, "The number of partitions exceeds the upper limit(4096)");
}
@ -95,6 +82,9 @@ CreatePartitionRequest::OnExecute() {
status = Status(milvus::SERVER_UNEXPECTED_ERROR, ""));
fiu_do_on("CreatePartitionRequest.OnExecute.throw_std_exception", throw std::exception());
if (!status.ok()) {
if (status.code() == DB_NOT_FOUND) {
return Status(SERVER_COLLECTION_NOT_EXIST, CollectionNotExistMsg(collection_name_));
}
// partition could exist
if (status.code() == DB_ALREADY_EXIST) {
return Status(SERVER_INVALID_COLLECTION_NAME, status.message());

5
core/unittest/server/test_config.cpp
View File

@ -343,6 +343,11 @@ TEST_F(ConfigTest, SERVER_CONFIG_VALID_TEST) {
ASSERT_TRUE(config.GetEngineConfigSimdType(str_val).ok());
ASSERT_TRUE(str_val == engine_simd_type);
int64_t max_partition = 1;
ASSERT_TRUE(config.SetEngineConfigMaxPartitionNum(std::to_string(max_partition)).ok());
ASSERT_TRUE(config.GetEngineConfigMaxPartitionNum(int64_val).ok());
ASSERT_TRUE(int64_val == max_partition);
#ifdef MILVUS_GPU_VERSION
int64_t engine_gpu_search_threshold = 800;
auto status = config.SetGpuResourceConfigGpuSearchThreshold(std::to_string(engine_gpu_search_threshold));