diff --git a/cpp/src/db/DBFactory.cpp b/cpp/src/db/DBFactory.cpp index eb13c309fc..91e03278d6 100644 --- a/cpp/src/db/DBFactory.cpp +++ b/cpp/src/db/DBFactory.cpp @@ -33,14 +33,14 @@ namespace zilliz { namespace milvus { namespace engine { -Options DBFactory::BuildOption() { +DBOptions DBFactory::BuildOption() { auto meta = MetaFactory::BuildOption(); - Options options; + DBOptions options; options.meta = meta; return options; } -DBPtr DBFactory::Build(const Options& options) { +DBPtr DBFactory::Build(const DBOptions& options) { return std::make_shared(options); } diff --git a/cpp/src/db/DBFactory.h b/cpp/src/db/DBFactory.h index 7de4396846..bb363118ed 100644 --- a/cpp/src/db/DBFactory.h +++ b/cpp/src/db/DBFactory.h @@ -29,9 +29,9 @@ namespace engine { class DBFactory { public: - static Options BuildOption(); + static DBOptions BuildOption(); - static DBPtr Build(const Options& options); + static DBPtr Build(const DBOptions& options); }; diff --git a/cpp/src/db/DBImpl.cpp b/cpp/src/db/DBImpl.cpp index 648038de25..0dec98b92b 100644 --- a/cpp/src/db/DBImpl.cpp +++ b/cpp/src/db/DBImpl.cpp @@ -51,7 +51,7 @@ constexpr uint64_t INDEX_ACTION_INTERVAL = 1; } -DBImpl::DBImpl(const Options& options) +DBImpl::DBImpl(const DBOptions& options) : options_(options), shutting_down_(true), compact_thread_pool_(1, 1), @@ -77,7 +77,7 @@ Status DBImpl::Start() { shutting_down_.store(false, std::memory_order_release); //for distribute version, some nodes are read only - if (options_.mode != Options::MODE::READ_ONLY) { + if (options_.mode != DBOptions::MODE::READ_ONLY) { ENGINE_LOG_TRACE << "StartTimerTasks"; bg_timer_thread_ = std::thread(&DBImpl::BackgroundTimerTask, this); } @@ -98,7 +98,7 @@ Status DBImpl::Stop() { //wait compaction/buildindex finish bg_timer_thread_.join(); - if (options_.mode != Options::MODE::READ_ONLY) { + if (options_.mode != DBOptions::MODE::READ_ONLY) { meta_ptr_->CleanUp(); } @@ -687,7 +687,7 @@ void DBImpl::BackgroundCompaction(std::set table_ids) { meta_ptr_->Archive(); int ttl = 5*meta::M_SEC;//default: file will be deleted after 5 minutes - if (options_.mode == Options::MODE::CLUSTER) { + if (options_.mode == DBOptions::MODE::CLUSTER) { ttl = meta::D_SEC; } meta_ptr_->CleanUpFilesWithTTL(ttl); diff --git a/cpp/src/db/DBImpl.h b/cpp/src/db/DBImpl.h index 85803b763f..30e9667994 100644 --- a/cpp/src/db/DBImpl.h +++ b/cpp/src/db/DBImpl.h @@ -44,7 +44,7 @@ class Meta; class DBImpl : public DB { public: - explicit DBImpl(const Options &options); + explicit DBImpl(const DBOptions &options); ~DBImpl(); Status Start() override; @@ -76,28 +76,28 @@ class DBImpl : public DB { Status DropIndex(const std::string& table_id) override; Status Query(const std::string &table_id, - uint64_t k, - uint64_t nq, - uint64_t nprobe, - const float *vectors, - QueryResults &results) override; + uint64_t k, + uint64_t nq, + uint64_t nprobe, + const float *vectors, + QueryResults &results) override; Status Query(const std::string &table_id, - uint64_t k, - uint64_t nq, - uint64_t nprobe, - const float *vectors, - const meta::DatesT &dates, - QueryResults &results) override; + uint64_t k, + uint64_t nq, + uint64_t nprobe, + const float *vectors, + const meta::DatesT &dates, + QueryResults &results) override; Status Query(const std::string &table_id, - const std::vector &file_ids, - uint64_t k, - uint64_t nq, - uint64_t nprobe, - const float *vectors, - const meta::DatesT &dates, - QueryResults &results) override; + const std::vector &file_ids, + uint64_t k, + uint64_t nq, + uint64_t nprobe, + const float *vectors, + const meta::DatesT &dates, + QueryResults &results) override; Status Size(uint64_t &result) override; @@ -130,7 +130,7 @@ class DBImpl : public DB { Status MemSerialize(); private: - const Options options_; + const DBOptions options_; std::atomic shutting_down_; diff --git a/cpp/src/db/Options.cpp b/cpp/src/db/Options.cpp index 9abc8e66ec..5796b781b2 100644 --- a/cpp/src/db/Options.cpp +++ b/cpp/src/db/Options.cpp @@ -27,9 +27,6 @@ namespace zilliz { namespace milvus { namespace engine { -Options::Options() { -} - ArchiveConf::ArchiveConf(const std::string& type, const std::string& criterias) { ParseType(type); ParseCritirias(criterias); diff --git a/cpp/src/db/Options.h b/cpp/src/db/Options.h index c5b4648c6d..663ac3fd09 100644 --- a/cpp/src/db/Options.h +++ b/cpp/src/db/Options.h @@ -58,16 +58,13 @@ struct DBMetaOptions { ArchiveConf archive_conf = ArchiveConf("delete"); }; // DBMetaOptions -struct Options { - +struct DBOptions { typedef enum { SINGLE, CLUSTER, READ_ONLY } MODE; - Options(); - uint16_t merge_trigger_number = 2; DBMetaOptions meta; int mode = MODE::SINGLE; diff --git a/cpp/src/db/insert/MemManagerImpl.h b/cpp/src/db/insert/MemManagerImpl.h index 51e8f1044e..767dc0eef0 100644 --- a/cpp/src/db/insert/MemManagerImpl.h +++ b/cpp/src/db/insert/MemManagerImpl.h @@ -38,7 +38,7 @@ class MemManagerImpl : public MemManager { public: using Ptr = std::shared_ptr; - MemManagerImpl(const meta::MetaPtr &meta, const Options &options) + MemManagerImpl(const meta::MetaPtr &meta, const DBOptions &options) : meta_(meta), options_(options) {} Status InsertVectors(const std::string &table_id, @@ -66,7 +66,7 @@ class MemManagerImpl : public MemManager { MemIdMap mem_id_map_; MemList immu_mem_list_; meta::MetaPtr meta_; - Options options_; + DBOptions options_; std::mutex mutex_; std::mutex serialization_mtx_; }; // NewMemManager diff --git a/cpp/src/db/insert/MemMenagerFactory.cpp b/cpp/src/db/insert/MemMenagerFactory.cpp index d8620e908c..d0ef51f61f 100644 --- a/cpp/src/db/insert/MemMenagerFactory.cpp +++ b/cpp/src/db/insert/MemMenagerFactory.cpp @@ -31,8 +31,7 @@ namespace zilliz { namespace milvus { namespace engine { -MemManagerPtr MemManagerFactory::Build(const std::shared_ptr& meta, - const Options& options) { +MemManagerPtr MemManagerFactory::Build(const std::shared_ptr& meta, const DBOptions& options) { return std::make_shared(meta, options); } diff --git a/cpp/src/db/insert/MemMenagerFactory.h b/cpp/src/db/insert/MemMenagerFactory.h index 4bf4267c87..d38404f5c0 100644 --- a/cpp/src/db/insert/MemMenagerFactory.h +++ b/cpp/src/db/insert/MemMenagerFactory.h @@ -26,7 +26,7 @@ namespace engine { class MemManagerFactory { public: - static MemManagerPtr Build(const std::shared_ptr &meta, const Options &options); + static MemManagerPtr Build(const std::shared_ptr &meta, const DBOptions &options); }; } diff --git a/cpp/src/db/insert/MemTable.cpp b/cpp/src/db/insert/MemTable.cpp index 7f4f497ba1..4be0695572 100644 --- a/cpp/src/db/insert/MemTable.cpp +++ b/cpp/src/db/insert/MemTable.cpp @@ -26,7 +26,7 @@ namespace engine { MemTable::MemTable(const std::string &table_id, const meta::MetaPtr &meta, - const Options &options) : + const DBOptions &options) : table_id_(table_id), meta_(meta), options_(options) { diff --git a/cpp/src/db/insert/MemTable.h b/cpp/src/db/insert/MemTable.h index def4a6dcca..c1b4125331 100644 --- a/cpp/src/db/insert/MemTable.h +++ b/cpp/src/db/insert/MemTable.h @@ -33,7 +33,7 @@ class MemTable { public: using MemTableFileList = std::vector; - MemTable(const std::string &table_id, const meta::MetaPtr &meta, const Options &options); + MemTable(const std::string &table_id, const meta::MetaPtr &meta, const DBOptions &options); Status Add(VectorSourcePtr &source, IDNumbers &vector_ids); @@ -56,7 +56,7 @@ class MemTable { meta::MetaPtr meta_; - Options options_; + DBOptions options_; std::mutex mutex_; diff --git a/cpp/src/db/insert/MemTableFile.cpp b/cpp/src/db/insert/MemTableFile.cpp index 57ef39c670..12dbbcf49f 100644 --- a/cpp/src/db/insert/MemTableFile.cpp +++ b/cpp/src/db/insert/MemTableFile.cpp @@ -31,7 +31,7 @@ namespace engine { MemTableFile::MemTableFile(const std::string &table_id, const meta::MetaPtr &meta, - const Options &options) : + const DBOptions &options) : table_id_(table_id), meta_(meta), options_(options) { diff --git a/cpp/src/db/insert/MemTableFile.h b/cpp/src/db/insert/MemTableFile.h index 5ac3e15ad2..168fabe7e9 100644 --- a/cpp/src/db/insert/MemTableFile.h +++ b/cpp/src/db/insert/MemTableFile.h @@ -31,7 +31,7 @@ namespace engine { class MemTableFile { public: - MemTableFile(const std::string &table_id, const meta::MetaPtr &meta, const Options &options); + MemTableFile(const std::string &table_id, const meta::MetaPtr &meta, const DBOptions &options); Status Add(const VectorSourcePtr &source, IDNumbers& vector_ids); @@ -53,7 +53,7 @@ class MemTableFile { meta::MetaPtr meta_; - Options options_; + DBOptions options_; size_t current_mem_; diff --git a/cpp/src/db/meta/MySQLMetaImpl.cpp b/cpp/src/db/meta/MySQLMetaImpl.cpp index ccd7e79ada..a07d4c82cf 100644 --- a/cpp/src/db/meta/MySQLMetaImpl.cpp +++ b/cpp/src/db/meta/MySQLMetaImpl.cpp @@ -135,7 +135,7 @@ Status MySQLMetaImpl::Initialize() { ENGINE_LOG_DEBUG << "MySQL connection pool: maximum pool size = " << std::to_string(maxPoolSize); try { - if (mode_ != Options::MODE::READ_ONLY) { + if (mode_ != DBOptions::MODE::READ_ONLY) { CleanUp(); } @@ -621,7 +621,7 @@ Status MySQLMetaImpl::DeleteTable(const std::string &table_id) { } //Scoped Connection - if (mode_ == Options::MODE::CLUSTER) { + if (mode_ == DBOptions::MODE::CLUSTER) { DeleteTableFiles(table_id); } diff --git a/cpp/src/main.cpp b/cpp/src/main.cpp index d9b160d6e0..3065e7d684 100644 --- a/cpp/src/main.cpp +++ b/cpp/src/main.cpp @@ -109,9 +109,9 @@ main(int argc, char *argv[]) { } } - server::Server* server_ptr = server::Server::Instance(); - server_ptr->Init(start_daemonized, pid_filename, config_filename, log_config_file); - return server_ptr->Start(); + server::Server& server = server::Server::Instance(); + server.Init(start_daemonized, pid_filename, config_filename, log_config_file); + return server.Start(); } void diff --git a/cpp/src/server/DBWrapper.cpp b/cpp/src/server/DBWrapper.cpp index 508d9552b1..115a636621 100644 --- a/cpp/src/server/DBWrapper.cpp +++ b/cpp/src/server/DBWrapper.cpp @@ -24,6 +24,7 @@ #include "utils/StringHelpFunctions.h" #include +#include namespace zilliz { namespace milvus { @@ -35,7 +36,7 @@ DBWrapper::DBWrapper() { Status DBWrapper::StartService() { //db config - zilliz::milvus::engine::Options opt; + engine::DBOptions opt; ConfigNode& db_config = ServerConfig::GetInstance().GetConfig(CONFIG_DB); opt.meta.backend_uri = db_config.GetValue(CONFIG_DB_URL); std::string db_path = db_config.GetValue(CONFIG_DB_PATH); @@ -51,13 +52,13 @@ Status DBWrapper::StartService() { ConfigNode& serverConfig = ServerConfig::GetInstance().GetConfig(CONFIG_SERVER); std::string mode = serverConfig.GetValue(CONFIG_CLUSTER_MODE, "single"); if (mode == "single") { - opt.mode = zilliz::milvus::engine::Options::MODE::SINGLE; + opt.mode = engine::DBOptions::MODE::SINGLE; } else if (mode == "cluster") { - opt.mode = zilliz::milvus::engine::Options::MODE::CLUSTER; + opt.mode = engine::DBOptions::MODE::CLUSTER; } else if (mode == "read_only") { - opt.mode = zilliz::milvus::engine::Options::MODE::READ_ONLY; + opt.mode = engine::DBOptions::MODE::READ_ONLY; } else { std::cerr << "ERROR: mode specified in server_config is not one of ['single', 'cluster', 'read_only']" << std::endl; @@ -78,6 +79,8 @@ Status DBWrapper::StartService() { } } + faiss::distance_compute_blas_threshold = engine_config.GetInt32Value(CONFIG_DCBT, 20);//init faiss global variable + //set archive config engine::ArchiveConf::CriteriaT criterial; int64_t disk = db_config.GetInt64Value(CONFIG_DB_ARCHIVE_DISK, 0); diff --git a/cpp/src/server/Server.cpp b/cpp/src/server/Server.cpp index e1e67a112f..252f92a5f0 100644 --- a/cpp/src/server/Server.cpp +++ b/cpp/src/server/Server.cpp @@ -42,10 +42,10 @@ namespace zilliz { namespace milvus { namespace server { -Server * +Server& Server::Instance() { static Server server; - return &server; + return server; } Server::Server() { diff --git a/cpp/src/server/Server.h b/cpp/src/server/Server.h index 43e8b28028..eaf19d6b15 100644 --- a/cpp/src/server/Server.h +++ b/cpp/src/server/Server.h @@ -27,15 +27,19 @@ namespace milvus { namespace server { class Server { - public: - static Server* Instance(); +public: + static Server &Instance(); + + void Init(int64_t daemonized, const std::string &pid_filename, const std::string &config_filename, + const std::string &log_config_file); - void Init(int64_t daemonized, const std::string& pid_filename, const std::string& config_filename, const std::string &log_config_file); int Start(); + void Stop(); - private: +private: Server(); + ~Server(); void Daemonize(); @@ -43,9 +47,10 @@ class Server { ErrorCode LoadConfig(); void StartService(); + void StopService(); - private: +private: int64_t daemonized_ = 0; int pid_fd = -1; std::string pid_filename_; diff --git a/cpp/src/server/grpc_impl/GrpcMilvusServer.cpp b/cpp/src/server/grpc_impl/GrpcMilvusServer.cpp index 73f8a7573e..6770e3111a 100644 --- a/cpp/src/server/grpc_impl/GrpcMilvusServer.cpp +++ b/cpp/src/server/grpc_impl/GrpcMilvusServer.cpp @@ -17,10 +17,9 @@ #include "milvus.grpc.pb.h" #include "GrpcMilvusServer.h" -#include "../ServerConfig.h" -#include "../DBWrapper.h" +#include "server/ServerConfig.h" +#include "server/DBWrapper.h" #include "utils/Log.h" -#include "faiss/utils.h" #include "GrpcRequestHandler.h" #include @@ -47,6 +46,7 @@ static std::unique_ptr<::grpc::Server> server; constexpr long MESSAGE_SIZE = -1; +//this class is to check port occupation during server start class NoReusePortOption : public ::grpc::ServerBuilderOption { public: void UpdateArguments(::grpc::ChannelArguments *args) override { @@ -71,8 +71,6 @@ GrpcMilvusServer::StartService() { std::string address = server_config.GetValue(CONFIG_SERVER_ADDRESS, "127.0.0.1"); int32_t port = server_config.GetInt32Value(CONFIG_SERVER_PORT, 19530); - faiss::distance_compute_blas_threshold = engine_config.GetInt32Value(CONFIG_DCBT, 20); - std::string server_address(address + ":" + std::to_string(port)); ::grpc::ServerBuilder builder; diff --git a/cpp/src/utils/BlockingQueue.inl b/cpp/src/utils/BlockingQueue.inl index 8df38dd547..86237f3322 100644 --- a/cpp/src/utils/BlockingQueue.inl +++ b/cpp/src/utils/BlockingQueue.inl @@ -18,8 +18,6 @@ #pragma once -//#include "Log.h" -#include "Error.h" namespace zilliz { namespace milvus { @@ -31,14 +29,6 @@ BlockingQueue::Put(const T &task) { std::unique_lock lock(mtx); full_.wait(lock, [this] { return (queue_.size() < capacity_); }); - if (queue_.size() >= capacity_) { - std::string error_msg = - "blocking queue is full, capacity: " + std::to_string(capacity_) + " queue_size: " + - std::to_string(queue_.size()); - //SERVER_LOG_ERROR << error_msg; - throw ServerException(SERVER_BLOCKING_QUEUE_EMPTY, error_msg); - } - queue_.push(task); empty_.notify_all(); } @@ -49,12 +39,6 @@ BlockingQueue::Take() { std::unique_lock lock(mtx); empty_.wait(lock, [this] { return !queue_.empty(); }); - if (queue_.empty()) { - std::string error_msg = "blocking queue empty"; - //SERVER_LOG_ERROR << error_msg; - throw ServerException(SERVER_BLOCKING_QUEUE_EMPTY, error_msg); - } - T front(queue_.front()); queue_.pop(); full_.notify_all(); @@ -73,11 +57,7 @@ T BlockingQueue::Front() { std::unique_lock lock(mtx); empty_.wait(lock, [this] { return !queue_.empty(); }); - if (queue_.empty()) { - std::string error_msg = "blocking queue empty"; - //SERVER_LOG_ERROR << error_msg; - throw ServerException(SERVER_BLOCKING_QUEUE_EMPTY, error_msg); - } + T front(queue_.front()); return front; } @@ -88,12 +68,6 @@ BlockingQueue::Back() { std::unique_lock lock(mtx); empty_.wait(lock, [this] { return !queue_.empty(); }); - if (queue_.empty()) { - std::string error_msg = "blocking queue empty"; - //SERVER_LOG_ERROR << error_msg; - throw ServerException(SERVER_BLOCKING_QUEUE_EMPTY, error_msg); - } - T back(queue_.back()); return back; } diff --git a/cpp/src/utils/CommonUtil.cpp b/cpp/src/utils/CommonUtil.cpp index 19a63006cd..ff985dcd7d 100644 --- a/cpp/src/utils/CommonUtil.cpp +++ b/cpp/src/utils/CommonUtil.cpp @@ -162,6 +162,11 @@ uint64_t CommonUtil::GetFileSize(const std::string &path) { } } +std::string CommonUtil::GetFileName(std::string filename) { + int pos = filename.find_last_of('/'); + return filename.substr(pos + 1); +} + std::string CommonUtil::GetExePath() { const size_t buf_len = 1024; char buf[buf_len]; diff --git a/cpp/src/utils/CommonUtil.h b/cpp/src/utils/CommonUtil.h index aee57f0446..7e1349bf9e 100755 --- a/cpp/src/utils/CommonUtil.h +++ b/cpp/src/utils/CommonUtil.h @@ -38,6 +38,7 @@ class CommonUtil { static Status CreateDirectory(const std::string &path); static Status DeleteDirectory(const std::string &path); + static std::string GetFileName(std::string filename); static std::string GetExePath(); static bool TimeStrToTime(const std::string& time_str, diff --git a/cpp/src/utils/LogUtil.cpp b/cpp/src/utils/LogUtil.cpp index 6224c854a5..e4af1104db 100644 --- a/cpp/src/utils/LogUtil.cpp +++ b/cpp/src/utils/LogUtil.cpp @@ -84,49 +84,15 @@ void RolloutHandler(const char *filename, std::size_t size, el::Level level) { } } -int32_t InitLog(const std::string &log_config_file) { -#if 0 - ServerConfig &config = ServerConfig::GetInstance(); - ConfigNode log_config = config.GetConfig(CONFIG_LOG); - const std::map& settings = log_config.GetChildren(); - - std::string str_config; - for(auto iter : settings) { - str_config += "* "; - str_config += iter.first; - str_config += ":"; - str_config.append("\n"); - - auto sub_configs = iter.second.GetConfig(); - for(auto it_sub : sub_configs) { - str_config += " "; - str_config += it_sub.first; - str_config += " = "; - std::string temp = it_sub.first; - std::transform(temp.begin(), temp.end(), temp.begin(), ::tolower); - bool is_text = (temp == "format" || temp == "filename"); - if(is_text){ - str_config += "\""; - } - str_config += it_sub.second; - if(is_text){ - str_config += "\""; - } - str_config.append("\n"); - } - } - - el::Configurations conf; - conf.parseFromText(str_config); -#else +Status InitLog(const std::string &log_config_file) { el::Configurations conf(log_config_file); -#endif el::Loggers::reconfigureAllLoggers(conf); el::Loggers::addFlag(el::LoggingFlag::StrictLogFileSizeCheck); el::Helpers::installPreRollOutCallback(RolloutHandler); el::Loggers::addFlag(el::LoggingFlag::DisableApplicationAbortOnFatalLog); - return 0; + + return Status::OK(); } diff --git a/cpp/src/utils/LogUtil.h b/cpp/src/utils/LogUtil.h index ebeb117aa2..d86b301d9f 100644 --- a/cpp/src/utils/LogUtil.h +++ b/cpp/src/utils/LogUtil.h @@ -17,6 +17,8 @@ #pragma once +#include "utils/Status.h" + #include #include #include "easylogging++.h" @@ -24,11 +26,9 @@ namespace zilliz { namespace milvus { namespace server { -int32_t InitLog(const std::string& log_config_file); -inline std::string GetFileName(std::string filename) { - int pos = filename.find_last_of('/'); - return filename.substr(pos + 1); -} + +Status InitLog(const std::string& log_config_file); + void RolloutHandler(const char *filename, std::size_t size, el::Level level); #define SHOW_LOCATION diff --git a/cpp/src/utils/SignalUtil.cpp b/cpp/src/utils/SignalUtil.cpp index 07c1001044..e380dc6c85 100644 --- a/cpp/src/utils/SignalUtil.cpp +++ b/cpp/src/utils/SignalUtil.cpp @@ -31,23 +31,19 @@ void SignalUtil::HandleSignal(int signum){ switch(signum){ case SIGINT: case SIGUSR2:{ - server::Server* server_ptr = server::Server::Instance(); - server_ptr->Stop(); + SERVER_LOG_INFO << "Server received signal:" << std::to_string(signum); + + server::Server& server_ptr = server::Server::Instance(); + server_ptr.Stop(); exit(0); } default:{ - SERVER_LOG_INFO << "Server received signal:" << std::to_string(signum); + SERVER_LOG_INFO << "Server received critical signal:" << std::to_string(signum); SignalUtil::PrintStacktrace(); - std::string info = "Server encounter critical signal:"; - info += std::to_string(signum); -// SendSignalMessage(signum, info); - - SERVER_LOG_INFO << info; - - server::Server* server_ptr = server::Server::Instance(); - server_ptr->Stop(); + server::Server& server_ptr = server::Server::Instance(); + server_ptr.Stop(); exit(1); } diff --git a/cpp/src/utils/ValidationUtil.h b/cpp/src/utils/ValidationUtil.h index 0c109e0e25..ad8ba5fee9 100644 --- a/cpp/src/utils/ValidationUtil.h +++ b/cpp/src/utils/ValidationUtil.h @@ -26,6 +26,9 @@ namespace milvus { namespace server { class ValidationUtil { +private: + ValidationUtil() = default; + public: static Status ValidateTableName(const std::string &table_name); diff --git a/cpp/unittest/db/mysql_meta_test.cpp b/cpp/unittest/db/mysql_meta_test.cpp index ab94c817fe..e99565886d 100644 --- a/cpp/unittest/db/mysql_meta_test.cpp +++ b/cpp/unittest/db/mysql_meta_test.cpp @@ -130,7 +130,7 @@ TEST_F(MySqlMetaTest, ARCHIVE_TEST_DAYS) { std::stringstream ss; ss << "days:" << days_num; options.archive_conf = ArchiveConf("delete", ss.str()); - int mode = Options::MODE::SINGLE; + int mode = DBOptions::MODE::SINGLE; meta::MySQLMetaImpl impl(options, mode); auto table_id = "meta_test_table"; @@ -190,7 +190,7 @@ TEST_F(MySqlMetaTest, ARCHIVE_TEST_DISK) { DBMetaOptions options = GetOptions().meta; options.archive_conf = ArchiveConf("delete", "disk:11"); - int mode = Options::MODE::SINGLE; + int mode = DBOptions::MODE::SINGLE; auto impl = meta::MySQLMetaImpl(options, mode); auto table_id = "meta_test_group"; diff --git a/cpp/unittest/db/utils.cpp b/cpp/unittest/db/utils.cpp index 4bc9699e05..aededfd0bd 100644 --- a/cpp/unittest/db/utils.cpp +++ b/cpp/unittest/db/utils.cpp @@ -66,7 +66,7 @@ void BaseTest::TearDown() { zilliz::knowhere::FaissGpuResourceMgr::GetInstance().Free(); } -engine::Options BaseTest::GetOptions() { +engine::DBOptions BaseTest::GetOptions() { auto options = engine::DBFactory::BuildOption(); options.meta.path = "/tmp/milvus_test"; options.meta.backend_uri = "sqlite://:@:/"; @@ -108,7 +108,7 @@ void DBTest::TearDown() { } ///////////////////////////////////////////////////////////////////////////////////////////////////////////////// -engine::Options DBTest2::GetOptions() { +engine::DBOptions DBTest2::GetOptions() { auto options = engine::DBFactory::BuildOption(); options.meta.path = "/tmp/milvus_test"; options.meta.archive_conf = engine::ArchiveConf("delete", "disk:1"); @@ -134,7 +134,7 @@ void MetaTest::TearDown() { } ///////////////////////////////////////////////////////////////////////////////////////////////////////////////// -engine::Options MySqlDBTest::GetOptions() { +engine::DBOptions MySqlDBTest::GetOptions() { auto options = engine::DBFactory::BuildOption(); options.meta.path = "/tmp/milvus_test"; options.meta.backend_uri = DBTestEnvironment::getURI(); @@ -163,7 +163,7 @@ void MySqlMetaTest::TearDown() { boost::filesystem::remove_all(options.meta.path); } -zilliz::milvus::engine::Options MySqlMetaTest::GetOptions() { +engine::DBOptions MySqlMetaTest::GetOptions() { auto options = engine::DBFactory::BuildOption(); options.meta.path = "/tmp/milvus_test"; options.meta.backend_uri = DBTestEnvironment::getURI(); diff --git a/cpp/unittest/db/utils.h b/cpp/unittest/db/utils.h index c1dc730089..7a1320ef6b 100644 --- a/cpp/unittest/db/utils.h +++ b/cpp/unittest/db/utils.h @@ -50,7 +50,7 @@ protected: virtual void SetUp() override; virtual void TearDown() override; - virtual zilliz::milvus::engine::Options GetOptions(); + virtual zilliz::milvus::engine::DBOptions GetOptions(); }; ///////////////////////////////////////////////////////////////////////////////////////////////////////////////// @@ -65,7 +65,7 @@ class DBTest : public BaseTest { ///////////////////////////////////////////////////////////////////////////////////////////////////////////////// class DBTest2 : public DBTest { protected: - virtual zilliz::milvus::engine::Options GetOptions() override; + virtual zilliz::milvus::engine::DBOptions GetOptions() override; }; ///////////////////////////////////////////////////////////////////////////////////////////////////////////////// @@ -84,7 +84,7 @@ class MetaTest : public BaseTest { ///////////////////////////////////////////////////////////////////////////////////////////////////////////////// class MySqlDBTest : public DBTest { protected: - zilliz::milvus::engine::Options GetOptions(); + zilliz::milvus::engine::DBOptions GetOptions(); }; ///////////////////////////////////////////////////////////////////////////////////////////////////////////////// @@ -94,7 +94,7 @@ class MySqlMetaTest : public BaseTest { virtual void SetUp() override; virtual void TearDown() override; - zilliz::milvus::engine::Options GetOptions(); + zilliz::milvus::engine::DBOptions GetOptions(); }; diff --git a/cpp/unittest/metrics/utils.cpp b/cpp/unittest/metrics/utils.cpp index 4dc370c952..49ccc53d30 100644 --- a/cpp/unittest/metrics/utils.cpp +++ b/cpp/unittest/metrics/utils.cpp @@ -52,7 +52,7 @@ void MetricTest::InitLog() { el::Loggers::reconfigureLogger("default", defaultConf); } -engine::Options MetricTest::GetOptions() { +engine::DBOptions MetricTest::GetOptions() { auto options = engine::DBFactory::BuildOption(); options.meta.path = "/tmp/milvus_test"; options.meta.backend_uri = "sqlite://:@:/"; diff --git a/cpp/unittest/metrics/utils.h b/cpp/unittest/metrics/utils.h index 7f093dd091..37c21c82f9 100644 --- a/cpp/unittest/metrics/utils.h +++ b/cpp/unittest/metrics/utils.h @@ -72,5 +72,5 @@ protected: void InitLog(); virtual void SetUp() override; virtual void TearDown() override; - virtual zilliz::milvus::engine::Options GetOptions(); + virtual zilliz::milvus::engine::DBOptions GetOptions(); }; \ No newline at end of file diff --git a/cpp/unittest/server/rpc_test.cpp b/cpp/unittest/server/rpc_test.cpp index b87ead099a..f89c98eb9d 100644 --- a/cpp/unittest/server/rpc_test.cpp +++ b/cpp/unittest/server/rpc_test.cpp @@ -34,11 +34,9 @@ #include "scheduler/ResourceFactory.h" #include "utils/CommonUtil.h" +using namespace zilliz::milvus; -namespace zilliz { -namespace milvus { -namespace server { -namespace grpc { +namespace { static const char *TABLE_NAME = "test_grpc"; static constexpr int64_t TABLE_DIM = 256; @@ -65,35 +63,35 @@ class RpcHandlerTest : public testing::Test { res_mgr->Start(); engine::SchedInst::GetInstance()->Start(); - zilliz::milvus::engine::Options opt; + engine::DBOptions opt; - ConfigNode &db_config = ServerConfig::GetInstance().GetConfig(CONFIG_DB); - db_config.SetValue(CONFIG_DB_URL, "sqlite://:@:/"); - db_config.SetValue(CONFIG_DB_PATH, "/tmp/milvus_test"); - db_config.SetValue(CONFIG_DB_SLAVE_PATH, ""); - db_config.SetValue(CONFIG_DB_ARCHIVE_DISK, ""); - db_config.SetValue(CONFIG_DB_ARCHIVE_DAYS, ""); + server::ConfigNode &db_config = server::ServerConfig::GetInstance().GetConfig(server::CONFIG_DB); + db_config.SetValue(server::CONFIG_DB_URL, "sqlite://:@:/"); + db_config.SetValue(server::CONFIG_DB_PATH, "/tmp/milvus_test"); + db_config.SetValue(server::CONFIG_DB_SLAVE_PATH, ""); + db_config.SetValue(server::CONFIG_DB_ARCHIVE_DISK, ""); + db_config.SetValue(server::CONFIG_DB_ARCHIVE_DAYS, ""); - ConfigNode &cache_config = ServerConfig::GetInstance().GetConfig(CONFIG_CACHE); - cache_config.SetValue(CONFIG_INSERT_CACHE_IMMEDIATELY, ""); + server::ConfigNode &cache_config = server::ServerConfig::GetInstance().GetConfig(server::CONFIG_CACHE); + cache_config.SetValue(server::CONFIG_INSERT_CACHE_IMMEDIATELY, ""); - ConfigNode &engine_config = ServerConfig::GetInstance().GetConfig(CONFIG_ENGINE); - engine_config.SetValue(CONFIG_OMP_THREAD_NUM, ""); + server::ConfigNode &engine_config = server::ServerConfig::GetInstance().GetConfig(server::CONFIG_ENGINE); + engine_config.SetValue(server::CONFIG_OMP_THREAD_NUM, ""); - ConfigNode &serverConfig = ServerConfig::GetInstance().GetConfig(CONFIG_SERVER); -// serverConfig.SetValue(CONFIG_CLUSTER_MODE, "cluster"); + server::ConfigNode &serverConfig = server::ServerConfig::GetInstance().GetConfig(server::CONFIG_SERVER); +// serverConfig.SetValue(server::CONFIG_CLUSTER_MODE, "cluster"); // DBWrapper::GetInstance().GetInstance().StartService(); // DBWrapper::GetInstance().GetInstance().StopService(); // -// serverConfig.SetValue(CONFIG_CLUSTER_MODE, "read_only"); +// serverConfig.SetValue(server::CONFIG_CLUSTER_MODE, "read_only"); // DBWrapper::GetInstance().GetInstance().StartService(); // DBWrapper::GetInstance().GetInstance().StopService(); - serverConfig.SetValue(CONFIG_CLUSTER_MODE, "single"); - DBWrapper::GetInstance().GetInstance().StartService(); + serverConfig.SetValue(server::CONFIG_CLUSTER_MODE, "single"); + server::DBWrapper::GetInstance().StartService(); //initialize handler, create table - handler = std::make_shared(); + handler = std::make_shared(); ::grpc::ServerContext context; ::milvus::grpc::TableSchema request; ::milvus::grpc::Status status; @@ -106,16 +104,15 @@ class RpcHandlerTest : public testing::Test { void TearDown() override { - DBWrapper::GetInstance().StopService(); + server::DBWrapper::GetInstance().StopService(); engine::ResMgrInst::GetInstance()->Stop(); engine::SchedInst::GetInstance()->Stop(); boost::filesystem::remove_all("/tmp/milvus_test"); } protected: - std::shared_ptr handler; + std::shared_ptr handler; }; -namespace { void BuildVectors(int64_t from, int64_t to, std::vector> &vector_record_array) { if (to <= from) { @@ -146,6 +143,7 @@ std::string CurrentTmDate(int64_t offset_day = 0) { return str; } + } TEST_F(RpcHandlerTest, HasTableTest) { @@ -429,26 +427,27 @@ TEST_F(RpcHandlerTest, DeleteByRangeTest) { } ////////////////////////////////////////////////////////////////////// -class DummyTask : public GrpcBaseTask { - public: +namespace { +class DummyTask : public server::grpc::GrpcBaseTask { +public: Status OnExecute() override { return Status::OK(); } - static BaseTaskPtr - Create(std::string& dummy) { - return std::shared_ptr(new DummyTask(dummy)); + static server::grpc::BaseTaskPtr + Create(std::string &dummy) { + return std::shared_ptr(new DummyTask(dummy)); } - public: +public: explicit DummyTask(std::string &dummy) : GrpcBaseTask(dummy) { } }; class RpcSchedulerTest : public testing::Test { - protected: +protected: void SetUp() override { std::string dummy = "dql"; @@ -458,25 +457,22 @@ class RpcSchedulerTest : public testing::Test { std::shared_ptr task_ptr; }; +} + TEST_F(RpcSchedulerTest, BaseTaskTest){ auto status = task_ptr->Execute(); ASSERT_TRUE(status.ok()); - GrpcRequestScheduler::GetInstance().Start(); + server::grpc::GrpcRequestScheduler::GetInstance().Start(); ::milvus::grpc::Status grpc_status; std::string dummy = "dql"; - BaseTaskPtr base_task_ptr = DummyTask::Create(dummy); - GrpcRequestScheduler::GetInstance().ExecTask(base_task_ptr, &grpc_status); + server::grpc::BaseTaskPtr base_task_ptr = DummyTask::Create(dummy); + server::grpc::GrpcRequestScheduler::GetInstance().ExecTask(base_task_ptr, &grpc_status); - GrpcRequestScheduler::GetInstance().ExecuteTask(task_ptr); + server::grpc::GrpcRequestScheduler::GetInstance().ExecuteTask(task_ptr); task_ptr = nullptr; - GrpcRequestScheduler::GetInstance().ExecuteTask(task_ptr); + server::grpc::GrpcRequestScheduler::GetInstance().ExecuteTask(task_ptr); - GrpcRequestScheduler::GetInstance().Stop(); -} - -} -} -} + server::grpc::GrpcRequestScheduler::GetInstance().Stop(); } diff --git a/cpp/unittest/server/util_test.cpp b/cpp/unittest/server/util_test.cpp index 00be2f3f13..b582c9a17a 100644 --- a/cpp/unittest/server/util_test.cpp +++ b/cpp/unittest/server/util_test.cpp @@ -21,6 +21,7 @@ #include #include #include +#include #include "utils/CommonUtil.h" #include "utils/Error.h" @@ -51,6 +52,10 @@ TEST(UtilTest, EXCEPTION_TEST) { ASSERT_EQ(msg, err_msg); } +TEST(UtilTest, SIGNAL_TEST) { + server::SignalUtil::PrintStacktrace(); +} + TEST(UtilTest, COMMON_TEST) { unsigned long total_mem = 0, free_mem = 0; server::CommonUtil::GetSystemMemInfo(total_mem, free_mem); @@ -167,13 +172,13 @@ TEST(UtilTest, BLOCKINGQUEUE_TEST) { } TEST(UtilTest, LOG_TEST) { - int32_t res = server::InitLog(LOG_FILE_PATH); - ASSERT_EQ(res, 0); + auto status = server::InitLog(LOG_FILE_PATH); + ASSERT_TRUE(status.ok()); EXPECT_FALSE(el::Loggers::hasFlag(el::LoggingFlag::NewLineForContainer)); EXPECT_FALSE(el::Loggers::hasFlag(el::LoggingFlag::LogDetailedCrashReason)); - std::string fname = server::GetFileName(LOG_FILE_PATH); + std::string fname = server::CommonUtil::GetFileName(LOG_FILE_PATH); ASSERT_EQ(fname, "log_config.conf"); }