From 0c05cffa725e4a96d2777acec1cde5916607a371 Mon Sep 17 00:00:00 2001 From: yu yunfeng Date: Mon, 24 Jun 2019 19:52:31 +0800 Subject: [PATCH 001/189] comment unfixed function Former-commit-id: 4f6fa3e940e98099995d751e945a67539c6a8339 --- cpp/unittest/metrics/metrics_test.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/unittest/metrics/metrics_test.cpp b/cpp/unittest/metrics/metrics_test.cpp index 82e1d99b93..00075fbb5f 100644 --- a/cpp/unittest/metrics/metrics_test.cpp +++ b/cpp/unittest/metrics/metrics_test.cpp @@ -33,7 +33,7 @@ TEST_F(DBTest, Metric_Tes) { server::Metrics::GetInstance().Init(); // server::PrometheusMetrics::GetInstance().exposer_ptr()->RegisterCollectable(server::PrometheusMetrics::GetInstance().registry_ptr()); - zilliz::milvus::cache::CpuCacheMgr::GetInstance()->SetCapacity(2UL*1024*1024*1024); + zilliz::milvus::cache::CpuCacheMgr::GetInstance()->SetCapacity(1UL*1024*1024*1024); std::cout<CacheCapacity()< Date: Tue, 25 Jun 2019 18:42:18 +0800 Subject: [PATCH 002/189] MS-110 - Avoid huge file size Former-commit-id: 53e865a853cc15b5ffe91159ab158d37ec7cb634 --- cpp/CHANGELOG.md | 1 + cpp/src/db/DBImpl.cpp | 10 +++-- cpp/src/db/MemManager.cpp | 91 +++++++++++++++++++++++++-------------- cpp/src/db/MemManager.h | 27 ++++++------ 4 files changed, 79 insertions(+), 50 deletions(-) diff --git a/cpp/CHANGELOG.md b/cpp/CHANGELOG.md index e88a80fb33..04ca984858 100644 --- a/cpp/CHANGELOG.md +++ b/cpp/CHANGELOG.md @@ -11,6 +11,7 @@ Please mark all change in change log and use the ticket from JIRA. - MS-89 - Fix compile failed, libgpufaiss.a link missing - MS-90 - Fix arch match incorrect on ARM - MS-99 - Fix compilation bug +- MS-110 - Avoid huge file size ## Improvement - MS-82 - Update server startup welcome message diff --git a/cpp/src/db/DBImpl.cpp b/cpp/src/db/DBImpl.cpp index 279c2a5636..17280b37d3 100644 --- a/cpp/src/db/DBImpl.cpp +++ b/cpp/src/db/DBImpl.cpp @@ -479,7 +479,7 @@ void DBImpl::StartCompactionTask() { } //serialize memory data - std::vector temp_table_ids; + std::set temp_table_ids; mem_mgr_->Serialize(temp_table_ids); for(auto& id : temp_table_ids) { compact_table_ids_.insert(id); @@ -550,7 +550,8 @@ Status DBImpl::MergeFiles(const std::string& table_id, const meta::DateT& date, ENGINE_LOG_DEBUG << "New merged file " << table_file.file_id_ << " of size=" << index->PhysicalSize()/(1024*1024) << " M"; - index->Cache(); + //current disable this line to avoid memory + //index->Cache(); return status; } @@ -670,7 +671,8 @@ Status DBImpl::BuildIndex(const meta::TableFileSchema& file) { << index->PhysicalSize()/(1024*1024) << " M" << " from file " << to_remove.file_id_; - index->Cache(); + //current disable this line to avoid memory + //index->Cache(); } catch (std::exception& ex) { return Status::Error("Build index encounter exception", ex.what()); @@ -709,7 +711,7 @@ Status DBImpl::Size(uint64_t& result) { DBImpl::~DBImpl() { shutting_down_.store(true, std::memory_order_release); bg_timer_thread_.join(); - std::vector ids; + std::set ids; mem_mgr_->Serialize(ids); } diff --git a/cpp/src/db/MemManager.cpp b/cpp/src/db/MemManager.cpp index d7a2087ee0..fa7f3c54b0 100644 --- a/cpp/src/db/MemManager.cpp +++ b/cpp/src/db/MemManager.cpp @@ -20,36 +20,54 @@ namespace engine { MemVectors::MemVectors(const std::shared_ptr& meta_ptr, const meta::TableFileSchema& schema, const Options& options) - : pMeta_(meta_ptr), + : meta_(meta_ptr), options_(options), schema_(schema), - pIdGenerator_(new SimpleIDGenerator()), - pEE_(EngineFactory::Build(schema_.dimension_, schema_.location_, (EngineType)schema_.engine_type_)) { + id_generator_(new SimpleIDGenerator()), + active_engine_(EngineFactory::Build(schema_.dimension_, schema_.location_, (EngineType)schema_.engine_type_)) { } -void MemVectors::Add(size_t n_, const float* vectors_, IDNumbers& vector_ids_) { +Status MemVectors::Add(size_t n_, const float* vectors_, IDNumbers& vector_ids_) { + if(active_engine_ == nullptr) { + return Status::Error("index engine is null"); + } + auto start_time = METRICS_NOW_TIME; - pIdGenerator_->GetNextIDNumbers(n_, vector_ids_); - pEE_->AddWithIds(n_, vectors_, vector_ids_.data()); + id_generator_->GetNextIDNumbers(n_, vector_ids_); + Status status = active_engine_->AddWithIds(n_, vectors_, vector_ids_.data()); auto end_time = METRICS_NOW_TIME; auto total_time = METRICS_MICROSECONDS(start_time, end_time); server::Metrics::GetInstance().AddVectorsPerSecondGaugeSet(static_cast(n_), static_cast(schema_.dimension_), total_time); + + return status; } -size_t MemVectors::Total() const { - return pEE_->Count(); +size_t MemVectors::RowCount() const { + if(active_engine_ == nullptr) { + return 0; + } + + return active_engine_->Count(); } -size_t MemVectors::ApproximateSize() const { - return pEE_->Size(); +size_t MemVectors::Size() const { + if(active_engine_ == nullptr) { + return 0; + } + + return active_engine_->Size(); } Status MemVectors::Serialize(std::string& table_id) { + if(active_engine_ == nullptr) { + return Status::Error("index engine is null"); + } + table_id = schema_.table_id_; - auto size = ApproximateSize(); + auto size = Size(); auto start_time = METRICS_NOW_TIME; - pEE_->Serialize(); + active_engine_->Serialize(); auto end_time = METRICS_NOW_TIME; auto total_time = METRICS_MICROSECONDS(start_time, end_time); schema_.size_ = size; @@ -59,20 +77,20 @@ Status MemVectors::Serialize(std::string& table_id) { schema_.file_type_ = (size >= options_.index_trigger_size) ? meta::TableFileSchema::TO_INDEX : meta::TableFileSchema::RAW; - auto status = pMeta_->UpdateTableFile(schema_); + auto status = meta_->UpdateTableFile(schema_); LOG(DEBUG) << "New " << ((schema_.file_type_ == meta::TableFileSchema::RAW) ? "raw" : "to_index") - << " file " << schema_.file_id_ << " of size " << (double)(pEE_->Size()) / (double)meta::M << " M"; + << " file " << schema_.file_id_ << " of size " << (double)(active_engine_->Size()) / (double)meta::M << " M"; - pEE_->Cache(); + active_engine_->Cache(); return status; } MemVectors::~MemVectors() { - if (pIdGenerator_ != nullptr) { - delete pIdGenerator_; - pIdGenerator_ = nullptr; + if (id_generator_ != nullptr) { + delete id_generator_; + id_generator_ = nullptr; } } @@ -81,20 +99,20 @@ MemVectors::~MemVectors() { */ MemManager::MemVectorsPtr MemManager::GetMemByTable( const std::string& table_id) { - auto memIt = memMap_.find(table_id); - if (memIt != memMap_.end()) { + auto memIt = mem_id_map_.find(table_id); + if (memIt != mem_id_map_.end()) { return memIt->second; } meta::TableFileSchema table_file; table_file.table_id_ = table_id; - auto status = pMeta_->CreateTableFile(table_file); + auto status = meta_->CreateTableFile(table_file); if (!status.ok()) { return nullptr; } - memMap_[table_id] = MemVectorsPtr(new MemVectors(pMeta_, table_file, options_)); - return memMap_[table_id]; + mem_id_map_[table_id] = MemVectorsPtr(new MemVectors(meta_, table_file, options_)); + return mem_id_map_[table_id]; } Status MemManager::InsertVectors(const std::string& table_id_, @@ -114,37 +132,44 @@ Status MemManager::InsertVectorsNoLock(const std::string& table_id, if (mem == nullptr) { return Status::NotFound("Group " + table_id + " not found!"); } - mem->Add(n, vectors, vector_ids); - return Status::OK(); + //makesure each file size less than index_trigger_size + if(mem->Size() > options_.index_trigger_size) { + std::unique_lock lock(serialization_mtx_); + immu_mem_list_.push_back(mem); + mem_id_map_.erase(table_id); + return InsertVectorsNoLock(table_id, n, vectors, vector_ids); + } else { + return mem->Add(n, vectors, vector_ids); + } } Status MemManager::ToImmutable() { std::unique_lock lock(mutex_); - for (auto& kv: memMap_) { - immMems_.push_back(kv.second); + for (auto& kv: mem_id_map_) { + immu_mem_list_.push_back(kv.second); } - memMap_.clear(); + mem_id_map_.clear(); return Status::OK(); } -Status MemManager::Serialize(std::vector& table_ids) { +Status MemManager::Serialize(std::set& table_ids) { ToImmutable(); std::unique_lock lock(serialization_mtx_); std::string table_id; table_ids.clear(); - for (auto& mem : immMems_) { + for (auto& mem : immu_mem_list_) { mem->Serialize(table_id); - table_ids.push_back(table_id); + table_ids.insert(table_id); } - immMems_.clear(); + immu_mem_list_.clear(); return Status::OK(); } Status MemManager::EraseMemVector(const std::string& table_id) { std::unique_lock lock(mutex_); - memMap_.erase(table_id); + mem_id_map_.erase(table_id); return Status::OK(); } diff --git a/cpp/src/db/MemManager.h b/cpp/src/db/MemManager.h index 1b329f971b..2aa0183898 100644 --- a/cpp/src/db/MemManager.h +++ b/cpp/src/db/MemManager.h @@ -15,6 +15,7 @@ #include #include #include +#include namespace zilliz { namespace milvus { @@ -32,11 +33,11 @@ public: explicit MemVectors(const std::shared_ptr&, const meta::TableFileSchema&, const Options&); - void Add(size_t n_, const float* vectors_, IDNumbers& vector_ids_); + Status Add(size_t n_, const float* vectors_, IDNumbers& vector_ids_); - size_t Total() const; + size_t RowCount() const; - size_t ApproximateSize() const; + size_t Size() const; Status Serialize(std::string& table_id); @@ -49,11 +50,11 @@ private: MemVectors(const MemVectors&) = delete; MemVectors& operator=(const MemVectors&) = delete; - MetaPtr pMeta_; + MetaPtr meta_; Options options_; meta::TableFileSchema schema_; - IDGenerator* pIdGenerator_; - ExecutionEnginePtr pEE_; + IDGenerator* id_generator_; + ExecutionEnginePtr active_engine_; }; // MemVectors @@ -66,14 +67,14 @@ public: using Ptr = std::shared_ptr; MemManager(const std::shared_ptr& meta, const Options& options) - : pMeta_(meta), options_(options) {} + : meta_(meta), options_(options) {} MemVectorsPtr GetMemByTable(const std::string& table_id); Status InsertVectors(const std::string& table_id, size_t n, const float* vectors, IDNumbers& vector_ids); - Status Serialize(std::vector& table_ids); + Status Serialize(std::set& table_ids); Status EraseMemVector(const std::string& table_id); @@ -82,11 +83,11 @@ private: size_t n, const float* vectors, IDNumbers& vector_ids); Status ToImmutable(); - using MemMap = std::map; - using ImmMemPool = std::vector; - MemMap memMap_; - ImmMemPool immMems_; - MetaPtr pMeta_; + using MemIdMap = std::map; + using MemList = std::vector; + MemIdMap mem_id_map_; + MemList immu_mem_list_; + MetaPtr meta_; Options options_; std::mutex mutex_; std::mutex serialization_mtx_; From 411c29b22c9ffe365f57cb03d615cb0b1bf06dea Mon Sep 17 00:00:00 2001 From: yu yunfeng Date: Wed, 26 Jun 2019 11:21:55 +0800 Subject: [PATCH 003/189] alter server_config Former-commit-id: fe8d5604005b155cfd602d449b0e346d65fddb89 --- cpp/conf/server_config.yaml | 2 +- cpp/src/metrics/PrometheusMetrics.cpp | 3 ++- cpp/unittest/metrics/metrics_test.cpp | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/cpp/conf/server_config.yaml b/cpp/conf/server_config.yaml index 9f60f0f212..95fa6156c8 100644 --- a/cpp/conf/server_config.yaml +++ b/cpp/conf/server_config.yaml @@ -14,7 +14,7 @@ db_config: index_building_threshold: 1024 #build index file when raw data file size larger than this value, unit: MB metric_config: - is_startup: true # true is on, false is off + is_startup: off # on is activated, otherwise is down. note: case sensitive collector: prometheus # prometheus, now we only have prometheus prometheus_config: collect_type: pull # pull means prometheus pull the message from server, push means server push metric to push gateway diff --git a/cpp/src/metrics/PrometheusMetrics.cpp b/cpp/src/metrics/PrometheusMetrics.cpp index 07e1d2ee71..d0d50800ad 100644 --- a/cpp/src/metrics/PrometheusMetrics.cpp +++ b/cpp/src/metrics/PrometheusMetrics.cpp @@ -17,7 +17,8 @@ ServerError PrometheusMetrics::Init() { try { ConfigNode &configNode = ServerConfig::GetInstance().GetConfig(CONFIG_METRIC); - startup_ = configNode.GetValue(CONFIG_METRIC_IS_STARTUP) == "true" ? true : false; + startup_ = configNode.GetValue(CONFIG_METRIC_IS_STARTUP) == "on"; + if(!startup_) return SERVER_SUCCESS; // Following should be read from config file. const std::string bind_address = configNode.GetChild(CONFIG_PROMETHEUS).GetValue(CONFIG_METRIC_PROMETHEUS_PORT); const std::string uri = std::string("/metrics"); diff --git a/cpp/unittest/metrics/metrics_test.cpp b/cpp/unittest/metrics/metrics_test.cpp index 00075fbb5f..72596dc79e 100644 --- a/cpp/unittest/metrics/metrics_test.cpp +++ b/cpp/unittest/metrics/metrics_test.cpp @@ -102,7 +102,7 @@ TEST_F(DBTest, Metric_Tes) { } }); - int loop = 10; + int loop = 10000; for (auto i=0; i Date: Wed, 26 Jun 2019 09:19:12 +0800 Subject: [PATCH 004/189] remove unuse config Former-commit-id: dad8d4734bb4344d7999a74cdbef9a0eb8781daf --- cpp/conf/server_config.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/cpp/conf/server_config.yaml b/cpp/conf/server_config.yaml index 95fa6156c8..cfb7ff8cf8 100644 --- a/cpp/conf/server_config.yaml +++ b/cpp/conf/server_config.yaml @@ -1,8 +1,6 @@ server_config: address: 0.0.0.0 port: 19530 - transfer_protocol: binary #optional: binary, compact, json - server_mode: thread_pool #optional: simple, thread_pool gpu_index: 0 #which gpu to be used mode: single #optional: single, cluster From 9f23b8e2493c8e9dd654dc7eb4ffd80ef24aea77 Mon Sep 17 00:00:00 2001 From: starlord Date: Wed, 26 Jun 2019 10:13:11 +0800 Subject: [PATCH 005/189] remove unuse config Former-commit-id: 6b5468ba77c440ca2ad52c8846a0205d7f651f1b --- cpp/conf/server_config.yaml | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/cpp/conf/server_config.yaml b/cpp/conf/server_config.yaml index cfb7ff8cf8..2db841abaa 100644 --- a/cpp/conf/server_config.yaml +++ b/cpp/conf/server_config.yaml @@ -1,8 +1,8 @@ server_config: address: 0.0.0.0 - port: 19530 - gpu_index: 0 #which gpu to be used - mode: single #optional: single, cluster + port: 19530 # the port milvus listen to, default: 19530, range: 1025 ~ 65534 + gpu_index: 0 # the gpu milvus use, default: 0, range: 0 ~ gpu number - 1 + mode: single # milvus deployment type: single, cluster db_config: db_path: /tmp/milvus @@ -12,16 +12,16 @@ db_config: index_building_threshold: 1024 #build index file when raw data file size larger than this value, unit: MB metric_config: - is_startup: off # on is activated, otherwise is down. note: case sensitive - collector: prometheus # prometheus, now we only have prometheus - prometheus_config: - collect_type: pull # pull means prometheus pull the message from server, push means server push metric to push gateway - port: 8080 - push_gateway_ip_address: 127.0.0.1 - push_gateway_port: 9091 + is_startup: true # if monitoring start: on, off + collector: prometheus # metrics collector: prometheus + prometheus_config: # following are prometheus configure + collect_type: pull # prometheus collect data method + port: 8080 # the port prometheus use to fetch metrics + push_gateway_ip_address: 127.0.0.1 # push method configure: push gateway ip address + push_gateway_port: 9091 # push method configure: push gateway port -license_config: - license_path: "/tmp/system.license" +license_config: # license configure + license_path: "/tmp/system.license" # license file path -cache_config: - cpu_cache_capacity: 16 # memory pool to hold index data, unit: GB \ No newline at end of file +cache_config: # cache configure + cpu_cache_capacity: 16 # how many memory are used as cache, unit: GB, range: 0 ~ less than total memory \ No newline at end of file From 330ed5f9b32bdeb2bfe6e9192f67e812daf265e2 Mon Sep 17 00:00:00 2001 From: yu yunfeng Date: Wed, 26 Jun 2019 11:51:47 +0800 Subject: [PATCH 006/189] change CHANGELOG Former-commit-id: e76335a6ee5fadc0d72aabbed324822436b88ada --- cpp/CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/CHANGELOG.md b/cpp/CHANGELOG.md index 04ca984858..4e7a4448f2 100644 --- a/cpp/CHANGELOG.md +++ b/cpp/CHANGELOG.md @@ -20,7 +20,7 @@ Please mark all change in change log and use the ticket from JIRA. - MS-22 - Enhancement for MemVector size control - MS-92 - Unify behavior of debug and release build - MS-98 - Install all unit test to installation directory - +- MS-115 - Change is_startup of metric_config switch from true to on ## New Feature - MS-57 - Implement index load/search pipeline From 44bd91038f2eb506293ba4eb889269ce799670fe Mon Sep 17 00:00:00 2001 From: yu yunfeng Date: Wed, 26 Jun 2019 11:54:51 +0800 Subject: [PATCH 007/189] off Former-commit-id: ec106e7523c3aad142f345d9370fa8d24776def1 --- cpp/conf/server_config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/conf/server_config.yaml b/cpp/conf/server_config.yaml index 2db841abaa..dcf23d17b9 100644 --- a/cpp/conf/server_config.yaml +++ b/cpp/conf/server_config.yaml @@ -12,7 +12,7 @@ db_config: index_building_threshold: 1024 #build index file when raw data file size larger than this value, unit: MB metric_config: - is_startup: true # if monitoring start: on, off + is_startup: off # if monitoring start: on, off collector: prometheus # metrics collector: prometheus prometheus_config: # following are prometheus configure collect_type: pull # prometheus collect data method From 97296aca7a2631dd983a391002060f0cae405e55 Mon Sep 17 00:00:00 2001 From: zhiru Date: Thu, 27 Jun 2019 14:10:02 +0800 Subject: [PATCH 008/189] fix Former-commit-id: 66326961f0cff4aea03c408870de758d928a9cff --- cpp/src/db/MySQLMetaImpl.cpp | 73 +++++++++++++++++++-------------- cpp/src/server/DBWrapper.cpp | 2 +- cpp/src/server/MilvusServer.cpp | 7 ++-- cpp/src/server/RequestTask.cpp | 70 ++++++++++++++++--------------- 4 files changed, 84 insertions(+), 68 deletions(-) diff --git a/cpp/src/db/MySQLMetaImpl.cpp b/cpp/src/db/MySQLMetaImpl.cpp index c860c667c4..4384e26d82 100644 --- a/cpp/src/db/MySQLMetaImpl.cpp +++ b/cpp/src/db/MySQLMetaImpl.cpp @@ -961,6 +961,10 @@ namespace meta { // std::lock_guard lock(mysql_mutex); + if (ids.empty()) { + return Status::OK(); + } + std::stringstream idSS; for (auto& id : ids) { idSS << "id = " << std::to_string(id) << " OR "; @@ -1405,17 +1409,22 @@ namespace meta { idsToDelete.emplace_back(std::to_string(table_file.id_)); } - std::stringstream idsToDeleteSS; - for (auto &id : idsToDelete) { - idsToDeleteSS << "id = " << id << " OR "; - } - std::string idsToDeleteStr = idsToDeleteSS.str(); - idsToDeleteStr = idsToDeleteStr.substr(0, idsToDeleteStr.size() - 4); //remove the last " OR " - cleanUpFilesWithTTLQuery << "DELETE FROM TableFiles WHERE " << - idsToDeleteStr << ";"; - if (!cleanUpFilesWithTTLQuery.exec()) { - ENGINE_LOG_ERROR << "QUERY ERROR WHEN CLEANING UP FILES WITH TTL"; - return Status::DBTransactionError("CleanUpFilesWithTTL Error", cleanUpFilesWithTTLQuery.error()); + if (!idsToDelete.empty()) { + + std::stringstream idsToDeleteSS; + for (auto &id : idsToDelete) { + idsToDeleteSS << "id = " << id << " OR "; + } + + std::string idsToDeleteStr = idsToDeleteSS.str(); + idsToDeleteStr = idsToDeleteStr.substr(0, idsToDeleteStr.size() - 4); //remove the last " OR " + cleanUpFilesWithTTLQuery << "DELETE FROM TableFiles WHERE " << + idsToDeleteStr << ";"; + if (!cleanUpFilesWithTTLQuery.exec()) { + ENGINE_LOG_ERROR << "QUERY ERROR WHEN CLEANING UP FILES WITH TTL"; + return Status::DBTransactionError("CleanUpFilesWithTTL Error", + cleanUpFilesWithTTLQuery.error()); + } } } //Scoped Connection @@ -1442,29 +1451,33 @@ namespace meta { StoreQueryResult res = cleanUpFilesWithTTLQuery.store(); assert(res); // std::cout << res.num_rows() << std::endl; - std::stringstream idsToDeleteSS; - for (auto &resRow : res) { - size_t id = resRow["id"]; - std::string table_id; - resRow["table_id"].to_string(table_id); - auto table_path = GetTablePath(table_id); + if (!res.empty()) { - ENGINE_LOG_DEBUG << "Remove table folder: " << table_path; - boost::filesystem::remove_all(table_path); + std::stringstream idsToDeleteSS; + for (auto &resRow : res) { + size_t id = resRow["id"]; + std::string table_id; + resRow["table_id"].to_string(table_id); - idsToDeleteSS << "id = " << std::to_string(id) << " OR "; + auto table_path = GetTablePath(table_id); + + ENGINE_LOG_DEBUG << "Remove table folder: " << table_path; + boost::filesystem::remove_all(table_path); + + idsToDeleteSS << "id = " << std::to_string(id) << " OR "; + } + std::string idsToDeleteStr = idsToDeleteSS.str(); + idsToDeleteStr = idsToDeleteStr.substr(0, idsToDeleteStr.size() - 4); //remove the last " OR " + cleanUpFilesWithTTLQuery << "DELETE FROM Tables WHERE " << + idsToDeleteStr << ";"; + if (!cleanUpFilesWithTTLQuery.exec()) { + ENGINE_LOG_ERROR << "QUERY ERROR WHEN CLEANING UP FILES WITH TTL"; + return Status::DBTransactionError("QUERY ERROR WHEN CLEANING UP FILES WITH TTL", + cleanUpFilesWithTTLQuery.error()); + } } - std::string idsToDeleteStr = idsToDeleteSS.str(); - idsToDeleteStr = idsToDeleteStr.substr(0, idsToDeleteStr.size() - 4); //remove the last " OR " - cleanUpFilesWithTTLQuery << "DELETE FROM Tables WHERE " << - idsToDeleteStr << ";"; - if (!cleanUpFilesWithTTLQuery.exec()) { - ENGINE_LOG_ERROR << "QUERY ERROR WHEN CLEANING UP FILES WITH TTL"; - return Status::DBTransactionError("QUERY ERROR WHEN CLEANING UP FILES WITH TTL", - cleanUpFilesWithTTLQuery.error()); - } - } //Scoped Connection + } //Scoped Connection } catch (const BadQuery& er) { // Handle any query errors diff --git a/cpp/src/server/DBWrapper.cpp b/cpp/src/server/DBWrapper.cpp index 4cae31ea6b..0721de1325 100644 --- a/cpp/src/server/DBWrapper.cpp +++ b/cpp/src/server/DBWrapper.cpp @@ -28,7 +28,7 @@ DBWrapper::DBWrapper() { zilliz::milvus::engine::DB::Open(opt, &db_); if(db_ == nullptr) { - SERVER_LOG_ERROR << "Failed to open db"; + SERVER_LOG_ERROR << "Failed to open db. Provided database uri = " << opt.meta.backend_uri; throw ServerException(SERVER_NULL_POINTER, "Failed to open db"); } } diff --git a/cpp/src/server/MilvusServer.cpp b/cpp/src/server/MilvusServer.cpp index 5e42eb3d7a..1b1b85e883 100644 --- a/cpp/src/server/MilvusServer.cpp +++ b/cpp/src/server/MilvusServer.cpp @@ -8,6 +8,7 @@ #include "ServerConfig.h" #include "ThreadPoolServer.h" #include "DBWrapper.h" +#include "utils/Log.h" #include "milvus_types.h" #include "milvus_constants.h" @@ -67,7 +68,7 @@ MilvusServer::StartService() { } else if (protocol == "compact") { protocol_factory.reset(new TCompactProtocolFactory()); } else { - //SERVER_LOG_INFO << "Service protocol: " << protocol << " is not supported currently"; + SERVER_LOG_ERROR << "Service protocol: " << protocol << " is not supported currently"; return; } @@ -88,11 +89,11 @@ MilvusServer::StartService() { threadManager)); s_server->serve(); } else { - //SERVER_LOG_INFO << "Service mode: " << mode << " is not supported currently"; + SERVER_LOG_ERROR << "Service mode: " << mode << " is not supported currently"; return; } } catch (apache::thrift::TException& ex) { - //SERVER_LOG_ERROR << "Server encounter exception: " << ex.what(); + SERVER_LOG_ERROR << "Server encounter exception: " << ex.what(); } } diff --git a/cpp/src/server/RequestTask.cpp b/cpp/src/server/RequestTask.cpp index dbe4e6a740..77396d0046 100644 --- a/cpp/src/server/RequestTask.cpp +++ b/cpp/src/server/RequestTask.cpp @@ -140,7 +140,9 @@ ServerError CreateTableTask::OnExecute() { //step 1: check arguments if(schema_.table_name.empty() || schema_.dimension <= 0) { error_code_ = SERVER_INVALID_ARGUMENT; - error_msg_ = "Invalid table name or dimension"; +// error_msg_ = schema_.table_name.empty() ? + error_msg_ = "CreateTableTask: Invalid table name or dimension. table name = " + schema_.table_name + + "dimension = " + std::to_string(schema_.dimension); SERVER_LOG_ERROR << error_msg_; return error_code_; } @@ -148,7 +150,7 @@ ServerError CreateTableTask::OnExecute() { engine::EngineType engine_type = EngineType(schema_.index_type); if(engine_type == engine::EngineType::INVALID) { error_code_ = SERVER_INVALID_ARGUMENT; - error_msg_ = "Invalid index type"; + error_msg_ = "CreateTableTask: Invalid index type. type = " + std::to_string(schema_.index_type); SERVER_LOG_ERROR << error_msg_; return error_code_; } @@ -164,7 +166,7 @@ ServerError CreateTableTask::OnExecute() { engine::Status stat = DBWrapper::DB()->CreateTable(table_info); if(!stat.ok()) {//table could exist error_code_ = SERVER_UNEXPECTED_ERROR; - error_msg_ = "Engine failed: " + stat.ToString(); + error_msg_ = "CreateTableTask: Engine failed: " + stat.ToString(); SERVER_LOG_ERROR << error_msg_; return error_code_; } @@ -172,7 +174,7 @@ ServerError CreateTableTask::OnExecute() { } catch (std::exception& ex) { error_code_ = SERVER_UNEXPECTED_ERROR; error_msg_ = ex.what(); - SERVER_LOG_ERROR << error_msg_; + SERVER_LOG_ERROR << "CreateTableTask: " << error_msg_; return error_code_; } @@ -200,7 +202,7 @@ ServerError DescribeTableTask::OnExecute() { //step 1: check arguments if(table_name_.empty()) { error_code_ = SERVER_INVALID_ARGUMENT; - error_msg_ = "Table name cannot be empty"; + error_msg_ = "DescribeTableTask: Table name cannot be empty"; SERVER_LOG_ERROR << error_msg_; return error_code_; } @@ -211,7 +213,7 @@ ServerError DescribeTableTask::OnExecute() { engine::Status stat = DBWrapper::DB()->DescribeTable(table_info); if(!stat.ok()) { error_code_ = SERVER_TABLE_NOT_EXIST; - error_msg_ = "Engine failed: " + stat.ToString(); + error_msg_ = "DescribeTableTask: Engine failed: " + stat.ToString(); SERVER_LOG_ERROR << error_msg_; return error_code_; } @@ -224,7 +226,7 @@ ServerError DescribeTableTask::OnExecute() { } catch (std::exception& ex) { error_code_ = SERVER_UNEXPECTED_ERROR; error_msg_ = ex.what(); - SERVER_LOG_ERROR << error_msg_; + SERVER_LOG_ERROR << "DescribeTableTask: " << error_msg_; return SERVER_UNEXPECTED_ERROR; } @@ -251,7 +253,7 @@ ServerError DeleteTableTask::OnExecute() { //step 1: check arguments if (table_name_.empty()) { error_code_ = SERVER_INVALID_ARGUMENT; - error_msg_ = "Table name cannot be empty"; + error_msg_ = "DeleteTableTask: Table name cannot be empty"; SERVER_LOG_ERROR << error_msg_; return error_code_; } @@ -262,7 +264,7 @@ ServerError DeleteTableTask::OnExecute() { engine::Status stat = DBWrapper::DB()->DescribeTable(table_info); if(!stat.ok()) { error_code_ = SERVER_TABLE_NOT_EXIST; - error_msg_ = "Engine failed: " + stat.ToString(); + error_msg_ = "DeleteTableTask: Engine failed: " + stat.ToString(); SERVER_LOG_ERROR << error_msg_; return error_code_; } @@ -273,16 +275,16 @@ ServerError DeleteTableTask::OnExecute() { std::vector dates; stat = DBWrapper::DB()->DeleteTable(table_name_, dates); if(!stat.ok()) { - SERVER_LOG_ERROR << "Engine failed: " << stat.ToString(); + SERVER_LOG_ERROR << "DeleteTableTask: Engine failed: " << stat.ToString(); return SERVER_UNEXPECTED_ERROR; } rc.Record("deleta table"); - rc.Elapse("totally cost"); + rc.Elapse("total cost"); } catch (std::exception& ex) { error_code_ = SERVER_UNEXPECTED_ERROR; error_msg_ = ex.what(); - SERVER_LOG_ERROR << error_msg_; + SERVER_LOG_ERROR << "DeleteTableTask: " << error_msg_; return error_code_; } @@ -305,7 +307,7 @@ ServerError ShowTablesTask::OnExecute() { engine::Status stat = DBWrapper::DB()->AllTables(schema_array); if(!stat.ok()) { error_code_ = SERVER_UNEXPECTED_ERROR; - error_msg_ = "Engine failed: " + stat.ToString(); + error_msg_ = "ShowTablesTask: Engine failed: " + stat.ToString(); SERVER_LOG_ERROR << error_msg_; return error_code_; } @@ -342,14 +344,14 @@ ServerError AddVectorTask::OnExecute() { //step 1: check arguments if (table_name_.empty()) { error_code_ = SERVER_INVALID_ARGUMENT; - error_msg_ = "Table name cannot be empty"; + error_msg_ = "AddVectorTask: Table name cannot be empty"; SERVER_LOG_ERROR << error_msg_; return error_code_; } if(record_array_.empty()) { error_code_ = SERVER_INVALID_ARGUMENT; - error_msg_ = "Row record array is empty"; + error_msg_ = "AddVectorTask: Row record array is empty"; SERVER_LOG_ERROR << error_msg_; return error_code_; } @@ -360,7 +362,7 @@ ServerError AddVectorTask::OnExecute() { engine::Status stat = DBWrapper::DB()->DescribeTable(table_info); if(!stat.ok()) { error_code_ = SERVER_TABLE_NOT_EXIST; - error_msg_ = "Engine failed: " + stat.ToString(); + error_msg_ = "AddVectorTask: Engine failed when DescribeTable: " + stat.ToString(); SERVER_LOG_ERROR << error_msg_; return error_code_; } @@ -371,7 +373,7 @@ ServerError AddVectorTask::OnExecute() { std::vector vec_f; error_code_ = ConvertRowRecordToFloatArray(record_array_, table_info.dimension_, vec_f); if(error_code_ != SERVER_SUCCESS) { - error_msg_ = "Invalid row record data"; + error_msg_ = "AddVectorTask when ConvertRowRecordToFloatArray: Invalid row record data"; return error_code_; } @@ -383,23 +385,23 @@ ServerError AddVectorTask::OnExecute() { rc.Record("add vectors to engine"); if(!stat.ok()) { error_code_ = SERVER_UNEXPECTED_ERROR; - error_msg_ = "Engine failed: " + stat.ToString(); + error_msg_ = "AddVectorTask: Engine failed when InsertVectors: " + stat.ToString(); SERVER_LOG_ERROR << error_msg_; return error_code_; } if(record_ids_.size() != vec_count) { - SERVER_LOG_ERROR << "Vector ID not returned"; + SERVER_LOG_ERROR << "AddVectorTask: Vector ID not returned"; return SERVER_UNEXPECTED_ERROR; } rc.Record("do insert"); - rc.Elapse("totally cost"); + rc.Elapse("total cost"); } catch (std::exception& ex) { error_code_ = SERVER_UNEXPECTED_ERROR; error_msg_ = ex.what(); - SERVER_LOG_ERROR << error_msg_; + SERVER_LOG_ERROR << "AddVectorTask: " << error_msg_; return error_code_; } @@ -440,14 +442,14 @@ ServerError SearchVectorTask::OnExecute() { //step 1: check arguments if (table_name_.empty()) { error_code_ = SERVER_INVALID_ARGUMENT; - error_msg_ = "Table name cannot be empty"; + error_msg_ = "SearchVectorTask: Table name cannot be empty"; SERVER_LOG_ERROR << error_msg_; return error_code_; } if(top_k_ <= 0 || record_array_.empty()) { error_code_ = SERVER_INVALID_ARGUMENT; - error_msg_ = "Invalid topk value, or query record array is empty"; + error_msg_ = "SearchVectorTask: Invalid topk value, or query record array is empty"; SERVER_LOG_ERROR << error_msg_; return error_code_; } @@ -458,7 +460,7 @@ ServerError SearchVectorTask::OnExecute() { engine::Status stat = DBWrapper::DB()->DescribeTable(table_info); if(!stat.ok()) { error_code_ = SERVER_TABLE_NOT_EXIST; - error_msg_ = "Engine failed: " + stat.ToString(); + error_msg_ = "SearchVectorTask: Engine failed when DescribeTable: " + stat.ToString(); SERVER_LOG_ERROR << error_msg_; return error_code_; } @@ -467,7 +469,7 @@ ServerError SearchVectorTask::OnExecute() { std::vector dates; error_code_ = ConvertTimeRangeToDBDates(range_array_, dates); if(error_code_ != SERVER_SUCCESS) { - error_msg_ = "Invalid query range"; + error_msg_ = "SearchVectorTask: Invalid query range when ConvertTimeRangeToDBDates"; return error_code_; } @@ -477,7 +479,7 @@ ServerError SearchVectorTask::OnExecute() { std::vector vec_f; error_code_ = ConvertRowRecordToFloatArray(record_array_, table_info.dimension_, vec_f); if(error_code_ != SERVER_SUCCESS) { - error_msg_ = "Invalid row record data"; + error_msg_ = "Invalid row record data when ConvertRowRecordToFloatArray"; return error_code_; } @@ -495,12 +497,12 @@ ServerError SearchVectorTask::OnExecute() { rc.Record("search vectors from engine"); if(!stat.ok()) { - SERVER_LOG_ERROR << "Engine failed: " << stat.ToString(); + SERVER_LOG_ERROR << "SearchVectorTask: Engine failed: " << stat.ToString(); return SERVER_UNEXPECTED_ERROR; } if(results.size() != record_count) { - SERVER_LOG_ERROR << "Search result not returned"; + SERVER_LOG_ERROR << "SearchVectorTask: Search result not returned"; return SERVER_UNEXPECTED_ERROR; } @@ -523,12 +525,12 @@ ServerError SearchVectorTask::OnExecute() { result_array_.emplace_back(thrift_topk_result); } rc.Record("construct result"); - rc.Elapse("totally cost"); + rc.Elapse("total cost"); } catch (std::exception& ex) { error_code_ = SERVER_UNEXPECTED_ERROR; error_msg_ = ex.what(); - SERVER_LOG_ERROR << error_msg_; + SERVER_LOG_ERROR << "SearchVectorTask: " << error_msg_; return error_code_; } @@ -554,7 +556,7 @@ ServerError GetTableRowCountTask::OnExecute() { //step 1: check arguments if (table_name_.empty()) { error_code_ = SERVER_INVALID_ARGUMENT; - error_msg_ = "Table name cannot be empty"; + error_msg_ = "GetTableRowCountTask: Table name cannot be empty"; SERVER_LOG_ERROR << error_msg_; return error_code_; } @@ -564,19 +566,19 @@ ServerError GetTableRowCountTask::OnExecute() { engine::Status stat = DBWrapper::DB()->GetTableRowCount(table_name_, row_count); if (!stat.ok()) { error_code_ = SERVER_UNEXPECTED_ERROR; - error_msg_ = "Engine failed: " + stat.ToString(); + error_msg_ = "GetTableRowCountTask: Engine failed: " + stat.ToString(); SERVER_LOG_ERROR << error_msg_; return error_code_; } row_count_ = (int64_t) row_count; - rc.Elapse("totally cost"); + rc.Elapse("total cost"); } catch (std::exception& ex) { error_code_ = SERVER_UNEXPECTED_ERROR; error_msg_ = ex.what(); - SERVER_LOG_ERROR << error_msg_; + SERVER_LOG_ERROR << "GetTableRowCountTask: " << error_msg_; return error_code_; } From bc182c8cc3a438398ccea0125ce5da79e65d9a0d Mon Sep 17 00:00:00 2001 From: zhiru Date: Thu, 27 Jun 2019 14:52:21 +0800 Subject: [PATCH 009/189] update Former-commit-id: c77a5add40c22104ea65dff1dcecfac0066bebd3 --- cpp/src/db/DBImpl.cpp | 8 +++++++- cpp/src/db/Factories.cpp | 2 ++ cpp/src/db/Options.h | 1 + cpp/src/server/DBWrapper.cpp | 1 + cpp/src/server/ServerConfig.h | 1 + 5 files changed, 12 insertions(+), 1 deletion(-) diff --git a/cpp/src/db/DBImpl.cpp b/cpp/src/db/DBImpl.cpp index 17280b37d3..e39942d35d 100644 --- a/cpp/src/db/DBImpl.cpp +++ b/cpp/src/db/DBImpl.cpp @@ -13,6 +13,7 @@ #include "scheduler/context/SearchContext.h" #include "scheduler/context/DeleteContext.h" #include "utils/TimeRecorder.h" +#include "MetaConsts.h" #include #include @@ -595,7 +596,12 @@ void DBImpl::BackgroundCompaction(std::set table_ids) { } meta_ptr_->Archive(); - meta_ptr_->CleanUpFilesWithTTL(1); + + int ttl = 1; + if (options_.mode == "cluster") { + ttl = meta::D_SEC; + } + meta_ptr_->CleanUpFilesWithTTL(ttl); } void DBImpl::StartBuildIndexTask() { diff --git a/cpp/src/db/Factories.cpp b/cpp/src/db/Factories.cpp index fd005d1ff5..a6998d10dc 100644 --- a/cpp/src/db/Factories.cpp +++ b/cpp/src/db/Factories.cpp @@ -81,9 +81,11 @@ std::shared_ptr DBMetaImplFactory::Build(const DBMetaOptions& metaOp std::string dialect = pieces_match[1].str(); std::transform(dialect.begin(), dialect.end(), dialect.begin(), ::tolower); if (dialect.find("mysql") != std::string::npos) { + ENGINE_LOG_DEBUG << "Using MySQL"; return std::make_shared(meta::MySQLMetaImpl(metaOptions)); } else if (dialect.find("sqlite") != std::string::npos) { + ENGINE_LOG_DEBUG << "Using SQLite"; return std::make_shared(meta::DBMetaImpl(metaOptions)); } else { diff --git a/cpp/src/db/Options.h b/cpp/src/db/Options.h index ddb7d1bff5..039134ebaa 100644 --- a/cpp/src/db/Options.h +++ b/cpp/src/db/Options.h @@ -47,6 +47,7 @@ struct Options { uint16_t merge_trigger_number = 2; size_t index_trigger_size = ONE_GB; //unit: byte DBMetaOptions meta; + std::string mode; }; // Options diff --git a/cpp/src/server/DBWrapper.cpp b/cpp/src/server/DBWrapper.cpp index 0721de1325..c38c5d8843 100644 --- a/cpp/src/server/DBWrapper.cpp +++ b/cpp/src/server/DBWrapper.cpp @@ -23,6 +23,7 @@ DBWrapper::DBWrapper() { if(index_size > 0) {//ensure larger than zero, unit is MB opt.index_trigger_size = (size_t)index_size * engine::ONE_MB; } + opt.mode = config.GetValue(CONFIG_CLUSTER_MODE, "single"); CommonUtil::CreateDirectory(opt.meta.path); diff --git a/cpp/src/server/ServerConfig.h b/cpp/src/server/ServerConfig.h index dd7c9d2966..aa77b081d7 100644 --- a/cpp/src/server/ServerConfig.h +++ b/cpp/src/server/ServerConfig.h @@ -19,6 +19,7 @@ static const std::string CONFIG_SERVER_ADDRESS = "address"; static const std::string CONFIG_SERVER_PORT = "port"; static const std::string CONFIG_SERVER_PROTOCOL = "transfer_protocol"; static const std::string CONFIG_SERVER_MODE = "server_mode"; +static const std::string CONFIG_CLUSTER_MODE = "mode"; static const std::string CONFIG_DB = "db_config"; static const std::string CONFIG_DB_URL = "db_backend_url"; From 9a230c6f2f072a36c552634a833f9334174342ce Mon Sep 17 00:00:00 2001 From: zhiru Date: Thu, 27 Jun 2019 15:52:22 +0800 Subject: [PATCH 010/189] update Former-commit-id: bd64e84da45739f21df32dcb88249016337dabc0 --- cpp/src/db/DBImpl.cpp | 1 + cpp/src/db/MySQLConnectionPool.h | 2 +- cpp/src/db/MySQLMetaImpl.cpp | 5 +++-- cpp/src/server/DBWrapper.cpp | 4 +++- 4 files changed, 8 insertions(+), 4 deletions(-) diff --git a/cpp/src/db/DBImpl.cpp b/cpp/src/db/DBImpl.cpp index e39942d35d..c7ee986c96 100644 --- a/cpp/src/db/DBImpl.cpp +++ b/cpp/src/db/DBImpl.cpp @@ -600,6 +600,7 @@ void DBImpl::BackgroundCompaction(std::set table_ids) { int ttl = 1; if (options_.mode == "cluster") { ttl = meta::D_SEC; + ENGINE_LOG_DEBUG << "Server mode is cluster. Clean up files with ttl = " << std::to_string(ttl) << "seconds."; } meta_ptr_->CleanUpFilesWithTTL(ttl); } diff --git a/cpp/src/db/MySQLConnectionPool.h b/cpp/src/db/MySQLConnectionPool.h index 63ae65829d..ade150a6cb 100644 --- a/cpp/src/db/MySQLConnectionPool.h +++ b/cpp/src/db/MySQLConnectionPool.h @@ -56,7 +56,7 @@ public: // ENGINE_LOG_DEBUG << "conns_in_use_ in release: " << conns_in_use_ << std::endl; --conns_in_use_; if (conns_in_use_ < 0) { - ENGINE_LOG_ERROR << "conns_in_use_ in release less than zero: " << conns_in_use_ << std::endl; + ENGINE_LOG_WARNING << "MySQLConnetionPool::release: conns_in_use_ is less than zero. conns_in_use_ = " << conns_in_use_ << std::endl; } } diff --git a/cpp/src/db/MySQLMetaImpl.cpp b/cpp/src/db/MySQLMetaImpl.cpp index 4384e26d82..5c0725ef93 100644 --- a/cpp/src/db/MySQLMetaImpl.cpp +++ b/cpp/src/db/MySQLMetaImpl.cpp @@ -448,9 +448,10 @@ namespace meta { Query deleteTableFilesQuery = connectionPtr->query(); // deleteTableFilesQuery << "UPDATE TableFiles " << - "SET state = " << std::to_string(TableSchema::TO_DELETE) << ", " << + "SET file_type = " << std::to_string(TableSchema::TO_DELETE) << ", " << "updated_time = " << std::to_string(utils::GetMicroSecTimeStamp()) << " " << - "WHERE table_id = " << quote << table_id << ";"; + "WHERE table_id = " << quote << table_id << " AND " << + "file_type <> " << std::to_string(TableSchema::TO_DELETE) << ";"; if (!deleteTableFilesQuery.exec()) { ENGINE_LOG_ERROR << "QUERY ERROR WHEN DELETING TABLE FILES"; diff --git a/cpp/src/server/DBWrapper.cpp b/cpp/src/server/DBWrapper.cpp index c38c5d8843..6ad308a8d6 100644 --- a/cpp/src/server/DBWrapper.cpp +++ b/cpp/src/server/DBWrapper.cpp @@ -23,7 +23,9 @@ DBWrapper::DBWrapper() { if(index_size > 0) {//ensure larger than zero, unit is MB opt.index_trigger_size = (size_t)index_size * engine::ONE_MB; } - opt.mode = config.GetValue(CONFIG_CLUSTER_MODE, "single"); + ConfigNode& serverConfig = ServerConfig::GetInstance().GetConfig(CONFIG_SERVER); + opt.mode = serverConfig.GetValue(CONFIG_CLUSTER_MODE, "single"); +// std::cout << "mode = " << opt.mode << std::endl; CommonUtil::CreateDirectory(opt.meta.path); From de751bd680a4ffa6ade2cc3b4eadd9f62d8ebe2d Mon Sep 17 00:00:00 2001 From: zhiru Date: Thu, 27 Jun 2019 16:54:16 +0800 Subject: [PATCH 011/189] temporarily disable decrementing conns_in_use_ when it's already <= 0 Former-commit-id: d2c874147a4b0ae529a5b3b44c9186566ac2d174 --- cpp/conf/log_config.conf | 4 ++-- cpp/conf/server_config.yaml | 9 +++++---- cpp/src/db/MySQLConnectionPool.h | 6 ++++-- 3 files changed, 11 insertions(+), 8 deletions(-) diff --git a/cpp/conf/log_config.conf b/cpp/conf/log_config.conf index 80710b570e..79a3965719 100644 --- a/cpp/conf/log_config.conf +++ b/cpp/conf/log_config.conf @@ -20,8 +20,8 @@ TO_STANDARD_OUTPUT = false ## Error logs * ERROR: - ENABLED = false + ENABLED = true FILENAME = "/tmp/milvus/logs/milvus-%datetime{%H:%m}-error.log" * FATAL: - ENABLED = false + ENABLED = true FILENAME = "/tmp/milvus/logs/milvus-%datetime{%H:%m}-fatal.log" \ No newline at end of file diff --git a/cpp/conf/server_config.yaml b/cpp/conf/server_config.yaml index dcf23d17b9..c918c74908 100644 --- a/cpp/conf/server_config.yaml +++ b/cpp/conf/server_config.yaml @@ -1,15 +1,16 @@ server_config: address: 0.0.0.0 - port: 19530 # the port milvus listen to, default: 19530, range: 1025 ~ 65534 + port: 19531 # the port milvus listen to, default: 19530, range: 1025 ~ 65534 gpu_index: 0 # the gpu milvus use, default: 0, range: 0 ~ gpu number - 1 - mode: single # milvus deployment type: single, cluster + mode: cluster # milvus deployment type: single, cluster db_config: db_path: /tmp/milvus #URI format: dialect://username:password@host:port/database #All parts except dialect are optional, but you MUST include the delimiters - db_backend_url: mysql://root:1234@:/test - index_building_threshold: 1024 #build index file when raw data file size larger than this value, unit: MB + #Currently supports mysql or sqlite + db_backend_url: mysql://root:1234@:/test # meta database uri + index_building_threshold: 1024 # index building trigger threshold, default: 1024, unit: MB metric_config: is_startup: off # if monitoring start: on, off diff --git a/cpp/src/db/MySQLConnectionPool.h b/cpp/src/db/MySQLConnectionPool.h index ade150a6cb..c1ea2e83bc 100644 --- a/cpp/src/db/MySQLConnectionPool.h +++ b/cpp/src/db/MySQLConnectionPool.h @@ -54,10 +54,12 @@ public: void release(const mysqlpp::Connection* pc) override { mysqlpp::ConnectionPool::release(pc); // ENGINE_LOG_DEBUG << "conns_in_use_ in release: " << conns_in_use_ << std::endl; - --conns_in_use_; - if (conns_in_use_ < 0) { + if (conns_in_use_ <= 0) { ENGINE_LOG_WARNING << "MySQLConnetionPool::release: conns_in_use_ is less than zero. conns_in_use_ = " << conns_in_use_ << std::endl; } + else { + --conns_in_use_; + } } void set_max_idle_time(int max_idle) { From e8c4390327dc415ba575477744daafb3e7b082e3 Mon Sep 17 00:00:00 2001 From: starlord Date: Thu, 27 Jun 2019 11:39:27 +0800 Subject: [PATCH 012/189] archive config Former-commit-id: c54cb683d0eae72f9eaa82c6159dc6bb411d6edf --- cpp/conf/server_config.yaml | 2 ++ cpp/src/db/DBMetaImpl.cpp | 6 +++--- cpp/src/db/Options.cpp | 6 ++++++ cpp/src/db/Options.h | 8 +++++++- cpp/src/server/DBWrapper.cpp | 13 +++++++++++++ cpp/src/server/ServerConfig.h | 2 ++ 6 files changed, 33 insertions(+), 4 deletions(-) diff --git a/cpp/conf/server_config.yaml b/cpp/conf/server_config.yaml index c918c74908..b59fa78b67 100644 --- a/cpp/conf/server_config.yaml +++ b/cpp/conf/server_config.yaml @@ -11,6 +11,8 @@ db_config: #Currently supports mysql or sqlite db_backend_url: mysql://root:1234@:/test # meta database uri index_building_threshold: 1024 # index building trigger threshold, default: 1024, unit: MB + archive_disk_threshold: 512 # triger archive action if storage size exceed this value, unit: GB + archive_days_threshold: 30 # files older than x days will be archived, unit: day metric_config: is_startup: off # if monitoring start: on, off diff --git a/cpp/src/db/DBMetaImpl.cpp b/cpp/src/db/DBMetaImpl.cpp index 6549a1740a..9a44820c7a 100644 --- a/cpp/src/db/DBMetaImpl.cpp +++ b/cpp/src/db/DBMetaImpl.cpp @@ -656,7 +656,7 @@ Status DBMetaImpl::Archive() { for (auto kv : criterias) { auto &criteria = kv.first; auto &limit = kv.second; - if (criteria == "days") { + if (criteria == engine::ARCHIVE_CONF_DAYS) { long usecs = limit * D_SEC * US_PS; long now = utils::GetMicroSecTimeStamp(); try { @@ -672,11 +672,11 @@ Status DBMetaImpl::Archive() { return HandleException("Encounter exception when update table files", e); } } - if (criteria == "disk") { + if (criteria == engine::ARCHIVE_CONF_DISK) { uint64_t sum = 0; Size(sum); - auto to_delete = (sum - limit * G); + int64_t to_delete = (int64_t)sum - limit * G; DiscardFiles(to_delete); } } diff --git a/cpp/src/db/Options.cpp b/cpp/src/db/Options.cpp index ff6404751f..5f591dcb0f 100644 --- a/cpp/src/db/Options.cpp +++ b/cpp/src/db/Options.cpp @@ -24,6 +24,12 @@ ArchiveConf::ArchiveConf(const std::string& type, const std::string& criterias) ParseCritirias(criterias); } +void ArchiveConf::SetCriterias(const ArchiveConf::CriteriaT& criterial) { + for(auto& pair : criterial) { + criterias_[pair.first] = pair.second; + } +} + void ArchiveConf::ParseCritirias(const std::string& criterias) { std::stringstream ss(criterias); std::vector tokens; diff --git a/cpp/src/db/Options.h b/cpp/src/db/Options.h index 039134ebaa..dbe80f8d5f 100644 --- a/cpp/src/db/Options.h +++ b/cpp/src/db/Options.h @@ -19,14 +19,20 @@ static constexpr uint64_t ONE_KB = 1024; static constexpr uint64_t ONE_MB = ONE_KB*ONE_KB; static constexpr uint64_t ONE_GB = ONE_KB*ONE_MB; +static const std::string ARCHIVE_CONF_DISK = "disk"; +static const std::string ARCHIVE_CONF_DAYS = "days"; +static const std::string ARCHIVE_CONF_DEFAULT = ARCHIVE_CONF_DISK + ":512"; + struct ArchiveConf { using CriteriaT = std::map; - ArchiveConf(const std::string& type, const std::string& criterias = "disk:512"); + ArchiveConf(const std::string& type, const std::string& criterias = ARCHIVE_CONF_DEFAULT); const std::string& GetType() const { return type_; } const CriteriaT GetCriterias() const { return criterias_; } + void SetCriterias(const ArchiveConf::CriteriaT& criterial); + private: void ParseCritirias(const std::string& type); void ParseType(const std::string& criterias); diff --git a/cpp/src/server/DBWrapper.cpp b/cpp/src/server/DBWrapper.cpp index 6ad308a8d6..c36bf2621e 100644 --- a/cpp/src/server/DBWrapper.cpp +++ b/cpp/src/server/DBWrapper.cpp @@ -27,6 +27,19 @@ DBWrapper::DBWrapper() { opt.mode = serverConfig.GetValue(CONFIG_CLUSTER_MODE, "single"); // std::cout << "mode = " << opt.mode << std::endl; + //set archive config + engine::ArchiveConf::CriteriaT criterial; + int64_t disk = config.GetInt64Value(CONFIG_DB_ARCHIVE_DISK, 0); + int64_t days = config.GetInt64Value(CONFIG_DB_ARCHIVE_DAYS, 0); + if(disk > 0) { + criterial[engine::ARCHIVE_CONF_DISK] = disk; + } + if(days > 0) { + criterial[engine::ARCHIVE_CONF_DAYS] = days; + } + opt.meta.archive_conf.SetCriterias(criterial); + + //create db root folder CommonUtil::CreateDirectory(opt.meta.path); zilliz::milvus::engine::DB::Open(opt, &db_); diff --git a/cpp/src/server/ServerConfig.h b/cpp/src/server/ServerConfig.h index aa77b081d7..f337275a46 100644 --- a/cpp/src/server/ServerConfig.h +++ b/cpp/src/server/ServerConfig.h @@ -25,6 +25,8 @@ static const std::string CONFIG_DB = "db_config"; static const std::string CONFIG_DB_URL = "db_backend_url"; static const std::string CONFIG_DB_PATH = "db_path"; static const std::string CONFIG_DB_INDEX_TRIGGER_SIZE = "index_building_threshold"; +static const std::string CONFIG_DB_ARCHIVE_DISK = "archive_disk_threshold"; +static const std::string CONFIG_DB_ARCHIVE_DAYS = "archive_days_threshold"; static const std::string CONFIG_LOG = "log_config"; From 762520b1b2a68e13793fcc8b73dcc1cdd6a48633 Mon Sep 17 00:00:00 2001 From: starlord Date: Thu, 27 Jun 2019 11:56:43 +0800 Subject: [PATCH 013/189] archive config Former-commit-id: d0ee5e22c4889364b10dce305ae3eeb1b9171d11 --- cpp/CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cpp/CHANGELOG.md b/cpp/CHANGELOG.md index 4e7a4448f2..a37edff271 100644 --- a/cpp/CHANGELOG.md +++ b/cpp/CHANGELOG.md @@ -21,6 +21,8 @@ Please mark all change in change log and use the ticket from JIRA. - MS-92 - Unify behavior of debug and release build - MS-98 - Install all unit test to installation directory - MS-115 - Change is_startup of metric_config switch from true to on +- MS-122 - Archive criteria config + ## New Feature - MS-57 - Implement index load/search pipeline From 0fea89ba18d8417bc57dd504ce6ea95fe52c4160 Mon Sep 17 00:00:00 2001 From: starlord Date: Thu, 27 Jun 2019 14:45:53 +0800 Subject: [PATCH 014/189] fix portential bug Former-commit-id: e35df7be95da3495c8ec1bfb1cddffc50212c211 --- cpp/src/db/MemManager.cpp | 24 +++++++++++++++++++++--- cpp/src/db/MemManager.h | 2 ++ 2 files changed, 23 insertions(+), 3 deletions(-) diff --git a/cpp/src/db/MemManager.cpp b/cpp/src/db/MemManager.cpp index fa7f3c54b0..e36b0c45ba 100644 --- a/cpp/src/db/MemManager.cpp +++ b/cpp/src/db/MemManager.cpp @@ -146,11 +146,16 @@ Status MemManager::InsertVectorsNoLock(const std::string& table_id, Status MemManager::ToImmutable() { std::unique_lock lock(mutex_); + MemIdMap temp_map; for (auto& kv: mem_id_map_) { + if(kv.second->RowCount() == 0) { + temp_map.insert(kv); + continue;//empty vector, no need to serialize + } immu_mem_list_.push_back(kv.second); } - mem_id_map_.clear(); + mem_id_map_.swap(temp_map); return Status::OK(); } @@ -168,8 +173,21 @@ Status MemManager::Serialize(std::set& table_ids) { } Status MemManager::EraseMemVector(const std::string& table_id) { - std::unique_lock lock(mutex_); - mem_id_map_.erase(table_id); + {//erase MemVector from rapid-insert cache + std::unique_lock lock(mutex_); + mem_id_map_.erase(table_id); + } + + {//erase MemVector from serialize cache + std::unique_lock lock(serialization_mtx_); + MemList temp_list; + for (auto& mem : immu_mem_list_) { + if(mem->TableId() != table_id) { + temp_list.push_back(mem); + } + } + immu_mem_list_.swap(temp_list); + } return Status::OK(); } diff --git a/cpp/src/db/MemManager.h b/cpp/src/db/MemManager.h index 2aa0183898..0ce88d504d 100644 --- a/cpp/src/db/MemManager.h +++ b/cpp/src/db/MemManager.h @@ -45,6 +45,8 @@ public: const std::string& Location() const { return schema_.location_; } + std::string TableId() const { return schema_.table_id_; } + private: MemVectors() = delete; MemVectors(const MemVectors&) = delete; From af094c3b307597e50a8076671bb6332a8c979c3e Mon Sep 17 00:00:00 2001 From: starlord Date: Thu, 27 Jun 2019 15:51:25 +0800 Subject: [PATCH 015/189] MS-124 HasTable interface Former-commit-id: 0edfe9be5b105d4423eaa7d0186307d90a16dfc9 --- cpp/CHANGELOG.md | 1 + .../sdk/examples/simple/src/ClientTest.cpp | 5 + cpp/src/sdk/include/MilvusApi.h | 12 + cpp/src/sdk/src/client/ClientProxy.cpp | 9 + cpp/src/sdk/src/client/ClientProxy.h | 2 + cpp/src/sdk/src/interface/ConnectionImpl.cpp | 5 + cpp/src/sdk/src/interface/ConnectionImpl.h | 2 + cpp/src/server/RequestHandler.cpp | 9 + cpp/src/server/RequestHandler.h | 32 +- cpp/src/server/RequestTask.cpp | 44 +- cpp/src/server/RequestTask.h | 16 + cpp/src/thrift/gen-cpp/MilvusService.cpp | 413 ++++++++++++++++++ cpp/src/thrift/gen-cpp/MilvusService.h | 145 ++++++ .../gen-cpp/MilvusService_server.skeleton.cpp | 15 + cpp/src/thrift/milvus.thrift | 10 + 15 files changed, 707 insertions(+), 13 deletions(-) diff --git a/cpp/CHANGELOG.md b/cpp/CHANGELOG.md index a37edff271..1245b7efd5 100644 --- a/cpp/CHANGELOG.md +++ b/cpp/CHANGELOG.md @@ -22,6 +22,7 @@ Please mark all change in change log and use the ticket from JIRA. - MS-98 - Install all unit test to installation directory - MS-115 - Change is_startup of metric_config switch from true to on - MS-122 - Archive criteria config +- MS-124 - HasTable interface ## New Feature diff --git a/cpp/src/sdk/examples/simple/src/ClientTest.cpp b/cpp/src/sdk/examples/simple/src/ClientTest.cpp index 78145446a6..19c764fd0a 100644 --- a/cpp/src/sdk/examples/simple/src/ClientTest.cpp +++ b/cpp/src/sdk/examples/simple/src/ClientTest.cpp @@ -165,6 +165,11 @@ ClientTest::Test(const std::string& address, const std::string& port) { Status stat = conn->CreateTable(tb_schema); std::cout << "CreateTable function call status: " << stat.ToString() << std::endl; PrintTableSchema(tb_schema); + + bool has_table = conn->HasTable(tb_schema.table_name); + if(has_table) { + std::cout << "Table is created" << std::endl; + } } {//describe table diff --git a/cpp/src/sdk/include/MilvusApi.h b/cpp/src/sdk/include/MilvusApi.h index 2f4532b761..302871c48b 100644 --- a/cpp/src/sdk/include/MilvusApi.h +++ b/cpp/src/sdk/include/MilvusApi.h @@ -156,6 +156,18 @@ public: virtual Status CreateTable(const TableSchema ¶m) = 0; + /** + * @brief Test table existence method + * + * This method is used to create table + * + * @param table_name, table name is going to be tested. + * + * @return Indicate if table is cexist + */ + virtual bool HasTable(const std::string &table_name) = 0; + + /** * @brief Delete table method * diff --git a/cpp/src/sdk/src/client/ClientProxy.cpp b/cpp/src/sdk/src/client/ClientProxy.cpp index 6f68344fac..1185e4988c 100644 --- a/cpp/src/sdk/src/client/ClientProxy.cpp +++ b/cpp/src/sdk/src/client/ClientProxy.cpp @@ -102,6 +102,15 @@ ClientProxy::CreateTable(const TableSchema ¶m) { return Status::OK(); } +bool +ClientProxy::HasTable(const std::string &table_name) { + if(!IsConnected()) { + return false; + } + + return ClientPtr()->interface()->HasTable(table_name); +} + Status ClientProxy::DeleteTable(const std::string &table_name) { if(!IsConnected()) { diff --git a/cpp/src/sdk/src/client/ClientProxy.h b/cpp/src/sdk/src/client/ClientProxy.h index 601bcddd8a..a2ede77c40 100644 --- a/cpp/src/sdk/src/client/ClientProxy.h +++ b/cpp/src/sdk/src/client/ClientProxy.h @@ -23,6 +23,8 @@ public: virtual Status CreateTable(const TableSchema ¶m) override; + virtual bool HasTable(const std::string &table_name) override; + virtual Status DeleteTable(const std::string &table_name) override; virtual Status AddVector(const std::string &table_name, diff --git a/cpp/src/sdk/src/interface/ConnectionImpl.cpp b/cpp/src/sdk/src/interface/ConnectionImpl.cpp index e303cf0aa0..efb5f61b1b 100644 --- a/cpp/src/sdk/src/interface/ConnectionImpl.cpp +++ b/cpp/src/sdk/src/interface/ConnectionImpl.cpp @@ -56,6 +56,11 @@ ConnectionImpl::CreateTable(const TableSchema ¶m) { return client_proxy_->CreateTable(param); } +bool +ConnectionImpl::HasTable(const std::string &table_name) { + return client_proxy_->HasTable(table_name); +} + Status ConnectionImpl::DeleteTable(const std::string &table_name) { return client_proxy_->DeleteTable(table_name); diff --git a/cpp/src/sdk/src/interface/ConnectionImpl.h b/cpp/src/sdk/src/interface/ConnectionImpl.h index 61e11c9390..0f9cd14e39 100644 --- a/cpp/src/sdk/src/interface/ConnectionImpl.h +++ b/cpp/src/sdk/src/interface/ConnectionImpl.h @@ -25,6 +25,8 @@ public: virtual Status CreateTable(const TableSchema ¶m) override; + virtual bool HasTable(const std::string &table_name) override; + virtual Status DeleteTable(const std::string &table_name) override; virtual Status AddVector(const std::string &table_name, diff --git a/cpp/src/server/RequestHandler.cpp b/cpp/src/server/RequestHandler.cpp index 62ce0711b4..037f80e0db 100644 --- a/cpp/src/server/RequestHandler.cpp +++ b/cpp/src/server/RequestHandler.cpp @@ -24,6 +24,15 @@ RequestHandler::CreateTable(const thrift::TableSchema ¶m) { RequestScheduler::ExecTask(task_ptr); } +bool +RequestHandler::HasTable(const std::string &table_name) { + bool has_table = false; + BaseTaskPtr task_ptr = HasTableTask::Create(table_name, has_table); + RequestScheduler::ExecTask(task_ptr); + + return has_table; +} + void RequestHandler::DeleteTable(const std::string &table_name) { BaseTaskPtr task_ptr = DeleteTableTask::Create(table_name); diff --git a/cpp/src/server/RequestHandler.h b/cpp/src/server/RequestHandler.h index 8f3eca576b..e736b7593f 100644 --- a/cpp/src/server/RequestHandler.h +++ b/cpp/src/server/RequestHandler.h @@ -19,16 +19,28 @@ public: RequestHandler(); /** - * @brief Create table method - * - * This method is used to create table - * - * @param param, use to provide table information to be created. - * - * - * @param param - */ - void CreateTable(const ::milvus::thrift::TableSchema& param); + * @brief Create table method + * + * This method is used to create table + * + * @param param, use to provide table information to be created. + * + * + * @param param + */ + void CreateTable(const ::milvus::thrift::TableSchema ¶m); + + /** + * @brief Test table existence method + * + * This method is used to test table existence. + * + * @param table_name, table name is going to be tested. + * + * + * @param table_name + */ + bool HasTable(const std::string &table_name); /** * @brief Delete table method diff --git a/cpp/src/server/RequestTask.cpp b/cpp/src/server/RequestTask.cpp index 77396d0046..a4733425e2 100644 --- a/cpp/src/server/RequestTask.cpp +++ b/cpp/src/server/RequestTask.cpp @@ -70,7 +70,7 @@ namespace { uint64_t vec_dim = record.vector_data.size()/sizeof(double);//how many double value? if(vec_dim != dimension) { SERVER_LOG_ERROR << "Invalid vector dimension: " << vec_dim - << " vs. group dimension:" << dimension; + << " vs. table dimension:" << dimension; error_code = SERVER_INVALID_VECTOR_DIMENSION; return error_code; } @@ -235,6 +235,44 @@ ServerError DescribeTableTask::OnExecute() { return SERVER_SUCCESS; } +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +HasTableTask::HasTableTask(const std::string& table_name, bool& has_table) + : BaseTask(DDL_DML_TASK_GROUP), + table_name_(table_name), + has_table_(has_table) { + +} + +BaseTaskPtr HasTableTask::Create(const std::string& table_name, bool& has_table) { + return std::shared_ptr(new HasTableTask(table_name, has_table)); +} + +ServerError HasTableTask::OnExecute() { + try { + TimeRecorder rc("HasTableTask"); + + //step 1: check arguments + if (table_name_.empty()) { + error_code_ = SERVER_INVALID_ARGUMENT; + error_msg_ = "Table name cannot be empty"; + SERVER_LOG_ERROR << error_msg_; + return error_code_; + } + + //step 2: check table existence + engine::Status stat = DBWrapper::DB()->HasTable(table_name_, has_table_); + + rc.Elapse("totally cost"); + } catch (std::exception& ex) { + error_code_ = SERVER_UNEXPECTED_ERROR; + error_msg_ = ex.what(); + SERVER_LOG_ERROR << error_msg_; + return error_code_; + } + + return SERVER_SUCCESS; +} + //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// DeleteTableTask::DeleteTableTask(const std::string& table_name) : BaseTask(DDL_DML_TASK_GROUP), @@ -242,8 +280,8 @@ DeleteTableTask::DeleteTableTask(const std::string& table_name) } -BaseTaskPtr DeleteTableTask::Create(const std::string& group_id) { - return std::shared_ptr(new DeleteTableTask(group_id)); +BaseTaskPtr DeleteTableTask::Create(const std::string& table_name) { + return std::shared_ptr(new DeleteTableTask(table_name)); } ServerError DeleteTableTask::OnExecute() { diff --git a/cpp/src/server/RequestTask.h b/cpp/src/server/RequestTask.h index b4ddc69726..3061b3b75d 100644 --- a/cpp/src/server/RequestTask.h +++ b/cpp/src/server/RequestTask.h @@ -33,6 +33,22 @@ private: const ::milvus::thrift::TableSchema& schema_; }; +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +class HasTableTask : public BaseTask { +public: + static BaseTaskPtr Create(const std::string& table_name, bool& has_table); + +protected: + HasTableTask(const std::string& table_name, bool& has_table); + + ServerError OnExecute() override; + + +private: + std::string table_name_; + bool& has_table_; +}; + //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// class DescribeTableTask : public BaseTask { public: diff --git a/cpp/src/thrift/gen-cpp/MilvusService.cpp b/cpp/src/thrift/gen-cpp/MilvusService.cpp index 7e0a120bc0..7a591276f6 100644 --- a/cpp/src/thrift/gen-cpp/MilvusService.cpp +++ b/cpp/src/thrift/gen-cpp/MilvusService.cpp @@ -196,6 +196,213 @@ uint32_t MilvusService_CreateTable_presult::read(::apache::thrift::protocol::TPr } +MilvusService_HasTable_args::~MilvusService_HasTable_args() throw() { +} + + +uint32_t MilvusService_HasTable_args::read(::apache::thrift::protocol::TProtocol* iprot) { + + ::apache::thrift::protocol::TInputRecursionTracker tracker(*iprot); + uint32_t xfer = 0; + std::string fname; + ::apache::thrift::protocol::TType ftype; + int16_t fid; + + xfer += iprot->readStructBegin(fname); + + using ::apache::thrift::protocol::TProtocolException; + + + while (true) + { + xfer += iprot->readFieldBegin(fname, ftype, fid); + if (ftype == ::apache::thrift::protocol::T_STOP) { + break; + } + switch (fid) + { + case 2: + if (ftype == ::apache::thrift::protocol::T_STRING) { + xfer += iprot->readString(this->table_name); + this->__isset.table_name = true; + } else { + xfer += iprot->skip(ftype); + } + break; + default: + xfer += iprot->skip(ftype); + break; + } + xfer += iprot->readFieldEnd(); + } + + xfer += iprot->readStructEnd(); + + return xfer; +} + +uint32_t MilvusService_HasTable_args::write(::apache::thrift::protocol::TProtocol* oprot) const { + uint32_t xfer = 0; + ::apache::thrift::protocol::TOutputRecursionTracker tracker(*oprot); + xfer += oprot->writeStructBegin("MilvusService_HasTable_args"); + + xfer += oprot->writeFieldBegin("table_name", ::apache::thrift::protocol::T_STRING, 2); + xfer += oprot->writeString(this->table_name); + xfer += oprot->writeFieldEnd(); + + xfer += oprot->writeFieldStop(); + xfer += oprot->writeStructEnd(); + return xfer; +} + + +MilvusService_HasTable_pargs::~MilvusService_HasTable_pargs() throw() { +} + + +uint32_t MilvusService_HasTable_pargs::write(::apache::thrift::protocol::TProtocol* oprot) const { + uint32_t xfer = 0; + ::apache::thrift::protocol::TOutputRecursionTracker tracker(*oprot); + xfer += oprot->writeStructBegin("MilvusService_HasTable_pargs"); + + xfer += oprot->writeFieldBegin("table_name", ::apache::thrift::protocol::T_STRING, 2); + xfer += oprot->writeString((*(this->table_name))); + xfer += oprot->writeFieldEnd(); + + xfer += oprot->writeFieldStop(); + xfer += oprot->writeStructEnd(); + return xfer; +} + + +MilvusService_HasTable_result::~MilvusService_HasTable_result() throw() { +} + + +uint32_t MilvusService_HasTable_result::read(::apache::thrift::protocol::TProtocol* iprot) { + + ::apache::thrift::protocol::TInputRecursionTracker tracker(*iprot); + uint32_t xfer = 0; + std::string fname; + ::apache::thrift::protocol::TType ftype; + int16_t fid; + + xfer += iprot->readStructBegin(fname); + + using ::apache::thrift::protocol::TProtocolException; + + + while (true) + { + xfer += iprot->readFieldBegin(fname, ftype, fid); + if (ftype == ::apache::thrift::protocol::T_STOP) { + break; + } + switch (fid) + { + case 0: + if (ftype == ::apache::thrift::protocol::T_BOOL) { + xfer += iprot->readBool(this->success); + this->__isset.success = true; + } else { + xfer += iprot->skip(ftype); + } + break; + case 1: + if (ftype == ::apache::thrift::protocol::T_STRUCT) { + xfer += this->e.read(iprot); + this->__isset.e = true; + } else { + xfer += iprot->skip(ftype); + } + break; + default: + xfer += iprot->skip(ftype); + break; + } + xfer += iprot->readFieldEnd(); + } + + xfer += iprot->readStructEnd(); + + return xfer; +} + +uint32_t MilvusService_HasTable_result::write(::apache::thrift::protocol::TProtocol* oprot) const { + + uint32_t xfer = 0; + + xfer += oprot->writeStructBegin("MilvusService_HasTable_result"); + + if (this->__isset.success) { + xfer += oprot->writeFieldBegin("success", ::apache::thrift::protocol::T_BOOL, 0); + xfer += oprot->writeBool(this->success); + xfer += oprot->writeFieldEnd(); + } else if (this->__isset.e) { + xfer += oprot->writeFieldBegin("e", ::apache::thrift::protocol::T_STRUCT, 1); + xfer += this->e.write(oprot); + xfer += oprot->writeFieldEnd(); + } + xfer += oprot->writeFieldStop(); + xfer += oprot->writeStructEnd(); + return xfer; +} + + +MilvusService_HasTable_presult::~MilvusService_HasTable_presult() throw() { +} + + +uint32_t MilvusService_HasTable_presult::read(::apache::thrift::protocol::TProtocol* iprot) { + + ::apache::thrift::protocol::TInputRecursionTracker tracker(*iprot); + uint32_t xfer = 0; + std::string fname; + ::apache::thrift::protocol::TType ftype; + int16_t fid; + + xfer += iprot->readStructBegin(fname); + + using ::apache::thrift::protocol::TProtocolException; + + + while (true) + { + xfer += iprot->readFieldBegin(fname, ftype, fid); + if (ftype == ::apache::thrift::protocol::T_STOP) { + break; + } + switch (fid) + { + case 0: + if (ftype == ::apache::thrift::protocol::T_BOOL) { + xfer += iprot->readBool((*(this->success))); + this->__isset.success = true; + } else { + xfer += iprot->skip(ftype); + } + break; + case 1: + if (ftype == ::apache::thrift::protocol::T_STRUCT) { + xfer += this->e.read(iprot); + this->__isset.e = true; + } else { + xfer += iprot->skip(ftype); + } + break; + default: + xfer += iprot->skip(ftype); + break; + } + xfer += iprot->readFieldEnd(); + } + + xfer += iprot->readStructEnd(); + + return xfer; +} + + MilvusService_DeleteTable_args::~MilvusService_DeleteTable_args() throw() { } @@ -2290,6 +2497,67 @@ void MilvusServiceClient::recv_CreateTable() return; } +bool MilvusServiceClient::HasTable(const std::string& table_name) +{ + send_HasTable(table_name); + return recv_HasTable(); +} + +void MilvusServiceClient::send_HasTable(const std::string& table_name) +{ + int32_t cseqid = 0; + oprot_->writeMessageBegin("HasTable", ::apache::thrift::protocol::T_CALL, cseqid); + + MilvusService_HasTable_pargs args; + args.table_name = &table_name; + args.write(oprot_); + + oprot_->writeMessageEnd(); + oprot_->getTransport()->writeEnd(); + oprot_->getTransport()->flush(); +} + +bool MilvusServiceClient::recv_HasTable() +{ + + int32_t rseqid = 0; + std::string fname; + ::apache::thrift::protocol::TMessageType mtype; + + iprot_->readMessageBegin(fname, mtype, rseqid); + if (mtype == ::apache::thrift::protocol::T_EXCEPTION) { + ::apache::thrift::TApplicationException x; + x.read(iprot_); + iprot_->readMessageEnd(); + iprot_->getTransport()->readEnd(); + throw x; + } + if (mtype != ::apache::thrift::protocol::T_REPLY) { + iprot_->skip(::apache::thrift::protocol::T_STRUCT); + iprot_->readMessageEnd(); + iprot_->getTransport()->readEnd(); + } + if (fname.compare("HasTable") != 0) { + iprot_->skip(::apache::thrift::protocol::T_STRUCT); + iprot_->readMessageEnd(); + iprot_->getTransport()->readEnd(); + } + bool _return; + MilvusService_HasTable_presult result; + result.success = &_return; + result.read(iprot_); + iprot_->readMessageEnd(); + iprot_->getTransport()->readEnd(); + + if (result.__isset.success) { + return _return; + } + if (result.__isset.e) { + throw result.e; + } + throw ::apache::thrift::TApplicationException(::apache::thrift::TApplicationException::MISSING_RESULT, "HasTable failed: unknown result"); +} + void MilvusServiceClient::DeleteTable(const std::string& table_name) { send_DeleteTable(table_name); @@ -2855,6 +3123,63 @@ void MilvusServiceProcessor::process_CreateTable(int32_t seqid, ::apache::thrift } } +void MilvusServiceProcessor::process_HasTable(int32_t seqid, ::apache::thrift::protocol::TProtocol* iprot, ::apache::thrift::protocol::TProtocol* oprot, void* callContext) +{ + void* ctx = NULL; + if (this->eventHandler_.get() != NULL) { + ctx = this->eventHandler_->getContext("MilvusService.HasTable", callContext); + } + ::apache::thrift::TProcessorContextFreer freer(this->eventHandler_.get(), ctx, "MilvusService.HasTable"); + + if (this->eventHandler_.get() != NULL) { + this->eventHandler_->preRead(ctx, "MilvusService.HasTable"); + } + + MilvusService_HasTable_args args; + args.read(iprot); + iprot->readMessageEnd(); + uint32_t bytes = iprot->getTransport()->readEnd(); + + if (this->eventHandler_.get() != NULL) { + this->eventHandler_->postRead(ctx, "MilvusService.HasTable", bytes); + } + + MilvusService_HasTable_result result; + try { + result.success = iface_->HasTable(args.table_name); + result.__isset.success = true; + } catch (Exception &e) { + result.e = e; + result.__isset.e = true; + } catch (const std::exception& e) { + if (this->eventHandler_.get() != NULL) { + this->eventHandler_->handlerError(ctx, "MilvusService.HasTable"); + } + + ::apache::thrift::TApplicationException x(e.what()); + oprot->writeMessageBegin("HasTable", ::apache::thrift::protocol::T_EXCEPTION, seqid); + x.write(oprot); + oprot->writeMessageEnd(); + oprot->getTransport()->writeEnd(); + oprot->getTransport()->flush(); + return; + } + + if (this->eventHandler_.get() != NULL) { + this->eventHandler_->preWrite(ctx, "MilvusService.HasTable"); + } + + oprot->writeMessageBegin("HasTable", ::apache::thrift::protocol::T_REPLY, seqid); + result.write(oprot); + oprot->writeMessageEnd(); + bytes = oprot->getTransport()->writeEnd(); + oprot->getTransport()->flush(); + + if (this->eventHandler_.get() != NULL) { + this->eventHandler_->postWrite(ctx, "MilvusService.HasTable", bytes); + } +} + void MilvusServiceProcessor::process_DeleteTable(int32_t seqid, ::apache::thrift::protocol::TProtocol* iprot, ::apache::thrift::protocol::TProtocol* oprot, void* callContext) { void* ctx = NULL; @@ -3399,6 +3724,94 @@ void MilvusServiceConcurrentClient::recv_CreateTable(const int32_t seqid) } // end while(true) } +bool MilvusServiceConcurrentClient::HasTable(const std::string& table_name) +{ + int32_t seqid = send_HasTable(table_name); + return recv_HasTable(seqid); +} + +int32_t MilvusServiceConcurrentClient::send_HasTable(const std::string& table_name) +{ + int32_t cseqid = this->sync_.generateSeqId(); + ::apache::thrift::async::TConcurrentSendSentry sentry(&this->sync_); + oprot_->writeMessageBegin("HasTable", ::apache::thrift::protocol::T_CALL, cseqid); + + MilvusService_HasTable_pargs args; + args.table_name = &table_name; + args.write(oprot_); + + oprot_->writeMessageEnd(); + oprot_->getTransport()->writeEnd(); + oprot_->getTransport()->flush(); + + sentry.commit(); + return cseqid; +} + +bool MilvusServiceConcurrentClient::recv_HasTable(const int32_t seqid) +{ + + int32_t rseqid = 0; + std::string fname; + ::apache::thrift::protocol::TMessageType mtype; + + // the read mutex gets dropped and reacquired as part of waitForWork() + // The destructor of this sentry wakes up other clients + ::apache::thrift::async::TConcurrentRecvSentry sentry(&this->sync_, seqid); + + while(true) { + if(!this->sync_.getPending(fname, mtype, rseqid)) { + iprot_->readMessageBegin(fname, mtype, rseqid); + } + if(seqid == rseqid) { + if (mtype == ::apache::thrift::protocol::T_EXCEPTION) { + ::apache::thrift::TApplicationException x; + x.read(iprot_); + iprot_->readMessageEnd(); + iprot_->getTransport()->readEnd(); + sentry.commit(); + throw x; + } + if (mtype != ::apache::thrift::protocol::T_REPLY) { + iprot_->skip(::apache::thrift::protocol::T_STRUCT); + iprot_->readMessageEnd(); + iprot_->getTransport()->readEnd(); + } + if (fname.compare("HasTable") != 0) { + iprot_->skip(::apache::thrift::protocol::T_STRUCT); + iprot_->readMessageEnd(); + iprot_->getTransport()->readEnd(); + + // in a bad state, don't commit + using ::apache::thrift::protocol::TProtocolException; + throw TProtocolException(TProtocolException::INVALID_DATA); + } + bool _return; + MilvusService_HasTable_presult result; + result.success = &_return; + result.read(iprot_); + iprot_->readMessageEnd(); + iprot_->getTransport()->readEnd(); + + if (result.__isset.success) { + sentry.commit(); + return _return; + } + if (result.__isset.e) { + sentry.commit(); + throw result.e; + } + // in a bad state, don't commit + throw ::apache::thrift::TApplicationException(::apache::thrift::TApplicationException::MISSING_RESULT, "HasTable failed: unknown result"); + } + // seqid != rseqid + this->sync_.updatePending(fname, mtype, rseqid); + + // this will temporarily unlock the readMutex, and let other clients get work done + this->sync_.waitForWork(seqid); + } // end while(true) +} + void MilvusServiceConcurrentClient::DeleteTable(const std::string& table_name) { int32_t seqid = send_DeleteTable(table_name); diff --git a/cpp/src/thrift/gen-cpp/MilvusService.h b/cpp/src/thrift/gen-cpp/MilvusService.h index 4c1183de60..3868681463 100644 --- a/cpp/src/thrift/gen-cpp/MilvusService.h +++ b/cpp/src/thrift/gen-cpp/MilvusService.h @@ -34,6 +34,18 @@ class MilvusServiceIf { */ virtual void CreateTable(const TableSchema& param) = 0; + /** + * @brief Test table existence method + * + * This method is used to test table existence. + * + * @param table_name, table name is going to be tested. + * + * + * @param table_name + */ + virtual bool HasTable(const std::string& table_name) = 0; + /** * @brief Delete table method * @@ -178,6 +190,10 @@ class MilvusServiceNull : virtual public MilvusServiceIf { void CreateTable(const TableSchema& /* param */) { return; } + bool HasTable(const std::string& /* table_name */) { + bool _return = false; + return _return; + } void DeleteTable(const std::string& /* table_name */) { return; } @@ -309,6 +325,118 @@ class MilvusService_CreateTable_presult { }; +typedef struct _MilvusService_HasTable_args__isset { + _MilvusService_HasTable_args__isset() : table_name(false) {} + bool table_name :1; +} _MilvusService_HasTable_args__isset; + +class MilvusService_HasTable_args { + public: + + MilvusService_HasTable_args(const MilvusService_HasTable_args&); + MilvusService_HasTable_args& operator=(const MilvusService_HasTable_args&); + MilvusService_HasTable_args() : table_name() { + } + + virtual ~MilvusService_HasTable_args() throw(); + std::string table_name; + + _MilvusService_HasTable_args__isset __isset; + + void __set_table_name(const std::string& val); + + bool operator == (const MilvusService_HasTable_args & rhs) const + { + if (!(table_name == rhs.table_name)) + return false; + return true; + } + bool operator != (const MilvusService_HasTable_args &rhs) const { + return !(*this == rhs); + } + + bool operator < (const MilvusService_HasTable_args & ) const; + + uint32_t read(::apache::thrift::protocol::TProtocol* iprot); + uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const; + +}; + + +class MilvusService_HasTable_pargs { + public: + + + virtual ~MilvusService_HasTable_pargs() throw(); + const std::string* table_name; + + uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const; + +}; + +typedef struct _MilvusService_HasTable_result__isset { + _MilvusService_HasTable_result__isset() : success(false), e(false) {} + bool success :1; + bool e :1; +} _MilvusService_HasTable_result__isset; + +class MilvusService_HasTable_result { + public: + + MilvusService_HasTable_result(const MilvusService_HasTable_result&); + MilvusService_HasTable_result& operator=(const MilvusService_HasTable_result&); + MilvusService_HasTable_result() : success(0) { + } + + virtual ~MilvusService_HasTable_result() throw(); + bool success; + Exception e; + + _MilvusService_HasTable_result__isset __isset; + + void __set_success(const bool val); + + void __set_e(const Exception& val); + + bool operator == (const MilvusService_HasTable_result & rhs) const + { + if (!(success == rhs.success)) + return false; + if (!(e == rhs.e)) + return false; + return true; + } + bool operator != (const MilvusService_HasTable_result &rhs) const { + return !(*this == rhs); + } + + bool operator < (const MilvusService_HasTable_result & ) const; + + uint32_t read(::apache::thrift::protocol::TProtocol* iprot); + uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const; + +}; + +typedef struct _MilvusService_HasTable_presult__isset { + _MilvusService_HasTable_presult__isset() : success(false), e(false) {} + bool success :1; + bool e :1; +} _MilvusService_HasTable_presult__isset; + +class MilvusService_HasTable_presult { + public: + + + virtual ~MilvusService_HasTable_presult() throw(); + bool* success; + Exception e; + + _MilvusService_HasTable_presult__isset __isset; + + uint32_t read(::apache::thrift::protocol::TProtocol* iprot); + +}; + typedef struct _MilvusService_DeleteTable_args__isset { _MilvusService_DeleteTable_args__isset() : table_name(false) {} bool table_name :1; @@ -1269,6 +1397,9 @@ class MilvusServiceClient : virtual public MilvusServiceIf { void CreateTable(const TableSchema& param); void send_CreateTable(const TableSchema& param); void recv_CreateTable(); + bool HasTable(const std::string& table_name); + void send_HasTable(const std::string& table_name); + bool recv_HasTable(); void DeleteTable(const std::string& table_name); void send_DeleteTable(const std::string& table_name); void recv_DeleteTable(); @@ -1309,6 +1440,7 @@ class MilvusServiceProcessor : public ::apache::thrift::TDispatchProcessor { typedef std::map ProcessMap; ProcessMap processMap_; void process_CreateTable(int32_t seqid, ::apache::thrift::protocol::TProtocol* iprot, ::apache::thrift::protocol::TProtocol* oprot, void* callContext); + void process_HasTable(int32_t seqid, ::apache::thrift::protocol::TProtocol* iprot, ::apache::thrift::protocol::TProtocol* oprot, void* callContext); void process_DeleteTable(int32_t seqid, ::apache::thrift::protocol::TProtocol* iprot, ::apache::thrift::protocol::TProtocol* oprot, void* callContext); void process_AddVector(int32_t seqid, ::apache::thrift::protocol::TProtocol* iprot, ::apache::thrift::protocol::TProtocol* oprot, void* callContext); void process_SearchVector(int32_t seqid, ::apache::thrift::protocol::TProtocol* iprot, ::apache::thrift::protocol::TProtocol* oprot, void* callContext); @@ -1321,6 +1453,7 @@ class MilvusServiceProcessor : public ::apache::thrift::TDispatchProcessor { MilvusServiceProcessor(::apache::thrift::stdcxx::shared_ptr iface) : iface_(iface) { processMap_["CreateTable"] = &MilvusServiceProcessor::process_CreateTable; + processMap_["HasTable"] = &MilvusServiceProcessor::process_HasTable; processMap_["DeleteTable"] = &MilvusServiceProcessor::process_DeleteTable; processMap_["AddVector"] = &MilvusServiceProcessor::process_AddVector; processMap_["SearchVector"] = &MilvusServiceProcessor::process_SearchVector; @@ -1366,6 +1499,15 @@ class MilvusServiceMultiface : virtual public MilvusServiceIf { ifaces_[i]->CreateTable(param); } + bool HasTable(const std::string& table_name) { + size_t sz = ifaces_.size(); + size_t i = 0; + for (; i < (sz - 1); ++i) { + ifaces_[i]->HasTable(table_name); + } + return ifaces_[i]->HasTable(table_name); + } + void DeleteTable(const std::string& table_name) { size_t sz = ifaces_.size(); size_t i = 0; @@ -1477,6 +1619,9 @@ class MilvusServiceConcurrentClient : virtual public MilvusServiceIf { void CreateTable(const TableSchema& param); int32_t send_CreateTable(const TableSchema& param); void recv_CreateTable(const int32_t seqid); + bool HasTable(const std::string& table_name); + int32_t send_HasTable(const std::string& table_name); + bool recv_HasTable(const int32_t seqid); void DeleteTable(const std::string& table_name); int32_t send_DeleteTable(const std::string& table_name); void recv_DeleteTable(const int32_t seqid); diff --git a/cpp/src/thrift/gen-cpp/MilvusService_server.skeleton.cpp b/cpp/src/thrift/gen-cpp/MilvusService_server.skeleton.cpp index 55d73b9642..ecb22c0b62 100644 --- a/cpp/src/thrift/gen-cpp/MilvusService_server.skeleton.cpp +++ b/cpp/src/thrift/gen-cpp/MilvusService_server.skeleton.cpp @@ -35,6 +35,21 @@ class MilvusServiceHandler : virtual public MilvusServiceIf { printf("CreateTable\n"); } + /** + * @brief Test table existence method + * + * This method is used to test table existence. + * + * @param table_name, table name is going to be tested. + * + * + * @param table_name + */ + bool HasTable(const std::string& table_name) { + // Your implementation goes here + printf("HasTable\n"); + } + /** * @brief Delete table method * diff --git a/cpp/src/thrift/milvus.thrift b/cpp/src/thrift/milvus.thrift index 2936b85a5c..48116256e8 100644 --- a/cpp/src/thrift/milvus.thrift +++ b/cpp/src/thrift/milvus.thrift @@ -80,6 +80,16 @@ service MilvusService { */ void CreateTable(2: TableSchema param) throws(1: Exception e); + /** + * @brief Test table existence method + * + * This method is used to test table existence. + * + * @param table_name, table name is going to be tested. + * + */ + bool HasTable(2: string table_name) throws(1: Exception e); + /** * @brief Delete table method From abe34a45863e3fda31438b0af889949c8369617d Mon Sep 17 00:00:00 2001 From: starlord Date: Thu, 27 Jun 2019 18:03:52 +0800 Subject: [PATCH 016/189] MS-126 Add more error code Former-commit-id: 43496ac3c9a578105131219f6802de3bec78746d --- cpp/CHANGELOG.md | 1 + cpp/src/db/DBMetaImpl.cpp | 12 +- cpp/src/server/RequestScheduler.cpp | 52 +++--- cpp/src/server/RequestScheduler.h | 2 + cpp/src/server/RequestTask.cpp | 237 +++++++++--------------- cpp/src/thrift/gen-cpp/milvus_types.cpp | 34 +++- cpp/src/thrift/gen-cpp/milvus_types.h | 26 ++- cpp/src/thrift/milvus.thrift | 14 ++ cpp/src/utils/Error.h | 41 ++-- cpp/unittest/db/meta_tests.cpp | 70 +++---- 10 files changed, 261 insertions(+), 228 deletions(-) diff --git a/cpp/CHANGELOG.md b/cpp/CHANGELOG.md index 1245b7efd5..be7024672f 100644 --- a/cpp/CHANGELOG.md +++ b/cpp/CHANGELOG.md @@ -23,6 +23,7 @@ Please mark all change in change log and use the ticket from JIRA. - MS-115 - Change is_startup of metric_config switch from true to on - MS-122 - Archive criteria config - MS-124 - HasTable interface +- MS-126 - Add more error code ## New Feature diff --git a/cpp/src/db/DBMetaImpl.cpp b/cpp/src/db/DBMetaImpl.cpp index 9a44820c7a..8c56c863e7 100644 --- a/cpp/src/db/DBMetaImpl.cpp +++ b/cpp/src/db/DBMetaImpl.cpp @@ -193,9 +193,11 @@ Status DBMetaImpl::CreateTable(TableSchema &table_schema) { auto table = ConnectorPtr->select(columns(&TableSchema::state_), where(c(&TableSchema::table_id_) == table_schema.table_id_)); if (table.size() == 1) { - std::string msg = (TableSchema::TO_DELETE == std::get<0>(table[0])) ? - "Table already exists and it is in delete state, please wait a second" : "Table already exists"; - return Status::Error(msg); + if(TableSchema::TO_DELETE == std::get<0>(table[0])) { + return Status::Error("Table already exists and it is in delete state, please wait a second"); + } else { + return Status::OK();//table already exists, no error + } } } @@ -329,7 +331,7 @@ Status DBMetaImpl::HasTable(const std::string &table_id, bool &has_or_not) { } } catch (std::exception &e) { - HandleException("Encounter exception when lookup table", e); + return HandleException("Encounter exception when lookup table", e); } return Status::OK(); @@ -359,7 +361,7 @@ Status DBMetaImpl::AllTables(std::vector& table_schema_array) { } } catch (std::exception &e) { - HandleException("Encounter exception when lookup all tables", e); + return HandleException("Encounter exception when lookup all tables", e); } return Status::OK(); diff --git a/cpp/src/server/RequestScheduler.cpp b/cpp/src/server/RequestScheduler.cpp index 99ae36701e..36df155b62 100644 --- a/cpp/src/server/RequestScheduler.cpp +++ b/cpp/src/server/RequestScheduler.cpp @@ -18,35 +18,35 @@ using namespace ::milvus; namespace { const std::map &ErrorMap() { static const std::map code_map = { - {SERVER_UNEXPECTED_ERROR, thrift::ErrorCode::ILLEGAL_ARGUMENT}, - {SERVER_NULL_POINTER, thrift::ErrorCode::ILLEGAL_ARGUMENT}, + {SERVER_UNEXPECTED_ERROR, thrift::ErrorCode::UNEXPECTED_ERROR}, + {SERVER_UNSUPPORTED_ERROR, thrift::ErrorCode::UNEXPECTED_ERROR}, + {SERVER_NULL_POINTER, thrift::ErrorCode::UNEXPECTED_ERROR}, {SERVER_INVALID_ARGUMENT, thrift::ErrorCode::ILLEGAL_ARGUMENT}, - {SERVER_FILE_NOT_FOUND, thrift::ErrorCode::ILLEGAL_ARGUMENT}, - {SERVER_NOT_IMPLEMENT, thrift::ErrorCode::ILLEGAL_ARGUMENT}, - {SERVER_BLOCKING_QUEUE_EMPTY, thrift::ErrorCode::ILLEGAL_ARGUMENT}, + {SERVER_FILE_NOT_FOUND, thrift::ErrorCode::FILE_NOT_FOUND}, + {SERVER_NOT_IMPLEMENT, thrift::ErrorCode::UNEXPECTED_ERROR}, + {SERVER_BLOCKING_QUEUE_EMPTY, thrift::ErrorCode::UNEXPECTED_ERROR}, + {SERVER_CANNOT_CREATE_FOLDER, thrift::ErrorCode::CANNOT_CREATE_FOLDER}, + {SERVER_CANNOT_CREATE_FILE, thrift::ErrorCode::CANNOT_CREATE_FILE}, + {SERVER_CANNOT_DELETE_FOLDER, thrift::ErrorCode::CANNOT_DELETE_FOLDER}, + {SERVER_CANNOT_DELETE_FILE, thrift::ErrorCode::CANNOT_DELETE_FILE}, {SERVER_TABLE_NOT_EXIST, thrift::ErrorCode::TABLE_NOT_EXISTS}, + {SERVER_INVALID_TABLE_NAME, thrift::ErrorCode::ILLEGAL_TABLE_NAME}, + {SERVER_INVALID_TABLE_DIMENSION, thrift::ErrorCode::ILLEGAL_DIMENSION}, {SERVER_INVALID_TIME_RANGE, thrift::ErrorCode::ILLEGAL_RANGE}, {SERVER_INVALID_VECTOR_DIMENSION, thrift::ErrorCode::ILLEGAL_DIMENSION}, + + {SERVER_INVALID_INDEX_TYPE, thrift::ErrorCode::ILLEGAL_INDEX_TYPE}, + {SERVER_INVALID_ROWRECORD, thrift::ErrorCode::ILLEGAL_ROWRECORD}, + {SERVER_INVALID_ROWRECORD_ARRAY, thrift::ErrorCode::ILLEGAL_ROWRECORD}, + {SERVER_INVALID_TOPK, thrift::ErrorCode::ILLEGAL_TOPK}, + {SERVER_ILLEGAL_VECTOR_ID, thrift::ErrorCode::ILLEGAL_VECTOR_ID}, + {SERVER_ILLEGAL_SEARCH_RESULT, thrift::ErrorCode::ILLEGAL_SEARCH_RESULT}, + {SERVER_CACHE_ERROR, thrift::ErrorCode::CACHE_FAILED}, + {DB_META_TRANSACTION_FAILED, thrift::ErrorCode::META_FAILED}, }; return code_map; } - - const std::map &ErrorMessage() { - static const std::map msg_map = { - {SERVER_UNEXPECTED_ERROR, "unexpected error occurs"}, - {SERVER_NULL_POINTER, "null pointer error"}, - {SERVER_INVALID_ARGUMENT, "invalid argument"}, - {SERVER_FILE_NOT_FOUND, "file not found"}, - {SERVER_NOT_IMPLEMENT, "not implemented"}, - {SERVER_BLOCKING_QUEUE_EMPTY, "queue empty"}, - {SERVER_TABLE_NOT_EXIST, "table not exist"}, - {SERVER_INVALID_TIME_RANGE, "invalid time range"}, - {SERVER_INVALID_VECTOR_DIMENSION, "invalid vector dimension"}, - }; - - return msg_map; - } } //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// @@ -69,6 +69,14 @@ ServerError BaseTask::Execute() { return error_code_; } +ServerError BaseTask::SetError(ServerError error_code, const std::string& error_msg) { + error_code_ = error_code; + error_msg_ = error_msg; + + SERVER_LOG_ERROR << error_msg_; + return error_code_; +} + ServerError BaseTask::WaitToFinish() { std::unique_lock lock(finish_mtx_); finish_cond_.wait(lock, [this] { return done_; }); @@ -102,7 +110,7 @@ void RequestScheduler::ExecTask(BaseTaskPtr& task_ptr) { ex.__set_code(ErrorMap().at(err)); std::string msg = task_ptr->ErrorMsg(); if(msg.empty()){ - msg = ErrorMessage().at(err); + msg = "Error message not set"; } ex.__set_reason(msg); throw ex; diff --git a/cpp/src/server/RequestScheduler.h b/cpp/src/server/RequestScheduler.h index b7562cc367..d4b1e1c826 100644 --- a/cpp/src/server/RequestScheduler.h +++ b/cpp/src/server/RequestScheduler.h @@ -34,6 +34,8 @@ public: protected: virtual ServerError OnExecute() = 0; + ServerError SetError(ServerError error_code, const std::string& msg); + protected: mutable std::mutex finish_mtx_; std::condition_variable finish_cond_; diff --git a/cpp/src/server/RequestTask.cpp b/cpp/src/server/RequestTask.cpp index a4733425e2..bb0c437219 100644 --- a/cpp/src/server/RequestTask.cpp +++ b/cpp/src/server/RequestTask.cpp @@ -53,26 +53,27 @@ namespace { return map_type[type]; } - ServerError + void ConvertRowRecordToFloatArray(const std::vector& record_array, uint64_t dimension, - std::vector& float_array) { - ServerError error_code; + std::vector& float_array, + ServerError& error_code, + std::string& error_msg) { uint64_t vec_count = record_array.size(); float_array.resize(vec_count*dimension);//allocate enough memory for(uint64_t i = 0; i < vec_count; i++) { const auto& record = record_array[i]; if(record.vector_data.empty()) { - error_code = SERVER_INVALID_ARGUMENT; - SERVER_LOG_ERROR << "No vector provided in record"; - return error_code; + error_code = SERVER_INVALID_ROWRECORD; + error_msg = "Rowrecord float array is empty"; + return; } uint64_t vec_dim = record.vector_data.size()/sizeof(double);//how many double value? if(vec_dim != dimension) { - SERVER_LOG_ERROR << "Invalid vector dimension: " << vec_dim - << " vs. table dimension:" << dimension; error_code = SERVER_INVALID_VECTOR_DIMENSION; - return error_code; + error_msg = "Invalid rowrecord dimension: " + std::to_string(vec_dim) + + " vs. table dimension:" + std::to_string(dimension); + return; } //convert double array to float array(thrift has no float type) @@ -81,30 +82,29 @@ namespace { float_array[i*vec_dim + d] = (float)(d_p[d]); } } - - return SERVER_SUCCESS; } static constexpr long DAY_SECONDS = 86400; - ServerError + void ConvertTimeRangeToDBDates(const std::vector &range_array, - std::vector& dates) { + std::vector& dates, + ServerError& error_code, + std::string& error_msg) { dates.clear(); - ServerError error_code; for(auto& range : range_array) { time_t tt_start, tt_end; tm tm_start, tm_end; if(!CommonUtil::TimeStrToTime(range.start_value, tt_start, tm_start)){ error_code = SERVER_INVALID_TIME_RANGE; - SERVER_LOG_ERROR << "Invalid time range: " << range.start_value; - return error_code; + error_msg = "Invalid time range: " + range.start_value; + return; } if(!CommonUtil::TimeStrToTime(range.end_value, tt_end, tm_end)){ error_code = SERVER_INVALID_TIME_RANGE; - SERVER_LOG_ERROR << "Invalid time range: " << range.end_value; - return error_code; + error_msg = "Invalid time range: " + range.start_value; + return; } long days = (tt_end > tt_start) ? (tt_end - tt_start)/DAY_SECONDS : (tt_start - tt_end)/DAY_SECONDS; @@ -117,8 +117,6 @@ namespace { dates.push_back(date); } } - - return SERVER_SUCCESS; } } @@ -138,21 +136,16 @@ ServerError CreateTableTask::OnExecute() { try { //step 1: check arguments - if(schema_.table_name.empty() || schema_.dimension <= 0) { - error_code_ = SERVER_INVALID_ARGUMENT; -// error_msg_ = schema_.table_name.empty() ? - error_msg_ = "CreateTableTask: Invalid table name or dimension. table name = " + schema_.table_name - + "dimension = " + std::to_string(schema_.dimension); - SERVER_LOG_ERROR << error_msg_; - return error_code_; + if(schema_.table_name.empty()) { + return SetError(SERVER_INVALID_TABLE_NAME, "Empty table name"); + } + if(schema_.dimension <= 0) { + return SetError(SERVER_INVALID_TABLE_DIMENSION, "Invalid table dimension: " + std::to_string(schema_.dimension)); } engine::EngineType engine_type = EngineType(schema_.index_type); if(engine_type == engine::EngineType::INVALID) { - error_code_ = SERVER_INVALID_ARGUMENT; - error_msg_ = "CreateTableTask: Invalid index type. type = " + std::to_string(schema_.index_type); - SERVER_LOG_ERROR << error_msg_; - return error_code_; + return SetError(SERVER_INVALID_INDEX_TYPE, "Invalid index type: " + std::to_string(schema_.index_type)); } //step 2: construct table schema @@ -165,17 +158,11 @@ ServerError CreateTableTask::OnExecute() { //step 3: create table engine::Status stat = DBWrapper::DB()->CreateTable(table_info); if(!stat.ok()) {//table could exist - error_code_ = SERVER_UNEXPECTED_ERROR; - error_msg_ = "CreateTableTask: Engine failed: " + stat.ToString(); - SERVER_LOG_ERROR << error_msg_; - return error_code_; + return SetError(DB_META_TRANSACTION_FAILED, "Engine failed: " + stat.ToString()); } } catch (std::exception& ex) { - error_code_ = SERVER_UNEXPECTED_ERROR; - error_msg_ = ex.what(); - SERVER_LOG_ERROR << "CreateTableTask: " << error_msg_; - return error_code_; + return SetError(SERVER_UNEXPECTED_ERROR, ex.what()); } rc.Record("done"); @@ -201,10 +188,7 @@ ServerError DescribeTableTask::OnExecute() { try { //step 1: check arguments if(table_name_.empty()) { - error_code_ = SERVER_INVALID_ARGUMENT; - error_msg_ = "DescribeTableTask: Table name cannot be empty"; - SERVER_LOG_ERROR << error_msg_; - return error_code_; + return SetError(SERVER_INVALID_TABLE_NAME, "Empty table name"); } //step 2: get table info @@ -212,10 +196,7 @@ ServerError DescribeTableTask::OnExecute() { table_info.table_id_ = table_name_; engine::Status stat = DBWrapper::DB()->DescribeTable(table_info); if(!stat.ok()) { - error_code_ = SERVER_TABLE_NOT_EXIST; - error_msg_ = "DescribeTableTask: Engine failed: " + stat.ToString(); - SERVER_LOG_ERROR << error_msg_; - return error_code_; + return SetError(DB_META_TRANSACTION_FAILED, "Engine failed: " + stat.ToString()); } schema_.table_name = table_info.table_id_; @@ -224,10 +205,7 @@ ServerError DescribeTableTask::OnExecute() { schema_.store_raw_vector = table_info.store_raw_data_; } catch (std::exception& ex) { - error_code_ = SERVER_UNEXPECTED_ERROR; - error_msg_ = ex.what(); - SERVER_LOG_ERROR << "DescribeTableTask: " << error_msg_; - return SERVER_UNEXPECTED_ERROR; + return SetError(SERVER_UNEXPECTED_ERROR, ex.what()); } rc.Record("done"); @@ -252,22 +230,19 @@ ServerError HasTableTask::OnExecute() { TimeRecorder rc("HasTableTask"); //step 1: check arguments - if (table_name_.empty()) { - error_code_ = SERVER_INVALID_ARGUMENT; - error_msg_ = "Table name cannot be empty"; - SERVER_LOG_ERROR << error_msg_; - return error_code_; + if(table_name_.empty()) { + return SetError(SERVER_INVALID_TABLE_NAME, "Empty table name"); } //step 2: check table existence engine::Status stat = DBWrapper::DB()->HasTable(table_name_, has_table_); + if(!stat.ok()) { + return SetError(DB_META_TRANSACTION_FAILED, "Engine failed: " + stat.ToString()); + } rc.Elapse("totally cost"); } catch (std::exception& ex) { - error_code_ = SERVER_UNEXPECTED_ERROR; - error_msg_ = ex.what(); - SERVER_LOG_ERROR << error_msg_; - return error_code_; + return SetError(SERVER_UNEXPECTED_ERROR, ex.what()); } return SERVER_SUCCESS; @@ -290,10 +265,7 @@ ServerError DeleteTableTask::OnExecute() { //step 1: check arguments if (table_name_.empty()) { - error_code_ = SERVER_INVALID_ARGUMENT; - error_msg_ = "DeleteTableTask: Table name cannot be empty"; - SERVER_LOG_ERROR << error_msg_; - return error_code_; + return SetError(SERVER_INVALID_TABLE_NAME, "Empty table name"); } //step 2: check table existence @@ -301,10 +273,11 @@ ServerError DeleteTableTask::OnExecute() { table_info.table_id_ = table_name_; engine::Status stat = DBWrapper::DB()->DescribeTable(table_info); if(!stat.ok()) { - error_code_ = SERVER_TABLE_NOT_EXIST; - error_msg_ = "DeleteTableTask: Engine failed: " + stat.ToString(); - SERVER_LOG_ERROR << error_msg_; - return error_code_; + if(stat.IsNotFound()) { + return SetError(SERVER_TABLE_NOT_EXIST, "Table " + table_name_ + " not exists"); + } else { + return SetError(DB_META_TRANSACTION_FAILED, "Engine failed: " + stat.ToString()); + } } rc.Record("check validation"); @@ -313,17 +286,13 @@ ServerError DeleteTableTask::OnExecute() { std::vector dates; stat = DBWrapper::DB()->DeleteTable(table_name_, dates); if(!stat.ok()) { - SERVER_LOG_ERROR << "DeleteTableTask: Engine failed: " << stat.ToString(); - return SERVER_UNEXPECTED_ERROR; + return SetError(DB_META_TRANSACTION_FAILED, "Engine failed: " + stat.ToString()); } rc.Record("deleta table"); rc.Elapse("total cost"); } catch (std::exception& ex) { - error_code_ = SERVER_UNEXPECTED_ERROR; - error_msg_ = ex.what(); - SERVER_LOG_ERROR << "DeleteTableTask: " << error_msg_; - return error_code_; + return SetError(SERVER_UNEXPECTED_ERROR, ex.what()); } return SERVER_SUCCESS; @@ -344,10 +313,7 @@ ServerError ShowTablesTask::OnExecute() { std::vector schema_array; engine::Status stat = DBWrapper::DB()->AllTables(schema_array); if(!stat.ok()) { - error_code_ = SERVER_UNEXPECTED_ERROR; - error_msg_ = "ShowTablesTask: Engine failed: " + stat.ToString(); - SERVER_LOG_ERROR << error_msg_; - return error_code_; + return SetError(DB_META_TRANSACTION_FAILED, "Engine failed: " + stat.ToString()); } tables_.clear(); @@ -381,17 +347,11 @@ ServerError AddVectorTask::OnExecute() { //step 1: check arguments if (table_name_.empty()) { - error_code_ = SERVER_INVALID_ARGUMENT; - error_msg_ = "AddVectorTask: Table name cannot be empty"; - SERVER_LOG_ERROR << error_msg_; - return error_code_; + return SetError(SERVER_INVALID_TABLE_NAME, "Empty table name"); } if(record_array_.empty()) { - error_code_ = SERVER_INVALID_ARGUMENT; - error_msg_ = "AddVectorTask: Row record array is empty"; - SERVER_LOG_ERROR << error_msg_; - return error_code_; + return SetError(SERVER_INVALID_ROWRECORD_ARRAY, "Row record array is empty"); } //step 2: check table existence @@ -399,20 +359,22 @@ ServerError AddVectorTask::OnExecute() { table_info.table_id_ = table_name_; engine::Status stat = DBWrapper::DB()->DescribeTable(table_info); if(!stat.ok()) { - error_code_ = SERVER_TABLE_NOT_EXIST; - error_msg_ = "AddVectorTask: Engine failed when DescribeTable: " + stat.ToString(); - SERVER_LOG_ERROR << error_msg_; - return error_code_; + if(stat.IsNotFound()) { + return SetError(SERVER_TABLE_NOT_EXIST, "Table " + table_name_ + " not exists"); + } else { + return SetError(DB_META_TRANSACTION_FAILED, "Engine failed: " + stat.ToString()); + } } rc.Record("check validation"); //step 3: prepare float data std::vector vec_f; - error_code_ = ConvertRowRecordToFloatArray(record_array_, table_info.dimension_, vec_f); - if(error_code_ != SERVER_SUCCESS) { - error_msg_ = "AddVectorTask when ConvertRowRecordToFloatArray: Invalid row record data"; - return error_code_; + ServerError error_code = SERVER_SUCCESS; + std::string error_msg; + ConvertRowRecordToFloatArray(record_array_, table_info.dimension_, vec_f, error_code, error_msg); + if(error_code != SERVER_SUCCESS) { + return SetError(error_code, error_msg); } rc.Record("prepare vectors data"); @@ -422,25 +384,20 @@ ServerError AddVectorTask::OnExecute() { stat = DBWrapper::DB()->InsertVectors(table_name_, vec_count, vec_f.data(), record_ids_); rc.Record("add vectors to engine"); if(!stat.ok()) { - error_code_ = SERVER_UNEXPECTED_ERROR; - error_msg_ = "AddVectorTask: Engine failed when InsertVectors: " + stat.ToString(); - SERVER_LOG_ERROR << error_msg_; - return error_code_; + return SetError(SERVER_CACHE_ERROR, "Cache error: " + stat.ToString()); } if(record_ids_.size() != vec_count) { - SERVER_LOG_ERROR << "AddVectorTask: Vector ID not returned"; - return SERVER_UNEXPECTED_ERROR; + std::string msg = "Add " + std::to_string(vec_count) + " vectors but only return " + + std::to_string(record_ids_.size()) + " id"; + return SetError(SERVER_ILLEGAL_VECTOR_ID, msg); } rc.Record("do insert"); rc.Elapse("total cost"); } catch (std::exception& ex) { - error_code_ = SERVER_UNEXPECTED_ERROR; - error_msg_ = ex.what(); - SERVER_LOG_ERROR << "AddVectorTask: " << error_msg_; - return error_code_; + return SetError(SERVER_UNEXPECTED_ERROR, ex.what()); } return SERVER_SUCCESS; @@ -479,17 +436,14 @@ ServerError SearchVectorTask::OnExecute() { //step 1: check arguments if (table_name_.empty()) { - error_code_ = SERVER_INVALID_ARGUMENT; - error_msg_ = "SearchVectorTask: Table name cannot be empty"; - SERVER_LOG_ERROR << error_msg_; - return error_code_; + return SetError(SERVER_INVALID_TABLE_NAME, "Empty table name"); } - if(top_k_ <= 0 || record_array_.empty()) { - error_code_ = SERVER_INVALID_ARGUMENT; - error_msg_ = "SearchVectorTask: Invalid topk value, or query record array is empty"; - SERVER_LOG_ERROR << error_msg_; - return error_code_; + if(top_k_ <= 0) { + return SetError(SERVER_INVALID_TOPK, "Invalid topk: " + std::to_string(top_k_)); + } + if(record_array_.empty()) { + return SetError(SERVER_INVALID_ROWRECORD_ARRAY, "Row record array is empty"); } //step 2: check table existence @@ -497,28 +451,29 @@ ServerError SearchVectorTask::OnExecute() { table_info.table_id_ = table_name_; engine::Status stat = DBWrapper::DB()->DescribeTable(table_info); if(!stat.ok()) { - error_code_ = SERVER_TABLE_NOT_EXIST; - error_msg_ = "SearchVectorTask: Engine failed when DescribeTable: " + stat.ToString(); - SERVER_LOG_ERROR << error_msg_; - return error_code_; + if(stat.IsNotFound()) { + return SetError(SERVER_TABLE_NOT_EXIST, "Table " + table_name_ + " not exists"); + } else { + return SetError(DB_META_TRANSACTION_FAILED, "Engine failed: " + stat.ToString()); + } } //step 3: check date range, and convert to db dates std::vector dates; - error_code_ = ConvertTimeRangeToDBDates(range_array_, dates); - if(error_code_ != SERVER_SUCCESS) { - error_msg_ = "SearchVectorTask: Invalid query range when ConvertTimeRangeToDBDates"; - return error_code_; + ServerError error_code = SERVER_SUCCESS; + std::string error_msg; + ConvertTimeRangeToDBDates(range_array_, dates, error_code, error_msg); + if(error_code != SERVER_SUCCESS) { + return SetError(error_code, error_msg); } rc.Record("check validation"); //step 3: prepare float data std::vector vec_f; - error_code_ = ConvertRowRecordToFloatArray(record_array_, table_info.dimension_, vec_f); - if(error_code_ != SERVER_SUCCESS) { - error_msg_ = "Invalid row record data when ConvertRowRecordToFloatArray"; - return error_code_; + ConvertRowRecordToFloatArray(record_array_, table_info.dimension_, vec_f, error_code, error_msg); + if(error_code != SERVER_SUCCESS) { + return SetError(error_code, error_msg); } rc.Record("prepare vector data"); @@ -535,13 +490,17 @@ ServerError SearchVectorTask::OnExecute() { rc.Record("search vectors from engine"); if(!stat.ok()) { - SERVER_LOG_ERROR << "SearchVectorTask: Engine failed: " << stat.ToString(); - return SERVER_UNEXPECTED_ERROR; + return SetError(DB_META_TRANSACTION_FAILED, "Engine failed: " + stat.ToString()); + } + + if(results.empty()) { + return SERVER_SUCCESS; //empty table } if(results.size() != record_count) { - SERVER_LOG_ERROR << "SearchVectorTask: Search result not returned"; - return SERVER_UNEXPECTED_ERROR; + std::string msg = "Search " + std::to_string(record_count) + " vectors but only return " + + std::to_string(results.size()) + " results"; + return SetError(SERVER_ILLEGAL_SEARCH_RESULT, msg); } rc.Record("do search"); @@ -566,10 +525,7 @@ ServerError SearchVectorTask::OnExecute() { rc.Elapse("total cost"); } catch (std::exception& ex) { - error_code_ = SERVER_UNEXPECTED_ERROR; - error_msg_ = ex.what(); - SERVER_LOG_ERROR << "SearchVectorTask: " << error_msg_; - return error_code_; + return SetError(SERVER_UNEXPECTED_ERROR, ex.what()); } return SERVER_SUCCESS; @@ -593,20 +549,14 @@ ServerError GetTableRowCountTask::OnExecute() { //step 1: check arguments if (table_name_.empty()) { - error_code_ = SERVER_INVALID_ARGUMENT; - error_msg_ = "GetTableRowCountTask: Table name cannot be empty"; - SERVER_LOG_ERROR << error_msg_; - return error_code_; + return SetError(SERVER_INVALID_TABLE_NAME, "Empty table name"); } //step 2: get row count uint64_t row_count = 0; engine::Status stat = DBWrapper::DB()->GetTableRowCount(table_name_, row_count); if (!stat.ok()) { - error_code_ = SERVER_UNEXPECTED_ERROR; - error_msg_ = "GetTableRowCountTask: Engine failed: " + stat.ToString(); - SERVER_LOG_ERROR << error_msg_; - return error_code_; + return SetError(DB_META_TRANSACTION_FAILED, "Engine failed: " + stat.ToString()); } row_count_ = (int64_t) row_count; @@ -614,10 +564,7 @@ ServerError GetTableRowCountTask::OnExecute() { rc.Elapse("total cost"); } catch (std::exception& ex) { - error_code_ = SERVER_UNEXPECTED_ERROR; - error_msg_ = ex.what(); - SERVER_LOG_ERROR << "GetTableRowCountTask: " << error_msg_; - return error_code_; + return SetError(SERVER_UNEXPECTED_ERROR, ex.what()); } return SERVER_SUCCESS; diff --git a/cpp/src/thrift/gen-cpp/milvus_types.cpp b/cpp/src/thrift/gen-cpp/milvus_types.cpp index 203faa2e52..af77fda0ad 100644 --- a/cpp/src/thrift/gen-cpp/milvus_types.cpp +++ b/cpp/src/thrift/gen-cpp/milvus_types.cpp @@ -15,23 +15,51 @@ namespace milvus { namespace thrift { int _kErrorCodeValues[] = { ErrorCode::SUCCESS, + ErrorCode::UNEXPECTED_ERROR, ErrorCode::CONNECT_FAILED, ErrorCode::PERMISSION_DENIED, ErrorCode::TABLE_NOT_EXISTS, ErrorCode::ILLEGAL_ARGUMENT, ErrorCode::ILLEGAL_RANGE, - ErrorCode::ILLEGAL_DIMENSION + ErrorCode::ILLEGAL_DIMENSION, + ErrorCode::ILLEGAL_INDEX_TYPE, + ErrorCode::ILLEGAL_TABLE_NAME, + ErrorCode::ILLEGAL_TOPK, + ErrorCode::ILLEGAL_ROWRECORD, + ErrorCode::ILLEGAL_VECTOR_ID, + ErrorCode::ILLEGAL_SEARCH_RESULT, + ErrorCode::FILE_NOT_FOUND, + ErrorCode::META_FAILED, + ErrorCode::CACHE_FAILED, + ErrorCode::CANNOT_CREATE_FOLDER, + ErrorCode::CANNOT_CREATE_FILE, + ErrorCode::CANNOT_DELETE_FOLDER, + ErrorCode::CANNOT_DELETE_FILE }; const char* _kErrorCodeNames[] = { "SUCCESS", + "UNEXPECTED_ERROR", "CONNECT_FAILED", "PERMISSION_DENIED", "TABLE_NOT_EXISTS", "ILLEGAL_ARGUMENT", "ILLEGAL_RANGE", - "ILLEGAL_DIMENSION" + "ILLEGAL_DIMENSION", + "ILLEGAL_INDEX_TYPE", + "ILLEGAL_TABLE_NAME", + "ILLEGAL_TOPK", + "ILLEGAL_ROWRECORD", + "ILLEGAL_VECTOR_ID", + "ILLEGAL_SEARCH_RESULT", + "FILE_NOT_FOUND", + "META_FAILED", + "CACHE_FAILED", + "CANNOT_CREATE_FOLDER", + "CANNOT_CREATE_FILE", + "CANNOT_DELETE_FOLDER", + "CANNOT_DELETE_FILE" }; -const std::map _ErrorCode_VALUES_TO_NAMES(::apache::thrift::TEnumIterator(7, _kErrorCodeValues, _kErrorCodeNames), ::apache::thrift::TEnumIterator(-1, NULL, NULL)); +const std::map _ErrorCode_VALUES_TO_NAMES(::apache::thrift::TEnumIterator(21, _kErrorCodeValues, _kErrorCodeNames), ::apache::thrift::TEnumIterator(-1, NULL, NULL)); std::ostream& operator<<(std::ostream& out, const ErrorCode::type& val) { std::map::const_iterator it = _ErrorCode_VALUES_TO_NAMES.find(val); diff --git a/cpp/src/thrift/gen-cpp/milvus_types.h b/cpp/src/thrift/gen-cpp/milvus_types.h index ea4178aa99..8840e94fd9 100644 --- a/cpp/src/thrift/gen-cpp/milvus_types.h +++ b/cpp/src/thrift/gen-cpp/milvus_types.h @@ -23,12 +23,26 @@ namespace milvus { namespace thrift { struct ErrorCode { enum type { SUCCESS = 0, - CONNECT_FAILED = 1, - PERMISSION_DENIED = 2, - TABLE_NOT_EXISTS = 3, - ILLEGAL_ARGUMENT = 4, - ILLEGAL_RANGE = 5, - ILLEGAL_DIMENSION = 6 + UNEXPECTED_ERROR = 1, + CONNECT_FAILED = 2, + PERMISSION_DENIED = 3, + TABLE_NOT_EXISTS = 4, + ILLEGAL_ARGUMENT = 5, + ILLEGAL_RANGE = 6, + ILLEGAL_DIMENSION = 7, + ILLEGAL_INDEX_TYPE = 8, + ILLEGAL_TABLE_NAME = 9, + ILLEGAL_TOPK = 10, + ILLEGAL_ROWRECORD = 11, + ILLEGAL_VECTOR_ID = 12, + ILLEGAL_SEARCH_RESULT = 13, + FILE_NOT_FOUND = 14, + META_FAILED = 15, + CACHE_FAILED = 16, + CANNOT_CREATE_FOLDER = 17, + CANNOT_CREATE_FILE = 18, + CANNOT_DELETE_FOLDER = 19, + CANNOT_DELETE_FILE = 20 }; }; diff --git a/cpp/src/thrift/milvus.thrift b/cpp/src/thrift/milvus.thrift index 48116256e8..88ba223f02 100644 --- a/cpp/src/thrift/milvus.thrift +++ b/cpp/src/thrift/milvus.thrift @@ -15,12 +15,26 @@ namespace netcore milvus.thrift enum ErrorCode { SUCCESS = 0, + UNEXPECTED_ERROR, CONNECT_FAILED, PERMISSION_DENIED, TABLE_NOT_EXISTS, ILLEGAL_ARGUMENT, ILLEGAL_RANGE, ILLEGAL_DIMENSION, + ILLEGAL_INDEX_TYPE, + ILLEGAL_TABLE_NAME, + ILLEGAL_TOPK, + ILLEGAL_ROWRECORD, + ILLEGAL_VECTOR_ID, + ILLEGAL_SEARCH_RESULT, + FILE_NOT_FOUND, + META_FAILED, + CACHE_FAILED, + CANNOT_CREATE_FOLDER, + CANNOT_CREATE_FILE, + CANNOT_DELETE_FOLDER, + CANNOT_DELETE_FILE, } exception Exception { diff --git a/cpp/src/utils/Error.h b/cpp/src/utils/Error.h index bfb19f47a9..8c4da70339 100644 --- a/cpp/src/utils/Error.h +++ b/cpp/src/utils/Error.h @@ -24,18 +24,35 @@ ToGlobalServerErrorCode(const ServerError error_code) { return SERVER_ERROR_CODE_BASE + error_code; } -constexpr ServerError SERVER_UNEXPECTED_ERROR = ToGlobalServerErrorCode(0x001); -constexpr ServerError SERVER_UNSUPPORTED_ERROR = ToGlobalServerErrorCode(0x002); -constexpr ServerError SERVER_NULL_POINTER = ToGlobalServerErrorCode(0x003); -constexpr ServerError SERVER_INVALID_ARGUMENT = ToGlobalServerErrorCode(0x004); -constexpr ServerError SERVER_FILE_NOT_FOUND = ToGlobalServerErrorCode(0x005); -constexpr ServerError SERVER_NOT_IMPLEMENT = ToGlobalServerErrorCode(0x006); -constexpr ServerError SERVER_BLOCKING_QUEUE_EMPTY = ToGlobalServerErrorCode(0x007); -constexpr ServerError SERVER_TABLE_NOT_EXIST = ToGlobalServerErrorCode(0x008); -constexpr ServerError SERVER_INVALID_TIME_RANGE = ToGlobalServerErrorCode(0x009); -constexpr ServerError SERVER_INVALID_VECTOR_DIMENSION = ToGlobalServerErrorCode(0x00a); -constexpr ServerError SERVER_LICENSE_VALIDATION_FAIL = ToGlobalServerErrorCode(0x00b); -constexpr ServerError SERVER_LICENSE_FILE_NOT_EXIST = ToGlobalServerErrorCode(0x00c); +constexpr ServerError SERVER_UNEXPECTED_ERROR = ToGlobalServerErrorCode(1); +constexpr ServerError SERVER_UNSUPPORTED_ERROR = ToGlobalServerErrorCode(2); +constexpr ServerError SERVER_NULL_POINTER = ToGlobalServerErrorCode(3); +constexpr ServerError SERVER_INVALID_ARGUMENT = ToGlobalServerErrorCode(4); +constexpr ServerError SERVER_FILE_NOT_FOUND = ToGlobalServerErrorCode(5); +constexpr ServerError SERVER_NOT_IMPLEMENT = ToGlobalServerErrorCode(6); +constexpr ServerError SERVER_BLOCKING_QUEUE_EMPTY = ToGlobalServerErrorCode(7); +constexpr ServerError SERVER_CANNOT_CREATE_FOLDER = ToGlobalServerErrorCode(8); +constexpr ServerError SERVER_CANNOT_CREATE_FILE = ToGlobalServerErrorCode(9); +constexpr ServerError SERVER_CANNOT_DELETE_FOLDER = ToGlobalServerErrorCode(10); +constexpr ServerError SERVER_CANNOT_DELETE_FILE = ToGlobalServerErrorCode(11); + +constexpr ServerError SERVER_TABLE_NOT_EXIST = ToGlobalServerErrorCode(100); +constexpr ServerError SERVER_INVALID_TABLE_NAME = ToGlobalServerErrorCode(101); +constexpr ServerError SERVER_INVALID_TABLE_DIMENSION = ToGlobalServerErrorCode(102); +constexpr ServerError SERVER_INVALID_TIME_RANGE = ToGlobalServerErrorCode(103); +constexpr ServerError SERVER_INVALID_VECTOR_DIMENSION = ToGlobalServerErrorCode(104); +constexpr ServerError SERVER_INVALID_INDEX_TYPE = ToGlobalServerErrorCode(105); +constexpr ServerError SERVER_INVALID_ROWRECORD = ToGlobalServerErrorCode(106); +constexpr ServerError SERVER_INVALID_ROWRECORD_ARRAY = ToGlobalServerErrorCode(107); +constexpr ServerError SERVER_INVALID_TOPK = ToGlobalServerErrorCode(108); +constexpr ServerError SERVER_ILLEGAL_VECTOR_ID = ToGlobalServerErrorCode(109); +constexpr ServerError SERVER_ILLEGAL_SEARCH_RESULT = ToGlobalServerErrorCode(110); +constexpr ServerError SERVER_CACHE_ERROR = ToGlobalServerErrorCode(111); + +constexpr ServerError SERVER_LICENSE_FILE_NOT_EXIST = ToGlobalServerErrorCode(500); +constexpr ServerError SERVER_LICENSE_VALIDATION_FAIL = ToGlobalServerErrorCode(501); + +constexpr ServerError DB_META_TRANSACTION_FAILED = ToGlobalServerErrorCode(1000); class ServerException : public std::exception { public: diff --git a/cpp/unittest/db/meta_tests.cpp b/cpp/unittest/db/meta_tests.cpp index 7bc9db090f..a7933829c9 100644 --- a/cpp/unittest/db/meta_tests.cpp +++ b/cpp/unittest/db/meta_tests.cpp @@ -17,39 +17,39 @@ using namespace zilliz::milvus::engine; -TEST_F(MetaTest, GROUP_TEST) { - auto table_id = "meta_test_group"; +TEST_F(MetaTest, TABLE_TEST) { + auto table_id = "meta_test_table"; - meta::TableSchema group; - group.table_id_ = table_id; - auto status = impl_->CreateTable(group); + meta::TableSchema table; + table.table_id_ = table_id; + auto status = impl_->CreateTable(table); ASSERT_TRUE(status.ok()); - auto gid = group.id_; - group.id_ = -1; - status = impl_->DescribeTable(group); + auto gid = table.id_; + table.id_ = -1; + status = impl_->DescribeTable(table); ASSERT_TRUE(status.ok()); - ASSERT_EQ(group.id_, gid); - ASSERT_EQ(group.table_id_, table_id); + ASSERT_EQ(table.id_, gid); + ASSERT_EQ(table.table_id_, table_id); - group.table_id_ = "not_found"; - status = impl_->DescribeTable(group); + table.table_id_ = "not_found"; + status = impl_->DescribeTable(table); ASSERT_TRUE(!status.ok()); - group.table_id_ = table_id; - status = impl_->CreateTable(group); - ASSERT_TRUE(!status.ok()); + table.table_id_ = table_id; + status = impl_->CreateTable(table); + ASSERT_TRUE(status.ok()); } -TEST_F(MetaTest, table_file_TEST) { - auto table_id = "meta_test_group"; +TEST_F(MetaTest, TABLE_FILE_TEST) { + auto table_id = "meta_test_table"; - meta::TableSchema group; - group.table_id_ = table_id; - auto status = impl_->CreateTable(group); + meta::TableSchema table; + table.table_id_ = table_id; + auto status = impl_->CreateTable(table); meta::TableFileSchema table_file; - table_file.table_id_ = group.table_id_; + table_file.table_id_ = table.table_id_; status = impl_->CreateTableFile(table_file); ASSERT_TRUE(status.ok()); ASSERT_EQ(table_file.file_type_, meta::TableFileSchema::NEW); @@ -104,15 +104,15 @@ TEST_F(MetaTest, ARCHIVE_TEST_DAYS) { options.archive_conf = ArchiveConf("delete", ss.str()); auto impl = meta::DBMetaImpl(options); - auto table_id = "meta_test_group"; + auto table_id = "meta_test_table"; - meta::TableSchema group; - group.table_id_ = table_id; - auto status = impl.CreateTable(group); + meta::TableSchema table; + table.table_id_ = table_id; + auto status = impl.CreateTable(table); meta::TableFilesSchema files; meta::TableFileSchema table_file; - table_file.table_id_ = group.table_id_; + table_file.table_id_ = table.table_id_; auto cnt = 100; long ts = utils::GetMicroSecTimeStamp(); @@ -156,13 +156,13 @@ TEST_F(MetaTest, ARCHIVE_TEST_DISK) { auto impl = meta::DBMetaImpl(options); auto table_id = "meta_test_group"; - meta::TableSchema group; - group.table_id_ = table_id; - auto status = impl.CreateTable(group); + meta::TableSchema table; + table.table_id_ = table_id; + auto status = impl.CreateTable(table); meta::TableFilesSchema files; meta::TableFileSchema table_file; - table_file.table_id_ = group.table_id_; + table_file.table_id_ = table.table_id_; auto cnt = 10; auto each_size = 2UL; @@ -198,9 +198,9 @@ TEST_F(MetaTest, ARCHIVE_TEST_DISK) { TEST_F(MetaTest, TABLE_FILES_TEST) { auto table_id = "meta_test_group"; - meta::TableSchema group; - group.table_id_ = table_id; - auto status = impl_->CreateTable(group); + meta::TableSchema table; + table.table_id_ = table_id; + auto status = impl_->CreateTable(table); int new_files_cnt = 4; int raw_files_cnt = 5; @@ -208,7 +208,7 @@ TEST_F(MetaTest, TABLE_FILES_TEST) { int index_files_cnt = 7; meta::TableFileSchema table_file; - table_file.table_id_ = group.table_id_; + table_file.table_id_ = table.table_id_; for (auto i=0; iCreateTableFile(table_file); @@ -241,7 +241,7 @@ TEST_F(MetaTest, TABLE_FILES_TEST) { ASSERT_EQ(files.size(), to_index_files_cnt); meta::DatePartionedTableFilesSchema dated_files; - status = impl_->FilesToMerge(group.table_id_, dated_files); + status = impl_->FilesToMerge(table.table_id_, dated_files); ASSERT_TRUE(status.ok()); ASSERT_EQ(dated_files[table_file.date_].size(), raw_files_cnt); From 69cafce6cf90f6295d943de45969ada2f0c1f382 Mon Sep 17 00:00:00 2001 From: starlord Date: Thu, 27 Jun 2019 19:57:11 +0800 Subject: [PATCH 017/189] add more error handling Former-commit-id: 3039ab0d82bd1414f09c99f4edc4c21cfd6682b9 --- cpp/src/server/DBWrapper.cpp | 10 +++++++--- cpp/src/server/MilvusServer.cpp | 4 +++- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/cpp/src/server/DBWrapper.cpp b/cpp/src/server/DBWrapper.cpp index c36bf2621e..af02740829 100644 --- a/cpp/src/server/DBWrapper.cpp +++ b/cpp/src/server/DBWrapper.cpp @@ -40,12 +40,16 @@ DBWrapper::DBWrapper() { opt.meta.archive_conf.SetCriterias(criterial); //create db root folder - CommonUtil::CreateDirectory(opt.meta.path); + ServerError err = CommonUtil::CreateDirectory(opt.meta.path); + if(err != SERVER_SUCCESS) { + std::cout << "ERROR! Failed to create database root path: " << opt.meta.path << std::endl; + kill(0, SIGUSR1); + } zilliz::milvus::engine::DB::Open(opt, &db_); if(db_ == nullptr) { - SERVER_LOG_ERROR << "Failed to open db. Provided database uri = " << opt.meta.backend_uri; - throw ServerException(SERVER_NULL_POINTER, "Failed to open db"); + std::cout << "ERROR! Failed to open database" << std::endl; + kill(0, SIGUSR1); } } diff --git a/cpp/src/server/MilvusServer.cpp b/cpp/src/server/MilvusServer.cpp index 1b1b85e883..6e6d53c565 100644 --- a/cpp/src/server/MilvusServer.cpp +++ b/cpp/src/server/MilvusServer.cpp @@ -24,6 +24,7 @@ #include #include +#include namespace zilliz { namespace milvus { @@ -93,7 +94,8 @@ MilvusServer::StartService() { return; } } catch (apache::thrift::TException& ex) { - SERVER_LOG_ERROR << "Server encounter exception: " << ex.what(); + std::cout << "ERROR! " << ex.what() << std::endl; + kill(0, SIGUSR1); } } From 18600537975f5d71380ed4e136f5a1a2faa54567 Mon Sep 17 00:00:00 2001 From: zhiru Date: Thu, 27 Jun 2019 21:17:07 +0800 Subject: [PATCH 018/189] update Former-commit-id: ec6ccb24c3271ee1d2638fe2cbd31cf3a5113713 --- cpp/src/server/DBWrapper.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/cpp/src/server/DBWrapper.cpp b/cpp/src/server/DBWrapper.cpp index af02740829..1c56ece0e3 100644 --- a/cpp/src/server/DBWrapper.cpp +++ b/cpp/src/server/DBWrapper.cpp @@ -25,7 +25,6 @@ DBWrapper::DBWrapper() { } ConfigNode& serverConfig = ServerConfig::GetInstance().GetConfig(CONFIG_SERVER); opt.mode = serverConfig.GetValue(CONFIG_CLUSTER_MODE, "single"); -// std::cout << "mode = " << opt.mode << std::endl; //set archive config engine::ArchiveConf::CriteriaT criterial; From 623f837548d55c3aff4c24b09c261098c600cae7 Mon Sep 17 00:00:00 2001 From: zhiru Date: Fri, 28 Jun 2019 11:08:10 +0800 Subject: [PATCH 019/189] update Former-commit-id: 82f47e4997edb9d3ce71442ca51f972f05e808a0 --- cpp/src/db/DBImpl.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/src/db/DBImpl.cpp b/cpp/src/db/DBImpl.cpp index c7ee986c96..b028dd3ea3 100644 --- a/cpp/src/db/DBImpl.cpp +++ b/cpp/src/db/DBImpl.cpp @@ -600,7 +600,7 @@ void DBImpl::BackgroundCompaction(std::set table_ids) { int ttl = 1; if (options_.mode == "cluster") { ttl = meta::D_SEC; - ENGINE_LOG_DEBUG << "Server mode is cluster. Clean up files with ttl = " << std::to_string(ttl) << "seconds."; +// ENGINE_LOG_DEBUG << "Server mode is cluster. Clean up files with ttl = " << std::to_string(ttl) << "seconds."; } meta_ptr_->CleanUpFilesWithTTL(ttl); } From f663bdcac35e79a358ef80400011fbebe5795bdf Mon Sep 17 00:00:00 2001 From: zhiru Date: Fri, 28 Jun 2019 15:28:31 +0800 Subject: [PATCH 020/189] update Former-commit-id: 1760ea04488fc283868352dec36d9dda8c4c0fd7 --- cpp/src/db/DBImpl.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/cpp/src/db/DBImpl.cpp b/cpp/src/db/DBImpl.cpp index b028dd3ea3..fec7035c1f 100644 --- a/cpp/src/db/DBImpl.cpp +++ b/cpp/src/db/DBImpl.cpp @@ -139,7 +139,9 @@ DBImpl::DBImpl(const Options& options) meta_ptr_ = DBMetaImplFactory::Build(options.meta); mem_mgr_ = std::make_shared(meta_ptr_, options_); // mem_mgr_ = (MemManagerPtr)(new MemManager(meta_ptr_, options_)); - StartTimerTasks(); + if (options.mode != "read_only") { + StartTimerTasks(); + } } Status DBImpl::CreateTable(meta::TableSchema& table_schema) { From 1b7e0def5efe4ad6a4cb8cb948fc752e0b777569 Mon Sep 17 00:00:00 2001 From: starlord Date: Fri, 28 Jun 2019 10:52:16 +0800 Subject: [PATCH 021/189] change default db path Former-commit-id: 25df16a0e0f747dea0a2b663549f524613bbd4b9 --- cpp/.gitignore | 1 + cpp/CMakeLists.txt | 6 ++++++ cpp/conf/log_config.conf | 2 +- cpp/conf/log_config.template | 27 +++++++++++++++++++++++++++ cpp/conf/server_config.template | 28 ++++++++++++++++++++++++++++ cpp/conf/server_config.yaml | 4 ++-- 6 files changed, 65 insertions(+), 3 deletions(-) create mode 100644 cpp/conf/log_config.template create mode 100644 cpp/conf/server_config.template diff --git a/cpp/.gitignore b/cpp/.gitignore index b9ba83ac8a..d8368bd79b 100644 --- a/cpp/.gitignore +++ b/cpp/.gitignore @@ -1,4 +1,5 @@ milvus/ conf/server_config.yaml +conf/log_config.conf version.h megasearch/ diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index f4558f5303..76283185df 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -142,6 +142,10 @@ endif(BUILD_UNIT_TEST) add_custom_target(Clean-All COMMAND ${CMAKE_BUILD_TOOL} clean) +set(MILVUS_PATH "/tmp/milvus") +configure_file(${CMAKE_CURRENT_SOURCE_DIR}/conf/server_config.template ${CMAKE_CURRENT_SOURCE_DIR}/conf/server_config.yaml) +configure_file(${CMAKE_CURRENT_SOURCE_DIR}/conf/log_config.template ${CMAKE_CURRENT_SOURCE_DIR}/conf/log_config.conf) + #install install(FILES scripts/start_server.sh @@ -154,4 +158,6 @@ install(FILES DESTINATION conf) + + config_summary() diff --git a/cpp/conf/log_config.conf b/cpp/conf/log_config.conf index 79a3965719..29d46a7fe5 100644 --- a/cpp/conf/log_config.conf +++ b/cpp/conf/log_config.conf @@ -24,4 +24,4 @@ FILENAME = "/tmp/milvus/logs/milvus-%datetime{%H:%m}-error.log" * FATAL: ENABLED = true - FILENAME = "/tmp/milvus/logs/milvus-%datetime{%H:%m}-fatal.log" \ No newline at end of file + FILENAME = "/tmp/milvus/logs/milvus-%datetime{%H:%m}-fatal.log" diff --git a/cpp/conf/log_config.template b/cpp/conf/log_config.template new file mode 100644 index 0000000000..ce421bac53 --- /dev/null +++ b/cpp/conf/log_config.template @@ -0,0 +1,27 @@ +* GLOBAL: + FORMAT = "%datetime | %level | %logger | %msg" + FILENAME = "@MILVUS_PATH@/logs/milvus-%datetime{%H:%m}-global.log" + ENABLED = true + TO_FILE = true + TO_STANDARD_OUTPUT = false + SUBSECOND_PRECISION = 3 + PERFORMANCE_TRACKING = false + MAX_LOG_FILE_SIZE = 2097152 ## Throw log files away after 2MB +* DEBUG: + FILENAME = "@MILVUS_PATH@/logs/milvus-%datetime{%H:%m}-debug.log" + ENABLED = true +* WARNING: + FILENAME = "@MILVUS_PATH@/logs/milvus-%datetime{%H:%m}-warning.log" +* TRACE: + FILENAME = "@MILVUS_PATH@/logs/milvus-%datetime{%H:%m}-trace.log" +* VERBOSE: + FORMAT = "%datetime{%d/%M/%y} | %level-%vlevel | %msg" + TO_FILE = false + TO_STANDARD_OUTPUT = false +## Error logs +* ERROR: + ENABLED = true + FILENAME = "@MILVUS_PATH@/logs/milvus-%datetime{%H:%m}-error.log" +* FATAL: + ENABLED = true + FILENAME = "@MILVUS_PATH@/logs/milvus-%datetime{%H:%m}-fatal.log" \ No newline at end of file diff --git a/cpp/conf/server_config.template b/cpp/conf/server_config.template new file mode 100644 index 0000000000..b5f6ee2993 --- /dev/null +++ b/cpp/conf/server_config.template @@ -0,0 +1,28 @@ +server_config: + address: 0.0.0.0 + port: 19530 # the port milvus listen to, default: 19530, range: 1025 ~ 65534 + gpu_index: 0 # the gpu milvus use, default: 0, range: 0 ~ gpu number - 1 + mode: single # milvus deployment type: single, cluster + +db_config: + db_path: @MILVUS_PATH@ # milvus data storage path + db_backend_url: http://127.0.0.1 # meta database uri + index_building_threshold: 1024 # index building trigger threshold, default: 1024, unit: MB + archive_disk_threshold: 512 # triger archive action if storage size exceed this value, unit: GB + archive_days_threshold: 30 # files older than x days will be archived, unit: day + +metric_config: + is_startup: off # if monitoring start: on, off + collector: prometheus # metrics collector: prometheus + prometheus_config: # following are prometheus configure + collect_type: pull # prometheus collect data method + port: 8080 # the port prometheus use to fetch metrics + push_gateway_ip_address: 127.0.0.1 # push method configure: push gateway ip address + push_gateway_port: 9091 # push method configure: push gateway port + + +license_config: # license configure + license_path: "@MILVUS_PATH@/system.license" # license file path + +cache_config: # cache configure + cpu_cache_capacity: 16 # how many memory are used as cache, unit: GB, range: 0 ~ less than total memory \ No newline at end of file diff --git a/cpp/conf/server_config.yaml b/cpp/conf/server_config.yaml index b59fa78b67..6829a44181 100644 --- a/cpp/conf/server_config.yaml +++ b/cpp/conf/server_config.yaml @@ -24,7 +24,7 @@ metric_config: push_gateway_port: 9091 # push method configure: push gateway port license_config: # license configure - license_path: "/tmp/system.license" # license file path + license_path: "/tmp/milvus/system.license" # license file path cache_config: # cache configure - cpu_cache_capacity: 16 # how many memory are used as cache, unit: GB, range: 0 ~ less than total memory \ No newline at end of file + cpu_cache_capacity: 16 # how many memory are used as cache, unit: GB, range: 0 ~ less than total memory From 7964ec43911692a34d14956c6ea8a72ea9a0fb87 Mon Sep 17 00:00:00 2001 From: starlord Date: Fri, 28 Jun 2019 11:06:27 +0800 Subject: [PATCH 022/189] change default db path Former-commit-id: e92ad72765df751ffd7f5b401b8a6daeab301d21 --- cpp/CMakeLists.txt | 7 ++++++- cpp/build.sh | 2 ++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 76283185df..b7428421df 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -142,7 +142,12 @@ endif(BUILD_UNIT_TEST) add_custom_target(Clean-All COMMAND ${CMAKE_BUILD_TOOL} clean) -set(MILVUS_PATH "/tmp/milvus") + +if("${MILVUS_DB_PATHE}" STREQUAL "") + set(MILVUS_PATH "/tmp/milvus") +else() + set(MILVUS_PATH ${MILVUS_DB_PATHE}) +endif() configure_file(${CMAKE_CURRENT_SOURCE_DIR}/conf/server_config.template ${CMAKE_CURRENT_SOURCE_DIR}/conf/server_config.yaml) configure_file(${CMAKE_CURRENT_SOURCE_DIR}/conf/log_config.template ${CMAKE_CURRENT_SOURCE_DIR}/conf/log_config.conf) diff --git a/cpp/build.sh b/cpp/build.sh index a7ece5fee6..de9a8d3205 100755 --- a/cpp/build.sh +++ b/cpp/build.sh @@ -6,6 +6,7 @@ LICENSE_CHECK="OFF" INSTALL_PREFIX=$(pwd)/milvus MAKE_CLEAN="OFF" BUILD_COVERAGE="OFF" +DB_PATH="/opt/milvus" while getopts "p:t:uhlrc" arg do @@ -71,6 +72,7 @@ if [[ ${MAKE_CLEAN} == "ON" ]]; then -DCMAKE_CUDA_COMPILER=${CUDA_COMPILER} \ -DCMAKE_LICENSE_CHECK=${LICENSE_CHECK} \ -DBUILD_COVERAGE=${BUILD_COVERAGE} \ + -DMILVUS_DB_PATHE=${DB_PATH} \ $@ ../" echo ${CMAKE_CMD} From 4ca69f2a0fc02cf39c7c72e506f6dc822f263057 Mon Sep 17 00:00:00 2001 From: starlord Date: Fri, 28 Jun 2019 11:23:12 +0800 Subject: [PATCH 023/189] change default db path Former-commit-id: 88942447c02cc51cd725a7adf7254c13f47d71e5 --- cpp/CMakeLists.txt | 8 +++----- cpp/build.sh | 24 ++++++++++++++---------- cpp/conf/log_config.template | 12 ++++++------ cpp/conf/server_config.template | 4 ++-- 4 files changed, 25 insertions(+), 23 deletions(-) diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index b7428421df..4b95854f77 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -136,17 +136,15 @@ if (BUILD_COVERAGE STREQUAL "ON") endif() -if (BUILD_UNIT_TEST) +if ("${BUILD_UNIT_TEST}" STREQUAL "ON") add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/unittest) endif(BUILD_UNIT_TEST) add_custom_target(Clean-All COMMAND ${CMAKE_BUILD_TOOL} clean) -if("${MILVUS_DB_PATHE}" STREQUAL "") - set(MILVUS_PATH "/tmp/milvus") -else() - set(MILVUS_PATH ${MILVUS_DB_PATHE}) +if("${MILVUS_DB_PATH}" STREQUAL "") + set(MILVUS_DB_PATH "/tmp/milvus") endif() configure_file(${CMAKE_CURRENT_SOURCE_DIR}/conf/server_config.template ${CMAKE_CURRENT_SOURCE_DIR}/conf/server_config.yaml) configure_file(${CMAKE_CURRENT_SOURCE_DIR}/conf/log_config.template ${CMAKE_CURRENT_SOURCE_DIR}/conf/log_config.conf) diff --git a/cpp/build.sh b/cpp/build.sh index de9a8d3205..80be1d7ddb 100755 --- a/cpp/build.sh +++ b/cpp/build.sh @@ -1,14 +1,14 @@ #!/bin/bash BUILD_TYPE="Debug" -BUILD_UNITTEST="off" +BUILD_UNITTEST="OFF" LICENSE_CHECK="OFF" INSTALL_PREFIX=$(pwd)/milvus MAKE_CLEAN="OFF" BUILD_COVERAGE="OFF" DB_PATH="/opt/milvus" -while getopts "p:t:uhlrc" arg +while getopts "p:d:t:uhlrc" arg do case $arg in t) @@ -16,11 +16,14 @@ do ;; u) echo "Build and run unittest cases" ; - BUILD_UNITTEST="on"; + BUILD_UNITTEST="ON"; ;; p) INSTALL_PREFIX=$OPTARG ;; + d) + DB_PATH=$OPTARG + ;; l) LICENSE_CHECK="ON" ;; @@ -37,12 +40,13 @@ do echo " parameter: --t: build type --u: building unit test options --p: install prefix --l: build license version --r: remove previous build directory --c: code coverage +-t: build type(default: Debug) +-u: building unit test options(default: OFF) +-p: install prefix(default: $(pwd)/milvus) +-d: db path(default: /opt/milvus) +-l: build license version(default: OFF) +-r: remove previous build directory(default: OFF) +-c: code coverage(default: OFF) usage: ./build.sh -t \${BUILD_TYPE} [-u] [-h] [-g] [-r] [-c] @@ -72,7 +76,7 @@ if [[ ${MAKE_CLEAN} == "ON" ]]; then -DCMAKE_CUDA_COMPILER=${CUDA_COMPILER} \ -DCMAKE_LICENSE_CHECK=${LICENSE_CHECK} \ -DBUILD_COVERAGE=${BUILD_COVERAGE} \ - -DMILVUS_DB_PATHE=${DB_PATH} \ + -DMILVUS_DB_PATH=${DB_PATH} \ $@ ../" echo ${CMAKE_CMD} diff --git a/cpp/conf/log_config.template b/cpp/conf/log_config.template index ce421bac53..f4f3d3684c 100644 --- a/cpp/conf/log_config.template +++ b/cpp/conf/log_config.template @@ -1,6 +1,6 @@ * GLOBAL: FORMAT = "%datetime | %level | %logger | %msg" - FILENAME = "@MILVUS_PATH@/logs/milvus-%datetime{%H:%m}-global.log" + FILENAME = "@MILVUS_DB_PATH@/logs/milvus-%datetime{%H:%m}-global.log" ENABLED = true TO_FILE = true TO_STANDARD_OUTPUT = false @@ -8,12 +8,12 @@ PERFORMANCE_TRACKING = false MAX_LOG_FILE_SIZE = 2097152 ## Throw log files away after 2MB * DEBUG: - FILENAME = "@MILVUS_PATH@/logs/milvus-%datetime{%H:%m}-debug.log" + FILENAME = "@MILVUS_DB_PATH@/logs/milvus-%datetime{%H:%m}-debug.log" ENABLED = true * WARNING: - FILENAME = "@MILVUS_PATH@/logs/milvus-%datetime{%H:%m}-warning.log" + FILENAME = "@MILVUS_DB_PATH@/logs/milvus-%datetime{%H:%m}-warning.log" * TRACE: - FILENAME = "@MILVUS_PATH@/logs/milvus-%datetime{%H:%m}-trace.log" + FILENAME = "@MILVUS_DB_PATH@/logs/milvus-%datetime{%H:%m}-trace.log" * VERBOSE: FORMAT = "%datetime{%d/%M/%y} | %level-%vlevel | %msg" TO_FILE = false @@ -21,7 +21,7 @@ ## Error logs * ERROR: ENABLED = true - FILENAME = "@MILVUS_PATH@/logs/milvus-%datetime{%H:%m}-error.log" + FILENAME = "@MILVUS_DB_PATH@/logs/milvus-%datetime{%H:%m}-error.log" * FATAL: ENABLED = true - FILENAME = "@MILVUS_PATH@/logs/milvus-%datetime{%H:%m}-fatal.log" \ No newline at end of file + FILENAME = "@MILVUS_DB_PATH@/logs/milvus-%datetime{%H:%m}-fatal.log" \ No newline at end of file diff --git a/cpp/conf/server_config.template b/cpp/conf/server_config.template index b5f6ee2993..1a1c8303f2 100644 --- a/cpp/conf/server_config.template +++ b/cpp/conf/server_config.template @@ -5,7 +5,7 @@ server_config: mode: single # milvus deployment type: single, cluster db_config: - db_path: @MILVUS_PATH@ # milvus data storage path + db_path: @MILVUS_DB_PATH@ # milvus data storage path db_backend_url: http://127.0.0.1 # meta database uri index_building_threshold: 1024 # index building trigger threshold, default: 1024, unit: MB archive_disk_threshold: 512 # triger archive action if storage size exceed this value, unit: GB @@ -22,7 +22,7 @@ metric_config: license_config: # license configure - license_path: "@MILVUS_PATH@/system.license" # license file path + license_path: "@MILVUS_DB_PATH@/system.license" # license file path cache_config: # cache configure cpu_cache_capacity: 16 # how many memory are used as cache, unit: GB, range: 0 ~ less than total memory \ No newline at end of file From 007cde0e0a55f4ee6f6424d5de8a81407aef7121 Mon Sep 17 00:00:00 2001 From: starlord Date: Fri, 28 Jun 2019 11:48:50 +0800 Subject: [PATCH 024/189] change default db path Former-commit-id: a39f8c4976038df464c3ac73ba9d05f0d63443f1 --- cpp/CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/cpp/CHANGELOG.md b/cpp/CHANGELOG.md index be7024672f..95e2eb4f09 100644 --- a/cpp/CHANGELOG.md +++ b/cpp/CHANGELOG.md @@ -24,6 +24,7 @@ Please mark all change in change log and use the ticket from JIRA. - MS-122 - Archive criteria config - MS-124 - HasTable interface - MS-126 - Add more error code +- MS-128 - Change default db path ## New Feature From 0f46d463dfcb1000e06a03238089d17a872b577d Mon Sep 17 00:00:00 2001 From: starlord Date: Fri, 28 Jun 2019 13:42:21 +0800 Subject: [PATCH 025/189] MS-129 Add more unitest Former-commit-id: 6d0b616c8efca6f281aca0a9db82055bcde19abf --- cpp/src/sdk/src/client/ClientProxy.cpp | 1 - cpp/src/sdk/src/util/ConvertUtil.cpp | 44 ---- cpp/src/sdk/src/util/ConvertUtil.h | 18 -- cpp/src/storage/s3/S3ClientWrapper.cpp | 318 ++++++++++++------------- cpp/src/storage/s3/S3ClientWrapper.h | 90 +++---- cpp/unittest/server/cache_test.cpp | 37 ++- cpp/unittest/server/config_test.cpp | 77 ++++++ cpp/unittest/storage/S3ClientTest.cpp | 132 +++++----- 8 files changed, 380 insertions(+), 337 deletions(-) delete mode 100644 cpp/src/sdk/src/util/ConvertUtil.cpp delete mode 100644 cpp/src/sdk/src/util/ConvertUtil.h create mode 100644 cpp/unittest/server/config_test.cpp diff --git a/cpp/src/sdk/src/client/ClientProxy.cpp b/cpp/src/sdk/src/client/ClientProxy.cpp index 1185e4988c..35c774b52b 100644 --- a/cpp/src/sdk/src/client/ClientProxy.cpp +++ b/cpp/src/sdk/src/client/ClientProxy.cpp @@ -4,7 +4,6 @@ * Proprietary and confidential. ******************************************************************************/ #include "ClientProxy.h" -#include "util/ConvertUtil.h" namespace milvus { diff --git a/cpp/src/sdk/src/util/ConvertUtil.cpp b/cpp/src/sdk/src/util/ConvertUtil.cpp deleted file mode 100644 index 867221dc89..0000000000 --- a/cpp/src/sdk/src/util/ConvertUtil.cpp +++ /dev/null @@ -1,44 +0,0 @@ -/******************************************************************************* - * Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved - * Unauthorized copying of this file, via any medium is strictly prohibited. - * Proprietary and confidential. - ******************************************************************************/ -#include "ConvertUtil.h" -#include "Exception.h" - -#include - -namespace milvus { - -static const std::string INDEX_RAW = "raw"; -static const std::string INDEX_IVFFLAT = "ivfflat"; - -std::string ConvertUtil::IndexType2Str(IndexType index) { - static const std::map s_index2str = { - {IndexType::cpu_idmap, INDEX_RAW}, - {IndexType::gpu_ivfflat, INDEX_IVFFLAT} - }; - - const auto& iter = s_index2str.find(index); - if(iter == s_index2str.end()) { - throw Exception(StatusCode::InvalidAgument, "Invalid index type"); - } - - return iter->second; -} - -IndexType ConvertUtil::Str2IndexType(const std::string& type) { - static const std::map s_str2index = { - {INDEX_RAW, IndexType::cpu_idmap}, - {INDEX_IVFFLAT, IndexType::gpu_ivfflat} - }; - - const auto& iter = s_str2index.find(type); - if(iter == s_str2index.end()) { - throw Exception(StatusCode::InvalidAgument, "Invalid index type"); - } - - return iter->second; -} - -} \ No newline at end of file diff --git a/cpp/src/sdk/src/util/ConvertUtil.h b/cpp/src/sdk/src/util/ConvertUtil.h deleted file mode 100644 index 1411f3eb6d..0000000000 --- a/cpp/src/sdk/src/util/ConvertUtil.h +++ /dev/null @@ -1,18 +0,0 @@ -/******************************************************************************* - * Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved - * Unauthorized copying of this file, via any medium is strictly prohibited. - * Proprietary and confidential. - ******************************************************************************/ -#pragma once - -#include "MilvusApi.h" - -namespace milvus { - -class ConvertUtil { -public: - static std::string IndexType2Str(IndexType index); - static IndexType Str2IndexType(const std::string& type); -}; - -} diff --git a/cpp/src/storage/s3/S3ClientWrapper.cpp b/cpp/src/storage/s3/S3ClientWrapper.cpp index fcbab4f9bd..b739bc1e28 100644 --- a/cpp/src/storage/s3/S3ClientWrapper.cpp +++ b/cpp/src/storage/s3/S3ClientWrapper.cpp @@ -1,159 +1,159 @@ -#include "S3ClientWrapper.h" - -#include -#include -#include -#include -#include - -#include -#include - - -namespace zilliz { -namespace milvus { -namespace engine { -namespace storage { - -Status -S3ClientWrapper::Create(const std::string &ip_address, - const std::string &port, - const std::string &access_key, - const std::string &secret_key) { - Aws::InitAPI(options_); - Aws::Client::ClientConfiguration cfg; - - // TODO: ip_address need to be validated. - - cfg.endpointOverride = ip_address + ":" + port; // S3 server ip address and port - cfg.scheme = Aws::Http::Scheme::HTTP; - cfg.verifySSL = - false; //Aws::Auth::AWSCredentials cred("RPW421T9GSIO4A45Y9ZR", "2owKYy9emSS90Q0pXuyqpX1OxBCyEDYodsiBemcq"); // 认证的Key - client_ = - new S3Client(Aws::Auth::AWSCredentials(access_key, secret_key), - cfg, - Aws::Client::AWSAuthV4Signer::PayloadSigningPolicy::Always, - false); - if (client_ == nullptr) { - std::string error = "Can't connect server."; - return Status::Error(error); - } else { - return Status::OK(); - } -} - - -Status -S3ClientWrapper::Close() { - if (client_ != nullptr) { - delete client_; - client_ = nullptr; - } - Aws::ShutdownAPI(options_); - return Status::OK(); -} - -Status -S3ClientWrapper::CreateBucket(std::string& bucket_name) { - Aws::S3::Model::CreateBucketRequest request; - request.SetBucket(bucket_name); - - auto outcome = client_->CreateBucket(request); - - if (outcome.IsSuccess()) - { - return Status::OK(); - } - else - { - std::cout << "CreateBucket error: " - << outcome.GetError().GetExceptionName() << std::endl - << outcome.GetError().GetMessage() << std::endl; - switch(outcome.GetError().GetErrorType()) { - case Aws::S3::S3Errors::BUCKET_ALREADY_EXISTS: - case Aws::S3::S3Errors::BUCKET_ALREADY_OWNED_BY_YOU: - return Status::AlreadyExist(outcome.GetError().GetMessage()); - default: - return Status::Error(outcome.GetError().GetMessage()); - } - } -} - -Status -S3ClientWrapper::DeleteBucket(std::string& bucket_name) { - Aws::S3::Model::DeleteBucketRequest bucket_request; - bucket_request.SetBucket(bucket_name); - - auto outcome = client_->DeleteBucket(bucket_request); - - if (outcome.IsSuccess()) - { - return Status::OK(); - } - else - { - std::cout << "DeleteBucket error: " - << outcome.GetError().GetExceptionName() << " - " - << outcome.GetError().GetMessage() << std::endl; - return Status::Error(outcome.GetError().GetMessage()); - } -} - -Status -S3ClientWrapper::UploadFile(std::string &BucketName, std::string &objectKey, std::string &pathkey) { - - PutObjectRequest putObjectRequest; - putObjectRequest.WithBucket(BucketName.c_str()).WithKey(objectKey.c_str()); - - auto input_data = Aws::MakeShared("PutObjectInputStream", - pathkey.c_str(), - std::ios_base::in | std::ios_base::binary); - putObjectRequest.SetBody(input_data); - auto put_object_result = client_->PutObject(putObjectRequest); - if (put_object_result.IsSuccess()) { - return Status::OK(); - } else { - std::cout << "PutObject error: " << put_object_result.GetError().GetExceptionName() << " " - << put_object_result.GetError().GetMessage() << std::endl; - return Status::Error(put_object_result.GetError().GetMessage()); - } -} - -Status -S3ClientWrapper::DownloadFile(std::string &BucketName, std::string &objectKey, std::string &pathkey) { - GetObjectRequest object_request; - object_request.WithBucket(BucketName.c_str()).WithKey(objectKey.c_str()); - auto get_object_outcome = client_->GetObject(object_request); - if (get_object_outcome.IsSuccess()) { - Aws::OFStream local_file(pathkey.c_str(), std::ios::out | std::ios::binary); - local_file << get_object_outcome.GetResult().GetBody().rdbuf(); - return Status::OK(); - } else { - std::cout << "GetObject error: " << get_object_outcome.GetError().GetExceptionName() << " " - << get_object_outcome.GetError().GetMessage() << std::endl; - return Status::Error(get_object_outcome.GetError().GetMessage()); - } -} - -Status -S3ClientWrapper::DeleteFile(std::string &bucket_name, std::string &object_key) { - Aws::S3::Model::DeleteObjectRequest object_request; - object_request.WithBucket(bucket_name).WithKey(object_key); - - auto delete_object_outcome = client_->DeleteObject(object_request); - - if (delete_object_outcome.IsSuccess()) { - return Status::OK(); - } else { - std::cout << "DeleteObject error: " << - delete_object_outcome.GetError().GetExceptionName() << " " << - delete_object_outcome.GetError().GetMessage() << std::endl; - - return Status::Error(delete_object_outcome.GetError().GetMessage()); - } -} - -} -} -} -} \ No newline at end of file +//#include "S3ClientWrapper.h" +// +//#include +//#include +//#include +//#include +//#include +// +//#include +//#include +// +// +//namespace zilliz { +//namespace milvus { +//namespace engine { +//namespace storage { +// +//Status +//S3ClientWrapper::Create(const std::string &ip_address, +// const std::string &port, +// const std::string &access_key, +// const std::string &secret_key) { +// Aws::InitAPI(options_); +// Aws::Client::ClientConfiguration cfg; +// +// // TODO: ip_address need to be validated. +// +// cfg.endpointOverride = ip_address + ":" + port; // S3 server ip address and port +// cfg.scheme = Aws::Http::Scheme::HTTP; +// cfg.verifySSL = +// false; //Aws::Auth::AWSCredentials cred("RPW421T9GSIO4A45Y9ZR", "2owKYy9emSS90Q0pXuyqpX1OxBCyEDYodsiBemcq"); // 认证的Key +// client_ = +// new S3Client(Aws::Auth::AWSCredentials(access_key, secret_key), +// cfg, +// Aws::Client::AWSAuthV4Signer::PayloadSigningPolicy::Always, +// false); +// if (client_ == nullptr) { +// std::string error = "Can't connect server."; +// return Status::Error(error); +// } else { +// return Status::OK(); +// } +//} +// +// +//Status +//S3ClientWrapper::Close() { +// if (client_ != nullptr) { +// delete client_; +// client_ = nullptr; +// } +// Aws::ShutdownAPI(options_); +// return Status::OK(); +//} +// +//Status +//S3ClientWrapper::CreateBucket(std::string& bucket_name) { +// Aws::S3::Model::CreateBucketRequest request; +// request.SetBucket(bucket_name); +// +// auto outcome = client_->CreateBucket(request); +// +// if (outcome.IsSuccess()) +// { +// return Status::OK(); +// } +// else +// { +// std::cout << "CreateBucket error: " +// << outcome.GetError().GetExceptionName() << std::endl +// << outcome.GetError().GetMessage() << std::endl; +// switch(outcome.GetError().GetErrorType()) { +// case Aws::S3::S3Errors::BUCKET_ALREADY_EXISTS: +// case Aws::S3::S3Errors::BUCKET_ALREADY_OWNED_BY_YOU: +// return Status::AlreadyExist(outcome.GetError().GetMessage()); +// default: +// return Status::Error(outcome.GetError().GetMessage()); +// } +// } +//} +// +//Status +//S3ClientWrapper::DeleteBucket(std::string& bucket_name) { +// Aws::S3::Model::DeleteBucketRequest bucket_request; +// bucket_request.SetBucket(bucket_name); +// +// auto outcome = client_->DeleteBucket(bucket_request); +// +// if (outcome.IsSuccess()) +// { +// return Status::OK(); +// } +// else +// { +// std::cout << "DeleteBucket error: " +// << outcome.GetError().GetExceptionName() << " - " +// << outcome.GetError().GetMessage() << std::endl; +// return Status::Error(outcome.GetError().GetMessage()); +// } +//} +// +//Status +//S3ClientWrapper::UploadFile(std::string &BucketName, std::string &objectKey, std::string &pathkey) { +// +// PutObjectRequest putObjectRequest; +// putObjectRequest.WithBucket(BucketName.c_str()).WithKey(objectKey.c_str()); +// +// auto input_data = Aws::MakeShared("PutObjectInputStream", +// pathkey.c_str(), +// std::ios_base::in | std::ios_base::binary); +// putObjectRequest.SetBody(input_data); +// auto put_object_result = client_->PutObject(putObjectRequest); +// if (put_object_result.IsSuccess()) { +// return Status::OK(); +// } else { +// std::cout << "PutObject error: " << put_object_result.GetError().GetExceptionName() << " " +// << put_object_result.GetError().GetMessage() << std::endl; +// return Status::Error(put_object_result.GetError().GetMessage()); +// } +//} +// +//Status +//S3ClientWrapper::DownloadFile(std::string &BucketName, std::string &objectKey, std::string &pathkey) { +// GetObjectRequest object_request; +// object_request.WithBucket(BucketName.c_str()).WithKey(objectKey.c_str()); +// auto get_object_outcome = client_->GetObject(object_request); +// if (get_object_outcome.IsSuccess()) { +// Aws::OFStream local_file(pathkey.c_str(), std::ios::out | std::ios::binary); +// local_file << get_object_outcome.GetResult().GetBody().rdbuf(); +// return Status::OK(); +// } else { +// std::cout << "GetObject error: " << get_object_outcome.GetError().GetExceptionName() << " " +// << get_object_outcome.GetError().GetMessage() << std::endl; +// return Status::Error(get_object_outcome.GetError().GetMessage()); +// } +//} +// +//Status +//S3ClientWrapper::DeleteFile(std::string &bucket_name, std::string &object_key) { +// Aws::S3::Model::DeleteObjectRequest object_request; +// object_request.WithBucket(bucket_name).WithKey(object_key); +// +// auto delete_object_outcome = client_->DeleteObject(object_request); +// +// if (delete_object_outcome.IsSuccess()) { +// return Status::OK(); +// } else { +// std::cout << "DeleteObject error: " << +// delete_object_outcome.GetError().GetExceptionName() << " " << +// delete_object_outcome.GetError().GetMessage() << std::endl; +// +// return Status::Error(delete_object_outcome.GetError().GetMessage()); +// } +//} +// +//} +//} +//} +//} \ No newline at end of file diff --git a/cpp/src/storage/s3/S3ClientWrapper.h b/cpp/src/storage/s3/S3ClientWrapper.h index 9ef60174cf..fc66304a5a 100644 --- a/cpp/src/storage/s3/S3ClientWrapper.h +++ b/cpp/src/storage/s3/S3ClientWrapper.h @@ -1,45 +1,45 @@ -#pragma once - -#include "storage/IStorage.h" - - -#include -#include -#include - - -using namespace Aws::S3; -using namespace Aws::S3::Model; - -namespace zilliz { -namespace milvus { -namespace engine { -namespace storage { - -class S3ClientWrapper : public IStorage { - public: - - S3ClientWrapper() = default; - ~S3ClientWrapper() = default; - - Status Create(const std::string &ip_address, - const std::string &port, - const std::string &access_key, - const std::string &secret_key) override; - Status Close() override; - - Status CreateBucket(std::string& bucket_name) override; - Status DeleteBucket(std::string& bucket_name) override; - Status UploadFile(std::string &BucketName, std::string &objectKey, std::string &pathkey) override; - Status DownloadFile(std::string &BucketName, std::string &objectKey, std::string &pathkey) override; - Status DeleteFile(std::string &bucket_name, std::string &object_key) override; - - private: - S3Client *client_ = nullptr; - Aws::SDKOptions options_; -}; - -} -} -} -} \ No newline at end of file +//#pragma once +// +//#include "storage/IStorage.h" +// +// +//#include +//#include +//#include +// +// +//using namespace Aws::S3; +//using namespace Aws::S3::Model; +// +//namespace zilliz { +//namespace milvus { +//namespace engine { +//namespace storage { +// +//class S3ClientWrapper : public IStorage { +// public: +// +// S3ClientWrapper() = default; +// ~S3ClientWrapper() = default; +// +// Status Create(const std::string &ip_address, +// const std::string &port, +// const std::string &access_key, +// const std::string &secret_key) override; +// Status Close() override; +// +// Status CreateBucket(std::string& bucket_name) override; +// Status DeleteBucket(std::string& bucket_name) override; +// Status UploadFile(std::string &BucketName, std::string &objectKey, std::string &pathkey) override; +// Status DownloadFile(std::string &BucketName, std::string &objectKey, std::string &pathkey) override; +// Status DeleteFile(std::string &bucket_name, std::string &object_key) override; +// +// private: +// S3Client *client_ = nullptr; +// Aws::SDKOptions options_; +//}; +// +//} +//} +//} +//} \ No newline at end of file diff --git a/cpp/unittest/server/cache_test.cpp b/cpp/unittest/server/cache_test.cpp index 98c06f788c..05c292b11b 100644 --- a/cpp/unittest/server/cache_test.cpp +++ b/cpp/unittest/server/cache_test.cpp @@ -11,18 +11,18 @@ using namespace zilliz::milvus; -TEST(CacheTest, CACHE_TEST) { - cache::CacheMgr* cpu_mgr = cache::CpuCacheMgr::GetInstance(); +TEST(CacheTest, CPU_CACHE_TEST) { + cache::CacheMgr *cpu_mgr = cache::CpuCacheMgr::GetInstance(); const int64_t gbyte = 1 << 30; int64_t g_num = 16; - int64_t cap = g_num*gbyte; + int64_t cap = g_num * gbyte; cpu_mgr->SetCapacity(cap); ASSERT_EQ(cpu_mgr->CacheCapacity(), cap); const int dim = 256; - for(int i = 0; i < 20; i++) { + for (int i = 0; i < 20; i++) { std::shared_ptr raw_index(faiss::index_factory(dim, "IDMap,Flat")); engine::Index_ptr index = std::make_shared(raw_index); index->ntotal = 1000000;//less 1G per index @@ -31,6 +31,12 @@ TEST(CacheTest, CACHE_TEST) { } ASSERT_LT(cpu_mgr->ItemCount(), g_num); + auto obj = cpu_mgr->GetIndex("index_0"); + ASSERT_TRUE(obj == nullptr); + + obj = cpu_mgr->GetIndex("index_19"); + ASSERT_TRUE(obj != nullptr); + { std::string item = "index_15"; ASSERT_TRUE(cpu_mgr->ItemExists(item)); @@ -49,4 +55,27 @@ TEST(CacheTest, CACHE_TEST) { cpu_mgr->InsertItem("index_6g", index); ASSERT_EQ(cpu_mgr->ItemCount(), 0);//data greater than capacity can not be inserted sucessfully } + + cpu_mgr->PrintInfo(); +} + +TEST(CacheTest, GPU_CACHE_TEST) { + cache::CacheMgr* gpu_mgr = cache::GpuCacheMgr::GetInstance(); + + const int dim = 256; + + for(int i = 0; i < 20; i++) { + std::shared_ptr raw_index(faiss::index_factory(dim, "IDMap,Flat")); + engine::Index_ptr index = std::make_shared(raw_index); + index->ntotal = 1000; + + cache::DataObjPtr obj = std::make_shared(index); + + gpu_mgr->InsertItem("index_" + std::to_string(i), obj); + } + + auto obj = gpu_mgr->GetItem("index_0"); + + gpu_mgr->ClearCache(); + ASSERT_EQ(gpu_mgr->ItemCount(), 0); } \ No newline at end of file diff --git a/cpp/unittest/server/config_test.cpp b/cpp/unittest/server/config_test.cpp new file mode 100644 index 0000000000..4f76707dd2 --- /dev/null +++ b/cpp/unittest/server/config_test.cpp @@ -0,0 +1,77 @@ +//////////////////////////////////////////////////////////////////////////////// +// Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved +// Unauthorized copying of this file, via any medium is strictly prohibited. +// Proprietary and confidential. +//////////////////////////////////////////////////////////////////////////////// +#include + +#include "config/IConfigMgr.h" +#include "server/ServerConfig.h" + +using namespace zilliz::milvus; + +static const std::string CONFIG_FILE_PATH = "../../../conf/server_config.yaml"; + +TEST(ConfigTest, CONFIG_TEST) { + server::IConfigMgr* config_mgr = server::IConfigMgr::GetInstance(); + server::ServerError err = config_mgr->LoadConfigFile(CONFIG_FILE_PATH); + ASSERT_EQ(err, server::SERVER_SUCCESS); + + server::ConfigNode& root_config = config_mgr->GetRootNode(); + server::ConfigNode& server_config = root_config.GetChild("server_config"); + server::ConfigNode& db_config = root_config.GetChild("db_config"); + server::ConfigNode& metric_config = root_config.GetChild("metric_config"); + server::ConfigNode& cache_config = root_config.GetChild("cache_config"); + + std::string address = server_config.GetValue("address"); + ASSERT_TRUE(!address.empty()); + int64_t port = server_config.GetInt64Value("port"); + ASSERT_TRUE(port != 0); + + server_config.SetValue("test", "2.5"); + double test = server_config.GetDoubleValue("test"); + ASSERT_EQ(test, 2.5); + + server::ConfigNode fake; + server_config.AddChild("fake", fake); + fake = server_config.GetChild("fake"); + server::ConfigNodeArr arr; + server_config.GetChildren(arr); + ASSERT_EQ(arr.size(), 1UL); + + server_config.ClearChildren(); + auto children = server_config.GetChildren(); + ASSERT_TRUE(children.empty()); + + root_config.PrintAll(); + std::string all = root_config.DumpString(); + ASSERT_TRUE(!all.empty()); + + server_config.ClearConfig(); + auto configs = server_config.GetConfig(); + ASSERT_TRUE(configs.empty()); + + server_config.AddSequenceItem("seq", "aaa"); + server_config.AddSequenceItem("seq", "bbb"); + auto seq = server_config.GetSequence("seq"); + ASSERT_EQ(seq.size(), 2UL); + + server_config.ClearSequences(); + auto seqs = server_config.GetSequences(); + ASSERT_TRUE(seqs.empty()); +} + +TEST(ConfigTest, SERVER_CONFIG_TEST) { + server::ServerConfig& config = server::ServerConfig::GetInstance(); + server::ServerError err = config.LoadConfigFile(CONFIG_FILE_PATH); + ASSERT_EQ(err, server::SERVER_SUCCESS); + + server::ConfigNode node1 = config.GetConfig("server_config"); + server::ConfigNode& node2 = config.GetConfig("cache_config"); + node1.Combine(node2); + + int32_t cap = node1.GetInt32Value("cpu_cache_capacity"); + ASSERT_GT(cap, 0); + + config.PrintAll(); +} \ No newline at end of file diff --git a/cpp/unittest/storage/S3ClientTest.cpp b/cpp/unittest/storage/S3ClientTest.cpp index c83209ecf6..5147f701ae 100644 --- a/cpp/unittest/storage/S3ClientTest.cpp +++ b/cpp/unittest/storage/S3ClientTest.cpp @@ -1,66 +1,66 @@ -//////////////////////////////////////////////////////////////////////////////// -// Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved -// Unauthorized copying of this file, via any medium is strictly prohibited. -// Proprietary and confidential. -//////////////////////////////////////////////////////////////////////////////// - -#include "storage/IStorage.h" -#include "storage/s3/S3ClientWrapper.h" -#include -#include -#include - - -using namespace zilliz::milvus::engine; - -TEST(s3_client_wrapper, CLIENT_WRAPPER_TEST) { - - std::shared_ptr storage_ptr = std::make_shared(); - - std::string ip_address = "127.0.0.1"; - std::string port = "9000"; - std::string access_key = "AKIAIOSFODNN7EXAMPLE"; - std::string secret_key = "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"; - - Status status = storage_ptr->Create(ip_address, port, access_key, secret_key); - ASSERT_TRUE(status.ok()); - - std::string filename = "/tmp/s3_test_file"; - std::string bucket_name = "bucktname"; - std::string object_name = "test_file"; - - status = storage_ptr->CreateBucket(bucket_name); - std::cout << status.IsAlreadyExist() << std::endl; - if (status.IsAlreadyExist()) { - status = storage_ptr->DeleteBucket(bucket_name); - status = storage_ptr->CreateBucket(bucket_name); - } - - ASSERT_TRUE(status.ok()); - - std::ofstream ofile(filename); - std::stringstream ss; - for (int i = 0; i < 1024; ++i) { - ss << i; - } - ofile << ss.str() << std::endl; - ofile.close(); - status = storage_ptr->UploadFile(bucket_name, object_name, filename); - ASSERT_TRUE(status.ok()); - - status = storage_ptr->DownloadFile(bucket_name, object_name, filename); - std::ifstream infile(filename); - std::string in_buffer; - infile >> in_buffer; - ASSERT_STREQ(in_buffer.c_str(), ss.str().c_str()); - - status = storage_ptr->DeleteFile(bucket_name, object_name); - ASSERT_TRUE(status.ok()); - - status = storage_ptr->DeleteBucket(bucket_name); - ASSERT_TRUE(status.ok()); - - status = storage_ptr->Close(); - ASSERT_TRUE(status.ok()); -} - +////////////////////////////////////////////////////////////////////////////////// +//// Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved +//// Unauthorized copying of this file, via any medium is strictly prohibited. +//// Proprietary and confidential. +////////////////////////////////////////////////////////////////////////////////// +// +//#include "storage/IStorage.h" +//#include "storage/s3/S3ClientWrapper.h" +//#include +//#include +//#include +// +// +//using namespace zilliz::milvus::engine; +// +//TEST(s3_client_wrapper, CLIENT_WRAPPER_TEST) { +// +// std::shared_ptr storage_ptr = std::make_shared(); +// +// std::string ip_address = "127.0.0.1"; +// std::string port = "9000"; +// std::string access_key = "AKIAIOSFODNN7EXAMPLE"; +// std::string secret_key = "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"; +// +// Status status = storage_ptr->Create(ip_address, port, access_key, secret_key); +// ASSERT_TRUE(status.ok()); +// +// std::string filename = "/tmp/s3_test_file"; +// std::string bucket_name = "bucktname"; +// std::string object_name = "test_file"; +// +// status = storage_ptr->CreateBucket(bucket_name); +// std::cout << status.IsAlreadyExist() << std::endl; +// if (status.IsAlreadyExist()) { +// status = storage_ptr->DeleteBucket(bucket_name); +// status = storage_ptr->CreateBucket(bucket_name); +// } +// +// ASSERT_TRUE(status.ok()); +// +// std::ofstream ofile(filename); +// std::stringstream ss; +// for (int i = 0; i < 1024; ++i) { +// ss << i; +// } +// ofile << ss.str() << std::endl; +// ofile.close(); +// status = storage_ptr->UploadFile(bucket_name, object_name, filename); +// ASSERT_TRUE(status.ok()); +// +// status = storage_ptr->DownloadFile(bucket_name, object_name, filename); +// std::ifstream infile(filename); +// std::string in_buffer; +// infile >> in_buffer; +// ASSERT_STREQ(in_buffer.c_str(), ss.str().c_str()); +// +// status = storage_ptr->DeleteFile(bucket_name, object_name); +// ASSERT_TRUE(status.ok()); +// +// status = storage_ptr->DeleteBucket(bucket_name); +// ASSERT_TRUE(status.ok()); +// +// status = storage_ptr->Close(); +// ASSERT_TRUE(status.ok()); +//} +// From bc8f7c8c5e9e6c62cd039ee373d1be869b09b2d8 Mon Sep 17 00:00:00 2001 From: starlord Date: Fri, 28 Jun 2019 15:05:01 +0800 Subject: [PATCH 026/189] MS-129 Add more unitest Former-commit-id: 8103255d0c3725d3fa58a35f167748788da69c79 --- cpp/src/server/RequestTask.h | 1 - cpp/src/utils/AttributeSerializer.cpp | 50 ------------ cpp/src/utils/AttributeSerializer.h | 27 ------- cpp/src/utils/StringHelpFunctions.cpp | 7 -- cpp/src/utils/StringHelpFunctions.h | 2 - cpp/unittest/server/CMakeLists.txt | 7 +- cpp/unittest/server/common_test.cpp | 26 ------- cpp/unittest/server/util_test.cpp | 106 +++++++++++++++++++++----- 8 files changed, 91 insertions(+), 135 deletions(-) delete mode 100644 cpp/src/utils/AttributeSerializer.cpp delete mode 100644 cpp/src/utils/AttributeSerializer.h delete mode 100644 cpp/unittest/server/common_test.cpp diff --git a/cpp/src/server/RequestTask.h b/cpp/src/server/RequestTask.h index 3061b3b75d..4bf9f964e0 100644 --- a/cpp/src/server/RequestTask.h +++ b/cpp/src/server/RequestTask.h @@ -7,7 +7,6 @@ #include "RequestScheduler.h" #include "utils/Error.h" -#include "utils/AttributeSerializer.h" #include "db/Types.h" #include "milvus_types.h" diff --git a/cpp/src/utils/AttributeSerializer.cpp b/cpp/src/utils/AttributeSerializer.cpp deleted file mode 100644 index b351262e60..0000000000 --- a/cpp/src/utils/AttributeSerializer.cpp +++ /dev/null @@ -1,50 +0,0 @@ -/******************************************************************************* - * Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved - * Unauthorized copying of this file, via any medium is strictly prohibited. - * Proprietary and confidential. - ******************************************************************************/ - -#include "AttributeSerializer.h" -#include "StringHelpFunctions.h" - -namespace zilliz { -namespace milvus { -namespace server { - - -ServerError AttributeSerializer::Encode(const AttribMap& attrib_map, std::string& attrib_str) { - attrib_str = ""; - for(auto iter : attrib_map) { - attrib_str += iter.first; - attrib_str += ":\""; - attrib_str += iter.second; - attrib_str += "\";"; - } - - return SERVER_SUCCESS; -} - -ServerError AttributeSerializer::Decode(const std::string& attrib_str, AttribMap& attrib_map) { - attrib_map.clear(); - - std::vector kv_pairs; - StringHelpFunctions::SplitStringByQuote(attrib_str, ";", "\"", kv_pairs); - for(std::string& str : kv_pairs) { - std::string key, val; - size_t index = str.find_first_of(":", 0); - if (index != std::string::npos) { - key = str.substr(0, index); - val = str.substr(index + 1); - } else { - key = str; - } - - attrib_map.insert(std::make_pair(key, val)); - } - - return SERVER_SUCCESS; -} - -} -} -} diff --git a/cpp/src/utils/AttributeSerializer.h b/cpp/src/utils/AttributeSerializer.h deleted file mode 100644 index 8d0341aa2e..0000000000 --- a/cpp/src/utils/AttributeSerializer.h +++ /dev/null @@ -1,27 +0,0 @@ -/******************************************************************************* - * Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved - * Unauthorized copying of this file, via any medium is strictly prohibited. - * Proprietary and confidential. - ******************************************************************************/ -#pragma once - -#include - -#include "Error.h" - -namespace zilliz { -namespace milvus { -namespace server { - -using AttribMap = std::map; - -class AttributeSerializer { -public: - static ServerError Encode(const AttribMap& attrib_map, std::string& attrib_str); - static ServerError Decode(const std::string& attrib_str, AttribMap& attrib_map); -}; - - -} -} -} diff --git a/cpp/src/utils/StringHelpFunctions.cpp b/cpp/src/utils/StringHelpFunctions.cpp index aaefd7236a..36ba017beb 100644 --- a/cpp/src/utils/StringHelpFunctions.cpp +++ b/cpp/src/utils/StringHelpFunctions.cpp @@ -9,13 +9,6 @@ namespace zilliz { namespace milvus { namespace server { -void StringHelpFunctions::TrimStringLineBreak(std::string &string) { - if (!string.empty()) { - static std::string s_format("\n\r"); - string.erase(string.find_last_not_of(s_format) + 1); - } -} - void StringHelpFunctions::TrimStringBlank(std::string &string) { if (!string.empty()) { static std::string s_format(" \n\r\t"); diff --git a/cpp/src/utils/StringHelpFunctions.h b/cpp/src/utils/StringHelpFunctions.h index 2521c48e22..b5d8ce22b7 100644 --- a/cpp/src/utils/StringHelpFunctions.h +++ b/cpp/src/utils/StringHelpFunctions.h @@ -18,8 +18,6 @@ private: StringHelpFunctions() = default; public: - static void TrimStringLineBreak(std::string &string); - static void TrimStringBlank(std::string &string); static void TrimStringQuote(std::string &string, const std::string &qoute); diff --git a/cpp/unittest/server/CMakeLists.txt b/cpp/unittest/server/CMakeLists.txt index d844b7a9a3..4d34f0e2b7 100644 --- a/cpp/unittest/server/CMakeLists.txt +++ b/cpp/unittest/server/CMakeLists.txt @@ -14,9 +14,10 @@ aux_source_directory(../../src/cache cache_srcs) aux_source_directory(../../src/wrapper wrapper_src) aux_source_directory(./ test_srcs) -set(server_src_files +set(utils_srcs ${MILVUS_ENGINE_SRC}/utils/StringHelpFunctions.cpp - ${MILVUS_ENGINE_SRC}/utils/AttributeSerializer.cpp + ${MILVUS_ENGINE_SRC}/utils/TimeRecorder.cpp + ${MILVUS_ENGINE_SRC}/utils/CommonUtil.cpp ) cuda_add_executable(server_test @@ -25,7 +26,7 @@ cuda_add_executable(server_test ${cache_srcs} ${wrapper_src} ${test_srcs} - ${server_src_files} + ${utils_srcs} ${require_files} ) diff --git a/cpp/unittest/server/common_test.cpp b/cpp/unittest/server/common_test.cpp deleted file mode 100644 index 27d8c522e5..0000000000 --- a/cpp/unittest/server/common_test.cpp +++ /dev/null @@ -1,26 +0,0 @@ -//////////////////////////////////////////////////////////////////////////////// -// Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved -// Unauthorized copying of this file, via any medium is strictly prohibited. -// Proprietary and confidential. -//////////////////////////////////////////////////////////////////////////////// -#include -#include "utils/CommonUtil.h" -#include "utils/Error.h" - -using namespace zilliz::milvus; - - -TEST(CommonTest, COMMON_TEST) { - std::string path1 = "/tmp/milvus_test/"; - std::string path2 = path1 + "common_test_12345/"; - std::string path3 = path2 + "abcdef"; - server::ServerError err = server::CommonUtil::CreateDirectory(path3); - ASSERT_EQ(err, server::SERVER_SUCCESS); - - ASSERT_TRUE(server::CommonUtil::IsDirectoryExit(path3)); - - err = server::CommonUtil::DeleteDirectory(path1); - ASSERT_EQ(err, server::SERVER_SUCCESS); - - ASSERT_FALSE(server::CommonUtil::IsDirectoryExit(path1)); -} diff --git a/cpp/unittest/server/util_test.cpp b/cpp/unittest/server/util_test.cpp index 0bc951d88e..9a38092799 100644 --- a/cpp/unittest/server/util_test.cpp +++ b/cpp/unittest/server/util_test.cpp @@ -4,31 +4,99 @@ // Proprietary and confidential. //////////////////////////////////////////////////////////////////////////////// #include +#include -#include "utils/AttributeSerializer.h" +#include "utils/CommonUtil.h" +#include "utils/Error.h" #include "utils/StringHelpFunctions.h" +#include "utils/TimeRecorder.h" +#include "utils/BlockingQueue.h" using namespace zilliz::milvus; -TEST(AttribSerializeTest, ATTRIBSERIAL_TEST) { - std::map attrib; - attrib["uid"] = "ABCDEF"; - attrib["color"] = "red"; - attrib["number"] = "9900"; - attrib["comment"] = "please note: it is a car, not a ship"; - attrib["address"] = " china;shanghai "; +namespace { - std::string attri_str; - server::AttributeSerializer::Encode(attrib, attri_str); - - std::map attrib_out; - server::ServerError err = server::AttributeSerializer::Decode(attri_str, attrib_out); - ASSERT_EQ(err, server::SERVER_SUCCESS); - - ASSERT_EQ(attrib_out.size(), attrib.size()); - for(auto iter : attrib) { - ASSERT_EQ(attrib_out[iter.first], attrib_out[iter.first]); - } +using TimeUnit = server::TimeRecorder::TimeDisplayUnit; +double TestTimeRecorder(TimeUnit unit, int64_t log_level) { + server::TimeRecorder rc("test rc", unit, log_level); + rc.Record("begin"); + std::this_thread::sleep_for(std::chrono::microseconds(10)); + rc.Elapse("end"); + return rc.Span(); +} } +TEST(CommonTest, COMMON_TEST) { + std::string path1 = "/tmp/milvus_test/"; + std::string path2 = path1 + "common_test_12345/"; + std::string path3 = path2 + "abcdef"; + server::ServerError err = server::CommonUtil::CreateDirectory(path3); + ASSERT_EQ(err, server::SERVER_SUCCESS); + + ASSERT_TRUE(server::CommonUtil::IsDirectoryExit(path3)); + + err = server::CommonUtil::DeleteDirectory(path1); + ASSERT_EQ(err, server::SERVER_SUCCESS); + + ASSERT_FALSE(server::CommonUtil::IsDirectoryExit(path1)); +} + +TEST(UtilTest, STRINGFUNCTIONS_TEST) { + std::string str = " test zilliz"; + server::StringHelpFunctions::TrimStringBlank(str); + ASSERT_EQ(str, "test zilliz"); + + str = "a,b,c"; + std::vector result; + server::StringHelpFunctions::SplitStringByDelimeter(str , ",", result); + ASSERT_EQ(result.size(), 3UL); + + str = "55,\"aa,gg,yy\",b"; + result.clear(); + server::StringHelpFunctions::SplitStringByQuote(str , ",", "\"", result); + ASSERT_EQ(result.size(), 3UL); +} + +TEST(UtilTest, TIMERECORDER_TEST) { + double span = TestTimeRecorder(TimeUnit::eTimeAutoUnit, 0); + ASSERT_GT(span, 0.0); + span = TestTimeRecorder(TimeUnit::eTimeHourUnit, 1); + ASSERT_GT(span, 0.0); + span = TestTimeRecorder(TimeUnit::eTimeMinuteUnit, 2); + ASSERT_GT(span, 0.0); + span = TestTimeRecorder(TimeUnit::eTimeSecondUnit, 3); + ASSERT_GT(span, 0.0); + span = TestTimeRecorder(TimeUnit::eTimeMilliSecUnit, 4); + ASSERT_GT(span, 0.0); + span = TestTimeRecorder(TimeUnit::eTimeMicroSecUnit, -1); + ASSERT_GT(span, 0.0); +} + +TEST(UtilTest, BLOCKINGQUEUE_TEST) { + server::BlockingQueue bq; + + static const size_t count = 10; + bq.SetCapacity(count); + + for(size_t i = 1; i <= count; i++) { + std::string id = "No." + std::to_string(i); + bq.Put(id); + } + + ASSERT_EQ(bq.Size(), count); + ASSERT_FALSE(bq.Empty()); + + std::string str = bq.Front(); + ASSERT_EQ(str, "No.1"); + + str = bq.Back(); + ASSERT_EQ(str, "No." + std::to_string(count)); + + for(size_t i = 1; i <= count; i++) { + std::string id = "No." + std::to_string(i); + str = bq.Take(); + ASSERT_EQ(id, str); + } +} + From 394087fd3449a696ef1d9d9da594b6b6761a7bec Mon Sep 17 00:00:00 2001 From: zhiru Date: Sun, 30 Jun 2019 10:43:06 +0800 Subject: [PATCH 027/189] update Former-commit-id: 471118fe238938668b89db35a46db446a21758a3 --- cpp/conf/server_config.yaml | 4 ++-- cpp/src/db/DBImpl.cpp | 2 +- cpp/src/db/Factories.cpp | 13 +++++++------ cpp/src/db/Factories.h | 2 +- cpp/src/db/MySQLMetaImpl.cpp | 17 +++++++++++++---- cpp/src/db/MySQLMetaImpl.h | 3 ++- cpp/src/server/DBWrapper.cpp | 4 ++++ 7 files changed, 30 insertions(+), 15 deletions(-) diff --git a/cpp/conf/server_config.yaml b/cpp/conf/server_config.yaml index 6829a44181..35f3747a07 100644 --- a/cpp/conf/server_config.yaml +++ b/cpp/conf/server_config.yaml @@ -1,8 +1,8 @@ server_config: address: 0.0.0.0 - port: 19531 # the port milvus listen to, default: 19530, range: 1025 ~ 65534 + port: 19530 # the port milvus listen to, default: 19530, range: 1025 ~ 65534 gpu_index: 0 # the gpu milvus use, default: 0, range: 0 ~ gpu number - 1 - mode: cluster # milvus deployment type: single, cluster + mode: single # milvus deployment type: single, cluster, read_only db_config: db_path: /tmp/milvus diff --git a/cpp/src/db/DBImpl.cpp b/cpp/src/db/DBImpl.cpp index fec7035c1f..def216358b 100644 --- a/cpp/src/db/DBImpl.cpp +++ b/cpp/src/db/DBImpl.cpp @@ -136,7 +136,7 @@ DBImpl::DBImpl(const Options& options) shutting_down_(false), compact_thread_pool_(1, 1), index_thread_pool_(1, 1) { - meta_ptr_ = DBMetaImplFactory::Build(options.meta); + meta_ptr_ = DBMetaImplFactory::Build(options.meta, options.mode); mem_mgr_ = std::make_shared(meta_ptr_, options_); // mem_mgr_ = (MemManagerPtr)(new MemManager(meta_ptr_, options_)); if (options.mode != "read_only") { diff --git a/cpp/src/db/Factories.cpp b/cpp/src/db/Factories.cpp index a6998d10dc..97e89e0994 100644 --- a/cpp/src/db/Factories.cpp +++ b/cpp/src/db/Factories.cpp @@ -53,7 +53,8 @@ std::shared_ptr DBMetaImplFactory::Build() { return std::shared_ptr(new meta::DBMetaImpl(options)); } -std::shared_ptr DBMetaImplFactory::Build(const DBMetaOptions& metaOptions) { +std::shared_ptr DBMetaImplFactory::Build(const DBMetaOptions& metaOptions, + const std::string& mode) { std::string uri = metaOptions.backend_uri; // if (uri.empty()) { @@ -81,21 +82,21 @@ std::shared_ptr DBMetaImplFactory::Build(const DBMetaOptions& metaOp std::string dialect = pieces_match[1].str(); std::transform(dialect.begin(), dialect.end(), dialect.begin(), ::tolower); if (dialect.find("mysql") != std::string::npos) { - ENGINE_LOG_DEBUG << "Using MySQL"; - return std::make_shared(meta::MySQLMetaImpl(metaOptions)); + ENGINE_LOG_INFO << "Using MySQL"; + return std::make_shared(meta::MySQLMetaImpl(metaOptions, mode)); } else if (dialect.find("sqlite") != std::string::npos) { ENGINE_LOG_DEBUG << "Using SQLite"; return std::make_shared(meta::DBMetaImpl(metaOptions)); } else { - LOG(ERROR) << "Invalid dialect in URI: dialect = " << dialect; + ENGINE_LOG_ERROR << "Invalid dialect in URI: dialect = " << dialect; throw InvalidArgumentException("URI dialect is not mysql / sqlite"); } } else { - LOG(ERROR) << "Wrong URI format: URI = " << uri; - throw InvalidArgumentException("Wrong URI format"); + ENGINE_LOG_ERROR << "Wrong URI format: URI = " << uri; + throw InvalidArgumentException("Wrong URI format "); } } diff --git a/cpp/src/db/Factories.h b/cpp/src/db/Factories.h index 31afd2b5ba..48bea9b291 100644 --- a/cpp/src/db/Factories.h +++ b/cpp/src/db/Factories.h @@ -28,7 +28,7 @@ struct OptionsFactory { struct DBMetaImplFactory { static std::shared_ptr Build(); - static std::shared_ptr Build(const DBMetaOptions& metaOptions); + static std::shared_ptr Build(const DBMetaOptions& metaOptions, const std::string& mode); }; struct DBFactory { diff --git a/cpp/src/db/MySQLMetaImpl.cpp b/cpp/src/db/MySQLMetaImpl.cpp index 5c0725ef93..7ee9231b41 100644 --- a/cpp/src/db/MySQLMetaImpl.cpp +++ b/cpp/src/db/MySQLMetaImpl.cpp @@ -103,8 +103,9 @@ namespace meta { return Status::OK(); } - MySQLMetaImpl::MySQLMetaImpl(const DBMetaOptions &options_) - : options_(options_) { + MySQLMetaImpl::MySQLMetaImpl(const DBMetaOptions &options_, const std::string& mode) + : options_(options_), + mode_(mode) { Initialize(); } @@ -424,6 +425,14 @@ namespace meta { } } //Scoped Connection + + +// ConfigNode& serverConfig = ServerConfig::GetInstance().GetConfig(CONFIG_SERVER); +// opt.mode = serverConfig.GetValue(CONFIG_CLUSTER_MODE, "single"); + if (mode_ != "single") { + DeleteTableFiles(table_id); + } + } catch (const BadQuery& er) { // Handle any query errors ENGINE_LOG_ERROR << "GENERAL ERROR WHEN DELETING TABLE" << ": " << er.what(); @@ -448,10 +457,10 @@ namespace meta { Query deleteTableFilesQuery = connectionPtr->query(); // deleteTableFilesQuery << "UPDATE TableFiles " << - "SET file_type = " << std::to_string(TableSchema::TO_DELETE) << ", " << + "SET file_type = " << std::to_string(TableFileSchema::TO_DELETE) << ", " << "updated_time = " << std::to_string(utils::GetMicroSecTimeStamp()) << " " << "WHERE table_id = " << quote << table_id << " AND " << - "file_type <> " << std::to_string(TableSchema::TO_DELETE) << ";"; + "file_type <> " << std::to_string(TableFileSchema::TO_DELETE) << ";"; if (!deleteTableFilesQuery.exec()) { ENGINE_LOG_ERROR << "QUERY ERROR WHEN DELETING TABLE FILES"; diff --git a/cpp/src/db/MySQLMetaImpl.h b/cpp/src/db/MySQLMetaImpl.h index 033fa99453..4805076d45 100644 --- a/cpp/src/db/MySQLMetaImpl.h +++ b/cpp/src/db/MySQLMetaImpl.h @@ -22,7 +22,7 @@ namespace meta { class MySQLMetaImpl : public Meta { public: - MySQLMetaImpl(const DBMetaOptions& options_); + MySQLMetaImpl(const DBMetaOptions& options_, const std::string& mode); virtual Status CreateTable(TableSchema& table_schema) override; virtual Status DescribeTable(TableSchema& group_info_) override; @@ -77,6 +77,7 @@ namespace meta { Status Initialize(); const DBMetaOptions options_; + const std::string mode_; std::shared_ptr mySQLConnectionPool_; bool safe_grab = false; diff --git a/cpp/src/server/DBWrapper.cpp b/cpp/src/server/DBWrapper.cpp index 1c56ece0e3..ceaa6e8130 100644 --- a/cpp/src/server/DBWrapper.cpp +++ b/cpp/src/server/DBWrapper.cpp @@ -25,6 +25,10 @@ DBWrapper::DBWrapper() { } ConfigNode& serverConfig = ServerConfig::GetInstance().GetConfig(CONFIG_SERVER); opt.mode = serverConfig.GetValue(CONFIG_CLUSTER_MODE, "single"); + if (opt.mode != "single" && opt.mode != "cluster" && opt.mode != "read_only") { + std::cout << "ERROR: mode specified in server_config is not one of ['single', 'cluster', 'read_only']" << std::endl; + kill(0, SIGUSR1); + } //set archive config engine::ArchiveConf::CriteriaT criterial; From 082944c9a06f01b5a92064192b99d76ab0f7bff4 Mon Sep 17 00:00:00 2001 From: yu yunfeng Date: Fri, 28 Jun 2019 15:36:42 +0800 Subject: [PATCH 028/189] code coverage Former-commit-id: 284787742def99a04a9a2f822c707e149a614bfe --- cpp/src/metrics/MetricBase.h | 27 ++---------- cpp/src/metrics/PrometheusMetrics.cpp | 23 +--------- cpp/src/metrics/PrometheusMetrics.h | 29 ------------ cpp/src/metrics/SystemInfo.cpp | 40 ----------------- cpp/src/metrics/SystemInfo.h | 2 - cpp/unittest/metrics/CMakeLists.txt | 1 + cpp/unittest/metrics/metrics_test.cpp | 1 - cpp/unittest/metrics/prometheus_test.cpp | 56 ++++++++++++++++++++++++ 8 files changed, 62 insertions(+), 117 deletions(-) create mode 100644 cpp/unittest/metrics/prometheus_test.cpp diff --git a/cpp/src/metrics/MetricBase.h b/cpp/src/metrics/MetricBase.h index c18e857e4e..fe9b246503 100644 --- a/cpp/src/metrics/MetricBase.h +++ b/cpp/src/metrics/MetricBase.h @@ -22,40 +22,21 @@ class MetricsBase{ } virtual ServerError Init() {}; - virtual void AddGroupSuccessTotalIncrement(double value = 1) {}; - virtual void AddGroupFailTotalIncrement(double value = 1) {}; - virtual void HasGroupSuccessTotalIncrement(double value = 1) {}; - virtual void HasGroupFailTotalIncrement(double value = 1) {}; - virtual void GetGroupSuccessTotalIncrement(double value = 1) {}; - virtual void GetGroupFailTotalIncrement(double value = 1) {}; - virtual void GetGroupFilesSuccessTotalIncrement(double value = 1) {}; - virtual void GetGroupFilesFailTotalIncrement(double value = 1) {}; + virtual void AddVectorsSuccessTotalIncrement(double value = 1) {}; virtual void AddVectorsFailTotalIncrement(double value = 1) {}; virtual void AddVectorsDurationHistogramOberve(double value) {}; - virtual void SearchSuccessTotalIncrement(double value = 1) {}; - virtual void SearchFailTotalIncrement(double value = 1) {}; - virtual void SearchDurationHistogramObserve(double value) {}; + virtual void RawFileSizeHistogramObserve(double value) {}; virtual void IndexFileSizeHistogramObserve(double value) {}; virtual void BuildIndexDurationSecondsHistogramObserve(double value) {}; - virtual void AllBuildIndexDurationSecondsHistogramObserve(double value) {}; - virtual void CacheUsageGaugeIncrement(double value = 1) {}; - virtual void CacheUsageGaugeDecrement(double value = 1) {}; + virtual void CacheUsageGaugeSet(double value) {}; - virtual void MetaVisitTotalIncrement(double value = 1) {}; - virtual void MetaVisitDurationSecondsHistogramObserve(double value) {}; - virtual void MemUsagePercentGaugeSet(double value) {}; - virtual void MemUsagePercentGaugeIncrement(double value = 1) {}; - virtual void MemUsagePercentGaugeDecrement(double value = 1) {}; - virtual void MemUsageTotalGaugeSet(double value) {}; - virtual void MemUsageTotalGaugeIncrement(double value = 1) {}; - virtual void MemUsageTotalGaugeDecrement(double value = 1) {}; + virtual void MetaAccessTotalIncrement(double value = 1) {}; virtual void MetaAccessDurationSecondsHistogramObserve(double value) {}; virtual void FaissDiskLoadDurationSecondsHistogramObserve(double value) {}; virtual void FaissDiskLoadSizeBytesHistogramObserve(double value) {}; - virtual void FaissDiskLoadIOSpeedHistogramObserve(double value) {}; virtual void CacheAccessTotalIncrement(double value = 1) {}; virtual void MemTableMergeDurationSecondsHistogramObserve(double value) {}; virtual void SearchIndexDataDurationSecondsHistogramObserve(double value) {}; diff --git a/cpp/src/metrics/PrometheusMetrics.cpp b/cpp/src/metrics/PrometheusMetrics.cpp index d0d50800ad..df0516344e 100644 --- a/cpp/src/metrics/PrometheusMetrics.cpp +++ b/cpp/src/metrics/PrometheusMetrics.cpp @@ -60,9 +60,6 @@ PrometheusMetrics::GPUPercentGaugeSet() { if(!startup_) return; int numDevide = server::SystemInfo::GetInstance().num_device(); std::vector values = server::SystemInfo::GetInstance().GPUPercent(); -// for (int i = 0; i < numDevide; ++i) { -// GPU_percent_gauges_[i].Set(static_cast(values[i])); -// } if(numDevide >= 1) GPU0_percent_gauge_.Set(static_cast(values[0])); if(numDevide >= 2) GPU1_percent_gauge_.Set(static_cast(values[1])); if(numDevide >= 3) GPU2_percent_gauge_.Set(static_cast(values[2])); @@ -77,13 +74,10 @@ PrometheusMetrics::GPUPercentGaugeSet() { void PrometheusMetrics::GPUMemoryUsageGaugeSet() { if(!startup_) return; - int numDevide = server::SystemInfo::GetInstance().num_device(); std::vector values = server::SystemInfo::GetInstance().GPUMemoryUsed(); constexpr unsigned long long MtoB = 1024*1024; int numDevice = values.size(); -// for (int i = 0; i < numDevice; ++i) { -// GPU_memory_usage_gauges_[i].Set(values[i]/MtoB); -// } + if(numDevice >=1) GPU0_memory_usage_gauge_.Set(values[0]/MtoB); if(numDevice >=2) GPU1_memory_usage_gauge_.Set(values[1]/MtoB); if(numDevice >=3) GPU2_memory_usage_gauge_.Set(values[2]/MtoB); @@ -146,21 +140,6 @@ void PrometheusMetrics::OctetsSet() { outoctets_gauge_.Set((in_and_out_octets.second-old_outoctets)/total_second); } -//void PrometheusMetrics::GpuPercentInit() { -// int num_device = SystemInfo::GetInstance().num_device(); -// constexpr char device_number[] = "DeviceNum"; -// for(int i = 0; i < num_device; ++ i) { -// GPU_percent_gauges_.emplace_back(GPU_percent_.Add({{device_number,std::to_string(i)}})); -// } -// -//} -//void PrometheusMetrics::GpuMemoryInit() { -// int num_device = SystemInfo::GetInstance().num_device(); -// constexpr char device_number[] = "DeviceNum"; -// for(int i = 0; i < num_device; ++ i) { -// GPU_memory_usage_gauges_.emplace_back(GPU_memory_usage_.Add({{device_number,std::to_string(i)}})); -// } -//} } diff --git a/cpp/src/metrics/PrometheusMetrics.h b/cpp/src/metrics/PrometheusMetrics.h index 902e79b5a5..b7e991e1c2 100644 --- a/cpp/src/metrics/PrometheusMetrics.h +++ b/cpp/src/metrics/PrometheusMetrics.h @@ -34,10 +34,6 @@ class PrometheusMetrics: public MetricsBase { public: static PrometheusMetrics & GetInstance() { -// switch(MetricCollectorType) { -// case: prometheus:: -// static -// } static PrometheusMetrics instance; return instance; } @@ -49,46 +45,21 @@ class PrometheusMetrics: public MetricsBase { std::shared_ptr exposer_ptr_; std::shared_ptr registry_ = std::make_shared(); bool startup_ = false; -// void GpuPercentInit(); -// void GpuMemoryInit(); public: - void AddGroupSuccessTotalIncrement(double value = 1.0) override { if(startup_) add_group_success_total_.Increment(value);}; - void AddGroupFailTotalIncrement(double value = 1.0) override { if(startup_) add_group_fail_total_.Increment(value);}; - void HasGroupSuccessTotalIncrement(double value = 1.0) override { if(startup_) has_group_success_total_.Increment(value);}; - void HasGroupFailTotalIncrement(double value = 1.0) override { if(startup_) has_group_fail_total_.Increment(value);}; - void GetGroupSuccessTotalIncrement(double value = 1.0) override { if(startup_) get_group_success_total_.Increment(value);}; - void GetGroupFailTotalIncrement(double value = 1.0) override { if(startup_) get_group_fail_total_.Increment(value);}; - void GetGroupFilesSuccessTotalIncrement(double value = 1.0) override { if(startup_) get_group_files_success_total_.Increment(value);}; - void GetGroupFilesFailTotalIncrement(double value = 1.0) override { if(startup_) get_group_files_fail_total_.Increment(value);}; void AddVectorsSuccessTotalIncrement(double value = 1.0) override { if(startup_) add_vectors_success_total_.Increment(value);}; void AddVectorsFailTotalIncrement(double value = 1.0) override { if(startup_) add_vectors_fail_total_.Increment(value);}; void AddVectorsDurationHistogramOberve(double value) override { if(startup_) add_vectors_duration_histogram_.Observe(value);}; - void SearchSuccessTotalIncrement(double value = 1.0) override { if(startup_) search_success_total_.Increment(value);}; - void SearchFailTotalIncrement(double value = 1.0) override { if(startup_) search_fail_total_.Increment(value); }; - void SearchDurationHistogramObserve(double value) override { if(startup_) search_duration_histogram_.Observe(value);}; void RawFileSizeHistogramObserve(double value) override { if(startup_) raw_files_size_histogram_.Observe(value);}; void IndexFileSizeHistogramObserve(double value) override { if(startup_) index_files_size_histogram_.Observe(value);}; void BuildIndexDurationSecondsHistogramObserve(double value) override { if(startup_) build_index_duration_seconds_histogram_.Observe(value);}; - void AllBuildIndexDurationSecondsHistogramObserve(double value) override { if(startup_) all_build_index_duration_seconds_histogram_.Observe(value);}; - void CacheUsageGaugeIncrement(double value = 1.0) override { if(startup_) cache_usage_gauge_.Increment(value);}; - void CacheUsageGaugeDecrement(double value = 1.0) override { if(startup_) cache_usage_gauge_.Decrement(value);}; void CacheUsageGaugeSet(double value) override { if(startup_) cache_usage_gauge_.Set(value);}; -// void MetaVisitTotalIncrement(double value = 1.0) override { meta_visit_total_.Increment(value);}; -// void MetaVisitDurationSecondsHistogramObserve(double value) override { meta_visit_duration_seconds_histogram_.Observe(value);}; - void MemUsagePercentGaugeSet(double value) override { if(startup_) mem_usage_percent_gauge_.Set(value);}; - void MemUsagePercentGaugeIncrement(double value = 1.0) override { if(startup_) mem_usage_percent_gauge_.Increment(value);}; - void MemUsagePercentGaugeDecrement(double value = 1.0) override { if(startup_) mem_usage_percent_gauge_.Decrement(value);}; - void MemUsageTotalGaugeSet(double value) override { if(startup_) mem_usage_total_gauge_.Set(value);}; - void MemUsageTotalGaugeIncrement(double value = 1.0) override { if(startup_) mem_usage_total_gauge_.Increment(value);}; - void MemUsageTotalGaugeDecrement(double value = 1.0) override { if(startup_) mem_usage_total_gauge_.Decrement(value);}; void MetaAccessTotalIncrement(double value = 1) override { if(startup_) meta_access_total_.Increment(value);}; void MetaAccessDurationSecondsHistogramObserve(double value) override { if(startup_) meta_access_duration_seconds_histogram_.Observe(value);}; void FaissDiskLoadDurationSecondsHistogramObserve(double value) override { if(startup_) faiss_disk_load_duration_seconds_histogram_.Observe(value);}; void FaissDiskLoadSizeBytesHistogramObserve(double value) override { if(startup_) faiss_disk_load_size_bytes_histogram_.Observe(value);}; -// void FaissDiskLoadIOSpeedHistogramObserve(double value) { if(startup_) faiss_disk_load_IO_speed_histogram_.Observe(value);}; void FaissDiskLoadIOSpeedGaugeSet(double value) override { if(startup_) faiss_disk_load_IO_speed_gauge_.Set(value);}; void CacheAccessTotalIncrement(double value = 1) override { if(startup_) cache_access_total_.Increment(value);}; diff --git a/cpp/src/metrics/SystemInfo.cpp b/cpp/src/metrics/SystemInfo.cpp index d4ea2de575..a64cbc4992 100644 --- a/cpp/src/metrics/SystemInfo.cpp +++ b/cpp/src/metrics/SystemInfo.cpp @@ -135,46 +135,6 @@ SystemInfo::CPUPercent() { return percent; } -//std::unordered_map> -//SystemInfo::GetGPUMemPercent(){ -// // return GPUID: MEM% -// -// //write GPU info to a file -// system("nvidia-smi pmon -c 1 > GPUInfo.txt"); -// int pid = (int)getpid(); -// -// //parse line -// std::ifstream read_file; -// read_file.open("GPUInfo.txt"); -// std::string line; -// while(getline(read_file, line)){ -// std::vector words = split(line); -// // 0 1 2 3 4 5 6 7 -// //words stand for gpuindex, pid, type, sm, mem, enc, dec, command respectively -// if(std::stoi(words[1]) != pid) continue; -// int GPUindex = std::stoi(words[0]); -// double sm_percent = std::stod(words[3]); -// double mem_percent = std::stod(words[4]); -// -// } -// -//} - -//std::vector -//SystemInfo::split(std::string input) { -// std::vector words; -// input += " "; -// int word_start = 0; -// for (int i = 0; i < input.size(); ++i) { -// if(input[i] != ' ') continue; -// if(input[i] == ' ') { -// word_start = i + 1; -// continue; -// } -// words.push_back(input.substr(word_start,i-word_start)); -// } -// return words; -//} std::vector SystemInfo::GPUPercent() { diff --git a/cpp/src/metrics/SystemInfo.h b/cpp/src/metrics/SystemInfo.h index eeff4b61f5..2562e316e4 100644 --- a/cpp/src/metrics/SystemInfo.h +++ b/cpp/src/metrics/SystemInfo.h @@ -59,8 +59,6 @@ class SystemInfo { double MemoryPercent(); double CPUPercent(); std::pair Octets(); -// std::unordered_map> GetGPUMemPercent() {}; -// std::vector split(std::string input) {}; std::vector GPUPercent(); std::vector GPUMemoryUsed(); diff --git a/cpp/unittest/metrics/CMakeLists.txt b/cpp/unittest/metrics/CMakeLists.txt index 72e1b43867..8c5b09638d 100644 --- a/cpp/unittest/metrics/CMakeLists.txt +++ b/cpp/unittest/metrics/CMakeLists.txt @@ -55,6 +55,7 @@ set(count_test_src ${wrapper_src} ${metrics_src} metrics_test.cpp + prometheus_test.cpp ../db/utils.cpp ) diff --git a/cpp/unittest/metrics/metrics_test.cpp b/cpp/unittest/metrics/metrics_test.cpp index 72596dc79e..923c7b717b 100644 --- a/cpp/unittest/metrics/metrics_test.cpp +++ b/cpp/unittest/metrics/metrics_test.cpp @@ -24,7 +24,6 @@ using namespace zilliz::milvus; - TEST_F(DBTest, Metric_Tes) { server::SystemInfo::GetInstance().Init(); diff --git a/cpp/unittest/metrics/prometheus_test.cpp b/cpp/unittest/metrics/prometheus_test.cpp new file mode 100644 index 0000000000..f1e239d73b --- /dev/null +++ b/cpp/unittest/metrics/prometheus_test.cpp @@ -0,0 +1,56 @@ +/******************************************************************************* + * Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved + * Unauthorized copying of this file, via any medium is strictly prohibited. + * Proprietary and confidential. + ******************************************************************************/ + +#include "metrics/PrometheusMetrics.h" + +#include +#include + +using namespace zilliz::milvus; + +TEST(PrometheusTest, Prometheus_Test){ + server::PrometheusMetrics instance = server::PrometheusMetrics::GetInstance(); + instance.Init(); + server::SystemInfo::GetInstance().Init(); + instance.AddVectorsSuccessTotalIncrement(); + instance.AddVectorsFailTotalIncrement(); + instance.AddVectorsDurationHistogramOberve(1.0); + instance.RawFileSizeHistogramObserve(1.0); + instance.IndexFileSizeHistogramObserve(1.0); + instance.BuildIndexDurationSecondsHistogramObserve(1.0); + instance.CacheUsageGaugeSet(1.0); + instance.MetaAccessTotalIncrement(); + instance.MetaAccessDurationSecondsHistogramObserve(1.0); + instance.FaissDiskLoadDurationSecondsHistogramObserve(1.0); + instance.FaissDiskLoadSizeBytesHistogramObserve(1.0); + instance.FaissDiskLoadIOSpeedGaugeSet(1.0); + instance.CacheAccessTotalIncrement(); + instance.MemTableMergeDurationSecondsHistogramObserve(1.0); + instance.SearchIndexDataDurationSecondsHistogramObserve(1.0); + instance.SearchRawDataDurationSecondsHistogramObserve(1.0); + instance.IndexFileSizeTotalIncrement(); + instance.RawFileSizeTotalIncrement(); + instance.IndexFileSizeGaugeSet(1.0); + instance.RawFileSizeGaugeSet(1.0); + instance.QueryResponseSummaryObserve(1.0); + instance.DiskStoreIOSpeedGaugeSet(1.0); + instance.DataFileSizeGaugeSet(1.0); + instance.AddVectorsSuccessGaugeSet(1.0); + instance.AddVectorsFailGaugeSet(1.0); + instance.QueryVectorResponseSummaryObserve(1.0, 1); + instance.QueryVectorResponsePerSecondGaugeSet(1.0); + instance.CPUUsagePercentSet(); + instance.RAMUsagePercentSet(); + instance.QueryResponsePerSecondGaugeSet(1.0); + instance.GPUPercentGaugeSet(); + instance.GPUMemoryUsageGaugeSet(); + instance.AddVectorsPerSecondGaugeSet(1,1,1); + instance.QueryIndexTypePerSecondSet("IVF", 1.0); + instance.ConnectionGaugeIncrement(); + instance.ConnectionGaugeDecrement(); + instance.KeepingAliveCounterIncrement(); + instance.OctetsSet(); +} \ No newline at end of file From 0ca416055188fe784447cda91e79faf5acd46b72 Mon Sep 17 00:00:00 2001 From: yu yunfeng Date: Fri, 28 Jun 2019 15:40:47 +0800 Subject: [PATCH 029/189] changelog Former-commit-id: d273fd47baaf39f51b9fb4c58254d612821d3f9d --- cpp/CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/cpp/CHANGELOG.md b/cpp/CHANGELOG.md index 95e2eb4f09..eaac6a78ee 100644 --- a/cpp/CHANGELOG.md +++ b/cpp/CHANGELOG.md @@ -48,6 +48,7 @@ Please mark all change in change log and use the ticket from JIRA. - MS-97 - Add S3 SDK for MinIO Storage - MS-105 - Add MySQL +- MS-130 - Add prometheus_test ## Task - MS-74 - Change README.md in cpp - MS-88 - Add support for arm architecture From 46b26f075dea2753b245b94a5c036c2a4126fc07 Mon Sep 17 00:00:00 2001 From: yu yunfeng Date: Fri, 28 Jun 2019 16:20:17 +0800 Subject: [PATCH 030/189] line cov Former-commit-id: 98c5cb5ea36ee17f16966a9b6b3da1a084999299 --- cpp/src/metrics/PrometheusMetrics.h | 2 +- cpp/unittest/metrics/prometheus_test.cpp | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/cpp/src/metrics/PrometheusMetrics.h b/cpp/src/metrics/PrometheusMetrics.h index b7e991e1c2..5b651ec14f 100644 --- a/cpp/src/metrics/PrometheusMetrics.h +++ b/cpp/src/metrics/PrometheusMetrics.h @@ -46,7 +46,7 @@ class PrometheusMetrics: public MetricsBase { std::shared_ptr registry_ = std::make_shared(); bool startup_ = false; public: - + void SetStartup(bool startup) {startup_ = startup;}; void AddVectorsSuccessTotalIncrement(double value = 1.0) override { if(startup_) add_vectors_success_total_.Increment(value);}; void AddVectorsFailTotalIncrement(double value = 1.0) override { if(startup_) add_vectors_fail_total_.Increment(value);}; void AddVectorsDurationHistogramOberve(double value) override { if(startup_) add_vectors_duration_histogram_.Observe(value);}; diff --git a/cpp/unittest/metrics/prometheus_test.cpp b/cpp/unittest/metrics/prometheus_test.cpp index f1e239d73b..885abed566 100644 --- a/cpp/unittest/metrics/prometheus_test.cpp +++ b/cpp/unittest/metrics/prometheus_test.cpp @@ -14,6 +14,7 @@ using namespace zilliz::milvus; TEST(PrometheusTest, Prometheus_Test){ server::PrometheusMetrics instance = server::PrometheusMetrics::GetInstance(); instance.Init(); + instance.SetStartup(true); server::SystemInfo::GetInstance().Init(); instance.AddVectorsSuccessTotalIncrement(); instance.AddVectorsFailTotalIncrement(); From 5fb1285f39f67015eaddf4f3f940124b149a2c21 Mon Sep 17 00:00:00 2001 From: yu yunfeng Date: Fri, 28 Jun 2019 16:33:05 +0800 Subject: [PATCH 031/189] func coverage Former-commit-id: bf3053c42c39ef42a6d7191f8e5fa73677fd1235 --- cpp/conf/log_config.conf | 12 ++--- cpp/conf/server_config.yaml | 2 +- cpp/unittest/metrics/CMakeLists.txt | 2 +- cpp/unittest/metrics/metricbase_test.cpp | 56 ++++++++++++++++++++++++ 4 files changed, 64 insertions(+), 8 deletions(-) create mode 100644 cpp/unittest/metrics/metricbase_test.cpp diff --git a/cpp/conf/log_config.conf b/cpp/conf/log_config.conf index 29d46a7fe5..c16a6a2ebe 100644 --- a/cpp/conf/log_config.conf +++ b/cpp/conf/log_config.conf @@ -1,6 +1,6 @@ * GLOBAL: FORMAT = "%datetime | %level | %logger | %msg" - FILENAME = "/tmp/milvus/logs/milvus-%datetime{%H:%m}-global.log" + FILENAME = "/opt/milvus/logs/milvus-%datetime{%H:%m}-global.log" ENABLED = true TO_FILE = true TO_STANDARD_OUTPUT = false @@ -8,12 +8,12 @@ PERFORMANCE_TRACKING = false MAX_LOG_FILE_SIZE = 2097152 ## Throw log files away after 2MB * DEBUG: - FILENAME = "/tmp/milvus/logs/milvus-%datetime{%H:%m}-debug.log" + FILENAME = "/opt/milvus/logs/milvus-%datetime{%H:%m}-debug.log" ENABLED = true * WARNING: - FILENAME = "/tmp/milvus/logs/milvus-%datetime{%H:%m}-warning.log" + FILENAME = "/opt/milvus/logs/milvus-%datetime{%H:%m}-warning.log" * TRACE: - FILENAME = "/tmp/milvus/logs/milvus-%datetime{%H:%m}-trace.log" + FILENAME = "/opt/milvus/logs/milvus-%datetime{%H:%m}-trace.log" * VERBOSE: FORMAT = "%datetime{%d/%M/%y} | %level-%vlevel | %msg" TO_FILE = false @@ -21,7 +21,7 @@ ## Error logs * ERROR: ENABLED = true - FILENAME = "/tmp/milvus/logs/milvus-%datetime{%H:%m}-error.log" + FILENAME = "/opt/milvus/logs/milvus-%datetime{%H:%m}-error.log" * FATAL: ENABLED = true - FILENAME = "/tmp/milvus/logs/milvus-%datetime{%H:%m}-fatal.log" + FILENAME = "/opt/milvus/logs/milvus-%datetime{%H:%m}-fatal.log" diff --git a/cpp/conf/server_config.yaml b/cpp/conf/server_config.yaml index 35f3747a07..2069daa15c 100644 --- a/cpp/conf/server_config.yaml +++ b/cpp/conf/server_config.yaml @@ -24,7 +24,7 @@ metric_config: push_gateway_port: 9091 # push method configure: push gateway port license_config: # license configure - license_path: "/tmp/milvus/system.license" # license file path + license_path: "/opt/milvus/system.license" # license file path cache_config: # cache configure cpu_cache_capacity: 16 # how many memory are used as cache, unit: GB, range: 0 ~ less than total memory diff --git a/cpp/unittest/metrics/CMakeLists.txt b/cpp/unittest/metrics/CMakeLists.txt index 8c5b09638d..80210772a8 100644 --- a/cpp/unittest/metrics/CMakeLists.txt +++ b/cpp/unittest/metrics/CMakeLists.txt @@ -57,7 +57,7 @@ set(count_test_src metrics_test.cpp prometheus_test.cpp ../db/utils.cpp - ) + metricbase_test.cpp) add_executable(metrics_test ${count_test_src} ${require_files} ) diff --git a/cpp/unittest/metrics/metricbase_test.cpp b/cpp/unittest/metrics/metricbase_test.cpp new file mode 100644 index 0000000000..ac850c7b48 --- /dev/null +++ b/cpp/unittest/metrics/metricbase_test.cpp @@ -0,0 +1,56 @@ +/******************************************************************************* + * Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved + * Unauthorized copying of this file, via any medium is strictly prohibited. + * Proprietary and confidential. + ******************************************************************************/ + +#include "metrics/Metrics.h" + +#include +#include + +using namespace zilliz::milvus; + +TEST(MetricbaseTest, Metricbase_Test){ + server::MetricsBase instance = server::MetricsBase::GetInstance(); + instance.Init(); + server::SystemInfo::GetInstance().Init(); + instance.AddVectorsSuccessTotalIncrement(); + instance.AddVectorsFailTotalIncrement(); + instance.AddVectorsDurationHistogramOberve(1.0); + instance.RawFileSizeHistogramObserve(1.0); + instance.IndexFileSizeHistogramObserve(1.0); + instance.BuildIndexDurationSecondsHistogramObserve(1.0); + instance.CacheUsageGaugeSet(1.0); + instance.MetaAccessTotalIncrement(); + instance.MetaAccessDurationSecondsHistogramObserve(1.0); + instance.FaissDiskLoadDurationSecondsHistogramObserve(1.0); + instance.FaissDiskLoadSizeBytesHistogramObserve(1.0); + instance.FaissDiskLoadIOSpeedGaugeSet(1.0); + instance.CacheAccessTotalIncrement(); + instance.MemTableMergeDurationSecondsHistogramObserve(1.0); + instance.SearchIndexDataDurationSecondsHistogramObserve(1.0); + instance.SearchRawDataDurationSecondsHistogramObserve(1.0); + instance.IndexFileSizeTotalIncrement(); + instance.RawFileSizeTotalIncrement(); + instance.IndexFileSizeGaugeSet(1.0); + instance.RawFileSizeGaugeSet(1.0); + instance.QueryResponseSummaryObserve(1.0); + instance.DiskStoreIOSpeedGaugeSet(1.0); + instance.DataFileSizeGaugeSet(1.0); + instance.AddVectorsSuccessGaugeSet(1.0); + instance.AddVectorsFailGaugeSet(1.0); + instance.QueryVectorResponseSummaryObserve(1.0, 1); + instance.QueryVectorResponsePerSecondGaugeSet(1.0); + instance.CPUUsagePercentSet(); + instance.RAMUsagePercentSet(); + instance.QueryResponsePerSecondGaugeSet(1.0); + instance.GPUPercentGaugeSet(); + instance.GPUMemoryUsageGaugeSet(); + instance.AddVectorsPerSecondGaugeSet(1,1,1); + instance.QueryIndexTypePerSecondSet("IVF", 1.0); + instance.ConnectionGaugeIncrement(); + instance.ConnectionGaugeDecrement(); + instance.KeepingAliveCounterIncrement(); + instance.OctetsSet(); +} \ No newline at end of file From 1ec0296513724c9516746c70206791e99b6421c4 Mon Sep 17 00:00:00 2001 From: starlord Date: Fri, 28 Jun 2019 18:25:57 +0800 Subject: [PATCH 032/189] MS-129 Add more unitest Former-commit-id: 6c1909d834b2eefefb1fec6518816c2245f7360d --- cpp/.gitignore | 5 + cpp/CMakeLists.txt | 4 +- cpp/README.md | 5 + cpp/conf/log_config.conf | 12 +- cpp/conf/server_config.yaml | 9 +- cpp/src/license/GPUInfoFile.h | 138 ++-- cpp/src/license/GetSysInfo.cpp | 166 ++--- cpp/src/license/LicenseCheck.cpp | 262 +++---- cpp/src/license/LicenseCheck.h | 104 +-- cpp/src/license/LicenseFile.h | 172 ++--- cpp/src/license/LicenseGenerator.cpp | 240 +++---- cpp/src/license/LicenseLibrary.cpp | 652 +++++++++--------- cpp/src/license/LicenseLibrary.h | 204 +++--- cpp/src/utils/CommonUtil.cpp | 4 +- cpp/src/utils/CommonUtil.h | 2 +- cpp/unittest/CMakeLists.txt | 2 +- cpp/unittest/license/license_check_test.cpp | 364 +++++----- .../license/license_library_tests.cpp | 408 +++++------ cpp/unittest/server/CMakeLists.txt | 9 + cpp/unittest/server/appendix/log_config.conf | 27 + .../server/appendix/server_config.yaml | 28 + cpp/unittest/server/cache_test.cpp | 48 ++ cpp/unittest/server/config_test.cpp | 40 +- cpp/unittest/server/util_test.cpp | 103 ++- 24 files changed, 1616 insertions(+), 1392 deletions(-) create mode 100644 cpp/unittest/server/appendix/log_config.conf create mode 100644 cpp/unittest/server/appendix/server_config.yaml diff --git a/cpp/.gitignore b/cpp/.gitignore index d8368bd79b..03149cde32 100644 --- a/cpp/.gitignore +++ b/cpp/.gitignore @@ -3,3 +3,8 @@ conf/server_config.yaml conf/log_config.conf version.h megasearch/ +lcov_out/ +base.info +output.info +output_new.info +server.info \ No newline at end of file diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 4b95854f77..58124c5296 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -136,9 +136,9 @@ if (BUILD_COVERAGE STREQUAL "ON") endif() -if ("${BUILD_UNIT_TEST}" STREQUAL "ON") +if (BUILD_UNIT_TEST STREQUAL "ON") add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/unittest) -endif(BUILD_UNIT_TEST) +endif() add_custom_target(Clean-All COMMAND ${CMAKE_BUILD_TOOL} clean) diff --git a/cpp/README.md b/cpp/README.md index 94bf2241ec..c1cd381442 100644 --- a/cpp/README.md +++ b/cpp/README.md @@ -45,6 +45,11 @@ If you encounter the following error when building: or ./build.sh --unittest +#### To run code coverage + + apt-get install lcov + ./build.sh -u -c + ### Launch server Set config in cpp/conf/server_config.yaml diff --git a/cpp/conf/log_config.conf b/cpp/conf/log_config.conf index c16a6a2ebe..29d46a7fe5 100644 --- a/cpp/conf/log_config.conf +++ b/cpp/conf/log_config.conf @@ -1,6 +1,6 @@ * GLOBAL: FORMAT = "%datetime | %level | %logger | %msg" - FILENAME = "/opt/milvus/logs/milvus-%datetime{%H:%m}-global.log" + FILENAME = "/tmp/milvus/logs/milvus-%datetime{%H:%m}-global.log" ENABLED = true TO_FILE = true TO_STANDARD_OUTPUT = false @@ -8,12 +8,12 @@ PERFORMANCE_TRACKING = false MAX_LOG_FILE_SIZE = 2097152 ## Throw log files away after 2MB * DEBUG: - FILENAME = "/opt/milvus/logs/milvus-%datetime{%H:%m}-debug.log" + FILENAME = "/tmp/milvus/logs/milvus-%datetime{%H:%m}-debug.log" ENABLED = true * WARNING: - FILENAME = "/opt/milvus/logs/milvus-%datetime{%H:%m}-warning.log" + FILENAME = "/tmp/milvus/logs/milvus-%datetime{%H:%m}-warning.log" * TRACE: - FILENAME = "/opt/milvus/logs/milvus-%datetime{%H:%m}-trace.log" + FILENAME = "/tmp/milvus/logs/milvus-%datetime{%H:%m}-trace.log" * VERBOSE: FORMAT = "%datetime{%d/%M/%y} | %level-%vlevel | %msg" TO_FILE = false @@ -21,7 +21,7 @@ ## Error logs * ERROR: ENABLED = true - FILENAME = "/opt/milvus/logs/milvus-%datetime{%H:%m}-error.log" + FILENAME = "/tmp/milvus/logs/milvus-%datetime{%H:%m}-error.log" * FATAL: ENABLED = true - FILENAME = "/opt/milvus/logs/milvus-%datetime{%H:%m}-fatal.log" + FILENAME = "/tmp/milvus/logs/milvus-%datetime{%H:%m}-fatal.log" diff --git a/cpp/conf/server_config.yaml b/cpp/conf/server_config.yaml index 2069daa15c..8dc9e1d551 100644 --- a/cpp/conf/server_config.yaml +++ b/cpp/conf/server_config.yaml @@ -5,11 +5,8 @@ server_config: mode: single # milvus deployment type: single, cluster, read_only db_config: - db_path: /tmp/milvus - #URI format: dialect://username:password@host:port/database - #All parts except dialect are optional, but you MUST include the delimiters - #Currently supports mysql or sqlite - db_backend_url: mysql://root:1234@:/test # meta database uri + db_path: /tmp/milvus # milvus data storage path + db_backend_url: http://127.0.0.1 # meta database uri index_building_threshold: 1024 # index building trigger threshold, default: 1024, unit: MB archive_disk_threshold: 512 # triger archive action if storage size exceed this value, unit: GB archive_days_threshold: 30 # files older than x days will be archived, unit: day @@ -24,7 +21,7 @@ metric_config: push_gateway_port: 9091 # push method configure: push gateway port license_config: # license configure - license_path: "/opt/milvus/system.license" # license file path + license_path: "/tmp/milvus/system.license" # license file path cache_config: # cache configure cpu_cache_capacity: 16 # how many memory are used as cache, unit: GB, range: 0 ~ less than total memory diff --git a/cpp/src/license/GPUInfoFile.h b/cpp/src/license/GPUInfoFile.h index d22765bd71..eaad3efefe 100644 --- a/cpp/src/license/GPUInfoFile.h +++ b/cpp/src/license/GPUInfoFile.h @@ -1,69 +1,69 @@ -/******************************************************************************* - * Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved - * Unauthorized copying of this file, via any medium is strictly prohibited. - * Proprietary and confidential. - ******************************************************************************/ -#pragma once - -#include -#include -#include - - -class GPUInfoFile { - public: - GPUInfoFile() = default; - - GPUInfoFile(const int &device_count, const std::map &uuid_encryption_map) - : device_count_(device_count), uuid_encryption_map_(uuid_encryption_map) {} - - int get_device_count() { - return device_count_; - } - std::map &get_uuid_encryption_map() { - return uuid_encryption_map_; - } - - - public: - friend class boost::serialization::access; - - template - void serialize(Archive &ar, const unsigned int version) { - ar & device_count_; - ar & uuid_encryption_map_; - } - - public: - int device_count_ = 0; - std::map uuid_encryption_map_; -}; - -class SerializedGPUInfoFile { - public: - ~SerializedGPUInfoFile() { - if (gpu_info_file_ != nullptr) { - delete (gpu_info_file_); - gpu_info_file_ = nullptr; - } - } - - void - set_gpu_info_file(GPUInfoFile *gpu_info_file) { - gpu_info_file_ = gpu_info_file; - } - - GPUInfoFile *get_gpu_info_file() { - return gpu_info_file_; - } - private: - friend class boost::serialization::access; - - template - void serialize(Archive &ar, const unsigned int version) { - ar & gpu_info_file_; - } - - private: - GPUInfoFile *gpu_info_file_ = nullptr; -}; +///******************************************************************************* +// * Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved +// * Unauthorized copying of this file, via any medium is strictly prohibited. +// * Proprietary and confidential. +// ******************************************************************************/ +//#pragma once +// +//#include +//#include +//#include +// +// +//class GPUInfoFile { +// public: +// GPUInfoFile() = default; +// +// GPUInfoFile(const int &device_count, const std::map &uuid_encryption_map) +// : device_count_(device_count), uuid_encryption_map_(uuid_encryption_map) {} +// +// int get_device_count() { +// return device_count_; +// } +// std::map &get_uuid_encryption_map() { +// return uuid_encryption_map_; +// } +// +// +// public: +// friend class boost::serialization::access; +// +// template +// void serialize(Archive &ar, const unsigned int version) { +// ar & device_count_; +// ar & uuid_encryption_map_; +// } +// +// public: +// int device_count_ = 0; +// std::map uuid_encryption_map_; +//}; +// +//class SerializedGPUInfoFile { +// public: +// ~SerializedGPUInfoFile() { +// if (gpu_info_file_ != nullptr) { +// delete (gpu_info_file_); +// gpu_info_file_ = nullptr; +// } +// } +// +// void +// set_gpu_info_file(GPUInfoFile *gpu_info_file) { +// gpu_info_file_ = gpu_info_file; +// } +// +// GPUInfoFile *get_gpu_info_file() { +// return gpu_info_file_; +// } +// private: +// friend class boost::serialization::access; +// +// template +// void serialize(Archive &ar, const unsigned int version) { +// ar & gpu_info_file_; +// } +// +// private: +// GPUInfoFile *gpu_info_file_ = nullptr; +//}; diff --git a/cpp/src/license/GetSysInfo.cpp b/cpp/src/license/GetSysInfo.cpp index 4746fe5eac..7a30083469 100644 --- a/cpp/src/license/GetSysInfo.cpp +++ b/cpp/src/license/GetSysInfo.cpp @@ -1,83 +1,83 @@ - -#include "utils/Log.h" -#include "LicenseLibrary.h" -#include "utils/Error.h" - -#include -#include -#include -// Not provide path: current work path will be used and system.info. -using namespace zilliz::milvus; - -void -print_usage(const std::string &app_name) { - printf("\n Usage: %s [OPTIONS]\n\n", app_name.c_str()); - printf(" Options:\n"); - printf(" -h --help Print this help\n"); - printf(" -s --sysinfo filename Generate system info file as given name\n"); - printf("\n"); -} - -int main(int argc, char *argv[]) { - std::string app_name = argv[0]; - if (argc != 1 && argc != 3) { - print_usage(app_name); - return EXIT_FAILURE; - } - - static struct option long_options[] = {{"system_info", required_argument, 0, 's'}, - {"help", no_argument, 0, 'h'}, - {NULL, 0, 0, 0}}; - int value = 0; - int option_index = 0; - std::string system_info_filename = "./system.info"; - while ((value = getopt_long(argc, argv, "s:h", long_options, &option_index)) != -1) { - switch (value) { - case 's': { - char *system_info_filename_ptr = strdup(optarg); - system_info_filename = system_info_filename_ptr; - free(system_info_filename_ptr); -// printf("Generate system info file: %s\n", system_info_filename.c_str()); - break; - } - case 'h':print_usage(app_name); - return EXIT_SUCCESS; - case '?':print_usage(app_name); - return EXIT_FAILURE; - default:print_usage(app_name); - break; - } - } - - int device_count = 0; - server::ServerError err = server::LicenseLibrary::GetDeviceCount(device_count); - if (err != server::SERVER_SUCCESS) return -1; - - // 1. Get All GPU UUID - std::vector uuid_array; - err = server::LicenseLibrary::GetUUID(device_count, uuid_array); - if (err != server::SERVER_SUCCESS) return -1; - - // 2. Get UUID SHA256 - std::vector uuid_sha256_array; - err = server::LicenseLibrary::GetUUIDSHA256(device_count, uuid_array, uuid_sha256_array); - if (err != server::SERVER_SUCCESS) return -1; - - // 3. Generate GPU ID map with GPU UUID - std::map uuid_encrption_map; - for (int i = 0; i < device_count; ++i) { - uuid_encrption_map[i] = uuid_sha256_array[i]; - } - - - // 4. Generate GPU_info File - err = server::LicenseLibrary::GPUinfoFileSerialization(system_info_filename, - device_count, - uuid_encrption_map); - if (err != server::SERVER_SUCCESS) return -1; - - printf("Generate GPU_info File Success\n"); - - - return 0; -} \ No newline at end of file +// +//#include "utils/Log.h" +//#include "LicenseLibrary.h" +//#include "utils/Error.h" +// +//#include +//#include +//#include +//// Not provide path: current work path will be used and system.info. +//using namespace zilliz::milvus; +// +//void +//print_usage(const std::string &app_name) { +// printf("\n Usage: %s [OPTIONS]\n\n", app_name.c_str()); +// printf(" Options:\n"); +// printf(" -h --help Print this help\n"); +// printf(" -s --sysinfo filename Generate system info file as given name\n"); +// printf("\n"); +//} +// +//int main(int argc, char *argv[]) { +// std::string app_name = argv[0]; +// if (argc != 1 && argc != 3) { +// print_usage(app_name); +// return EXIT_FAILURE; +// } +// +// static struct option long_options[] = {{"system_info", required_argument, 0, 's'}, +// {"help", no_argument, 0, 'h'}, +// {NULL, 0, 0, 0}}; +// int value = 0; +// int option_index = 0; +// std::string system_info_filename = "./system.info"; +// while ((value = getopt_long(argc, argv, "s:h", long_options, &option_index)) != -1) { +// switch (value) { +// case 's': { +// char *system_info_filename_ptr = strdup(optarg); +// system_info_filename = system_info_filename_ptr; +// free(system_info_filename_ptr); +//// printf("Generate system info file: %s\n", system_info_filename.c_str()); +// break; +// } +// case 'h':print_usage(app_name); +// return EXIT_SUCCESS; +// case '?':print_usage(app_name); +// return EXIT_FAILURE; +// default:print_usage(app_name); +// break; +// } +// } +// +// int device_count = 0; +// server::ServerError err = server::LicenseLibrary::GetDeviceCount(device_count); +// if (err != server::SERVER_SUCCESS) return -1; +// +// // 1. Get All GPU UUID +// std::vector uuid_array; +// err = server::LicenseLibrary::GetUUID(device_count, uuid_array); +// if (err != server::SERVER_SUCCESS) return -1; +// +// // 2. Get UUID SHA256 +// std::vector uuid_sha256_array; +// err = server::LicenseLibrary::GetUUIDSHA256(device_count, uuid_array, uuid_sha256_array); +// if (err != server::SERVER_SUCCESS) return -1; +// +// // 3. Generate GPU ID map with GPU UUID +// std::map uuid_encrption_map; +// for (int i = 0; i < device_count; ++i) { +// uuid_encrption_map[i] = uuid_sha256_array[i]; +// } +// +// +// // 4. Generate GPU_info File +// err = server::LicenseLibrary::GPUinfoFileSerialization(system_info_filename, +// device_count, +// uuid_encrption_map); +// if (err != server::SERVER_SUCCESS) return -1; +// +// printf("Generate GPU_info File Success\n"); +// +// +// return 0; +//} \ No newline at end of file diff --git a/cpp/src/license/LicenseCheck.cpp b/cpp/src/license/LicenseCheck.cpp index 8821795540..8ea396ef41 100644 --- a/cpp/src/license/LicenseCheck.cpp +++ b/cpp/src/license/LicenseCheck.cpp @@ -1,131 +1,131 @@ -#include "LicenseCheck.h" -#include -#include - -#include -#include -//#include -//#include -#include -#include -#include -#include -#include - - -namespace zilliz { -namespace milvus { -namespace server { - -LicenseCheck::LicenseCheck() { - -} - -LicenseCheck::~LicenseCheck() { - StopCountingDown(); -} - -ServerError -LicenseCheck::LegalityCheck(const std::string &license_file_path) { - - int device_count; - LicenseLibrary::GetDeviceCount(device_count); - std::vector uuid_array; - LicenseLibrary::GetUUID(device_count, uuid_array); - - std::vector sha_array; - LicenseLibrary::GetUUIDSHA256(device_count, uuid_array, sha_array); - - int output_device_count; - std::map uuid_encryption_map; - time_t starting_time; - time_t end_time; - ServerError err = LicenseLibrary::LicenseFileDeserialization(license_file_path, - output_device_count, - uuid_encryption_map, - starting_time, - end_time); - if(err !=SERVER_SUCCESS) - { - std::cout << "License check error: 01" << std::endl; - return SERVER_UNEXPECTED_ERROR; - } - time_t system_time; - LicenseLibrary::GetSystemTime(system_time); - - if (device_count != output_device_count) { - std::cout << "License check error: 02" << std::endl; - return SERVER_UNEXPECTED_ERROR; - } - for (int i = 0; i < device_count; ++i) { - if (sha_array[i] != uuid_encryption_map[i]) { - std::cout << "License check error: 03" << std::endl; - return SERVER_UNEXPECTED_ERROR; - } - } - if (system_time < starting_time || system_time > end_time) { - std::cout << "License check error: 04" << std::endl; - return SERVER_UNEXPECTED_ERROR; - } - std::cout << "Legality Check Success" << std::endl; - return SERVER_SUCCESS; -} - -// Part 2: Timing check license - -ServerError -LicenseCheck::AlterFile(const std::string &license_file_path, - const boost::system::error_code &ec, - boost::asio::deadline_timer *pt) { - - ServerError err = LicenseCheck::LegalityCheck(license_file_path); - if(err!=SERVER_SUCCESS) { - std::cout << "license file check error" << std::endl; - exit(1); - } - - std::cout << "---runing---" << std::endl; - pt->expires_at(pt->expires_at() + boost::posix_time::hours(1)); - pt->async_wait(boost::bind(LicenseCheck::AlterFile, license_file_path, boost::asio::placeholders::error, pt)); - - return SERVER_SUCCESS; - -} - -ServerError -LicenseCheck::StartCountingDown(const std::string &license_file_path) { - - if (!LicenseLibrary::IsFileExistent(license_file_path)) { - std::cout << "license file not exist" << std::endl; - exit(1); - } - - //create a thread to run AlterFile - if(counting_thread_ == nullptr) { - counting_thread_ = std::make_shared([&]() { - boost::asio::deadline_timer t(io_service_, boost::posix_time::hours(1)); - t.async_wait(boost::bind(LicenseCheck::AlterFile, license_file_path, boost::asio::placeholders::error, &t)); - io_service_.run();//this thread will block here - }); - } - - return SERVER_SUCCESS; -} - -ServerError -LicenseCheck::StopCountingDown() { - if(!io_service_.stopped()) { - io_service_.stop(); - } - - if(counting_thread_ != nullptr) { - counting_thread_->join(); - counting_thread_ = nullptr; - } - - return SERVER_SUCCESS; -} - -} -} -} \ No newline at end of file +//#include "LicenseCheck.h" +//#include +//#include +// +//#include +//#include +////#include +////#include +//#include +//#include +//#include +//#include +//#include +// +// +//namespace zilliz { +//namespace milvus { +//namespace server { +// +//LicenseCheck::LicenseCheck() { +// +//} +// +//LicenseCheck::~LicenseCheck() { +// StopCountingDown(); +//} +// +//ServerError +//LicenseCheck::LegalityCheck(const std::string &license_file_path) { +// +// int device_count; +// LicenseLibrary::GetDeviceCount(device_count); +// std::vector uuid_array; +// LicenseLibrary::GetUUID(device_count, uuid_array); +// +// std::vector sha_array; +// LicenseLibrary::GetUUIDSHA256(device_count, uuid_array, sha_array); +// +// int output_device_count; +// std::map uuid_encryption_map; +// time_t starting_time; +// time_t end_time; +// ServerError err = LicenseLibrary::LicenseFileDeserialization(license_file_path, +// output_device_count, +// uuid_encryption_map, +// starting_time, +// end_time); +// if(err !=SERVER_SUCCESS) +// { +// std::cout << "License check error: 01" << std::endl; +// return SERVER_UNEXPECTED_ERROR; +// } +// time_t system_time; +// LicenseLibrary::GetSystemTime(system_time); +// +// if (device_count != output_device_count) { +// std::cout << "License check error: 02" << std::endl; +// return SERVER_UNEXPECTED_ERROR; +// } +// for (int i = 0; i < device_count; ++i) { +// if (sha_array[i] != uuid_encryption_map[i]) { +// std::cout << "License check error: 03" << std::endl; +// return SERVER_UNEXPECTED_ERROR; +// } +// } +// if (system_time < starting_time || system_time > end_time) { +// std::cout << "License check error: 04" << std::endl; +// return SERVER_UNEXPECTED_ERROR; +// } +// std::cout << "Legality Check Success" << std::endl; +// return SERVER_SUCCESS; +//} +// +//// Part 2: Timing check license +// +//ServerError +//LicenseCheck::AlterFile(const std::string &license_file_path, +// const boost::system::error_code &ec, +// boost::asio::deadline_timer *pt) { +// +// ServerError err = LicenseCheck::LegalityCheck(license_file_path); +// if(err!=SERVER_SUCCESS) { +// std::cout << "license file check error" << std::endl; +// exit(1); +// } +// +// std::cout << "---runing---" << std::endl; +// pt->expires_at(pt->expires_at() + boost::posix_time::hours(1)); +// pt->async_wait(boost::bind(LicenseCheck::AlterFile, license_file_path, boost::asio::placeholders::error, pt)); +// +// return SERVER_SUCCESS; +// +//} +// +//ServerError +//LicenseCheck::StartCountingDown(const std::string &license_file_path) { +// +// if (!LicenseLibrary::IsFileExistent(license_file_path)) { +// std::cout << "license file not exist" << std::endl; +// exit(1); +// } +// +// //create a thread to run AlterFile +// if(counting_thread_ == nullptr) { +// counting_thread_ = std::make_shared([&]() { +// boost::asio::deadline_timer t(io_service_, boost::posix_time::hours(1)); +// t.async_wait(boost::bind(LicenseCheck::AlterFile, license_file_path, boost::asio::placeholders::error, &t)); +// io_service_.run();//this thread will block here +// }); +// } +// +// return SERVER_SUCCESS; +//} +// +//ServerError +//LicenseCheck::StopCountingDown() { +// if(!io_service_.stopped()) { +// io_service_.stop(); +// } +// +// if(counting_thread_ != nullptr) { +// counting_thread_->join(); +// counting_thread_ = nullptr; +// } +// +// return SERVER_SUCCESS; +//} +// +//} +//} +//} \ No newline at end of file diff --git a/cpp/src/license/LicenseCheck.h b/cpp/src/license/LicenseCheck.h index aa66d132d0..7b20d014b5 100644 --- a/cpp/src/license/LicenseCheck.h +++ b/cpp/src/license/LicenseCheck.h @@ -1,52 +1,52 @@ -#pragma once - -#include "utils/Error.h" -#include "LicenseLibrary.h" - -#include - -#include -#include - -namespace zilliz { -namespace milvus { -namespace server { - -class LicenseCheck { -private: - LicenseCheck(); - ~LicenseCheck(); - -public: - static LicenseCheck & - GetInstance() { - static LicenseCheck instance; - return instance; - }; - - static ServerError - LegalityCheck(const std::string &license_file_path); - - ServerError - StartCountingDown(const std::string &license_file_path); - - ServerError - StopCountingDown(); - -private: - static ServerError - AlterFile(const std::string &license_file_path, - const boost::system::error_code &ec, - boost::asio::deadline_timer *pt); - -private: - boost::asio::io_service io_service_; - std::shared_ptr counting_thread_; - -}; - -} -} -} - - +//#pragma once +// +//#include "utils/Error.h" +//#include "LicenseLibrary.h" +// +//#include +// +//#include +//#include +// +//namespace zilliz { +//namespace milvus { +//namespace server { +// +//class LicenseCheck { +//private: +// LicenseCheck(); +// ~LicenseCheck(); +// +//public: +// static LicenseCheck & +// GetInstance() { +// static LicenseCheck instance; +// return instance; +// }; +// +// static ServerError +// LegalityCheck(const std::string &license_file_path); +// +// ServerError +// StartCountingDown(const std::string &license_file_path); +// +// ServerError +// StopCountingDown(); +// +//private: +// static ServerError +// AlterFile(const std::string &license_file_path, +// const boost::system::error_code &ec, +// boost::asio::deadline_timer *pt); +// +//private: +// boost::asio::io_service io_service_; +// std::shared_ptr counting_thread_; +// +//}; +// +//} +//} +//} +// +// diff --git a/cpp/src/license/LicenseFile.h b/cpp/src/license/LicenseFile.h index bf076b0d68..deefe19d14 100644 --- a/cpp/src/license/LicenseFile.h +++ b/cpp/src/license/LicenseFile.h @@ -1,86 +1,86 @@ -/******************************************************************************* - * Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved - * Unauthorized copying of this file, via any medium is strictly prohibited. - * Proprietary and confidential. - ******************************************************************************/ -#pragma once - - -#include -#include -#include - - -class LicenseFile { - public: - LicenseFile() = default; - - LicenseFile(const int &device_count, - const std::map &uuid_encryption_map, - const time_t &starting_time, - const time_t &end_time) - : device_count_(device_count), - uuid_encryption_map_(uuid_encryption_map), - starting_time_(starting_time), - end_time_(end_time) {} - - int get_device_count() { - return device_count_; - } - std::map &get_uuid_encryption_map() { - return uuid_encryption_map_; - } - time_t get_starting_time() { - return starting_time_; - } - time_t get_end_time() { - return end_time_; - } - - public: - friend class boost::serialization::access; - - template - void serialize(Archive &ar, const unsigned int version) { - ar & device_count_; - ar & uuid_encryption_map_; - ar & starting_time_; - ar & end_time_; - } - - public: - int device_count_ = 0; - std::map uuid_encryption_map_; - time_t starting_time_ = 0; - time_t end_time_ = 0; -}; - -class SerializedLicenseFile { - public: - ~SerializedLicenseFile() { - if (license_file_ != nullptr) { - delete (license_file_); - license_file_ = nullptr; - } - } - - void - set_license_file(LicenseFile *license_file) { - license_file_ = license_file; - } - - LicenseFile *get_license_file() { - return license_file_; - } - private: - friend class boost::serialization::access; - - template - void serialize(Archive &ar, const unsigned int version) { - ar & license_file_; - } - - private: - LicenseFile *license_file_ = nullptr; -}; - +///******************************************************************************* +// * Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved +// * Unauthorized copying of this file, via any medium is strictly prohibited. +// * Proprietary and confidential. +// ******************************************************************************/ +//#pragma once +// +// +//#include +//#include +//#include +// +// +//class LicenseFile { +// public: +// LicenseFile() = default; +// +// LicenseFile(const int &device_count, +// const std::map &uuid_encryption_map, +// const time_t &starting_time, +// const time_t &end_time) +// : device_count_(device_count), +// uuid_encryption_map_(uuid_encryption_map), +// starting_time_(starting_time), +// end_time_(end_time) {} +// +// int get_device_count() { +// return device_count_; +// } +// std::map &get_uuid_encryption_map() { +// return uuid_encryption_map_; +// } +// time_t get_starting_time() { +// return starting_time_; +// } +// time_t get_end_time() { +// return end_time_; +// } +// +// public: +// friend class boost::serialization::access; +// +// template +// void serialize(Archive &ar, const unsigned int version) { +// ar & device_count_; +// ar & uuid_encryption_map_; +// ar & starting_time_; +// ar & end_time_; +// } +// +// public: +// int device_count_ = 0; +// std::map uuid_encryption_map_; +// time_t starting_time_ = 0; +// time_t end_time_ = 0; +//}; +// +//class SerializedLicenseFile { +// public: +// ~SerializedLicenseFile() { +// if (license_file_ != nullptr) { +// delete (license_file_); +// license_file_ = nullptr; +// } +// } +// +// void +// set_license_file(LicenseFile *license_file) { +// license_file_ = license_file; +// } +// +// LicenseFile *get_license_file() { +// return license_file_; +// } +// private: +// friend class boost::serialization::access; +// +// template +// void serialize(Archive &ar, const unsigned int version) { +// ar & license_file_; +// } +// +// private: +// LicenseFile *license_file_ = nullptr; +//}; +// diff --git a/cpp/src/license/LicenseGenerator.cpp b/cpp/src/license/LicenseGenerator.cpp index 2a83b6b2d1..a5064eb4c3 100644 --- a/cpp/src/license/LicenseGenerator.cpp +++ b/cpp/src/license/LicenseGenerator.cpp @@ -1,121 +1,121 @@ - -#include -#include -#include - -#include "utils/Log.h" -#include "license/LicenseLibrary.h" -#include "utils/Error.h" - - -using namespace zilliz::milvus; -// Not provide path: current work path will be used and system.info. - -void -print_usage(const std::string &app_name) { - printf("\n Usage: %s [OPTIONS]\n\n", app_name.c_str()); - printf(" Options:\n"); - printf(" -h --help Print this help\n"); - printf(" -s --sysinfo filename sysinfo file location\n"); - printf(" -l --license filename Generate license file as given name\n"); - printf(" -b --starting time Set start time (format: YYYY-MM-DD)\n"); - printf(" -e --end time Set end time (format: YYYY-MM-DD)\n"); - printf("\n"); -} - -int main(int argc, char *argv[]) { - std::string app_name = argv[0]; -// if (argc != 1 && argc != 3) { -// print_usage(app_name); -// return EXIT_FAILURE; +// +//#include +//#include +//#include +// +//#include "utils/Log.h" +//#include "license/LicenseLibrary.h" +//#include "utils/Error.h" +// +// +//using namespace zilliz::milvus; +//// Not provide path: current work path will be used and system.info. +// +//void +//print_usage(const std::string &app_name) { +// printf("\n Usage: %s [OPTIONS]\n\n", app_name.c_str()); +// printf(" Options:\n"); +// printf(" -h --help Print this help\n"); +// printf(" -s --sysinfo filename sysinfo file location\n"); +// printf(" -l --license filename Generate license file as given name\n"); +// printf(" -b --starting time Set start time (format: YYYY-MM-DD)\n"); +// printf(" -e --end time Set end time (format: YYYY-MM-DD)\n"); +// printf("\n"); +//} +// +//int main(int argc, char *argv[]) { +// std::string app_name = argv[0]; +//// if (argc != 1 && argc != 3) { +//// print_usage(app_name); +//// return EXIT_FAILURE; +//// } +// static struct option long_options[] = {{"system_info", required_argument, 0, 's'}, +// {"license", optional_argument, 0, 'l'}, +// {"help", no_argument, 0, 'h'}, +// {"starting_time", required_argument, 0, 'b'}, +// {"end_time", required_argument, 0, 'e'}, +// {NULL, 0, 0, 0}}; +// server::ServerError err; +// int value = 0; +// int option_index = 0; +// std::string system_info_filename = "./system.info"; +// std::string license_filename = "./system.license"; +// char *string_starting_time = NULL; +// char *string_end_time = NULL; +// time_t starting_time = 0; +// time_t end_time = 0; +// int flag_s = 1; +// int flag_b = 1; +// int flag_e = 1; +// while ((value = getopt_long(argc, argv, "hl:s:b:e:", long_options, NULL)) != -1) { +// switch (value) { +// case 's': { +// flag_s = 0; +// system_info_filename = (std::string) (optarg); +// break; +// } +// case 'b': { +// flag_b = 0; +// string_starting_time = optarg; +// break; +// } +// case 'e': { +// flag_e = 0; +// string_end_time = optarg; +// break; +// } +// case 'l': { +// license_filename = (std::string) (optarg); +// break; +// } +// case 'h':print_usage(app_name); +// return EXIT_SUCCESS; +// case '?':print_usage(app_name); +// return EXIT_FAILURE; +// default:print_usage(app_name); +// break; +// } +// // } - static struct option long_options[] = {{"system_info", required_argument, 0, 's'}, - {"license", optional_argument, 0, 'l'}, - {"help", no_argument, 0, 'h'}, - {"starting_time", required_argument, 0, 'b'}, - {"end_time", required_argument, 0, 'e'}, - {NULL, 0, 0, 0}}; - server::ServerError err; - int value = 0; - int option_index = 0; - std::string system_info_filename = "./system.info"; - std::string license_filename = "./system.license"; - char *string_starting_time = NULL; - char *string_end_time = NULL; - time_t starting_time = 0; - time_t end_time = 0; - int flag_s = 1; - int flag_b = 1; - int flag_e = 1; - while ((value = getopt_long(argc, argv, "hl:s:b:e:", long_options, NULL)) != -1) { - switch (value) { - case 's': { - flag_s = 0; - system_info_filename = (std::string) (optarg); - break; - } - case 'b': { - flag_b = 0; - string_starting_time = optarg; - break; - } - case 'e': { - flag_e = 0; - string_end_time = optarg; - break; - } - case 'l': { - license_filename = (std::string) (optarg); - break; - } - case 'h':print_usage(app_name); - return EXIT_SUCCESS; - case '?':print_usage(app_name); - return EXIT_FAILURE; - default:print_usage(app_name); - break; - } - - } - if (flag_s) { - printf("Error: sysinfo file location must be entered\n"); - return 1; - } - if (flag_b) { - printf("Error: start time must be entered\n"); - return 1; - } - if (flag_e) { - printf("Error: end time must be entered\n"); - return 1; - } - - err = server::LicenseLibrary::GetDateTime(string_starting_time, starting_time); - if (err != server::SERVER_SUCCESS) return -1; - - err = server::LicenseLibrary::GetDateTime(string_end_time, end_time); - if (err != server::SERVER_SUCCESS) return -1; - - - int output_info_device_count = 0; - std::map output_info_uuid_encrption_map; - - - err = server::LicenseLibrary::GPUinfoFileDeserialization(system_info_filename, - output_info_device_count, - output_info_uuid_encrption_map); - if (err != server::SERVER_SUCCESS) return -1; - - - err = server::LicenseLibrary::LicenseFileSerialization(license_filename, - output_info_device_count, - output_info_uuid_encrption_map, - starting_time, - end_time); - if (err != server::SERVER_SUCCESS) return -1; - - - printf("Generate License File Success\n"); - - return 0; -} +// if (flag_s) { +// printf("Error: sysinfo file location must be entered\n"); +// return 1; +// } +// if (flag_b) { +// printf("Error: start time must be entered\n"); +// return 1; +// } +// if (flag_e) { +// printf("Error: end time must be entered\n"); +// return 1; +// } +// +// err = server::LicenseLibrary::GetDateTime(string_starting_time, starting_time); +// if (err != server::SERVER_SUCCESS) return -1; +// +// err = server::LicenseLibrary::GetDateTime(string_end_time, end_time); +// if (err != server::SERVER_SUCCESS) return -1; +// +// +// int output_info_device_count = 0; +// std::map output_info_uuid_encrption_map; +// +// +// err = server::LicenseLibrary::GPUinfoFileDeserialization(system_info_filename, +// output_info_device_count, +// output_info_uuid_encrption_map); +// if (err != server::SERVER_SUCCESS) return -1; +// +// +// err = server::LicenseLibrary::LicenseFileSerialization(license_filename, +// output_info_device_count, +// output_info_uuid_encrption_map, +// starting_time, +// end_time); +// if (err != server::SERVER_SUCCESS) return -1; +// +// +// printf("Generate License File Success\n"); +// +// return 0; +//} diff --git a/cpp/src/license/LicenseLibrary.cpp b/cpp/src/license/LicenseLibrary.cpp index 0bb616f5ea..bf377650a6 100644 --- a/cpp/src/license/LicenseLibrary.cpp +++ b/cpp/src/license/LicenseLibrary.cpp @@ -1,345 +1,345 @@ -//////////////////////////////////////////////////////////////////////////////// -// Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved -// Unauthorized copying of this file, via any medium is strictly prohibited. -// Proprietary and confidential. -//////////////////////////////////////////////////////////////////////////////// - -#include "LicenseLibrary.h" -#include "utils/Log.h" -#include -#include -#include -#include - -#include -#include -//#include -//#include -#include -#include -#include - - -namespace zilliz { -namespace milvus { -namespace server { - -constexpr int LicenseLibrary::sha256_length_; - -// Part 0: File check -bool -LicenseLibrary::IsFileExistent(const std::string &path) { - - boost::system::error_code error; - auto file_status = boost::filesystem::status(path, error); - if (error) { - return false; - } - - if (!boost::filesystem::exists(file_status)) { - return false; - } - - return !boost::filesystem::is_directory(file_status); -} - -// Part 1: Get GPU Info -ServerError -LicenseLibrary::GetDeviceCount(int &device_count) { - nvmlReturn_t result = nvmlInit(); - if (NVML_SUCCESS != result) { - printf("Failed to initialize NVML: %s\n", nvmlErrorString(result)); - return SERVER_UNEXPECTED_ERROR; - } - cudaError_t error_id = cudaGetDeviceCount(&device_count); - if (error_id != cudaSuccess) { - printf("cudaGetDeviceCount returned %d\n-> %s\n", (int) error_id, cudaGetErrorString(error_id)); - printf("Result = FAIL\n"); - return SERVER_UNEXPECTED_ERROR; - } - return SERVER_SUCCESS; -} - -ServerError -LicenseLibrary::GetUUID(int device_count, std::vector &uuid_array) { - if (device_count == 0) { - printf("There are no available device(s) that support CUDA\n"); - return SERVER_UNEXPECTED_ERROR; - } - - for (int dev = 0; dev < device_count; ++dev) { - nvmlDevice_t device; - nvmlReturn_t result = nvmlDeviceGetHandleByIndex(dev, &device); - if (NVML_SUCCESS != result) { - printf("Failed to get handle for device %i: %s\n", dev, nvmlErrorString(result)); - return SERVER_UNEXPECTED_ERROR; - } - - char uuid[80]; - unsigned int length = 80; - nvmlReturn_t err = nvmlDeviceGetUUID(device, uuid, length); - if (err != NVML_SUCCESS) { - printf("nvmlDeviceGetUUID error: %d\n", err); - return SERVER_UNEXPECTED_ERROR; - } - - uuid_array.emplace_back(uuid); - } - return SERVER_SUCCESS; -} - -ServerError -LicenseLibrary::GetUUIDMD5(int device_count, - std::vector &uuid_array, - std::vector &md5_array) { - MD5_CTX ctx; - unsigned char outmd[16]; - char temp[2]; - std::string md5; - for (int dev = 0; dev < device_count; ++dev) { - md5.clear(); - memset(outmd, 0, sizeof(outmd)); - MD5_Init(&ctx); - MD5_Update(&ctx, uuid_array[dev].c_str(), uuid_array[dev].size()); - MD5_Final(outmd, &ctx); - for (int i = 0; i < 16; ++i) { - std::snprintf(temp, 2, "%02X", outmd[i]); - md5 += temp; - } - md5_array.push_back(md5); - } - return SERVER_SUCCESS; -} - -ServerError -LicenseLibrary::GetUUIDSHA256(const int &device_count, - std::vector &uuid_array, - std::vector &sha_array) { - SHA256_CTX ctx; - unsigned char outmd[sha256_length_]; - char temp[2]; - std::string sha; - for (int dev = 0; dev < device_count; ++dev) { - sha.clear(); - memset(outmd, 0, sizeof(outmd)); - SHA256_Init(&ctx); - SHA256_Update(&ctx, uuid_array[dev].c_str(), uuid_array[dev].size()); - SHA256_Final(outmd, &ctx); - for (int i = 0; i < sha256_length_; ++i) { - std::snprintf(temp, 2, "%02X", outmd[i]); - sha += temp; - } - sha_array.push_back(sha); - } - return SERVER_SUCCESS; -} - -ServerError -LicenseLibrary::GetSystemTime(time_t &system_time) { - system_time = time(NULL); - return SERVER_SUCCESS; -} - -// Part 2: Handle License File -ServerError -LicenseLibrary::LicenseFileSerialization(const std::string &path, - int device_count, - const std::map &uuid_encrption_map, - time_t starting_time, - time_t end_time) { - - std::ofstream file(path); - boost::archive::binary_oarchive oa(file); - oa.register_type(); - - SerializedLicenseFile serialized_license_file; - - serialized_license_file.set_license_file(new LicenseFile(device_count, - uuid_encrption_map, - starting_time, - end_time)); - oa << serialized_license_file; - - file.close(); - return SERVER_SUCCESS; -} - -ServerError -LicenseLibrary::LicenseFileDeserialization(const std::string &path, - int &device_count, - std::map &uuid_encrption_map, - time_t &starting_time, - time_t &end_time) { - if (!IsFileExistent(path)) return SERVER_LICENSE_FILE_NOT_EXIST; - std::ifstream file(path); - boost::archive::binary_iarchive ia(file); - ia.register_type(); - - SerializedLicenseFile serialized_license_file; - ia >> serialized_license_file; - - device_count = serialized_license_file.get_license_file()->get_device_count(); - uuid_encrption_map = serialized_license_file.get_license_file()->get_uuid_encryption_map(); - starting_time = serialized_license_file.get_license_file()->get_starting_time(); - end_time = serialized_license_file.get_license_file()->get_end_time(); - - file.close(); - return SERVER_SUCCESS; -} - +////////////////////////////////////////////////////////////////////////////////// +//// Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved +//// Unauthorized copying of this file, via any medium is strictly prohibited. +//// Proprietary and confidential. +////////////////////////////////////////////////////////////////////////////////// +// +//#include "LicenseLibrary.h" +//#include "utils/Log.h" +//#include +//#include +//#include +//#include +// +//#include +//#include +////#include +////#include +//#include +//#include +//#include +// +// +//namespace zilliz { +//namespace milvus { +//namespace server { +// +//constexpr int LicenseLibrary::sha256_length_; +// +//// Part 0: File check +//bool +//LicenseLibrary::IsFileExistent(const std::string &path) { +// +// boost::system::error_code error; +// auto file_status = boost::filesystem::status(path, error); +// if (error) { +// return false; +// } +// +// if (!boost::filesystem::exists(file_status)) { +// return false; +// } +// +// return !boost::filesystem::is_directory(file_status); +//} +// +//// Part 1: Get GPU Info //ServerError -//LicenseLibrary::SecretFileSerialization(const std::string &path, -// const time_t &update_time, -// const off_t &file_size, -// const time_t &starting_time, -// const time_t &end_time, -// const std::string &file_md5) { +//LicenseLibrary::GetDeviceCount(int &device_count) { +// nvmlReturn_t result = nvmlInit(); +// if (NVML_SUCCESS != result) { +// printf("Failed to initialize NVML: %s\n", nvmlErrorString(result)); +// return SERVER_UNEXPECTED_ERROR; +// } +// cudaError_t error_id = cudaGetDeviceCount(&device_count); +// if (error_id != cudaSuccess) { +// printf("cudaGetDeviceCount returned %d\n-> %s\n", (int) error_id, cudaGetErrorString(error_id)); +// printf("Result = FAIL\n"); +// return SERVER_UNEXPECTED_ERROR; +// } +// return SERVER_SUCCESS; +//} +// +//ServerError +//LicenseLibrary::GetUUID(int device_count, std::vector &uuid_array) { +// if (device_count == 0) { +// printf("There are no available device(s) that support CUDA\n"); +// return SERVER_UNEXPECTED_ERROR; +// } +// +// for (int dev = 0; dev < device_count; ++dev) { +// nvmlDevice_t device; +// nvmlReturn_t result = nvmlDeviceGetHandleByIndex(dev, &device); +// if (NVML_SUCCESS != result) { +// printf("Failed to get handle for device %i: %s\n", dev, nvmlErrorString(result)); +// return SERVER_UNEXPECTED_ERROR; +// } +// +// char uuid[80]; +// unsigned int length = 80; +// nvmlReturn_t err = nvmlDeviceGetUUID(device, uuid, length); +// if (err != NVML_SUCCESS) { +// printf("nvmlDeviceGetUUID error: %d\n", err); +// return SERVER_UNEXPECTED_ERROR; +// } +// +// uuid_array.emplace_back(uuid); +// } +// return SERVER_SUCCESS; +//} +// +//ServerError +//LicenseLibrary::GetUUIDMD5(int device_count, +// std::vector &uuid_array, +// std::vector &md5_array) { +// MD5_CTX ctx; +// unsigned char outmd[16]; +// char temp[2]; +// std::string md5; +// for (int dev = 0; dev < device_count; ++dev) { +// md5.clear(); +// memset(outmd, 0, sizeof(outmd)); +// MD5_Init(&ctx); +// MD5_Update(&ctx, uuid_array[dev].c_str(), uuid_array[dev].size()); +// MD5_Final(outmd, &ctx); +// for (int i = 0; i < 16; ++i) { +// std::snprintf(temp, 2, "%02X", outmd[i]); +// md5 += temp; +// } +// md5_array.push_back(md5); +// } +// return SERVER_SUCCESS; +//} +// +//ServerError +//LicenseLibrary::GetUUIDSHA256(const int &device_count, +// std::vector &uuid_array, +// std::vector &sha_array) { +// SHA256_CTX ctx; +// unsigned char outmd[sha256_length_]; +// char temp[2]; +// std::string sha; +// for (int dev = 0; dev < device_count; ++dev) { +// sha.clear(); +// memset(outmd, 0, sizeof(outmd)); +// SHA256_Init(&ctx); +// SHA256_Update(&ctx, uuid_array[dev].c_str(), uuid_array[dev].size()); +// SHA256_Final(outmd, &ctx); +// for (int i = 0; i < sha256_length_; ++i) { +// std::snprintf(temp, 2, "%02X", outmd[i]); +// sha += temp; +// } +// sha_array.push_back(sha); +// } +// return SERVER_SUCCESS; +//} +// +//ServerError +//LicenseLibrary::GetSystemTime(time_t &system_time) { +// system_time = time(NULL); +// return SERVER_SUCCESS; +//} +// +//// Part 2: Handle License File +//ServerError +//LicenseLibrary::LicenseFileSerialization(const std::string &path, +// int device_count, +// const std::map &uuid_encrption_map, +// time_t starting_time, +// time_t end_time) { +// // std::ofstream file(path); // boost::archive::binary_oarchive oa(file); -// oa.register_type(); +// oa.register_type(); // -// SerializedSecretFile serialized_secret_file; +// SerializedLicenseFile serialized_license_file; // -// serialized_secret_file.set_secret_file(new SecretFile(update_time, file_size, starting_time, end_time, file_md5)); -// oa << serialized_secret_file; +// serialized_license_file.set_license_file(new LicenseFile(device_count, +// uuid_encrption_map, +// starting_time, +// end_time)); +// oa << serialized_license_file; // // file.close(); // return SERVER_SUCCESS; //} // //ServerError -//LicenseLibrary::SecretFileDeserialization(const std::string &path, -// time_t &update_time, -// off_t &file_size, -// time_t &starting_time, -// time_t &end_time, -// std::string &file_md5) { +//LicenseLibrary::LicenseFileDeserialization(const std::string &path, +// int &device_count, +// std::map &uuid_encrption_map, +// time_t &starting_time, +// time_t &end_time) { +// if (!IsFileExistent(path)) return SERVER_LICENSE_FILE_NOT_EXIST; +// std::ifstream file(path); +// boost::archive::binary_iarchive ia(file); +// ia.register_type(); +// +// SerializedLicenseFile serialized_license_file; +// ia >> serialized_license_file; +// +// device_count = serialized_license_file.get_license_file()->get_device_count(); +// uuid_encrption_map = serialized_license_file.get_license_file()->get_uuid_encryption_map(); +// starting_time = serialized_license_file.get_license_file()->get_starting_time(); +// end_time = serialized_license_file.get_license_file()->get_end_time(); +// +// file.close(); +// return SERVER_SUCCESS; +//} +// +////ServerError +////LicenseLibrary::SecretFileSerialization(const std::string &path, +//// const time_t &update_time, +//// const off_t &file_size, +//// const time_t &starting_time, +//// const time_t &end_time, +//// const std::string &file_md5) { +//// std::ofstream file(path); +//// boost::archive::binary_oarchive oa(file); +//// oa.register_type(); +//// +//// SerializedSecretFile serialized_secret_file; +//// +//// serialized_secret_file.set_secret_file(new SecretFile(update_time, file_size, starting_time, end_time, file_md5)); +//// oa << serialized_secret_file; +//// +//// file.close(); +//// return SERVER_SUCCESS; +////} +//// +////ServerError +////LicenseLibrary::SecretFileDeserialization(const std::string &path, +//// time_t &update_time, +//// off_t &file_size, +//// time_t &starting_time, +//// time_t &end_time, +//// std::string &file_md5) { +//// if (!IsFileExistent(path)) return SERVER_LICENSE_FILE_NOT_EXIST; +//// +//// std::ifstream file(path); +//// boost::archive::binary_iarchive ia(file); +//// ia.register_type(); +//// SerializedSecretFile serialized_secret_file; +//// +//// ia >> serialized_secret_file; +//// update_time = serialized_secret_file.get_secret_file()->get_update_time(); +//// file_size = serialized_secret_file.get_secret_file()->get_file_size(); +//// starting_time = serialized_secret_file.get_secret_file()->get_starting_time(); +//// end_time = serialized_secret_file.get_secret_file()->get_end_time(); +//// file_md5 = serialized_secret_file.get_secret_file()->get_file_md5(); +//// file.close(); +//// return SERVER_SUCCESS; +////} +// +// +// +//// Part 3: File attribute: UpdateTime Time/ Size/ MD5 +//ServerError +//LicenseLibrary::GetFileUpdateTimeAndSize(const std::string &path, time_t &update_time, off_t &file_size) { +// +// if (!IsFileExistent(path)) return SERVER_LICENSE_FILE_NOT_EXIST; +// +// struct stat buf; +// int err_no = stat(path.c_str(), &buf); +// if (err_no != 0) { +// std::cout << strerror(err_no) << std::endl; +// return SERVER_UNEXPECTED_ERROR; +// } +// +// update_time = buf.st_mtime; +// file_size = buf.st_size; +// +// return SERVER_SUCCESS; +//} +// +//ServerError +//LicenseLibrary::GetFileMD5(const std::string &path, std::string &filemd5) { +// +// if (!IsFileExistent(path)) return SERVER_LICENSE_FILE_NOT_EXIST; +// +// filemd5.clear(); +// +// std::ifstream file(path.c_str(), std::ifstream::binary); +// if (!file) { +// return -1; +// } +// +// MD5_CTX md5Context; +// MD5_Init(&md5Context); +// +// char buf[1024 * 16]; +// while (file.good()) { +// file.read(buf, sizeof(buf)); +// MD5_Update(&md5Context, buf, file.gcount()); +// } +// +// unsigned char result[MD5_DIGEST_LENGTH]; +// MD5_Final(result, &md5Context); +// +// char hex[35]; +// memset(hex, 0, sizeof(hex)); +// for (int i = 0; i < MD5_DIGEST_LENGTH; ++i) { +// sprintf(hex + i * 2, "%02X", result[i]); +// } +// hex[32] = '\0'; +// filemd5 = std::string(hex); +// +// return SERVER_SUCCESS; +//} +//// Part 4: GPU Info File Serialization/Deserialization +//ServerError +//LicenseLibrary::GPUinfoFileSerialization(const std::string &path, +// int device_count, +// const std::map &uuid_encrption_map) { +// std::ofstream file(path); +// boost::archive::binary_oarchive oa(file); +// oa.register_type(); +// +// SerializedGPUInfoFile serialized_gpu_info_file; +// +// serialized_gpu_info_file.set_gpu_info_file(new GPUInfoFile(device_count, uuid_encrption_map)); +// oa << serialized_gpu_info_file; +// +// file.close(); +// return SERVER_SUCCESS; +//} +//ServerError +//LicenseLibrary::GPUinfoFileDeserialization(const std::string &path, +// int &device_count, +// std::map &uuid_encrption_map) { // if (!IsFileExistent(path)) return SERVER_LICENSE_FILE_NOT_EXIST; // // std::ifstream file(path); // boost::archive::binary_iarchive ia(file); -// ia.register_type(); -// SerializedSecretFile serialized_secret_file; +// ia.register_type(); +// +// SerializedGPUInfoFile serialized_gpu_info_file; +// ia >> serialized_gpu_info_file; +// +// device_count = serialized_gpu_info_file.get_gpu_info_file()->get_device_count(); +// uuid_encrption_map = serialized_gpu_info_file.get_gpu_info_file()->get_uuid_encryption_map(); // -// ia >> serialized_secret_file; -// update_time = serialized_secret_file.get_secret_file()->get_update_time(); -// file_size = serialized_secret_file.get_secret_file()->get_file_size(); -// starting_time = serialized_secret_file.get_secret_file()->get_starting_time(); -// end_time = serialized_secret_file.get_secret_file()->get_end_time(); -// file_md5 = serialized_secret_file.get_secret_file()->get_file_md5(); // file.close(); // return SERVER_SUCCESS; //} - - - -// Part 3: File attribute: UpdateTime Time/ Size/ MD5 -ServerError -LicenseLibrary::GetFileUpdateTimeAndSize(const std::string &path, time_t &update_time, off_t &file_size) { - - if (!IsFileExistent(path)) return SERVER_LICENSE_FILE_NOT_EXIST; - - struct stat buf; - int err_no = stat(path.c_str(), &buf); - if (err_no != 0) { - std::cout << strerror(err_no) << std::endl; - return SERVER_UNEXPECTED_ERROR; - } - - update_time = buf.st_mtime; - file_size = buf.st_size; - - return SERVER_SUCCESS; -} - -ServerError -LicenseLibrary::GetFileMD5(const std::string &path, std::string &filemd5) { - - if (!IsFileExistent(path)) return SERVER_LICENSE_FILE_NOT_EXIST; - - filemd5.clear(); - - std::ifstream file(path.c_str(), std::ifstream::binary); - if (!file) { - return -1; - } - - MD5_CTX md5Context; - MD5_Init(&md5Context); - - char buf[1024 * 16]; - while (file.good()) { - file.read(buf, sizeof(buf)); - MD5_Update(&md5Context, buf, file.gcount()); - } - - unsigned char result[MD5_DIGEST_LENGTH]; - MD5_Final(result, &md5Context); - - char hex[35]; - memset(hex, 0, sizeof(hex)); - for (int i = 0; i < MD5_DIGEST_LENGTH; ++i) { - sprintf(hex + i * 2, "%02X", result[i]); - } - hex[32] = '\0'; - filemd5 = std::string(hex); - - return SERVER_SUCCESS; -} -// Part 4: GPU Info File Serialization/Deserialization -ServerError -LicenseLibrary::GPUinfoFileSerialization(const std::string &path, - int device_count, - const std::map &uuid_encrption_map) { - std::ofstream file(path); - boost::archive::binary_oarchive oa(file); - oa.register_type(); - - SerializedGPUInfoFile serialized_gpu_info_file; - - serialized_gpu_info_file.set_gpu_info_file(new GPUInfoFile(device_count, uuid_encrption_map)); - oa << serialized_gpu_info_file; - - file.close(); - return SERVER_SUCCESS; -} -ServerError -LicenseLibrary::GPUinfoFileDeserialization(const std::string &path, - int &device_count, - std::map &uuid_encrption_map) { - if (!IsFileExistent(path)) return SERVER_LICENSE_FILE_NOT_EXIST; - - std::ifstream file(path); - boost::archive::binary_iarchive ia(file); - ia.register_type(); - - SerializedGPUInfoFile serialized_gpu_info_file; - ia >> serialized_gpu_info_file; - - device_count = serialized_gpu_info_file.get_gpu_info_file()->get_device_count(); - uuid_encrption_map = serialized_gpu_info_file.get_gpu_info_file()->get_uuid_encryption_map(); - - file.close(); - return SERVER_SUCCESS; -} - -ServerError -LicenseLibrary::GetDateTime(const char *cha, time_t &data_time) { - tm tm_; - int year, month, day; - sscanf(cha, "%d-%d-%d", &year, &month, &day); - tm_.tm_year = year - 1900; - tm_.tm_mon = month - 1; - tm_.tm_mday = day; - tm_.tm_hour = 0; - tm_.tm_min = 0; - tm_.tm_sec = 0; - tm_.tm_isdst = 0; - data_time = mktime(&tm_); - return SERVER_SUCCESS; - -} - -} -} -} \ No newline at end of file +// +//ServerError +//LicenseLibrary::GetDateTime(const char *cha, time_t &data_time) { +// tm tm_; +// int year, month, day; +// sscanf(cha, "%d-%d-%d", &year, &month, &day); +// tm_.tm_year = year - 1900; +// tm_.tm_mon = month - 1; +// tm_.tm_mday = day; +// tm_.tm_hour = 0; +// tm_.tm_min = 0; +// tm_.tm_sec = 0; +// tm_.tm_isdst = 0; +// data_time = mktime(&tm_); +// return SERVER_SUCCESS; +// +//} +// +//} +//} +//} \ No newline at end of file diff --git a/cpp/src/license/LicenseLibrary.h b/cpp/src/license/LicenseLibrary.h index 3340e60caa..427dda8d47 100644 --- a/cpp/src/license/LicenseLibrary.h +++ b/cpp/src/license/LicenseLibrary.h @@ -1,105 +1,105 @@ -#pragma once - -#include "LicenseFile.h" -#include "GPUInfoFile.h" - -#include "utils/Error.h" - -#include -#include -#include - -#include -#include -#include - - -namespace zilliz { -namespace milvus { -namespace server { - -class LicenseLibrary { - public: - // Part 0: File check - static bool - IsFileExistent(const std::string &path); - - // Part 1: Get GPU Info - static ServerError - GetDeviceCount(int &device_count); - - static ServerError - GetUUID(int device_count, std::vector &uuid_array); - - static ServerError - GetUUIDMD5(int device_count, std::vector &uuid_array, std::vector &md5_array); - - - static ServerError - GetUUIDSHA256(const int &device_count, - std::vector &uuid_array, - std::vector &sha_array); - - static ServerError - GetSystemTime(time_t &system_time); - - // Part 2: Handle License File - static ServerError - LicenseFileSerialization(const std::string &path, - int device_count, - const std::map &uuid_encrption_map, - time_t starting_time, - time_t end_time); - - static ServerError - LicenseFileDeserialization(const std::string &path, - int &device_count, - std::map &uuid_encrption_map, - time_t &starting_time, - time_t &end_time); - +//#pragma once +// +//#include "LicenseFile.h" +//#include "GPUInfoFile.h" +// +//#include "utils/Error.h" +// +//#include +//#include +//#include +// +//#include +//#include +//#include +// +// +//namespace zilliz { +//namespace milvus { +//namespace server { +// +//class LicenseLibrary { +// public: +// // Part 0: File check +// static bool +// IsFileExistent(const std::string &path); +// +// // Part 1: Get GPU Info // static ServerError -// SecretFileSerialization(const std::string &path, -// const time_t &update_time, -// const off_t &file_size, -// const time_t &starting_time, -// const time_t &end_time, -// const std::string &file_md5); +// GetDeviceCount(int &device_count); // // static ServerError -// SecretFileDeserialization(const std::string &path, -// time_t &update_time, -// off_t &file_size, -// time_t &starting_time, -// time_t &end_time, -// std::string &file_md5); - - // Part 3: File attribute: UpdateTime Time/ Size/ MD5 - static ServerError - GetFileUpdateTimeAndSize(const std::string &path, time_t &update_time, off_t &file_size); - - static ServerError - GetFileMD5(const std::string &path, std::string &filemd5); - - // Part 4: GPU Info File Serialization/Deserialization - static ServerError - GPUinfoFileSerialization(const std::string &path, - int device_count, - const std::map &uuid_encrption_map); - static ServerError - GPUinfoFileDeserialization(const std::string &path, - int &device_count, - std::map &uuid_encrption_map); - - static ServerError - GetDateTime(const char *cha, time_t &data_time); - - - private: - static constexpr int sha256_length_ = 32; -}; - - -} -} -} +// GetUUID(int device_count, std::vector &uuid_array); +// +// static ServerError +// GetUUIDMD5(int device_count, std::vector &uuid_array, std::vector &md5_array); +// +// +// static ServerError +// GetUUIDSHA256(const int &device_count, +// std::vector &uuid_array, +// std::vector &sha_array); +// +// static ServerError +// GetSystemTime(time_t &system_time); +// +// // Part 2: Handle License File +// static ServerError +// LicenseFileSerialization(const std::string &path, +// int device_count, +// const std::map &uuid_encrption_map, +// time_t starting_time, +// time_t end_time); +// +// static ServerError +// LicenseFileDeserialization(const std::string &path, +// int &device_count, +// std::map &uuid_encrption_map, +// time_t &starting_time, +// time_t &end_time); +// +//// static ServerError +//// SecretFileSerialization(const std::string &path, +//// const time_t &update_time, +//// const off_t &file_size, +//// const time_t &starting_time, +//// const time_t &end_time, +//// const std::string &file_md5); +//// +//// static ServerError +//// SecretFileDeserialization(const std::string &path, +//// time_t &update_time, +//// off_t &file_size, +//// time_t &starting_time, +//// time_t &end_time, +//// std::string &file_md5); +// +// // Part 3: File attribute: UpdateTime Time/ Size/ MD5 +// static ServerError +// GetFileUpdateTimeAndSize(const std::string &path, time_t &update_time, off_t &file_size); +// +// static ServerError +// GetFileMD5(const std::string &path, std::string &filemd5); +// +// // Part 4: GPU Info File Serialization/Deserialization +// static ServerError +// GPUinfoFileSerialization(const std::string &path, +// int device_count, +// const std::map &uuid_encrption_map); +// static ServerError +// GPUinfoFileDeserialization(const std::string &path, +// int &device_count, +// std::map &uuid_encrption_map); +// +// static ServerError +// GetDateTime(const char *cha, time_t &data_time); +// +// +// private: +// static constexpr int sha256_length_ = 32; +//}; +// +// +//} +//} +//} diff --git a/cpp/src/utils/CommonUtil.cpp b/cpp/src/utils/CommonUtil.cpp index 0583150d86..19157af9f2 100644 --- a/cpp/src/utils/CommonUtil.cpp +++ b/cpp/src/utils/CommonUtil.cpp @@ -51,7 +51,7 @@ bool CommonUtil::GetSystemAvailableThreads(unsigned int &threadCnt) { return true; } -bool CommonUtil::IsDirectoryExit(const std::string &path) +bool CommonUtil::IsDirectoryExist(const std::string &path) { DIR *dp = nullptr; if ((dp = opendir(path.c_str())) == nullptr) { @@ -182,7 +182,7 @@ void CommonUtil::ConvertTime(time_t time_integer, tm &time_struct) { memcpy(&time_struct, t_m, sizeof(tm)); } -void ConvertTime(tm time_struct, time_t &time_integer) { +void CommonUtil::ConvertTime(tm time_struct, time_t &time_integer) { time_integer = mktime(&time_struct); } diff --git a/cpp/src/utils/CommonUtil.h b/cpp/src/utils/CommonUtil.h index 4de87badbc..f1db3c059d 100755 --- a/cpp/src/utils/CommonUtil.h +++ b/cpp/src/utils/CommonUtil.h @@ -20,7 +20,7 @@ class CommonUtil { static bool GetSystemAvailableThreads(unsigned int &threadCnt); static bool IsFileExist(const std::string &path); - static bool IsDirectoryExit(const std::string &path); + static bool IsDirectoryExist(const std::string &path); static ServerError CreateDirectory(const std::string &path); static ServerError DeleteDirectory(const std::string &path); diff --git a/cpp/unittest/CMakeLists.txt b/cpp/unittest/CMakeLists.txt index c9d2a2c83e..38046617ae 100644 --- a/cpp/unittest/CMakeLists.txt +++ b/cpp/unittest/CMakeLists.txt @@ -42,6 +42,6 @@ set(unittest_libs add_subdirectory(server) add_subdirectory(db) add_subdirectory(faiss_wrapper) -add_subdirectory(license) +#add_subdirectory(license) add_subdirectory(metrics) add_subdirectory(storage) \ No newline at end of file diff --git a/cpp/unittest/license/license_check_test.cpp b/cpp/unittest/license/license_check_test.cpp index 2621ff58ae..0ad719bb60 100644 --- a/cpp/unittest/license/license_check_test.cpp +++ b/cpp/unittest/license/license_check_test.cpp @@ -1,184 +1,184 @@ +//// +//// Created by zilliz on 19-5-13. +//// // -// Created by zilliz on 19-5-13. +//#include "utils/Log.h" +//#include "license/LicenseCheck.h" +//#include "utils/Error.h" +// +//#include +//#include +// +// +//using namespace zilliz::milvus; +// +//TEST(LicenseLibraryTest, CHECK_TEST) { +// +// +// server::ServerError err; +// // 1. Set license file name +// std::string license_file_path("/tmp/milvus/abc.license"); +// +// // 2. Legality check +// err = server::LicenseCheck::LegalityCheck(license_file_path); +// ASSERT_EQ(err, server::SERVER_SUCCESS); +// +//} +// +//TEST(LicenseLibraryTest, CHECK_ERROR1_TEST){ +// +// server::ServerError err; +// // 1. Set license file name +// std::string license_file_path("/tmp/milvus/abc"); +// +// // 2. Legality check +// err = server::LicenseCheck::LegalityCheck(license_file_path); +// ASSERT_EQ(err, server::SERVER_SUCCESS); +//} +// +//TEST(LicenseLibraryTest, CHECK_ERROR2_TEST){ +// +// server::ServerError err; +// // 1. Set license file name +// std::string license_file_path("/tmp/milvus/abc.license"); +// +// // 2. Define output var +// int device_count; +// std::map uuid_encryption_map; +// time_t starting_time; +// time_t end_time; +// +// // 3. Read License File +// err = server::LicenseLibrary::LicenseFileDeserialization(license_file_path, +// device_count, +// uuid_encryption_map, +// starting_time, +// end_time); +// ASSERT_EQ(err, server::SERVER_SUCCESS); +// +// // 4. Change device count +// ++device_count; +// err = server::LicenseLibrary::LicenseFileSerialization(license_file_path, +// device_count, +// uuid_encryption_map, +// starting_time, +// end_time); +// ASSERT_EQ(err, server::SERVER_SUCCESS); +// +// // 5. Legality check +// err = server::LicenseCheck::LegalityCheck(license_file_path); +// ASSERT_EQ(err, server::SERVER_SUCCESS); +//} +// +//TEST(LicenseLibraryTest, CHECK_ERROR3_TEST){ +// +// server::ServerError err; +// // 1. Set license file name +// std::string license_file_path("/tmp/milvus/abc.license"); +// +// // 2. Define output var +// int device_count; +// std::map uuid_encryption_map; +// time_t starting_time; +// time_t end_time; +// +// // 3. Read License File +// err = server::LicenseLibrary::LicenseFileDeserialization(license_file_path, +// device_count, +// uuid_encryption_map, +// starting_time, +// end_time); +// ASSERT_EQ(err, server::SERVER_SUCCESS); +// +// // 4. Change device count +// if(device_count) uuid_encryption_map[0]+="u"; +// err = server::LicenseLibrary::LicenseFileSerialization(license_file_path, +// device_count, +// uuid_encryption_map, +// starting_time, +// end_time); +// ASSERT_EQ(err, server::SERVER_SUCCESS); +// +// // 5. Legality check +// err = server::LicenseCheck::LegalityCheck(license_file_path); +// ASSERT_EQ(err, server::SERVER_SUCCESS); +//} +// +//TEST(LicenseLibraryTest, CHECK_ERROR4_1_TEST){ +// +// server::ServerError err; +// // 1. Set license file name +// std::string license_file_path("/tmp/milvus/abc.license"); +// +// // 2. Define output var +// int device_count; +// std::map uuid_encryption_map; +// time_t starting_time; +// time_t end_time; +// +// // 3. Read License File +// err = server::LicenseLibrary::LicenseFileDeserialization(license_file_path, +// device_count, +// uuid_encryption_map, +// starting_time, +// end_time); +// ASSERT_EQ(err, server::SERVER_SUCCESS); +// +// // 4. Change starting time +// time_t system_time; +// err = server::LicenseLibrary::GetSystemTime(system_time); +// ASSERT_EQ(err, server::SERVER_SUCCESS); +// +// system_time+=60*60*24; +// +// err = server::LicenseLibrary::LicenseFileSerialization(license_file_path, +// device_count, +// uuid_encryption_map, +// system_time, +// end_time); +// ASSERT_EQ(err, server::SERVER_SUCCESS); +// +// // 5. Legality check +// err = server::LicenseCheck::LegalityCheck(license_file_path); +// ASSERT_EQ(err, server::SERVER_SUCCESS); +//} +// +//TEST(LicenseLibraryTest, CHECK_ERROR4_2_TEST){ +// +// server::ServerError err; +// // 1. Set license file name +// std::string license_file_path("/tmp/milvus/abc.license"); +// +// // 2. Define output var +// int device_count; +// std::map uuid_encryption_map; +// time_t starting_time; +// time_t end_time; +// +// // 3. Read License File +// err = server::LicenseLibrary::LicenseFileDeserialization(license_file_path, +// device_count, +// uuid_encryption_map, +// starting_time, +// end_time); +// ASSERT_EQ(err, server::SERVER_SUCCESS); +// +// // 4. Change end time +// time_t system_time; +// err = server::LicenseLibrary::GetSystemTime(system_time); +// ASSERT_EQ(err, server::SERVER_SUCCESS); +// +// system_time-=100; +// +// err = server::LicenseLibrary::LicenseFileSerialization(license_file_path, +// device_count, +// uuid_encryption_map, +// starting_time, +// system_time); +// ASSERT_EQ(err, server::SERVER_SUCCESS); +// +// // 5. Legality check +// err = server::LicenseCheck::LegalityCheck(license_file_path); +// ASSERT_EQ(err, server::SERVER_SUCCESS); +//} // - -#include "utils/Log.h" -#include "license/LicenseCheck.h" -#include "utils/Error.h" - -#include -#include - - -using namespace zilliz::milvus; - -TEST(LicenseLibraryTest, CHECK_TEST) { - - - server::ServerError err; - // 1. Set license file name - std::string license_file_path("/tmp/milvus/abc.license"); - - // 2. Legality check - err = server::LicenseCheck::LegalityCheck(license_file_path); - ASSERT_EQ(err, server::SERVER_SUCCESS); - -} - -TEST(LicenseLibraryTest, CHECK_ERROR1_TEST){ - - server::ServerError err; - // 1. Set license file name - std::string license_file_path("/tmp/milvus/abc"); - - // 2. Legality check - err = server::LicenseCheck::LegalityCheck(license_file_path); - ASSERT_EQ(err, server::SERVER_SUCCESS); -} - -TEST(LicenseLibraryTest, CHECK_ERROR2_TEST){ - - server::ServerError err; - // 1. Set license file name - std::string license_file_path("/tmp/milvus/abc.license"); - - // 2. Define output var - int device_count; - std::map uuid_encryption_map; - time_t starting_time; - time_t end_time; - - // 3. Read License File - err = server::LicenseLibrary::LicenseFileDeserialization(license_file_path, - device_count, - uuid_encryption_map, - starting_time, - end_time); - ASSERT_EQ(err, server::SERVER_SUCCESS); - - // 4. Change device count - ++device_count; - err = server::LicenseLibrary::LicenseFileSerialization(license_file_path, - device_count, - uuid_encryption_map, - starting_time, - end_time); - ASSERT_EQ(err, server::SERVER_SUCCESS); - - // 5. Legality check - err = server::LicenseCheck::LegalityCheck(license_file_path); - ASSERT_EQ(err, server::SERVER_SUCCESS); -} - -TEST(LicenseLibraryTest, CHECK_ERROR3_TEST){ - - server::ServerError err; - // 1. Set license file name - std::string license_file_path("/tmp/milvus/abc.license"); - - // 2. Define output var - int device_count; - std::map uuid_encryption_map; - time_t starting_time; - time_t end_time; - - // 3. Read License File - err = server::LicenseLibrary::LicenseFileDeserialization(license_file_path, - device_count, - uuid_encryption_map, - starting_time, - end_time); - ASSERT_EQ(err, server::SERVER_SUCCESS); - - // 4. Change device count - if(device_count) uuid_encryption_map[0]+="u"; - err = server::LicenseLibrary::LicenseFileSerialization(license_file_path, - device_count, - uuid_encryption_map, - starting_time, - end_time); - ASSERT_EQ(err, server::SERVER_SUCCESS); - - // 5. Legality check - err = server::LicenseCheck::LegalityCheck(license_file_path); - ASSERT_EQ(err, server::SERVER_SUCCESS); -} - -TEST(LicenseLibraryTest, CHECK_ERROR4_1_TEST){ - - server::ServerError err; - // 1. Set license file name - std::string license_file_path("/tmp/milvus/abc.license"); - - // 2. Define output var - int device_count; - std::map uuid_encryption_map; - time_t starting_time; - time_t end_time; - - // 3. Read License File - err = server::LicenseLibrary::LicenseFileDeserialization(license_file_path, - device_count, - uuid_encryption_map, - starting_time, - end_time); - ASSERT_EQ(err, server::SERVER_SUCCESS); - - // 4. Change starting time - time_t system_time; - err = server::LicenseLibrary::GetSystemTime(system_time); - ASSERT_EQ(err, server::SERVER_SUCCESS); - - system_time+=60*60*24; - - err = server::LicenseLibrary::LicenseFileSerialization(license_file_path, - device_count, - uuid_encryption_map, - system_time, - end_time); - ASSERT_EQ(err, server::SERVER_SUCCESS); - - // 5. Legality check - err = server::LicenseCheck::LegalityCheck(license_file_path); - ASSERT_EQ(err, server::SERVER_SUCCESS); -} - -TEST(LicenseLibraryTest, CHECK_ERROR4_2_TEST){ - - server::ServerError err; - // 1. Set license file name - std::string license_file_path("/tmp/milvus/abc.license"); - - // 2. Define output var - int device_count; - std::map uuid_encryption_map; - time_t starting_time; - time_t end_time; - - // 3. Read License File - err = server::LicenseLibrary::LicenseFileDeserialization(license_file_path, - device_count, - uuid_encryption_map, - starting_time, - end_time); - ASSERT_EQ(err, server::SERVER_SUCCESS); - - // 4. Change end time - time_t system_time; - err = server::LicenseLibrary::GetSystemTime(system_time); - ASSERT_EQ(err, server::SERVER_SUCCESS); - - system_time-=100; - - err = server::LicenseLibrary::LicenseFileSerialization(license_file_path, - device_count, - uuid_encryption_map, - starting_time, - system_time); - ASSERT_EQ(err, server::SERVER_SUCCESS); - - // 5. Legality check - err = server::LicenseCheck::LegalityCheck(license_file_path); - ASSERT_EQ(err, server::SERVER_SUCCESS); -} - diff --git a/cpp/unittest/license/license_library_tests.cpp b/cpp/unittest/license/license_library_tests.cpp index 406fd863c1..e96a1ecb2b 100644 --- a/cpp/unittest/license/license_library_tests.cpp +++ b/cpp/unittest/license/license_library_tests.cpp @@ -1,214 +1,214 @@ -/******************************************************************************* - * Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved - * Unauthorized copying of this file, via any medium is strictly prohibited. - * Proprietary and confidential. - ******************************************************************************/ - - -#include "utils/Log.h" -#include "license/LicenseLibrary.h" -#include "utils/Error.h" - -#include -#include - - -using namespace zilliz::milvus; - -TEST(LicenseLibraryTest, FILE_EXISTENT_TEST) { - - std::string hosts_file = "/etc/hosts"; - ASSERT_EQ(server::LicenseLibrary::IsFileExistent(hosts_file), true); - - std::string no_exist_file = "/temp/asdaasd"; - ASSERT_EQ(server::LicenseLibrary::IsFileExistent(no_exist_file), false); - - std::string directory = "/tmp"; - ASSERT_EQ(server::LicenseLibrary::IsFileExistent(directory), false); -} - -TEST(LicenseLibraryTest, GPU_INFO_TEST) { - - int device_count = 0; - server::ServerError err = server::LicenseLibrary::GetDeviceCount(device_count); - ASSERT_EQ(err, server::SERVER_SUCCESS); - std::cout << "Device Count: " << device_count << std::endl; - - std::vector uuid_array; - err = server::LicenseLibrary::GetUUID(device_count, uuid_array); - ASSERT_EQ(err, server::SERVER_SUCCESS); - for (long i = 0; i < device_count; ++i) { - std::cout << "Device Id: " << i << ", UUID: " << uuid_array[i] << std::endl; - } - - std::vector uuid_md5_array; - err = server::LicenseLibrary::GetUUIDMD5(device_count, uuid_array, uuid_md5_array); - ASSERT_EQ(err, server::SERVER_SUCCESS); - for (long i = 0; i < device_count; ++i) { - std::cout << "Device Id: " << i << ", UUID: " << uuid_array[i] << ", UUID_MD5: " << uuid_md5_array[i] - << std::endl; - } - - std::vector uuid_sha256_array; - err = server::LicenseLibrary::GetUUIDSHA256(device_count, uuid_array, uuid_sha256_array); - ASSERT_EQ(err, server::SERVER_SUCCESS); - for (long i = 0; i < device_count; ++i) { - std::cout << "Device Id: " << i << ", UUID: " << uuid_array[i] << ", UUID_SHA256: " - << uuid_sha256_array[i] << std::endl; - } - - time_t systemtime; - err = server::LicenseLibrary::GetSystemTime(systemtime); - ASSERT_EQ(err, server::SERVER_SUCCESS); - std::cout << "System Time: " << systemtime << std::endl; - -} - -TEST(LicenseLibraryTest, LICENSE_FILE_TEST) { - - // 0. File check - std::string test("/tmp/a.test"); - bool is = server::LicenseLibrary::IsFileExistent(test); - ASSERT_EQ(is, false); - - // 1. Get Device Count - int device_count = 0; - server::ServerError err = server::LicenseLibrary::GetDeviceCount(device_count); - ASSERT_EQ(err, server::SERVER_SUCCESS); - - // 2. Get All GPU UUID - std::vector uuid_array; - err = server::LicenseLibrary::GetUUID(device_count, uuid_array); - ASSERT_EQ(err, server::SERVER_SUCCESS); - - // 3. Get UUID SHA256 - std::vector uuid_sha256_array; - err = server::LicenseLibrary::GetUUIDSHA256(device_count, uuid_array, uuid_sha256_array); - ASSERT_EQ(err, server::SERVER_SUCCESS); - - // 4. Generate GPU ID map with GPU UUID - std::map uuid_encrption_map; - for (int i = 0; i < device_count; ++i) { - uuid_encrption_map[i] = uuid_sha256_array[i]; - } - - // 5.GPU_info File - std::string GPU_info_file_path("/tmp/milvus.info"); - - - // 6. Generate GPU_info File - err = server::LicenseLibrary::GPUinfoFileSerialization(GPU_info_file_path, - device_count, - uuid_encrption_map); - ASSERT_EQ(err, server::SERVER_SUCCESS); - - // 7. Define output var - int output_info_device_count = 0; - std::map output_info_uuid_encrption_map; - - // 8. Read GPU_info File - err = server::LicenseLibrary::GPUinfoFileDeserialization(GPU_info_file_path, - output_info_device_count, - output_info_uuid_encrption_map); - ASSERT_EQ(err, server::SERVER_SUCCESS); - - ASSERT_EQ(device_count, output_info_device_count); - for (int i = 0; i < device_count; ++i) { - ASSERT_EQ(uuid_encrption_map[i], output_info_uuid_encrption_map[i]); - } - - // 9. Set license file name - std::string license_file_path("/tmp/milvus.license"); - - // 10. Get System Time/starting_time ans End Time - time_t sysyem_time; - err = server::LicenseLibrary::GetSystemTime(sysyem_time); - ASSERT_EQ(err, server::SERVER_SUCCESS); - - // 11.GetDateTime - time_t starting_time; - time_t end_time; - const char *string_starting_time = "2019-05-10"; - const char *string_end_time = "2022-05-10"; - err = server::LicenseLibrary::GetDateTime(string_starting_time, starting_time); - ASSERT_EQ(err, server::SERVER_SUCCESS); - err = server::LicenseLibrary::GetDateTime(string_end_time, end_time); - ASSERT_EQ(err, server::SERVER_SUCCESS); - - // 12. Generate License File - err = server::LicenseLibrary::LicenseFileSerialization(license_file_path, - device_count, - uuid_encrption_map, - starting_time, - end_time); - ASSERT_EQ(err, server::SERVER_SUCCESS); - - // 13. Define output var - int output_device_count = 0; - std::map output_uuid_encrption_map; - time_t output_starting_time; - time_t output_end_time; - - // 14. Read License File - err = server::LicenseLibrary::LicenseFileDeserialization(license_file_path, - output_device_count, - output_uuid_encrption_map, - output_starting_time, - output_end_time); - ASSERT_EQ(err, server::SERVER_SUCCESS); - - ASSERT_EQ(device_count, output_device_count); - ASSERT_EQ(starting_time, output_starting_time); - ASSERT_EQ(end_time, output_end_time); - - for (int i = 0; i < device_count; ++i) { - ASSERT_EQ(uuid_encrption_map[i], output_uuid_encrption_map[i]); - } - -// // 15. Get License File Attribute -// time_t update_time; -// off_t file_size; -// err = server::LicenseLibrary::GetFileUpdateTimeAndSize(license_file_path, update_time, file_size); +///******************************************************************************* +// * Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved +// * Unauthorized copying of this file, via any medium is strictly prohibited. +// * Proprietary and confidential. +// ******************************************************************************/ +// +// +//#include "utils/Log.h" +//#include "license/LicenseLibrary.h" +//#include "utils/Error.h" +// +//#include +//#include +// +// +//using namespace zilliz::milvus; +// +//TEST(LicenseLibraryTest, FILE_EXISTENT_TEST) { +// +// std::string hosts_file = "/etc/hosts"; +// ASSERT_EQ(server::LicenseLibrary::IsFileExistent(hosts_file), true); +// +// std::string no_exist_file = "/temp/asdaasd"; +// ASSERT_EQ(server::LicenseLibrary::IsFileExistent(no_exist_file), false); +// +// std::string directory = "/tmp"; +// ASSERT_EQ(server::LicenseLibrary::IsFileExistent(directory), false); +//} +// +//TEST(LicenseLibraryTest, GPU_INFO_TEST) { +// +// int device_count = 0; +// server::ServerError err = server::LicenseLibrary::GetDeviceCount(device_count); +// ASSERT_EQ(err, server::SERVER_SUCCESS); +// std::cout << "Device Count: " << device_count << std::endl; +// +// std::vector uuid_array; +// err = server::LicenseLibrary::GetUUID(device_count, uuid_array); +// ASSERT_EQ(err, server::SERVER_SUCCESS); +// for (long i = 0; i < device_count; ++i) { +// std::cout << "Device Id: " << i << ", UUID: " << uuid_array[i] << std::endl; +// } +// +// std::vector uuid_md5_array; +// err = server::LicenseLibrary::GetUUIDMD5(device_count, uuid_array, uuid_md5_array); +// ASSERT_EQ(err, server::SERVER_SUCCESS); +// for (long i = 0; i < device_count; ++i) { +// std::cout << "Device Id: " << i << ", UUID: " << uuid_array[i] << ", UUID_MD5: " << uuid_md5_array[i] +// << std::endl; +// } +// +// std::vector uuid_sha256_array; +// err = server::LicenseLibrary::GetUUIDSHA256(device_count, uuid_array, uuid_sha256_array); +// ASSERT_EQ(err, server::SERVER_SUCCESS); +// for (long i = 0; i < device_count; ++i) { +// std::cout << "Device Id: " << i << ", UUID: " << uuid_array[i] << ", UUID_SHA256: " +// << uuid_sha256_array[i] << std::endl; +// } +// +// time_t systemtime; +// err = server::LicenseLibrary::GetSystemTime(systemtime); +// ASSERT_EQ(err, server::SERVER_SUCCESS); +// std::cout << "System Time: " << systemtime << std::endl; +// +//} +// +//TEST(LicenseLibraryTest, LICENSE_FILE_TEST) { +// +// // 0. File check +// std::string test("/tmp/a.test"); +// bool is = server::LicenseLibrary::IsFileExistent(test); +// ASSERT_EQ(is, false); +// +// // 1. Get Device Count +// int device_count = 0; +// server::ServerError err = server::LicenseLibrary::GetDeviceCount(device_count); // ASSERT_EQ(err, server::SERVER_SUCCESS); // -// // 16. Get License File MD5 -// std::string file_md5; -// err = server::LicenseLibrary::GetFileMD5(license_file_path, file_md5); +// // 2. Get All GPU UUID +// std::vector uuid_array; +// err = server::LicenseLibrary::GetUUID(device_count, uuid_array); // ASSERT_EQ(err, server::SERVER_SUCCESS); - - - -// // 17. Generate Secret File -// std::string secret_file_path("/tmp/milvus.secret"); -// err = server::LicenseLibrary::SecretFileSerialization(secret_file_path, -// update_time, -// file_size, -// starting_time, -// end_time, -// file_md5); +// +// // 3. Get UUID SHA256 +// std::vector uuid_sha256_array; +// err = server::LicenseLibrary::GetUUIDSHA256(device_count, uuid_array, uuid_sha256_array); // ASSERT_EQ(err, server::SERVER_SUCCESS); - -// // 18. Define output var -// time_t output_update_time; -// off_t output_file_size; +// +// // 4. Generate GPU ID map with GPU UUID +// std::map uuid_encrption_map; +// for (int i = 0; i < device_count; ++i) { +// uuid_encrption_map[i] = uuid_sha256_array[i]; +// } +// +// // 5.GPU_info File +// std::string GPU_info_file_path("/tmp/milvus.info"); +// +// +// // 6. Generate GPU_info File +// err = server::LicenseLibrary::GPUinfoFileSerialization(GPU_info_file_path, +// device_count, +// uuid_encrption_map); +// ASSERT_EQ(err, server::SERVER_SUCCESS); +// +// // 7. Define output var +// int output_info_device_count = 0; +// std::map output_info_uuid_encrption_map; +// +// // 8. Read GPU_info File +// err = server::LicenseLibrary::GPUinfoFileDeserialization(GPU_info_file_path, +// output_info_device_count, +// output_info_uuid_encrption_map); +// ASSERT_EQ(err, server::SERVER_SUCCESS); +// +// ASSERT_EQ(device_count, output_info_device_count); +// for (int i = 0; i < device_count; ++i) { +// ASSERT_EQ(uuid_encrption_map[i], output_info_uuid_encrption_map[i]); +// } +// +// // 9. Set license file name +// std::string license_file_path("/tmp/milvus.license"); +// +// // 10. Get System Time/starting_time ans End Time +// time_t sysyem_time; +// err = server::LicenseLibrary::GetSystemTime(sysyem_time); +// ASSERT_EQ(err, server::SERVER_SUCCESS); +// +// // 11.GetDateTime +// time_t starting_time; +// time_t end_time; +// const char *string_starting_time = "2019-05-10"; +// const char *string_end_time = "2022-05-10"; +// err = server::LicenseLibrary::GetDateTime(string_starting_time, starting_time); +// ASSERT_EQ(err, server::SERVER_SUCCESS); +// err = server::LicenseLibrary::GetDateTime(string_end_time, end_time); +// ASSERT_EQ(err, server::SERVER_SUCCESS); +// +// // 12. Generate License File +// err = server::LicenseLibrary::LicenseFileSerialization(license_file_path, +// device_count, +// uuid_encrption_map, +// starting_time, +// end_time); +// ASSERT_EQ(err, server::SERVER_SUCCESS); +// +// // 13. Define output var +// int output_device_count = 0; +// std::map output_uuid_encrption_map; // time_t output_starting_time; // time_t output_end_time; -// std::string output_file_md5; - -// // 19. Read License File -// err = server::LicenseLibrary::SecretFileDeserialization(secret_file_path, -// output_update_time, -// output_file_size, -// output_starting_time, -// output_end_time, -// output_file_md5); +// +// // 14. Read License File +// err = server::LicenseLibrary::LicenseFileDeserialization(license_file_path, +// output_device_count, +// output_uuid_encrption_map, +// output_starting_time, +// output_end_time); // ASSERT_EQ(err, server::SERVER_SUCCESS); // -// ASSERT_EQ(update_time, output_update_time); -// ASSERT_EQ(file_size, output_file_size); +// ASSERT_EQ(device_count, output_device_count); // ASSERT_EQ(starting_time, output_starting_time); // ASSERT_EQ(end_time, output_end_time); -// ASSERT_EQ(file_md5, output_file_md5); - - -} +// +// for (int i = 0; i < device_count; ++i) { +// ASSERT_EQ(uuid_encrption_map[i], output_uuid_encrption_map[i]); +// } +// +//// // 15. Get License File Attribute +//// time_t update_time; +//// off_t file_size; +//// err = server::LicenseLibrary::GetFileUpdateTimeAndSize(license_file_path, update_time, file_size); +//// ASSERT_EQ(err, server::SERVER_SUCCESS); +//// +//// // 16. Get License File MD5 +//// std::string file_md5; +//// err = server::LicenseLibrary::GetFileMD5(license_file_path, file_md5); +//// ASSERT_EQ(err, server::SERVER_SUCCESS); +// +// +// +//// // 17. Generate Secret File +//// std::string secret_file_path("/tmp/milvus.secret"); +//// err = server::LicenseLibrary::SecretFileSerialization(secret_file_path, +//// update_time, +//// file_size, +//// starting_time, +//// end_time, +//// file_md5); +//// ASSERT_EQ(err, server::SERVER_SUCCESS); +// +//// // 18. Define output var +//// time_t output_update_time; +//// off_t output_file_size; +//// time_t output_starting_time; +//// time_t output_end_time; +//// std::string output_file_md5; +// +//// // 19. Read License File +//// err = server::LicenseLibrary::SecretFileDeserialization(secret_file_path, +//// output_update_time, +//// output_file_size, +//// output_starting_time, +//// output_end_time, +//// output_file_md5); +//// ASSERT_EQ(err, server::SERVER_SUCCESS); +//// +//// ASSERT_EQ(update_time, output_update_time); +//// ASSERT_EQ(file_size, output_file_size); +//// ASSERT_EQ(starting_time, output_starting_time); +//// ASSERT_EQ(end_time, output_end_time); +//// ASSERT_EQ(file_md5, output_file_md5); +// +// +//} diff --git a/cpp/unittest/server/CMakeLists.txt b/cpp/unittest/server/CMakeLists.txt index 4d34f0e2b7..c4112cda9e 100644 --- a/cpp/unittest/server/CMakeLists.txt +++ b/cpp/unittest/server/CMakeLists.txt @@ -18,6 +18,7 @@ set(utils_srcs ${MILVUS_ENGINE_SRC}/utils/StringHelpFunctions.cpp ${MILVUS_ENGINE_SRC}/utils/TimeRecorder.cpp ${MILVUS_ENGINE_SRC}/utils/CommonUtil.cpp + ${MILVUS_ENGINE_SRC}/utils/LogUtil.cpp ) cuda_add_executable(server_test @@ -54,3 +55,11 @@ target_link_libraries(server_test ) install(TARGETS server_test DESTINATION bin) + +configure_file(appendix/server_config.yaml + "${CMAKE_CURRENT_BINARY_DIR}/milvus/conf/server_config.yaml" + COPYONLY) + +configure_file(appendix/log_config.conf + "${CMAKE_CURRENT_BINARY_DIR}/milvus/conf/log_config.conf" + COPYONLY) diff --git a/cpp/unittest/server/appendix/log_config.conf b/cpp/unittest/server/appendix/log_config.conf new file mode 100644 index 0000000000..29d46a7fe5 --- /dev/null +++ b/cpp/unittest/server/appendix/log_config.conf @@ -0,0 +1,27 @@ +* GLOBAL: + FORMAT = "%datetime | %level | %logger | %msg" + FILENAME = "/tmp/milvus/logs/milvus-%datetime{%H:%m}-global.log" + ENABLED = true + TO_FILE = true + TO_STANDARD_OUTPUT = false + SUBSECOND_PRECISION = 3 + PERFORMANCE_TRACKING = false + MAX_LOG_FILE_SIZE = 2097152 ## Throw log files away after 2MB +* DEBUG: + FILENAME = "/tmp/milvus/logs/milvus-%datetime{%H:%m}-debug.log" + ENABLED = true +* WARNING: + FILENAME = "/tmp/milvus/logs/milvus-%datetime{%H:%m}-warning.log" +* TRACE: + FILENAME = "/tmp/milvus/logs/milvus-%datetime{%H:%m}-trace.log" +* VERBOSE: + FORMAT = "%datetime{%d/%M/%y} | %level-%vlevel | %msg" + TO_FILE = false + TO_STANDARD_OUTPUT = false +## Error logs +* ERROR: + ENABLED = true + FILENAME = "/tmp/milvus/logs/milvus-%datetime{%H:%m}-error.log" +* FATAL: + ENABLED = true + FILENAME = "/tmp/milvus/logs/milvus-%datetime{%H:%m}-fatal.log" diff --git a/cpp/unittest/server/appendix/server_config.yaml b/cpp/unittest/server/appendix/server_config.yaml new file mode 100644 index 0000000000..9019461940 --- /dev/null +++ b/cpp/unittest/server/appendix/server_config.yaml @@ -0,0 +1,28 @@ +server_config: + address: 0.0.0.0 + port: 19530 # the port milvus listen to, default: 19530, range: 1025 ~ 65534 + gpu_index: 0 # the gpu milvus use, default: 0, range: 0 ~ gpu number - 1 + mode: single # milvus deployment type: single, cluster + +db_config: + db_path: /tmp/milvus # milvus data storage path + db_backend_url: http://127.0.0.1 # meta database uri + index_building_threshold: 1024 # index building trigger threshold, default: 1024, unit: MB + archive_disk_threshold: 512 # triger archive action if storage size exceed this value, unit: GB + archive_days_threshold: 30 # files older than x days will be archived, unit: day + +metric_config: + is_startup: off # if monitoring start: on, off + collector: prometheus # metrics collector: prometheus + prometheus_config: # following are prometheus configure + collect_type: pull # prometheus collect data method + port: 8080 # the port prometheus use to fetch metrics + push_gateway_ip_address: 127.0.0.1 # push method configure: push gateway ip address + push_gateway_port: 9091 # push method configure: push gateway port + + +license_config: # license configure + license_path: "/tmp/milvus/system.license" # license file path + +cache_config: # cache configure + cpu_cache_capacity: 16 # how many memory are used as cache, unit: GB, range: 0 ~ less than total memory diff --git a/cpp/unittest/server/cache_test.cpp b/cpp/unittest/server/cache_test.cpp index 05c292b11b..8c323d81d3 100644 --- a/cpp/unittest/server/cache_test.cpp +++ b/cpp/unittest/server/cache_test.cpp @@ -11,6 +11,23 @@ using namespace zilliz::milvus; +namespace { + +class InvalidCacheMgr : public cache::CacheMgr { +public: + InvalidCacheMgr() { + } +}; + +class LessItemCacheMgr : public cache::CacheMgr { +public: + LessItemCacheMgr() { + cache_ = std::make_shared(1UL << 12, 10); + } +}; + +} + TEST(CacheTest, CPU_CACHE_TEST) { cache::CacheMgr *cpu_mgr = cache::CpuCacheMgr::GetInstance(); @@ -78,4 +95,35 @@ TEST(CacheTest, GPU_CACHE_TEST) { gpu_mgr->ClearCache(); ASSERT_EQ(gpu_mgr->ItemCount(), 0); +} + +TEST(CacheTest, INVALID_TEST) { + { + InvalidCacheMgr mgr; + ASSERT_EQ(mgr.ItemCount(), 0); + ASSERT_FALSE(mgr.ItemExists("test")); + ASSERT_EQ(mgr.GetItem("test"), nullptr); + + mgr.InsertItem("test", cache::DataObjPtr()); + mgr.InsertItem("test", engine::Index_ptr(nullptr)); + mgr.EraseItem("test"); + mgr.PrintInfo(); + mgr.ClearCache(); + mgr.SetCapacity(100); + ASSERT_EQ(mgr.CacheCapacity(), 0); + ASSERT_EQ(mgr.CacheUsage(), 0); + } + + { + LessItemCacheMgr mgr; + for(int i = 0; i < 20; i++) { + std::shared_ptr raw_index(faiss::index_factory(2, "IDMap,Flat")); + engine::Index_ptr index = std::make_shared(raw_index); + index->ntotal = 2; + + cache::DataObjPtr obj = std::make_shared(index); + mgr.InsertItem("index_" + std::to_string(i), obj); + } + ASSERT_EQ(mgr.GetItem("index_0"), nullptr); + } } \ No newline at end of file diff --git a/cpp/unittest/server/config_test.cpp b/cpp/unittest/server/config_test.cpp index 4f76707dd2..4b10327cfb 100644 --- a/cpp/unittest/server/config_test.cpp +++ b/cpp/unittest/server/config_test.cpp @@ -10,13 +10,27 @@ using namespace zilliz::milvus; -static const std::string CONFIG_FILE_PATH = "../../../conf/server_config.yaml"; +namespace { + +static const std::string CONFIG_FILE_PATH = "./milvus/conf/server_config.yaml"; +static const std::string LOG_FILE_PATH = "./milvus/conf/log_config.conf"; + +} TEST(ConfigTest, CONFIG_TEST) { server::IConfigMgr* config_mgr = server::IConfigMgr::GetInstance(); - server::ServerError err = config_mgr->LoadConfigFile(CONFIG_FILE_PATH); + + server::ServerError err = config_mgr->LoadConfigFile(""); + ASSERT_EQ(err, server::SERVER_UNEXPECTED_ERROR); + + err = config_mgr->LoadConfigFile(LOG_FILE_PATH); + ASSERT_EQ(err, server::SERVER_UNEXPECTED_ERROR); + + err = config_mgr->LoadConfigFile(CONFIG_FILE_PATH); ASSERT_EQ(err, server::SERVER_SUCCESS); + config_mgr->Print(); + server::ConfigNode& root_config = config_mgr->GetRootNode(); server::ConfigNode& server_config = root_config.GetChild("server_config"); server::ConfigNode& db_config = root_config.GetChild("db_config"); @@ -43,10 +57,6 @@ TEST(ConfigTest, CONFIG_TEST) { auto children = server_config.GetChildren(); ASSERT_TRUE(children.empty()); - root_config.PrintAll(); - std::string all = root_config.DumpString(); - ASSERT_TRUE(!all.empty()); - server_config.ClearConfig(); auto configs = server_config.GetConfig(); ASSERT_TRUE(configs.empty()); @@ -56,9 +66,20 @@ TEST(ConfigTest, CONFIG_TEST) { auto seq = server_config.GetSequence("seq"); ASSERT_EQ(seq.size(), 2UL); + server::ConfigNode combine; + combine.Combine(server_config); + + combine.PrintAll(); + std::string all = combine.DumpString(); + ASSERT_TRUE(!all.empty()); + server_config.ClearSequences(); auto seqs = server_config.GetSequences(); ASSERT_TRUE(seqs.empty()); + + const server::ConfigNode const_node = root_config.GetChild("cache_config"); + float flt = const_node.GetFloatValue("cpu_cache_capacity"); + ASSERT_GT(flt, 0.0); } TEST(ConfigTest, SERVER_CONFIG_TEST) { @@ -73,5 +94,12 @@ TEST(ConfigTest, SERVER_CONFIG_TEST) { int32_t cap = node1.GetInt32Value("cpu_cache_capacity"); ASSERT_GT(cap, 0); + node1.SetValue("bool", "true"); + bool bt = node1.GetBoolValue("bool"); + ASSERT_TRUE(bt); + config.PrintAll(); + + const server::ServerConfig const_config = config; + server::ConfigNode node = const_config.GetConfig("aaa"); } \ No newline at end of file diff --git a/cpp/unittest/server/util_test.cpp b/cpp/unittest/server/util_test.cpp index 9a38092799..200bdc9776 100644 --- a/cpp/unittest/server/util_test.cpp +++ b/cpp/unittest/server/util_test.cpp @@ -11,35 +11,83 @@ #include "utils/StringHelpFunctions.h" #include "utils/TimeRecorder.h" #include "utils/BlockingQueue.h" +#include "utils/LogUtil.h" using namespace zilliz::milvus; namespace { +static const std::string LOG_FILE_PATH = "./milvus/conf/log_config.conf"; + using TimeUnit = server::TimeRecorder::TimeDisplayUnit; -double TestTimeRecorder(TimeUnit unit, int64_t log_level) { +double TestTimeRecorder(TimeUnit unit, int64_t log_level, int64_t sleep_ms) { server::TimeRecorder rc("test rc", unit, log_level); rc.Record("begin"); - std::this_thread::sleep_for(std::chrono::microseconds(10)); + std::this_thread::sleep_for(std::chrono::milliseconds(sleep_ms)); rc.Elapse("end"); return rc.Span(); } } -TEST(CommonTest, COMMON_TEST) { +TEST(UtilTest, EXCEPTION_TEST) { + std::string err_msg = "failed"; + server::ServerException ex(server::SERVER_UNEXPECTED_ERROR, err_msg); + ASSERT_EQ(ex.error_code(), server::SERVER_UNEXPECTED_ERROR); + std::string msg = ex.what(); + ASSERT_EQ(msg, err_msg); +} + +TEST(UtilTest, COMMON_TEST) { + unsigned long total_mem = 0, free_mem = 0; + server::CommonUtil::GetSystemMemInfo(total_mem, free_mem); + ASSERT_GT(total_mem, 0); + ASSERT_GT(free_mem, 0); + + unsigned int thread_cnt = 0; + server::CommonUtil::GetSystemAvailableThreads(thread_cnt); + ASSERT_GT(thread_cnt, 0); + std::string path1 = "/tmp/milvus_test/"; std::string path2 = path1 + "common_test_12345/"; std::string path3 = path2 + "abcdef"; server::ServerError err = server::CommonUtil::CreateDirectory(path3); ASSERT_EQ(err, server::SERVER_SUCCESS); + //test again + err = server::CommonUtil::CreateDirectory(path3); + ASSERT_EQ(err, server::SERVER_SUCCESS); - ASSERT_TRUE(server::CommonUtil::IsDirectoryExit(path3)); + ASSERT_TRUE(server::CommonUtil::IsDirectoryExist(path3)); + err = server::CommonUtil::DeleteDirectory(path1); + ASSERT_EQ(err, server::SERVER_SUCCESS); + //test again err = server::CommonUtil::DeleteDirectory(path1); ASSERT_EQ(err, server::SERVER_SUCCESS); - ASSERT_FALSE(server::CommonUtil::IsDirectoryExit(path1)); + ASSERT_FALSE(server::CommonUtil::IsDirectoryExist(path1)); + ASSERT_FALSE(server::CommonUtil::IsFileExist(path1)); + + std::string exe_path = server::CommonUtil::GetExePath(); + ASSERT_FALSE(exe_path.empty()); + + time_t tt; + time( &tt ); + tm time_struct; + memset(&time_struct, 0, sizeof(tm)); + server::CommonUtil::ConvertTime(tt, time_struct); + ASSERT_GT(time_struct.tm_year, 0); + ASSERT_GT(time_struct.tm_mon, 0); + ASSERT_GT(time_struct.tm_mday, 0); + server::CommonUtil::ConvertTime(time_struct, tt); + ASSERT_GT(tt, 0); + + bool res = server::CommonUtil::TimeStrToTime("2019-03-23", tt, time_struct); + ASSERT_EQ(time_struct.tm_year, 119); + ASSERT_EQ(time_struct.tm_mon, 2); + ASSERT_EQ(time_struct.tm_mday, 23); + ASSERT_GT(tt, 0); + ASSERT_TRUE(res); } TEST(UtilTest, STRINGFUNCTIONS_TEST) { @@ -47,29 +95,49 @@ TEST(UtilTest, STRINGFUNCTIONS_TEST) { server::StringHelpFunctions::TrimStringBlank(str); ASSERT_EQ(str, "test zilliz"); + str = "\"test zilliz\""; + server::StringHelpFunctions::TrimStringQuote(str, "\""); + ASSERT_EQ(str, "test zilliz"); + str = "a,b,c"; std::vector result; - server::StringHelpFunctions::SplitStringByDelimeter(str , ",", result); + server::ServerError err = server::StringHelpFunctions::SplitStringByDelimeter(str , ",", result); + ASSERT_EQ(err, server::SERVER_SUCCESS); + ASSERT_EQ(result.size(), 3UL); + + result.clear(); + err = server::StringHelpFunctions::SplitStringByQuote(str , ",", "\"", result); + ASSERT_EQ(err, server::SERVER_SUCCESS); + ASSERT_EQ(result.size(), 3UL); + + result.clear(); + err = server::StringHelpFunctions::SplitStringByQuote(str , ",", "", result); + ASSERT_EQ(err, server::SERVER_SUCCESS); ASSERT_EQ(result.size(), 3UL); str = "55,\"aa,gg,yy\",b"; result.clear(); - server::StringHelpFunctions::SplitStringByQuote(str , ",", "\"", result); + err = server::StringHelpFunctions::SplitStringByQuote(str , ",", "\"", result); + ASSERT_EQ(err, server::SERVER_SUCCESS); ASSERT_EQ(result.size(), 3UL); + + } TEST(UtilTest, TIMERECORDER_TEST) { - double span = TestTimeRecorder(TimeUnit::eTimeAutoUnit, 0); + double span = TestTimeRecorder(TimeUnit::eTimeAutoUnit, 0, 1001); ASSERT_GT(span, 0.0); - span = TestTimeRecorder(TimeUnit::eTimeHourUnit, 1); + span = TestTimeRecorder(TimeUnit::eTimeAutoUnit, 0, 101); ASSERT_GT(span, 0.0); - span = TestTimeRecorder(TimeUnit::eTimeMinuteUnit, 2); + span = TestTimeRecorder(TimeUnit::eTimeHourUnit, 1, 10); ASSERT_GT(span, 0.0); - span = TestTimeRecorder(TimeUnit::eTimeSecondUnit, 3); + span = TestTimeRecorder(TimeUnit::eTimeMinuteUnit, 2, 10); ASSERT_GT(span, 0.0); - span = TestTimeRecorder(TimeUnit::eTimeMilliSecUnit, 4); + span = TestTimeRecorder(TimeUnit::eTimeSecondUnit, 3, 10); ASSERT_GT(span, 0.0); - span = TestTimeRecorder(TimeUnit::eTimeMicroSecUnit, -1); + span = TestTimeRecorder(TimeUnit::eTimeMilliSecUnit, 4, 10); + ASSERT_GT(span, 0.0); + span = TestTimeRecorder(TimeUnit::eTimeMicroSecUnit, -1, 10); ASSERT_GT(span, 0.0); } @@ -98,5 +166,14 @@ TEST(UtilTest, BLOCKINGQUEUE_TEST) { str = bq.Take(); ASSERT_EQ(id, str); } + + ASSERT_EQ(bq.Size(), 0); } +TEST(UtilTest, LOG_TEST) { + int32_t res = server::InitLog(LOG_FILE_PATH); + ASSERT_EQ(res, 0); + + std::string fname = server::GetFileName(LOG_FILE_PATH); + ASSERT_EQ(fname, "log_config.conf"); +} From dd010b4dda84f6541478a946ef960a4bb30be6e9 Mon Sep 17 00:00:00 2001 From: starlord Date: Fri, 28 Jun 2019 18:27:37 +0800 Subject: [PATCH 033/189] remove ignore files Former-commit-id: b716231263036f3b02c4953f26efa4e0bc361c9c --- cpp/conf/log_config.conf | 27 --------------------------- cpp/conf/server_config.yaml | 27 --------------------------- 2 files changed, 54 deletions(-) delete mode 100644 cpp/conf/log_config.conf delete mode 100644 cpp/conf/server_config.yaml diff --git a/cpp/conf/log_config.conf b/cpp/conf/log_config.conf deleted file mode 100644 index 29d46a7fe5..0000000000 --- a/cpp/conf/log_config.conf +++ /dev/null @@ -1,27 +0,0 @@ -* GLOBAL: - FORMAT = "%datetime | %level | %logger | %msg" - FILENAME = "/tmp/milvus/logs/milvus-%datetime{%H:%m}-global.log" - ENABLED = true - TO_FILE = true - TO_STANDARD_OUTPUT = false - SUBSECOND_PRECISION = 3 - PERFORMANCE_TRACKING = false - MAX_LOG_FILE_SIZE = 2097152 ## Throw log files away after 2MB -* DEBUG: - FILENAME = "/tmp/milvus/logs/milvus-%datetime{%H:%m}-debug.log" - ENABLED = true -* WARNING: - FILENAME = "/tmp/milvus/logs/milvus-%datetime{%H:%m}-warning.log" -* TRACE: - FILENAME = "/tmp/milvus/logs/milvus-%datetime{%H:%m}-trace.log" -* VERBOSE: - FORMAT = "%datetime{%d/%M/%y} | %level-%vlevel | %msg" - TO_FILE = false - TO_STANDARD_OUTPUT = false -## Error logs -* ERROR: - ENABLED = true - FILENAME = "/tmp/milvus/logs/milvus-%datetime{%H:%m}-error.log" -* FATAL: - ENABLED = true - FILENAME = "/tmp/milvus/logs/milvus-%datetime{%H:%m}-fatal.log" diff --git a/cpp/conf/server_config.yaml b/cpp/conf/server_config.yaml deleted file mode 100644 index 8dc9e1d551..0000000000 --- a/cpp/conf/server_config.yaml +++ /dev/null @@ -1,27 +0,0 @@ -server_config: - address: 0.0.0.0 - port: 19530 # the port milvus listen to, default: 19530, range: 1025 ~ 65534 - gpu_index: 0 # the gpu milvus use, default: 0, range: 0 ~ gpu number - 1 - mode: single # milvus deployment type: single, cluster, read_only - -db_config: - db_path: /tmp/milvus # milvus data storage path - db_backend_url: http://127.0.0.1 # meta database uri - index_building_threshold: 1024 # index building trigger threshold, default: 1024, unit: MB - archive_disk_threshold: 512 # triger archive action if storage size exceed this value, unit: GB - archive_days_threshold: 30 # files older than x days will be archived, unit: day - -metric_config: - is_startup: off # if monitoring start: on, off - collector: prometheus # metrics collector: prometheus - prometheus_config: # following are prometheus configure - collect_type: pull # prometheus collect data method - port: 8080 # the port prometheus use to fetch metrics - push_gateway_ip_address: 127.0.0.1 # push method configure: push gateway ip address - push_gateway_port: 9091 # push method configure: push gateway port - -license_config: # license configure - license_path: "/tmp/milvus/system.license" # license file path - -cache_config: # cache configure - cpu_cache_capacity: 16 # how many memory are used as cache, unit: GB, range: 0 ~ less than total memory From 5a31d921a329793db63d5142b39d7e301d775c1b Mon Sep 17 00:00:00 2001 From: zhiru Date: Sun, 30 Jun 2019 13:28:38 +0800 Subject: [PATCH 034/189] update Former-commit-id: 1cbe55a405e356f7361cd807a80662b81ee522f8 --- cpp/CMakeLists.txt | 8 - cpp/mysqlNotes | 8 - cpp/src/db/DBImpl.cpp | 4 +- cpp/src/db/Factories.cpp | 14 +- cpp/src/db/Factories.h | 2 +- cpp/src/db/MySQLConnectionPool.h | 18 +- cpp/src/db/MySQLMetaImpl.cpp | 308 +++++++++++++++++++++++++++---- cpp/src/db/MySQLMetaImpl.h | 6 +- cpp/src/db/Options.h | 10 +- cpp/src/server/DBWrapper.cpp | 25 ++- cpp/src/server/ServerConfig.h | 1 + 11 files changed, 323 insertions(+), 81 deletions(-) delete mode 100644 cpp/mysqlNotes diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 58124c5296..116f30026d 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -113,21 +113,13 @@ link_directories(${MILVUS_BINARY_DIR}) set(MILVUS_ENGINE_INCLUDE ${PROJECT_SOURCE_DIR}/include) set(MILVUS_ENGINE_SRC ${PROJECT_SOURCE_DIR}/src) -#set(MILVUS_THIRD_PARTY ${CMAKE_CURRENT_SOURCE_DIR}/third_party) -#set(MILVUS_THIRD_PARTY_BUILD ${CMAKE_CURRENT_SOURCE_DIR}/third_party/build) add_compile_definitions(PROFILER=${PROFILER}) include_directories(${MILVUS_ENGINE_INCLUDE}) include_directories(${MILVUS_ENGINE_SRC}) -include_directories(/usr/local/cuda/include) -#include_directories(${MILVUS_THIRD_PARTY_BUILD}/include) link_directories(${CMAKE_CURRRENT_BINARY_DIR}) -#link_directories(${MILVUS_THIRD_PARTY_BUILD}/lib) -#link_directories(${MILVUS_THIRD_PARTY_BUILD}/lib64) -#execute_process(COMMAND bash build.sh -# WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/third_party) add_subdirectory(src) diff --git a/cpp/mysqlNotes b/cpp/mysqlNotes deleted file mode 100644 index 8aa151efff..0000000000 --- a/cpp/mysqlNotes +++ /dev/null @@ -1,8 +0,0 @@ -sudo apt-get install mysql-server -sudo apt-get install libmysqlclient-dev -sudo ln -s libmysqlclient.so libmysqlclient_r.so - -Install MySQL++ -./configure --enable-thread-check LDFLAGS='-pthread' -make -sudo make install diff --git a/cpp/src/db/DBImpl.cpp b/cpp/src/db/DBImpl.cpp index def216358b..92bbff2b84 100644 --- a/cpp/src/db/DBImpl.cpp +++ b/cpp/src/db/DBImpl.cpp @@ -139,7 +139,7 @@ DBImpl::DBImpl(const Options& options) meta_ptr_ = DBMetaImplFactory::Build(options.meta, options.mode); mem_mgr_ = std::make_shared(meta_ptr_, options_); // mem_mgr_ = (MemManagerPtr)(new MemManager(meta_ptr_, options_)); - if (options.mode != "read_only") { + if (options.mode != Options::MODE::READ_ONLY) { StartTimerTasks(); } } @@ -600,7 +600,7 @@ void DBImpl::BackgroundCompaction(std::set table_ids) { meta_ptr_->Archive(); int ttl = 1; - if (options_.mode == "cluster") { + if (options_.mode == Options::MODE::CLUSTER) { ttl = meta::D_SEC; // ENGINE_LOG_DEBUG << "Server mode is cluster. Clean up files with ttl = " << std::to_string(ttl) << "seconds."; } diff --git a/cpp/src/db/Factories.cpp b/cpp/src/db/Factories.cpp index 97e89e0994..231c3cce4f 100644 --- a/cpp/src/db/Factories.cpp +++ b/cpp/src/db/Factories.cpp @@ -29,15 +29,8 @@ DBMetaOptions DBMetaOptionsFactory::Build(const std::string& path) { p = ss.str(); } -// std::string uri; -// const char* uri_p = getenv("MILVUS_DB_META_URI"); -// if (uri_p) { -// uri = uri_p; -// } - DBMetaOptions meta; meta.path = p; -// meta.backend_uri = uri; return meta; } @@ -54,14 +47,9 @@ std::shared_ptr DBMetaImplFactory::Build() { } std::shared_ptr DBMetaImplFactory::Build(const DBMetaOptions& metaOptions, - const std::string& mode) { + const int& mode) { std::string uri = metaOptions.backend_uri; -// if (uri.empty()) { -// //Default to sqlite if uri is empty -//// return std::make_shared(new meta::DBMetaImpl(metaOptions)); -// return std::shared_ptr(new meta::DBMetaImpl(metaOptions)); -// } std::string dialectRegex = "(.*)"; std::string usernameRegex = "(.*)"; diff --git a/cpp/src/db/Factories.h b/cpp/src/db/Factories.h index 48bea9b291..889922b17a 100644 --- a/cpp/src/db/Factories.h +++ b/cpp/src/db/Factories.h @@ -28,7 +28,7 @@ struct OptionsFactory { struct DBMetaImplFactory { static std::shared_ptr Build(); - static std::shared_ptr Build(const DBMetaOptions& metaOptions, const std::string& mode); + static std::shared_ptr Build(const DBMetaOptions& metaOptions, const int& mode); }; struct DBFactory { diff --git a/cpp/src/db/MySQLConnectionPool.h b/cpp/src/db/MySQLConnectionPool.h index c1ea2e83bc..0978d77a7c 100644 --- a/cpp/src/db/MySQLConnectionPool.h +++ b/cpp/src/db/MySQLConnectionPool.h @@ -20,12 +20,12 @@ public: password_(passWord), server_(serverIp), port_(port), - maxPoolSize_(maxPoolSize) + max_pool_size_(maxPoolSize) { conns_in_use_ = 0; - maxIdleTime_ = 300; //300ms + max_idle_time_ = 10; //10 seconds } // The destructor. We _must_ call ConnectionPool::clear() here, @@ -40,12 +40,10 @@ public: // connections actually in use, not those created. Also note that // we keep our own count; ConnectionPool::size() isn't the same! mysqlpp::Connection* grab() override { - while (conns_in_use_ > maxPoolSize_) { -// cout.put('R'); cout.flush(); // indicate waiting for release + while (conns_in_use_ > max_pool_size_) { sleep(1); } -// ENGINE_LOG_DEBUG << "conns_in_use_ in grab: " << conns_in_use_ << std::endl; ++conns_in_use_; return mysqlpp::ConnectionPool::grab(); } @@ -63,7 +61,7 @@ public: } void set_max_idle_time(int max_idle) { - maxIdleTime_ = max_idle; + max_idle_time_ = max_idle; } protected: @@ -72,7 +70,6 @@ protected: mysqlpp::Connection* create() override { // Create connection using the parameters we were passed upon // creation. -// cout.put('C'); cout.flush(); // indicate connection creation mysqlpp::Connection* conn = new mysqlpp::Connection(); conn->set_option(new mysqlpp::ReconnectOption(true)); conn->connect(db_.empty() ? 0 : db_.c_str(), @@ -86,12 +83,11 @@ protected: void destroy(mysqlpp::Connection* cp) override { // Our superclass can't know how we created the Connection, so // it delegates destruction to us, to be safe. -// cout.put('D'); cout.flush(); // indicate connection destruction delete cp; } unsigned int max_idle_time() override { - return maxIdleTime_; + return max_idle_time_; } private: @@ -102,7 +98,7 @@ private: std::string db_, user_, password_, server_; int port_; - int maxPoolSize_; + int max_pool_size_; - unsigned int maxIdleTime_; + unsigned int max_idle_time_; }; \ No newline at end of file diff --git a/cpp/src/db/MySQLMetaImpl.cpp b/cpp/src/db/MySQLMetaImpl.cpp index 7ee9231b41..25d2c77777 100644 --- a/cpp/src/db/MySQLMetaImpl.cpp +++ b/cpp/src/db/MySQLMetaImpl.cpp @@ -103,7 +103,7 @@ namespace meta { return Status::OK(); } - MySQLMetaImpl::MySQLMetaImpl(const DBMetaOptions &options_, const std::string& mode) + MySQLMetaImpl::MySQLMetaImpl(const DBMetaOptions &options_, const int& mode) : options_(options_), mode_(mode) { Initialize(); @@ -157,7 +157,7 @@ namespace meta { // connectionPtr->set_option(new mysqlpp::ReconnectOption(true)); int threadHint = std::thread::hardware_concurrency(); int maxPoolSize = threadHint == 0 ? 8 : threadHint; - mySQLConnectionPool_ = std::make_shared(dbName, username, password, serverAddress, port, maxPoolSize); + mysql_connection_pool_ = std::make_shared(dbName, username, password, serverAddress, port, maxPoolSize); // std::cout << "MySQL++ thread aware:" << std::to_string(connectionPtr->thread_aware()) << std::endl; try { @@ -165,7 +165,9 @@ namespace meta { CleanUp(); { - ScopedConnection connectionPtr(*mySQLConnectionPool_, safe_grab); + ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab); + +// ENGINE_LOG_DEBUG << "MySQLMetaImpl::Initialize: connections in use = " << mysql_connection_pool_->getConnectionsInUse(); // if (!connectionPtr->connect(dbName, serverAddress, username, password, port)) { // return Status::Error("DB connection failed: ", connectionPtr->error()); // } @@ -190,6 +192,11 @@ namespace meta { "files_cnt BIGINT DEFAULT 0 NOT NULL, " << "engine_type INT DEFAULT 1 NOT NULL, " << "store_raw_data BOOL DEFAULT false NOT NULL);"; + + if (options_.sql_echo) { + ENGINE_LOG_DEBUG << "MySQLMetaImpl::Initialize: " << InitializeQuery.str(); + } + if (!InitializeQuery.exec()) { return Status::DBTransactionError("Initialization Error", InitializeQuery.error()); } @@ -204,6 +211,11 @@ namespace meta { "updated_time BIGINT NOT NULL, " << "created_on BIGINT NOT NULL, " << "date INT DEFAULT -1 NOT NULL);"; + + if (options_.sql_echo) { + ENGINE_LOG_DEBUG << "MySQLMetaImpl::Initialize: " << InitializeQuery.str(); + } + if (!InitializeQuery.exec()) { return Status::DBTransactionError("Initialization Error", InitializeQuery.error()); } @@ -280,7 +292,11 @@ namespace meta { dateListStr = dateListStr.substr(0, dateListStr.size() - 2); //remove the last ", " { - ScopedConnection connectionPtr(*mySQLConnectionPool_, safe_grab); + ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab); + +// if (mysql_connection_pool_->getConnectionsInUse() <= 0) { +// ENGINE_LOG_WARNING << "MySQLMetaImpl::DropPartitionsByDates connection in use = " << mysql_connection_pool_->getConnectionsInUse(); +// } Query dropPartitionsByDatesQuery = connectionPtr->query(); @@ -289,6 +305,10 @@ namespace meta { "WHERE table_id = " << quote << table_id << " AND " << "date in (" << dateListStr << ");"; + if (options_.sql_echo) { + ENGINE_LOG_DEBUG << "MySQLMetaImpl::DropPartitionsByDates: " << dropPartitionsByDatesQuery.str(); + } + if (!dropPartitionsByDatesQuery.exec()) { ENGINE_LOG_ERROR << "QUERY ERROR WHEN DROPPING PARTITIONS BY DATES"; return Status::DBTransactionError("QUERY ERROR WHEN DROPPING PARTITIONS BY DATES", @@ -318,7 +338,11 @@ namespace meta { MetricCollector metric; { - ScopedConnection connectionPtr(*mySQLConnectionPool_, safe_grab); + ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab); + +// if (mysql_connection_pool_->getConnectionsInUse() <= 0) { +// ENGINE_LOG_WARNING << "MySQLMetaImpl::CreateTable connection in use = " << mysql_connection_pool_->getConnectionsInUse(); +// } Query createTableQuery = connectionPtr->query(); ENGINE_LOG_DEBUG << "Create Table in"; @@ -328,6 +352,11 @@ namespace meta { createTableQuery << "SELECT state FROM Tables " << "WHERE table_id = " << quote << table_schema.table_id_ << ";"; // ENGINE_LOG_DEBUG << "Create Table : " << createTableQuery.str(); + + if (options_.sql_echo) { + ENGINE_LOG_DEBUG << "MySQLMetaImpl::CreateTable: " << createTableQuery.str(); + } + StoreQueryResult res = createTableQuery.store(); assert(res && res.num_rows() <= 1); if (res.num_rows() == 1) { @@ -360,6 +389,11 @@ namespace meta { "(" << id << ", " << quote << table_id << ", " << state << ", " << dimension << ", " << created_on << ", " << files_cnt << ", " << engine_type << ", " << store_raw_data << ");"; // ENGINE_LOG_DEBUG << "Create Table : " << createTableQuery.str(); + + if (options_.sql_echo) { + ENGINE_LOG_DEBUG << "MySQLMetaImpl::CreateTable: " << createTableQuery.str(); + } + if (SimpleResult res = createTableQuery.execute()) { table_schema.id_ = res.insert_id(); //Might need to use SELECT LAST_INSERT_ID()? // std::cout << table_schema.id_ << std::endl; @@ -410,7 +444,11 @@ namespace meta { MetricCollector metric; { - ScopedConnection connectionPtr(*mySQLConnectionPool_, safe_grab); + ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab); + +// if (mysql_connection_pool_->getConnectionsInUse() <= 0) { +// ENGINE_LOG_WARNING << "MySQLMetaImpl::DeleteTable connection in use = " << mysql_connection_pool_->getConnectionsInUse(); +// } //soft delete table Query deleteTableQuery = connectionPtr->query(); @@ -419,6 +457,10 @@ namespace meta { "SET state = " << std::to_string(TableSchema::TO_DELETE) << " " << "WHERE table_id = " << quote << table_id << ";"; + if (options_.sql_echo) { + ENGINE_LOG_DEBUG << "MySQLMetaImpl::DeleteTable: " << deleteTableQuery.str(); + } + if (!deleteTableQuery.exec()) { ENGINE_LOG_ERROR << "QUERY ERROR WHEN DELETING TABLE"; return Status::DBTransactionError("QUERY ERROR WHEN DELETING TABLE", deleteTableQuery.error()); @@ -427,9 +469,7 @@ namespace meta { } //Scoped Connection -// ConfigNode& serverConfig = ServerConfig::GetInstance().GetConfig(CONFIG_SERVER); -// opt.mode = serverConfig.GetValue(CONFIG_CLUSTER_MODE, "single"); - if (mode_ != "single") { + if (mode_ != Options::MODE::SINGLE) { DeleteTableFiles(table_id); } @@ -451,7 +491,11 @@ namespace meta { MetricCollector metric; { - ScopedConnection connectionPtr(*mySQLConnectionPool_, safe_grab); + ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab); + +// if (mysql_connection_pool_->getConnectionsInUse() <= 0) { +// ENGINE_LOG_WARNING << "MySQLMetaImpl::DeleteTableFiles connection in use = " << mysql_connection_pool_->getConnectionsInUse(); +// } //soft delete table files Query deleteTableFilesQuery = connectionPtr->query(); @@ -462,6 +506,10 @@ namespace meta { "WHERE table_id = " << quote << table_id << " AND " << "file_type <> " << std::to_string(TableFileSchema::TO_DELETE) << ";"; + if (options_.sql_echo) { + ENGINE_LOG_DEBUG << "MySQLMetaImpl::DeleteTableFiles: " << deleteTableFilesQuery.str(); + } + if (!deleteTableFilesQuery.exec()) { ENGINE_LOG_ERROR << "QUERY ERROR WHEN DELETING TABLE FILES"; return Status::DBTransactionError("QUERY ERROR WHEN DELETING TABLE", deleteTableFilesQuery.error()); @@ -491,13 +539,22 @@ namespace meta { StoreQueryResult res; { - ScopedConnection connectionPtr(*mySQLConnectionPool_, safe_grab); + ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab); + +// if (mysql_connection_pool_->getConnectionsInUse() <= 0) { +// ENGINE_LOG_WARNING << "MySQLMetaImpl::DescribeTable connection in use = " << mysql_connection_pool_->getConnectionsInUse(); +// } Query describeTableQuery = connectionPtr->query(); describeTableQuery << "SELECT id, dimension, files_cnt, engine_type, store_raw_data " << "FROM Tables " << "WHERE table_id = " << quote << table_schema.table_id_ << " " << "AND state <> " << std::to_string(TableSchema::TO_DELETE) << ";"; + + if (options_.sql_echo) { + ENGINE_LOG_DEBUG << "MySQLMetaImpl::DescribeTable: " << describeTableQuery.str(); + } + res = describeTableQuery.store(); } //Scoped Connection @@ -513,7 +570,7 @@ namespace meta { table_schema.engine_type_ = resRow["engine_type"]; - table_schema.store_raw_data_ = (resRow["store_raw_data"].compare("true") == 0); + table_schema.store_raw_data_ = (resRow["store_raw_data"] == 1); } else { return Status::NotFound("Table " + table_schema.table_id_ + " not found"); @@ -546,7 +603,11 @@ namespace meta { StoreQueryResult res; { - ScopedConnection connectionPtr(*mySQLConnectionPool_, safe_grab); + ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab); + +// if (mysql_connection_pool_->getConnectionsInUse() <= 0) { +// ENGINE_LOG_WARNING << "MySQLMetaImpl::HasTable connection in use = " << mysql_connection_pool_->getConnectionsInUse(); +// } Query hasTableQuery = connectionPtr->query(); //since table_id is a unique column we just need to check whether it exists or not @@ -555,6 +616,11 @@ namespace meta { "WHERE table_id = " << quote << table_id << " " << "AND state <> " << std::to_string(TableSchema::TO_DELETE) << ") " << "AS " << quote << "check" << ";"; + + if (options_.sql_echo) { + ENGINE_LOG_DEBUG << "MySQLMetaImpl::HasTable: " << hasTableQuery.str(); + } + res = hasTableQuery.store(); } //Scoped Connection @@ -586,12 +652,21 @@ namespace meta { StoreQueryResult res; { - ScopedConnection connectionPtr(*mySQLConnectionPool_, safe_grab); + ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab); + +// if (mysql_connection_pool_->getConnectionsInUse() <= 0) { +// ENGINE_LOG_WARNING << "MySQLMetaImpl::AllTables connection in use = " << mysql_connection_pool_->getConnectionsInUse(); +// } Query allTablesQuery = connectionPtr->query(); allTablesQuery << "SELECT id, table_id, dimension, files_cnt, engine_type, store_raw_data " << "FROM Tables " << "WHERE state <> " << std::to_string(TableSchema::TO_DELETE) << ";"; + + if (options_.sql_echo) { + ENGINE_LOG_DEBUG << "MySQLMetaImpl::AllTables: " << allTablesQuery.str(); + } + res = allTablesQuery.store(); } //Scoped Connection @@ -610,7 +685,7 @@ namespace meta { table_schema.engine_type_ = resRow["engine_type"]; - table_schema.store_raw_data_ = (resRow["store_raw_data"].compare("true") == 0); + table_schema.store_raw_data_ = (resRow["store_raw_data"] == 1); table_schema_array.emplace_back(table_schema); } @@ -665,7 +740,11 @@ namespace meta { std::string date = std::to_string(file_schema.date_); { - ScopedConnection connectionPtr(*mySQLConnectionPool_, safe_grab); + ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab); + +// if (mysql_connection_pool_->getConnectionsInUse() <= 0) { +// ENGINE_LOG_WARNING << "MySQLMetaImpl::CreateTableFile connection in use = " << mysql_connection_pool_->getConnectionsInUse(); +// } Query createTableFileQuery = connectionPtr->query(); @@ -674,6 +753,10 @@ namespace meta { quote << file_id << ", " << file_type << ", " << size << ", " << updated_time << ", " << created_on << ", " << date << ");"; + if (options_.sql_echo) { + ENGINE_LOG_DEBUG << "MySQLMetaImpl::CreateTableFile: " << createTableFileQuery.str(); + } + if (SimpleResult res = createTableFileQuery.execute()) { file_schema.id_ = res.insert_id(); //Might need to use SELECT LAST_INSERT_ID()? @@ -725,12 +808,21 @@ namespace meta { StoreQueryResult res; { - ScopedConnection connectionPtr(*mySQLConnectionPool_, safe_grab); + ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab); + +// if (mysql_connection_pool_->getConnectionsInUse() <= 0) { +// ENGINE_LOG_WARNING << "MySQLMetaImpl::FilesToIndex connection in use = " << mysql_connection_pool_->getConnectionsInUse(); +// } Query filesToIndexQuery = connectionPtr->query(); filesToIndexQuery << "SELECT id, table_id, engine_type, file_id, file_type, size, date " << "FROM TableFiles " << "WHERE file_type = " << std::to_string(TableFileSchema::TO_INDEX) << ";"; + + if (options_.sql_echo) { + ENGINE_LOG_DEBUG << "MySQLMetaImpl::FilesToIndex: " << filesToIndexQuery.str(); + } + res = filesToIndexQuery.store(); } //Scoped Connection @@ -801,7 +893,11 @@ namespace meta { StoreQueryResult res; { - ScopedConnection connectionPtr(*mySQLConnectionPool_, safe_grab); + ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab); + +// if (mysql_connection_pool_->getConnectionsInUse() <= 0) { +// ENGINE_LOG_WARNING << "MySQLMetaImpl::FilesToSearch connection in use = " << mysql_connection_pool_->getConnectionsInUse(); +// } if (partition.empty()) { @@ -812,6 +908,11 @@ namespace meta { "(file_type = " << std::to_string(TableFileSchema::RAW) << " OR " << "file_type = " << std::to_string(TableFileSchema::TO_INDEX) << " OR " << "file_type = " << std::to_string(TableFileSchema::INDEX) << ");"; + + if (options_.sql_echo) { + ENGINE_LOG_DEBUG << "MySQLMetaImpl::FilesToSearch: " << filesToSearchQuery.str(); + } + res = filesToSearchQuery.store(); } else { @@ -832,6 +933,11 @@ namespace meta { "(file_type = " << std::to_string(TableFileSchema::RAW) << " OR " << "file_type = " << std::to_string(TableFileSchema::TO_INDEX) << " OR " << "file_type = " << std::to_string(TableFileSchema::INDEX) << ");"; + + if (options_.sql_echo) { + ENGINE_LOG_DEBUG << "MySQLMetaImpl::FilesToSearch: " << filesToSearchQuery.str(); + } + res = filesToSearchQuery.store(); } @@ -902,7 +1008,11 @@ namespace meta { StoreQueryResult res; { - ScopedConnection connectionPtr(*mySQLConnectionPool_, safe_grab); + ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab); + +// if (mysql_connection_pool_->getConnectionsInUse() <= 0) { +// ENGINE_LOG_WARNING << "MySQLMetaImpl::FilesToMerge connection in use = " << mysql_connection_pool_->getConnectionsInUse(); +// } Query filesToMergeQuery = connectionPtr->query(); filesToMergeQuery << "SELECT id, table_id, file_id, file_type, size, date " << @@ -910,6 +1020,11 @@ namespace meta { "WHERE table_id = " << quote << table_id << " AND " << "file_type = " << std::to_string(TableFileSchema::RAW) << " " << "ORDER BY size DESC" << ";"; + + if (options_.sql_echo) { + ENGINE_LOG_DEBUG << "MySQLMetaImpl::FilesToMerge: " << filesToMergeQuery.str(); + } + res = filesToMergeQuery.store(); } //Scoped Connection @@ -987,13 +1102,22 @@ namespace meta { StoreQueryResult res; { - ScopedConnection connectionPtr(*mySQLConnectionPool_, safe_grab); + ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab); + +// if (mysql_connection_pool_->getConnectionsInUse() <= 0) { +// ENGINE_LOG_WARNING << "MySQLMetaImpl::GetTableFiles connection in use = " << mysql_connection_pool_->getConnectionsInUse(); +// } Query getTableFileQuery = connectionPtr->query(); getTableFileQuery << "SELECT engine_type, file_id, file_type, size, date " << "FROM TableFiles " << "WHERE table_id = " << quote << table_id << " AND " << "(" << idStr << ");"; + + if (options_.sql_echo) { + ENGINE_LOG_DEBUG << "MySQLMetaImpl::GetTableFiles: " << getTableFileQuery.str(); + } + res = getTableFileQuery.store(); } //Scoped Connection @@ -1062,13 +1186,22 @@ namespace meta { try { - ScopedConnection connectionPtr(*mySQLConnectionPool_, safe_grab); + ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab); + +// if (mysql_connection_pool_->getConnectionsInUse() <= 0) { +// ENGINE_LOG_WARNING << "MySQLMetaImpl::Archive connection in use = " << mysql_connection_pool_->getConnectionsInUse(); +// } Query archiveQuery = connectionPtr->query(); archiveQuery << "UPDATE TableFiles " << "SET file_type = " << std::to_string(TableFileSchema::TO_DELETE) << " " << "WHERE created_on < " << std::to_string(now - usecs) << " AND " << "file_type <> " << std::to_string(TableFileSchema::TO_DELETE) << ";"; + + if (options_.sql_echo) { + ENGINE_LOG_DEBUG << "MySQLMetaImpl::Archive: " << archiveQuery.str(); + } + if (!archiveQuery.exec()) { return Status::DBTransactionError("QUERY ERROR DURING ARCHIVE", archiveQuery.error()); } @@ -1105,12 +1238,21 @@ namespace meta { StoreQueryResult res; { - ScopedConnection connectionPtr(*mySQLConnectionPool_, safe_grab); + ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab); + +// if (mysql_connection_pool_->getConnectionsInUse() <= 0) { +// ENGINE_LOG_WARNING << "MySQLMetaImpl::Size connection in use = " << mysql_connection_pool_->getConnectionsInUse(); +// } Query getSizeQuery = connectionPtr->query(); getSizeQuery << "SELECT SUM(size) AS sum " << "FROM TableFiles " << "WHERE file_type <> " << std::to_string(TableFileSchema::TO_DELETE) << ";"; + + if (options_.sql_echo) { + ENGINE_LOG_DEBUG << "MySQLMetaImpl::Size: " << getSizeQuery.str(); + } + res = getSizeQuery.store(); } //Scoped Connection @@ -1158,7 +1300,11 @@ namespace meta { bool status; { - ScopedConnection connectionPtr(*mySQLConnectionPool_, safe_grab); + ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab); + +// if (mysql_connection_pool_->getConnectionsInUse() <= 0) { +// ENGINE_LOG_WARNING << "MySQLMetaImpl::DiscardFiles connection in use = " << mysql_connection_pool_->getConnectionsInUse(); +// } Query discardFilesQuery = connectionPtr->query(); discardFilesQuery << "SELECT id, size " << @@ -1166,7 +1312,12 @@ namespace meta { "WHERE file_type <> " << std::to_string(TableFileSchema::TO_DELETE) << " " << "ORDER BY id ASC " << "LIMIT 10;"; -// std::cout << discardFilesQuery.str() << std::endl; + + if (options_.sql_echo) { + ENGINE_LOG_DEBUG << "MySQLMetaImpl::DiscardFiles: " << discardFilesQuery.str(); + } + + // std::cout << discardFilesQuery.str() << std::endl; StoreQueryResult res = discardFilesQuery.store(); assert(res); @@ -1196,6 +1347,10 @@ namespace meta { "updated_time = " << std::to_string(utils::GetMicroSecTimeStamp()) << " " << "WHERE " << idsToDiscardStr << ";"; + if (options_.sql_echo) { + ENGINE_LOG_DEBUG << "MySQLMetaImpl::DiscardFiles: " << discardFilesQuery.str(); + } + status = discardFilesQuery.exec(); if (!status) { ENGINE_LOG_ERROR << "QUERY ERROR WHEN DISCARDING FILES"; @@ -1227,7 +1382,11 @@ namespace meta { MetricCollector metric; { - ScopedConnection connectionPtr(*mySQLConnectionPool_, safe_grab); + ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab); + +// if (mysql_connection_pool_->getConnectionsInUse() <= 0) { +// ENGINE_LOG_WARNING << "MySQLMetaImpl::UpdateTableFile connection in use = " << mysql_connection_pool_->getConnectionsInUse(); +// } Query updateTableFileQuery = connectionPtr->query(); @@ -1235,6 +1394,11 @@ namespace meta { //clean thread will delete the file later updateTableFileQuery << "SELECT state FROM Tables " << "WHERE table_id = " << quote << file_schema.table_id_ << ";"; + + if (options_.sql_echo) { + ENGINE_LOG_DEBUG << "MySQLMetaImpl::UpdateTableFile: " << updateTableFileQuery.str(); + } + StoreQueryResult res = updateTableFileQuery.store(); assert(res && res.num_rows() <= 1); @@ -1268,7 +1432,11 @@ namespace meta { "date = " << date << " " << "WHERE id = " << id << ";"; -// std::cout << updateTableFileQuery.str() << std::endl; + if (options_.sql_echo) { + ENGINE_LOG_DEBUG << "MySQLMetaImpl::UpdateTableFile: " << updateTableFileQuery.str(); + } + + // std::cout << updateTableFileQuery.str() << std::endl; if (!updateTableFileQuery.exec()) { ENGINE_LOG_DEBUG << "table_id= " << file_schema.table_id_ << " file_id=" << file_schema.file_id_; @@ -1300,7 +1468,11 @@ namespace meta { MetricCollector metric; { - ScopedConnection connectionPtr(*mySQLConnectionPool_, safe_grab); + ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab); + +// if (mysql_connection_pool_->getConnectionsInUse() <= 0) { +// ENGINE_LOG_WARNING << "MySQLMetaImpl::UpdateTableFiles connection in use = " << mysql_connection_pool_->getConnectionsInUse(); +// } Query updateTableFilesQuery = connectionPtr->query(); @@ -1316,6 +1488,11 @@ namespace meta { "WHERE table_id = " << quote << file_schema.table_id_ << " " << "AND state <> " << std::to_string(TableSchema::TO_DELETE) << ") " << "AS " << quote << "check" << ";"; + + if (options_.sql_echo) { + ENGINE_LOG_DEBUG << "MySQLMetaImpl::UpdateTableFiles: " << updateTableFilesQuery.str(); + } + StoreQueryResult res = updateTableFilesQuery.store(); assert(res && res.num_rows() == 1); @@ -1351,6 +1528,10 @@ namespace meta { "date = " << date << " " << "WHERE id = " << id << ";"; + if (options_.sql_echo) { + ENGINE_LOG_DEBUG << "MySQLMetaImpl::UpdateTableFiles: " << updateTableFilesQuery.str(); + } + if (!updateTableFilesQuery.exec()) { ENGINE_LOG_ERROR << "QUERY ERROR WHEN UPDATING TABLE FILES"; return Status::DBTransactionError("QUERY ERROR WHEN UPDATING TABLE FILES", @@ -1382,13 +1563,27 @@ namespace meta { MetricCollector metric; { - ScopedConnection connectionPtr(*mySQLConnectionPool_, safe_grab); + +// ENGINE_LOG_WARNING << "MySQLMetaImpl::CleanUpFilesWithTTL: clean table files: connection in use before creating ScopedConnection = " +// << mysql_connection_pool_->getConnectionsInUse(); + + ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab); + +// if (mysql_connection_pool_->getConnectionsInUse() <= 0) { +// ENGINE_LOG_WARNING << "MySQLMetaImpl::CleanUpFilesWithTTL: clean table files: connection in use after creating ScopedConnection = " +// << mysql_connection_pool_->getConnectionsInUse(); +// } Query cleanUpFilesWithTTLQuery = connectionPtr->query(); cleanUpFilesWithTTLQuery << "SELECT id, table_id, file_id, date " << "FROM TableFiles " << "WHERE file_type = " << std::to_string(TableFileSchema::TO_DELETE) << " AND " << "updated_time < " << std::to_string(now - seconds * US_PS) << ";"; + + if (options_.sql_echo) { + ENGINE_LOG_DEBUG << "MySQLMetaImpl::CleanUpFilesWithTTL: " << cleanUpFilesWithTTLQuery.str(); + } + StoreQueryResult res = cleanUpFilesWithTTLQuery.store(); assert(res); @@ -1430,6 +1625,11 @@ namespace meta { idsToDeleteStr = idsToDeleteStr.substr(0, idsToDeleteStr.size() - 4); //remove the last " OR " cleanUpFilesWithTTLQuery << "DELETE FROM TableFiles WHERE " << idsToDeleteStr << ";"; + + if (options_.sql_echo) { + ENGINE_LOG_DEBUG << "MySQLMetaImpl::CleanUpFilesWithTTL: " << cleanUpFilesWithTTLQuery.str(); + } + if (!cleanUpFilesWithTTLQuery.exec()) { ENGINE_LOG_ERROR << "QUERY ERROR WHEN CLEANING UP FILES WITH TTL"; return Status::DBTransactionError("CleanUpFilesWithTTL Error", @@ -1452,12 +1652,25 @@ namespace meta { MetricCollector metric; { - ScopedConnection connectionPtr(*mySQLConnectionPool_, safe_grab); +// ENGINE_LOG_WARNING << "MySQLMetaImpl::CleanUpFilesWithTTL: clean tables: connection in use before creating ScopedConnection = " +// << mysql_connection_pool_->getConnectionsInUse(); + + ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab); + +// if (mysql_connection_pool_->getConnectionsInUse() <= 0) { +// ENGINE_LOG_WARNING << "MySQLMetaImpl::CleanUpFilesWithTTL: clean tables: connection in use after creating ScopedConnection = " +// << mysql_connection_pool_->getConnectionsInUse(); +// } Query cleanUpFilesWithTTLQuery = connectionPtr->query(); cleanUpFilesWithTTLQuery << "SELECT id, table_id " << "FROM Tables " << "WHERE state = " << std::to_string(TableSchema::TO_DELETE) << ";"; + + if (options_.sql_echo) { + ENGINE_LOG_DEBUG << "MySQLMetaImpl::CleanUpFilesWithTTL: " << cleanUpFilesWithTTLQuery.str(); + } + StoreQueryResult res = cleanUpFilesWithTTLQuery.store(); assert(res); // std::cout << res.num_rows() << std::endl; @@ -1481,6 +1694,11 @@ namespace meta { idsToDeleteStr = idsToDeleteStr.substr(0, idsToDeleteStr.size() - 4); //remove the last " OR " cleanUpFilesWithTTLQuery << "DELETE FROM Tables WHERE " << idsToDeleteStr << ";"; + + if (options_.sql_echo) { + ENGINE_LOG_DEBUG << "MySQLMetaImpl::CleanUpFilesWithTTL: " << cleanUpFilesWithTTLQuery.str(); + } + if (!cleanUpFilesWithTTLQuery.exec()) { ENGINE_LOG_ERROR << "QUERY ERROR WHEN CLEANING UP FILES WITH TTL"; return Status::DBTransactionError("QUERY ERROR WHEN CLEANING UP FILES WITH TTL", @@ -1507,12 +1725,20 @@ namespace meta { // std::lock_guard lock(mysql_mutex); try { - ScopedConnection connectionPtr(*mySQLConnectionPool_, safe_grab); + ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab); + +// if (mysql_connection_pool_->getConnectionsInUse() <= 0) { +// ENGINE_LOG_WARNING << "MySQLMetaImpl::CleanUp: connection in use = " << mysql_connection_pool_->getConnectionsInUse(); +// } ENGINE_LOG_DEBUG << "Remove table file type as NEW"; Query cleanUpQuery = connectionPtr->query(); cleanUpQuery << "DELETE FROM TableFiles WHERE file_type = " << std::to_string(TableFileSchema::NEW) << ";"; + if (options_.sql_echo) { + ENGINE_LOG_DEBUG << "MySQLMetaImpl::CleanUp: " << cleanUpQuery.str(); + } + if (!cleanUpQuery.exec()) { ENGINE_LOG_ERROR << "QUERY ERROR WHEN CLEANING UP FILES"; return Status::DBTransactionError("Clean up Error", cleanUpQuery.error()); @@ -1549,7 +1775,11 @@ namespace meta { StoreQueryResult res; { - ScopedConnection connectionPtr(*mySQLConnectionPool_, safe_grab); + ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab); + +// if (mysql_connection_pool_->getConnectionsInUse() <= 0) { +// ENGINE_LOG_WARNING << "MySQLMetaImpl::Count: connection in use = " << mysql_connection_pool_->getConnectionsInUse(); +// } Query countQuery = connectionPtr->query(); countQuery << "SELECT size " << @@ -1558,6 +1788,11 @@ namespace meta { "(file_type = " << std::to_string(TableFileSchema::RAW) << " OR " << "file_type = " << std::to_string(TableFileSchema::TO_INDEX) << " OR " << "file_type = " << std::to_string(TableFileSchema::INDEX) << ");"; + + if (options_.sql_echo) { + ENGINE_LOG_DEBUG << "MySQLMetaImpl::Count: " << countQuery.str(); + } + res = countQuery.store(); } //Scoped Connection @@ -1592,10 +1827,19 @@ namespace meta { } try { - ScopedConnection connectionPtr(*mySQLConnectionPool_, safe_grab); + ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab); + +// if (mysql_connection_pool_->getConnectionsInUse() <= 0) { +// ENGINE_LOG_WARNING << "MySQLMetaImpl::DropAll: connection in use = " << mysql_connection_pool_->getConnectionsInUse(); +// } Query dropTableQuery = connectionPtr->query(); dropTableQuery << "DROP TABLE IF EXISTS Tables, TableFiles;"; + + if (options_.sql_echo) { + ENGINE_LOG_DEBUG << "MySQLMetaImpl::DropAll: " << dropTableQuery.str(); + } + if (dropTableQuery.exec()) { return Status::OK(); } diff --git a/cpp/src/db/MySQLMetaImpl.h b/cpp/src/db/MySQLMetaImpl.h index 4805076d45..9ff8254b60 100644 --- a/cpp/src/db/MySQLMetaImpl.h +++ b/cpp/src/db/MySQLMetaImpl.h @@ -22,7 +22,7 @@ namespace meta { class MySQLMetaImpl : public Meta { public: - MySQLMetaImpl(const DBMetaOptions& options_, const std::string& mode); + MySQLMetaImpl(const DBMetaOptions& options_, const int& mode); virtual Status CreateTable(TableSchema& table_schema) override; virtual Status DescribeTable(TableSchema& group_info_) override; @@ -77,9 +77,9 @@ namespace meta { Status Initialize(); const DBMetaOptions options_; - const std::string mode_; + const int mode_; - std::shared_ptr mySQLConnectionPool_; + std::shared_ptr mysql_connection_pool_; bool safe_grab = false; // std::mutex connectionMutex_; diff --git a/cpp/src/db/Options.h b/cpp/src/db/Options.h index dbe80f8d5f..609e3ca245 100644 --- a/cpp/src/db/Options.h +++ b/cpp/src/db/Options.h @@ -45,15 +45,23 @@ struct DBMetaOptions { std::string path; std::string backend_uri; ArchiveConf archive_conf = ArchiveConf("delete"); + bool sql_echo = false; }; // DBMetaOptions struct Options { + + typedef enum { + SINGLE, + CLUSTER, + READ_ONLY + } MODE; + Options(); uint16_t memory_sync_interval = 1; //unit: second uint16_t merge_trigger_number = 2; size_t index_trigger_size = ONE_GB; //unit: byte DBMetaOptions meta; - std::string mode; + int mode = MODE::SINGLE; }; // Options diff --git a/cpp/src/server/DBWrapper.cpp b/cpp/src/server/DBWrapper.cpp index ceaa6e8130..a3db0bf110 100644 --- a/cpp/src/server/DBWrapper.cpp +++ b/cpp/src/server/DBWrapper.cpp @@ -23,9 +23,30 @@ DBWrapper::DBWrapper() { if(index_size > 0) {//ensure larger than zero, unit is MB opt.index_trigger_size = (size_t)index_size * engine::ONE_MB; } + std::string sql_echo = config.GetValue(CONFIG_DB_SQL_ECHO, "off"); + if (sql_echo == "on") { + opt.meta.sql_echo = true; + } + else if (sql_echo == "off") { + opt.meta.sql_echo = false; + } + else { + std::cout << "ERROR: sql_echo specified in db_config is not one of ['on', 'off']" << std::endl; + kill(0, SIGUSR1); + } + ConfigNode& serverConfig = ServerConfig::GetInstance().GetConfig(CONFIG_SERVER); - opt.mode = serverConfig.GetValue(CONFIG_CLUSTER_MODE, "single"); - if (opt.mode != "single" && opt.mode != "cluster" && opt.mode != "read_only") { + std::string mode = serverConfig.GetValue(CONFIG_CLUSTER_MODE, "single"); + if (mode == "single") { + opt.mode = zilliz::milvus::engine::Options::MODE::SINGLE; + } + else if (mode == "cluster") { + opt.mode = zilliz::milvus::engine::Options::MODE::CLUSTER; + } + else if (mode == "read_only") { + opt.mode = zilliz::milvus::engine::Options::MODE::READ_ONLY; + } + else { std::cout << "ERROR: mode specified in server_config is not one of ['single', 'cluster', 'read_only']" << std::endl; kill(0, SIGUSR1); } diff --git a/cpp/src/server/ServerConfig.h b/cpp/src/server/ServerConfig.h index f337275a46..768430f023 100644 --- a/cpp/src/server/ServerConfig.h +++ b/cpp/src/server/ServerConfig.h @@ -27,6 +27,7 @@ static const std::string CONFIG_DB_PATH = "db_path"; static const std::string CONFIG_DB_INDEX_TRIGGER_SIZE = "index_building_threshold"; static const std::string CONFIG_DB_ARCHIVE_DISK = "archive_disk_threshold"; static const std::string CONFIG_DB_ARCHIVE_DAYS = "archive_days_threshold"; +static const std::string CONFIG_DB_SQL_ECHO = "sql_echo"; static const std::string CONFIG_LOG = "log_config"; From 7bc5873743e24ef3dcd291c4aa39138994a81f43 Mon Sep 17 00:00:00 2001 From: zhiru Date: Sun, 30 Jun 2019 13:32:54 +0800 Subject: [PATCH 035/189] update Former-commit-id: 0f8c129028a1625d683abf11d6464b0a8fa4fb8c --- cpp/src/db/MySQLMetaImpl.cpp | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/cpp/src/db/MySQLMetaImpl.cpp b/cpp/src/db/MySQLMetaImpl.cpp index 25d2c77777..328e27f4d7 100644 --- a/cpp/src/db/MySQLMetaImpl.cpp +++ b/cpp/src/db/MySQLMetaImpl.cpp @@ -570,7 +570,8 @@ namespace meta { table_schema.engine_type_ = resRow["engine_type"]; - table_schema.store_raw_data_ = (resRow["store_raw_data"] == 1); + int store_raw_data = resRow["store_raw_data"]; + table_schema.store_raw_data_ = (store_raw_data == 1); } else { return Status::NotFound("Table " + table_schema.table_id_ + " not found"); @@ -685,7 +686,8 @@ namespace meta { table_schema.engine_type_ = resRow["engine_type"]; - table_schema.store_raw_data_ = (resRow["store_raw_data"] == 1); + int store_raw_data = resRow["store_raw_data"]; + table_schema.store_raw_data_ = (store_raw_data == 1); table_schema_array.emplace_back(table_schema); } From f41a834b92db595ebd95393319642f2c43aa773a Mon Sep 17 00:00:00 2001 From: zhiru Date: Sun, 30 Jun 2019 14:05:45 +0800 Subject: [PATCH 036/189] update Former-commit-id: 8c424908b64391585954bd72c65eb8572df4915b --- cpp/src/db/MySQLMetaImpl.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/src/db/MySQLMetaImpl.cpp b/cpp/src/db/MySQLMetaImpl.cpp index 328e27f4d7..4d5fc00050 100644 --- a/cpp/src/db/MySQLMetaImpl.cpp +++ b/cpp/src/db/MySQLMetaImpl.cpp @@ -1247,7 +1247,7 @@ namespace meta { // } Query getSizeQuery = connectionPtr->query(); - getSizeQuery << "SELECT SUM(size) AS sum " << + getSizeQuery << "SELECT IFNULL(SUM(size),0) AS sum " << "FROM TableFiles " << "WHERE file_type <> " << std::to_string(TableFileSchema::TO_DELETE) << ";"; From 269f727f1385996f2d2db4e145aedcbb7ec2a7fe Mon Sep 17 00:00:00 2001 From: zhiru Date: Sun, 30 Jun 2019 15:20:08 +0800 Subject: [PATCH 037/189] update Former-commit-id: aa1429a2525f95f6f57777f6cddd0d1caf11060b --- cpp/coverage.sh | 19 +++++++----- cpp/unittest/db/MySQLMetaImpl_test.cpp | 40 +++++++++++++++++--------- cpp/unittest/db/db_tests.cpp | 30 +++++++++++-------- cpp/unittest/db/utils.cpp | 2 ++ 4 files changed, 59 insertions(+), 32 deletions(-) diff --git a/cpp/coverage.sh b/cpp/coverage.sh index 3415e752bd..54cf480703 100755 --- a/cpp/coverage.sh +++ b/cpp/coverage.sh @@ -20,15 +20,20 @@ if [ $? -ne 0 ]; then fi for test in `ls ${DIR_UNITTEST}`; do - echo $test - case ${test} in + case ${test} in + db_test) + # set run args for db_test + args="mysql://root:Fantast1c@192.168.1.194:3306/test" + ;; *_test) - # run unittest - ./${DIR_UNITTEST}/${test} - if [ $? -ne 0 ]; then - echo ${DIR_UNITTEST}/${test} "run failed" - fi + args="" + ;; esac + # run unittest + ./${DIR_UNITTEST}/${test} "${args}" + if [ $? -ne 0 ]; then + echo ${DIR_UNITTEST}/${test} "run failed" + fi done # gen test converage diff --git a/cpp/unittest/db/MySQLMetaImpl_test.cpp b/cpp/unittest/db/MySQLMetaImpl_test.cpp index 57fad28468..93e0ce9b28 100644 --- a/cpp/unittest/db/MySQLMetaImpl_test.cpp +++ b/cpp/unittest/db/MySQLMetaImpl_test.cpp @@ -36,7 +36,8 @@ TEST_F(MySQLTest, core) { // //dialect+driver://username:password@host:port/database // options.backend_uri = "mysql://root:1234@:/test"; // options.path = "/tmp/vecwise_test"; - meta::MySQLMetaImpl impl(getDBMetaOptions()); + int mode = Options::MODE::SINGLE; + meta::MySQLMetaImpl impl(getDBMetaOptions(), mode); // auto status = impl.Initialize(); // ASSERT_TRUE(status.ok()); @@ -58,7 +59,7 @@ TEST_F(MySQLTest, core) { status = impl.CreateTable(schema2); // std::cout << status.ToString() << std::endl; // ASSERT_THROW(impl.CreateTable(schema), mysqlpp::BadQuery); - ASSERT_FALSE(status.ok()); + ASSERT_TRUE(status.ok()); status = impl.DeleteTable(schema2.table_id_); // std::cout << status.ToString() << std::endl; @@ -191,8 +192,9 @@ TEST_F(MySQLTest, core) { } TEST_F(MySQLTest, GROUP_TEST) { - - meta::MySQLMetaImpl impl(getDBMetaOptions()); + + int mode = Options::MODE::SINGLE; + meta::MySQLMetaImpl impl(getDBMetaOptions(), mode); auto table_id = "meta_test_group"; @@ -214,7 +216,12 @@ TEST_F(MySQLTest, GROUP_TEST) { group.table_id_ = table_id; status = impl.CreateTable(group); - ASSERT_TRUE(!status.ok()); + ASSERT_TRUE(status.ok()); + + group.table_id_ = ""; + status = impl.CreateTable(group); + ASSERT_TRUE(status.ok()); + status = impl.DropAll(); ASSERT_TRUE(status.ok()); @@ -222,12 +229,14 @@ TEST_F(MySQLTest, GROUP_TEST) { TEST_F(MySQLTest, table_file_TEST) { - meta::MySQLMetaImpl impl(getDBMetaOptions()); + int mode = Options::MODE::SINGLE; + meta::MySQLMetaImpl impl(getDBMetaOptions(), mode); auto table_id = "meta_test_group"; meta::TableSchema group; group.table_id_ = table_id; + group.dimension_ = 256; auto status = impl.CreateTable(group); meta::TableFileSchema table_file; @@ -237,6 +246,11 @@ TEST_F(MySQLTest, table_file_TEST) { ASSERT_TRUE(status.ok()); ASSERT_EQ(table_file.file_type_, meta::TableFileSchema::NEW); + uint64_t cnt = 0; + status = impl.Count(table_id, cnt); + ASSERT_TRUE(status.ok()); + ASSERT_EQ(cnt, 0UL); + auto file_id = table_file.file_id_; auto new_file_type = meta::TableFileSchema::INDEX; @@ -287,8 +301,8 @@ TEST_F(MySQLTest, ARCHIVE_TEST_DAYS) { std::stringstream ss; ss << "days:" << days_num; options.archive_conf = ArchiveConf("delete", ss.str()); - - meta::MySQLMetaImpl impl(options); + int mode = Options::MODE::SINGLE; + meta::MySQLMetaImpl impl(options, mode); auto table_id = "meta_test_group"; @@ -336,11 +350,10 @@ TEST_F(MySQLTest, ARCHIVE_TEST_DAYS) { } TEST_F(MySQLTest, ARCHIVE_TEST_DISK) { - DBMetaOptions options; - options.path = "/tmp/milvus_test"; + DBMetaOptions options = getDBMetaOptions(); options.archive_conf = ArchiveConf("delete", "disk:11"); - - auto impl = meta::DBMetaImpl(options); + int mode = Options::MODE::SINGLE; + auto impl = meta::MySQLMetaImpl(options, mode); auto table_id = "meta_test_group"; meta::TableSchema group; @@ -385,7 +398,8 @@ TEST_F(MySQLTest, ARCHIVE_TEST_DISK) { TEST_F(MySQLTest, TABLE_FILES_TEST) { - auto impl = meta::DBMetaImpl(getDBMetaOptions()); + int mode = Options::MODE::SINGLE; + auto impl = meta::MySQLMetaImpl(getDBMetaOptions(), mode); auto table_id = "meta_test_group"; diff --git a/cpp/unittest/db/db_tests.cpp b/cpp/unittest/db/db_tests.cpp index b1738abcdf..b5eb8e642b 100644 --- a/cpp/unittest/db/db_tests.cpp +++ b/cpp/unittest/db/db_tests.cpp @@ -293,20 +293,14 @@ TEST_F(MySQLDBTest, DB_TEST) { auto options = GetOptions(); auto db_ = engine::DBFactory::Build(options); - static const std::string table_name = "test_group"; - static const int table_dim = 256; - - engine::meta::TableSchema table_info; - table_info.dimension_ = table_dim; - table_info.table_id_ = table_name; - table_info.engine_type_ = (int)engine::EngineType::FAISS_IDMAP; + engine::meta::TableSchema table_info = BuildTableSchema(); engine::Status stat = db_->CreateTable(table_info); engine::meta::TableSchema table_info_get; - table_info_get.table_id_ = table_name; + table_info_get.table_id_ = TABLE_NAME; stat = db_->DescribeTable(table_info_get); ASSERT_STATS(stat); - ASSERT_EQ(table_info_get.dimension_, table_dim); + ASSERT_EQ(table_info_get.dimension_, TABLE_DIM); engine::IDNumbers vector_ids; engine::IDNumbers target_ids; @@ -335,7 +329,7 @@ TEST_F(MySQLDBTest, DB_TEST) { prev_count = count; START_TIMER; - stat = db_->Query(table_name, k, qb, qxb.data(), results); + stat = db_->Query(TABLE_NAME, k, qb, qxb.data(), results); ss << "Search " << j << " With Size " << count/engine::meta::M << " M"; STOP_TIMER(ss.str()); @@ -358,10 +352,10 @@ TEST_F(MySQLDBTest, DB_TEST) { for (auto i=0; iInsertVectors(table_name, qb, qxb.data(), target_ids); + db_->InsertVectors(TABLE_NAME, qb, qxb.data(), target_ids); ASSERT_EQ(target_ids.size(), qb); } else { - db_->InsertVectors(table_name, nb, xb.data(), vector_ids); + db_->InsertVectors(TABLE_NAME, nb, xb.data(), vector_ids); } std::this_thread::sleep_for(std::chrono::microseconds(1)); } @@ -447,6 +441,18 @@ TEST_F(MySQLDBTest, ARHIVE_DISK_CHECK) { engine::meta::TableSchema table_info = BuildTableSchema(); engine::Status stat = db_->CreateTable(table_info); + std::vector table_schema_array; + stat = db_->AllTables(table_schema_array); + ASSERT_STATS(stat); + bool bfound = false; + for(auto& schema : table_schema_array) { + if(schema.table_id_ == TABLE_NAME) { + bfound = true; + break; + } + } + ASSERT_TRUE(bfound); + engine::meta::TableSchema table_info_get; table_info_get.table_id_ = TABLE_NAME; stat = db_->DescribeTable(table_info_get); diff --git a/cpp/unittest/db/utils.cpp b/cpp/unittest/db/utils.cpp index 1f2ec5399c..6b1fc1e407 100644 --- a/cpp/unittest/db/utils.cpp +++ b/cpp/unittest/db/utils.cpp @@ -53,6 +53,7 @@ void DBTest::InitLog() { engine::Options DBTest::GetOptions() { auto options = engine::OptionsFactory::Build(); options.meta.path = "/tmp/milvus_test"; + options.meta.backend_uri = "sqlite://:@:/"; return options; } @@ -71,6 +72,7 @@ engine::Options DBTest2::GetOptions() { auto options = engine::OptionsFactory::Build(); options.meta.path = "/tmp/milvus_test"; options.meta.archive_conf = engine::ArchiveConf("delete", "disk:1"); + options.meta.backend_uri = "sqlite://:@:/"; return options; } From 46ed8749b2b6f745c04a5412aae0882c8aee0d7d Mon Sep 17 00:00:00 2001 From: starlord Date: Sun, 30 Jun 2019 11:54:57 +0800 Subject: [PATCH 038/189] add more unitest Former-commit-id: 7c7d2337f0d0c7abd82533fc2ff2ded0ee07344b --- cpp/src/db/scheduler/TaskDispatchQueue.cpp | 14 -- cpp/src/db/scheduler/TaskDispatchStrategy.cpp | 18 ++- cpp/unittest/db/CMakeLists.txt | 6 +- cpp/unittest/db/scheduler_test.cpp | 124 ++++++++++++++++++ 4 files changed, 138 insertions(+), 24 deletions(-) create mode 100644 cpp/unittest/db/scheduler_test.cpp diff --git a/cpp/src/db/scheduler/TaskDispatchQueue.cpp b/cpp/src/db/scheduler/TaskDispatchQueue.cpp index 2ce0e933b4..b728e925a9 100644 --- a/cpp/src/db/scheduler/TaskDispatchQueue.cpp +++ b/cpp/src/db/scheduler/TaskDispatchQueue.cpp @@ -24,14 +24,6 @@ TaskDispatchQueue::Put(const ScheduleContextPtr &context) { return; } - if (queue_.size() >= capacity_) { - std::string error_msg = - "blocking queue is full, capacity: " + std::to_string(capacity_) + " queue_size: " + - std::to_string(queue_.size()); - SERVER_LOG_ERROR << error_msg; - throw server::ServerException(server::SERVER_BLOCKING_QUEUE_EMPTY, error_msg); - } - TaskDispatchStrategy::Schedule(context, queue_); empty_.notify_all(); @@ -42,12 +34,6 @@ TaskDispatchQueue::Take() { std::unique_lock lock(mtx); empty_.wait(lock, [this] { return !queue_.empty(); }); - if (queue_.empty()) { - std::string error_msg = "blocking queue empty"; - SERVER_LOG_ERROR << error_msg; - throw server::ServerException(server::SERVER_BLOCKING_QUEUE_EMPTY, error_msg); - } - ScheduleTaskPtr front(queue_.front()); queue_.pop_front(); full_.notify_all(); diff --git a/cpp/src/db/scheduler/TaskDispatchStrategy.cpp b/cpp/src/db/scheduler/TaskDispatchStrategy.cpp index 7200f2584f..985f86cb09 100644 --- a/cpp/src/db/scheduler/TaskDispatchStrategy.cpp +++ b/cpp/src/db/scheduler/TaskDispatchStrategy.cpp @@ -74,20 +74,26 @@ public: } std::string table_id = context->table_id(); - for(auto iter = task_list.begin(); iter != task_list.end(); ++iter) { + + //put delete task to proper position + //for example: task_list has 10 IndexLoadTask, only the No.5 IndexLoadTask is for table1 + //if user want to delete table1, the DeleteTask will be insert into No.6 position + for(std::list::reverse_iterator iter = task_list.rbegin(); iter != task_list.rend(); ++iter) { if((*iter)->type() != ScheduleTaskType::kIndexLoad) { continue; } - //put delete task to proper position IndexLoadTaskPtr loader = std::static_pointer_cast(*iter); - if(loader->file_->table_id_ == table_id) { - - task_list.insert(++iter, delete_task); - break; + if(loader->file_->table_id_ != table_id) { + continue; } + + task_list.insert(iter.base(), delete_task); + return true; } + //no task is searching this table, put DeleteTask to front of list so that the table will be delete asap + task_list.push_front(delete_task); return true; } }; diff --git a/cpp/unittest/db/CMakeLists.txt b/cpp/unittest/db/CMakeLists.txt index 0d69aba803..5bae9190f5 100644 --- a/cpp/unittest/db/CMakeLists.txt +++ b/cpp/unittest/db/CMakeLists.txt @@ -7,6 +7,7 @@ aux_source_directory(${MILVUS_ENGINE_SRC}/db db_srcs) aux_source_directory(${MILVUS_ENGINE_SRC}/config config_files) aux_source_directory(${MILVUS_ENGINE_SRC}/cache cache_srcs) aux_source_directory(${MILVUS_ENGINE_SRC}/wrapper wrapper_src) +aux_source_directory(./ test_srcs) aux_source_directory(${MILVUS_ENGINE_SRC}/db/scheduler scheduler_files) aux_source_directory(${MILVUS_ENGINE_SRC}/db/scheduler/context scheduler_context_files) @@ -30,10 +31,7 @@ set(db_test_src ${db_scheduler_srcs} ${wrapper_src} ${require_files} - MySQLMetaImpl_test.cpp - utils.cpp - db_tests.cpp - meta_tests.cpp) + ${test_srcs}) cuda_add_executable(db_test ${db_test_src}) diff --git a/cpp/unittest/db/scheduler_test.cpp b/cpp/unittest/db/scheduler_test.cpp new file mode 100644 index 0000000000..01a7057e00 --- /dev/null +++ b/cpp/unittest/db/scheduler_test.cpp @@ -0,0 +1,124 @@ +//////////////////////////////////////////////////////////////////////////////// +// Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved +// Unauthorized copying of this file, via any medium is strictly prohibited. +// Proprietary and confidential. +//////////////////////////////////////////////////////////////////////////////// +#include +#include +#include +#include + +#include "db/scheduler/TaskScheduler.h" +#include "db/scheduler/TaskDispatchStrategy.h" +#include "db/scheduler/TaskDispatchQueue.h" +#include "db/scheduler/task/SearchTask.h" +#include "db/scheduler/task/DeleteTask.h" +#include "db/scheduler/task/IndexLoadTask.h" + +using namespace zilliz::milvus; + +namespace { + +engine::TableFileSchemaPtr CreateTabileFileStruct(size_t id, const std::string& table_id) { + auto file = std::make_shared(); + file->id_ = id; + file->table_id_ = table_id; + return file; +} + +} + +TEST(DBSchedulerTest, TASK_QUEUE_TEST) { + engine::TaskDispatchQueue queue; + queue.SetCapacity(1000); + queue.Put(nullptr); + ASSERT_EQ(queue.Size(), 1UL); + + auto ptr = queue.Take(); + ASSERT_EQ(ptr, nullptr); + ASSERT_TRUE(queue.Empty()); + + engine::SearchContextPtr context_ptr = std::make_shared(1, 1, nullptr); + for(size_t i = 0; i < 10; i++) { + auto file = CreateTabileFileStruct(i, "tbl"); + context_ptr->AddIndexFile(file); + } + + queue.Put(context_ptr); + ASSERT_EQ(queue.Size(), 10); + + auto index_files = context_ptr->GetIndexMap(); + + ptr = queue.Front(); + ASSERT_EQ(ptr->type(), engine::ScheduleTaskType::kIndexLoad); + engine::IndexLoadTaskPtr load_task = std::static_pointer_cast(ptr); + ASSERT_EQ(load_task->file_->id_, index_files.begin()->first); + + ptr = queue.Back(); + ASSERT_EQ(ptr->type(), engine::ScheduleTaskType::kIndexLoad); +} + +TEST(DBSchedulerTest, SEARCH_SCHEDULER_TEST) { + std::list task_list; + bool ret = engine::TaskDispatchStrategy::Schedule(nullptr, task_list); + ASSERT_FALSE(ret); + + for(size_t i = 10; i < 30; i++) { + engine::IndexLoadTaskPtr task_ptr = std::make_shared(); + task_ptr->file_ = CreateTabileFileStruct(i, "tbl"); + task_list.push_back(task_ptr); + } + + engine::SearchContextPtr context_ptr = std::make_shared(1, 1, nullptr); + for(size_t i = 0; i < 20; i++) { + auto file = CreateTabileFileStruct(i, "tbl"); + context_ptr->AddIndexFile(file); + } + + ret = engine::TaskDispatchStrategy::Schedule(context_ptr, task_list); + ASSERT_TRUE(ret); + ASSERT_EQ(task_list.size(), 30); +} + +TEST(DBSchedulerTest, DELETE_SCHEDULER_TEST) { + std::list task_list; + bool ret = engine::TaskDispatchStrategy::Schedule(nullptr, task_list); + ASSERT_FALSE(ret); + + const std::string table_id = "to_delete_table"; + for(size_t i = 0; i < 10; i++) { + engine::IndexLoadTaskPtr task_ptr = std::make_shared(); + task_ptr->file_ = CreateTabileFileStruct(i, table_id); + task_list.push_back(task_ptr); + } + + for(size_t i = 0; i < 10; i++) { + engine::IndexLoadTaskPtr task_ptr = std::make_shared(); + task_ptr->file_ = CreateTabileFileStruct(i, "other_table"); + task_list.push_back(task_ptr); + } + + engine::meta::Meta::Ptr meta_ptr; + engine::DeleteContextPtr context_ptr = std::make_shared(table_id, meta_ptr); + ret = engine::TaskDispatchStrategy::Schedule(context_ptr, task_list); + ASSERT_TRUE(ret); + ASSERT_EQ(task_list.size(), 21); + + auto temp_list = task_list; + for(size_t i = 0; ; i++) { + engine::ScheduleTaskPtr task_ptr = temp_list.front(); + temp_list.pop_front(); + if(task_ptr->type() == engine::ScheduleTaskType::kDelete) { + ASSERT_EQ(i, 10); + break; + } + } + + context_ptr = std::make_shared("no_task_table", meta_ptr); + ret = engine::TaskDispatchStrategy::Schedule(context_ptr, task_list); + ASSERT_TRUE(ret); + ASSERT_EQ(task_list.size(), 22); + + engine::ScheduleTaskPtr task_ptr = task_list.front(); + ASSERT_EQ(task_ptr->type(), engine::ScheduleTaskType::kDelete); +} From fa4f8643eb585ddd3ed59457f31f747cb168c6ab Mon Sep 17 00:00:00 2001 From: starlord Date: Sun, 30 Jun 2019 14:20:56 +0800 Subject: [PATCH 039/189] add more unitest Former-commit-id: 92cb23d025f006182a3ab7db2e90bc1639016359 --- cpp/src/db/DBImpl.cpp | 276 ++++++++++++++++----------------- cpp/src/db/DBImpl.h | 4 +- cpp/src/db/ExecutionEngine.cpp | 11 +- cpp/src/db/ExecutionEngine.h | 3 +- cpp/unittest/db/db_tests.cpp | 30 ++-- cpp/unittest/db/meta_tests.cpp | 10 ++ cpp/unittest/db/misc_test.cpp | 103 ++++++++++++ 7 files changed, 275 insertions(+), 162 deletions(-) create mode 100644 cpp/unittest/db/misc_test.cpp diff --git a/cpp/src/db/DBImpl.cpp b/cpp/src/db/DBImpl.cpp index 92bbff2b84..a1bcc7ff44 100644 --- a/cpp/src/db/DBImpl.cpp +++ b/cpp/src/db/DBImpl.cpp @@ -253,144 +253,144 @@ Status DBImpl::Query(const std::string& table_id, const std::vector return QueryAsync(table_id, files_array, k, nq, vectors, dates, results); } -Status DBImpl::QuerySync(const std::string& table_id, uint64_t k, uint64_t nq, - const float* vectors, const meta::DatesT& dates, QueryResults& results) { - meta::DatePartionedTableFilesSchema files; - auto status = meta_ptr_->FilesToSearch(table_id, dates, files); - if (!status.ok()) { return status; } - - ENGINE_LOG_DEBUG << "Search DateT Size = " << files.size(); - - meta::TableFilesSchema index_files; - meta::TableFilesSchema raw_files; - for (auto &day_files : files) { - for (auto &file : day_files.second) { - file.file_type_ == meta::TableFileSchema::INDEX ? - index_files.push_back(file) : raw_files.push_back(file); - } - } - - int dim = 0; - if (!index_files.empty()) { - dim = index_files[0].dimension_; - } else if (!raw_files.empty()) { - dim = raw_files[0].dimension_; - } else { - ENGINE_LOG_DEBUG << "no files to search"; - return Status::OK(); - } - - { - // [{ids, distence}, ...] - using SearchResult = std::pair, std::vector>; - std::vector batchresult(nq); // allocate nq cells. - - auto cluster = [&](long *nns, float *dis, const int& k) -> void { - for (int i = 0; i < nq; ++i) { - auto f_begin = batchresult[i].first.cbegin(); - auto s_begin = batchresult[i].second.cbegin(); - batchresult[i].first.insert(f_begin, nns + i * k, nns + i * k + k); - batchresult[i].second.insert(s_begin, dis + i * k, dis + i * k + k); - } - }; - - // Allocate Memory - float *output_distence; - long *output_ids; - output_distence = (float *) malloc(k * nq * sizeof(float)); - output_ids = (long *) malloc(k * nq * sizeof(long)); - memset(output_distence, 0, k * nq * sizeof(float)); - memset(output_ids, 0, k * nq * sizeof(long)); - - long search_set_size = 0; - - auto search_in_index = [&](meta::TableFilesSchema& file_vec) -> void { - for (auto &file : file_vec) { - - ExecutionEnginePtr index = EngineFactory::Build(file.dimension_, file.location_, (EngineType)file.engine_type_); - index->Load(); - auto file_size = index->PhysicalSize(); - search_set_size += file_size; - - ENGINE_LOG_DEBUG << "Search file_type " << file.file_type_ << " Of Size: " - << file_size/(1024*1024) << " M"; - - int inner_k = index->Count() < k ? index->Count() : k; - auto start_time = METRICS_NOW_TIME; - index->Search(nq, vectors, inner_k, output_distence, output_ids); - auto end_time = METRICS_NOW_TIME; - auto total_time = METRICS_MICROSECONDS(start_time, end_time); - CollectFileMetrics(file.file_type_, file_size, total_time); - cluster(output_ids, output_distence, inner_k); // cluster to each query - memset(output_distence, 0, k * nq * sizeof(float)); - memset(output_ids, 0, k * nq * sizeof(long)); - } - }; - - auto topk_cpu = [](const std::vector &input_data, - const int &k, - float *output_distence, - long *output_ids) -> void { - std::map> inverted_table; - for (int i = 0; i < input_data.size(); ++i) { - if (inverted_table.count(input_data[i]) == 1) { - auto& ori_vec = inverted_table[input_data[i]]; - ori_vec.push_back(i); - } - else { - inverted_table[input_data[i]] = std::vector{i}; - } - } - - int count = 0; - for (auto &item : inverted_table){ - if (count == k) break; - for (auto &id : item.second){ - output_distence[count] = item.first; - output_ids[count] = id; - if (++count == k) break; - } - } - }; - auto cluster_topk = [&]() -> void { - QueryResult res; - for (auto &result_pair : batchresult) { - auto &dis = result_pair.second; - auto &nns = result_pair.first; - - topk_cpu(dis, k, output_distence, output_ids); - - int inner_k = dis.size() < k ? dis.size() : k; - for (int i = 0; i < inner_k; ++i) { - res.emplace_back(std::make_pair(nns[output_ids[i]], output_distence[i])); // mapping - } - results.push_back(res); // append to result list - res.clear(); - memset(output_distence, 0, k * nq * sizeof(float)); - memset(output_ids, 0, k * nq * sizeof(long)); - } - }; - - search_in_index(raw_files); - search_in_index(index_files); - - ENGINE_LOG_DEBUG << "Search Overall Set Size = " << search_set_size << " M"; - cluster_topk(); - - free(output_distence); - free(output_ids); - } - - if (results.empty()) { - return Status::NotFound("Group " + table_id + ", search result not found!"); - } - - QueryResults temp_results; - CalcScore(nq, vectors, dim, results, temp_results); - results.swap(temp_results); - - return Status::OK(); -} +//Status DBImpl::QuerySync(const std::string& table_id, uint64_t k, uint64_t nq, +// const float* vectors, const meta::DatesT& dates, QueryResults& results) { +// meta::DatePartionedTableFilesSchema files; +// auto status = meta_ptr_->FilesToSearch(table_id, dates, files); +// if (!status.ok()) { return status; } +// +// ENGINE_LOG_DEBUG << "Search DateT Size = " << files.size(); +// +// meta::TableFilesSchema index_files; +// meta::TableFilesSchema raw_files; +// for (auto &day_files : files) { +// for (auto &file : day_files.second) { +// file.file_type_ == meta::TableFileSchema::INDEX ? +// index_files.push_back(file) : raw_files.push_back(file); +// } +// } +// +// int dim = 0; +// if (!index_files.empty()) { +// dim = index_files[0].dimension_; +// } else if (!raw_files.empty()) { +// dim = raw_files[0].dimension_; +// } else { +// ENGINE_LOG_DEBUG << "no files to search"; +// return Status::OK(); +// } +// +// { +// // [{ids, distence}, ...] +// using SearchResult = std::pair, std::vector>; +// std::vector batchresult(nq); // allocate nq cells. +// +// auto cluster = [&](long *nns, float *dis, const int& k) -> void { +// for (int i = 0; i < nq; ++i) { +// auto f_begin = batchresult[i].first.cbegin(); +// auto s_begin = batchresult[i].second.cbegin(); +// batchresult[i].first.insert(f_begin, nns + i * k, nns + i * k + k); +// batchresult[i].second.insert(s_begin, dis + i * k, dis + i * k + k); +// } +// }; +// +// // Allocate Memory +// float *output_distence; +// long *output_ids; +// output_distence = (float *) malloc(k * nq * sizeof(float)); +// output_ids = (long *) malloc(k * nq * sizeof(long)); +// memset(output_distence, 0, k * nq * sizeof(float)); +// memset(output_ids, 0, k * nq * sizeof(long)); +// +// long search_set_size = 0; +// +// auto search_in_index = [&](meta::TableFilesSchema& file_vec) -> void { +// for (auto &file : file_vec) { +// +// ExecutionEnginePtr index = EngineFactory::Build(file.dimension_, file.location_, (EngineType)file.engine_type_); +// index->Load(); +// auto file_size = index->PhysicalSize(); +// search_set_size += file_size; +// +// ENGINE_LOG_DEBUG << "Search file_type " << file.file_type_ << " Of Size: " +// << file_size/(1024*1024) << " M"; +// +// int inner_k = index->Count() < k ? index->Count() : k; +// auto start_time = METRICS_NOW_TIME; +// index->Search(nq, vectors, inner_k, output_distence, output_ids); +// auto end_time = METRICS_NOW_TIME; +// auto total_time = METRICS_MICROSECONDS(start_time, end_time); +// CollectFileMetrics(file.file_type_, file_size, total_time); +// cluster(output_ids, output_distence, inner_k); // cluster to each query +// memset(output_distence, 0, k * nq * sizeof(float)); +// memset(output_ids, 0, k * nq * sizeof(long)); +// } +// }; +// +// auto topk_cpu = [](const std::vector &input_data, +// const int &k, +// float *output_distence, +// long *output_ids) -> void { +// std::map> inverted_table; +// for (int i = 0; i < input_data.size(); ++i) { +// if (inverted_table.count(input_data[i]) == 1) { +// auto& ori_vec = inverted_table[input_data[i]]; +// ori_vec.push_back(i); +// } +// else { +// inverted_table[input_data[i]] = std::vector{i}; +// } +// } +// +// int count = 0; +// for (auto &item : inverted_table){ +// if (count == k) break; +// for (auto &id : item.second){ +// output_distence[count] = item.first; +// output_ids[count] = id; +// if (++count == k) break; +// } +// } +// }; +// auto cluster_topk = [&]() -> void { +// QueryResult res; +// for (auto &result_pair : batchresult) { +// auto &dis = result_pair.second; +// auto &nns = result_pair.first; +// +// topk_cpu(dis, k, output_distence, output_ids); +// +// int inner_k = dis.size() < k ? dis.size() : k; +// for (int i = 0; i < inner_k; ++i) { +// res.emplace_back(std::make_pair(nns[output_ids[i]], output_distence[i])); // mapping +// } +// results.push_back(res); // append to result list +// res.clear(); +// memset(output_distence, 0, k * nq * sizeof(float)); +// memset(output_ids, 0, k * nq * sizeof(long)); +// } +// }; +// +// search_in_index(raw_files); +// search_in_index(index_files); +// +// ENGINE_LOG_DEBUG << "Search Overall Set Size = " << search_set_size << " M"; +// cluster_topk(); +// +// free(output_distence); +// free(output_ids); +// } +// +// if (results.empty()) { +// return Status::NotFound("Group " + table_id + ", search result not found!"); +// } +// +// QueryResults temp_results; +// CalcScore(nq, vectors, dim, results, temp_results); +// results.swap(temp_results); +// +// return Status::OK(); +//} Status DBImpl::QueryAsync(const std::string& table_id, const meta::TableFilesSchema& files, uint64_t k, uint64_t nq, const float* vectors, diff --git a/cpp/src/db/DBImpl.h b/cpp/src/db/DBImpl.h index b4d60a27a9..43627cbace 100644 --- a/cpp/src/db/DBImpl.h +++ b/cpp/src/db/DBImpl.h @@ -62,8 +62,8 @@ public: virtual ~DBImpl(); private: - Status QuerySync(const std::string& table_id, uint64_t k, uint64_t nq, - const float* vectors, const meta::DatesT& dates, QueryResults& results); +// Status QuerySync(const std::string& table_id, uint64_t k, uint64_t nq, +// const float* vectors, const meta::DatesT& dates, QueryResults& results); Status QueryAsync(const std::string& table_id, const meta::TableFilesSchema& files, uint64_t k, uint64_t nq, const float* vectors, diff --git a/cpp/src/db/ExecutionEngine.cpp b/cpp/src/db/ExecutionEngine.cpp index f27d04dfa0..3412eb34bd 100644 --- a/cpp/src/db/ExecutionEngine.cpp +++ b/cpp/src/db/ExecutionEngine.cpp @@ -11,14 +11,9 @@ namespace zilliz { namespace milvus { namespace engine { -Status ExecutionEngine::AddWithIds(const std::vector& vectors, const std::vector& vector_ids) { - long n1 = (long)vectors.size(); - long n2 = (long)vector_ids.size(); - if (n1 != n2) { - LOG(ERROR) << "vectors size is not equal to the size of vector_ids: " << n1 << "!=" << n2; - return Status::Error("Error: AddWithIds"); - } - return AddWithIds(n1, vectors.data(), vector_ids.data()); +Status ExecutionEngine::AddWithIdArray(const std::vector& vectors, const std::vector& vector_ids) { + long n = (long)vector_ids.size(); + return AddWithIds(n, vectors.data(), vector_ids.data()); } diff --git a/cpp/src/db/ExecutionEngine.h b/cpp/src/db/ExecutionEngine.h index f26dce6371..d2b4d01e67 100644 --- a/cpp/src/db/ExecutionEngine.h +++ b/cpp/src/db/ExecutionEngine.h @@ -23,8 +23,7 @@ enum class EngineType { class ExecutionEngine { public: - virtual Status AddWithIds(const std::vector& vectors, - const std::vector& vector_ids); + virtual Status AddWithIdArray(const std::vector& vectors, const std::vector& vector_ids); virtual Status AddWithIds(long n, const float *xdata, const long *xids) = 0; diff --git a/cpp/unittest/db/db_tests.cpp b/cpp/unittest/db/db_tests.cpp index b5eb8e642b..8e50b7403b 100644 --- a/cpp/unittest/db/db_tests.cpp +++ b/cpp/unittest/db/db_tests.cpp @@ -89,20 +89,14 @@ TEST_F(DBTest, CONFIG_TEST) { TEST_F(DBTest, DB_TEST) { - static const std::string table_name = "test_group"; - static const int table_dim = 256; - - engine::meta::TableSchema table_info; - table_info.dimension_ = table_dim; - table_info.table_id_ = table_name; - table_info.engine_type_ = (int)engine::EngineType::FAISS_IDMAP; + engine::meta::TableSchema table_info = BuildTableSchema(); engine::Status stat = db_->CreateTable(table_info); engine::meta::TableSchema table_info_get; - table_info_get.table_id_ = table_name; + table_info_get.table_id_ = TABLE_NAME; stat = db_->DescribeTable(table_info_get); ASSERT_STATS(stat); - ASSERT_EQ(table_info_get.dimension_, table_dim); + ASSERT_EQ(table_info_get.dimension_, TABLE_DIM); engine::IDNumbers vector_ids; engine::IDNumbers target_ids; @@ -131,7 +125,7 @@ TEST_F(DBTest, DB_TEST) { prev_count = count; START_TIMER; - stat = db_->Query(table_name, k, qb, qxb.data(), results); + stat = db_->Query(TABLE_NAME, k, qb, qxb.data(), results); ss << "Search " << j << " With Size " << count/engine::meta::M << " M"; STOP_TIMER(ss.str()); @@ -154,10 +148,10 @@ TEST_F(DBTest, DB_TEST) { for (auto i=0; iInsertVectors(table_name, qb, qxb.data(), target_ids); + db_->InsertVectors(TABLE_NAME, qb, qxb.data(), target_ids); ASSERT_EQ(target_ids.size(), qb); } else { - db_->InsertVectors(table_name, nb, xb.data(), vector_ids); + db_->InsertVectors(TABLE_NAME, nb, xb.data(), vector_ids); } std::this_thread::sleep_for(std::chrono::microseconds(1)); } @@ -224,6 +218,18 @@ TEST_F(DBTest2, ARHIVE_DISK_CHECK) { engine::meta::TableSchema table_info = BuildTableSchema(); engine::Status stat = db_->CreateTable(table_info); + std::vector table_schema_array; + stat = db_->AllTables(table_schema_array); + ASSERT_STATS(stat); + bool bfound = false; + for(auto& schema : table_schema_array) { + if(schema.table_id_ == TABLE_NAME) { + bfound = true; + break; + } + } + ASSERT_TRUE(bfound); + engine::meta::TableSchema table_info_get; table_info_get.table_id_ = TABLE_NAME; stat = db_->DescribeTable(table_info_get); diff --git a/cpp/unittest/db/meta_tests.cpp b/cpp/unittest/db/meta_tests.cpp index a7933829c9..f6c9551aa7 100644 --- a/cpp/unittest/db/meta_tests.cpp +++ b/cpp/unittest/db/meta_tests.cpp @@ -39,6 +39,10 @@ TEST_F(MetaTest, TABLE_TEST) { table.table_id_ = table_id; status = impl_->CreateTable(table); ASSERT_TRUE(status.ok()); + + table.table_id_ = ""; + status = impl_->CreateTable(table); + ASSERT_TRUE(status.ok()); } TEST_F(MetaTest, TABLE_FILE_TEST) { @@ -46,6 +50,7 @@ TEST_F(MetaTest, TABLE_FILE_TEST) { meta::TableSchema table; table.table_id_ = table_id; + table.dimension_ = 256; auto status = impl_->CreateTable(table); meta::TableFileSchema table_file; @@ -54,6 +59,11 @@ TEST_F(MetaTest, TABLE_FILE_TEST) { ASSERT_TRUE(status.ok()); ASSERT_EQ(table_file.file_type_, meta::TableFileSchema::NEW); + uint64_t cnt = 0; + status = impl_->Count(table_id, cnt); + ASSERT_TRUE(status.ok()); + ASSERT_EQ(cnt, 0UL); + auto file_id = table_file.file_id_; auto new_file_type = meta::TableFileSchema::INDEX; diff --git a/cpp/unittest/db/misc_test.cpp b/cpp/unittest/db/misc_test.cpp new file mode 100644 index 0000000000..6f1f87c874 --- /dev/null +++ b/cpp/unittest/db/misc_test.cpp @@ -0,0 +1,103 @@ +//////////////////////////////////////////////////////////////////////////////// +// Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved +// Unauthorized copying of this file, via any medium is strictly prohibited. +// Proprietary and confidential. +//////////////////////////////////////////////////////////////////////////////// +#include +#include +#include +#include + +#include "db/FaissExecutionEngine.h" +#include "db/Exception.h" +#include "db/Status.h" +#include "db/Options.h" +#include "db/DBMetaImpl.h" + +#include + +using namespace zilliz::milvus; + +TEST(DBMiscTest, ENGINE_API_TEST) { + //engine api AddWithIdArray + const uint16_t dim = 512; + const long n = 10; + engine::FaissExecutionEngine engine(512, "/tmp/1", "IDMap", "IDMap,Flat"); + std::vector vectors; + std::vector ids; + for (long i = 0; i < n; i++) { + for (uint16_t k = 0; k < dim; k++) { + vectors.push_back((float) k); + } + ids.push_back(i); + } + + auto status = engine.AddWithIdArray(vectors, ids); + ASSERT_TRUE(status.ok()); +} + +TEST(DBMiscTest, EXCEPTION_TEST) { + engine::Exception ex1(""); + std::string what = ex1.what(); + ASSERT_FALSE(what.empty()); + + engine::OutOfRangeException ex2; + what = ex2.what(); + ASSERT_FALSE(what.empty()); +} + +TEST(DBMiscTest, STATUS_TEST) { + engine::Status status = engine::Status::OK(); + std::string str = status.ToString(); + ASSERT_FALSE(str.empty()); + + status = engine::Status::Error("wrong", "mistake"); + ASSERT_TRUE(status.IsError()); + str = status.ToString(); + ASSERT_FALSE(str.empty()); + + status = engine::Status::NotFound("wrong", "mistake"); + ASSERT_TRUE(status.IsNotFound()); + str = status.ToString(); + ASSERT_FALSE(str.empty()); + + status = engine::Status::DBTransactionError("wrong", "mistake"); + ASSERT_TRUE(status.IsDBTransactionError()); + str = status.ToString(); + ASSERT_FALSE(str.empty()); +} + +TEST(DBMiscTest, OPTIONS_TEST) { + try { + engine::ArchiveConf archive("$$##"); + } catch (std::exception& ex) { + ASSERT_TRUE(true); + } + + { + engine::ArchiveConf archive("delete", "no"); + ASSERT_TRUE(archive.GetCriterias().empty()); + } + + { + engine::ArchiveConf archive("delete", "1:2"); + ASSERT_TRUE(archive.GetCriterias().empty()); + } + + { + engine::ArchiveConf archive("delete", "1:2:3"); + ASSERT_TRUE(archive.GetCriterias().empty()); + } +} + +TEST(DBMiscTest, META_TEST) { + engine::DBMetaOptions options; + options.path = "/tmp/milvus_test"; + engine::meta::DBMetaImpl impl(options); + + time_t tt; + time( &tt ); + int delta = 10; + engine::meta::DateT dt = impl.GetDate(tt, delta); + ASSERT_GT(dt, 0); +} \ No newline at end of file From 8ffb89d2d74b88d43dadbb5006d3c23592b088ed Mon Sep 17 00:00:00 2001 From: starlord Date: Sun, 30 Jun 2019 15:20:36 +0800 Subject: [PATCH 040/189] add more unittest Former-commit-id: 853d82c0f7b81579d1332c5f1eb08234bc49a065 --- cpp/unittest/db/meta_tests.cpp | 5 ++++- cpp/unittest/db/misc_test.cpp | 34 ++++++++++++++++++++++++++++++++++ 2 files changed, 38 insertions(+), 1 deletion(-) diff --git a/cpp/unittest/db/meta_tests.cpp b/cpp/unittest/db/meta_tests.cpp index f6c9551aa7..0f3a92af09 100644 --- a/cpp/unittest/db/meta_tests.cpp +++ b/cpp/unittest/db/meta_tests.cpp @@ -265,5 +265,8 @@ TEST_F(MetaTest, TABLE_FILES_TEST) { ASSERT_EQ(dated_files[table_file.date_].size(), to_index_files_cnt+raw_files_cnt+index_files_cnt); - + status = impl_->FilesToSearch(table_id, meta::DatesT(), dated_files); + ASSERT_TRUE(status.ok()); + ASSERT_EQ(dated_files[table_file.date_].size(), + to_index_files_cnt+raw_files_cnt+index_files_cnt); } diff --git a/cpp/unittest/db/misc_test.cpp b/cpp/unittest/db/misc_test.cpp index 6f1f87c874..4356746fc2 100644 --- a/cpp/unittest/db/misc_test.cpp +++ b/cpp/unittest/db/misc_test.cpp @@ -13,11 +13,19 @@ #include "db/Status.h" #include "db/Options.h" #include "db/DBMetaImpl.h" +#include "db/EngineFactory.h" #include using namespace zilliz::milvus; +namespace { + void CopyStatus(engine::Status& st1, engine::Status& st2) { + st1 = st2; + } + +} + TEST(DBMiscTest, ENGINE_API_TEST) { //engine api AddWithIdArray const uint16_t dim = 512; @@ -34,6 +42,15 @@ TEST(DBMiscTest, ENGINE_API_TEST) { auto status = engine.AddWithIdArray(vectors, ids); ASSERT_TRUE(status.ok()); + + auto engine_ptr = engine::EngineFactory::Build(128, "/tmp", engine::EngineType::INVALID); + ASSERT_EQ(engine_ptr, nullptr); + + engine_ptr = engine::EngineFactory::Build(128, "/tmp", engine::EngineType::FAISS_IVFFLAT); + ASSERT_NE(engine_ptr, nullptr); + + engine_ptr = engine::EngineFactory::Build(128, "/tmp", engine::EngineType::FAISS_IDMAP); + ASSERT_NE(engine_ptr, nullptr); } TEST(DBMiscTest, EXCEPTION_TEST) { @@ -65,6 +82,10 @@ TEST(DBMiscTest, STATUS_TEST) { ASSERT_TRUE(status.IsDBTransactionError()); str = status.ToString(); ASSERT_FALSE(str.empty()); + + engine::Status status_copy = engine::Status::OK(); + CopyStatus(status_copy, status); + ASSERT_TRUE(status.IsDBTransactionError()); } TEST(DBMiscTest, OPTIONS_TEST) { @@ -88,6 +109,19 @@ TEST(DBMiscTest, OPTIONS_TEST) { engine::ArchiveConf archive("delete", "1:2:3"); ASSERT_TRUE(archive.GetCriterias().empty()); } + + { + engine::ArchiveConf archive("delete"); + engine::ArchiveConf::CriteriaT criterial = { + {"disk", 1024}, + {"days", 100} + }; + archive.SetCriterias(criterial); + + auto crit = archive.GetCriterias(); + ASSERT_EQ(criterial["disk"], 1024); + ASSERT_EQ(criterial["days"], 100); + } } TEST(DBMiscTest, META_TEST) { From 57cff92d04354b5741842606d4c0b836210da9bf Mon Sep 17 00:00:00 2001 From: starlord Date: Sun, 30 Jun 2019 16:19:04 +0800 Subject: [PATCH 041/189] fix build error Former-commit-id: da010fb2e477164a9dcc190839bcd7f9d0980b02 --- cpp/conf/server_config.template | 2 +- cpp/src/server/DBWrapper.cpp | 10 ++- cpp/unittest/metrics/CMakeLists.txt | 10 +-- cpp/unittest/metrics/metricbase_test.cpp | 2 +- cpp/unittest/metrics/metrics_test.cpp | 4 +- cpp/unittest/metrics/prometheus_test.cpp | 2 +- cpp/unittest/metrics/utils.cpp | 79 ++++++++++++++++++++++++ cpp/unittest/metrics/utils.h | 64 +++++++++++++++++++ 8 files changed, 161 insertions(+), 12 deletions(-) create mode 100644 cpp/unittest/metrics/utils.cpp create mode 100644 cpp/unittest/metrics/utils.h diff --git a/cpp/conf/server_config.template b/cpp/conf/server_config.template index 1a1c8303f2..6082969fae 100644 --- a/cpp/conf/server_config.template +++ b/cpp/conf/server_config.template @@ -6,7 +6,7 @@ server_config: db_config: db_path: @MILVUS_DB_PATH@ # milvus data storage path - db_backend_url: http://127.0.0.1 # meta database uri + db_backend_url: sqlite://:@:/ # meta database uri index_building_threshold: 1024 # index building trigger threshold, default: 1024, unit: MB archive_disk_threshold: 512 # triger archive action if storage size exceed this value, unit: GB archive_days_threshold: 30 # files older than x days will be archived, unit: day diff --git a/cpp/src/server/DBWrapper.cpp b/cpp/src/server/DBWrapper.cpp index a3db0bf110..bf859b3b4f 100644 --- a/cpp/src/server/DBWrapper.cpp +++ b/cpp/src/server/DBWrapper.cpp @@ -70,9 +70,15 @@ DBWrapper::DBWrapper() { kill(0, SIGUSR1); } - zilliz::milvus::engine::DB::Open(opt, &db_); + std::string msg = opt.meta.path; + try { + zilliz::milvus::engine::DB::Open(opt, &db_); + } catch(std::exception& ex) { + msg = ex.what(); + } + if(db_ == nullptr) { - std::cout << "ERROR! Failed to open database" << std::endl; + std::cout << "ERROR! Failed to open database: " << msg << std::endl; kill(0, SIGUSR1); } } diff --git a/cpp/unittest/metrics/CMakeLists.txt b/cpp/unittest/metrics/CMakeLists.txt index 80210772a8..d31e44c056 100644 --- a/cpp/unittest/metrics/CMakeLists.txt +++ b/cpp/unittest/metrics/CMakeLists.txt @@ -17,6 +17,7 @@ aux_source_directory(../../src/config config_files) aux_source_directory(../../src/cache cache_srcs) aux_source_directory(../../src/wrapper wrapper_src) aux_source_directory(../../src/metrics metrics_src) +aux_source_directory(./ test_srcs) aux_source_directory(${MILVUS_ENGINE_SRC}/db/scheduler scheduler_files) aux_source_directory(${MILVUS_ENGINE_SRC}/db/scheduler/context scheduler_context_files) @@ -35,6 +36,7 @@ link_directories("/usr/local/cuda/lib64") #include_directories(../db/utils.h) include_directories(../../src/metrics) +include_directories(/usr/include/mysql) #set(metrics_src_files # ../../src/metrics/Metrics.cpp @@ -47,17 +49,14 @@ include_directories(../../src/metrics) # ) set(count_test_src - ${unittest_srcs} ${config_files} ${cache_srcs} ${db_srcs} ${db_scheduler_srcs} ${wrapper_src} ${metrics_src} - metrics_test.cpp - prometheus_test.cpp - ../db/utils.cpp - metricbase_test.cpp) + ${test_srcs} + ) add_executable(metrics_test ${count_test_src} ${require_files} ) @@ -75,6 +74,7 @@ target_link_libraries(metrics_test gtest pthread z + mysqlpp ${unittest_libs} ) diff --git a/cpp/unittest/metrics/metricbase_test.cpp b/cpp/unittest/metrics/metricbase_test.cpp index ac850c7b48..1997748fdd 100644 --- a/cpp/unittest/metrics/metricbase_test.cpp +++ b/cpp/unittest/metrics/metricbase_test.cpp @@ -11,7 +11,7 @@ using namespace zilliz::milvus; -TEST(MetricbaseTest, Metricbase_Test){ +TEST(MetricbaseTest, METRICBASE_TEST){ server::MetricsBase instance = server::MetricsBase::GetInstance(); instance.Init(); server::SystemInfo::GetInstance().Init(); diff --git a/cpp/unittest/metrics/metrics_test.cpp b/cpp/unittest/metrics/metrics_test.cpp index 923c7b717b..883e63ed03 100644 --- a/cpp/unittest/metrics/metrics_test.cpp +++ b/cpp/unittest/metrics/metrics_test.cpp @@ -15,7 +15,7 @@ #include #include "metrics/Metrics.h" -#include "../db/utils.h" +#include "utils.h" #include "db/DB.h" #include "db/DBMetaImpl.h" #include "db/Factories.h" @@ -24,7 +24,7 @@ using namespace zilliz::milvus; -TEST_F(DBTest, Metric_Tes) { +TEST_F(MetricTest, Metric_Tes) { server::SystemInfo::GetInstance().Init(); // server::Metrics::GetInstance().Init(); diff --git a/cpp/unittest/metrics/prometheus_test.cpp b/cpp/unittest/metrics/prometheus_test.cpp index 885abed566..521e00fc5c 100644 --- a/cpp/unittest/metrics/prometheus_test.cpp +++ b/cpp/unittest/metrics/prometheus_test.cpp @@ -11,7 +11,7 @@ using namespace zilliz::milvus; -TEST(PrometheusTest, Prometheus_Test){ +TEST(PrometheusTest, PROMETHEUS_TEST){ server::PrometheusMetrics instance = server::PrometheusMetrics::GetInstance(); instance.Init(); instance.SetStartup(true); diff --git a/cpp/unittest/metrics/utils.cpp b/cpp/unittest/metrics/utils.cpp new file mode 100644 index 0000000000..81e924a87e --- /dev/null +++ b/cpp/unittest/metrics/utils.cpp @@ -0,0 +1,79 @@ +//////////////////////////////////////////////////////////////////////////////// +// Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved +// Unauthorized copying of this file, via any medium is strictly prohibited. +// Proprietary and confidential. +//////////////////////////////////////////////////////////////////////////////// + +#include +#include +#include +#include + +#include "utils.h" +#include "db/Factories.h" +#include "db/Options.h" + +INITIALIZE_EASYLOGGINGPP + +using namespace zilliz::milvus; + +static std::string uri; + +class DBTestEnvironment : public ::testing::Environment { +public: + +// explicit DBTestEnvironment(std::string uri) : uri_(uri) {} + + static std::string getURI() { + return uri; + } + + void SetUp() override { + getURI(); + } + +}; + +void ASSERT_STATS(engine::Status& stat) { + ASSERT_TRUE(stat.ok()); + if(!stat.ok()) { + std::cout << stat.ToString() << std::endl; + } +} + + +void MetricTest::InitLog() { + el::Configurations defaultConf; + defaultConf.setToDefault(); + defaultConf.set(el::Level::Debug, + el::ConfigurationType::Format, "[%thread-%datetime-%level]: %msg (%fbase:%line)"); + el::Loggers::reconfigureLogger("default", defaultConf); +} + +engine::Options MetricTest::GetOptions() { + auto options = engine::OptionsFactory::Build(); + options.meta.path = "/tmp/milvus_test"; + options.meta.backend_uri = "sqlite://:@:/"; + return options; +} + +void MetricTest::SetUp() { + InitLog(); + auto options = GetOptions(); + db_ = engine::DBFactory::Build(options); +} + +void MetricTest::TearDown() { + delete db_; + boost::filesystem::remove_all("/tmp/milvus_test"); +} + +int main(int argc, char **argv) { + ::testing::InitGoogleTest(&argc, argv); + if (argc > 1) { + uri = argv[1]; + } +// std::cout << uri << std::endl; + ::testing::AddGlobalTestEnvironment(new DBTestEnvironment); + return RUN_ALL_TESTS(); +} diff --git a/cpp/unittest/metrics/utils.h b/cpp/unittest/metrics/utils.h new file mode 100644 index 0000000000..1badce00f2 --- /dev/null +++ b/cpp/unittest/metrics/utils.h @@ -0,0 +1,64 @@ +//////////////////////////////////////////////////////////////////////////////// +// Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved +// Unauthorized copying of this file, via any medium is strictly prohibited. +// Proprietary and confidential. +//////////////////////////////////////////////////////////////////////////////// + +#pragma once + +#include +#include +//#include + +#include "db/DB.h" +#include "db/DBMetaImpl.h" +#include "db/MySQLMetaImpl.h" + + +#define TIMING + +#ifdef TIMING +#define INIT_TIMER auto start = std::chrono::high_resolution_clock::now(); +#define START_TIMER start = std::chrono::high_resolution_clock::now(); +#define STOP_TIMER(name) LOG(DEBUG) << "RUNTIME of " << name << ": " << \ + std::chrono::duration_cast( \ + std::chrono::high_resolution_clock::now()-start \ + ).count() << " ms "; +#else +#define INIT_TIMER +#define START_TIMER +#define STOP_TIMER(name) +#endif + +void ASSERT_STATS(zilliz::milvus::engine::Status& stat); + +//class TestEnv : public ::testing::Environment { +//public: +// +// static std::string getURI() { +// if (const char* uri = std::getenv("MILVUS_DBMETA_URI")) { +// return uri; +// } +// else { +// return ""; +// } +// } +// +// void SetUp() override { +// getURI(); +// } +// +//}; +// +//::testing::Environment* const test_env = +// ::testing::AddGlobalTestEnvironment(new TestEnv); + +class MetricTest : public ::testing::Test { +protected: + zilliz::milvus::engine::DB* db_; + + void InitLog(); + virtual void SetUp() override; + virtual void TearDown() override; + virtual zilliz::milvus::engine::Options GetOptions(); +}; \ No newline at end of file From 2f19e1ca183735a49f175dbb8810b941139d0413 Mon Sep 17 00:00:00 2001 From: zhiru Date: Sun, 30 Jun 2019 15:58:22 +0800 Subject: [PATCH 042/189] edit README.md Former-commit-id: 150f84fc982b23a055664e606b0ef22494ad650a --- cpp/README.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/cpp/README.md b/cpp/README.md index c1cd381442..1b2f507db2 100644 --- a/cpp/README.md +++ b/cpp/README.md @@ -53,6 +53,12 @@ If you encounter the following error when building: ### Launch server Set config in cpp/conf/server_config.yaml +Add milvus/bin/lib to LD_LIBRARY_PATH + +``` +export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/path/to/milvus/bin/lib +``` + Then launch server with config: cd [build output path] start_server.sh From 72877cf3d98365d87427ca52b675f12a9c5e4ff0 Mon Sep 17 00:00:00 2001 From: zhiru Date: Sun, 30 Jun 2019 16:08:56 +0800 Subject: [PATCH 043/189] edit server_config.template Former-commit-id: 288b49e77638b34aa3882f7ba2800de46bb446d8 --- cpp/conf/server_config.template | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/cpp/conf/server_config.template b/cpp/conf/server_config.template index 6082969fae..6b432056c5 100644 --- a/cpp/conf/server_config.template +++ b/cpp/conf/server_config.template @@ -6,7 +6,12 @@ server_config: db_config: db_path: @MILVUS_DB_PATH@ # milvus data storage path - db_backend_url: sqlite://:@:/ # meta database uri + + # URI format: dialect://username:password@host:port/database + # All parts except dialect are optional, but you MUST include the delimiters + # Currently dialect supports mysql or sqlite + db_backend_url: dialect://username:password@host:port/database # meta database uri + index_building_threshold: 1024 # index building trigger threshold, default: 1024, unit: MB archive_disk_threshold: 512 # triger archive action if storage size exceed this value, unit: GB archive_days_threshold: 30 # files older than x days will be archived, unit: day From 4bd3e1f2dabf4b9208ef3e03a52e66d2829c1d05 Mon Sep 17 00:00:00 2001 From: starlord Date: Sun, 30 Jun 2019 16:42:12 +0800 Subject: [PATCH 044/189] avoid unitest hang Former-commit-id: 95e1d10cb61ff5f8c7e5b8b38564f2c7ab78865c --- cpp/unittest/db/MySQLMetaImpl_test.cpp | 56 ++++++++++++++++++++++---- cpp/unittest/db/utils.cpp | 5 +++ 2 files changed, 54 insertions(+), 7 deletions(-) diff --git a/cpp/unittest/db/MySQLMetaImpl_test.cpp b/cpp/unittest/db/MySQLMetaImpl_test.cpp index 93e0ce9b28..a370fd1cd9 100644 --- a/cpp/unittest/db/MySQLMetaImpl_test.cpp +++ b/cpp/unittest/db/MySQLMetaImpl_test.cpp @@ -32,12 +32,19 @@ using namespace zilliz::milvus::engine; //} TEST_F(MySQLTest, core) { -// DBMetaOptions options; + DBMetaOptions options; // //dialect+driver://username:password@host:port/database // options.backend_uri = "mysql://root:1234@:/test"; // options.path = "/tmp/vecwise_test"; + try { + options = getDBMetaOptions(); + } catch(std::exception& ex) { + ASSERT_TRUE(false); + return; + } + int mode = Options::MODE::SINGLE; - meta::MySQLMetaImpl impl(getDBMetaOptions(), mode); + meta::MySQLMetaImpl impl(options, mode); // auto status = impl.Initialize(); // ASSERT_TRUE(status.ok()); @@ -192,9 +199,16 @@ TEST_F(MySQLTest, core) { } TEST_F(MySQLTest, GROUP_TEST) { + DBMetaOptions options; + try { + options = getDBMetaOptions(); + } catch(std::exception& ex) { + ASSERT_TRUE(false); + return; + } int mode = Options::MODE::SINGLE; - meta::MySQLMetaImpl impl(getDBMetaOptions(), mode); + meta::MySQLMetaImpl impl(options, mode); auto table_id = "meta_test_group"; @@ -228,9 +242,16 @@ TEST_F(MySQLTest, GROUP_TEST) { } TEST_F(MySQLTest, table_file_TEST) { + DBMetaOptions options; + try { + options = getDBMetaOptions(); + } catch(std::exception& ex) { + ASSERT_TRUE(false); + return; + } int mode = Options::MODE::SINGLE; - meta::MySQLMetaImpl impl(getDBMetaOptions(), mode); + meta::MySQLMetaImpl impl(options, mode); auto table_id = "meta_test_group"; @@ -296,7 +317,14 @@ TEST_F(MySQLTest, table_file_TEST) { TEST_F(MySQLTest, ARCHIVE_TEST_DAYS) { srand(time(0)); - DBMetaOptions options = getDBMetaOptions(); + DBMetaOptions options; + try { + options = getDBMetaOptions(); + } catch(std::exception& ex) { + ASSERT_TRUE(false); + return; + } + int days_num = rand() % 100; std::stringstream ss; ss << "days:" << days_num; @@ -350,7 +378,14 @@ TEST_F(MySQLTest, ARCHIVE_TEST_DAYS) { } TEST_F(MySQLTest, ARCHIVE_TEST_DISK) { - DBMetaOptions options = getDBMetaOptions(); + DBMetaOptions options; + try { + options = getDBMetaOptions(); + } catch(std::exception& ex) { + ASSERT_TRUE(false); + return; + } + options.archive_conf = ArchiveConf("delete", "disk:11"); int mode = Options::MODE::SINGLE; auto impl = meta::MySQLMetaImpl(options, mode); @@ -397,9 +432,16 @@ TEST_F(MySQLTest, ARCHIVE_TEST_DISK) { } TEST_F(MySQLTest, TABLE_FILES_TEST) { + DBMetaOptions options; + try { + options = getDBMetaOptions(); + } catch(std::exception& ex) { + ASSERT_TRUE(false); + return; + } int mode = Options::MODE::SINGLE; - auto impl = meta::MySQLMetaImpl(getDBMetaOptions(), mode); + auto impl = meta::MySQLMetaImpl(options, mode); auto table_id = "meta_test_group"; diff --git a/cpp/unittest/db/utils.cpp b/cpp/unittest/db/utils.cpp index 6b1fc1e407..70c0712549 100644 --- a/cpp/unittest/db/utils.cpp +++ b/cpp/unittest/db/utils.cpp @@ -91,6 +91,11 @@ zilliz::milvus::engine::DBMetaOptions MySQLTest::getDBMetaOptions() { zilliz::milvus::engine::DBMetaOptions options; options.path = "/tmp/milvus_test"; options.backend_uri = DBTestEnvironment::getURI(); + + if(options.backend_uri.empty()) { + throw std::exception(); + } + return options; } From 0f2f6bed6f1c270d2133b9a73c70035ded204a62 Mon Sep 17 00:00:00 2001 From: starlord Date: Sun, 30 Jun 2019 17:05:46 +0800 Subject: [PATCH 045/189] avoid unitest hang Former-commit-id: 3913dca183379d1cb658fc905c6a11d2fdabddcb --- .../db/{MySQLMetaImpl_test.cpp => mysql_meta_test.cpp} | 5 +++++ 1 file changed, 5 insertions(+) rename cpp/unittest/db/{MySQLMetaImpl_test.cpp => mysql_meta_test.cpp} (98%) diff --git a/cpp/unittest/db/MySQLMetaImpl_test.cpp b/cpp/unittest/db/mysql_meta_test.cpp similarity index 98% rename from cpp/unittest/db/MySQLMetaImpl_test.cpp rename to cpp/unittest/db/mysql_meta_test.cpp index a370fd1cd9..436086acb3 100644 --- a/cpp/unittest/db/MySQLMetaImpl_test.cpp +++ b/cpp/unittest/db/mysql_meta_test.cpp @@ -502,6 +502,11 @@ TEST_F(MySQLTest, TABLE_FILES_TEST) { ASSERT_EQ(dated_files[table_file.date_].size(), to_index_files_cnt+raw_files_cnt+index_files_cnt); + status = impl.FilesToSearch(table_id, meta::DatesT(), dated_files); + ASSERT_TRUE(status.ok()); + ASSERT_EQ(dated_files[table_file.date_].size(), + to_index_files_cnt+raw_files_cnt+index_files_cnt); + status = impl.DropAll(); ASSERT_TRUE(status.ok()); } From 5f485d7eaa71df41fcaa5a5bc70fc7f67e17e8fe Mon Sep 17 00:00:00 2001 From: starlord Date: Sun, 30 Jun 2019 18:53:43 +0800 Subject: [PATCH 046/189] refine unitest code Former-commit-id: e7206cd997cff9d4cdab8cf531438468370ebaf2 --- cpp/unittest/db/db_tests.cpp | 258 +------------------------- cpp/unittest/db/mysql_db_test.cpp | 293 ++++++++++++++++++++++++++++++ 2 files changed, 299 insertions(+), 252 deletions(-) create mode 100644 cpp/unittest/db/mysql_db_test.cpp diff --git a/cpp/unittest/db/db_tests.cpp b/cpp/unittest/db/db_tests.cpp index 8e50b7403b..d505320e86 100644 --- a/cpp/unittest/db/db_tests.cpp +++ b/cpp/unittest/db/db_tests.cpp @@ -20,6 +20,8 @@ namespace { static const std::string TABLE_NAME = "test_group"; static constexpr int64_t TABLE_DIM = 256; + static constexpr int64_t VECTOR_COUNT = 250000; + static constexpr int64_t INSERT_LOOP = 100000; engine::meta::TableSchema BuildTableSchema() { engine::meta::TableSchema table_info; @@ -144,7 +146,7 @@ TEST_F(DBTest, DB_TEST) { } }); - int loop = 100000; + int loop = INSERT_LOOP; for (auto i=0; i xb(nb*TABLE_DIM); @@ -246,7 +248,7 @@ TEST_F(DBTest2, ARHIVE_DISK_CHECK) { std::vector xb; BuildVectors(nb, xb); - int loop = 100000; + int loop = INSERT_LOOP; for (auto i=0; iInsertVectors(TABLE_NAME, nb, xb.data(), vector_ids); std::this_thread::sleep_for(std::chrono::microseconds(1)); @@ -277,7 +279,7 @@ TEST_F(DBTest2, DELETE_TEST) { uint64_t size; db_->Size(size); - int64_t nb = 100000; + int64_t nb = INSERT_LOOP; std::vector xb; BuildVectors(nb, xb); @@ -293,251 +295,3 @@ TEST_F(DBTest2, DELETE_TEST) { ASSERT_TRUE(stat.ok()); ASSERT_FALSE(boost::filesystem::exists(table_info_get.location_)); }; - -TEST_F(MySQLDBTest, DB_TEST) { - - auto options = GetOptions(); - auto db_ = engine::DBFactory::Build(options); - - engine::meta::TableSchema table_info = BuildTableSchema(); - engine::Status stat = db_->CreateTable(table_info); - - engine::meta::TableSchema table_info_get; - table_info_get.table_id_ = TABLE_NAME; - stat = db_->DescribeTable(table_info_get); - ASSERT_STATS(stat); - ASSERT_EQ(table_info_get.dimension_, TABLE_DIM); - - engine::IDNumbers vector_ids; - engine::IDNumbers target_ids; - - int64_t nb = 50; - std::vector xb; - BuildVectors(nb, xb); - - int64_t qb = 5; - std::vector qxb; - BuildVectors(qb, qxb); - - std::thread search([&]() { - engine::QueryResults results; - int k = 10; - std::this_thread::sleep_for(std::chrono::seconds(2)); - - INIT_TIMER; - std::stringstream ss; - uint64_t count = 0; - uint64_t prev_count = 0; - - for (auto j=0; j<10; ++j) { - ss.str(""); - db_->Size(count); - prev_count = count; - - START_TIMER; - stat = db_->Query(TABLE_NAME, k, qb, qxb.data(), results); - ss << "Search " << j << " With Size " << count/engine::meta::M << " M"; - STOP_TIMER(ss.str()); - - ASSERT_STATS(stat); - for (auto k=0; k= prev_count); - std::this_thread::sleep_for(std::chrono::seconds(1)); - } - }); - - int loop = 100000; - - for (auto i=0; iInsertVectors(TABLE_NAME, qb, qxb.data(), target_ids); - ASSERT_EQ(target_ids.size(), qb); - } else { - db_->InsertVectors(TABLE_NAME, nb, xb.data(), vector_ids); - } - std::this_thread::sleep_for(std::chrono::microseconds(1)); - } - - search.join(); - - delete db_; - - auto dummyDB = engine::DBFactory::Build(options); - dummyDB->DropAll(); - delete dummyDB; -}; - -TEST_F(MySQLDBTest, SEARCH_TEST) { - auto options = GetOptions(); - auto db_ = engine::DBFactory::Build(options); - - engine::meta::TableSchema table_info = BuildTableSchema(); - engine::Status stat = db_->CreateTable(table_info); - - engine::meta::TableSchema table_info_get; - table_info_get.table_id_ = TABLE_NAME; - stat = db_->DescribeTable(table_info_get); - ASSERT_STATS(stat); - ASSERT_EQ(table_info_get.dimension_, TABLE_DIM); - - // prepare raw data - size_t nb = 250000; - size_t nq = 10; - size_t k = 5; - std::vector xb(nb*TABLE_DIM); - std::vector xq(nq*TABLE_DIM); - std::vector ids(nb); - - std::random_device rd; - std::mt19937 gen(rd()); - std::uniform_real_distribution<> dis_xt(-1.0, 1.0); - for (size_t i = 0; i < nb*TABLE_DIM; i++) { - xb[i] = dis_xt(gen); - if (i < nb){ - ids[i] = i; - } - } - for (size_t i = 0; i < nq*TABLE_DIM; i++) { - xq[i] = dis_xt(gen); - } - - // result data - //std::vector nns_gt(k*nq); - std::vector nns(k*nq); // nns = nearst neg search - //std::vector dis_gt(k*nq); - std::vector dis(k*nq); - - // insert data - const int batch_size = 100; - for (int j = 0; j < nb / batch_size; ++j) { - stat = db_->InsertVectors(TABLE_NAME, batch_size, xb.data()+batch_size*j*TABLE_DIM, ids); - if (j == 200){ sleep(1);} - ASSERT_STATS(stat); - } - - sleep(2); // wait until build index finish - - engine::QueryResults results; - stat = db_->Query(TABLE_NAME, k, nq, xq.data(), results); - ASSERT_STATS(stat); - - delete db_; - - auto dummyDB = engine::DBFactory::Build(options); - dummyDB->DropAll(); - delete dummyDB; - - // TODO(linxj): add groundTruth assert -}; - -TEST_F(MySQLDBTest, ARHIVE_DISK_CHECK) { - - auto options = GetOptions(); - options.meta.archive_conf = engine::ArchiveConf("delete", "disk:1"); - auto db_ = engine::DBFactory::Build(options); - - engine::meta::TableSchema table_info = BuildTableSchema(); - engine::Status stat = db_->CreateTable(table_info); - - std::vector table_schema_array; - stat = db_->AllTables(table_schema_array); - ASSERT_STATS(stat); - bool bfound = false; - for(auto& schema : table_schema_array) { - if(schema.table_id_ == TABLE_NAME) { - bfound = true; - break; - } - } - ASSERT_TRUE(bfound); - - engine::meta::TableSchema table_info_get; - table_info_get.table_id_ = TABLE_NAME; - stat = db_->DescribeTable(table_info_get); - ASSERT_STATS(stat); - ASSERT_EQ(table_info_get.dimension_, TABLE_DIM); - - engine::IDNumbers vector_ids; - engine::IDNumbers target_ids; - - uint64_t size; - db_->Size(size); - - int64_t nb = 10; - std::vector xb; - BuildVectors(nb, xb); - - int loop = 100000; - for (auto i=0; iInsertVectors(TABLE_NAME, nb, xb.data(), vector_ids); - std::this_thread::sleep_for(std::chrono::microseconds(1)); - } - - std::this_thread::sleep_for(std::chrono::seconds(1)); - - db_->Size(size); - LOG(DEBUG) << "size=" << size; - ASSERT_LE(size, 1 * engine::meta::G); - - delete db_; - - auto dummyDB = engine::DBFactory::Build(options); - dummyDB->DropAll(); - delete dummyDB; -}; - -TEST_F(MySQLDBTest, DELETE_TEST) { - - auto options = GetOptions(); - options.meta.archive_conf = engine::ArchiveConf("delete", "disk:1"); - auto db_ = engine::DBFactory::Build(options); - - engine::meta::TableSchema table_info = BuildTableSchema(); - engine::Status stat = db_->CreateTable(table_info); -// std::cout << stat.ToString() << std::endl; - - engine::meta::TableSchema table_info_get; - table_info_get.table_id_ = TABLE_NAME; - stat = db_->DescribeTable(table_info_get); - ASSERT_STATS(stat); - -// std::cout << "location: " << table_info_get.location_ << std::endl; - ASSERT_TRUE(boost::filesystem::exists(table_info_get.location_)); - - engine::IDNumbers vector_ids; - - uint64_t size; - db_->Size(size); - - int64_t nb = 100000; - std::vector xb; - BuildVectors(nb, xb); - - int loop = 20; - for (auto i=0; iInsertVectors(TABLE_NAME, nb, xb.data(), vector_ids); - std::this_thread::sleep_for(std::chrono::microseconds(1)); - } - - std::vector dates; - stat = db_->DeleteTable(TABLE_NAME, dates); -// std::cout << "5 sec start" << std::endl; - std::this_thread::sleep_for(std::chrono::seconds(5)); -// std::cout << "5 sec finish" << std::endl; - ASSERT_TRUE(stat.ok()); -// ASSERT_FALSE(boost::filesystem::exists(table_info_get.location_)); - - delete db_; - - auto dummyDB = engine::DBFactory::Build(options); - dummyDB->DropAll(); - delete dummyDB; -}; diff --git a/cpp/unittest/db/mysql_db_test.cpp b/cpp/unittest/db/mysql_db_test.cpp new file mode 100644 index 0000000000..db3c84751e --- /dev/null +++ b/cpp/unittest/db/mysql_db_test.cpp @@ -0,0 +1,293 @@ +//////////////////////////////////////////////////////////////////////////////// +// Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved +// Unauthorized copying of this file, via any medium is strictly prohibited. +// Proprietary and confidential. +//////////////////////////////////////////////////////////////////////////////// +#include +#include +#include +#include + +#include "utils.h" +#include "db/DB.h" +#include "db/DBImpl.h" +#include "db/MetaConsts.h" +#include "db/Factories.h" + +using namespace zilliz::milvus; + +namespace { + +static const std::string TABLE_NAME = "test_group"; +static constexpr int64_t TABLE_DIM = 256; +static constexpr int64_t VECTOR_COUNT = 250000; +static constexpr int64_t INSERT_LOOP = 100000; + +engine::meta::TableSchema BuildTableSchema() { + engine::meta::TableSchema table_info; + table_info.dimension_ = TABLE_DIM; + table_info.table_id_ = TABLE_NAME; + table_info.engine_type_ = (int)engine::EngineType::FAISS_IDMAP; + return table_info; +} + +void BuildVectors(int64_t n, std::vector& vectors) { + vectors.clear(); + vectors.resize(n*TABLE_DIM); + float* data = vectors.data(); + for(int i = 0; i < n; i++) { + for(int j = 0; j < TABLE_DIM; j++) data[TABLE_DIM * i + j] = drand48(); + data[TABLE_DIM * i] += i / 2000.; + } +} + +} + + +TEST_F(MySQLDBTest, DB_TEST) { + + auto options = GetOptions(); + auto db_ = engine::DBFactory::Build(options); + + engine::meta::TableSchema table_info = BuildTableSchema(); + engine::Status stat = db_->CreateTable(table_info); + + engine::meta::TableSchema table_info_get; + table_info_get.table_id_ = TABLE_NAME; + stat = db_->DescribeTable(table_info_get); + ASSERT_STATS(stat); + ASSERT_EQ(table_info_get.dimension_, TABLE_DIM); + + engine::IDNumbers vector_ids; + engine::IDNumbers target_ids; + + int64_t nb = 50; + std::vector xb; + BuildVectors(nb, xb); + + int64_t qb = 5; + std::vector qxb; + BuildVectors(qb, qxb); + + std::thread search([&]() { + engine::QueryResults results; + int k = 10; + std::this_thread::sleep_for(std::chrono::seconds(2)); + + INIT_TIMER; + std::stringstream ss; + uint64_t count = 0; + uint64_t prev_count = 0; + + for (auto j=0; j<10; ++j) { + ss.str(""); + db_->Size(count); + prev_count = count; + + START_TIMER; + stat = db_->Query(TABLE_NAME, k, qb, qxb.data(), results); + ss << "Search " << j << " With Size " << count/engine::meta::M << " M"; + STOP_TIMER(ss.str()); + + ASSERT_STATS(stat); + for (auto k=0; k= prev_count); + std::this_thread::sleep_for(std::chrono::seconds(1)); + } + }); + + int loop = INSERT_LOOP; + + for (auto i=0; iInsertVectors(TABLE_NAME, qb, qxb.data(), target_ids); + ASSERT_EQ(target_ids.size(), qb); + } else { + db_->InsertVectors(TABLE_NAME, nb, xb.data(), vector_ids); + } + std::this_thread::sleep_for(std::chrono::microseconds(1)); + } + + search.join(); + + delete db_; + + auto dummyDB = engine::DBFactory::Build(options); + dummyDB->DropAll(); + delete dummyDB; +}; + +TEST_F(MySQLDBTest, SEARCH_TEST) { + auto options = GetOptions(); + auto db_ = engine::DBFactory::Build(options); + + engine::meta::TableSchema table_info = BuildTableSchema(); + engine::Status stat = db_->CreateTable(table_info); + + engine::meta::TableSchema table_info_get; + table_info_get.table_id_ = TABLE_NAME; + stat = db_->DescribeTable(table_info_get); + ASSERT_STATS(stat); + ASSERT_EQ(table_info_get.dimension_, TABLE_DIM); + + // prepare raw data + size_t nb = VECTOR_COUNT; + size_t nq = 10; + size_t k = 5; + std::vector xb(nb*TABLE_DIM); + std::vector xq(nq*TABLE_DIM); + std::vector ids(nb); + + std::random_device rd; + std::mt19937 gen(rd()); + std::uniform_real_distribution<> dis_xt(-1.0, 1.0); + for (size_t i = 0; i < nb*TABLE_DIM; i++) { + xb[i] = dis_xt(gen); + if (i < nb){ + ids[i] = i; + } + } + for (size_t i = 0; i < nq*TABLE_DIM; i++) { + xq[i] = dis_xt(gen); + } + + // result data + //std::vector nns_gt(k*nq); + std::vector nns(k*nq); // nns = nearst neg search + //std::vector dis_gt(k*nq); + std::vector dis(k*nq); + + // insert data + const int batch_size = 100; + for (int j = 0; j < nb / batch_size; ++j) { + stat = db_->InsertVectors(TABLE_NAME, batch_size, xb.data()+batch_size*j*TABLE_DIM, ids); + if (j == 200){ sleep(1);} + ASSERT_STATS(stat); + } + + sleep(2); // wait until build index finish + + engine::QueryResults results; + stat = db_->Query(TABLE_NAME, k, nq, xq.data(), results); + ASSERT_STATS(stat); + + delete db_; + + auto dummyDB = engine::DBFactory::Build(options); + dummyDB->DropAll(); + delete dummyDB; + + // TODO(linxj): add groundTruth assert +}; + +TEST_F(MySQLDBTest, ARHIVE_DISK_CHECK) { + + auto options = GetOptions(); + options.meta.archive_conf = engine::ArchiveConf("delete", "disk:1"); + auto db_ = engine::DBFactory::Build(options); + + engine::meta::TableSchema table_info = BuildTableSchema(); + engine::Status stat = db_->CreateTable(table_info); + + std::vector table_schema_array; + stat = db_->AllTables(table_schema_array); + ASSERT_STATS(stat); + bool bfound = false; + for(auto& schema : table_schema_array) { + if(schema.table_id_ == TABLE_NAME) { + bfound = true; + break; + } + } + ASSERT_TRUE(bfound); + + engine::meta::TableSchema table_info_get; + table_info_get.table_id_ = TABLE_NAME; + stat = db_->DescribeTable(table_info_get); + ASSERT_STATS(stat); + ASSERT_EQ(table_info_get.dimension_, TABLE_DIM); + + engine::IDNumbers vector_ids; + engine::IDNumbers target_ids; + + uint64_t size; + db_->Size(size); + + int64_t nb = 10; + std::vector xb; + BuildVectors(nb, xb); + + int loop = INSERT_LOOP; + for (auto i=0; iInsertVectors(TABLE_NAME, nb, xb.data(), vector_ids); + std::this_thread::sleep_for(std::chrono::microseconds(1)); + } + + std::this_thread::sleep_for(std::chrono::seconds(1)); + + db_->Size(size); + LOG(DEBUG) << "size=" << size; + ASSERT_LE(size, 1 * engine::meta::G); + + delete db_; + + auto dummyDB = engine::DBFactory::Build(options); + dummyDB->DropAll(); + delete dummyDB; +}; + +TEST_F(MySQLDBTest, DELETE_TEST) { + + auto options = GetOptions(); + options.meta.archive_conf = engine::ArchiveConf("delete", "disk:1"); + auto db_ = engine::DBFactory::Build(options); + + engine::meta::TableSchema table_info = BuildTableSchema(); + engine::Status stat = db_->CreateTable(table_info); +// std::cout << stat.ToString() << std::endl; + + engine::meta::TableSchema table_info_get; + table_info_get.table_id_ = TABLE_NAME; + stat = db_->DescribeTable(table_info_get); + ASSERT_STATS(stat); + +// std::cout << "location: " << table_info_get.location_ << std::endl; + ASSERT_TRUE(boost::filesystem::exists(table_info_get.location_)); + + engine::IDNumbers vector_ids; + + uint64_t size; + db_->Size(size); + + int64_t nb = INSERT_LOOP; + std::vector xb; + BuildVectors(nb, xb); + + int loop = 20; + for (auto i=0; iInsertVectors(TABLE_NAME, nb, xb.data(), vector_ids); + std::this_thread::sleep_for(std::chrono::microseconds(1)); + } + + std::vector dates; + stat = db_->DeleteTable(TABLE_NAME, dates); +// std::cout << "5 sec start" << std::endl; + std::this_thread::sleep_for(std::chrono::seconds(5)); +// std::cout << "5 sec finish" << std::endl; + ASSERT_TRUE(stat.ok()); +// ASSERT_FALSE(boost::filesystem::exists(table_info_get.location_)); + + delete db_; + + auto dummyDB = engine::DBFactory::Build(options); + dummyDB->DropAll(); + delete dummyDB; +}; From 121fba6e10b34021fb950e5327697daa258fcd0c Mon Sep 17 00:00:00 2001 From: starlord Date: Sun, 30 Jun 2019 19:07:00 +0800 Subject: [PATCH 047/189] ignore c sdk Former-commit-id: b6e3d44be7bf3db02d855ac75d100aff76e7d255 --- cpp/src/CMakeLists.txt | 2 +- cpp/src/sdk/CMakeLists.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cpp/src/CMakeLists.txt b/cpp/src/CMakeLists.txt index 6c1bd12771..4cde99bb0d 100644 --- a/cpp/src/CMakeLists.txt +++ b/cpp/src/CMakeLists.txt @@ -189,4 +189,4 @@ install(FILES ${CMAKE_BINARY_DIR}/mysqlpp_ep-prefix/src/mysqlpp_ep/lib/${CMAKE_SHARED_LIBRARY_PREFIX}mysqlpp${CMAKE_SHARED_LIBRARY_SUFFIX}.3.2.4 DESTINATION bin/lib) #need to copy libmysqlpp.so -add_subdirectory(sdk) +#add_subdirectory(sdk) diff --git a/cpp/src/sdk/CMakeLists.txt b/cpp/src/sdk/CMakeLists.txt index a43f0b85de..b51c2d5e09 100644 --- a/cpp/src/sdk/CMakeLists.txt +++ b/cpp/src/sdk/CMakeLists.txt @@ -32,4 +32,4 @@ target_link_libraries(milvus_sdk add_subdirectory(examples) -install(TARGETS milvus_sdk DESTINATION bin) +install(TARGETS milvus_sdk DESTINATION lib) From 763194dc2be901e557986c171d74508ee20343e3 Mon Sep 17 00:00:00 2001 From: zhiru Date: Sun, 30 Jun 2019 19:00:53 +0800 Subject: [PATCH 048/189] update Former-commit-id: ac5140491b67a1fbbf87e61ef5194e39992796e4 --- cpp/coverage.sh | 2 ++ cpp/src/CMakeLists.txt | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/cpp/coverage.sh b/cpp/coverage.sh index 54cf480703..cc509b611b 100755 --- a/cpp/coverage.sh +++ b/cpp/coverage.sh @@ -1,5 +1,7 @@ #!/bin/bash +export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$(pwd)/milvus/lib + LCOV_CMD="lcov" LCOV_GEN_CMD="genhtml" diff --git a/cpp/src/CMakeLists.txt b/cpp/src/CMakeLists.txt index 4cde99bb0d..d0029d5175 100644 --- a/cpp/src/CMakeLists.txt +++ b/cpp/src/CMakeLists.txt @@ -187,6 +187,6 @@ install(FILES ${CMAKE_BINARY_DIR}/mysqlpp_ep-prefix/src/mysqlpp_ep/lib/${CMAKE_SHARED_LIBRARY_PREFIX}mysqlpp${CMAKE_SHARED_LIBRARY_SUFFIX} ${CMAKE_BINARY_DIR}/mysqlpp_ep-prefix/src/mysqlpp_ep/lib/${CMAKE_SHARED_LIBRARY_PREFIX}mysqlpp${CMAKE_SHARED_LIBRARY_SUFFIX}.3 ${CMAKE_BINARY_DIR}/mysqlpp_ep-prefix/src/mysqlpp_ep/lib/${CMAKE_SHARED_LIBRARY_PREFIX}mysqlpp${CMAKE_SHARED_LIBRARY_SUFFIX}.3.2.4 - DESTINATION bin/lib) #need to copy libmysqlpp.so + DESTINATION lib) #need to copy libmysqlpp.so #add_subdirectory(sdk) From 63a71e04cb3840540a097c9b8e1daccbc95beff1 Mon Sep 17 00:00:00 2001 From: zhiru Date: Mon, 1 Jul 2019 15:29:12 +0800 Subject: [PATCH 049/189] update Former-commit-id: 9690faeda806fb563ed3c2f7a104e9d77b63f92a --- cpp/conf/server_config.template | 1 + cpp/coverage.sh | 31 +++- cpp/src/db/MySQLConnectionPool.h | 4 + cpp/src/db/MySQLMetaImpl.cpp | 23 ++- cpp/unittest/db/mysql_meta_test.cpp | 253 ++++------------------------ 5 files changed, 85 insertions(+), 227 deletions(-) diff --git a/cpp/conf/server_config.template b/cpp/conf/server_config.template index 6b432056c5..6339b0c440 100644 --- a/cpp/conf/server_config.template +++ b/cpp/conf/server_config.template @@ -15,6 +15,7 @@ db_config: index_building_threshold: 1024 # index building trigger threshold, default: 1024, unit: MB archive_disk_threshold: 512 # triger archive action if storage size exceed this value, unit: GB archive_days_threshold: 30 # files older than x days will be archived, unit: day + sql_echo: on # print sql statement in debug log metric_config: is_startup: off # if monitoring start: on, off diff --git a/cpp/coverage.sh b/cpp/coverage.sh index cc509b611b..dd2b9418f5 100755 --- a/cpp/coverage.sh +++ b/cpp/coverage.sh @@ -1,7 +1,5 @@ #!/bin/bash -export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$(pwd)/milvus/lib - LCOV_CMD="lcov" LCOV_GEN_CMD="genhtml" @@ -14,6 +12,26 @@ DIR_LCOV_OUTPUT="lcov_out" DIR_GCNO="cmake_build" DIR_UNITTEST="milvus/bin" +MYSQL_USER_NAME=root +MYSQL_PASSWORD=Fantast1c +MYSQL_HOST='192.168.1.194' +MYSQL_PORT='3306' + +MYSQL_DB_NAME=`date +%s%N` + +function mysql_exc() +{ + cmd=$1 + mysql -h${MYSQL_HOST} -u${MYSQL_USER_NAME} -p${MYSQL_PASSWORD} -e "${cmd}" + if [ $? -ne 0 ]; then + echo "mysql $cmd run failed" + fi +} + +mysql_exc "CREATE DATABASE IF NOT EXISTS ${MYSQL_DB_NAME};" +mysql_exc "GRANT ALL PRIVILEGES ON ${MYSQL_DB_NAME}.* TO '${MYSQL_USER_NAME}'@'%';" +mysql_exc "FLUSH PRIVILEGES;" + # get baseline ${LCOV_CMD} -c -i -d ${DIR_GCNO} -o "${FILE_INFO_BASE}" if [ $? -ne 0 ]; then @@ -22,14 +40,13 @@ if [ $? -ne 0 ]; then fi for test in `ls ${DIR_UNITTEST}`; do + echo $test case ${test} in db_test) # set run args for db_test - args="mysql://root:Fantast1c@192.168.1.194:3306/test" - ;; + args="mysql://${MYSQL_USER_NAME}:${MYSQL_PASSWORD}@${MYSQL_HOST}:${MYSQL_PORT}/${MYSQL_DB_NAME}" *_test) args="" - ;; esac # run unittest ./${DIR_UNITTEST}/${test} "${args}" @@ -38,6 +55,8 @@ for test in `ls ${DIR_UNITTEST}`; do fi done +mysql_exc "DROP DATABASE IF EXISTS ${MYSQL_DB_NAME};" + # gen test converage ${LCOV_CMD} -d ${DIR_GCNO} -o "${FILE_INFO_MILVUS}" -c # merge coverage @@ -50,4 +69,4 @@ ${LCOV_CMD} -r "${FILE_INFO_OUTPUT}" -o "${FILE_INFO_OUTPUT_NEW}" \ "*/cmake_build/*_ep-prefix/*" \ # gen html report -${LCOV_GEN_CMD} "${FILE_INFO_OUTPUT_NEW}" --output-directory ${DIR_LCOV_OUTPUT}/ +${LCOV_GEN_CMD} "${FILE_INFO_OUTPUT_NEW}" --output-directory ${DIR_LCOV_OUTPUT}/ \ No newline at end of file diff --git a/cpp/src/db/MySQLConnectionPool.h b/cpp/src/db/MySQLConnectionPool.h index 0978d77a7c..143e68be06 100644 --- a/cpp/src/db/MySQLConnectionPool.h +++ b/cpp/src/db/MySQLConnectionPool.h @@ -64,6 +64,10 @@ public: max_idle_time_ = max_idle; } + std::string getDB() { + return db_; + } + protected: // Superclass overrides diff --git a/cpp/src/db/MySQLMetaImpl.cpp b/cpp/src/db/MySQLMetaImpl.cpp index 4d5fc00050..a64cf4cb6d 100644 --- a/cpp/src/db/MySQLMetaImpl.cpp +++ b/cpp/src/db/MySQLMetaImpl.cpp @@ -1733,17 +1733,30 @@ namespace meta { // ENGINE_LOG_WARNING << "MySQLMetaImpl::CleanUp: connection in use = " << mysql_connection_pool_->getConnectionsInUse(); // } - ENGINE_LOG_DEBUG << "Remove table file type as NEW"; Query cleanUpQuery = connectionPtr->query(); - cleanUpQuery << "DELETE FROM TableFiles WHERE file_type = " << std::to_string(TableFileSchema::NEW) << ";"; + cleanUpQuery << "SELECT table_name " << + "FROM information_schema.tables " << + "WHERE table_schema = " << quote << mysql_connection_pool_->getDB() << quote << " " << + "AND table_name = " << quote << "TableFiles" << quote << ";"; if (options_.sql_echo) { ENGINE_LOG_DEBUG << "MySQLMetaImpl::CleanUp: " << cleanUpQuery.str(); } - if (!cleanUpQuery.exec()) { - ENGINE_LOG_ERROR << "QUERY ERROR WHEN CLEANING UP FILES"; - return Status::DBTransactionError("Clean up Error", cleanUpQuery.error()); + StoreQueryResult res = cleanUpQuery.store(); + assert(res); + if (!res.empty()) { + ENGINE_LOG_DEBUG << "Remove table file type as NEW"; + cleanUpQuery << "DELETE FROM TableFiles WHERE file_type = " << std::to_string(TableFileSchema::NEW) << ";"; + + if (options_.sql_echo) { + ENGINE_LOG_DEBUG << "MySQLMetaImpl::CleanUp: " << cleanUpQuery.str(); + } + + if (!cleanUpQuery.exec()) { + ENGINE_LOG_ERROR << "QUERY ERROR WHEN CLEANING UP FILES"; + return Status::DBTransactionError("Clean up Error", cleanUpQuery.error()); + } } } catch (const BadQuery& er) { diff --git a/cpp/unittest/db/mysql_meta_test.cpp b/cpp/unittest/db/mysql_meta_test.cpp index 436086acb3..76d7846362 100644 --- a/cpp/unittest/db/mysql_meta_test.cpp +++ b/cpp/unittest/db/mysql_meta_test.cpp @@ -21,184 +21,7 @@ using namespace zilliz::milvus::engine; -//TEST_F(MySQLTest, InitializeTest) { -// DBMetaOptions options; -// //dialect+driver://username:password@host:port/database -// options.backend_uri = "mysql://root:1234@:/test"; -// meta::MySQLMetaImpl impl(options); -// auto status = impl.Initialize(); -// std::cout << status.ToString() << std::endl; -// ASSERT_TRUE(status.ok()); -//} - -TEST_F(MySQLTest, core) { - DBMetaOptions options; -// //dialect+driver://username:password@host:port/database -// options.backend_uri = "mysql://root:1234@:/test"; -// options.path = "/tmp/vecwise_test"; - try { - options = getDBMetaOptions(); - } catch(std::exception& ex) { - ASSERT_TRUE(false); - return; - } - - int mode = Options::MODE::SINGLE; - meta::MySQLMetaImpl impl(options, mode); -// auto status = impl.Initialize(); -// ASSERT_TRUE(status.ok()); - - meta::TableSchema schema1; - schema1.table_id_ = "test1"; - schema1.dimension_ = 123; - - auto status = impl.CreateTable(schema1); -// std::cout << status.ToString() << std::endl; - ASSERT_TRUE(status.ok()); - - meta::TableSchema schema2; - schema2.table_id_ = "test2"; - schema2.dimension_ = 321; - status = impl.CreateTable(schema2); -// std::cout << status.ToString() << std::endl; - ASSERT_TRUE(status.ok()); - - status = impl.CreateTable(schema2); -// std::cout << status.ToString() << std::endl; -// ASSERT_THROW(impl.CreateTable(schema), mysqlpp::BadQuery); - ASSERT_TRUE(status.ok()); - - status = impl.DeleteTable(schema2.table_id_); -// std::cout << status.ToString() << std::endl; - ASSERT_TRUE(status.ok()); - - size_t id1 = schema1.id_; - long created_on1 = schema1.created_on_; - status = impl.DescribeTable(schema1); - ASSERT_TRUE(status.ok()); - ASSERT_EQ(schema1.id_, id1); - ASSERT_EQ(schema1.table_id_, "test1"); - ASSERT_EQ(schema1.created_on_, created_on1); - ASSERT_EQ(schema1.files_cnt_, 0); - ASSERT_EQ(schema1.engine_type_, 1); - ASSERT_EQ(schema1.store_raw_data_, false); - - bool check; - status = impl.HasTable("test1", check); - ASSERT_TRUE(status.ok()); - ASSERT_EQ(check, true); - - std::vector table_schema_array; - status = impl.AllTables(table_schema_array); - ASSERT_TRUE(status.ok()); - ASSERT_EQ(table_schema_array.size(), 1); - meta::TableSchema resultSchema = table_schema_array[0]; - ASSERT_EQ(resultSchema.id_, id1); - ASSERT_EQ(resultSchema.table_id_, "test1"); - ASSERT_EQ(resultSchema.dimension_, 123); - ASSERT_EQ(resultSchema.files_cnt_, 0); - ASSERT_EQ(resultSchema.engine_type_, 1); - ASSERT_EQ(resultSchema.store_raw_data_, false); - - meta::TableFileSchema tableFileSchema; - tableFileSchema.table_id_ = "test1"; - - status = impl.CreateTableFile(tableFileSchema); -// std::cout << status.ToString() << std::endl; - ASSERT_TRUE(status.ok()); - - tableFileSchema.file_type_ = meta::TableFileSchema::TO_INDEX; - status = impl.UpdateTableFile(tableFileSchema); -// std::cout << status.ToString() << std::endl; - ASSERT_TRUE(status.ok()); - - meta::TableFilesSchema filesToIndex; - status = impl.FilesToIndex(filesToIndex); - ASSERT_TRUE(status.ok()); - ASSERT_EQ(filesToIndex.size(), 1); - meta::TableFileSchema fileToIndex = filesToIndex[0]; - ASSERT_EQ(fileToIndex.table_id_, "test1"); - ASSERT_EQ(fileToIndex.dimension_, 123); - -// meta::TableFilesSchema filesToIndex; -// status = impl.FilesToIndex(filesToIndex); -// ASSERT_TRUE(status.ok()); -// ASSERT_EQ(filesToIndex.size(), 0); - - meta::DatesT partition; - partition.push_back(tableFileSchema.date_); - meta::DatePartionedTableFilesSchema filesToSearch; - status = impl.FilesToSearch(tableFileSchema.table_id_, partition, filesToSearch); - ASSERT_TRUE(status.ok()); - ASSERT_EQ(filesToSearch.size(), 1); - ASSERT_EQ(filesToSearch[tableFileSchema.date_].size(), 1); - meta::TableFileSchema fileToSearch = filesToSearch[tableFileSchema.date_][0]; - ASSERT_EQ(fileToSearch.table_id_, "test1"); - ASSERT_EQ(fileToSearch.dimension_, 123); - - tableFileSchema.file_type_ = meta::TableFileSchema::RAW; - status = impl.UpdateTableFile(tableFileSchema); - ASSERT_TRUE(status.ok()); - - meta::DatePartionedTableFilesSchema filesToMerge; - status = impl.FilesToMerge(tableFileSchema.table_id_, filesToMerge); -// std::cout << status.ToString() << std::endl; - ASSERT_TRUE(status.ok()); - ASSERT_EQ(filesToMerge.size(), 1); - ASSERT_EQ(filesToMerge[tableFileSchema.date_].size(), 1); - meta::TableFileSchema fileToMerge = filesToMerge[tableFileSchema.date_][0]; - ASSERT_EQ(fileToMerge.table_id_, "test1"); - ASSERT_EQ(fileToMerge.dimension_, 123); - - meta::TableFilesSchema resultTableFilesSchema; - std::vector ids; - ids.push_back(tableFileSchema.id_); - status = impl.GetTableFiles(tableFileSchema.table_id_, ids, resultTableFilesSchema); - ASSERT_TRUE(status.ok()); - ASSERT_EQ(resultTableFilesSchema.size(), 1); - meta::TableFileSchema resultTableFileSchema = resultTableFilesSchema[0]; -// ASSERT_EQ(resultTableFileSchema.id_, tableFileSchema.id_); - ASSERT_EQ(resultTableFileSchema.table_id_, tableFileSchema.table_id_); - ASSERT_EQ(resultTableFileSchema.file_id_, tableFileSchema.file_id_); - ASSERT_EQ(resultTableFileSchema.file_type_, tableFileSchema.file_type_); - ASSERT_EQ(resultTableFileSchema.size_, tableFileSchema.size_); - ASSERT_EQ(resultTableFileSchema.date_, tableFileSchema.date_); - ASSERT_EQ(resultTableFileSchema.engine_type_, tableFileSchema.engine_type_); - ASSERT_EQ(resultTableFileSchema.dimension_, tableFileSchema.dimension_); - - tableFileSchema.size_ = 234; - meta::TableSchema schema3; - schema3.table_id_ = "test3"; - schema3.dimension_ = 321; - status = impl.CreateTable(schema3); - ASSERT_TRUE(status.ok()); - meta::TableFileSchema tableFileSchema2; - tableFileSchema2.table_id_ = "test3"; - tableFileSchema2.size_ = 345; - status = impl.CreateTableFile(tableFileSchema2); - ASSERT_TRUE(status.ok()); - meta::TableFilesSchema filesToUpdate; - filesToUpdate.emplace_back(tableFileSchema); - filesToUpdate.emplace_back(tableFileSchema2); - status = impl.UpdateTableFile(tableFileSchema); - ASSERT_TRUE(status.ok()); - - uint64_t resultSize; - status = impl.Size(resultSize); -// std::cout << status.ToString() << std::endl; - ASSERT_TRUE(status.ok()); - ASSERT_EQ(resultSize, tableFileSchema.size_ + tableFileSchema2.size_); - - uint64_t countResult; - status = impl.Count(tableFileSchema.table_id_, countResult); - ASSERT_TRUE(status.ok()); - - status = impl.DropAll(); - ASSERT_TRUE(status.ok()); - -} - -TEST_F(MySQLTest, GROUP_TEST) { +TEST_F(MySQLTest, TABLE_TEST) { DBMetaOptions options; try { options = getDBMetaOptions(); @@ -210,38 +33,37 @@ TEST_F(MySQLTest, GROUP_TEST) { int mode = Options::MODE::SINGLE; meta::MySQLMetaImpl impl(options, mode); - auto table_id = "meta_test_group"; + auto table_id = "meta_test_table"; - meta::TableSchema group; - group.table_id_ = table_id; - auto status = impl.CreateTable(group); + meta::TableSchema table; + table.table_id_ = table_id; + auto status = impl.CreateTable(table); ASSERT_TRUE(status.ok()); - auto gid = group.id_; - group.id_ = -1; - status = impl.DescribeTable(group); + auto gid = table.id_; + table.id_ = -1; + status = impl.DescribeTable(table); ASSERT_TRUE(status.ok()); - ASSERT_EQ(group.id_, gid); - ASSERT_EQ(group.table_id_, table_id); + ASSERT_EQ(table.id_, gid); + ASSERT_EQ(table.table_id_, table_id); - group.table_id_ = "not_found"; - status = impl.DescribeTable(group); + table.table_id_ = "not_found"; + status = impl.DescribeTable(table); ASSERT_TRUE(!status.ok()); - group.table_id_ = table_id; - status = impl.CreateTable(group); + table.table_id_ = table_id; + status = impl.CreateTable(table); ASSERT_TRUE(status.ok()); - group.table_id_ = ""; - status = impl.CreateTable(group); + table.table_id_ = ""; + status = impl.CreateTable(table); ASSERT_TRUE(status.ok()); - status = impl.DropAll(); ASSERT_TRUE(status.ok()); } -TEST_F(MySQLTest, table_file_TEST) { +TEST_F(MySQLTest, TABLE_FILE_TEST) { DBMetaOptions options; try { options = getDBMetaOptions(); @@ -253,17 +75,16 @@ TEST_F(MySQLTest, table_file_TEST) { int mode = Options::MODE::SINGLE; meta::MySQLMetaImpl impl(options, mode); - auto table_id = "meta_test_group"; + auto table_id = "meta_test_table"; - meta::TableSchema group; - group.table_id_ = table_id; - group.dimension_ = 256; - auto status = impl.CreateTable(group); + meta::TableSchema table; + table.table_id_ = table_id; + table.dimension_ = 256; + auto status = impl.CreateTable(table); meta::TableFileSchema table_file; - table_file.table_id_ = group.table_id_; + table_file.table_id_ = table.table_id_; status = impl.CreateTableFile(table_file); -// std::cout << status.ToString() << std::endl; ASSERT_TRUE(status.ok()); ASSERT_EQ(table_file.file_type_, meta::TableFileSchema::NEW); @@ -332,15 +153,15 @@ TEST_F(MySQLTest, ARCHIVE_TEST_DAYS) { int mode = Options::MODE::SINGLE; meta::MySQLMetaImpl impl(options, mode); - auto table_id = "meta_test_group"; + auto table_id = "meta_test_table"; - meta::TableSchema group; - group.table_id_ = table_id; - auto status = impl.CreateTable(group); + meta::TableSchema table; + table.table_id_ = table_id; + auto status = impl.CreateTable(table); meta::TableFilesSchema files; meta::TableFileSchema table_file; - table_file.table_id_ = group.table_id_; + table_file.table_id_ = table.table_id_; auto cnt = 100; long ts = utils::GetMicroSecTimeStamp(); @@ -391,13 +212,13 @@ TEST_F(MySQLTest, ARCHIVE_TEST_DISK) { auto impl = meta::MySQLMetaImpl(options, mode); auto table_id = "meta_test_group"; - meta::TableSchema group; - group.table_id_ = table_id; - auto status = impl.CreateTable(group); + meta::TableSchema table; + table.table_id_ = table_id; + auto status = impl.CreateTable(table); meta::TableFilesSchema files; meta::TableFileSchema table_file; - table_file.table_id_ = group.table_id_; + table_file.table_id_ = table.table_id_; auto cnt = 10; auto each_size = 2UL; @@ -445,9 +266,9 @@ TEST_F(MySQLTest, TABLE_FILES_TEST) { auto table_id = "meta_test_group"; - meta::TableSchema group; - group.table_id_ = table_id; - auto status = impl.CreateTable(group); + meta::TableSchema table; + table.table_id_ = table_id; + auto status = impl.CreateTable(table); int new_files_cnt = 4; int raw_files_cnt = 5; @@ -455,7 +276,7 @@ TEST_F(MySQLTest, TABLE_FILES_TEST) { int index_files_cnt = 7; meta::TableFileSchema table_file; - table_file.table_id_ = group.table_id_; + table_file.table_id_ = table.table_id_; for (auto i=0; i Date: Mon, 1 Jul 2019 15:49:48 +0800 Subject: [PATCH 050/189] update Former-commit-id: 83ee5fefa2ad0646d7c5a61b0491affd3e45fbee --- cpp/coverage.sh | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/cpp/coverage.sh b/cpp/coverage.sh index dd2b9418f5..d302170ff8 100755 --- a/cpp/coverage.sh +++ b/cpp/coverage.sh @@ -11,13 +11,13 @@ DIR_LCOV_OUTPUT="lcov_out" DIR_GCNO="cmake_build" DIR_UNITTEST="milvus/bin" - + MYSQL_USER_NAME=root MYSQL_PASSWORD=Fantast1c MYSQL_HOST='192.168.1.194' MYSQL_PORT='3306' -MYSQL_DB_NAME=`date +%s%N` +MYSQL_DB_NAME=milvus_`date +%s%N` function mysql_exc() { @@ -45,8 +45,10 @@ for test in `ls ${DIR_UNITTEST}`; do db_test) # set run args for db_test args="mysql://${MYSQL_USER_NAME}:${MYSQL_PASSWORD}@${MYSQL_HOST}:${MYSQL_PORT}/${MYSQL_DB_NAME}" + ;; *_test) args="" + ;; esac # run unittest ./${DIR_UNITTEST}/${test} "${args}" From 11e26506b6f5526159b0731ef1d455ce8e6c91ad Mon Sep 17 00:00:00 2001 From: jinhai Date: Thu, 27 Jun 2019 17:43:19 +0800 Subject: [PATCH 051/189] MS-125 Update CHANGELOG.md for branch-0.3.1 Former-commit-id: bcea2800679f1ce42ce3b0b08fd4c814d75bbeba --- cpp/CHANGELOG.md | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/cpp/CHANGELOG.md b/cpp/CHANGELOG.md index eaac6a78ee..26c96b907b 100644 --- a/cpp/CHANGELOG.md +++ b/cpp/CHANGELOG.md @@ -2,7 +2,20 @@ Please mark all change in change log and use the ticket from JIRA. -# Milvus 0.3.0 (TBD) + +# Milvus 0.3.1 (2019-07-10) + +## Bug + +## Improvement + +## New Feature + +## Task + +- MS-125 - Create 0.3.1 release branch + +# Milvus 0.3.0 (2019-06-30) ## Bug - MS-104 - Fix unittest lcov execution error From 9a85a1746695c712df7eaa62b0a1a3df2ea0e710 Mon Sep 17 00:00:00 2001 From: jinhai Date: Thu, 27 Jun 2019 20:06:04 +0800 Subject: [PATCH 052/189] MS-125 Update version of CMakeLists.txt Former-commit-id: f833f8943711f1b4dc2a0fef92f24a5d0a7799ca --- cpp/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 116f30026d..947759f793 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -52,7 +52,7 @@ if(MILVUS_VERSION_MAJOR STREQUAL "" OR MILVUS_VERSION_MINOR STREQUAL "" OR MILVUS_VERSION_PATCH STREQUAL "") message(WARNING "Failed to determine Milvus version from git branch name") - set(MILVUS_VERSION "0.3.0") + set(MILVUS_VERSION "0.3.1") endif() message(STATUS "Build version = ${MILVUS_VERSION}") From aa840917e525bd5e05037634147001d2c93e07ec Mon Sep 17 00:00:00 2001 From: jinhai Date: Sun, 30 Jun 2019 13:02:42 +0800 Subject: [PATCH 053/189] MS-133 Change score to distance Former-commit-id: b66f48e93fbfe461776055f637b996712c8f5881 --- cpp/src/db/DBImpl.cpp | 343 ++++++++---------- cpp/src/db/DBImpl.h | 101 ++++-- cpp/src/db/Status.h | 31 +- .../sdk/examples/simple/src/ClientTest.cpp | 2 +- cpp/src/sdk/include/MilvusApi.h | 2 +- cpp/src/sdk/src/client/ClientProxy.cpp | 2 +- cpp/src/thrift/gen-cpp/milvus_types.cpp | 20 +- cpp/src/thrift/gen-cpp/milvus_types.h | 12 +- cpp/src/thrift/milvus.thrift | 2 +- 9 files changed, 253 insertions(+), 262 deletions(-) diff --git a/cpp/src/db/DBImpl.cpp b/cpp/src/db/DBImpl.cpp index a1bcc7ff44..46101dcf93 100644 --- a/cpp/src/db/DBImpl.cpp +++ b/cpp/src/db/DBImpl.cpp @@ -10,7 +10,7 @@ #include "Factories.h" #include "metrics/Metrics.h" #include "scheduler/TaskScheduler.h" -#include "scheduler/context/SearchContext.h" + #include "scheduler/context/DeleteContext.h" #include "utils/TimeRecorder.h" #include "MetaConsts.h" @@ -29,9 +29,9 @@ namespace engine { namespace { -static constexpr uint64_t METRIC_ACTION_INTERVAL = 1; -static constexpr uint64_t COMPACT_ACTION_INTERVAL = 1; -static constexpr uint64_t INDEX_ACTION_INTERVAL = 1; +constexpr uint64_t METRIC_ACTION_INTERVAL = 1; +constexpr uint64_t COMPACT_ACTION_INTERVAL = 1; +constexpr uint64_t INDEX_ACTION_INTERVAL = 1; void CollectInsertMetrics(double total_time, size_t n, bool succeed) { double avg_time = total_time / n; @@ -78,56 +78,6 @@ void CollectFileMetrics(int file_type, size_t file_size, double total_time) { } } } - -void CalcScore(uint64_t vector_count, - const float *vectors_data, - uint64_t dimension, - const SearchContext::ResultSet &result_src, - SearchContext::ResultSet &result_target) { - result_target.clear(); - if(result_src.empty()){ - return; - } - - server::TimeRecorder rc("Calculate Score"); - int vec_index = 0; - for(auto& result : result_src) { - const float * vec_data = vectors_data + vec_index*dimension; - double vec_len = 0; - for(uint64_t i = 0; i < dimension; i++) { - vec_len += vec_data[i]*vec_data[i]; - } - vec_index++; - - double max_score = 0.0; - for(auto& pair : result) { - if(max_score < pair.second) { - max_score = pair.second; - } - } - - //makesure socre is less than 100 - if(max_score > vec_len) { - vec_len = max_score; - } - - //avoid divided by zero - static constexpr double TOLERANCE = std::numeric_limits::epsilon(); - if(vec_len < TOLERANCE) { - vec_len = TOLERANCE; - } - - SearchContext::Id2ScoreMap score_array; - double vec_len_inverse = 1.0/vec_len; - for(auto& pair : result) { - score_array.push_back(std::make_pair(pair.first, (1 - pair.second*vec_len_inverse)*100.0)); - } - result_target.emplace_back(score_array); - } - - rc.Elapse("totally cost"); -} - } @@ -237,7 +187,7 @@ Status DBImpl::Query(const std::string& table_id, const std::vector meta::TableFileSchema table_file; table_file.table_id_ = table_id; std::string::size_type sz; - ids.push_back(std::stol(id, &sz)); + ids.push_back(std::stoul(id, &sz)); } meta::TableFilesSchema files_array; @@ -253,144 +203,140 @@ Status DBImpl::Query(const std::string& table_id, const std::vector return QueryAsync(table_id, files_array, k, nq, vectors, dates, results); } -//Status DBImpl::QuerySync(const std::string& table_id, uint64_t k, uint64_t nq, -// const float* vectors, const meta::DatesT& dates, QueryResults& results) { -// meta::DatePartionedTableFilesSchema files; -// auto status = meta_ptr_->FilesToSearch(table_id, dates, files); -// if (!status.ok()) { return status; } -// -// ENGINE_LOG_DEBUG << "Search DateT Size = " << files.size(); -// -// meta::TableFilesSchema index_files; -// meta::TableFilesSchema raw_files; -// for (auto &day_files : files) { -// for (auto &file : day_files.second) { -// file.file_type_ == meta::TableFileSchema::INDEX ? -// index_files.push_back(file) : raw_files.push_back(file); -// } -// } -// -// int dim = 0; -// if (!index_files.empty()) { -// dim = index_files[0].dimension_; -// } else if (!raw_files.empty()) { -// dim = raw_files[0].dimension_; -// } else { -// ENGINE_LOG_DEBUG << "no files to search"; -// return Status::OK(); -// } -// -// { -// // [{ids, distence}, ...] -// using SearchResult = std::pair, std::vector>; -// std::vector batchresult(nq); // allocate nq cells. -// -// auto cluster = [&](long *nns, float *dis, const int& k) -> void { -// for (int i = 0; i < nq; ++i) { -// auto f_begin = batchresult[i].first.cbegin(); -// auto s_begin = batchresult[i].second.cbegin(); -// batchresult[i].first.insert(f_begin, nns + i * k, nns + i * k + k); -// batchresult[i].second.insert(s_begin, dis + i * k, dis + i * k + k); -// } -// }; -// -// // Allocate Memory -// float *output_distence; -// long *output_ids; -// output_distence = (float *) malloc(k * nq * sizeof(float)); -// output_ids = (long *) malloc(k * nq * sizeof(long)); -// memset(output_distence, 0, k * nq * sizeof(float)); -// memset(output_ids, 0, k * nq * sizeof(long)); -// -// long search_set_size = 0; -// -// auto search_in_index = [&](meta::TableFilesSchema& file_vec) -> void { -// for (auto &file : file_vec) { -// -// ExecutionEnginePtr index = EngineFactory::Build(file.dimension_, file.location_, (EngineType)file.engine_type_); -// index->Load(); -// auto file_size = index->PhysicalSize(); -// search_set_size += file_size; -// -// ENGINE_LOG_DEBUG << "Search file_type " << file.file_type_ << " Of Size: " -// << file_size/(1024*1024) << " M"; -// -// int inner_k = index->Count() < k ? index->Count() : k; -// auto start_time = METRICS_NOW_TIME; -// index->Search(nq, vectors, inner_k, output_distence, output_ids); -// auto end_time = METRICS_NOW_TIME; -// auto total_time = METRICS_MICROSECONDS(start_time, end_time); -// CollectFileMetrics(file.file_type_, file_size, total_time); -// cluster(output_ids, output_distence, inner_k); // cluster to each query -// memset(output_distence, 0, k * nq * sizeof(float)); -// memset(output_ids, 0, k * nq * sizeof(long)); -// } -// }; -// -// auto topk_cpu = [](const std::vector &input_data, -// const int &k, -// float *output_distence, -// long *output_ids) -> void { -// std::map> inverted_table; -// for (int i = 0; i < input_data.size(); ++i) { -// if (inverted_table.count(input_data[i]) == 1) { -// auto& ori_vec = inverted_table[input_data[i]]; -// ori_vec.push_back(i); -// } -// else { -// inverted_table[input_data[i]] = std::vector{i}; -// } -// } -// -// int count = 0; -// for (auto &item : inverted_table){ -// if (count == k) break; -// for (auto &id : item.second){ -// output_distence[count] = item.first; -// output_ids[count] = id; -// if (++count == k) break; -// } -// } -// }; -// auto cluster_topk = [&]() -> void { -// QueryResult res; -// for (auto &result_pair : batchresult) { -// auto &dis = result_pair.second; -// auto &nns = result_pair.first; -// -// topk_cpu(dis, k, output_distence, output_ids); -// -// int inner_k = dis.size() < k ? dis.size() : k; -// for (int i = 0; i < inner_k; ++i) { -// res.emplace_back(std::make_pair(nns[output_ids[i]], output_distence[i])); // mapping -// } -// results.push_back(res); // append to result list -// res.clear(); -// memset(output_distence, 0, k * nq * sizeof(float)); -// memset(output_ids, 0, k * nq * sizeof(long)); -// } -// }; -// -// search_in_index(raw_files); -// search_in_index(index_files); -// -// ENGINE_LOG_DEBUG << "Search Overall Set Size = " << search_set_size << " M"; -// cluster_topk(); -// -// free(output_distence); -// free(output_ids); -// } -// -// if (results.empty()) { -// return Status::NotFound("Group " + table_id + ", search result not found!"); -// } -// -// QueryResults temp_results; -// CalcScore(nq, vectors, dim, results, temp_results); -// results.swap(temp_results); -// -// return Status::OK(); -//} +Status DBImpl::QuerySync(const std::string& table_id, uint64_t k, uint64_t nq, + const float* vectors, const meta::DatesT& dates, QueryResults& results) { + meta::DatePartionedTableFilesSchema files; + auto status = meta_ptr_->FilesToSearch(table_id, dates, files); + if (!status.ok()) { return status; } + + ENGINE_LOG_DEBUG << "Search DateT Size = " << files.size(); + + meta::TableFilesSchema index_files; + meta::TableFilesSchema raw_files; + for (auto &day_files : files) { + for (auto &file : day_files.second) { + file.file_type_ == meta::TableFileSchema::INDEX ? + index_files.push_back(file) : raw_files.push_back(file); + } + } + + int dim = 0; + if (!index_files.empty()) { + dim = index_files[0].dimension_; + } else if (!raw_files.empty()) { + dim = raw_files[0].dimension_; + } else { + ENGINE_LOG_DEBUG << "no files to search"; + return Status::OK(); + } + + { + // [{ids, distence}, ...] + using SearchResult = std::pair, std::vector>; + std::vector batchresult(nq); // allocate nq cells. + + auto cluster = [&](long *nns, float *dis, const int& k) -> void { + for (int i = 0; i < nq; ++i) { + auto f_begin = batchresult[i].first.cbegin(); + auto s_begin = batchresult[i].second.cbegin(); + batchresult[i].first.insert(f_begin, nns + i * k, nns + i * k + k); + batchresult[i].second.insert(s_begin, dis + i * k, dis + i * k + k); + } + }; + + // Allocate Memory + float *output_distence; + long *output_ids; + output_distence = (float *) malloc(k * nq * sizeof(float)); + output_ids = (long *) malloc(k * nq * sizeof(long)); + memset(output_distence, 0, k * nq * sizeof(float)); + memset(output_ids, 0, k * nq * sizeof(long)); + + long search_set_size = 0; + + auto search_in_index = [&](meta::TableFilesSchema& file_vec) -> void { + for (auto &file : file_vec) { + + ExecutionEnginePtr index = EngineFactory::Build(file.dimension_, file.location_, (EngineType)file.engine_type_); + index->Load(); + auto file_size = index->PhysicalSize(); + search_set_size += file_size; + + ENGINE_LOG_DEBUG << "Search file_type " << file.file_type_ << " Of Size: " + << file_size/(1024*1024) << " M"; + + int inner_k = index->Count() < k ? index->Count() : k; + auto start_time = METRICS_NOW_TIME; + index->Search(nq, vectors, inner_k, output_distence, output_ids); + auto end_time = METRICS_NOW_TIME; + auto total_time = METRICS_MICROSECONDS(start_time, end_time); + CollectFileMetrics(file.file_type_, file_size, total_time); + cluster(output_ids, output_distence, inner_k); // cluster to each query + memset(output_distence, 0, k * nq * sizeof(float)); + memset(output_ids, 0, k * nq * sizeof(long)); + } + }; + + auto topk_cpu = [](const std::vector &input_data, + const int &k, + float *output_distence, + long *output_ids) -> void { + std::map> inverted_table; + for (int i = 0; i < input_data.size(); ++i) { + if (inverted_table.count(input_data[i]) == 1) { + auto& ori_vec = inverted_table[input_data[i]]; + ori_vec.push_back(i); + } + else { + inverted_table[input_data[i]] = std::vector{i}; + } + } + + int count = 0; + for (auto &item : inverted_table){ + if (count == k) break; + for (auto &id : item.second){ + output_distence[count] = item.first; + output_ids[count] = id; + if (++count == k) break; + } + } + }; + auto cluster_topk = [&]() -> void { + QueryResult res; + for (auto &result_pair : batchresult) { + auto &dis = result_pair.second; + auto &nns = result_pair.first; + + topk_cpu(dis, k, output_distence, output_ids); + + int inner_k = dis.size() < k ? dis.size() : k; + for (int i = 0; i < inner_k; ++i) { + res.emplace_back(std::make_pair(nns[output_ids[i]], output_distence[i])); // mapping + } + results.push_back(res); // append to result list + res.clear(); + memset(output_distence, 0, k * nq * sizeof(float)); + memset(output_ids, 0, k * nq * sizeof(long)); + } + }; + + search_in_index(raw_files); + search_in_index(index_files); + + ENGINE_LOG_DEBUG << "Search Overall Set Size = " << search_set_size << " M"; + cluster_topk(); + + free(output_distence); + free(output_ids); + } + + if (results.empty()) { + return Status::NotFound("Group " + table_id + ", search result not found!"); + } + + return Status::OK(); +} Status DBImpl::QueryAsync(const std::string& table_id, const meta::TableFilesSchema& files, uint64_t k, uint64_t nq, const float* vectors, @@ -410,13 +356,8 @@ Status DBImpl::QueryAsync(const std::string& table_id, const meta::TableFilesSch context->WaitResult(); - //step 3: construct results, calculate score between 0 ~ 100 - auto& context_result = context->GetResult(); - meta::TableSchema table_schema; - table_schema.table_id_ = table_id; - meta_ptr_->DescribeTable(table_schema); - - CalcScore(context->nq(), context->vectors(), table_schema.dimension_, context_result, results); + //step 3: construct results + results = context->GetResult(); return Status::OK(); } @@ -589,7 +530,7 @@ void DBImpl::BackgroundCompaction(std::set table_ids) { // std::cout << "BackgroundCompaction: " << b_count << std::endl; Status status; - for (auto table_id : table_ids) { + for (auto& table_id : table_ids) { status = BackgroundMergeFiles(table_id); if (!status.ok()) { bg_error_ = status; diff --git a/cpp/src/db/DBImpl.h b/cpp/src/db/DBImpl.h index 43627cbace..cc632847c1 100644 --- a/cpp/src/db/DBImpl.h +++ b/cpp/src/db/DBImpl.h @@ -17,6 +17,8 @@ #include #include #include +#include "scheduler/context/SearchContext.h" + namespace zilliz { namespace milvus { @@ -25,49 +27,80 @@ namespace engine { class Env; namespace meta { - class Meta; +class Meta; } class DBImpl : public DB { -public: + public: using MetaPtr = meta::Meta::Ptr; using MemManagerPtr = typename MemManager::Ptr; - DBImpl(const Options& options); + explicit DBImpl(const Options &options); - virtual Status CreateTable(meta::TableSchema& table_schema) override; - virtual Status DeleteTable(const std::string& table_id, const meta::DatesT& dates) override; - virtual Status DescribeTable(meta::TableSchema& table_schema) override; - virtual Status HasTable(const std::string& table_id, bool& has_or_not) override; - virtual Status AllTables(std::vector& table_schema_array) override; - virtual Status GetTableRowCount(const std::string& table_id, uint64_t& row_count) override; + Status + CreateTable(meta::TableSchema &table_schema) override; - virtual Status InsertVectors(const std::string& table_id, - uint64_t n, const float* vectors, IDNumbers& vector_ids) override; + Status + DeleteTable(const std::string &table_id, const meta::DatesT &dates) override; - virtual Status Query(const std::string& table_id, uint64_t k, uint64_t nq, - const float* vectors, QueryResults& results) override; + Status + DescribeTable(meta::TableSchema &table_schema) override; - virtual Status Query(const std::string& table_id, uint64_t k, uint64_t nq, - const float* vectors, const meta::DatesT& dates, QueryResults& results) override; + Status + HasTable(const std::string &table_id, bool &has_or_not) override; - virtual Status Query(const std::string& table_id, const std::vector& file_ids, - uint64_t k, uint64_t nq, const float* vectors, - const meta::DatesT& dates, QueryResults& results) override; + Status + AllTables(std::vector &table_schema_array) override; - virtual Status DropAll() override; + Status + GetTableRowCount(const std::string &table_id, uint64_t &row_count) override; - virtual Status Size(uint64_t& result) override; + Status + InsertVectors(const std::string &table_id, uint64_t n, const float *vectors, IDNumbers &vector_ids) override; - virtual ~DBImpl(); + Status + Query(const std::string &table_id, uint64_t k, uint64_t nq, const float *vectors, QueryResults &results) override; -private: -// Status QuerySync(const std::string& table_id, uint64_t k, uint64_t nq, -// const float* vectors, const meta::DatesT& dates, QueryResults& results); + Status + Query(const std::string &table_id, + uint64_t k, + uint64_t nq, + const float *vectors, + const meta::DatesT &dates, + QueryResults &results) override; - Status QueryAsync(const std::string& table_id, const meta::TableFilesSchema& files, - uint64_t k, uint64_t nq, const float* vectors, - const meta::DatesT& dates, QueryResults& results); + Status + Query(const std::string &table_id, + const std::vector &file_ids, + uint64_t k, + uint64_t nq, + const float *vectors, + const meta::DatesT &dates, + QueryResults &results) override; + + Status DropAll() override; + + Status Size(uint64_t &result) override; + + ~DBImpl() override; + + private: + Status + QuerySync(const std::string &table_id, + uint64_t k, + uint64_t nq, + const float *vectors, + const meta::DatesT &dates, + QueryResults &results); + + Status + QueryAsync(const std::string &table_id, + const meta::TableFilesSchema &files, + uint64_t k, + uint64_t nq, + const float *vectors, + const meta::DatesT &dates, + QueryResults &results); void StartTimerTasks(); @@ -76,15 +109,19 @@ private: void StartMetricTask(); void StartCompactionTask(); - Status MergeFiles(const std::string& table_id, - const meta::DateT& date, - const meta::TableFilesSchema& files); - Status BackgroundMergeFiles(const std::string& table_id); + Status MergeFiles(const std::string &table_id, + const meta::DateT &date, + const meta::TableFilesSchema &files); + Status BackgroundMergeFiles(const std::string &table_id); void BackgroundCompaction(std::set table_ids); void StartBuildIndexTask(); void BackgroundBuildIndex(); - Status BuildIndex(const meta::TableFileSchema&); + + Status + BuildIndex(const meta::TableFileSchema &); + + private: const Options options_; diff --git a/cpp/src/db/Status.h b/cpp/src/db/Status.h index 52123eb8ce..8d58b96e85 100644 --- a/cpp/src/db/Status.h +++ b/cpp/src/db/Status.h @@ -15,33 +15,46 @@ namespace engine { class Status { public: Status() noexcept : state_(nullptr) {} + ~Status() { delete[] state_; } Status(const Status &rhs); - Status &operator=(const Status &rhs); + + Status & + operator=(const Status &rhs); Status(Status &&rhs) noexcept : state_(rhs.state_) { rhs.state_ = nullptr; } - Status &operator=(Status &&rhs_) noexcept; - static Status OK() { return Status(); } - static Status NotFound(const std::string &msg, const std::string &msg2 = "") { + Status & + operator=(Status &&rhs_) noexcept; + + static Status + OK() { return Status(); } + + static Status + NotFound(const std::string &msg, const std::string &msg2 = "") { return Status(kNotFound, msg, msg2); } - static Status Error(const std::string &msg, const std::string &msg2 = "") { + static Status + Error(const std::string &msg, const std::string &msg2 = "") { return Status(kError, msg, msg2); } - static Status InvalidDBPath(const std::string &msg, const std::string &msg2 = "") { + static Status + InvalidDBPath(const std::string &msg, const std::string &msg2 = "") { return Status(kInvalidDBPath, msg, msg2); } - static Status GroupError(const std::string &msg, const std::string &msg2 = "") { + static Status + GroupError(const std::string &msg, const std::string &msg2 = "") { return Status(kGroupError, msg, msg2); } - static Status DBTransactionError(const std::string &msg, const std::string &msg2 = "") { + static Status + DBTransactionError(const std::string &msg, const std::string &msg2 = "") { return Status(kDBTransactionError, msg, msg2); } - static Status AlreadyExist(const std::string &msg, const std::string &msg2 = "") { + static Status + AlreadyExist(const std::string &msg, const std::string &msg2 = "") { return Status(kAlreadyExist, msg, msg2); } diff --git a/cpp/src/sdk/examples/simple/src/ClientTest.cpp b/cpp/src/sdk/examples/simple/src/ClientTest.cpp index 19c764fd0a..b356128898 100644 --- a/cpp/src/sdk/examples/simple/src/ClientTest.cpp +++ b/cpp/src/sdk/examples/simple/src/ClientTest.cpp @@ -56,7 +56,7 @@ namespace { << std::to_string(result.query_result_arrays.size()) << " search result:" << std::endl; for(auto& item : result.query_result_arrays) { - std::cout << "\t" << std::to_string(item.id) << "\tscore:" << std::to_string(item.score); + std::cout << "\t" << std::to_string(item.id) << "\tdistance:" << std::to_string(item.distance); std::cout << std::endl; } } diff --git a/cpp/src/sdk/include/MilvusApi.h b/cpp/src/sdk/include/MilvusApi.h index 302871c48b..cb6d1f753e 100644 --- a/cpp/src/sdk/include/MilvusApi.h +++ b/cpp/src/sdk/include/MilvusApi.h @@ -59,7 +59,7 @@ struct RowRecord { */ struct QueryResult { int64_t id; ///< Output result - double score; ///< Vector similarity score: 0 ~ 100 + double distance; ///< Vector similarity distance }; /** diff --git a/cpp/src/sdk/src/client/ClientProxy.cpp b/cpp/src/sdk/src/client/ClientProxy.cpp index 35c774b52b..d26007cd0b 100644 --- a/cpp/src/sdk/src/client/ClientProxy.cpp +++ b/cpp/src/sdk/src/client/ClientProxy.cpp @@ -203,7 +203,7 @@ ClientProxy::SearchVector(const std::string &table_name, for(auto& thrift_query_result : thrift_topk_result.query_result_arrays) { QueryResult query_result; query_result.id = thrift_query_result.id; - query_result.score = thrift_query_result.score; + query_result.distance = thrift_query_result.distance; result.query_result_arrays.emplace_back(query_result); } diff --git a/cpp/src/thrift/gen-cpp/milvus_types.cpp b/cpp/src/thrift/gen-cpp/milvus_types.cpp index af77fda0ad..09aedec0bf 100644 --- a/cpp/src/thrift/gen-cpp/milvus_types.cpp +++ b/cpp/src/thrift/gen-cpp/milvus_types.cpp @@ -564,8 +564,8 @@ void QueryResult::__set_id(const int64_t val) { this->id = val; } -void QueryResult::__set_score(const double val) { - this->score = val; +void QueryResult::__set_distance(const double val) { + this->distance = val; } std::ostream& operator<<(std::ostream& out, const QueryResult& obj) { @@ -605,8 +605,8 @@ uint32_t QueryResult::read(::apache::thrift::protocol::TProtocol* iprot) { break; case 2: if (ftype == ::apache::thrift::protocol::T_DOUBLE) { - xfer += iprot->readDouble(this->score); - this->__isset.score = true; + xfer += iprot->readDouble(this->distance); + this->__isset.distance = true; } else { xfer += iprot->skip(ftype); } @@ -632,8 +632,8 @@ uint32_t QueryResult::write(::apache::thrift::protocol::TProtocol* oprot) const xfer += oprot->writeI64(this->id); xfer += oprot->writeFieldEnd(); - xfer += oprot->writeFieldBegin("score", ::apache::thrift::protocol::T_DOUBLE, 2); - xfer += oprot->writeDouble(this->score); + xfer += oprot->writeFieldBegin("distance", ::apache::thrift::protocol::T_DOUBLE, 2); + xfer += oprot->writeDouble(this->distance); xfer += oprot->writeFieldEnd(); xfer += oprot->writeFieldStop(); @@ -644,18 +644,18 @@ uint32_t QueryResult::write(::apache::thrift::protocol::TProtocol* oprot) const void swap(QueryResult &a, QueryResult &b) { using ::std::swap; swap(a.id, b.id); - swap(a.score, b.score); + swap(a.distance, b.distance); swap(a.__isset, b.__isset); } QueryResult::QueryResult(const QueryResult& other9) { id = other9.id; - score = other9.score; + distance = other9.distance; __isset = other9.__isset; } QueryResult& QueryResult::operator=(const QueryResult& other10) { id = other10.id; - score = other10.score; + distance = other10.distance; __isset = other10.__isset; return *this; } @@ -663,7 +663,7 @@ void QueryResult::printTo(std::ostream& out) const { using ::apache::thrift::to_string; out << "QueryResult("; out << "id=" << to_string(id); - out << ", " << "score=" << to_string(score); + out << ", " << "distance=" << to_string(distance); out << ")"; } diff --git a/cpp/src/thrift/gen-cpp/milvus_types.h b/cpp/src/thrift/gen-cpp/milvus_types.h index 8840e94fd9..34c5e084ce 100644 --- a/cpp/src/thrift/gen-cpp/milvus_types.h +++ b/cpp/src/thrift/gen-cpp/milvus_types.h @@ -256,9 +256,9 @@ void swap(RowRecord &a, RowRecord &b); std::ostream& operator<<(std::ostream& out, const RowRecord& obj); typedef struct _QueryResult__isset { - _QueryResult__isset() : id(false), score(false) {} + _QueryResult__isset() : id(false), distance(false) {} bool id :1; - bool score :1; + bool distance :1; } _QueryResult__isset; class QueryResult : public virtual ::apache::thrift::TBase { @@ -266,24 +266,24 @@ class QueryResult : public virtual ::apache::thrift::TBase { QueryResult(const QueryResult&); QueryResult& operator=(const QueryResult&); - QueryResult() : id(0), score(0) { + QueryResult() : id(0), distance(0) { } virtual ~QueryResult() throw(); int64_t id; - double score; + double distance; _QueryResult__isset __isset; void __set_id(const int64_t val); - void __set_score(const double val); + void __set_distance(const double val); bool operator == (const QueryResult & rhs) const { if (!(id == rhs.id)) return false; - if (!(score == rhs.score)) + if (!(distance == rhs.distance)) return false; return true; } diff --git a/cpp/src/thrift/milvus.thrift b/cpp/src/thrift/milvus.thrift index 88ba223f02..813026fdd8 100644 --- a/cpp/src/thrift/milvus.thrift +++ b/cpp/src/thrift/milvus.thrift @@ -73,7 +73,7 @@ struct RowRecord { */ struct QueryResult { 1: i64 id; ///< Output result - 2: double score; ///< Vector similarity score: 0 ~ 100 + 2: double distance; ///< Vector similarity distance } /** From 3d005212674a0c78ae167e604aaca7159330b780 Mon Sep 17 00:00:00 2001 From: jinhai Date: Sun, 30 Jun 2019 13:49:08 +0800 Subject: [PATCH 054/189] MS-133 Change score to distance Former-commit-id: 8e9e1c59187e046b3c5d85e708c27790140bc950 --- cpp/src/server/RequestTask.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/src/server/RequestTask.cpp b/cpp/src/server/RequestTask.cpp index bb0c437219..f312b7e605 100644 --- a/cpp/src/server/RequestTask.cpp +++ b/cpp/src/server/RequestTask.cpp @@ -514,7 +514,7 @@ ServerError SearchVectorTask::OnExecute() { for(auto& pair : result) { thrift::QueryResult thrift_result; thrift_result.__set_id(pair.first); - thrift_result.__set_score(pair.second); + thrift_result.__set_distance(pair.second); thrift_topk_result.query_result_arrays.emplace_back(thrift_result); } From c969a6bbb7aca3c9b93927be8674f8635cafe575 Mon Sep 17 00:00:00 2001 From: zhiru Date: Mon, 1 Jul 2019 17:11:33 +0800 Subject: [PATCH 055/189] test Former-commit-id: 99b4a0f5b9ba565d0fe3d2eb7241da45f71691e1 --- cpp/unittest/db/mysql_db_test.cpp | 47 ++++++++++++++++--------------- 1 file changed, 25 insertions(+), 22 deletions(-) diff --git a/cpp/unittest/db/mysql_db_test.cpp b/cpp/unittest/db/mysql_db_test.cpp index db3c84751e..14d9446d47 100644 --- a/cpp/unittest/db/mysql_db_test.cpp +++ b/cpp/unittest/db/mysql_db_test.cpp @@ -18,28 +18,28 @@ using namespace zilliz::milvus; namespace { -static const std::string TABLE_NAME = "test_group"; -static constexpr int64_t TABLE_DIM = 256; -static constexpr int64_t VECTOR_COUNT = 250000; -static constexpr int64_t INSERT_LOOP = 100000; + static const std::string TABLE_NAME = "test_group"; + static constexpr int64_t TABLE_DIM = 256; + static constexpr int64_t VECTOR_COUNT = 250000; + static constexpr int64_t INSERT_LOOP = 100000; -engine::meta::TableSchema BuildTableSchema() { - engine::meta::TableSchema table_info; - table_info.dimension_ = TABLE_DIM; - table_info.table_id_ = TABLE_NAME; - table_info.engine_type_ = (int)engine::EngineType::FAISS_IDMAP; - return table_info; -} - -void BuildVectors(int64_t n, std::vector& vectors) { - vectors.clear(); - vectors.resize(n*TABLE_DIM); - float* data = vectors.data(); - for(int i = 0; i < n; i++) { - for(int j = 0; j < TABLE_DIM; j++) data[TABLE_DIM * i + j] = drand48(); - data[TABLE_DIM * i] += i / 2000.; + engine::meta::TableSchema BuildTableSchema() { + engine::meta::TableSchema table_info; + table_info.dimension_ = TABLE_DIM; + table_info.table_id_ = TABLE_NAME; + table_info.engine_type_ = (int)engine::EngineType::FAISS_IDMAP; + return table_info; + } + + void BuildVectors(int64_t n, std::vector& vectors) { + vectors.clear(); + vectors.resize(n*TABLE_DIM); + float* data = vectors.data(); + for(int i = 0; i < n; i++) { + for(int j = 0; j < TABLE_DIM; j++) data[TABLE_DIM * i + j] = drand48(); + data[TABLE_DIM * i] += i / 2000.; + } } -} } @@ -72,7 +72,7 @@ TEST_F(MySQLDBTest, DB_TEST) { std::thread search([&]() { engine::QueryResults results; int k = 10; - std::this_thread::sleep_for(std::chrono::seconds(2)); + std::this_thread::sleep_for(std::chrono::seconds(3)); INIT_TIMER; std::stringstream ss; @@ -91,6 +91,7 @@ TEST_F(MySQLDBTest, DB_TEST) { ASSERT_STATS(stat); for (auto k=0; k= prev_count); - std::this_thread::sleep_for(std::chrono::seconds(1)); + std::this_thread::sleep_for(std::chrono::seconds(3)); } }); @@ -118,6 +119,8 @@ TEST_F(MySQLDBTest, DB_TEST) { search.join(); +// db_->DropAll(); + delete db_; auto dummyDB = engine::DBFactory::Build(options); From 4d83c24bd7785ca0382768f7e1c4d3ebae7df39d Mon Sep 17 00:00:00 2001 From: "zhiru.zhu" Date: Mon, 1 Jul 2019 15:38:35 +0800 Subject: [PATCH 056/189] Revert "Merge branch 'mysql-0.3.0' into 'branch-0.3.0'" This reverts merge request !143 Former-commit-id: 8099ff77584b0af7a8d39d0c6289e8e31b2120c8 --- cpp/conf/server_config.template | 1 - cpp/coverage.sh | 7 +- cpp/src/db/MySQLConnectionPool.h | 4 - cpp/src/db/MySQLMetaImpl.cpp | 23 +-- cpp/unittest/db/mysql_meta_test.cpp | 253 ++++++++++++++++++++++++---- 5 files changed, 224 insertions(+), 64 deletions(-) diff --git a/cpp/conf/server_config.template b/cpp/conf/server_config.template index 6339b0c440..6b432056c5 100644 --- a/cpp/conf/server_config.template +++ b/cpp/conf/server_config.template @@ -15,7 +15,6 @@ db_config: index_building_threshold: 1024 # index building trigger threshold, default: 1024, unit: MB archive_disk_threshold: 512 # triger archive action if storage size exceed this value, unit: GB archive_days_threshold: 30 # files older than x days will be archived, unit: day - sql_echo: on # print sql statement in debug log metric_config: is_startup: off # if monitoring start: on, off diff --git a/cpp/coverage.sh b/cpp/coverage.sh index d302170ff8..7638dad4e8 100755 --- a/cpp/coverage.sh +++ b/cpp/coverage.sh @@ -1,5 +1,7 @@ #!/bin/bash +export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$(pwd)/milvus/lib + LCOV_CMD="lcov" LCOV_GEN_CMD="genhtml" @@ -40,7 +42,6 @@ if [ $? -ne 0 ]; then fi for test in `ls ${DIR_UNITTEST}`; do - echo $test case ${test} in db_test) # set run args for db_test @@ -57,8 +58,6 @@ for test in `ls ${DIR_UNITTEST}`; do fi done -mysql_exc "DROP DATABASE IF EXISTS ${MYSQL_DB_NAME};" - # gen test converage ${LCOV_CMD} -d ${DIR_GCNO} -o "${FILE_INFO_MILVUS}" -c # merge coverage @@ -71,4 +70,4 @@ ${LCOV_CMD} -r "${FILE_INFO_OUTPUT}" -o "${FILE_INFO_OUTPUT_NEW}" \ "*/cmake_build/*_ep-prefix/*" \ # gen html report -${LCOV_GEN_CMD} "${FILE_INFO_OUTPUT_NEW}" --output-directory ${DIR_LCOV_OUTPUT}/ \ No newline at end of file +${LCOV_GEN_CMD} "${FILE_INFO_OUTPUT_NEW}" --output-directory ${DIR_LCOV_OUTPUT}/ diff --git a/cpp/src/db/MySQLConnectionPool.h b/cpp/src/db/MySQLConnectionPool.h index 143e68be06..0978d77a7c 100644 --- a/cpp/src/db/MySQLConnectionPool.h +++ b/cpp/src/db/MySQLConnectionPool.h @@ -64,10 +64,6 @@ public: max_idle_time_ = max_idle; } - std::string getDB() { - return db_; - } - protected: // Superclass overrides diff --git a/cpp/src/db/MySQLMetaImpl.cpp b/cpp/src/db/MySQLMetaImpl.cpp index a64cf4cb6d..4d5fc00050 100644 --- a/cpp/src/db/MySQLMetaImpl.cpp +++ b/cpp/src/db/MySQLMetaImpl.cpp @@ -1733,30 +1733,17 @@ namespace meta { // ENGINE_LOG_WARNING << "MySQLMetaImpl::CleanUp: connection in use = " << mysql_connection_pool_->getConnectionsInUse(); // } + ENGINE_LOG_DEBUG << "Remove table file type as NEW"; Query cleanUpQuery = connectionPtr->query(); - cleanUpQuery << "SELECT table_name " << - "FROM information_schema.tables " << - "WHERE table_schema = " << quote << mysql_connection_pool_->getDB() << quote << " " << - "AND table_name = " << quote << "TableFiles" << quote << ";"; + cleanUpQuery << "DELETE FROM TableFiles WHERE file_type = " << std::to_string(TableFileSchema::NEW) << ";"; if (options_.sql_echo) { ENGINE_LOG_DEBUG << "MySQLMetaImpl::CleanUp: " << cleanUpQuery.str(); } - StoreQueryResult res = cleanUpQuery.store(); - assert(res); - if (!res.empty()) { - ENGINE_LOG_DEBUG << "Remove table file type as NEW"; - cleanUpQuery << "DELETE FROM TableFiles WHERE file_type = " << std::to_string(TableFileSchema::NEW) << ";"; - - if (options_.sql_echo) { - ENGINE_LOG_DEBUG << "MySQLMetaImpl::CleanUp: " << cleanUpQuery.str(); - } - - if (!cleanUpQuery.exec()) { - ENGINE_LOG_ERROR << "QUERY ERROR WHEN CLEANING UP FILES"; - return Status::DBTransactionError("Clean up Error", cleanUpQuery.error()); - } + if (!cleanUpQuery.exec()) { + ENGINE_LOG_ERROR << "QUERY ERROR WHEN CLEANING UP FILES"; + return Status::DBTransactionError("Clean up Error", cleanUpQuery.error()); } } catch (const BadQuery& er) { diff --git a/cpp/unittest/db/mysql_meta_test.cpp b/cpp/unittest/db/mysql_meta_test.cpp index 76d7846362..436086acb3 100644 --- a/cpp/unittest/db/mysql_meta_test.cpp +++ b/cpp/unittest/db/mysql_meta_test.cpp @@ -21,7 +21,184 @@ using namespace zilliz::milvus::engine; -TEST_F(MySQLTest, TABLE_TEST) { +//TEST_F(MySQLTest, InitializeTest) { +// DBMetaOptions options; +// //dialect+driver://username:password@host:port/database +// options.backend_uri = "mysql://root:1234@:/test"; +// meta::MySQLMetaImpl impl(options); +// auto status = impl.Initialize(); +// std::cout << status.ToString() << std::endl; +// ASSERT_TRUE(status.ok()); +//} + +TEST_F(MySQLTest, core) { + DBMetaOptions options; +// //dialect+driver://username:password@host:port/database +// options.backend_uri = "mysql://root:1234@:/test"; +// options.path = "/tmp/vecwise_test"; + try { + options = getDBMetaOptions(); + } catch(std::exception& ex) { + ASSERT_TRUE(false); + return; + } + + int mode = Options::MODE::SINGLE; + meta::MySQLMetaImpl impl(options, mode); +// auto status = impl.Initialize(); +// ASSERT_TRUE(status.ok()); + + meta::TableSchema schema1; + schema1.table_id_ = "test1"; + schema1.dimension_ = 123; + + auto status = impl.CreateTable(schema1); +// std::cout << status.ToString() << std::endl; + ASSERT_TRUE(status.ok()); + + meta::TableSchema schema2; + schema2.table_id_ = "test2"; + schema2.dimension_ = 321; + status = impl.CreateTable(schema2); +// std::cout << status.ToString() << std::endl; + ASSERT_TRUE(status.ok()); + + status = impl.CreateTable(schema2); +// std::cout << status.ToString() << std::endl; +// ASSERT_THROW(impl.CreateTable(schema), mysqlpp::BadQuery); + ASSERT_TRUE(status.ok()); + + status = impl.DeleteTable(schema2.table_id_); +// std::cout << status.ToString() << std::endl; + ASSERT_TRUE(status.ok()); + + size_t id1 = schema1.id_; + long created_on1 = schema1.created_on_; + status = impl.DescribeTable(schema1); + ASSERT_TRUE(status.ok()); + ASSERT_EQ(schema1.id_, id1); + ASSERT_EQ(schema1.table_id_, "test1"); + ASSERT_EQ(schema1.created_on_, created_on1); + ASSERT_EQ(schema1.files_cnt_, 0); + ASSERT_EQ(schema1.engine_type_, 1); + ASSERT_EQ(schema1.store_raw_data_, false); + + bool check; + status = impl.HasTable("test1", check); + ASSERT_TRUE(status.ok()); + ASSERT_EQ(check, true); + + std::vector table_schema_array; + status = impl.AllTables(table_schema_array); + ASSERT_TRUE(status.ok()); + ASSERT_EQ(table_schema_array.size(), 1); + meta::TableSchema resultSchema = table_schema_array[0]; + ASSERT_EQ(resultSchema.id_, id1); + ASSERT_EQ(resultSchema.table_id_, "test1"); + ASSERT_EQ(resultSchema.dimension_, 123); + ASSERT_EQ(resultSchema.files_cnt_, 0); + ASSERT_EQ(resultSchema.engine_type_, 1); + ASSERT_EQ(resultSchema.store_raw_data_, false); + + meta::TableFileSchema tableFileSchema; + tableFileSchema.table_id_ = "test1"; + + status = impl.CreateTableFile(tableFileSchema); +// std::cout << status.ToString() << std::endl; + ASSERT_TRUE(status.ok()); + + tableFileSchema.file_type_ = meta::TableFileSchema::TO_INDEX; + status = impl.UpdateTableFile(tableFileSchema); +// std::cout << status.ToString() << std::endl; + ASSERT_TRUE(status.ok()); + + meta::TableFilesSchema filesToIndex; + status = impl.FilesToIndex(filesToIndex); + ASSERT_TRUE(status.ok()); + ASSERT_EQ(filesToIndex.size(), 1); + meta::TableFileSchema fileToIndex = filesToIndex[0]; + ASSERT_EQ(fileToIndex.table_id_, "test1"); + ASSERT_EQ(fileToIndex.dimension_, 123); + +// meta::TableFilesSchema filesToIndex; +// status = impl.FilesToIndex(filesToIndex); +// ASSERT_TRUE(status.ok()); +// ASSERT_EQ(filesToIndex.size(), 0); + + meta::DatesT partition; + partition.push_back(tableFileSchema.date_); + meta::DatePartionedTableFilesSchema filesToSearch; + status = impl.FilesToSearch(tableFileSchema.table_id_, partition, filesToSearch); + ASSERT_TRUE(status.ok()); + ASSERT_EQ(filesToSearch.size(), 1); + ASSERT_EQ(filesToSearch[tableFileSchema.date_].size(), 1); + meta::TableFileSchema fileToSearch = filesToSearch[tableFileSchema.date_][0]; + ASSERT_EQ(fileToSearch.table_id_, "test1"); + ASSERT_EQ(fileToSearch.dimension_, 123); + + tableFileSchema.file_type_ = meta::TableFileSchema::RAW; + status = impl.UpdateTableFile(tableFileSchema); + ASSERT_TRUE(status.ok()); + + meta::DatePartionedTableFilesSchema filesToMerge; + status = impl.FilesToMerge(tableFileSchema.table_id_, filesToMerge); +// std::cout << status.ToString() << std::endl; + ASSERT_TRUE(status.ok()); + ASSERT_EQ(filesToMerge.size(), 1); + ASSERT_EQ(filesToMerge[tableFileSchema.date_].size(), 1); + meta::TableFileSchema fileToMerge = filesToMerge[tableFileSchema.date_][0]; + ASSERT_EQ(fileToMerge.table_id_, "test1"); + ASSERT_EQ(fileToMerge.dimension_, 123); + + meta::TableFilesSchema resultTableFilesSchema; + std::vector ids; + ids.push_back(tableFileSchema.id_); + status = impl.GetTableFiles(tableFileSchema.table_id_, ids, resultTableFilesSchema); + ASSERT_TRUE(status.ok()); + ASSERT_EQ(resultTableFilesSchema.size(), 1); + meta::TableFileSchema resultTableFileSchema = resultTableFilesSchema[0]; +// ASSERT_EQ(resultTableFileSchema.id_, tableFileSchema.id_); + ASSERT_EQ(resultTableFileSchema.table_id_, tableFileSchema.table_id_); + ASSERT_EQ(resultTableFileSchema.file_id_, tableFileSchema.file_id_); + ASSERT_EQ(resultTableFileSchema.file_type_, tableFileSchema.file_type_); + ASSERT_EQ(resultTableFileSchema.size_, tableFileSchema.size_); + ASSERT_EQ(resultTableFileSchema.date_, tableFileSchema.date_); + ASSERT_EQ(resultTableFileSchema.engine_type_, tableFileSchema.engine_type_); + ASSERT_EQ(resultTableFileSchema.dimension_, tableFileSchema.dimension_); + + tableFileSchema.size_ = 234; + meta::TableSchema schema3; + schema3.table_id_ = "test3"; + schema3.dimension_ = 321; + status = impl.CreateTable(schema3); + ASSERT_TRUE(status.ok()); + meta::TableFileSchema tableFileSchema2; + tableFileSchema2.table_id_ = "test3"; + tableFileSchema2.size_ = 345; + status = impl.CreateTableFile(tableFileSchema2); + ASSERT_TRUE(status.ok()); + meta::TableFilesSchema filesToUpdate; + filesToUpdate.emplace_back(tableFileSchema); + filesToUpdate.emplace_back(tableFileSchema2); + status = impl.UpdateTableFile(tableFileSchema); + ASSERT_TRUE(status.ok()); + + uint64_t resultSize; + status = impl.Size(resultSize); +// std::cout << status.ToString() << std::endl; + ASSERT_TRUE(status.ok()); + ASSERT_EQ(resultSize, tableFileSchema.size_ + tableFileSchema2.size_); + + uint64_t countResult; + status = impl.Count(tableFileSchema.table_id_, countResult); + ASSERT_TRUE(status.ok()); + + status = impl.DropAll(); + ASSERT_TRUE(status.ok()); + +} + +TEST_F(MySQLTest, GROUP_TEST) { DBMetaOptions options; try { options = getDBMetaOptions(); @@ -33,37 +210,38 @@ TEST_F(MySQLTest, TABLE_TEST) { int mode = Options::MODE::SINGLE; meta::MySQLMetaImpl impl(options, mode); - auto table_id = "meta_test_table"; + auto table_id = "meta_test_group"; - meta::TableSchema table; - table.table_id_ = table_id; - auto status = impl.CreateTable(table); + meta::TableSchema group; + group.table_id_ = table_id; + auto status = impl.CreateTable(group); ASSERT_TRUE(status.ok()); - auto gid = table.id_; - table.id_ = -1; - status = impl.DescribeTable(table); + auto gid = group.id_; + group.id_ = -1; + status = impl.DescribeTable(group); ASSERT_TRUE(status.ok()); - ASSERT_EQ(table.id_, gid); - ASSERT_EQ(table.table_id_, table_id); + ASSERT_EQ(group.id_, gid); + ASSERT_EQ(group.table_id_, table_id); - table.table_id_ = "not_found"; - status = impl.DescribeTable(table); + group.table_id_ = "not_found"; + status = impl.DescribeTable(group); ASSERT_TRUE(!status.ok()); - table.table_id_ = table_id; - status = impl.CreateTable(table); + group.table_id_ = table_id; + status = impl.CreateTable(group); ASSERT_TRUE(status.ok()); - table.table_id_ = ""; - status = impl.CreateTable(table); + group.table_id_ = ""; + status = impl.CreateTable(group); ASSERT_TRUE(status.ok()); + status = impl.DropAll(); ASSERT_TRUE(status.ok()); } -TEST_F(MySQLTest, TABLE_FILE_TEST) { +TEST_F(MySQLTest, table_file_TEST) { DBMetaOptions options; try { options = getDBMetaOptions(); @@ -75,16 +253,17 @@ TEST_F(MySQLTest, TABLE_FILE_TEST) { int mode = Options::MODE::SINGLE; meta::MySQLMetaImpl impl(options, mode); - auto table_id = "meta_test_table"; + auto table_id = "meta_test_group"; - meta::TableSchema table; - table.table_id_ = table_id; - table.dimension_ = 256; - auto status = impl.CreateTable(table); + meta::TableSchema group; + group.table_id_ = table_id; + group.dimension_ = 256; + auto status = impl.CreateTable(group); meta::TableFileSchema table_file; - table_file.table_id_ = table.table_id_; + table_file.table_id_ = group.table_id_; status = impl.CreateTableFile(table_file); +// std::cout << status.ToString() << std::endl; ASSERT_TRUE(status.ok()); ASSERT_EQ(table_file.file_type_, meta::TableFileSchema::NEW); @@ -153,15 +332,15 @@ TEST_F(MySQLTest, ARCHIVE_TEST_DAYS) { int mode = Options::MODE::SINGLE; meta::MySQLMetaImpl impl(options, mode); - auto table_id = "meta_test_table"; + auto table_id = "meta_test_group"; - meta::TableSchema table; - table.table_id_ = table_id; - auto status = impl.CreateTable(table); + meta::TableSchema group; + group.table_id_ = table_id; + auto status = impl.CreateTable(group); meta::TableFilesSchema files; meta::TableFileSchema table_file; - table_file.table_id_ = table.table_id_; + table_file.table_id_ = group.table_id_; auto cnt = 100; long ts = utils::GetMicroSecTimeStamp(); @@ -212,13 +391,13 @@ TEST_F(MySQLTest, ARCHIVE_TEST_DISK) { auto impl = meta::MySQLMetaImpl(options, mode); auto table_id = "meta_test_group"; - meta::TableSchema table; - table.table_id_ = table_id; - auto status = impl.CreateTable(table); + meta::TableSchema group; + group.table_id_ = table_id; + auto status = impl.CreateTable(group); meta::TableFilesSchema files; meta::TableFileSchema table_file; - table_file.table_id_ = table.table_id_; + table_file.table_id_ = group.table_id_; auto cnt = 10; auto each_size = 2UL; @@ -266,9 +445,9 @@ TEST_F(MySQLTest, TABLE_FILES_TEST) { auto table_id = "meta_test_group"; - meta::TableSchema table; - table.table_id_ = table_id; - auto status = impl.CreateTable(table); + meta::TableSchema group; + group.table_id_ = table_id; + auto status = impl.CreateTable(group); int new_files_cnt = 4; int raw_files_cnt = 5; @@ -276,7 +455,7 @@ TEST_F(MySQLTest, TABLE_FILES_TEST) { int index_files_cnt = 7; meta::TableFileSchema table_file; - table_file.table_id_ = table.table_id_; + table_file.table_id_ = group.table_id_; for (auto i=0; i Date: Mon, 1 Jul 2019 19:49:42 +0800 Subject: [PATCH 057/189] update Former-commit-id: d9c2609d1e78c194cafb3a0db8d10be3c104ac1b --- cpp/conf/server_config.template | 1 + cpp/src/db/MySQLConnectionPool.h | 4 + cpp/src/db/MySQLMetaImpl.cpp | 23 ++- cpp/unittest/db/mysql_db_test.cpp | 4 +- cpp/unittest/db/mysql_meta_test.cpp | 253 ++++------------------------ 5 files changed, 61 insertions(+), 224 deletions(-) diff --git a/cpp/conf/server_config.template b/cpp/conf/server_config.template index 6b432056c5..6339b0c440 100644 --- a/cpp/conf/server_config.template +++ b/cpp/conf/server_config.template @@ -15,6 +15,7 @@ db_config: index_building_threshold: 1024 # index building trigger threshold, default: 1024, unit: MB archive_disk_threshold: 512 # triger archive action if storage size exceed this value, unit: GB archive_days_threshold: 30 # files older than x days will be archived, unit: day + sql_echo: on # print sql statement in debug log metric_config: is_startup: off # if monitoring start: on, off diff --git a/cpp/src/db/MySQLConnectionPool.h b/cpp/src/db/MySQLConnectionPool.h index 0978d77a7c..143e68be06 100644 --- a/cpp/src/db/MySQLConnectionPool.h +++ b/cpp/src/db/MySQLConnectionPool.h @@ -64,6 +64,10 @@ public: max_idle_time_ = max_idle; } + std::string getDB() { + return db_; + } + protected: // Superclass overrides diff --git a/cpp/src/db/MySQLMetaImpl.cpp b/cpp/src/db/MySQLMetaImpl.cpp index 4d5fc00050..2051c12f49 100644 --- a/cpp/src/db/MySQLMetaImpl.cpp +++ b/cpp/src/db/MySQLMetaImpl.cpp @@ -1733,17 +1733,30 @@ namespace meta { // ENGINE_LOG_WARNING << "MySQLMetaImpl::CleanUp: connection in use = " << mysql_connection_pool_->getConnectionsInUse(); // } - ENGINE_LOG_DEBUG << "Remove table file type as NEW"; Query cleanUpQuery = connectionPtr->query(); - cleanUpQuery << "DELETE FROM TableFiles WHERE file_type = " << std::to_string(TableFileSchema::NEW) << ";"; + cleanUpQuery << "SELECT table_name " << + "FROM information_schema.tables " << + "WHERE table_schema = " << quote << mysql_connection_pool_->getDB() << quote << " " << + "AND table_name = " << quote << "TableFiles" << quote << ";"; if (options_.sql_echo) { ENGINE_LOG_DEBUG << "MySQLMetaImpl::CleanUp: " << cleanUpQuery.str(); } - if (!cleanUpQuery.exec()) { - ENGINE_LOG_ERROR << "QUERY ERROR WHEN CLEANING UP FILES"; - return Status::DBTransactionError("Clean up Error", cleanUpQuery.error()); + StoreQueryResult res = cleanUpQuery.store(); + assert(res); + if (!res.empty()) { + ENGINE_LOG_DEBUG << "Remove table file type as NEW"; + cleanUpQuery << "DELETE FROM TableFiles WHERE file_type = " << std::to_string(TableFileSchema::NEW) << ";"; + + if (options_.sql_echo) { + ENGINE_LOG_DEBUG << "MySQLMetaImpl::CleanUp: " << cleanUpQuery.str(); + } + + if (!cleanUpQuery.exec()) { + ENGINE_LOG_ERROR << "QUERY ERROR WHEN CLEANING UP FILES"; + return Status::DBTransactionError("Clean up Error", cleanUpQuery.error()); + } } } catch (const BadQuery& er) { diff --git a/cpp/unittest/db/mysql_db_test.cpp b/cpp/unittest/db/mysql_db_test.cpp index 14d9446d47..85fe749460 100644 --- a/cpp/unittest/db/mysql_db_test.cpp +++ b/cpp/unittest/db/mysql_db_test.cpp @@ -72,7 +72,7 @@ TEST_F(MySQLDBTest, DB_TEST) { std::thread search([&]() { engine::QueryResults results; int k = 10; - std::this_thread::sleep_for(std::chrono::seconds(3)); + std::this_thread::sleep_for(std::chrono::seconds(5)); INIT_TIMER; std::stringstream ss; @@ -119,8 +119,6 @@ TEST_F(MySQLDBTest, DB_TEST) { search.join(); -// db_->DropAll(); - delete db_; auto dummyDB = engine::DBFactory::Build(options); diff --git a/cpp/unittest/db/mysql_meta_test.cpp b/cpp/unittest/db/mysql_meta_test.cpp index 436086acb3..76d7846362 100644 --- a/cpp/unittest/db/mysql_meta_test.cpp +++ b/cpp/unittest/db/mysql_meta_test.cpp @@ -21,184 +21,7 @@ using namespace zilliz::milvus::engine; -//TEST_F(MySQLTest, InitializeTest) { -// DBMetaOptions options; -// //dialect+driver://username:password@host:port/database -// options.backend_uri = "mysql://root:1234@:/test"; -// meta::MySQLMetaImpl impl(options); -// auto status = impl.Initialize(); -// std::cout << status.ToString() << std::endl; -// ASSERT_TRUE(status.ok()); -//} - -TEST_F(MySQLTest, core) { - DBMetaOptions options; -// //dialect+driver://username:password@host:port/database -// options.backend_uri = "mysql://root:1234@:/test"; -// options.path = "/tmp/vecwise_test"; - try { - options = getDBMetaOptions(); - } catch(std::exception& ex) { - ASSERT_TRUE(false); - return; - } - - int mode = Options::MODE::SINGLE; - meta::MySQLMetaImpl impl(options, mode); -// auto status = impl.Initialize(); -// ASSERT_TRUE(status.ok()); - - meta::TableSchema schema1; - schema1.table_id_ = "test1"; - schema1.dimension_ = 123; - - auto status = impl.CreateTable(schema1); -// std::cout << status.ToString() << std::endl; - ASSERT_TRUE(status.ok()); - - meta::TableSchema schema2; - schema2.table_id_ = "test2"; - schema2.dimension_ = 321; - status = impl.CreateTable(schema2); -// std::cout << status.ToString() << std::endl; - ASSERT_TRUE(status.ok()); - - status = impl.CreateTable(schema2); -// std::cout << status.ToString() << std::endl; -// ASSERT_THROW(impl.CreateTable(schema), mysqlpp::BadQuery); - ASSERT_TRUE(status.ok()); - - status = impl.DeleteTable(schema2.table_id_); -// std::cout << status.ToString() << std::endl; - ASSERT_TRUE(status.ok()); - - size_t id1 = schema1.id_; - long created_on1 = schema1.created_on_; - status = impl.DescribeTable(schema1); - ASSERT_TRUE(status.ok()); - ASSERT_EQ(schema1.id_, id1); - ASSERT_EQ(schema1.table_id_, "test1"); - ASSERT_EQ(schema1.created_on_, created_on1); - ASSERT_EQ(schema1.files_cnt_, 0); - ASSERT_EQ(schema1.engine_type_, 1); - ASSERT_EQ(schema1.store_raw_data_, false); - - bool check; - status = impl.HasTable("test1", check); - ASSERT_TRUE(status.ok()); - ASSERT_EQ(check, true); - - std::vector table_schema_array; - status = impl.AllTables(table_schema_array); - ASSERT_TRUE(status.ok()); - ASSERT_EQ(table_schema_array.size(), 1); - meta::TableSchema resultSchema = table_schema_array[0]; - ASSERT_EQ(resultSchema.id_, id1); - ASSERT_EQ(resultSchema.table_id_, "test1"); - ASSERT_EQ(resultSchema.dimension_, 123); - ASSERT_EQ(resultSchema.files_cnt_, 0); - ASSERT_EQ(resultSchema.engine_type_, 1); - ASSERT_EQ(resultSchema.store_raw_data_, false); - - meta::TableFileSchema tableFileSchema; - tableFileSchema.table_id_ = "test1"; - - status = impl.CreateTableFile(tableFileSchema); -// std::cout << status.ToString() << std::endl; - ASSERT_TRUE(status.ok()); - - tableFileSchema.file_type_ = meta::TableFileSchema::TO_INDEX; - status = impl.UpdateTableFile(tableFileSchema); -// std::cout << status.ToString() << std::endl; - ASSERT_TRUE(status.ok()); - - meta::TableFilesSchema filesToIndex; - status = impl.FilesToIndex(filesToIndex); - ASSERT_TRUE(status.ok()); - ASSERT_EQ(filesToIndex.size(), 1); - meta::TableFileSchema fileToIndex = filesToIndex[0]; - ASSERT_EQ(fileToIndex.table_id_, "test1"); - ASSERT_EQ(fileToIndex.dimension_, 123); - -// meta::TableFilesSchema filesToIndex; -// status = impl.FilesToIndex(filesToIndex); -// ASSERT_TRUE(status.ok()); -// ASSERT_EQ(filesToIndex.size(), 0); - - meta::DatesT partition; - partition.push_back(tableFileSchema.date_); - meta::DatePartionedTableFilesSchema filesToSearch; - status = impl.FilesToSearch(tableFileSchema.table_id_, partition, filesToSearch); - ASSERT_TRUE(status.ok()); - ASSERT_EQ(filesToSearch.size(), 1); - ASSERT_EQ(filesToSearch[tableFileSchema.date_].size(), 1); - meta::TableFileSchema fileToSearch = filesToSearch[tableFileSchema.date_][0]; - ASSERT_EQ(fileToSearch.table_id_, "test1"); - ASSERT_EQ(fileToSearch.dimension_, 123); - - tableFileSchema.file_type_ = meta::TableFileSchema::RAW; - status = impl.UpdateTableFile(tableFileSchema); - ASSERT_TRUE(status.ok()); - - meta::DatePartionedTableFilesSchema filesToMerge; - status = impl.FilesToMerge(tableFileSchema.table_id_, filesToMerge); -// std::cout << status.ToString() << std::endl; - ASSERT_TRUE(status.ok()); - ASSERT_EQ(filesToMerge.size(), 1); - ASSERT_EQ(filesToMerge[tableFileSchema.date_].size(), 1); - meta::TableFileSchema fileToMerge = filesToMerge[tableFileSchema.date_][0]; - ASSERT_EQ(fileToMerge.table_id_, "test1"); - ASSERT_EQ(fileToMerge.dimension_, 123); - - meta::TableFilesSchema resultTableFilesSchema; - std::vector ids; - ids.push_back(tableFileSchema.id_); - status = impl.GetTableFiles(tableFileSchema.table_id_, ids, resultTableFilesSchema); - ASSERT_TRUE(status.ok()); - ASSERT_EQ(resultTableFilesSchema.size(), 1); - meta::TableFileSchema resultTableFileSchema = resultTableFilesSchema[0]; -// ASSERT_EQ(resultTableFileSchema.id_, tableFileSchema.id_); - ASSERT_EQ(resultTableFileSchema.table_id_, tableFileSchema.table_id_); - ASSERT_EQ(resultTableFileSchema.file_id_, tableFileSchema.file_id_); - ASSERT_EQ(resultTableFileSchema.file_type_, tableFileSchema.file_type_); - ASSERT_EQ(resultTableFileSchema.size_, tableFileSchema.size_); - ASSERT_EQ(resultTableFileSchema.date_, tableFileSchema.date_); - ASSERT_EQ(resultTableFileSchema.engine_type_, tableFileSchema.engine_type_); - ASSERT_EQ(resultTableFileSchema.dimension_, tableFileSchema.dimension_); - - tableFileSchema.size_ = 234; - meta::TableSchema schema3; - schema3.table_id_ = "test3"; - schema3.dimension_ = 321; - status = impl.CreateTable(schema3); - ASSERT_TRUE(status.ok()); - meta::TableFileSchema tableFileSchema2; - tableFileSchema2.table_id_ = "test3"; - tableFileSchema2.size_ = 345; - status = impl.CreateTableFile(tableFileSchema2); - ASSERT_TRUE(status.ok()); - meta::TableFilesSchema filesToUpdate; - filesToUpdate.emplace_back(tableFileSchema); - filesToUpdate.emplace_back(tableFileSchema2); - status = impl.UpdateTableFile(tableFileSchema); - ASSERT_TRUE(status.ok()); - - uint64_t resultSize; - status = impl.Size(resultSize); -// std::cout << status.ToString() << std::endl; - ASSERT_TRUE(status.ok()); - ASSERT_EQ(resultSize, tableFileSchema.size_ + tableFileSchema2.size_); - - uint64_t countResult; - status = impl.Count(tableFileSchema.table_id_, countResult); - ASSERT_TRUE(status.ok()); - - status = impl.DropAll(); - ASSERT_TRUE(status.ok()); - -} - -TEST_F(MySQLTest, GROUP_TEST) { +TEST_F(MySQLTest, TABLE_TEST) { DBMetaOptions options; try { options = getDBMetaOptions(); @@ -210,38 +33,37 @@ TEST_F(MySQLTest, GROUP_TEST) { int mode = Options::MODE::SINGLE; meta::MySQLMetaImpl impl(options, mode); - auto table_id = "meta_test_group"; + auto table_id = "meta_test_table"; - meta::TableSchema group; - group.table_id_ = table_id; - auto status = impl.CreateTable(group); + meta::TableSchema table; + table.table_id_ = table_id; + auto status = impl.CreateTable(table); ASSERT_TRUE(status.ok()); - auto gid = group.id_; - group.id_ = -1; - status = impl.DescribeTable(group); + auto gid = table.id_; + table.id_ = -1; + status = impl.DescribeTable(table); ASSERT_TRUE(status.ok()); - ASSERT_EQ(group.id_, gid); - ASSERT_EQ(group.table_id_, table_id); + ASSERT_EQ(table.id_, gid); + ASSERT_EQ(table.table_id_, table_id); - group.table_id_ = "not_found"; - status = impl.DescribeTable(group); + table.table_id_ = "not_found"; + status = impl.DescribeTable(table); ASSERT_TRUE(!status.ok()); - group.table_id_ = table_id; - status = impl.CreateTable(group); + table.table_id_ = table_id; + status = impl.CreateTable(table); ASSERT_TRUE(status.ok()); - group.table_id_ = ""; - status = impl.CreateTable(group); + table.table_id_ = ""; + status = impl.CreateTable(table); ASSERT_TRUE(status.ok()); - status = impl.DropAll(); ASSERT_TRUE(status.ok()); } -TEST_F(MySQLTest, table_file_TEST) { +TEST_F(MySQLTest, TABLE_FILE_TEST) { DBMetaOptions options; try { options = getDBMetaOptions(); @@ -253,17 +75,16 @@ TEST_F(MySQLTest, table_file_TEST) { int mode = Options::MODE::SINGLE; meta::MySQLMetaImpl impl(options, mode); - auto table_id = "meta_test_group"; + auto table_id = "meta_test_table"; - meta::TableSchema group; - group.table_id_ = table_id; - group.dimension_ = 256; - auto status = impl.CreateTable(group); + meta::TableSchema table; + table.table_id_ = table_id; + table.dimension_ = 256; + auto status = impl.CreateTable(table); meta::TableFileSchema table_file; - table_file.table_id_ = group.table_id_; + table_file.table_id_ = table.table_id_; status = impl.CreateTableFile(table_file); -// std::cout << status.ToString() << std::endl; ASSERT_TRUE(status.ok()); ASSERT_EQ(table_file.file_type_, meta::TableFileSchema::NEW); @@ -332,15 +153,15 @@ TEST_F(MySQLTest, ARCHIVE_TEST_DAYS) { int mode = Options::MODE::SINGLE; meta::MySQLMetaImpl impl(options, mode); - auto table_id = "meta_test_group"; + auto table_id = "meta_test_table"; - meta::TableSchema group; - group.table_id_ = table_id; - auto status = impl.CreateTable(group); + meta::TableSchema table; + table.table_id_ = table_id; + auto status = impl.CreateTable(table); meta::TableFilesSchema files; meta::TableFileSchema table_file; - table_file.table_id_ = group.table_id_; + table_file.table_id_ = table.table_id_; auto cnt = 100; long ts = utils::GetMicroSecTimeStamp(); @@ -391,13 +212,13 @@ TEST_F(MySQLTest, ARCHIVE_TEST_DISK) { auto impl = meta::MySQLMetaImpl(options, mode); auto table_id = "meta_test_group"; - meta::TableSchema group; - group.table_id_ = table_id; - auto status = impl.CreateTable(group); + meta::TableSchema table; + table.table_id_ = table_id; + auto status = impl.CreateTable(table); meta::TableFilesSchema files; meta::TableFileSchema table_file; - table_file.table_id_ = group.table_id_; + table_file.table_id_ = table.table_id_; auto cnt = 10; auto each_size = 2UL; @@ -445,9 +266,9 @@ TEST_F(MySQLTest, TABLE_FILES_TEST) { auto table_id = "meta_test_group"; - meta::TableSchema group; - group.table_id_ = table_id; - auto status = impl.CreateTable(group); + meta::TableSchema table; + table.table_id_ = table_id; + auto status = impl.CreateTable(table); int new_files_cnt = 4; int raw_files_cnt = 5; @@ -455,7 +276,7 @@ TEST_F(MySQLTest, TABLE_FILES_TEST) { int index_files_cnt = 7; meta::TableFileSchema table_file; - table_file.table_id_ = group.table_id_; + table_file.table_id_ = table.table_id_; for (auto i=0; i Date: Mon, 1 Jul 2019 19:59:26 +0800 Subject: [PATCH 058/189] update Former-commit-id: 2e5ec910257824103ab706b7e71a51ea224acfd4 --- cpp/src/db/MySQLMetaImpl.cpp | 128 +++++++++------------------------- cpp/src/db/Options.h | 1 - cpp/src/server/DBWrapper.cpp | 11 --- cpp/src/server/ServerConfig.h | 1 - 4 files changed, 32 insertions(+), 109 deletions(-) diff --git a/cpp/src/db/MySQLMetaImpl.cpp b/cpp/src/db/MySQLMetaImpl.cpp index 2051c12f49..c226abf56b 100644 --- a/cpp/src/db/MySQLMetaImpl.cpp +++ b/cpp/src/db/MySQLMetaImpl.cpp @@ -193,9 +193,7 @@ namespace meta { "engine_type INT DEFAULT 1 NOT NULL, " << "store_raw_data BOOL DEFAULT false NOT NULL);"; - if (options_.sql_echo) { - ENGINE_LOG_DEBUG << "MySQLMetaImpl::Initialize: " << InitializeQuery.str(); - } + ENGINE_LOG_DEBUG << "MySQLMetaImpl::Initialize: " << InitializeQuery.str(); if (!InitializeQuery.exec()) { return Status::DBTransactionError("Initialization Error", InitializeQuery.error()); @@ -212,9 +210,7 @@ namespace meta { "created_on BIGINT NOT NULL, " << "date INT DEFAULT -1 NOT NULL);"; - if (options_.sql_echo) { - ENGINE_LOG_DEBUG << "MySQLMetaImpl::Initialize: " << InitializeQuery.str(); - } + ENGINE_LOG_DEBUG << "MySQLMetaImpl::Initialize: " << InitializeQuery.str(); if (!InitializeQuery.exec()) { return Status::DBTransactionError("Initialization Error", InitializeQuery.error()); @@ -305,9 +301,7 @@ namespace meta { "WHERE table_id = " << quote << table_id << " AND " << "date in (" << dateListStr << ");"; - if (options_.sql_echo) { - ENGINE_LOG_DEBUG << "MySQLMetaImpl::DropPartitionsByDates: " << dropPartitionsByDatesQuery.str(); - } + ENGINE_LOG_DEBUG << "MySQLMetaImpl::DropPartitionsByDates: " << dropPartitionsByDatesQuery.str(); if (!dropPartitionsByDatesQuery.exec()) { ENGINE_LOG_ERROR << "QUERY ERROR WHEN DROPPING PARTITIONS BY DATES"; @@ -353,9 +347,7 @@ namespace meta { "WHERE table_id = " << quote << table_schema.table_id_ << ";"; // ENGINE_LOG_DEBUG << "Create Table : " << createTableQuery.str(); - if (options_.sql_echo) { - ENGINE_LOG_DEBUG << "MySQLMetaImpl::CreateTable: " << createTableQuery.str(); - } + ENGINE_LOG_DEBUG << "MySQLMetaImpl::CreateTable: " << createTableQuery.str(); StoreQueryResult res = createTableQuery.store(); assert(res && res.num_rows() <= 1); @@ -390,9 +382,7 @@ namespace meta { created_on << ", " << files_cnt << ", " << engine_type << ", " << store_raw_data << ");"; // ENGINE_LOG_DEBUG << "Create Table : " << createTableQuery.str(); - if (options_.sql_echo) { - ENGINE_LOG_DEBUG << "MySQLMetaImpl::CreateTable: " << createTableQuery.str(); - } + ENGINE_LOG_DEBUG << "MySQLMetaImpl::CreateTable: " << createTableQuery.str(); if (SimpleResult res = createTableQuery.execute()) { table_schema.id_ = res.insert_id(); //Might need to use SELECT LAST_INSERT_ID()? @@ -457,9 +447,7 @@ namespace meta { "SET state = " << std::to_string(TableSchema::TO_DELETE) << " " << "WHERE table_id = " << quote << table_id << ";"; - if (options_.sql_echo) { - ENGINE_LOG_DEBUG << "MySQLMetaImpl::DeleteTable: " << deleteTableQuery.str(); - } + ENGINE_LOG_DEBUG << "MySQLMetaImpl::DeleteTable: " << deleteTableQuery.str(); if (!deleteTableQuery.exec()) { ENGINE_LOG_ERROR << "QUERY ERROR WHEN DELETING TABLE"; @@ -506,9 +494,7 @@ namespace meta { "WHERE table_id = " << quote << table_id << " AND " << "file_type <> " << std::to_string(TableFileSchema::TO_DELETE) << ";"; - if (options_.sql_echo) { - ENGINE_LOG_DEBUG << "MySQLMetaImpl::DeleteTableFiles: " << deleteTableFilesQuery.str(); - } + ENGINE_LOG_DEBUG << "MySQLMetaImpl::DeleteTableFiles: " << deleteTableFilesQuery.str(); if (!deleteTableFilesQuery.exec()) { ENGINE_LOG_ERROR << "QUERY ERROR WHEN DELETING TABLE FILES"; @@ -551,9 +537,7 @@ namespace meta { "WHERE table_id = " << quote << table_schema.table_id_ << " " << "AND state <> " << std::to_string(TableSchema::TO_DELETE) << ";"; - if (options_.sql_echo) { - ENGINE_LOG_DEBUG << "MySQLMetaImpl::DescribeTable: " << describeTableQuery.str(); - } + ENGINE_LOG_DEBUG << "MySQLMetaImpl::DescribeTable: " << describeTableQuery.str(); res = describeTableQuery.store(); } //Scoped Connection @@ -618,9 +602,7 @@ namespace meta { "AND state <> " << std::to_string(TableSchema::TO_DELETE) << ") " << "AS " << quote << "check" << ";"; - if (options_.sql_echo) { - ENGINE_LOG_DEBUG << "MySQLMetaImpl::HasTable: " << hasTableQuery.str(); - } + ENGINE_LOG_DEBUG << "MySQLMetaImpl::HasTable: " << hasTableQuery.str(); res = hasTableQuery.store(); } //Scoped Connection @@ -664,9 +646,7 @@ namespace meta { "FROM Tables " << "WHERE state <> " << std::to_string(TableSchema::TO_DELETE) << ";"; - if (options_.sql_echo) { - ENGINE_LOG_DEBUG << "MySQLMetaImpl::AllTables: " << allTablesQuery.str(); - } + ENGINE_LOG_DEBUG << "MySQLMetaImpl::AllTables: " << allTablesQuery.str(); res = allTablesQuery.store(); } //Scoped Connection @@ -755,9 +735,7 @@ namespace meta { quote << file_id << ", " << file_type << ", " << size << ", " << updated_time << ", " << created_on << ", " << date << ");"; - if (options_.sql_echo) { - ENGINE_LOG_DEBUG << "MySQLMetaImpl::CreateTableFile: " << createTableFileQuery.str(); - } + ENGINE_LOG_DEBUG << "MySQLMetaImpl::CreateTableFile: " << createTableFileQuery.str(); if (SimpleResult res = createTableFileQuery.execute()) { file_schema.id_ = res.insert_id(); //Might need to use SELECT LAST_INSERT_ID()? @@ -821,9 +799,7 @@ namespace meta { "FROM TableFiles " << "WHERE file_type = " << std::to_string(TableFileSchema::TO_INDEX) << ";"; - if (options_.sql_echo) { - ENGINE_LOG_DEBUG << "MySQLMetaImpl::FilesToIndex: " << filesToIndexQuery.str(); - } + ENGINE_LOG_DEBUG << "MySQLMetaImpl::FilesToIndex: " << filesToIndexQuery.str(); res = filesToIndexQuery.store(); } //Scoped Connection @@ -911,9 +887,7 @@ namespace meta { "file_type = " << std::to_string(TableFileSchema::TO_INDEX) << " OR " << "file_type = " << std::to_string(TableFileSchema::INDEX) << ");"; - if (options_.sql_echo) { - ENGINE_LOG_DEBUG << "MySQLMetaImpl::FilesToSearch: " << filesToSearchQuery.str(); - } + ENGINE_LOG_DEBUG << "MySQLMetaImpl::FilesToSearch: " << filesToSearchQuery.str(); res = filesToSearchQuery.store(); @@ -936,9 +910,7 @@ namespace meta { "file_type = " << std::to_string(TableFileSchema::TO_INDEX) << " OR " << "file_type = " << std::to_string(TableFileSchema::INDEX) << ");"; - if (options_.sql_echo) { - ENGINE_LOG_DEBUG << "MySQLMetaImpl::FilesToSearch: " << filesToSearchQuery.str(); - } + ENGINE_LOG_DEBUG << "MySQLMetaImpl::FilesToSearch: " << filesToSearchQuery.str(); res = filesToSearchQuery.store(); @@ -1023,9 +995,7 @@ namespace meta { "file_type = " << std::to_string(TableFileSchema::RAW) << " " << "ORDER BY size DESC" << ";"; - if (options_.sql_echo) { - ENGINE_LOG_DEBUG << "MySQLMetaImpl::FilesToMerge: " << filesToMergeQuery.str(); - } + ENGINE_LOG_DEBUG << "MySQLMetaImpl::FilesToMerge: " << filesToMergeQuery.str(); res = filesToMergeQuery.store(); } //Scoped Connection @@ -1116,9 +1086,7 @@ namespace meta { "WHERE table_id = " << quote << table_id << " AND " << "(" << idStr << ");"; - if (options_.sql_echo) { - ENGINE_LOG_DEBUG << "MySQLMetaImpl::GetTableFiles: " << getTableFileQuery.str(); - } + ENGINE_LOG_DEBUG << "MySQLMetaImpl::GetTableFiles: " << getTableFileQuery.str(); res = getTableFileQuery.store(); } //Scoped Connection @@ -1200,9 +1168,7 @@ namespace meta { "WHERE created_on < " << std::to_string(now - usecs) << " AND " << "file_type <> " << std::to_string(TableFileSchema::TO_DELETE) << ";"; - if (options_.sql_echo) { - ENGINE_LOG_DEBUG << "MySQLMetaImpl::Archive: " << archiveQuery.str(); - } + ENGINE_LOG_DEBUG << "MySQLMetaImpl::Archive: " << archiveQuery.str(); if (!archiveQuery.exec()) { return Status::DBTransactionError("QUERY ERROR DURING ARCHIVE", archiveQuery.error()); @@ -1251,9 +1217,7 @@ namespace meta { "FROM TableFiles " << "WHERE file_type <> " << std::to_string(TableFileSchema::TO_DELETE) << ";"; - if (options_.sql_echo) { - ENGINE_LOG_DEBUG << "MySQLMetaImpl::Size: " << getSizeQuery.str(); - } + ENGINE_LOG_DEBUG << "MySQLMetaImpl::Size: " << getSizeQuery.str(); res = getSizeQuery.store(); } //Scoped Connection @@ -1315,9 +1279,7 @@ namespace meta { "ORDER BY id ASC " << "LIMIT 10;"; - if (options_.sql_echo) { - ENGINE_LOG_DEBUG << "MySQLMetaImpl::DiscardFiles: " << discardFilesQuery.str(); - } + ENGINE_LOG_DEBUG << "MySQLMetaImpl::DiscardFiles: " << discardFilesQuery.str(); // std::cout << discardFilesQuery.str() << std::endl; StoreQueryResult res = discardFilesQuery.store(); @@ -1349,9 +1311,7 @@ namespace meta { "updated_time = " << std::to_string(utils::GetMicroSecTimeStamp()) << " " << "WHERE " << idsToDiscardStr << ";"; - if (options_.sql_echo) { - ENGINE_LOG_DEBUG << "MySQLMetaImpl::DiscardFiles: " << discardFilesQuery.str(); - } + ENGINE_LOG_DEBUG << "MySQLMetaImpl::DiscardFiles: " << discardFilesQuery.str(); status = discardFilesQuery.exec(); if (!status) { @@ -1397,9 +1357,7 @@ namespace meta { updateTableFileQuery << "SELECT state FROM Tables " << "WHERE table_id = " << quote << file_schema.table_id_ << ";"; - if (options_.sql_echo) { - ENGINE_LOG_DEBUG << "MySQLMetaImpl::UpdateTableFile: " << updateTableFileQuery.str(); - } + ENGINE_LOG_DEBUG << "MySQLMetaImpl::UpdateTableFile: " << updateTableFileQuery.str(); StoreQueryResult res = updateTableFileQuery.store(); @@ -1434,9 +1392,7 @@ namespace meta { "date = " << date << " " << "WHERE id = " << id << ";"; - if (options_.sql_echo) { - ENGINE_LOG_DEBUG << "MySQLMetaImpl::UpdateTableFile: " << updateTableFileQuery.str(); - } + ENGINE_LOG_DEBUG << "MySQLMetaImpl::UpdateTableFile: " << updateTableFileQuery.str(); // std::cout << updateTableFileQuery.str() << std::endl; @@ -1491,9 +1447,7 @@ namespace meta { "AND state <> " << std::to_string(TableSchema::TO_DELETE) << ") " << "AS " << quote << "check" << ";"; - if (options_.sql_echo) { - ENGINE_LOG_DEBUG << "MySQLMetaImpl::UpdateTableFiles: " << updateTableFilesQuery.str(); - } + ENGINE_LOG_DEBUG << "MySQLMetaImpl::UpdateTableFiles: " << updateTableFilesQuery.str(); StoreQueryResult res = updateTableFilesQuery.store(); @@ -1530,9 +1484,7 @@ namespace meta { "date = " << date << " " << "WHERE id = " << id << ";"; - if (options_.sql_echo) { - ENGINE_LOG_DEBUG << "MySQLMetaImpl::UpdateTableFiles: " << updateTableFilesQuery.str(); - } + ENGINE_LOG_DEBUG << "MySQLMetaImpl::UpdateTableFiles: " << updateTableFilesQuery.str(); if (!updateTableFilesQuery.exec()) { ENGINE_LOG_ERROR << "QUERY ERROR WHEN UPDATING TABLE FILES"; @@ -1582,9 +1534,7 @@ namespace meta { "WHERE file_type = " << std::to_string(TableFileSchema::TO_DELETE) << " AND " << "updated_time < " << std::to_string(now - seconds * US_PS) << ";"; - if (options_.sql_echo) { - ENGINE_LOG_DEBUG << "MySQLMetaImpl::CleanUpFilesWithTTL: " << cleanUpFilesWithTTLQuery.str(); - } + ENGINE_LOG_DEBUG << "MySQLMetaImpl::CleanUpFilesWithTTL: " << cleanUpFilesWithTTLQuery.str(); StoreQueryResult res = cleanUpFilesWithTTLQuery.store(); @@ -1628,9 +1578,7 @@ namespace meta { cleanUpFilesWithTTLQuery << "DELETE FROM TableFiles WHERE " << idsToDeleteStr << ";"; - if (options_.sql_echo) { - ENGINE_LOG_DEBUG << "MySQLMetaImpl::CleanUpFilesWithTTL: " << cleanUpFilesWithTTLQuery.str(); - } + ENGINE_LOG_DEBUG << "MySQLMetaImpl::CleanUpFilesWithTTL: " << cleanUpFilesWithTTLQuery.str(); if (!cleanUpFilesWithTTLQuery.exec()) { ENGINE_LOG_ERROR << "QUERY ERROR WHEN CLEANING UP FILES WITH TTL"; @@ -1669,9 +1617,7 @@ namespace meta { "FROM Tables " << "WHERE state = " << std::to_string(TableSchema::TO_DELETE) << ";"; - if (options_.sql_echo) { - ENGINE_LOG_DEBUG << "MySQLMetaImpl::CleanUpFilesWithTTL: " << cleanUpFilesWithTTLQuery.str(); - } + ENGINE_LOG_DEBUG << "MySQLMetaImpl::CleanUpFilesWithTTL: " << cleanUpFilesWithTTLQuery.str(); StoreQueryResult res = cleanUpFilesWithTTLQuery.store(); assert(res); @@ -1697,9 +1643,7 @@ namespace meta { cleanUpFilesWithTTLQuery << "DELETE FROM Tables WHERE " << idsToDeleteStr << ";"; - if (options_.sql_echo) { - ENGINE_LOG_DEBUG << "MySQLMetaImpl::CleanUpFilesWithTTL: " << cleanUpFilesWithTTLQuery.str(); - } + ENGINE_LOG_DEBUG << "MySQLMetaImpl::CleanUpFilesWithTTL: " << cleanUpFilesWithTTLQuery.str(); if (!cleanUpFilesWithTTLQuery.exec()) { ENGINE_LOG_ERROR << "QUERY ERROR WHEN CLEANING UP FILES WITH TTL"; @@ -1739,9 +1683,7 @@ namespace meta { "WHERE table_schema = " << quote << mysql_connection_pool_->getDB() << quote << " " << "AND table_name = " << quote << "TableFiles" << quote << ";"; - if (options_.sql_echo) { - ENGINE_LOG_DEBUG << "MySQLMetaImpl::CleanUp: " << cleanUpQuery.str(); - } + ENGINE_LOG_DEBUG << "MySQLMetaImpl::CleanUp: " << cleanUpQuery.str(); StoreQueryResult res = cleanUpQuery.store(); assert(res); @@ -1749,9 +1691,7 @@ namespace meta { ENGINE_LOG_DEBUG << "Remove table file type as NEW"; cleanUpQuery << "DELETE FROM TableFiles WHERE file_type = " << std::to_string(TableFileSchema::NEW) << ";"; - if (options_.sql_echo) { - ENGINE_LOG_DEBUG << "MySQLMetaImpl::CleanUp: " << cleanUpQuery.str(); - } + ENGINE_LOG_DEBUG << "MySQLMetaImpl::CleanUp: " << cleanUpQuery.str(); if (!cleanUpQuery.exec()) { ENGINE_LOG_ERROR << "QUERY ERROR WHEN CLEANING UP FILES"; @@ -1804,9 +1744,7 @@ namespace meta { "file_type = " << std::to_string(TableFileSchema::TO_INDEX) << " OR " << "file_type = " << std::to_string(TableFileSchema::INDEX) << ");"; - if (options_.sql_echo) { - ENGINE_LOG_DEBUG << "MySQLMetaImpl::Count: " << countQuery.str(); - } + ENGINE_LOG_DEBUG << "MySQLMetaImpl::Count: " << countQuery.str(); res = countQuery.store(); } //Scoped Connection @@ -1851,9 +1789,7 @@ namespace meta { Query dropTableQuery = connectionPtr->query(); dropTableQuery << "DROP TABLE IF EXISTS Tables, TableFiles;"; - if (options_.sql_echo) { - ENGINE_LOG_DEBUG << "MySQLMetaImpl::DropAll: " << dropTableQuery.str(); - } + ENGINE_LOG_DEBUG << "MySQLMetaImpl::DropAll: " << dropTableQuery.str(); if (dropTableQuery.exec()) { return Status::OK(); diff --git a/cpp/src/db/Options.h b/cpp/src/db/Options.h index 609e3ca245..39d0a15019 100644 --- a/cpp/src/db/Options.h +++ b/cpp/src/db/Options.h @@ -45,7 +45,6 @@ struct DBMetaOptions { std::string path; std::string backend_uri; ArchiveConf archive_conf = ArchiveConf("delete"); - bool sql_echo = false; }; // DBMetaOptions struct Options { diff --git a/cpp/src/server/DBWrapper.cpp b/cpp/src/server/DBWrapper.cpp index bf859b3b4f..fca15cb65a 100644 --- a/cpp/src/server/DBWrapper.cpp +++ b/cpp/src/server/DBWrapper.cpp @@ -23,17 +23,6 @@ DBWrapper::DBWrapper() { if(index_size > 0) {//ensure larger than zero, unit is MB opt.index_trigger_size = (size_t)index_size * engine::ONE_MB; } - std::string sql_echo = config.GetValue(CONFIG_DB_SQL_ECHO, "off"); - if (sql_echo == "on") { - opt.meta.sql_echo = true; - } - else if (sql_echo == "off") { - opt.meta.sql_echo = false; - } - else { - std::cout << "ERROR: sql_echo specified in db_config is not one of ['on', 'off']" << std::endl; - kill(0, SIGUSR1); - } ConfigNode& serverConfig = ServerConfig::GetInstance().GetConfig(CONFIG_SERVER); std::string mode = serverConfig.GetValue(CONFIG_CLUSTER_MODE, "single"); diff --git a/cpp/src/server/ServerConfig.h b/cpp/src/server/ServerConfig.h index 768430f023..f337275a46 100644 --- a/cpp/src/server/ServerConfig.h +++ b/cpp/src/server/ServerConfig.h @@ -27,7 +27,6 @@ static const std::string CONFIG_DB_PATH = "db_path"; static const std::string CONFIG_DB_INDEX_TRIGGER_SIZE = "index_building_threshold"; static const std::string CONFIG_DB_ARCHIVE_DISK = "archive_disk_threshold"; static const std::string CONFIG_DB_ARCHIVE_DAYS = "archive_days_threshold"; -static const std::string CONFIG_DB_SQL_ECHO = "sql_echo"; static const std::string CONFIG_LOG = "log_config"; From b56964f179936b1d33ad8466149c057b6243b20e Mon Sep 17 00:00:00 2001 From: zhiru Date: Mon, 1 Jul 2019 20:10:25 +0800 Subject: [PATCH 059/189] update Former-commit-id: 736501fe9c806903d75ef38140b8c9470ac3adc0 --- cpp/conf/server_config.template | 1 - 1 file changed, 1 deletion(-) diff --git a/cpp/conf/server_config.template b/cpp/conf/server_config.template index 6339b0c440..6b432056c5 100644 --- a/cpp/conf/server_config.template +++ b/cpp/conf/server_config.template @@ -15,7 +15,6 @@ db_config: index_building_threshold: 1024 # index building trigger threshold, default: 1024, unit: MB archive_disk_threshold: 512 # triger archive action if storage size exceed this value, unit: GB archive_days_threshold: 30 # files older than x days will be archived, unit: day - sql_echo: on # print sql statement in debug log metric_config: is_startup: off # if monitoring start: on, off From 3ba3c03a01d44b4e818e10b75fdad00ab4038779 Mon Sep 17 00:00:00 2001 From: yu yunfeng Date: Tue, 2 Jul 2019 15:03:21 +0800 Subject: [PATCH 060/189] Update faiss parameter Former-commit-id: f8d459c5b34053cee90ea7d0f6d4a6e27284b37e --- cpp/src/wrapper/Index.cpp | 4 ++++ cpp/src/wrapper/Operand.cpp | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/cpp/src/wrapper/Index.cpp b/cpp/src/wrapper/Index.cpp index 35dd646fdf..c306b65e1a 100644 --- a/cpp/src/wrapper/Index.cpp +++ b/cpp/src/wrapper/Index.cpp @@ -13,6 +13,7 @@ #include "Index.h" #include "faiss/index_io.h" +#include "faiss/IndexIVF.h" namespace zilliz { namespace milvus { @@ -55,6 +56,9 @@ bool Index::add_with_ids(idx_t n, const float *xdata, const long *xids) { bool Index::search(idx_t n, const float *data, idx_t k, float *distances, long *labels) const { try { + if(auto ivf_index = std::dynamic_pointer_cast(index_)) { + ivf_index->nprobe = 100; + } index_->search(n, data, k, distances, labels); } catch (std::exception &e) { diff --git a/cpp/src/wrapper/Operand.cpp b/cpp/src/wrapper/Operand.cpp index 25341676a6..30e31067fd 100644 --- a/cpp/src/wrapper/Operand.cpp +++ b/cpp/src/wrapper/Operand.cpp @@ -39,7 +39,7 @@ string Operand::get_index_type(const int &nb) { } case IVF: { index_str += (ncent != 0 ? index_type + std::to_string(ncent) : - index_type + std::to_string(int(nb / 1000000.0 * 16384))); + index_type + std::to_string(int(nb / 1000000.0 * 1638))); break; } case IDMAP: { From b1212c58edabdd105ca8d31cb6544f741bf8f123 Mon Sep 17 00:00:00 2001 From: jinhai Date: Tue, 2 Jul 2019 18:17:39 +0800 Subject: [PATCH 061/189] Add Milvus EULA Former-commit-id: 6eee85795efc8ce3bbb2c6ccd04e9bba62af90db --- cpp/CMakeLists.txt | 7 ++- cpp/Milvus-EULA-cn.md | 119 ++++++++++++++++++++++++++++++++++++++ cpp/Milvus-EULA-en.md | 129 ++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 254 insertions(+), 1 deletion(-) create mode 100644 cpp/Milvus-EULA-cn.md create mode 100644 cpp/Milvus-EULA-en.md diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 947759f793..4d8f43b5b9 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -152,7 +152,12 @@ install(FILES conf/log_config.conf DESTINATION conf) - +install(FILES + ./Milvus-EULA-cn.md + ./Milvus-EULA-en.md + DESTINATION + license + ) config_summary() diff --git a/cpp/Milvus-EULA-cn.md b/cpp/Milvus-EULA-cn.md new file mode 100644 index 0000000000..e9a25dcc6c --- /dev/null +++ b/cpp/Milvus-EULA-cn.md @@ -0,0 +1,119 @@ +# **Milvus**终端用户授权许可条款及条件 + +#### 2019-06-30 版 + + + +本条款和条件(下称“本协议”)适用于使用由上海赜睿信息科技有限公司(下称“**ZILLIZ**”)所提供的Milvus产品(参见如下定义) 的用户。 + +**请仔细阅读如下条款：** + +**若您（下称“您”或“用户”）代表某公司或者其他机构使用任何产品时， 您特此陈述您作为该公司或该等其他机构的员工或代理，您有权代表该公司或该等其他机构接受本协议项下所要求的全部条款和条件。** + +**若使用任何产品，您知晓并同意:** + +**(A)您已阅读本协议中所有的条款和条件;** + +**(B)您已理解本协议中所有的条款和条件;** + +**(C)您已同意本协议中所有条款和条件对您具有法律约束力。** + +**如果您不同意本协议所述条款和条件中的任意内容，则可以选择不使用产品的任何部分。** + +**本协议的“生效日期”是指您第一次下载任何产品的日期。** + +1. **产品**，指本协议项下任何 **ZILLIZ** 的Milvus产品和软件，包括： Milvus向量检索数据库Docker版与其相关的升级、更新、故障修复或修改版本（统称“更新软件”）。无论本协议是否另有规定： + + （a）仅Milvus向量检索数据库Docker版是免费授权用户的版本，且ZILLIZ保留收回该授权的权力； + + （b）任何使用或者试用Milvus向量检索数据库Docker版的个人与组织，需要通过support@zilliz.com向ZILLIZ告知个人或者组织的身份、联系方式以及使用Milvus的目的。 + + （c）制作和使用额外的副本仅限于必要的备份目的。 + +2. **全部协议**，本协议包括本授权许可条款及条件以及任何[Milvus官方网站](https://milvus.io)展示或者网页链接所附或引用的全部条款。本协议是双方就相关事项达成的完整协议，取代 **ZILLIZ** 与用户之间就本条款相关事项所达成的其他任何协议，无论是口头的还是书面的。 + +3. **使用许可**，**ZILLIZ** 授予用户非排他性的、不可转让的、非可再授权的、可撤回的和有限的许可进行访问和使用第1条所定义的产品，该访问和使用许可仅限于用户内部使用之目的。通过电子下载或其他经许可的来源获得产品的用户均应受限于本协议的内容。 + +4. **许可限制**，除非本协议另有明文规定，否则用户将不被允许： + + （a）修改、翻译或制造产品的衍生作品； + + （b）反向编译、反向工程、破解产品的任何部分或试图发现有关产品的任何源代码、基本理念或运算方法； （c）销售、分派、再授权、出租、出借、出质、提供或另行翻译全部或部分产品； + + （d）制造、获取非法制造的、再版或复制产品； + + （e）删除或更改与产品相关联的任何商标、标志、版权或其他专有标 ； + + （f）不得在没有 **ZILLIZ** 明确书面授权的情况下，使用或许可他人使用产品为第三方提供服务，无论是在产品服务过程中使用或采用分时的方式； + + （g）引起或许可任何其他方进行上述任何一种禁止行为。 + +5. **所有权**，**ZILLIZ** 和用户在本协议项下的许可需明确，**ZILLIZ** 有以下各项的全部权利、所有权和相关利益：（a）产品（包括但不限于，任何更新软件、修订版本或其衍生作品）； + + （b）在 **ZILLIZ** 根据本协议提供任何服务的过程中或作为其提供服务的结果，由 **ZILLIZ** 发现、 产生或发展出来的所有的概念、发明、发现、改进、信息、创意作品等； + + （c）前述各项所含的任何知识产权权利。在本协议项下，“知识产权”是指在任何管辖区域经申请和注册获得认可和保护的全部专利、版权、道德权利、商标、商业秘密和任何其他形式的权利。**ZILLIZ** 与用户同意，在受限于法律法规规定及本协议全部条款和条件的前提下，用户拥有使用产品而产生的数据的权利、所有权等相关利益。本协议中无任何默示许可，**ZILLIZ** 保留本协议项下未明确授权的全部权利。除非本协议明确约定，**ZILLIZ** 在本协议下未授予用户任何许可权利，无论是通过暗示、默许或其他方式。 + +6. **保密**，保密信息是指，无论是在本协议生效前或生效后，由 **ZILLIZ** 披露给用户的与本协议或与 **ZILLIZ** 相关的所有信息（无论是以口头、书面或其他有形、无形的形式）。保密信息包括但不限于，商业计划的内容、产品、发明、设计图纸、财务计划、计算机程序、发明、用户信息、战略和其他类似信息。在本协议期限内，除非获得明确许可， 用户需保证保密信息的秘密性，并确保不会使用上述保密信息。用户将采用与保护其自身保密信息的同等谨慎程度（不论在何种情况下均不低于合理的谨慎程度）来保护 **ZILLIZ** 的保密信息，来避免使得保密信息被未经授权的使用和披露。保密信息只供用户根据本协议规定使用产品之目的而使用。此外，用户将： + + （a）除非用户为了根据本协议的规定而使用产品之目的外，不得以任何形式复制、使用或披露保密信息； （b）只向为确保用户可根据本协议使用产品而必需知道该保密信息的员工和顾问披露保密信息，前提是上述员工和顾问已签署了包含保密义务不低于本条所述内容的保密协议。 + + 保密信息不包括下列信息： + + （a） 非因用户过错违反本协议导致已进入公共领域可被第三方获取的； + + （b） 用户能合理证明其在通过 **ZILLIZ** 获得之前已知晓的； + + （c）用户能证明没有使用或参考该保密信息而独立获得的； + + （d）用户从其他无披露限制或无保密义务的第三方获得的。如无另行说明，由用户提供给 **ZILLIZ** 有关产品的任何建议、评论或者其他反馈（统称“反馈信息”）将同样构成保密信息。 + + 此外，**ZILLIZ** 有权使用、披露、复制、许可和利用上述反馈信息，而无需承担任何知识产权负担或其他任何形式的义务或限制。根据相关法律法规，与本协议的履行和用户使用 **ZILLIZ** 产品相关的情况下： + + （a）**ZILLIZ** 同意不会要求用户提供任何个人身份信息； + + （b）用户同意不提供任何个人身份信息给 **ZILLIZ**。 + +7. **免责声明**，用户陈述、保证及承诺如下： + + （a）其所有员工和顾问都将遵守本协议的全部条款； + + （b）在履行本协议时将遵守全部可适用的政府部门颁发的法律、法规、规章、命令和其他要求（无论是现行有效还是之后生效的）。 + + 无论本协议是否另有规定，用户将持续对其雇员或顾问的全部作为或不作为承担责任，如同该等作为或不作为系其自身所为。 + + 产品系按照原状或现状提供给用户，不含任何形式的陈述、保证、 承诺或条件。**ZILLIZ** 及其供应商不保证任何产品将无任何故障、错误或漏洞。**ZILLIZ** 和其供应商不为产品的如下内容提供任何陈述和保证（无论是明示或暗示，口头或书面），不论该内容是否依据法律之规定，行业惯例，交易习惯或其他原因而要求的： + + （a）保证适销性； + + （b）保证可适用于任何目的（不论 **ZILLIZ** 是否知晓、应当知晓、被建议或另行得知该目的）； + + （c）保证不侵权和拥有全部所有权。用户已明确知悉并同意产品上无任何陈述和保证。此外，鉴于进行入侵和网络攻击的新技术在不断发展，**ZILLIZ** 并不保证产品或产品所使用的系统或网络将免于任何入侵或攻击。 + +8. **损害赔偿**，用户应赔偿、保护或使得 **ZILLIZ** 及其董事、高管、 雇员、供应商、顾问、承包商和代理商（统称为“**ZILLIZ **受保障方”）免受所有现存或潜在的针对 **ZILLIZ** 受保障方因提起请求、诉讼或其他程序而引起的要求其赔偿损害损失、支付费用、罚款、调解、 损失费用等支出（包括但不限于律师费、费用、罚款、利息和垫付款），用户承担上述责任的前提是该请求、诉讼或其他程序，不论是否成功系在如下情况发生时导致、引起的，或以任何形式与下述情况相关： + + （a）任何对本协议的违反（包括但不限于，任何违反用户陈述和保证或约定的情况）； + + （b）用户过失或故意产生的过错行为； + + （c）引起争议的数据和信息系在产品的使用过程中产生或收集的。 + +9. **责任限制**，除了 **ZILLIZ** 存在欺诈或故意的过错行为，在任何情况下： + + （a）**ZILLIZ** 都不会赔偿用户或任何第三方的因本协议或产品（包括用户使用或无法使用产品的情况）而遭受的任何利润损失、数 据损失、使用损失、收入损失、商誉损失、任何经营活动的中断，任何其他商业损害或损失，或任何间接的、特殊的、附带的、惩戒性、惩罚性或伴随的损失，不论上述损失系因合同、侵权、严格责任或其他原因而确认的，即使 **ZILLIZ** 已被通知或因其他可能的渠道知晓上述损失发生的可能性； + + （b）**ZILLIZ** 因本协议所需承担的全部赔偿责任不应超过用户已支付或将支付给 **ZILLIZ** 的全部款项总额（若有），多项请求亦不得超过该金额限制。上述限制、排除情况及声明应在相关法律允许的最大范围内得以适用，即便任何补偿无法达到其实质目的。 + +10. **第三方供应商**，产品可能包括由第三方供应商许可提供的软件或其他代码（下称“第三方软件”）。用户已知悉第三方供应商不对产品或其任何部分提供任何陈述和保证，**ZILLIZ** 不承担因产品或用户对第三方软件的使用或不能使用的情况而产生的任何责任。 + +11. **诊断和报告**，用户了解并同意该产品包含诊断功能作为其默认配置。 诊断功能用于收集有关使用环境和产品使用过程中的配置文件、节点数、 软件版本、日志文档和其他信息，并将上述信息报告给 **ZILLIZ** 用于提前识别潜在的支持问题、了解用户的使用环境、并提高产品的使用性能。虽然用户可以选择更改诊断功能来禁用自动定时报告或仅用于报告服务记录，但用户需同意，每季度须至少运行一次诊断功能并将结果报告给**ZILLIZ**。 + +12. **终止**，本协议期限从生效之日起直到 **ZILLIZ** 网站规定的期限终止，除非本协议因用户违反本协议中条款而提前终止。无论本协议是否另有规定，在用户存在违反第3、4、5或7条时，**ZILLIZ**有权立即终止本协议。本协议期满或提前终止时： + + （a）根据本协议所授予给用户的所有权利将立即终止，在此情况下用户应立即停止使用产品； + + （b） 用户应及时将届时仍由其占有的所有保密信息及其副本（包括但不限于产品）交还给 **ZILLIZ**，或根据 **ZILLIZ** 的自行审慎决定及指示， 销毁该等保密信息全部副本，未经 **ZILLIZ** 书面同意，用户不得擅自保留任何由 **ZILLIZ** 提供的保密信息及其副本。 + +13. **第三方资源**， **ZILLIZ** 供应的产品可能包括对其他网站、内容或资源的超链接（下称“第三方资源”），且 **ZILLIZ** 此类产品的正常使用可能依赖于第三方资源的可用性。**ZILLIZ** 无法控制任何第三方资源。用户承认并同意，**ZILLIZ** 不就第三方资源的可用性及安全性承担任何责任，也不对该等第三方资源所涉及的或从其中获得的任何广告、产品或其他材料提供保证。用户承认并同意，**ZILLIZ** 不应因第三方资源的可用性及安全性、或用户依赖于第三方资源所涉及的或从其中获得的任何广告、产品或其他材料的完整性、准确性及存续而可能遭受的损失或损害承担任何责任。 + +14. **其他**，本协议全部内容均在中华人民共和国境内履行，受中华人民共和国法律管辖并根据其解释（但不适用相关冲突法的法律条款）。用 **ZILLIZ** 同意与本协议有关的任何争议将向上海市徐汇区人民法院提出，且不可撤销无条件的同意上述法院对因本协议提起的全部诉讼、争议拥有排他的管辖权。一旦确定任何条款无效、非法或无法执行， **ZILLIZ** 保留修改和解释该条款的权利。任何需要发送给用户的通知如公布在 **ZILLIZ** 的网站上则被视为已有效、合法地发送给用户。除了本合同项下应支付款项的义务外，任何一方将不对因不可抗力而导致的无法合理控制的全部或部分未能履行或延迟履行本协议的行为负责， 不可抗力包括但不限于火灾、暴风雨、洪水、地震、内乱、电信中断、 电力中断或其他基础设施的中断、**ZILLIZ** 使用的服务提供商存在问题导致服务中断或终止、罢工、故意毁坏事件、电缆被切断、病毒入侵或其他任意第三方故意或非法的行为引起的其他类似事件。在上述迟延履行情况出现时，可延迟履行协议的时间为因上述原因引起的延迟时间。 本协议另有明确规定外，本协议所要求或认可的通知或通讯均需以书面形式经一方有权代表签署或授权并以直接呈递、隔夜快递，经确认的电子邮件发送，经确认的传真或邮寄挂号信、挂号邮件保留回单等方式送达。对本协议的任何修改、补充或删除或权利放弃，必须通过书面由双方适当授权的代表签署确认后方为有效。任何一方对任何权利或救济的不履行或迟延履行（部分或全部）不构成对该等权利或救济的放弃，也不影响任何其他权利或救济。本协议项下的所有权利主张和救济均可为累积的且不排除本协议中包含的或法律所规定的其他任何权利或救济。 对本协议中任何一项违约责任的豁免或延迟行使任何权利，并不构成对其他后续违约责任的豁免。 \ No newline at end of file diff --git a/cpp/Milvus-EULA-en.md b/cpp/Milvus-EULA-en.md new file mode 100644 index 0000000000..3444dd722b --- /dev/null +++ b/cpp/Milvus-EULA-en.md @@ -0,0 +1,129 @@ +# ZILLIZ End-User License Agreement + +#### Last updated: 2019-06-30 + + + +This End-user License Agreement ("Agreement") is applicable to all users who uses Milvus provided by ZILLIZ company. + +**Please read this agreement carefully before clicking the I Agree button, downloading or using this Application.** + +**If you ("You" or "User") use any product on behalf of a company or other organization, you hereby state that you are an employee or agent of the company or such other institution, and you have the right to represent the company or such institutions to accept all the terms and conditions required under this Agreement. ** + +**If you use any product, you acknowledge and agree:** + +**(A) You have read all the terms and conditions in the Agreement;** + +**(B) You have understand all the terms and conditions in the Agreement;** + +**(C) You have agreed that all the terms and conditions of this Agreement are legally binding on you.** + +**If you do not agree to any of the terms and conditions set forth in this Agreement, you may choose not to use any part of the product.** + +**This agreement takes effect immediately the first time you download the application**. + +1. **Product**. In this Agreement, it refers to Milvus and other related software products of **ZILLIZ**, including Milvus vector indexing database and its updates, higher versions, maintenance or patch releases ("Updated Software"). + + (a) Only the Docker version of Milvus vector indexing database is granted free to the User. **ZILLIZ** retains the right to revoke this grant; + + (b) Any person or organization that intend to use or try the Docker version of Milvus vector indexing database need to inform **ZILLIZ** of the personal identity, contact information and purposes of using the Product by sending an email to: support@zilliz.com; + + (c）Making or using additional copy of the Product is only restricted to necessary copy purposes. + +2. **Related Agreements**. The Related Agreements includes this Agreement and all other related terms and conditions that appear in [Milvus official website](https://milvus.io). This Agreement is the entire and final agreement that replaces all other terms agreed between the User and **ZILLIZ** about issues listed here, oral or written. + +3. **License Grant**. **ZILLIZ** grant You a revocable, non-exclusive, non-transferable limited right to install and use the Application defined above for your personal, non-commercial purposes. The User who uses the Application through downloading and other permitted channels are also subject to this Agreement; + +4. **Restrictions on Use.** You shall use the Application in accordance with the terms in the Agreement, and shall not: + + （a）Make any modification, translation or derivative work from the Application; + + （b）Decompile, reverse engineer, disassemble, attempt to derive the source code or algorithm of the Application; + + （c）Sell, distribute, license re-granting or provide translation of the whole or part of the Application; + + （d）Use the Application for creating a product, service or software. + + （e）Remove, alter or obscure any proprietary notice, trademark, or copyright of the Company and Application; + + （f）Install or use the Application to provide service to third-party partners, without acquiring formal grant of **ZILLIZ** ; + + （g）Perform or permit any behaviors that might lead to one of the above prohibited actions. + +5. **Ownership**. **ZILLIZ** enjoys the ownership of the following： + + （a）Products (includes but is not restricted to any updated software, patch releases, or derivative products); + + （b）All concepts, innovations, discoveries, improvements, information, or creative products developed and discovered by **ZILLIZ** as a result of or arising out of the service providing process; + + （c）Intellectual property rights of the above mentioned products and innovations. In this Agreement, "Intellectual Property" refers to trademarks, patents, designations of origin, industrial designs and models and copyright. **ZILLIZ** and the User agree that the User enjoy all the rights to use data produced by using the Product, while **ZILLIZ** keeps all other rights not explicitly stated in the Agreement. Unless otherwise stated, **ZILLIZ** has not granted any additional rights to Users, either implied, acquiesced or in other ways. + +6. **Non-disclosure**. Confidential Information refers to any and all information revealed to the User by **ZILLIZ**, either oral or written, tangible or intangible, before or after the Agreement takes effect. Confidential information includes but is not restricted to business plans and strategies, product, innovations, design papers, financial plans, computer programs, User information, etc. Within the term of this Agreement, unless granted definite permission, the User shall hold and maintain the Confidential Information in strictest confidence for the sole and exclusive benefit of **ZILLIZ** and using the Product. In addition: + + （a）You shall not copy, use or disclose Confidential Information for purposes other than using the Product agreed in this Agreement; + + （b）You shall carefully restrict access to Confidential Information to employees, contractors, and third parties as is reasonably required and shall require those persons to sign nondisclosure restrictions at least as protective as those in this Agreement. + + Confidential Information does not include: + + （a）Information that can be obtained by third-parties not due to User's violation of the Agreement; + + （b）Information that can be proven to be provided to Users not by **ZILLIZ** ; + + （c） Information that are obtained with no reference to Confidential Information; + + （d）Information the User gets from third-parties that are not subject to non-disclosure agreement. Unless otherwise stated, any comments, suggestions or other feedback ("Feedback Information") about the Product by the User to **ZILLIZ** will also be counted as Confidential Information. + + Furthermore, **ZILLIZ** has the right to use, disclose, copy or use above Feedback Information, and bearing no intellectual property burden or restrictions. According to related laws and regulations, during the fulfillment of this Agreement: + + （a）**ZILLIZ** agree not to require the User to provide any information regarding personal identities; + + （b）The User agree not to provide **ZILLIZ** with any personal information. + +7. **Disclaimer of Warranties**. You acknowledge, agree and promise that: + + (a) All employees and consultants will obey all terms in the Agreement; + + （b）Application of the Agreement is subject to all laws, terms, acts, commands and other requirements issued by the government (no matter these laws are in effect now or will be effective in the future). + + The User shall be held responsible for all the behaviors in relation to the Application. + + The Application is provided on an "As is" or "As available" basis, and that You use or reliance on the Application is at your sole risk and discretion. **ZILLIZ** and its partners make no warranty that the Application will meet all Your requirements and expectations. + + **ZILLIZ** and its suppliers hereby disclaim any and all representations, warranties and guaranties regarding the Application, whether expressed, implied or statutory: + + （a）The implied warranty of merchantability; + + （b）Fitness for a particular purpose; + + （c）Non-infringement. + + Further more, considering the continuous advancement of Internet hacking and attaching technologies, **ZILLIZ** make no guarantee that the Application or the systems and Internet it uses will be exempt from any hack or attack. + +8. **Damages and Penalties**. The User shall pay, protect or prevent **ZILLIZ** and its board members, executives, employees, consultants or representative agencies (**ZILLIZ** Protected Party) from any existing or potential damage loss, fees, penalties and other outgoing payments (include but are not limited to lawyer fees, fines, interests and advance payment) arising out of legal request, litigation or other processes. The prerequisite condition of the above obligations are that the legal request, litigation or process are caused by any of the following situations: + + （a）Any violation of the Agreement; + + （b）User fault or deliberate behavior; + + （c）Controversial data is produced or collected during the usage of the Product. + +9. **Limitation of Liability**. Unless due to deliberate fraud or error from **ZILLIZ**, below terms are applicable: + + （a）Under no circumstances shall **ZILLIZ** be held liable for any profit loss, data loss, revenue loss, termination of operations, any indirect, special, exemplary or consequential damages arising out or in connection with Your access or use of the Application; + + （b）Without limiting the generality of the foregoing, **ZILLIZ**'s aggregate liability to You shall not exceed the total amount of money You already paid or will pay to **ZILLIZ** (if any). + +10. **Third-party Suppliers**. The User acknowledge that no statement and guarantee should be expected from Third-party Suppliers about the Product or its components. **ZILLIZ** hold no obligations to the Users' usage of the softwares provided by third-party Suppliers. + +11. **Diagnosis and Report**. The User know and agree that Diagnosis is part of the configuration of the Product. Diagnosis is used to collect the configuration files, node numbers, software version, logs and related information, and send a Report to **ZILLIZ** to recognize potential support problems, get to know User environment, and to enhance product features. Although You can choose to turn off the Diagnosis function of automatic report sending, however, You shall run the Diagnosis at least once every quarter and send the Report to **ZILLIZ**. + +12. **Termination of Licensing**. This Agreement is valid from the day it takes effect to the termination dated defined in **ZILLIZ** website, unless the User has disobeyed the terms and caused the Agreement to end in advance. Whether or not listed, if the User has violated terms in Clause 3, 4, 5 or 7, **ZILLIZ** may, in its sole and absolute discretion, terminate this License and the rights afforded to You. Upon the expiration or termination of the License: + + （a）All rights afforded to the User based upon this Agreement will be terminated. You shall ease use of the Product and uninstall related software; + + （b）The User shall return all confidential information and the copy (includes but not restricted to Product) back to **ZILLIZ**, or destroy all copy of confidential information on permission of **ZILLIZ**. Without the written approval of **ZILLIZ**, the User is not allowed to keep any confidential information or its copy provided by **ZILLIZ**. + +13. **Third-party Resources**. Products supplied by **ZILLIZ** may include hyperlinks to other websites, content or resources ("Third Party Resources"), and the normal use of such products may depend on the availability of third party resources. **ZILLIZ** is unable to control any third-party resources. The User acknowledges and agrees that **ZILLIZ** is not responsible for the availability and security of third-party resources and does not guarantee any advertising, products or other materials that are or are derived from such third party resources. The User acknowledges and agrees that **ZILLIZ** shall not hold obligations about any liability for loss or damage that may be suffered due to the availability and security of third party resources, or the integrity or accuracy of any advertisements, products or other materials that the User relies on or obtains from third party resources. + +14. **Other**. The entire contents of this Agreement are performed within the territory of the People's Republic of China and are governed by and construed in accordance with the laws of the People's Republic of China (but not applicable to the relevant conflict laws). **ZILLIZ** agrees that any disputes relating to this Agreement will be submitted to the Xuhui District People's Court of Shanghai, and irrevocably and unconditionally agree that the above courts have exclusive jurisdiction over all litigations and disputes brought about by this Agreement. Once it is determined that any provision is invalid, illegal or unenforceable, **ZILLIZ** reserves the right to modify and interpret the terms. Any notice that needs to be sent to the user, if posted on the **ZILLIZ** website, is deemed to have been validly and legally sent to the user. Except for the obligation to pay under this contract, neither party will be liable for failure to perform or delayed performance of this Agreement in whole or in part due to force majeure. The force majeure includes but is not limited to fire, storm, flood , earthquake, civil strife, telecommunications disruption, power outage or other infrastructure disruption, service interruption or termination caused by **ZILLIZ** service provider problems, strikes, intentional destruction events, cable cuts, virus intrusion or any other similar incidents caused by intentional or illegal acts by third parties. In the case of the above-mentioned delayed performance, the delay in fulfilling the agreement may be the delay time due to the above reasons. Unless otherwise stated in this Agreement, notices or communications required or endorsed by this Agreement must be signed or authorized in writing by a party, and delivered by direct delivery, overnight delivery, confirmed email, confirmed fax or by mailing a registered letter, registered mail, and returning the order, etc. Any modification, addition or deletion or waiver of this Agreement must be confirmed by a written confirmation by a suitably authorized representative of both parties. The non-performance or delay in the performance of any right or remedy by any party (partially or wholly) does not constitute a waiver of such rights or remedies, nor does it affect any other rights or remedies. All claims and remedies under this Agreement may be cumulative and do not exclude any other rights or remedies contained in this Agreement or as required by law. Exemption from the waiver or delay of any liability for breach of contract in this Agreement does not constitute an exemption from other subsequent breach of contract obligations. \ No newline at end of file From e93892eae3ae7d1a5be746887578083c9279aa14 Mon Sep 17 00:00:00 2001 From: "xj.lin" Date: Tue, 2 Jul 2019 19:58:15 +0800 Subject: [PATCH 062/189] update README.md Former-commit-id: 4bceb291ad1c31c02fee159340233ed3a1a1402b --- cpp/README.md | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/cpp/README.md b/cpp/README.md index 1b2f507db2..b30cdbfae1 100644 --- a/cpp/README.md +++ b/cpp/README.md @@ -1,13 +1,12 @@ ### Compilation #### Step 1: install necessery tools - Install MySQL centos7 : - yum install gfortran qt4 flex bison mysql-devel + yum install gfortran qt4 flex bison mysql-devel mysql ubuntu16.04 : - sudo apt-get install gfortran qt4-qmake flex bison libmysqlclient-dev + sudo apt-get install gfortran qt4-qmake flex bison libmysqlclient-dev mysql-client If `libmysqlclient_r.so` does not exist after installing MySQL Development Files, you need to create a symbolic link: @@ -53,10 +52,10 @@ If you encounter the following error when building: ### Launch server Set config in cpp/conf/server_config.yaml -Add milvus/bin/lib to LD_LIBRARY_PATH +Add milvus/lib to LD_LIBRARY_PATH ``` -export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/path/to/milvus/bin/lib +export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/path/to/milvus/lib ``` Then launch server with config: From a8c4def6666b37850df7739df8c91a76ad986d72 Mon Sep 17 00:00:00 2001 From: yu yunfeng Date: Tue, 2 Jul 2019 19:36:06 +0800 Subject: [PATCH 063/189] alter nprobe Former-commit-id: 6cd1ba96ac2ca3831a022337ba79026db357822c --- cpp/src/wrapper/Index.cpp | 31 ++++++++++++++++++- cpp/src/wrapper/Operand.cpp | 2 +- .../server/appendix/server_config.yaml | 3 ++ 3 files changed, 34 insertions(+), 2 deletions(-) diff --git a/cpp/src/wrapper/Index.cpp b/cpp/src/wrapper/Index.cpp index c306b65e1a..b051bfb07e 100644 --- a/cpp/src/wrapper/Index.cpp +++ b/cpp/src/wrapper/Index.cpp @@ -14,6 +14,8 @@ #include "Index.h" #include "faiss/index_io.h" #include "faiss/IndexIVF.h" +#include +#include "server/ServerConfig.h" namespace zilliz { namespace milvus { @@ -23,6 +25,32 @@ using std::string; using std::unordered_map; using std::vector; +class Nprobe { + public: + static Nprobe &GetInstance() { + static Nprobe instance; + return instance; + } + + void SelectNprobe() { + using namespace zilliz::milvus::server; + ServerConfig &config = ServerConfig::GetInstance(); + ConfigNode engine_config = config.GetConfig(CONFIG_ENGINE); + nprobe_ = engine_config.GetInt32Value(CONFIG_NPROBE, 1000); + } + + size_t GetNprobe() { + return nprobe_; + } + + private: + Nprobe() : nprobe_(1000) { SelectNprobe(); } + + private: + size_t nprobe_; +}; + + Index::Index(const std::shared_ptr &raw_index) { index_ = raw_index; dim = index_->d; @@ -57,7 +85,8 @@ bool Index::add_with_ids(idx_t n, const float *xdata, const long *xids) { bool Index::search(idx_t n, const float *data, idx_t k, float *distances, long *labels) const { try { if(auto ivf_index = std::dynamic_pointer_cast(index_)) { - ivf_index->nprobe = 100; + ivf_index->nprobe = Nprobe::GetInstance().GetNprobe(); + std::cout << "nprobe = " << ivf_index->nprobe << std::endl; } index_->search(n, data, k, distances, labels); } diff --git a/cpp/src/wrapper/Operand.cpp b/cpp/src/wrapper/Operand.cpp index 30e31067fd..25341676a6 100644 --- a/cpp/src/wrapper/Operand.cpp +++ b/cpp/src/wrapper/Operand.cpp @@ -39,7 +39,7 @@ string Operand::get_index_type(const int &nb) { } case IVF: { index_str += (ncent != 0 ? index_type + std::to_string(ncent) : - index_type + std::to_string(int(nb / 1000000.0 * 1638))); + index_type + std::to_string(int(nb / 1000000.0 * 16384))); break; } case IDMAP: { diff --git a/cpp/unittest/server/appendix/server_config.yaml b/cpp/unittest/server/appendix/server_config.yaml index 9019461940..359f8d86bf 100644 --- a/cpp/unittest/server/appendix/server_config.yaml +++ b/cpp/unittest/server/appendix/server_config.yaml @@ -26,3 +26,6 @@ license_config: # license configure cache_config: # cache configure cpu_cache_capacity: 16 # how many memory are used as cache, unit: GB, range: 0 ~ less than total memory + +engine_config: + nprobe: 3000 \ No newline at end of file From a0a83a5de02aeda45bc0c78e8dd62b6f7abc9dc4 Mon Sep 17 00:00:00 2001 From: jinhai Date: Tue, 2 Jul 2019 20:06:00 +0800 Subject: [PATCH 064/189] Fix typo Former-commit-id: d5ca33aa338466c319b44a2564bd250a384f0085 --- cpp/src/main.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/src/main.cpp b/cpp/src/main.cpp index bfdafcc1c3..ec536c2ee2 100644 --- a/cpp/src/main.cpp +++ b/cpp/src/main.cpp @@ -26,7 +26,7 @@ using namespace zilliz::milvus; int main(int argc, char *argv[]) { - std::cout << std::endl << "Welcome to use Milvus by Zillz!" << std::endl; + std::cout << std::endl << "Welcome to use Milvus by Zilliz!" << std::endl; std::cout << "Milvus " << BUILD_TYPE << " version: v" << MILVUS_VERSION << " built at " << BUILD_TIME << std::endl; signal(SIGINT, server::SignalUtil::HandleSignal); From ee4cf853bc307c9dbf81bcb118a198b2a302fbad Mon Sep 17 00:00:00 2001 From: yu yunfeng Date: Tue, 2 Jul 2019 20:13:10 +0800 Subject: [PATCH 065/189] add engine config Former-commit-id: 2255522486d6b5cf2c1ab391c9d70f09c3e64e1c --- cpp/src/server/ServerConfig.h | 3 +++ 1 file changed, 3 insertions(+) diff --git a/cpp/src/server/ServerConfig.h b/cpp/src/server/ServerConfig.h index f337275a46..412581bc1f 100644 --- a/cpp/src/server/ServerConfig.h +++ b/cpp/src/server/ServerConfig.h @@ -43,6 +43,9 @@ static const std::string CONFIG_METRIC_COLLECTOR = "collector"; static const std::string CONFIG_PROMETHEUS = "prometheus_config"; static const std::string CONFIG_METRIC_PROMETHEUS_PORT = "port"; +static const std::string CONFIG_ENGINE = "engine_config"; +static const std::string CONFIG_NPROBE = "nprobe"; + class ServerConfig { public: static ServerConfig &GetInstance(); From 6ae16689ff29823df23674d3e8b2426fff942f3b Mon Sep 17 00:00:00 2001 From: yu yunfeng Date: Tue, 2 Jul 2019 20:24:41 +0800 Subject: [PATCH 066/189] update Former-commit-id: 4fea994502a6f8930efa1f2d821c1bbaaae9f4e2 --- cpp/src/wrapper/Index.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/cpp/src/wrapper/Index.cpp b/cpp/src/wrapper/Index.cpp index b051bfb07e..57c462a201 100644 --- a/cpp/src/wrapper/Index.cpp +++ b/cpp/src/wrapper/Index.cpp @@ -14,7 +14,7 @@ #include "Index.h" #include "faiss/index_io.h" #include "faiss/IndexIVF.h" -#include +#include "faiss/IVFlib.h" #include "server/ServerConfig.h" namespace zilliz { @@ -86,7 +86,6 @@ bool Index::search(idx_t n, const float *data, idx_t k, float *distances, long * try { if(auto ivf_index = std::dynamic_pointer_cast(index_)) { ivf_index->nprobe = Nprobe::GetInstance().GetNprobe(); - std::cout << "nprobe = " << ivf_index->nprobe << std::endl; } index_->search(n, data, k, distances, labels); } From 0d25f30b38f0ef835e606195bc73f7897a6d13d9 Mon Sep 17 00:00:00 2001 From: yu yunfeng Date: Wed, 3 Jul 2019 11:09:17 +0800 Subject: [PATCH 067/189] update IVF nprobe Former-commit-id: d2e93de87f9c8b45551f1453b5c0303508cab0c5 --- cpp/conf/server_config.template | 5 ++++- cpp/src/db/EngineFactory.cpp | 34 +++++++++++++++++++++-------- cpp/src/db/ExecutionEngine.h | 2 ++ cpp/src/db/FaissExecutionEngine.cpp | 30 ++++++++++++++++++++++++- cpp/src/db/FaissExecutionEngine.h | 9 ++++---- cpp/src/wrapper/Index.cpp | 29 ------------------------ 6 files changed, 65 insertions(+), 44 deletions(-) diff --git a/cpp/conf/server_config.template b/cpp/conf/server_config.template index 6b432056c5..260ede4f2d 100644 --- a/cpp/conf/server_config.template +++ b/cpp/conf/server_config.template @@ -30,4 +30,7 @@ license_config: # license configure license_path: "@MILVUS_DB_PATH@/system.license" # license file path cache_config: # cache configure - cpu_cache_capacity: 16 # how many memory are used as cache, unit: GB, range: 0 ~ less than total memory \ No newline at end of file + cpu_cache_capacity: 16 # how many memory are used as cache, unit: GB, range: 0 ~ less than total memory + +engine_config: + nprobe: 10 \ No newline at end of file diff --git a/cpp/src/db/EngineFactory.cpp b/cpp/src/db/EngineFactory.cpp index 26ef639c88..bacce70ce4 100644 --- a/cpp/src/db/EngineFactory.cpp +++ b/cpp/src/db/EngineFactory.cpp @@ -7,23 +7,39 @@ #include "FaissExecutionEngine.h" #include "Log.h" + namespace zilliz { namespace milvus { namespace engine { ExecutionEnginePtr EngineFactory::Build(uint16_t dimension, - const std::string& location, - EngineType type) { - switch(type) { - case EngineType::FAISS_IDMAP: - return ExecutionEnginePtr(new FaissExecutionEngine(dimension, location, "IDMap", "IDMap,Flat")); - case EngineType::FAISS_IVFFLAT: - return ExecutionEnginePtr(new FaissExecutionEngine(dimension, location, "IVF", "IDMap,Flat")); - default: - ENGINE_LOG_ERROR << "Unsupportted engine type"; + const std::string &location, + EngineType type) { + + ExecutionEnginePtr execution_engine_ptr; + + switch (type) { + case EngineType::FAISS_IDMAP: { + execution_engine_ptr = + ExecutionEnginePtr(new FaissExecutionEngine(dimension, location, "IDMap", "IDMap,Flat")); + break; + } + + case EngineType::FAISS_IVFFLAT: { + execution_engine_ptr = + ExecutionEnginePtr(new FaissExecutionEngine(dimension, location, "IVF", "IDMap,Flat")); + break; + } + + default: { + ENGINE_LOG_ERROR << "Unsupported engine type"; return nullptr; + } } + + execution_engine_ptr->Init(); + return execution_engine_ptr; } } diff --git a/cpp/src/db/ExecutionEngine.h b/cpp/src/db/ExecutionEngine.h index d2b4d01e67..f8c05f6f9d 100644 --- a/cpp/src/db/ExecutionEngine.h +++ b/cpp/src/db/ExecutionEngine.h @@ -50,6 +50,8 @@ public: virtual std::shared_ptr BuildIndex(const std::string&) = 0; virtual Status Cache() = 0; + + virtual Status Init() = 0; }; using ExecutionEnginePtr = std::shared_ptr; diff --git a/cpp/src/db/FaissExecutionEngine.cpp b/cpp/src/db/FaissExecutionEngine.cpp index 9dfdd978c3..20bd530e78 100644 --- a/cpp/src/db/FaissExecutionEngine.cpp +++ b/cpp/src/db/FaissExecutionEngine.cpp @@ -13,6 +13,7 @@ #include #include #include +#include "faiss/IndexIVF.h" #include "metrics/Metrics.h" @@ -135,7 +136,16 @@ Status FaissExecutionEngine::Search(long n, float *distances, long *labels) const { auto start_time = METRICS_NOW_TIME; - pIndex_->search(n, data, k, distances, labels); + + std::shared_ptr ivf_index = std::dynamic_pointer_cast(pIndex_); + if(ivf_index) { + ENGINE_LOG_DEBUG << "Index type: IVFFLAT nProbe: " << nprobe_; + ivf_index->nprobe = nprobe_; + ivf_index->search(n, data, k, distances, labels); + } else { + pIndex_->search(n, data, k, distances, labels); + } + auto end_time = METRICS_NOW_TIME; auto total_time = METRICS_MICROSECONDS(start_time,end_time); server::Metrics::GetInstance().QueryIndexTypePerSecondSet(build_index_type_, double(n)/double(total_time)); @@ -149,6 +159,24 @@ Status FaissExecutionEngine::Cache() { return Status::OK(); } +Status FaissExecutionEngine::Init() { + + if(build_index_type_ == "IVF") { + + using namespace zilliz::milvus::server; + ServerConfig &config = ServerConfig::GetInstance(); + ConfigNode engine_config = config.GetConfig(CONFIG_ENGINE); + nprobe_ = engine_config.GetInt32Value(CONFIG_NPROBE, 1000); + + } else if(build_index_type_ == "IDMap") { + ; + } else { + return Status::Error("Wrong index type: ", build_index_type_); + } + + return Status::OK(); +} + } // namespace engine } // namespace milvus diff --git a/cpp/src/db/FaissExecutionEngine.h b/cpp/src/db/FaissExecutionEngine.h index 5667df34ea..f9f37ad978 100644 --- a/cpp/src/db/FaissExecutionEngine.h +++ b/cpp/src/db/FaissExecutionEngine.h @@ -6,14 +6,11 @@ #pragma once #include "ExecutionEngine.h" +#include "faiss/Index.h" #include #include -namespace faiss { - class Index; -} - namespace zilliz { namespace milvus { namespace engine { @@ -58,12 +55,16 @@ public: Status Cache() override; + Status Init() override; + protected: std::shared_ptr pIndex_; std::string location_; std::string build_index_type_; std::string raw_index_type_; + + size_t nprobe_ = 0; }; diff --git a/cpp/src/wrapper/Index.cpp b/cpp/src/wrapper/Index.cpp index 57c462a201..18e20d830a 100644 --- a/cpp/src/wrapper/Index.cpp +++ b/cpp/src/wrapper/Index.cpp @@ -25,32 +25,6 @@ using std::string; using std::unordered_map; using std::vector; -class Nprobe { - public: - static Nprobe &GetInstance() { - static Nprobe instance; - return instance; - } - - void SelectNprobe() { - using namespace zilliz::milvus::server; - ServerConfig &config = ServerConfig::GetInstance(); - ConfigNode engine_config = config.GetConfig(CONFIG_ENGINE); - nprobe_ = engine_config.GetInt32Value(CONFIG_NPROBE, 1000); - } - - size_t GetNprobe() { - return nprobe_; - } - - private: - Nprobe() : nprobe_(1000) { SelectNprobe(); } - - private: - size_t nprobe_; -}; - - Index::Index(const std::shared_ptr &raw_index) { index_ = raw_index; dim = index_->d; @@ -84,9 +58,6 @@ bool Index::add_with_ids(idx_t n, const float *xdata, const long *xids) { bool Index::search(idx_t n, const float *data, idx_t k, float *distances, long *labels) const { try { - if(auto ivf_index = std::dynamic_pointer_cast(index_)) { - ivf_index->nprobe = Nprobe::GetInstance().GetNprobe(); - } index_->search(n, data, k, distances, labels); } catch (std::exception &e) { From ca5d12b6e4ac87ffb55f8ac6024607a73c41a764 Mon Sep 17 00:00:00 2001 From: yu yunfeng Date: Wed, 3 Jul 2019 11:18:34 +0800 Subject: [PATCH 068/189] change CHANGELOG Former-commit-id: 3af614370acd069b8af356536ab7a3e47a750aff --- cpp/CHANGELOG.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/cpp/CHANGELOG.md b/cpp/CHANGELOG.md index 26c96b907b..668ac1ba02 100644 --- a/cpp/CHANGELOG.md +++ b/cpp/CHANGELOG.md @@ -60,6 +60,9 @@ Please mark all change in change log and use the ticket from JIRA. - MS-96 - add new query interface for specified files - MS-97 - Add S3 SDK for MinIO Storage - MS-105 - Add MySQL +- MS-130 - Add prometheus_test +- MS-144 - Add nprobe config +- MS-147 - Enable IVF - MS-130 - Add prometheus_test ## Task From f267b5646f874997891df5d92909ae2a48a426b4 Mon Sep 17 00:00:00 2001 From: "peng.xu" Date: Wed, 3 Jul 2019 15:17:28 +0800 Subject: [PATCH 069/189] Revert "Merge branch 'integrate_knowhere-0.3.1' into 'branch-0.3.1'" This reverts merge request !152 Former-commit-id: 96413424d145e007f2a6ce1956b4bc57bd880486 --- cpp/README.md | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/cpp/README.md b/cpp/README.md index b30cdbfae1..1b2f507db2 100644 --- a/cpp/README.md +++ b/cpp/README.md @@ -1,12 +1,13 @@ ### Compilation #### Step 1: install necessery tools + Install MySQL centos7 : - yum install gfortran qt4 flex bison mysql-devel mysql + yum install gfortran qt4 flex bison mysql-devel ubuntu16.04 : - sudo apt-get install gfortran qt4-qmake flex bison libmysqlclient-dev mysql-client + sudo apt-get install gfortran qt4-qmake flex bison libmysqlclient-dev If `libmysqlclient_r.so` does not exist after installing MySQL Development Files, you need to create a symbolic link: @@ -52,10 +53,10 @@ If you encounter the following error when building: ### Launch server Set config in cpp/conf/server_config.yaml -Add milvus/lib to LD_LIBRARY_PATH +Add milvus/bin/lib to LD_LIBRARY_PATH ``` -export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/path/to/milvus/lib +export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/path/to/milvus/bin/lib ``` Then launch server with config: From 28cb8ae5075d67a6d0b47098f114b9f5aa0ac678 Mon Sep 17 00:00:00 2001 From: quicksilver Date: Wed, 3 Jul 2019 15:54:15 +0800 Subject: [PATCH 070/189] setting script file permissions to 700 Former-commit-id: d2b4c02d842c34207ca95589436f2e37c3b79705 --- cpp/CMakeLists.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 4d8f43b5b9..c58d63178a 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -145,6 +145,7 @@ configure_file(${CMAKE_CURRENT_SOURCE_DIR}/conf/log_config.template ${CMAKE_CURR install(FILES scripts/start_server.sh scripts/stop_server.sh + FILE_PERMISSIONS OWNER_EXECUTE OWNER_WRITE OWNER_READ DESTINATION scripts) install(FILES From dcc04ae5ecb45176d57e7a1358ab2a6cd5506d0a Mon Sep 17 00:00:00 2001 From: quicksilver Date: Wed, 3 Jul 2019 16:11:18 +0800 Subject: [PATCH 071/189] setting script file permissions to 744 Former-commit-id: 78ab09c270b127e3ae7855008640618bf2177cc6 --- cpp/CMakeLists.txt | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index c58d63178a..18abb9a72c 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -142,12 +142,12 @@ configure_file(${CMAKE_CURRENT_SOURCE_DIR}/conf/server_config.template ${CMAKE_C configure_file(${CMAKE_CURRENT_SOURCE_DIR}/conf/log_config.template ${CMAKE_CURRENT_SOURCE_DIR}/conf/log_config.conf) #install -install(FILES - scripts/start_server.sh - scripts/stop_server.sh +install(DIRECTORY scripts + DESTINATION scripts FILE_PERMISSIONS OWNER_EXECUTE OWNER_WRITE OWNER_READ - DESTINATION - scripts) + GROUP_EXECUTE GROUP_READ + WORLD_EXECUTE WORLD_READ + FILES_MATCHING PATTERN "*.sh") install(FILES conf/server_config.yaml conf/log_config.conf From e7ad142a080d94e94ea521244f82713d50e5e986 Mon Sep 17 00:00:00 2001 From: quicksilver Date: Wed, 3 Jul 2019 16:35:19 +0800 Subject: [PATCH 072/189] setting script file permissions to 755 Former-commit-id: 768f01856137fef36250df7bf3c1158738445c0d --- cpp/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index 18abb9a72c..3a0e8900a6 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -142,7 +142,7 @@ configure_file(${CMAKE_CURRENT_SOURCE_DIR}/conf/server_config.template ${CMAKE_C configure_file(${CMAKE_CURRENT_SOURCE_DIR}/conf/log_config.template ${CMAKE_CURRENT_SOURCE_DIR}/conf/log_config.conf) #install -install(DIRECTORY scripts +install(DIRECTORY scripts/ DESTINATION scripts FILE_PERMISSIONS OWNER_EXECUTE OWNER_WRITE OWNER_READ GROUP_EXECUTE GROUP_READ From 24e533a3a333c8580ef21d0801ae853a57f8f52d Mon Sep 17 00:00:00 2001 From: zhiru Date: Wed, 3 Jul 2019 14:25:02 +0800 Subject: [PATCH 073/189] Disable cleanup if mode is read only Former-commit-id: 8bb50a5c070e1f8bb7c73191481a476f08ee7281 --- cpp/CHANGELOG.md | 1 + cpp/src/db/MySQLMetaImpl.cpp | 6 ++++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/cpp/CHANGELOG.md b/cpp/CHANGELOG.md index 668ac1ba02..bbc4ad150b 100644 --- a/cpp/CHANGELOG.md +++ b/cpp/CHANGELOG.md @@ -25,6 +25,7 @@ Please mark all change in change log and use the ticket from JIRA. - MS-90 - Fix arch match incorrect on ARM - MS-99 - Fix compilation bug - MS-110 - Avoid huge file size +- MS-148 - Disable cleanup if mode is read only ## Improvement - MS-82 - Update server startup welcome message diff --git a/cpp/src/db/MySQLMetaImpl.cpp b/cpp/src/db/MySQLMetaImpl.cpp index c226abf56b..ebb56af809 100644 --- a/cpp/src/db/MySQLMetaImpl.cpp +++ b/cpp/src/db/MySQLMetaImpl.cpp @@ -162,7 +162,9 @@ namespace meta { try { - CleanUp(); + if (mode_ != Options::MODE::READ_ONLY) { + CleanUp(); + } { ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab); @@ -457,7 +459,7 @@ namespace meta { } //Scoped Connection - if (mode_ != Options::MODE::SINGLE) { + if (mode_ == Options::MODE::CLUSTER) { DeleteTableFiles(table_id); } From ad606851fa664873a22b1703c3feb13752d70a42 Mon Sep 17 00:00:00 2001 From: zhiru Date: Wed, 3 Jul 2019 14:27:19 +0800 Subject: [PATCH 074/189] Disable cleanup if mode is read only Former-commit-id: 8de4aa4c6c81e4d6840441527c164bb26fe948e4 --- cpp/src/db/MySQLMetaImpl.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/cpp/src/db/MySQLMetaImpl.cpp b/cpp/src/db/MySQLMetaImpl.cpp index ebb56af809..ab02419969 100644 --- a/cpp/src/db/MySQLMetaImpl.cpp +++ b/cpp/src/db/MySQLMetaImpl.cpp @@ -1814,7 +1814,9 @@ namespace meta { MySQLMetaImpl::~MySQLMetaImpl() { // std::lock_guard lock(mysql_mutex); - CleanUp(); + if (mode_ != Options::MODE::READ_ONLY) { + CleanUp(); + } } } // namespace meta From 1ecae09ab975e62a7eb50c90ff09851b87262b7e Mon Sep 17 00:00:00 2001 From: zhiru Date: Wed, 3 Jul 2019 15:48:21 +0800 Subject: [PATCH 075/189] update Former-commit-id: 0b1cecd26527c40069ef869028afd977304e34da --- cpp/src/server/RequestTask.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/cpp/src/server/RequestTask.cpp b/cpp/src/server/RequestTask.cpp index f312b7e605..0d432ccd90 100644 --- a/cpp/src/server/RequestTask.cpp +++ b/cpp/src/server/RequestTask.cpp @@ -482,6 +482,7 @@ ServerError SearchVectorTask::OnExecute() { engine::QueryResults results; uint64_t record_count = (uint64_t)record_array_.size(); + SERVER_LOG_DEBUG << file_id_array_ << std::endl; if(file_id_array_.empty()) { stat = DBWrapper::DB()->Query(table_name_, (size_t) top_k_, record_count, vec_f.data(), dates, results); } else { From 3d2b5006cee842d326b3aaacb9b48fbf073d735d Mon Sep 17 00:00:00 2001 From: zhiru Date: Wed, 3 Jul 2019 16:04:00 +0800 Subject: [PATCH 076/189] update Former-commit-id: ebe0147420bf01f3efe61b6b64bab27dc1f46acb --- cpp/src/server/RequestTask.cpp | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/cpp/src/server/RequestTask.cpp b/cpp/src/server/RequestTask.cpp index 0d432ccd90..682cdd3578 100644 --- a/cpp/src/server/RequestTask.cpp +++ b/cpp/src/server/RequestTask.cpp @@ -482,7 +482,11 @@ ServerError SearchVectorTask::OnExecute() { engine::QueryResults results; uint64_t record_count = (uint64_t)record_array_.size(); - SERVER_LOG_DEBUG << file_id_array_ << std::endl; + SERVER_LOG_DEBUG << "file_id_array_: "; + for (auto& file_id : file_id_array_) { + SERVER_LOG_DEBUG << file_id; + } + if(file_id_array_.empty()) { stat = DBWrapper::DB()->Query(table_name_, (size_t) top_k_, record_count, vec_f.data(), dates, results); } else { From 74b0ab8eb93accd9bfa22e1ca60a2a955530212b Mon Sep 17 00:00:00 2001 From: zhiru Date: Wed, 3 Jul 2019 17:24:36 +0800 Subject: [PATCH 077/189] update Former-commit-id: 99808a6e2bf44a240b4f72d5e6aa9b72c887c0fa --- cpp/src/db/DBImpl.cpp | 4 ++++ cpp/src/db/MySQLMetaImpl.cpp | 10 ++++++---- cpp/src/server/RequestHandler.cpp | 2 ++ cpp/src/server/RequestTask.cpp | 5 ----- 4 files changed, 12 insertions(+), 9 deletions(-) diff --git a/cpp/src/db/DBImpl.cpp b/cpp/src/db/DBImpl.cpp index 46101dcf93..83aa70f563 100644 --- a/cpp/src/db/DBImpl.cpp +++ b/cpp/src/db/DBImpl.cpp @@ -196,6 +196,10 @@ Status DBImpl::Query(const std::string& table_id, const std::vector return status; } + for (auto& file_schema : files_array) { + ENGINE_LOG_DEBUG << "file_id: " << file_schema.file_id_; + } + if(files_array.empty()) { return Status::Error("Invalid file id"); } diff --git a/cpp/src/db/MySQLMetaImpl.cpp b/cpp/src/db/MySQLMetaImpl.cpp index ab02419969..22da874a45 100644 --- a/cpp/src/db/MySQLMetaImpl.cpp +++ b/cpp/src/db/MySQLMetaImpl.cpp @@ -1083,10 +1083,10 @@ namespace meta { // } Query getTableFileQuery = connectionPtr->query(); - getTableFileQuery << "SELECT engine_type, file_id, file_type, size, date " << - "FROM TableFiles " << - "WHERE table_id = " << quote << table_id << " AND " << - "(" << idStr << ");"; + getTableFileQuery << "SELECT id, engine_type, file_id, file_type, size, date " << + "FROM TableFiles " << + "WHERE table_id = " << quote << table_id << " AND " << + "(" << idStr << ");"; ENGINE_LOG_DEBUG << "MySQLMetaImpl::GetTableFiles: " << getTableFileQuery.str(); @@ -1106,6 +1106,8 @@ namespace meta { TableFileSchema file_schema; + file_schema.id_ = resRow["id"]; + file_schema.table_id_ = table_id; file_schema.engine_type_ = resRow["engine_type"]; diff --git a/cpp/src/server/RequestHandler.cpp b/cpp/src/server/RequestHandler.cpp index 037f80e0db..a4dc182c35 100644 --- a/cpp/src/server/RequestHandler.cpp +++ b/cpp/src/server/RequestHandler.cpp @@ -53,6 +53,7 @@ RequestHandler::SearchVector(std::vector &_return, const std::vector &query_record_array, const std::vector &query_range_array, const int64_t topk) { +// SERVER_LOG_DEBUG << "Entering RequestHandler::SearchVector"; BaseTaskPtr task_ptr = SearchVectorTask::Create(table_name, std::vector(), query_record_array, query_range_array, topk, _return); RequestScheduler::ExecTask(task_ptr); @@ -65,6 +66,7 @@ RequestHandler::SearchVectorInFiles(std::vector<::milvus::thrift::TopKQueryResul const std::vector<::milvus::thrift::RowRecord> &query_record_array, const std::vector<::milvus::thrift::Range> &query_range_array, const int64_t topk) { +// SERVER_LOG_DEBUG << "Entering RequestHandler::SearchVectorInFiles. file_id_array size = " << std::to_string(file_id_array.size()); BaseTaskPtr task_ptr = SearchVectorTask::Create(table_name, file_id_array, query_record_array, query_range_array, topk, _return); RequestScheduler::ExecTask(task_ptr); diff --git a/cpp/src/server/RequestTask.cpp b/cpp/src/server/RequestTask.cpp index 682cdd3578..f312b7e605 100644 --- a/cpp/src/server/RequestTask.cpp +++ b/cpp/src/server/RequestTask.cpp @@ -482,11 +482,6 @@ ServerError SearchVectorTask::OnExecute() { engine::QueryResults results; uint64_t record_count = (uint64_t)record_array_.size(); - SERVER_LOG_DEBUG << "file_id_array_: "; - for (auto& file_id : file_id_array_) { - SERVER_LOG_DEBUG << file_id; - } - if(file_id_array_.empty()) { stat = DBWrapper::DB()->Query(table_name_, (size_t) top_k_, record_count, vec_f.data(), dates, results); } else { From 72a1e54acceba46b616d73ed138c9ae61b970f1a Mon Sep 17 00:00:00 2001 From: zhiru Date: Wed, 3 Jul 2019 18:31:45 +0800 Subject: [PATCH 078/189] update Former-commit-id: 021df2529cba274cabb92b4e6a1cc77186a2ebf5 --- cpp/src/db/DBImpl.cpp | 4 ---- 1 file changed, 4 deletions(-) diff --git a/cpp/src/db/DBImpl.cpp b/cpp/src/db/DBImpl.cpp index 83aa70f563..46101dcf93 100644 --- a/cpp/src/db/DBImpl.cpp +++ b/cpp/src/db/DBImpl.cpp @@ -196,10 +196,6 @@ Status DBImpl::Query(const std::string& table_id, const std::vector return status; } - for (auto& file_schema : files_array) { - ENGINE_LOG_DEBUG << "file_id: " << file_schema.file_id_; - } - if(files_array.empty()) { return Status::Error("Invalid file id"); } From 4cfee2499571e0a143ba88b8d9117ec303fa2ba8 Mon Sep 17 00:00:00 2001 From: zhiru Date: Wed, 3 Jul 2019 18:44:29 +0800 Subject: [PATCH 079/189] update Former-commit-id: 4b5f386c0207addda63030c941df6453e5e0afaa --- cpp/CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/cpp/CHANGELOG.md b/cpp/CHANGELOG.md index bbc4ad150b..eb933db3c3 100644 --- a/cpp/CHANGELOG.md +++ b/cpp/CHANGELOG.md @@ -26,6 +26,7 @@ Please mark all change in change log and use the ticket from JIRA. - MS-99 - Fix compilation bug - MS-110 - Avoid huge file size - MS-148 - Disable cleanup if mode is read only +- MS-149 - Fixed searching only one index file issue in distributed mode ## Improvement - MS-82 - Update server startup welcome message From a954f7a46321c8057572e7fa904ede7998f24f7e Mon Sep 17 00:00:00 2001 From: yu yunfeng Date: Wed, 3 Jul 2019 20:07:11 +0800 Subject: [PATCH 080/189] acc test Former-commit-id: 73a1cf40d4227a8e30fa00c43d67f4483592eccb --- cpp/src/db/DBMetaImpl.cpp | 14 ++++++++------ cpp/src/db/FaissExecutionEngine.cpp | 1 + 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/cpp/src/db/DBMetaImpl.cpp b/cpp/src/db/DBMetaImpl.cpp index 8c56c863e7..d13899dca0 100644 --- a/cpp/src/db/DBMetaImpl.cpp +++ b/cpp/src/db/DBMetaImpl.cpp @@ -612,7 +612,8 @@ Status DBMetaImpl::GetTableFiles(const std::string& table_id, TableFilesSchema& table_files) { try { table_files.clear(); - auto files = ConnectorPtr->select(columns(&TableFileSchema::file_id_, + auto files = ConnectorPtr->select(columns(&TableFileSchema::id_, + &TableFileSchema::file_id_, &TableFileSchema::file_type_, &TableFileSchema::size_, &TableFileSchema::date_, @@ -631,11 +632,12 @@ Status DBMetaImpl::GetTableFiles(const std::string& table_id, for (auto &file : files) { TableFileSchema file_schema; file_schema.table_id_ = table_id; - file_schema.file_id_ = std::get<0>(file); - file_schema.file_type_ = std::get<1>(file); - file_schema.size_ = std::get<2>(file); - file_schema.date_ = std::get<3>(file); - file_schema.engine_type_ = std::get<4>(file); + file_schema.id_ = std::get<0>(file); + file_schema.file_id_ = std::get<1>(file); + file_schema.file_type_ = std::get<2>(file); + file_schema.size_ = std::get<3>(file); + file_schema.date_ = std::get<4>(file); + file_schema.engine_type_ = std::get<5>(file); file_schema.dimension_ = table_schema.dimension_; GetTableFilePath(file_schema); diff --git a/cpp/src/db/FaissExecutionEngine.cpp b/cpp/src/db/FaissExecutionEngine.cpp index 20bd530e78..201c07dbcf 100644 --- a/cpp/src/db/FaissExecutionEngine.cpp +++ b/cpp/src/db/FaissExecutionEngine.cpp @@ -138,6 +138,7 @@ Status FaissExecutionEngine::Search(long n, auto start_time = METRICS_NOW_TIME; std::shared_ptr ivf_index = std::dynamic_pointer_cast(pIndex_); + //ENGINE_LOG_DEBUG << "Index nlist: " << ivf_index->nlist << ", ntotal: "<< ivf_index->ntotal; if(ivf_index) { ENGINE_LOG_DEBUG << "Index type: IVFFLAT nProbe: " << nprobe_; ivf_index->nprobe = nprobe_; From e22078a0c3fcc6788a30036fd980b915ec275275 Mon Sep 17 00:00:00 2001 From: jinhai Date: Wed, 3 Jul 2019 20:23:21 +0800 Subject: [PATCH 081/189] Remove unused code Former-commit-id: f6b16666a397b7e6c06f79795874ebd91baabe40 --- cpp/src/server/MilvusServer.cpp | 33 ++++++++++++--------------------- cpp/src/server/ServerConfig.h | 1 - 2 files changed, 12 insertions(+), 22 deletions(-) diff --git a/cpp/src/server/MilvusServer.cpp b/cpp/src/server/MilvusServer.cpp index 6e6d53c565..53bdc07dde 100644 --- a/cpp/src/server/MilvusServer.cpp +++ b/cpp/src/server/MilvusServer.cpp @@ -51,7 +51,6 @@ MilvusServer::StartService() { std::string address = server_config.GetValue(CONFIG_SERVER_ADDRESS, "127.0.0.1"); int32_t port = server_config.GetInt32Value(CONFIG_SERVER_PORT, 19530); std::string protocol = server_config.GetValue(CONFIG_SERVER_PROTOCOL, "binary"); - std::string mode = server_config.GetValue(CONFIG_SERVER_MODE, "thread_pool"); try { DBWrapper::DB();//initialize db @@ -69,30 +68,22 @@ MilvusServer::StartService() { } else if (protocol == "compact") { protocol_factory.reset(new TCompactProtocolFactory()); } else { - SERVER_LOG_ERROR << "Service protocol: " << protocol << " is not supported currently"; + // SERVER_LOG_INFO << "Service protocol: " << protocol << " is not supported currently"; return; } - std::string mode = "thread_pool"; - if (mode == "simple") { - s_server.reset(new TSimpleServer(processor, server_transport, transport_factory, protocol_factory)); - s_server->serve(); - } else if (mode == "thread_pool") { - stdcxx::shared_ptr threadManager(ThreadManager::newSimpleThreadManager()); - stdcxx::shared_ptr threadFactory(new PosixThreadFactory()); - threadManager->threadFactory(threadFactory); - threadManager->start(); + stdcxx::shared_ptr threadManager(ThreadManager::newSimpleThreadManager()); + stdcxx::shared_ptr threadFactory(new PosixThreadFactory()); + threadManager->threadFactory(threadFactory); + threadManager->start(); + + s_server.reset(new ThreadPoolServer(processor, + server_transport, + transport_factory, + protocol_factory, + threadManager)); + s_server->serve(); - s_server.reset(new ThreadPoolServer(processor, - server_transport, - transport_factory, - protocol_factory, - threadManager)); - s_server->serve(); - } else { - SERVER_LOG_ERROR << "Service mode: " << mode << " is not supported currently"; - return; - } } catch (apache::thrift::TException& ex) { std::cout << "ERROR! " << ex.what() << std::endl; kill(0, SIGUSR1); diff --git a/cpp/src/server/ServerConfig.h b/cpp/src/server/ServerConfig.h index 412581bc1f..0ec04eed8c 100644 --- a/cpp/src/server/ServerConfig.h +++ b/cpp/src/server/ServerConfig.h @@ -18,7 +18,6 @@ static const std::string CONFIG_SERVER = "server_config"; static const std::string CONFIG_SERVER_ADDRESS = "address"; static const std::string CONFIG_SERVER_PORT = "port"; static const std::string CONFIG_SERVER_PROTOCOL = "transfer_protocol"; -static const std::string CONFIG_SERVER_MODE = "server_mode"; static const std::string CONFIG_CLUSTER_MODE = "mode"; static const std::string CONFIG_DB = "db_config"; From f185a8b6060d87678644ee56978ec3b51a4422e1 Mon Sep 17 00:00:00 2001 From: yu yunfeng Date: Wed, 3 Jul 2019 20:59:36 +0800 Subject: [PATCH 082/189] ADD LOG Former-commit-id: 864d502d90f0b5e1b75da7eee8fcafc9a8c305dd --- cpp/src/db/scheduler/task/SearchTask.cpp | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/cpp/src/db/scheduler/task/SearchTask.cpp b/cpp/src/db/scheduler/task/SearchTask.cpp index d04f270331..80c7291753 100644 --- a/cpp/src/db/scheduler/task/SearchTask.cpp +++ b/cpp/src/db/scheduler/task/SearchTask.cpp @@ -167,6 +167,14 @@ std::shared_ptr SearchTask::Execute() { ClusterResult(output_ids, output_distence, context->nq(), inner_k, result_set); rc.Record("cluster result"); + + SERVER_LOG_DEBUG << "Query Result: "; + for(auto& id2score_vector: result_set) { + for(auto& pair: id2score_vector) { + SERVER_LOG_DEBUG << "id: " << pair.first << ", distance: " << pair.second; + } + } + //step 4: pick up topk result TopkResult(result_set, inner_k, context->GetResult()); rc.Record("reduce topk"); From 854dfeb85baf4700eb46771fda93a47963b25eae Mon Sep 17 00:00:00 2001 From: yu yunfeng Date: Thu, 4 Jul 2019 12:41:24 +0800 Subject: [PATCH 083/189] MS-151 Fix topk problem Former-commit-id: 40f952e4c6e1ed9f5ca3a300601c51d410c8e86b --- cpp/src/db/FaissExecutionEngine.cpp | 1 - cpp/src/db/scheduler/task/SearchTask.cpp | 8 -------- 2 files changed, 9 deletions(-) diff --git a/cpp/src/db/FaissExecutionEngine.cpp b/cpp/src/db/FaissExecutionEngine.cpp index 201c07dbcf..20bd530e78 100644 --- a/cpp/src/db/FaissExecutionEngine.cpp +++ b/cpp/src/db/FaissExecutionEngine.cpp @@ -138,7 +138,6 @@ Status FaissExecutionEngine::Search(long n, auto start_time = METRICS_NOW_TIME; std::shared_ptr ivf_index = std::dynamic_pointer_cast(pIndex_); - //ENGINE_LOG_DEBUG << "Index nlist: " << ivf_index->nlist << ", ntotal: "<< ivf_index->ntotal; if(ivf_index) { ENGINE_LOG_DEBUG << "Index type: IVFFLAT nProbe: " << nprobe_; ivf_index->nprobe = nprobe_; diff --git a/cpp/src/db/scheduler/task/SearchTask.cpp b/cpp/src/db/scheduler/task/SearchTask.cpp index 80c7291753..d04f270331 100644 --- a/cpp/src/db/scheduler/task/SearchTask.cpp +++ b/cpp/src/db/scheduler/task/SearchTask.cpp @@ -167,14 +167,6 @@ std::shared_ptr SearchTask::Execute() { ClusterResult(output_ids, output_distence, context->nq(), inner_k, result_set); rc.Record("cluster result"); - - SERVER_LOG_DEBUG << "Query Result: "; - for(auto& id2score_vector: result_set) { - for(auto& pair: id2score_vector) { - SERVER_LOG_DEBUG << "id: " << pair.first << ", distance: " << pair.second; - } - } - //step 4: pick up topk result TopkResult(result_set, inner_k, context->GetResult()); rc.Record("reduce topk"); From 2036586acfee466f5cad1e7f933917d602b8f903 Mon Sep 17 00:00:00 2001 From: starlord Date: Thu, 4 Jul 2019 13:02:47 +0800 Subject: [PATCH 084/189] reduce unittest time cost Former-commit-id: b568b26516a7894bf7a69471a3876f8222e2a180 --- cpp/unittest/db/db_tests.cpp | 2 +- cpp/unittest/db/mysql_db_test.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cpp/unittest/db/db_tests.cpp b/cpp/unittest/db/db_tests.cpp index d505320e86..bd17081af8 100644 --- a/cpp/unittest/db/db_tests.cpp +++ b/cpp/unittest/db/db_tests.cpp @@ -21,7 +21,7 @@ namespace { static const std::string TABLE_NAME = "test_group"; static constexpr int64_t TABLE_DIM = 256; static constexpr int64_t VECTOR_COUNT = 250000; - static constexpr int64_t INSERT_LOOP = 100000; + static constexpr int64_t INSERT_LOOP = 10000; engine::meta::TableSchema BuildTableSchema() { engine::meta::TableSchema table_info; diff --git a/cpp/unittest/db/mysql_db_test.cpp b/cpp/unittest/db/mysql_db_test.cpp index 85fe749460..a202859c3d 100644 --- a/cpp/unittest/db/mysql_db_test.cpp +++ b/cpp/unittest/db/mysql_db_test.cpp @@ -21,7 +21,7 @@ namespace { static const std::string TABLE_NAME = "test_group"; static constexpr int64_t TABLE_DIM = 256; static constexpr int64_t VECTOR_COUNT = 250000; - static constexpr int64_t INSERT_LOOP = 100000; + static constexpr int64_t INSERT_LOOP = 10000; engine::meta::TableSchema BuildTableSchema() { engine::meta::TableSchema table_info; From 434ba458cc8584c5dc13f0c88309d19f8d807947 Mon Sep 17 00:00:00 2001 From: zhiru Date: Thu, 4 Jul 2019 14:04:57 +0800 Subject: [PATCH 085/189] update Former-commit-id: 12267838390a1c152994efe234d79bc74e299f75 --- cpp/coverage.sh | 1 + cpp/src/db/MySQLConnectionPool.cpp | 78 +++++++++++++++++++ cpp/src/db/MySQLConnectionPool.h | 73 ++++++------------ cpp/src/db/MySQLMetaImpl.cpp | 116 ++++++++++++++++++++++++----- 4 files changed, 199 insertions(+), 69 deletions(-) create mode 100644 cpp/src/db/MySQLConnectionPool.cpp diff --git a/cpp/coverage.sh b/cpp/coverage.sh index 7638dad4e8..567e2d58cd 100755 --- a/cpp/coverage.sh +++ b/cpp/coverage.sh @@ -33,6 +33,7 @@ function mysql_exc() mysql_exc "CREATE DATABASE IF NOT EXISTS ${MYSQL_DB_NAME};" mysql_exc "GRANT ALL PRIVILEGES ON ${MYSQL_DB_NAME}.* TO '${MYSQL_USER_NAME}'@'%';" mysql_exc "FLUSH PRIVILEGES;" +mysql_exc "USE ${MYSQL_DB_NAME};" # get baseline ${LCOV_CMD} -c -i -d ${DIR_GCNO} -o "${FILE_INFO_BASE}" diff --git a/cpp/src/db/MySQLConnectionPool.cpp b/cpp/src/db/MySQLConnectionPool.cpp new file mode 100644 index 0000000000..b43126920e --- /dev/null +++ b/cpp/src/db/MySQLConnectionPool.cpp @@ -0,0 +1,78 @@ +#include "MySQLConnectionPool.h" + +namespace zilliz { +namespace milvus { +namespace engine { +namespace meta { + + // Do a simple form of in-use connection limiting: wait to return + // a connection until there are a reasonably low number in use + // already. Can't do this in create() because we're interested in + // connections actually in use, not those created. Also note that + // we keep our own count; ConnectionPool::size() isn't the same! + mysqlpp::Connection *MySQLConnectionPool::grab() { + while (conns_in_use_ > max_pool_size_) { + sleep(1); + } + + ++conns_in_use_; + return mysqlpp::ConnectionPool::grab(); + } + + // Other half of in-use conn count limit + void MySQLConnectionPool::release(const mysqlpp::Connection *pc) { + mysqlpp::ConnectionPool::release(pc); + + if (conns_in_use_ <= 0) { + ENGINE_LOG_WARNING << "MySQLConnetionPool::release: conns_in_use_ is less than zero. conns_in_use_ = " << conns_in_use_; + } else { + --conns_in_use_; + } + } + + int MySQLConnectionPool::getConnectionsInUse() { + return conns_in_use_; + } + + void MySQLConnectionPool::set_max_idle_time(int max_idle) { + max_idle_time_ = max_idle; + } + + std::string MySQLConnectionPool::getDB() { + return db_; + } + + // Superclass overrides + mysqlpp::Connection *MySQLConnectionPool::create() { + + try { + // Create connection using the parameters we were passed upon + // creation. + mysqlpp::Connection *conn = new mysqlpp::Connection(); + conn->set_option(new mysqlpp::ReconnectOption(true)); + conn->connect(db_.empty() ? 0 : db_.c_str(), + server_.empty() ? 0 : server_.c_str(), + user_.empty() ? 0 : user_.c_str(), + password_.empty() ? 0 : password_.c_str(), + port_); + return conn; + } catch (const mysqlpp::ConnectionFailed& er) { + ENGINE_LOG_ERROR << "Failed to connect to database server" << ": " << er.what(); + return nullptr; + } + } + + void MySQLConnectionPool::destroy(mysqlpp::Connection *cp) { + // Our superclass can't know how we created the Connection, so + // it delegates destruction to us, to be safe. + delete cp; + } + + unsigned int MySQLConnectionPool::max_idle_time() { + return max_idle_time_; + } + +} // namespace meta +} // namespace engine +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/db/MySQLConnectionPool.h b/cpp/src/db/MySQLConnectionPool.h index 143e68be06..5fcc15f312 100644 --- a/cpp/src/db/MySQLConnectionPool.h +++ b/cpp/src/db/MySQLConnectionPool.h @@ -5,6 +5,11 @@ #include "Log.h" +namespace zilliz { +namespace milvus { +namespace engine { +namespace meta { + class MySQLConnectionPool : public mysqlpp::ConnectionPool { public: @@ -20,8 +25,7 @@ public: password_(passWord), server_(serverIp), port_(port), - max_pool_size_(maxPoolSize) - { + max_pool_size_(maxPoolSize) { conns_in_use_ = 0; @@ -34,65 +38,25 @@ public: clear(); } - // Do a simple form of in-use connection limiting: wait to return - // a connection until there are a reasonably low number in use - // already. Can't do this in create() because we're interested in - // connections actually in use, not those created. Also note that - // we keep our own count; ConnectionPool::size() isn't the same! - mysqlpp::Connection* grab() override { - while (conns_in_use_ > max_pool_size_) { - sleep(1); - } - - ++conns_in_use_; - return mysqlpp::ConnectionPool::grab(); - } + mysqlpp::Connection *grab() override; // Other half of in-use conn count limit - void release(const mysqlpp::Connection* pc) override { - mysqlpp::ConnectionPool::release(pc); -// ENGINE_LOG_DEBUG << "conns_in_use_ in release: " << conns_in_use_ << std::endl; - if (conns_in_use_ <= 0) { - ENGINE_LOG_WARNING << "MySQLConnetionPool::release: conns_in_use_ is less than zero. conns_in_use_ = " << conns_in_use_ << std::endl; - } - else { - --conns_in_use_; - } - } + void release(const mysqlpp::Connection *pc) override; - void set_max_idle_time(int max_idle) { - max_idle_time_ = max_idle; - } + int getConnectionsInUse(); - std::string getDB() { - return db_; - } + void set_max_idle_time(int max_idle); + + std::string getDB(); protected: // Superclass overrides - mysqlpp::Connection* create() override { - // Create connection using the parameters we were passed upon - // creation. - mysqlpp::Connection* conn = new mysqlpp::Connection(); - conn->set_option(new mysqlpp::ReconnectOption(true)); - conn->connect(db_.empty() ? 0 : db_.c_str(), - server_.empty() ? 0 : server_.c_str(), - user_.empty() ? 0 : user_.c_str(), - password_.empty() ? 0 : password_.c_str(), - port_); - return conn; - } + mysqlpp::Connection *create() override; - void destroy(mysqlpp::Connection* cp) override { - // Our superclass can't know how we created the Connection, so - // it delegates destruction to us, to be safe. - delete cp; - } + void destroy(mysqlpp::Connection *cp) override; - unsigned int max_idle_time() override { - return max_idle_time_; - } + unsigned int max_idle_time() override; private: // Number of connections currently in use @@ -105,4 +69,9 @@ private: int max_pool_size_; unsigned int max_idle_time_; -}; \ No newline at end of file +}; + +} // namespace meta +} // namespace engine +} // namespace milvus +} // namespace zilliz \ No newline at end of file diff --git a/cpp/src/db/MySQLMetaImpl.cpp b/cpp/src/db/MySQLMetaImpl.cpp index 22da874a45..b951285d37 100644 --- a/cpp/src/db/MySQLMetaImpl.cpp +++ b/cpp/src/db/MySQLMetaImpl.cpp @@ -169,6 +169,10 @@ namespace meta { { ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab); + if (connectionPtr == nullptr) { + return Status::Error("Failed to connect to database server"); + } + // ENGINE_LOG_DEBUG << "MySQLMetaImpl::Initialize: connections in use = " << mysql_connection_pool_->getConnectionsInUse(); // if (!connectionPtr->connect(dbName, serverAddress, username, password, port)) { // return Status::Error("DB connection failed: ", connectionPtr->error()); @@ -234,9 +238,6 @@ namespace meta { // } else { // return Status::DBTransactionError("Initialization Error", InitializeQuery.error()); // } - } catch (const ConnectionFailed& er) { - ENGINE_LOG_ERROR << "Failed to connect to database server" << ": " << er.what(); - return Status::DBTransactionError("Failed to connect to database server", er.what()); } catch (const BadQuery& er) { // Handle any query errors ENGINE_LOG_ERROR << "QUERY ERROR DURING INITIALIZATION" << ": " << er.what(); @@ -292,6 +293,10 @@ namespace meta { { ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab); + if (connectionPtr == nullptr) { + return Status::Error("Failed to connect to database server"); + } + // if (mysql_connection_pool_->getConnectionsInUse() <= 0) { // ENGINE_LOG_WARNING << "MySQLMetaImpl::DropPartitionsByDates connection in use = " << mysql_connection_pool_->getConnectionsInUse(); // } @@ -336,6 +341,10 @@ namespace meta { { ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab); + if (connectionPtr == nullptr) { + return Status::Error("Failed to connect to database server"); + } + // if (mysql_connection_pool_->getConnectionsInUse() <= 0) { // ENGINE_LOG_WARNING << "MySQLMetaImpl::CreateTable connection in use = " << mysql_connection_pool_->getConnectionsInUse(); // } @@ -352,7 +361,7 @@ namespace meta { ENGINE_LOG_DEBUG << "MySQLMetaImpl::CreateTable: " << createTableQuery.str(); StoreQueryResult res = createTableQuery.store(); - assert(res && res.num_rows() <= 1); + if (res.num_rows() == 1) { int state = res[0]["state"]; std::string msg = (TableSchema::TO_DELETE == state) ? @@ -438,6 +447,10 @@ namespace meta { { ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab); + if (connectionPtr == nullptr) { + return Status::Error("Failed to connect to database server"); + } + // if (mysql_connection_pool_->getConnectionsInUse() <= 0) { // ENGINE_LOG_WARNING << "MySQLMetaImpl::DeleteTable connection in use = " << mysql_connection_pool_->getConnectionsInUse(); // } @@ -483,6 +496,10 @@ namespace meta { { ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab); + if (connectionPtr == nullptr) { + return Status::Error("Failed to connect to database server"); + } + // if (mysql_connection_pool_->getConnectionsInUse() <= 0) { // ENGINE_LOG_WARNING << "MySQLMetaImpl::DeleteTableFiles connection in use = " << mysql_connection_pool_->getConnectionsInUse(); // } @@ -529,6 +546,10 @@ namespace meta { { ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab); + if (connectionPtr == nullptr) { + return Status::Error("Failed to connect to database server"); + } + // if (mysql_connection_pool_->getConnectionsInUse() <= 0) { // ENGINE_LOG_WARNING << "MySQLMetaImpl::DescribeTable connection in use = " << mysql_connection_pool_->getConnectionsInUse(); // } @@ -544,7 +565,6 @@ namespace meta { res = describeTableQuery.store(); } //Scoped Connection - assert(res && res.num_rows() <= 1); if (res.num_rows() == 1) { const Row& resRow = res[0]; @@ -592,6 +612,10 @@ namespace meta { { ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab); + if (connectionPtr == nullptr) { + return Status::Error("Failed to connect to database server"); + } + // if (mysql_connection_pool_->getConnectionsInUse() <= 0) { // ENGINE_LOG_WARNING << "MySQLMetaImpl::HasTable connection in use = " << mysql_connection_pool_->getConnectionsInUse(); // } @@ -609,7 +633,6 @@ namespace meta { res = hasTableQuery.store(); } //Scoped Connection - assert(res && res.num_rows() == 1); int check = res[0]["check"]; has_or_not = (check == 1); @@ -639,6 +662,10 @@ namespace meta { { ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab); + if (connectionPtr == nullptr) { + return Status::Error("Failed to connect to database server"); + } + // if (mysql_connection_pool_->getConnectionsInUse() <= 0) { // ENGINE_LOG_WARNING << "MySQLMetaImpl::AllTables connection in use = " << mysql_connection_pool_->getConnectionsInUse(); // } @@ -726,6 +753,10 @@ namespace meta { { ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab); + if (connectionPtr == nullptr) { + return Status::Error("Failed to connect to database server"); + } + // if (mysql_connection_pool_->getConnectionsInUse() <= 0) { // ENGINE_LOG_WARNING << "MySQLMetaImpl::CreateTableFile connection in use = " << mysql_connection_pool_->getConnectionsInUse(); // } @@ -792,6 +823,10 @@ namespace meta { { ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab); + if (connectionPtr == nullptr) { + return Status::Error("Failed to connect to database server"); + } + // if (mysql_connection_pool_->getConnectionsInUse() <= 0) { // ENGINE_LOG_WARNING << "MySQLMetaImpl::FilesToIndex connection in use = " << mysql_connection_pool_->getConnectionsInUse(); // } @@ -875,6 +910,9 @@ namespace meta { { ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab); + if (connectionPtr == nullptr) { + return Status::Error("Failed to connect to database server"); + } // if (mysql_connection_pool_->getConnectionsInUse() <= 0) { // ENGINE_LOG_WARNING << "MySQLMetaImpl::FilesToSearch connection in use = " << mysql_connection_pool_->getConnectionsInUse(); // } @@ -986,6 +1024,10 @@ namespace meta { { ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab); + if (connectionPtr == nullptr) { + return Status::Error("Failed to connect to database server"); + } + // if (mysql_connection_pool_->getConnectionsInUse() <= 0) { // ENGINE_LOG_WARNING << "MySQLMetaImpl::FilesToMerge connection in use = " << mysql_connection_pool_->getConnectionsInUse(); // } @@ -1078,6 +1120,10 @@ namespace meta { { ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab); + if (connectionPtr == nullptr) { + return Status::Error("Failed to connect to database server"); + } + // if (mysql_connection_pool_->getConnectionsInUse() <= 0) { // ENGINE_LOG_WARNING << "MySQLMetaImpl::GetTableFiles connection in use = " << mysql_connection_pool_->getConnectionsInUse(); // } @@ -1093,8 +1139,6 @@ namespace meta { res = getTableFileQuery.store(); } //Scoped Connection - assert(res); - TableSchema table_schema; table_schema.table_id_ = table_id; auto status = DescribeTable(table_schema); @@ -1162,6 +1206,10 @@ namespace meta { ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab); + if (connectionPtr == nullptr) { + return Status::Error("Failed to connect to database server"); + } + // if (mysql_connection_pool_->getConnectionsInUse() <= 0) { // ENGINE_LOG_WARNING << "MySQLMetaImpl::Archive connection in use = " << mysql_connection_pool_->getConnectionsInUse(); // } @@ -1212,6 +1260,10 @@ namespace meta { { ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab); + if (connectionPtr == nullptr) { + return Status::Error("Failed to connect to database server"); + } + // if (mysql_connection_pool_->getConnectionsInUse() <= 0) { // ENGINE_LOG_WARNING << "MySQLMetaImpl::Size connection in use = " << mysql_connection_pool_->getConnectionsInUse(); // } @@ -1226,7 +1278,6 @@ namespace meta { res = getSizeQuery.store(); } //Scoped Connection - assert(res && res.num_rows() == 1); // if (!res) { //// std::cout << "result is NULL" << std::endl; // return Status::DBTransactionError("QUERY ERROR WHEN RETRIEVING SIZE", getSizeQuery.error()); @@ -1272,6 +1323,10 @@ namespace meta { { ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab); + if (connectionPtr == nullptr) { + return Status::Error("Failed to connect to database server"); + } + // if (mysql_connection_pool_->getConnectionsInUse() <= 0) { // ENGINE_LOG_WARNING << "MySQLMetaImpl::DiscardFiles connection in use = " << mysql_connection_pool_->getConnectionsInUse(); // } @@ -1288,7 +1343,6 @@ namespace meta { // std::cout << discardFilesQuery.str() << std::endl; StoreQueryResult res = discardFilesQuery.store(); - assert(res); if (res.num_rows() == 0) { return Status::OK(); } @@ -1350,6 +1404,10 @@ namespace meta { { ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab); + if (connectionPtr == nullptr) { + return Status::Error("Failed to connect to database server"); + } + // if (mysql_connection_pool_->getConnectionsInUse() <= 0) { // ENGINE_LOG_WARNING << "MySQLMetaImpl::UpdateTableFile connection in use = " << mysql_connection_pool_->getConnectionsInUse(); // } @@ -1365,7 +1423,6 @@ namespace meta { StoreQueryResult res = updateTableFileQuery.store(); - assert(res && res.num_rows() <= 1); if (res.num_rows() == 1) { int state = res[0]["state"]; if (state == TableSchema::TO_DELETE) { @@ -1432,6 +1489,10 @@ namespace meta { { ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab); + if (connectionPtr == nullptr) { + return Status::Error("Failed to connect to database server"); + } + // if (mysql_connection_pool_->getConnectionsInUse() <= 0) { // ENGINE_LOG_WARNING << "MySQLMetaImpl::UpdateTableFiles connection in use = " << mysql_connection_pool_->getConnectionsInUse(); // } @@ -1455,7 +1516,6 @@ namespace meta { StoreQueryResult res = updateTableFilesQuery.store(); - assert(res && res.num_rows() == 1); int check = res[0]["check"]; has_tables[file_schema.table_id_] = (check == 1); } @@ -1527,6 +1587,10 @@ namespace meta { ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab); + if (connectionPtr == nullptr) { + return Status::Error("Failed to connect to database server"); + } + // if (mysql_connection_pool_->getConnectionsInUse() <= 0) { // ENGINE_LOG_WARNING << "MySQLMetaImpl::CleanUpFilesWithTTL: clean table files: connection in use after creating ScopedConnection = " // << mysql_connection_pool_->getConnectionsInUse(); @@ -1542,8 +1606,6 @@ namespace meta { StoreQueryResult res = cleanUpFilesWithTTLQuery.store(); - assert(res); - TableFileSchema table_file; std::vector idsToDelete; @@ -1611,6 +1673,10 @@ namespace meta { ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab); + if (connectionPtr == nullptr) { + return Status::Error("Failed to connect to database server"); + } + // if (mysql_connection_pool_->getConnectionsInUse() <= 0) { // ENGINE_LOG_WARNING << "MySQLMetaImpl::CleanUpFilesWithTTL: clean tables: connection in use after creating ScopedConnection = " // << mysql_connection_pool_->getConnectionsInUse(); @@ -1624,7 +1690,6 @@ namespace meta { ENGINE_LOG_DEBUG << "MySQLMetaImpl::CleanUpFilesWithTTL: " << cleanUpFilesWithTTLQuery.str(); StoreQueryResult res = cleanUpFilesWithTTLQuery.store(); - assert(res); // std::cout << res.num_rows() << std::endl; if (!res.empty()) { @@ -1677,6 +1742,10 @@ namespace meta { try { ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab); + if (connectionPtr == nullptr) { + return Status::Error("Failed to connect to database server"); + } + // if (mysql_connection_pool_->getConnectionsInUse() <= 0) { // ENGINE_LOG_WARNING << "MySQLMetaImpl::CleanUp: connection in use = " << mysql_connection_pool_->getConnectionsInUse(); // } @@ -1690,7 +1759,7 @@ namespace meta { ENGINE_LOG_DEBUG << "MySQLMetaImpl::CleanUp: " << cleanUpQuery.str(); StoreQueryResult res = cleanUpQuery.store(); - assert(res); + if (!res.empty()) { ENGINE_LOG_DEBUG << "Remove table file type as NEW"; cleanUpQuery << "DELETE FROM TableFiles WHERE file_type = " << std::to_string(TableFileSchema::NEW) << ";"; @@ -1736,6 +1805,10 @@ namespace meta { { ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab); + if (connectionPtr == nullptr) { + return Status::Error("Failed to connect to database server"); + } + // if (mysql_connection_pool_->getConnectionsInUse() <= 0) { // ENGINE_LOG_WARNING << "MySQLMetaImpl::Count: connection in use = " << mysql_connection_pool_->getConnectionsInUse(); // } @@ -1759,7 +1832,12 @@ namespace meta { result += size; } - assert(table_schema.dimension_ != 0); + if (table_schema.dimension_ <= 0) { + std::stringstream errorMsg; + errorMsg << "MySQLMetaImpl::Count: " << "table dimension = " << std::to_string(table_schema.dimension_) << ", table_id = " << table_id; + ENGINE_LOG_ERROR << errorMsg.str(); + return Status::Error(errorMsg.str()); + } result /= table_schema.dimension_; result /= sizeof(float); @@ -1786,6 +1864,10 @@ namespace meta { ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab); + if (connectionPtr == nullptr) { + return Status::Error("Failed to connect to database server"); + } + // if (mysql_connection_pool_->getConnectionsInUse() <= 0) { // ENGINE_LOG_WARNING << "MySQLMetaImpl::DropAll: connection in use = " << mysql_connection_pool_->getConnectionsInUse(); // } From 107d8efdf205ed1ea8d62ae08c42eccf89f89e19 Mon Sep 17 00:00:00 2001 From: zhiru Date: Thu, 4 Jul 2019 14:20:59 +0800 Subject: [PATCH 086/189] update Former-commit-id: 14a8abe10984046f4ca1c38a96ecd5c9c0297cc4 --- cpp/CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/cpp/CHANGELOG.md b/cpp/CHANGELOG.md index eb933db3c3..13e66cf29d 100644 --- a/cpp/CHANGELOG.md +++ b/cpp/CHANGELOG.md @@ -40,6 +40,7 @@ Please mark all change in change log and use the ticket from JIRA. - MS-124 - HasTable interface - MS-126 - Add more error code - MS-128 - Change default db path +- MS-152 - Delete assert in MySQLMetaImpl and change MySQLConnectionPool impl ## New Feature From 1c441df72e4b5d60557fb915790ca420769456a8 Mon Sep 17 00:00:00 2001 From: zhiru Date: Thu, 4 Jul 2019 15:58:28 +0800 Subject: [PATCH 087/189] fix c_str error when connecting to MySQL Former-commit-id: 6c7b477cac2330c169a076550553dbba7efd533e --- cpp/src/db/MySQLMetaImpl.cpp | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/cpp/src/db/MySQLMetaImpl.cpp b/cpp/src/db/MySQLMetaImpl.cpp index b951285d37..1031da66a9 100644 --- a/cpp/src/db/MySQLMetaImpl.cpp +++ b/cpp/src/db/MySQLMetaImpl.cpp @@ -144,15 +144,15 @@ namespace meta { if (dialect.find("mysql") == std::string::npos) { return Status::Error("URI's dialect is not MySQL"); } - const char* username = pieces_match[2].str().c_str(); - const char* password = pieces_match[3].str().c_str(); - const char* serverAddress = pieces_match[4].str().c_str(); + std::string username = pieces_match[2].str(); + std::string password = pieces_match[3].str(); + std::string serverAddress = pieces_match[4].str(); unsigned int port = 0; if (!pieces_match[5].str().empty()) { port = std::stoi(pieces_match[5].str()); } - const char* dbName = pieces_match[6].str().c_str(); - //std::cout << dbName << " " << serverAddress << " " << username << " " << password << " " << port << std::endl; + std::string dbName = pieces_match[6].str(); +// std::cout << dbName << " " << serverAddress << " " << username << " " << password << " " << port << std::endl; // connectionPtr->set_option(new MultiStatementsOption(true)); // connectionPtr->set_option(new mysqlpp::ReconnectOption(true)); int threadHint = std::thread::hardware_concurrency(); @@ -1753,8 +1753,8 @@ namespace meta { Query cleanUpQuery = connectionPtr->query(); cleanUpQuery << "SELECT table_name " << "FROM information_schema.tables " << - "WHERE table_schema = " << quote << mysql_connection_pool_->getDB() << quote << " " << - "AND table_name = " << quote << "TableFiles" << quote << ";"; + "WHERE table_schema = " << quote << mysql_connection_pool_->getDB() << " " << + "AND table_name = " << quote << "TableFiles" << ";"; ENGINE_LOG_DEBUG << "MySQLMetaImpl::CleanUp: " << cleanUpQuery.str(); From b4f1e9cbb7ce5bc2579547f7f63ad48cb30d06d0 Mon Sep 17 00:00:00 2001 From: zhiru Date: Thu, 4 Jul 2019 15:58:38 +0800 Subject: [PATCH 088/189] fix c_str error when connecting to MySQL Former-commit-id: 4546e50abcbad6b8bc5d2cc19e20653672edaa3c --- cpp/CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/cpp/CHANGELOG.md b/cpp/CHANGELOG.md index 13e66cf29d..59de1f0fa0 100644 --- a/cpp/CHANGELOG.md +++ b/cpp/CHANGELOG.md @@ -27,6 +27,7 @@ Please mark all change in change log and use the ticket from JIRA. - MS-110 - Avoid huge file size - MS-148 - Disable cleanup if mode is read only - MS-149 - Fixed searching only one index file issue in distributed mode +- MS-153 - fix c_str error when connecting to MySQL ## Improvement - MS-82 - Update server startup welcome message From 6207e83000bf9c9c7bfbe6dabe5e3aaffc8034da Mon Sep 17 00:00:00 2001 From: starlord Date: Thu, 4 Jul 2019 16:58:40 +0800 Subject: [PATCH 089/189] add uiittest for merge result functions Former-commit-id: 071b7cd18d8acbd6bd6bdceb7dae9a7cf1d1a86c --- cpp/CHANGELOG.md | 1 + cpp/src/db/scheduler/context/SearchContext.h | 4 +- cpp/src/db/scheduler/task/SearchTask.cpp | 216 ++++++++++--------- cpp/src/db/scheduler/task/SearchTask.h | 14 ++ cpp/unittest/db/search_test.cpp | 162 ++++++++++++++ 5 files changed, 295 insertions(+), 102 deletions(-) create mode 100644 cpp/unittest/db/search_test.cpp diff --git a/cpp/CHANGELOG.md b/cpp/CHANGELOG.md index 59de1f0fa0..ccff7dcdba 100644 --- a/cpp/CHANGELOG.md +++ b/cpp/CHANGELOG.md @@ -8,6 +8,7 @@ Please mark all change in change log and use the ticket from JIRA. ## Bug ## Improvement +- MS-156 - Add unittest for merge result functions ## New Feature diff --git a/cpp/src/db/scheduler/context/SearchContext.h b/cpp/src/db/scheduler/context/SearchContext.h index 1997b80764..e81622eb32 100644 --- a/cpp/src/db/scheduler/context/SearchContext.h +++ b/cpp/src/db/scheduler/context/SearchContext.h @@ -32,8 +32,8 @@ public: using Id2IndexMap = std::unordered_map; const Id2IndexMap& GetIndexMap() const { return map_index_files_; } - using Id2ScoreMap = std::vector>; - using ResultSet = std::vector; + using Id2DistanceMap = std::vector>; + using ResultSet = std::vector; const ResultSet& GetResult() const { return result_; } ResultSet& GetResult() { return result_; } diff --git a/cpp/src/db/scheduler/task/SearchTask.cpp b/cpp/src/db/scheduler/task/SearchTask.cpp index d04f270331..91ac123dec 100644 --- a/cpp/src/db/scheduler/task/SearchTask.cpp +++ b/cpp/src/db/scheduler/task/SearchTask.cpp @@ -13,104 +13,6 @@ namespace milvus { namespace engine { namespace { -void ClusterResult(const std::vector &output_ids, - const std::vector &output_distence, - uint64_t nq, - uint64_t topk, - SearchContext::ResultSet &result_set) { - result_set.clear(); - result_set.reserve(nq); - for (auto i = 0; i < nq; i++) { - SearchContext::Id2ScoreMap id_score; - id_score.reserve(topk); - for (auto k = 0; k < topk; k++) { - uint64_t index = i * topk + k; - if(output_ids[index] < 0) { - continue; - } - id_score.push_back(std::make_pair(output_ids[index], output_distence[index])); - } - result_set.emplace_back(id_score); - } -} - -void MergeResult(SearchContext::Id2ScoreMap &score_src, - SearchContext::Id2ScoreMap &score_target, - uint64_t topk) { - //Note: the score_src and score_target are already arranged by score in ascending order - if(score_src.empty()) { - return; - } - - if(score_target.empty()) { - score_target.swap(score_src); - return; - } - - size_t src_count = score_src.size(); - size_t target_count = score_target.size(); - SearchContext::Id2ScoreMap score_merged; - score_merged.reserve(topk); - size_t src_index = 0, target_index = 0; - while(true) { - //all score_src items are merged, if score_merged.size() still less than topk - //move items from score_target to score_merged until score_merged.size() equal topk - if(src_index >= src_count) { - for(size_t i = target_index; i < target_count && score_merged.size() < topk; ++i) { - score_merged.push_back(score_target[i]); - } - break; - } - - //all score_target items are merged, if score_merged.size() still less than topk - //move items from score_src to score_merged until score_merged.size() equal topk - if(target_index >= target_count) { - for(size_t i = src_index; i < src_count && score_merged.size() < topk; ++i) { - score_merged.push_back(score_src[i]); - } - break; - } - - //compare score, put smallest score to score_merged one by one - auto& src_pair = score_src[src_index]; - auto& target_pair = score_target[target_index]; - if(src_pair.second > target_pair.second) { - score_merged.push_back(target_pair); - target_index++; - } else { - score_merged.push_back(src_pair); - src_index++; - } - - //score_merged.size() already equal topk - if(score_merged.size() >= topk) { - break; - } - } - - score_target.swap(score_merged); -} - -void TopkResult(SearchContext::ResultSet &result_src, - uint64_t topk, - SearchContext::ResultSet &result_target) { - if (result_target.empty()) { - result_target.swap(result_src); - return; - } - - if (result_src.size() != result_target.size()) { - SERVER_LOG_ERROR << "Invalid result set"; - return; - } - - for (size_t i = 0; i < result_src.size(); i++) { - SearchContext::Id2ScoreMap &score_src = result_src[i]; - SearchContext::Id2ScoreMap &score_target = result_target[i]; - MergeResult(score_src, score_target, topk); - } -} - void CollectDurationMetrics(int index_type, double total_time) { switch(index_type) { case meta::TableFileSchema::RAW: { @@ -164,11 +66,12 @@ std::shared_ptr SearchTask::Execute() { //step 3: cluster result SearchContext::ResultSet result_set; - ClusterResult(output_ids, output_distence, context->nq(), inner_k, result_set); + auto spec_k = index_engine_->Count() < context->topk() ? index_engine_->Count() : context->topk(); + SearchTask::ClusterResult(output_ids, output_distence, context->nq(), spec_k, result_set); rc.Record("cluster result"); //step 4: pick up topk result - TopkResult(result_set, inner_k, context->GetResult()); + SearchTask::TopkResult(result_set, inner_k, context->GetResult()); rc.Record("reduce topk"); } catch (std::exception& ex) { @@ -190,6 +93,119 @@ std::shared_ptr SearchTask::Execute() { return nullptr; } +Status SearchTask::ClusterResult(const std::vector &output_ids, + const std::vector &output_distence, + uint64_t nq, + uint64_t topk, + SearchContext::ResultSet &result_set) { + if(output_ids.size() != nq*topk || output_distence.size() != nq*topk) { + std::string msg = "Invalid id array size: " + std::to_string(output_ids.size()) + + " distance array size: " + std::to_string(output_distence.size()); + SERVER_LOG_ERROR << msg; + return Status::Error(msg); + } + + result_set.clear(); + result_set.reserve(nq); + for (auto i = 0; i < nq; i++) { + SearchContext::Id2DistanceMap id_distance; + id_distance.reserve(topk); + for (auto k = 0; k < topk; k++) { + uint64_t index = i * topk + k; + if(output_ids[index] < 0) { + continue; + } + id_distance.push_back(std::make_pair(output_ids[index], output_distence[index])); + } + result_set.emplace_back(id_distance); + } + + return Status::OK(); +} + +Status SearchTask::MergeResult(SearchContext::Id2DistanceMap &distance_src, + SearchContext::Id2DistanceMap &distance_target, + uint64_t topk) { + //Note: the score_src and score_target are already arranged by score in ascending order + if(distance_src.empty()) { + SERVER_LOG_WARNING << "Empty distance source array"; + return Status::OK(); + } + + if(distance_target.empty()) { + distance_target.swap(distance_src); + return Status::OK(); + } + + size_t src_count = distance_src.size(); + size_t target_count = distance_target.size(); + SearchContext::Id2DistanceMap distance_merged; + distance_merged.reserve(topk); + size_t src_index = 0, target_index = 0; + while(true) { + //all score_src items are merged, if score_merged.size() still less than topk + //move items from score_target to score_merged until score_merged.size() equal topk + if(src_index >= src_count) { + for(size_t i = target_index; i < target_count && distance_merged.size() < topk; ++i) { + distance_merged.push_back(distance_target[i]); + } + break; + } + + //all score_target items are merged, if score_merged.size() still less than topk + //move items from score_src to score_merged until score_merged.size() equal topk + if(target_index >= target_count) { + for(size_t i = src_index; i < src_count && distance_merged.size() < topk; ++i) { + distance_merged.push_back(distance_src[i]); + } + break; + } + + //compare score, put smallest score to score_merged one by one + auto& src_pair = distance_src[src_index]; + auto& target_pair = distance_target[target_index]; + if(src_pair.second > target_pair.second) { + distance_merged.push_back(target_pair); + target_index++; + } else { + distance_merged.push_back(src_pair); + src_index++; + } + + //score_merged.size() already equal topk + if(distance_merged.size() >= topk) { + break; + } + } + + distance_target.swap(distance_merged); + + return Status::OK(); +} + +Status SearchTask::TopkResult(SearchContext::ResultSet &result_src, + uint64_t topk, + SearchContext::ResultSet &result_target) { + if (result_target.empty()) { + result_target.swap(result_src); + return Status::OK(); + } + + if (result_src.size() != result_target.size()) { + std::string msg = "Invalid result set size"; + SERVER_LOG_ERROR << msg; + return Status::Error(msg); + } + + for (size_t i = 0; i < result_src.size(); i++) { + SearchContext::Id2DistanceMap &score_src = result_src[i]; + SearchContext::Id2DistanceMap &score_target = result_target[i]; + SearchTask::MergeResult(score_src, score_target, topk); + } + + return Status::OK(); +} + } } } diff --git a/cpp/src/db/scheduler/task/SearchTask.h b/cpp/src/db/scheduler/task/SearchTask.h index 0b3a236ce4..e4f0d872b1 100644 --- a/cpp/src/db/scheduler/task/SearchTask.h +++ b/cpp/src/db/scheduler/task/SearchTask.h @@ -19,6 +19,20 @@ public: virtual std::shared_ptr Execute() override; + static Status ClusterResult(const std::vector &output_ids, + const std::vector &output_distence, + uint64_t nq, + uint64_t topk, + SearchContext::ResultSet &result_set); + + static Status MergeResult(SearchContext::Id2DistanceMap &distance_src, + SearchContext::Id2DistanceMap &distance_target, + uint64_t topk); + + static Status TopkResult(SearchContext::ResultSet &result_src, + uint64_t topk, + SearchContext::ResultSet &result_target); + public: size_t index_id_ = 0; int index_type_ = 0; //for metrics diff --git a/cpp/unittest/db/search_test.cpp b/cpp/unittest/db/search_test.cpp new file mode 100644 index 0000000000..db10bcbadf --- /dev/null +++ b/cpp/unittest/db/search_test.cpp @@ -0,0 +1,162 @@ +//////////////////////////////////////////////////////////////////////////////// +// Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved +// Unauthorized copying of this file, via any medium is strictly prohibited. +// Proprietary and confidential. +//////////////////////////////////////////////////////////////////////////////// +#include + +#include "db/scheduler/task/SearchTask.h" + +#include + +using namespace zilliz::milvus; + +namespace { + +static constexpr uint64_t NQ = 15; +static constexpr uint64_t TOP_K = 64; + +void BuildResult(uint64_t nq, + uint64_t top_k, + std::vector &output_ids, + std::vector &output_distence) { + output_ids.clear(); + output_ids.resize(nq*top_k); + output_distence.clear(); + output_distence.resize(nq*top_k); + + for(uint64_t i = 0; i < nq; i++) { + for(uint64_t j = 0; j < top_k; j++) { + output_ids[i * top_k + j] = (long)(drand48()*100000); + output_distence[i * top_k + j] = j + drand48(); + } + } +} + +void CheckResult(const engine::SearchContext::Id2DistanceMap& src_1, + const engine::SearchContext::Id2DistanceMap& src_2, + const engine::SearchContext::Id2DistanceMap& target) { + for(uint64_t i = 0; i < target.size() - 1; i++) { + ASSERT_LE(target[i].second, target[i + 1].second); + } + + using ID2DistMap = std::map; + ID2DistMap src_map_1, src_map_2; + for(const auto& pair : src_1) { + src_map_1.insert(pair); + } + for(const auto& pair : src_2) { + src_map_2.insert(pair); + } + + for(const auto& pair : target) { + ASSERT_TRUE(src_map_1.find(pair.first) != src_map_1.end() || src_map_2.find(pair.first) != src_map_2.end()); + + float dist = src_map_1.find(pair.first) != src_map_1.end() ? src_map_1[pair.first] : src_map_2[pair.first]; + ASSERT_LT(fabs(pair.second - dist), std::numeric_limits::epsilon()); + } +} + +} + +TEST(DBSearchTest, TOPK_TEST) { + std::vector target_ids; + std::vector target_distence; + engine::SearchContext::ResultSet src_result; + auto status = engine::SearchTask::ClusterResult(target_ids, target_distence, NQ, TOP_K, src_result); + ASSERT_FALSE(status.ok()); + ASSERT_TRUE(src_result.empty()); + + BuildResult(NQ, TOP_K, target_ids, target_distence); + status = engine::SearchTask::ClusterResult(target_ids, target_distence, NQ, TOP_K, src_result); + ASSERT_TRUE(status.ok()); + ASSERT_EQ(src_result.size(), NQ); + + engine::SearchContext::ResultSet target_result; + status = engine::SearchTask::TopkResult(target_result, TOP_K, target_result); + ASSERT_TRUE(status.ok()); + + status = engine::SearchTask::TopkResult(target_result, TOP_K, src_result); + ASSERT_FALSE(status.ok()); + + status = engine::SearchTask::TopkResult(src_result, TOP_K, target_result); + ASSERT_TRUE(status.ok()); + ASSERT_TRUE(src_result.empty()); + ASSERT_EQ(target_result.size(), NQ); + + std::vector src_ids; + std::vector src_distence; + uint64_t wrong_topk = TOP_K - 10; + BuildResult(NQ, wrong_topk, src_ids, src_distence); + + status = engine::SearchTask::ClusterResult(src_ids, src_distence, NQ, wrong_topk, src_result); + ASSERT_TRUE(status.ok()); + + status = engine::SearchTask::TopkResult(src_result, TOP_K, target_result); + ASSERT_TRUE(status.ok()); + for(uint64_t i = 0; i < NQ; i++) { + ASSERT_EQ(target_result[i].size(), TOP_K); + } + + wrong_topk = TOP_K + 10; + BuildResult(NQ, wrong_topk, src_ids, src_distence); + + status = engine::SearchTask::TopkResult(src_result, TOP_K, target_result); + ASSERT_TRUE(status.ok()); + for(uint64_t i = 0; i < NQ; i++) { + ASSERT_EQ(target_result[i].size(), TOP_K); + } +} + +TEST(DBSearchTest, MERGE_TEST) { + std::vector target_ids; + std::vector target_distence; + std::vector src_ids; + std::vector src_distence; + engine::SearchContext::ResultSet src_result, target_result; + + uint64_t src_count = 5, target_count = 8; + BuildResult(1, src_count, src_ids, src_distence); + BuildResult(1, target_count, target_ids, target_distence); + auto status = engine::SearchTask::ClusterResult(src_ids, src_distence, 1, src_count, src_result); + ASSERT_TRUE(status.ok()); + status = engine::SearchTask::ClusterResult(target_ids, target_distence, 1, target_count, target_result); + ASSERT_TRUE(status.ok()); + + { + engine::SearchContext::Id2DistanceMap src = src_result[0]; + engine::SearchContext::Id2DistanceMap target = target_result[0]; + status = engine::SearchTask::MergeResult(src, target, 10); + ASSERT_TRUE(status.ok()); + ASSERT_EQ(target.size(), 10); + CheckResult(src_result[0], target_result[0], target); + } + + { + engine::SearchContext::Id2DistanceMap src = src_result[0]; + engine::SearchContext::Id2DistanceMap target; + status = engine::SearchTask::MergeResult(src, target, 10); + ASSERT_TRUE(status.ok()); + ASSERT_EQ(target.size(), src_count); + ASSERT_TRUE(src.empty()); + CheckResult(src_result[0], target_result[0], target); + } + + { + engine::SearchContext::Id2DistanceMap src = src_result[0]; + engine::SearchContext::Id2DistanceMap target = target_result[0]; + status = engine::SearchTask::MergeResult(src, target, 30); + ASSERT_TRUE(status.ok()); + ASSERT_EQ(target.size(), src_count + target_count); + CheckResult(src_result[0], target_result[0], target); + } + + { + engine::SearchContext::Id2DistanceMap target = src_result[0]; + engine::SearchContext::Id2DistanceMap src = target_result[0]; + status = engine::SearchTask::MergeResult(src, target, 30); + ASSERT_TRUE(status.ok()); + ASSERT_EQ(target.size(), src_count + target_count); + CheckResult(src_result[0], target_result[0], target); + } +} \ No newline at end of file From 54b0857cb2a9bab74e704932ae0a45b6f21ebe7b Mon Sep 17 00:00:00 2001 From: zhiru Date: Thu, 4 Jul 2019 17:15:38 +0800 Subject: [PATCH 090/189] fix changelog Former-commit-id: 9d7996e2d76f39f514ccc257a55f82fedbd20a0f --- cpp/CHANGELOG.md | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/cpp/CHANGELOG.md b/cpp/CHANGELOG.md index ccff7dcdba..a4f30642d4 100644 --- a/cpp/CHANGELOG.md +++ b/cpp/CHANGELOG.md @@ -7,9 +7,15 @@ Please mark all change in change log and use the ticket from JIRA. ## Bug +- MS-148 - Disable cleanup if mode is read only +- MS-149 - Fixed searching only one index file issue in distributed mode +- MS-153 - fix c_str error when connecting to MySQL + ## Improvement - MS-156 - Add unittest for merge result functions +- MS-152 - Delete assert in MySQLMetaImpl and change MySQLConnectionPool impl + ## New Feature ## Task @@ -26,9 +32,6 @@ Please mark all change in change log and use the ticket from JIRA. - MS-90 - Fix arch match incorrect on ARM - MS-99 - Fix compilation bug - MS-110 - Avoid huge file size -- MS-148 - Disable cleanup if mode is read only -- MS-149 - Fixed searching only one index file issue in distributed mode -- MS-153 - fix c_str error when connecting to MySQL ## Improvement - MS-82 - Update server startup welcome message @@ -42,7 +45,6 @@ Please mark all change in change log and use the ticket from JIRA. - MS-124 - HasTable interface - MS-126 - Add more error code - MS-128 - Change default db path -- MS-152 - Delete assert in MySQLMetaImpl and change MySQLConnectionPool impl ## New Feature From ed21eb299339430a75bb79f255d360551942fada Mon Sep 17 00:00:00 2001 From: zhiru Date: Thu, 4 Jul 2019 17:17:09 +0800 Subject: [PATCH 091/189] fix changelog Former-commit-id: 8923ccec2ae5b9334b60c0ac48c3e9ce6e6fbdfe --- cpp/CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/cpp/CHANGELOG.md b/cpp/CHANGELOG.md index a4f30642d4..1ddd47ef24 100644 --- a/cpp/CHANGELOG.md +++ b/cpp/CHANGELOG.md @@ -10,6 +10,7 @@ Please mark all change in change log and use the ticket from JIRA. - MS-148 - Disable cleanup if mode is read only - MS-149 - Fixed searching only one index file issue in distributed mode - MS-153 - fix c_str error when connecting to MySQL +- MS-157 - fix changelog ## Improvement - MS-156 - Add unittest for merge result functions From 7d4c2632fb98c8f1bed2648a96aeeb94106f9af5 Mon Sep 17 00:00:00 2001 From: zhiru Date: Fri, 5 Jul 2019 15:03:40 +0800 Subject: [PATCH 092/189] add mem impl Former-commit-id: d0e652cb4c7454cbe2cf6102622343b5a2eee158 --- cpp/src/db/Constants.h | 20 ++++ cpp/src/db/MemTable.cpp | 51 ++++++++++ cpp/src/db/MemTable.h | 40 ++++++++ cpp/src/db/MemTableFile.cpp | 66 +++++++++++++ cpp/src/db/MemTableFile.h | 44 +++++++++ cpp/src/db/VectorSource.cpp | 60 +++++++++++ cpp/src/db/VectorSource.h | 41 ++++++++ cpp/unittest/db/mem_test.cpp | 187 +++++++++++++++++++++++++++++++++++ 8 files changed, 509 insertions(+) create mode 100644 cpp/src/db/Constants.h create mode 100644 cpp/src/db/MemTable.cpp create mode 100644 cpp/src/db/MemTable.h create mode 100644 cpp/src/db/MemTableFile.cpp create mode 100644 cpp/src/db/MemTableFile.h create mode 100644 cpp/src/db/VectorSource.cpp create mode 100644 cpp/src/db/VectorSource.h create mode 100644 cpp/unittest/db/mem_test.cpp diff --git a/cpp/src/db/Constants.h b/cpp/src/db/Constants.h new file mode 100644 index 0000000000..2bb2e0a064 --- /dev/null +++ b/cpp/src/db/Constants.h @@ -0,0 +1,20 @@ +/******************************************************************************* + * Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved + * Unauthorized copying of this file, via any medium is strictly prohibited. + * Proprietary and confidential. + ******************************************************************************/ +#pragma once + +namespace zilliz { +namespace milvus { +namespace engine { + +const size_t K = 1024UL; +const size_t M = K*K; +const size_t MAX_TABLE_FILE_MEM = 128 * M; + +const int VECTOR_TYPE_SIZE = sizeof(float); + +} // namespace engine +} // namespace milvus +} // namespace zilliz diff --git a/cpp/src/db/MemTable.cpp b/cpp/src/db/MemTable.cpp new file mode 100644 index 0000000000..032d479999 --- /dev/null +++ b/cpp/src/db/MemTable.cpp @@ -0,0 +1,51 @@ +#include "MemTable.h" +#include "Log.h" + +namespace zilliz { +namespace milvus { +namespace engine { + +MemTable::MemTable(const std::string& table_id, + const std::shared_ptr& meta) : + table_id_(table_id), + meta_(meta) { + +} + +Status MemTable::Add(VectorSource::Ptr& source) { + while (!source->AllAdded()) { + MemTableFile::Ptr currentMemTableFile; + if (!mem_table_file_stack_.empty()) { + currentMemTableFile = mem_table_file_stack_.top(); + } + Status status; + if (mem_table_file_stack_.empty() || currentMemTableFile->isFull()) { + MemTableFile::Ptr newMemTableFile = std::make_shared(table_id_, meta_); + status = newMemTableFile->Add(source); + if (status.ok()) { + mem_table_file_stack_.push(newMemTableFile); + } + } + else { + status = currentMemTableFile->Add(source); + } + if (!status.ok()) { + std::string errMsg = "MemTable::Add failed: " + status.ToString(); + ENGINE_LOG_ERROR << errMsg; + return Status::Error(errMsg); + } + } + return Status::OK(); +} + +void MemTable::GetCurrentMemTableFile(MemTableFile::Ptr& mem_table_file) { + mem_table_file = mem_table_file_stack_.top(); +} + +size_t MemTable::GetStackSize() { + return mem_table_file_stack_.size(); +} + +} // namespace engine +} // namespace milvus +} // namespace zilliz \ No newline at end of file diff --git a/cpp/src/db/MemTable.h b/cpp/src/db/MemTable.h new file mode 100644 index 0000000000..b9fe4147d8 --- /dev/null +++ b/cpp/src/db/MemTable.h @@ -0,0 +1,40 @@ +#pragma once + +#include "Status.h" +#include "MemTableFile.h" +#include "VectorSource.h" + +#include + +namespace zilliz { +namespace milvus { +namespace engine { + +class MemTable { + +public: + + using Ptr = std::shared_ptr; + using MemTableFileStack = std::stack; + using MetaPtr = meta::Meta::Ptr; + + MemTable(const std::string& table_id, const std::shared_ptr& meta); + + Status Add(VectorSource::Ptr& source); + + void GetCurrentMemTableFile(MemTableFile::Ptr& mem_table_file); + + size_t GetStackSize(); + +private: + const std::string table_id_; + + MemTableFileStack mem_table_file_stack_; + + MetaPtr meta_; + +}; //MemTable + +} // namespace engine +} // namespace milvus +} // namespace zilliz \ No newline at end of file diff --git a/cpp/src/db/MemTableFile.cpp b/cpp/src/db/MemTableFile.cpp new file mode 100644 index 0000000000..26bc0d38e9 --- /dev/null +++ b/cpp/src/db/MemTableFile.cpp @@ -0,0 +1,66 @@ +#include "MemTableFile.h" +#include "Constants.h" +#include "Log.h" + +#include + +namespace zilliz { +namespace milvus { +namespace engine { + +MemTableFile::MemTableFile(const std::string& table_id, + const std::shared_ptr& meta) : + table_id_(table_id), + meta_(meta) { + + current_mem_ = 0; + CreateTableFile(); +} + +Status MemTableFile::CreateTableFile() { + + meta::TableFileSchema table_file_schema; + table_file_schema.table_id_ = table_id_; + auto status = meta_->CreateTableFile(table_file_schema); + if (status.ok()) { + table_file_schema_ = table_file_schema; + } + else { + std::string errMsg = "MemTableFile::CreateTableFile failed: " + status.ToString(); + ENGINE_LOG_ERROR << errMsg; + } + return status; +} + +Status MemTableFile::Add(const VectorSource::Ptr& source) { + + size_t singleVectorMemSize = table_file_schema_.dimension_ * VECTOR_TYPE_SIZE; + size_t memLeft = GetMemLeft(); + if (memLeft >= singleVectorMemSize) { + size_t numVectorsToAdd = std::ceil(memLeft / singleVectorMemSize); + size_t numVectorsAdded; + auto status = source->Add(table_file_schema_, numVectorsToAdd, numVectorsAdded); + if (status.ok()) { + current_mem_ += (numVectorsAdded * singleVectorMemSize); + } + return status; + } + return Status::OK(); +} + +size_t MemTableFile::GetCurrentMem() { + return current_mem_; +} + +size_t MemTableFile::GetMemLeft() { + return (MAX_TABLE_FILE_MEM - current_mem_); +} + +bool MemTableFile::isFull() { + size_t singleVectorMemSize = table_file_schema_.dimension_ * VECTOR_TYPE_SIZE; + return (GetMemLeft() < singleVectorMemSize); +} + +} // namespace engine +} // namespace milvus +} // namespace zilliz \ No newline at end of file diff --git a/cpp/src/db/MemTableFile.h b/cpp/src/db/MemTableFile.h new file mode 100644 index 0000000000..1efe4c0bfe --- /dev/null +++ b/cpp/src/db/MemTableFile.h @@ -0,0 +1,44 @@ +#pragma once + +#include "Status.h" +#include "Meta.h" +#include "VectorSource.h" + +namespace zilliz { +namespace milvus { +namespace engine { + +class MemTableFile { + +public: + + using Ptr = std::shared_ptr; + using MetaPtr = meta::Meta::Ptr; + + MemTableFile(const std::string& table_id, const std::shared_ptr& meta); + + Status Add(const VectorSource::Ptr& source); + + size_t GetCurrentMem(); + + size_t GetMemLeft(); + + bool isFull(); + +private: + + Status CreateTableFile(); + + const std::string table_id_; + + meta::TableFileSchema table_file_schema_; + + MetaPtr meta_; + + size_t current_mem_; + +}; //MemTableFile + +} // namespace engine +} // namespace milvus +} // namespace zilliz \ No newline at end of file diff --git a/cpp/src/db/VectorSource.cpp b/cpp/src/db/VectorSource.cpp new file mode 100644 index 0000000000..dff5423c6f --- /dev/null +++ b/cpp/src/db/VectorSource.cpp @@ -0,0 +1,60 @@ +#include "VectorSource.h" +#include "ExecutionEngine.h" +#include "EngineFactory.h" +#include "Log.h" + +namespace zilliz { +namespace milvus { +namespace engine { + + +VectorSource::VectorSource(const size_t &n, + const float *vectors) : + n_(n), + vectors_(vectors), + id_generator_(new SimpleIDGenerator()) { + current_num_vectors_added = 0; +} + +Status VectorSource::Add(const meta::TableFileSchema& table_file_schema, const size_t& num_vectors_to_add, size_t& num_vectors_added) { + + if (table_file_schema.dimension_ <= 0) { + std::string errMsg = "VectorSource::Add: table_file_schema dimension = " + + std::to_string(table_file_schema.dimension_) + ", table_id = " + table_file_schema.table_id_; + ENGINE_LOG_ERROR << errMsg; + return Status::Error(errMsg); + } + ExecutionEnginePtr engine = EngineFactory::Build(table_file_schema.dimension_, + table_file_schema.location_, + (EngineType)table_file_schema.engine_type_); + + num_vectors_added = current_num_vectors_added + num_vectors_to_add <= n_ ? num_vectors_to_add : n_ - current_num_vectors_added; + IDNumbers vector_ids_to_add; + id_generator_->GetNextIDNumbers(num_vectors_added, vector_ids_to_add); + Status status = engine->AddWithIds(num_vectors_added, vectors_ + current_num_vectors_added, vector_ids_to_add.data()); + if (status.ok()) { + current_num_vectors_added += num_vectors_added; + vector_ids_.insert(vector_ids_.end(), vector_ids_to_add.begin(), vector_ids_to_add.end()); + } + else { + ENGINE_LOG_ERROR << "VectorSource::Add failed: " + status.ToString(); + } + + return status; +} + +size_t VectorSource::GetNumVectorsAdded() { + return current_num_vectors_added; +} + +bool VectorSource::AllAdded() { + return (current_num_vectors_added == n_); +} + +IDNumbers VectorSource::GetVectorIds() { + return vector_ids_; +} + +} // namespace engine +} // namespace milvus +} // namespace zilliz \ No newline at end of file diff --git a/cpp/src/db/VectorSource.h b/cpp/src/db/VectorSource.h new file mode 100644 index 0000000000..170f3634cf --- /dev/null +++ b/cpp/src/db/VectorSource.h @@ -0,0 +1,41 @@ +#pragma once + +#include "Status.h" +#include "Meta.h" +#include "IDGenerator.h" + +namespace zilliz { +namespace milvus { +namespace engine { + +class VectorSource { + +public: + + using Ptr = std::shared_ptr; + + VectorSource(const size_t& n, const float* vectors); + + Status Add(const meta::TableFileSchema& table_file_schema, const size_t& num_vectors_to_add, size_t& num_vectors_added); + + size_t GetNumVectorsAdded(); + + bool AllAdded(); + + IDNumbers GetVectorIds(); + +private: + + const size_t n_; + const float* vectors_; + IDNumbers vector_ids_; + + size_t current_num_vectors_added; + + IDGenerator* id_generator_; + +}; //VectorSource + +} // namespace engine +} // namespace milvus +} // namespace zilliz \ No newline at end of file diff --git a/cpp/unittest/db/mem_test.cpp b/cpp/unittest/db/mem_test.cpp new file mode 100644 index 0000000000..8418b9cd2d --- /dev/null +++ b/cpp/unittest/db/mem_test.cpp @@ -0,0 +1,187 @@ +#include "gtest/gtest.h" + +#include "db/VectorSource.h" +#include "db/MemTableFile.h" +#include "db/MemTable.h" +#include "utils.h" +#include "db/Factories.h" +#include "db/Constants.h" + +using namespace zilliz::milvus; + +namespace { + + static const std::string TABLE_NAME = "test_group"; + static constexpr int64_t TABLE_DIM = 256; + static constexpr int64_t VECTOR_COUNT = 250000; + static constexpr int64_t INSERT_LOOP = 10000; + + engine::meta::TableSchema BuildTableSchema() { + engine::meta::TableSchema table_info; + table_info.dimension_ = TABLE_DIM; + table_info.table_id_ = TABLE_NAME; + table_info.engine_type_ = (int)engine::EngineType::FAISS_IDMAP; + return table_info; + } + + void BuildVectors(int64_t n, std::vector& vectors) { + vectors.clear(); + vectors.resize(n*TABLE_DIM); + float* data = vectors.data(); + for(int i = 0; i < n; i++) { + for(int j = 0; j < TABLE_DIM; j++) data[TABLE_DIM * i + j] = drand48(); + data[TABLE_DIM * i] += i / 2000.; + } + } +} + +TEST(MEM_TEST, VECTOR_SOURCE_TEST) { + + std::shared_ptr impl_ = engine::DBMetaImplFactory::Build(); + + engine::meta::TableSchema table_schema = BuildTableSchema(); + auto status = impl_->CreateTable(table_schema); + ASSERT_TRUE(status.ok()); + + engine::meta::TableFileSchema table_file_schema; + table_file_schema.table_id_ = TABLE_NAME; + status = impl_->CreateTableFile(table_file_schema); + ASSERT_TRUE(status.ok()); + + int64_t n = 100; + std::vector vectors; + BuildVectors(n, vectors); + + engine::VectorSource source(n, vectors.data()); + + size_t num_vectors_added; + status = source.Add(table_file_schema, 50, num_vectors_added); + ASSERT_TRUE(status.ok()); + + ASSERT_EQ(num_vectors_added, 50); + + engine::IDNumbers vector_ids = source.GetVectorIds(); + ASSERT_EQ(vector_ids.size(), 50); + + status = source.Add(table_file_schema, 60, num_vectors_added); + ASSERT_TRUE(status.ok()); + + ASSERT_EQ(num_vectors_added, 50); + + vector_ids = source.GetVectorIds(); + ASSERT_EQ(vector_ids.size(), 100); + +// for (auto& id : vector_ids) { +// std::cout << id << std::endl; +// } + + status = impl_->DropAll(); + ASSERT_TRUE(status.ok()); +} + +TEST(MEM_TEST, MEM_TABLE_FILE_TEST) { + + std::shared_ptr impl_ = engine::DBMetaImplFactory::Build(); + + engine::meta::TableSchema table_schema = BuildTableSchema(); + auto status = impl_->CreateTable(table_schema); + ASSERT_TRUE(status.ok()); + + engine::MemTableFile memTableFile(TABLE_NAME, impl_); + + int64_t n_100 = 100; + std::vector vectors_100; + BuildVectors(n_100, vectors_100); + + engine::VectorSource::Ptr source = std::make_shared(n_100, vectors_100.data()); + + status = memTableFile.Add(source); + ASSERT_TRUE(status.ok()); + +// std::cout << memTableFile.GetCurrentMem() << " " << memTableFile.GetMemLeft() << std::endl; + + engine::IDNumbers vector_ids = source->GetVectorIds(); + ASSERT_EQ(vector_ids.size(), 100); + + size_t singleVectorMem = sizeof(float) * TABLE_DIM; + ASSERT_EQ(memTableFile.GetCurrentMem(), n_100 * singleVectorMem); + + int64_t n_max = engine::MAX_TABLE_FILE_MEM / singleVectorMem; + std::vector vectors_128M; + BuildVectors(n_max, vectors_128M); + + engine::VectorSource::Ptr source_128M = std::make_shared(n_max, vectors_128M.data()); + status = memTableFile.Add(source_128M); + + vector_ids = source_128M->GetVectorIds(); + ASSERT_EQ(vector_ids.size(), n_max - n_100); + + ASSERT_TRUE(memTableFile.isFull()); + + status = impl_->DropAll(); + ASSERT_TRUE(status.ok()); +} + +TEST(MEM_TEST, MEM_TABLE_TEST) { + + std::shared_ptr impl_ = engine::DBMetaImplFactory::Build(); + + engine::meta::TableSchema table_schema = BuildTableSchema(); + auto status = impl_->CreateTable(table_schema); + ASSERT_TRUE(status.ok()); + + int64_t n_100 = 100; + std::vector vectors_100; + BuildVectors(n_100, vectors_100); + + engine::VectorSource::Ptr source_100 = std::make_shared(n_100, vectors_100.data()); + + engine::MemTable memTable(TABLE_NAME, impl_); + + status = memTable.Add(source_100); + ASSERT_TRUE(status.ok()); + + engine::IDNumbers vector_ids = source_100->GetVectorIds(); + ASSERT_EQ(vector_ids.size(), 100); + + engine::MemTableFile::Ptr memTableFile; + memTable.GetCurrentMemTableFile(memTableFile); + size_t singleVectorMem = sizeof(float) * TABLE_DIM; + ASSERT_EQ(memTableFile->GetCurrentMem(), n_100 * singleVectorMem); + + int64_t n_max = engine::MAX_TABLE_FILE_MEM / singleVectorMem; + std::vector vectors_128M; + BuildVectors(n_max, vectors_128M); + + engine::VectorSource::Ptr source_128M = std::make_shared(n_max, vectors_128M.data()); + status = memTable.Add(source_128M); + ASSERT_TRUE(status.ok()); + + vector_ids = source_128M->GetVectorIds(); + ASSERT_EQ(vector_ids.size(), n_max); + + memTable.GetCurrentMemTableFile(memTableFile); + ASSERT_EQ(memTableFile->GetCurrentMem(), n_100 * singleVectorMem); + + ASSERT_EQ(memTable.GetStackSize(), 2); + + int64_t n_1G = 1024000; + std::vector vectors_1G; + BuildVectors(n_1G, vectors_1G); + + engine::VectorSource::Ptr source_1G = std::make_shared(n_1G, vectors_1G.data()); + + status = memTable.Add(source_1G); + ASSERT_TRUE(status.ok()); + + vector_ids = source_1G->GetVectorIds(); + ASSERT_EQ(vector_ids.size(), n_1G); + + int expectedStackSize = 2 + std::ceil((n_1G - n_100) * singleVectorMem / engine::MAX_TABLE_FILE_MEM); + ASSERT_EQ(memTable.GetStackSize(), expectedStackSize); + + status = impl_->DropAll(); + ASSERT_TRUE(status.ok()); +} + + From d2f205d148aff80ed11a179336ce0dd265be7501 Mon Sep 17 00:00:00 2001 From: zhiru Date: Fri, 5 Jul 2019 15:57:49 +0800 Subject: [PATCH 093/189] update Former-commit-id: 1f59974937316725298b1d328c0a38308a4ce367 --- cpp/src/db/MemTableFile.cpp | 10 ++++++++-- cpp/src/db/MemTableFile.h | 3 +++ cpp/src/db/VectorSource.cpp | 10 +++++----- cpp/src/db/VectorSource.h | 8 +++++++- cpp/unittest/db/mem_test.cpp | 8 ++++++-- 5 files changed, 29 insertions(+), 10 deletions(-) diff --git a/cpp/src/db/MemTableFile.cpp b/cpp/src/db/MemTableFile.cpp index 26bc0d38e9..58b76ab834 100644 --- a/cpp/src/db/MemTableFile.cpp +++ b/cpp/src/db/MemTableFile.cpp @@ -1,6 +1,7 @@ #include "MemTableFile.h" #include "Constants.h" #include "Log.h" +#include "EngineFactory.h" #include @@ -14,7 +15,12 @@ MemTableFile::MemTableFile(const std::string& table_id, meta_(meta) { current_mem_ = 0; - CreateTableFile(); + auto status = CreateTableFile(); + if (status.ok()) { + execution_engine_ = EngineFactory::Build(table_file_schema_.dimension_, + table_file_schema_.location_, + (EngineType)table_file_schema_.engine_type_); + } } Status MemTableFile::CreateTableFile() { @@ -39,7 +45,7 @@ Status MemTableFile::Add(const VectorSource::Ptr& source) { if (memLeft >= singleVectorMemSize) { size_t numVectorsToAdd = std::ceil(memLeft / singleVectorMemSize); size_t numVectorsAdded; - auto status = source->Add(table_file_schema_, numVectorsToAdd, numVectorsAdded); + auto status = source->Add(execution_engine_, table_file_schema_, numVectorsToAdd, numVectorsAdded); if (status.ok()) { current_mem_ += (numVectorsAdded * singleVectorMemSize); } diff --git a/cpp/src/db/MemTableFile.h b/cpp/src/db/MemTableFile.h index 1efe4c0bfe..04f30178ea 100644 --- a/cpp/src/db/MemTableFile.h +++ b/cpp/src/db/MemTableFile.h @@ -3,6 +3,7 @@ #include "Status.h" #include "Meta.h" #include "VectorSource.h" +#include "ExecutionEngine.h" namespace zilliz { namespace milvus { @@ -37,6 +38,8 @@ private: size_t current_mem_; + ExecutionEnginePtr execution_engine_; + }; //MemTableFile } // namespace engine diff --git a/cpp/src/db/VectorSource.cpp b/cpp/src/db/VectorSource.cpp index dff5423c6f..f7cef994fa 100644 --- a/cpp/src/db/VectorSource.cpp +++ b/cpp/src/db/VectorSource.cpp @@ -16,7 +16,10 @@ VectorSource::VectorSource(const size_t &n, current_num_vectors_added = 0; } -Status VectorSource::Add(const meta::TableFileSchema& table_file_schema, const size_t& num_vectors_to_add, size_t& num_vectors_added) { +Status VectorSource::Add(const ExecutionEnginePtr& execution_engine, + const meta::TableFileSchema& table_file_schema, + const size_t& num_vectors_to_add, + size_t& num_vectors_added) { if (table_file_schema.dimension_ <= 0) { std::string errMsg = "VectorSource::Add: table_file_schema dimension = " + @@ -24,14 +27,11 @@ Status VectorSource::Add(const meta::TableFileSchema& table_file_schema, const s ENGINE_LOG_ERROR << errMsg; return Status::Error(errMsg); } - ExecutionEnginePtr engine = EngineFactory::Build(table_file_schema.dimension_, - table_file_schema.location_, - (EngineType)table_file_schema.engine_type_); num_vectors_added = current_num_vectors_added + num_vectors_to_add <= n_ ? num_vectors_to_add : n_ - current_num_vectors_added; IDNumbers vector_ids_to_add; id_generator_->GetNextIDNumbers(num_vectors_added, vector_ids_to_add); - Status status = engine->AddWithIds(num_vectors_added, vectors_ + current_num_vectors_added, vector_ids_to_add.data()); + Status status = execution_engine->AddWithIds(num_vectors_added, vectors_ + current_num_vectors_added, vector_ids_to_add.data()); if (status.ok()) { current_num_vectors_added += num_vectors_added; vector_ids_.insert(vector_ids_.end(), vector_ids_to_add.begin(), vector_ids_to_add.end()); diff --git a/cpp/src/db/VectorSource.h b/cpp/src/db/VectorSource.h index 170f3634cf..597eee4ad8 100644 --- a/cpp/src/db/VectorSource.h +++ b/cpp/src/db/VectorSource.h @@ -3,6 +3,7 @@ #include "Status.h" #include "Meta.h" #include "IDGenerator.h" +#include "ExecutionEngine.h" namespace zilliz { namespace milvus { @@ -16,7 +17,10 @@ public: VectorSource(const size_t& n, const float* vectors); - Status Add(const meta::TableFileSchema& table_file_schema, const size_t& num_vectors_to_add, size_t& num_vectors_added); + Status Add(const ExecutionEnginePtr& execution_engine, + const meta::TableFileSchema& table_file_schema, + const size_t& num_vectors_to_add, + size_t& num_vectors_added); size_t GetNumVectorsAdded(); @@ -24,6 +28,8 @@ public: IDNumbers GetVectorIds(); +// Status Serialize(); + private: const size_t n_; diff --git a/cpp/unittest/db/mem_test.cpp b/cpp/unittest/db/mem_test.cpp index 8418b9cd2d..111914f8a9 100644 --- a/cpp/unittest/db/mem_test.cpp +++ b/cpp/unittest/db/mem_test.cpp @@ -6,6 +6,7 @@ #include "utils.h" #include "db/Factories.h" #include "db/Constants.h" +#include "db/EngineFactory.h" using namespace zilliz::milvus; @@ -55,7 +56,10 @@ TEST(MEM_TEST, VECTOR_SOURCE_TEST) { engine::VectorSource source(n, vectors.data()); size_t num_vectors_added; - status = source.Add(table_file_schema, 50, num_vectors_added); + engine::ExecutionEnginePtr execution_engine_ = engine::EngineFactory::Build(table_file_schema.dimension_, + table_file_schema.location_, + (engine::EngineType)table_file_schema.engine_type_); + status = source.Add(execution_engine_, table_file_schema, 50, num_vectors_added); ASSERT_TRUE(status.ok()); ASSERT_EQ(num_vectors_added, 50); @@ -63,7 +67,7 @@ TEST(MEM_TEST, VECTOR_SOURCE_TEST) { engine::IDNumbers vector_ids = source.GetVectorIds(); ASSERT_EQ(vector_ids.size(), 50); - status = source.Add(table_file_schema, 60, num_vectors_added); + status = source.Add(execution_engine_, table_file_schema, 60, num_vectors_added); ASSERT_TRUE(status.ok()); ASSERT_EQ(num_vectors_added, 50); From b6d3c66476f7ea5e89edc4324f2eaeabe17b5e65 Mon Sep 17 00:00:00 2001 From: zhiru Date: Fri, 5 Jul 2019 16:46:15 +0800 Subject: [PATCH 094/189] Implemented add and serialize Former-commit-id: 35a61d2fd0fc9c703ec4efbc9fe8399162e79b8e --- cpp/src/db/MemTable.cpp | 32 +++++++++++++++++++-------- cpp/src/db/MemTable.h | 10 ++++++--- cpp/src/db/MemTableFile.cpp | 42 +++++++++++++++++++++++++++++++++--- cpp/src/db/MemTableFile.h | 8 +++++-- cpp/src/db/VectorSource.cpp | 12 +++++------ cpp/src/db/VectorSource.h | 2 -- cpp/unittest/db/mem_test.cpp | 11 +++++++--- 7 files changed, 89 insertions(+), 28 deletions(-) diff --git a/cpp/src/db/MemTable.cpp b/cpp/src/db/MemTable.cpp index 032d479999..86554695c8 100644 --- a/cpp/src/db/MemTable.cpp +++ b/cpp/src/db/MemTable.cpp @@ -6,24 +6,26 @@ namespace milvus { namespace engine { MemTable::MemTable(const std::string& table_id, - const std::shared_ptr& meta) : + const std::shared_ptr& meta, + const Options& options) : table_id_(table_id), - meta_(meta) { + meta_(meta), + options_(options) { } Status MemTable::Add(VectorSource::Ptr& source) { while (!source->AllAdded()) { MemTableFile::Ptr currentMemTableFile; - if (!mem_table_file_stack_.empty()) { - currentMemTableFile = mem_table_file_stack_.top(); + if (!mem_table_file_list_.empty()) { + currentMemTableFile = mem_table_file_list_.back(); } Status status; - if (mem_table_file_stack_.empty() || currentMemTableFile->isFull()) { - MemTableFile::Ptr newMemTableFile = std::make_shared(table_id_, meta_); + if (mem_table_file_list_.empty() || currentMemTableFile->IsFull()) { + MemTableFile::Ptr newMemTableFile = std::make_shared(table_id_, meta_, options_); status = newMemTableFile->Add(source); if (status.ok()) { - mem_table_file_stack_.push(newMemTableFile); + mem_table_file_list_.emplace_back(newMemTableFile); } } else { @@ -39,11 +41,23 @@ Status MemTable::Add(VectorSource::Ptr& source) { } void MemTable::GetCurrentMemTableFile(MemTableFile::Ptr& mem_table_file) { - mem_table_file = mem_table_file_stack_.top(); + mem_table_file = mem_table_file_list_.back(); } size_t MemTable::GetStackSize() { - return mem_table_file_stack_.size(); + return mem_table_file_list_.size(); +} + +Status MemTable::Serialize() { + for (auto& memTableFile : mem_table_file_list_) { + auto status = memTableFile->Serialize(); + if (!status.ok()) { + std::string errMsg = "MemTable::Serialize failed: " + status.ToString(); + ENGINE_LOG_ERROR << errMsg; + return Status::Error(errMsg); + } + } + return Status::OK(); } } // namespace engine diff --git a/cpp/src/db/MemTable.h b/cpp/src/db/MemTable.h index b9fe4147d8..d5c7cc9e85 100644 --- a/cpp/src/db/MemTable.h +++ b/cpp/src/db/MemTable.h @@ -15,10 +15,10 @@ class MemTable { public: using Ptr = std::shared_ptr; - using MemTableFileStack = std::stack; + using MemTableFileList = std::vector; using MetaPtr = meta::Meta::Ptr; - MemTable(const std::string& table_id, const std::shared_ptr& meta); + MemTable(const std::string& table_id, const std::shared_ptr& meta, const Options& options); Status Add(VectorSource::Ptr& source); @@ -26,13 +26,17 @@ public: size_t GetStackSize(); + Status Serialize(); + private: const std::string table_id_; - MemTableFileStack mem_table_file_stack_; + MemTableFileList mem_table_file_list_; MetaPtr meta_; + Options options_; + }; //MemTable } // namespace engine diff --git a/cpp/src/db/MemTableFile.cpp b/cpp/src/db/MemTableFile.cpp index 58b76ab834..0ff91de00b 100644 --- a/cpp/src/db/MemTableFile.cpp +++ b/cpp/src/db/MemTableFile.cpp @@ -2,6 +2,7 @@ #include "Constants.h" #include "Log.h" #include "EngineFactory.h" +#include "metrics/Metrics.h" #include @@ -10,9 +11,11 @@ namespace milvus { namespace engine { MemTableFile::MemTableFile(const std::string& table_id, - const std::shared_ptr& meta) : + const std::shared_ptr& meta, + const Options& options) : table_id_(table_id), - meta_(meta) { + meta_(meta), + options_(options) { current_mem_ = 0; auto status = CreateTableFile(); @@ -40,6 +43,13 @@ Status MemTableFile::CreateTableFile() { Status MemTableFile::Add(const VectorSource::Ptr& source) { + if (table_file_schema_.dimension_ <= 0) { + std::string errMsg = "MemTableFile::Add: table_file_schema dimension = " + + std::to_string(table_file_schema_.dimension_) + ", table_id = " + table_file_schema_.table_id_; + ENGINE_LOG_ERROR << errMsg; + return Status::Error(errMsg); + } + size_t singleVectorMemSize = table_file_schema_.dimension_ * VECTOR_TYPE_SIZE; size_t memLeft = GetMemLeft(); if (memLeft >= singleVectorMemSize) { @@ -62,11 +72,37 @@ size_t MemTableFile::GetMemLeft() { return (MAX_TABLE_FILE_MEM - current_mem_); } -bool MemTableFile::isFull() { +bool MemTableFile::IsFull() { size_t singleVectorMemSize = table_file_schema_.dimension_ * VECTOR_TYPE_SIZE; return (GetMemLeft() < singleVectorMemSize); } +Status MemTableFile::Serialize() { + + auto start_time = METRICS_NOW_TIME; + + auto size = GetCurrentMem(); + + execution_engine_->Serialize(); + auto end_time = METRICS_NOW_TIME; + auto total_time = METRICS_MICROSECONDS(start_time, end_time); + table_file_schema_.size_ = size; + + server::Metrics::GetInstance().DiskStoreIOSpeedGaugeSet((double)size/total_time); + + table_file_schema_.file_type_ = (size >= options_.index_trigger_size) ? + meta::TableFileSchema::TO_INDEX : meta::TableFileSchema::RAW; + + auto status = meta_->UpdateTableFile(table_file_schema_); + + LOG(DEBUG) << "New " << ((table_file_schema_.file_type_ == meta::TableFileSchema::RAW) ? "raw" : "to_index") + << " file " << table_file_schema_.file_id_ << " of size " << (double)size / (double)M << " M"; + + execution_engine_->Cache(); + + return status; +} + } // namespace engine } // namespace milvus } // namespace zilliz \ No newline at end of file diff --git a/cpp/src/db/MemTableFile.h b/cpp/src/db/MemTableFile.h index 04f30178ea..1be0ae78ba 100644 --- a/cpp/src/db/MemTableFile.h +++ b/cpp/src/db/MemTableFile.h @@ -16,7 +16,7 @@ public: using Ptr = std::shared_ptr; using MetaPtr = meta::Meta::Ptr; - MemTableFile(const std::string& table_id, const std::shared_ptr& meta); + MemTableFile(const std::string& table_id, const std::shared_ptr& meta, const Options& options); Status Add(const VectorSource::Ptr& source); @@ -24,7 +24,9 @@ public: size_t GetMemLeft(); - bool isFull(); + bool IsFull(); + + Status Serialize(); private: @@ -36,6 +38,8 @@ private: MetaPtr meta_; + Options options_; + size_t current_mem_; ExecutionEnginePtr execution_engine_; diff --git a/cpp/src/db/VectorSource.cpp b/cpp/src/db/VectorSource.cpp index f7cef994fa..b113b9ad5e 100644 --- a/cpp/src/db/VectorSource.cpp +++ b/cpp/src/db/VectorSource.cpp @@ -2,6 +2,7 @@ #include "ExecutionEngine.h" #include "EngineFactory.h" #include "Log.h" +#include "metrics/Metrics.h" namespace zilliz { namespace milvus { @@ -21,12 +22,7 @@ Status VectorSource::Add(const ExecutionEnginePtr& execution_engine, const size_t& num_vectors_to_add, size_t& num_vectors_added) { - if (table_file_schema.dimension_ <= 0) { - std::string errMsg = "VectorSource::Add: table_file_schema dimension = " + - std::to_string(table_file_schema.dimension_) + ", table_id = " + table_file_schema.table_id_; - ENGINE_LOG_ERROR << errMsg; - return Status::Error(errMsg); - } + auto start_time = METRICS_NOW_TIME; num_vectors_added = current_num_vectors_added + num_vectors_to_add <= n_ ? num_vectors_to_add : n_ - current_num_vectors_added; IDNumbers vector_ids_to_add; @@ -40,6 +36,10 @@ Status VectorSource::Add(const ExecutionEnginePtr& execution_engine, ENGINE_LOG_ERROR << "VectorSource::Add failed: " + status.ToString(); } + auto end_time = METRICS_NOW_TIME; + auto total_time = METRICS_MICROSECONDS(start_time, end_time); + server::Metrics::GetInstance().AddVectorsPerSecondGaugeSet(static_cast(n_), static_cast(table_file_schema.dimension_), total_time); + return status; } diff --git a/cpp/src/db/VectorSource.h b/cpp/src/db/VectorSource.h index 597eee4ad8..dec31f39e1 100644 --- a/cpp/src/db/VectorSource.h +++ b/cpp/src/db/VectorSource.h @@ -28,8 +28,6 @@ public: IDNumbers GetVectorIds(); -// Status Serialize(); - private: const size_t n_; diff --git a/cpp/unittest/db/mem_test.cpp b/cpp/unittest/db/mem_test.cpp index 111914f8a9..f68d1eb8e3 100644 --- a/cpp/unittest/db/mem_test.cpp +++ b/cpp/unittest/db/mem_test.cpp @@ -86,12 +86,13 @@ TEST(MEM_TEST, VECTOR_SOURCE_TEST) { TEST(MEM_TEST, MEM_TABLE_FILE_TEST) { std::shared_ptr impl_ = engine::DBMetaImplFactory::Build(); + auto options = engine::OptionsFactory::Build(); engine::meta::TableSchema table_schema = BuildTableSchema(); auto status = impl_->CreateTable(table_schema); ASSERT_TRUE(status.ok()); - engine::MemTableFile memTableFile(TABLE_NAME, impl_); + engine::MemTableFile memTableFile(TABLE_NAME, impl_, options); int64_t n_100 = 100; std::vector vectors_100; @@ -120,7 +121,7 @@ TEST(MEM_TEST, MEM_TABLE_FILE_TEST) { vector_ids = source_128M->GetVectorIds(); ASSERT_EQ(vector_ids.size(), n_max - n_100); - ASSERT_TRUE(memTableFile.isFull()); + ASSERT_TRUE(memTableFile.IsFull()); status = impl_->DropAll(); ASSERT_TRUE(status.ok()); @@ -129,6 +130,7 @@ TEST(MEM_TEST, MEM_TABLE_FILE_TEST) { TEST(MEM_TEST, MEM_TABLE_TEST) { std::shared_ptr impl_ = engine::DBMetaImplFactory::Build(); + auto options = engine::OptionsFactory::Build(); engine::meta::TableSchema table_schema = BuildTableSchema(); auto status = impl_->CreateTable(table_schema); @@ -140,7 +142,7 @@ TEST(MEM_TEST, MEM_TABLE_TEST) { engine::VectorSource::Ptr source_100 = std::make_shared(n_100, vectors_100.data()); - engine::MemTable memTable(TABLE_NAME, impl_); + engine::MemTable memTable(TABLE_NAME, impl_, options); status = memTable.Add(source_100); ASSERT_TRUE(status.ok()); @@ -184,6 +186,9 @@ TEST(MEM_TEST, MEM_TABLE_TEST) { int expectedStackSize = 2 + std::ceil((n_1G - n_100) * singleVectorMem / engine::MAX_TABLE_FILE_MEM); ASSERT_EQ(memTable.GetStackSize(), expectedStackSize); + status = memTable.Serialize(); + ASSERT_TRUE(status.ok()); + status = impl_->DropAll(); ASSERT_TRUE(status.ok()); } From 6cf90e61f885aa8a43b20f4f243bd853c705aaa0 Mon Sep 17 00:00:00 2001 From: quicksilver Date: Fri, 5 Jul 2019 18:07:40 +0800 Subject: [PATCH 095/189] MS-161 - Add CI / CD Module to Milvus Project Former-commit-id: 251e2ffd798f653920767f1550d70b2ee729bdff --- CHANGELOGS.md | 1 + ci/function/file_transfer.groovy | 10 + ci/jenkinsfile/cleanup_dev.groovy | 12 + ci/jenkinsfile/deploy2dev.groovy | 11 + ci/jenkinsfile/dev_test.groovy | 17 ++ ci/jenkinsfile/milvus_build.groovy | 17 ++ ci/jenkinsfile/milvus_build_no_ut.groovy | 17 ++ ci/jenkinsfile/packaged_milvus.groovy | 44 ++++ ci/jenkinsfile/packaged_milvus_no_ut.groovy | 26 ++ ci/jenkinsfile/publish_docker.groovy | 31 +++ ci/jenkinsfile/upload_dev_test_out.groovy | 26 ++ ci/main_jenkinsfile | 256 ++++++++++++++++++++ ci/main_jenkinsfile_no_ut | 256 ++++++++++++++++++++ ci/pod_containers/milvus-engine-build.yaml | 13 + ci/pod_containers/milvus-testframework.yaml | 13 + ci/pod_containers/publish-docker.yaml | 22 ++ 16 files changed, 772 insertions(+) create mode 100644 ci/function/file_transfer.groovy create mode 100644 ci/jenkinsfile/cleanup_dev.groovy create mode 100644 ci/jenkinsfile/deploy2dev.groovy create mode 100644 ci/jenkinsfile/dev_test.groovy create mode 100644 ci/jenkinsfile/milvus_build.groovy create mode 100644 ci/jenkinsfile/milvus_build_no_ut.groovy create mode 100644 ci/jenkinsfile/packaged_milvus.groovy create mode 100644 ci/jenkinsfile/packaged_milvus_no_ut.groovy create mode 100644 ci/jenkinsfile/publish_docker.groovy create mode 100644 ci/jenkinsfile/upload_dev_test_out.groovy create mode 100644 ci/main_jenkinsfile create mode 100644 ci/main_jenkinsfile_no_ut create mode 100644 ci/pod_containers/milvus-engine-build.yaml create mode 100644 ci/pod_containers/milvus-testframework.yaml create mode 100644 ci/pod_containers/publish-docker.yaml diff --git a/CHANGELOGS.md b/CHANGELOGS.md index a5d7bfec58..def4965a41 100644 --- a/CHANGELOGS.md +++ b/CHANGELOGS.md @@ -15,3 +15,4 @@ Please mark all change in change log and use the ticket from JIRA. ### Task - MS-1 - Add CHANGELOG.md +- MS-161 - Add CI / CD Module to Milvus Project diff --git a/ci/function/file_transfer.groovy b/ci/function/file_transfer.groovy new file mode 100644 index 0000000000..bebae14832 --- /dev/null +++ b/ci/function/file_transfer.groovy @@ -0,0 +1,10 @@ +def FileTransfer (sourceFiles, remoteDirectory, remoteIP, protocol = "ftp", makeEmptyDirs = true) { + if (protocol == "ftp") { + ftpPublisher masterNodeName: '', paramPublish: [parameterName: ''], alwaysPublishFromMaster: false, continueOnError: false, failOnError: true, publishers: [ + [configName: "${remoteIP}", transfers: [ + [asciiMode: false, cleanRemote: false, excludes: '', flatten: false, makeEmptyDirs: "${makeEmptyDirs}", noDefaultExcludes: false, patternSeparator: '[, ]+', remoteDirectory: "${remoteDirectory}", remoteDirectorySDF: false, removePrefix: '', sourceFiles: "${sourceFiles}"]], usePromotionTimestamp: true, useWorkspaceInPromotion: false, verbose: true + ] + ] + } +} +return this diff --git a/ci/jenkinsfile/cleanup_dev.groovy b/ci/jenkinsfile/cleanup_dev.groovy new file mode 100644 index 0000000000..32ee43d3b1 --- /dev/null +++ b/ci/jenkinsfile/cleanup_dev.groovy @@ -0,0 +1,12 @@ +try { + sh "helm del --purge ${env.JOB_NAME}-${env.BUILD_NUMBER}" + + if (currentBuild.result == 'ABORTED') { + throw new hudson.AbortException("Dev Test Aborted !") + } else if (currentBuild.result == 'FAILURE') { + error("Dev Test Failure !") + } +} catch (exc) { + updateGitlabCommitStatus name: 'Cleanup Dev', state: 'failed' + throw exc +} diff --git a/ci/jenkinsfile/deploy2dev.groovy b/ci/jenkinsfile/deploy2dev.groovy new file mode 100644 index 0000000000..6e4a23cfe7 --- /dev/null +++ b/ci/jenkinsfile/deploy2dev.groovy @@ -0,0 +1,11 @@ +try { + sh 'helm init --client-only --skip-refresh' + sh 'helm repo add milvus https://registry.zilliz.com/chartrepo/milvus' + sh 'helm repo update' + sh "helm install --set engine.image.repository=registry.zilliz.com/${PROJECT_NAME}/engine --set engine.image.tag=${DOCKER_VERSION} --set expose.type=clusterIP --name ${env.JOB_NAME}-${env.BUILD_NUMBER} --version 0.3.0 milvus/milvus-gpu" +} catch (exc) { + updateGitlabCommitStatus name: 'Deloy to Dev', state: 'failed' + echo 'Helm running failed!' + sh "helm del --purge ${env.JOB_NAME}-${env.BUILD_NUMBER}" + throw exc +} diff --git a/ci/jenkinsfile/dev_test.groovy b/ci/jenkinsfile/dev_test.groovy new file mode 100644 index 0000000000..f5808cef40 --- /dev/null +++ b/ci/jenkinsfile/dev_test.groovy @@ -0,0 +1,17 @@ +container('milvus-testframework') { + timeout(time: 10, unit: 'MINUTES') { + gitlabCommitStatus(name: 'Dev Test') { + try { + dir ("${PROJECT_NAME}_test") { + checkout([$class: 'GitSCM', branches: [[name: "${SEMVER}"]], doGenerateSubmoduleConfigurations: false, extensions: [], submoduleCfg: [], userRemoteConfigs: [[credentialsId: "${params.GIT_USER}", url: "git@192.168.1.105:Test/milvus_test.git"]]]) + sh 'python3 -m pip install -r requirements.txt' + sh "pytest . --alluredir=test_out --ip ${env.JOB_NAME}-${env.BUILD_NUMBER}-milvus-gpu-engine.kube-opt.svc.cluster.local" + } + } catch (exc) { + updateGitlabCommitStatus name: 'Dev Test', state: 'failed' + currentBuild.result = 'FAILURE' + echo 'Milvus Test Failed !' + } + } + } +} diff --git a/ci/jenkinsfile/milvus_build.groovy b/ci/jenkinsfile/milvus_build.groovy new file mode 100644 index 0000000000..ed07d2b992 --- /dev/null +++ b/ci/jenkinsfile/milvus_build.groovy @@ -0,0 +1,17 @@ +container('milvus-build-env') { + timeout(time: 20, unit: 'MINUTES') { + gitlabCommitStatus(name: 'Build Engine') { + dir ("milvus_engine") { + try { + checkout([$class: 'GitSCM', branches: [[name: "${SEMVER}"]], doGenerateSubmoduleConfigurations: false, extensions: [[$class: 'SubmoduleOption',disableSubmodules: false,parentCredentials: true,recursiveSubmodules: true,reference: '',trackingSubmodules: false]], submoduleCfg: [], userRemoteConfigs: [[credentialsId: "${params.GIT_USER}", url: "git@192.168.1.105:megasearch/milvus.git"]]]) + dir ("cpp") { + sh "./build.sh -t ${params.BUILD_TYPE} -u -c" + } + } catch (exc) { + updateGitlabCommitStatus name: 'Build Engine', state: 'failed' + throw exc + } + } + } + } +} diff --git a/ci/jenkinsfile/milvus_build_no_ut.groovy b/ci/jenkinsfile/milvus_build_no_ut.groovy new file mode 100644 index 0000000000..02b971de2f --- /dev/null +++ b/ci/jenkinsfile/milvus_build_no_ut.groovy @@ -0,0 +1,17 @@ +container('milvus-build-env') { + timeout(time: 20, unit: 'MINUTES') { + gitlabCommitStatus(name: 'Build Engine') { + dir ("milvus_engine") { + try { + checkout([$class: 'GitSCM', branches: [[name: "${SEMVER}"]], doGenerateSubmoduleConfigurations: false, extensions: [[$class: 'SubmoduleOption',disableSubmodules: false,parentCredentials: true,recursiveSubmodules: true,reference: '',trackingSubmodules: false]], submoduleCfg: [], userRemoteConfigs: [[credentialsId: "${params.GIT_USER}", url: "git@192.168.1.105:megasearch/milvus.git"]]]) + dir ("cpp") { + sh "./build.sh -t ${params.BUILD_TYPE}" + } + } catch (exc) { + updateGitlabCommitStatus name: 'Build Engine', state: 'failed' + throw exc + } + } + } + } +} diff --git a/ci/jenkinsfile/packaged_milvus.groovy b/ci/jenkinsfile/packaged_milvus.groovy new file mode 100644 index 0000000000..407b100589 --- /dev/null +++ b/ci/jenkinsfile/packaged_milvus.groovy @@ -0,0 +1,44 @@ +container('milvus-build-env') { + timeout(time: 5, unit: 'MINUTES') { + dir ("milvus_engine") { + dir ("cpp") { + gitlabCommitStatus(name: 'Packaged Engine') { + if (fileExists('milvus')) { + try { + sh "tar -zcvf ./${PROJECT_NAME}-engine-${PACKAGE_VERSION}.tar.gz ./milvus" + def fileTransfer = load "${env.WORKSPACE}/ci/function/file_transfer.groovy" + fileTransfer.FileTransfer("${PROJECT_NAME}-engine-${PACKAGE_VERSION}.tar.gz", "${PROJECT_NAME}/engine/${JOB_NAME}-${BUILD_ID}", 'nas storage') + if (currentBuild.resultIsBetterOrEqualTo('SUCCESS')) { + echo "Download Milvus Engine Binary Viewer \"http://192.168.1.126:8080/${PROJECT_NAME}/engine/${JOB_NAME}-${BUILD_ID}/${PROJECT_NAME}-engine-${PACKAGE_VERSION}.tar.gz\"" + } + } catch (exc) { + updateGitlabCommitStatus name: 'Packaged Engine', state: 'failed' + throw exc + } + } else { + updateGitlabCommitStatus name: 'Packaged Engine', state: 'failed' + error("Milvus binary directory don't exists!") + } + } + + gitlabCommitStatus(name: 'Packaged Engine lcov') { + if (fileExists('lcov_out')) { + try { + def fileTransfer = load "${env.WORKSPACE}/ci/function/file_transfer.groovy" + fileTransfer.FileTransfer("lcov_out/", "${PROJECT_NAME}/lcov/${JOB_NAME}-${BUILD_ID}", 'nas storage') + if (currentBuild.resultIsBetterOrEqualTo('SUCCESS')) { + echo "Milvus lcov out Viewer \"http://192.168.1.126:8080/${PROJECT_NAME}/lcov/${JOB_NAME}-${BUILD_ID}/lcov_out/\"" + } + } catch (exc) { + updateGitlabCommitStatus name: 'Packaged Engine lcov', state: 'failed' + throw exc + } + } else { + updateGitlabCommitStatus name: 'Packaged Engine lcov', state: 'failed' + error("Milvus lcov out directory don't exists!") + } + } + } + } + } +} diff --git a/ci/jenkinsfile/packaged_milvus_no_ut.groovy b/ci/jenkinsfile/packaged_milvus_no_ut.groovy new file mode 100644 index 0000000000..b6c31540a1 --- /dev/null +++ b/ci/jenkinsfile/packaged_milvus_no_ut.groovy @@ -0,0 +1,26 @@ +container('milvus-build-env') { + timeout(time: 5, unit: 'MINUTES') { + dir ("milvus_engine") { + dir ("cpp") { + gitlabCommitStatus(name: 'Packaged Engine') { + if (fileExists('milvus')) { + try { + sh "tar -zcvf ./${PROJECT_NAME}-engine-${PACKAGE_VERSION}.tar.gz ./milvus" + def fileTransfer = load "${env.WORKSPACE}/ci/function/file_transfer.groovy" + fileTransfer.FileTransfer("${PROJECT_NAME}-engine-${PACKAGE_VERSION}.tar.gz", "${PROJECT_NAME}/engine/${JOB_NAME}-${BUILD_ID}", 'nas storage') + if (currentBuild.resultIsBetterOrEqualTo('SUCCESS')) { + echo "Download Milvus Engine Binary Viewer \"http://192.168.1.126:8080/${PROJECT_NAME}/engine/${JOB_NAME}-${BUILD_ID}/${PROJECT_NAME}-engine-${PACKAGE_VERSION}.tar.gz\"" + } + } catch (exc) { + updateGitlabCommitStatus name: 'Packaged Engine', state: 'failed' + throw exc + } + } else { + updateGitlabCommitStatus name: 'Packaged Engine', state: 'failed' + error("Milvus binary directory don't exists!") + } + } + } + } + } +} diff --git a/ci/jenkinsfile/publish_docker.groovy b/ci/jenkinsfile/publish_docker.groovy new file mode 100644 index 0000000000..04f1a8567d --- /dev/null +++ b/ci/jenkinsfile/publish_docker.groovy @@ -0,0 +1,31 @@ +container('publish-docker') { + timeout(time: 15, unit: 'MINUTES') { + gitlabCommitStatus(name: 'Publish Engine Docker') { + try { + dir ("${PROJECT_NAME}_build") { + checkout([$class: 'GitSCM', branches: [[name: "${SEMVER}"]], doGenerateSubmoduleConfigurations: false, extensions: [], submoduleCfg: [], userRemoteConfigs: [[credentialsId: "${params.GIT_USER}", url: "git@192.168.1.105:build/milvus_build.git"]]]) + dir ("docker/deploy/ubuntu16.04/free_version") { + sh "curl -O -u anonymous: ftp://192.168.1.126/data/${PROJECT_NAME}/engine/${JOB_NAME}-${BUILD_ID}/${PROJECT_NAME}-engine-${PACKAGE_VERSION}.tar.gz" + sh "tar zxvf ${PROJECT_NAME}-engine-${PACKAGE_VERSION}.tar.gz" + try { + docker.withRegistry('https://registry.zilliz.com', 'a54e38ef-c424-4ea9-9224-b25fc20e3924') { + def customImage = docker.build("${PROJECT_NAME}/engine:${DOCKER_VERSION}") + customImage.push() + } + echo "Docker Pull Command: docker pull registry.zilliz.com/${PROJECT_NAME}/engine:${DOCKER_VERSION}" + } catch (exc) { + updateGitlabCommitStatus name: 'Publish Engine Docker', state: 'canceled' + throw exc + } finally { + sh "docker rmi ${PROJECT_NAME}/engine:${DOCKER_VERSION}" + } + } + } + } catch (exc) { + updateGitlabCommitStatus name: 'Publish Engine Docker', state: 'failed' + echo 'Publish docker failed!' + throw exc + } + } + } +} diff --git a/ci/jenkinsfile/upload_dev_test_out.groovy b/ci/jenkinsfile/upload_dev_test_out.groovy new file mode 100644 index 0000000000..c401b16608 --- /dev/null +++ b/ci/jenkinsfile/upload_dev_test_out.groovy @@ -0,0 +1,26 @@ +container('milvus-testframework') { + timeout(time: 5, unit: 'MINUTES') { + dir ("${PROJECT_NAME}_test") { + gitlabCommitStatus(name: 'Upload Dev Test Out') { + if (fileExists('test_out')) { + try { + def fileTransfer = load "${env.WORKSPACE}/ci/function/file_transfer.groovy" + fileTransfer.FileTransfer("test_out/", "${PROJECT_NAME}/test/${JOB_NAME}-${BUILD_ID}", 'nas storage') + if (currentBuild.resultIsBetterOrEqualTo('SUCCESS')) { + echo "Milvus Dev Test Out Viewer \"ftp://192.168.1.126/data/${PROJECT_NAME}/test/${JOB_NAME}-${BUILD_ID}\"" + } + } catch (hudson.AbortException ae) { + updateGitlabCommitStatus name: 'Upload Dev Test Out', state: 'canceled' + currentBuild.result = 'ABORTED' + } catch (exc) { + updateGitlabCommitStatus name: 'Upload Dev Test Out', state: 'failed' + currentBuild.result = 'FAILURE' + } + } else { + updateGitlabCommitStatus name: 'Upload Dev Test Out', state: 'failed' + echo "Milvus Dev Test Out directory don't exists!" + } + } + } + } +} diff --git a/ci/main_jenkinsfile b/ci/main_jenkinsfile new file mode 100644 index 0000000000..c144c46685 --- /dev/null +++ b/ci/main_jenkinsfile @@ -0,0 +1,256 @@ +pipeline { + agent none + + options { + timestamps() + } + + environment { + PROJECT_NAME = "milvus" + LOWER_BUILD_TYPE = BUILD_TYPE.toLowerCase() + SEMVER = "${env.gitlabSourceBranch == null ? params.ENGINE_BRANCH.substring(params.ENGINE_BRANCH.lastIndexOf('/') + 1) : env.gitlabSourceBranch}" + GITLAB_AFTER_COMMIT = "${env.gitlabAfter == null ? null : env.gitlabAfter}" + SUFFIX_VERSION_NAME = "${env.gitlabAfter == null ? null : env.gitlabAfter.substring(0, 6)}" + DOCKER_VERSION_STR = "${env.gitlabAfter == null ? "${SEMVER}-${LOWER_BUILD_TYPE}" : "${SEMVER}-${LOWER_BUILD_TYPE}-${SUFFIX_VERSION_NAME}"}" + } + + stages { + stage("Ubuntu 16.04") { + environment { + PACKAGE_VERSION = VersionNumber([ + versionNumberString : '${SEMVER}-${LOWER_BUILD_TYPE}-${BUILD_DATE_FORMATTED, "yyyyMMdd"}' + ]); + + DOCKER_VERSION = VersionNumber([ + versionNumberString : '${DOCKER_VERSION_STR}' + ]); + } + + stages { + stage("Run Build") { + agent { + kubernetes { + cloud 'build-kubernetes' + label 'build' + defaultContainer 'jnlp' + containerTemplate { + name 'milvus-build-env' + image 'registry.zilliz.com/milvus/milvus-build-env:v0.10' + ttyEnabled true + command 'cat' + } + } + } + stages { + stage('Build') { + steps { + gitlabCommitStatus(name: 'Build') { + script { + load "${env.WORKSPACE}/ci/jenkinsfile/milvus_build.groovy" + load "${env.WORKSPACE}/ci/jenkinsfile/packaged_milvus.groovy" + } + } + } + } + } + post { + aborted { + script { + updateGitlabCommitStatus name: 'Build', state: 'canceled' + echo "Milvus Build aborted !" + } + } + + failure { + script { + updateGitlabCommitStatus name: 'Build', state: 'failed' + echo "Milvus Build failure !" + } + } + } + } + + stage("Publish docker and helm") { + agent { + kubernetes { + label 'publish' + defaultContainer 'jnlp' + yaml """ +apiVersion: v1 +kind: Pod +metadata: + labels: + app: publish + componet: docker +spec: + containers: + - name: publish-docker + image: registry.zilliz.com/library/zilliz_docker:v1.0.0 + securityContext: + privileged: true + command: + - cat + tty: true + volumeMounts: + - name: docker-sock + mountPath: /var/run/docker.sock + volumes: + - name: docker-sock + hostPath: + path: /var/run/docker.sock +""" + } + } + stages { + stage('Publish Docker') { + steps { + gitlabCommitStatus(name: 'Publish Docker') { + script { + load "${env.WORKSPACE}/ci/jenkinsfile/publish_docker.groovy" + } + } + } + } + } + post { + aborted { + script { + updateGitlabCommitStatus name: 'Publish Docker', state: 'canceled' + echo "Milvus Publish Docker aborted !" + } + } + + failure { + script { + updateGitlabCommitStatus name: 'Publish Docker', state: 'failed' + echo "Milvus Publish Docker failure !" + } + } + } + } + + stage("Deploy to Development") { + stages { + stage("Deploy to Dev") { + agent { + kubernetes { + label 'jenkins-slave' + defaultContainer 'jnlp' + } + } + stages { + stage('Deploy') { + steps { + gitlabCommitStatus(name: 'Deloy to Dev') { + script { + load "${env.WORKSPACE}/ci/jenkinsfile/deploy2dev.groovy" + } + } + } + } + } + post { + aborted { + script { + updateGitlabCommitStatus name: 'Deloy to Dev', state: 'canceled' + echo "Milvus Deloy to Dev aborted !" + } + } + + failure { + script { + updateGitlabCommitStatus name: 'Deloy to Dev', state: 'failed' + echo "Milvus Deloy to Dev failure !" + } + } + } + } + + stage("Dev Test") { + agent { + kubernetes { + label 'test' + defaultContainer 'jnlp' + containerTemplate { + name 'milvus-testframework' + image 'registry.zilliz.com/milvus/milvus-test:v0.1' + ttyEnabled true + command 'cat' + } + } + } + stages { + stage('Test') { + steps { + script { + load "${env.WORKSPACE}/ci/jenkinsfile/dev_test.groovy" + load "${env.WORKSPACE}/ci/jenkinsfile/upload_dev_test_out.groovy" + } + } + } + } + } + + stage ("Cleanup Dev") { + agent { + kubernetes { + label 'jenkins-slave' + defaultContainer 'jnlp' + } + } + stages { + stage('Cleanup') { + steps { + gitlabCommitStatus(name: 'Cleanup Dev') { + script { + load "${env.WORKSPACE}/ci/jenkinsfile/cleanup_dev.groovy" + } + } + } + } + } + post { + aborted { + script { + updateGitlabCommitStatus name: 'Cleanup Dev', state: 'canceled' + echo "Milvus Cleanup Dev aborted !" + } + } + + failure { + script { + updateGitlabCommitStatus name: 'Cleanup Dev', state: 'failed' + echo "Milvus Cleanup Dev failure !" + } + } + } + } + } + } + } + } + } + + post { + success { + script { + updateGitlabCommitStatus name: 'CI/CD', state: 'success' + echo "Milvus CI/CD success !" + } + } + + aborted { + script { + updateGitlabCommitStatus name: 'CI/CD', state: 'canceled' + echo "Milvus CI/CD aborted !" + } + } + + failure { + script { + updateGitlabCommitStatus name: 'CI/CD', state: 'failed' + echo "Milvus CI/CD failure !" + } + } + } +} diff --git a/ci/main_jenkinsfile_no_ut b/ci/main_jenkinsfile_no_ut new file mode 100644 index 0000000000..277ec155a5 --- /dev/null +++ b/ci/main_jenkinsfile_no_ut @@ -0,0 +1,256 @@ +pipeline { + agent none + + options { + timestamps() + } + + environment { + PROJECT_NAME = "milvus" + LOWER_BUILD_TYPE = BUILD_TYPE.toLowerCase() + SEMVER = "${env.gitlabSourceBranch == null ? params.ENGINE_BRANCH.substring(params.ENGINE_BRANCH.lastIndexOf('/') + 1) : env.gitlabSourceBranch}" + GITLAB_AFTER_COMMIT = "${env.gitlabAfter == null ? null : env.gitlabAfter}" + SUFFIX_VERSION_NAME = "${env.gitlabAfter == null ? null : env.gitlabAfter.substring(0, 6)}" + DOCKER_VERSION_STR = "${env.gitlabAfter == null ? "${SEMVER}-${LOWER_BUILD_TYPE}" : "${SEMVER}-${LOWER_BUILD_TYPE}-${SUFFIX_VERSION_NAME}"}" + } + + stages { + stage("Ubuntu 16.04") { + environment { + PACKAGE_VERSION = VersionNumber([ + versionNumberString : '${SEMVER}-${LOWER_BUILD_TYPE}-${BUILD_DATE_FORMATTED, "yyyyMMdd"}' + ]); + + DOCKER_VERSION = VersionNumber([ + versionNumberString : '${DOCKER_VERSION_STR}' + ]); + } + + stages { + stage("Run Build") { + agent { + kubernetes { + cloud 'build-kubernetes' + label 'build' + defaultContainer 'jnlp' + containerTemplate { + name 'milvus-build-env' + image 'registry.zilliz.com/milvus/milvus-build-env:v0.10' + ttyEnabled true + command 'cat' + } + } + } + stages { + stage('Build') { + steps { + gitlabCommitStatus(name: 'Build') { + script { + load "${env.WORKSPACE}/ci/jenkinsfile/milvus_build_no_ut.groovy" + load "${env.WORKSPACE}/ci/jenkinsfile/packaged_milvus_no_ut.groovy" + } + } + } + } + } + post { + aborted { + script { + updateGitlabCommitStatus name: 'Build', state: 'canceled' + echo "Milvus Build aborted !" + } + } + + failure { + script { + updateGitlabCommitStatus name: 'Build', state: 'failed' + echo "Milvus Build failure !" + } + } + } + } + + stage("Publish docker and helm") { + agent { + kubernetes { + label 'publish' + defaultContainer 'jnlp' + yaml """ +apiVersion: v1 +kind: Pod +metadata: + labels: + app: publish + componet: docker +spec: + containers: + - name: publish-docker + image: registry.zilliz.com/library/zilliz_docker:v1.0.0 + securityContext: + privileged: true + command: + - cat + tty: true + volumeMounts: + - name: docker-sock + mountPath: /var/run/docker.sock + volumes: + - name: docker-sock + hostPath: + path: /var/run/docker.sock +""" + } + } + stages { + stage('Publish Docker') { + steps { + gitlabCommitStatus(name: 'Publish Docker') { + script { + load "${env.WORKSPACE}/ci/jenkinsfile/publish_docker.groovy" + } + } + } + } + } + post { + aborted { + script { + updateGitlabCommitStatus name: 'Publish Docker', state: 'canceled' + echo "Milvus Publish Docker aborted !" + } + } + + failure { + script { + updateGitlabCommitStatus name: 'Publish Docker', state: 'failed' + echo "Milvus Publish Docker failure !" + } + } + } + } + + stage("Deploy to Development") { + stages { + stage("Deploy to Dev") { + agent { + kubernetes { + label 'jenkins-slave' + defaultContainer 'jnlp' + } + } + stages { + stage('Deploy') { + steps { + gitlabCommitStatus(name: 'Deloy to Dev') { + script { + load "${env.WORKSPACE}/ci/jenkinsfile/deploy2dev.groovy" + } + } + } + } + } + post { + aborted { + script { + updateGitlabCommitStatus name: 'Deloy to Dev', state: 'canceled' + echo "Milvus Deloy to Dev aborted !" + } + } + + failure { + script { + updateGitlabCommitStatus name: 'Deloy to Dev', state: 'failed' + echo "Milvus Deloy to Dev failure !" + } + } + } + } + + stage("Dev Test") { + agent { + kubernetes { + label 'test' + defaultContainer 'jnlp' + containerTemplate { + name 'milvus-testframework' + image 'registry.zilliz.com/milvus/milvus-test:v0.1' + ttyEnabled true + command 'cat' + } + } + } + stages { + stage('Test') { + steps { + script { + load "${env.WORKSPACE}/ci/jenkinsfile/dev_test.groovy" + load "${env.WORKSPACE}/ci/jenkinsfile/upload_dev_test_out.groovy" + } + } + } + } + } + + stage ("Cleanup Dev") { + agent { + kubernetes { + label 'jenkins-slave' + defaultContainer 'jnlp' + } + } + stages { + stage('Cleanup') { + steps { + gitlabCommitStatus(name: 'Cleanup Dev') { + script { + load "${env.WORKSPACE}/ci/jenkinsfile/cleanup_dev.groovy" + } + } + } + } + } + post { + aborted { + script { + updateGitlabCommitStatus name: 'Cleanup Dev', state: 'canceled' + echo "Milvus Cleanup Dev aborted !" + } + } + + failure { + script { + updateGitlabCommitStatus name: 'Cleanup Dev', state: 'failed' + echo "Milvus Cleanup Dev failure !" + } + } + } + } + } + } + } + } + } + + post { + success { + script { + updateGitlabCommitStatus name: 'CI/CD', state: 'success' + echo "Milvus CI/CD success !" + } + } + + aborted { + script { + updateGitlabCommitStatus name: 'CI/CD', state: 'canceled' + echo "Milvus CI/CD aborted !" + } + } + + failure { + script { + updateGitlabCommitStatus name: 'CI/CD', state: 'failed' + echo "Milvus CI/CD failure !" + } + } + } +} diff --git a/ci/pod_containers/milvus-engine-build.yaml b/ci/pod_containers/milvus-engine-build.yaml new file mode 100644 index 0000000000..cd5352ffef --- /dev/null +++ b/ci/pod_containers/milvus-engine-build.yaml @@ -0,0 +1,13 @@ +apiVersion: v1 +kind: Pod +metadata: + labels: + app: milvus + componet: build-env +spec: + containers: + - name: milvus-build-env + image: registry.zilliz.com/milvus/milvus-build-env:v0.9 + command: + - cat + tty: true diff --git a/ci/pod_containers/milvus-testframework.yaml b/ci/pod_containers/milvus-testframework.yaml new file mode 100644 index 0000000000..7a98fbca8e --- /dev/null +++ b/ci/pod_containers/milvus-testframework.yaml @@ -0,0 +1,13 @@ +apiVersion: v1 +kind: Pod +metadata: + labels: + app: milvus + componet: testframework +spec: + containers: + - name: milvus-testframework + image: registry.zilliz.com/milvus/milvus-test:v0.1 + command: + - cat + tty: true diff --git a/ci/pod_containers/publish-docker.yaml b/ci/pod_containers/publish-docker.yaml new file mode 100644 index 0000000000..268afb1331 --- /dev/null +++ b/ci/pod_containers/publish-docker.yaml @@ -0,0 +1,22 @@ +apiVersion: v1 +kind: Pod +metadata: + labels: + app: publish + componet: docker +spec: + containers: + - name: publish-docker + image: registry.zilliz.com/library/zilliz_docker:v1.0.0 + securityContext: + privileged: true + command: + - cat + tty: true + volumeMounts: + - name: docker-sock + mountPath: /var/run/docker.sock + volumes: + - name: docker-sock + hostPath: + path: /var/run/docker.sock From ad4e841664ef55a13c012f12f0933090c5ebc3b2 Mon Sep 17 00:00:00 2001 From: zhiru Date: Sun, 7 Jul 2019 13:50:39 +0800 Subject: [PATCH 096/189] add mem manager Former-commit-id: 88ceef178607ea2e49eec64c533959a077c3e4a2 --- cpp/src/db/DBImpl.cpp | 3 +- cpp/src/db/DBImpl.h | 4 +- cpp/src/db/Factories.cpp | 11 +++ cpp/src/db/Factories.h | 5 ++ cpp/src/db/MemManager.h | 14 ++-- cpp/src/db/MemManagerAbstract.h | 25 ++++++ cpp/src/db/MemTable.cpp | 10 ++- cpp/src/db/MemTable.h | 6 +- cpp/src/db/NewMemManager.cpp | 92 +++++++++++++++++++++ cpp/src/db/NewMemManager.h | 54 +++++++++++++ cpp/src/db/VectorSource.cpp | 15 +++- cpp/unittest/db/mem_test.cpp | 137 +++++++++++++++++++++++++++++++- 12 files changed, 356 insertions(+), 20 deletions(-) create mode 100644 cpp/src/db/MemManagerAbstract.h create mode 100644 cpp/src/db/NewMemManager.cpp create mode 100644 cpp/src/db/NewMemManager.h diff --git a/cpp/src/db/DBImpl.cpp b/cpp/src/db/DBImpl.cpp index 46101dcf93..62d6b9c3cc 100644 --- a/cpp/src/db/DBImpl.cpp +++ b/cpp/src/db/DBImpl.cpp @@ -87,8 +87,7 @@ DBImpl::DBImpl(const Options& options) compact_thread_pool_(1, 1), index_thread_pool_(1, 1) { meta_ptr_ = DBMetaImplFactory::Build(options.meta, options.mode); - mem_mgr_ = std::make_shared(meta_ptr_, options_); - // mem_mgr_ = (MemManagerPtr)(new MemManager(meta_ptr_, options_)); + mem_mgr_ = MemManagerFactory::Build(meta_ptr_, options_); if (options.mode != Options::MODE::READ_ONLY) { StartTimerTasks(); } diff --git a/cpp/src/db/DBImpl.h b/cpp/src/db/DBImpl.h index cc632847c1..4853a0fb4d 100644 --- a/cpp/src/db/DBImpl.h +++ b/cpp/src/db/DBImpl.h @@ -9,6 +9,7 @@ #include "MemManager.h" #include "Types.h" #include "utils/ThreadPool.h" +#include "MemManagerAbstract.h" #include #include @@ -33,7 +34,6 @@ class Meta; class DBImpl : public DB { public: using MetaPtr = meta::Meta::Ptr; - using MemManagerPtr = typename MemManager::Ptr; explicit DBImpl(const Options &options); @@ -131,7 +131,7 @@ class DBImpl : public DB { std::thread bg_timer_thread_; MetaPtr meta_ptr_; - MemManagerPtr mem_mgr_; + MemManagerAbstractPtr mem_mgr_; server::ThreadPool compact_thread_pool_; std::list> compact_thread_results_; diff --git a/cpp/src/db/Factories.cpp b/cpp/src/db/Factories.cpp index 231c3cce4f..abcc0821ab 100644 --- a/cpp/src/db/Factories.cpp +++ b/cpp/src/db/Factories.cpp @@ -6,6 +6,8 @@ #include #include "Factories.h" #include "DBImpl.h" +#include "MemManager.h" +#include "NewMemManager.h" #include #include @@ -98,6 +100,15 @@ DB* DBFactory::Build(const Options& options) { return new DBImpl(options); } +MemManagerAbstractPtr MemManagerFactory::Build(const std::shared_ptr& meta, + const Options& options) { + bool useNew = true; + if (useNew) { + return std::make_shared(meta, options); + } + return std::make_shared(meta, options); +} + } // namespace engine } // namespace milvus } // namespace zilliz diff --git a/cpp/src/db/Factories.h b/cpp/src/db/Factories.h index 889922b17a..567bc0a8bc 100644 --- a/cpp/src/db/Factories.h +++ b/cpp/src/db/Factories.h @@ -10,6 +10,7 @@ #include "MySQLMetaImpl.h" #include "Options.h" #include "ExecutionEngine.h" +#include "MemManagerAbstract.h" #include #include @@ -36,6 +37,10 @@ struct DBFactory { static DB* Build(const Options&); }; +struct MemManagerFactory { + static MemManagerAbstractPtr Build(const std::shared_ptr& meta, const Options& options); +}; + } // namespace engine } // namespace milvus } // namespace zilliz diff --git a/cpp/src/db/MemManager.h b/cpp/src/db/MemManager.h index 0ce88d504d..95303889db 100644 --- a/cpp/src/db/MemManager.h +++ b/cpp/src/db/MemManager.h @@ -9,13 +9,13 @@ #include "IDGenerator.h" #include "Status.h" #include "Meta.h" +#include "MemManagerAbstract.h" #include #include #include #include #include -#include namespace zilliz { namespace milvus { @@ -62,7 +62,7 @@ private: -class MemManager { +class MemManager : public MemManagerAbstract { public: using MetaPtr = meta::Meta::Ptr; using MemVectorsPtr = typename MemVectors::Ptr; @@ -71,16 +71,16 @@ public: MemManager(const std::shared_ptr& meta, const Options& options) : meta_(meta), options_(options) {} - MemVectorsPtr GetMemByTable(const std::string& table_id); - Status InsertVectors(const std::string& table_id, - size_t n, const float* vectors, IDNumbers& vector_ids); + size_t n, const float* vectors, IDNumbers& vector_ids) override; - Status Serialize(std::set& table_ids); + Status Serialize(std::set& table_ids) override; - Status EraseMemVector(const std::string& table_id); + Status EraseMemVector(const std::string& table_id) override; private: + MemVectorsPtr GetMemByTable(const std::string& table_id); + Status InsertVectorsNoLock(const std::string& table_id, size_t n, const float* vectors, IDNumbers& vector_ids); Status ToImmutable(); diff --git a/cpp/src/db/MemManagerAbstract.h b/cpp/src/db/MemManagerAbstract.h new file mode 100644 index 0000000000..74222df1e8 --- /dev/null +++ b/cpp/src/db/MemManagerAbstract.h @@ -0,0 +1,25 @@ +#pragma once + +#include + +namespace zilliz { +namespace milvus { +namespace engine { + +class MemManagerAbstract { +public: + + virtual Status InsertVectors(const std::string& table_id, + size_t n, const float* vectors, IDNumbers& vector_ids) = 0; + + virtual Status Serialize(std::set& table_ids) = 0; + + virtual Status EraseMemVector(const std::string& table_id) = 0; + +}; // MemManagerAbstract + +using MemManagerAbstractPtr = std::shared_ptr; + +} // namespace engine +} // namespace milvus +} // namespace zilliz \ No newline at end of file diff --git a/cpp/src/db/MemTable.cpp b/cpp/src/db/MemTable.cpp index 86554695c8..b282ad375a 100644 --- a/cpp/src/db/MemTable.cpp +++ b/cpp/src/db/MemTable.cpp @@ -44,7 +44,7 @@ void MemTable::GetCurrentMemTableFile(MemTableFile::Ptr& mem_table_file) { mem_table_file = mem_table_file_list_.back(); } -size_t MemTable::GetStackSize() { +size_t MemTable::GetTableFileCount() { return mem_table_file_list_.size(); } @@ -60,6 +60,14 @@ Status MemTable::Serialize() { return Status::OK(); } +bool MemTable::Empty() { + return mem_table_file_list_.empty(); +} + +std::string MemTable::GetTableId() { + return table_id_; +} + } // namespace engine } // namespace milvus } // namespace zilliz \ No newline at end of file diff --git a/cpp/src/db/MemTable.h b/cpp/src/db/MemTable.h index d5c7cc9e85..e09d6ddac1 100644 --- a/cpp/src/db/MemTable.h +++ b/cpp/src/db/MemTable.h @@ -24,10 +24,14 @@ public: void GetCurrentMemTableFile(MemTableFile::Ptr& mem_table_file); - size_t GetStackSize(); + size_t GetTableFileCount(); Status Serialize(); + bool Empty(); + + std::string GetTableId(); + private: const std::string table_id_; diff --git a/cpp/src/db/NewMemManager.cpp b/cpp/src/db/NewMemManager.cpp new file mode 100644 index 0000000000..19aba68eb7 --- /dev/null +++ b/cpp/src/db/NewMemManager.cpp @@ -0,0 +1,92 @@ +#include "NewMemManager.h" +#include "VectorSource.h" + +namespace zilliz { +namespace milvus { +namespace engine { + +NewMemManager::MemTablePtr NewMemManager::GetMemByTable(const std::string& table_id) { + auto memIt = mem_id_map_.find(table_id); + if (memIt != mem_id_map_.end()) { + return memIt->second; + } + + mem_id_map_[table_id] = std::make_shared(table_id, meta_, options_); + return mem_id_map_[table_id]; +} + +Status NewMemManager::InsertVectors(const std::string& table_id_, + size_t n_, + const float* vectors_, + IDNumbers& vector_ids_) { + + + std::unique_lock lock(mutex_); + + return InsertVectorsNoLock(table_id_, n_, vectors_, vector_ids_); +} + +Status NewMemManager::InsertVectorsNoLock(const std::string& table_id, + size_t n, + const float* vectors, + IDNumbers& vector_ids) { + MemTablePtr mem = GetMemByTable(table_id); + VectorSource::Ptr source = std::make_shared(n, vectors); + + auto status = mem->Add(source); + if (status.ok()) { + vector_ids = source->GetVectorIds(); + } + return status; +} + +Status NewMemManager::ToImmutable() { + std::unique_lock lock(mutex_); + MemIdMap temp_map; + for (auto& kv: mem_id_map_) { + if(kv.second->Empty()) { + temp_map.insert(kv); + continue;//empty table, no need to serialize + } + immu_mem_list_.push_back(kv.second); + } + + mem_id_map_.swap(temp_map); + return Status::OK(); +} + +Status NewMemManager::Serialize(std::set& table_ids) { + ToImmutable(); + std::unique_lock lock(serialization_mtx_); + table_ids.clear(); + for (auto& mem : immu_mem_list_) { + mem->Serialize(); + table_ids.insert(mem->GetTableId()); + } + immu_mem_list_.clear(); + return Status::OK(); +} + +Status NewMemManager::EraseMemVector(const std::string& table_id) { + {//erase MemVector from rapid-insert cache + std::unique_lock lock(mutex_); + mem_id_map_.erase(table_id); + } + + {//erase MemVector from serialize cache + std::unique_lock lock(serialization_mtx_); + MemList temp_list; + for (auto& mem : immu_mem_list_) { + if(mem->GetTableId() != table_id) { + temp_list.push_back(mem); + } + } + immu_mem_list_.swap(temp_list); + } + + return Status::OK(); +} + +} // namespace engine +} // namespace milvus +} // namespace zilliz \ No newline at end of file diff --git a/cpp/src/db/NewMemManager.h b/cpp/src/db/NewMemManager.h new file mode 100644 index 0000000000..a5f5a9ca13 --- /dev/null +++ b/cpp/src/db/NewMemManager.h @@ -0,0 +1,54 @@ +#pragma once + +#include "Meta.h" +#include "MemTable.h" +#include "Status.h" +#include "MemManagerAbstract.h" + +#include +#include +#include +#include +#include + +namespace zilliz { +namespace milvus { +namespace engine { + +class NewMemManager : public MemManagerAbstract { +public: + using MetaPtr = meta::Meta::Ptr; + using Ptr = std::shared_ptr; + using MemTablePtr = typename MemTable::Ptr; + + NewMemManager(const std::shared_ptr& meta, const Options& options) + : meta_(meta), options_(options) {} + + Status InsertVectors(const std::string& table_id, + size_t n, const float* vectors, IDNumbers& vector_ids) override; + + Status Serialize(std::set& table_ids) override; + + Status EraseMemVector(const std::string& table_id) override; + +private: + MemTablePtr GetMemByTable(const std::string& table_id); + + Status InsertVectorsNoLock(const std::string& table_id, + size_t n, const float* vectors, IDNumbers& vector_ids); + Status ToImmutable(); + + using MemIdMap = std::map; + using MemList = std::vector; + MemIdMap mem_id_map_; + MemList immu_mem_list_; + MetaPtr meta_; + Options options_; + std::mutex mutex_; + std::mutex serialization_mtx_; +}; // NewMemManager + + +} // namespace engine +} // namespace milvus +} // namespace zilliz \ No newline at end of file diff --git a/cpp/src/db/VectorSource.cpp b/cpp/src/db/VectorSource.cpp index b113b9ad5e..d032be51f6 100644 --- a/cpp/src/db/VectorSource.cpp +++ b/cpp/src/db/VectorSource.cpp @@ -24,13 +24,18 @@ Status VectorSource::Add(const ExecutionEnginePtr& execution_engine, auto start_time = METRICS_NOW_TIME; - num_vectors_added = current_num_vectors_added + num_vectors_to_add <= n_ ? num_vectors_to_add : n_ - current_num_vectors_added; + num_vectors_added = current_num_vectors_added + num_vectors_to_add <= n_ ? + num_vectors_to_add : n_ - current_num_vectors_added; IDNumbers vector_ids_to_add; id_generator_->GetNextIDNumbers(num_vectors_added, vector_ids_to_add); - Status status = execution_engine->AddWithIds(num_vectors_added, vectors_ + current_num_vectors_added, vector_ids_to_add.data()); + Status status = execution_engine->AddWithIds(num_vectors_added, + vectors_ + current_num_vectors_added * table_file_schema.dimension_, + vector_ids_to_add.data()); if (status.ok()) { current_num_vectors_added += num_vectors_added; - vector_ids_.insert(vector_ids_.end(), vector_ids_to_add.begin(), vector_ids_to_add.end()); + vector_ids_.insert(vector_ids_.end(), + std::make_move_iterator(vector_ids_to_add.begin()), + std::make_move_iterator(vector_ids_to_add.end())); } else { ENGINE_LOG_ERROR << "VectorSource::Add failed: " + status.ToString(); @@ -38,7 +43,9 @@ Status VectorSource::Add(const ExecutionEnginePtr& execution_engine, auto end_time = METRICS_NOW_TIME; auto total_time = METRICS_MICROSECONDS(start_time, end_time); - server::Metrics::GetInstance().AddVectorsPerSecondGaugeSet(static_cast(n_), static_cast(table_file_schema.dimension_), total_time); + server::Metrics::GetInstance().AddVectorsPerSecondGaugeSet(static_cast(n_), + static_cast(table_file_schema.dimension_), + total_time); return status; } diff --git a/cpp/unittest/db/mem_test.cpp b/cpp/unittest/db/mem_test.cpp index f68d1eb8e3..915610adcc 100644 --- a/cpp/unittest/db/mem_test.cpp +++ b/cpp/unittest/db/mem_test.cpp @@ -7,6 +7,11 @@ #include "db/Factories.h" #include "db/Constants.h" #include "db/EngineFactory.h" +#include "metrics/Metrics.h" + +#include +#include +#include using namespace zilliz::milvus; @@ -29,6 +34,9 @@ namespace { vectors.clear(); vectors.resize(n*TABLE_DIM); float* data = vectors.data(); +// std::random_device rd; +// std::mt19937 gen(rd()); +// std::uniform_real_distribution<> dis(0.0, 1.0); for(int i = 0; i < n; i++) { for(int j = 0; j < TABLE_DIM; j++) data[TABLE_DIM * i + j] = drand48(); data[TABLE_DIM * i] += i / 2000.; @@ -169,7 +177,7 @@ TEST(MEM_TEST, MEM_TABLE_TEST) { memTable.GetCurrentMemTableFile(memTableFile); ASSERT_EQ(memTableFile->GetCurrentMem(), n_100 * singleVectorMem); - ASSERT_EQ(memTable.GetStackSize(), 2); + ASSERT_EQ(memTable.GetTableFileCount(), 2); int64_t n_1G = 1024000; std::vector vectors_1G; @@ -183,8 +191,8 @@ TEST(MEM_TEST, MEM_TABLE_TEST) { vector_ids = source_1G->GetVectorIds(); ASSERT_EQ(vector_ids.size(), n_1G); - int expectedStackSize = 2 + std::ceil((n_1G - n_100) * singleVectorMem / engine::MAX_TABLE_FILE_MEM); - ASSERT_EQ(memTable.GetStackSize(), expectedStackSize); + int expectedTableFileCount = 2 + std::ceil((n_1G - n_100) * singleVectorMem / engine::MAX_TABLE_FILE_MEM); + ASSERT_EQ(memTable.GetTableFileCount(), expectedTableFileCount); status = memTable.Serialize(); ASSERT_TRUE(status.ok()); @@ -193,4 +201,127 @@ TEST(MEM_TEST, MEM_TABLE_TEST) { ASSERT_TRUE(status.ok()); } +TEST(MEM_TEST, MEM_MANAGER_TEST) { + + auto options = engine::OptionsFactory::Build(); + options.meta.path = "/tmp/milvus_test"; + options.meta.backend_uri = "sqlite://:@:/"; + auto db_ = engine::DBFactory::Build(options); + + engine::meta::TableSchema table_info = BuildTableSchema(); + engine::Status stat = db_->CreateTable(table_info); + + engine::meta::TableSchema table_info_get; + table_info_get.table_id_ = TABLE_NAME; + stat = db_->DescribeTable(table_info_get); + ASSERT_STATS(stat); + ASSERT_EQ(table_info_get.dimension_, TABLE_DIM); + + std::map> search_vectors; +// std::map> vectors_ids_map; + { + engine::IDNumbers vector_ids; + int64_t nb = 1024000; + std::vector xb; + BuildVectors(nb, xb); + engine::Status status = db_->InsertVectors(TABLE_NAME, nb, xb.data(), vector_ids); + ASSERT_TRUE(status.ok()); + +// std::ofstream myfile("mem_test.txt"); +// for (int64_t i = 0; i < nb; ++i) { +// int64_t vector_id = vector_ids[i]; +// std::vector vectors; +// for (int64_t j = 0; j < TABLE_DIM; j++) { +// vectors.emplace_back(xb[i*TABLE_DIM + j]); +//// std::cout << xb[i*TABLE_DIM + j] << std::endl; +// } +// vectors_ids_map[vector_id] = vectors; +// } + + std::this_thread::sleep_for(std::chrono::seconds(3)); + + std::random_device rd; + std::mt19937 gen(rd()); + std::uniform_int_distribution dis(0, nb - 1); + + int64_t numQuery = 1000; + for (int64_t i = 0; i < numQuery; ++i) { + int64_t index = dis(gen); + std::vector search; + for (int64_t j = 0; j < TABLE_DIM; j++) { + search.push_back(xb[index * TABLE_DIM + j]); + } + search_vectors.insert(std::make_pair(vector_ids[index], search)); +// std::cout << "index: " << index << " vector_ids[index]: " << vector_ids[index] << std::endl; + } + +// for (int64_t i = 0; i < nb; i += 100000) { +// std::vector search; +// for (int64_t j = 0; j < TABLE_DIM; j++) { +// search.push_back(xb[i * TABLE_DIM + j]); +// } +// search_vectors.insert(std::make_pair(vector_ids[i], search)); +// } + + } + + int k = 10; + for(auto& pair : search_vectors) { + auto& search = pair.second; + engine::QueryResults results; + stat = db_->Query(TABLE_NAME, k, 1, search.data(), results); + for(int t = 0; t < k; t++) { +// std::cout << "ID=" << results[0][t].first << " DISTANCE=" << results[0][t].second << std::endl; + +// std::cout << vectors_ids_map[results[0][t].first].size() << std::endl; +// for (auto& data : vectors_ids_map[results[0][t].first]) { +// std::cout << data << " "; +// } +// std::cout << std::endl; + } + // std::cout << "results[0][0].first: " << results[0][0].first << " pair.first: " << pair.first << " results[0][0].second: " << results[0][0].second << std::endl; + ASSERT_EQ(results[0][0].first, pair.first); + ASSERT_LT(results[0][0].second, 0.00001); + } + + stat = db_->DropAll(); + ASSERT_TRUE(stat.ok()); + +} + +TEST(MEM_TEST, INSERT_TEST) { + + auto options = engine::OptionsFactory::Build(); + options.meta.path = "/tmp/milvus_test"; + options.meta.backend_uri = "sqlite://:@:/"; + auto db_ = engine::DBFactory::Build(options); + + engine::meta::TableSchema table_info = BuildTableSchema(); + engine::Status stat = db_->CreateTable(table_info); + + engine::meta::TableSchema table_info_get; + table_info_get.table_id_ = TABLE_NAME; + stat = db_->DescribeTable(table_info_get); + ASSERT_STATS(stat); + ASSERT_EQ(table_info_get.dimension_, TABLE_DIM); + + auto start_time = METRICS_NOW_TIME; + + int insert_loop = 1000; + for (int i = 0; i < insert_loop; ++i) { + int64_t nb = 204800; + std::vector xb; + BuildVectors(nb, xb); + engine::IDNumbers vector_ids; + engine::Status status = db_->InsertVectors(TABLE_NAME, nb, xb.data(), vector_ids); + ASSERT_TRUE(status.ok()); + } + auto end_time = METRICS_NOW_TIME; + auto total_time = METRICS_MICROSECONDS(start_time, end_time); + std::cout << "total_time(ms) : " << total_time << std::endl; + + stat = db_->DropAll(); + ASSERT_TRUE(stat.ok()); + +} From e28f3397ef62500bdc9d3fac9adc77476275ba38 Mon Sep 17 00:00:00 2001 From: jinhai Date: Sun, 7 Jul 2019 19:16:39 +0800 Subject: [PATCH 097/189] MS-176 Add create table parameter check Former-commit-id: a7bc3606576f90a9e579bb401f65efb4477c036f --- cpp/src/server/RequestTask.cpp | 60 +++++++++++------ cpp/src/utils/ValidationUtil.cpp | 74 +++++++++++++++++++++ cpp/src/utils/ValidationUtil.h | 20 ++++++ cpp/unittest/CMakeLists.txt | 4 +- cpp/unittest/db/db_tests.cpp | 13 ++-- cpp/unittest/db/mysql_db_test.cpp | 12 ++-- cpp/unittest/db/search_test.cpp | 5 +- cpp/unittest/faiss_wrapper/wrapper_test.cpp | 5 +- cpp/unittest/utils/CMakeLists.txt | 30 +++++++++ cpp/unittest/utils/ValidationUtilTest.cpp | 61 +++++++++++++++++ 10 files changed, 247 insertions(+), 37 deletions(-) create mode 100644 cpp/src/utils/ValidationUtil.cpp create mode 100644 cpp/src/utils/ValidationUtil.h create mode 100644 cpp/unittest/utils/CMakeLists.txt create mode 100644 cpp/unittest/utils/ValidationUtilTest.cpp diff --git a/cpp/src/server/RequestTask.cpp b/cpp/src/server/RequestTask.cpp index f312b7e605..01cf866b36 100644 --- a/cpp/src/server/RequestTask.cpp +++ b/cpp/src/server/RequestTask.cpp @@ -8,6 +8,7 @@ #include "utils/CommonUtil.h" #include "utils/Log.h" #include "utils/TimeRecorder.h" +#include "utils/ValidationUtil.h" #include "DBWrapper.h" #include "version.h" @@ -133,19 +134,23 @@ BaseTaskPtr CreateTableTask::Create(const thrift::TableSchema& schema) { ServerError CreateTableTask::OnExecute() { TimeRecorder rc("CreateTableTask"); - + try { //step 1: check arguments - if(schema_.table_name.empty()) { - return SetError(SERVER_INVALID_TABLE_NAME, "Empty table name"); - } - if(schema_.dimension <= 0) { - return SetError(SERVER_INVALID_TABLE_DIMENSION, "Invalid table dimension: " + std::to_string(schema_.dimension)); + ServerError res = SERVER_SUCCESS; + res = ValidateTableName(schema_.table_name); + if(res != SERVER_SUCCESS) { + return res; } - engine::EngineType engine_type = EngineType(schema_.index_type); - if(engine_type == engine::EngineType::INVALID) { - return SetError(SERVER_INVALID_INDEX_TYPE, "Invalid index type: " + std::to_string(schema_.index_type)); + res = ValidateTableDimension(schema_.dimension); + if(res != SERVER_SUCCESS) { + return res; + } + + res = ValidateTableIndexType(schema_.index_type); + if(res != SERVER_SUCCESS) { + return res; } //step 2: construct table schema @@ -187,8 +192,10 @@ ServerError DescribeTableTask::OnExecute() { try { //step 1: check arguments - if(table_name_.empty()) { - return SetError(SERVER_INVALID_TABLE_NAME, "Empty table name"); + ServerError res = SERVER_SUCCESS; + res = ValidateTableName(table_name_); + if(res != SERVER_SUCCESS) { + return res; } //step 2: get table info @@ -230,10 +237,11 @@ ServerError HasTableTask::OnExecute() { TimeRecorder rc("HasTableTask"); //step 1: check arguments - if(table_name_.empty()) { - return SetError(SERVER_INVALID_TABLE_NAME, "Empty table name"); + ServerError res = SERVER_SUCCESS; + res = ValidateTableName(table_name_); + if(res != SERVER_SUCCESS) { + return res; } - //step 2: check table existence engine::Status stat = DBWrapper::DB()->HasTable(table_name_, has_table_); if(!stat.ok()) { @@ -264,8 +272,10 @@ ServerError DeleteTableTask::OnExecute() { TimeRecorder rc("DeleteTableTask"); //step 1: check arguments - if (table_name_.empty()) { - return SetError(SERVER_INVALID_TABLE_NAME, "Empty table name"); + ServerError res = SERVER_SUCCESS; + res = ValidateTableName(table_name_); + if(res != SERVER_SUCCESS) { + return res; } //step 2: check table existence @@ -346,8 +356,10 @@ ServerError AddVectorTask::OnExecute() { TimeRecorder rc("AddVectorTask"); //step 1: check arguments - if (table_name_.empty()) { - return SetError(SERVER_INVALID_TABLE_NAME, "Empty table name"); + ServerError res = SERVER_SUCCESS; + res = ValidateTableName(table_name_); + if(res != SERVER_SUCCESS) { + return res; } if(record_array_.empty()) { @@ -435,8 +447,10 @@ ServerError SearchVectorTask::OnExecute() { TimeRecorder rc("SearchVectorTask"); //step 1: check arguments - if (table_name_.empty()) { - return SetError(SERVER_INVALID_TABLE_NAME, "Empty table name"); + ServerError res = SERVER_SUCCESS; + res = ValidateTableName(table_name_); + if(res != SERVER_SUCCESS) { + return res; } if(top_k_ <= 0) { @@ -548,8 +562,10 @@ ServerError GetTableRowCountTask::OnExecute() { TimeRecorder rc("GetTableRowCountTask"); //step 1: check arguments - if (table_name_.empty()) { - return SetError(SERVER_INVALID_TABLE_NAME, "Empty table name"); + ServerError res = SERVER_SUCCESS; + res = ValidateTableName(table_name_); + if(res != SERVER_SUCCESS) { + return res; } //step 2: get row count diff --git a/cpp/src/utils/ValidationUtil.cpp b/cpp/src/utils/ValidationUtil.cpp new file mode 100644 index 0000000000..b4bbd3346a --- /dev/null +++ b/cpp/src/utils/ValidationUtil.cpp @@ -0,0 +1,74 @@ +#include +#include "ValidationUtil.h" +#include "Log.h" + + +namespace zilliz { +namespace milvus { +namespace server { + +constexpr size_t table_name_size_limit = 16384; +constexpr int64_t table_dimension_limit = 16384; + +ServerError +ValidateTableName(const std::string &table_name) { + + // Table name shouldn't be empty. + if (table_name.empty()) { + SERVER_LOG_ERROR << "Empty table name"; + return SERVER_INVALID_TABLE_NAME; + } + + // Table name size shouldn't exceed 16384. + if (table_name.size() > table_name_size_limit) { + SERVER_LOG_ERROR << "Table name size exceed the limitation"; + return SERVER_INVALID_TABLE_NAME; + } + + // Table name first character should be underscore or character. + char first_char = table_name[0]; + if (first_char != '_' && std::isalpha(first_char) == 0) { + SERVER_LOG_ERROR << "Table name first character isn't underscore or character: " << first_char; + return SERVER_INVALID_TABLE_NAME; + } + + int64_t table_name_size = table_name.size(); + for (int64_t i = 1; i < table_name_size; ++i) { + char name_char = table_name[i]; + if (name_char != '_' && std::isalnum(name_char) == 0) { + SERVER_LOG_ERROR << "Table name character isn't underscore or alphanumber: " << name_char; + return SERVER_INVALID_TABLE_NAME; + } + } + + return SERVER_SUCCESS; +} + +ServerError +ValidateTableDimension(int64_t dimension) { + if (dimension <= 0 || dimension > table_dimension_limit) { + SERVER_LOG_ERROR << "Table dimension excceed the limitation: " << table_dimension_limit; + return SERVER_INVALID_VECTOR_DIMENSION; + } else { + return SERVER_SUCCESS; + } +} + +ServerError +ValidateTableIndexType(int32_t index_type) { + auto engine_type = engine::EngineType(index_type); + switch (engine_type) { + case engine::EngineType::FAISS_IDMAP: + case engine::EngineType::FAISS_IVFFLAT: { + SERVER_LOG_DEBUG << "Index type: " << index_type; + return SERVER_SUCCESS; + } + default: { + return SERVER_INVALID_INDEX_TYPE; + } + } +} + +} +} +} \ No newline at end of file diff --git a/cpp/src/utils/ValidationUtil.h b/cpp/src/utils/ValidationUtil.h new file mode 100644 index 0000000000..608ac22682 --- /dev/null +++ b/cpp/src/utils/ValidationUtil.h @@ -0,0 +1,20 @@ +#pragma once + +#include "Error.h" + +namespace zilliz { +namespace milvus { +namespace server { + +ServerError +ValidateTableName(const std::string& table_name); + +ServerError +ValidateTableDimension(int64_t dimension); + +ServerError +ValidateTableIndexType(int32_t index_type); + +} +} +} \ No newline at end of file diff --git a/cpp/unittest/CMakeLists.txt b/cpp/unittest/CMakeLists.txt index 38046617ae..62e32f6d1d 100644 --- a/cpp/unittest/CMakeLists.txt +++ b/cpp/unittest/CMakeLists.txt @@ -12,7 +12,6 @@ aux_source_directory(${MILVUS_ENGINE_SRC}/config config_files) set(unittest_srcs ${CMAKE_CURRENT_SOURCE_DIR}/main.cpp) - #${EASYLOGGINGPP_INCLUDE_DIR}/easylogging++.cc) set(require_files ${MILVUS_ENGINE_SRC}/server/ServerConfig.cpp @@ -44,4 +43,5 @@ add_subdirectory(db) add_subdirectory(faiss_wrapper) #add_subdirectory(license) add_subdirectory(metrics) -add_subdirectory(storage) \ No newline at end of file +add_subdirectory(storage) +add_subdirectory(utils) \ No newline at end of file diff --git a/cpp/unittest/db/db_tests.cpp b/cpp/unittest/db/db_tests.cpp index bd17081af8..625211cae7 100644 --- a/cpp/unittest/db/db_tests.cpp +++ b/cpp/unittest/db/db_tests.cpp @@ -3,17 +3,20 @@ // Unauthorized copying of this file, via any medium is strictly prohibited. // Proprietary and confidential. //////////////////////////////////////////////////////////////////////////////// -#include -#include -#include -#include - #include "utils.h" #include "db/DB.h" #include "db/DBImpl.h" #include "db/MetaConsts.h" #include "db/Factories.h" +#include +#include + +#include + +#include +#include + using namespace zilliz::milvus; namespace { diff --git a/cpp/unittest/db/mysql_db_test.cpp b/cpp/unittest/db/mysql_db_test.cpp index a202859c3d..58eea83995 100644 --- a/cpp/unittest/db/mysql_db_test.cpp +++ b/cpp/unittest/db/mysql_db_test.cpp @@ -3,17 +3,19 @@ // Unauthorized copying of this file, via any medium is strictly prohibited. // Proprietary and confidential. //////////////////////////////////////////////////////////////////////////////// -#include -#include -#include -#include - #include "utils.h" #include "db/DB.h" #include "db/DBImpl.h" #include "db/MetaConsts.h" #include "db/Factories.h" +#include +#include +#include + +#include +#include + using namespace zilliz::milvus; namespace { diff --git a/cpp/unittest/db/search_test.cpp b/cpp/unittest/db/search_test.cpp index db10bcbadf..ce99ea78f7 100644 --- a/cpp/unittest/db/search_test.cpp +++ b/cpp/unittest/db/search_test.cpp @@ -3,10 +3,11 @@ // Unauthorized copying of this file, via any medium is strictly prohibited. // Proprietary and confidential. //////////////////////////////////////////////////////////////////////////////// -#include - #include "db/scheduler/task/SearchTask.h" +#include + +#include #include using namespace zilliz::milvus; diff --git a/cpp/unittest/faiss_wrapper/wrapper_test.cpp b/cpp/unittest/faiss_wrapper/wrapper_test.cpp index 67a6c3cde8..6f4a651a55 100644 --- a/cpp/unittest/faiss_wrapper/wrapper_test.cpp +++ b/cpp/unittest/faiss_wrapper/wrapper_test.cpp @@ -4,12 +4,15 @@ // Proprietary and confidential. //////////////////////////////////////////////////////////////////////////////// -#include + #include "wrapper/Operand.h" #include "wrapper/Index.h" #include "wrapper/IndexBuilder.h" +#include +#include + using namespace zilliz::milvus::engine; diff --git a/cpp/unittest/utils/CMakeLists.txt b/cpp/unittest/utils/CMakeLists.txt new file mode 100644 index 0000000000..a46a3b05e1 --- /dev/null +++ b/cpp/unittest/utils/CMakeLists.txt @@ -0,0 +1,30 @@ +#------------------------------------------------------------------------------- +# Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved +# Unauthorized copying of this file, via any medium is strictly prohibited. +# Proprietary and confidential. +#------------------------------------------------------------------------------- + +# Make sure that your call to link_directories takes place before your call to the relevant add_executable. +include_directories("${CUDA_TOOLKIT_ROOT_DIR}/include") +link_directories("${CUDA_TOOLKIT_ROOT_DIR}/lib64") + +set(validation_util_src + ${MILVUS_ENGINE_SRC}/utils/ValidationUtil.cpp + ${MILVUS_ENGINE_SRC}/utils/ValidationUtil.h) + +set(validation_util_test_src + ${unittest_srcs} + ${validation_util_src} + ${require_files} + ValidationUtilTest.cpp + ) + +add_executable(valication_util_test + ${validation_util_test_src} + ${config_files}) + +target_link_libraries(valication_util_test + ${unittest_libs} + boost_filesystem) + +install(TARGETS valication_util_test DESTINATION bin) \ No newline at end of file diff --git a/cpp/unittest/utils/ValidationUtilTest.cpp b/cpp/unittest/utils/ValidationUtilTest.cpp new file mode 100644 index 0000000000..095614e325 --- /dev/null +++ b/cpp/unittest/utils/ValidationUtilTest.cpp @@ -0,0 +1,61 @@ +//////////////////////////////////////////////////////////////////////////////// +// Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved +// Unauthorized copying of this file, via any medium is strictly prohibited. +// Proprietary and confidential. +//////////////////////////////////////////////////////////////////////////////// +#include + +#include "utils/ValidationUtil.h" +#include "utils/Error.h" + +#include + +using namespace zilliz::milvus::server; + +TEST(ValidationUtilTest, TableNameTest) { + std::string table_name = "Normal123_"; + ServerError res = ValidateTableName(table_name); + ASSERT_EQ(res, SERVER_SUCCESS); + + table_name = "12sds"; + res = ValidateTableName(table_name); + ASSERT_EQ(res, SERVER_INVALID_TABLE_NAME); + + table_name = ""; + res = ValidateTableName(table_name); + ASSERT_EQ(res, SERVER_INVALID_TABLE_NAME); + + table_name = "_asdasd"; + res = ValidateTableName(table_name); + ASSERT_EQ(res, SERVER_SUCCESS); + + table_name = "!@#!@"; + res = ValidateTableName(table_name); + ASSERT_EQ(res, SERVER_INVALID_TABLE_NAME); + + table_name = "中文"; + res = ValidateTableName(table_name); + ASSERT_EQ(res, SERVER_INVALID_TABLE_NAME); + + + table_name = std::string('a', 32768); + res = ValidateTableName(table_name); + ASSERT_EQ(res, SERVER_INVALID_TABLE_NAME); +} + + +TEST(ValidationUtilTest, TableDimensionTest) { + ASSERT_EQ(ValidateTableDimension(-1), SERVER_INVALID_VECTOR_DIMENSION); + ASSERT_EQ(ValidateTableDimension(0), SERVER_INVALID_VECTOR_DIMENSION); + ASSERT_EQ(ValidateTableDimension(16385), SERVER_INVALID_VECTOR_DIMENSION); + ASSERT_EQ(ValidateTableDimension(16384), SERVER_SUCCESS); + ASSERT_EQ(ValidateTableDimension(1), SERVER_SUCCESS); +} + +TEST(ValidationUtilTest, TableIndexTypeTest) { + ASSERT_EQ(ValidateTableIndexType(0), SERVER_INVALID_INDEX_TYPE); + ASSERT_EQ(ValidateTableIndexType(1), SERVER_SUCCESS); + ASSERT_EQ(ValidateTableIndexType(2), SERVER_SUCCESS); + ASSERT_EQ(ValidateTableIndexType(3), SERVER_INVALID_INDEX_TYPE); + ASSERT_EQ(ValidateTableIndexType(4), SERVER_INVALID_INDEX_TYPE); +} From b67500c44ba742023503abd5a2163a465a626924 Mon Sep 17 00:00:00 2001 From: jinhai Date: Sun, 7 Jul 2019 19:55:16 +0800 Subject: [PATCH 098/189] MS-176 Update table name length Former-commit-id: 261a0a56335e00d72b92c84dcf2a8c4d233f1820 --- cpp/src/utils/ValidationUtil.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/src/utils/ValidationUtil.cpp b/cpp/src/utils/ValidationUtil.cpp index b4bbd3346a..65cd81e670 100644 --- a/cpp/src/utils/ValidationUtil.cpp +++ b/cpp/src/utils/ValidationUtil.cpp @@ -7,7 +7,7 @@ namespace zilliz { namespace milvus { namespace server { -constexpr size_t table_name_size_limit = 16384; +constexpr size_t table_name_size_limit = 255; constexpr int64_t table_dimension_limit = 16384; ServerError From f7e11e873f741fb4500eced4a22aeff995c65354 Mon Sep 17 00:00:00 2001 From: starlord Date: Sun, 7 Jul 2019 20:41:52 +0800 Subject: [PATCH 099/189] date range check Former-commit-id: 20a461530f625d7aa9c4ba14cfc6fbc09b78bf06 --- cpp/src/server/RequestTask.cpp | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/cpp/src/server/RequestTask.cpp b/cpp/src/server/RequestTask.cpp index 01cf866b36..4c207546b8 100644 --- a/cpp/src/server/RequestTask.cpp +++ b/cpp/src/server/RequestTask.cpp @@ -109,7 +109,13 @@ namespace { } long days = (tt_end > tt_start) ? (tt_end - tt_start)/DAY_SECONDS : (tt_start - tt_end)/DAY_SECONDS; - for(long i = 0; i <= days; i++) { + if(days == 0) { + error_code = SERVER_INVALID_TIME_RANGE; + error_msg = "Invalid time range: " + range.start_value + " to " + range.end_value; + return ; + } + + for(long i = 0; i < days; i++) { time_t tt_day = tt_start + DAY_SECONDS*i; tm tm_day; CommonUtil::ConvertTime(tt_day, tm_day); From 48626e338c01392bb0d39644a3358cafbead3a0e Mon Sep 17 00:00:00 2001 From: zhiru Date: Mon, 8 Jul 2019 11:14:28 +0800 Subject: [PATCH 100/189] Add new mem manager Former-commit-id: 639e28832e5dfb1cdbdbc7a47177520a9ad4f60f --- cpp/CHANGELOG.md | 2 + cpp/conf/server_config.template | 4 +- cpp/src/db/Constants.h | 3 + cpp/src/db/MemManager.cpp | 25 ++++++ cpp/src/db/MemManager.h | 6 ++ cpp/src/db/MemManagerAbstract.h | 6 ++ cpp/src/db/MemTable.cpp | 17 +++- cpp/src/db/MemTable.h | 8 +- cpp/src/db/NewMemManager.cpp | 38 +++++++++ cpp/src/db/NewMemManager.h | 6 ++ cpp/src/db/Options.h | 1 + cpp/src/server/DBWrapper.cpp | 8 ++ cpp/src/server/ServerConfig.h | 1 + cpp/unittest/db/mem_test.cpp | 144 +++++++++++++++++++++----------- cpp/unittest/db/utils.cpp | 12 +++ cpp/unittest/db/utils.h | 5 ++ 16 files changed, 232 insertions(+), 54 deletions(-) diff --git a/cpp/CHANGELOG.md b/cpp/CHANGELOG.md index 1ddd47ef24..8887c2436f 100644 --- a/cpp/CHANGELOG.md +++ b/cpp/CHANGELOG.md @@ -18,6 +18,8 @@ Please mark all change in change log and use the ticket from JIRA. - MS-152 - Delete assert in MySQLMetaImpl and change MySQLConnectionPool impl ## New Feature +- MS-137 - Integrate knowhere +- MS-180 - Add new mem manager ## Task diff --git a/cpp/conf/server_config.template b/cpp/conf/server_config.template index 260ede4f2d..24dd3e1363 100644 --- a/cpp/conf/server_config.template +++ b/cpp/conf/server_config.template @@ -2,7 +2,7 @@ server_config: address: 0.0.0.0 port: 19530 # the port milvus listen to, default: 19530, range: 1025 ~ 65534 gpu_index: 0 # the gpu milvus use, default: 0, range: 0 ~ gpu number - 1 - mode: single # milvus deployment type: single, cluster + mode: single # milvus deployment type: single, cluster, read_only db_config: db_path: @MILVUS_DB_PATH@ # milvus data storage path @@ -15,6 +15,8 @@ db_config: index_building_threshold: 1024 # index building trigger threshold, default: 1024, unit: MB archive_disk_threshold: 512 # triger archive action if storage size exceed this value, unit: GB archive_days_threshold: 30 # files older than x days will be archived, unit: day + maximum_memory: 4 # maximum memory allowed, default: 4, unit: GB, should be at least 1 GB. + # the sum of maximum_memory and cpu_cache_capacity should be less than total memory metric_config: is_startup: off # if monitoring start: on, off diff --git a/cpp/src/db/Constants.h b/cpp/src/db/Constants.h index 2bb2e0a064..1ba02b1d55 100644 --- a/cpp/src/db/Constants.h +++ b/cpp/src/db/Constants.h @@ -11,6 +11,9 @@ namespace engine { const size_t K = 1024UL; const size_t M = K*K; +const size_t G = K*M; +const size_t T = K*G; + const size_t MAX_TABLE_FILE_MEM = 128 * M; const int VECTOR_TYPE_SIZE = sizeof(float); diff --git a/cpp/src/db/MemManager.cpp b/cpp/src/db/MemManager.cpp index e36b0c45ba..ba8517cdbd 100644 --- a/cpp/src/db/MemManager.cpp +++ b/cpp/src/db/MemManager.cpp @@ -8,6 +8,7 @@ #include "MetaConsts.h" #include "EngineFactory.h" #include "metrics/Metrics.h" +#include "Log.h" #include #include @@ -128,6 +129,10 @@ Status MemManager::InsertVectorsNoLock(const std::string& table_id, size_t n, const float* vectors, IDNumbers& vector_ids) { + + LOG(DEBUG) << "MemManager::InsertVectorsNoLock: mutable mem = " << GetCurrentMutableMem() << + ", immutable mem = " << GetCurrentImmutableMem() << ", total mem = " << GetCurrentMem(); + MemVectorsPtr mem = GetMemByTable(table_id); if (mem == nullptr) { return Status::NotFound("Group " + table_id + " not found!"); @@ -192,6 +197,26 @@ Status MemManager::EraseMemVector(const std::string& table_id) { return Status::OK(); } +size_t MemManager::GetCurrentMutableMem() { + size_t totalMem = 0; + for (auto& kv : mem_id_map_) { + auto memVector = kv.second; + totalMem += memVector->Size(); + } + return totalMem; +} + +size_t MemManager::GetCurrentImmutableMem() { + size_t totalMem = 0; + for (auto& memVector : immu_mem_list_) { + totalMem += memVector->Size(); + } + return totalMem; +} + +size_t MemManager::GetCurrentMem() { + return GetCurrentMutableMem() + GetCurrentImmutableMem(); +} } // namespace engine } // namespace milvus diff --git a/cpp/src/db/MemManager.h b/cpp/src/db/MemManager.h index 95303889db..e8460c7a6d 100644 --- a/cpp/src/db/MemManager.h +++ b/cpp/src/db/MemManager.h @@ -78,6 +78,12 @@ public: Status EraseMemVector(const std::string& table_id) override; + size_t GetCurrentMutableMem() override; + + size_t GetCurrentImmutableMem() override; + + size_t GetCurrentMem() override; + private: MemVectorsPtr GetMemByTable(const std::string& table_id); diff --git a/cpp/src/db/MemManagerAbstract.h b/cpp/src/db/MemManagerAbstract.h index 74222df1e8..58c73ba6f8 100644 --- a/cpp/src/db/MemManagerAbstract.h +++ b/cpp/src/db/MemManagerAbstract.h @@ -16,6 +16,12 @@ public: virtual Status EraseMemVector(const std::string& table_id) = 0; + virtual size_t GetCurrentMutableMem() = 0; + + virtual size_t GetCurrentImmutableMem() = 0; + + virtual size_t GetCurrentMem() = 0; + }; // MemManagerAbstract using MemManagerAbstractPtr = std::shared_ptr; diff --git a/cpp/src/db/MemTable.cpp b/cpp/src/db/MemTable.cpp index b282ad375a..ba3875fbb5 100644 --- a/cpp/src/db/MemTable.cpp +++ b/cpp/src/db/MemTable.cpp @@ -49,13 +49,15 @@ size_t MemTable::GetTableFileCount() { } Status MemTable::Serialize() { - for (auto& memTableFile : mem_table_file_list_) { - auto status = memTableFile->Serialize(); + for (auto memTableFile = mem_table_file_list_.begin(); memTableFile != mem_table_file_list_.end(); ) { + auto status = (*memTableFile)->Serialize(); if (!status.ok()) { std::string errMsg = "MemTable::Serialize failed: " + status.ToString(); ENGINE_LOG_ERROR << errMsg; return Status::Error(errMsg); } + std::lock_guard lock(mutex_); + memTableFile = mem_table_file_list_.erase(memTableFile); } return Status::OK(); } @@ -64,10 +66,19 @@ bool MemTable::Empty() { return mem_table_file_list_.empty(); } -std::string MemTable::GetTableId() { +const std::string& MemTable::GetTableId() const { return table_id_; } +size_t MemTable::GetCurrentMem() { + std::lock_guard lock(mutex_); + size_t totalMem = 0; + for (auto& memTableFile : mem_table_file_list_) { + totalMem += memTableFile->GetCurrentMem(); + } + return totalMem; +} + } // namespace engine } // namespace milvus } // namespace zilliz \ No newline at end of file diff --git a/cpp/src/db/MemTable.h b/cpp/src/db/MemTable.h index e09d6ddac1..9bae932e62 100644 --- a/cpp/src/db/MemTable.h +++ b/cpp/src/db/MemTable.h @@ -4,7 +4,7 @@ #include "MemTableFile.h" #include "VectorSource.h" -#include +#include namespace zilliz { namespace milvus { @@ -30,7 +30,9 @@ public: bool Empty(); - std::string GetTableId(); + const std::string& GetTableId() const; + + size_t GetCurrentMem(); private: const std::string table_id_; @@ -41,6 +43,8 @@ private: Options options_; + std::mutex mutex_; + }; //MemTable } // namespace engine diff --git a/cpp/src/db/NewMemManager.cpp b/cpp/src/db/NewMemManager.cpp index 19aba68eb7..3c78f37101 100644 --- a/cpp/src/db/NewMemManager.cpp +++ b/cpp/src/db/NewMemManager.cpp @@ -1,5 +1,9 @@ #include "NewMemManager.h" #include "VectorSource.h" +#include "Log.h" +#include "Constants.h" + +#include namespace zilliz { namespace milvus { @@ -20,6 +24,9 @@ Status NewMemManager::InsertVectors(const std::string& table_id_, const float* vectors_, IDNumbers& vector_ids_) { + while (GetCurrentMem() > options_.maximum_memory) { + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + } std::unique_lock lock(mutex_); @@ -30,6 +37,10 @@ Status NewMemManager::InsertVectorsNoLock(const std::string& table_id, size_t n, const float* vectors, IDNumbers& vector_ids) { + + LOG(DEBUG) << "NewMemManager::InsertVectorsNoLock: mutable mem = " << GetCurrentMutableMem() << + ", immutable mem = " << GetCurrentImmutableMem() << ", total mem = " << GetCurrentMem(); + MemTablePtr mem = GetMemByTable(table_id); VectorSource::Ptr source = std::make_shared(n, vectors); @@ -64,6 +75,12 @@ Status NewMemManager::Serialize(std::set& table_ids) { table_ids.insert(mem->GetTableId()); } immu_mem_list_.clear(); +// for (auto mem = immu_mem_list_.begin(); mem != immu_mem_list_.end(); ) { +// (*mem)->Serialize(); +// table_ids.insert((*mem)->GetTableId()); +// mem = immu_mem_list_.erase(mem); +// LOG(DEBUG) << "immu_mem_list_ size = " << immu_mem_list_.size(); +// } return Status::OK(); } @@ -87,6 +104,27 @@ Status NewMemManager::EraseMemVector(const std::string& table_id) { return Status::OK(); } +size_t NewMemManager::GetCurrentMutableMem() { + size_t totalMem = 0; + for (auto& kv : mem_id_map_) { + auto memTable = kv.second; + totalMem += memTable->GetCurrentMem(); + } + return totalMem; +} + +size_t NewMemManager::GetCurrentImmutableMem() { + size_t totalMem = 0; + for (auto& memTable : immu_mem_list_) { + totalMem += memTable->GetCurrentMem(); + } + return totalMem; +} + +size_t NewMemManager::GetCurrentMem() { + return GetCurrentMutableMem() + GetCurrentImmutableMem(); +} + } // namespace engine } // namespace milvus } // namespace zilliz \ No newline at end of file diff --git a/cpp/src/db/NewMemManager.h b/cpp/src/db/NewMemManager.h index a5f5a9ca13..9883480404 100644 --- a/cpp/src/db/NewMemManager.h +++ b/cpp/src/db/NewMemManager.h @@ -31,6 +31,12 @@ public: Status EraseMemVector(const std::string& table_id) override; + size_t GetCurrentMutableMem() override; + + size_t GetCurrentImmutableMem() override; + + size_t GetCurrentMem() override; + private: MemTablePtr GetMemByTable(const std::string& table_id); diff --git a/cpp/src/db/Options.h b/cpp/src/db/Options.h index 39d0a15019..47bbb45bbc 100644 --- a/cpp/src/db/Options.h +++ b/cpp/src/db/Options.h @@ -61,6 +61,7 @@ struct Options { size_t index_trigger_size = ONE_GB; //unit: byte DBMetaOptions meta; int mode = MODE::SINGLE; + float maximum_memory = 4 * ONE_GB; }; // Options diff --git a/cpp/src/server/DBWrapper.cpp b/cpp/src/server/DBWrapper.cpp index fca15cb65a..bed4440d5e 100644 --- a/cpp/src/server/DBWrapper.cpp +++ b/cpp/src/server/DBWrapper.cpp @@ -23,6 +23,14 @@ DBWrapper::DBWrapper() { if(index_size > 0) {//ensure larger than zero, unit is MB opt.index_trigger_size = (size_t)index_size * engine::ONE_MB; } + float maximum_memory = config.GetFloatValue(CONFIG_MAXMIMUM_MEMORY); + if (maximum_memory > 1.0) { + opt.maximum_memory = maximum_memory * engine::ONE_GB; + } + else { + std::cout << "ERROR: maximum_memory should be at least 1 GB" << std::endl; + kill(0, SIGUSR1); + } ConfigNode& serverConfig = ServerConfig::GetInstance().GetConfig(CONFIG_SERVER); std::string mode = serverConfig.GetValue(CONFIG_CLUSTER_MODE, "single"); diff --git a/cpp/src/server/ServerConfig.h b/cpp/src/server/ServerConfig.h index 0ec04eed8c..b3b95eb8b6 100644 --- a/cpp/src/server/ServerConfig.h +++ b/cpp/src/server/ServerConfig.h @@ -26,6 +26,7 @@ static const std::string CONFIG_DB_PATH = "db_path"; static const std::string CONFIG_DB_INDEX_TRIGGER_SIZE = "index_building_threshold"; static const std::string CONFIG_DB_ARCHIVE_DISK = "archive_disk_threshold"; static const std::string CONFIG_DB_ARCHIVE_DAYS = "archive_days_threshold"; +static const std::string CONFIG_MAXMIMUM_MEMORY = "maximum_memory"; static const std::string CONFIG_LOG = "log_config"; diff --git a/cpp/unittest/db/mem_test.cpp b/cpp/unittest/db/mem_test.cpp index 915610adcc..818c3a6388 100644 --- a/cpp/unittest/db/mem_test.cpp +++ b/cpp/unittest/db/mem_test.cpp @@ -8,6 +8,8 @@ #include "db/Constants.h" #include "db/EngineFactory.h" #include "metrics/Metrics.h" +#include "db/MetaConsts.h" +#include "boost/filesystem.hpp" #include #include @@ -34,9 +36,6 @@ namespace { vectors.clear(); vectors.resize(n*TABLE_DIM); float* data = vectors.data(); -// std::random_device rd; -// std::mt19937 gen(rd()); -// std::uniform_real_distribution<> dis(0.0, 1.0); for(int i = 0; i < n; i++) { for(int j = 0; j < TABLE_DIM; j++) data[TABLE_DIM * i + j] = drand48(); data[TABLE_DIM * i] += i / 2000.; @@ -44,7 +43,7 @@ namespace { } } -TEST(MEM_TEST, VECTOR_SOURCE_TEST) { +TEST_F(NewMemManagerTest, VECTOR_SOURCE_TEST) { std::shared_ptr impl_ = engine::DBMetaImplFactory::Build(); @@ -91,7 +90,7 @@ TEST(MEM_TEST, VECTOR_SOURCE_TEST) { ASSERT_TRUE(status.ok()); } -TEST(MEM_TEST, MEM_TABLE_FILE_TEST) { +TEST_F(NewMemManagerTest, MEM_TABLE_FILE_TEST) { std::shared_ptr impl_ = engine::DBMetaImplFactory::Build(); auto options = engine::OptionsFactory::Build(); @@ -135,7 +134,7 @@ TEST(MEM_TEST, MEM_TABLE_FILE_TEST) { ASSERT_TRUE(status.ok()); } -TEST(MEM_TEST, MEM_TABLE_TEST) { +TEST_F(NewMemManagerTest, MEM_TABLE_TEST) { std::shared_ptr impl_ = engine::DBMetaImplFactory::Build(); auto options = engine::OptionsFactory::Build(); @@ -201,7 +200,7 @@ TEST(MEM_TEST, MEM_TABLE_TEST) { ASSERT_TRUE(status.ok()); } -TEST(MEM_TEST, MEM_MANAGER_TEST) { +TEST_F(NewMemManagerTest, SERIAL_INSERT_SEARCH_TEST) { auto options = engine::OptionsFactory::Build(); options.meta.path = "/tmp/milvus_test"; @@ -218,7 +217,6 @@ TEST(MEM_TEST, MEM_MANAGER_TEST) { ASSERT_EQ(table_info_get.dimension_, TABLE_DIM); std::map> search_vectors; -// std::map> vectors_ids_map; { engine::IDNumbers vector_ids; int64_t nb = 1024000; @@ -227,24 +225,13 @@ TEST(MEM_TEST, MEM_MANAGER_TEST) { engine::Status status = db_->InsertVectors(TABLE_NAME, nb, xb.data(), vector_ids); ASSERT_TRUE(status.ok()); -// std::ofstream myfile("mem_test.txt"); -// for (int64_t i = 0; i < nb; ++i) { -// int64_t vector_id = vector_ids[i]; -// std::vector vectors; -// for (int64_t j = 0; j < TABLE_DIM; j++) { -// vectors.emplace_back(xb[i*TABLE_DIM + j]); -//// std::cout << xb[i*TABLE_DIM + j] << std::endl; -// } -// vectors_ids_map[vector_id] = vectors; -// } - std::this_thread::sleep_for(std::chrono::seconds(3)); std::random_device rd; std::mt19937 gen(rd()); std::uniform_int_distribution dis(0, nb - 1); - int64_t numQuery = 1000; + int64_t numQuery = 20; for (int64_t i = 0; i < numQuery; ++i) { int64_t index = dis(gen); std::vector search; @@ -252,17 +239,7 @@ TEST(MEM_TEST, MEM_MANAGER_TEST) { search.push_back(xb[index * TABLE_DIM + j]); } search_vectors.insert(std::make_pair(vector_ids[index], search)); -// std::cout << "index: " << index << " vector_ids[index]: " << vector_ids[index] << std::endl; } - -// for (int64_t i = 0; i < nb; i += 100000) { -// std::vector search; -// for (int64_t j = 0; j < TABLE_DIM; j++) { -// search.push_back(xb[i * TABLE_DIM + j]); -// } -// search_vectors.insert(std::make_pair(vector_ids[i], search)); -// } - } int k = 10; @@ -270,26 +247,16 @@ TEST(MEM_TEST, MEM_MANAGER_TEST) { auto& search = pair.second; engine::QueryResults results; stat = db_->Query(TABLE_NAME, k, 1, search.data(), results); - for(int t = 0; t < k; t++) { -// std::cout << "ID=" << results[0][t].first << " DISTANCE=" << results[0][t].second << std::endl; - -// std::cout << vectors_ids_map[results[0][t].first].size() << std::endl; -// for (auto& data : vectors_ids_map[results[0][t].first]) { -// std::cout << data << " "; -// } -// std::cout << std::endl; - } - // std::cout << "results[0][0].first: " << results[0][0].first << " pair.first: " << pair.first << " results[0][0].second: " << results[0][0].second << std::endl; ASSERT_EQ(results[0][0].first, pair.first); ASSERT_LT(results[0][0].second, 0.00001); } - stat = db_->DropAll(); - ASSERT_TRUE(stat.ok()); + delete db_; + boost::filesystem::remove_all(options.meta.path); } -TEST(MEM_TEST, INSERT_TEST) { +TEST_F(NewMemManagerTest, INSERT_TEST) { auto options = engine::OptionsFactory::Build(); options.meta.path = "/tmp/milvus_test"; @@ -307,9 +274,9 @@ TEST(MEM_TEST, INSERT_TEST) { auto start_time = METRICS_NOW_TIME; - int insert_loop = 1000; + int insert_loop = 20; for (int i = 0; i < insert_loop; ++i) { - int64_t nb = 204800; + int64_t nb = 409600; std::vector xb; BuildVectors(nb, xb); engine::IDNumbers vector_ids; @@ -318,10 +285,91 @@ TEST(MEM_TEST, INSERT_TEST) { } auto end_time = METRICS_NOW_TIME; auto total_time = METRICS_MICROSECONDS(start_time, end_time); - std::cout << "total_time(ms) : " << total_time << std::endl; + LOG(DEBUG) << "total_time spent in INSERT_TEST (ms) : " << total_time; - stat = db_->DropAll(); - ASSERT_TRUE(stat.ok()); + delete db_; + boost::filesystem::remove_all(options.meta.path); } +TEST_F(NewMemManagerTest, CONCURRENT_INSERT_SEARCH_TEST) { + + auto options = engine::OptionsFactory::Build(); + options.meta.path = "/tmp/milvus_test"; + options.meta.backend_uri = "sqlite://:@:/"; + auto db_ = engine::DBFactory::Build(options); + + engine::meta::TableSchema table_info = BuildTableSchema(); + engine::Status stat = db_->CreateTable(table_info); + + engine::meta::TableSchema table_info_get; + table_info_get.table_id_ = TABLE_NAME; + stat = db_->DescribeTable(table_info_get); + ASSERT_STATS(stat); + ASSERT_EQ(table_info_get.dimension_, TABLE_DIM); + + engine::IDNumbers vector_ids; + engine::IDNumbers target_ids; + + int64_t nb = 409600; + std::vector xb; + BuildVectors(nb, xb); + + int64_t qb = 5; + std::vector qxb; + BuildVectors(qb, qxb); + + std::thread search([&]() { + engine::QueryResults results; + int k = 10; + std::this_thread::sleep_for(std::chrono::seconds(2)); + + INIT_TIMER; + std::stringstream ss; + uint64_t count = 0; + uint64_t prev_count = 0; + + for (auto j=0; j<10; ++j) { + ss.str(""); + db_->Size(count); + prev_count = count; + + START_TIMER; + stat = db_->Query(TABLE_NAME, k, qb, qxb.data(), results); + ss << "Search " << j << " With Size " << count/engine::meta::M << " M"; + STOP_TIMER(ss.str()); + + ASSERT_STATS(stat); + for (auto k=0; k= prev_count); + std::this_thread::sleep_for(std::chrono::seconds(1)); + } + }); + + int loop = 20; + + for (auto i=0; iInsertVectors(TABLE_NAME, qb, qxb.data(), target_ids); + ASSERT_EQ(target_ids.size(), qb); + } else { + db_->InsertVectors(TABLE_NAME, nb, xb.data(), vector_ids); + } + std::this_thread::sleep_for(std::chrono::microseconds(1)); + } + + search.join(); + + delete db_; + boost::filesystem::remove_all(options.meta.path); + +}; + diff --git a/cpp/unittest/db/utils.cpp b/cpp/unittest/db/utils.cpp index 70c0712549..ae05c59d3b 100644 --- a/cpp/unittest/db/utils.cpp +++ b/cpp/unittest/db/utils.cpp @@ -106,6 +106,18 @@ zilliz::milvus::engine::Options MySQLDBTest::GetOptions() { return options; } +void NewMemManagerTest::InitLog() { + el::Configurations defaultConf; + defaultConf.setToDefault(); + defaultConf.set(el::Level::Debug, + el::ConfigurationType::Format, "[%thread-%datetime-%level]: %msg (%fbase:%line)"); + el::Loggers::reconfigureLogger("default", defaultConf); +} + +void NewMemManagerTest::SetUp() { + InitLog(); +} + int main(int argc, char **argv) { ::testing::InitGoogleTest(&argc, argv); if (argc > 1) { diff --git a/cpp/unittest/db/utils.h b/cpp/unittest/db/utils.h index 361c24b4be..d06500de5c 100644 --- a/cpp/unittest/db/utils.h +++ b/cpp/unittest/db/utils.h @@ -87,3 +87,8 @@ class MySQLDBTest : public ::testing::Test { protected: zilliz::milvus::engine::Options GetOptions(); }; + +class NewMemManagerTest : public ::testing::Test { + void InitLog(); + virtual void SetUp() override; +}; From 65419a157f29c59201b4eb447afa07a023abee47 Mon Sep 17 00:00:00 2001 From: zhiru Date: Mon, 8 Jul 2019 15:07:03 +0800 Subject: [PATCH 101/189] update Former-commit-id: 966c56781878a740a8b31f7aeea99ed4dd562de7 --- cpp/src/db/Constants.h | 6 +- cpp/src/db/Factories.cpp | 21 ++++--- cpp/src/db/Factories.h | 9 +-- cpp/src/db/MemManager.cpp | 83 ++++++++++++++------------ cpp/src/db/MemManager.h | 43 +++++++------- cpp/src/db/MemManagerAbstract.h | 11 ++-- cpp/src/db/MemTable.cpp | 66 +++++++++++---------- cpp/src/db/MemTable.h | 13 ++-- cpp/src/db/MemTableFile.cpp | 56 +++++++++--------- cpp/src/db/MemTableFile.h | 9 +-- cpp/src/db/NewMemManager.cpp | 63 ++++++++++---------- cpp/src/db/NewMemManager.h | 23 ++++---- cpp/src/db/VectorSource.cpp | 18 +++--- cpp/src/db/VectorSource.h | 19 +++--- cpp/unittest/db/mem_test.cpp | 101 ++++++++++++++++---------------- cpp/unittest/db/utils.h | 20 +++---- 16 files changed, 285 insertions(+), 276 deletions(-) diff --git a/cpp/src/db/Constants.h b/cpp/src/db/Constants.h index 1ba02b1d55..055b10ca9a 100644 --- a/cpp/src/db/Constants.h +++ b/cpp/src/db/Constants.h @@ -10,9 +10,9 @@ namespace milvus { namespace engine { const size_t K = 1024UL; -const size_t M = K*K; -const size_t G = K*M; -const size_t T = K*G; +const size_t M = K * K; +const size_t G = K * M; +const size_t T = K * G; const size_t MAX_TABLE_FILE_MEM = 128 * M; diff --git a/cpp/src/db/Factories.cpp b/cpp/src/db/Factories.cpp index abcc0821ab..65c7484a50 100644 --- a/cpp/src/db/Factories.cpp +++ b/cpp/src/db/Factories.cpp @@ -22,6 +22,8 @@ namespace zilliz { namespace milvus { namespace engine { +#define USE_NEW_MEM_MANAGER 1 + DBMetaOptions DBMetaOptionsFactory::Build(const std::string& path) { auto p = path; if(p == "") { @@ -74,17 +76,14 @@ std::shared_ptr DBMetaImplFactory::Build(const DBMetaOptions& metaOp if (dialect.find("mysql") != std::string::npos) { ENGINE_LOG_INFO << "Using MySQL"; return std::make_shared(meta::MySQLMetaImpl(metaOptions, mode)); - } - else if (dialect.find("sqlite") != std::string::npos) { - ENGINE_LOG_DEBUG << "Using SQLite"; + } else if (dialect.find("sqlite") != std::string::npos) { + ENGINE_LOG_INFO << "Using SQLite"; return std::make_shared(meta::DBMetaImpl(metaOptions)); - } - else { + } else { ENGINE_LOG_ERROR << "Invalid dialect in URI: dialect = " << dialect; throw InvalidArgumentException("URI dialect is not mysql / sqlite"); } - } - else { + } else { ENGINE_LOG_ERROR << "Wrong URI format: URI = " << uri; throw InvalidArgumentException("Wrong URI format "); } @@ -102,11 +101,11 @@ DB* DBFactory::Build(const Options& options) { MemManagerAbstractPtr MemManagerFactory::Build(const std::shared_ptr& meta, const Options& options) { - bool useNew = true; - if (useNew) { - return std::make_shared(meta, options); - } +#ifdef USE_NEW_MEM_MANAGER + return std::make_shared(meta, options); +#else return std::make_shared(meta, options); +#endif } } // namespace engine diff --git a/cpp/src/db/Factories.h b/cpp/src/db/Factories.h index 567bc0a8bc..8b6e7b100f 100644 --- a/cpp/src/db/Factories.h +++ b/cpp/src/db/Factories.h @@ -15,12 +15,13 @@ #include #include + namespace zilliz { namespace milvus { namespace engine { struct DBMetaOptionsFactory { - static DBMetaOptions Build(const std::string& path = ""); + static DBMetaOptions Build(const std::string &path = ""); }; struct OptionsFactory { @@ -29,16 +30,16 @@ struct OptionsFactory { struct DBMetaImplFactory { static std::shared_ptr Build(); - static std::shared_ptr Build(const DBMetaOptions& metaOptions, const int& mode); + static std::shared_ptr Build(const DBMetaOptions &metaOptions, const int &mode); }; struct DBFactory { static std::shared_ptr Build(); - static DB* Build(const Options&); + static DB *Build(const Options &); }; struct MemManagerFactory { - static MemManagerAbstractPtr Build(const std::shared_ptr& meta, const Options& options); + static MemManagerAbstractPtr Build(const std::shared_ptr &meta, const Options &options); }; } // namespace engine diff --git a/cpp/src/db/MemManager.cpp b/cpp/src/db/MemManager.cpp index ba8517cdbd..dbf0703173 100644 --- a/cpp/src/db/MemManager.cpp +++ b/cpp/src/db/MemManager.cpp @@ -15,22 +15,23 @@ #include #include + namespace zilliz { namespace milvus { namespace engine { -MemVectors::MemVectors(const std::shared_ptr& meta_ptr, - const meta::TableFileSchema& schema, const Options& options) - : meta_(meta_ptr), - options_(options), - schema_(schema), - id_generator_(new SimpleIDGenerator()), - active_engine_(EngineFactory::Build(schema_.dimension_, schema_.location_, (EngineType)schema_.engine_type_)) { +MemVectors::MemVectors(const std::shared_ptr &meta_ptr, + const meta::TableFileSchema &schema, const Options &options) + : meta_(meta_ptr), + options_(options), + schema_(schema), + id_generator_(new SimpleIDGenerator()), + active_engine_(EngineFactory::Build(schema_.dimension_, schema_.location_, (EngineType) schema_.engine_type_)) { } -Status MemVectors::Add(size_t n_, const float* vectors_, IDNumbers& vector_ids_) { - if(active_engine_ == nullptr) { +Status MemVectors::Add(size_t n_, const float *vectors_, IDNumbers &vector_ids_) { + if (active_engine_ == nullptr) { return Status::Error("index engine is null"); } @@ -39,13 +40,15 @@ Status MemVectors::Add(size_t n_, const float* vectors_, IDNumbers& vector_ids_) Status status = active_engine_->AddWithIds(n_, vectors_, vector_ids_.data()); auto end_time = METRICS_NOW_TIME; auto total_time = METRICS_MICROSECONDS(start_time, end_time); - server::Metrics::GetInstance().AddVectorsPerSecondGaugeSet(static_cast(n_), static_cast(schema_.dimension_), total_time); + server::Metrics::GetInstance().AddVectorsPerSecondGaugeSet(static_cast(n_), + static_cast(schema_.dimension_), + total_time); return status; } size_t MemVectors::RowCount() const { - if(active_engine_ == nullptr) { + if (active_engine_ == nullptr) { return 0; } @@ -53,15 +56,15 @@ size_t MemVectors::RowCount() const { } size_t MemVectors::Size() const { - if(active_engine_ == nullptr) { + if (active_engine_ == nullptr) { return 0; } return active_engine_->Size(); } -Status MemVectors::Serialize(std::string& table_id) { - if(active_engine_ == nullptr) { +Status MemVectors::Serialize(std::string &table_id) { + if (active_engine_ == nullptr) { return Status::Error("index engine is null"); } @@ -73,15 +76,16 @@ Status MemVectors::Serialize(std::string& table_id) { auto total_time = METRICS_MICROSECONDS(start_time, end_time); schema_.size_ = size; - server::Metrics::GetInstance().DiskStoreIOSpeedGaugeSet(size/total_time); + server::Metrics::GetInstance().DiskStoreIOSpeedGaugeSet(size / total_time); schema_.file_type_ = (size >= options_.index_trigger_size) ? - meta::TableFileSchema::TO_INDEX : meta::TableFileSchema::RAW; + meta::TableFileSchema::TO_INDEX : meta::TableFileSchema::RAW; auto status = meta_->UpdateTableFile(schema_); LOG(DEBUG) << "New " << ((schema_.file_type_ == meta::TableFileSchema::RAW) ? "raw" : "to_index") - << " file " << schema_.file_id_ << " of size " << (double)(active_engine_->Size()) / (double)meta::M << " M"; + << " file " << schema_.file_id_ << " of size " << (double) (active_engine_->Size()) / (double) meta::M + << " M"; active_engine_->Cache(); @@ -99,7 +103,7 @@ MemVectors::~MemVectors() { * MemManager */ MemManager::MemVectorsPtr MemManager::GetMemByTable( - const std::string& table_id) { + const std::string &table_id) { auto memIt = mem_id_map_.find(table_id); if (memIt != mem_id_map_.end()) { return memIt->second; @@ -116,22 +120,23 @@ MemManager::MemVectorsPtr MemManager::GetMemByTable( return mem_id_map_[table_id]; } -Status MemManager::InsertVectors(const std::string& table_id_, - size_t n_, - const float* vectors_, - IDNumbers& vector_ids_) { +Status MemManager::InsertVectors(const std::string &table_id_, + size_t n_, + const float *vectors_, + IDNumbers &vector_ids_) { + + LOG(DEBUG) << "MemManager::InsertVectors: mutable mem = " << GetCurrentMutableMem() << + ", immutable mem = " << GetCurrentImmutableMem() << ", total mem = " << GetCurrentMem(); + std::unique_lock lock(mutex_); return InsertVectorsNoLock(table_id_, n_, vectors_, vector_ids_); } -Status MemManager::InsertVectorsNoLock(const std::string& table_id, - size_t n, - const float* vectors, - IDNumbers& vector_ids) { - - LOG(DEBUG) << "MemManager::InsertVectorsNoLock: mutable mem = " << GetCurrentMutableMem() << - ", immutable mem = " << GetCurrentImmutableMem() << ", total mem = " << GetCurrentMem(); +Status MemManager::InsertVectorsNoLock(const std::string &table_id, + size_t n, + const float *vectors, + IDNumbers &vector_ids) { MemVectorsPtr mem = GetMemByTable(table_id); if (mem == nullptr) { @@ -139,7 +144,7 @@ Status MemManager::InsertVectorsNoLock(const std::string& table_id, } //makesure each file size less than index_trigger_size - if(mem->Size() > options_.index_trigger_size) { + if (mem->Size() > options_.index_trigger_size) { std::unique_lock lock(serialization_mtx_); immu_mem_list_.push_back(mem); mem_id_map_.erase(table_id); @@ -152,8 +157,8 @@ Status MemManager::InsertVectorsNoLock(const std::string& table_id, Status MemManager::ToImmutable() { std::unique_lock lock(mutex_); MemIdMap temp_map; - for (auto& kv: mem_id_map_) { - if(kv.second->RowCount() == 0) { + for (auto &kv: mem_id_map_) { + if (kv.second->RowCount() == 0) { temp_map.insert(kv); continue;//empty vector, no need to serialize } @@ -164,12 +169,12 @@ Status MemManager::ToImmutable() { return Status::OK(); } -Status MemManager::Serialize(std::set& table_ids) { +Status MemManager::Serialize(std::set &table_ids) { ToImmutable(); std::unique_lock lock(serialization_mtx_); std::string table_id; table_ids.clear(); - for (auto& mem : immu_mem_list_) { + for (auto &mem : immu_mem_list_) { mem->Serialize(table_id); table_ids.insert(table_id); } @@ -177,7 +182,7 @@ Status MemManager::Serialize(std::set& table_ids) { return Status::OK(); } -Status MemManager::EraseMemVector(const std::string& table_id) { +Status MemManager::EraseMemVector(const std::string &table_id) { {//erase MemVector from rapid-insert cache std::unique_lock lock(mutex_); mem_id_map_.erase(table_id); @@ -186,8 +191,8 @@ Status MemManager::EraseMemVector(const std::string& table_id) { {//erase MemVector from serialize cache std::unique_lock lock(serialization_mtx_); MemList temp_list; - for (auto& mem : immu_mem_list_) { - if(mem->TableId() != table_id) { + for (auto &mem : immu_mem_list_) { + if (mem->TableId() != table_id) { temp_list.push_back(mem); } } @@ -199,7 +204,7 @@ Status MemManager::EraseMemVector(const std::string& table_id) { size_t MemManager::GetCurrentMutableMem() { size_t totalMem = 0; - for (auto& kv : mem_id_map_) { + for (auto &kv : mem_id_map_) { auto memVector = kv.second; totalMem += memVector->Size(); } @@ -208,7 +213,7 @@ size_t MemManager::GetCurrentMutableMem() { size_t MemManager::GetCurrentImmutableMem() { size_t totalMem = 0; - for (auto& memVector : immu_mem_list_) { + for (auto &memVector : immu_mem_list_) { totalMem += memVector->Size(); } return totalMem; diff --git a/cpp/src/db/MemManager.h b/cpp/src/db/MemManager.h index e8460c7a6d..5ad3d08b63 100644 --- a/cpp/src/db/MemManager.h +++ b/cpp/src/db/MemManager.h @@ -17,45 +17,46 @@ #include #include + namespace zilliz { namespace milvus { namespace engine { namespace meta { - class Meta; +class Meta; } class MemVectors { -public: + public: using MetaPtr = meta::Meta::Ptr; using Ptr = std::shared_ptr; - explicit MemVectors(const std::shared_ptr&, - const meta::TableFileSchema&, const Options&); + explicit MemVectors(const std::shared_ptr &, + const meta::TableFileSchema &, const Options &); - Status Add(size_t n_, const float* vectors_, IDNumbers& vector_ids_); + Status Add(size_t n_, const float *vectors_, IDNumbers &vector_ids_); size_t RowCount() const; size_t Size() const; - Status Serialize(std::string& table_id); + Status Serialize(std::string &table_id); ~MemVectors(); - const std::string& Location() const { return schema_.location_; } + const std::string &Location() const { return schema_.location_; } std::string TableId() const { return schema_.table_id_; } -private: + private: MemVectors() = delete; - MemVectors(const MemVectors&) = delete; - MemVectors& operator=(const MemVectors&) = delete; + MemVectors(const MemVectors &) = delete; + MemVectors &operator=(const MemVectors &) = delete; MetaPtr meta_; Options options_; meta::TableFileSchema schema_; - IDGenerator* id_generator_; + IDGenerator *id_generator_; ExecutionEnginePtr active_engine_; }; // MemVectors @@ -63,20 +64,20 @@ private: class MemManager : public MemManagerAbstract { -public: + public: using MetaPtr = meta::Meta::Ptr; using MemVectorsPtr = typename MemVectors::Ptr; using Ptr = std::shared_ptr; - MemManager(const std::shared_ptr& meta, const Options& options) + MemManager(const std::shared_ptr &meta, const Options &options) : meta_(meta), options_(options) {} - Status InsertVectors(const std::string& table_id, - size_t n, const float* vectors, IDNumbers& vector_ids) override; + Status InsertVectors(const std::string &table_id, + size_t n, const float *vectors, IDNumbers &vector_ids) override; - Status Serialize(std::set& table_ids) override; + Status Serialize(std::set &table_ids) override; - Status EraseMemVector(const std::string& table_id) override; + Status EraseMemVector(const std::string &table_id) override; size_t GetCurrentMutableMem() override; @@ -84,11 +85,11 @@ public: size_t GetCurrentMem() override; -private: - MemVectorsPtr GetMemByTable(const std::string& table_id); + private: + MemVectorsPtr GetMemByTable(const std::string &table_id); - Status InsertVectorsNoLock(const std::string& table_id, - size_t n, const float* vectors, IDNumbers& vector_ids); + Status InsertVectorsNoLock(const std::string &table_id, + size_t n, const float *vectors, IDNumbers &vector_ids); Status ToImmutable(); using MemIdMap = std::map; diff --git a/cpp/src/db/MemManagerAbstract.h b/cpp/src/db/MemManagerAbstract.h index 58c73ba6f8..943c454e46 100644 --- a/cpp/src/db/MemManagerAbstract.h +++ b/cpp/src/db/MemManagerAbstract.h @@ -2,19 +2,20 @@ #include + namespace zilliz { namespace milvus { namespace engine { class MemManagerAbstract { -public: + public: - virtual Status InsertVectors(const std::string& table_id, - size_t n, const float* vectors, IDNumbers& vector_ids) = 0; + virtual Status InsertVectors(const std::string &table_id, + size_t n, const float *vectors, IDNumbers &vector_ids) = 0; - virtual Status Serialize(std::set& table_ids) = 0; + virtual Status Serialize(std::set &table_ids) = 0; - virtual Status EraseMemVector(const std::string& table_id) = 0; + virtual Status EraseMemVector(const std::string &table_id) = 0; virtual size_t GetCurrentMutableMem() = 0; diff --git a/cpp/src/db/MemTable.cpp b/cpp/src/db/MemTable.cpp index ba3875fbb5..e05aa058ac 100644 --- a/cpp/src/db/MemTable.cpp +++ b/cpp/src/db/MemTable.cpp @@ -1,46 +1,50 @@ #include "MemTable.h" #include "Log.h" + namespace zilliz { namespace milvus { namespace engine { -MemTable::MemTable(const std::string& table_id, - const std::shared_ptr& meta, - const Options& options) : - table_id_(table_id), - meta_(meta), - options_(options) { +MemTable::MemTable(const std::string &table_id, + const std::shared_ptr &meta, + const Options &options) : + table_id_(table_id), + meta_(meta), + options_(options) { } -Status MemTable::Add(VectorSource::Ptr& source) { +Status MemTable::Add(VectorSource::Ptr &source) { + while (!source->AllAdded()) { - MemTableFile::Ptr currentMemTableFile; + + MemTableFile::Ptr current_mem_table_file; if (!mem_table_file_list_.empty()) { - currentMemTableFile = mem_table_file_list_.back(); + current_mem_table_file = mem_table_file_list_.back(); } + Status status; - if (mem_table_file_list_.empty() || currentMemTableFile->IsFull()) { - MemTableFile::Ptr newMemTableFile = std::make_shared(table_id_, meta_, options_); - status = newMemTableFile->Add(source); + if (mem_table_file_list_.empty() || current_mem_table_file->IsFull()) { + MemTableFile::Ptr new_mem_table_file = std::make_shared(table_id_, meta_, options_); + status = new_mem_table_file->Add(source); if (status.ok()) { - mem_table_file_list_.emplace_back(newMemTableFile); + mem_table_file_list_.emplace_back(new_mem_table_file); } + } else { + status = current_mem_table_file->Add(source); } - else { - status = currentMemTableFile->Add(source); - } + if (!status.ok()) { - std::string errMsg = "MemTable::Add failed: " + status.ToString(); - ENGINE_LOG_ERROR << errMsg; - return Status::Error(errMsg); + std::string err_msg = "MemTable::Add failed: " + status.ToString(); + ENGINE_LOG_ERROR << err_msg; + return Status::Error(err_msg); } } return Status::OK(); } -void MemTable::GetCurrentMemTableFile(MemTableFile::Ptr& mem_table_file) { +void MemTable::GetCurrentMemTableFile(MemTableFile::Ptr &mem_table_file) { mem_table_file = mem_table_file_list_.back(); } @@ -49,15 +53,15 @@ size_t MemTable::GetTableFileCount() { } Status MemTable::Serialize() { - for (auto memTableFile = mem_table_file_list_.begin(); memTableFile != mem_table_file_list_.end(); ) { - auto status = (*memTableFile)->Serialize(); + for (auto mem_table_file = mem_table_file_list_.begin(); mem_table_file != mem_table_file_list_.end();) { + auto status = (*mem_table_file)->Serialize(); if (!status.ok()) { - std::string errMsg = "MemTable::Serialize failed: " + status.ToString(); - ENGINE_LOG_ERROR << errMsg; - return Status::Error(errMsg); + std::string err_msg = "MemTable::Serialize failed: " + status.ToString(); + ENGINE_LOG_ERROR << err_msg; + return Status::Error(err_msg); } std::lock_guard lock(mutex_); - memTableFile = mem_table_file_list_.erase(memTableFile); + mem_table_file = mem_table_file_list_.erase(mem_table_file); } return Status::OK(); } @@ -66,17 +70,17 @@ bool MemTable::Empty() { return mem_table_file_list_.empty(); } -const std::string& MemTable::GetTableId() const { +const std::string &MemTable::GetTableId() const { return table_id_; } size_t MemTable::GetCurrentMem() { std::lock_guard lock(mutex_); - size_t totalMem = 0; - for (auto& memTableFile : mem_table_file_list_) { - totalMem += memTableFile->GetCurrentMem(); + size_t total_mem = 0; + for (auto &mem_table_file : mem_table_file_list_) { + total_mem += mem_table_file->GetCurrentMem(); } - return totalMem; + return total_mem; } } // namespace engine diff --git a/cpp/src/db/MemTable.h b/cpp/src/db/MemTable.h index 9bae932e62..198fcc228a 100644 --- a/cpp/src/db/MemTable.h +++ b/cpp/src/db/MemTable.h @@ -6,23 +6,24 @@ #include + namespace zilliz { namespace milvus { namespace engine { class MemTable { -public: + public: using Ptr = std::shared_ptr; using MemTableFileList = std::vector; using MetaPtr = meta::Meta::Ptr; - MemTable(const std::string& table_id, const std::shared_ptr& meta, const Options& options); + MemTable(const std::string &table_id, const std::shared_ptr &meta, const Options &options); - Status Add(VectorSource::Ptr& source); + Status Add(VectorSource::Ptr &source); - void GetCurrentMemTableFile(MemTableFile::Ptr& mem_table_file); + void GetCurrentMemTableFile(MemTableFile::Ptr &mem_table_file); size_t GetTableFileCount(); @@ -30,11 +31,11 @@ public: bool Empty(); - const std::string& GetTableId() const; + const std::string &GetTableId() const; size_t GetCurrentMem(); -private: + private: const std::string table_id_; MemTableFileList mem_table_file_list_; diff --git a/cpp/src/db/MemTableFile.cpp b/cpp/src/db/MemTableFile.cpp index 0ff91de00b..649a680cf3 100644 --- a/cpp/src/db/MemTableFile.cpp +++ b/cpp/src/db/MemTableFile.cpp @@ -6,23 +6,24 @@ #include + namespace zilliz { namespace milvus { namespace engine { -MemTableFile::MemTableFile(const std::string& table_id, - const std::shared_ptr& meta, - const Options& options) : - table_id_(table_id), - meta_(meta), - options_(options) { +MemTableFile::MemTableFile(const std::string &table_id, + const std::shared_ptr &meta, + const Options &options) : + table_id_(table_id), + meta_(meta), + options_(options) { current_mem_ = 0; auto status = CreateTableFile(); if (status.ok()) { execution_engine_ = EngineFactory::Build(table_file_schema_.dimension_, table_file_schema_.location_, - (EngineType)table_file_schema_.engine_type_); + (EngineType) table_file_schema_.engine_type_); } } @@ -33,31 +34,30 @@ Status MemTableFile::CreateTableFile() { auto status = meta_->CreateTableFile(table_file_schema); if (status.ok()) { table_file_schema_ = table_file_schema; - } - else { - std::string errMsg = "MemTableFile::CreateTableFile failed: " + status.ToString(); - ENGINE_LOG_ERROR << errMsg; + } else { + std::string err_msg = "MemTableFile::CreateTableFile failed: " + status.ToString(); + ENGINE_LOG_ERROR << err_msg; } return status; } -Status MemTableFile::Add(const VectorSource::Ptr& source) { +Status MemTableFile::Add(const VectorSource::Ptr &source) { if (table_file_schema_.dimension_ <= 0) { - std::string errMsg = "MemTableFile::Add: table_file_schema dimension = " + - std::to_string(table_file_schema_.dimension_) + ", table_id = " + table_file_schema_.table_id_; - ENGINE_LOG_ERROR << errMsg; - return Status::Error(errMsg); + std::string err_msg = "MemTableFile::Add: table_file_schema dimension = " + + std::to_string(table_file_schema_.dimension_) + ", table_id = " + table_file_schema_.table_id_; + ENGINE_LOG_ERROR << err_msg; + return Status::Error(err_msg); } - size_t singleVectorMemSize = table_file_schema_.dimension_ * VECTOR_TYPE_SIZE; - size_t memLeft = GetMemLeft(); - if (memLeft >= singleVectorMemSize) { - size_t numVectorsToAdd = std::ceil(memLeft / singleVectorMemSize); - size_t numVectorsAdded; - auto status = source->Add(execution_engine_, table_file_schema_, numVectorsToAdd, numVectorsAdded); + size_t single_vector_mem_size = table_file_schema_.dimension_ * VECTOR_TYPE_SIZE; + size_t mem_left = GetMemLeft(); + if (mem_left >= single_vector_mem_size) { + size_t num_vectors_to_add = std::ceil(mem_left / single_vector_mem_size); + size_t num_vectors_added; + auto status = source->Add(execution_engine_, table_file_schema_, num_vectors_to_add, num_vectors_added); if (status.ok()) { - current_mem_ += (numVectorsAdded * singleVectorMemSize); + current_mem_ += (num_vectors_added * single_vector_mem_size); } return status; } @@ -73,8 +73,8 @@ size_t MemTableFile::GetMemLeft() { } bool MemTableFile::IsFull() { - size_t singleVectorMemSize = table_file_schema_.dimension_ * VECTOR_TYPE_SIZE; - return (GetMemLeft() < singleVectorMemSize); + size_t single_vector_mem_size = table_file_schema_.dimension_ * VECTOR_TYPE_SIZE; + return (GetMemLeft() < single_vector_mem_size); } Status MemTableFile::Serialize() { @@ -88,15 +88,15 @@ Status MemTableFile::Serialize() { auto total_time = METRICS_MICROSECONDS(start_time, end_time); table_file_schema_.size_ = size; - server::Metrics::GetInstance().DiskStoreIOSpeedGaugeSet((double)size/total_time); + server::Metrics::GetInstance().DiskStoreIOSpeedGaugeSet((double) size / total_time); table_file_schema_.file_type_ = (size >= options_.index_trigger_size) ? - meta::TableFileSchema::TO_INDEX : meta::TableFileSchema::RAW; + meta::TableFileSchema::TO_INDEX : meta::TableFileSchema::RAW; auto status = meta_->UpdateTableFile(table_file_schema_); LOG(DEBUG) << "New " << ((table_file_schema_.file_type_ == meta::TableFileSchema::RAW) ? "raw" : "to_index") - << " file " << table_file_schema_.file_id_ << " of size " << (double)size / (double)M << " M"; + << " file " << table_file_schema_.file_id_ << " of size " << (double) size / (double) M << " M"; execution_engine_->Cache(); diff --git a/cpp/src/db/MemTableFile.h b/cpp/src/db/MemTableFile.h index 1be0ae78ba..4d0011b362 100644 --- a/cpp/src/db/MemTableFile.h +++ b/cpp/src/db/MemTableFile.h @@ -5,20 +5,21 @@ #include "VectorSource.h" #include "ExecutionEngine.h" + namespace zilliz { namespace milvus { namespace engine { class MemTableFile { -public: + public: using Ptr = std::shared_ptr; using MetaPtr = meta::Meta::Ptr; - MemTableFile(const std::string& table_id, const std::shared_ptr& meta, const Options& options); + MemTableFile(const std::string &table_id, const std::shared_ptr &meta, const Options &options); - Status Add(const VectorSource::Ptr& source); + Status Add(const VectorSource::Ptr &source); size_t GetCurrentMem(); @@ -28,7 +29,7 @@ public: Status Serialize(); -private: + private: Status CreateTableFile(); diff --git a/cpp/src/db/NewMemManager.cpp b/cpp/src/db/NewMemManager.cpp index 3c78f37101..b0fcc9d4ae 100644 --- a/cpp/src/db/NewMemManager.cpp +++ b/cpp/src/db/NewMemManager.cpp @@ -5,11 +5,12 @@ #include + namespace zilliz { namespace milvus { namespace engine { -NewMemManager::MemTablePtr NewMemManager::GetMemByTable(const std::string& table_id) { +NewMemManager::MemTablePtr NewMemManager::GetMemByTable(const std::string &table_id) { auto memIt = mem_id_map_.find(table_id); if (memIt != mem_id_map_.end()) { return memIt->second; @@ -19,27 +20,27 @@ NewMemManager::MemTablePtr NewMemManager::GetMemByTable(const std::string& table return mem_id_map_[table_id]; } -Status NewMemManager::InsertVectors(const std::string& table_id_, +Status NewMemManager::InsertVectors(const std::string &table_id_, size_t n_, - const float* vectors_, - IDNumbers& vector_ids_) { + const float *vectors_, + IDNumbers &vector_ids_) { while (GetCurrentMem() > options_.maximum_memory) { std::this_thread::sleep_for(std::chrono::milliseconds(1)); } + LOG(DEBUG) << "NewMemManager::InsertVectors: mutable mem = " << GetCurrentMutableMem() << + ", immutable mem = " << GetCurrentImmutableMem() << ", total mem = " << GetCurrentMem(); + std::unique_lock lock(mutex_); return InsertVectorsNoLock(table_id_, n_, vectors_, vector_ids_); } -Status NewMemManager::InsertVectorsNoLock(const std::string& table_id, +Status NewMemManager::InsertVectorsNoLock(const std::string &table_id, size_t n, - const float* vectors, - IDNumbers& vector_ids) { - - LOG(DEBUG) << "NewMemManager::InsertVectorsNoLock: mutable mem = " << GetCurrentMutableMem() << - ", immutable mem = " << GetCurrentImmutableMem() << ", total mem = " << GetCurrentMem(); + const float *vectors, + IDNumbers &vector_ids) { MemTablePtr mem = GetMemByTable(table_id); VectorSource::Ptr source = std::make_shared(n, vectors); @@ -54,37 +55,33 @@ Status NewMemManager::InsertVectorsNoLock(const std::string& table_id, Status NewMemManager::ToImmutable() { std::unique_lock lock(mutex_); MemIdMap temp_map; - for (auto& kv: mem_id_map_) { - if(kv.second->Empty()) { + for (auto &kv: mem_id_map_) { + if (kv.second->Empty()) { + //empty table, no need to serialize temp_map.insert(kv); - continue;//empty table, no need to serialize + } else { + immu_mem_list_.push_back(kv.second); } - immu_mem_list_.push_back(kv.second); } mem_id_map_.swap(temp_map); return Status::OK(); } -Status NewMemManager::Serialize(std::set& table_ids) { +Status NewMemManager::Serialize(std::set &table_ids) { ToImmutable(); std::unique_lock lock(serialization_mtx_); table_ids.clear(); - for (auto& mem : immu_mem_list_) { + for (auto &mem : immu_mem_list_) { mem->Serialize(); table_ids.insert(mem->GetTableId()); } immu_mem_list_.clear(); -// for (auto mem = immu_mem_list_.begin(); mem != immu_mem_list_.end(); ) { -// (*mem)->Serialize(); -// table_ids.insert((*mem)->GetTableId()); -// mem = immu_mem_list_.erase(mem); -// LOG(DEBUG) << "immu_mem_list_ size = " << immu_mem_list_.size(); -// } + return Status::OK(); } -Status NewMemManager::EraseMemVector(const std::string& table_id) { +Status NewMemManager::EraseMemVector(const std::string &table_id) { {//erase MemVector from rapid-insert cache std::unique_lock lock(mutex_); mem_id_map_.erase(table_id); @@ -93,8 +90,8 @@ Status NewMemManager::EraseMemVector(const std::string& table_id) { {//erase MemVector from serialize cache std::unique_lock lock(serialization_mtx_); MemList temp_list; - for (auto& mem : immu_mem_list_) { - if(mem->GetTableId() != table_id) { + for (auto &mem : immu_mem_list_) { + if (mem->GetTableId() != table_id) { temp_list.push_back(mem); } } @@ -105,20 +102,20 @@ Status NewMemManager::EraseMemVector(const std::string& table_id) { } size_t NewMemManager::GetCurrentMutableMem() { - size_t totalMem = 0; - for (auto& kv : mem_id_map_) { + size_t total_mem = 0; + for (auto &kv : mem_id_map_) { auto memTable = kv.second; - totalMem += memTable->GetCurrentMem(); + total_mem += memTable->GetCurrentMem(); } - return totalMem; + return total_mem; } size_t NewMemManager::GetCurrentImmutableMem() { - size_t totalMem = 0; - for (auto& memTable : immu_mem_list_) { - totalMem += memTable->GetCurrentMem(); + size_t total_mem = 0; + for (auto &mem_table : immu_mem_list_) { + total_mem += mem_table->GetCurrentMem(); } - return totalMem; + return total_mem; } size_t NewMemManager::GetCurrentMem() { diff --git a/cpp/src/db/NewMemManager.h b/cpp/src/db/NewMemManager.h index 9883480404..5b933c94ca 100644 --- a/cpp/src/db/NewMemManager.h +++ b/cpp/src/db/NewMemManager.h @@ -11,25 +11,26 @@ #include #include + namespace zilliz { namespace milvus { namespace engine { class NewMemManager : public MemManagerAbstract { -public: + public: using MetaPtr = meta::Meta::Ptr; using Ptr = std::shared_ptr; using MemTablePtr = typename MemTable::Ptr; - NewMemManager(const std::shared_ptr& meta, const Options& options) - : meta_(meta), options_(options) {} + NewMemManager(const std::shared_ptr &meta, const Options &options) + : meta_(meta), options_(options) {} - Status InsertVectors(const std::string& table_id, - size_t n, const float* vectors, IDNumbers& vector_ids) override; + Status InsertVectors(const std::string &table_id, + size_t n, const float *vectors, IDNumbers &vector_ids) override; - Status Serialize(std::set& table_ids) override; + Status Serialize(std::set &table_ids) override; - Status EraseMemVector(const std::string& table_id) override; + Status EraseMemVector(const std::string &table_id) override; size_t GetCurrentMutableMem() override; @@ -37,11 +38,11 @@ public: size_t GetCurrentMem() override; -private: - MemTablePtr GetMemByTable(const std::string& table_id); + private: + MemTablePtr GetMemByTable(const std::string &table_id); - Status InsertVectorsNoLock(const std::string& table_id, - size_t n, const float* vectors, IDNumbers& vector_ids); + Status InsertVectorsNoLock(const std::string &table_id, + size_t n, const float *vectors, IDNumbers &vector_ids); Status ToImmutable(); using MemIdMap = std::map; diff --git a/cpp/src/db/VectorSource.cpp b/cpp/src/db/VectorSource.cpp index d032be51f6..74c07ae1f6 100644 --- a/cpp/src/db/VectorSource.cpp +++ b/cpp/src/db/VectorSource.cpp @@ -4,6 +4,7 @@ #include "Log.h" #include "metrics/Metrics.h" + namespace zilliz { namespace milvus { namespace engine { @@ -11,16 +12,16 @@ namespace engine { VectorSource::VectorSource(const size_t &n, const float *vectors) : - n_(n), - vectors_(vectors), - id_generator_(new SimpleIDGenerator()) { + n_(n), + vectors_(vectors), + id_generator_(new SimpleIDGenerator()) { current_num_vectors_added = 0; } -Status VectorSource::Add(const ExecutionEnginePtr& execution_engine, - const meta::TableFileSchema& table_file_schema, - const size_t& num_vectors_to_add, - size_t& num_vectors_added) { +Status VectorSource::Add(const ExecutionEnginePtr &execution_engine, + const meta::TableFileSchema &table_file_schema, + const size_t &num_vectors_to_add, + size_t &num_vectors_added) { auto start_time = METRICS_NOW_TIME; @@ -36,8 +37,7 @@ Status VectorSource::Add(const ExecutionEnginePtr& execution_engine, vector_ids_.insert(vector_ids_.end(), std::make_move_iterator(vector_ids_to_add.begin()), std::make_move_iterator(vector_ids_to_add.end())); - } - else { + } else { ENGINE_LOG_ERROR << "VectorSource::Add failed: " + status.ToString(); } diff --git a/cpp/src/db/VectorSource.h b/cpp/src/db/VectorSource.h index dec31f39e1..7092805a6d 100644 --- a/cpp/src/db/VectorSource.h +++ b/cpp/src/db/VectorSource.h @@ -5,22 +5,23 @@ #include "IDGenerator.h" #include "ExecutionEngine.h" + namespace zilliz { namespace milvus { namespace engine { class VectorSource { -public: + public: using Ptr = std::shared_ptr; - VectorSource(const size_t& n, const float* vectors); + VectorSource(const size_t &n, const float *vectors); - Status Add(const ExecutionEnginePtr& execution_engine, - const meta::TableFileSchema& table_file_schema, - const size_t& num_vectors_to_add, - size_t& num_vectors_added); + Status Add(const ExecutionEnginePtr &execution_engine, + const meta::TableFileSchema &table_file_schema, + const size_t &num_vectors_to_add, + size_t &num_vectors_added); size_t GetNumVectorsAdded(); @@ -28,15 +29,15 @@ public: IDNumbers GetVectorIds(); -private: + private: const size_t n_; - const float* vectors_; + const float *vectors_; IDNumbers vector_ids_; size_t current_num_vectors_added; - IDGenerator* id_generator_; + IDGenerator *id_generator_; }; //VectorSource diff --git a/cpp/unittest/db/mem_test.cpp b/cpp/unittest/db/mem_test.cpp index 818c3a6388..5b7972ec35 100644 --- a/cpp/unittest/db/mem_test.cpp +++ b/cpp/unittest/db/mem_test.cpp @@ -15,33 +15,34 @@ #include #include + using namespace zilliz::milvus; namespace { - static const std::string TABLE_NAME = "test_group"; - static constexpr int64_t TABLE_DIM = 256; - static constexpr int64_t VECTOR_COUNT = 250000; - static constexpr int64_t INSERT_LOOP = 10000; +static const std::string TABLE_NAME = "test_group"; +static constexpr int64_t TABLE_DIM = 256; +static constexpr int64_t VECTOR_COUNT = 250000; +static constexpr int64_t INSERT_LOOP = 10000; - engine::meta::TableSchema BuildTableSchema() { - engine::meta::TableSchema table_info; - table_info.dimension_ = TABLE_DIM; - table_info.table_id_ = TABLE_NAME; - table_info.engine_type_ = (int)engine::EngineType::FAISS_IDMAP; - return table_info; - } +engine::meta::TableSchema BuildTableSchema() { + engine::meta::TableSchema table_info; + table_info.dimension_ = TABLE_DIM; + table_info.table_id_ = TABLE_NAME; + table_info.engine_type_ = (int) engine::EngineType::FAISS_IDMAP; + return table_info; +} - void BuildVectors(int64_t n, std::vector& vectors) { - vectors.clear(); - vectors.resize(n*TABLE_DIM); - float* data = vectors.data(); - for(int i = 0; i < n; i++) { - for(int j = 0; j < TABLE_DIM; j++) data[TABLE_DIM * i + j] = drand48(); - data[TABLE_DIM * i] += i / 2000.; - } +void BuildVectors(int64_t n, std::vector &vectors) { + vectors.clear(); + vectors.resize(n * TABLE_DIM); + float *data = vectors.data(); + for (int i = 0; i < n; i++) { + for (int j = 0; j < TABLE_DIM; j++) data[TABLE_DIM * i + j] = drand48(); + data[TABLE_DIM * i] += i / 2000.; } } +} TEST_F(NewMemManagerTest, VECTOR_SOURCE_TEST) { @@ -65,7 +66,7 @@ TEST_F(NewMemManagerTest, VECTOR_SOURCE_TEST) { size_t num_vectors_added; engine::ExecutionEnginePtr execution_engine_ = engine::EngineFactory::Build(table_file_schema.dimension_, table_file_schema.location_, - (engine::EngineType)table_file_schema.engine_type_); + (engine::EngineType) table_file_schema.engine_type_); status = source.Add(execution_engine_, table_file_schema, 50, num_vectors_added); ASSERT_TRUE(status.ok()); @@ -82,10 +83,6 @@ TEST_F(NewMemManagerTest, VECTOR_SOURCE_TEST) { vector_ids = source.GetVectorIds(); ASSERT_EQ(vector_ids.size(), 100); -// for (auto& id : vector_ids) { -// std::cout << id << std::endl; -// } - status = impl_->DropAll(); ASSERT_TRUE(status.ok()); } @@ -99,7 +96,7 @@ TEST_F(NewMemManagerTest, MEM_TABLE_FILE_TEST) { auto status = impl_->CreateTable(table_schema); ASSERT_TRUE(status.ok()); - engine::MemTableFile memTableFile(TABLE_NAME, impl_, options); + engine::MemTableFile mem_table_file(TABLE_NAME, impl_, options); int64_t n_100 = 100; std::vector vectors_100; @@ -107,28 +104,28 @@ TEST_F(NewMemManagerTest, MEM_TABLE_FILE_TEST) { engine::VectorSource::Ptr source = std::make_shared(n_100, vectors_100.data()); - status = memTableFile.Add(source); + status = mem_table_file.Add(source); ASSERT_TRUE(status.ok()); -// std::cout << memTableFile.GetCurrentMem() << " " << memTableFile.GetMemLeft() << std::endl; +// std::cout << mem_table_file.GetCurrentMem() << " " << mem_table_file.GetMemLeft() << std::endl; engine::IDNumbers vector_ids = source->GetVectorIds(); ASSERT_EQ(vector_ids.size(), 100); size_t singleVectorMem = sizeof(float) * TABLE_DIM; - ASSERT_EQ(memTableFile.GetCurrentMem(), n_100 * singleVectorMem); + ASSERT_EQ(mem_table_file.GetCurrentMem(), n_100 * singleVectorMem); int64_t n_max = engine::MAX_TABLE_FILE_MEM / singleVectorMem; std::vector vectors_128M; BuildVectors(n_max, vectors_128M); engine::VectorSource::Ptr source_128M = std::make_shared(n_max, vectors_128M.data()); - status = memTableFile.Add(source_128M); + status = mem_table_file.Add(source_128M); vector_ids = source_128M->GetVectorIds(); ASSERT_EQ(vector_ids.size(), n_max - n_100); - ASSERT_TRUE(memTableFile.IsFull()); + ASSERT_TRUE(mem_table_file.IsFull()); status = impl_->DropAll(); ASSERT_TRUE(status.ok()); @@ -149,34 +146,34 @@ TEST_F(NewMemManagerTest, MEM_TABLE_TEST) { engine::VectorSource::Ptr source_100 = std::make_shared(n_100, vectors_100.data()); - engine::MemTable memTable(TABLE_NAME, impl_, options); + engine::MemTable mem_table(TABLE_NAME, impl_, options); - status = memTable.Add(source_100); + status = mem_table.Add(source_100); ASSERT_TRUE(status.ok()); engine::IDNumbers vector_ids = source_100->GetVectorIds(); ASSERT_EQ(vector_ids.size(), 100); - engine::MemTableFile::Ptr memTableFile; - memTable.GetCurrentMemTableFile(memTableFile); + engine::MemTableFile::Ptr mem_table_file; + mem_table.GetCurrentMemTableFile(mem_table_file); size_t singleVectorMem = sizeof(float) * TABLE_DIM; - ASSERT_EQ(memTableFile->GetCurrentMem(), n_100 * singleVectorMem); + ASSERT_EQ(mem_table_file->GetCurrentMem(), n_100 * singleVectorMem); int64_t n_max = engine::MAX_TABLE_FILE_MEM / singleVectorMem; std::vector vectors_128M; BuildVectors(n_max, vectors_128M); engine::VectorSource::Ptr source_128M = std::make_shared(n_max, vectors_128M.data()); - status = memTable.Add(source_128M); + status = mem_table.Add(source_128M); ASSERT_TRUE(status.ok()); vector_ids = source_128M->GetVectorIds(); ASSERT_EQ(vector_ids.size(), n_max); - memTable.GetCurrentMemTableFile(memTableFile); - ASSERT_EQ(memTableFile->GetCurrentMem(), n_100 * singleVectorMem); + mem_table.GetCurrentMemTableFile(mem_table_file); + ASSERT_EQ(mem_table_file->GetCurrentMem(), n_100 * singleVectorMem); - ASSERT_EQ(memTable.GetTableFileCount(), 2); + ASSERT_EQ(mem_table.GetTableFileCount(), 2); int64_t n_1G = 1024000; std::vector vectors_1G; @@ -184,16 +181,16 @@ TEST_F(NewMemManagerTest, MEM_TABLE_TEST) { engine::VectorSource::Ptr source_1G = std::make_shared(n_1G, vectors_1G.data()); - status = memTable.Add(source_1G); + status = mem_table.Add(source_1G); ASSERT_TRUE(status.ok()); vector_ids = source_1G->GetVectorIds(); ASSERT_EQ(vector_ids.size(), n_1G); int expectedTableFileCount = 2 + std::ceil((n_1G - n_100) * singleVectorMem / engine::MAX_TABLE_FILE_MEM); - ASSERT_EQ(memTable.GetTableFileCount(), expectedTableFileCount); + ASSERT_EQ(mem_table.GetTableFileCount(), expectedTableFileCount); - status = memTable.Serialize(); + status = mem_table.Serialize(); ASSERT_TRUE(status.ok()); status = impl_->DropAll(); @@ -216,7 +213,7 @@ TEST_F(NewMemManagerTest, SERIAL_INSERT_SEARCH_TEST) { ASSERT_STATS(stat); ASSERT_EQ(table_info_get.dimension_, TABLE_DIM); - std::map> search_vectors; + std::map> search_vectors; { engine::IDNumbers vector_ids; int64_t nb = 1024000; @@ -231,8 +228,8 @@ TEST_F(NewMemManagerTest, SERIAL_INSERT_SEARCH_TEST) { std::mt19937 gen(rd()); std::uniform_int_distribution dis(0, nb - 1); - int64_t numQuery = 20; - for (int64_t i = 0; i < numQuery; ++i) { + int64_t num_query = 20; + for (int64_t i = 0; i < num_query; ++i) { int64_t index = dis(gen); std::vector search; for (int64_t j = 0; j < TABLE_DIM; j++) { @@ -243,8 +240,8 @@ TEST_F(NewMemManagerTest, SERIAL_INSERT_SEARCH_TEST) { } int k = 10; - for(auto& pair : search_vectors) { - auto& search = pair.second; + for (auto &pair : search_vectors) { + auto &search = pair.second; engine::QueryResults results; stat = db_->Query(TABLE_NAME, k, 1, search.data(), results); ASSERT_EQ(results[0][0].first, pair.first); @@ -329,18 +326,18 @@ TEST_F(NewMemManagerTest, CONCURRENT_INSERT_SEARCH_TEST) { uint64_t count = 0; uint64_t prev_count = 0; - for (auto j=0; j<10; ++j) { + for (auto j = 0; j < 10; ++j) { ss.str(""); db_->Size(count); prev_count = count; START_TIMER; stat = db_->Query(TABLE_NAME, k, qb, qxb.data(), results); - ss << "Search " << j << " With Size " << count/engine::meta::M << " M"; + ss << "Search " << j << " With Size " << count / engine::meta::M << " M"; STOP_TIMER(ss.str()); ASSERT_STATS(stat); - for (auto k=0; kInsertVectors(TABLE_NAME, qb, qxb.data(), target_ids); ASSERT_EQ(target_ids.size(), qb); } else { diff --git a/cpp/unittest/db/utils.h b/cpp/unittest/db/utils.h index d06500de5c..9c126030c2 100644 --- a/cpp/unittest/db/utils.h +++ b/cpp/unittest/db/utils.h @@ -30,7 +30,7 @@ #define STOP_TIMER(name) #endif -void ASSERT_STATS(zilliz::milvus::engine::Status& stat); +void ASSERT_STATS(zilliz::milvus::engine::Status &stat); //class TestEnv : public ::testing::Environment { //public: @@ -54,8 +54,8 @@ void ASSERT_STATS(zilliz::milvus::engine::Status& stat); // ::testing::AddGlobalTestEnvironment(new TestEnv); class DBTest : public ::testing::Test { -protected: - zilliz::milvus::engine::DB* db_; + protected: + zilliz::milvus::engine::DB *db_; void InitLog(); virtual void SetUp() override; @@ -64,13 +64,13 @@ protected: }; class DBTest2 : public DBTest { -protected: + protected: virtual zilliz::milvus::engine::Options GetOptions() override; }; class MetaTest : public DBTest { -protected: + protected: std::shared_ptr impl_; virtual void SetUp() override; @@ -78,17 +78,17 @@ protected: }; class MySQLTest : public ::testing::Test { -protected: + protected: // std::shared_ptr impl_; zilliz::milvus::engine::DBMetaOptions getDBMetaOptions(); }; -class MySQLDBTest : public ::testing::Test { -protected: +class MySQLDBTest : public ::testing::Test { + protected: zilliz::milvus::engine::Options GetOptions(); }; -class NewMemManagerTest : public ::testing::Test { +class NewMemManagerTest : public ::testing::Test { void InitLog(); - virtual void SetUp() override; + void SetUp() override; }; From e095face907844fd1692c182c95d9873a59cff2b Mon Sep 17 00:00:00 2001 From: zhiru Date: Wed, 3 Jul 2019 14:25:02 +0800 Subject: [PATCH 102/189] Disable cleanup if mode is read only Former-commit-id: 99eb96aa039dfc73eb5bef99d9a9416eae5d396e --- cpp/CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/cpp/CHANGELOG.md b/cpp/CHANGELOG.md index 8887c2436f..574eeea835 100644 --- a/cpp/CHANGELOG.md +++ b/cpp/CHANGELOG.md @@ -35,6 +35,7 @@ Please mark all change in change log and use the ticket from JIRA. - MS-90 - Fix arch match incorrect on ARM - MS-99 - Fix compilation bug - MS-110 - Avoid huge file size +- MS-148 - Disable cleanup if mode is read only ## Improvement - MS-82 - Update server startup welcome message From 2dc45d2a7bff9336d79abc49fbc8b4c25ecc3e0e Mon Sep 17 00:00:00 2001 From: zhiru Date: Wed, 3 Jul 2019 15:48:21 +0800 Subject: [PATCH 103/189] update Former-commit-id: d1cdc65f12f76335a7118fb3ebee3df201b26a43 --- cpp/src/server/RequestTask.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/cpp/src/server/RequestTask.cpp b/cpp/src/server/RequestTask.cpp index 4c207546b8..979dd99224 100644 --- a/cpp/src/server/RequestTask.cpp +++ b/cpp/src/server/RequestTask.cpp @@ -502,6 +502,7 @@ ServerError SearchVectorTask::OnExecute() { engine::QueryResults results; uint64_t record_count = (uint64_t)record_array_.size(); + SERVER_LOG_DEBUG << file_id_array_ << std::endl; if(file_id_array_.empty()) { stat = DBWrapper::DB()->Query(table_name_, (size_t) top_k_, record_count, vec_f.data(), dates, results); } else { From 7e96b554a0eaee69b8715e1829ffeac228667099 Mon Sep 17 00:00:00 2001 From: zhiru Date: Wed, 3 Jul 2019 16:04:00 +0800 Subject: [PATCH 104/189] update Former-commit-id: d5dba29858cecbf0781cad8b02a24de7c7fb080d --- cpp/src/server/RequestTask.cpp | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/cpp/src/server/RequestTask.cpp b/cpp/src/server/RequestTask.cpp index 979dd99224..c8bca99b18 100644 --- a/cpp/src/server/RequestTask.cpp +++ b/cpp/src/server/RequestTask.cpp @@ -502,7 +502,11 @@ ServerError SearchVectorTask::OnExecute() { engine::QueryResults results; uint64_t record_count = (uint64_t)record_array_.size(); - SERVER_LOG_DEBUG << file_id_array_ << std::endl; + SERVER_LOG_DEBUG << "file_id_array_: "; + for (auto& file_id : file_id_array_) { + SERVER_LOG_DEBUG << file_id; + } + if(file_id_array_.empty()) { stat = DBWrapper::DB()->Query(table_name_, (size_t) top_k_, record_count, vec_f.data(), dates, results); } else { From 5a04f56aad16a7b916bc679098e575f6bc2a4626 Mon Sep 17 00:00:00 2001 From: zhiru Date: Wed, 3 Jul 2019 17:24:36 +0800 Subject: [PATCH 105/189] update Former-commit-id: 5232df1758b4041adf936d8df46fe97ff89668cc --- cpp/src/db/DBImpl.cpp | 4 ++++ cpp/src/server/RequestTask.cpp | 5 ----- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/cpp/src/db/DBImpl.cpp b/cpp/src/db/DBImpl.cpp index 62d6b9c3cc..1179a756eb 100644 --- a/cpp/src/db/DBImpl.cpp +++ b/cpp/src/db/DBImpl.cpp @@ -195,6 +195,10 @@ Status DBImpl::Query(const std::string& table_id, const std::vector return status; } + for (auto& file_schema : files_array) { + ENGINE_LOG_DEBUG << "file_id: " << file_schema.file_id_; + } + if(files_array.empty()) { return Status::Error("Invalid file id"); } diff --git a/cpp/src/server/RequestTask.cpp b/cpp/src/server/RequestTask.cpp index c8bca99b18..4c207546b8 100644 --- a/cpp/src/server/RequestTask.cpp +++ b/cpp/src/server/RequestTask.cpp @@ -502,11 +502,6 @@ ServerError SearchVectorTask::OnExecute() { engine::QueryResults results; uint64_t record_count = (uint64_t)record_array_.size(); - SERVER_LOG_DEBUG << "file_id_array_: "; - for (auto& file_id : file_id_array_) { - SERVER_LOG_DEBUG << file_id; - } - if(file_id_array_.empty()) { stat = DBWrapper::DB()->Query(table_name_, (size_t) top_k_, record_count, vec_f.data(), dates, results); } else { From 46336733ee43a74b129c1345c7ba8fa9b618e8f1 Mon Sep 17 00:00:00 2001 From: zhiru Date: Wed, 3 Jul 2019 18:31:45 +0800 Subject: [PATCH 106/189] update Former-commit-id: 7e7f7df70a19e58f1c65ec487f2c5624e133ee47 --- cpp/src/db/DBImpl.cpp | 4 ---- 1 file changed, 4 deletions(-) diff --git a/cpp/src/db/DBImpl.cpp b/cpp/src/db/DBImpl.cpp index 1179a756eb..62d6b9c3cc 100644 --- a/cpp/src/db/DBImpl.cpp +++ b/cpp/src/db/DBImpl.cpp @@ -195,10 +195,6 @@ Status DBImpl::Query(const std::string& table_id, const std::vector return status; } - for (auto& file_schema : files_array) { - ENGINE_LOG_DEBUG << "file_id: " << file_schema.file_id_; - } - if(files_array.empty()) { return Status::Error("Invalid file id"); } From 640f3f3917f90713c3852ba373b07e73b70e36d9 Mon Sep 17 00:00:00 2001 From: zhiru Date: Wed, 3 Jul 2019 18:44:29 +0800 Subject: [PATCH 107/189] update Former-commit-id: c6e3e88df71a755ae3b93df822ad426179a86386 --- cpp/CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/cpp/CHANGELOG.md b/cpp/CHANGELOG.md index 574eeea835..dfccf0208a 100644 --- a/cpp/CHANGELOG.md +++ b/cpp/CHANGELOG.md @@ -36,6 +36,7 @@ Please mark all change in change log and use the ticket from JIRA. - MS-99 - Fix compilation bug - MS-110 - Avoid huge file size - MS-148 - Disable cleanup if mode is read only +- MS-149 - Fixed searching only one index file issue in distributed mode ## Improvement - MS-82 - Update server startup welcome message From e94b70829c01de33e0f0132e17280769532ce0f9 Mon Sep 17 00:00:00 2001 From: yu yunfeng Date: Wed, 3 Jul 2019 20:07:11 +0800 Subject: [PATCH 108/189] acc test Former-commit-id: 1f6af0b88d5d9d3b70ab5c891e8ce6c3a6f004a1 --- cpp/src/db/FaissExecutionEngine.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/cpp/src/db/FaissExecutionEngine.cpp b/cpp/src/db/FaissExecutionEngine.cpp index 20bd530e78..201c07dbcf 100644 --- a/cpp/src/db/FaissExecutionEngine.cpp +++ b/cpp/src/db/FaissExecutionEngine.cpp @@ -138,6 +138,7 @@ Status FaissExecutionEngine::Search(long n, auto start_time = METRICS_NOW_TIME; std::shared_ptr ivf_index = std::dynamic_pointer_cast(pIndex_); + //ENGINE_LOG_DEBUG << "Index nlist: " << ivf_index->nlist << ", ntotal: "<< ivf_index->ntotal; if(ivf_index) { ENGINE_LOG_DEBUG << "Index type: IVFFLAT nProbe: " << nprobe_; ivf_index->nprobe = nprobe_; From e0a6469f8a219ddd210d2f05a255e9e577a1b39b Mon Sep 17 00:00:00 2001 From: yu yunfeng Date: Wed, 3 Jul 2019 20:59:36 +0800 Subject: [PATCH 109/189] ADD LOG Former-commit-id: f8ce9610b58cedf3505afc956e78b656c2152e31 --- cpp/src/db/scheduler/task/SearchTask.cpp | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/cpp/src/db/scheduler/task/SearchTask.cpp b/cpp/src/db/scheduler/task/SearchTask.cpp index 91ac123dec..bd6837f363 100644 --- a/cpp/src/db/scheduler/task/SearchTask.cpp +++ b/cpp/src/db/scheduler/task/SearchTask.cpp @@ -70,6 +70,14 @@ std::shared_ptr SearchTask::Execute() { SearchTask::ClusterResult(output_ids, output_distence, context->nq(), spec_k, result_set); rc.Record("cluster result"); + + SERVER_LOG_DEBUG << "Query Result: "; + for(auto& id2score_vector: result_set) { + for(auto& pair: id2score_vector) { + SERVER_LOG_DEBUG << "id: " << pair.first << ", distance: " << pair.second; + } + } + //step 4: pick up topk result SearchTask::TopkResult(result_set, inner_k, context->GetResult()); rc.Record("reduce topk"); From 1f6a6458b7e3d029b7ba4c0fa492b7b55ddbd1e0 Mon Sep 17 00:00:00 2001 From: yu yunfeng Date: Thu, 4 Jul 2019 12:41:24 +0800 Subject: [PATCH 110/189] MS-151 Fix topk problem Former-commit-id: ead499ba9f5f152a6c2f9d0c6e2578f6f3a80f8e --- cpp/src/db/FaissExecutionEngine.cpp | 1 - cpp/src/db/scheduler/task/SearchTask.cpp | 12 ++---------- 2 files changed, 2 insertions(+), 11 deletions(-) diff --git a/cpp/src/db/FaissExecutionEngine.cpp b/cpp/src/db/FaissExecutionEngine.cpp index 201c07dbcf..20bd530e78 100644 --- a/cpp/src/db/FaissExecutionEngine.cpp +++ b/cpp/src/db/FaissExecutionEngine.cpp @@ -138,7 +138,6 @@ Status FaissExecutionEngine::Search(long n, auto start_time = METRICS_NOW_TIME; std::shared_ptr ivf_index = std::dynamic_pointer_cast(pIndex_); - //ENGINE_LOG_DEBUG << "Index nlist: " << ivf_index->nlist << ", ntotal: "<< ivf_index->ntotal; if(ivf_index) { ENGINE_LOG_DEBUG << "Index type: IVFFLAT nProbe: " << nprobe_; ivf_index->nprobe = nprobe_; diff --git a/cpp/src/db/scheduler/task/SearchTask.cpp b/cpp/src/db/scheduler/task/SearchTask.cpp index bd6837f363..8bea768a23 100644 --- a/cpp/src/db/scheduler/task/SearchTask.cpp +++ b/cpp/src/db/scheduler/task/SearchTask.cpp @@ -53,7 +53,7 @@ std::shared_ptr SearchTask::Execute() { std::vector output_distence; for(auto& context : search_contexts_) { //step 1: allocate memory - auto inner_k = index_engine_->Count() < context->topk() ? index_engine_->Count() : context->topk(); + auto inner_k = context->topk(); output_ids.resize(inner_k*context->nq()); output_distence.resize(inner_k*context->nq()); @@ -67,17 +67,9 @@ std::shared_ptr SearchTask::Execute() { //step 3: cluster result SearchContext::ResultSet result_set; auto spec_k = index_engine_->Count() < context->topk() ? index_engine_->Count() : context->topk(); - SearchTask::ClusterResult(output_ids, output_distence, context->nq(), spec_k, result_set); + ClusterResult(output_ids, output_distence, context->nq(), spec_k, result_set); rc.Record("cluster result"); - - SERVER_LOG_DEBUG << "Query Result: "; - for(auto& id2score_vector: result_set) { - for(auto& pair: id2score_vector) { - SERVER_LOG_DEBUG << "id: " << pair.first << ", distance: " << pair.second; - } - } - //step 4: pick up topk result SearchTask::TopkResult(result_set, inner_k, context->GetResult()); rc.Record("reduce topk"); From bad5da3b31dd0af0b542c73da651a94fec054df5 Mon Sep 17 00:00:00 2001 From: zhiru Date: Thu, 4 Jul 2019 14:20:59 +0800 Subject: [PATCH 111/189] update Former-commit-id: 85a14245eabca8806706a3807e701d0db91fd074 --- cpp/CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/cpp/CHANGELOG.md b/cpp/CHANGELOG.md index dfccf0208a..15c32f5e82 100644 --- a/cpp/CHANGELOG.md +++ b/cpp/CHANGELOG.md @@ -50,6 +50,7 @@ Please mark all change in change log and use the ticket from JIRA. - MS-124 - HasTable interface - MS-126 - Add more error code - MS-128 - Change default db path +- MS-152 - Delete assert in MySQLMetaImpl and change MySQLConnectionPool impl ## New Feature From a55fef07544a881d184cdc217fd618f744f37e5e Mon Sep 17 00:00:00 2001 From: zhiru Date: Thu, 4 Jul 2019 15:58:38 +0800 Subject: [PATCH 112/189] fix c_str error when connecting to MySQL Former-commit-id: 6c3fe3ff323cd8baa045d00f343bd48de312aa41 --- cpp/CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/cpp/CHANGELOG.md b/cpp/CHANGELOG.md index 15c32f5e82..39db1b5e7d 100644 --- a/cpp/CHANGELOG.md +++ b/cpp/CHANGELOG.md @@ -37,6 +37,7 @@ Please mark all change in change log and use the ticket from JIRA. - MS-110 - Avoid huge file size - MS-148 - Disable cleanup if mode is read only - MS-149 - Fixed searching only one index file issue in distributed mode +- MS-153 - fix c_str error when connecting to MySQL ## Improvement - MS-82 - Update server startup welcome message From 79d2d6630808a838b35ede275cb165a9ea63912d Mon Sep 17 00:00:00 2001 From: zhiru Date: Thu, 4 Jul 2019 17:15:38 +0800 Subject: [PATCH 113/189] fix changelog Former-commit-id: 32d04d3eb829ffbd0537a930cff80398f3ea2c85 --- cpp/CHANGELOG.md | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/cpp/CHANGELOG.md b/cpp/CHANGELOG.md index 39db1b5e7d..552f0c20f9 100644 --- a/cpp/CHANGELOG.md +++ b/cpp/CHANGELOG.md @@ -10,13 +10,14 @@ Please mark all change in change log and use the ticket from JIRA. - MS-148 - Disable cleanup if mode is read only - MS-149 - Fixed searching only one index file issue in distributed mode - MS-153 - fix c_str error when connecting to MySQL -- MS-157 - fix changelog ## Improvement - MS-156 - Add unittest for merge result functions - MS-152 - Delete assert in MySQLMetaImpl and change MySQLConnectionPool impl +- MS-152 - Delete assert in MySQLMetaImpl and change MySQLConnectionPool impl + ## New Feature - MS-137 - Integrate knowhere - MS-180 - Add new mem manager @@ -35,9 +36,6 @@ Please mark all change in change log and use the ticket from JIRA. - MS-90 - Fix arch match incorrect on ARM - MS-99 - Fix compilation bug - MS-110 - Avoid huge file size -- MS-148 - Disable cleanup if mode is read only -- MS-149 - Fixed searching only one index file issue in distributed mode -- MS-153 - fix c_str error when connecting to MySQL ## Improvement - MS-82 - Update server startup welcome message @@ -51,7 +49,6 @@ Please mark all change in change log and use the ticket from JIRA. - MS-124 - HasTable interface - MS-126 - Add more error code - MS-128 - Change default db path -- MS-152 - Delete assert in MySQLMetaImpl and change MySQLConnectionPool impl ## New Feature From 33ec9af47b3b33bd4d90271512faaa967a661edc Mon Sep 17 00:00:00 2001 From: zhiru Date: Thu, 4 Jul 2019 17:17:09 +0800 Subject: [PATCH 114/189] fix changelog Former-commit-id: 5f5edf4f22a3458d49dd42f65bb888801322b68e --- cpp/CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/cpp/CHANGELOG.md b/cpp/CHANGELOG.md index 552f0c20f9..8d78dfba99 100644 --- a/cpp/CHANGELOG.md +++ b/cpp/CHANGELOG.md @@ -10,6 +10,7 @@ Please mark all change in change log and use the ticket from JIRA. - MS-148 - Disable cleanup if mode is read only - MS-149 - Fixed searching only one index file issue in distributed mode - MS-153 - fix c_str error when connecting to MySQL +- MS-157 - fix changelog ## Improvement - MS-156 - Add unittest for merge result functions From fc74f013a06f11e535f8dd613fd69320fb8516d4 Mon Sep 17 00:00:00 2001 From: starlord Date: Thu, 4 Jul 2019 16:58:40 +0800 Subject: [PATCH 115/189] add uiittest for merge result functions Former-commit-id: 810194f590afbd4b4625840875e1447b464011b0 --- cpp/CHANGELOG.md | 3 --- cpp/src/db/scheduler/task/SearchTask.cpp | 2 +- cpp/unittest/db/search_test.cpp | 2 -- 3 files changed, 1 insertion(+), 6 deletions(-) diff --git a/cpp/CHANGELOG.md b/cpp/CHANGELOG.md index 8d78dfba99..40d5ef2cd8 100644 --- a/cpp/CHANGELOG.md +++ b/cpp/CHANGELOG.md @@ -14,9 +14,6 @@ Please mark all change in change log and use the ticket from JIRA. ## Improvement - MS-156 - Add unittest for merge result functions - -- MS-152 - Delete assert in MySQLMetaImpl and change MySQLConnectionPool impl - - MS-152 - Delete assert in MySQLMetaImpl and change MySQLConnectionPool impl ## New Feature diff --git a/cpp/src/db/scheduler/task/SearchTask.cpp b/cpp/src/db/scheduler/task/SearchTask.cpp index 8bea768a23..708bcc8708 100644 --- a/cpp/src/db/scheduler/task/SearchTask.cpp +++ b/cpp/src/db/scheduler/task/SearchTask.cpp @@ -67,7 +67,7 @@ std::shared_ptr SearchTask::Execute() { //step 3: cluster result SearchContext::ResultSet result_set; auto spec_k = index_engine_->Count() < context->topk() ? index_engine_->Count() : context->topk(); - ClusterResult(output_ids, output_distence, context->nq(), spec_k, result_set); + SearchTask::ClusterResult(output_ids, output_distence, context->nq(), spec_k, result_set); rc.Record("cluster result"); //step 4: pick up topk result diff --git a/cpp/unittest/db/search_test.cpp b/cpp/unittest/db/search_test.cpp index ce99ea78f7..60d210956f 100644 --- a/cpp/unittest/db/search_test.cpp +++ b/cpp/unittest/db/search_test.cpp @@ -4,9 +4,7 @@ // Proprietary and confidential. //////////////////////////////////////////////////////////////////////////////// #include "db/scheduler/task/SearchTask.h" - #include - #include #include From ba37fb954cac46e45fd6991c7f0ee040e9ee7e60 Mon Sep 17 00:00:00 2001 From: zhiru Date: Fri, 5 Jul 2019 15:03:40 +0800 Subject: [PATCH 116/189] add mem impl Former-commit-id: 36990028d0a289df532e31e53480dbb16ca0558e --- cpp/src/db/MemTable.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/cpp/src/db/MemTable.cpp b/cpp/src/db/MemTable.cpp index e05aa058ac..c88e7f1e04 100644 --- a/cpp/src/db/MemTable.cpp +++ b/cpp/src/db/MemTable.cpp @@ -1,7 +1,6 @@ #include "MemTable.h" #include "Log.h" - namespace zilliz { namespace milvus { namespace engine { From f4b5ae43f6d72f090d4b97a0942392c821b045e6 Mon Sep 17 00:00:00 2001 From: zhiru Date: Fri, 5 Jul 2019 15:57:49 +0800 Subject: [PATCH 117/189] update Former-commit-id: 1a16b8b1d26c704dc72282087207390adee03d0b --- cpp/src/db/MemTableFile.h | 1 - cpp/src/db/VectorSource.h | 1 - 2 files changed, 2 deletions(-) diff --git a/cpp/src/db/MemTableFile.h b/cpp/src/db/MemTableFile.h index 4d0011b362..36d5c1bb25 100644 --- a/cpp/src/db/MemTableFile.h +++ b/cpp/src/db/MemTableFile.h @@ -5,7 +5,6 @@ #include "VectorSource.h" #include "ExecutionEngine.h" - namespace zilliz { namespace milvus { namespace engine { diff --git a/cpp/src/db/VectorSource.h b/cpp/src/db/VectorSource.h index 7092805a6d..9d2abec3cc 100644 --- a/cpp/src/db/VectorSource.h +++ b/cpp/src/db/VectorSource.h @@ -5,7 +5,6 @@ #include "IDGenerator.h" #include "ExecutionEngine.h" - namespace zilliz { namespace milvus { namespace engine { From e96bd671c06b88fe3e8a2358ae9531137b941081 Mon Sep 17 00:00:00 2001 From: zhiru Date: Fri, 5 Jul 2019 16:46:15 +0800 Subject: [PATCH 118/189] Implemented add and serialize Former-commit-id: 8bcf593791b3f04c055132aec838c439dec9c3a0 --- cpp/src/db/MemTableFile.cpp | 26 ++++++++++++++++++++++++++ cpp/src/db/VectorSource.cpp | 1 - cpp/unittest/db/mem_test.cpp | 7 +++---- 3 files changed, 29 insertions(+), 5 deletions(-) diff --git a/cpp/src/db/MemTableFile.cpp b/cpp/src/db/MemTableFile.cpp index 649a680cf3..fb77a6ac3f 100644 --- a/cpp/src/db/MemTableFile.cpp +++ b/cpp/src/db/MemTableFile.cpp @@ -103,6 +103,32 @@ Status MemTableFile::Serialize() { return status; } +Status MemTableFile::Serialize() { + + auto start_time = METRICS_NOW_TIME; + + auto size = GetCurrentMem(); + + execution_engine_->Serialize(); + auto end_time = METRICS_NOW_TIME; + auto total_time = METRICS_MICROSECONDS(start_time, end_time); + table_file_schema_.size_ = size; + + server::Metrics::GetInstance().DiskStoreIOSpeedGaugeSet((double)size/total_time); + + table_file_schema_.file_type_ = (size >= options_.index_trigger_size) ? + meta::TableFileSchema::TO_INDEX : meta::TableFileSchema::RAW; + + auto status = meta_->UpdateTableFile(table_file_schema_); + + LOG(DEBUG) << "New " << ((table_file_schema_.file_type_ == meta::TableFileSchema::RAW) ? "raw" : "to_index") + << " file " << table_file_schema_.file_id_ << " of size " << (double)size / (double)M << " M"; + + execution_engine_->Cache(); + + return status; +} + } // namespace engine } // namespace milvus } // namespace zilliz \ No newline at end of file diff --git a/cpp/src/db/VectorSource.cpp b/cpp/src/db/VectorSource.cpp index 74c07ae1f6..52b3e40342 100644 --- a/cpp/src/db/VectorSource.cpp +++ b/cpp/src/db/VectorSource.cpp @@ -4,7 +4,6 @@ #include "Log.h" #include "metrics/Metrics.h" - namespace zilliz { namespace milvus { namespace engine { diff --git a/cpp/unittest/db/mem_test.cpp b/cpp/unittest/db/mem_test.cpp index 5b7972ec35..6925dba1eb 100644 --- a/cpp/unittest/db/mem_test.cpp +++ b/cpp/unittest/db/mem_test.cpp @@ -146,11 +146,7 @@ TEST_F(NewMemManagerTest, MEM_TABLE_TEST) { engine::VectorSource::Ptr source_100 = std::make_shared(n_100, vectors_100.data()); - engine::MemTable mem_table(TABLE_NAME, impl_, options); - - status = mem_table.Add(source_100); ASSERT_TRUE(status.ok()); - engine::IDNumbers vector_ids = source_100->GetVectorIds(); ASSERT_EQ(vector_ids.size(), 100); @@ -193,6 +189,9 @@ TEST_F(NewMemManagerTest, MEM_TABLE_TEST) { status = mem_table.Serialize(); ASSERT_TRUE(status.ok()); + status = memTable.Serialize(); + ASSERT_TRUE(status.ok()); + status = impl_->DropAll(); ASSERT_TRUE(status.ok()); } From 35cd901c3fc94dd0b549c05976b7126e8ffa1bb1 Mon Sep 17 00:00:00 2001 From: zhiru Date: Sun, 7 Jul 2019 13:50:39 +0800 Subject: [PATCH 119/189] add mem manager Former-commit-id: 1efc6fb4b565c7793a8a25904c3e656c6c36762d --- cpp/src/db/Factories.h | 4 ++++ cpp/src/db/MemManager.h | 1 - cpp/src/db/MemManagerAbstract.h | 1 - cpp/src/db/NewMemManager.h | 1 - cpp/unittest/db/mem_test.cpp | 1 - 5 files changed, 4 insertions(+), 4 deletions(-) diff --git a/cpp/src/db/Factories.h b/cpp/src/db/Factories.h index 8b6e7b100f..aff7f80e0e 100644 --- a/cpp/src/db/Factories.h +++ b/cpp/src/db/Factories.h @@ -42,6 +42,10 @@ struct MemManagerFactory { static MemManagerAbstractPtr Build(const std::shared_ptr &meta, const Options &options); }; +struct MemManagerFactory { + static MemManagerAbstractPtr Build(const std::shared_ptr& meta, const Options& options); +}; + } // namespace engine } // namespace milvus } // namespace zilliz diff --git a/cpp/src/db/MemManager.h b/cpp/src/db/MemManager.h index 5ad3d08b63..0d328bf364 100644 --- a/cpp/src/db/MemManager.h +++ b/cpp/src/db/MemManager.h @@ -17,7 +17,6 @@ #include #include - namespace zilliz { namespace milvus { namespace engine { diff --git a/cpp/src/db/MemManagerAbstract.h b/cpp/src/db/MemManagerAbstract.h index 943c454e46..012858f78c 100644 --- a/cpp/src/db/MemManagerAbstract.h +++ b/cpp/src/db/MemManagerAbstract.h @@ -2,7 +2,6 @@ #include - namespace zilliz { namespace milvus { namespace engine { diff --git a/cpp/src/db/NewMemManager.h b/cpp/src/db/NewMemManager.h index 5b933c94ca..f83af90064 100644 --- a/cpp/src/db/NewMemManager.h +++ b/cpp/src/db/NewMemManager.h @@ -11,7 +11,6 @@ #include #include - namespace zilliz { namespace milvus { namespace engine { diff --git a/cpp/unittest/db/mem_test.cpp b/cpp/unittest/db/mem_test.cpp index 6925dba1eb..6b07a05993 100644 --- a/cpp/unittest/db/mem_test.cpp +++ b/cpp/unittest/db/mem_test.cpp @@ -15,7 +15,6 @@ #include #include - using namespace zilliz::milvus; namespace { From 1cf7999f9625b0ed68e2badb44a24c0c5c428250 Mon Sep 17 00:00:00 2001 From: zhiru Date: Mon, 8 Jul 2019 11:14:28 +0800 Subject: [PATCH 120/189] Add new mem manager Former-commit-id: 577ceb1dbbe4b2aa0565f3363a3ca4d495a4f1b6 --- cpp/CHANGELOG.md | 1 - cpp/src/db/MemManagerAbstract.h | 6 ++++++ cpp/src/db/MemTable.h | 1 - cpp/src/db/NewMemManager.cpp | 2 -- 4 files changed, 6 insertions(+), 4 deletions(-) diff --git a/cpp/CHANGELOG.md b/cpp/CHANGELOG.md index 40d5ef2cd8..99ddda36ca 100644 --- a/cpp/CHANGELOG.md +++ b/cpp/CHANGELOG.md @@ -17,7 +17,6 @@ Please mark all change in change log and use the ticket from JIRA. - MS-152 - Delete assert in MySQLMetaImpl and change MySQLConnectionPool impl ## New Feature -- MS-137 - Integrate knowhere - MS-180 - Add new mem manager ## Task diff --git a/cpp/src/db/MemManagerAbstract.h b/cpp/src/db/MemManagerAbstract.h index 012858f78c..195f0dde70 100644 --- a/cpp/src/db/MemManagerAbstract.h +++ b/cpp/src/db/MemManagerAbstract.h @@ -22,6 +22,12 @@ class MemManagerAbstract { virtual size_t GetCurrentMem() = 0; + virtual size_t GetCurrentMutableMem() = 0; + + virtual size_t GetCurrentImmutableMem() = 0; + + virtual size_t GetCurrentMem() = 0; + }; // MemManagerAbstract using MemManagerAbstractPtr = std::shared_ptr; diff --git a/cpp/src/db/MemTable.h b/cpp/src/db/MemTable.h index 198fcc228a..f973b3f267 100644 --- a/cpp/src/db/MemTable.h +++ b/cpp/src/db/MemTable.h @@ -6,7 +6,6 @@ #include - namespace zilliz { namespace milvus { namespace engine { diff --git a/cpp/src/db/NewMemManager.cpp b/cpp/src/db/NewMemManager.cpp index b0fcc9d4ae..57d44a9031 100644 --- a/cpp/src/db/NewMemManager.cpp +++ b/cpp/src/db/NewMemManager.cpp @@ -5,7 +5,6 @@ #include - namespace zilliz { namespace milvus { namespace engine { @@ -77,7 +76,6 @@ Status NewMemManager::Serialize(std::set &table_ids) { table_ids.insert(mem->GetTableId()); } immu_mem_list_.clear(); - return Status::OK(); } From 9d9e46c3464fb732f885f44bea22363f68cf386f Mon Sep 17 00:00:00 2001 From: zhiru Date: Mon, 8 Jul 2019 15:07:03 +0800 Subject: [PATCH 121/189] update Former-commit-id: 45b443d3533ec64f2a029ccdd8cecb1b38142b9b --- cpp/src/db/Factories.h | 4 ---- cpp/src/db/MemManager.h | 1 + cpp/src/db/MemManagerAbstract.h | 7 +------ cpp/src/db/MemTable.cpp | 1 + cpp/src/db/MemTable.h | 1 + cpp/src/db/MemTableFile.cpp | 26 -------------------------- cpp/src/db/MemTableFile.h | 1 + cpp/src/db/NewMemManager.cpp | 1 + cpp/src/db/NewMemManager.h | 1 + cpp/src/db/VectorSource.cpp | 1 + cpp/src/db/VectorSource.h | 1 + cpp/unittest/db/mem_test.cpp | 7 ++++--- 12 files changed, 13 insertions(+), 39 deletions(-) diff --git a/cpp/src/db/Factories.h b/cpp/src/db/Factories.h index aff7f80e0e..8b6e7b100f 100644 --- a/cpp/src/db/Factories.h +++ b/cpp/src/db/Factories.h @@ -42,10 +42,6 @@ struct MemManagerFactory { static MemManagerAbstractPtr Build(const std::shared_ptr &meta, const Options &options); }; -struct MemManagerFactory { - static MemManagerAbstractPtr Build(const std::shared_ptr& meta, const Options& options); -}; - } // namespace engine } // namespace milvus } // namespace zilliz diff --git a/cpp/src/db/MemManager.h b/cpp/src/db/MemManager.h index 0d328bf364..5ad3d08b63 100644 --- a/cpp/src/db/MemManager.h +++ b/cpp/src/db/MemManager.h @@ -17,6 +17,7 @@ #include #include + namespace zilliz { namespace milvus { namespace engine { diff --git a/cpp/src/db/MemManagerAbstract.h b/cpp/src/db/MemManagerAbstract.h index 195f0dde70..943c454e46 100644 --- a/cpp/src/db/MemManagerAbstract.h +++ b/cpp/src/db/MemManagerAbstract.h @@ -2,6 +2,7 @@ #include + namespace zilliz { namespace milvus { namespace engine { @@ -22,12 +23,6 @@ class MemManagerAbstract { virtual size_t GetCurrentMem() = 0; - virtual size_t GetCurrentMutableMem() = 0; - - virtual size_t GetCurrentImmutableMem() = 0; - - virtual size_t GetCurrentMem() = 0; - }; // MemManagerAbstract using MemManagerAbstractPtr = std::shared_ptr; diff --git a/cpp/src/db/MemTable.cpp b/cpp/src/db/MemTable.cpp index c88e7f1e04..e05aa058ac 100644 --- a/cpp/src/db/MemTable.cpp +++ b/cpp/src/db/MemTable.cpp @@ -1,6 +1,7 @@ #include "MemTable.h" #include "Log.h" + namespace zilliz { namespace milvus { namespace engine { diff --git a/cpp/src/db/MemTable.h b/cpp/src/db/MemTable.h index f973b3f267..198fcc228a 100644 --- a/cpp/src/db/MemTable.h +++ b/cpp/src/db/MemTable.h @@ -6,6 +6,7 @@ #include + namespace zilliz { namespace milvus { namespace engine { diff --git a/cpp/src/db/MemTableFile.cpp b/cpp/src/db/MemTableFile.cpp index fb77a6ac3f..649a680cf3 100644 --- a/cpp/src/db/MemTableFile.cpp +++ b/cpp/src/db/MemTableFile.cpp @@ -103,32 +103,6 @@ Status MemTableFile::Serialize() { return status; } -Status MemTableFile::Serialize() { - - auto start_time = METRICS_NOW_TIME; - - auto size = GetCurrentMem(); - - execution_engine_->Serialize(); - auto end_time = METRICS_NOW_TIME; - auto total_time = METRICS_MICROSECONDS(start_time, end_time); - table_file_schema_.size_ = size; - - server::Metrics::GetInstance().DiskStoreIOSpeedGaugeSet((double)size/total_time); - - table_file_schema_.file_type_ = (size >= options_.index_trigger_size) ? - meta::TableFileSchema::TO_INDEX : meta::TableFileSchema::RAW; - - auto status = meta_->UpdateTableFile(table_file_schema_); - - LOG(DEBUG) << "New " << ((table_file_schema_.file_type_ == meta::TableFileSchema::RAW) ? "raw" : "to_index") - << " file " << table_file_schema_.file_id_ << " of size " << (double)size / (double)M << " M"; - - execution_engine_->Cache(); - - return status; -} - } // namespace engine } // namespace milvus } // namespace zilliz \ No newline at end of file diff --git a/cpp/src/db/MemTableFile.h b/cpp/src/db/MemTableFile.h index 36d5c1bb25..4d0011b362 100644 --- a/cpp/src/db/MemTableFile.h +++ b/cpp/src/db/MemTableFile.h @@ -5,6 +5,7 @@ #include "VectorSource.h" #include "ExecutionEngine.h" + namespace zilliz { namespace milvus { namespace engine { diff --git a/cpp/src/db/NewMemManager.cpp b/cpp/src/db/NewMemManager.cpp index 57d44a9031..e4903d75a9 100644 --- a/cpp/src/db/NewMemManager.cpp +++ b/cpp/src/db/NewMemManager.cpp @@ -5,6 +5,7 @@ #include + namespace zilliz { namespace milvus { namespace engine { diff --git a/cpp/src/db/NewMemManager.h b/cpp/src/db/NewMemManager.h index f83af90064..5b933c94ca 100644 --- a/cpp/src/db/NewMemManager.h +++ b/cpp/src/db/NewMemManager.h @@ -11,6 +11,7 @@ #include #include + namespace zilliz { namespace milvus { namespace engine { diff --git a/cpp/src/db/VectorSource.cpp b/cpp/src/db/VectorSource.cpp index 52b3e40342..74c07ae1f6 100644 --- a/cpp/src/db/VectorSource.cpp +++ b/cpp/src/db/VectorSource.cpp @@ -4,6 +4,7 @@ #include "Log.h" #include "metrics/Metrics.h" + namespace zilliz { namespace milvus { namespace engine { diff --git a/cpp/src/db/VectorSource.h b/cpp/src/db/VectorSource.h index 9d2abec3cc..7092805a6d 100644 --- a/cpp/src/db/VectorSource.h +++ b/cpp/src/db/VectorSource.h @@ -5,6 +5,7 @@ #include "IDGenerator.h" #include "ExecutionEngine.h" + namespace zilliz { namespace milvus { namespace engine { diff --git a/cpp/unittest/db/mem_test.cpp b/cpp/unittest/db/mem_test.cpp index 6b07a05993..860a665386 100644 --- a/cpp/unittest/db/mem_test.cpp +++ b/cpp/unittest/db/mem_test.cpp @@ -15,6 +15,7 @@ #include #include + using namespace zilliz::milvus; namespace { @@ -145,6 +146,9 @@ TEST_F(NewMemManagerTest, MEM_TABLE_TEST) { engine::VectorSource::Ptr source_100 = std::make_shared(n_100, vectors_100.data()); + engine::MemTable mem_table(TABLE_NAME, impl_, options); + + status = mem_table.Add(source_100); ASSERT_TRUE(status.ok()); engine::IDNumbers vector_ids = source_100->GetVectorIds(); ASSERT_EQ(vector_ids.size(), 100); @@ -188,9 +192,6 @@ TEST_F(NewMemManagerTest, MEM_TABLE_TEST) { status = mem_table.Serialize(); ASSERT_TRUE(status.ok()); - status = memTable.Serialize(); - ASSERT_TRUE(status.ok()); - status = impl_->DropAll(); ASSERT_TRUE(status.ok()); } From c7f3061f57eb029b0c9ba28cdd5dd299ec5bf5a2 Mon Sep 17 00:00:00 2001 From: quicksilver Date: Tue, 9 Jul 2019 13:47:25 +0800 Subject: [PATCH 122/189] configure git global user information Former-commit-id: e384c419f60a4470846ace909f5107cc90c2a70c --- ci/jenkinsfile/milvus_build.groovy | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ci/jenkinsfile/milvus_build.groovy b/ci/jenkinsfile/milvus_build.groovy index ed07d2b992..243b6dc2da 100644 --- a/ci/jenkinsfile/milvus_build.groovy +++ b/ci/jenkinsfile/milvus_build.groovy @@ -5,6 +5,8 @@ container('milvus-build-env') { try { checkout([$class: 'GitSCM', branches: [[name: "${SEMVER}"]], doGenerateSubmoduleConfigurations: false, extensions: [[$class: 'SubmoduleOption',disableSubmodules: false,parentCredentials: true,recursiveSubmodules: true,reference: '',trackingSubmodules: false]], submoduleCfg: [], userRemoteConfigs: [[credentialsId: "${params.GIT_USER}", url: "git@192.168.1.105:megasearch/milvus.git"]]]) dir ("cpp") { + sh "git config --global user.email \"test@zilliz.com\"" + sh "git config --global user.name \"test\"" sh "./build.sh -t ${params.BUILD_TYPE} -u -c" } } catch (exc) { From 8c1bffa68e04b334c1f256ab36a3effa0820a729 Mon Sep 17 00:00:00 2001 From: quicksilver Date: Tue, 9 Jul 2019 14:30:00 +0800 Subject: [PATCH 123/189] add nightly_main_jenkinsfile Former-commit-id: 7f28950ed13acdcdba5903d94302aacc18d6809a --- ci/nightly_main_jenkinsfile | 256 ++++++++++++++++++++++++++++++++++++ 1 file changed, 256 insertions(+) create mode 100644 ci/nightly_main_jenkinsfile diff --git a/ci/nightly_main_jenkinsfile b/ci/nightly_main_jenkinsfile new file mode 100644 index 0000000000..9c54b6b8c0 --- /dev/null +++ b/ci/nightly_main_jenkinsfile @@ -0,0 +1,256 @@ +pipeline { + agent none + + options { + timestamps() + } + + environment { + PROJECT_NAME = "milvus" + LOWER_BUILD_TYPE = BUILD_TYPE.toLowerCase() + SEMVER = "${env.gitlabSourceBranch == null ? params.ENGINE_BRANCH.substring(params.ENGINE_BRANCH.lastIndexOf('/') + 1) : env.gitlabSourceBranch}" + GITLAB_AFTER_COMMIT = "${env.gitlabAfter == null ? null : env.gitlabAfter}" + SUFFIX_VERSION_NAME = "${env.gitlabAfter == null ? null : env.gitlabAfter.substring(0, 6)}" + DOCKER_VERSION_STR = "${env.gitlabAfter == null ? "${SEMVER}-${LOWER_BUILD_TYPE}-\$\{BUILD_DATE_FORMATTED\, \"yyyyMMdd\"\}" : "${SEMVER}-${LOWER_BUILD_TYPE}-${SUFFIX_VERSION_NAME}"}" + } + + stages { + stage("Ubuntu 16.04") { + environment { + PACKAGE_VERSION = VersionNumber([ + versionNumberString : '${SEMVER}-${LOWER_BUILD_TYPE}-${BUILD_DATE_FORMATTED, "yyyyMMdd"}' + ]); + + DOCKER_VERSION = VersionNumber([ + versionNumberString : '${DOCKER_VERSION_STR}' + ]); + } + + stages { + stage("Run Build") { + agent { + kubernetes { + cloud 'build-kubernetes' + label 'build' + defaultContainer 'jnlp' + containerTemplate { + name 'milvus-build-env' + image 'registry.zilliz.com/milvus/milvus-build-env:v0.10' + ttyEnabled true + command 'cat' + } + } + } + stages { + stage('Build') { + steps { + gitlabCommitStatus(name: 'Build') { + script { + load "${env.WORKSPACE}/ci/jenkinsfile/milvus_build.groovy" + load "${env.WORKSPACE}/ci/jenkinsfile/packaged_milvus.groovy" + } + } + } + } + } + post { + aborted { + script { + updateGitlabCommitStatus name: 'Build', state: 'canceled' + echo "Milvus Build aborted !" + } + } + + failure { + script { + updateGitlabCommitStatus name: 'Build', state: 'failed' + echo "Milvus Build failure !" + } + } + } + } + + stage("Publish docker and helm") { + agent { + kubernetes { + label 'publish' + defaultContainer 'jnlp' + yaml """ +apiVersion: v1 +kind: Pod +metadata: + labels: + app: publish + componet: docker +spec: + containers: + - name: publish-docker + image: registry.zilliz.com/library/zilliz_docker:v1.0.0 + securityContext: + privileged: true + command: + - cat + tty: true + volumeMounts: + - name: docker-sock + mountPath: /var/run/docker.sock + volumes: + - name: docker-sock + hostPath: + path: /var/run/docker.sock +""" + } + } + stages { + stage('Publish Docker') { + steps { + gitlabCommitStatus(name: 'Publish Docker') { + script { + load "${env.WORKSPACE}/ci/jenkinsfile/publish_docker.groovy" + } + } + } + } + } + post { + aborted { + script { + updateGitlabCommitStatus name: 'Publish Docker', state: 'canceled' + echo "Milvus Publish Docker aborted !" + } + } + + failure { + script { + updateGitlabCommitStatus name: 'Publish Docker', state: 'failed' + echo "Milvus Publish Docker failure !" + } + } + } + } + + stage("Deploy to Development") { + stages { + stage("Deploy to Dev") { + agent { + kubernetes { + label 'jenkins-slave' + defaultContainer 'jnlp' + } + } + stages { + stage('Deploy') { + steps { + gitlabCommitStatus(name: 'Deloy to Dev') { + script { + load "${env.WORKSPACE}/ci/jenkinsfile/deploy2dev.groovy" + } + } + } + } + } + post { + aborted { + script { + updateGitlabCommitStatus name: 'Deloy to Dev', state: 'canceled' + echo "Milvus Deloy to Dev aborted !" + } + } + + failure { + script { + updateGitlabCommitStatus name: 'Deloy to Dev', state: 'failed' + echo "Milvus Deloy to Dev failure !" + } + } + } + } + + stage("Dev Test") { + agent { + kubernetes { + label 'test' + defaultContainer 'jnlp' + containerTemplate { + name 'milvus-testframework' + image 'registry.zilliz.com/milvus/milvus-test:v0.1' + ttyEnabled true + command 'cat' + } + } + } + stages { + stage('Test') { + steps { + script { + load "${env.WORKSPACE}/ci/jenkinsfile/dev_test.groovy" + load "${env.WORKSPACE}/ci/jenkinsfile/upload_dev_test_out.groovy" + } + } + } + } + } + + stage ("Cleanup Dev") { + agent { + kubernetes { + label 'jenkins-slave' + defaultContainer 'jnlp' + } + } + stages { + stage('Cleanup') { + steps { + gitlabCommitStatus(name: 'Cleanup Dev') { + script { + load "${env.WORKSPACE}/ci/jenkinsfile/cleanup_dev.groovy" + } + } + } + } + } + post { + aborted { + script { + updateGitlabCommitStatus name: 'Cleanup Dev', state: 'canceled' + echo "Milvus Cleanup Dev aborted !" + } + } + + failure { + script { + updateGitlabCommitStatus name: 'Cleanup Dev', state: 'failed' + echo "Milvus Cleanup Dev failure !" + } + } + } + } + } + } + } + } + } + + post { + success { + script { + updateGitlabCommitStatus name: 'CI/CD', state: 'success' + echo "Milvus CI/CD success !" + } + } + + aborted { + script { + updateGitlabCommitStatus name: 'CI/CD', state: 'canceled' + echo "Milvus CI/CD aborted !" + } + } + + failure { + script { + updateGitlabCommitStatus name: 'CI/CD', state: 'failed' + echo "Milvus CI/CD failure !" + } + } + } +} From 9e59ead85c3f47968ba32f166e38ad057fc81ee8 Mon Sep 17 00:00:00 2001 From: quicksilver Date: Tue, 9 Jul 2019 15:06:26 +0800 Subject: [PATCH 124/189] fix nightly_main_jenkinsfile docker version bug Former-commit-id: 13df60a581ffd52cdafbcc41d3de04d6656348c9 --- ci/nightly_main_jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/nightly_main_jenkinsfile b/ci/nightly_main_jenkinsfile index 9c54b6b8c0..eff34a5de2 100644 --- a/ci/nightly_main_jenkinsfile +++ b/ci/nightly_main_jenkinsfile @@ -11,7 +11,7 @@ pipeline { SEMVER = "${env.gitlabSourceBranch == null ? params.ENGINE_BRANCH.substring(params.ENGINE_BRANCH.lastIndexOf('/') + 1) : env.gitlabSourceBranch}" GITLAB_AFTER_COMMIT = "${env.gitlabAfter == null ? null : env.gitlabAfter}" SUFFIX_VERSION_NAME = "${env.gitlabAfter == null ? null : env.gitlabAfter.substring(0, 6)}" - DOCKER_VERSION_STR = "${env.gitlabAfter == null ? "${SEMVER}-${LOWER_BUILD_TYPE}-\$\{BUILD_DATE_FORMATTED\, \"yyyyMMdd\"\}" : "${SEMVER}-${LOWER_BUILD_TYPE}-${SUFFIX_VERSION_NAME}"}" + DOCKER_VERSION_STR = "${env.gitlabAfter == null ? '${SEMVER}-${LOWER_BUILD_TYPE}-${BUILD_DATE_FORMATTED, \"yyyyMMdd\"}' : '${SEMVER}-${LOWER_BUILD_TYPE}-${SUFFIX_VERSION_NAME}'}" } stages { From b660b49cb5be2cdd6af4485bcb2a5d1aac0fcfba Mon Sep 17 00:00:00 2001 From: zhiru Date: Tue, 9 Jul 2019 17:07:52 +0800 Subject: [PATCH 125/189] use env variable to switch mem manager and fix cmake Former-commit-id: 30fa1d2ca1721631298ea3233841cc15f45f87f7 --- cpp/src/db/Factories.cpp | 22 ++++++++++++++++------ cpp/unittest/metrics/CMakeLists.txt | 2 +- 2 files changed, 17 insertions(+), 7 deletions(-) diff --git a/cpp/src/db/Factories.cpp b/cpp/src/db/Factories.cpp index 65c7484a50..3cfb9a6eb3 100644 --- a/cpp/src/db/Factories.cpp +++ b/cpp/src/db/Factories.cpp @@ -3,12 +3,14 @@ // Unauthorized copying of this file, via any medium is strictly prohibited. // Proprietary and confidential. //////////////////////////////////////////////////////////////////////////////// -#include + #include "Factories.h" #include "DBImpl.h" #include "MemManager.h" #include "NewMemManager.h" +#include "Exception.h" +#include #include #include #include @@ -16,7 +18,9 @@ #include #include #include -#include "Exception.h" +#include +#include +#include namespace zilliz { namespace milvus { @@ -101,11 +105,17 @@ DB* DBFactory::Build(const Options& options) { MemManagerAbstractPtr MemManagerFactory::Build(const std::shared_ptr& meta, const Options& options) { -#ifdef USE_NEW_MEM_MANAGER + if (const char* env = getenv("MILVUS_USE_OLD_MEM_MANAGER")) { + std::string env_str = env; + std::transform(env_str.begin(), env_str.end(), env_str.begin(), ::toupper); + if (env_str == "ON") { + return std::make_shared(meta, options); + } + else { + return std::make_shared(meta, options); + } + } return std::make_shared(meta, options); -#else - return std::make_shared(meta, options); -#endif } } // namespace engine diff --git a/cpp/unittest/metrics/CMakeLists.txt b/cpp/unittest/metrics/CMakeLists.txt index d31e44c056..f5839cafe9 100644 --- a/cpp/unittest/metrics/CMakeLists.txt +++ b/cpp/unittest/metrics/CMakeLists.txt @@ -66,7 +66,7 @@ target_link_libraries(metrics_test faiss cudart cublas - sqlite3 + sqlite boost_system boost_filesystem lz4 From 661d1127aa68fb9f156a92d7a780243497d515c8 Mon Sep 17 00:00:00 2001 From: zhiru Date: Tue, 9 Jul 2019 17:09:22 +0800 Subject: [PATCH 126/189] use env variable to switch mem manager and fix cmake Former-commit-id: 887e191a29d7b88dc646f2d59628e1ca7693841a --- cpp/CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/cpp/CHANGELOG.md b/cpp/CHANGELOG.md index 99ddda36ca..40b8dbbf0b 100644 --- a/cpp/CHANGELOG.md +++ b/cpp/CHANGELOG.md @@ -11,6 +11,7 @@ Please mark all change in change log and use the ticket from JIRA. - MS-149 - Fixed searching only one index file issue in distributed mode - MS-153 - fix c_str error when connecting to MySQL - MS-157 - fix changelog +- MS-190 - use env variable to switch mem manager and fix cmake ## Improvement - MS-156 - Add unittest for merge result functions From c7d0f49637561588fd5e5ea823fcc746724bdc08 Mon Sep 17 00:00:00 2001 From: zhiru Date: Tue, 9 Jul 2019 17:31:02 +0800 Subject: [PATCH 127/189] fix cmake Former-commit-id: 70f81e37a82ca2b6e7280d95e65e6c8eb541a577 --- cpp/unittest/db/CMakeLists.txt | 6 +++--- cpp/unittest/faiss_wrapper/CMakeLists.txt | 6 +++--- cpp/unittest/license/CMakeLists.txt | 6 +++--- cpp/unittest/metrics/CMakeLists.txt | 4 ++-- cpp/unittest/server/CMakeLists.txt | 6 +++--- cpp/unittest/storage/CMakeLists.txt | 2 +- cpp/unittest/utils/CMakeLists.txt | 2 +- 7 files changed, 16 insertions(+), 16 deletions(-) diff --git a/cpp/unittest/db/CMakeLists.txt b/cpp/unittest/db/CMakeLists.txt index 5bae9190f5..e893ed70ad 100644 --- a/cpp/unittest/db/CMakeLists.txt +++ b/cpp/unittest/db/CMakeLists.txt @@ -40,9 +40,9 @@ set(db_libs faiss cudart cublas - sqlite3 - boost_system - boost_filesystem + sqlite + boost_system_static + boost_filesystem_static lz4 mysqlpp ) diff --git a/cpp/unittest/faiss_wrapper/CMakeLists.txt b/cpp/unittest/faiss_wrapper/CMakeLists.txt index f044df8d8c..ff2535e3fd 100644 --- a/cpp/unittest/faiss_wrapper/CMakeLists.txt +++ b/cpp/unittest/faiss_wrapper/CMakeLists.txt @@ -22,13 +22,13 @@ add_executable(wrapper_test ${wrapper_test_src}) set(wrapper_libs stdc++ - boost_system - boost_filesystem + boost_system_static + boost_filesystem_static libgpufaiss.a faiss cudart cublas - sqlite3 + sqlite snappy bz2 z diff --git a/cpp/unittest/license/CMakeLists.txt b/cpp/unittest/license/CMakeLists.txt index 93af82ed56..39d0ede197 100644 --- a/cpp/unittest/license/CMakeLists.txt +++ b/cpp/unittest/license/CMakeLists.txt @@ -33,11 +33,11 @@ set(db_libs nvidia-ml cudart cublas - boost_system - boost_filesystem + boost_system_static + boost_filesystem_static lz4 crypto - boost_serialization + boost_serialization_static ) target_link_libraries(license_test ${db_libs} ${unittest_libs}) diff --git a/cpp/unittest/metrics/CMakeLists.txt b/cpp/unittest/metrics/CMakeLists.txt index f5839cafe9..513b37b8c2 100644 --- a/cpp/unittest/metrics/CMakeLists.txt +++ b/cpp/unittest/metrics/CMakeLists.txt @@ -67,8 +67,8 @@ target_link_libraries(metrics_test cudart cublas sqlite - boost_system - boost_filesystem + boost_system_static + boost_filesystem_static lz4 metrics gtest diff --git a/cpp/unittest/server/CMakeLists.txt b/cpp/unittest/server/CMakeLists.txt index c4112cda9e..e49aed500c 100644 --- a/cpp/unittest/server/CMakeLists.txt +++ b/cpp/unittest/server/CMakeLists.txt @@ -37,9 +37,9 @@ set(require_libs faiss cudart cublas - sqlite3 - boost_system - boost_filesystem + sqlite + boost_system_static + boost_filesystem_static snappy z bz2 diff --git a/cpp/unittest/storage/CMakeLists.txt b/cpp/unittest/storage/CMakeLists.txt index 6b4303b70a..d4deaefab8 100644 --- a/cpp/unittest/storage/CMakeLists.txt +++ b/cpp/unittest/storage/CMakeLists.txt @@ -25,7 +25,7 @@ set(s3_client_libs stdc++ aws-cpp-sdk-s3 aws-cpp-sdk-core - boost_filesystem + boost_filesystem_static ) target_link_libraries(s3_test ${s3_client_libs} diff --git a/cpp/unittest/utils/CMakeLists.txt b/cpp/unittest/utils/CMakeLists.txt index a46a3b05e1..61e4459462 100644 --- a/cpp/unittest/utils/CMakeLists.txt +++ b/cpp/unittest/utils/CMakeLists.txt @@ -25,6 +25,6 @@ add_executable(valication_util_test target_link_libraries(valication_util_test ${unittest_libs} - boost_filesystem) + boost_filesystem_static) install(TARGETS valication_util_test DESTINATION bin) \ No newline at end of file From 9d80667a724dd87d8c69b9e9a4000b1dc34f4264 Mon Sep 17 00:00:00 2001 From: zhiru Date: Tue, 9 Jul 2019 17:32:05 +0800 Subject: [PATCH 128/189] update Former-commit-id: 142163d37af3a3aab8aa3466f37651d3315ddef5 --- cpp/src/db/Factories.cpp | 2 -- 1 file changed, 2 deletions(-) diff --git a/cpp/src/db/Factories.cpp b/cpp/src/db/Factories.cpp index 3cfb9a6eb3..442dca2974 100644 --- a/cpp/src/db/Factories.cpp +++ b/cpp/src/db/Factories.cpp @@ -26,8 +26,6 @@ namespace zilliz { namespace milvus { namespace engine { -#define USE_NEW_MEM_MANAGER 1 - DBMetaOptions DBMetaOptionsFactory::Build(const std::string& path) { auto p = path; if(p == "") { From 034d8a6403b3aa19b3aba647afc30924417a4b74 Mon Sep 17 00:00:00 2001 From: zhiru Date: Tue, 9 Jul 2019 21:11:05 +0800 Subject: [PATCH 129/189] update Former-commit-id: f2172ed1070f1be0fa525781fb1a70e230ea8258 --- cpp/cmake/ThirdPartyPackages.cmake | 2 +- cpp/unittest/db/CMakeLists.txt | 4 +- cpp/unittest/db/mem_test.cpp | 160 ++++++++++++++--------------- 3 files changed, 84 insertions(+), 82 deletions(-) diff --git a/cpp/cmake/ThirdPartyPackages.cmake b/cpp/cmake/ThirdPartyPackages.cmake index 9aa3f62124..9e19b037f8 100644 --- a/cpp/cmake/ThirdPartyPackages.cmake +++ b/cpp/cmake/ThirdPartyPackages.cmake @@ -525,7 +525,7 @@ if(MILVUS_BOOST_VENDORED) "" ${EP_LOG_OPTIONS}) set(Boost_INCLUDE_DIR "${BOOST_PREFIX}") - set(Boost_INCLUDE_DIRS "${BOOST_INCLUDE_DIR}") + set(Boost_INCLUDE_DIRS "${Boost_INCLUDE_DIR}") add_dependencies(boost_system_static boost_ep) add_dependencies(boost_filesystem_static boost_ep) add_dependencies(boost_serialization_static boost_ep) diff --git a/cpp/unittest/db/CMakeLists.txt b/cpp/unittest/db/CMakeLists.txt index e893ed70ad..460159227b 100644 --- a/cpp/unittest/db/CMakeLists.txt +++ b/cpp/unittest/db/CMakeLists.txt @@ -23,6 +23,8 @@ link_directories("/usr/local/cuda/lib64") include_directories(/usr/include/mysql) +#add_definitions(-DBOOST_ERROR_CODE_HEADER_ONLY) + set(db_test_src #${unittest_srcs} ${config_files} @@ -41,8 +43,8 @@ set(db_libs cudart cublas sqlite - boost_system_static boost_filesystem_static + boost_system_static lz4 mysqlpp ) diff --git a/cpp/unittest/db/mem_test.cpp b/cpp/unittest/db/mem_test.cpp index 860a665386..f0e79d4d34 100644 --- a/cpp/unittest/db/mem_test.cpp +++ b/cpp/unittest/db/mem_test.cpp @@ -288,84 +288,84 @@ TEST_F(NewMemManagerTest, INSERT_TEST) { } -TEST_F(NewMemManagerTest, CONCURRENT_INSERT_SEARCH_TEST) { - - auto options = engine::OptionsFactory::Build(); - options.meta.path = "/tmp/milvus_test"; - options.meta.backend_uri = "sqlite://:@:/"; - auto db_ = engine::DBFactory::Build(options); - - engine::meta::TableSchema table_info = BuildTableSchema(); - engine::Status stat = db_->CreateTable(table_info); - - engine::meta::TableSchema table_info_get; - table_info_get.table_id_ = TABLE_NAME; - stat = db_->DescribeTable(table_info_get); - ASSERT_STATS(stat); - ASSERT_EQ(table_info_get.dimension_, TABLE_DIM); - - engine::IDNumbers vector_ids; - engine::IDNumbers target_ids; - - int64_t nb = 409600; - std::vector xb; - BuildVectors(nb, xb); - - int64_t qb = 5; - std::vector qxb; - BuildVectors(qb, qxb); - - std::thread search([&]() { - engine::QueryResults results; - int k = 10; - std::this_thread::sleep_for(std::chrono::seconds(2)); - - INIT_TIMER; - std::stringstream ss; - uint64_t count = 0; - uint64_t prev_count = 0; - - for (auto j = 0; j < 10; ++j) { - ss.str(""); - db_->Size(count); - prev_count = count; - - START_TIMER; - stat = db_->Query(TABLE_NAME, k, qb, qxb.data(), results); - ss << "Search " << j << " With Size " << count / engine::meta::M << " M"; - STOP_TIMER(ss.str()); - - ASSERT_STATS(stat); - for (auto k = 0; k < qb; ++k) { - ASSERT_EQ(results[k][0].first, target_ids[k]); - ss.str(""); - ss << "Result [" << k << "]:"; - for (auto result : results[k]) { - ss << result.first << " "; - } - /* LOG(DEBUG) << ss.str(); */ - } - ASSERT_TRUE(count >= prev_count); - std::this_thread::sleep_for(std::chrono::seconds(1)); - } - }); - - int loop = 20; - - for (auto i = 0; i < loop; ++i) { - if (i == 0) { - db_->InsertVectors(TABLE_NAME, qb, qxb.data(), target_ids); - ASSERT_EQ(target_ids.size(), qb); - } else { - db_->InsertVectors(TABLE_NAME, nb, xb.data(), vector_ids); - } - std::this_thread::sleep_for(std::chrono::microseconds(1)); - } - - search.join(); - - delete db_; - boost::filesystem::remove_all(options.meta.path); - -}; +//TEST_F(NewMemManagerTest, CONCURRENT_INSERT_SEARCH_TEST) { +// +// auto options = engine::OptionsFactory::Build(); +// options.meta.path = "/tmp/milvus_test"; +// options.meta.backend_uri = "sqlite://:@:/"; +// auto db_ = engine::DBFactory::Build(options); +// +// engine::meta::TableSchema table_info = BuildTableSchema(); +// engine::Status stat = db_->CreateTable(table_info); +// +// engine::meta::TableSchema table_info_get; +// table_info_get.table_id_ = TABLE_NAME; +// stat = db_->DescribeTable(table_info_get); +// ASSERT_STATS(stat); +// ASSERT_EQ(table_info_get.dimension_, TABLE_DIM); +// +// engine::IDNumbers vector_ids; +// engine::IDNumbers target_ids; +// +// int64_t nb = 409600; +// std::vector xb; +// BuildVectors(nb, xb); +// +// int64_t qb = 5; +// std::vector qxb; +// BuildVectors(qb, qxb); +// +// std::thread search([&]() { +// engine::QueryResults results; +// int k = 10; +// std::this_thread::sleep_for(std::chrono::seconds(5)); +// +// INIT_TIMER; +// std::stringstream ss; +// uint64_t count = 0; +// uint64_t prev_count = 0; +// +// for (auto j = 0; j < 10; ++j) { +// ss.str(""); +// db_->Size(count); +// prev_count = count; +// +// START_TIMER; +// stat = db_->Query(TABLE_NAME, k, qb, qxb.data(), results); +// ss << "Search " << j << " With Size " << count / engine::meta::M << " M"; +// STOP_TIMER(ss.str()); +// +// ASSERT_STATS(stat); +// for (auto k = 0; k < qb; ++k) { +// ASSERT_EQ(results[k][0].first, target_ids[k]); +// ss.str(""); +// ss << "Result [" << k << "]:"; +// for (auto result : results[k]) { +// ss << result.first << " "; +// } +// /* LOG(DEBUG) << ss.str(); */ +// } +// ASSERT_TRUE(count >= prev_count); +// std::this_thread::sleep_for(std::chrono::seconds(2)); +// } +// }); +// +// int loop = 20; +// +// for (auto i = 0; i < loop; ++i) { +// if (i == 0) { +// db_->InsertVectors(TABLE_NAME, qb, qxb.data(), target_ids); +// ASSERT_EQ(target_ids.size(), qb); +// } else { +// db_->InsertVectors(TABLE_NAME, nb, xb.data(), vector_ids); +// } +// std::this_thread::sleep_for(std::chrono::microseconds(1)); +// } +// +// search.join(); +// +// delete db_; +// boost::filesystem::remove_all(options.meta.path); +// +//}; From 8a1af93518b086536f8f38afff45253df8507b44 Mon Sep 17 00:00:00 2001 From: zhiru Date: Tue, 9 Jul 2019 21:13:55 +0800 Subject: [PATCH 130/189] update Former-commit-id: 3bda5a0b65c0ddf4bce76685e708ea90c951ad8f --- cpp/unittest/db/mem_test.cpp | 160 +++++++++++++++++------------------ 1 file changed, 80 insertions(+), 80 deletions(-) diff --git a/cpp/unittest/db/mem_test.cpp b/cpp/unittest/db/mem_test.cpp index f0e79d4d34..860a665386 100644 --- a/cpp/unittest/db/mem_test.cpp +++ b/cpp/unittest/db/mem_test.cpp @@ -288,84 +288,84 @@ TEST_F(NewMemManagerTest, INSERT_TEST) { } -//TEST_F(NewMemManagerTest, CONCURRENT_INSERT_SEARCH_TEST) { -// -// auto options = engine::OptionsFactory::Build(); -// options.meta.path = "/tmp/milvus_test"; -// options.meta.backend_uri = "sqlite://:@:/"; -// auto db_ = engine::DBFactory::Build(options); -// -// engine::meta::TableSchema table_info = BuildTableSchema(); -// engine::Status stat = db_->CreateTable(table_info); -// -// engine::meta::TableSchema table_info_get; -// table_info_get.table_id_ = TABLE_NAME; -// stat = db_->DescribeTable(table_info_get); -// ASSERT_STATS(stat); -// ASSERT_EQ(table_info_get.dimension_, TABLE_DIM); -// -// engine::IDNumbers vector_ids; -// engine::IDNumbers target_ids; -// -// int64_t nb = 409600; -// std::vector xb; -// BuildVectors(nb, xb); -// -// int64_t qb = 5; -// std::vector qxb; -// BuildVectors(qb, qxb); -// -// std::thread search([&]() { -// engine::QueryResults results; -// int k = 10; -// std::this_thread::sleep_for(std::chrono::seconds(5)); -// -// INIT_TIMER; -// std::stringstream ss; -// uint64_t count = 0; -// uint64_t prev_count = 0; -// -// for (auto j = 0; j < 10; ++j) { -// ss.str(""); -// db_->Size(count); -// prev_count = count; -// -// START_TIMER; -// stat = db_->Query(TABLE_NAME, k, qb, qxb.data(), results); -// ss << "Search " << j << " With Size " << count / engine::meta::M << " M"; -// STOP_TIMER(ss.str()); -// -// ASSERT_STATS(stat); -// for (auto k = 0; k < qb; ++k) { -// ASSERT_EQ(results[k][0].first, target_ids[k]); -// ss.str(""); -// ss << "Result [" << k << "]:"; -// for (auto result : results[k]) { -// ss << result.first << " "; -// } -// /* LOG(DEBUG) << ss.str(); */ -// } -// ASSERT_TRUE(count >= prev_count); -// std::this_thread::sleep_for(std::chrono::seconds(2)); -// } -// }); -// -// int loop = 20; -// -// for (auto i = 0; i < loop; ++i) { -// if (i == 0) { -// db_->InsertVectors(TABLE_NAME, qb, qxb.data(), target_ids); -// ASSERT_EQ(target_ids.size(), qb); -// } else { -// db_->InsertVectors(TABLE_NAME, nb, xb.data(), vector_ids); -// } -// std::this_thread::sleep_for(std::chrono::microseconds(1)); -// } -// -// search.join(); -// -// delete db_; -// boost::filesystem::remove_all(options.meta.path); -// -//}; +TEST_F(NewMemManagerTest, CONCURRENT_INSERT_SEARCH_TEST) { + + auto options = engine::OptionsFactory::Build(); + options.meta.path = "/tmp/milvus_test"; + options.meta.backend_uri = "sqlite://:@:/"; + auto db_ = engine::DBFactory::Build(options); + + engine::meta::TableSchema table_info = BuildTableSchema(); + engine::Status stat = db_->CreateTable(table_info); + + engine::meta::TableSchema table_info_get; + table_info_get.table_id_ = TABLE_NAME; + stat = db_->DescribeTable(table_info_get); + ASSERT_STATS(stat); + ASSERT_EQ(table_info_get.dimension_, TABLE_DIM); + + engine::IDNumbers vector_ids; + engine::IDNumbers target_ids; + + int64_t nb = 409600; + std::vector xb; + BuildVectors(nb, xb); + + int64_t qb = 5; + std::vector qxb; + BuildVectors(qb, qxb); + + std::thread search([&]() { + engine::QueryResults results; + int k = 10; + std::this_thread::sleep_for(std::chrono::seconds(2)); + + INIT_TIMER; + std::stringstream ss; + uint64_t count = 0; + uint64_t prev_count = 0; + + for (auto j = 0; j < 10; ++j) { + ss.str(""); + db_->Size(count); + prev_count = count; + + START_TIMER; + stat = db_->Query(TABLE_NAME, k, qb, qxb.data(), results); + ss << "Search " << j << " With Size " << count / engine::meta::M << " M"; + STOP_TIMER(ss.str()); + + ASSERT_STATS(stat); + for (auto k = 0; k < qb; ++k) { + ASSERT_EQ(results[k][0].first, target_ids[k]); + ss.str(""); + ss << "Result [" << k << "]:"; + for (auto result : results[k]) { + ss << result.first << " "; + } + /* LOG(DEBUG) << ss.str(); */ + } + ASSERT_TRUE(count >= prev_count); + std::this_thread::sleep_for(std::chrono::seconds(1)); + } + }); + + int loop = 20; + + for (auto i = 0; i < loop; ++i) { + if (i == 0) { + db_->InsertVectors(TABLE_NAME, qb, qxb.data(), target_ids); + ASSERT_EQ(target_ids.size(), qb); + } else { + db_->InsertVectors(TABLE_NAME, nb, xb.data(), vector_ids); + } + std::this_thread::sleep_for(std::chrono::microseconds(1)); + } + + search.join(); + + delete db_; + boost::filesystem::remove_all(options.meta.path); + +}; From 446513f004eb7c1560972ec778c0ef03314ce753 Mon Sep 17 00:00:00 2001 From: zhiru Date: Tue, 9 Jul 2019 21:15:54 +0800 Subject: [PATCH 131/189] update Former-commit-id: cb170ffdc137c7b897792f092f36438de892d0e8 --- cpp/unittest/db/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/unittest/db/CMakeLists.txt b/cpp/unittest/db/CMakeLists.txt index 460159227b..c0f3052848 100644 --- a/cpp/unittest/db/CMakeLists.txt +++ b/cpp/unittest/db/CMakeLists.txt @@ -43,8 +43,8 @@ set(db_libs cudart cublas sqlite - boost_filesystem_static boost_system_static + boost_filesystem_static lz4 mysqlpp ) From 618705f6175af25fcfa04a979af44b8ed7511d20 Mon Sep 17 00:00:00 2001 From: quicksilver Date: Wed, 10 Jul 2019 17:14:28 +0800 Subject: [PATCH 132/189] add email notify function Former-commit-id: 2c4a16473de125cd19c6c440d98171e190786179 --- ci/jenkinsfile/notify.groovy | 15 +++++++++++++++ ci/main_jenkinsfile | 6 ++++++ 2 files changed, 21 insertions(+) create mode 100644 ci/jenkinsfile/notify.groovy diff --git a/ci/jenkinsfile/notify.groovy b/ci/jenkinsfile/notify.groovy new file mode 100644 index 0000000000..0a257b8cd8 --- /dev/null +++ b/ci/jenkinsfile/notify.groovy @@ -0,0 +1,15 @@ +def notify() { + if (!currentBuild.resultIsBetterOrEqualTo('SUCCESS')) { + // Send an email only if the build status has changed from green/unstable to red + emailext subject: '$DEFAULT_SUBJECT', + body: '$DEFAULT_CONTENT', + recipientProviders: [ + [$class: 'DevelopersRecipientProvider'], + [$class: 'RequesterRecipientProvider'] + ], + replyTo: '$DEFAULT_REPLYTO', + to: '$DEFAULT_RECIPIENTS' + } +} +return this + diff --git a/ci/main_jenkinsfile b/ci/main_jenkinsfile index c144c46685..5c0fde587b 100644 --- a/ci/main_jenkinsfile +++ b/ci/main_jenkinsfile @@ -232,6 +232,12 @@ spec: } post { + always { + script { + def notify = load "${env.WORKSPACE}/ci/jenkinsfile/notify.groovy" + notify.notify() + } + } success { script { updateGitlabCommitStatus name: 'CI/CD', state: 'success' From 5e023cd47b48369ae79f25f7cc0fd4bef10c58a6 Mon Sep 17 00:00:00 2001 From: quicksilver Date: Wed, 10 Jul 2019 17:43:08 +0800 Subject: [PATCH 133/189] send email on master node Former-commit-id: 7eaec56894b5dbab27bc7971aace3d5dfc0687c5 --- ci/main_jenkinsfile | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/ci/main_jenkinsfile b/ci/main_jenkinsfile index 5c0fde587b..2783e84688 100644 --- a/ci/main_jenkinsfile +++ b/ci/main_jenkinsfile @@ -234,8 +234,10 @@ spec: post { always { script { - def notify = load "${env.WORKSPACE}/ci/jenkinsfile/notify.groovy" - notify.notify() + node('master') { + def notify = load "${env.WORKSPACE}/ci/jenkinsfile/notify.groovy" + notify.notify() + } } } success { From fe085bc56dbe04b19115d4f67afbb0b2a96b598a Mon Sep 17 00:00:00 2001 From: quicksilver Date: Wed, 10 Jul 2019 17:46:29 +0800 Subject: [PATCH 134/189] send email on master node Former-commit-id: a20a534b6b0411188e26fdccb8becb6508d1b8ba --- ci/main_jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/main_jenkinsfile b/ci/main_jenkinsfile index 2783e84688..2ab4344c01 100644 --- a/ci/main_jenkinsfile +++ b/ci/main_jenkinsfile @@ -234,7 +234,7 @@ spec: post { always { script { - node('master') { + node { def notify = load "${env.WORKSPACE}/ci/jenkinsfile/notify.groovy" notify.notify() } From 2e59ea0d32086568ea24dfdcc31175ff908e71c8 Mon Sep 17 00:00:00 2001 From: quicksilver Date: Wed, 10 Jul 2019 17:50:22 +0800 Subject: [PATCH 135/189] send email on master node Former-commit-id: 33fe31f8254b5ab2cc7a99e4b903a16730030f71 --- ci/main_jenkinsfile | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/ci/main_jenkinsfile b/ci/main_jenkinsfile index 2ab4344c01..1c7d8be75c 100644 --- a/ci/main_jenkinsfile +++ b/ci/main_jenkinsfile @@ -234,12 +234,20 @@ spec: post { always { script { - node { - def notify = load "${env.WORKSPACE}/ci/jenkinsfile/notify.groovy" - notify.notify() + if (!currentBuild.resultIsBetterOrEqualTo('SUCCESS')) { + // Send an email only if the build status has changed from green/unstable to red + emailext subject: '$DEFAULT_SUBJECT', + body: '$DEFAULT_CONTENT', + recipientProviders: [ + [$class: 'DevelopersRecipientProvider'], + [$class: 'RequesterRecipientProvider'] + ], + replyTo: '$DEFAULT_REPLYTO', + to: '$DEFAULT_RECIPIENTS' } } } + success { script { updateGitlabCommitStatus name: 'CI/CD', state: 'success' From 0bddcd5ef82afe15f87bb291d1f608eff566d8e6 Mon Sep 17 00:00:00 2001 From: quicksilver Date: Wed, 10 Jul 2019 17:54:47 +0800 Subject: [PATCH 136/189] MS-202 - Add Milvus Jenkins project email notification Former-commit-id: b4cb53acef46fc39df90515e751a1a1395f3b880 --- CHANGELOGS.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOGS.md b/CHANGELOGS.md index def4965a41..d9ef3d4c36 100644 --- a/CHANGELOGS.md +++ b/CHANGELOGS.md @@ -16,3 +16,4 @@ Please mark all change in change log and use the ticket from JIRA. - MS-1 - Add CHANGELOG.md - MS-161 - Add CI / CD Module to Milvus Project +- MS-202 - Add Milvus Jenkins project email notification From beb7be415e9b6e0297d76c504ca5985e98ac6054 Mon Sep 17 00:00:00 2001 From: quicksilver Date: Fri, 12 Jul 2019 14:26:16 +0800 Subject: [PATCH 137/189] add helm stable repo Former-commit-id: 75db992912aa62772bdf0e879921b0b3693a595c --- ci/jenkinsfile/deploy2dev.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/jenkinsfile/deploy2dev.groovy b/ci/jenkinsfile/deploy2dev.groovy index 6e4a23cfe7..47497e1648 100644 --- a/ci/jenkinsfile/deploy2dev.groovy +++ b/ci/jenkinsfile/deploy2dev.groovy @@ -1,5 +1,5 @@ try { - sh 'helm init --client-only --skip-refresh' + sh 'helm init --client-only --skip-refresh --stable-repo-url https://kubernetes.oss-cn-hangzhou.aliyuncs.com/charts' sh 'helm repo add milvus https://registry.zilliz.com/chartrepo/milvus' sh 'helm repo update' sh "helm install --set engine.image.repository=registry.zilliz.com/${PROJECT_NAME}/engine --set engine.image.tag=${DOCKER_VERSION} --set expose.type=clusterIP --name ${env.JOB_NAME}-${env.BUILD_NUMBER} --version 0.3.0 milvus/milvus-gpu" From 7c58bd30a21b3c7d13baec91598b7385c57c8728 Mon Sep 17 00:00:00 2001 From: jinhai Date: Sun, 14 Jul 2019 01:54:26 +0800 Subject: [PATCH 138/189] Remove redefinition Former-commit-id: 4dd734c7ebffa9a462c641b0e220a62668651859 --- cpp/src/server/DBWrapper.cpp | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/cpp/src/server/DBWrapper.cpp b/cpp/src/server/DBWrapper.cpp index 678dcb72ef..66a1d83479 100644 --- a/cpp/src/server/DBWrapper.cpp +++ b/cpp/src/server/DBWrapper.cpp @@ -53,22 +53,6 @@ DBWrapper::DBWrapper() { kill(0, SIGUSR1); } - ConfigNode& serverConfig = ServerConfig::GetInstance().GetConfig(CONFIG_SERVER); - std::string mode = serverConfig.GetValue(CONFIG_CLUSTER_MODE, "single"); - if (mode == "single") { - opt.mode = zilliz::milvus::engine::Options::MODE::SINGLE; - } - else if (mode == "cluster") { - opt.mode = zilliz::milvus::engine::Options::MODE::CLUSTER; - } - else if (mode == "read_only") { - opt.mode = zilliz::milvus::engine::Options::MODE::READ_ONLY; - } - else { - std::cout << "ERROR: mode specified in server_config is not one of ['single', 'cluster', 'read_only']" << std::endl; - kill(0, SIGUSR1); - } - //set archive config engine::ArchiveConf::CriteriaT criterial; int64_t disk = config.GetInt64Value(CONFIG_DB_ARCHIVE_DISK, 0); From 8103d78fd561f5f8922294c7e082610cb64c067f Mon Sep 17 00:00:00 2001 From: jinhai Date: Sun, 14 Jul 2019 02:08:56 +0800 Subject: [PATCH 139/189] Fix build fault Former-commit-id: 1c813d1f13c4bfd36d42ad11f21f1fe1bc827f17 --- cpp/unittest/db/mem_test.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/cpp/unittest/db/mem_test.cpp b/cpp/unittest/db/mem_test.cpp index 860a665386..dba36db2a5 100644 --- a/cpp/unittest/db/mem_test.cpp +++ b/cpp/unittest/db/mem_test.cpp @@ -14,6 +14,7 @@ #include #include #include +#include using namespace zilliz::milvus; From b19b4031b8bae39dccbd9976583ddd4888bac9ac Mon Sep 17 00:00:00 2001 From: jinhai Date: Sun, 14 Jul 2019 02:11:32 +0800 Subject: [PATCH 140/189] Fix building fault Former-commit-id: 4b82bd7958672d7762aa667845c9eb63a8da789d --- cpp/unittest/db/mem_test.cpp | 2 +- cpp/unittest/db/search_test.cpp | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/cpp/unittest/db/mem_test.cpp b/cpp/unittest/db/mem_test.cpp index dba36db2a5..2f6952c79d 100644 --- a/cpp/unittest/db/mem_test.cpp +++ b/cpp/unittest/db/mem_test.cpp @@ -15,7 +15,7 @@ #include #include #include - +#include using namespace zilliz::milvus; diff --git a/cpp/unittest/db/search_test.cpp b/cpp/unittest/db/search_test.cpp index d860295cb1..c4ee7a48b5 100644 --- a/cpp/unittest/db/search_test.cpp +++ b/cpp/unittest/db/search_test.cpp @@ -6,7 +6,7 @@ #include #include "db/scheduler/task/SearchTask.h" - +#include #include using namespace zilliz::milvus; @@ -159,4 +159,4 @@ TEST(DBSearchTest, MERGE_TEST) { ASSERT_EQ(target.size(), src_count + target_count); CheckResult(src_result[0], target_result[0], target); } -} \ No newline at end of file +} From 46a63f43f36c495089cca3e9a78f7901bee6981e Mon Sep 17 00:00:00 2001 From: quicksilver Date: Sun, 14 Jul 2019 09:33:35 +0800 Subject: [PATCH 141/189] MS-215 - add milvus cluster CI/CD groovy file Former-commit-id: 6d3159ab782c4acb0bc31831438e09015ed447fa --- ci/jenkinsfile/cluster_deploy2dev.groovy | 8 +++++--- ci/jenkinsfile/deploy2dev.groovy | 8 +++++--- ci/main_jenkinsfile | 22 ++++++++++++---------- 3 files changed, 22 insertions(+), 16 deletions(-) diff --git a/ci/jenkinsfile/cluster_deploy2dev.groovy b/ci/jenkinsfile/cluster_deploy2dev.groovy index 97062922ca..4347ea0938 100644 --- a/ci/jenkinsfile/cluster_deploy2dev.groovy +++ b/ci/jenkinsfile/cluster_deploy2dev.groovy @@ -7,9 +7,11 @@ try { dir ("milvus/milvus-cluster") { sh "helm install --set roServers.image.tag=${DOCKER_VERSION} --set woServers.image.tag=${DOCKER_VERSION} --set expose.type=clusterIP -f ci/values.yaml --name ${env.JOB_NAME}-${env.BUILD_NUMBER}-cluster --namespace milvus-cluster --version 0.1.0 . " } - waitUntil { - def result = sh script: "nc -z -w 2 ${env.JOB_NAME}-${env.BUILD_NUMBER}-cluster-milvus-cluster-proxy.milvus-cluster.svc.cluster.local 19530", returnStatus: true - return !result + timeout(time: 2, unit: 'MINUTES') { + waitUntil { + def result = sh script: "nc -z -w 2 ${env.JOB_NAME}-${env.BUILD_NUMBER}-cluster-milvus-cluster-proxy.milvus-cluster.svc.cluster.local 19530", returnStatus: true + return !result + } } } } catch (exc) { diff --git a/ci/jenkinsfile/deploy2dev.groovy b/ci/jenkinsfile/deploy2dev.groovy index f594d0ba57..607149f35d 100644 --- a/ci/jenkinsfile/deploy2dev.groovy +++ b/ci/jenkinsfile/deploy2dev.groovy @@ -3,9 +3,11 @@ try { sh 'helm repo add milvus https://registry.zilliz.com/chartrepo/milvus' sh 'helm repo update' sh "helm install --set engine.image.tag=${DOCKER_VERSION} --set expose.type=clusterIP --name ${env.JOB_NAME}-${env.BUILD_NUMBER} --version 0.3.0 milvus/milvus-gpu" - waitUntil { - def result = sh script: "nc -z -w 2 ${env.JOB_NAME}-${env.BUILD_NUMBER}-milvus-gpu-engine.kube-opt.svc.cluster.local 19530", returnStatus: true - return !result + timeout(time: 2, unit: 'MINUTES') { + waitUntil { + def result = sh script: "nc -z -w 2 ${env.JOB_NAME}-${env.BUILD_NUMBER}-milvus-gpu-engine.kube-opt.svc.cluster.local 19530", returnStatus: true + return !result + } } } catch (exc) { updateGitlabCommitStatus name: 'Deloy to Dev', state: 'failed' diff --git a/ci/main_jenkinsfile b/ci/main_jenkinsfile index 1f0d0c3d7f..a324ff4441 100644 --- a/ci/main_jenkinsfile +++ b/ci/main_jenkinsfile @@ -337,16 +337,18 @@ spec: post { always { script { - if (!currentBuild.resultIsBetterOrEqualTo('SUCCESS')) { - // Send an email only if the build status has changed from green/unstable to red - emailext subject: '$DEFAULT_SUBJECT', - body: '$DEFAULT_CONTENT', - recipientProviders: [ - [$class: 'DevelopersRecipientProvider'], - [$class: 'RequesterRecipientProvider'] - ], - replyTo: '$DEFAULT_REPLYTO', - to: '$DEFAULT_RECIPIENTS' + if (env.gitlabAfter != null) { + if (!currentBuild.resultIsBetterOrEqualTo('SUCCESS')) { + // Send an email only if the build status has changed from green/unstable to red + emailext subject: '$DEFAULT_SUBJECT', + body: '$DEFAULT_CONTENT', + recipientProviders: [ + [$class: 'DevelopersRecipientProvider'], + [$class: 'RequesterRecipientProvider'] + ], + replyTo: '$DEFAULT_REPLYTO', + to: '$DEFAULT_RECIPIENTS' + } } } } From f45037322c1a50f05f68daa58837647053bf8c5d Mon Sep 17 00:00:00 2001 From: quicksilver Date: Sun, 14 Jul 2019 09:34:47 +0800 Subject: [PATCH 142/189] change cpp CHANGELOGS.md Former-commit-id: f57d675bf412261f544708860ccdd486933ca79c --- CHANGELOGS.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOGS.md b/CHANGELOGS.md index def4965a41..3cc1b28d82 100644 --- a/CHANGELOGS.md +++ b/CHANGELOGS.md @@ -16,3 +16,4 @@ Please mark all change in change log and use the ticket from JIRA. - MS-1 - Add CHANGELOG.md - MS-161 - Add CI / CD Module to Milvus Project +- MS-215 - Add Milvus cluster CI/CD groovy file From 94d88ec0cb9a82924cf4ec7ba87757e37e1a5739 Mon Sep 17 00:00:00 2001 From: quicksilver Date: Sun, 14 Jul 2019 11:24:47 +0800 Subject: [PATCH 143/189] Update Chart Version to 0.3.1 Former-commit-id: 8295125e067afeb2e45b58cfeba53f4c14a19db3 --- ci/jenkinsfile/cluster_deploy2dev.groovy | 4 ++-- ci/jenkinsfile/deploy2dev.groovy | 7 ++++++- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/ci/jenkinsfile/cluster_deploy2dev.groovy b/ci/jenkinsfile/cluster_deploy2dev.groovy index 4347ea0938..dba2e52713 100644 --- a/ci/jenkinsfile/cluster_deploy2dev.groovy +++ b/ci/jenkinsfile/cluster_deploy2dev.groovy @@ -3,9 +3,9 @@ try { sh 'helm repo add milvus https://registry.zilliz.com/chartrepo/milvus' sh 'helm repo update' dir ("milvus-helm") { - checkout([$class: 'GitSCM', branches: [[name: "${SEMVER}"]], doGenerateSubmoduleConfigurations: false, extensions: [[$class: 'SubmoduleOption',disableSubmodules: false,parentCredentials: true,recursiveSubmodules: true,reference: '',trackingSubmodules: false]], submoduleCfg: [], userRemoteConfigs: [[credentialsId: "${params.GIT_USER}", url: "git@192.168.1.105:megasearch/milvus-helm.git", name: 'origin', refspec: "+refs/heads/${SEMVER}:refs/remotes/origin/${SEMVER}"]]]) + checkout([$class: 'GitSCM', branches: [[name: "${SEMVER}"]], doGenerateSubmoduleConfigurations: false, extensions: [], submoduleCfg: [], userRemoteConfigs: [[credentialsId: "${params.GIT_USER}", url: "git@192.168.1.105:megasearch/milvus-helm.git", name: 'origin', refspec: "+refs/heads/${SEMVER}:refs/remotes/origin/${SEMVER}"]]]) dir ("milvus/milvus-cluster") { - sh "helm install --set roServers.image.tag=${DOCKER_VERSION} --set woServers.image.tag=${DOCKER_VERSION} --set expose.type=clusterIP -f ci/values.yaml --name ${env.JOB_NAME}-${env.BUILD_NUMBER}-cluster --namespace milvus-cluster --version 0.1.0 . " + sh "helm install --set roServers.image.tag=${DOCKER_VERSION} --set woServers.image.tag=${DOCKER_VERSION} --set expose.type=clusterIP -f ci/values.yaml --name ${env.JOB_NAME}-${env.BUILD_NUMBER}-cluster --namespace milvus-cluster --version 0.3.1 . " } timeout(time: 2, unit: 'MINUTES') { waitUntil { diff --git a/ci/jenkinsfile/deploy2dev.groovy b/ci/jenkinsfile/deploy2dev.groovy index 607149f35d..26e2aec562 100644 --- a/ci/jenkinsfile/deploy2dev.groovy +++ b/ci/jenkinsfile/deploy2dev.groovy @@ -2,7 +2,12 @@ try { sh 'helm init --client-only --skip-refresh --stable-repo-url https://kubernetes.oss-cn-hangzhou.aliyuncs.com/charts' sh 'helm repo add milvus https://registry.zilliz.com/chartrepo/milvus' sh 'helm repo update' - sh "helm install --set engine.image.tag=${DOCKER_VERSION} --set expose.type=clusterIP --name ${env.JOB_NAME}-${env.BUILD_NUMBER} --version 0.3.0 milvus/milvus-gpu" + dir ("milvus-helm") { + checkout([$class: 'GitSCM', branches: [[name: "${SEMVER}"]], doGenerateSubmoduleConfigurations: false, extensions: [], submoduleCfg: [], userRemoteConfigs: [[credentialsId: "${params.GIT_USER}", url: "git@192.168.1.105:megasearch/milvus-helm.git", name: 'origin', refspec: "+refs/heads/${SEMVER}:refs/remotes/origin/${SEMVER}"]]]) + dir ("milvus/milvus-gpu") { + helm install --set engine.image.tag=${DOCKER_VERSION} --set expose.type=clusterIP --name ${env.JOB_NAME}-${env.BUILD_NUMBER} --version 0.3.1 . + } + } timeout(time: 2, unit: 'MINUTES') { waitUntil { def result = sh script: "nc -z -w 2 ${env.JOB_NAME}-${env.BUILD_NUMBER}-milvus-gpu-engine.kube-opt.svc.cluster.local 19530", returnStatus: true From eb14bc0d2d7cfe17e3fc3e66ab3501b20f39f512 Mon Sep 17 00:00:00 2001 From: starlord Date: Sun, 14 Jul 2019 11:43:14 +0800 Subject: [PATCH 144/189] Fix SQ8 row count bug Former-commit-id: 41749a648567e492bc9a718db9526ed98bb2532a --- cpp/CHANGELOG.md | 5 +++-- cpp/conf/server_config.template | 6 +++--- cpp/src/db/DBImpl.cpp | 2 +- cpp/src/db/DBMetaImpl.cpp | 22 ++++++++++++++-------- cpp/src/db/FaissExecutionEngine.cpp | 2 +- cpp/src/db/Options.h | 2 +- 6 files changed, 23 insertions(+), 16 deletions(-) diff --git a/cpp/CHANGELOG.md b/cpp/CHANGELOG.md index 9ac6cb11bc..3be585c7b6 100644 --- a/cpp/CHANGELOG.md +++ b/cpp/CHANGELOG.md @@ -9,8 +9,9 @@ Please mark all change in change log and use the ticket from JIRA. - MS-148 - Disable cleanup if mode is read only - MS-149 - Fixed searching only one index file issue in distributed mode -- MS-153 - fix c_str error when connecting to MySQL -- MS-157 - fix changelog +- MS-153 - Fix c_str error when connecting to MySQL +- MS-157 - Fix changelog +- MS-217 - Fix SQ8 row count bug ## Improvement - MS-156 - Add unittest for merge result functions diff --git a/cpp/conf/server_config.template b/cpp/conf/server_config.template index 3f3b004456..6de96ae895 100644 --- a/cpp/conf/server_config.template +++ b/cpp/conf/server_config.template @@ -14,8 +14,8 @@ db_config: db_backend_url: sqlite://:@:/ index_building_threshold: 1024 # index building trigger threshold, default: 1024, unit: MB - archive_disk_threshold: 512 # triger archive action if storage size exceed this value, unit: GB - archive_days_threshold: 30 # files older than x days will be archived, unit: day + archive_disk_threshold: 0 # triger archive action if storage size exceed this value, 0 means no limit, unit: GB + archive_days_threshold: 0 # files older than x days will be archived, 0 means no limit, unit: day metric_config: is_startup: off # if monitoring start: on, off @@ -37,4 +37,4 @@ engine_config: nprobe: 10 nlist: 16384 use_blas_threshold: 20 - metric_type: L2 #L2 or Inner Product \ No newline at end of file + metric_type: L2 # compare vectors by euclidean distance(L2) or inner product(IP), optional: L2 or IP \ No newline at end of file diff --git a/cpp/src/db/DBImpl.cpp b/cpp/src/db/DBImpl.cpp index 35a6f075f8..beae6c30d3 100644 --- a/cpp/src/db/DBImpl.cpp +++ b/cpp/src/db/DBImpl.cpp @@ -487,7 +487,7 @@ Status DBImpl::BuildIndex(const meta::TableFileSchema& file) { //step 6: update meta table_file.file_type_ = meta::TableFileSchema::INDEX; - table_file.size_ = index->PhysicalSize(); + table_file.size_ = index->Size(); auto to_remove = file; to_remove.file_type_ = meta::TableFileSchema::TO_DELETE; diff --git a/cpp/src/db/DBMetaImpl.cpp b/cpp/src/db/DBMetaImpl.cpp index fc67c6b360..99ae3b6711 100644 --- a/cpp/src/db/DBMetaImpl.cpp +++ b/cpp/src/db/DBMetaImpl.cpp @@ -674,16 +674,22 @@ Status DBMetaImpl::Archive() { Status DBMetaImpl::Size(uint64_t &result) { result = 0; try { - auto selected = ConnectorPtr->select(columns(sum(&TableFileSchema::size_)), - where( - c(&TableFileSchema::file_type_) != (int) TableFileSchema::TO_DELETE - )); + auto files = ConnectorPtr->select(columns(&TableFileSchema::size_, + &TableFileSchema::file_type_, + &TableFileSchema::engine_type_), + where( + c(&TableFileSchema::file_type_) != (int) TableFileSchema::TO_DELETE + )); - for (auto &sub_query : selected) { - if (!std::get<0>(sub_query)) { - continue; + for (auto &file : files) { + auto file_size = std::get<0>(file); + auto file_type = std::get<1>(file); + auto engine_type = std::get<2>(file); + if(file_type == (int)TableFileSchema::INDEX && engine_type == (int)EngineType::FAISS_IVFSQ8) { + result += (uint64_t)file_size/4;//hardcode for sq8 + } else { + result += (uint64_t)file_size; } - result += (uint64_t) (*std::get<0>(sub_query)); } } catch (std::exception &e) { return HandleException("Encounter exception when calculte db size", e); diff --git a/cpp/src/db/FaissExecutionEngine.cpp b/cpp/src/db/FaissExecutionEngine.cpp index d2cb835364..7183f43017 100644 --- a/cpp/src/db/FaissExecutionEngine.cpp +++ b/cpp/src/db/FaissExecutionEngine.cpp @@ -110,7 +110,7 @@ Status FaissExecutionEngine::Merge(const std::string& location) { if (location == location_) { return Status::Error("Cannot Merge Self"); } - ENGINE_LOG_DEBUG << "Merge index file: " << location << " to: " << location_; + ENGINE_LOG_DEBUG << "Merge raw file: " << location << " to: " << location_; auto to_merge = zilliz::milvus::cache::CpuCacheMgr::GetInstance()->GetIndex(location); if (!to_merge) { diff --git a/cpp/src/db/Options.h b/cpp/src/db/Options.h index 1c3c28b4ac..f6c09fb750 100644 --- a/cpp/src/db/Options.h +++ b/cpp/src/db/Options.h @@ -22,7 +22,7 @@ static constexpr uint64_t ONE_GB = ONE_KB*ONE_MB; static const std::string ARCHIVE_CONF_DISK = "disk"; static const std::string ARCHIVE_CONF_DAYS = "days"; -static const std::string ARCHIVE_CONF_DEFAULT = ARCHIVE_CONF_DISK + ":512"; +static const std::string ARCHIVE_CONF_DEFAULT = ""; struct ArchiveConf { using CriteriaT = std::map; From b30cefc5e5d402072e695eb2bd4ca460ddcc2baf Mon Sep 17 00:00:00 2001 From: quicksilver Date: Sun, 14 Jul 2019 12:02:19 +0800 Subject: [PATCH 145/189] fix milvus single deploy error Former-commit-id: 4fabf7de507cb7eebbc532d3753c4b10cf0d391d --- ci/jenkinsfile/deploy2dev.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/jenkinsfile/deploy2dev.groovy b/ci/jenkinsfile/deploy2dev.groovy index 26e2aec562..d20f1444bc 100644 --- a/ci/jenkinsfile/deploy2dev.groovy +++ b/ci/jenkinsfile/deploy2dev.groovy @@ -5,7 +5,7 @@ try { dir ("milvus-helm") { checkout([$class: 'GitSCM', branches: [[name: "${SEMVER}"]], doGenerateSubmoduleConfigurations: false, extensions: [], submoduleCfg: [], userRemoteConfigs: [[credentialsId: "${params.GIT_USER}", url: "git@192.168.1.105:megasearch/milvus-helm.git", name: 'origin', refspec: "+refs/heads/${SEMVER}:refs/remotes/origin/${SEMVER}"]]]) dir ("milvus/milvus-gpu") { - helm install --set engine.image.tag=${DOCKER_VERSION} --set expose.type=clusterIP --name ${env.JOB_NAME}-${env.BUILD_NUMBER} --version 0.3.1 . + sh "helm install --set engine.image.tag=${DOCKER_VERSION} --set expose.type=clusterIP --name ${env.JOB_NAME}-${env.BUILD_NUMBER} --version 0.3.1 ." } } timeout(time: 2, unit: 'MINUTES') { From 8fd3dd219fdca159ef90e263efe5aca585155894 Mon Sep 17 00:00:00 2001 From: quicksilver Date: Sun, 14 Jul 2019 14:43:47 +0800 Subject: [PATCH 146/189] format clean up stage Former-commit-id: cbe735d8a052e33816fba943057877135ed7212a --- ci/jenkinsfile/cleanup_dev.groovy | 6 -- ci/jenkinsfile/cluster_cleanup_dev.groovy | 6 -- ci/jenkinsfile/cluster_dev_test.groovy | 2 +- ci/jenkinsfile/dev_test.groovy | 2 +- ci/main_jenkinsfile | 89 ++++++----------- ci/main_jenkinsfile_no_ut | 111 ++++++++-------------- ci/nightly_main_jenkinsfile | 88 ++++++----------- 7 files changed, 98 insertions(+), 206 deletions(-) diff --git a/ci/jenkinsfile/cleanup_dev.groovy b/ci/jenkinsfile/cleanup_dev.groovy index 056b8fa8c5..664c452f34 100644 --- a/ci/jenkinsfile/cleanup_dev.groovy +++ b/ci/jenkinsfile/cleanup_dev.groovy @@ -1,11 +1,5 @@ try { sh "helm del --purge ${env.JOB_NAME}-${env.BUILD_NUMBER}" - - if (currentBuild.result == 'ABORTED') { - throw new hudson.AbortException("Dev Test Aborted !") - } else if (currentBuild.result == 'FAILURE') { - error("Dev Test Failure !") - } } catch (exc) { sh "helm del --purge ${env.JOB_NAME}-${env.BUILD_NUMBER}" updateGitlabCommitStatus name: 'Cleanup Dev', state: 'failed' diff --git a/ci/jenkinsfile/cluster_cleanup_dev.groovy b/ci/jenkinsfile/cluster_cleanup_dev.groovy index dc18067cdd..9708669b9e 100644 --- a/ci/jenkinsfile/cluster_cleanup_dev.groovy +++ b/ci/jenkinsfile/cluster_cleanup_dev.groovy @@ -1,11 +1,5 @@ try { sh "helm del --purge ${env.JOB_NAME}-${env.BUILD_NUMBER}-cluster" - - if (currentBuild.result == 'ABORTED') { - throw new hudson.AbortException("Cluster Dev Test Aborted !") - } else if (currentBuild.result == 'FAILURE') { - error("Dev Test Failure !") - } } catch (exc) { sh "helm del --purge ${env.JOB_NAME}-${env.BUILD_NUMBER}-cluster" updateGitlabCommitStatus name: 'Cleanup Dev', state: 'failed' diff --git a/ci/jenkinsfile/cluster_dev_test.groovy b/ci/jenkinsfile/cluster_dev_test.groovy index 50fd67f803..d3c6a19832 100644 --- a/ci/jenkinsfile/cluster_dev_test.groovy +++ b/ci/jenkinsfile/cluster_dev_test.groovy @@ -9,8 +9,8 @@ container('milvus-testframework') { } } catch (exc) { updateGitlabCommitStatus name: 'Dev Test', state: 'failed' - currentBuild.result = 'FAILURE' echo 'Milvus Test Failed !' + throw exc } } } diff --git a/ci/jenkinsfile/dev_test.groovy b/ci/jenkinsfile/dev_test.groovy index 13e1c868b8..4034872615 100644 --- a/ci/jenkinsfile/dev_test.groovy +++ b/ci/jenkinsfile/dev_test.groovy @@ -9,8 +9,8 @@ container('milvus-testframework') { } } catch (exc) { updateGitlabCommitStatus name: 'Dev Test', state: 'failed' - currentBuild.result = 'FAILURE' echo 'Milvus Test Failed !' + throw exc } } } diff --git a/ci/main_jenkinsfile b/ci/main_jenkinsfile index a324ff4441..dc5e879817 100644 --- a/ci/main_jenkinsfile +++ b/ci/main_jenkinsfile @@ -191,38 +191,22 @@ spec: } } } - } - - stage ("Cleanup Dev") { - agent { - kubernetes { - label 'jenkins-slave' - defaultContainer 'jnlp' - } - } - stages { - stage('Cleanup') { - steps { - gitlabCommitStatus(name: 'Cleanup Dev') { - script { - load "${env.WORKSPACE}/ci/jenkinsfile/cleanup_dev.groovy" + post { + always { + stage ("Cleanup Dev") { + agent { + kubernetes { + label 'jenkins-slave' + defaultContainer 'jnlp' + } + } + stages { + gitlabCommitStatus(name: 'Cleanup Dev') { + script { + load "${env.WORKSPACE}/ci/jenkinsfile/cleanup_dev.groovy" + } } } - } - } - } - post { - aborted { - script { - updateGitlabCommitStatus name: 'Cleanup Dev', state: 'canceled' - echo "Milvus Cleanup Dev aborted !" - } - } - - failure { - script { - updateGitlabCommitStatus name: 'Cleanup Dev', state: 'failed' - echo "Milvus Cleanup Dev failure !" } } } @@ -290,38 +274,22 @@ spec: } } } - } - - stage ("Cleanup Dev") { - agent { - kubernetes { - label 'jenkins-slave' - defaultContainer 'jnlp' - } - } - stages { - stage('Cleanup') { - steps { - gitlabCommitStatus(name: 'Cleanup Dev') { - script { - load "${env.WORKSPACE}/ci/jenkinsfile/cluster_cleanup_dev.groovy" + post { + always { + stage ("Cleanup Dev") { + agent { + kubernetes { + label 'jenkins-slave' + defaultContainer 'jnlp' + } + } + stages { + gitlabCommitStatus(name: 'Cleanup Dev') { + script { + load "${env.WORKSPACE}/ci/jenkinsfile/cluster_cleanup_dev.groovy" + } } } - } - } - } - post { - aborted { - script { - updateGitlabCommitStatus name: 'Cleanup Dev', state: 'canceled' - echo "Milvus Cleanup Dev aborted !" - } - } - - failure { - script { - updateGitlabCommitStatus name: 'Cleanup Dev', state: 'failed' - echo "Milvus Cleanup Dev failure !" } } } @@ -375,4 +343,3 @@ spec: } } } - diff --git a/ci/main_jenkinsfile_no_ut b/ci/main_jenkinsfile_no_ut index 203b185e14..35ebc64778 100644 --- a/ci/main_jenkinsfile_no_ut +++ b/ci/main_jenkinsfile_no_ut @@ -191,38 +191,22 @@ spec: } } } - } - - stage ("Cleanup Dev") { - agent { - kubernetes { - label 'jenkins-slave' - defaultContainer 'jnlp' - } - } - stages { - stage('Cleanup') { - steps { - gitlabCommitStatus(name: 'Cleanup Dev') { - script { - load "${env.WORKSPACE}/ci/jenkinsfile/cleanup_dev.groovy" + post { + always { + stage ("Cleanup Dev") { + agent { + kubernetes { + label 'jenkins-slave' + defaultContainer 'jnlp' + } + } + stages { + gitlabCommitStatus(name: 'Cleanup Dev') { + script { + load "${env.WORKSPACE}/ci/jenkinsfile/cleanup_dev.groovy" + } } } - } - } - } - post { - aborted { - script { - updateGitlabCommitStatus name: 'Cleanup Dev', state: 'canceled' - echo "Milvus Cleanup Dev aborted !" - } - } - - failure { - script { - updateGitlabCommitStatus name: 'Cleanup Dev', state: 'failed' - echo "Milvus Cleanup Dev failure !" } } } @@ -290,38 +274,22 @@ spec: } } } - } - - stage ("Cleanup Dev") { - agent { - kubernetes { - label 'jenkins-slave' - defaultContainer 'jnlp' - } - } - stages { - stage('Cleanup') { - steps { - gitlabCommitStatus(name: 'Cleanup Dev') { - script { - load "${env.WORKSPACE}/ci/jenkinsfile/cluster_cleanup_dev.groovy" + post { + always { + stage ("Cleanup Dev") { + agent { + kubernetes { + label 'jenkins-slave' + defaultContainer 'jnlp' + } + } + stages { + gitlabCommitStatus(name: 'Cleanup Dev') { + script { + load "${env.WORKSPACE}/ci/jenkinsfile/cluster_cleanup_dev.groovy" + } } } - } - } - } - post { - aborted { - script { - updateGitlabCommitStatus name: 'Cleanup Dev', state: 'canceled' - echo "Milvus Cleanup Dev aborted !" - } - } - - failure { - script { - updateGitlabCommitStatus name: 'Cleanup Dev', state: 'failed' - echo "Milvus Cleanup Dev failure !" } } } @@ -337,16 +305,18 @@ spec: post { always { script { - if (!currentBuild.resultIsBetterOrEqualTo('SUCCESS')) { - // Send an email only if the build status has changed from green/unstable to red - emailext subject: '$DEFAULT_SUBJECT', - body: '$DEFAULT_CONTENT', - recipientProviders: [ - [$class: 'DevelopersRecipientProvider'], - [$class: 'RequesterRecipientProvider'] - ], - replyTo: '$DEFAULT_REPLYTO', - to: '$DEFAULT_RECIPIENTS' + if (env.gitlabAfter != null) { + if (!currentBuild.resultIsBetterOrEqualTo('SUCCESS')) { + // Send an email only if the build status has changed from green/unstable to red + emailext subject: '$DEFAULT_SUBJECT', + body: '$DEFAULT_CONTENT', + recipientProviders: [ + [$class: 'DevelopersRecipientProvider'], + [$class: 'RequesterRecipientProvider'] + ], + replyTo: '$DEFAULT_REPLYTO', + to: '$DEFAULT_RECIPIENTS' + } } } } @@ -373,4 +343,3 @@ spec: } } } - diff --git a/ci/nightly_main_jenkinsfile b/ci/nightly_main_jenkinsfile index c375cc87ea..7b39ed386c 100644 --- a/ci/nightly_main_jenkinsfile +++ b/ci/nightly_main_jenkinsfile @@ -191,38 +191,23 @@ spec: } } } - } - stage ("Cleanup Dev") { - agent { - kubernetes { - label 'jenkins-slave' - defaultContainer 'jnlp' - } - } - stages { - stage('Cleanup') { - steps { - gitlabCommitStatus(name: 'Cleanup Dev') { - script { - load "${env.WORKSPACE}/ci/jenkinsfile/cleanup_dev.groovy" + post { + always { + stage ("Cleanup Dev") { + agent { + kubernetes { + label 'jenkins-slave' + defaultContainer 'jnlp' + } + } + stages { + gitlabCommitStatus(name: 'Cleanup Dev') { + script { + load "${env.WORKSPACE}/ci/jenkinsfile/cleanup_dev.groovy" + } } } - } - } - } - post { - aborted { - script { - updateGitlabCommitStatus name: 'Cleanup Dev', state: 'canceled' - echo "Milvus Cleanup Dev aborted !" - } - } - - failure { - script { - updateGitlabCommitStatus name: 'Cleanup Dev', state: 'failed' - echo "Milvus Cleanup Dev failure !" } } } @@ -290,38 +275,22 @@ spec: } } } - } - - stage ("Cleanup Dev") { - agent { - kubernetes { - label 'jenkins-slave' - defaultContainer 'jnlp' - } - } - stages { - stage('Cleanup') { - steps { - gitlabCommitStatus(name: 'Cleanup Dev') { - script { - load "${env.WORKSPACE}/ci/jenkinsfile/cluster_cleanup_dev.groovy" + post { + always { + stage ("Cleanup Dev") { + agent { + kubernetes { + label 'jenkins-slave' + defaultContainer 'jnlp' + } + } + stages { + gitlabCommitStatus(name: 'Cleanup Dev') { + script { + load "${env.WORKSPACE}/ci/jenkinsfile/cluster_cleanup_dev.groovy" + } } } - } - } - } - post { - aborted { - script { - updateGitlabCommitStatus name: 'Cleanup Dev', state: 'canceled' - echo "Milvus Cleanup Dev aborted !" - } - } - - failure { - script { - updateGitlabCommitStatus name: 'Cleanup Dev', state: 'failed' - echo "Milvus Cleanup Dev failure !" } } } @@ -373,4 +342,3 @@ spec: } } } - From fdc159c27dab2b00290376fec10c5cf83901bd45 Mon Sep 17 00:00:00 2001 From: quicksilver Date: Sun, 14 Jul 2019 14:53:00 +0800 Subject: [PATCH 147/189] fix cleanup bug Former-commit-id: e1a28efe242625615c79b3a1a2c0f7dc3fc1ace5 --- ci/main_jenkinsfile | 44 ++++++++++++++++++++----------------- ci/main_jenkinsfile_no_ut | 44 ++++++++++++++++++++----------------- ci/nightly_main_jenkinsfile | 44 ++++++++++++++++++++----------------- 3 files changed, 72 insertions(+), 60 deletions(-) diff --git a/ci/main_jenkinsfile b/ci/main_jenkinsfile index dc5e879817..cde27f64d0 100644 --- a/ci/main_jenkinsfile +++ b/ci/main_jenkinsfile @@ -193,17 +193,19 @@ spec: } post { always { - stage ("Cleanup Dev") { - agent { - kubernetes { - label 'jenkins-slave' - defaultContainer 'jnlp' + stages { + stage ("Cleanup Dev") { + agent { + kubernetes { + label 'jenkins-slave' + defaultContainer 'jnlp' + } } - } - stages { - gitlabCommitStatus(name: 'Cleanup Dev') { - script { - load "${env.WORKSPACE}/ci/jenkinsfile/cleanup_dev.groovy" + stages { + gitlabCommitStatus(name: 'Cleanup Dev') { + script { + load "${env.WORKSPACE}/ci/jenkinsfile/cleanup_dev.groovy" + } } } } @@ -276,17 +278,19 @@ spec: } post { always { - stage ("Cleanup Dev") { - agent { - kubernetes { - label 'jenkins-slave' - defaultContainer 'jnlp' + stages { + stage ("Cleanup Dev") { + agent { + kubernetes { + label 'jenkins-slave' + defaultContainer 'jnlp' + } } - } - stages { - gitlabCommitStatus(name: 'Cleanup Dev') { - script { - load "${env.WORKSPACE}/ci/jenkinsfile/cluster_cleanup_dev.groovy" + stages { + gitlabCommitStatus(name: 'Cleanup Dev') { + script { + load "${env.WORKSPACE}/ci/jenkinsfile/cluster_cleanup_dev.groovy" + } } } } diff --git a/ci/main_jenkinsfile_no_ut b/ci/main_jenkinsfile_no_ut index 35ebc64778..c6a5914baa 100644 --- a/ci/main_jenkinsfile_no_ut +++ b/ci/main_jenkinsfile_no_ut @@ -193,17 +193,19 @@ spec: } post { always { - stage ("Cleanup Dev") { - agent { - kubernetes { - label 'jenkins-slave' - defaultContainer 'jnlp' + stages { + stage ("Cleanup Dev") { + agent { + kubernetes { + label 'jenkins-slave' + defaultContainer 'jnlp' + } } - } - stages { - gitlabCommitStatus(name: 'Cleanup Dev') { - script { - load "${env.WORKSPACE}/ci/jenkinsfile/cleanup_dev.groovy" + stages { + gitlabCommitStatus(name: 'Cleanup Dev') { + script { + load "${env.WORKSPACE}/ci/jenkinsfile/cleanup_dev.groovy" + } } } } @@ -276,17 +278,19 @@ spec: } post { always { - stage ("Cleanup Dev") { - agent { - kubernetes { - label 'jenkins-slave' - defaultContainer 'jnlp' + stages { + stage ("Cleanup Dev") { + agent { + kubernetes { + label 'jenkins-slave' + defaultContainer 'jnlp' + } } - } - stages { - gitlabCommitStatus(name: 'Cleanup Dev') { - script { - load "${env.WORKSPACE}/ci/jenkinsfile/cluster_cleanup_dev.groovy" + stages { + gitlabCommitStatus(name: 'Cleanup Dev') { + script { + load "${env.WORKSPACE}/ci/jenkinsfile/cluster_cleanup_dev.groovy" + } } } } diff --git a/ci/nightly_main_jenkinsfile b/ci/nightly_main_jenkinsfile index 7b39ed386c..bedfce378e 100644 --- a/ci/nightly_main_jenkinsfile +++ b/ci/nightly_main_jenkinsfile @@ -194,17 +194,19 @@ spec: post { always { - stage ("Cleanup Dev") { - agent { - kubernetes { - label 'jenkins-slave' - defaultContainer 'jnlp' + stages { + stage ("Cleanup Dev") { + agent { + kubernetes { + label 'jenkins-slave' + defaultContainer 'jnlp' + } } - } - stages { - gitlabCommitStatus(name: 'Cleanup Dev') { - script { - load "${env.WORKSPACE}/ci/jenkinsfile/cleanup_dev.groovy" + stages { + gitlabCommitStatus(name: 'Cleanup Dev') { + script { + load "${env.WORKSPACE}/ci/jenkinsfile/cleanup_dev.groovy" + } } } } @@ -277,17 +279,19 @@ spec: } post { always { - stage ("Cleanup Dev") { - agent { - kubernetes { - label 'jenkins-slave' - defaultContainer 'jnlp' + stages { + stage ("Cleanup Dev") { + agent { + kubernetes { + label 'jenkins-slave' + defaultContainer 'jnlp' + } } - } - stages { - gitlabCommitStatus(name: 'Cleanup Dev') { - script { - load "${env.WORKSPACE}/ci/jenkinsfile/cluster_cleanup_dev.groovy" + stages { + gitlabCommitStatus(name: 'Cleanup Dev') { + script { + load "${env.WORKSPACE}/ci/jenkinsfile/cluster_cleanup_dev.groovy" + } } } } From 753ba5a1514f2ccdc7939c1364bed5082c1effe3 Mon Sep 17 00:00:00 2001 From: starlord Date: Sun, 14 Jul 2019 15:18:22 +0800 Subject: [PATCH 148/189] Fix SQ8 row count bug Former-commit-id: 7cd10610f84b0398b36909d9cc45bcfa2d77487b --- cpp/src/cache/DataObj.h | 10 ++++++++++ cpp/src/db/DBImpl.cpp | 10 ++++++++-- cpp/src/db/FaissExecutionEngine.cpp | 5 +++-- 3 files changed, 21 insertions(+), 4 deletions(-) diff --git a/cpp/src/cache/DataObj.h b/cpp/src/cache/DataObj.h index 1dff04027e..995711c6ab 100644 --- a/cpp/src/cache/DataObj.h +++ b/cpp/src/cache/DataObj.h @@ -20,6 +20,11 @@ public: : index_(index) {} + DataObj(const engine::Index_ptr& index, int64_t size) + : index_(index), + size_(size) + {} + engine::Index_ptr data() { return index_; } const engine::Index_ptr& data() const { return index_; } @@ -28,11 +33,16 @@ public: return 0; } + if(size_ > 0) { + return size_; + } + return index_->ntotal*(index_->dim*4); } private: engine::Index_ptr index_ = nullptr; + int64_t size_ = 0; }; using DataObjPtr = std::shared_ptr; diff --git a/cpp/src/db/DBImpl.cpp b/cpp/src/db/DBImpl.cpp index beae6c30d3..5025c67adf 100644 --- a/cpp/src/db/DBImpl.cpp +++ b/cpp/src/db/DBImpl.cpp @@ -170,7 +170,10 @@ Status DBImpl::Query(const std::string& table_id, uint64_t k, uint64_t nq, } } - return QueryAsync(table_id, file_id_array, k, nq, vectors, dates, results); + cache::CpuCacheMgr::GetInstance()->PrintInfo(); //print cache info before query + status = QueryAsync(table_id, file_id_array, k, nq, vectors, dates, results); + cache::CpuCacheMgr::GetInstance()->PrintInfo(); //print cache info after query + return status; } Status DBImpl::Query(const std::string& table_id, const std::vector& file_ids, @@ -195,7 +198,10 @@ Status DBImpl::Query(const std::string& table_id, const std::vector return Status::Error("Invalid file id"); } - return QueryAsync(table_id, files_array, k, nq, vectors, dates, results); + cache::CpuCacheMgr::GetInstance()->PrintInfo(); //print cache info before query + status = QueryAsync(table_id, files_array, k, nq, vectors, dates, results); + cache::CpuCacheMgr::GetInstance()->PrintInfo(); //print cache info after query + return status; } Status DBImpl::QueryAsync(const std::string& table_id, const meta::TableFilesSchema& files, diff --git a/cpp/src/db/FaissExecutionEngine.cpp b/cpp/src/db/FaissExecutionEngine.cpp index 7183f43017..e32363dd1d 100644 --- a/cpp/src/db/FaissExecutionEngine.cpp +++ b/cpp/src/db/FaissExecutionEngine.cpp @@ -165,8 +165,9 @@ Status FaissExecutionEngine::Search(long n, } Status FaissExecutionEngine::Cache() { - zilliz::milvus::cache::CpuCacheMgr::GetInstance( - )->InsertItem(location_, std::make_shared(pIndex_)); + auto index = std::make_shared(pIndex_); + cache::DataObjPtr data_obj = std::make_shared(index, PhysicalSize()); + zilliz::milvus::cache::CpuCacheMgr::GetInstance()->InsertItem(location_, data_obj); return Status::OK(); } From c1b312e257f9ce79c7c97e9ae510a70a27de6b0f Mon Sep 17 00:00:00 2001 From: starlord Date: Sun, 14 Jul 2019 16:09:09 +0800 Subject: [PATCH 149/189] Fix unittest error Former-commit-id: 106c17b0afc52ddc577c2bc8262b25dcb23f4902 --- cpp/src/db/Options.cpp | 4 ++++ cpp/src/db/scheduler/task/SearchTask.cpp | 2 +- cpp/src/sdk/examples/simple/src/ClientTest.cpp | 14 ++++++++------ cpp/src/server/RequestTask.cpp | 8 +++++++- 4 files changed, 20 insertions(+), 8 deletions(-) diff --git a/cpp/src/db/Options.cpp b/cpp/src/db/Options.cpp index 5f591dcb0f..81f97978ea 100644 --- a/cpp/src/db/Options.cpp +++ b/cpp/src/db/Options.cpp @@ -41,6 +41,10 @@ void ArchiveConf::ParseCritirias(const std::string& criterias) { } for (auto& token : tokens) { + if(token.empty()) { + continue; + } + std::vector kv; boost::algorithm::split(kv, token, boost::is_any_of(":")); if (kv.size() != 2) { diff --git a/cpp/src/db/scheduler/task/SearchTask.cpp b/cpp/src/db/scheduler/task/SearchTask.cpp index 8036cf986d..5606ba2c84 100644 --- a/cpp/src/db/scheduler/task/SearchTask.cpp +++ b/cpp/src/db/scheduler/task/SearchTask.cpp @@ -107,7 +107,7 @@ Status SearchTask::ClusterResult(const std::vector &output_ids, uint64_t nq, uint64_t topk, SearchContext::ResultSet &result_set) { - if(output_ids.size() != nq*topk || output_distence.size() != nq*topk) { + if(output_ids.size() < nq*topk || output_distence.size() < nq*topk) { std::string msg = "Invalid id array size: " + std::to_string(output_ids.size()) + " distance array size: " + std::to_string(output_distence.size()); SERVER_LOG_ERROR << msg; diff --git a/cpp/src/sdk/examples/simple/src/ClientTest.cpp b/cpp/src/sdk/examples/simple/src/ClientTest.cpp index 495e6ae861..f90b6cfe84 100644 --- a/cpp/src/sdk/examples/simple/src/ClientTest.cpp +++ b/cpp/src/sdk/examples/simple/src/ClientTest.cpp @@ -18,11 +18,12 @@ namespace { static const std::string TABLE_NAME = GetTableName(); static constexpr int64_t TABLE_DIMENSION = 512; - static constexpr int64_t BATCH_ROW_COUNT = 100000; + static constexpr int64_t BATCH_ROW_COUNT = 10000; static constexpr int64_t NQ = 10; static constexpr int64_t TOP_K = 10; static constexpr int64_t SEARCH_TARGET = 5000; //change this value, result is different - static constexpr int64_t ADD_VECTOR_LOOP = 10; + static constexpr int64_t ADD_VECTOR_LOOP = 1; + static constexpr int64_t SECONDS_EACH_HOUR = 3600; #define BLOCK_SPLITER std::cout << "===========================================" << std::endl; @@ -59,7 +60,7 @@ namespace { std::string CurrentTime() { time_t tt; time( &tt ); - tt = tt + 8*3600; + tt = tt + 8*SECONDS_EACH_HOUR; tm* t= gmtime( &tt ); std::string str = std::to_string(t->tm_year + 1900) + "_" + std::to_string(t->tm_mon + 1) @@ -69,10 +70,11 @@ namespace { return str; } - std::string CurrentTmDate() { + std::string CurrentTmDate(int64_t offset_day = 0) { time_t tt; time( &tt ); - tt = tt + 8*3600; + tt = tt + 8*SECONDS_EACH_HOUR; + tt = tt + 24*SECONDS_EACH_HOUR*offset_day; tm* t= gmtime( &tt ); std::string str = std::to_string(t->tm_year + 1900) + "-" + std::to_string(t->tm_mon + 1) @@ -160,7 +162,7 @@ namespace { std::vector query_range_array; Range rg; rg.start_value = CurrentTmDate(); - rg.end_value = CurrentTmDate(); + rg.end_value = CurrentTmDate(1); query_range_array.emplace_back(rg); std::vector record_array; diff --git a/cpp/src/server/RequestTask.cpp b/cpp/src/server/RequestTask.cpp index 3f42b68531..86681623c0 100644 --- a/cpp/src/server/RequestTask.cpp +++ b/cpp/src/server/RequestTask.cpp @@ -110,7 +110,13 @@ namespace { } long days = (tt_end > tt_start) ? (tt_end - tt_start)/DAY_SECONDS : (tt_start - tt_end)/DAY_SECONDS; - for(long i = 0; i <= days; i++) { + if(days == 0) { + error_code = SERVER_INVALID_TIME_RANGE; + error_msg = "Invalid time range: " + range.start_value + " to " + range.end_value; + return ; + } + + for(long i = 0; i < days; i++) { time_t tt_day = tt_start + DAY_SECONDS*i; tm tm_day; CommonUtil::ConvertTime(tt_day, tm_day); From db65b9a0b4423b1717e9de08c1b312441d665d69 Mon Sep 17 00:00:00 2001 From: starlord Date: Sun, 14 Jul 2019 16:10:17 +0800 Subject: [PATCH 150/189] Fix unittest error Former-commit-id: d8dc208227a2904098d899fca71b936a64a528bd --- cpp/src/sdk/examples/simple/src/ClientTest.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cpp/src/sdk/examples/simple/src/ClientTest.cpp b/cpp/src/sdk/examples/simple/src/ClientTest.cpp index f90b6cfe84..5caab0961d 100644 --- a/cpp/src/sdk/examples/simple/src/ClientTest.cpp +++ b/cpp/src/sdk/examples/simple/src/ClientTest.cpp @@ -18,11 +18,11 @@ namespace { static const std::string TABLE_NAME = GetTableName(); static constexpr int64_t TABLE_DIMENSION = 512; - static constexpr int64_t BATCH_ROW_COUNT = 10000; + static constexpr int64_t BATCH_ROW_COUNT = 100000; static constexpr int64_t NQ = 10; static constexpr int64_t TOP_K = 10; static constexpr int64_t SEARCH_TARGET = 5000; //change this value, result is different - static constexpr int64_t ADD_VECTOR_LOOP = 1; + static constexpr int64_t ADD_VECTOR_LOOP = 10; static constexpr int64_t SECONDS_EACH_HOUR = 3600; #define BLOCK_SPLITER std::cout << "===========================================" << std::endl; From 14fb1c408ea34704ce7c939dbba4d8bd3c84eb8f Mon Sep 17 00:00:00 2001 From: Jeff Date: Sun, 14 Jul 2019 16:18:13 +0800 Subject: [PATCH 151/189] Revert "fix cleanup bug" This reverts commit 02fb15b7af3f7007aac7c70440db64a56a4244ca [formerly fcdd0de9624858dbaa62280c8fcaad5e37b8c277] Former-commit-id: ae863c72b256fc2917a4b7ad783e7f43bbbf8e38 --- ci/main_jenkinsfile | 44 +++++++++++++++++-------------------- ci/main_jenkinsfile_no_ut | 44 +++++++++++++++++-------------------- ci/nightly_main_jenkinsfile | 44 +++++++++++++++++-------------------- 3 files changed, 60 insertions(+), 72 deletions(-) diff --git a/ci/main_jenkinsfile b/ci/main_jenkinsfile index cde27f64d0..dc5e879817 100644 --- a/ci/main_jenkinsfile +++ b/ci/main_jenkinsfile @@ -193,19 +193,17 @@ spec: } post { always { - stages { - stage ("Cleanup Dev") { - agent { - kubernetes { - label 'jenkins-slave' - defaultContainer 'jnlp' - } + stage ("Cleanup Dev") { + agent { + kubernetes { + label 'jenkins-slave' + defaultContainer 'jnlp' } - stages { - gitlabCommitStatus(name: 'Cleanup Dev') { - script { - load "${env.WORKSPACE}/ci/jenkinsfile/cleanup_dev.groovy" - } + } + stages { + gitlabCommitStatus(name: 'Cleanup Dev') { + script { + load "${env.WORKSPACE}/ci/jenkinsfile/cleanup_dev.groovy" } } } @@ -278,19 +276,17 @@ spec: } post { always { - stages { - stage ("Cleanup Dev") { - agent { - kubernetes { - label 'jenkins-slave' - defaultContainer 'jnlp' - } + stage ("Cleanup Dev") { + agent { + kubernetes { + label 'jenkins-slave' + defaultContainer 'jnlp' } - stages { - gitlabCommitStatus(name: 'Cleanup Dev') { - script { - load "${env.WORKSPACE}/ci/jenkinsfile/cluster_cleanup_dev.groovy" - } + } + stages { + gitlabCommitStatus(name: 'Cleanup Dev') { + script { + load "${env.WORKSPACE}/ci/jenkinsfile/cluster_cleanup_dev.groovy" } } } diff --git a/ci/main_jenkinsfile_no_ut b/ci/main_jenkinsfile_no_ut index c6a5914baa..35ebc64778 100644 --- a/ci/main_jenkinsfile_no_ut +++ b/ci/main_jenkinsfile_no_ut @@ -193,19 +193,17 @@ spec: } post { always { - stages { - stage ("Cleanup Dev") { - agent { - kubernetes { - label 'jenkins-slave' - defaultContainer 'jnlp' - } + stage ("Cleanup Dev") { + agent { + kubernetes { + label 'jenkins-slave' + defaultContainer 'jnlp' } - stages { - gitlabCommitStatus(name: 'Cleanup Dev') { - script { - load "${env.WORKSPACE}/ci/jenkinsfile/cleanup_dev.groovy" - } + } + stages { + gitlabCommitStatus(name: 'Cleanup Dev') { + script { + load "${env.WORKSPACE}/ci/jenkinsfile/cleanup_dev.groovy" } } } @@ -278,19 +276,17 @@ spec: } post { always { - stages { - stage ("Cleanup Dev") { - agent { - kubernetes { - label 'jenkins-slave' - defaultContainer 'jnlp' - } + stage ("Cleanup Dev") { + agent { + kubernetes { + label 'jenkins-slave' + defaultContainer 'jnlp' } - stages { - gitlabCommitStatus(name: 'Cleanup Dev') { - script { - load "${env.WORKSPACE}/ci/jenkinsfile/cluster_cleanup_dev.groovy" - } + } + stages { + gitlabCommitStatus(name: 'Cleanup Dev') { + script { + load "${env.WORKSPACE}/ci/jenkinsfile/cluster_cleanup_dev.groovy" } } } diff --git a/ci/nightly_main_jenkinsfile b/ci/nightly_main_jenkinsfile index bedfce378e..7b39ed386c 100644 --- a/ci/nightly_main_jenkinsfile +++ b/ci/nightly_main_jenkinsfile @@ -194,19 +194,17 @@ spec: post { always { - stages { - stage ("Cleanup Dev") { - agent { - kubernetes { - label 'jenkins-slave' - defaultContainer 'jnlp' - } + stage ("Cleanup Dev") { + agent { + kubernetes { + label 'jenkins-slave' + defaultContainer 'jnlp' } - stages { - gitlabCommitStatus(name: 'Cleanup Dev') { - script { - load "${env.WORKSPACE}/ci/jenkinsfile/cleanup_dev.groovy" - } + } + stages { + gitlabCommitStatus(name: 'Cleanup Dev') { + script { + load "${env.WORKSPACE}/ci/jenkinsfile/cleanup_dev.groovy" } } } @@ -279,19 +277,17 @@ spec: } post { always { - stages { - stage ("Cleanup Dev") { - agent { - kubernetes { - label 'jenkins-slave' - defaultContainer 'jnlp' - } + stage ("Cleanup Dev") { + agent { + kubernetes { + label 'jenkins-slave' + defaultContainer 'jnlp' } - stages { - gitlabCommitStatus(name: 'Cleanup Dev') { - script { - load "${env.WORKSPACE}/ci/jenkinsfile/cluster_cleanup_dev.groovy" - } + } + stages { + gitlabCommitStatus(name: 'Cleanup Dev') { + script { + load "${env.WORKSPACE}/ci/jenkinsfile/cluster_cleanup_dev.groovy" } } } From 6adf5688076110334ed61aaea497313c19372e27 Mon Sep 17 00:00:00 2001 From: Jeff Date: Sun, 14 Jul 2019 16:19:09 +0800 Subject: [PATCH 152/189] Revert "format clean up stage" This reverts commit f2eb4ca5040ce3348695608e9d36e36ad618ccde [formerly 29cd25f823901ecd46fd174fa26ddb57ef3c3620] Former-commit-id: 53d31f7f21b81ab8c110f115e2b99e6fd4645519 --- ci/jenkinsfile/cleanup_dev.groovy | 6 ++ ci/jenkinsfile/cluster_cleanup_dev.groovy | 6 ++ ci/jenkinsfile/cluster_dev_test.groovy | 2 +- ci/jenkinsfile/dev_test.groovy | 2 +- ci/main_jenkinsfile | 93 +++++++++++------ ci/main_jenkinsfile_no_ut | 115 ++++++++++++++-------- ci/nightly_main_jenkinsfile | 92 +++++++++++------ 7 files changed, 212 insertions(+), 104 deletions(-) diff --git a/ci/jenkinsfile/cleanup_dev.groovy b/ci/jenkinsfile/cleanup_dev.groovy index 664c452f34..056b8fa8c5 100644 --- a/ci/jenkinsfile/cleanup_dev.groovy +++ b/ci/jenkinsfile/cleanup_dev.groovy @@ -1,5 +1,11 @@ try { sh "helm del --purge ${env.JOB_NAME}-${env.BUILD_NUMBER}" + + if (currentBuild.result == 'ABORTED') { + throw new hudson.AbortException("Dev Test Aborted !") + } else if (currentBuild.result == 'FAILURE') { + error("Dev Test Failure !") + } } catch (exc) { sh "helm del --purge ${env.JOB_NAME}-${env.BUILD_NUMBER}" updateGitlabCommitStatus name: 'Cleanup Dev', state: 'failed' diff --git a/ci/jenkinsfile/cluster_cleanup_dev.groovy b/ci/jenkinsfile/cluster_cleanup_dev.groovy index 9708669b9e..dc18067cdd 100644 --- a/ci/jenkinsfile/cluster_cleanup_dev.groovy +++ b/ci/jenkinsfile/cluster_cleanup_dev.groovy @@ -1,5 +1,11 @@ try { sh "helm del --purge ${env.JOB_NAME}-${env.BUILD_NUMBER}-cluster" + + if (currentBuild.result == 'ABORTED') { + throw new hudson.AbortException("Cluster Dev Test Aborted !") + } else if (currentBuild.result == 'FAILURE') { + error("Dev Test Failure !") + } } catch (exc) { sh "helm del --purge ${env.JOB_NAME}-${env.BUILD_NUMBER}-cluster" updateGitlabCommitStatus name: 'Cleanup Dev', state: 'failed' diff --git a/ci/jenkinsfile/cluster_dev_test.groovy b/ci/jenkinsfile/cluster_dev_test.groovy index d3c6a19832..50fd67f803 100644 --- a/ci/jenkinsfile/cluster_dev_test.groovy +++ b/ci/jenkinsfile/cluster_dev_test.groovy @@ -9,8 +9,8 @@ container('milvus-testframework') { } } catch (exc) { updateGitlabCommitStatus name: 'Dev Test', state: 'failed' + currentBuild.result = 'FAILURE' echo 'Milvus Test Failed !' - throw exc } } } diff --git a/ci/jenkinsfile/dev_test.groovy b/ci/jenkinsfile/dev_test.groovy index 4034872615..13e1c868b8 100644 --- a/ci/jenkinsfile/dev_test.groovy +++ b/ci/jenkinsfile/dev_test.groovy @@ -9,8 +9,8 @@ container('milvus-testframework') { } } catch (exc) { updateGitlabCommitStatus name: 'Dev Test', state: 'failed' + currentBuild.result = 'FAILURE' echo 'Milvus Test Failed !' - throw exc } } } diff --git a/ci/main_jenkinsfile b/ci/main_jenkinsfile index dc5e879817..a324ff4441 100644 --- a/ci/main_jenkinsfile +++ b/ci/main_jenkinsfile @@ -191,22 +191,38 @@ spec: } } } + } + + stage ("Cleanup Dev") { + agent { + kubernetes { + label 'jenkins-slave' + defaultContainer 'jnlp' + } + } + stages { + stage('Cleanup') { + steps { + gitlabCommitStatus(name: 'Cleanup Dev') { + script { + load "${env.WORKSPACE}/ci/jenkinsfile/cleanup_dev.groovy" + } + } + } + } + } post { - always { - stage ("Cleanup Dev") { - agent { - kubernetes { - label 'jenkins-slave' - defaultContainer 'jnlp' - } - } - stages { - gitlabCommitStatus(name: 'Cleanup Dev') { - script { - load "${env.WORKSPACE}/ci/jenkinsfile/cleanup_dev.groovy" - } - } - } + aborted { + script { + updateGitlabCommitStatus name: 'Cleanup Dev', state: 'canceled' + echo "Milvus Cleanup Dev aborted !" + } + } + + failure { + script { + updateGitlabCommitStatus name: 'Cleanup Dev', state: 'failed' + echo "Milvus Cleanup Dev failure !" } } } @@ -274,22 +290,38 @@ spec: } } } + } + + stage ("Cleanup Dev") { + agent { + kubernetes { + label 'jenkins-slave' + defaultContainer 'jnlp' + } + } + stages { + stage('Cleanup') { + steps { + gitlabCommitStatus(name: 'Cleanup Dev') { + script { + load "${env.WORKSPACE}/ci/jenkinsfile/cluster_cleanup_dev.groovy" + } + } + } + } + } post { - always { - stage ("Cleanup Dev") { - agent { - kubernetes { - label 'jenkins-slave' - defaultContainer 'jnlp' - } - } - stages { - gitlabCommitStatus(name: 'Cleanup Dev') { - script { - load "${env.WORKSPACE}/ci/jenkinsfile/cluster_cleanup_dev.groovy" - } - } - } + aborted { + script { + updateGitlabCommitStatus name: 'Cleanup Dev', state: 'canceled' + echo "Milvus Cleanup Dev aborted !" + } + } + + failure { + script { + updateGitlabCommitStatus name: 'Cleanup Dev', state: 'failed' + echo "Milvus Cleanup Dev failure !" } } } @@ -343,3 +375,4 @@ spec: } } } + diff --git a/ci/main_jenkinsfile_no_ut b/ci/main_jenkinsfile_no_ut index 35ebc64778..203b185e14 100644 --- a/ci/main_jenkinsfile_no_ut +++ b/ci/main_jenkinsfile_no_ut @@ -191,22 +191,38 @@ spec: } } } + } + + stage ("Cleanup Dev") { + agent { + kubernetes { + label 'jenkins-slave' + defaultContainer 'jnlp' + } + } + stages { + stage('Cleanup') { + steps { + gitlabCommitStatus(name: 'Cleanup Dev') { + script { + load "${env.WORKSPACE}/ci/jenkinsfile/cleanup_dev.groovy" + } + } + } + } + } post { - always { - stage ("Cleanup Dev") { - agent { - kubernetes { - label 'jenkins-slave' - defaultContainer 'jnlp' - } - } - stages { - gitlabCommitStatus(name: 'Cleanup Dev') { - script { - load "${env.WORKSPACE}/ci/jenkinsfile/cleanup_dev.groovy" - } - } - } + aborted { + script { + updateGitlabCommitStatus name: 'Cleanup Dev', state: 'canceled' + echo "Milvus Cleanup Dev aborted !" + } + } + + failure { + script { + updateGitlabCommitStatus name: 'Cleanup Dev', state: 'failed' + echo "Milvus Cleanup Dev failure !" } } } @@ -274,22 +290,38 @@ spec: } } } + } + + stage ("Cleanup Dev") { + agent { + kubernetes { + label 'jenkins-slave' + defaultContainer 'jnlp' + } + } + stages { + stage('Cleanup') { + steps { + gitlabCommitStatus(name: 'Cleanup Dev') { + script { + load "${env.WORKSPACE}/ci/jenkinsfile/cluster_cleanup_dev.groovy" + } + } + } + } + } post { - always { - stage ("Cleanup Dev") { - agent { - kubernetes { - label 'jenkins-slave' - defaultContainer 'jnlp' - } - } - stages { - gitlabCommitStatus(name: 'Cleanup Dev') { - script { - load "${env.WORKSPACE}/ci/jenkinsfile/cluster_cleanup_dev.groovy" - } - } - } + aborted { + script { + updateGitlabCommitStatus name: 'Cleanup Dev', state: 'canceled' + echo "Milvus Cleanup Dev aborted !" + } + } + + failure { + script { + updateGitlabCommitStatus name: 'Cleanup Dev', state: 'failed' + echo "Milvus Cleanup Dev failure !" } } } @@ -305,18 +337,16 @@ spec: post { always { script { - if (env.gitlabAfter != null) { - if (!currentBuild.resultIsBetterOrEqualTo('SUCCESS')) { - // Send an email only if the build status has changed from green/unstable to red - emailext subject: '$DEFAULT_SUBJECT', - body: '$DEFAULT_CONTENT', - recipientProviders: [ - [$class: 'DevelopersRecipientProvider'], - [$class: 'RequesterRecipientProvider'] - ], - replyTo: '$DEFAULT_REPLYTO', - to: '$DEFAULT_RECIPIENTS' - } + if (!currentBuild.resultIsBetterOrEqualTo('SUCCESS')) { + // Send an email only if the build status has changed from green/unstable to red + emailext subject: '$DEFAULT_SUBJECT', + body: '$DEFAULT_CONTENT', + recipientProviders: [ + [$class: 'DevelopersRecipientProvider'], + [$class: 'RequesterRecipientProvider'] + ], + replyTo: '$DEFAULT_REPLYTO', + to: '$DEFAULT_RECIPIENTS' } } } @@ -343,3 +373,4 @@ spec: } } } + diff --git a/ci/nightly_main_jenkinsfile b/ci/nightly_main_jenkinsfile index 7b39ed386c..c375cc87ea 100644 --- a/ci/nightly_main_jenkinsfile +++ b/ci/nightly_main_jenkinsfile @@ -191,23 +191,38 @@ spec: } } } + } + stage ("Cleanup Dev") { + agent { + kubernetes { + label 'jenkins-slave' + defaultContainer 'jnlp' + } + } + stages { + stage('Cleanup') { + steps { + gitlabCommitStatus(name: 'Cleanup Dev') { + script { + load "${env.WORKSPACE}/ci/jenkinsfile/cleanup_dev.groovy" + } + } + } + } + } post { - always { - stage ("Cleanup Dev") { - agent { - kubernetes { - label 'jenkins-slave' - defaultContainer 'jnlp' - } - } - stages { - gitlabCommitStatus(name: 'Cleanup Dev') { - script { - load "${env.WORKSPACE}/ci/jenkinsfile/cleanup_dev.groovy" - } - } - } + aborted { + script { + updateGitlabCommitStatus name: 'Cleanup Dev', state: 'canceled' + echo "Milvus Cleanup Dev aborted !" + } + } + + failure { + script { + updateGitlabCommitStatus name: 'Cleanup Dev', state: 'failed' + echo "Milvus Cleanup Dev failure !" } } } @@ -275,22 +290,38 @@ spec: } } } + } + + stage ("Cleanup Dev") { + agent { + kubernetes { + label 'jenkins-slave' + defaultContainer 'jnlp' + } + } + stages { + stage('Cleanup') { + steps { + gitlabCommitStatus(name: 'Cleanup Dev') { + script { + load "${env.WORKSPACE}/ci/jenkinsfile/cluster_cleanup_dev.groovy" + } + } + } + } + } post { - always { - stage ("Cleanup Dev") { - agent { - kubernetes { - label 'jenkins-slave' - defaultContainer 'jnlp' - } - } - stages { - gitlabCommitStatus(name: 'Cleanup Dev') { - script { - load "${env.WORKSPACE}/ci/jenkinsfile/cluster_cleanup_dev.groovy" - } - } - } + aborted { + script { + updateGitlabCommitStatus name: 'Cleanup Dev', state: 'canceled' + echo "Milvus Cleanup Dev aborted !" + } + } + + failure { + script { + updateGitlabCommitStatus name: 'Cleanup Dev', state: 'failed' + echo "Milvus Cleanup Dev failure !" } } } @@ -342,3 +373,4 @@ spec: } } } + From 46d64af09c567695bb513e0d0dc615d120a5dd3a Mon Sep 17 00:00:00 2001 From: starlord Date: Sun, 14 Jul 2019 16:46:05 +0800 Subject: [PATCH 153/189] Fix small size index build crash Former-commit-id: d1255df5fc62aa037f4670049cccfbb8a9851732 --- cpp/src/wrapper/Operand.cpp | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/cpp/src/wrapper/Operand.cpp b/cpp/src/wrapper/Operand.cpp index 62a863ae0e..49d007ba2a 100644 --- a/cpp/src/wrapper/Operand.cpp +++ b/cpp/src/wrapper/Operand.cpp @@ -28,6 +28,15 @@ IndexType resolveIndexType(const string &index_type) { return IndexType::Invalid_Option; } +int CalcBacketCount(int nb, size_t nlist) { + int backet_count = int(nb / 1000000.0 * nlist); + if(backet_count == 0) { + backet_count = 1; //avoid faiss rash + } + + return backet_count; +} + // nb at least 100 string Operand::get_index_type(const int &nb) { if (!index_str.empty()) { return index_str; } @@ -45,7 +54,7 @@ string Operand::get_index_type(const int &nb) { size_t nlist = engine_config.GetInt32Value(CONFIG_NLIST, 16384); index_str += (ncent != 0 ? index_type + std::to_string(ncent) : - index_type + std::to_string(int(nb / 1000000.0 * nlist))); + index_type + std::to_string(CalcBacketCount(nb, nlist))); // std::cout<<"nlist = "< Date: Sun, 14 Jul 2019 20:43:32 +0800 Subject: [PATCH 155/189] fix labels error Former-commit-id: c3e6ca2d5ff47b79e0210f9f5475974cc3e050d9 --- ci/main_jenkinsfile | 4 ++-- ci/main_jenkinsfile_no_ut | 4 ++-- ci/nightly_main_jenkinsfile | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/ci/main_jenkinsfile b/ci/main_jenkinsfile index 8c636a7981..9577837bfd 100644 --- a/ci/main_jenkinsfile +++ b/ci/main_jenkinsfile @@ -134,7 +134,7 @@ spec: stage("Single Node") { agent { kubernetes { - label 'dev test' + label 'dev-test' defaultContainer 'jnlp' yaml """ apiVersion: v1 @@ -227,7 +227,7 @@ spec: stage("Cluster") { agent { kubernetes { - label 'dev test' + label 'dev-test' defaultContainer 'jnlp' yaml """ apiVersion: v1 diff --git a/ci/main_jenkinsfile_no_ut b/ci/main_jenkinsfile_no_ut index 0642a3723b..082433650c 100644 --- a/ci/main_jenkinsfile_no_ut +++ b/ci/main_jenkinsfile_no_ut @@ -134,7 +134,7 @@ spec: stage("Single Node") { agent { kubernetes { - label 'dev test' + label 'dev-test' defaultContainer 'jnlp' yaml """ apiVersion: v1 @@ -227,7 +227,7 @@ spec: stage("Cluster") { agent { kubernetes { - label 'dev test' + label 'dev-test' defaultContainer 'jnlp' yaml """ apiVersion: v1 diff --git a/ci/nightly_main_jenkinsfile b/ci/nightly_main_jenkinsfile index 89b4a14985..7f59561f82 100644 --- a/ci/nightly_main_jenkinsfile +++ b/ci/nightly_main_jenkinsfile @@ -134,7 +134,7 @@ spec: stage("Single Node") { agent { kubernetes { - label 'dev test' + label 'dev-test' defaultContainer 'jnlp' yaml """ apiVersion: v1 @@ -227,7 +227,7 @@ spec: stage("Cluster") { agent { kubernetes { - label 'dev test' + label 'dev-test' defaultContainer 'jnlp' yaml """ apiVersion: v1 From 2abe69ffb51f1b77e4fd3c551ee0bf061f117fb0 Mon Sep 17 00:00:00 2001 From: quicksilver Date: Sun, 14 Jul 2019 21:08:49 +0800 Subject: [PATCH 156/189] fix clean up bug Former-commit-id: 041f4f5f8beb650d9d0277d4745ef0d1b8820866 --- ci/jenkinsfile/cleanup_dev.groovy | 6 +++--- ci/jenkinsfile/cluster_cleanup_dev.groovy | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/ci/jenkinsfile/cleanup_dev.groovy b/ci/jenkinsfile/cleanup_dev.groovy index 1b7e67e1b0..2e9332fa6e 100644 --- a/ci/jenkinsfile/cleanup_dev.groovy +++ b/ci/jenkinsfile/cleanup_dev.groovy @@ -1,11 +1,11 @@ try { - def result = sh script: "helm status ${env.JOB_NAME}-${env.BUILD_NUMBER}", returnStatus: true - if (result) { + def result = sh script: "helm status ${env.JOB_NAME}-${env.BUILD_NUMBER}", returnStatus: true + if (!result) { sh "helm del --purge ${env.JOB_NAME}-${env.BUILD_NUMBER}" } } catch (exc) { def result = sh script: "helm status ${env.JOB_NAME}-${env.BUILD_NUMBER}", returnStatus: true - if (result) { + if (!result) { sh "helm del --purge ${env.JOB_NAME}-${env.BUILD_NUMBER}" } throw exc diff --git a/ci/jenkinsfile/cluster_cleanup_dev.groovy b/ci/jenkinsfile/cluster_cleanup_dev.groovy index 9a73d0a291..e57988fefe 100644 --- a/ci/jenkinsfile/cluster_cleanup_dev.groovy +++ b/ci/jenkinsfile/cluster_cleanup_dev.groovy @@ -1,11 +1,11 @@ try { def result = sh script: "helm status ${env.JOB_NAME}-${env.BUILD_NUMBER}-cluster", returnStatus: true - if (result) { + if (!result) { sh "helm del --purge ${env.JOB_NAME}-${env.BUILD_NUMBER}-cluster" } } catch (exc) { def result = sh script: "helm status ${env.JOB_NAME}-${env.BUILD_NUMBER}-cluster", returnStatus: true - if (result) { + if (!result) { sh "helm del --purge ${env.JOB_NAME}-${env.BUILD_NUMBER}-cluster" } throw exc From f9b0050cdb67edbc9b1a2fee6107063aacddbb71 Mon Sep 17 00:00:00 2001 From: quicksilver Date: Sun, 14 Jul 2019 21:50:48 +0800 Subject: [PATCH 157/189] MS-215 - add milvus cluster CI/CD groovy file Former-commit-id: 61d3f98595be12a26c8563ee369d0f5526c407e8 --- ci/jenkinsfile/cleanup_dev.groovy | 15 +- ci/jenkinsfile/cluster_cleanup_dev.groovy | 15 +- ci/jenkinsfile/cluster_deploy2dev.groovy | 9 +- ci/jenkinsfile/cluster_dev_test.groovy | 24 +- ci/jenkinsfile/deploy2dev.groovy | 16 +- ci/jenkinsfile/dev_test.groovy | 24 +- .../upload_dev_cluster_test_out.groovy | 32 +- ci/jenkinsfile/upload_dev_test_out.groovy | 31 +- ci/main_jenkinsfile | 297 +++++++++--------- ci/main_jenkinsfile_no_ut | 297 +++++++++--------- ci/nightly_main_jenkinsfile | 275 ++++++++-------- 11 files changed, 484 insertions(+), 551 deletions(-) diff --git a/ci/jenkinsfile/cleanup_dev.groovy b/ci/jenkinsfile/cleanup_dev.groovy index 056b8fa8c5..2e9332fa6e 100644 --- a/ci/jenkinsfile/cleanup_dev.groovy +++ b/ci/jenkinsfile/cleanup_dev.groovy @@ -1,14 +1,13 @@ try { - sh "helm del --purge ${env.JOB_NAME}-${env.BUILD_NUMBER}" - - if (currentBuild.result == 'ABORTED') { - throw new hudson.AbortException("Dev Test Aborted !") - } else if (currentBuild.result == 'FAILURE') { - error("Dev Test Failure !") + def result = sh script: "helm status ${env.JOB_NAME}-${env.BUILD_NUMBER}", returnStatus: true + if (!result) { + sh "helm del --purge ${env.JOB_NAME}-${env.BUILD_NUMBER}" } } catch (exc) { - sh "helm del --purge ${env.JOB_NAME}-${env.BUILD_NUMBER}" - updateGitlabCommitStatus name: 'Cleanup Dev', state: 'failed' + def result = sh script: "helm status ${env.JOB_NAME}-${env.BUILD_NUMBER}", returnStatus: true + if (!result) { + sh "helm del --purge ${env.JOB_NAME}-${env.BUILD_NUMBER}" + } throw exc } diff --git a/ci/jenkinsfile/cluster_cleanup_dev.groovy b/ci/jenkinsfile/cluster_cleanup_dev.groovy index dc18067cdd..e57988fefe 100644 --- a/ci/jenkinsfile/cluster_cleanup_dev.groovy +++ b/ci/jenkinsfile/cluster_cleanup_dev.groovy @@ -1,14 +1,13 @@ try { - sh "helm del --purge ${env.JOB_NAME}-${env.BUILD_NUMBER}-cluster" - - if (currentBuild.result == 'ABORTED') { - throw new hudson.AbortException("Cluster Dev Test Aborted !") - } else if (currentBuild.result == 'FAILURE') { - error("Dev Test Failure !") + def result = sh script: "helm status ${env.JOB_NAME}-${env.BUILD_NUMBER}-cluster", returnStatus: true + if (!result) { + sh "helm del --purge ${env.JOB_NAME}-${env.BUILD_NUMBER}-cluster" } } catch (exc) { - sh "helm del --purge ${env.JOB_NAME}-${env.BUILD_NUMBER}-cluster" - updateGitlabCommitStatus name: 'Cleanup Dev', state: 'failed' + def result = sh script: "helm status ${env.JOB_NAME}-${env.BUILD_NUMBER}-cluster", returnStatus: true + if (!result) { + sh "helm del --purge ${env.JOB_NAME}-${env.BUILD_NUMBER}-cluster" + } throw exc } diff --git a/ci/jenkinsfile/cluster_deploy2dev.groovy b/ci/jenkinsfile/cluster_deploy2dev.groovy index 97062922ca..cde361b9ea 100644 --- a/ci/jenkinsfile/cluster_deploy2dev.groovy +++ b/ci/jenkinsfile/cluster_deploy2dev.groovy @@ -3,17 +3,18 @@ try { sh 'helm repo add milvus https://registry.zilliz.com/chartrepo/milvus' sh 'helm repo update' dir ("milvus-helm") { - checkout([$class: 'GitSCM', branches: [[name: "${SEMVER}"]], doGenerateSubmoduleConfigurations: false, extensions: [[$class: 'SubmoduleOption',disableSubmodules: false,parentCredentials: true,recursiveSubmodules: true,reference: '',trackingSubmodules: false]], submoduleCfg: [], userRemoteConfigs: [[credentialsId: "${params.GIT_USER}", url: "git@192.168.1.105:megasearch/milvus-helm.git", name: 'origin', refspec: "+refs/heads/${SEMVER}:refs/remotes/origin/${SEMVER}"]]]) + checkout([$class: 'GitSCM', branches: [[name: "${SEMVER}"]], doGenerateSubmoduleConfigurations: false, extensions: [], submoduleCfg: [], userRemoteConfigs: [[credentialsId: "${params.GIT_USER}", url: "git@192.168.1.105:megasearch/milvus-helm.git", name: 'origin', refspec: "+refs/heads/${SEMVER}:refs/remotes/origin/${SEMVER}"]]]) dir ("milvus/milvus-cluster") { - sh "helm install --set roServers.image.tag=${DOCKER_VERSION} --set woServers.image.tag=${DOCKER_VERSION} --set expose.type=clusterIP -f ci/values.yaml --name ${env.JOB_NAME}-${env.BUILD_NUMBER}-cluster --namespace milvus-cluster --version 0.1.0 . " + sh "helm install --set roServers.image.tag=${DOCKER_VERSION} --set woServers.image.tag=${DOCKER_VERSION} --set expose.type=clusterIP -f ci/values.yaml --name ${env.JOB_NAME}-${env.BUILD_NUMBER}-cluster --namespace milvus-cluster --version 0.3.1 . " } + } + timeout(time: 2, unit: 'MINUTES') { waitUntil { - def result = sh script: "nc -z -w 2 ${env.JOB_NAME}-${env.BUILD_NUMBER}-cluster-milvus-cluster-proxy.milvus-cluster.svc.cluster.local 19530", returnStatus: true + def result = sh script: "nc -z -w 3 ${env.JOB_NAME}-${env.BUILD_NUMBER}-cluster-milvus-cluster-proxy.milvus-cluster.svc.cluster.local 19530", returnStatus: true return !result } } } catch (exc) { - updateGitlabCommitStatus name: 'Deloy to Dev', state: 'failed' echo 'Helm running failed!' sh "helm del --purge ${env.JOB_NAME}-${env.BUILD_NUMBER}-cluster" throw exc diff --git a/ci/jenkinsfile/cluster_dev_test.groovy b/ci/jenkinsfile/cluster_dev_test.groovy index 50fd67f803..8093f67d18 100644 --- a/ci/jenkinsfile/cluster_dev_test.groovy +++ b/ci/jenkinsfile/cluster_dev_test.groovy @@ -1,18 +1,12 @@ -container('milvus-testframework') { - timeout(time: 10, unit: 'MINUTES') { - gitlabCommitStatus(name: 'Dev Test') { - try { - dir ("${PROJECT_NAME}_test") { - checkout([$class: 'GitSCM', branches: [[name: "${SEMVER}"]], doGenerateSubmoduleConfigurations: false, extensions: [], submoduleCfg: [], userRemoteConfigs: [[credentialsId: "${params.GIT_USER}", url: "git@192.168.1.105:Test/milvus_test.git", name: 'origin', refspec: "+refs/heads/${SEMVER}:refs/remotes/origin/${SEMVER}"]]]) - sh 'python3 -m pip install -r requirements.txt' - sh "pytest . --alluredir=cluster_test_out --ip ${env.JOB_NAME}-${env.BUILD_NUMBER}-cluster-milvus-cluster-proxy.milvus-cluster.svc.cluster.local" - } - } catch (exc) { - updateGitlabCommitStatus name: 'Dev Test', state: 'failed' - currentBuild.result = 'FAILURE' - echo 'Milvus Test Failed !' - } +timeout(time: 10, unit: 'MINUTES') { + try { + dir ("${PROJECT_NAME}_test") { + checkout([$class: 'GitSCM', branches: [[name: "${SEMVER}"]], doGenerateSubmoduleConfigurations: false, extensions: [], submoduleCfg: [], userRemoteConfigs: [[credentialsId: "${params.GIT_USER}", url: "git@192.168.1.105:Test/milvus_test.git", name: 'origin', refspec: "+refs/heads/${SEMVER}:refs/remotes/origin/${SEMVER}"]]]) + sh 'python3 -m pip install -r requirements.txt' + sh "pytest . --alluredir=cluster_test_out --ip ${env.JOB_NAME}-${env.BUILD_NUMBER}-cluster-milvus-cluster-proxy.milvus-cluster.svc.cluster.local" } + } catch (exc) { + echo 'Milvus Test Failed !' + throw exc } } - diff --git a/ci/jenkinsfile/deploy2dev.groovy b/ci/jenkinsfile/deploy2dev.groovy index f594d0ba57..b0c537f2e1 100644 --- a/ci/jenkinsfile/deploy2dev.groovy +++ b/ci/jenkinsfile/deploy2dev.groovy @@ -2,13 +2,19 @@ try { sh 'helm init --client-only --skip-refresh --stable-repo-url https://kubernetes.oss-cn-hangzhou.aliyuncs.com/charts' sh 'helm repo add milvus https://registry.zilliz.com/chartrepo/milvus' sh 'helm repo update' - sh "helm install --set engine.image.tag=${DOCKER_VERSION} --set expose.type=clusterIP --name ${env.JOB_NAME}-${env.BUILD_NUMBER} --version 0.3.0 milvus/milvus-gpu" - waitUntil { - def result = sh script: "nc -z -w 2 ${env.JOB_NAME}-${env.BUILD_NUMBER}-milvus-gpu-engine.kube-opt.svc.cluster.local 19530", returnStatus: true - return !result + dir ("milvus-helm") { + checkout([$class: 'GitSCM', branches: [[name: "${SEMVER}"]], doGenerateSubmoduleConfigurations: false, extensions: [], submoduleCfg: [], userRemoteConfigs: [[credentialsId: "${params.GIT_USER}", url: "git@192.168.1.105:megasearch/milvus-helm.git", name: 'origin', refspec: "+refs/heads/${SEMVER}:refs/remotes/origin/${SEMVER}"]]]) + dir ("milvus/milvus-gpu") { + sh "helm install --set engine.image.tag=${DOCKER_VERSION} --set expose.type=clusterIP --name ${env.JOB_NAME}-${env.BUILD_NUMBER} --version 0.3.1 ." + } + } + timeout(time: 2, unit: 'MINUTES') { + waitUntil { + def result = sh script: "nc -z -w 3 ${env.JOB_NAME}-${env.BUILD_NUMBER}-milvus-gpu-engine.kube-opt.svc.cluster.local 19530", returnStatus: true + return !result + } } } catch (exc) { - updateGitlabCommitStatus name: 'Deloy to Dev', state: 'failed' echo 'Helm running failed!' sh "helm del --purge ${env.JOB_NAME}-${env.BUILD_NUMBER}" throw exc diff --git a/ci/jenkinsfile/dev_test.groovy b/ci/jenkinsfile/dev_test.groovy index 13e1c868b8..836ddbad05 100644 --- a/ci/jenkinsfile/dev_test.groovy +++ b/ci/jenkinsfile/dev_test.groovy @@ -1,18 +1,12 @@ -container('milvus-testframework') { - timeout(time: 10, unit: 'MINUTES') { - gitlabCommitStatus(name: 'Dev Test') { - try { - dir ("${PROJECT_NAME}_test") { - checkout([$class: 'GitSCM', branches: [[name: "${SEMVER}"]], doGenerateSubmoduleConfigurations: false, extensions: [], submoduleCfg: [], userRemoteConfigs: [[credentialsId: "${params.GIT_USER}", url: "git@192.168.1.105:Test/milvus_test.git", name: 'origin', refspec: "+refs/heads/${SEMVER}:refs/remotes/origin/${SEMVER}"]]]) - sh 'python3 -m pip install -r requirements.txt' - sh "pytest . --alluredir=test_out --ip ${env.JOB_NAME}-${env.BUILD_NUMBER}-milvus-gpu-engine.kube-opt.svc.cluster.local" - } - } catch (exc) { - updateGitlabCommitStatus name: 'Dev Test', state: 'failed' - currentBuild.result = 'FAILURE' - echo 'Milvus Test Failed !' - } +timeout(time: 10, unit: 'MINUTES') { + try { + dir ("${PROJECT_NAME}_test") { + checkout([$class: 'GitSCM', branches: [[name: "${SEMVER}"]], doGenerateSubmoduleConfigurations: false, extensions: [], submoduleCfg: [], userRemoteConfigs: [[credentialsId: "${params.GIT_USER}", url: "git@192.168.1.105:Test/milvus_test.git", name: 'origin', refspec: "+refs/heads/${SEMVER}:refs/remotes/origin/${SEMVER}"]]]) + sh 'python3 -m pip install -r requirements.txt' + sh "pytest . --alluredir=test_out --ip ${env.JOB_NAME}-${env.BUILD_NUMBER}-milvus-gpu-engine.kube-opt.svc.cluster.local" } + } catch (exc) { + echo 'Milvus Test Failed !' + throw exc } } - diff --git a/ci/jenkinsfile/upload_dev_cluster_test_out.groovy b/ci/jenkinsfile/upload_dev_cluster_test_out.groovy index 52fecc6c1d..6bbd8a649f 100644 --- a/ci/jenkinsfile/upload_dev_cluster_test_out.groovy +++ b/ci/jenkinsfile/upload_dev_cluster_test_out.groovy @@ -1,26 +1,14 @@ -container('milvus-testframework') { - timeout(time: 5, unit: 'MINUTES') { - dir ("${PROJECT_NAME}_test") { - gitlabCommitStatus(name: 'Upload Dev Test Out') { - if (fileExists('cluster_test_out')) { - try { - def fileTransfer = load "${env.WORKSPACE}/ci/function/file_transfer.groovy" - fileTransfer.FileTransfer("cluster_test_out/", "${PROJECT_NAME}/test/${JOB_NAME}-${BUILD_ID}", 'nas storage') - if (currentBuild.resultIsBetterOrEqualTo('SUCCESS')) { - echo "Milvus Dev Test Out Viewer \"ftp://192.168.1.126/data/${PROJECT_NAME}/test/${JOB_NAME}-${BUILD_ID}\"" - } - } catch (hudson.AbortException ae) { - updateGitlabCommitStatus name: 'Upload Dev Test Out', state: 'canceled' - currentBuild.result = 'ABORTED' - } catch (exc) { - updateGitlabCommitStatus name: 'Upload Dev Test Out', state: 'failed' - currentBuild.result = 'FAILURE' - } - } else { - updateGitlabCommitStatus name: 'Upload Dev Test Out', state: 'failed' - echo "Milvus Dev Test Out directory don't exists!" - } +timeout(time: 5, unit: 'MINUTES') { + dir ("${PROJECT_NAME}_test") { + if (fileExists('cluster_test_out')) { + def fileTransfer = load "${env.WORKSPACE}/ci/function/file_transfer.groovy" + fileTransfer.FileTransfer("cluster_test_out/", "${PROJECT_NAME}/test/${JOB_NAME}-${BUILD_ID}", 'nas storage') + if (currentBuild.resultIsBetterOrEqualTo('SUCCESS')) { + echo "Milvus Dev Test Out Viewer \"ftp://192.168.1.126/data/${PROJECT_NAME}/test/${JOB_NAME}-${BUILD_ID}\"" } + } else { + error("Milvus Dev Test Out directory don't exists!") } } } + diff --git a/ci/jenkinsfile/upload_dev_test_out.groovy b/ci/jenkinsfile/upload_dev_test_out.groovy index c401b16608..7e106c0296 100644 --- a/ci/jenkinsfile/upload_dev_test_out.groovy +++ b/ci/jenkinsfile/upload_dev_test_out.groovy @@ -1,26 +1,13 @@ -container('milvus-testframework') { - timeout(time: 5, unit: 'MINUTES') { - dir ("${PROJECT_NAME}_test") { - gitlabCommitStatus(name: 'Upload Dev Test Out') { - if (fileExists('test_out')) { - try { - def fileTransfer = load "${env.WORKSPACE}/ci/function/file_transfer.groovy" - fileTransfer.FileTransfer("test_out/", "${PROJECT_NAME}/test/${JOB_NAME}-${BUILD_ID}", 'nas storage') - if (currentBuild.resultIsBetterOrEqualTo('SUCCESS')) { - echo "Milvus Dev Test Out Viewer \"ftp://192.168.1.126/data/${PROJECT_NAME}/test/${JOB_NAME}-${BUILD_ID}\"" - } - } catch (hudson.AbortException ae) { - updateGitlabCommitStatus name: 'Upload Dev Test Out', state: 'canceled' - currentBuild.result = 'ABORTED' - } catch (exc) { - updateGitlabCommitStatus name: 'Upload Dev Test Out', state: 'failed' - currentBuild.result = 'FAILURE' - } - } else { - updateGitlabCommitStatus name: 'Upload Dev Test Out', state: 'failed' - echo "Milvus Dev Test Out directory don't exists!" - } +timeout(time: 5, unit: 'MINUTES') { + dir ("${PROJECT_NAME}_test") { + if (fileExists('test_out')) { + def fileTransfer = load "${env.WORKSPACE}/ci/function/file_transfer.groovy" + fileTransfer.FileTransfer("test_out/", "${PROJECT_NAME}/test/${JOB_NAME}-${BUILD_ID}", 'nas storage') + if (currentBuild.resultIsBetterOrEqualTo('SUCCESS')) { + echo "Milvus Dev Test Out Viewer \"ftp://192.168.1.126/data/${PROJECT_NAME}/test/${JOB_NAME}-${BUILD_ID}\"" } + } else { + error("Milvus Dev Test Out directory don't exists!") } } } diff --git a/ci/main_jenkinsfile b/ci/main_jenkinsfile index 1f0d0c3d7f..9577837bfd 100644 --- a/ci/main_jenkinsfile +++ b/ci/main_jenkinsfile @@ -131,59 +131,53 @@ spec: stage("Deploy to Development") { parallel { - stage("Single") { + stage("Single Node") { + agent { + kubernetes { + label 'dev-test' + defaultContainer 'jnlp' + yaml """ +apiVersion: v1 +kind: Pod +metadata: + labels: + app: milvus + componet: test +spec: + containers: + - name: milvus-testframework + image: registry.zilliz.com/milvus/milvus-test:v0.2 + command: + - cat + tty: true + volumeMounts: + - name: kubeconf + mountPath: /root/.kube/ + readOnly: true + volumes: + - name: kubeconf + secret: + secretName: test-cluster-config +""" + } + } + stages { stage("Deploy to Dev") { - agent { - kubernetes { - label 'jenkins-slave' - defaultContainer 'jnlp' - } - } - stages { - stage('Deploy') { - steps { - gitlabCommitStatus(name: 'Deloy to Dev') { - script { - load "${env.WORKSPACE}/ci/jenkinsfile/deploy2dev.groovy" - } + steps { + gitlabCommitStatus(name: 'Deloy to Dev') { + container('milvus-testframework') { + script { + load "${env.WORKSPACE}/ci/jenkinsfile/deploy2dev.groovy" } } } } - post { - aborted { - script { - updateGitlabCommitStatus name: 'Deloy to Dev', state: 'canceled' - echo "Milvus Deloy to Dev aborted !" - } - } - - failure { - script { - updateGitlabCommitStatus name: 'Deloy to Dev', state: 'failed' - echo "Milvus Deloy to Dev failure !" - } - } - } } - stage("Dev Test") { - agent { - kubernetes { - label 'test' - defaultContainer 'jnlp' - containerTemplate { - name 'milvus-testframework' - image 'registry.zilliz.com/milvus/milvus-test:v0.1' - ttyEnabled true - command 'cat' - } - } - } - stages { - stage('Test') { - steps { + steps { + gitlabCommitStatus(name: 'Deloy Test') { + container('milvus-testframework') { script { load "${env.WORKSPACE}/ci/jenkinsfile/dev_test.groovy" load "${env.WORKSPACE}/ci/jenkinsfile/upload_dev_test_out.groovy" @@ -192,140 +186,133 @@ spec: } } } - stage ("Cleanup Dev") { - agent { - kubernetes { - label 'jenkins-slave' - defaultContainer 'jnlp' - } - } - stages { - stage('Cleanup') { - steps { - gitlabCommitStatus(name: 'Cleanup Dev') { - script { - load "${env.WORKSPACE}/ci/jenkinsfile/cleanup_dev.groovy" - } + steps { + gitlabCommitStatus(name: 'Cleanup Dev') { + container('milvus-testframework') { + script { + load "${env.WORKSPACE}/ci/jenkinsfile/cleanup_dev.groovy" } } } } - post { - aborted { - script { - updateGitlabCommitStatus name: 'Cleanup Dev', state: 'canceled' - echo "Milvus Cleanup Dev aborted !" - } - } - - failure { - script { - updateGitlabCommitStatus name: 'Cleanup Dev', state: 'failed' - echo "Milvus Cleanup Dev failure !" - } + } + } + post { + always { + container('milvus-testframework') { + script { + load "${env.WORKSPACE}/ci/jenkinsfile/cleanup_dev.groovy" } } } + success { + script { + echo "Milvus Single Node CI/CD success !" + } + } + aborted { + script { + echo "Milvus Single Node CI/CD aborted !" + } + } + failure { + script { + echo "Milvus Single Node CI/CD failure !" + } + } } } stage("Cluster") { + agent { + kubernetes { + label 'dev-test' + defaultContainer 'jnlp' + yaml """ +apiVersion: v1 +kind: Pod +metadata: + labels: + app: milvus + componet: test +spec: + containers: + - name: milvus-testframework + image: registry.zilliz.com/milvus/milvus-test:v0.2 + command: + - cat + tty: true + volumeMounts: + - name: kubeconf + mountPath: /root/.kube/ + readOnly: true + volumes: + - name: kubeconf + secret: + secretName: test-cluster-config +""" + } + } stages { stage("Deploy to Dev") { - agent { - kubernetes { - label 'jenkins-slave' - defaultContainer 'jnlp' - } - } - stages { - stage('Deploy') { - steps { - gitlabCommitStatus(name: 'Deloy to Dev') { - script { - load "${env.WORKSPACE}/ci/jenkinsfile/cluster_deploy2dev.groovy" - } + steps { + gitlabCommitStatus(name: 'Deloy to Dev') { + container('milvus-testframework') { + script { + load "${env.WORKSPACE}/ci/jenkinsfile/cluster_deploy2dev.groovy" } } } } - post { - aborted { - script { - updateGitlabCommitStatus name: 'Deloy to Dev', state: 'canceled' - echo "Milvus Deloy to Dev aborted !" - } - } - - failure { - script { - updateGitlabCommitStatus name: 'Deloy to Dev', state: 'failed' - echo "Milvus Deloy to Dev failure !" - } - } - } } - stage("Dev Test") { - agent { - kubernetes { - label 'test' - defaultContainer 'jnlp' - containerTemplate { - name 'milvus-testframework' - image 'registry.zilliz.com/milvus/milvus-test:v0.1' - ttyEnabled true - command 'cat' - } - } - } - stages { - stage('Test') { - steps { + steps { + gitlabCommitStatus(name: 'Deloy Test') { + container('milvus-testframework') { script { load "${env.WORKSPACE}/ci/jenkinsfile/cluster_dev_test.groovy" - load "${env.WORKSPACE}/ci/jenkinsfile/upload_cluster_dev_test_out.groovy" + load "${env.WORKSPACE}/ci/jenkinsfile/upload_dev_cluster_test_out.groovy" } } } } } - stage ("Cleanup Dev") { - agent { - kubernetes { - label 'jenkins-slave' - defaultContainer 'jnlp' - } - } - stages { - stage('Cleanup') { - steps { - gitlabCommitStatus(name: 'Cleanup Dev') { - script { - load "${env.WORKSPACE}/ci/jenkinsfile/cluster_cleanup_dev.groovy" - } + steps { + gitlabCommitStatus(name: 'Cleanup Dev') { + container('milvus-testframework') { + script { + load "${env.WORKSPACE}/ci/jenkinsfile/cluster_cleanup_dev.groovy" } } } } - post { - aborted { - script { - updateGitlabCommitStatus name: 'Cleanup Dev', state: 'canceled' - echo "Milvus Cleanup Dev aborted !" - } - } - - failure { - script { - updateGitlabCommitStatus name: 'Cleanup Dev', state: 'failed' - echo "Milvus Cleanup Dev failure !" - } + } + } + post { + always { + container('milvus-testframework') { + script { + load "${env.WORKSPACE}/ci/jenkinsfile/cluster_cleanup_dev.groovy" } } } + success { + script { + echo "Milvus Cluster CI/CD success !" + } + } + aborted { + script { + echo "Milvus Cluster CI/CD aborted !" + } + } + failure { + script { + echo "Milvus Cluster CI/CD failure !" + } + } } } } @@ -337,16 +324,18 @@ spec: post { always { script { - if (!currentBuild.resultIsBetterOrEqualTo('SUCCESS')) { - // Send an email only if the build status has changed from green/unstable to red - emailext subject: '$DEFAULT_SUBJECT', - body: '$DEFAULT_CONTENT', - recipientProviders: [ - [$class: 'DevelopersRecipientProvider'], - [$class: 'RequesterRecipientProvider'] - ], - replyTo: '$DEFAULT_REPLYTO', - to: '$DEFAULT_RECIPIENTS' + if (env.gitlabAfter != null) { + if (!currentBuild.resultIsBetterOrEqualTo('SUCCESS')) { + // Send an email only if the build status has changed from green/unstable to red + emailext subject: '$DEFAULT_SUBJECT', + body: '$DEFAULT_CONTENT', + recipientProviders: [ + [$class: 'DevelopersRecipientProvider'], + [$class: 'RequesterRecipientProvider'] + ], + replyTo: '$DEFAULT_REPLYTO', + to: '$DEFAULT_RECIPIENTS' + } } } } diff --git a/ci/main_jenkinsfile_no_ut b/ci/main_jenkinsfile_no_ut index 203b185e14..082433650c 100644 --- a/ci/main_jenkinsfile_no_ut +++ b/ci/main_jenkinsfile_no_ut @@ -131,59 +131,53 @@ spec: stage("Deploy to Development") { parallel { - stage("Single") { + stage("Single Node") { + agent { + kubernetes { + label 'dev-test' + defaultContainer 'jnlp' + yaml """ +apiVersion: v1 +kind: Pod +metadata: + labels: + app: milvus + componet: test +spec: + containers: + - name: milvus-testframework + image: registry.zilliz.com/milvus/milvus-test:v0.2 + command: + - cat + tty: true + volumeMounts: + - name: kubeconf + mountPath: /root/.kube/ + readOnly: true + volumes: + - name: kubeconf + secret: + secretName: test-cluster-config +""" + } + } + stages { stage("Deploy to Dev") { - agent { - kubernetes { - label 'jenkins-slave' - defaultContainer 'jnlp' - } - } - stages { - stage('Deploy') { - steps { - gitlabCommitStatus(name: 'Deloy to Dev') { - script { - load "${env.WORKSPACE}/ci/jenkinsfile/deploy2dev.groovy" - } + steps { + gitlabCommitStatus(name: 'Deloy to Dev') { + container('milvus-testframework') { + script { + load "${env.WORKSPACE}/ci/jenkinsfile/deploy2dev.groovy" } } } } - post { - aborted { - script { - updateGitlabCommitStatus name: 'Deloy to Dev', state: 'canceled' - echo "Milvus Deloy to Dev aborted !" - } - } - - failure { - script { - updateGitlabCommitStatus name: 'Deloy to Dev', state: 'failed' - echo "Milvus Deloy to Dev failure !" - } - } - } } - stage("Dev Test") { - agent { - kubernetes { - label 'test' - defaultContainer 'jnlp' - containerTemplate { - name 'milvus-testframework' - image 'registry.zilliz.com/milvus/milvus-test:v0.1' - ttyEnabled true - command 'cat' - } - } - } - stages { - stage('Test') { - steps { + steps { + gitlabCommitStatus(name: 'Deloy Test') { + container('milvus-testframework') { script { load "${env.WORKSPACE}/ci/jenkinsfile/dev_test.groovy" load "${env.WORKSPACE}/ci/jenkinsfile/upload_dev_test_out.groovy" @@ -192,140 +186,133 @@ spec: } } } - stage ("Cleanup Dev") { - agent { - kubernetes { - label 'jenkins-slave' - defaultContainer 'jnlp' - } - } - stages { - stage('Cleanup') { - steps { - gitlabCommitStatus(name: 'Cleanup Dev') { - script { - load "${env.WORKSPACE}/ci/jenkinsfile/cleanup_dev.groovy" - } + steps { + gitlabCommitStatus(name: 'Cleanup Dev') { + container('milvus-testframework') { + script { + load "${env.WORKSPACE}/ci/jenkinsfile/cleanup_dev.groovy" } } } } - post { - aborted { - script { - updateGitlabCommitStatus name: 'Cleanup Dev', state: 'canceled' - echo "Milvus Cleanup Dev aborted !" - } - } - - failure { - script { - updateGitlabCommitStatus name: 'Cleanup Dev', state: 'failed' - echo "Milvus Cleanup Dev failure !" - } + } + } + post { + always { + container('milvus-testframework') { + script { + load "${env.WORKSPACE}/ci/jenkinsfile/cleanup_dev.groovy" } } } + success { + script { + echo "Milvus Single Node CI/CD success !" + } + } + aborted { + script { + echo "Milvus Single Node CI/CD aborted !" + } + } + failure { + script { + echo "Milvus Single Node CI/CD failure !" + } + } } } stage("Cluster") { + agent { + kubernetes { + label 'dev-test' + defaultContainer 'jnlp' + yaml """ +apiVersion: v1 +kind: Pod +metadata: + labels: + app: milvus + componet: test +spec: + containers: + - name: milvus-testframework + image: registry.zilliz.com/milvus/milvus-test:v0.2 + command: + - cat + tty: true + volumeMounts: + - name: kubeconf + mountPath: /root/.kube/ + readOnly: true + volumes: + - name: kubeconf + secret: + secretName: test-cluster-config +""" + } + } stages { stage("Deploy to Dev") { - agent { - kubernetes { - label 'jenkins-slave' - defaultContainer 'jnlp' - } - } - stages { - stage('Deploy') { - steps { - gitlabCommitStatus(name: 'Deloy to Dev') { - script { - load "${env.WORKSPACE}/ci/jenkinsfile/cluster_deploy2dev.groovy" - } + steps { + gitlabCommitStatus(name: 'Deloy to Dev') { + container('milvus-testframework') { + script { + load "${env.WORKSPACE}/ci/jenkinsfile/cluster_deploy2dev.groovy" } } } } - post { - aborted { - script { - updateGitlabCommitStatus name: 'Deloy to Dev', state: 'canceled' - echo "Milvus Deloy to Dev aborted !" - } - } - - failure { - script { - updateGitlabCommitStatus name: 'Deloy to Dev', state: 'failed' - echo "Milvus Deloy to Dev failure !" - } - } - } } - stage("Dev Test") { - agent { - kubernetes { - label 'test' - defaultContainer 'jnlp' - containerTemplate { - name 'milvus-testframework' - image 'registry.zilliz.com/milvus/milvus-test:v0.1' - ttyEnabled true - command 'cat' - } - } - } - stages { - stage('Test') { - steps { + steps { + gitlabCommitStatus(name: 'Deloy Test') { + container('milvus-testframework') { script { load "${env.WORKSPACE}/ci/jenkinsfile/cluster_dev_test.groovy" - load "${env.WORKSPACE}/ci/jenkinsfile/upload_cluster_dev_test_out.groovy" + load "${env.WORKSPACE}/ci/jenkinsfile/upload_dev_cluster_test_out.groovy" } } } } } - stage ("Cleanup Dev") { - agent { - kubernetes { - label 'jenkins-slave' - defaultContainer 'jnlp' - } - } - stages { - stage('Cleanup') { - steps { - gitlabCommitStatus(name: 'Cleanup Dev') { - script { - load "${env.WORKSPACE}/ci/jenkinsfile/cluster_cleanup_dev.groovy" - } + steps { + gitlabCommitStatus(name: 'Cleanup Dev') { + container('milvus-testframework') { + script { + load "${env.WORKSPACE}/ci/jenkinsfile/cluster_cleanup_dev.groovy" } } } } - post { - aborted { - script { - updateGitlabCommitStatus name: 'Cleanup Dev', state: 'canceled' - echo "Milvus Cleanup Dev aborted !" - } - } - - failure { - script { - updateGitlabCommitStatus name: 'Cleanup Dev', state: 'failed' - echo "Milvus Cleanup Dev failure !" - } + } + } + post { + always { + container('milvus-testframework') { + script { + load "${env.WORKSPACE}/ci/jenkinsfile/cluster_cleanup_dev.groovy" } } } + success { + script { + echo "Milvus Cluster CI/CD success !" + } + } + aborted { + script { + echo "Milvus Cluster CI/CD aborted !" + } + } + failure { + script { + echo "Milvus Cluster CI/CD failure !" + } + } } } } @@ -337,16 +324,18 @@ spec: post { always { script { - if (!currentBuild.resultIsBetterOrEqualTo('SUCCESS')) { - // Send an email only if the build status has changed from green/unstable to red - emailext subject: '$DEFAULT_SUBJECT', - body: '$DEFAULT_CONTENT', - recipientProviders: [ - [$class: 'DevelopersRecipientProvider'], - [$class: 'RequesterRecipientProvider'] - ], - replyTo: '$DEFAULT_REPLYTO', - to: '$DEFAULT_RECIPIENTS' + if (env.gitlabAfter != null) { + if (!currentBuild.resultIsBetterOrEqualTo('SUCCESS')) { + // Send an email only if the build status has changed from green/unstable to red + emailext subject: '$DEFAULT_SUBJECT', + body: '$DEFAULT_CONTENT', + recipientProviders: [ + [$class: 'DevelopersRecipientProvider'], + [$class: 'RequesterRecipientProvider'] + ], + replyTo: '$DEFAULT_REPLYTO', + to: '$DEFAULT_RECIPIENTS' + } } } } diff --git a/ci/nightly_main_jenkinsfile b/ci/nightly_main_jenkinsfile index c375cc87ea..7f59561f82 100644 --- a/ci/nightly_main_jenkinsfile +++ b/ci/nightly_main_jenkinsfile @@ -131,59 +131,53 @@ spec: stage("Deploy to Development") { parallel { - stage("Single") { + stage("Single Node") { + agent { + kubernetes { + label 'dev-test' + defaultContainer 'jnlp' + yaml """ +apiVersion: v1 +kind: Pod +metadata: + labels: + app: milvus + componet: test +spec: + containers: + - name: milvus-testframework + image: registry.zilliz.com/milvus/milvus-test:v0.2 + command: + - cat + tty: true + volumeMounts: + - name: kubeconf + mountPath: /root/.kube/ + readOnly: true + volumes: + - name: kubeconf + secret: + secretName: test-cluster-config +""" + } + } + stages { stage("Deploy to Dev") { - agent { - kubernetes { - label 'jenkins-slave' - defaultContainer 'jnlp' - } - } - stages { - stage('Deploy') { - steps { - gitlabCommitStatus(name: 'Deloy to Dev') { - script { - load "${env.WORKSPACE}/ci/jenkinsfile/deploy2dev.groovy" - } + steps { + gitlabCommitStatus(name: 'Deloy to Dev') { + container('milvus-testframework') { + script { + load "${env.WORKSPACE}/ci/jenkinsfile/deploy2dev.groovy" } } } } - post { - aborted { - script { - updateGitlabCommitStatus name: 'Deloy to Dev', state: 'canceled' - echo "Milvus Deloy to Dev aborted !" - } - } - - failure { - script { - updateGitlabCommitStatus name: 'Deloy to Dev', state: 'failed' - echo "Milvus Deloy to Dev failure !" - } - } - } } - stage("Dev Test") { - agent { - kubernetes { - label 'test' - defaultContainer 'jnlp' - containerTemplate { - name 'milvus-testframework' - image 'registry.zilliz.com/milvus/milvus-test:v0.1' - ttyEnabled true - command 'cat' - } - } - } - stages { - stage('Test') { - steps { + steps { + gitlabCommitStatus(name: 'Deloy Test') { + container('milvus-testframework') { script { load "${env.WORKSPACE}/ci/jenkinsfile/dev_test.groovy" load "${env.WORKSPACE}/ci/jenkinsfile/upload_dev_test_out.groovy" @@ -192,140 +186,133 @@ spec: } } } - stage ("Cleanup Dev") { - agent { - kubernetes { - label 'jenkins-slave' - defaultContainer 'jnlp' - } - } - stages { - stage('Cleanup') { - steps { - gitlabCommitStatus(name: 'Cleanup Dev') { - script { - load "${env.WORKSPACE}/ci/jenkinsfile/cleanup_dev.groovy" - } + steps { + gitlabCommitStatus(name: 'Cleanup Dev') { + container('milvus-testframework') { + script { + load "${env.WORKSPACE}/ci/jenkinsfile/cleanup_dev.groovy" } } } } - post { - aborted { - script { - updateGitlabCommitStatus name: 'Cleanup Dev', state: 'canceled' - echo "Milvus Cleanup Dev aborted !" - } - } - - failure { - script { - updateGitlabCommitStatus name: 'Cleanup Dev', state: 'failed' - echo "Milvus Cleanup Dev failure !" - } + } + } + post { + always { + container('milvus-testframework') { + script { + load "${env.WORKSPACE}/ci/jenkinsfile/cleanup_dev.groovy" } } } + success { + script { + echo "Milvus Single Node CI/CD success !" + } + } + aborted { + script { + echo "Milvus Single Node CI/CD aborted !" + } + } + failure { + script { + echo "Milvus Single Node CI/CD failure !" + } + } } } stage("Cluster") { + agent { + kubernetes { + label 'dev-test' + defaultContainer 'jnlp' + yaml """ +apiVersion: v1 +kind: Pod +metadata: + labels: + app: milvus + componet: test +spec: + containers: + - name: milvus-testframework + image: registry.zilliz.com/milvus/milvus-test:v0.2 + command: + - cat + tty: true + volumeMounts: + - name: kubeconf + mountPath: /root/.kube/ + readOnly: true + volumes: + - name: kubeconf + secret: + secretName: test-cluster-config +""" + } + } stages { stage("Deploy to Dev") { - agent { - kubernetes { - label 'jenkins-slave' - defaultContainer 'jnlp' - } - } - stages { - stage('Deploy') { - steps { - gitlabCommitStatus(name: 'Deloy to Dev') { - script { - load "${env.WORKSPACE}/ci/jenkinsfile/cluster_deploy2dev.groovy" - } + steps { + gitlabCommitStatus(name: 'Deloy to Dev') { + container('milvus-testframework') { + script { + load "${env.WORKSPACE}/ci/jenkinsfile/cluster_deploy2dev.groovy" } } } } - post { - aborted { - script { - updateGitlabCommitStatus name: 'Deloy to Dev', state: 'canceled' - echo "Milvus Deloy to Dev aborted !" - } - } - - failure { - script { - updateGitlabCommitStatus name: 'Deloy to Dev', state: 'failed' - echo "Milvus Deloy to Dev failure !" - } - } - } } - stage("Dev Test") { - agent { - kubernetes { - label 'test' - defaultContainer 'jnlp' - containerTemplate { - name 'milvus-testframework' - image 'registry.zilliz.com/milvus/milvus-test:v0.1' - ttyEnabled true - command 'cat' - } - } - } - stages { - stage('Test') { - steps { + steps { + gitlabCommitStatus(name: 'Deloy Test') { + container('milvus-testframework') { script { load "${env.WORKSPACE}/ci/jenkinsfile/cluster_dev_test.groovy" - load "${env.WORKSPACE}/ci/jenkinsfile/upload_cluster_dev_test_out.groovy" + load "${env.WORKSPACE}/ci/jenkinsfile/upload_dev_cluster_test_out.groovy" } } } } } - stage ("Cleanup Dev") { - agent { - kubernetes { - label 'jenkins-slave' - defaultContainer 'jnlp' - } - } - stages { - stage('Cleanup') { - steps { - gitlabCommitStatus(name: 'Cleanup Dev') { - script { - load "${env.WORKSPACE}/ci/jenkinsfile/cluster_cleanup_dev.groovy" - } + steps { + gitlabCommitStatus(name: 'Cleanup Dev') { + container('milvus-testframework') { + script { + load "${env.WORKSPACE}/ci/jenkinsfile/cluster_cleanup_dev.groovy" } } } } - post { - aborted { - script { - updateGitlabCommitStatus name: 'Cleanup Dev', state: 'canceled' - echo "Milvus Cleanup Dev aborted !" - } - } - - failure { - script { - updateGitlabCommitStatus name: 'Cleanup Dev', state: 'failed' - echo "Milvus Cleanup Dev failure !" - } + } + } + post { + always { + container('milvus-testframework') { + script { + load "${env.WORKSPACE}/ci/jenkinsfile/cluster_cleanup_dev.groovy" } } } + success { + script { + echo "Milvus Cluster CI/CD success !" + } + } + aborted { + script { + echo "Milvus Cluster CI/CD aborted !" + } + } + failure { + script { + echo "Milvus Cluster CI/CD failure !" + } + } } } } From 2428ff11608f567744f3b76e68810045369b3b50 Mon Sep 17 00:00:00 2001 From: starlord Date: Mon, 15 Jul 2019 09:25:55 +0800 Subject: [PATCH 158/189] Fix unittest error Former-commit-id: 9185fecbe5161694c6b741ad53fdccf89857899a --- cpp/unittest/db/db_tests.cpp | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/cpp/unittest/db/db_tests.cpp b/cpp/unittest/db/db_tests.cpp index 729317011d..a8f5d2bf8d 100644 --- a/cpp/unittest/db/db_tests.cpp +++ b/cpp/unittest/db/db_tests.cpp @@ -52,15 +52,13 @@ TEST_F(DBTest, CONFIG_TEST) { engine::ArchiveConf conf("delete"); ASSERT_EQ(conf.GetType(), "delete"); auto criterias = conf.GetCriterias(); - ASSERT_TRUE(criterias.size() == 1); - ASSERT_TRUE(criterias["disk"] == 512); + ASSERT_TRUE(criterias.size() == 0); } { engine::ArchiveConf conf("swap"); ASSERT_EQ(conf.GetType(), "swap"); auto criterias = conf.GetCriterias(); - ASSERT_TRUE(criterias.size() == 1); - ASSERT_TRUE(criterias["disk"] == 512); + ASSERT_TRUE(criterias.size() == 0); } { ASSERT_ANY_THROW(engine::ArchiveConf conf1("swap", "disk:")); From 2dffb947a94e275ca42138fed36044cf79965277 Mon Sep 17 00:00:00 2001 From: starlord Date: Mon, 15 Jul 2019 10:29:28 +0800 Subject: [PATCH 159/189] add more unitest cases Former-commit-id: 6367f0601b449098514ac4770eb72daa81d91607 --- cpp/unittest/db/db_tests.cpp | 18 ++++++++++++++---- cpp/unittest/db/misc_test.cpp | 29 +++++++++++++++++++++++++++++ 2 files changed, 43 insertions(+), 4 deletions(-) diff --git a/cpp/unittest/db/db_tests.cpp b/cpp/unittest/db/db_tests.cpp index a8f5d2bf8d..c9a9d89780 100644 --- a/cpp/unittest/db/db_tests.cpp +++ b/cpp/unittest/db/db_tests.cpp @@ -204,11 +204,21 @@ TEST_F(DBTest, SEARCH_TEST) { ASSERT_STATS(stat); } - sleep(2); // wait until build index finish + db_->BuildIndex(TABLE_NAME); // wait until build index finish - engine::QueryResults results; - stat = db_->Query(TABLE_NAME, k, nq, xq.data(), results); - ASSERT_STATS(stat); + { + engine::QueryResults results; + stat = db_->Query(TABLE_NAME, k, nq, xq.data(), results); + ASSERT_STATS(stat); + } + + {//search by specify index file + engine::meta::DatesT dates; + std::vector file_ids = {"1", "2", "3", "4"}; + engine::QueryResults results; + stat = db_->Query(TABLE_NAME, file_ids, k, nq, xq.data(), dates, results); + ASSERT_STATS(stat); + } // TODO(linxj): add groundTruth assert }; diff --git a/cpp/unittest/db/misc_test.cpp b/cpp/unittest/db/misc_test.cpp index 4356746fc2..9dd07fac8d 100644 --- a/cpp/unittest/db/misc_test.cpp +++ b/cpp/unittest/db/misc_test.cpp @@ -14,6 +14,7 @@ #include "db/Options.h" #include "db/DBMetaImpl.h" #include "db/EngineFactory.h" +#include "db/Utils.h" #include @@ -134,4 +135,32 @@ TEST(DBMiscTest, META_TEST) { int delta = 10; engine::meta::DateT dt = impl.GetDate(tt, delta); ASSERT_GT(dt, 0); +} + +TEST(DBMiscTest, UTILS_TEST) { + engine::DBMetaOptions options; + options.path = "/tmp/milvus_test/main"; + options.slave_paths.push_back("/tmp/milvus_test/slave_1"); + options.slave_paths.push_back("/tmp/milvus_test/slave_2"); + + const std::string TABLE_NAME = "test_tbl"; + auto status = engine::utils::CreateTablePath(options, TABLE_NAME); + ASSERT_TRUE(status.ok()); + ASSERT_TRUE(boost::filesystem::exists(options.path)); + for(auto& path : options.slave_paths) { + ASSERT_TRUE(boost::filesystem::exists(path)); + } + + engine::meta::TableFileSchema file; + file.id_ = 50; + file.table_id_ = TABLE_NAME; + file.file_type_ = 3; + file.date_ = 155000; + status = engine::utils::GetTableFilePath(options, file); + ASSERT_FALSE(status.ok()); + ASSERT_TRUE(file.location_.empty()); + + status = engine::utils::DeleteTablePath(options, TABLE_NAME); + ASSERT_TRUE(status.ok()); + } \ No newline at end of file From 8ca3b33cd8943d5ada3d977b96268d452433aaa2 Mon Sep 17 00:00:00 2001 From: zhiru Date: Mon, 15 Jul 2019 10:41:22 +0800 Subject: [PATCH 160/189] return AlreadyExist error Former-commit-id: b80220e68e8a5a53c1a2fef4e90d8a4729a85996 --- cpp/src/db/MySQLMetaImpl.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/src/db/MySQLMetaImpl.cpp b/cpp/src/db/MySQLMetaImpl.cpp index bd82e7e6a1..9aa5a85e9e 100644 --- a/cpp/src/db/MySQLMetaImpl.cpp +++ b/cpp/src/db/MySQLMetaImpl.cpp @@ -347,7 +347,7 @@ namespace meta { return Status::Error("Table already exists and it is in delete state, please wait a second"); } else { - return Status::OK();//table already exists, no error + return Status::AlreadyExist("Table already exists"); } } } From 4d51f9a122a15a121e63eb84bbc7b0d200140a25 Mon Sep 17 00:00:00 2001 From: zhiru Date: Mon, 15 Jul 2019 10:45:20 +0800 Subject: [PATCH 161/189] Return AlreadyExist status in MySQLMetaImpl::CreateTable if table already exists Former-commit-id: eb56307b2a6613a90df28fba0f63166745195cc3 --- cpp/CHANGELOG.md | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/cpp/CHANGELOG.md b/cpp/CHANGELOG.md index 7e7f88a2e9..946e08e99c 100644 --- a/cpp/CHANGELOG.md +++ b/cpp/CHANGELOG.md @@ -12,6 +12,7 @@ Please mark all change in change log and use the ticket from JIRA. - MS-153 - fix c_str error when connecting to MySQL - MS-157 - fix changelog - MS-190 - use env variable to switch mem manager and fix cmake +- MS-224 - Return AlreadyExist status in MySQLMetaImpl::CreateTable if table already exists ## Improvement - MS-156 - Add unittest for merge result functions @@ -21,11 +22,9 @@ Please mark all change in change log and use the ticket from JIRA. - MS-208 - Add buildinde interface for C++ SDK - MS-212 - Support Inner product metric type -## New Feature -- MS-195 - Add nlist and use_blas_threshold conf - ## New Feature - MS-180 - Add new mem manager +- MS-195 - Add nlist and use_blas_threshold conf ## Task From b18cb007d2306380816385339bb6fb125162bc87 Mon Sep 17 00:00:00 2001 From: quicksilver Date: Mon, 15 Jul 2019 11:09:19 +0800 Subject: [PATCH 162/189] format jenkins groovy file Former-commit-id: efd0150eea28f3d2a0bbebe95385ae651081805b --- ci/jenkinsfile/cluster_dev_test.groovy | 2 +- ci/jenkinsfile/publish_docker.groovy | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/ci/jenkinsfile/cluster_dev_test.groovy b/ci/jenkinsfile/cluster_dev_test.groovy index 8093f67d18..ad7b838e4f 100644 --- a/ci/jenkinsfile/cluster_dev_test.groovy +++ b/ci/jenkinsfile/cluster_dev_test.groovy @@ -6,7 +6,7 @@ timeout(time: 10, unit: 'MINUTES') { sh "pytest . --alluredir=cluster_test_out --ip ${env.JOB_NAME}-${env.BUILD_NUMBER}-cluster-milvus-cluster-proxy.milvus-cluster.svc.cluster.local" } } catch (exc) { - echo 'Milvus Test Failed !' + echo 'Milvus Cluster Test Failed !' throw exc } } diff --git a/ci/jenkinsfile/publish_docker.groovy b/ci/jenkinsfile/publish_docker.groovy index 7416c9cb17..d43b06c9d3 100644 --- a/ci/jenkinsfile/publish_docker.groovy +++ b/ci/jenkinsfile/publish_docker.groovy @@ -12,7 +12,10 @@ container('publish-docker') { def customImage = docker.build("${PROJECT_NAME}/engine:${DOCKER_VERSION}") customImage.push() } - echo "Docker Pull Command: docker pull registry.zilliz.com/${PROJECT_NAME}/engine:${DOCKER_VERSION}" + if (currentBuild.resultIsBetterOrEqualTo('SUCCESS')) { + updateGitlabCommitStatus name: 'Publish Engine Docker', state: 'success' + echo "Docker Pull Command: docker pull registry.zilliz.com/${PROJECT_NAME}/engine:${DOCKER_VERSION}" + } } catch (exc) { updateGitlabCommitStatus name: 'Publish Engine Docker', state: 'canceled' throw exc From cd9f317747983774e638688ac845f2ab16132ee3 Mon Sep 17 00:00:00 2001 From: starlord Date: Mon, 15 Jul 2019 11:10:57 +0800 Subject: [PATCH 163/189] fix index check error Former-commit-id: 9b876213adce723cb2da6d4aa40fb33be2a2c54a --- cpp/src/utils/ValidationUtil.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cpp/src/utils/ValidationUtil.cpp b/cpp/src/utils/ValidationUtil.cpp index 65cd81e670..a1e3f0dffc 100644 --- a/cpp/src/utils/ValidationUtil.cpp +++ b/cpp/src/utils/ValidationUtil.cpp @@ -59,7 +59,8 @@ ValidateTableIndexType(int32_t index_type) { auto engine_type = engine::EngineType(index_type); switch (engine_type) { case engine::EngineType::FAISS_IDMAP: - case engine::EngineType::FAISS_IVFFLAT: { + case engine::EngineType::FAISS_IVFFLAT: + case engine::EngineType::FAISS_IVFSQ8:{ SERVER_LOG_DEBUG << "Index type: " << index_type; return SERVER_SUCCESS; } From e950fff6726751a978a6581e50b3efaa3b70cbdd Mon Sep 17 00:00:00 2001 From: zhiru Date: Mon, 15 Jul 2019 11:36:01 +0800 Subject: [PATCH 164/189] add MySQLMetaImpl::UpdateTableFilesToIndex and set maximum_memory to default if config value = 0 Former-commit-id: f8943149b9737b2f0107308aa96a7f77acdf9ce4 --- cpp/src/db/MySQLMetaImpl.cpp | 3264 +++++++++++++++++----------------- cpp/src/db/Options.h | 2 +- cpp/src/server/DBWrapper.cpp | 4 +- 3 files changed, 1593 insertions(+), 1677 deletions(-) diff --git a/cpp/src/db/MySQLMetaImpl.cpp b/cpp/src/db/MySQLMetaImpl.cpp index 9aa5a85e9e..dc22c0307c 100644 --- a/cpp/src/db/MySQLMetaImpl.cpp +++ b/cpp/src/db/MySQLMetaImpl.cpp @@ -24,228 +24,853 @@ #include "mysql++/mysql++.h" + namespace zilliz { namespace milvus { namespace engine { namespace meta { - using namespace mysqlpp; +using namespace mysqlpp; + + -// static std::unique_ptr connectionPtr(new Connection()); -// std::recursive_mutex mysql_mutex; // -// std::unique_ptr& MySQLMetaImpl::getConnectionPtr() { -//// static std::recursive_mutex connectionMutex_; -// std::lock_guard lock(connectionMutex_); -// return connectionPtr; -// } - namespace { +// - Status HandleException(const std::string& desc, std::exception &e) { - ENGINE_LOG_ERROR << desc << ": " << e.what(); - return Status::DBTransactionError(desc, e.what()); + + + +namespace { + +Status HandleException(const std::string &desc, std::exception &e) { + ENGINE_LOG_ERROR << desc << ": " << e.what(); + return Status::DBTransactionError(desc, e.what()); +} + +class MetricCollector { + public: + MetricCollector() { + server::Metrics::GetInstance().MetaAccessTotalIncrement(); + start_time_ = METRICS_NOW_TIME; + } + + ~MetricCollector() { + auto end_time = METRICS_NOW_TIME; + auto total_time = METRICS_MICROSECONDS(start_time_, end_time); + server::Metrics::GetInstance().MetaAccessDurationSecondsHistogramObserve(total_time); + } + + private: + using TIME_POINT = std::chrono::system_clock::time_point; + TIME_POINT start_time_; +}; + +} + +Status MySQLMetaImpl::NextTableId(std::string &table_id) { + std::stringstream ss; + SimpleIDGenerator g; + ss << g.GetNextIDNumber(); + table_id = ss.str(); + return Status::OK(); +} + +Status MySQLMetaImpl::NextFileId(std::string &file_id) { + std::stringstream ss; + SimpleIDGenerator g; + ss << g.GetNextIDNumber(); + file_id = ss.str(); + return Status::OK(); +} + +MySQLMetaImpl::MySQLMetaImpl(const DBMetaOptions &options_, const int &mode) + : options_(options_), + mode_(mode) { + Initialize(); +} + +Status MySQLMetaImpl::Initialize() { + + + if (!boost::filesystem::is_directory(options_.path)) { + auto ret = boost::filesystem::create_directory(options_.path); + if (!ret) { + ENGINE_LOG_ERROR << "Failed to create db directory " << options_.path; + return Status::DBTransactionError("Failed to create db directory", options_.path); } - - class MetricCollector { - public: - MetricCollector() { - server::Metrics::GetInstance().MetaAccessTotalIncrement(); - start_time_ = METRICS_NOW_TIME; - } - - ~MetricCollector() { - auto end_time = METRICS_NOW_TIME; - auto total_time = METRICS_MICROSECONDS(start_time_, end_time); - server::Metrics::GetInstance().MetaAccessDurationSecondsHistogramObserve(total_time); - } - - private: - using TIME_POINT = std::chrono::system_clock::time_point; - TIME_POINT start_time_; - }; - } - Status MySQLMetaImpl::NextTableId(std::string &table_id) { - std::stringstream ss; - SimpleIDGenerator g; - ss << g.GetNextIDNumber(); - table_id = ss.str(); - return Status::OK(); - } + std::string uri = options_.backend_uri; - Status MySQLMetaImpl::NextFileId(std::string &file_id) { - std::stringstream ss; - SimpleIDGenerator g; - ss << g.GetNextIDNumber(); - file_id = ss.str(); - return Status::OK(); - } + std::string dialectRegex = "(.*)"; + std::string usernameRegex = "(.*)"; + std::string passwordRegex = "(.*)"; + std::string hostRegex = "(.*)"; + std::string portRegex = "(.*)"; + std::string dbNameRegex = "(.*)"; + std::string uriRegexStr = dialectRegex + "\\:\\/\\/" + + usernameRegex + "\\:" + + passwordRegex + "\\@" + + hostRegex + "\\:" + + portRegex + "\\/" + + dbNameRegex; + std::regex uriRegex(uriRegexStr); + std::smatch pieces_match; - MySQLMetaImpl::MySQLMetaImpl(const DBMetaOptions &options_, const int& mode) - : options_(options_), - mode_(mode) { - Initialize(); - } - - Status MySQLMetaImpl::Initialize() { - -// std::lock_guard lock(mysql_mutex); - - if (!boost::filesystem::is_directory(options_.path)) { - auto ret = boost::filesystem::create_directory(options_.path); - if (!ret) { - ENGINE_LOG_ERROR << "Failed to create db directory " << options_.path; - return Status::DBTransactionError("Failed to create db directory", options_.path); - } + if (std::regex_match(uri, pieces_match, uriRegex)) { + std::string dialect = pieces_match[1].str(); + std::transform(dialect.begin(), dialect.end(), dialect.begin(), ::tolower); + if (dialect.find("mysql") == std::string::npos) { + return Status::Error("URI's dialect is not MySQL"); } + std::string username = pieces_match[2].str(); + std::string password = pieces_match[3].str(); + std::string serverAddress = pieces_match[4].str(); + unsigned int port = 0; + if (!pieces_match[5].str().empty()) { + port = std::stoi(pieces_match[5].str()); + } + std::string dbName = pieces_match[6].str(); - std::string uri = options_.backend_uri; - std::string dialectRegex = "(.*)"; - std::string usernameRegex = "(.*)"; - std::string passwordRegex = "(.*)"; - std::string hostRegex = "(.*)"; - std::string portRegex = "(.*)"; - std::string dbNameRegex = "(.*)"; - std::string uriRegexStr = dialectRegex + "\\:\\/\\/" + - usernameRegex + "\\:" + - passwordRegex + "\\@" + - hostRegex + "\\:" + - portRegex + "\\/" + - dbNameRegex; - std::regex uriRegex(uriRegexStr); - std::smatch pieces_match; + int threadHint = std::thread::hardware_concurrency(); + int maxPoolSize = threadHint == 0 ? 8 : threadHint; + mysql_connection_pool_ = + std::make_shared(dbName, username, password, serverAddress, port, maxPoolSize); - if (std::regex_match(uri, pieces_match, uriRegex)) { - std::string dialect = pieces_match[1].str(); - std::transform(dialect.begin(), dialect.end(), dialect.begin(), ::tolower); - if (dialect.find("mysql") == std::string::npos) { - return Status::Error("URI's dialect is not MySQL"); + ENGINE_LOG_DEBUG << "MySQL connection pool: maximum pool size = " << std::to_string(maxPoolSize); + try { + + if (mode_ != Options::MODE::READ_ONLY) { + CleanUp(); } - std::string username = pieces_match[2].str(); - std::string password = pieces_match[3].str(); - std::string serverAddress = pieces_match[4].str(); - unsigned int port = 0; - if (!pieces_match[5].str().empty()) { - port = std::stoi(pieces_match[5].str()); - } - std::string dbName = pieces_match[6].str(); -// std::cout << dbName << " " << serverAddress << " " << username << " " << password << " " << port << std::endl; -// connectionPtr->set_option(new MultiStatementsOption(true)); -// connectionPtr->set_option(new mysqlpp::ReconnectOption(true)); - int threadHint = std::thread::hardware_concurrency(); - int maxPoolSize = threadHint == 0 ? 8 : threadHint; - mysql_connection_pool_ = std::make_shared(dbName, username, password, serverAddress, port, maxPoolSize); -// std::cout << "MySQL++ thread aware:" << std::to_string(connectionPtr->thread_aware()) << std::endl; - ENGINE_LOG_DEBUG << "MySQL connection pool: maximum pool size = " << std::to_string(maxPoolSize); - try { - if (mode_ != Options::MODE::READ_ONLY) { - CleanUp(); + { + ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab); + + if (connectionPtr == nullptr) { + return Status::Error("Failed to connect to database server"); } - { - ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab); - if (connectionPtr == nullptr) { - return Status::Error("Failed to connect to database server"); - } + if (!connectionPtr->thread_aware()) { + ENGINE_LOG_ERROR << "MySQL++ wasn't built with thread awareness! Can't run without it."; + return Status::Error("MySQL++ wasn't built with thread awareness! Can't run without it."); + } + Query InitializeQuery = connectionPtr->query(); -// ENGINE_LOG_DEBUG << "MySQLMetaImpl::Initialize: connections in use = " << mysql_connection_pool_->getConnectionsInUse(); -// if (!connectionPtr->connect(dbName, serverAddress, username, password, port)) { -// return Status::Error("DB connection failed: ", connectionPtr->error()); -// } - if (!connectionPtr->thread_aware()) { - ENGINE_LOG_ERROR << "MySQL++ wasn't built with thread awareness! Can't run without it."; - return Status::Error("MySQL++ wasn't built with thread awareness! Can't run without it."); - } - Query InitializeQuery = connectionPtr->query(); -// InitializeQuery << "SET max_allowed_packet=67108864;"; -// if (!InitializeQuery.exec()) { -// return Status::DBTransactionError("Initialization Error", InitializeQuery.error()); -// } + InitializeQuery << "CREATE TABLE IF NOT EXISTS Tables (" << + "id BIGINT PRIMARY KEY AUTO_INCREMENT, " << + "table_id VARCHAR(255) UNIQUE NOT NULL, " << + "state INT NOT NULL, " << + "dimension SMALLINT NOT NULL, " << + "created_on BIGINT NOT NULL, " << + "files_cnt BIGINT DEFAULT 0 NOT NULL, " << + "engine_type INT DEFAULT 1 NOT NULL, " << + "store_raw_data BOOL DEFAULT false NOT NULL);"; -// InitializeQuery << "DROP TABLE IF EXISTS Tables, TableFiles;"; - InitializeQuery << "CREATE TABLE IF NOT EXISTS Tables (" << - "id BIGINT PRIMARY KEY AUTO_INCREMENT, " << - "table_id VARCHAR(255) UNIQUE NOT NULL, " << - "state INT NOT NULL, " << - "dimension SMALLINT NOT NULL, " << - "created_on BIGINT NOT NULL, " << - "files_cnt BIGINT DEFAULT 0 NOT NULL, " << - "engine_type INT DEFAULT 1 NOT NULL, " << - "store_raw_data BOOL DEFAULT false NOT NULL);"; + ENGINE_LOG_DEBUG << "MySQLMetaImpl::Initialize: " << InitializeQuery.str(); - ENGINE_LOG_DEBUG << "MySQLMetaImpl::Initialize: " << InitializeQuery.str(); + if (!InitializeQuery.exec()) { + return Status::DBTransactionError("Initialization Error", InitializeQuery.error()); + } - if (!InitializeQuery.exec()) { - return Status::DBTransactionError("Initialization Error", InitializeQuery.error()); - } + InitializeQuery << "CREATE TABLE IF NOT EXISTS TableFiles (" << + "id BIGINT PRIMARY KEY AUTO_INCREMENT, " << + "table_id VARCHAR(255) NOT NULL, " << + "engine_type INT DEFAULT 1 NOT NULL, " << + "file_id VARCHAR(255) NOT NULL, " << + "file_type INT DEFAULT 0 NOT NULL, " << + "size BIGINT DEFAULT 0 NOT NULL, " << + "updated_time BIGINT NOT NULL, " << + "created_on BIGINT NOT NULL, " << + "date INT DEFAULT -1 NOT NULL);"; - InitializeQuery << "CREATE TABLE IF NOT EXISTS TableFiles (" << - "id BIGINT PRIMARY KEY AUTO_INCREMENT, " << - "table_id VARCHAR(255) NOT NULL, " << - "engine_type INT DEFAULT 1 NOT NULL, " << - "file_id VARCHAR(255) NOT NULL, " << - "file_type INT DEFAULT 0 NOT NULL, " << - "size BIGINT DEFAULT 0 NOT NULL, " << - "updated_time BIGINT NOT NULL, " << - "created_on BIGINT NOT NULL, " << - "date INT DEFAULT -1 NOT NULL);"; + ENGINE_LOG_DEBUG << "MySQLMetaImpl::Initialize: " << InitializeQuery.str(); - ENGINE_LOG_DEBUG << "MySQLMetaImpl::Initialize: " << InitializeQuery.str(); + if (!InitializeQuery.exec()) { + return Status::DBTransactionError("Initialization Error", InitializeQuery.error()); + } + } //Scoped Connection - if (!InitializeQuery.exec()) { - return Status::DBTransactionError("Initialization Error", InitializeQuery.error()); - } - } //Scoped Connection -// //Consume all results to avoid "Commands out of sync" error -// while (InitializeQuery.more_results()) { -// InitializeQuery.store_next(); -// } - return Status::OK(); -// if (InitializeQuery.exec()) { -// std::cout << "XXXXXXXXXXXXXXXXXXXXXXXXX" << std::endl; -// while (InitializeQuery.more_results()) { -// InitializeQuery.store_next(); -// } -// return Status::OK(); -// } else { -// return Status::DBTransactionError("Initialization Error", InitializeQuery.error()); -// } - } catch (const BadQuery& er) { - // Handle any query errors - ENGINE_LOG_ERROR << "QUERY ERROR DURING INITIALIZATION" << ": " << er.what(); - return Status::DBTransactionError("QUERY ERROR DURING INITIALIZATION", er.what()); - } catch (const Exception& er) { - // Catch-all for any other MySQL++ exceptions - ENGINE_LOG_ERROR << "GENERAL ERROR DURING INITIALIZATION" << ": " << er.what(); - return Status::DBTransactionError("GENERAL ERROR DURING INITIALIZATION", er.what()); - } catch (std::exception &e) { - return HandleException("Encounter exception during initialization", e); - } - } - else { - ENGINE_LOG_ERROR << "Wrong URI format. URI = " << uri; - return Status::Error("Wrong URI format"); + + + return Status::OK(); + + + } catch (const BadQuery &er) { + // Handle any query errors + ENGINE_LOG_ERROR << "QUERY ERROR DURING INITIALIZATION" << ": " << er.what(); + return Status::DBTransactionError("QUERY ERROR DURING INITIALIZATION", er.what()); + } catch (const Exception &er) { + // Catch-all for any other MySQL++ exceptions + ENGINE_LOG_ERROR << "GENERAL ERROR DURING INITIALIZATION" << ": " << er.what(); + return Status::DBTransactionError("GENERAL ERROR DURING INITIALIZATION", er.what()); + } catch (std::exception &e) { + return HandleException("Encounter exception during initialization", e); } + } else { + ENGINE_LOG_ERROR << "Wrong URI format. URI = " << uri; + return Status::Error("Wrong URI format"); } +} // PXU TODO: Temp solution. Will fix later - Status MySQLMetaImpl::DropPartitionsByDates(const std::string &table_id, - const DatesT &dates) { +Status MySQLMetaImpl::DropPartitionsByDates(const std::string &table_id, + const DatesT &dates) { -// std::lock_guard lock(mysql_mutex); - if (dates.empty()) { - return Status::OK(); + if (dates.empty()) { + return Status::OK(); + } + + TableSchema table_schema; + table_schema.table_id_ = table_id; + auto status = DescribeTable(table_schema); + if (!status.ok()) { + return status; + } + + try { + + auto yesterday = GetDateWithDelta(-1); + + for (auto &date : dates) { + if (date >= yesterday) { + return Status::Error("Could not delete partitions within 2 days"); + } } + std::stringstream dateListSS; + for (auto &date : dates) { + dateListSS << std::to_string(date) << ", "; + } + std::string dateListStr = dateListSS.str(); + dateListStr = dateListStr.substr(0, dateListStr.size() - 2); //remove the last ", " + + { + ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab); + + if (connectionPtr == nullptr) { + return Status::Error("Failed to connect to database server"); + } + + + Query dropPartitionsByDatesQuery = connectionPtr->query(); + + dropPartitionsByDatesQuery << "UPDATE TableFiles " << + "SET file_type = " << std::to_string(TableFileSchema::TO_DELETE) << " " << + "WHERE table_id = " << quote << table_id << " AND " << + "date in (" << dateListStr << ");"; + + ENGINE_LOG_DEBUG << "MySQLMetaImpl::DropPartitionsByDates: " << dropPartitionsByDatesQuery.str(); + + if (!dropPartitionsByDatesQuery.exec()) { + ENGINE_LOG_ERROR << "QUERY ERROR WHEN DROPPING PARTITIONS BY DATES"; + return Status::DBTransactionError("QUERY ERROR WHEN DROPPING PARTITIONS BY DATES", + dropPartitionsByDatesQuery.error()); + } + } //Scoped Connection + } catch (const BadQuery &er) { + // Handle any query errors + ENGINE_LOG_ERROR << "QUERY ERROR WHEN DROPPING PARTITIONS BY DATES" << ": " << er.what(); + return Status::DBTransactionError("QUERY ERROR WHEN DROPPING PARTITIONS BY DATES", er.what()); + } catch (const Exception &er) { + // Catch-all for any other MySQL++ exceptions + ENGINE_LOG_ERROR << "GENERAL ERROR WHEN DROPPING PARTITIONS BY DATES" << ": " << er.what(); + return Status::DBTransactionError("GENERAL ERROR WHEN DROPPING PARTITIONS BY DATES", er.what()); + } + return Status::OK(); +} + +Status MySQLMetaImpl::CreateTable(TableSchema &table_schema) { + + + try { + + MetricCollector metric; + + { + ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab); + + if (connectionPtr == nullptr) { + return Status::Error("Failed to connect to database server"); + } + + + Query createTableQuery = connectionPtr->query(); + + if (table_schema.table_id_.empty()) { + NextTableId(table_schema.table_id_); + } else { + createTableQuery << "SELECT state FROM Tables " << + "WHERE table_id = " << quote << table_schema.table_id_ << ";"; + + + ENGINE_LOG_DEBUG << "MySQLMetaImpl::CreateTable: " << createTableQuery.str(); + + StoreQueryResult res = createTableQuery.store(); + + if (res.num_rows() == 1) { + int state = res[0]["state"]; + if (TableSchema::TO_DELETE == state) { + return Status::Error("Table already exists and it is in delete state, please wait a second"); + } else { + return Status::AlreadyExist("Table already exists"); + } + } + } + + + table_schema.files_cnt_ = 0; + table_schema.id_ = -1; + table_schema.created_on_ = utils::GetMicroSecTimeStamp(); + + + std::string id = "NULL"; //auto-increment + std::string table_id = table_schema.table_id_; + std::string state = std::to_string(table_schema.state_); + std::string dimension = std::to_string(table_schema.dimension_); + std::string created_on = std::to_string(table_schema.created_on_); + std::string files_cnt = "0"; + std::string engine_type = std::to_string(table_schema.engine_type_); + std::string store_raw_data = table_schema.store_raw_data_ ? "true" : "false"; + + createTableQuery << "INSERT INTO Tables VALUES" << + "(" << id << ", " << quote << table_id << ", " << state << ", " << dimension << ", " << + created_on << ", " << files_cnt << ", " << engine_type << ", " << store_raw_data << ");"; + + + ENGINE_LOG_DEBUG << "MySQLMetaImpl::CreateTable: " << createTableQuery.str(); + + if (SimpleResult res = createTableQuery.execute()) { + table_schema.id_ = res.insert_id(); //Might need to use SELECT LAST_INSERT_ID()? + + //Consume all results to avoid "Commands out of sync" error + + + + } else { + ENGINE_LOG_ERROR << "Add Table Error"; + return Status::DBTransactionError("Add Table Error", createTableQuery.error()); + } + } //Scoped Connection + + + + + + return utils::CreateTablePath(options_, table_schema.table_id_); + + } catch (const BadQuery &er) { + // Handle any query errors + ENGINE_LOG_ERROR << "QUERY ERROR WHEN ADDING TABLE" << ": " << er.what(); + return Status::DBTransactionError("QUERY ERROR WHEN ADDING TABLE", er.what()); + } catch (const Exception &er) { + // Catch-all for any other MySQL++ exceptions + ENGINE_LOG_ERROR << "GENERAL ERROR WHEN ADDING TABLE" << ": " << er.what(); + return Status::DBTransactionError("GENERAL ERROR WHEN ADDING TABLE", er.what()); + } catch (std::exception &e) { + return HandleException("Encounter exception when create table", e); + } + + return Status::OK(); +} + +Status MySQLMetaImpl::HasNonIndexFiles(const std::string &table_id, bool &has) { + // TODO + return Status::OK(); +} + +Status MySQLMetaImpl::DeleteTable(const std::string &table_id) { + + + try { + + MetricCollector metric; + + { + ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab); + + if (connectionPtr == nullptr) { + return Status::Error("Failed to connect to database server"); + } + + + + + + //soft delete table + Query deleteTableQuery = connectionPtr->query(); +// + deleteTableQuery << "UPDATE Tables " << + "SET state = " << std::to_string(TableSchema::TO_DELETE) << " " << + "WHERE table_id = " << quote << table_id << ";"; + + ENGINE_LOG_DEBUG << "MySQLMetaImpl::DeleteTable: " << deleteTableQuery.str(); + + if (!deleteTableQuery.exec()) { + ENGINE_LOG_ERROR << "QUERY ERROR WHEN DELETING TABLE"; + return Status::DBTransactionError("QUERY ERROR WHEN DELETING TABLE", deleteTableQuery.error()); + } + + } //Scoped Connection + + + if (mode_ == Options::MODE::CLUSTER) { + DeleteTableFiles(table_id); + } + + } catch (const BadQuery &er) { + // Handle any query errors + ENGINE_LOG_ERROR << "GENERAL ERROR WHEN DELETING TABLE" << ": " << er.what(); + return Status::DBTransactionError("QUERY ERROR WHEN DELETING TABLE", er.what()); + } catch (const Exception &er) { + // Catch-all for any other MySQL++ exceptions + ENGINE_LOG_ERROR << "GENERAL ERROR WHEN DELETING TABLE" << ": " << er.what(); + return Status::DBTransactionError("GENERAL ERROR WHEN DELETING TABLE", er.what()); + } + + return Status::OK(); +} + +Status MySQLMetaImpl::DeleteTableFiles(const std::string &table_id) { + try { + MetricCollector metric; + + { + ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab); + + if (connectionPtr == nullptr) { + return Status::Error("Failed to connect to database server"); + } + + + + + + //soft delete table files + Query deleteTableFilesQuery = connectionPtr->query(); + // + deleteTableFilesQuery << "UPDATE TableFiles " << + "SET file_type = " << std::to_string(TableFileSchema::TO_DELETE) << ", " << + "updated_time = " << std::to_string(utils::GetMicroSecTimeStamp()) << " " << + "WHERE table_id = " << quote << table_id << " AND " << + "file_type <> " << std::to_string(TableFileSchema::TO_DELETE) << ";"; + + ENGINE_LOG_DEBUG << "MySQLMetaImpl::DeleteTableFiles: " << deleteTableFilesQuery.str(); + + if (!deleteTableFilesQuery.exec()) { + ENGINE_LOG_ERROR << "QUERY ERROR WHEN DELETING TABLE FILES"; + return Status::DBTransactionError("QUERY ERROR WHEN DELETING TABLE", deleteTableFilesQuery.error()); + } + } //Scoped Connection + } catch (const BadQuery &er) { + // Handle any query errors + ENGINE_LOG_ERROR << "QUERY ERROR WHEN DELETING TABLE FILES" << ": " << er.what(); + return Status::DBTransactionError("QUERY ERROR WHEN DELETING TABLE FILES", er.what()); + } catch (const Exception &er) { + // Catch-all for any other MySQL++ exceptions + ENGINE_LOG_ERROR << "GENERAL ERROR WHEN DELETING TABLE FILES" << ": " << er.what(); + return Status::DBTransactionError("GENERAL ERROR WHEN DELETING TABLE FILES", er.what()); + } + + return Status::OK(); +} + +Status MySQLMetaImpl::DescribeTable(TableSchema &table_schema) { + + + try { + + MetricCollector metric; + + StoreQueryResult res; + + { + ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab); + + if (connectionPtr == nullptr) { + return Status::Error("Failed to connect to database server"); + } + + + Query describeTableQuery = connectionPtr->query(); + describeTableQuery << "SELECT id, dimension, files_cnt, engine_type, store_raw_data " << + "FROM Tables " << + "WHERE table_id = " << quote << table_schema.table_id_ << " " << + "AND state <> " << std::to_string(TableSchema::TO_DELETE) << ";"; + + ENGINE_LOG_DEBUG << "MySQLMetaImpl::DescribeTable: " << describeTableQuery.str(); + + res = describeTableQuery.store(); + } //Scoped Connection + + if (res.num_rows() == 1) { + const Row &resRow = res[0]; + + table_schema.id_ = resRow["id"]; //implicit conversion + + table_schema.dimension_ = resRow["dimension"]; + + table_schema.files_cnt_ = resRow["files_cnt"]; + + table_schema.engine_type_ = resRow["engine_type"]; + + int store_raw_data = resRow["store_raw_data"]; + table_schema.store_raw_data_ = (store_raw_data == 1); + } else { + return Status::NotFound("Table " + table_schema.table_id_ + " not found"); + } + + } catch (const BadQuery &er) { + // Handle any query errors + ENGINE_LOG_ERROR << "QUERY ERROR WHEN DESCRIBING TABLE" << ": " << er.what(); + return Status::DBTransactionError("QUERY ERROR WHEN DESCRIBING TABLE", er.what()); + } catch (const Exception &er) { + // Catch-all for any other MySQL++ exceptions + ENGINE_LOG_ERROR << "GENERAL ERROR WHEN DESCRIBING TABLE" << ": " << er.what(); + return Status::DBTransactionError("GENERAL ERROR WHEN DESCRIBING TABLE", er.what()); + } + + return Status::OK(); +} + +Status MySQLMetaImpl::HasTable(const std::string &table_id, bool &has_or_not) { + + + try { + + MetricCollector metric; + + StoreQueryResult res; + + { + ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab); + + if (connectionPtr == nullptr) { + return Status::Error("Failed to connect to database server"); + } + + + Query hasTableQuery = connectionPtr->query(); + //since table_id is a unique column we just need to check whether it exists or not + hasTableQuery << "SELECT EXISTS " << + "(SELECT 1 FROM Tables " << + "WHERE table_id = " << quote << table_id << " " << + "AND state <> " << std::to_string(TableSchema::TO_DELETE) << ") " << + "AS " << quote << "check" << ";"; + + ENGINE_LOG_DEBUG << "MySQLMetaImpl::HasTable: " << hasTableQuery.str(); + + res = hasTableQuery.store(); + } //Scoped Connection + + int check = res[0]["check"]; + has_or_not = (check == 1); + + } catch (const BadQuery &er) { + // Handle any query errors + ENGINE_LOG_ERROR << "QUERY ERROR WHEN CHECKING IF TABLE EXISTS" << ": " << er.what(); + return Status::DBTransactionError("QUERY ERROR WHEN CHECKING IF TABLE EXISTS", er.what()); + } catch (const Exception &er) { + // Catch-all for any other MySQL++ exceptions + ENGINE_LOG_ERROR << "GENERAL ERROR WHEN CHECKING IF TABLE EXISTS" << ": " << er.what(); + return Status::DBTransactionError("GENERAL ERROR WHEN CHECKING IF TABLE EXISTS", er.what()); + } + + return Status::OK(); +} + +Status MySQLMetaImpl::AllTables(std::vector &table_schema_array) { + + + try { + + MetricCollector metric; + + StoreQueryResult res; + + { + ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab); + + if (connectionPtr == nullptr) { + return Status::Error("Failed to connect to database server"); + } + + + Query allTablesQuery = connectionPtr->query(); + allTablesQuery << "SELECT id, table_id, dimension, files_cnt, engine_type, store_raw_data " << + "FROM Tables " << + "WHERE state <> " << std::to_string(TableSchema::TO_DELETE) << ";"; + + ENGINE_LOG_DEBUG << "MySQLMetaImpl::AllTables: " << allTablesQuery.str(); + + res = allTablesQuery.store(); + } //Scoped Connection + + for (auto &resRow : res) { + TableSchema table_schema; + + table_schema.id_ = resRow["id"]; //implicit conversion + + std::string table_id; + resRow["table_id"].to_string(table_id); + table_schema.table_id_ = table_id; + + table_schema.dimension_ = resRow["dimension"]; + + table_schema.files_cnt_ = resRow["files_cnt"]; + + table_schema.engine_type_ = resRow["engine_type"]; + + int store_raw_data = resRow["store_raw_data"]; + table_schema.store_raw_data_ = (store_raw_data == 1); + + table_schema_array.emplace_back(table_schema); + } + } catch (const BadQuery &er) { + // Handle any query errors + ENGINE_LOG_ERROR << "QUERY ERROR WHEN DESCRIBING ALL TABLES" << ": " << er.what(); + return Status::DBTransactionError("QUERY ERROR WHEN DESCRIBING ALL TABLES", er.what()); + } catch (const Exception &er) { + // Catch-all for any other MySQL++ exceptions + ENGINE_LOG_ERROR << "GENERAL ERROR WHEN DESCRIBING ALL TABLES" << ": " << er.what(); + return Status::DBTransactionError("GENERAL ERROR WHEN DESCRIBING ALL TABLES", er.what()); + } + + return Status::OK(); +} + +Status MySQLMetaImpl::CreateTableFile(TableFileSchema &file_schema) { + + + if (file_schema.date_ == EmptyDate) { + file_schema.date_ = Meta::GetDate(); + } + TableSchema table_schema; + table_schema.table_id_ = file_schema.table_id_; + auto status = DescribeTable(table_schema); + if (!status.ok()) { + return status; + } + + try { + + MetricCollector metric; + + NextFileId(file_schema.file_id_); + file_schema.file_type_ = TableFileSchema::NEW; + file_schema.dimension_ = table_schema.dimension_; + file_schema.size_ = 0; + file_schema.created_on_ = utils::GetMicroSecTimeStamp(); + file_schema.updated_time_ = file_schema.created_on_; + file_schema.engine_type_ = table_schema.engine_type_; + utils::GetTableFilePath(options_, file_schema); + + std::string id = "NULL"; //auto-increment + std::string table_id = file_schema.table_id_; + std::string engine_type = std::to_string(file_schema.engine_type_); + std::string file_id = file_schema.file_id_; + std::string file_type = std::to_string(file_schema.file_type_); + std::string size = std::to_string(file_schema.size_); + std::string updated_time = std::to_string(file_schema.updated_time_); + std::string created_on = std::to_string(file_schema.created_on_); + std::string date = std::to_string(file_schema.date_); + + { + ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab); + + if (connectionPtr == nullptr) { + return Status::Error("Failed to connect to database server"); + } + + + Query createTableFileQuery = connectionPtr->query(); + + createTableFileQuery << "INSERT INTO TableFiles VALUES" << + "(" << id << ", " << quote << table_id << ", " << engine_type << ", " << + quote << file_id << ", " << file_type << ", " << size << ", " << + updated_time << ", " << created_on << ", " << date << ");"; + + ENGINE_LOG_DEBUG << "MySQLMetaImpl::CreateTableFile: " << createTableFileQuery.str(); + + if (SimpleResult res = createTableFileQuery.execute()) { + file_schema.id_ = res.insert_id(); //Might need to use SELECT LAST_INSERT_ID()? + + //Consume all results to avoid "Commands out of sync" error + + + + } else { + ENGINE_LOG_ERROR << "QUERY ERROR WHEN ADDING TABLE FILE"; + return Status::DBTransactionError("Add file Error", createTableFileQuery.error()); + } + } // Scoped Connection + + return utils::CreateTableFilePath(options_, file_schema); + + } catch (const BadQuery &er) { + // Handle any query errors + ENGINE_LOG_ERROR << "QUERY ERROR WHEN ADDING TABLE FILE" << ": " << er.what(); + return Status::DBTransactionError("QUERY ERROR WHEN ADDING TABLE FILE", er.what()); + } catch (const Exception &er) { + // Catch-all for any other MySQL++ exceptions + ENGINE_LOG_ERROR << "GENERAL ERROR WHEN ADDING TABLE FILE" << ": " << er.what(); + return Status::DBTransactionError("GENERAL ERROR WHEN ADDING TABLE FILE", er.what()); + } catch (std::exception &ex) { + return HandleException("Encounter exception when create table file", ex); + } + + return Status::OK(); +} + +Status MySQLMetaImpl::FilesToIndex(TableFilesSchema &files) { + + + files.clear(); + + try { + + MetricCollector metric; + + StoreQueryResult res; + + { + ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab); + + if (connectionPtr == nullptr) { + return Status::Error("Failed to connect to database server"); + } + + + Query filesToIndexQuery = connectionPtr->query(); + filesToIndexQuery << "SELECT id, table_id, engine_type, file_id, file_type, size, date " << + "FROM TableFiles " << + "WHERE file_type = " << std::to_string(TableFileSchema::TO_INDEX) << ";"; + + ENGINE_LOG_DEBUG << "MySQLMetaImpl::FilesToIndex: " << filesToIndexQuery.str(); + + res = filesToIndexQuery.store(); + } //Scoped Connection + + std::map groups; + TableFileSchema table_file; + for (auto &resRow : res) { + + table_file.id_ = resRow["id"]; //implicit conversion + + std::string table_id; + resRow["table_id"].to_string(table_id); + table_file.table_id_ = table_id; + + table_file.engine_type_ = resRow["engine_type"]; + + std::string file_id; + resRow["file_id"].to_string(file_id); + table_file.file_id_ = file_id; + + table_file.file_type_ = resRow["file_type"]; + + table_file.size_ = resRow["size"]; + + table_file.date_ = resRow["date"]; + + auto groupItr = groups.find(table_file.table_id_); + if (groupItr == groups.end()) { + TableSchema table_schema; + table_schema.table_id_ = table_file.table_id_; + auto status = DescribeTable(table_schema); + if (!status.ok()) { + return status; + } + groups[table_file.table_id_] = table_schema; + + } + table_file.dimension_ = groups[table_file.table_id_].dimension_; + + utils::GetTableFilePath(options_, table_file); + + files.push_back(table_file); + } + } catch (const BadQuery &er) { + // Handle any query errors + ENGINE_LOG_ERROR << "QUERY ERROR WHEN FINDING TABLE FILES TO INDEX" << ": " << er.what(); + return Status::DBTransactionError("QUERY ERROR WHEN FINDING TABLE FILES TO INDEX", er.what()); + } catch (const Exception &er) { + // Catch-all for any other MySQL++ exceptions + ENGINE_LOG_ERROR << "GENERAL ERROR WHEN FINDING TABLE FILES TO INDEX" << ": " << er.what(); + return Status::DBTransactionError("GENERAL ERROR WHEN FINDING TABLE FILES TO INDEX", er.what()); + } + + return Status::OK(); +} + +Status MySQLMetaImpl::FilesToSearch(const std::string &table_id, + const DatesT &partition, + DatePartionedTableFilesSchema &files) { + + + files.clear(); + + try { + + MetricCollector metric; + + StoreQueryResult res; + + { + ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab); + + if (connectionPtr == nullptr) { + return Status::Error("Failed to connect to database server"); + } + + + if (partition.empty()) { + + Query filesToSearchQuery = connectionPtr->query(); + filesToSearchQuery << "SELECT id, table_id, engine_type, file_id, file_type, size, date " << + "FROM TableFiles " << + "WHERE table_id = " << quote << table_id << " AND " << + "(file_type = " << std::to_string(TableFileSchema::RAW) << " OR " << + "file_type = " << std::to_string(TableFileSchema::TO_INDEX) << " OR " << + "file_type = " << std::to_string(TableFileSchema::INDEX) << ");"; + + ENGINE_LOG_DEBUG << "MySQLMetaImpl::FilesToSearch: " << filesToSearchQuery.str(); + + res = filesToSearchQuery.store(); + + } else { + + Query filesToSearchQuery = connectionPtr->query(); + + std::stringstream partitionListSS; + for (auto &date : partition) { + partitionListSS << std::to_string(date) << ", "; + } + std::string partitionListStr = partitionListSS.str(); + partitionListStr = partitionListStr.substr(0, partitionListStr.size() - 2); //remove the last ", " + + filesToSearchQuery << "SELECT id, table_id, engine_type, file_id, file_type, size, date " << + "FROM TableFiles " << + "WHERE table_id = " << quote << table_id << " AND " << + "date IN (" << partitionListStr << ") AND " << + "(file_type = " << std::to_string(TableFileSchema::RAW) << " OR " << + "file_type = " << std::to_string(TableFileSchema::TO_INDEX) << " OR " << + "file_type = " << std::to_string(TableFileSchema::INDEX) << ");"; + + ENGINE_LOG_DEBUG << "MySQLMetaImpl::FilesToSearch: " << filesToSearchQuery.str(); + + res = filesToSearchQuery.store(); + + } + } //Scoped Connection + TableSchema table_schema; table_schema.table_id_ = table_id; auto status = DescribeTable(table_schema); @@ -253,469 +878,450 @@ namespace meta { return status; } - try { + TableFileSchema table_file; + for (auto &resRow : res) { - auto yesterday = GetDateWithDelta(-1); + table_file.id_ = resRow["id"]; //implicit conversion - for (auto &date : dates) { - if (date >= yesterday) { - return Status::Error("Could not delete partitions within 2 days"); - } + std::string table_id_str; + resRow["table_id"].to_string(table_id_str); + table_file.table_id_ = table_id_str; + + table_file.engine_type_ = resRow["engine_type"]; + + std::string file_id; + resRow["file_id"].to_string(file_id); + table_file.file_id_ = file_id; + + table_file.file_type_ = resRow["file_type"]; + + table_file.size_ = resRow["size"]; + + table_file.date_ = resRow["date"]; + + table_file.dimension_ = table_schema.dimension_; + + utils::GetTableFilePath(options_, table_file); + + auto dateItr = files.find(table_file.date_); + if (dateItr == files.end()) { + files[table_file.date_] = TableFilesSchema(); } - std::stringstream dateListSS; - for (auto &date : dates) { - dateListSS << std::to_string(date) << ", "; - } - std::string dateListStr = dateListSS.str(); - dateListStr = dateListStr.substr(0, dateListStr.size() - 2); //remove the last ", " - - { - ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab); - - if (connectionPtr == nullptr) { - return Status::Error("Failed to connect to database server"); - } - -// if (mysql_connection_pool_->getConnectionsInUse() <= 0) { -// ENGINE_LOG_WARNING << "MySQLMetaImpl::DropPartitionsByDates connection in use = " << mysql_connection_pool_->getConnectionsInUse(); -// } - - Query dropPartitionsByDatesQuery = connectionPtr->query(); - - dropPartitionsByDatesQuery << "UPDATE TableFiles " << - "SET file_type = " << std::to_string(TableFileSchema::TO_DELETE) << " " << - "WHERE table_id = " << quote << table_id << " AND " << - "date in (" << dateListStr << ");"; - - ENGINE_LOG_DEBUG << "MySQLMetaImpl::DropPartitionsByDates: " << dropPartitionsByDatesQuery.str(); - - if (!dropPartitionsByDatesQuery.exec()) { - ENGINE_LOG_ERROR << "QUERY ERROR WHEN DROPPING PARTITIONS BY DATES"; - return Status::DBTransactionError("QUERY ERROR WHEN DROPPING PARTITIONS BY DATES", - dropPartitionsByDatesQuery.error()); - } - } //Scoped Connection - } catch (const BadQuery& er) { - // Handle any query errors - ENGINE_LOG_ERROR << "QUERY ERROR WHEN DROPPING PARTITIONS BY DATES" << ": " << er.what(); - return Status::DBTransactionError("QUERY ERROR WHEN DROPPING PARTITIONS BY DATES", er.what()); - } catch (const Exception& er) { - // Catch-all for any other MySQL++ exceptions - ENGINE_LOG_ERROR << "GENERAL ERROR WHEN DROPPING PARTITIONS BY DATES" << ": " << er.what(); - return Status::DBTransactionError("GENERAL ERROR WHEN DROPPING PARTITIONS BY DATES", er.what()); + files[table_file.date_].push_back(table_file); } - return Status::OK(); + } catch (const BadQuery &er) { + // Handle any query errors + ENGINE_LOG_ERROR << "QUERY ERROR WHEN FINDING TABLE FILES TO SEARCH" << ": " << er.what(); + return Status::DBTransactionError("QUERY ERROR WHEN FINDING TABLE FILES TO SEARCH", er.what()); + } catch (const Exception &er) { + // Catch-all for any other MySQL++ exceptions + ENGINE_LOG_ERROR << "GENERAL ERROR WHEN FINDING TABLE FILES TO SEARCH" << ": " << er.what(); + return Status::DBTransactionError("GENERAL ERROR WHEN FINDING TABLE FILES TO SEARCH", er.what()); } - Status MySQLMetaImpl::CreateTable(TableSchema &table_schema) { + return Status::OK(); +} -// std::lock_guard lock(mysql_mutex); - -// server::Metrics::GetInstance().MetaAccessTotalIncrement(); - try { - - MetricCollector metric; - - { - ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab); - - if (connectionPtr == nullptr) { - return Status::Error("Failed to connect to database server"); - } - -// if (mysql_connection_pool_->getConnectionsInUse() <= 0) { -// ENGINE_LOG_WARNING << "MySQLMetaImpl::CreateTable connection in use = " << mysql_connection_pool_->getConnectionsInUse(); -// } - - Query createTableQuery = connectionPtr->query(); -// ENGINE_LOG_DEBUG << "Create Table in"; - if (table_schema.table_id_.empty()) { - NextTableId(table_schema.table_id_); - } else { - createTableQuery << "SELECT state FROM Tables " << - "WHERE table_id = " << quote << table_schema.table_id_ << ";"; -// ENGINE_LOG_DEBUG << "Create Table : " << createTableQuery.str(); - - ENGINE_LOG_DEBUG << "MySQLMetaImpl::CreateTable: " << createTableQuery.str(); - - StoreQueryResult res = createTableQuery.store(); - - if (res.num_rows() == 1) { - int state = res[0]["state"]; - if (TableSchema::TO_DELETE == state) { - return Status::Error("Table already exists and it is in delete state, please wait a second"); - } - else { - return Status::AlreadyExist("Table already exists"); - } - } - } -// ENGINE_LOG_DEBUG << "Create Table start"; - - table_schema.files_cnt_ = 0; - table_schema.id_ = -1; - table_schema.created_on_ = utils::GetMicroSecTimeStamp(); - -// auto start_time = METRICS_NOW_TIME; - - std::string id = "NULL"; //auto-increment - std::string table_id = table_schema.table_id_; - std::string state = std::to_string(table_schema.state_); - std::string dimension = std::to_string(table_schema.dimension_); - std::string created_on = std::to_string(table_schema.created_on_); - std::string files_cnt = "0"; - std::string engine_type = std::to_string(table_schema.engine_type_); - std::string store_raw_data = table_schema.store_raw_data_ ? "true" : "false"; - - createTableQuery << "INSERT INTO Tables VALUES" << - "(" << id << ", " << quote << table_id << ", " << state << ", " << dimension << ", " << - created_on << ", " << files_cnt << ", " << engine_type << ", " << store_raw_data << ");"; -// ENGINE_LOG_DEBUG << "Create Table : " << createTableQuery.str(); - - ENGINE_LOG_DEBUG << "MySQLMetaImpl::CreateTable: " << createTableQuery.str(); - - if (SimpleResult res = createTableQuery.execute()) { - table_schema.id_ = res.insert_id(); //Might need to use SELECT LAST_INSERT_ID()? -// std::cout << table_schema.id_ << std::endl; - //Consume all results to avoid "Commands out of sync" error -// while (createTableQuery.more_results()) { -// createTableQuery.store_next(); -// } - } else { - ENGINE_LOG_ERROR << "Add Table Error"; - return Status::DBTransactionError("Add Table Error", createTableQuery.error()); - } - } //Scoped Connection - -// auto end_time = METRICS_NOW_TIME; -// auto total_time = METRICS_MICROSECONDS(start_time, end_time); -// server::Metrics::GetInstance().MetaAccessDurationSecondsHistogramObserve(total_time); - - return utils::CreateTablePath(options_, table_schema.table_id_); - - } catch (const BadQuery& er) { - // Handle any query errors - ENGINE_LOG_ERROR << "QUERY ERROR WHEN ADDING TABLE" << ": " << er.what(); - return Status::DBTransactionError("QUERY ERROR WHEN ADDING TABLE", er.what()); - } catch (const Exception& er) { - // Catch-all for any other MySQL++ exceptions - ENGINE_LOG_ERROR << "GENERAL ERROR WHEN ADDING TABLE" << ": " << er.what(); - return Status::DBTransactionError("GENERAL ERROR WHEN ADDING TABLE", er.what()); - } catch (std::exception &e) { - return HandleException("Encounter exception when create table", e); - } - - return Status::OK(); - } - - Status MySQLMetaImpl::HasNonIndexFiles(const std::string& table_id, bool& has) { - // TODO - return Status::OK(); - } - - Status MySQLMetaImpl::DeleteTable(const std::string& table_id) { - -// std::lock_guard lock(mysql_mutex); - - try { - - MetricCollector metric; - - { - ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab); - - if (connectionPtr == nullptr) { - return Status::Error("Failed to connect to database server"); - } - -// if (mysql_connection_pool_->getConnectionsInUse() <= 0) { -// ENGINE_LOG_WARNING << "MySQLMetaImpl::DeleteTable connection in use = " << mysql_connection_pool_->getConnectionsInUse(); -// } - - //soft delete table - Query deleteTableQuery = connectionPtr->query(); -// - deleteTableQuery << "UPDATE Tables " << - "SET state = " << std::to_string(TableSchema::TO_DELETE) << " " << - "WHERE table_id = " << quote << table_id << ";"; - - ENGINE_LOG_DEBUG << "MySQLMetaImpl::DeleteTable: " << deleteTableQuery.str(); - - if (!deleteTableQuery.exec()) { - ENGINE_LOG_ERROR << "QUERY ERROR WHEN DELETING TABLE"; - return Status::DBTransactionError("QUERY ERROR WHEN DELETING TABLE", deleteTableQuery.error()); - } - - } //Scoped Connection +Status MySQLMetaImpl::FilesToMerge(const std::string &table_id, + DatePartionedTableFilesSchema &files) { - if (mode_ == Options::MODE::CLUSTER) { - DeleteTableFiles(table_id); + files.clear(); + + try { + MetricCollector metric; + + StoreQueryResult res; + + { + ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab); + + if (connectionPtr == nullptr) { + return Status::Error("Failed to connect to database server"); } - } catch (const BadQuery& er) { - // Handle any query errors - ENGINE_LOG_ERROR << "GENERAL ERROR WHEN DELETING TABLE" << ": " << er.what(); - return Status::DBTransactionError("QUERY ERROR WHEN DELETING TABLE", er.what()); - } catch (const Exception& er) { - // Catch-all for any other MySQL++ exceptions - ENGINE_LOG_ERROR << "GENERAL ERROR WHEN DELETING TABLE" << ": " << er.what(); - return Status::DBTransactionError("GENERAL ERROR WHEN DELETING TABLE", er.what()); - } - return Status::OK(); - } + Query filesToMergeQuery = connectionPtr->query(); + filesToMergeQuery << "SELECT id, table_id, file_id, file_type, size, date " << + "FROM TableFiles " << + "WHERE table_id = " << quote << table_id << " AND " << + "file_type = " << std::to_string(TableFileSchema::RAW) << " " << + "ORDER BY size DESC" << ";"; - Status MySQLMetaImpl::DeleteTableFiles(const std::string& table_id) { - try { - MetricCollector metric; + ENGINE_LOG_DEBUG << "MySQLMetaImpl::FilesToMerge: " << filesToMergeQuery.str(); - { - ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab); + res = filesToMergeQuery.store(); + } //Scoped Connection - if (connectionPtr == nullptr) { - return Status::Error("Failed to connect to database server"); - } - -// if (mysql_connection_pool_->getConnectionsInUse() <= 0) { -// ENGINE_LOG_WARNING << "MySQLMetaImpl::DeleteTableFiles connection in use = " << mysql_connection_pool_->getConnectionsInUse(); -// } - - //soft delete table files - Query deleteTableFilesQuery = connectionPtr->query(); - // - deleteTableFilesQuery << "UPDATE TableFiles " << - "SET file_type = " << std::to_string(TableFileSchema::TO_DELETE) << ", " << - "updated_time = " << std::to_string(utils::GetMicroSecTimeStamp()) << " " << - "WHERE table_id = " << quote << table_id << " AND " << - "file_type <> " << std::to_string(TableFileSchema::TO_DELETE) << ";"; - - ENGINE_LOG_DEBUG << "MySQLMetaImpl::DeleteTableFiles: " << deleteTableFilesQuery.str(); - - if (!deleteTableFilesQuery.exec()) { - ENGINE_LOG_ERROR << "QUERY ERROR WHEN DELETING TABLE FILES"; - return Status::DBTransactionError("QUERY ERROR WHEN DELETING TABLE", deleteTableFilesQuery.error()); - } - } //Scoped Connection - } catch (const BadQuery& er) { - // Handle any query errors - ENGINE_LOG_ERROR << "QUERY ERROR WHEN DELETING TABLE FILES" << ": " << er.what(); - return Status::DBTransactionError("QUERY ERROR WHEN DELETING TABLE FILES", er.what()); - } catch (const Exception& er) { - // Catch-all for any other MySQL++ exceptions - ENGINE_LOG_ERROR << "GENERAL ERROR WHEN DELETING TABLE FILES" << ": " << er.what(); - return Status::DBTransactionError("GENERAL ERROR WHEN DELETING TABLE FILES", er.what()); - } - - return Status::OK(); - } - - Status MySQLMetaImpl::DescribeTable(TableSchema &table_schema) { - -// std::lock_guard lock(mysql_mutex); - - try { - - MetricCollector metric; - - StoreQueryResult res; - - { - ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab); - - if (connectionPtr == nullptr) { - return Status::Error("Failed to connect to database server"); - } - -// if (mysql_connection_pool_->getConnectionsInUse() <= 0) { -// ENGINE_LOG_WARNING << "MySQLMetaImpl::DescribeTable connection in use = " << mysql_connection_pool_->getConnectionsInUse(); -// } - - Query describeTableQuery = connectionPtr->query(); - describeTableQuery << "SELECT id, dimension, files_cnt, engine_type, store_raw_data " << - "FROM Tables " << - "WHERE table_id = " << quote << table_schema.table_id_ << " " << - "AND state <> " << std::to_string(TableSchema::TO_DELETE) << ";"; - - ENGINE_LOG_DEBUG << "MySQLMetaImpl::DescribeTable: " << describeTableQuery.str(); - - res = describeTableQuery.store(); - } //Scoped Connection - - if (res.num_rows() == 1) { - const Row& resRow = res[0]; - - table_schema.id_ = resRow["id"]; //implicit conversion - - table_schema.dimension_ = resRow["dimension"]; - - table_schema.files_cnt_ = resRow["files_cnt"]; - - table_schema.engine_type_ = resRow["engine_type"]; - - int store_raw_data = resRow["store_raw_data"]; - table_schema.store_raw_data_ = (store_raw_data == 1); - } - else { - return Status::NotFound("Table " + table_schema.table_id_ + " not found"); - } - - } catch (const BadQuery& er) { - // Handle any query errors - ENGINE_LOG_ERROR << "QUERY ERROR WHEN DESCRIBING TABLE" << ": " << er.what(); - return Status::DBTransactionError("QUERY ERROR WHEN DESCRIBING TABLE", er.what()); - } catch (const Exception& er) { - // Catch-all for any other MySQL++ exceptions - ENGINE_LOG_ERROR << "GENERAL ERROR WHEN DESCRIBING TABLE" << ": " << er.what(); - return Status::DBTransactionError("GENERAL ERROR WHEN DESCRIBING TABLE", er.what()); - } - - return Status::OK(); - } - - Status MySQLMetaImpl::HasTable(const std::string &table_id, bool &has_or_not) { - -// std::lock_guard lock(mysql_mutex); - - try { - - MetricCollector metric; - - StoreQueryResult res; - - { - ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab); - - if (connectionPtr == nullptr) { - return Status::Error("Failed to connect to database server"); - } - -// if (mysql_connection_pool_->getConnectionsInUse() <= 0) { -// ENGINE_LOG_WARNING << "MySQLMetaImpl::HasTable connection in use = " << mysql_connection_pool_->getConnectionsInUse(); -// } - - Query hasTableQuery = connectionPtr->query(); - //since table_id is a unique column we just need to check whether it exists or not - hasTableQuery << "SELECT EXISTS " << - "(SELECT 1 FROM Tables " << - "WHERE table_id = " << quote << table_id << " " << - "AND state <> " << std::to_string(TableSchema::TO_DELETE) << ") " << - "AS " << quote << "check" << ";"; - - ENGINE_LOG_DEBUG << "MySQLMetaImpl::HasTable: " << hasTableQuery.str(); - - res = hasTableQuery.store(); - } //Scoped Connection - - int check = res[0]["check"]; - has_or_not = (check == 1); - - } catch (const BadQuery& er) { - // Handle any query errors - ENGINE_LOG_ERROR << "QUERY ERROR WHEN CHECKING IF TABLE EXISTS" << ": " << er.what(); - return Status::DBTransactionError("QUERY ERROR WHEN CHECKING IF TABLE EXISTS", er.what()); - } catch (const Exception& er) { - // Catch-all for any other MySQL++ exceptions - ENGINE_LOG_ERROR << "GENERAL ERROR WHEN CHECKING IF TABLE EXISTS" << ": " << er.what(); - return Status::DBTransactionError("GENERAL ERROR WHEN CHECKING IF TABLE EXISTS", er.what()); - } - - return Status::OK(); - } - - Status MySQLMetaImpl::AllTables(std::vector& table_schema_array) { - -// std::lock_guard lock(mysql_mutex); - - try { - - MetricCollector metric; - - StoreQueryResult res; - - { - ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab); - - if (connectionPtr == nullptr) { - return Status::Error("Failed to connect to database server"); - } - -// if (mysql_connection_pool_->getConnectionsInUse() <= 0) { -// ENGINE_LOG_WARNING << "MySQLMetaImpl::AllTables connection in use = " << mysql_connection_pool_->getConnectionsInUse(); -// } - - Query allTablesQuery = connectionPtr->query(); - allTablesQuery << "SELECT id, table_id, dimension, files_cnt, engine_type, store_raw_data " << - "FROM Tables " << - "WHERE state <> " << std::to_string(TableSchema::TO_DELETE) << ";"; - - ENGINE_LOG_DEBUG << "MySQLMetaImpl::AllTables: " << allTablesQuery.str(); - - res = allTablesQuery.store(); - } //Scoped Connection - - for (auto& resRow : res) { - TableSchema table_schema; - - table_schema.id_ = resRow["id"]; //implicit conversion - - std::string table_id; - resRow["table_id"].to_string(table_id); - table_schema.table_id_ = table_id; - - table_schema.dimension_ = resRow["dimension"]; - - table_schema.files_cnt_ = resRow["files_cnt"]; - - table_schema.engine_type_ = resRow["engine_type"]; - - int store_raw_data = resRow["store_raw_data"]; - table_schema.store_raw_data_ = (store_raw_data == 1); - - table_schema_array.emplace_back(table_schema); - } - } catch (const BadQuery& er) { - // Handle any query errors - ENGINE_LOG_ERROR << "QUERY ERROR WHEN DESCRIBING ALL TABLES" << ": " << er.what(); - return Status::DBTransactionError("QUERY ERROR WHEN DESCRIBING ALL TABLES", er.what()); - } catch (const Exception& er) { - // Catch-all for any other MySQL++ exceptions - ENGINE_LOG_ERROR << "GENERAL ERROR WHEN DESCRIBING ALL TABLES" << ": " << er.what(); - return Status::DBTransactionError("GENERAL ERROR WHEN DESCRIBING ALL TABLES", er.what()); - } - - return Status::OK(); - } - - Status MySQLMetaImpl::CreateTableFile(TableFileSchema &file_schema) { - -// std::lock_guard lock(mysql_mutex); - - if (file_schema.date_ == EmptyDate) { - file_schema.date_ = Meta::GetDate(); - } TableSchema table_schema; - table_schema.table_id_ = file_schema.table_id_; + table_schema.table_id_ = table_id; + auto status = DescribeTable(table_schema); + + if (!status.ok()) { + return status; + } + + TableFileSchema table_file; + for (auto &resRow : res) { + + table_file.id_ = resRow["id"]; //implicit conversion + + std::string table_id_str; + resRow["table_id"].to_string(table_id_str); + table_file.table_id_ = table_id_str; + + std::string file_id; + resRow["file_id"].to_string(file_id); + table_file.file_id_ = file_id; + + table_file.file_type_ = resRow["file_type"]; + + table_file.size_ = resRow["size"]; + + table_file.date_ = resRow["date"]; + + table_file.dimension_ = table_schema.dimension_; + + utils::GetTableFilePath(options_, table_file); + + auto dateItr = files.find(table_file.date_); + if (dateItr == files.end()) { + files[table_file.date_] = TableFilesSchema(); + } + + files[table_file.date_].push_back(table_file); + } + + } catch (const BadQuery &er) { + // Handle any query errors + ENGINE_LOG_ERROR << "QUERY ERROR WHEN FINDING TABLE FILES TO MERGE" << ": " << er.what(); + return Status::DBTransactionError("QUERY ERROR WHEN FINDING TABLE FILES TO MERGE", er.what()); + } catch (const Exception &er) { + // Catch-all for any other MySQL++ exceptions + ENGINE_LOG_ERROR << "GENERAL ERROR WHEN FINDING TABLE FILES TO MERGE" << ": " << er.what(); + return Status::DBTransactionError("GENERAL ERROR WHEN FINDING TABLE FILES TO MERGE", er.what()); + } + + return Status::OK(); +} + +Status MySQLMetaImpl::GetTableFiles(const std::string &table_id, + const std::vector &ids, + TableFilesSchema &table_files) { + + + if (ids.empty()) { + return Status::OK(); + } + + std::stringstream idSS; + for (auto &id : ids) { + idSS << "id = " << std::to_string(id) << " OR "; + } + std::string idStr = idSS.str(); + idStr = idStr.substr(0, idStr.size() - 4); //remove the last " OR " + + try { + + StoreQueryResult res; + + { + ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab); + + if (connectionPtr == nullptr) { + return Status::Error("Failed to connect to database server"); + } + + + Query getTableFileQuery = connectionPtr->query(); + getTableFileQuery << "SELECT id, engine_type, file_id, file_type, size, date " << + "FROM TableFiles " << + "WHERE table_id = " << quote << table_id << " AND " << + "(" << idStr << ");"; + + ENGINE_LOG_DEBUG << "MySQLMetaImpl::GetTableFiles: " << getTableFileQuery.str(); + + res = getTableFileQuery.store(); + } //Scoped Connection + + TableSchema table_schema; + table_schema.table_id_ = table_id; auto status = DescribeTable(table_schema); if (!status.ok()) { return status; } - try { + for (auto &resRow : res) { - MetricCollector metric; + TableFileSchema file_schema; + + file_schema.id_ = resRow["id"]; + + file_schema.table_id_ = table_id; + + file_schema.engine_type_ = resRow["engine_type"]; + + std::string file_id; + resRow["file_id"].to_string(file_id); + file_schema.file_id_ = file_id; + + file_schema.file_type_ = resRow["file_type"]; + + file_schema.size_ = resRow["size"]; + + file_schema.date_ = resRow["date"]; - NextFileId(file_schema.file_id_); - file_schema.file_type_ = TableFileSchema::NEW; file_schema.dimension_ = table_schema.dimension_; - file_schema.size_ = 0; - file_schema.created_on_ = utils::GetMicroSecTimeStamp(); - file_schema.updated_time_ = file_schema.created_on_; - file_schema.engine_type_ = table_schema.engine_type_; + utils::GetTableFilePath(options_, file_schema); - std::string id = "NULL"; //auto-increment + table_files.emplace_back(file_schema); + } + } catch (const BadQuery &er) { + // Handle any query errors + ENGINE_LOG_ERROR << "QUERY ERROR WHEN RETRIEVING TABLE FILES" << ": " << er.what(); + return Status::DBTransactionError("QUERY ERROR WHEN RETRIEVING TABLE FILES", er.what()); + } catch (const Exception &er) { + // Catch-all for any other MySQL++ exceptions + ENGINE_LOG_ERROR << "GENERAL ERROR WHEN RETRIEVING TABLE FILES" << ": " << er.what(); + return Status::DBTransactionError("GENERAL ERROR WHEN RETRIEVING TABLE FILES", er.what()); + } + + return Status::OK(); +} + +// PXU TODO: Support Swap +Status MySQLMetaImpl::Archive() { + + + auto &criterias = options_.archive_conf.GetCriterias(); + if (criterias.empty()) { + return Status::OK(); + } + + for (auto &kv : criterias) { + auto &criteria = kv.first; + auto &limit = kv.second; + if (criteria == "days") { + size_t usecs = limit * D_SEC * US_PS; + long now = utils::GetMicroSecTimeStamp(); + + try { + + ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab); + + if (connectionPtr == nullptr) { + return Status::Error("Failed to connect to database server"); + } + + + Query archiveQuery = connectionPtr->query(); + archiveQuery << "UPDATE TableFiles " << + "SET file_type = " << std::to_string(TableFileSchema::TO_DELETE) << " " << + "WHERE created_on < " << std::to_string(now - usecs) << " AND " << + "file_type <> " << std::to_string(TableFileSchema::TO_DELETE) << ";"; + + ENGINE_LOG_DEBUG << "MySQLMetaImpl::Archive: " << archiveQuery.str(); + + if (!archiveQuery.exec()) { + return Status::DBTransactionError("QUERY ERROR DURING ARCHIVE", archiveQuery.error()); + } + + } catch (const BadQuery &er) { + // Handle any query errors + ENGINE_LOG_ERROR << "QUERY ERROR WHEN DURING ARCHIVE" << ": " << er.what(); + return Status::DBTransactionError("QUERY ERROR WHEN DURING ARCHIVE", er.what()); + } catch (const Exception &er) { + // Catch-all for any other MySQL++ exceptions + ENGINE_LOG_ERROR << "GENERAL ERROR WHEN DURING ARCHIVE" << ": " << er.what(); + return Status::DBTransactionError("GENERAL ERROR WHEN DURING ARCHIVE", er.what()); + } + } + if (criteria == "disk") { + uint64_t sum = 0; + Size(sum); + + auto to_delete = (sum - limit * G); + DiscardFiles(to_delete); + } + } + + return Status::OK(); +} + +Status MySQLMetaImpl::Size(uint64_t &result) { + + + result = 0; + try { + + StoreQueryResult res; + + { + ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab); + + if (connectionPtr == nullptr) { + return Status::Error("Failed to connect to database server"); + } + + + Query getSizeQuery = connectionPtr->query(); + getSizeQuery << "SELECT IFNULL(SUM(size),0) AS sum " << + "FROM TableFiles " << + "WHERE file_type <> " << std::to_string(TableFileSchema::TO_DELETE) << ";"; + + ENGINE_LOG_DEBUG << "MySQLMetaImpl::Size: " << getSizeQuery.str(); + + res = getSizeQuery.store(); + } //Scoped Connection + + +// + + + if (res.empty()) { + result = 0; + + } else { + result = res[0]["sum"]; + + } + + } catch (const BadQuery &er) { + // Handle any query errors + ENGINE_LOG_ERROR << "QUERY ERROR WHEN RETRIEVING SIZE" << ": " << er.what(); + return Status::DBTransactionError("QUERY ERROR WHEN RETRIEVING SIZE", er.what()); + } catch (const Exception &er) { + // Catch-all for any other MySQL++ exceptions + ENGINE_LOG_ERROR << "GENERAL ERROR WHEN RETRIEVING SIZE" << ": " << er.what(); + return Status::DBTransactionError("GENERAL ERROR WHEN RETRIEVING SIZE", er.what()); + } + + return Status::OK(); +} + +Status MySQLMetaImpl::DiscardFiles(long long to_discard_size) { + + + if (to_discard_size <= 0) { + + return Status::OK(); + } + ENGINE_LOG_DEBUG << "About to discard size=" << to_discard_size; + + try { + + MetricCollector metric; + + bool status; + + { + ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab); + + if (connectionPtr == nullptr) { + return Status::Error("Failed to connect to database server"); + } + + + Query discardFilesQuery = connectionPtr->query(); + discardFilesQuery << "SELECT id, size " << + "FROM TableFiles " << + "WHERE file_type <> " << std::to_string(TableFileSchema::TO_DELETE) << " " << + "ORDER BY id ASC " << + "LIMIT 10;"; + + ENGINE_LOG_DEBUG << "MySQLMetaImpl::DiscardFiles: " << discardFilesQuery.str(); + + + StoreQueryResult res = discardFilesQuery.store(); + + if (res.num_rows() == 0) { + return Status::OK(); + } + + TableFileSchema table_file; + std::stringstream idsToDiscardSS; + for (auto &resRow : res) { + if (to_discard_size <= 0) { + break; + } + table_file.id_ = resRow["id"]; + table_file.size_ = resRow["size"]; + idsToDiscardSS << "id = " << std::to_string(table_file.id_) << " OR "; + ENGINE_LOG_DEBUG << "Discard table_file.id=" << table_file.file_id_ + << " table_file.size=" << table_file.size_; + to_discard_size -= table_file.size_; + } + + std::string idsToDiscardStr = idsToDiscardSS.str(); + idsToDiscardStr = idsToDiscardStr.substr(0, idsToDiscardStr.size() - 4); //remove the last " OR " + + discardFilesQuery << "UPDATE TableFiles " << + "SET file_type = " << std::to_string(TableFileSchema::TO_DELETE) << ", " << + "updated_time = " << std::to_string(utils::GetMicroSecTimeStamp()) << " " << + "WHERE " << idsToDiscardStr << ";"; + + ENGINE_LOG_DEBUG << "MySQLMetaImpl::DiscardFiles: " << discardFilesQuery.str(); + + status = discardFilesQuery.exec(); + if (!status) { + ENGINE_LOG_ERROR << "QUERY ERROR WHEN DISCARDING FILES"; + return Status::DBTransactionError("QUERY ERROR WHEN DISCARDING FILES", discardFilesQuery.error()); + } + } //Scoped Connection + + return DiscardFiles(to_discard_size); + + } catch (const BadQuery &er) { + // Handle any query errors + ENGINE_LOG_ERROR << "QUERY ERROR WHEN DISCARDING FILES" << ": " << er.what(); + return Status::DBTransactionError("QUERY ERROR WHEN DISCARDING FILES", er.what()); + } catch (const Exception &er) { + // Catch-all for any other MySQL++ exceptions + ENGINE_LOG_ERROR << "GENERAL ERROR WHEN DISCARDING FILES" << ": " << er.what(); + return Status::DBTransactionError("GENERAL ERROR WHEN DISCARDING FILES", er.what()); + } +} + +//ZR: this function assumes all fields in file_schema have value +Status MySQLMetaImpl::UpdateTableFile(TableFileSchema &file_schema) { + + + file_schema.updated_time_ = utils::GetMicroSecTimeStamp(); + try { + + MetricCollector metric; + + { + ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab); + + if (connectionPtr == nullptr) { + return Status::Error("Failed to connect to database server"); + } + + + Query updateTableFileQuery = connectionPtr->query(); + + //if the table has been deleted, just mark the table file as TO_DELETE + //clean thread will delete the file later + updateTableFileQuery << "SELECT state FROM Tables " << + "WHERE table_id = " << quote << file_schema.table_id_ << ";"; + + ENGINE_LOG_DEBUG << "MySQLMetaImpl::UpdateTableFile: " << updateTableFileQuery.str(); + + StoreQueryResult res = updateTableFileQuery.store(); + + if (res.num_rows() == 1) { + int state = res[0]["state"]; + if (state == TableSchema::TO_DELETE) { + file_schema.file_type_ = TableFileSchema::TO_DELETE; + } + } else { + file_schema.file_type_ = TableFileSchema::TO_DELETE; + } + + std::string id = std::to_string(file_schema.id_); std::string table_id = file_schema.table_id_; std::string engine_type = std::to_string(file_schema.engine_type_); std::string file_id = file_schema.file_id_; @@ -725,679 +1331,115 @@ namespace meta { std::string created_on = std::to_string(file_schema.created_on_); std::string date = std::to_string(file_schema.date_); - { - ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab); + updateTableFileQuery << "UPDATE TableFiles " << + "SET table_id = " << quote << table_id << ", " << + "engine_type = " << engine_type << ", " << + "file_id = " << quote << file_id << ", " << + "file_type = " << file_type << ", " << + "size = " << size << ", " << + "updated_time = " << updated_time << ", " << + "created_on = " << created_on << ", " << + "date = " << date << " " << + "WHERE id = " << id << ";"; - if (connectionPtr == nullptr) { - return Status::Error("Failed to connect to database server"); - } + ENGINE_LOG_DEBUG << "MySQLMetaImpl::UpdateTableFile: " << updateTableFileQuery.str(); -// if (mysql_connection_pool_->getConnectionsInUse() <= 0) { -// ENGINE_LOG_WARNING << "MySQLMetaImpl::CreateTableFile connection in use = " << mysql_connection_pool_->getConnectionsInUse(); -// } - Query createTableFileQuery = connectionPtr->query(); + if (!updateTableFileQuery.exec()) { + ENGINE_LOG_DEBUG << "table_id= " << file_schema.table_id_ << " file_id=" << file_schema.file_id_; + ENGINE_LOG_ERROR << "QUERY ERROR WHEN UPDATING TABLE FILE"; + return Status::DBTransactionError("QUERY ERROR WHEN UPDATING TABLE FILE", + updateTableFileQuery.error()); + } + } //Scoped Connection - createTableFileQuery << "INSERT INTO TableFiles VALUES" << - "(" << id << ", " << quote << table_id << ", " << engine_type << ", " << - quote << file_id << ", " << file_type << ", " << size << ", " << - updated_time << ", " << created_on << ", " << date << ");"; + } catch (const BadQuery &er) { + // Handle any query errors + ENGINE_LOG_DEBUG << "table_id= " << file_schema.table_id_ << " file_id=" << file_schema.file_id_; + ENGINE_LOG_ERROR << "QUERY ERROR WHEN UPDATING TABLE FILE" << ": " << er.what(); + return Status::DBTransactionError("QUERY ERROR WHEN UPDATING TABLE FILE", er.what()); + } catch (const Exception &er) { + // Catch-all for any other MySQL++ exceptions + ENGINE_LOG_DEBUG << "table_id= " << file_schema.table_id_ << " file_id=" << file_schema.file_id_; + ENGINE_LOG_ERROR << "GENERAL ERROR WHEN UPDATING TABLE FILE" << ": " << er.what(); + return Status::DBTransactionError("GENERAL ERROR WHEN UPDATING TABLE FILE", er.what()); + } + return Status::OK(); +} - ENGINE_LOG_DEBUG << "MySQLMetaImpl::CreateTableFile: " << createTableFileQuery.str(); +Status MySQLMetaImpl::UpdateTableFilesToIndex(const std::string &table_id) { + try { + ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab); - if (SimpleResult res = createTableFileQuery.execute()) { - file_schema.id_ = res.insert_id(); //Might need to use SELECT LAST_INSERT_ID()? - - //Consume all results to avoid "Commands out of sync" error -// while (createTableFileQuery.more_results()) { -// createTableFileQuery.store_next(); -// } - } else { - ENGINE_LOG_ERROR << "QUERY ERROR WHEN ADDING TABLE FILE"; - return Status::DBTransactionError("Add file Error", createTableFileQuery.error()); - } - } // Scoped Connection - - return utils::CreateTableFilePath(options_, file_schema); - - } catch (const BadQuery& er) { - // Handle any query errors - ENGINE_LOG_ERROR << "QUERY ERROR WHEN ADDING TABLE FILE" << ": " << er.what(); - return Status::DBTransactionError("QUERY ERROR WHEN ADDING TABLE FILE", er.what()); - } catch (const Exception& er) { - // Catch-all for any other MySQL++ exceptions - ENGINE_LOG_ERROR << "GENERAL ERROR WHEN ADDING TABLE FILE" << ": " << er.what(); - return Status::DBTransactionError("GENERAL ERROR WHEN ADDING TABLE FILE", er.what()); - } catch (std::exception& ex) { - return HandleException("Encounter exception when create table file", ex); + if (connectionPtr == nullptr) { + return Status::Error("Failed to connect to database server"); } - return Status::OK(); + Query updateTableFilesToIndexQuery = connectionPtr->query(); + + updateTableFilesToIndexQuery << "UPDATE TableFiles " << + "SET file_type = " << std::to_string(TableFileSchema::TO_INDEX) << " " << + "WHERE table_id = " << quote << table_id << " AND " << + "file_type = " << std::to_string(TableFileSchema::RAW) << ";"; + + ENGINE_LOG_DEBUG << "MySQLMetaImpl::UpdateTableFile: " << updateTableFilesToIndexQuery.str(); + + } catch (const BadQuery &er) { + // Handle any query errors + ENGINE_LOG_ERROR << "QUERY ERROR WHEN UPDATING TABLE FILES TO INDEX" << ": " << er.what(); + return Status::DBTransactionError("QUERY ERROR WHEN UPDATING TABLE FILES TO INDEX", er.what()); + } catch (const Exception &er) { + // Catch-all for any other MySQL++ exceptions + ENGINE_LOG_ERROR << "GENERAL ERROR WHEN UPDATING TABLE FILES TO INDEX" << ": " << er.what(); + return Status::DBTransactionError("GENERAL ERROR WHEN UPDATING TABLE FILES TO INDEX", er.what()); } - Status MySQLMetaImpl::FilesToIndex(TableFilesSchema &files) { + return Status::OK(); +} -// std::lock_guard lock(mysql_mutex); +Status MySQLMetaImpl::UpdateTableFiles(TableFilesSchema &files) { - files.clear(); - try { + try { + MetricCollector metric; - MetricCollector metric; + { + ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab); - StoreQueryResult res; - - { - ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab); - - if (connectionPtr == nullptr) { - return Status::Error("Failed to connect to database server"); - } - -// if (mysql_connection_pool_->getConnectionsInUse() <= 0) { -// ENGINE_LOG_WARNING << "MySQLMetaImpl::FilesToIndex connection in use = " << mysql_connection_pool_->getConnectionsInUse(); -// } - - Query filesToIndexQuery = connectionPtr->query(); - filesToIndexQuery << "SELECT id, table_id, engine_type, file_id, file_type, size, date " << - "FROM TableFiles " << - "WHERE file_type = " << std::to_string(TableFileSchema::TO_INDEX) << ";"; - - ENGINE_LOG_DEBUG << "MySQLMetaImpl::FilesToIndex: " << filesToIndexQuery.str(); - - res = filesToIndexQuery.store(); - } //Scoped Connection - - std::map groups; - TableFileSchema table_file; - for (auto& resRow : res) { - - table_file.id_ = resRow["id"]; //implicit conversion - - std::string table_id; - resRow["table_id"].to_string(table_id); - table_file.table_id_ = table_id; - - table_file.engine_type_ = resRow["engine_type"]; - - std::string file_id; - resRow["file_id"].to_string(file_id); - table_file.file_id_ = file_id; - - table_file.file_type_ = resRow["file_type"]; - - table_file.size_ = resRow["size"]; - - table_file.date_ = resRow["date"]; - - auto groupItr = groups.find(table_file.table_id_); - if (groupItr == groups.end()) { - TableSchema table_schema; - table_schema.table_id_ = table_file.table_id_; - auto status = DescribeTable(table_schema); - if (!status.ok()) { - return status; - } - groups[table_file.table_id_] = table_schema; -// std::cout << table_schema.dimension_ << std::endl; - } - table_file.dimension_ = groups[table_file.table_id_].dimension_; - - utils::GetTableFilePath(options_, table_file); - - files.push_back(table_file); - } - } catch (const BadQuery& er) { - // Handle any query errors - ENGINE_LOG_ERROR << "QUERY ERROR WHEN FINDING TABLE FILES TO INDEX" << ": " << er.what(); - return Status::DBTransactionError("QUERY ERROR WHEN FINDING TABLE FILES TO INDEX", er.what()); - } catch (const Exception& er) { - // Catch-all for any other MySQL++ exceptions - ENGINE_LOG_ERROR << "GENERAL ERROR WHEN FINDING TABLE FILES TO INDEX" << ": " << er.what(); - return Status::DBTransactionError("GENERAL ERROR WHEN FINDING TABLE FILES TO INDEX", er.what()); - } - - return Status::OK(); - } - - Status MySQLMetaImpl::FilesToSearch(const std::string &table_id, - const DatesT &partition, - DatePartionedTableFilesSchema &files) { - -// std::lock_guard lock(mysql_mutex); - - files.clear(); - - try { - - MetricCollector metric; - - StoreQueryResult res; - - { - ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab); - - if (connectionPtr == nullptr) { - return Status::Error("Failed to connect to database server"); - } -// if (mysql_connection_pool_->getConnectionsInUse() <= 0) { -// ENGINE_LOG_WARNING << "MySQLMetaImpl::FilesToSearch connection in use = " << mysql_connection_pool_->getConnectionsInUse(); -// } - - if (partition.empty()) { - - Query filesToSearchQuery = connectionPtr->query(); - filesToSearchQuery << "SELECT id, table_id, engine_type, file_id, file_type, size, date " << - "FROM TableFiles " << - "WHERE table_id = " << quote << table_id << " AND " << - "(file_type = " << std::to_string(TableFileSchema::RAW) << " OR " << - "file_type = " << std::to_string(TableFileSchema::TO_INDEX) << " OR " << - "file_type = " << std::to_string(TableFileSchema::INDEX) << ");"; - - ENGINE_LOG_DEBUG << "MySQLMetaImpl::FilesToSearch: " << filesToSearchQuery.str(); - - res = filesToSearchQuery.store(); - - } else { - - Query filesToSearchQuery = connectionPtr->query(); - - std::stringstream partitionListSS; - for (auto &date : partition) { - partitionListSS << std::to_string(date) << ", "; - } - std::string partitionListStr = partitionListSS.str(); - partitionListStr = partitionListStr.substr(0, partitionListStr.size() - 2); //remove the last ", " - - filesToSearchQuery << "SELECT id, table_id, engine_type, file_id, file_type, size, date " << - "FROM TableFiles " << - "WHERE table_id = " << quote << table_id << " AND " << - "date IN (" << partitionListStr << ") AND " << - "(file_type = " << std::to_string(TableFileSchema::RAW) << " OR " << - "file_type = " << std::to_string(TableFileSchema::TO_INDEX) << " OR " << - "file_type = " << std::to_string(TableFileSchema::INDEX) << ");"; - - ENGINE_LOG_DEBUG << "MySQLMetaImpl::FilesToSearch: " << filesToSearchQuery.str(); - - res = filesToSearchQuery.store(); - - } - } //Scoped Connection - - TableSchema table_schema; - table_schema.table_id_ = table_id; - auto status = DescribeTable(table_schema); - if (!status.ok()) { - return status; + if (connectionPtr == nullptr) { + return Status::Error("Failed to connect to database server"); } - TableFileSchema table_file; - for (auto& resRow : res) { - table_file.id_ = resRow["id"]; //implicit conversion + Query updateTableFilesQuery = connectionPtr->query(); - std::string table_id_str; - resRow["table_id"].to_string(table_id_str); - table_file.table_id_ = table_id_str; + std::map has_tables; + for (auto &file_schema : files) { - table_file.engine_type_ = resRow["engine_type"]; - - std::string file_id; - resRow["file_id"].to_string(file_id); - table_file.file_id_ = file_id; - - table_file.file_type_ = resRow["file_type"]; - - table_file.size_ = resRow["size"]; - - table_file.date_ = resRow["date"]; - - table_file.dimension_ = table_schema.dimension_; - - utils::GetTableFilePath(options_, table_file); - - auto dateItr = files.find(table_file.date_); - if (dateItr == files.end()) { - files[table_file.date_] = TableFilesSchema(); + if (has_tables.find(file_schema.table_id_) != has_tables.end()) { + continue; } - files[table_file.date_].push_back(table_file); - } - } catch (const BadQuery& er) { - // Handle any query errors - ENGINE_LOG_ERROR << "QUERY ERROR WHEN FINDING TABLE FILES TO SEARCH" << ": " << er.what(); - return Status::DBTransactionError("QUERY ERROR WHEN FINDING TABLE FILES TO SEARCH", er.what()); - } catch (const Exception& er) { - // Catch-all for any other MySQL++ exceptions - ENGINE_LOG_ERROR << "GENERAL ERROR WHEN FINDING TABLE FILES TO SEARCH" << ": " << er.what(); - return Status::DBTransactionError("GENERAL ERROR WHEN FINDING TABLE FILES TO SEARCH", er.what()); - } + updateTableFilesQuery << "SELECT EXISTS " << + "(SELECT 1 FROM Tables " << + "WHERE table_id = " << quote << file_schema.table_id_ << " " << + "AND state <> " << std::to_string(TableSchema::TO_DELETE) << ") " << + "AS " << quote << "check" << ";"; - return Status::OK(); - } + ENGINE_LOG_DEBUG << "MySQLMetaImpl::UpdateTableFiles: " << updateTableFilesQuery.str(); - Status MySQLMetaImpl::FilesToMerge(const std::string &table_id, - DatePartionedTableFilesSchema &files) { + StoreQueryResult res = updateTableFilesQuery.store(); -// std::lock_guard lock(mysql_mutex); - - files.clear(); - - try { - MetricCollector metric; - - StoreQueryResult res; - - { - ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab); - - if (connectionPtr == nullptr) { - return Status::Error("Failed to connect to database server"); - } - -// if (mysql_connection_pool_->getConnectionsInUse() <= 0) { -// ENGINE_LOG_WARNING << "MySQLMetaImpl::FilesToMerge connection in use = " << mysql_connection_pool_->getConnectionsInUse(); -// } - - Query filesToMergeQuery = connectionPtr->query(); - filesToMergeQuery << "SELECT id, table_id, file_id, file_type, size, date " << - "FROM TableFiles " << - "WHERE table_id = " << quote << table_id << " AND " << - "file_type = " << std::to_string(TableFileSchema::RAW) << " " << - "ORDER BY size DESC" << ";"; - - ENGINE_LOG_DEBUG << "MySQLMetaImpl::FilesToMerge: " << filesToMergeQuery.str(); - - res = filesToMergeQuery.store(); - } //Scoped Connection - - TableSchema table_schema; - table_schema.table_id_ = table_id; - auto status = DescribeTable(table_schema); - - if (!status.ok()) { - return status; + int check = res[0]["check"]; + has_tables[file_schema.table_id_] = (check == 1); } - TableFileSchema table_file; - for (auto& resRow : res) { + for (auto &file_schema : files) { - table_file.id_ = resRow["id"]; //implicit conversion - - std::string table_id_str; - resRow["table_id"].to_string(table_id_str); - table_file.table_id_ = table_id_str; - - std::string file_id; - resRow["file_id"].to_string(file_id); - table_file.file_id_ = file_id; - - table_file.file_type_ = resRow["file_type"]; - - table_file.size_ = resRow["size"]; - - table_file.date_ = resRow["date"]; - - table_file.dimension_ = table_schema.dimension_; - - utils::GetTableFilePath(options_, table_file); - - auto dateItr = files.find(table_file.date_); - if (dateItr == files.end()) { - files[table_file.date_] = TableFilesSchema(); - } - - files[table_file.date_].push_back(table_file); - } - - } catch (const BadQuery& er) { - // Handle any query errors - ENGINE_LOG_ERROR << "QUERY ERROR WHEN FINDING TABLE FILES TO MERGE" << ": " << er.what(); - return Status::DBTransactionError("QUERY ERROR WHEN FINDING TABLE FILES TO MERGE", er.what()); - } catch (const Exception& er) { - // Catch-all for any other MySQL++ exceptions - ENGINE_LOG_ERROR << "GENERAL ERROR WHEN FINDING TABLE FILES TO MERGE" << ": " << er.what(); - return Status::DBTransactionError("GENERAL ERROR WHEN FINDING TABLE FILES TO MERGE", er.what()); - } - - return Status::OK(); - } - - Status MySQLMetaImpl::GetTableFiles(const std::string& table_id, - const std::vector& ids, - TableFilesSchema& table_files) { - -// std::lock_guard lock(mysql_mutex); - - if (ids.empty()) { - return Status::OK(); - } - - std::stringstream idSS; - for (auto& id : ids) { - idSS << "id = " << std::to_string(id) << " OR "; - } - std::string idStr = idSS.str(); - idStr = idStr.substr(0, idStr.size() - 4); //remove the last " OR " - - try { - - StoreQueryResult res; - - { - ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab); - - if (connectionPtr == nullptr) { - return Status::Error("Failed to connect to database server"); - } - -// if (mysql_connection_pool_->getConnectionsInUse() <= 0) { -// ENGINE_LOG_WARNING << "MySQLMetaImpl::GetTableFiles connection in use = " << mysql_connection_pool_->getConnectionsInUse(); -// } - - Query getTableFileQuery = connectionPtr->query(); - getTableFileQuery << "SELECT id, engine_type, file_id, file_type, size, date " << - "FROM TableFiles " << - "WHERE table_id = " << quote << table_id << " AND " << - "(" << idStr << ");"; - - ENGINE_LOG_DEBUG << "MySQLMetaImpl::GetTableFiles: " << getTableFileQuery.str(); - - res = getTableFileQuery.store(); - } //Scoped Connection - - TableSchema table_schema; - table_schema.table_id_ = table_id; - auto status = DescribeTable(table_schema); - if (!status.ok()) { - return status; - } - - for (auto& resRow : res) { - - TableFileSchema file_schema; - - file_schema.id_ = resRow["id"]; - - file_schema.table_id_ = table_id; - - file_schema.engine_type_ = resRow["engine_type"]; - - std::string file_id; - resRow["file_id"].to_string(file_id); - file_schema.file_id_ = file_id; - - file_schema.file_type_ = resRow["file_type"]; - - file_schema.size_ = resRow["size"]; - - file_schema.date_ = resRow["date"]; - - file_schema.dimension_ = table_schema.dimension_; - - utils::GetTableFilePath(options_, file_schema); - - table_files.emplace_back(file_schema); - } - } catch (const BadQuery& er) { - // Handle any query errors - ENGINE_LOG_ERROR << "QUERY ERROR WHEN RETRIEVING TABLE FILES" << ": " << er.what(); - return Status::DBTransactionError("QUERY ERROR WHEN RETRIEVING TABLE FILES", er.what()); - } catch (const Exception& er) { - // Catch-all for any other MySQL++ exceptions - ENGINE_LOG_ERROR << "GENERAL ERROR WHEN RETRIEVING TABLE FILES" << ": " << er.what(); - return Status::DBTransactionError("GENERAL ERROR WHEN RETRIEVING TABLE FILES", er.what()); - } - - return Status::OK(); - } - -// PXU TODO: Support Swap - Status MySQLMetaImpl::Archive() { - -// std::lock_guard lock(mysql_mutex); - - auto &criterias = options_.archive_conf.GetCriterias(); - if (criterias.empty()) { - return Status::OK(); - } - - for (auto& kv : criterias) { - auto &criteria = kv.first; - auto &limit = kv.second; - if (criteria == "days") { - size_t usecs = limit * D_SEC * US_PS; - long now = utils::GetMicroSecTimeStamp(); - - try { - - ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab); - - if (connectionPtr == nullptr) { - return Status::Error("Failed to connect to database server"); - } - -// if (mysql_connection_pool_->getConnectionsInUse() <= 0) { -// ENGINE_LOG_WARNING << "MySQLMetaImpl::Archive connection in use = " << mysql_connection_pool_->getConnectionsInUse(); -// } - - Query archiveQuery = connectionPtr->query(); - archiveQuery << "UPDATE TableFiles " << - "SET file_type = " << std::to_string(TableFileSchema::TO_DELETE) << " " << - "WHERE created_on < " << std::to_string(now - usecs) << " AND " << - "file_type <> " << std::to_string(TableFileSchema::TO_DELETE) << ";"; - - ENGINE_LOG_DEBUG << "MySQLMetaImpl::Archive: " << archiveQuery.str(); - - if (!archiveQuery.exec()) { - return Status::DBTransactionError("QUERY ERROR DURING ARCHIVE", archiveQuery.error()); - } - - } catch (const BadQuery& er) { - // Handle any query errors - ENGINE_LOG_ERROR << "QUERY ERROR WHEN DURING ARCHIVE" << ": " << er.what(); - return Status::DBTransactionError("QUERY ERROR WHEN DURING ARCHIVE", er.what()); - } catch (const Exception& er) { - // Catch-all for any other MySQL++ exceptions - ENGINE_LOG_ERROR << "GENERAL ERROR WHEN DURING ARCHIVE" << ": " << er.what(); - return Status::DBTransactionError("GENERAL ERROR WHEN DURING ARCHIVE", er.what()); - } - } - if (criteria == "disk") { - uint64_t sum = 0; - Size(sum); - - auto to_delete = (sum - limit * G); - DiscardFiles(to_delete); - } - } - - return Status::OK(); - } - - Status MySQLMetaImpl::Size(uint64_t &result) { - -// std::lock_guard lock(mysql_mutex); - - result = 0; - try { - - StoreQueryResult res; - - { - ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab); - - if (connectionPtr == nullptr) { - return Status::Error("Failed to connect to database server"); - } - -// if (mysql_connection_pool_->getConnectionsInUse() <= 0) { -// ENGINE_LOG_WARNING << "MySQLMetaImpl::Size connection in use = " << mysql_connection_pool_->getConnectionsInUse(); -// } - - Query getSizeQuery = connectionPtr->query(); - getSizeQuery << "SELECT IFNULL(SUM(size),0) AS sum " << - "FROM TableFiles " << - "WHERE file_type <> " << std::to_string(TableFileSchema::TO_DELETE) << ";"; - - ENGINE_LOG_DEBUG << "MySQLMetaImpl::Size: " << getSizeQuery.str(); - - res = getSizeQuery.store(); - } //Scoped Connection - -// if (!res) { -//// std::cout << "result is NULL" << std::endl; -// return Status::DBTransactionError("QUERY ERROR WHEN RETRIEVING SIZE", getSizeQuery.error()); -// } - if (res.empty()) { - result = 0; -// std::cout << "result = 0" << std::endl; - } - else { - result = res[0]["sum"]; -// std::cout << "result = " << std::to_string(result) << std::endl; - } - - } catch (const BadQuery& er) { - // Handle any query errors - ENGINE_LOG_ERROR << "QUERY ERROR WHEN RETRIEVING SIZE" << ": " << er.what(); - return Status::DBTransactionError("QUERY ERROR WHEN RETRIEVING SIZE", er.what()); - } catch (const Exception& er) { - // Catch-all for any other MySQL++ exceptions - ENGINE_LOG_ERROR << "GENERAL ERROR WHEN RETRIEVING SIZE" << ": " << er.what(); - return Status::DBTransactionError("GENERAL ERROR WHEN RETRIEVING SIZE", er.what()); - } - - return Status::OK(); - } - - Status MySQLMetaImpl::DiscardFiles(long long to_discard_size) { - -// std::lock_guard lock(mysql_mutex); - - if (to_discard_size <= 0) { -// std::cout << "in" << std::endl; - return Status::OK(); - } - ENGINE_LOG_DEBUG << "About to discard size=" << to_discard_size; - - try { - - MetricCollector metric; - - bool status; - - { - ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab); - - if (connectionPtr == nullptr) { - return Status::Error("Failed to connect to database server"); - } - -// if (mysql_connection_pool_->getConnectionsInUse() <= 0) { -// ENGINE_LOG_WARNING << "MySQLMetaImpl::DiscardFiles connection in use = " << mysql_connection_pool_->getConnectionsInUse(); -// } - - Query discardFilesQuery = connectionPtr->query(); - discardFilesQuery << "SELECT id, size " << - "FROM TableFiles " << - "WHERE file_type <> " << std::to_string(TableFileSchema::TO_DELETE) << " " << - "ORDER BY id ASC " << - "LIMIT 10;"; - - ENGINE_LOG_DEBUG << "MySQLMetaImpl::DiscardFiles: " << discardFilesQuery.str(); - - // std::cout << discardFilesQuery.str() << std::endl; - StoreQueryResult res = discardFilesQuery.store(); - - if (res.num_rows() == 0) { - return Status::OK(); - } - - TableFileSchema table_file; - std::stringstream idsToDiscardSS; - for (auto &resRow : res) { - if (to_discard_size <= 0) { - break; - } - table_file.id_ = resRow["id"]; - table_file.size_ = resRow["size"]; - idsToDiscardSS << "id = " << std::to_string(table_file.id_) << " OR "; - ENGINE_LOG_DEBUG << "Discard table_file.id=" << table_file.file_id_ - << " table_file.size=" << table_file.size_; - to_discard_size -= table_file.size_; - } - - std::string idsToDiscardStr = idsToDiscardSS.str(); - idsToDiscardStr = idsToDiscardStr.substr(0, idsToDiscardStr.size() - 4); //remove the last " OR " - - discardFilesQuery << "UPDATE TableFiles " << - "SET file_type = " << std::to_string(TableFileSchema::TO_DELETE) << ", " << - "updated_time = " << std::to_string(utils::GetMicroSecTimeStamp()) << " " << - "WHERE " << idsToDiscardStr << ";"; - - ENGINE_LOG_DEBUG << "MySQLMetaImpl::DiscardFiles: " << discardFilesQuery.str(); - - status = discardFilesQuery.exec(); - if (!status) { - ENGINE_LOG_ERROR << "QUERY ERROR WHEN DISCARDING FILES"; - return Status::DBTransactionError("QUERY ERROR WHEN DISCARDING FILES", discardFilesQuery.error()); - } - } //Scoped Connection - - return DiscardFiles(to_discard_size); - - } catch (const BadQuery& er) { - // Handle any query errors - ENGINE_LOG_ERROR << "QUERY ERROR WHEN DISCARDING FILES" << ": " << er.what(); - return Status::DBTransactionError("QUERY ERROR WHEN DISCARDING FILES", er.what()); - } catch (const Exception& er) { - // Catch-all for any other MySQL++ exceptions - ENGINE_LOG_ERROR << "GENERAL ERROR WHEN DISCARDING FILES" << ": " << er.what(); - return Status::DBTransactionError("GENERAL ERROR WHEN DISCARDING FILES", er.what()); - } - } - - //ZR: this function assumes all fields in file_schema have value - Status MySQLMetaImpl::UpdateTableFile(TableFileSchema &file_schema) { - -// std::lock_guard lock(mysql_mutex); - - file_schema.updated_time_ = utils::GetMicroSecTimeStamp(); - try { - - MetricCollector metric; - - { - ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab); - - if (connectionPtr == nullptr) { - return Status::Error("Failed to connect to database server"); - } - -// if (mysql_connection_pool_->getConnectionsInUse() <= 0) { -// ENGINE_LOG_WARNING << "MySQLMetaImpl::UpdateTableFile connection in use = " << mysql_connection_pool_->getConnectionsInUse(); -// } - - Query updateTableFileQuery = connectionPtr->query(); - - //if the table has been deleted, just mark the table file as TO_DELETE - //clean thread will delete the file later - updateTableFileQuery << "SELECT state FROM Tables " << - "WHERE table_id = " << quote << file_schema.table_id_ << ";"; - - ENGINE_LOG_DEBUG << "MySQLMetaImpl::UpdateTableFile: " << updateTableFileQuery.str(); - - StoreQueryResult res = updateTableFileQuery.store(); - - if (res.num_rows() == 1) { - int state = res[0]["state"]; - if (state == TableSchema::TO_DELETE) { - file_schema.file_type_ = TableFileSchema::TO_DELETE; - } - } else { + if (!has_tables[file_schema.table_id_]) { file_schema.file_type_ = TableFileSchema::TO_DELETE; } + file_schema.updated_time_ = utils::GetMicroSecTimeStamp(); std::string id = std::to_string(file_schema.id_); std::string table_id = file_schema.table_id_; @@ -1409,467 +1451,341 @@ namespace meta { std::string created_on = std::to_string(file_schema.created_on_); std::string date = std::to_string(file_schema.date_); - updateTableFileQuery << "UPDATE TableFiles " << - "SET table_id = " << quote << table_id << ", " << - "engine_type = " << engine_type << ", " << - "file_id = " << quote << file_id << ", " << - "file_type = " << file_type << ", " << - "size = " << size << ", " << - "updated_time = " << updated_time << ", " << - "created_on = " << created_on << ", " << - "date = " << date << " " << - "WHERE id = " << id << ";"; + updateTableFilesQuery << "UPDATE TableFiles " << + "SET table_id = " << quote << table_id << ", " << + "engine_type = " << engine_type << ", " << + "file_id = " << quote << file_id << ", " << + "file_type = " << file_type << ", " << + "size = " << size << ", " << + "updated_time = " << updated_time << ", " << + "created_on = " << created_on << ", " << + "date = " << date << " " << + "WHERE id = " << id << ";"; - ENGINE_LOG_DEBUG << "MySQLMetaImpl::UpdateTableFile: " << updateTableFileQuery.str(); + ENGINE_LOG_DEBUG << "MySQLMetaImpl::UpdateTableFiles: " << updateTableFilesQuery.str(); - // std::cout << updateTableFileQuery.str() << std::endl; - - if (!updateTableFileQuery.exec()) { - ENGINE_LOG_DEBUG << "table_id= " << file_schema.table_id_ << " file_id=" << file_schema.file_id_; - ENGINE_LOG_ERROR << "QUERY ERROR WHEN UPDATING TABLE FILE"; - return Status::DBTransactionError("QUERY ERROR WHEN UPDATING TABLE FILE", - updateTableFileQuery.error()); + if (!updateTableFilesQuery.exec()) { + ENGINE_LOG_ERROR << "QUERY ERROR WHEN UPDATING TABLE FILES"; + return Status::DBTransactionError("QUERY ERROR WHEN UPDATING TABLE FILES", + updateTableFilesQuery.error()); } - } //Scoped Connection + } + } //Scoped Connection - } catch (const BadQuery& er) { - // Handle any query errors - ENGINE_LOG_DEBUG << "table_id= " << file_schema.table_id_ << " file_id=" << file_schema.file_id_; - ENGINE_LOG_ERROR << "QUERY ERROR WHEN UPDATING TABLE FILE" << ": " << er.what(); - return Status::DBTransactionError("QUERY ERROR WHEN UPDATING TABLE FILE", er.what()); - } catch (const Exception& er) { - // Catch-all for any other MySQL++ exceptions - ENGINE_LOG_DEBUG << "table_id= " << file_schema.table_id_ << " file_id=" << file_schema.file_id_; - ENGINE_LOG_ERROR << "GENERAL ERROR WHEN UPDATING TABLE FILE" << ": " << er.what(); - return Status::DBTransactionError("GENERAL ERROR WHEN UPDATING TABLE FILE", er.what()); - } - return Status::OK(); + } catch (const BadQuery &er) { + // Handle any query errors + ENGINE_LOG_ERROR << "QUERY ERROR WHEN UPDATING TABLE FILES" << ": " << er.what(); + return Status::DBTransactionError("QUERY ERROR WHEN UPDATING TABLE FILES", er.what()); + } catch (const Exception &er) { + // Catch-all for any other MySQL++ exceptions + ENGINE_LOG_ERROR << "GENERAL ERROR WHEN UPDATING TABLE FILES" << ": " << er.what(); + return Status::DBTransactionError("GENERAL ERROR WHEN UPDATING TABLE FILES", er.what()); } + return Status::OK(); +} - Status MySQLMetaImpl::UpdateTableFilesToIndex(const std::string& table_id) { - // TODO - return Status::OK(); - } +Status MySQLMetaImpl::CleanUpFilesWithTTL(uint16_t seconds) { - Status MySQLMetaImpl::UpdateTableFiles(TableFilesSchema &files) { -// std::lock_guard lock(mysql_mutex); + auto now = utils::GetMicroSecTimeStamp(); + try { + MetricCollector metric; - try { - MetricCollector metric; + { - { - ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab); - if (connectionPtr == nullptr) { - return Status::Error("Failed to connect to database server"); - } - -// if (mysql_connection_pool_->getConnectionsInUse() <= 0) { -// ENGINE_LOG_WARNING << "MySQLMetaImpl::UpdateTableFiles connection in use = " << mysql_connection_pool_->getConnectionsInUse(); -// } - - Query updateTableFilesQuery = connectionPtr->query(); - - std::map has_tables; - for (auto &file_schema : files) { - - if (has_tables.find(file_schema.table_id_) != has_tables.end()) { - continue; - } - - updateTableFilesQuery << "SELECT EXISTS " << - "(SELECT 1 FROM Tables " << - "WHERE table_id = " << quote << file_schema.table_id_ << " " << - "AND state <> " << std::to_string(TableSchema::TO_DELETE) << ") " << - "AS " << quote << "check" << ";"; - - ENGINE_LOG_DEBUG << "MySQLMetaImpl::UpdateTableFiles: " << updateTableFilesQuery.str(); - - StoreQueryResult res = updateTableFilesQuery.store(); - - int check = res[0]["check"]; - has_tables[file_schema.table_id_] = (check == 1); - } - - for (auto &file_schema : files) { - - if (!has_tables[file_schema.table_id_]) { - file_schema.file_type_ = TableFileSchema::TO_DELETE; - } - file_schema.updated_time_ = utils::GetMicroSecTimeStamp(); - - std::string id = std::to_string(file_schema.id_); - std::string table_id = file_schema.table_id_; - std::string engine_type = std::to_string(file_schema.engine_type_); - std::string file_id = file_schema.file_id_; - std::string file_type = std::to_string(file_schema.file_type_); - std::string size = std::to_string(file_schema.size_); - std::string updated_time = std::to_string(file_schema.updated_time_); - std::string created_on = std::to_string(file_schema.created_on_); - std::string date = std::to_string(file_schema.date_); - - updateTableFilesQuery << "UPDATE TableFiles " << - "SET table_id = " << quote << table_id << ", " << - "engine_type = " << engine_type << ", " << - "file_id = " << quote << file_id << ", " << - "file_type = " << file_type << ", " << - "size = " << size << ", " << - "updated_time = " << updated_time << ", " << - "created_on = " << created_on << ", " << - "date = " << date << " " << - "WHERE id = " << id << ";"; - - ENGINE_LOG_DEBUG << "MySQLMetaImpl::UpdateTableFiles: " << updateTableFilesQuery.str(); - - if (!updateTableFilesQuery.exec()) { - ENGINE_LOG_ERROR << "QUERY ERROR WHEN UPDATING TABLE FILES"; - return Status::DBTransactionError("QUERY ERROR WHEN UPDATING TABLE FILES", - updateTableFilesQuery.error()); - } - } - } //Scoped Connection - - } catch (const BadQuery& er) { - // Handle any query errors - ENGINE_LOG_ERROR << "QUERY ERROR WHEN UPDATING TABLE FILES" << ": " << er.what(); - return Status::DBTransactionError("QUERY ERROR WHEN UPDATING TABLE FILES", er.what()); - } catch (const Exception& er) { - // Catch-all for any other MySQL++ exceptions - ENGINE_LOG_ERROR << "GENERAL ERROR WHEN UPDATING TABLE FILES" << ": " << er.what(); - return Status::DBTransactionError("GENERAL ERROR WHEN UPDATING TABLE FILES", er.what()); - } - return Status::OK(); - } - - Status MySQLMetaImpl::CleanUpFilesWithTTL(uint16_t seconds) { -// static int b_count = 0; -// b_count++; -// std::cout << "CleanUpFilesWithTTL: " << b_count << std::endl; -// std::lock_guard lock(mysql_mutex); - - auto now = utils::GetMicroSecTimeStamp(); - try { - MetricCollector metric; - - { - -// ENGINE_LOG_WARNING << "MySQLMetaImpl::CleanUpFilesWithTTL: clean table files: connection in use before creating ScopedConnection = " -// << mysql_connection_pool_->getConnectionsInUse(); - - ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab); - - if (connectionPtr == nullptr) { - return Status::Error("Failed to connect to database server"); - } - -// if (mysql_connection_pool_->getConnectionsInUse() <= 0) { -// ENGINE_LOG_WARNING << "MySQLMetaImpl::CleanUpFilesWithTTL: clean table files: connection in use after creating ScopedConnection = " -// << mysql_connection_pool_->getConnectionsInUse(); -// } - - Query cleanUpFilesWithTTLQuery = connectionPtr->query(); - cleanUpFilesWithTTLQuery << "SELECT id, table_id, file_id, date " << - "FROM TableFiles " << - "WHERE file_type = " << std::to_string(TableFileSchema::TO_DELETE) << " AND " << - "updated_time < " << std::to_string(now - seconds * US_PS) << ";"; - - ENGINE_LOG_DEBUG << "MySQLMetaImpl::CleanUpFilesWithTTL: " << cleanUpFilesWithTTLQuery.str(); - - StoreQueryResult res = cleanUpFilesWithTTLQuery.store(); - - TableFileSchema table_file; - std::vector idsToDelete; - - for (auto &resRow : res) { - - table_file.id_ = resRow["id"]; //implicit conversion - - std::string table_id; - resRow["table_id"].to_string(table_id); - table_file.table_id_ = table_id; - - std::string file_id; - resRow["file_id"].to_string(file_id); - table_file.file_id_ = file_id; - - table_file.date_ = resRow["date"]; - - utils::DeleteTableFilePath(options_, table_file); - - ENGINE_LOG_DEBUG << "Removing deleted id =" << table_file.id_ << " location = " - << table_file.location_ << std::endl; - - idsToDelete.emplace_back(std::to_string(table_file.id_)); - } - - if (!idsToDelete.empty()) { - - std::stringstream idsToDeleteSS; - for (auto &id : idsToDelete) { - idsToDeleteSS << "id = " << id << " OR "; - } - - std::string idsToDeleteStr = idsToDeleteSS.str(); - idsToDeleteStr = idsToDeleteStr.substr(0, idsToDeleteStr.size() - 4); //remove the last " OR " - cleanUpFilesWithTTLQuery << "DELETE FROM TableFiles WHERE " << - idsToDeleteStr << ";"; - - ENGINE_LOG_DEBUG << "MySQLMetaImpl::CleanUpFilesWithTTL: " << cleanUpFilesWithTTLQuery.str(); - - if (!cleanUpFilesWithTTLQuery.exec()) { - ENGINE_LOG_ERROR << "QUERY ERROR WHEN CLEANING UP FILES WITH TTL"; - return Status::DBTransactionError("CleanUpFilesWithTTL Error", - cleanUpFilesWithTTLQuery.error()); - } - } - } //Scoped Connection - - } catch (const BadQuery& er) { - // Handle any query errors - ENGINE_LOG_ERROR << "QUERY ERROR WHEN CLEANING UP FILES WITH TTL" << ": " << er.what(); - return Status::DBTransactionError("QUERY ERROR WHEN CLEANING UP FILES WITH TTL", er.what()); - } catch (const Exception& er) { - // Catch-all for any other MySQL++ exceptions - ENGINE_LOG_ERROR << "GENERAL ERROR WHEN CLEANING UP FILES WITH TTL" << ": " << er.what(); - return Status::DBTransactionError("GENERAL ERROR WHEN CLEANING UP FILES WITH TTL", er.what()); - } - - try { - MetricCollector metric; - - { -// ENGINE_LOG_WARNING << "MySQLMetaImpl::CleanUpFilesWithTTL: clean tables: connection in use before creating ScopedConnection = " -// << mysql_connection_pool_->getConnectionsInUse(); - - ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab); - - if (connectionPtr == nullptr) { - return Status::Error("Failed to connect to database server"); - } - -// if (mysql_connection_pool_->getConnectionsInUse() <= 0) { -// ENGINE_LOG_WARNING << "MySQLMetaImpl::CleanUpFilesWithTTL: clean tables: connection in use after creating ScopedConnection = " -// << mysql_connection_pool_->getConnectionsInUse(); -// } - - Query cleanUpFilesWithTTLQuery = connectionPtr->query(); - cleanUpFilesWithTTLQuery << "SELECT id, table_id " << - "FROM Tables " << - "WHERE state = " << std::to_string(TableSchema::TO_DELETE) << ";"; - - ENGINE_LOG_DEBUG << "MySQLMetaImpl::CleanUpFilesWithTTL: " << cleanUpFilesWithTTLQuery.str(); - - StoreQueryResult res = cleanUpFilesWithTTLQuery.store(); -// std::cout << res.num_rows() << std::endl; - - if (!res.empty()) { - - std::stringstream idsToDeleteSS; - for (auto &resRow : res) { - size_t id = resRow["id"]; - std::string table_id; - resRow["table_id"].to_string(table_id); - - utils::DeleteTablePath(options_, table_id); - - idsToDeleteSS << "id = " << std::to_string(id) << " OR "; - } - std::string idsToDeleteStr = idsToDeleteSS.str(); - idsToDeleteStr = idsToDeleteStr.substr(0, idsToDeleteStr.size() - 4); //remove the last " OR " - cleanUpFilesWithTTLQuery << "DELETE FROM Tables WHERE " << - idsToDeleteStr << ";"; - - ENGINE_LOG_DEBUG << "MySQLMetaImpl::CleanUpFilesWithTTL: " << cleanUpFilesWithTTLQuery.str(); - - if (!cleanUpFilesWithTTLQuery.exec()) { - ENGINE_LOG_ERROR << "QUERY ERROR WHEN CLEANING UP FILES WITH TTL"; - return Status::DBTransactionError("QUERY ERROR WHEN CLEANING UP FILES WITH TTL", - cleanUpFilesWithTTLQuery.error()); - } - } - } //Scoped Connection - - } catch (const BadQuery& er) { - // Handle any query errors - ENGINE_LOG_ERROR << "QUERY ERROR WHEN CLEANING UP FILES WITH TTL" << ": " << er.what(); - return Status::DBTransactionError("QUERY ERROR WHEN CLEANING UP FILES WITH TTL", er.what()); - } catch (const Exception& er) { - // Catch-all for any other MySQL++ exceptions - ENGINE_LOG_ERROR << "GENERAL ERROR WHEN CLEANING UP FILES WITH TTL" << ": " << er.what(); - return Status::DBTransactionError("GENERAL ERROR WHEN CLEANING UP FILES WITH TTL", er.what()); - } - - return Status::OK(); - } - - Status MySQLMetaImpl::CleanUp() { - -// std::lock_guard lock(mysql_mutex); - - try { ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab); if (connectionPtr == nullptr) { return Status::Error("Failed to connect to database server"); } -// if (mysql_connection_pool_->getConnectionsInUse() <= 0) { -// ENGINE_LOG_WARNING << "MySQLMetaImpl::CleanUp: connection in use = " << mysql_connection_pool_->getConnectionsInUse(); -// } - Query cleanUpQuery = connectionPtr->query(); - cleanUpQuery << "SELECT table_name " << - "FROM information_schema.tables " << - "WHERE table_schema = " << quote << mysql_connection_pool_->getDB() << " " << - "AND table_name = " << quote << "TableFiles" << ";"; + Query cleanUpFilesWithTTLQuery = connectionPtr->query(); + cleanUpFilesWithTTLQuery << "SELECT id, table_id, file_id, date " << + "FROM TableFiles " << + "WHERE file_type = " << std::to_string(TableFileSchema::TO_DELETE) << " AND " << + "updated_time < " << std::to_string(now - seconds * US_PS) << ";"; + + ENGINE_LOG_DEBUG << "MySQLMetaImpl::CleanUpFilesWithTTL: " << cleanUpFilesWithTTLQuery.str(); + + StoreQueryResult res = cleanUpFilesWithTTLQuery.store(); + + TableFileSchema table_file; + std::vector idsToDelete; + + for (auto &resRow : res) { + + table_file.id_ = resRow["id"]; //implicit conversion + + std::string table_id; + resRow["table_id"].to_string(table_id); + table_file.table_id_ = table_id; + + std::string file_id; + resRow["file_id"].to_string(file_id); + table_file.file_id_ = file_id; + + table_file.date_ = resRow["date"]; + + utils::DeleteTableFilePath(options_, table_file); + + ENGINE_LOG_DEBUG << "Removing deleted id =" << table_file.id_ << " location = " + << table_file.location_ << std::endl; + + idsToDelete.emplace_back(std::to_string(table_file.id_)); + } + + if (!idsToDelete.empty()) { + + std::stringstream idsToDeleteSS; + for (auto &id : idsToDelete) { + idsToDeleteSS << "id = " << id << " OR "; + } + + std::string idsToDeleteStr = idsToDeleteSS.str(); + idsToDeleteStr = idsToDeleteStr.substr(0, idsToDeleteStr.size() - 4); //remove the last " OR " + cleanUpFilesWithTTLQuery << "DELETE FROM TableFiles WHERE " << + idsToDeleteStr << ";"; + + ENGINE_LOG_DEBUG << "MySQLMetaImpl::CleanUpFilesWithTTL: " << cleanUpFilesWithTTLQuery.str(); + + if (!cleanUpFilesWithTTLQuery.exec()) { + ENGINE_LOG_ERROR << "QUERY ERROR WHEN CLEANING UP FILES WITH TTL"; + return Status::DBTransactionError("CleanUpFilesWithTTL Error", + cleanUpFilesWithTTLQuery.error()); + } + } + } //Scoped Connection + + } catch (const BadQuery &er) { + // Handle any query errors + ENGINE_LOG_ERROR << "QUERY ERROR WHEN CLEANING UP FILES WITH TTL" << ": " << er.what(); + return Status::DBTransactionError("QUERY ERROR WHEN CLEANING UP FILES WITH TTL", er.what()); + } catch (const Exception &er) { + // Catch-all for any other MySQL++ exceptions + ENGINE_LOG_ERROR << "GENERAL ERROR WHEN CLEANING UP FILES WITH TTL" << ": " << er.what(); + return Status::DBTransactionError("GENERAL ERROR WHEN CLEANING UP FILES WITH TTL", er.what()); + } + + try { + MetricCollector metric; + + { + + + ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab); + + if (connectionPtr == nullptr) { + return Status::Error("Failed to connect to database server"); + } + + + Query cleanUpFilesWithTTLQuery = connectionPtr->query(); + cleanUpFilesWithTTLQuery << "SELECT id, table_id " << + "FROM Tables " << + "WHERE state = " << std::to_string(TableSchema::TO_DELETE) << ";"; + + ENGINE_LOG_DEBUG << "MySQLMetaImpl::CleanUpFilesWithTTL: " << cleanUpFilesWithTTLQuery.str(); + + StoreQueryResult res = cleanUpFilesWithTTLQuery.store(); + + + if (!res.empty()) { + + std::stringstream idsToDeleteSS; + for (auto &resRow : res) { + size_t id = resRow["id"]; + std::string table_id; + resRow["table_id"].to_string(table_id); + + utils::DeleteTablePath(options_, table_id); + + idsToDeleteSS << "id = " << std::to_string(id) << " OR "; + } + std::string idsToDeleteStr = idsToDeleteSS.str(); + idsToDeleteStr = idsToDeleteStr.substr(0, idsToDeleteStr.size() - 4); //remove the last " OR " + cleanUpFilesWithTTLQuery << "DELETE FROM Tables WHERE " << + idsToDeleteStr << ";"; + + ENGINE_LOG_DEBUG << "MySQLMetaImpl::CleanUpFilesWithTTL: " << cleanUpFilesWithTTLQuery.str(); + + if (!cleanUpFilesWithTTLQuery.exec()) { + ENGINE_LOG_ERROR << "QUERY ERROR WHEN CLEANING UP FILES WITH TTL"; + return Status::DBTransactionError("QUERY ERROR WHEN CLEANING UP FILES WITH TTL", + cleanUpFilesWithTTLQuery.error()); + } + } + } //Scoped Connection + + } catch (const BadQuery &er) { + // Handle any query errors + ENGINE_LOG_ERROR << "QUERY ERROR WHEN CLEANING UP FILES WITH TTL" << ": " << er.what(); + return Status::DBTransactionError("QUERY ERROR WHEN CLEANING UP FILES WITH TTL", er.what()); + } catch (const Exception &er) { + // Catch-all for any other MySQL++ exceptions + ENGINE_LOG_ERROR << "GENERAL ERROR WHEN CLEANING UP FILES WITH TTL" << ": " << er.what(); + return Status::DBTransactionError("GENERAL ERROR WHEN CLEANING UP FILES WITH TTL", er.what()); + } + + return Status::OK(); +} + +Status MySQLMetaImpl::CleanUp() { + + + try { + ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab); + + if (connectionPtr == nullptr) { + return Status::Error("Failed to connect to database server"); + } + + + Query cleanUpQuery = connectionPtr->query(); + cleanUpQuery << "SELECT table_name " << + "FROM information_schema.tables " << + "WHERE table_schema = " << quote << mysql_connection_pool_->getDB() << " " << + "AND table_name = " << quote << "TableFiles" << ";"; + + ENGINE_LOG_DEBUG << "MySQLMetaImpl::CleanUp: " << cleanUpQuery.str(); + + StoreQueryResult res = cleanUpQuery.store(); + + if (!res.empty()) { + ENGINE_LOG_DEBUG << "Remove table file type as NEW"; + cleanUpQuery << "DELETE FROM TableFiles WHERE file_type = " << std::to_string(TableFileSchema::NEW) << ";"; ENGINE_LOG_DEBUG << "MySQLMetaImpl::CleanUp: " << cleanUpQuery.str(); - StoreQueryResult res = cleanUpQuery.store(); - - if (!res.empty()) { - ENGINE_LOG_DEBUG << "Remove table file type as NEW"; - cleanUpQuery << "DELETE FROM TableFiles WHERE file_type = " << std::to_string(TableFileSchema::NEW) << ";"; - - ENGINE_LOG_DEBUG << "MySQLMetaImpl::CleanUp: " << cleanUpQuery.str(); - - if (!cleanUpQuery.exec()) { - ENGINE_LOG_ERROR << "QUERY ERROR WHEN CLEANING UP FILES"; - return Status::DBTransactionError("Clean up Error", cleanUpQuery.error()); - } + if (!cleanUpQuery.exec()) { + ENGINE_LOG_ERROR << "QUERY ERROR WHEN CLEANING UP FILES"; + return Status::DBTransactionError("Clean up Error", cleanUpQuery.error()); } - - } catch (const BadQuery& er) { - // Handle any query errors - ENGINE_LOG_ERROR << "QUERY ERROR WHEN CLEANING UP FILES" << ": " << er.what(); - return Status::DBTransactionError("QUERY ERROR WHEN CLEANING UP FILES", er.what()); - } catch (const Exception& er) { - // Catch-all for any other MySQL++ exceptions - ENGINE_LOG_ERROR << "GENERAL ERROR WHEN CLEANING UP FILES" << ": " << er.what(); - return Status::DBTransactionError("GENERAL ERROR WHEN CLEANING UP FILES", er.what()); } - return Status::OK(); + } catch (const BadQuery &er) { + // Handle any query errors + ENGINE_LOG_ERROR << "QUERY ERROR WHEN CLEANING UP FILES" << ": " << er.what(); + return Status::DBTransactionError("QUERY ERROR WHEN CLEANING UP FILES", er.what()); + } catch (const Exception &er) { + // Catch-all for any other MySQL++ exceptions + ENGINE_LOG_ERROR << "GENERAL ERROR WHEN CLEANING UP FILES" << ": " << er.what(); + return Status::DBTransactionError("GENERAL ERROR WHEN CLEANING UP FILES", er.what()); } - Status MySQLMetaImpl::Count(const std::string &table_id, uint64_t &result) { + return Status::OK(); +} -// std::lock_guard lock(mysql_mutex); +Status MySQLMetaImpl::Count(const std::string &table_id, uint64_t &result) { - try { - MetricCollector metric; - TableSchema table_schema; - table_schema.table_id_ = table_id; - auto status = DescribeTable(table_schema); + try { + MetricCollector metric; - if (!status.ok()) { - return status; - } + TableSchema table_schema; + table_schema.table_id_ = table_id; + auto status = DescribeTable(table_schema); - StoreQueryResult res; - - { - ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab); - - if (connectionPtr == nullptr) { - return Status::Error("Failed to connect to database server"); - } - -// if (mysql_connection_pool_->getConnectionsInUse() <= 0) { -// ENGINE_LOG_WARNING << "MySQLMetaImpl::Count: connection in use = " << mysql_connection_pool_->getConnectionsInUse(); -// } - - Query countQuery = connectionPtr->query(); - countQuery << "SELECT size " << - "FROM TableFiles " << - "WHERE table_id = " << quote << table_id << " AND " << - "(file_type = " << std::to_string(TableFileSchema::RAW) << " OR " << - "file_type = " << std::to_string(TableFileSchema::TO_INDEX) << " OR " << - "file_type = " << std::to_string(TableFileSchema::INDEX) << ");"; - - ENGINE_LOG_DEBUG << "MySQLMetaImpl::Count: " << countQuery.str(); - - res = countQuery.store(); - } //Scoped Connection - - result = 0; - for (auto &resRow : res) { - size_t size = resRow["size"]; - result += size; - } - - if (table_schema.dimension_ <= 0) { - std::stringstream errorMsg; - errorMsg << "MySQLMetaImpl::Count: " << "table dimension = " << std::to_string(table_schema.dimension_) << ", table_id = " << table_id; - ENGINE_LOG_ERROR << errorMsg.str(); - return Status::Error(errorMsg.str()); - } - result /= table_schema.dimension_; - result /= sizeof(float); - - } catch (const BadQuery& er) { - // Handle any query errors - ENGINE_LOG_ERROR << "QUERY ERROR WHEN RETRIEVING COUNT" << ": " << er.what(); - return Status::DBTransactionError("QUERY ERROR WHEN RETRIEVING COUNT", er.what()); - } catch (const Exception& er) { - // Catch-all for any other MySQL++ exceptions - ENGINE_LOG_ERROR << "GENERAL ERROR WHEN RETRIEVING COUNT" << ": " << er.what(); - return Status::DBTransactionError("GENERAL ERROR WHEN RETRIEVING COUNT", er.what()); + if (!status.ok()) { + return status; } - return Status::OK(); - } - Status MySQLMetaImpl::DropAll() { - -// std::lock_guard lock(mysql_mutex); - - if (boost::filesystem::is_directory(options_.path)) { - boost::filesystem::remove_all(options_.path); - } - try { + StoreQueryResult res; + { ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab); if (connectionPtr == nullptr) { return Status::Error("Failed to connect to database server"); } -// if (mysql_connection_pool_->getConnectionsInUse() <= 0) { -// ENGINE_LOG_WARNING << "MySQLMetaImpl::DropAll: connection in use = " << mysql_connection_pool_->getConnectionsInUse(); -// } - Query dropTableQuery = connectionPtr->query(); - dropTableQuery << "DROP TABLE IF EXISTS Tables, TableFiles;"; + Query countQuery = connectionPtr->query(); + countQuery << "SELECT size " << + "FROM TableFiles " << + "WHERE table_id = " << quote << table_id << " AND " << + "(file_type = " << std::to_string(TableFileSchema::RAW) << " OR " << + "file_type = " << std::to_string(TableFileSchema::TO_INDEX) << " OR " << + "file_type = " << std::to_string(TableFileSchema::INDEX) << ");"; - ENGINE_LOG_DEBUG << "MySQLMetaImpl::DropAll: " << dropTableQuery.str(); + ENGINE_LOG_DEBUG << "MySQLMetaImpl::Count: " << countQuery.str(); - if (dropTableQuery.exec()) { - return Status::OK(); - } - else { - ENGINE_LOG_ERROR << "QUERY ERROR WHEN DROPPING TABLE"; - return Status::DBTransactionError("DROP TABLE ERROR", dropTableQuery.error()); - } - } catch (const BadQuery& er) { - // Handle any query errors - ENGINE_LOG_ERROR << "QUERY ERROR WHEN DROPPING TABLE" << ": " << er.what(); - return Status::DBTransactionError("QUERY ERROR WHEN DROPPING TABLE", er.what()); - } catch (const Exception& er) { - // Catch-all for any other MySQL++ exceptions - ENGINE_LOG_ERROR << "GENERAL ERROR WHEN DROPPING TABLE" << ": " << er.what(); - return Status::DBTransactionError("GENERAL ERROR WHEN DROPPING TABLE", er.what()); + res = countQuery.store(); + } //Scoped Connection + + result = 0; + for (auto &resRow : res) { + size_t size = resRow["size"]; + result += size; } - return Status::OK(); - } - MySQLMetaImpl::~MySQLMetaImpl() { -// std::lock_guard lock(mysql_mutex); - if (mode_ != Options::MODE::READ_ONLY) { - CleanUp(); + if (table_schema.dimension_ <= 0) { + std::stringstream errorMsg; + errorMsg << "MySQLMetaImpl::Count: " << "table dimension = " << std::to_string(table_schema.dimension_) + << ", table_id = " << table_id; + ENGINE_LOG_ERROR << errorMsg.str(); + return Status::Error(errorMsg.str()); } + result /= table_schema.dimension_; + result /= sizeof(float); + + } catch (const BadQuery &er) { + // Handle any query errors + ENGINE_LOG_ERROR << "QUERY ERROR WHEN RETRIEVING COUNT" << ": " << er.what(); + return Status::DBTransactionError("QUERY ERROR WHEN RETRIEVING COUNT", er.what()); + } catch (const Exception &er) { + // Catch-all for any other MySQL++ exceptions + ENGINE_LOG_ERROR << "GENERAL ERROR WHEN RETRIEVING COUNT" << ": " << er.what(); + return Status::DBTransactionError("GENERAL ERROR WHEN RETRIEVING COUNT", er.what()); } + return Status::OK(); +} + +Status MySQLMetaImpl::DropAll() { + + + if (boost::filesystem::is_directory(options_.path)) { + boost::filesystem::remove_all(options_.path); + } + try { + + ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab); + + if (connectionPtr == nullptr) { + return Status::Error("Failed to connect to database server"); + } + + + Query dropTableQuery = connectionPtr->query(); + dropTableQuery << "DROP TABLE IF EXISTS Tables, TableFiles;"; + + ENGINE_LOG_DEBUG << "MySQLMetaImpl::DropAll: " << dropTableQuery.str(); + + if (dropTableQuery.exec()) { + return Status::OK(); + } else { + ENGINE_LOG_ERROR << "QUERY ERROR WHEN DROPPING TABLE"; + return Status::DBTransactionError("DROP TABLE ERROR", dropTableQuery.error()); + } + } catch (const BadQuery &er) { + // Handle any query errors + ENGINE_LOG_ERROR << "QUERY ERROR WHEN DROPPING TABLE" << ": " << er.what(); + return Status::DBTransactionError("QUERY ERROR WHEN DROPPING TABLE", er.what()); + } catch (const Exception &er) { + // Catch-all for any other MySQL++ exceptions + ENGINE_LOG_ERROR << "GENERAL ERROR WHEN DROPPING TABLE" << ": " << er.what(); + return Status::DBTransactionError("GENERAL ERROR WHEN DROPPING TABLE", er.what()); + } + return Status::OK(); +} + +MySQLMetaImpl::~MySQLMetaImpl() { + + if (mode_ != Options::MODE::READ_ONLY) { + CleanUp(); + } +} } // namespace meta } // namespace engine diff --git a/cpp/src/db/Options.h b/cpp/src/db/Options.h index 515a84d663..4275e7fe00 100644 --- a/cpp/src/db/Options.h +++ b/cpp/src/db/Options.h @@ -63,7 +63,7 @@ struct Options { size_t index_trigger_size = ONE_GB; //unit: byte DBMetaOptions meta; int mode = MODE::SINGLE; - float maximum_memory = 4 * ONE_GB; + size_t maximum_memory = 4 * ONE_GB; }; // Options diff --git a/cpp/src/server/DBWrapper.cpp b/cpp/src/server/DBWrapper.cpp index 66a1d83479..c01e1acda3 100644 --- a/cpp/src/server/DBWrapper.cpp +++ b/cpp/src/server/DBWrapper.cpp @@ -28,11 +28,11 @@ DBWrapper::DBWrapper() { if(index_size > 0) {//ensure larger than zero, unit is MB opt.index_trigger_size = (size_t)index_size * engine::ONE_MB; } - float maximum_memory = config.GetFloatValue(CONFIG_MAXMIMUM_MEMORY); + int64_t maximum_memory = config.GetInt64Value(CONFIG_MAXMIMUM_MEMORY); if (maximum_memory > 1.0) { opt.maximum_memory = maximum_memory * engine::ONE_GB; } - else { + else if (maximum_memory != 0) { //if maximum_memory = 0, set it to default std::cout << "ERROR: maximum_memory should be at least 1 GB" << std::endl; kill(0, SIGUSR1); } From 50ca5b4bdee1fce3e390aeabc2b536a2bf042c52 Mon Sep 17 00:00:00 2001 From: zhiru Date: Mon, 15 Jul 2019 11:37:26 +0800 Subject: [PATCH 165/189] add MySQLMetaImpl::UpdateTableFilesToIndex and set maximum_memory to default if config value = 0 Former-commit-id: e3dba8250a2b8c28770811e5c202232bb59231c6 --- cpp/CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/cpp/CHANGELOG.md b/cpp/CHANGELOG.md index 946e08e99c..f6b794969d 100644 --- a/cpp/CHANGELOG.md +++ b/cpp/CHANGELOG.md @@ -13,6 +13,7 @@ Please mark all change in change log and use the ticket from JIRA. - MS-157 - fix changelog - MS-190 - use env variable to switch mem manager and fix cmake - MS-224 - Return AlreadyExist status in MySQLMetaImpl::CreateTable if table already exists +- MS-232 - add MySQLMetaImpl::UpdateTableFilesToIndex and set maximum_memory to default if config value = 0 ## Improvement - MS-156 - Add unittest for merge result functions From 3963c93dedd5559a7fe75a1228606db2e07973db Mon Sep 17 00:00:00 2001 From: zhiru Date: Mon, 15 Jul 2019 11:58:02 +0800 Subject: [PATCH 166/189] remove mem manager log Former-commit-id: 886277f2db3f150db9475a9ffe490b0bc6d7fc63 --- cpp/CHANGELOG.md | 1 + cpp/src/db/MemManager.cpp | 3 --- cpp/src/db/NewMemManager.cpp | 3 --- 3 files changed, 1 insertion(+), 6 deletions(-) diff --git a/cpp/CHANGELOG.md b/cpp/CHANGELOG.md index f6b794969d..0585d9218b 100644 --- a/cpp/CHANGELOG.md +++ b/cpp/CHANGELOG.md @@ -14,6 +14,7 @@ Please mark all change in change log and use the ticket from JIRA. - MS-190 - use env variable to switch mem manager and fix cmake - MS-224 - Return AlreadyExist status in MySQLMetaImpl::CreateTable if table already exists - MS-232 - add MySQLMetaImpl::UpdateTableFilesToIndex and set maximum_memory to default if config value = 0 +- MS-233 - remove mem manager log ## Improvement - MS-156 - Add unittest for merge result functions diff --git a/cpp/src/db/MemManager.cpp b/cpp/src/db/MemManager.cpp index dbf0703173..366188b547 100644 --- a/cpp/src/db/MemManager.cpp +++ b/cpp/src/db/MemManager.cpp @@ -125,9 +125,6 @@ Status MemManager::InsertVectors(const std::string &table_id_, const float *vectors_, IDNumbers &vector_ids_) { - LOG(DEBUG) << "MemManager::InsertVectors: mutable mem = " << GetCurrentMutableMem() << - ", immutable mem = " << GetCurrentImmutableMem() << ", total mem = " << GetCurrentMem(); - std::unique_lock lock(mutex_); return InsertVectorsNoLock(table_id_, n_, vectors_, vector_ids_); diff --git a/cpp/src/db/NewMemManager.cpp b/cpp/src/db/NewMemManager.cpp index e4903d75a9..405d3771fd 100644 --- a/cpp/src/db/NewMemManager.cpp +++ b/cpp/src/db/NewMemManager.cpp @@ -29,9 +29,6 @@ Status NewMemManager::InsertVectors(const std::string &table_id_, std::this_thread::sleep_for(std::chrono::milliseconds(1)); } - LOG(DEBUG) << "NewMemManager::InsertVectors: mutable mem = " << GetCurrentMutableMem() << - ", immutable mem = " << GetCurrentImmutableMem() << ", total mem = " << GetCurrentMem(); - std::unique_lock lock(mutex_); return InsertVectorsNoLock(table_id_, n_, vectors_, vector_ids_); From a01cdbf886509aff586add3698835ff89778cc98 Mon Sep 17 00:00:00 2001 From: starlord Date: Mon, 15 Jul 2019 13:47:24 +0800 Subject: [PATCH 167/189] MS-230 - Change parameter name: Maximum_memory to insert_buffer_size Former-commit-id: 8a27b0b4b272a89a2a1e4e4fa78c26baccaf139d --- cpp/conf/server_config.template | 4 ++-- cpp/src/db/NewMemManager.cpp | 2 +- cpp/src/db/Options.h | 2 +- cpp/src/server/DBWrapper.cpp | 10 +++++----- cpp/src/server/ServerConfig.h | 2 +- 5 files changed, 10 insertions(+), 10 deletions(-) diff --git a/cpp/conf/server_config.template b/cpp/conf/server_config.template index 43b6f3ba16..2dd8daa0f1 100644 --- a/cpp/conf/server_config.template +++ b/cpp/conf/server_config.template @@ -16,8 +16,8 @@ db_config: index_building_threshold: 1024 # index building trigger threshold, default: 1024, unit: MB archive_disk_threshold: 0 # triger archive action if storage size exceed this value, 0 means no limit, unit: GB archive_days_threshold: 0 # files older than x days will be archived, 0 means no limit, unit: day - maximum_memory: 4 # maximum memory allowed, default: 4, unit: GB, should be at least 1 GB. - # the sum of maximum_memory and cpu_cache_capacity should be less than total memory + insert_buffer_size: 4 # maximum insert buffer size allowed, default: 4, unit: GB, should be at least 1 GB. + # the sum of insert_buffer_size and cpu_cache_capacity should be less than total memory metric_config: is_startup: off # if monitoring start: on, off diff --git a/cpp/src/db/NewMemManager.cpp b/cpp/src/db/NewMemManager.cpp index 405d3771fd..c7291354ad 100644 --- a/cpp/src/db/NewMemManager.cpp +++ b/cpp/src/db/NewMemManager.cpp @@ -25,7 +25,7 @@ Status NewMemManager::InsertVectors(const std::string &table_id_, const float *vectors_, IDNumbers &vector_ids_) { - while (GetCurrentMem() > options_.maximum_memory) { + while (GetCurrentMem() > options_.insert_buffer_size) { std::this_thread::sleep_for(std::chrono::milliseconds(1)); } diff --git a/cpp/src/db/Options.h b/cpp/src/db/Options.h index a12afbfd80..efc5ee3604 100644 --- a/cpp/src/db/Options.h +++ b/cpp/src/db/Options.h @@ -63,7 +63,7 @@ struct Options { size_t index_trigger_size = ONE_GB; //unit: byte DBMetaOptions meta; int mode = MODE::SINGLE; - size_t maximum_memory = 4 * ONE_GB; + size_t insert_buffer_size = 4 * ONE_GB; }; // Options diff --git a/cpp/src/server/DBWrapper.cpp b/cpp/src/server/DBWrapper.cpp index c01e1acda3..6c67176571 100644 --- a/cpp/src/server/DBWrapper.cpp +++ b/cpp/src/server/DBWrapper.cpp @@ -28,12 +28,12 @@ DBWrapper::DBWrapper() { if(index_size > 0) {//ensure larger than zero, unit is MB opt.index_trigger_size = (size_t)index_size * engine::ONE_MB; } - int64_t maximum_memory = config.GetInt64Value(CONFIG_MAXMIMUM_MEMORY); - if (maximum_memory > 1.0) { - opt.maximum_memory = maximum_memory * engine::ONE_GB; + int64_t insert_buffer_size = config.GetInt64Value(CONFIG_DB_INSERT_BUFFER_SIZE, 4); + if (insert_buffer_size >= 1) { + opt.insert_buffer_size = insert_buffer_size * engine::ONE_GB; } - else if (maximum_memory != 0) { //if maximum_memory = 0, set it to default - std::cout << "ERROR: maximum_memory should be at least 1 GB" << std::endl; + else { + std::cout << "ERROR: insert_buffer_size should be at least 1 GB" << std::endl; kill(0, SIGUSR1); } diff --git a/cpp/src/server/ServerConfig.h b/cpp/src/server/ServerConfig.h index d3fb77a8cb..bc202adcf6 100644 --- a/cpp/src/server/ServerConfig.h +++ b/cpp/src/server/ServerConfig.h @@ -27,7 +27,7 @@ static const std::string CONFIG_DB_SLAVE_PATH = "db_slave_path"; static const std::string CONFIG_DB_INDEX_TRIGGER_SIZE = "index_building_threshold"; static const std::string CONFIG_DB_ARCHIVE_DISK = "archive_disk_threshold"; static const std::string CONFIG_DB_ARCHIVE_DAYS = "archive_days_threshold"; -static const std::string CONFIG_MAXMIMUM_MEMORY = "maximum_memory"; +static const std::string CONFIG_DB_INSERT_BUFFER_SIZE = "insert_buffer_size"; static const std::string CONFIG_LOG = "log_config"; From de09ef53a9a7ba976a29db45218d744ae704c354 Mon Sep 17 00:00:00 2001 From: starlord Date: Mon, 15 Jul 2019 13:47:36 +0800 Subject: [PATCH 168/189] MS-230 - Change parameter name: Maximum_memory to insert_buffer_size Former-commit-id: 1263b0c2edceb235163c48b34af681c3edb48d16 --- cpp/CHANGELOG.md | 11 ++++++----- cpp/src/CMakeLists.txt | 2 +- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/cpp/CHANGELOG.md b/cpp/CHANGELOG.md index 4a3288526c..3ac9db25a1 100644 --- a/cpp/CHANGELOG.md +++ b/cpp/CHANGELOG.md @@ -9,13 +9,14 @@ Please mark all change in change log and use the ticket from JIRA. - MS-148 - Disable cleanup if mode is read only - MS-149 - Fixed searching only one index file issue in distributed mode -- MS-153 - fix c_str error when connecting to MySQL -- MS-157 - fix changelog -- MS-190 - use env variable to switch mem manager and fix cmake +- MS-153 - Fix c_str error when connecting to MySQL +- MS-157 - Fix changelog +- MS-190 - Use env variable to switch mem manager and fix cmake - MS-217 - Fix SQ8 row count bug - MS-224 - Return AlreadyExist status in MySQLMetaImpl::CreateTable if table already exists -- MS-232 - add MySQLMetaImpl::UpdateTableFilesToIndex and set maximum_memory to default if config value = 0 -- MS-233 - remove mem manager log +- MS-232 - Add MySQLMetaImpl::UpdateTableFilesToIndex and set maximum_memory to default if config value = 0 +- MS-233 - Remove mem manager log +- MS-230 - Change parameter name: Maximum_memory to insert_buffer_size ## Improvement - MS-156 - Add unittest for merge result functions diff --git a/cpp/src/CMakeLists.txt b/cpp/src/CMakeLists.txt index d0029d5175..00e2e1e87d 100644 --- a/cpp/src/CMakeLists.txt +++ b/cpp/src/CMakeLists.txt @@ -189,4 +189,4 @@ install(FILES ${CMAKE_BINARY_DIR}/mysqlpp_ep-prefix/src/mysqlpp_ep/lib/${CMAKE_SHARED_LIBRARY_PREFIX}mysqlpp${CMAKE_SHARED_LIBRARY_SUFFIX}.3.2.4 DESTINATION lib) #need to copy libmysqlpp.so -#add_subdirectory(sdk) +add_subdirectory(sdk) From f3c6559a71591d1cb4fa3cded29b5cc8be3eb829 Mon Sep 17 00:00:00 2001 From: starlord Date: Mon, 15 Jul 2019 14:28:12 +0800 Subject: [PATCH 169/189] MS-207 build_index return success when table not existed Former-commit-id: 8e0e1a929afe5959f14c9fbe318303c14d6f8c6d --- cpp/src/server/RequestTask.cpp | 33 +++++++++++++++++++++------------ 1 file changed, 21 insertions(+), 12 deletions(-) diff --git a/cpp/src/server/RequestTask.cpp b/cpp/src/server/RequestTask.cpp index d4051eba66..fe31d51c5b 100644 --- a/cpp/src/server/RequestTask.cpp +++ b/cpp/src/server/RequestTask.cpp @@ -148,17 +148,17 @@ ServerError CreateTableTask::OnExecute() { ServerError res = SERVER_SUCCESS; res = ValidateTableName(schema_.table_name); if(res != SERVER_SUCCESS) { - return res; + return SetError(res, "Invalid table name: " + schema_.table_name); } res = ValidateTableDimension(schema_.dimension); if(res != SERVER_SUCCESS) { - return res; + return SetError(res, "Invalid table dimension: " + std::to_string(schema_.dimension)); } res = ValidateTableIndexType(schema_.index_type); if(res != SERVER_SUCCESS) { - return res; + return SetError(res, "Invalid index type: " + std::to_string(schema_.index_type)); } //step 2: construct table schema @@ -203,7 +203,7 @@ ServerError DescribeTableTask::OnExecute() { ServerError res = SERVER_SUCCESS; res = ValidateTableName(table_name_); if(res != SERVER_SUCCESS) { - return res; + return SetError(res, "Invalid table name: " + table_name_); } //step 2: get table info @@ -243,12 +243,20 @@ ServerError BuildIndexTask::OnExecute() { TimeRecorder rc("BuildIndexTask"); //step 1: check arguments - if(table_name_.empty()) { - return SetError(SERVER_INVALID_TABLE_NAME, "Empty table name"); + ServerError res = SERVER_SUCCESS; + res = ValidateTableName(table_name_); + if(res != SERVER_SUCCESS) { + return SetError(res, "Invalid table name: " + table_name_); + } + + bool has_table = false; + engine::Status stat = DBWrapper::DB()->HasTable(table_name_, has_table); + if(!has_table) { + return SetError(SERVER_TABLE_NOT_EXIST, "Table " + table_name_ + " not exists"); } //step 2: check table existence - engine::Status stat = DBWrapper::DB()->BuildIndex(table_name_); + stat = DBWrapper::DB()->BuildIndex(table_name_); if(!stat.ok()) { return SetError(SERVER_BUILD_INDEX_ERROR, "Engine failed: " + stat.ToString()); } @@ -281,8 +289,9 @@ ServerError HasTableTask::OnExecute() { ServerError res = SERVER_SUCCESS; res = ValidateTableName(table_name_); if(res != SERVER_SUCCESS) { - return res; + return SetError(res, "Invalid table name: " + table_name_); } + //step 2: check table existence engine::Status stat = DBWrapper::DB()->HasTable(table_name_, has_table_); if(!stat.ok()) { @@ -316,7 +325,7 @@ ServerError DeleteTableTask::OnExecute() { ServerError res = SERVER_SUCCESS; res = ValidateTableName(table_name_); if(res != SERVER_SUCCESS) { - return res; + return SetError(res, "Invalid table name: " + table_name_); } //step 2: check table existence @@ -400,7 +409,7 @@ ServerError AddVectorTask::OnExecute() { ServerError res = SERVER_SUCCESS; res = ValidateTableName(table_name_); if(res != SERVER_SUCCESS) { - return res; + return SetError(res, "Invalid table name: " + table_name_); } if(record_array_.empty()) { @@ -491,7 +500,7 @@ ServerError SearchVectorTask::OnExecute() { ServerError res = SERVER_SUCCESS; res = ValidateTableName(table_name_); if(res != SERVER_SUCCESS) { - return res; + return SetError(res, "Invalid table name: " + table_name_); } if(top_k_ <= 0) { @@ -606,7 +615,7 @@ ServerError GetTableRowCountTask::OnExecute() { ServerError res = SERVER_SUCCESS; res = ValidateTableName(table_name_); if(res != SERVER_SUCCESS) { - return res; + return SetError(res, "Invalid table name: " + table_name_); } //step 2: get row count From 50e4e8a12ff253829b7d1c2946f659c5128f9d26 Mon Sep 17 00:00:00 2001 From: starlord Date: Mon, 15 Jul 2019 15:22:39 +0800 Subject: [PATCH 170/189] MS-207 build_index return success when table not existed Former-commit-id: de940fd3aedb6492c85bbd6c6a9fdc04731d7ff9 --- cpp/src/db/ExecutionEngine.h | 1 + cpp/src/server/RequestTask.cpp | 2 +- cpp/src/utils/ValidationUtil.cpp | 17 ++++++----------- cpp/unittest/utils/ValidationUtilTest.cpp | 12 +++++++----- 4 files changed, 15 insertions(+), 17 deletions(-) diff --git a/cpp/src/db/ExecutionEngine.h b/cpp/src/db/ExecutionEngine.h index 4503e55650..cd9e071b9c 100644 --- a/cpp/src/db/ExecutionEngine.h +++ b/cpp/src/db/ExecutionEngine.h @@ -19,6 +19,7 @@ enum class EngineType { FAISS_IDMAP = 1, FAISS_IVFFLAT, FAISS_IVFSQ8, + MAX_VALUE = FAISS_IVFSQ8, }; class ExecutionEngine { diff --git a/cpp/src/server/RequestTask.cpp b/cpp/src/server/RequestTask.cpp index fe31d51c5b..af6264da90 100644 --- a/cpp/src/server/RequestTask.cpp +++ b/cpp/src/server/RequestTask.cpp @@ -291,7 +291,7 @@ ServerError HasTableTask::OnExecute() { if(res != SERVER_SUCCESS) { return SetError(res, "Invalid table name: " + table_name_); } - + //step 2: check table existence engine::Status stat = DBWrapper::DB()->HasTable(table_name_, has_table_); if(!stat.ok()) { diff --git a/cpp/src/utils/ValidationUtil.cpp b/cpp/src/utils/ValidationUtil.cpp index a1e3f0dffc..53f00a4fc7 100644 --- a/cpp/src/utils/ValidationUtil.cpp +++ b/cpp/src/utils/ValidationUtil.cpp @@ -56,18 +56,13 @@ ValidateTableDimension(int64_t dimension) { ServerError ValidateTableIndexType(int32_t index_type) { - auto engine_type = engine::EngineType(index_type); - switch (engine_type) { - case engine::EngineType::FAISS_IDMAP: - case engine::EngineType::FAISS_IVFFLAT: - case engine::EngineType::FAISS_IVFSQ8:{ - SERVER_LOG_DEBUG << "Index type: " << index_type; - return SERVER_SUCCESS; - } - default: { - return SERVER_INVALID_INDEX_TYPE; - } + int engine_type = (int)engine::EngineType(index_type); + if(engine_type <= 0 || engine_type > (int)engine::EngineType::MAX_VALUE) { + return SERVER_INVALID_INDEX_TYPE; } + + SERVER_LOG_DEBUG << "Index type: " << index_type; + return SERVER_SUCCESS; } } diff --git a/cpp/unittest/utils/ValidationUtilTest.cpp b/cpp/unittest/utils/ValidationUtilTest.cpp index 095614e325..38fc63a10d 100644 --- a/cpp/unittest/utils/ValidationUtilTest.cpp +++ b/cpp/unittest/utils/ValidationUtilTest.cpp @@ -7,9 +7,11 @@ #include "utils/ValidationUtil.h" #include "utils/Error.h" +#include "db/ExecutionEngine.h" #include +using namespace zilliz::milvus; using namespace zilliz::milvus::server; TEST(ValidationUtilTest, TableNameTest) { @@ -53,9 +55,9 @@ TEST(ValidationUtilTest, TableDimensionTest) { } TEST(ValidationUtilTest, TableIndexTypeTest) { - ASSERT_EQ(ValidateTableIndexType(0), SERVER_INVALID_INDEX_TYPE); - ASSERT_EQ(ValidateTableIndexType(1), SERVER_SUCCESS); - ASSERT_EQ(ValidateTableIndexType(2), SERVER_SUCCESS); - ASSERT_EQ(ValidateTableIndexType(3), SERVER_INVALID_INDEX_TYPE); - ASSERT_EQ(ValidateTableIndexType(4), SERVER_INVALID_INDEX_TYPE); + ASSERT_EQ(ValidateTableIndexType((int)engine::EngineType::INVALID), SERVER_INVALID_INDEX_TYPE); + for(int i = 1; i <= (int)engine::EngineType::MAX_VALUE; i++) { + ASSERT_EQ(ValidateTableIndexType(i), SERVER_SUCCESS); + } + ASSERT_EQ(ValidateTableIndexType((int)engine::EngineType::MAX_VALUE + 1), SERVER_INVALID_INDEX_TYPE); } From 732abc72c72696cc3bed16fc213b3ea187759685 Mon Sep 17 00:00:00 2001 From: "peng.xu" Date: Mon, 15 Jul 2019 15:55:35 +0800 Subject: [PATCH 171/189] change ci build time from 20 to 30 Former-commit-id: d289352f66750e7b3e4cb8990bc0f46580952c87 --- ci/jenkinsfile/milvus_build.groovy | 3 +-- ci/jenkinsfile/milvus_build_no_ut.groovy | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/ci/jenkinsfile/milvus_build.groovy b/ci/jenkinsfile/milvus_build.groovy index 99892c90ef..016acfdefa 100644 --- a/ci/jenkinsfile/milvus_build.groovy +++ b/ci/jenkinsfile/milvus_build.groovy @@ -1,5 +1,5 @@ container('milvus-build-env') { - timeout(time: 20, unit: 'MINUTES') { + timeout(time: 30, unit: 'MINUTES') { gitlabCommitStatus(name: 'Build Engine') { dir ("milvus_engine") { try { @@ -17,4 +17,3 @@ container('milvus-build-env') { } } } - diff --git a/ci/jenkinsfile/milvus_build_no_ut.groovy b/ci/jenkinsfile/milvus_build_no_ut.groovy index 6c30ce491c..f4e243f1cc 100644 --- a/ci/jenkinsfile/milvus_build_no_ut.groovy +++ b/ci/jenkinsfile/milvus_build_no_ut.groovy @@ -1,5 +1,5 @@ container('milvus-build-env') { - timeout(time: 20, unit: 'MINUTES') { + timeout(time: 30, unit: 'MINUTES') { gitlabCommitStatus(name: 'Build Engine') { dir ("milvus_engine") { try { @@ -17,4 +17,3 @@ container('milvus-build-env') { } } } - From 813afe79076aae3f07a755d316a2fa002cdb224f Mon Sep 17 00:00:00 2001 From: "peng.xu" Date: Mon, 15 Jul 2019 17:30:13 +0800 Subject: [PATCH 172/189] add log before bg_error Former-commit-id: ab002bc46b2758d4061d5a25ff5f301d0f1516d0 --- cpp/src/db/DBImpl.cpp | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/cpp/src/db/DBImpl.cpp b/cpp/src/db/DBImpl.cpp index 55fc9cea5a..baee3a84ab 100644 --- a/cpp/src/db/DBImpl.cpp +++ b/cpp/src/db/DBImpl.cpp @@ -89,6 +89,7 @@ DBImpl::DBImpl(const Options& options) meta_ptr_ = DBMetaImplFactory::Build(options.meta, options.mode); mem_mgr_ = MemManagerFactory::Build(meta_ptr_, options_); if (options.mode != Options::MODE::READ_ONLY) { + ENGINE_LOG_INFO << "StartTimerTasks"; StartTimerTasks(); } } @@ -390,14 +391,11 @@ Status DBImpl::BackgroundMergeFiles(const std::string& table_id) { } void DBImpl::BackgroundCompaction(std::set table_ids) { -// static int b_count = 0; -// b_count++; -// std::cout << "BackgroundCompaction: " << b_count << std::endl; - Status status; for (auto& table_id : table_ids) { status = BackgroundMergeFiles(table_id); if (!status.ok()) { + ENGINE_LOG_ERROR << "BGERROR found during merge files!"; bg_error_ = status; return; } @@ -408,7 +406,6 @@ void DBImpl::BackgroundCompaction(std::set table_ids) { int ttl = 1; if (options_.mode == Options::MODE::CLUSTER) { ttl = meta::D_SEC; -// ENGINE_LOG_DEBUG << "Server mode is cluster. Clean up files with ttl = " << std::to_string(ttl) << "seconds."; } meta_ptr_->CleanUpFilesWithTTL(ttl); } @@ -541,9 +538,9 @@ void DBImpl::BackgroundBuildIndex() { meta_ptr_->FilesToIndex(to_index_files); Status status; for (auto& file : to_index_files) { - /* ENGINE_LOG_DEBUG << "Buiding index for " << file.location; */ status = BuildIndex(file); if (!status.ok()) { + ENGINE_LOG_ERROR << "BGERROR found during build index!"; bg_error_ = status; return; } @@ -552,7 +549,6 @@ void DBImpl::BackgroundBuildIndex() { break; } } - /* ENGINE_LOG_DEBUG << "All Buiding index Done"; */ } Status DBImpl::DropAll() { From a8368f58464ea47043d0e8117952c87e2e53dec9 Mon Sep 17 00:00:00 2001 From: "peng.xu" Date: Mon, 15 Jul 2019 18:50:56 +0800 Subject: [PATCH 173/189] add more logging for bg_error break Former-commit-id: 3d3ddbe93c0770968d0ac4f86b82c0a94c0fa341 --- cpp/src/db/DBImpl.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cpp/src/db/DBImpl.cpp b/cpp/src/db/DBImpl.cpp index baee3a84ab..8638a0da41 100644 --- a/cpp/src/db/DBImpl.cpp +++ b/cpp/src/db/DBImpl.cpp @@ -395,7 +395,7 @@ void DBImpl::BackgroundCompaction(std::set table_ids) { for (auto& table_id : table_ids) { status = BackgroundMergeFiles(table_id); if (!status.ok()) { - ENGINE_LOG_ERROR << "BGERROR found during merge files!"; + ENGINE_LOG_ERROR << "BGERROR found during merge files: " << status.ToString(); bg_error_ = status; return; } @@ -540,7 +540,7 @@ void DBImpl::BackgroundBuildIndex() { for (auto& file : to_index_files) { status = BuildIndex(file); if (!status.ok()) { - ENGINE_LOG_ERROR << "BGERROR found during build index!"; + ENGINE_LOG_ERROR << "BGERROR found during build index: " << status.ToString(); bg_error_ = status; return; } From 7d3c60e9562369c60f583051f3b7eb65a7d10337 Mon Sep 17 00:00:00 2001 From: starlord Date: Mon, 15 Jul 2019 19:24:20 +0800 Subject: [PATCH 174/189] avoid background merge thread stop Former-commit-id: 9dca4b821b870fa6167ca4e0bab3febe5eb98169 --- cpp/src/db/DBImpl.cpp | 5 ++--- cpp/src/db/DBImpl.h | 2 -- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/cpp/src/db/DBImpl.cpp b/cpp/src/db/DBImpl.cpp index 55fc9cea5a..6e4c49fc35 100644 --- a/cpp/src/db/DBImpl.cpp +++ b/cpp/src/db/DBImpl.cpp @@ -235,7 +235,6 @@ void DBImpl::BackgroundTimerTask() { Status status; server::SystemInfo::GetInstance().Init(); while (true) { - if (!bg_error_.ok()) break; if (shutting_down_.load(std::memory_order_acquire)){ for(auto& iter : compact_thread_results_) { iter.wait(); @@ -398,7 +397,7 @@ void DBImpl::BackgroundCompaction(std::set table_ids) { for (auto& table_id : table_ids) { status = BackgroundMergeFiles(table_id); if (!status.ok()) { - bg_error_ = status; + ENGINE_LOG_ERROR << "Merge files for table " << table_id << " failed: " << status.ToString(); return; } } @@ -544,7 +543,7 @@ void DBImpl::BackgroundBuildIndex() { /* ENGINE_LOG_DEBUG << "Buiding index for " << file.location; */ status = BuildIndex(file); if (!status.ok()) { - bg_error_ = status; + ENGINE_LOG_ERROR << "Building index for " << file.id_ << " failed: " << status.ToString(); return; } diff --git a/cpp/src/db/DBImpl.h b/cpp/src/db/DBImpl.h index 1309154a3c..73d28779b7 100644 --- a/cpp/src/db/DBImpl.h +++ b/cpp/src/db/DBImpl.h @@ -118,10 +118,8 @@ class DBImpl : public DB { BuildIndex(const meta::TableFileSchema &); private: - const Options options_; - Status bg_error_; std::atomic shutting_down_; std::thread bg_timer_thread_; From 7b35e2cebec428dcc6d8de75c50671db608a8a3f Mon Sep 17 00:00:00 2001 From: starlord Date: Mon, 15 Jul 2019 19:48:11 +0800 Subject: [PATCH 175/189] MS-235 Some test cases random fail Former-commit-id: c77aab33541b0c5da7c3cdc6a080fb47c76bd301 --- cpp/CHANGELOG.md | 1 + cpp/src/db/DBMetaImpl.cpp | 2 ++ 2 files changed, 3 insertions(+) diff --git a/cpp/CHANGELOG.md b/cpp/CHANGELOG.md index 812e62eb2d..3e699d8cd1 100644 --- a/cpp/CHANGELOG.md +++ b/cpp/CHANGELOG.md @@ -18,6 +18,7 @@ Please mark all change in change log and use the ticket from JIRA. - MS-233 - Remove mem manager log - MS-230 - Change parameter name: Maximum_memory to insert_buffer_size - MS-234 - Some case cause background merge thread stop +- MS-235 - Some test cases random fail ## Improvement - MS-156 - Add unittest for merge result functions diff --git a/cpp/src/db/DBMetaImpl.cpp b/cpp/src/db/DBMetaImpl.cpp index 99ae3b6711..48e0fa46cd 100644 --- a/cpp/src/db/DBMetaImpl.cpp +++ b/cpp/src/db/DBMetaImpl.cpp @@ -291,6 +291,8 @@ Status DBMetaImpl::HasNonIndexFiles(const std::string& table_id, bool& has) { try { auto selected = ConnectorPtr->select(columns(&TableFileSchema::id_), where((c(&TableFileSchema::file_type_) == (int) TableFileSchema::RAW + or + c(&TableFileSchema::file_type_) == (int) TableFileSchema::NEW or c(&TableFileSchema::file_type_) == (int) TableFileSchema::TO_INDEX) and c(&TableFileSchema::table_id_) == table_id From bc5961209b61ceda37baa2b5124b0656de577e6a Mon Sep 17 00:00:00 2001 From: zhiru Date: Mon, 15 Jul 2019 20:40:03 +0800 Subject: [PATCH 176/189] Add MySQLMetaImpl::HasNonIndexFiles Former-commit-id: 31682c46358678817802ed49dd10327382501c8e --- cpp/src/db/MySQLMetaImpl.cpp | 46 ++++++++++++++++++++++++++++++++++-- 1 file changed, 44 insertions(+), 2 deletions(-) diff --git a/cpp/src/db/MySQLMetaImpl.cpp b/cpp/src/db/MySQLMetaImpl.cpp index dc22c0307c..98394688a4 100644 --- a/cpp/src/db/MySQLMetaImpl.cpp +++ b/cpp/src/db/MySQLMetaImpl.cpp @@ -382,7 +382,49 @@ Status MySQLMetaImpl::CreateTable(TableSchema &table_schema) { } Status MySQLMetaImpl::HasNonIndexFiles(const std::string &table_id, bool &has) { - // TODO + + has = false; + + try { + + StoreQueryResult res; + + { + ScopedConnection connectionPtr(*mysql_connection_pool_, safe_grab); + + if (connectionPtr == nullptr) { + return Status::Error("Failed to connect to database server"); + } + + + Query hasNonIndexFilesQuery = connectionPtr->query(); + //since table_id is a unique column we just need to check whether it exists or not + hasNonIndexFilesQuery << "SELECT EXISTS " << + "(SELECT 1 FROM TableFiles " << + "WHERE table_id = " << quote << table_id << " AND " << + "(file_type = " << std::to_string(TableFileSchema::RAW) << " OR " << + "file_type = " << std::to_string(TableFileSchema::NEW) << " OR " << + "file_type = " << std::to_string(TableFileSchema::TO_INDEX) << ") " << + "AS " << quote << "check" << ";"; + + ENGINE_LOG_DEBUG << "MySQLMetaImpl::HasNonIndexFiles: " << hasNonIndexFilesQuery.str(); + + res = hasNonIndexFilesQuery.store(); + } //Scoped Connection + + int check = res[0]["check"]; + has = (check == 1); + + } catch (const BadQuery &er) { + // Handle any query errors + ENGINE_LOG_ERROR << "QUERY ERROR WHEN CHECKING IF NON INDEX FILES EXISTS" << ": " << er.what(); + return Status::DBTransactionError("QUERY ERROR WHEN CHECKING IF NON INDEX FILES EXISTS", er.what()); + } catch (const Exception &er) { + // Catch-all for any other MySQL++ exceptions + ENGINE_LOG_ERROR << "GENERAL ERROR WHEN CHECKING IF NON INDEX FILES EXISTS" << ": " << er.what(); + return Status::DBTransactionError("GENERAL ERROR WHEN CHECKING IF NON INDEX FILES EXISTS", er.what()); + } + return Status::OK(); } @@ -1382,7 +1424,7 @@ Status MySQLMetaImpl::UpdateTableFilesToIndex(const std::string &table_id) { "WHERE table_id = " << quote << table_id << " AND " << "file_type = " << std::to_string(TableFileSchema::RAW) << ";"; - ENGINE_LOG_DEBUG << "MySQLMetaImpl::UpdateTableFile: " << updateTableFilesToIndexQuery.str(); + ENGINE_LOG_DEBUG << "MySQLMetaImpl::UpdateTableFilesToIndex: " << updateTableFilesToIndexQuery.str(); } catch (const BadQuery &er) { // Handle any query errors From 0b84da5101f48b2735d5b86bbcd474e49bfa39a2 Mon Sep 17 00:00:00 2001 From: zhiru Date: Mon, 15 Jul 2019 20:41:17 +0800 Subject: [PATCH 177/189] Add MySQLMetaImpl::HasNonIndexFiles Former-commit-id: 9db26bc1ecb99c3522f6a4743da75fc3088fbb5e --- cpp/CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/cpp/CHANGELOG.md b/cpp/CHANGELOG.md index 3e699d8cd1..a11d40d7c9 100644 --- a/cpp/CHANGELOG.md +++ b/cpp/CHANGELOG.md @@ -19,6 +19,7 @@ Please mark all change in change log and use the ticket from JIRA. - MS-230 - Change parameter name: Maximum_memory to insert_buffer_size - MS-234 - Some case cause background merge thread stop - MS-235 - Some test cases random fail +- MS-236 - Add MySQLMetaImpl::HasNonIndexFiles ## Improvement - MS-156 - Add unittest for merge result functions From 049309f8338f4b38b612bf3f7579ca9aad49ac28 Mon Sep 17 00:00:00 2001 From: zhiru Date: Tue, 16 Jul 2019 10:25:01 +0800 Subject: [PATCH 178/189] fix Former-commit-id: ebf2ae2d41c259da0e3a22864eb30812b177c2de --- cpp/src/db/MySQLMetaImpl.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/src/db/MySQLMetaImpl.cpp b/cpp/src/db/MySQLMetaImpl.cpp index 98394688a4..692cfb04c6 100644 --- a/cpp/src/db/MySQLMetaImpl.cpp +++ b/cpp/src/db/MySQLMetaImpl.cpp @@ -404,7 +404,7 @@ Status MySQLMetaImpl::HasNonIndexFiles(const std::string &table_id, bool &has) { "WHERE table_id = " << quote << table_id << " AND " << "(file_type = " << std::to_string(TableFileSchema::RAW) << " OR " << "file_type = " << std::to_string(TableFileSchema::NEW) << " OR " << - "file_type = " << std::to_string(TableFileSchema::TO_INDEX) << ") " << + "file_type = " << std::to_string(TableFileSchema::TO_INDEX) << ")) " << "AS " << quote << "check" << ";"; ENGINE_LOG_DEBUG << "MySQLMetaImpl::HasNonIndexFiles: " << hasNonIndexFilesQuery.str(); From 585b8401dcc0ffc5e26bb5dd3025d3ed92f2f025 Mon Sep 17 00:00:00 2001 From: zhiru Date: Tue, 16 Jul 2019 11:20:45 +0800 Subject: [PATCH 179/189] fix Former-commit-id: 8076613c56f3eaa959ef5d21e06933bb09f41889 --- cpp/src/db/MySQLMetaImpl.cpp | 24 +++++++++++++++--------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/cpp/src/db/MySQLMetaImpl.cpp b/cpp/src/db/MySQLMetaImpl.cpp index 692cfb04c6..fa055feb3e 100644 --- a/cpp/src/db/MySQLMetaImpl.cpp +++ b/cpp/src/db/MySQLMetaImpl.cpp @@ -400,12 +400,12 @@ Status MySQLMetaImpl::HasNonIndexFiles(const std::string &table_id, bool &has) { Query hasNonIndexFilesQuery = connectionPtr->query(); //since table_id is a unique column we just need to check whether it exists or not hasNonIndexFilesQuery << "SELECT EXISTS " << - "(SELECT 1 FROM TableFiles " << - "WHERE table_id = " << quote << table_id << " AND " << - "(file_type = " << std::to_string(TableFileSchema::RAW) << " OR " << - "file_type = " << std::to_string(TableFileSchema::NEW) << " OR " << - "file_type = " << std::to_string(TableFileSchema::TO_INDEX) << ")) " << - "AS " << quote << "check" << ";"; + "(SELECT 1 FROM TableFiles " << + "WHERE table_id = " << quote << table_id << " AND " << + "(file_type = " << std::to_string(TableFileSchema::RAW) << " OR " << + "file_type = " << std::to_string(TableFileSchema::NEW) << " OR " << + "file_type = " << std::to_string(TableFileSchema::TO_INDEX) << ")) " << + "AS " << quote << "check" << ";"; ENGINE_LOG_DEBUG << "MySQLMetaImpl::HasNonIndexFiles: " << hasNonIndexFilesQuery.str(); @@ -1420,12 +1420,18 @@ Status MySQLMetaImpl::UpdateTableFilesToIndex(const std::string &table_id) { Query updateTableFilesToIndexQuery = connectionPtr->query(); updateTableFilesToIndexQuery << "UPDATE TableFiles " << - "SET file_type = " << std::to_string(TableFileSchema::TO_INDEX) << " " << - "WHERE table_id = " << quote << table_id << " AND " << - "file_type = " << std::to_string(TableFileSchema::RAW) << ";"; + "SET file_type = " << std::to_string(TableFileSchema::TO_INDEX) << " " << + "WHERE table_id = " << quote << table_id << " AND " << + "file_type = " << std::to_string(TableFileSchema::RAW) << ";"; ENGINE_LOG_DEBUG << "MySQLMetaImpl::UpdateTableFilesToIndex: " << updateTableFilesToIndexQuery.str(); + if (!updateTableFilesToIndexQuery.exec()) { + ENGINE_LOG_ERROR << "QUERY ERROR WHEN UPDATING TABLE FILE"; + return Status::DBTransactionError("QUERY ERROR WHEN UPDATING TABLE FILE", + updateTableFilesToIndexQuery.error()); + } + } catch (const BadQuery &er) { // Handle any query errors ENGINE_LOG_ERROR << "QUERY ERROR WHEN UPDATING TABLE FILES TO INDEX" << ": " << er.what(); From 51ebdb93a9a5f0f19559ab92eb28fab48bbb30a8 Mon Sep 17 00:00:00 2001 From: starlord Date: Wed, 17 Jul 2019 10:44:55 +0800 Subject: [PATCH 180/189] increse thrift thread count Former-commit-id: 33cec22bf8733907c7b6d67b5ff8b8cc28a816f7 --- cpp/src/server/MilvusServer.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/src/server/MilvusServer.cpp b/cpp/src/server/MilvusServer.cpp index 92e06d0ccd..2f68c4b189 100644 --- a/cpp/src/server/MilvusServer.cpp +++ b/cpp/src/server/MilvusServer.cpp @@ -76,7 +76,7 @@ MilvusServer::StartService() { return; } - stdcxx::shared_ptr threadManager(ThreadManager::newSimpleThreadManager()); + stdcxx::shared_ptr threadManager(ThreadManager::newSimpleThreadManager(16)); stdcxx::shared_ptr threadFactory(new PosixThreadFactory()); threadManager->threadFactory(threadFactory); threadManager->start(); From 5acd9ac4eb71cb54b950f7a97a036533770c9046 Mon Sep 17 00:00:00 2001 From: zhiru Date: Wed, 17 Jul 2019 12:50:37 +0800 Subject: [PATCH 181/189] MS-241 MS-242 Former-commit-id: 608b3d0b14834591a55fca324855b7cbbcf34ca5 --- cpp/CHANGELOG.md | 2 + cpp/cmake/DefineOptions.cmake | 2 - cpp/cmake/ThirdPartyPackages.cmake | 517 +++++----------------- cpp/scripts/requirements.sh | 10 + cpp/src/CMakeLists.txt | 17 +- cpp/unittest/CMakeLists.txt | 1 - cpp/unittest/db/CMakeLists.txt | 8 +- cpp/unittest/faiss_wrapper/CMakeLists.txt | 4 + cpp/unittest/metrics/CMakeLists.txt | 16 +- cpp/unittest/server/CMakeLists.txt | 4 + 10 files changed, 145 insertions(+), 436 deletions(-) create mode 100755 cpp/scripts/requirements.sh diff --git a/cpp/CHANGELOG.md b/cpp/CHANGELOG.md index a11d40d7c9..d6aa72860d 100644 --- a/cpp/CHANGELOG.md +++ b/cpp/CHANGELOG.md @@ -28,6 +28,8 @@ Please mark all change in change log and use the ticket from JIRA. - MS-206 - Support SQ8 index type - MS-208 - Add buildinde interface for C++ SDK - MS-212 - Support Inner product metric type +- MS-241 - Build Faiss with MKL if using Intel CPU; else build with OpenBlas +- MS-242 - clean up cmake and change MAKE_BUILD_ARGS to be user defined variable ## New Feature - MS-180 - Add new mem manager diff --git a/cpp/cmake/DefineOptions.cmake b/cpp/cmake/DefineOptions.cmake index 147663d0db..6a903f5806 100644 --- a/cpp/cmake/DefineOptions.cmake +++ b/cpp/cmake/DefineOptions.cmake @@ -57,8 +57,6 @@ define_option(MILVUS_VERBOSE_THIRDPARTY_BUILD define_option(MILVUS_WITH_ARROW "Build with ARROW" OFF) -define_option(MILVUS_BOOST_USE_SHARED "Rely on boost shared libraries where relevant" OFF) - define_option(MILVUS_BOOST_VENDORED "Use vendored Boost instead of existing Boost. \ Note that this requires linking Boost statically" ON) diff --git a/cpp/cmake/ThirdPartyPackages.cmake b/cpp/cmake/ThirdPartyPackages.cmake index 9e19b037f8..1f7c84f7cc 100644 --- a/cpp/cmake/ThirdPartyPackages.cmake +++ b/cpp/cmake/ThirdPartyPackages.cmake @@ -6,7 +6,7 @@ # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 + # # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an @@ -96,12 +96,8 @@ endmacro() macro(resolve_dependency DEPENDENCY_NAME) if (${DEPENDENCY_NAME}_SOURCE STREQUAL "AUTO") - #message(STATUS "Finding ${DEPENDENCY_NAME} package") -# find_package(${DEPENDENCY_NAME} QUIET) -# if (NOT ${DEPENDENCY_NAME}_FOUND) - #message(STATUS "${DEPENDENCY_NAME} package not found") + #disable find_package for now build_dependency(${DEPENDENCY_NAME}) -# endif () elseif (${DEPENDENCY_NAME}_SOURCE STREQUAL "BUNDLED") build_dependency(${DEPENDENCY_NAME}) elseif (${DEPENDENCY_NAME}_SOURCE STREQUAL "SYSTEM") @@ -117,11 +113,9 @@ string(TOUPPER ${CMAKE_BUILD_TYPE} UPPERCASE_BUILD_TYPE) set(EP_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${CMAKE_CXX_FLAGS_${UPPERCASE_BUILD_TYPE}}") set(EP_C_FLAGS "${CMAKE_C_FLAGS} ${CMAKE_C_FLAGS_${UPPERCASE_BUILD_TYPE}}") -if(NOT MSVC) - # Set -fPIC on all external projects - set(EP_CXX_FLAGS "${EP_CXX_FLAGS} -fPIC") - set(EP_C_FLAGS "${EP_C_FLAGS} -fPIC") -endif() +# Set -fPIC on all external projects +set(EP_CXX_FLAGS "${EP_CXX_FLAGS} -fPIC") +set(EP_C_FLAGS "${EP_C_FLAGS} -fPIC") # CC/CXX environment variables are captured on the first invocation of the # builder (e.g make or ninja) instead of when CMake is invoked into to build @@ -159,20 +153,13 @@ endif() # Ensure that a default make is set if("${MAKE}" STREQUAL "") - if(NOT MSVC) - find_program(MAKE make) - endif() + find_program(MAKE make) endif() -set(MAKE_BUILD_ARGS "-j2") - -## Using make -j in sub-make is fragile -#if(${CMAKE_GENERATOR} MATCHES "Makefiles") -# set(MAKE_BUILD_ARGS "") -#else() -# # limit the maximum number of jobs for ninja -# set(MAKE_BUILD_ARGS "-j4") -#endif() +if (NOT DEFINED MAKE_BUILD_ARGS) + set(MAKE_BUILD_ARGS "-j8") +endif() +message(STATUS "Third Party MAKE_BUILD_ARGS = ${MAKE_BUILD_ARGS}") # ---------------------------------------------------------------------- # Find pthreads @@ -285,7 +272,6 @@ if (DEFINED ENV{MILVUS_PROMETHEUS_URL}) set(PROMETHEUS_SOURCE_URL "$ENV{PROMETHEUS_OPENBLAS_URL}") else () set(PROMETHEUS_SOURCE_URL - #"https://github.com/JinHai-CN/prometheus-cpp/archive/${PROMETHEUS_VERSION}.tar.gz" https://github.com/jupp0r/prometheus-cpp.git) endif() @@ -347,6 +333,7 @@ if(DEFINED ENV{MILVUS_AWS_URL}) else() set(AWS_SOURCE_URL "https://github.com/aws/aws-sdk-cpp/archive/${AWS_VERSION}.tar.gz") endif() + # ---------------------------------------------------------------------- # ARROW @@ -354,19 +341,13 @@ macro(build_arrow) message(STATUS "Building Apache ARROW-${ARROW_VERSION} from source") set(ARROW_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/arrow_ep-prefix/src/arrow_ep/cpp") set(ARROW_STATIC_LIB_NAME arrow) -# set(ARROW_CUDA_STATIC_LIB_NAME arrow_cuda) + set(ARROW_STATIC_LIB "${ARROW_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}${ARROW_STATIC_LIB_NAME}${CMAKE_STATIC_LIBRARY_SUFFIX}" ) -# set(ARROW_CUDA_STATIC_LIB -# "${ARROW_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}${ARROW_CUDA_STATIC_LIB_NAME}${CMAKE_STATIC_LIBRARY_SUFFIX}" -# ) set(ARROW_INCLUDE_DIR "${ARROW_PREFIX}/include") - set(ARROW_CMAKE_ARGS ${EP_COMMON_CMAKE_ARGS} -# "-DARROW_THRIFT_URL=${THRIFT_SOURCE_URL}" - #"env ARROW_THRIFT_URL=${THRIFT_SOURCE_URL}" -DARROW_BUILD_STATIC=ON -DARROW_BUILD_SHARED=OFF -DARROW_PARQUET=ON @@ -375,8 +356,6 @@ macro(build_arrow) "-DCMAKE_LIBRARY_PATH=${CUDA_TOOLKIT_ROOT_DIR}/lib64/stubs" -DCMAKE_BUILD_TYPE=Release) -# set($ENV{ARROW_THRIFT_URL} ${THRIFT_SOURCE_URL}) - externalproject_add(arrow_ep GIT_REPOSITORY ${ARROW_SOURCE_URL} @@ -384,14 +363,8 @@ macro(build_arrow) ${ARROW_VERSION} GIT_SHALLOW TRUE -# SOURCE_DIR -# ${ARROW_PREFIX} -# BINARY_DIR -# ${ARROW_PREFIX} SOURCE_SUBDIR cpp -# COMMAND -# "export \"ARROW_THRIFT_URL=${THRIFT_SOURCE_URL}\"" ${EP_LOG_OPTIONS} CMAKE_ARGS ${ARROW_CMAKE_ARGS} @@ -400,21 +373,16 @@ macro(build_arrow) ${MAKE_BUILD_ARGS} INSTALL_COMMAND ${MAKE} install -# BUILD_IN_SOURCE -# 1 BUILD_BYPRODUCTS "${ARROW_STATIC_LIB}" -# "${ARROW_CUDA_STATIC_LIB}" ) -# ExternalProject_Add_StepDependencies(arrow_ep build thrift_ep) - file(MAKE_DIRECTORY "${ARROW_PREFIX}/include") add_library(arrow STATIC IMPORTED) set_target_properties(arrow PROPERTIES IMPORTED_LOCATION "${ARROW_STATIC_LIB}" INTERFACE_INCLUDE_DIRECTORIES "${ARROW_INCLUDE_DIR}") -# INTERFACE_LINK_LIBRARIES thrift) + add_dependencies(arrow arrow_ep) set(JEMALLOC_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/arrow_ep-prefix/src/arrow_ep-build/jemalloc_ep-prefix/src/jemalloc_ep") @@ -438,9 +406,6 @@ endif() # Add Boost dependencies (code adapted from Apache Kudu (incubating)) set(Boost_USE_MULTITHREADED ON) -if(MSVC AND MILVUS_USE_STATIC_CRT) - set(Boost_USE_STATIC_RUNTIME ON) -endif() set(Boost_ADDITIONAL_VERSIONS "1.70.0" "1.70" @@ -530,59 +495,8 @@ if(MILVUS_BOOST_VENDORED) add_dependencies(boost_filesystem_static boost_ep) add_dependencies(boost_serialization_static boost_ep) -else() - if(MSVC) - # disable autolinking in boost - add_definitions(-DBOOST_ALL_NO_LIB) - endif() - -# if(DEFINED ENV{BOOST_ROOT} OR DEFINED BOOST_ROOT) -# # In older versions of CMake (such as 3.2), the system paths for Boost will -# # be looked in first even if we set $BOOST_ROOT or pass -DBOOST_ROOT -# set(Boost_NO_SYSTEM_PATHS ON) -# endif() - - if(MILVUS_BOOST_USE_SHARED) - # Find shared Boost libraries. - set(Boost_USE_STATIC_LIBS OFF) - set(BUILD_SHARED_LIBS_KEEP ${BUILD_SHARED_LIBS}) - set(BUILD_SHARED_LIBS ON) - - if(MSVC) - # force all boost libraries to dynamic link - add_definitions(-DBOOST_ALL_DYN_LINK) - endif() - - if(MILVUS_BOOST_HEADER_ONLY) - find_package(Boost REQUIRED) - else() - find_package(Boost COMPONENTS serialization system filesystem REQUIRED) - set(BOOST_SYSTEM_LIBRARY Boost::system) - set(BOOST_FILESYSTEM_LIBRARY Boost::filesystem) - set(BOOST_SERIALIZATION_LIBRARY Boost::serialization) - set(MILVUS_BOOST_LIBS ${BOOST_SYSTEM_LIBRARY} ${BOOST_FILESYSTEM_LIBRARY}) - endif() - set(BUILD_SHARED_LIBS ${BUILD_SHARED_LIBS_KEEP}) - unset(BUILD_SHARED_LIBS_KEEP) - else() - # Find static boost headers and libs - # TODO Differentiate here between release and debug builds - set(Boost_USE_STATIC_LIBS ON) - if(MILVUS_BOOST_HEADER_ONLY) - find_package(Boost REQUIRED) - else() - find_package(Boost COMPONENTS serialization system filesystem REQUIRED) - set(BOOST_SYSTEM_LIBRARY Boost::system) - set(BOOST_FILESYSTEM_LIBRARY Boost::filesystem) - set(BOOST_SERIALIZATION_LIBRARY Boost::serialization) - set(MILVUS_BOOST_LIBS ${BOOST_SYSTEM_LIBRARY} ${BOOST_FILESYSTEM_LIBRARY}) - endif() - endif() endif() -#message(STATUS "Boost include dir: " ${Boost_INCLUDE_DIR}) -#message(STATUS "Boost libraries: " ${Boost_LIBRARIES}) - include_directories(SYSTEM ${Boost_INCLUDE_DIR}) link_directories(SYSTEM ${BOOST_LIB_DIR}) @@ -726,13 +640,6 @@ macro(build_openblas) add_dependencies(openblas openblas_ep) endmacro() -#if(MILVUS_WITH_OPENBLAS) -# resolve_dependency(OpenBLAS) -# -# get_target_property(OPENBLAS_INCLUDE_DIR openblas INTERFACE_INCLUDE_DIRECTORIES) -# include_directories(SYSTEM "${OPENBLAS_INCLUDE_DIR}") -#endif() - # ---------------------------------------------------------------------- # LAPACK @@ -770,16 +677,23 @@ macro(build_lapack) add_dependencies(lapack lapack_ep) endmacro() -#if(MILVUS_WITH_LAPACK) -# resolve_dependency(LAPACK) -# -# get_target_property(LAPACK_INCLUDE_DIR lapack INTERFACE_INCLUDE_DIRECTORIES) -# include_directories(SYSTEM "${LAPACK_INCLUDE_DIR}") -#endif() - # ---------------------------------------------------------------------- # FAISS +set(BUILD_FAISS_WITH_MKL false) + +if(EXISTS "/proc/cpuinfo") + FILE(READ /proc/cpuinfo PROC_CPUINFO) + + SET(VENDOR_ID_RX "vendor_id[ \t]*:[ \t]*([a-zA-Z]+)\n") + STRING(REGEX MATCH "${VENDOR_ID_RX}" VENDOR_ID "${PROC_CPUINFO}") + STRING(REGEX REPLACE "${VENDOR_ID_RX}" "\\1" VENDOR_ID "${VENDOR_ID}") + + if(${VENDOR_ID} STREQUAL "GenuineIntel") + set(BUILD_FAISS_WITH_MKL true) + endif() +endif() + macro(build_faiss) message(STATUS "Building FAISS-${FAISS_VERSION} from source") set(FAISS_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/faiss_ep-prefix/src/faiss_ep") @@ -787,33 +701,34 @@ macro(build_faiss) set(FAISS_STATIC_LIB "${FAISS_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}faiss${CMAKE_STATIC_LIBRARY_SUFFIX}") -# add_custom_target(faiss_dependencies) -# add_dependencies(faiss_dependencies openblas_ep) -# add_dependencies(faiss_dependencies openblas) -# get_target_property(FAISS_OPENBLAS_LIB_DIR openblas IMPORTED_LOCATION) -# get_filename_component(FAISS_OPENBLAS_LIB "${FAISS_OPENBLAS_LIB_DIR}" DIRECTORY) - set(FAISS_CONFIGURE_ARGS "--prefix=${FAISS_PREFIX}" "CFLAGS=${EP_C_FLAGS}" "CXXFLAGS=${EP_CXX_FLAGS}" - "LDFLAGS=-L${OPENBLAS_PREFIX}/lib -L${LAPACK_PREFIX}/lib -lopenblas -llapack" --without-python) -# if(OPENBLAS_STATIC_LIB) -# set(OPENBLAS_LIBRARY ${OPENBLAS_STATIC_LIB}) -# else() -# set(OPENBLAS_LIBRARY ${OPENBLAS_SHARED_LIB}) -# endif() -# set(FAISS_DEPENDENCIES ${FAISS_DEPENDENCIES} ${OPENBLAS_LIBRARY}) + set(FAISS_CFLAGS ${EP_C_FLAGS}) + set(FAISS_CXXFLAGS ${EP_CXX_FLAGS}) + + if(${BUILD_FAISS_WITH_MKL} STREQUAL "true") + message(STATUS "Build Faiss with MKL") + if(NOT DEFINED MKL_LIB_PATH) + set(MKL_LIB_PATH "/opt/intel/compilers_and_libraries_2019.4.243/linux/mkl/lib/intel64") + endif() + set(FAISS_CONFIGURE_ARGS ${FAISS_CONFIGURE_ARGS} + "CPPFLAGS=-DMKL_ILP64 -m64 -I${MKL_LIB_PATH}/../../include" + "LDFLAGS=-L${MKL_LIB_PATH}" + "LIBS=-Wl,--start-group ${MKL_LIB_PATH}/libmkl_intel_ilp64.a ${MKL_LIB_PATH}/libmkl_gnu_thread.a ${MKL_LIB_PATH}/libmkl_core.a -Wl,--end-group -lgomp -lpthread -lm -ldl") + + else() + message(STATUS "Build Faiss with OpenBlas/LAPACK") + set(FAISS_CONFIGURE_ARGS ${FAISS_CONFIGURE_ARGS} + "LDFLAGS=-L${OPENBLAS_PREFIX}/lib -L${LAPACK_PREFIX}/lib") + endif() if(${MILVUS_WITH_FAISS_GPU_VERSION} STREQUAL "ON") set(FAISS_CONFIGURE_ARGS ${FAISS_CONFIGURE_ARGS} "--with-cuda=${CUDA_TOOLKIT_ROOT_DIR}" -# "with_cuda_arch=\"-gencode=arch=compute_35,code=compute_35 \\ -# -gencode=arch=compute_52,code=compute_52 \\ -# -gencode=arch=compute_60,code=compute_60 \\ -# -gencode=arch=compute_61,code=compute_61\"" "--with-cuda-arch=\"-gencode=arch=compute_35,code=compute_35\"" "--with-cuda-arch=\"-gencode=arch=compute_52,code=compute_52\"" "--with-cuda-arch=\"-gencode=arch=compute_60,code=compute_60\"" @@ -830,58 +745,66 @@ macro(build_faiss) CONFIGURE_COMMAND "./configure" ${FAISS_CONFIGURE_ARGS} -# BINARY_DIR -# ${FAISS_PREFIX} -# INSTALL_DIR -# ${FAISS_PREFIX} -# BUILD_COMMAND -# ${MAKE} ${MAKE_BUILD_ARGS} BUILD_COMMAND ${MAKE} ${MAKE_BUILD_ARGS} all COMMAND cd gpu && ${MAKE} ${MAKE_BUILD_ARGS} BUILD_IN_SOURCE 1 -# INSTALL_DIR -# ${FAISS_PREFIX} INSTALL_COMMAND ${MAKE} install COMMAND ln -s faiss_ep ../faiss BUILD_BYPRODUCTS ${FAISS_STATIC_LIB}) -# DEPENDS -# ${faiss_dependencies}) - - ExternalProject_Add_StepDependencies(faiss_ep build openblas_ep lapack_ep) + + if(${BUILD_FAISS_WITH_MKL} STREQUAL "false") + ExternalProject_Add_StepDependencies(faiss_ep build openblas_ep lapack_ep) + endif() file(MAKE_DIRECTORY "${FAISS_INCLUDE_DIR}") - add_library(faiss STATIC IMPORTED) - set_target_properties( - faiss - PROPERTIES IMPORTED_LOCATION "${FAISS_STATIC_LIB}" - INTERFACE_INCLUDE_DIRECTORIES "${FAISS_INCLUDE_DIR}" - INTERFACE_LINK_LIBRARIES "openblas;lapack" ) + add_library(faiss SHARED IMPORTED) + if(${BUILD_FAISS_WITH_MKL} STREQUAL "true") + set(MKL_LIBS ${MKL_LIB_PATH}/libmkl_intel_ilp64.a + ${MKL_LIB_PATH}/libmkl_gnu_thread.a + ${MKL_LIB_PATH}/libmkl_core.a) + + set_target_properties( + faiss + PROPERTIES IMPORTED_LOCATION "${FAISS_STATIC_LIB}" + INTERFACE_INCLUDE_DIRECTORIES "${FAISS_INCLUDE_DIR}" + INTERFACE_LINK_LIBRARIES "${MKL_LIBS}" ) + else() + set_target_properties( + faiss + PROPERTIES IMPORTED_LOCATION "${FAISS_STATIC_LIB}" + INTERFACE_INCLUDE_DIRECTORIES "${FAISS_INCLUDE_DIR}" + INTERFACE_LINK_LIBRARIES "openblas;lapack" ) + endif() + add_dependencies(faiss faiss_ep) - #add_dependencies(faiss openblas_ep) - #add_dependencies(faiss lapack_ep) - #target_link_libraries(faiss ${OPENBLAS_PREFIX}/lib) - #target_link_libraries(faiss ${LAPACK_PREFIX}/lib) + + if(${BUILD_FAISS_WITH_MKL} STREQUAL "false") + add_dependencies(faiss openblas_ep) + add_dependencies(faiss lapack_ep) + endif() endmacro() if(MILVUS_WITH_FAISS) - resolve_dependency(OpenBLAS) - get_target_property(OPENBLAS_INCLUDE_DIR openblas INTERFACE_INCLUDE_DIRECTORIES) - include_directories(SYSTEM "${OPENBLAS_INCLUDE_DIR}") - link_directories(SYSTEM ${OPENBLAS_PREFIX}/lib) + if(${BUILD_FAISS_WITH_MKL} STREQUAL "false") + resolve_dependency(OpenBLAS) + get_target_property(OPENBLAS_INCLUDE_DIR openblas INTERFACE_INCLUDE_DIRECTORIES) + include_directories(SYSTEM "${OPENBLAS_INCLUDE_DIR}") + link_directories(SYSTEM ${OPENBLAS_PREFIX}/lib) - resolve_dependency(LAPACK) - get_target_property(LAPACK_INCLUDE_DIR lapack INTERFACE_INCLUDE_DIRECTORIES) - include_directories(SYSTEM "${LAPACK_INCLUDE_DIR}") - link_directories(SYSTEM "${LAPACK_PREFIX}/lib") + resolve_dependency(LAPACK) + get_target_property(LAPACK_INCLUDE_DIR lapack INTERFACE_INCLUDE_DIRECTORIES) + include_directories(SYSTEM "${LAPACK_INCLUDE_DIR}") + link_directories(SYSTEM "${LAPACK_PREFIX}/lib") + endif() resolve_dependency(FAISS) get_target_property(FAISS_INCLUDE_DIR faiss INTERFACE_INCLUDE_DIRECTORIES) @@ -926,8 +849,6 @@ macro(build_gtest) set(GMOCK_STATIC_LIB "${GTEST_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}gmock${CMAKE_STATIC_LIBRARY_SUFFIX}" ) - - ExternalProject_Add(googletest_ep URL ${GTEST_SOURCE_URL} @@ -967,13 +888,11 @@ macro(build_gtest) endmacro() if (MILVUS_BUILD_TESTS) - #message(STATUS "Resolving gtest dependency") resolve_dependency(GTest) if(NOT GTEST_VENDORED) endif() - - # TODO: Don't use global includes but rather target_include_directories + get_target_property(GTEST_INCLUDE_DIR gtest INTERFACE_INCLUDE_DIRECTORIES) link_directories(SYSTEM "${GTEST_PREFIX}/lib") include_directories(SYSTEM ${GTEST_INCLUDE_DIR}) @@ -1011,32 +930,8 @@ macro(build_lz4) set(LZ4_BUILD_DIR "${CMAKE_CURRENT_BINARY_DIR}/lz4_ep-prefix/src/lz4_ep") set(LZ4_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/lz4_ep-prefix/") - if(MSVC) - if(MILVUS_USE_STATIC_CRT) - if(${UPPERCASE_BUILD_TYPE} STREQUAL "DEBUG") - set(LZ4_RUNTIME_LIBRARY_LINKAGE "/p:RuntimeLibrary=MultiThreadedDebug") - else() - set(LZ4_RUNTIME_LIBRARY_LINKAGE "/p:RuntimeLibrary=MultiThreaded") - endif() - endif() - set(LZ4_STATIC_LIB - "${LZ4_BUILD_DIR}/visual/VS2010/bin/x64_${CMAKE_BUILD_TYPE}/liblz4_static.lib") - set(LZ4_BUILD_COMMAND - BUILD_COMMAND - msbuild.exe - /m - /p:Configuration=${CMAKE_BUILD_TYPE} - /p:Platform=x64 - /p:PlatformToolset=v140 - ${LZ4_RUNTIME_LIBRARY_LINKAGE} - /t:Build - ${LZ4_BUILD_DIR}/visual/VS2010/lz4.sln) - else() - set(LZ4_STATIC_LIB "${LZ4_BUILD_DIR}/lib/liblz4.a") - #set(LZ4_BUILD_COMMAND BUILD_COMMAND ${CMAKE_SOURCE_DIR}/build-support/build-lz4-lib.sh - # "AR=${CMAKE_AR}") - set(LZ4_BUILD_COMMAND BUILD_COMMAND ${MAKE} ${MAKE_BUILD_ARGS} CFLAGS=${EP_C_FLAGS}) - endif() + set(LZ4_STATIC_LIB "${LZ4_BUILD_DIR}/lib/liblz4.a") + set(LZ4_BUILD_COMMAND BUILD_COMMAND ${MAKE} ${MAKE_BUILD_ARGS} CFLAGS=${EP_C_FLAGS}) # We need to copy the header in lib to directory outside of the build externalproject_add(lz4_ep @@ -1071,7 +966,6 @@ endmacro() if(MILVUS_WITH_LZ4) resolve_dependency(Lz4) - # TODO: Don't use global includes but rather target_include_directories get_target_property(LZ4_INCLUDE_DIR lz4 INTERFACE_INCLUDE_DIRECTORIES) link_directories(SYSTEM ${LZ4_BUILD_DIR}/lib/) include_directories(SYSTEM ${LZ4_INCLUDE_DIR}) @@ -1097,16 +991,8 @@ macro(build_mysqlpp) externalproject_add(mysqlpp_ep URL ${MYSQLPP_SOURCE_URL} -# GIT_REPOSITORY -# ${MYSQLPP_SOURCE_URL} -# GIT_TAG -# ${MYSQLPP_VERSION} -# GIT_SHALLOW -# TRUE ${EP_LOG_OPTIONS} CONFIGURE_COMMAND -# "./bootstrap" -# COMMAND "./configure" ${MYSQLPP_CONFIGURE_ARGS} BUILD_COMMAND @@ -1167,10 +1053,6 @@ macro(build_prometheus) ${PROMETHEUS_VERSION} GIT_SHALLOW TRUE -# GIT_CONFIG -# recurse-submodules=true -# URL -# ${PROMETHEUS_SOURCE_URL} ${EP_LOG_OPTIONS} CMAKE_ARGS ${PROMETHEUS_CMAKE_ARGS} @@ -1214,21 +1096,15 @@ if(MILVUS_WITH_PROMETHEUS) resolve_dependency(Prometheus) - # TODO: Don't use global includes but rather target_include_directories - #get_target_property(PROMETHEUS-core_INCLUDE_DIRS prometheus-core INTERFACE_INCLUDE_DIRECTORIES) - - #get_target_property(PROMETHEUS_PUSH_INCLUDE_DIRS prometheus_push INTERFACE_INCLUDE_DIRECTORIES) link_directories(SYSTEM ${PROMETHEUS_PREFIX}/push/) include_directories(SYSTEM ${PROMETHEUS_PREFIX}/push/include) - #get_target_property(PROMETHEUS_PULL_INCLUDE_DIRS prometheus_pull INTERFACE_INCLUDE_DIRECTORIES) link_directories(SYSTEM ${PROMETHEUS_PREFIX}/pull/) include_directories(SYSTEM ${PROMETHEUS_PREFIX}/pull/include) link_directories(SYSTEM ${PROMETHEUS_PREFIX}/core/) include_directories(SYSTEM ${PROMETHEUS_PREFIX}/core/include) - #link_directories(${PROMETHEUS_PREFIX}/civetweb_ep-prefix/src/civetweb_ep) endif() # ---------------------------------------------------------------------- @@ -1276,8 +1152,6 @@ if(MILVUS_WITH_ROCKSDB) resolve_dependency(RocksDB) - # TODO: Don't use global includes but rather target_include_directories -# get_target_property(ROCKSDB_INCLUDE_DIRS rocksdb INTERFACE_INCLUDE_DIRECTORIES) link_directories(SYSTEM ${ROCKSDB_PREFIX}/lib/lib/) include_directories(SYSTEM ${ROCKSDB_INCLUDE_DIRS}) endif() @@ -1326,34 +1200,9 @@ macro(build_snappy) endmacro() if(MILVUS_WITH_SNAPPY) -# if(Snappy_SOURCE STREQUAL "AUTO") -# # Normally *Config.cmake files reside in /usr/lib/cmake but Snappy -# # errornously places them in ${CMAKE_ROOT}/Modules/ -# # This is fixed in 1.1.7 but fedora (30) still installs into the wrong -# # location. -# # https://bugzilla.redhat.com/show_bug.cgi?id=1679727 -# # https://src.fedoraproject.org/rpms/snappy/pull-request/1 -# find_package(Snappy QUIET HINTS "${CMAKE_ROOT}/Modules/") -# if(NOT Snappy_FOUND) -# find_package(SnappyAlt) -# endif() -# if(NOT Snappy_FOUND AND NOT SnappyAlt_FOUND) -# build_snappy() -# endif() -# elseif(Snappy_SOURCE STREQUAL "BUNDLED") -# build_snappy() -# elseif(Snappy_SOURCE STREQUAL "SYSTEM") -# # SnappyConfig.cmake is not installed on Ubuntu/Debian -# # TODO: Make a bug report upstream -# find_package(Snappy HINTS "${CMAKE_ROOT}/Modules/") -# if(NOT Snappy_FOUND) -# find_package(SnappyAlt REQUIRED) -# endif() -# endif() resolve_dependency(Snappy) - - # TODO: Don't use global includes but rather target_include_directories + get_target_property(SNAPPY_INCLUDE_DIRS snappy INTERFACE_INCLUDE_DIRECTORIES) link_directories(SYSTEM ${SNAPPY_PREFIX}/lib/) include_directories(SYSTEM ${SNAPPY_INCLUDE_DIRS}) @@ -1425,75 +1274,11 @@ macro(build_sqlite_orm) endif () - #set(SQLITE_ORM_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/sqlite_orm_ep-prefix/src/sqlite_orm_ep") - #set(SQLITE_ORM_INCLUDE_DIR "${SQLITE_ORM_PREFIX}/include/sqlite_orm") - -# set(SQLITE_ORM_STATIC_LIB -# "${SQLITE_ORM_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}sqlite_orm${CMAKE_STATIC_LIBRARY_SUFFIX}") -# -# set(SQLITE_ORM_CMAKE_CXX_FLAGS "${EP_CXX_FLAGS} -std=c++14") -# set(SQLITE_ORM_CMAKE_CXX_FLAGS_DEBUG "${EP_CXX_FLAGS} -std=c++14") -# -# set(SQLITE_ORM_CMAKE_ARGS -# ${EP_COMMON_CMAKE_ARGS} -# "-DCMAKE_INSTALL_PREFIX=${SQLITE_ORM_PREFIX}" -# #"LDFLAGS=-L${SQLITE_PREFIX}" -# #"-DCMAKE_PREFIX_PATH=${SQLITE_PREFIX}/include" -# "-DCMAKE_INCLUDE_PATH=${SQLITE_PREFIX}/include" -# "-DCMAKE_CXX_FLAGS=${SQLITE_ORM_CMAKE_CXX_FLAGS}" -# "-DCMAKE_CXX_FLAGS_DEBUG=${SQLITE_ORM_CMAKE_CXX_FLAGS}" -# -DSqliteOrm_BuildTests=off -# -DBUILD_TESTING=off) -# message(STATUS "SQLITE_INCLUDE: ${SQLITE_ORM_CMAKE_ARGS}") -# -# message(STATUS "SQLITE_ORM_CMAKE_CXX_FLAGS: ${SQLITE_ORM_CMAKE_CXX_FLAGS}") - -# externalproject_add(sqlite_orm_ep -# URL -# ${SQLITE_ORM_SOURCE_URL} -# PREFIX ${CMAKE_CURRENT_BINARY_DIR}/sqlite_orm_ep-prefix -# CONFIGURE_COMMAND -# "" -# BUILD_COMMAND -# "" -# INSTALL_COMMAND -# "" - #${EP_LOG_OPTIONS} - #${EP_LOG_OPTIONS} -# CMAKE_ARGS -# ${SQLITE_ORM_CMAKE_ARGS} -# BUILD_COMMAND -# ${MAKE} -# ${MAKE_BUILD_ARGS} -# #"LDFLAGS=-L${SQLITE_PREFIX}" -# BUILD_IN_SOURCE -# 1 -# BUILD_BYPRODUCTS -# "${SQLITE_ORM_STATIC_LIB}" -# ) -# ExternalProject_Add_StepDependencies(sqlite_orm_ep build sqlite_ep) - - #set(SQLITE_ORM_SQLITE_HEADER ${SQLITE_INCLUDE_DIR}/sqlite3.h) -# file(MAKE_DIRECTORY "${SQLITE_ORM_INCLUDE_DIR}") -# add_library(sqlite_orm STATIC IMPORTED) -## message(STATUS "SQLITE_INCLUDE_DIR: ${SQLITE_INCLUDE_DIR}") -# set_target_properties( -# sqlite_orm -# PROPERTIES -# IMPORTED_LOCATION "${SQLITE_ORM_STATIC_LIB}" -# INTERFACE_INCLUDE_DIRECTORIES "${SQLITE_ORM_INCLUDE_DIR};${SQLITE_INCLUDE_DIR}") -# target_include_directories(sqlite_orm INTERFACE ${SQLITE_PREFIX} ${SQLITE_INCLUDE_DIR}) -# target_link_libraries(sqlite_orm INTERFACE sqlite) -# -# add_dependencies(sqlite_orm sqlite_orm_ep) endmacro() if(MILVUS_WITH_SQLITE_ORM) resolve_dependency(SQLite_ORM) -# ExternalProject_Get_Property(sqlite_orm_ep source_dir) -# set(SQLITE_ORM_INCLUDE_DIR ${source_dir}/sqlite_orm_ep) include_directories(SYSTEM "${SQLITE_ORM_INCLUDE_DIR}") - #message(STATUS "SQLITE_ORM_INCLUDE_DIR: ${SQLITE_ORM_INCLUDE_DIR}") endif() # ---------------------------------------------------------------------- @@ -1533,18 +1318,7 @@ macro(build_thrift) endif() set(THRIFT_STATIC_LIB_NAME "${CMAKE_STATIC_LIBRARY_PREFIX}thrift") - if(MSVC) - if(MILVUS_USE_STATIC_CRT) - set(THRIFT_STATIC_LIB_NAME "${THRIFT_STATIC_LIB_NAME}") - set(THRIFT_CMAKE_ARGS ${THRIFT_CMAKE_ARGS} "-DWITH_MT=ON") - else() - set(THRIFT_STATIC_LIB_NAME "${THRIFT_STATIC_LIB_NAME}") - set(THRIFT_CMAKE_ARGS ${THRIFT_CMAKE_ARGS} "-DWITH_MT=OFF") - endif() - endif() - if(${UPPERCASE_BUILD_TYPE} STREQUAL "DEBUG") - set(THRIFT_STATIC_LIB_NAME "${THRIFT_STATIC_LIB_NAME}") - endif() + set(THRIFT_STATIC_LIB "${THRIFT_PREFIX}/lib/${THRIFT_STATIC_LIB_NAME}${CMAKE_STATIC_LIBRARY_SUFFIX}") @@ -1555,60 +1329,6 @@ macro(build_thrift) endif() set(THRIFT_DEPENDENCIES ${THRIFT_DEPENDENCIES} ${ZLIB_LIBRARY}) - if(MSVC) - set(WINFLEXBISON_VERSION 2.4.9) - set(WINFLEXBISON_PREFIX - "${CMAKE_CURRENT_BINARY_DIR}/winflexbison_ep/src/winflexbison_ep-install") - externalproject_add( - winflexbison_ep - URL - https://github.com/lexxmark/winflexbison/releases/download/v.${WINFLEXBISON_VERSION}/win_flex_bison-${WINFLEXBISON_VERSION}.zip - URL_HASH - MD5=a2e979ea9928fbf8567e995e9c0df765 - SOURCE_DIR - ${WINFLEXBISON_PREFIX} - CONFIGURE_COMMAND - "" - BUILD_COMMAND - "" - INSTALL_COMMAND - "" - ${EP_LOG_OPTIONS}) - set(THRIFT_DEPENDENCIES ${THRIFT_DEPENDENCIES} winflexbison_ep) - - set(THRIFT_CMAKE_ARGS - "-DFLEX_EXECUTABLE=${WINFLEXBISON_PREFIX}/win_flex.exe" - "-DBISON_EXECUTABLE=${WINFLEXBISON_PREFIX}/win_bison.exe" - "-DZLIB_INCLUDE_DIR=${ZLIB_INCLUDE_DIR}" - "-DWITH_SHARED_LIB=OFF" - "-DWITH_PLUGIN=OFF" - ${THRIFT_CMAKE_ARGS}) - elseif(APPLE) - # Some other process always resets BISON_EXECUTABLE to the system default, - # thus we use our own variable here. - if(NOT DEFINED THRIFT_BISON_EXECUTABLE) - find_package(BISON 2.5.1) - - # In the case where we cannot find a system-wide installation, look for - # homebrew and ask for its bison installation. - if(NOT BISON_FOUND) - find_program(BREW_BIN brew) - if(BREW_BIN) - execute_process(COMMAND ${BREW_BIN} --prefix bison - OUTPUT_VARIABLE BISON_PREFIX - OUTPUT_STRIP_TRAILING_WHITESPACE) - set(BISON_EXECUTABLE "${BISON_PREFIX}/bin/bison") - find_package(BISON 2.5.1) - set(THRIFT_BISON_EXECUTABLE "${BISON_EXECUTABLE}") - endif() - else() - set(THRIFT_BISON_EXECUTABLE "${BISON_EXECUTABLE}") - endif() - endif() - set(THRIFT_CMAKE_ARGS "-DBISON_EXECUTABLE=${THRIFT_BISON_EXECUTABLE}" - ${THRIFT_CMAKE_ARGS}) - endif() - externalproject_add(thrift_ep URL ${THRIFT_SOURCE_URL} @@ -1637,8 +1357,7 @@ endmacro() if(MILVUS_WITH_THRIFT) resolve_dependency(Thrift) - # TODO: Don't use global includes but rather target_include_directories -# MESSAGE(STATUS ${THRIFT_PREFIX}/lib/) + link_directories(SYSTEM ${THRIFT_PREFIX}/lib/) link_directories(SYSTEM ${CMAKE_CURRENT_BINARY_DIR}/thrift_ep-prefix/src/thrift_ep-build/lib) include_directories(SYSTEM ${THRIFT_INCLUDE_DIR}) @@ -1684,8 +1403,7 @@ endmacro() if(MILVUS_WITH_YAMLCPP) resolve_dependency(yaml-cpp) - - # TODO: Don't use global includes but rather target_include_directories + get_target_property(YAMLCPP_INCLUDE_DIR yaml-cpp INTERFACE_INCLUDE_DIRECTORIES) link_directories(SYSTEM ${YAMLCPP_PREFIX}/lib/) include_directories(SYSTEM ${YAMLCPP_INCLUDE_DIR}) @@ -1697,15 +1415,7 @@ endif() macro(build_zlib) message(STATUS "Building ZLIB-${ZLIB_VERSION} from source") set(ZLIB_PREFIX "${CMAKE_CURRENT_BINARY_DIR}/zlib_ep-prefix/src/zlib_ep") - if(MSVC) - if(${UPPERCASE_BUILD_TYPE} STREQUAL "DEBUG") - set(ZLIB_STATIC_LIB_NAME zlibstaticd.lib) - else() - set(ZLIB_STATIC_LIB_NAME zlibstatic.lib) - endif() - else() - set(ZLIB_STATIC_LIB_NAME libz.a) - endif() + set(ZLIB_STATIC_LIB_NAME libz.a) set(ZLIB_STATIC_LIB "${ZLIB_PREFIX}/lib/${ZLIB_STATIC_LIB_NAME}") set(ZLIB_CMAKE_ARGS ${EP_COMMON_CMAKE_ARGS} "-DCMAKE_INSTALL_PREFIX=${ZLIB_PREFIX}" -DBUILD_SHARED_LIBS=OFF) @@ -1734,8 +1444,7 @@ endmacro() if(MILVUS_WITH_ZLIB) resolve_dependency(ZLIB) - - # TODO: Don't use global includes but rather target_include_directories + get_target_property(ZLIB_INCLUDE_DIR zlib INTERFACE_INCLUDE_DIRECTORIES) include_directories(SYSTEM ${ZLIB_INCLUDE_DIR}) endif() @@ -1757,22 +1466,15 @@ macro(build_zstd) -DZSTD_BUILD_STATIC=on -DZSTD_MULTITHREAD_SUPPORT=off) - if(MSVC) - set(ZSTD_STATIC_LIB "${ZSTD_PREFIX}/lib/zstd_static.lib") - if(MILVUS_USE_STATIC_CRT) - set(ZSTD_CMAKE_ARGS ${ZSTD_CMAKE_ARGS} "-DZSTD_USE_STATIC_RUNTIME=on") - endif() - else() - set(ZSTD_STATIC_LIB "${ZSTD_PREFIX}/lib/libzstd.a") - # Only pass our C flags on Unix as on MSVC it leads to a - # "incompatible command-line options" error - set(ZSTD_CMAKE_ARGS - ${ZSTD_CMAKE_ARGS} - -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER} - -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER} - -DCMAKE_C_FLAGS=${EP_C_FLAGS} - -DCMAKE_CXX_FLAGS=${EP_CXX_FLAGS}) - endif() + + set(ZSTD_STATIC_LIB "${ZSTD_PREFIX}/lib/libzstd.a") + + set(ZSTD_CMAKE_ARGS + ${ZSTD_CMAKE_ARGS} + -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER} + -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER} + -DCMAKE_C_FLAGS=${EP_C_FLAGS} + -DCMAKE_CXX_FLAGS=${EP_CXX_FLAGS}) if(CMAKE_VERSION VERSION_LESS 3.7) message(FATAL_ERROR "Building zstd using ExternalProject requires at least CMake 3.7") @@ -1806,8 +1508,7 @@ endmacro() if(MILVUS_WITH_ZSTD) resolve_dependency(ZSTD) - - # TODO: Don't use global includes but rather target_include_directories + get_target_property(ZSTD_INCLUDE_DIR zstd INTERFACE_INCLUDE_DIRECTORIES) link_directories(SYSTEM ${ZSTD_PREFIX}/lib) include_directories(SYSTEM ${ZSTD_INCLUDE_DIR}) @@ -1823,7 +1524,7 @@ macro(build_aws) ${EP_COMMON_TOOLCHAIN} "-DCMAKE_INSTALL_PREFIX=${AWS_PREFIX}" -DCMAKE_BUILD_TYPE=Release - -DCMAKE_INSTALL_LIBDIR=lib #${CMAKE_INSTALL_LIBDIR} + -DCMAKE_INSTALL_LIBDIR=lib -DBUILD_ONLY=s3 -DBUILD_SHARED_LIBS=off -DENABLE_TESTING=off @@ -1834,8 +1535,7 @@ macro(build_aws) "${AWS_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}aws-cpp-sdk-core${CMAKE_STATIC_LIBRARY_SUFFIX}") set(AWS_CPP_SDK_S3_STATIC_LIB "${AWS_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}aws-cpp-sdk-s3${CMAKE_STATIC_LIBRARY_SUFFIX}") - # Only pass our C flags on Unix as on MSVC it leads to a - # "incompatible command-line options" error + set(AWS_CMAKE_ARGS ${AWS_CMAKE_ARGS} -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER} @@ -1843,10 +1543,6 @@ macro(build_aws) -DCMAKE_C_FLAGS=${EP_C_FLAGS} -DCMAKE_CXX_FLAGS=${EP_CXX_FLAGS}) - if(CMAKE_VERSION VERSION_LESS 3.7) - message(FATAL_ERROR "Building AWS using ExternalProject requires at least CMake 3.7") - endif() - externalproject_add(aws_ep ${EP_LOG_OPTIONS} CMAKE_ARGS @@ -1861,8 +1557,6 @@ macro(build_aws) BUILD_BYPRODUCTS "${AWS_CPP_SDK_S3_STATIC_LIB}" "${AWS_CPP_SDK_CORE_STATIC_LIB}") - - file(MAKE_DIRECTORY "${AWS_PREFIX}/include") add_library(aws-cpp-sdk-s3 STATIC IMPORTED) @@ -1885,8 +1579,7 @@ endmacro() if(MILVUS_WITH_AWS) resolve_dependency(AWS) - - # TODO: Don't use global includes but rather target_include_directories + link_directories(SYSTEM ${AWS_PREFIX}/lib) get_target_property(AWS_CPP_SDK_S3_INCLUDE_DIR aws-cpp-sdk-s3 INTERFACE_INCLUDE_DIRECTORIES) diff --git a/cpp/scripts/requirements.sh b/cpp/scripts/requirements.sh new file mode 100755 index 0000000000..5f8b74ad2c --- /dev/null +++ b/cpp/scripts/requirements.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env bash + +wget -P /tmp https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS-2019.PUB +apt-key add /tmp/GPG-PUB-KEY-INTEL-SW-PRODUCTS-2019.PUB + +sh -c 'echo deb https://apt.repos.intel.com/mkl all main > /etc/apt/sources.list.d/intel-mkl.list' +apt -y update && apt-get -y install intel-mkl-gnu-2019.4-243 intel-mkl-core-2019.4-243 + +#sh -c 'echo export LD_LIBRARY_PATH=/opt/intel/compilers_and_libraries_2019.4.243/linux/mkl/lib/intel64:\$LD_LIBRARY_PATH > /etc/profile.d/mkl.sh' +#source /etc/profile diff --git a/cpp/src/CMakeLists.txt b/cpp/src/CMakeLists.txt index 00e2e1e87d..5bea7818c9 100644 --- a/cpp/src/CMakeLists.txt +++ b/cpp/src/CMakeLists.txt @@ -48,7 +48,6 @@ set(engine_files ${db_files} ${db_scheduler_files} ${wrapper_files} -# metrics/Metrics.cpp ${metrics_files} ) @@ -71,8 +70,6 @@ set(third_party_libs yaml-cpp libgpufaiss.a faiss - lapack - openblas prometheus-cpp-push prometheus-cpp-pull prometheus-cpp-core @@ -87,9 +84,19 @@ set(third_party_libs mysqlpp ${CUDA_TOOLKIT_ROOT_DIR}/lib64/stubs/libnvidia-ml.so ) + if (MEGASEARCH_WITH_ARROW STREQUAL "ON") set(third_party_libs ${third_party_libs} arrow) -endif() +endif() +if(${BUILD_FAISS_WITH_MKL} STREQUAL "true") + set(third_party_libs ${third_party_libs} + ${MKL_LIBS} + ${MKL_LIBS}) +else() + set(third_party_libs ${third_party_libs} + lapack + openblas) +endif() if (GPU_VERSION STREQUAL "ON") link_directories("${CUDA_TOOLKIT_ROOT_DIR}/lib64") @@ -187,6 +194,6 @@ install(FILES ${CMAKE_BINARY_DIR}/mysqlpp_ep-prefix/src/mysqlpp_ep/lib/${CMAKE_SHARED_LIBRARY_PREFIX}mysqlpp${CMAKE_SHARED_LIBRARY_SUFFIX} ${CMAKE_BINARY_DIR}/mysqlpp_ep-prefix/src/mysqlpp_ep/lib/${CMAKE_SHARED_LIBRARY_PREFIX}mysqlpp${CMAKE_SHARED_LIBRARY_SUFFIX}.3 ${CMAKE_BINARY_DIR}/mysqlpp_ep-prefix/src/mysqlpp_ep/lib/${CMAKE_SHARED_LIBRARY_PREFIX}mysqlpp${CMAKE_SHARED_LIBRARY_SUFFIX}.3.2.4 - DESTINATION lib) #need to copy libmysqlpp.so + DESTINATION lib) add_subdirectory(sdk) diff --git a/cpp/unittest/CMakeLists.txt b/cpp/unittest/CMakeLists.txt index 62e32f6d1d..6a6f94a632 100644 --- a/cpp/unittest/CMakeLists.txt +++ b/cpp/unittest/CMakeLists.txt @@ -28,7 +28,6 @@ set(unittest_libs easyloggingpp pthread metrics - openblas gfortran prometheus-cpp-pull prometheus-cpp-push diff --git a/cpp/unittest/db/CMakeLists.txt b/cpp/unittest/db/CMakeLists.txt index c0f3052848..736219952c 100644 --- a/cpp/unittest/db/CMakeLists.txt +++ b/cpp/unittest/db/CMakeLists.txt @@ -23,10 +23,7 @@ link_directories("/usr/local/cuda/lib64") include_directories(/usr/include/mysql) -#add_definitions(-DBOOST_ERROR_CODE_HEADER_ONLY) - set(db_test_src - #${unittest_srcs} ${config_files} ${cache_srcs} ${db_srcs} @@ -49,6 +46,11 @@ set(db_libs mysqlpp ) +if(${BUILD_FAISS_WITH_MKL} STREQUAL "true") + set(db_libs ${db_libs} ${MKL_LIBS} ${MKL_LIBS}) +endif() + target_link_libraries(db_test ${db_libs} ${unittest_libs}) install(TARGETS db_test DESTINATION bin) + diff --git a/cpp/unittest/faiss_wrapper/CMakeLists.txt b/cpp/unittest/faiss_wrapper/CMakeLists.txt index ff2535e3fd..c439250544 100644 --- a/cpp/unittest/faiss_wrapper/CMakeLists.txt +++ b/cpp/unittest/faiss_wrapper/CMakeLists.txt @@ -35,6 +35,10 @@ set(wrapper_libs zstd lz4 ) +if(${BUILD_FAISS_WITH_MKL} STREQUAL "true") + set(wrapper_libs ${wrapper_libs} ${MKL_LIBS} ${MKL_LIBS}) +endif() + target_link_libraries(wrapper_test ${wrapper_libs} ${unittest_libs}) set(topk_test_src diff --git a/cpp/unittest/metrics/CMakeLists.txt b/cpp/unittest/metrics/CMakeLists.txt index 513b37b8c2..418544d0ca 100644 --- a/cpp/unittest/metrics/CMakeLists.txt +++ b/cpp/unittest/metrics/CMakeLists.txt @@ -10,8 +10,6 @@ include_directories(../../src) - - aux_source_directory(../../src/db db_srcs) aux_source_directory(../../src/config config_files) aux_source_directory(../../src/cache cache_srcs) @@ -33,21 +31,10 @@ include_directories(../../third_party/build/include) link_directories(../../third_party/build/lib) include_directories(/usr/local/cuda/include) link_directories("/usr/local/cuda/lib64") -#include_directories(../db/utils.h) include_directories(../../src/metrics) include_directories(/usr/include/mysql) -#set(metrics_src_files -# ../../src/metrics/Metrics.cpp -# ../../src/metrics/Metrics.h -# ../../src/metrics/PrometheusMetrics.cpp -# ../../src/metrics/MetricBase.h -# ../../src/server/ServerConfig.cpp -# ../../src/utils/CommonUtil.cpp -# ../../src/utils/TimeRecorder.cpp -# ) - set(count_test_src ${config_files} ${cache_srcs} @@ -77,5 +64,8 @@ target_link_libraries(metrics_test mysqlpp ${unittest_libs} ) +if(${BUILD_FAISS_WITH_MKL} STREQUAL "true") + target_link_libraries(metrics_test ${MKL_LIBS} ${MKL_LIBS}) +endif() install(TARGETS metrics_test DESTINATION bin) \ No newline at end of file diff --git a/cpp/unittest/server/CMakeLists.txt b/cpp/unittest/server/CMakeLists.txt index e49aed500c..40f5b10128 100644 --- a/cpp/unittest/server/CMakeLists.txt +++ b/cpp/unittest/server/CMakeLists.txt @@ -48,6 +48,10 @@ set(require_libs pthread ) +if(${BUILD_FAISS_WITH_MKL} STREQUAL "true") + set(require_libs ${require_libs} ${MKL_LIBS} ${MKL_LIBS}) +endif() + target_link_libraries(server_test ${require_libs} ${cuda_library} From ab45ad89a163fc28269075b88fc91752d7022aa3 Mon Sep 17 00:00:00 2001 From: zhiru Date: Wed, 17 Jul 2019 13:02:40 +0800 Subject: [PATCH 182/189] update Former-commit-id: aba27ece245cadb9aff24943c957de29c6af2a00 --- cpp/cmake/ThirdPartyPackages.cmake | 3 ++- cpp/thirdparty/versions.txt | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/cpp/cmake/ThirdPartyPackages.cmake b/cpp/cmake/ThirdPartyPackages.cmake index 1f7c84f7cc..aba99b3c1f 100644 --- a/cpp/cmake/ThirdPartyPackages.cmake +++ b/cpp/cmake/ThirdPartyPackages.cmake @@ -713,7 +713,8 @@ macro(build_faiss) if(${BUILD_FAISS_WITH_MKL} STREQUAL "true") message(STATUS "Build Faiss with MKL") if(NOT DEFINED MKL_LIB_PATH) - set(MKL_LIB_PATH "/opt/intel/compilers_and_libraries_2019.4.243/linux/mkl/lib/intel64") + set(MKL_LIB_PATH "/opt/intel/compilers_and_libraries_${MKL_VERSION}/linux/mkl/lib/intel64") + message(STATUS "MKL_LIB_PATH = ${MKL_LIB_PATH}") endif() set(FAISS_CONFIGURE_ARGS ${FAISS_CONFIGURE_ARGS} "CPPFLAGS=-DMKL_ILP64 -m64 -I${MKL_LIB_PATH}/../../include" diff --git a/cpp/thirdparty/versions.txt b/cpp/thirdparty/versions.txt index 311760948d..530e9feeec 100644 --- a/cpp/thirdparty/versions.txt +++ b/cpp/thirdparty/versions.txt @@ -3,6 +3,7 @@ BOOST_VERSION=1.70.0 BZIP2_VERSION=1.0.6 EASYLOGGINGPP_VERSION=v9.96.7 FAISS_VERSION=7b07685 +MKL_VERSION=2019.4.243 GTEST_VERSION=1.8.1 JSONCONS_VERSION=0.126.0 LAPACK_VERSION=v3.8.0 From eea064a30f9d88c63c95f75d7bcce16d55db6345 Mon Sep 17 00:00:00 2001 From: zhiru Date: Wed, 17 Jul 2019 13:04:30 +0800 Subject: [PATCH 183/189] update Former-commit-id: 1c05bc1ae8136ff21aec4c257531a1e9cdbf1fc3 --- cpp/cmake/ThirdPartyPackages.cmake | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/cpp/cmake/ThirdPartyPackages.cmake b/cpp/cmake/ThirdPartyPackages.cmake index aba99b3c1f..d08edceaf0 100644 --- a/cpp/cmake/ThirdPartyPackages.cmake +++ b/cpp/cmake/ThirdPartyPackages.cmake @@ -5,8 +5,7 @@ # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at -# - +# http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an From 07d070ca7366563b693765e8de870778e3bc3a87 Mon Sep 17 00:00:00 2001 From: zhiru Date: Wed, 17 Jul 2019 13:48:29 +0800 Subject: [PATCH 184/189] update Former-commit-id: 1c4f73d9eabbbc3bd085ea0e2cabadf54e4785dc --- cpp/README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/cpp/README.md b/cpp/README.md index 1b2f507db2..b46515a1cf 100644 --- a/cpp/README.md +++ b/cpp/README.md @@ -1,13 +1,13 @@ ### Compilation #### Step 1: install necessery tools - Install MySQL - centos7 : - yum install gfortran qt4 flex bison mysql-devel + yum install gfortran qt4 flex bison mysql-devel mysql ubuntu16.04 : - sudo apt-get install gfortran qt4-qmake flex bison libmysqlclient-dev + sudo apt-get install gfortran qt4-qmake flex bison libmysqlclient-dev mysql-client + + cd scripts && sudo ./requirements.sh If `libmysqlclient_r.so` does not exist after installing MySQL Development Files, you need to create a symbolic link: From 6f0755d88e1374ae2bfb0806521aff31f9bd63c6 Mon Sep 17 00:00:00 2001 From: zhiru Date: Wed, 17 Jul 2019 13:50:03 +0800 Subject: [PATCH 185/189] update Former-commit-id: 3e475d6f8a0ca2d869c83b83f2ed3614c8f06172 --- ci/main_jenkinsfile | 2 +- ci/main_jenkinsfile_no_ut | 2 +- ci/nightly_main_jenkinsfile | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/ci/main_jenkinsfile b/ci/main_jenkinsfile index 9577837bfd..d4132e9ff1 100644 --- a/ci/main_jenkinsfile +++ b/ci/main_jenkinsfile @@ -35,7 +35,7 @@ pipeline { defaultContainer 'jnlp' containerTemplate { name 'milvus-build-env' - image 'registry.zilliz.com/milvus/milvus-build-env:v0.10' + image 'registry.zilliz.com/milvus/milvus-build-env:v0.11' ttyEnabled true command 'cat' } diff --git a/ci/main_jenkinsfile_no_ut b/ci/main_jenkinsfile_no_ut index 082433650c..0b2f90fd63 100644 --- a/ci/main_jenkinsfile_no_ut +++ b/ci/main_jenkinsfile_no_ut @@ -35,7 +35,7 @@ pipeline { defaultContainer 'jnlp' containerTemplate { name 'milvus-build-env' - image 'registry.zilliz.com/milvus/milvus-build-env:v0.10' + image 'registry.zilliz.com/milvus/milvus-build-env:v0.11' ttyEnabled true command 'cat' } diff --git a/ci/nightly_main_jenkinsfile b/ci/nightly_main_jenkinsfile index 7f59561f82..5458cf7632 100644 --- a/ci/nightly_main_jenkinsfile +++ b/ci/nightly_main_jenkinsfile @@ -35,7 +35,7 @@ pipeline { defaultContainer 'jnlp' containerTemplate { name 'milvus-build-env' - image 'registry.zilliz.com/milvus/milvus-build-env:v0.10' + image 'registry.zilliz.com/milvus/milvus-build-env:v0.11' ttyEnabled true command 'cat' } From 476d21927b79b18ff1f01576151db17f72fbf877 Mon Sep 17 00:00:00 2001 From: starlord Date: Wed, 17 Jul 2019 17:37:42 +0800 Subject: [PATCH 186/189] MS-245 Improve search result transfer performance Former-commit-id: 77ce402c5ab2c851afa11cb2d637acb69d9ad737 --- cpp/CHANGELOG.md | 3 +- cpp/src/sdk/src/client/ClientProxy.cpp | 18 +- cpp/src/server/RequestHandler.cpp | 15 +- cpp/src/server/RequestHandler.h | 23 + cpp/src/server/RequestTask.cpp | 136 ++- cpp/src/server/RequestTask.h | 64 +- cpp/src/thrift/gen-cpp/MilvusService.cpp | 847 +++++++++++++++--- cpp/src/thrift/gen-cpp/MilvusService.h | 173 ++++ .../gen-cpp/MilvusService_server.skeleton.cpp | 22 + cpp/src/thrift/gen-cpp/milvus_types.cpp | 115 +++ cpp/src/thrift/gen-cpp/milvus_types.h | 43 + cpp/src/thrift/milvus.thrift | 25 + 12 files changed, 1284 insertions(+), 200 deletions(-) diff --git a/cpp/CHANGELOG.md b/cpp/CHANGELOG.md index d6aa72860d..fd0d2dfae9 100644 --- a/cpp/CHANGELOG.md +++ b/cpp/CHANGELOG.md @@ -29,7 +29,8 @@ Please mark all change in change log and use the ticket from JIRA. - MS-208 - Add buildinde interface for C++ SDK - MS-212 - Support Inner product metric type - MS-241 - Build Faiss with MKL if using Intel CPU; else build with OpenBlas -- MS-242 - clean up cmake and change MAKE_BUILD_ARGS to be user defined variable +- MS-242 - Clean up cmake and change MAKE_BUILD_ARGS to be user defined variable +- MS-245 - Improve search result transfer performance ## New Feature - MS-180 - Add new mem manager diff --git a/cpp/src/sdk/src/client/ClientProxy.cpp b/cpp/src/sdk/src/client/ClientProxy.cpp index 81dc8d28e4..115795c2f3 100644 --- a/cpp/src/sdk/src/client/ClientProxy.cpp +++ b/cpp/src/sdk/src/client/ClientProxy.cpp @@ -209,17 +209,25 @@ ClientProxy::SearchVector(const std::string &table_name, } //step 3: search vectors - std::vector result_array; - ClientPtr()->interface()->SearchVector(result_array, table_name, thrift_records, thrift_ranges, topk); + std::vector result_array; + ClientPtr()->interface()->SearchVector2(result_array, table_name, thrift_records, thrift_ranges, topk); //step 4: convert result array for(auto& thrift_topk_result : result_array) { TopKQueryResult result; - for(auto& thrift_query_result : thrift_topk_result.query_result_arrays) { + size_t id_count = thrift_topk_result.id_array.size()/sizeof(int64_t); + size_t dist_count = thrift_topk_result.distance_array.size()/ sizeof(double); + if(id_count != dist_count) { + return Status(StatusCode::UnknownError, "illegal result"); + } + + int64_t* id_ptr = (int64_t*)thrift_topk_result.id_array.data(); + double* dist_ptr = (double*)thrift_topk_result.distance_array.data(); + for(size_t i = 0; i < id_count; i++) { QueryResult query_result; - query_result.id = thrift_query_result.id; - query_result.distance = thrift_query_result.distance; + query_result.id = id_ptr[i]; + query_result.distance = dist_ptr[i]; result.query_result_arrays.emplace_back(query_result); } diff --git a/cpp/src/server/RequestHandler.cpp b/cpp/src/server/RequestHandler.cpp index 5ff6b1784a..5074043148 100644 --- a/cpp/src/server/RequestHandler.cpp +++ b/cpp/src/server/RequestHandler.cpp @@ -60,11 +60,22 @@ RequestHandler::SearchVector(std::vector &_return, const std::vector &query_range_array, const int64_t topk) { // SERVER_LOG_DEBUG << "Entering RequestHandler::SearchVector"; - BaseTaskPtr task_ptr = SearchVectorTask::Create(table_name, std::vector(), query_record_array, + BaseTaskPtr task_ptr = SearchVectorTask1::Create(table_name, std::vector(), query_record_array, query_range_array, topk, _return); RequestScheduler::ExecTask(task_ptr); } +void +RequestHandler::SearchVector2(std::vector & _return, + const std::string& table_name, + const std::vector & query_record_array, + const std::vector & query_range_array, + const int64_t topk) { + BaseTaskPtr task_ptr = SearchVectorTask2::Create(table_name, std::vector(), query_record_array, + query_range_array, topk, _return); + RequestScheduler::ExecTask(task_ptr); +} + void RequestHandler::SearchVectorInFiles(std::vector<::milvus::thrift::TopKQueryResult> &_return, const std::string& table_name, @@ -73,7 +84,7 @@ RequestHandler::SearchVectorInFiles(std::vector<::milvus::thrift::TopKQueryResul const std::vector<::milvus::thrift::Range> &query_range_array, const int64_t topk) { // SERVER_LOG_DEBUG << "Entering RequestHandler::SearchVectorInFiles. file_id_array size = " << std::to_string(file_id_array.size()); - BaseTaskPtr task_ptr = SearchVectorTask::Create(table_name, file_id_array, query_record_array, + BaseTaskPtr task_ptr = SearchVectorTask1::Create(table_name, file_id_array, query_record_array, query_range_array, topk, _return); RequestScheduler::ExecTask(task_ptr); } diff --git a/cpp/src/server/RequestHandler.h b/cpp/src/server/RequestHandler.h index aeda4e5ed6..a79830b512 100644 --- a/cpp/src/server/RequestHandler.h +++ b/cpp/src/server/RequestHandler.h @@ -106,6 +106,29 @@ public: const std::vector<::milvus::thrift::Range> & query_range_array, const int64_t topk); + /** + * @brief Query vector + * + * This method is used to query vector in table. + * + * @param table_name, table_name is queried. + * @param query_record_array, all vector are going to be queried. + * @param query_range_array, optional ranges for conditional search. If not specified, search whole table + * @param topk, how many similarity vectors will be searched. + * + * @return query binary result array. + * + * @param table_name + * @param query_record_array + * @param query_range_array + * @param topk + */ + void SearchVector2(std::vector<::milvus::thrift::TopKQueryBinResult> & _return, + const std::string& table_name, + const std::vector<::milvus::thrift::RowRecord> & query_record_array, + const std::vector<::milvus::thrift::Range> & query_range_array, + const int64_t topk); + /** * @brief Internal use query interface * diff --git a/cpp/src/server/RequestTask.cpp b/cpp/src/server/RequestTask.cpp index af6264da90..67a04610c7 100644 --- a/cpp/src/server/RequestTask.cpp +++ b/cpp/src/server/RequestTask.cpp @@ -466,33 +466,21 @@ ServerError AddVectorTask::OnExecute() { } //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -SearchVectorTask::SearchVectorTask(const std::string &table_name, - const std::vector& file_id_array, - const std::vector &query_record_array, - const std::vector &query_range_array, - const int64_t top_k, - std::vector &result_array) +SearchVectorTaskBase::SearchVectorTaskBase(const std::string &table_name, + const std::vector& file_id_array, + const std::vector &query_record_array, + const std::vector &query_range_array, + const int64_t top_k) : BaseTask(DQL_TASK_GROUP), table_name_(table_name), file_id_array_(file_id_array), record_array_(query_record_array), range_array_(query_range_array), - top_k_(top_k), - result_array_(result_array) { + top_k_(top_k) { } -BaseTaskPtr SearchVectorTask::Create(const std::string& table_name, - const std::vector& file_id_array, - const std::vector & query_record_array, - const std::vector & query_range_array, - const int64_t top_k, - std::vector& result_array) { - return std::shared_ptr(new SearchVectorTask(table_name, file_id_array, - query_record_array, query_range_array, top_k, result_array)); -} - -ServerError SearchVectorTask::OnExecute() { +ServerError SearchVectorTaskBase::OnExecute() { try { TimeRecorder rc("SearchVectorTask"); @@ -570,21 +558,7 @@ ServerError SearchVectorTask::OnExecute() { rc.Record("do search"); //step 5: construct result array - for(uint64_t i = 0; i < record_count; i++) { - auto& result = results[i]; - const auto& record = record_array_[i]; - - thrift::TopKQueryResult thrift_topk_result; - for(auto& pair : result) { - thrift::QueryResult thrift_result; - thrift_result.__set_id(pair.first); - thrift_result.__set_distance(pair.second); - - thrift_topk_result.query_result_arrays.emplace_back(thrift_result); - } - - result_array_.emplace_back(thrift_topk_result); - } + ConstructResult(results); rc.Record("construct result"); rc.Elapse("total cost"); @@ -595,6 +569,100 @@ ServerError SearchVectorTask::OnExecute() { return SERVER_SUCCESS; } +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +SearchVectorTask1::SearchVectorTask1(const std::string &table_name, + const std::vector& file_id_array, + const std::vector &query_record_array, + const std::vector &query_range_array, + const int64_t top_k, + std::vector &result_array) + : SearchVectorTaskBase(table_name, file_id_array, query_record_array, query_range_array, top_k), + result_array_(result_array) { + +} + +BaseTaskPtr SearchVectorTask1::Create(const std::string& table_name, + const std::vector& file_id_array, + const std::vector & query_record_array, + const std::vector & query_range_array, + const int64_t top_k, + std::vector& result_array) { + return std::shared_ptr(new SearchVectorTask1(table_name, file_id_array, + query_record_array, query_range_array, top_k, result_array)); +} + +ServerError SearchVectorTask1::ConstructResult(engine::QueryResults& results) { + for(uint64_t i = 0; i < results.size(); i++) { + auto& result = results[i]; + const auto& record = record_array_[i]; + + thrift::TopKQueryResult thrift_topk_result; + for(auto& pair : result) { + thrift::QueryResult thrift_result; + thrift_result.__set_id(pair.first); + thrift_result.__set_distance(pair.second); + + thrift_topk_result.query_result_arrays.emplace_back(thrift_result); + } + + result_array_.emplace_back(thrift_topk_result); + } + + return SERVER_SUCCESS; +} + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +SearchVectorTask2::SearchVectorTask2(const std::string &table_name, + const std::vector& file_id_array, + const std::vector &query_record_array, + const std::vector &query_range_array, + const int64_t top_k, + std::vector &result_array) + : SearchVectorTaskBase(table_name, file_id_array, query_record_array, query_range_array, top_k), + result_array_(result_array) { + +} + +BaseTaskPtr SearchVectorTask2::Create(const std::string& table_name, + const std::vector& file_id_array, + const std::vector & query_record_array, + const std::vector & query_range_array, + const int64_t top_k, + std::vector& result_array) { + return std::shared_ptr(new SearchVectorTask2(table_name, file_id_array, + query_record_array, query_range_array, top_k, result_array)); +} + +ServerError SearchVectorTask2::ConstructResult(engine::QueryResults& results) { + for(size_t i = 0; i < results.size(); i++) { + auto& result = results[i]; + + thrift::TopKQueryBinResult thrift_topk_result; + if(result.empty()) { + result_array_.emplace_back(thrift_topk_result); + continue; + } + + std::string str_ids, str_distances; + str_ids.resize(sizeof(engine::IDNumber)*result.size()); + str_distances.resize(sizeof(double)*result.size()); + + engine::IDNumber* ids_ptr = (engine::IDNumber*)str_ids.data(); + double* distance_ptr = (double*)str_distances.data(); + for(size_t k = 0; k < results.size(); k++) { + auto& pair = result[k]; + ids_ptr[k] = pair.first; + distance_ptr[k] = pair.second; + } + + thrift_topk_result.__set_id_array(str_ids); + thrift_topk_result.__set_distance_array(str_distances); + result_array_.emplace_back(thrift_topk_result); + } + + return SERVER_SUCCESS; +} + //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// GetTableRowCountTask::GetTableRowCountTask(const std::string& table_name, int64_t& row_count) : BaseTask(DDL_DML_TASK_GROUP), diff --git a/cpp/src/server/RequestTask.h b/cpp/src/server/RequestTask.h index 37c04272fa..4a95ef5b61 100644 --- a/cpp/src/server/RequestTask.h +++ b/cpp/src/server/RequestTask.h @@ -129,7 +129,28 @@ private: }; //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -class SearchVectorTask : public BaseTask { +class SearchVectorTaskBase : public BaseTask { +protected: + SearchVectorTaskBase(const std::string& table_name, + const std::vector& file_id_array, + const std::vector<::milvus::thrift::RowRecord> & query_record_array, + const std::vector<::milvus::thrift::Range> & query_range_array, + const int64_t top_k); + + ServerError OnExecute() override; + + virtual ServerError ConstructResult(engine::QueryResults& results) = 0; + +protected: + std::string table_name_; + std::vector file_id_array_; + int64_t top_k_; + const std::vector<::milvus::thrift::RowRecord>& record_array_; + const std::vector<::milvus::thrift::Range>& range_array_; +}; + +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +class SearchVectorTask1 : public SearchVectorTaskBase { public: static BaseTaskPtr Create(const std::string& table_name, const std::vector& file_id_array, @@ -139,24 +160,43 @@ public: std::vector<::milvus::thrift::TopKQueryResult>& result_array); protected: - SearchVectorTask(const std::string& table_name, - const std::vector& file_id_array, - const std::vector<::milvus::thrift::RowRecord> & query_record_array, - const std::vector<::milvus::thrift::Range> & query_range_array, - const int64_t top_k, + SearchVectorTask1(const std::string& table_name, + const std::vector& file_id_array, + const std::vector<::milvus::thrift::RowRecord> & query_record_array, + const std::vector<::milvus::thrift::Range> & query_range_array, + const int64_t top_k, std::vector<::milvus::thrift::TopKQueryResult>& result_array); - ServerError OnExecute() override; + ServerError ConstructResult(engine::QueryResults& results) override; private: - std::string table_name_; - std::vector file_id_array_; - int64_t top_k_; - const std::vector<::milvus::thrift::RowRecord>& record_array_; - const std::vector<::milvus::thrift::Range>& range_array_; std::vector<::milvus::thrift::TopKQueryResult>& result_array_; }; +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +class SearchVectorTask2 : public SearchVectorTaskBase { +public: + static BaseTaskPtr Create(const std::string& table_name, + const std::vector& file_id_array, + const std::vector<::milvus::thrift::RowRecord> & query_record_array, + const std::vector<::milvus::thrift::Range> & query_range_array, + const int64_t top_k, + std::vector<::milvus::thrift::TopKQueryBinResult>& result_array); + +protected: + SearchVectorTask2(const std::string& table_name, + const std::vector& file_id_array, + const std::vector<::milvus::thrift::RowRecord> & query_record_array, + const std::vector<::milvus::thrift::Range> & query_range_array, + const int64_t top_k, + std::vector<::milvus::thrift::TopKQueryBinResult>& result_array); + + ServerError ConstructResult(engine::QueryResults& results) override; + +private: + std::vector<::milvus::thrift::TopKQueryBinResult>& result_array_; +}; + //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// class GetTableRowCountTask : public BaseTask { public: diff --git a/cpp/src/thrift/gen-cpp/MilvusService.cpp b/cpp/src/thrift/gen-cpp/MilvusService.cpp index fb66b0ecbe..3420b92b5d 100644 --- a/cpp/src/thrift/gen-cpp/MilvusService.cpp +++ b/cpp/src/thrift/gen-cpp/MilvusService.cpp @@ -814,14 +814,14 @@ uint32_t MilvusService_AddVector_args::read(::apache::thrift::protocol::TProtoco if (ftype == ::apache::thrift::protocol::T_LIST) { { this->record_array.clear(); - uint32_t _size19; - ::apache::thrift::protocol::TType _etype22; - xfer += iprot->readListBegin(_etype22, _size19); - this->record_array.resize(_size19); - uint32_t _i23; - for (_i23 = 0; _i23 < _size19; ++_i23) + uint32_t _size21; + ::apache::thrift::protocol::TType _etype24; + xfer += iprot->readListBegin(_etype24, _size21); + this->record_array.resize(_size21); + uint32_t _i25; + for (_i25 = 0; _i25 < _size21; ++_i25) { - xfer += this->record_array[_i23].read(iprot); + xfer += this->record_array[_i25].read(iprot); } xfer += iprot->readListEnd(); } @@ -854,10 +854,10 @@ uint32_t MilvusService_AddVector_args::write(::apache::thrift::protocol::TProtoc xfer += oprot->writeFieldBegin("record_array", ::apache::thrift::protocol::T_LIST, 3); { xfer += oprot->writeListBegin(::apache::thrift::protocol::T_STRUCT, static_cast(this->record_array.size())); - std::vector ::const_iterator _iter24; - for (_iter24 = this->record_array.begin(); _iter24 != this->record_array.end(); ++_iter24) + std::vector ::const_iterator _iter26; + for (_iter26 = this->record_array.begin(); _iter26 != this->record_array.end(); ++_iter26) { - xfer += (*_iter24).write(oprot); + xfer += (*_iter26).write(oprot); } xfer += oprot->writeListEnd(); } @@ -885,10 +885,10 @@ uint32_t MilvusService_AddVector_pargs::write(::apache::thrift::protocol::TProto xfer += oprot->writeFieldBegin("record_array", ::apache::thrift::protocol::T_LIST, 3); { xfer += oprot->writeListBegin(::apache::thrift::protocol::T_STRUCT, static_cast((*(this->record_array)).size())); - std::vector ::const_iterator _iter25; - for (_iter25 = (*(this->record_array)).begin(); _iter25 != (*(this->record_array)).end(); ++_iter25) + std::vector ::const_iterator _iter27; + for (_iter27 = (*(this->record_array)).begin(); _iter27 != (*(this->record_array)).end(); ++_iter27) { - xfer += (*_iter25).write(oprot); + xfer += (*_iter27).write(oprot); } xfer += oprot->writeListEnd(); } @@ -929,14 +929,14 @@ uint32_t MilvusService_AddVector_result::read(::apache::thrift::protocol::TProto if (ftype == ::apache::thrift::protocol::T_LIST) { { this->success.clear(); - uint32_t _size26; - ::apache::thrift::protocol::TType _etype29; - xfer += iprot->readListBegin(_etype29, _size26); - this->success.resize(_size26); - uint32_t _i30; - for (_i30 = 0; _i30 < _size26; ++_i30) + uint32_t _size28; + ::apache::thrift::protocol::TType _etype31; + xfer += iprot->readListBegin(_etype31, _size28); + this->success.resize(_size28); + uint32_t _i32; + for (_i32 = 0; _i32 < _size28; ++_i32) { - xfer += iprot->readI64(this->success[_i30]); + xfer += iprot->readI64(this->success[_i32]); } xfer += iprot->readListEnd(); } @@ -975,10 +975,10 @@ uint32_t MilvusService_AddVector_result::write(::apache::thrift::protocol::TProt xfer += oprot->writeFieldBegin("success", ::apache::thrift::protocol::T_LIST, 0); { xfer += oprot->writeListBegin(::apache::thrift::protocol::T_I64, static_cast(this->success.size())); - std::vector ::const_iterator _iter31; - for (_iter31 = this->success.begin(); _iter31 != this->success.end(); ++_iter31) + std::vector ::const_iterator _iter33; + for (_iter33 = this->success.begin(); _iter33 != this->success.end(); ++_iter33) { - xfer += oprot->writeI64((*_iter31)); + xfer += oprot->writeI64((*_iter33)); } xfer += oprot->writeListEnd(); } @@ -1023,14 +1023,14 @@ uint32_t MilvusService_AddVector_presult::read(::apache::thrift::protocol::TProt if (ftype == ::apache::thrift::protocol::T_LIST) { { (*(this->success)).clear(); - uint32_t _size32; - ::apache::thrift::protocol::TType _etype35; - xfer += iprot->readListBegin(_etype35, _size32); - (*(this->success)).resize(_size32); - uint32_t _i36; - for (_i36 = 0; _i36 < _size32; ++_i36) + uint32_t _size34; + ::apache::thrift::protocol::TType _etype37; + xfer += iprot->readListBegin(_etype37, _size34); + (*(this->success)).resize(_size34); + uint32_t _i38; + for (_i38 = 0; _i38 < _size34; ++_i38) { - xfer += iprot->readI64((*(this->success))[_i36]); + xfer += iprot->readI64((*(this->success))[_i38]); } xfer += iprot->readListEnd(); } @@ -1097,14 +1097,14 @@ uint32_t MilvusService_SearchVector_args::read(::apache::thrift::protocol::TProt if (ftype == ::apache::thrift::protocol::T_LIST) { { this->query_record_array.clear(); - uint32_t _size37; - ::apache::thrift::protocol::TType _etype40; - xfer += iprot->readListBegin(_etype40, _size37); - this->query_record_array.resize(_size37); - uint32_t _i41; - for (_i41 = 0; _i41 < _size37; ++_i41) + uint32_t _size39; + ::apache::thrift::protocol::TType _etype42; + xfer += iprot->readListBegin(_etype42, _size39); + this->query_record_array.resize(_size39); + uint32_t _i43; + for (_i43 = 0; _i43 < _size39; ++_i43) { - xfer += this->query_record_array[_i41].read(iprot); + xfer += this->query_record_array[_i43].read(iprot); } xfer += iprot->readListEnd(); } @@ -1117,14 +1117,14 @@ uint32_t MilvusService_SearchVector_args::read(::apache::thrift::protocol::TProt if (ftype == ::apache::thrift::protocol::T_LIST) { { this->query_range_array.clear(); - uint32_t _size42; - ::apache::thrift::protocol::TType _etype45; - xfer += iprot->readListBegin(_etype45, _size42); - this->query_range_array.resize(_size42); - uint32_t _i46; - for (_i46 = 0; _i46 < _size42; ++_i46) + uint32_t _size44; + ::apache::thrift::protocol::TType _etype47; + xfer += iprot->readListBegin(_etype47, _size44); + this->query_range_array.resize(_size44); + uint32_t _i48; + for (_i48 = 0; _i48 < _size44; ++_i48) { - xfer += this->query_range_array[_i46].read(iprot); + xfer += this->query_range_array[_i48].read(iprot); } xfer += iprot->readListEnd(); } @@ -1165,10 +1165,10 @@ uint32_t MilvusService_SearchVector_args::write(::apache::thrift::protocol::TPro xfer += oprot->writeFieldBegin("query_record_array", ::apache::thrift::protocol::T_LIST, 3); { xfer += oprot->writeListBegin(::apache::thrift::protocol::T_STRUCT, static_cast(this->query_record_array.size())); - std::vector ::const_iterator _iter47; - for (_iter47 = this->query_record_array.begin(); _iter47 != this->query_record_array.end(); ++_iter47) + std::vector ::const_iterator _iter49; + for (_iter49 = this->query_record_array.begin(); _iter49 != this->query_record_array.end(); ++_iter49) { - xfer += (*_iter47).write(oprot); + xfer += (*_iter49).write(oprot); } xfer += oprot->writeListEnd(); } @@ -1177,10 +1177,10 @@ uint32_t MilvusService_SearchVector_args::write(::apache::thrift::protocol::TPro xfer += oprot->writeFieldBegin("query_range_array", ::apache::thrift::protocol::T_LIST, 4); { xfer += oprot->writeListBegin(::apache::thrift::protocol::T_STRUCT, static_cast(this->query_range_array.size())); - std::vector ::const_iterator _iter48; - for (_iter48 = this->query_range_array.begin(); _iter48 != this->query_range_array.end(); ++_iter48) + std::vector ::const_iterator _iter50; + for (_iter50 = this->query_range_array.begin(); _iter50 != this->query_range_array.end(); ++_iter50) { - xfer += (*_iter48).write(oprot); + xfer += (*_iter50).write(oprot); } xfer += oprot->writeListEnd(); } @@ -1212,10 +1212,10 @@ uint32_t MilvusService_SearchVector_pargs::write(::apache::thrift::protocol::TPr xfer += oprot->writeFieldBegin("query_record_array", ::apache::thrift::protocol::T_LIST, 3); { xfer += oprot->writeListBegin(::apache::thrift::protocol::T_STRUCT, static_cast((*(this->query_record_array)).size())); - std::vector ::const_iterator _iter49; - for (_iter49 = (*(this->query_record_array)).begin(); _iter49 != (*(this->query_record_array)).end(); ++_iter49) + std::vector ::const_iterator _iter51; + for (_iter51 = (*(this->query_record_array)).begin(); _iter51 != (*(this->query_record_array)).end(); ++_iter51) { - xfer += (*_iter49).write(oprot); + xfer += (*_iter51).write(oprot); } xfer += oprot->writeListEnd(); } @@ -1224,10 +1224,10 @@ uint32_t MilvusService_SearchVector_pargs::write(::apache::thrift::protocol::TPr xfer += oprot->writeFieldBegin("query_range_array", ::apache::thrift::protocol::T_LIST, 4); { xfer += oprot->writeListBegin(::apache::thrift::protocol::T_STRUCT, static_cast((*(this->query_range_array)).size())); - std::vector ::const_iterator _iter50; - for (_iter50 = (*(this->query_range_array)).begin(); _iter50 != (*(this->query_range_array)).end(); ++_iter50) + std::vector ::const_iterator _iter52; + for (_iter52 = (*(this->query_range_array)).begin(); _iter52 != (*(this->query_range_array)).end(); ++_iter52) { - xfer += (*_iter50).write(oprot); + xfer += (*_iter52).write(oprot); } xfer += oprot->writeListEnd(); } @@ -1272,14 +1272,14 @@ uint32_t MilvusService_SearchVector_result::read(::apache::thrift::protocol::TPr if (ftype == ::apache::thrift::protocol::T_LIST) { { this->success.clear(); - uint32_t _size51; - ::apache::thrift::protocol::TType _etype54; - xfer += iprot->readListBegin(_etype54, _size51); - this->success.resize(_size51); - uint32_t _i55; - for (_i55 = 0; _i55 < _size51; ++_i55) + uint32_t _size53; + ::apache::thrift::protocol::TType _etype56; + xfer += iprot->readListBegin(_etype56, _size53); + this->success.resize(_size53); + uint32_t _i57; + for (_i57 = 0; _i57 < _size53; ++_i57) { - xfer += this->success[_i55].read(iprot); + xfer += this->success[_i57].read(iprot); } xfer += iprot->readListEnd(); } @@ -1318,10 +1318,10 @@ uint32_t MilvusService_SearchVector_result::write(::apache::thrift::protocol::TP xfer += oprot->writeFieldBegin("success", ::apache::thrift::protocol::T_LIST, 0); { xfer += oprot->writeListBegin(::apache::thrift::protocol::T_STRUCT, static_cast(this->success.size())); - std::vector ::const_iterator _iter56; - for (_iter56 = this->success.begin(); _iter56 != this->success.end(); ++_iter56) + std::vector ::const_iterator _iter58; + for (_iter58 = this->success.begin(); _iter58 != this->success.end(); ++_iter58) { - xfer += (*_iter56).write(oprot); + xfer += (*_iter58).write(oprot); } xfer += oprot->writeListEnd(); } @@ -1366,14 +1366,357 @@ uint32_t MilvusService_SearchVector_presult::read(::apache::thrift::protocol::TP if (ftype == ::apache::thrift::protocol::T_LIST) { { (*(this->success)).clear(); - uint32_t _size57; - ::apache::thrift::protocol::TType _etype60; - xfer += iprot->readListBegin(_etype60, _size57); - (*(this->success)).resize(_size57); - uint32_t _i61; - for (_i61 = 0; _i61 < _size57; ++_i61) + uint32_t _size59; + ::apache::thrift::protocol::TType _etype62; + xfer += iprot->readListBegin(_etype62, _size59); + (*(this->success)).resize(_size59); + uint32_t _i63; + for (_i63 = 0; _i63 < _size59; ++_i63) { - xfer += (*(this->success))[_i61].read(iprot); + xfer += (*(this->success))[_i63].read(iprot); + } + xfer += iprot->readListEnd(); + } + this->__isset.success = true; + } else { + xfer += iprot->skip(ftype); + } + break; + case 1: + if (ftype == ::apache::thrift::protocol::T_STRUCT) { + xfer += this->e.read(iprot); + this->__isset.e = true; + } else { + xfer += iprot->skip(ftype); + } + break; + default: + xfer += iprot->skip(ftype); + break; + } + xfer += iprot->readFieldEnd(); + } + + xfer += iprot->readStructEnd(); + + return xfer; +} + + +MilvusService_SearchVector2_args::~MilvusService_SearchVector2_args() throw() { +} + + +uint32_t MilvusService_SearchVector2_args::read(::apache::thrift::protocol::TProtocol* iprot) { + + ::apache::thrift::protocol::TInputRecursionTracker tracker(*iprot); + uint32_t xfer = 0; + std::string fname; + ::apache::thrift::protocol::TType ftype; + int16_t fid; + + xfer += iprot->readStructBegin(fname); + + using ::apache::thrift::protocol::TProtocolException; + + + while (true) + { + xfer += iprot->readFieldBegin(fname, ftype, fid); + if (ftype == ::apache::thrift::protocol::T_STOP) { + break; + } + switch (fid) + { + case 2: + if (ftype == ::apache::thrift::protocol::T_STRING) { + xfer += iprot->readString(this->table_name); + this->__isset.table_name = true; + } else { + xfer += iprot->skip(ftype); + } + break; + case 3: + if (ftype == ::apache::thrift::protocol::T_LIST) { + { + this->query_record_array.clear(); + uint32_t _size64; + ::apache::thrift::protocol::TType _etype67; + xfer += iprot->readListBegin(_etype67, _size64); + this->query_record_array.resize(_size64); + uint32_t _i68; + for (_i68 = 0; _i68 < _size64; ++_i68) + { + xfer += this->query_record_array[_i68].read(iprot); + } + xfer += iprot->readListEnd(); + } + this->__isset.query_record_array = true; + } else { + xfer += iprot->skip(ftype); + } + break; + case 4: + if (ftype == ::apache::thrift::protocol::T_LIST) { + { + this->query_range_array.clear(); + uint32_t _size69; + ::apache::thrift::protocol::TType _etype72; + xfer += iprot->readListBegin(_etype72, _size69); + this->query_range_array.resize(_size69); + uint32_t _i73; + for (_i73 = 0; _i73 < _size69; ++_i73) + { + xfer += this->query_range_array[_i73].read(iprot); + } + xfer += iprot->readListEnd(); + } + this->__isset.query_range_array = true; + } else { + xfer += iprot->skip(ftype); + } + break; + case 5: + if (ftype == ::apache::thrift::protocol::T_I64) { + xfer += iprot->readI64(this->topk); + this->__isset.topk = true; + } else { + xfer += iprot->skip(ftype); + } + break; + default: + xfer += iprot->skip(ftype); + break; + } + xfer += iprot->readFieldEnd(); + } + + xfer += iprot->readStructEnd(); + + return xfer; +} + +uint32_t MilvusService_SearchVector2_args::write(::apache::thrift::protocol::TProtocol* oprot) const { + uint32_t xfer = 0; + ::apache::thrift::protocol::TOutputRecursionTracker tracker(*oprot); + xfer += oprot->writeStructBegin("MilvusService_SearchVector2_args"); + + xfer += oprot->writeFieldBegin("table_name", ::apache::thrift::protocol::T_STRING, 2); + xfer += oprot->writeString(this->table_name); + xfer += oprot->writeFieldEnd(); + + xfer += oprot->writeFieldBegin("query_record_array", ::apache::thrift::protocol::T_LIST, 3); + { + xfer += oprot->writeListBegin(::apache::thrift::protocol::T_STRUCT, static_cast(this->query_record_array.size())); + std::vector ::const_iterator _iter74; + for (_iter74 = this->query_record_array.begin(); _iter74 != this->query_record_array.end(); ++_iter74) + { + xfer += (*_iter74).write(oprot); + } + xfer += oprot->writeListEnd(); + } + xfer += oprot->writeFieldEnd(); + + xfer += oprot->writeFieldBegin("query_range_array", ::apache::thrift::protocol::T_LIST, 4); + { + xfer += oprot->writeListBegin(::apache::thrift::protocol::T_STRUCT, static_cast(this->query_range_array.size())); + std::vector ::const_iterator _iter75; + for (_iter75 = this->query_range_array.begin(); _iter75 != this->query_range_array.end(); ++_iter75) + { + xfer += (*_iter75).write(oprot); + } + xfer += oprot->writeListEnd(); + } + xfer += oprot->writeFieldEnd(); + + xfer += oprot->writeFieldBegin("topk", ::apache::thrift::protocol::T_I64, 5); + xfer += oprot->writeI64(this->topk); + xfer += oprot->writeFieldEnd(); + + xfer += oprot->writeFieldStop(); + xfer += oprot->writeStructEnd(); + return xfer; +} + + +MilvusService_SearchVector2_pargs::~MilvusService_SearchVector2_pargs() throw() { +} + + +uint32_t MilvusService_SearchVector2_pargs::write(::apache::thrift::protocol::TProtocol* oprot) const { + uint32_t xfer = 0; + ::apache::thrift::protocol::TOutputRecursionTracker tracker(*oprot); + xfer += oprot->writeStructBegin("MilvusService_SearchVector2_pargs"); + + xfer += oprot->writeFieldBegin("table_name", ::apache::thrift::protocol::T_STRING, 2); + xfer += oprot->writeString((*(this->table_name))); + xfer += oprot->writeFieldEnd(); + + xfer += oprot->writeFieldBegin("query_record_array", ::apache::thrift::protocol::T_LIST, 3); + { + xfer += oprot->writeListBegin(::apache::thrift::protocol::T_STRUCT, static_cast((*(this->query_record_array)).size())); + std::vector ::const_iterator _iter76; + for (_iter76 = (*(this->query_record_array)).begin(); _iter76 != (*(this->query_record_array)).end(); ++_iter76) + { + xfer += (*_iter76).write(oprot); + } + xfer += oprot->writeListEnd(); + } + xfer += oprot->writeFieldEnd(); + + xfer += oprot->writeFieldBegin("query_range_array", ::apache::thrift::protocol::T_LIST, 4); + { + xfer += oprot->writeListBegin(::apache::thrift::protocol::T_STRUCT, static_cast((*(this->query_range_array)).size())); + std::vector ::const_iterator _iter77; + for (_iter77 = (*(this->query_range_array)).begin(); _iter77 != (*(this->query_range_array)).end(); ++_iter77) + { + xfer += (*_iter77).write(oprot); + } + xfer += oprot->writeListEnd(); + } + xfer += oprot->writeFieldEnd(); + + xfer += oprot->writeFieldBegin("topk", ::apache::thrift::protocol::T_I64, 5); + xfer += oprot->writeI64((*(this->topk))); + xfer += oprot->writeFieldEnd(); + + xfer += oprot->writeFieldStop(); + xfer += oprot->writeStructEnd(); + return xfer; +} + + +MilvusService_SearchVector2_result::~MilvusService_SearchVector2_result() throw() { +} + + +uint32_t MilvusService_SearchVector2_result::read(::apache::thrift::protocol::TProtocol* iprot) { + + ::apache::thrift::protocol::TInputRecursionTracker tracker(*iprot); + uint32_t xfer = 0; + std::string fname; + ::apache::thrift::protocol::TType ftype; + int16_t fid; + + xfer += iprot->readStructBegin(fname); + + using ::apache::thrift::protocol::TProtocolException; + + + while (true) + { + xfer += iprot->readFieldBegin(fname, ftype, fid); + if (ftype == ::apache::thrift::protocol::T_STOP) { + break; + } + switch (fid) + { + case 0: + if (ftype == ::apache::thrift::protocol::T_LIST) { + { + this->success.clear(); + uint32_t _size78; + ::apache::thrift::protocol::TType _etype81; + xfer += iprot->readListBegin(_etype81, _size78); + this->success.resize(_size78); + uint32_t _i82; + for (_i82 = 0; _i82 < _size78; ++_i82) + { + xfer += this->success[_i82].read(iprot); + } + xfer += iprot->readListEnd(); + } + this->__isset.success = true; + } else { + xfer += iprot->skip(ftype); + } + break; + case 1: + if (ftype == ::apache::thrift::protocol::T_STRUCT) { + xfer += this->e.read(iprot); + this->__isset.e = true; + } else { + xfer += iprot->skip(ftype); + } + break; + default: + xfer += iprot->skip(ftype); + break; + } + xfer += iprot->readFieldEnd(); + } + + xfer += iprot->readStructEnd(); + + return xfer; +} + +uint32_t MilvusService_SearchVector2_result::write(::apache::thrift::protocol::TProtocol* oprot) const { + + uint32_t xfer = 0; + + xfer += oprot->writeStructBegin("MilvusService_SearchVector2_result"); + + if (this->__isset.success) { + xfer += oprot->writeFieldBegin("success", ::apache::thrift::protocol::T_LIST, 0); + { + xfer += oprot->writeListBegin(::apache::thrift::protocol::T_STRUCT, static_cast(this->success.size())); + std::vector ::const_iterator _iter83; + for (_iter83 = this->success.begin(); _iter83 != this->success.end(); ++_iter83) + { + xfer += (*_iter83).write(oprot); + } + xfer += oprot->writeListEnd(); + } + xfer += oprot->writeFieldEnd(); + } else if (this->__isset.e) { + xfer += oprot->writeFieldBegin("e", ::apache::thrift::protocol::T_STRUCT, 1); + xfer += this->e.write(oprot); + xfer += oprot->writeFieldEnd(); + } + xfer += oprot->writeFieldStop(); + xfer += oprot->writeStructEnd(); + return xfer; +} + + +MilvusService_SearchVector2_presult::~MilvusService_SearchVector2_presult() throw() { +} + + +uint32_t MilvusService_SearchVector2_presult::read(::apache::thrift::protocol::TProtocol* iprot) { + + ::apache::thrift::protocol::TInputRecursionTracker tracker(*iprot); + uint32_t xfer = 0; + std::string fname; + ::apache::thrift::protocol::TType ftype; + int16_t fid; + + xfer += iprot->readStructBegin(fname); + + using ::apache::thrift::protocol::TProtocolException; + + + while (true) + { + xfer += iprot->readFieldBegin(fname, ftype, fid); + if (ftype == ::apache::thrift::protocol::T_STOP) { + break; + } + switch (fid) + { + case 0: + if (ftype == ::apache::thrift::protocol::T_LIST) { + { + (*(this->success)).clear(); + uint32_t _size84; + ::apache::thrift::protocol::TType _etype87; + xfer += iprot->readListBegin(_etype87, _size84); + (*(this->success)).resize(_size84); + uint32_t _i88; + for (_i88 = 0; _i88 < _size84; ++_i88) + { + xfer += (*(this->success))[_i88].read(iprot); } xfer += iprot->readListEnd(); } @@ -1440,14 +1783,14 @@ uint32_t MilvusService_SearchVectorInFiles_args::read(::apache::thrift::protocol if (ftype == ::apache::thrift::protocol::T_LIST) { { this->file_id_array.clear(); - uint32_t _size62; - ::apache::thrift::protocol::TType _etype65; - xfer += iprot->readListBegin(_etype65, _size62); - this->file_id_array.resize(_size62); - uint32_t _i66; - for (_i66 = 0; _i66 < _size62; ++_i66) + uint32_t _size89; + ::apache::thrift::protocol::TType _etype92; + xfer += iprot->readListBegin(_etype92, _size89); + this->file_id_array.resize(_size89); + uint32_t _i93; + for (_i93 = 0; _i93 < _size89; ++_i93) { - xfer += iprot->readString(this->file_id_array[_i66]); + xfer += iprot->readString(this->file_id_array[_i93]); } xfer += iprot->readListEnd(); } @@ -1460,14 +1803,14 @@ uint32_t MilvusService_SearchVectorInFiles_args::read(::apache::thrift::protocol if (ftype == ::apache::thrift::protocol::T_LIST) { { this->query_record_array.clear(); - uint32_t _size67; - ::apache::thrift::protocol::TType _etype70; - xfer += iprot->readListBegin(_etype70, _size67); - this->query_record_array.resize(_size67); - uint32_t _i71; - for (_i71 = 0; _i71 < _size67; ++_i71) + uint32_t _size94; + ::apache::thrift::protocol::TType _etype97; + xfer += iprot->readListBegin(_etype97, _size94); + this->query_record_array.resize(_size94); + uint32_t _i98; + for (_i98 = 0; _i98 < _size94; ++_i98) { - xfer += this->query_record_array[_i71].read(iprot); + xfer += this->query_record_array[_i98].read(iprot); } xfer += iprot->readListEnd(); } @@ -1480,14 +1823,14 @@ uint32_t MilvusService_SearchVectorInFiles_args::read(::apache::thrift::protocol if (ftype == ::apache::thrift::protocol::T_LIST) { { this->query_range_array.clear(); - uint32_t _size72; - ::apache::thrift::protocol::TType _etype75; - xfer += iprot->readListBegin(_etype75, _size72); - this->query_range_array.resize(_size72); - uint32_t _i76; - for (_i76 = 0; _i76 < _size72; ++_i76) + uint32_t _size99; + ::apache::thrift::protocol::TType _etype102; + xfer += iprot->readListBegin(_etype102, _size99); + this->query_range_array.resize(_size99); + uint32_t _i103; + for (_i103 = 0; _i103 < _size99; ++_i103) { - xfer += this->query_range_array[_i76].read(iprot); + xfer += this->query_range_array[_i103].read(iprot); } xfer += iprot->readListEnd(); } @@ -1528,10 +1871,10 @@ uint32_t MilvusService_SearchVectorInFiles_args::write(::apache::thrift::protoco xfer += oprot->writeFieldBegin("file_id_array", ::apache::thrift::protocol::T_LIST, 3); { xfer += oprot->writeListBegin(::apache::thrift::protocol::T_STRING, static_cast(this->file_id_array.size())); - std::vector ::const_iterator _iter77; - for (_iter77 = this->file_id_array.begin(); _iter77 != this->file_id_array.end(); ++_iter77) + std::vector ::const_iterator _iter104; + for (_iter104 = this->file_id_array.begin(); _iter104 != this->file_id_array.end(); ++_iter104) { - xfer += oprot->writeString((*_iter77)); + xfer += oprot->writeString((*_iter104)); } xfer += oprot->writeListEnd(); } @@ -1540,10 +1883,10 @@ uint32_t MilvusService_SearchVectorInFiles_args::write(::apache::thrift::protoco xfer += oprot->writeFieldBegin("query_record_array", ::apache::thrift::protocol::T_LIST, 4); { xfer += oprot->writeListBegin(::apache::thrift::protocol::T_STRUCT, static_cast(this->query_record_array.size())); - std::vector ::const_iterator _iter78; - for (_iter78 = this->query_record_array.begin(); _iter78 != this->query_record_array.end(); ++_iter78) + std::vector ::const_iterator _iter105; + for (_iter105 = this->query_record_array.begin(); _iter105 != this->query_record_array.end(); ++_iter105) { - xfer += (*_iter78).write(oprot); + xfer += (*_iter105).write(oprot); } xfer += oprot->writeListEnd(); } @@ -1552,10 +1895,10 @@ uint32_t MilvusService_SearchVectorInFiles_args::write(::apache::thrift::protoco xfer += oprot->writeFieldBegin("query_range_array", ::apache::thrift::protocol::T_LIST, 5); { xfer += oprot->writeListBegin(::apache::thrift::protocol::T_STRUCT, static_cast(this->query_range_array.size())); - std::vector ::const_iterator _iter79; - for (_iter79 = this->query_range_array.begin(); _iter79 != this->query_range_array.end(); ++_iter79) + std::vector ::const_iterator _iter106; + for (_iter106 = this->query_range_array.begin(); _iter106 != this->query_range_array.end(); ++_iter106) { - xfer += (*_iter79).write(oprot); + xfer += (*_iter106).write(oprot); } xfer += oprot->writeListEnd(); } @@ -1587,10 +1930,10 @@ uint32_t MilvusService_SearchVectorInFiles_pargs::write(::apache::thrift::protoc xfer += oprot->writeFieldBegin("file_id_array", ::apache::thrift::protocol::T_LIST, 3); { xfer += oprot->writeListBegin(::apache::thrift::protocol::T_STRING, static_cast((*(this->file_id_array)).size())); - std::vector ::const_iterator _iter80; - for (_iter80 = (*(this->file_id_array)).begin(); _iter80 != (*(this->file_id_array)).end(); ++_iter80) + std::vector ::const_iterator _iter107; + for (_iter107 = (*(this->file_id_array)).begin(); _iter107 != (*(this->file_id_array)).end(); ++_iter107) { - xfer += oprot->writeString((*_iter80)); + xfer += oprot->writeString((*_iter107)); } xfer += oprot->writeListEnd(); } @@ -1599,10 +1942,10 @@ uint32_t MilvusService_SearchVectorInFiles_pargs::write(::apache::thrift::protoc xfer += oprot->writeFieldBegin("query_record_array", ::apache::thrift::protocol::T_LIST, 4); { xfer += oprot->writeListBegin(::apache::thrift::protocol::T_STRUCT, static_cast((*(this->query_record_array)).size())); - std::vector ::const_iterator _iter81; - for (_iter81 = (*(this->query_record_array)).begin(); _iter81 != (*(this->query_record_array)).end(); ++_iter81) + std::vector ::const_iterator _iter108; + for (_iter108 = (*(this->query_record_array)).begin(); _iter108 != (*(this->query_record_array)).end(); ++_iter108) { - xfer += (*_iter81).write(oprot); + xfer += (*_iter108).write(oprot); } xfer += oprot->writeListEnd(); } @@ -1611,10 +1954,10 @@ uint32_t MilvusService_SearchVectorInFiles_pargs::write(::apache::thrift::protoc xfer += oprot->writeFieldBegin("query_range_array", ::apache::thrift::protocol::T_LIST, 5); { xfer += oprot->writeListBegin(::apache::thrift::protocol::T_STRUCT, static_cast((*(this->query_range_array)).size())); - std::vector ::const_iterator _iter82; - for (_iter82 = (*(this->query_range_array)).begin(); _iter82 != (*(this->query_range_array)).end(); ++_iter82) + std::vector ::const_iterator _iter109; + for (_iter109 = (*(this->query_range_array)).begin(); _iter109 != (*(this->query_range_array)).end(); ++_iter109) { - xfer += (*_iter82).write(oprot); + xfer += (*_iter109).write(oprot); } xfer += oprot->writeListEnd(); } @@ -1659,14 +2002,14 @@ uint32_t MilvusService_SearchVectorInFiles_result::read(::apache::thrift::protoc if (ftype == ::apache::thrift::protocol::T_LIST) { { this->success.clear(); - uint32_t _size83; - ::apache::thrift::protocol::TType _etype86; - xfer += iprot->readListBegin(_etype86, _size83); - this->success.resize(_size83); - uint32_t _i87; - for (_i87 = 0; _i87 < _size83; ++_i87) + uint32_t _size110; + ::apache::thrift::protocol::TType _etype113; + xfer += iprot->readListBegin(_etype113, _size110); + this->success.resize(_size110); + uint32_t _i114; + for (_i114 = 0; _i114 < _size110; ++_i114) { - xfer += this->success[_i87].read(iprot); + xfer += this->success[_i114].read(iprot); } xfer += iprot->readListEnd(); } @@ -1705,10 +2048,10 @@ uint32_t MilvusService_SearchVectorInFiles_result::write(::apache::thrift::proto xfer += oprot->writeFieldBegin("success", ::apache::thrift::protocol::T_LIST, 0); { xfer += oprot->writeListBegin(::apache::thrift::protocol::T_STRUCT, static_cast(this->success.size())); - std::vector ::const_iterator _iter88; - for (_iter88 = this->success.begin(); _iter88 != this->success.end(); ++_iter88) + std::vector ::const_iterator _iter115; + for (_iter115 = this->success.begin(); _iter115 != this->success.end(); ++_iter115) { - xfer += (*_iter88).write(oprot); + xfer += (*_iter115).write(oprot); } xfer += oprot->writeListEnd(); } @@ -1753,14 +2096,14 @@ uint32_t MilvusService_SearchVectorInFiles_presult::read(::apache::thrift::proto if (ftype == ::apache::thrift::protocol::T_LIST) { { (*(this->success)).clear(); - uint32_t _size89; - ::apache::thrift::protocol::TType _etype92; - xfer += iprot->readListBegin(_etype92, _size89); - (*(this->success)).resize(_size89); - uint32_t _i93; - for (_i93 = 0; _i93 < _size89; ++_i93) + uint32_t _size116; + ::apache::thrift::protocol::TType _etype119; + xfer += iprot->readListBegin(_etype119, _size116); + (*(this->success)).resize(_size116); + uint32_t _i120; + for (_i120 = 0; _i120 < _size116; ++_i120) { - xfer += (*(this->success))[_i93].read(iprot); + xfer += (*(this->success))[_i120].read(iprot); } xfer += iprot->readListEnd(); } @@ -2291,14 +2634,14 @@ uint32_t MilvusService_ShowTables_result::read(::apache::thrift::protocol::TProt if (ftype == ::apache::thrift::protocol::T_LIST) { { this->success.clear(); - uint32_t _size94; - ::apache::thrift::protocol::TType _etype97; - xfer += iprot->readListBegin(_etype97, _size94); - this->success.resize(_size94); - uint32_t _i98; - for (_i98 = 0; _i98 < _size94; ++_i98) + uint32_t _size121; + ::apache::thrift::protocol::TType _etype124; + xfer += iprot->readListBegin(_etype124, _size121); + this->success.resize(_size121); + uint32_t _i125; + for (_i125 = 0; _i125 < _size121; ++_i125) { - xfer += iprot->readString(this->success[_i98]); + xfer += iprot->readString(this->success[_i125]); } xfer += iprot->readListEnd(); } @@ -2337,10 +2680,10 @@ uint32_t MilvusService_ShowTables_result::write(::apache::thrift::protocol::TPro xfer += oprot->writeFieldBegin("success", ::apache::thrift::protocol::T_LIST, 0); { xfer += oprot->writeListBegin(::apache::thrift::protocol::T_STRING, static_cast(this->success.size())); - std::vector ::const_iterator _iter99; - for (_iter99 = this->success.begin(); _iter99 != this->success.end(); ++_iter99) + std::vector ::const_iterator _iter126; + for (_iter126 = this->success.begin(); _iter126 != this->success.end(); ++_iter126) { - xfer += oprot->writeString((*_iter99)); + xfer += oprot->writeString((*_iter126)); } xfer += oprot->writeListEnd(); } @@ -2385,14 +2728,14 @@ uint32_t MilvusService_ShowTables_presult::read(::apache::thrift::protocol::TPro if (ftype == ::apache::thrift::protocol::T_LIST) { { (*(this->success)).clear(); - uint32_t _size100; - ::apache::thrift::protocol::TType _etype103; - xfer += iprot->readListBegin(_etype103, _size100); - (*(this->success)).resize(_size100); - uint32_t _i104; - for (_i104 = 0; _i104 < _size100; ++_i104) + uint32_t _size127; + ::apache::thrift::protocol::TType _etype130; + xfer += iprot->readListBegin(_etype130, _size127); + (*(this->success)).resize(_size127); + uint32_t _i131; + for (_i131 = 0; _i131 < _size127; ++_i131) { - xfer += iprot->readString((*(this->success))[_i104]); + xfer += iprot->readString((*(this->success))[_i131]); } xfer += iprot->readListEnd(); } @@ -2983,6 +3326,70 @@ void MilvusServiceClient::recv_SearchVector(std::vector & _retu throw ::apache::thrift::TApplicationException(::apache::thrift::TApplicationException::MISSING_RESULT, "SearchVector failed: unknown result"); } +void MilvusServiceClient::SearchVector2(std::vector & _return, const std::string& table_name, const std::vector & query_record_array, const std::vector & query_range_array, const int64_t topk) +{ + send_SearchVector2(table_name, query_record_array, query_range_array, topk); + recv_SearchVector2(_return); +} + +void MilvusServiceClient::send_SearchVector2(const std::string& table_name, const std::vector & query_record_array, const std::vector & query_range_array, const int64_t topk) +{ + int32_t cseqid = 0; + oprot_->writeMessageBegin("SearchVector2", ::apache::thrift::protocol::T_CALL, cseqid); + + MilvusService_SearchVector2_pargs args; + args.table_name = &table_name; + args.query_record_array = &query_record_array; + args.query_range_array = &query_range_array; + args.topk = &topk; + args.write(oprot_); + + oprot_->writeMessageEnd(); + oprot_->getTransport()->writeEnd(); + oprot_->getTransport()->flush(); +} + +void MilvusServiceClient::recv_SearchVector2(std::vector & _return) +{ + + int32_t rseqid = 0; + std::string fname; + ::apache::thrift::protocol::TMessageType mtype; + + iprot_->readMessageBegin(fname, mtype, rseqid); + if (mtype == ::apache::thrift::protocol::T_EXCEPTION) { + ::apache::thrift::TApplicationException x; + x.read(iprot_); + iprot_->readMessageEnd(); + iprot_->getTransport()->readEnd(); + throw x; + } + if (mtype != ::apache::thrift::protocol::T_REPLY) { + iprot_->skip(::apache::thrift::protocol::T_STRUCT); + iprot_->readMessageEnd(); + iprot_->getTransport()->readEnd(); + } + if (fname.compare("SearchVector2") != 0) { + iprot_->skip(::apache::thrift::protocol::T_STRUCT); + iprot_->readMessageEnd(); + iprot_->getTransport()->readEnd(); + } + MilvusService_SearchVector2_presult result; + result.success = &_return; + result.read(iprot_); + iprot_->readMessageEnd(); + iprot_->getTransport()->readEnd(); + + if (result.__isset.success) { + // _return pointer has now been filled + return; + } + if (result.__isset.e) { + throw result.e; + } + throw ::apache::thrift::TApplicationException(::apache::thrift::TApplicationException::MISSING_RESULT, "SearchVector2 failed: unknown result"); +} + void MilvusServiceClient::SearchVectorInFiles(std::vector & _return, const std::string& table_name, const std::vector & file_id_array, const std::vector & query_record_array, const std::vector & query_range_array, const int64_t topk) { send_SearchVectorInFiles(table_name, file_id_array, query_record_array, query_range_array, topk); @@ -3649,6 +4056,63 @@ void MilvusServiceProcessor::process_SearchVector(int32_t seqid, ::apache::thrif } } +void MilvusServiceProcessor::process_SearchVector2(int32_t seqid, ::apache::thrift::protocol::TProtocol* iprot, ::apache::thrift::protocol::TProtocol* oprot, void* callContext) +{ + void* ctx = NULL; + if (this->eventHandler_.get() != NULL) { + ctx = this->eventHandler_->getContext("MilvusService.SearchVector2", callContext); + } + ::apache::thrift::TProcessorContextFreer freer(this->eventHandler_.get(), ctx, "MilvusService.SearchVector2"); + + if (this->eventHandler_.get() != NULL) { + this->eventHandler_->preRead(ctx, "MilvusService.SearchVector2"); + } + + MilvusService_SearchVector2_args args; + args.read(iprot); + iprot->readMessageEnd(); + uint32_t bytes = iprot->getTransport()->readEnd(); + + if (this->eventHandler_.get() != NULL) { + this->eventHandler_->postRead(ctx, "MilvusService.SearchVector2", bytes); + } + + MilvusService_SearchVector2_result result; + try { + iface_->SearchVector2(result.success, args.table_name, args.query_record_array, args.query_range_array, args.topk); + result.__isset.success = true; + } catch (Exception &e) { + result.e = e; + result.__isset.e = true; + } catch (const std::exception& e) { + if (this->eventHandler_.get() != NULL) { + this->eventHandler_->handlerError(ctx, "MilvusService.SearchVector2"); + } + + ::apache::thrift::TApplicationException x(e.what()); + oprot->writeMessageBegin("SearchVector2", ::apache::thrift::protocol::T_EXCEPTION, seqid); + x.write(oprot); + oprot->writeMessageEnd(); + oprot->getTransport()->writeEnd(); + oprot->getTransport()->flush(); + return; + } + + if (this->eventHandler_.get() != NULL) { + this->eventHandler_->preWrite(ctx, "MilvusService.SearchVector2"); + } + + oprot->writeMessageBegin("SearchVector2", ::apache::thrift::protocol::T_REPLY, seqid); + result.write(oprot); + oprot->writeMessageEnd(); + bytes = oprot->getTransport()->writeEnd(); + oprot->getTransport()->flush(); + + if (this->eventHandler_.get() != NULL) { + this->eventHandler_->postWrite(ctx, "MilvusService.SearchVector2", bytes); + } +} + void MilvusServiceProcessor::process_SearchVectorInFiles(int32_t seqid, ::apache::thrift::protocol::TProtocol* iprot, ::apache::thrift::protocol::TProtocol* oprot, void* callContext) { void* ctx = NULL; @@ -4455,6 +4919,97 @@ void MilvusServiceConcurrentClient::recv_SearchVector(std::vector & _return, const std::string& table_name, const std::vector & query_record_array, const std::vector & query_range_array, const int64_t topk) +{ + int32_t seqid = send_SearchVector2(table_name, query_record_array, query_range_array, topk); + recv_SearchVector2(_return, seqid); +} + +int32_t MilvusServiceConcurrentClient::send_SearchVector2(const std::string& table_name, const std::vector & query_record_array, const std::vector & query_range_array, const int64_t topk) +{ + int32_t cseqid = this->sync_.generateSeqId(); + ::apache::thrift::async::TConcurrentSendSentry sentry(&this->sync_); + oprot_->writeMessageBegin("SearchVector2", ::apache::thrift::protocol::T_CALL, cseqid); + + MilvusService_SearchVector2_pargs args; + args.table_name = &table_name; + args.query_record_array = &query_record_array; + args.query_range_array = &query_range_array; + args.topk = &topk; + args.write(oprot_); + + oprot_->writeMessageEnd(); + oprot_->getTransport()->writeEnd(); + oprot_->getTransport()->flush(); + + sentry.commit(); + return cseqid; +} + +void MilvusServiceConcurrentClient::recv_SearchVector2(std::vector & _return, const int32_t seqid) +{ + + int32_t rseqid = 0; + std::string fname; + ::apache::thrift::protocol::TMessageType mtype; + + // the read mutex gets dropped and reacquired as part of waitForWork() + // The destructor of this sentry wakes up other clients + ::apache::thrift::async::TConcurrentRecvSentry sentry(&this->sync_, seqid); + + while(true) { + if(!this->sync_.getPending(fname, mtype, rseqid)) { + iprot_->readMessageBegin(fname, mtype, rseqid); + } + if(seqid == rseqid) { + if (mtype == ::apache::thrift::protocol::T_EXCEPTION) { + ::apache::thrift::TApplicationException x; + x.read(iprot_); + iprot_->readMessageEnd(); + iprot_->getTransport()->readEnd(); + sentry.commit(); + throw x; + } + if (mtype != ::apache::thrift::protocol::T_REPLY) { + iprot_->skip(::apache::thrift::protocol::T_STRUCT); + iprot_->readMessageEnd(); + iprot_->getTransport()->readEnd(); + } + if (fname.compare("SearchVector2") != 0) { + iprot_->skip(::apache::thrift::protocol::T_STRUCT); + iprot_->readMessageEnd(); + iprot_->getTransport()->readEnd(); + + // in a bad state, don't commit + using ::apache::thrift::protocol::TProtocolException; + throw TProtocolException(TProtocolException::INVALID_DATA); + } + MilvusService_SearchVector2_presult result; + result.success = &_return; + result.read(iprot_); + iprot_->readMessageEnd(); + iprot_->getTransport()->readEnd(); + + if (result.__isset.success) { + // _return pointer has now been filled + sentry.commit(); + return; + } + if (result.__isset.e) { + sentry.commit(); + throw result.e; + } + // in a bad state, don't commit + throw ::apache::thrift::TApplicationException(::apache::thrift::TApplicationException::MISSING_RESULT, "SearchVector2 failed: unknown result"); + } + // seqid != rseqid + this->sync_.updatePending(fname, mtype, rseqid); + + // this will temporarily unlock the readMutex, and let other clients get work done + this->sync_.waitForWork(seqid); + } // end while(true) +} + void MilvusServiceConcurrentClient::SearchVectorInFiles(std::vector & _return, const std::string& table_name, const std::vector & file_id_array, const std::vector & query_record_array, const std::vector & query_range_array, const int64_t topk) { int32_t seqid = send_SearchVectorInFiles(table_name, file_id_array, query_record_array, query_range_array, topk); diff --git a/cpp/src/thrift/gen-cpp/MilvusService.h b/cpp/src/thrift/gen-cpp/MilvusService.h index 5b4204e6b8..c18a33009a 100644 --- a/cpp/src/thrift/gen-cpp/MilvusService.h +++ b/cpp/src/thrift/gen-cpp/MilvusService.h @@ -104,6 +104,25 @@ class MilvusServiceIf { */ virtual void SearchVector(std::vector & _return, const std::string& table_name, const std::vector & query_record_array, const std::vector & query_range_array, const int64_t topk) = 0; + /** + * @brief Query vector + * + * This method is used to query vector in table. + * + * @param table_name, table_name is queried. + * @param query_record_array, all vector are going to be queried. + * @param query_range_array, optional ranges for conditional search. If not specified, search whole table + * @param topk, how many similarity vectors will be searched. + * + * @return query binary result array. + * + * @param table_name + * @param query_record_array + * @param query_range_array + * @param topk + */ + virtual void SearchVector2(std::vector & _return, const std::string& table_name, const std::vector & query_record_array, const std::vector & query_range_array, const int64_t topk) = 0; + /** * @brief Internal use query interface * @@ -218,6 +237,9 @@ class MilvusServiceNull : virtual public MilvusServiceIf { void SearchVector(std::vector & /* _return */, const std::string& /* table_name */, const std::vector & /* query_record_array */, const std::vector & /* query_range_array */, const int64_t /* topk */) { return; } + void SearchVector2(std::vector & /* _return */, const std::string& /* table_name */, const std::vector & /* query_record_array */, const std::vector & /* query_range_array */, const int64_t /* topk */) { + return; + } void SearchVectorInFiles(std::vector & /* _return */, const std::string& /* table_name */, const std::vector & /* file_id_array */, const std::vector & /* query_record_array */, const std::vector & /* query_range_array */, const int64_t /* topk */) { return; } @@ -912,6 +934,139 @@ class MilvusService_SearchVector_presult { }; +typedef struct _MilvusService_SearchVector2_args__isset { + _MilvusService_SearchVector2_args__isset() : table_name(false), query_record_array(false), query_range_array(false), topk(false) {} + bool table_name :1; + bool query_record_array :1; + bool query_range_array :1; + bool topk :1; +} _MilvusService_SearchVector2_args__isset; + +class MilvusService_SearchVector2_args { + public: + + MilvusService_SearchVector2_args(const MilvusService_SearchVector2_args&); + MilvusService_SearchVector2_args& operator=(const MilvusService_SearchVector2_args&); + MilvusService_SearchVector2_args() : table_name(), topk(0) { + } + + virtual ~MilvusService_SearchVector2_args() throw(); + std::string table_name; + std::vector query_record_array; + std::vector query_range_array; + int64_t topk; + + _MilvusService_SearchVector2_args__isset __isset; + + void __set_table_name(const std::string& val); + + void __set_query_record_array(const std::vector & val); + + void __set_query_range_array(const std::vector & val); + + void __set_topk(const int64_t val); + + bool operator == (const MilvusService_SearchVector2_args & rhs) const + { + if (!(table_name == rhs.table_name)) + return false; + if (!(query_record_array == rhs.query_record_array)) + return false; + if (!(query_range_array == rhs.query_range_array)) + return false; + if (!(topk == rhs.topk)) + return false; + return true; + } + bool operator != (const MilvusService_SearchVector2_args &rhs) const { + return !(*this == rhs); + } + + bool operator < (const MilvusService_SearchVector2_args & ) const; + + uint32_t read(::apache::thrift::protocol::TProtocol* iprot); + uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const; + +}; + + +class MilvusService_SearchVector2_pargs { + public: + + + virtual ~MilvusService_SearchVector2_pargs() throw(); + const std::string* table_name; + const std::vector * query_record_array; + const std::vector * query_range_array; + const int64_t* topk; + + uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const; + +}; + +typedef struct _MilvusService_SearchVector2_result__isset { + _MilvusService_SearchVector2_result__isset() : success(false), e(false) {} + bool success :1; + bool e :1; +} _MilvusService_SearchVector2_result__isset; + +class MilvusService_SearchVector2_result { + public: + + MilvusService_SearchVector2_result(const MilvusService_SearchVector2_result&); + MilvusService_SearchVector2_result& operator=(const MilvusService_SearchVector2_result&); + MilvusService_SearchVector2_result() { + } + + virtual ~MilvusService_SearchVector2_result() throw(); + std::vector success; + Exception e; + + _MilvusService_SearchVector2_result__isset __isset; + + void __set_success(const std::vector & val); + + void __set_e(const Exception& val); + + bool operator == (const MilvusService_SearchVector2_result & rhs) const + { + if (!(success == rhs.success)) + return false; + if (!(e == rhs.e)) + return false; + return true; + } + bool operator != (const MilvusService_SearchVector2_result &rhs) const { + return !(*this == rhs); + } + + bool operator < (const MilvusService_SearchVector2_result & ) const; + + uint32_t read(::apache::thrift::protocol::TProtocol* iprot); + uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const; + +}; + +typedef struct _MilvusService_SearchVector2_presult__isset { + _MilvusService_SearchVector2_presult__isset() : success(false), e(false) {} + bool success :1; + bool e :1; +} _MilvusService_SearchVector2_presult__isset; + +class MilvusService_SearchVector2_presult { + public: + + + virtual ~MilvusService_SearchVector2_presult() throw(); + std::vector * success; + Exception e; + + _MilvusService_SearchVector2_presult__isset __isset; + + uint32_t read(::apache::thrift::protocol::TProtocol* iprot); + +}; + typedef struct _MilvusService_SearchVectorInFiles_args__isset { _MilvusService_SearchVectorInFiles_args__isset() : table_name(false), file_id_array(false), query_record_array(false), query_range_array(false), topk(false) {} bool table_name :1; @@ -1531,6 +1686,9 @@ class MilvusServiceClient : virtual public MilvusServiceIf { void SearchVector(std::vector & _return, const std::string& table_name, const std::vector & query_record_array, const std::vector & query_range_array, const int64_t topk); void send_SearchVector(const std::string& table_name, const std::vector & query_record_array, const std::vector & query_range_array, const int64_t topk); void recv_SearchVector(std::vector & _return); + void SearchVector2(std::vector & _return, const std::string& table_name, const std::vector & query_record_array, const std::vector & query_range_array, const int64_t topk); + void send_SearchVector2(const std::string& table_name, const std::vector & query_record_array, const std::vector & query_range_array, const int64_t topk); + void recv_SearchVector2(std::vector & _return); void SearchVectorInFiles(std::vector & _return, const std::string& table_name, const std::vector & file_id_array, const std::vector & query_record_array, const std::vector & query_range_array, const int64_t topk); void send_SearchVectorInFiles(const std::string& table_name, const std::vector & file_id_array, const std::vector & query_record_array, const std::vector & query_range_array, const int64_t topk); void recv_SearchVectorInFiles(std::vector & _return); @@ -1567,6 +1725,7 @@ class MilvusServiceProcessor : public ::apache::thrift::TDispatchProcessor { void process_BuildIndex(int32_t seqid, ::apache::thrift::protocol::TProtocol* iprot, ::apache::thrift::protocol::TProtocol* oprot, void* callContext); void process_AddVector(int32_t seqid, ::apache::thrift::protocol::TProtocol* iprot, ::apache::thrift::protocol::TProtocol* oprot, void* callContext); void process_SearchVector(int32_t seqid, ::apache::thrift::protocol::TProtocol* iprot, ::apache::thrift::protocol::TProtocol* oprot, void* callContext); + void process_SearchVector2(int32_t seqid, ::apache::thrift::protocol::TProtocol* iprot, ::apache::thrift::protocol::TProtocol* oprot, void* callContext); void process_SearchVectorInFiles(int32_t seqid, ::apache::thrift::protocol::TProtocol* iprot, ::apache::thrift::protocol::TProtocol* oprot, void* callContext); void process_DescribeTable(int32_t seqid, ::apache::thrift::protocol::TProtocol* iprot, ::apache::thrift::protocol::TProtocol* oprot, void* callContext); void process_GetTableRowCount(int32_t seqid, ::apache::thrift::protocol::TProtocol* iprot, ::apache::thrift::protocol::TProtocol* oprot, void* callContext); @@ -1581,6 +1740,7 @@ class MilvusServiceProcessor : public ::apache::thrift::TDispatchProcessor { processMap_["BuildIndex"] = &MilvusServiceProcessor::process_BuildIndex; processMap_["AddVector"] = &MilvusServiceProcessor::process_AddVector; processMap_["SearchVector"] = &MilvusServiceProcessor::process_SearchVector; + processMap_["SearchVector2"] = &MilvusServiceProcessor::process_SearchVector2; processMap_["SearchVectorInFiles"] = &MilvusServiceProcessor::process_SearchVectorInFiles; processMap_["DescribeTable"] = &MilvusServiceProcessor::process_DescribeTable; processMap_["GetTableRowCount"] = &MilvusServiceProcessor::process_GetTableRowCount; @@ -1670,6 +1830,16 @@ class MilvusServiceMultiface : virtual public MilvusServiceIf { return; } + void SearchVector2(std::vector & _return, const std::string& table_name, const std::vector & query_record_array, const std::vector & query_range_array, const int64_t topk) { + size_t sz = ifaces_.size(); + size_t i = 0; + for (; i < (sz - 1); ++i) { + ifaces_[i]->SearchVector2(_return, table_name, query_record_array, query_range_array, topk); + } + ifaces_[i]->SearchVector2(_return, table_name, query_record_array, query_range_array, topk); + return; + } + void SearchVectorInFiles(std::vector & _return, const std::string& table_name, const std::vector & file_id_array, const std::vector & query_record_array, const std::vector & query_range_array, const int64_t topk) { size_t sz = ifaces_.size(); size_t i = 0; @@ -1767,6 +1937,9 @@ class MilvusServiceConcurrentClient : virtual public MilvusServiceIf { void SearchVector(std::vector & _return, const std::string& table_name, const std::vector & query_record_array, const std::vector & query_range_array, const int64_t topk); int32_t send_SearchVector(const std::string& table_name, const std::vector & query_record_array, const std::vector & query_range_array, const int64_t topk); void recv_SearchVector(std::vector & _return, const int32_t seqid); + void SearchVector2(std::vector & _return, const std::string& table_name, const std::vector & query_record_array, const std::vector & query_range_array, const int64_t topk); + int32_t send_SearchVector2(const std::string& table_name, const std::vector & query_record_array, const std::vector & query_range_array, const int64_t topk); + void recv_SearchVector2(std::vector & _return, const int32_t seqid); void SearchVectorInFiles(std::vector & _return, const std::string& table_name, const std::vector & file_id_array, const std::vector & query_record_array, const std::vector & query_range_array, const int64_t topk); int32_t send_SearchVectorInFiles(const std::string& table_name, const std::vector & file_id_array, const std::vector & query_record_array, const std::vector & query_range_array, const int64_t topk); void recv_SearchVectorInFiles(std::vector & _return, const int32_t seqid); diff --git a/cpp/src/thrift/gen-cpp/MilvusService_server.skeleton.cpp b/cpp/src/thrift/gen-cpp/MilvusService_server.skeleton.cpp index f22cfc2894..efa8e95034 100644 --- a/cpp/src/thrift/gen-cpp/MilvusService_server.skeleton.cpp +++ b/cpp/src/thrift/gen-cpp/MilvusService_server.skeleton.cpp @@ -120,6 +120,28 @@ class MilvusServiceHandler : virtual public MilvusServiceIf { printf("SearchVector\n"); } + /** + * @brief Query vector + * + * This method is used to query vector in table. + * + * @param table_name, table_name is queried. + * @param query_record_array, all vector are going to be queried. + * @param query_range_array, optional ranges for conditional search. If not specified, search whole table + * @param topk, how many similarity vectors will be searched. + * + * @return query binary result array. + * + * @param table_name + * @param query_record_array + * @param query_range_array + * @param topk + */ + void SearchVector2(std::vector & _return, const std::string& table_name, const std::vector & query_record_array, const std::vector & query_range_array, const int64_t topk) { + // Your implementation goes here + printf("SearchVector2\n"); + } + /** * @brief Internal use query interface * diff --git a/cpp/src/thrift/gen-cpp/milvus_types.cpp b/cpp/src/thrift/gen-cpp/milvus_types.cpp index 06923c3a1a..a1ef24de52 100644 --- a/cpp/src/thrift/gen-cpp/milvus_types.cpp +++ b/cpp/src/thrift/gen-cpp/milvus_types.cpp @@ -781,4 +781,119 @@ void TopKQueryResult::printTo(std::ostream& out) const { out << ")"; } + +TopKQueryBinResult::~TopKQueryBinResult() throw() { +} + + +void TopKQueryBinResult::__set_id_array(const std::string& val) { + this->id_array = val; +} + +void TopKQueryBinResult::__set_distance_array(const std::string& val) { + this->distance_array = val; +} +std::ostream& operator<<(std::ostream& out, const TopKQueryBinResult& obj) +{ + obj.printTo(out); + return out; +} + + +uint32_t TopKQueryBinResult::read(::apache::thrift::protocol::TProtocol* iprot) { + + ::apache::thrift::protocol::TInputRecursionTracker tracker(*iprot); + uint32_t xfer = 0; + std::string fname; + ::apache::thrift::protocol::TType ftype; + int16_t fid; + + xfer += iprot->readStructBegin(fname); + + using ::apache::thrift::protocol::TProtocolException; + + bool isset_id_array = false; + bool isset_distance_array = false; + + while (true) + { + xfer += iprot->readFieldBegin(fname, ftype, fid); + if (ftype == ::apache::thrift::protocol::T_STOP) { + break; + } + switch (fid) + { + case 1: + if (ftype == ::apache::thrift::protocol::T_STRING) { + xfer += iprot->readBinary(this->id_array); + isset_id_array = true; + } else { + xfer += iprot->skip(ftype); + } + break; + case 2: + if (ftype == ::apache::thrift::protocol::T_STRING) { + xfer += iprot->readBinary(this->distance_array); + isset_distance_array = true; + } else { + xfer += iprot->skip(ftype); + } + break; + default: + xfer += iprot->skip(ftype); + break; + } + xfer += iprot->readFieldEnd(); + } + + xfer += iprot->readStructEnd(); + + if (!isset_id_array) + throw TProtocolException(TProtocolException::INVALID_DATA); + if (!isset_distance_array) + throw TProtocolException(TProtocolException::INVALID_DATA); + return xfer; +} + +uint32_t TopKQueryBinResult::write(::apache::thrift::protocol::TProtocol* oprot) const { + uint32_t xfer = 0; + ::apache::thrift::protocol::TOutputRecursionTracker tracker(*oprot); + xfer += oprot->writeStructBegin("TopKQueryBinResult"); + + xfer += oprot->writeFieldBegin("id_array", ::apache::thrift::protocol::T_STRING, 1); + xfer += oprot->writeBinary(this->id_array); + xfer += oprot->writeFieldEnd(); + + xfer += oprot->writeFieldBegin("distance_array", ::apache::thrift::protocol::T_STRING, 2); + xfer += oprot->writeBinary(this->distance_array); + xfer += oprot->writeFieldEnd(); + + xfer += oprot->writeFieldStop(); + xfer += oprot->writeStructEnd(); + return xfer; +} + +void swap(TopKQueryBinResult &a, TopKQueryBinResult &b) { + using ::std::swap; + swap(a.id_array, b.id_array); + swap(a.distance_array, b.distance_array); +} + +TopKQueryBinResult::TopKQueryBinResult(const TopKQueryBinResult& other19) { + id_array = other19.id_array; + distance_array = other19.distance_array; +} +TopKQueryBinResult& TopKQueryBinResult::operator=(const TopKQueryBinResult& other20) { + id_array = other20.id_array; + distance_array = other20.distance_array; + return *this; +} +void TopKQueryBinResult::printTo(std::ostream& out) const { + using ::apache::thrift::to_string; + out << "TopKQueryBinResult("; + out << "id_array=" << to_string(id_array); + out << ", " << "distance_array=" << to_string(distance_array); + out << ")"; +} + }} // namespace diff --git a/cpp/src/thrift/gen-cpp/milvus_types.h b/cpp/src/thrift/gen-cpp/milvus_types.h index 63e313d197..a75d883074 100644 --- a/cpp/src/thrift/gen-cpp/milvus_types.h +++ b/cpp/src/thrift/gen-cpp/milvus_types.h @@ -63,6 +63,8 @@ class QueryResult; class TopKQueryResult; +class TopKQueryBinResult; + typedef struct _Exception__isset { _Exception__isset() : code(false), reason(false) {} bool code :1; @@ -346,6 +348,47 @@ void swap(TopKQueryResult &a, TopKQueryResult &b); std::ostream& operator<<(std::ostream& out, const TopKQueryResult& obj); + +class TopKQueryBinResult : public virtual ::apache::thrift::TBase { + public: + + TopKQueryBinResult(const TopKQueryBinResult&); + TopKQueryBinResult& operator=(const TopKQueryBinResult&); + TopKQueryBinResult() : id_array(), distance_array() { + } + + virtual ~TopKQueryBinResult() throw(); + std::string id_array; + std::string distance_array; + + void __set_id_array(const std::string& val); + + void __set_distance_array(const std::string& val); + + bool operator == (const TopKQueryBinResult & rhs) const + { + if (!(id_array == rhs.id_array)) + return false; + if (!(distance_array == rhs.distance_array)) + return false; + return true; + } + bool operator != (const TopKQueryBinResult &rhs) const { + return !(*this == rhs); + } + + bool operator < (const TopKQueryBinResult & ) const; + + uint32_t read(::apache::thrift::protocol::TProtocol* iprot); + uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const; + + virtual void printTo(std::ostream& out) const; +}; + +void swap(TopKQueryBinResult &a, TopKQueryBinResult &b); + +std::ostream& operator<<(std::ostream& out, const TopKQueryBinResult& obj); + }} // namespace #endif diff --git a/cpp/src/thrift/milvus.thrift b/cpp/src/thrift/milvus.thrift index 001b27b04b..64f25ae387 100644 --- a/cpp/src/thrift/milvus.thrift +++ b/cpp/src/thrift/milvus.thrift @@ -84,6 +84,14 @@ struct TopKQueryResult { 1: list query_result_arrays; ///< TopK query result } +/** + * @brief TopK query binary result + */ +struct TopKQueryBinResult { + 1: required binary id_array; ///< id array, interger array + 2: required binary distance_array; ///< distance array, double array +} + service MilvusService { /** * @brief Create table method @@ -158,6 +166,23 @@ service MilvusService { 4: list query_range_array, 5: i64 topk) throws(1: Exception e); + /** + * @brief Query vector + * + * This method is used to query vector in table. + * + * @param table_name, table_name is queried. + * @param query_record_array, all vector are going to be queried. + * @param query_range_array, optional ranges for conditional search. If not specified, search whole table + * @param topk, how many similarity vectors will be searched. + * + * @return query binary result array. + */ + list SearchVector2(2: string table_name, + 3: list query_record_array, + 4: list query_range_array, + 5: i64 topk) throws(1: Exception e); + /** * @brief Internal use query interface * From 97e452f61e54aab7a243247f74321c5009d0a0f2 Mon Sep 17 00:00:00 2001 From: zhiru Date: Wed, 17 Jul 2019 18:07:59 +0800 Subject: [PATCH 187/189] fix Former-commit-id: 5716472aeaa5679dac13c8e212e279df2960fdea --- cpp/cmake/ThirdPartyPackages.cmake | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cpp/cmake/ThirdPartyPackages.cmake b/cpp/cmake/ThirdPartyPackages.cmake index d08edceaf0..7b220d7a7e 100644 --- a/cpp/cmake/ThirdPartyPackages.cmake +++ b/cpp/cmake/ThirdPartyPackages.cmake @@ -715,8 +715,9 @@ macro(build_faiss) set(MKL_LIB_PATH "/opt/intel/compilers_and_libraries_${MKL_VERSION}/linux/mkl/lib/intel64") message(STATUS "MKL_LIB_PATH = ${MKL_LIB_PATH}") endif() + set(FAISS_CONFIGURE_ARGS ${FAISS_CONFIGURE_ARGS} - "CPPFLAGS=-DMKL_ILP64 -m64 -I${MKL_LIB_PATH}/../../include" + "CPPFLAGS=-DFINTEGER=long -DMKL_ILP64 -m64 -I${MKL_LIB_PATH}/../../include" "LDFLAGS=-L${MKL_LIB_PATH}" "LIBS=-Wl,--start-group ${MKL_LIB_PATH}/libmkl_intel_ilp64.a ${MKL_LIB_PATH}/libmkl_gnu_thread.a ${MKL_LIB_PATH}/libmkl_core.a -Wl,--end-group -lgomp -lpthread -lm -ldl") From fd43613935decf89fda499b75a2dfa36109dd7ef Mon Sep 17 00:00:00 2001 From: starlord Date: Wed, 17 Jul 2019 19:53:37 +0800 Subject: [PATCH 188/189] reduce unittest time Former-commit-id: aafd3f7417aa6339a67456d9f260a6f55e1b0e8c --- cpp/unittest/db/mem_test.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cpp/unittest/db/mem_test.cpp b/cpp/unittest/db/mem_test.cpp index 2f6952c79d..17d87030cd 100644 --- a/cpp/unittest/db/mem_test.cpp +++ b/cpp/unittest/db/mem_test.cpp @@ -273,7 +273,7 @@ TEST_F(NewMemManagerTest, INSERT_TEST) { int insert_loop = 20; for (int i = 0; i < insert_loop; ++i) { - int64_t nb = 409600; + int64_t nb = 40960; std::vector xb; BuildVectors(nb, xb); engine::IDNumbers vector_ids; @@ -308,7 +308,7 @@ TEST_F(NewMemManagerTest, CONCURRENT_INSERT_SEARCH_TEST) { engine::IDNumbers vector_ids; engine::IDNumbers target_ids; - int64_t nb = 409600; + int64_t nb = 40960; std::vector xb; BuildVectors(nb, xb); From 0878da322c67ffdd965d0f68d7b1dc5eec36b73d Mon Sep 17 00:00:00 2001 From: quicksilver Date: Wed, 17 Jul 2019 21:38:03 +0800 Subject: [PATCH 189/189] add milvus single node helm CI Former-commit-id: f9b886c858c7b71bbbe40f8b5f5c0e2baf9f513b --- ci/jenkinsfile/deploy2dev.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/jenkinsfile/deploy2dev.groovy b/ci/jenkinsfile/deploy2dev.groovy index b0c537f2e1..5991b11360 100644 --- a/ci/jenkinsfile/deploy2dev.groovy +++ b/ci/jenkinsfile/deploy2dev.groovy @@ -5,7 +5,7 @@ try { dir ("milvus-helm") { checkout([$class: 'GitSCM', branches: [[name: "${SEMVER}"]], doGenerateSubmoduleConfigurations: false, extensions: [], submoduleCfg: [], userRemoteConfigs: [[credentialsId: "${params.GIT_USER}", url: "git@192.168.1.105:megasearch/milvus-helm.git", name: 'origin', refspec: "+refs/heads/${SEMVER}:refs/remotes/origin/${SEMVER}"]]]) dir ("milvus/milvus-gpu") { - sh "helm install --set engine.image.tag=${DOCKER_VERSION} --set expose.type=clusterIP --name ${env.JOB_NAME}-${env.BUILD_NUMBER} --version 0.3.1 ." + sh "helm install --set engine.image.tag=${DOCKER_VERSION} --set expose.type=clusterIP --name ${env.JOB_NAME}-${env.BUILD_NUMBER} -f ci/values.yaml --version 0.3.1 ." } } timeout(time: 2, unit: 'MINUTES') {