milvus-io
/
milvus
mirror of https://github.com/milvus-io/milvus.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'branch-0.5.0' into 'branch-0.5.0' 

format scheduler code

See merge request megasearch/milvus!627

Former-commit-id: 4b46d2b89a1309faa30580dbcf58111c13c68ae5
pull/191/head
peng.xu 2019-09-27 14:22:23 +08:00
parent d4babb6dde a0e4d19e4f
commit b046049ffd
97 changed files with 1694 additions and 1578 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
cpp/src/db/Constants.h
View File

@ -33,8 +33,8 @@ constexpr uint64_t MAX_TABLE_FILE_MEM = 128 * M;
constexpr int VECTOR_TYPE_SIZE = sizeof(float); constexpr int VECTOR_TYPE_SIZE = sizeof(float);
static constexpr uint64_t ONE_KB = K; static constexpr uint64_t ONE_KB = K;
static constexpr uint64_t ONE_MB = ONE_KB*ONE_KB; static constexpr uint64_t ONE_MB = ONE_KB * ONE_KB;
static constexpr uint64_t ONE_GB = ONE_KB*ONE_MB; static constexpr uint64_t ONE_GB = ONE_KB * ONE_MB;
} // namespace engine } // namespace engine
} // namespace milvus } // namespace milvus

48
cpp/src/db/DB.h
View File

@ -25,6 +25,7 @@
#include <string> #include <string>
#include <memory> #include <memory>
#include <vector>
namespace zilliz { namespace zilliz {
namespace milvus { namespace milvus {
@ -33,46 +34,45 @@ namespace engine {
class Env; class Env;
class DB { class DB {
public: public:
DB() = default; DB() = default;
DB(const DB&) = delete; DB(const DB &) = delete;
DB& operator=(const DB&) = delete; DB &operator=(const DB &) = delete;
virtual ~DB() = default; virtual ~DB() = default;
virtual Status Start() = 0; virtual Status Start() = 0;
virtual Status Stop() = 0; virtual Status Stop() = 0;
virtual Status CreateTable(meta::TableSchema& table_schema_) = 0; virtual Status CreateTable(meta::TableSchema &table_schema_) = 0;
virtual Status DeleteTable(const std::string& table_id, const meta::DatesT& dates) = 0; virtual Status DeleteTable(const std::string &table_id, const meta::DatesT &dates) = 0;
virtual Status DescribeTable(meta::TableSchema& table_schema_) = 0; virtual Status DescribeTable(meta::TableSchema &table_schema_) = 0;
virtual Status HasTable(const std::string& table_id, bool& has_or_not_) = 0; virtual Status HasTable(const std::string &table_id, bool &has_or_not_) = 0;
virtual Status AllTables(std::vector<meta::TableSchema>& table_schema_array) = 0; virtual Status AllTables(std::vector<meta::TableSchema> &table_schema_array) = 0;
virtual Status GetTableRowCount(const std::string& table_id, uint64_t& row_count) = 0; virtual Status GetTableRowCount(const std::string &table_id, uint64_t &row_count) = 0;
virtual Status PreloadTable(const std::string& table_id) = 0; virtual Status PreloadTable(const std::string &table_id) = 0;
virtual Status UpdateTableFlag(const std::string &table_id, int64_t flag) = 0; virtual Status UpdateTableFlag(const std::string &table_id, int64_t flag) = 0;
virtual Status InsertVectors(const std::string& table_id_, virtual Status InsertVectors(const std::string &table_id_,
uint64_t n, const float* vectors, IDNumbers& vector_ids_) = 0; uint64_t n, const float *vectors, IDNumbers &vector_ids_) = 0;
virtual Status Query(const std::string& table_id, uint64_t k, uint64_t nq, uint64_t nprobe, virtual Status Query(const std::string &table_id, uint64_t k, uint64_t nq, uint64_t nprobe,
const float* vectors, QueryResults& results) = 0; const float *vectors, QueryResults &results) = 0;
virtual Status Query(const std::string& table_id, uint64_t k, uint64_t nq, uint64_t nprobe, virtual Status Query(const std::string &table_id, uint64_t k, uint64_t nq, uint64_t nprobe,
const float* vectors, const meta::DatesT& dates, QueryResults& results) = 0; const float *vectors, const meta::DatesT &dates, QueryResults &results) = 0;
virtual Status Query(const std::string& table_id, const std::vector<std::string>& file_ids, virtual Status Query(const std::string &table_id, const std::vector<std::string> &file_ids,
uint64_t k, uint64_t nq, uint64_t nprobe, const float* vectors, uint64_t k, uint64_t nq, uint64_t nprobe, const float *vectors,
const meta::DatesT& dates, QueryResults& results) = 0; const meta::DatesT &dates, QueryResults &results) = 0;
virtual Status Size(uint64_t& result) = 0; virtual Status Size(uint64_t &result) = 0;
virtual Status CreateIndex(const std::string& table_id, const TableIndex& index) = 0; virtual Status CreateIndex(const std::string &table_id, const TableIndex &index) = 0;
virtual Status DescribeIndex(const std::string& table_id, TableIndex& index) = 0; virtual Status DescribeIndex(const std::string &table_id, TableIndex &index) = 0;
virtual Status DropIndex(const std::string& table_id) = 0; virtual Status DropIndex(const std::string &table_id) = 0;
virtual Status DropAll() = 0; virtual Status DropAll() = 0;
}; // DB }; // DB
using DBPtr = std::shared_ptr<DB>; using DBPtr = std::shared_ptr<DB>;

10
cpp/src/db/DBFactory.cpp
View File

@ -16,7 +16,7 @@
// under the License. // under the License.
#include "DBFactory.h" #include "db/DBFactory.h"
#include "DBImpl.h" #include "DBImpl.h"
#include "utils/Exception.h" #include "utils/Exception.h"
#include "meta/MetaFactory.h" #include "meta/MetaFactory.h"
@ -33,14 +33,16 @@ namespace zilliz {
namespace milvus { namespace milvus {
namespace engine { namespace engine {
DBOptions DBFactory::BuildOption() { DBOptions
DBFactory::BuildOption() {
auto meta = MetaFactory::BuildOption(); auto meta = MetaFactory::BuildOption();
DBOptions options; DBOptions options;
options.meta_ = meta; options.meta_ = meta;
return options; return options;
} }
DBPtr DBFactory::Build(const DBOptions& options) { DBPtr
DBFactory::Build(const DBOptions &options) {
return std::make_shared<DBImpl>(options); return std::make_shared<DBImpl>(options);
} }

11
cpp/src/db/DBFactory.h
View File

@ -28,13 +28,12 @@ namespace milvus {
namespace engine { namespace engine {
class DBFactory { class DBFactory {
public: public:
static DBOptions BuildOption(); static DBOptions BuildOption();
static DBPtr Build(const DBOptions& options); static DBPtr Build(const DBOptions &options);
}; };
} // namespace engine
} } // namespace milvus
} } // namespace zilliz
}

294
cpp/src/db/DBImpl.cpp
View File

@ -15,7 +15,7 @@
// specific language governing permissions and limitations // specific language governing permissions and limitations
// under the License. // under the License.
#include "DBImpl.h" #include "db/DBImpl.h"
#include "cache/CpuCacheMgr.h" #include "cache/CpuCacheMgr.h"
#include "cache/GpuCacheMgr.h" #include "cache/GpuCacheMgr.h"
#include "engine/EngineFactory.h" #include "engine/EngineFactory.h"
@ -36,6 +36,7 @@
#include <thread> #include <thread>
#include <iostream> #include <iostream>
#include <cstring> #include <cstring>
#include <algorithm>
#include <boost/filesystem.hpp> #include <boost/filesystem.hpp>
namespace zilliz { namespace zilliz {
@ -48,10 +49,9 @@ constexpr uint64_t METRIC_ACTION_INTERVAL = 1;
constexpr uint64_t COMPACT_ACTION_INTERVAL = 1; constexpr uint64_t COMPACT_ACTION_INTERVAL = 1;
constexpr uint64_t INDEX_ACTION_INTERVAL = 1; constexpr uint64_t INDEX_ACTION_INTERVAL = 1;
} } // namespace
DBImpl::DBImpl(const DBOptions &options)
DBImpl::DBImpl(const DBOptions& options)
: options_(options), : options_(options),
shutting_down_(true), shutting_down_(true),
compact_thread_pool_(1, 1), compact_thread_pool_(1, 1),
@ -68,8 +68,9 @@ DBImpl::~DBImpl() {
/////////////////////////////////////////////////////////////////////////////////////////////////////////////////// ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
//external api //external api
/////////////////////////////////////////////////////////////////////////////////////////////////////////////////// ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
Status DBImpl::Start() { Status
if (!shutting_down_.load(std::memory_order_acquire)){ DBImpl::Start() {
if (!shutting_down_.load(std::memory_order_acquire)) {
return Status::OK(); return Status::OK();
} }
@ -85,8 +86,9 @@ Status DBImpl::Start() {
return Status::OK(); return Status::OK();
} }
Status DBImpl::Stop() { Status
if (shutting_down_.load(std::memory_order_acquire)){ DBImpl::Stop() {
if (shutting_down_.load(std::memory_order_acquire)) {
return Status::OK(); return Status::OK();
} }
@ -106,12 +108,14 @@ Status DBImpl::Stop() {
return Status::OK(); return Status::OK();
} }
Status DBImpl::DropAll() { Status
DBImpl::DropAll() {
return meta_ptr_->DropAll(); return meta_ptr_->DropAll();
} }
Status DBImpl::CreateTable(meta::TableSchema& table_schema) { Status
if (shutting_down_.load(std::memory_order_acquire)){ DBImpl::CreateTable(meta::TableSchema &table_schema) {
if (shutting_down_.load(std::memory_order_acquire)) {
return Status(DB_ERROR, "Milsvus server is shutdown!"); return Status(DB_ERROR, "Milsvus server is shutdown!");
} }
@ -120,8 +124,9 @@ Status DBImpl::CreateTable(meta::TableSchema& table_schema) {
return meta_ptr_->CreateTable(temp_schema); return meta_ptr_->CreateTable(temp_schema);
} }
Status DBImpl::DeleteTable(const std::string& table_id, const meta::DatesT& dates) { Status
if (shutting_down_.load(std::memory_order_acquire)){ DBImpl::DeleteTable(const std::string &table_id, const meta::DatesT &dates) {
if (shutting_down_.load(std::memory_order_acquire)) {
return Status(DB_ERROR, "Milsvus server is shutdown!"); return Status(DB_ERROR, "Milsvus server is shutdown!");
} }
@ -144,8 +149,9 @@ Status DBImpl::DeleteTable(const std::string& table_id, const meta::DatesT& date
return Status::OK(); return Status::OK();
} }
Status DBImpl::DescribeTable(meta::TableSchema& table_schema) { Status
if (shutting_down_.load(std::memory_order_acquire)){ DBImpl::DescribeTable(meta::TableSchema &table_schema) {
if (shutting_down_.load(std::memory_order_acquire)) {
return Status(DB_ERROR, "Milsvus server is shutdown!"); return Status(DB_ERROR, "Milsvus server is shutdown!");
} }
@ -154,24 +160,27 @@ Status DBImpl::DescribeTable(meta::TableSchema& table_schema) {
return stat; return stat;
} }
Status DBImpl::HasTable(const std::string& table_id, bool& has_or_not) { Status
if (shutting_down_.load(std::memory_order_acquire)){ DBImpl::HasTable(const std::string &table_id, bool &has_or_not) {
if (shutting_down_.load(std::memory_order_acquire)) {
return Status(DB_ERROR, "Milsvus server is shutdown!"); return Status(DB_ERROR, "Milsvus server is shutdown!");
} }
return meta_ptr_->HasTable(table_id, has_or_not); return meta_ptr_->HasTable(table_id, has_or_not);
} }
Status DBImpl::AllTables(std::vector<meta::TableSchema>& table_schema_array) { Status
if (shutting_down_.load(std::memory_order_acquire)){ DBImpl::AllTables(std::vector<meta::TableSchema> &table_schema_array) {
if (shutting_down_.load(std::memory_order_acquire)) {
return Status(DB_ERROR, "Milsvus server is shutdown!"); return Status(DB_ERROR, "Milsvus server is shutdown!");
} }
return meta_ptr_->AllTables(table_schema_array); return meta_ptr_->AllTables(table_schema_array);
} }
Status DBImpl::PreloadTable(const std::string &table_id) { Status
if (shutting_down_.load(std::memory_order_acquire)){ DBImpl::PreloadTable(const std::string &table_id) {
if (shutting_down_.load(std::memory_order_acquire)) {
return Status(DB_ERROR, "Milsvus server is shutdown!"); return Status(DB_ERROR, "Milsvus server is shutdown!");
} }
@ -189,10 +198,14 @@ Status DBImpl::PreloadTable(const std::string &table_id) {
int64_t cache_usage = cache::CpuCacheMgr::GetInstance()->CacheUsage(); int64_t cache_usage = cache::CpuCacheMgr::GetInstance()->CacheUsage();
int64_t available_size = cache_total - cache_usage; int64_t available_size = cache_total - cache_usage;
for(auto &day_files : files) { for (auto &day_files : files) {
for (auto &file : day_files.second) { for (auto &file : day_files.second) {
ExecutionEnginePtr engine = EngineFactory::Build(file.dimension_, file.location_, (EngineType)file.engine_type_, (MetricType)file.metric_type_, file.nlist_); ExecutionEnginePtr engine = EngineFactory::Build(file.dimension_,
if(engine == nullptr) { file.location_,
(EngineType) file.engine_type_,
(MetricType) file.metric_type_,
file.nlist_);
if (engine == nullptr) {
ENGINE_LOG_ERROR << "Invalid engine type"; ENGINE_LOG_ERROR << "Invalid engine type";
return Status(DB_ERROR, "Invalid engine type"); return Status(DB_ERROR, "Invalid engine type");
} }
@ -215,33 +228,37 @@ Status DBImpl::PreloadTable(const std::string &table_id) {
return Status::OK(); return Status::OK();
} }
Status DBImpl::UpdateTableFlag(const std::string &table_id, int64_t flag) { Status
if (shutting_down_.load(std::memory_order_acquire)){ DBImpl::UpdateTableFlag(const std::string &table_id, int64_t flag) {
if (shutting_down_.load(std::memory_order_acquire)) {
return Status(DB_ERROR, "Milsvus server is shutdown!"); return Status(DB_ERROR, "Milsvus server is shutdown!");
} }
return meta_ptr_->UpdateTableFlag(table_id, flag); return meta_ptr_->UpdateTableFlag(table_id, flag);
} }
Status DBImpl::GetTableRowCount(const std::string& table_id, uint64_t& row_count) { Status
if (shutting_down_.load(std::memory_order_acquire)){ DBImpl::GetTableRowCount(const std::string &table_id, uint64_t &row_count) {
if (shutting_down_.load(std::memory_order_acquire)) {
return Status(DB_ERROR, "Milsvus server is shutdown!"); return Status(DB_ERROR, "Milsvus server is shutdown!");
} }
return meta_ptr_->Count(table_id, row_count); return meta_ptr_->Count(table_id, row_count);
} }
Status DBImpl::InsertVectors(const std::string& table_id_, Status
uint64_t n, const float* vectors, IDNumbers& vector_ids_) { DBImpl::InsertVectors(const std::string &table_id_,
uint64_t n, const float *vectors, IDNumbers &vector_ids_) {
// ENGINE_LOG_DEBUG << "Insert " << n << " vectors to cache"; // ENGINE_LOG_DEBUG << "Insert " << n << " vectors to cache";
if (shutting_down_.load(std::memory_order_acquire)){ if (shutting_down_.load(std::memory_order_acquire)) {
return Status(DB_ERROR, "Milsvus server is shutdown!"); return Status(DB_ERROR, "Milsvus server is shutdown!");
} }
Status status; Status status;
zilliz::milvus::server::CollectInsertMetrics metrics(n, status); zilliz::milvus::server::CollectInsertMetrics metrics(n, status);
status = mem_mgr_->InsertVectors(table_id_, n, vectors, vector_ids_); status = mem_mgr_->InsertVectors(table_id_, n, vectors, vector_ids_);
// std::chrono::microseconds time_span = std::chrono::duration_cast<std::chrono::microseconds>(end_time - start_time); // std::chrono::microseconds time_span =
// std::chrono::duration_cast<std::chrono::microseconds>(end_time - start_time);
// double average_time = double(time_span.count()) / n; // double average_time = double(time_span.count()) / n;
// ENGINE_LOG_DEBUG << "Insert vectors to cache finished"; // ENGINE_LOG_DEBUG << "Insert vectors to cache finished";
@ -249,14 +266,15 @@ Status DBImpl::InsertVectors(const std::string& table_id_,
return status; return status;
} }
Status DBImpl::CreateIndex(const std::string& table_id, const TableIndex& index) { Status
DBImpl::CreateIndex(const std::string &table_id, const TableIndex &index) {
{ {
std::unique_lock<std::mutex> lock(build_index_mutex_); std::unique_lock<std::mutex> lock(build_index_mutex_);
//step 1: check index difference //step 1: check index difference
TableIndex old_index; TableIndex old_index;
auto status = DescribeIndex(table_id, old_index); auto status = DescribeIndex(table_id, old_index);
if(!status.ok()) { if (!status.ok()) {
ENGINE_LOG_ERROR << "Failed to get table index info for table: " << table_id; ENGINE_LOG_ERROR << "Failed to get table index info for table: " << table_id;
return status; return status;
} }
@ -264,7 +282,7 @@ Status DBImpl::CreateIndex(const std::string& table_id, const TableIndex& index)
//step 2: update index info //step 2: update index info
TableIndex new_index = index; TableIndex new_index = index;
new_index.metric_type_ = old_index.metric_type_;//dont change metric type, it was defined by CreateTable new_index.metric_type_ = old_index.metric_type_;//dont change metric type, it was defined by CreateTable
if(!utils::IsSameIndex(old_index, new_index)) { if (!utils::IsSameIndex(old_index, new_index)) {
DropIndex(table_id); DropIndex(table_id);
status = meta_ptr_->UpdateTableIndex(table_id, new_index); status = meta_ptr_->UpdateTableIndex(table_id, new_index);
@ -283,18 +301,18 @@ Status DBImpl::CreateIndex(const std::string& table_id, const TableIndex& index)
//for IDMAP type, only wait all NEW file converted to RAW file //for IDMAP type, only wait all NEW file converted to RAW file
//for other type, wait NEW/RAW/NEW_MERGE/NEW_INDEX/TO_INDEX files converted to INDEX files //for other type, wait NEW/RAW/NEW_MERGE/NEW_INDEX/TO_INDEX files converted to INDEX files
std::vector<int> file_types; std::vector<int> file_types;
if(index.engine_type_ == (int)EngineType::FAISS_IDMAP) { if (index.engine_type_ == (int) EngineType::FAISS_IDMAP) {
file_types = { file_types = {
(int) meta::TableFileSchema::NEW, (int) meta::TableFileSchema::NEW,
(int) meta::TableFileSchema::NEW_MERGE, (int) meta::TableFileSchema::NEW_MERGE,
}; };
} else { } else {
file_types = { file_types = {
(int) meta::TableFileSchema::RAW, (int) meta::TableFileSchema::RAW,
(int) meta::TableFileSchema::NEW, (int) meta::TableFileSchema::NEW,
(int) meta::TableFileSchema::NEW_MERGE, (int) meta::TableFileSchema::NEW_MERGE,
(int) meta::TableFileSchema::NEW_INDEX, (int) meta::TableFileSchema::NEW_INDEX,
(int) meta::TableFileSchema::TO_INDEX, (int) meta::TableFileSchema::TO_INDEX,
}; };
} }
@ -304,11 +322,11 @@ Status DBImpl::CreateIndex(const std::string& table_id, const TableIndex& index)
while (!file_ids.empty()) { while (!file_ids.empty()) {
ENGINE_LOG_DEBUG << "Non index files detected! Will build index " << times; ENGINE_LOG_DEBUG << "Non index files detected! Will build index " << times;
if(index.engine_type_ != (int)EngineType::FAISS_IDMAP) { if (index.engine_type_ != (int) EngineType::FAISS_IDMAP) {
status = meta_ptr_->UpdateTableFilesToIndex(table_id); status = meta_ptr_->UpdateTableFilesToIndex(table_id);
} }
std::this_thread::sleep_for(std::chrono::milliseconds(std::min(10*1000, times*100))); std::this_thread::sleep_for(std::chrono::milliseconds(std::min(10 * 1000, times * 100)));
status = meta_ptr_->FilesByType(table_id, file_types, file_ids); status = meta_ptr_->FilesByType(table_id, file_types, file_ids);
times++; times++;
} }
@ -316,18 +334,21 @@ Status DBImpl::CreateIndex(const std::string& table_id, const TableIndex& index)
return Status::OK(); return Status::OK();
} }
Status DBImpl::DescribeIndex(const std::string& table_id, TableIndex& index) { Status
DBImpl::DescribeIndex(const std::string &table_id, TableIndex &index) {
return meta_ptr_->DescribeTableIndex(table_id, index); return meta_ptr_->DescribeTableIndex(table_id, index);
} }
Status DBImpl::DropIndex(const std::string& table_id) { Status
DBImpl::DropIndex(const std::string &table_id) {
ENGINE_LOG_DEBUG << "Drop index for table: " << table_id; ENGINE_LOG_DEBUG << "Drop index for table: " << table_id;
return meta_ptr_->DropTableIndex(table_id); return meta_ptr_->DropTableIndex(table_id);
} }
Status DBImpl::Query(const std::string &table_id, uint64_t k, uint64_t nq, uint64_t nprobe, Status
const float *vectors, QueryResults &results) { DBImpl::Query(const std::string &table_id, uint64_t k, uint64_t nq, uint64_t nprobe,
if (shutting_down_.load(std::memory_order_acquire)){ const float *vectors, QueryResults &results) {
if (shutting_down_.load(std::memory_order_acquire)) {
return Status(DB_ERROR, "Milsvus server is shutdown!"); return Status(DB_ERROR, "Milsvus server is shutdown!");
} }
@ -337,9 +358,10 @@ Status DBImpl::Query(const std::string &table_id, uint64_t k, uint64_t nq, uint6
return result; return result;
} }
Status DBImpl::Query(const std::string& table_id, uint64_t k, uint64_t nq, uint64_t nprobe, Status
const float* vectors, const meta::DatesT& dates, QueryResults& results) { DBImpl::Query(const std::string &table_id, uint64_t k, uint64_t nq, uint64_t nprobe,
if (shutting_down_.load(std::memory_order_acquire)){ const float *vectors, const meta::DatesT &dates, QueryResults &results) {
if (shutting_down_.load(std::memory_order_acquire)) {
return Status(DB_ERROR, "Milsvus server is shutdown!"); return Status(DB_ERROR, "Milsvus server is shutdown!");
} }
@ -364,10 +386,11 @@ Status DBImpl::Query(const std::string& table_id, uint64_t k, uint64_t nq, uint6
return status; return status;
} }
Status DBImpl::Query(const std::string& table_id, const std::vector<std::string>& file_ids, Status
uint64_t k, uint64_t nq, uint64_t nprobe, const float* vectors, DBImpl::Query(const std::string &table_id, const std::vector<std::string> &file_ids,
const meta::DatesT& dates, QueryResults& results) { uint64_t k, uint64_t nq, uint64_t nprobe, const float *vectors,
if (shutting_down_.load(std::memory_order_acquire)){ const meta::DatesT &dates, QueryResults &results) {
if (shutting_down_.load(std::memory_order_acquire)) {
return Status(DB_ERROR, "Milsvus server is shutdown!"); return Status(DB_ERROR, "Milsvus server is shutdown!");
} }
@ -395,7 +418,7 @@ Status DBImpl::Query(const std::string& table_id, const std::vector<std::string>
} }
} }
if(file_id_array.empty()) { if (file_id_array.empty()) {
return Status(DB_ERROR, "Invalid file id"); return Status(DB_ERROR, "Invalid file id");
} }
@ -405,36 +428,37 @@ Status DBImpl::Query(const std::string& table_id, const std::vector<std::string>
return status; return status;
} }
Status DBImpl::Size(uint64_t& result) { Status
if (shutting_down_.load(std::memory_order_acquire)){ DBImpl::Size(uint64_t &result) {
if (shutting_down_.load(std::memory_order_acquire)) {
return Status(DB_ERROR, "Milsvus server is shutdown!"); return Status(DB_ERROR, "Milsvus server is shutdown!");
} }
return meta_ptr_->Size(result); return meta_ptr_->Size(result);
} }
/////////////////////////////////////////////////////////////////////////////////////////////////////////////////// ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
//internal methods //internal methods
/////////////////////////////////////////////////////////////////////////////////////////////////////////////////// ///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
Status DBImpl::QueryAsync(const std::string& table_id, const meta::TableFilesSchema& files, Status
uint64_t k, uint64_t nq, uint64_t nprobe, const float* vectors, DBImpl::QueryAsync(const std::string &table_id, const meta::TableFilesSchema &files,
const meta::DatesT& dates, QueryResults& results) { uint64_t k, uint64_t nq, uint64_t nprobe, const float *vectors,
using namespace scheduler; const meta::DatesT &dates, QueryResults &results) {
server::CollectQueryMetrics metrics(nq); server::CollectQueryMetrics metrics(nq);
TimeRecorder rc(""); TimeRecorder rc("");
//step 1: get files to search //step 1: get files to search
ENGINE_LOG_DEBUG << "Engine query begin, index file count: " << files.size() << " date range count: " << dates.size(); ENGINE_LOG_DEBUG << "Engine query begin, index file count: " << files.size() << " date range count: "
SearchJobPtr job = std::make_shared<SearchJob>(0, k, nq, nprobe, vectors); << dates.size();
for (auto &file : files) { scheduler::SearchJobPtr job = std::make_shared<scheduler::SearchJob>(0, k, nq, nprobe, vectors);
TableFileSchemaPtr file_ptr = std::make_shared<meta::TableFileSchema>(file); for (auto &file : files) {
scheduler::TableFileSchemaPtr file_ptr = std::make_shared<meta::TableFileSchema>(file);
job->AddIndexFile(file_ptr); job->AddIndexFile(file_ptr);
} }
//step 2: put search task to scheduler //step 2: put search task to scheduler
JobMgrInst::GetInstance()->Put(job); scheduler::JobMgrInst::GetInstance()->Put(job);
job->WaitResult(); job->WaitResult();
if (!job->GetStatus().ok()) { if (!job->GetStatus().ok()) {
return job->GetStatus(); return job->GetStatus();
@ -453,9 +477,12 @@ Status DBImpl::QueryAsync(const std::string& table_id, const meta::TableFilesSch
// double search_percent = search_cost/total_cost; // double search_percent = search_cost/total_cost;
// double reduce_percent = reduce_cost/total_cost; // double reduce_percent = reduce_cost/total_cost;
// //
// ENGINE_LOG_DEBUG << "Engine load index totally cost: " << load_info << " percent: " << load_percent*100 << "%"; // ENGINE_LOG_DEBUG << "Engine load index totally cost: " << load_info
// ENGINE_LOG_DEBUG << "Engine search index totally cost: " << search_info << " percent: " << search_percent*100 << "%"; // << " percent: " << load_percent*100 << "%";
// ENGINE_LOG_DEBUG << "Engine reduce topk totally cost: " << reduce_info << " percent: " << reduce_percent*100 << "%"; // ENGINE_LOG_DEBUG << "Engine search index totally cost: " << search_info
// << " percent: " << search_percent*100 << "%";
// ENGINE_LOG_DEBUG << "Engine reduce topk totally cost: " << reduce_info
// << " percent: " << reduce_percent*100 << "%";
// } else { // } else {
// ENGINE_LOG_DEBUG << "Engine load cost: " << load_info // ENGINE_LOG_DEBUG << "Engine load cost: " << load_info
// << " search cost: " << search_info // << " search cost: " << search_info
@ -469,11 +496,12 @@ Status DBImpl::QueryAsync(const std::string& table_id, const meta::TableFilesSch
return Status::OK(); return Status::OK();
} }
void DBImpl::BackgroundTimerTask() { void
DBImpl::BackgroundTimerTask() {
Status status; Status status;
server::SystemInfo::GetInstance().Init(); server::SystemInfo::GetInstance().Init();
while (true) { while (true) {
if (shutting_down_.load(std::memory_order_acquire)){ if (shutting_down_.load(std::memory_order_acquire)) {
WaitMergeFileFinish(); WaitMergeFileFinish();
WaitBuildIndexFinish(); WaitBuildIndexFinish();
@ -489,24 +517,27 @@ void DBImpl::BackgroundTimerTask() {
} }
} }
void DBImpl::WaitMergeFileFinish() { void
DBImpl::WaitMergeFileFinish() {
std::lock_guard<std::mutex> lck(compact_result_mutex_); std::lock_guard<std::mutex> lck(compact_result_mutex_);
for(auto& iter : compact_thread_results_) { for (auto &iter : compact_thread_results_) {
iter.wait(); iter.wait();
} }
} }
void DBImpl::WaitBuildIndexFinish() { void
DBImpl::WaitBuildIndexFinish() {
std::lock_guard<std::mutex> lck(index_result_mutex_); std::lock_guard<std::mutex> lck(index_result_mutex_);
for(auto& iter : index_thread_results_) { for (auto &iter : index_thread_results_) {
iter.wait(); iter.wait();
} }
} }
void DBImpl::StartMetricTask() { void
DBImpl::StartMetricTask() {
static uint64_t metric_clock_tick = 0; static uint64_t metric_clock_tick = 0;
metric_clock_tick++; metric_clock_tick++;
if(metric_clock_tick%METRIC_ACTION_INTERVAL != 0) { if (metric_clock_tick % METRIC_ACTION_INTERVAL != 0) {
return; return;
} }
@ -515,7 +546,7 @@ void DBImpl::StartMetricTask() {
server::Metrics::GetInstance().KeepingAliveCounterIncrement(METRIC_ACTION_INTERVAL); server::Metrics::GetInstance().KeepingAliveCounterIncrement(METRIC_ACTION_INTERVAL);
int64_t cache_usage = cache::CpuCacheMgr::GetInstance()->CacheUsage(); int64_t cache_usage = cache::CpuCacheMgr::GetInstance()->CacheUsage();
int64_t cache_total = cache::CpuCacheMgr::GetInstance()->CacheCapacity(); int64_t cache_total = cache::CpuCacheMgr::GetInstance()->CacheCapacity();
server::Metrics::GetInstance().CpuCacheUsageGaugeSet(cache_usage*100/cache_total); server::Metrics::GetInstance().CpuCacheUsageGaugeSet(cache_usage * 100 / cache_total);
server::Metrics::GetInstance().GpuCacheUsageGaugeSet(); server::Metrics::GetInstance().GpuCacheUsageGaugeSet();
uint64_t size; uint64_t size;
Size(size); Size(size);
@ -533,25 +564,27 @@ void DBImpl::StartMetricTask() {
ENGINE_LOG_TRACE << "Metric task finished"; ENGINE_LOG_TRACE << "Metric task finished";
} }
Status DBImpl::MemSerialize() { Status
DBImpl::MemSerialize() {
std::lock_guard<std::mutex> lck(mem_serialize_mutex_); std::lock_guard<std::mutex> lck(mem_serialize_mutex_);
std::set<std::string> temp_table_ids; std::set<std::string> temp_table_ids;
mem_mgr_->Serialize(temp_table_ids); mem_mgr_->Serialize(temp_table_ids);
for(auto& id : temp_table_ids) { for (auto &id : temp_table_ids) {
compact_table_ids_.insert(id); compact_table_ids_.insert(id);
} }
if(!temp_table_ids.empty()) { if (!temp_table_ids.empty()) {
SERVER_LOG_DEBUG << "Insert cache serialized"; SERVER_LOG_DEBUG << "Insert cache serialized";
} }
return Status::OK(); return Status::OK();
} }
void DBImpl::StartCompactionTask() { void
DBImpl::StartCompactionTask() {
static uint64_t compact_clock_tick = 0; static uint64_t compact_clock_tick = 0;
compact_clock_tick++; compact_clock_tick++;
if(compact_clock_tick%COMPACT_ACTION_INTERVAL != 0) { if (compact_clock_tick % COMPACT_ACTION_INTERVAL != 0) {
return; return;
} }
@ -580,8 +613,9 @@ void DBImpl::StartCompactionTask() {
} }
} }
Status DBImpl::MergeFiles(const std::string& table_id, const meta::DateT& date, Status
const meta::TableFilesSchema& files) { DBImpl::MergeFiles(const std::string &table_id, const meta::DateT &date,
const meta::TableFilesSchema &files) {
ENGINE_LOG_DEBUG << "Merge files for table: " << table_id; ENGINE_LOG_DEBUG << "Merge files for table: " << table_id;
//step 1: create table file //step 1: create table file
@ -598,13 +632,13 @@ Status DBImpl::MergeFiles(const std::string& table_id, const meta::DateT& date,
//step 2: merge files //step 2: merge files
ExecutionEnginePtr index = ExecutionEnginePtr index =
EngineFactory::Build(table_file.dimension_, table_file.location_, (EngineType)table_file.engine_type_, EngineFactory::Build(table_file.dimension_, table_file.location_, (EngineType) table_file.engine_type_,
(MetricType)table_file.metric_type_, table_file.nlist_); (MetricType) table_file.metric_type_, table_file.nlist_);
meta::TableFilesSchema updated; meta::TableFilesSchema updated;
long index_size = 0; int64_t index_size = 0;
for (auto& file : files) { for (auto &file : files) {
server::CollectMergeFilesMetrics metrics; server::CollectMergeFilesMetrics metrics;
index->Merge(file.location_); index->Merge(file.location_);
@ -620,7 +654,7 @@ Status DBImpl::MergeFiles(const std::string& table_id, const meta::DateT& date,
//step 3: serialize to disk //step 3: serialize to disk
try { try {
index->Serialize(); index->Serialize();
} catch (std::exception& ex) { } catch (std::exception &ex) {
//typical error: out of disk space or permition denied //typical error: out of disk space or permition denied
std::string msg = "Serialize merged index encounter exception: " + std::string(ex.what()); std::string msg = "Serialize merged index encounter exception: " + std::string(ex.what());
ENGINE_LOG_ERROR << msg; ENGINE_LOG_ERROR << msg;
@ -638,7 +672,7 @@ Status DBImpl::MergeFiles(const std::string& table_id, const meta::DateT& date,
//step 4: update table files state //step 4: update table files state
//if index type isn't IDMAP, set file type to TO_INDEX if file size execeed index_file_size //if index type isn't IDMAP, set file type to TO_INDEX if file size execeed index_file_size
//else set file type to RAW, no need to build index //else set file type to RAW, no need to build index
if (table_file.engine_type_ != (int)EngineType::FAISS_IDMAP) { if (table_file.engine_type_ != (int) EngineType::FAISS_IDMAP) {
table_file.file_type_ = (index->PhysicalSize() >= table_file.index_file_size_) ? table_file.file_type_ = (index->PhysicalSize() >= table_file.index_file_size_) ?
meta::TableFileSchema::TO_INDEX : meta::TableFileSchema::RAW; meta::TableFileSchema::TO_INDEX : meta::TableFileSchema::RAW;
} else { } else {
@ -649,16 +683,17 @@ Status DBImpl::MergeFiles(const std::string& table_id, const meta::DateT& date,
updated.push_back(table_file); updated.push_back(table_file);
status = meta_ptr_->UpdateTableFiles(updated); status = meta_ptr_->UpdateTableFiles(updated);
ENGINE_LOG_DEBUG << "New merged file " << table_file.file_id_ << ENGINE_LOG_DEBUG << "New merged file " << table_file.file_id_ <<
" of size " << index->PhysicalSize() << " bytes"; " of size " << index->PhysicalSize() << " bytes";
if(options_.insert_cache_immediately_) { if (options_.insert_cache_immediately_) {
index->Cache(); index->Cache();
} }
return status; return status;
} }
Status DBImpl::BackgroundMergeFiles(const std::string& table_id) { Status
DBImpl::BackgroundMergeFiles(const std::string &table_id) {
meta::DatePartionedTableFilesSchema raw_files; meta::DatePartionedTableFilesSchema raw_files;
auto status = meta_ptr_->FilesToMerge(table_id, raw_files); auto status = meta_ptr_->FilesToMerge(table_id, raw_files);
if (!status.ok()) { if (!status.ok()) {
@ -667,7 +702,7 @@ Status DBImpl::BackgroundMergeFiles(const std::string& table_id) {
} }
bool has_merge = false; bool has_merge = false;
for (auto& kv : raw_files) { for (auto &kv : raw_files) {
auto files = kv.second; auto files = kv.second;
if (files.size() < options_.merge_trigger_number_) { if (files.size() < options_.merge_trigger_number_) {
ENGINE_LOG_DEBUG << "Files number not greater equal than merge trigger number, skip merge action"; ENGINE_LOG_DEBUG << "Files number not greater equal than merge trigger number, skip merge action";
@ -676,7 +711,7 @@ Status DBImpl::BackgroundMergeFiles(const std::string& table_id) {
has_merge = true; has_merge = true;
MergeFiles(table_id, kv.first, kv.second); MergeFiles(table_id, kv.first, kv.second);
if (shutting_down_.load(std::memory_order_acquire)){ if (shutting_down_.load(std::memory_order_acquire)) {
ENGINE_LOG_DEBUG << "Server will shutdown, skip merge action for table: " << table_id; ENGINE_LOG_DEBUG << "Server will shutdown, skip merge action for table: " << table_id;
break; break;
} }
@ -685,17 +720,18 @@ Status DBImpl::BackgroundMergeFiles(const std::string& table_id) {
return Status::OK(); return Status::OK();
} }
void DBImpl::BackgroundCompaction(std::set<std::string> table_ids) { void
DBImpl::BackgroundCompaction(std::set<std::string> table_ids) {
ENGINE_LOG_TRACE << " Background compaction thread start"; ENGINE_LOG_TRACE << " Background compaction thread start";
Status status; Status status;
for (auto& table_id : table_ids) { for (auto &table_id : table_ids) {
status = BackgroundMergeFiles(table_id); status = BackgroundMergeFiles(table_id);
if (!status.ok()) { if (!status.ok()) {
ENGINE_LOG_ERROR << "Merge files for table " << table_id << " failed: " << status.ToString(); ENGINE_LOG_ERROR << "Merge files for table " << table_id << " failed: " << status.ToString();
} }
if (shutting_down_.load(std::memory_order_acquire)){ if (shutting_down_.load(std::memory_order_acquire)) {
ENGINE_LOG_DEBUG << "Server will shutdown, skip merge action"; ENGINE_LOG_DEBUG << "Server will shutdown, skip merge action";
break; break;
} }
@ -703,7 +739,7 @@ void DBImpl::BackgroundCompaction(std::set<std::string> table_ids) {
meta_ptr_->Archive(); meta_ptr_->Archive();
int ttl = 5*meta::M_SEC;//default: file will be deleted after 5 minutes int ttl = 5 * meta::M_SEC;//default: file will be deleted after 5 minutes
if (options_.mode_ == DBOptions::MODE::CLUSTER_WRITABLE) { if (options_.mode_ == DBOptions::MODE::CLUSTER_WRITABLE) {
ttl = meta::D_SEC; ttl = meta::D_SEC;
} }
@ -712,10 +748,11 @@ void DBImpl::BackgroundCompaction(std::set<std::string> table_ids) {
ENGINE_LOG_TRACE << " Background compaction thread exit"; ENGINE_LOG_TRACE << " Background compaction thread exit";
} }
void DBImpl::StartBuildIndexTask(bool force) { void
DBImpl::StartBuildIndexTask(bool force) {
static uint64_t index_clock_tick = 0; static uint64_t index_clock_tick = 0;
index_clock_tick++; index_clock_tick++;
if(!force && (index_clock_tick%INDEX_ACTION_INTERVAL != 0)) { if (!force && (index_clock_tick % INDEX_ACTION_INTERVAL != 0)) {
return; return;
} }
@ -740,11 +777,12 @@ void DBImpl::StartBuildIndexTask(bool force) {
} }
} }
Status DBImpl::BuildIndex(const meta::TableFileSchema& file) { Status
DBImpl::BuildIndex(const meta::TableFileSchema &file) {
ExecutionEnginePtr to_index = ExecutionEnginePtr to_index =
EngineFactory::Build(file.dimension_, file.location_, (EngineType)file.engine_type_, EngineFactory::Build(file.dimension_, file.location_, (EngineType) file.engine_type_,
(MetricType)file.metric_type_, file.nlist_); (MetricType) file.metric_type_, file.nlist_);
if(to_index == nullptr) { if (to_index == nullptr) {
ENGINE_LOG_ERROR << "Invalid engine type"; ENGINE_LOG_ERROR << "Invalid engine type";
return Status(DB_ERROR, "Invalid engine type"); return Status(DB_ERROR, "Invalid engine type");
} }
@ -761,7 +799,8 @@ Status DBImpl::BuildIndex(const meta::TableFileSchema& file) {
meta::TableFileSchema table_file; meta::TableFileSchema table_file;
table_file.table_id_ = file.table_id_; table_file.table_id_ = file.table_id_;
table_file.date_ = file.date_; table_file.date_ = file.date_;
table_file.file_type_ = meta::TableFileSchema::NEW_INDEX; //for multi-db-path, distribute index file averagely to each path table_file.file_type_ =
meta::TableFileSchema::NEW_INDEX; //for multi-db-path, distribute index file averagely to each path
status = meta_ptr_->CreateTableFile(table_file); status = meta_ptr_->CreateTableFile(table_file);
if (!status.ok()) { if (!status.ok()) {
ENGINE_LOG_ERROR << "Failed to create table file: " << status.ToString(); ENGINE_LOG_ERROR << "Failed to create table file: " << status.ToString();
@ -773,16 +812,16 @@ Status DBImpl::BuildIndex(const meta::TableFileSchema& file) {
try { try {
server::CollectBuildIndexMetrics metrics; server::CollectBuildIndexMetrics metrics;
index = to_index->BuildIndex(table_file.location_, (EngineType)table_file.engine_type_); index = to_index->BuildIndex(table_file.location_, (EngineType) table_file.engine_type_);
if (index == nullptr) { if (index == nullptr) {
table_file.file_type_ = meta::TableFileSchema::TO_DELETE; table_file.file_type_ = meta::TableFileSchema::TO_DELETE;
status = meta_ptr_->UpdateTableFile(table_file); status = meta_ptr_->UpdateTableFile(table_file);
ENGINE_LOG_DEBUG << "Failed to update file to index, mark file: " << table_file.file_id_ << " to to_delete"; ENGINE_LOG_DEBUG << "Failed to update file to index, mark file: " << table_file.file_id_
<< " to to_delete";
return status; return status;
} }
} catch (std::exception &ex) {
} catch (std::exception& ex) {
//typical error: out of gpu memory //typical error: out of gpu memory
std::string msg = "BuildIndex encounter exception: " + std::string(ex.what()); std::string msg = "BuildIndex encounter exception: " + std::string(ex.what());
ENGINE_LOG_ERROR << msg; ENGINE_LOG_ERROR << msg;
@ -791,7 +830,8 @@ Status DBImpl::BuildIndex(const meta::TableFileSchema& file) {
status = meta_ptr_->UpdateTableFile(table_file); status = meta_ptr_->UpdateTableFile(table_file);
ENGINE_LOG_DEBUG << "Failed to update file to index, mark file: " << table_file.file_id_ << " to to_delete"; ENGINE_LOG_DEBUG << "Failed to update file to index, mark file: " << table_file.file_id_ << " to to_delete";
std::cout << "ERROR: failed to build index, index file is too large or gpu memory is not enough" << std::endl; std::cout << "ERROR: failed to build index, index file is too large or gpu memory is not enough"
<< std::endl;
return Status(DB_ERROR, msg); return Status(DB_ERROR, msg);
} }
@ -799,7 +839,7 @@ Status DBImpl::BuildIndex(const meta::TableFileSchema& file) {
//step 4: if table has been deleted, dont save index file //step 4: if table has been deleted, dont save index file
bool has_table = false; bool has_table = false;
meta_ptr_->HasTable(file.table_id_, has_table); meta_ptr_->HasTable(file.table_id_, has_table);
if(!has_table) { if (!has_table) {
meta_ptr_->DeleteTableFiles(file.table_id_); meta_ptr_->DeleteTableFiles(file.table_id_);
return Status::OK(); return Status::OK();
} }
@ -807,7 +847,7 @@ Status DBImpl::BuildIndex(const meta::TableFileSchema& file) {
//step 5: save index file //step 5: save index file
try { try {
index->Serialize(); index->Serialize();
} catch (std::exception& ex) { } catch (std::exception &ex) {
//typical error: out of disk space or permition denied //typical error: out of disk space or permition denied
std::string msg = "Serialize index encounter exception: " + std::string(ex.what()); std::string msg = "Serialize index encounter exception: " + std::string(ex.what());
ENGINE_LOG_ERROR << msg; ENGINE_LOG_ERROR << msg;
@ -817,7 +857,7 @@ Status DBImpl::BuildIndex(const meta::TableFileSchema& file) {
ENGINE_LOG_DEBUG << "Failed to update file to index, mark file: " << table_file.file_id_ << " to to_delete"; ENGINE_LOG_DEBUG << "Failed to update file to index, mark file: " << table_file.file_id_ << " to to_delete";
std::cout << "ERROR: failed to persist index file: " << table_file.location_ std::cout << "ERROR: failed to persist index file: " << table_file.location_
<< ", possible out of disk space" << std::endl; << ", possible out of disk space" << std::endl;
return Status(DB_ERROR, msg); return Status(DB_ERROR, msg);
} }
@ -832,12 +872,12 @@ Status DBImpl::BuildIndex(const meta::TableFileSchema& file) {
meta::TableFilesSchema update_files = {table_file, origin_file}; meta::TableFilesSchema update_files = {table_file, origin_file};
status = meta_ptr_->UpdateTableFiles(update_files); status = meta_ptr_->UpdateTableFiles(update_files);
if(status.ok()) { if (status.ok()) {
ENGINE_LOG_DEBUG << "New index file " << table_file.file_id_ << " of size " ENGINE_LOG_DEBUG << "New index file " << table_file.file_id_ << " of size "
<< index->PhysicalSize() << " bytes" << index->PhysicalSize() << " bytes"
<< " from file " << origin_file.file_id_; << " from file " << origin_file.file_id_;
if(options_.insert_cache_immediately_) { if (options_.insert_cache_immediately_) {
index->Cache(); index->Cache();
} }
} else { } else {
@ -850,8 +890,7 @@ Status DBImpl::BuildIndex(const meta::TableFileSchema& file) {
status = meta_ptr_->UpdateTableFile(table_file); status = meta_ptr_->UpdateTableFile(table_file);
ENGINE_LOG_DEBUG << "Failed to update file to index, mark file: " << table_file.file_id_ << " to to_delete"; ENGINE_LOG_DEBUG << "Failed to update file to index, mark file: " << table_file.file_id_ << " to to_delete";
} }
} catch (std::exception &ex) {
} catch (std::exception& ex) {
std::string msg = "Build index encounter exception: " + std::string(ex.what()); std::string msg = "Build index encounter exception: " + std::string(ex.what());
ENGINE_LOG_ERROR << msg; ENGINE_LOG_ERROR << msg;
return Status(DB_ERROR, msg); return Status(DB_ERROR, msg);
@ -860,20 +899,21 @@ Status DBImpl::BuildIndex(const meta::TableFileSchema& file) {
return Status::OK(); return Status::OK();
} }
void DBImpl::BackgroundBuildIndex() { void
DBImpl::BackgroundBuildIndex() {
ENGINE_LOG_TRACE << "Background build index thread start"; ENGINE_LOG_TRACE << "Background build index thread start";
std::unique_lock<std::mutex> lock(build_index_mutex_); std::unique_lock<std::mutex> lock(build_index_mutex_);
meta::TableFilesSchema to_index_files; meta::TableFilesSchema to_index_files;
meta_ptr_->FilesToIndex(to_index_files); meta_ptr_->FilesToIndex(to_index_files);
Status status; Status status;
for (auto& file : to_index_files) { for (auto &file : to_index_files) {
status = BuildIndex(file); status = BuildIndex(file);
if (!status.ok()) { if (!status.ok()) {
ENGINE_LOG_ERROR << "Building index for " << file.id_ << " failed: " << status.ToString(); ENGINE_LOG_ERROR << "Building index for " << file.id_ << " failed: " << status.ToString();
} }
if (shutting_down_.load(std::memory_order_acquire)){ if (shutting_down_.load(std::memory_order_acquire)) {
ENGINE_LOG_DEBUG << "Server will shutdown, skip build index action"; ENGINE_LOG_DEBUG << "Server will shutdown, skip build index action";
break; break;
} }

12
cpp/src/db/DBImpl.h
View File

@ -29,7 +29,8 @@
#include <thread> #include <thread>
#include <list> #include <list>
#include <set> #include <set>
#include <vector>
#include <string>
namespace zilliz { namespace zilliz {
namespace milvus { namespace milvus {
@ -68,11 +69,11 @@ class DBImpl : public DB {
Status InsertVectors(const std::string &table_id, uint64_t n, const float *vectors, IDNumbers &vector_ids) override; Status InsertVectors(const std::string &table_id, uint64_t n, const float *vectors, IDNumbers &vector_ids) override;
Status CreateIndex(const std::string& table_id, const TableIndex& index) override; Status CreateIndex(const std::string &table_id, const TableIndex &index) override;
Status DescribeIndex(const std::string& table_id, TableIndex& index) override; Status DescribeIndex(const std::string &table_id, TableIndex &index) override;
Status DropIndex(const std::string& table_id) override; Status DropIndex(const std::string &table_id) override;
Status Query(const std::string &table_id, Status Query(const std::string &table_id,
uint64_t k, uint64_t k,
@ -123,7 +124,7 @@ class DBImpl : public DB {
Status BackgroundMergeFiles(const std::string &table_id); Status BackgroundMergeFiles(const std::string &table_id);
void BackgroundCompaction(std::set<std::string> table_ids); void BackgroundCompaction(std::set<std::string> table_ids);
void StartBuildIndexTask(bool force=false); void StartBuildIndexTask(bool force = false);
void BackgroundBuildIndex(); void BackgroundBuildIndex();
Status BuildIndex(const meta::TableFileSchema &); Status BuildIndex(const meta::TableFileSchema &);
@ -151,7 +152,6 @@ class DBImpl : public DB {
std::list<std::future<void>> index_thread_results_; std::list<std::future<void>> index_thread_results_;
std::mutex build_index_mutex_; std::mutex build_index_mutex_;
}; // DBImpl }; // DBImpl

22
cpp/src/db/IDGenerator.cpp
View File

@ -15,7 +15,7 @@
// specific language governing permissions and limitations // specific language governing permissions and limitations
// under the License. // under the License.
#include "IDGenerator.h" #include "db/IDGenerator.h"
#include <chrono> #include <chrono>
#include <assert.h> #include <assert.h>
@ -29,16 +29,18 @@ IDGenerator::~IDGenerator() = default;
constexpr size_t SimpleIDGenerator::MAX_IDS_PER_MICRO; constexpr size_t SimpleIDGenerator::MAX_IDS_PER_MICRO;
IDNumber SimpleIDGenerator::GetNextIDNumber() { IDNumber
SimpleIDGenerator::GetNextIDNumber() {
auto now = std::chrono::system_clock::now(); auto now = std::chrono::system_clock::now();
auto micros = std::chrono::duration_cast<std::chrono::microseconds>( auto micros = std::chrono::duration_cast<std::chrono::microseconds>(
now.time_since_epoch()).count(); now.time_since_epoch()).count();
return micros * MAX_IDS_PER_MICRO; return micros * MAX_IDS_PER_MICRO;
} }
void SimpleIDGenerator::NextIDNumbers(size_t n, IDNumbers& ids) { void
SimpleIDGenerator::NextIDNumbers(size_t n, IDNumbers &ids) {
if (n > MAX_IDS_PER_MICRO) { if (n > MAX_IDS_PER_MICRO) {
NextIDNumbers(n-MAX_IDS_PER_MICRO, ids); NextIDNumbers(n - MAX_IDS_PER_MICRO, ids);
NextIDNumbers(MAX_IDS_PER_MICRO, ids); NextIDNumbers(MAX_IDS_PER_MICRO, ids);
return; return;
} }
@ -48,20 +50,20 @@ void SimpleIDGenerator::NextIDNumbers(size_t n, IDNumbers& ids) {
auto now = std::chrono::system_clock::now(); auto now = std::chrono::system_clock::now();
auto micros = std::chrono::duration_cast<std::chrono::microseconds>( auto micros = std::chrono::duration_cast<std::chrono::microseconds>(
now.time_since_epoch()).count(); now.time_since_epoch()).count();
micros *= MAX_IDS_PER_MICRO; micros *= MAX_IDS_PER_MICRO;
for (int pos=0; pos<n; ++pos) { for (int pos = 0; pos < n; ++pos) {
ids.push_back(micros+pos); ids.push_back(micros + pos);
} }
} }
void SimpleIDGenerator::GetNextIDNumbers(size_t n, IDNumbers& ids) { void
SimpleIDGenerator::GetNextIDNumbers(size_t n, IDNumbers &ids) {
ids.clear(); ids.clear();
NextIDNumbers(n, ids); NextIDNumbers(n, ids);
} }
} // namespace engine } // namespace engine
} // namespace milvus } // namespace milvus
} // namespace zilliz } // namespace zilliz

2
cpp/src/db/IDGenerator.h
View File

@ -22,7 +22,6 @@
#include <cstddef> #include <cstddef>
#include <vector> #include <vector>
namespace zilliz { namespace zilliz {
namespace milvus { namespace milvus {
namespace engine { namespace engine {
@ -55,7 +54,6 @@ class SimpleIDGenerator : public IDGenerator {
NextIDNumbers(size_t n, IDNumbers &ids); NextIDNumbers(size_t n, IDNumbers &ids);
static constexpr size_t MAX_IDS_PER_MICRO = 1000; static constexpr size_t MAX_IDS_PER_MICRO = 1000;
}; // SimpleIDGenerator }; // SimpleIDGenerator

23
cpp/src/db/Options.cpp
View File

@ -15,7 +15,7 @@
// specific language governing permissions and limitations // specific language governing permissions and limitations
// under the License. // under the License.
#include "Options.h" #include "db/Options.h"
#include "utils/Exception.h" #include "utils/Exception.h"
#include "utils/Log.h" #include "utils/Log.h"
@ -27,18 +27,20 @@ namespace zilliz {
namespace milvus { namespace milvus {
namespace engine { namespace engine {
ArchiveConf::ArchiveConf(const std::string& type, const std::string& criterias) { ArchiveConf::ArchiveConf(const std::string &type, const std::string &criterias) {
ParseType(type); ParseType(type);
ParseCritirias(criterias); ParseCritirias(criterias);
} }
void ArchiveConf::SetCriterias(const ArchiveConf::CriteriaT& criterial) { void
for(auto& pair : criterial) { ArchiveConf::SetCriterias(const ArchiveConf::CriteriaT &criterial) {
for (auto &pair : criterial) {
criterias_[pair.first] = pair.second; criterias_[pair.first] = pair.second;
} }
} }
void ArchiveConf::ParseCritirias(const std::string& criterias) { void
ArchiveConf::ParseCritirias(const std::string &criterias) {
std::stringstream ss(criterias); std::stringstream ss(criterias);
std::vector<std::string> tokens; std::vector<std::string> tokens;
@ -48,8 +50,8 @@ void ArchiveConf::ParseCritirias(const std::string& criterias) {
return; return;
} }
for (auto& token : tokens) { for (auto &token : tokens) {
if(token.empty()) { if (token.empty()) {
continue; continue;
} }
@ -67,12 +69,12 @@ void ArchiveConf::ParseCritirias(const std::string& criterias) {
auto value = std::stoi(kv[1]); auto value = std::stoi(kv[1]);
criterias_[kv[0]] = value; criterias_[kv[0]] = value;
} }
catch (std::out_of_range&){ catch (std::out_of_range &) {
std::string msg = "Out of range: '" + kv[1] + "'"; std::string msg = "Out of range: '" + kv[1] + "'";
ENGINE_LOG_ERROR << msg; ENGINE_LOG_ERROR << msg;
throw InvalidArgumentException(msg); throw InvalidArgumentException(msg);
} }
catch (...){ catch (...) {
std::string msg = "Invalid argument: '" + kv[1] + "'"; std::string msg = "Invalid argument: '" + kv[1] + "'";
ENGINE_LOG_ERROR << msg; ENGINE_LOG_ERROR << msg;
throw InvalidArgumentException(msg); throw InvalidArgumentException(msg);
@ -80,7 +82,8 @@ void ArchiveConf::ParseCritirias(const std::string& criterias) {
} }
} }
void ArchiveConf::ParseType(const std::string& type) { void
ArchiveConf::ParseType(const std::string &type) {
if (type != "delete" && type != "swap") { if (type != "delete" && type != "swap") {
std::string msg = "Invalid argument: type='" + type + "'"; std::string msg = "Invalid argument: type='" + type + "'";
throw InvalidArgumentException(msg); throw InvalidArgumentException(msg);

25
cpp/src/db/Options.h
View File

@ -30,22 +30,27 @@ namespace engine {
class Env; class Env;
static const char* ARCHIVE_CONF_DISK = "disk"; static const char *ARCHIVE_CONF_DISK = "disk";
static const char* ARCHIVE_CONF_DAYS = "days"; static const char *ARCHIVE_CONF_DAYS = "days";
struct ArchiveConf { struct ArchiveConf {
using CriteriaT = std::map<std::string, int>; using CriteriaT = std::map<std::string, int>;
ArchiveConf(const std::string& type, const std::string& criterias = std::string()); explicit ArchiveConf(const std::string &type, const std::string &criterias = std::string());
const std::string& GetType() const { return type_; } const std::string &GetType() const {
const CriteriaT GetCriterias() const { return criterias_; } return type_;
}
void SetCriterias(const ArchiveConf::CriteriaT& criterial); const CriteriaT GetCriterias() const {
return criterias_;
}
private: void SetCriterias(const ArchiveConf::CriteriaT &criterial);
void ParseCritirias(const std::string& type);
void ParseType(const std::string& criterias); private:
void ParseCritirias(const std::string &type);
void ParseType(const std::string &criterias);
std::string type_; std::string type_;
CriteriaT criterias_; CriteriaT criterias_;
@ -65,7 +70,7 @@ struct DBOptions {
CLUSTER_WRITABLE CLUSTER_WRITABLE
} MODE; } MODE;
uint16_t merge_trigger_number_ = 2; uint16_t merge_trigger_number_ = 2;
DBMetaOptions meta_; DBMetaOptions meta_;
int mode_ = MODE::SINGLE; int mode_ = MODE::SINGLE;

9
cpp/src/db/Types.h
View File

@ -21,22 +21,23 @@
#include <vector> #include <vector>
#include <stdint.h> #include <stdint.h>
#include <utility>
namespace zilliz { namespace zilliz {
namespace milvus { namespace milvus {
namespace engine { namespace engine {
typedef long IDNumber; typedef int64_t IDNumber;
typedef IDNumber* IDNumberPtr; typedef IDNumber *IDNumberPtr;
typedef std::vector<IDNumber> IDNumbers; typedef std::vector<IDNumber> IDNumbers;
typedef std::vector<std::pair<IDNumber, double>> QueryResult; typedef std::vector<std::pair<IDNumber, double>> QueryResult;
typedef std::vector<QueryResult> QueryResults; typedef std::vector<QueryResult> QueryResults;
struct TableIndex { struct TableIndex {
int32_t engine_type_ = (int)EngineType::FAISS_IDMAP; int32_t engine_type_ = (int) EngineType::FAISS_IDMAP;
int32_t nlist_ = 16384; int32_t nlist_ = 16384;
int32_t metric_type_ = (int)MetricType::L2; int32_t metric_type_ = (int) MetricType::L2;
}; };
} // namespace engine } // namespace engine

74
cpp/src/db/Utils.cpp
View File

@ -15,13 +15,14 @@
// specific language governing permissions and limitations // specific language governing permissions and limitations
// under the License. // under the License.
#include "Utils.h" #include "db/Utils.h"
#include "utils/CommonUtil.h" #include "utils/CommonUtil.h"
#include "utils/Log.h" #include "utils/Log.h"
#include <mutex> #include <mutex>
#include <chrono> #include <chrono>
#include <regex> #include <regex>
#include <vector>
#include <boost/filesystem.hpp> #include <boost/filesystem.hpp>
namespace zilliz { namespace zilliz {
@ -31,23 +32,25 @@ namespace utils {
namespace { namespace {
const char* TABLES_FOLDER = "/tables/"; const char *TABLES_FOLDER = "/tables/";
uint64_t index_file_counter = 0; uint64_t index_file_counter = 0;
std::mutex index_file_counter_mutex; std::mutex index_file_counter_mutex;
std::string ConstructParentFolder(const std::string& db_path, const meta::TableFileSchema& table_file) { std::string
ConstructParentFolder(const std::string &db_path, const meta::TableFileSchema &table_file) {
std::string table_path = db_path + TABLES_FOLDER + table_file.table_id_; std::string table_path = db_path + TABLES_FOLDER + table_file.table_id_;
std::string partition_path = table_path + "/" + std::to_string(table_file.date_); std::string partition_path = table_path + "/" + std::to_string(table_file.date_);
return partition_path; return partition_path;
} }
std::string GetTableFileParentFolder(const DBMetaOptions& options, const meta::TableFileSchema& table_file) { std::string
GetTableFileParentFolder(const DBMetaOptions &options, const meta::TableFileSchema &table_file) {
uint64_t path_count = options.slave_paths_.size() + 1; uint64_t path_count = options.slave_paths_.size() + 1;
std::string target_path = options.path_; std::string target_path = options.path_;
uint64_t index = 0; uint64_t index = 0;
if(meta::TableFileSchema::NEW_INDEX == table_file.file_type_) { if (meta::TableFileSchema::NEW_INDEX == table_file.file_type_) {
// index file is large file and to be persisted permanently // index file is large file and to be persisted permanently
// we need to distribute index files to each db_path averagely // we need to distribute index files to each db_path averagely
// round robin according to a file counter // round robin according to a file counter
@ -67,17 +70,19 @@ std::string GetTableFileParentFolder(const DBMetaOptions& options, const meta::T
return ConstructParentFolder(target_path, table_file); return ConstructParentFolder(target_path, table_file);
} }
} } // namespace
long GetMicroSecTimeStamp() { int64_t
GetMicroSecTimeStamp() {
auto now = std::chrono::system_clock::now(); auto now = std::chrono::system_clock::now();
auto micros = std::chrono::duration_cast<std::chrono::microseconds>( auto micros = std::chrono::duration_cast<std::chrono::microseconds>(
now.time_since_epoch()).count(); now.time_since_epoch()).count();
return micros; return micros;
} }
Status CreateTablePath(const DBMetaOptions& options, const std::string& table_id) { Status
CreateTablePath(const DBMetaOptions &options, const std::string &table_id) {
std::string db_path = options.path_; std::string db_path = options.path_;
std::string table_path = db_path + TABLES_FOLDER + table_id; std::string table_path = db_path + TABLES_FOLDER + table_id;
auto status = server::CommonUtil::CreateDirectory(table_path); auto status = server::CommonUtil::CreateDirectory(table_path);
@ -86,7 +91,7 @@ Status CreateTablePath(const DBMetaOptions& options, const std::string& table_id
return status; return status;
} }
for(auto& path : options.slave_paths_) { for (auto &path : options.slave_paths_) {
table_path = path + TABLES_FOLDER + table_id; table_path = path + TABLES_FOLDER + table_id;
status = server::CommonUtil::CreateDirectory(table_path); status = server::CommonUtil::CreateDirectory(table_path);
if (!status.ok()) { if (!status.ok()) {
@ -98,17 +103,18 @@ Status CreateTablePath(const DBMetaOptions& options, const std::string& table_id
return Status::OK(); return Status::OK();
} }
Status DeleteTablePath(const DBMetaOptions& options, const std::string& table_id, bool force) { Status
DeleteTablePath(const DBMetaOptions &options, const std::string &table_id, bool force) {
std::vector<std::string> paths = options.slave_paths_; std::vector<std::string> paths = options.slave_paths_;
paths.push_back(options.path_); paths.push_back(options.path_);
for(auto& path : paths) { for (auto &path : paths) {
std::string table_path = path + TABLES_FOLDER + table_id; std::string table_path = path + TABLES_FOLDER + table_id;
if(force) { if (force) {
boost::filesystem::remove_all(table_path); boost::filesystem::remove_all(table_path);
ENGINE_LOG_DEBUG << "Remove table folder: " << table_path; ENGINE_LOG_DEBUG << "Remove table folder: " << table_path;
} else if(boost::filesystem::exists(table_path) && } else if (boost::filesystem::exists(table_path) &&
boost::filesystem::is_empty(table_path)) { boost::filesystem::is_empty(table_path)) {
boost::filesystem::remove_all(table_path); boost::filesystem::remove_all(table_path);
ENGINE_LOG_DEBUG << "Remove table folder: " << table_path; ENGINE_LOG_DEBUG << "Remove table folder: " << table_path;
} }
@ -117,7 +123,8 @@ Status DeleteTablePath(const DBMetaOptions& options, const std::string& table_id
return Status::OK(); return Status::OK();
} }
Status CreateTableFilePath(const DBMetaOptions& options, meta::TableFileSchema& table_file) { Status
CreateTableFilePath(const DBMetaOptions &options, meta::TableFileSchema &table_file) {
std::string parent_path = GetTableFileParentFolder(options, table_file); std::string parent_path = GetTableFileParentFolder(options, table_file);
auto status = server::CommonUtil::CreateDirectory(parent_path); auto status = server::CommonUtil::CreateDirectory(parent_path);
@ -131,17 +138,18 @@ Status CreateTableFilePath(const DBMetaOptions& options, meta::TableFileSchema&
return Status::OK(); return Status::OK();
} }
Status GetTableFilePath(const DBMetaOptions& options, meta::TableFileSchema& table_file) { Status
GetTableFilePath(const DBMetaOptions &options, meta::TableFileSchema &table_file) {
std::string parent_path = ConstructParentFolder(options.path_, table_file); std::string parent_path = ConstructParentFolder(options.path_, table_file);
std::string file_path = parent_path + "/" + table_file.file_id_; std::string file_path = parent_path + "/" + table_file.file_id_;
if(boost::filesystem::exists(file_path)) { if (boost::filesystem::exists(file_path)) {
table_file.location_ = file_path; table_file.location_ = file_path;
return Status::OK(); return Status::OK();
} else { } else {
for(auto& path : options.slave_paths_) { for (auto &path : options.slave_paths_) {
parent_path = ConstructParentFolder(path, table_file); parent_path = ConstructParentFolder(path, table_file);
file_path = parent_path + "/" + table_file.file_id_; file_path = parent_path + "/" + table_file.file_id_;
if(boost::filesystem::exists(file_path)) { if (boost::filesystem::exists(file_path)) {
table_file.location_ = file_path; table_file.location_ = file_path;
return Status::OK(); return Status::OK();
} }
@ -155,49 +163,55 @@ Status GetTableFilePath(const DBMetaOptions& options, meta::TableFileSchema& tab
return Status(DB_ERROR, msg); return Status(DB_ERROR, msg);
} }
Status DeleteTableFilePath(const DBMetaOptions& options, meta::TableFileSchema& table_file) { Status
DeleteTableFilePath(const DBMetaOptions &options, meta::TableFileSchema &table_file) {
utils::GetTableFilePath(options, table_file); utils::GetTableFilePath(options, table_file);
boost::filesystem::remove(table_file.location_); boost::filesystem::remove(table_file.location_);
return Status::OK(); return Status::OK();
} }
bool IsSameIndex(const TableIndex& index1, const TableIndex& index2) { bool
IsSameIndex(const TableIndex &index1, const TableIndex &index2) {
return index1.engine_type_ == index2.engine_type_ return index1.engine_type_ == index2.engine_type_
&& index1.nlist_ == index2.nlist_ && index1.nlist_ == index2.nlist_
&& index1.metric_type_ == index2.metric_type_; && index1.metric_type_ == index2.metric_type_;
} }
meta::DateT GetDate(const std::time_t& t, int day_delta) { meta::DateT
GetDate(const std::time_t &t, int day_delta) {
struct tm ltm; struct tm ltm;
localtime_r(&t, &ltm); localtime_r(&t, &ltm);
if (day_delta > 0) { if (day_delta > 0) {
do { do {
++ltm.tm_mday; ++ltm.tm_mday;
--day_delta; --day_delta;
} while(day_delta > 0); } while (day_delta > 0);
mktime(&ltm); mktime(&ltm);
} else if (day_delta < 0) { } else if (day_delta < 0) {
do { do {
--ltm.tm_mday; --ltm.tm_mday;
++day_delta; ++day_delta;
} while(day_delta < 0); } while (day_delta < 0);
mktime(&ltm); mktime(&ltm);
} else { } else {
ltm.tm_mday; ltm.tm_mday;
} }
return ltm.tm_year*10000 + ltm.tm_mon*100 + ltm.tm_mday; return ltm.tm_year * 10000 + ltm.tm_mon * 100 + ltm.tm_mday;
} }
meta::DateT GetDateWithDelta(int day_delta) { meta::DateT
GetDateWithDelta(int day_delta) {
return GetDate(std::time(nullptr), day_delta); return GetDate(std::time(nullptr), day_delta);
} }
meta::DateT GetDate() { meta::DateT
GetDate() {
return GetDate(std::time(nullptr), 0); return GetDate(std::time(nullptr), 0);
} }
// URI format: dialect://username:password@host:port/database // URI format: dialect://username:password@host:port/database
Status ParseMetaUri(const std::string& uri, MetaUriInfo& info) { Status
ParseMetaUri(const std::string &uri, MetaUriInfo &info) {
std::string dialect_regex = "(.*)"; std::string dialect_regex = "(.*)";
std::string username_tegex = "(.*)"; std::string username_tegex = "(.*)";
std::string password_regex = "(.*)"; std::string password_regex = "(.*)";
@ -205,7 +219,7 @@ Status ParseMetaUri(const std::string& uri, MetaUriInfo& info) {
std::string port_regex = "(.*)"; std::string port_regex = "(.*)";
std::string db_name_regex = "(.*)"; std::string db_name_regex = "(.*)";
std::string uri_regex_str = std::string uri_regex_str =
dialect_regex + "\\:\\/\\/" + dialect_regex + "\\:\\/\\/" +
username_tegex + "\\:" + username_tegex + "\\:" +
password_regex + "\\@" + password_regex + "\\@" +
host_regex + "\\:" + host_regex + "\\:" +

33
cpp/src/db/Utils.h
View File

@ -29,20 +29,30 @@ namespace milvus {
namespace engine { namespace engine {
namespace utils { namespace utils {
long GetMicroSecTimeStamp(); int64_t
GetMicroSecTimeStamp();
Status CreateTablePath(const DBMetaOptions& options, const std::string& table_id); Status
Status DeleteTablePath(const DBMetaOptions& options, const std::string& table_id, bool force = true); CreateTablePath(const DBMetaOptions &options, const std::string &table_id);
Status
DeleteTablePath(const DBMetaOptions &options, const std::string &table_id, bool force = true);
Status CreateTableFilePath(const DBMetaOptions& options, meta::TableFileSchema& table_file); Status
Status GetTableFilePath(const DBMetaOptions& options, meta::TableFileSchema& table_file); CreateTableFilePath(const DBMetaOptions &options, meta::TableFileSchema &table_file);
Status DeleteTableFilePath(const DBMetaOptions& options, meta::TableFileSchema& table_file); Status
GetTableFilePath(const DBMetaOptions &options, meta::TableFileSchema &table_file);
Status
DeleteTableFilePath(const DBMetaOptions &options, meta::TableFileSchema &table_file);
bool IsSameIndex(const TableIndex& index1, const TableIndex& index2); bool
IsSameIndex(const TableIndex &index1, const TableIndex &index2);
meta::DateT GetDate(const std::time_t &t, int day_delta = 0); meta::DateT
meta::DateT GetDate(); GetDate(const std::time_t &t, int day_delta = 0);
meta::DateT GetDateWithDelta(int day_delta); meta::DateT
GetDate();
meta::DateT
GetDateWithDelta(int day_delta);
struct MetaUriInfo { struct MetaUriInfo {
std::string dialect_; std::string dialect_;
@ -53,7 +63,8 @@ struct MetaUriInfo {
std::string db_name_; std::string db_name_;
}; };
Status ParseMetaUri(const std::string& uri, MetaUriInfo& info); Status
ParseMetaUri(const std::string &uri, MetaUriInfo &info);
} // namespace utils } // namespace utils
} // namespace engine } // namespace engine

19
cpp/src/db/engine/EngineFactory.cpp
View File

@ -15,10 +15,12 @@
// specific language governing permissions and limitations // specific language governing permissions and limitations
// under the License. // under the License.
#include "EngineFactory.h" #include "db/engine/EngineFactory.h"
#include "ExecutionEngineImpl.h" #include "db/engine/ExecutionEngineImpl.h"
#include "utils/Log.h" #include "utils/Log.h"
#include <memory>
namespace zilliz { namespace zilliz {
namespace milvus { namespace milvus {
namespace engine { namespace engine {
@ -29,20 +31,19 @@ EngineFactory::Build(uint16_t dimension,
EngineType index_type, EngineType index_type,
MetricType metric_type, MetricType metric_type,
int32_t nlist) { int32_t nlist) {
if (index_type == EngineType::INVALID) {
if(index_type == EngineType::INVALID) {
ENGINE_LOG_ERROR << "Unsupported engine type"; ENGINE_LOG_ERROR << "Unsupported engine type";
return nullptr; return nullptr;
} }
ENGINE_LOG_DEBUG << "EngineFactory index type: " << (int)index_type; ENGINE_LOG_DEBUG << "EngineFactory index type: " << (int) index_type;
ExecutionEnginePtr execution_engine_ptr = ExecutionEnginePtr execution_engine_ptr =
std::make_shared<ExecutionEngineImpl>(dimension, location, index_type, metric_type, nlist); std::make_shared<ExecutionEngineImpl>(dimension, location, index_type, metric_type, nlist);
execution_engine_ptr->Init(); execution_engine_ptr->Init();
return execution_engine_ptr; return execution_engine_ptr;
} }
} } // namespace engine
} } // namespace milvus
} } // namespace zilliz

13
cpp/src/db/engine/EngineFactory.h
View File

@ -21,19 +21,22 @@
#include "ExecutionEngine.h" #include "ExecutionEngine.h"
#include "utils/Status.h" #include "utils/Status.h"
#include <string>
namespace zilliz { namespace zilliz {
namespace milvus { namespace milvus {
namespace engine { namespace engine {
class EngineFactory { class EngineFactory {
public: public:
static ExecutionEnginePtr Build(uint16_t dimension, static ExecutionEnginePtr Build(uint16_t dimension,
const std::string& location, const std::string &location,
EngineType index_type, EngineType index_type,
MetricType metric_type, MetricType metric_type,
int32_t nlist); int32_t nlist);
}; };
} } // namespace engine
} } // namespace milvus
} } // namespace zilliz

23
cpp/src/db/engine/ExecutionEngine.h
View File

@ -21,6 +21,7 @@
#include <vector> #include <vector>
#include <memory> #include <memory>
#include <string>
namespace zilliz { namespace zilliz {
namespace milvus { namespace milvus {
@ -41,9 +42,8 @@ enum class MetricType {
}; };
class ExecutionEngine { class ExecutionEngine {
public: public:
virtual Status AddWithIds(int64_t n, const float *xdata, const int64_t *xids) = 0;
virtual Status AddWithIds(long n, const float *xdata, const long *xids) = 0;
virtual size_t Count() const = 0; virtual size_t Count() const = 0;
@ -63,16 +63,16 @@ public:
virtual std::shared_ptr<ExecutionEngine> Clone() = 0; virtual std::shared_ptr<ExecutionEngine> Clone() = 0;
virtual Status Merge(const std::string& location) = 0; virtual Status Merge(const std::string &location) = 0;
virtual Status Search(long n, virtual Status Search(int64_t n,
const float *data, const float *data,
long k, int64_t k,
long nprobe, int64_t nprobe,
float *distances, float *distances,
long *labels) const = 0; int64_t *labels) const = 0;
virtual std::shared_ptr<ExecutionEngine> BuildIndex(const std::string& location, EngineType engine_type) = 0; virtual std::shared_ptr<ExecutionEngine> BuildIndex(const std::string &location, EngineType engine_type) = 0;
virtual Status Cache() = 0; virtual Status Cache() = 0;
@ -89,7 +89,6 @@ public:
using ExecutionEnginePtr = std::shared_ptr<ExecutionEngine>; using ExecutionEnginePtr = std::shared_ptr<ExecutionEngine>;
} // namespace engine } // namespace engine
} // namespace milvus } // namespace milvus
} // namespace zilliz } // namespace zilliz

87
cpp/src/db/engine/ExecutionEngineImpl.cpp
View File

@ -15,7 +15,7 @@
// specific language governing permissions and limitations // specific language governing permissions and limitations
// under the License. // under the License.
#include "ExecutionEngineImpl.h" #include "db/engine/ExecutionEngineImpl.h"
#include "cache/GpuCacheMgr.h" #include "cache/GpuCacheMgr.h"
#include "cache/CpuCacheMgr.h" #include "cache/CpuCacheMgr.h"
#include "metrics/Metrics.h" #include "metrics/Metrics.h"
@ -29,6 +29,7 @@
#include "server/Config.h" #include "server/Config.h"
#include <stdexcept> #include <stdexcept>
#include <utility>
namespace zilliz { namespace zilliz {
namespace milvus { namespace milvus {
@ -72,7 +73,8 @@ ExecutionEngineImpl::ExecutionEngineImpl(VecIndexPtr index,
nlist_(nlist) { nlist_(nlist) {
} }
VecIndexPtr ExecutionEngineImpl::CreatetVecIndex(EngineType type) { VecIndexPtr
ExecutionEngineImpl::CreatetVecIndex(EngineType type) {
std::shared_ptr<VecIndex> index; std::shared_ptr<VecIndex> index;
switch (type) { switch (type) {
case EngineType::FAISS_IDMAP: { case EngineType::FAISS_IDMAP: {
@ -99,41 +101,48 @@ VecIndexPtr ExecutionEngineImpl::CreatetVecIndex(EngineType type) {
return index; return index;
} }
Status ExecutionEngineImpl::AddWithIds(long n, const float *xdata, const long *xids) { Status
ExecutionEngineImpl::AddWithIds(int64_t n, const float *xdata, const int64_t *xids) {
auto status = index_->Add(n, xdata, xids); auto status = index_->Add(n, xdata, xids);
return status; return status;
} }
size_t ExecutionEngineImpl::Count() const { size_t
if(index_ == nullptr) { ExecutionEngineImpl::Count() const {
if (index_ == nullptr) {
ENGINE_LOG_ERROR << "ExecutionEngineImpl: index is null, return count 0"; ENGINE_LOG_ERROR << "ExecutionEngineImpl: index is null, return count 0";
return 0; return 0;
} }
return index_->Count(); return index_->Count();
} }
size_t ExecutionEngineImpl::Size() const { size_t
ExecutionEngineImpl::Size() const {
return (size_t) (Count() * Dimension()) * sizeof(float); return (size_t) (Count() * Dimension()) * sizeof(float);
} }
size_t ExecutionEngineImpl::Dimension() const { size_t
if(index_ == nullptr) { ExecutionEngineImpl::Dimension() const {
if (index_ == nullptr) {
ENGINE_LOG_ERROR << "ExecutionEngineImpl: index is null, return dimension " << dim_; ENGINE_LOG_ERROR << "ExecutionEngineImpl: index is null, return dimension " << dim_;
return dim_; return dim_;
} }
return index_->Dimension(); return index_->Dimension();
} }
size_t ExecutionEngineImpl::PhysicalSize() const { size_t
ExecutionEngineImpl::PhysicalSize() const {
return server::CommonUtil::GetFileSize(location_); return server::CommonUtil::GetFileSize(location_);
} }
Status ExecutionEngineImpl::Serialize() { Status
ExecutionEngineImpl::Serialize() {
auto status = write_index(index_, location_); auto status = write_index(index_, location_);
return status; return status;
} }
Status ExecutionEngineImpl::Load(bool to_cache) { Status
ExecutionEngineImpl::Load(bool to_cache) {
index_ = cache::CpuCacheMgr::GetInstance()->GetIndex(location_); index_ = cache::CpuCacheMgr::GetInstance()->GetIndex(location_);
bool already_in_cache = (index_ != nullptr); bool already_in_cache = (index_ != nullptr);
if (!already_in_cache) { if (!already_in_cache) {
@ -141,7 +150,7 @@ Status ExecutionEngineImpl::Load(bool to_cache) {
double physical_size = PhysicalSize(); double physical_size = PhysicalSize();
server::CollectExecutionEngineMetrics metrics(physical_size); server::CollectExecutionEngineMetrics metrics(physical_size);
index_ = read_index(location_); index_ = read_index(location_);
if(index_ == nullptr) { if (index_ == nullptr) {
std::string msg = "Failed to load index from " + location_; std::string msg = "Failed to load index from " + location_;
ENGINE_LOG_ERROR << msg; ENGINE_LOG_ERROR << msg;
return Status(DB_ERROR, msg); return Status(DB_ERROR, msg);
@ -160,13 +169,14 @@ Status ExecutionEngineImpl::Load(bool to_cache) {
return Status::OK(); return Status::OK();
} }
Status ExecutionEngineImpl::CopyToGpu(uint64_t device_id) { Status
ExecutionEngineImpl::CopyToGpu(uint64_t device_id) {
auto index = cache::GpuCacheMgr::GetInstance(device_id)->GetIndex(location_); auto index = cache::GpuCacheMgr::GetInstance(device_id)->GetIndex(location_);
bool already_in_cache = (index != nullptr); bool already_in_cache = (index != nullptr);
if (already_in_cache) { if (already_in_cache) {
index_ = index; index_ = index;
} else { } else {
if(index_ == nullptr) { if (index_ == nullptr) {
ENGINE_LOG_ERROR << "ExecutionEngineImpl: index is null, failed to copy to gpu"; ENGINE_LOG_ERROR << "ExecutionEngineImpl: index is null, failed to copy to gpu";
return Status(DB_ERROR, "index is null"); return Status(DB_ERROR, "index is null");
} }
@ -187,13 +197,14 @@ Status ExecutionEngineImpl::CopyToGpu(uint64_t device_id) {
return Status::OK(); return Status::OK();
} }
Status ExecutionEngineImpl::CopyToCpu() { Status
ExecutionEngineImpl::CopyToCpu() {
auto index = cache::CpuCacheMgr::GetInstance()->GetIndex(location_); auto index = cache::CpuCacheMgr::GetInstance()->GetIndex(location_);
bool already_in_cache = (index != nullptr); bool already_in_cache = (index != nullptr);
if (already_in_cache) { if (already_in_cache) {
index_ = index; index_ = index;
} else { } else {
if(index_ == nullptr) { if (index_ == nullptr) {
ENGINE_LOG_ERROR << "ExecutionEngineImpl: index is null, failed to copy to cpu"; ENGINE_LOG_ERROR << "ExecutionEngineImpl: index is null, failed to copy to cpu";
return Status(DB_ERROR, "index is null"); return Status(DB_ERROR, "index is null");
} }
@ -213,8 +224,9 @@ Status ExecutionEngineImpl::CopyToCpu() {
return Status::OK(); return Status::OK();
} }
ExecutionEnginePtr ExecutionEngineImpl::Clone() { ExecutionEnginePtr
if(index_ == nullptr) { ExecutionEngineImpl::Clone() {
if (index_ == nullptr) {
ENGINE_LOG_ERROR << "ExecutionEngineImpl: index is null, failed to clone"; ENGINE_LOG_ERROR << "ExecutionEngineImpl: index is null, failed to clone";
return nullptr; return nullptr;
} }
@ -225,7 +237,8 @@ ExecutionEnginePtr ExecutionEngineImpl::Clone() {
return ret; return ret;
} }
Status ExecutionEngineImpl::Merge(const std::string &location) { Status
ExecutionEngineImpl::Merge(const std::string &location) {
if (location == location_) { if (location == location_) {
return Status(DB_ERROR, "Cannot Merge Self"); return Status(DB_ERROR, "Cannot Merge Self");
} }
@ -243,7 +256,7 @@ Status ExecutionEngineImpl::Merge(const std::string &location) {
} }
} }
if(index_ == nullptr) { if (index_ == nullptr) {
ENGINE_LOG_ERROR << "ExecutionEngineImpl: index is null, failed to merge"; ENGINE_LOG_ERROR << "ExecutionEngineImpl: index is null, failed to merge";
return Status(DB_ERROR, "index is null"); return Status(DB_ERROR, "index is null");
} }
@ -264,7 +277,7 @@ ExecutionEngineImpl::BuildIndex(const std::string &location, EngineType engine_t
ENGINE_LOG_DEBUG << "Build index file: " << location << " from: " << location_; ENGINE_LOG_DEBUG << "Build index file: " << location << " from: " << location_;
auto from_index = std::dynamic_pointer_cast<BFIndex>(index_); auto from_index = std::dynamic_pointer_cast<BFIndex>(index_);
if(from_index == nullptr) { if (from_index == nullptr) {
ENGINE_LOG_ERROR << "ExecutionEngineImpl: from_index is null, failed to build index"; ENGINE_LOG_ERROR << "ExecutionEngineImpl: from_index is null, failed to build index";
return nullptr; return nullptr;
} }
@ -282,21 +295,22 @@ ExecutionEngineImpl::BuildIndex(const std::string &location, EngineType engine_t
AutoGenParams(to_index->GetType(), Count(), build_cfg); AutoGenParams(to_index->GetType(), Count(), build_cfg);
auto status = to_index->BuildAll(Count(), auto status = to_index->BuildAll(Count(),
from_index->GetRawVectors(), from_index->GetRawVectors(),
from_index->GetRawIds(), from_index->GetRawIds(),
build_cfg); build_cfg);
if (!status.ok()) { throw Exception(DB_ERROR, status.message()); } if (!status.ok()) { throw Exception(DB_ERROR, status.message()); }
return std::make_shared<ExecutionEngineImpl>(to_index, location, engine_type, metric_type_, nlist_); return std::make_shared<ExecutionEngineImpl>(to_index, location, engine_type, metric_type_, nlist_);
} }
Status ExecutionEngineImpl::Search(long n, Status
const float *data, ExecutionEngineImpl::Search(int64_t n,
long k, const float *data,
long nprobe, int64_t k,
float *distances, int64_t nprobe,
long *labels) const { float *distances,
if(index_ == nullptr) { int64_t *labels) const {
if (index_ == nullptr) {
ENGINE_LOG_ERROR << "ExecutionEngineImpl: index is null, failed to search"; ENGINE_LOG_ERROR << "ExecutionEngineImpl: index is null, failed to search";
return Status(DB_ERROR, "index is null"); return Status(DB_ERROR, "index is null");
} }
@ -310,14 +324,16 @@ Status ExecutionEngineImpl::Search(long n,
return status; return status;
} }
Status ExecutionEngineImpl::Cache() { Status
ExecutionEngineImpl::Cache() {
cache::DataObjPtr obj = std::make_shared<cache::DataObj>(index_, PhysicalSize()); cache::DataObjPtr obj = std::make_shared<cache::DataObj>(index_, PhysicalSize());
zilliz::milvus::cache::CpuCacheMgr::GetInstance()->InsertItem(location_, obj); zilliz::milvus::cache::CpuCacheMgr::GetInstance()->InsertItem(location_, obj);
return Status::OK(); return Status::OK();
} }
Status ExecutionEngineImpl::GpuCache(uint64_t gpu_id) { Status
ExecutionEngineImpl::GpuCache(uint64_t gpu_id) {
cache::DataObjPtr obj = std::make_shared<cache::DataObj>(index_, PhysicalSize()); cache::DataObjPtr obj = std::make_shared<cache::DataObj>(index_, PhysicalSize());
zilliz::milvus::cache::GpuCacheMgr::GetInstance(gpu_id)->InsertItem(location_, obj); zilliz::milvus::cache::GpuCacheMgr::GetInstance(gpu_id)->InsertItem(location_, obj);
@ -325,8 +341,8 @@ Status ExecutionEngineImpl::GpuCache(uint64_t gpu_id) {
} }
// TODO(linxj): remove. // TODO(linxj): remove.
Status ExecutionEngineImpl::Init() { Status
using namespace zilliz::milvus::server; ExecutionEngineImpl::Init() {
server::Config &config = server::Config::GetInstance(); server::Config &config = server::Config::GetInstance();
Status s = config.GetDBConfigBuildIndexGPU(gpu_num_); Status s = config.GetDBConfigBuildIndexGPU(gpu_num_);
if (!s.ok()) return s; if (!s.ok()) return s;
@ -334,7 +350,6 @@ Status ExecutionEngineImpl::Init() {
return Status::OK(); return Status::OK();
} }
} // namespace engine } // namespace engine
} // namespace milvus } // namespace milvus
} // namespace zilliz } // namespace zilliz

32
cpp/src/db/engine/ExecutionEngineImpl.h
View File

@ -23,15 +23,12 @@
#include <memory> #include <memory>
#include <string> #include <string>
namespace zilliz { namespace zilliz {
namespace milvus { namespace milvus {
namespace engine { namespace engine {
class ExecutionEngineImpl : public ExecutionEngine { class ExecutionEngineImpl : public ExecutionEngine {
public: public:
ExecutionEngineImpl(uint16_t dimension, ExecutionEngineImpl(uint16_t dimension,
const std::string &location, const std::string &location,
EngineType index_type, EngineType index_type,
@ -44,7 +41,7 @@ public:
MetricType metric_type, MetricType metric_type,
int32_t nlist); int32_t nlist);
Status AddWithIds(long n, const float *xdata, const long *xids) override; Status AddWithIds(int64_t n, const float *xdata, const int64_t *xids) override;
size_t Count() const override; size_t Count() const override;
@ -66,12 +63,12 @@ public:
Status Merge(const std::string &location) override; Status Merge(const std::string &location) override;
Status Search(long n, Status Search(int64_t n,
const float *data, const float *data,
long k, int64_t k,
long nprobe, int64_t nprobe,
float *distances, float *distances,
long *labels) const override; int64_t *labels) const override;
ExecutionEnginePtr BuildIndex(const std::string &location, EngineType engine_type) override; ExecutionEnginePtr BuildIndex(const std::string &location, EngineType engine_type) override;
@ -81,18 +78,24 @@ public:
Status Init() override; Status Init() override;
EngineType IndexEngineType() const override { return index_type_; } EngineType IndexEngineType() const override {
return index_type_;
}
MetricType IndexMetricType() const override { return metric_type_; } MetricType IndexMetricType() const override {
return metric_type_;
}
std::string GetLocation() const override { return location_; } std::string GetLocation() const override {
return location_;
}
private: private:
VecIndexPtr CreatetVecIndex(EngineType type); VecIndexPtr CreatetVecIndex(EngineType type);
VecIndexPtr Load(const std::string &location); VecIndexPtr Load(const std::string &location);
protected: protected:
VecIndexPtr index_ = nullptr; VecIndexPtr index_ = nullptr;
EngineType index_type_; EngineType index_type_;
MetricType metric_type_; MetricType metric_type_;
@ -104,7 +107,6 @@ protected:
int32_t gpu_num_ = 0; int32_t gpu_num_ = 0;
}; };
} // namespace engine } // namespace engine
} // namespace milvus } // namespace milvus
} // namespace zilliz } // namespace zilliz

5
cpp/src/db/insert/MemManager.h
View File

@ -23,6 +23,7 @@
#include <set> #include <set>
#include <memory> #include <memory>
#include <string>
namespace zilliz { namespace zilliz {
namespace milvus { namespace milvus {
@ -30,7 +31,6 @@ namespace engine {
class MemManager { class MemManager {
public: public:
virtual Status InsertVectors(const std::string &table_id, virtual Status InsertVectors(const std::string &table_id,
size_t n, const float *vectors, IDNumbers &vector_ids) = 0; size_t n, const float *vectors, IDNumbers &vector_ids) = 0;
@ -43,11 +43,10 @@ class MemManager {
virtual size_t GetCurrentImmutableMem() = 0; virtual size_t GetCurrentImmutableMem() = 0;
virtual size_t GetCurrentMem() = 0; virtual size_t GetCurrentMem() = 0;
}; // MemManagerAbstract }; // MemManagerAbstract
using MemManagerPtr = std::shared_ptr<MemManager>; using MemManagerPtr = std::shared_ptr<MemManager>;
} // namespace engine } // namespace engine
} // namespace milvus } // namespace milvus
} // namespace zilliz } // namespace zilliz

48
cpp/src/db/insert/MemManagerImpl.cpp
View File

@ -16,19 +16,19 @@
// under the License. // under the License.
#include "MemManagerImpl.h" #include "db/insert/MemManagerImpl.h"
#include "VectorSource.h" #include "VectorSource.h"
#include "utils/Log.h" #include "utils/Log.h"
#include "db/Constants.h" #include "db/Constants.h"
#include <thread> #include <thread>
namespace zilliz { namespace zilliz {
namespace milvus { namespace milvus {
namespace engine { namespace engine {
MemTablePtr MemManagerImpl::GetMemByTable(const std::string &table_id) { MemTablePtr
MemManagerImpl::GetMemByTable(const std::string &table_id) {
auto memIt = mem_id_map_.find(table_id); auto memIt = mem_id_map_.find(table_id);
if (memIt != mem_id_map_.end()) { if (memIt != mem_id_map_.end()) {
return memIt->second; return memIt->second;
@ -38,11 +38,11 @@ MemTablePtr MemManagerImpl::GetMemByTable(const std::string &table_id) {
return mem_id_map_[table_id]; return mem_id_map_[table_id];
} }
Status MemManagerImpl::InsertVectors(const std::string &table_id_, Status
size_t n_, MemManagerImpl::InsertVectors(const std::string &table_id_,
const float *vectors_, size_t n_,
IDNumbers &vector_ids_) { const float *vectors_,
IDNumbers &vector_ids_) {
while (GetCurrentMem() > options_.insert_buffer_size_) { while (GetCurrentMem() > options_.insert_buffer_size_) {
std::this_thread::sleep_for(std::chrono::milliseconds(1)); std::this_thread::sleep_for(std::chrono::milliseconds(1));
} }
@ -52,11 +52,11 @@ Status MemManagerImpl::InsertVectors(const std::string &table_id_,
return InsertVectorsNoLock(table_id_, n_, vectors_, vector_ids_); return InsertVectorsNoLock(table_id_, n_, vectors_, vector_ids_);
} }
Status MemManagerImpl::InsertVectorsNoLock(const std::string &table_id, Status
size_t n, MemManagerImpl::InsertVectorsNoLock(const std::string &table_id,
const float *vectors, size_t n,
IDNumbers &vector_ids) { const float *vectors,
IDNumbers &vector_ids) {
MemTablePtr mem = GetMemByTable(table_id); MemTablePtr mem = GetMemByTable(table_id);
VectorSourcePtr source = std::make_shared<VectorSource>(n, vectors); VectorSourcePtr source = std::make_shared<VectorSource>(n, vectors);
@ -69,10 +69,11 @@ Status MemManagerImpl::InsertVectorsNoLock(const std::string &table_id,
return status; return status;
} }
Status MemManagerImpl::ToImmutable() { Status
MemManagerImpl::ToImmutable() {
std::unique_lock<std::mutex> lock(mutex_); std::unique_lock<std::mutex> lock(mutex_);
MemIdMap temp_map; MemIdMap temp_map;
for (auto &kv: mem_id_map_) { for (auto &kv : mem_id_map_) {
if (kv.second->Empty()) { if (kv.second->Empty()) {
//empty table, no need to serialize //empty table, no need to serialize
temp_map.insert(kv); temp_map.insert(kv);
@ -85,7 +86,8 @@ Status MemManagerImpl::ToImmutable() {
return Status::OK(); return Status::OK();
} }
Status MemManagerImpl::Serialize(std::set<std::string> &table_ids) { Status
MemManagerImpl::Serialize(std::set<std::string> &table_ids) {
ToImmutable(); ToImmutable();
std::unique_lock<std::mutex> lock(serialization_mtx_); std::unique_lock<std::mutex> lock(serialization_mtx_);
table_ids.clear(); table_ids.clear();
@ -97,7 +99,8 @@ Status MemManagerImpl::Serialize(std::set<std::string> &table_ids) {
return Status::OK(); return Status::OK();
} }
Status MemManagerImpl::EraseMemVector(const std::string &table_id) { Status
MemManagerImpl::EraseMemVector(const std::string &table_id) {
{//erase MemVector from rapid-insert cache {//erase MemVector from rapid-insert cache
std::unique_lock<std::mutex> lock(mutex_); std::unique_lock<std::mutex> lock(mutex_);
mem_id_map_.erase(table_id); mem_id_map_.erase(table_id);
@ -117,7 +120,8 @@ Status MemManagerImpl::EraseMemVector(const std::string &table_id) {
return Status::OK(); return Status::OK();
} }
size_t MemManagerImpl::GetCurrentMutableMem() { size_t
MemManagerImpl::GetCurrentMutableMem() {
size_t total_mem = 0; size_t total_mem = 0;
for (auto &kv : mem_id_map_) { for (auto &kv : mem_id_map_) {
auto memTable = kv.second; auto memTable = kv.second;
@ -126,7 +130,8 @@ size_t MemManagerImpl::GetCurrentMutableMem() {
return total_mem; return total_mem;
} }
size_t MemManagerImpl::GetCurrentImmutableMem() { size_t
MemManagerImpl::GetCurrentImmutableMem() {
size_t total_mem = 0; size_t total_mem = 0;
for (auto &mem_table : immu_mem_list_) { for (auto &mem_table : immu_mem_list_) {
total_mem += mem_table->GetCurrentMem(); total_mem += mem_table->GetCurrentMem();
@ -134,10 +139,11 @@ size_t MemManagerImpl::GetCurrentImmutableMem() {
return total_mem; return total_mem;
} }
size_t MemManagerImpl::GetCurrentMem() { size_t
MemManagerImpl::GetCurrentMem() {
return GetCurrentMutableMem() + GetCurrentImmutableMem(); return GetCurrentMutableMem() + GetCurrentImmutableMem();
} }
} // namespace engine } // namespace engine
} // namespace milvus } // namespace milvus
} // namespace zilliz } // namespace zilliz

9
cpp/src/db/insert/MemManagerImpl.h
View File

@ -24,12 +24,13 @@
#include "utils/Status.h" #include "utils/Status.h"
#include <map> #include <map>
#include <set>
#include <vector>
#include <string> #include <string>
#include <ctime> #include <ctime>
#include <memory> #include <memory>
#include <mutex> #include <mutex>
namespace zilliz { namespace zilliz {
namespace milvus { namespace milvus {
namespace engine { namespace engine {
@ -39,7 +40,8 @@ class MemManagerImpl : public MemManager {
using Ptr = std::shared_ptr<MemManagerImpl>; using Ptr = std::shared_ptr<MemManagerImpl>;
MemManagerImpl(const meta::MetaPtr &meta, const DBOptions &options) MemManagerImpl(const meta::MetaPtr &meta, const DBOptions &options)
: meta_(meta), options_(options) {} : meta_(meta), options_(options) {
}
Status InsertVectors(const std::string &table_id, Status InsertVectors(const std::string &table_id,
size_t n, const float *vectors, IDNumbers &vector_ids) override; size_t n, const float *vectors, IDNumbers &vector_ids) override;
@ -71,7 +73,6 @@ class MemManagerImpl : public MemManager {
std::mutex serialization_mtx_; std::mutex serialization_mtx_;
}; // NewMemManager }; // NewMemManager
} // namespace engine } // namespace engine
} // namespace milvus } // namespace milvus
} // namespace zilliz } // namespace zilliz

6
cpp/src/db/insert/MemMenagerFactory.cpp
View File

@ -15,7 +15,7 @@
// specific language governing permissions and limitations // specific language governing permissions and limitations
// under the License. // under the License.
#include "MemMenagerFactory.h" #include "db/insert/MemMenagerFactory.h"
#include "MemManagerImpl.h" #include "MemManagerImpl.h"
#include "utils/Log.h" #include "utils/Log.h"
#include "utils/Exception.h" #include "utils/Exception.h"
@ -26,12 +26,14 @@
#include <cstdlib> #include <cstdlib>
#include <string> #include <string>
#include <regex> #include <regex>
#include <memory>
namespace zilliz { namespace zilliz {
namespace milvus { namespace milvus {
namespace engine { namespace engine {
MemManagerPtr MemManagerFactory::Build(const std::shared_ptr<meta::Meta>& meta, const DBOptions& options) { MemManagerPtr
MemManagerFactory::Build(const std::shared_ptr<meta::Meta> &meta, const DBOptions &options) {
return std::make_shared<MemManagerImpl>(meta, options); return std::make_shared<MemManagerImpl>(meta, options);
} }

10
cpp/src/db/insert/MemMenagerFactory.h
View File

@ -20,15 +20,17 @@
#include "MemManager.h" #include "MemManager.h"
#include "db/meta/Meta.h" #include "db/meta/Meta.h"
#include <memory>
namespace zilliz { namespace zilliz {
namespace milvus { namespace milvus {
namespace engine { namespace engine {
class MemManagerFactory { class MemManagerFactory {
public: public:
static MemManagerPtr Build(const std::shared_ptr<meta::Meta> &meta, const DBOptions &options); static MemManagerPtr Build(const std::shared_ptr<meta::Meta> &meta, const DBOptions &options);
}; };
} } // namespace engine
} } // namespace milvus
} } // namespace zilliz

30
cpp/src/db/insert/MemTable.cpp
View File

@ -16,9 +16,11 @@
// under the License. // under the License.
#include "MemTable.h" #include "db/insert/MemTable.h"
#include "utils/Log.h" #include "utils/Log.h"
#include <memory>
#include <string>
namespace zilliz { namespace zilliz {
namespace milvus { namespace milvus {
@ -30,13 +32,11 @@ MemTable::MemTable(const std::string &table_id,
table_id_(table_id), table_id_(table_id),
meta_(meta), meta_(meta),
options_(options) { options_(options) {
} }
Status MemTable::Add(VectorSourcePtr &source, IDNumbers &vector_ids) { Status
MemTable::Add(VectorSourcePtr &source, IDNumbers &vector_ids) {
while (!source->AllAdded()) { while (!source->AllAdded()) {
MemTableFilePtr current_mem_table_file; MemTableFilePtr current_mem_table_file;
if (!mem_table_file_list_.empty()) { if (!mem_table_file_list_.empty()) {
current_mem_table_file = mem_table_file_list_.back(); current_mem_table_file = mem_table_file_list_.back();
@ -62,15 +62,18 @@ Status MemTable::Add(VectorSourcePtr &source, IDNumbers &vector_ids) {
return Status::OK(); return Status::OK();
} }
void MemTable::GetCurrentMemTableFile(MemTableFilePtr &mem_table_file) { void
MemTable::GetCurrentMemTableFile(MemTableFilePtr &mem_table_file) {
mem_table_file = mem_table_file_list_.back(); mem_table_file = mem_table_file_list_.back();
} }
size_t MemTable::GetTableFileCount() { size_t
MemTable::GetTableFileCount() {
return mem_table_file_list_.size(); return mem_table_file_list_.size();
} }
Status MemTable::Serialize() { Status
MemTable::Serialize() {
for (auto mem_table_file = mem_table_file_list_.begin(); mem_table_file != mem_table_file_list_.end();) { for (auto mem_table_file = mem_table_file_list_.begin(); mem_table_file != mem_table_file_list_.end();) {
auto status = (*mem_table_file)->Serialize(); auto status = (*mem_table_file)->Serialize();
if (!status.ok()) { if (!status.ok()) {
@ -84,15 +87,18 @@ Status MemTable::Serialize() {
return Status::OK(); return Status::OK();
} }
bool MemTable::Empty() { bool
MemTable::Empty() {
return mem_table_file_list_.empty(); return mem_table_file_list_.empty();
} }
const std::string &MemTable::GetTableId() const { const std::string &
MemTable::GetTableId() const {
return table_id_; return table_id_;
} }
size_t MemTable::GetCurrentMem() { size_t
MemTable::GetCurrentMem() {
std::lock_guard<std::mutex> lock(mutex_); std::lock_guard<std::mutex> lock(mutex_);
size_t total_mem = 0; size_t total_mem = 0;
for (auto &mem_table_file : mem_table_file_list_) { for (auto &mem_table_file : mem_table_file_list_) {
@ -103,4 +109,4 @@ size_t MemTable::GetCurrentMem() {
} // namespace engine } // namespace engine
} // namespace milvus } // namespace milvus
} // namespace zilliz } // namespace zilliz

7
cpp/src/db/insert/MemTable.h
View File

@ -23,7 +23,9 @@
#include "utils/Status.h" #include "utils/Status.h"
#include <mutex> #include <mutex>
#include <vector>
#include <memory>
#include <string>
namespace zilliz { namespace zilliz {
namespace milvus { namespace milvus {
@ -59,11 +61,10 @@ class MemTable {
DBOptions options_; DBOptions options_;
std::mutex mutex_; std::mutex mutex_;
}; //MemTable }; //MemTable
using MemTablePtr = std::shared_ptr<MemTable>; using MemTablePtr = std::shared_ptr<MemTable>;
} // namespace engine } // namespace engine
} // namespace milvus } // namespace milvus
} // namespace zilliz } // namespace zilliz

38
cpp/src/db/insert/MemTableFile.cpp
View File

@ -16,14 +16,14 @@
// under the License. // under the License.
#include "MemTableFile.h" #include "db/insert/MemTableFile.h"
#include "db/Constants.h" #include "db/Constants.h"
#include "db/engine/EngineFactory.h" #include "db/engine/EngineFactory.h"
#include "metrics/Metrics.h" #include "metrics/Metrics.h"
#include "utils/Log.h" #include "utils/Log.h"
#include <cmath> #include <cmath>
#include <string>
namespace zilliz { namespace zilliz {
namespace milvus { namespace milvus {
@ -35,20 +35,19 @@ MemTableFile::MemTableFile(const std::string &table_id,
table_id_(table_id), table_id_(table_id),
meta_(meta), meta_(meta),
options_(options) { options_(options) {
current_mem_ = 0; current_mem_ = 0;
auto status = CreateTableFile(); auto status = CreateTableFile();
if (status.ok()) { if (status.ok()) {
execution_engine_ = EngineFactory::Build(table_file_schema_.dimension_, execution_engine_ = EngineFactory::Build(table_file_schema_.dimension_,
table_file_schema_.location_, table_file_schema_.location_,
(EngineType) table_file_schema_.engine_type_, (EngineType) table_file_schema_.engine_type_,
(MetricType)table_file_schema_.metric_type_, (MetricType) table_file_schema_.metric_type_,
table_file_schema_.nlist_); table_file_schema_.nlist_);
} }
} }
Status MemTableFile::CreateTableFile() { Status
MemTableFile::CreateTableFile() {
meta::TableFileSchema table_file_schema; meta::TableFileSchema table_file_schema;
table_file_schema.table_id_ = table_id_; table_file_schema.table_id_ = table_id_;
auto status = meta_->CreateTableFile(table_file_schema); auto status = meta_->CreateTableFile(table_file_schema);
@ -61,8 +60,8 @@ Status MemTableFile::CreateTableFile() {
return status; return status;
} }
Status MemTableFile::Add(const VectorSourcePtr &source, IDNumbers& vector_ids) { Status
MemTableFile::Add(const VectorSourcePtr &source, IDNumbers &vector_ids) {
if (table_file_schema_.dimension_ <= 0) { if (table_file_schema_.dimension_ <= 0) {
std::string err_msg = "MemTableFile::Add: table_file_schema dimension = " + std::string err_msg = "MemTableFile::Add: table_file_schema dimension = " +
std::to_string(table_file_schema_.dimension_) + ", table_id = " + table_file_schema_.table_id_; std::to_string(table_file_schema_.dimension_) + ", table_id = " + table_file_schema_.table_id_;
@ -75,7 +74,8 @@ Status MemTableFile::Add(const VectorSourcePtr &source, IDNumbers& vector_ids) {
if (mem_left >= single_vector_mem_size) { if (mem_left >= single_vector_mem_size) {
size_t num_vectors_to_add = std::ceil(mem_left / single_vector_mem_size); size_t num_vectors_to_add = std::ceil(mem_left / single_vector_mem_size);
size_t num_vectors_added; size_t num_vectors_added;
auto status = source->Add(execution_engine_, table_file_schema_, num_vectors_to_add, num_vectors_added, vector_ids); auto status =
source->Add(execution_engine_, table_file_schema_, num_vectors_to_add, num_vectors_added, vector_ids);
if (status.ok()) { if (status.ok()) {
current_mem_ += (num_vectors_added * single_vector_mem_size); current_mem_ += (num_vectors_added * single_vector_mem_size);
} }
@ -84,20 +84,24 @@ Status MemTableFile::Add(const VectorSourcePtr &source, IDNumbers& vector_ids) {
return Status::OK(); return Status::OK();
} }
size_t MemTableFile::GetCurrentMem() { size_t
MemTableFile::GetCurrentMem() {
return current_mem_; return current_mem_;
} }
size_t MemTableFile::GetMemLeft() { size_t
MemTableFile::GetMemLeft() {
return (MAX_TABLE_FILE_MEM - current_mem_); return (MAX_TABLE_FILE_MEM - current_mem_);
} }
bool MemTableFile::IsFull() { bool
MemTableFile::IsFull() {
size_t single_vector_mem_size = table_file_schema_.dimension_ * VECTOR_TYPE_SIZE; size_t single_vector_mem_size = table_file_schema_.dimension_ * VECTOR_TYPE_SIZE;
return (GetMemLeft() < single_vector_mem_size); return (GetMemLeft() < single_vector_mem_size);
} }
Status MemTableFile::Serialize() { Status
MemTableFile::Serialize() {
size_t size = GetCurrentMem(); size_t size = GetCurrentMem();
server::CollectSerializeMetrics metrics(size); server::CollectSerializeMetrics metrics(size);
@ -107,7 +111,7 @@ Status MemTableFile::Serialize() {
//if index type isn't IDMAP, set file type to TO_INDEX if file size execeed index_file_size //if index type isn't IDMAP, set file type to TO_INDEX if file size execeed index_file_size
//else set file type to RAW, no need to build index //else set file type to RAW, no need to build index
if (table_file_schema_.engine_type_ != (int)EngineType::FAISS_IDMAP) { if (table_file_schema_.engine_type_ != (int) EngineType::FAISS_IDMAP) {
table_file_schema_.file_type_ = (size >= table_file_schema_.index_file_size_) ? table_file_schema_.file_type_ = (size >= table_file_schema_.index_file_size_) ?
meta::TableFileSchema::TO_INDEX : meta::TableFileSchema::RAW; meta::TableFileSchema::TO_INDEX : meta::TableFileSchema::RAW;
} else { } else {
@ -117,9 +121,9 @@ Status MemTableFile::Serialize() {
auto status = meta_->UpdateTableFile(table_file_schema_); auto status = meta_->UpdateTableFile(table_file_schema_);
ENGINE_LOG_DEBUG << "New " << ((table_file_schema_.file_type_ == meta::TableFileSchema::RAW) ? "raw" : "to_index") ENGINE_LOG_DEBUG << "New " << ((table_file_schema_.file_type_ == meta::TableFileSchema::RAW) ? "raw" : "to_index")
<< " file " << table_file_schema_.file_id_ << " of size " << size << " bytes"; << " file " << table_file_schema_.file_id_ << " of size " << size << " bytes";
if(options_.insert_cache_immediately_) { if (options_.insert_cache_immediately_) {
execution_engine_->Cache(); execution_engine_->Cache();
} }
@ -128,4 +132,4 @@ Status MemTableFile::Serialize() {
} // namespace engine } // namespace engine
} // namespace milvus } // namespace milvus
} // namespace zilliz } // namespace zilliz

14
cpp/src/db/insert/MemTableFile.h
View File

@ -23,17 +23,18 @@
#include "db/engine/ExecutionEngine.h" #include "db/engine/ExecutionEngine.h"
#include "utils/Status.h" #include "utils/Status.h"
#include <string>
#include <memory>
namespace zilliz { namespace zilliz {
namespace milvus { namespace milvus {
namespace engine { namespace engine {
class MemTableFile { class MemTableFile {
public: public:
MemTableFile(const std::string &table_id, const meta::MetaPtr &meta, const DBOptions &options); MemTableFile(const std::string &table_id, const meta::MetaPtr &meta, const DBOptions &options);
Status Add(const VectorSourcePtr &source, IDNumbers& vector_ids); Status Add(const VectorSourcePtr &source, IDNumbers &vector_ids);
size_t GetCurrentMem(); size_t GetCurrentMem();
@ -44,25 +45,20 @@ class MemTableFile {
Status Serialize(); Status Serialize();
private: private:
Status CreateTableFile(); Status CreateTableFile();
private:
const std::string table_id_; const std::string table_id_;
meta::TableFileSchema table_file_schema_; meta::TableFileSchema table_file_schema_;
meta::MetaPtr meta_; meta::MetaPtr meta_;
DBOptions options_; DBOptions options_;
size_t current_mem_; size_t current_mem_;
ExecutionEnginePtr execution_engine_; ExecutionEnginePtr execution_engine_;
}; //MemTableFile }; //MemTableFile
using MemTableFilePtr = std::shared_ptr<MemTableFile>; using MemTableFilePtr = std::shared_ptr<MemTableFile>;
} // namespace engine } // namespace engine
} // namespace milvus } // namespace milvus
} // namespace zilliz } // namespace zilliz

35
cpp/src/db/insert/VectorSource.cpp
View File

@ -16,32 +16,30 @@
// under the License. // under the License.
#include "VectorSource.h" #include "db/insert/VectorSource.h"
#include "db/engine/ExecutionEngine.h" #include "db/engine/ExecutionEngine.h"
#include "db/engine/EngineFactory.h" #include "db/engine/EngineFactory.h"
#include "utils/Log.h" #include "utils/Log.h"
#include "metrics/Metrics.h" #include "metrics/Metrics.h"
namespace zilliz { namespace zilliz {
namespace milvus { namespace milvus {
namespace engine { namespace engine {
VectorSource::VectorSource(const size_t &n, VectorSource::VectorSource(const size_t &n,
const float *vectors) : const float *vectors) :
n_(n), n_(n),
vectors_(vectors), vectors_(vectors),
id_generator_(std::make_shared<SimpleIDGenerator>()) { id_generator_(std::make_shared<SimpleIDGenerator>()) {
current_num_vectors_added = 0; current_num_vectors_added = 0;
} }
Status VectorSource::Add(const ExecutionEnginePtr &execution_engine, Status
const meta::TableFileSchema &table_file_schema, VectorSource::Add(const ExecutionEnginePtr &execution_engine,
const size_t &num_vectors_to_add, const meta::TableFileSchema &table_file_schema,
size_t &num_vectors_added, const size_t &num_vectors_to_add,
IDNumbers &vector_ids) { size_t &num_vectors_added,
IDNumbers &vector_ids) {
server::CollectAddMetrics metrics(n_, table_file_schema.dimension_); server::CollectAddMetrics metrics(n_, table_file_schema.dimension_);
num_vectors_added = current_num_vectors_added + num_vectors_to_add <= n_ ? num_vectors_added = current_num_vectors_added + num_vectors_to_add <= n_ ?
@ -52,7 +50,7 @@ Status VectorSource::Add(const ExecutionEnginePtr &execution_engine,
} else { } else {
vector_ids_to_add.resize(num_vectors_added); vector_ids_to_add.resize(num_vectors_added);
for (int pos = current_num_vectors_added; pos < current_num_vectors_added + num_vectors_added; pos++) { for (int pos = current_num_vectors_added; pos < current_num_vectors_added + num_vectors_added; pos++) {
vector_ids_to_add[pos-current_num_vectors_added] = vector_ids[pos]; vector_ids_to_add[pos - current_num_vectors_added] = vector_ids[pos];
} }
} }
Status status = execution_engine->AddWithIds(num_vectors_added, Status status = execution_engine->AddWithIds(num_vectors_added,
@ -70,18 +68,21 @@ Status VectorSource::Add(const ExecutionEnginePtr &execution_engine,
return status; return status;
} }
size_t VectorSource::GetNumVectorsAdded() { size_t
VectorSource::GetNumVectorsAdded() {
return current_num_vectors_added; return current_num_vectors_added;
} }
bool VectorSource::AllAdded() { bool
VectorSource::AllAdded() {
return (current_num_vectors_added == n_); return (current_num_vectors_added == n_);
} }
IDNumbers VectorSource::GetVectorIds() { IDNumbers
VectorSource::GetVectorIds() {
return vector_ids_; return vector_ids_;
} }
} // namespace engine } // namespace engine
} // namespace milvus } // namespace milvus
} // namespace zilliz } // namespace zilliz

5
cpp/src/db/insert/VectorSource.h
View File

@ -23,6 +23,7 @@
#include "db/engine/ExecutionEngine.h" #include "db/engine/ExecutionEngine.h"
#include "utils/Status.h" #include "utils/Status.h"
#include <memory>
namespace zilliz { namespace zilliz {
namespace milvus { namespace milvus {
@ -45,7 +46,6 @@ class VectorSource {
IDNumbers GetVectorIds(); IDNumbers GetVectorIds();
private: private:
const size_t n_; const size_t n_;
const float *vectors_; const float *vectors_;
IDNumbers vector_ids_; IDNumbers vector_ids_;
@ -53,11 +53,10 @@ class VectorSource {
size_t current_num_vectors_added; size_t current_num_vectors_added;
std::shared_ptr<IDGenerator> id_generator_; std::shared_ptr<IDGenerator> id_generator_;
}; //VectorSource }; //VectorSource
using VectorSourcePtr = std::shared_ptr<VectorSource>; using VectorSourcePtr = std::shared_ptr<VectorSource>;
} // namespace engine } // namespace engine
} // namespace milvus } // namespace milvus
} // namespace zilliz } // namespace zilliz

13
cpp/src/db/meta/Meta.h
View File

@ -25,14 +25,16 @@
#include <cstddef> #include <cstddef>
#include <memory> #include <memory>
#include <vector>
#include <string>
namespace zilliz { namespace zilliz {
namespace milvus { namespace milvus {
namespace engine { namespace engine {
namespace meta { namespace meta {
static const char* META_TABLES = "Tables"; static const char *META_TABLES = "Tables";
static const char* META_TABLEFILES = "TableFiles"; static const char *META_TABLEFILES = "TableFiles";
class Meta { class Meta {
public: public:
@ -46,7 +48,7 @@ class Meta {
virtual Status AllTables(std::vector<TableSchema> &table_schema_array) = 0; virtual Status AllTables(std::vector<TableSchema> &table_schema_array) = 0;
virtual Status UpdateTableIndex(const std::string &table_id, const TableIndex& index) = 0; virtual Status UpdateTableIndex(const std::string &table_id, const TableIndex &index) = 0;
virtual Status UpdateTableFlag(const std::string &table_id, int64_t flag) = 0; virtual Status UpdateTableFlag(const std::string &table_id, int64_t flag) = 0;
@ -83,9 +85,9 @@ class Meta {
virtual Status FilesByType(const std::string &table_id, virtual Status FilesByType(const std::string &table_id,
const std::vector<int> &file_types, const std::vector<int> &file_types,
std::vector<std::string>& file_ids) = 0; std::vector<std::string> &file_ids) = 0;
virtual Status DescribeTableIndex(const std::string &table_id, TableIndex& index) = 0; virtual Status DescribeTableIndex(const std::string &table_id, TableIndex &index) = 0;
virtual Status DropTableIndex(const std::string &table_id) = 0; virtual Status DropTableIndex(const std::string &table_id) = 0;
@ -96,7 +98,6 @@ class Meta {
virtual Status DropAll() = 0; virtual Status DropAll() = 0;
virtual Status Count(const std::string &table_id, uint64_t &result) = 0; virtual Status Count(const std::string &table_id, uint64_t &result) = 0;
}; // MetaData }; // MetaData
using MetaPtr = std::shared_ptr<Meta>; using MetaPtr = std::shared_ptr<Meta>;

20
cpp/src/db/meta/MetaConsts.h
View File

@ -23,20 +23,20 @@ namespace engine {
namespace meta { namespace meta {
const size_t K = 1024UL; const size_t K = 1024UL;
const size_t M = K*K; const size_t M = K * K;
const size_t G = K*M; const size_t G = K * M;
const size_t T = K*G; const size_t T = K * G;
const size_t S_PS = 1UL; const size_t S_PS = 1UL;
const size_t MS_PS = 1000*S_PS; const size_t MS_PS = 1000 * S_PS;
const size_t US_PS = 1000*MS_PS; const size_t US_PS = 1000 * MS_PS;
const size_t NS_PS = 1000*US_PS; const size_t NS_PS = 1000 * US_PS;
const size_t SECOND = 1UL; const size_t SECOND = 1UL;
const size_t M_SEC = 60*SECOND; const size_t M_SEC = 60 * SECOND;
const size_t H_SEC = 60*M_SEC; const size_t H_SEC = 60 * M_SEC;
const size_t D_SEC = 24*H_SEC; const size_t D_SEC = 24 * H_SEC;
const size_t W_SEC = 7*D_SEC; const size_t W_SEC = 7 * D_SEC;
} // namespace meta } // namespace meta
} // namespace engine } // namespace engine

67
cpp/src/db/meta/MetaFactory.cpp
View File

@ -15,7 +15,7 @@
// specific language governing permissions and limitations // specific language governing permissions and limitations
// under the License. // under the License.
#include "MetaFactory.h" #include "db/meta/MetaFactory.h"
#include "SqliteMetaImpl.h" #include "SqliteMetaImpl.h"
#include "MySQLMetaImpl.h" #include "MySQLMetaImpl.h"
#include "utils/Log.h" #include "utils/Log.h"
@ -28,47 +28,52 @@
#include <cstdlib> #include <cstdlib>
#include <string> #include <string>
#include <string.h> #include <string.h>
#include <memory>
namespace zilliz { namespace zilliz {
namespace milvus { namespace milvus {
namespace engine { namespace engine {
DBMetaOptions MetaFactory::BuildOption(const std::string &path) { DBMetaOptions
auto p = path; MetaFactory::BuildOption(const std::string &path) {
if(p == "") { auto p = path;
srand(time(nullptr)); if (p == "") {
std::stringstream ss; srand(time(nullptr));
ss << "/tmp/" << rand(); std::stringstream ss;
p = ss.str(); uint32_t rd = 0;
} rand_r(&rd);
ss << "/tmp/" << rd;
DBMetaOptions meta; p = ss.str();
meta.path_ = p;
return meta;
} }
meta::MetaPtr MetaFactory::Build(const DBMetaOptions &metaOptions, const int &mode) { DBMetaOptions meta;
std::string uri = metaOptions.backend_uri_; meta.path_ = p;
return meta;
}
utils::MetaUriInfo uri_info; meta::MetaPtr
auto status = utils::ParseMetaUri(uri, uri_info); MetaFactory::Build(const DBMetaOptions &metaOptions, const int &mode) {
if(!status.ok()) { std::string uri = metaOptions.backend_uri_;
ENGINE_LOG_ERROR << "Wrong URI format: URI = " << uri;
throw InvalidArgumentException("Wrong URI format ");
}
if (strcasecmp(uri_info.dialect_.c_str(), "mysql") == 0) { utils::MetaUriInfo uri_info;
ENGINE_LOG_INFO << "Using MySQL"; auto status = utils::ParseMetaUri(uri, uri_info);
return std::make_shared<meta::MySQLMetaImpl>(metaOptions, mode); if (!status.ok()) {
} else if (strcasecmp(uri_info.dialect_.c_str(), "sqlite") == 0) { ENGINE_LOG_ERROR << "Wrong URI format: URI = " << uri;
ENGINE_LOG_INFO << "Using SQLite"; throw InvalidArgumentException("Wrong URI format ");
return std::make_shared<meta::SqliteMetaImpl>(metaOptions);
} else {
ENGINE_LOG_ERROR << "Invalid dialect in URI: dialect = " << uri_info.dialect_;
throw InvalidArgumentException("URI dialect is not mysql / sqlite");
}
} }
if (strcasecmp(uri_info.dialect_.c_str(), "mysql") == 0) {
ENGINE_LOG_INFO << "Using MySQL";
return std::make_shared<meta::MySQLMetaImpl>(metaOptions, mode);
} else if (strcasecmp(uri_info.dialect_.c_str(), "sqlite") == 0) {
ENGINE_LOG_INFO << "Using SQLite";
return std::make_shared<meta::SqliteMetaImpl>(metaOptions);
} else {
ENGINE_LOG_ERROR << "Invalid dialect in URI: dialect = " << uri_info.dialect_;
throw InvalidArgumentException("URI dialect is not mysql / sqlite");
}
}
} // namespace engine } // namespace engine
} // namespace milvus } // namespace milvus
} // namespace zilliz } // namespace zilliz

11
cpp/src/db/meta/MetaFactory.h
View File

@ -20,18 +20,19 @@
#include "Meta.h" #include "Meta.h"
#include "db/Options.h" #include "db/Options.h"
#include <string>
namespace zilliz { namespace zilliz {
namespace milvus { namespace milvus {
namespace engine { namespace engine {
class MetaFactory { class MetaFactory {
public: public:
static DBMetaOptions BuildOption(const std::string &path = ""); static DBMetaOptions BuildOption(const std::string &path = "");
static meta::MetaPtr Build(const DBMetaOptions &metaOptions, const int &mode); static meta::MetaPtr Build(const DBMetaOptions &metaOptions, const int &mode);
}; };
} // namespace engine
} } // namespace milvus
} } // namespace zilliz
}

11
cpp/src/db/meta/MetaTypes.h
View File

@ -23,21 +23,22 @@
#include <vector> #include <vector>
#include <map> #include <map>
#include <string> #include <string>
#include <memory>
namespace zilliz { namespace zilliz {
namespace milvus { namespace milvus {
namespace engine { namespace engine {
namespace meta { namespace meta {
constexpr int32_t DEFAULT_ENGINE_TYPE = (int)EngineType::FAISS_IDMAP; constexpr int32_t DEFAULT_ENGINE_TYPE = (int) EngineType::FAISS_IDMAP;
constexpr int32_t DEFAULT_NLIST = 16384; constexpr int32_t DEFAULT_NLIST = 16384;
constexpr int32_t DEFAULT_METRIC_TYPE = (int)MetricType::L2; constexpr int32_t DEFAULT_METRIC_TYPE = (int) MetricType::L2;
constexpr int32_t DEFAULT_INDEX_FILE_SIZE = ONE_GB; constexpr int32_t DEFAULT_INDEX_FILE_SIZE = ONE_GB;
constexpr int64_t FLAG_MASK_NO_USERID = 0x1; constexpr int64_t FLAG_MASK_NO_USERID = 0x1;
constexpr int64_t FLAG_MASK_HAS_USERID = 0x1<<1; constexpr int64_t FLAG_MASK_HAS_USERID = 0x1 << 1;
using DateT = int ; using DateT = int;
const DateT EmptyDate = -1; const DateT EmptyDate = -1;
using DatesT = std::vector<DateT>; using DatesT = std::vector<DateT>;
@ -49,7 +50,7 @@ struct TableSchema {
size_t id_ = 0; size_t id_ = 0;
std::string table_id_; std::string table_id_;
int32_t state_ = (int)NORMAL; int32_t state_ = (int) NORMAL;
uint16_t dimension_ = 0; uint16_t dimension_ = 0;
int64_t created_on_ = 0; int64_t created_on_ = 0;
int64_t flag_ = 0; int64_t flag_ = 0;

105
cpp/src/db/meta/MySQLConnectionPool.cpp
View File

@ -16,37 +16,39 @@
// under the License. // under the License.
#include "MySQLConnectionPool.h" #include "db/meta/MySQLConnectionPool.h"
namespace zilliz { namespace zilliz {
namespace milvus { namespace milvus {
namespace engine { namespace engine {
namespace meta { namespace meta {
// Do a simple form of in-use connection limiting: wait to return // Do a simple form of in-use connection limiting: wait to return
// a connection until there are a reasonably low number in use // a connection until there are a reasonably low number in use
// already. Can't do this in create() because we're interested in // already. Can't do this in create() because we're interested in
// connections actually in use, not those created. Also note that // connections actually in use, not those created. Also note that
// we keep our own count; ConnectionPool::size() isn't the same! // we keep our own count; ConnectionPool::size() isn't the same!
mysqlpp::Connection *MySQLConnectionPool::grab() { mysqlpp::Connection *
while (conns_in_use_ > max_pool_size_) { MySQLConnectionPool::grab() {
sleep(1); while (conns_in_use_ > max_pool_size_) {
} sleep(1);
++conns_in_use_;
return mysqlpp::ConnectionPool::grab();
} }
// Other half of in-use conn count limit ++conns_in_use_;
void MySQLConnectionPool::release(const mysqlpp::Connection *pc) { return mysqlpp::ConnectionPool::grab();
mysqlpp::ConnectionPool::release(pc); }
if (conns_in_use_ <= 0) { // Other half of in-use conn count limit
ENGINE_LOG_WARNING << "MySQLConnetionPool::release: conns_in_use_ is less than zero. conns_in_use_ = " << conns_in_use_; void
} else { MySQLConnectionPool::release(const mysqlpp::Connection *pc) {
--conns_in_use_; mysqlpp::ConnectionPool::release(pc);
} if (conns_in_use_ <= 0) {
ENGINE_LOG_WARNING << "MySQLConnetionPool::release: conns_in_use_ is less than zero. conns_in_use_ = "
<< conns_in_use_;
} else {
--conns_in_use_;
} }
}
// int MySQLConnectionPool::getConnectionsInUse() { // int MySQLConnectionPool::getConnectionsInUse() {
// return conns_in_use_; // return conns_in_use_;
@ -56,39 +58,42 @@ namespace meta {
// max_idle_time_ = max_idle; // max_idle_time_ = max_idle;
// } // }
std::string MySQLConnectionPool::getDB() { std::string
return db_; MySQLConnectionPool::getDB() {
} return db_;
}
// Superclass overrides // Superclass overrides
mysqlpp::Connection *MySQLConnectionPool::create() { mysqlpp::Connection *
MySQLConnectionPool::create() {
try { try {
// Create connection using the parameters we were passed upon // Create connection using the parameters we were passed upon
// creation. // creation.
mysqlpp::Connection *conn = new mysqlpp::Connection(); mysqlpp::Connection *conn = new mysqlpp::Connection();
conn->set_option(new mysqlpp::ReconnectOption(true)); conn->set_option(new mysqlpp::ReconnectOption(true));
conn->connect(db_.empty() ? 0 : db_.c_str(), conn->connect(db_.empty() ? 0 : db_.c_str(),
server_.empty() ? 0 : server_.c_str(), server_.empty() ? 0 : server_.c_str(),
user_.empty() ? 0 : user_.c_str(), user_.empty() ? 0 : user_.c_str(),
password_.empty() ? 0 : password_.c_str(), password_.empty() ? 0 : password_.c_str(),
port_); port_);
return conn; return conn;
} catch (const mysqlpp::ConnectionFailed& er) { } catch (const mysqlpp::ConnectionFailed &er) {
ENGINE_LOG_ERROR << "Failed to connect to database server" << ": " << er.what(); ENGINE_LOG_ERROR << "Failed to connect to database server" << ": " << er.what();
return nullptr; return nullptr;
}
} }
}
void MySQLConnectionPool::destroy(mysqlpp::Connection *cp) { void
// Our superclass can't know how we created the Connection, so MySQLConnectionPool::destroy(mysqlpp::Connection *cp) {
// it delegates destruction to us, to be safe. // Our superclass can't know how we created the Connection, so
delete cp; // it delegates destruction to us, to be safe.
} delete cp;
}
unsigned int MySQLConnectionPool::max_idle_time() { unsigned int
return max_idle_time_; MySQLConnectionPool::max_idle_time() {
} return max_idle_time_;
}
} // namespace meta } // namespace meta
} // namespace engine } // namespace engine

24
cpp/src/db/meta/MySQLConnectionPool.h
View File

@ -30,8 +30,7 @@ namespace engine {
namespace meta { namespace meta {
class MySQLConnectionPool : public mysqlpp::ConnectionPool { class MySQLConnectionPool : public mysqlpp::ConnectionPool {
public:
public:
// The object's only constructor // The object's only constructor
MySQLConnectionPool(std::string dbName, MySQLConnectionPool(std::string dbName,
std::string userName, std::string userName,
@ -39,15 +38,13 @@ public:
std::string serverIp, std::string serverIp,
int port = 0, int port = 0,
int maxPoolSize = 8) : int maxPoolSize = 8) :
db_(dbName), db_(dbName),
user_(userName), user_(userName),
password_(passWord), password_(passWord),
server_(serverIp), server_(serverIp),
port_(port), port_(port),
max_pool_size_(maxPoolSize) { max_pool_size_(maxPoolSize) {
conns_in_use_ = 0; conns_in_use_ = 0;
max_idle_time_ = 10; //10 seconds max_idle_time_ = 10; //10 seconds
} }
@ -68,8 +65,7 @@ public:
std::string getDB(); std::string getDB();
protected: protected:
// Superclass overrides // Superclass overrides
mysqlpp::Connection *create() override; mysqlpp::Connection *create() override;
@ -77,7 +73,7 @@ protected:
unsigned int max_idle_time() override; unsigned int max_idle_time() override;
private: private:
// Number of connections currently in use // Number of connections currently in use
std::atomic<int> conns_in_use_; std::atomic<int> conns_in_use_;
@ -93,4 +89,4 @@ private:
} // namespace meta } // namespace meta
} // namespace engine } // namespace engine
} // namespace milvus } // namespace milvus
} // namespace zilliz } // namespace zilliz

590
cpp/src/db/meta/MySQLMetaImpl.cpp
View File

File diff suppressed because it is too large Load Diff

21
cpp/src/db/meta/MySQLMetaImpl.h
View File

@ -21,26 +21,25 @@
#include "db/Options.h" #include "db/Options.h"
#include "MySQLConnectionPool.h" #include "MySQLConnectionPool.h"
#include "mysql++/mysql++.h" #include <mysql++/mysql++.h>
#include <mutex> #include <mutex>
#include <vector>
#include <string>
#include <memory>
namespace zilliz { namespace zilliz {
namespace milvus { namespace milvus {
namespace engine { namespace engine {
namespace meta { namespace meta {
// auto StoragePrototype(const std::string& path);
using namespace mysqlpp;
class MySQLMetaImpl : public Meta { class MySQLMetaImpl : public Meta {
public: public:
MySQLMetaImpl(const DBMetaOptions &options_, const int &mode); MySQLMetaImpl(const DBMetaOptions &options, const int &mode);
~MySQLMetaImpl(); ~MySQLMetaImpl();
Status CreateTable(TableSchema &table_schema) override; Status CreateTable(TableSchema &table_schema) override;
Status DescribeTable(TableSchema &group_info_) override; Status DescribeTable(TableSchema &table_schema) override;
Status HasTable(const std::string &table_id, bool &has_or_not) override; Status HasTable(const std::string &table_id, bool &has_or_not) override;
@ -63,11 +62,11 @@ class MySQLMetaImpl : public Meta {
const std::vector<int> &file_types, const std::vector<int> &file_types,
std::vector<std::string> &file_ids) override; std::vector<std::string> &file_ids) override;
Status UpdateTableIndex(const std::string &table_id, const TableIndex& index) override; Status UpdateTableIndex(const std::string &table_id, const TableIndex &index) override;
Status UpdateTableFlag(const std::string &table_id, int64_t flag) override; Status UpdateTableFlag(const std::string &table_id, int64_t flag) override;
Status DescribeTableIndex(const std::string &table_id, TableIndex& index) override; Status DescribeTableIndex(const std::string &table_id, TableIndex &index) override;
Status DropTableIndex(const std::string &table_id) override; Status DropTableIndex(const std::string &table_id) override;
@ -102,12 +101,12 @@ class MySQLMetaImpl : public Meta {
private: private:
Status NextFileId(std::string &file_id); Status NextFileId(std::string &file_id);
Status NextTableId(std::string &table_id); Status NextTableId(std::string &table_id);
Status DiscardFiles(long long to_discard_size); Status DiscardFiles(int64_t to_discard_size);
void ValidateMetaSchema(); void ValidateMetaSchema();
Status Initialize(); Status Initialize();
private: private:
const DBMetaOptions options_; const DBMetaOptions options_;
const int mode_; const int mode_;

438
cpp/src/db/meta/SqliteMetaImpl.cpp
View File

File diff suppressed because it is too large Load Diff

17
cpp/src/db/meta/SqliteMetaImpl.h
View File

@ -21,22 +21,25 @@
#include "db/Options.h" #include "db/Options.h"
#include <mutex> #include <mutex>
#include <vector>
#include <string>
namespace zilliz { namespace zilliz {
namespace milvus { namespace milvus {
namespace engine { namespace engine {
namespace meta { namespace meta {
auto StoragePrototype(const std::string &path); auto
StoragePrototype(const std::string &path);
class SqliteMetaImpl : public Meta { class SqliteMetaImpl : public Meta {
public: public:
explicit SqliteMetaImpl(const DBMetaOptions &options_); explicit SqliteMetaImpl(const DBMetaOptions &options);
~SqliteMetaImpl(); ~SqliteMetaImpl();
Status CreateTable(TableSchema &table_schema) override; Status CreateTable(TableSchema &table_schema) override;
Status DescribeTable(TableSchema &group_info_) override; Status DescribeTable(TableSchema &table_schema) override;
Status HasTable(const std::string &table_id, bool &has_or_not) override; Status HasTable(const std::string &table_id, bool &has_or_not) override;
@ -58,11 +61,11 @@ class SqliteMetaImpl : public Meta {
const std::vector<int> &file_types, const std::vector<int> &file_types,
std::vector<std::string> &file_ids) override; std::vector<std::string> &file_ids) override;
Status UpdateTableIndex(const std::string &table_id, const TableIndex& index) override; Status UpdateTableIndex(const std::string &table_id, const TableIndex &index) override;
Status UpdateTableFlag(const std::string &table_id, int64_t flag) override; Status UpdateTableFlag(const std::string &table_id, int64_t flag) override;
Status DescribeTableIndex(const std::string &table_id, TableIndex& index) override; Status DescribeTableIndex(const std::string &table_id, TableIndex &index) override;
Status DropTableIndex(const std::string &table_id) override; Status DropTableIndex(const std::string &table_id) override;
@ -96,12 +99,12 @@ class SqliteMetaImpl : public Meta {
private: private:
Status NextFileId(std::string &file_id); Status NextFileId(std::string &file_id);
Status NextTableId(std::string &table_id); Status NextTableId(std::string &table_id);
Status DiscardFiles(long to_discard_size); Status DiscardFiles(int64_t to_discard_size);
void ValidateMetaSchema(); void ValidateMetaSchema();
Status Initialize(); Status Initialize();
private: private:
const DBMetaOptions options_; const DBMetaOptions options_;
std::mutex meta_mutex_; std::mutex meta_mutex_;
}; // DBMetaImpl }; // DBMetaImpl

16
cpp/src/scheduler/Algorithm.cpp
View File

@ -16,20 +16,23 @@
// under the License. // under the License.
#include "Algorithm.h" #include "scheduler/Algorithm.h"
#include <limits>
#include <unordered_map>
#include <utility>
namespace zilliz { namespace zilliz {
namespace milvus { namespace milvus {
namespace scheduler { namespace scheduler {
constexpr uint64_t MAXINT = std::numeric_limits<uint32_t >::max(); constexpr uint64_t MAXINT = std::numeric_limits<uint32_t>::max();
uint64_t uint64_t
ShortestPath(const ResourcePtr &src, ShortestPath(const ResourcePtr &src,
const ResourcePtr &dest, const ResourcePtr &dest,
const ResourceMgrPtr &res_mgr, const ResourceMgrPtr &res_mgr,
std::vector<std::string> &path) { std::vector<std::string> &path) {
std::vector<std::vector<std::string>> paths; std::vector<std::vector<std::string>> paths;
uint64_t num_of_resources = res_mgr->GetAllResources().size(); uint64_t num_of_resources = res_mgr->GetAllResources().size();
@ -53,7 +56,6 @@ ShortestPath(const ResourcePtr &src,
std::vector<bool> vis(num_of_resources, false); std::vector<bool> vis(num_of_resources, false);
std::vector<uint64_t> dis(num_of_resources, MAXINT); std::vector<uint64_t> dis(num_of_resources, MAXINT);
for (auto &res : res_mgr->GetAllResources()) { for (auto &res : res_mgr->GetAllResources()) {
auto cur_node = std::static_pointer_cast<Node>(res); auto cur_node = std::static_pointer_cast<Node>(res);
auto cur_neighbours = cur_node->GetNeighbours(); auto cur_neighbours = cur_node->GetNeighbours();
@ -105,6 +107,6 @@ ShortestPath(const ResourcePtr &src,
return dis[name_id_map.at(dest->name())]; return dis[name_id_map.at(dest->name())];
} }
} } // namespace scheduler
} } // namespace milvus
} } // namespace zilliz

8
cpp/src/scheduler/Algorithm.h
View File

@ -30,8 +30,8 @@ uint64_t
ShortestPath(const ResourcePtr &src, ShortestPath(const ResourcePtr &src,
const ResourcePtr &dest, const ResourcePtr &dest,
const ResourceMgrPtr &res_mgr, const ResourceMgrPtr &res_mgr,
std::vector<std::string>& path); std::vector<std::string> &path);
} } // namespace scheduler
} } // namespace milvus
} } // namespace zilliz

7
cpp/src/scheduler/Definition.h
View File

@ -30,7 +30,6 @@
#include "db/engine/EngineFactory.h" #include "db/engine/EngineFactory.h"
#include "db/engine/ExecutionEngine.h" #include "db/engine/ExecutionEngine.h"
namespace zilliz { namespace zilliz {
namespace milvus { namespace milvus {
namespace scheduler { namespace scheduler {
@ -43,6 +42,6 @@ using EngineFactory = engine::EngineFactory;
using EngineType = engine::EngineType; using EngineType = engine::EngineType;
using MetricType = engine::MetricType; using MetricType = engine::MetricType;
} } // namespace scheduler
} } // namespace milvus
} } // namespace zilliz

18
cpp/src/scheduler/JobMgr.cpp
View File

@ -15,19 +15,19 @@
// specific language governing permissions and limitations // specific language governing permissions and limitations
// under the License. // under the License.
#include "JobMgr.h" #include "scheduler/JobMgr.h"
#include "task/Task.h" #include "task/Task.h"
#include "TaskCreator.h" #include "TaskCreator.h"
#include <utility>
namespace zilliz { namespace zilliz {
namespace milvus { namespace milvus {
namespace scheduler { namespace scheduler {
using namespace engine;
JobMgr::JobMgr(ResourceMgrPtr res_mgr) JobMgr::JobMgr(ResourceMgrPtr res_mgr)
: res_mgr_(std::move(res_mgr)) {} : res_mgr_(std::move(res_mgr)) {
}
void void
JobMgr::Start() { JobMgr::Start() {
@ -59,7 +59,9 @@ void
JobMgr::worker_function() { JobMgr::worker_function() {
while (running_) { while (running_) {
std::unique_lock<std::mutex> lock(mutex_); std::unique_lock<std::mutex> lock(mutex_);
cv_.wait(lock, [this] { return !queue_.empty(); }); cv_.wait(lock, [this] {
return !queue_.empty();
});
auto job = queue_.front(); auto job = queue_.front();
queue_.pop(); queue_.pop();
lock.unlock(); lock.unlock();
@ -84,6 +86,6 @@ JobMgr::build_task(const JobPtr &job) {
return TaskCreator::Create(job); return TaskCreator::Create(job);
} }
} } // namespace scheduler
} } // namespace milvus
} } // namespace zilliz

18
cpp/src/scheduler/JobMgr.h
View File

@ -31,15 +31,13 @@
#include "task/Task.h" #include "task/Task.h"
#include "ResourceMgr.h" #include "ResourceMgr.h"
namespace zilliz { namespace zilliz {
namespace milvus { namespace milvus {
namespace scheduler { namespace scheduler {
class JobMgr { class JobMgr {
public: public:
explicit explicit JobMgr(ResourceMgrPtr res_mgr);
JobMgr(ResourceMgrPtr res_mgr);
void void
Start(); Start();
@ -47,18 +45,18 @@ public:
void void
Stop(); Stop();
public: public:
void void
Put(const JobPtr &job); Put(const JobPtr &job);
private: private:
void void
worker_function(); worker_function();
std::vector<TaskPtr> std::vector<TaskPtr>
build_task(const JobPtr &job); build_task(const JobPtr &job);
private: private:
bool running_ = false; bool running_ = false;
std::queue<JobPtr> queue_; std::queue<JobPtr> queue_;
@ -72,6 +70,6 @@ private:
using JobMgrPtr = std::shared_ptr<JobMgr>; using JobMgrPtr = std::shared_ptr<JobMgr>;
} } // namespace scheduler
} } // namespace milvus
} } // namespace zilliz

9
cpp/src/scheduler/ResourceFactory.cpp
View File

@ -16,8 +16,7 @@
// under the License. // under the License.
#include "ResourceFactory.h" #include "scheduler/ResourceFactory.h"
namespace zilliz { namespace zilliz {
namespace milvus { namespace milvus {
@ -40,6 +39,6 @@ ResourceFactory::Create(const std::string &name,
} }
} }
} } // namespace scheduler
} } // namespace milvus
} } // namespace zilliz

11
cpp/src/scheduler/ResourceFactory.h
View File

@ -25,13 +25,12 @@
#include "resource/GpuResource.h" #include "resource/GpuResource.h"
#include "resource/DiskResource.h" #include "resource/DiskResource.h"
namespace zilliz { namespace zilliz {
namespace milvus { namespace milvus {
namespace scheduler { namespace scheduler {
class ResourceFactory { class ResourceFactory {
public: public:
static std::shared_ptr<Resource> static std::shared_ptr<Resource>
Create(const std::string &name, Create(const std::string &name,
const std::string &type, const std::string &type,
@ -40,8 +39,6 @@ public:
bool enable_executor = true); bool enable_executor = true);
}; };
} // namespace scheduler
} } // namespace milvus
} } // namespace zilliz
}

14
cpp/src/scheduler/ResourceMgr.cpp
View File

@ -16,15 +16,13 @@
// specific language governing permissions and limitations // specific language governing permissions and limitations
// under the License. // under the License.
#include "ResourceMgr.h" #include "scheduler/ResourceMgr.h"
#include "utils/Log.h" #include "utils/Log.h"
namespace zilliz { namespace zilliz {
namespace milvus { namespace milvus {
namespace scheduler { namespace scheduler {
void void
ResourceMgr::Start() { ResourceMgr::Start() {
std::lock_guard<std::mutex> lck(resources_mutex_); std::lock_guard<std::mutex> lck(resources_mutex_);
@ -186,7 +184,9 @@ void
ResourceMgr::event_process() { ResourceMgr::event_process() {
while (running_) { while (running_) {
std::unique_lock<std::mutex> lock(event_mutex_); std::unique_lock<std::mutex> lock(event_mutex_);
event_cv_.wait(lock, [this] { return !queue_.empty(); }); event_cv_.wait(lock, [this] {
return !queue_.empty();
});
auto event = queue_.front(); auto event = queue_.front();
queue_.pop(); queue_.pop();
@ -201,6 +201,6 @@ ResourceMgr::event_process() {
} }
} }
} } // namespace scheduler
} } // namespace milvus
} } // namespace zilliz

24
cpp/src/scheduler/ResourceMgr.h
View File

@ -22,21 +22,21 @@
#include <memory> #include <memory>
#include <mutex> #include <mutex>
#include <queue> #include <queue>
#include <utility>
#include <condition_variable> #include <condition_variable>
#include "resource/Resource.h" #include "resource/Resource.h"
#include "utils/Log.h" #include "utils/Log.h"
namespace zilliz { namespace zilliz {
namespace milvus { namespace milvus {
namespace scheduler { namespace scheduler {
class ResourceMgr { class ResourceMgr {
public: public:
ResourceMgr() = default; ResourceMgr() = default;
public: public:
/******** Management Interface ********/ /******** Management Interface ********/
void void
Start(); Start();
@ -58,7 +58,7 @@ public:
subscriber_ = std::move(subscriber); subscriber_ = std::move(subscriber);
} }
public: public:
/******** Management Interface ********/ /******** Management Interface ********/
inline std::vector<ResourceWPtr> & inline std::vector<ResourceWPtr> &
GetDiskResources() { GetDiskResources() {
@ -89,10 +89,10 @@ public:
uint64_t uint64_t
GetNumGpuResource() const; GetNumGpuResource() const;
public: public:
// TODO: add stats interface(low) // TODO: add stats interface(low)
public: public:
/******** Utility Functions ********/ /******** Utility Functions ********/
std::string std::string
Dump(); Dump();
@ -100,14 +100,14 @@ public:
std::string std::string
DumpTaskTables(); DumpTaskTables();
private: private:
void void
post_event(const EventPtr &event); post_event(const EventPtr &event);
void void
event_process(); event_process();
private: private:
bool running_ = false; bool running_ = false;
std::vector<ResourceWPtr> disk_resources_; std::vector<ResourceWPtr> disk_resources_;
@ -120,13 +120,11 @@ private:
std::condition_variable event_cv_; std::condition_variable event_cv_;
std::thread worker_thread_; std::thread worker_thread_;
}; };
using ResourceMgrPtr = std::shared_ptr<ResourceMgr>; using ResourceMgrPtr = std::shared_ptr<ResourceMgr>;
using ResourceMgrWPtr = std::weak_ptr<ResourceMgr>; using ResourceMgrWPtr = std::weak_ptr<ResourceMgr>;
} } // namespace scheduler
} } // namespace milvus
} } // namespace zilliz

13
cpp/src/scheduler/SchedInst.cpp
View File

@ -16,12 +16,16 @@
// under the License. // under the License.
#include "SchedInst.h" #include "scheduler/SchedInst.h"
#include "server/Config.h" #include "server/Config.h"
#include "ResourceFactory.h" #include "ResourceFactory.h"
#include "knowhere/index/vector_index/IndexGPUIVF.h" #include "knowhere/index/vector_index/IndexGPUIVF.h"
#include "Utils.h" #include "Utils.h"
#include <vector>
#include <set>
#include <utility>
#include <string>
namespace zilliz { namespace zilliz {
namespace milvus { namespace milvus {
@ -165,6 +169,7 @@ StopSchedulerService() {
SchedInst::GetInstance()->Stop(); SchedInst::GetInstance()->Stop();
ResMgrInst::GetInstance()->Stop(); ResMgrInst::GetInstance()->Stop();
} }
}
} } // namespace scheduler
} } // namespace milvus
} // namespace zilliz

19
cpp/src/scheduler/SchedInst.h
View File

@ -24,13 +24,12 @@
#include <mutex> #include <mutex>
#include <memory> #include <memory>
namespace zilliz { namespace zilliz {
namespace milvus { namespace milvus {
namespace scheduler { namespace scheduler {
class ResMgrInst { class ResMgrInst {
public: public:
static ResourceMgrPtr static ResourceMgrPtr
GetInstance() { GetInstance() {
if (instance == nullptr) { if (instance == nullptr) {
@ -42,13 +41,13 @@ public:
return instance; return instance;
} }
private: private:
static ResourceMgrPtr instance; static ResourceMgrPtr instance;
static std::mutex mutex_; static std::mutex mutex_;
}; };
class SchedInst { class SchedInst {
public: public:
static SchedulerPtr static SchedulerPtr
GetInstance() { GetInstance() {
if (instance == nullptr) { if (instance == nullptr) {
@ -60,13 +59,13 @@ public:
return instance; return instance;
} }
private: private:
static SchedulerPtr instance; static SchedulerPtr instance;
static std::mutex mutex_; static std::mutex mutex_;
}; };
class JobMgrInst { class JobMgrInst {
public: public:
static scheduler::JobMgrPtr static scheduler::JobMgrPtr
GetInstance() { GetInstance() {
if (instance == nullptr) { if (instance == nullptr) {
@ -78,7 +77,7 @@ public:
return instance; return instance;
} }
private: private:
static scheduler::JobMgrPtr instance; static scheduler::JobMgrPtr instance;
static std::mutex mutex_; static std::mutex mutex_;
}; };
@ -89,6 +88,6 @@ StartSchedulerService();
void void
StopSchedulerService(); StopSchedulerService();
} } // namespace scheduler
} } // namespace milvus
} } // namespace zilliz

17
cpp/src/scheduler/Scheduler.cpp
View File

@ -15,13 +15,13 @@
// specific language governing permissions and limitations // specific language governing permissions and limitations
// under the License. // under the License.
#include "scheduler/Scheduler.h"
#include "src/cache/GpuCacheMgr.h" #include "cache/GpuCacheMgr.h"
#include "event/LoadCompletedEvent.h" #include "event/LoadCompletedEvent.h"
#include "Scheduler.h"
#include "action/Action.h" #include "action/Action.h"
#include "Algorithm.h" #include "Algorithm.h"
#include <utility>
namespace zilliz { namespace zilliz {
namespace milvus { namespace milvus {
@ -43,7 +43,6 @@ Scheduler::Scheduler(ResourceMgrWPtr res_mgr)
std::bind(&Scheduler::OnFinishTask, this, std::placeholders::_1))); std::bind(&Scheduler::OnFinishTask, this, std::placeholders::_1)));
} }
void void
Scheduler::Start() { Scheduler::Start() {
running_ = true; running_ = true;
@ -79,7 +78,9 @@ void
Scheduler::worker_function() { Scheduler::worker_function() {
while (running_) { while (running_) {
std::unique_lock<std::mutex> lock(event_mutex_); std::unique_lock<std::mutex> lock(event_mutex_);
event_cv_.wait(lock, [this] { return !event_queue_.empty(); }); event_cv_.wait(lock, [this] {
return !event_queue_.empty();
});
auto event = event_queue_.front(); auto event = event_queue_.front();
event_queue_.pop(); event_queue_.pop();
if (event == nullptr) { if (event == nullptr) {
@ -142,6 +143,6 @@ Scheduler::OnTaskTableUpdated(const EventPtr &event) {
} }
} }
} } // namespace scheduler
} } // namespace milvus
} } // namespace zilliz

21
cpp/src/scheduler/Scheduler.h
View File

@ -22,22 +22,20 @@
#include <mutex> #include <mutex>
#include <thread> #include <thread>
#include <queue> #include <queue>
#include <unordered_map>
#include "resource/Resource.h" #include "resource/Resource.h"
#include "ResourceMgr.h" #include "ResourceMgr.h"
#include "utils/Log.h" #include "utils/Log.h"
namespace zilliz { namespace zilliz {
namespace milvus { namespace milvus {
namespace scheduler { namespace scheduler {
// TODO: refactor, not friendly to unittest, logical in framework code // TODO: refactor, not friendly to unittest, logical in framework code
class Scheduler { class Scheduler {
public: public:
explicit explicit Scheduler(ResourceMgrWPtr res_mgr);
Scheduler(ResourceMgrWPtr res_mgr);
Scheduler(const Scheduler &) = delete; Scheduler(const Scheduler &) = delete;
Scheduler(Scheduler &&) = delete; Scheduler(Scheduler &&) = delete;
@ -66,7 +64,7 @@ public:
std::string std::string
Dump(); Dump();
private: private:
/******** Events ********/ /******** Events ********/
/* /*
@ -106,7 +104,7 @@ private:
void void
OnTaskTableUpdated(const EventPtr &event); OnTaskTableUpdated(const EventPtr &event);
private: private:
/* /*
* Dispatch event to event handler; * Dispatch event to event handler;
*/ */
@ -119,7 +117,7 @@ private:
void void
worker_function(); worker_function();
private: private:
bool running_; bool running_;
std::unordered_map<uint64_t, std::function<void(EventPtr)>> event_register_; std::unordered_map<uint64_t, std::function<void(EventPtr)>> event_register_;
@ -133,7 +131,6 @@ private:
using SchedulerPtr = std::shared_ptr<Scheduler>; using SchedulerPtr = std::shared_ptr<Scheduler>;
} } // namespace scheduler
} } // namespace milvus
} } // namespace zilliz

13
cpp/src/scheduler/TaskCreator.cpp
View File

@ -15,11 +15,10 @@
// specific language governing permissions and limitations // specific language governing permissions and limitations
// under the License. // under the License.
#include <src/scheduler/tasklabel/BroadcastLabel.h> #include "scheduler/TaskCreator.h"
#include "TaskCreator.h" #include "scheduler/tasklabel/BroadcastLabel.h"
#include "tasklabel/DefaultLabel.h" #include "tasklabel/DefaultLabel.h"
namespace zilliz { namespace zilliz {
namespace milvus { namespace milvus {
namespace scheduler { namespace scheduler {
@ -64,8 +63,6 @@ TaskCreator::Create(const DeleteJobPtr &job) {
return tasks; return tasks;
} }
} // namespace scheduler
} } // namespace milvus
} } // namespace zilliz
}

11
cpp/src/scheduler/TaskCreator.h
View File

@ -34,17 +34,16 @@
#include "task/SearchTask.h" #include "task/SearchTask.h"
#include "task/DeleteTask.h" #include "task/DeleteTask.h"
namespace zilliz { namespace zilliz {
namespace milvus { namespace milvus {
namespace scheduler { namespace scheduler {
class TaskCreator { class TaskCreator {
public: public:
static std::vector<TaskPtr> static std::vector<TaskPtr>
Create(const JobPtr &job); Create(const JobPtr &job);
public: public:
static std::vector<TaskPtr> static std::vector<TaskPtr>
Create(const SearchJobPtr &job); Create(const SearchJobPtr &job);
@ -52,6 +51,6 @@ public:
Create(const DeleteJobPtr &job); Create(const DeleteJobPtr &job);
}; };
} } // namespace scheduler
} } // namespace milvus
} } // namespace zilliz

15
cpp/src/scheduler/TaskTable.cpp
View File

@ -16,7 +16,7 @@
// under the License. // under the License.
#include "TaskTable.h" #include "scheduler/TaskTable.h"
#include "event/TaskTableUpdatedEvent.h" #include "event/TaskTableUpdatedEvent.h"
#include "Utils.h" #include "Utils.h"
@ -24,7 +24,6 @@
#include <sstream> #include <sstream>
#include <ctime> #include <ctime>
namespace zilliz { namespace zilliz {
namespace milvus { namespace milvus {
namespace scheduler { namespace scheduler {
@ -75,6 +74,7 @@ TaskTableItem::Load() {
} }
return false; return false;
} }
bool bool
TaskTableItem::Loaded() { TaskTableItem::Loaded() {
std::unique_lock<std::mutex> lock(mutex); std::unique_lock<std::mutex> lock(mutex);
@ -86,6 +86,7 @@ TaskTableItem::Loaded() {
} }
return false; return false;
} }
bool bool
TaskTableItem::Execute() { TaskTableItem::Execute() {
std::unique_lock<std::mutex> lock(mutex); std::unique_lock<std::mutex> lock(mutex);
@ -97,6 +98,7 @@ TaskTableItem::Execute() {
} }
return false; return false;
} }
bool bool
TaskTableItem::Executed() { TaskTableItem::Executed() {
std::unique_lock<std::mutex> lock(mutex); std::unique_lock<std::mutex> lock(mutex);
@ -109,6 +111,7 @@ TaskTableItem::Executed() {
} }
return false; return false;
} }
bool bool
TaskTableItem::Move() { TaskTableItem::Move() {
std::unique_lock<std::mutex> lock(mutex); std::unique_lock<std::mutex> lock(mutex);
@ -120,6 +123,7 @@ TaskTableItem::Move() {
} }
return false; return false;
} }
bool bool
TaskTableItem::Moved() { TaskTableItem::Moved() {
std::unique_lock<std::mutex> lock(mutex); std::unique_lock<std::mutex> lock(mutex);
@ -206,7 +210,6 @@ TaskTable::Put(std::vector<TaskPtr> &tasks) {
} }
} }
TaskTableItemPtr TaskTableItemPtr
TaskTable::Get(uint64_t index) { TaskTable::Get(uint64_t index) {
return table_[index]; return table_[index];
@ -232,6 +235,6 @@ TaskTable::Dump() {
return ss.str(); return ss.str();
} }
} } // namespace scheduler
} } // namespace milvus
} } // namespace zilliz

36
cpp/src/scheduler/TaskTable.h
View File

@ -20,11 +20,13 @@
#include <vector> #include <vector>
#include <deque> #include <deque>
#include <mutex> #include <mutex>
#include <memory>
#include <utility>
#include <string>
#include "task/SearchTask.h" #include "task/SearchTask.h"
#include "event/Event.h" #include "event/Event.h"
namespace zilliz { namespace zilliz {
namespace milvus { namespace milvus {
namespace scheduler { namespace scheduler {
@ -52,7 +54,8 @@ struct TaskTimestamp {
}; };
struct TaskTableItem { struct TaskTableItem {
TaskTableItem() : id(0), task(nullptr), state(TaskTableItemState::INVALID), mutex() {} TaskTableItem() : id(0), task(nullptr), state(TaskTableItemState::INVALID), mutex() {
}
TaskTableItem(const TaskTableItem &src) = delete; TaskTableItem(const TaskTableItem &src) = delete;
TaskTableItem(TaskTableItem &&) = delete; TaskTableItem(TaskTableItem &&) = delete;
@ -91,7 +94,7 @@ struct TaskTableItem {
using TaskTableItemPtr = std::shared_ptr<TaskTableItem>; using TaskTableItemPtr = std::shared_ptr<TaskTableItem>;
class TaskTable { class TaskTable {
public: public:
TaskTable() = default; TaskTable() = default;
TaskTable(const TaskTable &) = delete; TaskTable(const TaskTable &) = delete;
@ -145,24 +148,28 @@ public:
return table_.size(); return table_.size();
} }
public: public:
TaskTableItemPtr & TaskTableItemPtr &
operator[](uint64_t index) { operator[](uint64_t index) {
return table_[index]; return table_[index];
} }
std::deque<TaskTableItemPtr>::iterator begin() { return table_.begin(); } std::deque<TaskTableItemPtr>::iterator begin() {
std::deque<TaskTableItemPtr>::iterator end() { return table_.end(); } return table_.begin();
}
public: std::deque<TaskTableItemPtr>::iterator end() {
return table_.end();
}
public:
std::vector<uint64_t> std::vector<uint64_t>
PickToLoad(uint64_t limit); PickToLoad(uint64_t limit);
std::vector<uint64_t> std::vector<uint64_t>
PickToExecute(uint64_t limit); PickToExecute(uint64_t limit);
public: public:
/******** Action ********/ /******** Action ********/
// TODO: bool to Status // TODO: bool to Status
@ -227,14 +234,14 @@ public:
return table_[index]->Moved(); return table_[index]->Moved();
} }
public: public:
/* /*
* Dump; * Dump;
*/ */
std::string std::string
Dump(); Dump();
private: private:
std::uint64_t id_ = 0; std::uint64_t id_ = 0;
mutable std::mutex id_mutex_; mutable std::mutex id_mutex_;
std::deque<TaskTableItemPtr> table_; std::deque<TaskTableItemPtr> table_;
@ -246,7 +253,6 @@ private:
uint64_t last_finish_ = -1; uint64_t last_finish_ = -1;
}; };
} // namespace scheduler
} } // namespace milvus
} } // namespace zilliz
}

9
cpp/src/scheduler/Utils.cpp
View File

@ -16,12 +16,11 @@
// under the License. // under the License.
#include "Utils.h" #include "scheduler/Utils.h"
#include <chrono> #include <chrono>
#include <cuda_runtime.h> #include <cuda_runtime.h>
namespace zilliz { namespace zilliz {
namespace milvus { namespace milvus {
namespace scheduler { namespace scheduler {
@ -41,6 +40,6 @@ get_num_gpu() {
return n_devices; return n_devices;
} }
} } // namespace scheduler
} } // namespace milvus
} } // namespace zilliz

7
cpp/src/scheduler/Utils.h
View File

@ -18,7 +18,6 @@
#include <cstdint> #include <cstdint>
namespace zilliz { namespace zilliz {
namespace milvus { namespace milvus {
namespace scheduler { namespace scheduler {
@ -29,6 +28,6 @@ get_current_timestamp();
uint64_t uint64_t
get_num_gpu(); get_num_gpu();
} } // namespace scheduler
} } // namespace milvus
} } // namespace zilliz

15
cpp/src/scheduler/action/Action.h
View File

@ -17,16 +17,17 @@
#pragma once #pragma once
#include "../resource/Resource.h" #include "scheduler/resource/Resource.h"
#include "../ResourceMgr.h" #include "scheduler/ResourceMgr.h"
#include <memory>
namespace zilliz { namespace zilliz {
namespace milvus { namespace milvus {
namespace scheduler { namespace scheduler {
class Action { class Action {
public: public:
static void static void
PushTaskToNeighbourRandomly(const TaskPtr &task, const ResourcePtr &self); PushTaskToNeighbourRandomly(const TaskPtr &task, const ResourcePtr &self);
@ -43,10 +44,8 @@ public:
SpecifiedResourceLabelTaskScheduler(ResourceMgrWPtr res_mgr, SpecifiedResourceLabelTaskScheduler(ResourceMgrWPtr res_mgr,
ResourcePtr resource, ResourcePtr resource,
std::shared_ptr<LoadCompletedEvent> event); std::shared_ptr<LoadCompletedEvent> event);
}; };
} // namespace scheduler
} } // namespace milvus
} } // namespace zilliz
}

22
cpp/src/scheduler/action/PushTaskToNeighbour.cpp
View File

@ -22,7 +22,6 @@
#include "src/cache/GpuCacheMgr.h" #include "src/cache/GpuCacheMgr.h"
#include "Action.h" #include "Action.h"
namespace zilliz { namespace zilliz {
namespace milvus { namespace milvus {
namespace scheduler { namespace scheduler {
@ -57,13 +56,12 @@ get_neighbours_with_connetion(const ResourcePtr &self) {
return neighbours; return neighbours;
} }
void void
Action::PushTaskToNeighbourRandomly(const TaskPtr &task, Action::PushTaskToNeighbourRandomly(const TaskPtr &task,
const ResourcePtr &self) { const ResourcePtr &self) {
auto neighbours = get_neighbours_with_connetion(self); auto neighbours = get_neighbours_with_connetion(self);
if (not neighbours.empty()) { if (not neighbours.empty()) {
std::vector<uint64_t > speeds; std::vector<uint64_t> speeds;
uint64_t total_speed = 0; uint64_t total_speed = 0;
for (auto &neighbour : neighbours) { for (auto &neighbour : neighbours) {
uint64_t speed = neighbour.second.speed(); uint64_t speed = neighbour.second.speed();
@ -87,7 +85,6 @@ Action::PushTaskToNeighbourRandomly(const TaskPtr &task,
} else { } else {
//TODO: process //TODO: process
} }
} }
void void
@ -99,14 +96,14 @@ Action::PushTaskToAllNeighbour(const TaskPtr &task, const ResourcePtr &self) {
} }
void void
Action::PushTaskToResource(const TaskPtr& task, const ResourcePtr& dest) { Action::PushTaskToResource(const TaskPtr &task, const ResourcePtr &dest) {
dest->task_table().Put(task); dest->task_table().Put(task);
} }
void void
Action::DefaultLabelTaskScheduler(ResourceMgrWPtr res_mgr, Action::DefaultLabelTaskScheduler(ResourceMgrWPtr res_mgr,
ResourcePtr resource, ResourcePtr resource,
std::shared_ptr<LoadCompletedEvent> event) { std::shared_ptr<LoadCompletedEvent> event) {
if (not resource->HasExecutor() && event->task_table_item_->Move()) { if (not resource->HasExecutor() && event->task_table_item_->Move()) {
auto task = event->task_table_item_->task; auto task = event->task_table_item_->task;
auto search_task = std::static_pointer_cast<XSearchTask>(task); auto search_task = std::static_pointer_cast<XSearchTask>(task);
@ -135,8 +132,8 @@ Action::DefaultLabelTaskScheduler(ResourceMgrWPtr res_mgr,
void void
Action::SpecifiedResourceLabelTaskScheduler(ResourceMgrWPtr res_mgr, Action::SpecifiedResourceLabelTaskScheduler(ResourceMgrWPtr res_mgr,
ResourcePtr resource, ResourcePtr resource,
std::shared_ptr<LoadCompletedEvent> event) { std::shared_ptr<LoadCompletedEvent> event) {
auto task = event->task_table_item_->task; auto task = event->task_table_item_->task;
if (resource->type() == ResourceType::DISK) { if (resource->type() == ResourceType::DISK) {
// step 1: calculate shortest path per resource, from disk to compute resource // step 1: calculate shortest path per resource, from disk to compute resource
@ -181,7 +178,6 @@ Action::SpecifiedResourceLabelTaskScheduler(ResourceMgrWPtr res_mgr,
} }
} }
} } // namespace scheduler
} } // namespace milvus
} } // namespace zilliz

15
cpp/src/scheduler/event/Event.h
View File

@ -18,6 +18,8 @@
#pragma once #pragma once
#include <memory> #include <memory>
#include <utility>
#include <string>
namespace zilliz { namespace zilliz {
namespace milvus { namespace milvus {
@ -33,11 +35,12 @@ enum class EventType {
class Resource; class Resource;
class Event { class Event {
public: public:
explicit explicit
Event(EventType type, std::weak_ptr<Resource> resource) Event(EventType type, std::weak_ptr<Resource> resource)
: type_(type), : type_(type),
resource_(std::move(resource)) {} resource_(std::move(resource)) {
}
inline EventType inline EventType
Type() const { Type() const {
@ -49,13 +52,13 @@ public:
friend std::ostream &operator<<(std::ostream &out, const Event &event); friend std::ostream &operator<<(std::ostream &out, const Event &event);
public: public:
EventType type_; EventType type_;
std::weak_ptr<Resource> resource_; std::weak_ptr<Resource> resource_;
}; };
using EventPtr = std::shared_ptr<Event>; using EventPtr = std::shared_ptr<Event>;
} } // namespace scheduler
} } // namespace milvus
} } // namespace zilliz

22
cpp/src/scheduler/event/EventDump.cpp
View File

@ -22,36 +22,40 @@
#include "FinishTaskEvent.h" #include "FinishTaskEvent.h"
#include "TaskTableUpdatedEvent.h" #include "TaskTableUpdatedEvent.h"
namespace zilliz { namespace zilliz {
namespace milvus { namespace milvus {
namespace scheduler { namespace scheduler {
std::ostream &operator<<(std::ostream &out, const Event &event) { std::ostream &
operator<<(std::ostream &out, const Event &event) {
out << event.Dump(); out << event.Dump();
return out; return out;
} }
std::ostream &operator<<(std::ostream &out, const StartUpEvent &event) { std::ostream &
operator<<(std::ostream &out, const StartUpEvent &event) {
out << event.Dump(); out << event.Dump();
return out; return out;
} }
std::ostream &operator<<(std::ostream &out, const LoadCompletedEvent &event) { std::ostream &
operator<<(std::ostream &out, const LoadCompletedEvent &event) {
out << event.Dump(); out << event.Dump();
return out; return out;
} }
std::ostream &operator<<(std::ostream &out, const FinishTaskEvent &event) { std::ostream &
operator<<(std::ostream &out, const FinishTaskEvent &event) {
out << event.Dump(); out << event.Dump();
return out; return out;
} }
std::ostream &operator<<(std::ostream &out, const TaskTableUpdatedEvent &event) { std::ostream &
operator<<(std::ostream &out, const TaskTableUpdatedEvent &event) {
out << event.Dump(); out << event.Dump();
return out; return out;
} }
} } // namespace scheduler
} } // namespace milvus
} } // namespace zilliz

18
cpp/src/scheduler/event/FinishTaskEvent.h
View File

@ -17,18 +17,22 @@
#pragma once #pragma once
#include "Event.h" #include "scheduler/event/Event.h"
#include <memory>
#include <utility>
#include <string>
namespace zilliz { namespace zilliz {
namespace milvus { namespace milvus {
namespace scheduler { namespace scheduler {
class FinishTaskEvent : public Event { class FinishTaskEvent : public Event {
public: public:
FinishTaskEvent(std::weak_ptr<Resource> resource, TaskTableItemPtr task_table_item) FinishTaskEvent(std::weak_ptr<Resource> resource, TaskTableItemPtr task_table_item)
: Event(EventType::FINISH_TASK, std::move(resource)), : Event(EventType::FINISH_TASK, std::move(resource)),
task_table_item_(std::move(task_table_item)) {} task_table_item_(std::move(task_table_item)) {
}
inline std::string inline std::string
Dump() const override { Dump() const override {
@ -37,10 +41,10 @@ public:
friend std::ostream &operator<<(std::ostream &out, const FinishTaskEvent &event); friend std::ostream &operator<<(std::ostream &out, const FinishTaskEvent &event);
public: public:
TaskTableItemPtr task_table_item_; TaskTableItemPtr task_table_item_;
}; };
} } // namespace scheduler
} } // namespace milvus
} } // namespace zilliz

20
cpp/src/scheduler/event/LoadCompletedEvent.h
View File

@ -17,19 +17,23 @@
#pragma once #pragma once
#include "Event.h" #include "scheduler/event/Event.h"
#include "../TaskTable.h" #include "scheduler/TaskTable.h"
#include <memory>
#include <utility>
#include <string>
namespace zilliz { namespace zilliz {
namespace milvus { namespace milvus {
namespace scheduler { namespace scheduler {
class LoadCompletedEvent : public Event { class LoadCompletedEvent : public Event {
public: public:
LoadCompletedEvent(std::weak_ptr<Resource> resource, TaskTableItemPtr task_table_item) LoadCompletedEvent(std::weak_ptr<Resource> resource, TaskTableItemPtr task_table_item)
: Event(EventType::LOAD_COMPLETED, std::move(resource)), : Event(EventType::LOAD_COMPLETED, std::move(resource)),
task_table_item_(std::move(task_table_item)) {} task_table_item_(std::move(task_table_item)) {
}
inline std::string inline std::string
Dump() const override { Dump() const override {
@ -38,10 +42,10 @@ public:
friend std::ostream &operator<<(std::ostream &out, const LoadCompletedEvent &event); friend std::ostream &operator<<(std::ostream &out, const LoadCompletedEvent &event);
public: public:
TaskTableItemPtr task_table_item_; TaskTableItemPtr task_table_item_;
}; };
} } // namespace scheduler
} } // namespace milvus
} } // namespace zilliz

19
cpp/src/scheduler/event/StartUpEvent.h
View File

@ -17,18 +17,21 @@
#pragma once #pragma once
#include "Event.h" #include "scheduler/event/Event.h"
#include <memory>
#include <utility>
#include <string>
namespace zilliz { namespace zilliz {
namespace milvus { namespace milvus {
namespace scheduler { namespace scheduler {
class StartUpEvent : public Event { class StartUpEvent : public Event {
public: public:
explicit explicit StartUpEvent(std::weak_ptr<Resource> resource)
StartUpEvent(std::weak_ptr<Resource> resource) : Event(EventType::START_UP, std::move(resource)) {
: Event(EventType::START_UP, std::move(resource)) {} }
inline std::string inline std::string
Dump() const override { Dump() const override {
@ -38,6 +41,6 @@ public:
friend std::ostream &operator<<(std::ostream &out, const StartUpEvent &event); friend std::ostream &operator<<(std::ostream &out, const StartUpEvent &event);
}; };
} } // namespace scheduler
} } // namespace milvus
} } // namespace zilliz

18
cpp/src/scheduler/event/TaskTableUpdatedEvent.h
View File

@ -19,16 +19,19 @@
#include "Event.h" #include "Event.h"
#include <memory>
#include <utility>
#include <string>
namespace zilliz { namespace zilliz {
namespace milvus { namespace milvus {
namespace scheduler { namespace scheduler {
class TaskTableUpdatedEvent : public Event { class TaskTableUpdatedEvent : public Event {
public: public:
explicit explicit TaskTableUpdatedEvent(std::weak_ptr<Resource> resource)
TaskTableUpdatedEvent(std::weak_ptr<Resource> resource) : Event(EventType::TASK_TABLE_UPDATED, std::move(resource)) {
: Event(EventType::TASK_TABLE_UPDATED, std::move(resource)) {} }
inline std::string inline std::string
Dump() const override { Dump() const override {
@ -38,7 +41,6 @@ public:
friend std::ostream &operator<<(std::ostream &out, const TaskTableUpdatedEvent &event); friend std::ostream &operator<<(std::ostream &out, const TaskTableUpdatedEvent &event);
}; };
} // namespace scheduler
} } // namespace milvus
} } // namespace zilliz
}

23
cpp/src/scheduler/job/DeleteJob.cpp
View File

@ -15,8 +15,9 @@
// specific language governing permissions and limitations // specific language governing permissions and limitations
// under the License. // under the License.
#include "DeleteJob.h" #include "scheduler/job/DeleteJob.h"
#include <utility>
namespace zilliz { namespace zilliz {
namespace milvus { namespace milvus {
@ -29,15 +30,20 @@ DeleteJob::DeleteJob(JobId id,
: Job(id, JobType::DELETE), : Job(id, JobType::DELETE),
table_id_(std::move(table_id)), table_id_(std::move(table_id)),
meta_ptr_(std::move(meta_ptr)), meta_ptr_(std::move(meta_ptr)),
num_resource_(num_resource) {} num_resource_(num_resource) {
}
void DeleteJob::WaitAndDelete() { void
DeleteJob::WaitAndDelete() {
std::unique_lock<std::mutex> lock(mutex_); std::unique_lock<std::mutex> lock(mutex_);
cv_.wait(lock, [&] { return done_resource == num_resource_; }); cv_.wait(lock, [&] {
return done_resource == num_resource_;
});
meta_ptr_->DeleteTableFiles(table_id_); meta_ptr_->DeleteTableFiles(table_id_);
} }
void DeleteJob::ResourceDone() { void
DeleteJob::ResourceDone() {
{ {
std::lock_guard<std::mutex> lock(mutex_); std::lock_guard<std::mutex> lock(mutex_);
++done_resource; ++done_resource;
@ -45,7 +51,6 @@ void DeleteJob::ResourceDone() {
cv_.notify_one(); cv_.notify_one();
} }
} } // namespace scheduler
} } // namespace milvus
} } // namespace zilliz

16
cpp/src/scheduler/job/DeleteJob.h
View File

@ -30,26 +30,25 @@
#include "Job.h" #include "Job.h"
#include "db/meta/Meta.h" #include "db/meta/Meta.h"
namespace zilliz { namespace zilliz {
namespace milvus { namespace milvus {
namespace scheduler { namespace scheduler {
class DeleteJob : public Job { class DeleteJob : public Job {
public: public:
DeleteJob(JobId id, DeleteJob(JobId id,
std::string table_id, std::string table_id,
engine::meta::MetaPtr meta_ptr, engine::meta::MetaPtr meta_ptr,
uint64_t num_resource); uint64_t num_resource);
public: public:
void void
WaitAndDelete(); WaitAndDelete();
void void
ResourceDone(); ResourceDone();
public: public:
std::string std::string
table_id() const { table_id() const {
return table_id_; return table_id_;
@ -60,7 +59,7 @@ public:
return meta_ptr_; return meta_ptr_;
} }
private: private:
std::string table_id_; std::string table_id_;
engine::meta::MetaPtr meta_ptr_; engine::meta::MetaPtr meta_ptr_;
@ -72,7 +71,6 @@ private:
using DeleteJobPtr = std::shared_ptr<DeleteJob>; using DeleteJobPtr = std::shared_ptr<DeleteJob>;
} } // namespace scheduler
} } // namespace milvus
} } // namespace zilliz

17
cpp/src/scheduler/job/Job.h
View File

@ -27,7 +27,6 @@
#include <condition_variable> #include <condition_variable>
#include <memory> #include <memory>
namespace zilliz { namespace zilliz {
namespace milvus { namespace milvus {
namespace scheduler { namespace scheduler {
@ -42,7 +41,7 @@ enum class JobType {
using JobId = std::uint64_t; using JobId = std::uint64_t;
class Job { class Job {
public: public:
inline JobId inline JobId
id() const { id() const {
return id_; return id_;
@ -53,10 +52,11 @@ public:
return type_; return type_;
} }
protected: protected:
Job(JobId id, JobType type) : id_(id), type_(type) {} Job(JobId id, JobType type) : id_(id), type_(type) {
}
private: private:
JobId id_; JobId id_;
JobType type_; JobType type_;
}; };
@ -64,7 +64,6 @@ private:
using JobPtr = std::shared_ptr<Job>; using JobPtr = std::shared_ptr<Job>;
using JobWPtr = std::weak_ptr<Job>; using JobWPtr = std::weak_ptr<Job>;
} } // namespace scheduler
} } // namespace milvus
} } // namespace zilliz

23
cpp/src/scheduler/job/SearchJob.cpp
View File

@ -15,11 +15,9 @@
// specific language governing permissions and limitations // specific language governing permissions and limitations
// under the License. // under the License.
#include "scheduler/job/SearchJob.h"
#include "utils/Log.h" #include "utils/Log.h"
#include "SearchJob.h"
namespace zilliz { namespace zilliz {
namespace milvus { namespace milvus {
namespace scheduler { namespace scheduler {
@ -33,7 +31,8 @@ SearchJob::SearchJob(zilliz::milvus::scheduler::JobId id,
topk_(topk), topk_(topk),
nq_(nq), nq_(nq),
nprobe_(nprobe), nprobe_(nprobe),
vectors_(vectors) {} vectors_(vectors) {
}
bool bool
SearchJob::AddIndexFile(const TableFileSchemaPtr &index_file) { SearchJob::AddIndexFile(const TableFileSchemaPtr &index_file) {
@ -48,11 +47,12 @@ SearchJob::AddIndexFile(const TableFileSchemaPtr &index_file) {
return true; return true;
} }
void void
SearchJob::WaitResult() { SearchJob::WaitResult() {
std::unique_lock<std::mutex> lock(mutex_); std::unique_lock<std::mutex> lock(mutex_);
cv_.wait(lock, [this] { return index_files_.empty(); }); cv_.wait(lock, [this] {
return index_files_.empty();
});
SERVER_LOG_DEBUG << "SearchJob " << id() << " all done"; SERVER_LOG_DEBUG << "SearchJob " << id() << " all done";
} }
@ -69,14 +69,11 @@ SearchJob::GetResult() {
return result_; return result_;
} }
Status& Status &
SearchJob::GetStatus() { SearchJob::GetStatus() {
return status_; return status_;
} }
} // namespace scheduler
} } // namespace milvus
} } // namespace zilliz
}

19
cpp/src/scheduler/job/SearchJob.h
View File

@ -26,16 +26,15 @@
#include <mutex> #include <mutex>
#include <condition_variable> #include <condition_variable>
#include <memory> #include <memory>
#include <utility>
#include "Job.h" #include "Job.h"
#include "db/meta/MetaTypes.h" #include "db/meta/MetaTypes.h"
namespace zilliz { namespace zilliz {
namespace milvus { namespace milvus {
namespace scheduler { namespace scheduler {
using engine::meta::TableFileSchemaPtr; using engine::meta::TableFileSchemaPtr;
using Id2IndexMap = std::unordered_map<size_t, TableFileSchemaPtr>; using Id2IndexMap = std::unordered_map<size_t, TableFileSchemaPtr>;
@ -43,10 +42,10 @@ using Id2DistanceMap = std::vector<std::pair<int64_t, double>>;
using ResultSet = std::vector<Id2DistanceMap>; using ResultSet = std::vector<Id2DistanceMap>;
class SearchJob : public Job { class SearchJob : public Job {
public: public:
SearchJob(JobId id, uint64_t topk, uint64_t nq, uint64_t nprobe, const float *vectors); SearchJob(JobId id, uint64_t topk, uint64_t nq, uint64_t nprobe, const float *vectors);
public: public:
bool bool
AddIndexFile(const TableFileSchemaPtr &index_file); AddIndexFile(const TableFileSchemaPtr &index_file);
@ -62,7 +61,7 @@ public:
Status & Status &
GetStatus(); GetStatus();
public: public:
uint64_t uint64_t
topk() const { topk() const {
return topk_; return topk_;
@ -77,6 +76,7 @@ public:
nprobe() const { nprobe() const {
return nprobe_; return nprobe_;
} }
const float * const float *
vectors() const { vectors() const {
return vectors_; return vectors_;
@ -87,7 +87,7 @@ public:
return index_files_; return index_files_;
} }
private: private:
uint64_t topk_ = 0; uint64_t topk_ = 0;
uint64_t nq_ = 0; uint64_t nq_ = 0;
uint64_t nprobe_ = 0; uint64_t nprobe_ = 0;
@ -105,7 +105,6 @@ private:
using SearchJobPtr = std::shared_ptr<SearchJob>; using SearchJobPtr = std::shared_ptr<SearchJob>;
} } // namespace scheduler
} } // namespace milvus
} } // namespace zilliz

18
cpp/src/scheduler/resource/Connection.h
View File

@ -19,17 +19,18 @@
#include <string> #include <string>
#include <sstream> #include <sstream>
#include <utility>
namespace zilliz { namespace zilliz {
namespace milvus { namespace milvus {
namespace scheduler { namespace scheduler {
class Connection { class Connection {
public: public:
// TODO: update construct function, speed: double->uint64_t // TODO: update construct function, speed: double->uint64_t
Connection(std::string name, double speed) Connection(std::string name, double speed)
: name_(std::move(name)), speed_(speed) {} : name_(std::move(name)), speed_(speed) {
}
const std::string & const std::string &
name() const { name() const {
@ -46,7 +47,7 @@ public:
return 1024 / speed_; return 1024 / speed_;
} }
public: public:
std::string std::string
Dump() const { Dump() const {
std::stringstream ss; std::stringstream ss;
@ -54,12 +55,11 @@ public:
return ss.str(); return ss.str();
} }
private: private:
std::string name_; std::string name_;
uint64_t speed_; uint64_t speed_;
}; };
} // namespace scheduler
} } // namespace milvus
} } // namespace zilliz
}

21
cpp/src/scheduler/resource/CpuResource.cpp
View File

@ -16,29 +16,34 @@
// under the License. // under the License.
#include "CpuResource.h" #include "scheduler/resource/CpuResource.h"
#include <utility>
namespace zilliz { namespace zilliz {
namespace milvus { namespace milvus {
namespace scheduler { namespace scheduler {
std::ostream &operator<<(std::ostream &out, const CpuResource &resource) { std::ostream &
operator<<(std::ostream &out, const CpuResource &resource) {
out << resource.Dump(); out << resource.Dump();
return out; return out;
} }
CpuResource::CpuResource(std::string name, uint64_t device_id, bool enable_loader, bool enable_executor) CpuResource::CpuResource(std::string name, uint64_t device_id, bool enable_loader, bool enable_executor)
: Resource(std::move(name), ResourceType::CPU, device_id, enable_loader, enable_executor) {} : Resource(std::move(name), ResourceType::CPU, device_id, enable_loader, enable_executor) {
}
void CpuResource::LoadFile(TaskPtr task) { void
CpuResource::LoadFile(TaskPtr task) {
task->Load(LoadType::DISK2CPU, 0); task->Load(LoadType::DISK2CPU, 0);
} }
void CpuResource::Process(TaskPtr task) { void
CpuResource::Process(TaskPtr task) {
task->Execute(); task->Execute();
} }
} } // namespace scheduler
} } // namespace milvus
} } // namespace zilliz

11
cpp/src/scheduler/resource/CpuResource.h
View File

@ -21,13 +21,12 @@
#include "Resource.h" #include "Resource.h"
namespace zilliz { namespace zilliz {
namespace milvus { namespace milvus {
namespace scheduler { namespace scheduler {
class CpuResource : public Resource { class CpuResource : public Resource {
public: public:
explicit explicit
CpuResource(std::string name, uint64_t device_id, bool enable_loader, bool enable_executor); CpuResource(std::string name, uint64_t device_id, bool enable_loader, bool enable_executor);
@ -38,7 +37,7 @@ public:
friend std::ostream &operator<<(std::ostream &out, const CpuResource &resource); friend std::ostream &operator<<(std::ostream &out, const CpuResource &resource);
protected: protected:
void void
LoadFile(TaskPtr task) override; LoadFile(TaskPtr task) override;
@ -46,6 +45,6 @@ protected:
Process(TaskPtr task) override; Process(TaskPtr task) override;
}; };
} } // namespace scheduler
} } // namespace milvus
} } // namespace zilliz

22
cpp/src/scheduler/resource/DiskResource.cpp
View File

@ -15,14 +15,17 @@
// specific language governing permissions and limitations // specific language governing permissions and limitations
// under the License. // under the License.
#include "DiskResource.h" #include "scheduler/resource/DiskResource.h"
#include <string>
#include <utility>
namespace zilliz { namespace zilliz {
namespace milvus { namespace milvus {
namespace scheduler { namespace scheduler {
std::ostream &operator<<(std::ostream &out, const DiskResource &resource) { std::ostream &
operator<<(std::ostream &out, const DiskResource &resource) {
out << resource.Dump(); out << resource.Dump();
return out; return out;
} }
@ -31,15 +34,14 @@ DiskResource::DiskResource(std::string name, uint64_t device_id, bool enable_loa
: Resource(std::move(name), ResourceType::DISK, device_id, enable_loader, enable_executor) { : Resource(std::move(name), ResourceType::DISK, device_id, enable_loader, enable_executor) {
} }
void DiskResource::LoadFile(TaskPtr task) { void
DiskResource::LoadFile(TaskPtr task) {
} }
void DiskResource::Process(TaskPtr task) { void
DiskResource::Process(TaskPtr task) {
} }
} } // namespace scheduler
} } // namespace milvus
} } // namespace zilliz

12
cpp/src/scheduler/resource/DiskResource.h
View File

@ -17,16 +17,16 @@
#pragma once #pragma once
#include "Resource.h" #include "Resource.h"
#include <string>
namespace zilliz { namespace zilliz {
namespace milvus { namespace milvus {
namespace scheduler { namespace scheduler {
class DiskResource : public Resource { class DiskResource : public Resource {
public: public:
explicit explicit
DiskResource(std::string name, uint64_t device_id, bool enable_loader, bool enable_executor); DiskResource(std::string name, uint64_t device_id, bool enable_loader, bool enable_executor);
@ -37,7 +37,7 @@ public:
friend std::ostream &operator<<(std::ostream &out, const DiskResource &resource); friend std::ostream &operator<<(std::ostream &out, const DiskResource &resource);
protected: protected:
void void
LoadFile(TaskPtr task) override; LoadFile(TaskPtr task) override;
@ -45,6 +45,6 @@ protected:
Process(TaskPtr task) override; Process(TaskPtr task) override;
}; };
} } // namespace scheduler
} } // namespace milvus
} } // namespace zilliz

21
cpp/src/scheduler/resource/GpuResource.cpp
View File

@ -16,29 +16,32 @@
// under the License. // under the License.
#include "GpuResource.h" #include "scheduler/resource/GpuResource.h"
namespace zilliz { namespace zilliz {
namespace milvus { namespace milvus {
namespace scheduler { namespace scheduler {
std::ostream &operator<<(std::ostream &out, const GpuResource &resource) { std::ostream &
operator<<(std::ostream &out, const GpuResource &resource) {
out << resource.Dump(); out << resource.Dump();
return out; return out;
} }
GpuResource::GpuResource(std::string name, uint64_t device_id, bool enable_loader, bool enable_executor) GpuResource::GpuResource(std::string name, uint64_t device_id, bool enable_loader, bool enable_executor)
: Resource(std::move(name), ResourceType::GPU, device_id, enable_loader, enable_executor) {} : Resource(std::move(name), ResourceType::GPU, device_id, enable_loader, enable_executor) {
}
void GpuResource::LoadFile(TaskPtr task) { void
GpuResource::LoadFile(TaskPtr task) {
task->Load(LoadType::CPU2GPU, device_id_); task->Load(LoadType::CPU2GPU, device_id_);
} }
void GpuResource::Process(TaskPtr task) { void
GpuResource::Process(TaskPtr task) {
task->Execute(); task->Execute();
} }
} } // namespace scheduler
} } // namespace milvus
} } // namespace zilliz

13
cpp/src/scheduler/resource/GpuResource.h
View File

@ -17,16 +17,17 @@
#pragma once #pragma once
#include "Resource.h" #include "Resource.h"
#include <string>
#include <utility>
namespace zilliz { namespace zilliz {
namespace milvus { namespace milvus {
namespace scheduler { namespace scheduler {
class GpuResource : public Resource { class GpuResource : public Resource {
public: public:
explicit explicit
GpuResource(std::string name, uint64_t device_id, bool enable_loader, bool enable_executor); GpuResource(std::string name, uint64_t device_id, bool enable_loader, bool enable_executor);
@ -37,7 +38,7 @@ public:
friend std::ostream &operator<<(std::ostream &out, const GpuResource &resource); friend std::ostream &operator<<(std::ostream &out, const GpuResource &resource);
protected: protected:
void void
LoadFile(TaskPtr task) override; LoadFile(TaskPtr task) override;
@ -45,6 +46,6 @@ protected:
Process(TaskPtr task) override; Process(TaskPtr task) override;
}; };
} } // namespace scheduler
} } // namespace milvus
} } // namespace zilliz

19
cpp/src/scheduler/resource/Node.cpp
View File

@ -15,10 +15,10 @@
// specific language governing permissions and limitations // specific language governing permissions and limitations
// under the License. // under the License.
#include "scheduler/resource/Node.h"
#include <atomic> #include <atomic>
#include "Node.h" #include <utility>
namespace zilliz { namespace zilliz {
namespace milvus { namespace milvus {
@ -29,7 +29,8 @@ Node::Node() {
id_ = counter++; id_ = counter++;
} }
std::vector<Neighbour> Node::GetNeighbours() { std::vector<Neighbour>
Node::GetNeighbours() {
std::lock_guard<std::mutex> lk(mutex_); std::lock_guard<std::mutex> lk(mutex_);
std::vector<Neighbour> ret; std::vector<Neighbour> ret;
for (auto &e : neighbours_) { for (auto &e : neighbours_) {
@ -38,7 +39,8 @@ std::vector<Neighbour> Node::GetNeighbours() {
return ret; return ret;
} }
std::string Node::Dump() { std::string
Node::Dump() {
std::stringstream ss; std::stringstream ss;
ss << "<Node, id=" << std::to_string(id_) << ">::neighbours:" << std::endl; ss << "<Node, id=" << std::to_string(id_) << ">::neighbours:" << std::endl;
for (auto &neighbour : neighbours_) { for (auto &neighbour : neighbours_) {
@ -48,7 +50,8 @@ std::string Node::Dump() {
return ss.str(); return ss.str();
} }
void Node::AddNeighbour(const NeighbourNodePtr &neighbour_node, Connection &connection) { void
Node::AddNeighbour(const NeighbourNodePtr &neighbour_node, Connection &connection) {
std::lock_guard<std::mutex> lk(mutex_); std::lock_guard<std::mutex> lk(mutex_);
if (auto s = neighbour_node.lock()) { if (auto s = neighbour_node.lock()) {
neighbours_.emplace(std::make_pair(s->id_, Neighbour(neighbour_node, connection))); neighbours_.emplace(std::make_pair(s->id_, Neighbour(neighbour_node, connection)));
@ -56,6 +59,6 @@ void Node::AddNeighbour(const NeighbourNodePtr &neighbour_node, Connection &conn
// else do nothing, consider it.. // else do nothing, consider it..
} }
} } // namespace scheduler
} } // namespace milvus
} } // namespace zilliz

21
cpp/src/scheduler/resource/Node.h
View File

@ -20,11 +20,11 @@
#include <vector> #include <vector>
#include <memory> #include <memory>
#include <map> #include <map>
#include <string>
#include "../TaskTable.h" #include "scheduler/TaskTable.h"
#include "Connection.h" #include "Connection.h"
namespace zilliz { namespace zilliz {
namespace milvus { namespace milvus {
namespace scheduler { namespace scheduler {
@ -34,8 +34,9 @@ class Node;
using NeighbourNodePtr = std::weak_ptr<Node>; using NeighbourNodePtr = std::weak_ptr<Node>;
struct Neighbour { struct Neighbour {
Neighbour(NeighbourNodePtr nei, Connection conn) Neighbour(NeighbourNodePtr nei, Connection conn)
: neighbour_node(nei), connection(conn) {} : neighbour_node(nei), connection(conn) {
}
NeighbourNodePtr neighbour_node; NeighbourNodePtr neighbour_node;
Connection connection; Connection connection;
@ -43,7 +44,7 @@ struct Neighbour {
// TODO(linxj): return type void -> Status // TODO(linxj): return type void -> Status
class Node { class Node {
public: public:
Node(); Node();
void void
@ -52,11 +53,11 @@ public:
std::vector<Neighbour> std::vector<Neighbour>
GetNeighbours(); GetNeighbours();
public: public:
std::string std::string
Dump(); Dump();
private: private:
std::mutex mutex_; std::mutex mutex_;
uint8_t id_; uint8_t id_;
std::map<uint8_t, Neighbour> neighbours_; std::map<uint8_t, Neighbour> neighbours_;
@ -65,6 +66,6 @@ private:
using NodePtr = std::shared_ptr<Node>; using NodePtr = std::shared_ptr<Node>;
using NodeWPtr = std::weak_ptr<Node>; using NodeWPtr = std::weak_ptr<Node>;
} } // namespace scheduler
} } // namespace milvus
} } // namespace zilliz

35
cpp/src/scheduler/resource/Resource.cpp
View File

@ -15,10 +15,11 @@
// specific language governing permissions and limitations // specific language governing permissions and limitations
// under the License. // under the License.
#include <iostream> #include "scheduler/resource/Resource.h"
#include "../Utils.h" #include "scheduler/Utils.h"
#include "Resource.h"
#include <iostream>
#include <utility>
namespace zilliz { namespace zilliz {
namespace milvus { namespace milvus {
@ -100,7 +101,8 @@ Resource::NumOfTaskToExec() {
return count; return count;
} }
TaskTableItemPtr Resource::pick_task_load() { TaskTableItemPtr
Resource::pick_task_load() {
auto indexes = task_table_.PickToLoad(10); auto indexes = task_table_.PickToLoad(10);
for (auto index : indexes) { for (auto index : indexes) {
// try to set one task loading, then return // try to set one task loading, then return
@ -111,7 +113,8 @@ TaskTableItemPtr Resource::pick_task_load() {
return nullptr; return nullptr;
} }
TaskTableItemPtr Resource::pick_task_execute() { TaskTableItemPtr
Resource::pick_task_execute() {
auto indexes = task_table_.PickToExecute(3); auto indexes = task_table_.PickToExecute(3);
for (auto index : indexes) { for (auto index : indexes) {
// try to set one task executing, then return // try to set one task executing, then return
@ -122,10 +125,13 @@ TaskTableItemPtr Resource::pick_task_execute() {
return nullptr; return nullptr;
} }
void Resource::loader_function() { void
Resource::loader_function() {
while (running_) { while (running_) {
std::unique_lock<std::mutex> lock(load_mutex_); std::unique_lock<std::mutex> lock(load_mutex_);
load_cv_.wait(lock, [&] { return load_flag_; }); load_cv_.wait(lock, [&] {
return load_flag_;
});
load_flag_ = false; load_flag_ = false;
lock.unlock(); lock.unlock();
while (true) { while (true) {
@ -140,18 +146,20 @@ void Resource::loader_function() {
subscriber_(std::static_pointer_cast<Event>(event)); subscriber_(std::static_pointer_cast<Event>(event));
} }
} }
} }
} }
void Resource::executor_function() { void
Resource::executor_function() {
if (subscriber_) { if (subscriber_) {
auto event = std::make_shared<StartUpEvent>(shared_from_this()); auto event = std::make_shared<StartUpEvent>(shared_from_this());
subscriber_(std::static_pointer_cast<Event>(event)); subscriber_(std::static_pointer_cast<Event>(event));
} }
while (running_) { while (running_) {
std::unique_lock<std::mutex> lock(exec_mutex_); std::unique_lock<std::mutex> lock(exec_mutex_);
exec_cv_.wait(lock, [&] { return exec_flag_; }); exec_cv_.wait(lock, [&] {
return exec_flag_;
});
exec_flag_ = false; exec_flag_ = false;
lock.unlock(); lock.unlock();
while (true) { while (true) {
@ -172,10 +180,9 @@ void Resource::executor_function() {
subscriber_(std::static_pointer_cast<Event>(event)); subscriber_(std::static_pointer_cast<Event>(event));
} }
} }
} }
} }
} } // namespace scheduler
} } // namespace milvus
} } // namespace zilliz

11
cpp/src/scheduler/resource/Resource.h
View File

@ -20,6 +20,7 @@
#include <string> #include <string>
#include <vector> #include <vector>
#include <memory> #include <memory>
#include <utility>
#include <thread> #include <thread>
#include <functional> #include <functional>
#include <condition_variable> #include <condition_variable>
@ -34,7 +35,6 @@
#include "Connection.h" #include "Connection.h"
#include "Node.h" #include "Node.h"
namespace zilliz { namespace zilliz {
namespace milvus { namespace milvus {
namespace scheduler { namespace scheduler {
@ -104,7 +104,7 @@ class Resource : public Node, public std::enable_shared_from_this<Resource> {
return task_table_; return task_table_;
} }
public: public:
inline bool inline bool
HasLoader() const { HasLoader() const {
return enable_loader_; return enable_loader_;
@ -212,7 +212,6 @@ public:
using ResourcePtr = std::shared_ptr<Resource>; using ResourcePtr = std::shared_ptr<Resource>;
using ResourceWPtr = std::weak_ptr<Resource>; using ResourceWPtr = std::weak_ptr<Resource>;
} } // namespace scheduler
} } // namespace milvus
} } // namespace zilliz

19
cpp/src/scheduler/resource/TestResource.cpp
View File

@ -15,14 +15,16 @@
// specific language governing permissions and limitations // specific language governing permissions and limitations
// under the License. // under the License.
#include "TestResource.h" #include "scheduler/resource/TestResource.h"
#include <string>
namespace zilliz { namespace zilliz {
namespace milvus { namespace milvus {
namespace scheduler { namespace scheduler {
std::ostream &operator<<(std::ostream &out, const TestResource &resource) { std::ostream &
operator<<(std::ostream &out, const TestResource &resource) {
out << resource.Dump(); out << resource.Dump();
return out; return out;
} }
@ -31,15 +33,16 @@ TestResource::TestResource(std::string name, uint64_t device_id, bool enable_loa
: Resource(std::move(name), ResourceType::TEST, device_id, enable_loader, enable_executor) { : Resource(std::move(name), ResourceType::TEST, device_id, enable_loader, enable_executor) {
} }
void TestResource::LoadFile(TaskPtr task) { void
TestResource::LoadFile(TaskPtr task) {
task->Load(LoadType::TEST, 0); task->Load(LoadType::TEST, 0);
} }
void TestResource::Process(TaskPtr task) { void
TestResource::Process(TaskPtr task) {
task->Execute(); task->Execute();
} }
} } // namespace scheduler
} } // namespace milvus
} } // namespace zilliz

13
cpp/src/scheduler/resource/TestResource.h
View File

@ -17,16 +17,17 @@
#pragma once #pragma once
#include "Resource.h" #include "Resource.h"
#include <utility>
#include <string>
namespace zilliz { namespace zilliz {
namespace milvus { namespace milvus {
namespace scheduler { namespace scheduler {
class TestResource : public Resource { class TestResource : public Resource {
public: public:
explicit explicit
TestResource(std::string name, uint64_t device_id, bool enable_loader, bool enable_executor); TestResource(std::string name, uint64_t device_id, bool enable_loader, bool enable_executor);
@ -37,7 +38,7 @@ public:
friend std::ostream &operator<<(std::ostream &out, const TestResource &resource); friend std::ostream &operator<<(std::ostream &out, const TestResource &resource);
protected: protected:
void void
LoadFile(TaskPtr task) override; LoadFile(TaskPtr task) override;
@ -45,6 +46,6 @@ protected:
Process(TaskPtr task) override; Process(TaskPtr task) override;
}; };
} } // namespace scheduler
} } // namespace milvus
} } // namespace zilliz

13
cpp/src/scheduler/task/DeleteTask.cpp
View File

@ -16,19 +16,18 @@
// under the License. // under the License.
#include "DeleteTask.h" #include "scheduler/task/DeleteTask.h"
namespace zilliz { namespace zilliz {
namespace milvus { namespace milvus {
namespace scheduler { namespace scheduler {
XDeleteTask::XDeleteTask(const scheduler::DeleteJobPtr &delete_job) XDeleteTask::XDeleteTask(const scheduler::DeleteJobPtr &delete_job)
: Task(TaskType::DeleteTask), delete_job_(delete_job) {} : Task(TaskType::DeleteTask), delete_job_(delete_job) {
}
void void
XDeleteTask::Load(LoadType type, uint8_t device_id) { XDeleteTask::Load(LoadType type, uint8_t device_id) {
} }
void void
@ -36,6 +35,6 @@ XDeleteTask::Execute() {
delete_job_->ResourceDone(); delete_job_->ResourceDone();
} }
} } // namespace scheduler
} } // namespace milvus
} } // namespace zilliz

14
cpp/src/scheduler/task/DeleteTask.h
View File

@ -20,15 +20,13 @@
#include "scheduler/job/DeleteJob.h" #include "scheduler/job/DeleteJob.h"
#include "Task.h" #include "Task.h"
namespace zilliz { namespace zilliz {
namespace milvus { namespace milvus {
namespace scheduler { namespace scheduler {
class XDeleteTask : public Task { class XDeleteTask : public Task {
public: public:
explicit explicit XDeleteTask(const scheduler::DeleteJobPtr &job);
XDeleteTask(const scheduler::DeleteJobPtr &job);
void void
Load(LoadType type, uint8_t device_id) override; Load(LoadType type, uint8_t device_id) override;
@ -36,10 +34,10 @@ public:
void void
Execute() override; Execute() override;
public: public:
scheduler::DeleteJobPtr delete_job_; scheduler::DeleteJobPtr delete_job_;
}; };
} } // namespace scheduler
} } // namespace milvus
} } // namespace zilliz

20
cpp/src/scheduler/task/Path.h
View File

@ -20,7 +20,6 @@
#include <vector> #include <vector>
#include <string> #include <string>
namespace zilliz { namespace zilliz {
namespace milvus { namespace milvus {
namespace scheduler { namespace scheduler {
@ -29,7 +28,8 @@ class Path {
public: public:
Path() = default; Path() = default;
Path(std::vector<std::string>& path, uint64_t index) : path_(path), index_(index) {} Path(std::vector<std::string> &path, uint64_t index) : path_(path), index_(index) {
}
void void
push_back(const std::string &str) { push_back(const std::string &str) {
@ -49,7 +49,6 @@ class Path {
} else { } else {
return nullptr; return nullptr;
} }
} }
std::string std::string
@ -67,14 +66,19 @@ class Path {
return path_[index]; return path_[index];
} }
std::vector<std::string>::iterator begin() { return path_.begin(); } std::vector<std::string>::iterator begin() {
std::vector<std::string>::iterator end() { return path_.end(); } return path_.begin();
}
std::vector<std::string>::iterator end() {
return path_.end();
}
public: public:
std::vector<std::string> path_; std::vector<std::string> path_;
uint64_t index_ = 0; uint64_t index_ = 0;
}; };
} } // namespace scheduler
} } // namespace milvus
} } // namespace zilliz

50
cpp/src/scheduler/task/SearchTask.cpp
View File

@ -15,15 +15,16 @@
// specific language governing permissions and limitations // specific language governing permissions and limitations
// under the License. // under the License.
#include "SearchTask.h" #include "scheduler/task/SearchTask.h"
#include "scheduler/job/SearchJob.h"
#include "metrics/Metrics.h" #include "metrics/Metrics.h"
#include "db/engine/EngineFactory.h" #include "db/engine/EngineFactory.h"
#include "utils/TimeRecorder.h" #include "utils/TimeRecorder.h"
#include "utils/Log.h" #include "utils/Log.h"
#include <thread> #include <thread>
#include "scheduler/job/SearchJob.h" #include <utility>
#include <string>
namespace zilliz { namespace zilliz {
namespace milvus { namespace milvus {
@ -104,7 +105,6 @@ XSearchTask::XSearchTask(TableFileSchemaPtr file)
(MetricType) file_->metric_type_, (MetricType) file_->metric_type_,
file_->nlist_); file_->nlist_);
} }
} }
void void
@ -144,7 +144,7 @@ XSearchTask::Load(LoadType type, uint8_t device_id) {
s = Status(SERVER_UNEXPECTED_ERROR, error_msg); s = Status(SERVER_UNEXPECTED_ERROR, error_msg);
} }
if (auto job = job_.lock()){ if (auto job = job_.lock()) {
auto search_job = std::static_pointer_cast<scheduler::SearchJob>(job); auto search_job = std::static_pointer_cast<scheduler::SearchJob>(job);
search_job->SearchDone(file_->id_); search_job->SearchDone(file_->id_);
search_job->GetStatus() = s; search_job->GetStatus() = s;
@ -183,7 +183,7 @@ XSearchTask::Execute() {
server::CollectDurationMetrics metrics(index_type_); server::CollectDurationMetrics metrics(index_type_);
std::vector<long> output_ids; std::vector<int64_t> output_ids;
std::vector<float> output_distance; std::vector<float> output_distance;
if (auto job = job_.lock()) { if (auto job = job_.lock()) {
@ -192,7 +192,7 @@ XSearchTask::Execute() {
uint64_t nq = search_job->nq(); uint64_t nq = search_job->nq();
uint64_t topk = search_job->topk(); uint64_t topk = search_job->topk();
uint64_t nprobe = search_job->nprobe(); uint64_t nprobe = search_job->nprobe();
const float* vectors = search_job->vectors(); const float *vectors = search_job->vectors();
output_ids.resize(topk * nq); output_ids.resize(topk * nq);
output_distance.resize(topk * nq); output_distance.resize(topk * nq);
@ -236,11 +236,12 @@ XSearchTask::Execute() {
index_engine_ = nullptr; index_engine_ = nullptr;
} }
Status XSearchTask::ClusterResult(const std::vector<long> &output_ids, Status
const std::vector<float> &output_distance, XSearchTask::ClusterResult(const std::vector<int64_t> &output_ids,
uint64_t nq, const std::vector<float> &output_distance,
uint64_t topk, uint64_t nq,
scheduler::ResultSet &result_set) { uint64_t topk,
scheduler::ResultSet &result_set) {
if (output_ids.size() < nq * topk || output_distance.size() < nq * topk) { if (output_ids.size() < nq * topk || output_distance.size() < nq * topk) {
std::string msg = "Invalid id array size: " + std::to_string(output_ids.size()) + std::string msg = "Invalid id array size: " + std::to_string(output_ids.size()) +
" distance array size: " + std::to_string(output_distance.size()); " distance array size: " + std::to_string(output_distance.size());
@ -275,10 +276,11 @@ Status XSearchTask::ClusterResult(const std::vector<long> &output_ids,
return Status::OK(); return Status::OK();
} }
Status XSearchTask::MergeResult(scheduler::Id2DistanceMap &distance_src, Status
scheduler::Id2DistanceMap &distance_target, XSearchTask::MergeResult(scheduler::Id2DistanceMap &distance_src,
uint64_t topk, scheduler::Id2DistanceMap &distance_target,
bool ascending) { uint64_t topk,
bool ascending) {
//Note: the score_src and score_target are already arranged by score in ascending order //Note: the score_src and score_target are already arranged by score in ascending order
if (distance_src.empty()) { if (distance_src.empty()) {
ENGINE_LOG_WARNING << "Empty distance source array"; ENGINE_LOG_WARNING << "Empty distance source array";
@ -349,10 +351,11 @@ Status XSearchTask::MergeResult(scheduler::Id2DistanceMap &distance_src,
return Status::OK(); return Status::OK();
} }
Status XSearchTask::TopkResult(scheduler::ResultSet &result_src, Status
uint64_t topk, XSearchTask::TopkResult(scheduler::ResultSet &result_src,
bool ascending, uint64_t topk,
scheduler::ResultSet &result_target) { bool ascending,
scheduler::ResultSet &result_target) {
if (result_target.empty()) { if (result_target.empty()) {
result_target.swap(result_src); result_target.swap(result_src);
return Status::OK(); return Status::OK();
@ -381,7 +384,6 @@ Status XSearchTask::TopkResult(scheduler::ResultSet &result_src,
return Status::OK(); return Status::OK();
} }
} // namespace scheduler
} } // namespace milvus
} } // namespace zilliz
}

18
cpp/src/scheduler/task/SearchTask.h
View File

@ -21,6 +21,7 @@
#include "scheduler/job/SearchJob.h" #include "scheduler/job/SearchJob.h"
#include "scheduler/Definition.h" #include "scheduler/Definition.h"
#include <vector>
namespace zilliz { namespace zilliz {
namespace milvus { namespace milvus {
@ -28,9 +29,8 @@ namespace scheduler {
// TODO: rewrite // TODO: rewrite
class XSearchTask : public Task { class XSearchTask : public Task {
public: public:
explicit explicit XSearchTask(TableFileSchemaPtr file);
XSearchTask(TableFileSchemaPtr file);
void void
Load(LoadType type, uint8_t device_id) override; Load(LoadType type, uint8_t device_id) override;
@ -38,8 +38,8 @@ public:
void void
Execute() override; Execute() override;
public: public:
static Status ClusterResult(const std::vector<long> &output_ids, static Status ClusterResult(const std::vector<int64_t> &output_ids,
const std::vector<float> &output_distence, const std::vector<float> &output_distence,
uint64_t nq, uint64_t nq,
uint64_t topk, uint64_t topk,
@ -55,7 +55,7 @@ public:
bool ascending, bool ascending,
scheduler::ResultSet &result_target); scheduler::ResultSet &result_target);
public: public:
TableFileSchemaPtr file_; TableFileSchemaPtr file_;
size_t index_id_ = 0; size_t index_id_ = 0;
@ -66,6 +66,6 @@ public:
static std::mutex merge_mutex_; static std::mutex merge_mutex_;
}; };
} } // namespace scheduler
} } // namespace milvus
} } // namespace zilliz

24
cpp/src/scheduler/task/Task.h
View File

@ -25,7 +25,6 @@
#include <string> #include <string>
#include <memory> #include <memory>
namespace zilliz { namespace zilliz {
namespace milvus { namespace milvus {
namespace scheduler { namespace scheduler {
@ -49,20 +48,22 @@ using TaskPtr = std::shared_ptr<Task>;
// TODO: re-design // TODO: re-design
class Task { class Task {
public: public:
explicit explicit Task(TaskType type) : type_(type) {
Task(TaskType type) : type_(type) {} }
/* /*
* Just Getter; * Just Getter;
*/ */
inline TaskType inline TaskType
Type() const { return type_; } Type() const {
return type_;
}
/* /*
* Transport path; * Transport path;
*/ */
inline Path& inline Path &
path() { path() {
return task_path_; return task_path_;
} }
@ -75,14 +76,14 @@ public:
return label_; return label_;
} }
public: public:
virtual void virtual void
Load(LoadType type, uint8_t device_id) = 0; Load(LoadType type, uint8_t device_id) = 0;
virtual void virtual void
Execute() = 0; Execute() = 0;
public: public:
Path task_path_; Path task_path_;
// std::vector<SearchContextPtr> search_contexts_; // std::vector<SearchContextPtr> search_contexts_;
scheduler::JobWPtr job_; scheduler::JobWPtr job_;
@ -90,7 +91,6 @@ public:
TaskLabelPtr label_ = nullptr; TaskLabelPtr label_ = nullptr;
}; };
} // namespace scheduler
} } // namespace milvus
} } // namespace zilliz
}

20
cpp/src/scheduler/task/TestTask.cpp
View File

@ -15,16 +15,15 @@
// specific language governing permissions and limitations // specific language governing permissions and limitations
// under the License. // under the License.
#include "scheduler/task/TestTask.h"
#include <src/cache/GpuCacheMgr.h> #include "cache/GpuCacheMgr.h"
#include "TestTask.h"
namespace zilliz { namespace zilliz {
namespace milvus { namespace milvus {
namespace scheduler { namespace scheduler {
TestTask::TestTask(TableFileSchemaPtr &file) : XSearchTask(file) {} TestTask::TestTask(TableFileSchemaPtr &file) : XSearchTask(file) {
}
void void
TestTask::Load(LoadType type, uint8_t device_id) { TestTask::Load(LoadType type, uint8_t device_id) {
@ -44,10 +43,11 @@ TestTask::Execute() {
void void
TestTask::Wait() { TestTask::Wait() {
std::unique_lock<std::mutex> lock(mutex_); std::unique_lock<std::mutex> lock(mutex_);
cv_.wait(lock, [&] { return done_; }); cv_.wait(lock, [&] {
} return done_;
});
}
}
} }
} // namespace scheduler
} // namespace milvus
} // namespace zilliz

17
cpp/src/scheduler/task/TestTask.h
View File

@ -19,17 +19,15 @@
#include "SearchTask.h" #include "SearchTask.h"
namespace zilliz { namespace zilliz {
namespace milvus { namespace milvus {
namespace scheduler { namespace scheduler {
class TestTask : public XSearchTask { class TestTask : public XSearchTask {
public: public:
explicit explicit TestTask(TableFileSchemaPtr &file);
TestTask(TableFileSchemaPtr& file);
public: public:
void void
Load(LoadType type, uint8_t device_id) override; Load(LoadType type, uint8_t device_id) override;
@ -39,7 +37,7 @@ public:
void void
Wait(); Wait();
public: public:
uint64_t load_count_ = 0; uint64_t load_count_ = 0;
uint64_t exec_count_ = 0; uint64_t exec_count_ = 0;
@ -48,7 +46,6 @@ public:
std::condition_variable cv_; std::condition_variable cv_;
}; };
} // namespace scheduler
} } // namespace milvus
} } // namespace zilliz
}

14
cpp/src/scheduler/tasklabel/BroadcastLabel.h
View File

@ -21,19 +21,19 @@
#include <memory> #include <memory>
namespace zilliz { namespace zilliz {
namespace milvus { namespace milvus {
namespace scheduler { namespace scheduler {
class BroadcastLabel : public TaskLabel { class BroadcastLabel : public TaskLabel {
public: public:
BroadcastLabel() : TaskLabel(TaskLabelType::BROADCAST) {} BroadcastLabel() : TaskLabel(TaskLabelType::BROADCAST) {
}
}; };
using BroadcastLabelPtr = std::shared_ptr<BroadcastLabel>; using BroadcastLabelPtr = std::shared_ptr<BroadcastLabel>;
} } // namespace scheduler
} } // namespace milvus
} } // namespace zilliz

14
cpp/src/scheduler/tasklabel/DefaultLabel.h
View File

@ -21,20 +21,18 @@
#include <memory> #include <memory>
namespace zilliz { namespace zilliz {
namespace milvus { namespace milvus {
namespace scheduler { namespace scheduler {
class DefaultLabel : public TaskLabel { class DefaultLabel : public TaskLabel {
public: public:
DefaultLabel() : TaskLabel(TaskLabelType::DEFAULT) {} DefaultLabel() : TaskLabel(TaskLabelType::DEFAULT) {
}
}; };
using DefaultLabelPtr = std::shared_ptr<DefaultLabel>; using DefaultLabelPtr = std::shared_ptr<DefaultLabel>;
} } // namespace scheduler
} } // namespace milvus
} } // namespace zilliz

17
cpp/src/scheduler/tasklabel/SpecResLabel.h
View File

@ -22,7 +22,6 @@
#include <string> #include <string>
#include <memory> #include <memory>
class Resource; class Resource;
using ResourceWPtr = std::weak_ptr<Resource>; using ResourceWPtr = std::weak_ptr<Resource>;
@ -32,9 +31,10 @@ namespace milvus {
namespace scheduler { namespace scheduler {
class SpecResLabel : public TaskLabel { class SpecResLabel : public TaskLabel {
public: public:
SpecResLabel(const ResourceWPtr &resource) explicit SpecResLabel(const ResourceWPtr &resource)
: TaskLabel(TaskLabelType::SPECIFIED_RESOURCE), resource_(resource) {} : TaskLabel(TaskLabelType::SPECIFIED_RESOURCE), resource_(resource) {
}
inline ResourceWPtr & inline ResourceWPtr &
resource() { resource() {
@ -46,14 +46,13 @@ public:
return resource_name_; return resource_name_;
} }
private: private:
ResourceWPtr resource_; ResourceWPtr resource_;
std::string resource_name_; std::string resource_name_;
}; };
using SpecResLabelPtr = std::shared_ptr<SpecResLabel>(); using SpecResLabelPtr = std::shared_ptr<SpecResLabel>();
} } // namespace scheduler
} } // namespace milvus
} } // namespace zilliz

17
cpp/src/scheduler/tasklabel/TaskLabel.h
View File

@ -30,23 +30,22 @@ enum class TaskLabelType {
}; };
class TaskLabel { class TaskLabel {
public: public:
inline TaskLabelType inline TaskLabelType
Type() const { Type() const {
return type_; return type_;
} }
protected: protected:
explicit explicit TaskLabel(TaskLabelType type) : type_(type) {
TaskLabel(TaskLabelType type) : type_(type) {} }
private: private:
TaskLabelType type_; TaskLabelType type_;
}; };
using TaskLabelPtr = std::shared_ptr<TaskLabel>; using TaskLabelPtr = std::shared_ptr<TaskLabel>;
} } // namespace scheduler
} } // namespace milvus
} } // namespace zilliz