milvus-io
/
milvus
mirror of https://github.com/milvus-io/milvus.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'branch-0.5.0' into 'branch-0.5.0' 

format scheduler code

See merge request megasearch/milvus!627

Former-commit-id: 4b46d2b89a1309faa30580dbcf58111c13c68ae5
pull/191/head
peng.xu 2019-09-27 14:22:23 +08:00
parent d4babb6dde a0e4d19e4f
commit b046049ffd
97 changed files with 1694 additions and 1578 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
cpp/src/db/Constants.h
View File

@ -33,8 +33,8 @@ constexpr uint64_t MAX_TABLE_FILE_MEM = 128 * M;
constexpr int VECTOR_TYPE_SIZE = sizeof(float);
static constexpr uint64_t ONE_KB = K;
static constexpr uint64_t ONE_MB = ONE_KB*ONE_KB;
static constexpr uint64_t ONE_GB = ONE_KB*ONE_MB;
static constexpr uint64_t ONE_MB = ONE_KB * ONE_KB;
static constexpr uint64_t ONE_GB = ONE_KB * ONE_MB;
} // namespace engine
} // namespace milvus

48
cpp/src/db/DB.h
View File

@ -25,6 +25,7 @@
#include <string>
#include <memory>
#include <vector>
namespace zilliz {
namespace milvus {
@ -33,46 +34,45 @@ namespace engine {
class Env;
class DB {
public:
public:
DB() = default;
DB(const DB&) = delete;
DB& operator=(const DB&) = delete;
DB(const DB &) = delete;
DB &operator=(const DB &) = delete;
virtual ~DB() = default;
virtual Status Start() = 0;
virtual Status Stop() = 0;
virtual Status CreateTable(meta::TableSchema& table_schema_) = 0;
virtual Status DeleteTable(const std::string& table_id, const meta::DatesT& dates) = 0;
virtual Status DescribeTable(meta::TableSchema& table_schema_) = 0;
virtual Status HasTable(const std::string& table_id, bool& has_or_not_) = 0;
virtual Status AllTables(std::vector<meta::TableSchema>& table_schema_array) = 0;
virtual Status GetTableRowCount(const std::string& table_id, uint64_t& row_count) = 0;
virtual Status PreloadTable(const std::string& table_id) = 0;
virtual Status CreateTable(meta::TableSchema &table_schema_) = 0;
virtual Status DeleteTable(const std::string &table_id, const meta::DatesT &dates) = 0;
virtual Status DescribeTable(meta::TableSchema &table_schema_) = 0;
virtual Status HasTable(const std::string &table_id, bool &has_or_not_) = 0;
virtual Status AllTables(std::vector<meta::TableSchema> &table_schema_array) = 0;
virtual Status GetTableRowCount(const std::string &table_id, uint64_t &row_count) = 0;
virtual Status PreloadTable(const std::string &table_id) = 0;
virtual Status UpdateTableFlag(const std::string &table_id, int64_t flag) = 0;
virtual Status InsertVectors(const std::string& table_id_,
uint64_t n, const float* vectors, IDNumbers& vector_ids_) = 0;
virtual Status InsertVectors(const std::string &table_id_,
uint64_t n, const float *vectors, IDNumbers &vector_ids_) = 0;
virtual Status Query(const std::string& table_id, uint64_t k, uint64_t nq, uint64_t nprobe,
const float* vectors, QueryResults& results) = 0;
virtual Status Query(const std::string &table_id, uint64_t k, uint64_t nq, uint64_t nprobe,
const float *vectors, QueryResults &results) = 0;
virtual Status Query(const std::string& table_id, uint64_t k, uint64_t nq, uint64_t nprobe,
const float* vectors, const meta::DatesT& dates, QueryResults& results) = 0;
virtual Status Query(const std::string &table_id, uint64_t k, uint64_t nq, uint64_t nprobe,
const float *vectors, const meta::DatesT &dates, QueryResults &results) = 0;
virtual Status Query(const std::string& table_id, const std::vector<std::string>& file_ids,
uint64_t k, uint64_t nq, uint64_t nprobe, const float* vectors,
const meta::DatesT& dates, QueryResults& results) = 0;
virtual Status Query(const std::string &table_id, const std::vector<std::string> &file_ids,
uint64_t k, uint64_t nq, uint64_t nprobe, const float *vectors,
const meta::DatesT &dates, QueryResults &results) = 0;
virtual Status Size(uint64_t& result) = 0;
virtual Status Size(uint64_t &result) = 0;
virtual Status CreateIndex(const std::string& table_id, const TableIndex& index) = 0;
virtual Status DescribeIndex(const std::string& table_id, TableIndex& index) = 0;
virtual Status DropIndex(const std::string& table_id) = 0;
virtual Status CreateIndex(const std::string &table_id, const TableIndex &index) = 0;
virtual Status DescribeIndex(const std::string &table_id, TableIndex &index) = 0;
virtual Status DropIndex(const std::string &table_id) = 0;
virtual Status DropAll() = 0;
}; // DB
using DBPtr = std::shared_ptr<DB>;

10
cpp/src/db/DBFactory.cpp
View File

@ -16,7 +16,7 @@
// under the License.
#include "DBFactory.h"
#include "db/DBFactory.h"
#include "DBImpl.h"
#include "utils/Exception.h"
#include "meta/MetaFactory.h"
@ -33,14 +33,16 @@ namespace zilliz {
namespace milvus {
namespace engine {
DBOptions DBFactory::BuildOption() {
DBOptions
DBFactory::BuildOption() {
auto meta = MetaFactory::BuildOption();
DBOptions options;
DBOptions options;
options.meta_ = meta;
return options;
}
DBPtr DBFactory::Build(const DBOptions& options) {
DBPtr
DBFactory::Build(const DBOptions &options) {
return std::make_shared<DBImpl>(options);
}

11
cpp/src/db/DBFactory.h
View File

@ -28,13 +28,12 @@ namespace milvus {
namespace engine {
class DBFactory {
public:
public:
static DBOptions BuildOption();
static DBPtr Build(const DBOptions& options);
static DBPtr Build(const DBOptions &options);
};
}
}
}
} // namespace engine
} // namespace milvus
} // namespace zilliz

294
cpp/src/db/DBImpl.cpp
View File

@ -15,7 +15,7 @@
// specific language governing permissions and limitations
// under the License.
#include "DBImpl.h"
#include "db/DBImpl.h"
#include "cache/CpuCacheMgr.h"
#include "cache/GpuCacheMgr.h"
#include "engine/EngineFactory.h"
@ -36,6 +36,7 @@
#include <thread>
#include <iostream>
#include <cstring>
#include <algorithm>
#include <boost/filesystem.hpp>
namespace zilliz {
@ -48,10 +49,9 @@ constexpr uint64_t METRIC_ACTION_INTERVAL = 1;
constexpr uint64_t COMPACT_ACTION_INTERVAL = 1;
constexpr uint64_t INDEX_ACTION_INTERVAL = 1;
}
} // namespace
DBImpl::DBImpl(const DBOptions& options)
DBImpl::DBImpl(const DBOptions &options)
: options_(options),
shutting_down_(true),
compact_thread_pool_(1, 1),
@ -68,8 +68,9 @@ DBImpl::~DBImpl() {
///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
//external api
///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
Status DBImpl::Start() {
if (!shutting_down_.load(std::memory_order_acquire)){
Status
DBImpl::Start() {
if (!shutting_down_.load(std::memory_order_acquire)) {
return Status::OK();
}
@ -85,8 +86,9 @@ Status DBImpl::Start() {
return Status::OK();
}
Status DBImpl::Stop() {
if (shutting_down_.load(std::memory_order_acquire)){
Status
DBImpl::Stop() {
if (shutting_down_.load(std::memory_order_acquire)) {
return Status::OK();
}
@ -106,12 +108,14 @@ Status DBImpl::Stop() {
return Status::OK();
}
Status DBImpl::DropAll() {
Status
DBImpl::DropAll() {
return meta_ptr_->DropAll();
}
Status DBImpl::CreateTable(meta::TableSchema& table_schema) {
if (shutting_down_.load(std::memory_order_acquire)){
Status
DBImpl::CreateTable(meta::TableSchema &table_schema) {
if (shutting_down_.load(std::memory_order_acquire)) {
return Status(DB_ERROR, "Milsvus server is shutdown!");
}
@ -120,8 +124,9 @@ Status DBImpl::CreateTable(meta::TableSchema& table_schema) {
return meta_ptr_->CreateTable(temp_schema);
}
Status DBImpl::DeleteTable(const std::string& table_id, const meta::DatesT& dates) {
if (shutting_down_.load(std::memory_order_acquire)){
Status
DBImpl::DeleteTable(const std::string &table_id, const meta::DatesT &dates) {
if (shutting_down_.load(std::memory_order_acquire)) {
return Status(DB_ERROR, "Milsvus server is shutdown!");
}
@ -144,8 +149,9 @@ Status DBImpl::DeleteTable(const std::string& table_id, const meta::DatesT& date
return Status::OK();
}
Status DBImpl::DescribeTable(meta::TableSchema& table_schema) {
if (shutting_down_.load(std::memory_order_acquire)){
Status
DBImpl::DescribeTable(meta::TableSchema &table_schema) {
if (shutting_down_.load(std::memory_order_acquire)) {
return Status(DB_ERROR, "Milsvus server is shutdown!");
}
@ -154,24 +160,27 @@ Status DBImpl::DescribeTable(meta::TableSchema& table_schema) {
return stat;
}
Status DBImpl::HasTable(const std::string& table_id, bool& has_or_not) {
if (shutting_down_.load(std::memory_order_acquire)){
Status
DBImpl::HasTable(const std::string &table_id, bool &has_or_not) {
if (shutting_down_.load(std::memory_order_acquire)) {
return Status(DB_ERROR, "Milsvus server is shutdown!");
}
return meta_ptr_->HasTable(table_id, has_or_not);
}
Status DBImpl::AllTables(std::vector<meta::TableSchema>& table_schema_array) {
if (shutting_down_.load(std::memory_order_acquire)){
Status
DBImpl::AllTables(std::vector<meta::TableSchema> &table_schema_array) {
if (shutting_down_.load(std::memory_order_acquire)) {
return Status(DB_ERROR, "Milsvus server is shutdown!");
}
return meta_ptr_->AllTables(table_schema_array);
}
Status DBImpl::PreloadTable(const std::string &table_id) {
if (shutting_down_.load(std::memory_order_acquire)){
Status
DBImpl::PreloadTable(const std::string &table_id) {
if (shutting_down_.load(std::memory_order_acquire)) {
return Status(DB_ERROR, "Milsvus server is shutdown!");
}
@ -189,10 +198,14 @@ Status DBImpl::PreloadTable(const std::string &table_id) {
int64_t cache_usage = cache::CpuCacheMgr::GetInstance()->CacheUsage();
int64_t available_size = cache_total - cache_usage;
for(auto &day_files : files) {
for (auto &day_files : files) {
for (auto &file : day_files.second) {
ExecutionEnginePtr engine = EngineFactory::Build(file.dimension_, file.location_, (EngineType)file.engine_type_, (MetricType)file.metric_type_, file.nlist_);
if(engine == nullptr) {
ExecutionEnginePtr engine = EngineFactory::Build(file.dimension_,
file.location_,
(EngineType) file.engine_type_,
(MetricType) file.metric_type_,
file.nlist_);
if (engine == nullptr) {
ENGINE_LOG_ERROR << "Invalid engine type";
return Status(DB_ERROR, "Invalid engine type");
}
@ -215,33 +228,37 @@ Status DBImpl::PreloadTable(const std::string &table_id) {
return Status::OK();
}
Status DBImpl::UpdateTableFlag(const std::string &table_id, int64_t flag) {
if (shutting_down_.load(std::memory_order_acquire)){
Status
DBImpl::UpdateTableFlag(const std::string &table_id, int64_t flag) {
if (shutting_down_.load(std::memory_order_acquire)) {
return Status(DB_ERROR, "Milsvus server is shutdown!");
}
return meta_ptr_->UpdateTableFlag(table_id, flag);
}
Status DBImpl::GetTableRowCount(const std::string& table_id, uint64_t& row_count) {
if (shutting_down_.load(std::memory_order_acquire)){
Status
DBImpl::GetTableRowCount(const std::string &table_id, uint64_t &row_count) {
if (shutting_down_.load(std::memory_order_acquire)) {
return Status(DB_ERROR, "Milsvus server is shutdown!");
}
return meta_ptr_->Count(table_id, row_count);
}
Status DBImpl::InsertVectors(const std::string& table_id_,
uint64_t n, const float* vectors, IDNumbers& vector_ids_) {
Status
DBImpl::InsertVectors(const std::string &table_id_,
uint64_t n, const float *vectors, IDNumbers &vector_ids_) {
// ENGINE_LOG_DEBUG << "Insert " << n << " vectors to cache";
if (shutting_down_.load(std::memory_order_acquire)){
if (shutting_down_.load(std::memory_order_acquire)) {
return Status(DB_ERROR, "Milsvus server is shutdown!");
}
Status status;
zilliz::milvus::server::CollectInsertMetrics metrics(n, status);
status = mem_mgr_->InsertVectors(table_id_, n, vectors, vector_ids_);
// std::chrono::microseconds time_span = std::chrono::duration_cast<std::chrono::microseconds>(end_time - start_time);
// std::chrono::microseconds time_span =
// std::chrono::duration_cast<std::chrono::microseconds>(end_time - start_time);
// double average_time = double(time_span.count()) / n;
// ENGINE_LOG_DEBUG << "Insert vectors to cache finished";
@ -249,14 +266,15 @@ Status DBImpl::InsertVectors(const std::string& table_id_,
return status;
}
Status DBImpl::CreateIndex(const std::string& table_id, const TableIndex& index) {
Status
DBImpl::CreateIndex(const std::string &table_id, const TableIndex &index) {
{
std::unique_lock<std::mutex> lock(build_index_mutex_);
//step 1: check index difference
TableIndex old_index;
auto status = DescribeIndex(table_id, old_index);
if(!status.ok()) {
if (!status.ok()) {
ENGINE_LOG_ERROR << "Failed to get table index info for table: " << table_id;
return status;
}
@ -264,7 +282,7 @@ Status DBImpl::CreateIndex(const std::string& table_id, const TableIndex& index)
//step 2: update index info
TableIndex new_index = index;
new_index.metric_type_ = old_index.metric_type_;//dont change metric type, it was defined by CreateTable
if(!utils::IsSameIndex(old_index, new_index)) {
if (!utils::IsSameIndex(old_index, new_index)) {
DropIndex(table_id);
status = meta_ptr_->UpdateTableIndex(table_id, new_index);
@ -283,18 +301,18 @@ Status DBImpl::CreateIndex(const std::string& table_id, const TableIndex& index)
//for IDMAP type, only wait all NEW file converted to RAW file
//for other type, wait NEW/RAW/NEW_MERGE/NEW_INDEX/TO_INDEX files converted to INDEX files
std::vector<int> file_types;
if(index.engine_type_ == (int)EngineType::FAISS_IDMAP) {
if (index.engine_type_ == (int) EngineType::FAISS_IDMAP) {
file_types = {
(int) meta::TableFileSchema::NEW,
(int) meta::TableFileSchema::NEW_MERGE,
(int) meta::TableFileSchema::NEW,
(int) meta::TableFileSchema::NEW_MERGE,
};
} else {
file_types = {
(int) meta::TableFileSchema::RAW,
(int) meta::TableFileSchema::NEW,
(int) meta::TableFileSchema::NEW_MERGE,
(int) meta::TableFileSchema::NEW_INDEX,
(int) meta::TableFileSchema::TO_INDEX,
(int) meta::TableFileSchema::RAW,
(int) meta::TableFileSchema::NEW,
(int) meta::TableFileSchema::NEW_MERGE,
(int) meta::TableFileSchema::NEW_INDEX,
(int) meta::TableFileSchema::TO_INDEX,
};
}
@ -304,11 +322,11 @@ Status DBImpl::CreateIndex(const std::string& table_id, const TableIndex& index)
while (!file_ids.empty()) {
ENGINE_LOG_DEBUG << "Non index files detected! Will build index " << times;
if(index.engine_type_ != (int)EngineType::FAISS_IDMAP) {
if (index.engine_type_ != (int) EngineType::FAISS_IDMAP) {
status = meta_ptr_->UpdateTableFilesToIndex(table_id);
}
std::this_thread::sleep_for(std::chrono::milliseconds(std::min(10*1000, times*100)));
std::this_thread::sleep_for(std::chrono::milliseconds(std::min(10 * 1000, times * 100)));
status = meta_ptr_->FilesByType(table_id, file_types, file_ids);
times++;
}
@ -316,18 +334,21 @@ Status DBImpl::CreateIndex(const std::string& table_id, const TableIndex& index)
return Status::OK();
}
Status DBImpl::DescribeIndex(const std::string& table_id, TableIndex& index) {
Status
DBImpl::DescribeIndex(const std::string &table_id, TableIndex &index) {
return meta_ptr_->DescribeTableIndex(table_id, index);
}
Status DBImpl::DropIndex(const std::string& table_id) {
Status
DBImpl::DropIndex(const std::string &table_id) {
ENGINE_LOG_DEBUG << "Drop index for table: " << table_id;
return meta_ptr_->DropTableIndex(table_id);
}
Status DBImpl::Query(const std::string &table_id, uint64_t k, uint64_t nq, uint64_t nprobe,
const float *vectors, QueryResults &results) {
if (shutting_down_.load(std::memory_order_acquire)){
Status
DBImpl::Query(const std::string &table_id, uint64_t k, uint64_t nq, uint64_t nprobe,
const float *vectors, QueryResults &results) {
if (shutting_down_.load(std::memory_order_acquire)) {
return Status(DB_ERROR, "Milsvus server is shutdown!");
}
@ -337,9 +358,10 @@ Status DBImpl::Query(const std::string &table_id, uint64_t k, uint64_t nq, uint6
return result;
}
Status DBImpl::Query(const std::string& table_id, uint64_t k, uint64_t nq, uint64_t nprobe,
const float* vectors, const meta::DatesT& dates, QueryResults& results) {
if (shutting_down_.load(std::memory_order_acquire)){
Status
DBImpl::Query(const std::string &table_id, uint64_t k, uint64_t nq, uint64_t nprobe,
const float *vectors, const meta::DatesT &dates, QueryResults &results) {
if (shutting_down_.load(std::memory_order_acquire)) {
return Status(DB_ERROR, "Milsvus server is shutdown!");
}
@ -364,10 +386,11 @@ Status DBImpl::Query(const std::string& table_id, uint64_t k, uint64_t nq, uint6
return status;
}
Status DBImpl::Query(const std::string& table_id, const std::vector<std::string>& file_ids,
uint64_t k, uint64_t nq, uint64_t nprobe, const float* vectors,
const meta::DatesT& dates, QueryResults& results) {
if (shutting_down_.load(std::memory_order_acquire)){
Status
DBImpl::Query(const std::string &table_id, const std::vector<std::string> &file_ids,
uint64_t k, uint64_t nq, uint64_t nprobe, const float *vectors,
const meta::DatesT &dates, QueryResults &results) {
if (shutting_down_.load(std::memory_order_acquire)) {
return Status(DB_ERROR, "Milsvus server is shutdown!");
}
@ -395,7 +418,7 @@ Status DBImpl::Query(const std::string& table_id, const std::vector<std::string>
}
}
if(file_id_array.empty()) {
if (file_id_array.empty()) {
return Status(DB_ERROR, "Invalid file id");
}
@ -405,36 +428,37 @@ Status DBImpl::Query(const std::string& table_id, const std::vector<std::string>
return status;
}
Status DBImpl::Size(uint64_t& result) {
if (shutting_down_.load(std::memory_order_acquire)){
Status
DBImpl::Size(uint64_t &result) {
if (shutting_down_.load(std::memory_order_acquire)) {
return Status(DB_ERROR, "Milsvus server is shutdown!");
}
return meta_ptr_->Size(result);
return meta_ptr_->Size(result);
}
///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
//internal methods
///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
Status DBImpl::QueryAsync(const std::string& table_id, const meta::TableFilesSchema& files,
uint64_t k, uint64_t nq, uint64_t nprobe, const float* vectors,
const meta::DatesT& dates, QueryResults& results) {
using namespace scheduler;
Status
DBImpl::QueryAsync(const std::string &table_id, const meta::TableFilesSchema &files,
uint64_t k, uint64_t nq, uint64_t nprobe, const float *vectors,
const meta::DatesT &dates, QueryResults &results) {
server::CollectQueryMetrics metrics(nq);
TimeRecorder rc("");
//step 1: get files to search
ENGINE_LOG_DEBUG << "Engine query begin, index file count: " << files.size() << " date range count: " << dates.size();
SearchJobPtr job = std::make_shared<SearchJob>(0, k, nq, nprobe, vectors);
for (auto &file : files) {
TableFileSchemaPtr file_ptr = std::make_shared<meta::TableFileSchema>(file);
ENGINE_LOG_DEBUG << "Engine query begin, index file count: " << files.size() << " date range count: "
<< dates.size();
scheduler::SearchJobPtr job = std::make_shared<scheduler::SearchJob>(0, k, nq, nprobe, vectors);
for (auto &file : files) {
scheduler::TableFileSchemaPtr file_ptr = std::make_shared<meta::TableFileSchema>(file);
job->AddIndexFile(file_ptr);
}
//step 2: put search task to scheduler
JobMgrInst::GetInstance()->Put(job);
scheduler::JobMgrInst::GetInstance()->Put(job);
job->WaitResult();
if (!job->GetStatus().ok()) {
return job->GetStatus();
@ -453,9 +477,12 @@ Status DBImpl::QueryAsync(const std::string& table_id, const meta::TableFilesSch
// double search_percent = search_cost/total_cost;
// double reduce_percent = reduce_cost/total_cost;
//
// ENGINE_LOG_DEBUG << "Engine load index totally cost: " << load_info << " percent: " << load_percent*100 << "%";
// ENGINE_LOG_DEBUG << "Engine search index totally cost: " << search_info << " percent: " << search_percent*100 << "%";
// ENGINE_LOG_DEBUG << "Engine reduce topk totally cost: " << reduce_info << " percent: " << reduce_percent*100 << "%";
// ENGINE_LOG_DEBUG << "Engine load index totally cost: " << load_info
// << " percent: " << load_percent*100 << "%";
// ENGINE_LOG_DEBUG << "Engine search index totally cost: " << search_info
// << " percent: " << search_percent*100 << "%";
// ENGINE_LOG_DEBUG << "Engine reduce topk totally cost: " << reduce_info
// << " percent: " << reduce_percent*100 << "%";
// } else {
// ENGINE_LOG_DEBUG << "Engine load cost: " << load_info
// << " search cost: " << search_info
@ -469,11 +496,12 @@ Status DBImpl::QueryAsync(const std::string& table_id, const meta::TableFilesSch
return Status::OK();
}
void DBImpl::BackgroundTimerTask() {
void
DBImpl::BackgroundTimerTask() {
Status status;
server::SystemInfo::GetInstance().Init();
while (true) {
if (shutting_down_.load(std::memory_order_acquire)){
if (shutting_down_.load(std::memory_order_acquire)) {
WaitMergeFileFinish();
WaitBuildIndexFinish();
@ -489,24 +517,27 @@ void DBImpl::BackgroundTimerTask() {
}
}
void DBImpl::WaitMergeFileFinish() {
void
DBImpl::WaitMergeFileFinish() {
std::lock_guard<std::mutex> lck(compact_result_mutex_);
for(auto& iter : compact_thread_results_) {
for (auto &iter : compact_thread_results_) {
iter.wait();
}
}
void DBImpl::WaitBuildIndexFinish() {
void
DBImpl::WaitBuildIndexFinish() {
std::lock_guard<std::mutex> lck(index_result_mutex_);
for(auto& iter : index_thread_results_) {
for (auto &iter : index_thread_results_) {
iter.wait();
}
}
void DBImpl::StartMetricTask() {
void
DBImpl::StartMetricTask() {
static uint64_t metric_clock_tick = 0;
metric_clock_tick++;
if(metric_clock_tick%METRIC_ACTION_INTERVAL != 0) {
if (metric_clock_tick % METRIC_ACTION_INTERVAL != 0) {
return;
}
@ -515,7 +546,7 @@ void DBImpl::StartMetricTask() {
server::Metrics::GetInstance().KeepingAliveCounterIncrement(METRIC_ACTION_INTERVAL);
int64_t cache_usage = cache::CpuCacheMgr::GetInstance()->CacheUsage();
int64_t cache_total = cache::CpuCacheMgr::GetInstance()->CacheCapacity();
server::Metrics::GetInstance().CpuCacheUsageGaugeSet(cache_usage*100/cache_total);
server::Metrics::GetInstance().CpuCacheUsageGaugeSet(cache_usage * 100 / cache_total);
server::Metrics::GetInstance().GpuCacheUsageGaugeSet();
uint64_t size;
Size(size);
@ -533,25 +564,27 @@ void DBImpl::StartMetricTask() {
ENGINE_LOG_TRACE << "Metric task finished";
}
Status DBImpl::MemSerialize() {
Status
DBImpl::MemSerialize() {
std::lock_guard<std::mutex> lck(mem_serialize_mutex_);
std::set<std::string> temp_table_ids;
mem_mgr_->Serialize(temp_table_ids);
for(auto& id : temp_table_ids) {
for (auto &id : temp_table_ids) {
compact_table_ids_.insert(id);
}
if(!temp_table_ids.empty()) {
if (!temp_table_ids.empty()) {
SERVER_LOG_DEBUG << "Insert cache serialized";
}
return Status::OK();
}
void DBImpl::StartCompactionTask() {
void
DBImpl::StartCompactionTask() {
static uint64_t compact_clock_tick = 0;
compact_clock_tick++;
if(compact_clock_tick%COMPACT_ACTION_INTERVAL != 0) {
if (compact_clock_tick % COMPACT_ACTION_INTERVAL != 0) {
return;
}
@ -580,8 +613,9 @@ void DBImpl::StartCompactionTask() {
}
}
Status DBImpl::MergeFiles(const std::string& table_id, const meta::DateT& date,
const meta::TableFilesSchema& files) {
Status
DBImpl::MergeFiles(const std::string &table_id, const meta::DateT &date,
const meta::TableFilesSchema &files) {
ENGINE_LOG_DEBUG << "Merge files for table: " << table_id;
//step 1: create table file
@ -598,13 +632,13 @@ Status DBImpl::MergeFiles(const std::string& table_id, const meta::DateT& date,
//step 2: merge files
ExecutionEnginePtr index =
EngineFactory::Build(table_file.dimension_, table_file.location_, (EngineType)table_file.engine_type_,
(MetricType)table_file.metric_type_, table_file.nlist_);
EngineFactory::Build(table_file.dimension_, table_file.location_, (EngineType) table_file.engine_type_,
(MetricType) table_file.metric_type_, table_file.nlist_);
meta::TableFilesSchema updated;
long index_size = 0;
int64_t index_size = 0;
for (auto& file : files) {
for (auto &file : files) {
server::CollectMergeFilesMetrics metrics;
index->Merge(file.location_);
@ -620,7 +654,7 @@ Status DBImpl::MergeFiles(const std::string& table_id, const meta::DateT& date,
//step 3: serialize to disk
try {
index->Serialize();
} catch (std::exception& ex) {
} catch (std::exception &ex) {
//typical error: out of disk space or permition denied
std::string msg = "Serialize merged index encounter exception: " + std::string(ex.what());
ENGINE_LOG_ERROR << msg;
@ -638,7 +672,7 @@ Status DBImpl::MergeFiles(const std::string& table_id, const meta::DateT& date,
//step 4: update table files state
//if index type isn't IDMAP, set file type to TO_INDEX if file size execeed index_file_size
//else set file type to RAW, no need to build index
if (table_file.engine_type_ != (int)EngineType::FAISS_IDMAP) {
if (table_file.engine_type_ != (int) EngineType::FAISS_IDMAP) {
table_file.file_type_ = (index->PhysicalSize() >= table_file.index_file_size_) ?
meta::TableFileSchema::TO_INDEX : meta::TableFileSchema::RAW;
} else {
@ -649,16 +683,17 @@ Status DBImpl::MergeFiles(const std::string& table_id, const meta::DateT& date,
updated.push_back(table_file);
status = meta_ptr_->UpdateTableFiles(updated);
ENGINE_LOG_DEBUG << "New merged file " << table_file.file_id_ <<
" of size " << index->PhysicalSize() << " bytes";
" of size " << index->PhysicalSize() << " bytes";
if(options_.insert_cache_immediately_) {
if (options_.insert_cache_immediately_) {
index->Cache();
}
return status;
}
Status DBImpl::BackgroundMergeFiles(const std::string& table_id) {
Status
DBImpl::BackgroundMergeFiles(const std::string &table_id) {
meta::DatePartionedTableFilesSchema raw_files;
auto status = meta_ptr_->FilesToMerge(table_id, raw_files);
if (!status.ok()) {
@ -667,7 +702,7 @@ Status DBImpl::BackgroundMergeFiles(const std::string& table_id) {
}
bool has_merge = false;
for (auto& kv : raw_files) {
for (auto &kv : raw_files) {
auto files = kv.second;
if (files.size() < options_.merge_trigger_number_) {
ENGINE_LOG_DEBUG << "Files number not greater equal than merge trigger number, skip merge action";
@ -676,7 +711,7 @@ Status DBImpl::BackgroundMergeFiles(const std::string& table_id) {
has_merge = true;
MergeFiles(table_id, kv.first, kv.second);
if (shutting_down_.load(std::memory_order_acquire)){
if (shutting_down_.load(std::memory_order_acquire)) {
ENGINE_LOG_DEBUG << "Server will shutdown, skip merge action for table: " << table_id;
break;
}
@ -685,17 +720,18 @@ Status DBImpl::BackgroundMergeFiles(const std::string& table_id) {
return Status::OK();
}
void DBImpl::BackgroundCompaction(std::set<std::string> table_ids) {
void
DBImpl::BackgroundCompaction(std::set<std::string> table_ids) {
ENGINE_LOG_TRACE << " Background compaction thread start";
Status status;
for (auto& table_id : table_ids) {
for (auto &table_id : table_ids) {
status = BackgroundMergeFiles(table_id);
if (!status.ok()) {
ENGINE_LOG_ERROR << "Merge files for table " << table_id << " failed: " << status.ToString();
}
if (shutting_down_.load(std::memory_order_acquire)){
if (shutting_down_.load(std::memory_order_acquire)) {
ENGINE_LOG_DEBUG << "Server will shutdown, skip merge action";
break;
}
@ -703,7 +739,7 @@ void DBImpl::BackgroundCompaction(std::set<std::string> table_ids) {
meta_ptr_->Archive();
int ttl = 5*meta::M_SEC;//default: file will be deleted after 5 minutes
int ttl = 5 * meta::M_SEC;//default: file will be deleted after 5 minutes
if (options_.mode_ == DBOptions::MODE::CLUSTER_WRITABLE) {
ttl = meta::D_SEC;
}
@ -712,10 +748,11 @@ void DBImpl::BackgroundCompaction(std::set<std::string> table_ids) {
ENGINE_LOG_TRACE << " Background compaction thread exit";
}
void DBImpl::StartBuildIndexTask(bool force) {
void
DBImpl::StartBuildIndexTask(bool force) {
static uint64_t index_clock_tick = 0;
index_clock_tick++;
if(!force && (index_clock_tick%INDEX_ACTION_INTERVAL != 0)) {
if (!force && (index_clock_tick % INDEX_ACTION_INTERVAL != 0)) {
return;
}
@ -740,11 +777,12 @@ void DBImpl::StartBuildIndexTask(bool force) {
}
}
Status DBImpl::BuildIndex(const meta::TableFileSchema& file) {
Status
DBImpl::BuildIndex(const meta::TableFileSchema &file) {
ExecutionEnginePtr to_index =
EngineFactory::Build(file.dimension_, file.location_, (EngineType)file.engine_type_,
(MetricType)file.metric_type_, file.nlist_);
if(to_index == nullptr) {
EngineFactory::Build(file.dimension_, file.location_, (EngineType) file.engine_type_,
(MetricType) file.metric_type_, file.nlist_);
if (to_index == nullptr) {
ENGINE_LOG_ERROR << "Invalid engine type";
return Status(DB_ERROR, "Invalid engine type");
}
@ -761,7 +799,8 @@ Status DBImpl::BuildIndex(const meta::TableFileSchema& file) {
meta::TableFileSchema table_file;
table_file.table_id_ = file.table_id_;
table_file.date_ = file.date_;
table_file.file_type_ = meta::TableFileSchema::NEW_INDEX; //for multi-db-path, distribute index file averagely to each path
table_file.file_type_ =
meta::TableFileSchema::NEW_INDEX; //for multi-db-path, distribute index file averagely to each path
status = meta_ptr_->CreateTableFile(table_file);
if (!status.ok()) {
ENGINE_LOG_ERROR << "Failed to create table file: " << status.ToString();
@ -773,16 +812,16 @@ Status DBImpl::BuildIndex(const meta::TableFileSchema& file) {
try {
server::CollectBuildIndexMetrics metrics;
index = to_index->BuildIndex(table_file.location_, (EngineType)table_file.engine_type_);
index = to_index->BuildIndex(table_file.location_, (EngineType) table_file.engine_type_);
if (index == nullptr) {
table_file.file_type_ = meta::TableFileSchema::TO_DELETE;
status = meta_ptr_->UpdateTableFile(table_file);
ENGINE_LOG_DEBUG << "Failed to update file to index, mark file: " << table_file.file_id_ << " to to_delete";
ENGINE_LOG_DEBUG << "Failed to update file to index, mark file: " << table_file.file_id_
<< " to to_delete";
return status;
}
} catch (std::exception& ex) {
} catch (std::exception &ex) {
//typical error: out of gpu memory
std::string msg = "BuildIndex encounter exception: " + std::string(ex.what());
ENGINE_LOG_ERROR << msg;
@ -791,7 +830,8 @@ Status DBImpl::BuildIndex(const meta::TableFileSchema& file) {
status = meta_ptr_->UpdateTableFile(table_file);
ENGINE_LOG_DEBUG << "Failed to update file to index, mark file: " << table_file.file_id_ << " to to_delete";
std::cout << "ERROR: failed to build index, index file is too large or gpu memory is not enough" << std::endl;
std::cout << "ERROR: failed to build index, index file is too large or gpu memory is not enough"
<< std::endl;
return Status(DB_ERROR, msg);
}
@ -799,7 +839,7 @@ Status DBImpl::BuildIndex(const meta::TableFileSchema& file) {
//step 4: if table has been deleted, dont save index file
bool has_table = false;
meta_ptr_->HasTable(file.table_id_, has_table);
if(!has_table) {
if (!has_table) {
meta_ptr_->DeleteTableFiles(file.table_id_);
return Status::OK();
}
@ -807,7 +847,7 @@ Status DBImpl::BuildIndex(const meta::TableFileSchema& file) {
//step 5: save index file
try {
index->Serialize();
} catch (std::exception& ex) {
} catch (std::exception &ex) {
//typical error: out of disk space or permition denied
std::string msg = "Serialize index encounter exception: " + std::string(ex.what());
ENGINE_LOG_ERROR << msg;
@ -817,7 +857,7 @@ Status DBImpl::BuildIndex(const meta::TableFileSchema& file) {
ENGINE_LOG_DEBUG << "Failed to update file to index, mark file: " << table_file.file_id_ << " to to_delete";
std::cout << "ERROR: failed to persist index file: " << table_file.location_
<< ", possible out of disk space" << std::endl;
<< ", possible out of disk space" << std::endl;
return Status(DB_ERROR, msg);
}
@ -832,12 +872,12 @@ Status DBImpl::BuildIndex(const meta::TableFileSchema& file) {
meta::TableFilesSchema update_files = {table_file, origin_file};
status = meta_ptr_->UpdateTableFiles(update_files);
if(status.ok()) {
if (status.ok()) {
ENGINE_LOG_DEBUG << "New index file " << table_file.file_id_ << " of size "
<< index->PhysicalSize() << " bytes"
<< " from file " << origin_file.file_id_;
if(options_.insert_cache_immediately_) {
if (options_.insert_cache_immediately_) {
index->Cache();
}
} else {
@ -850,8 +890,7 @@ Status DBImpl::BuildIndex(const meta::TableFileSchema& file) {
status = meta_ptr_->UpdateTableFile(table_file);
ENGINE_LOG_DEBUG << "Failed to update file to index, mark file: " << table_file.file_id_ << " to to_delete";
}
} catch (std::exception& ex) {
} catch (std::exception &ex) {
std::string msg = "Build index encounter exception: " + std::string(ex.what());
ENGINE_LOG_ERROR << msg;
return Status(DB_ERROR, msg);
@ -860,20 +899,21 @@ Status DBImpl::BuildIndex(const meta::TableFileSchema& file) {
return Status::OK();
}
void DBImpl::BackgroundBuildIndex() {
void
DBImpl::BackgroundBuildIndex() {
ENGINE_LOG_TRACE << "Background build index thread start";
std::unique_lock<std::mutex> lock(build_index_mutex_);
meta::TableFilesSchema to_index_files;
meta_ptr_->FilesToIndex(to_index_files);
Status status;
for (auto& file : to_index_files) {
for (auto &file : to_index_files) {
status = BuildIndex(file);
if (!status.ok()) {
ENGINE_LOG_ERROR << "Building index for " << file.id_ << " failed: " << status.ToString();
}
if (shutting_down_.load(std::memory_order_acquire)){
if (shutting_down_.load(std::memory_order_acquire)) {
ENGINE_LOG_DEBUG << "Server will shutdown, skip build index action";
break;
}

12
cpp/src/db/DBImpl.h
View File

@ -29,7 +29,8 @@
#include <thread>
#include <list>
#include <set>
#include <vector>
#include <string>
namespace zilliz {
namespace milvus {
@ -68,11 +69,11 @@ class DBImpl : public DB {
Status InsertVectors(const std::string &table_id, uint64_t n, const float *vectors, IDNumbers &vector_ids) override;
Status CreateIndex(const std::string& table_id, const TableIndex& index) override;
Status CreateIndex(const std::string &table_id, const TableIndex &index) override;
Status DescribeIndex(const std::string& table_id, TableIndex& index) override;
Status DescribeIndex(const std::string &table_id, TableIndex &index) override;
Status DropIndex(const std::string& table_id) override;
Status DropIndex(const std::string &table_id) override;
Status Query(const std::string &table_id,
uint64_t k,
@ -123,7 +124,7 @@ class DBImpl : public DB {
Status BackgroundMergeFiles(const std::string &table_id);
void BackgroundCompaction(std::set<std::string> table_ids);
void StartBuildIndexTask(bool force=false);
void StartBuildIndexTask(bool force = false);
void BackgroundBuildIndex();
Status BuildIndex(const meta::TableFileSchema &);
@ -151,7 +152,6 @@ class DBImpl : public DB {
std::list<std::future<void>> index_thread_results_;
std::mutex build_index_mutex_;
}; // DBImpl

22
cpp/src/db/IDGenerator.cpp
View File

@ -15,7 +15,7 @@
// specific language governing permissions and limitations
// under the License.
#include "IDGenerator.h"
#include "db/IDGenerator.h"
#include <chrono>
#include <assert.h>
@ -29,16 +29,18 @@ IDGenerator::~IDGenerator() = default;
constexpr size_t SimpleIDGenerator::MAX_IDS_PER_MICRO;
IDNumber SimpleIDGenerator::GetNextIDNumber() {
IDNumber
SimpleIDGenerator::GetNextIDNumber() {
auto now = std::chrono::system_clock::now();
auto micros = std::chrono::duration_cast<std::chrono::microseconds>(
now.time_since_epoch()).count();
now.time_since_epoch()).count();
return micros * MAX_IDS_PER_MICRO;
}
void SimpleIDGenerator::NextIDNumbers(size_t n, IDNumbers& ids) {
void
SimpleIDGenerator::NextIDNumbers(size_t n, IDNumbers &ids) {
if (n > MAX_IDS_PER_MICRO) {
NextIDNumbers(n-MAX_IDS_PER_MICRO, ids);
NextIDNumbers(n - MAX_IDS_PER_MICRO, ids);
NextIDNumbers(MAX_IDS_PER_MICRO, ids);
return;
}
@ -48,20 +50,20 @@ void SimpleIDGenerator::NextIDNumbers(size_t n, IDNumbers& ids) {
auto now = std::chrono::system_clock::now();
auto micros = std::chrono::duration_cast<std::chrono::microseconds>(
now.time_since_epoch()).count();
now.time_since_epoch()).count();
micros *= MAX_IDS_PER_MICRO;
for (int pos=0; pos<n; ++pos) {
ids.push_back(micros+pos);
for (int pos = 0; pos < n; ++pos) {
ids.push_back(micros + pos);
}
}
void SimpleIDGenerator::GetNextIDNumbers(size_t n, IDNumbers& ids) {
void
SimpleIDGenerator::GetNextIDNumbers(size_t n, IDNumbers &ids) {
ids.clear();
NextIDNumbers(n, ids);
}
} // namespace engine
} // namespace milvus
} // namespace zilliz

2
cpp/src/db/IDGenerator.h
View File

@ -22,7 +22,6 @@
#include <cstddef>
#include <vector>
namespace zilliz {
namespace milvus {
namespace engine {
@ -55,7 +54,6 @@ class SimpleIDGenerator : public IDGenerator {
NextIDNumbers(size_t n, IDNumbers &ids);
static constexpr size_t MAX_IDS_PER_MICRO = 1000;
}; // SimpleIDGenerator

23
cpp/src/db/Options.cpp
View File

@ -15,7 +15,7 @@
// specific language governing permissions and limitations
// under the License.
#include "Options.h"
#include "db/Options.h"
#include "utils/Exception.h"
#include "utils/Log.h"
@ -27,18 +27,20 @@ namespace zilliz {
namespace milvus {
namespace engine {
ArchiveConf::ArchiveConf(const std::string& type, const std::string& criterias) {
ArchiveConf::ArchiveConf(const std::string &type, const std::string &criterias) {
ParseType(type);
ParseCritirias(criterias);
}
void ArchiveConf::SetCriterias(const ArchiveConf::CriteriaT& criterial) {
for(auto& pair : criterial) {
void
ArchiveConf::SetCriterias(const ArchiveConf::CriteriaT &criterial) {
for (auto &pair : criterial) {
criterias_[pair.first] = pair.second;
}
}
void ArchiveConf::ParseCritirias(const std::string& criterias) {
void
ArchiveConf::ParseCritirias(const std::string &criterias) {
std::stringstream ss(criterias);
std::vector<std::string> tokens;
@ -48,8 +50,8 @@ void ArchiveConf::ParseCritirias(const std::string& criterias) {
return;
}
for (auto& token : tokens) {
if(token.empty()) {
for (auto &token : tokens) {
if (token.empty()) {
continue;
}
@ -67,12 +69,12 @@ void ArchiveConf::ParseCritirias(const std::string& criterias) {
auto value = std::stoi(kv[1]);
criterias_[kv[0]] = value;
}
catch (std::out_of_range&){
catch (std::out_of_range &) {
std::string msg = "Out of range: '" + kv[1] + "'";
ENGINE_LOG_ERROR << msg;
throw InvalidArgumentException(msg);
}
catch (...){
catch (...) {
std::string msg = "Invalid argument: '" + kv[1] + "'";
ENGINE_LOG_ERROR << msg;
throw InvalidArgumentException(msg);
@ -80,7 +82,8 @@ void ArchiveConf::ParseCritirias(const std::string& criterias) {
}
}
void ArchiveConf::ParseType(const std::string& type) {
void
ArchiveConf::ParseType(const std::string &type) {
if (type != "delete" && type != "swap") {
std::string msg = "Invalid argument: type='" + type + "'";
throw InvalidArgumentException(msg);

25
cpp/src/db/Options.h
View File

@ -30,22 +30,27 @@ namespace engine {
class Env;
static const char* ARCHIVE_CONF_DISK = "disk";
static const char* ARCHIVE_CONF_DAYS = "days";
static const char *ARCHIVE_CONF_DISK = "disk";
static const char *ARCHIVE_CONF_DAYS = "days";
struct ArchiveConf {
using CriteriaT = std::map<std::string, int>;
ArchiveConf(const std::string& type, const std::string& criterias = std::string());
explicit ArchiveConf(const std::string &type, const std::string &criterias = std::string());
const std::string& GetType() const { return type_; }
const CriteriaT GetCriterias() const { return criterias_; }
const std::string &GetType() const {
return type_;
}
void SetCriterias(const ArchiveConf::CriteriaT& criterial);
const CriteriaT GetCriterias() const {
return criterias_;
}
private:
void ParseCritirias(const std::string& type);
void ParseType(const std::string& criterias);
void SetCriterias(const ArchiveConf::CriteriaT &criterial);
private:
void ParseCritirias(const std::string &type);
void ParseType(const std::string &criterias);
std::string type_;
CriteriaT criterias_;
@ -65,7 +70,7 @@ struct DBOptions {
CLUSTER_WRITABLE
} MODE;
uint16_t merge_trigger_number_ = 2;
uint16_t merge_trigger_number_ = 2;
DBMetaOptions meta_;
int mode_ = MODE::SINGLE;

9
cpp/src/db/Types.h
View File

@ -21,22 +21,23 @@
#include <vector>
#include <stdint.h>
#include <utility>
namespace zilliz {
namespace milvus {
namespace engine {
typedef long IDNumber;
typedef IDNumber* IDNumberPtr;
typedef int64_t IDNumber;
typedef IDNumber *IDNumberPtr;
typedef std::vector<IDNumber> IDNumbers;
typedef std::vector<std::pair<IDNumber, double>> QueryResult;
typedef std::vector<QueryResult> QueryResults;
struct TableIndex {
int32_t engine_type_ = (int)EngineType::FAISS_IDMAP;
int32_t engine_type_ = (int) EngineType::FAISS_IDMAP;
int32_t nlist_ = 16384;
int32_t metric_type_ = (int)MetricType::L2;
int32_t metric_type_ = (int) MetricType::L2;
};
} // namespace engine

74
cpp/src/db/Utils.cpp
View File

@ -15,13 +15,14 @@
// specific language governing permissions and limitations
// under the License.
#include "Utils.h"
#include "db/Utils.h"
#include "utils/CommonUtil.h"
#include "utils/Log.h"
#include <mutex>
#include <chrono>
#include <regex>
#include <vector>
#include <boost/filesystem.hpp>
namespace zilliz {
@ -31,23 +32,25 @@ namespace utils {
namespace {
const char* TABLES_FOLDER = "/tables/";
const char *TABLES_FOLDER = "/tables/";
uint64_t index_file_counter = 0;
std::mutex index_file_counter_mutex;
std::string ConstructParentFolder(const std::string& db_path, const meta::TableFileSchema& table_file) {
std::string
ConstructParentFolder(const std::string &db_path, const meta::TableFileSchema &table_file) {
std::string table_path = db_path + TABLES_FOLDER + table_file.table_id_;
std::string partition_path = table_path + "/" + std::to_string(table_file.date_);
return partition_path;
}
std::string GetTableFileParentFolder(const DBMetaOptions& options, const meta::TableFileSchema& table_file) {
std::string
GetTableFileParentFolder(const DBMetaOptions &options, const meta::TableFileSchema &table_file) {
uint64_t path_count = options.slave_paths_.size() + 1;
std::string target_path = options.path_;
uint64_t index = 0;
if(meta::TableFileSchema::NEW_INDEX == table_file.file_type_) {
if (meta::TableFileSchema::NEW_INDEX == table_file.file_type_) {
// index file is large file and to be persisted permanently
// we need to distribute index files to each db_path averagely
// round robin according to a file counter
@ -67,17 +70,19 @@ std::string GetTableFileParentFolder(const DBMetaOptions& options, const meta::T
return ConstructParentFolder(target_path, table_file);
}
}
} // namespace
long GetMicroSecTimeStamp() {
int64_t
GetMicroSecTimeStamp() {
auto now = std::chrono::system_clock::now();
auto micros = std::chrono::duration_cast<std::chrono::microseconds>(
now.time_since_epoch()).count();
now.time_since_epoch()).count();
return micros;
}
Status CreateTablePath(const DBMetaOptions& options, const std::string& table_id) {
Status
CreateTablePath(const DBMetaOptions &options, const std::string &table_id) {
std::string db_path = options.path_;
std::string table_path = db_path + TABLES_FOLDER + table_id;
auto status = server::CommonUtil::CreateDirectory(table_path);
@ -86,7 +91,7 @@ Status CreateTablePath(const DBMetaOptions& options, const std::string& table_id
return status;
}
for(auto& path : options.slave_paths_) {
for (auto &path : options.slave_paths_) {
table_path = path + TABLES_FOLDER + table_id;
status = server::CommonUtil::CreateDirectory(table_path);
if (!status.ok()) {
@ -98,17 +103,18 @@ Status CreateTablePath(const DBMetaOptions& options, const std::string& table_id
return Status::OK();
}
Status DeleteTablePath(const DBMetaOptions& options, const std::string& table_id, bool force) {
Status
DeleteTablePath(const DBMetaOptions &options, const std::string &table_id, bool force) {
std::vector<std::string> paths = options.slave_paths_;
paths.push_back(options.path_);
for(auto& path : paths) {
for (auto &path : paths) {
std::string table_path = path + TABLES_FOLDER + table_id;
if(force) {
if (force) {
boost::filesystem::remove_all(table_path);
ENGINE_LOG_DEBUG << "Remove table folder: " << table_path;
} else if(boost::filesystem::exists(table_path) &&
boost::filesystem::is_empty(table_path)) {
} else if (boost::filesystem::exists(table_path) &&
boost::filesystem::is_empty(table_path)) {
boost::filesystem::remove_all(table_path);
ENGINE_LOG_DEBUG << "Remove table folder: " << table_path;
}
@ -117,7 +123,8 @@ Status DeleteTablePath(const DBMetaOptions& options, const std::string& table_id
return Status::OK();
}
Status CreateTableFilePath(const DBMetaOptions& options, meta::TableFileSchema& table_file) {
Status
CreateTableFilePath(const DBMetaOptions &options, meta::TableFileSchema &table_file) {
std::string parent_path = GetTableFileParentFolder(options, table_file);
auto status = server::CommonUtil::CreateDirectory(parent_path);
@ -131,17 +138,18 @@ Status CreateTableFilePath(const DBMetaOptions& options, meta::TableFileSchema&
return Status::OK();
}
Status GetTableFilePath(const DBMetaOptions& options, meta::TableFileSchema& table_file) {
Status
GetTableFilePath(const DBMetaOptions &options, meta::TableFileSchema &table_file) {
std::string parent_path = ConstructParentFolder(options.path_, table_file);
std::string file_path = parent_path + "/" + table_file.file_id_;
if(boost::filesystem::exists(file_path)) {
if (boost::filesystem::exists(file_path)) {
table_file.location_ = file_path;
return Status::OK();
} else {
for(auto& path : options.slave_paths_) {
for (auto &path : options.slave_paths_) {
parent_path = ConstructParentFolder(path, table_file);
file_path = parent_path + "/" + table_file.file_id_;
if(boost::filesystem::exists(file_path)) {
if (boost::filesystem::exists(file_path)) {
table_file.location_ = file_path;
return Status::OK();
}
@ -155,49 +163,55 @@ Status GetTableFilePath(const DBMetaOptions& options, meta::TableFileSchema& tab
return Status(DB_ERROR, msg);
}
Status DeleteTableFilePath(const DBMetaOptions& options, meta::TableFileSchema& table_file) {
Status
DeleteTableFilePath(const DBMetaOptions &options, meta::TableFileSchema &table_file) {
utils::GetTableFilePath(options, table_file);
boost::filesystem::remove(table_file.location_);
return Status::OK();
}
bool IsSameIndex(const TableIndex& index1, const TableIndex& index2) {
bool
IsSameIndex(const TableIndex &index1, const TableIndex &index2) {
return index1.engine_type_ == index2.engine_type_
&& index1.nlist_ == index2.nlist_
&& index1.metric_type_ == index2.metric_type_;
}
meta::DateT GetDate(const std::time_t& t, int day_delta) {
meta::DateT
GetDate(const std::time_t &t, int day_delta) {
struct tm ltm;
localtime_r(&t, &ltm);
if (day_delta > 0) {
do {
++ltm.tm_mday;
--day_delta;
} while(day_delta > 0);
} while (day_delta > 0);
mktime(&ltm);
} else if (day_delta < 0) {
do {
--ltm.tm_mday;
++day_delta;
} while(day_delta < 0);
} while (day_delta < 0);
mktime(&ltm);
} else {
ltm.tm_mday;
}
return ltm.tm_year*10000 + ltm.tm_mon*100 + ltm.tm_mday;
return ltm.tm_year * 10000 + ltm.tm_mon * 100 + ltm.tm_mday;
}
meta::DateT GetDateWithDelta(int day_delta) {
meta::DateT
GetDateWithDelta(int day_delta) {
return GetDate(std::time(nullptr), day_delta);
}
meta::DateT GetDate() {
meta::DateT
GetDate() {
return GetDate(std::time(nullptr), 0);
}
// URI format: dialect://username:password@host:port/database
Status ParseMetaUri(const std::string& uri, MetaUriInfo& info) {
Status
ParseMetaUri(const std::string &uri, MetaUriInfo &info) {
std::string dialect_regex = "(.*)";
std::string username_tegex = "(.*)";
std::string password_regex = "(.*)";
@ -205,7 +219,7 @@ Status ParseMetaUri(const std::string& uri, MetaUriInfo& info) {
std::string port_regex = "(.*)";
std::string db_name_regex = "(.*)";
std::string uri_regex_str =
dialect_regex + "\\:\\/\\/" +
dialect_regex + "\\:\\/\\/" +
username_tegex + "\\:" +
password_regex + "\\@" +
host_regex + "\\:" +

33
cpp/src/db/Utils.h
View File

@ -29,20 +29,30 @@ namespace milvus {
namespace engine {
namespace utils {
long GetMicroSecTimeStamp();
int64_t
GetMicroSecTimeStamp();
Status CreateTablePath(const DBMetaOptions& options, const std::string& table_id);
Status DeleteTablePath(const DBMetaOptions& options, const std::string& table_id, bool force = true);
Status
CreateTablePath(const DBMetaOptions &options, const std::string &table_id);
Status
DeleteTablePath(const DBMetaOptions &options, const std::string &table_id, bool force = true);
Status CreateTableFilePath(const DBMetaOptions& options, meta::TableFileSchema& table_file);
Status GetTableFilePath(const DBMetaOptions& options, meta::TableFileSchema& table_file);
Status DeleteTableFilePath(const DBMetaOptions& options, meta::TableFileSchema& table_file);
Status
CreateTableFilePath(const DBMetaOptions &options, meta::TableFileSchema &table_file);
Status
GetTableFilePath(const DBMetaOptions &options, meta::TableFileSchema &table_file);
Status
DeleteTableFilePath(const DBMetaOptions &options, meta::TableFileSchema &table_file);
bool IsSameIndex(const TableIndex& index1, const TableIndex& index2);
bool
IsSameIndex(const TableIndex &index1, const TableIndex &index2);
meta::DateT GetDate(const std::time_t &t, int day_delta = 0);
meta::DateT GetDate();
meta::DateT GetDateWithDelta(int day_delta);
meta::DateT
GetDate(const std::time_t &t, int day_delta = 0);
meta::DateT
GetDate();
meta::DateT
GetDateWithDelta(int day_delta);
struct MetaUriInfo {
std::string dialect_;
@ -53,7 +63,8 @@ struct MetaUriInfo {
std::string db_name_;
};
Status ParseMetaUri(const std::string& uri, MetaUriInfo& info);
Status
ParseMetaUri(const std::string &uri, MetaUriInfo &info);
} // namespace utils
} // namespace engine

19
cpp/src/db/engine/EngineFactory.cpp
View File

@ -15,10 +15,12 @@
// specific language governing permissions and limitations
// under the License.
#include "EngineFactory.h"
#include "ExecutionEngineImpl.h"
#include "db/engine/EngineFactory.h"
#include "db/engine/ExecutionEngineImpl.h"
#include "utils/Log.h"
#include <memory>
namespace zilliz {
namespace milvus {
namespace engine {
@ -29,20 +31,19 @@ EngineFactory::Build(uint16_t dimension,
EngineType index_type,
MetricType metric_type,
int32_t nlist) {
if(index_type == EngineType::INVALID) {
if (index_type == EngineType::INVALID) {
ENGINE_LOG_ERROR << "Unsupported engine type";
return nullptr;
}
ENGINE_LOG_DEBUG << "EngineFactory index type: " << (int)index_type;
ENGINE_LOG_DEBUG << "EngineFactory index type: " << (int) index_type;
ExecutionEnginePtr execution_engine_ptr =
std::make_shared<ExecutionEngineImpl>(dimension, location, index_type, metric_type, nlist);
std::make_shared<ExecutionEngineImpl>(dimension, location, index_type, metric_type, nlist);
execution_engine_ptr->Init();
return execution_engine_ptr;
}
}
}
}
} // namespace engine
} // namespace milvus
} // namespace zilliz

13
cpp/src/db/engine/EngineFactory.h
View File

@ -21,19 +21,22 @@
#include "ExecutionEngine.h"
#include "utils/Status.h"
#include <string>
namespace zilliz {
namespace milvus {
namespace engine {
class EngineFactory {
public:
public:
static ExecutionEnginePtr Build(uint16_t dimension,
const std::string& location,
const std::string &location,
EngineType index_type,
MetricType metric_type,
int32_t nlist);
};
}
}
}
} // namespace engine
} // namespace milvus
} // namespace zilliz

23
cpp/src/db/engine/ExecutionEngine.h
View File

@ -21,6 +21,7 @@
#include <vector>
#include <memory>
#include <string>
namespace zilliz {
namespace milvus {
@ -41,9 +42,8 @@ enum class MetricType {
};
class ExecutionEngine {
public:
virtual Status AddWithIds(long n, const float *xdata, const long *xids) = 0;
public:
virtual Status AddWithIds(int64_t n, const float *xdata, const int64_t *xids) = 0;
virtual size_t Count() const = 0;
@ -63,16 +63,16 @@ public:
virtual std::shared_ptr<ExecutionEngine> Clone() = 0;
virtual Status Merge(const std::string& location) = 0;
virtual Status Merge(const std::string &location) = 0;
virtual Status Search(long n,
const float *data,
long k,
long nprobe,
float *distances,
long *labels) const = 0;
virtual Status Search(int64_t n,
const float *data,
int64_t k,
int64_t nprobe,
float *distances,
int64_t *labels) const = 0;
virtual std::shared_ptr<ExecutionEngine> BuildIndex(const std::string& location, EngineType engine_type) = 0;
virtual std::shared_ptr<ExecutionEngine> BuildIndex(const std::string &location, EngineType engine_type) = 0;
virtual Status Cache() = 0;
@ -89,7 +89,6 @@ public:
using ExecutionEnginePtr = std::shared_ptr<ExecutionEngine>;
} // namespace engine
} // namespace milvus
} // namespace zilliz

87
cpp/src/db/engine/ExecutionEngineImpl.cpp
View File

@ -15,7 +15,7 @@
// specific language governing permissions and limitations
// under the License.
#include "ExecutionEngineImpl.h"
#include "db/engine/ExecutionEngineImpl.h"
#include "cache/GpuCacheMgr.h"
#include "cache/CpuCacheMgr.h"
#include "metrics/Metrics.h"
@ -29,6 +29,7 @@
#include "server/Config.h"
#include <stdexcept>
#include <utility>
namespace zilliz {
namespace milvus {
@ -72,7 +73,8 @@ ExecutionEngineImpl::ExecutionEngineImpl(VecIndexPtr index,
nlist_(nlist) {
}
VecIndexPtr ExecutionEngineImpl::CreatetVecIndex(EngineType type) {
VecIndexPtr
ExecutionEngineImpl::CreatetVecIndex(EngineType type) {
std::shared_ptr<VecIndex> index;
switch (type) {
case EngineType::FAISS_IDMAP: {
@ -99,41 +101,48 @@ VecIndexPtr ExecutionEngineImpl::CreatetVecIndex(EngineType type) {
return index;
}
Status ExecutionEngineImpl::AddWithIds(long n, const float *xdata, const long *xids) {
Status
ExecutionEngineImpl::AddWithIds(int64_t n, const float *xdata, const int64_t *xids) {
auto status = index_->Add(n, xdata, xids);
return status;
}
size_t ExecutionEngineImpl::Count() const {
if(index_ == nullptr) {
size_t
ExecutionEngineImpl::Count() const {
if (index_ == nullptr) {
ENGINE_LOG_ERROR << "ExecutionEngineImpl: index is null, return count 0";
return 0;
}
return index_->Count();
}
size_t ExecutionEngineImpl::Size() const {
size_t
ExecutionEngineImpl::Size() const {
return (size_t) (Count() * Dimension()) * sizeof(float);
}
size_t ExecutionEngineImpl::Dimension() const {
if(index_ == nullptr) {
size_t
ExecutionEngineImpl::Dimension() const {
if (index_ == nullptr) {
ENGINE_LOG_ERROR << "ExecutionEngineImpl: index is null, return dimension " << dim_;
return dim_;
}
return index_->Dimension();
}
size_t ExecutionEngineImpl::PhysicalSize() const {
size_t
ExecutionEngineImpl::PhysicalSize() const {
return server::CommonUtil::GetFileSize(location_);
}
Status ExecutionEngineImpl::Serialize() {
Status
ExecutionEngineImpl::Serialize() {
auto status = write_index(index_, location_);
return status;
}
Status ExecutionEngineImpl::Load(bool to_cache) {
Status
ExecutionEngineImpl::Load(bool to_cache) {
index_ = cache::CpuCacheMgr::GetInstance()->GetIndex(location_);
bool already_in_cache = (index_ != nullptr);
if (!already_in_cache) {
@ -141,7 +150,7 @@ Status ExecutionEngineImpl::Load(bool to_cache) {
double physical_size = PhysicalSize();
server::CollectExecutionEngineMetrics metrics(physical_size);
index_ = read_index(location_);
if(index_ == nullptr) {
if (index_ == nullptr) {
std::string msg = "Failed to load index from " + location_;
ENGINE_LOG_ERROR << msg;
return Status(DB_ERROR, msg);
@ -160,13 +169,14 @@ Status ExecutionEngineImpl::Load(bool to_cache) {
return Status::OK();
}
Status ExecutionEngineImpl::CopyToGpu(uint64_t device_id) {
Status
ExecutionEngineImpl::CopyToGpu(uint64_t device_id) {
auto index = cache::GpuCacheMgr::GetInstance(device_id)->GetIndex(location_);
bool already_in_cache = (index != nullptr);
if (already_in_cache) {
index_ = index;
} else {
if(index_ == nullptr) {
if (index_ == nullptr) {
ENGINE_LOG_ERROR << "ExecutionEngineImpl: index is null, failed to copy to gpu";
return Status(DB_ERROR, "index is null");
}
@ -187,13 +197,14 @@ Status ExecutionEngineImpl::CopyToGpu(uint64_t device_id) {
return Status::OK();
}
Status ExecutionEngineImpl::CopyToCpu() {
Status
ExecutionEngineImpl::CopyToCpu() {
auto index = cache::CpuCacheMgr::GetInstance()->GetIndex(location_);
bool already_in_cache = (index != nullptr);
if (already_in_cache) {
index_ = index;
} else {
if(index_ == nullptr) {
if (index_ == nullptr) {
ENGINE_LOG_ERROR << "ExecutionEngineImpl: index is null, failed to copy to cpu";
return Status(DB_ERROR, "index is null");
}
@ -213,8 +224,9 @@ Status ExecutionEngineImpl::CopyToCpu() {
return Status::OK();
}
ExecutionEnginePtr ExecutionEngineImpl::Clone() {
if(index_ == nullptr) {
ExecutionEnginePtr
ExecutionEngineImpl::Clone() {
if (index_ == nullptr) {
ENGINE_LOG_ERROR << "ExecutionEngineImpl: index is null, failed to clone";
return nullptr;
}
@ -225,7 +237,8 @@ ExecutionEnginePtr ExecutionEngineImpl::Clone() {
return ret;
}
Status ExecutionEngineImpl::Merge(const std::string &location) {
Status
ExecutionEngineImpl::Merge(const std::string &location) {
if (location == location_) {
return Status(DB_ERROR, "Cannot Merge Self");
}
@ -243,7 +256,7 @@ Status ExecutionEngineImpl::Merge(const std::string &location) {
}
}
if(index_ == nullptr) {
if (index_ == nullptr) {
ENGINE_LOG_ERROR << "ExecutionEngineImpl: index is null, failed to merge";
return Status(DB_ERROR, "index is null");
}
@ -264,7 +277,7 @@ ExecutionEngineImpl::BuildIndex(const std::string &location, EngineType engine_t
ENGINE_LOG_DEBUG << "Build index file: " << location << " from: " << location_;
auto from_index = std::dynamic_pointer_cast<BFIndex>(index_);
if(from_index == nullptr) {
if (from_index == nullptr) {
ENGINE_LOG_ERROR << "ExecutionEngineImpl: from_index is null, failed to build index";
return nullptr;
}
@ -282,21 +295,22 @@ ExecutionEngineImpl::BuildIndex(const std::string &location, EngineType engine_t
AutoGenParams(to_index->GetType(), Count(), build_cfg);
auto status = to_index->BuildAll(Count(),
from_index->GetRawVectors(),
from_index->GetRawIds(),
build_cfg);
from_index->GetRawVectors(),
from_index->GetRawIds(),
build_cfg);
if (!status.ok()) { throw Exception(DB_ERROR, status.message()); }
return std::make_shared<ExecutionEngineImpl>(to_index, location, engine_type, metric_type_, nlist_);
}
Status ExecutionEngineImpl::Search(long n,
const float *data,
long k,
long nprobe,
float *distances,
long *labels) const {
if(index_ == nullptr) {
Status
ExecutionEngineImpl::Search(int64_t n,
const float *data,
int64_t k,
int64_t nprobe,
float *distances,
int64_t *labels) const {
if (index_ == nullptr) {
ENGINE_LOG_ERROR << "ExecutionEngineImpl: index is null, failed to search";
return Status(DB_ERROR, "index is null");
}
@ -310,14 +324,16 @@ Status ExecutionEngineImpl::Search(long n,
return status;
}
Status ExecutionEngineImpl::Cache() {
Status
ExecutionEngineImpl::Cache() {
cache::DataObjPtr obj = std::make_shared<cache::DataObj>(index_, PhysicalSize());
zilliz::milvus::cache::CpuCacheMgr::GetInstance()->InsertItem(location_, obj);
return Status::OK();
}
Status ExecutionEngineImpl::GpuCache(uint64_t gpu_id) {
Status
ExecutionEngineImpl::GpuCache(uint64_t gpu_id) {
cache::DataObjPtr obj = std::make_shared<cache::DataObj>(index_, PhysicalSize());
zilliz::milvus::cache::GpuCacheMgr::GetInstance(gpu_id)->InsertItem(location_, obj);
@ -325,8 +341,8 @@ Status ExecutionEngineImpl::GpuCache(uint64_t gpu_id) {
}
// TODO(linxj): remove.
Status ExecutionEngineImpl::Init() {
using namespace zilliz::milvus::server;
Status
ExecutionEngineImpl::Init() {
server::Config &config = server::Config::GetInstance();
Status s = config.GetDBConfigBuildIndexGPU(gpu_num_);
if (!s.ok()) return s;
@ -334,7 +350,6 @@ Status ExecutionEngineImpl::Init() {
return Status::OK();
}
} // namespace engine
} // namespace milvus
} // namespace zilliz

32
cpp/src/db/engine/ExecutionEngineImpl.h
View File

@ -23,15 +23,12 @@
#include <memory>
#include <string>
namespace zilliz {
namespace milvus {
namespace engine {
class ExecutionEngineImpl : public ExecutionEngine {
public:
public:
ExecutionEngineImpl(uint16_t dimension,
const std::string &location,
EngineType index_type,
@ -44,7 +41,7 @@ public:
MetricType metric_type,
int32_t nlist);
Status AddWithIds(long n, const float *xdata, const long *xids) override;
Status AddWithIds(int64_t n, const float *xdata, const int64_t *xids) override;
size_t Count() const override;
@ -66,12 +63,12 @@ public:
Status Merge(const std::string &location) override;
Status Search(long n,
Status Search(int64_t n,
const float *data,
long k,
long nprobe,
int64_t k,
int64_t nprobe,
float *distances,
long *labels) const override;
int64_t *labels) const override;
ExecutionEnginePtr BuildIndex(const std::string &location, EngineType engine_type) override;
@ -81,18 +78,24 @@ public:
Status Init() override;
EngineType IndexEngineType() const override { return index_type_; }
EngineType IndexEngineType() const override {
return index_type_;
}
MetricType IndexMetricType() const override { return metric_type_; }
MetricType IndexMetricType() const override {
return metric_type_;
}
std::string GetLocation() const override { return location_; }
std::string GetLocation() const override {
return location_;
}
private:
private:
VecIndexPtr CreatetVecIndex(EngineType type);
VecIndexPtr Load(const std::string &location);
protected:
protected:
VecIndexPtr index_ = nullptr;
EngineType index_type_;
MetricType metric_type_;
@ -104,7 +107,6 @@ protected:
int32_t gpu_num_ = 0;
};
} // namespace engine
} // namespace milvus
} // namespace zilliz

5
cpp/src/db/insert/MemManager.h
View File

@ -23,6 +23,7 @@
#include <set>
#include <memory>
#include <string>
namespace zilliz {
namespace milvus {
@ -30,7 +31,6 @@ namespace engine {
class MemManager {
public:
virtual Status InsertVectors(const std::string &table_id,
size_t n, const float *vectors, IDNumbers &vector_ids) = 0;
@ -43,11 +43,10 @@ class MemManager {
virtual size_t GetCurrentImmutableMem() = 0;
virtual size_t GetCurrentMem() = 0;
}; // MemManagerAbstract
using MemManagerPtr = std::shared_ptr<MemManager>;
} // namespace engine
} // namespace milvus
} // namespace zilliz
} // namespace zilliz

48
cpp/src/db/insert/MemManagerImpl.cpp
View File

@ -16,19 +16,19 @@
// under the License.
#include "MemManagerImpl.h"
#include "db/insert/MemManagerImpl.h"
#include "VectorSource.h"
#include "utils/Log.h"
#include "db/Constants.h"
#include <thread>
namespace zilliz {
namespace milvus {
namespace engine {
MemTablePtr MemManagerImpl::GetMemByTable(const std::string &table_id) {
MemTablePtr
MemManagerImpl::GetMemByTable(const std::string &table_id) {
auto memIt = mem_id_map_.find(table_id);
if (memIt != mem_id_map_.end()) {
return memIt->second;
@ -38,11 +38,11 @@ MemTablePtr MemManagerImpl::GetMemByTable(const std::string &table_id) {
return mem_id_map_[table_id];
}
Status MemManagerImpl::InsertVectors(const std::string &table_id_,
size_t n_,
const float *vectors_,
IDNumbers &vector_ids_) {
Status
MemManagerImpl::InsertVectors(const std::string &table_id_,
size_t n_,
const float *vectors_,
IDNumbers &vector_ids_) {
while (GetCurrentMem() > options_.insert_buffer_size_) {
std::this_thread::sleep_for(std::chrono::milliseconds(1));
}
@ -52,11 +52,11 @@ Status MemManagerImpl::InsertVectors(const std::string &table_id_,
return InsertVectorsNoLock(table_id_, n_, vectors_, vector_ids_);
}
Status MemManagerImpl::InsertVectorsNoLock(const std::string &table_id,
size_t n,
const float *vectors,
IDNumbers &vector_ids) {
Status
MemManagerImpl::InsertVectorsNoLock(const std::string &table_id,
size_t n,
const float *vectors,
IDNumbers &vector_ids) {
MemTablePtr mem = GetMemByTable(table_id);
VectorSourcePtr source = std::make_shared<VectorSource>(n, vectors);
@ -69,10 +69,11 @@ Status MemManagerImpl::InsertVectorsNoLock(const std::string &table_id,
return status;
}
Status MemManagerImpl::ToImmutable() {
Status
MemManagerImpl::ToImmutable() {
std::unique_lock<std::mutex> lock(mutex_);
MemIdMap temp_map;
for (auto &kv: mem_id_map_) {
for (auto &kv : mem_id_map_) {
if (kv.second->Empty()) {
//empty table, no need to serialize
temp_map.insert(kv);
@ -85,7 +86,8 @@ Status MemManagerImpl::ToImmutable() {
return Status::OK();
}
Status MemManagerImpl::Serialize(std::set<std::string> &table_ids) {
Status
MemManagerImpl::Serialize(std::set<std::string> &table_ids) {
ToImmutable();
std::unique_lock<std::mutex> lock(serialization_mtx_);
table_ids.clear();
@ -97,7 +99,8 @@ Status MemManagerImpl::Serialize(std::set<std::string> &table_ids) {
return Status::OK();
}
Status MemManagerImpl::EraseMemVector(const std::string &table_id) {
Status
MemManagerImpl::EraseMemVector(const std::string &table_id) {
{//erase MemVector from rapid-insert cache
std::unique_lock<std::mutex> lock(mutex_);
mem_id_map_.erase(table_id);
@ -117,7 +120,8 @@ Status MemManagerImpl::EraseMemVector(const std::string &table_id) {
return Status::OK();
}
size_t MemManagerImpl::GetCurrentMutableMem() {
size_t
MemManagerImpl::GetCurrentMutableMem() {
size_t total_mem = 0;
for (auto &kv : mem_id_map_) {
auto memTable = kv.second;
@ -126,7 +130,8 @@ size_t MemManagerImpl::GetCurrentMutableMem() {
return total_mem;
}
size_t MemManagerImpl::GetCurrentImmutableMem() {
size_t
MemManagerImpl::GetCurrentImmutableMem() {
size_t total_mem = 0;
for (auto &mem_table : immu_mem_list_) {
total_mem += mem_table->GetCurrentMem();
@ -134,10 +139,11 @@ size_t MemManagerImpl::GetCurrentImmutableMem() {
return total_mem;
}
size_t MemManagerImpl::GetCurrentMem() {
size_t
MemManagerImpl::GetCurrentMem() {
return GetCurrentMutableMem() + GetCurrentImmutableMem();
}
} // namespace engine
} // namespace milvus
} // namespace zilliz
} // namespace zilliz

9
cpp/src/db/insert/MemManagerImpl.h
View File

@ -24,12 +24,13 @@
#include "utils/Status.h"
#include <map>
#include <set>
#include <vector>
#include <string>
#include <ctime>
#include <memory>
#include <mutex>
namespace zilliz {
namespace milvus {
namespace engine {
@ -39,7 +40,8 @@ class MemManagerImpl : public MemManager {
using Ptr = std::shared_ptr<MemManagerImpl>;
MemManagerImpl(const meta::MetaPtr &meta, const DBOptions &options)
: meta_(meta), options_(options) {}
: meta_(meta), options_(options) {
}
Status InsertVectors(const std::string &table_id,
size_t n, const float *vectors, IDNumbers &vector_ids) override;
@ -71,7 +73,6 @@ class MemManagerImpl : public MemManager {
std::mutex serialization_mtx_;
}; // NewMemManager
} // namespace engine
} // namespace milvus
} // namespace zilliz
} // namespace zilliz

6
cpp/src/db/insert/MemMenagerFactory.cpp
View File

@ -15,7 +15,7 @@
// specific language governing permissions and limitations
// under the License.
#include "MemMenagerFactory.h"
#include "db/insert/MemMenagerFactory.h"
#include "MemManagerImpl.h"
#include "utils/Log.h"
#include "utils/Exception.h"
@ -26,12 +26,14 @@
#include <cstdlib>
#include <string>
#include <regex>
#include <memory>
namespace zilliz {
namespace milvus {
namespace engine {
MemManagerPtr MemManagerFactory::Build(const std::shared_ptr<meta::Meta>& meta, const DBOptions& options) {
MemManagerPtr
MemManagerFactory::Build(const std::shared_ptr<meta::Meta> &meta, const DBOptions &options) {
return std::make_shared<MemManagerImpl>(meta, options);
}

10
cpp/src/db/insert/MemMenagerFactory.h
View File

@ -20,15 +20,17 @@
#include "MemManager.h"
#include "db/meta/Meta.h"
#include <memory>
namespace zilliz {
namespace milvus {
namespace engine {
class MemManagerFactory {
public:
public:
static MemManagerPtr Build(const std::shared_ptr<meta::Meta> &meta, const DBOptions &options);
};
}
}
}
} // namespace engine
} // namespace milvus
} // namespace zilliz

30
cpp/src/db/insert/MemTable.cpp
View File

@ -16,9 +16,11 @@
// under the License.
#include "MemTable.h"
#include "db/insert/MemTable.h"
#include "utils/Log.h"
#include <memory>
#include <string>
namespace zilliz {
namespace milvus {
@ -30,13 +32,11 @@ MemTable::MemTable(const std::string &table_id,
table_id_(table_id),
meta_(meta),
options_(options) {
}
Status MemTable::Add(VectorSourcePtr &source, IDNumbers &vector_ids) {
Status
MemTable::Add(VectorSourcePtr &source, IDNumbers &vector_ids) {
while (!source->AllAdded()) {
MemTableFilePtr current_mem_table_file;
if (!mem_table_file_list_.empty()) {
current_mem_table_file = mem_table_file_list_.back();
@ -62,15 +62,18 @@ Status MemTable::Add(VectorSourcePtr &source, IDNumbers &vector_ids) {
return Status::OK();
}
void MemTable::GetCurrentMemTableFile(MemTableFilePtr &mem_table_file) {
void
MemTable::GetCurrentMemTableFile(MemTableFilePtr &mem_table_file) {
mem_table_file = mem_table_file_list_.back();
}
size_t MemTable::GetTableFileCount() {
size_t
MemTable::GetTableFileCount() {
return mem_table_file_list_.size();
}
Status MemTable::Serialize() {
Status
MemTable::Serialize() {
for (auto mem_table_file = mem_table_file_list_.begin(); mem_table_file != mem_table_file_list_.end();) {
auto status = (*mem_table_file)->Serialize();
if (!status.ok()) {
@ -84,15 +87,18 @@ Status MemTable::Serialize() {
return Status::OK();
}
bool MemTable::Empty() {
bool
MemTable::Empty() {
return mem_table_file_list_.empty();
}
const std::string &MemTable::GetTableId() const {
const std::string &
MemTable::GetTableId() const {
return table_id_;
}
size_t MemTable::GetCurrentMem() {
size_t
MemTable::GetCurrentMem() {
std::lock_guard<std::mutex> lock(mutex_);
size_t total_mem = 0;
for (auto &mem_table_file : mem_table_file_list_) {
@ -103,4 +109,4 @@ size_t MemTable::GetCurrentMem() {
} // namespace engine
} // namespace milvus
} // namespace zilliz
} // namespace zilliz

7
cpp/src/db/insert/MemTable.h
View File

@ -23,7 +23,9 @@
#include "utils/Status.h"
#include <mutex>
#include <vector>
#include <memory>
#include <string>
namespace zilliz {
namespace milvus {
@ -59,11 +61,10 @@ class MemTable {
DBOptions options_;
std::mutex mutex_;
}; //MemTable
using MemTablePtr = std::shared_ptr<MemTable>;
} // namespace engine
} // namespace milvus
} // namespace zilliz
} // namespace zilliz

38
cpp/src/db/insert/MemTableFile.cpp
View File

@ -16,14 +16,14 @@
// under the License.
#include "MemTableFile.h"
#include "db/insert/MemTableFile.h"
#include "db/Constants.h"
#include "db/engine/EngineFactory.h"
#include "metrics/Metrics.h"
#include "utils/Log.h"
#include <cmath>
#include <string>
namespace zilliz {
namespace milvus {
@ -35,20 +35,19 @@ MemTableFile::MemTableFile(const std::string &table_id,
table_id_(table_id),
meta_(meta),
options_(options) {
current_mem_ = 0;
auto status = CreateTableFile();
if (status.ok()) {
execution_engine_ = EngineFactory::Build(table_file_schema_.dimension_,
table_file_schema_.location_,
(EngineType) table_file_schema_.engine_type_,
(MetricType)table_file_schema_.metric_type_,
(MetricType) table_file_schema_.metric_type_,
table_file_schema_.nlist_);
}
}
Status MemTableFile::CreateTableFile() {
Status
MemTableFile::CreateTableFile() {
meta::TableFileSchema table_file_schema;
table_file_schema.table_id_ = table_id_;
auto status = meta_->CreateTableFile(table_file_schema);
@ -61,8 +60,8 @@ Status MemTableFile::CreateTableFile() {
return status;
}
Status MemTableFile::Add(const VectorSourcePtr &source, IDNumbers& vector_ids) {
Status
MemTableFile::Add(const VectorSourcePtr &source, IDNumbers &vector_ids) {
if (table_file_schema_.dimension_ <= 0) {
std::string err_msg = "MemTableFile::Add: table_file_schema dimension = " +
std::to_string(table_file_schema_.dimension_) + ", table_id = " + table_file_schema_.table_id_;
@ -75,7 +74,8 @@ Status MemTableFile::Add(const VectorSourcePtr &source, IDNumbers& vector_ids) {
if (mem_left >= single_vector_mem_size) {
size_t num_vectors_to_add = std::ceil(mem_left / single_vector_mem_size);
size_t num_vectors_added;
auto status = source->Add(execution_engine_, table_file_schema_, num_vectors_to_add, num_vectors_added, vector_ids);
auto status =
source->Add(execution_engine_, table_file_schema_, num_vectors_to_add, num_vectors_added, vector_ids);
if (status.ok()) {
current_mem_ += (num_vectors_added * single_vector_mem_size);
}
@ -84,20 +84,24 @@ Status MemTableFile::Add(const VectorSourcePtr &source, IDNumbers& vector_ids) {
return Status::OK();
}
size_t MemTableFile::GetCurrentMem() {
size_t
MemTableFile::GetCurrentMem() {
return current_mem_;
}
size_t MemTableFile::GetMemLeft() {
size_t
MemTableFile::GetMemLeft() {
return (MAX_TABLE_FILE_MEM - current_mem_);
}
bool MemTableFile::IsFull() {
bool
MemTableFile::IsFull() {
size_t single_vector_mem_size = table_file_schema_.dimension_ * VECTOR_TYPE_SIZE;
return (GetMemLeft() < single_vector_mem_size);
}
Status MemTableFile::Serialize() {
Status
MemTableFile::Serialize() {
size_t size = GetCurrentMem();
server::CollectSerializeMetrics metrics(size);
@ -107,7 +111,7 @@ Status MemTableFile::Serialize() {
//if index type isn't IDMAP, set file type to TO_INDEX if file size execeed index_file_size
//else set file type to RAW, no need to build index
if (table_file_schema_.engine_type_ != (int)EngineType::FAISS_IDMAP) {
if (table_file_schema_.engine_type_ != (int) EngineType::FAISS_IDMAP) {
table_file_schema_.file_type_ = (size >= table_file_schema_.index_file_size_) ?
meta::TableFileSchema::TO_INDEX : meta::TableFileSchema::RAW;
} else {
@ -117,9 +121,9 @@ Status MemTableFile::Serialize() {
auto status = meta_->UpdateTableFile(table_file_schema_);
ENGINE_LOG_DEBUG << "New " << ((table_file_schema_.file_type_ == meta::TableFileSchema::RAW) ? "raw" : "to_index")
<< " file " << table_file_schema_.file_id_ << " of size " << size << " bytes";
<< " file " << table_file_schema_.file_id_ << " of size " << size << " bytes";
if(options_.insert_cache_immediately_) {
if (options_.insert_cache_immediately_) {
execution_engine_->Cache();
}
@ -128,4 +132,4 @@ Status MemTableFile::Serialize() {
} // namespace engine
} // namespace milvus
} // namespace zilliz
} // namespace zilliz

14
cpp/src/db/insert/MemTableFile.h
View File

@ -23,17 +23,18 @@
#include "db/engine/ExecutionEngine.h"
#include "utils/Status.h"
#include <string>
#include <memory>
namespace zilliz {
namespace milvus {
namespace engine {
class MemTableFile {
public:
MemTableFile(const std::string &table_id, const meta::MetaPtr &meta, const DBOptions &options);
Status Add(const VectorSourcePtr &source, IDNumbers& vector_ids);
Status Add(const VectorSourcePtr &source, IDNumbers &vector_ids);
size_t GetCurrentMem();
@ -44,25 +45,20 @@ class MemTableFile {
Status Serialize();
private:
Status CreateTableFile();
private:
const std::string table_id_;
meta::TableFileSchema table_file_schema_;
meta::MetaPtr meta_;
DBOptions options_;
size_t current_mem_;
ExecutionEnginePtr execution_engine_;
}; //MemTableFile
using MemTableFilePtr = std::shared_ptr<MemTableFile>;
} // namespace engine
} // namespace milvus
} // namespace zilliz
} // namespace zilliz

35
cpp/src/db/insert/VectorSource.cpp
View File

@ -16,32 +16,30 @@
// under the License.
#include "VectorSource.h"
#include "db/insert/VectorSource.h"
#include "db/engine/ExecutionEngine.h"
#include "db/engine/EngineFactory.h"
#include "utils/Log.h"
#include "metrics/Metrics.h"
namespace zilliz {
namespace milvus {
namespace engine {
VectorSource::VectorSource(const size_t &n,
const float *vectors) :
n_(n),
vectors_(vectors),
id_generator_(std::make_shared<SimpleIDGenerator>()) {
n_(n),
vectors_(vectors),
id_generator_(std::make_shared<SimpleIDGenerator>()) {
current_num_vectors_added = 0;
}
Status VectorSource::Add(const ExecutionEnginePtr &execution_engine,
const meta::TableFileSchema &table_file_schema,
const size_t &num_vectors_to_add,
size_t &num_vectors_added,
IDNumbers &vector_ids) {
Status
VectorSource::Add(const ExecutionEnginePtr &execution_engine,
const meta::TableFileSchema &table_file_schema,
const size_t &num_vectors_to_add,
size_t &num_vectors_added,
IDNumbers &vector_ids) {
server::CollectAddMetrics metrics(n_, table_file_schema.dimension_);
num_vectors_added = current_num_vectors_added + num_vectors_to_add <= n_ ?
@ -52,7 +50,7 @@ Status VectorSource::Add(const ExecutionEnginePtr &execution_engine,
} else {
vector_ids_to_add.resize(num_vectors_added);
for (int pos = current_num_vectors_added; pos < current_num_vectors_added + num_vectors_added; pos++) {
vector_ids_to_add[pos-current_num_vectors_added] = vector_ids[pos];
vector_ids_to_add[pos - current_num_vectors_added] = vector_ids[pos];
}
}
Status status = execution_engine->AddWithIds(num_vectors_added,
@ -70,18 +68,21 @@ Status VectorSource::Add(const ExecutionEnginePtr &execution_engine,
return status;
}
size_t VectorSource::GetNumVectorsAdded() {
size_t
VectorSource::GetNumVectorsAdded() {
return current_num_vectors_added;
}
bool VectorSource::AllAdded() {
bool
VectorSource::AllAdded() {
return (current_num_vectors_added == n_);
}
IDNumbers VectorSource::GetVectorIds() {
IDNumbers
VectorSource::GetVectorIds() {
return vector_ids_;
}
} // namespace engine
} // namespace milvus
} // namespace zilliz
} // namespace zilliz

5
cpp/src/db/insert/VectorSource.h
View File

@ -23,6 +23,7 @@
#include "db/engine/ExecutionEngine.h"
#include "utils/Status.h"
#include <memory>
namespace zilliz {
namespace milvus {
@ -45,7 +46,6 @@ class VectorSource {
IDNumbers GetVectorIds();
private:
const size_t n_;
const float *vectors_;
IDNumbers vector_ids_;
@ -53,11 +53,10 @@ class VectorSource {
size_t current_num_vectors_added;
std::shared_ptr<IDGenerator> id_generator_;
}; //VectorSource
using VectorSourcePtr = std::shared_ptr<VectorSource>;
} // namespace engine
} // namespace milvus
} // namespace zilliz
} // namespace zilliz

13
cpp/src/db/meta/Meta.h
View File

@ -25,14 +25,16 @@
#include <cstddef>
#include <memory>
#include <vector>
#include <string>
namespace zilliz {
namespace milvus {
namespace engine {
namespace meta {
static const char* META_TABLES = "Tables";
static const char* META_TABLEFILES = "TableFiles";
static const char *META_TABLES = "Tables";
static const char *META_TABLEFILES = "TableFiles";
class Meta {
public:
@ -46,7 +48,7 @@ class Meta {
virtual Status AllTables(std::vector<TableSchema> &table_schema_array) = 0;
virtual Status UpdateTableIndex(const std::string &table_id, const TableIndex& index) = 0;
virtual Status UpdateTableIndex(const std::string &table_id, const TableIndex &index) = 0;
virtual Status UpdateTableFlag(const std::string &table_id, int64_t flag) = 0;
@ -83,9 +85,9 @@ class Meta {
virtual Status FilesByType(const std::string &table_id,
const std::vector<int> &file_types,
std::vector<std::string>& file_ids) = 0;
std::vector<std::string> &file_ids) = 0;
virtual Status DescribeTableIndex(const std::string &table_id, TableIndex& index) = 0;
virtual Status DescribeTableIndex(const std::string &table_id, TableIndex &index) = 0;
virtual Status DropTableIndex(const std::string &table_id) = 0;
@ -96,7 +98,6 @@ class Meta {
virtual Status DropAll() = 0;
virtual Status Count(const std::string &table_id, uint64_t &result) = 0;
}; // MetaData
using MetaPtr = std::shared_ptr<Meta>;

20
cpp/src/db/meta/MetaConsts.h
View File

@ -23,20 +23,20 @@ namespace engine {
namespace meta {
const size_t K = 1024UL;
const size_t M = K*K;
const size_t G = K*M;
const size_t T = K*G;
const size_t M = K * K;
const size_t G = K * M;
const size_t T = K * G;
const size_t S_PS = 1UL;
const size_t MS_PS = 1000*S_PS;
const size_t US_PS = 1000*MS_PS;
const size_t NS_PS = 1000*US_PS;
const size_t MS_PS = 1000 * S_PS;
const size_t US_PS = 1000 * MS_PS;
const size_t NS_PS = 1000 * US_PS;
const size_t SECOND = 1UL;
const size_t M_SEC = 60*SECOND;
const size_t H_SEC = 60*M_SEC;
const size_t D_SEC = 24*H_SEC;
const size_t W_SEC = 7*D_SEC;
const size_t M_SEC = 60 * SECOND;
const size_t H_SEC = 60 * M_SEC;
const size_t D_SEC = 24 * H_SEC;
const size_t W_SEC = 7 * D_SEC;
} // namespace meta
} // namespace engine

67
cpp/src/db/meta/MetaFactory.cpp
View File

@ -15,7 +15,7 @@
// specific language governing permissions and limitations
// under the License.
#include "MetaFactory.h"
#include "db/meta/MetaFactory.h"
#include "SqliteMetaImpl.h"
#include "MySQLMetaImpl.h"
#include "utils/Log.h"
@ -28,47 +28,52 @@
#include <cstdlib>
#include <string>
#include <string.h>
#include <memory>
namespace zilliz {
namespace milvus {
namespace engine {
DBMetaOptions MetaFactory::BuildOption(const std::string &path) {
auto p = path;
if(p == "") {
srand(time(nullptr));
std::stringstream ss;
ss << "/tmp/" << rand();
p = ss.str();
}
DBMetaOptions meta;
meta.path_ = p;
return meta;
DBMetaOptions
MetaFactory::BuildOption(const std::string &path) {
auto p = path;
if (p == "") {
srand(time(nullptr));
std::stringstream ss;
uint32_t rd = 0;
rand_r(&rd);
ss << "/tmp/" << rd;
p = ss.str();
}
meta::MetaPtr MetaFactory::Build(const DBMetaOptions &metaOptions, const int &mode) {
std::string uri = metaOptions.backend_uri_;
DBMetaOptions meta;
meta.path_ = p;
return meta;
}
utils::MetaUriInfo uri_info;
auto status = utils::ParseMetaUri(uri, uri_info);
if(!status.ok()) {
ENGINE_LOG_ERROR << "Wrong URI format: URI = " << uri;
throw InvalidArgumentException("Wrong URI format ");
}
meta::MetaPtr
MetaFactory::Build(const DBMetaOptions &metaOptions, const int &mode) {
std::string uri = metaOptions.backend_uri_;
if (strcasecmp(uri_info.dialect_.c_str(), "mysql") == 0) {
ENGINE_LOG_INFO << "Using MySQL";
return std::make_shared<meta::MySQLMetaImpl>(metaOptions, mode);
} else if (strcasecmp(uri_info.dialect_.c_str(), "sqlite") == 0) {
ENGINE_LOG_INFO << "Using SQLite";
return std::make_shared<meta::SqliteMetaImpl>(metaOptions);
} else {
ENGINE_LOG_ERROR << "Invalid dialect in URI: dialect = " << uri_info.dialect_;
throw InvalidArgumentException("URI dialect is not mysql / sqlite");
}
utils::MetaUriInfo uri_info;
auto status = utils::ParseMetaUri(uri, uri_info);
if (!status.ok()) {
ENGINE_LOG_ERROR << "Wrong URI format: URI = " << uri;
throw InvalidArgumentException("Wrong URI format ");
}
if (strcasecmp(uri_info.dialect_.c_str(), "mysql") == 0) {
ENGINE_LOG_INFO << "Using MySQL";
return std::make_shared<meta::MySQLMetaImpl>(metaOptions, mode);
} else if (strcasecmp(uri_info.dialect_.c_str(), "sqlite") == 0) {
ENGINE_LOG_INFO << "Using SQLite";
return std::make_shared<meta::SqliteMetaImpl>(metaOptions);
} else {
ENGINE_LOG_ERROR << "Invalid dialect in URI: dialect = " << uri_info.dialect_;
throw InvalidArgumentException("URI dialect is not mysql / sqlite");
}
}
} // namespace engine
} // namespace milvus
} // namespace zilliz

11
cpp/src/db/meta/MetaFactory.h
View File

@ -20,18 +20,19 @@
#include "Meta.h"
#include "db/Options.h"
#include <string>
namespace zilliz {
namespace milvus {
namespace engine {
class MetaFactory {
public:
public:
static DBMetaOptions BuildOption(const std::string &path = "");
static meta::MetaPtr Build(const DBMetaOptions &metaOptions, const int &mode);
};
}
}
}
} // namespace engine
} // namespace milvus
} // namespace zilliz

11
cpp/src/db/meta/MetaTypes.h
View File

@ -23,21 +23,22 @@
#include <vector>
#include <map>
#include <string>
#include <memory>
namespace zilliz {
namespace milvus {
namespace engine {
namespace meta {
constexpr int32_t DEFAULT_ENGINE_TYPE = (int)EngineType::FAISS_IDMAP;
constexpr int32_t DEFAULT_ENGINE_TYPE = (int) EngineType::FAISS_IDMAP;
constexpr int32_t DEFAULT_NLIST = 16384;
constexpr int32_t DEFAULT_METRIC_TYPE = (int)MetricType::L2;
constexpr int32_t DEFAULT_METRIC_TYPE = (int) MetricType::L2;
constexpr int32_t DEFAULT_INDEX_FILE_SIZE = ONE_GB;
constexpr int64_t FLAG_MASK_NO_USERID = 0x1;
constexpr int64_t FLAG_MASK_HAS_USERID = 0x1<<1;
constexpr int64_t FLAG_MASK_HAS_USERID = 0x1 << 1;
using DateT = int ;
using DateT = int;
const DateT EmptyDate = -1;
using DatesT = std::vector<DateT>;
@ -49,7 +50,7 @@ struct TableSchema {
size_t id_ = 0;
std::string table_id_;
int32_t state_ = (int)NORMAL;
int32_t state_ = (int) NORMAL;
uint16_t dimension_ = 0;
int64_t created_on_ = 0;
int64_t flag_ = 0;

105
cpp/src/db/meta/MySQLConnectionPool.cpp
View File

@ -16,37 +16,39 @@
// under the License.
#include "MySQLConnectionPool.h"
#include "db/meta/MySQLConnectionPool.h"
namespace zilliz {
namespace milvus {
namespace engine {
namespace meta {
// Do a simple form of in-use connection limiting: wait to return
// a connection until there are a reasonably low number in use
// already. Can't do this in create() because we're interested in
// connections actually in use, not those created. Also note that
// we keep our own count; ConnectionPool::size() isn't the same!
mysqlpp::Connection *MySQLConnectionPool::grab() {
while (conns_in_use_ > max_pool_size_) {
sleep(1);
}
++conns_in_use_;
return mysqlpp::ConnectionPool::grab();
// Do a simple form of in-use connection limiting: wait to return
// a connection until there are a reasonably low number in use
// already. Can't do this in create() because we're interested in
// connections actually in use, not those created. Also note that
// we keep our own count; ConnectionPool::size() isn't the same!
mysqlpp::Connection *
MySQLConnectionPool::grab() {
while (conns_in_use_ > max_pool_size_) {
sleep(1);
}
// Other half of in-use conn count limit
void MySQLConnectionPool::release(const mysqlpp::Connection *pc) {
mysqlpp::ConnectionPool::release(pc);
++conns_in_use_;
return mysqlpp::ConnectionPool::grab();
}
if (conns_in_use_ <= 0) {
ENGINE_LOG_WARNING << "MySQLConnetionPool::release: conns_in_use_ is less than zero. conns_in_use_ = " << conns_in_use_;
} else {
--conns_in_use_;
}
// Other half of in-use conn count limit
void
MySQLConnectionPool::release(const mysqlpp::Connection *pc) {
mysqlpp::ConnectionPool::release(pc);
if (conns_in_use_ <= 0) {
ENGINE_LOG_WARNING << "MySQLConnetionPool::release: conns_in_use_ is less than zero. conns_in_use_ = "
<< conns_in_use_;
} else {
--conns_in_use_;
}
}
// int MySQLConnectionPool::getConnectionsInUse() {
// return conns_in_use_;
@ -56,39 +58,42 @@ namespace meta {
// max_idle_time_ = max_idle;
// }
std::string MySQLConnectionPool::getDB() {
return db_;
}
std::string
MySQLConnectionPool::getDB() {
return db_;
}
// Superclass overrides
mysqlpp::Connection *MySQLConnectionPool::create() {
try {
// Create connection using the parameters we were passed upon
// creation.
mysqlpp::Connection *conn = new mysqlpp::Connection();
conn->set_option(new mysqlpp::ReconnectOption(true));
conn->connect(db_.empty() ? 0 : db_.c_str(),
server_.empty() ? 0 : server_.c_str(),
user_.empty() ? 0 : user_.c_str(),
password_.empty() ? 0 : password_.c_str(),
port_);
return conn;
} catch (const mysqlpp::ConnectionFailed& er) {
ENGINE_LOG_ERROR << "Failed to connect to database server" << ": " << er.what();
return nullptr;
}
// Superclass overrides
mysqlpp::Connection *
MySQLConnectionPool::create() {
try {
// Create connection using the parameters we were passed upon
// creation.
mysqlpp::Connection *conn = new mysqlpp::Connection();
conn->set_option(new mysqlpp::ReconnectOption(true));
conn->connect(db_.empty() ? 0 : db_.c_str(),
server_.empty() ? 0 : server_.c_str(),
user_.empty() ? 0 : user_.c_str(),
password_.empty() ? 0 : password_.c_str(),
port_);
return conn;
} catch (const mysqlpp::ConnectionFailed &er) {
ENGINE_LOG_ERROR << "Failed to connect to database server" << ": " << er.what();
return nullptr;
}
}
void MySQLConnectionPool::destroy(mysqlpp::Connection *cp) {
// Our superclass can't know how we created the Connection, so
// it delegates destruction to us, to be safe.
delete cp;
}
void
MySQLConnectionPool::destroy(mysqlpp::Connection *cp) {
// Our superclass can't know how we created the Connection, so
// it delegates destruction to us, to be safe.
delete cp;
}
unsigned int MySQLConnectionPool::max_idle_time() {
return max_idle_time_;
}
unsigned int
MySQLConnectionPool::max_idle_time() {
return max_idle_time_;
}
} // namespace meta
} // namespace engine

24
cpp/src/db/meta/MySQLConnectionPool.h
View File

@ -30,8 +30,7 @@ namespace engine {
namespace meta {
class MySQLConnectionPool : public mysqlpp::ConnectionPool {
public:
public:
// The object's only constructor
MySQLConnectionPool(std::string dbName,
std::string userName,
@ -39,15 +38,13 @@ public:
std::string serverIp,
int port = 0,
int maxPoolSize = 8) :
db_(dbName),
user_(userName),
password_(passWord),
server_(serverIp),
port_(port),
max_pool_size_(maxPoolSize) {
db_(dbName),
user_(userName),
password_(passWord),
server_(serverIp),
port_(port),
max_pool_size_(maxPoolSize) {
conns_in_use_ = 0;
max_idle_time_ = 10; //10 seconds
}
@ -68,8 +65,7 @@ public:
std::string getDB();
protected:
protected:
// Superclass overrides
mysqlpp::Connection *create() override;
@ -77,7 +73,7 @@ protected:
unsigned int max_idle_time() override;
private:
private:
// Number of connections currently in use
std::atomic<int> conns_in_use_;
@ -93,4 +89,4 @@ private:
} // namespace meta
} // namespace engine
} // namespace milvus
} // namespace zilliz
} // namespace zilliz

590
cpp/src/db/meta/MySQLMetaImpl.cpp
View File

File diff suppressed because it is too large Load Diff

21
cpp/src/db/meta/MySQLMetaImpl.h
View File

@ -21,26 +21,25 @@
#include "db/Options.h"
#include "MySQLConnectionPool.h"
#include "mysql++/mysql++.h"
#include <mysql++/mysql++.h>
#include <mutex>
#include <vector>
#include <string>
#include <memory>
namespace zilliz {
namespace milvus {
namespace engine {
namespace meta {
// auto StoragePrototype(const std::string& path);
using namespace mysqlpp;
class MySQLMetaImpl : public Meta {
public:
MySQLMetaImpl(const DBMetaOptions &options_, const int &mode);
MySQLMetaImpl(const DBMetaOptions &options, const int &mode);
~MySQLMetaImpl();
Status CreateTable(TableSchema &table_schema) override;
Status DescribeTable(TableSchema &group_info_) override;
Status DescribeTable(TableSchema &table_schema) override;
Status HasTable(const std::string &table_id, bool &has_or_not) override;
@ -63,11 +62,11 @@ class MySQLMetaImpl : public Meta {
const std::vector<int> &file_types,
std::vector<std::string> &file_ids) override;
Status UpdateTableIndex(const std::string &table_id, const TableIndex& index) override;
Status UpdateTableIndex(const std::string &table_id, const TableIndex &index) override;
Status UpdateTableFlag(const std::string &table_id, int64_t flag) override;
Status DescribeTableIndex(const std::string &table_id, TableIndex& index) override;
Status DescribeTableIndex(const std::string &table_id, TableIndex &index) override;
Status DropTableIndex(const std::string &table_id) override;
@ -102,12 +101,12 @@ class MySQLMetaImpl : public Meta {
private:
Status NextFileId(std::string &file_id);
Status NextTableId(std::string &table_id);
Status DiscardFiles(long long to_discard_size);
Status DiscardFiles(int64_t to_discard_size);
void ValidateMetaSchema();
Status Initialize();
private:
private:
const DBMetaOptions options_;
const int mode_;

438
cpp/src/db/meta/SqliteMetaImpl.cpp
View File

File diff suppressed because it is too large Load Diff

17
cpp/src/db/meta/SqliteMetaImpl.h
View File

@ -21,22 +21,25 @@
#include "db/Options.h"
#include <mutex>
#include <vector>
#include <string>
namespace zilliz {
namespace milvus {
namespace engine {
namespace meta {
auto StoragePrototype(const std::string &path);
auto
StoragePrototype(const std::string &path);
class SqliteMetaImpl : public Meta {
public:
explicit SqliteMetaImpl(const DBMetaOptions &options_);
explicit SqliteMetaImpl(const DBMetaOptions &options);
~SqliteMetaImpl();
Status CreateTable(TableSchema &table_schema) override;
Status DescribeTable(TableSchema &group_info_) override;
Status DescribeTable(TableSchema &table_schema) override;
Status HasTable(const std::string &table_id, bool &has_or_not) override;
@ -58,11 +61,11 @@ class SqliteMetaImpl : public Meta {
const std::vector<int> &file_types,
std::vector<std::string> &file_ids) override;
Status UpdateTableIndex(const std::string &table_id, const TableIndex& index) override;
Status UpdateTableIndex(const std::string &table_id, const TableIndex &index) override;
Status UpdateTableFlag(const std::string &table_id, int64_t flag) override;
Status DescribeTableIndex(const std::string &table_id, TableIndex& index) override;
Status DescribeTableIndex(const std::string &table_id, TableIndex &index) override;
Status DropTableIndex(const std::string &table_id) override;
@ -96,12 +99,12 @@ class SqliteMetaImpl : public Meta {
private:
Status NextFileId(std::string &file_id);
Status NextTableId(std::string &table_id);
Status DiscardFiles(long to_discard_size);
Status DiscardFiles(int64_t to_discard_size);
void ValidateMetaSchema();
Status Initialize();
private:
private:
const DBMetaOptions options_;
std::mutex meta_mutex_;
}; // DBMetaImpl

16
cpp/src/scheduler/Algorithm.cpp
View File

@ -16,20 +16,23 @@
// under the License.
#include "Algorithm.h"
#include "scheduler/Algorithm.h"
#include <limits>
#include <unordered_map>
#include <utility>
namespace zilliz {
namespace milvus {
namespace scheduler {
constexpr uint64_t MAXINT = std::numeric_limits<uint32_t >::max();
constexpr uint64_t MAXINT = std::numeric_limits<uint32_t>::max();
uint64_t
ShortestPath(const ResourcePtr &src,
const ResourcePtr &dest,
const ResourceMgrPtr &res_mgr,
std::vector<std::string> &path) {
std::vector<std::vector<std::string>> paths;
uint64_t num_of_resources = res_mgr->GetAllResources().size();
@ -53,7 +56,6 @@ ShortestPath(const ResourcePtr &src,
std::vector<bool> vis(num_of_resources, false);
std::vector<uint64_t> dis(num_of_resources, MAXINT);
for (auto &res : res_mgr->GetAllResources()) {
auto cur_node = std::static_pointer_cast<Node>(res);
auto cur_neighbours = cur_node->GetNeighbours();
@ -105,6 +107,6 @@ ShortestPath(const ResourcePtr &src,
return dis[name_id_map.at(dest->name())];
}
}
}
}
} // namespace scheduler
} // namespace milvus
} // namespace zilliz

8
cpp/src/scheduler/Algorithm.h
View File

@ -30,8 +30,8 @@ uint64_t
ShortestPath(const ResourcePtr &src,
const ResourcePtr &dest,
const ResourceMgrPtr &res_mgr,
std::vector<std::string>& path);
std::vector<std::string> &path);
}
}
}
} // namespace scheduler
} // namespace milvus
} // namespace zilliz

7
cpp/src/scheduler/Definition.h
View File

@ -30,7 +30,6 @@
#include "db/engine/EngineFactory.h"
#include "db/engine/ExecutionEngine.h"
namespace zilliz {
namespace milvus {
namespace scheduler {
@ -43,6 +42,6 @@ using EngineFactory = engine::EngineFactory;
using EngineType = engine::EngineType;
using MetricType = engine::MetricType;
}
}
}
} // namespace scheduler
} // namespace milvus
} // namespace zilliz

18
cpp/src/scheduler/JobMgr.cpp
View File

@ -15,19 +15,19 @@
// specific language governing permissions and limitations
// under the License.
#include "JobMgr.h"
#include "scheduler/JobMgr.h"
#include "task/Task.h"
#include "TaskCreator.h"
#include <utility>
namespace zilliz {
namespace milvus {
namespace scheduler {
using namespace engine;
JobMgr::JobMgr(ResourceMgrPtr res_mgr)
: res_mgr_(std::move(res_mgr)) {}
: res_mgr_(std::move(res_mgr)) {
}
void
JobMgr::Start() {
@ -59,7 +59,9 @@ void
JobMgr::worker_function() {
while (running_) {
std::unique_lock<std::mutex> lock(mutex_);
cv_.wait(lock, [this] { return !queue_.empty(); });
cv_.wait(lock, [this] {
return !queue_.empty();
});
auto job = queue_.front();
queue_.pop();
lock.unlock();
@ -84,6 +86,6 @@ JobMgr::build_task(const JobPtr &job) {
return TaskCreator::Create(job);
}
}
}
}
} // namespace scheduler
} // namespace milvus
} // namespace zilliz

18
cpp/src/scheduler/JobMgr.h
View File

@ -31,15 +31,13 @@
#include "task/Task.h"
#include "ResourceMgr.h"
namespace zilliz {
namespace milvus {
namespace scheduler {
class JobMgr {
public:
explicit
JobMgr(ResourceMgrPtr res_mgr);
public:
explicit JobMgr(ResourceMgrPtr res_mgr);
void
Start();
@ -47,18 +45,18 @@ public:
void
Stop();
public:
public:
void
Put(const JobPtr &job);
private:
private:
void
worker_function();
std::vector<TaskPtr>
build_task(const JobPtr &job);
private:
private:
bool running_ = false;
std::queue<JobPtr> queue_;
@ -72,6 +70,6 @@ private:
using JobMgrPtr = std::shared_ptr<JobMgr>;
}
}
}
} // namespace scheduler
} // namespace milvus
} // namespace zilliz

9
cpp/src/scheduler/ResourceFactory.cpp
View File

@ -16,8 +16,7 @@
// under the License.
#include "ResourceFactory.h"
#include "scheduler/ResourceFactory.h"
namespace zilliz {
namespace milvus {
@ -40,6 +39,6 @@ ResourceFactory::Create(const std::string &name,
}
}
}
}
}
} // namespace scheduler
} // namespace milvus
} // namespace zilliz

11
cpp/src/scheduler/ResourceFactory.h
View File

@ -25,13 +25,12 @@
#include "resource/GpuResource.h"
#include "resource/DiskResource.h"
namespace zilliz {
namespace milvus {
namespace scheduler {
class ResourceFactory {
public:
public:
static std::shared_ptr<Resource>
Create(const std::string &name,
const std::string &type,
@ -40,8 +39,6 @@ public:
bool enable_executor = true);
};
}
}
}
} // namespace scheduler
} // namespace milvus
} // namespace zilliz

14
cpp/src/scheduler/ResourceMgr.cpp
View File

@ -16,15 +16,13 @@
// specific language governing permissions and limitations
// under the License.
#include "ResourceMgr.h"
#include "scheduler/ResourceMgr.h"
#include "utils/Log.h"
namespace zilliz {
namespace milvus {
namespace scheduler {
void
ResourceMgr::Start() {
std::lock_guard<std::mutex> lck(resources_mutex_);
@ -186,7 +184,9 @@ void
ResourceMgr::event_process() {
while (running_) {
std::unique_lock<std::mutex> lock(event_mutex_);
event_cv_.wait(lock, [this] { return !queue_.empty(); });
event_cv_.wait(lock, [this] {
return !queue_.empty();
});
auto event = queue_.front();
queue_.pop();
@ -201,6 +201,6 @@ ResourceMgr::event_process() {
}
}
}
}
}
} // namespace scheduler
} // namespace milvus
} // namespace zilliz

24
cpp/src/scheduler/ResourceMgr.h
View File

@ -22,21 +22,21 @@
#include <memory>
#include <mutex>
#include <queue>
#include <utility>
#include <condition_variable>
#include "resource/Resource.h"
#include "utils/Log.h"
namespace zilliz {
namespace milvus {
namespace scheduler {
class ResourceMgr {
public:
public:
ResourceMgr() = default;
public:
public:
/******** Management Interface ********/
void
Start();
@ -58,7 +58,7 @@ public:
subscriber_ = std::move(subscriber);
}
public:
public:
/******** Management Interface ********/
inline std::vector<ResourceWPtr> &
GetDiskResources() {
@ -89,10 +89,10 @@ public:
uint64_t
GetNumGpuResource() const;
public:
public:
// TODO: add stats interface(low)
public:
public:
/******** Utility Functions ********/
std::string
Dump();
@ -100,14 +100,14 @@ public:
std::string
DumpTaskTables();
private:
private:
void
post_event(const EventPtr &event);
void
event_process();
private:
private:
bool running_ = false;
std::vector<ResourceWPtr> disk_resources_;
@ -120,13 +120,11 @@ private:
std::condition_variable event_cv_;
std::thread worker_thread_;
};
using ResourceMgrPtr = std::shared_ptr<ResourceMgr>;
using ResourceMgrWPtr = std::weak_ptr<ResourceMgr>;
}
}
}
} // namespace scheduler
} // namespace milvus
} // namespace zilliz

13
cpp/src/scheduler/SchedInst.cpp
View File

@ -16,12 +16,16 @@
// under the License.
#include "SchedInst.h"
#include "scheduler/SchedInst.h"
#include "server/Config.h"
#include "ResourceFactory.h"
#include "knowhere/index/vector_index/IndexGPUIVF.h"
#include "Utils.h"
#include <vector>
#include <set>
#include <utility>
#include <string>
namespace zilliz {
namespace milvus {
@ -165,6 +169,7 @@ StopSchedulerService() {
SchedInst::GetInstance()->Stop();
ResMgrInst::GetInstance()->Stop();
}
}
}
}
} // namespace scheduler
} // namespace milvus
} // namespace zilliz

19
cpp/src/scheduler/SchedInst.h
View File

@ -24,13 +24,12 @@
#include <mutex>
#include <memory>
namespace zilliz {
namespace milvus {
namespace scheduler {
class ResMgrInst {
public:
public:
static ResourceMgrPtr
GetInstance() {
if (instance == nullptr) {
@ -42,13 +41,13 @@ public:
return instance;
}
private:
private:
static ResourceMgrPtr instance;
static std::mutex mutex_;
};
class SchedInst {
public:
public:
static SchedulerPtr
GetInstance() {
if (instance == nullptr) {
@ -60,13 +59,13 @@ public:
return instance;
}
private:
private:
static SchedulerPtr instance;
static std::mutex mutex_;
};
class JobMgrInst {
public:
public:
static scheduler::JobMgrPtr
GetInstance() {
if (instance == nullptr) {
@ -78,7 +77,7 @@ public:
return instance;
}
private:
private:
static scheduler::JobMgrPtr instance;
static std::mutex mutex_;
};
@ -89,6 +88,6 @@ StartSchedulerService();
void
StopSchedulerService();
}
}
}
} // namespace scheduler
} // namespace milvus
} // namespace zilliz

17
cpp/src/scheduler/Scheduler.cpp
View File

@ -15,13 +15,13 @@
// specific language governing permissions and limitations
// under the License.
#include "src/cache/GpuCacheMgr.h"
#include "scheduler/Scheduler.h"
#include "cache/GpuCacheMgr.h"
#include "event/LoadCompletedEvent.h"
#include "Scheduler.h"
#include "action/Action.h"
#include "Algorithm.h"
#include <utility>
namespace zilliz {
namespace milvus {
@ -43,7 +43,6 @@ Scheduler::Scheduler(ResourceMgrWPtr res_mgr)
std::bind(&Scheduler::OnFinishTask, this, std::placeholders::_1)));
}
void
Scheduler::Start() {
running_ = true;
@ -79,7 +78,9 @@ void
Scheduler::worker_function() {
while (running_) {
std::unique_lock<std::mutex> lock(event_mutex_);
event_cv_.wait(lock, [this] { return !event_queue_.empty(); });
event_cv_.wait(lock, [this] {
return !event_queue_.empty();
});
auto event = event_queue_.front();
event_queue_.pop();
if (event == nullptr) {
@ -142,6 +143,6 @@ Scheduler::OnTaskTableUpdated(const EventPtr &event) {
}
}
}
}
}
} // namespace scheduler
} // namespace milvus
} // namespace zilliz

21
cpp/src/scheduler/Scheduler.h
View File

@ -22,22 +22,20 @@
#include <mutex>
#include <thread>
#include <queue>
#include <unordered_map>
#include "resource/Resource.h"
#include "ResourceMgr.h"
#include "utils/Log.h"
namespace zilliz {
namespace milvus {
namespace scheduler {
// TODO: refactor, not friendly to unittest, logical in framework code
class Scheduler {
public:
explicit
Scheduler(ResourceMgrWPtr res_mgr);
public:
explicit Scheduler(ResourceMgrWPtr res_mgr);
Scheduler(const Scheduler &) = delete;
Scheduler(Scheduler &&) = delete;
@ -66,7 +64,7 @@ public:
std::string
Dump();
private:
private:
/******** Events ********/
/*
@ -106,7 +104,7 @@ private:
void
OnTaskTableUpdated(const EventPtr &event);
private:
private:
/*
* Dispatch event to event handler;
*/
@ -119,7 +117,7 @@ private:
void
worker_function();
private:
private:
bool running_;
std::unordered_map<uint64_t, std::function<void(EventPtr)>> event_register_;
@ -133,7 +131,6 @@ private:
using SchedulerPtr = std::shared_ptr<Scheduler>;
}
}
}
} // namespace scheduler
} // namespace milvus
} // namespace zilliz

13
cpp/src/scheduler/TaskCreator.cpp
View File

@ -15,11 +15,10 @@
// specific language governing permissions and limitations
// under the License.
#include <src/scheduler/tasklabel/BroadcastLabel.h>
#include "TaskCreator.h"
#include "scheduler/TaskCreator.h"
#include "scheduler/tasklabel/BroadcastLabel.h"
#include "tasklabel/DefaultLabel.h"
namespace zilliz {
namespace milvus {
namespace scheduler {
@ -64,8 +63,6 @@ TaskCreator::Create(const DeleteJobPtr &job) {
return tasks;
}
}
}
}
} // namespace scheduler
} // namespace milvus
} // namespace zilliz

11
cpp/src/scheduler/TaskCreator.h
View File

@ -34,17 +34,16 @@
#include "task/SearchTask.h"
#include "task/DeleteTask.h"
namespace zilliz {
namespace milvus {
namespace scheduler {
class TaskCreator {
public:
public:
static std::vector<TaskPtr>
Create(const JobPtr &job);
public:
public:
static std::vector<TaskPtr>
Create(const SearchJobPtr &job);
@ -52,6 +51,6 @@ public:
Create(const DeleteJobPtr &job);
};
}
}
}
} // namespace scheduler
} // namespace milvus
} // namespace zilliz

15
cpp/src/scheduler/TaskTable.cpp
View File

@ -16,7 +16,7 @@
// under the License.
#include "TaskTable.h"
#include "scheduler/TaskTable.h"
#include "event/TaskTableUpdatedEvent.h"
#include "Utils.h"
@ -24,7 +24,6 @@
#include <sstream>
#include <ctime>
namespace zilliz {
namespace milvus {
namespace scheduler {
@ -75,6 +74,7 @@ TaskTableItem::Load() {
}
return false;
}
bool
TaskTableItem::Loaded() {
std::unique_lock<std::mutex> lock(mutex);
@ -86,6 +86,7 @@ TaskTableItem::Loaded() {
}
return false;
}
bool
TaskTableItem::Execute() {
std::unique_lock<std::mutex> lock(mutex);
@ -97,6 +98,7 @@ TaskTableItem::Execute() {
}
return false;
}
bool
TaskTableItem::Executed() {
std::unique_lock<std::mutex> lock(mutex);
@ -109,6 +111,7 @@ TaskTableItem::Executed() {
}
return false;
}
bool
TaskTableItem::Move() {
std::unique_lock<std::mutex> lock(mutex);
@ -120,6 +123,7 @@ TaskTableItem::Move() {
}
return false;
}
bool
TaskTableItem::Moved() {
std::unique_lock<std::mutex> lock(mutex);
@ -206,7 +210,6 @@ TaskTable::Put(std::vector<TaskPtr> &tasks) {
}
}
TaskTableItemPtr
TaskTable::Get(uint64_t index) {
return table_[index];
@ -232,6 +235,6 @@ TaskTable::Dump() {
return ss.str();
}
}
}
}
} // namespace scheduler
} // namespace milvus
} // namespace zilliz

36
cpp/src/scheduler/TaskTable.h
View File

@ -20,11 +20,13 @@
#include <vector>
#include <deque>
#include <mutex>
#include <memory>
#include <utility>
#include <string>
#include "task/SearchTask.h"
#include "event/Event.h"
namespace zilliz {
namespace milvus {
namespace scheduler {
@ -52,7 +54,8 @@ struct TaskTimestamp {
};
struct TaskTableItem {
TaskTableItem() : id(0), task(nullptr), state(TaskTableItemState::INVALID), mutex() {}
TaskTableItem() : id(0), task(nullptr), state(TaskTableItemState::INVALID), mutex() {
}
TaskTableItem(const TaskTableItem &src) = delete;
TaskTableItem(TaskTableItem &&) = delete;
@ -91,7 +94,7 @@ struct TaskTableItem {
using TaskTableItemPtr = std::shared_ptr<TaskTableItem>;
class TaskTable {
public:
public:
TaskTable() = default;
TaskTable(const TaskTable &) = delete;
@ -145,24 +148,28 @@ public:
return table_.size();
}
public:
public:
TaskTableItemPtr &
operator[](uint64_t index) {
return table_[index];
}
std::deque<TaskTableItemPtr>::iterator begin() { return table_.begin(); }
std::deque<TaskTableItemPtr>::iterator end() { return table_.end(); }
std::deque<TaskTableItemPtr>::iterator begin() {
return table_.begin();
}
public:
std::deque<TaskTableItemPtr>::iterator end() {
return table_.end();
}
public:
std::vector<uint64_t>
PickToLoad(uint64_t limit);
std::vector<uint64_t>
PickToExecute(uint64_t limit);
public:
public:
/******** Action ********/
// TODO: bool to Status
@ -227,14 +234,14 @@ public:
return table_[index]->Moved();
}
public:
public:
/*
* Dump;
*/
std::string
Dump();
private:
private:
std::uint64_t id_ = 0;
mutable std::mutex id_mutex_;
std::deque<TaskTableItemPtr> table_;
@ -246,7 +253,6 @@ private:
uint64_t last_finish_ = -1;
};
}
}
}
} // namespace scheduler
} // namespace milvus
} // namespace zilliz

9
cpp/src/scheduler/Utils.cpp
View File

@ -16,12 +16,11 @@
// under the License.
#include "Utils.h"
#include "scheduler/Utils.h"
#include <chrono>
#include <cuda_runtime.h>
namespace zilliz {
namespace milvus {
namespace scheduler {
@ -41,6 +40,6 @@ get_num_gpu() {
return n_devices;
}
}
}
}
} // namespace scheduler
} // namespace milvus
} // namespace zilliz

7
cpp/src/scheduler/Utils.h
View File

@ -18,7 +18,6 @@
#include <cstdint>
namespace zilliz {
namespace milvus {
namespace scheduler {
@ -29,6 +28,6 @@ get_current_timestamp();
uint64_t
get_num_gpu();
}
}
}
} // namespace scheduler
} // namespace milvus
} // namespace zilliz

15
cpp/src/scheduler/action/Action.h
View File

@ -17,16 +17,17 @@
#pragma once
#include "../resource/Resource.h"
#include "../ResourceMgr.h"
#include "scheduler/resource/Resource.h"
#include "scheduler/ResourceMgr.h"
#include <memory>
namespace zilliz {
namespace milvus {
namespace scheduler {
class Action {
public:
public:
static void
PushTaskToNeighbourRandomly(const TaskPtr &task, const ResourcePtr &self);
@ -43,10 +44,8 @@ public:
SpecifiedResourceLabelTaskScheduler(ResourceMgrWPtr res_mgr,
ResourcePtr resource,
std::shared_ptr<LoadCompletedEvent> event);
};
}
}
}
} // namespace scheduler
} // namespace milvus
} // namespace zilliz

22
cpp/src/scheduler/action/PushTaskToNeighbour.cpp
View File

@ -22,7 +22,6 @@
#include "src/cache/GpuCacheMgr.h"
#include "Action.h"
namespace zilliz {
namespace milvus {
namespace scheduler {
@ -57,13 +56,12 @@ get_neighbours_with_connetion(const ResourcePtr &self) {
return neighbours;
}
void
Action::PushTaskToNeighbourRandomly(const TaskPtr &task,
const ResourcePtr &self) {
auto neighbours = get_neighbours_with_connetion(self);
if (not neighbours.empty()) {
std::vector<uint64_t > speeds;
std::vector<uint64_t> speeds;
uint64_t total_speed = 0;
for (auto &neighbour : neighbours) {
uint64_t speed = neighbour.second.speed();
@ -87,7 +85,6 @@ Action::PushTaskToNeighbourRandomly(const TaskPtr &task,
} else {
//TODO: process
}
}
void
@ -99,14 +96,14 @@ Action::PushTaskToAllNeighbour(const TaskPtr &task, const ResourcePtr &self) {
}
void
Action::PushTaskToResource(const TaskPtr& task, const ResourcePtr& dest) {
Action::PushTaskToResource(const TaskPtr &task, const ResourcePtr &dest) {
dest->task_table().Put(task);
}
void
Action::DefaultLabelTaskScheduler(ResourceMgrWPtr res_mgr,
ResourcePtr resource,
std::shared_ptr<LoadCompletedEvent> event) {
ResourcePtr resource,
std::shared_ptr<LoadCompletedEvent> event) {
if (not resource->HasExecutor() && event->task_table_item_->Move()) {
auto task = event->task_table_item_->task;
auto search_task = std::static_pointer_cast<XSearchTask>(task);
@ -135,8 +132,8 @@ Action::DefaultLabelTaskScheduler(ResourceMgrWPtr res_mgr,
void
Action::SpecifiedResourceLabelTaskScheduler(ResourceMgrWPtr res_mgr,
ResourcePtr resource,
std::shared_ptr<LoadCompletedEvent> event) {
ResourcePtr resource,
std::shared_ptr<LoadCompletedEvent> event) {
auto task = event->task_table_item_->task;
if (resource->type() == ResourceType::DISK) {
// step 1: calculate shortest path per resource, from disk to compute resource
@ -181,7 +178,6 @@ Action::SpecifiedResourceLabelTaskScheduler(ResourceMgrWPtr res_mgr,
}
}
}
}
}
} // namespace scheduler
} // namespace milvus
} // namespace zilliz

15
cpp/src/scheduler/event/Event.h
View File

@ -18,6 +18,8 @@
#pragma once
#include <memory>
#include <utility>
#include <string>
namespace zilliz {
namespace milvus {
@ -33,11 +35,12 @@ enum class EventType {
class Resource;
class Event {
public:
public:
explicit
Event(EventType type, std::weak_ptr<Resource> resource)
: type_(type),
resource_(std::move(resource)) {}
resource_(std::move(resource)) {
}
inline EventType
Type() const {
@ -49,13 +52,13 @@ public:
friend std::ostream &operator<<(std::ostream &out, const Event &event);
public:
public:
EventType type_;
std::weak_ptr<Resource> resource_;
};
using EventPtr = std::shared_ptr<Event>;
}
}
}
} // namespace scheduler
} // namespace milvus
} // namespace zilliz

22
cpp/src/scheduler/event/EventDump.cpp
View File

@ -22,36 +22,40 @@
#include "FinishTaskEvent.h"
#include "TaskTableUpdatedEvent.h"
namespace zilliz {
namespace milvus {
namespace scheduler {
std::ostream &operator<<(std::ostream &out, const Event &event) {
std::ostream &
operator<<(std::ostream &out, const Event &event) {
out << event.Dump();
return out;
}
std::ostream &operator<<(std::ostream &out, const StartUpEvent &event) {
std::ostream &
operator<<(std::ostream &out, const StartUpEvent &event) {
out << event.Dump();
return out;
}
std::ostream &operator<<(std::ostream &out, const LoadCompletedEvent &event) {
std::ostream &
operator<<(std::ostream &out, const LoadCompletedEvent &event) {
out << event.Dump();
return out;
}
std::ostream &operator<<(std::ostream &out, const FinishTaskEvent &event) {
std::ostream &
operator<<(std::ostream &out, const FinishTaskEvent &event) {
out << event.Dump();
return out;
}
std::ostream &operator<<(std::ostream &out, const TaskTableUpdatedEvent &event) {
std::ostream &
operator<<(std::ostream &out, const TaskTableUpdatedEvent &event) {
out << event.Dump();
return out;
}
}
}
}
} // namespace scheduler
} // namespace milvus
} // namespace zilliz

18
cpp/src/scheduler/event/FinishTaskEvent.h
View File

@ -17,18 +17,22 @@
#pragma once
#include "Event.h"
#include "scheduler/event/Event.h"
#include <memory>
#include <utility>
#include <string>
namespace zilliz {
namespace milvus {
namespace scheduler {
class FinishTaskEvent : public Event {
public:
public:
FinishTaskEvent(std::weak_ptr<Resource> resource, TaskTableItemPtr task_table_item)
: Event(EventType::FINISH_TASK, std::move(resource)),
task_table_item_(std::move(task_table_item)) {}
task_table_item_(std::move(task_table_item)) {
}
inline std::string
Dump() const override {
@ -37,10 +41,10 @@ public:
friend std::ostream &operator<<(std::ostream &out, const FinishTaskEvent &event);
public:
public:
TaskTableItemPtr task_table_item_;
};
}
}
}
} // namespace scheduler
} // namespace milvus
} // namespace zilliz

20
cpp/src/scheduler/event/LoadCompletedEvent.h
View File

@ -17,19 +17,23 @@
#pragma once
#include "Event.h"
#include "../TaskTable.h"
#include "scheduler/event/Event.h"
#include "scheduler/TaskTable.h"
#include <memory>
#include <utility>
#include <string>
namespace zilliz {
namespace milvus {
namespace scheduler {
class LoadCompletedEvent : public Event {
public:
public:
LoadCompletedEvent(std::weak_ptr<Resource> resource, TaskTableItemPtr task_table_item)
: Event(EventType::LOAD_COMPLETED, std::move(resource)),
task_table_item_(std::move(task_table_item)) {}
task_table_item_(std::move(task_table_item)) {
}
inline std::string
Dump() const override {
@ -38,10 +42,10 @@ public:
friend std::ostream &operator<<(std::ostream &out, const LoadCompletedEvent &event);
public:
public:
TaskTableItemPtr task_table_item_;
};
}
}
}
} // namespace scheduler
} // namespace milvus
} // namespace zilliz

19
cpp/src/scheduler/event/StartUpEvent.h
View File

@ -17,18 +17,21 @@
#pragma once
#include "Event.h"
#include "scheduler/event/Event.h"
#include <memory>
#include <utility>
#include <string>
namespace zilliz {
namespace milvus {
namespace scheduler {
class StartUpEvent : public Event {
public:
explicit
StartUpEvent(std::weak_ptr<Resource> resource)
: Event(EventType::START_UP, std::move(resource)) {}
public:
explicit StartUpEvent(std::weak_ptr<Resource> resource)
: Event(EventType::START_UP, std::move(resource)) {
}
inline std::string
Dump() const override {
@ -38,6 +41,6 @@ public:
friend std::ostream &operator<<(std::ostream &out, const StartUpEvent &event);
};
}
}
}
} // namespace scheduler
} // namespace milvus
} // namespace zilliz

18
cpp/src/scheduler/event/TaskTableUpdatedEvent.h
View File

@ -19,16 +19,19 @@
#include "Event.h"
#include <memory>
#include <utility>
#include <string>
namespace zilliz {
namespace milvus {
namespace scheduler {
class TaskTableUpdatedEvent : public Event {
public:
explicit
TaskTableUpdatedEvent(std::weak_ptr<Resource> resource)
: Event(EventType::TASK_TABLE_UPDATED, std::move(resource)) {}
public:
explicit TaskTableUpdatedEvent(std::weak_ptr<Resource> resource)
: Event(EventType::TASK_TABLE_UPDATED, std::move(resource)) {
}
inline std::string
Dump() const override {
@ -38,7 +41,6 @@ public:
friend std::ostream &operator<<(std::ostream &out, const TaskTableUpdatedEvent &event);
};
}
}
}
} // namespace scheduler
} // namespace milvus
} // namespace zilliz

23
cpp/src/scheduler/job/DeleteJob.cpp
View File

@ -15,8 +15,9 @@
// specific language governing permissions and limitations
// under the License.
#include "DeleteJob.h"
#include "scheduler/job/DeleteJob.h"
#include <utility>
namespace zilliz {
namespace milvus {
@ -29,15 +30,20 @@ DeleteJob::DeleteJob(JobId id,
: Job(id, JobType::DELETE),
table_id_(std::move(table_id)),
meta_ptr_(std::move(meta_ptr)),
num_resource_(num_resource) {}
num_resource_(num_resource) {
}
void DeleteJob::WaitAndDelete() {
void
DeleteJob::WaitAndDelete() {
std::unique_lock<std::mutex> lock(mutex_);
cv_.wait(lock, [&] { return done_resource == num_resource_; });
cv_.wait(lock, [&] {
return done_resource == num_resource_;
});
meta_ptr_->DeleteTableFiles(table_id_);
}
void DeleteJob::ResourceDone() {
void
DeleteJob::ResourceDone() {
{
std::lock_guard<std::mutex> lock(mutex_);
++done_resource;
@ -45,7 +51,6 @@ void DeleteJob::ResourceDone() {
cv_.notify_one();
}
}
}
}
} // namespace scheduler
} // namespace milvus
} // namespace zilliz

16
cpp/src/scheduler/job/DeleteJob.h
View File

@ -30,26 +30,25 @@
#include "Job.h"
#include "db/meta/Meta.h"
namespace zilliz {
namespace milvus {
namespace scheduler {
class DeleteJob : public Job {
public:
public:
DeleteJob(JobId id,
std::string table_id,
engine::meta::MetaPtr meta_ptr,
uint64_t num_resource);
public:
public:
void
WaitAndDelete();
void
ResourceDone();
public:
public:
std::string
table_id() const {
return table_id_;
@ -60,7 +59,7 @@ public:
return meta_ptr_;
}
private:
private:
std::string table_id_;
engine::meta::MetaPtr meta_ptr_;
@ -72,7 +71,6 @@ private:
using DeleteJobPtr = std::shared_ptr<DeleteJob>;
}
}
}
} // namespace scheduler
} // namespace milvus
} // namespace zilliz

17
cpp/src/scheduler/job/Job.h
View File

@ -27,7 +27,6 @@
#include <condition_variable>
#include <memory>
namespace zilliz {
namespace milvus {
namespace scheduler {
@ -42,7 +41,7 @@ enum class JobType {
using JobId = std::uint64_t;
class Job {
public:
public:
inline JobId
id() const {
return id_;
@ -53,10 +52,11 @@ public:
return type_;
}
protected:
Job(JobId id, JobType type) : id_(id), type_(type) {}
protected:
Job(JobId id, JobType type) : id_(id), type_(type) {
}
private:
private:
JobId id_;
JobType type_;
};
@ -64,7 +64,6 @@ private:
using JobPtr = std::shared_ptr<Job>;
using JobWPtr = std::weak_ptr<Job>;
}
}
}
} // namespace scheduler
} // namespace milvus
} // namespace zilliz

23
cpp/src/scheduler/job/SearchJob.cpp
View File

@ -15,11 +15,9 @@
// specific language governing permissions and limitations
// under the License.
#include "scheduler/job/SearchJob.h"
#include "utils/Log.h"
#include "SearchJob.h"
namespace zilliz {
namespace milvus {
namespace scheduler {
@ -33,7 +31,8 @@ SearchJob::SearchJob(zilliz::milvus::scheduler::JobId id,
topk_(topk),
nq_(nq),
nprobe_(nprobe),
vectors_(vectors) {}
vectors_(vectors) {
}
bool
SearchJob::AddIndexFile(const TableFileSchemaPtr &index_file) {
@ -48,11 +47,12 @@ SearchJob::AddIndexFile(const TableFileSchemaPtr &index_file) {
return true;
}
void
SearchJob::WaitResult() {
std::unique_lock<std::mutex> lock(mutex_);
cv_.wait(lock, [this] { return index_files_.empty(); });
cv_.wait(lock, [this] {
return index_files_.empty();
});
SERVER_LOG_DEBUG << "SearchJob " << id() << " all done";
}
@ -69,14 +69,11 @@ SearchJob::GetResult() {
return result_;
}
Status&
Status &
SearchJob::GetStatus() {
return status_;
}
}
}
}
} // namespace scheduler
} // namespace milvus
} // namespace zilliz

19
cpp/src/scheduler/job/SearchJob.h
View File

@ -26,16 +26,15 @@
#include <mutex>
#include <condition_variable>
#include <memory>
#include <utility>
#include "Job.h"
#include "db/meta/MetaTypes.h"
namespace zilliz {
namespace milvus {
namespace scheduler {
using engine::meta::TableFileSchemaPtr;
using Id2IndexMap = std::unordered_map<size_t, TableFileSchemaPtr>;
@ -43,10 +42,10 @@ using Id2DistanceMap = std::vector<std::pair<int64_t, double>>;
using ResultSet = std::vector<Id2DistanceMap>;
class SearchJob : public Job {
public:
public:
SearchJob(JobId id, uint64_t topk, uint64_t nq, uint64_t nprobe, const float *vectors);
public:
public:
bool
AddIndexFile(const TableFileSchemaPtr &index_file);
@ -62,7 +61,7 @@ public:
Status &
GetStatus();
public:
public:
uint64_t
topk() const {
return topk_;
@ -77,6 +76,7 @@ public:
nprobe() const {
return nprobe_;
}
const float *
vectors() const {
return vectors_;
@ -87,7 +87,7 @@ public:
return index_files_;
}
private:
private:
uint64_t topk_ = 0;
uint64_t nq_ = 0;
uint64_t nprobe_ = 0;
@ -105,7 +105,6 @@ private:
using SearchJobPtr = std::shared_ptr<SearchJob>;
}
}
}
} // namespace scheduler
} // namespace milvus
} // namespace zilliz

18
cpp/src/scheduler/resource/Connection.h
View File

@ -19,17 +19,18 @@
#include <string>
#include <sstream>
#include <utility>
namespace zilliz {
namespace milvus {
namespace scheduler {
class Connection {
public:
public:
// TODO: update construct function, speed: double->uint64_t
Connection(std::string name, double speed)
: name_(std::move(name)), speed_(speed) {}
: name_(std::move(name)), speed_(speed) {
}
const std::string &
name() const {
@ -46,7 +47,7 @@ public:
return 1024 / speed_;
}
public:
public:
std::string
Dump() const {
std::stringstream ss;
@ -54,12 +55,11 @@ public:
return ss.str();
}
private:
private:
std::string name_;
uint64_t speed_;
};
}
}
}
} // namespace scheduler
} // namespace milvus
} // namespace zilliz

21
cpp/src/scheduler/resource/CpuResource.cpp
View File

@ -16,29 +16,34 @@
// under the License.
#include "CpuResource.h"
#include "scheduler/resource/CpuResource.h"
#include <utility>
namespace zilliz {
namespace milvus {
namespace scheduler {
std::ostream &operator<<(std::ostream &out, const CpuResource &resource) {
std::ostream &
operator<<(std::ostream &out, const CpuResource &resource) {
out << resource.Dump();
return out;
}
CpuResource::CpuResource(std::string name, uint64_t device_id, bool enable_loader, bool enable_executor)
: Resource(std::move(name), ResourceType::CPU, device_id, enable_loader, enable_executor) {}
: Resource(std::move(name), ResourceType::CPU, device_id, enable_loader, enable_executor) {
}
void CpuResource::LoadFile(TaskPtr task) {
void
CpuResource::LoadFile(TaskPtr task) {
task->Load(LoadType::DISK2CPU, 0);
}
void CpuResource::Process(TaskPtr task) {
void
CpuResource::Process(TaskPtr task) {
task->Execute();
}
}
}
}
} // namespace scheduler
} // namespace milvus
} // namespace zilliz

11
cpp/src/scheduler/resource/CpuResource.h
View File

@ -21,13 +21,12 @@
#include "Resource.h"
namespace zilliz {
namespace milvus {
namespace scheduler {
class CpuResource : public Resource {
public:
public:
explicit
CpuResource(std::string name, uint64_t device_id, bool enable_loader, bool enable_executor);
@ -38,7 +37,7 @@ public:
friend std::ostream &operator<<(std::ostream &out, const CpuResource &resource);
protected:
protected:
void
LoadFile(TaskPtr task) override;
@ -46,6 +45,6 @@ protected:
Process(TaskPtr task) override;
};
}
}
}
} // namespace scheduler
} // namespace milvus
} // namespace zilliz

22
cpp/src/scheduler/resource/DiskResource.cpp
View File

@ -15,14 +15,17 @@
// specific language governing permissions and limitations
// under the License.
#include "DiskResource.h"
#include "scheduler/resource/DiskResource.h"
#include <string>
#include <utility>
namespace zilliz {
namespace milvus {
namespace scheduler {
std::ostream &operator<<(std::ostream &out, const DiskResource &resource) {
std::ostream &
operator<<(std::ostream &out, const DiskResource &resource) {
out << resource.Dump();
return out;
}
@ -31,15 +34,14 @@ DiskResource::DiskResource(std::string name, uint64_t device_id, bool enable_loa
: Resource(std::move(name), ResourceType::DISK, device_id, enable_loader, enable_executor) {
}
void DiskResource::LoadFile(TaskPtr task) {
void
DiskResource::LoadFile(TaskPtr task) {
}
void DiskResource::Process(TaskPtr task) {
void
DiskResource::Process(TaskPtr task) {
}
}
}
}
} // namespace scheduler
} // namespace milvus
} // namespace zilliz

12
cpp/src/scheduler/resource/DiskResource.h
View File

@ -17,16 +17,16 @@
#pragma once
#include "Resource.h"
#include <string>
namespace zilliz {
namespace milvus {
namespace scheduler {
class DiskResource : public Resource {
public:
public:
explicit
DiskResource(std::string name, uint64_t device_id, bool enable_loader, bool enable_executor);
@ -37,7 +37,7 @@ public:
friend std::ostream &operator<<(std::ostream &out, const DiskResource &resource);
protected:
protected:
void
LoadFile(TaskPtr task) override;
@ -45,6 +45,6 @@ protected:
Process(TaskPtr task) override;
};
}
}
}
} // namespace scheduler
} // namespace milvus
} // namespace zilliz

21
cpp/src/scheduler/resource/GpuResource.cpp
View File

@ -16,29 +16,32 @@
// under the License.
#include "GpuResource.h"
#include "scheduler/resource/GpuResource.h"
namespace zilliz {
namespace milvus {
namespace scheduler {
std::ostream &operator<<(std::ostream &out, const GpuResource &resource) {
std::ostream &
operator<<(std::ostream &out, const GpuResource &resource) {
out << resource.Dump();
return out;
}
GpuResource::GpuResource(std::string name, uint64_t device_id, bool enable_loader, bool enable_executor)
: Resource(std::move(name), ResourceType::GPU, device_id, enable_loader, enable_executor) {}
: Resource(std::move(name), ResourceType::GPU, device_id, enable_loader, enable_executor) {
}
void GpuResource::LoadFile(TaskPtr task) {
void
GpuResource::LoadFile(TaskPtr task) {
task->Load(LoadType::CPU2GPU, device_id_);
}
void GpuResource::Process(TaskPtr task) {
void
GpuResource::Process(TaskPtr task) {
task->Execute();
}
}
}
}
} // namespace scheduler
} // namespace milvus
} // namespace zilliz

13
cpp/src/scheduler/resource/GpuResource.h
View File

@ -17,16 +17,17 @@
#pragma once
#include "Resource.h"
#include <string>
#include <utility>
namespace zilliz {
namespace milvus {
namespace scheduler {
class GpuResource : public Resource {
public:
public:
explicit
GpuResource(std::string name, uint64_t device_id, bool enable_loader, bool enable_executor);
@ -37,7 +38,7 @@ public:
friend std::ostream &operator<<(std::ostream &out, const GpuResource &resource);
protected:
protected:
void
LoadFile(TaskPtr task) override;
@ -45,6 +46,6 @@ protected:
Process(TaskPtr task) override;
};
}
}
}
} // namespace scheduler
} // namespace milvus
} // namespace zilliz

19
cpp/src/scheduler/resource/Node.cpp
View File

@ -15,10 +15,10 @@
// specific language governing permissions and limitations
// under the License.
#include "scheduler/resource/Node.h"
#include <atomic>
#include "Node.h"
#include <utility>
namespace zilliz {
namespace milvus {
@ -29,7 +29,8 @@ Node::Node() {
id_ = counter++;
}
std::vector<Neighbour> Node::GetNeighbours() {
std::vector<Neighbour>
Node::GetNeighbours() {
std::lock_guard<std::mutex> lk(mutex_);
std::vector<Neighbour> ret;
for (auto &e : neighbours_) {
@ -38,7 +39,8 @@ std::vector<Neighbour> Node::GetNeighbours() {
return ret;
}
std::string Node::Dump() {
std::string
Node::Dump() {
std::stringstream ss;
ss << "<Node, id=" << std::to_string(id_) << ">::neighbours:" << std::endl;
for (auto &neighbour : neighbours_) {
@ -48,7 +50,8 @@ std::string Node::Dump() {
return ss.str();
}
void Node::AddNeighbour(const NeighbourNodePtr &neighbour_node, Connection &connection) {
void
Node::AddNeighbour(const NeighbourNodePtr &neighbour_node, Connection &connection) {
std::lock_guard<std::mutex> lk(mutex_);
if (auto s = neighbour_node.lock()) {
neighbours_.emplace(std::make_pair(s->id_, Neighbour(neighbour_node, connection)));
@ -56,6 +59,6 @@ void Node::AddNeighbour(const NeighbourNodePtr &neighbour_node, Connection &conn
// else do nothing, consider it..
}
}
}
}
} // namespace scheduler
} // namespace milvus
} // namespace zilliz

21
cpp/src/scheduler/resource/Node.h
View File

@ -20,11 +20,11 @@
#include <vector>
#include <memory>
#include <map>
#include <string>
#include "../TaskTable.h"
#include "scheduler/TaskTable.h"
#include "Connection.h"
namespace zilliz {
namespace milvus {
namespace scheduler {
@ -34,8 +34,9 @@ class Node;
using NeighbourNodePtr = std::weak_ptr<Node>;
struct Neighbour {
Neighbour(NeighbourNodePtr nei, Connection conn)
: neighbour_node(nei), connection(conn) {}
Neighbour(NeighbourNodePtr nei, Connection conn)
: neighbour_node(nei), connection(conn) {
}
NeighbourNodePtr neighbour_node;
Connection connection;
@ -43,7 +44,7 @@ struct Neighbour {
// TODO(linxj): return type void -> Status
class Node {
public:
public:
Node();
void
@ -52,11 +53,11 @@ public:
std::vector<Neighbour>
GetNeighbours();
public:
public:
std::string
Dump();
private:
private:
std::mutex mutex_;
uint8_t id_;
std::map<uint8_t, Neighbour> neighbours_;
@ -65,6 +66,6 @@ private:
using NodePtr = std::shared_ptr<Node>;
using NodeWPtr = std::weak_ptr<Node>;
}
}
}
} // namespace scheduler
} // namespace milvus
} // namespace zilliz

35
cpp/src/scheduler/resource/Resource.cpp
View File

@ -15,10 +15,11 @@
// specific language governing permissions and limitations
// under the License.
#include <iostream>
#include "../Utils.h"
#include "Resource.h"
#include "scheduler/resource/Resource.h"
#include "scheduler/Utils.h"
#include <iostream>
#include <utility>
namespace zilliz {
namespace milvus {
@ -100,7 +101,8 @@ Resource::NumOfTaskToExec() {
return count;
}
TaskTableItemPtr Resource::pick_task_load() {
TaskTableItemPtr
Resource::pick_task_load() {
auto indexes = task_table_.PickToLoad(10);
for (auto index : indexes) {
// try to set one task loading, then return
@ -111,7 +113,8 @@ TaskTableItemPtr Resource::pick_task_load() {
return nullptr;
}
TaskTableItemPtr Resource::pick_task_execute() {
TaskTableItemPtr
Resource::pick_task_execute() {
auto indexes = task_table_.PickToExecute(3);
for (auto index : indexes) {
// try to set one task executing, then return
@ -122,10 +125,13 @@ TaskTableItemPtr Resource::pick_task_execute() {
return nullptr;
}
void Resource::loader_function() {
void
Resource::loader_function() {
while (running_) {
std::unique_lock<std::mutex> lock(load_mutex_);
load_cv_.wait(lock, [&] { return load_flag_; });
load_cv_.wait(lock, [&] {
return load_flag_;
});
load_flag_ = false;
lock.unlock();
while (true) {
@ -140,18 +146,20 @@ void Resource::loader_function() {
subscriber_(std::static_pointer_cast<Event>(event));
}
}
}
}
void Resource::executor_function() {
void
Resource::executor_function() {
if (subscriber_) {
auto event = std::make_shared<StartUpEvent>(shared_from_this());
subscriber_(std::static_pointer_cast<Event>(event));
}
while (running_) {
std::unique_lock<std::mutex> lock(exec_mutex_);
exec_cv_.wait(lock, [&] { return exec_flag_; });
exec_cv_.wait(lock, [&] {
return exec_flag_;
});
exec_flag_ = false;
lock.unlock();
while (true) {
@ -172,10 +180,9 @@ void Resource::executor_function() {
subscriber_(std::static_pointer_cast<Event>(event));
}
}
}
}
}
}
}
} // namespace scheduler
} // namespace milvus
} // namespace zilliz

11
cpp/src/scheduler/resource/Resource.h
View File

@ -20,6 +20,7 @@
#include <string>
#include <vector>
#include <memory>
#include <utility>
#include <thread>
#include <functional>
#include <condition_variable>
@ -34,7 +35,6 @@
#include "Connection.h"
#include "Node.h"
namespace zilliz {
namespace milvus {
namespace scheduler {
@ -104,7 +104,7 @@ class Resource : public Node, public std::enable_shared_from_this<Resource> {
return task_table_;
}
public:
public:
inline bool
HasLoader() const {
return enable_loader_;
@ -212,7 +212,6 @@ public:
using ResourcePtr = std::shared_ptr<Resource>;
using ResourceWPtr = std::weak_ptr<Resource>;
}
}
}
} // namespace scheduler
} // namespace milvus
} // namespace zilliz

19
cpp/src/scheduler/resource/TestResource.cpp
View File

@ -15,14 +15,16 @@
// specific language governing permissions and limitations
// under the License.
#include "TestResource.h"
#include "scheduler/resource/TestResource.h"
#include <string>
namespace zilliz {
namespace milvus {
namespace scheduler {
std::ostream &operator<<(std::ostream &out, const TestResource &resource) {
std::ostream &
operator<<(std::ostream &out, const TestResource &resource) {
out << resource.Dump();
return out;
}
@ -31,15 +33,16 @@ TestResource::TestResource(std::string name, uint64_t device_id, bool enable_loa
: Resource(std::move(name), ResourceType::TEST, device_id, enable_loader, enable_executor) {
}
void TestResource::LoadFile(TaskPtr task) {
void
TestResource::LoadFile(TaskPtr task) {
task->Load(LoadType::TEST, 0);
}
void TestResource::Process(TaskPtr task) {
void
TestResource::Process(TaskPtr task) {
task->Execute();
}
}
}
}
} // namespace scheduler
} // namespace milvus
} // namespace zilliz

13
cpp/src/scheduler/resource/TestResource.h
View File

@ -17,16 +17,17 @@
#pragma once
#include "Resource.h"
#include <utility>
#include <string>
namespace zilliz {
namespace milvus {
namespace scheduler {
class TestResource : public Resource {
public:
public:
explicit
TestResource(std::string name, uint64_t device_id, bool enable_loader, bool enable_executor);
@ -37,7 +38,7 @@ public:
friend std::ostream &operator<<(std::ostream &out, const TestResource &resource);
protected:
protected:
void
LoadFile(TaskPtr task) override;
@ -45,6 +46,6 @@ protected:
Process(TaskPtr task) override;
};
}
}
}
} // namespace scheduler
} // namespace milvus
} // namespace zilliz

13
cpp/src/scheduler/task/DeleteTask.cpp
View File

@ -16,19 +16,18 @@
// under the License.
#include "DeleteTask.h"
#include "scheduler/task/DeleteTask.h"
namespace zilliz {
namespace milvus {
namespace scheduler {
XDeleteTask::XDeleteTask(const scheduler::DeleteJobPtr &delete_job)
: Task(TaskType::DeleteTask), delete_job_(delete_job) {}
: Task(TaskType::DeleteTask), delete_job_(delete_job) {
}
void
XDeleteTask::Load(LoadType type, uint8_t device_id) {
}
void
@ -36,6 +35,6 @@ XDeleteTask::Execute() {
delete_job_->ResourceDone();
}
}
}
}
} // namespace scheduler
} // namespace milvus
} // namespace zilliz

14
cpp/src/scheduler/task/DeleteTask.h
View File

@ -20,15 +20,13 @@
#include "scheduler/job/DeleteJob.h"
#include "Task.h"
namespace zilliz {
namespace milvus {
namespace scheduler {
class XDeleteTask : public Task {
public:
explicit
XDeleteTask(const scheduler::DeleteJobPtr &job);
public:
explicit XDeleteTask(const scheduler::DeleteJobPtr &job);
void
Load(LoadType type, uint8_t device_id) override;
@ -36,10 +34,10 @@ public:
void
Execute() override;
public:
public:
scheduler::DeleteJobPtr delete_job_;
};
}
}
}
} // namespace scheduler
} // namespace milvus
} // namespace zilliz

20
cpp/src/scheduler/task/Path.h
View File

@ -20,7 +20,6 @@
#include <vector>
#include <string>
namespace zilliz {
namespace milvus {
namespace scheduler {
@ -29,7 +28,8 @@ class Path {
public:
Path() = default;
Path(std::vector<std::string>& path, uint64_t index) : path_(path), index_(index) {}
Path(std::vector<std::string> &path, uint64_t index) : path_(path), index_(index) {
}
void
push_back(const std::string &str) {
@ -49,7 +49,6 @@ class Path {
} else {
return nullptr;
}
}
std::string
@ -67,14 +66,19 @@ class Path {
return path_[index];
}
std::vector<std::string>::iterator begin() { return path_.begin(); }
std::vector<std::string>::iterator end() { return path_.end(); }
std::vector<std::string>::iterator begin() {
return path_.begin();
}
std::vector<std::string>::iterator end() {
return path_.end();
}
public:
std::vector<std::string> path_;
uint64_t index_ = 0;
};
}
}
}
} // namespace scheduler
} // namespace milvus
} // namespace zilliz

50
cpp/src/scheduler/task/SearchTask.cpp
View File

@ -15,15 +15,16 @@
// specific language governing permissions and limitations
// under the License.
#include "SearchTask.h"
#include "scheduler/task/SearchTask.h"
#include "scheduler/job/SearchJob.h"
#include "metrics/Metrics.h"
#include "db/engine/EngineFactory.h"
#include "utils/TimeRecorder.h"
#include "utils/Log.h"
#include <thread>
#include "scheduler/job/SearchJob.h"
#include <utility>
#include <string>
namespace zilliz {
namespace milvus {
@ -104,7 +105,6 @@ XSearchTask::XSearchTask(TableFileSchemaPtr file)
(MetricType) file_->metric_type_,
file_->nlist_);
}
}
void
@ -144,7 +144,7 @@ XSearchTask::Load(LoadType type, uint8_t device_id) {
s = Status(SERVER_UNEXPECTED_ERROR, error_msg);
}
if (auto job = job_.lock()){
if (auto job = job_.lock()) {
auto search_job = std::static_pointer_cast<scheduler::SearchJob>(job);
search_job->SearchDone(file_->id_);
search_job->GetStatus() = s;
@ -183,7 +183,7 @@ XSearchTask::Execute() {
server::CollectDurationMetrics metrics(index_type_);
std::vector<long> output_ids;
std::vector<int64_t> output_ids;
std::vector<float> output_distance;
if (auto job = job_.lock()) {
@ -192,7 +192,7 @@ XSearchTask::Execute() {
uint64_t nq = search_job->nq();
uint64_t topk = search_job->topk();
uint64_t nprobe = search_job->nprobe();
const float* vectors = search_job->vectors();
const float *vectors = search_job->vectors();
output_ids.resize(topk * nq);
output_distance.resize(topk * nq);
@ -236,11 +236,12 @@ XSearchTask::Execute() {
index_engine_ = nullptr;
}
Status XSearchTask::ClusterResult(const std::vector<long> &output_ids,
const std::vector<float> &output_distance,
uint64_t nq,
uint64_t topk,
scheduler::ResultSet &result_set) {
Status
XSearchTask::ClusterResult(const std::vector<int64_t> &output_ids,
const std::vector<float> &output_distance,
uint64_t nq,
uint64_t topk,
scheduler::ResultSet &result_set) {
if (output_ids.size() < nq * topk || output_distance.size() < nq * topk) {
std::string msg = "Invalid id array size: " + std::to_string(output_ids.size()) +
" distance array size: " + std::to_string(output_distance.size());
@ -275,10 +276,11 @@ Status XSearchTask::ClusterResult(const std::vector<long> &output_ids,
return Status::OK();
}
Status XSearchTask::MergeResult(scheduler::Id2DistanceMap &distance_src,
scheduler::Id2DistanceMap &distance_target,
uint64_t topk,
bool ascending) {
Status
XSearchTask::MergeResult(scheduler::Id2DistanceMap &distance_src,
scheduler::Id2DistanceMap &distance_target,
uint64_t topk,
bool ascending) {
//Note: the score_src and score_target are already arranged by score in ascending order
if (distance_src.empty()) {
ENGINE_LOG_WARNING << "Empty distance source array";
@ -349,10 +351,11 @@ Status XSearchTask::MergeResult(scheduler::Id2DistanceMap &distance_src,
return Status::OK();
}
Status XSearchTask::TopkResult(scheduler::ResultSet &result_src,
uint64_t topk,
bool ascending,
scheduler::ResultSet &result_target) {
Status
XSearchTask::TopkResult(scheduler::ResultSet &result_src,
uint64_t topk,
bool ascending,
scheduler::ResultSet &result_target) {
if (result_target.empty()) {
result_target.swap(result_src);
return Status::OK();
@ -381,7 +384,6 @@ Status XSearchTask::TopkResult(scheduler::ResultSet &result_src,
return Status::OK();
}
}
}
}
} // namespace scheduler
} // namespace milvus
} // namespace zilliz

18
cpp/src/scheduler/task/SearchTask.h
View File

@ -21,6 +21,7 @@
#include "scheduler/job/SearchJob.h"
#include "scheduler/Definition.h"
#include <vector>
namespace zilliz {
namespace milvus {
@ -28,9 +29,8 @@ namespace scheduler {
// TODO: rewrite
class XSearchTask : public Task {
public:
explicit
XSearchTask(TableFileSchemaPtr file);
public:
explicit XSearchTask(TableFileSchemaPtr file);
void
Load(LoadType type, uint8_t device_id) override;
@ -38,8 +38,8 @@ public:
void
Execute() override;
public:
static Status ClusterResult(const std::vector<long> &output_ids,
public:
static Status ClusterResult(const std::vector<int64_t> &output_ids,
const std::vector<float> &output_distence,
uint64_t nq,
uint64_t topk,
@ -55,7 +55,7 @@ public:
bool ascending,
scheduler::ResultSet &result_target);
public:
public:
TableFileSchemaPtr file_;
size_t index_id_ = 0;
@ -66,6 +66,6 @@ public:
static std::mutex merge_mutex_;
};
}
}
}
} // namespace scheduler
} // namespace milvus
} // namespace zilliz

24
cpp/src/scheduler/task/Task.h
View File

@ -25,7 +25,6 @@
#include <string>
#include <memory>
namespace zilliz {
namespace milvus {
namespace scheduler {
@ -49,20 +48,22 @@ using TaskPtr = std::shared_ptr<Task>;
// TODO: re-design
class Task {
public:
explicit
Task(TaskType type) : type_(type) {}
public:
explicit Task(TaskType type) : type_(type) {
}
/*
* Just Getter;
*/
inline TaskType
Type() const { return type_; }
Type() const {
return type_;
}
/*
* Transport path;
*/
inline Path&
inline Path &
path() {
return task_path_;
}
@ -75,14 +76,14 @@ public:
return label_;
}
public:
public:
virtual void
Load(LoadType type, uint8_t device_id) = 0;
virtual void
Execute() = 0;
public:
public:
Path task_path_;
// std::vector<SearchContextPtr> search_contexts_;
scheduler::JobWPtr job_;
@ -90,7 +91,6 @@ public:
TaskLabelPtr label_ = nullptr;
};
}
}
}
} // namespace scheduler
} // namespace milvus
} // namespace zilliz

20
cpp/src/scheduler/task/TestTask.cpp
View File

@ -15,16 +15,15 @@
// specific language governing permissions and limitations
// under the License.
#include <src/cache/GpuCacheMgr.h>
#include "TestTask.h"
#include "scheduler/task/TestTask.h"
#include "cache/GpuCacheMgr.h"
namespace zilliz {
namespace milvus {
namespace scheduler {
TestTask::TestTask(TableFileSchemaPtr &file) : XSearchTask(file) {}
TestTask::TestTask(TableFileSchemaPtr &file) : XSearchTask(file) {
}
void
TestTask::Load(LoadType type, uint8_t device_id) {
@ -44,10 +43,11 @@ TestTask::Execute() {
void
TestTask::Wait() {
std::unique_lock<std::mutex> lock(mutex_);
cv_.wait(lock, [&] { return done_; });
}
}
}
cv_.wait(lock, [&] {
return done_;
});
}
} // namespace scheduler
} // namespace milvus
} // namespace zilliz

17
cpp/src/scheduler/task/TestTask.h
View File

@ -19,17 +19,15 @@
#include "SearchTask.h"
namespace zilliz {
namespace milvus {
namespace scheduler {
class TestTask : public XSearchTask {
public:
explicit
TestTask(TableFileSchemaPtr& file);
public:
explicit TestTask(TableFileSchemaPtr &file);
public:
public:
void
Load(LoadType type, uint8_t device_id) override;
@ -39,7 +37,7 @@ public:
void
Wait();
public:
public:
uint64_t load_count_ = 0;
uint64_t exec_count_ = 0;
@ -48,7 +46,6 @@ public:
std::condition_variable cv_;
};
}
}
}
} // namespace scheduler
} // namespace milvus
} // namespace zilliz

14
cpp/src/scheduler/tasklabel/BroadcastLabel.h
View File

@ -21,19 +21,19 @@
#include <memory>
namespace zilliz {
namespace milvus {
namespace scheduler {
class BroadcastLabel : public TaskLabel {
public:
BroadcastLabel() : TaskLabel(TaskLabelType::BROADCAST) {}
public:
BroadcastLabel() : TaskLabel(TaskLabelType::BROADCAST) {
}
};
using BroadcastLabelPtr = std::shared_ptr<BroadcastLabel>;
}
}
}
} // namespace scheduler
} // namespace milvus
} // namespace zilliz

14
cpp/src/scheduler/tasklabel/DefaultLabel.h
View File

@ -21,20 +21,18 @@
#include <memory>
namespace zilliz {
namespace milvus {
namespace scheduler {
class DefaultLabel : public TaskLabel {
public:
DefaultLabel() : TaskLabel(TaskLabelType::DEFAULT) {}
public:
DefaultLabel() : TaskLabel(TaskLabelType::DEFAULT) {
}
};
using DefaultLabelPtr = std::shared_ptr<DefaultLabel>;
}
}
}
} // namespace scheduler
} // namespace milvus
} // namespace zilliz

17
cpp/src/scheduler/tasklabel/SpecResLabel.h
View File

@ -22,7 +22,6 @@
#include <string>
#include <memory>
class Resource;
using ResourceWPtr = std::weak_ptr<Resource>;
@ -32,9 +31,10 @@ namespace milvus {
namespace scheduler {
class SpecResLabel : public TaskLabel {
public:
SpecResLabel(const ResourceWPtr &resource)
: TaskLabel(TaskLabelType::SPECIFIED_RESOURCE), resource_(resource) {}
public:
explicit SpecResLabel(const ResourceWPtr &resource)
: TaskLabel(TaskLabelType::SPECIFIED_RESOURCE), resource_(resource) {
}
inline ResourceWPtr &
resource() {
@ -46,14 +46,13 @@ public:
return resource_name_;
}
private:
private:
ResourceWPtr resource_;
std::string resource_name_;
};
using SpecResLabelPtr = std::shared_ptr<SpecResLabel>();
}
}
}
} // namespace scheduler
} // namespace milvus
} // namespace zilliz

17
cpp/src/scheduler/tasklabel/TaskLabel.h
View File

@ -30,23 +30,22 @@ enum class TaskLabelType {
};
class TaskLabel {
public:
public:
inline TaskLabelType
Type() const {
return type_;
}
protected:
explicit
TaskLabel(TaskLabelType type) : type_(type) {}
protected:
explicit TaskLabel(TaskLabelType type) : type_(type) {
}
private:
private:
TaskLabelType type_;
};
using TaskLabelPtr = std::shared_ptr<TaskLabel>;
}
}
}
} // namespace scheduler
} // namespace milvus
} // namespace zilliz