milvus-io
/
milvus
mirror of https://github.com/milvus-io/milvus.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'upstream/branch-0.4.0' into branch-0.4.0 

Former-commit-id: d918f411479abd408d66ae84741ffb5231ccce42
pull/191/head
Yu Kun 2019-09-11 19:08:35 +08:00
parent 789c6f9e4f aee725a3f2
commit 6afd997b7a
44 changed files with 375 additions and 322 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
cpp/conf/server_config.template
View File

@ -56,12 +56,12 @@ resource_config:
cpu:
type: CPU
device_id: 0
enable_executor: false
enable_executor: true
gpu0:
type: GPU
device_id: 0
enable_executor: true
enable_executor: false
gpu_resource_num: 2
pinned_memory: 300
temp_memory: 300

3
cpp/src/db/DB.h
View File

@ -11,6 +11,7 @@
#include "Types.h"
#include <string>
#include <memory>
namespace zilliz {
namespace milvus {
@ -61,6 +62,8 @@ public:
}; // DB
using DBPtr = std::shared_ptr<DB>;
} // namespace engine
} // namespace milvus
} // namespace zilliz

37
cpp/src/db/DBFactory.cpp Normal file
View File

@ -0,0 +1,37 @@
////////////////////////////////////////////////////////////////////////////////
// Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved
// Unauthorized copying of this file, via any medium is strictly prohibited.
// Proprietary and confidential.
////////////////////////////////////////////////////////////////////////////////
#include "DBFactory.h"
#include "DBImpl.h"
#include "Exception.h"
#include "meta/MetaFactory.h"
#include "meta/SqliteMetaImpl.h"
#include "meta/MySQLMetaImpl.h"
#include <stdlib.h>
#include <time.h>
#include <sstream>
#include <cstdlib>
#include <string>
namespace zilliz {
namespace milvus {
namespace engine {
Options DBFactory::BuildOption() {
auto meta = MetaFactory::BuildOption();
Options options;
options.meta = meta;
return options;
}
DBPtr DBFactory::Build(const Options& options) {
return std::make_shared<DBImpl>(options);
}
} // namespace engine
} // namespace milvus
} // namespace zilliz

28
cpp/src/db/DBFactory.h Normal file
View File

@ -0,0 +1,28 @@
////////////////////////////////////////////////////////////////////////////////
// Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved
// Unauthorized copying of this file, via any medium is strictly prohibited.
// Proprietary and confidential.
////////////////////////////////////////////////////////////////////////////////
#pragma once
#include "DB.h"
#include "Options.h"
#include <string>
#include <memory>
namespace zilliz {
namespace milvus {
namespace engine {
class DBFactory {
public:
static Options BuildOption();
static DBPtr Build(const Options& options);
};
}
}
}

5
cpp/src/db/DBImpl.cpp
View File

@ -8,7 +8,8 @@
#include "Log.h"
#include "Utils.h"
#include "engine/EngineFactory.h"
#include "Factories.h"
#include "insert/MemMenagerFactory.h"
#include "meta/MetaFactory.h"
#include "metrics/Metrics.h"
#include "scheduler/TaskScheduler.h"
@ -44,7 +45,7 @@ DBImpl::DBImpl(const Options& options)
shutting_down_(true),
compact_thread_pool_(1, 1),
index_thread_pool_(1, 1) {
meta_ptr_ = DBMetaImplFactory::Build(options.meta, options.mode);
meta_ptr_ = MetaFactory::Build(options.meta, options.mode);
mem_mgr_ = MemManagerFactory::Build(meta_ptr_, options_);
Start();
}

6
cpp/src/db/DBImpl.h
View File

@ -32,8 +32,6 @@ class Meta;
class DBImpl : public DB {
public:
using MetaPtr = meta::Meta::Ptr;
explicit DBImpl(const Options &options);
~DBImpl();
@ -126,8 +124,8 @@ class DBImpl : public DB {
std::thread bg_timer_thread_;
MetaPtr meta_ptr_;
MemManagerAbstractPtr mem_mgr_;
meta::MetaPtr meta_ptr_;
MemManagerPtr mem_mgr_;
std::mutex mem_serialize_mutex_;
server::ThreadPool compact_thread_pool_;

110
cpp/src/db/Factories.cpp
View File

@ -1,110 +0,0 @@
////////////////////////////////////////////////////////////////////////////////
// Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved
// Unauthorized copying of this file, via any medium is strictly prohibited.
// Proprietary and confidential.
////////////////////////////////////////////////////////////////////////////////
#include "Factories.h"
#include "DBImpl.h"
#include "src/db/insert/MemManagerImpl.h"
#include "Exception.h"
#include <stdlib.h>
#include <time.h>
#include <sstream>
#include <iostream>
#include <vector>
#include <assert.h>
#include <easylogging++.h>
#include <regex>
#include <cstdlib>
#include <string>
#include <algorithm>
namespace zilliz {
namespace milvus {
namespace engine {
DBMetaOptions DBMetaOptionsFactory::Build(const std::string& path) {
auto p = path;
if(p == "") {
srand(time(nullptr));
std::stringstream ss;
ss << "/tmp/" << rand();
p = ss.str();
}
DBMetaOptions meta;
meta.path = p;
return meta;
}
Options OptionsFactory::Build() {
auto meta = DBMetaOptionsFactory::Build();
Options options;
options.meta = meta;
return options;
}
std::shared_ptr<meta::SqliteMetaImpl> DBMetaImplFactory::Build() {
DBMetaOptions options = DBMetaOptionsFactory::Build();
return std::shared_ptr<meta::SqliteMetaImpl>(new meta::SqliteMetaImpl(options));
}
std::shared_ptr<meta::Meta> DBMetaImplFactory::Build(const DBMetaOptions& metaOptions,
const int& mode) {
std::string uri = metaOptions.backend_uri;
std::string dialectRegex = "(.*)";
std::string usernameRegex = "(.*)";
std::string passwordRegex = "(.*)";
std::string hostRegex = "(.*)";
std::string portRegex = "(.*)";
std::string dbNameRegex = "(.*)";
std::string uriRegexStr = dialectRegex + "\\:\\/\\/" +
usernameRegex + "\\:" +
passwordRegex + "\\@" +
hostRegex + "\\:" +
portRegex + "\\/" +
dbNameRegex;
std::regex uriRegex(uriRegexStr);
std::smatch pieces_match;
if (std::regex_match(uri, pieces_match, uriRegex)) {
std::string dialect = pieces_match[1].str();
std::transform(dialect.begin(), dialect.end(), dialect.begin(), ::tolower);
if (dialect.find("mysql") != std::string::npos) {
ENGINE_LOG_INFO << "Using MySQL";
return std::make_shared<meta::MySQLMetaImpl>(metaOptions, mode);
} else if (dialect.find("sqlite") != std::string::npos) {
ENGINE_LOG_INFO << "Using SQLite";
return std::make_shared<meta::SqliteMetaImpl>(metaOptions);
} else {
ENGINE_LOG_ERROR << "Invalid dialect in URI: dialect = " << dialect;
throw InvalidArgumentException("URI dialect is not mysql / sqlite");
}
} else {
ENGINE_LOG_ERROR << "Wrong URI format: URI = " << uri;
throw InvalidArgumentException("Wrong URI format ");
}
}
//std::shared_ptr<DB> DBFactory::Build() {
// auto options = OptionsFactory::Build();
// auto db = DBFactory::Build(options);
// return std::shared_ptr<DB>(db);
//}
DB* DBFactory::Build(const Options& options) {
return new DBImpl(options);
}
MemManagerAbstractPtr MemManagerFactory::Build(const std::shared_ptr<meta::Meta>& meta,
const Options& options) {
return std::make_shared<MemManagerImpl>(meta, options);
}
} // namespace engine
} // namespace milvus
} // namespace zilliz

46
cpp/src/db/Factories.h
View File

@ -1,46 +0,0 @@
////////////////////////////////////////////////////////////////////////////////
// Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved
// Unauthorized copying of this file, via any medium is strictly prohibited.
// Proprietary and confidential.
////////////////////////////////////////////////////////////////////////////////
#pragma once
#include "DB.h"
#include "src/db/meta/SqliteMetaImpl.h"
#include "meta/MySQLMetaImpl.h"
#include "Options.h"
#include "src/db/insert/MemManager.h"
#include <string>
#include <memory>
namespace zilliz {
namespace milvus {
namespace engine {
struct DBMetaOptionsFactory {
static DBMetaOptions Build(const std::string &path = "");
};
struct OptionsFactory {
static Options Build();
};
struct DBMetaImplFactory {
static std::shared_ptr<meta::SqliteMetaImpl> Build();
static std::shared_ptr<meta::Meta> Build(const DBMetaOptions &metaOptions, const int &mode);
};
struct DBFactory {
//static std::shared_ptr<DB> Build();
static DB *Build(const Options &);
};
struct MemManagerFactory {
static MemManagerAbstractPtr Build(const std::shared_ptr<meta::Meta> &meta, const Options &options);
};
} // namespace engine
} // namespace milvus
} // namespace zilliz

4
cpp/src/db/engine/ExecutionEngineImpl.cpp
View File

@ -320,8 +320,8 @@ Status ExecutionEngineImpl::GpuCache(uint64_t gpu_id) {
Status ExecutionEngineImpl::Init() {
using namespace zilliz::milvus::server;
ServerConfig &config = ServerConfig::GetInstance();
ConfigNode server_config = config.GetConfig(CONFIG_SERVER);
gpu_num_ = server_config.GetInt32Value("gpu_index", 0);
ConfigNode server_config = config.GetConfig(CONFIG_DB);
gpu_num_ = server_config.GetInt32Value(CONFIG_DB_BUILD_INDEX_GPU, 0);
return Status::OK();
}

7
cpp/src/db/insert/MemManager.h
View File

@ -1,7 +1,10 @@
#pragma once
#include <set>
#include "db/Status.h"
#include "db/Types.h"
#include <set>
#include <memory>
namespace zilliz {
namespace milvus {
@ -25,7 +28,7 @@ class MemManager {
}; // MemManagerAbstract
using MemManagerAbstractPtr = std::shared_ptr<MemManager>;
using MemManagerPtr = std::shared_ptr<MemManager>;
} // namespace engine
} // namespace milvus

4
cpp/src/db/insert/MemManagerImpl.cpp
View File

@ -10,7 +10,7 @@ namespace zilliz {
namespace milvus {
namespace engine {
MemManagerImpl::MemTablePtr MemManagerImpl::GetMemByTable(const std::string &table_id) {
MemTablePtr MemManagerImpl::GetMemByTable(const std::string &table_id) {
auto memIt = mem_id_map_.find(table_id);
if (memIt != mem_id_map_.end()) {
return memIt->second;
@ -40,7 +40,7 @@ Status MemManagerImpl::InsertVectorsNoLock(const std::string &table_id,
IDNumbers &vector_ids) {
MemTablePtr mem = GetMemByTable(table_id);
VectorSource::Ptr source = std::make_shared<VectorSource>(n, vectors);
VectorSourcePtr source = std::make_shared<VectorSource>(n, vectors);
auto status = mem->Add(source, vector_ids);
if (status.ok()) {

6
cpp/src/db/insert/MemManagerImpl.h
View File

@ -18,11 +18,9 @@ namespace engine {
class MemManagerImpl : public MemManager {
public:
using MetaPtr = meta::Meta::Ptr;
using Ptr = std::shared_ptr<MemManagerImpl>;
using MemTablePtr = typename MemTable::Ptr;
MemManagerImpl(const std::shared_ptr<meta::Meta> &meta, const Options &options)
MemManagerImpl(const meta::MetaPtr &meta, const Options &options)
: meta_(meta), options_(options) {}
Status InsertVectors(const std::string &table_id,
@ -49,7 +47,7 @@ class MemManagerImpl : public MemManager {
using MemList = std::vector<MemTablePtr>;
MemIdMap mem_id_map_;
MemList immu_mem_list_;
MetaPtr meta_;
meta::MetaPtr meta_;
Options options_;
std::mutex mutex_;
std::mutex serialization_mtx_;

30
cpp/src/db/insert/MemMenagerFactory.cpp Normal file
View File

@ -0,0 +1,30 @@
////////////////////////////////////////////////////////////////////////////////
// Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved
// Unauthorized copying of this file, via any medium is strictly prohibited.
// Proprietary and confidential.
////////////////////////////////////////////////////////////////////////////////
#include "MemMenagerFactory.h"
#include "MemManagerImpl.h"
#include "db/Log.h"
#include "db/Exception.h"
#include <stdlib.h>
#include <time.h>
#include <sstream>
#include <cstdlib>
#include <string>
#include <regex>
namespace zilliz {
namespace milvus {
namespace engine {
MemManagerPtr MemManagerFactory::Build(const std::shared_ptr<meta::Meta>& meta,
const Options& options) {
return std::make_shared<MemManagerImpl>(meta, options);
}
} // namespace engine
} // namespace milvus
} // namespace zilliz

22
cpp/src/db/insert/MemMenagerFactory.h Normal file
View File

@ -0,0 +1,22 @@
////////////////////////////////////////////////////////////////////////////////
// Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved
// Unauthorized copying of this file, via any medium is strictly prohibited.
// Proprietary and confidential.
////////////////////////////////////////////////////////////////////////////////
#pragma once
#include "MemManager.h"
#include "db/meta/Meta.h"
namespace zilliz {
namespace milvus {
namespace engine {
class MemManagerFactory {
public:
static MemManagerPtr Build(const std::shared_ptr<meta::Meta> &meta, const Options &options);
};
}
}
}

10
cpp/src/db/insert/MemTable.cpp
View File

@ -7,7 +7,7 @@ namespace milvus {
namespace engine {
MemTable::MemTable(const std::string &table_id,
const std::shared_ptr<meta::Meta> &meta,
const meta::MetaPtr &meta,
const Options &options) :
table_id_(table_id),
meta_(meta),
@ -15,18 +15,18 @@ MemTable::MemTable(const std::string &table_id,
}
Status MemTable::Add(VectorSource::Ptr &source, IDNumbers &vector_ids) {
Status MemTable::Add(VectorSourcePtr &source, IDNumbers &vector_ids) {
while (!source->AllAdded()) {
MemTableFile::Ptr current_mem_table_file;
MemTableFilePtr current_mem_table_file;
if (!mem_table_file_list_.empty()) {
current_mem_table_file = mem_table_file_list_.back();
}
Status status;
if (mem_table_file_list_.empty() || current_mem_table_file->IsFull()) {
MemTableFile::Ptr new_mem_table_file = std::make_shared<MemTableFile>(table_id_, meta_, options_);
MemTableFilePtr new_mem_table_file = std::make_shared<MemTableFile>(table_id_, meta_, options_);
status = new_mem_table_file->Add(source, vector_ids);
if (status.ok()) {
mem_table_file_list_.emplace_back(new_mem_table_file);
@ -44,7 +44,7 @@ Status MemTable::Add(VectorSource::Ptr &source, IDNumbers &vector_ids) {
return Status::OK();
}
void MemTable::GetCurrentMemTableFile(MemTableFile::Ptr &mem_table_file) {
void MemTable::GetCurrentMemTableFile(MemTableFilePtr &mem_table_file) {
mem_table_file = mem_table_file_list_.back();
}

16
cpp/src/db/insert/MemTable.h
View File

@ -12,18 +12,14 @@ namespace milvus {
namespace engine {
class MemTable {
public:
using MemTableFileList = std::vector<MemTableFilePtr>;
using Ptr = std::shared_ptr<MemTable>;
using MemTableFileList = std::vector<MemTableFile::Ptr>;
using MetaPtr = meta::Meta::Ptr;
MemTable(const std::string &table_id, const meta::MetaPtr &meta, const Options &options);
MemTable(const std::string &table_id, const std::shared_ptr<meta::Meta> &meta, const Options &options);
Status Add(VectorSourcePtr &source, IDNumbers &vector_ids);
Status Add(VectorSource::Ptr &source, IDNumbers &vector_ids);
void GetCurrentMemTableFile(MemTableFile::Ptr &mem_table_file);
void GetCurrentMemTableFile(MemTableFilePtr &mem_table_file);
size_t GetTableFileCount();
@ -40,7 +36,7 @@ class MemTable {
MemTableFileList mem_table_file_list_;
MetaPtr meta_;
meta::MetaPtr meta_;
Options options_;
@ -48,6 +44,8 @@ class MemTable {
}; //MemTable
using MemTablePtr = std::shared_ptr<MemTable>;
} // namespace engine
} // namespace milvus
} // namespace zilliz

4
cpp/src/db/insert/MemTableFile.cpp
View File

@ -12,7 +12,7 @@ namespace milvus {
namespace engine {
MemTableFile::MemTableFile(const std::string &table_id,
const std::shared_ptr<meta::Meta> &meta,
const meta::MetaPtr &meta,
const Options &options) :
table_id_(table_id),
meta_(meta),
@ -43,7 +43,7 @@ Status MemTableFile::CreateTableFile() {
return status;
}
Status MemTableFile::Add(const VectorSource::Ptr &source, IDNumbers& vector_ids) {
Status MemTableFile::Add(const VectorSourcePtr &source, IDNumbers& vector_ids) {
if (table_file_schema_.dimension_ <= 0) {
std::string err_msg = "MemTableFile::Add: table_file_schema dimension = " +

12
cpp/src/db/insert/MemTableFile.h
View File

@ -13,13 +13,9 @@ namespace engine {
class MemTableFile {
public:
MemTableFile(const std::string &table_id, const meta::MetaPtr &meta, const Options &options);
using Ptr = std::shared_ptr<MemTableFile>;
using MetaPtr = meta::Meta::Ptr;
MemTableFile(const std::string &table_id, const std::shared_ptr<meta::Meta> &meta, const Options &options);
Status Add(const VectorSource::Ptr &source, IDNumbers& vector_ids);
Status Add(const VectorSourcePtr &source, IDNumbers& vector_ids);
size_t GetCurrentMem();
@ -37,7 +33,7 @@ class MemTableFile {
meta::TableFileSchema table_file_schema_;
MetaPtr meta_;
meta::MetaPtr meta_;
Options options_;
@ -47,6 +43,8 @@ class MemTableFile {
}; //MemTableFile
using MemTableFilePtr = std::shared_ptr<MemTableFile>;
} // namespace engine
} // namespace milvus
} // namespace zilliz

6
cpp/src/db/insert/VectorSource.h
View File

@ -11,11 +11,7 @@ namespace milvus {
namespace engine {
class VectorSource {
public:
using Ptr = std::shared_ptr<VectorSource>;
VectorSource(const size_t &n, const float *vectors);
Status Add(const ExecutionEnginePtr &execution_engine,
@ -42,6 +38,8 @@ class VectorSource {
}; //VectorSource
using VectorSourcePtr = std::shared_ptr<VectorSource>;
} // namespace engine
} // namespace milvus
} // namespace zilliz

4
cpp/src/db/meta/Meta.h
View File

@ -20,8 +20,6 @@ namespace meta {
class Meta {
public:
using Ptr = std::shared_ptr<Meta>;
virtual ~Meta() = default;
virtual Status CreateTable(TableSchema &table_schema) = 0;
@ -85,6 +83,8 @@ class Meta {
}; // MetaData
using MetaPtr = std::shared_ptr<Meta>;
} // namespace meta
} // namespace engine
} // namespace milvus

77
cpp/src/db/meta/MetaFactory.cpp Normal file
View File

@ -0,0 +1,77 @@
////////////////////////////////////////////////////////////////////////////////
// Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved
// Unauthorized copying of this file, via any medium is strictly prohibited.
// Proprietary and confidential.
////////////////////////////////////////////////////////////////////////////////
#include "MetaFactory.h"
#include "SqliteMetaImpl.h"
#include "MySQLMetaImpl.h"
#include "db/Log.h"
#include "db/Exception.h"
#include <stdlib.h>
#include <time.h>
#include <sstream>
#include <cstdlib>
#include <string>
#include <regex>
namespace zilliz {
namespace milvus {
namespace engine {
DBMetaOptions MetaFactory::BuildOption(const std::string &path) {
auto p = path;
if(p == "") {
srand(time(nullptr));
std::stringstream ss;
ss << "/tmp/" << rand();
p = ss.str();
}
DBMetaOptions meta;
meta.path = p;
return meta;
}
meta::MetaPtr MetaFactory::Build(const DBMetaOptions &metaOptions, const int &mode) {
std::string uri = metaOptions.backend_uri;
std::string dialectRegex = "(.*)";
std::string usernameRegex = "(.*)";
std::string passwordRegex = "(.*)";
std::string hostRegex = "(.*)";
std::string portRegex = "(.*)";
std::string dbNameRegex = "(.*)";
std::string uriRegexStr = dialectRegex + "\\:\\/\\/" +
usernameRegex + "\\:" +
passwordRegex + "\\@" +
hostRegex + "\\:" +
portRegex + "\\/" +
dbNameRegex;
std::regex uriRegex(uriRegexStr);
std::smatch pieces_match;
if (std::regex_match(uri, pieces_match, uriRegex)) {
std::string dialect = pieces_match[1].str();
std::transform(dialect.begin(), dialect.end(), dialect.begin(), ::tolower);
if (dialect.find("mysql") != std::string::npos) {
ENGINE_LOG_INFO << "Using MySQL";
return std::make_shared<meta::MySQLMetaImpl>(metaOptions, mode);
} else if (dialect.find("sqlite") != std::string::npos) {
ENGINE_LOG_INFO << "Using SQLite";
return std::make_shared<meta::SqliteMetaImpl>(metaOptions);
} else {
ENGINE_LOG_ERROR << "Invalid dialect in URI: dialect = " << dialect;
throw InvalidArgumentException("URI dialect is not mysql / sqlite");
}
} else {
ENGINE_LOG_ERROR << "Wrong URI format: URI = " << uri;
throw InvalidArgumentException("Wrong URI format ");
}
}
} // namespace engine
} // namespace milvus
} // namespace zilliz

25
cpp/src/db/meta/MetaFactory.h Normal file
View File

@ -0,0 +1,25 @@
////////////////////////////////////////////////////////////////////////////////
// Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved
// Unauthorized copying of this file, via any medium is strictly prohibited.
// Proprietary and confidential.
////////////////////////////////////////////////////////////////////////////////
#pragma once
#include "Meta.h"
#include "db/Options.h"
namespace zilliz {
namespace milvus {
namespace engine {
class MetaFactory {
public:
static DBMetaOptions BuildOption(const std::string &path = "");
static meta::MetaPtr Build(const DBMetaOptions &metaOptions, const int &mode);
};
}
}
}

1
cpp/src/db/meta/MySQLMetaImpl.cpp
View File

@ -8,7 +8,6 @@
#include "db/Utils.h"
#include "db/Log.h"
#include "MetaConsts.h"
#include "db/Factories.h"
#include "metrics/Metrics.h"
#include <unistd.h>

1
cpp/src/db/meta/SqliteMetaImpl.cpp
View File

@ -8,7 +8,6 @@
#include "db/Utils.h"
#include "db/Log.h"
#include "MetaConsts.h"
#include "db/Factories.h"
#include "metrics/Metrics.h"
#include <unistd.h>

2
cpp/src/db/scheduler/context/DeleteContext.cpp
View File

@ -11,7 +11,7 @@ namespace zilliz {
namespace milvus {
namespace engine {
DeleteContext::DeleteContext(const std::string &table_id, meta::Meta::Ptr &meta_ptr, uint64_t num_resource)
DeleteContext::DeleteContext(const std::string &table_id, meta::MetaPtr &meta_ptr, uint64_t num_resource)
: IScheduleContext(ScheduleContextType::kDelete),
table_id_(table_id),
meta_ptr_(meta_ptr),

6
cpp/src/db/scheduler/context/DeleteContext.h
View File

@ -16,16 +16,16 @@ namespace engine {
class DeleteContext : public IScheduleContext {
public:
DeleteContext(const std::string& table_id, meta::Meta::Ptr& meta_ptr, uint64_t num_resource);
DeleteContext(const std::string& table_id, meta::MetaPtr& meta_ptr, uint64_t num_resource);
std::string table_id() const { return table_id_; }
meta::Meta::Ptr meta() const { return meta_ptr_; }
meta::MetaPtr meta() const { return meta_ptr_; }
void WaitAndDelete();
void ResourceDone();
private:
std::string table_id_;
meta::Meta::Ptr meta_ptr_;
meta::MetaPtr meta_ptr_;
uint64_t num_resource_;
uint64_t done_resource = 0;

1
cpp/src/db/scheduler/context/SearchContext.cpp
View File

@ -50,6 +50,7 @@ void
SearchContext::WaitResult() {
std::unique_lock <std::mutex> lock(mtx_);
done_cond_.wait(lock, [this] { return map_index_files_.empty(); });
SERVER_LOG_DEBUG << "SearchContext " << identity_ << " all done";
}
}

31
cpp/src/scheduler/task/SearchTask.cpp
View File

@ -152,10 +152,10 @@ XSearchTask::Execute() {
return;
}
ENGINE_LOG_DEBUG << "Searching in file id:" << index_id_ << " with "
ENGINE_LOG_DEBUG << "Searching in file id " << index_id_ << " with "
<< search_contexts_.size() << " tasks";
server::TimeRecorder rc("DoSearch file id:" + std::to_string(index_id_));
server::TimeRecorder rc("DoSearch file id " + std::to_string(index_id_));
server::CollectDurationMetrics metrics(index_type_);
@ -163,32 +163,37 @@ XSearchTask::Execute() {
std::vector<float> output_distance;
for (auto &context : search_contexts_) {
//step 1: allocate memory
auto inner_k = context->topk();
auto nq = context->nq();
auto topk = context->topk();
auto nprobe = context->nprobe();
output_ids.resize(inner_k * context->nq());
output_distance.resize(inner_k * context->nq());
auto vectors = context->vectors();
output_ids.resize(topk * nq);
output_distance.resize(topk * nq);
std::string hdr = "context " + context->Identity() +
" nq " + std::to_string(nq) +
" topk " + std::to_string(topk);
try {
//step 2: search
index_engine_->Search(context->nq(), context->vectors(), inner_k, nprobe, output_distance.data(),
output_ids.data());
index_engine_->Search(nq, vectors, topk, nprobe, output_distance.data(), output_ids.data());
double span = rc.RecordSection("do search for context:" + context->Identity());
double span = rc.RecordSection(hdr + ", do search");
context->AccumSearchCost(span);
//step 3: cluster result
SearchContext::ResultSet result_set;
auto spec_k = index_engine_->Count() < context->topk() ? index_engine_->Count() : context->topk();
XSearchTask::ClusterResult(output_ids, output_distance, context->nq(), spec_k, result_set);
auto spec_k = index_engine_->Count() < topk ? index_engine_->Count() : topk;
XSearchTask::ClusterResult(output_ids, output_distance, nq, spec_k, result_set);
span = rc.RecordSection("cluster result for context:" + context->Identity());
span = rc.RecordSection(hdr + ", cluster result");
context->AccumReduceCost(span);
// step 4: pick up topk result
XSearchTask::TopkResult(result_set, inner_k, metric_l2, context->GetResult());
XSearchTask::TopkResult(result_set, topk, metric_l2, context->GetResult());
span = rc.RecordSection("reduce topk for context:" + context->Identity());
span = rc.RecordSection(hdr + ", reduce topk");
context->AccumReduceCost(span);
} catch (std::exception &ex) {
ENGINE_LOG_ERROR << "SearchTask encounter exception: " << ex.what();

5
cpp/src/server/DBWrapper.cpp
View File

@ -6,7 +6,7 @@
#include "DBWrapper.h"
#include "ServerConfig.h"
#include "db/Factories.h"
#include "db/DBFactory.h"
#include "utils/CommonUtil.h"
#include "utils/Log.h"
#include "utils/StringHelpFunctions.h"
@ -96,8 +96,7 @@ ErrorCode DBWrapper::StartService() {
//create db instance
std::string msg = opt.meta.path;
try {
engine::DB* db = engine::DBFactory::Build(opt);
db_.reset(db);
db_ = engine::DBFactory::Build(opt);
} catch(std::exception& ex) {
msg = ex.what();
}

6
cpp/src/server/DBWrapper.h
View File

@ -25,19 +25,19 @@ public:
return wrapper;
}
static std::shared_ptr<engine::DB> DB() {
static engine::DBPtr DB() {
return GetInstance().EngineDB();
}
ErrorCode StartService();
ErrorCode StopService();
std::shared_ptr<engine::DB> EngineDB() {
engine::DBPtr EngineDB() {
return db_;
}
private:
std::shared_ptr<engine::DB> db_;
engine::DBPtr db_;
};
}

116
cpp/src/server/ServerConfig.cpp
View File

@ -86,11 +86,12 @@ ErrorCode ServerConfig::ValidateConfig() {
ErrorCode
ServerConfig::CheckServerConfig() {
/*
server_config:
address: 0.0.0.0 # milvus server ip address
port: 19530 # the port milvus listen to, default: 19530, range: 1025 ~ 65534
gpu_index: 0 # the gpu milvus use, default: 0, range: 0 ~ gpu number - 1
mode: single # milvus deployment type: single, cluster, read_only
server_config:
address: 0.0.0.0 # milvus server ip address (IPv4)
port: 19530 # the port milvus listen to, default: 19530, range: 1025 ~ 65534
mode: single # milvus deployment type: single, cluster, read_only
time_zone: UTC+8 # Use the UTC-x or UTC+x to specify a time zone. eg. UTC+8 for China Standard Time
*/
bool okay = true;
ConfigNode server_config = GetConfig(CONFIG_SERVER);
@ -144,20 +145,20 @@ ServerConfig::CheckServerConfig() {
ErrorCode
ServerConfig::CheckDBConfig() {
/*
db_config:
db_path: @MILVUS_DB_PATH@ # milvus data storage path
db_slave_path: # secondry data storage path, split by semicolon
parallel_reduce: false # use multi-threads to reduce topk result
db_config:
db_path: @MILVUS_DB_PATH@ # milvus data storage path
db_slave_path: # secondry data storage path, split by semicolon
# URI format: dialect://username:password@host:port/database
# All parts except dialect are optional, but you MUST include the delimiters
# Currently dialect supports mysql or sqlite
db_backend_url: sqlite://:@:/
# URI format: dialect://username:password@host:port/database
# All parts except dialect are optional, but you MUST include the delimiters
# Currently dialect supports mysql or sqlite
db_backend_url: sqlite://:@:/
archive_disk_threshold: 0 # triger archive action if storage size exceed this value, 0 means no limit, unit: GB
archive_days_threshold: 0 # files older than x days will be archived, 0 means no limit, unit: day
insert_buffer_size: 4 # maximum insert buffer size allowed, default: 4, unit: GB, should be at least 1 GB.
# the sum of insert_buffer_size and cpu_cache_capacity should be less than total memory, unit: GB
archive_disk_threshold: 0 # triger archive action if storage size exceed this value, 0 means no limit, unit: GB
archive_days_threshold: 0 # files older than x days will be archived, 0 means no limit, unit: day
insert_buffer_size: 4 # maximum insert buffer size allowed, default: 4, unit: GB, should be at least 1 GB.
# the sum of insert_buffer_size and cpu_cache_capacity should be less than total memory, unit: GB
build_index_gpu: 0 # which gpu is used to build index, default: 0, range: 0 ~ gpu number - 1
*/
bool okay = true;
ConfigNode db_config = GetConfig(CONFIG_DB);
@ -249,15 +250,13 @@ ServerConfig::CheckMetricConfig() {
ErrorCode
ServerConfig::CheckCacheConfig() {
/*
cache_config:
cpu_cache_capacity: 16 # how many memory are used as cache, unit: GB, range: 0 ~ less than total memory
cpu_cache_free_percent: 0.85 # old data will be erased from cache when cache is full, this value specify how much memory should be kept, range: greater than zero ~ 1.0
insert_cache_immediately: false # insert data will be load into cache immediately for hot query
gpu_cache_capacity: 5 # how many memory are used as cache in gpu, unit: GB, RANGE: 0 ~ less than total memory
gpu_cache_free_percent: 0.85 # old data will be erased from cache when cache is full, this value specify how much memory should be kept, range: greater than zero ~ 1.0
gpu_ids: # gpu id
- 0
- 1
cache_config:
cpu_cache_capacity: 16 # how many memory are used as cache, unit: GB, range: 0 ~ less than total memory
cpu_cache_free_percent: 0.85 # old data will be erased from cache when cache is full, this value specify how much memory should be kept, range: greater than zero ~ 1.0
insert_cache_immediately: false # insert data will be load into cache immediately for hot query
gpu_cache_capacity: 5 # how many memory are used as cache in gpu, unit: GB, RANGE: 0 ~ less than total memory
gpu_cache_free_percent: 0.85 # old data will be erased from cache when cache is full, this value specify how much memory should be kept, range: greater than zero ~ 1.0
*/
bool okay = true;
ConfigNode cache_config = GetConfig(CONFIG_CACHE);
@ -379,48 +378,47 @@ ServerConfig::CheckEngineConfig() {
ErrorCode
ServerConfig::CheckResourceConfig() {
/*
resource_config:
# resource list, length: 0~N
# please set a DISK resource and a CPU resource least, or system will not return query result.
#
# example:
# resource_name: # resource name, just using in connections below
# type: DISK # resource type, optional: DISK/CPU/GPU
# device_id: 0
# enable_executor: false # if is enable executor, optional: true, false
resource_config:
# resource list, length: 0~N
# please set a DISK resource and a CPU resource least, or system will not return query result.
#
# example:
# resource_name: # resource name, just using in connections below
# type: DISK # resource type, optional: DISK/CPU/GPU
# device_id: 0
# enable_executor: false # if is enable executor, optional: true, false
resources:
resources:
ssda:
type: DISK
device_id: 0
enable_executor: false
type: DISK
device_id: 0
enable_executor: false
cpu:
type: CPU
device_id: 0
enable_executor: false
type: CPU
device_id: 0
enable_executor: true
gpu0:
type: GPU
device_id: 0
enable_executor: true
gpu_resource_num: 2
pinned_memory: 300
temp_memory: 300
type: GPU
device_id: 0
enable_executor: false
gpu_resource_num: 2
pinned_memory: 300
temp_memory: 300
# connection list, length: 0~N
# example:
# connection_name:
# speed: 100 # unit: MS/s
# endpoint: ${resource_name}===${resource_name}
connections:
# connection list, length: 0~N
# example:
# connection_name:
# speed: 100 # unit: MS/s
# endpoint: ${resource_name}===${resource_name}
connections:
io:
speed: 500
endpoint: ssda===cpu
speed: 500
endpoint: ssda===cpu
pcie0:
speed: 11000
endpoint: cpu===gpu0
speed: 11000
endpoint: cpu===gpu0
*/
bool okay = true;
server::ConfigNode resource_config = GetConfig(CONFIG_RESOURCE);

8
cpp/src/server/grpc_impl/GrpcRequestTask.cpp
View File

@ -571,7 +571,11 @@ SearchTask::Create(const ::milvus::grpc::SearchParam *search_vector_infos,
ErrorCode
SearchTask::OnExecute() {
try {
TimeRecorder rc("SearchTask");
int64_t top_k = search_param_->topk();
int64_t nprobe = search_param_->nprobe();
std::string hdr = "SearchTask(k=" + std::to_string(top_k) + ", nprob=" + std::to_string(nprobe) + ")";
TimeRecorder rc(hdr);
//step 1: check table name
std::string table_name_ = search_param_->table_name();
@ -593,13 +597,11 @@ SearchTask::OnExecute() {
}
//step 3: check search parameter
int64_t top_k = search_param_->topk();
res = ValidationUtil::ValidateSearchTopk(top_k, table_info);
if (res != SERVER_SUCCESS) {
return SetError(res, "Invalid topk: " + std::to_string(top_k));
}
int64_t nprobe = search_param_->nprobe();
res = ValidationUtil::ValidateSearchNprobe(nprobe, table_info);
if (res != SERVER_SUCCESS) {
return SetError(res, "Invalid nprobe: " + std::to_string(nprobe));

4
cpp/src/utils/ValidationUtil.cpp
View File

@ -224,6 +224,9 @@ ValidationUtil::ValidateDbURI(const std::string &uri) {
okay = false;
}
/*
* Could be DNS, skip checking
*
std::string host = pieces_match[4].str();
if (!host.empty() && host != "localhost") {
if (ValidateIpAddress(host) != SERVER_SUCCESS) {
@ -231,6 +234,7 @@ ValidationUtil::ValidateDbURI(const std::string &uri) {
okay = false;
}
}
*/
std::string port = pieces_match[5].str();
if (!port.empty()) {

7
cpp/unittest/db/db_tests.cpp
View File

@ -7,7 +7,7 @@
#include "db/DB.h"
#include "db/DBImpl.h"
#include "db/meta/MetaConsts.h"
#include "db/Factories.h"
#include "db/DBFactory.h"
#include "cache/CpuCacheMgr.h"
#include "utils/CommonUtil.h"
@ -467,11 +467,6 @@ TEST_F(DBTest2, DELETE_TEST) {
};
TEST_F(DBTest2, DELETE_BY_RANGE_TEST) {
auto options = engine::OptionsFactory::Build();
options.meta.path = "/tmp/milvus_test";
options.meta.backend_uri = "sqlite://:@:/";
auto db_ = engine::DBFactory::Build(options);
engine::meta::TableSchema table_info = BuildTableSchema();
engine::Status stat = db_->CreateTable(table_info);

13
cpp/unittest/db/mem_test.cpp
View File

@ -3,7 +3,6 @@
#include "db/insert/VectorSource.h"
#include "db/insert/MemTableFile.h"
#include "db/insert/MemTable.h"
#include "db/Factories.h"
#include "db/Constants.h"
#include "db/engine/EngineFactory.h"
#include "db/meta/MetaConsts.h"
@ -106,7 +105,7 @@ TEST_F(MemManagerTest, MEM_TABLE_FILE_TEST) {
std::vector<float> vectors_100;
BuildVectors(n_100, vectors_100);
engine::VectorSource::Ptr source = std::make_shared<engine::VectorSource>(n_100, vectors_100.data());
engine::VectorSourcePtr source = std::make_shared<engine::VectorSource>(n_100, vectors_100.data());
engine::IDNumbers vector_ids;
status = mem_table_file.Add(source, vector_ids);
@ -124,7 +123,7 @@ TEST_F(MemManagerTest, MEM_TABLE_FILE_TEST) {
std::vector<float> vectors_128M;
BuildVectors(n_max, vectors_128M);
engine::VectorSource::Ptr source_128M = std::make_shared<engine::VectorSource>(n_max, vectors_128M.data());
engine::VectorSourcePtr source_128M = std::make_shared<engine::VectorSource>(n_max, vectors_128M.data());
vector_ids.clear();
status = mem_table_file.Add(source_128M, vector_ids);
@ -145,7 +144,7 @@ TEST_F(MemManagerTest, MEM_TABLE_TEST) {
std::vector<float> vectors_100;
BuildVectors(n_100, vectors_100);
engine::VectorSource::Ptr source_100 = std::make_shared<engine::VectorSource>(n_100, vectors_100.data());
engine::VectorSourcePtr source_100 = std::make_shared<engine::VectorSource>(n_100, vectors_100.data());
engine::MemTable mem_table(TABLE_NAME, impl_, options);
@ -155,7 +154,7 @@ TEST_F(MemManagerTest, MEM_TABLE_TEST) {
vector_ids = source_100->GetVectorIds();
ASSERT_EQ(vector_ids.size(), 100);
engine::MemTableFile::Ptr mem_table_file;
engine::MemTableFilePtr mem_table_file;
mem_table.GetCurrentMemTableFile(mem_table_file);
size_t singleVectorMem = sizeof(float) * TABLE_DIM;
ASSERT_EQ(mem_table_file->GetCurrentMem(), n_100 * singleVectorMem);
@ -165,7 +164,7 @@ TEST_F(MemManagerTest, MEM_TABLE_TEST) {
BuildVectors(n_max, vectors_128M);
vector_ids.clear();
engine::VectorSource::Ptr source_128M = std::make_shared<engine::VectorSource>(n_max, vectors_128M.data());
engine::VectorSourcePtr source_128M = std::make_shared<engine::VectorSource>(n_max, vectors_128M.data());
status = mem_table.Add(source_128M, vector_ids);
ASSERT_TRUE(status.ok());
@ -181,7 +180,7 @@ TEST_F(MemManagerTest, MEM_TABLE_TEST) {
std::vector<float> vectors_1G;
BuildVectors(n_1G, vectors_1G);
engine::VectorSource::Ptr source_1G = std::make_shared<engine::VectorSource>(n_1G, vectors_1G.data());
engine::VectorSourcePtr source_1G = std::make_shared<engine::VectorSource>(n_1G, vectors_1G.data());
vector_ids.clear();
status = mem_table.Add(source_1G, vector_ids);

1
cpp/unittest/db/meta_tests.cpp
View File

@ -11,7 +11,6 @@
#include "utils.h"
#include "db/meta/SqliteMetaImpl.h"
#include "db/Factories.h"
#include "db/Utils.h"
#include "db/meta/MetaConsts.h"

1
cpp/unittest/db/mysql_db_test.cpp
View File

@ -7,7 +7,6 @@
#include "db/DB.h"
#include "db/DBImpl.h"
#include "db/meta/MetaConsts.h"
#include "db/Factories.h"
#include <gtest/gtest.h>
#include <easylogging++.h>

1
cpp/unittest/db/mysql_meta_test.cpp
View File

@ -11,7 +11,6 @@
#include "utils.h"
#include "db/meta/MySQLMetaImpl.h"
#include "db/Factories.h"
#include "db/Utils.h"
#include "db/meta/MetaConsts.h"

2
cpp/unittest/db/scheduler_test.cpp
View File

@ -99,7 +99,7 @@ TEST(DBSchedulerTest, DELETE_SCHEDULER_TEST) {
task_list.push_back(task_ptr);
}
engine::meta::Meta::Ptr meta_ptr;
engine::meta::MetaPtr meta_ptr;
engine::DeleteContextPtr context_ptr = std::make_shared<engine::DeleteContext>(table_id, meta_ptr, 0);
ret = engine::TaskDispatchStrategy::Schedule(context_ptr, task_list);
ASSERT_TRUE(ret);

12
cpp/unittest/db/utils.cpp
View File

@ -5,12 +5,11 @@
////////////////////////////////////////////////////////////////////////////////
#include <iostream>
#include <easylogging++.h>
#include <thread>
#include <boost/filesystem.hpp>
#include "utils.h"
#include "db/Factories.h"
#include "db/DBFactory.h"
#include "db/Options.h"
#include "server/ServerConfig.h"
#include "knowhere/index/vector_index/gpu_ivf.h"
@ -56,7 +55,7 @@ void BaseTest::TearDown() {
}
engine::Options BaseTest::GetOptions() {
auto options = engine::OptionsFactory::Build();
auto options = engine::DBFactory::BuildOption();
options.meta.path = "/tmp/milvus_test";
options.meta.backend_uri = "sqlite://:@:/";
return options;
@ -86,7 +85,6 @@ void DBTest::SetUp() {
void DBTest::TearDown() {
db_->Stop();
db_->DropAll();
delete db_;
BaseTest::TearDown();
@ -99,7 +97,7 @@ void DBTest::TearDown() {
/////////////////////////////////////////////////////////////////////////////////////////////////////////////////
engine::Options DBTest2::GetOptions() {
auto options = engine::OptionsFactory::Build();
auto options = engine::DBFactory::BuildOption();
options.meta.path = "/tmp/milvus_test";
options.meta.archive_conf = engine::ArchiveConf("delete", "disk:1");
options.meta.backend_uri = "sqlite://:@:/";
@ -125,7 +123,7 @@ void MetaTest::TearDown() {
/////////////////////////////////////////////////////////////////////////////////////////////////////////////////
engine::Options MySqlDBTest::GetOptions() {
auto options = engine::OptionsFactory::Build();
auto options = engine::DBFactory::BuildOption();
options.meta.path = "/tmp/milvus_test";
options.meta.backend_uri = DBTestEnvironment::getURI();
@ -154,7 +152,7 @@ void MySqlMetaTest::TearDown() {
}
zilliz::milvus::engine::Options MySqlMetaTest::GetOptions() {
auto options = engine::OptionsFactory::Build();
auto options = engine::DBFactory::BuildOption();
options.meta.path = "/tmp/milvus_test";
options.meta.backend_uri = DBTestEnvironment::getURI();

2
cpp/unittest/db/utils.h
View File

@ -44,7 +44,7 @@ protected:
/////////////////////////////////////////////////////////////////////////////////////////////////////////////////
class DBTest : public BaseTest {
protected:
zilliz::milvus::engine::DB *db_;
zilliz::milvus::engine::DBPtr db_;
virtual void SetUp() override;
virtual void TearDown() override;

1
cpp/unittest/metrics/metrics_test.cpp
View File

@ -18,7 +18,6 @@
#include "utils.h"
#include "db/DB.h"
#include "db/meta/SqliteMetaImpl.h"
#include "db/Factories.h"
using namespace zilliz::milvus;

8
cpp/unittest/metrics/utils.cpp
View File

@ -5,13 +5,11 @@
////////////////////////////////////////////////////////////////////////////////
#include <iostream>
#include <easylogging++.h>
#include <thread>
#include <boost/filesystem.hpp>
#include "utils.h"
#include "db/Factories.h"
#include "db/Options.h"
#include "db/DBFactory.h"
INITIALIZE_EASYLOGGINGPP
@ -43,7 +41,7 @@ void MetricTest::InitLog() {
}
engine::Options MetricTest::GetOptions() {
auto options = engine::OptionsFactory::Build();
auto options = engine::DBFactory::BuildOption();
options.meta.path = "/tmp/milvus_test";
options.meta.backend_uri = "sqlite://:@:/";
return options;
@ -56,7 +54,7 @@ void MetricTest::SetUp() {
}
void MetricTest::TearDown() {
delete db_;
db_->Stop();
boost::filesystem::remove_all("/tmp/milvus_test");
}

2
cpp/unittest/metrics/utils.h
View File

@ -55,7 +55,7 @@ void ASSERT_STATS(zilliz::milvus::engine::Status& stat);
class MetricTest : public ::testing::Test {
protected:
zilliz::milvus::engine::DB* db_;
zilliz::milvus::engine::DBPtr db_;
void InitLog();
virtual void SetUp() override;