milvus-io
/
milvus
mirror of https://github.com/milvus-io/milvus.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'branch-0.5.0' into 'branch-0.5.0' 

format code by clang-format

See merge request megasearch/milvus!637

Former-commit-id: 5ef398b80b258437c7edd3543bb1c0721e8a4078
pull/191/head
peng.xu 2019-09-28 10:30:37 +08:00
parent c181e405e4 58b82398cc
commit 464a341523
164 changed files with 4353 additions and 4311 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
cpp/build.sh
View File

@ -104,13 +104,13 @@ if [[ ${RUN_CPPLINT} == "ON" ]]; then
fi
echo "cpplint check passed!"
# # clang-format check
# make check-clang-format
# if [ $? -ne 0 ]; then
# echo "ERROR! clang-format check failed"
# exit 1
# fi
# echo "clang-format check passed!"
# clang-format check
make check-clang-format
if [ $? -ne 0 ]; then
echo "ERROR! clang-format check failed"
exit 1
fi
echo "clang-format check passed!"
#
# # clang-tidy check
# make check-clang-tidy

56
cpp/src/cache/Cache.h vendored
View File

@ -15,55 +15,67 @@
// specific language governing permissions and limitations
// under the License.
#pragma once
#include "LRU.h"
#include "utils/Log.h"
#include <string>
#include <mutex>
#include <atomic>
#include <mutex>
#include <set>
#include <string>
namespace zilliz {
namespace milvus {
namespace cache {
template<typename ItemObj>
template <typename ItemObj>
class Cache {
public:
//mem_capacity, units:GB
// mem_capacity, units:GB
Cache(int64_t capacity_gb, uint64_t cache_max_count);
~Cache() = default;
int64_t usage() const {
int64_t
usage() const {
return usage_;
}
int64_t capacity() const {
int64_t
capacity() const {
return capacity_;
} //unit: BYTE
void set_capacity(int64_t capacity); //unit: BYTE
} // unit: BYTE
void
set_capacity(int64_t capacity); // unit: BYTE
double freemem_percent() const {
double
freemem_percent() const {
return freemem_percent_;
}
void set_freemem_percent(double percent) {
void
set_freemem_percent(double percent) {
freemem_percent_ = percent;
}
size_t size() const;
bool exists(const std::string &key);
ItemObj get(const std::string &key);
void insert(const std::string &key, const ItemObj &item);
void erase(const std::string &key);
void print();
void clear();
size_t
size() const;
bool
exists(const std::string& key);
ItemObj
get(const std::string& key);
void
insert(const std::string& key, const ItemObj& item);
void
erase(const std::string& key);
void
print();
void
clear();
private:
void free_memory();
void
free_memory();
private:
int64_t usage_;
@ -74,8 +86,8 @@ class Cache {
mutable std::mutex mutex_;
};
} // namespace cache
} // namespace milvus
} // namespace zilliz
} // namespace cache
} // namespace milvus
} // namespace zilliz
#include "cache/Cache.inl"

43
cpp/src/cache/CacheMgr.h vendored
View File

@ -15,40 +15,49 @@
// specific language governing permissions and limitations
// under the License.
#pragma once
#include "Cache.h"
#include "utils/Log.h"
#include "metrics/Metrics.h"
#include "utils/Log.h"
#include <string>
#include <memory>
#include <string>
namespace zilliz {
namespace milvus {
namespace cache {
template<typename ItemObj>
template <typename ItemObj>
class CacheMgr {
public:
virtual uint64_t ItemCount() const;
virtual uint64_t
ItemCount() const;
virtual bool ItemExists(const std::string &key);
virtual bool
ItemExists(const std::string& key);
virtual ItemObj GetItem(const std::string &key);
virtual ItemObj
GetItem(const std::string& key);
virtual void InsertItem(const std::string &key, const ItemObj &data);
virtual void
InsertItem(const std::string& key, const ItemObj& data);
virtual void EraseItem(const std::string &key);
virtual void
EraseItem(const std::string& key);
virtual void PrintInfo();
virtual void
PrintInfo();
virtual void ClearCache();
virtual void
ClearCache();
int64_t CacheUsage() const;
int64_t CacheCapacity() const;
void SetCapacity(int64_t capacity);
int64_t
CacheUsage() const;
int64_t
CacheCapacity() const;
void
SetCapacity(int64_t capacity);
protected:
CacheMgr();
@ -59,8 +68,8 @@ class CacheMgr {
CachePtr cache_;
};
} // namespace cache
} // namespace milvus
} // namespace zilliz
} // namespace cache
} // namespace milvus
} // namespace zilliz
#include "cache/CacheMgr.inl"

17
cpp/src/cache/CpuCacheMgr.cpp vendored
View File

@ -15,7 +15,6 @@
// specific language governing permissions and limitations
// under the License.
#include "cache/CpuCacheMgr.h"
#include "server/Config.h"
#include "utils/Log.h"
@ -31,7 +30,7 @@ constexpr int64_t unit = 1024 * 1024 * 1024;
}
CpuCacheMgr::CpuCacheMgr() {
server::Config &config = server::Config::GetInstance();
server::Config& config = server::Config::GetInstance();
Status s;
int32_t cpu_cache_cap;
@ -50,19 +49,19 @@ CpuCacheMgr::CpuCacheMgr() {
if (cpu_cache_threshold > 0.0 && cpu_cache_threshold <= 1.0) {
cache_->set_freemem_percent(cpu_cache_threshold);
} else {
SERVER_LOG_ERROR << "Invalid cpu_cache_threshold: " << cpu_cache_threshold
<< ", by default set to " << cache_->freemem_percent();
SERVER_LOG_ERROR << "Invalid cpu_cache_threshold: " << cpu_cache_threshold << ", by default set to "
<< cache_->freemem_percent();
}
}
CpuCacheMgr *
CpuCacheMgr*
CpuCacheMgr::GetInstance() {
static CpuCacheMgr s_mgr;
return &s_mgr;
}
engine::VecIndexPtr
CpuCacheMgr::GetIndex(const std::string &key) {
CpuCacheMgr::GetIndex(const std::string& key) {
DataObjPtr obj = GetItem(key);
if (obj != nullptr) {
return obj->data();
@ -71,6 +70,6 @@ CpuCacheMgr::GetIndex(const std::string &key) {
return nullptr;
}
} // namespace cache
} // namespace milvus
} // namespace zilliz
} // namespace cache
} // namespace milvus
} // namespace zilliz

16
cpp/src/cache/CpuCacheMgr.h vendored
View File

@ -20,8 +20,8 @@
#include "CacheMgr.h"
#include "DataObj.h"
#include <string>
#include <memory>
#include <string>
namespace zilliz {
namespace milvus {
@ -32,12 +32,14 @@ class CpuCacheMgr : public CacheMgr<DataObjPtr> {
CpuCacheMgr();
public:
//TODO: use smart pointer instead
static CpuCacheMgr *GetInstance();
// TODO: use smart pointer instead
static CpuCacheMgr*
GetInstance();
engine::VecIndexPtr GetIndex(const std::string &key);
engine::VecIndexPtr
GetIndex(const std::string& key);
};
} // namespace cache
} // namespace milvus
} // namespace zilliz
} // namespace cache
} // namespace milvus
} // namespace zilliz

23
cpp/src/cache/DataObj.h vendored
View File

@ -15,7 +15,6 @@
// specific language governing permissions and limitations
// under the License.
#pragma once
#include "src/wrapper/VecIndex.h"
@ -28,24 +27,24 @@ namespace cache {
class DataObj {
public:
explicit DataObj(const engine::VecIndexPtr &index)
: index_(index) {
explicit DataObj(const engine::VecIndexPtr& index) : index_(index) {
}
DataObj(const engine::VecIndexPtr &index, int64_t size)
: index_(index),
size_(size) {
DataObj(const engine::VecIndexPtr& index, int64_t size) : index_(index), size_(size) {
}
engine::VecIndexPtr data() {
engine::VecIndexPtr
data() {
return index_;
}
const engine::VecIndexPtr &data() const {
const engine::VecIndexPtr&
data() const {
return index_;
}
int64_t size() const {
int64_t
size() const {
if (index_ == nullptr) {
return 0;
}
@ -64,6 +63,6 @@ class DataObj {
using DataObjPtr = std::shared_ptr<DataObj>;
} // namespace cache
} // namespace milvus
} // namespace zilliz
} // namespace cache
} // namespace milvus
} // namespace zilliz

19
cpp/src/cache/GpuCacheMgr.cpp vendored
View File

@ -15,10 +15,9 @@
// specific language governing permissions and limitations
// under the License.
#include "cache/GpuCacheMgr.h"
#include "utils/Log.h"
#include "server/Config.h"
#include "utils/Log.h"
#include <sstream>
#include <utility>
@ -35,7 +34,7 @@ constexpr int64_t G_BYTE = 1024 * 1024 * 1024;
}
GpuCacheMgr::GpuCacheMgr() {
server::Config &config = server::Config::GetInstance();
server::Config& config = server::Config::GetInstance();
Status s;
int32_t gpu_cache_cap;
@ -54,12 +53,12 @@ GpuCacheMgr::GpuCacheMgr() {
if (gpu_mem_threshold > 0.0 && gpu_mem_threshold <= 1.0) {
cache_->set_freemem_percent(gpu_mem_threshold);
} else {
SERVER_LOG_ERROR << "Invalid gpu_mem_threshold: " << gpu_mem_threshold
<< ", by default set to " << cache_->freemem_percent();
SERVER_LOG_ERROR << "Invalid gpu_mem_threshold: " << gpu_mem_threshold << ", by default set to "
<< cache_->freemem_percent();
}
}
GpuCacheMgr *
GpuCacheMgr*
GpuCacheMgr::GetInstance(uint64_t gpu_id) {
if (instance_.find(gpu_id) == instance_.end()) {
std::lock_guard<std::mutex> lock(mutex_);
@ -74,7 +73,7 @@ GpuCacheMgr::GetInstance(uint64_t gpu_id) {
}
engine::VecIndexPtr
GpuCacheMgr::GetIndex(const std::string &key) {
GpuCacheMgr::GetIndex(const std::string& key) {
DataObjPtr obj = GetItem(key);
if (obj != nullptr) {
return obj->data();
@ -83,6 +82,6 @@ GpuCacheMgr::GetIndex(const std::string &key) {
return nullptr;
}
} // namespace cache
} // namespace milvus
} // namespace zilliz
} // namespace cache
} // namespace milvus
} // namespace zilliz

15
cpp/src/cache/GpuCacheMgr.h vendored
View File

@ -15,13 +15,12 @@
// specific language governing permissions and limitations
// under the License.
#include "CacheMgr.h"
#include "DataObj.h"
#include <unordered_map>
#include <memory>
#include <string>
#include <unordered_map>
namespace zilliz {
namespace milvus {
@ -34,15 +33,17 @@ class GpuCacheMgr : public CacheMgr<DataObjPtr> {
public:
GpuCacheMgr();
static GpuCacheMgr *GetInstance(uint64_t gpu_id);
static GpuCacheMgr*
GetInstance(uint64_t gpu_id);
engine::VecIndexPtr GetIndex(const std::string &key);
engine::VecIndexPtr
GetIndex(const std::string& key);
private:
static std::mutex mutex_;
static std::unordered_map<uint64_t, GpuCacheMgrPtr> instance_;
};
} // namespace cache
} // namespace milvus
} // namespace zilliz
} // namespace cache
} // namespace milvus
} // namespace zilliz

44
cpp/src/cache/LRU.h vendored
View File

@ -15,20 +15,19 @@
// specific language governing permissions and limitations
// under the License.
#pragma once
#include <unordered_map>
#include <list>
#include <cstddef>
#include <list>
#include <stdexcept>
#include <unordered_map>
#include <utility>
namespace zilliz {
namespace milvus {
namespace cache {
template<typename key_t, typename value_t>
template <typename key_t, typename value_t>
class LRU {
public:
typedef typename std::pair<key_t, value_t> key_value_pair_t;
@ -38,7 +37,8 @@ class LRU {
explicit LRU(size_t max_size) : max_size_(max_size) {
}
void put(const key_t &key, const value_t &value) {
void
put(const key_t& key, const value_t& value) {
auto it = cache_items_map_.find(key);
cache_items_list_.push_front(key_value_pair_t(key, value));
if (it != cache_items_map_.end()) {
@ -55,7 +55,8 @@ class LRU {
}
}
const value_t &get(const key_t &key) {
const value_t&
get(const key_t& key) {
auto it = cache_items_map_.find(key);
if (it == cache_items_map_.end()) {
throw std::range_error("There is no such key in cache");
@ -65,7 +66,8 @@ class LRU {
}
}
void erase(const key_t &key) {
void
erase(const key_t& key) {
auto it = cache_items_map_.find(key);
if (it != cache_items_map_.end()) {
cache_items_list_.erase(it->second);
@ -73,32 +75,39 @@ class LRU {
}
}
bool exists(const key_t &key) const {
bool
exists(const key_t& key) const {
return cache_items_map_.find(key) != cache_items_map_.end();
}
size_t size() const {
size_t
size() const {
return cache_items_map_.size();
}
list_iterator_t begin() {
list_iterator_t
begin() {
iter_ = cache_items_list_.begin();
return iter_;
}
list_iterator_t end() {
list_iterator_t
end() {
return cache_items_list_.end();
}
reverse_list_iterator_t rbegin() {
reverse_list_iterator_t
rbegin() {
return cache_items_list_.rbegin();
}
reverse_list_iterator_t rend() {
reverse_list_iterator_t
rend() {
return cache_items_list_.rend();
}
void clear() {
void
clear() {
cache_items_list_.clear();
cache_items_map_.clear();
}
@ -110,7 +119,6 @@ class LRU {
list_iterator_t iter_;
};
} // namespace cache
} // namespace milvus
} // namespace zilliz
} // namespace cache
} // namespace milvus
} // namespace zilliz

8
cpp/src/config/ConfigMgr.cpp
View File

@ -22,12 +22,12 @@ namespace zilliz {
namespace milvus {
namespace server {
ConfigMgr *
ConfigMgr*
ConfigMgr::GetInstance() {
static YamlConfigMgr mgr;
return &mgr;
}
} // namespace server
} // namespace milvus
} // namespace zilliz
} // namespace server
} // namespace milvus
} // namespace zilliz

26
cpp/src/config/ConfigMgr.h
View File

@ -17,8 +17,8 @@
#pragma once
#include "utils/Error.h"
#include "ConfigNode.h"
#include "utils/Error.h"
#include <string>
@ -42,16 +42,22 @@ namespace server {
class ConfigMgr {
public:
static ConfigMgr *GetInstance();
static ConfigMgr*
GetInstance();
virtual ErrorCode LoadConfigFile(const std::string &filename) = 0;
virtual void Print() const = 0;//will be deleted
virtual std::string DumpString() const = 0;
virtual ErrorCode
LoadConfigFile(const std::string& filename) = 0;
virtual void
Print() const = 0; // will be deleted
virtual std::string
DumpString() const = 0;
virtual const ConfigNode &GetRootNode() const = 0;
virtual ConfigNode &GetRootNode() = 0;
virtual const ConfigNode&
GetRootNode() const = 0;
virtual ConfigNode&
GetRootNode() = 0;
};
} // namespace server
} // namespace milvus
} // namespace zilliz
} // namespace server
} // namespace milvus
} // namespace zilliz

80
cpp/src/config/ConfigNode.cpp
View File

@ -19,51 +19,51 @@
#include "utils/Error.h"
#include "utils/Log.h"
#include <algorithm>
#include <sstream>
#include <string>
#include <algorithm>
namespace zilliz {
namespace milvus {
namespace server {
void
ConfigNode::Combine(const ConfigNode &target) {
const std::map<std::string, std::string> &kv = target.GetConfig();
ConfigNode::Combine(const ConfigNode& target) {
const std::map<std::string, std::string>& kv = target.GetConfig();
for (auto itr = kv.begin(); itr != kv.end(); ++itr) {
config_[itr->first] = itr->second;
}
const std::map<std::string, std::vector<std::string> > &sequences = target.GetSequences();
const std::map<std::string, std::vector<std::string> >& sequences = target.GetSequences();
for (auto itr = sequences.begin(); itr != sequences.end(); ++itr) {
sequences_[itr->first] = itr->second;
}
const std::map<std::string, ConfigNode> &children = target.GetChildren();
const std::map<std::string, ConfigNode>& children = target.GetChildren();
for (auto itr = children.begin(); itr != children.end(); ++itr) {
children_[itr->first] = itr->second;
}
}
//key/value pair config
// key/value pair config
void
ConfigNode::SetValue(const std::string &key, const std::string &value) {
ConfigNode::SetValue(const std::string& key, const std::string& value) {
config_[key] = value;
}
std::string
ConfigNode::GetValue(const std::string &param_key, const std::string &default_val) const {
ConfigNode::GetValue(const std::string& param_key, const std::string& default_val) const {
auto ref = config_.find(param_key);
if (ref != config_.end()) {
return ref->second;
}
//THROW_UNEXPECTED_ERROR("Can't find parameter key: " + param_key);
// THROW_UNEXPECTED_ERROR("Can't find parameter key: " + param_key);
return default_val;
}
bool
ConfigNode::GetBoolValue(const std::string &param_key, bool default_val) const {
ConfigNode::GetBoolValue(const std::string& param_key, bool default_val) const {
std::string val = GetValue(param_key);
if (!val.empty()) {
std::transform(val.begin(), val.end(), val.begin(), ::tolower);
@ -74,17 +74,17 @@ ConfigNode::GetBoolValue(const std::string &param_key, bool default_val) const {
}
int32_t
ConfigNode::GetInt32Value(const std::string &param_key, int32_t default_val) const {
ConfigNode::GetInt32Value(const std::string& param_key, int32_t default_val) const {
std::string val = GetValue(param_key);
if (!val.empty()) {
return (int32_t) std::strtol(val.c_str(), nullptr, 10);
return (int32_t)std::strtol(val.c_str(), nullptr, 10);
} else {
return default_val;
}
}
int64_t
ConfigNode::GetInt64Value(const std::string &param_key, int64_t default_val) const {
ConfigNode::GetInt64Value(const std::string& param_key, int64_t default_val) const {
std::string val = GetValue(param_key);
if (!val.empty()) {
return std::strtol(val.c_str(), nullptr, 10);
@ -94,7 +94,7 @@ ConfigNode::GetInt64Value(const std::string &param_key, int64_t default_val) con
}
float
ConfigNode::GetFloatValue(const std::string &param_key, float default_val) const {
ConfigNode::GetFloatValue(const std::string& param_key, float default_val) const {
std::string val = GetValue(param_key);
if (!val.empty()) {
return std::strtof(val.c_str(), nullptr);
@ -104,7 +104,7 @@ ConfigNode::GetFloatValue(const std::string &param_key, float default_val) const
}
double
ConfigNode::GetDoubleValue(const std::string &param_key, double default_val) const {
ConfigNode::GetDoubleValue(const std::string& param_key, double default_val) const {
std::string val = GetValue(param_key);
if (!val.empty()) {
return std::strtod(val.c_str(), nullptr);
@ -113,7 +113,7 @@ ConfigNode::GetDoubleValue(const std::string &param_key, double default_val) con
}
}
const std::map<std::string, std::string> &
const std::map<std::string, std::string>&
ConfigNode::GetConfig() const {
return config_;
}
@ -123,14 +123,14 @@ ConfigNode::ClearConfig() {
config_.clear();
}
//key/object config
// key/object config
void
ConfigNode::AddChild(const std::string &type_name, const ConfigNode &config) {
ConfigNode::AddChild(const std::string& type_name, const ConfigNode& config) {
children_[type_name] = config;
}
ConfigNode
ConfigNode::GetChild(const std::string &type_name) const {
ConfigNode::GetChild(const std::string& type_name) const {
auto ref = children_.find(type_name);
if (ref != children_.end()) {
return ref->second;
@ -140,20 +140,20 @@ ConfigNode::GetChild(const std::string &type_name) const {
return nc;
}
ConfigNode &
ConfigNode::GetChild(const std::string &type_name) {
ConfigNode&
ConfigNode::GetChild(const std::string& type_name) {
return children_[type_name];
}
void
ConfigNode::GetChildren(ConfigNodeArr &arr) const {
ConfigNode::GetChildren(ConfigNodeArr& arr) const {
arr.clear();
for (auto ref : children_) {
arr.push_back(ref.second);
}
}
const std::map<std::string, ConfigNode> &
const std::map<std::string, ConfigNode>&
ConfigNode::GetChildren() const {
return children_;
}
@ -163,14 +163,14 @@ ConfigNode::ClearChildren() {
children_.clear();
}
//key/sequence config
// key/sequence config
void
ConfigNode::AddSequenceItem(const std::string &key, const std::string &item) {
ConfigNode::AddSequenceItem(const std::string& key, const std::string& item) {
sequences_[key].push_back(item);
}
std::vector<std::string>
ConfigNode::GetSequence(const std::string &key) const {
ConfigNode::GetSequence(const std::string& key) const {
auto itr = sequences_.find(key);
if (itr != sequences_.end()) {
return itr->second;
@ -180,7 +180,7 @@ ConfigNode::GetSequence(const std::string &key) const {
}
}
const std::map<std::string, std::vector<std::string> > &
const std::map<std::string, std::vector<std::string> >&
ConfigNode::GetSequences() const {
return sequences_;
}
@ -191,40 +191,40 @@ ConfigNode::ClearSequences() {
}
void
ConfigNode::PrintAll(const std::string &prefix) const {
for (auto &elem : config_) {
ConfigNode::PrintAll(const std::string& prefix) const {
for (auto& elem : config_) {
SERVER_LOG_INFO << prefix << elem.first + ": " << elem.second;
}
for (auto &elem : sequences_) {
for (auto& elem : sequences_) {
SERVER_LOG_INFO << prefix << elem.first << ": ";
for (auto &str : elem.second) {
for (auto& str : elem.second) {
SERVER_LOG_INFO << prefix << " - " << str;
}
}
for (auto &elem : children_) {
for (auto& elem : children_) {
SERVER_LOG_INFO << prefix << elem.first << ": ";
elem.second.PrintAll(prefix + " ");
}
}
std::string
ConfigNode::DumpString(const std::string &prefix) const {
ConfigNode::DumpString(const std::string& prefix) const {
std::stringstream str_buffer;
const std::string endl = "\n";
for (auto &elem : config_) {
for (auto& elem : config_) {
str_buffer << prefix << elem.first << ": " << elem.second << endl;
}
for (auto &elem : sequences_) {
for (auto& elem : sequences_) {
str_buffer << prefix << elem.first << ": " << endl;
for (auto &str : elem.second) {
for (auto& str : elem.second) {
str_buffer << prefix + " - " << str << endl;
}
}
for (auto &elem : children_) {
for (auto& elem : children_) {
str_buffer << prefix << elem.first << ": " << endl;
str_buffer << elem.second.DumpString(prefix + " ") << endl;
}
@ -232,6 +232,6 @@ ConfigNode::DumpString(const std::string &prefix) const {
return str_buffer.str();
}
} // namespace server
} // namespace milvus
} // namespace zilliz
} // namespace server
} // namespace milvus
} // namespace zilliz

82
cpp/src/config/ConfigNode.h
View File

@ -17,9 +17,9 @@
#pragma once
#include <vector>
#include <string>
#include <map>
#include <string>
#include <vector>
namespace zilliz {
namespace milvus {
@ -30,39 +30,61 @@ typedef std::vector<ConfigNode> ConfigNodeArr;
class ConfigNode {
public:
void Combine(const ConfigNode &target);
void
Combine(const ConfigNode& target);
//key/value pair config
void SetValue(const std::string &key, const std::string &value);
// key/value pair config
void
SetValue(const std::string& key, const std::string& value);
std::string GetValue(const std::string &param_key, const std::string &default_val = "") const;
bool GetBoolValue(const std::string &param_key, bool default_val = false) const;
int32_t GetInt32Value(const std::string &param_key, int32_t default_val = 0) const;
int64_t GetInt64Value(const std::string &param_key, int64_t default_val = 0) const;
float GetFloatValue(const std::string &param_key, float default_val = 0.0) const;
double GetDoubleValue(const std::string &param_key, double default_val = 0.0) const;
std::string
GetValue(const std::string& param_key, const std::string& default_val = "") const;
bool
GetBoolValue(const std::string& param_key, bool default_val = false) const;
int32_t
GetInt32Value(const std::string& param_key, int32_t default_val = 0) const;
int64_t
GetInt64Value(const std::string& param_key, int64_t default_val = 0) const;
float
GetFloatValue(const std::string& param_key, float default_val = 0.0) const;
double
GetDoubleValue(const std::string& param_key, double default_val = 0.0) const;
const std::map<std::string, std::string> &GetConfig() const;
void ClearConfig();
const std::map<std::string, std::string>&
GetConfig() const;
void
ClearConfig();
//key/object config
void AddChild(const std::string &type_name, const ConfigNode &config);
ConfigNode GetChild(const std::string &type_name) const;
ConfigNode &GetChild(const std::string &type_name);
void GetChildren(ConfigNodeArr &arr) const;
// key/object config
void
AddChild(const std::string& type_name, const ConfigNode& config);
ConfigNode
GetChild(const std::string& type_name) const;
ConfigNode&
GetChild(const std::string& type_name);
void
GetChildren(ConfigNodeArr& arr) const;
const std::map<std::string, ConfigNode> &GetChildren() const;
void ClearChildren();
const std::map<std::string, ConfigNode>&
GetChildren() const;
void
ClearChildren();
//key/sequence config
void AddSequenceItem(const std::string &key, const std::string &item);
std::vector<std::string> GetSequence(const std::string &key) const;
// key/sequence config
void
AddSequenceItem(const std::string& key, const std::string& item);
std::vector<std::string>
GetSequence(const std::string& key) const;
const std::map<std::string, std::vector<std::string> > &GetSequences() const;
void ClearSequences();
const std::map<std::string, std::vector<std::string> >&
GetSequences() const;
void
ClearSequences();
void PrintAll(const std::string &prefix = "") const;
std::string DumpString(const std::string &prefix = "") const;
void
PrintAll(const std::string& prefix = "") const;
std::string
DumpString(const std::string& prefix = "") const;
private:
std::map<std::string, std::string> config_;
@ -70,6 +92,6 @@ class ConfigNode {
std::map<std::string, std::vector<std::string> > sequences_;
};
} // namespace server
} // namespace milvus
} // namespace zilliz
} // namespace server
} // namespace milvus
} // namespace zilliz

29
cpp/src/config/YamlConfigMgr.cpp
View File

@ -25,7 +25,7 @@ namespace milvus {
namespace server {
ErrorCode
YamlConfigMgr::LoadConfigFile(const std::string &filename) {
YamlConfigMgr::LoadConfigFile(const std::string& filename) {
struct stat directoryStat;
int statOK = stat(filename.c_str(), &directoryStat);
if (statOK != 0) {
@ -36,8 +36,7 @@ YamlConfigMgr::LoadConfigFile(const std::string &filename) {
try {
node_ = YAML::LoadFile(filename);
LoadConfigNode(node_, config_);
}
catch (YAML::Exception &e) {
} catch (YAML::Exception& e) {
SERVER_LOG_ERROR << "Failed to load config file: " << std::string(e.what());
return SERVER_UNEXPECTED_ERROR;
}
@ -56,20 +55,18 @@ YamlConfigMgr::DumpString() const {
return config_.DumpString("");
}
const ConfigNode &
const ConfigNode&
YamlConfigMgr::GetRootNode() const {
return config_;
}
ConfigNode &
ConfigNode&
YamlConfigMgr::GetRootNode() {
return config_;
}
bool
YamlConfigMgr::SetConfigValue(const YAML::Node &node,
const std::string &key,
ConfigNode &config) {
YamlConfigMgr::SetConfigValue(const YAML::Node& node, const std::string& key, ConfigNode& config) {
if (node[key].IsDefined()) {
config.SetValue(key, node[key].as<std::string>());
return true;
@ -78,9 +75,7 @@ YamlConfigMgr::SetConfigValue(const YAML::Node &node,
}
bool
YamlConfigMgr::SetChildConfig(const YAML::Node &node,
const std::string &child_name,
ConfigNode &config) {
YamlConfigMgr::SetChildConfig(const YAML::Node& node, const std::string& child_name, ConfigNode& config) {
if (node[child_name].IsDefined()) {
ConfigNode sub_config;
LoadConfigNode(node[child_name], sub_config);
@ -91,9 +86,7 @@ YamlConfigMgr::SetChildConfig(const YAML::Node &node,
}
bool
YamlConfigMgr::SetSequence(const YAML::Node &node,
const std::string &child_name,
ConfigNode &config) {
YamlConfigMgr::SetSequence(const YAML::Node& node, const std::string& child_name, ConfigNode& config) {
if (node[child_name].IsDefined()) {
size_t cnt = node[child_name].size();
for (size_t i = 0; i < cnt; i++) {
@ -105,7 +98,7 @@ YamlConfigMgr::SetSequence(const YAML::Node &node,
}
void
YamlConfigMgr::LoadConfigNode(const YAML::Node &node, ConfigNode &config) {
YamlConfigMgr::LoadConfigNode(const YAML::Node& node, ConfigNode& config) {
std::string key;
for (YAML::const_iterator it = node.begin(); it != node.end(); ++it) {
if (!it->first.IsNull()) {
@ -121,6 +114,6 @@ YamlConfigMgr::LoadConfigNode(const YAML::Node &node, ConfigNode &config) {
}
}
} // namespace server
} // namespace milvus
} // namespace zilliz
} // namespace server
} // namespace milvus
} // namespace zilliz

42
cpp/src/config/YamlConfigMgr.h
View File

@ -21,8 +21,8 @@
#include "ConfigNode.h"
#include "utils/Error.h"
#include <string>
#include <yaml-cpp/yaml.h>
#include <string>
namespace zilliz {
namespace milvus {
@ -30,34 +30,36 @@ namespace server {
class YamlConfigMgr : public ConfigMgr {
public:
virtual ErrorCode LoadConfigFile(const std::string &filename);
virtual void Print() const;
virtual std::string DumpString() const;
virtual ErrorCode
LoadConfigFile(const std::string& filename);
virtual void
Print() const;
virtual std::string
DumpString() const;
virtual const ConfigNode &GetRootNode() const;
virtual ConfigNode &GetRootNode();
virtual const ConfigNode&
GetRootNode() const;
virtual ConfigNode&
GetRootNode();
private:
bool SetConfigValue(const YAML::Node &node,
const std::string &key,
ConfigNode &config);
bool SetChildConfig(const YAML::Node &node,
const std::string &name,
ConfigNode &config);
bool
SetConfigValue(const YAML::Node& node, const std::string& key, ConfigNode& config);
bool
SetSequence(const YAML::Node &node,
const std::string &child_name,
ConfigNode &config);
SetChildConfig(const YAML::Node& node, const std::string& name, ConfigNode& config);
void LoadConfigNode(const YAML::Node &node, ConfigNode &config);
bool
SetSequence(const YAML::Node& node, const std::string& child_name, ConfigNode& config);
void
LoadConfigNode(const YAML::Node& node, ConfigNode& config);
private:
YAML::Node node_;
ConfigNode config_;
};
} // namespace server
} // namespace milvus
} // namespace zilliz
} // namespace server
} // namespace milvus
} // namespace zilliz

6
cpp/src/db/Constants.h
View File

@ -36,6 +36,6 @@ static constexpr uint64_t ONE_KB = K;
static constexpr uint64_t ONE_MB = ONE_KB * ONE_KB;
static constexpr uint64_t ONE_GB = ONE_KB * ONE_MB;
} // namespace engine
} // namespace milvus
} // namespace zilliz
} // namespace engine
} // namespace milvus
} // namespace zilliz

81
cpp/src/db/DB.h
View File

@ -15,7 +15,6 @@
// specific language governing permissions and limitations
// under the License.
#pragma once
#include "Options.h"
@ -23,8 +22,8 @@
#include "meta/Meta.h"
#include "utils/Status.h"
#include <string>
#include <memory>
#include <string>
#include <vector>
namespace zilliz {
@ -36,47 +35,65 @@ class Env;
class DB {
public:
DB() = default;
DB(const DB &) = delete;
DB &operator=(const DB &) = delete;
DB(const DB&) = delete;
DB&
operator=(const DB&) = delete;
virtual ~DB() = default;
virtual Status Start() = 0;
virtual Status Stop() = 0;
virtual Status
Start() = 0;
virtual Status
Stop() = 0;
virtual Status CreateTable(meta::TableSchema &table_schema_) = 0;
virtual Status DeleteTable(const std::string &table_id, const meta::DatesT &dates) = 0;
virtual Status DescribeTable(meta::TableSchema &table_schema_) = 0;
virtual Status HasTable(const std::string &table_id, bool &has_or_not_) = 0;
virtual Status AllTables(std::vector<meta::TableSchema> &table_schema_array) = 0;
virtual Status GetTableRowCount(const std::string &table_id, uint64_t &row_count) = 0;
virtual Status PreloadTable(const std::string &table_id) = 0;
virtual Status UpdateTableFlag(const std::string &table_id, int64_t flag) = 0;
virtual Status
CreateTable(meta::TableSchema& table_schema_) = 0;
virtual Status
DeleteTable(const std::string& table_id, const meta::DatesT& dates) = 0;
virtual Status
DescribeTable(meta::TableSchema& table_schema_) = 0;
virtual Status
HasTable(const std::string& table_id, bool& has_or_not_) = 0;
virtual Status
AllTables(std::vector<meta::TableSchema>& table_schema_array) = 0;
virtual Status
GetTableRowCount(const std::string& table_id, uint64_t& row_count) = 0;
virtual Status
PreloadTable(const std::string& table_id) = 0;
virtual Status
UpdateTableFlag(const std::string& table_id, int64_t flag) = 0;
virtual Status InsertVectors(const std::string &table_id_,
uint64_t n, const float *vectors, IDNumbers &vector_ids_) = 0;
virtual Status
InsertVectors(const std::string& table_id_, uint64_t n, const float* vectors, IDNumbers& vector_ids_) = 0;
virtual Status Query(const std::string &table_id, uint64_t k, uint64_t nq, uint64_t nprobe,
const float *vectors, QueryResults &results) = 0;
virtual Status
Query(const std::string& table_id, uint64_t k, uint64_t nq, uint64_t nprobe, const float* vectors,
QueryResults& results) = 0;
virtual Status Query(const std::string &table_id, uint64_t k, uint64_t nq, uint64_t nprobe,
const float *vectors, const meta::DatesT &dates, QueryResults &results) = 0;
virtual Status
Query(const std::string& table_id, uint64_t k, uint64_t nq, uint64_t nprobe, const float* vectors,
const meta::DatesT& dates, QueryResults& results) = 0;
virtual Status Query(const std::string &table_id, const std::vector<std::string> &file_ids,
uint64_t k, uint64_t nq, uint64_t nprobe, const float *vectors,
const meta::DatesT &dates, QueryResults &results) = 0;
virtual Status
Query(const std::string& table_id, const std::vector<std::string>& file_ids, uint64_t k, uint64_t nq,
uint64_t nprobe, const float* vectors, const meta::DatesT& dates, QueryResults& results) = 0;
virtual Status Size(uint64_t &result) = 0;
virtual Status
Size(uint64_t& result) = 0;
virtual Status CreateIndex(const std::string &table_id, const TableIndex &index) = 0;
virtual Status DescribeIndex(const std::string &table_id, TableIndex &index) = 0;
virtual Status DropIndex(const std::string &table_id) = 0;
virtual Status
CreateIndex(const std::string& table_id, const TableIndex& index) = 0;
virtual Status
DescribeIndex(const std::string& table_id, TableIndex& index) = 0;
virtual Status
DropIndex(const std::string& table_id) = 0;
virtual Status DropAll() = 0;
}; // DB
virtual Status
DropAll() = 0;
}; // DB
using DBPtr = std::shared_ptr<DB>;
} // namespace engine
} // namespace milvus
} // namespace zilliz
} // namespace engine
} // namespace milvus
} // namespace zilliz

15
cpp/src/db/DBFactory.cpp
View File

@ -15,18 +15,17 @@
// specific language governing permissions and limitations
// under the License.
#include "db/DBFactory.h"
#include "DBImpl.h"
#include "utils/Exception.h"
#include "meta/MetaFactory.h"
#include "meta/SqliteMetaImpl.h"
#include "meta/MySQLMetaImpl.h"
#include "meta/SqliteMetaImpl.h"
#include "utils/Exception.h"
#include <stdlib.h>
#include <time.h>
#include <sstream>
#include <cstdlib>
#include <sstream>
#include <string>
namespace zilliz {
@ -42,10 +41,10 @@ DBFactory::BuildOption() {
}
DBPtr
DBFactory::Build(const DBOptions &options) {
DBFactory::Build(const DBOptions& options) {
return std::make_shared<DBImpl>(options);
}
} // namespace engine
} // namespace milvus
} // namespace zilliz
} // namespace engine
} // namespace milvus
} // namespace zilliz

14
cpp/src/db/DBFactory.h
View File

@ -20,8 +20,8 @@
#include "DB.h"
#include "Options.h"
#include <string>
#include <memory>
#include <string>
namespace zilliz {
namespace milvus {
@ -29,11 +29,13 @@ namespace engine {
class DBFactory {
public:
static DBOptions BuildOption();
static DBOptions
BuildOption();
static DBPtr Build(const DBOptions &options);
static DBPtr
Build(const DBOptions& options);
};
} // namespace engine
} // namespace milvus
} // namespace zilliz
} // namespace engine
} // namespace milvus
} // namespace zilliz

347
cpp/src/db/DBImpl.cpp
View File

@ -16,28 +16,28 @@
// under the License.
#include "db/DBImpl.h"
#include "Utils.h"
#include "cache/CpuCacheMgr.h"
#include "cache/GpuCacheMgr.h"
#include "engine/EngineFactory.h"
#include "insert/MemMenagerFactory.h"
#include "meta/SqliteMetaImpl.h"
#include "meta/MetaFactory.h"
#include "meta/MetaConsts.h"
#include "meta/MetaFactory.h"
#include "meta/SqliteMetaImpl.h"
#include "metrics/Metrics.h"
#include "scheduler/job/SearchJob.h"
#include "scheduler/job/DeleteJob.h"
#include "scheduler/SchedInst.h"
#include "utils/TimeRecorder.h"
#include "scheduler/job/DeleteJob.h"
#include "scheduler/job/SearchJob.h"
#include "utils/Log.h"
#include "Utils.h"
#include "utils/TimeRecorder.h"
#include <assert.h>
#include <chrono>
#include <thread>
#include <iostream>
#include <cstring>
#include <algorithm>
#include <boost/filesystem.hpp>
#include <chrono>
#include <cstring>
#include <iostream>
#include <thread>
namespace zilliz {
namespace milvus {
@ -49,13 +49,10 @@ constexpr uint64_t METRIC_ACTION_INTERVAL = 1;
constexpr uint64_t COMPACT_ACTION_INTERVAL = 1;
constexpr uint64_t INDEX_ACTION_INTERVAL = 1;
} // namespace
} // namespace
DBImpl::DBImpl(const DBOptions &options)
: options_(options),
shutting_down_(true),
compact_thread_pool_(1, 1),
index_thread_pool_(1, 1) {
DBImpl::DBImpl(const DBOptions& options)
: options_(options), shutting_down_(true), compact_thread_pool_(1, 1), index_thread_pool_(1, 1) {
meta_ptr_ = MetaFactory::Build(options.meta_, options.mode_);
mem_mgr_ = MemManagerFactory::Build(meta_ptr_, options_);
Start();
@ -66,7 +63,7 @@ DBImpl::~DBImpl() {
}
///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
//external api
// external api
///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
Status
DBImpl::Start() {
@ -77,7 +74,7 @@ DBImpl::Start() {
ENGINE_LOG_TRACE << "DB service start";
shutting_down_.store(false, std::memory_order_release);
//for distribute version, some nodes are read only
// for distribute version, some nodes are read only
if (options_.mode_ != DBOptions::MODE::CLUSTER_READONLY) {
ENGINE_LOG_TRACE << "StartTimerTasks";
bg_timer_thread_ = std::thread(&DBImpl::BackgroundTimerTask, this);
@ -94,10 +91,10 @@ DBImpl::Stop() {
shutting_down_.store(true, std::memory_order_release);
//makesure all memory data serialized
// makesure all memory data serialized
MemSerialize();
//wait compaction/buildindex finish
// wait compaction/buildindex finish
bg_timer_thread_.join();
if (options_.mode_ != DBOptions::MODE::CLUSTER_READONLY) {
@ -114,30 +111,30 @@ DBImpl::DropAll() {
}
Status
DBImpl::CreateTable(meta::TableSchema &table_schema) {
DBImpl::CreateTable(meta::TableSchema& table_schema) {
if (shutting_down_.load(std::memory_order_acquire)) {
return Status(DB_ERROR, "Milsvus server is shutdown!");
}
meta::TableSchema temp_schema = table_schema;
temp_schema.index_file_size_ *= ONE_MB; //store as MB
temp_schema.index_file_size_ *= ONE_MB; // store as MB
return meta_ptr_->CreateTable(temp_schema);
}
Status
DBImpl::DeleteTable(const std::string &table_id, const meta::DatesT &dates) {
DBImpl::DeleteTable(const std::string& table_id, const meta::DatesT& dates) {
if (shutting_down_.load(std::memory_order_acquire)) {
return Status(DB_ERROR, "Milsvus server is shutdown!");
}
//dates partly delete files of the table but currently we don't support
// dates partly delete files of the table but currently we don't support
ENGINE_LOG_DEBUG << "Prepare to delete table " << table_id;
if (dates.empty()) {
mem_mgr_->EraseMemVector(table_id); //not allow insert
meta_ptr_->DeleteTable(table_id); //soft delete table
mem_mgr_->EraseMemVector(table_id); // not allow insert
meta_ptr_->DeleteTable(table_id); // soft delete table
//scheduler will determine when to delete table files
// scheduler will determine when to delete table files
auto nres = scheduler::ResMgrInst::GetInstance()->GetNumOfComputeResource();
scheduler::DeleteJobPtr job = std::make_shared<scheduler::DeleteJob>(0, table_id, meta_ptr_, nres);
scheduler::JobMgrInst::GetInstance()->Put(job);
@ -150,18 +147,18 @@ DBImpl::DeleteTable(const std::string &table_id, const meta::DatesT &dates) {
}
Status
DBImpl::DescribeTable(meta::TableSchema &table_schema) {
DBImpl::DescribeTable(meta::TableSchema& table_schema) {
if (shutting_down_.load(std::memory_order_acquire)) {
return Status(DB_ERROR, "Milsvus server is shutdown!");
}
auto stat = meta_ptr_->DescribeTable(table_schema);
table_schema.index_file_size_ /= ONE_MB; //return as MB
table_schema.index_file_size_ /= ONE_MB; // return as MB
return stat;
}
Status
DBImpl::HasTable(const std::string &table_id, bool &has_or_not) {
DBImpl::HasTable(const std::string& table_id, bool& has_or_not) {
if (shutting_down_.load(std::memory_order_acquire)) {
return Status(DB_ERROR, "Milsvus server is shutdown!");
}
@ -170,7 +167,7 @@ DBImpl::HasTable(const std::string &table_id, bool &has_or_not) {
}
Status
DBImpl::AllTables(std::vector<meta::TableSchema> &table_schema_array) {
DBImpl::AllTables(std::vector<meta::TableSchema>& table_schema_array) {
if (shutting_down_.load(std::memory_order_acquire)) {
return Status(DB_ERROR, "Milsvus server is shutdown!");
}
@ -179,7 +176,7 @@ DBImpl::AllTables(std::vector<meta::TableSchema> &table_schema_array) {
}
Status
DBImpl::PreloadTable(const std::string &table_id) {
DBImpl::PreloadTable(const std::string& table_id) {
if (shutting_down_.load(std::memory_order_acquire)) {
return Status(DB_ERROR, "Milsvus server is shutdown!");
}
@ -198,13 +195,11 @@ DBImpl::PreloadTable(const std::string &table_id) {
int64_t cache_usage = cache::CpuCacheMgr::GetInstance()->CacheUsage();
int64_t available_size = cache_total - cache_usage;
for (auto &day_files : files) {
for (auto &file : day_files.second) {
ExecutionEnginePtr engine = EngineFactory::Build(file.dimension_,
file.location_,
(EngineType) file.engine_type_,
(MetricType) file.metric_type_,
file.nlist_);
for (auto& day_files : files) {
for (auto& file : day_files.second) {
ExecutionEnginePtr engine =
EngineFactory::Build(file.dimension_, file.location_, (EngineType)file.engine_type_,
(MetricType)file.metric_type_, file.nlist_);
if (engine == nullptr) {
ENGINE_LOG_ERROR << "Invalid engine type";
return Status(DB_ERROR, "Invalid engine type");
@ -215,9 +210,9 @@ DBImpl::PreloadTable(const std::string &table_id) {
break;
} else {
try {
//step 1: load index
// step 1: load index
engine->Load(true);
} catch (std::exception &ex) {
} catch (std::exception& ex) {
std::string msg = "Pre-load table encounter exception: " + std::string(ex.what());
ENGINE_LOG_ERROR << msg;
return Status(DB_ERROR, msg);
@ -229,7 +224,7 @@ DBImpl::PreloadTable(const std::string &table_id) {
}
Status
DBImpl::UpdateTableFlag(const std::string &table_id, int64_t flag) {
DBImpl::UpdateTableFlag(const std::string& table_id, int64_t flag) {
if (shutting_down_.load(std::memory_order_acquire)) {
return Status(DB_ERROR, "Milsvus server is shutdown!");
}
@ -238,7 +233,7 @@ DBImpl::UpdateTableFlag(const std::string &table_id, int64_t flag) {
}
Status
DBImpl::GetTableRowCount(const std::string &table_id, uint64_t &row_count) {
DBImpl::GetTableRowCount(const std::string& table_id, uint64_t& row_count) {
if (shutting_down_.load(std::memory_order_acquire)) {
return Status(DB_ERROR, "Milsvus server is shutdown!");
}
@ -247,9 +242,8 @@ DBImpl::GetTableRowCount(const std::string &table_id, uint64_t &row_count) {
}
Status
DBImpl::InsertVectors(const std::string &table_id_,
uint64_t n, const float *vectors, IDNumbers &vector_ids_) {
// ENGINE_LOG_DEBUG << "Insert " << n << " vectors to cache";
DBImpl::InsertVectors(const std::string& table_id_, uint64_t n, const float* vectors, IDNumbers& vector_ids_) {
// ENGINE_LOG_DEBUG << "Insert " << n << " vectors to cache";
if (shutting_down_.load(std::memory_order_acquire)) {
return Status(DB_ERROR, "Milsvus server is shutdown!");
}
@ -257,21 +251,21 @@ DBImpl::InsertVectors(const std::string &table_id_,
Status status;
zilliz::milvus::server::CollectInsertMetrics metrics(n, status);
status = mem_mgr_->InsertVectors(table_id_, n, vectors, vector_ids_);
// std::chrono::microseconds time_span =
// std::chrono::duration_cast<std::chrono::microseconds>(end_time - start_time);
// double average_time = double(time_span.count()) / n;
// std::chrono::microseconds time_span =
// std::chrono::duration_cast<std::chrono::microseconds>(end_time - start_time);
// double average_time = double(time_span.count()) / n;
// ENGINE_LOG_DEBUG << "Insert vectors to cache finished";
// ENGINE_LOG_DEBUG << "Insert vectors to cache finished";
return status;
}
Status
DBImpl::CreateIndex(const std::string &table_id, const TableIndex &index) {
DBImpl::CreateIndex(const std::string& table_id, const TableIndex& index) {
{
std::unique_lock<std::mutex> lock(build_index_mutex_);
//step 1: check index difference
// step 1: check index difference
TableIndex old_index;
auto status = DescribeIndex(table_id, old_index);
if (!status.ok()) {
@ -279,9 +273,9 @@ DBImpl::CreateIndex(const std::string &table_id, const TableIndex &index) {
return status;
}
//step 2: update index info
// step 2: update index info
TableIndex new_index = index;
new_index.metric_type_ = old_index.metric_type_;//dont change metric type, it was defined by CreateTable
new_index.metric_type_ = old_index.metric_type_; // dont change metric type, it was defined by CreateTable
if (!utils::IsSameIndex(old_index, new_index)) {
DropIndex(table_id);
@ -293,26 +287,23 @@ DBImpl::CreateIndex(const std::string &table_id, const TableIndex &index) {
}
}
//step 3: let merge file thread finish
//to avoid duplicate data bug
// step 3: let merge file thread finish
// to avoid duplicate data bug
WaitMergeFileFinish();
//step 4: wait and build index
//for IDMAP type, only wait all NEW file converted to RAW file
//for other type, wait NEW/RAW/NEW_MERGE/NEW_INDEX/TO_INDEX files converted to INDEX files
// step 4: wait and build index
// for IDMAP type, only wait all NEW file converted to RAW file
// for other type, wait NEW/RAW/NEW_MERGE/NEW_INDEX/TO_INDEX files converted to INDEX files
std::vector<int> file_types;
if (index.engine_type_ == (int) EngineType::FAISS_IDMAP) {
if (index.engine_type_ == (int)EngineType::FAISS_IDMAP) {
file_types = {
(int) meta::TableFileSchema::NEW,
(int) meta::TableFileSchema::NEW_MERGE,
(int)meta::TableFileSchema::NEW, (int)meta::TableFileSchema::NEW_MERGE,
};
} else {
file_types = {
(int) meta::TableFileSchema::RAW,
(int) meta::TableFileSchema::NEW,
(int) meta::TableFileSchema::NEW_MERGE,
(int) meta::TableFileSchema::NEW_INDEX,
(int) meta::TableFileSchema::TO_INDEX,
(int)meta::TableFileSchema::RAW, (int)meta::TableFileSchema::NEW,
(int)meta::TableFileSchema::NEW_MERGE, (int)meta::TableFileSchema::NEW_INDEX,
(int)meta::TableFileSchema::TO_INDEX,
};
}
@ -322,7 +313,7 @@ DBImpl::CreateIndex(const std::string &table_id, const TableIndex &index) {
while (!file_ids.empty()) {
ENGINE_LOG_DEBUG << "Non index files detected! Will build index " << times;
if (index.engine_type_ != (int) EngineType::FAISS_IDMAP) {
if (index.engine_type_ != (int)EngineType::FAISS_IDMAP) {
status = meta_ptr_->UpdateTableFilesToIndex(table_id);
}
@ -335,19 +326,19 @@ DBImpl::CreateIndex(const std::string &table_id, const TableIndex &index) {
}
Status
DBImpl::DescribeIndex(const std::string &table_id, TableIndex &index) {
DBImpl::DescribeIndex(const std::string& table_id, TableIndex& index) {
return meta_ptr_->DescribeTableIndex(table_id, index);
}
Status
DBImpl::DropIndex(const std::string &table_id) {
DBImpl::DropIndex(const std::string& table_id) {
ENGINE_LOG_DEBUG << "Drop index for table: " << table_id;
return meta_ptr_->DropTableIndex(table_id);
}
Status
DBImpl::Query(const std::string &table_id, uint64_t k, uint64_t nq, uint64_t nprobe,
const float *vectors, QueryResults &results) {
DBImpl::Query(const std::string& table_id, uint64_t k, uint64_t nq, uint64_t nprobe, const float* vectors,
QueryResults& results) {
if (shutting_down_.load(std::memory_order_acquire)) {
return Status(DB_ERROR, "Milsvus server is shutdown!");
}
@ -359,46 +350,47 @@ DBImpl::Query(const std::string &table_id, uint64_t k, uint64_t nq, uint64_t npr
}
Status
DBImpl::Query(const std::string &table_id, uint64_t k, uint64_t nq, uint64_t nprobe,
const float *vectors, const meta::DatesT &dates, QueryResults &results) {
DBImpl::Query(const std::string& table_id, uint64_t k, uint64_t nq, uint64_t nprobe, const float* vectors,
const meta::DatesT& dates, QueryResults& results) {
if (shutting_down_.load(std::memory_order_acquire)) {
return Status(DB_ERROR, "Milsvus server is shutdown!");
}
ENGINE_LOG_DEBUG << "Query by dates for table: " << table_id;
//get all table files from table
// get all table files from table
meta::DatePartionedTableFilesSchema files;
std::vector<size_t> ids;
auto status = meta_ptr_->FilesToSearch(table_id, ids, dates, files);
if (!status.ok()) { return status; }
if (!status.ok()) {
return status;
}
meta::TableFilesSchema file_id_array;
for (auto &day_files : files) {
for (auto &file : day_files.second) {
for (auto& day_files : files) {
for (auto& file : day_files.second) {
file_id_array.push_back(file);
}
}
cache::CpuCacheMgr::GetInstance()->PrintInfo(); //print cache info before query
cache::CpuCacheMgr::GetInstance()->PrintInfo(); // print cache info before query
status = QueryAsync(table_id, file_id_array, k, nq, nprobe, vectors, dates, results);
cache::CpuCacheMgr::GetInstance()->PrintInfo(); //print cache info after query
cache::CpuCacheMgr::GetInstance()->PrintInfo(); // print cache info after query
return status;
}
Status
DBImpl::Query(const std::string &table_id, const std::vector<std::string> &file_ids,
uint64_t k, uint64_t nq, uint64_t nprobe, const float *vectors,
const meta::DatesT &dates, QueryResults &results) {
DBImpl::Query(const std::string& table_id, const std::vector<std::string>& file_ids, uint64_t k, uint64_t nq,
uint64_t nprobe, const float* vectors, const meta::DatesT& dates, QueryResults& results) {
if (shutting_down_.load(std::memory_order_acquire)) {
return Status(DB_ERROR, "Milsvus server is shutdown!");
}
ENGINE_LOG_DEBUG << "Query by file ids for table: " << table_id;
//get specified files
// get specified files
std::vector<size_t> ids;
for (auto &id : file_ids) {
for (auto& id : file_ids) {
meta::TableFileSchema table_file;
table_file.table_id_ = table_id;
std::string::size_type sz;
@ -412,8 +404,8 @@ DBImpl::Query(const std::string &table_id, const std::vector<std::string> &file_
}
meta::TableFilesSchema file_id_array;
for (auto &day_files : files_array) {
for (auto &file : day_files.second) {
for (auto& day_files : files_array) {
for (auto& file : day_files.second) {
file_id_array.push_back(file);
}
}
@ -422,14 +414,14 @@ DBImpl::Query(const std::string &table_id, const std::vector<std::string> &file_
return Status(DB_ERROR, "Invalid file id");
}
cache::CpuCacheMgr::GetInstance()->PrintInfo(); //print cache info before query
cache::CpuCacheMgr::GetInstance()->PrintInfo(); // print cache info before query
status = QueryAsync(table_id, file_id_array, k, nq, nprobe, vectors, dates, results);
cache::CpuCacheMgr::GetInstance()->PrintInfo(); //print cache info after query
cache::CpuCacheMgr::GetInstance()->PrintInfo(); // print cache info after query
return status;
}
Status
DBImpl::Size(uint64_t &result) {
DBImpl::Size(uint64_t& result) {
if (shutting_down_.load(std::memory_order_acquire)) {
return Status(DB_ERROR, "Milsvus server is shutdown!");
}
@ -438,58 +430,57 @@ DBImpl::Size(uint64_t &result) {
}
///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
//internal methods
// internal methods
///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
Status
DBImpl::QueryAsync(const std::string &table_id, const meta::TableFilesSchema &files,
uint64_t k, uint64_t nq, uint64_t nprobe, const float *vectors,
const meta::DatesT &dates, QueryResults &results) {
DBImpl::QueryAsync(const std::string& table_id, const meta::TableFilesSchema& files, uint64_t k, uint64_t nq,
uint64_t nprobe, const float* vectors, const meta::DatesT& dates, QueryResults& results) {
server::CollectQueryMetrics metrics(nq);
TimeRecorder rc("");
//step 1: get files to search
ENGINE_LOG_DEBUG << "Engine query begin, index file count: " << files.size() << " date range count: "
<< dates.size();
// step 1: get files to search
ENGINE_LOG_DEBUG << "Engine query begin, index file count: " << files.size()
<< " date range count: " << dates.size();
scheduler::SearchJobPtr job = std::make_shared<scheduler::SearchJob>(0, k, nq, nprobe, vectors);
for (auto &file : files) {
for (auto& file : files) {
scheduler::TableFileSchemaPtr file_ptr = std::make_shared<meta::TableFileSchema>(file);
job->AddIndexFile(file_ptr);
}
//step 2: put search task to scheduler
// step 2: put search task to scheduler
scheduler::JobMgrInst::GetInstance()->Put(job);
job->WaitResult();
if (!job->GetStatus().ok()) {
return job->GetStatus();
}
//step 3: print time cost information
// double load_cost = context->LoadCost();
// double search_cost = context->SearchCost();
// double reduce_cost = context->ReduceCost();
// std::string load_info = TimeRecorder::GetTimeSpanStr(load_cost);
// std::string search_info = TimeRecorder::GetTimeSpanStr(search_cost);
// std::string reduce_info = TimeRecorder::GetTimeSpanStr(reduce_cost);
// if(search_cost > 0.0 || reduce_cost > 0.0) {
// double total_cost = load_cost + search_cost + reduce_cost;
// double load_percent = load_cost/total_cost;
// double search_percent = search_cost/total_cost;
// double reduce_percent = reduce_cost/total_cost;
//
// ENGINE_LOG_DEBUG << "Engine load index totally cost: " << load_info
// << " percent: " << load_percent*100 << "%";
// ENGINE_LOG_DEBUG << "Engine search index totally cost: " << search_info
// << " percent: " << search_percent*100 << "%";
// ENGINE_LOG_DEBUG << "Engine reduce topk totally cost: " << reduce_info
// << " percent: " << reduce_percent*100 << "%";
// } else {
// ENGINE_LOG_DEBUG << "Engine load cost: " << load_info
// << " search cost: " << search_info
// << " reduce cost: " << reduce_info;
// }
// step 3: print time cost information
// double load_cost = context->LoadCost();
// double search_cost = context->SearchCost();
// double reduce_cost = context->ReduceCost();
// std::string load_info = TimeRecorder::GetTimeSpanStr(load_cost);
// std::string search_info = TimeRecorder::GetTimeSpanStr(search_cost);
// std::string reduce_info = TimeRecorder::GetTimeSpanStr(reduce_cost);
// if(search_cost > 0.0 || reduce_cost > 0.0) {
// double total_cost = load_cost + search_cost + reduce_cost;
// double load_percent = load_cost/total_cost;
// double search_percent = search_cost/total_cost;
// double reduce_percent = reduce_cost/total_cost;
//
// ENGINE_LOG_DEBUG << "Engine load index totally cost: " << load_info
// << " percent: " << load_percent*100 << "%";
// ENGINE_LOG_DEBUG << "Engine search index totally cost: " << search_info
// << " percent: " << search_percent*100 << "%";
// ENGINE_LOG_DEBUG << "Engine reduce topk totally cost: " << reduce_info
// << " percent: " << reduce_percent*100 << "%";
// } else {
// ENGINE_LOG_DEBUG << "Engine load cost: " << load_info
// << " search cost: " << search_info
// << " reduce cost: " << reduce_info;
// }
//step 4: construct results
// step 4: construct results
results = job->GetResult();
rc.ElapseFromBegin("Engine query totally cost");
@ -520,7 +511,7 @@ DBImpl::BackgroundTimerTask() {
void
DBImpl::WaitMergeFileFinish() {
std::lock_guard<std::mutex> lck(compact_result_mutex_);
for (auto &iter : compact_thread_results_) {
for (auto& iter : compact_thread_results_) {
iter.wait();
}
}
@ -528,7 +519,7 @@ DBImpl::WaitMergeFileFinish() {
void
DBImpl::WaitBuildIndexFinish() {
std::lock_guard<std::mutex> lck(index_result_mutex_);
for (auto &iter : index_thread_results_) {
for (auto& iter : index_thread_results_) {
iter.wait();
}
}
@ -569,7 +560,7 @@ DBImpl::MemSerialize() {
std::lock_guard<std::mutex> lck(mem_serialize_mutex_);
std::set<std::string> temp_table_ids;
mem_mgr_->Serialize(temp_table_ids);
for (auto &id : temp_table_ids) {
for (auto& id : temp_table_ids) {
compact_table_ids_.insert(id);
}
@ -588,10 +579,10 @@ DBImpl::StartCompactionTask() {
return;
}
//serialize memory data
// serialize memory data
MemSerialize();
//compactiong has been finished?
// compactiong has been finished?
{
std::lock_guard<std::mutex> lck(compact_result_mutex_);
if (!compact_thread_results_.empty()) {
@ -602,7 +593,7 @@ DBImpl::StartCompactionTask() {
}
}
//add new compaction task
// add new compaction task
{
std::lock_guard<std::mutex> lck(compact_result_mutex_);
if (compact_thread_results_.empty()) {
@ -614,11 +605,10 @@ DBImpl::StartCompactionTask() {
}
Status
DBImpl::MergeFiles(const std::string &table_id, const meta::DateT &date,
const meta::TableFilesSchema &files) {
DBImpl::MergeFiles(const std::string& table_id, const meta::DateT& date, const meta::TableFilesSchema& files) {
ENGINE_LOG_DEBUG << "Merge files for table: " << table_id;
//step 1: create table file
// step 1: create table file
meta::TableFileSchema table_file;
table_file.table_id_ = table_id;
table_file.date_ = date;
@ -630,15 +620,15 @@ DBImpl::MergeFiles(const std::string &table_id, const meta::DateT &date,
return status;
}
//step 2: merge files
// step 2: merge files
ExecutionEnginePtr index =
EngineFactory::Build(table_file.dimension_, table_file.location_, (EngineType) table_file.engine_type_,
(MetricType) table_file.metric_type_, table_file.nlist_);
EngineFactory::Build(table_file.dimension_, table_file.location_, (EngineType)table_file.engine_type_,
(MetricType)table_file.metric_type_, table_file.nlist_);
meta::TableFilesSchema updated;
int64_t index_size = 0;
for (auto &file : files) {
for (auto& file : files) {
server::CollectMergeFilesMetrics metrics;
index->Merge(file.location_);
@ -651,11 +641,11 @@ DBImpl::MergeFiles(const std::string &table_id, const meta::DateT &date,
if (index_size >= file_schema.index_file_size_) break;
}
//step 3: serialize to disk
// step 3: serialize to disk
try {
index->Serialize();
} catch (std::exception &ex) {
//typical error: out of disk space or permition denied
} catch (std::exception& ex) {
// typical error: out of disk space or permition denied
std::string msg = "Serialize merged index encounter exception: " + std::string(ex.what());
ENGINE_LOG_ERROR << msg;
@ -669,12 +659,12 @@ DBImpl::MergeFiles(const std::string &table_id, const meta::DateT &date,
return Status(DB_ERROR, msg);
}
//step 4: update table files state
//if index type isn't IDMAP, set file type to TO_INDEX if file size execeed index_file_size
//else set file type to RAW, no need to build index
if (table_file.engine_type_ != (int) EngineType::FAISS_IDMAP) {
table_file.file_type_ = (index->PhysicalSize() >= table_file.index_file_size_) ?
meta::TableFileSchema::TO_INDEX : meta::TableFileSchema::RAW;
// step 4: update table files state
// if index type isn't IDMAP, set file type to TO_INDEX if file size execeed index_file_size
// else set file type to RAW, no need to build index
if (table_file.engine_type_ != (int)EngineType::FAISS_IDMAP) {
table_file.file_type_ = (index->PhysicalSize() >= table_file.index_file_size_) ? meta::TableFileSchema::TO_INDEX
: meta::TableFileSchema::RAW;
} else {
table_file.file_type_ = meta::TableFileSchema::RAW;
}
@ -682,8 +672,7 @@ DBImpl::MergeFiles(const std::string &table_id, const meta::DateT &date,
table_file.row_count_ = index->Count();
updated.push_back(table_file);
status = meta_ptr_->UpdateTableFiles(updated);
ENGINE_LOG_DEBUG << "New merged file " << table_file.file_id_ <<
" of size " << index->PhysicalSize() << " bytes";
ENGINE_LOG_DEBUG << "New merged file " << table_file.file_id_ << " of size " << index->PhysicalSize() << " bytes";
if (options_.insert_cache_immediately_) {
index->Cache();
@ -693,7 +682,7 @@ DBImpl::MergeFiles(const std::string &table_id, const meta::DateT &date,
}
Status
DBImpl::BackgroundMergeFiles(const std::string &table_id) {
DBImpl::BackgroundMergeFiles(const std::string& table_id) {
meta::DatePartionedTableFilesSchema raw_files;
auto status = meta_ptr_->FilesToMerge(table_id, raw_files);
if (!status.ok()) {
@ -702,7 +691,7 @@ DBImpl::BackgroundMergeFiles(const std::string &table_id) {
}
bool has_merge = false;
for (auto &kv : raw_files) {
for (auto& kv : raw_files) {
auto files = kv.second;
if (files.size() < options_.merge_trigger_number_) {
ENGINE_LOG_DEBUG << "Files number not greater equal than merge trigger number, skip merge action";
@ -725,7 +714,7 @@ DBImpl::BackgroundCompaction(std::set<std::string> table_ids) {
ENGINE_LOG_TRACE << " Background compaction thread start";
Status status;
for (auto &table_id : table_ids) {
for (auto& table_id : table_ids) {
status = BackgroundMergeFiles(table_id);
if (!status.ok()) {
ENGINE_LOG_ERROR << "Merge files for table " << table_id << " failed: " << status.ToString();
@ -739,7 +728,7 @@ DBImpl::BackgroundCompaction(std::set<std::string> table_ids) {
meta_ptr_->Archive();
int ttl = 5 * meta::M_SEC;//default: file will be deleted after 5 minutes
int ttl = 5 * meta::M_SEC; // default: file will be deleted after 5 minutes
if (options_.mode_ == DBOptions::MODE::CLUSTER_WRITABLE) {
ttl = meta::D_SEC;
}
@ -756,7 +745,7 @@ DBImpl::StartBuildIndexTask(bool force) {
return;
}
//build index has been finished?
// build index has been finished?
{
std::lock_guard<std::mutex> lck(index_result_mutex_);
if (!index_thread_results_.empty()) {
@ -767,52 +756,50 @@ DBImpl::StartBuildIndexTask(bool force) {
}
}
//add new build index task
// add new build index task
{
std::lock_guard<std::mutex> lck(index_result_mutex_);
if (index_thread_results_.empty()) {
index_thread_results_.push_back(
index_thread_pool_.enqueue(&DBImpl::BackgroundBuildIndex, this));
index_thread_results_.push_back(index_thread_pool_.enqueue(&DBImpl::BackgroundBuildIndex, this));
}
}
}
Status
DBImpl::BuildIndex(const meta::TableFileSchema &file) {
ExecutionEnginePtr to_index =
EngineFactory::Build(file.dimension_, file.location_, (EngineType) file.engine_type_,
(MetricType) file.metric_type_, file.nlist_);
DBImpl::BuildIndex(const meta::TableFileSchema& file) {
ExecutionEnginePtr to_index = EngineFactory::Build(file.dimension_, file.location_, (EngineType)file.engine_type_,
(MetricType)file.metric_type_, file.nlist_);
if (to_index == nullptr) {
ENGINE_LOG_ERROR << "Invalid engine type";
return Status(DB_ERROR, "Invalid engine type");
}
try {
//step 1: load index
// step 1: load index
Status status = to_index->Load(options_.insert_cache_immediately_);
if (!status.ok()) {
ENGINE_LOG_ERROR << "Failed to load index file: " << status.ToString();
return status;
}
//step 2: create table file
// step 2: create table file
meta::TableFileSchema table_file;
table_file.table_id_ = file.table_id_;
table_file.date_ = file.date_;
table_file.file_type_ =
meta::TableFileSchema::NEW_INDEX; //for multi-db-path, distribute index file averagely to each path
meta::TableFileSchema::NEW_INDEX; // for multi-db-path, distribute index file averagely to each path
status = meta_ptr_->CreateTableFile(table_file);
if (!status.ok()) {
ENGINE_LOG_ERROR << "Failed to create table file: " << status.ToString();
return status;
}
//step 3: build index
// step 3: build index
std::shared_ptr<ExecutionEngine> index;
try {
server::CollectBuildIndexMetrics metrics;
index = to_index->BuildIndex(table_file.location_, (EngineType) table_file.engine_type_);
index = to_index->BuildIndex(table_file.location_, (EngineType)table_file.engine_type_);
if (index == nullptr) {
table_file.file_type_ = meta::TableFileSchema::TO_DELETE;
status = meta_ptr_->UpdateTableFile(table_file);
@ -821,8 +808,8 @@ DBImpl::BuildIndex(const meta::TableFileSchema &file) {
return status;
}
} catch (std::exception &ex) {
//typical error: out of gpu memory
} catch (std::exception& ex) {
// typical error: out of gpu memory
std::string msg = "BuildIndex encounter exception: " + std::string(ex.what());
ENGINE_LOG_ERROR << msg;
@ -836,7 +823,7 @@ DBImpl::BuildIndex(const meta::TableFileSchema &file) {
return Status(DB_ERROR, msg);
}
//step 4: if table has been deleted, dont save index file
// step 4: if table has been deleted, dont save index file
bool has_table = false;
meta_ptr_->HasTable(file.table_id_, has_table);
if (!has_table) {
@ -844,11 +831,11 @@ DBImpl::BuildIndex(const meta::TableFileSchema &file) {
return Status::OK();
}
//step 5: save index file
// step 5: save index file
try {
index->Serialize();
} catch (std::exception &ex) {
//typical error: out of disk space or permition denied
} catch (std::exception& ex) {
// typical error: out of disk space or permition denied
std::string msg = "Serialize index encounter exception: " + std::string(ex.what());
ENGINE_LOG_ERROR << msg;
@ -862,7 +849,7 @@ DBImpl::BuildIndex(const meta::TableFileSchema &file) {
return Status(DB_ERROR, msg);
}
//step 6: update meta
// step 6: update meta
table_file.file_type_ = meta::TableFileSchema::INDEX;
table_file.file_size_ = index->PhysicalSize();
table_file.row_count_ = index->Count();
@ -873,15 +860,15 @@ DBImpl::BuildIndex(const meta::TableFileSchema &file) {
meta::TableFilesSchema update_files = {table_file, origin_file};
status = meta_ptr_->UpdateTableFiles(update_files);
if (status.ok()) {
ENGINE_LOG_DEBUG << "New index file " << table_file.file_id_ << " of size "
<< index->PhysicalSize() << " bytes"
ENGINE_LOG_DEBUG << "New index file " << table_file.file_id_ << " of size " << index->PhysicalSize()
<< " bytes"
<< " from file " << origin_file.file_id_;
if (options_.insert_cache_immediately_) {
index->Cache();
}
} else {
//failed to update meta, mark the new file as to_delete, don't delete old file
// failed to update meta, mark the new file as to_delete, don't delete old file
origin_file.file_type_ = meta::TableFileSchema::TO_INDEX;
status = meta_ptr_->UpdateTableFile(origin_file);
ENGINE_LOG_DEBUG << "Failed to update file to index, mark file: " << origin_file.file_id_ << " to to_index";
@ -890,7 +877,7 @@ DBImpl::BuildIndex(const meta::TableFileSchema &file) {
status = meta_ptr_->UpdateTableFile(table_file);
ENGINE_LOG_DEBUG << "Failed to update file to index, mark file: " << table_file.file_id_ << " to to_delete";
}
} catch (std::exception &ex) {
} catch (std::exception& ex) {
std::string msg = "Build index encounter exception: " + std::string(ex.what());
ENGINE_LOG_ERROR << msg;
return Status(DB_ERROR, msg);
@ -907,7 +894,7 @@ DBImpl::BackgroundBuildIndex() {
meta::TableFilesSchema to_index_files;
meta_ptr_->FilesToIndex(to_index_files);
Status status;
for (auto &file : to_index_files) {
for (auto& file : to_index_files) {
status = BuildIndex(file);
if (!status.ok()) {
ENGINE_LOG_ERROR << "Building index for " << file.id_ << " failed: " << status.ToString();
@ -922,6 +909,6 @@ DBImpl::BackgroundBuildIndex() {
ENGINE_LOG_TRACE << "Background build index thread exit";
}
} // namespace engine
} // namespace milvus
} // namespace zilliz
} // namespace engine
} // namespace milvus
} // namespace zilliz

150
cpp/src/db/DBImpl.h
View File

@ -19,18 +19,18 @@
#include "DB.h"
#include "Types.h"
#include "utils/ThreadPool.h"
#include "src/db/insert/MemManager.h"
#include "utils/ThreadPool.h"
#include <mutex>
#include <condition_variable>
#include <memory>
#include <atomic>
#include <thread>
#include <condition_variable>
#include <list>
#include <memory>
#include <mutex>
#include <set>
#include <vector>
#include <string>
#include <thread>
#include <vector>
namespace zilliz {
namespace milvus {
@ -44,92 +44,101 @@ class Meta;
class DBImpl : public DB {
public:
explicit DBImpl(const DBOptions &options);
explicit DBImpl(const DBOptions& options);
~DBImpl();
Status Start() override;
Status Stop() override;
Status DropAll() override;
Status
Start() override;
Status
Stop() override;
Status
DropAll() override;
Status CreateTable(meta::TableSchema &table_schema) override;
Status
CreateTable(meta::TableSchema& table_schema) override;
Status DeleteTable(const std::string &table_id, const meta::DatesT &dates) override;
Status
DeleteTable(const std::string& table_id, const meta::DatesT& dates) override;
Status DescribeTable(meta::TableSchema &table_schema) override;
Status
DescribeTable(meta::TableSchema& table_schema) override;
Status HasTable(const std::string &table_id, bool &has_or_not) override;
Status
HasTable(const std::string& table_id, bool& has_or_not) override;
Status AllTables(std::vector<meta::TableSchema> &table_schema_array) override;
Status
AllTables(std::vector<meta::TableSchema>& table_schema_array) override;
Status PreloadTable(const std::string &table_id) override;
Status
PreloadTable(const std::string& table_id) override;
Status UpdateTableFlag(const std::string &table_id, int64_t flag);
Status
UpdateTableFlag(const std::string& table_id, int64_t flag);
Status GetTableRowCount(const std::string &table_id, uint64_t &row_count) override;
Status
GetTableRowCount(const std::string& table_id, uint64_t& row_count) override;
Status InsertVectors(const std::string &table_id, uint64_t n, const float *vectors, IDNumbers &vector_ids) override;
Status
InsertVectors(const std::string& table_id, uint64_t n, const float* vectors, IDNumbers& vector_ids) override;
Status CreateIndex(const std::string &table_id, const TableIndex &index) override;
Status
CreateIndex(const std::string& table_id, const TableIndex& index) override;
Status DescribeIndex(const std::string &table_id, TableIndex &index) override;
Status
DescribeIndex(const std::string& table_id, TableIndex& index) override;
Status DropIndex(const std::string &table_id) override;
Status
DropIndex(const std::string& table_id) override;
Status Query(const std::string &table_id,
uint64_t k,
uint64_t nq,
uint64_t nprobe,
const float *vectors,
QueryResults &results) override;
Status
Query(const std::string& table_id, uint64_t k, uint64_t nq, uint64_t nprobe, const float* vectors,
QueryResults& results) override;
Status Query(const std::string &table_id,
uint64_t k,
uint64_t nq,
uint64_t nprobe,
const float *vectors,
const meta::DatesT &dates,
QueryResults &results) override;
Status
Query(const std::string& table_id, uint64_t k, uint64_t nq, uint64_t nprobe, const float* vectors,
const meta::DatesT& dates, QueryResults& results) override;
Status Query(const std::string &table_id,
const std::vector<std::string> &file_ids,
uint64_t k,
uint64_t nq,
uint64_t nprobe,
const float *vectors,
const meta::DatesT &dates,
QueryResults &results) override;
Status
Query(const std::string& table_id, const std::vector<std::string>& file_ids, uint64_t k, uint64_t nq,
uint64_t nprobe, const float* vectors, const meta::DatesT& dates, QueryResults& results) override;
Status Size(uint64_t &result) override;
Status
Size(uint64_t& result) override;
private:
Status QueryAsync(const std::string &table_id,
const meta::TableFilesSchema &files,
uint64_t k,
uint64_t nq,
uint64_t nprobe,
const float *vectors,
const meta::DatesT &dates,
QueryResults &results);
Status
QueryAsync(const std::string& table_id, const meta::TableFilesSchema& files, uint64_t k, uint64_t nq,
uint64_t nprobe, const float* vectors, const meta::DatesT& dates, QueryResults& results);
void BackgroundTimerTask();
void WaitMergeFileFinish();
void WaitBuildIndexFinish();
void
BackgroundTimerTask();
void
WaitMergeFileFinish();
void
WaitBuildIndexFinish();
void StartMetricTask();
void
StartMetricTask();
void StartCompactionTask();
Status MergeFiles(const std::string &table_id,
const meta::DateT &date,
const meta::TableFilesSchema &files);
Status BackgroundMergeFiles(const std::string &table_id);
void BackgroundCompaction(std::set<std::string> table_ids);
void
StartCompactionTask();
Status
MergeFiles(const std::string& table_id, const meta::DateT& date, const meta::TableFilesSchema& files);
Status
BackgroundMergeFiles(const std::string& table_id);
void
BackgroundCompaction(std::set<std::string> table_ids);
void StartBuildIndexTask(bool force = false);
void BackgroundBuildIndex();
void
StartBuildIndexTask(bool force = false);
void
BackgroundBuildIndex();
Status BuildIndex(const meta::TableFileSchema &);
Status
BuildIndex(const meta::TableFileSchema&);
Status MemSerialize();
Status
MemSerialize();
private:
const DBOptions options_;
@ -152,9 +161,8 @@ class DBImpl : public DB {
std::list<std::future<void>> index_thread_results_;
std::mutex build_index_mutex_;
}; // DBImpl
}; // DBImpl
} // namespace engine
} // namespace milvus
} // namespace zilliz
} // namespace engine
} // namespace milvus
} // namespace zilliz

18
cpp/src/db/IDGenerator.cpp
View File

@ -17,8 +17,8 @@
#include "db/IDGenerator.h"
#include <chrono>
#include <assert.h>
#include <chrono>
#include <iostream>
namespace zilliz {
@ -32,13 +32,12 @@ constexpr size_t SimpleIDGenerator::MAX_IDS_PER_MICRO;
IDNumber
SimpleIDGenerator::GetNextIDNumber() {
auto now = std::chrono::system_clock::now();
auto micros = std::chrono::duration_cast<std::chrono::microseconds>(
now.time_since_epoch()).count();
auto micros = std::chrono::duration_cast<std::chrono::microseconds>(now.time_since_epoch()).count();
return micros * MAX_IDS_PER_MICRO;
}
void
SimpleIDGenerator::NextIDNumbers(size_t n, IDNumbers &ids) {
SimpleIDGenerator::NextIDNumbers(size_t n, IDNumbers& ids) {
if (n > MAX_IDS_PER_MICRO) {
NextIDNumbers(n - MAX_IDS_PER_MICRO, ids);
NextIDNumbers(MAX_IDS_PER_MICRO, ids);
@ -49,8 +48,7 @@ SimpleIDGenerator::NextIDNumbers(size_t n, IDNumbers &ids) {
}
auto now = std::chrono::system_clock::now();
auto micros = std::chrono::duration_cast<std::chrono::microseconds>(
now.time_since_epoch()).count();
auto micros = std::chrono::duration_cast<std::chrono::microseconds>(now.time_since_epoch()).count();
micros *= MAX_IDS_PER_MICRO;
for (int pos = 0; pos < n; ++pos) {
@ -59,11 +57,11 @@ SimpleIDGenerator::NextIDNumbers(size_t n, IDNumbers &ids) {
}
void
SimpleIDGenerator::GetNextIDNumbers(size_t n, IDNumbers &ids) {
SimpleIDGenerator::GetNextIDNumbers(size_t n, IDNumbers& ids) {
ids.clear();
NextIDNumbers(n, ids);
}
} // namespace engine
} // namespace milvus
} // namespace zilliz
} // namespace engine
} // namespace milvus
} // namespace zilliz

25
cpp/src/db/IDGenerator.h
View File

@ -28,16 +28,14 @@ namespace engine {
class IDGenerator {
public:
virtual
IDNumber GetNextIDNumber() = 0;
virtual IDNumber
GetNextIDNumber() = 0;
virtual void
GetNextIDNumbers(size_t n, IDNumbers &ids) = 0;
virtual
~IDGenerator() = 0;
}; // IDGenerator
GetNextIDNumbers(size_t n, IDNumbers& ids) = 0;
virtual ~IDGenerator() = 0;
}; // IDGenerator
class SimpleIDGenerator : public IDGenerator {
public:
@ -47,16 +45,15 @@ class SimpleIDGenerator : public IDGenerator {
GetNextIDNumber() override;
void
GetNextIDNumbers(size_t n, IDNumbers &ids) override;
GetNextIDNumbers(size_t n, IDNumbers& ids) override;
private:
void
NextIDNumbers(size_t n, IDNumbers &ids);
NextIDNumbers(size_t n, IDNumbers& ids);
static constexpr size_t MAX_IDS_PER_MICRO = 1000;
}; // SimpleIDGenerator
}; // SimpleIDGenerator
} // namespace engine
} // namespace milvus
} // namespace zilliz
} // namespace engine
} // namespace milvus
} // namespace zilliz

26
cpp/src/db/Options.cpp
View File

@ -19,28 +19,28 @@
#include "utils/Exception.h"
#include "utils/Log.h"
#include <stdlib.h>
#include <assert.h>
#include <stdlib.h>
#include <boost/algorithm/string.hpp>
namespace zilliz {
namespace milvus {
namespace engine {
ArchiveConf::ArchiveConf(const std::string &type, const std::string &criterias) {
ArchiveConf::ArchiveConf(const std::string& type, const std::string& criterias) {
ParseType(type);
ParseCritirias(criterias);
}
void
ArchiveConf::SetCriterias(const ArchiveConf::CriteriaT &criterial) {
for (auto &pair : criterial) {
ArchiveConf::SetCriterias(const ArchiveConf::CriteriaT& criterial) {
for (auto& pair : criterial) {
criterias_[pair.first] = pair.second;
}
}
void
ArchiveConf::ParseCritirias(const std::string &criterias) {
ArchiveConf::ParseCritirias(const std::string& criterias) {
std::stringstream ss(criterias);
std::vector<std::string> tokens;
@ -50,7 +50,7 @@ ArchiveConf::ParseCritirias(const std::string &criterias) {
return;
}
for (auto &token : tokens) {
for (auto& token : tokens) {
if (token.empty()) {
continue;
}
@ -68,13 +68,11 @@ ArchiveConf::ParseCritirias(const std::string &criterias) {
try {
auto value = std::stoi(kv[1]);
criterias_[kv[0]] = value;
}
catch (std::out_of_range &) {
} catch (std::out_of_range&) {
std::string msg = "Out of range: '" + kv[1] + "'";
ENGINE_LOG_ERROR << msg;
throw InvalidArgumentException(msg);
}
catch (...) {
} catch (...) {
std::string msg = "Invalid argument: '" + kv[1] + "'";
ENGINE_LOG_ERROR << msg;
throw InvalidArgumentException(msg);
@ -83,7 +81,7 @@ ArchiveConf::ParseCritirias(const std::string &criterias) {
}
void
ArchiveConf::ParseType(const std::string &type) {
ArchiveConf::ParseType(const std::string& type) {
if (type != "delete" && type != "swap") {
std::string msg = "Invalid argument: type='" + type + "'";
throw InvalidArgumentException(msg);
@ -91,6 +89,6 @@ ArchiveConf::ParseType(const std::string &type) {
type_ = type;
}
} // namespace engine
} // namespace milvus
} // namespace zilliz
} // namespace engine
} // namespace milvus
} // namespace zilliz

42
cpp/src/db/Options.h
View File

@ -19,9 +19,9 @@
#include "Constants.h"
#include <string>
#include <memory>
#include <map>
#include <memory>
#include <string>
#include <vector>
namespace zilliz {
@ -30,27 +30,32 @@ namespace engine {
class Env;
static const char *ARCHIVE_CONF_DISK = "disk";
static const char *ARCHIVE_CONF_DAYS = "days";
static const char* ARCHIVE_CONF_DISK = "disk";
static const char* ARCHIVE_CONF_DAYS = "days";
struct ArchiveConf {
using CriteriaT = std::map<std::string, int>;
explicit ArchiveConf(const std::string &type, const std::string &criterias = std::string());
explicit ArchiveConf(const std::string& type, const std::string& criterias = std::string());
const std::string &GetType() const {
const std::string&
GetType() const {
return type_;
}
const CriteriaT GetCriterias() const {
const CriteriaT
GetCriterias() const {
return criterias_;
}
void SetCriterias(const ArchiveConf::CriteriaT &criterial);
void
SetCriterias(const ArchiveConf::CriteriaT& criterial);
private:
void ParseCritirias(const std::string &type);
void ParseType(const std::string &criterias);
void
ParseCritirias(const std::string& type);
void
ParseType(const std::string& criterias);
std::string type_;
CriteriaT criterias_;
@ -61,14 +66,10 @@ struct DBMetaOptions {
std::vector<std::string> slave_paths_;
std::string backend_uri_;
ArchiveConf archive_conf_ = ArchiveConf("delete");
}; // DBMetaOptions
}; // DBMetaOptions
struct DBOptions {
typedef enum {
SINGLE = 0,
CLUSTER_READONLY,
CLUSTER_WRITABLE
} MODE;
typedef enum { SINGLE = 0, CLUSTER_READONLY, CLUSTER_WRITABLE } MODE;
uint16_t merge_trigger_number_ = 2;
DBMetaOptions meta_;
@ -76,9 +77,8 @@ struct DBOptions {
size_t insert_buffer_size_ = 4 * ONE_GB;
bool insert_cache_immediately_ = false;
}; // Options
}; // Options
} // namespace engine
} // namespace milvus
} // namespace zilliz
} // namespace engine
} // namespace milvus
} // namespace zilliz

14
cpp/src/db/Types.h
View File

@ -19,27 +19,27 @@
#include "db/engine/ExecutionEngine.h"
#include <vector>
#include <stdint.h>
#include <utility>
#include <vector>
namespace zilliz {
namespace milvus {
namespace engine {
typedef int64_t IDNumber;
typedef IDNumber *IDNumberPtr;
typedef IDNumber* IDNumberPtr;
typedef std::vector<IDNumber> IDNumbers;
typedef std::vector<std::pair<IDNumber, double>> QueryResult;
typedef std::vector<QueryResult> QueryResults;
struct TableIndex {
int32_t engine_type_ = (int) EngineType::FAISS_IDMAP;
int32_t engine_type_ = (int)EngineType::FAISS_IDMAP;
int32_t nlist_ = 16384;
int32_t metric_type_ = (int) MetricType::L2;
int32_t metric_type_ = (int)MetricType::L2;
};
} // namespace engine
} // namespace milvus
} // namespace zilliz
} // namespace engine
} // namespace milvus
} // namespace zilliz

67
cpp/src/db/Utils.cpp
View File

@ -19,11 +19,11 @@
#include "utils/CommonUtil.h"
#include "utils/Log.h"
#include <mutex>
#include <boost/filesystem.hpp>
#include <chrono>
#include <mutex>
#include <regex>
#include <vector>
#include <boost/filesystem.hpp>
namespace zilliz {
namespace milvus {
@ -32,20 +32,20 @@ namespace utils {
namespace {
const char *TABLES_FOLDER = "/tables/";
const char* TABLES_FOLDER = "/tables/";
uint64_t index_file_counter = 0;
std::mutex index_file_counter_mutex;
std::string
ConstructParentFolder(const std::string &db_path, const meta::TableFileSchema &table_file) {
ConstructParentFolder(const std::string& db_path, const meta::TableFileSchema& table_file) {
std::string table_path = db_path + TABLES_FOLDER + table_file.table_id_;
std::string partition_path = table_path + "/" + std::to_string(table_file.date_);
return partition_path;
}
std::string
GetTableFileParentFolder(const DBMetaOptions &options, const meta::TableFileSchema &table_file) {
GetTableFileParentFolder(const DBMetaOptions& options, const meta::TableFileSchema& table_file) {
uint64_t path_count = options.slave_paths_.size() + 1;
std::string target_path = options.path_;
uint64_t index = 0;
@ -70,19 +70,18 @@ GetTableFileParentFolder(const DBMetaOptions &options, const meta::TableFileSche
return ConstructParentFolder(target_path, table_file);
}
} // namespace
} // namespace
int64_t
GetMicroSecTimeStamp() {
auto now = std::chrono::system_clock::now();
auto micros = std::chrono::duration_cast<std::chrono::microseconds>(
now.time_since_epoch()).count();
auto micros = std::chrono::duration_cast<std::chrono::microseconds>(now.time_since_epoch()).count();
return micros;
}
Status
CreateTablePath(const DBMetaOptions &options, const std::string &table_id) {
CreateTablePath(const DBMetaOptions& options, const std::string& table_id) {
std::string db_path = options.path_;
std::string table_path = db_path + TABLES_FOLDER + table_id;
auto status = server::CommonUtil::CreateDirectory(table_path);
@ -91,7 +90,7 @@ CreateTablePath(const DBMetaOptions &options, const std::string &table_id) {
return status;
}
for (auto &path : options.slave_paths_) {
for (auto& path : options.slave_paths_) {
table_path = path + TABLES_FOLDER + table_id;
status = server::CommonUtil::CreateDirectory(table_path);
if (!status.ok()) {
@ -104,17 +103,16 @@ CreateTablePath(const DBMetaOptions &options, const std::string &table_id) {
}
Status
DeleteTablePath(const DBMetaOptions &options, const std::string &table_id, bool force) {
DeleteTablePath(const DBMetaOptions& options, const std::string& table_id, bool force) {
std::vector<std::string> paths = options.slave_paths_;
paths.push_back(options.path_);
for (auto &path : paths) {
for (auto& path : paths) {
std::string table_path = path + TABLES_FOLDER + table_id;
if (force) {
boost::filesystem::remove_all(table_path);
ENGINE_LOG_DEBUG << "Remove table folder: " << table_path;
} else if (boost::filesystem::exists(table_path) &&
boost::filesystem::is_empty(table_path)) {
} else if (boost::filesystem::exists(table_path) && boost::filesystem::is_empty(table_path)) {
boost::filesystem::remove_all(table_path);
ENGINE_LOG_DEBUG << "Remove table folder: " << table_path;
}
@ -124,7 +122,7 @@ DeleteTablePath(const DBMetaOptions &options, const std::string &table_id, bool
}
Status
CreateTableFilePath(const DBMetaOptions &options, meta::TableFileSchema &table_file) {
CreateTableFilePath(const DBMetaOptions& options, meta::TableFileSchema& table_file) {
std::string parent_path = GetTableFileParentFolder(options, table_file);
auto status = server::CommonUtil::CreateDirectory(parent_path);
@ -139,14 +137,14 @@ CreateTableFilePath(const DBMetaOptions &options, meta::TableFileSchema &table_f
}
Status
GetTableFilePath(const DBMetaOptions &options, meta::TableFileSchema &table_file) {
GetTableFilePath(const DBMetaOptions& options, meta::TableFileSchema& table_file) {
std::string parent_path = ConstructParentFolder(options.path_, table_file);
std::string file_path = parent_path + "/" + table_file.file_id_;
if (boost::filesystem::exists(file_path)) {
table_file.location_ = file_path;
return Status::OK();
} else {
for (auto &path : options.slave_paths_) {
for (auto& path : options.slave_paths_) {
parent_path = ConstructParentFolder(path, table_file);
file_path = parent_path + "/" + table_file.file_id_;
if (boost::filesystem::exists(file_path)) {
@ -157,28 +155,26 @@ GetTableFilePath(const DBMetaOptions &options, meta::TableFileSchema &table_file
}
std::string msg = "Table file doesn't exist: " + file_path;
ENGINE_LOG_ERROR << msg << " in path: " << options.path_
<< " for table: " << table_file.table_id_;
ENGINE_LOG_ERROR << msg << " in path: " << options.path_ << " for table: " << table_file.table_id_;
return Status(DB_ERROR, msg);
}
Status
DeleteTableFilePath(const DBMetaOptions &options, meta::TableFileSchema &table_file) {
DeleteTableFilePath(const DBMetaOptions& options, meta::TableFileSchema& table_file) {
utils::GetTableFilePath(options, table_file);
boost::filesystem::remove(table_file.location_);
return Status::OK();
}
bool
IsSameIndex(const TableIndex &index1, const TableIndex &index2) {
return index1.engine_type_ == index2.engine_type_
&& index1.nlist_ == index2.nlist_
&& index1.metric_type_ == index2.metric_type_;
IsSameIndex(const TableIndex& index1, const TableIndex& index2) {
return index1.engine_type_ == index2.engine_type_ && index1.nlist_ == index2.nlist_ &&
index1.metric_type_ == index2.metric_type_;
}
meta::DateT
GetDate(const std::time_t &t, int day_delta) {
GetDate(const std::time_t& t, int day_delta) {
struct tm ltm;
localtime_r(&t, &ltm);
if (day_delta > 0) {
@ -211,20 +207,15 @@ GetDate() {
// URI format: dialect://username:password@host:port/database
Status
ParseMetaUri(const std::string &uri, MetaUriInfo &info) {
ParseMetaUri(const std::string& uri, MetaUriInfo& info) {
std::string dialect_regex = "(.*)";
std::string username_tegex = "(.*)";
std::string password_regex = "(.*)";
std::string host_regex = "(.*)";
std::string port_regex = "(.*)";
std::string db_name_regex = "(.*)";
std::string uri_regex_str =
dialect_regex + "\\:\\/\\/" +
username_tegex + "\\:" +
password_regex + "\\@" +
host_regex + "\\:" +
port_regex + "\\/" +
db_name_regex;
std::string uri_regex_str = dialect_regex + "\\:\\/\\/" + username_tegex + "\\:" + password_regex + "\\@" +
host_regex + "\\:" + port_regex + "\\/" + db_name_regex;
std::regex uri_regex(uri_regex_str);
std::smatch pieces_match;
@ -237,7 +228,7 @@ ParseMetaUri(const std::string &uri, MetaUriInfo &info) {
info.port_ = pieces_match[5].str();
info.db_name_ = pieces_match[6].str();
//TODO: verify host, port...
// TODO: verify host, port...
} else {
return Status(DB_INVALID_META_URI, "Invalid meta uri: " + uri);
}
@ -245,7 +236,7 @@ ParseMetaUri(const std::string &uri, MetaUriInfo &info) {
return Status::OK();
}
} // namespace utils
} // namespace engine
} // namespace milvus
} // namespace zilliz
} // namespace utils
} // namespace engine
} // namespace milvus
} // namespace zilliz

28
cpp/src/db/Utils.h
View File

@ -18,11 +18,11 @@
#pragma once
#include "Options.h"
#include "db/meta/MetaTypes.h"
#include "db/Types.h"
#include "db/meta/MetaTypes.h"
#include <string>
#include <ctime>
#include <string>
namespace zilliz {
namespace milvus {
@ -33,22 +33,22 @@ int64_t
GetMicroSecTimeStamp();
Status
CreateTablePath(const DBMetaOptions &options, const std::string &table_id);
CreateTablePath(const DBMetaOptions& options, const std::string& table_id);
Status
DeleteTablePath(const DBMetaOptions &options, const std::string &table_id, bool force = true);
DeleteTablePath(const DBMetaOptions& options, const std::string& table_id, bool force = true);
Status
CreateTableFilePath(const DBMetaOptions &options, meta::TableFileSchema &table_file);
CreateTableFilePath(const DBMetaOptions& options, meta::TableFileSchema& table_file);
Status
GetTableFilePath(const DBMetaOptions &options, meta::TableFileSchema &table_file);
GetTableFilePath(const DBMetaOptions& options, meta::TableFileSchema& table_file);
Status
DeleteTableFilePath(const DBMetaOptions &options, meta::TableFileSchema &table_file);
DeleteTableFilePath(const DBMetaOptions& options, meta::TableFileSchema& table_file);
bool
IsSameIndex(const TableIndex &index1, const TableIndex &index2);
IsSameIndex(const TableIndex& index1, const TableIndex& index2);
meta::DateT
GetDate(const std::time_t &t, int day_delta = 0);
GetDate(const std::time_t& t, int day_delta = 0);
meta::DateT
GetDate();
meta::DateT
@ -64,9 +64,9 @@ struct MetaUriInfo {
};
Status
ParseMetaUri(const std::string &uri, MetaUriInfo &info);
ParseMetaUri(const std::string& uri, MetaUriInfo& info);
} // namespace utils
} // namespace engine
} // namespace milvus
} // namespace zilliz
} // namespace utils
} // namespace engine
} // namespace milvus
} // namespace zilliz

13
cpp/src/db/engine/EngineFactory.cpp
View File

@ -26,17 +26,14 @@ namespace milvus {
namespace engine {
ExecutionEnginePtr
EngineFactory::Build(uint16_t dimension,
const std::string &location,
EngineType index_type,
MetricType metric_type,
EngineFactory::Build(uint16_t dimension, const std::string& location, EngineType index_type, MetricType metric_type,
int32_t nlist) {
if (index_type == EngineType::INVALID) {
ENGINE_LOG_ERROR << "Unsupported engine type";
return nullptr;
}
ENGINE_LOG_DEBUG << "EngineFactory index type: " << (int) index_type;
ENGINE_LOG_DEBUG << "EngineFactory index type: " << (int)index_type;
ExecutionEnginePtr execution_engine_ptr =
std::make_shared<ExecutionEngineImpl>(dimension, location, index_type, metric_type, nlist);
@ -44,6 +41,6 @@ EngineFactory::Build(uint16_t dimension,
return execution_engine_ptr;
}
} // namespace engine
} // namespace milvus
} // namespace zilliz
} // namespace engine
} // namespace milvus
} // namespace zilliz

16
cpp/src/db/engine/EngineFactory.h
View File

@ -15,7 +15,6 @@
// specific language governing permissions and limitations
// under the License.
#pragma once
#include "ExecutionEngine.h"
@ -29,14 +28,11 @@ namespace engine {
class EngineFactory {
public:
static ExecutionEnginePtr Build(uint16_t dimension,
const std::string &location,
EngineType index_type,
MetricType metric_type,
int32_t nlist);
static ExecutionEnginePtr
Build(uint16_t dimension, const std::string& location, EngineType index_type, MetricType metric_type,
int32_t nlist);
};
} // namespace engine
} // namespace milvus
} // namespace zilliz
} // namespace engine
} // namespace milvus
} // namespace zilliz

70
cpp/src/db/engine/ExecutionEngine.h
View File

@ -19,9 +19,9 @@
#include "utils/Status.h"
#include <vector>
#include <memory>
#include <string>
#include <vector>
namespace zilliz {
namespace milvus {
@ -43,52 +43,66 @@ enum class MetricType {
class ExecutionEngine {
public:
virtual Status AddWithIds(int64_t n, const float *xdata, const int64_t *xids) = 0;
virtual Status
AddWithIds(int64_t n, const float* xdata, const int64_t* xids) = 0;
virtual size_t Count() const = 0;
virtual size_t
Count() const = 0;
virtual size_t Size() const = 0;
virtual size_t
Size() const = 0;
virtual size_t Dimension() const = 0;
virtual size_t
Dimension() const = 0;
virtual size_t PhysicalSize() const = 0;
virtual size_t
PhysicalSize() const = 0;
virtual Status Serialize() = 0;
virtual Status
Serialize() = 0;
virtual Status Load(bool to_cache = true) = 0;
virtual Status
Load(bool to_cache = true) = 0;
virtual Status CopyToGpu(uint64_t device_id) = 0;
virtual Status
CopyToGpu(uint64_t device_id) = 0;
virtual Status CopyToCpu() = 0;
virtual Status
CopyToCpu() = 0;
virtual std::shared_ptr<ExecutionEngine> Clone() = 0;
virtual std::shared_ptr<ExecutionEngine>
Clone() = 0;
virtual Status Merge(const std::string &location) = 0;
virtual Status
Merge(const std::string& location) = 0;
virtual Status Search(int64_t n,
const float *data,
int64_t k,
int64_t nprobe,
float *distances,
int64_t *labels) const = 0;
virtual Status
Search(int64_t n, const float* data, int64_t k, int64_t nprobe, float* distances, int64_t* labels) const = 0;
virtual std::shared_ptr<ExecutionEngine> BuildIndex(const std::string &location, EngineType engine_type) = 0;
virtual std::shared_ptr<ExecutionEngine>
BuildIndex(const std::string& location, EngineType engine_type) = 0;
virtual Status Cache() = 0;
virtual Status
Cache() = 0;
virtual Status GpuCache(uint64_t gpu_id) = 0;
virtual Status
GpuCache(uint64_t gpu_id) = 0;
virtual Status Init() = 0;
virtual Status
Init() = 0;
virtual EngineType IndexEngineType() const = 0;
virtual EngineType
IndexEngineType() const = 0;
virtual MetricType IndexMetricType() const = 0;
virtual MetricType
IndexMetricType() const = 0;
virtual std::string GetLocation() const = 0;
virtual std::string
GetLocation() const = 0;
};
using ExecutionEnginePtr = std::shared_ptr<ExecutionEngine>;
} // namespace engine
} // namespace milvus
} // namespace zilliz
} // namespace engine
} // namespace milvus
} // namespace zilliz

88
cpp/src/db/engine/ExecutionEngineImpl.cpp
View File

@ -16,20 +16,20 @@
// under the License.
#include "db/engine/ExecutionEngineImpl.h"
#include "cache/GpuCacheMgr.h"
#include "cache/CpuCacheMgr.h"
#include "cache/GpuCacheMgr.h"
#include "metrics/Metrics.h"
#include "utils/Log.h"
#include "utils/CommonUtil.h"
#include "utils/Exception.h"
#include "utils/Log.h"
#include "src/wrapper/VecIndex.h"
#include "src/wrapper/VecImpl.h"
#include "knowhere/common/Exception.h"
#include "knowhere/common/Config.h"
#include "wrapper/ConfAdapterMgr.h"
#include "wrapper/ConfAdapter.h"
#include "knowhere/common/Exception.h"
#include "server/Config.h"
#include "src/wrapper/VecImpl.h"
#include "src/wrapper/VecIndex.h"
#include "wrapper/ConfAdapter.h"
#include "wrapper/ConfAdapterMgr.h"
#include <stdexcept>
#include <utility>
@ -38,17 +38,9 @@ namespace zilliz {
namespace milvus {
namespace engine {
ExecutionEngineImpl::ExecutionEngineImpl(uint16_t dimension,
const std::string &location,
EngineType index_type,
MetricType metric_type,
int32_t nlist)
: location_(location),
dim_(dimension),
index_type_(index_type),
metric_type_(metric_type),
nlist_(nlist) {
ExecutionEngineImpl::ExecutionEngineImpl(uint16_t dimension, const std::string& location, EngineType index_type,
MetricType metric_type, int32_t nlist)
: location_(location), dim_(dimension), index_type_(index_type), metric_type_(metric_type), nlist_(nlist) {
index_ = CreatetVecIndex(EngineType::FAISS_IDMAP);
if (!index_) {
throw Exception(DB_ERROR, "Could not create VecIndex");
@ -57,8 +49,7 @@ ExecutionEngineImpl::ExecutionEngineImpl(uint16_t dimension,
TempMetaConf temp_conf;
temp_conf.gpu_id = gpu_num_;
temp_conf.dim = dimension;
temp_conf.metric_type = (metric_type_ == MetricType::IP) ?
knowhere::METRICTYPE::IP : knowhere::METRICTYPE::L2;
temp_conf.metric_type = (metric_type_ == MetricType::IP) ? knowhere::METRICTYPE::IP : knowhere::METRICTYPE::L2;
auto adapter = AdapterMgr::GetInstance().GetAdapter(index_->GetType());
auto conf = adapter->Match(temp_conf);
@ -68,16 +59,9 @@ ExecutionEngineImpl::ExecutionEngineImpl(uint16_t dimension,
}
}
ExecutionEngineImpl::ExecutionEngineImpl(VecIndexPtr index,
const std::string &location,
EngineType index_type,
MetricType metric_type,
int32_t nlist)
: index_(std::move(index)),
location_(location),
index_type_(index_type),
metric_type_(metric_type),
nlist_(nlist) {
ExecutionEngineImpl::ExecutionEngineImpl(VecIndexPtr index, const std::string& location, EngineType index_type,
MetricType metric_type, int32_t nlist)
: index_(std::move(index)), location_(location), index_type_(index_type), metric_type_(metric_type), nlist_(nlist) {
}
VecIndexPtr
@ -109,7 +93,7 @@ ExecutionEngineImpl::CreatetVecIndex(EngineType type) {
}
Status
ExecutionEngineImpl::AddWithIds(int64_t n, const float *xdata, const int64_t *xids) {
ExecutionEngineImpl::AddWithIds(int64_t n, const float* xdata, const int64_t* xids) {
auto status = index_->Add(n, xdata, xids);
return status;
}
@ -125,7 +109,7 @@ ExecutionEngineImpl::Count() const {
size_t
ExecutionEngineImpl::Size() const {
return (size_t) (Count() * Dimension()) * sizeof(float);
return (size_t)(Count() * Dimension()) * sizeof(float);
}
size_t
@ -164,7 +148,7 @@ ExecutionEngineImpl::Load(bool to_cache) {
} else {
ENGINE_LOG_DEBUG << "Disk io from: " << location_;
}
} catch (std::exception &e) {
} catch (std::exception& e) {
ENGINE_LOG_ERROR << e.what();
return Status(DB_ERROR, e.what());
}
@ -191,7 +175,7 @@ ExecutionEngineImpl::CopyToGpu(uint64_t device_id) {
try {
index_ = index_->CopyToGpu(device_id);
ENGINE_LOG_DEBUG << "CPU to GPU" << device_id;
} catch (std::exception &e) {
} catch (std::exception& e) {
ENGINE_LOG_ERROR << e.what();
return Status(DB_ERROR, e.what());
}
@ -219,7 +203,7 @@ ExecutionEngineImpl::CopyToCpu() {
try {
index_ = index_->CopyToCpu();
ENGINE_LOG_DEBUG << "GPU to CPU";
} catch (std::exception &e) {
} catch (std::exception& e) {
ENGINE_LOG_ERROR << e.what();
return Status(DB_ERROR, e.what());
}
@ -245,7 +229,7 @@ ExecutionEngineImpl::Clone() {
}
Status
ExecutionEngineImpl::Merge(const std::string &location) {
ExecutionEngineImpl::Merge(const std::string& location) {
if (location == location_) {
return Status(DB_ERROR, "Cannot Merge Self");
}
@ -257,7 +241,7 @@ ExecutionEngineImpl::Merge(const std::string &location) {
double physical_size = server::CommonUtil::GetFileSize(location);
server::CollectExecutionEngineMetrics metrics(physical_size);
to_merge = read_index(location);
} catch (std::exception &e) {
} catch (std::exception& e) {
ENGINE_LOG_ERROR << e.what();
return Status(DB_ERROR, e.what());
}
@ -280,7 +264,7 @@ ExecutionEngineImpl::Merge(const std::string &location) {
}
ExecutionEnginePtr
ExecutionEngineImpl::BuildIndex(const std::string &location, EngineType engine_type) {
ExecutionEngineImpl::BuildIndex(const std::string& location, EngineType engine_type) {
ENGINE_LOG_DEBUG << "Build index file: " << location << " from: " << location_;
auto from_index = std::dynamic_pointer_cast<BFIndex>(index_);
@ -298,29 +282,23 @@ ExecutionEngineImpl::BuildIndex(const std::string &location, EngineType engine_t
temp_conf.gpu_id = gpu_num_;
temp_conf.dim = Dimension();
temp_conf.nlist = nlist_;
temp_conf.metric_type = (metric_type_ == MetricType::IP) ?
knowhere::METRICTYPE::IP : knowhere::METRICTYPE::L2;
temp_conf.metric_type = (metric_type_ == MetricType::IP) ? knowhere::METRICTYPE::IP : knowhere::METRICTYPE::L2;
temp_conf.size = Count();
auto adapter = AdapterMgr::GetInstance().GetAdapter(to_index->GetType());
auto conf = adapter->Match(temp_conf);
auto status = to_index->BuildAll(Count(),
from_index->GetRawVectors(),
from_index->GetRawIds(),
conf);
if (!status.ok()) { throw Exception(DB_ERROR, status.message()); }
auto status = to_index->BuildAll(Count(), from_index->GetRawVectors(), from_index->GetRawIds(), conf);
if (!status.ok()) {
throw Exception(DB_ERROR, status.message());
}
return std::make_shared<ExecutionEngineImpl>(to_index, location, engine_type, metric_type_, nlist_);
}
Status
ExecutionEngineImpl::Search(int64_t n,
const float *data,
int64_t k,
int64_t nprobe,
float *distances,
int64_t *labels) const {
ExecutionEngineImpl::Search(int64_t n, const float* data, int64_t k, int64_t nprobe, float* distances,
int64_t* labels) const {
if (index_ == nullptr) {
ENGINE_LOG_ERROR << "ExecutionEngineImpl: index is null, failed to search";
return Status(DB_ERROR, "index is null");
@ -362,13 +340,13 @@ ExecutionEngineImpl::GpuCache(uint64_t gpu_id) {
// TODO(linxj): remove.
Status
ExecutionEngineImpl::Init() {
server::Config &config = server::Config::GetInstance();
server::Config& config = server::Config::GetInstance();
Status s = config.GetDBConfigBuildIndexGPU(gpu_num_);
if (!s.ok()) return s;
return Status::OK();
}
} // namespace engine
} // namespace milvus
} // namespace zilliz
} // namespace engine
} // namespace milvus
} // namespace zilliz

84
cpp/src/db/engine/ExecutionEngineImpl.h
View File

@ -29,71 +29,81 @@ namespace engine {
class ExecutionEngineImpl : public ExecutionEngine {
public:
ExecutionEngineImpl(uint16_t dimension,
const std::string &location,
EngineType index_type,
MetricType metric_type,
ExecutionEngineImpl(uint16_t dimension, const std::string& location, EngineType index_type, MetricType metric_type,
int32_t nlist);
ExecutionEngineImpl(VecIndexPtr index,
const std::string &location,
EngineType index_type,
MetricType metric_type,
ExecutionEngineImpl(VecIndexPtr index, const std::string& location, EngineType index_type, MetricType metric_type,
int32_t nlist);
Status AddWithIds(int64_t n, const float *xdata, const int64_t *xids) override;
Status
AddWithIds(int64_t n, const float* xdata, const int64_t* xids) override;
size_t Count() const override;
size_t
Count() const override;
size_t Size() const override;
size_t
Size() const override;
size_t Dimension() const override;
size_t
Dimension() const override;
size_t PhysicalSize() const override;
size_t
PhysicalSize() const override;
Status Serialize() override;
Status
Serialize() override;
Status Load(bool to_cache) override;
Status
Load(bool to_cache) override;
Status CopyToGpu(uint64_t device_id) override;
Status
CopyToGpu(uint64_t device_id) override;
Status CopyToCpu() override;
Status
CopyToCpu() override;
ExecutionEnginePtr Clone() override;
ExecutionEnginePtr
Clone() override;
Status Merge(const std::string &location) override;
Status
Merge(const std::string& location) override;
Status Search(int64_t n,
const float *data,
int64_t k,
int64_t nprobe,
float *distances,
int64_t *labels) const override;
Status
Search(int64_t n, const float* data, int64_t k, int64_t nprobe, float* distances, int64_t* labels) const override;
ExecutionEnginePtr BuildIndex(const std::string &location, EngineType engine_type) override;
ExecutionEnginePtr
BuildIndex(const std::string& location, EngineType engine_type) override;
Status Cache() override;
Status
Cache() override;
Status GpuCache(uint64_t gpu_id) override;
Status
GpuCache(uint64_t gpu_id) override;
Status Init() override;
Status
Init() override;
EngineType IndexEngineType() const override {
EngineType
IndexEngineType() const override {
return index_type_;
}
MetricType IndexMetricType() const override {
MetricType
IndexMetricType() const override {
return metric_type_;
}
std::string GetLocation() const override {
std::string
GetLocation() const override {
return location_;
}
private:
VecIndexPtr CreatetVecIndex(EngineType type);
VecIndexPtr
CreatetVecIndex(EngineType type);
VecIndexPtr Load(const std::string &location);
VecIndexPtr
Load(const std::string& location);
protected:
VecIndexPtr index_ = nullptr;
@ -107,6 +117,6 @@ class ExecutionEngineImpl : public ExecutionEngine {
int32_t gpu_num_ = 0;
};
} // namespace engine
} // namespace milvus
} // namespace zilliz
} // namespace engine
} // namespace milvus
} // namespace zilliz

32
cpp/src/db/insert/MemManager.h
View File

@ -15,14 +15,13 @@
// specific language governing permissions and limitations
// under the License.
#pragma once
#include "utils/Status.h"
#include "db/Types.h"
#include "utils/Status.h"
#include <set>
#include <memory>
#include <set>
#include <string>
namespace zilliz {
@ -31,22 +30,27 @@ namespace engine {
class MemManager {
public:
virtual Status InsertVectors(const std::string &table_id,
size_t n, const float *vectors, IDNumbers &vector_ids) = 0;
virtual Status
InsertVectors(const std::string& table_id, size_t n, const float* vectors, IDNumbers& vector_ids) = 0;
virtual Status Serialize(std::set<std::string> &table_ids) = 0;
virtual Status
Serialize(std::set<std::string>& table_ids) = 0;
virtual Status EraseMemVector(const std::string &table_id) = 0;
virtual Status
EraseMemVector(const std::string& table_id) = 0;
virtual size_t GetCurrentMutableMem() = 0;
virtual size_t
GetCurrentMutableMem() = 0;
virtual size_t GetCurrentImmutableMem() = 0;
virtual size_t
GetCurrentImmutableMem() = 0;
virtual size_t GetCurrentMem() = 0;
}; // MemManagerAbstract
virtual size_t
GetCurrentMem() = 0;
}; // MemManagerAbstract
using MemManagerPtr = std::shared_ptr<MemManager>;
} // namespace engine
} // namespace milvus
} // namespace zilliz
} // namespace engine
} // namespace milvus
} // namespace zilliz

42
cpp/src/db/insert/MemManagerImpl.cpp
View File

@ -15,11 +15,10 @@
// specific language governing permissions and limitations
// under the License.
#include "db/insert/MemManagerImpl.h"
#include "VectorSource.h"
#include "utils/Log.h"
#include "db/Constants.h"
#include "utils/Log.h"
#include <thread>
@ -28,7 +27,7 @@ namespace milvus {
namespace engine {
MemTablePtr
MemManagerImpl::GetMemByTable(const std::string &table_id) {
MemManagerImpl::GetMemByTable(const std::string& table_id) {
auto memIt = mem_id_map_.find(table_id);
if (memIt != mem_id_map_.end()) {
return memIt->second;
@ -39,10 +38,7 @@ MemManagerImpl::GetMemByTable(const std::string &table_id) {
}
Status
MemManagerImpl::InsertVectors(const std::string &table_id_,
size_t n_,
const float *vectors_,
IDNumbers &vector_ids_) {
MemManagerImpl::InsertVectors(const std::string& table_id_, size_t n_, const float* vectors_, IDNumbers& vector_ids_) {
while (GetCurrentMem() > options_.insert_buffer_size_) {
std::this_thread::sleep_for(std::chrono::milliseconds(1));
}
@ -53,10 +49,8 @@ MemManagerImpl::InsertVectors(const std::string &table_id_,
}
Status
MemManagerImpl::InsertVectorsNoLock(const std::string &table_id,
size_t n,
const float *vectors,
IDNumbers &vector_ids) {
MemManagerImpl::InsertVectorsNoLock(const std::string& table_id, size_t n, const float* vectors,
IDNumbers& vector_ids) {
MemTablePtr mem = GetMemByTable(table_id);
VectorSourcePtr source = std::make_shared<VectorSource>(n, vectors);
@ -73,9 +67,9 @@ Status
MemManagerImpl::ToImmutable() {
std::unique_lock<std::mutex> lock(mutex_);
MemIdMap temp_map;
for (auto &kv : mem_id_map_) {
for (auto& kv : mem_id_map_) {
if (kv.second->Empty()) {
//empty table, no need to serialize
// empty table, no need to serialize
temp_map.insert(kv);
} else {
immu_mem_list_.push_back(kv.second);
@ -87,11 +81,11 @@ MemManagerImpl::ToImmutable() {
}
Status
MemManagerImpl::Serialize(std::set<std::string> &table_ids) {
MemManagerImpl::Serialize(std::set<std::string>& table_ids) {
ToImmutable();
std::unique_lock<std::mutex> lock(serialization_mtx_);
table_ids.clear();
for (auto &mem : immu_mem_list_) {
for (auto& mem : immu_mem_list_) {
mem->Serialize();
table_ids.insert(mem->GetTableId());
}
@ -100,16 +94,16 @@ MemManagerImpl::Serialize(std::set<std::string> &table_ids) {
}
Status
MemManagerImpl::EraseMemVector(const std::string &table_id) {
{//erase MemVector from rapid-insert cache
MemManagerImpl::EraseMemVector(const std::string& table_id) {
{ // erase MemVector from rapid-insert cache
std::unique_lock<std::mutex> lock(mutex_);
mem_id_map_.erase(table_id);
}
{//erase MemVector from serialize cache
{ // erase MemVector from serialize cache
std::unique_lock<std::mutex> lock(serialization_mtx_);
MemList temp_list;
for (auto &mem : immu_mem_list_) {
for (auto& mem : immu_mem_list_) {
if (mem->GetTableId() != table_id) {
temp_list.push_back(mem);
}
@ -123,7 +117,7 @@ MemManagerImpl::EraseMemVector(const std::string &table_id) {
size_t
MemManagerImpl::GetCurrentMutableMem() {
size_t total_mem = 0;
for (auto &kv : mem_id_map_) {
for (auto& kv : mem_id_map_) {
auto memTable = kv.second;
total_mem += memTable->GetCurrentMem();
}
@ -133,7 +127,7 @@ MemManagerImpl::GetCurrentMutableMem() {
size_t
MemManagerImpl::GetCurrentImmutableMem() {
size_t total_mem = 0;
for (auto &mem_table : immu_mem_list_) {
for (auto& mem_table : immu_mem_list_) {
total_mem += mem_table->GetCurrentMem();
}
return total_mem;
@ -144,6 +138,6 @@ MemManagerImpl::GetCurrentMem() {
return GetCurrentMutableMem() + GetCurrentImmutableMem();
}
} // namespace engine
} // namespace milvus
} // namespace zilliz
} // namespace engine
} // namespace milvus
} // namespace zilliz

51
cpp/src/db/insert/MemManagerImpl.h
View File

@ -15,21 +15,20 @@
// specific language governing permissions and limitations
// under the License.
#pragma once
#include "MemTable.h"
#include "MemManager.h"
#include "MemTable.h"
#include "db/meta/Meta.h"
#include "utils/Status.h"
#include <map>
#include <set>
#include <vector>
#include <string>
#include <ctime>
#include <map>
#include <memory>
#include <mutex>
#include <set>
#include <string>
#include <vector>
namespace zilliz {
namespace milvus {
@ -39,29 +38,35 @@ class MemManagerImpl : public MemManager {
public:
using Ptr = std::shared_ptr<MemManagerImpl>;
MemManagerImpl(const meta::MetaPtr &meta, const DBOptions &options)
: meta_(meta), options_(options) {
MemManagerImpl(const meta::MetaPtr& meta, const DBOptions& options) : meta_(meta), options_(options) {
}
Status InsertVectors(const std::string &table_id,
size_t n, const float *vectors, IDNumbers &vector_ids) override;
Status
InsertVectors(const std::string& table_id, size_t n, const float* vectors, IDNumbers& vector_ids) override;
Status Serialize(std::set<std::string> &table_ids) override;
Status
Serialize(std::set<std::string>& table_ids) override;
Status EraseMemVector(const std::string &table_id) override;
Status
EraseMemVector(const std::string& table_id) override;
size_t GetCurrentMutableMem() override;
size_t
GetCurrentMutableMem() override;
size_t GetCurrentImmutableMem() override;
size_t
GetCurrentImmutableMem() override;
size_t GetCurrentMem() override;
size_t
GetCurrentMem() override;
private:
MemTablePtr GetMemByTable(const std::string &table_id);
MemTablePtr
GetMemByTable(const std::string& table_id);
Status InsertVectorsNoLock(const std::string &table_id,
size_t n, const float *vectors, IDNumbers &vector_ids);
Status ToImmutable();
Status
InsertVectorsNoLock(const std::string& table_id, size_t n, const float* vectors, IDNumbers& vector_ids);
Status
ToImmutable();
using MemIdMap = std::map<std::string, MemTablePtr>;
using MemList = std::vector<MemTablePtr>;
@ -71,8 +76,8 @@ class MemManagerImpl : public MemManager {
DBOptions options_;
std::mutex mutex_;
std::mutex serialization_mtx_;
}; // NewMemManager
}; // NewMemManager
} // namespace engine
} // namespace milvus
} // namespace zilliz
} // namespace engine
} // namespace milvus
} // namespace zilliz

16
cpp/src/db/insert/MemMenagerFactory.cpp
View File

@ -17,26 +17,26 @@
#include "db/insert/MemMenagerFactory.h"
#include "MemManagerImpl.h"
#include "utils/Log.h"
#include "utils/Exception.h"
#include "utils/Log.h"
#include <stdlib.h>
#include <time.h>
#include <sstream>
#include <cstdlib>
#include <string>
#include <regex>
#include <memory>
#include <regex>
#include <sstream>
#include <string>
namespace zilliz {
namespace milvus {
namespace engine {
MemManagerPtr
MemManagerFactory::Build(const std::shared_ptr<meta::Meta> &meta, const DBOptions &options) {
MemManagerFactory::Build(const std::shared_ptr<meta::Meta>& meta, const DBOptions& options) {
return std::make_shared<MemManagerImpl>(meta, options);
}
} // namespace engine
} // namespace milvus
} // namespace zilliz
} // namespace engine
} // namespace milvus
} // namespace zilliz

9
cpp/src/db/insert/MemMenagerFactory.h
View File

@ -28,9 +28,10 @@ namespace engine {
class MemManagerFactory {
public:
static MemManagerPtr Build(const std::shared_ptr<meta::Meta> &meta, const DBOptions &options);
static MemManagerPtr
Build(const std::shared_ptr<meta::Meta>& meta, const DBOptions& options);
};
} // namespace engine
} // namespace milvus
} // namespace zilliz
} // namespace engine
} // namespace milvus
} // namespace zilliz

23
cpp/src/db/insert/MemTable.cpp
View File

@ -15,7 +15,6 @@
// specific language governing permissions and limitations
// under the License.
#include "db/insert/MemTable.h"
#include "utils/Log.h"
@ -26,16 +25,12 @@ namespace zilliz {
namespace milvus {
namespace engine {
MemTable::MemTable(const std::string &table_id,
const meta::MetaPtr &meta,
const DBOptions &options) :
table_id_(table_id),
meta_(meta),
options_(options) {
MemTable::MemTable(const std::string& table_id, const meta::MetaPtr& meta, const DBOptions& options)
: table_id_(table_id), meta_(meta), options_(options) {
}
Status
MemTable::Add(VectorSourcePtr &source, IDNumbers &vector_ids) {
MemTable::Add(VectorSourcePtr& source, IDNumbers& vector_ids) {
while (!source->AllAdded()) {
MemTableFilePtr current_mem_table_file;
if (!mem_table_file_list_.empty()) {
@ -63,7 +58,7 @@ MemTable::Add(VectorSourcePtr &source, IDNumbers &vector_ids) {
}
void
MemTable::GetCurrentMemTableFile(MemTableFilePtr &mem_table_file) {
MemTable::GetCurrentMemTableFile(MemTableFilePtr& mem_table_file) {
mem_table_file = mem_table_file_list_.back();
}
@ -92,7 +87,7 @@ MemTable::Empty() {
return mem_table_file_list_.empty();
}
const std::string &
const std::string&
MemTable::GetTableId() const {
return table_id_;
}
@ -101,12 +96,12 @@ size_t
MemTable::GetCurrentMem() {
std::lock_guard<std::mutex> lock(mutex_);
size_t total_mem = 0;
for (auto &mem_table_file : mem_table_file_list_) {
for (auto& mem_table_file : mem_table_file_list_) {
total_mem += mem_table_file->GetCurrentMem();
}
return total_mem;
}
} // namespace engine
} // namespace milvus
} // namespace zilliz
} // namespace engine
} // namespace milvus
} // namespace zilliz

36
cpp/src/db/insert/MemTable.h
View File

@ -15,17 +15,16 @@
// specific language governing permissions and limitations
// under the License.
#pragma once
#include "MemTableFile.h"
#include "VectorSource.h"
#include "utils/Status.h"
#include <mutex>
#include <vector>
#include <memory>
#include <mutex>
#include <string>
#include <vector>
namespace zilliz {
namespace milvus {
@ -35,21 +34,28 @@ class MemTable {
public:
using MemTableFileList = std::vector<MemTableFilePtr>;
MemTable(const std::string &table_id, const meta::MetaPtr &meta, const DBOptions &options);
MemTable(const std::string& table_id, const meta::MetaPtr& meta, const DBOptions& options);
Status Add(VectorSourcePtr &source, IDNumbers &vector_ids);
Status
Add(VectorSourcePtr& source, IDNumbers& vector_ids);
void GetCurrentMemTableFile(MemTableFilePtr &mem_table_file);
void
GetCurrentMemTableFile(MemTableFilePtr& mem_table_file);
size_t GetTableFileCount();
size_t
GetTableFileCount();
Status Serialize();
Status
Serialize();
bool Empty();
bool
Empty();
const std::string &GetTableId() const;
const std::string&
GetTableId() const;
size_t GetCurrentMem();
size_t
GetCurrentMem();
private:
const std::string table_id_;
@ -61,10 +67,10 @@ class MemTable {
DBOptions options_;
std::mutex mutex_;
}; //MemTable
}; // MemTable
using MemTablePtr = std::shared_ptr<MemTable>;
} // namespace engine
} // namespace milvus
} // namespace zilliz
} // namespace engine
} // namespace milvus
} // namespace zilliz

38
cpp/src/db/insert/MemTableFile.cpp
View File

@ -15,7 +15,6 @@
// specific language governing permissions and limitations
// under the License.
#include "db/insert/MemTableFile.h"
#include "db/Constants.h"
#include "db/engine/EngineFactory.h"
@ -29,20 +28,14 @@ namespace zilliz {
namespace milvus {
namespace engine {
MemTableFile::MemTableFile(const std::string &table_id,
const meta::MetaPtr &meta,
const DBOptions &options) :
table_id_(table_id),
meta_(meta),
options_(options) {
MemTableFile::MemTableFile(const std::string& table_id, const meta::MetaPtr& meta, const DBOptions& options)
: table_id_(table_id), meta_(meta), options_(options) {
current_mem_ = 0;
auto status = CreateTableFile();
if (status.ok()) {
execution_engine_ = EngineFactory::Build(table_file_schema_.dimension_,
table_file_schema_.location_,
(EngineType) table_file_schema_.engine_type_,
(MetricType) table_file_schema_.metric_type_,
table_file_schema_.nlist_);
execution_engine_ = EngineFactory::Build(
table_file_schema_.dimension_, table_file_schema_.location_, (EngineType)table_file_schema_.engine_type_,
(MetricType)table_file_schema_.metric_type_, table_file_schema_.nlist_);
}
}
@ -61,10 +54,11 @@ MemTableFile::CreateTableFile() {
}
Status
MemTableFile::Add(const VectorSourcePtr &source, IDNumbers &vector_ids) {
MemTableFile::Add(const VectorSourcePtr& source, IDNumbers& vector_ids) {
if (table_file_schema_.dimension_ <= 0) {
std::string err_msg = "MemTableFile::Add: table_file_schema dimension = " +
std::to_string(table_file_schema_.dimension_) + ", table_id = " + table_file_schema_.table_id_;
std::to_string(table_file_schema_.dimension_) + ", table_id = " +
table_file_schema_.table_id_;
ENGINE_LOG_ERROR << err_msg;
return Status(DB_ERROR, "Not able to create table file");
}
@ -109,11 +103,11 @@ MemTableFile::Serialize() {
table_file_schema_.file_size_ = execution_engine_->PhysicalSize();
table_file_schema_.row_count_ = execution_engine_->Count();
//if index type isn't IDMAP, set file type to TO_INDEX if file size execeed index_file_size
//else set file type to RAW, no need to build index
if (table_file_schema_.engine_type_ != (int) EngineType::FAISS_IDMAP) {
table_file_schema_.file_type_ = (size >= table_file_schema_.index_file_size_) ?
meta::TableFileSchema::TO_INDEX : meta::TableFileSchema::RAW;
// if index type isn't IDMAP, set file type to TO_INDEX if file size execeed index_file_size
// else set file type to RAW, no need to build index
if (table_file_schema_.engine_type_ != (int)EngineType::FAISS_IDMAP) {
table_file_schema_.file_type_ = (size >= table_file_schema_.index_file_size_) ? meta::TableFileSchema::TO_INDEX
: meta::TableFileSchema::RAW;
} else {
table_file_schema_.file_type_ = meta::TableFileSchema::RAW;
}
@ -130,6 +124,6 @@ MemTableFile::Serialize() {
return status;
}
} // namespace engine
} // namespace milvus
} // namespace zilliz
} // namespace engine
} // namespace milvus
} // namespace zilliz

33
cpp/src/db/insert/MemTableFile.h
View File

@ -15,16 +15,15 @@
// specific language governing permissions and limitations
// under the License.
#pragma once
#include "VectorSource.h"
#include "db/meta/Meta.h"
#include "db/engine/ExecutionEngine.h"
#include "db/meta/Meta.h"
#include "utils/Status.h"
#include <string>
#include <memory>
#include <string>
namespace zilliz {
namespace milvus {
@ -32,20 +31,26 @@ namespace engine {
class MemTableFile {
public:
MemTableFile(const std::string &table_id, const meta::MetaPtr &meta, const DBOptions &options);
MemTableFile(const std::string& table_id, const meta::MetaPtr& meta, const DBOptions& options);
Status Add(const VectorSourcePtr &source, IDNumbers &vector_ids);
Status
Add(const VectorSourcePtr& source, IDNumbers& vector_ids);
size_t GetCurrentMem();
size_t
GetCurrentMem();
size_t GetMemLeft();
size_t
GetMemLeft();
bool IsFull();
bool
IsFull();
Status Serialize();
Status
Serialize();
private:
Status CreateTableFile();
Status
CreateTableFile();
private:
const std::string table_id_;
@ -55,10 +60,10 @@ class MemTableFile {
size_t current_mem_;
ExecutionEnginePtr execution_engine_;
}; //MemTableFile
}; // MemTableFile
using MemTableFilePtr = std::shared_ptr<MemTableFile>;
} // namespace engine
} // namespace milvus
} // namespace zilliz
} // namespace engine
} // namespace milvus
} // namespace zilliz

32
cpp/src/db/insert/VectorSource.cpp
View File

@ -15,35 +15,28 @@
// specific language governing permissions and limitations
// under the License.
#include "db/insert/VectorSource.h"
#include "db/engine/ExecutionEngine.h"
#include "db/engine/EngineFactory.h"
#include "utils/Log.h"
#include "db/engine/ExecutionEngine.h"
#include "metrics/Metrics.h"
#include "utils/Log.h"
namespace zilliz {
namespace milvus {
namespace engine {
VectorSource::VectorSource(const size_t &n,
const float *vectors) :
n_(n),
vectors_(vectors),
id_generator_(std::make_shared<SimpleIDGenerator>()) {
VectorSource::VectorSource(const size_t& n, const float* vectors)
: n_(n), vectors_(vectors), id_generator_(std::make_shared<SimpleIDGenerator>()) {
current_num_vectors_added = 0;
}
Status
VectorSource::Add(const ExecutionEnginePtr &execution_engine,
const meta::TableFileSchema &table_file_schema,
const size_t &num_vectors_to_add,
size_t &num_vectors_added,
IDNumbers &vector_ids) {
VectorSource::Add(const ExecutionEnginePtr& execution_engine, const meta::TableFileSchema& table_file_schema,
const size_t& num_vectors_to_add, size_t& num_vectors_added, IDNumbers& vector_ids) {
server::CollectAddMetrics metrics(n_, table_file_schema.dimension_);
num_vectors_added = current_num_vectors_added + num_vectors_to_add <= n_ ?
num_vectors_to_add : n_ - current_num_vectors_added;
num_vectors_added =
current_num_vectors_added + num_vectors_to_add <= n_ ? num_vectors_to_add : n_ - current_num_vectors_added;
IDNumbers vector_ids_to_add;
if (vector_ids.empty()) {
id_generator_->GetNextIDNumbers(num_vectors_added, vector_ids_to_add);
@ -58,8 +51,7 @@ VectorSource::Add(const ExecutionEnginePtr &execution_engine,
vector_ids_to_add.data());
if (status.ok()) {
current_num_vectors_added += num_vectors_added;
vector_ids_.insert(vector_ids_.end(),
std::make_move_iterator(vector_ids_to_add.begin()),
vector_ids_.insert(vector_ids_.end(), std::make_move_iterator(vector_ids_to_add.begin()),
std::make_move_iterator(vector_ids_to_add.end()));
} else {
ENGINE_LOG_ERROR << "VectorSource::Add failed: " + status.ToString();
@ -83,6 +75,6 @@ VectorSource::GetVectorIds() {
return vector_ids_;
}
} // namespace engine
} // namespace milvus
} // namespace zilliz
} // namespace engine
} // namespace milvus
} // namespace zilliz

32
cpp/src/db/insert/VectorSource.h
View File

@ -15,12 +15,11 @@
// specific language governing permissions and limitations
// under the License.
#pragma once
#include "db/meta/Meta.h"
#include "db/IDGenerator.h"
#include "db/engine/ExecutionEngine.h"
#include "db/meta/Meta.h"
#include "utils/Status.h"
#include <memory>
@ -31,32 +30,33 @@ namespace engine {
class VectorSource {
public:
VectorSource(const size_t &n, const float *vectors);
VectorSource(const size_t& n, const float* vectors);
Status Add(const ExecutionEnginePtr &execution_engine,
const meta::TableFileSchema &table_file_schema,
const size_t &num_vectors_to_add,
size_t &num_vectors_added,
IDNumbers &vector_ids);
Status
Add(const ExecutionEnginePtr& execution_engine, const meta::TableFileSchema& table_file_schema,
const size_t& num_vectors_to_add, size_t& num_vectors_added, IDNumbers& vector_ids);
size_t GetNumVectorsAdded();
size_t
GetNumVectorsAdded();
bool AllAdded();
bool
AllAdded();
IDNumbers GetVectorIds();
IDNumbers
GetVectorIds();
private:
const size_t n_;
const float *vectors_;
const float* vectors_;
IDNumbers vector_ids_;
size_t current_num_vectors_added;
std::shared_ptr<IDGenerator> id_generator_;
}; //VectorSource
}; // VectorSource
using VectorSourcePtr = std::shared_ptr<VectorSource>;
} // namespace engine
} // namespace milvus
} // namespace zilliz
} // namespace engine
} // namespace milvus
} // namespace zilliz

101
cpp/src/db/meta/Meta.h
View File

@ -15,7 +15,6 @@
// specific language governing permissions and limitations
// under the License.
#pragma once
#include "MetaTypes.h"
@ -25,84 +24,104 @@
#include <cstddef>
#include <memory>
#include <vector>
#include <string>
#include <vector>
namespace zilliz {
namespace milvus {
namespace engine {
namespace meta {
static const char *META_TABLES = "Tables";
static const char *META_TABLEFILES = "TableFiles";
static const char* META_TABLES = "Tables";
static const char* META_TABLEFILES = "TableFiles";
class Meta {
public:
virtual ~Meta() = default;
virtual Status CreateTable(TableSchema &table_schema) = 0;
virtual Status
CreateTable(TableSchema& table_schema) = 0;
virtual Status DescribeTable(TableSchema &table_schema) = 0;
virtual Status
DescribeTable(TableSchema& table_schema) = 0;
virtual Status HasTable(const std::string &table_id, bool &has_or_not) = 0;
virtual Status
HasTable(const std::string& table_id, bool& has_or_not) = 0;
virtual Status AllTables(std::vector<TableSchema> &table_schema_array) = 0;
virtual Status
AllTables(std::vector<TableSchema>& table_schema_array) = 0;
virtual Status UpdateTableIndex(const std::string &table_id, const TableIndex &index) = 0;
virtual Status
UpdateTableIndex(const std::string& table_id, const TableIndex& index) = 0;
virtual Status UpdateTableFlag(const std::string &table_id, int64_t flag) = 0;
virtual Status
UpdateTableFlag(const std::string& table_id, int64_t flag) = 0;
virtual Status DeleteTable(const std::string &table_id) = 0;
virtual Status
DeleteTable(const std::string& table_id) = 0;
virtual Status DeleteTableFiles(const std::string &table_id) = 0;
virtual Status
DeleteTableFiles(const std::string& table_id) = 0;
virtual Status CreateTableFile(TableFileSchema &file_schema) = 0;
virtual Status
CreateTableFile(TableFileSchema& file_schema) = 0;
virtual Status DropPartitionsByDates(const std::string &table_id, const DatesT &dates) = 0;
virtual Status
DropPartitionsByDates(const std::string& table_id, const DatesT& dates) = 0;
virtual Status GetTableFiles(const std::string &table_id,
const std::vector<size_t> &ids,
TableFilesSchema &table_files) = 0;
virtual Status
GetTableFiles(const std::string& table_id, const std::vector<size_t>& ids, TableFilesSchema& table_files) = 0;
virtual Status UpdateTableFilesToIndex(const std::string &table_id) = 0;
virtual Status
UpdateTableFilesToIndex(const std::string& table_id) = 0;
virtual Status UpdateTableFile(TableFileSchema &file_schema) = 0;
virtual Status
UpdateTableFile(TableFileSchema& file_schema) = 0;
virtual Status UpdateTableFiles(TableFilesSchema &files) = 0;
virtual Status
UpdateTableFiles(TableFilesSchema& files) = 0;
virtual Status FilesToSearch(const std::string &table_id,
const std::vector<size_t> &ids,
const DatesT &partition,
DatePartionedTableFilesSchema &files) = 0;
virtual Status
FilesToSearch(const std::string& table_id, const std::vector<size_t>& ids, const DatesT& partition,
DatePartionedTableFilesSchema& files) = 0;
virtual Status FilesToMerge(const std::string &table_id, DatePartionedTableFilesSchema &files) = 0;
virtual Status
FilesToMerge(const std::string& table_id, DatePartionedTableFilesSchema& files) = 0;
virtual Status Size(uint64_t &result) = 0;
virtual Status
Size(uint64_t& result) = 0;
virtual Status Archive() = 0;
virtual Status
Archive() = 0;
virtual Status FilesToIndex(TableFilesSchema &) = 0;
virtual Status
FilesToIndex(TableFilesSchema&) = 0;
virtual Status FilesByType(const std::string &table_id,
const std::vector<int> &file_types,
std::vector<std::string> &file_ids) = 0;
virtual Status
FilesByType(const std::string& table_id, const std::vector<int>& file_types,
std::vector<std::string>& file_ids) = 0;
virtual Status DescribeTableIndex(const std::string &table_id, TableIndex &index) = 0;
virtual Status
DescribeTableIndex(const std::string& table_id, TableIndex& index) = 0;
virtual Status DropTableIndex(const std::string &table_id) = 0;
virtual Status
DropTableIndex(const std::string& table_id) = 0;
virtual Status CleanUp() = 0;
virtual Status
CleanUp() = 0;
virtual Status CleanUpFilesWithTTL(uint16_t) = 0;
virtual Status DropAll() = 0;
virtual Status
DropAll() = 0;
virtual Status Count(const std::string &table_id, uint64_t &result) = 0;
}; // MetaData
virtual Status
Count(const std::string& table_id, uint64_t& result) = 0;
}; // MetaData
using MetaPtr = std::shared_ptr<Meta>;
} // namespace meta
} // namespace engine
} // namespace milvus
} // namespace zilliz
} // namespace meta
} // namespace engine
} // namespace milvus
} // namespace zilliz

8
cpp/src/db/meta/MetaConsts.h
View File

@ -38,7 +38,7 @@ const size_t H_SEC = 60 * M_SEC;
const size_t D_SEC = 24 * H_SEC;
const size_t W_SEC = 7 * D_SEC;
} // namespace meta
} // namespace engine
} // namespace milvus
} // namespace zilliz
} // namespace meta
} // namespace engine
} // namespace milvus
} // namespace zilliz

24
cpp/src/db/meta/MetaFactory.cpp
View File

@ -16,26 +16,26 @@
// under the License.
#include "db/meta/MetaFactory.h"
#include "SqliteMetaImpl.h"
#include "MySQLMetaImpl.h"
#include "utils/Log.h"
#include "utils/Exception.h"
#include "SqliteMetaImpl.h"
#include "db/Utils.h"
#include "utils/Exception.h"
#include "utils/Log.h"
#include <stdlib.h>
#include <time.h>
#include <sstream>
#include <cstdlib>
#include <string>
#include <string.h>
#include <time.h>
#include <cstdlib>
#include <memory>
#include <sstream>
#include <string>
namespace zilliz {
namespace milvus {
namespace engine {
DBMetaOptions
MetaFactory::BuildOption(const std::string &path) {
MetaFactory::BuildOption(const std::string& path) {
auto p = path;
if (p == "") {
srand(time(nullptr));
@ -51,7 +51,7 @@ MetaFactory::BuildOption(const std::string &path) {
}
meta::MetaPtr
MetaFactory::Build(const DBMetaOptions &metaOptions, const int &mode) {
MetaFactory::Build(const DBMetaOptions& metaOptions, const int& mode) {
std::string uri = metaOptions.backend_uri_;
utils::MetaUriInfo uri_info;
@ -73,6 +73,6 @@ MetaFactory::Build(const DBMetaOptions &metaOptions, const int &mode) {
}
}
} // namespace engine
} // namespace milvus
} // namespace zilliz
} // namespace engine
} // namespace milvus
} // namespace zilliz

12
cpp/src/db/meta/MetaFactory.h
View File

@ -28,11 +28,13 @@ namespace engine {
class MetaFactory {
public:
static DBMetaOptions BuildOption(const std::string &path = "");
static DBMetaOptions
BuildOption(const std::string& path = "");
static meta::MetaPtr Build(const DBMetaOptions &metaOptions, const int &mode);
static meta::MetaPtr
Build(const DBMetaOptions& metaOptions, const int& mode);
};
} // namespace engine
} // namespace milvus
} // namespace zilliz
} // namespace engine
} // namespace milvus
} // namespace zilliz

30
cpp/src/db/meta/MetaTypes.h
View File

@ -17,22 +17,22 @@
#pragma once
#include "db/engine/ExecutionEngine.h"
#include "db/Constants.h"
#include "db/engine/ExecutionEngine.h"
#include <vector>
#include <map>
#include <string>
#include <memory>
#include <string>
#include <vector>
namespace zilliz {
namespace milvus {
namespace engine {
namespace meta {
constexpr int32_t DEFAULT_ENGINE_TYPE = (int) EngineType::FAISS_IDMAP;
constexpr int32_t DEFAULT_ENGINE_TYPE = (int)EngineType::FAISS_IDMAP;
constexpr int32_t DEFAULT_NLIST = 16384;
constexpr int32_t DEFAULT_METRIC_TYPE = (int) MetricType::L2;
constexpr int32_t DEFAULT_METRIC_TYPE = (int)MetricType::L2;
constexpr int32_t DEFAULT_INDEX_FILE_SIZE = ONE_GB;
constexpr int64_t FLAG_MASK_NO_USERID = 0x1;
@ -50,7 +50,7 @@ struct TableSchema {
size_t id_ = 0;
std::string table_id_;
int32_t state_ = (int) NORMAL;
int32_t state_ = (int)NORMAL;
uint16_t dimension_ = 0;
int64_t created_on_ = 0;
int64_t flag_ = 0;
@ -58,7 +58,7 @@ struct TableSchema {
int32_t engine_type_ = DEFAULT_ENGINE_TYPE;
int32_t nlist_ = DEFAULT_NLIST;
int32_t metric_type_ = DEFAULT_METRIC_TYPE;
}; // TableSchema
}; // TableSchema
struct TableFileSchema {
typedef enum {
@ -83,17 +83,17 @@ struct TableFileSchema {
std::string location_;
int64_t updated_time_ = 0;
int64_t created_on_ = 0;
int64_t index_file_size_ = DEFAULT_INDEX_FILE_SIZE; //not persist to meta
int64_t index_file_size_ = DEFAULT_INDEX_FILE_SIZE; // not persist to meta
int32_t engine_type_ = DEFAULT_ENGINE_TYPE;
int32_t nlist_ = DEFAULT_NLIST; //not persist to meta
int32_t metric_type_ = DEFAULT_METRIC_TYPE; //not persist to meta
}; // TableFileSchema
int32_t nlist_ = DEFAULT_NLIST; // not persist to meta
int32_t metric_type_ = DEFAULT_METRIC_TYPE; // not persist to meta
}; // TableFileSchema
using TableFileSchemaPtr = std::shared_ptr<meta::TableFileSchema>;
using TableFilesSchema = std::vector<TableFileSchema>;
using DatePartionedTableFilesSchema = std::map<DateT, TableFilesSchema>;
} // namespace meta
} // namespace engine
} // namespace milvus
} // namespace zilliz
} // namespace meta
} // namespace engine
} // namespace milvus
} // namespace zilliz

31
cpp/src/db/meta/MySQLConnectionPool.cpp
View File

@ -15,7 +15,6 @@
// specific language governing permissions and limitations
// under the License.
#include "db/meta/MySQLConnectionPool.h"
namespace zilliz {
@ -28,7 +27,7 @@ namespace meta {
// already. Can't do this in create() because we're interested in
// connections actually in use, not those created. Also note that
// we keep our own count; ConnectionPool::size() isn't the same!
mysqlpp::Connection *
mysqlpp::Connection*
MySQLConnectionPool::grab() {
while (conns_in_use_ > max_pool_size_) {
sleep(1);
@ -40,7 +39,7 @@ MySQLConnectionPool::grab() {
// Other half of in-use conn count limit
void
MySQLConnectionPool::release(const mysqlpp::Connection *pc) {
MySQLConnectionPool::release(const mysqlpp::Connection* pc) {
mysqlpp::ConnectionPool::release(pc);
if (conns_in_use_ <= 0) {
ENGINE_LOG_WARNING << "MySQLConnetionPool::release: conns_in_use_ is less than zero. conns_in_use_ = "
@ -64,27 +63,25 @@ MySQLConnectionPool::getDB() {
}
// Superclass overrides
mysqlpp::Connection *
mysqlpp::Connection*
MySQLConnectionPool::create() {
try {
// Create connection using the parameters we were passed upon
// creation.
mysqlpp::Connection *conn = new mysqlpp::Connection();
mysqlpp::Connection* conn = new mysqlpp::Connection();
conn->set_option(new mysqlpp::ReconnectOption(true));
conn->connect(db_.empty() ? 0 : db_.c_str(),
server_.empty() ? 0 : server_.c_str(),
user_.empty() ? 0 : user_.c_str(),
password_.empty() ? 0 : password_.c_str(),
port_);
conn->connect(db_.empty() ? 0 : db_.c_str(), server_.empty() ? 0 : server_.c_str(),
user_.empty() ? 0 : user_.c_str(), password_.empty() ? 0 : password_.c_str(), port_);
return conn;
} catch (const mysqlpp::ConnectionFailed &er) {
ENGINE_LOG_ERROR << "Failed to connect to database server" << ": " << er.what();
} catch (const mysqlpp::ConnectionFailed& er) {
ENGINE_LOG_ERROR << "Failed to connect to database server"
<< ": " << er.what();
return nullptr;
}
}
void
MySQLConnectionPool::destroy(mysqlpp::Connection *cp) {
MySQLConnectionPool::destroy(mysqlpp::Connection* cp) {
// Our superclass can't know how we created the Connection, so
// it delegates destruction to us, to be safe.
delete cp;
@ -95,7 +92,7 @@ MySQLConnectionPool::max_idle_time() {
return max_idle_time_;
}
} // namespace meta
} // namespace engine
} // namespace milvus
} // namespace zilliz
} // namespace meta
} // namespace engine
} // namespace milvus
} // namespace zilliz

57
cpp/src/db/meta/MySQLConnectionPool.h
View File

@ -15,12 +15,11 @@
// specific language governing permissions and limitations
// under the License.
#include "mysql++/mysql++.h"
#include <string>
#include <unistd.h>
#include <atomic>
#include <string>
#include "utils/Log.h"
@ -32,20 +31,16 @@ namespace meta {
class MySQLConnectionPool : public mysqlpp::ConnectionPool {
public:
// The object's only constructor
MySQLConnectionPool(std::string dbName,
std::string userName,
std::string passWord,
std::string serverIp,
int port = 0,
int maxPoolSize = 8) :
db_(dbName),
user_(userName),
password_(passWord),
server_(serverIp),
port_(port),
max_pool_size_(maxPoolSize) {
MySQLConnectionPool(std::string dbName, std::string userName, std::string passWord, std::string serverIp,
int port = 0, int maxPoolSize = 8)
: db_(dbName),
user_(userName),
password_(passWord),
server_(serverIp),
port_(port),
max_pool_size_(maxPoolSize) {
conns_in_use_ = 0;
max_idle_time_ = 10; //10 seconds
max_idle_time_ = 10; // 10 seconds
}
// The destructor. We _must_ call ConnectionPool::clear() here,
@ -54,24 +49,30 @@ class MySQLConnectionPool : public mysqlpp::ConnectionPool {
clear();
}
mysqlpp::Connection *grab() override;
mysqlpp::Connection*
grab() override;
// Other half of in-use conn count limit
void release(const mysqlpp::Connection *pc) override;
void
release(const mysqlpp::Connection* pc) override;
// int getConnectionsInUse();
//
// void set_max_idle_time(int max_idle);
// int getConnectionsInUse();
//
// void set_max_idle_time(int max_idle);
std::string getDB();
std::string
getDB();
protected:
// Superclass overrides
mysqlpp::Connection *create() override;
mysqlpp::Connection*
create() override;
void destroy(mysqlpp::Connection *cp) override;
void
destroy(mysqlpp::Connection* cp) override;
unsigned int max_idle_time() override;
unsigned int
max_idle_time() override;
private:
// Number of connections currently in use
@ -86,7 +87,7 @@ class MySQLConnectionPool : public mysqlpp::ConnectionPool {
unsigned int max_idle_time_;
};
} // namespace meta
} // namespace engine
} // namespace milvus
} // namespace zilliz
} // namespace meta
} // namespace engine
} // namespace milvus
} // namespace zilliz

836
cpp/src/db/meta/MySQLMetaImpl.cpp
View File

File diff suppressed because it is too large Load Diff

126
cpp/src/db/meta/MySQLMetaImpl.h
View File

@ -18,14 +18,14 @@
#pragma once
#include "Meta.h"
#include "db/Options.h"
#include "MySQLConnectionPool.h"
#include "db/Options.h"
#include <mysql++/mysql++.h>
#include <mutex>
#include <vector>
#include <string>
#include <memory>
#include <mutex>
#include <string>
#include <vector>
namespace zilliz {
namespace milvus {
@ -34,77 +34,101 @@ namespace meta {
class MySQLMetaImpl : public Meta {
public:
MySQLMetaImpl(const DBMetaOptions &options, const int &mode);
MySQLMetaImpl(const DBMetaOptions& options, const int& mode);
~MySQLMetaImpl();
Status CreateTable(TableSchema &table_schema) override;
Status
CreateTable(TableSchema& table_schema) override;
Status DescribeTable(TableSchema &table_schema) override;
Status
DescribeTable(TableSchema& table_schema) override;
Status HasTable(const std::string &table_id, bool &has_or_not) override;
Status
HasTable(const std::string& table_id, bool& has_or_not) override;
Status AllTables(std::vector<TableSchema> &table_schema_array) override;
Status
AllTables(std::vector<TableSchema>& table_schema_array) override;
Status DeleteTable(const std::string &table_id) override;
Status
DeleteTable(const std::string& table_id) override;
Status DeleteTableFiles(const std::string &table_id) override;
Status
DeleteTableFiles(const std::string& table_id) override;
Status CreateTableFile(TableFileSchema &file_schema) override;
Status
CreateTableFile(TableFileSchema& file_schema) override;
Status DropPartitionsByDates(const std::string &table_id,
const DatesT &dates) override;
Status
DropPartitionsByDates(const std::string& table_id, const DatesT& dates) override;
Status GetTableFiles(const std::string &table_id,
const std::vector<size_t> &ids,
TableFilesSchema &table_files) override;
Status
GetTableFiles(const std::string& table_id, const std::vector<size_t>& ids, TableFilesSchema& table_files) override;
Status FilesByType(const std::string &table_id,
const std::vector<int> &file_types,
std::vector<std::string> &file_ids) override;
Status
FilesByType(const std::string& table_id, const std::vector<int>& file_types,
std::vector<std::string>& file_ids) override;
Status UpdateTableIndex(const std::string &table_id, const TableIndex &index) override;
Status
UpdateTableIndex(const std::string& table_id, const TableIndex& index) override;
Status UpdateTableFlag(const std::string &table_id, int64_t flag) override;
Status
UpdateTableFlag(const std::string& table_id, int64_t flag) override;
Status DescribeTableIndex(const std::string &table_id, TableIndex &index) override;
Status
DescribeTableIndex(const std::string& table_id, TableIndex& index) override;
Status DropTableIndex(const std::string &table_id) override;
Status
DropTableIndex(const std::string& table_id) override;
Status UpdateTableFile(TableFileSchema &file_schema) override;
Status
UpdateTableFile(TableFileSchema& file_schema) override;
Status UpdateTableFilesToIndex(const std::string &table_id) override;
Status
UpdateTableFilesToIndex(const std::string& table_id) override;
Status UpdateTableFiles(TableFilesSchema &files) override;
Status
UpdateTableFiles(TableFilesSchema& files) override;
Status FilesToSearch(const std::string &table_id,
const std::vector<size_t> &ids,
const DatesT &partition,
DatePartionedTableFilesSchema &files) override;
Status
FilesToSearch(const std::string& table_id, const std::vector<size_t>& ids, const DatesT& partition,
DatePartionedTableFilesSchema& files) override;
Status FilesToMerge(const std::string &table_id,
DatePartionedTableFilesSchema &files) override;
Status
FilesToMerge(const std::string& table_id, DatePartionedTableFilesSchema& files) override;
Status FilesToIndex(TableFilesSchema &) override;
Status
FilesToIndex(TableFilesSchema&) override;
Status Archive() override;
Status
Archive() override;
Status Size(uint64_t &result) override;
Status
Size(uint64_t& result) override;
Status CleanUp() override;
Status
CleanUp() override;
Status CleanUpFilesWithTTL(uint16_t seconds) override;
Status
CleanUpFilesWithTTL(uint16_t seconds) override;
Status DropAll() override;
Status
DropAll() override;
Status Count(const std::string &table_id, uint64_t &result) override;
Status
Count(const std::string& table_id, uint64_t& result) override;
private:
Status NextFileId(std::string &file_id);
Status NextTableId(std::string &table_id);
Status DiscardFiles(int64_t to_discard_size);
Status
NextFileId(std::string& file_id);
Status
NextTableId(std::string& table_id);
Status
DiscardFiles(int64_t to_discard_size);
void ValidateMetaSchema();
Status Initialize();
void
ValidateMetaSchema();
Status
Initialize();
private:
const DBMetaOptions options_;
@ -113,10 +137,10 @@ class MySQLMetaImpl : public Meta {
std::shared_ptr<MySQLConnectionPool> mysql_connection_pool_;
bool safe_grab_ = false;
// std::mutex connectionMutex_;
}; // DBMetaImpl
// std::mutex connectionMutex_;
}; // DBMetaImpl
} // namespace meta
} // namespace engine
} // namespace milvus
} // namespace zilliz
} // namespace meta
} // namespace engine
} // namespace milvus
} // namespace zilliz

118
cpp/src/db/meta/SqliteMetaImpl.h
View File

@ -21,8 +21,8 @@
#include "db/Options.h"
#include <mutex>
#include <vector>
#include <string>
#include <vector>
namespace zilliz {
namespace milvus {
@ -30,86 +30,112 @@ namespace engine {
namespace meta {
auto
StoragePrototype(const std::string &path);
StoragePrototype(const std::string& path);
class SqliteMetaImpl : public Meta {
public:
explicit SqliteMetaImpl(const DBMetaOptions &options);
explicit SqliteMetaImpl(const DBMetaOptions& options);
~SqliteMetaImpl();
Status CreateTable(TableSchema &table_schema) override;
Status
CreateTable(TableSchema& table_schema) override;
Status DescribeTable(TableSchema &table_schema) override;
Status
DescribeTable(TableSchema& table_schema) override;
Status HasTable(const std::string &table_id, bool &has_or_not) override;
Status
HasTable(const std::string& table_id, bool& has_or_not) override;
Status AllTables(std::vector<TableSchema> &table_schema_array) override;
Status
AllTables(std::vector<TableSchema>& table_schema_array) override;
Status DeleteTable(const std::string &table_id) override;
Status
DeleteTable(const std::string& table_id) override;
Status DeleteTableFiles(const std::string &table_id) override;
Status
DeleteTableFiles(const std::string& table_id) override;
Status CreateTableFile(TableFileSchema &file_schema) override;
Status
CreateTableFile(TableFileSchema& file_schema) override;
Status DropPartitionsByDates(const std::string &table_id, const DatesT &dates) override;
Status
DropPartitionsByDates(const std::string& table_id, const DatesT& dates) override;
Status GetTableFiles(const std::string &table_id,
const std::vector<size_t> &ids,
TableFilesSchema &table_files) override;
Status
GetTableFiles(const std::string& table_id, const std::vector<size_t>& ids, TableFilesSchema& table_files) override;
Status FilesByType(const std::string &table_id,
const std::vector<int> &file_types,
std::vector<std::string> &file_ids) override;
Status
FilesByType(const std::string& table_id, const std::vector<int>& file_types,
std::vector<std::string>& file_ids) override;
Status UpdateTableIndex(const std::string &table_id, const TableIndex &index) override;
Status
UpdateTableIndex(const std::string& table_id, const TableIndex& index) override;
Status UpdateTableFlag(const std::string &table_id, int64_t flag) override;
Status
UpdateTableFlag(const std::string& table_id, int64_t flag) override;
Status DescribeTableIndex(const std::string &table_id, TableIndex &index) override;
Status
DescribeTableIndex(const std::string& table_id, TableIndex& index) override;
Status DropTableIndex(const std::string &table_id) override;
Status
DropTableIndex(const std::string& table_id) override;
Status UpdateTableFilesToIndex(const std::string &table_id) override;
Status
UpdateTableFilesToIndex(const std::string& table_id) override;
Status UpdateTableFile(TableFileSchema &file_schema) override;
Status
UpdateTableFile(TableFileSchema& file_schema) override;
Status UpdateTableFiles(TableFilesSchema &files) override;
Status
UpdateTableFiles(TableFilesSchema& files) override;
Status FilesToSearch(const std::string &table_id,
const std::vector<size_t> &ids,
const DatesT &partition,
DatePartionedTableFilesSchema &files) override;
Status
FilesToSearch(const std::string& table_id, const std::vector<size_t>& ids, const DatesT& partition,
DatePartionedTableFilesSchema& files) override;
Status FilesToMerge(const std::string &table_id, DatePartionedTableFilesSchema &files) override;
Status
FilesToMerge(const std::string& table_id, DatePartionedTableFilesSchema& files) override;
Status FilesToIndex(TableFilesSchema &) override;
Status
FilesToIndex(TableFilesSchema&) override;
Status Archive() override;
Status
Archive() override;
Status Size(uint64_t &result) override;
Status
Size(uint64_t& result) override;
Status CleanUp() override;
Status
CleanUp() override;
Status CleanUpFilesWithTTL(uint16_t seconds) override;
Status
CleanUpFilesWithTTL(uint16_t seconds) override;
Status DropAll() override;
Status
DropAll() override;
Status Count(const std::string &table_id, uint64_t &result) override;
Status
Count(const std::string& table_id, uint64_t& result) override;
private:
Status NextFileId(std::string &file_id);
Status NextTableId(std::string &table_id);
Status DiscardFiles(int64_t to_discard_size);
Status
NextFileId(std::string& file_id);
Status
NextTableId(std::string& table_id);
Status
DiscardFiles(int64_t to_discard_size);
void ValidateMetaSchema();
Status Initialize();
void
ValidateMetaSchema();
Status
Initialize();
private:
const DBMetaOptions options_;
std::mutex meta_mutex_;
}; // DBMetaImpl
}; // DBMetaImpl
} // namespace meta
} // namespace engine
} // namespace milvus
} // namespace zilliz
} // namespace meta
} // namespace engine
} // namespace milvus
} // namespace zilliz

28
cpp/src/main.cpp
View File

@ -17,25 +17,25 @@
#include <getopt.h>
#include <libgen.h>
#include <cstring>
#include <string>
#include <signal.h>
#include <unistd.h>
#include <cstring>
#include <string>
#include "utils/easylogging++.h"
#include "utils/SignalUtil.h"
#include "utils/CommonUtil.h"
#include "../version.h"
#include "metrics/Metrics.h"
#include "server/Server.h"
#include "../version.h"
#include "utils/CommonUtil.h"
#include "utils/SignalUtil.h"
#include "utils/easylogging++.h"
INITIALIZE_EASYLOGGINGPP
void print_help(const std::string &app_name);
void
print_help(const std::string& app_name);
int
main(int argc, char *argv[]) {
main(int argc, char* argv[]) {
std::cout << std::endl << "Welcome to use Milvus by Zilliz!" << std::endl;
std::cout << "Milvus " << BUILD_TYPE << " version: v" << MILVUS_VERSION << " built at " << BUILD_TIME << std::endl;
@ -63,21 +63,21 @@ main(int argc, char *argv[]) {
while ((value = getopt_long(argc, argv, "c:l:p:dh", long_options, &option_index)) != -1) {
switch (value) {
case 'c': {
char *config_filename_ptr = strdup(optarg);
char* config_filename_ptr = strdup(optarg);
config_filename = config_filename_ptr;
free(config_filename_ptr);
std::cout << "Loading configuration from: " << config_filename << std::endl;
break;
}
case 'l': {
char *log_filename_ptr = strdup(optarg);
char* log_filename_ptr = strdup(optarg);
log_config_file = log_filename_ptr;
free(log_filename_ptr);
std::cout << "Initial log config from: " << log_config_file << std::endl;
break;
}
case 'p': {
char *pid_filename_ptr = strdup(optarg);
char* pid_filename_ptr = strdup(optarg);
pid_filename = pid_filename_ptr;
free(pid_filename_ptr);
std::cout << pid_filename << std::endl;
@ -106,7 +106,7 @@ main(int argc, char *argv[]) {
signal(SIGUSR2, zilliz::milvus::server::SignalUtil::HandleSignal);
signal(SIGTERM, zilliz::milvus::server::SignalUtil::HandleSignal);
zilliz::milvus::server::Server &server = zilliz::milvus::server::Server::GetInstance();
zilliz::milvus::server::Server& server = zilliz::milvus::server::Server::GetInstance();
server.Init(start_daemonized, pid_filename, config_filename, log_config_file);
server.Start();
@ -117,7 +117,7 @@ main(int argc, char *argv[]) {
}
void
print_help(const std::string &app_name) {
print_help(const std::string& app_name) {
std::cout << std::endl << "Usage: " << app_name << " [OPTIONS]" << std::endl << std::endl;
std::cout << " Options:" << std::endl;
std::cout << " -h --help Print this help" << std::endl;

140
cpp/src/metrics/MetricBase.h
View File

@ -15,11 +15,10 @@
// specific language governing permissions and limitations
// under the License.
#pragma once
#include "utils/Error.h"
#include "SystemInfo.h"
#include "utils/Error.h"
#include <string>
@ -28,142 +27,185 @@ namespace milvus {
namespace server {
class MetricsBase {
public:
static MetricsBase &
static MetricsBase&
GetInstance() {
static MetricsBase instance;
return instance;
}
virtual ErrorCode Init() {
virtual ErrorCode
Init() {
}
virtual void AddVectorsSuccessTotalIncrement(double value = 1) {
virtual void
AddVectorsSuccessTotalIncrement(double value = 1) {
}
virtual void AddVectorsFailTotalIncrement(double value = 1) {
virtual void
AddVectorsFailTotalIncrement(double value = 1) {
}
virtual void AddVectorsDurationHistogramOberve(double value) {
virtual void
AddVectorsDurationHistogramOberve(double value) {
}
virtual void RawFileSizeHistogramObserve(double value) {
virtual void
RawFileSizeHistogramObserve(double value) {
}
virtual void IndexFileSizeHistogramObserve(double value) {
virtual void
IndexFileSizeHistogramObserve(double value) {
}
virtual void BuildIndexDurationSecondsHistogramObserve(double value) {
virtual void
BuildIndexDurationSecondsHistogramObserve(double value) {
}
virtual void CpuCacheUsageGaugeSet(double value) {
virtual void
CpuCacheUsageGaugeSet(double value) {
}
virtual void GpuCacheUsageGaugeSet() {
virtual void
GpuCacheUsageGaugeSet() {
}
virtual void MetaAccessTotalIncrement(double value = 1) {
virtual void
MetaAccessTotalIncrement(double value = 1) {
}
virtual void MetaAccessDurationSecondsHistogramObserve(double value) {
virtual void
MetaAccessDurationSecondsHistogramObserve(double value) {
}
virtual void FaissDiskLoadDurationSecondsHistogramObserve(double value) {
virtual void
FaissDiskLoadDurationSecondsHistogramObserve(double value) {
}
virtual void FaissDiskLoadSizeBytesHistogramObserve(double value) {
virtual void
FaissDiskLoadSizeBytesHistogramObserve(double value) {
}
virtual void CacheAccessTotalIncrement(double value = 1) {
virtual void
CacheAccessTotalIncrement(double value = 1) {
}
virtual void MemTableMergeDurationSecondsHistogramObserve(double value) {
virtual void
MemTableMergeDurationSecondsHistogramObserve(double value) {
}
virtual void SearchIndexDataDurationSecondsHistogramObserve(double value) {
virtual void
SearchIndexDataDurationSecondsHistogramObserve(double value) {
}
virtual void SearchRawDataDurationSecondsHistogramObserve(double value) {
virtual void
SearchRawDataDurationSecondsHistogramObserve(double value) {
}
virtual void IndexFileSizeTotalIncrement(double value = 1) {
virtual void
IndexFileSizeTotalIncrement(double value = 1) {
}
virtual void RawFileSizeTotalIncrement(double value = 1) {
virtual void
RawFileSizeTotalIncrement(double value = 1) {
}
virtual void IndexFileSizeGaugeSet(double value) {
virtual void
IndexFileSizeGaugeSet(double value) {
}
virtual void RawFileSizeGaugeSet(double value) {
virtual void
RawFileSizeGaugeSet(double value) {
}
virtual void FaissDiskLoadIOSpeedGaugeSet(double value) {
virtual void
FaissDiskLoadIOSpeedGaugeSet(double value) {
}
virtual void QueryResponseSummaryObserve(double value) {
virtual void
QueryResponseSummaryObserve(double value) {
}
virtual void DiskStoreIOSpeedGaugeSet(double value) {
virtual void
DiskStoreIOSpeedGaugeSet(double value) {
}
virtual void DataFileSizeGaugeSet(double value) {
virtual void
DataFileSizeGaugeSet(double value) {
}
virtual void AddVectorsSuccessGaugeSet(double value) {
virtual void
AddVectorsSuccessGaugeSet(double value) {
}
virtual void AddVectorsFailGaugeSet(double value) {
virtual void
AddVectorsFailGaugeSet(double value) {
}
virtual void QueryVectorResponseSummaryObserve(double value, int count = 1) {
virtual void
QueryVectorResponseSummaryObserve(double value, int count = 1) {
}
virtual void QueryVectorResponsePerSecondGaugeSet(double value) {
virtual void
QueryVectorResponsePerSecondGaugeSet(double value) {
}
virtual void CPUUsagePercentSet() {
virtual void
CPUUsagePercentSet() {
}
virtual void RAMUsagePercentSet() {
virtual void
RAMUsagePercentSet() {
}
virtual void QueryResponsePerSecondGaugeSet(double value) {
virtual void
QueryResponsePerSecondGaugeSet(double value) {
}
virtual void GPUPercentGaugeSet() {
virtual void
GPUPercentGaugeSet() {
}
virtual void GPUMemoryUsageGaugeSet() {
virtual void
GPUMemoryUsageGaugeSet() {
}
virtual void AddVectorsPerSecondGaugeSet(int num_vector, int dim, double time) {
virtual void
AddVectorsPerSecondGaugeSet(int num_vector, int dim, double time) {
}
virtual void QueryIndexTypePerSecondSet(std::string type, double value) {
virtual void
QueryIndexTypePerSecondSet(std::string type, double value) {
}
virtual void ConnectionGaugeIncrement() {
virtual void
ConnectionGaugeIncrement() {
}
virtual void ConnectionGaugeDecrement() {
virtual void
ConnectionGaugeDecrement() {
}
virtual void KeepingAliveCounterIncrement(double value = 1) {
virtual void
KeepingAliveCounterIncrement(double value = 1) {
}
virtual void OctetsSet() {
virtual void
OctetsSet() {
}
virtual void CPUCoreUsagePercentSet() {
virtual void
CPUCoreUsagePercentSet() {
}
virtual void GPUTemperature() {
virtual void
GPUTemperature() {
}
virtual void CPUTemperature() {
virtual void
CPUTemperature() {
}
};
} // namespace server
} // namespace milvus
} // namespace zilliz
} // namespace server
} // namespace milvus
} // namespace zilliz

16
cpp/src/metrics/Metrics.cpp
View File

@ -16,8 +16,8 @@
// under the License.
#include "metrics/Metrics.h"
#include "server/Config.h"
#include "PrometheusMetrics.h"
#include "server/Config.h"
#include <string>
@ -25,15 +25,15 @@ namespace zilliz {
namespace milvus {
namespace server {
MetricsBase &
MetricsBase&
Metrics::GetInstance() {
static MetricsBase &instance = CreateMetricsCollector();
static MetricsBase& instance = CreateMetricsCollector();
return instance;
}
MetricsBase &
MetricsBase&
Metrics::CreateMetricsCollector() {
Config &config = Config::GetInstance();
Config& config = Config::GetInstance();
std::string collector_type_str;
config.GetMetricConfigCollector(collector_type_str);
@ -45,6 +45,6 @@ Metrics::CreateMetricsCollector() {
}
}
} // namespace server
} // namespace milvus
} // namespace zilliz
} // namespace server
} // namespace milvus
} // namespace zilliz

33
cpp/src/metrics/Metrics.h
View File

@ -15,7 +15,6 @@
// specific language governing permissions and limitations
// under the License.
#pragma once
#include "MetricBase.h"
@ -26,20 +25,18 @@ namespace milvus {
namespace server {
#define METRICS_NOW_TIME std::chrono::system_clock::now()
#define METRICS_MICROSECONDS(a, b) (std::chrono::duration_cast<std::chrono::microseconds> (b-a)).count();
#define METRICS_MICROSECONDS(a, b) (std::chrono::duration_cast<std::chrono::microseconds>(b - a)).count();
enum class MetricCollectorType {
INVALID,
PROMETHEUS,
ZABBIX
};
enum class MetricCollectorType { INVALID, PROMETHEUS, ZABBIX };
class Metrics {
public:
static MetricsBase &GetInstance();
static MetricsBase&
GetInstance();
private:
static MetricsBase &CreateMetricsCollector();
static MetricsBase&
CreateMetricsCollector();
};
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
class CollectMetricsBase {
@ -50,7 +47,8 @@ class CollectMetricsBase {
virtual ~CollectMetricsBase() = default;
double TimeFromBegine() {
double
TimeFromBegine() {
auto end_time = METRICS_NOW_TIME;
return METRICS_MICROSECONDS(start_time_, end_time);
}
@ -63,7 +61,7 @@ class CollectMetricsBase {
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
class CollectInsertMetrics : CollectMetricsBase {
public:
CollectInsertMetrics(size_t n, Status &status) : n_(n), status_(status) {
CollectInsertMetrics(size_t n, Status& status) : n_(n), status_(status) {
}
~CollectInsertMetrics() {
@ -87,7 +85,7 @@ class CollectInsertMetrics : CollectMetricsBase {
private:
size_t n_;
Status &status_;
Status& status_;
};
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
@ -162,7 +160,7 @@ class CollectSerializeMetrics : CollectMetricsBase {
~CollectSerializeMetrics() {
auto total_time = TimeFromBegine();
server::Metrics::GetInstance().DiskStoreIOSpeedGaugeSet((double) size_ / total_time);
server::Metrics::GetInstance().DiskStoreIOSpeedGaugeSet((double)size_ / total_time);
}
private:
@ -177,8 +175,7 @@ class CollectAddMetrics : CollectMetricsBase {
~CollectAddMetrics() {
auto total_time = TimeFromBegine();
server::Metrics::GetInstance().AddVectorsPerSecondGaugeSet(static_cast<int>(n_),
static_cast<int>(dimension_),
server::Metrics::GetInstance().AddVectorsPerSecondGaugeSet(static_cast<int>(n_), static_cast<int>(dimension_),
total_time);
}
@ -256,6 +253,6 @@ class MetricCollector : CollectMetricsBase {
}
};
} // namespace server
} // namespace milvus
} // namespace zilliz
} // namespace server
} // namespace milvus
} // namespace zilliz

48
cpp/src/metrics/PrometheusMetrics.cpp
View File

@ -15,12 +15,11 @@
// specific language governing permissions and limitations
// under the License.
#include "metrics/PrometheusMetrics.h"
#include "SystemInfo.h"
#include "cache/GpuCacheMgr.h"
#include "server/Config.h"
#include "utils/Log.h"
#include "SystemInfo.h"
#include <string>
#include <utility>
@ -32,7 +31,7 @@ namespace server {
ErrorCode
PrometheusMetrics::Init() {
try {
Config &config = Config::GetInstance();
Config& config = Config::GetInstance();
Status s = config.GetMetricConfigEnableMonitor(startup_);
if (!s.ok()) return s.code();
if (!startup_) return SERVER_SUCCESS;
@ -49,7 +48,7 @@ PrometheusMetrics::Init() {
// Exposer Registry
exposer_ptr_->RegisterCollectable(registry_);
} catch (std::exception &ex) {
} catch (std::exception& ex) {
SERVER_LOG_ERROR << "Failed to connect prometheus server: " << std::string(ex.what());
return SERVER_UNEXPECTED_ERROR;
}
@ -79,8 +78,8 @@ PrometheusMetrics::GPUPercentGaugeSet() {
std::vector<uint64_t> used_memory = server::SystemInfo::GetInstance().GPUMemoryUsed();
for (int i = 0; i < numDevice; ++i) {
prometheus::Gauge &GPU_percent = GPU_percent_.Add({{"DeviceNum", std::to_string(i)}});
double percent = (double) used_memory[i] / (double) used_total[i];
prometheus::Gauge& GPU_percent = GPU_percent_.Add({{"DeviceNum", std::to_string(i)}});
double percent = (double)used_memory[i] / (double)used_total[i];
GPU_percent.Set(percent * 100);
}
}
@ -93,7 +92,7 @@ PrometheusMetrics::GPUMemoryUsageGaugeSet() {
int numDevice = server::SystemInfo::GetInstance().num_device();
for (int i = 0; i < numDevice; ++i) {
prometheus::Gauge &GPU_memory = GPU_memory_usage_.Add({{"DeviceNum", std::to_string(i)}});
prometheus::Gauge& GPU_memory = GPU_memory_usage_.Add({{"DeviceNum", std::to_string(i)}});
GPU_memory.Set(values[i] / MtoB);
}
}
@ -155,54 +154,51 @@ PrometheusMetrics::OctetsSet() {
void
PrometheusMetrics::CPUCoreUsagePercentSet() {
if (!startup_)
return;
if (!startup_) return;
std::vector<double> cpu_core_percent = server::SystemInfo::GetInstance().CPUCorePercent();
for (int i = 0; i < cpu_core_percent.size(); ++i) {
prometheus::Gauge &core_percent = CPU_.Add({{"CPU", std::to_string(i)}});
prometheus::Gauge& core_percent = CPU_.Add({{"CPU", std::to_string(i)}});
core_percent.Set(cpu_core_percent[i]);
}
}
void
PrometheusMetrics::GPUTemperature() {
if (!startup_)
return;
if (!startup_) return;
std::vector<uint64_t> GPU_temperatures = server::SystemInfo::GetInstance().GPUTemperature();
for (int i = 0; i < GPU_temperatures.size(); ++i) {
prometheus::Gauge &gpu_temp = GPU_temperature_.Add({{"GPU", std::to_string(i)}});
prometheus::Gauge& gpu_temp = GPU_temperature_.Add({{"GPU", std::to_string(i)}});
gpu_temp.Set(GPU_temperatures[i]);
}
}
void
PrometheusMetrics::CPUTemperature() {
if (!startup_)
return;
if (!startup_) return;
std::vector<float> CPU_temperatures = server::SystemInfo::GetInstance().CPUTemperature();
for (int i = 0; i < CPU_temperatures.size(); ++i) {
prometheus::Gauge &cpu_temp = CPU_temperature_.Add({{"CPU", std::to_string(i)}});
prometheus::Gauge& cpu_temp = CPU_temperature_.Add({{"CPU", std::to_string(i)}});
cpu_temp.Set(CPU_temperatures[i]);
}
}
void
PrometheusMetrics::GpuCacheUsageGaugeSet() {
// std::vector<uint64_t > gpu_ids = {0};
// for(auto i = 0; i < gpu_ids.size(); ++i) {
// uint64_t cache_usage = cache::GpuCacheMgr::GetInstance(gpu_ids[i])->CacheUsage();
// uint64_t cache_capacity = cache::GpuCacheMgr::GetInstance(gpu_ids[i])->CacheCapacity();
// prometheus::Gauge &gpu_cache = gpu_cache_usage_.Add({{"GPU_Cache", std::to_string(i)}});
// gpu_cache.Set(cache_usage * 100 / cache_capacity);
// }
// std::vector<uint64_t > gpu_ids = {0};
// for(auto i = 0; i < gpu_ids.size(); ++i) {
// uint64_t cache_usage = cache::GpuCacheMgr::GetInstance(gpu_ids[i])->CacheUsage();
// uint64_t cache_capacity = cache::GpuCacheMgr::GetInstance(gpu_ids[i])->CacheCapacity();
// prometheus::Gauge &gpu_cache = gpu_cache_usage_.Add({{"GPU_Cache", std::to_string(i)}});
// gpu_cache.Set(cache_usage * 100 / cache_capacity);
// }
}
} // namespace server
} // namespace milvus
} // namespace zilliz
} // namespace server
} // namespace milvus
} // namespace zilliz

713
cpp/src/metrics/PrometheusMetrics.h
View File

@ -17,19 +17,19 @@
#pragma once
#include <memory>
#include <vector>
#include <prometheus/registry.h>
#include <prometheus/exposer.h>
#include <prometheus/registry.h>
#include <iostream>
#include <memory>
#include <string>
#include <vector>
#include "utils/Error.h"
#include "MetricBase.h"
#include "utils/Error.h"
#define METRICS_NOW_TIME std::chrono::system_clock::now()
//#define server::Metrics::GetInstance() server::GetInstance()
#define METRICS_MICROSECONDS(a, b) (std::chrono::duration_cast<std::chrono::microseconds> (b-a)).count();
#define METRICS_MICROSECONDS(a, b) (std::chrono::duration_cast<std::chrono::microseconds>(b - a)).count();
namespace zilliz {
namespace milvus {
@ -37,13 +37,14 @@ namespace server {
class PrometheusMetrics : public MetricsBase {
public:
static PrometheusMetrics &
static PrometheusMetrics&
GetInstance() {
static PrometheusMetrics instance;
return instance;
}
ErrorCode Init();
ErrorCode
Init();
private:
std::shared_ptr<prometheus::Exposer> exposer_ptr_;
@ -51,163 +52,191 @@ class PrometheusMetrics : public MetricsBase {
bool startup_ = false;
public:
void SetStartup(bool startup) {
void
SetStartup(bool startup) {
startup_ = startup;
}
void AddVectorsSuccessTotalIncrement(double value = 1.0) override {
void
AddVectorsSuccessTotalIncrement(double value = 1.0) override {
if (startup_) {
add_vectors_success_total_.Increment(value);
}
}
void AddVectorsFailTotalIncrement(double value = 1.0) override {
void
AddVectorsFailTotalIncrement(double value = 1.0) override {
if (startup_) {
add_vectors_fail_total_.Increment(value);
}
}
void AddVectorsDurationHistogramOberve(double value) override {
void
AddVectorsDurationHistogramOberve(double value) override {
if (startup_) {
add_vectors_duration_histogram_.Observe(value);
}
}
void RawFileSizeHistogramObserve(double value) override {
void
RawFileSizeHistogramObserve(double value) override {
if (startup_) {
raw_files_size_histogram_.Observe(value);
}
}
void IndexFileSizeHistogramObserve(double value) override {
void
IndexFileSizeHistogramObserve(double value) override {
if (startup_) {
index_files_size_histogram_.Observe(value);
}
}
void BuildIndexDurationSecondsHistogramObserve(double value) override {
void
BuildIndexDurationSecondsHistogramObserve(double value) override {
if (startup_) {
build_index_duration_seconds_histogram_.Observe(value);
}
}
void CpuCacheUsageGaugeSet(double value) override {
void
CpuCacheUsageGaugeSet(double value) override {
if (startup_) {
cpu_cache_usage_gauge_.Set(value);
}
}
void GpuCacheUsageGaugeSet() override;
void
GpuCacheUsageGaugeSet() override;
void MetaAccessTotalIncrement(double value = 1) override {
void
MetaAccessTotalIncrement(double value = 1) override {
if (startup_) {
meta_access_total_.Increment(value);
}
}
void MetaAccessDurationSecondsHistogramObserve(double value) override {
void
MetaAccessDurationSecondsHistogramObserve(double value) override {
if (startup_) {
meta_access_duration_seconds_histogram_.Observe(value);
}
}
void FaissDiskLoadDurationSecondsHistogramObserve(double value) override {
void
FaissDiskLoadDurationSecondsHistogramObserve(double value) override {
if (startup_) {
faiss_disk_load_duration_seconds_histogram_.Observe(value);
}
}
void FaissDiskLoadSizeBytesHistogramObserve(double value) override {
void
FaissDiskLoadSizeBytesHistogramObserve(double value) override {
if (startup_) {
faiss_disk_load_size_bytes_histogram_.Observe(value);
}
}
void FaissDiskLoadIOSpeedGaugeSet(double value) override {
void
FaissDiskLoadIOSpeedGaugeSet(double value) override {
if (startup_) {
faiss_disk_load_IO_speed_gauge_.Set(value);
}
}
void CacheAccessTotalIncrement(double value = 1) override {
void
CacheAccessTotalIncrement(double value = 1) override {
if (startup_) {
cache_access_total_.Increment(value);
}
}
void MemTableMergeDurationSecondsHistogramObserve(double value) override {
void
MemTableMergeDurationSecondsHistogramObserve(double value) override {
if (startup_) {
mem_table_merge_duration_seconds_histogram_.Observe(value);
}
}
void SearchIndexDataDurationSecondsHistogramObserve(double value) override {
void
SearchIndexDataDurationSecondsHistogramObserve(double value) override {
if (startup_) {
search_index_data_duration_seconds_histogram_.Observe(value);
}
}
void SearchRawDataDurationSecondsHistogramObserve(double value) override {
void
SearchRawDataDurationSecondsHistogramObserve(double value) override {
if (startup_) {
search_raw_data_duration_seconds_histogram_.Observe(value);
}
}
void IndexFileSizeTotalIncrement(double value = 1) override {
void
IndexFileSizeTotalIncrement(double value = 1) override {
if (startup_) {
index_file_size_total_.Increment(value);
}
}
void RawFileSizeTotalIncrement(double value = 1) override {
void
RawFileSizeTotalIncrement(double value = 1) override {
if (startup_) {
raw_file_size_total_.Increment(value);
}
}
void IndexFileSizeGaugeSet(double value) override {
void
IndexFileSizeGaugeSet(double value) override {
if (startup_) {
index_file_size_gauge_.Set(value);
}
}
void RawFileSizeGaugeSet(double value) override {
void
RawFileSizeGaugeSet(double value) override {
if (startup_) {
raw_file_size_gauge_.Set(value);
}
}
void QueryResponseSummaryObserve(double value) override {
void
QueryResponseSummaryObserve(double value) override {
if (startup_) {
query_response_summary_.Observe(value);
}
}
void DiskStoreIOSpeedGaugeSet(double value) override {
void
DiskStoreIOSpeedGaugeSet(double value) override {
if (startup_) {
disk_store_IO_speed_gauge_.Set(value);
}
}
void DataFileSizeGaugeSet(double value) override {
void
DataFileSizeGaugeSet(double value) override {
if (startup_) {
data_file_size_gauge_.Set(value);
}
}
void AddVectorsSuccessGaugeSet(double value) override {
void
AddVectorsSuccessGaugeSet(double value) override {
if (startup_) {
add_vectors_success_gauge_.Set(value);
}
}
void AddVectorsFailGaugeSet(double value) override {
void
AddVectorsFailGaugeSet(double value) override {
if (startup_) {
add_vectors_fail_gauge_.Set(value);
}
}
void QueryVectorResponseSummaryObserve(double value, int count = 1) override {
void
QueryVectorResponseSummaryObserve(double value, int count = 1) override {
if (startup_) {
for (int i = 0; i < count; ++i) {
query_vector_response_summary_.Observe(value);
@ -215,412 +244,414 @@ class PrometheusMetrics : public MetricsBase {
}
}
void QueryVectorResponsePerSecondGaugeSet(double value) override {
void
QueryVectorResponsePerSecondGaugeSet(double value) override {
if (startup_) {
query_vector_response_per_second_gauge_.Set(value);
}
}
void CPUUsagePercentSet() override;
void CPUCoreUsagePercentSet() override;
void
CPUUsagePercentSet() override;
void
CPUCoreUsagePercentSet() override;
void RAMUsagePercentSet() override;
void
RAMUsagePercentSet() override;
void QueryResponsePerSecondGaugeSet(double value) override {
void
QueryResponsePerSecondGaugeSet(double value) override {
if (startup_) {
query_response_per_second_gauge.Set(value);
}
}
void GPUPercentGaugeSet() override;
void GPUMemoryUsageGaugeSet() override;
void AddVectorsPerSecondGaugeSet(int num_vector, int dim, double time) override;
void QueryIndexTypePerSecondSet(std::string type, double value) override;
void ConnectionGaugeIncrement() override;
void ConnectionGaugeDecrement() override;
void
GPUPercentGaugeSet() override;
void
GPUMemoryUsageGaugeSet() override;
void
AddVectorsPerSecondGaugeSet(int num_vector, int dim, double time) override;
void
QueryIndexTypePerSecondSet(std::string type, double value) override;
void
ConnectionGaugeIncrement() override;
void
ConnectionGaugeDecrement() override;
void KeepingAliveCounterIncrement(double value = 1) override {
void
KeepingAliveCounterIncrement(double value = 1) override {
if (startup_) {
keeping_alive_counter_.Increment(value);
}
}
void OctetsSet() override;
void
OctetsSet() override;
void GPUTemperature() override;
void CPUTemperature() override;
void
GPUTemperature() override;
void
CPUTemperature() override;
std::shared_ptr<prometheus::Exposer> &exposer_ptr() {
std::shared_ptr<prometheus::Exposer>&
exposer_ptr() {
return exposer_ptr_;
}
// prometheus::Exposer& exposer() { return exposer_;}
std::shared_ptr<prometheus::Registry> &registry_ptr() {
// prometheus::Exposer& exposer() { return exposer_;}
std::shared_ptr<prometheus::Registry>&
registry_ptr() {
return registry_;
}
// .....
private:
////all from db_connection.cpp
// prometheus::Family<prometheus::Counter> &connect_request_ = prometheus::BuildCounter()
// .Name("connection_total")
// .Help("total number of connection has been made")
// .Register(*registry_);
// prometheus::Counter &connection_total_ = connect_request_.Add({});
// prometheus::Family<prometheus::Counter> &connect_request_ = prometheus::BuildCounter()
// .Name("connection_total")
// .Help("total number of connection has been made")
// .Register(*registry_);
// prometheus::Counter &connection_total_ = connect_request_.Add({});
////all from DBImpl.cpp
using BucketBoundaries = std::vector<double>;
//record add_group request
prometheus::Family<prometheus::Counter> &add_group_request_ = prometheus::BuildCounter()
.Name("add_group_request_total")
.Help("the number of add_group request")
.Register(*registry_);
// record add_group request
prometheus::Family<prometheus::Counter>& add_group_request_ = prometheus::BuildCounter()
.Name("add_group_request_total")
.Help("the number of add_group request")
.Register(*registry_);
prometheus::Counter &add_group_success_total_ = add_group_request_.Add({{"outcome", "success"}});
prometheus::Counter &add_group_fail_total_ = add_group_request_.Add({{"outcome", "fail"}});
prometheus::Counter& add_group_success_total_ = add_group_request_.Add({{"outcome", "success"}});
prometheus::Counter& add_group_fail_total_ = add_group_request_.Add({{"outcome", "fail"}});
//record get_group request
prometheus::Family<prometheus::Counter> &get_group_request_ = prometheus::BuildCounter()
.Name("get_group_request_total")
.Help("the number of get_group request")
.Register(*registry_);
// record get_group request
prometheus::Family<prometheus::Counter>& get_group_request_ = prometheus::BuildCounter()
.Name("get_group_request_total")
.Help("the number of get_group request")
.Register(*registry_);
prometheus::Counter &get_group_success_total_ = get_group_request_.Add({{"outcome", "success"}});
prometheus::Counter &get_group_fail_total_ = get_group_request_.Add({{"outcome", "fail"}});
prometheus::Counter& get_group_success_total_ = get_group_request_.Add({{"outcome", "success"}});
prometheus::Counter& get_group_fail_total_ = get_group_request_.Add({{"outcome", "fail"}});
//record has_group request
prometheus::Family<prometheus::Counter> &has_group_request_ = prometheus::BuildCounter()
.Name("has_group_request_total")
.Help("the number of has_group request")
.Register(*registry_);
// record has_group request
prometheus::Family<prometheus::Counter>& has_group_request_ = prometheus::BuildCounter()
.Name("has_group_request_total")
.Help("the number of has_group request")
.Register(*registry_);
prometheus::Counter &has_group_success_total_ = has_group_request_.Add({{"outcome", "success"}});
prometheus::Counter &has_group_fail_total_ = has_group_request_.Add({{"outcome", "fail"}});
prometheus::Counter& has_group_success_total_ = has_group_request_.Add({{"outcome", "success"}});
prometheus::Counter& has_group_fail_total_ = has_group_request_.Add({{"outcome", "fail"}});
//record get_group_files
prometheus::Family<prometheus::Counter> &get_group_files_request_ = prometheus::BuildCounter()
.Name("get_group_files_request_total")
.Help("the number of get_group_files request")
.Register(*registry_);
// record get_group_files
prometheus::Family<prometheus::Counter>& get_group_files_request_ =
prometheus::BuildCounter()
.Name("get_group_files_request_total")
.Help("the number of get_group_files request")
.Register(*registry_);
prometheus::Counter &get_group_files_success_total_ = get_group_files_request_.Add({{"outcome", "success"}});
prometheus::Counter &get_group_files_fail_total_ = get_group_files_request_.Add({{"outcome", "fail"}});
prometheus::Counter& get_group_files_success_total_ = get_group_files_request_.Add({{"outcome", "success"}});
prometheus::Counter& get_group_files_fail_total_ = get_group_files_request_.Add({{"outcome", "fail"}});
//record add_vectors count and average time
//need to be considered
prometheus::Family<prometheus::Counter> &add_vectors_request_ = prometheus::BuildCounter()
.Name("add_vectors_request_total")
.Help("the number of vectors added")
.Register(*registry_);
prometheus::Counter &add_vectors_success_total_ = add_vectors_request_.Add({{"outcome", "success"}});
prometheus::Counter &add_vectors_fail_total_ = add_vectors_request_.Add({{"outcome", "fail"}});
// record add_vectors count and average time
// need to be considered
prometheus::Family<prometheus::Counter>& add_vectors_request_ = prometheus::BuildCounter()
.Name("add_vectors_request_total")
.Help("the number of vectors added")
.Register(*registry_);
prometheus::Counter& add_vectors_success_total_ = add_vectors_request_.Add({{"outcome", "success"}});
prometheus::Counter& add_vectors_fail_total_ = add_vectors_request_.Add({{"outcome", "fail"}});
prometheus::Family<prometheus::Histogram> &add_vectors_duration_seconds_ = prometheus::BuildHistogram()
.Name("add_vector_duration_microseconds")
.Help("average time of adding every vector")
.Register(*registry_);
prometheus::Histogram &add_vectors_duration_histogram_ =
prometheus::Family<prometheus::Histogram>& add_vectors_duration_seconds_ =
prometheus::BuildHistogram()
.Name("add_vector_duration_microseconds")
.Help("average time of adding every vector")
.Register(*registry_);
prometheus::Histogram& add_vectors_duration_histogram_ =
add_vectors_duration_seconds_.Add({}, BucketBoundaries{0, 0.01, 0.02, 0.03, 0.04, 0.05, 0.08, 0.1, 0.5, 1});
//record search count and average time
prometheus::Family<prometheus::Counter> &search_request_ = prometheus::BuildCounter()
.Name("search_request_total")
.Help("the number of search request")
.Register(*registry_);
prometheus::Counter &search_success_total_ = search_request_.Add({{"outcome", "success"}});
prometheus::Counter &search_fail_total_ = search_request_.Add({{"outcome", "fail"}});
// record search count and average time
prometheus::Family<prometheus::Counter>& search_request_ = prometheus::BuildCounter()
.Name("search_request_total")
.Help("the number of search request")
.Register(*registry_);
prometheus::Counter& search_success_total_ = search_request_.Add({{"outcome", "success"}});
prometheus::Counter& search_fail_total_ = search_request_.Add({{"outcome", "fail"}});
prometheus::Family<prometheus::Histogram> &search_request_duration_seconds_ = prometheus::BuildHistogram()
.Name("search_request_duration_microsecond")
.Help("histogram of processing time for each search")
.Register(*registry_);
prometheus::Histogram
&search_duration_histogram_ = search_request_duration_seconds_.Add({}, BucketBoundaries{0.1, 1.0, 10.0});
prometheus::Family<prometheus::Histogram>& search_request_duration_seconds_ =
prometheus::BuildHistogram()
.Name("search_request_duration_microsecond")
.Help("histogram of processing time for each search")
.Register(*registry_);
prometheus::Histogram& search_duration_histogram_ =
search_request_duration_seconds_.Add({}, BucketBoundaries{0.1, 1.0, 10.0});
//record raw_files size histogram
prometheus::Family<prometheus::Histogram> &raw_files_size_ = prometheus::BuildHistogram()
.Name("search_raw_files_bytes")
.Help("histogram of raw files size by bytes")
.Register(*registry_);
prometheus::Histogram
&raw_files_size_histogram_ = raw_files_size_.Add({}, BucketBoundaries{1e9, 2e9, 4e9, 6e9, 8e9, 1e10});
// record raw_files size histogram
prometheus::Family<prometheus::Histogram>& raw_files_size_ = prometheus::BuildHistogram()
.Name("search_raw_files_bytes")
.Help("histogram of raw files size by bytes")
.Register(*registry_);
prometheus::Histogram& raw_files_size_histogram_ =
raw_files_size_.Add({}, BucketBoundaries{1e9, 2e9, 4e9, 6e9, 8e9, 1e10});
//record index_files size histogram
prometheus::Family<prometheus::Histogram> &index_files_size_ = prometheus::BuildHistogram()
.Name("search_index_files_bytes")
.Help("histogram of index files size by bytes")
.Register(*registry_);
prometheus::Histogram
&index_files_size_histogram_ = index_files_size_.Add({}, BucketBoundaries{1e9, 2e9, 4e9, 6e9, 8e9, 1e10});
// record index_files size histogram
prometheus::Family<prometheus::Histogram>& index_files_size_ = prometheus::BuildHistogram()
.Name("search_index_files_bytes")
.Help("histogram of index files size by bytes")
.Register(*registry_);
prometheus::Histogram& index_files_size_histogram_ =
index_files_size_.Add({}, BucketBoundaries{1e9, 2e9, 4e9, 6e9, 8e9, 1e10});
//record index and raw files size counter
prometheus::Family<prometheus::Counter> &file_size_total_ = prometheus::BuildCounter()
.Name("search_file_size_total")
.Help("searched index and raw file size")
.Register(*registry_);
prometheus::Counter &index_file_size_total_ = file_size_total_.Add({{"type", "index"}});
prometheus::Counter &raw_file_size_total_ = file_size_total_.Add({{"type", "raw"}});
// record index and raw files size counter
prometheus::Family<prometheus::Counter>& file_size_total_ = prometheus::BuildCounter()
.Name("search_file_size_total")
.Help("searched index and raw file size")
.Register(*registry_);
prometheus::Counter& index_file_size_total_ = file_size_total_.Add({{"type", "index"}});
prometheus::Counter& raw_file_size_total_ = file_size_total_.Add({{"type", "raw"}});
//record index and raw files size counter
prometheus::Family<prometheus::Gauge> &file_size_gauge_ = prometheus::BuildGauge()
.Name("search_file_size_gauge")
.Help("searched current index and raw file size")
.Register(*registry_);
prometheus::Gauge &index_file_size_gauge_ = file_size_gauge_.Add({{"type", "index"}});
prometheus::Gauge &raw_file_size_gauge_ = file_size_gauge_.Add({{"type", "raw"}});
// record index and raw files size counter
prometheus::Family<prometheus::Gauge>& file_size_gauge_ = prometheus::BuildGauge()
.Name("search_file_size_gauge")
.Help("searched current index and raw file size")
.Register(*registry_);
prometheus::Gauge& index_file_size_gauge_ = file_size_gauge_.Add({{"type", "index"}});
prometheus::Gauge& raw_file_size_gauge_ = file_size_gauge_.Add({{"type", "raw"}});
//record processing time for building index
prometheus::Family<prometheus::Histogram> &build_index_duration_seconds_ = prometheus::BuildHistogram()
.Name("build_index_duration_microseconds")
.Help("histogram of processing time for building index")
.Register(*registry_);
prometheus::Histogram &build_index_duration_seconds_histogram_ =
// record processing time for building index
prometheus::Family<prometheus::Histogram>& build_index_duration_seconds_ =
prometheus::BuildHistogram()
.Name("build_index_duration_microseconds")
.Help("histogram of processing time for building index")
.Register(*registry_);
prometheus::Histogram& build_index_duration_seconds_histogram_ =
build_index_duration_seconds_.Add({}, BucketBoundaries{5e5, 2e6, 4e6, 6e6, 8e6, 1e7});
//record processing time for all building index
prometheus::Family<prometheus::Histogram> &all_build_index_duration_seconds_ = prometheus::BuildHistogram()
.Name("all_build_index_duration_microseconds")
.Help("histogram of processing time for building index")
.Register(*registry_);
prometheus::Histogram &all_build_index_duration_seconds_histogram_ =
// record processing time for all building index
prometheus::Family<prometheus::Histogram>& all_build_index_duration_seconds_ =
prometheus::BuildHistogram()
.Name("all_build_index_duration_microseconds")
.Help("histogram of processing time for building index")
.Register(*registry_);
prometheus::Histogram& all_build_index_duration_seconds_histogram_ =
all_build_index_duration_seconds_.Add({}, BucketBoundaries{2e6, 4e6, 6e6, 8e6, 1e7});
//record duration of merging mem table
prometheus::Family<prometheus::Histogram> &mem_table_merge_duration_seconds_ = prometheus::BuildHistogram()
.Name("mem_table_merge_duration_microseconds")
.Help("histogram of processing time for merging mem tables")
.Register(*registry_);
prometheus::Histogram &mem_table_merge_duration_seconds_histogram_ =
// record duration of merging mem table
prometheus::Family<prometheus::Histogram>& mem_table_merge_duration_seconds_ =
prometheus::BuildHistogram()
.Name("mem_table_merge_duration_microseconds")
.Help("histogram of processing time for merging mem tables")
.Register(*registry_);
prometheus::Histogram& mem_table_merge_duration_seconds_histogram_ =
mem_table_merge_duration_seconds_.Add({}, BucketBoundaries{5e4, 1e5, 2e5, 4e5, 6e5, 8e5, 1e6});
//record search index and raw data duration
prometheus::Family<prometheus::Histogram> &search_data_duration_seconds_ = prometheus::BuildHistogram()
.Name("search_data_duration_microseconds")
.Help("histograms of processing time for search index and raw data")
.Register(*registry_);
prometheus::Histogram &search_index_data_duration_seconds_histogram_ =
// record search index and raw data duration
prometheus::Family<prometheus::Histogram>& search_data_duration_seconds_ =
prometheus::BuildHistogram()
.Name("search_data_duration_microseconds")
.Help("histograms of processing time for search index and raw data")
.Register(*registry_);
prometheus::Histogram& search_index_data_duration_seconds_histogram_ =
search_data_duration_seconds_.Add({{"type", "index"}}, BucketBoundaries{1e5, 2e5, 4e5, 6e5, 8e5});
prometheus::Histogram &search_raw_data_duration_seconds_histogram_ =
prometheus::Histogram& search_raw_data_duration_seconds_histogram_ =
search_data_duration_seconds_.Add({{"type", "raw"}}, BucketBoundaries{1e5, 2e5, 4e5, 6e5, 8e5});
////all form Cache.cpp
//record cache usage, when insert/erase/clear/free
// record cache usage, when insert/erase/clear/free
////all from Meta.cpp
//record meta visit count and time
// prometheus::Family<prometheus::Counter> &meta_visit_ = prometheus::BuildCounter()
// .Name("meta_visit_total")
// .Help("the number of accessing Meta")
// .Register(*registry_);
// prometheus::Counter &meta_visit_total_ = meta_visit_.Add({{}});
//
// prometheus::Family<prometheus::Histogram> &meta_visit_duration_seconds_ = prometheus::BuildHistogram()
// .Name("meta_visit_duration_seconds")
// .Help("histogram of processing time to get data from mata")
// .Register(*registry_);
// prometheus::Histogram &meta_visit_duration_seconds_histogram_ =
// meta_visit_duration_seconds_.Add({{}}, BucketBoundaries{0.1, 1.0, 10.0});
// record meta visit count and time
// prometheus::Family<prometheus::Counter> &meta_visit_ = prometheus::BuildCounter()
// .Name("meta_visit_total")
// .Help("the number of accessing Meta")
// .Register(*registry_);
// prometheus::Counter &meta_visit_total_ = meta_visit_.Add({{}});
//
// prometheus::Family<prometheus::Histogram> &meta_visit_duration_seconds_ = prometheus::BuildHistogram()
// .Name("meta_visit_duration_seconds")
// .Help("histogram of processing time to get data from mata")
// .Register(*registry_);
// prometheus::Histogram &meta_visit_duration_seconds_histogram_ =
// meta_visit_duration_seconds_.Add({{}}, BucketBoundaries{0.1, 1.0, 10.0});
////all from MemManager.cpp
//record memory usage percent
prometheus::Family<prometheus::Gauge> &mem_usage_percent_ = prometheus::BuildGauge()
.Name("memory_usage_percent")
.Help("memory usage percent")
.Register(*registry_);
prometheus::Gauge &mem_usage_percent_gauge_ = mem_usage_percent_.Add({});
// record memory usage percent
prometheus::Family<prometheus::Gauge>& mem_usage_percent_ =
prometheus::BuildGauge().Name("memory_usage_percent").Help("memory usage percent").Register(*registry_);
prometheus::Gauge& mem_usage_percent_gauge_ = mem_usage_percent_.Add({});
//record memory usage toal
prometheus::Family<prometheus::Gauge> &mem_usage_total_ = prometheus::BuildGauge()
.Name("memory_usage_total")
.Help("memory usage total")
.Register(*registry_);
prometheus::Gauge &mem_usage_total_gauge_ = mem_usage_total_.Add({});
// record memory usage toal
prometheus::Family<prometheus::Gauge>& mem_usage_total_ =
prometheus::BuildGauge().Name("memory_usage_total").Help("memory usage total").Register(*registry_);
prometheus::Gauge& mem_usage_total_gauge_ = mem_usage_total_.Add({});
////all from DBMetaImpl.cpp
//record meta access count
prometheus::Family<prometheus::Counter> &meta_access_ = prometheus::BuildCounter()
.Name("meta_access_total")
.Help("the number of meta accessing")
.Register(*registry_);
prometheus::Counter &meta_access_total_ = meta_access_.Add({});
// record meta access count
prometheus::Family<prometheus::Counter>& meta_access_ =
prometheus::BuildCounter().Name("meta_access_total").Help("the number of meta accessing").Register(*registry_);
prometheus::Counter& meta_access_total_ = meta_access_.Add({});
//record meta access duration
prometheus::Family<prometheus::Histogram> &meta_access_duration_seconds_ = prometheus::BuildHistogram()
.Name("meta_access_duration_microseconds")
.Help("histogram of processing time for accessing mata")
.Register(*registry_);
prometheus::Histogram &meta_access_duration_seconds_histogram_ =
// record meta access duration
prometheus::Family<prometheus::Histogram>& meta_access_duration_seconds_ =
prometheus::BuildHistogram()
.Name("meta_access_duration_microseconds")
.Help("histogram of processing time for accessing mata")
.Register(*registry_);
prometheus::Histogram& meta_access_duration_seconds_histogram_ =
meta_access_duration_seconds_.Add({}, BucketBoundaries{100, 300, 500, 700, 900, 2000, 4000, 6000, 8000, 20000});
////all from FaissExecutionEngine.cpp
//record data loading from disk count, size, duration, IO speed
prometheus::Family<prometheus::Histogram> &disk_load_duration_second_ = prometheus::BuildHistogram()
.Name("disk_load_duration_microseconds")
.Help("Histogram of processing time for loading data from disk")
.Register(*registry_);
prometheus::Histogram &faiss_disk_load_duration_seconds_histogram_ =
// record data loading from disk count, size, duration, IO speed
prometheus::Family<prometheus::Histogram>& disk_load_duration_second_ =
prometheus::BuildHistogram()
.Name("disk_load_duration_microseconds")
.Help("Histogram of processing time for loading data from disk")
.Register(*registry_);
prometheus::Histogram& faiss_disk_load_duration_seconds_histogram_ =
disk_load_duration_second_.Add({{"DB", "Faiss"}}, BucketBoundaries{2e5, 4e5, 6e5, 8e5});
prometheus::Family<prometheus::Histogram> &disk_load_size_bytes_ = prometheus::BuildHistogram()
.Name("disk_load_size_bytes")
.Help("Histogram of data size by bytes for loading data from disk")
.Register(*registry_);
prometheus::Histogram &faiss_disk_load_size_bytes_histogram_ =
prometheus::Family<prometheus::Histogram>& disk_load_size_bytes_ =
prometheus::BuildHistogram()
.Name("disk_load_size_bytes")
.Help("Histogram of data size by bytes for loading data from disk")
.Register(*registry_);
prometheus::Histogram& faiss_disk_load_size_bytes_histogram_ =
disk_load_size_bytes_.Add({{"DB", "Faiss"}}, BucketBoundaries{1e9, 2e9, 4e9, 6e9, 8e9});
// prometheus::Family<prometheus::Histogram> &disk_load_IO_speed_ = prometheus::BuildHistogram()
// .Name("disk_load_IO_speed_byte_per_sec")
// .Help("Histogram of IO speed for loading data from disk")
// .Register(*registry_);
// prometheus::Histogram &faiss_disk_load_IO_speed_histogram_ =
// disk_load_IO_speed_.Add({{"DB","Faiss"}},BucketBoundaries{1000, 2000, 3000, 4000, 6000, 8000});
// prometheus::Family<prometheus::Histogram> &disk_load_IO_speed_ = prometheus::BuildHistogram()
// .Name("disk_load_IO_speed_byte_per_sec")
// .Help("Histogram of IO speed for loading data from disk")
// .Register(*registry_);
// prometheus::Histogram &faiss_disk_load_IO_speed_histogram_ =
// disk_load_IO_speed_.Add({{"DB","Faiss"}},BucketBoundaries{1000, 2000, 3000, 4000, 6000, 8000});
prometheus::Family<prometheus::Gauge> &faiss_disk_load_IO_speed_ = prometheus::BuildGauge()
.Name("disk_load_IO_speed_byte_per_microsec")
.Help("disk IO speed ")
.Register(*registry_);
prometheus::Gauge &faiss_disk_load_IO_speed_gauge_ = faiss_disk_load_IO_speed_.Add({{"DB", "Faiss"}});
prometheus::Family<prometheus::Gauge>& faiss_disk_load_IO_speed_ = prometheus::BuildGauge()
.Name("disk_load_IO_speed_byte_per_microsec")
.Help("disk IO speed ")
.Register(*registry_);
prometheus::Gauge& faiss_disk_load_IO_speed_gauge_ = faiss_disk_load_IO_speed_.Add({{"DB", "Faiss"}});
////all from CacheMgr.cpp
//record cache access count
prometheus::Family<prometheus::Counter> &cache_access_ = prometheus::BuildCounter()
.Name("cache_access_total")
.Help("the count of accessing cache ")
.Register(*registry_);
prometheus::Counter &cache_access_total_ = cache_access_.Add({});
// record cache access count
prometheus::Family<prometheus::Counter>& cache_access_ = prometheus::BuildCounter()
.Name("cache_access_total")
.Help("the count of accessing cache ")
.Register(*registry_);
prometheus::Counter& cache_access_total_ = cache_access_.Add({});
// record CPU cache usage and %
prometheus::Family<prometheus::Gauge> &cpu_cache_usage_ = prometheus::BuildGauge()
.Name("cache_usage_bytes")
.Help("current cache usage by bytes")
.Register(*registry_);
prometheus::Gauge &cpu_cache_usage_gauge_ = cpu_cache_usage_.Add({});
prometheus::Family<prometheus::Gauge>& cpu_cache_usage_ =
prometheus::BuildGauge().Name("cache_usage_bytes").Help("current cache usage by bytes").Register(*registry_);
prometheus::Gauge& cpu_cache_usage_gauge_ = cpu_cache_usage_.Add({});
//record GPU cache usage and %
prometheus::Family<prometheus::Gauge> &gpu_cache_usage_ = prometheus::BuildGauge()
.Name("gpu_cache_usage_bytes")
.Help("current gpu cache usage by bytes")
.Register(*registry_);
// record GPU cache usage and %
prometheus::Family<prometheus::Gauge>& gpu_cache_usage_ = prometheus::BuildGauge()
.Name("gpu_cache_usage_bytes")
.Help("current gpu cache usage by bytes")
.Register(*registry_);
// record query response
using Quantiles = std::vector<prometheus::detail::CKMSQuantiles::Quantile>;
prometheus::Family<prometheus::Summary> &query_response_ = prometheus::BuildSummary()
.Name("query_response_summary")
.Help("query response summary")
.Register(*registry_);
prometheus::Summary
&query_response_summary_ = query_response_.Add({}, Quantiles{{0.95, 0.00}, {0.9, 0.05}, {0.8, 0.1}});
prometheus::Family<prometheus::Summary>& query_response_ =
prometheus::BuildSummary().Name("query_response_summary").Help("query response summary").Register(*registry_);
prometheus::Summary& query_response_summary_ =
query_response_.Add({}, Quantiles{{0.95, 0.00}, {0.9, 0.05}, {0.8, 0.1}});
prometheus::Family<prometheus::Summary> &query_vector_response_ = prometheus::BuildSummary()
.Name("query_vector_response_summary")
.Help("query each vector response summary")
.Register(*registry_);
prometheus::Summary &query_vector_response_summary_ =
prometheus::Family<prometheus::Summary>& query_vector_response_ = prometheus::BuildSummary()
.Name("query_vector_response_summary")
.Help("query each vector response summary")
.Register(*registry_);
prometheus::Summary& query_vector_response_summary_ =
query_vector_response_.Add({}, Quantiles{{0.95, 0.00}, {0.9, 0.05}, {0.8, 0.1}});
prometheus::Family<prometheus::Gauge> &query_vector_response_per_second_ = prometheus::BuildGauge()
.Name("query_vector_response_per_microsecond")
.Help("the number of vectors can be queried every second ")
.Register(*registry_);
prometheus::Gauge &query_vector_response_per_second_gauge_ = query_vector_response_per_second_.Add({});
prometheus::Family<prometheus::Gauge>& query_vector_response_per_second_ =
prometheus::BuildGauge()
.Name("query_vector_response_per_microsecond")
.Help("the number of vectors can be queried every second ")
.Register(*registry_);
prometheus::Gauge& query_vector_response_per_second_gauge_ = query_vector_response_per_second_.Add({});
prometheus::Family<prometheus::Gauge> &query_response_per_second_ = prometheus::BuildGauge()
.Name("query_response_per_microsecond")
.Help("the number of queries can be processed every microsecond")
.Register(*registry_);
prometheus::Gauge &query_response_per_second_gauge = query_response_per_second_.Add({});
prometheus::Family<prometheus::Gauge>& query_response_per_second_ =
prometheus::BuildGauge()
.Name("query_response_per_microsecond")
.Help("the number of queries can be processed every microsecond")
.Register(*registry_);
prometheus::Gauge& query_response_per_second_gauge = query_response_per_second_.Add({});
prometheus::Family<prometheus::Gauge> &disk_store_IO_speed_ = prometheus::BuildGauge()
.Name("disk_store_IO_speed_bytes_per_microseconds")
.Help("disk_store_IO_speed")
.Register(*registry_);
prometheus::Gauge &disk_store_IO_speed_gauge_ = disk_store_IO_speed_.Add({});
prometheus::Family<prometheus::Gauge>& disk_store_IO_speed_ =
prometheus::BuildGauge()
.Name("disk_store_IO_speed_bytes_per_microseconds")
.Help("disk_store_IO_speed")
.Register(*registry_);
prometheus::Gauge& disk_store_IO_speed_gauge_ = disk_store_IO_speed_.Add({});
prometheus::Family<prometheus::Gauge> &data_file_size_ = prometheus::BuildGauge()
.Name("data_file_size_bytes")
.Help("data file size by bytes")
.Register(*registry_);
prometheus::Gauge &data_file_size_gauge_ = data_file_size_.Add({});
prometheus::Family<prometheus::Gauge>& data_file_size_ =
prometheus::BuildGauge().Name("data_file_size_bytes").Help("data file size by bytes").Register(*registry_);
prometheus::Gauge& data_file_size_gauge_ = data_file_size_.Add({});
prometheus::Family<prometheus::Gauge> &add_vectors_ = prometheus::BuildGauge()
.Name("add_vectors")
.Help("current added vectors")
.Register(*registry_);
prometheus::Gauge &add_vectors_success_gauge_ = add_vectors_.Add({{"outcome", "success"}});
prometheus::Gauge &add_vectors_fail_gauge_ = add_vectors_.Add({{"outcome", "fail"}});
prometheus::Family<prometheus::Gauge>& add_vectors_ =
prometheus::BuildGauge().Name("add_vectors").Help("current added vectors").Register(*registry_);
prometheus::Gauge& add_vectors_success_gauge_ = add_vectors_.Add({{"outcome", "success"}});
prometheus::Gauge& add_vectors_fail_gauge_ = add_vectors_.Add({{"outcome", "fail"}});
prometheus::Family<prometheus::Gauge> &add_vectors_per_second_ = prometheus::BuildGauge()
.Name("add_vectors_throughput_per_microsecond")
.Help("add vectors throughput per microsecond")
.Register(*registry_);
prometheus::Gauge &add_vectors_per_second_gauge_ = add_vectors_per_second_.Add({});
prometheus::Family<prometheus::Gauge>& add_vectors_per_second_ = prometheus::BuildGauge()
.Name("add_vectors_throughput_per_microsecond")
.Help("add vectors throughput per microsecond")
.Register(*registry_);
prometheus::Gauge& add_vectors_per_second_gauge_ = add_vectors_per_second_.Add({});
prometheus::Family<prometheus::Gauge> &CPU_ = prometheus::BuildGauge()
.Name("CPU_usage_percent")
.Help("CPU usage percent by this this process")
.Register(*registry_);
prometheus::Gauge &CPU_usage_percent_ = CPU_.Add({{"CPU", "avg"}});
prometheus::Family<prometheus::Gauge>& CPU_ = prometheus::BuildGauge()
.Name("CPU_usage_percent")
.Help("CPU usage percent by this this process")
.Register(*registry_);
prometheus::Gauge& CPU_usage_percent_ = CPU_.Add({{"CPU", "avg"}});
prometheus::Family<prometheus::Gauge> &RAM_ = prometheus::BuildGauge()
.Name("RAM_usage_percent")
.Help("RAM usage percent by this process")
.Register(*registry_);
prometheus::Gauge &RAM_usage_percent_ = RAM_.Add({});
prometheus::Family<prometheus::Gauge>& RAM_ = prometheus::BuildGauge()
.Name("RAM_usage_percent")
.Help("RAM usage percent by this process")
.Register(*registry_);
prometheus::Gauge& RAM_usage_percent_ = RAM_.Add({});
//GPU Usage Percent
prometheus::Family<prometheus::Gauge> &GPU_percent_ = prometheus::BuildGauge()
.Name("Gpu_usage_percent")
.Help("GPU_usage_percent ")
.Register(*registry_);
// GPU Usage Percent
prometheus::Family<prometheus::Gauge>& GPU_percent_ =
prometheus::BuildGauge().Name("Gpu_usage_percent").Help("GPU_usage_percent ").Register(*registry_);
//GPU Mempry used
prometheus::Family<prometheus::Gauge> &GPU_memory_usage_ = prometheus::BuildGauge()
.Name("GPU_memory_usage_total")
.Help("GPU memory usage total ")
.Register(*registry_);
// GPU Mempry used
prometheus::Family<prometheus::Gauge>& GPU_memory_usage_ =
prometheus::BuildGauge().Name("GPU_memory_usage_total").Help("GPU memory usage total ").Register(*registry_);
prometheus::Family<prometheus::Gauge> &query_index_type_per_second_ = prometheus::BuildGauge()
.Name("query_index_throughtout_per_microsecond")
.Help("query index throughtout per microsecond")
.Register(*registry_);
prometheus::Gauge
&query_index_IVF_type_per_second_gauge_ = query_index_type_per_second_.Add({{"IndexType", "IVF"}});
prometheus::Gauge
&query_index_IDMAP_type_per_second_gauge_ = query_index_type_per_second_.Add({{"IndexType", "IDMAP"}});
prometheus::Family<prometheus::Gauge>& query_index_type_per_second_ =
prometheus::BuildGauge()
.Name("query_index_throughtout_per_microsecond")
.Help("query index throughtout per microsecond")
.Register(*registry_);
prometheus::Gauge& query_index_IVF_type_per_second_gauge_ =
query_index_type_per_second_.Add({{"IndexType", "IVF"}});
prometheus::Gauge& query_index_IDMAP_type_per_second_gauge_ =
query_index_type_per_second_.Add({{"IndexType", "IDMAP"}});
prometheus::Family<prometheus::Gauge> &connection_ = prometheus::BuildGauge()
.Name("connection_number")
.Help("the number of connections")
.Register(*registry_);
prometheus::Gauge &connection_gauge_ = connection_.Add({});
prometheus::Family<prometheus::Gauge>& connection_ =
prometheus::BuildGauge().Name("connection_number").Help("the number of connections").Register(*registry_);
prometheus::Gauge& connection_gauge_ = connection_.Add({});
prometheus::Family<prometheus::Counter> &keeping_alive_ = prometheus::BuildCounter()
.Name("keeping_alive_seconds_total")
.Help("total seconds of the serve alive")
.Register(*registry_);
prometheus::Counter &keeping_alive_counter_ = keeping_alive_.Add({});
prometheus::Family<prometheus::Counter>& keeping_alive_ = prometheus::BuildCounter()
.Name("keeping_alive_seconds_total")
.Help("total seconds of the serve alive")
.Register(*registry_);
prometheus::Counter& keeping_alive_counter_ = keeping_alive_.Add({});
prometheus::Family<prometheus::Gauge> &octets_ = prometheus::BuildGauge()
.Name("octets_bytes_per_second")
.Help("octets bytes per second")
.Register(*registry_);
prometheus::Gauge &inoctets_gauge_ = octets_.Add({{"type", "inoctets"}});
prometheus::Gauge &outoctets_gauge_ = octets_.Add({{"type", "outoctets"}});
prometheus::Family<prometheus::Gauge>& octets_ =
prometheus::BuildGauge().Name("octets_bytes_per_second").Help("octets bytes per second").Register(*registry_);
prometheus::Gauge& inoctets_gauge_ = octets_.Add({{"type", "inoctets"}});
prometheus::Gauge& outoctets_gauge_ = octets_.Add({{"type", "outoctets"}});
prometheus::Family<prometheus::Gauge> &GPU_temperature_ = prometheus::BuildGauge()
.Name("GPU_temperature")
.Help("GPU temperature")
.Register(*registry_);
prometheus::Family<prometheus::Gauge>& GPU_temperature_ =
prometheus::BuildGauge().Name("GPU_temperature").Help("GPU temperature").Register(*registry_);
prometheus::Family<prometheus::Gauge> &CPU_temperature_ = prometheus::BuildGauge()
.Name("CPU_temperature")
.Help("CPU temperature")
.Register(*registry_);
prometheus::Family<prometheus::Gauge>& CPU_temperature_ =
prometheus::BuildGauge().Name("CPU_temperature").Help("CPU temperature").Register(*registry_);
};
} // namespace server
} // namespace milvus
} // namespace zilliz
} // namespace server
} // namespace milvus
} // namespace zilliz

52
cpp/src/metrics/SystemInfo.cpp
View File

@ -15,14 +15,13 @@
// specific language governing permissions and limitations
// under the License.
#include "metrics/SystemInfo.h"
#include <nvml.h>
#include <sys/types.h>
#include <unistd.h>
#include <iostream>
#include <fstream>
#include <nvml.h>
#include <iostream>
#include <string>
#include <utility>
@ -37,7 +36,7 @@ SystemInfo::Init() {
initialized_ = true;
// initialize CPU information
FILE *file;
FILE* file;
struct tms time_sample;
char line[128];
last_cpu_ = times(&time_sample);
@ -54,7 +53,7 @@ SystemInfo::Init() {
total_ram_ = GetPhysicalMemory();
fclose(file);
//initialize GPU information
// initialize GPU information
nvmlReturn_t nvmlresult;
nvmlresult = nvmlInit();
if (NVML_SUCCESS != nvmlresult) {
@ -67,7 +66,7 @@ SystemInfo::Init() {
return;
}
//initialize network traffic information
// initialize network traffic information
std::pair<uint64_t, uint64_t> in_and_out_octets = Octets();
in_octets_ = in_and_out_octets.first;
out_octets_ = in_and_out_octets.second;
@ -75,10 +74,10 @@ SystemInfo::Init() {
}
uint64_t
SystemInfo::ParseLine(char *line) {
SystemInfo::ParseLine(char* line) {
// This assumes that a digit will be found and the line ends in " Kb".
int i = strlen(line);
const char *p = line;
const char* p = line;
while (*p < '0' || *p > '9') p++;
line[i - 3] = '\0';
i = atoi(p);
@ -90,15 +89,15 @@ SystemInfo::GetPhysicalMemory() {
struct sysinfo memInfo;
sysinfo(&memInfo);
uint64_t totalPhysMem = memInfo.totalram;
//Multiply in next statement to avoid int overflow on right hand side...
// Multiply in next statement to avoid int overflow on right hand side...
totalPhysMem *= memInfo.mem_unit;
return totalPhysMem;
}
uint64_t
SystemInfo::GetProcessUsedMemory() {
//Note: this value is in KB!
FILE *file = fopen("/proc/self/status", "r");
// Note: this value is in KB!
FILE* file = fopen("/proc/self/status", "r");
constexpr uint64_t line_length = 128;
uint64_t result = -1;
constexpr uint64_t KB_SIZE = 1024;
@ -118,7 +117,7 @@ SystemInfo::GetProcessUsedMemory() {
double
SystemInfo::MemoryPercent() {
if (!initialized_) Init();
return (double) (GetProcessUsedMemory() * 100) / (double) total_ram_;
return (double)(GetProcessUsedMemory() * 100) / (double)total_ram_;
}
std::vector<double>
@ -139,9 +138,9 @@ SystemInfo::CPUCorePercent() {
}
std::vector<uint64_t>
SystemInfo::getTotalCpuTime(std::vector<uint64_t> &work_time_array) {
SystemInfo::getTotalCpuTime(std::vector<uint64_t>& work_time_array) {
std::vector<uint64_t> total_time_array;
FILE *file = fopen("/proc/stat", "r");
FILE* file = fopen("/proc/stat", "r");
if (file == NULL) {
perror("Could not open stat file");
return total_time_array;
@ -152,16 +151,15 @@ SystemInfo::getTotalCpuTime(std::vector<uint64_t> &work_time_array) {
for (int i = 0; i < num_processors_; i++) {
char buffer[1024];
char *ret = fgets(buffer, sizeof(buffer) - 1, file);
char* ret = fgets(buffer, sizeof(buffer) - 1, file);
if (ret == NULL) {
perror("Could not read stat file");
fclose(file);
return total_time_array;
}
sscanf(buffer,
"cpu %16lu %16lu %16lu %16lu %16lu %16lu %16lu %16lu %16lu %16lu",
&user, &nice, &system, &idle, &iowait, &irq, &softirq, &steal, &guest, &guestnice);
sscanf(buffer, "cpu %16lu %16lu %16lu %16lu %16lu %16lu %16lu %16lu %16lu %16lu", &user, &nice, &system, &idle,
&iowait, &irq, &softirq, &steal, &guest, &guestnice);
work_time_array.push_back(user + nice + system);
total_time_array.push_back(user + nice + system + idle + iowait + irq + softirq + steal);
@ -179,13 +177,11 @@ SystemInfo::CPUPercent() {
double percent;
now = times(&time_sample);
if (now <= last_cpu_ || time_sample.tms_stime < last_sys_cpu_ ||
time_sample.tms_utime < last_user_cpu_) {
//Overflow detection. Just skip this value.
if (now <= last_cpu_ || time_sample.tms_stime < last_sys_cpu_ || time_sample.tms_utime < last_user_cpu_) {
// Overflow detection. Just skip this value.
percent = -1.0;
} else {
percent = (time_sample.tms_stime - last_sys_cpu_) +
(time_sample.tms_utime - last_user_cpu_);
percent = (time_sample.tms_stime - last_sys_cpu_) + (time_sample.tms_utime - last_user_cpu_);
percent /= (now - last_cpu_);
percent *= 100;
}
@ -230,7 +226,7 @@ SystemInfo::CPUTemperature() {
std::vector<float> result;
for (int i = 0; i <= num_physical_processors_; ++i) {
std::string path = "/sys/class/thermal/thermal_zone" + std::to_string(i) + "/temp";
FILE *file = fopen(path.data(), "r");
FILE* file = fopen(path.data(), "r");
if (file == NULL) {
perror("Could not open thermal file");
return result;
@ -261,7 +257,7 @@ SystemInfo::GPUMemoryUsed() {
std::pair<uint64_t, uint64_t>
SystemInfo::Octets() {
pid_t pid = getpid();
// const std::string filename = "/proc/"+std::to_string(pid)+"/net/netstat";
// const std::string filename = "/proc/"+std::to_string(pid)+"/net/netstat";
const std::string filename = "/proc/net/netstat";
std::ifstream file(filename);
std::string lastline = "";
@ -293,6 +289,6 @@ SystemInfo::Octets() {
return res;
}
} // namespace server
} // namespace milvus
} // namespace zilliz
} // namespace server
} // namespace milvus
} // namespace zilliz

87
cpp/src/metrics/SystemInfo.h
View File

@ -15,20 +15,19 @@
// specific language governing permissions and limitations
// under the License.
#pragma once
#include <sys/types.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <sys/sysinfo.h>
#include <sys/times.h>
#include <sys/types.h>
#include <sys/vtimes.h>
#include <stdlib.h>
#include <stdio.h>
#include <string.h>
#include <chrono>
#include <unordered_map>
#include <vector>
#include <utility>
#include <vector>
namespace zilliz {
namespace milvus {
@ -43,72 +42,94 @@ class SystemInfo {
std::chrono::system_clock::time_point net_time_ = std::chrono::system_clock::now();
int num_processors_ = 0;
int num_physical_processors_ = 0;
//number of GPU
// number of GPU
uint32_t num_device_ = 0;
uint64_t in_octets_ = 0;
uint64_t out_octets_ = 0;
bool initialized_ = false;
public:
static SystemInfo &
static SystemInfo&
GetInstance() {
static SystemInfo instance;
return instance;
}
void Init();
void
Init();
int num_processor() const {
int
num_processor() const {
return num_processors_;
}
int num_physical_processors() const {
int
num_physical_processors() const {
return num_physical_processors_;
}
uint32_t num_device() const {
uint32_t
num_device() const {
return num_device_;
}
uint64_t get_inoctets() {
uint64_t
get_inoctets() {
return in_octets_;
}
uint64_t get_octets() {
uint64_t
get_octets() {
return out_octets_;
}
std::chrono::system_clock::time_point get_nettime() {
std::chrono::system_clock::time_point
get_nettime() {
return net_time_;
}
void set_inoctets(uint64_t value) {
void
set_inoctets(uint64_t value) {
in_octets_ = value;
}
void set_outoctets(uint64_t value) {
void
set_outoctets(uint64_t value) {
out_octets_ = value;
}
void set_nettime() {
void
set_nettime() {
net_time_ = std::chrono::system_clock::now();
}
uint64_t ParseLine(char *line);
uint64_t GetPhysicalMemory();
uint64_t GetProcessUsedMemory();
double MemoryPercent();
double CPUPercent();
std::pair<uint64_t, uint64_t> Octets();
std::vector<uint64_t> GPUMemoryTotal();
std::vector<uint64_t> GPUMemoryUsed();
uint64_t
ParseLine(char* line);
uint64_t
GetPhysicalMemory();
uint64_t
GetProcessUsedMemory();
double
MemoryPercent();
double
CPUPercent();
std::pair<uint64_t, uint64_t>
Octets();
std::vector<uint64_t>
GPUMemoryTotal();
std::vector<uint64_t>
GPUMemoryUsed();
std::vector<double> CPUCorePercent();
std::vector<uint64_t> getTotalCpuTime(std::vector<uint64_t> &workTime);
std::vector<uint64_t> GPUTemperature();
std::vector<float> CPUTemperature();
std::vector<double>
CPUCorePercent();
std::vector<uint64_t>
getTotalCpuTime(std::vector<uint64_t>& workTime);
std::vector<uint64_t>
GPUTemperature();
std::vector<float>
CPUTemperature();
};
} // namespace server
} // namespace milvus
} // namespace zilliz
} // namespace server
} // namespace milvus
} // namespace zilliz

19
cpp/src/scheduler/Algorithm.cpp
View File

@ -15,7 +15,6 @@
// specific language governing permissions and limitations
// under the License.
#include "scheduler/Algorithm.h"
#include <limits>
@ -29,10 +28,8 @@ namespace scheduler {
constexpr uint64_t MAXINT = std::numeric_limits<uint32_t>::max();
uint64_t
ShortestPath(const ResourcePtr &src,
const ResourcePtr &dest,
const ResourceMgrPtr &res_mgr,
std::vector<std::string> &path) {
ShortestPath(const ResourcePtr& src, const ResourcePtr& dest, const ResourceMgrPtr& res_mgr,
std::vector<std::string>& path) {
std::vector<std::vector<std::string>> paths;
uint64_t num_of_resources = res_mgr->GetAllResources().size();
@ -43,7 +40,7 @@ ShortestPath(const ResourcePtr &src,
name_id_map.insert(std::make_pair(res_mgr->GetAllResources().at(i)->name(), i));
}
std::vector<std::vector<uint64_t> > dis_matrix;
std::vector<std::vector<uint64_t>> dis_matrix;
dis_matrix.resize(num_of_resources);
for (uint64_t i = 0; i < num_of_resources; ++i) {
dis_matrix[i].resize(num_of_resources);
@ -55,11 +52,11 @@ ShortestPath(const ResourcePtr &src,
std::vector<bool> vis(num_of_resources, false);
std::vector<uint64_t> dis(num_of_resources, MAXINT);
for (auto &res : res_mgr->GetAllResources()) {
for (auto& res : res_mgr->GetAllResources()) {
auto cur_node = std::static_pointer_cast<Node>(res);
auto cur_neighbours = cur_node->GetNeighbours();
for (auto &neighbour : cur_neighbours) {
for (auto& neighbour : cur_neighbours) {
auto neighbour_res = std::static_pointer_cast<Resource>(neighbour.neighbour_node.lock());
dis_matrix[name_id_map.at(res->name())][name_id_map.at(neighbour_res->name())] =
neighbour.connection.transport_cost();
@ -107,6 +104,6 @@ ShortestPath(const ResourcePtr &src,
return dis[name_id_map.at(dest->name())];
}
} // namespace scheduler
} // namespace milvus
} // namespace zilliz
} // namespace scheduler
} // namespace milvus
} // namespace zilliz

17
cpp/src/scheduler/Algorithm.h
View File

@ -15,23 +15,20 @@
// specific language governing permissions and limitations
// under the License.
#include "resource/Resource.h"
#include "ResourceMgr.h"
#include "resource/Resource.h"
#include <vector>
#include <string>
#include <vector>
namespace zilliz {
namespace milvus {
namespace scheduler {
uint64_t
ShortestPath(const ResourcePtr &src,
const ResourcePtr &dest,
const ResourceMgrPtr &res_mgr,
std::vector<std::string> &path);
ShortestPath(const ResourcePtr& src, const ResourcePtr& dest, const ResourceMgrPtr& res_mgr,
std::vector<std::string>& path);
} // namespace scheduler
} // namespace milvus
} // namespace zilliz
} // namespace scheduler
} // namespace milvus
} // namespace zilliz

24
cpp/src/scheduler/Definition.h
View File

@ -15,20 +15,20 @@
// specific language governing permissions and limitations
// under the License.
#include <string>
#include <vector>
#include <list>
#include <queue>
#include <deque>
#include <unordered_map>
#include <thread>
#include <mutex>
#include <condition_variable>
#include <deque>
#include <list>
#include <memory>
#include <mutex>
#include <queue>
#include <string>
#include <thread>
#include <unordered_map>
#include <vector>
#include "db/meta/MetaTypes.h"
#include "db/engine/EngineFactory.h"
#include "db/engine/ExecutionEngine.h"
#include "db/meta/MetaTypes.h"
namespace zilliz {
namespace milvus {
@ -42,6 +42,6 @@ using EngineFactory = engine::EngineFactory;
using EngineType = engine::EngineType;
using MetricType = engine::MetricType;
} // namespace scheduler
} // namespace milvus
} // namespace zilliz
} // namespace scheduler
} // namespace milvus
} // namespace zilliz

21
cpp/src/scheduler/JobMgr.cpp
View File

@ -16,8 +16,8 @@
// under the License.
#include "scheduler/JobMgr.h"
#include "task/Task.h"
#include "TaskCreator.h"
#include "task/Task.h"
#include <utility>
@ -25,8 +25,7 @@ namespace zilliz {
namespace milvus {
namespace scheduler {
JobMgr::JobMgr(ResourceMgrPtr res_mgr)
: res_mgr_(std::move(res_mgr)) {
JobMgr::JobMgr(ResourceMgrPtr res_mgr) : res_mgr_(std::move(res_mgr)) {
}
void
@ -47,7 +46,7 @@ JobMgr::Stop() {
}
void
JobMgr::Put(const JobPtr &job) {
JobMgr::Put(const JobPtr& job) {
{
std::lock_guard<std::mutex> lock(mutex_);
queue_.push(job);
@ -59,9 +58,7 @@ void
JobMgr::worker_function() {
while (running_) {
std::unique_lock<std::mutex> lock(mutex_);
cv_.wait(lock, [this] {
return !queue_.empty();
});
cv_.wait(lock, [this] { return !queue_.empty(); });
auto job = queue_.front();
queue_.pop();
lock.unlock();
@ -73,7 +70,7 @@ JobMgr::worker_function() {
auto disk_list = res_mgr_->GetDiskResources();
if (!disk_list.empty()) {
if (auto disk = disk_list[0].lock()) {
for (auto &task : tasks) {
for (auto& task : tasks) {
disk->task_table().Put(task);
}
}
@ -82,10 +79,10 @@ JobMgr::worker_function() {
}
std::vector<TaskPtr>
JobMgr::build_task(const JobPtr &job) {
JobMgr::build_task(const JobPtr& job) {
return TaskCreator::Create(job);
}
} // namespace scheduler
} // namespace milvus
} // namespace zilliz
} // namespace scheduler
} // namespace milvus
} // namespace zilliz

28
cpp/src/scheduler/JobMgr.h
View File

@ -16,20 +16,20 @@
// under the License.
#pragma once
#include <string>
#include <vector>
#include <list>
#include <queue>
#include <deque>
#include <unordered_map>
#include <thread>
#include <mutex>
#include <condition_variable>
#include <deque>
#include <list>
#include <memory>
#include <mutex>
#include <queue>
#include <string>
#include <thread>
#include <unordered_map>
#include <vector>
#include "ResourceMgr.h"
#include "job/Job.h"
#include "task/Task.h"
#include "ResourceMgr.h"
namespace zilliz {
namespace milvus {
@ -47,14 +47,14 @@ class JobMgr {
public:
void
Put(const JobPtr &job);
Put(const JobPtr& job);
private:
void
worker_function();
std::vector<TaskPtr>
build_task(const JobPtr &job);
build_task(const JobPtr& job);
private:
bool running_ = false;
@ -70,6 +70,6 @@ class JobMgr {
using JobMgrPtr = std::shared_ptr<JobMgr>;
} // namespace scheduler
} // namespace milvus
} // namespace zilliz
} // namespace scheduler
} // namespace milvus
} // namespace zilliz

12
cpp/src/scheduler/ResourceFactory.cpp
View File

@ -15,7 +15,6 @@
// specific language governing permissions and limitations
// under the License.
#include "scheduler/ResourceFactory.h"
namespace zilliz {
@ -23,10 +22,7 @@ namespace milvus {
namespace scheduler {
std::shared_ptr<Resource>
ResourceFactory::Create(const std::string &name,
const std::string &type,
uint64_t device_id,
bool enable_loader,
ResourceFactory::Create(const std::string& name, const std::string& type, uint64_t device_id, bool enable_loader,
bool enable_executor) {
if (type == "DISK") {
return std::make_shared<DiskResource>(name, device_id, enable_loader, enable_executor);
@ -39,6 +35,6 @@ ResourceFactory::Create(const std::string &name,
}
}
} // namespace scheduler
} // namespace milvus
} // namespace zilliz
} // namespace scheduler
} // namespace milvus
} // namespace zilliz

17
cpp/src/scheduler/ResourceFactory.h
View File

@ -17,13 +17,13 @@
#pragma once
#include <string>
#include <memory>
#include <string>
#include "resource/Resource.h"
#include "resource/CpuResource.h"
#include "resource/GpuResource.h"
#include "resource/DiskResource.h"
#include "resource/GpuResource.h"
#include "resource/Resource.h"
namespace zilliz {
namespace milvus {
@ -32,13 +32,10 @@ namespace scheduler {
class ResourceFactory {
public:
static std::shared_ptr<Resource>
Create(const std::string &name,
const std::string &type,
uint64_t device_id,
bool enable_loader = true,
Create(const std::string& name, const std::string& type, uint64_t device_id, bool enable_loader = true,
bool enable_executor = true);
};
} // namespace scheduler
} // namespace milvus
} // namespace zilliz
} // namespace scheduler
} // namespace milvus
} // namespace zilliz

38
cpp/src/scheduler/ResourceMgr.cpp
View File

@ -26,7 +26,7 @@ namespace scheduler {
void
ResourceMgr::Start() {
std::lock_guard<std::mutex> lck(resources_mutex_);
for (auto &resource : resources_) {
for (auto& resource : resources_) {
resource->Start();
}
running_ = true;
@ -44,13 +44,13 @@ ResourceMgr::Stop() {
worker_thread_.join();
std::lock_guard<std::mutex> lck(resources_mutex_);
for (auto &resource : resources_) {
for (auto& resource : resources_) {
resource->Stop();
}
}
ResourceWPtr
ResourceMgr::Add(ResourcePtr &&resource) {
ResourceMgr::Add(ResourcePtr&& resource) {
ResourceWPtr ret(resource);
std::lock_guard<std::mutex> lck(resources_mutex_);
@ -70,13 +70,13 @@ ResourceMgr::Add(ResourcePtr &&resource) {
}
bool
ResourceMgr::Connect(const std::string &name1, const std::string &name2, Connection &connection) {
ResourceMgr::Connect(const std::string& name1, const std::string& name2, Connection& connection) {
auto res1 = GetResource(name1);
auto res2 = GetResource(name2);
if (res1 && res2) {
res1->AddNeighbour(std::static_pointer_cast<Node>(res2), connection);
// TODO: enable when task balance supported
// res2->AddNeighbour(std::static_pointer_cast<Node>(res1), connection);
// res2->AddNeighbour(std::static_pointer_cast<Node>(res1), connection);
return true;
}
return false;
@ -92,7 +92,7 @@ ResourceMgr::Clear() {
std::vector<ResourcePtr>
ResourceMgr::GetComputeResources() {
std::vector<ResourcePtr> result;
for (auto &resource : resources_) {
for (auto& resource : resources_) {
if (resource->HasExecutor()) {
result.emplace_back(resource);
}
@ -102,7 +102,7 @@ ResourceMgr::GetComputeResources() {
ResourcePtr
ResourceMgr::GetResource(ResourceType type, uint64_t device_id) {
for (auto &resource : resources_) {
for (auto& resource : resources_) {
if (resource->type() == type && resource->device_id() == device_id) {
return resource;
}
@ -111,8 +111,8 @@ ResourceMgr::GetResource(ResourceType type, uint64_t device_id) {
}
ResourcePtr
ResourceMgr::GetResource(const std::string &name) {
for (auto &resource : resources_) {
ResourceMgr::GetResource(const std::string& name) {
for (auto& resource : resources_) {
if (resource->name() == name) {
return resource;
}
@ -128,7 +128,7 @@ ResourceMgr::GetNumOfResource() const {
uint64_t
ResourceMgr::GetNumOfComputeResource() const {
uint64_t count = 0;
for (auto &res : resources_) {
for (auto& res : resources_) {
if (res->HasExecutor()) {
++count;
}
@ -139,7 +139,7 @@ ResourceMgr::GetNumOfComputeResource() const {
uint64_t
ResourceMgr::GetNumGpuResource() const {
uint64_t num = 0;
for (auto &res : resources_) {
for (auto& res : resources_) {
if (res->type() == ResourceType::GPU) {
num++;
}
@ -153,7 +153,7 @@ ResourceMgr::Dump() {
for (uint64_t i = 0; i < resources_.size(); ++i) {
str += "Resource No." + std::to_string(i) + ":\n";
//str += resources_[i]->Dump();
// str += resources_[i]->Dump();
}
return str;
@ -163,7 +163,7 @@ std::string
ResourceMgr::DumpTaskTables() {
std::stringstream ss;
ss << ">>>>>>>>>>>>>>>ResourceMgr::DumpTaskTable<<<<<<<<<<<<<<<" << std::endl;
for (auto &resource : resources_) {
for (auto& resource : resources_) {
ss << resource->Dump() << std::endl;
ss << resource->task_table().Dump();
ss << resource->Dump() << std::endl << std::endl;
@ -172,7 +172,7 @@ ResourceMgr::DumpTaskTables() {
}
void
ResourceMgr::post_event(const EventPtr &event) {
ResourceMgr::post_event(const EventPtr& event) {
{
std::lock_guard<std::mutex> lock(event_mutex_);
queue_.emplace(event);
@ -184,9 +184,7 @@ void
ResourceMgr::event_process() {
while (running_) {
std::unique_lock<std::mutex> lock(event_mutex_);
event_cv_.wait(lock, [this] {
return !queue_.empty();
});
event_cv_.wait(lock, [this] { return !queue_.empty(); });
auto event = queue_.front();
queue_.pop();
@ -201,6 +199,6 @@ ResourceMgr::event_process() {
}
}
} // namespace scheduler
} // namespace milvus
} // namespace zilliz
} // namespace scheduler
} // namespace milvus
} // namespace zilliz

22
cpp/src/scheduler/ResourceMgr.h
View File

@ -17,13 +17,13 @@
#pragma once
#include <string>
#include <vector>
#include <condition_variable>
#include <memory>
#include <mutex>
#include <queue>
#include <string>
#include <utility>
#include <condition_variable>
#include <vector>
#include "resource/Resource.h"
#include "utils/Log.h"
@ -45,10 +45,10 @@ class ResourceMgr {
Stop();
ResourceWPtr
Add(ResourcePtr &&resource);
Add(ResourcePtr&& resource);
bool
Connect(const std::string &res1, const std::string &res2, Connection &connection);
Connect(const std::string& res1, const std::string& res2, Connection& connection);
void
Clear();
@ -60,7 +60,7 @@ class ResourceMgr {
public:
/******** Management Interface ********/
inline std::vector<ResourceWPtr> &
inline std::vector<ResourceWPtr>&
GetDiskResources() {
return disk_resources_;
}
@ -78,7 +78,7 @@ class ResourceMgr {
GetResource(ResourceType type, uint64_t device_id);
ResourcePtr
GetResource(const std::string &name);
GetResource(const std::string& name);
uint64_t
GetNumOfResource() const;
@ -102,7 +102,7 @@ class ResourceMgr {
private:
void
post_event(const EventPtr &event);
post_event(const EventPtr& event);
void
event_process();
@ -125,6 +125,6 @@ class ResourceMgr {
using ResourceMgrPtr = std::shared_ptr<ResourceMgr>;
using ResourceMgrWPtr = std::weak_ptr<ResourceMgr>;
} // namespace scheduler
} // namespace milvus
} // namespace zilliz
} // namespace scheduler
} // namespace milvus
} // namespace zilliz

153
cpp/src/scheduler/SchedInst.cpp
View File

@ -15,17 +15,16 @@
// specific language governing permissions and limitations
// under the License.
#include "scheduler/SchedInst.h"
#include "server/Config.h"
#include "ResourceFactory.h"
#include "knowhere/index/vector_index/IndexGPUIVF.h"
#include "Utils.h"
#include "knowhere/index/vector_index/IndexGPUIVF.h"
#include "server/Config.h"
#include <vector>
#include <set>
#include <utility>
#include <string>
#include <utility>
#include <vector>
namespace zilliz {
namespace milvus {
@ -42,7 +41,7 @@ std::mutex JobMgrInst::mutex_;
void
load_simple_config() {
server::Config &config = server::Config::GetInstance();
server::Config& config = server::Config::GetInstance();
std::string mode;
config.GetResourceConfigMode(mode);
std::vector<std::string> pool;
@ -50,7 +49,7 @@ load_simple_config() {
bool cpu = false;
std::set<uint64_t> gpu_ids;
for (auto &resource : pool) {
for (auto& resource : pool) {
if (resource == "cpu") {
cpu = true;
break;
@ -78,7 +77,7 @@ load_simple_config() {
ResMgrInst::GetInstance()->Connect("disk", "cpu", io);
auto pcie = Connection("pcie", 12000);
for (auto &gpu_id : gpu_ids) {
for (auto& gpu_id : gpu_ids) {
ResMgrInst::GetInstance()->Add(ResourceFactory::Create(std::to_string(gpu_id), "GPU", gpu_id, true, true));
ResMgrInst::GetInstance()->Connect("cpu", std::to_string(gpu_id), io);
}
@ -87,77 +86,77 @@ load_simple_config() {
void
load_advance_config() {
// try {
// server::ConfigNode &config = server::Config::GetInstance().GetConfig(server::CONFIG_RESOURCE);
//
// if (config.GetChildren().empty()) throw "resource_config null exception";
//
// auto resources = config.GetChild(server::CONFIG_RESOURCES).GetChildren();
//
// if (resources.empty()) throw "Children of resource_config null exception";
//
// for (auto &resource : resources) {
// auto &resname = resource.first;
// auto &resconf = resource.second;
// auto type = resconf.GetValue(server::CONFIG_RESOURCE_TYPE);
//// auto memory = resconf.GetInt64Value(server::CONFIG_RESOURCE_MEMORY);
// auto device_id = resconf.GetInt64Value(server::CONFIG_RESOURCE_DEVICE_ID);
//// auto enable_loader = resconf.GetBoolValue(server::CONFIG_RESOURCE_ENABLE_LOADER);
// auto enable_loader = true;
// auto enable_executor = resconf.GetBoolValue(server::CONFIG_RESOURCE_ENABLE_EXECUTOR);
// auto pinned_memory = resconf.GetInt64Value(server::CONFIG_RESOURCE_PIN_MEMORY);
// auto temp_memory = resconf.GetInt64Value(server::CONFIG_RESOURCE_TEMP_MEMORY);
// auto resource_num = resconf.GetInt64Value(server::CONFIG_RESOURCE_NUM);
//
// auto res = ResMgrInst::GetInstance()->Add(ResourceFactory::Create(resname,
// type,
// device_id,
// enable_loader,
// enable_executor));
//
// if (res.lock()->type() == ResourceType::GPU) {
// auto pinned_memory = resconf.GetInt64Value(server::CONFIG_RESOURCE_PIN_MEMORY, 300);
// auto temp_memory = resconf.GetInt64Value(server::CONFIG_RESOURCE_TEMP_MEMORY, 300);
// auto resource_num = resconf.GetInt64Value(server::CONFIG_RESOURCE_NUM, 2);
// pinned_memory = 1024 * 1024 * pinned_memory;
// temp_memory = 1024 * 1024 * temp_memory;
// knowhere::FaissGpuResourceMgr::GetInstance().InitDevice(device_id,
// pinned_memory,
// temp_memory,
// resource_num);
// }
// }
//
// knowhere::FaissGpuResourceMgr::GetInstance().InitResource();
//
// auto connections = config.GetChild(server::CONFIG_RESOURCE_CONNECTIONS).GetChildren();
// if (connections.empty()) throw "connections config null exception";
// for (auto &conn : connections) {
// auto &connect_name = conn.first;
// auto &connect_conf = conn.second;
// auto connect_speed = connect_conf.GetInt64Value(server::CONFIG_SPEED_CONNECTIONS);
// auto connect_endpoint = connect_conf.GetValue(server::CONFIG_ENDPOINT_CONNECTIONS);
//
// std::string delimiter = "===";
// std::string left = connect_endpoint.substr(0, connect_endpoint.find(delimiter));
// std::string right = connect_endpoint.substr(connect_endpoint.find(delimiter) + 3,
// connect_endpoint.length());
//
// auto connection = Connection(connect_name, connect_speed);
// ResMgrInst::GetInstance()->Connect(left, right, connection);
// }
// } catch (const char *msg) {
// SERVER_LOG_ERROR << msg;
// // TODO: throw exception instead
// exit(-1);
//// throw std::exception();
// }
// try {
// server::ConfigNode &config = server::Config::GetInstance().GetConfig(server::CONFIG_RESOURCE);
//
// if (config.GetChildren().empty()) throw "resource_config null exception";
//
// auto resources = config.GetChild(server::CONFIG_RESOURCES).GetChildren();
//
// if (resources.empty()) throw "Children of resource_config null exception";
//
// for (auto &resource : resources) {
// auto &resname = resource.first;
// auto &resconf = resource.second;
// auto type = resconf.GetValue(server::CONFIG_RESOURCE_TYPE);
//// auto memory = resconf.GetInt64Value(server::CONFIG_RESOURCE_MEMORY);
// auto device_id = resconf.GetInt64Value(server::CONFIG_RESOURCE_DEVICE_ID);
//// auto enable_loader = resconf.GetBoolValue(server::CONFIG_RESOURCE_ENABLE_LOADER);
// auto enable_loader = true;
// auto enable_executor = resconf.GetBoolValue(server::CONFIG_RESOURCE_ENABLE_EXECUTOR);
// auto pinned_memory = resconf.GetInt64Value(server::CONFIG_RESOURCE_PIN_MEMORY);
// auto temp_memory = resconf.GetInt64Value(server::CONFIG_RESOURCE_TEMP_MEMORY);
// auto resource_num = resconf.GetInt64Value(server::CONFIG_RESOURCE_NUM);
//
// auto res = ResMgrInst::GetInstance()->Add(ResourceFactory::Create(resname,
// type,
// device_id,
// enable_loader,
// enable_executor));
//
// if (res.lock()->type() == ResourceType::GPU) {
// auto pinned_memory = resconf.GetInt64Value(server::CONFIG_RESOURCE_PIN_MEMORY, 300);
// auto temp_memory = resconf.GetInt64Value(server::CONFIG_RESOURCE_TEMP_MEMORY, 300);
// auto resource_num = resconf.GetInt64Value(server::CONFIG_RESOURCE_NUM, 2);
// pinned_memory = 1024 * 1024 * pinned_memory;
// temp_memory = 1024 * 1024 * temp_memory;
// knowhere::FaissGpuResourceMgr::GetInstance().InitDevice(device_id,
// pinned_memory,
// temp_memory,
// resource_num);
// }
// }
//
// knowhere::FaissGpuResourceMgr::GetInstance().InitResource();
//
// auto connections = config.GetChild(server::CONFIG_RESOURCE_CONNECTIONS).GetChildren();
// if (connections.empty()) throw "connections config null exception";
// for (auto &conn : connections) {
// auto &connect_name = conn.first;
// auto &connect_conf = conn.second;
// auto connect_speed = connect_conf.GetInt64Value(server::CONFIG_SPEED_CONNECTIONS);
// auto connect_endpoint = connect_conf.GetValue(server::CONFIG_ENDPOINT_CONNECTIONS);
//
// std::string delimiter = "===";
// std::string left = connect_endpoint.substr(0, connect_endpoint.find(delimiter));
// std::string right = connect_endpoint.substr(connect_endpoint.find(delimiter) + 3,
// connect_endpoint.length());
//
// auto connection = Connection(connect_name, connect_speed);
// ResMgrInst::GetInstance()->Connect(left, right, connection);
// }
// } catch (const char *msg) {
// SERVER_LOG_ERROR << msg;
// // TODO: throw exception instead
// exit(-1);
//// throw std::exception();
// }
}
void
StartSchedulerService() {
load_simple_config();
// load_advance_config();
// load_advance_config();
ResMgrInst::GetInstance()->Start();
SchedInst::GetInstance()->Start();
JobMgrInst::GetInstance()->Start();
@ -170,6 +169,6 @@ StopSchedulerService() {
ResMgrInst::GetInstance()->Stop();
}
} // namespace scheduler
} // namespace milvus
} // namespace zilliz
} // namespace scheduler
} // namespace milvus
} // namespace zilliz

10
cpp/src/scheduler/SchedInst.h
View File

@ -17,12 +17,12 @@
#pragma once
#include "JobMgr.h"
#include "ResourceMgr.h"
#include "Scheduler.h"
#include "JobMgr.h"
#include <mutex>
#include <memory>
#include <mutex>
namespace zilliz {
namespace milvus {
@ -88,6 +88,6 @@ StartSchedulerService();
void
StopSchedulerService();
} // namespace scheduler
} // namespace milvus
} // namespace zilliz
} // namespace scheduler
} // namespace milvus
} // namespace zilliz

34
cpp/src/scheduler/Scheduler.cpp
View File

@ -16,10 +16,10 @@
// under the License.
#include "scheduler/Scheduler.h"
#include "Algorithm.h"
#include "action/Action.h"
#include "cache/GpuCacheMgr.h"
#include "event/LoadCompletedEvent.h"
#include "action/Action.h"
#include "Algorithm.h"
#include <utility>
@ -27,9 +27,7 @@ namespace zilliz {
namespace milvus {
namespace scheduler {
Scheduler::Scheduler(ResourceMgrWPtr res_mgr)
: running_(false),
res_mgr_(std::move(res_mgr)) {
Scheduler::Scheduler(ResourceMgrWPtr res_mgr) : running_(false), res_mgr_(std::move(res_mgr)) {
if (auto mgr = res_mgr_.lock()) {
mgr->RegisterSubscriber(std::bind(&Scheduler::PostEvent, this, std::placeholders::_1));
}
@ -61,7 +59,7 @@ Scheduler::Stop() {
}
void
Scheduler::PostEvent(const EventPtr &event) {
Scheduler::PostEvent(const EventPtr& event) {
{
std::lock_guard<std::mutex> lock(event_mutex_);
event_queue_.push(event);
@ -78,9 +76,7 @@ void
Scheduler::worker_function() {
while (running_) {
std::unique_lock<std::mutex> lock(event_mutex_);
event_cv_.wait(lock, [this] {
return !event_queue_.empty();
});
event_cv_.wait(lock, [this] { return !event_queue_.empty(); });
auto event = event_queue_.front();
event_queue_.pop();
if (event == nullptr) {
@ -92,14 +88,14 @@ Scheduler::worker_function() {
}
void
Scheduler::Process(const EventPtr &event) {
Scheduler::Process(const EventPtr& event) {
auto process_event = event_register_.at(static_cast<int>(event->Type()));
process_event(event);
}
// TODO: refactor the function
void
Scheduler::OnLoadCompleted(const EventPtr &event) {
Scheduler::OnLoadCompleted(const EventPtr& event) {
auto load_completed_event = std::static_pointer_cast<LoadCompletedEvent>(event);
if (auto resource = event->resource_.lock()) {
resource->WakeupExecutor();
@ -118,31 +114,29 @@ Scheduler::OnLoadCompleted(const EventPtr &event) {
Action::PushTaskToAllNeighbour(load_completed_event->task_table_item_->task, resource);
break;
}
default: {
break;
}
default: { break; }
}
}
}
void
Scheduler::OnStartUp(const EventPtr &event) {
Scheduler::OnStartUp(const EventPtr& event) {
if (auto resource = event->resource_.lock()) {
resource->WakeupLoader();
}
}
void
Scheduler::OnFinishTask(const EventPtr &event) {
Scheduler::OnFinishTask(const EventPtr& event) {
}
void
Scheduler::OnTaskTableUpdated(const EventPtr &event) {
Scheduler::OnTaskTableUpdated(const EventPtr& event) {
if (auto resource = event->resource_.lock()) {
resource->WakeupLoader();
}
}
} // namespace scheduler
} // namespace milvus
} // namespace zilliz
} // namespace scheduler
} // namespace milvus
} // namespace zilliz

28
cpp/src/scheduler/Scheduler.h
View File

@ -18,14 +18,14 @@
#pragma once
#include <memory>
#include <string>
#include <mutex>
#include <thread>
#include <queue>
#include <string>
#include <thread>
#include <unordered_map>
#include "resource/Resource.h"
#include "ResourceMgr.h"
#include "resource/Resource.h"
#include "utils/Log.h"
namespace zilliz {
@ -37,8 +37,8 @@ class Scheduler {
public:
explicit Scheduler(ResourceMgrWPtr res_mgr);
Scheduler(const Scheduler &) = delete;
Scheduler(Scheduler &&) = delete;
Scheduler(const Scheduler&) = delete;
Scheduler(Scheduler&&) = delete;
/*
* Start worker thread;
@ -56,7 +56,7 @@ class Scheduler {
* Post event to scheduler event queue;
*/
void
PostEvent(const EventPtr &event);
PostEvent(const EventPtr& event);
/*
* Dump as string;
@ -74,7 +74,7 @@ class Scheduler {
* Pull task from neighbours;
*/
void
OnStartUp(const EventPtr &event);
OnStartUp(const EventPtr& event);
/*
* Process finish task events;
@ -83,7 +83,7 @@ class Scheduler {
* Pull task from neighbours;
*/
void
OnFinishTask(const EventPtr &event);
OnFinishTask(const EventPtr& event);
/*
* Process copy completed events;
@ -93,7 +93,7 @@ class Scheduler {
* Pull task from neighbours;
*/
void
OnLoadCompleted(const EventPtr &event);
OnLoadCompleted(const EventPtr& event);
/*
* Process task table updated events, which happened on task_table->put;
@ -102,14 +102,14 @@ class Scheduler {
* Push task to neighbours;
*/
void
OnTaskTableUpdated(const EventPtr &event);
OnTaskTableUpdated(const EventPtr& event);
private:
/*
* Dispatch event to event handler;
*/
void
Process(const EventPtr &event);
Process(const EventPtr& event);
/*
* Called by worker_thread_;
@ -131,6 +131,6 @@ class Scheduler {
using SchedulerPtr = std::shared_ptr<Scheduler>;
} // namespace scheduler
} // namespace milvus
} // namespace zilliz
} // namespace scheduler
} // namespace milvus
} // namespace zilliz

14
cpp/src/scheduler/TaskCreator.cpp
View File

@ -24,7 +24,7 @@ namespace milvus {
namespace scheduler {
std::vector<TaskPtr>
TaskCreator::Create(const JobPtr &job) {
TaskCreator::Create(const JobPtr& job) {
switch (job->type()) {
case JobType::SEARCH: {
return Create(std::static_pointer_cast<SearchJob>(job));
@ -40,9 +40,9 @@ TaskCreator::Create(const JobPtr &job) {
}
std::vector<TaskPtr>
TaskCreator::Create(const SearchJobPtr &job) {
TaskCreator::Create(const SearchJobPtr& job) {
std::vector<TaskPtr> tasks;
for (auto &index_file : job->index_files()) {
for (auto& index_file : job->index_files()) {
auto task = std::make_shared<XSearchTask>(index_file.second);
task->label() = std::make_shared<DefaultLabel>();
task->job_ = job;
@ -53,7 +53,7 @@ TaskCreator::Create(const SearchJobPtr &job) {
}
std::vector<TaskPtr>
TaskCreator::Create(const DeleteJobPtr &job) {
TaskCreator::Create(const DeleteJobPtr& job) {
std::vector<TaskPtr> tasks;
auto task = std::make_shared<XDeleteTask>(job);
task->label() = std::make_shared<BroadcastLabel>();
@ -63,6 +63,6 @@ TaskCreator::Create(const DeleteJobPtr &job) {
return tasks;
}
} // namespace scheduler
} // namespace milvus
} // namespace zilliz
} // namespace scheduler
} // namespace milvus
} // namespace zilliz

34
cpp/src/scheduler/TaskCreator.h
View File

@ -16,23 +16,23 @@
// under the License.
#pragma once
#include <string>
#include <vector>
#include <list>
#include <queue>
#include <deque>
#include <unordered_map>
#include <thread>
#include <mutex>
#include <condition_variable>
#include <deque>
#include <list>
#include <memory>
#include <mutex>
#include <queue>
#include <string>
#include <thread>
#include <unordered_map>
#include <vector>
#include "job/DeleteJob.h"
#include "job/Job.h"
#include "job/SearchJob.h"
#include "job/DeleteJob.h"
#include "task/Task.h"
#include "task/SearchTask.h"
#include "task/DeleteTask.h"
#include "task/SearchTask.h"
#include "task/Task.h"
namespace zilliz {
namespace milvus {
@ -41,16 +41,16 @@ namespace scheduler {
class TaskCreator {
public:
static std::vector<TaskPtr>
Create(const JobPtr &job);
Create(const JobPtr& job);
public:
static std::vector<TaskPtr>
Create(const SearchJobPtr &job);
Create(const SearchJobPtr& job);
static std::vector<TaskPtr>
Create(const DeleteJobPtr &job);
Create(const DeleteJobPtr& job);
};
} // namespace scheduler
} // namespace milvus
} // namespace zilliz
} // namespace scheduler
} // namespace milvus
} // namespace zilliz

53
cpp/src/scheduler/TaskTable.cpp
View File

@ -15,14 +15,13 @@
// specific language governing permissions and limitations
// under the License.
#include "scheduler/TaskTable.h"
#include "event/TaskTableUpdatedEvent.h"
#include "Utils.h"
#include "event/TaskTableUpdatedEvent.h"
#include <vector>
#include <sstream>
#include <ctime>
#include <sstream>
#include <vector>
namespace zilliz {
namespace milvus {
@ -31,20 +30,29 @@ namespace scheduler {
std::string
ToString(TaskTableItemState state) {
switch (state) {
case TaskTableItemState::INVALID: return "INVALID";
case TaskTableItemState::START: return "START";
case TaskTableItemState::LOADING: return "LOADING";
case TaskTableItemState::LOADED: return "LOADED";
case TaskTableItemState::EXECUTING: return "EXECUTING";
case TaskTableItemState::EXECUTED: return "EXECUTED";
case TaskTableItemState::MOVING: return "MOVING";
case TaskTableItemState::MOVED: return "MOVED";
default: return "";
case TaskTableItemState::INVALID:
return "INVALID";
case TaskTableItemState::START:
return "START";
case TaskTableItemState::LOADING:
return "LOADING";
case TaskTableItemState::LOADED:
return "LOADED";
case TaskTableItemState::EXECUTING:
return "EXECUTING";
case TaskTableItemState::EXECUTED:
return "EXECUTED";
case TaskTableItemState::MOVING:
return "MOVING";
case TaskTableItemState::MOVED:
return "MOVED";
default:
return "";
}
}
std::string
ToString(const TaskTimestamp &timestamp) {
ToString(const TaskTimestamp& timestamp) {
std::stringstream ss;
ss << "<start=" << timestamp.start;
ss << ", load=" << timestamp.load;
@ -195,9 +203,9 @@ TaskTable::Put(TaskPtr task) {
}
void
TaskTable::Put(std::vector<TaskPtr> &tasks) {
TaskTable::Put(std::vector<TaskPtr>& tasks) {
std::lock_guard<std::mutex> lock(id_mutex_);
for (auto &task : tasks) {
for (auto& task : tasks) {
auto item = std::make_shared<TaskTableItem>();
item->id = id_++;
item->task = std::move(task);
@ -215,8 +223,8 @@ TaskTable::Get(uint64_t index) {
return table_[index];
}
//void
//TaskTable::Clear() {
// void
// TaskTable::Clear() {
//// find first task is NOT (done or moved), erase from begin to it;
//// auto iterator = table_.begin();
//// while (iterator->state == TaskTableItemState::EXECUTED or
@ -225,16 +233,15 @@ TaskTable::Get(uint64_t index) {
//// table_.erase(table_.begin(), iterator);
//}
std::string
TaskTable::Dump() {
std::stringstream ss;
for (auto &item : table_) {
for (auto& item : table_) {
ss << item->Dump() << std::endl;
}
return ss.str();
}
} // namespace scheduler
} // namespace milvus
} // namespace zilliz
} // namespace scheduler
} // namespace milvus
} // namespace zilliz

61
cpp/src/scheduler/TaskTable.h
View File

@ -17,16 +17,16 @@
#pragma once
#include <vector>
#include <deque>
#include <mutex>
#include <memory>
#include <utility>
#include <string>
#include <functional>
#include <memory>
#include <mutex>
#include <string>
#include <utility>
#include <vector>
#include "task/SearchTask.h"
#include "event/Event.h"
#include "task/SearchTask.h"
namespace zilliz {
namespace milvus {
@ -34,13 +34,13 @@ namespace scheduler {
enum class TaskTableItemState {
INVALID,
START, // idle
LOADING, // loading data from other resource
LOADED, // ready to exec or move
EXECUTING, // executing, locking util executed or failed
EXECUTED, // executed, termination state
MOVING, // moving to another resource, locking util executed or failed
MOVED, // moved, termination state
START, // idle
LOADING, // loading data from other resource
LOADED, // ready to exec or move
EXECUTING, // executing, locking util executed or failed
EXECUTED, // executed, termination state
MOVING, // moving to another resource, locking util executed or failed
MOVED, // moved, termination state
};
struct TaskTimestamp {
@ -58,12 +58,12 @@ struct TaskTableItem {
TaskTableItem() : id(0), task(nullptr), state(TaskTableItemState::INVALID), mutex() {
}
TaskTableItem(const TaskTableItem &src) = delete;
TaskTableItem(TaskTableItem &&) = delete;
TaskTableItem(const TaskTableItem& src) = delete;
TaskTableItem(TaskTableItem&&) = delete;
uint64_t id; // auto increment from 0;
TaskPtr task; // the task;
TaskTableItemState state; // the state;
uint64_t id; // auto increment from 0;
TaskPtr task; // the task;
TaskTableItemState state; // the state;
std::mutex mutex;
TaskTimestamp timestamp;
@ -98,8 +98,8 @@ class TaskTable {
public:
TaskTable() = default;
TaskTable(const TaskTable &) = delete;
TaskTable(TaskTable &&) = delete;
TaskTable(const TaskTable&) = delete;
TaskTable(TaskTable&&) = delete;
inline void
RegisterSubscriber(std::function<void(void)> subscriber) {
@ -117,7 +117,7 @@ class TaskTable {
* Called by DBImpl;
*/
void
Put(std::vector<TaskPtr> &tasks);
Put(std::vector<TaskPtr>& tasks);
/*
* Return task table item reference;
@ -130,8 +130,8 @@ class TaskTable {
* Remove sequence task which is DONE or MOVED from front;
* Called by ?
*/
// void
// Clear();
// void
// Clear();
/*
* Return true if task table empty, otherwise false;
@ -150,16 +150,17 @@ class TaskTable {
}
public:
TaskTableItemPtr &
operator[](uint64_t index) {
TaskTableItemPtr& operator[](uint64_t index) {
return table_[index];
}
std::deque<TaskTableItemPtr>::iterator begin() {
std::deque<TaskTableItemPtr>::iterator
begin() {
return table_.begin();
}
std::deque<TaskTableItemPtr>::iterator end() {
std::deque<TaskTableItemPtr>::iterator
end() {
return table_.end();
}
@ -254,6 +255,6 @@ class TaskTable {
uint64_t last_finish_ = -1;
};
} // namespace scheduler
} // namespace milvus
} // namespace zilliz
} // namespace scheduler
} // namespace milvus
} // namespace zilliz

9
cpp/src/scheduler/Utils.cpp
View File

@ -15,11 +15,10 @@
// specific language governing permissions and limitations
// under the License.
#include "scheduler/Utils.h"
#include <chrono>
#include <cuda_runtime.h>
#include <chrono>
namespace zilliz {
namespace milvus {
@ -40,6 +39,6 @@ get_num_gpu() {
return n_devices;
}
} // namespace scheduler
} // namespace milvus
} // namespace zilliz
} // namespace scheduler
} // namespace milvus
} // namespace zilliz

7
cpp/src/scheduler/Utils.h
View File

@ -15,7 +15,6 @@
// specific language governing permissions and limitations
// under the License.
#include <cstdint>
namespace zilliz {
@ -28,6 +27,6 @@ get_current_timestamp();
uint64_t
get_num_gpu();
} // namespace scheduler
} // namespace milvus
} // namespace zilliz
} // namespace scheduler
} // namespace milvus
} // namespace zilliz

17
cpp/src/scheduler/action/Action.h
View File

@ -17,8 +17,8 @@
#pragma once
#include "scheduler/resource/Resource.h"
#include "scheduler/ResourceMgr.h"
#include "scheduler/resource/Resource.h"
#include <memory>
@ -29,23 +29,22 @@ namespace scheduler {
class Action {
public:
static void
PushTaskToNeighbourRandomly(const TaskPtr &task, const ResourcePtr &self);
PushTaskToNeighbourRandomly(const TaskPtr& task, const ResourcePtr& self);
static void
PushTaskToAllNeighbour(const TaskPtr &task, const ResourcePtr &self);
PushTaskToAllNeighbour(const TaskPtr& task, const ResourcePtr& self);
static void
PushTaskToResource(const TaskPtr &task, const ResourcePtr &dest);
PushTaskToResource(const TaskPtr& task, const ResourcePtr& dest);
static void
DefaultLabelTaskScheduler(ResourceMgrWPtr res_mgr, ResourcePtr resource, std::shared_ptr<LoadCompletedEvent> event);
static void
SpecifiedResourceLabelTaskScheduler(ResourceMgrWPtr res_mgr,
ResourcePtr resource,
SpecifiedResourceLabelTaskScheduler(ResourceMgrWPtr res_mgr, ResourcePtr resource,
std::shared_ptr<LoadCompletedEvent> event);
};
} // namespace scheduler
} // namespace milvus
} // namespace zilliz
} // namespace scheduler
} // namespace milvus
} // namespace zilliz

48
cpp/src/scheduler/action/PushTaskToNeighbour.cpp
View File

@ -15,26 +15,25 @@
// specific language governing permissions and limitations
// under the License.
#include <list>
#include <random>
#include "../Algorithm.h"
#include "src/cache/GpuCacheMgr.h"
#include "Action.h"
#include "src/cache/GpuCacheMgr.h"
namespace zilliz {
namespace milvus {
namespace scheduler {
std::vector<ResourcePtr>
get_neighbours(const ResourcePtr &self) {
get_neighbours(const ResourcePtr& self) {
std::vector<ResourcePtr> neighbours;
for (auto &neighbour_node : self->GetNeighbours()) {
for (auto& neighbour_node : self->GetNeighbours()) {
auto node = neighbour_node.neighbour_node.lock();
if (not node) continue;
auto resource = std::static_pointer_cast<Resource>(node);
// if (not resource->HasExecutor()) continue;
// if (not resource->HasExecutor()) continue;
neighbours.emplace_back(resource);
}
@ -42,14 +41,14 @@ get_neighbours(const ResourcePtr &self) {
}
std::vector<std::pair<ResourcePtr, Connection>>
get_neighbours_with_connetion(const ResourcePtr &self) {
get_neighbours_with_connetion(const ResourcePtr& self) {
std::vector<std::pair<ResourcePtr, Connection>> neighbours;
for (auto &neighbour_node : self->GetNeighbours()) {
for (auto& neighbour_node : self->GetNeighbours()) {
auto node = neighbour_node.neighbour_node.lock();
if (not node) continue;
auto resource = std::static_pointer_cast<Resource>(node);
// if (not resource->HasExecutor()) continue;
// if (not resource->HasExecutor()) continue;
Connection conn = neighbour_node.connection;
neighbours.emplace_back(std::make_pair(resource, conn));
}
@ -57,13 +56,12 @@ get_neighbours_with_connetion(const ResourcePtr &self) {
}
void
Action::PushTaskToNeighbourRandomly(const TaskPtr &task,
const ResourcePtr &self) {
Action::PushTaskToNeighbourRandomly(const TaskPtr& task, const ResourcePtr& self) {
auto neighbours = get_neighbours_with_connetion(self);
if (not neighbours.empty()) {
std::vector<uint64_t> speeds;
uint64_t total_speed = 0;
for (auto &neighbour : neighbours) {
for (auto& neighbour : neighbours) {
uint64_t speed = neighbour.second.speed();
speeds.emplace_back(speed);
total_speed += speed;
@ -83,33 +81,32 @@ Action::PushTaskToNeighbourRandomly(const TaskPtr &task,
}
} else {
//TODO: process
// TODO: process
}
}
void
Action::PushTaskToAllNeighbour(const TaskPtr &task, const ResourcePtr &self) {
Action::PushTaskToAllNeighbour(const TaskPtr& task, const ResourcePtr& self) {
auto neighbours = get_neighbours(self);
for (auto &neighbour : neighbours) {
for (auto& neighbour : neighbours) {
neighbour->task_table().Put(task);
}
}
void
Action::PushTaskToResource(const TaskPtr &task, const ResourcePtr &dest) {
Action::PushTaskToResource(const TaskPtr& task, const ResourcePtr& dest) {
dest->task_table().Put(task);
}
void
Action::DefaultLabelTaskScheduler(ResourceMgrWPtr res_mgr,
ResourcePtr resource,
Action::DefaultLabelTaskScheduler(ResourceMgrWPtr res_mgr, ResourcePtr resource,
std::shared_ptr<LoadCompletedEvent> event) {
if (not resource->HasExecutor() && event->task_table_item_->Move()) {
auto task = event->task_table_item_->task;
auto search_task = std::static_pointer_cast<XSearchTask>(task);
bool moved = false;
//to support test task, REFACTOR
// to support test task, REFACTOR
if (auto index_engine = search_task->index_engine_) {
auto location = index_engine->GetLocation();
@ -131,8 +128,7 @@ Action::DefaultLabelTaskScheduler(ResourceMgrWPtr res_mgr,
}
void
Action::SpecifiedResourceLabelTaskScheduler(ResourceMgrWPtr res_mgr,
ResourcePtr resource,
Action::SpecifiedResourceLabelTaskScheduler(ResourceMgrWPtr res_mgr, ResourcePtr resource,
std::shared_ptr<LoadCompletedEvent> event) {
auto task = event->task_table_item_->task;
if (resource->type() == ResourceType::DISK) {
@ -140,7 +136,7 @@ Action::SpecifiedResourceLabelTaskScheduler(ResourceMgrWPtr res_mgr,
auto compute_resources = res_mgr.lock()->GetComputeResources();
std::vector<std::vector<std::string>> paths;
std::vector<uint64_t> transport_costs;
for (auto &res : compute_resources) {
for (auto& res : compute_resources) {
std::vector<std::string> path;
uint64_t transport_cost = ShortestPath(resource, res, res_mgr.lock(), path);
transport_costs.push_back(transport_cost);
@ -155,8 +151,8 @@ Action::SpecifiedResourceLabelTaskScheduler(ResourceMgrWPtr res_mgr,
min_cost_idx = i;
break;
}
uint64_t cost = compute_resources[i]->TaskAvgCost() * compute_resources[i]->NumOfTaskToExec()
+ transport_costs[i];
uint64_t cost =
compute_resources[i]->TaskAvgCost() * compute_resources[i]->NumOfTaskToExec() + transport_costs[i];
if (min_cost > cost) {
min_cost = cost;
min_cost_idx = i;
@ -178,6 +174,6 @@ Action::SpecifiedResourceLabelTaskScheduler(ResourceMgrWPtr res_mgr,
}
}
} // namespace scheduler
} // namespace milvus
} // namespace zilliz
} // namespace scheduler
} // namespace milvus
} // namespace zilliz

23
cpp/src/scheduler/event/Event.h
View File

@ -18,28 +18,20 @@
#pragma once
#include <memory>
#include <utility>
#include <string>
#include <utility>
namespace zilliz {
namespace milvus {
namespace scheduler {
enum class EventType {
START_UP,
LOAD_COMPLETED,
FINISH_TASK,
TASK_TABLE_UPDATED
};
enum class EventType { START_UP, LOAD_COMPLETED, FINISH_TASK, TASK_TABLE_UPDATED };
class Resource;
class Event {
public:
explicit
Event(EventType type, std::weak_ptr<Resource> resource)
: type_(type),
resource_(std::move(resource)) {
explicit Event(EventType type, std::weak_ptr<Resource> resource) : type_(type), resource_(std::move(resource)) {
}
inline EventType
@ -50,7 +42,8 @@ class Event {
virtual std::string
Dump() const = 0;
friend std::ostream &operator<<(std::ostream &out, const Event &event);
friend std::ostream&
operator<<(std::ostream& out, const Event& event);
public:
EventType type_;
@ -59,6 +52,6 @@ class Event {
using EventPtr = std::shared_ptr<Event>;
} // namespace scheduler
} // namespace milvus
} // namespace zilliz
} // namespace scheduler
} // namespace milvus
} // namespace zilliz

31
cpp/src/scheduler/event/EventDump.cpp
View File

@ -15,47 +15,46 @@
// specific language governing permissions and limitations
// under the License.
#include "Event.h"
#include "StartUpEvent.h"
#include "LoadCompletedEvent.h"
#include "FinishTaskEvent.h"
#include "LoadCompletedEvent.h"
#include "StartUpEvent.h"
#include "TaskTableUpdatedEvent.h"
namespace zilliz {
namespace milvus {
namespace scheduler {
std::ostream &
operator<<(std::ostream &out, const Event &event) {
std::ostream&
operator<<(std::ostream& out, const Event& event) {
out << event.Dump();
return out;
}
std::ostream &
operator<<(std::ostream &out, const StartUpEvent &event) {
std::ostream&
operator<<(std::ostream& out, const StartUpEvent& event) {
out << event.Dump();
return out;
}
std::ostream &
operator<<(std::ostream &out, const LoadCompletedEvent &event) {
std::ostream&
operator<<(std::ostream& out, const LoadCompletedEvent& event) {
out << event.Dump();
return out;
}
std::ostream &
operator<<(std::ostream &out, const FinishTaskEvent &event) {
std::ostream&
operator<<(std::ostream& out, const FinishTaskEvent& event) {
out << event.Dump();
return out;
}
std::ostream &
operator<<(std::ostream &out, const TaskTableUpdatedEvent &event) {
std::ostream&
operator<<(std::ostream& out, const TaskTableUpdatedEvent& event) {
out << event.Dump();
return out;
}
} // namespace scheduler
} // namespace milvus
} // namespace zilliz
} // namespace scheduler
} // namespace milvus
} // namespace zilliz

15
cpp/src/scheduler/event/FinishTaskEvent.h
View File

@ -17,11 +17,12 @@
#pragma once
#include "scheduler/TaskTable.h"
#include "scheduler/event/Event.h"
#include <memory>
#include <utility>
#include <string>
#include <utility>
namespace zilliz {
namespace milvus {
@ -30,8 +31,7 @@ namespace scheduler {
class FinishTaskEvent : public Event {
public:
FinishTaskEvent(std::weak_ptr<Resource> resource, TaskTableItemPtr task_table_item)
: Event(EventType::FINISH_TASK, std::move(resource)),
task_table_item_(std::move(task_table_item)) {
: Event(EventType::FINISH_TASK, std::move(resource)), task_table_item_(std::move(task_table_item)) {
}
inline std::string
@ -39,12 +39,13 @@ class FinishTaskEvent : public Event {
return "<FinishTaskEvent>";
}
friend std::ostream &operator<<(std::ostream &out, const FinishTaskEvent &event);
friend std::ostream&
operator<<(std::ostream& out, const FinishTaskEvent& event);
public:
TaskTableItemPtr task_table_item_;
};
} // namespace scheduler
} // namespace milvus
} // namespace zilliz
} // namespace scheduler
} // namespace milvus
} // namespace zilliz

16
cpp/src/scheduler/event/LoadCompletedEvent.h
View File

@ -17,12 +17,12 @@
#pragma once
#include "scheduler/event/Event.h"
#include "scheduler/TaskTable.h"
#include "scheduler/event/Event.h"
#include <memory>
#include <utility>
#include <string>
#include <utility>
namespace zilliz {
namespace milvus {
@ -31,8 +31,7 @@ namespace scheduler {
class LoadCompletedEvent : public Event {
public:
LoadCompletedEvent(std::weak_ptr<Resource> resource, TaskTableItemPtr task_table_item)
: Event(EventType::LOAD_COMPLETED, std::move(resource)),
task_table_item_(std::move(task_table_item)) {
: Event(EventType::LOAD_COMPLETED, std::move(resource)), task_table_item_(std::move(task_table_item)) {
}
inline std::string
@ -40,12 +39,13 @@ class LoadCompletedEvent : public Event {
return "<LoadCompletedEvent>";
}
friend std::ostream &operator<<(std::ostream &out, const LoadCompletedEvent &event);
friend std::ostream&
operator<<(std::ostream& out, const LoadCompletedEvent& event);
public:
TaskTableItemPtr task_table_item_;
};
} // namespace scheduler
} // namespace milvus
} // namespace zilliz
} // namespace scheduler
} // namespace milvus
} // namespace zilliz

14
cpp/src/scheduler/event/StartUpEvent.h
View File

@ -20,8 +20,8 @@
#include "scheduler/event/Event.h"
#include <memory>
#include <utility>
#include <string>
#include <utility>
namespace zilliz {
namespace milvus {
@ -29,8 +29,7 @@ namespace scheduler {
class StartUpEvent : public Event {
public:
explicit StartUpEvent(std::weak_ptr<Resource> resource)
: Event(EventType::START_UP, std::move(resource)) {
explicit StartUpEvent(std::weak_ptr<Resource> resource) : Event(EventType::START_UP, std::move(resource)) {
}
inline std::string
@ -38,9 +37,10 @@ class StartUpEvent : public Event {
return "<StartUpEvent>";
}
friend std::ostream &operator<<(std::ostream &out, const StartUpEvent &event);
friend std::ostream&
operator<<(std::ostream& out, const StartUpEvent& event);
};
} // namespace scheduler
} // namespace milvus
} // namespace zilliz
} // namespace scheduler
} // namespace milvus
} // namespace zilliz

11
cpp/src/scheduler/event/TaskTableUpdatedEvent.h
View File

@ -20,8 +20,8 @@
#include "Event.h"
#include <memory>
#include <utility>
#include <string>
#include <utility>
namespace zilliz {
namespace milvus {
@ -38,9 +38,10 @@ class TaskTableUpdatedEvent : public Event {
return "<TaskTableUpdatedEvent>";
}
friend std::ostream &operator<<(std::ostream &out, const TaskTableUpdatedEvent &event);
friend std::ostream&
operator<<(std::ostream& out, const TaskTableUpdatedEvent& event);
};
} // namespace scheduler
} // namespace milvus
} // namespace zilliz
} // namespace scheduler
} // namespace milvus
} // namespace zilliz

15
cpp/src/scheduler/job/DeleteJob.cpp
View File

@ -23,10 +23,7 @@ namespace zilliz {
namespace milvus {
namespace scheduler {
DeleteJob::DeleteJob(JobId id,
std::string table_id,
engine::meta::MetaPtr meta_ptr,
uint64_t num_resource)
DeleteJob::DeleteJob(JobId id, std::string table_id, engine::meta::MetaPtr meta_ptr, uint64_t num_resource)
: Job(id, JobType::DELETE),
table_id_(std::move(table_id)),
meta_ptr_(std::move(meta_ptr)),
@ -36,9 +33,7 @@ DeleteJob::DeleteJob(JobId id,
void
DeleteJob::WaitAndDelete() {
std::unique_lock<std::mutex> lock(mutex_);
cv_.wait(lock, [&] {
return done_resource == num_resource_;
});
cv_.wait(lock, [&] { return done_resource == num_resource_; });
meta_ptr_->DeleteTableFiles(table_id_);
}
@ -51,6 +46,6 @@ DeleteJob::ResourceDone() {
cv_.notify_one();
}
} // namespace scheduler
} // namespace milvus
} // namespace zilliz
} // namespace scheduler
} // namespace milvus
} // namespace zilliz

27
cpp/src/scheduler/job/DeleteJob.h
View File

@ -16,16 +16,16 @@
// under the License.
#pragma once
#include <string>
#include <vector>
#include <list>
#include <queue>
#include <deque>
#include <unordered_map>
#include <thread>
#include <mutex>
#include <condition_variable>
#include <deque>
#include <list>
#include <memory>
#include <mutex>
#include <queue>
#include <string>
#include <thread>
#include <unordered_map>
#include <vector>
#include "Job.h"
#include "db/meta/Meta.h"
@ -36,10 +36,7 @@ namespace scheduler {
class DeleteJob : public Job {
public:
DeleteJob(JobId id,
std::string table_id,
engine::meta::MetaPtr meta_ptr,
uint64_t num_resource);
DeleteJob(JobId id, std::string table_id, engine::meta::MetaPtr meta_ptr, uint64_t num_resource);
public:
void
@ -71,6 +68,6 @@ class DeleteJob : public Job {
using DeleteJobPtr = std::shared_ptr<DeleteJob>;
} // namespace scheduler
} // namespace milvus
} // namespace zilliz
} // namespace scheduler
} // namespace milvus
} // namespace zilliz

22
cpp/src/scheduler/job/Job.h
View File

@ -16,16 +16,16 @@
// under the License.
#pragma once
#include <string>
#include <vector>
#include <list>
#include <queue>
#include <deque>
#include <unordered_map>
#include <thread>
#include <mutex>
#include <condition_variable>
#include <deque>
#include <list>
#include <memory>
#include <mutex>
#include <queue>
#include <string>
#include <thread>
#include <unordered_map>
#include <vector>
namespace zilliz {
namespace milvus {
@ -64,6 +64,6 @@ class Job {
using JobPtr = std::shared_ptr<Job>;
using JobWPtr = std::weak_ptr<Job>;
} // namespace scheduler
} // namespace milvus
} // namespace zilliz
} // namespace scheduler
} // namespace milvus
} // namespace zilliz

29
cpp/src/scheduler/job/SearchJob.cpp
View File

@ -22,20 +22,13 @@ namespace zilliz {
namespace milvus {
namespace scheduler {
SearchJob::SearchJob(zilliz::milvus::scheduler::JobId id,
uint64_t topk,
uint64_t nq,
uint64_t nprobe,
const float *vectors)
: Job(id, JobType::SEARCH),
topk_(topk),
nq_(nq),
nprobe_(nprobe),
vectors_(vectors) {
SearchJob::SearchJob(zilliz::milvus::scheduler::JobId id, uint64_t topk, uint64_t nq, uint64_t nprobe,
const float* vectors)
: Job(id, JobType::SEARCH), topk_(topk), nq_(nq), nprobe_(nprobe), vectors_(vectors) {
}
bool
SearchJob::AddIndexFile(const TableFileSchemaPtr &index_file) {
SearchJob::AddIndexFile(const TableFileSchemaPtr& index_file) {
std::unique_lock<std::mutex> lock(mutex_);
if (index_file == nullptr || index_files_.find(index_file->id_) != index_files_.end()) {
return false;
@ -50,9 +43,7 @@ SearchJob::AddIndexFile(const TableFileSchemaPtr &index_file) {
void
SearchJob::WaitResult() {
std::unique_lock<std::mutex> lock(mutex_);
cv_.wait(lock, [this] {
return index_files_.empty();
});
cv_.wait(lock, [this] { return index_files_.empty(); });
SERVER_LOG_DEBUG << "SearchJob " << id() << " all done";
}
@ -64,16 +55,16 @@ SearchJob::SearchDone(size_t index_id) {
SERVER_LOG_DEBUG << "SearchJob " << id() << " finish index file: " << index_id;
}
ResultSet &
ResultSet&
SearchJob::GetResult() {
return result_;
}
Status &
Status&
SearchJob::GetStatus() {
return status_;
}
} // namespace scheduler
} // namespace milvus
} // namespace zilliz
} // namespace scheduler
} // namespace milvus
} // namespace zilliz

36
cpp/src/scheduler/job/SearchJob.h
View File

@ -16,17 +16,17 @@
// under the License.
#pragma once
#include <string>
#include <vector>
#include <list>
#include <queue>
#include <deque>
#include <unordered_map>
#include <thread>
#include <mutex>
#include <condition_variable>
#include <deque>
#include <list>
#include <memory>
#include <mutex>
#include <queue>
#include <string>
#include <thread>
#include <unordered_map>
#include <utility>
#include <vector>
#include "Job.h"
#include "db/meta/MetaTypes.h"
@ -43,11 +43,11 @@ using ResultSet = std::vector<Id2DistanceMap>;
class SearchJob : public Job {
public:
SearchJob(JobId id, uint64_t topk, uint64_t nq, uint64_t nprobe, const float *vectors);
SearchJob(JobId id, uint64_t topk, uint64_t nq, uint64_t nprobe, const float* vectors);
public:
bool
AddIndexFile(const TableFileSchemaPtr &index_file);
AddIndexFile(const TableFileSchemaPtr& index_file);
void
WaitResult();
@ -55,10 +55,10 @@ class SearchJob : public Job {
void
SearchDone(size_t index_id);
ResultSet &
ResultSet&
GetResult();
Status &
Status&
GetStatus();
public:
@ -77,12 +77,12 @@ class SearchJob : public Job {
return nprobe_;
}
const float *
const float*
vectors() const {
return vectors_;
}
Id2IndexMap &
Id2IndexMap&
index_files() {
return index_files_;
}
@ -92,7 +92,7 @@ class SearchJob : public Job {
uint64_t nq_ = 0;
uint64_t nprobe_ = 0;
// TODO: smart pointer
const float *vectors_ = nullptr;
const float* vectors_ = nullptr;
Id2IndexMap index_files_;
// TODO: column-base better ?
@ -105,6 +105,6 @@ class SearchJob : public Job {
using SearchJobPtr = std::shared_ptr<SearchJob>;
} // namespace scheduler
} // namespace milvus
} // namespace zilliz
} // namespace scheduler
} // namespace milvus
} // namespace zilliz

13
cpp/src/scheduler/resource/Connection.h
View File

@ -17,8 +17,8 @@
#pragma once
#include <string>
#include <sstream>
#include <string>
#include <utility>
namespace zilliz {
@ -28,11 +28,10 @@ namespace scheduler {
class Connection {
public:
// TODO: update construct function, speed: double->uint64_t
Connection(std::string name, double speed)
: name_(std::move(name)), speed_(speed) {
Connection(std::string name, double speed) : name_(std::move(name)), speed_(speed) {
}
const std::string &
const std::string&
name() const {
return name_;
}
@ -60,6 +59,6 @@ class Connection {
uint64_t speed_;
};
} // namespace scheduler
} // namespace milvus
} // namespace zilliz
} // namespace scheduler
} // namespace milvus
} // namespace zilliz

11
cpp/src/scheduler/resource/CpuResource.cpp
View File

@ -15,7 +15,6 @@
// specific language governing permissions and limitations
// under the License.
#include "scheduler/resource/CpuResource.h"
#include <utility>
@ -24,8 +23,8 @@ namespace zilliz {
namespace milvus {
namespace scheduler {
std::ostream &
operator<<(std::ostream &out, const CpuResource &resource) {
std::ostream&
operator<<(std::ostream& out, const CpuResource& resource) {
out << resource.Dump();
return out;
}
@ -44,6 +43,6 @@ CpuResource::Process(TaskPtr task) {
task->Execute();
}
} // namespace scheduler
} // namespace milvus
} // namespace zilliz
} // namespace scheduler
} // namespace milvus
} // namespace zilliz

12
cpp/src/scheduler/resource/CpuResource.h
View File

@ -27,15 +27,15 @@ namespace scheduler {
class CpuResource : public Resource {
public:
explicit
CpuResource(std::string name, uint64_t device_id, bool enable_loader, bool enable_executor);
explicit CpuResource(std::string name, uint64_t device_id, bool enable_loader, bool enable_executor);
inline std::string
Dump() const override {
return "<CpuResource, name=" + name_ + ">";
}
friend std::ostream &operator<<(std::ostream &out, const CpuResource &resource);
friend std::ostream&
operator<<(std::ostream& out, const CpuResource& resource);
protected:
void
@ -45,6 +45,6 @@ class CpuResource : public Resource {
Process(TaskPtr task) override;
};
} // namespace scheduler
} // namespace milvus
} // namespace zilliz
} // namespace scheduler
} // namespace milvus
} // namespace zilliz

10
cpp/src/scheduler/resource/DiskResource.cpp
View File

@ -24,8 +24,8 @@ namespace zilliz {
namespace milvus {
namespace scheduler {
std::ostream &
operator<<(std::ostream &out, const DiskResource &resource) {
std::ostream&
operator<<(std::ostream& out, const DiskResource& resource) {
out << resource.Dump();
return out;
}
@ -42,6 +42,6 @@ void
DiskResource::Process(TaskPtr task) {
}
} // namespace scheduler
} // namespace milvus
} // namespace zilliz
} // namespace scheduler
} // namespace milvus
} // namespace zilliz

12
cpp/src/scheduler/resource/DiskResource.h
View File

@ -27,15 +27,15 @@ namespace scheduler {
class DiskResource : public Resource {
public:
explicit
DiskResource(std::string name, uint64_t device_id, bool enable_loader, bool enable_executor);
explicit DiskResource(std::string name, uint64_t device_id, bool enable_loader, bool enable_executor);
inline std::string
Dump() const override {
return "<DiskResource, name=" + name_ + ">";
}
friend std::ostream &operator<<(std::ostream &out, const DiskResource &resource);
friend std::ostream&
operator<<(std::ostream& out, const DiskResource& resource);
protected:
void
@ -45,6 +45,6 @@ class DiskResource : public Resource {
Process(TaskPtr task) override;
};
} // namespace scheduler
} // namespace milvus
} // namespace zilliz
} // namespace scheduler
} // namespace milvus
} // namespace zilliz

11
cpp/src/scheduler/resource/GpuResource.cpp
View File

@ -15,15 +15,14 @@
// specific language governing permissions and limitations
// under the License.
#include "scheduler/resource/GpuResource.h"
namespace zilliz {
namespace milvus {
namespace scheduler {
std::ostream &
operator<<(std::ostream &out, const GpuResource &resource) {
std::ostream&
operator<<(std::ostream& out, const GpuResource& resource) {
out << resource.Dump();
return out;
}
@ -42,6 +41,6 @@ GpuResource::Process(TaskPtr task) {
task->Execute();
}
} // namespace scheduler
} // namespace milvus
} // namespace zilliz
} // namespace scheduler
} // namespace milvus
} // namespace zilliz

Some files were not shown because too many files have changed in this diff Show More