mirror of https://github.com/milvus-io/milvus.git
groot
GitHub
parent
d6bf0c928f
commit
73c9ab43e0
|
|
@ -27,7 +27,6 @@ add_subdirectory( exceptions )
|
|||
add_subdirectory( utils )
|
||||
add_subdirectory( log )
|
||||
add_subdirectory( pb )
|
||||
add_subdirectory( storage )
|
||||
add_subdirectory( segcore )
|
||||
add_subdirectory( query )
|
||||
add_subdirectory( common )
|
||||
|
|
|
|||
|
|
@ -1,104 +0,0 @@
|
|||
// Copyright (C) 2019-2020 Zilliz. All rights reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software distributed under the License
|
||||
// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
|
||||
// or implied. See the License for the specific language governing permissions and limitations under the License.
|
||||
|
||||
#pragma once
|
||||
|
||||
#include "LRU.h"
|
||||
#include "utils/Log.h"
|
||||
|
||||
#include <atomic>
|
||||
#include <mutex>
|
||||
#include <set>
|
||||
#include <string>
|
||||
|
||||
namespace milvus {
|
||||
namespace cache {
|
||||
|
||||
template <typename ItemObj>
|
||||
class Cache {
|
||||
public:
|
||||
// mem_capacity, units:GB
|
||||
Cache(int64_t capacity_gb, int64_t cache_max_count, const std::string& header = "");
|
||||
~Cache() = default;
|
||||
|
||||
int64_t
|
||||
usage() const {
|
||||
return usage_;
|
||||
}
|
||||
|
||||
// unit: BYTE
|
||||
int64_t
|
||||
capacity() const {
|
||||
return capacity_;
|
||||
}
|
||||
|
||||
// unit: BYTE
|
||||
void
|
||||
set_capacity(int64_t capacity);
|
||||
|
||||
double
|
||||
freemem_percent() const {
|
||||
return freemem_percent_;
|
||||
}
|
||||
|
||||
void
|
||||
set_freemem_percent(double percent) {
|
||||
freemem_percent_ = percent;
|
||||
}
|
||||
|
||||
size_t
|
||||
size() const;
|
||||
|
||||
bool
|
||||
exists(const std::string& key);
|
||||
|
||||
ItemObj
|
||||
get(const std::string& key);
|
||||
|
||||
void
|
||||
insert(const std::string& key, const ItemObj& item);
|
||||
|
||||
void
|
||||
erase(const std::string& key);
|
||||
|
||||
bool
|
||||
reserve(const int64_t size);
|
||||
|
||||
void
|
||||
print();
|
||||
|
||||
void
|
||||
clear();
|
||||
|
||||
private:
|
||||
void
|
||||
insert_internal(const std::string& key, const ItemObj& item);
|
||||
|
||||
void
|
||||
erase_internal(const std::string& key);
|
||||
|
||||
void
|
||||
free_memory_internal(const int64_t target_size);
|
||||
|
||||
private:
|
||||
std::string header_;
|
||||
int64_t usage_;
|
||||
int64_t capacity_;
|
||||
double freemem_percent_;
|
||||
|
||||
LRU<std::string, ItemObj> lru_;
|
||||
mutable std::mutex mutex_;
|
||||
};
|
||||
|
||||
} // namespace cache
|
||||
} // namespace milvus
|
||||
|
||||
#include "cache/Cache.inl"
|
||||
|
|
@ -1,191 +0,0 @@
|
|||
// Copyright (C) 2019-2020 Zilliz. All rights reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software distributed under the License
|
||||
// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
|
||||
// or implied. See the License for the specific language governing permissions and limitations under the License.
|
||||
|
||||
namespace milvus {
|
||||
namespace cache {
|
||||
|
||||
constexpr double DEFAULT_THRESHOLD_PERCENT = 0.7;
|
||||
|
||||
template <typename ItemObj>
|
||||
Cache<ItemObj>::Cache(int64_t capacity, int64_t cache_max_count, const std::string& header)
|
||||
: header_(header),
|
||||
usage_(0),
|
||||
capacity_(capacity),
|
||||
freemem_percent_(DEFAULT_THRESHOLD_PERCENT),
|
||||
lru_(cache_max_count) {
|
||||
}
|
||||
|
||||
template <typename ItemObj>
|
||||
void
|
||||
Cache<ItemObj>::set_capacity(int64_t capacity) {
|
||||
std::lock_guard<std::mutex> lock(mutex_);
|
||||
if (capacity > 0) {
|
||||
capacity_ = capacity;
|
||||
free_memory_internal(capacity);
|
||||
}
|
||||
}
|
||||
|
||||
template <typename ItemObj>
|
||||
size_t
|
||||
Cache<ItemObj>::size() const {
|
||||
std::lock_guard<std::mutex> lock(mutex_);
|
||||
return lru_.size();
|
||||
}
|
||||
|
||||
template <typename ItemObj>
|
||||
bool
|
||||
Cache<ItemObj>::exists(const std::string& key) {
|
||||
std::lock_guard<std::mutex> lock(mutex_);
|
||||
return lru_.exists(key);
|
||||
}
|
||||
|
||||
template <typename ItemObj>
|
||||
ItemObj
|
||||
Cache<ItemObj>::get(const std::string& key) {
|
||||
std::lock_guard<std::mutex> lock(mutex_);
|
||||
if (!lru_.exists(key)) {
|
||||
return nullptr;
|
||||
}
|
||||
return lru_.get(key);
|
||||
}
|
||||
|
||||
template <typename ItemObj>
|
||||
void
|
||||
Cache<ItemObj>::insert(const std::string& key, const ItemObj& item) {
|
||||
std::lock_guard<std::mutex> lock(mutex_);
|
||||
insert_internal(key, item);
|
||||
}
|
||||
|
||||
template <typename ItemObj>
|
||||
void
|
||||
Cache<ItemObj>::erase(const std::string& key) {
|
||||
std::lock_guard<std::mutex> lock(mutex_);
|
||||
erase_internal(key);
|
||||
}
|
||||
|
||||
template <typename ItemObj>
|
||||
bool
|
||||
Cache<ItemObj>::reserve(const int64_t item_size) {
|
||||
std::lock_guard<std::mutex> lock(mutex_);
|
||||
if (item_size > capacity_) {
|
||||
LOG_SERVER_ERROR_ << header_ << " item size " << (item_size >> 20) << "MB too big to insert into cache capacity"
|
||||
<< (capacity_ >> 20) << "MB";
|
||||
return false;
|
||||
}
|
||||
if (item_size > capacity_ - usage_) {
|
||||
free_memory_internal(capacity_ - item_size);
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
template <typename ItemObj>
|
||||
void
|
||||
Cache<ItemObj>::clear() {
|
||||
std::lock_guard<std::mutex> lock(mutex_);
|
||||
lru_.clear();
|
||||
usage_ = 0;
|
||||
LOG_SERVER_DEBUG_ << header_ << " Clear cache !";
|
||||
}
|
||||
|
||||
|
||||
template <typename ItemObj>
|
||||
void
|
||||
Cache<ItemObj>::print() {
|
||||
std::lock_guard<std::mutex> lock(mutex_);
|
||||
size_t cache_count = lru_.size();
|
||||
// for (auto it = lru_.begin(); it != lru_.end(); ++it) {
|
||||
// LOG_SERVER_DEBUG_ << it->first;
|
||||
// }
|
||||
LOG_SERVER_DEBUG_ << header_ << " [item count]: " << cache_count << ", [usage] " << (usage_ >> 20)
|
||||
<< "MB, [capacity] " << (capacity_ >> 20) << "MB";
|
||||
}
|
||||
|
||||
template <typename ItemObj>
|
||||
void
|
||||
Cache<ItemObj>::insert_internal(const std::string& key, const ItemObj& item) {
|
||||
if (item == nullptr) {
|
||||
return;
|
||||
}
|
||||
|
||||
size_t item_size = item->Size();
|
||||
|
||||
// if key already exist, subtract old item size
|
||||
if (lru_.exists(key)) {
|
||||
const ItemObj& old_item = lru_.get(key);
|
||||
usage_ -= old_item->Size();
|
||||
}
|
||||
|
||||
// plus new item size
|
||||
usage_ += item_size;
|
||||
|
||||
// if usage exceed capacity, free some items
|
||||
if (usage_ > capacity_) {
|
||||
LOG_SERVER_DEBUG_ << header_ << " Current usage " << (usage_ >> 20) << "MB is too high for capacity "
|
||||
<< (capacity_ >> 20) << "MB, start free memory";
|
||||
free_memory_internal(capacity_);
|
||||
}
|
||||
|
||||
// insert new item
|
||||
lru_.put(key, item);
|
||||
LOG_SERVER_DEBUG_ << header_ << " Insert " << key << " size: " << (item_size >> 20) << "MB into cache";
|
||||
LOG_SERVER_DEBUG_ << header_ << " Count: " << lru_.size() << ", Usage: " << (usage_ >> 20) << "MB, Capacity: "
|
||||
<< (capacity_ >> 20) << "MB";
|
||||
}
|
||||
|
||||
template <typename ItemObj>
|
||||
void
|
||||
Cache<ItemObj>::erase_internal(const std::string& key) {
|
||||
if (!lru_.exists(key)) {
|
||||
return;
|
||||
}
|
||||
|
||||
const ItemObj& item = lru_.get(key);
|
||||
size_t item_size = item->Size();
|
||||
|
||||
lru_.erase(key);
|
||||
|
||||
usage_ -= item_size;
|
||||
LOG_SERVER_DEBUG_ << header_ << " Erase " << key << " size: " << (item_size >> 20) << "MB from cache";
|
||||
LOG_SERVER_DEBUG_ << header_ << " Count: " << lru_.size() << ", Usage: " << (usage_ >> 20) << "MB, Capacity: "
|
||||
<< (capacity_ >> 20) << "MB";
|
||||
}
|
||||
|
||||
template <typename ItemObj>
|
||||
void
|
||||
Cache<ItemObj>::free_memory_internal(const int64_t target_size) {
|
||||
int64_t threshold = std::min((int64_t)(capacity_ * freemem_percent_), target_size);
|
||||
int64_t delta_size = usage_ - threshold;
|
||||
if (delta_size <= 0) {
|
||||
delta_size = 1; // ensure at least one item erased
|
||||
}
|
||||
|
||||
std::set<std::string> key_array;
|
||||
int64_t released_size = 0;
|
||||
|
||||
auto it = lru_.rbegin();
|
||||
while (it != lru_.rend() && released_size < delta_size) {
|
||||
auto& key = it->first;
|
||||
auto& obj_ptr = it->second;
|
||||
|
||||
key_array.emplace(key);
|
||||
released_size += obj_ptr->Size();
|
||||
++it;
|
||||
}
|
||||
|
||||
LOG_SERVER_DEBUG_ << header_ << " To be released memory size: " << (released_size >> 20) << "MB";
|
||||
|
||||
for (auto& key : key_array) {
|
||||
erase_internal(key);
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace cache
|
||||
} // namespace milvus
|
||||
|
|
@ -1,71 +0,0 @@
|
|||
// Copyright (C) 2019-2020 Zilliz. All rights reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software distributed under the License
|
||||
// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
|
||||
// or implied. See the License for the specific language governing permissions and limitations under the License.
|
||||
|
||||
#pragma once
|
||||
|
||||
#include "Cache.h"
|
||||
#include "utils/Log.h"
|
||||
|
||||
#include <memory>
|
||||
#include <string>
|
||||
|
||||
namespace milvus {
|
||||
namespace cache {
|
||||
|
||||
template <typename ItemObj>
|
||||
class CacheMgr {
|
||||
public:
|
||||
virtual uint64_t
|
||||
ItemCount() const;
|
||||
|
||||
virtual bool
|
||||
ItemExists(const std::string& key);
|
||||
|
||||
virtual ItemObj
|
||||
GetItem(const std::string& key);
|
||||
|
||||
virtual void
|
||||
InsertItem(const std::string& key, const ItemObj& data);
|
||||
|
||||
virtual void
|
||||
EraseItem(const std::string& key);
|
||||
|
||||
virtual bool
|
||||
Reserve(const int64_t size);
|
||||
|
||||
virtual void
|
||||
PrintInfo();
|
||||
|
||||
virtual void
|
||||
ClearCache();
|
||||
|
||||
int64_t
|
||||
CacheUsage() const;
|
||||
|
||||
int64_t
|
||||
CacheCapacity() const;
|
||||
|
||||
void
|
||||
SetCapacity(int64_t capacity);
|
||||
|
||||
protected:
|
||||
CacheMgr();
|
||||
|
||||
virtual ~CacheMgr();
|
||||
|
||||
protected:
|
||||
std::shared_ptr<Cache<ItemObj>> cache_;
|
||||
};
|
||||
|
||||
} // namespace cache
|
||||
} // namespace milvus
|
||||
|
||||
#include "cache/CacheMgr.inl"
|
||||
|
|
@ -1,134 +0,0 @@
|
|||
// Copyright (C) 2019-2020 Zilliz. All rights reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software distributed under the License
|
||||
// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
|
||||
// or implied. See the License for the specific language governing permissions and limitations under the License.
|
||||
|
||||
namespace milvus {
|
||||
namespace cache {
|
||||
|
||||
template <typename ItemObj>
|
||||
CacheMgr<ItemObj>::CacheMgr() {
|
||||
}
|
||||
|
||||
template <typename ItemObj>
|
||||
CacheMgr<ItemObj>::~CacheMgr() {
|
||||
}
|
||||
|
||||
template <typename ItemObj>
|
||||
uint64_t
|
||||
CacheMgr<ItemObj>::ItemCount() const {
|
||||
if (cache_ == nullptr) {
|
||||
LOG_SERVER_ERROR_ << "Cache doesn't exist";
|
||||
return 0;
|
||||
}
|
||||
return (uint64_t)(cache_->size());
|
||||
}
|
||||
|
||||
template <typename ItemObj>
|
||||
bool
|
||||
CacheMgr<ItemObj>::ItemExists(const std::string& key) {
|
||||
if (cache_ == nullptr) {
|
||||
LOG_SERVER_ERROR_ << "Cache doesn't exist";
|
||||
return false;
|
||||
}
|
||||
return cache_->exists(key);
|
||||
}
|
||||
|
||||
template <typename ItemObj>
|
||||
ItemObj
|
||||
CacheMgr<ItemObj>::GetItem(const std::string& key) {
|
||||
if (cache_ == nullptr) {
|
||||
LOG_SERVER_ERROR_ << "Cache doesn't exist";
|
||||
return nullptr;
|
||||
}
|
||||
return cache_->get(key);
|
||||
}
|
||||
|
||||
template <typename ItemObj>
|
||||
void
|
||||
CacheMgr<ItemObj>::InsertItem(const std::string& key, const ItemObj& data) {
|
||||
if (cache_ == nullptr) {
|
||||
LOG_SERVER_ERROR_ << "Cache doesn't exist";
|
||||
return;
|
||||
}
|
||||
cache_->insert(key, data);
|
||||
}
|
||||
|
||||
template <typename ItemObj>
|
||||
void
|
||||
CacheMgr<ItemObj>::EraseItem(const std::string& key) {
|
||||
if (cache_ == nullptr) {
|
||||
LOG_SERVER_ERROR_ << "Cache doesn't exist";
|
||||
return;
|
||||
}
|
||||
cache_->erase(key);
|
||||
}
|
||||
|
||||
template <typename ItemObj>
|
||||
bool
|
||||
CacheMgr<ItemObj>::Reserve(const int64_t size) {
|
||||
if (cache_ == nullptr) {
|
||||
LOG_SERVER_ERROR_ << "Cache doesn't exist";
|
||||
return false;
|
||||
}
|
||||
return cache_->reserve(size);
|
||||
}
|
||||
|
||||
template <typename ItemObj>
|
||||
void
|
||||
CacheMgr<ItemObj>::PrintInfo() {
|
||||
if (cache_ == nullptr) {
|
||||
LOG_SERVER_ERROR_ << "Cache doesn't exist";
|
||||
return;
|
||||
}
|
||||
cache_->print();
|
||||
}
|
||||
|
||||
template <typename ItemObj>
|
||||
void
|
||||
CacheMgr<ItemObj>::ClearCache() {
|
||||
if (cache_ == nullptr) {
|
||||
LOG_SERVER_ERROR_ << "Cache doesn't exist";
|
||||
return;
|
||||
}
|
||||
cache_->clear();
|
||||
}
|
||||
|
||||
template <typename ItemObj>
|
||||
int64_t
|
||||
CacheMgr<ItemObj>::CacheUsage() const {
|
||||
if (cache_ == nullptr) {
|
||||
LOG_SERVER_ERROR_ << "Cache doesn't exist";
|
||||
return 0;
|
||||
}
|
||||
return cache_->usage();
|
||||
}
|
||||
|
||||
template <typename ItemObj>
|
||||
int64_t
|
||||
CacheMgr<ItemObj>::CacheCapacity() const {
|
||||
if (cache_ == nullptr) {
|
||||
LOG_SERVER_ERROR_ << "Cache doesn't exist";
|
||||
return 0;
|
||||
}
|
||||
return cache_->capacity();
|
||||
}
|
||||
|
||||
template <typename ItemObj>
|
||||
void
|
||||
CacheMgr<ItemObj>::SetCapacity(int64_t capacity) {
|
||||
if (cache_ == nullptr) {
|
||||
LOG_SERVER_ERROR_ << "Cache doesn't exist";
|
||||
return;
|
||||
}
|
||||
cache_->set_capacity(capacity);
|
||||
}
|
||||
|
||||
} // namespace cache
|
||||
} // namespace milvus
|
||||
|
|
@ -1,55 +0,0 @@
|
|||
// Copyright (C) 2019-2020 Zilliz. All rights reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software distributed under the License
|
||||
// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
|
||||
// or implied. See the License for the specific language governing permissions and limitations under the License.
|
||||
|
||||
#include "cache/CpuCacheMgr.h"
|
||||
|
||||
#include <utility>
|
||||
|
||||
#include <fiu/fiu-local.h>
|
||||
|
||||
#include "utils/Log.h"
|
||||
#include "value/config/ServerConfig.h"
|
||||
|
||||
namespace milvus {
|
||||
namespace cache {
|
||||
|
||||
CpuCacheMgr&
|
||||
CpuCacheMgr::GetInstance() {
|
||||
static CpuCacheMgr s_mgr;
|
||||
return s_mgr;
|
||||
}
|
||||
|
||||
CpuCacheMgr::CpuCacheMgr() {
|
||||
cache_ = std::make_shared<Cache<DataObjPtr>>(config.cache.cache_size(), 1UL << 32, "[CACHE CPU]");
|
||||
|
||||
if (config.cache.cpu_cache_threshold() > 0.0) {
|
||||
cache_->set_freemem_percent(config.cache.cpu_cache_threshold());
|
||||
}
|
||||
ConfigMgr::GetInstance().Attach("cache.cache_size", this);
|
||||
}
|
||||
|
||||
CpuCacheMgr::~CpuCacheMgr() {
|
||||
ConfigMgr::GetInstance().Detach("cache.cache_size", this);
|
||||
}
|
||||
|
||||
DataObjPtr
|
||||
CpuCacheMgr::GetItem(const std::string& key) {
|
||||
auto ret = CacheMgr<DataObjPtr>::GetItem(key);
|
||||
return ret;
|
||||
}
|
||||
|
||||
void
|
||||
CpuCacheMgr::ConfigUpdate(const std::string& name) {
|
||||
SetCapacity(config.cache.cache_size());
|
||||
}
|
||||
|
||||
} // namespace cache
|
||||
} // namespace milvus
|
||||
|
|
@ -1,44 +0,0 @@
|
|||
// Copyright (C) 2019-2020 Zilliz. All rights reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software distributed under the License
|
||||
// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
|
||||
// or implied. See the License for the specific language governing permissions and limitations under the License.
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <memory>
|
||||
#include <string>
|
||||
|
||||
#include "cache/CacheMgr.h"
|
||||
#include "cache/DataObj.h"
|
||||
#include "value/config/ConfigMgr.h"
|
||||
|
||||
namespace milvus {
|
||||
namespace cache {
|
||||
|
||||
class CpuCacheMgr : public CacheMgr<DataObjPtr>, public ConfigObserver {
|
||||
public:
|
||||
static CpuCacheMgr&
|
||||
GetInstance();
|
||||
|
||||
private:
|
||||
CpuCacheMgr();
|
||||
|
||||
~CpuCacheMgr();
|
||||
|
||||
public:
|
||||
DataObjPtr
|
||||
GetItem(const std::string& key) override;
|
||||
|
||||
public:
|
||||
void
|
||||
ConfigUpdate(const std::string& name) override;
|
||||
};
|
||||
|
||||
} // namespace cache
|
||||
} // namespace milvus
|
||||
|
|
@ -1,63 +0,0 @@
|
|||
// Copyright (C) 2019-2020 Zilliz. All rights reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software distributed under the License
|
||||
// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
|
||||
// or implied. See the License for the specific language governing permissions and limitations under the License.
|
||||
|
||||
#include "cache/GpuCacheMgr.h"
|
||||
#include "utils/Log.h"
|
||||
#include "value/config/ServerConfig.h"
|
||||
|
||||
#include <fiu/fiu-local.h>
|
||||
#include <sstream>
|
||||
#include <utility>
|
||||
|
||||
namespace milvus {
|
||||
namespace cache {
|
||||
|
||||
#ifdef MILVUS_GPU_VERSION
|
||||
std::mutex GpuCacheMgr::global_mutex_;
|
||||
std::unordered_map<int64_t, GpuCacheMgrPtr> GpuCacheMgr::instance_;
|
||||
|
||||
GpuCacheMgr::GpuCacheMgr(int64_t gpu_id) : gpu_id_(gpu_id) {
|
||||
std::string header = "[CACHE GPU" + std::to_string(gpu_id) + "]";
|
||||
cache_ = std::make_shared<Cache<DataObjPtr>>(config.gpu.cache_size(), 1UL << 32, header);
|
||||
|
||||
if (config.gpu.cache_threshold() > 0.0) {
|
||||
cache_->set_freemem_percent(config.gpu.cache_threshold());
|
||||
}
|
||||
ConfigMgr::GetInstance().Attach("gpu.cache_threshold", this);
|
||||
}
|
||||
|
||||
GpuCacheMgr::~GpuCacheMgr() {
|
||||
ConfigMgr::GetInstance().Detach("gpu.cache_threshold", this);
|
||||
}
|
||||
|
||||
GpuCacheMgrPtr
|
||||
GpuCacheMgr::GetInstance(int64_t gpu_id) {
|
||||
if (instance_.find(gpu_id) == instance_.end()) {
|
||||
std::lock_guard<std::mutex> lock(global_mutex_);
|
||||
if (instance_.find(gpu_id) == instance_.end()) {
|
||||
instance_[gpu_id] = std::make_shared<GpuCacheMgr>(gpu_id);
|
||||
}
|
||||
}
|
||||
return instance_[gpu_id];
|
||||
}
|
||||
|
||||
void
|
||||
GpuCacheMgr::ConfigUpdate(const std::string& name) {
|
||||
std::lock_guard<std::mutex> lock(global_mutex_);
|
||||
for (auto& it : instance_) {
|
||||
it.second->SetCapacity(config.gpu.cache_size());
|
||||
}
|
||||
}
|
||||
|
||||
#endif
|
||||
|
||||
} // namespace cache
|
||||
} // namespace milvus
|
||||
|
|
@ -1,51 +0,0 @@
|
|||
// Copyright (C) 2019-2020 Zilliz. All rights reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software distributed under the License
|
||||
// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
|
||||
// or implied. See the License for the specific language governing permissions and limitations under the License.
|
||||
|
||||
#include <memory>
|
||||
#include <mutex>
|
||||
#include <string>
|
||||
#include <unordered_map>
|
||||
#include <utility>
|
||||
|
||||
#include "cache/CacheMgr.h"
|
||||
#include "cache/DataObj.h"
|
||||
#include "value/config/ConfigMgr.h"
|
||||
|
||||
namespace milvus {
|
||||
namespace cache {
|
||||
|
||||
#ifdef MILVUS_GPU_VERSION
|
||||
class GpuCacheMgr;
|
||||
using GpuCacheMgrPtr = std::shared_ptr<GpuCacheMgr>;
|
||||
using MutexPtr = std::shared_ptr<std::mutex>;
|
||||
|
||||
class GpuCacheMgr : public CacheMgr<DataObjPtr>, public ConfigObserver {
|
||||
public:
|
||||
explicit GpuCacheMgr(int64_t gpu_id);
|
||||
|
||||
~GpuCacheMgr();
|
||||
|
||||
static GpuCacheMgrPtr
|
||||
GetInstance(int64_t gpu_id);
|
||||
|
||||
public:
|
||||
void
|
||||
ConfigUpdate(const std::string& name) override;
|
||||
|
||||
private:
|
||||
int64_t gpu_id_;
|
||||
static std::mutex global_mutex_;
|
||||
static std::unordered_map<int64_t, GpuCacheMgrPtr> instance_;
|
||||
};
|
||||
#endif
|
||||
|
||||
} // namespace cache
|
||||
} // namespace milvus
|
||||
|
|
@ -1,116 +0,0 @@
|
|||
// Copyright (C) 2019-2020 Zilliz. All rights reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software distributed under the License
|
||||
// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
|
||||
// or implied. See the License for the specific language governing permissions and limitations under the License.
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <cstddef>
|
||||
#include <list>
|
||||
#include <stdexcept>
|
||||
#include <unordered_map>
|
||||
#include <utility>
|
||||
|
||||
namespace milvus {
|
||||
namespace cache {
|
||||
|
||||
template <typename key_t, typename value_t>
|
||||
class LRU {
|
||||
public:
|
||||
typedef typename std::pair<key_t, value_t> key_value_pair_t;
|
||||
typedef typename std::list<key_value_pair_t>::iterator list_iterator_t;
|
||||
typedef typename std::list<key_value_pair_t>::reverse_iterator reverse_list_iterator_t;
|
||||
|
||||
explicit LRU(size_t max_size) : max_size_(max_size) {
|
||||
}
|
||||
|
||||
void
|
||||
put(const key_t& key, const value_t& value) {
|
||||
auto it = cache_items_map_.find(key);
|
||||
cache_items_list_.push_front(key_value_pair_t(key, value));
|
||||
if (it != cache_items_map_.end()) {
|
||||
cache_items_list_.erase(it->second);
|
||||
cache_items_map_.erase(it);
|
||||
}
|
||||
cache_items_map_[key] = cache_items_list_.begin();
|
||||
|
||||
if (cache_items_map_.size() > max_size_) {
|
||||
auto last = cache_items_list_.end();
|
||||
last--;
|
||||
cache_items_map_.erase(last->first);
|
||||
cache_items_list_.pop_back();
|
||||
}
|
||||
}
|
||||
|
||||
const value_t&
|
||||
get(const key_t& key) {
|
||||
auto it = cache_items_map_.find(key);
|
||||
if (it == cache_items_map_.end()) {
|
||||
throw std::range_error("There is no such key in cache");
|
||||
} else {
|
||||
cache_items_list_.splice(cache_items_list_.begin(), cache_items_list_, it->second);
|
||||
return it->second->second;
|
||||
}
|
||||
}
|
||||
|
||||
void
|
||||
erase(const key_t& key) {
|
||||
auto it = cache_items_map_.find(key);
|
||||
if (it != cache_items_map_.end()) {
|
||||
cache_items_list_.erase(it->second);
|
||||
cache_items_map_.erase(it);
|
||||
}
|
||||
}
|
||||
|
||||
bool
|
||||
exists(const key_t& key) const {
|
||||
return cache_items_map_.find(key) != cache_items_map_.end();
|
||||
}
|
||||
|
||||
size_t
|
||||
size() const {
|
||||
return cache_items_map_.size();
|
||||
}
|
||||
|
||||
list_iterator_t
|
||||
begin() {
|
||||
iter_ = cache_items_list_.begin();
|
||||
return iter_;
|
||||
}
|
||||
|
||||
list_iterator_t
|
||||
end() {
|
||||
return cache_items_list_.end();
|
||||
}
|
||||
|
||||
reverse_list_iterator_t
|
||||
rbegin() {
|
||||
return cache_items_list_.rbegin();
|
||||
}
|
||||
|
||||
reverse_list_iterator_t
|
||||
rend() {
|
||||
return cache_items_list_.rend();
|
||||
}
|
||||
|
||||
void
|
||||
clear() {
|
||||
cache_items_list_.clear();
|
||||
cache_items_map_.clear();
|
||||
}
|
||||
|
||||
private:
|
||||
std::list<key_value_pair_t> cache_items_list_;
|
||||
std::unordered_map<key_t, list_iterator_t> cache_items_map_;
|
||||
size_t max_size_;
|
||||
list_iterator_t iter_;
|
||||
};
|
||||
|
||||
} // namespace cache
|
||||
} // namespace milvus
|
||||
|
|
@ -1,129 +0,0 @@
|
|||
// Copyright (C) 2019-2020 Zilliz. All rights reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software distributed under the License
|
||||
// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
|
||||
// or implied. See the License for the specific language governing permissions and limitations under the License.
|
||||
|
||||
#pragma once
|
||||
|
||||
/*
|
||||
* Please use LOG_MODULE_LEVEL_C macro in member function of class
|
||||
* and LOG_MODULE_LEVEL_ macro in other functions.
|
||||
*/
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
#define SERVER_MODULE_NAME "SERVER"
|
||||
#define SERVER_MODULE_CLASS_FUNCTION \
|
||||
LogOut("[%s][%s::%s][%s] ", SERVER_MODULE_NAME, (typeid(*this).name()), __FUNCTION__, \
|
||||
milvus::knowhere::GetThreadName().c_str())
|
||||
#define SERVER_MODULE_FUNCTION \
|
||||
LogOut("[%s][%s][%s] ", SERVER_MODULE_NAME, __FUNCTION__, milvus::knowhere::GetThreadName().c_str())
|
||||
|
||||
#define LOG_SERVER_TRACE_C LOG(TRACE) << SERVER_MODULE_CLASS_FUNCTION
|
||||
#define LOG_SERVER_DEBUG_C LOG(DEBUG) << SERVER_MODULE_CLASS_FUNCTION
|
||||
#define LOG_SERVER_INFO_C LOG(INFO) << SERVER_MODULE_CLASS_FUNCTION
|
||||
#define LOG_SERVER_WARNING_C LOG(WARNING) << SERVER_MODULE_CLASS_FUNCTION
|
||||
#define LOG_SERVER_ERROR_C LOG(ERROR) << SERVER_MODULE_CLASS_FUNCTION
|
||||
#define LOG_SERVER_FATAL_C LOG(FATAL) << SERVER_MODULE_CLASS_FUNCTION
|
||||
|
||||
#define LOG_SERVER_TRACE_ LOG(TRACE) << SERVER_MODULE_FUNCTION
|
||||
#define LOG_SERVER_DEBUG_ LOG(DEBUG) << SERVER_MODULE_FUNCTION
|
||||
#define LOG_SERVER_INFO_ LOG(INFO) << SERVER_MODULE_FUNCTION
|
||||
#define LOG_SERVER_WARNING_ LOG(WARNING) << SERVER_MODULE_FUNCTION
|
||||
#define LOG_SERVER_ERROR_ LOG(ERROR) << SERVER_MODULE_FUNCTION
|
||||
#define LOG_SERVER_FATAL_ LOG(FATAL) << SERVER_MODULE_FUNCTION
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
#define ENGINE_MODULE_NAME "ENGINE"
|
||||
#define ENGINE_MODULE_CLASS_FUNCTION \
|
||||
LogOut("[%s][%s::%s][%s] ", ENGINE_MODULE_NAME, (typeid(*this).name()), __FUNCTION__, \
|
||||
milvus::knowhere::GetThreadName().c_str())
|
||||
#define ENGINE_MODULE_FUNCTION \
|
||||
LogOut("[%s][%s][%s] ", ENGINE_MODULE_NAME, __FUNCTION__, milvus::knowhere::GetThreadName().c_str())
|
||||
|
||||
#define LOG_ENGINE_TRACE_C LOG(TRACE) << ENGINE_MODULE_CLASS_FUNCTION
|
||||
#define LOG_ENGINE_DEBUG_C LOG(DEBUG) << ENGINE_MODULE_CLASS_FUNCTION
|
||||
#define LOG_ENGINE_INFO_C LOG(INFO) << ENGINE_MODULE_CLASS_FUNCTION
|
||||
#define LOG_ENGINE_WARNING_C LOG(WARNING) << ENGINE_MODULE_CLASS_FUNCTION
|
||||
#define LOG_ENGINE_ERROR_C LOG(ERROR) << ENGINE_MODULE_CLASS_FUNCTION
|
||||
#define LOG_ENGINE_FATAL_C LOG(FATAL) << ENGINE_MODULE_CLASS_FUNCTION
|
||||
|
||||
#define LOG_ENGINE_TRACE_ LOG(TRACE) << ENGINE_MODULE_FUNCTION
|
||||
#define LOG_ENGINE_DEBUG_ LOG(DEBUG) << ENGINE_MODULE_FUNCTION
|
||||
#define LOG_ENGINE_INFO_ LOG(INFO) << ENGINE_MODULE_FUNCTION
|
||||
#define LOG_ENGINE_WARNING_ LOG(WARNING) << ENGINE_MODULE_FUNCTION
|
||||
#define LOG_ENGINE_ERROR_ LOG(ERROR) << ENGINE_MODULE_FUNCTION
|
||||
#define LOG_ENGINE_FATAL_ LOG(FATAL) << ENGINE_MODULE_FUNCTION
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
#define WRAPPER_MODULE_NAME "WRAPPER"
|
||||
#define WRAPPER_MODULE_CLASS_FUNCTION \
|
||||
LogOut("[%s][%s::%s][%s] ", WRAPPER_MODULE_NAME, (typeid(*this).name()), __FUNCTION__, \
|
||||
milvus::knowhere::GetThreadName().c_str())
|
||||
#define WRAPPER_MODULE_FUNCTION \
|
||||
LogOut("[%s][%s][%s] ", WRAPPER_MODULE_NAME, __FUNCTION__, milvus::knowhere::GetThreadName().c_str())
|
||||
|
||||
#define LOG_WRAPPER_TRACE_C LOG(TRACE) << WRAPPER_MODULE_CLASS_FUNCTION
|
||||
#define LOG_WRAPPER_DEBUG_C LOG(DEBUG) << WRAPPER_MODULE_CLASS_FUNCTION
|
||||
#define LOG_WRAPPER_INFO_C LOG(INFO) << WRAPPER_MODULE_CLASS_FUNCTION
|
||||
#define LOG_WRAPPER_WARNING_C LOG(WARNING) << WRAPPER_MODULE_CLASS_FUNCTION
|
||||
#define LOG_WRAPPER_ERROR_C LOG(ERROR) << WRAPPER_MODULE_CLASS_FUNCTION
|
||||
#define LOG_WRAPPER_FATAL_C LOG(FATAL) << WRAPPER_MODULE_CLASS_FUNCTION
|
||||
|
||||
#define LOG_WRAPPER_TRACE_ LOG(TRACE) << WRAPPER_MODULE_FUNCTION
|
||||
#define LOG_WRAPPER_DEBUG_ LOG(DEBUG) << WRAPPER_MODULE_FUNCTION
|
||||
#define LOG_WRAPPER_INFO_ LOG(INFO) << WRAPPER_MODULE_FUNCTION
|
||||
#define LOG_WRAPPER_WARNING_ LOG(WARNING) << WRAPPER_MODULE_FUNCTION
|
||||
#define LOG_WRAPPER_ERROR_ LOG(ERROR) << WRAPPER_MODULE_FUNCTION
|
||||
#define LOG_WRAPPER_FATAL_ LOG(FATAL) << WRAPPER_MODULE_FUNCTION
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
#define STORAGE_MODULE_NAME "STORAGE"
|
||||
#define STORAGE_MODULE_CLASS_FUNCTION \
|
||||
LogOut("[%s][%s::%s][%s] ", STORAGE_MODULE_NAME, (typeid(*this).name()), __FUNCTION__, \
|
||||
milvus::knowhere::GetThreadName().c_str())
|
||||
#define STORAGE_MODULE_FUNCTION \
|
||||
LogOut("[%s][%s][%s] ", STORAGE_MODULE_NAME, __FUNCTION__, milvus::knowhere::GetThreadName().c_str())
|
||||
|
||||
#define LOG_STORAGE_TRACE_C LOG(TRACE) << STORAGE_MODULE_CLASS_FUNCTION
|
||||
#define LOG_STORAGE_DEBUG_C LOG(DEBUG) << STORAGE_MODULE_CLASS_FUNCTION
|
||||
#define LOG_STORAGE_INFO_C LOG(INFO) << STORAGE_MODULE_CLASS_FUNCTION
|
||||
#define LOG_STORAGE_WARNING_C LOG(WARNING) << STORAGE_MODULE_CLASS_FUNCTION
|
||||
#define LOG_STORAGE_ERROR_C LOG(ERROR) << STORAGE_MODULE_CLASS_FUNCTION
|
||||
#define LOG_STORAGE_FATAL_C LOG(FATAL) << STORAGE_MODULE_CLASS_FUNCTION
|
||||
|
||||
#define LOG_STORAGE_TRACE_ LOG(TRACE) << STORAGE_MODULE_FUNCTION
|
||||
#define LOG_STORAGE_DEBUG_ LOG(DEBUG) << STORAGE_MODULE_FUNCTION
|
||||
#define LOG_STORAGE_INFO_ LOG(INFO) << STORAGE_MODULE_FUNCTION
|
||||
#define LOG_STORAGE_WARNING_ LOG(WARNING) << STORAGE_MODULE_FUNCTION
|
||||
#define LOG_STORAGE_ERROR_ LOG(ERROR) << STORAGE_MODULE_FUNCTION
|
||||
#define LOG_STORAGE_FATAL_ LOG(FATAL) << STORAGE_MODULE_FUNCTION
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
#define WAL_MODULE_NAME "WAL"
|
||||
#define WAL_MODULE_CLASS_FUNCTION \
|
||||
LogOut("[%s][%s::%s][%s] ", WAL_MODULE_NAME, (typeid(*this).name()), __FUNCTION__, \
|
||||
milvus::knowhere::GetThreadName().c_str())
|
||||
#define WAL_MODULE_FUNCTION \
|
||||
LogOut("[%s][%s][%s] ", WAL_MODULE_NAME, __FUNCTION__, milvus::knowhere::GetThreadName().c_str())
|
||||
|
||||
#define LOG_WAL_TRACE_C LOG(TRACE) << WAL_MODULE_CLASS_FUNCTION
|
||||
#define LOG_WAL_DEBUG_C LOG(DEBUG) << WAL_MODULE_CLASS_FUNCTION
|
||||
#define LOG_WAL_INFO_C LOG(INFO) << WAL_MODULE_CLASS_FUNCTION
|
||||
#define LOG_WAL_WARNING_C LOG(WARNING) << WAL_MODULE_CLASS_FUNCTION
|
||||
#define LOG_WAL_ERROR_C LOG(ERROR) << WAL_MODULE_CLASS_FUNCTION
|
||||
#define LOG_WAL_FATAL_C LOG(FATAL) << WAL_MODULE_CLASS_FUNCTION
|
||||
|
||||
#define LOG_WAL_TRACE_ LOG(TRACE) << WAL_MODULE_FUNCTION
|
||||
#define LOG_WAL_DEBUG_ LOG(DEBUG) << WAL_MODULE_FUNCTION
|
||||
#define LOG_WAL_INFO_ LOG(INFO) << WAL_MODULE_FUNCTION
|
||||
#define LOG_WAL_WARNING_ LOG(WARNING) << WAL_MODULE_FUNCTION
|
||||
#define LOG_WAL_ERROR_ LOG(ERROR) << WAL_MODULE_FUNCTION
|
||||
#define LOG_WAL_FATAL_ LOG(FATAL) << WAL_MODULE_FUNCTION
|
||||
|
||||
/////////////////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
|
@ -1,108 +0,0 @@
|
|||
// Copyright (C) 2019-2020 Zilliz. All rights reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software distributed under the License
|
||||
// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
|
||||
// or implied. See the License for the specific language governing permissions and limitations under the License.
|
||||
|
||||
#include "log/Log.h"
|
||||
INITIALIZE_EASYLOGGINGPP
|
||||
|
||||
#include <chrono>
|
||||
#include <cstdarg>
|
||||
#include <cstdio>
|
||||
#include <memory>
|
||||
#include <stdexcept>
|
||||
#include <string>
|
||||
|
||||
namespace milvus {
|
||||
|
||||
std::string
|
||||
LogOut(const char* pattern, ...) {
|
||||
size_t len = strnlen(pattern, 1024) + 256;
|
||||
auto str_p = std::make_unique<char[]>(len);
|
||||
memset(str_p.get(), 0, len);
|
||||
|
||||
va_list vl;
|
||||
va_start(vl, pattern);
|
||||
vsnprintf(str_p.get(), len, pattern, vl); // NOLINT
|
||||
va_end(vl);
|
||||
|
||||
return std::string(str_p.get());
|
||||
}
|
||||
|
||||
void
|
||||
SetThreadName(const std::string& name) {
|
||||
// Note: the name cannot exceed 16 bytes
|
||||
pthread_setname_np(pthread_self(), name.c_str());
|
||||
}
|
||||
|
||||
std::string
|
||||
GetThreadName() {
|
||||
std::string thread_name = "unamed";
|
||||
char name[16];
|
||||
size_t len = 16;
|
||||
auto err = pthread_getname_np(pthread_self(), name, len);
|
||||
if (not err) {
|
||||
thread_name = name;
|
||||
}
|
||||
|
||||
return thread_name;
|
||||
}
|
||||
|
||||
int64_t
|
||||
get_now_timestamp() {
|
||||
auto now = std::chrono::system_clock::now().time_since_epoch();
|
||||
return std::chrono::duration_cast<std::chrono::seconds>(now).count();
|
||||
}
|
||||
|
||||
int64_t
|
||||
get_system_boottime() {
|
||||
FILE* uptime = fopen("/proc/uptime", "r");
|
||||
float since_sys_boot, _;
|
||||
auto ret = fscanf(uptime, "%f %f", &since_sys_boot, &_);
|
||||
fclose(uptime);
|
||||
if (ret != 2) {
|
||||
throw std::runtime_error("read /proc/uptime failed.");
|
||||
}
|
||||
return static_cast<int64_t>(since_sys_boot);
|
||||
}
|
||||
|
||||
int64_t
|
||||
get_thread_starttime() {
|
||||
int64_t tid = gettid();
|
||||
int64_t pid = getpid();
|
||||
char filename[256];
|
||||
snprintf(filename, sizeof(filename), "/proc/%ld/task/%ld/stat", pid, tid);
|
||||
|
||||
int64_t val = 0;
|
||||
char comm[16], state;
|
||||
FILE* thread_stat = fopen(filename, "r");
|
||||
auto ret = fscanf(thread_stat, "%ld %s %s ", &val, comm, &state);
|
||||
for (auto i = 4; i < 23; i++) {
|
||||
ret = fscanf(thread_stat, "%ld ", &val);
|
||||
if (i == 22) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
fclose(thread_stat);
|
||||
if (ret != 1) {
|
||||
throw std::runtime_error("read " + std::string(filename) + " failed.");
|
||||
}
|
||||
return val / sysconf(_SC_CLK_TCK);
|
||||
}
|
||||
|
||||
int64_t
|
||||
get_thread_start_timestamp() {
|
||||
try {
|
||||
return get_now_timestamp() - get_system_boottime() + get_thread_starttime();
|
||||
} catch (...) {
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace milvus
|
||||
|
|
@ -1,77 +0,0 @@
|
|||
// Copyright (C) 2019-2020 Zilliz. All rights reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software distributed under the License
|
||||
// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
|
||||
// or implied. See the License for the specific language governing permissions and limitations under the License.
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <string>
|
||||
|
||||
#include <sys/types.h>
|
||||
#include <unistd.h>
|
||||
#include "easyloggingpp/easylogging++.h"
|
||||
|
||||
#include "log/DeprecatedLog.h"
|
||||
|
||||
namespace milvus {
|
||||
|
||||
#if __GLIBC__ == 2 && __GLIBC_MINOR__ < 30
|
||||
#include <sys/syscall.h>
|
||||
#define gettid() syscall(SYS_gettid)
|
||||
#endif
|
||||
|
||||
// Log message format: %timestamp | %request_id | %level | %collection_name | %client_id | %client_tag | %client_ipport
|
||||
// | %thread_id | %thread_start_timestamp | %command_tag | %module | %error_code | %message
|
||||
|
||||
#define VAR_REQUEST_ID (context->request_id())
|
||||
#define VAR_COLLECTION_NAME (context->collection_name())
|
||||
#define VAR_CLIENT_ID ("")
|
||||
#define VAR_CLIENT_TAG (context->client_tag())
|
||||
#define VAR_CLIENT_IPPORT (context->client_ipport())
|
||||
#define VAR_THREAD_ID (gettid())
|
||||
#define VAR_THREAD_START_TIMESTAMP (get_thread_start_timestamp())
|
||||
#define VAR_COMMAND_TAG (context->command_tag())
|
||||
|
||||
// Use this macro whenever possible
|
||||
// Depends variables: context Context
|
||||
#define MLOG(level, module, error_code) \
|
||||
LOG(level) << " | " << VAR_REQUEST_ID << " | " << #level << " | " << VAR_COLLECTION_NAME << " | " << VAR_CLIENT_ID \
|
||||
<< " | " << VAR_CLIENT_TAG << " | " << VAR_CLIENT_IPPORT << " | " << VAR_THREAD_ID << " | " \
|
||||
<< VAR_THREAD_START_TIMESTAMP << " | " << VAR_COMMAND_TAG << " | " << #module << " | " << error_code \
|
||||
<< " | "
|
||||
|
||||
// Use in some background process only
|
||||
#define MLOG_(level, module, error_code) \
|
||||
LOG(level) << " | " \
|
||||
<< "" \
|
||||
<< " | " << #level << " | " \
|
||||
<< "" \
|
||||
<< " | " \
|
||||
<< "" \
|
||||
<< " | " \
|
||||
<< "" \
|
||||
<< " | " \
|
||||
<< "" \
|
||||
<< " | " << VAR_THREAD_ID << " | " << VAR_THREAD_START_TIMESTAMP << " | " \
|
||||
<< "" \
|
||||
<< " | " << #module << " | " << error_code << " | "
|
||||
|
||||
std::string
|
||||
LogOut(const char* pattern, ...);
|
||||
|
||||
void
|
||||
SetThreadName(const std::string& name);
|
||||
|
||||
std::string
|
||||
GetThreadName();
|
||||
|
||||
int64_t
|
||||
get_thread_start_timestamp();
|
||||
|
||||
} // namespace milvus
|
||||
|
|
@ -1,262 +0,0 @@
|
|||
// Copyright (C) 2019-2020 Zilliz. All rights reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software distributed under the License
|
||||
// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
|
||||
// or implied. See the License for the specific language governing permissions and limitations under the License.
|
||||
|
||||
#include <fiu/fiu-local.h>
|
||||
#include <libgen.h>
|
||||
#include <cctype>
|
||||
#include <string>
|
||||
#include <unordered_map>
|
||||
#include <utility>
|
||||
#include <vector>
|
||||
|
||||
#include <boost/filesystem.hpp>
|
||||
|
||||
#include "log/LogMgr.h"
|
||||
#include "utils/Status.h"
|
||||
#include "value/config/ServerConfig.h"
|
||||
|
||||
namespace milvus {
|
||||
|
||||
int LogMgr::trace_idx = 0;
|
||||
int LogMgr::global_idx = 0;
|
||||
int LogMgr::debug_idx = 0;
|
||||
int LogMgr::info_idx = 0;
|
||||
int LogMgr::warning_idx = 0;
|
||||
int LogMgr::error_idx = 0;
|
||||
int LogMgr::fatal_idx = 0;
|
||||
int64_t LogMgr::logs_delete_exceeds = 1;
|
||||
bool LogMgr::enable_log_delete = false;
|
||||
|
||||
Status
|
||||
LogMgr::InitLog(bool trace_enable,
|
||||
const std::string& level,
|
||||
const std::string& logs_path,
|
||||
const std::string& filename,
|
||||
int64_t max_log_file_size,
|
||||
int64_t log_rotate_num,
|
||||
bool log_to_stdout,
|
||||
bool log_to_file) {
|
||||
try {
|
||||
auto enables = parse_level(level);
|
||||
enables["trace"] = trace_enable;
|
||||
LogMgr log_mgr(logs_path);
|
||||
log_mgr.Default()
|
||||
.Filename(filename)
|
||||
.Level(enables)
|
||||
.To(log_to_stdout, log_to_file)
|
||||
.Rotate(max_log_file_size, log_rotate_num)
|
||||
.Setup();
|
||||
} catch (std::exception& ex) {
|
||||
return Status(SERVER_UNEXPECTED_ERROR, ex.what());
|
||||
}
|
||||
|
||||
return Status::OK();
|
||||
}
|
||||
|
||||
// TODO(yzb) : change the easylogging library to get the log level from parameter rather than filename
|
||||
void
|
||||
LogMgr::RolloutHandler(const char* filename, std::size_t size, el::Level level) {
|
||||
char* dirc = strdup(filename);
|
||||
char* basec = strdup(filename);
|
||||
char* dir = dirname(dirc);
|
||||
char* base = basename(basec);
|
||||
|
||||
std::string s(base);
|
||||
std::vector<std::string> list = {"\\", " ", "\'", "\"", "*", "\?", "{", "}", ";", "<",
|
||||
">", "|", "^", "&", "$", "#", "!", "`", "~"};
|
||||
std::string::size_type position;
|
||||
for (auto& substr : list) {
|
||||
position = 0;
|
||||
while ((position = s.find_first_of(substr, position)) != std::string::npos) {
|
||||
s.insert(position, "\\");
|
||||
position += 2;
|
||||
}
|
||||
}
|
||||
std::string m(std::string(dir) + "/" + s);
|
||||
try {
|
||||
switch (level) {
|
||||
case el::Level::Trace: {
|
||||
rename_and_delete(m, ++trace_idx);
|
||||
break;
|
||||
}
|
||||
case el::Level::Global: {
|
||||
rename_and_delete(m, ++global_idx);
|
||||
break;
|
||||
}
|
||||
case el::Level::Debug: {
|
||||
rename_and_delete(m, ++debug_idx);
|
||||
break;
|
||||
}
|
||||
case el::Level::Info: {
|
||||
rename_and_delete(m, ++info_idx);
|
||||
break;
|
||||
}
|
||||
case el::Level::Warning: {
|
||||
rename_and_delete(m, ++warning_idx);
|
||||
break;
|
||||
}
|
||||
case el::Level::Error: {
|
||||
rename_and_delete(m, ++error_idx);
|
||||
break;
|
||||
}
|
||||
case el::Level::Fatal: {
|
||||
rename_and_delete(m, ++fatal_idx);
|
||||
break;
|
||||
}
|
||||
default: {
|
||||
break;
|
||||
}
|
||||
}
|
||||
} catch (const std::exception& exc) {
|
||||
std::cerr << exc.what() << ". Exception throws from RolloutHandler." << std::endl;
|
||||
}
|
||||
}
|
||||
|
||||
LogMgr::LogMgr(std::string log_path) : logs_path_(std::move(log_path)) {
|
||||
}
|
||||
|
||||
LogMgr&
|
||||
LogMgr::Default() {
|
||||
el_config_.setToDefault();
|
||||
el_config_.setGlobally(el::ConfigurationType::Format, "[%datetime][%level]%msg");
|
||||
el_config_.setGlobally(el::ConfigurationType::SubsecondPrecision, "3");
|
||||
el_config_.setGlobally(el::ConfigurationType::PerformanceTracking, "false");
|
||||
|
||||
return *this;
|
||||
}
|
||||
|
||||
LogMgr&
|
||||
LogMgr::Filename(const std::string& filename) {
|
||||
std::string logs_reg_path = logs_path_.rfind('/') == logs_path_.length() - 1 ? logs_path_ : logs_path_ + "/";
|
||||
|
||||
std::string log_file = logs_reg_path + filename;
|
||||
|
||||
/* Set set log file at Global level to make all level log output to the same log file*/
|
||||
el_config_.set(el::Level::Global, el::ConfigurationType::Filename, log_file.c_str());
|
||||
|
||||
return *this;
|
||||
}
|
||||
|
||||
LogMgr&
|
||||
LogMgr::Level(std::unordered_map<std::string, bool>& enables) {
|
||||
fiu_do_on("LogMgr.Level.trace_enable_to_false", enables["trace"] = false);
|
||||
enable(el_config_, el::Level::Trace, enables["trace"]);
|
||||
|
||||
fiu_do_on("LogMgr.Level.info_enable_to_false", enables["info"] = false);
|
||||
enable(el_config_, el::Level::Info, enables["info"]);
|
||||
|
||||
fiu_do_on("LogMgr.Level.debug_enable_to_false", enables["debug"] = false);
|
||||
enable(el_config_, el::Level::Debug, enables["debug"]);
|
||||
|
||||
fiu_do_on("LogMgr.Level.warning_enable_to_false", enables["warning"] = false);
|
||||
enable(el_config_, el::Level::Warning, enables["warning"]);
|
||||
|
||||
fiu_do_on("LogMgr.Level.error_enable_to_false", enables["error"] = false);
|
||||
enable(el_config_, el::Level::Error, enables["error"]);
|
||||
|
||||
fiu_do_on("LogMgr.Level.fatal_enable_to_false", enables["fatal"] = false);
|
||||
enable(el_config_, el::Level::Fatal, enables["fatal"]);
|
||||
|
||||
return *this;
|
||||
}
|
||||
|
||||
LogMgr&
|
||||
LogMgr::To(bool log_to_stdout, bool log_to_file) {
|
||||
el_config_.setGlobally(el::ConfigurationType::ToStandardOutput, (log_to_stdout ? "true" : "false"));
|
||||
el_config_.setGlobally(el::ConfigurationType::ToFile, (log_to_file ? "true" : "false"));
|
||||
|
||||
return *this;
|
||||
}
|
||||
|
||||
LogMgr&
|
||||
LogMgr::Rotate(int64_t max_log_file_size, int64_t log_rotate_num) {
|
||||
fiu_do_on("LogMgr.Rotate.set_max_log_size_small_than_min", max_log_file_size = MAX_LOG_FILE_SIZE_MIN - 1);
|
||||
if (max_log_file_size < MAX_LOG_FILE_SIZE_MIN || max_log_file_size > MAX_LOG_FILE_SIZE_MAX) {
|
||||
std::string msg = "max_log_file_size must in range[" + std::to_string(MAX_LOG_FILE_SIZE_MIN) + ", " +
|
||||
std::to_string(MAX_LOG_FILE_SIZE_MAX) + "], now is " + std::to_string(max_log_file_size);
|
||||
throw std::runtime_error(msg);
|
||||
}
|
||||
|
||||
el_config_.setGlobally(el::ConfigurationType::MaxLogFileSize, std::to_string(max_log_file_size));
|
||||
el::Loggers::addFlag(el::LoggingFlag::StrictLogFileSizeCheck);
|
||||
el::Helpers::installPreRollOutCallback(LogMgr::RolloutHandler);
|
||||
el::Loggers::addFlag(el::LoggingFlag::DisableApplicationAbortOnFatalLog);
|
||||
|
||||
// set delete_exceeds = 0 means disable throw away log file even they reach certain limit.
|
||||
if (log_rotate_num != 0) {
|
||||
fiu_do_on("LogMgr.Rotate.delete_exceeds_small_than_min", log_rotate_num = LOG_ROTATE_NUM_MIN - 1);
|
||||
if (log_rotate_num < LOG_ROTATE_NUM_MIN || log_rotate_num > LOG_ROTATE_NUM_MAX) {
|
||||
std::string msg = "log_rotate_num must in range[" + std::to_string(LOG_ROTATE_NUM_MIN) + ", " +
|
||||
std::to_string(LOG_ROTATE_NUM_MAX) + "], now is " + std::to_string(log_rotate_num);
|
||||
throw std::runtime_error(msg);
|
||||
}
|
||||
|
||||
/* global variable */
|
||||
enable_log_delete = true;
|
||||
logs_delete_exceeds = log_rotate_num;
|
||||
}
|
||||
|
||||
return *this;
|
||||
}
|
||||
|
||||
void
|
||||
LogMgr::Setup() {
|
||||
el::Loggers::reconfigureLogger("default", el_config_);
|
||||
}
|
||||
|
||||
void
|
||||
LogMgr::rename_and_delete(const std::string& filename, int64_t idx) {
|
||||
std::string target_filename = filename + "." + std::to_string(idx);
|
||||
rename(filename.c_str(), target_filename.c_str());
|
||||
if (enable_log_delete && idx - logs_delete_exceeds > 0) {
|
||||
std::string to_delete = filename + "." + std::to_string(trace_idx - logs_delete_exceeds);
|
||||
boost::filesystem::remove(to_delete);
|
||||
}
|
||||
}
|
||||
|
||||
std::unordered_map<std::string, bool>
|
||||
LogMgr::parse_level(const std::string& level) {
|
||||
std::unordered_map<std::string, bool> enables{
|
||||
{"debug", false}, {"info", false}, {"warning", false}, {"error", false}, {"fatal", false},
|
||||
};
|
||||
|
||||
std::unordered_map<std::string, int64_t> level_to_int{
|
||||
{"debug", 5}, {"info", 4}, {"warning", 3}, {"error", 2}, {"fatal", 1},
|
||||
};
|
||||
|
||||
switch (level_to_int[level]) {
|
||||
case 5:
|
||||
enables["debug"] = true;
|
||||
case 4:
|
||||
enables["info"] = true;
|
||||
case 3:
|
||||
enables["warning"] = true;
|
||||
case 2:
|
||||
enables["error"] = true;
|
||||
case 1:
|
||||
enables["fatal"] = true;
|
||||
break;
|
||||
default: {
|
||||
std::string msg = "Cannot parse level " + level +
|
||||
": invalid log level, must be one of debug, info, warning, error, fatal.";
|
||||
throw std::runtime_error(msg);
|
||||
}
|
||||
}
|
||||
return enables;
|
||||
}
|
||||
|
||||
void
|
||||
LogMgr::enable(el::Configurations& default_conf, el::Level level, bool enable) {
|
||||
std::string enable_str = enable ? "true" : "false";
|
||||
default_conf.set(level, el::ConfigurationType::Enabled, enable_str);
|
||||
}
|
||||
|
||||
} // namespace milvus
|
||||
|
|
@ -1,95 +0,0 @@
|
|||
// Copyright (C) 2019-2020 Zilliz. All rights reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software distributed under the License
|
||||
// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
|
||||
// or implied. See the License for the specific language governing permissions and limitations under the License.
|
||||
|
||||
#pragma once
|
||||
|
||||
#include "easyloggingpp/easylogging++.h"
|
||||
#include "utils/Status.h"
|
||||
#include "value/config/ServerConfig.h"
|
||||
|
||||
#include <sstream>
|
||||
#include <string>
|
||||
#include <unordered_map>
|
||||
|
||||
namespace milvus {
|
||||
|
||||
class LogMgr {
|
||||
public:
|
||||
static Status
|
||||
InitLog(bool trace_enable,
|
||||
const std::string& level,
|
||||
const std::string& logs_path,
|
||||
const std::string& filename,
|
||||
int64_t max_log_file_size,
|
||||
int64_t delete_exceeds,
|
||||
bool log_to_stdout,
|
||||
bool log_to_file);
|
||||
|
||||
static void
|
||||
RolloutHandler(const char* filename, std::size_t size, el::Level level);
|
||||
|
||||
private:
|
||||
explicit LogMgr(std::string log_path);
|
||||
|
||||
LogMgr&
|
||||
Default();
|
||||
|
||||
LogMgr&
|
||||
Filename(const std::string& filename);
|
||||
|
||||
/* Non-const for fiu to injecting error */
|
||||
LogMgr&
|
||||
Level(std::unordered_map<std::string, bool>& enables);
|
||||
|
||||
LogMgr&
|
||||
To(bool log_to_stdout, bool log_to_file);
|
||||
|
||||
LogMgr&
|
||||
Rotate(int64_t max_log_file_size, int64_t log_rotate_num);
|
||||
|
||||
void
|
||||
Setup();
|
||||
|
||||
private:
|
||||
static void
|
||||
rename_and_delete(const std::string& filename, int64_t idx);
|
||||
|
||||
static std::unordered_map<std::string, bool>
|
||||
parse_level(const std::string& level);
|
||||
|
||||
/**
|
||||
* @brief Configures if output corresponding level log
|
||||
*/
|
||||
static void
|
||||
enable(el::Configurations& default_conf, el::Level level, bool enable);
|
||||
|
||||
private:
|
||||
el::Configurations el_config_;
|
||||
std::string logs_path_;
|
||||
|
||||
private:
|
||||
static int trace_idx;
|
||||
static int global_idx;
|
||||
static int debug_idx;
|
||||
static int info_idx;
|
||||
static int warning_idx;
|
||||
static int error_idx;
|
||||
static int fatal_idx;
|
||||
static int64_t logs_delete_exceeds;
|
||||
static bool enable_log_delete;
|
||||
|
||||
const int64_t MAX_LOG_FILE_SIZE_MIN = 536870912; /* 512 MB */
|
||||
const int64_t MAX_LOG_FILE_SIZE_MAX = 4294967296; /* 4 GB */
|
||||
const int64_t LOG_ROTATE_NUM_MIN = 0;
|
||||
const int64_t LOG_ROTATE_NUM_MAX = 1024;
|
||||
};
|
||||
|
||||
} // namespace milvus
|
||||
|
|
@ -40,7 +40,6 @@ target_link_libraries(milvus_segcore
|
|||
milvus_common
|
||||
milvus_query
|
||||
milvus_utils
|
||||
milvus_storage
|
||||
milvus_config
|
||||
)
|
||||
|
||||
|
|
|
|||
|
|
@ -1,17 +0,0 @@
|
|||
# Copyright (C) 2019-2020 Zilliz. All rights reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software distributed under the License
|
||||
# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
|
||||
# or implied. See the License for the specific language governing permissions and limitations under the License
|
||||
|
||||
set(MILVUS_STORAGE_SRCS
|
||||
StorageCache.cpp
|
||||
)
|
||||
|
||||
add_library(milvus_storage ${MILVUS_STORAGE_SRCS})
|
||||
target_link_libraries(milvus_storage milvus_proto milvus_utils knowhere boost_bitset_ext)
|
||||
|
|
@ -1,12 +0,0 @@
|
|||
// Copyright (C) 2019-2020 Zilliz. All rights reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software distributed under the License
|
||||
// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
|
||||
// or implied. See the License for the specific language governing permissions and limitations under the License
|
||||
|
||||
#include "storage/StorageCache.h"
|
||||
|
|
@ -1,32 +0,0 @@
|
|||
// Copyright (C) 2019-2020 Zilliz. All rights reserved.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
|
||||
// with the License. You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software distributed under the License
|
||||
// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
|
||||
// or implied. See the License for the specific language governing permissions and limitations under the License
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <optional>
|
||||
#include "common/Types.h"
|
||||
|
||||
namespace milvus::storage {
|
||||
|
||||
// struct DataBlock {
|
||||
// // TODO
|
||||
// };
|
||||
|
||||
// struct BlockId {
|
||||
// int64_t segment_id_;
|
||||
// FieldId field_id_;
|
||||
// };
|
||||
|
||||
// class StorageCacheEngine {
|
||||
// // TODO
|
||||
// };
|
||||
|
||||
} // namespace milvus::storage
|
||||
Loading…
Reference in New Issue