milvus-io
/
milvus
mirror of https://github.com/milvus-io/milvus.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

fix(db): fix mem runtime error 

Former-commit-id: c12e6e99c8c7de1b62c2d2e7f9fecf3775715326
pull/191/head
Xu Peng 2019-04-15 20:44:11 +08:00 committed by xj.lin
parent 8eebbf65b4
commit 803686b8c6
2 changed files with 25 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
cpp/src/db/memvectors.cpp
View File

@ -2,6 +2,8 @@
#include <faiss/MetaIndexes.h> #include <faiss/MetaIndexes.h>
#include <faiss/index_io.h> #include <faiss/index_io.h>
#include <iostream> #include <iostream>
#include <sstream>
#include <thread>
#include "memvectors.h" #include "memvectors.h"
#include "db_meta.h" #include "db_meta.h"
@ -12,7 +14,7 @@ namespace vecwise {
namespace engine { namespace engine {
MemVectors::MemVectors(size_t dimension_, const std::string& file_location_) : MemVectors::MemVectors(size_t dimension_, const std::string& file_location_) :
_file_location(file_location_.c_str()), _file_location(file_location_),
_pIdGenerator(new SimpleIDGenerator()), _pIdGenerator(new SimpleIDGenerator()),
_dimension(dimension_), _dimension(dimension_),
_pInnerIndex(new faiss::IndexFlat(_dimension)), _pInnerIndex(new faiss::IndexFlat(_dimension)),
@ -20,8 +22,11 @@ MemVectors::MemVectors(size_t dimension_, const std::string& file_location_) :
} }
void MemVectors::add(size_t n_, const float* vectors_, IDNumbers& vector_ids_) { void MemVectors::add(size_t n_, const float* vectors_, IDNumbers& vector_ids_) {
vector_ids_ = _pIdGenerator->getNextIDNumbers(n_); _pIdGenerator->getNextIDNumbers(n_, vector_ids_);
_pIdMapIndex->add_with_ids(n_, vectors_, &vector_ids_[0]); _pIdMapIndex->add_with_ids(n_, vectors_, &vector_ids_[0]);
for(auto i=0 ; i<n_; i++) {
vector_ids_.push_back(i);
}
} }
size_t MemVectors::total() const { size_t MemVectors::total() const {
@ -33,7 +38,12 @@ size_t MemVectors::approximate_size() const {
} }
void MemVectors::serialize() { void MemVectors::serialize() {
faiss::write_index(_pIdMapIndex, _file_location); /* std::stringstream ss; */
/* ss << "/tmp/test/" << _pIdGenerator->getNextIDNumber(); */
/* faiss::write_index(_pIdMapIndex, ss.str().c_str()); */
/* std::cout << _pIdMapIndex->ntotal << std::endl; */
/* std::cout << _file_location << std::endl; */
faiss::write_index(_pIdMapIndex, _file_location.c_str());
} }
MemVectors::~MemVectors() { MemVectors::~MemVectors() {
@ -66,6 +76,7 @@ VectorsPtr MemManager::get_mem_by_group(const std::string& group_id) {
if (!status.ok()) { if (!status.ok()) {
return nullptr; return nullptr;
} }
_memMap[group_id] = std::shared_ptr<MemVectors>(new MemVectors(group_info.dimension, _memMap[group_id] = std::shared_ptr<MemVectors>(new MemVectors(group_info.dimension,
group_info.next_file_location)); group_info.next_file_location));
return _memMap[group_id]; return _memMap[group_id];
@ -83,7 +94,7 @@ Status MemManager::add_vectors_no_lock(const std::string& group_id,
size_t n, size_t n,
const float* vectors, const float* vectors,
IDNumbers& vector_ids) { IDNumbers& vector_ids) {
auto mem = get_mem_by_group(group_id); std::shared_ptr<MemVectors> mem = get_mem_by_group(group_id);
if (mem == nullptr) { if (mem == nullptr) {
return Status::NotFound("Group " + group_id + " not found!"); return Status::NotFound("Group " + group_id + " not found!");
} }
@ -97,6 +108,7 @@ Status MemManager::mark_memory_as_immutable() {
for (auto& kv: _memMap) { for (auto& kv: _memMap) {
_immMems.push_back(kv.second); _immMems.push_back(kv.second);
} }
_memMap.clear(); _memMap.clear();
return Status::OK(); return Status::OK();
} }

12
cpp/src/db/memvectors.h
View File

@ -33,8 +33,14 @@ public:
~MemVectors(); ~MemVectors();
const std::string& location() const { return _file_location; }
private: private:
const char* _file_location; MemVectors() = delete;
MemVectors(const MemVectors&) = delete;
MemVectors& operator=(const MemVectors&) = delete;
const std::string _file_location;
IDGenerator* _pIdGenerator; IDGenerator* _pIdGenerator;
size_t _dimension; size_t _dimension;
faiss::Index* _pInnerIndex; faiss::Index* _pInnerIndex;
@ -49,7 +55,7 @@ typedef std::shared_ptr<MemVectors> VectorsPtr;
class MemManager { class MemManager {
public: public:
MemManager(const std::shared_ptr<Meta>& meta_) MemManager(const std::shared_ptr<Meta>& meta_)
: _pMeta(meta_), _last_compact_time(std::time(nullptr)) {} : _pMeta(meta_) /*_last_compact_time(std::time(nullptr))*/ {}
VectorsPtr get_mem_by_group(const std::string& group_id_); VectorsPtr get_mem_by_group(const std::string& group_id_);
@ -68,7 +74,7 @@ private:
MemMap _memMap; MemMap _memMap;
ImmMemPool _immMems; ImmMemPool _immMems;
std::shared_ptr<Meta> _pMeta; std::shared_ptr<Meta> _pMeta;
std::time_t _last_compact_time; /* std::time_t _last_compact_time; */
std::mutex _mutex; std::mutex _mutex;
}; // MemManager }; // MemManager