fix(db): fix compile error

Former-commit-id: dbf6519efd3a0bac6f8ef2c617fcc08b879543f4
pull/191/head
Xu Peng 2019-04-15 15:45:33 +08:00 committed by xj.lin
parent 424b938c82
commit ff120dba16
5 changed files with 34 additions and 26 deletions

View File

@ -1,19 +1,21 @@
#include <IndexFlat.h> #include <faiss/IndexFlat.h>
#include <MetaIndexes.h> #include <faiss/MetaIndexes.h>
#include <index_io.h> #include <faiss/index_io.h>
#include "memvectors.h" #include "memvectors.h"
#include "db_meta.h" #include "db_meta.h"
namespace vecengine { namespace zilliz {
namespace vecwise {
namespace engine {
MemVectors::MemVectors(size_t dimension_, const std::string& file_location_) : MemVectors::MemVectors(size_t dimension_, const std::string& file_location_) :
_file_location(file_location_), _file_location(file_location_.c_str()),
_pIdGenerator(new SimpleIDGenerator()), _pIdGenerator(new SimpleIDGenerator()),
_dimension(dimension_), _dimension(dimension_),
_pInnerIndex(new faiss::IndexFlat(_dimension)), _pInnerIndex(new faiss::IndexFlat(_dimension)),
_pIdMapIndex = new faiss::IndexIDMap(_pInnerIndex) { _pIdMapIndex(new faiss::IndexIDMap(_pInnerIndex)) {
} }
void MemVectors::add(size_t n_, const float* vectors_, IDNumbers& vector_ids_) { void MemVectors::add(size_t n_, const float* vectors_, IDNumbers& vector_ids_) {
@ -52,14 +54,14 @@ MemVectors::~MemVectors() {
* MemManager * MemManager
*/ */
VectorsPtr MemManager::get_mem_by_group(const std::string& group_id_) { VectorsPtr MemManager::get_mem_by_group(const std::string& group_id) {
auto memIt = _memMap.find(group_id_); auto memIt = _memMap.find(group_id);
if memIt != _memMap.end() { if (memIt != _memMap.end()) {
return &(memIt->second); return memIt->second;
} }
GroupSchema group_info; GroupSchema group_info;
Status status = _pMeta->get_group(group_id_, group_info); Status status = _pMeta->get_group(group_id, group_info);
if (!status.ok()) { if (!status.ok()) {
return nullptr; return nullptr;
} }
@ -76,15 +78,17 @@ Status MemManager::add_vectors(const std::string& group_id_,
return add_vectors_no_lock(group_id_, n_, vectors_, vector_ids_); return add_vectors_no_lock(group_id_, n_, vectors_, vector_ids_);
} }
Status MemManager::add_vectors_no_lock(const std::string& group_id_, Status MemManager::add_vectors_no_lock(const std::string& group_id,
size_t n, size_t n,
const float* vectors, const float* vectors,
IDNumbers& vector_ids_) { IDNumbers& vector_ids) {
auto mem = get_mem_by_group(group_id_); auto mem = get_mem_by_group(group_id);
if (mem == nullptr) { if (mem == nullptr) {
return Status::NotFound("Group " + group_id_ " not found!"); return Status::NotFound("Group " + group_id + " not found!");
} }
return mem->add(n, vectors, vector_ids_); mem->add(n, vectors, vector_ids);
return Status::OK();
} }
Status MemManager::mark_memory_as_immutable() { Status MemManager::mark_memory_as_immutable() {
@ -111,11 +115,13 @@ Status MemManager::mark_memory_as_immutable() {
Status MemManager::serialize() { Status MemManager::serialize() {
mark_memory_as_immutable(); mark_memory_as_immutable();
for (auto& mem : _immMems) { for (auto& mem : _immMems) {
mem->serialize() mem->serialize();
} }
_immMems.clear(); _immMems.clear();
/* _last_compact_time = std::time(nullptr); */ /* _last_compact_time = std::time(nullptr); */
} }
} // namespace vecengine } // namespace engine
} // namespace vecwise
} // namespace zilliz

View File

@ -4,6 +4,8 @@
#include <map> #include <map>
#include <string> #include <string>
#include <ctime> #include <ctime>
#include <memory>
#include <mutex>
#include "id_generators.h" #include "id_generators.h"
#include "status.h" #include "status.h"
@ -21,7 +23,7 @@ class MemVectors {
public: public:
explicit MemVectors(size_t dimension_, const std::string& file_location_); explicit MemVectors(size_t dimension_, const std::string& file_location_);
IDNumbers&& add(size_t n, const float* vectors); void add(size_t n_, const float* vectors_, IDNumbers& vector_ids_);
size_t total() const; size_t total() const;
@ -32,7 +34,7 @@ public:
~MemVectors(); ~MemVectors();
private: private:
std::string _file_location; const char* _file_location;
IDGenerator* _pIdGenerator; IDGenerator* _pIdGenerator;
size_t _dimension; size_t _dimension;
faiss::Index* _pInnerIndex; faiss::Index* _pInnerIndex;
@ -42,10 +44,10 @@ private:
class Meta; class Meta;
typedef std::shared_ptr<MemVectors> VectorsPtr;
class MemManager { class MemManager {
public: public:
typedef std::shared_ptr<MemVectors> VectorsPtr;
MemManager(const std::shared_ptr<Meta>& meta_) MemManager(const std::shared_ptr<Meta>& meta_)
: _pMeta(meta_), _last_compact_time(std::time(nullptr)) {} : _pMeta(meta_), _last_compact_time(std::time(nullptr)) {}
@ -59,6 +61,7 @@ public:
private: private:
Status add_vectors_no_lock(const std::string& group_id_, Status add_vectors_no_lock(const std::string& group_id_,
size_t n_, const float* vectors_, IDNumbers& vector_ids_); size_t n_, const float* vectors_, IDNumbers& vector_ids_);
Status mark_memory_as_immutable();
typedef std::map<std::string, VectorsPtr> MemMap; typedef std::map<std::string, VectorsPtr> MemMap;
typedef std::vector<VectorsPtr> ImmMemPool; typedef std::vector<VectorsPtr> ImmMemPool;

View File

@ -1,4 +1,5 @@
#include "options.h" #include "options.h"
#include "env.h"
namespace zilliz { namespace zilliz {
namespace vecwise { namespace vecwise {

View File

@ -1,5 +1,4 @@
#ifndef VECENGINE_OPTIONS_H_ #pragma once
#define VECENGINE_OPTIONS_H_
#include <string> #include <string>
#include <memory> #include <memory>
@ -12,6 +11,7 @@ class MetaOptions;
class Env; class Env;
struct Options { struct Options {
Options();
uint16_t memory_sync_interval = 10; uint16_t memory_sync_interval = 10;
uint16_t raw_file_merge_trigger_number = 100; uint16_t raw_file_merge_trigger_number = 100;
size_t raw_to_index_trigger_size = 100000; size_t raw_to_index_trigger_size = 100000;
@ -39,5 +39,3 @@ struct DBMetaOptions : public MetaOptions {
} // namespace engine } // namespace engine
} // namespace vecwise } // namespace vecwise
} // namespace zilliz } // namespace zilliz
#endif // VECENGINE_OPTIONS_H_

View File

@ -6,7 +6,7 @@ namespace zilliz {
namespace vecwise { namespace vecwise {
namespace engine { namespace engine {
typedef uint64_t IDNumber; typedef long IDNumber;
typedef IDNumber* IDNumberPtr; typedef IDNumber* IDNumberPtr;
typedef std::vector<IDNumber> IDNumbers; typedef std::vector<IDNumber> IDNumbers;