milvus-io
/
milvus
mirror of https://github.com/milvus-io/milvus.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

redefine id mapper interface 

Former-commit-id: 6fa296aa09aae1019ed1e49a481c3a1e34440071
pull/191/head
groot 2019-04-23 16:34:43 +08:00
parent 82fe888083
commit f642a7b38e
4 changed files with 59 additions and 74 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

59
cpp/src/server/VecIdMapper.cpp
View File

@ -40,28 +40,24 @@ SimpleIdMapper::~SimpleIdMapper() {
}
ServerError SimpleIdMapper::Put(INTEGER_ID nid, const std::string& sid) {
ServerError SimpleIdMapper::Put(const std::string& nid, const std::string& sid) {
ids_[nid] = sid;
return SERVER_SUCCESS;
}
ServerError SimpleIdMapper::Put(const std::vector<INTEGER_ID>& nid, const std::vector<std::string>& sid) {
return Put(nid.data(), nid.size(), sid);
}
ServerError SimpleIdMapper::Put(const INTEGER_ID *nid, uint64_t count, const std::vector<std::string>& sid) {
if(count != sid.size()) {
ServerError SimpleIdMapper::Put(const std::vector<std::string>& nid, const std::vector<std::string>& sid) {
if(nid.size() != sid.size()) {
return SERVER_INVALID_ARGUMENT;
}
for(int64_t i = 0; i < count; i++) {
for(size_t i = 0; i < nid.size(); i++) {
ids_[nid[i]] = sid[i];
}
return SERVER_SUCCESS;
}
ServerError SimpleIdMapper::Get(INTEGER_ID nid, std::string& sid) const {
ServerError SimpleIdMapper::Get(const std::string& nid, std::string& sid) const {
auto iter = ids_.find(nid);
if(iter == ids_.end()) {
return SERVER_INVALID_ARGUMENT;
@ -72,15 +68,11 @@ ServerError SimpleIdMapper::Get(INTEGER_ID nid, std::string& sid) const {
return SERVER_SUCCESS;
}
ServerError SimpleIdMapper::Get(const std::vector<INTEGER_ID>& nid, std::vector<std::string>& sid) const {
return Get(nid.data(), nid.size(), sid);
}
ServerError SimpleIdMapper::Get(const INTEGER_ID *nid, uint64_t count, std::vector<std::string>& sid) const {
ServerError SimpleIdMapper::Get(const std::vector<std::string>& nid, std::vector<std::string>& sid) const {
sid.clear();
ServerError err = SERVER_SUCCESS;
for(uint64_t i = 0; i < count; i++) {
for(size_t i = 0; i < nid.size(); i++) {
auto iter = ids_.find(nid[i]);
if(iter == ids_.end()) {
sid.push_back("");
@ -95,7 +87,7 @@ ServerError SimpleIdMapper::Get(const INTEGER_ID *nid, uint64_t count, std::vect
return err;
}
ServerError SimpleIdMapper::Delete(INTEGER_ID nid) {
ServerError SimpleIdMapper::Delete(const std::string& nid) {
ids_.erase(nid);
return SERVER_SUCCESS;
}
@ -124,13 +116,12 @@ RocksIdMapper::~RocksIdMapper() {
}
}
ServerError RocksIdMapper::Put(INTEGER_ID nid, const std::string& sid) {
ServerError RocksIdMapper::Put(const std::string& nid, const std::string& sid) {
if(db_ == nullptr) {
return SERVER_NULL_POINTER;
}
std::string str_id = std::to_string(nid);//NOTE: keep a local virible here, since the Slice require a char* pointer
rocksdb::Slice key(str_id);
rocksdb::Slice key(nid);
rocksdb::Slice value(sid);
rocksdb::Status s = db_->Put(rocksdb::WriteOptions(), key, value);
if(!s.ok()) {
@ -141,33 +132,28 @@ ServerError RocksIdMapper::Put(INTEGER_ID nid, const std::string& sid) {
return SERVER_SUCCESS;
}
ServerError RocksIdMapper::Put(const std::vector<INTEGER_ID>& nid, const std::vector<std::string>& sid) {
return Put(nid.data(), nid.size(), sid);
}
ServerError RocksIdMapper::Put(const INTEGER_ID *nid, uint64_t count, const std::vector<std::string>& sid) {
if(count != sid.size()) {
ServerError RocksIdMapper::Put(const std::vector<std::string>& nid, const std::vector<std::string>& sid) {
if(nid.size() != sid.size()) {
return SERVER_INVALID_ARGUMENT;
}
ServerError err = SERVER_SUCCESS;
for(int64_t i = 0; i < count; i++) {
for(size_t i = 0; i < nid.size(); i++) {
err = Put(nid[i], sid[i]);
if(err != SERVER_SUCCESS) {
return err;
}
}
return SERVER_SUCCESS;
return err;
}
ServerError RocksIdMapper::Get(INTEGER_ID nid, std::string& sid) const {
ServerError RocksIdMapper::Get(const std::string& nid, std::string& sid) const {
if(db_ == nullptr) {
return SERVER_NULL_POINTER;
}
std::string str_id = std::to_string(nid);//NOTE: keep a local virible here, since the Slice require a char* pointer
rocksdb::Slice key(str_id);
rocksdb::Slice key(nid);
rocksdb::Status s = db_->Get(rocksdb::ReadOptions(), key, &sid);
if(!s.ok()) {
SERVER_LOG_ERROR << "ID mapper failed to get:" << s.ToString();
@ -177,15 +163,11 @@ ServerError RocksIdMapper::Get(INTEGER_ID nid, std::string& sid) const {
return SERVER_SUCCESS;
}
ServerError RocksIdMapper::Get(const std::vector<INTEGER_ID>& nid, std::vector<std::string>& sid) const {
return Get(nid.data(), nid.size(), sid);
}
ServerError RocksIdMapper::Get(const INTEGER_ID *nid, uint64_t count, std::vector<std::string>& sid) const {
ServerError RocksIdMapper::Get(const std::vector<std::string>& nid, std::vector<std::string>& sid) const {
sid.clear();
ServerError err = SERVER_SUCCESS;
for(uint64_t i = 0; i < count; i++) {
for(size_t i = 0; i < nid.size(); i++) {
std::string str_id;
ServerError temp_err = Get(nid[i], str_id);
if(temp_err != SERVER_SUCCESS) {
@ -201,13 +183,12 @@ ServerError RocksIdMapper::Get(const INTEGER_ID *nid, uint64_t count, std::vecto
return err;
}
ServerError RocksIdMapper::Delete(INTEGER_ID nid) {
ServerError RocksIdMapper::Delete(const std::string& nid) {
if(db_ == nullptr) {
return SERVER_NULL_POINTER;
}
std::string str_id = std::to_string(nid);//NOTE: keep a local virible here, since the Slice require a char* pointer
rocksdb::Slice key(str_id);
rocksdb::Slice key(nid);
rocksdb::Status s = db_->Delete(rocksdb::WriteOptions(), key);
if(!s.ok()) {
SERVER_LOG_ERROR << "ID mapper failed to delete:" << s.ToString();

41
cpp/src/server/VecIdMapper.h
View File

@ -19,23 +19,20 @@ namespace zilliz {
namespace vecwise {
namespace server {
using INTEGER_ID = int64_t;
class IVecIdMapper {
public:
static IVecIdMapper* GetInstance();
virtual ~IVecIdMapper(){}
virtual ServerError Put(INTEGER_ID nid, const std::string& sid) = 0;
virtual ServerError Put(const std::vector<INTEGER_ID>& nid, const std::vector<std::string>& sid) = 0;
virtual ServerError Put(const INTEGER_ID *nid, uint64_t count, const std::vector<std::string>& sid) = 0;
virtual ServerError Put(const std::string& nid, const std::string& sid) = 0;
virtual ServerError Put(const std::vector<std::string>& nid, const std::vector<std::string>& sid) = 0;
virtual ServerError Get(INTEGER_ID nid, std::string& sid) const = 0;
virtual ServerError Get(const std::vector<INTEGER_ID>& nid, std::vector<std::string>& sid) const = 0;
virtual ServerError Get(const INTEGER_ID *nid, uint64_t count, std::vector<std::string>& sid) const = 0;
virtual ServerError Get(const std::string& nid, std::string& sid) const = 0;
//NOTE: the 'sid' will be cleared at begin of the function
virtual ServerError Get(const std::vector<std::string>& nid, std::vector<std::string>& sid) const = 0;
virtual ServerError Delete(INTEGER_ID nid) = 0;
virtual ServerError Delete(const std::string& nid) = 0;
};
class SimpleIdMapper : public IVecIdMapper{
@ -43,18 +40,16 @@ public:
SimpleIdMapper();
~SimpleIdMapper();
ServerError Put(INTEGER_ID nid, const std::string& sid) override;
ServerError Put(const std::vector<INTEGER_ID>& nid, const std::vector<std::string>& sid) override;
ServerError Put(const INTEGER_ID *nid, uint64_t count, const std::vector<std::string>& sid) override;
ServerError Put(const std::string& nid, const std::string& sid) override;
ServerError Put(const std::vector<std::string>& nid, const std::vector<std::string>& sid) override;
ServerError Get(INTEGER_ID nid, std::string& sid) const override;
ServerError Get(const std::vector<INTEGER_ID>& nid, std::vector<std::string>& sid) const override;
ServerError Get(const INTEGER_ID *nid, uint64_t count, std::vector<std::string>& sid) const override;
ServerError Get(const std::string& nid, std::string& sid) const override;
ServerError Get(const std::vector<std::string>& nid, std::vector<std::string>& sid) const override;
ServerError Delete(INTEGER_ID nid) override;
ServerError Delete(const std::string& nid) override;
private:
std::unordered_map<INTEGER_ID, std::string> ids_;
std::unordered_map<std::string, std::string> ids_;
};
class RocksIdMapper : public IVecIdMapper{
@ -62,15 +57,13 @@ public:
RocksIdMapper(const std::string& store_path);
~RocksIdMapper();
ServerError Put(INTEGER_ID nid, const std::string& sid) override;
ServerError Put(const std::vector<INTEGER_ID>& nid, const std::vector<std::string>& sid) override;
ServerError Put(const INTEGER_ID *nid, uint64_t count, const std::vector<std::string>& sid) override;
ServerError Put(const std::string& nid, const std::string& sid) override;
ServerError Put(const std::vector<std::string>& nid, const std::vector<std::string>& sid) override;
ServerError Get(INTEGER_ID nid, std::string& sid) const override;
ServerError Get(const std::vector<INTEGER_ID>& nid, std::vector<std::string>& sid) const override;
ServerError Get(const INTEGER_ID *nid, uint64_t count, std::vector<std::string>& sid) const override;
ServerError Get(const std::string& nid, std::string& sid) const override;
ServerError Get(const std::vector<std::string>& nid, std::vector<std::string>& sid) const override;
ServerError Delete(INTEGER_ID nid) override;
ServerError Delete(const std::string& nid) override;
private:
rocksdb::DB* db_;

20
cpp/src/server/VecServiceHandler.cpp
View File

@ -100,7 +100,8 @@ VecServiceHandler::add_vector(const std::string &group_id, const VecTensor &tens
if(vector_ids.size() != 1) {
SERVER_LOG_ERROR << "Vector ID not returned";
} else {
IVecIdMapper::GetInstance()->Put(vector_ids[0], tensor.uid);
std::string nid = group_id + "_" + std::to_string(vector_ids[0]);
IVecIdMapper::GetInstance()->Put(nid, tensor.uid);
}
}
@ -131,8 +132,10 @@ VecServiceHandler::add_vector_batch(const std::string &group_id,
if(vector_ids.size() != tensor_list.tensor_list.size()) {
SERVER_LOG_ERROR << "Vector ID not returned";
} else {
std::string nid_prefix = group_id + "_";
for(size_t i = 0; i < vector_ids.size(); i++) {
IVecIdMapper::GetInstance()->Put(vector_ids[i], tensor_list.tensor_list[i].uid);
std::string nid = nid_prefix + std::to_string(vector_ids[i]);
IVecIdMapper::GetInstance()->Put(nid, tensor_list.tensor_list[i].uid);
}
}
}
@ -163,9 +166,11 @@ VecServiceHandler::search_vector(VecSearchResult &_return,
SERVER_LOG_ERROR << "Engine failed: " << stat.ToString();
} else {
if(!results.empty()) {
std::string nid_prefix = group_id + "_";
for(auto id : results[0]) {
std::string sid;
IVecIdMapper::GetInstance()->Get(id, sid);
std::string nid = nid_prefix + std::to_string(id);
IVecIdMapper::GetInstance()->Get(nid, sid);
_return.id_list.push_back(sid);
}
}
@ -201,7 +206,14 @@ VecServiceHandler::search_vector_batch(VecSearchResultList &_return,
} else {
for(engine::QueryResult& res : results){
VecSearchResult v_res;
IVecIdMapper::GetInstance()->Get(res.data(), res.size(), v_res.id_list);
std::string nid_prefix = group_id + "_";
for(auto id : results[0]) {
std::string sid;
std::string nid = nid_prefix + std::to_string(id);
IVecIdMapper::GetInstance()->Get(nid, sid);
v_res.id_list.push_back(sid);
}
_return.result_list.push_back(v_res);
}
}

13
cpp/unittest/server/idmapper_test.cpp
View File

@ -19,7 +19,7 @@ TEST(IdMapperTest, IDMAPPER_TEST) {
server::IVecIdMapper* mapper = server::IVecIdMapper::GetInstance();
std::vector<int64_t> nid = {1,50, 900, 10000};
std::vector<std::string> nid = {"1", "50", "900", "10000"};
std::vector<std::string> sid = {"one", "fifty", "nine zero zero", "many"};
server::ServerError err = mapper->Put(nid, sid);
ASSERT_EQ(err, server::SERVER_SUCCESS);
@ -35,22 +35,21 @@ TEST(IdMapperTest, IDMAPPER_TEST) {
}
std::string str_id;
err = mapper->Get(50, str_id);
err = mapper->Get(nid[1], str_id);
ASSERT_EQ(str_id, "fifty");
err = mapper->Delete(900);
err = mapper->Delete(nid[2]);
ASSERT_EQ(err, server::SERVER_SUCCESS);
err = mapper->Get(900, str_id);
err = mapper->Get(nid[2], str_id);
ASSERT_NE(err, server::SERVER_SUCCESS);
//performance?
//test performance
nid.clear();
sid.clear();
const int64_t count = 1000000;
for(int64_t i = 0; i < count; i++) {
nid.push_back(i+100000);
nid.push_back(std::to_string(i+100000));
sid.push_back("val_" + std::to_string(i));
}