milvus-io
/
milvus
mirror of https://github.com/milvus-io/milvus.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'mega/branch-0.3.0' into branch-0.3.0 

Former-commit-id: 925bf53361e59ad626892bfce3299d49f5a2560a
pull/191/head
Yang Xuan 2019-06-10 10:01:21 +08:00
parent c2beaa9296 a1eda477c9
commit 6b612ffcc7
59 changed files with 1957 additions and 4647 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
cpp/CHANGELOG.md
View File

@ -12,6 +12,8 @@ Please mark all change in change log and use the ticket from JIRA.
- MS-57 - Implement index load/search pipeline
- MS-56 - Add version information when server is started
- MS-64 - Different table can have different index type
- MS-52 - Return search score
## Task

2
cpp/src/CMakeLists.txt
View File

@ -85,7 +85,6 @@ set(third_party_libs
prometheus-cpp-pull
prometheus-cpp-core
civetweb
rocksdb
boost_system_static
boost_filesystem_static
boost_serialization_static
@ -165,7 +164,6 @@ endif ()
set(server_libs
vecwise_engine
librocksdb.a
libthrift.a
pthread
libyaml-cpp.a

10
cpp/src/db/DB.cpp
View File

@ -16,15 +16,7 @@ namespace engine {
DB::~DB() {}
void DB::Open(const Options& options, DB** dbptr) {
*dbptr = nullptr;
#ifdef GPU_VERSION
std::string default_index_type{"Faiss,IVF"};
#else
std::string default_index_type{"Faiss,IDMap"};
#endif
*dbptr = DBFactory::Build(options, default_index_type);
*dbptr = DBFactory::Build(options);
return;
}

296
cpp/src/db/DBImpl.inl → cpp/src/db/DBImpl.cpp
View File

@ -3,11 +3,10 @@
* Unauthorized copying of this file, via any medium is strictly prohibited.
* Proprietary and confidential.
******************************************************************************/
#pragma once
#include "DBImpl.h"
#include "DBMetaImpl.h"
#include "Env.h"
#include "EngineFactory.h"
#include "metrics/Metrics.h"
#include "scheduler/SearchScheduler.h"
@ -23,80 +22,107 @@ namespace zilliz {
namespace vecwise {
namespace engine {
namespace {
template<typename EngineT>
DBImpl<EngineT>::DBImpl(const Options& options)
: env_(options.env),
options_(options),
bg_compaction_scheduled_(false),
shutting_down_(false),
bg_build_index_started_(false),
pMeta_(new meta::DBMetaImpl(options_.meta)),
pMemMgr_(new MemManager<EngineT>(pMeta_, options_)) {
StartTimerTasks(options_.memory_sync_interval);
}
template<typename EngineT>
Status DBImpl<EngineT>::CreateTable(meta::TableSchema& table_schema) {
return pMeta_->CreateTable(table_schema);
}
template<typename EngineT>
Status DBImpl<EngineT>::DescribeTable(meta::TableSchema& table_schema) {
return pMeta_->DescribeTable(table_schema);
}
template<typename EngineT>
Status DBImpl<EngineT>::HasTable(const std::string& table_id, bool& has_or_not) {
return pMeta_->HasTable(table_id, has_or_not);
}
template<typename EngineT>
Status DBImpl<EngineT>::InsertVectors(const std::string& table_id_,
size_t n, const float* vectors, IDNumbers& vector_ids_) {
auto start_time = METRICS_NOW_TIME;
Status status = pMemMgr_->InsertVectors(table_id_, n, vectors, vector_ids_);
auto end_time = METRICS_NOW_TIME;
// std::chrono::microseconds time_span = std::chrono::duration_cast<std::chrono::microseconds>(end_time - start_time);
// double average_time = double(time_span.count()) / n;
double total_time = METRICS_MICROSECONDS(start_time,end_time);
void CollectInsertMetrics(double total_time, size_t n, bool succeed) {
double avg_time = total_time / n;
for (int i = 0; i < n; ++i) {
server::Metrics::GetInstance().AddVectorsDurationHistogramOberve(avg_time);
}
// server::Metrics::GetInstance().add_vector_duration_seconds_quantiles().Observe((average_time));
if (!status.ok()) {
if (succeed) {
server::Metrics::GetInstance().AddVectorsSuccessTotalIncrement(n);
server::Metrics::GetInstance().AddVectorsSuccessGaugeSet(n);
}
else {
server::Metrics::GetInstance().AddVectorsFailTotalIncrement(n);
server::Metrics::GetInstance().AddVectorsFailGaugeSet(n);
return status;
}
server::Metrics::GetInstance().AddVectorsSuccessTotalIncrement(n);
server::Metrics::GetInstance().AddVectorsSuccessGaugeSet(n);
}
template<typename EngineT>
Status DBImpl<EngineT>::Query(const std::string &table_id, size_t k, size_t nq,
void CollectQueryMetrics(double total_time, size_t nq) {
for (int i = 0; i < nq; ++i) {
server::Metrics::GetInstance().QueryResponseSummaryObserve(total_time);
}
auto average_time = total_time / nq;
server::Metrics::GetInstance().QueryVectorResponseSummaryObserve(average_time, nq);
server::Metrics::GetInstance().QueryVectorResponsePerSecondGaugeSet(double (nq) / total_time);
}
void CollectFileMetrics(int file_type, size_t file_size, double total_time) {
switch(file_type) {
case meta::TableFileSchema::RAW:
case meta::TableFileSchema::TO_INDEX: {
server::Metrics::GetInstance().SearchRawDataDurationSecondsHistogramObserve(total_time);
server::Metrics::GetInstance().RawFileSizeHistogramObserve(file_size);
server::Metrics::GetInstance().RawFileSizeTotalIncrement(file_size);
server::Metrics::GetInstance().RawFileSizeGaugeSet(file_size);
break;
}
default: {
server::Metrics::GetInstance().SearchIndexDataDurationSecondsHistogramObserve(total_time);
server::Metrics::GetInstance().IndexFileSizeHistogramObserve(file_size);
server::Metrics::GetInstance().IndexFileSizeTotalIncrement(file_size);
server::Metrics::GetInstance().IndexFileSizeGaugeSet(file_size);
break;
}
}
}
}
DBImpl::DBImpl(const Options& options)
: env_(options.env),
options_(options),
bg_compaction_scheduled_(false),
shutting_down_(false),
bg_build_index_started_(false),
pMeta_(new meta::DBMetaImpl(options_.meta)),
pMemMgr_(new MemManager(pMeta_, options_)) {
StartTimerTasks(options_.memory_sync_interval);
}
Status DBImpl::CreateTable(meta::TableSchema& table_schema) {
return pMeta_->CreateTable(table_schema);
}
Status DBImpl::DescribeTable(meta::TableSchema& table_schema) {
return pMeta_->DescribeTable(table_schema);
}
Status DBImpl::HasTable(const std::string& table_id, bool& has_or_not) {
return pMeta_->HasTable(table_id, has_or_not);
}
Status DBImpl::InsertVectors(const std::string& table_id_,
size_t n, const float* vectors, IDNumbers& vector_ids_) {
auto start_time = METRICS_NOW_TIME;
Status status = pMemMgr_->InsertVectors(table_id_, n, vectors, vector_ids_);
auto end_time = METRICS_NOW_TIME;
double total_time = METRICS_MICROSECONDS(start_time,end_time);
// std::chrono::microseconds time_span = std::chrono::duration_cast<std::chrono::microseconds>(end_time - start_time);
// double average_time = double(time_span.count()) / n;
CollectInsertMetrics(total_time, n, status.ok());
return status;
}
Status DBImpl::Query(const std::string &table_id, size_t k, size_t nq,
const float *vectors, QueryResults &results) {
auto start_time = METRICS_NOW_TIME;
meta::DatesT dates = {meta::Meta::GetDate()};
Status result = Query(table_id, k, nq, vectors, dates, results);
auto end_time = METRICS_NOW_TIME;
auto total_time = METRICS_MICROSECONDS(start_time,end_time);
auto average_time = total_time / nq;
for (int i = 0; i < nq; ++i) {
server::Metrics::GetInstance().QueryResponseSummaryObserve(total_time);
}
server::Metrics::GetInstance().QueryVectorResponseSummaryObserve(average_time, nq);
server::Metrics::GetInstance().QueryVectorResponsePerSecondGaugeSet(double (nq) / total_time);
CollectQueryMetrics(total_time, nq);
return result;
}
template<typename EngineT>
Status DBImpl<EngineT>::Query(const std::string& table_id, size_t k, size_t nq,
Status DBImpl::Query(const std::string& table_id, size_t k, size_t nq,
const float* vectors, const meta::DatesT& dates, QueryResults& results) {
#if 0
return QuerySync(table_id, k, nq, vectors, dates, results);
@ -105,8 +131,7 @@ Status DBImpl<EngineT>::Query(const std::string& table_id, size_t k, size_t nq,
#endif
}
template<typename EngineT>
Status DBImpl<EngineT>::QuerySync(const std::string& table_id, size_t k, size_t nq,
Status DBImpl::QuerySync(const std::string& table_id, size_t k, size_t nq,
const float* vectors, const meta::DatesT& dates, QueryResults& results) {
meta::DatePartionedTableFilesSchema files;
auto status = pMeta_->FilesToSearch(table_id, dates, files);
@ -118,16 +143,16 @@ Status DBImpl<EngineT>::QuerySync(const std::string& table_id, size_t k, size_t
meta::TableFilesSchema raw_files;
for (auto &day_files : files) {
for (auto &file : day_files.second) {
file.file_type == meta::TableFileSchema::INDEX ?
file.file_type_ == meta::TableFileSchema::INDEX ?
index_files.push_back(file) : raw_files.push_back(file);
}
}
int dim = 0;
if (!index_files.empty()) {
dim = index_files[0].dimension;
dim = index_files[0].dimension_;
} else if (!raw_files.empty()) {
dim = raw_files[0].dimension;
dim = raw_files[0].dimension_;
} else {
LOG(DEBUG) << "no files to search";
return Status::OK();
@ -160,38 +185,20 @@ Status DBImpl<EngineT>::QuerySync(const std::string& table_id, size_t k, size_t
auto search_in_index = [&](meta::TableFilesSchema& file_vec) -> void {
for (auto &file : file_vec) {
EngineT index(file.dimension, file.location);
index.Load();
auto file_size = index.PhysicalSize()/(1024*1024);
ExecutionEnginePtr index = EngineFactory::Build(file.dimension_, file.location_, (EngineType)file.engine_type_);
index->Load();
auto file_size = index->PhysicalSize();
search_set_size += file_size;
LOG(DEBUG) << "Search file_type " << file.file_type << " Of Size: "
<< file_size << " M";
LOG(DEBUG) << "Search file_type " << file.file_type_ << " Of Size: "
<< file_size/(1024*1024) << " M";
int inner_k = index.Count() < k ? index.Count() : k;
int inner_k = index->Count() < k ? index->Count() : k;
auto start_time = METRICS_NOW_TIME;
index.Search(nq, vectors, inner_k, output_distence, output_ids);
index->Search(nq, vectors, inner_k, output_distence, output_ids);
auto end_time = METRICS_NOW_TIME;
auto total_time = METRICS_MICROSECONDS(start_time, end_time);
if(file.file_type == meta::TableFileSchema::RAW) {
server::Metrics::GetInstance().SearchRawDataDurationSecondsHistogramObserve(total_time);
server::Metrics::GetInstance().RawFileSizeHistogramObserve(file_size*1024*1024);
server::Metrics::GetInstance().RawFileSizeTotalIncrement(file_size*1024*1024);
server::Metrics::GetInstance().RawFileSizeGaugeSet(file_size*1024*1024);
} else if(file.file_type == meta::TableFileSchema::TO_INDEX) {
server::Metrics::GetInstance().SearchRawDataDurationSecondsHistogramObserve(total_time);
server::Metrics::GetInstance().RawFileSizeHistogramObserve(file_size*1024*1024);
server::Metrics::GetInstance().RawFileSizeTotalIncrement(file_size*1024*1024);
server::Metrics::GetInstance().RawFileSizeGaugeSet(file_size*1024*1024);
} else {
server::Metrics::GetInstance().SearchIndexDataDurationSecondsHistogramObserve(total_time);
server::Metrics::GetInstance().IndexFileSizeHistogramObserve(file_size*1024*1024);
server::Metrics::GetInstance().IndexFileSizeTotalIncrement(file_size*1024*1024);
server::Metrics::GetInstance().IndexFileSizeGaugeSet(file_size*1024*1024);
}
CollectFileMetrics(file.file_type_, file_size, total_time);
cluster(output_ids, output_distence, inner_k); // cluster to each query
memset(output_distence, 0, k * nq * sizeof(float));
memset(output_ids, 0, k * nq * sizeof(long));
@ -233,7 +240,7 @@ Status DBImpl<EngineT>::QuerySync(const std::string& table_id, size_t k, size_t
int inner_k = dis.size() < k ? dis.size() : k;
for (int i = 0; i < inner_k; ++i) {
res.emplace_back(nns[output_ids[i]]); // mapping
res.emplace_back(std::make_pair(nns[output_ids[i]], output_distence[i])); // mapping
}
results.push_back(res); // append to result list
res.clear();
@ -258,9 +265,10 @@ Status DBImpl<EngineT>::QuerySync(const std::string& table_id, size_t k, size_t
return Status::OK();
}
template<typename EngineT>
Status DBImpl<EngineT>::QueryAsync(const std::string& table_id, size_t k, size_t nq,
Status DBImpl::QueryAsync(const std::string& table_id, size_t k, size_t nq,
const float* vectors, const meta::DatesT& dates, QueryResults& results) {
//step 1: get files to search
meta::DatePartionedTableFilesSchema files;
auto status = pMeta_->FilesToSearch(table_id, dates, files);
if (!status.ok()) { return status; }
@ -276,29 +284,25 @@ Status DBImpl<EngineT>::QueryAsync(const std::string& table_id, size_t k, size_t
}
}
//step 2: put search task to scheduler
SearchScheduler& scheduler = SearchScheduler::GetInstance();
scheduler.ScheduleSearchTask(context);
context->WaitResult();
//step 3: construct results
auto& context_result = context->GetResult();
for(auto& topk_result : context_result) {
QueryResult ids;
for(auto& pair : topk_result) {
ids.push_back(pair.second);
}
results.emplace_back(ids);
}
results.swap(context_result);
return Status::OK();
}
template<typename EngineT>
void DBImpl<EngineT>::StartTimerTasks(int interval) {
bg_timer_thread_ = std::thread(&DBImpl<EngineT>::BackgroundTimerTask, this, interval);
void DBImpl::StartTimerTasks(int interval) {
bg_timer_thread_ = std::thread(&DBImpl::BackgroundTimerTask, this, interval);
}
template<typename EngineT>
void DBImpl<EngineT>::BackgroundTimerTask(int interval) {
void DBImpl::BackgroundTimerTask(int interval) {
Status status;
while (true) {
if (!bg_error_.ok()) break;
@ -315,22 +319,19 @@ void DBImpl<EngineT>::BackgroundTimerTask(int interval) {
}
}
template<typename EngineT>
void DBImpl<EngineT>::TrySchedule() {
void DBImpl::TrySchedule() {
if (bg_compaction_scheduled_) return;
if (!bg_error_.ok()) return;
bg_compaction_scheduled_ = true;
env_->Schedule(&DBImpl<EngineT>::BGWork, this);
env_->Schedule(&DBImpl::BGWork, this);
}
template<typename EngineT>
void DBImpl<EngineT>::BGWork(void* db_) {
void DBImpl::BGWork(void* db_) {
reinterpret_cast<DBImpl*>(db_)->BackgroundCall();
}
template<typename EngineT>
void DBImpl<EngineT>::BackgroundCall() {
void DBImpl::BackgroundCall() {
std::lock_guard<std::mutex> lock(mutex_);
assert(bg_compaction_scheduled_);
@ -343,13 +344,11 @@ void DBImpl<EngineT>::BackgroundCall() {
bg_work_finish_signal_.notify_all();
}
template<typename EngineT>
Status DBImpl<EngineT>::MergeFiles(const std::string& table_id, const meta::DateT& date,
Status DBImpl::MergeFiles(const std::string& table_id, const meta::DateT& date,
const meta::TableFilesSchema& files) {
meta::TableFileSchema table_file;
table_file.table_id = table_id;
table_file.date = date;
table_file.table_id_ = table_id;
table_file.date_ = date;
Status status = pMeta_->CreateTableFile(table_file);
if (!status.ok()) {
@ -357,7 +356,8 @@ Status DBImpl<EngineT>::MergeFiles(const std::string& table_id, const meta::Date
return status;
}
EngineT index(table_file.dimension, table_file.location);
ExecutionEnginePtr index =
EngineFactory::Build(table_file.dimension_, table_file.location_, (EngineType)table_file.engine_type_);
meta::TableFilesSchema updated;
long index_size = 0;
@ -365,41 +365,40 @@ Status DBImpl<EngineT>::MergeFiles(const std::string& table_id, const meta::Date
for (auto& file : files) {
auto start_time = METRICS_NOW_TIME;
index.Merge(file.location);
index->Merge(file.location_);
auto file_schema = file;
auto end_time = METRICS_NOW_TIME;
auto total_time = METRICS_MICROSECONDS(start_time,end_time);
server::Metrics::GetInstance().MemTableMergeDurationSecondsHistogramObserve(total_time);
file_schema.file_type = meta::TableFileSchema::TO_DELETE;
file_schema.file_type_ = meta::TableFileSchema::TO_DELETE;
updated.push_back(file_schema);
LOG(DEBUG) << "Merging file " << file_schema.file_id;
index_size = index.Size();
LOG(DEBUG) << "Merging file " << file_schema.file_id_;
index_size = index->Size();
if (index_size >= options_.index_trigger_size) break;
}
index.Serialize();
index->Serialize();
if (index_size >= options_.index_trigger_size) {
table_file.file_type = meta::TableFileSchema::TO_INDEX;
table_file.file_type_ = meta::TableFileSchema::TO_INDEX;
} else {
table_file.file_type = meta::TableFileSchema::RAW;
table_file.file_type_ = meta::TableFileSchema::RAW;
}
table_file.size = index_size;
table_file.size_ = index_size;
updated.push_back(table_file);
status = pMeta_->UpdateTableFiles(updated);
LOG(DEBUG) << "New merged file " << table_file.file_id <<
" of size=" << index.PhysicalSize()/(1024*1024) << " M";
LOG(DEBUG) << "New merged file " << table_file.file_id_ <<
" of size=" << index->PhysicalSize()/(1024*1024) << " M";
index.Cache();
index->Cache();
return status;
}
template<typename EngineT>
Status DBImpl<EngineT>::BackgroundMergeFiles(const std::string& table_id) {
Status DBImpl::BackgroundMergeFiles(const std::string& table_id) {
meta::DatePartionedTableFilesSchema raw_files;
auto status = pMeta_->FilesToMerge(table_id, raw_files);
if (!status.ok()) {
@ -426,37 +425,36 @@ Status DBImpl<EngineT>::BackgroundMergeFiles(const std::string& table_id) {
return Status::OK();
}
template<typename EngineT>
Status DBImpl<EngineT>::BuildIndex(const meta::TableFileSchema& file) {
Status DBImpl::BuildIndex(const meta::TableFileSchema& file) {
meta::TableFileSchema table_file;
table_file.table_id = file.table_id;
table_file.date = file.date;
table_file.table_id_ = file.table_id_;
table_file.date_ = file.date_;
Status status = pMeta_->CreateTableFile(table_file);
if (!status.ok()) {
return status;
}
EngineT to_index(file.dimension, file.location);
ExecutionEnginePtr to_index = EngineFactory::Build(file.dimension_, file.location_, (EngineType)file.engine_type_);
to_index.Load();
to_index->Load();
auto start_time = METRICS_NOW_TIME;
auto index = to_index.BuildIndex(table_file.location);
auto index = to_index->BuildIndex(table_file.location_);
auto end_time = METRICS_NOW_TIME;
auto total_time = METRICS_MICROSECONDS(start_time, end_time);
server::Metrics::GetInstance().BuildIndexDurationSecondsHistogramObserve(total_time);
table_file.file_type = meta::TableFileSchema::INDEX;
table_file.size = index->Size();
table_file.file_type_ = meta::TableFileSchema::INDEX;
table_file.size_ = index->Size();
auto to_remove = file;
to_remove.file_type = meta::TableFileSchema::TO_DELETE;
to_remove.file_type_ = meta::TableFileSchema::TO_DELETE;
meta::TableFilesSchema update_files = {to_remove, table_file};
pMeta_->UpdateTableFiles(update_files);
LOG(DEBUG) << "New index file " << table_file.file_id << " of size "
LOG(DEBUG) << "New index file " << table_file.file_id_ << " of size "
<< index->PhysicalSize()/(1024*1024) << " M"
<< " from file " << to_remove.file_id;
<< " from file " << to_remove.file_id_;
index->Cache();
pMeta_->Archive();
@ -464,8 +462,7 @@ Status DBImpl<EngineT>::BuildIndex(const meta::TableFileSchema& file) {
return Status::OK();
}
template<typename EngineT>
void DBImpl<EngineT>::BackgroundBuildIndex() {
void DBImpl::BackgroundBuildIndex() {
std::lock_guard<std::mutex> lock(build_index_mutex_);
assert(bg_build_index_started_);
meta::TableFilesSchema to_index_files;
@ -485,18 +482,16 @@ void DBImpl<EngineT>::BackgroundBuildIndex() {
bg_build_index_finish_signal_.notify_all();
}
template<typename EngineT>
Status DBImpl<EngineT>::TryBuildIndex() {
Status DBImpl::TryBuildIndex() {
if (bg_build_index_started_) return Status::OK();
if (shutting_down_.load(std::memory_order_acquire)) return Status::OK();
bg_build_index_started_ = true;
std::thread build_index_task(&DBImpl<EngineT>::BackgroundBuildIndex, this);
std::thread build_index_task(&DBImpl::BackgroundBuildIndex, this);
build_index_task.detach();
return Status::OK();
}
template<typename EngineT>
void DBImpl<EngineT>::BackgroundCompaction() {
void DBImpl::BackgroundCompaction() {
std::vector<std::string> table_ids;
pMemMgr_->Serialize(table_ids);
@ -510,18 +505,15 @@ void DBImpl<EngineT>::BackgroundCompaction() {
}
}
template<typename EngineT>
Status DBImpl<EngineT>::DropAll() {
Status DBImpl::DropAll() {
return pMeta_->DropAll();
}
template<typename EngineT>
Status DBImpl<EngineT>::Size(long& result) {
Status DBImpl::Size(long& result) {
return pMeta_->Size(result);
}
template<typename EngineT>
DBImpl<EngineT>::~DBImpl() {
DBImpl::~DBImpl() {
{
std::unique_lock<std::mutex> lock(mutex_);
shutting_down_.store(true, std::memory_order_release);

7
cpp/src/db/DBImpl.h
View File

@ -8,12 +8,12 @@
#include "DB.h"
#include "MemManager.h"
#include "Types.h"
#include "Traits.h"
#include <mutex>
#include <condition_variable>
#include <memory>
#include <atomic>
#include <thread>
namespace zilliz {
namespace vecwise {
@ -25,11 +25,10 @@ namespace meta {
class Meta;
}
template <typename EngineT>
class DBImpl : public DB {
public:
using MetaPtr = meta::Meta::Ptr;
using MemManagerPtr = typename MemManager<EngineT>::Ptr;
using MemManagerPtr = typename MemManager::Ptr;
DBImpl(const Options& options);
@ -100,5 +99,3 @@ private:
} // namespace engine
} // namespace vecwise
} // namespace zilliz
#include "DBImpl.inl"

430
cpp/src/db/DBMetaImpl.cpp
View File

@ -7,6 +7,7 @@
#include "IDGenerator.h"
#include "Utils.h"
#include "MetaConsts.h"
#include "Factories.h"
#include "metrics/Metrics.h"
#include <unistd.h>
@ -29,27 +30,30 @@ using namespace sqlite_orm;
inline auto StoragePrototype(const std::string &path) {
return make_storage(path,
make_table("Table",
make_column("id", &TableSchema::id, primary_key()),
make_column("table_id", &TableSchema::table_id, unique()),
make_column("dimension", &TableSchema::dimension),
make_column("created_on", &TableSchema::created_on),
make_column("files_cnt", &TableSchema::files_cnt, default_value(0))),
make_column("id", &TableSchema::id_, primary_key()),
make_column("table_id", &TableSchema::table_id_, unique()),
make_column("dimension", &TableSchema::dimension_),
make_column("created_on", &TableSchema::created_on_),
make_column("files_cnt", &TableSchema::files_cnt_, default_value(0)),
make_column("engine_type", &TableSchema::engine_type_),
make_column("store_raw_data", &TableSchema::store_raw_data_)),
make_table("TableFile",
make_column("id", &TableFileSchema::id, primary_key()),
make_column("table_id", &TableFileSchema::table_id),
make_column("file_id", &TableFileSchema::file_id),
make_column("file_type", &TableFileSchema::file_type),
make_column("size", &TableFileSchema::size, default_value(0)),
make_column("updated_time", &TableFileSchema::updated_time),
make_column("created_on", &TableFileSchema::created_on),
make_column("date", &TableFileSchema::date))
make_column("id", &TableFileSchema::id_, primary_key()),
make_column("table_id", &TableFileSchema::table_id_),
make_column("engine_type", &TableFileSchema::engine_type_),
make_column("file_id", &TableFileSchema::file_id_),
make_column("file_type", &TableFileSchema::file_type_),
make_column("size", &TableFileSchema::size_, default_value(0)),
make_column("updated_time", &TableFileSchema::updated_time_),
make_column("created_on", &TableFileSchema::created_on_),
make_column("date", &TableFileSchema::date_))
);
}
using ConnectorT = decltype(StoragePrototype(""));
static std::unique_ptr<ConnectorT> ConnectorPtr;
using ConditionT = decltype(c(&TableFileSchema::id) == 1UL);
using ConditionT = decltype(c(&TableFileSchema::id_) == 1UL);
std::string DBMetaImpl::GetTablePath(const std::string &table_id) {
return options_.path + "/tables/" + table_id;
@ -62,13 +66,13 @@ std::string DBMetaImpl::GetTableDatePartitionPath(const std::string &table_id, D
}
void DBMetaImpl::GetTableFilePath(TableFileSchema &group_file) {
if (group_file.date == EmptyDate) {
group_file.date = Meta::GetDate();
if (group_file.date_ == EmptyDate) {
group_file.date_ = Meta::GetDate();
}
std::stringstream ss;
ss << GetTableDatePartitionPath(group_file.table_id, group_file.date)
<< "/" << group_file.file_id;
group_file.location = ss.str();
ss << GetTableDatePartitionPath(group_file.table_id_, group_file.date_)
<< "/" << group_file.file_id_;
group_file.location_ = ss.str();
}
Status DBMetaImpl::NextTableId(std::string &table_id) {
@ -120,7 +124,7 @@ Status DBMetaImpl::DropPartitionsByDates(const std::string &table_id,
}
TableSchema table_schema;
table_schema.table_id = table_id;
table_schema.table_id_ = table_id;
auto status = DescribeTable(table_schema);
if (!status.ok()) {
return status;
@ -137,11 +141,11 @@ Status DBMetaImpl::DropPartitionsByDates(const std::string &table_id,
try {
ConnectorPtr->update_all(
set(
c(&TableFileSchema::file_type) = (int) TableFileSchema::TO_DELETE
c(&TableFileSchema::file_type_) = (int) TableFileSchema::TO_DELETE
),
where(
c(&TableFileSchema::table_id) == table_id and
in(&TableFileSchema::date, dates)
c(&TableFileSchema::table_id_) == table_id and
in(&TableFileSchema::date_, dates)
));
} catch (std::exception &e) {
LOG(DEBUG) << e.what();
@ -152,17 +156,17 @@ Status DBMetaImpl::DropPartitionsByDates(const std::string &table_id,
Status DBMetaImpl::CreateTable(TableSchema &table_schema) {
server::Metrics::GetInstance().MetaAccessTotalIncrement();
if (table_schema.table_id == "") {
NextTableId(table_schema.table_id);
if (table_schema.table_id_ == "") {
NextTableId(table_schema.table_id_);
}
table_schema.files_cnt = 0;
table_schema.id = -1;
table_schema.created_on = utils::GetMicroSecTimeStamp();
table_schema.files_cnt_ = 0;
table_schema.id_ = -1;
table_schema.created_on_ = utils::GetMicroSecTimeStamp();
auto start_time = METRICS_NOW_TIME;
{
try {
auto id = ConnectorPtr->insert(table_schema);
table_schema.id = id;
table_schema.id_ = id;
} catch (...) {
return Status::DBTransactionError("Add Table Error");
}
@ -171,7 +175,7 @@ Status DBMetaImpl::CreateTable(TableSchema &table_schema) {
auto total_time = METRICS_MICROSECONDS(start_time, end_time);
server::Metrics::GetInstance().MetaAccessDurationSecondsHistogramObserve(total_time);
auto group_path = GetTablePath(table_schema.table_id);
auto group_path = GetTablePath(table_schema.table_id_);
if (!boost::filesystem::is_directory(group_path)) {
auto ret = boost::filesystem::create_directories(group_path);
@ -188,21 +192,25 @@ Status DBMetaImpl::DescribeTable(TableSchema &table_schema) {
try {
server::Metrics::GetInstance().MetaAccessTotalIncrement();
auto start_time = METRICS_NOW_TIME;
auto groups = ConnectorPtr->select(columns(&TableSchema::id,
&TableSchema::table_id,
&TableSchema::files_cnt,
&TableSchema::dimension),
where(c(&TableSchema::table_id) == table_schema.table_id));
auto groups = ConnectorPtr->select(columns(&TableSchema::id_,
&TableSchema::table_id_,
&TableSchema::files_cnt_,
&TableSchema::dimension_,
&TableSchema::engine_type_,
&TableSchema::store_raw_data_),
where(c(&TableSchema::table_id_) == table_schema.table_id_));
auto end_time = METRICS_NOW_TIME;
auto total_time = METRICS_MICROSECONDS(start_time, end_time);
server::Metrics::GetInstance().MetaAccessDurationSecondsHistogramObserve(total_time);
assert(groups.size() <= 1);
if (groups.size() == 1) {
table_schema.id = std::get<0>(groups[0]);
table_schema.files_cnt = std::get<2>(groups[0]);
table_schema.dimension = std::get<3>(groups[0]);
table_schema.id_ = std::get<0>(groups[0]);
table_schema.files_cnt_ = std::get<2>(groups[0]);
table_schema.dimension_ = std::get<3>(groups[0]);
table_schema.engine_type_ = std::get<4>(groups[0]);
table_schema.store_raw_data_ = std::get<5>(groups[0]);
} else {
return Status::NotFound("Table " + table_schema.table_id + " not found");
return Status::NotFound("Table " + table_schema.table_id_ + " not found");
}
} catch (std::exception &e) {
LOG(DEBUG) << e.what();
@ -217,8 +225,8 @@ Status DBMetaImpl::HasTable(const std::string &table_id, bool &has_or_not) {
server::Metrics::GetInstance().MetaAccessTotalIncrement();
auto start_time = METRICS_NOW_TIME;
auto tables = ConnectorPtr->select(columns(&TableSchema::id),
where(c(&TableSchema::table_id) == table_id));
auto tables = ConnectorPtr->select(columns(&TableSchema::id_),
where(c(&TableSchema::table_id_) == table_id));
auto end_time = METRICS_NOW_TIME;
auto total_time = METRICS_MICROSECONDS(start_time, end_time);
server::Metrics::GetInstance().MetaAccessDurationSecondsHistogramObserve(total_time);
@ -236,22 +244,23 @@ Status DBMetaImpl::HasTable(const std::string &table_id, bool &has_or_not) {
}
Status DBMetaImpl::CreateTableFile(TableFileSchema &file_schema) {
if (file_schema.date == EmptyDate) {
file_schema.date = Meta::GetDate();
if (file_schema.date_ == EmptyDate) {
file_schema.date_ = Meta::GetDate();
}
TableSchema table_schema;
table_schema.table_id = file_schema.table_id;
table_schema.table_id_ = file_schema.table_id_;
auto status = DescribeTable(table_schema);
if (!status.ok()) {
return status;
}
NextFileId(file_schema.file_id);
file_schema.file_type = TableFileSchema::NEW;
file_schema.dimension = table_schema.dimension;
file_schema.size = 0;
file_schema.created_on = utils::GetMicroSecTimeStamp();
file_schema.updated_time = file_schema.created_on;
NextFileId(file_schema.file_id_);
file_schema.file_type_ = TableFileSchema::NEW;
file_schema.dimension_ = table_schema.dimension_;
file_schema.size_ = 0;
file_schema.created_on_ = utils::GetMicroSecTimeStamp();
file_schema.updated_time_ = file_schema.created_on_;
file_schema.engine_type_ = table_schema.engine_type_;
GetTableFilePath(file_schema);
{
@ -259,7 +268,7 @@ Status DBMetaImpl::CreateTableFile(TableFileSchema &file_schema) {
server::Metrics::GetInstance().MetaAccessTotalIncrement();
auto start_time = METRICS_NOW_TIME;
auto id = ConnectorPtr->insert(file_schema);
file_schema.id = id;
file_schema.id_ = id;
auto end_time = METRICS_NOW_TIME;
auto total_time = METRICS_MICROSECONDS(start_time, end_time);
server::Metrics::GetInstance().MetaAccessDurationSecondsHistogramObserve(total_time);
@ -268,7 +277,7 @@ Status DBMetaImpl::CreateTableFile(TableFileSchema &file_schema) {
}
}
auto partition_path = GetTableDatePartitionPath(file_schema.table_id, file_schema.date);
auto partition_path = GetTableDatePartitionPath(file_schema.table_id_, file_schema.date_);
if (!boost::filesystem::is_directory(partition_path)) {
auto ret = boost::filesystem::create_directory(partition_path);
@ -287,13 +296,14 @@ Status DBMetaImpl::FilesToIndex(TableFilesSchema &files) {
try {
server::Metrics::GetInstance().MetaAccessTotalIncrement();
auto start_time = METRICS_NOW_TIME;
auto selected = ConnectorPtr->select(columns(&TableFileSchema::id,
&TableFileSchema::table_id,
&TableFileSchema::file_id,
&TableFileSchema::file_type,
&TableFileSchema::size,
&TableFileSchema::date),
where(c(&TableFileSchema::file_type)
auto selected = ConnectorPtr->select(columns(&TableFileSchema::id_,
&TableFileSchema::table_id_,
&TableFileSchema::file_id_,
&TableFileSchema::file_type_,
&TableFileSchema::size_,
&TableFileSchema::date_,
&TableFileSchema::engine_type_),
where(c(&TableFileSchema::file_type_)
== (int) TableFileSchema::TO_INDEX));
auto end_time = METRICS_NOW_TIME;
auto total_time = METRICS_MICROSECONDS(start_time, end_time);
@ -303,24 +313,26 @@ Status DBMetaImpl::FilesToIndex(TableFilesSchema &files) {
TableFileSchema table_file;
for (auto &file : selected) {
table_file.id = std::get<0>(file);
table_file.table_id = std::get<1>(file);
table_file.file_id = std::get<2>(file);
table_file.file_type = std::get<3>(file);
table_file.size = std::get<4>(file);
table_file.date = std::get<5>(file);
table_file.id_ = std::get<0>(file);
table_file.table_id_ = std::get<1>(file);
table_file.file_id_ = std::get<2>(file);
table_file.file_type_ = std::get<3>(file);
table_file.size_ = std::get<4>(file);
table_file.date_ = std::get<5>(file);
table_file.engine_type_ = std::get<6>(file);
GetTableFilePath(table_file);
auto groupItr = groups.find(table_file.table_id);
auto groupItr = groups.find(table_file.table_id_);
if (groupItr == groups.end()) {
TableSchema table_schema;
table_schema.table_id = table_file.table_id;
table_schema.table_id_ = table_file.table_id_;
auto status = DescribeTable(table_schema);
if (!status.ok()) {
return status;
}
groups[table_file.table_id] = table_schema;
groups[table_file.table_id_] = table_schema;
}
table_file.dimension = groups[table_file.table_id].dimension;
table_file.dimension_ = groups[table_file.table_id_].dimension_;
files.push_back(table_file);
}
} catch (std::exception &e) {
@ -340,23 +352,24 @@ Status DBMetaImpl::FilesToSearch(const std::string &table_id,
server::Metrics::GetInstance().MetaAccessTotalIncrement();
auto start_time = METRICS_NOW_TIME;
if (partition.empty()) {
auto selected = ConnectorPtr->select(columns(&TableFileSchema::id,
&TableFileSchema::table_id,
&TableFileSchema::file_id,
&TableFileSchema::file_type,
&TableFileSchema::size,
&TableFileSchema::date),
where(c(&TableFileSchema::table_id) == table_id and
(c(&TableFileSchema::file_type) == (int) TableFileSchema::RAW or
c(&TableFileSchema::file_type)
auto selected = ConnectorPtr->select(columns(&TableFileSchema::id_,
&TableFileSchema::table_id_,
&TableFileSchema::file_id_,
&TableFileSchema::file_type_,
&TableFileSchema::size_,
&TableFileSchema::date_,
&TableFileSchema::engine_type_),
where(c(&TableFileSchema::table_id_) == table_id and
(c(&TableFileSchema::file_type_) == (int) TableFileSchema::RAW or
c(&TableFileSchema::file_type_)
== (int) TableFileSchema::TO_INDEX or
c(&TableFileSchema::file_type)
c(&TableFileSchema::file_type_)
== (int) TableFileSchema::INDEX)));
auto end_time = METRICS_NOW_TIME;
auto total_time = METRICS_MICROSECONDS(start_time, end_time);
server::Metrics::GetInstance().MetaAccessDurationSecondsHistogramObserve(total_time);
TableSchema table_schema;
table_schema.table_id = table_id;
table_schema.table_id_ = table_id;
auto status = DescribeTable(table_schema);
if (!status.ok()) {
return status;
@ -365,40 +378,41 @@ Status DBMetaImpl::FilesToSearch(const std::string &table_id,
TableFileSchema table_file;
for (auto &file : selected) {
table_file.id = std::get<0>(file);
table_file.table_id = std::get<1>(file);
table_file.file_id = std::get<2>(file);
table_file.file_type = std::get<3>(file);
table_file.size = std::get<4>(file);
table_file.date = std::get<5>(file);
table_file.dimension = table_schema.dimension;
table_file.id_ = std::get<0>(file);
table_file.table_id_ = std::get<1>(file);
table_file.file_id_ = std::get<2>(file);
table_file.file_type_ = std::get<3>(file);
table_file.size_ = std::get<4>(file);
table_file.date_ = std::get<5>(file);
table_file.engine_type_ = std::get<6>(file);
table_file.dimension_ = table_schema.dimension_;
GetTableFilePath(table_file);
auto dateItr = files.find(table_file.date);
auto dateItr = files.find(table_file.date_);
if (dateItr == files.end()) {
files[table_file.date] = TableFilesSchema();
files[table_file.date_] = TableFilesSchema();
}
files[table_file.date].push_back(table_file);
files[table_file.date_].push_back(table_file);
}
}
else {
auto selected = ConnectorPtr->select(columns(&TableFileSchema::id,
&TableFileSchema::table_id,
&TableFileSchema::file_id,
&TableFileSchema::file_type,
&TableFileSchema::size,
&TableFileSchema::date),
where(c(&TableFileSchema::table_id) == table_id and
in(&TableFileSchema::date, partition) and
(c(&TableFileSchema::file_type) == (int) TableFileSchema::RAW or
c(&TableFileSchema::file_type)
auto selected = ConnectorPtr->select(columns(&TableFileSchema::id_,
&TableFileSchema::table_id_,
&TableFileSchema::file_id_,
&TableFileSchema::file_type_,
&TableFileSchema::size_,
&TableFileSchema::date_),
where(c(&TableFileSchema::table_id_) == table_id and
in(&TableFileSchema::date_, partition) and
(c(&TableFileSchema::file_type_) == (int) TableFileSchema::RAW or
c(&TableFileSchema::file_type_)
== (int) TableFileSchema::TO_INDEX or
c(&TableFileSchema::file_type)
c(&TableFileSchema::file_type_)
== (int) TableFileSchema::INDEX)));
auto end_time = METRICS_NOW_TIME;
auto total_time = METRICS_MICROSECONDS(start_time, end_time);
server::Metrics::GetInstance().MetaAccessDurationSecondsHistogramObserve(total_time);
TableSchema table_schema;
table_schema.table_id = table_id;
table_schema.table_id_ = table_id;
auto status = DescribeTable(table_schema);
if (!status.ok()) {
return status;
@ -407,19 +421,19 @@ Status DBMetaImpl::FilesToSearch(const std::string &table_id,
TableFileSchema table_file;
for (auto &file : selected) {
table_file.id = std::get<0>(file);
table_file.table_id = std::get<1>(file);
table_file.file_id = std::get<2>(file);
table_file.file_type = std::get<3>(file);
table_file.size = std::get<4>(file);
table_file.date = std::get<5>(file);
table_file.dimension = table_schema.dimension;
table_file.id_ = std::get<0>(file);
table_file.table_id_ = std::get<1>(file);
table_file.file_id_ = std::get<2>(file);
table_file.file_type_ = std::get<3>(file);
table_file.size_ = std::get<4>(file);
table_file.date_ = std::get<5>(file);
table_file.dimension_ = table_schema.dimension_;
GetTableFilePath(table_file);
auto dateItr = files.find(table_file.date);
auto dateItr = files.find(table_file.date_);
if (dateItr == files.end()) {
files[table_file.date] = TableFilesSchema();
files[table_file.date_] = TableFilesSchema();
}
files[table_file.date].push_back(table_file);
files[table_file.date_].push_back(table_file);
}
}
@ -438,19 +452,19 @@ Status DBMetaImpl::FilesToMerge(const std::string &table_id,
try {
server::Metrics::GetInstance().MetaAccessTotalIncrement();
auto start_time = METRICS_NOW_TIME;
auto selected = ConnectorPtr->select(columns(&TableFileSchema::id,
&TableFileSchema::table_id,
&TableFileSchema::file_id,
&TableFileSchema::file_type,
&TableFileSchema::size,
&TableFileSchema::date),
where(c(&TableFileSchema::file_type) == (int) TableFileSchema::RAW and
c(&TableFileSchema::table_id) == table_id));
auto selected = ConnectorPtr->select(columns(&TableFileSchema::id_,
&TableFileSchema::table_id_,
&TableFileSchema::file_id_,
&TableFileSchema::file_type_,
&TableFileSchema::size_,
&TableFileSchema::date_),
where(c(&TableFileSchema::file_type_) == (int) TableFileSchema::RAW and
c(&TableFileSchema::table_id_) == table_id));
auto end_time = METRICS_NOW_TIME;
auto total_time = METRICS_MICROSECONDS(start_time, end_time);
server::Metrics::GetInstance().MetaAccessDurationSecondsHistogramObserve(total_time);
TableSchema table_schema;
table_schema.table_id = table_id;
table_schema.table_id_ = table_id;
auto status = DescribeTable(table_schema);
if (!status.ok()) {
@ -459,19 +473,19 @@ Status DBMetaImpl::FilesToMerge(const std::string &table_id,
TableFileSchema table_file;
for (auto &file : selected) {
table_file.id = std::get<0>(file);
table_file.table_id = std::get<1>(file);
table_file.file_id = std::get<2>(file);
table_file.file_type = std::get<3>(file);
table_file.size = std::get<4>(file);
table_file.date = std::get<5>(file);
table_file.dimension = table_schema.dimension;
table_file.id_ = std::get<0>(file);
table_file.table_id_ = std::get<1>(file);
table_file.file_id_ = std::get<2>(file);
table_file.file_type_ = std::get<3>(file);
table_file.size_ = std::get<4>(file);
table_file.date_ = std::get<5>(file);
table_file.dimension_ = table_schema.dimension_;
GetTableFilePath(table_file);
auto dateItr = files.find(table_file.date);
auto dateItr = files.find(table_file.date_);
if (dateItr == files.end()) {
files[table_file.date] = TableFilesSchema();
files[table_file.date_] = TableFilesSchema();
}
files[table_file.date].push_back(table_file);
files[table_file.date_].push_back(table_file);
}
} catch (std::exception &e) {
LOG(DEBUG) << e.what();
@ -484,26 +498,26 @@ Status DBMetaImpl::FilesToMerge(const std::string &table_id,
Status DBMetaImpl::GetTableFile(TableFileSchema &file_schema) {
try {
auto files = ConnectorPtr->select(columns(&TableFileSchema::id,
&TableFileSchema::table_id,
&TableFileSchema::file_id,
&TableFileSchema::file_type,
&TableFileSchema::size,
&TableFileSchema::date),
where(c(&TableFileSchema::file_id) == file_schema.file_id and
c(&TableFileSchema::table_id) == file_schema.table_id
auto files = ConnectorPtr->select(columns(&TableFileSchema::id_,
&TableFileSchema::table_id_,
&TableFileSchema::file_id_,
&TableFileSchema::file_type_,
&TableFileSchema::size_,
&TableFileSchema::date_),
where(c(&TableFileSchema::file_id_) == file_schema.file_id_ and
c(&TableFileSchema::table_id_) == file_schema.table_id_
));
assert(files.size() <= 1);
if (files.size() == 1) {
file_schema.id = std::get<0>(files[0]);
file_schema.table_id = std::get<1>(files[0]);
file_schema.file_id = std::get<2>(files[0]);
file_schema.file_type = std::get<3>(files[0]);
file_schema.size = std::get<4>(files[0]);
file_schema.date = std::get<5>(files[0]);
file_schema.id_ = std::get<0>(files[0]);
file_schema.table_id_ = std::get<1>(files[0]);
file_schema.file_id_ = std::get<2>(files[0]);
file_schema.file_type_ = std::get<3>(files[0]);
file_schema.size_ = std::get<4>(files[0]);
file_schema.date_ = std::get<5>(files[0]);
} else {
return Status::NotFound("Table:" + file_schema.table_id +
" File:" + file_schema.file_id + " not found");
return Status::NotFound("Table:" + file_schema.table_id_ +
" File:" + file_schema.file_id_ + " not found");
}
} catch (std::exception &e) {
LOG(DEBUG) << e.what();
@ -529,11 +543,11 @@ Status DBMetaImpl::Archive() {
try {
ConnectorPtr->update_all(
set(
c(&TableFileSchema::file_type) = (int) TableFileSchema::TO_DELETE
c(&TableFileSchema::file_type_) = (int) TableFileSchema::TO_DELETE
),
where(
c(&TableFileSchema::created_on) < (long) (now - usecs) and
c(&TableFileSchema::file_type) != (int) TableFileSchema::TO_DELETE
c(&TableFileSchema::created_on_) < (long) (now - usecs) and
c(&TableFileSchema::file_type_) != (int) TableFileSchema::TO_DELETE
));
} catch (std::exception &e) {
LOG(DEBUG) << e.what();
@ -555,9 +569,9 @@ Status DBMetaImpl::Archive() {
Status DBMetaImpl::Size(long &result) {
result = 0;
try {
auto selected = ConnectorPtr->select(columns(sum(&TableFileSchema::size)),
auto selected = ConnectorPtr->select(columns(sum(&TableFileSchema::size_)),
where(
c(&TableFileSchema::file_type) != (int) TableFileSchema::TO_DELETE
c(&TableFileSchema::file_type_) != (int) TableFileSchema::TO_DELETE
));
for (auto &sub_query : selected) {
@ -580,11 +594,11 @@ Status DBMetaImpl::DiscardFiles(long to_discard_size) {
return Status::OK();
}
try {
auto selected = ConnectorPtr->select(columns(&TableFileSchema::id,
&TableFileSchema::size),
where(c(&TableFileSchema::file_type)
auto selected = ConnectorPtr->select(columns(&TableFileSchema::id_,
&TableFileSchema::size_),
where(c(&TableFileSchema::file_type_)
!= (int) TableFileSchema::TO_DELETE),
order_by(&TableFileSchema::id),
order_by(&TableFileSchema::id_),
limit(10));
std::vector<int> ids;
@ -592,11 +606,11 @@ Status DBMetaImpl::DiscardFiles(long to_discard_size) {
for (auto &file : selected) {
if (to_discard_size <= 0) break;
table_file.id = std::get<0>(file);
table_file.size = std::get<1>(file);
ids.push_back(table_file.id);
LOG(DEBUG) << "Discard table_file.id=" << table_file.file_id << " table_file.size=" << table_file.size;
to_discard_size -= table_file.size;
table_file.id_ = std::get<0>(file);
table_file.size_ = std::get<1>(file);
ids.push_back(table_file.id_);
LOG(DEBUG) << "Discard table_file.id=" << table_file.file_id_ << " table_file.size=" << table_file.size_;
to_discard_size -= table_file.size_;
}
if (ids.size() == 0) {
@ -605,10 +619,10 @@ Status DBMetaImpl::DiscardFiles(long to_discard_size) {
ConnectorPtr->update_all(
set(
c(&TableFileSchema::file_type) = (int) TableFileSchema::TO_DELETE
c(&TableFileSchema::file_type_) = (int) TableFileSchema::TO_DELETE
),
where(
in(&TableFileSchema::id, ids)
in(&TableFileSchema::id_, ids)
));
} catch (std::exception &e) {
@ -621,7 +635,7 @@ Status DBMetaImpl::DiscardFiles(long to_discard_size) {
}
Status DBMetaImpl::UpdateTableFile(TableFileSchema &file_schema) {
file_schema.updated_time = utils::GetMicroSecTimeStamp();
file_schema.updated_time_ = utils::GetMicroSecTimeStamp();
try {
server::Metrics::GetInstance().MetaAccessTotalIncrement();
auto start_time = METRICS_NOW_TIME;
@ -631,7 +645,7 @@ Status DBMetaImpl::UpdateTableFile(TableFileSchema &file_schema) {
server::Metrics::GetInstance().MetaAccessDurationSecondsHistogramObserve(total_time);
} catch (std::exception &e) {
LOG(DEBUG) << e.what();
LOG(DEBUG) << "table_id= " << file_schema.table_id << " file_id=" << file_schema.file_id;
LOG(DEBUG) << "table_id= " << file_schema.table_id_ << " file_id=" << file_schema.file_id_;
throw e;
}
return Status::OK();
@ -643,7 +657,7 @@ Status DBMetaImpl::UpdateTableFiles(TableFilesSchema &files) {
auto start_time = METRICS_NOW_TIME;
auto commited = ConnectorPtr->transaction([&]() mutable {
for (auto &file : files) {
file.updated_time = utils::GetMicroSecTimeStamp();
file.updated_time_ = utils::GetMicroSecTimeStamp();
ConnectorPtr->update(file);
}
auto end_time = METRICS_NOW_TIME;
@ -664,33 +678,33 @@ Status DBMetaImpl::UpdateTableFiles(TableFilesSchema &files) {
Status DBMetaImpl::CleanUpFilesWithTTL(uint16_t seconds) {
auto now = utils::GetMicroSecTimeStamp();
try {
auto selected = ConnectorPtr->select(columns(&TableFileSchema::id,
&TableFileSchema::table_id,
&TableFileSchema::file_id,
&TableFileSchema::file_type,
&TableFileSchema::size,
&TableFileSchema::date),
auto selected = ConnectorPtr->select(columns(&TableFileSchema::id_,
&TableFileSchema::table_id_,
&TableFileSchema::file_id_,
&TableFileSchema::file_type_,
&TableFileSchema::size_,
&TableFileSchema::date_),
where(
c(&TableFileSchema::file_type) == (int) TableFileSchema::TO_DELETE
c(&TableFileSchema::file_type_) == (int) TableFileSchema::TO_DELETE
and
c(&TableFileSchema::updated_time)
c(&TableFileSchema::updated_time_)
> now - seconds * US_PS));
TableFilesSchema updated;
TableFileSchema table_file;
for (auto &file : selected) {
table_file.id = std::get<0>(file);
table_file.table_id = std::get<1>(file);
table_file.file_id = std::get<2>(file);
table_file.file_type = std::get<3>(file);
table_file.size = std::get<4>(file);
table_file.date = std::get<5>(file);
table_file.id_ = std::get<0>(file);
table_file.table_id_ = std::get<1>(file);
table_file.file_id_ = std::get<2>(file);
table_file.file_type_ = std::get<3>(file);
table_file.size_ = std::get<4>(file);
table_file.date_ = std::get<5>(file);
GetTableFilePath(table_file);
if (table_file.file_type == TableFileSchema::TO_DELETE) {
boost::filesystem::remove(table_file.location);
if (table_file.file_type_ == TableFileSchema::TO_DELETE) {
boost::filesystem::remove(table_file.location_);
}
ConnectorPtr->remove<TableFileSchema>(table_file.id);
ConnectorPtr->remove<TableFileSchema>(table_file.id_);
/* LOG(DEBUG) << "Removing deleted id=" << table_file.id << " location=" << table_file.location << std::endl; */
}
} catch (std::exception &e) {
@ -703,33 +717,33 @@ Status DBMetaImpl::CleanUpFilesWithTTL(uint16_t seconds) {
Status DBMetaImpl::CleanUp() {
try {
auto selected = ConnectorPtr->select(columns(&TableFileSchema::id,
&TableFileSchema::table_id,
&TableFileSchema::file_id,
&TableFileSchema::file_type,
&TableFileSchema::size,
&TableFileSchema::date),
auto selected = ConnectorPtr->select(columns(&TableFileSchema::id_,
&TableFileSchema::table_id_,
&TableFileSchema::file_id_,
&TableFileSchema::file_type_,
&TableFileSchema::size_,
&TableFileSchema::date_),
where(
c(&TableFileSchema::file_type) == (int) TableFileSchema::TO_DELETE
c(&TableFileSchema::file_type_) == (int) TableFileSchema::TO_DELETE
or
c(&TableFileSchema::file_type)
c(&TableFileSchema::file_type_)
== (int) TableFileSchema::NEW));
TableFilesSchema updated;
TableFileSchema table_file;
for (auto &file : selected) {
table_file.id = std::get<0>(file);
table_file.table_id = std::get<1>(file);
table_file.file_id = std::get<2>(file);
table_file.file_type = std::get<3>(file);
table_file.size = std::get<4>(file);
table_file.date = std::get<5>(file);
table_file.id_ = std::get<0>(file);
table_file.table_id_ = std::get<1>(file);
table_file.file_id_ = std::get<2>(file);
table_file.file_type_ = std::get<3>(file);
table_file.size_ = std::get<4>(file);
table_file.date_ = std::get<5>(file);
GetTableFilePath(table_file);
if (table_file.file_type == TableFileSchema::TO_DELETE) {
boost::filesystem::remove(table_file.location);
if (table_file.file_type_ == TableFileSchema::TO_DELETE) {
boost::filesystem::remove(table_file.location_);
}
ConnectorPtr->remove<TableFileSchema>(table_file.id);
ConnectorPtr->remove<TableFileSchema>(table_file.id_);
/* LOG(DEBUG) << "Removing id=" << table_file.id << " location=" << table_file.location << std::endl; */
}
} catch (std::exception &e) {
@ -746,19 +760,19 @@ Status DBMetaImpl::Count(const std::string &table_id, long &result) {
server::Metrics::GetInstance().MetaAccessTotalIncrement();
auto start_time = METRICS_NOW_TIME;
auto selected = ConnectorPtr->select(columns(&TableFileSchema::size,
&TableFileSchema::date),
where((c(&TableFileSchema::file_type) == (int) TableFileSchema::RAW or
c(&TableFileSchema::file_type) == (int) TableFileSchema::TO_INDEX
auto selected = ConnectorPtr->select(columns(&TableFileSchema::size_,
&TableFileSchema::date_),
where((c(&TableFileSchema::file_type_) == (int) TableFileSchema::RAW or
c(&TableFileSchema::file_type_) == (int) TableFileSchema::TO_INDEX
or
c(&TableFileSchema::file_type) == (int) TableFileSchema::INDEX)
c(&TableFileSchema::file_type_) == (int) TableFileSchema::INDEX)
and
c(&TableFileSchema::table_id) == table_id));
c(&TableFileSchema::table_id_) == table_id));
auto end_time = METRICS_NOW_TIME;
auto total_time = METRICS_MICROSECONDS(start_time, end_time);
server::Metrics::GetInstance().MetaAccessDurationSecondsHistogramObserve(total_time);
TableSchema table_schema;
table_schema.table_id = table_id;
table_schema.table_id_ = table_id;
auto status = DescribeTable(table_schema);
if (!status.ok()) {
@ -770,7 +784,7 @@ Status DBMetaImpl::Count(const std::string &table_id, long &result) {
result += std::get<0>(file);
}
result /= table_schema.dimension;
result /= table_schema.dimension_;
} catch (std::exception &e) {
LOG(DEBUG) << e.what();

31
cpp/src/db/EngineFactory.cpp Normal file
View File

@ -0,0 +1,31 @@
/*******************************************************************************
* Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved
* Unauthorized copying of this file, via any medium is strictly prohibited.
* Proprietary and confidential.
******************************************************************************/
#include "EngineFactory.h"
#include "FaissExecutionEngine.h"
#include "Log.h"
namespace zilliz {
namespace vecwise {
namespace engine {
ExecutionEnginePtr
EngineFactory::Build(uint16_t dimension,
const std::string& location,
EngineType type) {
switch(type) {
case EngineType::FAISS_IDMAP:
return ExecutionEnginePtr(new FaissExecutionEngine(dimension, location, "IDMap", "IDMap,Flat"));
case EngineType::FAISS_IVFFLAT:
return ExecutionEnginePtr(new FaissExecutionEngine(dimension, location, "IVF", "IDMap,Flat"));
default:
ENGINE_LOG_ERROR << "Unsupportted engine type";
return nullptr;
}
}
}
}
}

22
cpp/src/db/Traits.h → cpp/src/db/EngineFactory.h
View File

@ -5,22 +5,20 @@
******************************************************************************/
#pragma once
#include "Status.h"
#include "ExecutionEngine.h"
namespace zilliz {
namespace vecwise {
namespace engine {
struct IVFIndexTrait {
static const char* BuildIndexType;
static const char* RawIndexType;
class EngineFactory {
public:
static ExecutionEnginePtr Build(uint16_t dimension,
const std::string& location,
EngineType type);
};
struct IDMapIndexTrait {
static const char* BuildIndexType;
static const char* RawIndexType;
};
} // namespace engine
} // namespace vecwise
} // namespace zilliz
}
}
}

57
cpp/src/db/ExecutionEngine.cpp
View File

@ -11,8 +11,7 @@ namespace zilliz {
namespace vecwise {
namespace engine {
template<typename Derived>
Status ExecutionEngine<Derived>::AddWithIds(const std::vector<float>& vectors, const std::vector<long>& vector_ids) {
Status ExecutionEngine::AddWithIds(const std::vector<float>& vectors, const std::vector<long>& vector_ids) {
long n1 = (long)vectors.size();
long n2 = (long)vector_ids.size();
if (n1 != n2) {
@ -22,60 +21,6 @@ Status ExecutionEngine<Derived>::AddWithIds(const std::vector<float>& vectors, c
return AddWithIds(n1, vectors.data(), vector_ids.data());
}
template<typename Derived>
Status ExecutionEngine<Derived>::AddWithIds(long n, const float *xdata, const long *xids) {
return static_cast<Derived*>(this)->AddWithIds(n, xdata, xids);
}
template<typename Derived>
size_t ExecutionEngine<Derived>::Count() const {
return static_cast<Derived*>(this)->Count();
}
template<typename Derived>
size_t ExecutionEngine<Derived>::Size() const {
return static_cast<Derived*>(this)->Size();
}
template<typename Derived>
size_t ExecutionEngine<Derived>::PhysicalSize() const {
return static_cast<Derived*>(this)->PhysicalSize();
}
template<typename Derived>
Status ExecutionEngine<Derived>::Serialize() {
return static_cast<Derived*>(this)->Serialize();
}
template<typename Derived>
Status ExecutionEngine<Derived>::Load() {
return static_cast<Derived*>(this)->Load();
}
template<typename Derived>
Status ExecutionEngine<Derived>::Merge(const std::string& location) {
return static_cast<Derived*>(this)->Merge(location);
}
template<typename Derived>
Status ExecutionEngine<Derived>::Search(long n,
const float *data,
long k,
float *distances,
long *labels) const {
return static_cast<Derived*>(this)->Search(n, data, k, distances, labels);
}
template<typename Derived>
Status ExecutionEngine<Derived>::Cache() {
return static_cast<Derived*>(this)->Cache();
}
template<typename Derived>
std::shared_ptr<Derived> ExecutionEngine<Derived>::BuildIndex(const std::string& location) {
return static_cast<Derived*>(this)->BuildIndex(location);
}
} // namespace engine
} // namespace vecwise

37
cpp/src/db/ExecutionEngine.h
View File

@ -14,38 +14,47 @@ namespace zilliz {
namespace vecwise {
namespace engine {
template <typename Derived>
enum class EngineType {
INVALID = 0,
FAISS_IDMAP = 1,
FAISS_IVFFLAT,
};
class ExecutionEngine {
public:
Status AddWithIds(const std::vector<float>& vectors,
const std::vector<long>& vector_ids);
virtual Status AddWithIds(const std::vector<float>& vectors,
const std::vector<long>& vector_ids);
Status AddWithIds(long n, const float *xdata, const long *xids);
virtual Status AddWithIds(long n, const float *xdata, const long *xids) = 0;
size_t Count() const;
virtual size_t Count() const = 0;
size_t Size() const;
virtual size_t Size() const = 0;
size_t PhysicalSize() const;
virtual size_t Dimension() const = 0;
Status Serialize();
virtual size_t PhysicalSize() const = 0;
Status Load();
virtual Status Serialize() = 0;
Status Merge(const std::string& location);
virtual Status Load() = 0;
Status Search(long n,
virtual Status Merge(const std::string& location) = 0;
virtual Status Search(long n,
const float *data,
long k,
float *distances,
long *labels) const;
long *labels) const = 0;
std::shared_ptr<Derived> BuildIndex(const std::string&);
virtual std::shared_ptr<ExecutionEngine> BuildIndex(const std::string&) = 0;
Status Cache();
virtual Status Cache() = 0;
};
using ExecutionEnginePtr = std::shared_ptr<ExecutionEngine>;
} // namespace engine
} // namespace vecwise

24
cpp/src/db/Factories.cpp
View File

@ -5,8 +5,6 @@
////////////////////////////////////////////////////////////////////////////////
#include "Factories.h"
#include "DBImpl.h"
#include "FaissExecutionEngine.h"
#include "Traits.h"
#include <stdlib.h>
#include <time.h>
@ -45,28 +43,14 @@ std::shared_ptr<meta::DBMetaImpl> DBMetaImplFactory::Build() {
return std::shared_ptr<meta::DBMetaImpl>(new meta::DBMetaImpl(options));
}
std::shared_ptr<DB> DBFactory::Build(const std::string& db_type) {
std::shared_ptr<DB> DBFactory::Build() {
auto options = OptionsFactory::Build();
auto db = DBFactory::Build(options, db_type);
auto db = DBFactory::Build(options);
return std::shared_ptr<DB>(db);
}
DB* DBFactory::Build(const Options& options, const std::string& db_type) {
std::stringstream ss(db_type);
std::string token;
std::vector<std::string> tokens;
while (std::getline(ss, token, ',')) {
tokens.push_back(token);
}
assert(tokens.size()==2);
assert(tokens[0]=="Faiss");
if (tokens[1] == "IVF") {
return new DBImpl<FaissExecutionEngine<IVFIndexTrait>>(options);
} else if (tokens[1] == "IDMap") {
return new DBImpl<FaissExecutionEngine<IDMapIndexTrait>>(options);
}
return nullptr;
DB* DBFactory::Build(const Options& options) {
return new DBImpl(options);
}
} // namespace engine

5
cpp/src/db/Factories.h
View File

@ -8,6 +8,7 @@
#include "DB.h"
#include "DBMetaImpl.h"
#include "Options.h"
#include "ExecutionEngine.h"
#include <string>
#include <memory>
@ -29,8 +30,8 @@ struct DBMetaImplFactory {
};
struct DBFactory {
static std::shared_ptr<DB> Build(const std::string& db_type = "Faiss,IVF");
static DB* Build(const Options&, const std::string& db_type = "Faiss,IVF");
static std::shared_ptr<DB> Build();
static DB* Build(const Options&);
};
} // namespace engine

64
cpp/src/db/FaissExecutionEngine.inl → cpp/src/db/FaissExecutionEngine.cpp
View File

@ -3,8 +3,6 @@
* Unauthorized copying of this file, via any medium is strictly prohibited.
* Proprietary and confidential.
******************************************************************************/
#pragma once
#include "FaissExecutionEngine.h"
#include <easylogging++.h>
@ -23,47 +21,53 @@ namespace vecwise {
namespace engine {
template<class IndexTrait>
FaissExecutionEngine<IndexTrait>::FaissExecutionEngine(uint16_t dimension, const std::string& location)
: pIndex_(faiss::index_factory(dimension, IndexTrait::RawIndexType)),
location_(location) {
FaissExecutionEngine::FaissExecutionEngine(uint16_t dimension,
const std::string& location,
const std::string& build_index_type,
const std::string& raw_index_type)
: pIndex_(faiss::index_factory(dimension, raw_index_type.c_str())),
location_(location),
build_index_type_(build_index_type),
raw_index_type_(raw_index_type) {
}
template<class IndexTrait>
FaissExecutionEngine<IndexTrait>::FaissExecutionEngine(std::shared_ptr<faiss::Index> index, const std::string& location)
FaissExecutionEngine::FaissExecutionEngine(std::shared_ptr<faiss::Index> index,
const std::string& location,
const std::string& build_index_type,
const std::string& raw_index_type)
: pIndex_(index),
location_(location) {
location_(location),
build_index_type_(build_index_type),
raw_index_type_(raw_index_type) {
}
template<class IndexTrait>
Status FaissExecutionEngine<IndexTrait>::AddWithIds(long n, const float *xdata, const long *xids) {
Status FaissExecutionEngine::AddWithIds(long n, const float *xdata, const long *xids) {
pIndex_->add_with_ids(n, xdata, xids);
return Status::OK();
}
template<class IndexTrait>
size_t FaissExecutionEngine<IndexTrait>::Count() const {
size_t FaissExecutionEngine::Count() const {
return (size_t)(pIndex_->ntotal);
}
template<class IndexTrait>
size_t FaissExecutionEngine<IndexTrait>::Size() const {
size_t FaissExecutionEngine::Size() const {
return (size_t)(Count() * pIndex_->d)*sizeof(float);
}
template<class IndexTrait>
size_t FaissExecutionEngine<IndexTrait>::PhysicalSize() const {
size_t FaissExecutionEngine::Dimension() const {
return pIndex_->d;
}
size_t FaissExecutionEngine::PhysicalSize() const {
return (size_t)(Count() * pIndex_->d)*sizeof(float);
}
template<class IndexTrait>
Status FaissExecutionEngine<IndexTrait>::Serialize() {
Status FaissExecutionEngine::Serialize() {
write_index(pIndex_.get(), location_.c_str());
return Status::OK();
}
template<class IndexTrait>
Status FaissExecutionEngine<IndexTrait>::Load() {
Status FaissExecutionEngine::Load() {
auto index = zilliz::vecwise::cache::CpuCacheMgr::GetInstance()->GetIndex(location_);
bool to_cache = false;
auto start_time = METRICS_NOW_TIME;
@ -90,8 +94,7 @@ Status FaissExecutionEngine<IndexTrait>::Load() {
return Status::OK();
}
template<class IndexTrait>
Status FaissExecutionEngine<IndexTrait>::Merge(const std::string& location) {
Status FaissExecutionEngine::Merge(const std::string& location) {
if (location == location_) {
return Status::Error("Cannot Merge Self");
}
@ -105,12 +108,11 @@ Status FaissExecutionEngine<IndexTrait>::Merge(const std::string& location) {
return Status::OK();
}
template<class IndexTrait>
typename FaissExecutionEngine<IndexTrait>::Ptr
FaissExecutionEngine<IndexTrait>::BuildIndex(const std::string& location) {
ExecutionEnginePtr
FaissExecutionEngine::BuildIndex(const std::string& location) {
auto opd = std::make_shared<Operand>();
opd->d = pIndex_->d;
opd->index_type = IndexTrait::BuildIndexType;
opd->index_type = build_index_type_;
IndexBuilderPtr pBuilder = GetIndexBuilder(opd);
auto from_index = dynamic_cast<faiss::IndexIDMap*>(pIndex_.get());
@ -119,13 +121,12 @@ FaissExecutionEngine<IndexTrait>::BuildIndex(const std::string& location) {
dynamic_cast<faiss::IndexFlat*>(from_index->index)->xb.data(),
from_index->id_map.data());
Ptr new_ee(new FaissExecutionEngine<IndexTrait>(index->data(), location));
ExecutionEnginePtr new_ee(new FaissExecutionEngine(index->data(), location, build_index_type_, raw_index_type_));
new_ee->Serialize();
return new_ee;
}
template<class IndexTrait>
Status FaissExecutionEngine<IndexTrait>::Search(long n,
Status FaissExecutionEngine::Search(long n,
const float *data,
long k,
float *distances,
@ -135,8 +136,7 @@ Status FaissExecutionEngine<IndexTrait>::Search(long n,
return Status::OK();
}
template<class IndexTrait>
Status FaissExecutionEngine<IndexTrait>::Cache() {
Status FaissExecutionEngine::Cache() {
zilliz::vecwise::cache::CpuCacheMgr::GetInstance(
)->InsertItem(location_, std::make_shared<Index>(pIndex_));

44
cpp/src/db/FaissExecutionEngine.h
View File

@ -19,50 +19,54 @@ namespace vecwise {
namespace engine {
template<class IndexTrait>
class FaissExecutionEngine : public ExecutionEngine<FaissExecutionEngine<IndexTrait>> {
class FaissExecutionEngine : public ExecutionEngine {
public:
using Ptr = std::shared_ptr<FaissExecutionEngine<IndexTrait>>;
FaissExecutionEngine(uint16_t dimension, const std::string& location);
FaissExecutionEngine(std::shared_ptr<faiss::Index> index, const std::string& location);
FaissExecutionEngine(uint16_t dimension,
const std::string& location,
const std::string& build_index_type,
const std::string& raw_index_type);
Status AddWithIds(const std::vector<float>& vectors,
const std::vector<long>& vector_ids);
FaissExecutionEngine(std::shared_ptr<faiss::Index> index,
const std::string& location,
const std::string& build_index_type,
const std::string& raw_index_type);
Status AddWithIds(long n, const float *xdata, const long *xids);
Status AddWithIds(long n, const float *xdata, const long *xids) override;
size_t Count() const;
size_t Count() const override;
size_t Size() const;
size_t Size() const override;
size_t PhysicalSize() const;
size_t Dimension() const override;
Status Serialize();
size_t PhysicalSize() const override;
Status Load();
Status Serialize() override;
Status Merge(const std::string& location);
Status Load() override;
Status Merge(const std::string& location) override;
Status Search(long n,
const float *data,
long k,
float *distances,
long *labels) const;
long *labels) const override;
Ptr BuildIndex(const std::string&);
ExecutionEnginePtr BuildIndex(const std::string&) override;
Status Cache();
Status Cache() override;
protected:
std::shared_ptr<faiss::Index> pIndex_;
std::string location_;
std::string build_index_type_;
std::string raw_index_type_;
};
} // namespace engine
} // namespace vecwise
} // namespace zilliz
#include "FaissExecutionEngine.inl"

26
cpp/src/db/Log.h Normal file
View File

@ -0,0 +1,26 @@
/*******************************************************************************
* Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved
* Unauthorized copying of this file, via any medium is strictly prohibited.
* Proprietary and confidential.
******************************************************************************/
#pragma once
#include <easylogging++.h>
namespace zilliz {
namespace vecwise {
namespace engine {
#define ENGINE_DOMAIN_NAME "[ENGINE] "
#define ENGINE_ERROR_TEXT "ENGINE Error:"
#define ENGINE_LOG_TRACE LOG(TRACE) << ENGINE_DOMAIN_NAME
#define ENGINE_LOG_DEBUG LOG(DEBUG) << ENGINE_DOMAIN_NAME
#define ENGINE_LOG_INFO LOG(INFO) << ENGINE_DOMAIN_NAME
#define ENGINE_LOG_WARNING LOG(WARNING) << ENGINE_DOMAIN_NAME
#define ENGINE_LOG_ERROR LOG(ERROR) << ENGINE_DOMAIN_NAME
#define ENGINE_LOG_FATAL LOG(FATAL) << ENGINE_DOMAIN_NAME
} // namespace sql
} // namespace zilliz
} // namespace server

53
cpp/src/db/MemManager.inl → cpp/src/db/MemManager.cpp
View File

@ -3,11 +3,10 @@
* Unauthorized copying of this file, via any medium is strictly prohibited.
* Proprietary and confidential.
******************************************************************************/
#pragma once
#include "MemManager.h"
#include "Meta.h"
#include "MetaConsts.h"
#include "EngineFactory.h"
#include "metrics/Metrics.h"
#include <iostream>
@ -19,59 +18,53 @@ namespace zilliz {
namespace vecwise {
namespace engine {
template<typename EngineT>
MemVectors<EngineT>::MemVectors(const std::shared_ptr<meta::Meta>& meta_ptr,
MemVectors::MemVectors(const std::shared_ptr<meta::Meta>& meta_ptr,
const meta::TableFileSchema& schema, const Options& options)
: pMeta_(meta_ptr),
options_(options),
schema_(schema),
pIdGenerator_(new SimpleIDGenerator()),
pEE_(new EngineT(schema_.dimension, schema_.location)) {
pEE_(EngineFactory::Build(schema_.dimension_, schema_.location_, (EngineType)schema_.engine_type_)) {
}
template<typename EngineT>
void MemVectors<EngineT>::Add(size_t n_, const float* vectors_, IDNumbers& vector_ids_) {
void MemVectors::Add(size_t n_, const float* vectors_, IDNumbers& vector_ids_) {
pIdGenerator_->GetNextIDNumbers(n_, vector_ids_);
pEE_->AddWithIds(n_, vectors_, vector_ids_.data());
}
template<typename EngineT>
size_t MemVectors<EngineT>::Total() const {
size_t MemVectors::Total() const {
return pEE_->Count();
}
template<typename EngineT>
size_t MemVectors<EngineT>::ApproximateSize() const {
size_t MemVectors::ApproximateSize() const {
return pEE_->Size();
}
template<typename EngineT>
Status MemVectors<EngineT>::Serialize(std::string& table_id) {
table_id = schema_.table_id;
Status MemVectors::Serialize(std::string& table_id) {
table_id = schema_.table_id_;
auto size = ApproximateSize();
auto start_time = METRICS_NOW_TIME;
pEE_->Serialize();
auto end_time = METRICS_NOW_TIME;
auto total_time = METRICS_MICROSECONDS(start_time, end_time);
schema_.size = size;
schema_.size_ = size;
server::Metrics::GetInstance().DiskStoreIOSpeedGaugeSet(size/total_time);
schema_.file_type = (size >= options_.index_trigger_size) ?
schema_.file_type_ = (size >= options_.index_trigger_size) ?
meta::TableFileSchema::TO_INDEX : meta::TableFileSchema::RAW;
auto status = pMeta_->UpdateTableFile(schema_);
LOG(DEBUG) << "New " << ((schema_.file_type == meta::TableFileSchema::RAW) ? "raw" : "to_index")
<< " file " << schema_.file_id << " of size " << pEE_->Size() / meta::M << " M";
LOG(DEBUG) << "New " << ((schema_.file_type_ == meta::TableFileSchema::RAW) ? "raw" : "to_index")
<< " file " << schema_.file_id_ << " of size " << pEE_->Size() / meta::M << " M";
pEE_->Cache();
return status;
}
template<typename EngineT>
MemVectors<EngineT>::~MemVectors() {
MemVectors::~MemVectors() {
if (pIdGenerator_ != nullptr) {
delete pIdGenerator_;
pIdGenerator_ = nullptr;
@ -81,9 +74,7 @@ MemVectors<EngineT>::~MemVectors() {
/*
* MemManager
*/
template<typename EngineT>
typename MemManager<EngineT>::MemVectorsPtr MemManager<EngineT>::GetMemByTable(
MemManager::MemVectorsPtr MemManager::GetMemByTable(
const std::string& table_id) {
auto memIt = memMap_.find(table_id);
if (memIt != memMap_.end()) {
@ -91,18 +82,17 @@ typename MemManager<EngineT>::MemVectorsPtr MemManager<EngineT>::GetMemByTable(
}
meta::TableFileSchema table_file;
table_file.table_id = table_id;
table_file.table_id_ = table_id;
auto status = pMeta_->CreateTableFile(table_file);
if (!status.ok()) {
return nullptr;
}
memMap_[table_id] = MemVectorsPtr(new MemVectors<EngineT>(pMeta_, table_file, options_));
memMap_[table_id] = MemVectorsPtr(new MemVectors(pMeta_, table_file, options_));
return memMap_[table_id];
}
template<typename EngineT>
Status MemManager<EngineT>::InsertVectors(const std::string& table_id_,
Status MemManager::InsertVectors(const std::string& table_id_,
size_t n_,
const float* vectors_,
IDNumbers& vector_ids_) {
@ -110,8 +100,7 @@ Status MemManager<EngineT>::InsertVectors(const std::string& table_id_,
return InsertVectorsNoLock(table_id_, n_, vectors_, vector_ids_);
}
template<typename EngineT>
Status MemManager<EngineT>::InsertVectorsNoLock(const std::string& table_id,
Status MemManager::InsertVectorsNoLock(const std::string& table_id,
size_t n,
const float* vectors,
IDNumbers& vector_ids) {
@ -124,8 +113,7 @@ Status MemManager<EngineT>::InsertVectorsNoLock(const std::string& table_id,
return Status::OK();
}
template<typename EngineT>
Status MemManager<EngineT>::ToImmutable() {
Status MemManager::ToImmutable() {
std::unique_lock<std::mutex> lock(mutex_);
for (auto& kv: memMap_) {
immMems_.push_back(kv.second);
@ -135,8 +123,7 @@ Status MemManager<EngineT>::ToImmutable() {
return Status::OK();
}
template<typename EngineT>
Status MemManager<EngineT>::Serialize(std::vector<std::string>& table_ids) {
Status MemManager::Serialize(std::vector<std::string>& table_ids) {
ToImmutable();
std::unique_lock<std::mutex> lock(serialization_mtx_);
std::string table_id;

15
cpp/src/db/MemManager.h
View File

@ -5,6 +5,7 @@
******************************************************************************/
#pragma once
#include "ExecutionEngine.h"
#include "IDGenerator.h"
#include "Status.h"
#include "Meta.h"
@ -23,12 +24,10 @@ namespace meta {
class Meta;
}
template <typename EngineT>
class MemVectors {
public:
using EnginePtr = typename EngineT::Ptr;
using MetaPtr = meta::Meta::Ptr;
using Ptr = std::shared_ptr<MemVectors<EngineT>>;
using Ptr = std::shared_ptr<MemVectors>;
explicit MemVectors(const std::shared_ptr<meta::Meta>&,
const meta::TableFileSchema&, const Options&);
@ -43,7 +42,7 @@ public:
~MemVectors();
const std::string& Location() const { return schema_.location; }
const std::string& Location() const { return schema_.location_; }
private:
MemVectors() = delete;
@ -54,18 +53,17 @@ private:
Options options_;
meta::TableFileSchema schema_;
IDGenerator* pIdGenerator_;
EnginePtr pEE_;
ExecutionEnginePtr pEE_;
}; // MemVectors
template<typename EngineT>
class MemManager {
public:
using MetaPtr = meta::Meta::Ptr;
using MemVectorsPtr = typename MemVectors<EngineT>::Ptr;
using Ptr = std::shared_ptr<MemManager<EngineT>>;
using MemVectorsPtr = typename MemVectors::Ptr;
using Ptr = std::shared_ptr<MemManager>;
MemManager(const std::shared_ptr<meta::Meta>& meta, const Options& options)
: pMeta_(meta), options_(options) {}
@ -96,4 +94,3 @@ private:
} // namespace engine
} // namespace vecwise
} // namespace zilliz
#include "MemManager.inl"

37
cpp/src/db/MetaTypes.h
View File

@ -5,6 +5,8 @@
******************************************************************************/
#pragma once
#include "ExecutionEngine.h"
#include <vector>
#include <map>
#include <string>
@ -19,12 +21,14 @@ const DateT EmptyDate = -1;
typedef std::vector<DateT> DatesT;
struct TableSchema {
size_t id;
std::string table_id;
size_t files_cnt = 0;
uint16_t dimension;
std::string location;
long created_on;
size_t id_;
std::string table_id_;
size_t files_cnt_ = 0;
uint16_t dimension_;
std::string location_;
long created_on_;
int engine_type_ = (int)EngineType::FAISS_IDMAP;
bool store_raw_data_ = false;
}; // TableSchema
struct TableFileSchema {
@ -36,16 +40,17 @@ struct TableFileSchema {
TO_DELETE,
} FILE_TYPE;
size_t id;
std::string table_id;
std::string file_id;
int file_type = NEW;
size_t size;
DateT date = EmptyDate;
uint16_t dimension;
std::string location;
long updated_time;
long created_on;
size_t id_;
std::string table_id_;
int engine_type_ = (int)EngineType::FAISS_IDMAP;
std::string file_id_;
int file_type_ = NEW;
size_t size_;
DateT date_ = EmptyDate;
uint16_t dimension_;
std::string location_;
long updated_time_;
long created_on_;
}; // TableFileSchema
typedef std::vector<TableFileSchema> TableFilesSchema;

20
cpp/src/db/Traits.cpp
View File

@ -1,20 +0,0 @@
/*******************************************************************************
* Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved
* Unauthorized copying of this file, via any medium is strictly prohibited.
* Proprietary and confidential.
******************************************************************************/
#include "Traits.h"
namespace zilliz {
namespace vecwise {
namespace engine {
const char* IVFIndexTrait::BuildIndexType = "IVF";
const char* IVFIndexTrait::RawIndexType = "IDMap,Flat";
const char* IDMapIndexTrait::BuildIndexType = "IDMap";
const char* IDMapIndexTrait::RawIndexType = "IDMap,Flat";
} // namespace engine
} // namespace vecwise
} // namespace zilliz

2
cpp/src/db/Types.h
View File

@ -15,7 +15,7 @@ typedef long IDNumber;
typedef IDNumber* IDNumberPtr;
typedef std::vector<IDNumber> IDNumbers;
typedef std::vector<IDNumber> QueryResult;
typedef std::vector<std::pair<IDNumber, double>> QueryResult;
typedef std::vector<QueryResult> QueryResults;

8
cpp/src/db/scheduler/ScheduleStrategy.cpp
View File

@ -24,21 +24,21 @@ public:
SearchContext::Id2IndexMap index_files = search_context->GetIndexMap();
//some index loader alread exists
for(auto& loader : loader_list) {
if(index_files.find(loader->file_->id) != index_files.end()){
if(index_files.find(loader->file_->id_) != index_files.end()){
SERVER_LOG_INFO << "Append SearchContext to exist IndexLoaderContext";
index_files.erase(loader->file_->id);
index_files.erase(loader->file_->id_);
loader->search_contexts_.push_back(search_context);
}
}
//index_files still contains some index files, create new loader
for(auto& pair : index_files) {
SERVER_LOG_INFO << "Create new IndexLoaderContext for: " << pair.second->location;
SERVER_LOG_INFO << "Create new IndexLoaderContext for: " << pair.second->location_;
IndexLoaderContextPtr new_loader = std::make_shared<IndexLoaderContext>();
new_loader->search_contexts_.push_back(search_context);
new_loader->file_ = pair.second;
auto index = zilliz::vecwise::cache::CpuCacheMgr::GetInstance()->GetIndex(pair.second->location);
auto index = zilliz::vecwise::cache::CpuCacheMgr::GetInstance()->GetIndex(pair.second->location_);
if(index != nullptr) {
//if the index file has been in memory, increase its priority
loader_list.push_front(new_loader);

6
cpp/src/db/scheduler/SearchContext.cpp
View File

@ -26,13 +26,13 @@ SearchContext::SearchContext(uint64_t topk, uint64_t nq, const float* vectors)
bool
SearchContext::AddIndexFile(TableFileSchemaPtr& index_file) {
std::unique_lock <std::mutex> lock(mtx_);
if(index_file == nullptr || map_index_files_.find(index_file->id) != map_index_files_.end()) {
if(index_file == nullptr || map_index_files_.find(index_file->id_) != map_index_files_.end()) {
return false;
}
SERVER_LOG_INFO << "SearchContext " << identity_ << " add index file: " << index_file->id;
SERVER_LOG_INFO << "SearchContext " << identity_ << " add index file: " << index_file->id_;
map_index_files_[index_file->id] = index_file;
map_index_files_[index_file->id_] = index_file;
return true;
}

4
cpp/src/db/scheduler/SearchContext.h
View File

@ -31,8 +31,8 @@ public:
using Id2IndexMap = std::unordered_map<size_t, TableFileSchemaPtr>;
const Id2IndexMap& GetIndexMap() const { return map_index_files_; }
using Score2IdMap = std::map<float, int64_t>;
using ResultSet = std::vector<Score2IdMap>;
using Id2ScoreMap = std::vector<std::pair<int64_t, double>>;
using ResultSet = std::vector<Id2ScoreMap>;
const ResultSet& GetResult() const { return result_; }
ResultSet& GetResult() { return result_; }

94
cpp/src/db/scheduler/SearchScheduler.cpp
View File

@ -10,11 +10,50 @@
#include "utils/Log.h"
#include "utils/TimeRecorder.h"
#include "metrics/Metrics.h"
#include "db/EngineFactory.h"
namespace zilliz {
namespace vecwise {
namespace engine {
namespace {
void CollectFileMetrics(int file_type, size_t file_size) {
switch(file_type) {
case meta::TableFileSchema::RAW:
case meta::TableFileSchema::TO_INDEX: {
server::Metrics::GetInstance().RawFileSizeHistogramObserve(file_size);
server::Metrics::GetInstance().RawFileSizeTotalIncrement(file_size);
server::Metrics::GetInstance().RawFileSizeGaugeSet(file_size);
break;
}
default: {
server::Metrics::GetInstance().IndexFileSizeHistogramObserve(file_size);
server::Metrics::GetInstance().IndexFileSizeTotalIncrement(file_size);
server::Metrics::GetInstance().IndexFileSizeGaugeSet(file_size);
break;
}
}
}
void CollectDurationMetrics(int index_type, double total_time) {
switch(index_type) {
case meta::TableFileSchema::RAW: {
server::Metrics::GetInstance().SearchRawDataDurationSecondsHistogramObserve(total_time);
break;
}
case meta::TableFileSchema::TO_INDEX: {
server::Metrics::GetInstance().SearchRawDataDurationSecondsHistogramObserve(total_time);
break;
}
default: {
server::Metrics::GetInstance().SearchIndexDataDurationSecondsHistogramObserve(total_time);
break;
}
}
}
}
SearchScheduler::SearchScheduler()
: thread_pool_(2),
stopped_(true) {
@ -75,45 +114,27 @@ SearchScheduler::IndexLoadWorker() {
break;//exit
}
SERVER_LOG_INFO << "Loading index(" << context->file_->id << ") from location: " << context->file_->location;
SERVER_LOG_INFO << "Loading index(" << context->file_->id_ << ") from location: " << context->file_->location_;
server::TimeRecorder rc("Load index");
//load index
IndexEnginePtr index_ptr = std::make_shared<IndexClass>(context->file_->dimension, context->file_->location);
//step 1: load index
ExecutionEnginePtr index_ptr = EngineFactory::Build(context->file_->dimension_,
context->file_->location_,
(EngineType)context->file_->engine_type_);
index_ptr->Load();
rc.Record("load index file to memory");
size_t file_size = index_ptr->PhysicalSize();
LOG(DEBUG) << "Index file type " << context->file_->file_type << " Of Size: "
LOG(DEBUG) << "Index file type " << context->file_->file_type_ << " Of Size: "
<< file_size/(1024*1024) << " M";
//metric
switch(context->file_->file_type) {
case meta::TableFileSchema::RAW: {
server::Metrics::GetInstance().RawFileSizeHistogramObserve(file_size);
server::Metrics::GetInstance().RawFileSizeTotalIncrement(file_size);
server::Metrics::GetInstance().RawFileSizeGaugeSet(file_size);
break;
}
case meta::TableFileSchema::TO_INDEX: {
server::Metrics::GetInstance().RawFileSizeHistogramObserve(file_size);
server::Metrics::GetInstance().RawFileSizeTotalIncrement(file_size);
server::Metrics::GetInstance().RawFileSizeGaugeSet(file_size);
break;
}
default: {
server::Metrics::GetInstance().IndexFileSizeHistogramObserve(file_size);
server::Metrics::GetInstance().IndexFileSizeTotalIncrement(file_size);
server::Metrics::GetInstance().IndexFileSizeGaugeSet(file_size);
break;
}
}
CollectFileMetrics(context->file_->file_type_, file_size);
//put search task to another queue
SearchTaskPtr task_ptr = std::make_shared<SearchTaskClass>();
task_ptr->index_id_ = context->file_->id;
task_ptr->index_type_ = context->file_->file_type;
//step 2: put search task into another queue
SearchTaskPtr task_ptr = std::make_shared<SearchTask>();
task_ptr->index_id_ = context->file_->id_;
task_ptr->index_type_ = context->file_->file_type_;
task_ptr->index_engine_ = index_ptr;
task_ptr->search_contexts_.swap(context->search_contexts_);
search_queue.Put(task_ptr);
@ -140,20 +161,7 @@ SearchScheduler::SearchWorker() {
task_ptr->DoSearch();
auto end_time = METRICS_NOW_TIME;
auto total_time = METRICS_MICROSECONDS(start_time, end_time);
switch(task_ptr->index_type_) {
case meta::TableFileSchema::RAW: {
server::Metrics::GetInstance().SearchRawDataDurationSecondsHistogramObserve(total_time);
break;
}
case meta::TableFileSchema::TO_INDEX: {
server::Metrics::GetInstance().SearchRawDataDurationSecondsHistogramObserve(total_time);
break;
}
default: {
server::Metrics::GetInstance().SearchIndexDataDurationSecondsHistogramObserve(total_time);
break;
}
}
CollectDurationMetrics(task_ptr->index_type_, total_time);
}
return true;

81
cpp/src/db/scheduler/SearchTaskQueue.inl → cpp/src/db/scheduler/SearchTaskQueue.cpp
View File

@ -3,8 +3,6 @@
* Unauthorized copying of this file, via any medium is strictly prohibited.
* Proprietary and confidential.
******************************************************************************/
#pragma once
#include "SearchTaskQueue.h"
#include "utils/Log.h"
#include "utils/TimeRecorder.h"
@ -21,12 +19,29 @@ void ClusterResult(const std::vector<long> &output_ids,
SearchContext::ResultSet &result_set) {
result_set.clear();
for (auto i = 0; i < nq; i++) {
SearchContext::Score2IdMap score2id;
SearchContext::Id2ScoreMap id_score;
for (auto k = 0; k < topk; k++) {
uint64_t index = i * nq + k;
score2id.insert(std::make_pair(output_distence[index], output_ids[index]));
id_score.push_back(std::make_pair(output_ids[index], output_distence[index]));
}
result_set.emplace_back(score2id);
result_set.emplace_back(id_score);
}
}
void MergeResult(SearchContext::Id2ScoreMap &score_src,
SearchContext::Id2ScoreMap &score_target,
uint64_t topk) {
for (auto& pair_src : score_src) {
for (auto iter = score_target.begin(); iter != score_target.end(); ++iter) {
if(pair_src.second > iter->second) {
score_target.insert(iter, pair_src);
}
}
}
//remove unused items
while (score_target.size() > topk) {
score_target.pop_back();
}
}
@ -44,18 +59,39 @@ void TopkResult(SearchContext::ResultSet &result_src,
}
for (size_t i = 0; i < result_src.size(); i++) {
SearchContext::Score2IdMap &score2id_src = result_src[i];
SearchContext::Score2IdMap &score2id_target = result_target[i];
for (auto iter = score2id_src.begin(); iter != score2id_src.end(); ++iter) {
score2id_target.insert(std::make_pair(iter->first, iter->second));
}
//remove unused items
while (score2id_target.size() > topk) {
score2id_target.erase(score2id_target.rbegin()->first);
}
SearchContext::Id2ScoreMap &score_src = result_src[i];
SearchContext::Id2ScoreMap &score_target = result_target[i];
MergeResult(score_src, score_target, topk);
}
}
void CalcScore(uint64_t vector_count,
const float *vectors_data,
uint64_t dimension,
const SearchContext::ResultSet &result_src,
SearchContext::ResultSet &result_target) {
result_target.clear();
if(result_src.empty()){
return;
}
int vec_index = 0;
for(auto& result : result_src) {
const float * vec_data = vectors_data + vec_index*dimension;
double vec_len = 0;
for(uint64_t i = 0; i < dimension; i++) {
vec_len += vec_data[i]*vec_data[i];
}
vec_index++;
SearchContext::Id2ScoreMap score_array;
for(auto& pair : result) {
score_array.push_back(std::make_pair(pair.first, (1 - pair.second/vec_len)*100.0));
}
result_target.emplace_back(score_array);
}
}
}
@ -70,8 +106,7 @@ SearchTaskQueue::GetInstance() {
return instance;
}
template<typename trait>
bool SearchTask<trait>::DoSearch() {
bool SearchTask::DoSearch() {
if(index_engine_ == nullptr) {
return false;
}
@ -81,10 +116,12 @@ bool SearchTask<trait>::DoSearch() {
std::vector<long> output_ids;
std::vector<float> output_distence;
for(auto& context : search_contexts_) {
//step 1: allocate memory
auto inner_k = index_engine_->Count() < context->topk() ? index_engine_->Count() : context->topk();
output_ids.resize(inner_k*context->nq());
output_distence.resize(inner_k*context->nq());
//step 2: search
try {
index_engine_->Search(context->nq(), context->vectors(), inner_k, output_distence.data(),
output_ids.data());
@ -96,11 +133,21 @@ bool SearchTask<trait>::DoSearch() {
rc.Record("do search");
//step 3: cluster result
SearchContext::ResultSet result_set;
ClusterResult(output_ids, output_distence, context->nq(), inner_k, result_set);
rc.Record("cluster result");
//step 4: pick up topk result
TopkResult(result_set, inner_k, context->GetResult());
rc.Record("reduce topk");
//step 5: calculate score between 0 ~ 100
CalcScore(context->nq(), context->vectors(), index_engine_->Dimension(), context->GetResult(), result_set);
context->GetResult().swap(result_set);
rc.Record("calculate score");
//step 6: notify to send result to client
context->IndexSearchDone(index_id_);
}

22
cpp/src/db/scheduler/SearchTaskQueue.h
View File

@ -7,8 +7,7 @@
#include "SearchContext.h"
#include "utils/BlockingQueue.h"
#include "../FaissExecutionEngine.h"
#include "../Traits.h"
#include "db/ExecutionEngine.h"
#include <memory>
@ -16,16 +15,6 @@ namespace zilliz {
namespace vecwise {
namespace engine {
#ifdef GPU_VERSION
using IndexTraitClass = IVFIndexTrait;
#else
using IndexTraitClass = IDMapIndexTrait;
#endif
using IndexClass = FaissExecutionEngine<IndexTraitClass>;
using IndexEnginePtr = std::shared_ptr<IndexClass>;
template <typename trait>
class SearchTask {
public:
bool DoSearch();
@ -33,12 +22,11 @@ public:
public:
size_t index_id_ = 0;
int index_type_ = 0; //for metrics
IndexEnginePtr index_engine_;
ExecutionEnginePtr index_engine_;
std::vector<SearchContextPtr> search_contexts_;
};
using SearchTaskClass = SearchTask<IndexTraitClass>;
using SearchTaskPtr = std::shared_ptr<SearchTaskClass>;
using SearchTaskPtr = std::shared_ptr<SearchTask>;
class SearchTaskQueue : public server::BlockingQueue<SearchTaskPtr> {
private:
@ -58,6 +46,4 @@ private:
}
}
}
#include "SearchTaskQueue.inl"
}

82
cpp/src/sdk/examples/simple/src/ClientTest.cpp
View File

@ -16,9 +16,6 @@ namespace {
std::string GetTableName();
static const std::string TABLE_NAME = GetTableName();
static const std::string VECTOR_COLUMN_NAME = "face_vector";
static const std::string ID_COLUMN_NAME = "aid";
static const std::string CITY_COLUMN_NAME = "city";
static constexpr int64_t TABLE_DIMENSION = 512;
static constexpr int64_t TOTAL_ROW_COUNT = 100000;
static constexpr int64_t TOP_K = 10;
@ -29,9 +26,9 @@ namespace {
void PrintTableSchema(const megasearch::TableSchema& tb_schema) {
BLOCK_SPLITER
std::cout << "Table name: " << tb_schema.table_name << std::endl;
std::cout << "Table vectors: " << tb_schema.vector_column_array.size() << std::endl;
std::cout << "Table attributes: " << tb_schema.attribute_column_array.size() << std::endl;
std::cout << "Table partitions: " << tb_schema.partition_column_name_array.size() << std::endl;
std::cout << "Table index type: " << (int)tb_schema.index_type << std::endl;
std::cout << "Table dimension: " << tb_schema.dimension << std::endl;
std::cout << "Table store raw data: " << tb_schema.store_raw_vector << std::endl;
BLOCK_SPLITER
}
@ -58,9 +55,6 @@ namespace {
<< " search result:" << std::endl;
for(auto& item : result.query_result_arrays) {
std::cout << "\t" << std::to_string(item.id) << "\tscore:" << std::to_string(item.score);
for(auto& attribute : item.column_map) {
std::cout << "\t" << attribute.first << ":" << attribute.second;
}
std::cout << std::endl;
}
}
@ -88,73 +82,30 @@ namespace {
TableSchema BuildTableSchema() {
TableSchema tb_schema;
VectorColumn col1;
col1.name = VECTOR_COLUMN_NAME;
col1.dimension = TABLE_DIMENSION;
col1.store_raw_vector = true;
tb_schema.vector_column_array.emplace_back(col1);
Column col2 = {ColumnType::int8, ID_COLUMN_NAME};
tb_schema.attribute_column_array.emplace_back(col2);
Column col3 = {ColumnType::int16, CITY_COLUMN_NAME};
tb_schema.attribute_column_array.emplace_back(col3);
tb_schema.table_name = TABLE_NAME;
tb_schema.index_type = IndexType::gpu_ivfflat;
tb_schema.dimension = TABLE_DIMENSION;
tb_schema.store_raw_vector = true;
return tb_schema;
}
void BuildVectors(int64_t from, int64_t to,
std::vector<RowRecord>* vector_record_array,
std::vector<QueryRecord>* query_record_array) {
std::vector<RowRecord>& vector_record_array) {
if(to <= from){
return;
}
if(vector_record_array) {
vector_record_array->clear();
}
if(query_record_array) {
query_record_array->clear();
}
static const std::map<int64_t , std::string> CITY_MAP = {
{0, "Beijing"},
{1, "Shanhai"},
{2, "Hangzhou"},
{3, "Guangzhou"},
{4, "Shenzheng"},
{5, "Wuhan"},
{6, "Chengdu"},
{7, "Chongqin"},
{8, "Tianjing"},
{9, "Hongkong"},
};
vector_record_array.clear();
for (int64_t k = from; k < to; k++) {
std::vector<float> f_p;
f_p.resize(TABLE_DIMENSION);
RowRecord record;
record.data.resize(TABLE_DIMENSION);
for(int64_t i = 0; i < TABLE_DIMENSION; i++) {
f_p[i] = (float)(i + k);
record.data[i] = (float)(i + k);
}
if(vector_record_array) {
RowRecord record;
record.vector_map.insert(std::make_pair(VECTOR_COLUMN_NAME, f_p));
record.attribute_map[ID_COLUMN_NAME] = std::to_string(k);
record.attribute_map[CITY_COLUMN_NAME] = CITY_MAP.at(k%CITY_MAP.size());
vector_record_array->emplace_back(record);
}
if(query_record_array) {
QueryRecord record;
record.vector_map.insert(std::make_pair(VECTOR_COLUMN_NAME, f_p));
record.selected_column_array.push_back(ID_COLUMN_NAME);
record.selected_column_array.push_back(CITY_COLUMN_NAME);
query_record_array->emplace_back(record);
}
vector_record_array.emplace_back(record);
}
}
}
@ -205,7 +156,7 @@ ClientTest::Test(const std::string& address, const std::string& port) {
{//add vectors
std::vector<RowRecord> record_array;
BuildVectors(0, TOTAL_ROW_COUNT, &record_array, nullptr);
BuildVectors(0, TOTAL_ROW_COUNT, record_array);
std::vector<int64_t> record_ids;
Status stat = conn->AddVector(TABLE_NAME, record_array, record_ids);
std::cout << "AddVector function call status: " << stat.ToString() << std::endl;
@ -215,11 +166,12 @@ ClientTest::Test(const std::string& address, const std::string& port) {
{//search vectors
std::cout << "Waiting data persist. Sleep 10 seconds ..." << std::endl;
sleep(10);
std::vector<QueryRecord> record_array;
BuildVectors(SEARCH_TARGET, SEARCH_TARGET + 10, nullptr, &record_array);
std::vector<RowRecord> record_array;
BuildVectors(SEARCH_TARGET, SEARCH_TARGET + 10, record_array);
std::vector<Range> query_range_array;
std::vector<TopKQueryResult> topk_query_result_array;
Status stat = conn->SearchVector(TABLE_NAME, record_array, topk_query_result_array, TOP_K);
Status stat = conn->SearchVector(TABLE_NAME, record_array, query_range_array, TOP_K, topk_query_result_array);
std::cout << "SearchVector function call status: " << stat.ToString() << std::endl;
PrintSearchResult(topk_query_result_array);
}

146
cpp/src/sdk/include/MegaSearch.h
View File

@ -4,7 +4,6 @@
#include <string>
#include <vector>
#include <map>
#include <memory>
/** \brief MegaSearch SDK namespace
@ -12,129 +11,70 @@
namespace megasearch {
/**
* @brief Column Type
*/
enum class ColumnType {
invalid,
int8,
int16,
int32,
int64,
float32,
float64,
date,
vector
};
/**
* @brief Index Type
*/
enum class IndexType {
raw,
ivfflat
invalid = 0,
cpu_idmap,
gpu_ivfflat,
};
/**
* @brief Connect API parameter
*/
struct ConnectParam {
std::string ip_address; ///< Server IP address
std::string port; ///< Server PORT
};
/**
* @brief Table column description
*/
struct Column {
ColumnType type = ColumnType::invalid; ///< Column Type: enum ColumnType
std::string name; ///< Column name
};
/**
* @brief Table vector column description
*/
struct VectorColumn : public Column {
VectorColumn() { type = ColumnType::vector; }
int64_t dimension = 0; ///< Vector dimension
IndexType index_type = IndexType::raw; ///< Index type
bool store_raw_vector = false; ///< Is vector self stored in the table
std::string ip_address; ///< Server IP address
std::string port; ///< Server PORT
};
/**
* @brief Table Schema
*/
struct TableSchema {
std::string table_name; ///< Table name
std::vector<VectorColumn> vector_column_array; ///< Vector column description
std::vector<Column> attribute_column_array; ///< Columns description
std::vector<std::string> partition_column_name_array; ///< Partition column name
std::string table_name; ///< Table name
IndexType index_type = IndexType::invalid; ///< Index type
int64_t dimension = 0; ///< Vector dimension, must be a positive value
bool store_raw_vector = false; ///< Is vector raw data stored in the table
};
/**
* @brief Range information
* for DATE partition, the format is like: 'year-month-day'
*/
struct Range {
std::string start_value; ///< Range start
std::string end_value; ///< Range stop
};
/**
* @brief Create table partition parameters
*/
struct CreateTablePartitionParam {
std::string table_name; ///< Table name, vector/float32/float64 type column is not allowed for partition
std::string partition_name; ///< Partition name, created partition name
std::map<std::string, Range> range_map; ///< Column name to PartitionRange map
};
/**
* @brief Delete table partition parameters
*/
struct DeleteTablePartitionParam {
std::string table_name; ///< Table name
std::vector<std::string> partition_name_array; ///< Partition name array
std::string start_value; ///< Range start
std::string end_value; ///< Range stop
};
/**
* @brief Record inserted
*/
struct RowRecord {
std::map<std::string, std::vector<float>> vector_map; ///< Vector columns
std::map<std::string, std::string> attribute_map; ///< Other attribute columns
};
/**
* @brief Query record
*/
struct QueryRecord {
std::map<std::string, std::vector<float>> vector_map; ///< Query vectors
std::vector<std::string> selected_column_array; ///< Output column array
std::map<std::string, std::vector<Range>> partition_filter_column_map; ///< Range used to select partitions
std::vector<float> data; ///< Vector raw data
};
/**
* @brief Query result
*/
struct QueryResult {
int64_t id; ///< Output result
double score; ///< Vector similarity score: 0 ~ 100
std::map<std::string, std::string> column_map; ///< Other column
int64_t id; ///< Output result
double score; ///< Vector similarity score: 0 ~ 100
};
/**
* @brief TopK query result
*/
struct TopKQueryResult {
std::vector<QueryResult> query_result_arrays; ///< TopK query result
std::vector<QueryResult> query_result_arrays; ///< TopK query result
};
/**
* @brief SDK main class
*/
class Connection {
public:
public:
/**
* @brief CreateConnection
@ -228,30 +168,6 @@ class Connection {
virtual Status DeleteTable(const std::string &table_name) = 0;
/**
* @brief Create table partition
*
* This method is used to create table partition.
*
* @param param, use to provide partition information to be created.
*
* @return Indicate if table partition is created successfully.
*/
virtual Status CreateTablePartition(const CreateTablePartitionParam &param) = 0;
/**
* @brief Delete table partition
*
* This method is used to delete table partition.
*
* @param param, use to provide partition information to be deleted.
*
* @return Indicate if table partition is delete successfully.
*/
virtual Status DeleteTablePartition(const DeleteTablePartitionParam &param) = 0;
/**
* @brief Add vector to table
*
@ -264,8 +180,8 @@ class Connection {
* @return Indicate if vector array are inserted successfully
*/
virtual Status AddVector(const std::string &table_name,
const std::vector<RowRecord> &record_array,
std::vector<int64_t> &id_array) = 0;
const std::vector<RowRecord> &record_array,
std::vector<int64_t> &id_array) = 0;
/**
@ -275,15 +191,17 @@ class Connection {
*
* @param table_name, table_name is queried.
* @param query_record_array, all vector are going to be queried.
* @param topk_query_result_array, result array.
* @param query_range_array, time ranges, if not specified, will search in whole table
* @param topk, how many similarity vectors will be searched.
* @param topk_query_result_array, result array.
*
* @return Indicate if query is successful.
*/
virtual Status SearchVector(const std::string &table_name,
const std::vector<QueryRecord> &query_record_array,
std::vector<TopKQueryResult> &topk_query_result_array,
int64_t topk) = 0;
const std::vector<RowRecord> &query_record_array,
const std::vector<Range> &query_range_array,
int64_t topk,
std::vector<TopKQueryResult> &topk_query_result_array) = 0;
/**
* @brief Show table description
@ -297,6 +215,18 @@ class Connection {
*/
virtual Status DescribeTable(const std::string &table_name, TableSchema &table_schema) = 0;
/**
* @brief Get table row count
*
* This method is used to get table row count.
*
* @param table_name, table's name.
* @param row_count, table total row count.
*
* @return Indicate if this operation is successful.
*/
virtual Status GetTableRowCount(const std::string &table_name, int64_t &row_count) = 0;
/**
* @brief Show all tables in database
*

15
cpp/src/sdk/include/Status.h
View File

@ -12,10 +12,13 @@ namespace megasearch {
*/
enum class StatusCode {
OK = 0,
Invalid = 1,
UnknownError = 2,
NotSupported = 3,
NotConnected = 4
// system error section
UnknownError = 1,
NotSupported,
NotConnected,
// function error section
InvalidAgument = 1000,
};
/**
@ -171,7 +174,7 @@ class Status {
*/
template<typename... Args>
static Status Invalid(Args &&... args) {
return Status(StatusCode::Invalid,
return Status(StatusCode::InvalidAgument,
MessageBuilder(std::forward<Args>(args)...));
}
@ -221,7 +224,7 @@ class Status {
* @return, if the status indicates invalid.
*
*/
bool IsInvalid() const { return code() == StatusCode::Invalid; }
bool IsInvalid() const { return code() == StatusCode::InvalidAgument; }
/**
* @brief IsUnknownError

151
cpp/src/sdk/src/client/ClientProxy.cpp
View File

@ -89,27 +89,9 @@ ClientProxy::CreateTable(const TableSchema &param) {
try {
thrift::TableSchema schema;
schema.__set_table_name(param.table_name);
std::vector<thrift::VectorColumn> vector_column_array;
for(auto& column : param.vector_column_array) {
thrift::VectorColumn col;
col.__set_dimension(column.dimension);
col.__set_index_type(ConvertUtil::IndexType2Str(column.index_type));
col.__set_store_raw_vector(column.store_raw_vector);
vector_column_array.emplace_back(col);
}
schema.__set_vector_column_array(vector_column_array);
std::vector<thrift::Column> attribute_column_array;
for(auto& column : param.attribute_column_array) {
thrift::Column col;
col.__set_name(col.name);
col.__set_type(col.type);
attribute_column_array.emplace_back(col);
}
schema.__set_attribute_column_array(attribute_column_array);
schema.__set_partition_column_name_array(param.partition_column_name_array);
schema.__set_index_type((int)param.index_type);
schema.__set_dimension(param.dimension);
schema.__set_store_raw_vector(param.store_raw_vector);
ClientPtr()->interface()->CreateTable(schema);
@ -120,55 +102,6 @@ ClientProxy::CreateTable(const TableSchema &param) {
return Status::OK();
}
Status
ClientProxy::CreateTablePartition(const CreateTablePartitionParam &param) {
if(!IsConnected()) {
return Status(StatusCode::NotConnected, "not connected to server");
}
try {
thrift::CreateTablePartitionParam partition_param;
partition_param.__set_table_name(param.table_name);
partition_param.__set_partition_name(param.partition_name);
std::map<std::string, thrift::Range> range_map;
for(auto& pair : param.range_map) {
thrift::Range range;
range.__set_start_value(pair.second.start_value);
range.__set_end_value(pair.second.end_value);
range_map.insert(std::make_pair(pair.first, range));
}
partition_param.__set_range_map(range_map);
ClientPtr()->interface()->CreateTablePartition(partition_param);
} catch ( std::exception& ex) {
return Status(StatusCode::UnknownError, "failed to create table partition: " + std::string(ex.what()));
}
return Status::OK();
}
Status
ClientProxy::DeleteTablePartition(const DeleteTablePartitionParam &param) {
if(!IsConnected()) {
return Status(StatusCode::NotConnected, "not connected to server");
}
try {
thrift::DeleteTablePartitionParam partition_param;
partition_param.__set_table_name(param.table_name);
partition_param.__set_partition_name_array(param.partition_name_array);
ClientPtr()->interface()->DeleteTablePartition(partition_param);
} catch ( std::exception& ex) {
return Status(StatusCode::UnknownError, "failed to delete table partition: " + std::string(ex.what()));
}
return Status::OK();
}
Status
ClientProxy::DeleteTable(const std::string &table_name) {
if(!IsConnected()) {
@ -197,17 +130,13 @@ ClientProxy::AddVector(const std::string &table_name,
std::vector<thrift::RowRecord> thrift_records;
for(auto& record : record_array) {
thrift::RowRecord thrift_record;
thrift_record.__set_attribute_map(record.attribute_map);
for(auto& pair : record.vector_map) {
size_t dim = pair.second.size();
std::string& thrift_vector = thrift_record.vector_map[pair.first];
thrift_vector.resize(dim * sizeof(double));
double *dbl = (double *) (const_cast<char *>(thrift_vector.data()));
for (size_t i = 0; i < dim; i++) {
dbl[i] = (double) (pair.second[i]);
}
thrift_record.vector_data.resize(record.data.size() * sizeof(double));
double *dbl = (double *) (const_cast<char *>(thrift_record.vector_data.data()));
for (size_t i = 0; i < record.data.size(); i++) {
dbl[i] = (double) (record.data[i]);
}
thrift_records.emplace_back(thrift_record);
}
ClientPtr()->interface()->AddVector(id_array, table_name, thrift_records);
@ -221,33 +150,31 @@ ClientProxy::AddVector(const std::string &table_name,
Status
ClientProxy::SearchVector(const std::string &table_name,
const std::vector<QueryRecord> &query_record_array,
std::vector<TopKQueryResult> &topk_query_result_array,
int64_t topk) {
const std::vector<RowRecord> &query_record_array,
const std::vector<Range> &query_range_array,
int64_t topk,
std::vector<TopKQueryResult> &topk_query_result_array) {
if(!IsConnected()) {
return Status(StatusCode::NotConnected, "not connected to server");
}
try {
std::vector<thrift::QueryRecord> thrift_records;
std::vector<thrift::RowRecord> thrift_records;
for(auto& record : query_record_array) {
thrift::QueryRecord thrift_record;
thrift_record.__set_selected_column_array(record.selected_column_array);
thrift::RowRecord thrift_record;
for(auto& pair : record.vector_map) {
size_t dim = pair.second.size();
std::string& thrift_vector = thrift_record.vector_map[pair.first];
thrift_vector.resize(dim * sizeof(double));
double *dbl = (double *) (const_cast<char *>(thrift_vector.data()));
for (size_t i = 0; i < dim; i++) {
dbl[i] = (double) (pair.second[i]);
}
thrift_record.vector_data.resize(record.data.size() * sizeof(double));
double *dbl = (double *) (const_cast<char *>(thrift_record.vector_data.data()));
for (size_t i = 0; i < record.data.size(); i++) {
dbl[i] = (double) (record.data[i]);
}
thrift_records.emplace_back(thrift_record);
}
std::vector<thrift::Range> thrift_ranges;
std::vector<thrift::TopKQueryResult> result_array;
ClientPtr()->interface()->SearchVector(result_array, table_name, thrift_records, topk);
ClientPtr()->interface()->SearchVector(result_array, table_name, thrift_records, thrift_ranges, topk);
for(auto& thrift_topk_result : result_array) {
TopKQueryResult result;
@ -255,7 +182,6 @@ ClientProxy::SearchVector(const std::string &table_name,
for(auto& thrift_query_result : thrift_topk_result.query_result_arrays) {
QueryResult query_result;
query_result.id = thrift_query_result.id;
query_result.column_map = thrift_query_result.column_map;
query_result.score = thrift_query_result.score;
result.query_result_arrays.emplace_back(query_result);
}
@ -281,24 +207,7 @@ ClientProxy::DescribeTable(const std::string &table_name, TableSchema &table_sch
ClientPtr()->interface()->DescribeTable(thrift_schema, table_name);
table_schema.table_name = thrift_schema.table_name;
table_schema.partition_column_name_array = thrift_schema.partition_column_name_array;
for(auto& thrift_col : thrift_schema.attribute_column_array) {
Column col;
col.name = col.name;
col.type = col.type;
table_schema.attribute_column_array.emplace_back(col);
}
for(auto& thrift_col : thrift_schema.vector_column_array) {
VectorColumn col;
col.store_raw_vector = thrift_col.store_raw_vector;
col.index_type = ConvertUtil::Str2IndexType(thrift_col.index_type);
col.dimension = thrift_col.dimension;
col.name = thrift_col.base.name;
col.type = (ColumnType)thrift_col.base.type;
table_schema.vector_column_array.emplace_back(col);
}
table_schema.index_type = (IndexType)thrift_schema.index_type;
} catch ( std::exception& ex) {
return Status(StatusCode::UnknownError, "failed to describe table: " + std::string(ex.what()));
@ -307,6 +216,22 @@ ClientProxy::DescribeTable(const std::string &table_name, TableSchema &table_sch
return Status::OK();
}
Status
ClientProxy::GetTableRowCount(const std::string &table_name, int64_t &row_count) {
if(!IsConnected()) {
return Status(StatusCode::NotConnected, "not connected to server");
}
try {
row_count = ClientPtr()->interface()->GetTableRowCount(table_name);
} catch ( std::exception& ex) {
return Status(StatusCode::UnknownError, "failed to show tables: " + std::string(ex.what()));
}
return Status::OK();
}
Status
ClientProxy::ShowTables(std::vector<std::string> &table_array) {
if(!IsConnected()) {

13
cpp/src/sdk/src/client/ClientProxy.h
View File

@ -25,21 +25,20 @@ public:
virtual Status DeleteTable(const std::string &table_name) override;
virtual Status CreateTablePartition(const CreateTablePartitionParam &param) override;
virtual Status DeleteTablePartition(const DeleteTablePartitionParam &param) override;
virtual Status AddVector(const std::string &table_name,
const std::vector<RowRecord> &record_array,
std::vector<int64_t> &id_array) override;
virtual Status SearchVector(const std::string &table_name,
const std::vector<QueryRecord> &query_record_array,
std::vector<TopKQueryResult> &topk_query_result_array,
int64_t topk) override;
const std::vector<RowRecord> &query_record_array,
const std::vector<Range> &query_range_array,
int64_t topk,
std::vector<TopKQueryResult> &topk_query_result_array) override;
virtual Status DescribeTable(const std::string &table_name, TableSchema &table_schema) override;
virtual Status GetTableRowCount(const std::string &table_name, int64_t &row_count) override;
virtual Status ShowTables(std::vector<std::string> &table_array) override;
virtual std::string ClientVersion() const override;

2
cpp/src/sdk/src/client/ThriftClient.cpp
View File

@ -58,7 +58,7 @@ ThriftClient::Connect(const std::string& address, int32_t port, const std::strin
protocol_ptr.reset(new TCompactProtocol(transport_ptr));
} else {
//CLIENT_LOG_ERROR << "Service protocol: " << protocol << " is not supported currently";
return Status(StatusCode::Invalid, "unsupported protocol");
return Status(StatusCode::InvalidAgument, "unsupported protocol");
}
transport_ptr->open();

24
cpp/src/sdk/src/interface/ConnectionImpl.cpp
View File

@ -55,16 +55,6 @@ ConnectionImpl::CreateTable(const TableSchema &param) {
return client_proxy_->CreateTable(param);
}
Status
ConnectionImpl::CreateTablePartition(const CreateTablePartitionParam &param) {
return client_proxy_->CreateTablePartition(param);
}
Status
ConnectionImpl::DeleteTablePartition(const DeleteTablePartitionParam &param) {
return client_proxy_->DeleteTablePartition(param);
}
Status
ConnectionImpl::DeleteTable(const std::string &table_name) {
return client_proxy_->DeleteTable(table_name);
@ -79,10 +69,11 @@ ConnectionImpl::AddVector(const std::string &table_name,
Status
ConnectionImpl::SearchVector(const std::string &table_name,
const std::vector<QueryRecord> &query_record_array,
std::vector<TopKQueryResult> &topk_query_result_array,
int64_t topk) {
return client_proxy_->SearchVector(table_name, query_record_array, topk_query_result_array, topk);
const std::vector<RowRecord> &query_record_array,
const std::vector<Range> &query_range_array,
int64_t topk,
std::vector<TopKQueryResult> &topk_query_result_array) {
return client_proxy_->SearchVector(table_name, query_record_array, query_range_array, topk, topk_query_result_array);
}
Status
@ -90,6 +81,11 @@ ConnectionImpl::DescribeTable(const std::string &table_name, TableSchema &table_
return client_proxy_->DescribeTable(table_name, table_schema);
}
Status
ConnectionImpl::GetTableRowCount(const std::string &table_name, int64_t &row_count) {
return client_proxy_->GetTableRowCount(table_name, row_count);
}
Status
ConnectionImpl::ShowTables(std::vector<std::string> &table_array) {
return client_proxy_->ShowTables(table_array);

13
cpp/src/sdk/src/interface/ConnectionImpl.h
View File

@ -27,21 +27,20 @@ public:
virtual Status DeleteTable(const std::string &table_name) override;
virtual Status CreateTablePartition(const CreateTablePartitionParam &param) override;
virtual Status DeleteTablePartition(const DeleteTablePartitionParam &param) override;
virtual Status AddVector(const std::string &table_name,
const std::vector<RowRecord> &record_array,
std::vector<int64_t> &id_array) override;
virtual Status SearchVector(const std::string &table_name,
const std::vector<QueryRecord> &query_record_array,
std::vector<TopKQueryResult> &topk_query_result_array,
int64_t topk) override;
const std::vector<RowRecord> &query_record_array,
const std::vector<Range> &query_range_array,
int64_t topk,
std::vector<TopKQueryResult> &topk_query_result_array) override;
virtual Status DescribeTable(const std::string &table_name, TableSchema &table_schema) override;
virtual Status GetTableRowCount(const std::string &table_name, int64_t &row_count) override;
virtual Status ShowTables(std::vector<std::string> &table_array) override;
virtual std::string ClientVersion() const override;

2
cpp/src/sdk/src/interface/Status.cpp
View File

@ -95,7 +95,7 @@ std::string Status::CodeAsString() const {
switch (code()) {
case StatusCode::OK: type = "OK";
break;
case StatusCode::Invalid: type = "Invalid";
case StatusCode::InvalidAgument: type = "Invalid agument";
break;
case StatusCode::UnknownError: type = "Unknown error";
break;

12
cpp/src/sdk/src/util/ConvertUtil.cpp
View File

@ -15,13 +15,13 @@ static const std::string INDEX_IVFFLAT = "ivfflat";
std::string ConvertUtil::IndexType2Str(megasearch::IndexType index) {
static const std::map<megasearch::IndexType, std::string> s_index2str = {
{megasearch::IndexType::raw, INDEX_RAW},
{megasearch::IndexType::ivfflat, INDEX_IVFFLAT}
{megasearch::IndexType::cpu_idmap, INDEX_RAW},
{megasearch::IndexType::gpu_ivfflat, INDEX_IVFFLAT}
};
const auto& iter = s_index2str.find(index);
if(iter == s_index2str.end()) {
throw Exception(StatusCode::Invalid, "Invalid index type");
throw Exception(StatusCode::InvalidAgument, "Invalid index type");
}
return iter->second;
@ -29,13 +29,13 @@ std::string ConvertUtil::IndexType2Str(megasearch::IndexType index) {
megasearch::IndexType ConvertUtil::Str2IndexType(const std::string& type) {
static const std::map<std::string, megasearch::IndexType> s_str2index = {
{INDEX_RAW, megasearch::IndexType::raw},
{INDEX_IVFFLAT, megasearch::IndexType::ivfflat}
{INDEX_RAW, megasearch::IndexType::cpu_idmap},
{INDEX_IVFFLAT, megasearch::IndexType::gpu_ivfflat}
};
const auto& iter = s_str2index.find(type);
if(iter == s_str2index.end()) {
throw Exception(StatusCode::Invalid, "Invalid index type");
throw Exception(StatusCode::InvalidAgument, "Invalid index type");
}
return iter->second;

35
cpp/src/server/MegasearchHandler.cpp
View File

@ -30,18 +30,6 @@ MegasearchServiceHandler::DeleteTable(const std::string &table_name) {
MegasearchScheduler::ExecTask(task_ptr);
}
void
MegasearchServiceHandler::CreateTablePartition(const thrift::CreateTablePartitionParam &param) {
BaseTaskPtr task_ptr = CreateTablePartitionTask::Create(param);
MegasearchScheduler::ExecTask(task_ptr);
}
void
MegasearchServiceHandler::DeleteTablePartition(const thrift::DeleteTablePartitionParam &param) {
BaseTaskPtr task_ptr = DeleteTablePartitionTask::Create(param);
MegasearchScheduler::ExecTask(task_ptr);
}
void
MegasearchServiceHandler::AddVector(std::vector<int64_t> &_return,
const std::string &table_name,
@ -51,11 +39,12 @@ MegasearchServiceHandler::AddVector(std::vector<int64_t> &_return,
}
void
MegasearchServiceHandler::SearchVector(std::vector<thrift::TopKQueryResult> &_return,
const std::string &table_name,
const std::vector<thrift::QueryRecord> &query_record_array,
const int64_t topk) {
BaseTaskPtr task_ptr = SearchVectorTask::Create(table_name, query_record_array, topk, _return);
MegasearchServiceHandler::SearchVector(std::vector<megasearch::thrift::TopKQueryResult> & _return,
const std::string& table_name,
const std::vector<megasearch::thrift::RowRecord> & query_record_array,
const std::vector<megasearch::thrift::Range> & query_range_array,
const int64_t topk) {
BaseTaskPtr task_ptr = SearchVectorTask::Create(table_name, query_record_array, query_range_array, topk, _return);
MegasearchScheduler::ExecTask(task_ptr);
}
@ -65,6 +54,18 @@ MegasearchServiceHandler::DescribeTable(thrift::TableSchema &_return, const std:
MegasearchScheduler::ExecTask(task_ptr);
}
int64_t
MegasearchServiceHandler::GetTableRowCount(const std::string& table_name) {
int64_t row_count = 0;
{
BaseTaskPtr task_ptr = GetTableRowCountTask::Create(table_name, row_count);
MegasearchScheduler::ExecTask(task_ptr);
task_ptr->WaitToFinish();
}
return row_count;
}
void
MegasearchServiceHandler::ShowTables(std::vector<std::string> &_return) {
BaseTaskPtr task_ptr = ShowTablesTask::Create(_return);

66
cpp/src/server/MegasearchHandler.h
View File

@ -19,15 +19,15 @@ public:
MegasearchServiceHandler();
/**
* @brief Create table method
*
* This method is used to create table
*
* @param param, use to provide table information to be created.
*
*
* @param param
*/
* @brief Create table method
*
* This method is used to create table
*
* @param param, use to provide table information to be created.
*
*
* @param param
*/
void CreateTable(const megasearch::thrift::TableSchema& param);
/**
@ -42,30 +42,6 @@ public:
*/
void DeleteTable(const std::string& table_name);
/**
* @brief Create table partition
*
* This method is used to create table partition.
*
* @param param, use to provide partition information to be created.
*
*
* @param param
*/
void CreateTablePartition(const megasearch::thrift::CreateTablePartitionParam& param);
/**
* @brief Delete table partition
*
* This method is used to delete table partition.
*
* @param param, use to provide partition information to be deleted.
*
*
* @param param
*/
void DeleteTablePartition(const megasearch::thrift::DeleteTablePartitionParam& param);
/**
* @brief Add vector array to table
*
@ -90,25 +66,28 @@ public:
*
* @param table_name, table_name is queried.
* @param query_record_array, all vector are going to be queried.
* @param query_range_array, optional ranges for conditional search. If not specified, search whole table
* @param topk, how many similarity vectors will be searched.
*
* @return query result array.
*
* @param table_name
* @param query_record_array
* @param query_range_array
* @param topk
*/
void SearchVector(std::vector<megasearch::thrift::TopKQueryResult> & _return,
const std::string& table_name,
const std::vector<megasearch::thrift::QueryRecord> & query_record_array,
const std::vector<megasearch::thrift::RowRecord> & query_record_array,
const std::vector<megasearch::thrift::Range> & query_range_array,
const int64_t topk);
/**
* @brief Show table information
* @brief Get table schema
*
* This method is used to show table information.
* This method is used to get table schema.
*
* @param table_name, which table is show.
* @param table_name, target table name.
*
* @return table schema
*
@ -116,6 +95,19 @@ public:
*/
void DescribeTable(megasearch::thrift::TableSchema& _return, const std::string& table_name);
/**
* @brief Get table row count
*
* This method is used to get table row count.
*
* @param table_name, target table name.
*
* @return table row count
*
* @param table_name
*/
int64_t GetTableRowCount(const std::string& table_name);
/**
* @brief List all tables in database
*

4
cpp/src/server/MegasearchScheduler.cpp
View File

@ -24,7 +24,7 @@ namespace {
{SERVER_FILE_NOT_FOUND, thrift::ErrorCode::ILLEGAL_ARGUMENT},
{SERVER_NOT_IMPLEMENT, thrift::ErrorCode::ILLEGAL_ARGUMENT},
{SERVER_BLOCKING_QUEUE_EMPTY, thrift::ErrorCode::ILLEGAL_ARGUMENT},
{SERVER_GROUP_NOT_EXIST, thrift::ErrorCode::TABLE_NOT_EXISTS},
{SERVER_TABLE_NOT_EXIST, thrift::ErrorCode::TABLE_NOT_EXISTS},
{SERVER_INVALID_TIME_RANGE, thrift::ErrorCode::ILLEGAL_RANGE},
{SERVER_INVALID_VECTOR_DIMENSION, thrift::ErrorCode::ILLEGAL_DIMENSION},
};
@ -40,7 +40,7 @@ namespace {
{SERVER_FILE_NOT_FOUND, "file not found"},
{SERVER_NOT_IMPLEMENT, "not implemented"},
{SERVER_BLOCKING_QUEUE_EMPTY, "queue empty"},
{SERVER_GROUP_NOT_EXIST, "group not exist"},
{SERVER_TABLE_NOT_EXIST, "table not exist"},
{SERVER_INVALID_TIME_RANGE, "invalid time range"},
{SERVER_INVALID_VECTOR_DIMENSION, "invalid vector dimension"},
};

202
cpp/src/server/MegasearchTask.cpp
View File

@ -5,15 +5,13 @@
******************************************************************************/
#include "MegasearchTask.h"
#include "ServerConfig.h"
#include "VecIdMapper.h"
#include "utils/CommonUtil.h"
#include "utils/Log.h"
#include "utils/TimeRecorder.h"
#include "utils/ThreadPool.h"
#include "db/DB.h"
#include "db/Env.h"
#include "db/Meta.h"
#include "version.h"
namespace zilliz {
namespace vecwise {
@ -64,9 +62,18 @@ namespace {
return db_wrapper.DB();
}
ThreadPool& GetThreadPool() {
static ThreadPool pool(6);
return pool;
engine::EngineType EngineType(int type) {
static std::map<int, engine::EngineType> map_type = {
{0, engine::EngineType::INVALID},
{1, engine::EngineType::FAISS_IDMAP},
{2, engine::EngineType::FAISS_IVFFLAT},
};
if(map_type.find(type) == map_type.end()) {
return engine::EngineType::INVALID;
}
return map_type[type];
}
}
@ -85,16 +92,20 @@ ServerError CreateTableTask::OnExecute() {
TimeRecorder rc("CreateTableTask");
try {
if(schema_.vector_column_array.empty()) {
if(schema_.table_name.empty() || schema_.dimension == 0 || schema_.index_type == 0) {
return SERVER_INVALID_ARGUMENT;
}
IVecIdMapper::GetInstance()->AddGroup(schema_.table_name);
//step 1: construct table schema
engine::meta::TableSchema table_info;
table_info.dimension = (uint16_t)schema_.vector_column_array[0].dimension;
table_info.table_id = schema_.table_name;
table_info.dimension_ = (uint16_t)schema_.dimension;
table_info.table_id_ = schema_.table_name;
table_info.engine_type_ = (int)EngineType(schema_.index_type);
table_info.store_raw_data_ = schema_.store_raw_vector;
//step 2: create table
engine::Status stat = DB()->CreateTable(table_info);
if(!stat.ok()) {//could exist
if(!stat.ok()) {//table could exist
error_msg_ = "Engine failed: " + stat.ToString();
SERVER_LOG_ERROR << error_msg_;
return SERVER_SUCCESS;
@ -129,10 +140,10 @@ ServerError DescribeTableTask::OnExecute() {
try {
engine::meta::TableSchema table_info;
table_info.table_id = table_name_;
table_info.table_id_ = table_name_;
engine::Status stat = DB()->DescribeTable(table_info);
if(!stat.ok()) {
error_code_ = SERVER_GROUP_NOT_EXIST;
error_code_ = SERVER_TABLE_NOT_EXIST;
error_msg_ = "Engine failed: " + stat.ToString();
SERVER_LOG_ERROR << error_msg_;
return error_code_;
@ -168,46 +179,6 @@ ServerError DeleteTableTask::OnExecute() {
error_msg_ = "delete table not implemented";
SERVER_LOG_ERROR << error_msg_;
IVecIdMapper::GetInstance()->DeleteGroup(table_name_);
return SERVER_NOT_IMPLEMENT;
}
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
CreateTablePartitionTask::CreateTablePartitionTask(const thrift::CreateTablePartitionParam &param)
: BaseTask(DDL_DML_TASK_GROUP),
param_(param) {
}
BaseTaskPtr CreateTablePartitionTask::Create(const thrift::CreateTablePartitionParam &param) {
return std::shared_ptr<BaseTask>(new CreateTablePartitionTask(param));
}
ServerError CreateTablePartitionTask::OnExecute() {
error_code_ = SERVER_NOT_IMPLEMENT;
error_msg_ = "create table partition not implemented";
SERVER_LOG_ERROR << error_msg_;
return SERVER_NOT_IMPLEMENT;
}
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
DeleteTablePartitionTask::DeleteTablePartitionTask(const thrift::DeleteTablePartitionParam &param)
: BaseTask(DDL_DML_TASK_GROUP),
param_(param) {
}
BaseTaskPtr DeleteTablePartitionTask::Create(const thrift::DeleteTablePartitionParam &param) {
return std::shared_ptr<BaseTask>(new DeleteTablePartitionTask(param));
}
ServerError DeleteTablePartitionTask::OnExecute() {
error_code_ = SERVER_NOT_IMPLEMENT;
error_msg_ = "delete table partition not implemented";
SERVER_LOG_ERROR << error_msg_;
return SERVER_NOT_IMPLEMENT;
}
@ -223,7 +194,6 @@ BaseTaskPtr ShowTablesTask::Create(std::vector<std::string>& tables) {
}
ServerError ShowTablesTask::OnExecute() {
IVecIdMapper::GetInstance()->AllGroups(tables_);
return SERVER_SUCCESS;
}
@ -253,31 +223,33 @@ ServerError AddVectorTask::OnExecute() {
return SERVER_SUCCESS;
}
//step 1: check table existence
engine::meta::TableSchema table_info;
table_info.table_id = table_name_;
table_info.table_id_ = table_name_;
engine::Status stat = DB()->DescribeTable(table_info);
if(!stat.ok()) {
error_code_ = SERVER_GROUP_NOT_EXIST;
error_code_ = SERVER_TABLE_NOT_EXIST;
error_msg_ = "Engine failed: " + stat.ToString();
SERVER_LOG_ERROR << error_msg_;
return error_code_;
}
rc.Record("get group info");
rc.Record("check validation");
//step 2: prepare float data
uint64_t vec_count = (uint64_t)record_array_.size();
uint64_t group_dim = table_info.dimension;
uint64_t group_dim = table_info.dimension_;
std::vector<float> vec_f;
vec_f.resize(vec_count*group_dim);//allocate enough memory
for(uint64_t i = 0; i < vec_count; i++) {
const auto& record = record_array_[i];
if(record.vector_map.empty()) {
if(record.vector_data.empty()) {
error_code_ = SERVER_INVALID_ARGUMENT;
error_msg_ = "No vector provided in record";
SERVER_LOG_ERROR << error_msg_;
return error_code_;
}
uint64_t vec_dim = record.vector_map.begin()->second.size()/sizeof(double);//how many double value?
uint64_t vec_dim = record.vector_data.size()/sizeof(double);//how many double value?
if(vec_dim != group_dim) {
SERVER_LOG_ERROR << "Invalid vector dimension: " << vec_dim
<< " vs. group dimension:" << group_dim;
@ -287,7 +259,7 @@ ServerError AddVectorTask::OnExecute() {
}
//convert double array to float array(thrift has no float type)
const double* d_p = reinterpret_cast<const double*>(record.vector_map.begin()->second.data());
const double* d_p = reinterpret_cast<const double*>(record.vector_data.data());
for(uint64_t d = 0; d < vec_dim; d++) {
vec_f[i*vec_dim + d] = (float)(d_p[d]);
}
@ -295,6 +267,7 @@ ServerError AddVectorTask::OnExecute() {
rc.Record("prepare vectors data");
//step 3: insert vectors
stat = DB()->InsertVectors(table_name_, vec_count, vec_f.data(), record_ids_);
rc.Record("add vectors to engine");
if(!stat.ok()) {
@ -309,22 +282,8 @@ ServerError AddVectorTask::OnExecute() {
return SERVER_UNEXPECTED_ERROR;
}
//persist attributes
for(uint64_t i = 0; i < vec_count; i++) {
const auto &record = record_array_[i];
//any attributes?
if(record.attribute_map.empty()) {
continue;
}
std::string nid = std::to_string(record_ids_[i]);
std::string attrib_str;
AttributeSerializer::Encode(record.attribute_map, attrib_str);
IVecIdMapper::GetInstance()->Put(nid, attrib_str, table_name_);
}
rc.Record("persist vector attributes");
rc.Record("do insert");
rc.Elapse("totally cost");
} catch (std::exception& ex) {
error_code_ = SERVER_UNEXPECTED_ERROR;
@ -338,28 +297,33 @@ ServerError AddVectorTask::OnExecute() {
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
SearchVectorTask::SearchVectorTask(const std::string& table_name,
const std::vector<thrift::RowRecord> & query_record_array,
const std::vector<megasearch::thrift::Range> & query_range_array,
const int64_t top_k,
const std::vector<thrift::QueryRecord>& record_array,
std::vector<thrift::TopKQueryResult>& result_array)
: BaseTask(DQL_TASK_GROUP),
table_name_(table_name),
record_array_(query_record_array),
range_array_(query_range_array),
top_k_(top_k),
record_array_(record_array),
result_array_(result_array) {
}
BaseTaskPtr SearchVectorTask::Create(const std::string& table_name,
const std::vector<thrift::QueryRecord>& record_array,
const std::vector<thrift::RowRecord> & query_record_array,
const std::vector<megasearch::thrift::Range> & query_range_array,
const int64_t top_k,
std::vector<thrift::TopKQueryResult>& result_array) {
return std::shared_ptr<BaseTask>(new SearchVectorTask(table_name, top_k, record_array, result_array));
return std::shared_ptr<BaseTask>(new SearchVectorTask(table_name,
query_record_array, query_range_array, top_k, result_array));
}
ServerError SearchVectorTask::OnExecute() {
try {
TimeRecorder rc("SearchVectorTask");
//step 1: check validation
if(top_k_ <= 0 || record_array_.empty()) {
error_code_ = SERVER_INVALID_ARGUMENT;
error_msg_ = "Invalid topk value, or query record array is empty";
@ -367,40 +331,44 @@ ServerError SearchVectorTask::OnExecute() {
return error_code_;
}
//step 2: check table existence
engine::meta::TableSchema table_info;
table_info.table_id = table_name_;
table_info.table_id_ = table_name_;
engine::Status stat = DB()->DescribeTable(table_info);
if(!stat.ok()) {
error_code_ = SERVER_GROUP_NOT_EXIST;
error_code_ = SERVER_TABLE_NOT_EXIST;
error_msg_ = "Engine failed: " + stat.ToString();
SERVER_LOG_ERROR << error_msg_;
return error_code_;
}
rc.Record("check validation");
//step 3: prepare float data
std::vector<float> vec_f;
uint64_t record_count = (uint64_t)record_array_.size();
vec_f.resize(record_count*table_info.dimension);
vec_f.resize(record_count*table_info.dimension_);
for(uint64_t i = 0; i < record_array_.size(); i++) {
const auto& record = record_array_[i];
if (record.vector_map.empty()) {
if (record.vector_data.empty()) {
error_code_ = SERVER_INVALID_ARGUMENT;
error_msg_ = "Query record has no vector";
SERVER_LOG_ERROR << error_msg_;
return error_code_;
}
uint64_t vec_dim = record.vector_map.begin()->second.size() / sizeof(double);//how many double value?
if (vec_dim != table_info.dimension) {
uint64_t vec_dim = record.vector_data.size() / sizeof(double);//how many double value?
if (vec_dim != table_info.dimension_) {
SERVER_LOG_ERROR << "Invalid vector dimension: " << vec_dim
<< " vs. group dimension:" << table_info.dimension;
<< " vs. group dimension:" << table_info.dimension_;
error_code_ = SERVER_INVALID_VECTOR_DIMENSION;
error_msg_ = "Engine failed: " + stat.ToString();
return error_code_;
}
//convert double array to float array(thrift has no float type)
const double* d_p = reinterpret_cast<const double*>(record.vector_map.begin()->second.data());
const double* d_p = reinterpret_cast<const double*>(record.vector_data.data());
for(uint64_t d = 0; d < vec_dim; d++) {
vec_f[i*vec_dim + d] = (float)(d_p[d]);
}
@ -408,6 +376,8 @@ ServerError SearchVectorTask::OnExecute() {
rc.Record("prepare vector data");
//step 4: search vectors
std::vector<DB_DATE> dates;
engine::QueryResults results;
stat = DB()->Query(table_name_, (size_t)top_k_, record_count, vec_f.data(), dates, results);
@ -422,32 +392,18 @@ ServerError SearchVectorTask::OnExecute() {
return SERVER_UNEXPECTED_ERROR;
}
//construct result array
rc.Record("do search");
//step 5: construct result array
for(uint64_t i = 0; i < record_count; i++) {
auto& result = results[i];
const auto& record = record_array_[i];
thrift::TopKQueryResult thrift_topk_result;
for(auto id : result) {
for(auto& pair : result) {
thrift::QueryResult thrift_result;
thrift_result.__set_id(id);
//need get attributes?
if(record.selected_column_array.empty()) {
thrift_topk_result.query_result_arrays.emplace_back(thrift_result);
continue;
}
std::string nid = std::to_string(id);
std::string attrib_str;
IVecIdMapper::GetInstance()->Get(nid, attrib_str, table_name_);
AttribMap attrib_map;
AttributeSerializer::Decode(attrib_str, attrib_map);
for(auto& attribute : record.selected_column_array) {
thrift_result.column_map[attribute] = attrib_map[attribute];
}
thrift_result.__set_id(pair.first);
thrift_result.__set_score(pair.second);
thrift_topk_result.query_result_arrays.emplace_back(thrift_result);
}
@ -466,6 +422,32 @@ ServerError SearchVectorTask::OnExecute() {
return SERVER_SUCCESS;
}
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
GetTableRowCountTask::GetTableRowCountTask(const std::string& table_name, int64_t& row_count)
: BaseTask(DQL_TASK_GROUP),
table_name_(table_name),
row_count_(row_count) {
}
BaseTaskPtr GetTableRowCountTask::Create(const std::string& table_name, int64_t& row_count) {
return std::shared_ptr<BaseTask>(new GetTableRowCountTask(table_name, row_count));
}
ServerError GetTableRowCountTask::OnExecute() {
if(table_name_.empty()) {
error_code_ = SERVER_UNEXPECTED_ERROR;
error_msg_ = "Table name cannot be empty";
SERVER_LOG_ERROR << error_msg_;
return error_code_;
}
error_code_ = SERVER_NOT_IMPLEMENT;
error_msg_ = "Not implemented";
SERVER_LOG_ERROR << error_msg_;
return error_code_;
}
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
PingTask::PingTask(const std::string& cmd, std::string& result)
: BaseTask(PING_TASK_GROUP),
@ -480,7 +462,7 @@ BaseTaskPtr PingTask::Create(const std::string& cmd, std::string& result) {
ServerError PingTask::OnExecute() {
if(cmd_ == "version") {
result_ = "v1.2.0";//currently hardcode
result_ = MEGASEARCH_VERSION;
}
return SERVER_SUCCESS;

54
cpp/src/server/MegasearchTask.h
View File

@ -65,36 +65,6 @@ private:
std::string table_name_;
};
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
class CreateTablePartitionTask : public BaseTask {
public:
static BaseTaskPtr Create(const thrift::CreateTablePartitionParam &param);
protected:
CreateTablePartitionTask(const thrift::CreateTablePartitionParam &param);
ServerError OnExecute() override;
private:
const thrift::CreateTablePartitionParam &param_;
};
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
class DeleteTablePartitionTask : public BaseTask {
public:
static BaseTaskPtr Create(const thrift::DeleteTablePartitionParam &param);
protected:
DeleteTablePartitionTask(const thrift::DeleteTablePartitionParam &param);
ServerError OnExecute() override;
private:
const thrift::DeleteTablePartitionParam &param_;
};
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
class ShowTablesTask : public BaseTask {
public:
@ -133,14 +103,16 @@ private:
class SearchVectorTask : public BaseTask {
public:
static BaseTaskPtr Create(const std::string& table_name,
const std::vector<thrift::QueryRecord>& record_array,
const std::vector<thrift::RowRecord> & query_record_array,
const std::vector<megasearch::thrift::Range> & query_range_array,
const int64_t top_k,
std::vector<thrift::TopKQueryResult>& result_array);
protected:
SearchVectorTask(const std::string& table_name,
const std::vector<thrift::RowRecord> & query_record_array,
const std::vector<megasearch::thrift::Range> & query_range_array,
const int64_t top_k,
const std::vector<thrift::QueryRecord>& record_array,
std::vector<thrift::TopKQueryResult>& result_array);
ServerError OnExecute() override;
@ -148,10 +120,26 @@ protected:
private:
std::string table_name_;
int64_t top_k_;
const std::vector<thrift::QueryRecord>& record_array_;
const std::vector<thrift::RowRecord>& record_array_;
const std::vector<megasearch::thrift::Range>& range_array_;
std::vector<thrift::TopKQueryResult>& result_array_;
};
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
class GetTableRowCountTask : public BaseTask {
public:
static BaseTaskPtr Create(const std::string& table_name, int64_t& row_count);
protected:
GetTableRowCountTask(const std::string& table_name, int64_t& row_count);
ServerError OnExecute() override;
private:
std::string table_name_;
int64_t& row_count_;
};
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
class PingTask : public BaseTask {
public:

327
cpp/src/server/RocksIdMapper.cpp
View File

@ -1,327 +0,0 @@
/*******************************************************************************
* Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved
* Unauthorized copying of this file, via any medium is strictly prohibited.
* Proprietary and confidential.
******************************************************************************/
#include "RocksIdMapper.h"
#include "ServerConfig.h"
#include "utils/Log.h"
#include "utils/CommonUtil.h"
#include "rocksdb/db.h"
#include "rocksdb/slice.h"
#include "rocksdb/options.h"
#include <exception>
namespace zilliz {
namespace vecwise {
namespace server {
static const std::string ROCKSDB_DEFAULT_GROUP = "default";
RocksIdMapper::RocksIdMapper()
: db_(nullptr) {
OpenDb();
}
RocksIdMapper::~RocksIdMapper() {
CloseDb();
}
void RocksIdMapper::OpenDb() {
std::lock_guard<std::mutex> lck(db_mutex_);
if(db_) {
return;
}
ConfigNode& config = ServerConfig::GetInstance().GetConfig(CONFIG_DB);
std::string db_path = config.GetValue(CONFIG_DB_PATH);
db_path += "/id_mapping";
CommonUtil::CreateDirectory(db_path);
rocksdb::Options options;
// Optimize RocksDB. This is the easiest way to get RocksDB to perform well
options.IncreaseParallelism();
options.OptimizeLevelStyleCompaction();
// create the DB if it's not already present
options.create_if_missing = true;
options.max_open_files = config.GetInt32Value(CONFIG_DB_IDMAPPER_MAX_FILE, 512);
//load column families
std::vector<std::string> column_names;
rocksdb::Status s = rocksdb::DB::ListColumnFamilies(options, db_path, &column_names);
if (!s.ok()) {
SERVER_LOG_ERROR << "ID mapper failed to initialize:" << s.ToString();
}
if(column_names.empty()) {
column_names.push_back("default");
}
SERVER_LOG_INFO << "ID mapper has " << std::to_string(column_names.size()) << " groups";
std::vector<rocksdb::ColumnFamilyDescriptor> column_families;
for(auto& column_name : column_names) {
rocksdb::ColumnFamilyDescriptor desc;
desc.name = column_name;
column_families.emplace_back(desc);
}
// open DB
std::vector<rocksdb::ColumnFamilyHandle*> column_handles;
s = rocksdb::DB::Open(options, db_path, column_families, &column_handles, &db_);
if(!s.ok()) {
SERVER_LOG_ERROR << "ID mapper failed to initialize:" << s.ToString();
db_ = nullptr;
}
column_handles_.clear();
for(auto handler : column_handles) {
column_handles_.insert(std::make_pair(handler->GetName(), handler));
}
}
void RocksIdMapper::CloseDb() {
std::lock_guard<std::mutex> lck(db_mutex_);
for(auto& iter : column_handles_) {
delete iter.second;
}
column_handles_.clear();
if(db_) {
db_->Close();
delete db_;
}
}
ServerError RocksIdMapper::AddGroup(const std::string& group) {
std::lock_guard<std::mutex> lck(db_mutex_);
return AddGroupInternal(group);
}
bool RocksIdMapper::IsGroupExist(const std::string& group) const {
std::lock_guard<std::mutex> lck(db_mutex_);
return IsGroupExistInternal(group);
}
ServerError RocksIdMapper::AllGroups(std::vector<std::string>& groups) const {
groups.clear();
std::lock_guard<std::mutex> lck(db_mutex_);
for(auto& pair : column_handles_) {
if(pair.first == ROCKSDB_DEFAULT_GROUP) {
continue;
}
groups.push_back(pair.first);
}
return SERVER_SUCCESS;
}
ServerError RocksIdMapper::Put(const std::string& nid, const std::string& sid, const std::string& group) {
std::lock_guard<std::mutex> lck(db_mutex_);
return PutInternal(nid, sid, group);
}
ServerError RocksIdMapper::Put(const std::vector<std::string>& nid, const std::vector<std::string>& sid, const std::string& group) {
if(nid.size() != sid.size()) {
return SERVER_INVALID_ARGUMENT;
}
std::lock_guard<std::mutex> lck(db_mutex_);
ServerError err = SERVER_SUCCESS;
for(size_t i = 0; i < nid.size(); i++) {
err = PutInternal(nid[i], sid[i], group);
if(err != SERVER_SUCCESS) {
return err;
}
}
return err;
}
ServerError RocksIdMapper::Get(const std::string& nid, std::string& sid, const std::string& group) const {
std::lock_guard<std::mutex> lck(db_mutex_);
return GetInternal(nid, sid, group);
}
ServerError RocksIdMapper::Get(const std::vector<std::string>& nid, std::vector<std::string>& sid, const std::string& group) const {
sid.clear();
std::lock_guard<std::mutex> lck(db_mutex_);
ServerError err = SERVER_SUCCESS;
for(size_t i = 0; i < nid.size(); i++) {
std::string str_id;
ServerError temp_err = GetInternal(nid[i], str_id, group);
if(temp_err != SERVER_SUCCESS) {
sid.push_back("");
SERVER_LOG_ERROR << "ID mapper failed to get id: " << nid[i];
err = temp_err;
continue;
}
sid.push_back(str_id);
}
return err;
}
ServerError RocksIdMapper::Delete(const std::string& nid, const std::string& group) {
std::lock_guard<std::mutex> lck(db_mutex_);
return DeleteInternal(nid, group);
}
ServerError RocksIdMapper::DeleteGroup(const std::string& group) {
std::lock_guard<std::mutex> lck(db_mutex_);
return DeleteGroupInternal(group);
}
//internal methods(whitout lock)
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
ServerError RocksIdMapper::AddGroupInternal(const std::string& group) {
if(!IsGroupExistInternal(group)) {
if(db_ == nullptr) {
return SERVER_NULL_POINTER;
}
try {//add group
rocksdb::ColumnFamilyHandle *cfh = nullptr;
rocksdb::Status s = db_->CreateColumnFamily(rocksdb::ColumnFamilyOptions(), group, &cfh);
if (!s.ok()) {
SERVER_LOG_ERROR << "ID mapper failed to create group:" << s.ToString();
return SERVER_UNEXPECTED_ERROR;
} else {
column_handles_.insert(std::make_pair(group, cfh));
}
} catch(std::exception& ex) {
SERVER_LOG_ERROR << "ID mapper failed to create group: " << ex.what();
return SERVER_UNEXPECTED_ERROR;
}
}
return SERVER_SUCCESS;
}
bool RocksIdMapper::IsGroupExistInternal(const std::string& group) const {
std::string group_name = group;
if(group_name.empty()){
group_name = ROCKSDB_DEFAULT_GROUP;
}
return (column_handles_.count(group_name) > 0 && column_handles_[group_name] != nullptr);
}
ServerError RocksIdMapper::PutInternal(const std::string& nid, const std::string& sid, const std::string& group) {
if(db_ == nullptr) {
return SERVER_NULL_POINTER;
}
rocksdb::Slice key(nid);
rocksdb::Slice value(sid);
if(group.empty()) {//to default group
rocksdb::Status s = db_->Put(rocksdb::WriteOptions(), key, value);
if (!s.ok()) {
SERVER_LOG_ERROR << "ID mapper failed to put:" << s.ToString();
return SERVER_UNEXPECTED_ERROR;
}
} else {
//try create group
if(AddGroupInternal(group) != SERVER_SUCCESS){
return SERVER_UNEXPECTED_ERROR;
}
rocksdb::ColumnFamilyHandle *cfh = column_handles_[group];
rocksdb::Status s = db_->Put(rocksdb::WriteOptions(), cfh, key, value);
if (!s.ok()) {
SERVER_LOG_ERROR << "ID mapper failed to put:" << s.ToString();
return SERVER_UNEXPECTED_ERROR;
}
}
return SERVER_SUCCESS;
}
ServerError RocksIdMapper::GetInternal(const std::string& nid, std::string& sid, const std::string& group) const {
sid = "";
if(db_ == nullptr) {
return SERVER_NULL_POINTER;
}
rocksdb::ColumnFamilyHandle *cfh = nullptr;
if(column_handles_.count(group) != 0) {
cfh = column_handles_.at(group);
}
rocksdb::Slice key(nid);
rocksdb::Status s;
if(cfh){
s = db_->Get(rocksdb::ReadOptions(), cfh, key, &sid);
} else {
s = db_->Get(rocksdb::ReadOptions(), key, &sid);
}
if(!s.ok()) {
SERVER_LOG_ERROR << "ID mapper failed to get:" << s.ToString();
return SERVER_UNEXPECTED_ERROR;
}
return SERVER_SUCCESS;
}
ServerError RocksIdMapper::DeleteInternal(const std::string& nid, const std::string& group) {
if(db_ == nullptr) {
return SERVER_NULL_POINTER;
}
rocksdb::ColumnFamilyHandle *cfh = nullptr;
if(column_handles_.count(group) != 0) {
cfh = column_handles_.at(group);
}
rocksdb::Slice key(nid);
rocksdb::Status s;
if(cfh){
s = db_->Delete(rocksdb::WriteOptions(), cfh, key);
} else {
s = db_->Delete(rocksdb::WriteOptions(), key);
}
if(!s.ok()) {
SERVER_LOG_ERROR << "ID mapper failed to delete:" << s.ToString();
return SERVER_UNEXPECTED_ERROR;
}
return SERVER_SUCCESS;
}
ServerError RocksIdMapper::DeleteGroupInternal(const std::string& group) {
if(db_ == nullptr) {
return SERVER_NULL_POINTER;
}
rocksdb::ColumnFamilyHandle *cfh = nullptr;
if(column_handles_.count(group) != 0) {
cfh = column_handles_.at(group);
}
if(cfh) {
db_->DropColumnFamily(cfh);
db_->DestroyColumnFamilyHandle(cfh);
column_handles_.erase(group);
}
return SERVER_SUCCESS;
}
}
}
}

68
cpp/src/server/RocksIdMapper.h
View File

@ -1,68 +0,0 @@
/*******************************************************************************
* Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved
* Unauthorized copying of this file, via any medium is strictly prohibited.
* Proprietary and confidential.
******************************************************************************/
#pragma once
#include "utils/Error.h"
#include "VecIdMapper.h"
#include <string>
#include <vector>
#include <unordered_map>
#include <mutex>
namespace rocksdb {
class DB;
class ColumnFamilyHandle;
}
namespace zilliz {
namespace vecwise {
namespace server {
class RocksIdMapper : public IVecIdMapper{
public:
RocksIdMapper();
~RocksIdMapper();
ServerError AddGroup(const std::string& group) override;
bool IsGroupExist(const std::string& group) const override;
ServerError AllGroups(std::vector<std::string>& groups) const override;
ServerError Put(const std::string& nid, const std::string& sid, const std::string& group = "") override;
ServerError Put(const std::vector<std::string>& nid, const std::vector<std::string>& sid, const std::string& group = "") override;
ServerError Get(const std::string& nid, std::string& sid, const std::string& group = "") const override;
ServerError Get(const std::vector<std::string>& nid, std::vector<std::string>& sid, const std::string& group = "") const override;
ServerError Delete(const std::string& nid, const std::string& group = "") override;
ServerError DeleteGroup(const std::string& group) override;
private:
void OpenDb();
void CloseDb();
ServerError AddGroupInternal(const std::string& group);
bool IsGroupExistInternal(const std::string& group) const;
ServerError PutInternal(const std::string& nid, const std::string& sid, const std::string& group);
ServerError GetInternal(const std::string& nid, std::string& sid, const std::string& group) const;
ServerError DeleteInternal(const std::string& nid, const std::string& group);
ServerError DeleteGroupInternal(const std::string& group);
private:
rocksdb::DB* db_;
mutable std::unordered_map<std::string, rocksdb::ColumnFamilyHandle*> column_handles_;
mutable std::mutex db_mutex_;
};
}
}
}

162
cpp/src/server/VecIdMapper.cpp
View File

@ -1,162 +0,0 @@
/*******************************************************************************
* Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved
* Unauthorized copying of this file, via any medium is strictly prohibited.
* Proprietary and confidential.
******************************************************************************/
#include "VecIdMapper.h"
#include "RocksIdMapper.h"
#include "ServerConfig.h"
#include "utils/Log.h"
#include "utils/CommonUtil.h"
#include "rocksdb/db.h"
#include "rocksdb/slice.h"
#include "rocksdb/options.h"
#include <exception>
#include <unordered_map>
namespace zilliz {
namespace vecwise {
namespace server {
IVecIdMapper* IVecIdMapper::GetInstance() {
#if 0
static SimpleIdMapper s_mapper;
return &s_mapper;
#else
static RocksIdMapper s_mapper;
return &s_mapper;
#endif
}
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
class SimpleIdMapper : public IVecIdMapper{
public:
SimpleIdMapper();
~SimpleIdMapper();
ServerError AddGroup(const std::string& group) override;
bool IsGroupExist(const std::string& group) const override;
ServerError AllGroups(std::vector<std::string>& groups) const override;
ServerError Put(const std::string& nid, const std::string& sid, const std::string& group = "") override;
ServerError Put(const std::vector<std::string>& nid, const std::vector<std::string>& sid, const std::string& group = "") override;
ServerError Get(const std::string& nid, std::string& sid, const std::string& group = "") const override;
ServerError Get(const std::vector<std::string>& nid, std::vector<std::string>& sid, const std::string& group = "") const override;
ServerError Delete(const std::string& nid, const std::string& group = "") override;
ServerError DeleteGroup(const std::string& group) override;
private:
using ID_MAPPING = std::unordered_map<std::string, std::string>;
mutable std::unordered_map<std::string, ID_MAPPING> id_groups_;
};
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
SimpleIdMapper::SimpleIdMapper() {
}
SimpleIdMapper::~SimpleIdMapper() {
}
ServerError
SimpleIdMapper::AddGroup(const std::string& group) {
if(id_groups_.count(group) == 0) {
id_groups_.insert(std::make_pair(group, ID_MAPPING()));
}
}
//not thread-safe
bool
SimpleIdMapper::IsGroupExist(const std::string& group) const {
return id_groups_.count(group) > 0;
}
ServerError SimpleIdMapper::AllGroups(std::vector<std::string>& groups) const {
groups.clear();
for(auto& pair : id_groups_) {
groups.push_back(pair.first);
}
return SERVER_SUCCESS;
}
//not thread-safe
ServerError SimpleIdMapper::Put(const std::string& nid, const std::string& sid, const std::string& group) {
ID_MAPPING& mapping = id_groups_[group];
mapping[nid] = sid;
return SERVER_SUCCESS;
}
//not thread-safe
ServerError SimpleIdMapper::Put(const std::vector<std::string>& nid, const std::vector<std::string>& sid, const std::string& group) {
if(nid.size() != sid.size()) {
return SERVER_INVALID_ARGUMENT;
}
ID_MAPPING& mapping = id_groups_[group];
for(size_t i = 0; i < nid.size(); i++) {
mapping[nid[i]] = sid[i];
}
return SERVER_SUCCESS;
}
//not thread-safe
ServerError SimpleIdMapper::Get(const std::string& nid, std::string& sid, const std::string& group) const {
ID_MAPPING& mapping = id_groups_[group];
auto iter = mapping.find(nid);
if(iter == mapping.end()) {
return SERVER_INVALID_ARGUMENT;
}
sid = iter->second;
return SERVER_SUCCESS;
}
//not thread-safe
ServerError SimpleIdMapper::Get(const std::vector<std::string>& nid, std::vector<std::string>& sid, const std::string& group) const {
sid.clear();
ID_MAPPING& mapping = id_groups_[group];
ServerError err = SERVER_SUCCESS;
for(size_t i = 0; i < nid.size(); i++) {
auto iter = mapping.find(nid[i]);
if(iter == mapping.end()) {
sid.push_back("");
SERVER_LOG_ERROR << "ID mapper failed to find id: " << nid[i];
err = SERVER_INVALID_ARGUMENT;
continue;
}
sid.push_back(iter->second);
}
return err;
}
//not thread-safe
ServerError SimpleIdMapper::Delete(const std::string& nid, const std::string& group) {
ID_MAPPING& mapping = id_groups_[group];
mapping.erase(nid);
return SERVER_SUCCESS;
}
//not thread-safe
ServerError SimpleIdMapper::DeleteGroup(const std::string& group) {
id_groups_.erase(group);
return SERVER_SUCCESS;
}
}
}
}

40
cpp/src/server/VecIdMapper.h
View File

@ -1,40 +0,0 @@
/*******************************************************************************
* Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved
* Unauthorized copying of this file, via any medium is strictly prohibited.
* Proprietary and confidential.
******************************************************************************/
#pragma once
#include "utils/Error.h"
#include <string>
#include <vector>
namespace zilliz {
namespace vecwise {
namespace server {
class IVecIdMapper {
public:
static IVecIdMapper* GetInstance();
virtual ~IVecIdMapper(){}
virtual ServerError AddGroup(const std::string& group) = 0;
virtual bool IsGroupExist(const std::string& group) const = 0;
virtual ServerError AllGroups(std::vector<std::string>& groups) const = 0;
virtual ServerError Put(const std::string& nid, const std::string& sid, const std::string& group = "") = 0;
virtual ServerError Put(const std::vector<std::string>& nid, const std::vector<std::string>& sid, const std::string& group = "") = 0;
virtual ServerError Get(const std::string& nid, std::string& sid, const std::string& group = "") const = 0;
//NOTE: the 'sid' will be cleared at begin of the function
virtual ServerError Get(const std::vector<std::string>& nid, std::vector<std::string>& sid, const std::string& group = "") const = 0;
virtual ServerError Delete(const std::string& nid, const std::string& group = "") = 0;
virtual ServerError DeleteGroup(const std::string& group) = 0;
};
}
}
}

1393
cpp/src/thrift/gen-cpp/MegasearchService.cpp
View File

File diff suppressed because it is too large Load Diff

459
cpp/src/thrift/gen-cpp/MegasearchService.h
View File

@ -46,30 +46,6 @@ class MegasearchServiceIf {
*/
virtual void DeleteTable(const std::string& table_name) = 0;
/**
* @brief Create table partition
*
* This method is used to create table partition.
*
* @param param, use to provide partition information to be created.
*
*
* @param param
*/
virtual void CreateTablePartition(const CreateTablePartitionParam& param) = 0;
/**
* @brief Delete table partition
*
* This method is used to delete table partition.
*
* @param param, use to provide partition information to be deleted.
*
*
* @param param
*/
virtual void DeleteTablePartition(const DeleteTablePartitionParam& param) = 0;
/**
* @brief Add vector array to table
*
@ -92,22 +68,24 @@ class MegasearchServiceIf {
*
* @param table_name, table_name is queried.
* @param query_record_array, all vector are going to be queried.
* @param query_range_array, optional ranges for conditional search. If not specified, search whole table
* @param topk, how many similarity vectors will be searched.
*
* @return query result array.
*
* @param table_name
* @param query_record_array
* @param query_range_array
* @param topk
*/
virtual void SearchVector(std::vector<TopKQueryResult> & _return, const std::string& table_name, const std::vector<QueryRecord> & query_record_array, const int64_t topk) = 0;
virtual void SearchVector(std::vector<TopKQueryResult> & _return, const std::string& table_name, const std::vector<RowRecord> & query_record_array, const std::vector<Range> & query_range_array, const int64_t topk) = 0;
/**
* @brief Show table information
* @brief Get table schema
*
* This method is used to show table information.
* This method is used to get table schema.
*
* @param table_name, which table is show.
* @param table_name, target table name.
*
* @return table schema
*
@ -115,6 +93,19 @@ class MegasearchServiceIf {
*/
virtual void DescribeTable(TableSchema& _return, const std::string& table_name) = 0;
/**
* @brief Get table row count
*
* This method is used to get table row count.
*
* @param table_name, target table name.
*
* @return table row count
*
* @param table_name
*/
virtual int64_t GetTableRowCount(const std::string& table_name) = 0;
/**
* @brief List all tables in database
*
@ -170,21 +161,19 @@ class MegasearchServiceNull : virtual public MegasearchServiceIf {
void DeleteTable(const std::string& /* table_name */) {
return;
}
void CreateTablePartition(const CreateTablePartitionParam& /* param */) {
return;
}
void DeleteTablePartition(const DeleteTablePartitionParam& /* param */) {
return;
}
void AddVector(std::vector<int64_t> & /* _return */, const std::string& /* table_name */, const std::vector<RowRecord> & /* record_array */) {
return;
}
void SearchVector(std::vector<TopKQueryResult> & /* _return */, const std::string& /* table_name */, const std::vector<QueryRecord> & /* query_record_array */, const int64_t /* topk */) {
void SearchVector(std::vector<TopKQueryResult> & /* _return */, const std::string& /* table_name */, const std::vector<RowRecord> & /* query_record_array */, const std::vector<Range> & /* query_range_array */, const int64_t /* topk */) {
return;
}
void DescribeTable(TableSchema& /* _return */, const std::string& /* table_name */) {
return;
}
int64_t GetTableRowCount(const std::string& /* table_name */) {
int64_t _return = 0;
return _return;
}
void ShowTables(std::vector<std::string> & /* _return */) {
return;
}
@ -401,214 +390,6 @@ class MegasearchService_DeleteTable_presult {
};
typedef struct _MegasearchService_CreateTablePartition_args__isset {
_MegasearchService_CreateTablePartition_args__isset() : param(false) {}
bool param :1;
} _MegasearchService_CreateTablePartition_args__isset;
class MegasearchService_CreateTablePartition_args {
public:
MegasearchService_CreateTablePartition_args(const MegasearchService_CreateTablePartition_args&);
MegasearchService_CreateTablePartition_args& operator=(const MegasearchService_CreateTablePartition_args&);
MegasearchService_CreateTablePartition_args() {
}
virtual ~MegasearchService_CreateTablePartition_args() throw();
CreateTablePartitionParam param;
_MegasearchService_CreateTablePartition_args__isset __isset;
void __set_param(const CreateTablePartitionParam& val);
bool operator == (const MegasearchService_CreateTablePartition_args & rhs) const
{
if (!(param == rhs.param))
return false;
return true;
}
bool operator != (const MegasearchService_CreateTablePartition_args &rhs) const {
return !(*this == rhs);
}
bool operator < (const MegasearchService_CreateTablePartition_args & ) const;
uint32_t read(::apache::thrift::protocol::TProtocol* iprot);
uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const;
};
class MegasearchService_CreateTablePartition_pargs {
public:
virtual ~MegasearchService_CreateTablePartition_pargs() throw();
const CreateTablePartitionParam* param;
uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const;
};
typedef struct _MegasearchService_CreateTablePartition_result__isset {
_MegasearchService_CreateTablePartition_result__isset() : e(false) {}
bool e :1;
} _MegasearchService_CreateTablePartition_result__isset;
class MegasearchService_CreateTablePartition_result {
public:
MegasearchService_CreateTablePartition_result(const MegasearchService_CreateTablePartition_result&);
MegasearchService_CreateTablePartition_result& operator=(const MegasearchService_CreateTablePartition_result&);
MegasearchService_CreateTablePartition_result() {
}
virtual ~MegasearchService_CreateTablePartition_result() throw();
Exception e;
_MegasearchService_CreateTablePartition_result__isset __isset;
void __set_e(const Exception& val);
bool operator == (const MegasearchService_CreateTablePartition_result & rhs) const
{
if (!(e == rhs.e))
return false;
return true;
}
bool operator != (const MegasearchService_CreateTablePartition_result &rhs) const {
return !(*this == rhs);
}
bool operator < (const MegasearchService_CreateTablePartition_result & ) const;
uint32_t read(::apache::thrift::protocol::TProtocol* iprot);
uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const;
};
typedef struct _MegasearchService_CreateTablePartition_presult__isset {
_MegasearchService_CreateTablePartition_presult__isset() : e(false) {}
bool e :1;
} _MegasearchService_CreateTablePartition_presult__isset;
class MegasearchService_CreateTablePartition_presult {
public:
virtual ~MegasearchService_CreateTablePartition_presult() throw();
Exception e;
_MegasearchService_CreateTablePartition_presult__isset __isset;
uint32_t read(::apache::thrift::protocol::TProtocol* iprot);
};
typedef struct _MegasearchService_DeleteTablePartition_args__isset {
_MegasearchService_DeleteTablePartition_args__isset() : param(false) {}
bool param :1;
} _MegasearchService_DeleteTablePartition_args__isset;
class MegasearchService_DeleteTablePartition_args {
public:
MegasearchService_DeleteTablePartition_args(const MegasearchService_DeleteTablePartition_args&);
MegasearchService_DeleteTablePartition_args& operator=(const MegasearchService_DeleteTablePartition_args&);
MegasearchService_DeleteTablePartition_args() {
}
virtual ~MegasearchService_DeleteTablePartition_args() throw();
DeleteTablePartitionParam param;
_MegasearchService_DeleteTablePartition_args__isset __isset;
void __set_param(const DeleteTablePartitionParam& val);
bool operator == (const MegasearchService_DeleteTablePartition_args & rhs) const
{
if (!(param == rhs.param))
return false;
return true;
}
bool operator != (const MegasearchService_DeleteTablePartition_args &rhs) const {
return !(*this == rhs);
}
bool operator < (const MegasearchService_DeleteTablePartition_args & ) const;
uint32_t read(::apache::thrift::protocol::TProtocol* iprot);
uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const;
};
class MegasearchService_DeleteTablePartition_pargs {
public:
virtual ~MegasearchService_DeleteTablePartition_pargs() throw();
const DeleteTablePartitionParam* param;
uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const;
};
typedef struct _MegasearchService_DeleteTablePartition_result__isset {
_MegasearchService_DeleteTablePartition_result__isset() : e(false) {}
bool e :1;
} _MegasearchService_DeleteTablePartition_result__isset;
class MegasearchService_DeleteTablePartition_result {
public:
MegasearchService_DeleteTablePartition_result(const MegasearchService_DeleteTablePartition_result&);
MegasearchService_DeleteTablePartition_result& operator=(const MegasearchService_DeleteTablePartition_result&);
MegasearchService_DeleteTablePartition_result() {
}
virtual ~MegasearchService_DeleteTablePartition_result() throw();
Exception e;
_MegasearchService_DeleteTablePartition_result__isset __isset;
void __set_e(const Exception& val);
bool operator == (const MegasearchService_DeleteTablePartition_result & rhs) const
{
if (!(e == rhs.e))
return false;
return true;
}
bool operator != (const MegasearchService_DeleteTablePartition_result &rhs) const {
return !(*this == rhs);
}
bool operator < (const MegasearchService_DeleteTablePartition_result & ) const;
uint32_t read(::apache::thrift::protocol::TProtocol* iprot);
uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const;
};
typedef struct _MegasearchService_DeleteTablePartition_presult__isset {
_MegasearchService_DeleteTablePartition_presult__isset() : e(false) {}
bool e :1;
} _MegasearchService_DeleteTablePartition_presult__isset;
class MegasearchService_DeleteTablePartition_presult {
public:
virtual ~MegasearchService_DeleteTablePartition_presult() throw();
Exception e;
_MegasearchService_DeleteTablePartition_presult__isset __isset;
uint32_t read(::apache::thrift::protocol::TProtocol* iprot);
};
typedef struct _MegasearchService_AddVector_args__isset {
_MegasearchService_AddVector_args__isset() : table_name(false), record_array(false) {}
bool table_name :1;
@ -729,9 +510,10 @@ class MegasearchService_AddVector_presult {
};
typedef struct _MegasearchService_SearchVector_args__isset {
_MegasearchService_SearchVector_args__isset() : table_name(false), query_record_array(false), topk(false) {}
_MegasearchService_SearchVector_args__isset() : table_name(false), query_record_array(false), query_range_array(false), topk(false) {}
bool table_name :1;
bool query_record_array :1;
bool query_range_array :1;
bool topk :1;
} _MegasearchService_SearchVector_args__isset;
@ -745,14 +527,17 @@ class MegasearchService_SearchVector_args {
virtual ~MegasearchService_SearchVector_args() throw();
std::string table_name;
std::vector<QueryRecord> query_record_array;
std::vector<RowRecord> query_record_array;
std::vector<Range> query_range_array;
int64_t topk;
_MegasearchService_SearchVector_args__isset __isset;
void __set_table_name(const std::string& val);
void __set_query_record_array(const std::vector<QueryRecord> & val);
void __set_query_record_array(const std::vector<RowRecord> & val);
void __set_query_range_array(const std::vector<Range> & val);
void __set_topk(const int64_t val);
@ -762,6 +547,8 @@ class MegasearchService_SearchVector_args {
return false;
if (!(query_record_array == rhs.query_record_array))
return false;
if (!(query_range_array == rhs.query_range_array))
return false;
if (!(topk == rhs.topk))
return false;
return true;
@ -784,7 +571,8 @@ class MegasearchService_SearchVector_pargs {
virtual ~MegasearchService_SearchVector_pargs() throw();
const std::string* table_name;
const std::vector<QueryRecord> * query_record_array;
const std::vector<RowRecord> * query_record_array;
const std::vector<Range> * query_range_array;
const int64_t* topk;
uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const;
@ -966,6 +754,118 @@ class MegasearchService_DescribeTable_presult {
};
typedef struct _MegasearchService_GetTableRowCount_args__isset {
_MegasearchService_GetTableRowCount_args__isset() : table_name(false) {}
bool table_name :1;
} _MegasearchService_GetTableRowCount_args__isset;
class MegasearchService_GetTableRowCount_args {
public:
MegasearchService_GetTableRowCount_args(const MegasearchService_GetTableRowCount_args&);
MegasearchService_GetTableRowCount_args& operator=(const MegasearchService_GetTableRowCount_args&);
MegasearchService_GetTableRowCount_args() : table_name() {
}
virtual ~MegasearchService_GetTableRowCount_args() throw();
std::string table_name;
_MegasearchService_GetTableRowCount_args__isset __isset;
void __set_table_name(const std::string& val);
bool operator == (const MegasearchService_GetTableRowCount_args & rhs) const
{
if (!(table_name == rhs.table_name))
return false;
return true;
}
bool operator != (const MegasearchService_GetTableRowCount_args &rhs) const {
return !(*this == rhs);
}
bool operator < (const MegasearchService_GetTableRowCount_args & ) const;
uint32_t read(::apache::thrift::protocol::TProtocol* iprot);
uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const;
};
class MegasearchService_GetTableRowCount_pargs {
public:
virtual ~MegasearchService_GetTableRowCount_pargs() throw();
const std::string* table_name;
uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const;
};
typedef struct _MegasearchService_GetTableRowCount_result__isset {
_MegasearchService_GetTableRowCount_result__isset() : success(false), e(false) {}
bool success :1;
bool e :1;
} _MegasearchService_GetTableRowCount_result__isset;
class MegasearchService_GetTableRowCount_result {
public:
MegasearchService_GetTableRowCount_result(const MegasearchService_GetTableRowCount_result&);
MegasearchService_GetTableRowCount_result& operator=(const MegasearchService_GetTableRowCount_result&);
MegasearchService_GetTableRowCount_result() : success(0) {
}
virtual ~MegasearchService_GetTableRowCount_result() throw();
int64_t success;
Exception e;
_MegasearchService_GetTableRowCount_result__isset __isset;
void __set_success(const int64_t val);
void __set_e(const Exception& val);
bool operator == (const MegasearchService_GetTableRowCount_result & rhs) const
{
if (!(success == rhs.success))
return false;
if (!(e == rhs.e))
return false;
return true;
}
bool operator != (const MegasearchService_GetTableRowCount_result &rhs) const {
return !(*this == rhs);
}
bool operator < (const MegasearchService_GetTableRowCount_result & ) const;
uint32_t read(::apache::thrift::protocol::TProtocol* iprot);
uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const;
};
typedef struct _MegasearchService_GetTableRowCount_presult__isset {
_MegasearchService_GetTableRowCount_presult__isset() : success(false), e(false) {}
bool success :1;
bool e :1;
} _MegasearchService_GetTableRowCount_presult__isset;
class MegasearchService_GetTableRowCount_presult {
public:
virtual ~MegasearchService_GetTableRowCount_presult() throw();
int64_t* success;
Exception e;
_MegasearchService_GetTableRowCount_presult__isset __isset;
uint32_t read(::apache::thrift::protocol::TProtocol* iprot);
};
class MegasearchService_ShowTables_args {
public:
@ -1209,21 +1109,18 @@ class MegasearchServiceClient : virtual public MegasearchServiceIf {
void DeleteTable(const std::string& table_name);
void send_DeleteTable(const std::string& table_name);
void recv_DeleteTable();
void CreateTablePartition(const CreateTablePartitionParam& param);
void send_CreateTablePartition(const CreateTablePartitionParam& param);
void recv_CreateTablePartition();
void DeleteTablePartition(const DeleteTablePartitionParam& param);
void send_DeleteTablePartition(const DeleteTablePartitionParam& param);
void recv_DeleteTablePartition();
void AddVector(std::vector<int64_t> & _return, const std::string& table_name, const std::vector<RowRecord> & record_array);
void send_AddVector(const std::string& table_name, const std::vector<RowRecord> & record_array);
void recv_AddVector(std::vector<int64_t> & _return);
void SearchVector(std::vector<TopKQueryResult> & _return, const std::string& table_name, const std::vector<QueryRecord> & query_record_array, const int64_t topk);
void send_SearchVector(const std::string& table_name, const std::vector<QueryRecord> & query_record_array, const int64_t topk);
void SearchVector(std::vector<TopKQueryResult> & _return, const std::string& table_name, const std::vector<RowRecord> & query_record_array, const std::vector<Range> & query_range_array, const int64_t topk);
void send_SearchVector(const std::string& table_name, const std::vector<RowRecord> & query_record_array, const std::vector<Range> & query_range_array, const int64_t topk);
void recv_SearchVector(std::vector<TopKQueryResult> & _return);
void DescribeTable(TableSchema& _return, const std::string& table_name);
void send_DescribeTable(const std::string& table_name);
void recv_DescribeTable(TableSchema& _return);
int64_t GetTableRowCount(const std::string& table_name);
void send_GetTableRowCount(const std::string& table_name);
int64_t recv_GetTableRowCount();
void ShowTables(std::vector<std::string> & _return);
void send_ShowTables();
void recv_ShowTables(std::vector<std::string> & _return);
@ -1247,11 +1144,10 @@ class MegasearchServiceProcessor : public ::apache::thrift::TDispatchProcessor {
ProcessMap processMap_;
void process_CreateTable(int32_t seqid, ::apache::thrift::protocol::TProtocol* iprot, ::apache::thrift::protocol::TProtocol* oprot, void* callContext);
void process_DeleteTable(int32_t seqid, ::apache::thrift::protocol::TProtocol* iprot, ::apache::thrift::protocol::TProtocol* oprot, void* callContext);
void process_CreateTablePartition(int32_t seqid, ::apache::thrift::protocol::TProtocol* iprot, ::apache::thrift::protocol::TProtocol* oprot, void* callContext);
void process_DeleteTablePartition(int32_t seqid, ::apache::thrift::protocol::TProtocol* iprot, ::apache::thrift::protocol::TProtocol* oprot, void* callContext);
void process_AddVector(int32_t seqid, ::apache::thrift::protocol::TProtocol* iprot, ::apache::thrift::protocol::TProtocol* oprot, void* callContext);
void process_SearchVector(int32_t seqid, ::apache::thrift::protocol::TProtocol* iprot, ::apache::thrift::protocol::TProtocol* oprot, void* callContext);
void process_DescribeTable(int32_t seqid, ::apache::thrift::protocol::TProtocol* iprot, ::apache::thrift::protocol::TProtocol* oprot, void* callContext);
void process_GetTableRowCount(int32_t seqid, ::apache::thrift::protocol::TProtocol* iprot, ::apache::thrift::protocol::TProtocol* oprot, void* callContext);
void process_ShowTables(int32_t seqid, ::apache::thrift::protocol::TProtocol* iprot, ::apache::thrift::protocol::TProtocol* oprot, void* callContext);
void process_Ping(int32_t seqid, ::apache::thrift::protocol::TProtocol* iprot, ::apache::thrift::protocol::TProtocol* oprot, void* callContext);
public:
@ -1259,11 +1155,10 @@ class MegasearchServiceProcessor : public ::apache::thrift::TDispatchProcessor {
iface_(iface) {
processMap_["CreateTable"] = &MegasearchServiceProcessor::process_CreateTable;
processMap_["DeleteTable"] = &MegasearchServiceProcessor::process_DeleteTable;
processMap_["CreateTablePartition"] = &MegasearchServiceProcessor::process_CreateTablePartition;
processMap_["DeleteTablePartition"] = &MegasearchServiceProcessor::process_DeleteTablePartition;
processMap_["AddVector"] = &MegasearchServiceProcessor::process_AddVector;
processMap_["SearchVector"] = &MegasearchServiceProcessor::process_SearchVector;
processMap_["DescribeTable"] = &MegasearchServiceProcessor::process_DescribeTable;
processMap_["GetTableRowCount"] = &MegasearchServiceProcessor::process_GetTableRowCount;
processMap_["ShowTables"] = &MegasearchServiceProcessor::process_ShowTables;
processMap_["Ping"] = &MegasearchServiceProcessor::process_Ping;
}
@ -1312,24 +1207,6 @@ class MegasearchServiceMultiface : virtual public MegasearchServiceIf {
ifaces_[i]->DeleteTable(table_name);
}
void CreateTablePartition(const CreateTablePartitionParam& param) {
size_t sz = ifaces_.size();
size_t i = 0;
for (; i < (sz - 1); ++i) {
ifaces_[i]->CreateTablePartition(param);
}
ifaces_[i]->CreateTablePartition(param);
}
void DeleteTablePartition(const DeleteTablePartitionParam& param) {
size_t sz = ifaces_.size();
size_t i = 0;
for (; i < (sz - 1); ++i) {
ifaces_[i]->DeleteTablePartition(param);
}
ifaces_[i]->DeleteTablePartition(param);
}
void AddVector(std::vector<int64_t> & _return, const std::string& table_name, const std::vector<RowRecord> & record_array) {
size_t sz = ifaces_.size();
size_t i = 0;
@ -1340,13 +1217,13 @@ class MegasearchServiceMultiface : virtual public MegasearchServiceIf {
return;
}
void SearchVector(std::vector<TopKQueryResult> & _return, const std::string& table_name, const std::vector<QueryRecord> & query_record_array, const int64_t topk) {
void SearchVector(std::vector<TopKQueryResult> & _return, const std::string& table_name, const std::vector<RowRecord> & query_record_array, const std::vector<Range> & query_range_array, const int64_t topk) {
size_t sz = ifaces_.size();
size_t i = 0;
for (; i < (sz - 1); ++i) {
ifaces_[i]->SearchVector(_return, table_name, query_record_array, topk);
ifaces_[i]->SearchVector(_return, table_name, query_record_array, query_range_array, topk);
}
ifaces_[i]->SearchVector(_return, table_name, query_record_array, topk);
ifaces_[i]->SearchVector(_return, table_name, query_record_array, query_range_array, topk);
return;
}
@ -1360,6 +1237,15 @@ class MegasearchServiceMultiface : virtual public MegasearchServiceIf {
return;
}
int64_t GetTableRowCount(const std::string& table_name) {
size_t sz = ifaces_.size();
size_t i = 0;
for (; i < (sz - 1); ++i) {
ifaces_[i]->GetTableRowCount(table_name);
}
return ifaces_[i]->GetTableRowCount(table_name);
}
void ShowTables(std::vector<std::string> & _return) {
size_t sz = ifaces_.size();
size_t i = 0;
@ -1416,21 +1302,18 @@ class MegasearchServiceConcurrentClient : virtual public MegasearchServiceIf {
void DeleteTable(const std::string& table_name);
int32_t send_DeleteTable(const std::string& table_name);
void recv_DeleteTable(const int32_t seqid);
void CreateTablePartition(const CreateTablePartitionParam& param);
int32_t send_CreateTablePartition(const CreateTablePartitionParam& param);
void recv_CreateTablePartition(const int32_t seqid);
void DeleteTablePartition(const DeleteTablePartitionParam& param);
int32_t send_DeleteTablePartition(const DeleteTablePartitionParam& param);
void recv_DeleteTablePartition(const int32_t seqid);
void AddVector(std::vector<int64_t> & _return, const std::string& table_name, const std::vector<RowRecord> & record_array);
int32_t send_AddVector(const std::string& table_name, const std::vector<RowRecord> & record_array);
void recv_AddVector(std::vector<int64_t> & _return, const int32_t seqid);
void SearchVector(std::vector<TopKQueryResult> & _return, const std::string& table_name, const std::vector<QueryRecord> & query_record_array, const int64_t topk);
int32_t send_SearchVector(const std::string& table_name, const std::vector<QueryRecord> & query_record_array, const int64_t topk);
void SearchVector(std::vector<TopKQueryResult> & _return, const std::string& table_name, const std::vector<RowRecord> & query_record_array, const std::vector<Range> & query_range_array, const int64_t topk);
int32_t send_SearchVector(const std::string& table_name, const std::vector<RowRecord> & query_record_array, const std::vector<Range> & query_range_array, const int64_t topk);
void recv_SearchVector(std::vector<TopKQueryResult> & _return, const int32_t seqid);
void DescribeTable(TableSchema& _return, const std::string& table_name);
int32_t send_DescribeTable(const std::string& table_name);
void recv_DescribeTable(TableSchema& _return, const int32_t seqid);
int64_t GetTableRowCount(const std::string& table_name);
int32_t send_GetTableRowCount(const std::string& table_name);
int64_t recv_GetTableRowCount(const int32_t seqid);
void ShowTables(std::vector<std::string> & _return);
int32_t send_ShowTables();
void recv_ShowTables(std::vector<std::string> & _return, const int32_t seqid);

56
cpp/src/thrift/gen-cpp/MegasearchService_server.skeleton.cpp
View File

@ -50,36 +50,6 @@ class MegasearchServiceHandler : virtual public MegasearchServiceIf {
printf("DeleteTable\n");
}
/**
* @brief Create table partition
*
* This method is used to create table partition.
*
* @param param, use to provide partition information to be created.
*
*
* @param param
*/
void CreateTablePartition(const CreateTablePartitionParam& param) {
// Your implementation goes here
printf("CreateTablePartition\n");
}
/**
* @brief Delete table partition
*
* This method is used to delete table partition.
*
* @param param, use to provide partition information to be deleted.
*
*
* @param param
*/
void DeleteTablePartition(const DeleteTablePartitionParam& param) {
// Your implementation goes here
printf("DeleteTablePartition\n");
}
/**
* @brief Add vector array to table
*
@ -105,25 +75,27 @@ class MegasearchServiceHandler : virtual public MegasearchServiceIf {
*
* @param table_name, table_name is queried.
* @param query_record_array, all vector are going to be queried.
* @param query_range_array, optional ranges for conditional search. If not specified, search whole table
* @param topk, how many similarity vectors will be searched.
*
* @return query result array.
*
* @param table_name
* @param query_record_array
* @param query_range_array
* @param topk
*/
void SearchVector(std::vector<TopKQueryResult> & _return, const std::string& table_name, const std::vector<QueryRecord> & query_record_array, const int64_t topk) {
void SearchVector(std::vector<TopKQueryResult> & _return, const std::string& table_name, const std::vector<RowRecord> & query_record_array, const std::vector<Range> & query_range_array, const int64_t topk) {
// Your implementation goes here
printf("SearchVector\n");
}
/**
* @brief Show table information
* @brief Get table schema
*
* This method is used to show table information.
* This method is used to get table schema.
*
* @param table_name, which table is show.
* @param table_name, target table name.
*
* @return table schema
*
@ -134,6 +106,22 @@ class MegasearchServiceHandler : virtual public MegasearchServiceIf {
printf("DescribeTable\n");
}
/**
* @brief Get table row count
*
* This method is used to get table row count.
*
* @param table_name, target table name.
*
* @return table row count
*
* @param table_name
*/
int64_t GetTableRowCount(const std::string& table_name) {
// Your implementation goes here
printf("GetTableRowCount\n");
}
/**
* @brief List all tables in database
*

1175
cpp/src/thrift/gen-cpp/megasearch_types.cpp
View File

File diff suppressed because it is too large Load Diff

324
cpp/src/thrift/gen-cpp/megasearch_types.h
View File

@ -26,10 +26,9 @@ struct ErrorCode {
CONNECT_FAILED = 1,
PERMISSION_DENIED = 2,
TABLE_NOT_EXISTS = 3,
PARTITION_NOT_EXIST = 4,
ILLEGAL_ARGUMENT = 5,
ILLEGAL_RANGE = 6,
ILLEGAL_DIMENSION = 7
ILLEGAL_ARGUMENT = 4,
ILLEGAL_RANGE = 5,
ILLEGAL_DIMENSION = 6
};
};
@ -39,22 +38,12 @@ std::ostream& operator<<(std::ostream& out, const ErrorCode::type& val);
class Exception;
class Column;
class VectorColumn;
class TableSchema;
class Range;
class CreateTablePartitionParam;
class DeleteTablePartitionParam;
class RowRecord;
class QueryRecord;
class QueryResult;
class TopKQueryResult;
@ -109,108 +98,11 @@ void swap(Exception &a, Exception &b);
std::ostream& operator<<(std::ostream& out, const Exception& obj);
class Column : public virtual ::apache::thrift::TBase {
public:
Column(const Column&);
Column& operator=(const Column&);
Column() : type(0), name() {
}
virtual ~Column() throw();
int32_t type;
std::string name;
void __set_type(const int32_t val);
void __set_name(const std::string& val);
bool operator == (const Column & rhs) const
{
if (!(type == rhs.type))
return false;
if (!(name == rhs.name))
return false;
return true;
}
bool operator != (const Column &rhs) const {
return !(*this == rhs);
}
bool operator < (const Column & ) const;
uint32_t read(::apache::thrift::protocol::TProtocol* iprot);
uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const;
virtual void printTo(std::ostream& out) const;
};
void swap(Column &a, Column &b);
std::ostream& operator<<(std::ostream& out, const Column& obj);
typedef struct _VectorColumn__isset {
_VectorColumn__isset() : store_raw_vector(true) {}
bool store_raw_vector :1;
} _VectorColumn__isset;
class VectorColumn : public virtual ::apache::thrift::TBase {
public:
VectorColumn(const VectorColumn&);
VectorColumn& operator=(const VectorColumn&);
VectorColumn() : dimension(0), index_type(), store_raw_vector(false) {
}
virtual ~VectorColumn() throw();
Column base;
int64_t dimension;
std::string index_type;
bool store_raw_vector;
_VectorColumn__isset __isset;
void __set_base(const Column& val);
void __set_dimension(const int64_t val);
void __set_index_type(const std::string& val);
void __set_store_raw_vector(const bool val);
bool operator == (const VectorColumn & rhs) const
{
if (!(base == rhs.base))
return false;
if (!(dimension == rhs.dimension))
return false;
if (!(index_type == rhs.index_type))
return false;
if (!(store_raw_vector == rhs.store_raw_vector))
return false;
return true;
}
bool operator != (const VectorColumn &rhs) const {
return !(*this == rhs);
}
bool operator < (const VectorColumn & ) const;
uint32_t read(::apache::thrift::protocol::TProtocol* iprot);
uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const;
virtual void printTo(std::ostream& out) const;
};
void swap(VectorColumn &a, VectorColumn &b);
std::ostream& operator<<(std::ostream& out, const VectorColumn& obj);
typedef struct _TableSchema__isset {
_TableSchema__isset() : attribute_column_array(false), partition_column_name_array(false) {}
bool attribute_column_array :1;
bool partition_column_name_array :1;
_TableSchema__isset() : index_type(true), dimension(true), store_raw_vector(true) {}
bool index_type :1;
bool dimension :1;
bool store_raw_vector :1;
} _TableSchema__isset;
class TableSchema : public virtual ::apache::thrift::TBase {
@ -218,38 +110,34 @@ class TableSchema : public virtual ::apache::thrift::TBase {
TableSchema(const TableSchema&);
TableSchema& operator=(const TableSchema&);
TableSchema() : table_name() {
TableSchema() : table_name(), index_type(0), dimension(0LL), store_raw_vector(false) {
}
virtual ~TableSchema() throw();
std::string table_name;
std::vector<VectorColumn> vector_column_array;
std::vector<Column> attribute_column_array;
std::vector<std::string> partition_column_name_array;
int32_t index_type;
int64_t dimension;
bool store_raw_vector;
_TableSchema__isset __isset;
void __set_table_name(const std::string& val);
void __set_vector_column_array(const std::vector<VectorColumn> & val);
void __set_index_type(const int32_t val);
void __set_attribute_column_array(const std::vector<Column> & val);
void __set_dimension(const int64_t val);
void __set_partition_column_name_array(const std::vector<std::string> & val);
void __set_store_raw_vector(const bool val);
bool operator == (const TableSchema & rhs) const
{
if (!(table_name == rhs.table_name))
return false;
if (!(vector_column_array == rhs.vector_column_array))
if (!(index_type == rhs.index_type))
return false;
if (__isset.attribute_column_array != rhs.__isset.attribute_column_array)
if (!(dimension == rhs.dimension))
return false;
else if (__isset.attribute_column_array && !(attribute_column_array == rhs.attribute_column_array))
return false;
if (__isset.partition_column_name_array != rhs.__isset.partition_column_name_array)
return false;
else if (__isset.partition_column_name_array && !(partition_column_name_array == rhs.partition_column_name_array))
if (!(store_raw_vector == rhs.store_raw_vector))
return false;
return true;
}
@ -269,6 +157,11 @@ void swap(TableSchema &a, TableSchema &b);
std::ostream& operator<<(std::ostream& out, const TableSchema& obj);
typedef struct _Range__isset {
_Range__isset() : start_value(false), end_value(false) {}
bool start_value :1;
bool end_value :1;
} _Range__isset;
class Range : public virtual ::apache::thrift::TBase {
public:
@ -282,6 +175,8 @@ class Range : public virtual ::apache::thrift::TBase {
std::string start_value;
std::string end_value;
_Range__isset __isset;
void __set_start_value(const std::string& val);
void __set_end_value(const std::string& val);
@ -311,120 +206,22 @@ void swap(Range &a, Range &b);
std::ostream& operator<<(std::ostream& out, const Range& obj);
class CreateTablePartitionParam : public virtual ::apache::thrift::TBase {
public:
CreateTablePartitionParam(const CreateTablePartitionParam&);
CreateTablePartitionParam& operator=(const CreateTablePartitionParam&);
CreateTablePartitionParam() : table_name(), partition_name() {
}
virtual ~CreateTablePartitionParam() throw();
std::string table_name;
std::string partition_name;
std::map<std::string, Range> range_map;
void __set_table_name(const std::string& val);
void __set_partition_name(const std::string& val);
void __set_range_map(const std::map<std::string, Range> & val);
bool operator == (const CreateTablePartitionParam & rhs) const
{
if (!(table_name == rhs.table_name))
return false;
if (!(partition_name == rhs.partition_name))
return false;
if (!(range_map == rhs.range_map))
return false;
return true;
}
bool operator != (const CreateTablePartitionParam &rhs) const {
return !(*this == rhs);
}
bool operator < (const CreateTablePartitionParam & ) const;
uint32_t read(::apache::thrift::protocol::TProtocol* iprot);
uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const;
virtual void printTo(std::ostream& out) const;
};
void swap(CreateTablePartitionParam &a, CreateTablePartitionParam &b);
std::ostream& operator<<(std::ostream& out, const CreateTablePartitionParam& obj);
class DeleteTablePartitionParam : public virtual ::apache::thrift::TBase {
public:
DeleteTablePartitionParam(const DeleteTablePartitionParam&);
DeleteTablePartitionParam& operator=(const DeleteTablePartitionParam&);
DeleteTablePartitionParam() : table_name() {
}
virtual ~DeleteTablePartitionParam() throw();
std::string table_name;
std::vector<std::string> partition_name_array;
void __set_table_name(const std::string& val);
void __set_partition_name_array(const std::vector<std::string> & val);
bool operator == (const DeleteTablePartitionParam & rhs) const
{
if (!(table_name == rhs.table_name))
return false;
if (!(partition_name_array == rhs.partition_name_array))
return false;
return true;
}
bool operator != (const DeleteTablePartitionParam &rhs) const {
return !(*this == rhs);
}
bool operator < (const DeleteTablePartitionParam & ) const;
uint32_t read(::apache::thrift::protocol::TProtocol* iprot);
uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const;
virtual void printTo(std::ostream& out) const;
};
void swap(DeleteTablePartitionParam &a, DeleteTablePartitionParam &b);
std::ostream& operator<<(std::ostream& out, const DeleteTablePartitionParam& obj);
typedef struct _RowRecord__isset {
_RowRecord__isset() : attribute_map(false) {}
bool attribute_map :1;
} _RowRecord__isset;
class RowRecord : public virtual ::apache::thrift::TBase {
public:
RowRecord(const RowRecord&);
RowRecord& operator=(const RowRecord&);
RowRecord() {
RowRecord() : vector_data() {
}
virtual ~RowRecord() throw();
std::map<std::string, std::string> vector_map;
std::map<std::string, std::string> attribute_map;
std::string vector_data;
_RowRecord__isset __isset;
void __set_vector_map(const std::map<std::string, std::string> & val);
void __set_attribute_map(const std::map<std::string, std::string> & val);
void __set_vector_data(const std::string& val);
bool operator == (const RowRecord & rhs) const
{
if (!(vector_map == rhs.vector_map))
return false;
if (!(attribute_map == rhs.attribute_map))
if (!(vector_data == rhs.vector_data))
return false;
return true;
}
@ -444,68 +241,10 @@ void swap(RowRecord &a, RowRecord &b);
std::ostream& operator<<(std::ostream& out, const RowRecord& obj);
typedef struct _QueryRecord__isset {
_QueryRecord__isset() : selected_column_array(false), partition_filter_column_map(false) {}
bool selected_column_array :1;
bool partition_filter_column_map :1;
} _QueryRecord__isset;
class QueryRecord : public virtual ::apache::thrift::TBase {
public:
QueryRecord(const QueryRecord&);
QueryRecord& operator=(const QueryRecord&);
QueryRecord() {
}
virtual ~QueryRecord() throw();
std::map<std::string, std::string> vector_map;
std::vector<std::string> selected_column_array;
std::map<std::string, std::vector<Range> > partition_filter_column_map;
_QueryRecord__isset __isset;
void __set_vector_map(const std::map<std::string, std::string> & val);
void __set_selected_column_array(const std::vector<std::string> & val);
void __set_partition_filter_column_map(const std::map<std::string, std::vector<Range> > & val);
bool operator == (const QueryRecord & rhs) const
{
if (!(vector_map == rhs.vector_map))
return false;
if (__isset.selected_column_array != rhs.__isset.selected_column_array)
return false;
else if (__isset.selected_column_array && !(selected_column_array == rhs.selected_column_array))
return false;
if (__isset.partition_filter_column_map != rhs.__isset.partition_filter_column_map)
return false;
else if (__isset.partition_filter_column_map && !(partition_filter_column_map == rhs.partition_filter_column_map))
return false;
return true;
}
bool operator != (const QueryRecord &rhs) const {
return !(*this == rhs);
}
bool operator < (const QueryRecord & ) const;
uint32_t read(::apache::thrift::protocol::TProtocol* iprot);
uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const;
virtual void printTo(std::ostream& out) const;
};
void swap(QueryRecord &a, QueryRecord &b);
std::ostream& operator<<(std::ostream& out, const QueryRecord& obj);
typedef struct _QueryResult__isset {
_QueryResult__isset() : id(false), score(false), column_map(false) {}
_QueryResult__isset() : id(false), score(false) {}
bool id :1;
bool score :1;
bool column_map :1;
} _QueryResult__isset;
class QueryResult : public virtual ::apache::thrift::TBase {
@ -519,7 +258,6 @@ class QueryResult : public virtual ::apache::thrift::TBase {
virtual ~QueryResult() throw();
int64_t id;
double score;
std::map<std::string, std::string> column_map;
_QueryResult__isset __isset;
@ -527,16 +265,12 @@ class QueryResult : public virtual ::apache::thrift::TBase {
void __set_score(const double val);
void __set_column_map(const std::map<std::string, std::string> & val);
bool operator == (const QueryResult & rhs) const
{
if (!(id == rhs.id))
return false;
if (!(score == rhs.score))
return false;
if (!(column_map == rhs.column_map))
return false;
return true;
}
bool operator != (const QueryResult &rhs) const {

128
cpp/src/thrift/megasearch.thrift
View File

@ -17,104 +17,56 @@ enum ErrorCode {
SUCCESS = 0,
CONNECT_FAILED,
PERMISSION_DENIED,
TABLE_NOT_EXISTS,
PARTITION_NOT_EXIST,
ILLEGAL_ARGUMENT,
ILLEGAL_RANGE,
ILLEGAL_DIMENSION,
TABLE_NOT_EXISTS,
ILLEGAL_ARGUMENT,
ILLEGAL_RANGE,
ILLEGAL_DIMENSION,
}
exception Exception {
1: ErrorCode code;
2: string reason;
1: ErrorCode code;
2: string reason;
}
/**
* @brief Table column description
*/
struct Column {
1: required i32 type; ///< Column Type: 0:invealid/1:int8/2:int16/3:int32/4:int64/5:float32/6:float64/7:date/8:vector
2: required string name; ///< Column name
}
/**
* @brief Table vector column description
*/
struct VectorColumn {
1: required Column base; ///< Base column schema
2: required i64 dimension; ///< Vector dimension
3: required string index_type; ///< Index type, optional: raw, ivf
4: bool store_raw_vector = false; ///< Is vector self stored in the table
}
/**
* @brief Table Schema
*/
struct TableSchema {
1: required string table_name; ///< Table name
2: required list<VectorColumn> vector_column_array; ///< Vector column description
3: optional list<Column> attribute_column_array; ///< Columns description
4: optional list<string> partition_column_name_array; ///< Partition column name
1: required string table_name; ///< Table name
2: i32 index_type = 0; ///< Index type, optional: 0-invalid, 1-idmap, 2-ivflat
3: i64 dimension = 0; ///< Vector dimension
4: bool store_raw_vector = false; ///< Store raw data
}
/**
* @brief Range Schema
*/
struct Range {
1: required string start_value; ///< Range start
2: required string end_value; ///< Range stop
}
/**
* @brief Create table partition parameters
*/
struct CreateTablePartitionParam {
1: required string table_name; ///< Table name, vector/float32/float64 type column is not allowed for partition
2: required string partition_name; ///< Partition name, created partition name
3: required map<string, Range> range_map; ///< Column name to Range map
}
/**
* @brief Delete table partition parameters
*/
struct DeleteTablePartitionParam {
1: required string table_name; ///< Table name
2: required list<string> partition_name_array; ///< Partition name array
1: string start_value; ///< Range start
2: string end_value; ///< Range stop
}
/**
* @brief Record inserted
*/
struct RowRecord {
1: required map<string, binary> vector_map; ///< Vector columns
2: map<string, string> attribute_map; ///< Other attribute columns
}
/**
* @brief Query record
*/
struct QueryRecord {
1: required map<string, binary> vector_map; ///< Query vectors
2: optional list<string> selected_column_array; ///< Output column array
3: optional map<string, list<Range>> partition_filter_column_map; ///< Range used to select partitions
1: required binary vector_data; ///< Vector data, double array
}
/**
* @brief Query result
*/
struct QueryResult {
1: i64 id; ///< Output result
2: double score; ///< Vector similarity score: 0 ~ 100
3: map<string, string> column_map; ///< Other column
1: i64 id; ///< Output result
2: double score; ///< Vector similarity score: 0 ~ 100
}
/**
* @brief TopK query result
*/
struct TopKQueryResult {
1: list<QueryResult> query_result_arrays; ///< TopK query result
1: list<QueryResult> query_result_arrays; ///< TopK query result
}
service MegasearchService {
@ -140,28 +92,6 @@ service MegasearchService {
void DeleteTable(2: string table_name) throws(1: Exception e);
/**
* @brief Create table partition
*
* This method is used to create table partition.
*
* @param param, use to provide partition information to be created.
*
*/
void CreateTablePartition(2: CreateTablePartitionParam param) throws(1: Exception e);
/**
* @brief Delete table partition
*
* This method is used to delete table partition.
*
* @param param, use to provide partition information to be deleted.
*
*/
void DeleteTablePartition(2: DeleteTablePartitionParam param) throws(1: Exception e);
/**
* @brief Add vector array to table
*
@ -183,25 +113,40 @@ service MegasearchService {
*
* @param table_name, table_name is queried.
* @param query_record_array, all vector are going to be queried.
* @param query_range_array, optional ranges for conditional search. If not specified, search whole table
* @param topk, how many similarity vectors will be searched.
*
* @return query result array.
*/
list<TopKQueryResult> SearchVector(2: string table_name,
3: list<QueryRecord> query_record_array,
4: i64 topk) throws(1: Exception e);
3: list<RowRecord> query_record_array,
4: list<Range> query_range_array,
5: i64 topk) throws(1: Exception e);
/**
* @brief Show table information
* @brief Get table schema
*
* This method is used to show table information.
* This method is used to get table schema.
*
* @param table_name, which table is show.
* @param table_name, target table name.
*
* @return table schema
*/
TableSchema DescribeTable(2: string table_name) throws(1: Exception e);
/**
* @brief Get table row count
*
* This method is used to get table row count.
*
* @param table_name, target table name.
*
* @return table row count
*/
i64 GetTableRowCount(2: string table_name) throws(1: Exception e);
/**
* @brief List all tables in database
*
@ -212,6 +157,7 @@ service MegasearchService {
*/
list<string> ShowTables() throws(1: Exception e);
/**
* @brief Give the server status
*

2
cpp/src/utils/Error.h
View File

@ -31,7 +31,7 @@ constexpr ServerError SERVER_INVALID_ARGUMENT = ToGlobalServerErrorCode(0x004);
constexpr ServerError SERVER_FILE_NOT_FOUND = ToGlobalServerErrorCode(0x005);
constexpr ServerError SERVER_NOT_IMPLEMENT = ToGlobalServerErrorCode(0x006);
constexpr ServerError SERVER_BLOCKING_QUEUE_EMPTY = ToGlobalServerErrorCode(0x007);
constexpr ServerError SERVER_GROUP_NOT_EXIST = ToGlobalServerErrorCode(0x008);
constexpr ServerError SERVER_TABLE_NOT_EXIST = ToGlobalServerErrorCode(0x008);
constexpr ServerError SERVER_INVALID_TIME_RANGE = ToGlobalServerErrorCode(0x009);
constexpr ServerError SERVER_INVALID_VECTOR_DIMENSION = ToGlobalServerErrorCode(0x00a);
constexpr ServerError SERVER_LICENSE_VALIDATION_FAIL = ToGlobalServerErrorCode(0x00b);

28
cpp/unittest/db/db_tests.cpp
View File

@ -67,15 +67,15 @@ TEST_F(DBTest2, ARHIVE_DISK_CHECK) {
long size;
engine::meta::TableSchema group_info;
group_info.dimension = group_dim;
group_info.table_id = group_name;
group_info.dimension_ = group_dim;
group_info.table_id_ = group_name;
engine::Status stat = db_->CreateTable(group_info);
engine::meta::TableSchema group_info_get;
group_info_get.table_id = group_name;
group_info_get.table_id_ = group_name;
stat = db_->DescribeTable(group_info_get);
ASSERT_STATS(stat);
ASSERT_EQ(group_info_get.dimension, group_dim);
ASSERT_EQ(group_info_get.dimension_, group_dim);
engine::IDNumbers vector_ids;
engine::IDNumbers target_ids;
@ -114,15 +114,15 @@ TEST_F(DBTest, DB_TEST) {
static const int group_dim = 256;
engine::meta::TableSchema group_info;
group_info.dimension = group_dim;
group_info.table_id = group_name;
group_info.dimension_ = group_dim;
group_info.table_id_ = group_name;
engine::Status stat = db_->CreateTable(group_info);
engine::meta::TableSchema group_info_get;
group_info_get.table_id = group_name;
group_info_get.table_id_ = group_name;
stat = db_->DescribeTable(group_info_get);
ASSERT_STATS(stat);
ASSERT_EQ(group_info_get.dimension, group_dim);
ASSERT_EQ(group_info_get.dimension_, group_dim);
engine::IDNumbers vector_ids;
engine::IDNumbers target_ids;
@ -164,11 +164,11 @@ TEST_F(DBTest, DB_TEST) {
ASSERT_STATS(stat);
for (auto k=0; k<qb; ++k) {
ASSERT_EQ(results[k][0], target_ids[k]);
ASSERT_EQ(results[k][0].first, target_ids[k]);
ss.str("");
ss << "Result [" << k << "]:";
for (auto result : results[k]) {
ss << result << " ";
ss << result.first << " ";
}
/* LOG(DEBUG) << ss.str(); */
}
@ -200,15 +200,15 @@ TEST_F(DBTest, SEARCH_TEST) {
static const int group_dim = 256;
engine::meta::TableSchema group_info;
group_info.dimension = group_dim;
group_info.table_id = group_name;
group_info.dimension_ = group_dim;
group_info.table_id_ = group_name;
engine::Status stat = db_->CreateTable(group_info);
engine::meta::TableSchema group_info_get;
group_info_get.table_id = group_name;
group_info_get.table_id_ = group_name;
stat = db_->DescribeTable(group_info_get);
ASSERT_STATS(stat);
ASSERT_EQ(group_info_get.dimension, group_dim);
ASSERT_EQ(group_info_get.dimension_, group_dim);
// prepare raw data
size_t nb = 250000;

86
cpp/unittest/db/meta_tests.cpp
View File

@ -21,22 +21,22 @@ TEST_F(MetaTest, GROUP_TEST) {
auto table_id = "meta_test_group";
meta::TableSchema group;
group.table_id = table_id;
group.table_id_ = table_id;
auto status = impl_->CreateTable(group);
ASSERT_TRUE(status.ok());
auto gid = group.id;
group.id = -1;
auto gid = group.id_;
group.id_ = -1;
status = impl_->DescribeTable(group);
ASSERT_TRUE(status.ok());
ASSERT_EQ(group.id, gid);
ASSERT_EQ(group.table_id, table_id);
ASSERT_EQ(group.id_, gid);
ASSERT_EQ(group.table_id_, table_id);
group.table_id = "not_found";
group.table_id_ = "not_found";
status = impl_->DescribeTable(group);
ASSERT_TRUE(!status.ok());
group.table_id = table_id;
group.table_id_ = table_id;
status = impl_->CreateTable(group);
ASSERT_TRUE(!status.ok());
}
@ -45,49 +45,49 @@ TEST_F(MetaTest, table_file_TEST) {
auto table_id = "meta_test_group";
meta::TableSchema group;
group.table_id = table_id;
group.table_id_ = table_id;
auto status = impl_->CreateTable(group);
meta::TableFileSchema table_file;
table_file.table_id = group.table_id;
table_file.table_id_ = group.table_id_;
status = impl_->CreateTableFile(table_file);
ASSERT_TRUE(status.ok());
ASSERT_EQ(table_file.file_type, meta::TableFileSchema::NEW);
ASSERT_EQ(table_file.file_type_, meta::TableFileSchema::NEW);
auto file_id = table_file.file_id;
auto file_id = table_file.file_id_;
auto new_file_type = meta::TableFileSchema::INDEX;
table_file.file_type = new_file_type;
table_file.file_type_ = new_file_type;
status = impl_->UpdateTableFile(table_file);
ASSERT_TRUE(status.ok());
ASSERT_EQ(table_file.file_type, new_file_type);
ASSERT_EQ(table_file.file_type_, new_file_type);
meta::DatesT dates;
dates.push_back(meta::Meta::GetDate());
status = impl_->DropPartitionsByDates(table_file.table_id, dates);
status = impl_->DropPartitionsByDates(table_file.table_id_, dates);
ASSERT_FALSE(status.ok());
dates.clear();
for (auto i=2; i < 10; ++i) {
dates.push_back(meta::Meta::GetDateWithDelta(-1*i));
}
status = impl_->DropPartitionsByDates(table_file.table_id, dates);
status = impl_->DropPartitionsByDates(table_file.table_id_, dates);
ASSERT_TRUE(status.ok());
table_file.date = meta::Meta::GetDateWithDelta(-2);
table_file.date_ = meta::Meta::GetDateWithDelta(-2);
status = impl_->UpdateTableFile(table_file);
ASSERT_TRUE(status.ok());
ASSERT_EQ(table_file.date, meta::Meta::GetDateWithDelta(-2));
ASSERT_FALSE(table_file.file_type == meta::TableFileSchema::TO_DELETE);
ASSERT_EQ(table_file.date_, meta::Meta::GetDateWithDelta(-2));
ASSERT_FALSE(table_file.file_type_ == meta::TableFileSchema::TO_DELETE);
dates.clear();
dates.push_back(table_file.date);
status = impl_->DropPartitionsByDates(table_file.table_id, dates);
dates.push_back(table_file.date_);
status = impl_->DropPartitionsByDates(table_file.table_id_, dates);
ASSERT_TRUE(status.ok());
status = impl_->GetTableFile(table_file);
ASSERT_TRUE(status.ok());
ASSERT_TRUE(table_file.file_type == meta::TableFileSchema::TO_DELETE);
ASSERT_TRUE(table_file.file_type_ == meta::TableFileSchema::TO_DELETE);
}
TEST_F(MetaTest, ARCHIVE_TEST_DAYS) {
@ -103,21 +103,21 @@ TEST_F(MetaTest, ARCHIVE_TEST_DAYS) {
auto table_id = "meta_test_group";
meta::TableSchema group;
group.table_id = table_id;
group.table_id_ = table_id;
auto status = impl.CreateTable(group);
meta::TableFilesSchema files;
meta::TableFileSchema table_file;
table_file.table_id = group.table_id;
table_file.table_id_ = group.table_id_;
auto cnt = 100;
long ts = utils::GetMicroSecTimeStamp();
std::vector<int> days;
for (auto i=0; i<cnt; ++i) {
status = impl.CreateTableFile(table_file);
table_file.file_type = meta::TableFileSchema::NEW;
table_file.file_type_ = meta::TableFileSchema::NEW;
int day = rand() % (days_num*2);
table_file.created_on = ts - day*meta::D_SEC*meta::US_PS - 10000;
table_file.created_on_ = ts - day*meta::D_SEC*meta::US_PS - 10000;
status = impl.UpdateTableFile(table_file);
files.push_back(table_file);
days.push_back(day);
@ -130,9 +130,9 @@ TEST_F(MetaTest, ARCHIVE_TEST_DAYS) {
status = impl.GetTableFile(file);
ASSERT_TRUE(status.ok());
if (days[i] < days_num) {
ASSERT_EQ(file.file_type, meta::TableFileSchema::NEW);
ASSERT_EQ(file.file_type_, meta::TableFileSchema::NEW);
} else {
ASSERT_EQ(file.file_type, meta::TableFileSchema::TO_DELETE);
ASSERT_EQ(file.file_type_, meta::TableFileSchema::TO_DELETE);
}
i++;
}
@ -149,19 +149,19 @@ TEST_F(MetaTest, ARCHIVE_TEST_DISK) {
auto table_id = "meta_test_group";
meta::TableSchema group;
group.table_id = table_id;
group.table_id_ = table_id;
auto status = impl.CreateTable(group);
meta::TableFilesSchema files;
meta::TableFileSchema table_file;
table_file.table_id = group.table_id;
table_file.table_id_ = group.table_id_;
auto cnt = 10;
auto each_size = 2UL;
for (auto i=0; i<cnt; ++i) {
status = impl.CreateTableFile(table_file);
table_file.file_type = meta::TableFileSchema::NEW;
table_file.size = each_size * meta::G;
table_file.file_type_ = meta::TableFileSchema::NEW;
table_file.size_ = each_size * meta::G;
status = impl.UpdateTableFile(table_file);
files.push_back(table_file);
}
@ -173,9 +173,9 @@ TEST_F(MetaTest, ARCHIVE_TEST_DISK) {
status = impl.GetTableFile(file);
ASSERT_TRUE(status.ok());
if (i < 5) {
ASSERT_TRUE(file.file_type == meta::TableFileSchema::TO_DELETE);
ASSERT_TRUE(file.file_type_ == meta::TableFileSchema::TO_DELETE);
} else {
ASSERT_EQ(file.file_type, meta::TableFileSchema::NEW);
ASSERT_EQ(file.file_type_, meta::TableFileSchema::NEW);
}
++i;
}
@ -187,7 +187,7 @@ TEST_F(MetaTest, TABLE_FILES_TEST) {
auto table_id = "meta_test_group";
meta::TableSchema group;
group.table_id = table_id;
group.table_id_ = table_id;
auto status = impl_->CreateTable(group);
int new_files_cnt = 4;
@ -196,29 +196,29 @@ TEST_F(MetaTest, TABLE_FILES_TEST) {
int index_files_cnt = 7;
meta::TableFileSchema table_file;
table_file.table_id = group.table_id;
table_file.table_id_ = group.table_id_;
for (auto i=0; i<new_files_cnt; ++i) {
status = impl_->CreateTableFile(table_file);
table_file.file_type = meta::TableFileSchema::NEW;
table_file.file_type_ = meta::TableFileSchema::NEW;
status = impl_->UpdateTableFile(table_file);
}
for (auto i=0; i<raw_files_cnt; ++i) {
status = impl_->CreateTableFile(table_file);
table_file.file_type = meta::TableFileSchema::RAW;
table_file.file_type_ = meta::TableFileSchema::RAW;
status = impl_->UpdateTableFile(table_file);
}
for (auto i=0; i<to_index_files_cnt; ++i) {
status = impl_->CreateTableFile(table_file);
table_file.file_type = meta::TableFileSchema::TO_INDEX;
table_file.file_type_ = meta::TableFileSchema::TO_INDEX;
status = impl_->UpdateTableFile(table_file);
}
for (auto i=0; i<index_files_cnt; ++i) {
status = impl_->CreateTableFile(table_file);
table_file.file_type = meta::TableFileSchema::INDEX;
table_file.file_type_ = meta::TableFileSchema::INDEX;
status = impl_->UpdateTableFile(table_file);
}
@ -229,17 +229,17 @@ TEST_F(MetaTest, TABLE_FILES_TEST) {
ASSERT_EQ(files.size(), to_index_files_cnt);
meta::DatePartionedTableFilesSchema dated_files;
status = impl_->FilesToMerge(group.table_id, dated_files);
status = impl_->FilesToMerge(group.table_id_, dated_files);
ASSERT_TRUE(status.ok());
ASSERT_EQ(dated_files[table_file.date].size(), raw_files_cnt);
ASSERT_EQ(dated_files[table_file.date_].size(), raw_files_cnt);
status = impl_->FilesToIndex(files);
ASSERT_TRUE(status.ok());
ASSERT_EQ(files.size(), to_index_files_cnt);
meta::DatesT dates = {table_file.date};
meta::DatesT dates = {table_file.date_};
status = impl_->FilesToSearch(table_id, dates, dated_files);
ASSERT_TRUE(status.ok());
ASSERT_EQ(dated_files[table_file.date].size(),
ASSERT_EQ(dated_files[table_file.date_].size(),
to_index_files_cnt+raw_files_cnt+index_files_cnt);
}

2
cpp/unittest/db/utils.cpp
View File

@ -38,7 +38,7 @@ engine::Options DBTest::GetOptions() {
void DBTest::SetUp() {
InitLog();
auto options = GetOptions();
db_ = engine::DBFactory::Build(options, "Faiss,IDMap");
db_ = engine::DBFactory::Build(options);
}
void DBTest::TearDown() {

3
cpp/unittest/faiss_wrapper/CMakeLists.txt
View File

@ -13,8 +13,6 @@ include_directories(/usr/local/cuda/include)
link_directories("/usr/local/cuda/lib64")
set(require_files
../../src/server/VecIdMapper.cpp
../../src/server/RocksIdMapper.cpp
../../src/server/ServerConfig.cpp
../../src/utils/CommonUtil.cpp
../../src/utils/TimeRecorder.cpp
@ -38,7 +36,6 @@ set(wrapper_libs
cudart
cublas
sqlite3
rocksdb
snappy
bz2
z

10
cpp/unittest/metrics/metrics_test.cpp
View File

@ -38,12 +38,12 @@ TEST_F(DBTest, Metric_Tes) {
static const int group_dim = 256;
engine::meta::TableSchema group_info;
group_info.dimension = group_dim;
group_info.table_id = group_name;
group_info.dimension_ = group_dim;
group_info.table_id_ = group_name;
engine::Status stat = db_->CreateTable(group_info);
engine::meta::TableSchema group_info_get;
group_info_get.table_id = group_name;
group_info_get.table_id_ = group_name;
stat = db_->DescribeTable(group_info_get);
@ -87,11 +87,11 @@ TEST_F(DBTest, Metric_Tes) {
ASSERT_STATS(stat);
for (auto k=0; k<qb; ++k) {
ASSERT_EQ(results[k][0], target_ids[k]);
ASSERT_EQ(results[k][0].first, target_ids[k]);
ss.str("");
ss << "Result [" << k << "]:";
for (auto result : results[k]) {
ss << result << " ";
ss << result.first << " ";
}
/* LOG(DEBUG) << ss.str(); */
}

3
cpp/unittest/server/CMakeLists.txt
View File

@ -15,8 +15,6 @@ aux_source_directory(../../src/wrapper wrapper_src)
aux_source_directory(./ test_srcs)
set(require_files
../../src/server/VecIdMapper.cpp
../../src/server/RocksIdMapper.cpp
../../src/server/ServerConfig.cpp
../../src/utils/CommonUtil.cpp
../../src/utils/TimeRecorder.cpp
@ -35,7 +33,6 @@ cuda_add_executable(server_test
set(require_libs
stdc++
rocksdb
faiss
cudart
cublas

118
cpp/unittest/server/idmapper_test.cpp
View File

@ -1,118 +0,0 @@
////////////////////////////////////////////////////////////////////////////////
// Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved
// Unauthorized copying of this file, via any medium is strictly prohibited.
// Proprietary and confidential.
////////////////////////////////////////////////////////////////////////////////
#include <gtest/gtest.h>
#include "server/ServerConfig.h"
#include "server/VecIdMapper.h"
#include "utils/TimeRecorder.h"
#include "utils/CommonUtil.h"
#include <time.h>
using namespace zilliz::vecwise;
namespace {
std::string CurrentTime() {
time_t tt;
time(&tt);
tt = tt + 8 * 3600;
tm *t = gmtime(&tt);
std::string str = std::to_string(t->tm_year + 1900) + "_" + std::to_string(t->tm_mon + 1)
+ "_" + std::to_string(t->tm_mday) + "_" + std::to_string(t->tm_hour)
+ "_" + std::to_string(t->tm_min) + "_" + std::to_string(t->tm_sec);
return str;
}
std::string GetGroupID() {
static std::string s_id(CurrentTime());
return s_id;
}
}
TEST(IdMapperTest, IDMAPPER_TEST) {
server::IVecIdMapper* mapper = server::IVecIdMapper::GetInstance();
std::string group_id = GetGroupID();
std::vector<std::string> nid = {"1", "50", "900", "10000"};
std::vector<std::string> sid = {"one", "fifty", "nine zero zero", "many"};
server::ServerError err = mapper->Put(nid, sid, group_id);
ASSERT_EQ(err, server::SERVER_SUCCESS);
err = mapper->Put(nid, std::vector<std::string>(), group_id);
ASSERT_NE(err, server::SERVER_SUCCESS);
std::vector<std::string> res;
err = mapper->Get(nid, res, group_id);
ASSERT_EQ(res.size(), nid.size());
for(size_t i = 0; i < res.size(); i++) {
ASSERT_EQ(res[i], sid[i]);
}
std::string str_id;
err = mapper->Get(nid[1], str_id, group_id);
ASSERT_EQ(str_id, "fifty");
err = mapper->Get(nid[1], str_id);
ASSERT_EQ(str_id, "");
err = mapper->Get(nid[2], str_id, group_id);
ASSERT_EQ(str_id, "nine zero zero");
err = mapper->Delete(nid[2], group_id);
ASSERT_EQ(err, server::SERVER_SUCCESS);
err = mapper->Get(nid[2], str_id, group_id);
ASSERT_EQ(str_id, "");
err = mapper->Get(nid[3], str_id, group_id);
ASSERT_EQ(str_id, "many");
err = mapper->DeleteGroup(group_id);
ASSERT_EQ(err, server::SERVER_SUCCESS);
err = mapper->Get(nid[3], str_id, group_id);
ASSERT_EQ(str_id, "");
std::string ct = CurrentTime();
err = mapper->Put("current_time", ct, "time");
ASSERT_EQ(err, server::SERVER_SUCCESS);
err = mapper->Get("current_time", str_id, "time");
ASSERT_EQ(str_id, ct);
//test performance
nid.clear();
sid.clear();
const int64_t count = 1000000;
{
server::TimeRecorder rc("prepare id data");
for (int64_t i = 0; i < count; i++) {
nid.push_back(std::to_string(i + 100000));
sid.push_back("val_" + std::to_string(i));
}
rc.Record("done!");
}
{
std::string str_info = "Insert " + std::to_string(count) + " k/v into mapper";
server::TimeRecorder rc(str_info);
err = mapper->Put(nid, sid);
ASSERT_EQ(err, server::SERVER_SUCCESS);
rc.Record("done!");
}
{
std::string str_info = "Get " + std::to_string(count) + " k/v from mapper";
server::TimeRecorder rc(str_info);
std::vector<std::string> res;
err = mapper->Get(nid, res);
ASSERT_EQ(res.size(), nid.size());
rc.Record("done!");
}
}