milvus-io
/
milvus
mirror of https://github.com/milvus-io/milvus.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

refactor(db): change groupschema and groupfileschema to table* 

Former-commit-id: fbc69151149faa8ed32547bcdd1a9935dac34070
pull/191/head
Xu Peng 2019-05-27 17:16:16 +08:00
parent aa60c62870
commit e94499cdff
13 changed files with 209 additions and 209 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
cpp/src/db/DB.h
View File

@ -21,8 +21,8 @@ class DB {
public: public:
static void Open(const Options& options, DB** dbptr); static void Open(const Options& options, DB** dbptr);
virtual Status add_group(meta::GroupSchema& group_info_) = 0; virtual Status add_group(meta::TableSchema& group_info_) = 0;
virtual Status get_group(meta::GroupSchema& group_info_) = 0; virtual Status get_group(meta::TableSchema& group_info_) = 0;
virtual Status delete_vectors(const std::string& group_id, virtual Status delete_vectors(const std::string& group_id,
const meta::DatesT& dates) = 0; const meta::DatesT& dates) = 0;
virtual Status has_group(const std::string& group_id_, bool& has_or_not_) = 0; virtual Status has_group(const std::string& group_id_, bool& has_or_not_) = 0;

22
cpp/src/db/DBImpl.cpp
View File

@ -35,12 +35,12 @@ DBImpl<EngineT>::DBImpl(const Options& options)
} }
template<typename EngineT> template<typename EngineT>
Status DBImpl<EngineT>::add_group(meta::GroupSchema& group_info) { Status DBImpl<EngineT>::add_group(meta::TableSchema& group_info) {
return _pMeta->add_group(group_info); return _pMeta->add_group(group_info);
} }
template<typename EngineT> template<typename EngineT>
Status DBImpl<EngineT>::get_group(meta::GroupSchema& group_info) { Status DBImpl<EngineT>::get_group(meta::TableSchema& group_info) {
return _pMeta->get_group(group_info); return _pMeta->get_group(group_info);
} }
@ -93,7 +93,7 @@ Status DBImpl<EngineT>::search(const std::string& group_id, size_t k, size_t nq,
meta::GroupFilesSchema raw_files; meta::GroupFilesSchema raw_files;
for (auto &day_files : files) { for (auto &day_files : files) {
for (auto &file : day_files.second) { for (auto &file : day_files.second) {
file.file_type == meta::GroupFileSchema::INDEX ? file.file_type == meta::TableFileSchema::INDEX ?
index_files.push_back(file) : raw_files.push_back(file); index_files.push_back(file) : raw_files.push_back(file);
} }
} }
@ -259,7 +259,7 @@ void DBImpl<EngineT>::background_call() {
template<typename EngineT> template<typename EngineT>
Status DBImpl<EngineT>::merge_files(const std::string& group_id, const meta::DateT& date, Status DBImpl<EngineT>::merge_files(const std::string& group_id, const meta::DateT& date,
const meta::GroupFilesSchema& files) { const meta::GroupFilesSchema& files) {
meta::GroupFileSchema group_file; meta::TableFileSchema group_file;
group_file.group_id = group_id; group_file.group_id = group_id;
group_file.date = date; group_file.date = date;
Status status = _pMeta->add_group_file(group_file); Status status = _pMeta->add_group_file(group_file);
@ -277,7 +277,7 @@ Status DBImpl<EngineT>::merge_files(const std::string& group_id, const meta::Dat
for (auto& file : files) { for (auto& file : files) {
index.Merge(file.location); index.Merge(file.location);
auto file_schema = file; auto file_schema = file;
file_schema.file_type = meta::GroupFileSchema::TO_DELETE; file_schema.file_type = meta::TableFileSchema::TO_DELETE;
updated.push_back(file_schema); updated.push_back(file_schema);
LOG(DEBUG) << "Merging file " << file_schema.file_id; LOG(DEBUG) << "Merging file " << file_schema.file_id;
index_size = index.Size(); index_size = index.Size();
@ -288,9 +288,9 @@ Status DBImpl<EngineT>::merge_files(const std::string& group_id, const meta::Dat
index.Serialize(); index.Serialize();
if (index_size >= _options.index_trigger_size) { if (index_size >= _options.index_trigger_size) {
group_file.file_type = meta::GroupFileSchema::TO_INDEX; group_file.file_type = meta::TableFileSchema::TO_INDEX;
} else { } else {
group_file.file_type = meta::GroupFileSchema::RAW; group_file.file_type = meta::TableFileSchema::RAW;
} }
group_file.size = index_size; group_file.size = index_size;
updated.push_back(group_file); updated.push_back(group_file);
@ -336,8 +336,8 @@ Status DBImpl<EngineT>::background_merge_files(const std::string& group_id) {
} }
template<typename EngineT> template<typename EngineT>
Status DBImpl<EngineT>::build_index(const meta::GroupFileSchema& file) { Status DBImpl<EngineT>::build_index(const meta::TableFileSchema& file) {
meta::GroupFileSchema group_file; meta::TableFileSchema group_file;
group_file.group_id = file.group_id; group_file.group_id = file.group_id;
group_file.date = file.date; group_file.date = file.date;
Status status = _pMeta->add_group_file(group_file); Status status = _pMeta->add_group_file(group_file);
@ -350,11 +350,11 @@ Status DBImpl<EngineT>::build_index(const meta::GroupFileSchema& file) {
to_index.Load(); to_index.Load();
auto index = to_index.BuildIndex(group_file.location); auto index = to_index.BuildIndex(group_file.location);
group_file.file_type = meta::GroupFileSchema::INDEX; group_file.file_type = meta::TableFileSchema::INDEX;
group_file.size = index->Size(); group_file.size = index->Size();
auto to_remove = file; auto to_remove = file;
to_remove.file_type = meta::GroupFileSchema::TO_DELETE; to_remove.file_type = meta::TableFileSchema::TO_DELETE;
meta::GroupFilesSchema update_files = {to_remove, group_file}; meta::GroupFilesSchema update_files = {to_remove, group_file};
_pMeta->update_files(update_files); _pMeta->update_files(update_files);

6
cpp/src/db/DBImpl.h
View File

@ -33,8 +33,8 @@ public:
DBImpl(const Options& options); DBImpl(const Options& options);
virtual Status add_group(meta::GroupSchema& group_info) override; virtual Status add_group(meta::TableSchema& group_info) override;
virtual Status get_group(meta::GroupSchema& group_info) override; virtual Status get_group(meta::TableSchema& group_info) override;
virtual Status delete_vectors(const std::string& group_id, const meta::DatesT& dates) override; virtual Status delete_vectors(const std::string& group_id, const meta::DatesT& dates) override;
virtual Status has_group(const std::string& group_id_, bool& has_or_not_) override; virtual Status has_group(const std::string& group_id_, bool& has_or_not_) override;
@ -62,7 +62,7 @@ public:
private: private:
void background_build_index(); void background_build_index();
Status build_index(const meta::GroupFileSchema&); Status build_index(const meta::TableFileSchema&);
Status try_build_index(); Status try_build_index();
Status merge_files(const std::string& group_id, Status merge_files(const std::string& group_id,
const meta::DateT& date, const meta::DateT& date,

230
cpp/src/db/DBMetaImpl.cpp
View File

@ -28,20 +28,20 @@ using namespace sqlite_orm;
inline auto StoragePrototype(const std::string& path) { inline auto StoragePrototype(const std::string& path) {
return make_storage(path, return make_storage(path,
make_table("Group", make_table("Group",
make_column("id", &GroupSchema::id, primary_key()), make_column("id", &TableSchema::id, primary_key()),
make_column("group_id", &GroupSchema::group_id, unique()), make_column("group_id", &TableSchema::group_id, unique()),
make_column("dimension", &GroupSchema::dimension), make_column("dimension", &TableSchema::dimension),
make_column("created_on", &GroupSchema::created_on), make_column("created_on", &TableSchema::created_on),
make_column("files_cnt", &GroupSchema::files_cnt, default_value(0))), make_column("files_cnt", &TableSchema::files_cnt, default_value(0))),
make_table("GroupFile", make_table("GroupFile",
make_column("id", &GroupFileSchema::id, primary_key()), make_column("id", &TableFileSchema::id, primary_key()),
make_column("group_id", &GroupFileSchema::group_id), make_column("group_id", &TableFileSchema::group_id),
make_column("file_id", &GroupFileSchema::file_id), make_column("file_id", &TableFileSchema::file_id),
make_column("file_type", &GroupFileSchema::file_type), make_column("file_type", &TableFileSchema::file_type),
make_column("size", &GroupFileSchema::size, default_value(0)), make_column("size", &TableFileSchema::size, default_value(0)),
make_column("updated_time", &GroupFileSchema::updated_time), make_column("updated_time", &TableFileSchema::updated_time),
make_column("created_on", &GroupFileSchema::created_on), make_column("created_on", &TableFileSchema::created_on),
make_column("date", &GroupFileSchema::date)) make_column("date", &TableFileSchema::date))
); );
} }
@ -59,7 +59,7 @@ std::string DBMetaImpl::GetGroupDatePartitionPath(const std::string& group_id, D
return ss.str(); return ss.str();
} }
void DBMetaImpl::GetGroupFilePath(GroupFileSchema& group_file) { void DBMetaImpl::GetGroupFilePath(TableFileSchema& group_file) {
if (group_file.date == EmptyDate) { if (group_file.date == EmptyDate) {
group_file.date = Meta::GetDate(); group_file.date = Meta::GetDate();
} }
@ -117,7 +117,7 @@ Status DBMetaImpl::delete_group_partitions(const std::string& group_id,
return Status::OK(); return Status::OK();
} }
GroupSchema group_info; TableSchema group_info;
group_info.group_id = group_id; group_info.group_id = group_id;
auto status = get_group(group_info); auto status = get_group(group_info);
if (!status.ok()) { if (!status.ok()) {
@ -135,11 +135,11 @@ Status DBMetaImpl::delete_group_partitions(const std::string& group_id,
try { try {
ConnectorPtr->update_all( ConnectorPtr->update_all(
set( set(
c(&GroupFileSchema::file_type) = (int)GroupFileSchema::TO_DELETE c(&TableFileSchema::file_type) = (int)TableFileSchema::TO_DELETE
), ),
where( where(
c(&GroupFileSchema::group_id) == group_id and c(&TableFileSchema::group_id) == group_id and
in(&GroupFileSchema::date, dates) in(&TableFileSchema::date, dates)
)); ));
} catch (std::exception & e) { } catch (std::exception & e) {
LOG(DEBUG) << e.what(); LOG(DEBUG) << e.what();
@ -148,7 +148,7 @@ Status DBMetaImpl::delete_group_partitions(const std::string& group_id,
return Status::OK(); return Status::OK();
} }
Status DBMetaImpl::add_group(GroupSchema& group_info) { Status DBMetaImpl::add_group(TableSchema& group_info) {
if (group_info.group_id == "") { if (group_info.group_id == "") {
NextGroupId(group_info.group_id); NextGroupId(group_info.group_id);
} }
@ -178,17 +178,17 @@ Status DBMetaImpl::add_group(GroupSchema& group_info) {
return Status::OK(); return Status::OK();
} }
Status DBMetaImpl::get_group(GroupSchema& group_info) { Status DBMetaImpl::get_group(TableSchema& group_info) {
return get_group_no_lock(group_info); return get_group_no_lock(group_info);
} }
Status DBMetaImpl::get_group_no_lock(GroupSchema& group_info) { Status DBMetaImpl::get_group_no_lock(TableSchema& group_info) {
try { try {
auto groups = ConnectorPtr->select(columns(&GroupSchema::id, auto groups = ConnectorPtr->select(columns(&TableSchema::id,
&GroupSchema::group_id, &TableSchema::group_id,
&GroupSchema::files_cnt, &TableSchema::files_cnt,
&GroupSchema::dimension), &TableSchema::dimension),
where(c(&GroupSchema::group_id) == group_info.group_id)); where(c(&TableSchema::group_id) == group_info.group_id));
assert(groups.size() <= 1); assert(groups.size() <= 1);
if (groups.size() == 1) { if (groups.size() == 1) {
group_info.id = std::get<0>(groups[0]); group_info.id = std::get<0>(groups[0]);
@ -207,8 +207,8 @@ Status DBMetaImpl::get_group_no_lock(GroupSchema& group_info) {
Status DBMetaImpl::has_group(const std::string& group_id, bool& has_or_not) { Status DBMetaImpl::has_group(const std::string& group_id, bool& has_or_not) {
try { try {
auto groups = ConnectorPtr->select(columns(&GroupSchema::id), auto groups = ConnectorPtr->select(columns(&TableSchema::id),
where(c(&GroupSchema::group_id) == group_id)); where(c(&TableSchema::group_id) == group_id));
assert(groups.size() <= 1); assert(groups.size() <= 1);
if (groups.size() == 1) { if (groups.size() == 1) {
has_or_not = true; has_or_not = true;
@ -222,11 +222,11 @@ Status DBMetaImpl::has_group(const std::string& group_id, bool& has_or_not) {
return Status::OK(); return Status::OK();
} }
Status DBMetaImpl::add_group_file(GroupFileSchema& group_file) { Status DBMetaImpl::add_group_file(TableFileSchema& group_file) {
if (group_file.date == EmptyDate) { if (group_file.date == EmptyDate) {
group_file.date = Meta::GetDate(); group_file.date = Meta::GetDate();
} }
GroupSchema group_info; TableSchema group_info;
group_info.group_id = group_file.group_id; group_info.group_id = group_file.group_id;
auto status = get_group(group_info); auto status = get_group(group_info);
if (!status.ok()) { if (!status.ok()) {
@ -234,7 +234,7 @@ Status DBMetaImpl::add_group_file(GroupFileSchema& group_file) {
} }
NextFileId(group_file.file_id); NextFileId(group_file.file_id);
group_file.file_type = GroupFileSchema::NEW; group_file.file_type = TableFileSchema::NEW;
group_file.dimension = group_info.dimension; group_file.dimension = group_info.dimension;
group_file.size = 0; group_file.size = 0;
group_file.created_on = utils::GetMicroSecTimeStamp(); group_file.created_on = utils::GetMicroSecTimeStamp();
@ -267,18 +267,18 @@ Status DBMetaImpl::files_to_index(GroupFilesSchema& files) {
files.clear(); files.clear();
try { try {
auto selected = ConnectorPtr->select(columns(&GroupFileSchema::id, auto selected = ConnectorPtr->select(columns(&TableFileSchema::id,
&GroupFileSchema::group_id, &TableFileSchema::group_id,
&GroupFileSchema::file_id, &TableFileSchema::file_id,
&GroupFileSchema::file_type, &TableFileSchema::file_type,
&GroupFileSchema::size, &TableFileSchema::size,
&GroupFileSchema::date), &TableFileSchema::date),
where(c(&GroupFileSchema::file_type) == (int)GroupFileSchema::TO_INDEX)); where(c(&TableFileSchema::file_type) == (int)TableFileSchema::TO_INDEX));
std::map<std::string, GroupSchema> groups; std::map<std::string, TableSchema> groups;
for (auto& file : selected) { for (auto& file : selected) {
GroupFileSchema group_file; TableFileSchema group_file;
group_file.id = std::get<0>(file); group_file.id = std::get<0>(file);
group_file.group_id = std::get<1>(file); group_file.group_id = std::get<1>(file);
group_file.file_id = std::get<2>(file); group_file.file_id = std::get<2>(file);
@ -288,7 +288,7 @@ Status DBMetaImpl::files_to_index(GroupFilesSchema& files) {
GetGroupFilePath(group_file); GetGroupFilePath(group_file);
auto groupItr = groups.find(group_file.group_id); auto groupItr = groups.find(group_file.group_id);
if (groupItr == groups.end()) { if (groupItr == groups.end()) {
GroupSchema group_info; TableSchema group_info;
group_info.group_id = group_file.group_id; group_info.group_id = group_file.group_id;
auto status = get_group_no_lock(group_info); auto status = get_group_no_lock(group_info);
if (!status.ok()) { if (!status.ok()) {
@ -315,19 +315,19 @@ Status DBMetaImpl::files_to_search(const std::string &group_id,
const DatesT& dates = (partition.empty() == true) ? today : partition; const DatesT& dates = (partition.empty() == true) ? today : partition;
try { try {
auto selected = ConnectorPtr->select(columns(&GroupFileSchema::id, auto selected = ConnectorPtr->select(columns(&TableFileSchema::id,
&GroupFileSchema::group_id, &TableFileSchema::group_id,
&GroupFileSchema::file_id, &TableFileSchema::file_id,
&GroupFileSchema::file_type, &TableFileSchema::file_type,
&GroupFileSchema::size, &TableFileSchema::size,
&GroupFileSchema::date), &TableFileSchema::date),
where(c(&GroupFileSchema::group_id) == group_id and where(c(&TableFileSchema::group_id) == group_id and
in(&GroupFileSchema::date, dates) and in(&TableFileSchema::date, dates) and
(c(&GroupFileSchema::file_type) == (int) GroupFileSchema::RAW or (c(&TableFileSchema::file_type) == (int) TableFileSchema::RAW or
c(&GroupFileSchema::file_type) == (int) GroupFileSchema::TO_INDEX or c(&TableFileSchema::file_type) == (int) TableFileSchema::TO_INDEX or
c(&GroupFileSchema::file_type) == (int) GroupFileSchema::INDEX))); c(&TableFileSchema::file_type) == (int) TableFileSchema::INDEX)));
GroupSchema group_info; TableSchema group_info;
group_info.group_id = group_id; group_info.group_id = group_id;
auto status = get_group_no_lock(group_info); auto status = get_group_no_lock(group_info);
if (!status.ok()) { if (!status.ok()) {
@ -335,7 +335,7 @@ Status DBMetaImpl::files_to_search(const std::string &group_id,
} }
for (auto& file : selected) { for (auto& file : selected) {
GroupFileSchema group_file; TableFileSchema group_file;
group_file.id = std::get<0>(file); group_file.id = std::get<0>(file);
group_file.group_id = std::get<1>(file); group_file.group_id = std::get<1>(file);
group_file.file_id = std::get<2>(file); group_file.file_id = std::get<2>(file);
@ -363,16 +363,16 @@ Status DBMetaImpl::files_to_merge(const std::string& group_id,
files.clear(); files.clear();
try { try {
auto selected = ConnectorPtr->select(columns(&GroupFileSchema::id, auto selected = ConnectorPtr->select(columns(&TableFileSchema::id,
&GroupFileSchema::group_id, &TableFileSchema::group_id,
&GroupFileSchema::file_id, &TableFileSchema::file_id,
&GroupFileSchema::file_type, &TableFileSchema::file_type,
&GroupFileSchema::size, &TableFileSchema::size,
&GroupFileSchema::date), &TableFileSchema::date),
where(c(&GroupFileSchema::file_type) == (int)GroupFileSchema::RAW and where(c(&TableFileSchema::file_type) == (int)TableFileSchema::RAW and
c(&GroupFileSchema::group_id) == group_id)); c(&TableFileSchema::group_id) == group_id));
GroupSchema group_info; TableSchema group_info;
group_info.group_id = group_id; group_info.group_id = group_id;
auto status = get_group_no_lock(group_info); auto status = get_group_no_lock(group_info);
if (!status.ok()) { if (!status.ok()) {
@ -380,7 +380,7 @@ Status DBMetaImpl::files_to_merge(const std::string& group_id,
} }
for (auto& file : selected) { for (auto& file : selected) {
GroupFileSchema group_file; TableFileSchema group_file;
group_file.id = std::get<0>(file); group_file.id = std::get<0>(file);
group_file.group_id = std::get<1>(file); group_file.group_id = std::get<1>(file);
group_file.file_id = std::get<2>(file); group_file.file_id = std::get<2>(file);
@ -412,16 +412,16 @@ Status DBMetaImpl::has_group_file(const std::string& group_id_,
Status DBMetaImpl::get_group_file(const std::string& group_id_, Status DBMetaImpl::get_group_file(const std::string& group_id_,
const std::string& file_id_, const std::string& file_id_,
GroupFileSchema& group_file_info_) { TableFileSchema& group_file_info_) {
try { try {
auto files = ConnectorPtr->select(columns(&GroupFileSchema::id, auto files = ConnectorPtr->select(columns(&TableFileSchema::id,
&GroupFileSchema::group_id, &TableFileSchema::group_id,
&GroupFileSchema::file_id, &TableFileSchema::file_id,
&GroupFileSchema::file_type, &TableFileSchema::file_type,
&GroupFileSchema::size, &TableFileSchema::size,
&GroupFileSchema::date), &TableFileSchema::date),
where(c(&GroupFileSchema::file_id) == file_id_ and where(c(&TableFileSchema::file_id) == file_id_ and
c(&GroupFileSchema::group_id) == group_id_ c(&TableFileSchema::group_id) == group_id_
)); ));
assert(files.size() <= 1); assert(files.size() <= 1);
if (files.size() == 1) { if (files.size() == 1) {
@ -466,11 +466,11 @@ Status DBMetaImpl::archive_files() {
{ {
ConnectorPtr->update_all( ConnectorPtr->update_all(
set( set(
c(&GroupFileSchema::file_type) = (int)GroupFileSchema::TO_DELETE c(&TableFileSchema::file_type) = (int)TableFileSchema::TO_DELETE
), ),
where( where(
c(&GroupFileSchema::created_on) < (long)(now - usecs) and c(&TableFileSchema::created_on) < (long)(now - usecs) and
c(&GroupFileSchema::file_type) != (int)GroupFileSchema::TO_DELETE c(&TableFileSchema::file_type) != (int)TableFileSchema::TO_DELETE
)); ));
} catch (std::exception & e) { } catch (std::exception & e) {
LOG(DEBUG) << e.what(); LOG(DEBUG) << e.what();
@ -493,9 +493,9 @@ Status DBMetaImpl::archive_files() {
Status DBMetaImpl::size(long& result) { Status DBMetaImpl::size(long& result) {
result = 0; result = 0;
try { try {
auto selected = ConnectorPtr->select(columns(sum(&GroupFileSchema::size)), auto selected = ConnectorPtr->select(columns(sum(&TableFileSchema::size)),
where( where(
c(&GroupFileSchema::file_type) != (int)GroupFileSchema::TO_DELETE c(&TableFileSchema::file_type) != (int)TableFileSchema::TO_DELETE
)); ));
for (auto& sub_query : selected) { for (auto& sub_query : selected) {
@ -518,16 +518,16 @@ Status DBMetaImpl::discard_files_of_size(long to_discard_size) {
return Status::OK(); return Status::OK();
} }
try { try {
auto selected = ConnectorPtr->select(columns(&GroupFileSchema::id, auto selected = ConnectorPtr->select(columns(&TableFileSchema::id,
&GroupFileSchema::size), &TableFileSchema::size),
where(c(&GroupFileSchema::file_type) != (int)GroupFileSchema::TO_DELETE), where(c(&TableFileSchema::file_type) != (int)TableFileSchema::TO_DELETE),
order_by(&GroupFileSchema::id), order_by(&TableFileSchema::id),
limit(10)); limit(10));
std::vector<int> ids; std::vector<int> ids;
for (auto& file : selected) { for (auto& file : selected) {
if (to_discard_size <= 0) break; if (to_discard_size <= 0) break;
GroupFileSchema group_file; TableFileSchema group_file;
group_file.id = std::get<0>(file); group_file.id = std::get<0>(file);
group_file.size = std::get<1>(file); group_file.size = std::get<1>(file);
ids.push_back(group_file.id); ids.push_back(group_file.id);
@ -541,10 +541,10 @@ Status DBMetaImpl::discard_files_of_size(long to_discard_size) {
ConnectorPtr->update_all( ConnectorPtr->update_all(
set( set(
c(&GroupFileSchema::file_type) = (int)GroupFileSchema::TO_DELETE c(&TableFileSchema::file_type) = (int)TableFileSchema::TO_DELETE
), ),
where( where(
in(&GroupFileSchema::id, ids) in(&TableFileSchema::id, ids)
)); ));
} catch (std::exception & e) { } catch (std::exception & e) {
@ -556,7 +556,7 @@ Status DBMetaImpl::discard_files_of_size(long to_discard_size) {
return discard_files_of_size(to_discard_size); return discard_files_of_size(to_discard_size);
} }
Status DBMetaImpl::update_group_file(GroupFileSchema& group_file) { Status DBMetaImpl::update_group_file(TableFileSchema& group_file) {
group_file.updated_time = utils::GetMicroSecTimeStamp(); group_file.updated_time = utils::GetMicroSecTimeStamp();
try { try {
ConnectorPtr->update(group_file); ConnectorPtr->update(group_file);
@ -590,19 +590,19 @@ Status DBMetaImpl::update_files(GroupFilesSchema& files) {
Status DBMetaImpl::cleanup_ttl_files(uint16_t seconds) { Status DBMetaImpl::cleanup_ttl_files(uint16_t seconds) {
auto now = utils::GetMicroSecTimeStamp(); auto now = utils::GetMicroSecTimeStamp();
try { try {
auto selected = ConnectorPtr->select(columns(&GroupFileSchema::id, auto selected = ConnectorPtr->select(columns(&TableFileSchema::id,
&GroupFileSchema::group_id, &TableFileSchema::group_id,
&GroupFileSchema::file_id, &TableFileSchema::file_id,
&GroupFileSchema::file_type, &TableFileSchema::file_type,
&GroupFileSchema::size, &TableFileSchema::size,
&GroupFileSchema::date), &TableFileSchema::date),
where(c(&GroupFileSchema::file_type) == (int)GroupFileSchema::TO_DELETE and where(c(&TableFileSchema::file_type) == (int)TableFileSchema::TO_DELETE and
c(&GroupFileSchema::updated_time) > now - seconds*US_PS)); c(&TableFileSchema::updated_time) > now - seconds*US_PS));
GroupFilesSchema updated; GroupFilesSchema updated;
for (auto& file : selected) { for (auto& file : selected) {
GroupFileSchema group_file; TableFileSchema group_file;
group_file.id = std::get<0>(file); group_file.id = std::get<0>(file);
group_file.group_id = std::get<1>(file); group_file.group_id = std::get<1>(file);
group_file.file_id = std::get<2>(file); group_file.file_id = std::get<2>(file);
@ -610,10 +610,10 @@ Status DBMetaImpl::cleanup_ttl_files(uint16_t seconds) {
group_file.size = std::get<4>(file); group_file.size = std::get<4>(file);
group_file.date = std::get<5>(file); group_file.date = std::get<5>(file);
GetGroupFilePath(group_file); GetGroupFilePath(group_file);
if (group_file.file_type == GroupFileSchema::TO_DELETE) { if (group_file.file_type == TableFileSchema::TO_DELETE) {
boost::filesystem::remove(group_file.location); boost::filesystem::remove(group_file.location);
} }
ConnectorPtr->remove<GroupFileSchema>(group_file.id); ConnectorPtr->remove<TableFileSchema>(group_file.id);
/* LOG(DEBUG) << "Removing deleted id=" << group_file.id << " location=" << group_file.location << std::endl; */ /* LOG(DEBUG) << "Removing deleted id=" << group_file.id << " location=" << group_file.location << std::endl; */
} }
} catch (std::exception & e) { } catch (std::exception & e) {
@ -626,19 +626,19 @@ Status DBMetaImpl::cleanup_ttl_files(uint16_t seconds) {
Status DBMetaImpl::cleanup() { Status DBMetaImpl::cleanup() {
try { try {
auto selected = ConnectorPtr->select(columns(&GroupFileSchema::id, auto selected = ConnectorPtr->select(columns(&TableFileSchema::id,
&GroupFileSchema::group_id, &TableFileSchema::group_id,
&GroupFileSchema::file_id, &TableFileSchema::file_id,
&GroupFileSchema::file_type, &TableFileSchema::file_type,
&GroupFileSchema::size, &TableFileSchema::size,
&GroupFileSchema::date), &TableFileSchema::date),
where(c(&GroupFileSchema::file_type) == (int)GroupFileSchema::TO_DELETE or where(c(&TableFileSchema::file_type) == (int)TableFileSchema::TO_DELETE or
c(&GroupFileSchema::file_type) == (int)GroupFileSchema::NEW)); c(&TableFileSchema::file_type) == (int)TableFileSchema::NEW));
GroupFilesSchema updated; GroupFilesSchema updated;
for (auto& file : selected) { for (auto& file : selected) {
GroupFileSchema group_file; TableFileSchema group_file;
group_file.id = std::get<0>(file); group_file.id = std::get<0>(file);
group_file.group_id = std::get<1>(file); group_file.group_id = std::get<1>(file);
group_file.file_id = std::get<2>(file); group_file.file_id = std::get<2>(file);
@ -646,10 +646,10 @@ Status DBMetaImpl::cleanup() {
group_file.size = std::get<4>(file); group_file.size = std::get<4>(file);
group_file.date = std::get<5>(file); group_file.date = std::get<5>(file);
GetGroupFilePath(group_file); GetGroupFilePath(group_file);
if (group_file.file_type == GroupFileSchema::TO_DELETE) { if (group_file.file_type == TableFileSchema::TO_DELETE) {
boost::filesystem::remove(group_file.location); boost::filesystem::remove(group_file.location);
} }
ConnectorPtr->remove<GroupFileSchema>(group_file.id); ConnectorPtr->remove<TableFileSchema>(group_file.id);
/* LOG(DEBUG) << "Removing id=" << group_file.id << " location=" << group_file.location << std::endl; */ /* LOG(DEBUG) << "Removing id=" << group_file.id << " location=" << group_file.location << std::endl; */
} }
} catch (std::exception & e) { } catch (std::exception & e) {
@ -663,14 +663,14 @@ Status DBMetaImpl::cleanup() {
Status DBMetaImpl::count(const std::string& group_id, long& result) { Status DBMetaImpl::count(const std::string& group_id, long& result) {
try { try {
auto selected = ConnectorPtr->select(columns(&GroupFileSchema::size, auto selected = ConnectorPtr->select(columns(&TableFileSchema::size,
&GroupFileSchema::date), &TableFileSchema::date),
where((c(&GroupFileSchema::file_type) == (int)GroupFileSchema::RAW or where((c(&TableFileSchema::file_type) == (int)TableFileSchema::RAW or
c(&GroupFileSchema::file_type) == (int)GroupFileSchema::TO_INDEX or c(&TableFileSchema::file_type) == (int)TableFileSchema::TO_INDEX or
c(&GroupFileSchema::file_type) == (int)GroupFileSchema::INDEX) and c(&TableFileSchema::file_type) == (int)TableFileSchema::INDEX) and
c(&GroupFileSchema::group_id) == group_id)); c(&TableFileSchema::group_id) == group_id));
GroupSchema group_info; TableSchema group_info;
group_info.group_id = group_id; group_info.group_id = group_id;
auto status = get_group_no_lock(group_info); auto status = get_group_no_lock(group_info);
if (!status.ok()) { if (!status.ok()) {

14
cpp/src/db/DBMetaImpl.h
View File

@ -19,11 +19,11 @@ class DBMetaImpl : public Meta {
public: public:
DBMetaImpl(const DBMetaOptions& options_); DBMetaImpl(const DBMetaOptions& options_);
virtual Status add_group(GroupSchema& group_info) override; virtual Status add_group(TableSchema& group_info) override;
virtual Status get_group(GroupSchema& group_info_) override; virtual Status get_group(TableSchema& group_info_) override;
virtual Status has_group(const std::string& group_id_, bool& has_or_not_) override; virtual Status has_group(const std::string& group_id_, bool& has_or_not_) override;
virtual Status add_group_file(GroupFileSchema& group_file_info) override; virtual Status add_group_file(TableFileSchema& group_file_info) override;
virtual Status delete_group_partitions(const std::string& group_id, virtual Status delete_group_partitions(const std::string& group_id,
const meta::DatesT& dates) override; const meta::DatesT& dates) override;
@ -32,8 +32,8 @@ public:
bool& has_or_not_) override; bool& has_or_not_) override;
virtual Status get_group_file(const std::string& group_id_, virtual Status get_group_file(const std::string& group_id_,
const std::string& file_id_, const std::string& file_id_,
GroupFileSchema& group_file_info_) override; TableFileSchema& group_file_info_) override;
virtual Status update_group_file(GroupFileSchema& group_file_) override; virtual Status update_group_file(TableFileSchema& group_file_) override;
virtual Status get_group_files(const std::string& group_id_, virtual Status get_group_files(const std::string& group_id_,
const int date_delta_, const int date_delta_,
@ -68,10 +68,10 @@ private:
Status NextFileId(std::string& file_id); Status NextFileId(std::string& file_id);
Status NextGroupId(std::string& group_id); Status NextGroupId(std::string& group_id);
Status discard_files_of_size(long to_discard_size); Status discard_files_of_size(long to_discard_size);
Status get_group_no_lock(GroupSchema& group_info); Status get_group_no_lock(TableSchema& group_info);
std::string GetGroupPath(const std::string& group_id); std::string GetGroupPath(const std::string& group_id);
std::string GetGroupDatePartitionPath(const std::string& group_id, DateT& date); std::string GetGroupDatePartitionPath(const std::string& group_id, DateT& date);
void GetGroupFilePath(GroupFileSchema& group_file); void GetGroupFilePath(TableFileSchema& group_file);
Status initialize(); Status initialize();
const DBMetaOptions _options; const DBMetaOptions _options;

28
cpp/src/db/LocalMetaImpl.cpp
View File

@ -38,8 +38,8 @@ std::string LocalMetaImpl::GetGroupDatePartitionPath(const std::string& group_id
} }
std::string LocalMetaImpl::GetNextGroupFileLocationByPartition(const std::string& group_id, DateT& date, std::string LocalMetaImpl::GetNextGroupFileLocationByPartition(const std::string& group_id, DateT& date,
GroupFileSchema::FILE_TYPE file_type) { TableFileSchema::FILE_TYPE file_type) {
std::string suffix = (file_type == GroupFileSchema::RAW) ? ".raw" : ".index"; std::string suffix = (file_type == TableFileSchema::RAW) ? ".raw" : ".index";
SimpleIDGenerator g; SimpleIDGenerator g;
std::stringstream ss; std::stringstream ss;
ss << GetGroupPath(group_id) << "/" << date << "/" << g.getNextIDNumber() << suffix; ss << GetGroupPath(group_id) << "/" << date << "/" << g.getNextIDNumber() << suffix;
@ -54,7 +54,7 @@ std::string LocalMetaImpl::GetGroupMetaPath(const std::string& group_id) {
return GetGroupMetaPathByGroupPath(GetGroupPath(group_id)); return GetGroupMetaPathByGroupPath(GetGroupPath(group_id));
} }
Status LocalMetaImpl::GetGroupMetaInfoByPath(const std::string& path, GroupSchema& group_info) { Status LocalMetaImpl::GetGroupMetaInfoByPath(const std::string& path, TableSchema& group_info) {
boost::property_tree::ptree ptree; boost::property_tree::ptree ptree;
boost::property_tree::read_json(path, ptree); boost::property_tree::read_json(path, ptree);
auto files_cnt = ptree.get_child("files_cnt").data(); auto files_cnt = ptree.get_child("files_cnt").data();
@ -71,7 +71,7 @@ Status LocalMetaImpl::GetGroupMetaInfoByPath(const std::string& path, GroupSchem
} }
Status LocalMetaImpl::GetGroupMetaInfo(const std::string& group_id, GroupSchema& group_info) { Status LocalMetaImpl::GetGroupMetaInfo(const std::string& group_id, TableSchema& group_info) {
group_info.group_id = group_id; group_info.group_id = group_id;
return GetGroupMetaInfoByPath(GetGroupMetaPath(group_id), group_info); return GetGroupMetaInfoByPath(GetGroupMetaPath(group_id), group_info);
} }
@ -90,7 +90,7 @@ Status LocalMetaImpl::initialize() {
return Status::OK(); return Status::OK();
} }
Status LocalMetaImpl::add_group(GroupSchema& group_info) { Status LocalMetaImpl::add_group(TableSchema& group_info) {
std::string real_gid; std::string real_gid;
size_t id = SimpleIDGenerator().getNextIDNumber(); size_t id = SimpleIDGenerator().getNextIDNumber();
if (group_info.group_id == "") { if (group_info.group_id == "") {
@ -123,7 +123,7 @@ Status LocalMetaImpl::add_group(GroupSchema& group_info) {
return Status::OK(); return Status::OK();
} }
Status LocalMetaImpl::get_group(GroupSchema& group_info) { Status LocalMetaImpl::get_group(TableSchema& group_info) {
bool group_exist; bool group_exist;
has_group(group_info.group_id, group_exist); has_group(group_info.group_id, group_exist);
if (!group_exist) { if (!group_exist) {
@ -138,8 +138,8 @@ Status LocalMetaImpl::has_group(const std::string& group_id, bool& has_or_not) {
return Status::OK(); return Status::OK();
} }
Status LocalMetaImpl::add_group_file(GroupFileSchema& group_file_info) { Status LocalMetaImpl::add_group_file(TableFileSchema& group_file_info) {
GroupSchema group_info; TableSchema group_info;
/* auto status = get_group(group_info); */ /* auto status = get_group(group_info); */
/* if (!status.ok()) { */ /* if (!status.ok()) { */
/* return status; */ /* return status; */
@ -159,7 +159,7 @@ Status LocalMetaImpl::files_to_index(GroupFilesSchema& files) {
boost::filesystem::directory_iterator end_itr; boost::filesystem::directory_iterator end_itr;
for (boost::filesystem::directory_iterator itr(_options.path); itr != end_itr; ++itr) { for (boost::filesystem::directory_iterator itr(_options.path); itr != end_itr; ++itr) {
auto group_path = itr->path().string(); auto group_path = itr->path().string();
GroupSchema group_info; TableSchema group_info;
GetGroupMetaInfoByPath(GetGroupMetaPathByGroupPath(group_path), group_info); GetGroupMetaInfoByPath(GetGroupMetaPathByGroupPath(group_path), group_info);
for (boost::filesystem::directory_iterator innerItr(group_path); innerItr != end_itr; ++innerItr) { for (boost::filesystem::directory_iterator innerItr(group_path); innerItr != end_itr; ++innerItr) {
auto partition_path = innerItr->path().string(); auto partition_path = innerItr->path().string();
@ -169,7 +169,7 @@ Status LocalMetaImpl::files_to_index(GroupFilesSchema& files) {
if (suffix == "index") continue; if (suffix == "index") continue;
if (INDEX_TRIGGER_SIZE >= GetFileSize(location)) continue; if (INDEX_TRIGGER_SIZE >= GetFileSize(location)) continue;
std::cout << "[About to index] " << location << std::endl; std::cout << "[About to index] " << location << std::endl;
GroupFileSchema f; TableFileSchema f;
f.location = location; f.location = location;
/* f.group_id = group_id; */ /* f.group_id = group_id; */
f.dimension = group_info.dimension; f.dimension = group_info.dimension;
@ -188,7 +188,7 @@ Status LocalMetaImpl::files_to_merge(const std::string& group_id,
/* boost::filesystem::directory_iterator end_itr; */ /* boost::filesystem::directory_iterator end_itr; */
/* for (boost::filesystem::directory_iterator itr(_options.path); itr != end_itr; ++itr) { */ /* for (boost::filesystem::directory_iterator itr(_options.path); itr != end_itr; ++itr) { */
/* auto group_path = itr->path().string(); */ /* auto group_path = itr->path().string(); */
/* GroupSchema group_info; */ /* TableSchema group_info; */
/* GetGroupMetaInfoByPath(GetGroupMetaPathByGroupPath(group_path), group_info); */ /* GetGroupMetaInfoByPath(GetGroupMetaPathByGroupPath(group_path), group_info); */
/* for (boost::filesystem::directory_iterator innerItr(group_path); innerItr != end_itr; ++innerItr) { */ /* for (boost::filesystem::directory_iterator innerItr(group_path); innerItr != end_itr; ++innerItr) { */
/* auto partition_path = innerItr->path().string(); */ /* auto partition_path = innerItr->path().string(); */
@ -198,7 +198,7 @@ Status LocalMetaImpl::files_to_merge(const std::string& group_id,
/* if (suffix == "index") continue; */ /* if (suffix == "index") continue; */
/* if (INDEX_TRIGGER_SIZE < GetFileSize(location)) continue; */ /* if (INDEX_TRIGGER_SIZE < GetFileSize(location)) continue; */
/* std::cout << "[About to index] " << location << std::endl; */ /* std::cout << "[About to index] " << location << std::endl; */
/* GroupFileSchema f; */ /* TableFileSchema f; */
/* f.location = location; */ /* f.location = location; */
/* f.group_id = group_id; */ /* f.group_id = group_id; */
/* f.dimension = group_info.dimension; */ /* f.dimension = group_info.dimension; */
@ -219,7 +219,7 @@ Status LocalMetaImpl::has_group_file(const std::string& group_id_,
Status LocalMetaImpl::get_group_file(const std::string& group_id_, Status LocalMetaImpl::get_group_file(const std::string& group_id_,
const std::string& file_id_, const std::string& file_id_,
GroupFileSchema& group_file_info_) { TableFileSchema& group_file_info_) {
//PXU TODO //PXU TODO
return Status::OK(); return Status::OK();
} }
@ -231,7 +231,7 @@ Status LocalMetaImpl::get_group_files(const std::string& group_id_,
return Status::OK(); return Status::OK();
} }
Status LocalMetaImpl::update_group_file(GroupFileSchema& group_file_) { Status LocalMetaImpl::update_group_file(TableFileSchema& group_file_) {
//PXU TODO //PXU TODO
return Status::OK(); return Status::OK();
} }

18
cpp/src/db/LocalMetaImpl.h
View File

@ -18,11 +18,11 @@ public:
const size_t INDEX_TRIGGER_SIZE = 1024*1024*500; const size_t INDEX_TRIGGER_SIZE = 1024*1024*500;
LocalMetaImpl(const DBMetaOptions& options_); LocalMetaImpl(const DBMetaOptions& options_);
virtual Status add_group(GroupSchema& group_info_) override; virtual Status add_group(TableSchema& group_info_) override;
virtual Status get_group(GroupSchema& group_info_) override; virtual Status get_group(TableSchema& group_info_) override;
virtual Status has_group(const std::string& group_id_, bool& has_or_not_) override; virtual Status has_group(const std::string& group_id_, bool& has_or_not_) override;
virtual Status add_group_file(GroupFileSchema& group_file_info) override; virtual Status add_group_file(TableFileSchema& group_file_info) override;
/* virtual Status delete_group_partitions(const std::string& group_id, */ /* virtual Status delete_group_partitions(const std::string& group_id, */
/* const meta::DatesT& dates) override; */ /* const meta::DatesT& dates) override; */
@ -31,8 +31,8 @@ public:
bool& has_or_not_) override; bool& has_or_not_) override;
virtual Status get_group_file(const std::string& group_id_, virtual Status get_group_file(const std::string& group_id_,
const std::string& file_id_, const std::string& file_id_,
GroupFileSchema& group_file_info_) override; TableFileSchema& group_file_info_) override;
virtual Status update_group_file(GroupFileSchema& group_file_) override; virtual Status update_group_file(TableFileSchema& group_file_) override;
virtual Status get_group_files(const std::string& group_id_, virtual Status get_group_files(const std::string& group_id_,
const int date_delta_, const int date_delta_,
@ -59,16 +59,16 @@ public:
private: private:
Status GetGroupMetaInfoByPath(const std::string& path, GroupSchema& group_info); Status GetGroupMetaInfoByPath(const std::string& path, TableSchema& group_info);
std::string GetGroupMetaPathByGroupPath(const std::string& group_path); std::string GetGroupMetaPathByGroupPath(const std::string& group_path);
Status GetGroupMetaInfo(const std::string& group_id, GroupSchema& group_info); Status GetGroupMetaInfo(const std::string& group_id, TableSchema& group_info);
std::string GetNextGroupFileLocationByPartition(const std::string& group_id, DateT& date, std::string GetNextGroupFileLocationByPartition(const std::string& group_id, DateT& date,
GroupFileSchema::FILE_TYPE file_type); TableFileSchema::FILE_TYPE file_type);
std::string GetGroupDatePartitionPath(const std::string& group_id, DateT& date); std::string GetGroupDatePartitionPath(const std::string& group_id, DateT& date);
std::string GetGroupPath(const std::string& group_id); std::string GetGroupPath(const std::string& group_id);
std::string GetGroupMetaPath(const std::string& group_id); std::string GetGroupMetaPath(const std::string& group_id);
Status CreateGroupMeta(const GroupSchema& group_schema); Status CreateGroupMeta(const TableSchema& group_schema);
long GetFileSize(const std::string& filename); long GetFileSize(const std::string& filename);
Status initialize(); Status initialize();

8
cpp/src/db/MemManager.cpp
View File

@ -22,7 +22,7 @@ namespace engine {
template<typename EngineT> template<typename EngineT>
MemVectors<EngineT>::MemVectors(const std::shared_ptr<meta::Meta>& meta_ptr, MemVectors<EngineT>::MemVectors(const std::shared_ptr<meta::Meta>& meta_ptr,
const meta::GroupFileSchema& schema, const Options& options) const meta::TableFileSchema& schema, const Options& options)
: pMeta_(meta_ptr), : pMeta_(meta_ptr),
options_(options), options_(options),
schema_(schema), schema_(schema),
@ -53,11 +53,11 @@ Status MemVectors<EngineT>::serialize(std::string& group_id) {
pEE_->Serialize(); pEE_->Serialize();
schema_.size = size; schema_.size = size;
schema_.file_type = (size >= options_.index_trigger_size) ? schema_.file_type = (size >= options_.index_trigger_size) ?
meta::GroupFileSchema::TO_INDEX : meta::GroupFileSchema::RAW; meta::TableFileSchema::TO_INDEX : meta::TableFileSchema::RAW;
auto status = pMeta_->update_group_file(schema_); auto status = pMeta_->update_group_file(schema_);
LOG(DEBUG) << "New " << ((schema_.file_type == meta::GroupFileSchema::RAW) ? "raw" : "to_index") LOG(DEBUG) << "New " << ((schema_.file_type == meta::TableFileSchema::RAW) ? "raw" : "to_index")
<< " file " << schema_.file_id << " of size " << pEE_->Size() / meta::M << " M"; << " file " << schema_.file_id << " of size " << pEE_->Size() / meta::M << " M";
pEE_->Cache(); pEE_->Cache();
@ -85,7 +85,7 @@ typename MemManager<EngineT>::MemVectorsPtr MemManager<EngineT>::get_mem_by_grou
return memIt->second; return memIt->second;
} }
meta::GroupFileSchema group_file; meta::TableFileSchema group_file;
group_file.group_id = group_id; group_file.group_id = group_id;
auto status = _pMeta->add_group_file(group_file); auto status = _pMeta->add_group_file(group_file);
if (!status.ok()) { if (!status.ok()) {

4
cpp/src/db/MemManager.h
View File

@ -31,7 +31,7 @@ public:
typedef std::shared_ptr<MemVectors<EngineT>> Ptr; typedef std::shared_ptr<MemVectors<EngineT>> Ptr;
explicit MemVectors(const std::shared_ptr<meta::Meta>&, explicit MemVectors(const std::shared_ptr<meta::Meta>&,
const meta::GroupFileSchema&, const Options&); const meta::TableFileSchema&, const Options&);
void add(size_t n_, const float* vectors_, IDNumbers& vector_ids_); void add(size_t n_, const float* vectors_, IDNumbers& vector_ids_);
@ -52,7 +52,7 @@ private:
MetaPtr pMeta_; MetaPtr pMeta_;
Options options_; Options options_;
meta::GroupFileSchema schema_; meta::TableFileSchema schema_;
IDGenerator* _pIdGenerator; IDGenerator* _pIdGenerator;
EnginePtr pEE_; EnginePtr pEE_;

12
cpp/src/db/Meta.h
View File

@ -20,13 +20,13 @@ namespace meta {
class Meta { class Meta {
public: public:
typedef std::shared_ptr<Meta> Ptr; using Ptr = std::shared_ptr<Meta>;
virtual Status add_group(GroupSchema& group_info) = 0; virtual Status add_group(TableSchema& group_info) = 0;
virtual Status get_group(GroupSchema& group_info) = 0; virtual Status get_group(TableSchema& group_info) = 0;
virtual Status has_group(const std::string& group_id_, bool& has_or_not_) = 0; virtual Status has_group(const std::string& group_id_, bool& has_or_not_) = 0;
virtual Status add_group_file(GroupFileSchema& group_file_info) = 0; virtual Status add_group_file(TableFileSchema& group_file_info) = 0;
virtual Status delete_group_partitions(const std::string& group_id, virtual Status delete_group_partitions(const std::string& group_id,
const meta::DatesT& dates) = 0; const meta::DatesT& dates) = 0;
@ -35,8 +35,8 @@ public:
bool& has_or_not_) = 0; bool& has_or_not_) = 0;
virtual Status get_group_file(const std::string& group_id_, virtual Status get_group_file(const std::string& group_id_,
const std::string& file_id_, const std::string& file_id_,
GroupFileSchema& group_file_info_) = 0; TableFileSchema& group_file_info_) = 0;
virtual Status update_group_file(GroupFileSchema& group_file_) = 0; virtual Status update_group_file(TableFileSchema& group_file_) = 0;
virtual Status get_group_files(const std::string& group_id_, virtual Status get_group_files(const std::string& group_id_,
const int date_delta_, const int date_delta_,

14
cpp/src/db/MetaTypes.h
View File

@ -18,16 +18,16 @@ typedef int DateT;
const DateT EmptyDate = -1; const DateT EmptyDate = -1;
typedef std::vector<DateT> DatesT; typedef std::vector<DateT> DatesT;
struct GroupSchema { struct TableSchema {
size_t id; size_t id;
std::string group_id; std::string group_id;
size_t files_cnt = 0; size_t files_cnt = 0;
uint16_t dimension; uint16_t dimension;
std::string location = ""; std::string location;
long created_on; long created_on;
}; // GroupSchema }; // TableSchema
struct GroupFileSchema { struct TableFileSchema {
typedef enum { typedef enum {
NEW, NEW,
RAW, RAW,
@ -43,12 +43,12 @@ struct GroupFileSchema {
size_t size; size_t size;
DateT date = EmptyDate; DateT date = EmptyDate;
uint16_t dimension; uint16_t dimension;
std::string location = ""; std::string location;
long updated_time; long updated_time;
long created_on; long created_on;
}; // GroupFileSchema }; // TableFileSchema
typedef std::vector<GroupFileSchema> GroupFilesSchema; typedef std::vector<TableFileSchema> GroupFilesSchema;
typedef std::map<DateT, GroupFilesSchema> DatePartionedGroupFilesSchema; typedef std::map<DateT, GroupFilesSchema> DatePartionedGroupFilesSchema;
} // namespace meta } // namespace meta

12
cpp/unittest/db/db_tests.cpp
View File

@ -66,12 +66,12 @@ TEST_F(DBTest2, ARHIVE_DISK_CHECK) {
static const int group_dim = 256; static const int group_dim = 256;
long size; long size;
engine::meta::GroupSchema group_info; engine::meta::TableSchema group_info;
group_info.dimension = group_dim; group_info.dimension = group_dim;
group_info.group_id = group_name; group_info.group_id = group_name;
engine::Status stat = db_->add_group(group_info); engine::Status stat = db_->add_group(group_info);
engine::meta::GroupSchema group_info_get; engine::meta::TableSchema group_info_get;
group_info_get.group_id = group_name; group_info_get.group_id = group_name;
stat = db_->get_group(group_info_get); stat = db_->get_group(group_info_get);
ASSERT_STATS(stat); ASSERT_STATS(stat);
@ -111,12 +111,12 @@ TEST_F(DBTest, DB_TEST) {
static const std::string group_name = "test_group"; static const std::string group_name = "test_group";
static const int group_dim = 256; static const int group_dim = 256;
engine::meta::GroupSchema group_info; engine::meta::TableSchema group_info;
group_info.dimension = group_dim; group_info.dimension = group_dim;
group_info.group_id = group_name; group_info.group_id = group_name;
engine::Status stat = db_->add_group(group_info); engine::Status stat = db_->add_group(group_info);
engine::meta::GroupSchema group_info_get; engine::meta::TableSchema group_info_get;
group_info_get.group_id = group_name; group_info_get.group_id = group_name;
stat = db_->get_group(group_info_get); stat = db_->get_group(group_info_get);
ASSERT_STATS(stat); ASSERT_STATS(stat);
@ -197,12 +197,12 @@ TEST_F(DBTest, SEARCH_TEST) {
static const std::string group_name = "test_group"; static const std::string group_name = "test_group";
static const int group_dim = 256; static const int group_dim = 256;
engine::meta::GroupSchema group_info; engine::meta::TableSchema group_info;
group_info.dimension = group_dim; group_info.dimension = group_dim;
group_info.group_id = group_name; group_info.group_id = group_name;
engine::Status stat = db_->add_group(group_info); engine::Status stat = db_->add_group(group_info);
engine::meta::GroupSchema group_info_get; engine::meta::TableSchema group_info_get;
group_info_get.group_id = group_name; group_info_get.group_id = group_name;
stat = db_->get_group(group_info_get); stat = db_->get_group(group_info_get);
ASSERT_STATS(stat); ASSERT_STATS(stat);

46
cpp/unittest/db/meta_tests.cpp
View File

@ -20,7 +20,7 @@ using namespace zilliz::vecwise::engine;
TEST_F(MetaTest, GROUP_TEST) { TEST_F(MetaTest, GROUP_TEST) {
auto group_id = "meta_test_group"; auto group_id = "meta_test_group";
meta::GroupSchema group; meta::TableSchema group;
group.group_id = group_id; group.group_id = group_id;
auto status = impl_->add_group(group); auto status = impl_->add_group(group);
ASSERT_TRUE(status.ok()); ASSERT_TRUE(status.ok());
@ -44,19 +44,19 @@ TEST_F(MetaTest, GROUP_TEST) {
TEST_F(MetaTest, GROUP_FILE_TEST) { TEST_F(MetaTest, GROUP_FILE_TEST) {
auto group_id = "meta_test_group"; auto group_id = "meta_test_group";
meta::GroupSchema group; meta::TableSchema group;
group.group_id = group_id; group.group_id = group_id;
auto status = impl_->add_group(group); auto status = impl_->add_group(group);
meta::GroupFileSchema group_file; meta::TableFileSchema group_file;
group_file.group_id = group.group_id; group_file.group_id = group.group_id;
status = impl_->add_group_file(group_file); status = impl_->add_group_file(group_file);
ASSERT_TRUE(status.ok()); ASSERT_TRUE(status.ok());
ASSERT_EQ(group_file.file_type, meta::GroupFileSchema::NEW); ASSERT_EQ(group_file.file_type, meta::TableFileSchema::NEW);
auto file_id = group_file.file_id; auto file_id = group_file.file_id;
auto new_file_type = meta::GroupFileSchema::INDEX; auto new_file_type = meta::TableFileSchema::INDEX;
group_file.file_type = new_file_type; group_file.file_type = new_file_type;
status = impl_->update_group_file(group_file); status = impl_->update_group_file(group_file);
@ -79,7 +79,7 @@ TEST_F(MetaTest, GROUP_FILE_TEST) {
status = impl_->update_group_file(group_file); status = impl_->update_group_file(group_file);
ASSERT_TRUE(status.ok()); ASSERT_TRUE(status.ok());
ASSERT_EQ(group_file.date, meta::Meta::GetDateWithDelta(-2)); ASSERT_EQ(group_file.date, meta::Meta::GetDateWithDelta(-2));
ASSERT_FALSE(group_file.file_type == meta::GroupFileSchema::TO_DELETE); ASSERT_FALSE(group_file.file_type == meta::TableFileSchema::TO_DELETE);
dates.clear(); dates.clear();
dates.push_back(group_file.date); dates.push_back(group_file.date);
@ -87,7 +87,7 @@ TEST_F(MetaTest, GROUP_FILE_TEST) {
ASSERT_TRUE(status.ok()); ASSERT_TRUE(status.ok());
status = impl_->get_group_file(group_file.group_id, group_file.file_id, group_file); status = impl_->get_group_file(group_file.group_id, group_file.file_id, group_file);
ASSERT_TRUE(status.ok()); ASSERT_TRUE(status.ok());
ASSERT_TRUE(group_file.file_type == meta::GroupFileSchema::TO_DELETE); ASSERT_TRUE(group_file.file_type == meta::TableFileSchema::TO_DELETE);
} }
TEST_F(MetaTest, ARCHIVE_TEST_DAYS) { TEST_F(MetaTest, ARCHIVE_TEST_DAYS) {
@ -102,12 +102,12 @@ TEST_F(MetaTest, ARCHIVE_TEST_DAYS) {
auto impl = meta::DBMetaImpl(options); auto impl = meta::DBMetaImpl(options);
auto group_id = "meta_test_group"; auto group_id = "meta_test_group";
meta::GroupSchema group; meta::TableSchema group;
group.group_id = group_id; group.group_id = group_id;
auto status = impl.add_group(group); auto status = impl.add_group(group);
meta::GroupFilesSchema files; meta::GroupFilesSchema files;
meta::GroupFileSchema group_file; meta::TableFileSchema group_file;
group_file.group_id = group.group_id; group_file.group_id = group.group_id;
auto cnt = 100; auto cnt = 100;
@ -115,7 +115,7 @@ TEST_F(MetaTest, ARCHIVE_TEST_DAYS) {
std::vector<int> days; std::vector<int> days;
for (auto i=0; i<cnt; ++i) { for (auto i=0; i<cnt; ++i) {
status = impl.add_group_file(group_file); status = impl.add_group_file(group_file);
group_file.file_type = meta::GroupFileSchema::NEW; group_file.file_type = meta::TableFileSchema::NEW;
int day = rand() % (days_num*2); int day = rand() % (days_num*2);
group_file.created_on = ts - day*meta::D_SEC*meta::US_PS - 10000; group_file.created_on = ts - day*meta::D_SEC*meta::US_PS - 10000;
status = impl.update_group_file(group_file); status = impl.update_group_file(group_file);
@ -130,9 +130,9 @@ TEST_F(MetaTest, ARCHIVE_TEST_DAYS) {
status = impl.get_group_file(file.group_id, file.file_id, file); status = impl.get_group_file(file.group_id, file.file_id, file);
ASSERT_TRUE(status.ok()); ASSERT_TRUE(status.ok());
if (days[i] < days_num) { if (days[i] < days_num) {
ASSERT_EQ(file.file_type, meta::GroupFileSchema::NEW); ASSERT_EQ(file.file_type, meta::TableFileSchema::NEW);
} else { } else {
ASSERT_EQ(file.file_type, meta::GroupFileSchema::TO_DELETE); ASSERT_EQ(file.file_type, meta::TableFileSchema::TO_DELETE);
} }
i++; i++;
} }
@ -148,19 +148,19 @@ TEST_F(MetaTest, ARCHIVE_TEST_DISK) {
auto impl = meta::DBMetaImpl(options); auto impl = meta::DBMetaImpl(options);
auto group_id = "meta_test_group"; auto group_id = "meta_test_group";
meta::GroupSchema group; meta::TableSchema group;
group.group_id = group_id; group.group_id = group_id;
auto status = impl.add_group(group); auto status = impl.add_group(group);
meta::GroupFilesSchema files; meta::GroupFilesSchema files;
meta::GroupFileSchema group_file; meta::TableFileSchema group_file;
group_file.group_id = group.group_id; group_file.group_id = group.group_id;
auto cnt = 10; auto cnt = 10;
auto each_size = 2UL; auto each_size = 2UL;
for (auto i=0; i<cnt; ++i) { for (auto i=0; i<cnt; ++i) {
status = impl.add_group_file(group_file); status = impl.add_group_file(group_file);
group_file.file_type = meta::GroupFileSchema::NEW; group_file.file_type = meta::TableFileSchema::NEW;
group_file.size = each_size * meta::G; group_file.size = each_size * meta::G;
status = impl.update_group_file(group_file); status = impl.update_group_file(group_file);
files.push_back(group_file); files.push_back(group_file);
@ -173,9 +173,9 @@ TEST_F(MetaTest, ARCHIVE_TEST_DISK) {
status = impl.get_group_file(file.group_id, file.file_id, file); status = impl.get_group_file(file.group_id, file.file_id, file);
ASSERT_TRUE(status.ok()); ASSERT_TRUE(status.ok());
if (i < 5) { if (i < 5) {
ASSERT_TRUE(file.file_type == meta::GroupFileSchema::TO_DELETE); ASSERT_TRUE(file.file_type == meta::TableFileSchema::TO_DELETE);
} else { } else {
ASSERT_EQ(file.file_type, meta::GroupFileSchema::NEW); ASSERT_EQ(file.file_type, meta::TableFileSchema::NEW);
} }
++i; ++i;
} }
@ -186,7 +186,7 @@ TEST_F(MetaTest, ARCHIVE_TEST_DISK) {
TEST_F(MetaTest, GROUP_FILES_TEST) { TEST_F(MetaTest, GROUP_FILES_TEST) {
auto group_id = "meta_test_group"; auto group_id = "meta_test_group";
meta::GroupSchema group; meta::TableSchema group;
group.group_id = group_id; group.group_id = group_id;
auto status = impl_->add_group(group); auto status = impl_->add_group(group);
@ -195,30 +195,30 @@ TEST_F(MetaTest, GROUP_FILES_TEST) {
int to_index_files_cnt = 6; int to_index_files_cnt = 6;
int index_files_cnt = 7; int index_files_cnt = 7;
meta::GroupFileSchema group_file; meta::TableFileSchema group_file;
group_file.group_id = group.group_id; group_file.group_id = group.group_id;
for (auto i=0; i<new_files_cnt; ++i) { for (auto i=0; i<new_files_cnt; ++i) {
status = impl_->add_group_file(group_file); status = impl_->add_group_file(group_file);
group_file.file_type = meta::GroupFileSchema::NEW; group_file.file_type = meta::TableFileSchema::NEW;
status = impl_->update_group_file(group_file); status = impl_->update_group_file(group_file);
} }
for (auto i=0; i<raw_files_cnt; ++i) { for (auto i=0; i<raw_files_cnt; ++i) {
status = impl_->add_group_file(group_file); status = impl_->add_group_file(group_file);
group_file.file_type = meta::GroupFileSchema::RAW; group_file.file_type = meta::TableFileSchema::RAW;
status = impl_->update_group_file(group_file); status = impl_->update_group_file(group_file);
} }
for (auto i=0; i<to_index_files_cnt; ++i) { for (auto i=0; i<to_index_files_cnt; ++i) {
status = impl_->add_group_file(group_file); status = impl_->add_group_file(group_file);
group_file.file_type = meta::GroupFileSchema::TO_INDEX; group_file.file_type = meta::TableFileSchema::TO_INDEX;
status = impl_->update_group_file(group_file); status = impl_->update_group_file(group_file);
} }
for (auto i=0; i<index_files_cnt; ++i) { for (auto i=0; i<index_files_cnt; ++i) {
status = impl_->add_group_file(group_file); status = impl_->add_group_file(group_file);
group_file.file_type = meta::GroupFileSchema::INDEX; group_file.file_type = meta::TableFileSchema::INDEX;
status = impl_->update_group_file(group_file); status = impl_->update_group_file(group_file);
} }