milvus-io
/
milvus
mirror of https://github.com/milvus-io/milvus.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

refactor(db): change groupschema and groupfileschema to table* 

Former-commit-id: fbc69151149faa8ed32547bcdd1a9935dac34070
pull/191/head
Xu Peng 2019-05-27 17:16:16 +08:00
parent aa60c62870
commit e94499cdff
13 changed files with 209 additions and 209 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
cpp/src/db/DB.h
View File

@ -21,8 +21,8 @@ class DB {
public:
static void Open(const Options& options, DB** dbptr);
virtual Status add_group(meta::GroupSchema& group_info_) = 0;
virtual Status get_group(meta::GroupSchema& group_info_) = 0;
virtual Status add_group(meta::TableSchema& group_info_) = 0;
virtual Status get_group(meta::TableSchema& group_info_) = 0;
virtual Status delete_vectors(const std::string& group_id,
const meta::DatesT& dates) = 0;
virtual Status has_group(const std::string& group_id_, bool& has_or_not_) = 0;

22
cpp/src/db/DBImpl.cpp
View File

@ -35,12 +35,12 @@ DBImpl<EngineT>::DBImpl(const Options& options)
}
template<typename EngineT>
Status DBImpl<EngineT>::add_group(meta::GroupSchema& group_info) {
Status DBImpl<EngineT>::add_group(meta::TableSchema& group_info) {
return _pMeta->add_group(group_info);
}
template<typename EngineT>
Status DBImpl<EngineT>::get_group(meta::GroupSchema& group_info) {
Status DBImpl<EngineT>::get_group(meta::TableSchema& group_info) {
return _pMeta->get_group(group_info);
}
@ -93,7 +93,7 @@ Status DBImpl<EngineT>::search(const std::string& group_id, size_t k, size_t nq,
meta::GroupFilesSchema raw_files;
for (auto &day_files : files) {
for (auto &file : day_files.second) {
file.file_type == meta::GroupFileSchema::INDEX ?
file.file_type == meta::TableFileSchema::INDEX ?
index_files.push_back(file) : raw_files.push_back(file);
}
}
@ -259,7 +259,7 @@ void DBImpl<EngineT>::background_call() {
template<typename EngineT>
Status DBImpl<EngineT>::merge_files(const std::string& group_id, const meta::DateT& date,
const meta::GroupFilesSchema& files) {
meta::GroupFileSchema group_file;
meta::TableFileSchema group_file;
group_file.group_id = group_id;
group_file.date = date;
Status status = _pMeta->add_group_file(group_file);
@ -277,7 +277,7 @@ Status DBImpl<EngineT>::merge_files(const std::string& group_id, const meta::Dat
for (auto& file : files) {
index.Merge(file.location);
auto file_schema = file;
file_schema.file_type = meta::GroupFileSchema::TO_DELETE;
file_schema.file_type = meta::TableFileSchema::TO_DELETE;
updated.push_back(file_schema);
LOG(DEBUG) << "Merging file " << file_schema.file_id;
index_size = index.Size();
@ -288,9 +288,9 @@ Status DBImpl<EngineT>::merge_files(const std::string& group_id, const meta::Dat
index.Serialize();
if (index_size >= _options.index_trigger_size) {
group_file.file_type = meta::GroupFileSchema::TO_INDEX;
group_file.file_type = meta::TableFileSchema::TO_INDEX;
} else {
group_file.file_type = meta::GroupFileSchema::RAW;
group_file.file_type = meta::TableFileSchema::RAW;
}
group_file.size = index_size;
updated.push_back(group_file);
@ -336,8 +336,8 @@ Status DBImpl<EngineT>::background_merge_files(const std::string& group_id) {
}
template<typename EngineT>
Status DBImpl<EngineT>::build_index(const meta::GroupFileSchema& file) {
meta::GroupFileSchema group_file;
Status DBImpl<EngineT>::build_index(const meta::TableFileSchema& file) {
meta::TableFileSchema group_file;
group_file.group_id = file.group_id;
group_file.date = file.date;
Status status = _pMeta->add_group_file(group_file);
@ -350,11 +350,11 @@ Status DBImpl<EngineT>::build_index(const meta::GroupFileSchema& file) {
to_index.Load();
auto index = to_index.BuildIndex(group_file.location);
group_file.file_type = meta::GroupFileSchema::INDEX;
group_file.file_type = meta::TableFileSchema::INDEX;
group_file.size = index->Size();
auto to_remove = file;
to_remove.file_type = meta::GroupFileSchema::TO_DELETE;
to_remove.file_type = meta::TableFileSchema::TO_DELETE;
meta::GroupFilesSchema update_files = {to_remove, group_file};
_pMeta->update_files(update_files);

6
cpp/src/db/DBImpl.h
View File

@ -33,8 +33,8 @@ public:
DBImpl(const Options& options);
virtual Status add_group(meta::GroupSchema& group_info) override;
virtual Status get_group(meta::GroupSchema& group_info) override;
virtual Status add_group(meta::TableSchema& group_info) override;
virtual Status get_group(meta::TableSchema& group_info) override;
virtual Status delete_vectors(const std::string& group_id, const meta::DatesT& dates) override;
virtual Status has_group(const std::string& group_id_, bool& has_or_not_) override;
@ -62,7 +62,7 @@ public:
private:
void background_build_index();
Status build_index(const meta::GroupFileSchema&);
Status build_index(const meta::TableFileSchema&);
Status try_build_index();
Status merge_files(const std::string& group_id,
const meta::DateT& date,

230
cpp/src/db/DBMetaImpl.cpp
View File

@ -28,20 +28,20 @@ using namespace sqlite_orm;
inline auto StoragePrototype(const std::string& path) {
return make_storage(path,
make_table("Group",
make_column("id", &GroupSchema::id, primary_key()),
make_column("group_id", &GroupSchema::group_id, unique()),
make_column("dimension", &GroupSchema::dimension),
make_column("created_on", &GroupSchema::created_on),
make_column("files_cnt", &GroupSchema::files_cnt, default_value(0))),
make_column("id", &TableSchema::id, primary_key()),
make_column("group_id", &TableSchema::group_id, unique()),
make_column("dimension", &TableSchema::dimension),
make_column("created_on", &TableSchema::created_on),
make_column("files_cnt", &TableSchema::files_cnt, default_value(0))),
make_table("GroupFile",
make_column("id", &GroupFileSchema::id, primary_key()),
make_column("group_id", &GroupFileSchema::group_id),
make_column("file_id", &GroupFileSchema::file_id),
make_column("file_type", &GroupFileSchema::file_type),
make_column("size", &GroupFileSchema::size, default_value(0)),
make_column("updated_time", &GroupFileSchema::updated_time),
make_column("created_on", &GroupFileSchema::created_on),
make_column("date", &GroupFileSchema::date))
make_column("id", &TableFileSchema::id, primary_key()),
make_column("group_id", &TableFileSchema::group_id),
make_column("file_id", &TableFileSchema::file_id),
make_column("file_type", &TableFileSchema::file_type),
make_column("size", &TableFileSchema::size, default_value(0)),
make_column("updated_time", &TableFileSchema::updated_time),
make_column("created_on", &TableFileSchema::created_on),
make_column("date", &TableFileSchema::date))
);
}
@ -59,7 +59,7 @@ std::string DBMetaImpl::GetGroupDatePartitionPath(const std::string& group_id, D
return ss.str();
}
void DBMetaImpl::GetGroupFilePath(GroupFileSchema& group_file) {
void DBMetaImpl::GetGroupFilePath(TableFileSchema& group_file) {
if (group_file.date == EmptyDate) {
group_file.date = Meta::GetDate();
}
@ -117,7 +117,7 @@ Status DBMetaImpl::delete_group_partitions(const std::string& group_id,
return Status::OK();
}
GroupSchema group_info;
TableSchema group_info;
group_info.group_id = group_id;
auto status = get_group(group_info);
if (!status.ok()) {
@ -135,11 +135,11 @@ Status DBMetaImpl::delete_group_partitions(const std::string& group_id,
try {
ConnectorPtr->update_all(
set(
c(&GroupFileSchema::file_type) = (int)GroupFileSchema::TO_DELETE
c(&TableFileSchema::file_type) = (int)TableFileSchema::TO_DELETE
),
where(
c(&GroupFileSchema::group_id) == group_id and
in(&GroupFileSchema::date, dates)
c(&TableFileSchema::group_id) == group_id and
in(&TableFileSchema::date, dates)
));
} catch (std::exception & e) {
LOG(DEBUG) << e.what();
@ -148,7 +148,7 @@ Status DBMetaImpl::delete_group_partitions(const std::string& group_id,
return Status::OK();
}
Status DBMetaImpl::add_group(GroupSchema& group_info) {
Status DBMetaImpl::add_group(TableSchema& group_info) {
if (group_info.group_id == "") {
NextGroupId(group_info.group_id);
}
@ -178,17 +178,17 @@ Status DBMetaImpl::add_group(GroupSchema& group_info) {
return Status::OK();
}
Status DBMetaImpl::get_group(GroupSchema& group_info) {
Status DBMetaImpl::get_group(TableSchema& group_info) {
return get_group_no_lock(group_info);
}
Status DBMetaImpl::get_group_no_lock(GroupSchema& group_info) {
Status DBMetaImpl::get_group_no_lock(TableSchema& group_info) {
try {
auto groups = ConnectorPtr->select(columns(&GroupSchema::id,
&GroupSchema::group_id,
&GroupSchema::files_cnt,
&GroupSchema::dimension),
where(c(&GroupSchema::group_id) == group_info.group_id));
auto groups = ConnectorPtr->select(columns(&TableSchema::id,
&TableSchema::group_id,
&TableSchema::files_cnt,
&TableSchema::dimension),
where(c(&TableSchema::group_id) == group_info.group_id));
assert(groups.size() <= 1);
if (groups.size() == 1) {
group_info.id = std::get<0>(groups[0]);
@ -207,8 +207,8 @@ Status DBMetaImpl::get_group_no_lock(GroupSchema& group_info) {
Status DBMetaImpl::has_group(const std::string& group_id, bool& has_or_not) {
try {
auto groups = ConnectorPtr->select(columns(&GroupSchema::id),
where(c(&GroupSchema::group_id) == group_id));
auto groups = ConnectorPtr->select(columns(&TableSchema::id),
where(c(&TableSchema::group_id) == group_id));
assert(groups.size() <= 1);
if (groups.size() == 1) {
has_or_not = true;
@ -222,11 +222,11 @@ Status DBMetaImpl::has_group(const std::string& group_id, bool& has_or_not) {
return Status::OK();
}
Status DBMetaImpl::add_group_file(GroupFileSchema& group_file) {
Status DBMetaImpl::add_group_file(TableFileSchema& group_file) {
if (group_file.date == EmptyDate) {
group_file.date = Meta::GetDate();
}
GroupSchema group_info;
TableSchema group_info;
group_info.group_id = group_file.group_id;
auto status = get_group(group_info);
if (!status.ok()) {
@ -234,7 +234,7 @@ Status DBMetaImpl::add_group_file(GroupFileSchema& group_file) {
}
NextFileId(group_file.file_id);
group_file.file_type = GroupFileSchema::NEW;
group_file.file_type = TableFileSchema::NEW;
group_file.dimension = group_info.dimension;
group_file.size = 0;
group_file.created_on = utils::GetMicroSecTimeStamp();
@ -267,18 +267,18 @@ Status DBMetaImpl::files_to_index(GroupFilesSchema& files) {
files.clear();
try {
auto selected = ConnectorPtr->select(columns(&GroupFileSchema::id,
&GroupFileSchema::group_id,
&GroupFileSchema::file_id,
&GroupFileSchema::file_type,
&GroupFileSchema::size,
&GroupFileSchema::date),
where(c(&GroupFileSchema::file_type) == (int)GroupFileSchema::TO_INDEX));
auto selected = ConnectorPtr->select(columns(&TableFileSchema::id,
&TableFileSchema::group_id,
&TableFileSchema::file_id,
&TableFileSchema::file_type,
&TableFileSchema::size,
&TableFileSchema::date),
where(c(&TableFileSchema::file_type) == (int)TableFileSchema::TO_INDEX));
std::map<std::string, GroupSchema> groups;
std::map<std::string, TableSchema> groups;
for (auto& file : selected) {
GroupFileSchema group_file;
TableFileSchema group_file;
group_file.id = std::get<0>(file);
group_file.group_id = std::get<1>(file);
group_file.file_id = std::get<2>(file);
@ -288,7 +288,7 @@ Status DBMetaImpl::files_to_index(GroupFilesSchema& files) {
GetGroupFilePath(group_file);
auto groupItr = groups.find(group_file.group_id);
if (groupItr == groups.end()) {
GroupSchema group_info;
TableSchema group_info;
group_info.group_id = group_file.group_id;
auto status = get_group_no_lock(group_info);
if (!status.ok()) {
@ -315,19 +315,19 @@ Status DBMetaImpl::files_to_search(const std::string &group_id,
const DatesT& dates = (partition.empty() == true) ? today : partition;
try {
auto selected = ConnectorPtr->select(columns(&GroupFileSchema::id,
&GroupFileSchema::group_id,
&GroupFileSchema::file_id,
&GroupFileSchema::file_type,
&GroupFileSchema::size,
&GroupFileSchema::date),
where(c(&GroupFileSchema::group_id) == group_id and
in(&GroupFileSchema::date, dates) and
(c(&GroupFileSchema::file_type) == (int) GroupFileSchema::RAW or
c(&GroupFileSchema::file_type) == (int) GroupFileSchema::TO_INDEX or
c(&GroupFileSchema::file_type) == (int) GroupFileSchema::INDEX)));
auto selected = ConnectorPtr->select(columns(&TableFileSchema::id,
&TableFileSchema::group_id,
&TableFileSchema::file_id,
&TableFileSchema::file_type,
&TableFileSchema::size,
&TableFileSchema::date),
where(c(&TableFileSchema::group_id) == group_id and
in(&TableFileSchema::date, dates) and
(c(&TableFileSchema::file_type) == (int) TableFileSchema::RAW or
c(&TableFileSchema::file_type) == (int) TableFileSchema::TO_INDEX or
c(&TableFileSchema::file_type) == (int) TableFileSchema::INDEX)));
GroupSchema group_info;
TableSchema group_info;
group_info.group_id = group_id;
auto status = get_group_no_lock(group_info);
if (!status.ok()) {
@ -335,7 +335,7 @@ Status DBMetaImpl::files_to_search(const std::string &group_id,
}
for (auto& file : selected) {
GroupFileSchema group_file;
TableFileSchema group_file;
group_file.id = std::get<0>(file);
group_file.group_id = std::get<1>(file);
group_file.file_id = std::get<2>(file);
@ -363,16 +363,16 @@ Status DBMetaImpl::files_to_merge(const std::string& group_id,
files.clear();
try {
auto selected = ConnectorPtr->select(columns(&GroupFileSchema::id,
&GroupFileSchema::group_id,
&GroupFileSchema::file_id,
&GroupFileSchema::file_type,
&GroupFileSchema::size,
&GroupFileSchema::date),
where(c(&GroupFileSchema::file_type) == (int)GroupFileSchema::RAW and
c(&GroupFileSchema::group_id) == group_id));
auto selected = ConnectorPtr->select(columns(&TableFileSchema::id,
&TableFileSchema::group_id,
&TableFileSchema::file_id,
&TableFileSchema::file_type,
&TableFileSchema::size,
&TableFileSchema::date),
where(c(&TableFileSchema::file_type) == (int)TableFileSchema::RAW and
c(&TableFileSchema::group_id) == group_id));
GroupSchema group_info;
TableSchema group_info;
group_info.group_id = group_id;
auto status = get_group_no_lock(group_info);
if (!status.ok()) {
@ -380,7 +380,7 @@ Status DBMetaImpl::files_to_merge(const std::string& group_id,
}
for (auto& file : selected) {
GroupFileSchema group_file;
TableFileSchema group_file;
group_file.id = std::get<0>(file);
group_file.group_id = std::get<1>(file);
group_file.file_id = std::get<2>(file);
@ -412,16 +412,16 @@ Status DBMetaImpl::has_group_file(const std::string& group_id_,
Status DBMetaImpl::get_group_file(const std::string& group_id_,
const std::string& file_id_,
GroupFileSchema& group_file_info_) {
TableFileSchema& group_file_info_) {
try {
auto files = ConnectorPtr->select(columns(&GroupFileSchema::id,
&GroupFileSchema::group_id,
&GroupFileSchema::file_id,
&GroupFileSchema::file_type,
&GroupFileSchema::size,
&GroupFileSchema::date),
where(c(&GroupFileSchema::file_id) == file_id_ and
c(&GroupFileSchema::group_id) == group_id_
auto files = ConnectorPtr->select(columns(&TableFileSchema::id,
&TableFileSchema::group_id,
&TableFileSchema::file_id,
&TableFileSchema::file_type,
&TableFileSchema::size,
&TableFileSchema::date),
where(c(&TableFileSchema::file_id) == file_id_ and
c(&TableFileSchema::group_id) == group_id_
));
assert(files.size() <= 1);
if (files.size() == 1) {
@ -466,11 +466,11 @@ Status DBMetaImpl::archive_files() {
{
ConnectorPtr->update_all(
set(
c(&GroupFileSchema::file_type) = (int)GroupFileSchema::TO_DELETE
c(&TableFileSchema::file_type) = (int)TableFileSchema::TO_DELETE
),
where(
c(&GroupFileSchema::created_on) < (long)(now - usecs) and
c(&GroupFileSchema::file_type) != (int)GroupFileSchema::TO_DELETE
c(&TableFileSchema::created_on) < (long)(now - usecs) and
c(&TableFileSchema::file_type) != (int)TableFileSchema::TO_DELETE
));
} catch (std::exception & e) {
LOG(DEBUG) << e.what();
@ -493,9 +493,9 @@ Status DBMetaImpl::archive_files() {
Status DBMetaImpl::size(long& result) {
result = 0;
try {
auto selected = ConnectorPtr->select(columns(sum(&GroupFileSchema::size)),
auto selected = ConnectorPtr->select(columns(sum(&TableFileSchema::size)),
where(
c(&GroupFileSchema::file_type) != (int)GroupFileSchema::TO_DELETE
c(&TableFileSchema::file_type) != (int)TableFileSchema::TO_DELETE
));
for (auto& sub_query : selected) {
@ -518,16 +518,16 @@ Status DBMetaImpl::discard_files_of_size(long to_discard_size) {
return Status::OK();
}
try {
auto selected = ConnectorPtr->select(columns(&GroupFileSchema::id,
&GroupFileSchema::size),
where(c(&GroupFileSchema::file_type) != (int)GroupFileSchema::TO_DELETE),
order_by(&GroupFileSchema::id),
auto selected = ConnectorPtr->select(columns(&TableFileSchema::id,
&TableFileSchema::size),
where(c(&TableFileSchema::file_type) != (int)TableFileSchema::TO_DELETE),
order_by(&TableFileSchema::id),
limit(10));
std::vector<int> ids;
for (auto& file : selected) {
if (to_discard_size <= 0) break;
GroupFileSchema group_file;
TableFileSchema group_file;
group_file.id = std::get<0>(file);
group_file.size = std::get<1>(file);
ids.push_back(group_file.id);
@ -541,10 +541,10 @@ Status DBMetaImpl::discard_files_of_size(long to_discard_size) {
ConnectorPtr->update_all(
set(
c(&GroupFileSchema::file_type) = (int)GroupFileSchema::TO_DELETE
c(&TableFileSchema::file_type) = (int)TableFileSchema::TO_DELETE
),
where(
in(&GroupFileSchema::id, ids)
in(&TableFileSchema::id, ids)
));
} catch (std::exception & e) {
@ -556,7 +556,7 @@ Status DBMetaImpl::discard_files_of_size(long to_discard_size) {
return discard_files_of_size(to_discard_size);
}
Status DBMetaImpl::update_group_file(GroupFileSchema& group_file) {
Status DBMetaImpl::update_group_file(TableFileSchema& group_file) {
group_file.updated_time = utils::GetMicroSecTimeStamp();
try {
ConnectorPtr->update(group_file);
@ -590,19 +590,19 @@ Status DBMetaImpl::update_files(GroupFilesSchema& files) {
Status DBMetaImpl::cleanup_ttl_files(uint16_t seconds) {
auto now = utils::GetMicroSecTimeStamp();
try {
auto selected = ConnectorPtr->select(columns(&GroupFileSchema::id,
&GroupFileSchema::group_id,
&GroupFileSchema::file_id,
&GroupFileSchema::file_type,
&GroupFileSchema::size,
&GroupFileSchema::date),
where(c(&GroupFileSchema::file_type) == (int)GroupFileSchema::TO_DELETE and
c(&GroupFileSchema::updated_time) > now - seconds*US_PS));
auto selected = ConnectorPtr->select(columns(&TableFileSchema::id,
&TableFileSchema::group_id,
&TableFileSchema::file_id,
&TableFileSchema::file_type,
&TableFileSchema::size,
&TableFileSchema::date),
where(c(&TableFileSchema::file_type) == (int)TableFileSchema::TO_DELETE and
c(&TableFileSchema::updated_time) > now - seconds*US_PS));
GroupFilesSchema updated;
for (auto& file : selected) {
GroupFileSchema group_file;
TableFileSchema group_file;
group_file.id = std::get<0>(file);
group_file.group_id = std::get<1>(file);
group_file.file_id = std::get<2>(file);
@ -610,10 +610,10 @@ Status DBMetaImpl::cleanup_ttl_files(uint16_t seconds) {
group_file.size = std::get<4>(file);
group_file.date = std::get<5>(file);
GetGroupFilePath(group_file);
if (group_file.file_type == GroupFileSchema::TO_DELETE) {
if (group_file.file_type == TableFileSchema::TO_DELETE) {
boost::filesystem::remove(group_file.location);
}
ConnectorPtr->remove<GroupFileSchema>(group_file.id);
ConnectorPtr->remove<TableFileSchema>(group_file.id);
/* LOG(DEBUG) << "Removing deleted id=" << group_file.id << " location=" << group_file.location << std::endl; */
}
} catch (std::exception & e) {
@ -626,19 +626,19 @@ Status DBMetaImpl::cleanup_ttl_files(uint16_t seconds) {
Status DBMetaImpl::cleanup() {
try {
auto selected = ConnectorPtr->select(columns(&GroupFileSchema::id,
&GroupFileSchema::group_id,
&GroupFileSchema::file_id,
&GroupFileSchema::file_type,
&GroupFileSchema::size,
&GroupFileSchema::date),
where(c(&GroupFileSchema::file_type) == (int)GroupFileSchema::TO_DELETE or
c(&GroupFileSchema::file_type) == (int)GroupFileSchema::NEW));
auto selected = ConnectorPtr->select(columns(&TableFileSchema::id,
&TableFileSchema::group_id,
&TableFileSchema::file_id,
&TableFileSchema::file_type,
&TableFileSchema::size,
&TableFileSchema::date),
where(c(&TableFileSchema::file_type) == (int)TableFileSchema::TO_DELETE or
c(&TableFileSchema::file_type) == (int)TableFileSchema::NEW));
GroupFilesSchema updated;
for (auto& file : selected) {
GroupFileSchema group_file;
TableFileSchema group_file;
group_file.id = std::get<0>(file);
group_file.group_id = std::get<1>(file);
group_file.file_id = std::get<2>(file);
@ -646,10 +646,10 @@ Status DBMetaImpl::cleanup() {
group_file.size = std::get<4>(file);
group_file.date = std::get<5>(file);
GetGroupFilePath(group_file);
if (group_file.file_type == GroupFileSchema::TO_DELETE) {
if (group_file.file_type == TableFileSchema::TO_DELETE) {
boost::filesystem::remove(group_file.location);
}
ConnectorPtr->remove<GroupFileSchema>(group_file.id);
ConnectorPtr->remove<TableFileSchema>(group_file.id);
/* LOG(DEBUG) << "Removing id=" << group_file.id << " location=" << group_file.location << std::endl; */
}
} catch (std::exception & e) {
@ -663,14 +663,14 @@ Status DBMetaImpl::cleanup() {
Status DBMetaImpl::count(const std::string& group_id, long& result) {
try {
auto selected = ConnectorPtr->select(columns(&GroupFileSchema::size,
&GroupFileSchema::date),
where((c(&GroupFileSchema::file_type) == (int)GroupFileSchema::RAW or
c(&GroupFileSchema::file_type) == (int)GroupFileSchema::TO_INDEX or
c(&GroupFileSchema::file_type) == (int)GroupFileSchema::INDEX) and
c(&GroupFileSchema::group_id) == group_id));
auto selected = ConnectorPtr->select(columns(&TableFileSchema::size,
&TableFileSchema::date),
where((c(&TableFileSchema::file_type) == (int)TableFileSchema::RAW or
c(&TableFileSchema::file_type) == (int)TableFileSchema::TO_INDEX or
c(&TableFileSchema::file_type) == (int)TableFileSchema::INDEX) and
c(&TableFileSchema::group_id) == group_id));
GroupSchema group_info;
TableSchema group_info;
group_info.group_id = group_id;
auto status = get_group_no_lock(group_info);
if (!status.ok()) {

14
cpp/src/db/DBMetaImpl.h
View File

@ -19,11 +19,11 @@ class DBMetaImpl : public Meta {
public:
DBMetaImpl(const DBMetaOptions& options_);
virtual Status add_group(GroupSchema& group_info) override;
virtual Status get_group(GroupSchema& group_info_) override;
virtual Status add_group(TableSchema& group_info) override;
virtual Status get_group(TableSchema& group_info_) override;
virtual Status has_group(const std::string& group_id_, bool& has_or_not_) override;
virtual Status add_group_file(GroupFileSchema& group_file_info) override;
virtual Status add_group_file(TableFileSchema& group_file_info) override;
virtual Status delete_group_partitions(const std::string& group_id,
const meta::DatesT& dates) override;
@ -32,8 +32,8 @@ public:
bool& has_or_not_) override;
virtual Status get_group_file(const std::string& group_id_,
const std::string& file_id_,
GroupFileSchema& group_file_info_) override;
virtual Status update_group_file(GroupFileSchema& group_file_) override;
TableFileSchema& group_file_info_) override;
virtual Status update_group_file(TableFileSchema& group_file_) override;
virtual Status get_group_files(const std::string& group_id_,
const int date_delta_,
@ -68,10 +68,10 @@ private:
Status NextFileId(std::string& file_id);
Status NextGroupId(std::string& group_id);
Status discard_files_of_size(long to_discard_size);
Status get_group_no_lock(GroupSchema& group_info);
Status get_group_no_lock(TableSchema& group_info);
std::string GetGroupPath(const std::string& group_id);
std::string GetGroupDatePartitionPath(const std::string& group_id, DateT& date);
void GetGroupFilePath(GroupFileSchema& group_file);
void GetGroupFilePath(TableFileSchema& group_file);
Status initialize();
const DBMetaOptions _options;

28
cpp/src/db/LocalMetaImpl.cpp
View File

@ -38,8 +38,8 @@ std::string LocalMetaImpl::GetGroupDatePartitionPath(const std::string& group_id
}
std::string LocalMetaImpl::GetNextGroupFileLocationByPartition(const std::string& group_id, DateT& date,
GroupFileSchema::FILE_TYPE file_type) {
std::string suffix = (file_type == GroupFileSchema::RAW) ? ".raw" : ".index";
TableFileSchema::FILE_TYPE file_type) {
std::string suffix = (file_type == TableFileSchema::RAW) ? ".raw" : ".index";
SimpleIDGenerator g;
std::stringstream ss;
ss << GetGroupPath(group_id) << "/" << date << "/" << g.getNextIDNumber() << suffix;
@ -54,7 +54,7 @@ std::string LocalMetaImpl::GetGroupMetaPath(const std::string& group_id) {
return GetGroupMetaPathByGroupPath(GetGroupPath(group_id));
}
Status LocalMetaImpl::GetGroupMetaInfoByPath(const std::string& path, GroupSchema& group_info) {
Status LocalMetaImpl::GetGroupMetaInfoByPath(const std::string& path, TableSchema& group_info) {
boost::property_tree::ptree ptree;
boost::property_tree::read_json(path, ptree);
auto files_cnt = ptree.get_child("files_cnt").data();
@ -71,7 +71,7 @@ Status LocalMetaImpl::GetGroupMetaInfoByPath(const std::string& path, GroupSchem
}
Status LocalMetaImpl::GetGroupMetaInfo(const std::string& group_id, GroupSchema& group_info) {
Status LocalMetaImpl::GetGroupMetaInfo(const std::string& group_id, TableSchema& group_info) {
group_info.group_id = group_id;
return GetGroupMetaInfoByPath(GetGroupMetaPath(group_id), group_info);
}
@ -90,7 +90,7 @@ Status LocalMetaImpl::initialize() {
return Status::OK();
}
Status LocalMetaImpl::add_group(GroupSchema& group_info) {
Status LocalMetaImpl::add_group(TableSchema& group_info) {
std::string real_gid;
size_t id = SimpleIDGenerator().getNextIDNumber();
if (group_info.group_id == "") {
@ -123,7 +123,7 @@ Status LocalMetaImpl::add_group(GroupSchema& group_info) {
return Status::OK();
}
Status LocalMetaImpl::get_group(GroupSchema& group_info) {
Status LocalMetaImpl::get_group(TableSchema& group_info) {
bool group_exist;
has_group(group_info.group_id, group_exist);
if (!group_exist) {
@ -138,8 +138,8 @@ Status LocalMetaImpl::has_group(const std::string& group_id, bool& has_or_not) {
return Status::OK();
}
Status LocalMetaImpl::add_group_file(GroupFileSchema& group_file_info) {
GroupSchema group_info;
Status LocalMetaImpl::add_group_file(TableFileSchema& group_file_info) {
TableSchema group_info;
/* auto status = get_group(group_info); */
/* if (!status.ok()) { */
/* return status; */
@ -159,7 +159,7 @@ Status LocalMetaImpl::files_to_index(GroupFilesSchema& files) {
boost::filesystem::directory_iterator end_itr;
for (boost::filesystem::directory_iterator itr(_options.path); itr != end_itr; ++itr) {
auto group_path = itr->path().string();
GroupSchema group_info;
TableSchema group_info;
GetGroupMetaInfoByPath(GetGroupMetaPathByGroupPath(group_path), group_info);
for (boost::filesystem::directory_iterator innerItr(group_path); innerItr != end_itr; ++innerItr) {
auto partition_path = innerItr->path().string();
@ -169,7 +169,7 @@ Status LocalMetaImpl::files_to_index(GroupFilesSchema& files) {
if (suffix == "index") continue;
if (INDEX_TRIGGER_SIZE >= GetFileSize(location)) continue;
std::cout << "[About to index] " << location << std::endl;
GroupFileSchema f;
TableFileSchema f;
f.location = location;
/* f.group_id = group_id; */
f.dimension = group_info.dimension;
@ -188,7 +188,7 @@ Status LocalMetaImpl::files_to_merge(const std::string& group_id,
/* boost::filesystem::directory_iterator end_itr; */
/* for (boost::filesystem::directory_iterator itr(_options.path); itr != end_itr; ++itr) { */
/* auto group_path = itr->path().string(); */
/* GroupSchema group_info; */
/* TableSchema group_info; */
/* GetGroupMetaInfoByPath(GetGroupMetaPathByGroupPath(group_path), group_info); */
/* for (boost::filesystem::directory_iterator innerItr(group_path); innerItr != end_itr; ++innerItr) { */
/* auto partition_path = innerItr->path().string(); */
@ -198,7 +198,7 @@ Status LocalMetaImpl::files_to_merge(const std::string& group_id,
/* if (suffix == "index") continue; */
/* if (INDEX_TRIGGER_SIZE < GetFileSize(location)) continue; */
/* std::cout << "[About to index] " << location << std::endl; */
/* GroupFileSchema f; */
/* TableFileSchema f; */
/* f.location = location; */
/* f.group_id = group_id; */
/* f.dimension = group_info.dimension; */
@ -219,7 +219,7 @@ Status LocalMetaImpl::has_group_file(const std::string& group_id_,
Status LocalMetaImpl::get_group_file(const std::string& group_id_,
const std::string& file_id_,
GroupFileSchema& group_file_info_) {
TableFileSchema& group_file_info_) {
//PXU TODO
return Status::OK();
}
@ -231,7 +231,7 @@ Status LocalMetaImpl::get_group_files(const std::string& group_id_,
return Status::OK();
}
Status LocalMetaImpl::update_group_file(GroupFileSchema& group_file_) {
Status LocalMetaImpl::update_group_file(TableFileSchema& group_file_) {
//PXU TODO
return Status::OK();
}

18
cpp/src/db/LocalMetaImpl.h
View File

@ -18,11 +18,11 @@ public:
const size_t INDEX_TRIGGER_SIZE = 1024*1024*500;
LocalMetaImpl(const DBMetaOptions& options_);
virtual Status add_group(GroupSchema& group_info_) override;
virtual Status get_group(GroupSchema& group_info_) override;
virtual Status add_group(TableSchema& group_info_) override;
virtual Status get_group(TableSchema& group_info_) override;
virtual Status has_group(const std::string& group_id_, bool& has_or_not_) override;
virtual Status add_group_file(GroupFileSchema& group_file_info) override;
virtual Status add_group_file(TableFileSchema& group_file_info) override;
/* virtual Status delete_group_partitions(const std::string& group_id, */
/* const meta::DatesT& dates) override; */
@ -31,8 +31,8 @@ public:
bool& has_or_not_) override;
virtual Status get_group_file(const std::string& group_id_,
const std::string& file_id_,
GroupFileSchema& group_file_info_) override;
virtual Status update_group_file(GroupFileSchema& group_file_) override;
TableFileSchema& group_file_info_) override;
virtual Status update_group_file(TableFileSchema& group_file_) override;
virtual Status get_group_files(const std::string& group_id_,
const int date_delta_,
@ -59,16 +59,16 @@ public:
private:
Status GetGroupMetaInfoByPath(const std::string& path, GroupSchema& group_info);
Status GetGroupMetaInfoByPath(const std::string& path, TableSchema& group_info);
std::string GetGroupMetaPathByGroupPath(const std::string& group_path);
Status GetGroupMetaInfo(const std::string& group_id, GroupSchema& group_info);
Status GetGroupMetaInfo(const std::string& group_id, TableSchema& group_info);
std::string GetNextGroupFileLocationByPartition(const std::string& group_id, DateT& date,
GroupFileSchema::FILE_TYPE file_type);
TableFileSchema::FILE_TYPE file_type);
std::string GetGroupDatePartitionPath(const std::string& group_id, DateT& date);
std::string GetGroupPath(const std::string& group_id);
std::string GetGroupMetaPath(const std::string& group_id);
Status CreateGroupMeta(const GroupSchema& group_schema);
Status CreateGroupMeta(const TableSchema& group_schema);
long GetFileSize(const std::string& filename);
Status initialize();

8
cpp/src/db/MemManager.cpp
View File

@ -22,7 +22,7 @@ namespace engine {
template<typename EngineT>
MemVectors<EngineT>::MemVectors(const std::shared_ptr<meta::Meta>& meta_ptr,
const meta::GroupFileSchema& schema, const Options& options)
const meta::TableFileSchema& schema, const Options& options)
: pMeta_(meta_ptr),
options_(options),
schema_(schema),
@ -53,11 +53,11 @@ Status MemVectors<EngineT>::serialize(std::string& group_id) {
pEE_->Serialize();
schema_.size = size;
schema_.file_type = (size >= options_.index_trigger_size) ?
meta::GroupFileSchema::TO_INDEX : meta::GroupFileSchema::RAW;
meta::TableFileSchema::TO_INDEX : meta::TableFileSchema::RAW;
auto status = pMeta_->update_group_file(schema_);
LOG(DEBUG) << "New " << ((schema_.file_type == meta::GroupFileSchema::RAW) ? "raw" : "to_index")
LOG(DEBUG) << "New " << ((schema_.file_type == meta::TableFileSchema::RAW) ? "raw" : "to_index")
<< " file " << schema_.file_id << " of size " << pEE_->Size() / meta::M << " M";
pEE_->Cache();
@ -85,7 +85,7 @@ typename MemManager<EngineT>::MemVectorsPtr MemManager<EngineT>::get_mem_by_grou
return memIt->second;
}
meta::GroupFileSchema group_file;
meta::TableFileSchema group_file;
group_file.group_id = group_id;
auto status = _pMeta->add_group_file(group_file);
if (!status.ok()) {

4
cpp/src/db/MemManager.h
View File

@ -31,7 +31,7 @@ public:
typedef std::shared_ptr<MemVectors<EngineT>> Ptr;
explicit MemVectors(const std::shared_ptr<meta::Meta>&,
const meta::GroupFileSchema&, const Options&);
const meta::TableFileSchema&, const Options&);
void add(size_t n_, const float* vectors_, IDNumbers& vector_ids_);
@ -52,7 +52,7 @@ private:
MetaPtr pMeta_;
Options options_;
meta::GroupFileSchema schema_;
meta::TableFileSchema schema_;
IDGenerator* _pIdGenerator;
EnginePtr pEE_;

12
cpp/src/db/Meta.h
View File

@ -20,13 +20,13 @@ namespace meta {
class Meta {
public:
typedef std::shared_ptr<Meta> Ptr;
using Ptr = std::shared_ptr<Meta>;
virtual Status add_group(GroupSchema& group_info) = 0;
virtual Status get_group(GroupSchema& group_info) = 0;
virtual Status add_group(TableSchema& group_info) = 0;
virtual Status get_group(TableSchema& group_info) = 0;
virtual Status has_group(const std::string& group_id_, bool& has_or_not_) = 0;
virtual Status add_group_file(GroupFileSchema& group_file_info) = 0;
virtual Status add_group_file(TableFileSchema& group_file_info) = 0;
virtual Status delete_group_partitions(const std::string& group_id,
const meta::DatesT& dates) = 0;
@ -35,8 +35,8 @@ public:
bool& has_or_not_) = 0;
virtual Status get_group_file(const std::string& group_id_,
const std::string& file_id_,
GroupFileSchema& group_file_info_) = 0;
virtual Status update_group_file(GroupFileSchema& group_file_) = 0;
TableFileSchema& group_file_info_) = 0;
virtual Status update_group_file(TableFileSchema& group_file_) = 0;
virtual Status get_group_files(const std::string& group_id_,
const int date_delta_,

14
cpp/src/db/MetaTypes.h
View File

@ -18,16 +18,16 @@ typedef int DateT;
const DateT EmptyDate = -1;
typedef std::vector<DateT> DatesT;
struct GroupSchema {
struct TableSchema {
size_t id;
std::string group_id;
size_t files_cnt = 0;
uint16_t dimension;
std::string location = "";
std::string location;
long created_on;
}; // GroupSchema
}; // TableSchema
struct GroupFileSchema {
struct TableFileSchema {
typedef enum {
NEW,
RAW,
@ -43,12 +43,12 @@ struct GroupFileSchema {
size_t size;
DateT date = EmptyDate;
uint16_t dimension;
std::string location = "";
std::string location;
long updated_time;
long created_on;
}; // GroupFileSchema
}; // TableFileSchema
typedef std::vector<GroupFileSchema> GroupFilesSchema;
typedef std::vector<TableFileSchema> GroupFilesSchema;
typedef std::map<DateT, GroupFilesSchema> DatePartionedGroupFilesSchema;
} // namespace meta

12
cpp/unittest/db/db_tests.cpp
View File

@ -66,12 +66,12 @@ TEST_F(DBTest2, ARHIVE_DISK_CHECK) {
static const int group_dim = 256;
long size;
engine::meta::GroupSchema group_info;
engine::meta::TableSchema group_info;
group_info.dimension = group_dim;
group_info.group_id = group_name;
engine::Status stat = db_->add_group(group_info);
engine::meta::GroupSchema group_info_get;
engine::meta::TableSchema group_info_get;
group_info_get.group_id = group_name;
stat = db_->get_group(group_info_get);
ASSERT_STATS(stat);
@ -111,12 +111,12 @@ TEST_F(DBTest, DB_TEST) {
static const std::string group_name = "test_group";
static const int group_dim = 256;
engine::meta::GroupSchema group_info;
engine::meta::TableSchema group_info;
group_info.dimension = group_dim;
group_info.group_id = group_name;
engine::Status stat = db_->add_group(group_info);
engine::meta::GroupSchema group_info_get;
engine::meta::TableSchema group_info_get;
group_info_get.group_id = group_name;
stat = db_->get_group(group_info_get);
ASSERT_STATS(stat);
@ -197,12 +197,12 @@ TEST_F(DBTest, SEARCH_TEST) {
static const std::string group_name = "test_group";
static const int group_dim = 256;
engine::meta::GroupSchema group_info;
engine::meta::TableSchema group_info;
group_info.dimension = group_dim;
group_info.group_id = group_name;
engine::Status stat = db_->add_group(group_info);
engine::meta::GroupSchema group_info_get;
engine::meta::TableSchema group_info_get;
group_info_get.group_id = group_name;
stat = db_->get_group(group_info_get);
ASSERT_STATS(stat);

46
cpp/unittest/db/meta_tests.cpp
View File

@ -20,7 +20,7 @@ using namespace zilliz::vecwise::engine;
TEST_F(MetaTest, GROUP_TEST) {
auto group_id = "meta_test_group";
meta::GroupSchema group;
meta::TableSchema group;
group.group_id = group_id;
auto status = impl_->add_group(group);
ASSERT_TRUE(status.ok());
@ -44,19 +44,19 @@ TEST_F(MetaTest, GROUP_TEST) {
TEST_F(MetaTest, GROUP_FILE_TEST) {
auto group_id = "meta_test_group";
meta::GroupSchema group;
meta::TableSchema group;
group.group_id = group_id;
auto status = impl_->add_group(group);
meta::GroupFileSchema group_file;
meta::TableFileSchema group_file;
group_file.group_id = group.group_id;
status = impl_->add_group_file(group_file);
ASSERT_TRUE(status.ok());
ASSERT_EQ(group_file.file_type, meta::GroupFileSchema::NEW);
ASSERT_EQ(group_file.file_type, meta::TableFileSchema::NEW);
auto file_id = group_file.file_id;
auto new_file_type = meta::GroupFileSchema::INDEX;
auto new_file_type = meta::TableFileSchema::INDEX;
group_file.file_type = new_file_type;
status = impl_->update_group_file(group_file);
@ -79,7 +79,7 @@ TEST_F(MetaTest, GROUP_FILE_TEST) {
status = impl_->update_group_file(group_file);
ASSERT_TRUE(status.ok());
ASSERT_EQ(group_file.date, meta::Meta::GetDateWithDelta(-2));
ASSERT_FALSE(group_file.file_type == meta::GroupFileSchema::TO_DELETE);
ASSERT_FALSE(group_file.file_type == meta::TableFileSchema::TO_DELETE);
dates.clear();
dates.push_back(group_file.date);
@ -87,7 +87,7 @@ TEST_F(MetaTest, GROUP_FILE_TEST) {
ASSERT_TRUE(status.ok());
status = impl_->get_group_file(group_file.group_id, group_file.file_id, group_file);
ASSERT_TRUE(status.ok());
ASSERT_TRUE(group_file.file_type == meta::GroupFileSchema::TO_DELETE);
ASSERT_TRUE(group_file.file_type == meta::TableFileSchema::TO_DELETE);
}
TEST_F(MetaTest, ARCHIVE_TEST_DAYS) {
@ -102,12 +102,12 @@ TEST_F(MetaTest, ARCHIVE_TEST_DAYS) {
auto impl = meta::DBMetaImpl(options);
auto group_id = "meta_test_group";
meta::GroupSchema group;
meta::TableSchema group;
group.group_id = group_id;
auto status = impl.add_group(group);
meta::GroupFilesSchema files;
meta::GroupFileSchema group_file;
meta::TableFileSchema group_file;
group_file.group_id = group.group_id;
auto cnt = 100;
@ -115,7 +115,7 @@ TEST_F(MetaTest, ARCHIVE_TEST_DAYS) {
std::vector<int> days;
for (auto i=0; i<cnt; ++i) {
status = impl.add_group_file(group_file);
group_file.file_type = meta::GroupFileSchema::NEW;
group_file.file_type = meta::TableFileSchema::NEW;
int day = rand() % (days_num*2);
group_file.created_on = ts - day*meta::D_SEC*meta::US_PS - 10000;
status = impl.update_group_file(group_file);
@ -130,9 +130,9 @@ TEST_F(MetaTest, ARCHIVE_TEST_DAYS) {
status = impl.get_group_file(file.group_id, file.file_id, file);
ASSERT_TRUE(status.ok());
if (days[i] < days_num) {
ASSERT_EQ(file.file_type, meta::GroupFileSchema::NEW);
ASSERT_EQ(file.file_type, meta::TableFileSchema::NEW);
} else {
ASSERT_EQ(file.file_type, meta::GroupFileSchema::TO_DELETE);
ASSERT_EQ(file.file_type, meta::TableFileSchema::TO_DELETE);
}
i++;
}
@ -148,19 +148,19 @@ TEST_F(MetaTest, ARCHIVE_TEST_DISK) {
auto impl = meta::DBMetaImpl(options);
auto group_id = "meta_test_group";
meta::GroupSchema group;
meta::TableSchema group;
group.group_id = group_id;
auto status = impl.add_group(group);
meta::GroupFilesSchema files;
meta::GroupFileSchema group_file;
meta::TableFileSchema group_file;
group_file.group_id = group.group_id;
auto cnt = 10;
auto each_size = 2UL;
for (auto i=0; i<cnt; ++i) {
status = impl.add_group_file(group_file);
group_file.file_type = meta::GroupFileSchema::NEW;
group_file.file_type = meta::TableFileSchema::NEW;
group_file.size = each_size * meta::G;
status = impl.update_group_file(group_file);
files.push_back(group_file);
@ -173,9 +173,9 @@ TEST_F(MetaTest, ARCHIVE_TEST_DISK) {
status = impl.get_group_file(file.group_id, file.file_id, file);
ASSERT_TRUE(status.ok());
if (i < 5) {
ASSERT_TRUE(file.file_type == meta::GroupFileSchema::TO_DELETE);
ASSERT_TRUE(file.file_type == meta::TableFileSchema::TO_DELETE);
} else {
ASSERT_EQ(file.file_type, meta::GroupFileSchema::NEW);
ASSERT_EQ(file.file_type, meta::TableFileSchema::NEW);
}
++i;
}
@ -186,7 +186,7 @@ TEST_F(MetaTest, ARCHIVE_TEST_DISK) {
TEST_F(MetaTest, GROUP_FILES_TEST) {
auto group_id = "meta_test_group";
meta::GroupSchema group;
meta::TableSchema group;
group.group_id = group_id;
auto status = impl_->add_group(group);
@ -195,30 +195,30 @@ TEST_F(MetaTest, GROUP_FILES_TEST) {
int to_index_files_cnt = 6;
int index_files_cnt = 7;
meta::GroupFileSchema group_file;
meta::TableFileSchema group_file;
group_file.group_id = group.group_id;
for (auto i=0; i<new_files_cnt; ++i) {
status = impl_->add_group_file(group_file);
group_file.file_type = meta::GroupFileSchema::NEW;
group_file.file_type = meta::TableFileSchema::NEW;
status = impl_->update_group_file(group_file);
}
for (auto i=0; i<raw_files_cnt; ++i) {
status = impl_->add_group_file(group_file);
group_file.file_type = meta::GroupFileSchema::RAW;
group_file.file_type = meta::TableFileSchema::RAW;
status = impl_->update_group_file(group_file);
}
for (auto i=0; i<to_index_files_cnt; ++i) {
status = impl_->add_group_file(group_file);
group_file.file_type = meta::GroupFileSchema::TO_INDEX;
group_file.file_type = meta::TableFileSchema::TO_INDEX;
status = impl_->update_group_file(group_file);
}
for (auto i=0; i<index_files_cnt; ++i) {
status = impl_->add_group_file(group_file);
group_file.file_type = meta::GroupFileSchema::INDEX;
group_file.file_type = meta::TableFileSchema::INDEX;
status = impl_->update_group_file(group_file);
}