milvus-io
/
milvus
mirror of https://github.com/milvus-io/milvus.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

refactor(db): change group_info to table_schema and add_group to CreateTable 

Former-commit-id: e8d48ec8e54429bf5d1080997ca329b42ddcab82
pull/191/head
Xu Peng 2019-05-27 17:45:39 +08:00
parent f2bb20c90d
commit 3d330e0aa4
9 changed files with 91 additions and 91 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
cpp/src/db/DB.h
View File

@ -21,8 +21,8 @@ class DB {
public:
static void Open(const Options& options, DB** dbptr);
virtual Status add_group(meta::TableSchema& group_info_) = 0;
virtual Status get_group(meta::TableSchema& group_info_) = 0;
virtual Status add_group(meta::TableSchema& table_schema_) = 0;
virtual Status get_group(meta::TableSchema& table_schema_) = 0;
virtual Status delete_vectors(const std::string& table_id,
const meta::DatesT& dates) = 0;
virtual Status has_group(const std::string& table_id_, bool& has_or_not_) = 0;

8
cpp/src/db/DBImpl.cpp
View File

@ -35,13 +35,13 @@ DBImpl<EngineT>::DBImpl(const Options& options)
}
template<typename EngineT>
Status DBImpl<EngineT>::add_group(meta::TableSchema& group_info) {
return _pMeta->add_group(group_info);
Status DBImpl<EngineT>::add_group(meta::TableSchema& table_schema) {
return _pMeta->CreateTable(table_schema);
}
template<typename EngineT>
Status DBImpl<EngineT>::get_group(meta::TableSchema& group_info) {
return _pMeta->get_group(group_info);
Status DBImpl<EngineT>::get_group(meta::TableSchema& table_schema) {
return _pMeta->get_group(table_schema);
}
template<typename EngineT>

4
cpp/src/db/DBImpl.h
View File

@ -33,8 +33,8 @@ public:
DBImpl(const Options& options);
virtual Status add_group(meta::TableSchema& group_info) override;
virtual Status get_group(meta::TableSchema& group_info) override;
virtual Status add_group(meta::TableSchema& table_schema) override;
virtual Status get_group(meta::TableSchema& table_schema) override;
virtual Status delete_vectors(const std::string& table_id, const meta::DatesT& dates) override;
virtual Status has_group(const std::string& table_id_, bool& has_or_not_) override;

80
cpp/src/db/DBMetaImpl.cpp
View File

@ -117,9 +117,9 @@ Status DBMetaImpl::delete_group_partitions(const std::string& table_id,
return Status::OK();
}
TableSchema group_info;
group_info.table_id = table_id;
auto status = get_group(group_info);
TableSchema table_schema;
table_schema.table_id = table_id;
auto status = get_group(table_schema);
if (!status.ok()) {
return status;
}
@ -148,24 +148,24 @@ Status DBMetaImpl::delete_group_partitions(const std::string& table_id,
return Status::OK();
}
Status DBMetaImpl::add_group(TableSchema& group_info) {
if (group_info.table_id == "") {
NextGroupId(group_info.table_id);
Status DBMetaImpl::CreateTable(TableSchema& table_schema) {
if (table_schema.table_id == "") {
NextGroupId(table_schema.table_id);
}
group_info.files_cnt = 0;
group_info.id = -1;
group_info.created_on = utils::GetMicroSecTimeStamp();
table_schema.files_cnt = 0;
table_schema.id = -1;
table_schema.created_on = utils::GetMicroSecTimeStamp();
{
try {
auto id = ConnectorPtr->insert(group_info);
group_info.id = id;
auto id = ConnectorPtr->insert(table_schema);
table_schema.id = id;
} catch (...) {
return Status::DBTransactionError("Add Group Error");
}
}
auto group_path = GetGroupPath(group_info.table_id);
auto group_path = GetGroupPath(table_schema.table_id);
if (!boost::filesystem::is_directory(group_path)) {
auto ret = boost::filesystem::create_directories(group_path);
@ -178,24 +178,24 @@ Status DBMetaImpl::add_group(TableSchema& group_info) {
return Status::OK();
}
Status DBMetaImpl::get_group(TableSchema& group_info) {
return get_group_no_lock(group_info);
Status DBMetaImpl::get_group(TableSchema& table_schema) {
return get_group_no_lock(table_schema);
}
Status DBMetaImpl::get_group_no_lock(TableSchema& group_info) {
Status DBMetaImpl::get_group_no_lock(TableSchema& table_schema) {
try {
auto groups = ConnectorPtr->select(columns(&TableSchema::id,
&TableSchema::table_id,
&TableSchema::files_cnt,
&TableSchema::dimension),
where(c(&TableSchema::table_id) == group_info.table_id));
where(c(&TableSchema::table_id) == table_schema.table_id));
assert(groups.size() <= 1);
if (groups.size() == 1) {
group_info.id = std::get<0>(groups[0]);
group_info.files_cnt = std::get<2>(groups[0]);
group_info.dimension = std::get<3>(groups[0]);
table_schema.id = std::get<0>(groups[0]);
table_schema.files_cnt = std::get<2>(groups[0]);
table_schema.dimension = std::get<3>(groups[0]);
} else {
return Status::NotFound("Group " + group_info.table_id + " not found");
return Status::NotFound("Group " + table_schema.table_id + " not found");
}
} catch (std::exception &e) {
LOG(DEBUG) << e.what();
@ -226,16 +226,16 @@ Status DBMetaImpl::add_group_file(TableFileSchema& group_file) {
if (group_file.date == EmptyDate) {
group_file.date = Meta::GetDate();
}
TableSchema group_info;
group_info.table_id = group_file.table_id;
auto status = get_group(group_info);
TableSchema table_schema;
table_schema.table_id = group_file.table_id;
auto status = get_group(table_schema);
if (!status.ok()) {
return status;
}
NextFileId(group_file.file_id);
group_file.file_type = TableFileSchema::NEW;
group_file.dimension = group_info.dimension;
group_file.dimension = table_schema.dimension;
group_file.size = 0;
group_file.created_on = utils::GetMicroSecTimeStamp();
group_file.updated_time = group_file.created_on;
@ -288,13 +288,13 @@ Status DBMetaImpl::files_to_index(TableFilesSchema& files) {
GetGroupFilePath(group_file);
auto groupItr = groups.find(group_file.table_id);
if (groupItr == groups.end()) {
TableSchema group_info;
group_info.table_id = group_file.table_id;
auto status = get_group_no_lock(group_info);
TableSchema table_schema;
table_schema.table_id = group_file.table_id;
auto status = get_group_no_lock(table_schema);
if (!status.ok()) {
return status;
}
groups[group_file.table_id] = group_info;
groups[group_file.table_id] = table_schema;
}
group_file.dimension = groups[group_file.table_id].dimension;
files.push_back(group_file);
@ -327,9 +327,9 @@ Status DBMetaImpl::files_to_search(const std::string &table_id,
c(&TableFileSchema::file_type) == (int) TableFileSchema::TO_INDEX or
c(&TableFileSchema::file_type) == (int) TableFileSchema::INDEX)));
TableSchema group_info;
group_info.table_id = table_id;
auto status = get_group_no_lock(group_info);
TableSchema table_schema;
table_schema.table_id = table_id;
auto status = get_group_no_lock(table_schema);
if (!status.ok()) {
return status;
}
@ -342,7 +342,7 @@ Status DBMetaImpl::files_to_search(const std::string &table_id,
group_file.file_type = std::get<3>(file);
group_file.size = std::get<4>(file);
group_file.date = std::get<5>(file);
group_file.dimension = group_info.dimension;
group_file.dimension = table_schema.dimension;
GetGroupFilePath(group_file);
auto dateItr = files.find(group_file.date);
if (dateItr == files.end()) {
@ -372,9 +372,9 @@ Status DBMetaImpl::files_to_merge(const std::string& table_id,
where(c(&TableFileSchema::file_type) == (int)TableFileSchema::RAW and
c(&TableFileSchema::table_id) == table_id));
TableSchema group_info;
group_info.table_id = table_id;
auto status = get_group_no_lock(group_info);
TableSchema table_schema;
table_schema.table_id = table_id;
auto status = get_group_no_lock(table_schema);
if (!status.ok()) {
return status;
}
@ -387,7 +387,7 @@ Status DBMetaImpl::files_to_merge(const std::string& table_id,
group_file.file_type = std::get<3>(file);
group_file.size = std::get<4>(file);
group_file.date = std::get<5>(file);
group_file.dimension = group_info.dimension;
group_file.dimension = table_schema.dimension;
GetGroupFilePath(group_file);
auto dateItr = files.find(group_file.date);
if (dateItr == files.end()) {
@ -670,9 +670,9 @@ Status DBMetaImpl::count(const std::string& table_id, long& result) {
c(&TableFileSchema::file_type) == (int)TableFileSchema::INDEX) and
c(&TableFileSchema::table_id) == table_id));
TableSchema group_info;
group_info.table_id = table_id;
auto status = get_group_no_lock(group_info);
TableSchema table_schema;
table_schema.table_id = table_id;
auto status = get_group_no_lock(table_schema);
if (!status.ok()) {
return status;
}
@ -682,7 +682,7 @@ Status DBMetaImpl::count(const std::string& table_id, long& result) {
result += std::get<0>(file);
}
result /= group_info.dimension;
result /= table_schema.dimension;
} catch (std::exception & e) {
LOG(DEBUG) << e.what();

2
cpp/src/db/DBMetaImpl.h
View File

@ -19,7 +19,7 @@ class DBMetaImpl : public Meta {
public:
DBMetaImpl(const DBMetaOptions& options_);
virtual Status add_group(TableSchema& group_info) override;
virtual Status CreateTable(TableSchema& table_schema) override;
virtual Status get_group(TableSchema& group_info_) override;
virtual Status has_group(const std::string& table_id_, bool& has_or_not_) override;

60
cpp/src/db/LocalMetaImpl.cpp
View File

@ -54,7 +54,7 @@ std::string LocalMetaImpl::GetGroupMetaPath(const std::string& table_id) {
return GetGroupMetaPathByGroupPath(GetGroupPath(table_id));
}
Status LocalMetaImpl::GetGroupMetaInfoByPath(const std::string& path, TableSchema& group_info) {
Status LocalMetaImpl::GetGroupMetaInfoByPath(const std::string& path, TableSchema& table_schema) {
boost::property_tree::ptree ptree;
boost::property_tree::read_json(path, ptree);
auto files_cnt = ptree.get_child("files_cnt").data();
@ -62,18 +62,18 @@ Status LocalMetaImpl::GetGroupMetaInfoByPath(const std::string& path, TableSchem
/* std::cout << dimension << std::endl; */
/* std::cout << files_cnt << std::endl; */
group_info.id = std::stoi(group_info.table_id);
group_info.files_cnt = std::stoi(files_cnt);
group_info.dimension = std::stoi(dimension);
group_info.location = GetGroupPath(group_info.table_id);
table_schema.id = std::stoi(table_schema.table_id);
table_schema.files_cnt = std::stoi(files_cnt);
table_schema.dimension = std::stoi(dimension);
table_schema.location = GetGroupPath(table_schema.table_id);
return Status::OK();
}
Status LocalMetaImpl::GetGroupMetaInfo(const std::string& table_id, TableSchema& group_info) {
group_info.table_id = table_id;
return GetGroupMetaInfoByPath(GetGroupMetaPath(table_id), group_info);
Status LocalMetaImpl::GetGroupMetaInfo(const std::string& table_id, TableSchema& table_schema) {
table_schema.table_id = table_id;
return GetGroupMetaInfoByPath(GetGroupMetaPath(table_id), table_schema);
}
LocalMetaImpl::LocalMetaImpl(const DBMetaOptions& options_)
@ -90,15 +90,15 @@ Status LocalMetaImpl::initialize() {
return Status::OK();
}
Status LocalMetaImpl::add_group(TableSchema& group_info) {
Status LocalMetaImpl::CreateTable(TableSchema& table_schema) {
std::string real_gid;
size_t id = SimpleIDGenerator().getNextIDNumber();
if (group_info.table_id == "") {
if (table_schema.table_id == "") {
std::stringstream ss;
ss << id;
real_gid = ss.str();
} else {
real_gid = group_info.table_id;
real_gid = table_schema.table_id;
}
bool group_exist;
@ -110,27 +110,27 @@ Status LocalMetaImpl::add_group(TableSchema& group_info) {
return Status::GroupError("Cannot Create Group " + real_gid);
}
group_info.table_id = real_gid;
group_info.files_cnt = 0;
group_info.id = 0;
group_info.location = GetGroupPath(real_gid);
table_schema.table_id = real_gid;
table_schema.files_cnt = 0;
table_schema.id = 0;
table_schema.location = GetGroupPath(real_gid);
boost::property_tree::ptree out;
out.put("files_cnt", group_info.files_cnt);
out.put("dimension", group_info.dimension);
out.put("files_cnt", table_schema.files_cnt);
out.put("dimension", table_schema.dimension);
boost::property_tree::write_json(GetGroupMetaPath(real_gid), out);
return Status::OK();
}
Status LocalMetaImpl::get_group(TableSchema& group_info) {
Status LocalMetaImpl::get_group(TableSchema& table_schema) {
bool group_exist;
has_group(group_info.table_id, group_exist);
has_group(table_schema.table_id, group_exist);
if (!group_exist) {
return Status::NotFound("Group " + group_info.table_id + " Not Found");
return Status::NotFound("Group " + table_schema.table_id + " Not Found");
}
return GetGroupMetaInfo(group_info.table_id, group_info);
return GetGroupMetaInfo(table_schema.table_id, table_schema);
}
Status LocalMetaImpl::has_group(const std::string& table_id, bool& has_or_not) {
@ -139,14 +139,14 @@ Status LocalMetaImpl::has_group(const std::string& table_id, bool& has_or_not) {
}
Status LocalMetaImpl::add_group_file(TableFileSchema& group_file_info) {
TableSchema group_info;
/* auto status = get_group(group_info); */
TableSchema table_schema;
/* auto status = get_group(table_schema); */
/* if (!status.ok()) { */
/* return status; */
/* } */
/* auto location = GetNextGroupFileLocationByPartition(table_id, date, file_type); */
/* group_file_info.table_id = table_id; */
/* group_file_info.dimension = group_info.dimension; */
/* group_file_info.dimension = table_schema.dimension; */
/* group_file_info.location = location; */
/* group_file_info.date = date; */
return Status::OK();
@ -159,8 +159,8 @@ Status LocalMetaImpl::files_to_index(TableFilesSchema& files) {
boost::filesystem::directory_iterator end_itr;
for (boost::filesystem::directory_iterator itr(_options.path); itr != end_itr; ++itr) {
auto group_path = itr->path().string();
TableSchema group_info;
GetGroupMetaInfoByPath(GetGroupMetaPathByGroupPath(group_path), group_info);
TableSchema table_schema;
GetGroupMetaInfoByPath(GetGroupMetaPathByGroupPath(group_path), table_schema);
for (boost::filesystem::directory_iterator innerItr(group_path); innerItr != end_itr; ++innerItr) {
auto partition_path = innerItr->path().string();
for (boost::filesystem::directory_iterator fItr(partition_path); fItr != end_itr; ++fItr) {
@ -172,7 +172,7 @@ Status LocalMetaImpl::files_to_index(TableFilesSchema& files) {
TableFileSchema f;
f.location = location;
/* f.table_id = table_id; */
f.dimension = group_info.dimension;
f.dimension = table_schema.dimension;
files.push_back(f);
}
}
@ -188,8 +188,8 @@ Status LocalMetaImpl::files_to_merge(const std::string& table_id,
/* boost::filesystem::directory_iterator end_itr; */
/* for (boost::filesystem::directory_iterator itr(_options.path); itr != end_itr; ++itr) { */
/* auto group_path = itr->path().string(); */
/* TableSchema group_info; */
/* GetGroupMetaInfoByPath(GetGroupMetaPathByGroupPath(group_path), group_info); */
/* TableSchema table_schema; */
/* GetGroupMetaInfoByPath(GetGroupMetaPathByGroupPath(group_path), table_schema); */
/* for (boost::filesystem::directory_iterator innerItr(group_path); innerItr != end_itr; ++innerItr) { */
/* auto partition_path = innerItr->path().string(); */
/* for (boost::filesystem::directory_iterator fItr(partition_path); fItr != end_itr; ++fItr) { */
@ -201,7 +201,7 @@ Status LocalMetaImpl::files_to_merge(const std::string& table_id,
/* TableFileSchema f; */
/* f.location = location; */
/* f.table_id = table_id; */
/* f.dimension = group_info.dimension; */
/* f.dimension = table_schema.dimension; */
/* files.push_back(f); */
/* } */
/* } */

8
cpp/src/db/LocalMetaImpl.h
View File

@ -18,8 +18,8 @@ public:
const size_t INDEX_TRIGGER_SIZE = 1024*1024*500;
LocalMetaImpl(const DBMetaOptions& options_);
virtual Status add_group(TableSchema& group_info_) override;
virtual Status get_group(TableSchema& group_info_) override;
virtual Status CreateTable(TableSchema& table_schema) override;
virtual Status get_group(TableSchema& table_schema_) override;
virtual Status has_group(const std::string& table_id_, bool& has_or_not_) override;
virtual Status add_group_file(TableFileSchema& group_file_info) override;
@ -59,9 +59,9 @@ public:
private:
Status GetGroupMetaInfoByPath(const std::string& path, TableSchema& group_info);
Status GetGroupMetaInfoByPath(const std::string& path, TableSchema& table_schema);
std::string GetGroupMetaPathByGroupPath(const std::string& group_path);
Status GetGroupMetaInfo(const std::string& table_id, TableSchema& group_info);
Status GetGroupMetaInfo(const std::string& table_id, TableSchema& table_schema);
std::string GetNextGroupFileLocationByPartition(const std::string& table_id, DateT& date,
TableFileSchema::FILE_TYPE file_type);
std::string GetGroupDatePartitionPath(const std::string& table_id, DateT& date);

4
cpp/src/db/Meta.h
View File

@ -22,8 +22,8 @@ class Meta {
public:
using Ptr = std::shared_ptr<Meta>;
virtual Status add_group(TableSchema& group_info) = 0;
virtual Status get_group(TableSchema& group_info) = 0;
virtual Status CreateTable(TableSchema& table_schema) = 0;
virtual Status get_group(TableSchema& table_schema) = 0;
virtual Status has_group(const std::string& table_id_, bool& has_or_not_) = 0;
virtual Status add_group_file(TableFileSchema& group_file_info) = 0;

12
cpp/unittest/db/meta_tests.cpp
View File

@ -22,7 +22,7 @@ TEST_F(MetaTest, GROUP_TEST) {
meta::TableSchema group;
group.table_id = table_id;
auto status = impl_->add_group(group);
auto status = impl_->CreateTable(group);
ASSERT_TRUE(status.ok());
auto gid = group.id;
@ -37,7 +37,7 @@ TEST_F(MetaTest, GROUP_TEST) {
ASSERT_TRUE(!status.ok());
group.table_id = table_id;
status = impl_->add_group(group);
status = impl_->CreateTable(group);
ASSERT_TRUE(!status.ok());
}
@ -46,7 +46,7 @@ TEST_F(MetaTest, GROUP_FILE_TEST) {
meta::TableSchema group;
group.table_id = table_id;
auto status = impl_->add_group(group);
auto status = impl_->CreateTable(group);
meta::TableFileSchema group_file;
group_file.table_id = group.table_id;
@ -104,7 +104,7 @@ TEST_F(MetaTest, ARCHIVE_TEST_DAYS) {
meta::TableSchema group;
group.table_id = table_id;
auto status = impl.add_group(group);
auto status = impl.CreateTable(group);
meta::TableFilesSchema files;
meta::TableFileSchema group_file;
@ -150,7 +150,7 @@ TEST_F(MetaTest, ARCHIVE_TEST_DISK) {
meta::TableSchema group;
group.table_id = table_id;
auto status = impl.add_group(group);
auto status = impl.CreateTable(group);
meta::TableFilesSchema files;
meta::TableFileSchema group_file;
@ -188,7 +188,7 @@ TEST_F(MetaTest, GROUP_FILES_TEST) {
meta::TableSchema group;
group.table_id = table_id;
auto status = impl_->add_group(group);
auto status = impl_->CreateTable(group);
int new_files_cnt = 4;
int raw_files_cnt = 5;