milvus-io
/
milvus
mirror of https://github.com/milvus-io/milvus.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

refactor(db): DropAll -Count CleanUp CleanUpFilesWithTTL 

Former-commit-id: b718bb89945651de19a1263eaf64eab22d11eb6c
pull/191/head
Xu Peng 2019-05-27 19:22:58 +08:00
parent 874eeb8f7a
commit ca33fbbe14
9 changed files with 45 additions and 54 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
cpp/src/db/DB.h
View File

@ -38,8 +38,6 @@ public:
virtual Status drop_all() = 0; virtual Status drop_all() = 0;
virtual Status count(const std::string& table_id, long& result) = 0;
DB() = default; DB() = default;
DB(const DB&) = delete; DB(const DB&) = delete;
DB& operator=(const DB&) = delete; DB& operator=(const DB&) = delete;

9
cpp/src/db/DBImpl.cpp
View File

@ -316,7 +316,7 @@ Status DBImpl<EngineT>::background_merge_files(const std::string& table_id) {
try_build_index(); try_build_index();
_pMeta->cleanup_ttl_files(1); _pMeta->CleanUpFilesWithTTL(1);
return Status::OK(); return Status::OK();
} }
@ -403,12 +403,7 @@ void DBImpl<EngineT>::background_compaction() {
template<typename EngineT> template<typename EngineT>
Status DBImpl<EngineT>::drop_all() { Status DBImpl<EngineT>::drop_all() {
return _pMeta->drop_all(); return _pMeta->DropAll();
}
template<typename EngineT>
Status DBImpl<EngineT>::count(const std::string& table_id, long& result) {
return _pMeta->count(table_id, result);
} }
template<typename EngineT> template<typename EngineT>

2
cpp/src/db/DBImpl.h
View File

@ -48,8 +48,6 @@ public:
virtual Status drop_all() override; virtual Status drop_all() override;
virtual Status count(const std::string& table_id, long& result) override;
virtual Status size(long& result) override; virtual Status size(long& result) override;
virtual ~DBImpl(); virtual ~DBImpl();

60
cpp/src/db/DBMetaImpl.cpp
View File

@ -105,7 +105,7 @@ Status DBMetaImpl::initialize() {
ConnectorPtr->open_forever(); // thread safe option ConnectorPtr->open_forever(); // thread safe option
ConnectorPtr->pragma.journal_mode(journal_mode::WAL); // WAL => write ahead log ConnectorPtr->pragma.journal_mode(journal_mode::WAL); // WAL => write ahead log
cleanup(); CleanUp();
return Status::OK(); return Status::OK();
} }
@ -570,7 +570,7 @@ Status DBMetaImpl::UpdateTableFiles(TableFilesSchema& files) {
return Status::OK(); return Status::OK();
} }
Status DBMetaImpl::cleanup_ttl_files(uint16_t seconds) { Status DBMetaImpl::CleanUpFilesWithTTL(uint16_t seconds) {
auto now = utils::GetMicroSecTimeStamp(); auto now = utils::GetMicroSecTimeStamp();
try { try {
auto selected = ConnectorPtr->select(columns(&TableFileSchema::id, auto selected = ConnectorPtr->select(columns(&TableFileSchema::id,
@ -583,21 +583,21 @@ Status DBMetaImpl::cleanup_ttl_files(uint16_t seconds) {
c(&TableFileSchema::updated_time) > now - seconds*US_PS)); c(&TableFileSchema::updated_time) > now - seconds*US_PS));
TableFilesSchema updated; TableFilesSchema updated;
TableFileSchema table_file;
for (auto& file : selected) { for (auto& file : selected) {
TableFileSchema group_file; table_file.id = std::get<0>(file);
group_file.id = std::get<0>(file); table_file.table_id = std::get<1>(file);
group_file.table_id = std::get<1>(file); table_file.file_id = std::get<2>(file);
group_file.file_id = std::get<2>(file); table_file.file_type = std::get<3>(file);
group_file.file_type = std::get<3>(file); table_file.size = std::get<4>(file);
group_file.size = std::get<4>(file); table_file.date = std::get<5>(file);
group_file.date = std::get<5>(file); GetGroupFilePath(table_file);
GetGroupFilePath(group_file); if (table_file.file_type == TableFileSchema::TO_DELETE) {
if (group_file.file_type == TableFileSchema::TO_DELETE) { boost::filesystem::remove(table_file.location);
boost::filesystem::remove(group_file.location);
} }
ConnectorPtr->remove<TableFileSchema>(group_file.id); ConnectorPtr->remove<TableFileSchema>(table_file.id);
/* LOG(DEBUG) << "Removing deleted id=" << group_file.id << " location=" << group_file.location << std::endl; */ /* LOG(DEBUG) << "Removing deleted id=" << table_file.id << " location=" << table_file.location << std::endl; */
} }
} catch (std::exception & e) { } catch (std::exception & e) {
LOG(DEBUG) << e.what(); LOG(DEBUG) << e.what();
@ -607,7 +607,7 @@ Status DBMetaImpl::cleanup_ttl_files(uint16_t seconds) {
return Status::OK(); return Status::OK();
} }
Status DBMetaImpl::cleanup() { Status DBMetaImpl::CleanUp() {
try { try {
auto selected = ConnectorPtr->select(columns(&TableFileSchema::id, auto selected = ConnectorPtr->select(columns(&TableFileSchema::id,
&TableFileSchema::table_id, &TableFileSchema::table_id,
@ -619,21 +619,21 @@ Status DBMetaImpl::cleanup() {
c(&TableFileSchema::file_type) == (int)TableFileSchema::NEW)); c(&TableFileSchema::file_type) == (int)TableFileSchema::NEW));
TableFilesSchema updated; TableFilesSchema updated;
TableFileSchema table_file;
for (auto& file : selected) { for (auto& file : selected) {
TableFileSchema group_file; table_file.id = std::get<0>(file);
group_file.id = std::get<0>(file); table_file.table_id = std::get<1>(file);
group_file.table_id = std::get<1>(file); table_file.file_id = std::get<2>(file);
group_file.file_id = std::get<2>(file); table_file.file_type = std::get<3>(file);
group_file.file_type = std::get<3>(file); table_file.size = std::get<4>(file);
group_file.size = std::get<4>(file); table_file.date = std::get<5>(file);
group_file.date = std::get<5>(file); GetGroupFilePath(table_file);
GetGroupFilePath(group_file); if (table_file.file_type == TableFileSchema::TO_DELETE) {
if (group_file.file_type == TableFileSchema::TO_DELETE) { boost::filesystem::remove(table_file.location);
boost::filesystem::remove(group_file.location);
} }
ConnectorPtr->remove<TableFileSchema>(group_file.id); ConnectorPtr->remove<TableFileSchema>(table_file.id);
/* LOG(DEBUG) << "Removing id=" << group_file.id << " location=" << group_file.location << std::endl; */ /* LOG(DEBUG) << "Removing id=" << table_file.id << " location=" << table_file.location << std::endl; */
} }
} catch (std::exception & e) { } catch (std::exception & e) {
LOG(DEBUG) << e.what(); LOG(DEBUG) << e.what();
@ -643,7 +643,7 @@ Status DBMetaImpl::cleanup() {
return Status::OK(); return Status::OK();
} }
Status DBMetaImpl::count(const std::string& table_id, long& result) { Status DBMetaImpl::Count(const std::string& table_id, long& result) {
try { try {
auto selected = ConnectorPtr->select(columns(&TableFileSchema::size, auto selected = ConnectorPtr->select(columns(&TableFileSchema::size,
@ -674,7 +674,7 @@ Status DBMetaImpl::count(const std::string& table_id, long& result) {
return Status::OK(); return Status::OK();
} }
Status DBMetaImpl::drop_all() { Status DBMetaImpl::DropAll() {
if (boost::filesystem::is_directory(_options.path)) { if (boost::filesystem::is_directory(_options.path)) {
boost::filesystem::remove_all(_options.path); boost::filesystem::remove_all(_options.path);
} }
@ -682,7 +682,7 @@ Status DBMetaImpl::drop_all() {
} }
DBMetaImpl::~DBMetaImpl() { DBMetaImpl::~DBMetaImpl() {
cleanup(); CleanUp();
} }
} // namespace meta } // namespace meta

8
cpp/src/db/DBMetaImpl.h
View File

@ -46,13 +46,13 @@ public:
virtual Status Size(long& result) override; virtual Status Size(long& result) override;
virtual Status cleanup() override; virtual Status CleanUp() override;
virtual Status cleanup_ttl_files(uint16_t seconds) override; virtual Status CleanUpFilesWithTTL(uint16_t seconds) override;
virtual Status drop_all() override; virtual Status DropAll() override;
virtual Status count(const std::string& table_id, long& result) override; virtual Status Count(const std::string& table_id, long& result) override;
virtual ~DBMetaImpl(); virtual ~DBMetaImpl();

8
cpp/src/db/Meta.h
View File

@ -48,12 +48,12 @@ public:
virtual Status FilesToIndex(TableFilesSchema&) = 0; virtual Status FilesToIndex(TableFilesSchema&) = 0;
virtual Status cleanup() = 0; virtual Status CleanUp() = 0;
virtual Status cleanup_ttl_files(uint16_t) = 0; virtual Status CleanUpFilesWithTTL(uint16_t) = 0;
virtual Status drop_all() = 0; virtual Status DropAll() = 0;
virtual Status count(const std::string& table_id, long& result) = 0; virtual Status Count(const std::string& table_id, long& result) = 0;
static DateT GetDate(const std::time_t& t, int day_delta = 0); static DateT GetDate(const std::time_t& t, int day_delta = 0);
static DateT GetDate(); static DateT GetDate();

4
cpp/unittest/db/db_tests.cpp
View File

@ -152,12 +152,12 @@ TEST_F(DBTest, DB_TEST) {
for (auto j=0; j<10; ++j) { for (auto j=0; j<10; ++j) {
ss.str(""); ss.str("");
db_->count(group_name, count); db_->size(count);
prev_count = count; prev_count = count;
START_TIMER; START_TIMER;
stat = db_->search(group_name, k, qb, qxb, results); stat = db_->search(group_name, k, qb, qxb, results);
ss << "Search " << j << " With Size " << (float)(count*group_dim*sizeof(float))/engine::meta::M << " M"; ss << "Search " << j << " With Size " << count/engine::meta::M << " M";
STOP_TIMER(ss.str()); STOP_TIMER(ss.str());
ASSERT_STATS(stat); ASSERT_STATS(stat);

4
cpp/unittest/db/meta_tests.cpp
View File

@ -137,7 +137,7 @@ TEST_F(MetaTest, ARCHIVE_TEST_DAYS) {
i++; i++;
} }
impl.drop_all(); impl.DropAll();
} }
TEST_F(MetaTest, ARCHIVE_TEST_DISK) { TEST_F(MetaTest, ARCHIVE_TEST_DISK) {
@ -180,7 +180,7 @@ TEST_F(MetaTest, ARCHIVE_TEST_DISK) {
++i; ++i;
} }
impl.drop_all(); impl.DropAll();
} }
TEST_F(MetaTest, TABLE_FILES_TEST) { TEST_F(MetaTest, TABLE_FILES_TEST) {

2
cpp/unittest/db/utils.cpp
View File

@ -59,5 +59,5 @@ void MetaTest::SetUp() {
} }
void MetaTest::TearDown() { void MetaTest::TearDown() {
impl_->drop_all(); impl_->DropAll();
} }