milvus-io
/
milvus
mirror of https://github.com/milvus-io/milvus.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

(db/snapshot): retry remove if remove_all fails due to bug in filesystem remove_all (#3187) 

Signed-off-by: peng.xu <peng.xu@zilliz.com>
pull/3186/head^2
XuPeng-SH 2020-08-09 23:19:30 +08:00 committed by GitHub
parent d37e0a39e5
commit c0f77a7e47
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 42 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

55
core/src/db/snapshot/ResourceGCEvent.h
View File

@ -43,21 +43,50 @@ class ResourceGCEvent : public GCEvent {
/* TODO: physically clean resource */
auto res_prefix = store->GetRootPath();
std::string res_path = GetResPath<ResourceT>(res_prefix, res_);
if (res_path.empty()) {
/* std::cout << "[GC] No remove action for " << res_->ToString() << std::endl; */
} else if (std::experimental::filesystem::is_directory(res_path)) {
auto ok = std::experimental::filesystem::remove_all(res_path);
std::cout << "[GC] Remove DIR " << res_->ToString() << " " << res_path << " " << ok << std::endl;
} else if (std::experimental::filesystem::is_regular_file(res_path)) {
auto ok = std::experimental::filesystem::remove(res_path);
std::cout << "[GC] Remove FILE " << res_->ToString() << " " << res_path << " " << ok << std::endl;
} else {
RemoveWithSuffix<ResourceT>(res_, res_path, store->GetSuffixSet());
auto do_remove = [&](bool do_throw) -> Status {
try {
if (res_path.empty()) {
/* std::cout << "[GC] No remove action for " << res_->ToString() << std::endl; */
} else if (std::experimental::filesystem::is_directory(res_path)) {
std::cout << "[GC] Remove DIR " << res_->ToString() << " " << res_path << std::endl;
auto ok = std::experimental::filesystem::remove_all(res_path);
} else if (std::experimental::filesystem::is_regular_file(res_path)) {
std::cout << "[GC] Remove FILE " << res_->ToString() << " " << res_path << std::endl;
auto ok = std::experimental::filesystem::remove(res_path);
} else {
RemoveWithSuffix<ResourceT>(res_, res_path, store->GetSuffixSet());
}
} catch (const std::experimental::filesystem::filesystem_error& er) {
std::cout << "[GC] " << er.what() << std::endl;
if (do_throw) {
throw;
}
return Status(DB_ERROR, er.what());
}
return Status::OK();
};
auto retry = 3;
Status status;
while (--retry >= 0) {
status = do_remove(false);
if (status.ok()) {
break;
}
std::this_thread::sleep_for(std::chrono::milliseconds(1));
std::cout << "[GC] Retry remove: " << res_path << std::endl;
}
/* remove resource from meta */
auto hd_op = std::make_shared<HardDeleteOperation<ResourceT>>(res_->GetID());
STATUS_CHECK((*hd_op)(store));
auto hard_delete_meta = [](ID_TYPE id, StorePtr store) -> Status {
auto hd_op = std::make_shared<HardDeleteOperation<ResourceT>>(id);
return ((*hd_op)(store));
};
if (!status.ok()) {
std::cout << "[GC] Stale Resource: " << res_path << " need to be cleanup later" << std::endl;
return status;
}
STATUS_CHECK(hard_delete_meta(res_->GetID(), store));
return Status::OK();
}