milvus-io
/
milvus
mirror of https://github.com/milvus-io/milvus.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

refactor(db): delete local meta impl and refactor HasTable 

Former-commit-id: 4849b994d40cf433797892eb37ceeac46fc65d3d
pull/191/head
Xu Peng 2019-05-27 18:02:22 +08:00
parent 624e34f10d
commit 9d4591612d
6 changed files with 8 additions and 368 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
cpp/src/db/DBImpl.cpp
View File

@ -51,8 +51,8 @@ Status DBImpl<EngineT>::delete_vectors(const std::string& table_id,
}
template<typename EngineT>
Status DBImpl<EngineT>::has_group(const std::string& table_id_, bool& has_or_not_) {
return _pMeta->has_group(table_id_, has_or_not_);
Status DBImpl<EngineT>::has_group(const std::string& table_id, bool& has_or_not) {
return _pMeta->HasTable(table_id, has_or_not);
}
template<typename EngineT>

8
cpp/src/db/DBMetaImpl.cpp
View File

@ -201,12 +201,12 @@ Status DBMetaImpl::DescribeTable(TableSchema& table_schema) {
return Status::OK();
}
Status DBMetaImpl::has_group(const std::string& table_id, bool& has_or_not) {
Status DBMetaImpl::HasTable(const std::string& table_id, bool& has_or_not) {
try {
auto groups = ConnectorPtr->select(columns(&TableSchema::id),
auto tables = ConnectorPtr->select(columns(&TableSchema::id),
where(c(&TableSchema::table_id) == table_id));
assert(groups.size() <= 1);
if (groups.size() == 1) {
assert(tables.size() <= 1);
if (tables.size() == 1) {
has_or_not = true;
} else {
has_or_not = false;

2
cpp/src/db/DBMetaImpl.h
View File

@ -21,7 +21,7 @@ public:
virtual Status CreateTable(TableSchema& table_schema) override;
virtual Status DescribeTable(TableSchema& group_info_) override;
virtual Status has_group(const std::string& table_id_, bool& has_or_not_) override;
virtual Status HasTable(const std::string& table_id, bool& has_or_not) override;
virtual Status add_group_file(TableFileSchema& group_file_info) override;
virtual Status delete_group_partitions(const std::string& table_id,

277
cpp/src/db/LocalMetaImpl.cpp
View File

@ -1,277 +0,0 @@
////////////////////////////////////////////////////////////////////////////////
// Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved
// Unauthorized copying of this file, via any medium is strictly prohibited.
// Proprietary and confidential.
////////////////////////////////////////////////////////////////////////////////
#include <sys/stat.h>
#include <unistd.h>
#include <sstream>
#include <iostream>
#include <boost/filesystem.hpp>
#include <boost/property_tree/ptree.hpp>
#include <boost/property_tree/json_parser.hpp>
#include <fstream>
#include "LocalMetaImpl.h"
#include "IDGenerator.h"
namespace zilliz {
namespace vecwise {
namespace engine {
namespace meta {
long LocalMetaImpl::GetFileSize(const std::string& filename)
{
struct stat stat_buf;
int rc = stat(filename.c_str(), &stat_buf);
return rc == 0 ? stat_buf.st_size : -1;
}
std::string LocalMetaImpl::GetGroupPath(const std::string& table_id) {
return _options.path + "/" + table_id;
}
std::string LocalMetaImpl::GetGroupDatePartitionPath(const std::string& table_id, DateT& date) {
std::stringstream ss;
ss << GetGroupPath(table_id) << "/" << date;
return ss.str();
}
std::string LocalMetaImpl::GetNextGroupFileLocationByPartition(const std::string& table_id, DateT& date,
TableFileSchema::FILE_TYPE file_type) {
std::string suffix = (file_type == TableFileSchema::RAW) ? ".raw" : ".index";
SimpleIDGenerator g;
std::stringstream ss;
ss << GetGroupPath(table_id) << "/" << date << "/" << g.getNextIDNumber() << suffix;
return ss.str();
}
std::string LocalMetaImpl::GetGroupMetaPathByGroupPath(const std::string& group_path) {
return group_path + "/" + "meta";
}
std::string LocalMetaImpl::GetGroupMetaPath(const std::string& table_id) {
return GetGroupMetaPathByGroupPath(GetGroupPath(table_id));
}
Status LocalMetaImpl::GetGroupMetaInfoByPath(const std::string& path, TableSchema& table_schema) {
boost::property_tree::ptree ptree;
boost::property_tree::read_json(path, ptree);
auto files_cnt = ptree.get_child("files_cnt").data();
auto dimension = ptree.get_child("dimension").data();
/* std::cout << dimension << std::endl; */
/* std::cout << files_cnt << std::endl; */
table_schema.id = std::stoi(table_schema.table_id);
table_schema.files_cnt = std::stoi(files_cnt);
table_schema.dimension = std::stoi(dimension);
table_schema.location = GetGroupPath(table_schema.table_id);
return Status::OK();
}
Status LocalMetaImpl::GetGroupMetaInfo(const std::string& table_id, TableSchema& table_schema) {
table_schema.table_id = table_id;
return GetGroupMetaInfoByPath(GetGroupMetaPath(table_id), table_schema);
}
LocalMetaImpl::LocalMetaImpl(const DBMetaOptions& options_)
: _options(options_) {
initialize();
}
Status LocalMetaImpl::initialize() {
if (boost::filesystem::is_directory(_options.path)) {
}
else if (!boost::filesystem::create_directory(_options.path)) {
return Status::InvalidDBPath("Cannot Create " + _options.path);
}
return Status::OK();
}
Status LocalMetaImpl::CreateTable(TableSchema& table_schema) {
std::string real_gid;
size_t id = SimpleIDGenerator().getNextIDNumber();
if (table_schema.table_id == "") {
std::stringstream ss;
ss << id;
real_gid = ss.str();
} else {
real_gid = table_schema.table_id;
}
bool group_exist;
has_group(real_gid, group_exist);
if (group_exist) {
return Status::GroupError("Group Already Existed " + real_gid);
}
if (!boost::filesystem::create_directory(GetGroupPath(real_gid))) {
return Status::GroupError("Cannot Create Group " + real_gid);
}
table_schema.table_id = real_gid;
table_schema.files_cnt = 0;
table_schema.id = 0;
table_schema.location = GetGroupPath(real_gid);
boost::property_tree::ptree out;
out.put("files_cnt", table_schema.files_cnt);
out.put("dimension", table_schema.dimension);
boost::property_tree::write_json(GetGroupMetaPath(real_gid), out);
return Status::OK();
}
Status LocalMetaImpl::DescribeTable(TableSchema& table_schema) {
bool exist;
has_group(table_schema.table_id, exist);
if (!exist) {
return Status::NotFound("Table " + table_schema.table_id + " Not Found");
}
return GetGroupMetaInfo(table_schema.table_id, table_schema);
}
Status LocalMetaImpl::has_group(const std::string& table_id, bool& has_or_not) {
has_or_not = boost::filesystem::is_directory(GetGroupPath(table_id));
return Status::OK();
}
Status LocalMetaImpl::add_group_file(TableFileSchema& group_file_info) {
TableSchema table_schema;
/* auto status = get_group(table_schema); */
/* if (!status.ok()) { */
/* return status; */
/* } */
/* auto location = GetNextGroupFileLocationByPartition(table_id, date, file_type); */
/* group_file_info.table_id = table_id; */
/* group_file_info.dimension = table_schema.dimension; */
/* group_file_info.location = location; */
/* group_file_info.date = date; */
return Status::OK();
}
Status LocalMetaImpl::files_to_index(TableFilesSchema& files) {
files.clear();
std::string suffix;
boost::filesystem::directory_iterator end_itr;
for (boost::filesystem::directory_iterator itr(_options.path); itr != end_itr; ++itr) {
auto group_path = itr->path().string();
TableSchema table_schema;
GetGroupMetaInfoByPath(GetGroupMetaPathByGroupPath(group_path), table_schema);
for (boost::filesystem::directory_iterator innerItr(group_path); innerItr != end_itr; ++innerItr) {
auto partition_path = innerItr->path().string();
for (boost::filesystem::directory_iterator fItr(partition_path); fItr != end_itr; ++fItr) {
auto location = fItr->path().string();
suffix = location.substr(location.find_last_of('.') + 1);
if (suffix == "index") continue;
if (INDEX_TRIGGER_SIZE >= GetFileSize(location)) continue;
std::cout << "[About to index] " << location << std::endl;
TableFileSchema f;
f.location = location;
/* f.table_id = table_id; */
f.dimension = table_schema.dimension;
files.push_back(f);
}
}
}
return Status::OK();
}
Status LocalMetaImpl::files_to_merge(const std::string& table_id,
DatePartionedTableFilesSchema& files) {
files.clear();
/* std::string suffix; */
/* boost::filesystem::directory_iterator end_itr; */
/* for (boost::filesystem::directory_iterator itr(_options.path); itr != end_itr; ++itr) { */
/* auto group_path = itr->path().string(); */
/* TableSchema table_schema; */
/* GetGroupMetaInfoByPath(GetGroupMetaPathByGroupPath(group_path), table_schema); */
/* for (boost::filesystem::directory_iterator innerItr(group_path); innerItr != end_itr; ++innerItr) { */
/* auto partition_path = innerItr->path().string(); */
/* for (boost::filesystem::directory_iterator fItr(partition_path); fItr != end_itr; ++fItr) { */
/* auto location = fItr->path().string(); */
/* suffix = location.substr(location.find_last_of('.') + 1); */
/* if (suffix == "index") continue; */
/* if (INDEX_TRIGGER_SIZE < GetFileSize(location)) continue; */
/* std::cout << "[About to index] " << location << std::endl; */
/* TableFileSchema f; */
/* f.location = location; */
/* f.table_id = table_id; */
/* f.dimension = table_schema.dimension; */
/* files.push_back(f); */
/* } */
/* } */
/* } */
return Status::OK();
}
Status LocalMetaImpl::has_group_file(const std::string& table_id_,
const std::string& file_id_,
bool& has_or_not_) {
//PXU TODO
return Status::OK();
}
Status LocalMetaImpl::get_group_file(const std::string& table_id_,
const std::string& file_id_,
TableFileSchema& group_file_info_) {
//PXU TODO
return Status::OK();
}
Status LocalMetaImpl::get_group_files(const std::string& table_id_,
const int date_delta_,
TableFilesSchema& group_files_info_) {
// PXU TODO
return Status::OK();
}
Status LocalMetaImpl::update_group_file(TableFileSchema& group_file_) {
//PXU TODO
return Status::OK();
}
Status LocalMetaImpl::update_files(TableFilesSchema& files) {
//PXU TODO
return Status::OK();
}
Status LocalMetaImpl::archive_files() {
//PXU TODO
return Status::OK();
}
Status LocalMetaImpl::cleanup() {
//PXU TODO
return Status::OK();
}
Status LocalMetaImpl::cleanup_ttl_files(uint16_t seconds) {
// PXU TODO
return Status::OK();
}
Status LocalMetaImpl::drop_all() {
// PXU TODO
return Status::OK();
}
Status LocalMetaImpl::size(long& result) {
// PXU TODO
return Status::OK();
}
Status LocalMetaImpl::count(const std::string& table_id, long& result) {
// PXU TODO
return Status::OK();
}
} // namespace meta
} // namespace engine
} // namespace vecwise
} // namespace zilliz

83
cpp/src/db/LocalMetaImpl.h
View File

@ -1,83 +0,0 @@
////////////////////////////////////////////////////////////////////////////////
// Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved
// Unauthorized copying of this file, via any medium is strictly prohibited.
// Proprietary and confidential.
////////////////////////////////////////////////////////////////////////////////
#pragma once
#include "Meta.h"
#include "Options.h"
namespace zilliz {
namespace vecwise {
namespace engine {
namespace meta {
class LocalMetaImpl : public Meta {
public:
const size_t INDEX_TRIGGER_SIZE = 1024*1024*500;
LocalMetaImpl(const DBMetaOptions& options_);
virtual Status CreateTable(TableSchema& table_schema) override;
virtual Status DescribeTable(TableSchema& table_schema_) override;
virtual Status has_group(const std::string& table_id_, bool& has_or_not_) override;
virtual Status add_group_file(TableFileSchema& group_file_info) override;
/* virtual Status delete_group_partitions(const std::string& table_id, */
/* const meta::DatesT& dates) override; */
virtual Status has_group_file(const std::string& table_id_,
const std::string& file_id_,
bool& has_or_not_) override;
virtual Status get_group_file(const std::string& table_id_,
const std::string& file_id_,
TableFileSchema& group_file_info_) override;
virtual Status update_group_file(TableFileSchema& group_file_) override;
virtual Status get_group_files(const std::string& table_id_,
const int date_delta_,
TableFilesSchema& group_files_info_) override;
virtual Status update_files(TableFilesSchema& files) override;
virtual Status cleanup() override;
virtual Status files_to_merge(const std::string& table_id,
DatePartionedTableFilesSchema& files) override;
virtual Status files_to_index(TableFilesSchema&) override;
virtual Status archive_files() override;
virtual Status cleanup_ttl_files(uint16_t seconds) override;
virtual Status count(const std::string& table_id, long& result) override;
virtual Status drop_all() override;
virtual Status size(long& result) override;
private:
Status GetGroupMetaInfoByPath(const std::string& path, TableSchema& table_schema);
std::string GetGroupMetaPathByGroupPath(const std::string& group_path);
Status GetGroupMetaInfo(const std::string& table_id, TableSchema& table_schema);
std::string GetNextGroupFileLocationByPartition(const std::string& table_id, DateT& date,
TableFileSchema::FILE_TYPE file_type);
std::string GetGroupDatePartitionPath(const std::string& table_id, DateT& date);
std::string GetGroupPath(const std::string& table_id);
std::string GetGroupMetaPath(const std::string& table_id);
Status CreateGroupMeta(const TableSchema& group_schema);
long GetFileSize(const std::string& filename);
Status initialize();
const DBMetaOptions _options;
}; // LocalMetaImpl
} // namespace meta
} // namespace engine
} // namespace vecwise
} // namespace zilliz

2
cpp/src/db/Meta.h
View File

@ -24,7 +24,7 @@ public:
virtual Status CreateTable(TableSchema& table_schema) = 0;
virtual Status DescribeTable(TableSchema& table_schema) = 0;
virtual Status has_group(const std::string& table_id_, bool& has_or_not_) = 0;
virtual Status HasTable(const std::string& table_id, bool& has_or_not) = 0;
virtual Status add_group_file(TableFileSchema& group_file_info) = 0;
virtual Status delete_group_partitions(const std::string& table_id,