refactor(db): delete local meta impl and refactor HasTable

Former-commit-id: 4849b994d40cf433797892eb37ceeac46fc65d3d

cpp/src/db/DBImpl.cpp

@@ -51,8 +51,8 @@ Status DBImpl<EngineT>::delete_vectors(const std::string& table_id,
 }
 
 template<typename EngineT>
-Status DBImpl<EngineT>::has_group(const std::string& table_id_, bool& has_or_not_) {
+Status DBImpl<EngineT>::has_group(const std::string& table_id, bool& has_or_not) {
-    return _pMeta->has_group(table_id_, has_or_not_);
+    return _pMeta->HasTable(table_id, has_or_not);
 }
 
 template<typename EngineT>

cpp/src/db/DBMetaImpl.cpp

@@ -201,12 +201,12 @@ Status DBMetaImpl::DescribeTable(TableSchema& table_schema) {
     return Status::OK();
 }
 
-Status DBMetaImpl::has_group(const std::string& table_id, bool& has_or_not) {
+Status DBMetaImpl::HasTable(const std::string& table_id, bool& has_or_not) {
     try {
-        auto groups = ConnectorPtr->select(columns(&TableSchema::id),
+        auto tables = ConnectorPtr->select(columns(&TableSchema::id),
                                           where(c(&TableSchema::table_id) == table_id));
-        assert(groups.size() <= 1);
+        assert(tables.size() <= 1);
-        if (groups.size() == 1) {
+        if (tables.size() == 1) {
             has_or_not = true;
         } else {
             has_or_not = false;

cpp/src/db/DBMetaImpl.h

@@ -21,7 +21,7 @@ public:
 
     virtual Status CreateTable(TableSchema& table_schema) override;
     virtual Status DescribeTable(TableSchema& group_info_) override;
-    virtual Status has_group(const std::string& table_id_, bool& has_or_not_) override;
+    virtual Status HasTable(const std::string& table_id, bool& has_or_not) override;
 
     virtual Status add_group_file(TableFileSchema& group_file_info) override;
     virtual Status delete_group_partitions(const std::string& table_id,

cpp/src/db/LocalMetaImpl.cpp

@@ -1,277 +0,0 @@
-////////////////////////////////////////////////////////////////////////////////
-// Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved
-// Unauthorized copying of this file, via any medium is strictly prohibited.
-// Proprietary and confidential.
-////////////////////////////////////////////////////////////////////////////////
-#include <sys/stat.h>
-#include <unistd.h>
-#include <sstream>
-#include <iostream>
-#include <boost/filesystem.hpp>
-#include <boost/property_tree/ptree.hpp>
-#include <boost/property_tree/json_parser.hpp>
-
-#include <fstream>
-#include "LocalMetaImpl.h"
-#include "IDGenerator.h"
-
-namespace zilliz {
-namespace vecwise {
-namespace engine {
-namespace meta {
-
-long LocalMetaImpl::GetFileSize(const std::string& filename)
-{
-    struct stat stat_buf;
-    int rc = stat(filename.c_str(), &stat_buf);
-    return rc == 0 ? stat_buf.st_size : -1;
-}
-
-std::string LocalMetaImpl::GetGroupPath(const std::string& table_id) {
-    return _options.path + "/" + table_id;
-}
-
-std::string LocalMetaImpl::GetGroupDatePartitionPath(const std::string& table_id, DateT& date) {
-    std::stringstream ss;
-    ss << GetGroupPath(table_id) << "/" << date;
-    return ss.str();
-}
-
-std::string LocalMetaImpl::GetNextGroupFileLocationByPartition(const std::string& table_id, DateT& date,
-        TableFileSchema::FILE_TYPE file_type) {
-    std::string suffix = (file_type == TableFileSchema::RAW) ? ".raw" : ".index";
-    SimpleIDGenerator g;
-    std::stringstream ss;
-    ss << GetGroupPath(table_id) << "/" << date << "/" << g.getNextIDNumber() << suffix;
-    return ss.str();
-}
-
-std::string LocalMetaImpl::GetGroupMetaPathByGroupPath(const std::string& group_path) {
-    return group_path + "/" + "meta";
-}
-
-std::string LocalMetaImpl::GetGroupMetaPath(const std::string& table_id) {
-    return GetGroupMetaPathByGroupPath(GetGroupPath(table_id));
-}
-
-Status LocalMetaImpl::GetGroupMetaInfoByPath(const std::string& path, TableSchema& table_schema) {
-    boost::property_tree::ptree ptree;
-    boost::property_tree::read_json(path, ptree);
-    auto files_cnt = ptree.get_child("files_cnt").data();
-    auto dimension = ptree.get_child("dimension").data();
-    /* std::cout << dimension << std::endl; */
-    /* std::cout << files_cnt << std::endl; */
-
-    table_schema.id = std::stoi(table_schema.table_id);
-    table_schema.files_cnt = std::stoi(files_cnt);
-    table_schema.dimension = std::stoi(dimension);
-    table_schema.location = GetGroupPath(table_schema.table_id);
-
-    return Status::OK();
-
-}
-
-Status LocalMetaImpl::GetGroupMetaInfo(const std::string& table_id, TableSchema& table_schema) {
-    table_schema.table_id = table_id;
-    return GetGroupMetaInfoByPath(GetGroupMetaPath(table_id), table_schema);
-}
-
-LocalMetaImpl::LocalMetaImpl(const DBMetaOptions& options_)
-    : _options(options_) {
-    initialize();
-}
-
-Status LocalMetaImpl::initialize() {
-    if (boost::filesystem::is_directory(_options.path)) {
-    }
-    else if (!boost::filesystem::create_directory(_options.path)) {
-        return Status::InvalidDBPath("Cannot Create " + _options.path);
-    }
-    return Status::OK();
-}
-
-Status LocalMetaImpl::CreateTable(TableSchema& table_schema) {
-    std::string real_gid;
-    size_t id = SimpleIDGenerator().getNextIDNumber();
-    if (table_schema.table_id == "") {
-        std::stringstream ss;
-        ss << id;
-        real_gid = ss.str();
-    } else {
-        real_gid = table_schema.table_id;
-    }
-
-    bool group_exist;
-    has_group(real_gid, group_exist);
-    if (group_exist) {
-        return Status::GroupError("Group Already Existed " + real_gid);
-    }
-    if (!boost::filesystem::create_directory(GetGroupPath(real_gid))) {
-        return Status::GroupError("Cannot Create Group " + real_gid);
-    }
-
-    table_schema.table_id = real_gid;
-    table_schema.files_cnt = 0;
-    table_schema.id = 0;
-    table_schema.location = GetGroupPath(real_gid);
-
-    boost::property_tree::ptree out;
-    out.put("files_cnt", table_schema.files_cnt);
-    out.put("dimension", table_schema.dimension);
-    boost::property_tree::write_json(GetGroupMetaPath(real_gid), out);
-
-    return Status::OK();
-}
-
-Status LocalMetaImpl::DescribeTable(TableSchema& table_schema) {
-    bool exist;
-    has_group(table_schema.table_id, exist);
-    if (!exist) {
-        return Status::NotFound("Table " + table_schema.table_id + " Not Found");
-    }
-
-    return GetGroupMetaInfo(table_schema.table_id, table_schema);
-}
-
-Status LocalMetaImpl::has_group(const std::string& table_id, bool& has_or_not) {
-    has_or_not = boost::filesystem::is_directory(GetGroupPath(table_id));
-    return Status::OK();
-}
-
-Status LocalMetaImpl::add_group_file(TableFileSchema& group_file_info) {
-    TableSchema table_schema;
-    /* auto status = get_group(table_schema); */
-    /* if (!status.ok()) { */
-    /*     return status; */
-    /* } */
-    /* auto location = GetNextGroupFileLocationByPartition(table_id, date, file_type); */
-    /* group_file_info.table_id = table_id; */
-    /* group_file_info.dimension = table_schema.dimension; */
-    /* group_file_info.location = location; */
-    /* group_file_info.date = date; */
-    return Status::OK();
-}
-
-Status LocalMetaImpl::files_to_index(TableFilesSchema& files) {
-    files.clear();
-
-    std::string suffix;
-    boost::filesystem::directory_iterator end_itr;
-    for (boost::filesystem::directory_iterator itr(_options.path); itr != end_itr; ++itr) {
-        auto group_path = itr->path().string();
-        TableSchema table_schema;
-        GetGroupMetaInfoByPath(GetGroupMetaPathByGroupPath(group_path), table_schema);
-        for (boost::filesystem::directory_iterator innerItr(group_path); innerItr != end_itr; ++innerItr) {
-            auto partition_path = innerItr->path().string();
-            for (boost::filesystem::directory_iterator fItr(partition_path); fItr != end_itr; ++fItr) {
-                auto location = fItr->path().string();
-                suffix = location.substr(location.find_last_of('.') + 1);
-                if (suffix == "index") continue;
-                if (INDEX_TRIGGER_SIZE >= GetFileSize(location)) continue;
-                std::cout << "[About to index] " << location << std::endl;
-                TableFileSchema f;
-                f.location = location;
-                /* f.table_id = table_id; */
-                f.dimension = table_schema.dimension;
-                files.push_back(f);
-            }
-        }
-    }
-
-    return Status::OK();
-}
-
-Status LocalMetaImpl::files_to_merge(const std::string& table_id,
-        DatePartionedTableFilesSchema& files) {
-    files.clear();
-    /* std::string suffix; */
-    /* boost::filesystem::directory_iterator end_itr; */
-    /* for (boost::filesystem::directory_iterator itr(_options.path); itr != end_itr; ++itr) { */
-    /*     auto group_path = itr->path().string(); */
-    /*     TableSchema table_schema; */
-    /*     GetGroupMetaInfoByPath(GetGroupMetaPathByGroupPath(group_path), table_schema); */
-    /*     for (boost::filesystem::directory_iterator innerItr(group_path); innerItr != end_itr; ++innerItr) { */
-    /*         auto partition_path = innerItr->path().string(); */
-    /*         for (boost::filesystem::directory_iterator fItr(partition_path); fItr != end_itr; ++fItr) { */
-    /*             auto location = fItr->path().string(); */
-    /*             suffix = location.substr(location.find_last_of('.') + 1); */
-    /*             if (suffix == "index") continue; */
-    /*             if (INDEX_TRIGGER_SIZE < GetFileSize(location)) continue; */
-    /*             std::cout << "[About to index] " << location << std::endl; */
-    /*             TableFileSchema f; */
-    /*             f.location = location; */
-    /*             f.table_id = table_id; */
-    /*             f.dimension = table_schema.dimension; */
-    /*             files.push_back(f); */
-    /*         } */
-    /*     } */
-    /* } */
-
-    return Status::OK();
-}
-
-Status LocalMetaImpl::has_group_file(const std::string& table_id_,
-                              const std::string& file_id_,
-                              bool& has_or_not_) {
-    //PXU TODO
-    return Status::OK();
-}
-
-Status LocalMetaImpl::get_group_file(const std::string& table_id_,
-                              const std::string& file_id_,
-                              TableFileSchema& group_file_info_) {
-    //PXU TODO
-    return Status::OK();
-}
-
-Status LocalMetaImpl::get_group_files(const std::string& table_id_,
-                               const int date_delta_,
-                               TableFilesSchema& group_files_info_) {
-    // PXU TODO
-    return Status::OK();
-}
-
-Status LocalMetaImpl::update_group_file(TableFileSchema& group_file_) {
-    //PXU TODO
-    return Status::OK();
-}
-
-Status LocalMetaImpl::update_files(TableFilesSchema& files) {
-    //PXU TODO
-    return Status::OK();
-}
-
-Status LocalMetaImpl::archive_files() {
-    //PXU TODO
-    return Status::OK();
-}
-
-Status LocalMetaImpl::cleanup() {
-    //PXU TODO
-    return Status::OK();
-}
-
-Status LocalMetaImpl::cleanup_ttl_files(uint16_t seconds) {
-    // PXU TODO
-    return Status::OK();
-}
-
-Status LocalMetaImpl::drop_all() {
-    // PXU TODO
-    return Status::OK();
-}
-
-Status LocalMetaImpl::size(long& result) {
-    // PXU TODO
-    return Status::OK();
-}
-
-Status LocalMetaImpl::count(const std::string& table_id, long& result) {
-    // PXU TODO
-    return Status::OK();
-}
-
-} // namespace meta
-} // namespace engine
-} // namespace vecwise
-} // namespace zilliz

cpp/src/db/LocalMetaImpl.h

@@ -1,83 +0,0 @@
-////////////////////////////////////////////////////////////////////////////////
-// Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved
-// Unauthorized copying of this file, via any medium is strictly prohibited.
-// Proprietary and confidential.
-////////////////////////////////////////////////////////////////////////////////
-#pragma once
-
-#include "Meta.h"
-#include "Options.h"
-
-namespace zilliz {
-namespace vecwise {
-namespace engine {
-namespace meta {
-
-class LocalMetaImpl : public Meta {
-public:
-    const size_t INDEX_TRIGGER_SIZE = 1024*1024*500;
-    LocalMetaImpl(const DBMetaOptions& options_);
-
-    virtual Status CreateTable(TableSchema& table_schema) override;
-    virtual Status DescribeTable(TableSchema& table_schema_) override;
-    virtual Status has_group(const std::string& table_id_, bool& has_or_not_) override;
-
-    virtual Status add_group_file(TableFileSchema& group_file_info) override;
-    /* virtual Status delete_group_partitions(const std::string& table_id, */
-    /*         const meta::DatesT& dates) override; */
-
-    virtual Status has_group_file(const std::string& table_id_,
-                                  const std::string& file_id_,
-                                  bool& has_or_not_) override;
-    virtual Status get_group_file(const std::string& table_id_,
-                                  const std::string& file_id_,
-                                  TableFileSchema& group_file_info_) override;
-    virtual Status update_group_file(TableFileSchema& group_file_) override;
-
-    virtual Status get_group_files(const std::string& table_id_,
-                                   const int date_delta_,
-                                   TableFilesSchema& group_files_info_) override;
-
-    virtual Status update_files(TableFilesSchema& files) override;
-
-    virtual Status cleanup() override;
-
-    virtual Status files_to_merge(const std::string& table_id,
-            DatePartionedTableFilesSchema& files) override;
-
-    virtual Status files_to_index(TableFilesSchema&) override;
-
-    virtual Status archive_files() override;
-
-    virtual Status cleanup_ttl_files(uint16_t seconds) override;
-
-    virtual Status count(const std::string& table_id, long& result) override;
-
-    virtual Status drop_all() override;
-
-    virtual Status size(long& result) override;
-
-private:
-
-    Status GetGroupMetaInfoByPath(const std::string& path, TableSchema& table_schema);
-    std::string GetGroupMetaPathByGroupPath(const std::string& group_path);
-    Status GetGroupMetaInfo(const std::string& table_id, TableSchema& table_schema);
-    std::string GetNextGroupFileLocationByPartition(const std::string& table_id, DateT& date,
-        TableFileSchema::FILE_TYPE file_type);
-    std::string GetGroupDatePartitionPath(const std::string& table_id, DateT& date);
-    std::string GetGroupPath(const std::string& table_id);
-    std::string GetGroupMetaPath(const std::string& table_id);
-
-    Status CreateGroupMeta(const TableSchema& group_schema);
-    long GetFileSize(const std::string& filename);
-
-    Status initialize();
-
-    const DBMetaOptions _options;
-
-}; // LocalMetaImpl
-
-} // namespace meta
-} // namespace engine
-} // namespace vecwise
-} // namespace zilliz

cpp/src/db/Meta.h

@@ -24,7 +24,7 @@ public:
 
     virtual Status CreateTable(TableSchema& table_schema) = 0;
     virtual Status DescribeTable(TableSchema& table_schema) = 0;
-    virtual Status has_group(const std::string& table_id_, bool& has_or_not_) = 0;
+    virtual Status HasTable(const std::string& table_id, bool& has_or_not) = 0;
 
     virtual Status add_group_file(TableFileSchema& group_file_info) = 0;
     virtual Status delete_group_partitions(const std::string& table_id,