feat(db): mock build_index related

Former-commit-id: a4908e8eaf9b8ba57d873e0f9b1128980edbf3ff

cpp/src/db/DBImpl.cpp

@@ -166,14 +166,14 @@ Status DBImpl::background_merge_files(const std::string& group_id) {
 
 Status DBImpl::build_index(const meta::GroupFileSchema& file) {
     //PXU TODO
+    std::cout << ">>Building Index for: " << file.location << std::endl;
     return Status::OK();
 }
 
 Status DBImpl::background_build_index() {
     assert(bg_build_index_started_);
     meta::GroupFilesSchema to_index_files;
-    // PXU TODO
-    /* _pMeta->files_to_index(to_index_files); */
+    _pMeta->files_to_index(to_index_files);
     Status status;
     for (auto& file : to_index_files) {
         status = build_index(file);

cpp/src/db/DBMetaImpl.cpp

@@ -1,6 +1,9 @@
+#include <sys/stat.h>
+#include <unistd.h>
 #include <sstream>
 #include <iostream>
 #include <boost/filesystem.hpp>
+#include <fstream>
 #include "DBMetaImpl.h"
 #include "IDGenerator.h"
 
@@ -9,6 +12,13 @@ namespace vecwise {
 namespace engine {
 namespace meta {
 
+long GetFileSize(const std::string& filename)
+{
+    struct stat stat_buf;
+    int rc = stat(filename.c_str(), &stat_buf);
+    return rc == 0 ? stat_buf.st_size : -1;
+}
+
 DBMetaImpl::DBMetaImpl(const MetaOptions& options_)
     : _options(static_cast<const DBMetaOptions&>(options_)) {
     initialize();
@@ -57,6 +67,24 @@ Status DBMetaImpl::add_group_file(const std::string& group_id,
     return Status::OK();
 }
 
+Status DBMetaImpl::files_to_index(GroupFilesSchema& files) {
+    // PXU TODO
+    files.clear();
+    std::stringstream ss;
+    ss << "/tmp/test/" << Meta::GetDate();
+    boost::filesystem::path path(ss.str().c_str());
+    boost::filesystem::directory_iterator end_itr;
+    for (boost::filesystem::directory_iterator itr(path); itr != end_itr; ++itr) {
+        std::cout << itr->path().string() << std::endl;
+        GroupFileSchema f;
+        f.location = itr->path().string();
+        if (1024*1024*50 >= GetFileSize(f.location)) continue;
+        std::cout << "About to index " << f.location << std::endl;
+        files.push_back(f);
+    }
+    return Status::OK();
+}
+
 Status DBMetaImpl::files_to_merge(const std::string& group_id,
         DatePartionedGroupFilesSchema& files) {
     //PXU TODO
@@ -72,6 +100,8 @@ Status DBMetaImpl::files_to_merge(const std::string& group_id,
         std::cout << itr->path().string() << std::endl;
         GroupFileSchema f;
         f.location = itr->path().string();
+        if (1024*1024*50 < GetFileSize(f.location)) continue;
+        std::cout << "About to merge " << f.location << std::endl;
         files[date].push_back(f);
     }
 

cpp/src/db/DBMetaImpl.h

@@ -41,6 +41,8 @@ public:
     virtual Status files_to_merge(const std::string& group_id,
             DatePartionedGroupFilesSchema& files) override;
 
+    virtual Status files_to_index(GroupFilesSchema&) override;
+
 private:
 
     Status initialize();

cpp/src/db/Meta.h

@@ -77,6 +77,8 @@ public:
     virtual Status files_to_merge(const std::string& group_id,
             DatePartionedGroupFilesSchema& files) = 0;
 
+    virtual Status files_to_index(GroupFilesSchema&) = 0;
+
     static DateT GetDate(const std::time_t& t);
     static DateT GetDate();