Merge branch 'branch-0.3.0' into 'branch-0.3.0'

fix build error

See merge request megasearch/vecwise_engine!139

Former-commit-id: 526da01ddcf4a660f08c3b0a2d0cde22624b6b00

cpp/conf/server_config.template

@@ -10,7 +10,7 @@ db_config:
   # URI format: dialect://username:password@host:port/database
   # All parts except dialect are optional, but you MUST include the delimiters
   # Currently dialect supports mysql or sqlite
-  db_backend_url: dialect://username:password@host:port/database # meta database uri
+  db_backend_url: sqlite://:@:/
 
   index_building_threshold: 1024   # index building trigger threshold, default: 1024, unit: MB
   archive_disk_threshold: 512      # triger archive action if storage size exceed this value, unit: GB

cpp/src/server/DBWrapper.cpp

@@ -70,9 +70,15 @@ DBWrapper::DBWrapper() {
         kill(0, SIGUSR1);
     }
 
-    zilliz::milvus::engine::DB::Open(opt, &db_);
+    std::string msg = opt.meta.path;
+    try {
+        zilliz::milvus::engine::DB::Open(opt, &db_);
+    } catch(std::exception& ex) {
+        msg = ex.what();
+    }
+
     if(db_ == nullptr) {
-        std::cout << "ERROR! Failed to open database" << std::endl;
+        std::cout << "ERROR! Failed to open database: " << msg << std::endl;
         kill(0, SIGUSR1);
     }
 }

cpp/unittest/db/mysql_meta_test.cpp

@@ -32,12 +32,19 @@ using namespace zilliz::milvus::engine;
 //}
 
 TEST_F(MySQLTest, core) {
-//    DBMetaOptions options;
+    DBMetaOptions options;
 //    //dialect+driver://username:password@host:port/database
 //    options.backend_uri = "mysql://root:1234@:/test";
 //    options.path = "/tmp/vecwise_test";
+    try {
+        options = getDBMetaOptions();
+    } catch(std::exception& ex) {
+        ASSERT_TRUE(false);
+        return;
+    }
+
     int mode = Options::MODE::SINGLE;
-    meta::MySQLMetaImpl impl(getDBMetaOptions(), mode);
+    meta::MySQLMetaImpl impl(options, mode);
 //    auto status = impl.Initialize();
 //    ASSERT_TRUE(status.ok());
 
@@ -192,9 +199,16 @@ TEST_F(MySQLTest, core) {
 }
 
 TEST_F(MySQLTest, GROUP_TEST) {
+    DBMetaOptions options;
+    try {
+        options = getDBMetaOptions();
+    } catch(std::exception& ex) {
+        ASSERT_TRUE(false);
+        return;
+    }
 
     int mode = Options::MODE::SINGLE;
-    meta::MySQLMetaImpl impl(getDBMetaOptions(), mode);
+    meta::MySQLMetaImpl impl(options, mode);
 
     auto table_id = "meta_test_group";
 
@@ -228,9 +242,16 @@ TEST_F(MySQLTest, GROUP_TEST) {
 }
 
 TEST_F(MySQLTest, table_file_TEST) {
+    DBMetaOptions options;
+    try {
+        options = getDBMetaOptions();
+    } catch(std::exception& ex) {
+        ASSERT_TRUE(false);
+        return;
+    }
 
     int mode = Options::MODE::SINGLE;
-    meta::MySQLMetaImpl impl(getDBMetaOptions(), mode);
+    meta::MySQLMetaImpl impl(options, mode);
 
     auto table_id = "meta_test_group";
 
@@ -296,7 +317,14 @@ TEST_F(MySQLTest, table_file_TEST) {
 
 TEST_F(MySQLTest, ARCHIVE_TEST_DAYS) {
     srand(time(0));
-    DBMetaOptions options = getDBMetaOptions();
+    DBMetaOptions options;
+    try {
+        options = getDBMetaOptions();
+    } catch(std::exception& ex) {
+        ASSERT_TRUE(false);
+        return;
+    }
+
     int days_num = rand() % 100;
     std::stringstream ss;
     ss << "days:" << days_num;
@@ -350,7 +378,14 @@ TEST_F(MySQLTest, ARCHIVE_TEST_DAYS) {
 }
 
 TEST_F(MySQLTest, ARCHIVE_TEST_DISK) {
-    DBMetaOptions options = getDBMetaOptions();
+    DBMetaOptions options;
+    try {
+        options = getDBMetaOptions();
+    } catch(std::exception& ex) {
+        ASSERT_TRUE(false);
+        return;
+    }
+
     options.archive_conf = ArchiveConf("delete", "disk:11");
     int mode = Options::MODE::SINGLE;
     auto impl = meta::MySQLMetaImpl(options, mode);
@@ -397,9 +432,16 @@ TEST_F(MySQLTest, ARCHIVE_TEST_DISK) {
 }
 
 TEST_F(MySQLTest, TABLE_FILES_TEST) {
+    DBMetaOptions options;
+    try {
+        options = getDBMetaOptions();
+    } catch(std::exception& ex) {
+        ASSERT_TRUE(false);
+        return;
+    }
 
     int mode = Options::MODE::SINGLE;
-    auto impl = meta::MySQLMetaImpl(getDBMetaOptions(), mode);
+    auto impl = meta::MySQLMetaImpl(options, mode);
 
     auto table_id = "meta_test_group";
 
@@ -460,6 +502,11 @@ TEST_F(MySQLTest, TABLE_FILES_TEST) {
     ASSERT_EQ(dated_files[table_file.date_].size(),
               to_index_files_cnt+raw_files_cnt+index_files_cnt);
 
+    status = impl.FilesToSearch(table_id, meta::DatesT(), dated_files);
+    ASSERT_TRUE(status.ok());
+    ASSERT_EQ(dated_files[table_file.date_].size(),
+              to_index_files_cnt+raw_files_cnt+index_files_cnt);
+
     status = impl.DropAll();
     ASSERT_TRUE(status.ok());
 }

cpp/unittest/db/utils.cpp

@@ -91,6 +91,11 @@ zilliz::milvus::engine::DBMetaOptions MySQLTest::getDBMetaOptions() {
     zilliz::milvus::engine::DBMetaOptions options;
     options.path = "/tmp/milvus_test";
     options.backend_uri = DBTestEnvironment::getURI();
+    
+    if(options.backend_uri.empty()) {
+        throw std::exception();
+    }
+
     return options;
 }
 

cpp/unittest/metrics/CMakeLists.txt

@@ -17,6 +17,7 @@ aux_source_directory(../../src/config config_files)
 aux_source_directory(../../src/cache cache_srcs)
 aux_source_directory(../../src/wrapper wrapper_src)
 aux_source_directory(../../src/metrics metrics_src)
+aux_source_directory(./ test_srcs)
 
 aux_source_directory(${MILVUS_ENGINE_SRC}/db/scheduler scheduler_files)
 aux_source_directory(${MILVUS_ENGINE_SRC}/db/scheduler/context scheduler_context_files)
@@ -35,6 +36,7 @@ link_directories("/usr/local/cuda/lib64")
 #include_directories(../db/utils.h)
 include_directories(../../src/metrics)
 
+include_directories(/usr/include/mysql)
 
 #set(metrics_src_files
 #        ../../src/metrics/Metrics.cpp
@@ -47,17 +49,14 @@ include_directories(../../src/metrics)
 #        )
 
 set(count_test_src
-        ${unittest_srcs}
         ${config_files}
         ${cache_srcs}
         ${db_srcs}
         ${db_scheduler_srcs}
         ${wrapper_src}
         ${metrics_src}
-        metrics_test.cpp
-        prometheus_test.cpp
-        ../db/utils.cpp
-        metricbase_test.cpp)
+        ${test_srcs}
+        )
 
 
 add_executable(metrics_test ${count_test_src} ${require_files} )
@@ -75,6 +74,7 @@ target_link_libraries(metrics_test
         gtest
         pthread
         z
+        mysqlpp
         ${unittest_libs}
         )
 

cpp/unittest/metrics/metricbase_test.cpp

@@ -11,7 +11,7 @@
 
 using namespace zilliz::milvus;
 
-TEST(MetricbaseTest, Metricbase_Test){
+TEST(MetricbaseTest, METRICBASE_TEST){
     server::MetricsBase instance = server::MetricsBase::GetInstance();
     instance.Init();
     server::SystemInfo::GetInstance().Init();

cpp/unittest/metrics/metrics_test.cpp

@@ -15,7 +15,7 @@
 #include <cache/CpuCacheMgr.h>
 
 #include "metrics/Metrics.h"
-#include "../db/utils.h"
+#include "utils.h"
 #include "db/DB.h"
 #include "db/DBMetaImpl.h"
 #include "db/Factories.h"
@@ -24,7 +24,7 @@
 using namespace zilliz::milvus;
 
 
-TEST_F(DBTest, Metric_Tes) {
+TEST_F(MetricTest, Metric_Tes) {
 
     server::SystemInfo::GetInstance().Init();
 //    server::Metrics::GetInstance().Init();

cpp/unittest/metrics/prometheus_test.cpp

@@ -11,7 +11,7 @@
 
 using namespace zilliz::milvus;
 
-TEST(PrometheusTest, Prometheus_Test){
+TEST(PrometheusTest, PROMETHEUS_TEST){
     server::PrometheusMetrics instance = server::PrometheusMetrics::GetInstance();
     instance.Init();
     instance.SetStartup(true);

cpp/unittest/metrics/utils.cpp

@@ -0,0 +1,79 @@
+////////////////////////////////////////////////////////////////////////////////
+// Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved
+// Unauthorized copying of this file, via any medium is strictly prohibited.
+// Proprietary and confidential.
+////////////////////////////////////////////////////////////////////////////////
+
+#include <iostream>
+#include <easylogging++.h>
+#include <thread>
+#include <boost/filesystem.hpp>
+
+#include "utils.h"
+#include "db/Factories.h"
+#include "db/Options.h"
+
+INITIALIZE_EASYLOGGINGPP
+
+using namespace zilliz::milvus;
+
+static std::string uri;
+
+class DBTestEnvironment : public ::testing::Environment {
+public:
+
+//    explicit DBTestEnvironment(std::string uri) : uri_(uri) {}
+
+    static std::string getURI() {
+        return uri;
+    }
+
+    void SetUp() override {
+        getURI();
+    }
+
+};
+
+void ASSERT_STATS(engine::Status& stat) {
+    ASSERT_TRUE(stat.ok());
+    if(!stat.ok()) {
+        std::cout << stat.ToString() << std::endl;
+    }
+}
+
+
+void MetricTest::InitLog() {
+    el::Configurations defaultConf;
+    defaultConf.setToDefault();
+    defaultConf.set(el::Level::Debug,
+            el::ConfigurationType::Format, "[%thread-%datetime-%level]: %msg (%fbase:%line)");
+    el::Loggers::reconfigureLogger("default", defaultConf);
+}
+
+engine::Options MetricTest::GetOptions() {
+    auto options = engine::OptionsFactory::Build();
+    options.meta.path = "/tmp/milvus_test";
+    options.meta.backend_uri = "sqlite://:@:/";
+    return options;
+}
+
+void MetricTest::SetUp() {
+    InitLog();
+    auto options = GetOptions();
+    db_ = engine::DBFactory::Build(options);
+}
+
+void MetricTest::TearDown() {
+    delete db_;
+    boost::filesystem::remove_all("/tmp/milvus_test");
+}
+
+int main(int argc, char **argv) {
+    ::testing::InitGoogleTest(&argc, argv);
+    if (argc > 1) {
+        uri = argv[1];
+    }
+//    std::cout << uri << std::endl;
+    ::testing::AddGlobalTestEnvironment(new DBTestEnvironment);
+    return RUN_ALL_TESTS();
+}

cpp/unittest/metrics/utils.h

@@ -0,0 +1,64 @@
+////////////////////////////////////////////////////////////////////////////////
+// Copyright 上海赜睿信息科技有限公司(Zilliz) - All Rights Reserved
+// Unauthorized copying of this file, via any medium is strictly prohibited.
+// Proprietary and confidential.
+////////////////////////////////////////////////////////////////////////////////
+
+#pragma once
+
+#include <gtest/gtest.h>
+#include <chrono>
+//#include <src/db/MySQLMetaImpl.h>
+
+#include "db/DB.h"
+#include "db/DBMetaImpl.h"
+#include "db/MySQLMetaImpl.h"
+
+
+#define TIMING
+
+#ifdef TIMING
+#define INIT_TIMER auto start = std::chrono::high_resolution_clock::now();
+#define START_TIMER  start = std::chrono::high_resolution_clock::now();
+#define STOP_TIMER(name)  LOG(DEBUG) << "RUNTIME of " << name << ": " << \
+    std::chrono::duration_cast<std::chrono::milliseconds>( \
+            std::chrono::high_resolution_clock::now()-start \
+    ).count() << " ms ";
+#else
+#define INIT_TIMER
+#define START_TIMER
+#define STOP_TIMER(name)
+#endif
+
+void ASSERT_STATS(zilliz::milvus::engine::Status& stat);
+
+//class TestEnv : public ::testing::Environment {
+//public:
+//
+//    static std::string getURI() {
+//        if (const char* uri = std::getenv("MILVUS_DBMETA_URI")) {
+//            return uri;
+//        }
+//        else {
+//            return "";
+//        }
+//    }
+//
+//    void SetUp() override {
+//        getURI();
+//    }
+//
+//};
+//
+//::testing::Environment* const test_env =
+//        ::testing::AddGlobalTestEnvironment(new TestEnv);
+
+class MetricTest : public ::testing::Test {
+protected:
+    zilliz::milvus::engine::DB* db_;
+
+    void InitLog();
+    virtual void SetUp() override;
+    virtual void TearDown() override;
+    virtual zilliz::milvus::engine::Options GetOptions();
+};