Merge branch 'branch-0.5.0' into 'branch-0.5.0'

MS-557 merge Log.h

See merge request megasearch/milvus!572

Former-commit-id: a525b9e9d1cc0edbe001c7350c646a2895efcb8a

cpp/CHANGELOG.md

@@ -9,6 +9,7 @@ Please mark all change in change log and use the ticket from JIRA.
 ## Improvement
 - MS-552 - Add and change the easylogging library
 - MS-553 - Refine cache code
+- MS-557 - Merge Log.h
 - MS-556 - Add Job Definition in Scheduler
 
 ## New Feature

cpp/src/core/todo

@@ -1,2 +0,0 @@
-1. Support L2 and IP
-2. replace with RapidJson

cpp/src/db/DBFactory.cpp

@@ -18,7 +18,7 @@
 
 #include "DBFactory.h"
 #include "DBImpl.h"
-#include "Exception.h"
+#include "utils/Exception.h"
 #include "meta/MetaFactory.h"
 #include "meta/SqliteMetaImpl.h"
 #include "meta/MySQLMetaImpl.h"

cpp/src/db/DBImpl.cpp

@@ -16,28 +16,27 @@
 // under the License.
 
 #include "DBImpl.h"
-#include "src/db/meta/SqliteMetaImpl.h"
-#include "Log.h"
-#include "Utils.h"
+#include "cache/CpuCacheMgr.h"
+#include "cache/GpuCacheMgr.h"
 #include "engine/EngineFactory.h"
 #include "insert/MemMenagerFactory.h"
+#include "meta/SqliteMetaImpl.h"
 #include "meta/MetaFactory.h"
+#include "meta/MetaConsts.h"
 #include "metrics/Metrics.h"
 #include "scheduler/TaskScheduler.h"
-
 #include "scheduler/context/DeleteContext.h"
+#include "scheduler/SchedInst.h"
 #include "utils/TimeRecorder.h"
-#include "meta/MetaConsts.h"
+#include "utils/Log.h"
+#include "Utils.h"
 
 #include <assert.h>
 #include <chrono>
 #include <thread>
 #include <iostream>
 #include <cstring>
-#include <cache/CpuCacheMgr.h>
 #include <boost/filesystem.hpp>
-#include "scheduler/SchedInst.h"
-#include <src/cache/GpuCacheMgr.h>
 
 namespace zilliz {
 namespace milvus {
@@ -421,7 +420,7 @@ Status DBImpl::QueryAsync(const std::string& table_id, const meta::TableFilesSch
                           const meta::DatesT& dates, QueryResults& results) {
     server::CollectQueryMetrics metrics(nq);
 
-    server::TimeRecorder rc("");
+    TimeRecorder rc("");
 
     //step 1: get files to search
     ENGINE_LOG_DEBUG << "Engine query begin, index file count: " << files.size() << " date range count: " << dates.size();
@@ -444,9 +443,9 @@ Status DBImpl::QueryAsync(const std::string& table_id, const meta::TableFilesSch
     double load_cost = context->LoadCost();
     double search_cost = context->SearchCost();
     double reduce_cost = context->ReduceCost();
-    std::string load_info = server::TimeRecorder::GetTimeSpanStr(load_cost);
-    std::string search_info = server::TimeRecorder::GetTimeSpanStr(search_cost);
-    std::string reduce_info = server::TimeRecorder::GetTimeSpanStr(reduce_cost);
+    std::string load_info = TimeRecorder::GetTimeSpanStr(load_cost);
+    std::string search_info = TimeRecorder::GetTimeSpanStr(search_cost);
+    std::string reduce_info = TimeRecorder::GetTimeSpanStr(reduce_cost);
     if(search_cost > 0.0 || reduce_cost > 0.0) {
         double total_cost = load_cost + search_cost + reduce_cost;
         double load_percent = load_cost/total_cost;

cpp/src/db/DBImpl.h

@@ -140,11 +140,11 @@ class DBImpl : public DB {
     MemManagerPtr mem_mgr_;
     std::mutex mem_serialize_mutex_;
 
-    server::ThreadPool compact_thread_pool_;
+    ThreadPool compact_thread_pool_;
     std::list<std::future<void>> compact_thread_results_;
     std::set<std::string> compact_table_ids_;
 
-    server::ThreadPool index_thread_pool_;
+    ThreadPool index_thread_pool_;
     std::list<std::future<void>> index_thread_results_;
 
     std::mutex build_index_mutex_;

cpp/src/db/Log.h

@@ -1,38 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements.  See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership.  The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-#pragma once
-
-#include "utils/easylogging++.h"
-
-namespace zilliz {
-namespace milvus {
-namespace engine {
-
-#define ENGINE_DOMAIN_NAME "[ENGINE] "
-#define ENGINE_ERROR_TEXT "ENGINE Error:"
-
-#define ENGINE_LOG_TRACE LOG(TRACE) << ENGINE_DOMAIN_NAME
-#define ENGINE_LOG_DEBUG LOG(DEBUG) << ENGINE_DOMAIN_NAME
-#define ENGINE_LOG_INFO LOG(INFO) << ENGINE_DOMAIN_NAME
-#define ENGINE_LOG_WARNING LOG(WARNING) << ENGINE_DOMAIN_NAME
-#define ENGINE_LOG_ERROR LOG(ERROR) << ENGINE_DOMAIN_NAME
-#define ENGINE_LOG_FATAL LOG(FATAL) << ENGINE_DOMAIN_NAME
-
-} // namespace sql
-} // namespace zilliz
-} // namespace server

cpp/src/db/Options.cpp

@@ -15,15 +15,14 @@
 // specific language governing permissions and limitations
 // under the License.
 
+#include "Options.h"
+#include "utils/Exception.h"
+#include "utils/easylogging++.h"
+
 #include <stdlib.h>
 #include <assert.h>
-#include "utils/easylogging++.h"
 #include <boost/algorithm/string.hpp>
 
-#include "Options.h"
-#include "db/meta/SqliteMetaImpl.h"
-#include "Exception.h"
-
 namespace zilliz {
 namespace milvus {
 namespace engine {

cpp/src/db/Utils.cpp

@@ -17,7 +17,7 @@
 
 #include "Utils.h"
 #include "utils/CommonUtil.h"
-#include "Log.h"
+#include "utils/Log.h"
 
 #include <mutex>
 #include <chrono>

cpp/src/db/engine/EngineFactory.cpp

@@ -17,7 +17,7 @@
 
 #include "EngineFactory.h"
 #include "ExecutionEngineImpl.h"
-#include "db/Log.h"
+#include "utils/Log.h"
 
 namespace zilliz {
 namespace milvus {

cpp/src/db/engine/ExecutionEngineImpl.cpp

@@ -15,20 +15,19 @@
 // specific language governing permissions and limitations
 // under the License.
 
-#include <stdexcept>
-#include "src/cache/GpuCacheMgr.h"
-
-#include "src/metrics/Metrics.h"
-#include "db/Log.h"
-#include "utils/CommonUtil.h"
-
-#include "src/cache/CpuCacheMgr.h"
 #include "ExecutionEngineImpl.h"
+#include "cache/GpuCacheMgr.h"
+#include "cache/CpuCacheMgr.h"
+#include "metrics/Metrics.h"
+#include "utils/Log.h"
+#include "utils/CommonUtil.h"
+#include "utils/Exception.h"
+
 #include "wrapper/knowhere/vec_index.h"
 #include "wrapper/knowhere/vec_impl.h"
 #include "knowhere/common/exception.h"
-#include "db/Exception.h"
 
+#include <stdexcept>
 
 namespace zilliz {
 namespace milvus {

cpp/src/db/insert/MemManagerImpl.cpp

@@ -18,7 +18,7 @@
 
 #include "MemManagerImpl.h"
 #include "VectorSource.h"
-#include "db/Log.h"
+#include "utils/Log.h"
 #include "db/Constants.h"
 
 #include <thread>

cpp/src/db/insert/MemMenagerFactory.cpp

@@ -15,11 +15,10 @@
 // specific language governing permissions and limitations
 // under the License.
 
-
 #include "MemMenagerFactory.h"
 #include "MemManagerImpl.h"
-#include "db/Log.h"
-#include "db/Exception.h"
+#include "utils/Log.h"
+#include "utils/Exception.h"
 
 #include <stdlib.h>
 #include <time.h>

cpp/src/db/insert/MemTable.cpp

@@ -17,7 +17,7 @@
 
 
 #include "MemTable.h"
-#include "db/Log.h"
+#include "utils/Log.h"
 
 
 namespace zilliz {

cpp/src/db/insert/MemTableFile.cpp

@@ -18,9 +18,9 @@
 
 #include "MemTableFile.h"
 #include "db/Constants.h"
-#include "db/Log.h"
 #include "db/engine/EngineFactory.h"
 #include "metrics/Metrics.h"
+#include "utils/Log.h"
 
 #include <cmath>
 

cpp/src/db/insert/VectorSource.cpp

@@ -19,7 +19,7 @@
 #include "VectorSource.h"
 #include "db/engine/ExecutionEngine.h"
 #include "db/engine/EngineFactory.h"
-#include "db/Log.h"
+#include "utils/Log.h"
 #include "metrics/Metrics.h"
 
 

cpp/src/db/meta/MetaFactory.cpp

@@ -15,12 +15,11 @@
 // specific language governing permissions and limitations
 // under the License.
 
-
 #include "MetaFactory.h"
 #include "SqliteMetaImpl.h"
 #include "MySQLMetaImpl.h"
-#include "db/Log.h"
-#include "db/Exception.h"
+#include "utils/Log.h"
+#include "utils/Exception.h"
 
 #include <stdlib.h>
 #include <time.h>

cpp/src/db/meta/MySQLConnectionPool.h

@@ -22,7 +22,7 @@
 #include <unistd.h>
 #include <atomic>
 
-#include "db/Log.h"
+#include "utils/Log.h"
 
 namespace zilliz {
 namespace milvus {

cpp/src/db/meta/MySQLMetaImpl.cpp

@@ -18,7 +18,7 @@
 #include "MySQLMetaImpl.h"
 #include "db/IDGenerator.h"
 #include "db/Utils.h"
-#include "db/Log.h"
+#include "utils/Log.h"
 #include "MetaConsts.h"
 #include "metrics/Metrics.h"
 

cpp/src/db/meta/SqliteMetaImpl.cpp

@@ -18,7 +18,7 @@
 #include "SqliteMetaImpl.h"
 #include "db/IDGenerator.h"
 #include "db/Utils.h"
-#include "db/Log.h"
+#include "utils/Log.h"
 #include "MetaConsts.h"
 #include "metrics/Metrics.h"
 

cpp/src/db/scheduler/TaskDispatchStrategy.cpp

@@ -22,7 +22,7 @@
 #include "task/DeleteTask.h"
 #include "cache/CpuCacheMgr.h"
 #include "utils/Error.h"
-#include "db/Log.h"
+#include "utils/Log.h"
 
 namespace zilliz {
 namespace milvus {

cpp/src/db/scheduler/task/IndexLoadTask.cpp

@@ -18,8 +18,8 @@
 
 #include "IndexLoadTask.h"
 #include "SearchTask.h"
-#include "db/Log.h"
 #include "db/engine/EngineFactory.h"
+#include "utils/Log.h"
 #include "utils/TimeRecorder.h"
 #include "metrics/Metrics.h"
 

cpp/src/db/scheduler/task/SearchTask.cpp

@@ -17,7 +17,7 @@
 
 #include "SearchTask.h"
 #include "metrics/Metrics.h"
-#include "db/Log.h"
+#include "utils/Log.h"
 #include "utils/TimeRecorder.h"
 
 #include <thread>

cpp/src/scheduler/ResourceMgr.cpp

@@ -17,7 +17,7 @@
 // under the License.
 
 #include "ResourceMgr.h"
-#include "db/Log.h"
+#include "utils/Log.h"
 
 
 namespace zilliz {

cpp/src/scheduler/task/SearchTask.cpp

@@ -15,12 +15,11 @@
 // specific language governing permissions and limitations
 // under the License.
 
-
-#include "src/metrics/Metrics.h"
-#include "src/utils/TimeRecorder.h"
-#include "src/db/engine/EngineFactory.h"
-#include "src/db/Log.h"
 #include "SearchTask.h"
+#include "metrics/Metrics.h"
+#include "db/engine/EngineFactory.h"
+#include "utils/TimeRecorder.h"
+#include "utils/Log.h"
 
 #include <thread>
 
@@ -109,7 +108,7 @@ XSearchTask::XSearchTask(TableFileSchemaPtr file)
 
 void
 XSearchTask::Load(LoadType type, uint8_t device_id) {
-    server::TimeRecorder rc("");
+    TimeRecorder rc("");
     Status stat = Status::OK();
     std::string error_msg;
     std::string type_str;
@@ -178,7 +177,7 @@ XSearchTask::Execute() {
     ENGINE_LOG_DEBUG << "Searching in file id:" << index_id_ << " with "
                      << search_contexts_.size() << " tasks";
 
-    server::TimeRecorder rc("DoSearch file id:" + std::to_string(index_id_));
+    TimeRecorder rc("DoSearch file id:" + std::to_string(index_id_));
 
     server::CollectDurationMetrics metrics(index_type_);
 

cpp/src/utils/Exception.h

@@ -22,7 +22,6 @@
 
 namespace zilliz {
 namespace milvus {
-namespace engine {
 
 class Exception : public std::exception {
 public:
@@ -61,6 +60,5 @@ public:
     OutOfRangeException(const std::string& message) : Exception(message) {};
 };
 
-} // namespace engine
 } // namespace milvus
 } // namespace zilliz

cpp/src/utils/Log.h

@@ -17,13 +17,12 @@
 
 #pragma once
 
-#include "Error.h"
 #include "easylogging++.h"
 
 namespace zilliz {
 namespace milvus {
-namespace server {
 
+/////////////////////////////////////////////////////////////////////////////////////////////////
 #define SERVER_DOMAIN_NAME "[SERVER] "
 #define SERVER_ERROR_TEXT "SERVER Error:"
 
@@ -34,20 +33,27 @@ namespace server {
 #define SERVER_LOG_ERROR LOG(ERROR) << SERVER_DOMAIN_NAME
 #define SERVER_LOG_FATAL LOG(FATAL) << SERVER_DOMAIN_NAME
 
-#define SERVER_ERROR(error)                             \
-    ({                                                  \
-        SERVER_LOG_ERROR << SERVER_ERROR_TEXT << error; \
-        (error);                                        \
-    })
+/////////////////////////////////////////////////////////////////////////////////////////////////
+#define ENGINE_DOMAIN_NAME "[ENGINE] "
+#define ENGINE_ERROR_TEXT "ENGINE Error:"
 
-#define SERVER_CHECK(func)                                  \
-    {                                                       \
-        zilliz::milvus::server::ServerError error = func;  \
-        if (error != zilliz::milvus::server::SERVER_SUCCESS) { \
-            return SERVER_ERROR(error);                     \
-        }                                                   \
-    }                                                       \
+#define ENGINE_LOG_TRACE LOG(TRACE) << ENGINE_DOMAIN_NAME
+#define ENGINE_LOG_DEBUG LOG(DEBUG) << ENGINE_DOMAIN_NAME
+#define ENGINE_LOG_INFO LOG(INFO) << ENGINE_DOMAIN_NAME
+#define ENGINE_LOG_WARNING LOG(WARNING) << ENGINE_DOMAIN_NAME
+#define ENGINE_LOG_ERROR LOG(ERROR) << ENGINE_DOMAIN_NAME
+#define ENGINE_LOG_FATAL LOG(FATAL) << ENGINE_DOMAIN_NAME
 
-} // namespace sql
+/////////////////////////////////////////////////////////////////////////////////////////////////
+#define WRAPPER_DOMAIN_NAME "[WRAPPER] "
+#define WRAPPER_ERROR_TEXT "WRAPPER Error:"
+
+#define WRAPPER_LOG_TRACE LOG(TRACE) << WRAPPER_DOMAIN_NAME
+#define WRAPPER_LOG_DEBUG LOG(DEBUG) << WRAPPER_DOMAIN_NAME
+#define WRAPPER_LOG_INFO LOG(INFO) << WRAPPER_DOMAIN_NAME
+#define WRAPPER_LOG_WARNING LOG(WARNING) << WRAPPER_DOMAIN_NAME
+#define WRAPPER_LOG_ERROR LOG(ERROR) << WRAPPER_DOMAIN_NAME
+#define WRAPPER_LOG_FATAL LOG(FATAL) << WRAPPER_DOMAIN_NAME
+
+} // namespace milvus
 } // namespace zilliz
-} // namespace server

cpp/src/utils/ThreadPool.h

@@ -32,7 +32,6 @@
 
 namespace zilliz {
 namespace milvus {
-namespace server {
 
 class ThreadPool {
 public:
@@ -126,5 +125,4 @@ inline ThreadPool::~ThreadPool() {
 
 }
 }
-}
 

cpp/src/utils/TimeRecorder.cpp

@@ -21,7 +21,6 @@
 
 namespace zilliz {
 namespace milvus {
-namespace server {
 
 TimeRecorder::TimeRecorder(const std::string &header,
                            int64_t log_level) :
@@ -103,4 +102,3 @@ TimeRecorder::ElapseFromBegin(const std::string &msg) {
 
 }
 }
-}

cpp/src/utils/TimeRecorder.h

@@ -23,7 +23,6 @@
 
 namespace zilliz {
 namespace milvus {
-namespace server {
 
 class TimeRecorder {
     using stdclock = std::chrono::high_resolution_clock;
@@ -52,4 +51,3 @@ private:
 
 }
 }
-}

cpp/src/wrapper/knowhere/vec_impl.cpp

@@ -16,7 +16,7 @@
 // under the License.
 
 
-#include <src/utils/Log.h>
+#include "utils/Log.h"
 #include "knowhere/index/vector_index/idmap.h"
 #include "knowhere/index/vector_index/gpu_ivf.h"
 #include "knowhere/common/exception.h"
@@ -24,7 +24,6 @@
 
 #include "vec_impl.h"
 #include "data_transfer.h"
-#include "wrapper_log.h"
 
 
 namespace zilliz {

cpp/src/wrapper/knowhere/vec_index.cpp

@@ -24,7 +24,7 @@
 
 #include "vec_index.h"
 #include "vec_impl.h"
-#include "wrapper_log.h"
+#include "utils/Log.h"
 
 #include <cuda.h>
 

cpp/src/wrapper/knowhere/wrapper_log.h

@@ -1,40 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements.  See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership.  The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-
-#pragma once
-
-#include "utils/easylogging++.h"
-
-namespace zilliz {
-namespace milvus {
-namespace engine {
-
-#define WRAPPER_DOMAIN_NAME "[WRAPPER] "
-#define WRAPPER_ERROR_TEXT "WRAPPER Error:"
-
-#define WRAPPER_LOG_TRACE LOG(TRACE) << WRAPPER_DOMAIN_NAME
-#define WRAPPER_LOG_DEBUG LOG(DEBUG) << WRAPPER_DOMAIN_NAME
-#define WRAPPER_LOG_INFO LOG(INFO) << WRAPPER_DOMAIN_NAME
-#define WRAPPER_LOG_WARNING LOG(WARNING) << WRAPPER_DOMAIN_NAME
-#define WRAPPER_LOG_ERROR LOG(ERROR) << WRAPPER_DOMAIN_NAME
-#define WRAPPER_LOG_FATAL LOG(FATAL) << WRAPPER_DOMAIN_NAME
-
-}
-}
-}
-

cpp/unittest/db/misc_test.cpp

@@ -15,18 +15,17 @@
 // specific language governing permissions and limitations
 // under the License.
 
-#include <gtest/gtest.h>
-#include <thread>
-#include "utils/easylogging++.h"
-#include <boost/filesystem.hpp>
-
-#include "db/Exception.h"
 #include "db/Status.h"
 #include "db/Options.h"
 #include "db/meta/SqliteMetaImpl.h"
 #include "db/engine/EngineFactory.h"
 #include "db/Utils.h"
+#include "utils/Exception.h"
+#include "utils/easylogging++.h"
 
+#include <gtest/gtest.h>
+#include <thread>
+#include <boost/filesystem.hpp>
 #include <vector>
 
 using namespace zilliz::milvus;
@@ -39,11 +38,11 @@ namespace {
 }
 
 TEST(DBMiscTest, EXCEPTION_TEST) {
-    engine::Exception ex1("");
+    Exception ex1("");
     std::string what = ex1.what();
     ASSERT_FALSE(what.empty());
 
-    engine::OutOfRangeException ex2;
+    OutOfRangeException ex2;
     what = ex2.what();
     ASSERT_FALSE(what.empty());
 }

cpp/unittest/db/search_test.cpp

@@ -238,7 +238,7 @@ TEST(DBSearchTest, PARALLEL_CLUSTER_TEST) {
     engine::SearchContext::ResultSet src_result;
 
     auto DoCluster = [&](int64_t nq, int64_t topk) {
-        server::TimeRecorder rc("DoCluster");
+        TimeRecorder rc("DoCluster");
         src_result.clear();
         BuildResult(nq, topk, ascending, target_ids, target_distence);
         rc.RecordSection("build id/dietance map");
@@ -280,7 +280,7 @@ TEST(DBSearchTest, PARALLEL_TOPK_TEST) {
         src_result.clear();
         insufficient_result.clear();
 
-        server::TimeRecorder rc("DoCluster");
+        TimeRecorder rc("DoCluster");
 
         BuildResult(nq, topk, ascending, target_ids, target_distence);
         auto status = engine::XSearchTask::ClusterResult(target_ids, target_distence, nq, topk, src_result);

cpp/unittest/server/util_test.cpp

@@ -178,7 +178,7 @@ TEST(UtilTest, TIMERECORDER_TEST) {
         if(log_level == 5) {
             continue; //skip fatal
         }
-        server::TimeRecorder rc("time", log_level);
+        TimeRecorder rc("time", log_level);
         rc.RecordSection("end");
     }
 }