Merge remote-tracking branch 'mega/branch-0.3.0' into branch-0.3.0

Former-commit-id: eeb07fe4825bd1c21ec900cea5451c235f3053b8
pull/191/head
Yang Xuan 2019-06-13 17:15:32 +08:00
commit e7021fc234
135 changed files with 291 additions and 324 deletions

View File

@ -5,8 +5,11 @@ Please mark all change in change log and use the ticket from JIRA.
# MegaSearch 0.3.0 (TBD)
## Bug
- MS-80 - Fix server hang issue
## Improvement
- MS-82 - Update server startup welcome message
- MS-83 - Update vecwise to Milvus
## New Feature
@ -23,6 +26,7 @@ Please mark all change in change log and use the ticket from JIRA.
- MS-65 - Implement GetTableRowCount interface
- MS-45 - Implement DeleteTable interface
- MS-75 - cmake: change faiss version to 1.5.2; add CUDA gencode
- MS-81 - fix faiss ptx issue; change cuda gencode
## Task
- MS-74 - Change README.md in cpp
@ -34,6 +38,7 @@ Please mark all change in change log and use the ticket from JIRA.
- MS-32 - Fix thrift error
- MS-34 - Fix prometheus-cpp thirdparty
- MS-67 - Fix license check bug
- MS-76 - Fix pipeline crash bug
## Improvement

View File

@ -44,33 +44,6 @@ if("${MAKE}" STREQUAL "")
endif()
endif()
set(GCC_VERSION 5.3.0)
set(GCC_DIR "${CMAKE_CURRENT_BINARY_DIR}/gcc")
set(GCC_BUILD_DIR "${CMAKE_CURRENT_BINARY_DIR}/gcc/gcc_build")
set(GCC_TAR_NAME "${GCC_BUILD_DIR}/gcc-${GCC_VERSION}.tar.gz")
if (NOT EXISTS ${GCC_TAR_NAME})
file(MAKE_DIRECTORY ${GCC_BUILD_DIR})
file(DOWNLOAD http://ftp.tsukuba.wide.ad.jp/software/gcc/releases/gcc-${GCC_VERSION}/${GCC_TAR_NAME}
${GCC_TAR_NAME})
message(STATUS "Building GCC-${GCC_VERSION} from source")
execute_process(
COMMAND
"${CMAKE_COMMAND} -E tar -xf ${GCC_TAR_NAME}"
COMMAND
"./configure --prefix=${GCC_DIR} --with-local-prefix=${GCC_DIR}
--enable-languages=cxx,fortran
--enable-checking=release
--disable-bootstrap"
COMMAND
"${MAKE} -j4"
COMMAND
"${MAKE} install"
RESULT_VARIABLE _status
#OUTPUT_QUIET
#ERROR_QUIET
WORKING_DIRECTORY ${GCC_BUILD_DIR})
endif ()
set(MEGASEARCH_VERSION_MAJOR "${megasearch_VERSION_MAJOR}")
set(MEGASEARCH_VERSION_MINOR "${megasearch_VERSION_MINOR}")
set(MEGASEARCH_VERSION_PATCH "${megasearch_VERSION_PATCH}")

View File

@ -70,8 +70,8 @@ define_option(MEGASEARCH_WITH_FAISS "Build with FAISS library" ON)
define_option(MEGASEARCH_WITH_FAISS_GPU_VERSION "Build with FAISS GPU version" ON)
define_option_string(MEGASEARCH_FAISS_GPU_ARCH "Specifying which GPU architectures to build against"
"-gencode=arch=compute_35,code=sm_35 -gencode=arch=compute_52,code=sm_52 -gencode=arch=compute_60,code=sm_60 -gencode=arch=compute_61,code=sm_61")
#define_option_string(MEGASEARCH_FAISS_GPU_ARCH "Specifying which GPU architectures to build against"
# "-gencode=arch=compute_35,code=compute_35 -gencode=arch=compute_52,code=compute_52 -gencode=arch=compute_60,code=compute_60 -gencode=arch=compute_61,code=compute_61")
define_option(MEGASEARCH_WITH_LAPACK "Build with LAPACK library" ON)

View File

@ -686,7 +686,15 @@ macro(build_faiss)
if(${MEGASEARCH_WITH_FAISS_GPU_VERSION} STREQUAL "ON")
set(FAISS_CONFIGURE_ARGS ${FAISS_CONFIGURE_ARGS}
"--with-cuda=${CUDA_TOOLKIT_ROOT_DIR}"
"--with-cuda-arch=${MEGASEARCH_FAISS_GPU_ARCH}")
# "with_cuda_arch=\"-gencode=arch=compute_35,code=compute_35 \\
# -gencode=arch=compute_52,code=compute_52 \\
# -gencode=arch=compute_60,code=compute_60 \\
# -gencode=arch=compute_61,code=compute_61\""
"--with-cuda-arch=\"-gencode=arch=compute_35,code=compute_35\""
"--with-cuda-arch=\"-gencode=arch=compute_52,code=compute_52\""
"--with-cuda-arch=\"-gencode=arch=compute_60,code=compute_60\""
"--with-cuda-arch=\"-gencode=arch=compute_61,code=compute_61\""
)
else()
set(FAISS_CONFIGURE_ARGS ${FAISS_CONFIGURE_ARGS} --without-cuda)
endif()

View File

@ -6,7 +6,7 @@ server_config:
gpu_index: 0 #which gpu to be used
db_config:
db_path: /tmp/vecwise
db_path: /tmp/milvus
db_backend_url: http://127.0.0.1
db_flush_interval: 5 #unit: second
idmapper_max_open_file: 128

View File

@ -6,7 +6,7 @@ server_config:
gpu_index: 0 #which gpu to be used
db_config:
db_path: /tmp/vecwise
db_path: /tmp/milvus
db_backend_url: http://127.0.0.1
db_flush_interval: 5 #unit: second
idmapper_max_open_file: 128

View File

@ -1,6 +1,6 @@
* GLOBAL:
FORMAT = "%datetime | %level | %logger | %msg"
FILENAME = "/tmp/vecwise/logs/vecwise_engine-%datetime{%H:%m}-global.log"
FILENAME = "/tmp/milvus/logs/vecwise_engine-%datetime{%H:%m}-global.log"
ENABLED = true
TO_FILE = true
TO_STANDARD_OUTPUT = true
@ -8,12 +8,12 @@
PERFORMANCE_TRACKING = false
MAX_LOG_FILE_SIZE = 2097152 ## Throw log files away after 2MB
* DEBUG:
FILENAME = "/tmp/vecwise/logs/vecwise_engine-%datetime{%H:%m}-debug.log"
FILENAME = "/tmp/milvus/logs/vecwise_engine-%datetime{%H:%m}-debug.log"
ENABLED = true
* WARNING:
FILENAME = "/tmp/vecwise/logs/vecwise_engine-%datetime{%H:%m}-warning.log"
FILENAME = "/tmp/milvus/logs/vecwise_engine-%datetime{%H:%m}-warning.log"
* TRACE:
FILENAME = "/tmp/vecwise/logs/vecwise_engine-%datetime{%H:%m}-trace.log"
FILENAME = "/tmp/milvus/logs/vecwise_engine-%datetime{%H:%m}-trace.log"
* VERBOSE:
FORMAT = "%datetime{%d/%M/%y} | %level-%vlevel | %msg"
TO_FILE = false
@ -21,7 +21,7 @@
## Error logs
* ERROR:
ENABLED = false
FILENAME = "/tmp/vecwise/logs/vecwise_engine-%datetime{%H:%m}-error.log"
FILENAME = "/tmp/milvus/logs/vecwise_engine-%datetime{%H:%m}-error.log"
* FATAL:
ENABLED = false
FILENAME = "/tmp/vecwise/logs/vecwise_engine-%datetime{%H:%m}-fatal.log"
FILENAME = "/tmp/milvus/logs/vecwise_engine-%datetime{%H:%m}-fatal.log"

View File

@ -10,7 +10,7 @@ function kill_progress()
STATUS=$(kill_progress "vecwise_server" )
if [[ ${STATUS} == "false" ]];then
echo "vecwise_server closed abnormally!"
echo "Milvus server closed abnormally!"
else
echo "vecwise_server closed successfully!"
echo "Milvus server closed successfully!"
fi

View File

@ -10,7 +10,7 @@
#include <set>
namespace zilliz {
namespace vecwise {
namespace milvus {
namespace cache {
Cache::Cache(int64_t capacity, uint64_t cache_max_count)
@ -218,6 +218,6 @@ void Cache::print() {
}
} // cache
} // vecwise
} // milvus
} // zilliz

View File

@ -14,7 +14,7 @@
#include "DataObj.h"
namespace zilliz {
namespace vecwise {
namespace milvus {
namespace cache {
const std::string SWAP_DIR = ".CACHE";
@ -65,6 +65,6 @@ private:
using CachePtr = std::shared_ptr<Cache>;
} // cache
} // vecwise
} // milvus
} // zilliz

View File

@ -8,7 +8,7 @@
#include "metrics/Metrics.h"
namespace zilliz {
namespace vecwise {
namespace milvus {
namespace cache {
CacheMgr::CacheMgr() {

View File

@ -9,7 +9,7 @@
#include "Cache.h"
namespace zilliz {
namespace vecwise {
namespace milvus {
namespace cache {
class CacheMgr {

View File

@ -8,7 +8,7 @@
#include "server/ServerConfig.h"
namespace zilliz {
namespace vecwise {
namespace milvus {
namespace cache {
CpuCacheMgr::CpuCacheMgr() {

View File

@ -8,7 +8,7 @@
#include "CacheMgr.h"
namespace zilliz {
namespace vecwise {
namespace milvus {
namespace cache {
class CpuCacheMgr : public CacheMgr {

View File

@ -11,7 +11,7 @@
#include <memory>
namespace zilliz {
namespace vecwise {
namespace milvus {
namespace cache {
class DataObj {

View File

@ -8,7 +8,7 @@
#include "server/ServerConfig.h"
namespace zilliz {
namespace vecwise {
namespace milvus {
namespace cache {
GpuCacheMgr::GpuCacheMgr() {

View File

@ -7,7 +7,7 @@
#include "CacheMgr.h"
namespace zilliz {
namespace vecwise {
namespace milvus {
namespace cache {
class GpuCacheMgr : public CacheMgr {

4
cpp/src/cache/LRU.h vendored
View File

@ -12,7 +12,7 @@
#include <stdexcept>
namespace zilliz {
namespace vecwise {
namespace milvus {
namespace cache {
template<typename key_t, typename value_t>
@ -97,6 +97,6 @@ private:
};
} // cache
} // vecwise
} // milvus
} // zilliz

View File

@ -12,7 +12,7 @@
#include <algorithm>
namespace zilliz {
namespace vecwise {
namespace milvus {
namespace server {
void ConfigNode::Combine(const ConfigNode& target) {

View File

@ -10,7 +10,7 @@
#include <map>
namespace zilliz {
namespace vecwise {
namespace milvus {
namespace server {
class ConfigNode;

View File

@ -7,7 +7,7 @@
#include "YamlConfigMgr.h"
namespace zilliz {
namespace vecwise {
namespace milvus {
namespace server {
IConfigMgr * IConfigMgr::GetInstance() {

View File

@ -9,7 +9,7 @@
#include "ConfigNode.h"
namespace zilliz {
namespace vecwise {
namespace milvus {
namespace server {
// this class can parse nested config file and return config item

View File

@ -9,7 +9,7 @@
#include <sys/stat.h>
namespace zilliz {
namespace vecwise {
namespace milvus {
namespace server {
ServerError YamlConfigMgr::LoadConfigFile(const std::string &filename) {

View File

@ -12,7 +12,7 @@
#include <yaml-cpp/yaml.h>
namespace zilliz {
namespace vecwise {
namespace milvus {
namespace server {
class YamlConfigMgr : public IConfigMgr {

View File

@ -10,7 +10,7 @@
#include "Factories.h"
namespace zilliz {
namespace vecwise {
namespace milvus {
namespace engine {
DB::~DB() {}
@ -21,5 +21,5 @@ void DB::Open(const Options& options, DB** dbptr) {
}
} // namespace engine
} // namespace vecwise
} // namespace milvus
} // namespace zilliz

View File

@ -13,7 +13,7 @@
#include <string>
namespace zilliz {
namespace vecwise {
namespace milvus {
namespace engine {
class Env;
@ -50,5 +50,5 @@ public:
}; // DB
} // namespace engine
} // namespace vecwise
} // namespace milvus
} // namespace zilliz

View File

@ -20,7 +20,7 @@
#include <boost/filesystem.hpp>
namespace zilliz {
namespace vecwise {
namespace milvus {
namespace engine {
namespace {
@ -580,5 +580,5 @@ DBImpl::~DBImpl() {
}
} // namespace engine
} // namespace vecwise
} // namespace milvus
} // namespace zilliz

View File

@ -16,7 +16,7 @@
#include <thread>
namespace zilliz {
namespace vecwise {
namespace milvus {
namespace engine {
class Env;
@ -100,5 +100,5 @@ private:
} // namespace engine
} // namespace vecwise
} // namespace milvus
} // namespace zilliz

View File

@ -21,7 +21,7 @@
namespace zilliz {
namespace vecwise {
namespace milvus {
namespace engine {
namespace meta {
@ -923,5 +923,5 @@ DBMetaImpl::~DBMetaImpl() {
} // namespace meta
} // namespace engine
} // namespace vecwise
} // namespace milvus
} // namespace zilliz

View File

@ -9,7 +9,7 @@
#include "Options.h"
namespace zilliz {
namespace vecwise {
namespace milvus {
namespace engine {
namespace meta {
@ -76,5 +76,5 @@ private:
} // namespace meta
} // namespace engine
} // namespace vecwise
} // namespace milvus
} // namespace zilliz

View File

@ -8,7 +8,7 @@
#include "Log.h"
namespace zilliz {
namespace vecwise {
namespace milvus {
namespace engine {
ExecutionEnginePtr

View File

@ -9,7 +9,7 @@
#include "ExecutionEngine.h"
namespace zilliz {
namespace vecwise {
namespace milvus {
namespace engine {
class EngineFactory {

View File

@ -9,7 +9,7 @@
#include "Env.h"
namespace zilliz {
namespace vecwise {
namespace milvus {
namespace engine {
Env::Env()
@ -83,5 +83,5 @@ Env* Env::Default() {
}
} // namespace engine
} // namespace vecwise
} // namespace milvus
} // namespace zilliz

View File

@ -12,7 +12,7 @@
#include <atomic>
namespace zilliz {
namespace vecwise {
namespace milvus {
namespace engine {
class Env {
@ -52,5 +52,5 @@ protected:
}; // Env
} // namespace engine
} // namespace vecwise
} // namespace milvus
} // namespace zilliz

View File

@ -9,7 +9,7 @@
#include <string>
namespace zilliz {
namespace vecwise {
namespace milvus {
namespace engine {
class Exception : public std::exception {
@ -50,5 +50,5 @@ public:
};
} // namespace engine
} // namespace vecwise
} // namespace milvus
} // namespace zilliz

View File

@ -8,7 +8,7 @@
#include <easylogging++.h>
namespace zilliz {
namespace vecwise {
namespace milvus {
namespace engine {
Status ExecutionEngine::AddWithIds(const std::vector<float>& vectors, const std::vector<long>& vector_ids) {
@ -23,5 +23,5 @@ Status ExecutionEngine::AddWithIds(const std::vector<float>& vectors, const std:
} // namespace engine
} // namespace vecwise
} // namespace milvus
} // namespace zilliz

View File

@ -11,7 +11,7 @@
#include <memory>
namespace zilliz {
namespace vecwise {
namespace milvus {
namespace engine {
enum class EngineType {
@ -57,5 +57,5 @@ using ExecutionEnginePtr = std::shared_ptr<ExecutionEngine>;
} // namespace engine
} // namespace vecwise
} // namespace milvus
} // namespace zilliz

View File

@ -15,7 +15,7 @@
#include <easylogging++.h>
namespace zilliz {
namespace vecwise {
namespace milvus {
namespace engine {
DBMetaOptions DBMetaOptionsFactory::Build(const std::string& path) {
@ -54,5 +54,5 @@ DB* DBFactory::Build(const Options& options) {
}
} // namespace engine
} // namespace vecwise
} // namespace milvus
} // namespace zilliz

View File

@ -14,7 +14,7 @@
#include <memory>
namespace zilliz {
namespace vecwise {
namespace milvus {
namespace engine {
struct DBMetaOptionsFactory {
@ -35,5 +35,5 @@ struct DBFactory {
};
} // namespace engine
} // namespace vecwise
} // namespace milvus
} // namespace zilliz

View File

@ -17,7 +17,7 @@
namespace zilliz {
namespace vecwise {
namespace milvus {
namespace engine {
@ -68,7 +68,7 @@ Status FaissExecutionEngine::Serialize() {
}
Status FaissExecutionEngine::Load() {
auto index = zilliz::vecwise::cache::CpuCacheMgr::GetInstance()->GetIndex(location_);
auto index = zilliz::milvus::cache::CpuCacheMgr::GetInstance()->GetIndex(location_);
bool to_cache = false;
auto start_time = METRICS_NOW_TIME;
if (!index) {
@ -98,7 +98,7 @@ Status FaissExecutionEngine::Merge(const std::string& location) {
if (location == location_) {
return Status::Error("Cannot Merge Self");
}
auto to_merge = zilliz::vecwise::cache::CpuCacheMgr::GetInstance()->GetIndex(location);
auto to_merge = zilliz::milvus::cache::CpuCacheMgr::GetInstance()->GetIndex(location);
if (!to_merge) {
to_merge = read_index(location);
}
@ -140,7 +140,7 @@ Status FaissExecutionEngine::Search(long n,
}
Status FaissExecutionEngine::Cache() {
zilliz::vecwise::cache::CpuCacheMgr::GetInstance(
zilliz::milvus::cache::CpuCacheMgr::GetInstance(
)->InsertItem(location_, std::make_shared<Index>(pIndex_));
return Status::OK();
@ -148,5 +148,5 @@ Status FaissExecutionEngine::Cache() {
} // namespace engine
} // namespace vecwise
} // namespace milvus
} // namespace zilliz

View File

@ -15,7 +15,7 @@ namespace faiss {
}
namespace zilliz {
namespace vecwise {
namespace milvus {
namespace engine {
@ -68,5 +68,5 @@ protected:
} // namespace engine
} // namespace vecwise
} // namespace milvus
} // namespace zilliz

View File

@ -10,7 +10,7 @@
#include <iostream>
namespace zilliz {
namespace vecwise {
namespace milvus {
namespace engine {
IDGenerator::~IDGenerator() {}
@ -49,5 +49,5 @@ void SimpleIDGenerator::GetNextIDNumbers(size_t n, IDNumbers& ids) {
} // namespace engine
} // namespace vecwise
} // namespace milvus
} // namespace zilliz

View File

@ -11,7 +11,7 @@
#include <vector>
namespace zilliz {
namespace vecwise {
namespace milvus {
namespace engine {
class IDGenerator {
@ -37,5 +37,5 @@ private:
} // namespace engine
} // namespace vecwise
} // namespace milvus
} // namespace zilliz

View File

@ -8,7 +8,7 @@
#include <easylogging++.h>
namespace zilliz {
namespace vecwise {
namespace milvus {
namespace engine {
#define ENGINE_DOMAIN_NAME "[ENGINE] "

View File

@ -15,7 +15,7 @@
#include <easylogging++.h>
namespace zilliz {
namespace vecwise {
namespace milvus {
namespace engine {
MemVectors::MemVectors(const std::shared_ptr<meta::Meta>& meta_ptr,
@ -144,5 +144,5 @@ Status MemManager::Serialize(std::vector<std::string>& table_ids) {
} // namespace engine
} // namespace vecwise
} // namespace milvus
} // namespace zilliz

View File

@ -17,7 +17,7 @@
#include <mutex>
namespace zilliz {
namespace vecwise {
namespace milvus {
namespace engine {
namespace meta {
@ -92,5 +92,5 @@ private:
} // namespace engine
} // namespace vecwise
} // namespace milvus
} // namespace zilliz

View File

@ -9,7 +9,7 @@
#include <stdio.h>
namespace zilliz {
namespace vecwise {
namespace milvus {
namespace engine {
namespace meta {
@ -44,5 +44,5 @@ DateT Meta::GetDate() {
} // namespace meta
} // namespace engine
} // namespace vecwise
} // namespace milvus
} // namespace zilliz

View File

@ -14,7 +14,7 @@
#include <memory>
namespace zilliz {
namespace vecwise {
namespace milvus {
namespace engine {
namespace meta {
@ -70,5 +70,5 @@ public:
} // namespace meta
} // namespace engine
} // namespace vecwise
} // namespace milvus
} // namespace zilliz

View File

@ -6,7 +6,7 @@
#pragma once
namespace zilliz {
namespace vecwise {
namespace milvus {
namespace engine {
namespace meta {
@ -28,5 +28,5 @@ const size_t W_SEC = 7*D_SEC;
} // namespace meta
} // namespace engine
} // namespace vecwise
} // namespace milvus
} // namespace zilliz

View File

@ -12,7 +12,7 @@
#include <string>
namespace zilliz {
namespace vecwise {
namespace milvus {
namespace engine {
namespace meta {
@ -58,5 +58,5 @@ typedef std::map<DateT, TableFilesSchema> DatePartionedTableFilesSchema;
} // namespace meta
} // namespace engine
} // namespace vecwise
} // namespace milvus
} // namespace zilliz

View File

@ -14,7 +14,7 @@
#include "Exception.h"
namespace zilliz {
namespace vecwise {
namespace milvus {
namespace engine {
Options::Options()
@ -71,5 +71,5 @@ void ArchiveConf::ParseType(const std::string& type) {
}
} // namespace engine
} // namespace vecwise
} // namespace milvus
} // namespace zilliz

View File

@ -10,7 +10,7 @@
#include <map>
namespace zilliz {
namespace vecwise {
namespace milvus {
namespace engine {
class Env;
@ -49,5 +49,5 @@ struct Options {
} // namespace engine
} // namespace vecwise
} // namespace milvus
} // namespace zilliz

View File

@ -9,7 +9,7 @@
#include "Status.h"
namespace zilliz {
namespace vecwise {
namespace milvus {
namespace engine {
const char* Status::CopyState(const char* state) {
@ -63,5 +63,5 @@ std::string Status::ToString() const {
}
} // namespace engine
} // namespace vecwise
} // namespace milvus
} // namespace zilliz

View File

@ -8,7 +8,7 @@
#include <string>
namespace zilliz {
namespace vecwise {
namespace milvus {
namespace engine {
class Status {
@ -90,5 +90,5 @@ inline Status& Status::operator=(Status&& rhs) noexcept {
}
} // namespace engine
} // namespace vecwise
} // namespace milvus
} // namespace zilliz

View File

@ -8,7 +8,7 @@
#include <vector>
namespace zilliz {
namespace vecwise {
namespace milvus {
namespace engine {
typedef long IDNumber;
@ -20,5 +20,5 @@ typedef std::vector<QueryResult> QueryResults;
} // namespace engine
} // namespace vecwise
} // namespace milvus
} // namespace zilliz

View File

@ -8,7 +8,7 @@
#include <chrono>
namespace zilliz {
namespace vecwise {
namespace milvus {
namespace engine {
namespace utils {
@ -22,5 +22,5 @@ long GetMicroSecTimeStamp() {
} // namespace utils
} // namespace engine
} // namespace vecwise
} // namespace milvus
} // namespace zilliz

View File

@ -7,7 +7,7 @@
namespace zilliz {
namespace vecwise {
namespace milvus {
namespace engine {
namespace utils {
@ -15,5 +15,5 @@ long GetMicroSecTimeStamp();
} // namespace utils
} // namespace engine
} // namespace vecwise
} // namespace milvus
} // namespace zilliz

View File

@ -9,7 +9,7 @@
#include "SearchContext.h"
namespace zilliz {
namespace vecwise {
namespace milvus {
namespace engine {
class IScheduleStrategy {

View File

@ -10,15 +10,9 @@
#include "utils/Log.h"
namespace zilliz {
namespace vecwise {
namespace milvus {
namespace engine {
IndexLoaderQueue&
IndexLoaderQueue::GetInstance() {
static IndexLoaderQueue instance;
return instance;
}
void
IndexLoaderQueue::Put(const SearchContextPtr &search_context) {
std::unique_lock <std::mutex> lock(mtx);
@ -26,6 +20,7 @@ IndexLoaderQueue::Put(const SearchContextPtr &search_context) {
if(search_context == nullptr) {
queue_.push_back(nullptr);
empty_.notify_all();
return;
}

View File

@ -14,7 +14,7 @@
namespace zilliz {
namespace vecwise {
namespace milvus {
namespace engine {
@ -26,18 +26,15 @@ public:
using IndexLoaderContextPtr = std::shared_ptr<IndexLoaderContext>;
class IndexLoaderQueue {
private:
public:
IndexLoaderQueue() : mtx(), full_(), empty_() {}
IndexLoaderQueue(const IndexLoaderQueue &rhs) = delete;
IndexLoaderQueue &operator=(const IndexLoaderQueue &rhs) = delete;
public:
using LoaderQueue = std::list<IndexLoaderContextPtr>;
static IndexLoaderQueue& GetInstance();
void Put(const SearchContextPtr &search_context);
IndexLoaderContextPtr Take();

View File

@ -11,7 +11,7 @@
#include "utils/Log.h"
namespace zilliz {
namespace vecwise {
namespace milvus {
namespace engine {
class MemScheduleStrategy : public IScheduleStrategy {
@ -38,7 +38,7 @@ public:
new_loader->search_contexts_.push_back(search_context);
new_loader->file_ = pair.second;
auto index = zilliz::vecwise::cache::CpuCacheMgr::GetInstance()->GetIndex(pair.second->location_);
auto index = zilliz::milvus::cache::CpuCacheMgr::GetInstance()->GetIndex(pair.second->location_);
if(index != nullptr) {
//if the index file has been in memory, increase its priority
loader_list.push_front(new_loader);

View File

@ -8,7 +8,7 @@
#include "IScheduleStrategy.h"
namespace zilliz {
namespace vecwise {
namespace milvus {
namespace engine {
class StrategyFactory {

View File

@ -10,7 +10,7 @@
#include <chrono>
namespace zilliz {
namespace vecwise {
namespace milvus {
namespace engine {
SearchContext::SearchContext(uint64_t topk, uint64_t nq, const float* vectors)

View File

@ -13,7 +13,7 @@
#include <condition_variable>
namespace zilliz {
namespace vecwise {
namespace milvus {
namespace engine {
using TableFileSchemaPtr = std::shared_ptr<meta::TableFileSchema>;

View File

@ -13,7 +13,7 @@
#include "db/EngineFactory.h"
namespace zilliz {
namespace vecwise {
namespace milvus {
namespace engine {
namespace {
@ -55,8 +55,7 @@ void CollectDurationMetrics(int index_type, double total_time) {
}
SearchScheduler::SearchScheduler()
: thread_pool_(2),
stopped_(true) {
: stopped_(true) {
Start();
}
@ -75,8 +74,13 @@ SearchScheduler::Start() {
return true;
}
thread_pool_.enqueue(&SearchScheduler::IndexLoadWorker, this);
thread_pool_.enqueue(&SearchScheduler::SearchWorker, this);
stopped_ = false;
search_queue_.SetCapacity(2);
index_load_thread_ = std::make_shared<std::thread>(&SearchScheduler::IndexLoadWorker, this);
search_thread_ = std::make_shared<std::thread>(&SearchScheduler::SearchWorker, this);
return true;
}
@ -86,29 +90,34 @@ SearchScheduler::Stop() {
return true;
}
IndexLoaderQueue& index_queue = IndexLoaderQueue::GetInstance();
index_queue.Put(nullptr);
if(index_load_thread_) {
index_load_queue_.Put(nullptr);
index_load_thread_->join();
index_load_thread_ = nullptr;
}
SearchTaskQueue& search_queue = SearchTaskQueue::GetInstance();
search_queue.Put(nullptr);
if(search_thread_) {
search_queue_.Put(nullptr);
search_thread_->join();
search_thread_ = nullptr;
}
stopped_ = true;
return true;
}
bool
SearchScheduler::ScheduleSearchTask(SearchContextPtr& search_context) {
IndexLoaderQueue& index_queue = IndexLoaderQueue::GetInstance();
index_queue.Put(search_context);
index_load_queue_.Put(search_context);
return true;
}
bool
SearchScheduler::IndexLoadWorker() {
IndexLoaderQueue& index_queue = IndexLoaderQueue::GetInstance();
SearchTaskQueue& search_queue = SearchTaskQueue::GetInstance();
while(true) {
IndexLoaderContextPtr context = index_queue.Take();
IndexLoaderContextPtr context = index_load_queue_.Take();
if(context == nullptr) {
SERVER_LOG_INFO << "Stop thread for index loading";
break;//exit
@ -137,7 +146,7 @@ SearchScheduler::IndexLoadWorker() {
task_ptr->index_type_ = context->file_->file_type_;
task_ptr->index_engine_ = index_ptr;
task_ptr->search_contexts_.swap(context->search_contexts_);
search_queue.Put(task_ptr);
search_queue_.Put(task_ptr);
}
return true;
@ -145,9 +154,8 @@ SearchScheduler::IndexLoadWorker() {
bool
SearchScheduler::SearchWorker() {
SearchTaskQueue& search_queue = SearchTaskQueue::GetInstance();
while(true) {
SearchTaskPtr task_ptr = search_queue.Take();
SearchTaskPtr task_ptr = search_queue_.Take();
if(task_ptr == nullptr) {
SERVER_LOG_INFO << "Stop thread for searching";
break;//exit

View File

@ -6,10 +6,11 @@
#pragma once
#include "SearchContext.h"
#include "utils/ThreadPool.h"
#include "IndexLoaderQueue.h"
#include "SearchTaskQueue.h"
namespace zilliz {
namespace vecwise {
namespace milvus {
namespace engine {
class SearchScheduler {
@ -30,7 +31,12 @@ private:
bool SearchWorker();
private:
server::ThreadPool thread_pool_;
std::shared_ptr<std::thread> index_load_thread_;
std::shared_ptr<std::thread> search_thread_;
IndexLoaderQueue index_load_queue_;
SearchTaskQueue search_queue_;
bool stopped_ = true;
};

View File

@ -8,7 +8,7 @@
#include "utils/TimeRecorder.h"
namespace zilliz {
namespace vecwise {
namespace milvus {
namespace engine {
namespace {
@ -21,7 +21,7 @@ void ClusterResult(const std::vector<long> &output_ids,
for (auto i = 0; i < nq; i++) {
SearchContext::Id2ScoreMap id_score;
for (auto k = 0; k < topk; k++) {
uint64_t index = i * nq + k;
uint64_t index = i * topk + k;
id_score.push_back(std::make_pair(output_ids[index], output_distence[index]));
}
result_set.emplace_back(id_score);
@ -94,18 +94,6 @@ void CalcScore(uint64_t vector_count,
}
SearchTaskQueue::SearchTaskQueue() {
SetCapacity(4);
}
SearchTaskQueue&
SearchTaskQueue::GetInstance() {
static SearchTaskQueue instance;
return instance;
}
bool SearchTask::DoSearch() {
if(index_engine_ == nullptr) {
return false;
@ -125,28 +113,29 @@ bool SearchTask::DoSearch() {
try {
index_engine_->Search(context->nq(), context->vectors(), inner_k, output_distence.data(),
output_ids.data());
rc.Record("do search");
//step 3: cluster result
SearchContext::ResultSet result_set;
ClusterResult(output_ids, output_distence, context->nq(), inner_k, result_set);
rc.Record("cluster result");
//step 4: pick up topk result
TopkResult(result_set, inner_k, context->GetResult());
rc.Record("reduce topk");
//step 5: calculate score between 0 ~ 100
CalcScore(context->nq(), context->vectors(), index_engine_->Dimension(), context->GetResult(), result_set);
context->GetResult().swap(result_set);
rc.Record("calculate score");
} catch (std::exception& ex) {
SERVER_LOG_ERROR << "SearchTask encounter exception: " << ex.what();
context->IndexSearchDone(index_id_);//mark as done avoid dead lock, even search failed
continue;
}
rc.Record("do search");
//step 3: cluster result
SearchContext::ResultSet result_set;
ClusterResult(output_ids, output_distence, context->nq(), inner_k, result_set);
rc.Record("cluster result");
//step 4: pick up topk result
TopkResult(result_set, inner_k, context->GetResult());
rc.Record("reduce topk");
//step 5: calculate score between 0 ~ 100
CalcScore(context->nq(), context->vectors(), index_engine_->Dimension(), context->GetResult(), result_set);
context->GetResult().swap(result_set);
rc.Record("calculate score");
//step 6: notify to send result to client
context->IndexSearchDone(index_id_);
}

View File

@ -12,7 +12,7 @@
#include <memory>
namespace zilliz {
namespace vecwise {
namespace milvus {
namespace engine {
class SearchTask {
@ -27,21 +27,7 @@ public:
};
using SearchTaskPtr = std::shared_ptr<SearchTask>;
class SearchTaskQueue : public server::BlockingQueue<SearchTaskPtr> {
private:
SearchTaskQueue();
SearchTaskQueue(const SearchTaskQueue &rhs) = delete;
SearchTaskQueue &operator=(const SearchTaskQueue &rhs) = delete;
public:
static SearchTaskQueue& GetInstance();
private:
};
using SearchTaskQueue = server::BlockingQueue<SearchTaskPtr>;
}

View File

@ -7,7 +7,7 @@
#include <getopt.h>
#include <memory.h>
// Not provide path: current work path will be used and system.info.
using namespace zilliz::vecwise;
using namespace zilliz::milvus;
void
print_usage(const std::string &app_name) {

View File

@ -14,7 +14,7 @@
namespace zilliz {
namespace vecwise {
namespace milvus {
namespace server {
LicenseCheck::LicenseCheck() {

View File

@ -9,7 +9,7 @@
#include <memory>
namespace zilliz {
namespace vecwise {
namespace milvus {
namespace server {
class LicenseCheck {

View File

@ -8,7 +8,7 @@
#include "utils/Error.h"
using namespace zilliz::vecwise;
using namespace zilliz::milvus;
// Not provide path: current work path will be used and system.info.
void

View File

@ -21,7 +21,7 @@
namespace zilliz {
namespace vecwise {
namespace milvus {
namespace server {
constexpr int LicenseLibrary::sha256_length_;

View File

@ -15,7 +15,7 @@
namespace zilliz {
namespace vecwise {
namespace milvus {
namespace server {
class LicenseLibrary {

View File

@ -22,12 +22,12 @@ INITIALIZE_EASYLOGGINGPP
void print_help(const std::string &app_name);
using namespace zilliz::vecwise;
using namespace zilliz::milvus;
int
main(int argc, char *argv[]) {
printf("Megasearch %s version: v%s built at %s\n", BUILD_TYPE, MEGASEARCH_VERSION, BUILD_TIME);
printf("Megasearch server start...\n");
printf("\nWelcome to use Milvus by Zillz!\n");
printf("Milvus %s version: v%s built at %s\n", BUILD_TYPE, MEGASEARCH_VERSION, BUILD_TIME);
signal(SIGINT, server::SignalUtil::HandleSignal);
signal(SIGSEGV, server::SignalUtil::HandleSignal);
@ -53,7 +53,7 @@ main(int argc, char *argv[]) {
if(argc < 2) {
print_help(app_name);
printf("Vecwise engine server exit...\n");
printf("Milvus server exit...\n");
return EXIT_FAILURE;
}
@ -98,7 +98,7 @@ main(int argc, char *argv[]) {
}
}
zilliz::vecwise::server::InitLog(log_config_file);
zilliz::milvus::server::InitLog(log_config_file);
server::Server* server_ptr = server::Server::Instance();
server_ptr->Init(start_daemonized, pid_filename, config_filename);

View File

@ -11,7 +11,7 @@
#include "SystemInfo.h"
namespace zilliz {
namespace vecwise {
namespace milvus {
namespace server {
class MetricsBase{
public:

View File

@ -9,7 +9,7 @@
#include "PrometheusMetrics.h"
namespace zilliz {
namespace vecwise {
namespace milvus {
namespace server {
MetricsBase &

View File

@ -16,7 +16,7 @@
//#include "PrometheusMetrics.h"
namespace zilliz {
namespace vecwise {
namespace milvus {
namespace server {
#define METRICS_NOW_TIME std::chrono::system_clock::now()

View File

@ -10,7 +10,7 @@
namespace zilliz {
namespace vecwise {
namespace milvus {
namespace server {
ServerError

View File

@ -22,7 +22,7 @@
namespace zilliz {
namespace vecwise {
namespace milvus {
namespace server {

View File

@ -17,7 +17,7 @@
namespace zilliz {
namespace vecwise {
namespace milvus {
namespace server {
void SystemInfo::Init() {

View File

@ -20,7 +20,7 @@
namespace zilliz {
namespace vecwise {
namespace milvus {
namespace server {
class SystemInfo {

View File

@ -9,7 +9,7 @@
#include "utils/TimeRecorder.h"
namespace zilliz {
namespace vecwise {
namespace milvus {
namespace server {
using namespace megasearch;

View File

@ -11,7 +11,7 @@
#include "MegasearchService.h"
namespace zilliz {
namespace vecwise {
namespace milvus {
namespace server {
class MegasearchServiceHandler : virtual public megasearch::thrift::MegasearchServiceIf {

View File

@ -10,7 +10,7 @@
#include "megasearch_constants.h"
namespace zilliz {
namespace vecwise {
namespace milvus {
namespace server {
using namespace megasearch;

View File

@ -12,7 +12,7 @@
#include <thread>
namespace zilliz {
namespace vecwise {
namespace milvus {
namespace server {
class BaseTask {

View File

@ -23,7 +23,7 @@
#include <thread>
namespace zilliz {
namespace vecwise {
namespace milvus {
namespace server {
using namespace megasearch::thrift;

View File

@ -9,7 +9,7 @@
#include <string>
namespace zilliz {
namespace vecwise {
namespace milvus {
namespace server {
class MegasearchServer {

View File

@ -14,21 +14,21 @@
#include "version.h"
namespace zilliz {
namespace vecwise {
namespace milvus {
namespace server {
static const std::string DQL_TASK_GROUP = "dql";
static const std::string DDL_DML_TASK_GROUP = "ddl_dml";
static const std::string PING_TASK_GROUP = "ping";
using DB_META = zilliz::vecwise::engine::meta::Meta;
using DB_DATE = zilliz::vecwise::engine::meta::DateT;
using DB_META = zilliz::milvus::engine::meta::Meta;
using DB_DATE = zilliz::milvus::engine::meta::DateT;
namespace {
class DBWrapper {
public:
DBWrapper() {
zilliz::vecwise::engine::Options opt;
zilliz::milvus::engine::Options opt;
ConfigNode& config = ServerConfig::GetInstance().GetConfig(CONFIG_DB);
opt.meta.backend_uri = config.GetValue(CONFIG_DB_URL);
std::string db_path = config.GetValue(CONFIG_DB_PATH);
@ -37,7 +37,7 @@ namespace {
CommonUtil::CreateDirectory(opt.meta.path);
zilliz::vecwise::engine::DB::Open(opt, &db_);
zilliz::milvus::engine::DB::Open(opt, &db_);
if(db_ == nullptr) {
SERVER_LOG_ERROR << "Failed to open db";
throw ServerException(SERVER_NULL_POINTER, "Failed to open db");
@ -48,13 +48,13 @@ namespace {
delete db_;
}
zilliz::vecwise::engine::DB* DB() { return db_; }
zilliz::milvus::engine::DB* DB() { return db_; }
private:
zilliz::vecwise::engine::DB* db_ = nullptr;
zilliz::milvus::engine::DB* db_ = nullptr;
};
zilliz::vecwise::engine::DB* DB() {
zilliz::milvus::engine::DB* DB() {
static DBWrapper db_wrapper;
return db_wrapper.DB();
}

View File

@ -16,7 +16,7 @@
#include <memory>
namespace zilliz {
namespace vecwise {
namespace milvus {
namespace server {
using namespace megasearch;

View File

@ -7,7 +7,7 @@
#include "MegasearchThreadPoolServer.h"
namespace zilliz {
namespace vecwise {
namespace milvus {
namespace server {
void
@ -21,7 +21,7 @@ MegasearchThreadPoolServer::onClientDisconnected(apache::thrift::server::TConnec
server::Metrics::GetInstance().ConnectionGaugeDecrement();
TThreadPoolServer::onClientDisconnected(pClient);
}
zilliz::vecwise::server::MegasearchThreadPoolServer::MegasearchThreadPoolServer(const std::shared_ptr<apache::thrift::TProcessor> &processor,
zilliz::milvus::server::MegasearchThreadPoolServer::MegasearchThreadPoolServer(const std::shared_ptr<apache::thrift::TProcessor> &processor,
const std::shared_ptr<apache::thrift::transport::TServerTransport> &serverTransport,
const std::shared_ptr<apache::thrift::transport::TTransportFactory> &transportFactory,
const std::shared_ptr<apache::thrift::protocol::TProtocolFactory> &protocolFactory,

View File

@ -10,7 +10,7 @@
namespace zilliz {
namespace vecwise {
namespace milvus {
namespace server {
class MegasearchThreadPoolServer : public apache::thrift::server::TThreadPoolServer {

View File

@ -23,7 +23,7 @@
#include "metrics/Metrics.h"
namespace zilliz {
namespace vecwise {
namespace milvus {
namespace server {
Server*
@ -52,7 +52,7 @@ Server::Daemonize() {
return;
}
SERVER_LOG_INFO << "Vecwise server run in daemonize mode";
SERVER_LOG_INFO << "Milvus server run in daemonize mode";
// std::string log_path(GetLogDirFullPath());
// log_path += "zdb_server.(INFO/WARNNING/ERROR/CRITICAL)";
@ -152,9 +152,6 @@ Server::Start() {
ServerConfig &config = ServerConfig::GetInstance();
ConfigNode server_config = config.GetConfig(CONFIG_SERVER);
//print config into console and log
config.PrintAll();
#ifdef ENABLE_LICENSE
ConfigNode license_config = config.GetConfig(CONFIG_LICENSE);
std::string license_file_path = license_config.GetValue(CONFIG_LICENSE_PATH);
@ -174,11 +171,11 @@ Server::Start() {
signal(SIGTERM, SignalUtil::HandleSignal);
server::Metrics::GetInstance().Init();
server::SystemInfo::GetInstance().Init();
SERVER_LOG_INFO << "Vecwise server is running...";
printf("Milvus server start successfully.\n");
StartService();
} catch(std::exception& ex){
SERVER_LOG_ERROR << "Vecwise server encounter exception: " << std::string(ex.what())
SERVER_LOG_ERROR << "Milvus server encounter exception: " << std::string(ex.what())
<< "Is another server instance running?";
break;
}
@ -190,17 +187,19 @@ Server::Start() {
void
Server::Stop() {
SERVER_LOG_INFO << "Vecwise server will be closed";
printf("Milvus server is going to shutdown ...\n");
// Unlock and close lockfile
if (pid_fd != -1) {
int ret = lockf(pid_fd, F_ULOCK, 0);
if(ret != 0){
printf("Can't lock file: %s\n", strerror(errno));
exit(0);
}
ret = close(pid_fd);
if(ret != 0){
printf("Can't close file: %s\n", strerror(errno));
exit(0);
}
}
@ -208,7 +207,8 @@ Server::Stop() {
if (!pid_filename_.empty()) {
int ret = unlink(pid_filename_.c_str());
if(ret != 0){
printf("Can't unlink file: %s\n", strerror(errno));
exit(0);
}
}
@ -219,8 +219,7 @@ Server::Stop() {
#ifdef ENABLE_LICENSE
server::LicenseCheck::GetInstance().StopCountingDown();
#endif
SERVER_LOG_INFO << "Vecwise server closed";
printf("Milvus server is closed!\n");
}

View File

@ -11,7 +11,7 @@
#include <string>
namespace zilliz {
namespace vecwise {
namespace milvus {
namespace server {
class Server {

View File

@ -14,7 +14,7 @@
#include "config/IConfigMgr.h"
namespace zilliz {
namespace vecwise {
namespace milvus {
namespace server {
ServerConfig&

View File

@ -11,7 +11,7 @@
#include "yaml-cpp/yaml.h"
namespace zilliz {
namespace vecwise {
namespace milvus {
namespace server {
static const std::string CONFIG_SERVER = "server_config";

View File

@ -8,7 +8,7 @@
#include "StringHelpFunctions.h"
namespace zilliz {
namespace vecwise {
namespace milvus {
namespace server {

View File

@ -10,7 +10,7 @@
#include "Error.h"
namespace zilliz {
namespace vecwise {
namespace milvus {
namespace server {
using AttribMap = std::map<std::string, std::string>;

View File

@ -12,7 +12,7 @@
#include <vector>
namespace zilliz {
namespace vecwise {
namespace milvus {
namespace server {
template<typename T>

View File

@ -4,7 +4,7 @@
#include "Error.h"
namespace zilliz {
namespace vecwise {
namespace milvus {
namespace server {
template<typename T>

Some files were not shown because too many files have changed in this diff Show More