milvus-io
/
milvus
mirror of https://github.com/milvus-io/milvus.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'branch-0.4.0' into change_top_k_limitation 

Former-commit-id: 3d27803138b5e5371de1e68e9ea2ba167acdaac5
pull/191/head
Heisenberg 2019-09-06 21:35:26 +08:00
parent 248bdbf08d 26b30c9a1d
commit 3cb39ba157
48 changed files with 794 additions and 840 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
cpp/build.sh
View File

@ -83,10 +83,6 @@ if [[ ! -d cmake_build ]]; then
MAKE_CLEAN="ON" MAKE_CLEAN="ON"
fi fi
#pushd `pwd`/src/core
#./build.sh ${KNOWHERE_OPTIONS}
#popd
cd cmake_build cd cmake_build
CUDA_COMPILER=/usr/local/cuda/bin/nvcc CUDA_COMPILER=/usr/local/cuda/bin/nvcc

2
cpp/src/config/ConfigMgr.h
View File

@ -30,7 +30,7 @@ class ConfigMgr {
public: public:
static ConfigMgr* GetInstance(); static ConfigMgr* GetInstance();
virtual ServerError LoadConfigFile(const std::string &filename) = 0; virtual ErrorCode LoadConfigFile(const std::string &filename) = 0;
virtual void Print() const = 0;//will be deleted virtual void Print() const = 0;//will be deleted
virtual std::string DumpString() const = 0; virtual std::string DumpString() const = 0;

2
cpp/src/config/YamlConfigMgr.cpp
View File

@ -12,7 +12,7 @@ namespace zilliz {
namespace milvus { namespace milvus {
namespace server { namespace server {
ServerError YamlConfigMgr::LoadConfigFile(const std::string &filename) { ErrorCode YamlConfigMgr::LoadConfigFile(const std::string &filename) {
struct stat directoryStat; struct stat directoryStat;
int statOK = stat(filename.c_str(), &directoryStat); int statOK = stat(filename.c_str(), &directoryStat);
if (statOK != 0) { if (statOK != 0) {

2
cpp/src/config/YamlConfigMgr.h
View File

@ -17,7 +17,7 @@ namespace server {
class YamlConfigMgr : public ConfigMgr { class YamlConfigMgr : public ConfigMgr {
public: public:
virtual ServerError LoadConfigFile(const std::string &filename); virtual ErrorCode LoadConfigFile(const std::string &filename);
virtual void Print() const; virtual void Print() const;
virtual std::string DumpString() const; virtual std::string DumpString() const;

42
cpp/src/db/DBImpl.cpp
View File

@ -100,7 +100,7 @@ Status DBImpl::DropAll() {
Status DBImpl::CreateTable(meta::TableSchema& table_schema) { Status DBImpl::CreateTable(meta::TableSchema& table_schema) {
if (shutting_down_.load(std::memory_order_acquire)){ if (shutting_down_.load(std::memory_order_acquire)){
return Status::Error("Milsvus server is shutdown!"); return Status(DB_ERROR, "Milsvus server is shutdown!");
} }
meta::TableSchema temp_schema = table_schema; meta::TableSchema temp_schema = table_schema;
@ -110,7 +110,7 @@ Status DBImpl::CreateTable(meta::TableSchema& table_schema) {
Status DBImpl::DeleteTable(const std::string& table_id, const meta::DatesT& dates) { Status DBImpl::DeleteTable(const std::string& table_id, const meta::DatesT& dates) {
if (shutting_down_.load(std::memory_order_acquire)){ if (shutting_down_.load(std::memory_order_acquire)){
return Status::Error("Milsvus server is shutdown!"); return Status(DB_ERROR, "Milsvus server is shutdown!");
} }
//dates partly delete files of the table but currently we don't support //dates partly delete files of the table but currently we don't support
@ -136,7 +136,7 @@ Status DBImpl::DeleteTable(const std::string& table_id, const meta::DatesT& date
Status DBImpl::DescribeTable(meta::TableSchema& table_schema) { Status DBImpl::DescribeTable(meta::TableSchema& table_schema) {
if (shutting_down_.load(std::memory_order_acquire)){ if (shutting_down_.load(std::memory_order_acquire)){
return Status::Error("Milsvus server is shutdown!"); return Status(DB_ERROR, "Milsvus server is shutdown!");
} }
auto stat = meta_ptr_->DescribeTable(table_schema); auto stat = meta_ptr_->DescribeTable(table_schema);
@ -146,7 +146,7 @@ Status DBImpl::DescribeTable(meta::TableSchema& table_schema) {
Status DBImpl::HasTable(const std::string& table_id, bool& has_or_not) { Status DBImpl::HasTable(const std::string& table_id, bool& has_or_not) {
if (shutting_down_.load(std::memory_order_acquire)){ if (shutting_down_.load(std::memory_order_acquire)){
return Status::Error("Milsvus server is shutdown!"); return Status(DB_ERROR, "Milsvus server is shutdown!");
} }
return meta_ptr_->HasTable(table_id, has_or_not); return meta_ptr_->HasTable(table_id, has_or_not);
@ -154,7 +154,7 @@ Status DBImpl::HasTable(const std::string& table_id, bool& has_or_not) {
Status DBImpl::AllTables(std::vector<meta::TableSchema>& table_schema_array) { Status DBImpl::AllTables(std::vector<meta::TableSchema>& table_schema_array) {
if (shutting_down_.load(std::memory_order_acquire)){ if (shutting_down_.load(std::memory_order_acquire)){
return Status::Error("Milsvus server is shutdown!"); return Status(DB_ERROR, "Milsvus server is shutdown!");
} }
return meta_ptr_->AllTables(table_schema_array); return meta_ptr_->AllTables(table_schema_array);
@ -162,7 +162,7 @@ Status DBImpl::AllTables(std::vector<meta::TableSchema>& table_schema_array) {
Status DBImpl::PreloadTable(const std::string &table_id) { Status DBImpl::PreloadTable(const std::string &table_id) {
if (shutting_down_.load(std::memory_order_acquire)){ if (shutting_down_.load(std::memory_order_acquire)){
return Status::Error("Milsvus server is shutdown!"); return Status(DB_ERROR, "Milsvus server is shutdown!");
} }
meta::DatePartionedTableFilesSchema files; meta::DatePartionedTableFilesSchema files;
@ -184,7 +184,7 @@ Status DBImpl::PreloadTable(const std::string &table_id) {
ExecutionEnginePtr engine = EngineFactory::Build(file.dimension_, file.location_, (EngineType)file.engine_type_, (MetricType)file.metric_type_, file.nlist_); ExecutionEnginePtr engine = EngineFactory::Build(file.dimension_, file.location_, (EngineType)file.engine_type_, (MetricType)file.metric_type_, file.nlist_);
if(engine == nullptr) { if(engine == nullptr) {
ENGINE_LOG_ERROR << "Invalid engine type"; ENGINE_LOG_ERROR << "Invalid engine type";
return Status::Error("Invalid engine type"); return Status(DB_ERROR, "Invalid engine type");
} }
size += engine->PhysicalSize(); size += engine->PhysicalSize();
@ -197,7 +197,7 @@ Status DBImpl::PreloadTable(const std::string &table_id) {
} catch (std::exception &ex) { } catch (std::exception &ex) {
std::string msg = "Pre-load table encounter exception: " + std::string(ex.what()); std::string msg = "Pre-load table encounter exception: " + std::string(ex.what());
ENGINE_LOG_ERROR << msg; ENGINE_LOG_ERROR << msg;
return Status::Error(msg); return Status(DB_ERROR, msg);
} }
} }
} }
@ -207,7 +207,7 @@ Status DBImpl::PreloadTable(const std::string &table_id) {
Status DBImpl::UpdateTableFlag(const std::string &table_id, int64_t flag) { Status DBImpl::UpdateTableFlag(const std::string &table_id, int64_t flag) {
if (shutting_down_.load(std::memory_order_acquire)){ if (shutting_down_.load(std::memory_order_acquire)){
return Status::Error("Milsvus server is shutdown!"); return Status(DB_ERROR, "Milsvus server is shutdown!");
} }
return meta_ptr_->UpdateTableFlag(table_id, flag); return meta_ptr_->UpdateTableFlag(table_id, flag);
@ -215,7 +215,7 @@ Status DBImpl::UpdateTableFlag(const std::string &table_id, int64_t flag) {
Status DBImpl::GetTableRowCount(const std::string& table_id, uint64_t& row_count) { Status DBImpl::GetTableRowCount(const std::string& table_id, uint64_t& row_count) {
if (shutting_down_.load(std::memory_order_acquire)){ if (shutting_down_.load(std::memory_order_acquire)){
return Status::Error("Milsvus server is shutdown!"); return Status(DB_ERROR, "Milsvus server is shutdown!");
} }
return meta_ptr_->Count(table_id, row_count); return meta_ptr_->Count(table_id, row_count);
@ -225,7 +225,7 @@ Status DBImpl::InsertVectors(const std::string& table_id_,
uint64_t n, const float* vectors, IDNumbers& vector_ids_) { uint64_t n, const float* vectors, IDNumbers& vector_ids_) {
// ENGINE_LOG_DEBUG << "Insert " << n << " vectors to cache"; // ENGINE_LOG_DEBUG << "Insert " << n << " vectors to cache";
if (shutting_down_.load(std::memory_order_acquire)){ if (shutting_down_.load(std::memory_order_acquire)){
return Status::Error("Milsvus server is shutdown!"); return Status(DB_ERROR, "Milsvus server is shutdown!");
} }
Status status; Status status;
@ -314,7 +314,7 @@ Status DBImpl::DropIndex(const std::string& table_id) {
Status DBImpl::Query(const std::string &table_id, uint64_t k, uint64_t nq, uint64_t nprobe, Status DBImpl::Query(const std::string &table_id, uint64_t k, uint64_t nq, uint64_t nprobe,
const float *vectors, QueryResults &results) { const float *vectors, QueryResults &results) {
if (shutting_down_.load(std::memory_order_acquire)){ if (shutting_down_.load(std::memory_order_acquire)){
return Status::Error("Milsvus server is shutdown!"); return Status(DB_ERROR, "Milsvus server is shutdown!");
} }
meta::DatesT dates = {utils::GetDate()}; meta::DatesT dates = {utils::GetDate()};
@ -326,7 +326,7 @@ Status DBImpl::Query(const std::string &table_id, uint64_t k, uint64_t nq, uint6
Status DBImpl::Query(const std::string& table_id, uint64_t k, uint64_t nq, uint64_t nprobe, Status DBImpl::Query(const std::string& table_id, uint64_t k, uint64_t nq, uint64_t nprobe,
const float* vectors, const meta::DatesT& dates, QueryResults& results) { const float* vectors, const meta::DatesT& dates, QueryResults& results) {
if (shutting_down_.load(std::memory_order_acquire)){ if (shutting_down_.load(std::memory_order_acquire)){
return Status::Error("Milsvus server is shutdown!"); return Status(DB_ERROR, "Milsvus server is shutdown!");
} }
ENGINE_LOG_DEBUG << "Query by dates for table: " << table_id; ENGINE_LOG_DEBUG << "Query by dates for table: " << table_id;
@ -354,7 +354,7 @@ Status DBImpl::Query(const std::string& table_id, const std::vector<std::string>
uint64_t k, uint64_t nq, uint64_t nprobe, const float* vectors, uint64_t k, uint64_t nq, uint64_t nprobe, const float* vectors,
const meta::DatesT& dates, QueryResults& results) { const meta::DatesT& dates, QueryResults& results) {
if (shutting_down_.load(std::memory_order_acquire)){ if (shutting_down_.load(std::memory_order_acquire)){
return Status::Error("Milsvus server is shutdown!"); return Status(DB_ERROR, "Milsvus server is shutdown!");
} }
ENGINE_LOG_DEBUG << "Query by file ids for table: " << table_id; ENGINE_LOG_DEBUG << "Query by file ids for table: " << table_id;
@ -382,7 +382,7 @@ Status DBImpl::Query(const std::string& table_id, const std::vector<std::string>
} }
if(file_id_array.empty()) { if(file_id_array.empty()) {
return Status::Error("Invalid file id"); return Status(DB_ERROR, "Invalid file id");
} }
cache::CpuCacheMgr::GetInstance()->PrintInfo(); //print cache info before query cache::CpuCacheMgr::GetInstance()->PrintInfo(); //print cache info before query
@ -393,7 +393,7 @@ Status DBImpl::Query(const std::string& table_id, const std::vector<std::string>
Status DBImpl::Size(uint64_t& result) { Status DBImpl::Size(uint64_t& result) {
if (shutting_down_.load(std::memory_order_acquire)){ if (shutting_down_.load(std::memory_order_acquire)){
return Status::Error("Milsvus server is shutdown!"); return Status(DB_ERROR, "Milsvus server is shutdown!");
} }
return meta_ptr_->Size(result); return meta_ptr_->Size(result);
@ -600,7 +600,7 @@ Status DBImpl::MergeFiles(const std::string& table_id, const meta::DateT& date,
std::cout << "ERROR: failed to persist merged index file: " << table_file.location_ std::cout << "ERROR: failed to persist merged index file: " << table_file.location_
<< ", possible out of disk space" << std::endl; << ", possible out of disk space" << std::endl;
return Status::Error(msg); return Status(DB_ERROR, msg);
} }
//step 4: update table files state //step 4: update table files state
@ -708,7 +708,7 @@ Status DBImpl::BuildIndex(const meta::TableFileSchema& file) {
(MetricType)file.metric_type_, file.nlist_); (MetricType)file.metric_type_, file.nlist_);
if(to_index == nullptr) { if(to_index == nullptr) {
ENGINE_LOG_ERROR << "Invalid engine type"; ENGINE_LOG_ERROR << "Invalid engine type";
return Status::Error("Invalid engine type"); return Status(DB_ERROR, "Invalid engine type");
} }
try { try {
@ -755,7 +755,7 @@ Status DBImpl::BuildIndex(const meta::TableFileSchema& file) {
std::cout << "ERROR: failed to build index, index file is too large or gpu memory is not enough" << std::endl; std::cout << "ERROR: failed to build index, index file is too large or gpu memory is not enough" << std::endl;
return Status::Error(msg); return Status(DB_ERROR, msg);
} }
//step 4: if table has been deleted, dont save index file //step 4: if table has been deleted, dont save index file
@ -781,7 +781,7 @@ Status DBImpl::BuildIndex(const meta::TableFileSchema& file) {
std::cout << "ERROR: failed to persist index file: " << table_file.location_ std::cout << "ERROR: failed to persist index file: " << table_file.location_
<< ", possible out of disk space" << std::endl; << ", possible out of disk space" << std::endl;
return Status::Error(msg); return Status(DB_ERROR, msg);
} }
//step 6: update meta //step 6: update meta
@ -816,7 +816,7 @@ Status DBImpl::BuildIndex(const meta::TableFileSchema& file) {
} catch (std::exception& ex) { } catch (std::exception& ex) {
std::string msg = "Build index encounter exception: " + std::string(ex.what()); std::string msg = "Build index encounter exception: " + std::string(ex.what());
ENGINE_LOG_ERROR << msg; ENGINE_LOG_ERROR << msg;
return Status::Error(msg); return Status(DB_ERROR, msg);
} }
return Status::OK(); return Status::OK();

81
cpp/src/db/Status.cpp
View File

@ -12,57 +12,62 @@ namespace zilliz {
namespace milvus { namespace milvus {
namespace engine { namespace engine {
const char* Status::CopyState(const char* state) { constexpr int CODE_WIDTH = sizeof(ErrorCode);
uint32_t size;
std::memcpy(&size, state, sizeof(size)); Status::Status(ErrorCode code, const std::string& msg) {
char* result = new char[size+5]; //4 bytes store code
memcpy(result, state, size+5); //4 bytes store message length
return result; //the left bytes store message string
const uint32_t length = (uint32_t)msg.size();
char* result = new char[length + sizeof(length) + CODE_WIDTH];
std::memcpy(result, &code, CODE_WIDTH);
std::memcpy(result + CODE_WIDTH, &length, sizeof(length));
memcpy(result + sizeof(length) + CODE_WIDTH, msg.data(), length);
state_ = result;
} }
Status::Status(Code code, const std::string& msg, const std::string& msg2) { Status::Status()
assert(code != kOK); : state_(nullptr) {
const uint32_t len1 = msg.size();
const uint32_t len2 = msg2.size(); }
const uint32_t size = len1 + (len2 ? (2+len2) : 0);
char* result = new char[size+5]; Status::~Status() {
std::memcpy(result, &size, sizeof(size)); delete[] state_;
result[4] = static_cast<char>(code); }
memcpy(result+5, msg.data(), len1);
if (len2) { const char* Status::CopyState(const char* state) {
result[5 + len1] = ':'; uint32_t length = 0;
result[6 + len1] = ' '; std::memcpy(&length, state + CODE_WIDTH, sizeof(length));
memcpy(result + 7 + len1, msg2.data(), len2); int buff_len = length + sizeof(length) + CODE_WIDTH;
} char* result = new char[buff_len];
state_ = result; memcpy(result, state, buff_len);
return result;
} }
std::string Status::ToString() const { std::string Status::ToString() const {
if (state_ == nullptr) return "OK"; if (state_ == nullptr) return "OK";
char tmp[30]; char tmp[32];
const char* type; const char* type;
switch (code()) { switch (code()) {
case kOK: case DB_SUCCESS:
type = "OK"; type = "OK";
break; break;
case kNotFound: case DB_ERROR:
type = "NotFound: ";
break;
case kError:
type = "Error: "; type = "Error: ";
break; break;
case kInvalidDBPath: case DB_META_TRANSACTION_FAILED:
type = "InvalidDBPath: ";
break;
case kGroupError:
type = "GroupError: ";
break;
case kDBTransactionError:
type = "DBTransactionError: "; type = "DBTransactionError: ";
break; break;
case kAlreadyExist: case DB_NOT_FOUND:
type = "NotFound: ";
break;
case DB_ALREADY_EXIST:
type = "AlreadyExist: "; type = "AlreadyExist: ";
break; break;
case DB_INVALID_PATH:
type = "InvalidPath: ";
break;
default: default:
snprintf(tmp, sizeof(tmp), "Unkown code(%d): ", snprintf(tmp, sizeof(tmp), "Unkown code(%d): ",
static_cast<int>(code())); static_cast<int>(code()));
@ -71,9 +76,9 @@ std::string Status::ToString() const {
} }
std::string result(type); std::string result(type);
uint32_t length; uint32_t length = 0;
memcpy(&length, state_, sizeof(length)); memcpy(&length, state_ + CODE_WIDTH, sizeof(length));
result.append(state_ + 5, length); result.append(state_ + sizeof(length) + CODE_WIDTH, length);
return result; return result;
} }

66
cpp/src/db/Status.h
View File

@ -5,8 +5,9 @@
******************************************************************************/ ******************************************************************************/
#pragma once #pragma once
#include <string> #include "utils/Error.h"
#include <string>
namespace zilliz { namespace zilliz {
namespace milvus { namespace milvus {
@ -14,9 +15,9 @@ namespace engine {
class Status { class Status {
public: public:
Status() noexcept : state_(nullptr) {} Status(ErrorCode code, const std::string &msg);
Status();
~Status() { delete[] state_; } ~Status();
Status(const Status &rhs); Status(const Status &rhs);
@ -31,64 +32,17 @@ class Status {
static Status static Status
OK() { return Status(); } OK() { return Status(); }
static Status bool ok() const { return state_ == nullptr || code() == DB_SUCCESS; }
NotFound(const std::string &msg, const std::string &msg2 = "") {
return Status(kNotFound, msg, msg2);
}
static Status
Error(const std::string &msg, const std::string &msg2 = "") {
return Status(kError, msg, msg2);
}
static Status
InvalidDBPath(const std::string &msg, const std::string &msg2 = "") {
return Status(kInvalidDBPath, msg, msg2);
}
static Status
GroupError(const std::string &msg, const std::string &msg2 = "") {
return Status(kGroupError, msg, msg2);
}
static Status
DBTransactionError(const std::string &msg, const std::string &msg2 = "") {
return Status(kDBTransactionError, msg, msg2);
}
static Status
AlreadyExist(const std::string &msg, const std::string &msg2 = "") {
return Status(kAlreadyExist, msg, msg2);
}
bool ok() const { return state_ == nullptr; }
bool IsNotFound() const { return code() == kNotFound; }
bool IsError() const { return code() == kError; }
bool IsInvalidDBPath() const { return code() == kInvalidDBPath; }
bool IsGroupError() const { return code() == kGroupError; }
bool IsDBTransactionError() const { return code() == kDBTransactionError; }
bool IsAlreadyExist() const { return code() == kAlreadyExist; }
std::string ToString() const; std::string ToString() const;
ErrorCode code() const {
return (state_ == nullptr) ? DB_SUCCESS : *(ErrorCode*)(state_);
}
private: private:
const char *state_ = nullptr; const char *state_ = nullptr;
enum Code {
kOK = 0,
kNotFound,
kError,
kInvalidDBPath,
kGroupError,
kDBTransactionError,
kAlreadyExist,
};
Code code() const {
return (state_ == nullptr) ? kOK : static_cast<Code>(state_[4]);
}
Status(Code code, const std::string &msg, const std::string &msg2);
static const char *CopyState(const char *s); static const char *CopyState(const char *s);
}; // Status }; // Status

8
cpp/src/db/Utils.cpp
View File

@ -70,7 +70,7 @@ Status CreateTablePath(const DBMetaOptions& options, const std::string& table_id
auto status = server::CommonUtil::CreateDirectory(table_path); auto status = server::CommonUtil::CreateDirectory(table_path);
if (status != 0) { if (status != 0) {
ENGINE_LOG_ERROR << "Create directory " << table_path << " Error"; ENGINE_LOG_ERROR << "Create directory " << table_path << " Error";
return Status::Error("Failed to create table path"); return Status(DB_ERROR, "Failed to create table path");
} }
for(auto& path : options.slave_paths) { for(auto& path : options.slave_paths) {
@ -78,7 +78,7 @@ Status CreateTablePath(const DBMetaOptions& options, const std::string& table_id
status = server::CommonUtil::CreateDirectory(table_path); status = server::CommonUtil::CreateDirectory(table_path);
if (status != 0) { if (status != 0) {
ENGINE_LOG_ERROR << "Create directory " << table_path << " Error"; ENGINE_LOG_ERROR << "Create directory " << table_path << " Error";
return Status::Error("Failed to create table path"); return Status(DB_ERROR, "Failed to create table path");
} }
} }
@ -110,7 +110,7 @@ Status CreateTableFilePath(const DBMetaOptions& options, meta::TableFileSchema&
auto status = server::CommonUtil::CreateDirectory(parent_path); auto status = server::CommonUtil::CreateDirectory(parent_path);
if (status != 0) { if (status != 0) {
ENGINE_LOG_ERROR << "Create directory " << parent_path << " Error"; ENGINE_LOG_ERROR << "Create directory " << parent_path << " Error";
return Status::DBTransactionError("Failed to create partition directory"); return Status(DB_ERROR, "Failed to create partition directory");
} }
table_file.location_ = parent_path + "/" + table_file.file_id_; table_file.location_ = parent_path + "/" + table_file.file_id_;
@ -137,7 +137,7 @@ Status GetTableFilePath(const DBMetaOptions& options, meta::TableFileSchema& tab
std::string msg = "Table file doesn't exist: " + table_file.file_id_; std::string msg = "Table file doesn't exist: " + table_file.file_id_;
ENGINE_LOG_ERROR << msg; ENGINE_LOG_ERROR << msg;
return Status::Error(msg); return Status(DB_ERROR, msg);
} }
Status DeleteTableFilePath(const DBMetaOptions& options, meta::TableFileSchema& table_file) { Status DeleteTableFilePath(const DBMetaOptions& options, meta::TableFileSchema& table_file) {

50
cpp/src/db/engine/ExecutionEngineImpl.cpp
View File

@ -41,7 +41,7 @@ ExecutionEngineImpl::ExecutionEngineImpl(uint16_t dimension,
build_cfg["metric_type"] = (metric_type_ == MetricType::IP) ? "IP" : "L2"; build_cfg["metric_type"] = (metric_type_ == MetricType::IP) ? "IP" : "L2";
AutoGenParams(index_->GetType(), 0, build_cfg); AutoGenParams(index_->GetType(), 0, build_cfg);
auto ec = std::static_pointer_cast<BFIndex>(index_)->Build(build_cfg); auto ec = std::static_pointer_cast<BFIndex>(index_)->Build(build_cfg);
if (ec != server::KNOWHERE_SUCCESS) { throw Exception("Build index error"); } if (ec != KNOWHERE_SUCCESS) { throw Exception("Build index error"); }
} }
ExecutionEngineImpl::ExecutionEngineImpl(VecIndexPtr index, ExecutionEngineImpl::ExecutionEngineImpl(VecIndexPtr index,
@ -85,8 +85,8 @@ VecIndexPtr ExecutionEngineImpl::CreatetVecIndex(EngineType type) {
Status ExecutionEngineImpl::AddWithIds(long n, const float *xdata, const long *xids) { Status ExecutionEngineImpl::AddWithIds(long n, const float *xdata, const long *xids) {
auto ec = index_->Add(n, xdata, xids); auto ec = index_->Add(n, xdata, xids);
if (ec != server::KNOWHERE_SUCCESS) { if (ec != KNOWHERE_SUCCESS) {
return Status::Error("Add error"); return Status(DB_ERROR, "Add error");
} }
return Status::OK(); return Status::OK();
} }
@ -117,8 +117,8 @@ size_t ExecutionEngineImpl::PhysicalSize() const {
Status ExecutionEngineImpl::Serialize() { Status ExecutionEngineImpl::Serialize() {
auto ec = write_index(index_, location_); auto ec = write_index(index_, location_);
if (ec != server::KNOWHERE_SUCCESS) { if (ec != KNOWHERE_SUCCESS) {
return Status::Error("Serialize: write to disk error"); return Status(DB_ERROR, "Serialize: write to disk error");
} }
return Status::OK(); return Status::OK();
} }
@ -134,15 +134,15 @@ Status ExecutionEngineImpl::Load(bool to_cache) {
if(index_ == nullptr) { if(index_ == nullptr) {
std::string msg = "Failed to load index from " + location_; std::string msg = "Failed to load index from " + location_;
ENGINE_LOG_ERROR << msg; ENGINE_LOG_ERROR << msg;
return Status::Error(msg); return Status(DB_ERROR, msg);
} else { } else {
ENGINE_LOG_DEBUG << "Disk io from: " << location_; ENGINE_LOG_DEBUG << "Disk io from: " << location_;
} }
} catch (knowhere::KnowhereException &e) { } catch (knowhere::KnowhereException &e) {
ENGINE_LOG_ERROR << e.what(); ENGINE_LOG_ERROR << e.what();
return Status::Error(e.what()); return Status(DB_ERROR, e.what());
} catch (std::exception &e) { } catch (std::exception &e) {
return Status::Error(e.what()); return Status(DB_ERROR, e.what());
} }
} }
@ -160,7 +160,7 @@ Status ExecutionEngineImpl::CopyToGpu(uint64_t device_id) {
} else { } else {
if(index_ == nullptr) { if(index_ == nullptr) {
ENGINE_LOG_ERROR << "ExecutionEngineImpl: index is null, failed to copy to gpu"; ENGINE_LOG_ERROR << "ExecutionEngineImpl: index is null, failed to copy to gpu";
return Status::Error("index is null"); return Status(DB_ERROR, "index is null");
} }
try { try {
@ -168,9 +168,9 @@ Status ExecutionEngineImpl::CopyToGpu(uint64_t device_id) {
ENGINE_LOG_DEBUG << "CPU to GPU" << device_id; ENGINE_LOG_DEBUG << "CPU to GPU" << device_id;
} catch (knowhere::KnowhereException &e) { } catch (knowhere::KnowhereException &e) {
ENGINE_LOG_ERROR << e.what(); ENGINE_LOG_ERROR << e.what();
return Status::Error(e.what()); return Status(DB_ERROR, e.what());
} catch (std::exception &e) { } catch (std::exception &e) {
return Status::Error(e.what()); return Status(DB_ERROR, e.what());
} }
} }
@ -189,7 +189,7 @@ Status ExecutionEngineImpl::CopyToCpu() {
} else { } else {
if(index_ == nullptr) { if(index_ == nullptr) {
ENGINE_LOG_ERROR << "ExecutionEngineImpl: index is null, failed to copy to cpu"; ENGINE_LOG_ERROR << "ExecutionEngineImpl: index is null, failed to copy to cpu";
return Status::Error("index is null"); return Status(DB_ERROR, "index is null");
} }
try { try {
@ -197,9 +197,9 @@ Status ExecutionEngineImpl::CopyToCpu() {
ENGINE_LOG_DEBUG << "GPU to CPU"; ENGINE_LOG_DEBUG << "GPU to CPU";
} catch (knowhere::KnowhereException &e) { } catch (knowhere::KnowhereException &e) {
ENGINE_LOG_ERROR << e.what(); ENGINE_LOG_ERROR << e.what();
return Status::Error(e.what()); return Status(DB_ERROR, e.what());
} catch (std::exception &e) { } catch (std::exception &e) {
return Status::Error(e.what()); return Status(DB_ERROR, e.what());
} }
} }
@ -223,7 +223,7 @@ ExecutionEnginePtr ExecutionEngineImpl::Clone() {
Status ExecutionEngineImpl::Merge(const std::string &location) { Status ExecutionEngineImpl::Merge(const std::string &location) {
if (location == location_) { if (location == location_) {
return Status::Error("Cannot Merge Self"); return Status(DB_ERROR, "Cannot Merge Self");
} }
ENGINE_LOG_DEBUG << "Merge index file: " << location << " to: " << location_; ENGINE_LOG_DEBUG << "Merge index file: " << location << " to: " << location_;
@ -235,26 +235,26 @@ Status ExecutionEngineImpl::Merge(const std::string &location) {
to_merge = read_index(location); to_merge = read_index(location);
} catch (knowhere::KnowhereException &e) { } catch (knowhere::KnowhereException &e) {
ENGINE_LOG_ERROR << e.what(); ENGINE_LOG_ERROR << e.what();
return Status::Error(e.what()); return Status(DB_ERROR, e.what());
} catch (std::exception &e) { } catch (std::exception &e) {
return Status::Error(e.what()); return Status(DB_ERROR, e.what());
} }
} }
if(index_ == nullptr) { if(index_ == nullptr) {
ENGINE_LOG_ERROR << "ExecutionEngineImpl: index is null, failed to merge"; ENGINE_LOG_ERROR << "ExecutionEngineImpl: index is null, failed to merge";
return Status::Error("index is null"); return Status(DB_ERROR, "index is null");
} }
if (auto file_index = std::dynamic_pointer_cast<BFIndex>(to_merge)) { if (auto file_index = std::dynamic_pointer_cast<BFIndex>(to_merge)) {
auto ec = index_->Add(file_index->Count(), file_index->GetRawVectors(), file_index->GetRawIds()); auto ec = index_->Add(file_index->Count(), file_index->GetRawVectors(), file_index->GetRawIds());
if (ec != server::KNOWHERE_SUCCESS) { if (ec != KNOWHERE_SUCCESS) {
ENGINE_LOG_ERROR << "Merge: Add Error"; ENGINE_LOG_ERROR << "Merge: Add Error";
return Status::Error("Merge: Add Error"); return Status(DB_ERROR, "Merge: Add Error");
} }
return Status::OK(); return Status::OK();
} else { } else {
return Status::Error("file index type is not idmap"); return Status(DB_ERROR, "file index type is not idmap");
} }
} }
@ -284,7 +284,7 @@ ExecutionEngineImpl::BuildIndex(const std::string &location, EngineType engine_t
from_index->GetRawVectors(), from_index->GetRawVectors(),
from_index->GetRawIds(), from_index->GetRawIds(),
build_cfg); build_cfg);
if (ec != server::KNOWHERE_SUCCESS) { throw Exception("Build index error"); } if (ec != KNOWHERE_SUCCESS) { throw Exception("Build index error"); }
return std::make_shared<ExecutionEngineImpl>(to_index, location, engine_type, metric_type_, nlist_); return std::make_shared<ExecutionEngineImpl>(to_index, location, engine_type, metric_type_, nlist_);
} }
@ -297,15 +297,15 @@ Status ExecutionEngineImpl::Search(long n,
long *labels) const { long *labels) const {
if(index_ == nullptr) { if(index_ == nullptr) {
ENGINE_LOG_ERROR << "ExecutionEngineImpl: index is null, failed to search"; ENGINE_LOG_ERROR << "ExecutionEngineImpl: index is null, failed to search";
return Status::Error("index is null"); return Status(DB_ERROR, "index is null");
} }
ENGINE_LOG_DEBUG << "Search Params: [k] " << k << " [nprobe] " << nprobe; ENGINE_LOG_DEBUG << "Search Params: [k] " << k << " [nprobe] " << nprobe;
auto cfg = Config::object{{"k", k}, {"nprobe", nprobe}}; auto cfg = Config::object{{"k", k}, {"nprobe", nprobe}};
auto ec = index_->Search(n, data, distances, labels, cfg); auto ec = index_->Search(n, data, distances, labels, cfg);
if (ec != server::KNOWHERE_SUCCESS) { if (ec != KNOWHERE_SUCCESS) {
ENGINE_LOG_ERROR << "Search error"; ENGINE_LOG_ERROR << "Search error";
return Status::Error("Search: Search Error"); return Status(DB_ERROR, "Search: Search Error");
} }
return Status::OK(); return Status::OK();
} }

4
cpp/src/db/insert/MemTable.cpp
View File

@ -38,7 +38,7 @@ Status MemTable::Add(VectorSource::Ptr &source, IDNumbers &vector_ids) {
if (!status.ok()) { if (!status.ok()) {
std::string err_msg = "MemTable::Add failed: " + status.ToString(); std::string err_msg = "MemTable::Add failed: " + status.ToString();
ENGINE_LOG_ERROR << err_msg; ENGINE_LOG_ERROR << err_msg;
return Status::Error(err_msg); return Status(DB_ERROR, err_msg);
} }
} }
return Status::OK(); return Status::OK();
@ -58,7 +58,7 @@ Status MemTable::Serialize() {
if (!status.ok()) { if (!status.ok()) {
std::string err_msg = "MemTable::Serialize failed: " + status.ToString(); std::string err_msg = "MemTable::Serialize failed: " + status.ToString();
ENGINE_LOG_ERROR << err_msg; ENGINE_LOG_ERROR << err_msg;
return Status::Error(err_msg); return Status(DB_ERROR, err_msg);
} }
std::lock_guard<std::mutex> lock(mutex_); std::lock_guard<std::mutex> lock(mutex_);
mem_table_file = mem_table_file_list_.erase(mem_table_file); mem_table_file = mem_table_file_list_.erase(mem_table_file);

2
cpp/src/db/insert/MemTableFile.cpp
View File

@ -49,7 +49,7 @@ Status MemTableFile::Add(const VectorSource::Ptr &source, IDNumbers& vector_ids)
std::string err_msg = "MemTableFile::Add: table_file_schema dimension = " + std::string err_msg = "MemTableFile::Add: table_file_schema dimension = " +
std::to_string(table_file_schema_.dimension_) + ", table_id = " + table_file_schema_.table_id_; std::to_string(table_file_schema_.dimension_) + ", table_id = " + table_file_schema_.table_id_;
ENGINE_LOG_ERROR << err_msg; ENGINE_LOG_ERROR << err_msg;
return Status::Error(err_msg); return Status(DB_ERROR, err_msg);
} }
size_t single_vector_mem_size = table_file_schema_.dimension_ * VECTOR_TYPE_SIZE; size_t single_vector_mem_size = table_file_schema_.dimension_ * VECTOR_TYPE_SIZE;

537
cpp/src/db/meta/MySQLMetaImpl.cpp
View File

File diff suppressed because it is too large Load Diff

133
cpp/src/db/meta/SqliteMetaImpl.cpp
View File

@ -29,9 +29,15 @@ using namespace sqlite_orm;
namespace { namespace {
Status HandleException(const std::string& desc, std::exception &e) { Status HandleException(const std::string &desc, const char* what = nullptr) {
ENGINE_LOG_ERROR << desc << ": " << e.what(); if(what == nullptr) {
return Status::DBTransactionError(desc, e.what()); ENGINE_LOG_ERROR << desc;
return Status(DB_META_TRANSACTION_FAILED, desc);
} else {
std::string msg = desc + ":" + what;
ENGINE_LOG_ERROR << msg;
return Status(DB_META_TRANSACTION_FAILED, msg);
}
} }
} }
@ -97,8 +103,9 @@ Status SqliteMetaImpl::Initialize() {
if (!boost::filesystem::is_directory(options_.path)) { if (!boost::filesystem::is_directory(options_.path)) {
auto ret = boost::filesystem::create_directory(options_.path); auto ret = boost::filesystem::create_directory(options_.path);
if (!ret) { if (!ret) {
ENGINE_LOG_ERROR << "Failed to create db directory " << options_.path; std::string msg = "Failed to create db directory " + options_.path;
return Status::InvalidDBPath("Failed to create db directory", options_.path); ENGINE_LOG_ERROR << msg;
return Status(DB_INVALID_PATH, msg);
} }
} }
@ -141,7 +148,7 @@ Status SqliteMetaImpl::DropPartitionsByDates(const std::string &table_id,
in(&TableFileSchema::date_, dates) in(&TableFileSchema::date_, dates)
)); ));
} catch (std::exception &e) { } catch (std::exception &e) {
return HandleException("Encounter exception when drop partition", e); return HandleException("Encounter exception when drop partition", e.what());
} }
return Status::OK(); return Status::OK();
@ -162,10 +169,10 @@ Status SqliteMetaImpl::CreateTable(TableSchema &table_schema) {
where(c(&TableSchema::table_id_) == table_schema.table_id_)); where(c(&TableSchema::table_id_) == table_schema.table_id_));
if (table.size() == 1) { if (table.size() == 1) {
if(TableSchema::TO_DELETE == std::get<0>(table[0])) { if(TableSchema::TO_DELETE == std::get<0>(table[0])) {
return Status::Error("Table already exists and it is in delete state, please wait a second"); return Status(DB_ERROR, "Table already exists and it is in delete state, please wait a second");
} else { } else {
// Change from no error to already exist. // Change from no error to already exist.
return Status::AlreadyExist("Table already exists"); return Status(DB_ALREADY_EXIST, "Table already exists");
} }
} }
} }
@ -176,15 +183,14 @@ Status SqliteMetaImpl::CreateTable(TableSchema &table_schema) {
try { try {
auto id = ConnectorPtr->insert(table_schema); auto id = ConnectorPtr->insert(table_schema);
table_schema.id_ = id; table_schema.id_ = id;
} catch (...) { } catch (std::exception &e) {
ENGINE_LOG_ERROR << "sqlite transaction failed"; return HandleException("Encounter exception when create table", e.what());
return Status::DBTransactionError("Add Table Error");
} }
return utils::CreateTablePath(options_, table_schema.table_id_); return utils::CreateTablePath(options_, table_schema.table_id_);
} catch (std::exception &e) { } catch (std::exception &e) {
return HandleException("Encounter exception when create table", e); return HandleException("Encounter exception when create table", e.what());
} }
return Status::OK(); return Status::OK();
@ -208,7 +214,7 @@ Status SqliteMetaImpl::DeleteTable(const std::string& table_id) {
)); ));
} catch (std::exception &e) { } catch (std::exception &e) {
return HandleException("Encounter exception when delete table", e); return HandleException("Encounter exception when delete table", e.what());
} }
return Status::OK(); return Status::OK();
@ -233,7 +239,7 @@ Status SqliteMetaImpl::DeleteTableFiles(const std::string& table_id) {
)); ));
} catch (std::exception &e) { } catch (std::exception &e) {
return HandleException("Encounter exception when delete table files", e); return HandleException("Encounter exception when delete table files", e.what());
} }
return Status::OK(); return Status::OK();
@ -266,11 +272,11 @@ Status SqliteMetaImpl::DescribeTable(TableSchema &table_schema) {
table_schema.nlist_ = std::get<7>(groups[0]); table_schema.nlist_ = std::get<7>(groups[0]);
table_schema.metric_type_ = std::get<8>(groups[0]); table_schema.metric_type_ = std::get<8>(groups[0]);
} else { } else {
return Status::NotFound("Table " + table_schema.table_id_ + " not found"); return Status(DB_NOT_FOUND, "Table " + table_schema.table_id_ + " not found");
} }
} catch (std::exception &e) { } catch (std::exception &e) {
return HandleException("Encounter exception when describe table", e); return HandleException("Encounter exception when describe table", e.what());
} }
return Status::OK(); return Status::OK();
@ -280,7 +286,7 @@ Status SqliteMetaImpl::FilesByType(const std::string& table_id,
const std::vector<int>& file_types, const std::vector<int>& file_types,
std::vector<std::string>& file_ids) { std::vector<std::string>& file_ids) {
if(file_types.empty()) { if(file_types.empty()) {
return Status::Error("file types array is empty"); return Status(DB_ERROR, "file types array is empty");
} }
try { try {
@ -330,7 +336,7 @@ Status SqliteMetaImpl::FilesByType(const std::string& table_id,
} }
} catch (std::exception &e) { } catch (std::exception &e) {
return HandleException("Encounter exception when check non index files", e); return HandleException("Encounter exception when check non index files", e.what());
} }
return Status::OK(); return Status::OK();
} }
@ -366,7 +372,7 @@ Status SqliteMetaImpl::UpdateTableIndexParam(const std::string &table_id, const
ConnectorPtr->update(table_schema); ConnectorPtr->update(table_schema);
} else { } else {
return Status::NotFound("Table " + table_id + " not found"); return Status(DB_NOT_FOUND, "Table " + table_id + " not found");
} }
//set all backup file to raw //set all backup file to raw
@ -382,7 +388,7 @@ Status SqliteMetaImpl::UpdateTableIndexParam(const std::string &table_id, const
} catch (std::exception &e) { } catch (std::exception &e) {
std::string msg = "Encounter exception when update table index: table_id = " + table_id; std::string msg = "Encounter exception when update table index: table_id = " + table_id;
return HandleException(msg, e); return HandleException(msg, e.what());
} }
return Status::OK(); return Status::OK();
@ -403,7 +409,7 @@ Status SqliteMetaImpl::UpdateTableFlag(const std::string &table_id, int64_t flag
} catch (std::exception &e) { } catch (std::exception &e) {
std::string msg = "Encounter exception when update table flag: table_id = " + table_id; std::string msg = "Encounter exception when update table flag: table_id = " + table_id;
return HandleException(msg, e); return HandleException(msg, e.what());
} }
return Status::OK(); return Status::OK();
@ -424,11 +430,11 @@ Status SqliteMetaImpl::DescribeTableIndex(const std::string &table_id, TableInde
index.nlist_ = std::get<1>(groups[0]); index.nlist_ = std::get<1>(groups[0]);
index.metric_type_ = std::get<2>(groups[0]); index.metric_type_ = std::get<2>(groups[0]);
} else { } else {
return Status::NotFound("Table " + table_id + " not found"); return Status(DB_NOT_FOUND, "Table " + table_id + " not found");
} }
} catch (std::exception &e) { } catch (std::exception &e) {
return HandleException("Encounter exception when describe index", e); return HandleException("Encounter exception when describe index", e.what());
} }
return Status::OK(); return Status::OK();
@ -475,7 +481,7 @@ Status SqliteMetaImpl::DropTableIndex(const std::string &table_id) {
)); ));
} catch (std::exception &e) { } catch (std::exception &e) {
return HandleException("Encounter exception when delete table index files", e); return HandleException("Encounter exception when delete table index files", e.what());
} }
return Status::OK(); return Status::OK();
@ -496,7 +502,7 @@ Status SqliteMetaImpl::HasTable(const std::string &table_id, bool &has_or_not) {
} }
} catch (std::exception &e) { } catch (std::exception &e) {
return HandleException("Encounter exception when lookup table", e); return HandleException("Encounter exception when lookup table", e.what());
} }
return Status::OK(); return Status::OK();
@ -532,7 +538,7 @@ Status SqliteMetaImpl::AllTables(std::vector<TableSchema>& table_schema_array) {
} }
} catch (std::exception &e) { } catch (std::exception &e) {
return HandleException("Encounter exception when lookup all tables", e); return HandleException("Encounter exception when lookup all tables", e.what());
} }
return Status::OK(); return Status::OK();
@ -571,8 +577,8 @@ Status SqliteMetaImpl::CreateTableFile(TableFileSchema &file_schema) {
return utils::CreateTableFilePath(options_, file_schema); return utils::CreateTableFilePath(options_, file_schema);
} catch (std::exception& ex) { } catch (std::exception& e) {
return HandleException("Encounter exception when create table file", ex); return HandleException("Encounter exception when create table file", e.what());
} }
return Status::OK(); return Status::OK();
@ -629,7 +635,7 @@ Status SqliteMetaImpl::FilesToIndex(TableFilesSchema &files) {
} }
} catch (std::exception &e) { } catch (std::exception &e) {
return HandleException("Encounter exception when iterate raw files", e); return HandleException("Encounter exception when iterate raw files", e.what());
} }
return Status::OK(); return Status::OK();
@ -654,8 +660,12 @@ Status SqliteMetaImpl::FilesToSearch(const std::string &table_id,
auto match_tableid = c(&TableFileSchema::table_id_) == table_id; auto match_tableid = c(&TableFileSchema::table_id_) == table_id;
std::vector<int> file_type = {(int) TableFileSchema::RAW, (int) TableFileSchema::TO_INDEX, (int) TableFileSchema::INDEX}; std::vector<int> file_types = {
auto match_type = in(&TableFileSchema::file_type_, file_type); (int) TableFileSchema::RAW,
(int) TableFileSchema::TO_INDEX,
(int) TableFileSchema::INDEX
};
auto match_type = in(&TableFileSchema::file_type_, file_types);
TableSchema table_schema; TableSchema table_schema;
table_schema.table_id_ = table_id; table_schema.table_id_ = table_id;
@ -710,7 +720,7 @@ Status SqliteMetaImpl::FilesToSearch(const std::string &table_id,
ENGINE_LOG_ERROR << "No file to search for table: " << table_id; ENGINE_LOG_ERROR << "No file to search for table: " << table_id;
} }
} catch (std::exception &e) { } catch (std::exception &e) {
return HandleException("Encounter exception when iterate index files", e); return HandleException("Encounter exception when iterate index files", e.what());
} }
@ -773,7 +783,7 @@ Status SqliteMetaImpl::FilesToMerge(const std::string &table_id,
files[table_file.date_].push_back(table_file); files[table_file.date_].push_back(table_file);
} }
} catch (std::exception &e) { } catch (std::exception &e) {
return HandleException("Encounter exception when iterate merge files", e); return HandleException("Encounter exception when iterate merge files", e.what());
} }
return Status::OK(); return Status::OK();
@ -824,7 +834,7 @@ Status SqliteMetaImpl::GetTableFiles(const std::string& table_id,
table_files.emplace_back(file_schema); table_files.emplace_back(file_schema);
} }
} catch (std::exception &e) { } catch (std::exception &e) {
return HandleException("Encounter exception when lookup table files", e); return HandleException("Encounter exception when lookup table files", e.what());
} }
return Status::OK(); return Status::OK();
@ -856,7 +866,7 @@ Status SqliteMetaImpl::Archive() {
c(&TableFileSchema::file_type_) != (int) TableFileSchema::TO_DELETE c(&TableFileSchema::file_type_) != (int) TableFileSchema::TO_DELETE
)); ));
} catch (std::exception &e) { } catch (std::exception &e) {
return HandleException("Encounter exception when update table files", e); return HandleException("Encounter exception when update table files", e.what());
} }
} }
if (criteria == engine::ARCHIVE_CONF_DISK) { if (criteria == engine::ARCHIVE_CONF_DISK) {
@ -886,7 +896,7 @@ Status SqliteMetaImpl::Size(uint64_t &result) {
} }
} catch (std::exception &e) { } catch (std::exception &e) {
return HandleException("Encounter exception when calculte db size", e); return HandleException("Encounter exception when calculte db size", e.what());
} }
return Status::OK(); return Status::OK();
@ -943,12 +953,11 @@ Status SqliteMetaImpl::DiscardFiles(long to_discard_size) {
}); });
if (!commited) { if (!commited) {
ENGINE_LOG_ERROR << "sqlite transaction failed"; return HandleException("DiscardFiles error: sqlite transaction failed");
return Status::DBTransactionError("Update table file error");
} }
} catch (std::exception &e) { } catch (std::exception &e) {
return HandleException("Encounter exception when discard table file", e); return HandleException("Encounter exception when discard table file", e.what());
} }
return DiscardFiles(to_discard_size); return DiscardFiles(to_discard_size);
@ -976,7 +985,7 @@ Status SqliteMetaImpl::UpdateTableFile(TableFileSchema &file_schema) {
} catch (std::exception &e) { } catch (std::exception &e) {
std::string msg = "Exception update table file: table_id = " + file_schema.table_id_ std::string msg = "Exception update table file: table_id = " + file_schema.table_id_
+ " file_id = " + file_schema.file_id_; + " file_id = " + file_schema.file_id_;
return HandleException(msg, e); return HandleException(msg, e.what());
} }
return Status::OK(); return Status::OK();
} }
@ -997,7 +1006,7 @@ Status SqliteMetaImpl::UpdateTableFilesToIndex(const std::string& table_id) {
c(&TableFileSchema::file_type_) == (int) TableFileSchema::RAW c(&TableFileSchema::file_type_) == (int) TableFileSchema::RAW
)); ));
} catch (std::exception &e) { } catch (std::exception &e) {
return HandleException("Encounter exception when update table files to to_index", e); return HandleException("Encounter exception when update table files to to_index", e.what());
} }
return Status::OK(); return Status::OK();
@ -1038,12 +1047,11 @@ Status SqliteMetaImpl::UpdateTableFiles(TableFilesSchema &files) {
}); });
if (!commited) { if (!commited) {
ENGINE_LOG_ERROR << "sqlite transaction failed"; return HandleException("UpdateTableFiles error: sqlite transaction failed");
return Status::DBTransactionError("Update table files error");
} }
} catch (std::exception &e) { } catch (std::exception &e) {
return HandleException("Encounter exception when update table files", e); return HandleException("Encounter exception when update table files", e.what());
} }
return Status::OK(); return Status::OK();
} }
@ -1088,12 +1096,11 @@ Status SqliteMetaImpl::CleanUpFilesWithTTL(uint16_t seconds) {
}); });
if (!commited) { if (!commited) {
ENGINE_LOG_ERROR << "sqlite transaction failed"; return HandleException("CleanUpFilesWithTTL error: sqlite transaction failed");
return Status::DBTransactionError("Clean files error");
} }
} catch (std::exception &e) { } catch (std::exception &e) {
return HandleException("Encounter exception when clean table files", e); return HandleException("Encounter exception when clean table files", e.what());
} }
//remove to_delete tables //remove to_delete tables
@ -1117,12 +1124,11 @@ Status SqliteMetaImpl::CleanUpFilesWithTTL(uint16_t seconds) {
}); });
if (!commited) { if (!commited) {
ENGINE_LOG_ERROR << "sqlite transaction failed"; return HandleException("CleanUpFilesWithTTL error: sqlite transaction failed");
return Status::DBTransactionError("Clean files error");
} }
} catch (std::exception &e) { } catch (std::exception &e) {
return HandleException("Encounter exception when clean table files", e); return HandleException("Encounter exception when clean table files", e.what());
} }
//remove deleted table folder //remove deleted table folder
@ -1139,7 +1145,7 @@ Status SqliteMetaImpl::CleanUpFilesWithTTL(uint16_t seconds) {
} }
} catch (std::exception &e) { } catch (std::exception &e) {
return HandleException("Encounter exception when delete table folder", e); return HandleException("Encounter exception when delete table folder", e.what());
} }
return Status::OK(); return Status::OK();
@ -1152,8 +1158,12 @@ Status SqliteMetaImpl::CleanUp() {
//multi-threads call sqlite update may get exception('bad logic', etc), so we add a lock here //multi-threads call sqlite update may get exception('bad logic', etc), so we add a lock here
std::lock_guard<std::mutex> meta_lock(meta_mutex_); std::lock_guard<std::mutex> meta_lock(meta_mutex_);
std::vector<int> file_type = {(int) TableFileSchema::NEW, (int) TableFileSchema::NEW_INDEX, (int) TableFileSchema::NEW_MERGE}; std::vector<int> file_types = {
auto files = ConnectorPtr->select(columns(&TableFileSchema::id_), where(in(&TableFileSchema::file_type_, file_type))); (int) TableFileSchema::NEW,
(int) TableFileSchema::NEW_INDEX,
(int) TableFileSchema::NEW_MERGE
};
auto files = ConnectorPtr->select(columns(&TableFileSchema::id_), where(in(&TableFileSchema::file_type_, file_types)));
auto commited = ConnectorPtr->transaction([&]() mutable { auto commited = ConnectorPtr->transaction([&]() mutable {
for (auto &file : files) { for (auto &file : files) {
@ -1164,12 +1174,11 @@ Status SqliteMetaImpl::CleanUp() {
}); });
if (!commited) { if (!commited) {
ENGINE_LOG_ERROR << "sqlite transaction failed"; return HandleException("CleanUp error: sqlite transaction failed");
return Status::DBTransactionError("Clean files error");
} }
} catch (std::exception &e) { } catch (std::exception &e) {
return HandleException("Encounter exception when clean table file", e); return HandleException("Encounter exception when clean table file", e.what());
} }
return Status::OK(); return Status::OK();
@ -1180,9 +1189,13 @@ Status SqliteMetaImpl::Count(const std::string &table_id, uint64_t &result) {
try { try {
server::MetricCollector metric; server::MetricCollector metric;
std::vector<int> file_type = {(int) TableFileSchema::RAW, (int) TableFileSchema::TO_INDEX, (int) TableFileSchema::INDEX}; std::vector<int> file_types = {
(int) TableFileSchema::RAW,
(int) TableFileSchema::TO_INDEX,
(int) TableFileSchema::INDEX
};
auto selected = ConnectorPtr->select(columns(&TableFileSchema::row_count_), auto selected = ConnectorPtr->select(columns(&TableFileSchema::row_count_),
where(in(&TableFileSchema::file_type_, file_type) where(in(&TableFileSchema::file_type_, file_types)
and c(&TableFileSchema::table_id_) == table_id)); and c(&TableFileSchema::table_id_) == table_id));
TableSchema table_schema; TableSchema table_schema;
@ -1199,7 +1212,7 @@ Status SqliteMetaImpl::Count(const std::string &table_id, uint64_t &result) {
} }
} catch (std::exception &e) { } catch (std::exception &e) {
return HandleException("Encounter exception when calculate table file size", e); return HandleException("Encounter exception when calculate table file size", e.what());
} }
return Status::OK(); return Status::OK();
} }
@ -1211,7 +1224,7 @@ Status SqliteMetaImpl::DropAll() {
ConnectorPtr->drop_table("Tables"); ConnectorPtr->drop_table("Tables");
ConnectorPtr->drop_table("TableFiles"); ConnectorPtr->drop_table("TableFiles");
} catch (std::exception &e) { } catch (std::exception &e) {
return HandleException("Encounter exception when drop all meta", e); return HandleException("Encounter exception when drop all meta", e.what());
} }
return Status::OK(); return Status::OK();

4
cpp/src/db/scheduler/TaskDispatchQueue.cpp
View File

@ -53,7 +53,7 @@ TaskDispatchQueue::Front() {
if (queue_.empty()) { if (queue_.empty()) {
std::string error_msg = "blocking queue empty"; std::string error_msg = "blocking queue empty";
SERVER_LOG_ERROR << error_msg; SERVER_LOG_ERROR << error_msg;
throw server::ServerException(server::SERVER_BLOCKING_QUEUE_EMPTY, error_msg); throw server::ServerException(SERVER_BLOCKING_QUEUE_EMPTY, error_msg);
} }
ScheduleTaskPtr front(queue_.front()); ScheduleTaskPtr front(queue_.front());
return front; return front;
@ -67,7 +67,7 @@ TaskDispatchQueue::Back() {
if (queue_.empty()) { if (queue_.empty()) {
std::string error_msg = "blocking queue empty"; std::string error_msg = "blocking queue empty";
SERVER_LOG_ERROR << error_msg; SERVER_LOG_ERROR << error_msg;
throw server::ServerException(server::SERVER_BLOCKING_QUEUE_EMPTY, error_msg); throw server::ServerException(SERVER_BLOCKING_QUEUE_EMPTY, error_msg);
} }
ScheduleTaskPtr back(queue_.back()); ScheduleTaskPtr back(queue_.back());

4
cpp/src/db/scheduler/task/SearchTask.cpp
View File

@ -134,7 +134,7 @@ Status SearchTask::ClusterResult(const std::vector<long> &output_ids,
std::string msg = "Invalid id array size: " + std::to_string(output_ids.size()) + std::string msg = "Invalid id array size: " + std::to_string(output_ids.size()) +
" distance array size: " + std::to_string(output_distence.size()); " distance array size: " + std::to_string(output_distence.size());
ENGINE_LOG_ERROR << msg; ENGINE_LOG_ERROR << msg;
return Status::Error(msg); return Status(DB_ERROR, msg);
} }
result_set.clear(); result_set.clear();
@ -249,7 +249,7 @@ Status SearchTask::TopkResult(SearchContext::ResultSet &result_src,
if (result_src.size() != result_target.size()) { if (result_src.size() != result_target.size()) {
std::string msg = "Invalid result set size"; std::string msg = "Invalid result set size";
ENGINE_LOG_ERROR << msg; ENGINE_LOG_ERROR << msg;
return Status::Error(msg); return Status(DB_ERROR, msg);
} }
std::function<void(size_t, size_t)> ReduceWorker = [&](size_t from_index, size_t to_index) { std::function<void(size_t, size_t)> ReduceWorker = [&](size_t from_index, size_t to_index) {

2
cpp/src/metrics/MetricBase.h
View File

@ -21,7 +21,7 @@ class MetricsBase{
return instance; return instance;
} }
virtual ServerError Init() {}; virtual ErrorCode Init() {};
virtual void AddVectorsSuccessTotalIncrement(double value = 1) {}; virtual void AddVectorsSuccessTotalIncrement(double value = 1) {};
virtual void AddVectorsFailTotalIncrement(double value = 1) {}; virtual void AddVectorsFailTotalIncrement(double value = 1) {};

2
cpp/src/metrics/PrometheusMetrics.cpp
View File

@ -14,7 +14,7 @@ namespace zilliz {
namespace milvus { namespace milvus {
namespace server { namespace server {
ServerError ErrorCode
PrometheusMetrics::Init() { PrometheusMetrics::Init() {
try { try {
ConfigNode &configNode = ServerConfig::GetInstance().GetConfig(CONFIG_METRIC); ConfigNode &configNode = ServerConfig::GetInstance().GetConfig(CONFIG_METRIC);

3
cpp/src/metrics/PrometheusMetrics.h
View File

@ -39,8 +39,7 @@ class PrometheusMetrics: public MetricsBase {
return instance; return instance;
} }
ServerError ErrorCode Init();
Init();
private: private:
std::shared_ptr<prometheus::Exposer> exposer_ptr_; std::shared_ptr<prometheus::Exposer> exposer_ptr_;

4
cpp/src/scheduler/task/SearchTask.cpp
View File

@ -202,7 +202,7 @@ Status XSearchTask::ClusterResult(const std::vector<long> &output_ids,
std::string msg = "Invalid id array size: " + std::to_string(output_ids.size()) + std::string msg = "Invalid id array size: " + std::to_string(output_ids.size()) +
" distance array size: " + std::to_string(output_distence.size()); " distance array size: " + std::to_string(output_distence.size());
ENGINE_LOG_ERROR << msg; ENGINE_LOG_ERROR << msg;
return Status::Error(msg); return Status(DB_ERROR, msg);
} }
result_set.clear(); result_set.clear();
@ -317,7 +317,7 @@ Status XSearchTask::TopkResult(SearchContext::ResultSet &result_src,
if (result_src.size() != result_target.size()) { if (result_src.size() != result_target.size()) {
std::string msg = "Invalid result set size"; std::string msg = "Invalid result set size";
ENGINE_LOG_ERROR << msg; ENGINE_LOG_ERROR << msg;
return Status::Error(msg); return Status(DB_ERROR, msg);
} }
std::function<void(size_t, size_t)> ReduceWorker = [&](size_t from_index, size_t to_index) { std::function<void(size_t, size_t)> ReduceWorker = [&](size_t from_index, size_t to_index) {

6
cpp/src/server/DBWrapper.cpp
View File

@ -21,7 +21,7 @@ DBWrapper::DBWrapper() {
} }
ServerError DBWrapper::StartService() { ErrorCode DBWrapper::StartService() {
//db config //db config
zilliz::milvus::engine::Options opt; zilliz::milvus::engine::Options opt;
ConfigNode& db_config = ServerConfig::GetInstance().GetConfig(CONFIG_DB); ConfigNode& db_config = ServerConfig::GetInstance().GetConfig(CONFIG_DB);
@ -79,7 +79,7 @@ ServerError DBWrapper::StartService() {
opt.meta.archive_conf.SetCriterias(criterial); opt.meta.archive_conf.SetCriterias(criterial);
//create db root folder //create db root folder
ServerError err = CommonUtil::CreateDirectory(opt.meta.path); ErrorCode err = CommonUtil::CreateDirectory(opt.meta.path);
if(err != SERVER_SUCCESS) { if(err != SERVER_SUCCESS) {
std::cout << "ERROR! Failed to create database root path: " << opt.meta.path << std::endl; std::cout << "ERROR! Failed to create database root path: " << opt.meta.path << std::endl;
kill(0, SIGUSR1); kill(0, SIGUSR1);
@ -112,7 +112,7 @@ ServerError DBWrapper::StartService() {
return SERVER_SUCCESS; return SERVER_SUCCESS;
} }
ServerError DBWrapper::StopService() { ErrorCode DBWrapper::StopService() {
if(db_) { if(db_) {
db_->Stop(); db_->Stop();
} }

4
cpp/src/server/DBWrapper.h
View File

@ -29,8 +29,8 @@ public:
return GetInstance().EngineDB(); return GetInstance().EngineDB();
} }
ServerError StartService(); ErrorCode StartService();
ServerError StopService(); ErrorCode StopService();
std::shared_ptr<engine::DB> EngineDB() { std::shared_ptr<engine::DB> EngineDB() {
return db_; return db_;

4
cpp/src/server/Server.cpp
View File

@ -210,10 +210,10 @@ Server::Stop() {
} }
ServerError ErrorCode
Server::LoadConfig() { Server::LoadConfig() {
ServerConfig::GetInstance().LoadConfigFile(config_filename_); ServerConfig::GetInstance().LoadConfigFile(config_filename_);
ServerError err = ServerConfig::GetInstance().ValidateConfig(); ErrorCode err = ServerConfig::GetInstance().ValidateConfig();
if(err != SERVER_SUCCESS){ if(err != SERVER_SUCCESS){
exit(0); exit(0);
} }

2
cpp/src/server/Server.h
View File

@ -29,7 +29,7 @@ class Server {
void Daemonize(); void Daemonize();
static void HandleSignal(int signal); static void HandleSignal(int signal);
ServerError LoadConfig(); ErrorCode LoadConfig();
void StartService(); void StartService();
void StopService(); void StopService();

6
cpp/src/server/ServerConfig.cpp
View File

@ -28,7 +28,7 @@ ServerConfig::GetInstance() {
return config; return config;
} }
ServerError ErrorCode
ServerConfig::LoadConfigFile(const std::string& config_filename) { ServerConfig::LoadConfigFile(const std::string& config_filename) {
std::string filename = config_filename; std::string filename = config_filename;
if(filename.empty()){ if(filename.empty()){
@ -44,7 +44,7 @@ ServerConfig::LoadConfigFile(const std::string& config_filename) {
try { try {
ConfigMgr* mgr = const_cast<ConfigMgr*>(ConfigMgr::GetInstance()); ConfigMgr* mgr = const_cast<ConfigMgr*>(ConfigMgr::GetInstance());
ServerError err = mgr->LoadConfigFile(filename); ErrorCode err = mgr->LoadConfigFile(filename);
if(err != 0) { if(err != 0) {
std::cout << "Server failed to load config file" << std::endl; std::cout << "Server failed to load config file" << std::endl;
exit(1);//directly exit program if the config file is illegal exit(1);//directly exit program if the config file is illegal
@ -58,7 +58,7 @@ ServerConfig::LoadConfigFile(const std::string& config_filename) {
return SERVER_SUCCESS; return SERVER_SUCCESS;
} }
ServerError ServerConfig::ValidateConfig() const { ErrorCode ServerConfig::ValidateConfig() const {
//server config validation //server config validation
ConfigNode server_config = GetConfig(CONFIG_SERVER); ConfigNode server_config = GetConfig(CONFIG_SERVER);
uint32_t gpu_index = (uint32_t)server_config.GetInt32Value(CONFIG_GPU_INDEX, 0); uint32_t gpu_index = (uint32_t)server_config.GetInt32Value(CONFIG_GPU_INDEX, 0);

4
cpp/src/server/ServerConfig.h
View File

@ -68,8 +68,8 @@ class ServerConfig {
public: public:
static ServerConfig &GetInstance(); static ServerConfig &GetInstance();
ServerError LoadConfigFile(const std::string& config_filename); ErrorCode LoadConfigFile(const std::string& config_filename);
ServerError ValidateConfig() const; ErrorCode ValidateConfig() const;
void PrintAll() const; void PrintAll() const;
ConfigNode GetConfig(const std::string& name) const; ConfigNode GetConfig(const std::string& name) const;

20
cpp/src/server/grpc_impl/GrpcRequestScheduler.cpp
View File

@ -16,8 +16,8 @@ namespace grpc {
using namespace ::milvus; using namespace ::milvus;
namespace { namespace {
const std::map<ServerError, ::milvus::grpc::ErrorCode> &ErrorMap() { const std::map<ErrorCode, ::milvus::grpc::ErrorCode> &ErrorMap() {
static const std::map<ServerError, ::milvus::grpc::ErrorCode> code_map = { static const std::map<ErrorCode, ::milvus::grpc::ErrorCode> code_map = {
{SERVER_UNEXPECTED_ERROR, ::milvus::grpc::ErrorCode::UNEXPECTED_ERROR}, {SERVER_UNEXPECTED_ERROR, ::milvus::grpc::ErrorCode::UNEXPECTED_ERROR},
{SERVER_UNSUPPORTED_ERROR, ::milvus::grpc::ErrorCode::UNEXPECTED_ERROR}, {SERVER_UNSUPPORTED_ERROR, ::milvus::grpc::ErrorCode::UNEXPECTED_ERROR},
{SERVER_NULL_POINTER, ::milvus::grpc::ErrorCode::UNEXPECTED_ERROR}, {SERVER_NULL_POINTER, ::milvus::grpc::ErrorCode::UNEXPECTED_ERROR},
@ -66,7 +66,7 @@ GrpcBaseTask::~GrpcBaseTask() {
WaitToFinish(); WaitToFinish();
} }
ServerError GrpcBaseTask::Execute() { ErrorCode GrpcBaseTask::Execute() {
error_code_ = OnExecute(); error_code_ = OnExecute();
Done(); Done();
return error_code_; return error_code_;
@ -77,7 +77,7 @@ void GrpcBaseTask::Done() {
finish_cond_.notify_all(); finish_cond_.notify_all();
} }
ServerError GrpcBaseTask::SetError(ServerError error_code, const std::string &error_msg) { ErrorCode GrpcBaseTask::SetError(ErrorCode error_code, const std::string &error_msg) {
error_code_ = error_code; error_code_ = error_code;
error_msg_ = error_msg; error_msg_ = error_msg;
@ -85,7 +85,7 @@ ServerError GrpcBaseTask::SetError(ServerError error_code, const std::string &er
return error_code_; return error_code_;
} }
ServerError GrpcBaseTask::WaitToFinish() { ErrorCode GrpcBaseTask::WaitToFinish() {
std::unique_lock<std::mutex> lock(finish_mtx_); std::unique_lock<std::mutex> lock(finish_mtx_);
finish_cond_.wait(lock, [this] { return done_; }); finish_cond_.wait(lock, [this] { return done_; });
@ -112,7 +112,7 @@ void GrpcRequestScheduler::ExecTask(BaseTaskPtr &task_ptr, ::milvus::grpc::Statu
if (!task_ptr->IsAsync()) { if (!task_ptr->IsAsync()) {
task_ptr->WaitToFinish(); task_ptr->WaitToFinish();
ServerError err = task_ptr->ErrorCode(); ErrorCode err = task_ptr->ErrorID();
if (err != SERVER_SUCCESS) { if (err != SERVER_SUCCESS) {
grpc_status->set_reason(task_ptr->ErrorMsg()); grpc_status->set_reason(task_ptr->ErrorMsg());
grpc_status->set_error_code(ErrorMap().at(err)); grpc_status->set_error_code(ErrorMap().at(err));
@ -153,12 +153,12 @@ void GrpcRequestScheduler::Stop() {
SERVER_LOG_INFO << "Scheduler stopped"; SERVER_LOG_INFO << "Scheduler stopped";
} }
ServerError GrpcRequestScheduler::ExecuteTask(const BaseTaskPtr &task_ptr) { ErrorCode GrpcRequestScheduler::ExecuteTask(const BaseTaskPtr &task_ptr) {
if (task_ptr == nullptr) { if (task_ptr == nullptr) {
return SERVER_NULL_POINTER; return SERVER_NULL_POINTER;
} }
ServerError err = PutTaskToQueue(task_ptr); ErrorCode err = PutTaskToQueue(task_ptr);
if (err != SERVER_SUCCESS) { if (err != SERVER_SUCCESS) {
SERVER_LOG_ERROR << "Put task to queue failed with code: " << err; SERVER_LOG_ERROR << "Put task to queue failed with code: " << err;
return err; return err;
@ -185,7 +185,7 @@ void GrpcRequestScheduler::TakeTaskToExecute(TaskQueuePtr task_queue) {
} }
try { try {
ServerError err = task->Execute(); ErrorCode err = task->Execute();
if (err != SERVER_SUCCESS) { if (err != SERVER_SUCCESS) {
SERVER_LOG_ERROR << "Task failed with code: " << err; SERVER_LOG_ERROR << "Task failed with code: " << err;
} }
@ -195,7 +195,7 @@ void GrpcRequestScheduler::TakeTaskToExecute(TaskQueuePtr task_queue) {
} }
} }
ServerError GrpcRequestScheduler::PutTaskToQueue(const BaseTaskPtr &task_ptr) { ErrorCode GrpcRequestScheduler::PutTaskToQueue(const BaseTaskPtr &task_ptr) {
std::lock_guard<std::mutex> lock(queue_mtx_); std::lock_guard<std::mutex> lock(queue_mtx_);
std::string group_name = task_ptr->TaskGroup(); std::string group_name = task_ptr->TaskGroup();

16
cpp/src/server/grpc_impl/GrpcRequestScheduler.h
View File

@ -25,24 +25,24 @@ protected:
virtual ~GrpcBaseTask(); virtual ~GrpcBaseTask();
public: public:
ServerError Execute(); ErrorCode Execute();
void Done(); void Done();
ServerError WaitToFinish(); ErrorCode WaitToFinish();
std::string TaskGroup() const { return task_group_; } std::string TaskGroup() const { return task_group_; }
ServerError ErrorCode() const { return error_code_; } ErrorCode ErrorID() const { return error_code_; }
std::string ErrorMsg() const { return error_msg_; } std::string ErrorMsg() const { return error_msg_; }
bool IsAsync() const { return async_; } bool IsAsync() const { return async_; }
protected: protected:
virtual ServerError OnExecute() = 0; virtual ErrorCode OnExecute() = 0;
ServerError SetError(ServerError error_code, const std::string &msg); ErrorCode SetError(ErrorCode error_code, const std::string &msg);
protected: protected:
mutable std::mutex finish_mtx_; mutable std::mutex finish_mtx_;
@ -51,7 +51,7 @@ protected:
std::string task_group_; std::string task_group_;
bool async_; bool async_;
bool done_; bool done_;
ServerError error_code_; ErrorCode error_code_;
std::string error_msg_; std::string error_msg_;
}; };
@ -71,7 +71,7 @@ public:
void Stop(); void Stop();
ServerError ExecuteTask(const BaseTaskPtr &task_ptr); ErrorCode ExecuteTask(const BaseTaskPtr &task_ptr);
static void ExecTask(BaseTaskPtr &task_ptr, ::milvus::grpc::Status *grpc_status); static void ExecTask(BaseTaskPtr &task_ptr, ::milvus::grpc::Status *grpc_status);
@ -82,7 +82,7 @@ protected:
void TakeTaskToExecute(TaskQueuePtr task_queue); void TakeTaskToExecute(TaskQueuePtr task_queue);
ServerError PutTaskToQueue(const BaseTaskPtr &task_ptr); ErrorCode PutTaskToQueue(const BaseTaskPtr &task_ptr);
private: private:
mutable std::mutex queue_mtx_; mutable std::mutex queue_mtx_;

72
cpp/src/server/grpc_impl/GrpcRequestTask.cpp
View File

@ -67,7 +67,7 @@ namespace {
void void
ConvertTimeRangeToDBDates(const std::vector<::milvus::grpc::Range> &range_array, ConvertTimeRangeToDBDates(const std::vector<::milvus::grpc::Range> &range_array,
std::vector<DB_DATE> &dates, std::vector<DB_DATE> &dates,
ServerError &error_code, ErrorCode &error_code,
std::string &error_msg) { std::string &error_msg) {
dates.clear(); dates.clear();
for (auto &range : range_array) { for (auto &range : range_array) {
@ -123,13 +123,13 @@ CreateTableTask::Create(const ::milvus::grpc::TableSchema *schema) {
return std::shared_ptr<GrpcBaseTask>(new CreateTableTask(schema)); return std::shared_ptr<GrpcBaseTask>(new CreateTableTask(schema));
} }
ServerError ErrorCode
CreateTableTask::OnExecute() { CreateTableTask::OnExecute() {
TimeRecorder rc("CreateTableTask"); TimeRecorder rc("CreateTableTask");
try { try {
//step 1: check arguments //step 1: check arguments
ServerError res = ValidationUtil::ValidateTableName(schema_->table_name().table_name()); ErrorCode res = ValidationUtil::ValidateTableName(schema_->table_name().table_name());
if (res != SERVER_SUCCESS) { if (res != SERVER_SUCCESS) {
return SetError(res, "Invalid table name: " + schema_->table_name().table_name()); return SetError(res, "Invalid table name: " + schema_->table_name().table_name());
} }
@ -160,7 +160,7 @@ CreateTableTask::OnExecute() {
engine::Status stat = DBWrapper::DB()->CreateTable(table_info); engine::Status stat = DBWrapper::DB()->CreateTable(table_info);
if (!stat.ok()) { if (!stat.ok()) {
//table could exist //table could exist
if(stat.IsAlreadyExist()) { if(stat.code() == DB_ALREADY_EXIST) {
return SetError(SERVER_INVALID_TABLE_NAME, stat.ToString()); return SetError(SERVER_INVALID_TABLE_NAME, stat.ToString());
} }
return SetError(DB_META_TRANSACTION_FAILED, stat.ToString()); return SetError(DB_META_TRANSACTION_FAILED, stat.ToString());
@ -187,13 +187,13 @@ DescribeTableTask::Create(const std::string &table_name, ::milvus::grpc::TableSc
return std::shared_ptr<GrpcBaseTask>(new DescribeTableTask(table_name, schema)); return std::shared_ptr<GrpcBaseTask>(new DescribeTableTask(table_name, schema));
} }
ServerError ErrorCode
DescribeTableTask::OnExecute() { DescribeTableTask::OnExecute() {
TimeRecorder rc("DescribeTableTask"); TimeRecorder rc("DescribeTableTask");
try { try {
//step 1: check arguments //step 1: check arguments
ServerError res = ValidationUtil::ValidateTableName(table_name_); ErrorCode res = ValidationUtil::ValidateTableName(table_name_);
if (res != SERVER_SUCCESS) { if (res != SERVER_SUCCESS) {
return SetError(res, "Invalid table name: " + table_name_); return SetError(res, "Invalid table name: " + table_name_);
} }
@ -235,14 +235,14 @@ CreateIndexTask::Create(const ::milvus::grpc::IndexParam *index_param) {
return std::shared_ptr<GrpcBaseTask>(new CreateIndexTask(index_param)); return std::shared_ptr<GrpcBaseTask>(new CreateIndexTask(index_param));
} }
ServerError ErrorCode
CreateIndexTask::OnExecute() { CreateIndexTask::OnExecute() {
try { try {
TimeRecorder rc("CreateIndexTask"); TimeRecorder rc("CreateIndexTask");
//step 1: check arguments //step 1: check arguments
std::string table_name_ = index_param_->table_name().table_name(); std::string table_name_ = index_param_->table_name().table_name();
ServerError res = ValidationUtil::ValidateTableName(table_name_); ErrorCode res = ValidationUtil::ValidateTableName(table_name_);
if (res != SERVER_SUCCESS) { if (res != SERVER_SUCCESS) {
return SetError(res, "Invalid table name: " + table_name_); return SetError(res, "Invalid table name: " + table_name_);
} }
@ -298,13 +298,13 @@ HasTableTask::Create(const std::string &table_name, bool &has_table) {
return std::shared_ptr<GrpcBaseTask>(new HasTableTask(table_name, has_table)); return std::shared_ptr<GrpcBaseTask>(new HasTableTask(table_name, has_table));
} }
ServerError ErrorCode
HasTableTask::OnExecute() { HasTableTask::OnExecute() {
try { try {
TimeRecorder rc("HasTableTask"); TimeRecorder rc("HasTableTask");
//step 1: check arguments //step 1: check arguments
ServerError res = ValidationUtil::ValidateTableName(table_name_); ErrorCode res = ValidationUtil::ValidateTableName(table_name_);
if (res != SERVER_SUCCESS) { if (res != SERVER_SUCCESS) {
return SetError(res, "Invalid table name: " + table_name_); return SetError(res, "Invalid table name: " + table_name_);
} }
@ -335,13 +335,13 @@ DropTableTask::Create(const std::string &table_name) {
return std::shared_ptr<GrpcBaseTask>(new DropTableTask(table_name)); return std::shared_ptr<GrpcBaseTask>(new DropTableTask(table_name));
} }
ServerError ErrorCode
DropTableTask::OnExecute() { DropTableTask::OnExecute() {
try { try {
TimeRecorder rc("DropTableTask"); TimeRecorder rc("DropTableTask");
//step 1: check arguments //step 1: check arguments
ServerError res = ValidationUtil::ValidateTableName(table_name_); ErrorCode res = ValidationUtil::ValidateTableName(table_name_);
if (res != SERVER_SUCCESS) { if (res != SERVER_SUCCESS) {
return SetError(res, "Invalid table name: " + table_name_); return SetError(res, "Invalid table name: " + table_name_);
} }
@ -351,7 +351,7 @@ DropTableTask::OnExecute() {
table_info.table_id_ = table_name_; table_info.table_id_ = table_name_;
engine::Status stat = DBWrapper::DB()->DescribeTable(table_info); engine::Status stat = DBWrapper::DB()->DescribeTable(table_info);
if (!stat.ok()) { if (!stat.ok()) {
if (stat.IsNotFound()) { if (stat.code() == DB_NOT_FOUND) {
return SetError(SERVER_TABLE_NOT_EXIST, "Table " + table_name_ + " not exists"); return SetError(SERVER_TABLE_NOT_EXIST, "Table " + table_name_ + " not exists");
} else { } else {
return SetError(DB_META_TRANSACTION_FAILED, stat.ToString()); return SetError(DB_META_TRANSACTION_FAILED, stat.ToString());
@ -387,7 +387,7 @@ ShowTablesTask::Create(::grpc::ServerWriter<::milvus::grpc::TableName> *writer)
return std::shared_ptr<GrpcBaseTask>(new ShowTablesTask(writer)); return std::shared_ptr<GrpcBaseTask>(new ShowTablesTask(writer));
} }
ServerError ErrorCode
ShowTablesTask::OnExecute() { ShowTablesTask::OnExecute() {
std::vector<engine::meta::TableSchema> schema_array; std::vector<engine::meta::TableSchema> schema_array;
engine::Status stat = DBWrapper::DB()->AllTables(schema_array); engine::Status stat = DBWrapper::DB()->AllTables(schema_array);
@ -424,13 +424,13 @@ InsertTask::Create(const ::milvus::grpc::InsertParam *insert_param,
return std::shared_ptr<GrpcBaseTask>(new InsertTask(insert_param, record_ids)); return std::shared_ptr<GrpcBaseTask>(new InsertTask(insert_param, record_ids));
} }
ServerError ErrorCode
InsertTask::OnExecute() { InsertTask::OnExecute() {
try { try {
TimeRecorder rc("InsertVectorTask"); TimeRecorder rc("InsertVectorTask");
//step 1: check arguments //step 1: check arguments
ServerError res = ValidationUtil::ValidateTableName(insert_param_->table_name()); ErrorCode res = ValidationUtil::ValidateTableName(insert_param_->table_name());
if (res != SERVER_SUCCESS) { if (res != SERVER_SUCCESS) {
return SetError(res, "Invalid table name: " + insert_param_->table_name()); return SetError(res, "Invalid table name: " + insert_param_->table_name());
} }
@ -450,7 +450,7 @@ InsertTask::OnExecute() {
table_info.table_id_ = insert_param_->table_name(); table_info.table_id_ = insert_param_->table_name();
engine::Status stat = DBWrapper::DB()->DescribeTable(table_info); engine::Status stat = DBWrapper::DB()->DescribeTable(table_info);
if (!stat.ok()) { if (!stat.ok()) {
if (stat.IsNotFound()) { if (stat.code() == DB_NOT_FOUND) {
return SetError(SERVER_TABLE_NOT_EXIST, return SetError(SERVER_TABLE_NOT_EXIST,
"Table " + insert_param_->table_name() + " not exists"); "Table " + insert_param_->table_name() + " not exists");
} else { } else {
@ -489,7 +489,7 @@ InsertTask::OnExecute() {
} }
uint64_t vec_dim = insert_param_->row_record_array(i).vector_data().size(); uint64_t vec_dim = insert_param_->row_record_array(i).vector_data().size();
if (vec_dim != table_info.dimension_) { if (vec_dim != table_info.dimension_) {
ServerError error_code = SERVER_INVALID_VECTOR_DIMENSION; ErrorCode error_code = SERVER_INVALID_VECTOR_DIMENSION;
std::string error_msg = "Invalid rowrecord dimension: " + std::to_string(vec_dim) std::string error_msg = "Invalid rowrecord dimension: " + std::to_string(vec_dim)
+ " vs. table dimension:" + + " vs. table dimension:" +
std::to_string(table_info.dimension_); std::to_string(table_info.dimension_);
@ -569,14 +569,14 @@ SearchTask::Create(const ::milvus::grpc::SearchParam *search_vector_infos,
response)); response));
} }
ServerError ErrorCode
SearchTask::OnExecute() { SearchTask::OnExecute() {
try { try {
TimeRecorder rc("SearchTask"); TimeRecorder rc("SearchTask");
//step 1: check table name //step 1: check table name
std::string table_name_ = search_param_->table_name(); std::string table_name_ = search_param_->table_name();
ServerError res = ValidationUtil::ValidateTableName(table_name_); ErrorCode res = ValidationUtil::ValidateTableName(table_name_);
if (res != SERVER_SUCCESS) { if (res != SERVER_SUCCESS) {
return SetError(res, "Invalid table name: " + table_name_); return SetError(res, "Invalid table name: " + table_name_);
} }
@ -586,7 +586,7 @@ SearchTask::OnExecute() {
table_info.table_id_ = table_name_; table_info.table_id_ = table_name_;
engine::Status stat = DBWrapper::DB()->DescribeTable(table_info); engine::Status stat = DBWrapper::DB()->DescribeTable(table_info);
if (!stat.ok()) { if (!stat.ok()) {
if (stat.IsNotFound()) { if (stat.code() == DB_NOT_FOUND) {
return SetError(SERVER_TABLE_NOT_EXIST, "Table " + table_name_ + " not exists"); return SetError(SERVER_TABLE_NOT_EXIST, "Table " + table_name_ + " not exists");
} else { } else {
return SetError(DB_META_TRANSACTION_FAILED, stat.ToString()); return SetError(DB_META_TRANSACTION_FAILED, stat.ToString());
@ -612,7 +612,7 @@ SearchTask::OnExecute() {
//step 4: check date range, and convert to db dates //step 4: check date range, and convert to db dates
std::vector<DB_DATE> dates; std::vector<DB_DATE> dates;
ServerError error_code = SERVER_SUCCESS; ErrorCode error_code = SERVER_SUCCESS;
std::string error_msg; std::string error_msg;
std::vector<::milvus::grpc::Range> range_array; std::vector<::milvus::grpc::Range> range_array;
@ -642,7 +642,7 @@ SearchTask::OnExecute() {
} }
uint64_t query_vec_dim = search_param_->query_record_array(i).vector_data().size(); uint64_t query_vec_dim = search_param_->query_record_array(i).vector_data().size();
if (query_vec_dim != table_info.dimension_) { if (query_vec_dim != table_info.dimension_) {
ServerError error_code = SERVER_INVALID_VECTOR_DIMENSION; ErrorCode error_code = SERVER_INVALID_VECTOR_DIMENSION;
std::string error_msg = "Invalid rowrecord dimension: " + std::to_string(query_vec_dim) std::string error_msg = "Invalid rowrecord dimension: " + std::to_string(query_vec_dim)
+ " vs. table dimension:" + std::to_string(table_info.dimension_); + " vs. table dimension:" + std::to_string(table_info.dimension_);
return SetError(error_code, error_msg); return SetError(error_code, error_msg);
@ -722,13 +722,13 @@ CountTableTask::Create(const std::string &table_name, int64_t &row_count) {
return std::shared_ptr<GrpcBaseTask>(new CountTableTask(table_name, row_count)); return std::shared_ptr<GrpcBaseTask>(new CountTableTask(table_name, row_count));
} }
ServerError ErrorCode
CountTableTask::OnExecute() { CountTableTask::OnExecute() {
try { try {
TimeRecorder rc("GetTableRowCountTask"); TimeRecorder rc("GetTableRowCountTask");
//step 1: check arguments //step 1: check arguments
ServerError res = SERVER_SUCCESS; ErrorCode res = SERVER_SUCCESS;
res = ValidationUtil::ValidateTableName(table_name_); res = ValidationUtil::ValidateTableName(table_name_);
if (res != SERVER_SUCCESS) { if (res != SERVER_SUCCESS) {
return SetError(res, "Invalid table name: " + table_name_); return SetError(res, "Invalid table name: " + table_name_);
@ -765,7 +765,7 @@ CmdTask::Create(const std::string &cmd, std::string &result) {
return std::shared_ptr<GrpcBaseTask>(new CmdTask(cmd, result)); return std::shared_ptr<GrpcBaseTask>(new CmdTask(cmd, result));
} }
ServerError ErrorCode
CmdTask::OnExecute() { CmdTask::OnExecute() {
if (cmd_ == "version") { if (cmd_ == "version") {
result_ = MILVUS_VERSION; result_ = MILVUS_VERSION;
@ -794,14 +794,14 @@ DeleteByRangeTask::Create(const ::milvus::grpc::DeleteByRangeParam *delete_by_ra
return std::shared_ptr<GrpcBaseTask>(new DeleteByRangeTask(delete_by_range_param)); return std::shared_ptr<GrpcBaseTask>(new DeleteByRangeTask(delete_by_range_param));
} }
ServerError ErrorCode
DeleteByRangeTask::OnExecute() { DeleteByRangeTask::OnExecute() {
try { try {
TimeRecorder rc("DeleteByRangeTask"); TimeRecorder rc("DeleteByRangeTask");
//step 1: check arguments //step 1: check arguments
std::string table_name = delete_by_range_param_->table_name(); std::string table_name = delete_by_range_param_->table_name();
ServerError res = ValidationUtil::ValidateTableName(table_name); ErrorCode res = ValidationUtil::ValidateTableName(table_name);
if (res != SERVER_SUCCESS) { if (res != SERVER_SUCCESS) {
return SetError(res, "Invalid table name: " + table_name); return SetError(res, "Invalid table name: " + table_name);
} }
@ -811,7 +811,7 @@ DeleteByRangeTask::OnExecute() {
table_info.table_id_ = table_name; table_info.table_id_ = table_name;
engine::Status stat = DBWrapper::DB()->DescribeTable(table_info); engine::Status stat = DBWrapper::DB()->DescribeTable(table_info);
if (!stat.ok()) { if (!stat.ok()) {
if (stat.IsNotFound()) { if (stat.code(), DB_NOT_FOUND) {
return SetError(SERVER_TABLE_NOT_EXIST, "Table " + table_name + " not exists"); return SetError(SERVER_TABLE_NOT_EXIST, "Table " + table_name + " not exists");
} else { } else {
return SetError(DB_META_TRANSACTION_FAILED, stat.ToString()); return SetError(DB_META_TRANSACTION_FAILED, stat.ToString());
@ -822,7 +822,7 @@ DeleteByRangeTask::OnExecute() {
//step 3: check date range, and convert to db dates //step 3: check date range, and convert to db dates
std::vector<DB_DATE> dates; std::vector<DB_DATE> dates;
ServerError error_code = SERVER_SUCCESS; ErrorCode error_code = SERVER_SUCCESS;
std::string error_msg; std::string error_msg;
std::vector<::milvus::grpc::Range> range_array; std::vector<::milvus::grpc::Range> range_array;
@ -862,13 +862,13 @@ PreloadTableTask::Create(const std::string &table_name){
return std::shared_ptr<GrpcBaseTask>(new PreloadTableTask(table_name)); return std::shared_ptr<GrpcBaseTask>(new PreloadTableTask(table_name));
} }
ServerError ErrorCode
PreloadTableTask::OnExecute() { PreloadTableTask::OnExecute() {
try { try {
TimeRecorder rc("PreloadTableTask"); TimeRecorder rc("PreloadTableTask");
//step 1: check arguments //step 1: check arguments
ServerError res = ValidationUtil::ValidateTableName(table_name_); ErrorCode res = ValidationUtil::ValidateTableName(table_name_);
if (res != SERVER_SUCCESS) { if (res != SERVER_SUCCESS) {
return SetError(res, "Invalid table name: " + table_name_); return SetError(res, "Invalid table name: " + table_name_);
} }
@ -902,13 +902,13 @@ DescribeIndexTask::Create(const std::string &table_name,
return std::shared_ptr<GrpcBaseTask>(new DescribeIndexTask(table_name, index_param)); return std::shared_ptr<GrpcBaseTask>(new DescribeIndexTask(table_name, index_param));
} }
ServerError ErrorCode
DescribeIndexTask::OnExecute() { DescribeIndexTask::OnExecute() {
try { try {
TimeRecorder rc("DescribeIndexTask"); TimeRecorder rc("DescribeIndexTask");
//step 1: check arguments //step 1: check arguments
ServerError res = ValidationUtil::ValidateTableName(table_name_); ErrorCode res = ValidationUtil::ValidateTableName(table_name_);
if (res != SERVER_SUCCESS) { if (res != SERVER_SUCCESS) {
return SetError(res, "Invalid table name: " + table_name_); return SetError(res, "Invalid table name: " + table_name_);
} }
@ -944,13 +944,13 @@ DropIndexTask::Create(const std::string &table_name){
return std::shared_ptr<GrpcBaseTask>(new DropIndexTask(table_name)); return std::shared_ptr<GrpcBaseTask>(new DropIndexTask(table_name));
} }
ServerError ErrorCode
DropIndexTask::OnExecute() { DropIndexTask::OnExecute() {
try { try {
TimeRecorder rc("DropIndexTask"); TimeRecorder rc("DropIndexTask");
//step 1: check arguments //step 1: check arguments
ServerError res = ValidationUtil::ValidateTableName(table_name_); ErrorCode res = ValidationUtil::ValidateTableName(table_name_);
if (res != SERVER_SUCCESS) { if (res != SERVER_SUCCESS) {
return SetError(res, "Invalid table name: " + table_name_); return SetError(res, "Invalid table name: " + table_name_);
} }

28
cpp/src/server/grpc_impl/GrpcRequestTask.h
View File

@ -29,7 +29,7 @@ protected:
explicit explicit
CreateTableTask(const ::milvus::grpc::TableSchema *request); CreateTableTask(const ::milvus::grpc::TableSchema *request);
ServerError ErrorCode
OnExecute() override; OnExecute() override;
private: private:
@ -45,7 +45,7 @@ public:
protected: protected:
HasTableTask(const std::string &request, bool &has_table); HasTableTask(const std::string &request, bool &has_table);
ServerError ErrorCode
OnExecute() override; OnExecute() override;
@ -63,7 +63,7 @@ public:
protected: protected:
DescribeTableTask(const std::string &table_name, ::milvus::grpc::TableSchema *schema); DescribeTableTask(const std::string &table_name, ::milvus::grpc::TableSchema *schema);
ServerError ErrorCode
OnExecute() override; OnExecute() override;
@ -82,7 +82,7 @@ protected:
explicit explicit
DropTableTask(const std::string &table_name); DropTableTask(const std::string &table_name);
ServerError ErrorCode
OnExecute() override; OnExecute() override;
@ -100,7 +100,7 @@ protected:
explicit explicit
CreateIndexTask(const ::milvus::grpc::IndexParam *index_Param); CreateIndexTask(const ::milvus::grpc::IndexParam *index_Param);
ServerError ErrorCode
OnExecute() override; OnExecute() override;
@ -118,7 +118,7 @@ protected:
explicit explicit
ShowTablesTask(::grpc::ServerWriter<::milvus::grpc::TableName> *writer); ShowTablesTask(::grpc::ServerWriter<::milvus::grpc::TableName> *writer);
ServerError ErrorCode
OnExecute() override; OnExecute() override;
private: private:
@ -136,7 +136,7 @@ protected:
InsertTask(const ::milvus::grpc::InsertParam *insert_Param, InsertTask(const ::milvus::grpc::InsertParam *insert_Param,
::milvus::grpc::VectorIds *record_ids_); ::milvus::grpc::VectorIds *record_ids_);
ServerError ErrorCode
OnExecute() override; OnExecute() override;
private: private:
@ -157,7 +157,7 @@ protected:
const std::vector<std::string> &file_id_array, const std::vector<std::string> &file_id_array,
::milvus::grpc::TopKQueryResultList *response); ::milvus::grpc::TopKQueryResultList *response);
ServerError ErrorCode
OnExecute() override; OnExecute() override;
private: private:
@ -175,7 +175,7 @@ public:
protected: protected:
CountTableTask(const std::string &table_name, int64_t &row_count); CountTableTask(const std::string &table_name, int64_t &row_count);
ServerError ErrorCode
OnExecute() override; OnExecute() override;
private: private:
@ -192,7 +192,7 @@ public:
protected: protected:
CmdTask(const std::string &cmd, std::string &result); CmdTask(const std::string &cmd, std::string &result);
ServerError ErrorCode
OnExecute() override; OnExecute() override;
private: private:
@ -209,7 +209,7 @@ public:
protected: protected:
DeleteByRangeTask(const ::milvus::grpc::DeleteByRangeParam *delete_by_range_param); DeleteByRangeTask(const ::milvus::grpc::DeleteByRangeParam *delete_by_range_param);
ServerError ErrorCode
OnExecute() override; OnExecute() override;
private: private:
@ -225,7 +225,7 @@ public:
protected: protected:
PreloadTableTask(const std::string &table_name); PreloadTableTask(const std::string &table_name);
ServerError ErrorCode
OnExecute() override; OnExecute() override;
private: private:
@ -243,7 +243,7 @@ protected:
DescribeIndexTask(const std::string &table_name, DescribeIndexTask(const std::string &table_name,
::milvus::grpc::IndexParam *index_param); ::milvus::grpc::IndexParam *index_param);
ServerError ErrorCode
OnExecute() override; OnExecute() override;
private: private:
@ -260,7 +260,7 @@ public:
protected: protected:
DropIndexTask(const std::string &table_name); DropIndexTask(const std::string &table_name);
ServerError ErrorCode
OnExecute() override; OnExecute() override;
private: private:

6
cpp/src/utils/CommonUtil.cpp
View File

@ -61,7 +61,7 @@ bool CommonUtil::IsDirectoryExist(const std::string &path) {
return true; return true;
} }
ServerError CommonUtil::CreateDirectory(const std::string &path) { ErrorCode CommonUtil::CreateDirectory(const std::string &path) {
if(path.empty()) { if(path.empty()) {
return SERVER_SUCCESS; return SERVER_SUCCESS;
} }
@ -74,7 +74,7 @@ ServerError CommonUtil::CreateDirectory(const std::string &path) {
fs::path fs_path(path); fs::path fs_path(path);
fs::path parent_path = fs_path.parent_path(); fs::path parent_path = fs_path.parent_path();
ServerError err = CreateDirectory(parent_path.string()); ErrorCode err = CreateDirectory(parent_path.string());
if(err != SERVER_SUCCESS){ if(err != SERVER_SUCCESS){
return err; return err;
} }
@ -122,7 +122,7 @@ namespace {
} }
} }
ServerError CommonUtil::DeleteDirectory(const std::string &path) { ErrorCode CommonUtil::DeleteDirectory(const std::string &path) {
if(path.empty()) { if(path.empty()) {
return SERVER_SUCCESS; return SERVER_SUCCESS;
} }

4
cpp/src/utils/CommonUtil.h
View File

@ -22,8 +22,8 @@ class CommonUtil {
static bool IsFileExist(const std::string &path); static bool IsFileExist(const std::string &path);
static uint64_t GetFileSize(const std::string &path); static uint64_t GetFileSize(const std::string &path);
static bool IsDirectoryExist(const std::string &path); static bool IsDirectoryExist(const std::string &path);
static ServerError CreateDirectory(const std::string &path); static ErrorCode CreateDirectory(const std::string &path);
static ServerError DeleteDirectory(const std::string &path); static ErrorCode DeleteDirectory(const std::string &path);
static std::string GetExePath(); static std::string GetExePath();

135
cpp/src/utils/Error.h
View File

@ -11,82 +11,95 @@
namespace zilliz { namespace zilliz {
namespace milvus { namespace milvus {
namespace server {
using ServerError = int32_t; using ErrorCode = int32_t;
constexpr ServerError SERVER_SUCCESS = 0; constexpr ErrorCode SERVER_SUCCESS = 0;
constexpr ErrorCode SERVER_ERROR_CODE_BASE = 0x30000;
constexpr ServerError SERVER_ERROR_CODE_BASE = 0x30000; constexpr ErrorCode
ToServerErrorCode(const ErrorCode error_code) {
constexpr ServerError
ToGlobalServerErrorCode(const ServerError error_code) {
return SERVER_ERROR_CODE_BASE + error_code; return SERVER_ERROR_CODE_BASE + error_code;
} }
constexpr ServerError SERVER_UNEXPECTED_ERROR = ToGlobalServerErrorCode(1); constexpr ErrorCode DB_SUCCESS = 0;
constexpr ServerError SERVER_UNSUPPORTED_ERROR = ToGlobalServerErrorCode(2); constexpr ErrorCode DB_ERROR_CODE_BASE = 0x40000;
constexpr ServerError SERVER_NULL_POINTER = ToGlobalServerErrorCode(3); constexpr ErrorCode
constexpr ServerError SERVER_INVALID_ARGUMENT = ToGlobalServerErrorCode(4); ToDbErrorCode(const ErrorCode error_code) {
constexpr ServerError SERVER_FILE_NOT_FOUND = ToGlobalServerErrorCode(5); return DB_ERROR_CODE_BASE + error_code;
constexpr ServerError SERVER_NOT_IMPLEMENT = ToGlobalServerErrorCode(6); }
constexpr ServerError SERVER_BLOCKING_QUEUE_EMPTY = ToGlobalServerErrorCode(7);
constexpr ServerError SERVER_CANNOT_CREATE_FOLDER = ToGlobalServerErrorCode(8);
constexpr ServerError SERVER_CANNOT_CREATE_FILE = ToGlobalServerErrorCode(9);
constexpr ServerError SERVER_CANNOT_DELETE_FOLDER = ToGlobalServerErrorCode(10);
constexpr ServerError SERVER_CANNOT_DELETE_FILE = ToGlobalServerErrorCode(11);
constexpr ServerError SERVER_BUILD_INDEX_ERROR = ToGlobalServerErrorCode(12);
constexpr ServerError SERVER_TABLE_NOT_EXIST = ToGlobalServerErrorCode(100); constexpr ErrorCode KNOWHERE_SUCCESS = 0;
constexpr ServerError SERVER_INVALID_TABLE_NAME = ToGlobalServerErrorCode(101); constexpr ErrorCode KNOWHERE_ERROR_CODE_BASE = 0x50000;
constexpr ServerError SERVER_INVALID_TABLE_DIMENSION = ToGlobalServerErrorCode(102); constexpr ErrorCode
constexpr ServerError SERVER_INVALID_TIME_RANGE = ToGlobalServerErrorCode(103); ToKnowhereErrorCode(const ErrorCode error_code) {
constexpr ServerError SERVER_INVALID_VECTOR_DIMENSION = ToGlobalServerErrorCode(104); return KNOWHERE_ERROR_CODE_BASE + error_code;
constexpr ServerError SERVER_INVALID_INDEX_TYPE = ToGlobalServerErrorCode(105); }
constexpr ServerError SERVER_INVALID_ROWRECORD = ToGlobalServerErrorCode(106);
constexpr ServerError SERVER_INVALID_ROWRECORD_ARRAY = ToGlobalServerErrorCode(107);
constexpr ServerError SERVER_INVALID_TOPK = ToGlobalServerErrorCode(108);
constexpr ServerError SERVER_ILLEGAL_VECTOR_ID = ToGlobalServerErrorCode(109);
constexpr ServerError SERVER_ILLEGAL_SEARCH_RESULT = ToGlobalServerErrorCode(110);
constexpr ServerError SERVER_CACHE_ERROR = ToGlobalServerErrorCode(111);
constexpr ServerError SERVER_WRITE_ERROR = ToGlobalServerErrorCode(112);
constexpr ServerError SERVER_INVALID_NPROBE = ToGlobalServerErrorCode(113);
constexpr ServerError SERVER_INVALID_INDEX_NLIST = ToGlobalServerErrorCode(114);
constexpr ServerError SERVER_INVALID_INDEX_METRIC_TYPE = ToGlobalServerErrorCode(115);
constexpr ServerError SERVER_INVALID_INDEX_FILE_SIZE = ToGlobalServerErrorCode(116);
//server error code
constexpr ErrorCode SERVER_UNEXPECTED_ERROR = ToServerErrorCode(1);
constexpr ErrorCode SERVER_UNSUPPORTED_ERROR = ToServerErrorCode(2);
constexpr ErrorCode SERVER_NULL_POINTER = ToServerErrorCode(3);
constexpr ErrorCode SERVER_INVALID_ARGUMENT = ToServerErrorCode(4);
constexpr ErrorCode SERVER_FILE_NOT_FOUND = ToServerErrorCode(5);
constexpr ErrorCode SERVER_NOT_IMPLEMENT = ToServerErrorCode(6);
constexpr ErrorCode SERVER_BLOCKING_QUEUE_EMPTY = ToServerErrorCode(7);
constexpr ErrorCode SERVER_CANNOT_CREATE_FOLDER = ToServerErrorCode(8);
constexpr ErrorCode SERVER_CANNOT_CREATE_FILE = ToServerErrorCode(9);
constexpr ErrorCode SERVER_CANNOT_DELETE_FOLDER = ToServerErrorCode(10);
constexpr ErrorCode SERVER_CANNOT_DELETE_FILE = ToServerErrorCode(11);
constexpr ErrorCode SERVER_BUILD_INDEX_ERROR = ToServerErrorCode(12);
constexpr ServerError SERVER_LICENSE_FILE_NOT_EXIST = ToGlobalServerErrorCode(500); constexpr ErrorCode SERVER_TABLE_NOT_EXIST = ToServerErrorCode(100);
constexpr ServerError SERVER_LICENSE_VALIDATION_FAIL = ToGlobalServerErrorCode(501); constexpr ErrorCode SERVER_INVALID_TABLE_NAME = ToServerErrorCode(101);
constexpr ErrorCode SERVER_INVALID_TABLE_DIMENSION = ToServerErrorCode(102);
constexpr ErrorCode SERVER_INVALID_TIME_RANGE = ToServerErrorCode(103);
constexpr ErrorCode SERVER_INVALID_VECTOR_DIMENSION = ToServerErrorCode(104);
constexpr ErrorCode SERVER_INVALID_INDEX_TYPE = ToServerErrorCode(105);
constexpr ErrorCode SERVER_INVALID_ROWRECORD = ToServerErrorCode(106);
constexpr ErrorCode SERVER_INVALID_ROWRECORD_ARRAY = ToServerErrorCode(107);
constexpr ErrorCode SERVER_INVALID_TOPK = ToServerErrorCode(108);
constexpr ErrorCode SERVER_ILLEGAL_VECTOR_ID = ToServerErrorCode(109);
constexpr ErrorCode SERVER_ILLEGAL_SEARCH_RESULT = ToServerErrorCode(110);
constexpr ErrorCode SERVER_CACHE_ERROR = ToServerErrorCode(111);
constexpr ErrorCode SERVER_WRITE_ERROR = ToServerErrorCode(112);
constexpr ErrorCode SERVER_INVALID_NPROBE = ToServerErrorCode(113);
constexpr ErrorCode SERVER_INVALID_INDEX_NLIST = ToServerErrorCode(114);
constexpr ErrorCode SERVER_INVALID_INDEX_METRIC_TYPE = ToServerErrorCode(115);
constexpr ErrorCode SERVER_INVALID_INDEX_FILE_SIZE = ToServerErrorCode(116);
constexpr ServerError DB_META_TRANSACTION_FAILED = ToGlobalServerErrorCode(1000); //db error code
constexpr ErrorCode DB_META_TRANSACTION_FAILED = ToDbErrorCode(1);
constexpr ErrorCode DB_ERROR = ToDbErrorCode(2);
constexpr ErrorCode DB_NOT_FOUND = ToDbErrorCode(3);
constexpr ErrorCode DB_ALREADY_EXIST = ToDbErrorCode(4);
constexpr ErrorCode DB_INVALID_PATH = ToDbErrorCode(5);
using KnowhereError = int32_t; //knowhere error code
constexpr KnowhereError KNOWHERE_SUCCESS = 0; constexpr ErrorCode KNOWHERE_ERROR = ToKnowhereErrorCode(1);
constexpr KnowhereError KNOWHERE_ERROR = ToGlobalServerErrorCode(1); constexpr ErrorCode KNOWHERE_INVALID_ARGUMENT = ToKnowhereErrorCode(2);
constexpr KnowhereError KNOWHERE_INVALID_ARGUMENT = ToGlobalServerErrorCode(2); constexpr ErrorCode KNOWHERE_UNEXPECTED_ERROR = ToKnowhereErrorCode(3);
constexpr KnowhereError KNOWHERE_UNEXPECTED_ERROR = ToGlobalServerErrorCode(3);
class ServerException : public std::exception { namespace server {
public: class ServerException : public std::exception {
ServerException(ServerError error_code, public:
const std::string &message = std::string()) ServerException(ErrorCode error_code,
const std::string &message = std::string())
: error_code_(error_code), message_(message) {} : error_code_(error_code), message_(message) {}
public: public:
ServerError error_code() const { ErrorCode error_code() const {
return error_code_; return error_code_;
} }
virtual const char *what() const noexcept { virtual const char *what() const noexcept {
return message_.c_str(); return message_.c_str();
} }
private: private:
ServerError error_code_; ErrorCode error_code_;
std::string message_; std::string message_;
}; };
}
} // namespace server
} // namespace milvus } // namespace milvus
} // namespace zilliz } // namespace zilliz

14
cpp/src/utils/StringHelpFunctions.cpp
View File

@ -24,9 +24,9 @@ void StringHelpFunctions::TrimStringQuote(std::string &string, const std::string
} }
} }
ServerError StringHelpFunctions::SplitStringByDelimeter(const std::string &str, ErrorCode StringHelpFunctions::SplitStringByDelimeter(const std::string &str,
const std::string &delimeter, const std::string &delimeter,
std::vector<std::string> &result) { std::vector<std::string> &result) {
if(str.empty()) { if(str.empty()) {
return SERVER_SUCCESS; return SERVER_SUCCESS;
} }
@ -46,10 +46,10 @@ ServerError StringHelpFunctions::SplitStringByDelimeter(const std::string &str,
return SERVER_SUCCESS; return SERVER_SUCCESS;
} }
ServerError StringHelpFunctions::SplitStringByQuote(const std::string &str, ErrorCode StringHelpFunctions::SplitStringByQuote(const std::string &str,
const std::string &delimeter, const std::string &delimeter,
const std::string &quote, const std::string &quote,
std::vector<std::string> &result) { std::vector<std::string> &result) {
if (quote.empty()) { if (quote.empty()) {
return SplitStringByDelimeter(str, delimeter, result); return SplitStringByDelimeter(str, delimeter, result);
} }

14
cpp/src/utils/StringHelpFunctions.h
View File

@ -29,9 +29,9 @@ public:
// ,b, | b | // ,b, | b |
// ,, | | // ,, | |
// a a // a a
static ServerError SplitStringByDelimeter(const std::string &str, static ErrorCode SplitStringByDelimeter(const std::string &str,
const std::string &delimeter, const std::string &delimeter,
std::vector<std::string> &result); std::vector<std::string> &result);
//assume the table has two columns, quote='\"', delimeter=',' //assume the table has two columns, quote='\"', delimeter=','
// a,b a | b // a,b a | b
@ -40,10 +40,10 @@ public:
// "aa,bb" aa,bb // "aa,bb" aa,bb
// 55,1122\"aa,bb\",yyy,\"kkk\" 55 | 1122aa,bb | yyy | kkk // 55,1122\"aa,bb\",yyy,\"kkk\" 55 | 1122aa,bb | yyy | kkk
// "55,1122"aa,bb",yyy,"kkk" illegal // "55,1122"aa,bb",yyy,"kkk" illegal
static ServerError SplitStringByQuote(const std::string &str, static ErrorCode SplitStringByQuote(const std::string &str,
const std::string &delimeter, const std::string &delimeter,
const std::string &quote, const std::string &quote,
std::vector<std::string> &result); std::vector<std::string> &result);
}; };

20
cpp/src/utils/ValidationUtil.cpp
View File

@ -12,7 +12,7 @@ constexpr size_t table_name_size_limit = 255;
constexpr int64_t table_dimension_limit = 16384; constexpr int64_t table_dimension_limit = 16384;
constexpr int32_t index_file_size_limit = 4096; //index trigger size max = 4096 MB constexpr int32_t index_file_size_limit = 4096; //index trigger size max = 4096 MB
ServerError ErrorCode
ValidationUtil::ValidateTableName(const std::string &table_name) { ValidationUtil::ValidateTableName(const std::string &table_name) {
// Table name shouldn't be empty. // Table name shouldn't be empty.
@ -46,7 +46,7 @@ ValidationUtil::ValidateTableName(const std::string &table_name) {
return SERVER_SUCCESS; return SERVER_SUCCESS;
} }
ServerError ErrorCode
ValidationUtil::ValidateTableDimension(int64_t dimension) { ValidationUtil::ValidateTableDimension(int64_t dimension) {
if (dimension <= 0 || dimension > table_dimension_limit) { if (dimension <= 0 || dimension > table_dimension_limit) {
SERVER_LOG_ERROR << "Table dimension excceed the limitation: " << table_dimension_limit; SERVER_LOG_ERROR << "Table dimension excceed the limitation: " << table_dimension_limit;
@ -56,7 +56,7 @@ ValidationUtil::ValidateTableDimension(int64_t dimension) {
} }
} }
ServerError ErrorCode
ValidationUtil::ValidateTableIndexType(int32_t index_type) { ValidationUtil::ValidateTableIndexType(int32_t index_type) {
int engine_type = (int)engine::EngineType(index_type); int engine_type = (int)engine::EngineType(index_type);
if(engine_type <= 0 || engine_type > (int)engine::EngineType::MAX_VALUE) { if(engine_type <= 0 || engine_type > (int)engine::EngineType::MAX_VALUE) {
@ -66,7 +66,7 @@ ValidationUtil::ValidateTableIndexType(int32_t index_type) {
return SERVER_SUCCESS; return SERVER_SUCCESS;
} }
ServerError ErrorCode
ValidationUtil::ValidateTableIndexNlist(int32_t nlist) { ValidationUtil::ValidateTableIndexNlist(int32_t nlist) {
if(nlist <= 0) { if(nlist <= 0) {
return SERVER_INVALID_INDEX_NLIST; return SERVER_INVALID_INDEX_NLIST;
@ -75,7 +75,7 @@ ValidationUtil::ValidateTableIndexNlist(int32_t nlist) {
return SERVER_SUCCESS; return SERVER_SUCCESS;
} }
ServerError ErrorCode
ValidationUtil::ValidateTableIndexFileSize(int64_t index_file_size) { ValidationUtil::ValidateTableIndexFileSize(int64_t index_file_size) {
if(index_file_size <= 0 || index_file_size > index_file_size_limit) { if(index_file_size <= 0 || index_file_size > index_file_size_limit) {
return SERVER_INVALID_INDEX_FILE_SIZE; return SERVER_INVALID_INDEX_FILE_SIZE;
@ -84,7 +84,7 @@ ValidationUtil::ValidateTableIndexFileSize(int64_t index_file_size) {
return SERVER_SUCCESS; return SERVER_SUCCESS;
} }
ServerError ErrorCode
ValidationUtil::ValidateTableIndexMetricType(int32_t metric_type) { ValidationUtil::ValidateTableIndexMetricType(int32_t metric_type) {
if(metric_type != (int32_t)engine::MetricType::L2 && metric_type != (int32_t)engine::MetricType::IP) { if(metric_type != (int32_t)engine::MetricType::L2 && metric_type != (int32_t)engine::MetricType::IP) {
return SERVER_INVALID_INDEX_METRIC_TYPE; return SERVER_INVALID_INDEX_METRIC_TYPE;
@ -92,7 +92,7 @@ ValidationUtil::ValidateTableIndexMetricType(int32_t metric_type) {
return SERVER_SUCCESS; return SERVER_SUCCESS;
} }
ServerError ErrorCode
ValidationUtil::ValidateSearchTopk(int64_t top_k, const engine::meta::TableSchema& table_schema) { ValidationUtil::ValidateSearchTopk(int64_t top_k, const engine::meta::TableSchema& table_schema) {
if (top_k <= 0) { if (top_k <= 0) {
return SERVER_INVALID_TOPK; return SERVER_INVALID_TOPK;
@ -101,7 +101,7 @@ ValidationUtil::ValidateSearchTopk(int64_t top_k, const engine::meta::TableSchem
return SERVER_SUCCESS; return SERVER_SUCCESS;
} }
ServerError ErrorCode
ValidationUtil::ValidateSearchNprobe(int64_t nprobe, const engine::meta::TableSchema& table_schema) { ValidationUtil::ValidateSearchNprobe(int64_t nprobe, const engine::meta::TableSchema& table_schema) {
if (nprobe <= 0 || nprobe > table_schema.nlist_) { if (nprobe <= 0 || nprobe > table_schema.nlist_) {
return SERVER_INVALID_NPROBE; return SERVER_INVALID_NPROBE;
@ -110,7 +110,7 @@ ValidationUtil::ValidateSearchNprobe(int64_t nprobe, const engine::meta::TableSc
return SERVER_SUCCESS; return SERVER_SUCCESS;
} }
ServerError ErrorCode
ValidationUtil::ValidateGpuIndex(uint32_t gpu_index) { ValidationUtil::ValidateGpuIndex(uint32_t gpu_index) {
int num_devices = 0; int num_devices = 0;
auto cuda_err = cudaGetDeviceCount(&num_devices); auto cuda_err = cudaGetDeviceCount(&num_devices);
@ -126,7 +126,7 @@ ValidationUtil::ValidateGpuIndex(uint32_t gpu_index) {
return SERVER_SUCCESS; return SERVER_SUCCESS;
} }
ServerError ErrorCode
ValidationUtil::GetGpuMemory(uint32_t gpu_index, size_t& memory) { ValidationUtil::GetGpuMemory(uint32_t gpu_index, size_t& memory) {
cudaDeviceProp deviceProp; cudaDeviceProp deviceProp;
auto cuda_err = cudaGetDeviceProperties(&deviceProp, gpu_index); auto cuda_err = cudaGetDeviceProperties(&deviceProp, gpu_index);

20
cpp/src/utils/ValidationUtil.h
View File

@ -9,34 +9,34 @@ namespace server {
class ValidationUtil { class ValidationUtil {
public: public:
static ServerError static ErrorCode
ValidateTableName(const std::string &table_name); ValidateTableName(const std::string &table_name);
static ServerError static ErrorCode
ValidateTableDimension(int64_t dimension); ValidateTableDimension(int64_t dimension);
static ServerError static ErrorCode
ValidateTableIndexType(int32_t index_type); ValidateTableIndexType(int32_t index_type);
static ServerError static ErrorCode
ValidateTableIndexNlist(int32_t nlist); ValidateTableIndexNlist(int32_t nlist);
static ServerError static ErrorCode
ValidateTableIndexFileSize(int64_t index_file_size); ValidateTableIndexFileSize(int64_t index_file_size);
static ServerError static ErrorCode
ValidateTableIndexMetricType(int32_t metric_type); ValidateTableIndexMetricType(int32_t metric_type);
static ServerError static ErrorCode
ValidateSearchTopk(int64_t top_k, const engine::meta::TableSchema& table_schema); ValidateSearchTopk(int64_t top_k, const engine::meta::TableSchema& table_schema);
static ServerError static ErrorCode
ValidateSearchNprobe(int64_t nprobe, const engine::meta::TableSchema& table_schema); ValidateSearchNprobe(int64_t nprobe, const engine::meta::TableSchema& table_schema);
static ServerError static ErrorCode
ValidateGpuIndex(uint32_t gpu_index); ValidateGpuIndex(uint32_t gpu_index);
static ServerError static ErrorCode
GetGpuMemory(uint32_t gpu_index, size_t &memory); GetGpuMemory(uint32_t gpu_index, size_t &memory);
}; };

70
cpp/src/wrapper/knowhere/vec_impl.cpp
View File

@ -21,7 +21,7 @@ namespace engine {
using namespace zilliz::knowhere; using namespace zilliz::knowhere;
server::KnowhereError VecIndexImpl::BuildAll(const long &nb, ErrorCode VecIndexImpl::BuildAll(const long &nb,
const float *xb, const float *xb,
const long *ids, const long *ids,
const Config &cfg, const Config &cfg,
@ -38,36 +38,36 @@ server::KnowhereError VecIndexImpl::BuildAll(const long &nb,
index_->Add(dataset, cfg); index_->Add(dataset, cfg);
} catch (KnowhereException &e) { } catch (KnowhereException &e) {
WRAPPER_LOG_ERROR << e.what(); WRAPPER_LOG_ERROR << e.what();
return server::KNOWHERE_UNEXPECTED_ERROR; return KNOWHERE_UNEXPECTED_ERROR;
} catch (jsoncons::json_exception &e) { } catch (jsoncons::json_exception &e) {
WRAPPER_LOG_ERROR << e.what(); WRAPPER_LOG_ERROR << e.what();
return server::KNOWHERE_INVALID_ARGUMENT; return KNOWHERE_INVALID_ARGUMENT;
} catch (std::exception &e) { } catch (std::exception &e) {
WRAPPER_LOG_ERROR << e.what(); WRAPPER_LOG_ERROR << e.what();
return server::KNOWHERE_ERROR; return KNOWHERE_ERROR;
} }
return server::KNOWHERE_SUCCESS; return KNOWHERE_SUCCESS;
} }
server::KnowhereError VecIndexImpl::Add(const long &nb, const float *xb, const long *ids, const Config &cfg) { ErrorCode VecIndexImpl::Add(const long &nb, const float *xb, const long *ids, const Config &cfg) {
try { try {
auto dataset = GenDatasetWithIds(nb, dim, xb, ids); auto dataset = GenDatasetWithIds(nb, dim, xb, ids);
index_->Add(dataset, cfg); index_->Add(dataset, cfg);
} catch (KnowhereException &e) { } catch (KnowhereException &e) {
WRAPPER_LOG_ERROR << e.what(); WRAPPER_LOG_ERROR << e.what();
return server::KNOWHERE_UNEXPECTED_ERROR; return KNOWHERE_UNEXPECTED_ERROR;
} catch (jsoncons::json_exception &e) { } catch (jsoncons::json_exception &e) {
WRAPPER_LOG_ERROR << e.what(); WRAPPER_LOG_ERROR << e.what();
return server::KNOWHERE_INVALID_ARGUMENT; return KNOWHERE_INVALID_ARGUMENT;
} catch (std::exception &e) { } catch (std::exception &e) {
WRAPPER_LOG_ERROR << e.what(); WRAPPER_LOG_ERROR << e.what();
return server::KNOWHERE_ERROR; return KNOWHERE_ERROR;
} }
return server::KNOWHERE_SUCCESS; return KNOWHERE_SUCCESS;
} }
server::KnowhereError VecIndexImpl::Search(const long &nq, const float *xq, float *dist, long *ids, const Config &cfg) { ErrorCode VecIndexImpl::Search(const long &nq, const float *xq, float *dist, long *ids, const Config &cfg) {
try { try {
auto k = cfg["k"].as<int>(); auto k = cfg["k"].as<int>();
auto dataset = GenDataset(nq, dim, xq); auto dataset = GenDataset(nq, dim, xq);
@ -105,15 +105,15 @@ server::KnowhereError VecIndexImpl::Search(const long &nq, const float *xq, floa
memcpy(dist, p_dist, sizeof(float) * nq * k); memcpy(dist, p_dist, sizeof(float) * nq * k);
} catch (KnowhereException &e) { } catch (KnowhereException &e) {
WRAPPER_LOG_ERROR << e.what(); WRAPPER_LOG_ERROR << e.what();
return server::KNOWHERE_UNEXPECTED_ERROR; return KNOWHERE_UNEXPECTED_ERROR;
} catch (jsoncons::json_exception &e) { } catch (jsoncons::json_exception &e) {
WRAPPER_LOG_ERROR << e.what(); WRAPPER_LOG_ERROR << e.what();
return server::KNOWHERE_INVALID_ARGUMENT; return KNOWHERE_INVALID_ARGUMENT;
} catch (std::exception &e) { } catch (std::exception &e) {
WRAPPER_LOG_ERROR << e.what(); WRAPPER_LOG_ERROR << e.what();
return server::KNOWHERE_ERROR; return KNOWHERE_ERROR;
} }
return server::KNOWHERE_SUCCESS; return KNOWHERE_SUCCESS;
} }
zilliz::knowhere::BinarySet VecIndexImpl::Serialize() { zilliz::knowhere::BinarySet VecIndexImpl::Serialize() {
@ -121,10 +121,10 @@ zilliz::knowhere::BinarySet VecIndexImpl::Serialize() {
return index_->Serialize(); return index_->Serialize();
} }
server::KnowhereError VecIndexImpl::Load(const zilliz::knowhere::BinarySet &index_binary) { ErrorCode VecIndexImpl::Load(const zilliz::knowhere::BinarySet &index_binary) {
index_->Load(index_binary); index_->Load(index_binary);
dim = Dimension(); dim = Dimension();
return server::KNOWHERE_SUCCESS; return KNOWHERE_SUCCESS;
} }
int64_t VecIndexImpl::Dimension() { int64_t VecIndexImpl::Dimension() {
@ -180,24 +180,24 @@ int64_t *BFIndex::GetRawIds() {
return std::static_pointer_cast<IDMAP>(index_)->GetRawIds(); return std::static_pointer_cast<IDMAP>(index_)->GetRawIds();
} }
server::KnowhereError BFIndex::Build(const Config &cfg) { ErrorCode BFIndex::Build(const Config &cfg) {
try { try {
dim = cfg["dim"].as<int>(); dim = cfg["dim"].as<int>();
std::static_pointer_cast<IDMAP>(index_)->Train(cfg); std::static_pointer_cast<IDMAP>(index_)->Train(cfg);
} catch (KnowhereException &e) { } catch (KnowhereException &e) {
WRAPPER_LOG_ERROR << e.what(); WRAPPER_LOG_ERROR << e.what();
return server::KNOWHERE_UNEXPECTED_ERROR; return KNOWHERE_UNEXPECTED_ERROR;
} catch (jsoncons::json_exception &e) { } catch (jsoncons::json_exception &e) {
WRAPPER_LOG_ERROR << e.what(); WRAPPER_LOG_ERROR << e.what();
return server::KNOWHERE_INVALID_ARGUMENT; return KNOWHERE_INVALID_ARGUMENT;
} catch (std::exception &e) { } catch (std::exception &e) {
WRAPPER_LOG_ERROR << e.what(); WRAPPER_LOG_ERROR << e.what();
return server::KNOWHERE_ERROR; return KNOWHERE_ERROR;
} }
return server::KNOWHERE_SUCCESS; return KNOWHERE_SUCCESS;
} }
server::KnowhereError BFIndex::BuildAll(const long &nb, ErrorCode BFIndex::BuildAll(const long &nb,
const float *xb, const float *xb,
const long *ids, const long *ids,
const Config &cfg, const Config &cfg,
@ -211,19 +211,19 @@ server::KnowhereError BFIndex::BuildAll(const long &nb,
index_->Add(dataset, cfg); index_->Add(dataset, cfg);
} catch (KnowhereException &e) { } catch (KnowhereException &e) {
WRAPPER_LOG_ERROR << e.what(); WRAPPER_LOG_ERROR << e.what();
return server::KNOWHERE_UNEXPECTED_ERROR; return KNOWHERE_UNEXPECTED_ERROR;
} catch (jsoncons::json_exception &e) { } catch (jsoncons::json_exception &e) {
WRAPPER_LOG_ERROR << e.what(); WRAPPER_LOG_ERROR << e.what();
return server::KNOWHERE_INVALID_ARGUMENT; return KNOWHERE_INVALID_ARGUMENT;
} catch (std::exception &e) { } catch (std::exception &e) {
WRAPPER_LOG_ERROR << e.what(); WRAPPER_LOG_ERROR << e.what();
return server::KNOWHERE_ERROR; return KNOWHERE_ERROR;
} }
return server::KNOWHERE_SUCCESS; return KNOWHERE_SUCCESS;
} }
// TODO(linxj): add lock here. // TODO(linxj): add lock here.
server::KnowhereError IVFMixIndex::BuildAll(const long &nb, ErrorCode IVFMixIndex::BuildAll(const long &nb,
const float *xb, const float *xb,
const long *ids, const long *ids,
const Config &cfg, const Config &cfg,
@ -245,26 +245,26 @@ server::KnowhereError IVFMixIndex::BuildAll(const long &nb,
type = ConvertToCpuIndexType(type); type = ConvertToCpuIndexType(type);
} else { } else {
WRAPPER_LOG_ERROR << "Build IVFMIXIndex Failed"; WRAPPER_LOG_ERROR << "Build IVFMIXIndex Failed";
return server::KNOWHERE_ERROR; return KNOWHERE_ERROR;
} }
} catch (KnowhereException &e) { } catch (KnowhereException &e) {
WRAPPER_LOG_ERROR << e.what(); WRAPPER_LOG_ERROR << e.what();
return server::KNOWHERE_UNEXPECTED_ERROR; return KNOWHERE_UNEXPECTED_ERROR;
} catch (jsoncons::json_exception &e) { } catch (jsoncons::json_exception &e) {
WRAPPER_LOG_ERROR << e.what(); WRAPPER_LOG_ERROR << e.what();
return server::KNOWHERE_INVALID_ARGUMENT; return KNOWHERE_INVALID_ARGUMENT;
} catch (std::exception &e) { } catch (std::exception &e) {
WRAPPER_LOG_ERROR << e.what(); WRAPPER_LOG_ERROR << e.what();
return server::KNOWHERE_ERROR; return KNOWHERE_ERROR;
} }
return server::KNOWHERE_SUCCESS; return KNOWHERE_SUCCESS;
} }
server::KnowhereError IVFMixIndex::Load(const zilliz::knowhere::BinarySet &index_binary) { ErrorCode IVFMixIndex::Load(const zilliz::knowhere::BinarySet &index_binary) {
//index_ = std::make_shared<IVF>(); //index_ = std::make_shared<IVF>();
index_->Load(index_binary); index_->Load(index_binary);
dim = Dimension(); dim = Dimension();
return server::KNOWHERE_SUCCESS; return KNOWHERE_SUCCESS;
} }
} }

16
cpp/src/wrapper/knowhere/vec_impl.h
View File

@ -19,7 +19,7 @@ class VecIndexImpl : public VecIndex {
public: public:
explicit VecIndexImpl(std::shared_ptr<zilliz::knowhere::VectorIndex> index, const IndexType &type) explicit VecIndexImpl(std::shared_ptr<zilliz::knowhere::VectorIndex> index, const IndexType &type)
: index_(std::move(index)), type(type) {}; : index_(std::move(index)), type(type) {};
server::KnowhereError BuildAll(const long &nb, ErrorCode BuildAll(const long &nb,
const float *xb, const float *xb,
const long *ids, const long *ids,
const Config &cfg, const Config &cfg,
@ -30,12 +30,12 @@ class VecIndexImpl : public VecIndex {
IndexType GetType() override; IndexType GetType() override;
int64_t Dimension() override; int64_t Dimension() override;
int64_t Count() override; int64_t Count() override;
server::KnowhereError Add(const long &nb, const float *xb, const long *ids, const Config &cfg) override; ErrorCode Add(const long &nb, const float *xb, const long *ids, const Config &cfg) override;
zilliz::knowhere::BinarySet Serialize() override; zilliz::knowhere::BinarySet Serialize() override;
server::KnowhereError Load(const zilliz::knowhere::BinarySet &index_binary) override; ErrorCode Load(const zilliz::knowhere::BinarySet &index_binary) override;
VecIndexPtr Clone() override; VecIndexPtr Clone() override;
int64_t GetDeviceId() override; int64_t GetDeviceId() override;
server::KnowhereError Search(const long &nq, const float *xq, float *dist, long *ids, const Config &cfg) override; ErrorCode Search(const long &nq, const float *xq, float *dist, long *ids, const Config &cfg) override;
protected: protected:
int64_t dim = 0; int64_t dim = 0;
@ -48,22 +48,22 @@ class IVFMixIndex : public VecIndexImpl {
explicit IVFMixIndex(std::shared_ptr<zilliz::knowhere::VectorIndex> index, const IndexType &type) explicit IVFMixIndex(std::shared_ptr<zilliz::knowhere::VectorIndex> index, const IndexType &type)
: VecIndexImpl(std::move(index), type) {}; : VecIndexImpl(std::move(index), type) {};
server::KnowhereError BuildAll(const long &nb, ErrorCode BuildAll(const long &nb,
const float *xb, const float *xb,
const long *ids, const long *ids,
const Config &cfg, const Config &cfg,
const long &nt, const long &nt,
const float *xt) override; const float *xt) override;
server::KnowhereError Load(const zilliz::knowhere::BinarySet &index_binary) override; ErrorCode Load(const zilliz::knowhere::BinarySet &index_binary) override;
}; };
class BFIndex : public VecIndexImpl { class BFIndex : public VecIndexImpl {
public: public:
explicit BFIndex(std::shared_ptr<zilliz::knowhere::VectorIndex> index) : VecIndexImpl(std::move(index), explicit BFIndex(std::shared_ptr<zilliz::knowhere::VectorIndex> index) : VecIndexImpl(std::move(index),
IndexType::FAISS_IDMAP) {}; IndexType::FAISS_IDMAP) {};
server::KnowhereError Build(const Config& cfg); ErrorCode Build(const Config& cfg);
float *GetRawVectors(); float *GetRawVectors();
server::KnowhereError BuildAll(const long &nb, ErrorCode BuildAll(const long &nb,
const float *xb, const float *xb,
const long *ids, const long *ids,
const Config &cfg, const Config &cfg,

8
cpp/src/wrapper/knowhere/vec_index.cpp
View File

@ -174,7 +174,7 @@ VecIndexPtr read_index(const std::string &location) {
return LoadVecIndex(current_type, load_data_list); return LoadVecIndex(current_type, load_data_list);
} }
server::KnowhereError write_index(VecIndexPtr index, const std::string &location) { ErrorCode write_index(VecIndexPtr index, const std::string &location) {
try { try {
auto binaryset = index->Serialize(); auto binaryset = index->Serialize();
auto index_type = index->GetType(); auto index_type = index->GetType();
@ -194,12 +194,12 @@ server::KnowhereError write_index(VecIndexPtr index, const std::string &location
} }
} catch (knowhere::KnowhereException &e) { } catch (knowhere::KnowhereException &e) {
WRAPPER_LOG_ERROR << e.what(); WRAPPER_LOG_ERROR << e.what();
return server::KNOWHERE_UNEXPECTED_ERROR; return KNOWHERE_UNEXPECTED_ERROR;
} catch (std::exception &e) { } catch (std::exception &e) {
WRAPPER_LOG_ERROR << e.what(); WRAPPER_LOG_ERROR << e.what();
return server::KNOWHERE_ERROR; return KNOWHERE_ERROR;
} }
return server::KNOWHERE_SUCCESS; return KNOWHERE_SUCCESS;
} }

34
cpp/src/wrapper/knowhere/vec_index.h
View File

@ -42,23 +42,23 @@ using VecIndexPtr = std::shared_ptr<VecIndex>;
class VecIndex { class VecIndex {
public: public:
virtual server::KnowhereError BuildAll(const long &nb, virtual ErrorCode BuildAll(const long &nb,
const float *xb, const float *xb,
const long *ids, const long *ids,
const Config &cfg, const Config &cfg,
const long &nt = 0, const long &nt = 0,
const float *xt = nullptr) = 0; const float *xt = nullptr) = 0;
virtual server::KnowhereError Add(const long &nb, virtual ErrorCode Add(const long &nb,
const float *xb, const float *xb,
const long *ids, const long *ids,
const Config &cfg = Config()) = 0; const Config &cfg = Config()) = 0;
virtual server::KnowhereError Search(const long &nq, virtual ErrorCode Search(const long &nq,
const float *xq, const float *xq,
float *dist, float *dist,
long *ids, long *ids,
const Config &cfg = Config()) = 0; const Config &cfg = Config()) = 0;
virtual VecIndexPtr CopyToGpu(const int64_t &device_id, virtual VecIndexPtr CopyToGpu(const int64_t &device_id,
const Config &cfg = Config()) = 0; const Config &cfg = Config()) = 0;
@ -77,10 +77,10 @@ class VecIndex {
virtual zilliz::knowhere::BinarySet Serialize() = 0; virtual zilliz::knowhere::BinarySet Serialize() = 0;
virtual server::KnowhereError Load(const zilliz::knowhere::BinarySet &index_binary) = 0; virtual ErrorCode Load(const zilliz::knowhere::BinarySet &index_binary) = 0;
}; };
extern server::KnowhereError write_index(VecIndexPtr index, const std::string &location); extern ErrorCode write_index(VecIndexPtr index, const std::string &location);
extern VecIndexPtr read_index(const std::string &location); extern VecIndexPtr read_index(const std::string &location);

4
cpp/unittest/CMakeLists.txt
View File

@ -46,5 +46,5 @@ add_subdirectory(server)
add_subdirectory(db) add_subdirectory(db)
add_subdirectory(knowhere) add_subdirectory(knowhere)
add_subdirectory(metrics) add_subdirectory(metrics)
add_subdirectory(scheduler) #add_subdirectory(scheduler)
add_subdirectory(storage) #add_subdirectory(storage)

3
cpp/unittest/db/meta_tests.cpp
View File

@ -15,6 +15,7 @@
#include "db/Utils.h" #include "db/Utils.h"
#include "db/meta/MetaConsts.h" #include "db/meta/MetaConsts.h"
using namespace zilliz::milvus;
using namespace zilliz::milvus::engine; using namespace zilliz::milvus::engine;
TEST_F(MetaTest, TABLE_TEST) { TEST_F(MetaTest, TABLE_TEST) {
@ -38,7 +39,7 @@ TEST_F(MetaTest, TABLE_TEST) {
table.table_id_ = table_id; table.table_id_ = table_id;
status = impl_->CreateTable(table); status = impl_->CreateTable(table);
ASSERT_TRUE(status.IsAlreadyExist()); ASSERT_EQ(status.code(), DB_ALREADY_EXIST);
table.table_id_ = ""; table.table_id_ = "";
status = impl_->CreateTable(table); status = impl_->CreateTable(table);

19
cpp/unittest/db/misc_test.cpp
View File

@ -41,24 +41,29 @@ TEST(DBMiscTest, STATUS_TEST) {
std::string str = status.ToString(); std::string str = status.ToString();
ASSERT_FALSE(str.empty()); ASSERT_FALSE(str.empty());
status = engine::Status::Error("wrong", "mistake"); status = engine::Status(DB_ERROR, "mistake");
ASSERT_TRUE(status.IsError()); ASSERT_EQ(status.code(), DB_ERROR);
str = status.ToString(); str = status.ToString();
ASSERT_FALSE(str.empty()); ASSERT_FALSE(str.empty());
status = engine::Status::NotFound("wrong", "mistake"); status = engine::Status(DB_NOT_FOUND, "mistake");
ASSERT_TRUE(status.IsNotFound()); ASSERT_EQ(status.code(), DB_NOT_FOUND);
str = status.ToString(); str = status.ToString();
ASSERT_FALSE(str.empty()); ASSERT_FALSE(str.empty());
status = engine::Status::DBTransactionError("wrong", "mistake"); status = engine::Status(DB_ALREADY_EXIST, "mistake");
ASSERT_TRUE(status.IsDBTransactionError()); ASSERT_EQ(status.code(), DB_ALREADY_EXIST);
str = status.ToString();
ASSERT_FALSE(str.empty());
status = engine::Status(DB_META_TRANSACTION_FAILED, "mistake");
ASSERT_EQ(status.code(), DB_META_TRANSACTION_FAILED);
str = status.ToString(); str = status.ToString();
ASSERT_FALSE(str.empty()); ASSERT_FALSE(str.empty());
engine::Status status_copy = engine::Status::OK(); engine::Status status_copy = engine::Status::OK();
CopyStatus(status_copy, status); CopyStatus(status_copy, status);
ASSERT_TRUE(status.IsDBTransactionError()); ASSERT_EQ(status.code(), DB_META_TRANSACTION_FAILED);
} }
TEST(DBMiscTest, OPTIONS_TEST) { TEST(DBMiscTest, OPTIONS_TEST) {

3
cpp/unittest/db/mysql_meta_test.cpp
View File

@ -19,6 +19,7 @@
#include <iostream> #include <iostream>
using namespace zilliz::milvus;
using namespace zilliz::milvus::engine; using namespace zilliz::milvus::engine;
TEST_F(MySqlMetaTest, TABLE_TEST) { TEST_F(MySqlMetaTest, TABLE_TEST) {
@ -42,7 +43,7 @@ TEST_F(MySqlMetaTest, TABLE_TEST) {
table.table_id_ = table_id; table.table_id_ = table_id;
status = impl_->CreateTable(table); status = impl_->CreateTable(table);
ASSERT_TRUE(status.IsAlreadyExist()); ASSERT_EQ(status.code(), DB_ALREADY_EXIST);
table.table_id_ = ""; table.table_id_ = "";
status = impl_->CreateTable(table); status = impl_->CreateTable(table);

2
cpp/unittest/metrics/metrics_test.cpp
View File

@ -127,7 +127,7 @@ TEST_F(MetricTest, Metric_Tes) {
TEST_F(MetricTest, Collector_Metrics_Test){ TEST_F(MetricTest, Collector_Metrics_Test){
engine::Status status = engine::Status::OK(); engine::Status status = engine::Status::OK();
server::CollectInsertMetrics insert_metrics0(0, status); server::CollectInsertMetrics insert_metrics0(0, status);
status = engine::Status::Error("error"); status = engine::Status(DB_ERROR, "error");
server::CollectInsertMetrics insert_metrics1(0, status); server::CollectInsertMetrics insert_metrics1(0, status);
server::CollectQueryMetrics query_metrics(10); server::CollectQueryMetrics query_metrics(10);

32
cpp/unittest/server/cache_test.cpp
View File

@ -30,12 +30,12 @@ public:
class MockVecIndex : public engine::VecIndex { class MockVecIndex : public engine::VecIndex {
public: public:
virtual server::KnowhereError BuildAll(const long &nb, virtual ErrorCode BuildAll(const long &nb,
const float *xb, const float *xb,
const long *ids, const long *ids,
const engine::Config &cfg, const engine::Config &cfg,
const long &nt = 0, const long &nt = 0,
const float *xt = nullptr) { const float *xt = nullptr) {
} }
@ -51,18 +51,18 @@ public:
return engine::IndexType::INVALID; return engine::IndexType::INVALID;
} }
virtual server::KnowhereError Add(const long &nb, virtual ErrorCode Add(const long &nb,
const float *xb, const float *xb,
const long *ids, const long *ids,
const engine::Config &cfg = engine::Config()) { const engine::Config &cfg = engine::Config()) {
} }
virtual server::KnowhereError Search(const long &nq, virtual ErrorCode Search(const long &nq,
const float *xq, const float *xq,
float *dist, float *dist,
long *ids, long *ids,
const engine::Config &cfg = engine::Config()) { const engine::Config &cfg = engine::Config()) {
} }
@ -87,7 +87,7 @@ public:
return binset; return binset;
} }
virtual server::KnowhereError Load(const zilliz::knowhere::BinarySet &index_binary) { virtual ErrorCode Load(const zilliz::knowhere::BinarySet &index_binary) {
} }

24
cpp/unittest/server/config_test.cpp
View File

@ -27,14 +27,14 @@ static constexpr uint64_t GB = MB*1024;
TEST(ConfigTest, CONFIG_TEST) { TEST(ConfigTest, CONFIG_TEST) {
server::ConfigMgr* config_mgr = server::ConfigMgr::GetInstance(); server::ConfigMgr* config_mgr = server::ConfigMgr::GetInstance();
server::ServerError err = config_mgr->LoadConfigFile(""); ErrorCode err = config_mgr->LoadConfigFile("");
ASSERT_EQ(err, server::SERVER_UNEXPECTED_ERROR); ASSERT_EQ(err, SERVER_UNEXPECTED_ERROR);
err = config_mgr->LoadConfigFile(LOG_FILE_PATH); err = config_mgr->LoadConfigFile(LOG_FILE_PATH);
ASSERT_EQ(err, server::SERVER_UNEXPECTED_ERROR); ASSERT_EQ(err, SERVER_UNEXPECTED_ERROR);
err = config_mgr->LoadConfigFile(CONFIG_FILE_PATH); err = config_mgr->LoadConfigFile(CONFIG_FILE_PATH);
ASSERT_EQ(err, server::SERVER_SUCCESS); ASSERT_EQ(err, SERVER_SUCCESS);
config_mgr->Print(); config_mgr->Print();
@ -95,11 +95,11 @@ TEST(ConfigTest, CONFIG_TEST) {
TEST(ConfigTest, SERVER_CONFIG_TEST) { TEST(ConfigTest, SERVER_CONFIG_TEST) {
server::ServerConfig& config = server::ServerConfig::GetInstance(); server::ServerConfig& config = server::ServerConfig::GetInstance();
server::ServerError err = config.LoadConfigFile(CONFIG_FILE_PATH); ErrorCode err = config.LoadConfigFile(CONFIG_FILE_PATH);
ASSERT_EQ(err, server::SERVER_SUCCESS); ASSERT_EQ(err, SERVER_SUCCESS);
err = server::ServerConfig::GetInstance().ValidateConfig(); err = server::ServerConfig::GetInstance().ValidateConfig();
ASSERT_EQ(err, server::SERVER_SUCCESS); ASSERT_EQ(err, SERVER_SUCCESS);
server::ConfigNode node1 = config.GetConfig("server_config"); server::ConfigNode node1 = config.GetConfig("server_config");
server::ConfigNode& node2 = config.GetConfig("cache_config"); server::ConfigNode& node2 = config.GetConfig("cache_config");
@ -125,26 +125,26 @@ TEST(ConfigTest, SERVER_CONFIG_TEST) {
server::ConfigNode& cache_config = config.GetConfig(server::CONFIG_CACHE); server::ConfigNode& cache_config = config.GetConfig(server::CONFIG_CACHE);
cache_config.SetValue(server::CACHE_FREE_PERCENT, "2.0"); cache_config.SetValue(server::CACHE_FREE_PERCENT, "2.0");
err = config.ValidateConfig(); err = config.ValidateConfig();
ASSERT_NE(err, server::SERVER_SUCCESS); ASSERT_NE(err, SERVER_SUCCESS);
size_t cache_cap = 16; size_t cache_cap = 16;
size_t insert_buffer_size = (total_mem - cache_cap*GB + 1*GB)/GB; size_t insert_buffer_size = (total_mem - cache_cap*GB + 1*GB)/GB;
db_config.SetValue(server::CONFIG_DB_INSERT_BUFFER_SIZE, std::to_string(insert_buffer_size)); db_config.SetValue(server::CONFIG_DB_INSERT_BUFFER_SIZE, std::to_string(insert_buffer_size));
cache_config.SetValue(server::CONFIG_CPU_CACHE_CAPACITY, std::to_string(cache_cap)); cache_config.SetValue(server::CONFIG_CPU_CACHE_CAPACITY, std::to_string(cache_cap));
err = config.ValidateConfig(); err = config.ValidateConfig();
ASSERT_NE(err, server::SERVER_SUCCESS); ASSERT_NE(err, SERVER_SUCCESS);
cache_cap = total_mem/GB + 2; cache_cap = total_mem/GB + 2;
cache_config.SetValue(server::CONFIG_CPU_CACHE_CAPACITY, std::to_string(cache_cap)); cache_config.SetValue(server::CONFIG_CPU_CACHE_CAPACITY, std::to_string(cache_cap));
err = config.ValidateConfig(); err = config.ValidateConfig();
ASSERT_NE(err, server::SERVER_SUCCESS); ASSERT_NE(err, SERVER_SUCCESS);
insert_buffer_size = total_mem/GB + 2; insert_buffer_size = total_mem/GB + 2;
db_config.SetValue(server::CONFIG_DB_INSERT_BUFFER_SIZE, std::to_string(insert_buffer_size)); db_config.SetValue(server::CONFIG_DB_INSERT_BUFFER_SIZE, std::to_string(insert_buffer_size));
err = config.ValidateConfig(); err = config.ValidateConfig();
ASSERT_NE(err, server::SERVER_SUCCESS); ASSERT_NE(err, SERVER_SUCCESS);
server_config.SetValue(server::CONFIG_GPU_INDEX, "9999"); server_config.SetValue(server::CONFIG_GPU_INDEX, "9999");
err = config.ValidateConfig(); err = config.ValidateConfig();
ASSERT_NE(err, server::SERVER_SUCCESS); ASSERT_NE(err, SERVER_SUCCESS);
} }

92
cpp/unittest/server/util_test.cpp
View File

@ -25,8 +25,8 @@ static const char* LOG_FILE_PATH = "./milvus/conf/log_config.conf";
TEST(UtilTest, EXCEPTION_TEST) { TEST(UtilTest, EXCEPTION_TEST) {
std::string err_msg = "failed"; std::string err_msg = "failed";
server::ServerException ex(server::SERVER_UNEXPECTED_ERROR, err_msg); server::ServerException ex(SERVER_UNEXPECTED_ERROR, err_msg);
ASSERT_EQ(ex.error_code(), server::SERVER_UNEXPECTED_ERROR); ASSERT_EQ(ex.error_code(), SERVER_UNEXPECTED_ERROR);
std::string msg = ex.what(); std::string msg = ex.what();
ASSERT_EQ(msg, err_msg); ASSERT_EQ(msg, err_msg);
} }
@ -44,19 +44,19 @@ TEST(UtilTest, COMMON_TEST) {
std::string path1 = "/tmp/milvus_test/"; std::string path1 = "/tmp/milvus_test/";
std::string path2 = path1 + "common_test_12345/"; std::string path2 = path1 + "common_test_12345/";
std::string path3 = path2 + "abcdef"; std::string path3 = path2 + "abcdef";
server::ServerError err = server::CommonUtil::CreateDirectory(path3); ErrorCode err = server::CommonUtil::CreateDirectory(path3);
ASSERT_EQ(err, server::SERVER_SUCCESS); ASSERT_EQ(err, SERVER_SUCCESS);
//test again //test again
err = server::CommonUtil::CreateDirectory(path3); err = server::CommonUtil::CreateDirectory(path3);
ASSERT_EQ(err, server::SERVER_SUCCESS); ASSERT_EQ(err, SERVER_SUCCESS);
ASSERT_TRUE(server::CommonUtil::IsDirectoryExist(path3)); ASSERT_TRUE(server::CommonUtil::IsDirectoryExist(path3));
err = server::CommonUtil::DeleteDirectory(path1); err = server::CommonUtil::DeleteDirectory(path1);
ASSERT_EQ(err, server::SERVER_SUCCESS); ASSERT_EQ(err, SERVER_SUCCESS);
//test again //test again
err = server::CommonUtil::DeleteDirectory(path1); err = server::CommonUtil::DeleteDirectory(path1);
ASSERT_EQ(err, server::SERVER_SUCCESS); ASSERT_EQ(err, SERVER_SUCCESS);
ASSERT_FALSE(server::CommonUtil::IsDirectoryExist(path1)); ASSERT_FALSE(server::CommonUtil::IsDirectoryExist(path1));
ASSERT_FALSE(server::CommonUtil::IsFileExist(path1)); ASSERT_FALSE(server::CommonUtil::IsFileExist(path1));
@ -94,24 +94,24 @@ TEST(UtilTest, STRINGFUNCTIONS_TEST) {
str = "a,b,c"; str = "a,b,c";
std::vector<std::string> result; std::vector<std::string> result;
server::ServerError err = server::StringHelpFunctions::SplitStringByDelimeter(str , ",", result); ErrorCode err = server::StringHelpFunctions::SplitStringByDelimeter(str , ",", result);
ASSERT_EQ(err, server::SERVER_SUCCESS); ASSERT_EQ(err, SERVER_SUCCESS);
ASSERT_EQ(result.size(), 3UL); ASSERT_EQ(result.size(), 3UL);
result.clear(); result.clear();
err = server::StringHelpFunctions::SplitStringByQuote(str , ",", "\"", result); err = server::StringHelpFunctions::SplitStringByQuote(str , ",", "\"", result);
ASSERT_EQ(err, server::SERVER_SUCCESS); ASSERT_EQ(err, SERVER_SUCCESS);
ASSERT_EQ(result.size(), 3UL); ASSERT_EQ(result.size(), 3UL);
result.clear(); result.clear();
err = server::StringHelpFunctions::SplitStringByQuote(str , ",", "", result); err = server::StringHelpFunctions::SplitStringByQuote(str , ",", "", result);
ASSERT_EQ(err, server::SERVER_SUCCESS); ASSERT_EQ(err, SERVER_SUCCESS);
ASSERT_EQ(result.size(), 3UL); ASSERT_EQ(result.size(), 3UL);
str = "55,\"aa,gg,yy\",b"; str = "55,\"aa,gg,yy\",b";
result.clear(); result.clear();
err = server::StringHelpFunctions::SplitStringByQuote(str , ",", "\"", result); err = server::StringHelpFunctions::SplitStringByQuote(str , ",", "\"", result);
ASSERT_EQ(err, server::SERVER_SUCCESS); ASSERT_EQ(err, SERVER_SUCCESS);
ASSERT_EQ(result.size(), 3UL); ASSERT_EQ(result.size(), 3UL);
@ -159,86 +159,86 @@ TEST(UtilTest, LOG_TEST) {
TEST(UtilTest, VALIDATE_TABLENAME_TEST) { TEST(UtilTest, VALIDATE_TABLENAME_TEST) {
std::string table_name = "Normal123_"; std::string table_name = "Normal123_";
server:: ServerError res = server::ValidationUtil::ValidateTableName(table_name); ErrorCode res = server::ValidationUtil::ValidateTableName(table_name);
ASSERT_EQ(res, server::SERVER_SUCCESS); ASSERT_EQ(res, SERVER_SUCCESS);
table_name = "12sds"; table_name = "12sds";
res = server::ValidationUtil::ValidateTableName(table_name); res = server::ValidationUtil::ValidateTableName(table_name);
ASSERT_EQ(res, server::SERVER_INVALID_TABLE_NAME); ASSERT_EQ(res, SERVER_INVALID_TABLE_NAME);
table_name = ""; table_name = "";
res = server::ValidationUtil::ValidateTableName(table_name); res = server::ValidationUtil::ValidateTableName(table_name);
ASSERT_EQ(res, server::SERVER_INVALID_TABLE_NAME); ASSERT_EQ(res, SERVER_INVALID_TABLE_NAME);
table_name = "_asdasd"; table_name = "_asdasd";
res = server::ValidationUtil::ValidateTableName(table_name); res = server::ValidationUtil::ValidateTableName(table_name);
ASSERT_EQ(res, server::SERVER_SUCCESS); ASSERT_EQ(res, SERVER_SUCCESS);
table_name = "!@#!@"; table_name = "!@#!@";
res = server::ValidationUtil::ValidateTableName(table_name); res = server::ValidationUtil::ValidateTableName(table_name);
ASSERT_EQ(res, server::SERVER_INVALID_TABLE_NAME); ASSERT_EQ(res, SERVER_INVALID_TABLE_NAME);
table_name = "_!@#!@"; table_name = "_!@#!@";
res = server::ValidationUtil::ValidateTableName(table_name); res = server::ValidationUtil::ValidateTableName(table_name);
ASSERT_EQ(res, server::SERVER_INVALID_TABLE_NAME); ASSERT_EQ(res, SERVER_INVALID_TABLE_NAME);
table_name = "中文"; table_name = "中文";
res = server::ValidationUtil::ValidateTableName(table_name); res = server::ValidationUtil::ValidateTableName(table_name);
ASSERT_EQ(res, server::SERVER_INVALID_TABLE_NAME); ASSERT_EQ(res, SERVER_INVALID_TABLE_NAME);
table_name = std::string(10000, 'a'); table_name = std::string(10000, 'a');
res = server::ValidationUtil::ValidateTableName(table_name); res = server::ValidationUtil::ValidateTableName(table_name);
ASSERT_EQ(res, server::SERVER_INVALID_TABLE_NAME); ASSERT_EQ(res, SERVER_INVALID_TABLE_NAME);
} }
TEST(UtilTest, VALIDATE_DIMENSIONTEST) { TEST(UtilTest, VALIDATE_DIMENSIONTEST) {
ASSERT_EQ(server::ValidationUtil::ValidateTableDimension(-1), server::SERVER_INVALID_VECTOR_DIMENSION); ASSERT_EQ(server::ValidationUtil::ValidateTableDimension(-1), SERVER_INVALID_VECTOR_DIMENSION);
ASSERT_EQ(server::ValidationUtil::ValidateTableDimension(0), server::SERVER_INVALID_VECTOR_DIMENSION); ASSERT_EQ(server::ValidationUtil::ValidateTableDimension(0), SERVER_INVALID_VECTOR_DIMENSION);
ASSERT_EQ(server::ValidationUtil::ValidateTableDimension(16385), server::SERVER_INVALID_VECTOR_DIMENSION); ASSERT_EQ(server::ValidationUtil::ValidateTableDimension(16385), SERVER_INVALID_VECTOR_DIMENSION);
ASSERT_EQ(server::ValidationUtil::ValidateTableDimension(16384), server::SERVER_SUCCESS); ASSERT_EQ(server::ValidationUtil::ValidateTableDimension(16384), SERVER_SUCCESS);
ASSERT_EQ(server::ValidationUtil::ValidateTableDimension(1), server::SERVER_SUCCESS); ASSERT_EQ(server::ValidationUtil::ValidateTableDimension(1), SERVER_SUCCESS);
} }
TEST(UtilTest, VALIDATE_INDEX_TEST) { TEST(UtilTest, VALIDATE_INDEX_TEST) {
ASSERT_EQ(server::ValidationUtil::ValidateTableIndexType((int)engine::EngineType::INVALID), server::SERVER_INVALID_INDEX_TYPE); ASSERT_EQ(server::ValidationUtil::ValidateTableIndexType((int)engine::EngineType::INVALID), SERVER_INVALID_INDEX_TYPE);
for(int i = 1; i <= (int)engine::EngineType::MAX_VALUE; i++) { for(int i = 1; i <= (int)engine::EngineType::MAX_VALUE; i++) {
ASSERT_EQ(server::ValidationUtil::ValidateTableIndexType(i), server::SERVER_SUCCESS); ASSERT_EQ(server::ValidationUtil::ValidateTableIndexType(i), SERVER_SUCCESS);
} }
ASSERT_EQ(server::ValidationUtil::ValidateTableIndexType((int)engine::EngineType::MAX_VALUE + 1), server::SERVER_INVALID_INDEX_TYPE); ASSERT_EQ(server::ValidationUtil::ValidateTableIndexType((int)engine::EngineType::MAX_VALUE + 1), SERVER_INVALID_INDEX_TYPE);
ASSERT_EQ(server::ValidationUtil::ValidateTableIndexNlist(0), server::SERVER_INVALID_INDEX_NLIST); ASSERT_EQ(server::ValidationUtil::ValidateTableIndexNlist(0), SERVER_INVALID_INDEX_NLIST);
ASSERT_EQ(server::ValidationUtil::ValidateTableIndexNlist(100), server::SERVER_SUCCESS); ASSERT_EQ(server::ValidationUtil::ValidateTableIndexNlist(100), SERVER_SUCCESS);
ASSERT_EQ(server::ValidationUtil::ValidateTableIndexFileSize(0), server::SERVER_INVALID_INDEX_FILE_SIZE); ASSERT_EQ(server::ValidationUtil::ValidateTableIndexFileSize(0), SERVER_INVALID_INDEX_FILE_SIZE);
ASSERT_EQ(server::ValidationUtil::ValidateTableIndexFileSize(100), server::SERVER_SUCCESS); ASSERT_EQ(server::ValidationUtil::ValidateTableIndexFileSize(100), SERVER_SUCCESS);
ASSERT_EQ(server::ValidationUtil::ValidateTableIndexMetricType(0), server::SERVER_INVALID_INDEX_METRIC_TYPE); ASSERT_EQ(server::ValidationUtil::ValidateTableIndexMetricType(0), SERVER_INVALID_INDEX_METRIC_TYPE);
ASSERT_EQ(server::ValidationUtil::ValidateTableIndexMetricType(1), server::SERVER_SUCCESS); ASSERT_EQ(server::ValidationUtil::ValidateTableIndexMetricType(1), SERVER_SUCCESS);
ASSERT_EQ(server::ValidationUtil::ValidateTableIndexMetricType(2), server::SERVER_SUCCESS); ASSERT_EQ(server::ValidationUtil::ValidateTableIndexMetricType(2), SERVER_SUCCESS);
} }
TEST(ValidationUtilTest, ValidateTopkTest) { TEST(ValidationUtilTest, ValidateTopkTest) {
engine::meta::TableSchema schema; engine::meta::TableSchema schema;
ASSERT_EQ(server::ValidationUtil::ValidateSearchTopk(10, schema), server::SERVER_SUCCESS); ASSERT_EQ(server::ValidationUtil::ValidateSearchTopk(10, schema), SERVER_SUCCESS);
ASSERT_NE(server::ValidationUtil::ValidateSearchTopk(65536, schema), server::SERVER_SUCCESS); ASSERT_NE(server::ValidationUtil::ValidateSearchTopk(65536, schema), SERVER_SUCCESS);
ASSERT_NE(server::ValidationUtil::ValidateSearchTopk(0, schema), server::SERVER_SUCCESS); ASSERT_NE(server::ValidationUtil::ValidateSearchTopk(0, schema), SERVER_SUCCESS);
} }
TEST(ValidationUtilTest, ValidateNprobeTest) { TEST(ValidationUtilTest, ValidateNprobeTest) {
engine::meta::TableSchema schema; engine::meta::TableSchema schema;
schema.nlist_ = 100; schema.nlist_ = 100;
ASSERT_EQ(server::ValidationUtil::ValidateSearchNprobe(10, schema), server::SERVER_SUCCESS); ASSERT_EQ(server::ValidationUtil::ValidateSearchNprobe(10, schema), SERVER_SUCCESS);
ASSERT_NE(server::ValidationUtil::ValidateSearchNprobe(0, schema), server::SERVER_SUCCESS); ASSERT_NE(server::ValidationUtil::ValidateSearchNprobe(0, schema), SERVER_SUCCESS);
ASSERT_NE(server::ValidationUtil::ValidateSearchNprobe(101, schema), server::SERVER_SUCCESS); ASSERT_NE(server::ValidationUtil::ValidateSearchNprobe(101, schema), SERVER_SUCCESS);
} }
TEST(ValidationUtilTest, ValidateGpuTest) { TEST(ValidationUtilTest, ValidateGpuTest) {
ASSERT_EQ(server::ValidationUtil::ValidateGpuIndex(0), server::SERVER_SUCCESS); ASSERT_EQ(server::ValidationUtil::ValidateGpuIndex(0), SERVER_SUCCESS);
ASSERT_NE(server::ValidationUtil::ValidateGpuIndex(100), server::SERVER_SUCCESS); ASSERT_NE(server::ValidationUtil::ValidateGpuIndex(100), SERVER_SUCCESS);
size_t memory = 0; size_t memory = 0;
ASSERT_EQ(server::ValidationUtil::GetGpuMemory(0, memory), server::SERVER_SUCCESS); ASSERT_EQ(server::ValidationUtil::GetGpuMemory(0, memory), SERVER_SUCCESS);
ASSERT_NE(server::ValidationUtil::GetGpuMemory(100, memory), server::SERVER_SUCCESS); ASSERT_NE(server::ValidationUtil::GetGpuMemory(100, memory), SERVER_SUCCESS);
} }
TEST(UtilTest, TIMERECORDER_TEST) { TEST(UtilTest, TIMERECORDER_TEST) {