milvus-io
/
milvus
mirror of https://github.com/milvus-io/milvus.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Remove redundant files under internal/core (#10658) 

Signed-off-by: yudong.cai <yudong.cai@zilliz.com>
pull/10672/head
Cai Yudong 2021-10-26 17:26:30 +08:00 committed by GitHub
parent 5b42a3223c
commit fa01a888a0
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
41 changed files with 14 additions and 3848 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

28
internal/core/src/index/knowhere/knowhere/archive/KnowhereConfig.cpp
View File

@ -9,27 +9,21 @@
// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
// or implied. See the License for the specific language governing permissions and limitations under the License.
#include <string>
#include "KnowhereConfig.h"
#include "NGT/lib/NGT/defines.h"
#include "faiss/Clustering.h"
#include "faiss/FaissHook.h"
#include "faiss/common.h"
#include "faiss/utils/distances.h"
#include "faiss/utils/utils.h"
#include "knowhere/common/Exception.h"
#include "knowhere/common/Log.h"
#include "knowhere/index/vector_index/IndexHNSW.h"
#ifdef MILVUS_GPU_VERSION
#include "knowhere/index/vector_index/helpers/FaissGpuResourceMgr.h"
#endif
#include <faiss/Clustering.h>
#include <faiss/utils/distances.h>
#include "NGT/lib/NGT/defines.h"
#include "faiss/FaissHook.h"
#include "faiss/common.h"
#include "faiss/utils/utils.h"
#include "knowhere/common/Log.h"
#include "knowhere/index/IndexType.h"
#include "knowhere/index/vector_index/IndexHNSW.h"
#include "knowhere/index/vector_index/helpers/FaissIO.h"
#include "utils/ConfigUtils.h"
#include "utils/Error.h"
#include "knowhere/common/Exception.h"
#include <string>
#include <vector>
namespace milvus {
namespace engine {

95
internal/core/src/index/knowhere/utils/BlockingQueue.h
View File

@ -1,95 +0,0 @@
// Copyright (C) 2019-2020 Zilliz. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software distributed under the License
// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
// or implied. See the License for the specific language governing permissions and limitations under the License.
#pragma once
#include <assert.h>
#include <condition_variable>
#include <iostream>
#include <queue>
#include <vector>
namespace milvus {
template <typename T>
class BlockingQueue {
public:
BlockingQueue() : mtx(), full_(), empty_() {
}
virtual ~BlockingQueue() {
}
BlockingQueue(const BlockingQueue& rhs) = delete;
BlockingQueue&
operator=(const BlockingQueue& rhs) = delete;
void
Put(const T& task) {
std::unique_lock<std::mutex> lock(mtx);
full_.wait(lock, [this] { return (queue_.size() < capacity_); });
queue_.push(task);
empty_.notify_all();
}
T
Take() {
std::unique_lock<std::mutex> lock(mtx);
empty_.wait(lock, [this] { return !queue_.empty(); });
T front(queue_.front());
queue_.pop();
full_.notify_all();
return front;
}
T
Front() {
std::unique_lock<std::mutex> lock(mtx);
empty_.wait(lock, [this] { return !queue_.empty(); });
T front(queue_.front());
return front;
}
T
Back() {
std::unique_lock<std::mutex> lock(mtx);
empty_.wait(lock, [this] { return !queue_.empty(); });
T back(queue_.back());
return back;
}
size_t
Size() const {
std::lock_guard<std::mutex> lock(mtx);
return queue_.size();
}
bool
Empty() const {
std::unique_lock<std::mutex> lock(mtx);
return queue_.empty();
}
void
SetCapacity(const size_t capacity) {
capacity_ = (capacity > 0 ? capacity : capacity_);
}
protected:
mutable std::mutex mtx;
std::condition_variable full_;
std::condition_variable empty_;
std::queue<T> queue_;
size_t capacity_ = 32;
};
} // namespace milvus

243
internal/core/src/index/knowhere/utils/CommonUtil.cpp
View File

@ -1,243 +0,0 @@
// Copyright (C) 2019-2020 Zilliz. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software distributed under the License
// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
// or implied. See the License for the specific language governing permissions and limitations under the License.
#include "utils/CommonUtil.h"
#include "utils/Log.h"
#include <dirent.h>
#include <fiu/fiu-local.h>
#include <pwd.h>
#include <sys/stat.h>
#include <unistd.h>
#include <boost/filesystem.hpp>
#include <iostream>
#include <vector>
namespace milvus {
namespace fs = boost::filesystem;
bool
CommonUtil::IsDirectoryExist(const std::string& path) {
DIR* dp = nullptr;
if ((dp = opendir(path.c_str())) == nullptr) {
return false;
}
closedir(dp);
return true;
}
Status
CommonUtil::CreateDirectory(const std::string& path) {
if (path.empty()) {
return Status::OK();
}
struct stat directory_stat;
int status = stat(path.c_str(), &directory_stat);
if (status == 0) {
return Status::OK(); // already exist
}
fs::path fs_path(path);
fs::path parent_path = fs_path.parent_path();
Status err_status = CreateDirectory(parent_path.string());
fiu_do_on("CommonUtil.CreateDirectory.create_parent_fail", err_status = Status(SERVER_INVALID_ARGUMENT, ""));
if (!err_status.ok()) {
return err_status;
}
status = stat(path.c_str(), &directory_stat);
if (status == 0) {
return Status::OK(); // already exist
}
int makeOK = mkdir(path.c_str(), S_IRWXU | S_IRGRP | S_IROTH);
fiu_do_on("CommonUtil.CreateDirectory.create_dir_fail", makeOK = 1);
if (makeOK != 0) {
return Status(SERVER_UNEXPECTED_ERROR, "failed to create directory: " + path);
}
return Status::OK();
}
namespace {
void
RemoveDirectory(const std::string& path) {
DIR* dir = nullptr;
const int32_t buf_size = 256;
char file_name[buf_size];
std::string folder_name = path + "/%s";
if ((dir = opendir(path.c_str())) != nullptr) {
struct dirent* dmsg;
while ((dmsg = readdir(dir)) != nullptr) {
if (strcmp(dmsg->d_name, ".") != 0 && strcmp(dmsg->d_name, "..") != 0) {
snprintf(file_name, buf_size, folder_name.c_str(), dmsg->d_name);
std::string tmp = file_name;
if (tmp.find('.') == std::string::npos) {
RemoveDirectory(file_name);
}
remove(file_name);
}
}
}
if (dir != nullptr) {
closedir(dir);
}
remove(path.c_str());
}
} // namespace
Status
CommonUtil::DeleteDirectory(const std::string& path) {
if (path.empty()) {
return Status::OK();
}
struct stat directory_stat;
int statOK = stat(path.c_str(), &directory_stat);
if (statOK != 0) {
return Status::OK();
}
RemoveDirectory(path);
return Status::OK();
}
bool
CommonUtil::IsFileExist(const std::string& path) {
return (access(path.c_str(), F_OK) == 0);
}
uint64_t
CommonUtil::GetFileSize(const std::string& path) {
struct stat file_info;
if (stat(path.c_str(), &file_info) < 0) {
return 0;
}
return static_cast<uint64_t>(file_info.st_size);
}
std::string
CommonUtil::GetFileName(std::string filename) {
int pos = filename.find_last_of('/');
return filename.substr(pos + 1);
}
std::string
CommonUtil::GetExePath() {
const int64_t buf_len = 1024;
char buf[buf_len];
int64_t cnt = readlink("/proc/self/exe", buf, buf_len);
fiu_do_on("CommonUtil.GetExePath.readlink_fail", cnt = -1);
if (cnt < 0 || cnt >= buf_len) {
return "";
}
buf[cnt] = '\0';
std::string exe_path = buf;
fiu_do_on("CommonUtil.GetExePath.exe_path_error", exe_path = "/");
if (exe_path.rfind('/') != exe_path.length() - 1) {
std::string sub_str = exe_path.substr(0, exe_path.rfind('/'));
return sub_str + "/";
}
return exe_path;
}
bool
CommonUtil::TimeStrToTime(const std::string& time_str,
time_t& time_integer,
tm& time_struct,
const std::string& format) {
time_integer = 0;
memset(&time_struct, 0, sizeof(tm));
int ret = sscanf(time_str.c_str(), format.c_str(), &(time_struct.tm_year), &(time_struct.tm_mon),
&(time_struct.tm_mday), &(time_struct.tm_hour), &(time_struct.tm_min), &(time_struct.tm_sec));
if (ret <= 0) {
return false;
}
time_struct.tm_year -= 1900;
time_struct.tm_mon--;
time_integer = mktime(&time_struct);
return true;
}
void
CommonUtil::GetCurrentTimeStr(std::string& time_str) {
auto t = std::time(nullptr);
struct tm ltm;
localtime_r(&t, &ltm);
time_str = "";
time_str += std::to_string(ltm.tm_year + 1900);
time_str += "-";
time_str += std::to_string(ltm.tm_mon + 1);
time_str += "-";
time_str += std::to_string(ltm.tm_mday);
time_str += "_";
time_str += std::to_string(ltm.tm_hour);
time_str += ":";
time_str += std::to_string(ltm.tm_min);
time_str += ":";
time_str += std::to_string(ltm.tm_sec);
}
void
CommonUtil::ConvertTime(time_t time_integer, tm& time_struct) {
localtime_r(&time_integer, &time_struct);
}
void
CommonUtil::ConvertTime(tm time_struct, time_t& time_integer) {
time_integer = mktime(&time_struct);
}
std::string
CommonUtil::ConvertSize(int64_t size) {
const int64_t gb = 1024ll * 1024 * 1024;
const int64_t mb = 1024ll * 1024;
const int64_t kb = 1024ll;
if (size % gb == 0) {
return std::to_string(size / gb) + "GB";
} else if (size % mb == 0) {
return std::to_string(size / mb) + "MB";
} else if (size % kb == 0) {
return std::to_string(size / kb) + "KB";
} else {
return std::to_string(size);
}
}
#ifdef ENABLE_CPU_PROFILING
std::string
CommonUtil::GetCurrentTimeStr() {
time_t tt;
time(&tt);
tt = tt + 8 * 60;
tm t;
gmtime_r(&tt, &t);
std::string str = std::to_string(t.tm_year + 1900) + "_" + std::to_string(t.tm_mon + 1) + "_" +
std::to_string(t.tm_mday) + "_" + std::to_string(t.tm_hour) + "_" + std::to_string(t.tm_min) +
"_" + std::to_string(t.tm_sec);
return str;
}
#endif
} // namespace milvus

62
internal/core/src/index/knowhere/utils/CommonUtil.h
View File

@ -1,62 +0,0 @@
// Copyright (C) 2019-2020 Zilliz. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software distributed under the License
// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
// or implied. See the License for the specific language governing permissions and limitations under the License.
#pragma once
#include "utils/Status.h"
#include <time.h>
#include <string>
namespace milvus {
class CommonUtil {
public:
static bool
IsFileExist(const std::string& path);
static uint64_t
GetFileSize(const std::string& path);
static bool
IsDirectoryExist(const std::string& path);
static Status
CreateDirectory(const std::string& path);
static Status
DeleteDirectory(const std::string& path);
static std::string
GetFileName(std::string filename);
static std::string
GetExePath();
static bool
TimeStrToTime(const std::string& time_str,
time_t& time_integer,
tm& time_struct,
const std::string& format = "%d-%d-%d %d:%d:%d");
static void
GetCurrentTimeStr(std::string& time_str);
static void
ConvertTime(time_t time_integer, tm& time_struct);
static void
ConvertTime(tm time_struct, time_t& time_integer);
static std::string
ConvertSize(int64_t size);
#ifdef ENABLE_CPU_PROFILING
static std::string
GetCurrentTimeStr();
#endif
};
} // namespace milvus

320
internal/core/src/index/knowhere/utils/ConfigUtils.cpp
View File

@ -1,320 +0,0 @@
// Copyright (C) 2019-2020 Zilliz. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software distributed under the License
// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
// or implied. See the License for the specific language governing permissions and limitations under the License.
#include "utils/ConfigUtils.h"
#include "utils/Log.h"
#include "utils/StringHelpFunctions.h"
#include <arpa/inet.h>
#include <algorithm>
#include <cmath>
#ifdef MILVUS_GPU_VERSION
#include <cuda_runtime.h>
#endif
#include <fiu/fiu-local.h>
#include <sys/stat.h>
#include <sys/sysinfo.h>
#include <unistd.h>
#include <limits>
#include <regex>
#include <set>
#include <unordered_map>
#if defined(__x86_64__)
#define THREAD_MULTIPLY_CPU 1
#elif defined(__powerpc64__)
#define THREAD_MULTIPLY_CPU 4
#else
#define THREAD_MULTIPLY_CPU 1
#endif
namespace milvus {
namespace server {
std::unordered_map<std::string, int64_t> BYTE_UNITS = {
{"b", 1},
{"k", 1024},
{"m", 1024 * 1024},
{"g", 1024 * 1024 * 1024},
};
bool
is_number(const std::string& s) {
return !s.empty() && std::find_if(s.begin(), s.end(), [](unsigned char c) { return !std::isdigit(c); }) == s.end();
}
bool
is_alpha(const std::string& s) {
return !s.empty() && std::find_if(s.begin(), s.end(), [](unsigned char c) { return !std::isalpha(c); }) == s.end();
}
std::string
str_tolower(std::string s) {
std::transform(s.begin(), s.end(), s.begin(), [](unsigned char c) { return std::tolower(c); });
return s;
}
int64_t
parse_bytes(const std::string& str, std::string& err) {
try {
std::string s = str;
if (is_number(s)) {
return std::stoll(s);
}
if (s.length() == 0) {
return 0;
}
auto last_two = s.substr(s.length() - 2, 2);
auto last_one = s.substr(s.length() - 1);
if (is_alpha(last_two) && is_alpha(last_one)) {
if (last_one == "b" or last_one == "B") {
s = s.substr(0, s.length() - 1);
}
}
auto& units = BYTE_UNITS;
auto suffix = str_tolower(s.substr(s.length() - 1));
std::string digits_part;
if (is_number(suffix)) {
digits_part = s;
suffix = 'b';
} else {
digits_part = s.substr(0, s.length() - 1);
}
if (units.find(suffix) != units.end() or is_number(suffix)) {
auto digits = std::stoll(digits_part);
return digits * units[suffix];
} else {
std::stringstream ss;
ss << "The specified value for memory (" << str << ") should specify the units."
<< "The postfix should be one of the `b` `k` `m` `g` characters";
err = ss.str();
}
} catch (...) {
err = "Unknown error happened on parse bytes.";
}
return 0;
}
bool
GetSystemMemInfo(int64_t& total_mem, int64_t& free_mem) {
struct sysinfo info;
int ret = sysinfo(&info);
total_mem = info.totalram;
free_mem = info.freeram;
return ret == 0; // succeed 0, failed -1
}
bool
GetSystemAvailableThreads(int64_t& thread_count) {
// threadCnt = std::thread::hardware_concurrency();
thread_count = sysconf(_SC_NPROCESSORS_CONF);
thread_count *= THREAD_MULTIPLY_CPU;
fiu_do_on("GetSystemAvailableThreads.zero_thread", thread_count = 0);
if (thread_count == 0) {
thread_count = 8;
}
return true;
}
Status
ValidateGpuIndex(int32_t gpu_index) {
#ifdef MILVUS_GPU_VERSION
int num_devices = 0;
auto cuda_err = cudaGetDeviceCount(&num_devices);
fiu_do_on("config.ValidateGpuIndex.get_device_count_fail", cuda_err = cudaError::cudaErrorUnknown);
if (cuda_err != cudaSuccess) {
std::string msg = "Failed to get gpu card number, cuda error:" + std::to_string(cuda_err);
LOG_SERVER_ERROR_ << msg;
return Status(SERVER_UNEXPECTED_ERROR, msg);
}
if (gpu_index >= num_devices) {
std::string msg = "Invalid gpu index: " + std::to_string(gpu_index);
LOG_SERVER_ERROR_ << msg;
return Status(SERVER_INVALID_ARGUMENT, msg);
}
#endif
return Status::OK();
}
#ifdef MILVUS_GPU_VERSION
Status
GetGpuMemory(int32_t gpu_index, int64_t& memory) {
fiu_return_on("config.GetGpuMemory.return_error", Status(SERVER_UNEXPECTED_ERROR, ""));
cudaDeviceProp deviceProp;
auto cuda_err = cudaGetDeviceProperties(&deviceProp, gpu_index);
if (cuda_err) {
std::string msg = "Failed to get gpu properties for gpu" + std::to_string(gpu_index) +
" , cuda error:" + std::to_string(cuda_err);
LOG_SERVER_ERROR_ << msg;
return Status(SERVER_UNEXPECTED_ERROR, msg);
}
memory = deviceProp.totalGlobalMem;
return Status::OK();
}
#endif
Status
ValidateIpAddress(const std::string& ip_address) {
struct in_addr address;
int result = inet_pton(AF_INET, ip_address.c_str(), &address);
fiu_do_on("config.ValidateIpAddress.error_ip_result", result = 2);
switch (result) {
case 1:
return Status::OK();
case 0: {
std::string msg = "Invalid IP address: " + ip_address;
LOG_SERVER_ERROR_ << msg;
return Status(SERVER_INVALID_ARGUMENT, msg);
}
default: {
std::string msg = "IP address conversion error: " + ip_address;
LOG_SERVER_ERROR_ << msg;
return Status(SERVER_UNEXPECTED_ERROR, msg);
}
}
}
Status
ValidateStringIsNumber(const std::string& str) {
if (str.empty() || !std::all_of(str.begin(), str.end(), ::isdigit)) {
return Status(SERVER_INVALID_ARGUMENT, "Invalid number");
}
try {
int64_t value = std::stol(str);
fiu_do_on("config.ValidateStringIsNumber.throw_exception", throw std::exception());
if (value < 0) {
return Status(SERVER_INVALID_ARGUMENT, "Negative number");
}
} catch (...) {
return Status(SERVER_INVALID_ARGUMENT, "Invalid number");
}
return Status::OK();
}
Status
ValidateStringIsBool(const std::string& str) {
fiu_return_on("ValidateStringNotBool", Status(SERVER_INVALID_ARGUMENT, "Invalid boolean: " + str));
std::string s = str;
std::transform(s.begin(), s.end(), s.begin(), ::tolower);
if (s == "true" || s == "on" || s == "yes" || s == "1" || s == "false" || s == "off" || s == "no" || s == "0" ||
s.empty()) {
return Status::OK();
}
return Status(SERVER_INVALID_ARGUMENT, "Invalid boolean: " + str);
}
Status
ValidateStringIsFloat(const std::string& str) {
try {
float val = std::stof(str);
if (val < 0.0) {
return Status(SERVER_INVALID_ARGUMENT, "Negative float: " + str);
}
} catch (...) {
return Status(SERVER_INVALID_ARGUMENT, "Invalid float: " + str);
}
return Status::OK();
}
Status
ValidateDbURI(const std::string& uri) {
std::string dialectRegex = "(.*)";
std::string usernameRegex = "(.*)";
std::string passwordRegex = "(.*)";
std::string hostRegex = "(.*)";
std::string portRegex = "(.*)";
std::string dbNameRegex = "(.*)";
std::string uriRegexStr = dialectRegex + R"(\:\/\/)" + usernameRegex + R"(\:)" + passwordRegex + R"(\@)" +
hostRegex + R"(\:)" + portRegex + R"(\/)" + dbNameRegex;
std::regex uriRegex(uriRegexStr);
std::smatch pieces_match;
bool okay = true;
if (std::regex_match(uri, pieces_match, uriRegex)) {
std::string dialect = pieces_match[1].str();
std::transform(dialect.begin(), dialect.end(), dialect.begin(), ::tolower);
if (dialect.find("mysql") == std::string::npos && dialect.find("sqlite") == std::string::npos &&
dialect.find("mock") == std::string::npos) {
LOG_SERVER_ERROR_ << "Invalid dialect in URI: dialect = " << dialect;
okay = false;
}
/*
* Could be DNS, skip checking
*
std::string host = pieces_match[4].str();
if (!host.empty() && host != "localhost") {
if (ValidateIpAddress(host) != SERVER_SUCCESS) {
LOG_SERVER_ERROR_ << "Invalid host ip address in uri = " << host;
okay = false;
}
}
*/
std::string port = pieces_match[5].str();
if (!port.empty()) {
auto status = ValidateStringIsNumber(port);
if (!status.ok()) {
LOG_SERVER_ERROR_ << "Invalid port in uri = " << port;
okay = false;
}
}
} else {
LOG_SERVER_ERROR_ << "Wrong URI format: URI = " << uri;
okay = false;
}
return (okay ? Status::OK() : Status(SERVER_INVALID_ARGUMENT, "Invalid db backend uri"));
}
Status
ValidateStoragePath(const std::string& path) {
// Validate storage path if is valid, only correct absolute path will be validated pass
// Invalid path only contain character[a-zA-Z], number[0-9], '-', and '_',
// and path must start with '/'.
// examples below are invalid
// '/a//a', '/a--/a', '/-a/a', '/a@#/a', 'aaa/sfs'
std::string path_pattern = R"(^\/(\w+-?\/?)+$)";
std::regex regex(path_pattern);
return std::regex_match(path, regex) ? Status::OK() : Status(SERVER_INVALID_ARGUMENT, "Invalid file path");
}
Status
ValidateLogLevel(const std::string& level) {
std::set<std::string> supported_level{"debug", "info", "warning", "error", "fatal"};
return supported_level.find(level) != supported_level.end()
? Status::OK()
: Status(SERVER_INVALID_ARGUMENT, "Log level must be one of debug, info, warning, error and fatal.");
}
bool
IsNumber(const std::string& s) {
return !s.empty() && std::all_of(s.begin(), s.end(), ::isdigit);
}
} // namespace server
} // namespace milvus

63
internal/core/src/index/knowhere/utils/ConfigUtils.h
View File

@ -1,63 +0,0 @@
// Copyright (C) 2019-2020 Zilliz. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software distributed under the License
// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
// or implied. See the License for the specific language governing permissions and limitations under the License.
#pragma once
#include <string>
#include <vector>
#include "utils/Status.h"
namespace milvus {
namespace server {
extern int64_t
parse_bytes(const std::string& str, std::string& err);
extern bool
GetSystemMemInfo(int64_t& total_mem, int64_t& free_mem);
extern bool
GetSystemAvailableThreads(int64_t& thread_count);
extern Status
ValidateGpuIndex(int32_t gpu_index);
#ifdef MILVUS_GPU_VERSION
extern Status
GetGpuMemory(int32_t gpu_index, int64_t& memory);
#endif
extern Status
ValidateIpAddress(const std::string& ip_address);
extern Status
ValidateStringIsNumber(const std::string& str);
extern Status
ValidateStringIsBool(const std::string& str);
extern Status
ValidateStringIsFloat(const std::string& str);
extern Status
ValidateDbURI(const std::string& uri);
extern Status
ValidateStoragePath(const std::string& path);
extern Status
ValidateLogLevel(const std::string& level);
extern bool
IsNumber(const std::string& s);
} // namespace server
} // namespace milvus

145
internal/core/src/index/knowhere/utils/Error.h
View File

@ -1,145 +0,0 @@
// Copyright (C) 2019-2020 Zilliz. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software distributed under the License
// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
// or implied. See the License for the specific language governing permissions and limitations under the License.
#pragma once
#include <cstdint>
#include <exception>
#include <string>
namespace milvus {
using ErrorCode = int32_t;
constexpr ErrorCode SERVER_SUCCESS = 0;
constexpr ErrorCode SERVER_ERROR_CODE_BASE = 30000;
constexpr ErrorCode
ToServerErrorCode(const ErrorCode error_code) {
return SERVER_ERROR_CODE_BASE + error_code;
}
constexpr ErrorCode DB_SUCCESS = 0;
constexpr ErrorCode DB_ERROR_CODE_BASE = 40000;
constexpr ErrorCode
ToDbErrorCode(const ErrorCode error_code) {
return DB_ERROR_CODE_BASE + error_code;
}
constexpr ErrorCode KNOWHERE_SUCCESS = 0;
constexpr ErrorCode KNOWHERE_ERROR_CODE_BASE = 50000;
constexpr ErrorCode
ToKnowhereErrorCode(const ErrorCode error_code) {
return KNOWHERE_ERROR_CODE_BASE + error_code;
}
constexpr ErrorCode WAL_SUCCESS = 0;
constexpr ErrorCode WAL_ERROR_CODE_BASE = 60000;
constexpr ErrorCode
ToWalErrorCode(const ErrorCode error_code) {
return WAL_ERROR_CODE_BASE + error_code;
}
constexpr ErrorCode SS_SUCCESS = 0;
constexpr ErrorCode SS_ERROR_CODE_BASE = 70000;
constexpr ErrorCode
ToSSErrorCode(const ErrorCode error_code) {
return SS_ERROR_CODE_BASE + error_code;
}
// server error code
constexpr ErrorCode SERVER_UNEXPECTED_ERROR = ToServerErrorCode(1);
constexpr ErrorCode SERVER_UNSUPPORTED_ERROR = ToServerErrorCode(2);
constexpr ErrorCode SERVER_NULL_POINTER = ToServerErrorCode(3);
constexpr ErrorCode SERVER_INVALID_ARGUMENT = ToServerErrorCode(4);
constexpr ErrorCode SERVER_FILE_NOT_FOUND = ToServerErrorCode(5);
constexpr ErrorCode SERVER_NOT_IMPLEMENT = ToServerErrorCode(6);
constexpr ErrorCode SERVER_CANNOT_CREATE_FOLDER = ToServerErrorCode(8);
constexpr ErrorCode SERVER_CANNOT_CREATE_FILE = ToServerErrorCode(9);
constexpr ErrorCode SERVER_CANNOT_DELETE_FOLDER = ToServerErrorCode(10);
constexpr ErrorCode SERVER_CANNOT_DELETE_FILE = ToServerErrorCode(11);
constexpr ErrorCode SERVER_BUILD_INDEX_ERROR = ToServerErrorCode(12);
constexpr ErrorCode SERVER_CANNOT_OPEN_FILE = ToServerErrorCode(13);
constexpr ErrorCode SERVER_FILE_MAGIC_BYTES_ERROR = ToServerErrorCode(14);
constexpr ErrorCode SERVER_FILE_SUM_BYTES_ERROR = ToServerErrorCode(15);
constexpr ErrorCode SERVER_CANNOT_READ_FILE = ToServerErrorCode(16);
constexpr ErrorCode SERVER_COLLECTION_NOT_EXIST = ToServerErrorCode(100);
constexpr ErrorCode SERVER_INVALID_COLLECTION_NAME = ToServerErrorCode(101);
constexpr ErrorCode SERVER_INVALID_COLLECTION_DIMENSION = ToServerErrorCode(102);
constexpr ErrorCode SERVER_INVALID_VECTOR_DIMENSION = ToServerErrorCode(104);
constexpr ErrorCode SERVER_INVALID_INDEX_TYPE = ToServerErrorCode(105);
constexpr ErrorCode SERVER_INVALID_ROWRECORD = ToServerErrorCode(106);
constexpr ErrorCode SERVER_INVALID_ROWRECORD_ARRAY = ToServerErrorCode(107);
constexpr ErrorCode SERVER_INVALID_TOPK = ToServerErrorCode(108);
constexpr ErrorCode SERVER_ILLEGAL_VECTOR_ID = ToServerErrorCode(109);
constexpr ErrorCode SERVER_ILLEGAL_SEARCH_RESULT = ToServerErrorCode(110);
constexpr ErrorCode SERVER_CACHE_FULL = ToServerErrorCode(111);
constexpr ErrorCode SERVER_WRITE_ERROR = ToServerErrorCode(112);
constexpr ErrorCode SERVER_INVALID_NPROBE = ToServerErrorCode(113);
constexpr ErrorCode SERVER_INVALID_INDEX_NLIST = ToServerErrorCode(114);
constexpr ErrorCode SERVER_INVALID_INDEX_METRIC_TYPE = ToServerErrorCode(115);
constexpr ErrorCode SERVER_INVALID_SEGMENT_ROW_COUNT = ToServerErrorCode(116);
constexpr ErrorCode SERVER_OUT_OF_MEMORY = ToServerErrorCode(117);
constexpr ErrorCode SERVER_INVALID_PARTITION_TAG = ToServerErrorCode(118);
constexpr ErrorCode SERVER_INVALID_BINARY_QUERY = ToServerErrorCode(119);
constexpr ErrorCode SERVER_INVALID_DSL_PARAMETER = ToServerErrorCode(120);
constexpr ErrorCode SERVER_INVALID_FIELD_NAME = ToServerErrorCode(121);
constexpr ErrorCode SERVER_INVALID_FIELD_NUM = ToServerErrorCode(122);
// db error code
constexpr ErrorCode DB_META_TRANSACTION_FAILED = ToDbErrorCode(1);
constexpr ErrorCode DB_ERROR = ToDbErrorCode(2);
constexpr ErrorCode DB_NOT_FOUND = ToDbErrorCode(3);
constexpr ErrorCode DB_ALREADY_EXIST = ToDbErrorCode(4);
constexpr ErrorCode DB_INVALID_PATH = ToDbErrorCode(5);
constexpr ErrorCode DB_INCOMPATIB_META = ToDbErrorCode(6);
constexpr ErrorCode DB_INVALID_META_URI = ToDbErrorCode(7);
constexpr ErrorCode DB_EMPTY_COLLECTION = ToDbErrorCode(8);
constexpr ErrorCode DB_BLOOM_FILTER_ERROR = ToDbErrorCode(9);
constexpr ErrorCode DB_PARTITION_NOT_FOUND = ToDbErrorCode(10);
constexpr ErrorCode DB_OUT_OF_STORAGE = ToDbErrorCode(11);
constexpr ErrorCode DB_META_QUERY_FAILED = ToDbErrorCode(12);
constexpr ErrorCode DB_FILE_NOT_FOUND = ToDbErrorCode(13);
constexpr ErrorCode DB_PERMISSION_ERROR = ToDbErrorCode(14);
// knowhere error code
constexpr ErrorCode KNOWHERE_ERROR = ToKnowhereErrorCode(1);
constexpr ErrorCode KNOWHERE_INVALID_ARGUMENT = ToKnowhereErrorCode(2);
constexpr ErrorCode KNOWHERE_UNEXPECTED_ERROR = ToKnowhereErrorCode(3);
constexpr ErrorCode KNOWHERE_NO_SPACE = ToKnowhereErrorCode(4);
// knowhere error code
constexpr ErrorCode WAL_ERROR = ToWalErrorCode(1);
constexpr ErrorCode WAL_META_ERROR = ToWalErrorCode(2);
constexpr ErrorCode WAL_FILE_ERROR = ToWalErrorCode(3);
constexpr ErrorCode WAL_PATH_ERROR = ToWalErrorCode(4);
// Snapshot error code
constexpr ErrorCode SS_ERROR = ToSSErrorCode(1);
constexpr ErrorCode SS_STALE_ERROR = ToSSErrorCode(2);
constexpr ErrorCode SS_NOT_FOUND_ERROR = ToSSErrorCode(3);
constexpr ErrorCode SS_INVALID_CONTEX_ERROR = ToSSErrorCode(4);
constexpr ErrorCode SS_DUPLICATED_ERROR = ToSSErrorCode(5);
constexpr ErrorCode SS_NOT_ACTIVE_ERROR = ToSSErrorCode(6);
constexpr ErrorCode SS_CONSTRAINT_CHECK_ERROR = ToSSErrorCode(7);
constexpr ErrorCode SS_INVALID_ARGUMENT_ERROR = ToSSErrorCode(8);
constexpr ErrorCode SS_OPERATION_PENDING = ToSSErrorCode(9);
constexpr ErrorCode SS_TIMEOUT = ToSSErrorCode(10);
constexpr ErrorCode SS_NOT_COMMITED = ToSSErrorCode(11);
constexpr ErrorCode SS_COLLECTION_DROPPED = ToSSErrorCode(12);
constexpr ErrorCode SS_EMPTY_HOLDER = ToSSErrorCode(13);
} // namespace milvus

61
internal/core/src/index/knowhere/utils/Exception.h
View File

@ -1,61 +0,0 @@
// Copyright (C) 2019-2020 Zilliz. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software distributed under the License
// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
// or implied. See the License for the specific language governing permissions and limitations under the License.
#pragma once
#include "utils/Error.h"
#include <exception>
#include <string>
#include <utility>
namespace milvus {
#define THROW_ERROR(err_code, err_msg) \
LOG_ENGINE_ERROR_ << err_msg; \
throw Exception(err_code, err_msg);
class Exception : public std::exception {
public:
Exception(ErrorCode code, std::string msg) : code_(code), message_(std::move(msg)) {
}
ErrorCode
code() const noexcept {
return code_;
}
const char*
what() const noexcept override {
if (message_.empty()) {
return "Default Exception.";
} else {
return message_.c_str();
}
}
~Exception() noexcept override = default;
protected:
ErrorCode code_;
std::string message_;
};
class InvalidArgumentException : public Exception {
public:
InvalidArgumentException() : Exception(SERVER_INVALID_ARGUMENT, "Invalid Argument") {
}
explicit InvalidArgumentException(const std::string& message) : Exception(SERVER_INVALID_ARGUMENT, message) {
}
};
} // namespace milvus

24
internal/core/src/index/knowhere/utils/Json.h
View File

@ -9,26 +9,4 @@
// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
// or implied. See the License for the specific language governing permissions and limitations under the License.
#pragma once
#include "nlohmann/json.hpp"
namespace milvus {
using json = nlohmann::json;
#define JSON_NULL_CHECK(json) \
do { \
if (json.empty()) { \
return Status{SERVER_INVALID_ARGUMENT, "Json is null"}; \
} \
} while (false)
#define JSON_OBJECT_CHECK(json) \
do { \
if (!json.is_object()) { \
return Status{SERVER_INVALID_ARGUMENT, "Json is not a json object"}; \
} \
} while (false)
} // namespace milvus
#include "../../../utils/Json.h"

60
internal/core/src/index/knowhere/utils/SignalHandler.cpp
View File

@ -1,60 +0,0 @@
// Copyright (C) 2019-2020 Zilliz. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software distributed under the License
// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
// or implied. See the License for the specific language governing permissions and limitations under the License.
#include "utils/SignalHandler.h"
#include "utils/Log.h"
#include <execinfo.h>
#include <string>
namespace milvus {
signal_func_ptr signal_routine_func = nullptr;
void
HandleSignal(int signum) {
int32_t exit_code = 1; /* 0: normal exit; 1: exception */
switch (signum) {
case SIGINT:
case SIGUSR2:
exit_code = 0;
/* no break */
default: {
if (exit_code == 0) {
LOG_SERVER_INFO_ << "Server received signal: " << signum;
} else {
LOG_SERVER_INFO_ << "Server received critical signal: " << signum;
PrintStacktrace();
}
if (signal_routine_func != nullptr) {
(*signal_routine_func)(exit_code);
}
}
}
}
void
PrintStacktrace() {
const int bt_depth = 128;
void* array[bt_depth];
int stack_num = backtrace(array, bt_depth);
char** stacktrace = backtrace_symbols(array, stack_num);
LOG_SERVER_INFO_ << "Call stack:";
for (int i = 0; i < stack_num; ++i) {
std::string info = stacktrace[i];
std::cout << "No." << i << ": " << info << std::endl;
LOG_SERVER_INFO_ << info;
}
free(stacktrace);
}
} // namespace milvus

28
internal/core/src/index/knowhere/utils/SignalHandler.h
View File

@ -1,28 +0,0 @@
// Copyright (C) 2019-2020 Zilliz. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software distributed under the License
// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
// or implied. See the License for the specific language governing permissions and limitations under the License.
#pragma once
#include <cstdint>
namespace milvus {
typedef void (*signal_func_ptr)(int32_t);
extern signal_func_ptr signal_routine_func;
extern void
HandleSignal(int signum);
extern void
PrintStacktrace();
} // namespace milvus

127
internal/core/src/index/knowhere/utils/Status.cpp
View File

@ -1,127 +0,0 @@
// Copyright (C) 2019-2020 Zilliz. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software distributed under the License
// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
// or implied. See the License for the specific language governing permissions and limitations under the License.
#include "utils/Status.h"
#include "memory"
#include <cstring>
namespace milvus {
constexpr int CODE_WIDTH = sizeof(StatusCode);
Status::Status(StatusCode code, const std::string& msg) {
// 4 bytes store code
// 4 bytes store message length
// the left bytes store message string
auto length = static_cast<uint32_t>(msg.size());
// auto result = new char[length + sizeof(length) + CODE_WIDTH];
state_.resize(length + sizeof(length) + CODE_WIDTH);
std::memcpy(state_.data(), &code, CODE_WIDTH);
std::memcpy(state_.data() + CODE_WIDTH, &length, sizeof(length));
memcpy(state_.data() + sizeof(length) + CODE_WIDTH, msg.data(), length);
}
Status::~Status() {
}
Status::Status(const Status& s) {
CopyFrom(s);
}
Status::Status(Status&& s) noexcept {
MoveFrom(s);
}
Status&
Status::operator=(const Status& s) {
CopyFrom(s);
return *this;
}
Status&
Status::operator=(Status&& s) noexcept {
MoveFrom(s);
return *this;
}
void
Status::CopyFrom(const Status& s) {
state_.clear();
if (s.state_.empty()) {
return;
}
uint32_t length = 0;
memcpy(&length, s.state_.data() + CODE_WIDTH, sizeof(length));
int buff_len = length + sizeof(length) + CODE_WIDTH;
state_.resize(buff_len);
memcpy(state_.data(), s.state_.data(), buff_len);
}
void
Status::MoveFrom(Status& s) {
state_ = s.state_;
s.state_.clear();
}
std::string
Status::message() const {
if (state_.empty()) {
return "OK";
}
std::string msg;
uint32_t length = 0;
memcpy(&length, state_.data() + CODE_WIDTH, sizeof(length));
if (length > 0) {
msg.append(state_.data() + sizeof(length) + CODE_WIDTH, length);
}
return msg;
}
std::string
Status::ToString() const {
if (state_.empty()) {
return "OK";
}
std::string result;
switch (code()) {
case DB_SUCCESS:
result = "OK ";
break;
case DB_ERROR:
result = "Error: ";
break;
case DB_META_TRANSACTION_FAILED:
result = "Database error: ";
break;
case DB_NOT_FOUND:
result = "Not found: ";
break;
case DB_ALREADY_EXIST:
result = "Already exist: ";
break;
case DB_INVALID_PATH:
result = "Invalid path: ";
break;
default:
result = "Error code(" + std::to_string(code()) + "): ";
break;
}
result += message();
return result;
}
} // namespace milvus

69
internal/core/src/index/knowhere/utils/Status.h
View File

@ -9,71 +9,4 @@
// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
// or implied. See the License for the specific language governing permissions and limitations under the License.
#pragma once
#include "utils/Error.h"
#include <string>
namespace milvus {
class Status;
#define STATUS_CHECK(func) \
do { \
Status s = func; \
if (!s.ok()) { \
return s; \
} \
} while (false)
using StatusCode = ErrorCode;
class Status {
public:
Status(StatusCode code, const std::string& msg);
Status() = default;
virtual ~Status();
Status(const Status& s);
Status(Status&& s) noexcept;
Status&
operator=(const Status& s);
Status&
operator=(Status&& s) noexcept;
static Status
OK() {
return Status();
}
bool
ok() const {
return state_.empty() || code() == 0;
}
StatusCode
code() const {
return (state_.empty()) ? 0 : *(StatusCode*)(state_.data());
}
std::string
message() const;
std::string
ToString() const;
private:
inline void
CopyFrom(const Status& s);
inline void
MoveFrom(Status& s);
private:
std::string state_;
}; // Status
} // namespace milvus
#include "../../../utils/Status.h"

175
internal/core/src/index/knowhere/utils/StringHelpFunctions.cpp
View File

@ -1,175 +0,0 @@
// Copyright (C) 2019-2020 Zilliz. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software distributed under the License
// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
// or implied. See the License for the specific language governing permissions and limitations under the License.
#include "utils/StringHelpFunctions.h"
#include "utils/Log.h"
#include <fiu/fiu-local.h>
#include <algorithm>
#include <regex>
#include <string>
namespace milvus {
void
StringHelpFunctions::TrimStringBlank(std::string& string) {
if (!string.empty()) {
static std::string s_format(" \n\r\t");
string.erase(0, string.find_first_not_of(s_format));
string.erase(string.find_last_not_of(s_format) + 1);
}
}
void
StringHelpFunctions::TrimStringQuote(std::string& string, const std::string& qoute) {
if (!string.empty()) {
string.erase(0, string.find_first_not_of(qoute));
string.erase(string.find_last_not_of(qoute) + 1);
}
}
void
StringHelpFunctions::SplitStringByDelimeter(const std::string& str,
const std::string& delimeter,
std::vector<std::string>& result) {
if (str.empty()) {
return;
}
size_t prev = 0;
while (true) {
size_t pos = str.find_first_of(delimeter, prev);
if (pos == std::string::npos) {
result.emplace_back(str.substr(prev));
break;
} else {
result.emplace_back(str.substr(prev, pos - prev));
prev = pos + 1;
}
}
}
void
StringHelpFunctions::MergeStringWithDelimeter(const std::vector<std::string>& strs,
const std::string& delimeter,
std::string& result) {
if (strs.empty()) {
result = "";
return;
}
result = strs[0];
for (size_t i = 1; i < strs.size(); i++) {
result = result + delimeter + strs[i];
}
}
Status
StringHelpFunctions::SplitStringByQuote(const std::string& str,
const std::string& delimeter,
const std::string& quote,
std::vector<std::string>& result) {
if (quote.empty()) {
SplitStringByDelimeter(str, delimeter, result);
return Status::OK();
}
size_t last = 0;
size_t index = str.find_first_of(quote, last);
if (index == std::string::npos) {
SplitStringByDelimeter(str, delimeter, result);
return Status::OK();
}
std::string process_str = str;
while (index != std::string::npos) {
std::string prefix = process_str.substr(last, index - last);
std::string append_prefix;
if (!prefix.empty()) {
std::vector<std::string> prefix_split;
SplitStringByDelimeter(prefix, delimeter, prefix_split);
for (size_t i = 0; i < prefix_split.size() - 1; i++) {
result.push_back(prefix_split[i]);
}
append_prefix = prefix_split[prefix_split.size() - 1];
}
last = index + 1;
std::string postfix = process_str.substr(last);
index = postfix.find_first_of(quote, 0);
fiu_do_on("StringHelpFunctions.SplitStringByQuote.invalid_index", index = std::string::npos);
if (index == std::string::npos) {
return Status(SERVER_UNEXPECTED_ERROR, "");
}
std::string quoted_text = postfix.substr(0, index);
append_prefix += quoted_text;
last = index + 1;
index = postfix.find_first_of(delimeter, last);
fiu_do_on("StringHelpFunctions.SplitStringByQuote.index_gt_last", last = 0);
fiu_do_on("StringHelpFunctions.SplitStringByQuote.invalid_index2", index = std::string::npos);
if (index != std::string::npos) {
if (index > last) {
append_prefix += postfix.substr(last, index - last);
}
} else {
append_prefix += postfix.substr(last);
}
result.emplace_back(append_prefix);
fiu_do_on("StringHelpFunctions.SplitStringByQuote.last_is_end", last = postfix.length());
if (last == postfix.length()) {
return Status::OK();
}
process_str = postfix.substr(index + 1);
last = 0;
index = process_str.find_first_of(quote, last);
}
if (!process_str.empty()) {
SplitStringByDelimeter(process_str, delimeter, result);
}
return Status::OK();
}
bool
StringHelpFunctions::IsRegexMatch(const std::string& target_str, const std::string& pattern_str) {
// if target_str equals pattern_str, return true
if (target_str == pattern_str) {
return true;
}
// regex match
// for illegal regex expression, the std::regex will throw exception, regard as unmatch
try {
std::regex pattern(pattern_str);
std::smatch results;
return std::regex_match(target_str, results, pattern);
} catch (std::exception& e) {
LOG_SERVER_ERROR_ << "Regex exception: " << e.what();
}
return false;
}
Status
StringHelpFunctions::ConvertToBoolean(const std::string& str, bool& value) {
std::string s = str;
std::transform(s.begin(), s.end(), s.begin(), ::tolower);
value = s == "true" || s == "on" || s == "yes" || s == "1";
return Status::OK();
}
} // namespace milvus

76
internal/core/src/index/knowhere/utils/StringHelpFunctions.h
View File

@ -1,76 +0,0 @@
// Copyright (C) 2019-2020 Zilliz. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software distributed under the License
// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
// or implied. See the License for the specific language governing permissions and limitations under the License.
#pragma once
#include "utils/Status.h"
#include <string>
#include <vector>
namespace milvus {
class StringHelpFunctions {
private:
StringHelpFunctions() = default;
public:
// trim blanks from begin and end
// " a b c " => "a b c"
static void
TrimStringBlank(std::string& string);
// trim quotes from begin and end
// "'abc'" => "abc"
static void
TrimStringQuote(std::string& string, const std::string& qoute);
// split string by delimeter ','
// a,b,c a | b | c
// a,b, a | b |
// ,b,c | b | c
// ,b, | b |
// ,, | |
// a a
static void
SplitStringByDelimeter(const std::string& str, const std::string& delimeter, std::vector<std::string>& result);
// merge strings with delimeter
// "a", "b", "c" => "a,b,c"
static void
MergeStringWithDelimeter(const std::vector<std::string>& strs, const std::string& delimeter, std::string& result);
// assume the collection has two columns, quote='\"', delimeter=','
// a,b a | b
// "aa,gg,yy",b aa,gg,yy | b
// aa"dd,rr"kk,pp aadd,rrkk | pp
// "aa,bb" aa,bb
// 55,1122\"aa,bb\",yyy,\"kkk\" 55 | 1122aa,bb | yyy | kkk
// "55,1122"aa,bb",yyy,"kkk" illegal
static Status
SplitStringByQuote(const std::string& str,
const std::string& delimeter,
const std::string& quote,
std::vector<std::string>& result);
// std regex match function
// regex grammar reference: http://www.cplusplus.com/reference/regex/ECMAScript/
static bool
IsRegexMatch(const std::string& target_str, const std::string& pattern);
// conversion rules refer to ValidationUtil::ValidateStringIsBool()
// "true", "on", "yes", "1" ==> true
// "false", "off", "no", "0", "" ==> false
static Status
ConvertToBoolean(const std::string& str, bool& value);
};
} // namespace milvus

128
internal/core/src/index/knowhere/utils/ThreadPool.h
View File

@ -1,128 +0,0 @@
// Copyright (C) 2019-2020 Zilliz. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software distributed under the License
// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
// or implied. See the License for the specific language governing permissions and limitations under the License.
#pragma once
#include <fiu/fiu-local.h>
#include <atomic>
#include <condition_variable>
#include <functional>
#include <future>
#include <memory>
#include <mutex>
#include <queue>
#include <stdexcept>
#include <thread>
#include <utility>
#include <vector>
#define MAX_THREADS_NUM 32
namespace milvus {
class ThreadPool {
public:
explicit ThreadPool(size_t threads, size_t queue_size = 1000);
template <class F, class... Args>
auto
enqueue(F&& f, Args&&... args) -> std::future<typename std::result_of<F(Args...)>::type>;
void
Stop();
~ThreadPool();
private:
// need to keep track of threads so we can join them
std::vector<std::thread> workers_;
// the task queue
std::queue<std::function<void()>> tasks_;
size_t max_queue_size_;
// synchronization
std::mutex queue_mutex_;
std::condition_variable condition_;
std::atomic_bool stop_;
};
// the constructor just launches some amount of workers
inline ThreadPool::ThreadPool(size_t threads, size_t queue_size) : max_queue_size_(queue_size), stop_(false) {
for (size_t i = 0; i < threads; ++i)
workers_.emplace_back([this] {
for (;;) {
std::function<void()> task;
{
std::unique_lock<std::mutex> lock(this->queue_mutex_);
this->condition_.wait(lock, [this] { return this->stop_ || !this->tasks_.empty(); });
if (this->stop_ && this->tasks_.empty())
return;
task = std::move(this->tasks_.front());
this->tasks_.pop();
}
this->condition_.notify_all();
task();
}
});
}
// add new work item to the pool
template <class F, class... Args>
auto
ThreadPool::enqueue(F&& f, Args&&... args) -> std::future<typename std::result_of<F(Args...)>::type> {
using return_type = typename std::result_of<F(Args...)>::type;
auto task =
std::make_shared<std::packaged_task<return_type()>>(std::bind(std::forward<F>(f), std::forward<Args>(args)...));
fiu_do_on("ThreadPool.enqueue.stop_is_true", stop_ = true);
std::future<return_type> res = task->get_future();
{
std::unique_lock<std::mutex> lock(queue_mutex_);
this->condition_.wait(lock, [this] { return this->tasks_.size() < max_queue_size_; });
// don't allow enqueueing after stopping the pool
if (stop_)
throw std::runtime_error("enqueue on stopped ThreadPool");
tasks_.emplace([task]() { (*task)(); });
}
condition_.notify_all();
return res;
}
inline void
ThreadPool::Stop() {
if (stop_) {
return;
}
{
std::unique_lock<std::mutex> lock(queue_mutex_);
stop_ = true;
}
condition_.notify_all();
for (std::thread& worker : workers_) {
worker.join();
}
}
// the destructor joins all threads
inline ThreadPool::~ThreadPool() {
Stop();
}
using ThreadPoolPtr = std::shared_ptr<ThreadPool>;
} // namespace milvus

93
internal/core/src/index/knowhere/utils/TimeRecorder.cpp
View File

@ -1,93 +0,0 @@
// Copyright (C) 2019-2020 Zilliz. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software distributed under the License
// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
// or implied. See the License for the specific language governing permissions and limitations under the License.
#include <utility>
#include "utils/Log.h"
#include "utils/TimeRecorder.h"
namespace milvus {
TimeRecorder::TimeRecorder(std::string hdr, int64_t log_level) : header_(std::move(hdr)), log_level_(log_level) {
start_ = last_ = stdclock::now();
}
std::string
TimeRecorder::GetTimeSpanStr(double span) {
std::string str_sec = std::to_string(span * 0.000001) + ((span > 1000000) ? " seconds" : " second");
std::string str_ms = std::to_string(span * 0.001) + " ms";
return str_sec + " [" + str_ms + "]";
}
void
TimeRecorder::PrintTimeRecord(const std::string& msg, double span) {
std::string str_log;
if (!header_.empty()) {
str_log += header_ + ": ";
}
str_log += msg;
str_log += " (";
str_log += TimeRecorder::GetTimeSpanStr(span);
str_log += ")";
switch (log_level_) {
case 0:
LOG_SERVER_TRACE_ << str_log;
break;
case 1:
LOG_SERVER_DEBUG_ << str_log;
break;
case 2:
LOG_SERVER_INFO_ << str_log;
break;
case 3:
LOG_SERVER_WARNING_ << str_log;
break;
case 4:
LOG_SERVER_ERROR_ << str_log;
break;
case 5:
LOG_SERVER_FATAL_ << str_log;
break;
default:
LOG_SERVER_INFO_ << str_log;
break;
}
}
double
TimeRecorder::RecordSection(const std::string& msg) {
stdclock::time_point curr = stdclock::now();
double span = (std::chrono::duration<double, std::micro>(curr - last_)).count();
last_ = curr;
PrintTimeRecord(msg, span);
return span;
}
double
TimeRecorder::ElapseFromBegin(const std::string& msg) {
stdclock::time_point curr = stdclock::now();
double span = (std::chrono::duration<double, std::micro>(curr - start_)).count();
PrintTimeRecord(msg, span);
return span;
}
TimeRecorderAuto::TimeRecorderAuto(std::string hdr, int64_t log_level) : TimeRecorder(hdr, log_level) {
}
TimeRecorderAuto::~TimeRecorderAuto() {
ElapseFromBegin("totally cost");
}
} // namespace milvus

65
internal/core/src/index/knowhere/utils/TimeRecorder.h
View File

@ -1,65 +0,0 @@
// Copyright (C) 2019-2020 Zilliz. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software distributed under the License
// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
// or implied. See the License for the specific language governing permissions and limitations under the License.
#pragma once
#include <chrono>
#include <string>
#include "utils/Log.h"
namespace milvus {
inline void
print_timestamp(const std::string& message) {
std::chrono::time_point<std::chrono::system_clock> now = std::chrono::system_clock::now();
auto duration = now.time_since_epoch();
auto micros = std::chrono::duration_cast<std::chrono::microseconds>(duration).count();
micros %= 1000000;
double millisecond = (double)micros / 1000.0;
LOG_SERVER_DEBUG_ << std::fixed << " " << millisecond << "(ms) [timestamp]" << message;
}
class TimeRecorder {
using stdclock = std::chrono::high_resolution_clock;
public:
// trace = 0, debug = 1, info = 2, warn = 3, error = 4, critical = 5
explicit TimeRecorder(std::string hdr, int64_t log_level = 1);
virtual ~TimeRecorder() = default;
double
RecordSection(const std::string& msg);
double
ElapseFromBegin(const std::string& msg);
static std::string
GetTimeSpanStr(double span);
private:
void
PrintTimeRecord(const std::string& msg, double span);
private:
std::string header_;
stdclock::time_point start_;
stdclock::time_point last_;
int64_t log_level_;
};
class TimeRecorderAuto : public TimeRecorder {
public:
explicit TimeRecorderAuto(std::string hdr, int64_t log_level = 1);
~TimeRecorderAuto() override;
};
} // namespace milvus

76
internal/core/src/index/knowhere/utils/TimerContext.h
View File

@ -1,76 +0,0 @@
// Copyright (C) 2019-2020 Zilliz. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software distributed under the License
// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
// or implied. See the License for the specific language governing permissions and limitations under the License.
#pragma once
#include <boost/asio.hpp>
#include <boost/date_time/posix_time/posix_time.hpp>
#include <functional>
#include <memory>
#include <string>
#include <vector>
#include "utils/Log.h"
#include "utils/ThreadPool.h"
namespace milvus {
struct TimerContext {
using HandlerT = std::function<void(const boost::system::error_code&)>;
struct Context {
/* Context(int interval_us, HandlerT& handler, ThreadPoolPtr pool = nullptr) */
/* : interval_(interval_us), handler_(handler), timer_(io, interval_), pool_(pool) { */
/* } */
int interval_us;
HandlerT handler;
ThreadPoolPtr pool = nullptr;
};
TimerContext(boost::asio::io_service& io, int interval_us, HandlerT& handler, ThreadPoolPtr pool)
: io_(io), interval_(interval_us), handler_(handler), timer_(io, interval_), pool_(pool) {
}
TimerContext(boost::asio::io_service& io, Context& context)
: io_(io),
interval_(context.interval_us),
handler_(context.handler),
timer_(io, interval_),
pool_(context.pool) {
}
void
Reschedule(const boost::system::error_code& ec);
boost::asio::io_service& io_;
boost::posix_time::microseconds interval_;
boost::asio::deadline_timer timer_;
HandlerT handler_;
ThreadPoolPtr pool_;
};
inline void
TimerContext::Reschedule(const boost::system::error_code& ec) {
try {
pool_->enqueue(handler_, ec);
} catch (std::exception& ex) {
LOG_SERVER_ERROR_ << "Fail to enqueue handler: " << std::string(ex.what());
}
boost::system::error_code e;
auto new_expires = timer_.expires_at() + interval_;
timer_.expires_at(new_expires, e);
if (e) {
LOG_SERVER_ERROR_ << "Fail to Reschedule: " << e;
}
timer_.async_wait(std::bind(&TimerContext::Reschedule, this, std::placeholders::_1));
}
using TimerContextPtr = std::shared_ptr<TimerContext>;
} // namespace milvus

85
internal/core/src/index/knowhere/utils/TimerManager.cpp
View File

@ -1,85 +0,0 @@
// Copyright (C) 2019-2020 Zilliz. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software distributed under the License
// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
// or implied. See the License for the specific language governing permissions and limitations under the License.
#include <utility>
#include "utils/Log.h"
#include "utils/TimerManager.h"
namespace milvus {
TimerManager::TimerManager(unsigned int pool_size) : pool_size_(pool_size) {
}
TimerManager::~TimerManager() {
}
Status
TimerManager::SetPoolSize(unsigned int pool_size) {
if (timer_exeutors_) {
return Status(SERVER_UNEXPECTED_ERROR, "Cannot set pool size since it has been set already");
}
pool_size_ = pool_size;
return Status::OK();
}
Status
TimerManager::Run() {
boost::system::error_code ec;
io_.run(ec);
Status status;
if (ec) {
status = Status(SERVER_UNEXPECTED_ERROR, ec.message());
}
return status;
}
Status
TimerManager::Start() {
for (auto& timer : timers_) {
timer->timer_.async_wait(std::bind(&TimerContext::Reschedule, timer, std::placeholders::_1));
}
return Status::OK();
}
void
TimerManager::Stop() {
boost::system::error_code ec;
for (auto& timer : timers_) {
timer->timer_.cancel(ec);
if (ec) {
LOG_SERVER_ERROR_ << "Fail to cancel timer: " << ec;
}
}
if (timer_exeutors_) {
timer_exeutors_->Stop();
}
}
void
TimerManager::AddTimer(int interval_us, TimerContext::HandlerT handler) {
if (!timer_exeutors_) {
timer_exeutors_ = std::make_shared<ThreadPool>(pool_size_);
}
timers_.emplace_back(std::make_shared<TimerContext>(io_, interval_us, handler, timer_exeutors_));
}
void
TimerManager::AddTimer(const TimerContext::Context& ctx) {
if (!timer_exeutors_) {
timer_exeutors_ = std::make_shared<ThreadPool>(pool_size_);
}
TimerContext::Context context(ctx);
context.pool = timer_exeutors_;
timers_.emplace_back(std::make_shared<TimerContext>(io_, context));
}
} // namespace milvus

57
internal/core/src/index/knowhere/utils/TimerManager.h
View File

@ -1,57 +0,0 @@
// Copyright (C) 2019-2020 Zilliz. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software distributed under the License
// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
// or implied. See the License for the specific language governing permissions and limitations under the License.
#pragma once
#include <boost/asio.hpp>
#include <functional>
#include <memory>
#include <vector>
#include "utils/Status.h"
#include "utils/ThreadPool.h"
#include "utils/TimerContext.h"
namespace milvus {
class TimerManager {
public:
TimerManager() = default;
explicit TimerManager(unsigned int pool_size);
Status
SetPoolSize(unsigned int pool_size);
void
AddTimer(int interval_us, TimerContext::HandlerT handler);
void
AddTimer(const TimerContext::Context& ctx);
virtual Status
Run();
virtual Status
Start();
virtual void
Stop();
virtual ~TimerManager();
protected:
boost::asio::io_service io_;
ThreadPoolPtr timer_exeutors_;
unsigned int pool_size_;
std::vector<TimerContextPtr> timers_;
};
using TimerManagerPtr = std::shared_ptr<TimerManager>;
} // namespace milvus

12
internal/core/src/index/knowhere/utils/Types.cpp
View File

@ -1,12 +0,0 @@
// Copyright (C) 2019-2020 Zilliz. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software distributed under the License
// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
// or implied. See the License for the specific language governing permissions and limitations under the License
#include "Types.h"

51
internal/core/src/index/knowhere/utils/Types.h
View File

@ -1,51 +0,0 @@
// Copyright (C) 2019-2020 Zilliz. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software distributed under the License
// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
// or implied. See the License for the specific language governing permissions and limitations under the License.
#pragma once
// #include <faiss/Index.h>
#include <cstdint>
#include <map>
#include <memory>
#include <set>
#include <string>
#include <unordered_map>
#include <utility>
#include <vector>
namespace milvus::engine {
using idx_t = int64_t;
using offset_t = int32_t;
using date_t = int32_t;
using distance_t = float;
using IDNumbers = std::vector<idx_t>;
enum class DataType {
NONE = 0,
BOOL = 1,
INT8 = 2,
INT16 = 3,
INT32 = 4,
INT64 = 5,
FLOAT = 10,
DOUBLE = 11,
STRING = 20,
VECTOR_BINARY = 100,
VECTOR_FLOAT = 101,
};
} // namespace milvus::engine

30
internal/core/src/index/knowhere/utils/tools.h
View File

@ -1,30 +0,0 @@
// Copyright (C) 2019-2020 Zilliz. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software distributed under the License
// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
// or implied. See the License for the specific language governing permissions and limitations under the License
#pragma once
#include <cstdint>
#include "exceptions/EasyAssert.h"
namespace milvus {
inline int64_t
upper_align(int64_t value, int64_t align) {
Assert(align > 0);
auto groups = (value + align - 1) / align;
return groups * align;
}
inline int64_t
upper_div(int64_t value, int64_t align) {
Assert(align > 0);
auto groups = (value + align - 1) / align;
return groups;
}
} // namespace milvus

95
internal/core/src/utils/BlockingQueue.h
View File

@ -1,95 +0,0 @@
// Copyright (C) 2019-2020 Zilliz. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software distributed under the License
// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
// or implied. See the License for the specific language governing permissions and limitations under the License.
#pragma once
#include <assert.h>
#include <condition_variable>
#include <iostream>
#include <queue>
#include <vector>
namespace milvus {
template <typename T>
class BlockingQueue {
public:
BlockingQueue() : mtx(), full_(), empty_() {
}
virtual ~BlockingQueue() {
}
BlockingQueue(const BlockingQueue& rhs) = delete;
BlockingQueue&
operator=(const BlockingQueue& rhs) = delete;
void
Put(const T& task) {
std::unique_lock<std::mutex> lock(mtx);
full_.wait(lock, [this] { return (queue_.size() < capacity_); });
queue_.push(task);
empty_.notify_all();
}
T
Take() {
std::unique_lock<std::mutex> lock(mtx);
empty_.wait(lock, [this] { return !queue_.empty(); });
T front(queue_.front());
queue_.pop();
full_.notify_all();
return front;
}
T
Front() {
std::unique_lock<std::mutex> lock(mtx);
empty_.wait(lock, [this] { return !queue_.empty(); });
T front(queue_.front());
return front;
}
T
Back() {
std::unique_lock<std::mutex> lock(mtx);
empty_.wait(lock, [this] { return !queue_.empty(); });
T back(queue_.back());
return back;
}
size_t
Size() const {
std::lock_guard<std::mutex> lock(mtx);
return queue_.size();
}
bool
Empty() const {
std::unique_lock<std::mutex> lock(mtx);
return queue_.empty();
}
void
SetCapacity(const size_t capacity) {
capacity_ = (capacity > 0 ? capacity : capacity_);
}
protected:
mutable std::mutex mtx;
std::condition_variable full_;
std::condition_variable empty_;
std::queue<T> queue_;
size_t capacity_ = 32;
};
} // namespace milvus

243
internal/core/src/utils/CommonUtil.cpp
View File

@ -1,243 +0,0 @@
// Copyright (C) 2019-2020 Zilliz. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software distributed under the License
// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
// or implied. See the License for the specific language governing permissions and limitations under the License.
#include "utils/CommonUtil.h"
#include "utils/Log.h"
#include <dirent.h>
#include <fiu/fiu-local.h>
#include <pwd.h>
#include <sys/stat.h>
#include <unistd.h>
#include <boost/filesystem.hpp>
#include <iostream>
#include <vector>
namespace milvus {
namespace fs = boost::filesystem;
bool
CommonUtil::IsDirectoryExist(const std::string& path) {
DIR* dp = nullptr;
if ((dp = opendir(path.c_str())) == nullptr) {
return false;
}
closedir(dp);
return true;
}
Status
CommonUtil::CreateDirectory(const std::string& path) {
if (path.empty()) {
return Status::OK();
}
struct stat directory_stat;
int status = stat(path.c_str(), &directory_stat);
if (status == 0) {
return Status::OK(); // already exist
}
fs::path fs_path(path);
fs::path parent_path = fs_path.parent_path();
Status err_status = CreateDirectory(parent_path.string());
fiu_do_on("CommonUtil.CreateDirectory.create_parent_fail", err_status = Status(SERVER_INVALID_ARGUMENT, ""));
if (!err_status.ok()) {
return err_status;
}
status = stat(path.c_str(), &directory_stat);
if (status == 0) {
return Status::OK(); // already exist
}
int makeOK = mkdir(path.c_str(), S_IRWXU | S_IRGRP | S_IROTH);
fiu_do_on("CommonUtil.CreateDirectory.create_dir_fail", makeOK = 1);
if (makeOK != 0) {
return Status(SERVER_UNEXPECTED_ERROR, "failed to create directory: " + path);
}
return Status::OK();
}
namespace {
void
RemoveDirectory(const std::string& path) {
DIR* dir = nullptr;
const int32_t buf_size = 256;
char file_name[buf_size];
std::string folder_name = path + "/%s";
if ((dir = opendir(path.c_str())) != nullptr) {
struct dirent* dmsg;
while ((dmsg = readdir(dir)) != nullptr) {
if (strcmp(dmsg->d_name, ".") != 0 && strcmp(dmsg->d_name, "..") != 0) {
snprintf(file_name, buf_size, folder_name.c_str(), dmsg->d_name);
std::string tmp = file_name;
if (tmp.find('.') == std::string::npos) {
RemoveDirectory(file_name);
}
remove(file_name);
}
}
}
if (dir != nullptr) {
closedir(dir);
}
remove(path.c_str());
}
} // namespace
Status
CommonUtil::DeleteDirectory(const std::string& path) {
if (path.empty()) {
return Status::OK();
}
struct stat directory_stat;
int statOK = stat(path.c_str(), &directory_stat);
if (statOK != 0) {
return Status::OK();
}
RemoveDirectory(path);
return Status::OK();
}
bool
CommonUtil::IsFileExist(const std::string& path) {
return (access(path.c_str(), F_OK) == 0);
}
uint64_t
CommonUtil::GetFileSize(const std::string& path) {
struct stat file_info;
if (stat(path.c_str(), &file_info) < 0) {
return 0;
}
return static_cast<uint64_t>(file_info.st_size);
}
std::string
CommonUtil::GetFileName(std::string filename) {
int pos = filename.find_last_of('/');
return filename.substr(pos + 1);
}
std::string
CommonUtil::GetExePath() {
const int64_t buf_len = 1024;
char buf[buf_len];
int64_t cnt = readlink("/proc/self/exe", buf, buf_len);
fiu_do_on("CommonUtil.GetExePath.readlink_fail", cnt = -1);
if (cnt < 0 || cnt >= buf_len) {
return "";
}
buf[cnt] = '\0';
std::string exe_path = buf;
fiu_do_on("CommonUtil.GetExePath.exe_path_error", exe_path = "/");
if (exe_path.rfind('/') != exe_path.length() - 1) {
std::string sub_str = exe_path.substr(0, exe_path.rfind('/'));
return sub_str + "/";
}
return exe_path;
}
bool
CommonUtil::TimeStrToTime(const std::string& time_str,
time_t& time_integer,
tm& time_struct,
const std::string& format) {
time_integer = 0;
memset(&time_struct, 0, sizeof(tm));
int ret = sscanf(time_str.c_str(), format.c_str(), &(time_struct.tm_year), &(time_struct.tm_mon),
&(time_struct.tm_mday), &(time_struct.tm_hour), &(time_struct.tm_min), &(time_struct.tm_sec));
if (ret <= 0) {
return false;
}
time_struct.tm_year -= 1900;
time_struct.tm_mon--;
time_integer = mktime(&time_struct);
return true;
}
void
CommonUtil::GetCurrentTimeStr(std::string& time_str) {
auto t = std::time(nullptr);
struct tm ltm;
localtime_r(&t, &ltm);
time_str = "";
time_str += std::to_string(ltm.tm_year + 1900);
time_str += "-";
time_str += std::to_string(ltm.tm_mon + 1);
time_str += "-";
time_str += std::to_string(ltm.tm_mday);
time_str += "_";
time_str += std::to_string(ltm.tm_hour);
time_str += ":";
time_str += std::to_string(ltm.tm_min);
time_str += ":";
time_str += std::to_string(ltm.tm_sec);
}
void
CommonUtil::ConvertTime(time_t time_integer, tm& time_struct) {
localtime_r(&time_integer, &time_struct);
}
void
CommonUtil::ConvertTime(tm time_struct, time_t& time_integer) {
time_integer = mktime(&time_struct);
}
std::string
CommonUtil::ConvertSize(int64_t size) {
const int64_t gb = 1024ll * 1024 * 1024;
const int64_t mb = 1024ll * 1024;
const int64_t kb = 1024ll;
if (size % gb == 0) {
return std::to_string(size / gb) + "GB";
} else if (size % mb == 0) {
return std::to_string(size / mb) + "MB";
} else if (size % kb == 0) {
return std::to_string(size / kb) + "KB";
} else {
return std::to_string(size);
}
}
#ifdef ENABLE_CPU_PROFILING
std::string
CommonUtil::GetCurrentTimeStr() {
time_t tt;
time(&tt);
tt = tt + 8 * 60;
tm t;
gmtime_r(&tt, &t);
std::string str = std::to_string(t.tm_year + 1900) + "_" + std::to_string(t.tm_mon + 1) + "_" +
std::to_string(t.tm_mday) + "_" + std::to_string(t.tm_hour) + "_" + std::to_string(t.tm_min) +
"_" + std::to_string(t.tm_sec);
return str;
}
#endif
} // namespace milvus

62
internal/core/src/utils/CommonUtil.h
View File

@ -1,62 +0,0 @@
// Copyright (C) 2019-2020 Zilliz. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software distributed under the License
// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
// or implied. See the License for the specific language governing permissions and limitations under the License.
#pragma once
#include "utils/Status.h"
#include <time.h>
#include <string>
namespace milvus {
class CommonUtil {
public:
static bool
IsFileExist(const std::string& path);
static uint64_t
GetFileSize(const std::string& path);
static bool
IsDirectoryExist(const std::string& path);
static Status
CreateDirectory(const std::string& path);
static Status
DeleteDirectory(const std::string& path);
static std::string
GetFileName(std::string filename);
static std::string
GetExePath();
static bool
TimeStrToTime(const std::string& time_str,
time_t& time_integer,
tm& time_struct,
const std::string& format = "%d-%d-%d %d:%d:%d");
static void
GetCurrentTimeStr(std::string& time_str);
static void
ConvertTime(time_t time_integer, tm& time_struct);
static void
ConvertTime(tm time_struct, time_t& time_integer);
static std::string
ConvertSize(int64_t size);
#ifdef ENABLE_CPU_PROFILING
static std::string
GetCurrentTimeStr();
#endif
};
} // namespace milvus

320
internal/core/src/utils/ConfigUtils.cpp
View File

@ -1,320 +0,0 @@
// Copyright (C) 2019-2020 Zilliz. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software distributed under the License
// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
// or implied. See the License for the specific language governing permissions and limitations under the License.
#include "utils/ConfigUtils.h"
#include "utils/Log.h"
#include "utils/StringHelpFunctions.h"
#include <arpa/inet.h>
#include <algorithm>
#include <cmath>
#ifdef MILVUS_GPU_VERSION
#include <cuda_runtime.h>
#endif
#include <fiu/fiu-local.h>
#include <sys/stat.h>
#include <sys/sysinfo.h>
#include <unistd.h>
#include <limits>
#include <regex>
#include <set>
#include <unordered_map>
#if defined(__x86_64__)
#define THREAD_MULTIPLY_CPU 1
#elif defined(__powerpc64__)
#define THREAD_MULTIPLY_CPU 4
#else
#define THREAD_MULTIPLY_CPU 1
#endif
namespace milvus {
namespace server {
std::unordered_map<std::string, int64_t> BYTE_UNITS = {
{"b", 1},
{"k", 1024},
{"m", 1024 * 1024},
{"g", 1024 * 1024 * 1024},
};
bool
is_number(const std::string& s) {
return !s.empty() && std::find_if(s.begin(), s.end(), [](unsigned char c) { return !std::isdigit(c); }) == s.end();
}
bool
is_alpha(const std::string& s) {
return !s.empty() && std::find_if(s.begin(), s.end(), [](unsigned char c) { return !std::isalpha(c); }) == s.end();
}
std::string
str_tolower(std::string s) {
std::transform(s.begin(), s.end(), s.begin(), [](unsigned char c) { return std::tolower(c); });
return s;
}
int64_t
parse_bytes(const std::string& str, std::string& err) {
try {
std::string s = str;
if (is_number(s)) {
return std::stoll(s);
}
if (s.length() == 0) {
return 0;
}
auto last_two = s.substr(s.length() - 2, 2);
auto last_one = s.substr(s.length() - 1);
if (is_alpha(last_two) && is_alpha(last_one)) {
if (last_one == "b" or last_one == "B") {
s = s.substr(0, s.length() - 1);
}
}
auto& units = BYTE_UNITS;
auto suffix = str_tolower(s.substr(s.length() - 1));
std::string digits_part;
if (is_number(suffix)) {
digits_part = s;
suffix = 'b';
} else {
digits_part = s.substr(0, s.length() - 1);
}
if (units.find(suffix) != units.end() or is_number(suffix)) {
auto digits = std::stoll(digits_part);
return digits * units[suffix];
} else {
std::stringstream ss;
ss << "The specified value for memory (" << str << ") should specify the units."
<< "The postfix should be one of the `b` `k` `m` `g` characters";
err = ss.str();
}
} catch (...) {
err = "Unknown error happened on parse bytes.";
}
return 0;
}
bool
GetSystemMemInfo(int64_t& total_mem, int64_t& free_mem) {
struct sysinfo info;
int ret = sysinfo(&info);
total_mem = info.totalram;
free_mem = info.freeram;
return ret == 0; // succeed 0, failed -1
}
bool
GetSystemAvailableThreads(int64_t& thread_count) {
// threadCnt = std::thread::hardware_concurrency();
thread_count = sysconf(_SC_NPROCESSORS_CONF);
thread_count *= THREAD_MULTIPLY_CPU;
fiu_do_on("GetSystemAvailableThreads.zero_thread", thread_count = 0);
if (thread_count == 0) {
thread_count = 8;
}
return true;
}
Status
ValidateGpuIndex(int32_t gpu_index) {
#ifdef MILVUS_GPU_VERSION
int num_devices = 0;
auto cuda_err = cudaGetDeviceCount(&num_devices);
fiu_do_on("config.ValidateGpuIndex.get_device_count_fail", cuda_err = cudaError::cudaErrorUnknown);
if (cuda_err != cudaSuccess) {
std::string msg = "Failed to get gpu card number, cuda error:" + std::to_string(cuda_err);
LOG_SERVER_ERROR_ << msg;
return Status(SERVER_UNEXPECTED_ERROR, msg);
}
if (gpu_index >= num_devices) {
std::string msg = "Invalid gpu index: " + std::to_string(gpu_index);
LOG_SERVER_ERROR_ << msg;
return Status(SERVER_INVALID_ARGUMENT, msg);
}
#endif
return Status::OK();
}
#ifdef MILVUS_GPU_VERSION
Status
GetGpuMemory(int32_t gpu_index, int64_t& memory) {
fiu_return_on("config.GetGpuMemory.return_error", Status(SERVER_UNEXPECTED_ERROR, ""));
cudaDeviceProp deviceProp;
auto cuda_err = cudaGetDeviceProperties(&deviceProp, gpu_index);
if (cuda_err) {
std::string msg = "Failed to get gpu properties for gpu" + std::to_string(gpu_index) +
" , cuda error:" + std::to_string(cuda_err);
LOG_SERVER_ERROR_ << msg;
return Status(SERVER_UNEXPECTED_ERROR, msg);
}
memory = deviceProp.totalGlobalMem;
return Status::OK();
}
#endif
Status
ValidateIpAddress(const std::string& ip_address) {
struct in_addr address;
int result = inet_pton(AF_INET, ip_address.c_str(), &address);
fiu_do_on("config.ValidateIpAddress.error_ip_result", result = 2);
switch (result) {
case 1:
return Status::OK();
case 0: {
std::string msg = "Invalid IP address: " + ip_address;
LOG_SERVER_ERROR_ << msg;
return Status(SERVER_INVALID_ARGUMENT, msg);
}
default: {
std::string msg = "IP address conversion error: " + ip_address;
LOG_SERVER_ERROR_ << msg;
return Status(SERVER_UNEXPECTED_ERROR, msg);
}
}
}
Status
ValidateStringIsNumber(const std::string& str) {
if (str.empty() || !std::all_of(str.begin(), str.end(), ::isdigit)) {
return Status(SERVER_INVALID_ARGUMENT, "Invalid number");
}
try {
int64_t value = std::stol(str);
fiu_do_on("config.ValidateStringIsNumber.throw_exception", throw std::exception());
if (value < 0) {
return Status(SERVER_INVALID_ARGUMENT, "Negative number");
}
} catch (...) {
return Status(SERVER_INVALID_ARGUMENT, "Invalid number");
}
return Status::OK();
}
Status
ValidateStringIsBool(const std::string& str) {
fiu_return_on("ValidateStringNotBool", Status(SERVER_INVALID_ARGUMENT, "Invalid boolean: " + str));
std::string s = str;
std::transform(s.begin(), s.end(), s.begin(), ::tolower);
if (s == "true" || s == "on" || s == "yes" || s == "1" || s == "false" || s == "off" || s == "no" || s == "0" ||
s.empty()) {
return Status::OK();
}
return Status(SERVER_INVALID_ARGUMENT, "Invalid boolean: " + str);
}
Status
ValidateStringIsFloat(const std::string& str) {
try {
float val = std::stof(str);
if (val < 0.0) {
return Status(SERVER_INVALID_ARGUMENT, "Negative float: " + str);
}
} catch (...) {
return Status(SERVER_INVALID_ARGUMENT, "Invalid float: " + str);
}
return Status::OK();
}
Status
ValidateDbURI(const std::string& uri) {
std::string dialectRegex = "(.*)";
std::string usernameRegex = "(.*)";
std::string passwordRegex = "(.*)";
std::string hostRegex = "(.*)";
std::string portRegex = "(.*)";
std::string dbNameRegex = "(.*)";
std::string uriRegexStr = dialectRegex + R"(\:\/\/)" + usernameRegex + R"(\:)" + passwordRegex + R"(\@)" +
hostRegex + R"(\:)" + portRegex + R"(\/)" + dbNameRegex;
std::regex uriRegex(uriRegexStr);
std::smatch pieces_match;
bool okay = true;
if (std::regex_match(uri, pieces_match, uriRegex)) {
std::string dialect = pieces_match[1].str();
std::transform(dialect.begin(), dialect.end(), dialect.begin(), ::tolower);
if (dialect.find("mysql") == std::string::npos && dialect.find("sqlite") == std::string::npos &&
dialect.find("mock") == std::string::npos) {
LOG_SERVER_ERROR_ << "Invalid dialect in URI: dialect = " << dialect;
okay = false;
}
/*
* Could be DNS, skip checking
*
std::string host = pieces_match[4].str();
if (!host.empty() && host != "localhost") {
if (ValidateIpAddress(host) != SERVER_SUCCESS) {
LOG_SERVER_ERROR_ << "Invalid host ip address in uri = " << host;
okay = false;
}
}
*/
std::string port = pieces_match[5].str();
if (!port.empty()) {
auto status = ValidateStringIsNumber(port);
if (!status.ok()) {
LOG_SERVER_ERROR_ << "Invalid port in uri = " << port;
okay = false;
}
}
} else {
LOG_SERVER_ERROR_ << "Wrong URI format: URI = " << uri;
okay = false;
}
return (okay ? Status::OK() : Status(SERVER_INVALID_ARGUMENT, "Invalid db backend uri"));
}
Status
ValidateStoragePath(const std::string& path) {
// Validate storage path if is valid, only correct absolute path will be validated pass
// Invalid path only contain character[a-zA-Z], number[0-9], '-', and '_',
// and path must start with '/'.
// examples below are invalid
// '/a//a', '/a--/a', '/-a/a', '/a@#/a', 'aaa/sfs'
std::string path_pattern = R"(^\/(\w+-?\/?)+$)";
std::regex regex(path_pattern);
return std::regex_match(path, regex) ? Status::OK() : Status(SERVER_INVALID_ARGUMENT, "Invalid file path");
}
Status
ValidateLogLevel(const std::string& level) {
std::set<std::string> supported_level{"debug", "info", "warning", "error", "fatal"};
return supported_level.find(level) != supported_level.end()
? Status::OK()
: Status(SERVER_INVALID_ARGUMENT, "Log level must be one of debug, info, warning, error and fatal.");
}
bool
IsNumber(const std::string& s) {
return !s.empty() && std::all_of(s.begin(), s.end(), ::isdigit);
}
} // namespace server
} // namespace milvus

63
internal/core/src/utils/ConfigUtils.h
View File

@ -1,63 +0,0 @@
// Copyright (C) 2019-2020 Zilliz. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software distributed under the License
// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
// or implied. See the License for the specific language governing permissions and limitations under the License.
#pragma once
#include <string>
#include <vector>
#include "utils/Status.h"
namespace milvus {
namespace server {
extern int64_t
parse_bytes(const std::string& str, std::string& err);
extern bool
GetSystemMemInfo(int64_t& total_mem, int64_t& free_mem);
extern bool
GetSystemAvailableThreads(int64_t& thread_count);
extern Status
ValidateGpuIndex(int32_t gpu_index);
#ifdef MILVUS_GPU_VERSION
extern Status
GetGpuMemory(int32_t gpu_index, int64_t& memory);
#endif
extern Status
ValidateIpAddress(const std::string& ip_address);
extern Status
ValidateStringIsNumber(const std::string& str);
extern Status
ValidateStringIsBool(const std::string& str);
extern Status
ValidateStringIsFloat(const std::string& str);
extern Status
ValidateDbURI(const std::string& uri);
extern Status
ValidateStoragePath(const std::string& path);
extern Status
ValidateLogLevel(const std::string& level);
extern bool
IsNumber(const std::string& s);
} // namespace server
} // namespace milvus

61
internal/core/src/utils/Exception.h
View File

@ -1,61 +0,0 @@
// Copyright (C) 2019-2020 Zilliz. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software distributed under the License
// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
// or implied. See the License for the specific language governing permissions and limitations under the License.
#pragma once
#include "utils/Error.h"
#include <exception>
#include <string>
#include <utility>
namespace milvus {
#define THROW_ERROR(err_code, err_msg) \
LOG_ENGINE_ERROR_ << err_msg; \
throw Exception(err_code, err_msg);
class Exception : public std::exception {
public:
Exception(ErrorCode code, std::string msg) : code_(code), message_(std::move(msg)) {
}
ErrorCode
code() const noexcept {
return code_;
}
const char*
what() const noexcept override {
if (message_.empty()) {
return "Default Exception.";
} else {
return message_.c_str();
}
}
~Exception() noexcept override = default;
protected:
ErrorCode code_;
std::string message_;
};
class InvalidArgumentException : public Exception {
public:
InvalidArgumentException() : Exception(SERVER_INVALID_ARGUMENT, "Invalid Argument") {
}
explicit InvalidArgumentException(const std::string& message) : Exception(SERVER_INVALID_ARGUMENT, message) {
}
};
} // namespace milvus

60
internal/core/src/utils/SignalHandler.cpp
View File

@ -1,60 +0,0 @@
// Copyright (C) 2019-2020 Zilliz. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software distributed under the License
// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
// or implied. See the License for the specific language governing permissions and limitations under the License.
#include "utils/SignalHandler.h"
#include "utils/Log.h"
#include <execinfo.h>
#include <string>
namespace milvus {
signal_func_ptr signal_routine_func = nullptr;
void
HandleSignal(int signum) {
int32_t exit_code = 1; /* 0: normal exit; 1: exception */
switch (signum) {
case SIGINT:
case SIGUSR2:
exit_code = 0;
/* no break */
default: {
if (exit_code == 0) {
LOG_SERVER_INFO_ << "Server received signal: " << signum;
} else {
LOG_SERVER_INFO_ << "Server received critical signal: " << signum;
PrintStacktrace();
}
if (signal_routine_func != nullptr) {
(*signal_routine_func)(exit_code);
}
}
}
}
void
PrintStacktrace() {
const int bt_depth = 128;
void* array[bt_depth];
int stack_num = backtrace(array, bt_depth);
char** stacktrace = backtrace_symbols(array, stack_num);
LOG_SERVER_INFO_ << "Call stack:";
for (int i = 0; i < stack_num; ++i) {
std::string info = stacktrace[i];
std::cout << "No." << i << ": " << info << std::endl;
LOG_SERVER_INFO_ << info;
}
free(stacktrace);
}
} // namespace milvus

28
internal/core/src/utils/SignalHandler.h
View File

@ -1,28 +0,0 @@
// Copyright (C) 2019-2020 Zilliz. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software distributed under the License
// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
// or implied. See the License for the specific language governing permissions and limitations under the License.
#pragma once
#include <cstdint>
namespace milvus {
typedef void (*signal_func_ptr)(int32_t);
extern signal_func_ptr signal_routine_func;
extern void
HandleSignal(int signum);
extern void
PrintStacktrace();
} // namespace milvus

2
internal/core/src/utils/Status.h
View File

@ -11,7 +11,7 @@
#pragma once
#include "utils/Error.h"
#include "Error.h"
#include <string>

175
internal/core/src/utils/StringHelpFunctions.cpp
View File

@ -1,175 +0,0 @@
// Copyright (C) 2019-2020 Zilliz. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software distributed under the License
// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
// or implied. See the License for the specific language governing permissions and limitations under the License.
#include "utils/StringHelpFunctions.h"
#include "utils/Log.h"
#include <fiu/fiu-local.h>
#include <algorithm>
#include <regex>
#include <string>
namespace milvus {
void
StringHelpFunctions::TrimStringBlank(std::string& string) {
if (!string.empty()) {
static std::string s_format(" \n\r\t");
string.erase(0, string.find_first_not_of(s_format));
string.erase(string.find_last_not_of(s_format) + 1);
}
}
void
StringHelpFunctions::TrimStringQuote(std::string& string, const std::string& qoute) {
if (!string.empty()) {
string.erase(0, string.find_first_not_of(qoute));
string.erase(string.find_last_not_of(qoute) + 1);
}
}
void
StringHelpFunctions::SplitStringByDelimeter(const std::string& str,
const std::string& delimeter,
std::vector<std::string>& result) {
if (str.empty()) {
return;
}
size_t prev = 0;
while (true) {
size_t pos = str.find_first_of(delimeter, prev);
if (pos == std::string::npos) {
result.emplace_back(str.substr(prev));
break;
} else {
result.emplace_back(str.substr(prev, pos - prev));
prev = pos + 1;
}
}
}
void
StringHelpFunctions::MergeStringWithDelimeter(const std::vector<std::string>& strs,
const std::string& delimeter,
std::string& result) {
if (strs.empty()) {
result = "";
return;
}
result = strs[0];
for (size_t i = 1; i < strs.size(); i++) {
result = result + delimeter + strs[i];
}
}
Status
StringHelpFunctions::SplitStringByQuote(const std::string& str,
const std::string& delimeter,
const std::string& quote,
std::vector<std::string>& result) {
if (quote.empty()) {
SplitStringByDelimeter(str, delimeter, result);
return Status::OK();
}
size_t last = 0;
size_t index = str.find_first_of(quote, last);
if (index == std::string::npos) {
SplitStringByDelimeter(str, delimeter, result);
return Status::OK();
}
std::string process_str = str;
while (index != std::string::npos) {
std::string prefix = process_str.substr(last, index - last);
std::string append_prefix;
if (!prefix.empty()) {
std::vector<std::string> prefix_split;
SplitStringByDelimeter(prefix, delimeter, prefix_split);
for (size_t i = 0; i < prefix_split.size() - 1; i++) {
result.push_back(prefix_split[i]);
}
append_prefix = prefix_split[prefix_split.size() - 1];
}
last = index + 1;
std::string postfix = process_str.substr(last);
index = postfix.find_first_of(quote, 0);
fiu_do_on("StringHelpFunctions.SplitStringByQuote.invalid_index", index = std::string::npos);
if (index == std::string::npos) {
return Status(SERVER_UNEXPECTED_ERROR, "");
}
std::string quoted_text = postfix.substr(0, index);
append_prefix += quoted_text;
last = index + 1;
index = postfix.find_first_of(delimeter, last);
fiu_do_on("StringHelpFunctions.SplitStringByQuote.index_gt_last", last = 0);
fiu_do_on("StringHelpFunctions.SplitStringByQuote.invalid_index2", index = std::string::npos);
if (index != std::string::npos) {
if (index > last) {
append_prefix += postfix.substr(last, index - last);
}
} else {
append_prefix += postfix.substr(last);
}
result.emplace_back(append_prefix);
fiu_do_on("StringHelpFunctions.SplitStringByQuote.last_is_end", last = postfix.length());
if (last == postfix.length()) {
return Status::OK();
}
process_str = postfix.substr(index + 1);
last = 0;
index = process_str.find_first_of(quote, last);
}
if (!process_str.empty()) {
SplitStringByDelimeter(process_str, delimeter, result);
}
return Status::OK();
}
bool
StringHelpFunctions::IsRegexMatch(const std::string& target_str, const std::string& pattern_str) {
// if target_str equals pattern_str, return true
if (target_str == pattern_str) {
return true;
}
// regex match
// for illegal regex expression, the std::regex will throw exception, regard as unmatch
try {
std::regex pattern(pattern_str);
std::smatch results;
return std::regex_match(target_str, results, pattern);
} catch (std::exception& e) {
LOG_SERVER_ERROR_ << "Regex exception: " << e.what();
}
return false;
}
Status
StringHelpFunctions::ConvertToBoolean(const std::string& str, bool& value) {
std::string s = str;
std::transform(s.begin(), s.end(), s.begin(), ::tolower);
value = s == "true" || s == "on" || s == "yes" || s == "1";
return Status::OK();
}
} // namespace milvus

76
internal/core/src/utils/StringHelpFunctions.h
View File

@ -1,76 +0,0 @@
// Copyright (C) 2019-2020 Zilliz. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software distributed under the License
// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
// or implied. See the License for the specific language governing permissions and limitations under the License.
#pragma once
#include "utils/Status.h"
#include <string>
#include <vector>
namespace milvus {
class StringHelpFunctions {
private:
StringHelpFunctions() = default;
public:
// trim blanks from begin and end
// " a b c " => "a b c"
static void
TrimStringBlank(std::string& string);
// trim quotes from begin and end
// "'abc'" => "abc"
static void
TrimStringQuote(std::string& string, const std::string& qoute);
// split string by delimeter ','
// a,b,c a | b | c
// a,b, a | b |
// ,b,c | b | c
// ,b, | b |
// ,, | |
// a a
static void
SplitStringByDelimeter(const std::string& str, const std::string& delimeter, std::vector<std::string>& result);
// merge strings with delimeter
// "a", "b", "c" => "a,b,c"
static void
MergeStringWithDelimeter(const std::vector<std::string>& strs, const std::string& delimeter, std::string& result);
// assume the collection has two columns, quote='\"', delimeter=','
// a,b a | b
// "aa,gg,yy",b aa,gg,yy | b
// aa"dd,rr"kk,pp aadd,rrkk | pp
// "aa,bb" aa,bb
// 55,1122\"aa,bb\",yyy,\"kkk\" 55 | 1122aa,bb | yyy | kkk
// "55,1122"aa,bb",yyy,"kkk" illegal
static Status
SplitStringByQuote(const std::string& str,
const std::string& delimeter,
const std::string& quote,
std::vector<std::string>& result);
// std regex match function
// regex grammar reference: http://www.cplusplus.com/reference/regex/ECMAScript/
static bool
IsRegexMatch(const std::string& target_str, const std::string& pattern);
// conversion rules refer to ValidationUtil::ValidateStringIsBool()
// "true", "on", "yes", "1" ==> true
// "false", "off", "no", "0", "" ==> false
static Status
ConvertToBoolean(const std::string& str, bool& value);
};
} // namespace milvus

128
internal/core/src/utils/ThreadPool.h
View File

@ -1,128 +0,0 @@
// Copyright (C) 2019-2020 Zilliz. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software distributed under the License
// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
// or implied. See the License for the specific language governing permissions and limitations under the License.
#pragma once
#include <fiu/fiu-local.h>
#include <atomic>
#include <condition_variable>
#include <functional>
#include <future>
#include <memory>
#include <mutex>
#include <queue>
#include <stdexcept>
#include <thread>
#include <utility>
#include <vector>
#define MAX_THREADS_NUM 32
namespace milvus {
class ThreadPool {
public:
explicit ThreadPool(size_t threads, size_t queue_size = 1000);
template <class F, class... Args>
auto
enqueue(F&& f, Args&&... args) -> std::future<typename std::result_of<F(Args...)>::type>;
void
Stop();
~ThreadPool();
private:
// need to keep track of threads so we can join them
std::vector<std::thread> workers_;
// the task queue
std::queue<std::function<void()>> tasks_;
size_t max_queue_size_;
// synchronization
std::mutex queue_mutex_;
std::condition_variable condition_;
std::atomic_bool stop_;
};
// the constructor just launches some amount of workers
inline ThreadPool::ThreadPool(size_t threads, size_t queue_size) : max_queue_size_(queue_size), stop_(false) {
for (size_t i = 0; i < threads; ++i)
workers_.emplace_back([this] {
for (;;) {
std::function<void()> task;
{
std::unique_lock<std::mutex> lock(this->queue_mutex_);
this->condition_.wait(lock, [this] { return this->stop_ || !this->tasks_.empty(); });
if (this->stop_ && this->tasks_.empty())
return;
task = std::move(this->tasks_.front());
this->tasks_.pop();
}
this->condition_.notify_all();
task();
}
});
}
// add new work item to the pool
template <class F, class... Args>
auto
ThreadPool::enqueue(F&& f, Args&&... args) -> std::future<typename std::result_of<F(Args...)>::type> {
using return_type = typename std::result_of<F(Args...)>::type;
auto task =
std::make_shared<std::packaged_task<return_type()>>(std::bind(std::forward<F>(f), std::forward<Args>(args)...));
fiu_do_on("ThreadPool.enqueue.stop_is_true", stop_ = true);
std::future<return_type> res = task->get_future();
{
std::unique_lock<std::mutex> lock(queue_mutex_);
this->condition_.wait(lock, [this] { return this->tasks_.size() < max_queue_size_; });
// don't allow enqueueing after stopping the pool
if (stop_)
throw std::runtime_error("enqueue on stopped ThreadPool");
tasks_.emplace([task]() { (*task)(); });
}
condition_.notify_all();
return res;
}
inline void
ThreadPool::Stop() {
if (stop_) {
return;
}
{
std::unique_lock<std::mutex> lock(queue_mutex_);
stop_ = true;
}
condition_.notify_all();
for (std::thread& worker : workers_) {
worker.join();
}
}
// the destructor joins all threads
inline ThreadPool::~ThreadPool() {
Stop();
}
using ThreadPoolPtr = std::shared_ptr<ThreadPool>;
} // namespace milvus

93
internal/core/src/utils/TimeRecorder.cpp
View File

@ -1,93 +0,0 @@
// Copyright (C) 2019-2020 Zilliz. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software distributed under the License
// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
// or implied. See the License for the specific language governing permissions and limitations under the License.
#include <utility>
#include "utils/Log.h"
#include "utils/TimeRecorder.h"
namespace milvus {
TimeRecorder::TimeRecorder(std::string hdr, int64_t log_level) : header_(std::move(hdr)), log_level_(log_level) {
start_ = last_ = stdclock::now();
}
std::string
TimeRecorder::GetTimeSpanStr(double span) {
std::string str_sec = std::to_string(span * 0.000001) + ((span > 1000000) ? " seconds" : " second");
std::string str_ms = std::to_string(span * 0.001) + " ms";
return str_sec + " [" + str_ms + "]";
}
void
TimeRecorder::PrintTimeRecord(const std::string& msg, double span) {
std::string str_log;
if (!header_.empty()) {
str_log += header_ + ": ";
}
str_log += msg;
str_log += " (";
str_log += TimeRecorder::GetTimeSpanStr(span);
str_log += ")";
switch (log_level_) {
case 0:
LOG_SERVER_TRACE_ << str_log;
break;
case 1:
LOG_SERVER_DEBUG_ << str_log;
break;
case 2:
LOG_SERVER_INFO_ << str_log;
break;
case 3:
LOG_SERVER_WARNING_ << str_log;
break;
case 4:
LOG_SERVER_ERROR_ << str_log;
break;
case 5:
LOG_SERVER_FATAL_ << str_log;
break;
default:
LOG_SERVER_INFO_ << str_log;
break;
}
}
double
TimeRecorder::RecordSection(const std::string& msg) {
stdclock::time_point curr = stdclock::now();
double span = (std::chrono::duration<double, std::micro>(curr - last_)).count();
last_ = curr;
PrintTimeRecord(msg, span);
return span;
}
double
TimeRecorder::ElapseFromBegin(const std::string& msg) {
stdclock::time_point curr = stdclock::now();
double span = (std::chrono::duration<double, std::micro>(curr - start_)).count();
PrintTimeRecord(msg, span);
return span;
}
TimeRecorderAuto::TimeRecorderAuto(std::string hdr, int64_t log_level) : TimeRecorder(hdr, log_level) {
}
TimeRecorderAuto::~TimeRecorderAuto() {
ElapseFromBegin("totally cost");
}
} // namespace milvus

65
internal/core/src/utils/TimeRecorder.h
View File

@ -1,65 +0,0 @@
// Copyright (C) 2019-2020 Zilliz. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software distributed under the License
// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
// or implied. See the License for the specific language governing permissions and limitations under the License.
#pragma once
#include <chrono>
#include <string>
#include "utils/Log.h"
namespace milvus {
inline void
print_timestamp(const std::string& message) {
std::chrono::time_point<std::chrono::system_clock> now = std::chrono::system_clock::now();
auto duration = now.time_since_epoch();
auto micros = std::chrono::duration_cast<std::chrono::microseconds>(duration).count();
micros %= 1000000;
double millisecond = (double)micros / 1000.0;
LOG_SERVER_DEBUG_ << std::fixed << " " << millisecond << "(ms) [timestamp]" << message;
}
class TimeRecorder {
using stdclock = std::chrono::high_resolution_clock;
public:
// trace = 0, debug = 1, info = 2, warn = 3, error = 4, critical = 5
explicit TimeRecorder(std::string hdr, int64_t log_level = 1);
virtual ~TimeRecorder() = default;
double
RecordSection(const std::string& msg);
double
ElapseFromBegin(const std::string& msg);
static std::string
GetTimeSpanStr(double span);
private:
void
PrintTimeRecord(const std::string& msg, double span);
private:
std::string header_;
stdclock::time_point start_;
stdclock::time_point last_;
int64_t log_level_;
};
class TimeRecorderAuto : public TimeRecorder {
public:
explicit TimeRecorderAuto(std::string hdr, int64_t log_level = 1);
~TimeRecorderAuto() override;
};
} // namespace milvus

76
internal/core/src/utils/TimerContext.h
View File

@ -1,76 +0,0 @@
// Copyright (C) 2019-2020 Zilliz. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software distributed under the License
// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
// or implied. See the License for the specific language governing permissions and limitations under the License.
#pragma once
#include <boost/asio.hpp>
#include <boost/date_time/posix_time/posix_time.hpp>
#include <functional>
#include <memory>
#include <string>
#include <vector>
#include "utils/Log.h"
#include "utils/ThreadPool.h"
namespace milvus {
struct TimerContext {
using HandlerT = std::function<void(const boost::system::error_code&)>;
struct Context {
/* Context(int interval_us, HandlerT& handler, ThreadPoolPtr pool = nullptr) */
/* : interval_(interval_us), handler_(handler), timer_(io, interval_), pool_(pool) { */
/* } */
int interval_us;
HandlerT handler;
ThreadPoolPtr pool = nullptr;
};
TimerContext(boost::asio::io_service& io, int interval_us, HandlerT& handler, ThreadPoolPtr pool)
: io_(io), interval_(interval_us), handler_(handler), timer_(io, interval_), pool_(pool) {
}
TimerContext(boost::asio::io_service& io, Context& context)
: io_(io),
interval_(context.interval_us),
handler_(context.handler),
timer_(io, interval_),
pool_(context.pool) {
}
void
Reschedule(const boost::system::error_code& ec);
boost::asio::io_service& io_;
boost::posix_time::microseconds interval_;
boost::asio::deadline_timer timer_;
HandlerT handler_;
ThreadPoolPtr pool_;
};
inline void
TimerContext::Reschedule(const boost::system::error_code& ec) {
try {
pool_->enqueue(handler_, ec);
} catch (std::exception& ex) {
LOG_SERVER_ERROR_ << "Fail to enqueue handler: " << std::string(ex.what());
}
boost::system::error_code e;
auto new_expires = timer_.expires_at() + interval_;
timer_.expires_at(new_expires, e);
if (e) {
LOG_SERVER_ERROR_ << "Fail to Reschedule: " << e;
}
timer_.async_wait(std::bind(&TimerContext::Reschedule, this, std::placeholders::_1));
}
using TimerContextPtr = std::shared_ptr<TimerContext>;
} // namespace milvus

85
internal/core/src/utils/TimerManager.cpp
View File

@ -1,85 +0,0 @@
// Copyright (C) 2019-2020 Zilliz. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software distributed under the License
// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
// or implied. See the License for the specific language governing permissions and limitations under the License.
#include <utility>
#include "utils/Log.h"
#include "utils/TimerManager.h"
namespace milvus {
TimerManager::TimerManager(unsigned int pool_size) : pool_size_(pool_size) {
}
TimerManager::~TimerManager() {
}
Status
TimerManager::SetPoolSize(unsigned int pool_size) {
if (timer_exeutors_) {
return Status(SERVER_UNEXPECTED_ERROR, "Cannot set pool size since it has been set already");
}
pool_size_ = pool_size;
return Status::OK();
}
Status
TimerManager::Run() {
boost::system::error_code ec;
io_.run(ec);
Status status;
if (ec) {
status = Status(SERVER_UNEXPECTED_ERROR, ec.message());
}
return status;
}
Status
TimerManager::Start() {
for (auto& timer : timers_) {
timer->timer_.async_wait(std::bind(&TimerContext::Reschedule, timer, std::placeholders::_1));
}
return Status::OK();
}
void
TimerManager::Stop() {
boost::system::error_code ec;
for (auto& timer : timers_) {
timer->timer_.cancel(ec);
if (ec) {
LOG_SERVER_ERROR_ << "Fail to cancel timer: " << ec;
}
}
if (timer_exeutors_) {
timer_exeutors_->Stop();
}
}
void
TimerManager::AddTimer(int interval_us, TimerContext::HandlerT handler) {
if (!timer_exeutors_) {
timer_exeutors_ = std::make_shared<ThreadPool>(pool_size_);
}
timers_.emplace_back(std::make_shared<TimerContext>(io_, interval_us, handler, timer_exeutors_));
}
void
TimerManager::AddTimer(const TimerContext::Context& ctx) {
if (!timer_exeutors_) {
timer_exeutors_ = std::make_shared<ThreadPool>(pool_size_);
}
TimerContext::Context context(ctx);
context.pool = timer_exeutors_;
timers_.emplace_back(std::make_shared<TimerContext>(io_, context));
}
} // namespace milvus

57
internal/core/src/utils/TimerManager.h
View File

@ -1,57 +0,0 @@
// Copyright (C) 2019-2020 Zilliz. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software distributed under the License
// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
// or implied. See the License for the specific language governing permissions and limitations under the License.
#pragma once
#include <boost/asio.hpp>
#include <functional>
#include <memory>
#include <vector>
#include "utils/Status.h"
#include "utils/ThreadPool.h"
#include "utils/TimerContext.h"
namespace milvus {
class TimerManager {
public:
TimerManager() = default;
explicit TimerManager(unsigned int pool_size);
Status
SetPoolSize(unsigned int pool_size);
void
AddTimer(int interval_us, TimerContext::HandlerT handler);
void
AddTimer(const TimerContext::Context& ctx);
virtual Status
Run();
virtual Status
Start();
virtual void
Stop();
virtual ~TimerManager();
protected:
boost::asio::io_service io_;
ThreadPoolPtr timer_exeutors_;
unsigned int pool_size_;
std::vector<TimerContextPtr> timers_;
};
using TimerManagerPtr = std::shared_ptr<TimerManager>;
} // namespace milvus