milvus-io
/
milvus
mirror of https://github.com/milvus-io/milvus.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

avoid relative header path 

Former-commit-id: d4a01b720ac7011e17076b7c36a20af0ebe2b58a
pull/191/head
starlord 2019-10-23 15:05:22 +08:00
parent 25e49ef1a2
commit e7876f6343
14 changed files with 90 additions and 107 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
core/CMakeLists.txt
View File

@ -71,7 +71,7 @@ if(MILVUS_VERSION_MAJOR STREQUAL ""
endif()
message(STATUS "Build version = ${MILVUS_VERSION}")
configure_file(${CMAKE_CURRENT_SOURCE_DIR}/version.h.macro ${CMAKE_CURRENT_SOURCE_DIR}/version.h)
configure_file(${CMAKE_CURRENT_SOURCE_DIR}/src/version.h.macro ${CMAKE_CURRENT_SOURCE_DIR}/src/version.h)
message(STATUS "Milvus version: "
"${MILVUS_VERSION_MAJOR}.${MILVUS_VERSION_MINOR}.${MILVUS_VERSION_PATCH} "

2
core/src/main.cpp
View File

@ -22,10 +22,10 @@
#include <cstring>
#include <string>
#include "../version.h"
#include "external/easyloggingpp/easylogging++.h"
#include "metrics/Metrics.h"
#include "server/Server.h"
#include "src/version.h"
#include "utils/CommonUtil.h"
#include "utils/SignalUtil.h"

2
core/src/sdk/grpc/ClientProxy.cpp
View File

@ -16,8 +16,8 @@
// under the License.
#include "sdk/grpc/ClientProxy.h"
#include "../../../version.h"
#include "grpc/gen-milvus/milvus.grpc.pb.h"
#include "src/version.h"
#include <memory>
#include <string>

2
core/src/server/Server.cpp
View File

@ -19,13 +19,13 @@
#include <string.h>
#include <unistd.h>
#include "../../version.h"
#include "metrics/Metrics.h"
#include "scheduler/SchedInst.h"
#include "server/Config.h"
#include "server/DBWrapper.h"
#include "server/Server.h"
#include "server/grpc_impl/GrpcServer.h"
#include "src/version.h"
#include "utils/Log.h"
#include "utils/LogUtil.h"
#include "utils/SignalUtil.h"

2
core/src/server/grpc_impl/GrpcRequestTask.cpp
View File

@ -23,12 +23,12 @@
#include <vector>
//#include <gperftools/profiler.h>
#include "../../../version.h"
#include "GrpcServer.h"
#include "db/Utils.h"
#include "scheduler/SchedInst.h"
#include "server/DBWrapper.h"
#include "server/Server.h"
#include "src/version.h"
#include "utils/CommonUtil.h"
#include "utils/Log.h"
#include "utils/TimeRecorder.h"

0
core/version.h.macro → core/src/version.h.macro
View File

7
core/unittest/db/CMakeLists.txt
View File

@ -31,12 +31,5 @@ target_link_libraries(test_db
install(TARGETS test_db DESTINATION unittest)
configure_file(appendix/server_config.yaml
"${CMAKE_CURRENT_BINARY_DIR}/milvus/conf/server_config.yaml"
COPYONLY)
configure_file(appendix/log_config.conf
"${CMAKE_CURRENT_BINARY_DIR}/milvus/conf/log_config.conf"
COPYONLY)

27
core/unittest/db/appendix/log_config.conf
View File

@ -1,27 +0,0 @@
* GLOBAL:
FORMAT = "%datetime | %level | %logger | %msg"
FILENAME = "/tmp/milvus/logs/milvus-%datetime{%y-%M-%d-%H:%m}-global.log"
ENABLED = true
TO_FILE = true
TO_STANDARD_OUTPUT = false
SUBSECOND_PRECISION = 3
PERFORMANCE_TRACKING = false
MAX_LOG_FILE_SIZE = 209715200 ## Throw log files away after 200MB
* DEBUG:
FILENAME = "/tmp/milvus/logs/milvus-%datetime{%y-%M-%d-%H:%m}-debug.log"
ENABLED = true
* WARNING:
FILENAME = "/tmp/milvus/logs/milvus-%datetime{%y-%M-%d-%H:%m}-warning.log"
* TRACE:
FILENAME = "/tmp/milvus/logs/milvus-%datetime{%y-%M-%d-%H:%m}-trace.log"
* VERBOSE:
FORMAT = "%datetime{%d/%M/%y} | %level-%vlevel | %msg"
TO_FILE = false
TO_STANDARD_OUTPUT = false
## Error logs
* ERROR:
ENABLED = true
FILENAME = "/tmp/milvus/logs/milvus-%datetime{%y-%M-%d-%H:%m}-error.log"
* FATAL:
ENABLED = true
FILENAME = "/tmp/milvus/logs/milvus-%datetime{%y-%M-%d-%H:%m}-fatal.log"

37
core/unittest/db/appendix/server_config.yaml
View File

@ -1,37 +0,0 @@
# All the following configurations are default values.
server_config:
address: 0.0.0.0 # milvus server ip address (IPv4)
port: 19530 # port range: 1025 ~ 65534
deploy_mode: single # deployment type: single, cluster_readonly, cluster_writable
time_zone: UTC+8
db_config:
primary_path: /tmp/milvus # path used to store data and meta
secondary_path: # path used to store data only, split by semicolon
backend_url: sqlite://:@:/ # URI format: dialect://username:password@host:port/database
# Keep 'dialect://:@:/', and replace other texts with real values.
# Replace 'dialect' with 'mysql' or 'sqlite'
insert_buffer_size: 4 # GB, maximum insert buffer size allowed
build_index_gpu: 0 # gpu id used for building index
metric_config:
enable_monitor: false # enable monitoring or not
collector: prometheus # prometheus
prometheus_config:
port: 8080 # port prometheus used to fetch metrics
cache_config:
cpu_mem_capacity: 16 # GB, CPU memory used for cache
cpu_mem_threshold: 0.85 # percentage of data kept when cache cleanup triggered
cache_insert_data: false # whether load inserted data into cache
engine_config:
blas_threshold: 20
resource_config:
resource_pool:
- cpu
- gpu0

6
core/unittest/db/test_db.cpp
View File

@ -33,8 +33,6 @@
namespace {
static const char *CONFIG_FILE_PATH = "./milvus/conf/server_config.yaml";
static const char *TABLE_NAME = "test_group";
static constexpr int64_t TABLE_DIM = 256;
static constexpr int64_t VECTOR_COUNT = 25000;
@ -232,8 +230,10 @@ TEST_F(DBTest, DB_TEST) {
}
TEST_F(DBTest, SEARCH_TEST) {
std::string config_path(CONFIG_PATH);
config_path += CONFIG_FILE;
milvus::server::Config &config = milvus::server::Config::GetInstance();
milvus::Status s = config.LoadConfigFile(CONFIG_FILE_PATH);
milvus::Status s = config.LoadConfigFile(config_path);
milvus::engine::meta::TableSchema table_info = BuildTableSchema();
auto stat = db_->CreateTable(table_info);

53
core/unittest/db/utils.cpp
View File

@ -28,11 +28,59 @@
#include "db/DBFactory.h"
#include "db/Options.h"
#include "knowhere/index/vector_index/helpers/FaissGpuResourceMgr.h"
#include "utils/CommonUtil.h"
INITIALIZE_EASYLOGGINGPP
namespace {
static const char
* CONFIG_STR = "# All the following configurations are default values.\n"
"\n"
"server_config:\n"
" address: 0.0.0.0 # milvus server ip address (IPv4)\n"
" port: 19530 # port range: 1025 ~ 65534\n"
" deploy_mode: single \n"
" time_zone: UTC+8\n"
"\n"
"db_config:\n"
" primary_path: /tmp/milvus # path used to store data and meta\n"
" secondary_path: # path used to store data only, split by semicolon\n"
"\n"
" backend_url: sqlite://:@:/ \n"
" \n"
" # Replace 'dialect' with 'mysql' or 'sqlite'\n"
"\n"
" insert_buffer_size: 4 # GB, maximum insert buffer size allowed\n"
"\n"
"metric_config:\n"
" enable_monitor: false # enable monitoring or not\n"
" collector: prometheus # prometheus\n"
" prometheus_config:\n"
" port: 8080 # port prometheus used to fetch metrics\n"
"\n"
"cache_config:\n"
" cpu_mem_capacity: 16 # GB, CPU memory used for cache\n"
" cpu_mem_threshold: 0.85 # percentage of data kept when cache cleanup triggered\n"
" cache_insert_data: false # whether load inserted data into cache\n"
"\n"
"engine_config:\n"
" blas_threshold: 20\n"
"\n"
"resource_config:\n"
" resource_pool:\n"
" - gpu0\n"
" index_build_device: gpu0 # GPU used for building index";
void
WriteToFile(const std::string& file_path, const char* content) {
std::fstream fs(file_path.c_str(), std::ios_base::out);
//write data to file
fs << content;
fs.close();
}
class DBTestEnvironment : public ::testing::Environment {
public:
explicit DBTestEnvironment(const std::string& uri)
@ -84,7 +132,7 @@ BaseTest::TearDown() {
milvus::engine::DBOptions
BaseTest::GetOptions() {
auto options = milvus::engine::DBFactory::BuildOption();
options.meta_.path_ = "/tmp/milvus_test";
options.meta_.path_ = CONFIG_PATH;
options.meta_.backend_uri_ = "sqlite://:@:/";
return options;
}
@ -111,6 +159,9 @@ DBTest::SetUp() {
auto options = GetOptions();
db_ = milvus::engine::DBFactory::Build(options);
std::string config_path(options.meta_.path_ + CONFIG_FILE);
WriteToFile(config_path, CONFIG_STR);
}
void

3
core/unittest/db/utils.h
View File

@ -42,6 +42,9 @@
#define STOP_TIMER(name)
#endif
static const char *CONFIG_PATH = "/tmp/milvus_test";
static const char *CONFIG_FILE = "/server_config.yaml";
class BaseTest : public ::testing::Test {
protected:
void InitLog();

16
core/unittest/server/test_config.cpp
View File

@ -33,13 +33,13 @@ static constexpr uint64_t GB = MB * 1024;
} // namespace
TEST_F(ConfigTest, CONFIG_TEST) {
milvus::server::ConfigMgr *config_mgr = milvus::server::YamlConfigMgr::GetInstance();
milvus::server::ConfigMgr* config_mgr = milvus::server::YamlConfigMgr::GetInstance();
milvus::Status s = config_mgr->LoadConfigFile("");
ASSERT_FALSE(s.ok());
std::string config_path(CONFIG_PATH);
s = config_mgr->LoadConfigFile(config_path+ INVALID_CONFIG_FILE);
s = config_mgr->LoadConfigFile(config_path + INVALID_CONFIG_FILE);
ASSERT_FALSE(s.ok());
s = config_mgr->LoadConfigFile(config_path + VALID_CONFIG_FILE);
@ -47,11 +47,11 @@ TEST_F(ConfigTest, CONFIG_TEST) {
config_mgr->Print();
milvus::server::ConfigNode &root_config = config_mgr->GetRootNode();
milvus::server::ConfigNode &server_config = root_config.GetChild("server_config");
milvus::server::ConfigNode &db_config = root_config.GetChild("db_config");
milvus::server::ConfigNode &metric_config = root_config.GetChild("metric_config");
milvus::server::ConfigNode &cache_config = root_config.GetChild("cache_config");
milvus::server::ConfigNode& root_config = config_mgr->GetRootNode();
milvus::server::ConfigNode& server_config = root_config.GetChild("server_config");
milvus::server::ConfigNode& db_config = root_config.GetChild("db_config");
milvus::server::ConfigNode& metric_config = root_config.GetChild("metric_config");
milvus::server::ConfigNode& cache_config = root_config.GetChild("cache_config");
milvus::server::ConfigNode invalid_config = root_config.GetChild("invalid_config");
auto valus = invalid_config.GetSequence("not_exist");
float ff = invalid_config.GetFloatValue("not_exist", 3.0);
@ -100,7 +100,7 @@ TEST_F(ConfigTest, CONFIG_TEST) {
TEST_F(ConfigTest, SERVER_CONFIG_TEST) {
std::string config_path(CONFIG_PATH);
milvus::server::Config &config = milvus::server::Config::GetInstance();
milvus::server::Config& config = milvus::server::Config::GetInstance();
milvus::Status s = config.LoadConfigFile(config_path + VALID_CONFIG_FILE);
ASSERT_TRUE(s.ok());

38
core/unittest/server/test_rpc.cpp
View File

@ -23,7 +23,7 @@
#include "server/grpc_impl/GrpcRequestHandler.h"
#include "server/grpc_impl/GrpcRequestScheduler.h"
#include "server/grpc_impl/GrpcRequestTask.h"
#include "../version.h"
#include "src/version.h"
#include "grpc/gen-milvus/milvus.grpc.pb.h"
#include "grpc/gen-status/status.pb.h"
@ -36,7 +36,7 @@
namespace {
static const char *TABLE_NAME = "test_grpc";
static const char* TABLE_NAME = "test_grpc";
static constexpr int64_t TABLE_DIM = 256;
static constexpr int64_t INDEX_FILE_SIZE = 1024;
static constexpr int64_t VECTOR_COUNT = 1000;
@ -109,7 +109,7 @@ class RpcHandlerTest : public testing::Test {
void
BuildVectors(int64_t from, int64_t to,
std::vector<std::vector<float >> &vector_record_array) {
std::vector<std::vector<float >>& vector_record_array) {
if (to <= from) {
return;
}
@ -119,7 +119,7 @@ BuildVectors(int64_t from, int64_t to,
std::vector<float> record;
record.resize(TABLE_DIM);
for (int64_t i = 0; i < TABLE_DIM; i++) {
record[i] = (float) (k % (i + 1));
record[i] = (float)(k % (i + 1));
}
vector_record_array.emplace_back(record);
@ -136,7 +136,7 @@ CurrentTmDate(int64_t offset_day = 0) {
gmtime_r(&tt, &t);
std::string str = std::to_string(t.tm_year + 1900) + "-" + std::to_string(t.tm_mon + 1)
+ "-" + std::to_string(t.tm_mday);
+ "-" + std::to_string(t.tm_mday);
return str;
}
@ -200,8 +200,8 @@ TEST_F(RpcHandlerTest, INSERT_TEST) {
std::vector<std::vector<float>> record_array;
BuildVectors(0, VECTOR_COUNT, record_array);
::milvus::grpc::VectorIds vector_ids;
for (auto &record : record_array) {
::milvus::grpc::RowRecord *grpc_record = request.add_row_record_array();
for (auto& record : record_array) {
::milvus::grpc::RowRecord* grpc_record = request.add_row_record_array();
for (size_t i = 0; i < record.size(); i++) {
grpc_record->add_vector_data(record[i]);
}
@ -239,8 +239,8 @@ TEST_F(RpcHandlerTest, SEARCH_TEST) {
std::vector<std::vector<float>> record_array;
BuildVectors(0, VECTOR_COUNT, record_array);
::milvus::grpc::InsertParam insert_param;
for (auto &record : record_array) {
::milvus::grpc::RowRecord *grpc_record = insert_param.add_row_record_array();
for (auto& record : record_array) {
::milvus::grpc::RowRecord* grpc_record = insert_param.add_row_record_array();
for (size_t i = 0; i < record.size(); i++) {
grpc_record->add_vector_data(record[i]);
}
@ -252,16 +252,16 @@ TEST_F(RpcHandlerTest, SEARCH_TEST) {
sleep(7);
BuildVectors(0, 10, record_array);
for (auto &record : record_array) {
::milvus::grpc::RowRecord *row_record = request.add_query_record_array();
for (auto &rec : record) {
for (auto& record : record_array) {
::milvus::grpc::RowRecord* row_record = request.add_query_record_array();
for (auto& rec : record) {
row_record->add_vector_data(rec);
}
}
handler->Search(&context, &request, &response);
//test search with range
::milvus::grpc::Range *range = request.mutable_query_range_array()->Add();
::milvus::grpc::Range* range = request.mutable_query_range_array()->Add();
range->set_start_value(CurrentTmDate(-2));
range->set_end_value(CurrentTmDate(-3));
handler->Search(&context, &request, &response);
@ -273,7 +273,7 @@ TEST_F(RpcHandlerTest, SEARCH_TEST) {
handler->Search(&context, &request, &response);
::milvus::grpc::SearchInFilesParam search_in_files_param;
std::string *file_id = search_in_files_param.add_file_id_array();
std::string* file_id = search_in_files_param.add_file_id_array();
*file_id = "test_tbl";
handler->SearchInFiles(&context, &search_in_files_param, &response);
}
@ -323,8 +323,8 @@ TEST_F(RpcHandlerTest, TABLES_TEST) {
//test empty row record
handler->Insert(&context, &request, &vector_ids);
for (auto &record : record_array) {
::milvus::grpc::RowRecord *grpc_record = request.add_row_record_array();
for (auto& record : record_array) {
::milvus::grpc::RowRecord* grpc_record = request.add_row_record_array();
for (size_t i = 0; i < record.size(); i++) {
grpc_record->add_vector_data(record[i]);
}
@ -341,7 +341,7 @@ TEST_F(RpcHandlerTest, TABLES_TEST) {
request.clear_row_record_array();
vector_ids.clear_vector_id_array();
for (uint64_t i = 0; i < 10; ++i) {
::milvus::grpc::RowRecord *grpc_record = request.add_row_record_array();
::milvus::grpc::RowRecord* grpc_record = request.add_row_record_array();
for (size_t j = 0; j < 10; j++) {
grpc_record->add_vector_data(record_array[i][j]);
}
@ -431,12 +431,12 @@ class DummyTask : public milvus::server::grpc::GrpcBaseTask {
}
static milvus::server::grpc::BaseTaskPtr
Create(std::string &dummy) {
Create(std::string& dummy) {
return std::shared_ptr<milvus::server::grpc::GrpcBaseTask>(new DummyTask(dummy));
}
public:
explicit DummyTask(std::string &dummy) : GrpcBaseTask(dummy) {
explicit DummyTask(std::string& dummy) : GrpcBaseTask(dummy) {
}
};