milvus-io
/
milvus
mirror of https://github.com/milvus-io/milvus.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

MS-653 fix when config check fail, Milvus close without message 

Former-commit-id: b9ff8c2228d07b6abe9d72f87e06bee823a05363
pull/191/head
yudong.cai 2019-10-15 15:59:56 +08:00
parent c11820aff1
commit 30534bee23
13 changed files with 121 additions and 164 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
core/CHANGELOG.md
View File

@ -11,16 +11,17 @@ Please mark all change in change log and use the ticket from JIRA.
- MS-587 - Count get wrong result after adding vectors and index built immediately
- MS-599 - Search wrong result when table created with metric_type: IP
- MS-601 - Docker logs error caused by get CPUTemperature error
- MS-622 - Delete vectors should be failed if date range is invalid
- MS-620 - Get table row counts display wrong error code
- MS-622 - Delete vectors should be failed if date range is invalid
- MS-624 - Search vectors failed if time ranges long enough
- MS-637 - Out of memory when load too many tasks
- MS-639 - SQ8H index created failed and server hang
- MS-640 - Cache object size calculate incorrect
- MS-641 - Segment fault(signal 11) in PickToLoad
- MS-639 - SQ8H index created failed and server hang
- MS-647 - [monitor] grafana display average cpu-temp
- MS-644 - Search crashed with index-type: flat
- MS-624 - Search vectors failed if time ranges long enough
- MS-647 - [monitor] grafana display average cpu-temp
- MS-652 - IVFSQH quantization double free
- MS-653 - When config check fail, Milvus close without message
## Improvement
- MS-552 - Add and change the easylogging library
@ -38,8 +39,8 @@ Please mark all change in change log and use the ticket from JIRA.
- MS-608 - Update TODO names
- MS-609 - Update task construct function
- MS-611 - Add resources validity check in ResourceMgr
- MS-619 - Add optimizer class in scheduler
- MS-614 - Preload table at startup
- MS-619 - Add optimizer class in scheduler
- MS-626 - Refactor DataObj to support cache any type data
- MS-648 - Improve unittest
@ -59,8 +60,8 @@ Please mark all change in change log and use the ticket from JIRA.
- MS-600 - Reconstruct unittest code
- MS-602 - Remove zilliz namespace
- MS-610 - Change error code base value from hex to decimal
- MS-635 - Add compile option to support customized faiss
- MS-624 - Re-organize project directory for open-source
- MS-635 - Add compile option to support customized faiss
# Milvus 0.4.0 (2019-09-12)

31
core/src/config/ConfigMgr.cpp
View File

@ -1,31 +0,0 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
#include "config/ConfigMgr.h"
#include "YamlConfigMgr.h"
namespace milvus {
namespace server {
ConfigMgr*
ConfigMgr::GetInstance() {
static YamlConfigMgr mgr;
return &mgr;
}
} // namespace server
} // namespace milvus

28
core/src/config/ConfigMgr.h
View File

@ -17,42 +17,28 @@
#pragma once
#include "ConfigNode.h"
#include "utils/Error.h"
#include <string>
#include "ConfigNode.h"
#include "utils/Status.h"
namespace milvus {
namespace server {
// this class can parse nested config file and return config item
// config file example(yaml style)
// AAA: 1
// BBB:
// CCC: hello
// DDD: 23.5
//
// usage
// const ConfigMgr* mgr = ConfigMgr::GetInstance();
// const ConfigNode& node = mgr->GetRootNode();
// std::string val = node.GetValue("AAA"); // return '1'
// const ConfigNode& child = node.GetChild("BBB");
// val = child.GetValue("CCC"); //return 'hello'
class ConfigMgr {
public:
static ConfigMgr*
GetInstance();
virtual ErrorCode
virtual Status
LoadConfigFile(const std::string& filename) = 0;
virtual void
Print() const = 0; // will be deleted
virtual std::string
DumpString() const = 0;
virtual const ConfigNode&
GetRootNode() const = 0;
virtual ConfigNode&
GetRootNode() = 0;
};

17
core/src/config/YamlConfigMgr.cpp
View File

@ -18,29 +18,20 @@
#include "config/YamlConfigMgr.h"
#include "utils/Log.h"
#include <sys/stat.h>
namespace milvus {
namespace server {
ErrorCode
Status
YamlConfigMgr::LoadConfigFile(const std::string& filename) {
struct stat directoryStat;
int statOK = stat(filename.c_str(), &directoryStat);
if (statOK != 0) {
SERVER_LOG_ERROR << "File not found: " << filename;
return SERVER_UNEXPECTED_ERROR;
}
try {
node_ = YAML::LoadFile(filename);
LoadConfigNode(node_, config_);
} catch (YAML::Exception& e) {
SERVER_LOG_ERROR << "Failed to load config file: " << std::string(e.what());
return SERVER_UNEXPECTED_ERROR;
std::string str = "Exception: load config file fail: " + std::string(e.what());
return Status(SERVER_UNEXPECTED_ERROR, str);
}
return SERVER_SUCCESS;
return Status::OK();
}
void

18
core/src/config/YamlConfigMgr.h
View File

@ -17,27 +17,35 @@
#pragma once
#include "ConfigMgr.h"
#include "ConfigNode.h"
#include "utils/Error.h"
#include <yaml-cpp/yaml.h>
#include <string>
#include "ConfigMgr.h"
#include "utils/Status.h"
namespace milvus {
namespace server {
class YamlConfigMgr : public ConfigMgr {
public:
virtual ErrorCode
static ConfigMgr*
GetInstance() {
static YamlConfigMgr mgr;
return &mgr;
}
virtual Status
LoadConfigFile(const std::string& filename);
virtual void
Print() const;
virtual std::string
DumpString() const;
virtual const ConfigNode&
GetRootNode() const;
virtual ConfigNode&
GetRootNode();

56
core/src/main.cpp
View File

@ -32,12 +32,32 @@
INITIALIZE_EASYLOGGINGPP
void
print_help(const std::string& app_name);
print_help(const std::string& app_name) {
std::cout << std::endl << "Usage: " << app_name << " [OPTIONS]" << std::endl << std::endl;
std::cout << " Options:" << std::endl;
std::cout << " -h --help Print this help" << std::endl;
std::cout << " -c --conf_file filename Read configuration from the file" << std::endl;
std::cout << " -d --daemon Daemonize this application" << std::endl;
std::cout << " -p --pid_file filename PID file used by daemonized app" << std::endl;
std::cout << std::endl;
}
void
print_banner() {
std::cout << std::endl;
std::cout << " __ _________ _ ____ ______ " << std::endl;
std::cout << " / |/ / _/ /| | / / / / / __/ " << std::endl;
std::cout << " / /|_/ // // /_| |/ / /_/ /\\ \\ " << std::endl;
std::cout << " /_/ /_/___/____/___/\\____/___/ " << std::endl;
std::cout << std::endl;
std::cout << "Welcome to use Milvus by Zilliz!" << std::endl;
std::cout << "Milvus " << BUILD_TYPE << " version: v" << MILVUS_VERSION << ", built at " << BUILD_TIME << std::endl;
std::cout << std::endl;
}
int
main(int argc, char* argv[]) {
std::cout << std::endl << "Welcome to use Milvus by Zilliz!" << std::endl;
std::cout << "Milvus " << BUILD_TYPE << " version: v" << MILVUS_VERSION << " built at " << BUILD_TIME << std::endl;
print_banner();
static struct option long_options[] = {{"conf_file", required_argument, nullptr, 'c'},
{"log_conf_file", required_argument, nullptr, 'l'},
@ -53,10 +73,12 @@ main(int argc, char* argv[]) {
std::string pid_filename;
std::string app_name = argv[0];
milvus::server::Server& server = milvus::server::Server::GetInstance();
milvus::Status s;
if (argc < 2) {
print_help(app_name);
std::cout << "Milvus server exit..." << std::endl;
return EXIT_FAILURE;
goto FAIL;
}
int value;
@ -106,23 +128,21 @@ main(int argc, char* argv[]) {
signal(SIGUSR2, milvus::server::SignalUtil::HandleSignal);
signal(SIGTERM, milvus::server::SignalUtil::HandleSignal);
milvus::server::Server& server = milvus::server::Server::GetInstance();
server.Init(start_daemonized, pid_filename, config_filename, log_config_file);
server.Start();
s = server.Start();
if (s.ok()) {
std::cout << "Milvus server start successfully." << std::endl;
} else {
goto FAIL;
}
/* wait signal */
pause();
return 0;
}
return EXIT_SUCCESS;
void
print_help(const std::string& app_name) {
std::cout << std::endl << "Usage: " << app_name << " [OPTIONS]" << std::endl << std::endl;
std::cout << " Options:" << std::endl;
std::cout << " -h --help Print this help" << std::endl;
std::cout << " -c --conf_file filename Read configuration from the file" << std::endl;
std::cout << " -d --daemon Daemonize this application" << std::endl;
std::cout << " -p --pid_file filename PID file used by daemonized app" << std::endl;
std::cout << std::endl;
FAIL:
std::cout << "Milvus server exit..." << std::endl;
return EXIT_FAILURE;
}

4
core/src/scheduler/task/SearchTask.cpp
View File

@ -160,8 +160,8 @@ XSearchTask::Load(LoadType type, uint8_t device_id) {
size_t file_size = index_engine_->PhysicalSize();
std::string info = "Load file id:" + std::to_string(file_->id_) +
" file type:" + std::to_string(file_->file_type_) + " size:" + std::to_string(file_size) +
std::string info = "Load file id:" + std::to_string(file_->id_) + " file type:" +
std::to_string(file_->file_type_) + " size:" + std::to_string(file_size) +
" bytes from location: " + file_->location_ + " totally cost";
double span = rc.ElapseFromBegin(info);
// for (auto &context : search_contexts_) {

36
core/src/server/Config.cpp
View File

@ -15,18 +15,14 @@
// specific language governing permissions and limitations
// under the License.
#include "server/Config.h"
#include <stdlib.h>
#include <sys/stat.h>
#include <sys/types.h>
#include <unistd.h>
#include <algorithm>
#include <iostream>
#include <string>
#include <vector>
#include "config/ConfigMgr.h"
#include "config/YamlConfigMgr.h"
#include "server/Config.h"
#include "utils/CommonUtil.h"
#include "utils/ValidationUtil.h"
@ -44,26 +40,24 @@ Config::GetInstance() {
Status
Config::LoadConfigFile(const std::string& filename) {
if (filename.empty()) {
std::cerr << "ERROR: need specify config file" << std::endl;
exit(1);
return Status(SERVER_UNEXPECTED_ERROR, "No specified config file");
}
struct stat dirStat;
int statOK = stat(filename.c_str(), &dirStat);
if (statOK != 0) {
std::cerr << "ERROR: Config file not exist: " << filename << std::endl;
exit(1);
struct stat file_stat;
if (stat(filename.c_str(), &file_stat) != 0) {
std::string str = "Config file not exist: " + filename;
return Status(SERVER_FILE_NOT_FOUND, str);
}
try {
ConfigMgr* mgr = const_cast<ConfigMgr*>(ConfigMgr::GetInstance());
ErrorCode err = mgr->LoadConfigFile(filename);
if (err != 0) {
std::cerr << "Server failed to load config file: " << filename << std::endl;
exit(1);
ConfigMgr* mgr = YamlConfigMgr::GetInstance();
Status s = mgr->LoadConfigFile(filename);
if (!s.ok()) {
return s;
}
} catch (YAML::Exception& e) {
std::cerr << "Server failed to load config file: " << filename << std::endl;
exit(1);
std::string str = "Exception occurs when loading config file: " + filename;
return Status(SERVER_UNEXPECTED_ERROR, str);
}
return Status::OK();
@ -632,7 +626,7 @@ Config::CheckResourceConfigPool(const std::vector<std::string>& value) {
////////////////////////////////////////////////////////////////////////////////
ConfigNode&
Config::GetConfigNode(const std::string& name) {
ConfigMgr* mgr = ConfigMgr::GetInstance();
ConfigMgr* mgr = YamlConfigMgr::GetInstance();
ConfigNode& root_node = mgr->GetRootNode();
return root_node.GetChild(name);
}

4
core/src/server/Config.h
View File

@ -17,7 +17,6 @@
#pragma once
#include <yaml-cpp/yaml.h>
#include <mutex>
#include <string>
#include <unordered_map>
@ -110,13 +109,10 @@ class Config {
private:
ConfigNode&
GetConfigNode(const std::string& name);
Status
GetConfigValueInMem(const std::string& parent_key, const std::string& child_key, std::string& value);
void
SetConfigValueInMem(const std::string& parent_key, const std::string& child_key, const std::string& value);
void
PrintConfigSection(const std::string& config_node_name);

16
core/src/server/DBWrapper.cpp
View File

@ -15,19 +15,19 @@
// specific language governing permissions and limitations
// under the License.
#include "server/DBWrapper.h"
#include "Config.h"
#include "db/DBFactory.h"
#include "utils/CommonUtil.h"
#include "utils/Log.h"
#include "utils/StringHelpFunctions.h"
#include <faiss/utils.h>
#include <omp.h>
#include <cmath>
#include <string>
#include <vector>
#include "db/DBFactory.h"
#include "server/Config.h"
#include "server/DBWrapper.h"
#include "utils/CommonUtil.h"
#include "utils/Log.h"
#include "utils/StringHelpFunctions.h"
namespace milvus {
namespace server {
@ -35,9 +35,9 @@ Status
DBWrapper::StartService() {
Config& config = Config::GetInstance();
Status s;
// db config
engine::DBOptions opt;
s = config.GetDBConfigBackendUrl(opt.meta_.backend_uri_);
if (!s.ok()) {
return s;

5
core/src/server/DBWrapper.h
View File

@ -17,12 +17,11 @@
#pragma once
#include <string>
#include "db/DB.h"
#include "utils/Status.h"
#include <memory>
#include <string>
namespace milvus {
namespace server {

49
core/src/server/Server.cpp
View File

@ -15,21 +15,15 @@
// specific language governing permissions and limitations
// under the License.
#include "server/Server.h"
#include <fcntl.h>
#include <sys/stat.h>
#include <sys/types.h>
#include <csignal>
#include <thread>
//#include <numaif.h>
#include <string.h>
#include <unistd.h>
#include "DBWrapper.h"
#include "metrics/Metrics.h"
#include "scheduler/SchedInst.h"
#include "server/Config.h"
#include "server/DBWrapper.h"
#include "server/Server.h"
#include "server/grpc_impl/GrpcServer.h"
#include "utils/Log.h"
#include "utils/LogUtil.h"
@ -143,7 +137,7 @@ Server::Daemonize() {
}
}
void
Status
Server::Start() {
if (daemonized_ != 0) {
Daemonize();
@ -151,18 +145,19 @@ Server::Start() {
try {
/* Read config file */
if (LoadConfig() != SERVER_SUCCESS) {
std::cerr << "Milvus server fail to load config file" << std::endl;
return;
Status s = LoadConfig();
if (!s.ok()) {
std::cerr << "ERROR: Milvus server fail to load config file" << std::endl;
return s;
}
/* log path is defined in Config file, so InitLog must be called after LoadConfig */
Config& config = Config::GetInstance();
std::string time_zone;
Status s = config.GetServerConfigTimeZone(time_zone);
s = config.GetServerConfigTimeZone(time_zone);
if (!s.ok()) {
std::cerr << "Fail to get server config timezone" << std::endl;
return;
return s;
}
if (time_zone.length() == 3) {
@ -179,8 +174,7 @@ Server::Start() {
}
if (setenv("TZ", time_zone.c_str(), 1) != 0) {
std::cerr << "Fail to setenv" << std::endl;
return;
return Status(SERVER_UNEXPECTED_ERROR, "Fail to setenv");
}
tzset();
@ -190,9 +184,10 @@ Server::Start() {
server::SystemInfo::GetInstance().Init();
StartService();
std::cout << "Milvus server start successfully." << std::endl;
return Status::OK();
} catch (std::exception& ex) {
std::cerr << "Milvus server encounter exception: " << ex.what();
std::string str = "Milvus server encounter exception: " + std::string(ex.what());
return Status(SERVER_UNEXPECTED_ERROR, str);
}
}
@ -204,12 +199,12 @@ Server::Stop() {
if (pid_fd_ != -1) {
int ret = lockf(pid_fd_, F_ULOCK, 0);
if (ret != 0) {
std::cerr << "Can't lock file: " << strerror(errno) << std::endl;
std::cerr << "ERROR: Can't lock file: " << strerror(errno) << std::endl;
exit(0);
}
ret = close(pid_fd_);
if (ret != 0) {
std::cerr << "Can't close file: " << strerror(errno) << std::endl;
std::cerr << "ERROR: Can't close file: " << strerror(errno) << std::endl;
exit(0);
}
}
@ -218,31 +213,31 @@ Server::Stop() {
if (!pid_filename_.empty()) {
int ret = unlink(pid_filename_.c_str());
if (ret != 0) {
std::cerr << "Can't unlink file: " << strerror(errno) << std::endl;
std::cerr << "ERROR: Can't unlink file: " << strerror(errno) << std::endl;
exit(0);
}
}
StopService();
std::cerr << "Milvus server is closed!" << std::endl;
std::cerr << "Milvus server exit..." << std::endl;
}
ErrorCode
Status
Server::LoadConfig() {
Config& config = Config::GetInstance();
Status s = config.LoadConfigFile(config_filename_);
if (!s.ok()) {
std::cerr << "Failed to load config file: " << config_filename_ << std::endl;
exit(0);
std::cerr << s.message() << std::endl;
return s;
}
s = config.ValidateConfig();
if (!s.ok()) {
std::cerr << "Config check fail: " << s.message() << std::endl;
exit(0);
return s;
}
return SERVER_SUCCESS;
return milvus::Status::OK();
}
void

8
core/src/server/Server.h
View File

@ -17,10 +17,8 @@
#pragma once
#include "utils/Status.h"
#include <cstdint>
#include <string>
#include "utils/Status.h"
namespace milvus {
namespace server {
@ -34,7 +32,7 @@ class Server {
Init(int64_t daemonized, const std::string& pid_filename, const std::string& config_filename,
const std::string& log_config_file);
void
Status
Start();
void
Stop();
@ -46,7 +44,7 @@ class Server {
void
Daemonize();
ErrorCode
Status
LoadConfig();
void