milvus-io
/
milvus
mirror of https://github.com/milvus-io/milvus.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Support easylogging config for segcore and knowhere (#16751) 

Signed-off-by: yudong.cai <yudong.cai@zilliz.com>
pull/16762/head
Cai Yudong 2022-05-03 08:39:49 +08:00 committed by GitHub
parent d5db4ae463
commit 6a62ff18bf
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
12 changed files with 59 additions and 39 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

25
configs/easylogging.yaml Normal file
View File

@ -0,0 +1,25 @@
* GLOBAL:
FORMAT = "%datetime | %level | %logger | %msg"
FILENAME = "/var/lib/milvus/logs/milvus-%datetime{%y-%M-%d-%H:%m}-easylog.log"
ENABLED = false
TO_FILE = false
TO_STANDARD_OUTPUT = false
SUBSECOND_PRECISION = 3
PERFORMANCE_TRACKING = false
MAX_LOG_FILE_SIZE = 209715200 ## Throw log files away after 200MB
* INFO:
ENABLED = false
* TRACE:
ENABLED = false
* DEBUG:
ENABLED = false
* WARNING:
ENABLED = false
* ERROR:
ENABLED = true
* FATAL:
ENABLED = true
* VERBOSE:
FORMAT = "%datetime{%d/%M/%y} | %level-%vlevel | %msg"
TO_FILE = false
TO_STANDARD_OUTPUT = false

25
internal/core/src/config/ConfigKnowhere.cpp
View File

@ -27,23 +27,24 @@ namespace milvus::config {
std::once_flag init_knowhere_once_;
void
KnowhereInitImpl() {
auto init = []() {
KnowhereInitImpl(const char* conf_file) {
auto init = [&]() {
knowhere::KnowhereConfig::SetBlasThreshold(16384);
knowhere::KnowhereConfig::SetEarlyStopThreshold(0);
knowhere::KnowhereConfig::SetLogHandler();
knowhere::KnowhereConfig::SetStatisticsLevel(0);
#ifdef EMBEDDED_MILVUS
// always disable all logs for embedded milvus.
el::Configurations el_conf;
el_conf.setGlobally(el::ConfigurationType::Enabled, std::to_string(false));
#if defined(EMBEDDED_MILVUS)
// Disable all logs for embedded milvus.
el_conf.set(el::Level::Trace, el::ConfigurationType::Enabled, "false");
el_conf.set(el::Level::Debug, el::ConfigurationType::Enabled, "false");
el_conf.set(el::Level::Info, el::ConfigurationType::Enabled, "false");
el_conf.set(el::Level::Warning, el::ConfigurationType::Enabled, "false");
el_conf.set(el::Level::Error, el::ConfigurationType::Enabled, "false");
el_conf.set(el::Level::Fatal, el::ConfigurationType::Enabled, "false");
el::Loggers::reconfigureLogger("default", el_conf);
el_conf.setGlobally(el::ConfigurationType::Enabled, "false");
el::Loggers::reconfigureAllLoggers(el_conf);
#else
if (conf_file != nullptr) {
el::Configurations el_conf(conf_file);
el::Loggers::reconfigureAllLoggers(el_conf);
LOG_SERVER_DEBUG_ << "Config easylogging with yaml file: " << conf_file;
}
#endif
};

2
internal/core/src/config/ConfigKnowhere.h
View File

@ -20,7 +20,7 @@
namespace milvus::config {
void
KnowhereInitImpl();
KnowhereInitImpl(const char*);
std::string
KnowhereSetSimdType(const char*);

4
internal/core/src/indexbuilder/init_c.cpp
View File

@ -14,8 +14,8 @@
#include "indexbuilder/init_c.h"
void
IndexBuilderInit() {
milvus::config::KnowhereInitImpl();
IndexBuilderInit(const char* conf_file) {
milvus::config::KnowhereInitImpl(conf_file);
}
// return value must be freed by the caller

2
internal/core/src/indexbuilder/init_c.h
View File

@ -16,7 +16,7 @@ extern "C" {
#endif
void
IndexBuilderInit();
IndexBuilderInit(const char*);
// return value must be freed by the caller
char*

16
internal/core/src/segcore/segcore_init_c.cpp
View File

@ -16,20 +16,8 @@
namespace milvus::segcore {
extern "C" void
SegcoreInit() {
milvus::config::KnowhereInitImpl();
#if defined(EMBEDDED_MILVUS)
el::Configurations defaultConf;
defaultConf.setToDefault();
// Disable all logs for embedded milvus.
defaultConf.set(el::Level::Trace, el::ConfigurationType::Enabled, "false");
defaultConf.set(el::Level::Debug, el::ConfigurationType::Enabled, "false");
defaultConf.set(el::Level::Info, el::ConfigurationType::Enabled, "false");
defaultConf.set(el::Level::Warning, el::ConfigurationType::Enabled, "false");
defaultConf.set(el::Level::Error, el::ConfigurationType::Enabled, "false");
defaultConf.set(el::Level::Fatal, el::ConfigurationType::Enabled, "false");
el::Loggers::reconfigureLogger("default", defaultConf);
#endif
SegcoreInit(const char* conf_file) {
milvus::config::KnowhereInitImpl(conf_file);
}
extern "C" void

2
internal/core/src/segcore/segcore_init_c.h
View File

@ -16,7 +16,7 @@ extern "C" {
#endif
void
SegcoreInit();
SegcoreInit(const char*);
void
SegcoreSetChunkRows(const int64_t);

2
internal/core/unittest/test_init.cpp
View File

@ -17,7 +17,7 @@
TEST(Init, Naive) {
using namespace milvus;
using namespace milvus::segcore;
SegcoreInit();
SegcoreInit(nullptr);
SegcoreSetChunkRows(32768);
SegcoreSetSimdType("auto");
}

5
internal/indexnode/indexnode.go
View File

@ -36,6 +36,7 @@ import (
"io"
"math/rand"
"os"
"path"
"strconv"
"sync"
"sync/atomic"
@ -146,7 +147,9 @@ func (i *IndexNode) Register() error {
}
func (i *IndexNode) initKnowhere() {
C.IndexBuilderInit()
cEasyloggingYaml := C.CString(path.Join(Params.BaseTable.GetConfigDir(), paramtable.DefaultEasyloggingYaml))
C.IndexBuilderInit(cEasyloggingYaml)
C.free(unsafe.Pointer(cEasyloggingYaml))
// override index builder SIMD type
cSimdType := C.CString(Params.CommonCfg.SimdType)

5
internal/querynode/query_node.go
View File

@ -36,6 +36,7 @@ import (
"errors"
"fmt"
"os"
"path"
"path/filepath"
"strconv"
"sync"
@ -176,7 +177,9 @@ func (node *QueryNode) Register() error {
// InitSegcore set init params of segCore, such as chunckRows, SIMD type...
func (node *QueryNode) InitSegcore() {
C.SegcoreInit()
cEasyloggingYaml := C.CString(path.Join(Params.BaseTable.GetConfigDir(), paramtable.DefaultEasyloggingYaml))
C.SegcoreInit(cEasyloggingYaml)
C.free(unsafe.Pointer(cEasyloggingYaml))
// override segcore chunk size
cChunkRows := C.int64_t(Params.QueryNodeCfg.ChunkRows)

8
internal/util/paramtable/base_table.go
View File

@ -35,6 +35,8 @@ import (
type UniqueID = typeutil.UniqueID
const (
DefaultMilvusYaml = "milvus.yaml"
DefaultEasyloggingYaml = "easylogging.yaml"
DefaultMinioHost = "localhost"
DefaultMinioPort = "9000"
DefaultMinioAccessKey = "minioadmin"
@ -49,7 +51,7 @@ const (
DefaultEnvPrefix = "milvus"
)
var DefaultYaml = "milvus.yaml"
var defaultYaml = DefaultMilvusYaml
// Base abstracts BaseTable
// TODO: it's never used, consider to substitute BaseTable or to remove it
@ -79,7 +81,7 @@ type BaseTable struct {
// GlobalInitWithYaml should be called only in standalone and embedded Milvus.
func (gp *BaseTable) GlobalInitWithYaml(yaml string) {
gp.once.Do(func() {
DefaultYaml = yaml
defaultYaml = yaml
gp.Init()
})
}
@ -88,7 +90,7 @@ func (gp *BaseTable) GlobalInitWithYaml(yaml string) {
func (gp *BaseTable) Init() {
gp.params = memkv.NewMemoryKV()
gp.configDir = gp.initConfPath()
gp.loadFromYaml(DefaultYaml)
gp.loadFromYaml(defaultYaml)
gp.tryLoadFromEnv()
gp.InitLogCfg()
}

2
internal/util/paramtable/base_table_test.go
View File

@ -22,8 +22,6 @@ import (
"github.com/stretchr/testify/assert"
)
const defaultYaml = "milvus.yaml"
var baseParams = BaseTable{}
func TestMain(m *testing.M) {