milvus-io
/
milvus
mirror of https://github.com/milvus-io/milvus.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Rename log file based on node id (#8517) 

Signed-off-by: xiaofan-luan <xiaofan.luan@zilliz.com>
pull/8517/merge
Xiaofan 2021-09-26 21:18:11 +08:00 committed by GitHub
parent d0074c3641
commit d38adda8c2
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
14 changed files with 54 additions and 220 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

30
internal/datacoord/param_table.go
View File

@ -12,14 +12,10 @@
package datacoord
import (
"path"
"strconv"
"strings"
"sync"
"time"
"github.com/milvus-io/milvus/internal/log"
"github.com/milvus-io/milvus/internal/util/paramtable"
)
@ -62,8 +58,6 @@ type ParamTable struct {
CreatedTime time.Time
UpdatedTime time.Time
Log log.Config
}
var Params ParamTable
@ -242,29 +236,7 @@ func (p *ParamTable) initDataCoordSubscriptionName() {
}
func (p *ParamTable) initLogCfg() {
p.Log = log.Config{}
format, err := p.Load("log.format")
if err != nil {
panic(err)
}
p.Log.Format = format
level, err := p.Load("log.level")
if err != nil {
panic(err)
}
p.Log.Level = level
p.Log.File.MaxSize = p.ParseInt("log.file.maxSize")
p.Log.File.MaxBackups = p.ParseInt("log.file.maxBackups")
p.Log.File.MaxDays = p.ParseInt("log.file.maxAge")
rootPath, err := p.Load("log.file.rootPath")
if err != nil {
panic(err)
}
if len(rootPath) != 0 {
p.Log.File.Filename = path.Join(rootPath, "datacoord-"+strconv.FormatInt(p.NodeID, 10)+".log")
} else {
p.Log.File.Filename = ""
}
p.InitLogCfg("datacoord", 0)
}
func (p *ParamTable) initFlushStreamPosSubPath() {

1
internal/datanode/data_node.go
View File

@ -157,6 +157,7 @@ func (node *DataNode) Register() error {
go node.StartWatchChannels(node.ctx)
Params.initMsgChannelSubName()
Params.initLogCfg()
log.Debug("DataNode Init",
zap.String("MsgChannelSubName", Params.MsgChannelSubName),
)

27
internal/datanode/param_table.go
View File

@ -18,7 +18,6 @@ import (
"sync"
"time"
"github.com/milvus-io/milvus/internal/log"
"github.com/milvus-io/milvus/internal/util/paramtable"
)
@ -35,7 +34,6 @@ type ParamTable struct {
FlushInsertBufferSize int64
InsertBinlogRootPath string
StatsBinlogRootPath string
Log log.Config
Alias string // Different datanode in one machine
// === DataNode External Components Configs ===
@ -102,7 +100,6 @@ func (p *ParamTable) Init() {
p.initFlushInsertBufferSize()
p.initInsertBinlogRootPath()
p.initStatsBinlogRootPath()
p.initLogCfg()
// === DataNode External Components Configs ===
// --- Pulsar ---
@ -279,27 +276,5 @@ func (p *ParamTable) initMinioBucketName() {
}
func (p *ParamTable) initLogCfg() {
p.Log = log.Config{}
format, err := p.Load("log.format")
if err != nil {
panic(err)
}
p.Log.Format = format
level, err := p.Load("log.level")
if err != nil {
panic(err)
}
p.Log.Level = level
p.Log.File.MaxSize = p.ParseInt("log.file.maxSize")
p.Log.File.MaxBackups = p.ParseInt("log.file.maxBackups")
p.Log.File.MaxDays = p.ParseInt("log.file.maxAge")
rootPath, err := p.Load("log.file.rootPath")
if err != nil {
panic(err)
}
if len(rootPath) != 0 {
p.Log.File.Filename = path.Join(rootPath, "datanode"+p.Alias+".log")
} else {
p.Log.File.Filename = ""
}
p.InitLogCfg("datanode", p.NodeID)
}

28
internal/indexcoord/param_table.go
View File

@ -12,13 +12,11 @@
package indexcoord
import (
"path"
"strconv"
"strings"
"sync"
"time"
"github.com/milvus-io/milvus/internal/log"
"github.com/milvus-io/milvus/internal/util/paramtable"
)
@ -40,8 +38,6 @@ type ParamTable struct {
CreatedTime time.Time
UpdatedTime time.Time
Log log.Config
}
var Params ParamTable
@ -148,27 +144,5 @@ func (pt *ParamTable) initMinioBucketName() {
}
func (pt *ParamTable) initLogCfg() {
pt.Log = log.Config{}
format, err := pt.Load("log.format")
if err != nil {
panic(err)
}
pt.Log.Format = format
level, err := pt.Load("log.level")
if err != nil {
panic(err)
}
pt.Log.Level = level
pt.Log.File.MaxSize = pt.ParseInt("log.file.maxSize")
pt.Log.File.MaxBackups = pt.ParseInt("log.file.maxBackups")
pt.Log.File.MaxDays = pt.ParseInt("log.file.maxAge")
rootPath, err := pt.Load("log.file.rootPath")
if err != nil {
panic(err)
}
if len(rootPath) != 0 {
pt.Log.File.Filename = path.Join(rootPath, "indexcoord.log")
} else {
pt.Log.File.Filename = ""
}
pt.InitLogCfg("indexcoord", 0)
}

1
internal/indexnode/indexnode.go
View File

@ -106,6 +106,7 @@ func (i *IndexNode) Register() error {
}
i.liveCh = i.session.Init(typeutil.IndexNodeRole, Params.IP+":"+strconv.Itoa(Params.Port), false)
Params.NodeID = i.session.ServerID
Params.initLogCfg()
return nil
}

28
internal/indexnode/param_table.go
View File

@ -12,7 +12,6 @@
package indexnode
import (
"fmt"
"path"
"strconv"
"strings"
@ -53,8 +52,6 @@ type ParamTable struct {
CreatedTime time.Time
UpdatedTime time.Time
Log log.Config
}
var Params ParamTable
@ -76,7 +73,6 @@ func (pt *ParamTable) Init() {
panic(err)
}*/
pt.initLogCfg()
pt.initParams()
pt.initKnowhereSimdType()
}
@ -170,29 +166,7 @@ func (pt *ParamTable) initMinioBucketName() {
}
func (pt *ParamTable) initLogCfg() {
pt.Log = log.Config{}
format, err := pt.Load("log.format")
if err != nil {
panic(err)
}
pt.Log.Format = format
level, err := pt.Load("log.level")
if err != nil {
panic(err)
}
pt.Log.Level = level
pt.Log.File.MaxSize = pt.ParseInt("log.file.maxSize")
pt.Log.File.MaxBackups = pt.ParseInt("log.file.maxBackups")
pt.Log.File.MaxDays = pt.ParseInt("log.file.maxAge")
rootPath, err := pt.Load("log.file.rootPath")
if err != nil {
panic(err)
}
if len(rootPath) != 0 {
pt.Log.File.Filename = path.Join(rootPath, fmt.Sprintf("indexnode-%s.log", pt.Alias))
} else {
pt.Log.File.Filename = ""
}
pt.InitLogCfg("indexnode", pt.NodeID)
}
func (pt *ParamTable) initKnowhereSimdType() {

28
internal/proxy/param_table.go
View File

@ -19,7 +19,6 @@ import (
"sync"
"time"
"github.com/milvus-io/milvus/internal/log"
"github.com/milvus-io/milvus/internal/util/paramtable"
)
@ -37,7 +36,6 @@ type ParamTable struct {
NetworkAddress string
// TODO(dragondriver): maybe using the Proxy + ProxyID as the alias is more reasonable
Alias string
EtcdEndpoints []string
@ -69,7 +67,6 @@ type ParamTable struct {
MaxTaskNum int64
PulsarMaxMessageSize int
Log log.Config
RoleName string
}
@ -89,7 +86,6 @@ func (pt *ParamTable) Init() {
panic(err)
}
pt.initLogCfg()
pt.initEtcdEndpoints()
pt.initMetaRootPath()
pt.initPulsarAddress()
@ -277,29 +273,7 @@ func (pt *ParamTable) initPulsarMaxMessageSize() {
}
func (pt *ParamTable) initLogCfg() {
pt.Log = log.Config{}
format, err := pt.Load("log.format")
if err != nil {
panic(err)
}
pt.Log.Format = format
level, err := pt.Load("log.level")
if err != nil {
panic(err)
}
pt.Log.Level = level
pt.Log.File.MaxSize = pt.ParseInt("log.file.maxSize")
pt.Log.File.MaxBackups = pt.ParseInt("log.file.maxBackups")
pt.Log.File.MaxDays = pt.ParseInt("log.file.maxAge")
rootPath, err := pt.Load("log.file.rootPath")
if err != nil {
panic(err)
}
if len(rootPath) != 0 {
pt.Log.File.Filename = path.Join(rootPath, fmt.Sprintf("proxy-%s.log", pt.Alias))
} else {
pt.Log.File.Filename = ""
}
pt.InitLogCfg("proxy", pt.ProxyID)
}
func (pt *ParamTable) initRoleName() {

1
internal/proxy/proxy.go
View File

@ -100,6 +100,7 @@ func (node *Proxy) Register() error {
node.session.Init(typeutil.ProxyRole, Params.NetworkAddress, false)
Params.ProxyID = node.session.ServerID
Params.initProxySubName()
Params.initLogCfg()
return nil
}

29
internal/querycoord/param_table.go
View File

@ -40,7 +40,6 @@ type ParamTable struct {
// timetick
TimeTickChannelName string
Log log.Config
RoleName string
// channels
@ -81,7 +80,7 @@ func (p *ParamTable) Init() {
panic(err)
}
p.initLogCfg()
p.BaseTable.InitLogCfg("queryCoord", 0)
p.initQueryCoordAddress()
p.initRoleName()
@ -106,32 +105,6 @@ func (p *ParamTable) Init() {
p.initMinioBucketName()
}
func (p *ParamTable) initLogCfg() {
p.Log = log.Config{}
format, err := p.Load("log.format")
if err != nil {
panic(err)
}
p.Log.Format = format
level, err := p.Load("log.level")
if err != nil {
panic(err)
}
p.Log.Level = level
p.Log.File.MaxSize = p.ParseInt("log.file.maxSize")
p.Log.File.MaxBackups = p.ParseInt("log.file.maxBackups")
p.Log.File.MaxDays = p.ParseInt("log.file.maxAge")
rootPath, err := p.Load("log.file.rootPath")
if err != nil {
panic(err)
}
if len(rootPath) != 0 {
p.Log.File.Filename = path.Join(rootPath, fmt.Sprintf("queryCoord-%d.log", p.NodeID))
} else {
p.Log.File.Filename = ""
}
}
func (p *ParamTable) initQueryCoordAddress() {
url, err := p.Load("_QueryCoordAddress")
if err != nil {

30
internal/querynode/param_table.go
View File

@ -12,8 +12,6 @@
package querynode
import (
"fmt"
"path"
"strconv"
"strings"
"sync"
@ -74,8 +72,6 @@ type ParamTable struct {
// segcore
ChunkRows int64
SimdType string
Log log.Config
}
var Params ParamTable
@ -130,8 +126,6 @@ func (p *ParamTable) Init() {
p.initSegcoreChunkRows()
p.initKnowhereSimdType()
p.initLogCfg()
}
// ---------------------------------------------------------- minio
@ -298,27 +292,5 @@ func (p *ParamTable) initKnowhereSimdType() {
}
func (p *ParamTable) initLogCfg() {
p.Log = log.Config{}
format, err := p.Load("log.format")
if err != nil {
panic(err)
}
p.Log.Format = format
level, err := p.Load("log.level")
if err != nil {
panic(err)
}
p.Log.Level = level
p.Log.File.MaxSize = p.ParseInt("log.file.maxSize")
p.Log.File.MaxBackups = p.ParseInt("log.file.maxBackups")
p.Log.File.MaxDays = p.ParseInt("log.file.maxAge")
rootPath, err := p.Load("log.file.rootPath")
if err != nil {
panic(err)
}
if len(rootPath) != 0 {
p.Log.File.Filename = path.Join(rootPath, fmt.Sprintf("querynode-%s.log", p.Alias))
} else {
p.Log.File.Filename = ""
}
p.InitLogCfg("querynode", p.QueryNodeID)
}

1
internal/querynode/query_node.go
View File

@ -109,6 +109,7 @@ func (node *QueryNode) Register() error {
// This param needs valid QueryNodeID
Params.initMsgChannelSubName()
Params.initLogCfg()
return nil
}

28
internal/rootcoord/param_table.go
View File

@ -12,12 +12,10 @@
package rootcoord
import (
"path"
"strings"
"sync"
"time"
"github.com/milvus-io/milvus/internal/log"
"github.com/milvus-io/milvus/internal/util/paramtable"
)
@ -55,8 +53,6 @@ type ParamTable struct {
CreatedTime time.Time
UpdatedTime time.Time
Log log.Config
RoleName string
}
@ -222,29 +218,7 @@ func (p *ParamTable) initTimeTickInterval() {
}
func (p *ParamTable) initLogCfg() {
p.Log = log.Config{}
format, err := p.Load("log.format")
if err != nil {
panic(err)
}
p.Log.Format = format
level, err := p.Load("log.level")
if err != nil {
panic(err)
}
p.Log.Level = level
p.Log.File.MaxSize = p.ParseInt("log.file.maxSize")
p.Log.File.MaxBackups = p.ParseInt("log.file.maxBackups")
p.Log.File.MaxDays = p.ParseInt("log.file.maxAge")
rootPath, err := p.Load("log.file.rootPath")
if err != nil {
panic(err)
}
if len(rootPath) != 0 {
p.Log.File.Filename = path.Join(rootPath, "rootcoord.log")
} else {
p.Log.File.Filename = ""
}
p.InitLogCfg("rootcoord", 0)
}
func (p *ParamTable) initRoleName() {

28
internal/util/paramtable/basetable.go
View File

@ -43,6 +43,8 @@ type Base interface {
type BaseTable struct {
params *memkv.MemoryKV
configDir string
Log log.Config
}
func (gp *BaseTable) Init() {
@ -413,3 +415,29 @@ func ConvertRangeToIntSlice(rangeStr, sep string) []int {
}
return ret
}
func (gp *BaseTable) InitLogCfg(role string, id UniqueID) {
gp.Log = log.Config{}
format, err := gp.Load("log.format")
if err != nil {
panic(err)
}
gp.Log.Format = format
level, err := gp.Load("log.level")
if err != nil {
panic(err)
}
gp.Log.Level = level
gp.Log.File.MaxSize = gp.ParseInt("log.file.maxSize")
gp.Log.File.MaxBackups = gp.ParseInt("log.file.maxBackups")
gp.Log.File.MaxDays = gp.ParseInt("log.file.maxAge")
rootPath, err := gp.Load("log.file.rootPath")
if err != nil {
panic(err)
}
if len(rootPath) != 0 {
gp.Log.File.Filename = path.Join(rootPath, role+"-"+strconv.FormatInt(id, 10)+".log")
} else {
gp.Log.File.Filename = ""
}
}

14
internal/util/paramtable/basetable_test.go
View File

@ -225,3 +225,17 @@ func Test_ConvertRangeToIntSlice(t *testing.T) {
assert.Panics(t, func() { ConvertRangeToIntSlice("9,0", ",") })
})
}
func Test_InitLogCfg(t *testing.T) {
t.Run("TestInitLogCfg", func(t *testing.T) {
baseParams.InitLogCfg("rootcoord", 0)
assert.Equal(t, baseParams.Log.File.Filename, "")
baseParams.Save("log.file.rootPath", "/")
baseParams.InitLogCfg("rootcoord", 0)
assert.Equal(t, baseParams.Log.File.Filename, "/rootcoord-0.log")
baseParams.InitLogCfg("datanode", 8)
assert.Equal(t, baseParams.Log.File.Filename, "/datanode-8.log")
})
}