Support print log to file and console at the same time (#21946)

Signed-off-by: wayblink <anyang.wang@zilliz.com>

configs/milvus.yaml

@@ -333,9 +333,10 @@ dataNode:
 # Configures the system log output.
 log:
   level: debug # Only supports debug, info, warn, error, panic, or fatal. Default 'info'.
+  stdout: "true" # default true, print log to stdout
   file:
     # please adjust in embedded Milvus: /tmp/milvus/logs
-    rootPath: "" # default to stdout, stderr
+    rootPath: "" # root dir path to put logs, default "" means no log file will print
     maxSize: 300 # MB
     maxAge: 10 # Maximum time for log retention in day.
     maxBackups: 20

internal/log/config.go

@@ -48,6 +48,8 @@ type Config struct {
 	Format string `toml:"format" json:"format"`
 	// Disable automatic timestamps in output.
 	DisableTimestamp bool `toml:"disable-timestamp" json:"disable-timestamp"`
+	// Stdout enable or not.
+	Stdout bool `toml:"stdout" json:"stdout"`
 	// File log config.
 	File FileLogConfig `toml:"file" json:"file"`
 	// Development puts the logger in development mode, which changes the

internal/log/log.go

@@ -32,6 +32,8 @@ package log
 import (
 	"fmt"
 	"os"
+	"path/filepath"
+	"strings"
 	"sync"
 	"sync/atomic"
 
@@ -68,23 +70,25 @@ func init() {
 
 // InitLogger initializes a zap logger.
 func InitLogger(cfg *Config, opts ...zap.Option) (*zap.Logger, *ZapProperties, error) {
-	var output zapcore.WriteSyncer
+	var outputs []zapcore.WriteSyncer
 	if len(cfg.File.Filename) > 0 {
 		lg, err := initFileLog(&cfg.File)
 		if err != nil {
 			return nil, nil, err
 		}
-		output = zapcore.AddSync(lg)
-	} else {
+		outputs = append(outputs, zapcore.AddSync(lg))
+	}
+	if cfg.Stdout {
 		stdOut, _, err := zap.Open([]string{"stdout"}...)
 		if err != nil {
 			return nil, nil, err
 		}
-		output = stdOut
+		outputs = append(outputs, stdOut)
 	}
 	debugCfg := *cfg
 	debugCfg.Level = "debug"
-	debugL, r, err := InitLoggerWithWriteSyncer(&debugCfg, output, opts...)
+	outputsWriter := zap.CombineWriteSyncers(outputs...)
+	debugL, r, err := InitLoggerWithWriteSyncer(&debugCfg, outputsWriter, opts...)
 	if err != nil {
 		return nil, nil, err
 	}
@@ -129,7 +133,8 @@ func InitLoggerWithWriteSyncer(cfg *Config, output zapcore.WriteSyncer, opts ...
 
 // initFileLog initializes file based logging options.
 func initFileLog(cfg *FileLogConfig) (*lumberjack.Logger, error) {
-	if st, err := os.Stat(cfg.Filename); err == nil {
+	logPath := strings.Join([]string{cfg.RootPath, cfg.Filename}, string(filepath.Separator))
+	if st, err := os.Stat(logPath); err == nil {
 		if st.IsDir() {
 			return nil, errors.New("can't use directory as log file name")
 		}
@@ -140,7 +145,7 @@ func initFileLog(cfg *FileLogConfig) (*lumberjack.Logger, error) {
 
 	// use lumberjack to logrotate
 	return &lumberjack.Logger{
-		Filename:   cfg.Filename,
+		Filename:   logPath,
 		MaxSize:    cfg.MaxSize,
 		MaxBackups: cfg.MaxBackups,
 		MaxAge:     cfg.MaxDays,
@@ -149,7 +154,7 @@ func initFileLog(cfg *FileLogConfig) (*lumberjack.Logger, error) {
 }
 
 func newStdLogger() (*zap.Logger, *ZapProperties) {
-	conf := &Config{Level: "debug", File: FileLogConfig{}}
+	conf := &Config{Level: "debug", Stdout: true}
 	lg, r, _ := InitLogger(conf)
 	return lg, r
 }

internal/log/log_test.go

@@ -35,6 +35,8 @@ import (
 	"bytes"
 	"context"
 	"fmt"
+	"os"
+	"path/filepath"
 	"testing"
 	"time"
 
@@ -254,3 +256,33 @@ func TestLeveledLogger(t *testing.T) {
 	SetLevel(orgLevel)
 
 }
+
+func TestStdAndFileLogger(t *testing.T) {
+	tmpDir := t.TempDir()
+
+	fileConf := FileLogConfig{
+		RootPath: tmpDir,
+		Filename: "TestStdAndFileLogger",
+	}
+	fmt.Println(tmpDir)
+	conf := &Config{Level: "debug", Stdout: true, File: fileConf}
+
+	logger, _, err := InitLogger(conf)
+
+	assert.NoError(t, err)
+	logger.Info("1234567")
+
+	fileInfo, err := os.Stat(fileConf.RootPath + string(filepath.Separator) + fileConf.Filename)
+	assert.NoError(t, err)
+	assert.NotEmpty(t, fileInfo)
+	assert.True(t, fileInfo.Size() > 0)
+}
+
+func TestStdLogger(t *testing.T) {
+	conf := &Config{Level: "debug", Stdout: true}
+
+	logger, _, err := InitLogger(conf)
+
+	assert.NoError(t, err)
+	logger.Info("1234567")
+}

internal/util/paramtable/base_table.go

@@ -49,7 +49,7 @@ const (
 	DefaultRootPath        = ""
 )
 
-//Const of Global Config List
+// Const of Global Config List
 func globalConfigPrefixs() []string {
 	return []string{"metastore.", "localStorage.", "etcd.", "mysql.", "minio.", "pulsar.", "kafka.", "rocksmq.", "log.", "grpc.", "common.", "quotaAndLimits."}
 }
@@ -226,6 +226,12 @@ func (gp *BaseTable) initLog() {
 	gp.Log.File.MaxBackups, _ = strconv.Atoi(gp.GetWithDefault("log.file.maxBackups", "10"))
 	gp.Log.File.MaxDays, _ = strconv.Atoi(gp.GetWithDefault("log.file.maxAge", "20"))
 	gp.Log.File.RootPath = gp.GetWithDefault("log.file.rootPath", DefaultRootPath)
+	stdout, err := strconv.ParseBool(gp.GetWithDefault("log.stdout", "true"))
+	if err != nil {
+		gp.Log.Stdout = true
+	} else {
+		gp.Log.Stdout = stdout
+	}
 
 	grpclog, err := gp.Load("grpc.log.level")
 	if err != nil {