Fix datarace at Setlogger (#20376)

Signed-off-by: Enwei Jiao <enwei.jiao@zilliz.com>

Signed-off-by: Enwei Jiao <enwei.jiao@zilliz.com>

cmd/milvus/run.go

@@ -102,9 +102,6 @@ func (c *run) execute(args []string, flags *flag.FlagSet) {
 		} else {
 			params.Init()
 		}
-		params.SetLogConfig()
-		params.RoleName = c.serverType
-		params.SetLogger(0)
 	}
 
 	runtimeDir := createRuntimeDir(c.serverType)

cmd/roles/roles.go

@@ -20,6 +20,8 @@ import (
 	"context"
 	"os"
 	"os/signal"
+	"path"
+	"strconv"
 	"strings"
 	"sync"
 	"syscall"
@@ -39,6 +41,7 @@ import (
 	"github.com/milvus-io/milvus/internal/querynode"
 	"github.com/milvus-io/milvus/internal/util/dependency"
 	"github.com/milvus-io/milvus/internal/util/etcd"
+	"github.com/milvus-io/milvus/internal/util/logutil"
 	"github.com/milvus-io/milvus/internal/util/metricsinfo"
 	"github.com/milvus-io/milvus/internal/util/paramtable"
 	"github.com/milvus-io/milvus/internal/util/trace"
@@ -76,7 +79,6 @@ func runComponent[T component](ctx context.Context,
 
 	wg.Add(1)
 	go func() {
-		params := paramtable.Get()
 		if extraInit != nil {
 			extraInit()
 		}
@@ -84,9 +86,9 @@ func runComponent[T component](ctx context.Context,
 		var err error
 		role, err = creator(ctx, factory)
 		if localMsg {
-			params.SetLogConfig(typeutil.StandaloneRole)
+			paramtable.SetRole(typeutil.StandaloneRole)
 		} else {
-			params.SetLogConfig(role.GetName())
+			paramtable.SetRole(role.GetName())
 		}
 		if err != nil {
 			panic(err)
@@ -180,6 +182,24 @@ func (mr *MilvusRoles) runIndexNode(ctx context.Context, localMsg bool, alias st
 		metrics.RegisterIndexNode)
 }
 
+func (mr *MilvusRoles) setupLogger() {
+	logConfig := paramtable.Get().Log
+	id := paramtable.GetNodeID()
+	roleName := paramtable.GetRole()
+	rootPath := logConfig.File.RootPath
+	if rootPath != "" {
+		if id < 0 {
+			logConfig.File.Filename = path.Join(rootPath, roleName+".log")
+		} else {
+			logConfig.File.Filename = path.Join(rootPath, roleName+"-"+strconv.FormatInt(id, 10)+".log")
+		}
+	} else {
+		logConfig.File.Filename = ""
+	}
+
+	logutil.SetupLogger(&logConfig)
+}
+
 // Run Milvus components.
 func (mr *MilvusRoles) Run(local bool, alias string) {
 	log.Info("starting running Milvus components")
@@ -218,6 +238,8 @@ func (mr *MilvusRoles) Run(local bool, alias string) {
 		paramtable.Init()
 	}
 
+	mr.setupLogger()
+
 	if os.Getenv(metricsinfo.DeployModeEnvKey) == metricsinfo.StandaloneDeployMode {
 		closer := trace.InitTracing("standalone")
 		if closer != nil {

internal/datacoord/server.go

@@ -245,7 +245,6 @@ func (s *Server) initSession() error {
 	s.session.Init(typeutil.DataCoordRole, s.address, true, true)
 	s.session.SetEnableActiveStandBy(s.enableActiveStandBy)
 	paramtable.SetNodeID(s.session.ServerID)
-	Params.SetLogger(paramtable.GetNodeID())
 	return nil
 }
 

internal/datanode/data_node.go

@@ -217,7 +217,6 @@ func (node *DataNode) initSession() error {
 	}
 	node.session.Init(typeutil.DataNodeRole, node.address, false, true)
 	paramtable.SetNodeID(node.session.ServerID)
-	Params.SetLogger(paramtable.GetNodeID())
 	return nil
 }
 

internal/indexcoord/index_coord.go

@@ -164,7 +164,6 @@ func (i *IndexCoord) initSession() error {
 	}
 	i.session.Init(typeutil.IndexCoordRole, i.address, true, true)
 	i.session.SetEnableActiveStandBy(i.enableActiveStandBy)
-	Params.SetLogger(i.session.ServerID)
 	i.serverID = i.session.ServerID
 	return nil
 }

internal/indexnode/indexnode.go

@@ -171,7 +171,6 @@ func (i *IndexNode) initSession() error {
 	}
 	i.session.Init(typeutil.IndexNodeRole, i.address, false, true)
 	paramtable.SetNodeID(i.session.ServerID)
-	Params.SetLogger(i.session.ServerID)
 	return nil
 }
 

internal/proxy/proxy.go

@@ -155,7 +155,6 @@ func (node *Proxy) initSession() error {
 	}
 	node.session.Init(typeutil.ProxyRole, node.address, false, true)
 	paramtable.SetNodeID(node.session.ServerID)
-	Params.SetLogger(node.session.ServerID)
 	return nil
 }
 

internal/querycoordv2/server.go

@@ -157,7 +157,6 @@ func (s *Server) Init() error {
 	s.enableActiveStandBy = Params.QueryCoordCfg.EnableActiveStandby
 	s.session.SetEnableActiveStandBy(s.enableActiveStandBy)
 	paramtable.SetNodeID(s.session.ServerID)
-	Params.SetLogger(s.session.ServerID)
 	s.factory.Init(Params)
 
 	// Init KV

internal/querynode/query_node.go

@@ -150,7 +150,6 @@ func (node *QueryNode) initSession() error {
 	}
 	node.session.Init(typeutil.QueryNodeRole, node.address, false, true)
 	paramtable.SetNodeID(node.session.ServerID)
-	Params.SetLogger(paramtable.GetNodeID())
 	log.Info("QueryNode init session", zap.Int64("nodeID", paramtable.GetNodeID()), zap.String("node address", node.session.Address))
 	return nil
 }

internal/rootcoord/root_coord.go

@@ -332,7 +332,6 @@ func (c *Core) initSession() error {
 	}
 	c.session.Init(typeutil.RootCoordRole, c.address, true, true)
 	c.session.SetEnableActiveStandBy(c.enableActiveStandBy)
-	Params.SetLogger(c.session.ServerID)
 	return nil
 }
 

internal/util/paramtable/base_table.go

@@ -23,7 +23,6 @@ import (
 
 	config "github.com/milvus-io/milvus/internal/config"
 	"github.com/milvus-io/milvus/internal/log"
-	"github.com/milvus-io/milvus/internal/util/logutil"
 	"github.com/milvus-io/milvus/internal/util/typeutil"
 	"go.uber.org/zap"
 )
@@ -76,9 +75,7 @@ type BaseTable struct {
 
 	configDir string
 
-	RoleName   string
-	Log        log.Config
-	LogCfgFunc func(log.Config)
+	Log log.Config
 
 	YamlFile string
 }
@@ -398,37 +395,13 @@ func (gp *BaseTable) InitLogCfg() {
 	gp.Log.File.MaxSize = gp.ParseIntWithDefault("log.file.maxSize", DefaultMaxSize)
 	gp.Log.File.MaxBackups = gp.ParseIntWithDefault("log.file.maxBackups", DefaultMaxBackups)
 	gp.Log.File.MaxDays = gp.ParseIntWithDefault("log.file.maxAge", DefaultMaxAge)
-}
+	gp.Log.File.RootPath = gp.LoadWithDefault("log.file.rootPath", DefaultRootPath)
 
-// SetLogConfig set log config of the base table
-func (gp *BaseTable) SetLogConfig() {
-	gp.LogCfgFunc = func(cfg log.Config) {
-		var err error
-		grpclog, err := gp.Load("grpc.log.level")
-		if err != nil {
-			cfg.GrpcLevel = DefaultLogLevel
-		} else {
-			cfg.GrpcLevel = strings.ToUpper(grpclog)
-		}
-		logutil.SetupLogger(&cfg)
-		defer log.Sync()
-	}
-}
-
-// SetLogger sets the logger file by given id
-func (gp *BaseTable) SetLogger(id UniqueID) {
-	rootPath := gp.LoadWithDefault("log.file.rootPath", DefaultRootPath)
-	if rootPath != "" {
-		if id < 0 {
-			gp.Log.File.Filename = path.Join(rootPath, gp.RoleName+".log")
-		} else {
-			gp.Log.File.Filename = path.Join(rootPath, gp.RoleName+"-"+strconv.FormatInt(id, 10)+".log")
-		}
+	grpclog, err := gp.Load("grpc.log.level")
+	if err != nil {
+		gp.Log.GrpcLevel = DefaultLogLevel
 	} else {
-		gp.Log.File.Filename = ""
+		gp.Log.GrpcLevel = strings.ToUpper(grpclog)
 	}
 
-	if gp.LogCfgFunc != nil {
-		gp.LogCfgFunc(gp.Log)
-	}
 }

internal/util/paramtable/base_table_test.go

@@ -17,7 +17,6 @@ import (
 	"testing"
 
 	"github.com/stretchr/testify/assert"
-	"google.golang.org/grpc/grpclog"
 )
 
 var baseParams = BaseTable{}
@@ -257,33 +256,6 @@ func Test_ConvertRangeToIntSlice(t *testing.T) {
 	})
 }
 
-func Test_SetLogger(t *testing.T) {
-	t.Run("TestSetLooger", func(t *testing.T) {
-		baseParams.RoleName = "rootcoord"
-		baseParams.Save("log.file.rootPath", ".")
-		baseParams.SetLogger(UniqueID(-1))
-		assert.Equal(t, "rootcoord.log", baseParams.Log.File.Filename)
-
-		baseParams.RoleName = "datanode"
-		baseParams.SetLogger(UniqueID(1))
-		assert.Equal(t, "datanode-1.log", baseParams.Log.File.Filename)
-
-		baseParams.RoleName = "datanode"
-		baseParams.SetLogger(UniqueID(0))
-		assert.Equal(t, "datanode-0.log", baseParams.Log.File.Filename)
-	})
-
-	t.Run("TestGrpclog", func(t *testing.T) {
-		baseParams.Save("grpc.log.level", "Warning")
-		baseParams.SetLogConfig()
-
-		baseParams.SetLogger(UniqueID(1))
-		assert.Equal(t, false, grpclog.V(0))
-		assert.Equal(t, true, grpclog.V(1))
-		assert.Equal(t, true, grpclog.V(2))
-	})
-}
-
 func TestNewBaseTableFromYamlOnly(t *testing.T) {
 	var yaml string
 	var gp *BaseTable

internal/util/paramtable/component_param.go

@@ -94,12 +94,6 @@ func (p *ComponentParam) Init() {
 	p.HookCfg.init()
 }
 
-// SetLogConfig set log config with given role
-func (p *ComponentParam) SetLogConfig(role string) {
-	p.BaseTable.RoleName = role
-	p.BaseTable.SetLogConfig()
-}
-
 func (p *ComponentParam) RocksmqEnable() bool {
 	return p.RocksmqCfg.Path != ""
 }

internal/util/paramtable/runtime.go

@@ -17,6 +17,7 @@ import (
 
 const (
 	runtimeNodeIDKey = "runtime.nodeID"
+	runtimeRoleKey   = "runtime.role"
 )
 
 var params ComponentParam
@@ -40,3 +41,11 @@ func GetNodeID() UniqueID {
 	}
 	return nodeID
 }
+
+func SetRole(role string) {
+	params.Save(runtimeRoleKey, role)
+}
+
+func GetRole() string {
+	return params.Get(runtimeRoleKey)
+}