Make paramtable init only once (#21782)

Signed-off-by: Congqi Xia <congqi.xia@zilliz.com>
pull/21829/head
congqixia 2023-01-19 14:53:44 +08:00 committed by GitHub
parent 1dcd9eeb79
commit 5986106037
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
38 changed files with 88 additions and 89 deletions

View File

@ -601,7 +601,7 @@ func TestIndexBuilder(t *testing.T) {
nodeID = UniqueID(700)
)
Params.InitOnce()
Params.Init()
ctx := context.Background()
catalog := catalogmocks.NewDataCoordCatalog(t)
catalog.On("CreateSegmentIndex",

View File

@ -3735,6 +3735,7 @@ func Test_newChunkManagerFactory(t *testing.T) {
paramtable.Get().Save(Params.DataCoordCfg.EnableGarbageCollection.Key, "true")
t.Run("err_minio_bad_address", func(t *testing.T) {
paramtable.Get().Save(Params.CommonCfg.StorageType.Key, "minio")
paramtable.Get().Save(Params.MinioCfg.Address.Key, "host:9000:bad")
defer paramtable.Get().Reset(Params.MinioCfg.Address.Key)
storageCli, err := server.newChunkManagerFactory()

View File

@ -11,7 +11,7 @@ import (
)
func Test_getOrCreateIOPool(t *testing.T) {
Params.InitOnce()
Params.Init()
ioConcurrency := Params.DataNodeCfg.IOConcurrency
paramtable.Get().Save(Params.DataNodeCfg.IOConcurrency.Key, "64")
defer func() { Params.DataNodeCfg.IOConcurrency = ioConcurrency }()

View File

@ -30,7 +30,7 @@ import (
)
func Test_NewClient(t *testing.T) {
proxy.Params.InitOnce()
proxy.Params.Init()
ctx := context.Background()
etcdCli, err := etcd.GetEtcdClient(

View File

@ -30,7 +30,7 @@ import (
)
func Test_NewClient(t *testing.T) {
proxy.Params.InitOnce()
proxy.Params.Init()
ctx := context.Background()
client, err := NewClient(ctx, "")
assert.Nil(t, client)

View File

@ -44,7 +44,7 @@ func TestIndexNodeServer(t *testing.T) {
assert.NotNil(t, server)
inm := indexnode.NewIndexNodeMock()
ParamsGlobal.InitOnce()
ParamsGlobal.Init()
etcdCli, err := etcd.GetEtcdClient(
ParamsGlobal.EtcdCfg.UseEmbedEtcd.GetAsBool(),
ParamsGlobal.EtcdCfg.EtcdUseSSL.GetAsBool(),

View File

@ -30,7 +30,7 @@ import (
)
func Test_NewClient(t *testing.T) {
proxy.Params.InitOnce()
proxy.Params.Init()
ctx := context.Background()
client, err := NewClient(ctx, "")

View File

@ -31,7 +31,7 @@ import (
)
func Test_NewClient(t *testing.T) {
proxy.Params.InitOnce()
proxy.Params.Init()
ctx := context.Background()

View File

@ -32,7 +32,7 @@ import (
)
func Test_NewClient(t *testing.T) {
proxy.Params.InitOnce()
proxy.Params.Init()
ctx := context.Background()
etcdCli, err := etcd.GetEtcdClient(

View File

@ -31,6 +31,7 @@ import (
"github.com/milvus-io/milvus/internal/util/metautil"
"github.com/milvus-io/milvus/internal/util/metricsinfo"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func genStorageConfig() *indexpb.StorageConfig {
@ -48,7 +49,8 @@ func genStorageConfig() *indexpb.StorageConfig {
func TestIndexNodeSimple(t *testing.T) {
in, err := NewMockIndexNodeComponent(context.TODO())
assert.Nil(t, err)
require.Nil(t, err)
defer in.Stop()
ctx := context.TODO()
state, err := in.GetComponentStates(ctx)
assert.Nil(t, err)
@ -226,7 +228,8 @@ func TestIndexNodeComplex(t *testing.T) {
}
)
in, err := NewMockIndexNodeComponent(context.TODO())
assert.Nil(t, err)
require.Nil(t, err)
defer in.Stop()
ctx := context.TODO()
state, err := in.GetComponentStates(ctx)
assert.Nil(t, err)
@ -386,6 +389,7 @@ func TestGetMetrics(t *testing.T) {
)
in, err := NewMockIndexNodeComponent(ctx)
assert.Nil(t, err)
defer in.Stop()
resp, err := in.GetMetrics(ctx, metricReq)
assert.Nil(t, err)
assert.Equal(t, resp.Status.ErrorCode, commonpb.ErrorCode_Success)
@ -399,6 +403,7 @@ func TestGetMetricsError(t *testing.T) {
in, err := NewMockIndexNodeComponent(ctx)
assert.Nil(t, err)
defer in.Stop()
errReq := &milvuspb.GetMetricsRequest{
Request: `{"metric_typ": "system_info"}`,
}

View File

@ -30,7 +30,7 @@ import (
func TestEtcdConfigLoad(te *testing.T) {
te.Setenv(metricsinfo.DeployModeEnvKey, metricsinfo.StandaloneDeployMode)
param := new(paramtable.ServiceParam)
param := new(paramtable.ComponentParam)
te.Setenv("etcd.use.embed", "true")
te.Setenv("etcd.config.path", "../../../configs/advanced/etcd.yaml")

View File

@ -35,7 +35,7 @@ import (
func TestEmbedEtcd(te *testing.T) {
te.Setenv(metricsinfo.DeployModeEnvKey, metricsinfo.StandaloneDeployMode)
param := new(paramtable.ServiceParam)
param := new(paramtable.ComponentParam)
te.Setenv("etcd.use.embed", "true")
te.Setenv("etcd.config.path", "../../../configs/advanced/etcd.yaml")

View File

@ -32,7 +32,7 @@ func TestEtcdRestartLoad(te *testing.T) {
etcdDataDir := "/tmp/_etcd_data"
te.Setenv(metricsinfo.DeployModeEnvKey, metricsinfo.StandaloneDeployMode)
te.Setenv("ETCD_USE_EMBED", "true")
param := new(paramtable.ServiceParam)
param := new(paramtable.ComponentParam)
param.Init()
param.BaseTable.Save("etcd.config.path", "../../../configs/advanced/etcd.yaml")
param.BaseTable.Save("etcd.data.dir", etcdDataDir)

View File

@ -168,8 +168,7 @@ func TestRocksmq_RegisterConsumer(t *testing.T) {
defer os.RemoveAll(rocksdbPath + kvSuffix)
defer os.RemoveAll(rocksdbPath)
var params paramtable.BaseTable
params.Init(0)
paramtable.Init()
rmq, err := NewRocksMQ(rocksdbPath, idAllocator)
assert.NoError(t, err)
defer rmq.Close()
@ -233,8 +232,7 @@ func TestRocksmq_Basic(t *testing.T) {
rocksdbPath := rmqPath + suffix
defer os.RemoveAll(rocksdbPath + kvSuffix)
defer os.RemoveAll(rocksdbPath)
var params paramtable.BaseTable
params.Init(0)
paramtable.Init()
rmq, err := NewRocksMQ(rocksdbPath, idAllocator)
assert.Nil(t, err)
defer rmq.Close()
@ -370,8 +368,7 @@ func TestRocksmq_Dummy(t *testing.T) {
rocksdbPath := rmqPath + suffix
defer os.RemoveAll(rocksdbPath + kvSuffix)
defer os.RemoveAll(rocksdbPath)
var params paramtable.BaseTable
params.Init(0)
paramtable.Init()
rmq, err := NewRocksMQ(rocksdbPath, idAllocator)
assert.Nil(t, err)
defer rmq.Close()
@ -441,8 +438,7 @@ func TestRocksmq_Seek(t *testing.T) {
defer os.RemoveAll(rocksdbPath + kvSuffix)
defer os.RemoveAll(rocksdbPath)
var params paramtable.BaseTable
params.Init(0)
paramtable.Init()
rmq, err := NewRocksMQ(rocksdbPath, idAllocator)
assert.Nil(t, err)
defer rmq.Close()
@ -509,8 +505,8 @@ func TestRocksmq_Loop(t *testing.T) {
kvName := name + "_meta_kv"
_ = os.RemoveAll(kvName)
defer os.RemoveAll(kvName)
var params paramtable.BaseTable
params.Init(0)
paramtable.Init()
rmq, err := NewRocksMQ(name, idAllocator)
assert.Nil(t, err)
defer rmq.Close()
@ -581,8 +577,8 @@ func TestRocksmq_Goroutines(t *testing.T) {
kvName := name + "_meta_kv"
_ = os.RemoveAll(kvName)
defer os.RemoveAll(kvName)
var params paramtable.BaseTable
params.Init(0)
paramtable.Init()
rmq, err := NewRocksMQ(name, idAllocator)
assert.Nil(t, err)
defer rmq.Close()
@ -660,8 +656,8 @@ func TestRocksmq_Throughout(t *testing.T) {
kvName := name + "_meta_kv"
_ = os.RemoveAll(kvName)
defer os.RemoveAll(kvName)
var params paramtable.BaseTable
params.Init(0)
paramtable.Init()
rmq, err := NewRocksMQ(name, idAllocator)
assert.Nil(t, err)
defer rmq.Close()
@ -725,8 +721,8 @@ func TestRocksmq_MultiChan(t *testing.T) {
kvName := name + "_meta_kv"
_ = os.RemoveAll(kvName)
defer os.RemoveAll(kvName)
var params paramtable.BaseTable
params.Init(0)
paramtable.Init()
rmq, err := NewRocksMQ(name, idAllocator)
assert.Nil(t, err)
defer rmq.Close()
@ -779,8 +775,8 @@ func TestRocksmq_CopyData(t *testing.T) {
kvName := name + "_meta_kv"
_ = os.RemoveAll(kvName)
defer os.RemoveAll(kvName)
var params paramtable.BaseTable
params.Init(0)
paramtable.Init()
rmq, err := NewRocksMQ(name, idAllocator)
assert.Nil(t, err)
defer rmq.Close()
@ -847,8 +843,8 @@ func TestRocksmq_SeekToLatest(t *testing.T) {
kvName := name + "_meta_kv"
_ = os.RemoveAll(kvName)
defer os.RemoveAll(kvName)
var params paramtable.BaseTable
params.Init(0)
paramtable.Init()
rmq, err := NewRocksMQ(name, idAllocator)
assert.Nil(t, err)
defer rmq.Close()

View File

@ -411,8 +411,8 @@ func TestRetentionInfo_InitRetentionInfo(t *testing.T) {
defer os.RemoveAll(metaPath)
var params paramtable.BaseTable
params.Init(0)
params := paramtable.Get()
params.Init()
rmq, err := NewRocksMQ(rocksdbPath, idAllocator)
assert.Nil(t, err)
assert.NotNil(t, rmq)

View File

@ -22,10 +22,10 @@ import (
"go.uber.org/zap"
)
var Params paramtable.BaseTable
var Params = paramtable.Get()
func TestMain(m *testing.M) {
Params.Init(0)
Params.Init()
mockCluster, err := kafka.NewMockCluster(1)
defer mockCluster.Close()
if err != nil {

View File

@ -44,10 +44,10 @@ const (
DefaultPulsarNamespace = "default"
)
var Params paramtable.BaseTable
var Params = paramtable.Get()
func TestMain(m *testing.M) {
Params.Init(0)
Params.Init()
exitCode := m.Run()
os.Exit(exitCode)
}

View File

@ -32,11 +32,11 @@ func TestProxyRpcLimit(t *testing.T) {
localMsg := true
factory := dependency.NewDefaultFactory(localMsg)
base := paramtable.BaseTable{}
base.Init(0)
base := &paramtable.ComponentParam{}
base.Init()
var p paramtable.GrpcServerConfig
assert.NoError(t, err)
p.Init(typeutil.ProxyRole, &base)
p.Init(typeutil.ProxyRole, &base.BaseTable)
base.Save("proxy.grpc.serverMaxRecvSize", "1")
assert.Equal(t, p.ServerMaxRecvSize.GetAsInt(), 1)

View File

@ -474,10 +474,10 @@ func TestProxy(t *testing.T) {
testServer := newProxyTestServer(proxy)
wg.Add(1)
base := paramtable.BaseTable{}
base.Init(0)
base := &paramtable.ComponentParam{}
base.Init()
var p paramtable.GrpcServerConfig
p.Init(typeutil.ProxyRole, &base)
p.Init(typeutil.ProxyRole, &base.BaseTable)
testServer.Proxy.SetAddress(p.GetAddress())
assert.Equal(t, p.GetAddress(), testServer.Proxy.GetAddress())

View File

@ -50,7 +50,7 @@ func Test_dropPartitionTask_Prepare(t *testing.T) {
})
t.Run("normal case", func(t *testing.T) {
Params.InitOnce()
Params.Init()
collectionName := funcutil.GenRandomStr()
coll := &model.Collection{Name: collectionName}

View File

@ -447,7 +447,7 @@ func TestMetaTable_getCollectionByIDInternal(t *testing.T) {
})
t.Run("normal case, filter unavailable partitions", func(t *testing.T) {
Params.InitOnce()
Params.Init()
meta := &MetaTable{
collID2Meta: map[typeutil.UniqueID]*model.Collection{
100: {
@ -469,7 +469,7 @@ func TestMetaTable_getCollectionByIDInternal(t *testing.T) {
})
t.Run("get latest version", func(t *testing.T) {
Params.InitOnce()
Params.Init()
meta := &MetaTable{
collID2Meta: map[typeutil.UniqueID]*model.Collection{
100: {

View File

@ -501,7 +501,7 @@ func withTtSynchronizer(ticker *timetickSync) Opt {
}
func newRocksMqTtSynchronizer() *timetickSync {
Params.InitOnce()
Params.Init()
paramtable.Get().Save(Params.RootCoordCfg.DmlChannelNum.Key, "4")
ctx := context.Background()
factory := dependency.NewDefaultFactory(true)
@ -893,7 +893,7 @@ func newTickerWithMockNormalStream() *timetickSync {
}
func newTickerWithFactory(factory msgstream.Factory) *timetickSync {
Params.InitOnce()
Params.Init()
paramtable.Get().Save(Params.RootCoordCfg.DmlChannelNum.Key, "4")
ctx := context.Background()
chans := map[UniqueID][]string{}

View File

@ -754,7 +754,7 @@ func TestRootCoord_ShowConfigurations(t *testing.T) {
})
t.Run("normal case", func(t *testing.T) {
Params.InitOnce()
Params.Init()
pattern := "rootcoord.Port"
req := &internalpb.ShowConfigurationsRequest{

View File

@ -186,7 +186,7 @@ func Test_scheduler_updateDdlMinTsLoop(t *testing.T) {
}
ctx := context.Background()
s := newScheduler(ctx, idAlloc, tsoAlloc)
Params.InitOnce()
Params.Init()
paramtable.Get().Save(Params.ProxyCfg.TimeTickInterval.Key, "1")
s.Start()
@ -217,7 +217,7 @@ func Test_scheduler_updateDdlMinTsLoop(t *testing.T) {
}
ctx := context.Background()
s := newScheduler(ctx, idAlloc, tsoAlloc)
Params.InitOnce()
Params.Init()
paramtable.Get().Save(Params.ProxyCfg.TimeTickInterval.Key, "1")
s.Start()

View File

@ -20,7 +20,7 @@ import (
)
func getTestEtcdCli() *clientv3.Client {
Params.InitOnce()
Params.Init()
cli, err := etcd.GetEtcdClient(
Params.EtcdCfg.UseEmbedEtcd.GetAsBool(),
Params.EtcdCfg.EtcdUseSSL.GetAsBool(),

View File

@ -84,7 +84,7 @@ func TestMinIOCMFail(t *testing.T) {
}
func TestMinIOCM(t *testing.T) {
Params.Init(0)
Params.Init()
testBucket, err := Params.Load("minio.bucketName")
require.NoError(t, err)

View File

@ -139,11 +139,11 @@ func buildVectorChunkManager(localPath string, localCacheEnable bool) (*VectorCh
return vcm, cancel, nil
}
var Params paramtable.BaseTable
var Params = paramtable.Get()
var localPath = "/tmp/milvus_test/chunkmanager/"
func TestMain(m *testing.M) {
Params.Init(0)
Params.Init()
exitCode := m.Run()
err := os.RemoveAll(localPath)
if err != nil {

View File

@ -287,9 +287,7 @@ func genIndexCase() []indexTestCase {
}
func genStorageConfig() *indexpb.StorageConfig {
InitOnce.Do(func() {
Params.Init()
})
Params.Init()
return &indexpb.StorageConfig{
Address: Params.MinioCfg.Address.GetValue(),

View File

@ -2,7 +2,6 @@ package indexcgowrapper
import (
"strconv"
"sync"
"testing"
"github.com/stretchr/testify/assert"
@ -40,8 +39,7 @@ const (
ef = 200
)
var Params paramtable.ServiceParam
var InitOnce sync.Once
var Params paramtable.ComponentParam
type vecTestCase struct {
indexType string

View File

@ -76,9 +76,9 @@ func NewBaseTableFromYamlOnly(yaml string) *BaseTable {
return gp
}
// Init initializes the param table.
// init initializes the param table.
// if refreshInterval greater than 0 will auto refresh config from source
func (gp *BaseTable) Init(refreshInterval int) {
func (gp *BaseTable) init(refreshInterval int) {
formatter := func(key string) string {
ret := strings.ToLower(key)
ret = strings.TrimPrefix(ret, "milvus.")

View File

@ -23,7 +23,7 @@ import (
var baseParams = BaseTable{}
func TestMain(m *testing.M) {
baseParams.Init(0)
baseParams.init(0)
code := m.Run()
os.Exit(code)
}
@ -111,7 +111,7 @@ func TestBaseTable_Get(t *testing.T) {
func TestBaseTable_Pulsar(t *testing.T) {
//test PULSAR ADDRESS
t.Setenv("PULSAR_ADDRESS", "pulsar://localhost:6650")
baseParams.Init(0)
baseParams.init(0)
address := baseParams.Get("pulsar.address")
assert.Equal(t, "pulsar://localhost:6650", address)
@ -124,7 +124,7 @@ func TestBaseTable_Env(t *testing.T) {
t.Setenv("milvus.test", "test")
t.Setenv("milvus.test.test2", "test2")
baseParams.Init(0)
baseParams.init(0)
result, _ := baseParams.Load("test")
assert.Equal(t, result, "test")
@ -133,7 +133,7 @@ func TestBaseTable_Env(t *testing.T) {
t.Setenv("milvus.invalid", "xxx=test")
baseParams.Init(0)
baseParams.init(0)
result, _ = baseParams.Load("invalid")
assert.Equal(t, result, "xxx=test")
}

View File

@ -88,16 +88,17 @@ type ComponentParam struct {
IntegrationTestCfg integrationTestConfig
}
// InitOnce initialize once
func (p *ComponentParam) InitOnce() {
// Init initialize once
func (p *ComponentParam) Init() {
p.once.Do(func() {
p.Init()
p.init()
})
}
// Init initialize the global param table
func (p *ComponentParam) Init() {
p.ServiceParam.Init()
// init initialize the global param table
func (p *ComponentParam) init() {
p.ServiceParam.init()
p.CommonCfg.init(&p.BaseTable)
p.QuotaConfig.init(&p.BaseTable)

View File

@ -21,10 +21,10 @@ import (
func TestGrpcServerParams(t *testing.T) {
role := typeutil.DataNodeRole
base := BaseTable{}
base.Init(0)
base := &ComponentParam{}
base.Init()
var serverConfig GrpcServerConfig
serverConfig.Init(role, &base)
serverConfig.Init(role, &base.BaseTable)
assert.Equal(t, serverConfig.Domain, role)
t.Logf("Domain = %s", serverConfig.Domain)
@ -63,10 +63,10 @@ func TestGrpcServerParams(t *testing.T) {
}
func TestGrpcClientParams(t *testing.T) {
role := typeutil.DataNodeRole
base := BaseTable{}
base.Init(0)
base := ComponentParam{}
base.Init()
var clientConfig GrpcClientConfig
clientConfig.Init(role, &base)
clientConfig.Init(role, &base.BaseTable)
assert.Equal(t, clientConfig.Domain, role)
t.Logf("Domain = %s", clientConfig.Domain)

View File

@ -9,7 +9,7 @@ type hookConfig struct {
func (h *hookConfig) init() {
base := &BaseTable{YamlFile: hookYamlFile}
base.Init(0)
base.init(0)
h.SoPath = ParamItem{
Key: "soPath",

View File

@ -8,7 +8,7 @@ import (
func TestHTTPConfig_Init(t *testing.T) {
params := ComponentParam{}
params.InitOnce()
params.Init()
cf := params.HTTPCfg
assert.Equal(t, cf.Enabled.GetAsBool(), true)
assert.Equal(t, cf.DebugMode.GetAsBool(), false)

View File

@ -26,7 +26,7 @@ const (
var params ComponentParam
func Init() {
params.InitOnce()
params.Init()
}
func Get() *ComponentParam {

View File

@ -53,8 +53,8 @@ type ServiceParam struct {
MinioCfg MinioConfig
}
func (p *ServiceParam) Init() {
p.BaseTable.Init(10)
func (p *ServiceParam) init() {
p.BaseTable.init(10)
p.LocalStorageCfg.Init(&p.BaseTable)
p.MetaStoreCfg.Init(&p.BaseTable)

View File

@ -22,7 +22,7 @@ import (
func TestServiceParam(t *testing.T) {
var SParams ServiceParam
SParams.Init()
SParams.init()
t.Run("test etcdConfig", func(t *testing.T) {
Params := &SParams.EtcdCfg
@ -54,11 +54,11 @@ func TestServiceParam(t *testing.T) {
// test UseEmbedEtcd
t.Setenv("etcd.use.embed", "true")
t.Setenv(metricsinfo.DeployModeEnvKey, metricsinfo.ClusterDeployMode)
assert.Panics(t, func() { SParams.Init() })
assert.Panics(t, func() { SParams.init() })
t.Setenv(metricsinfo.DeployModeEnvKey, metricsinfo.StandaloneDeployMode)
t.Setenv("etcd.use.embed", "false")
SParams.Init()
SParams.init()
})
t.Run("test pulsarConfig", func(t *testing.T) {