milvus-io
/
milvus
mirror of https://github.com/milvus-io/milvus.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

enhance: support to config the default db properties (#38035) 

- issue: #38034

Signed-off-by: SimFG <bang.fu@zilliz.com>
pull/35835/head
SimFG 2024-11-27 10:04:34 +08:00 committed by GitHub
parent 971b4f17ae
commit 49ee46ec1d
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
12 changed files with 74 additions and 24 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
internal/metastore/model/database.go
View File

@ -31,8 +31,8 @@ func NewDatabase(id int64, name string, state pb.DatabaseState, properties []*co
}
}
func NewDefaultDatabase() *Database {
return NewDatabase(util.DefaultDBID, util.DefaultDBName, pb.DatabaseState_DatabaseCreated, nil)
func NewDefaultDatabase(prop []*commonpb.KeyValuePair) *Database {
return NewDatabase(util.DefaultDBID, util.DefaultDBName, pb.DatabaseState_DatabaseCreated, prop)
}
func (c *Database) Available() bool {

2
internal/metastore/model/database_test.go
View File

@ -59,5 +59,5 @@ func TestDatabaseCloneAndEqual(t *testing.T) {
func TestDatabaseAvailable(t *testing.T) {
assert.True(t, dbModel.Available())
assert.True(t, NewDefaultDatabase().Available())
assert.True(t, NewDefaultDatabase(nil).Available())
}

4
internal/rootcoord/create_collection_task_test.go
View File

@ -728,7 +728,7 @@ func Test_createCollectionTask_Prepare(t *testing.T) {
mock.Anything,
mock.Anything,
mock.Anything,
).Return(model.NewDefaultDatabase(), nil)
).Return(model.NewDefaultDatabase(nil), nil)
meta.On("ListAllAvailCollections",
mock.Anything,
).Return(map[int64][]int64{
@ -1154,7 +1154,7 @@ func Test_createCollectionTask_PartitionKey(t *testing.T) {
mock.Anything,
mock.Anything,
mock.Anything,
).Return(model.NewDefaultDatabase(), nil)
).Return(model.NewDefaultDatabase(nil), nil)
meta.On("ListAllAvailCollections",
mock.Anything,
).Return(map[int64][]int64{

4
internal/rootcoord/create_db_task_test.go
View File

@ -54,7 +54,7 @@ func Test_CreateDBTask_Prepare(t *testing.T) {
len := cfgMaxDatabaseNum + 1
dbs := make([]*model.Database, 0, len)
for i := 0; i < len; i++ {
dbs = append(dbs, model.NewDefaultDatabase())
dbs = append(dbs, model.NewDefaultDatabase(nil))
}
meta.On("ListDatabases",
mock.Anything,
@ -81,7 +81,7 @@ func Test_CreateDBTask_Prepare(t *testing.T) {
meta.On("ListDatabases",
mock.Anything,
mock.Anything).
Return([]*model.Database{model.NewDefaultDatabase()}, nil)
Return([]*model.Database{model.NewDefaultDatabase(nil)}, nil)
core := newTestCore(withMeta(meta), withValidIDAllocator())
paramtable.Get().Save(Params.RootCoordCfg.MaxDatabaseNum.Key, strconv.Itoa(10))

2
internal/rootcoord/describe_db_task_test.go
View File

@ -48,7 +48,7 @@ func Test_describeDatabaseTask_Execute(t *testing.T) {
t.Run("describe with empty database name", func(t *testing.T) {
meta := mockrootcoord.NewIMetaTable(t)
meta.EXPECT().GetDatabaseByName(mock.Anything, mock.Anything, mock.Anything).
Return(model.NewDefaultDatabase(), nil)
Return(model.NewDefaultDatabase(nil), nil)
core := newTestCore(withMeta(meta))
task := &describeDBTask{

4
internal/rootcoord/list_db_task_test.go
View File

@ -58,7 +58,7 @@ func Test_ListDBTask(t *testing.T) {
})
t.Run("ok", func(t *testing.T) {
ret := []*model.Database{model.NewDefaultDatabase()}
ret := []*model.Database{model.NewDefaultDatabase(nil)}
meta := mockrootcoord.NewIMetaTable(t)
meta.On("ListDatabases",
mock.Anything,
@ -89,7 +89,7 @@ func Test_ListDBTask(t *testing.T) {
t.Run("list db with auth", func(t *testing.T) {
Params.Save(Params.CommonCfg.AuthorizationEnabled.Key, "true")
defer Params.Reset(Params.CommonCfg.AuthorizationEnabled.Key)
ret := []*model.Database{model.NewDefaultDatabase()}
ret := []*model.Database{model.NewDefaultDatabase(nil)}
meta := mockrootcoord.NewIMetaTable(t)
core := newTestCore(withMeta(meta))

8
internal/rootcoord/meta_table.go
View File

@ -260,7 +260,13 @@ func (mt *MetaTable) createDefaultDb() error {
return err
}
return mt.createDatabasePrivate(mt.ctx, model.NewDefaultDatabase(), ts)
s := Params.RootCoordCfg.DefaultDBProperties.GetValue()
defaultProperties, err := funcutil.String2KeyValuePair(s)
if err != nil {
return err
}
return mt.createDatabasePrivate(mt.ctx, model.NewDefaultDatabase(defaultProperties), ts)
}
func (mt *MetaTable) CreateDatabase(ctx context.Context, db *model.Database, ts typeutil.Timestamp) error {

26
internal/rootcoord/meta_table_test.go
View File

@ -461,7 +461,7 @@ func TestMetaTable_getCollectionByIDInternal(t *testing.T) {
meta := &MetaTable{
catalog: catalog,
dbName2Meta: map[string]*model.Database{
util.DefaultDBName: model.NewDefaultDatabase(),
util.DefaultDBName: model.NewDefaultDatabase(nil),
},
collID2Meta: map[typeutil.UniqueID]*model.Collection{},
}
@ -481,7 +481,7 @@ func TestMetaTable_getCollectionByIDInternal(t *testing.T) {
meta := &MetaTable{
catalog: catalog,
dbName2Meta: map[string]*model.Database{
util.DefaultDBName: model.NewDefaultDatabase(),
util.DefaultDBName: model.NewDefaultDatabase(nil),
},
collID2Meta: map[typeutil.UniqueID]*model.Collection{},
}
@ -595,7 +595,7 @@ func TestMetaTable_GetCollectionByName(t *testing.T) {
).Return(nil, errors.New("error mock GetCollectionByName"))
meta := &MetaTable{
dbName2Meta: map[string]*model.Database{
util.DefaultDBName: model.NewDefaultDatabase(),
util.DefaultDBName: model.NewDefaultDatabase(nil),
},
names: newNameDb(),
aliases: newNameDb(),
@ -616,7 +616,7 @@ func TestMetaTable_GetCollectionByName(t *testing.T) {
).Return(&model.Collection{State: pb.CollectionState_CollectionDropped}, nil)
meta := &MetaTable{
dbName2Meta: map[string]*model.Database{
util.DefaultDBName: model.NewDefaultDatabase(),
util.DefaultDBName: model.NewDefaultDatabase(nil),
},
names: newNameDb(),
aliases: newNameDb(),
@ -646,7 +646,7 @@ func TestMetaTable_GetCollectionByName(t *testing.T) {
meta := &MetaTable{
dbName2Meta: map[string]*model.Database{
util.DefaultDBName: model.NewDefaultDatabase(),
util.DefaultDBName: model.NewDefaultDatabase(nil),
},
names: newNameDb(),
aliases: newNameDb(),
@ -1177,7 +1177,7 @@ func TestMetaTable_reload(t *testing.T) {
catalog.On("ListDatabases",
mock.Anything,
mock.Anything,
).Return([]*model.Database{model.NewDefaultDatabase()}, nil)
).Return([]*model.Database{model.NewDefaultDatabase(nil)}, nil)
catalog.On("ListCollections",
mock.Anything,
mock.Anything,
@ -1479,7 +1479,7 @@ func TestMetaTable_RenameCollection(t *testing.T) {
t.Run("new collection name already exist-1", func(t *testing.T) {
meta := &MetaTable{
dbName2Meta: map[string]*model.Database{
util.DefaultDBName: model.NewDefaultDatabase(),
util.DefaultDBName: model.NewDefaultDatabase(nil),
},
names: newNameDb(),
aliases: newNameDb(),
@ -1506,7 +1506,7 @@ func TestMetaTable_RenameCollection(t *testing.T) {
).Return(nil, errors.New("error mock GetCollectionByID"))
meta := &MetaTable{
dbName2Meta: map[string]*model.Database{
util.DefaultDBName: model.NewDefaultDatabase(),
util.DefaultDBName: model.NewDefaultDatabase(nil),
},
catalog: catalog,
names: newNameDb(),
@ -1536,7 +1536,7 @@ func TestMetaTable_RenameCollection(t *testing.T) {
meta := &MetaTable{
dbName2Meta: map[string]*model.Database{
util.DefaultDBName: model.NewDefaultDatabase(),
util.DefaultDBName: model.NewDefaultDatabase(nil),
},
catalog: catalog,
names: newNameDb(),
@ -1563,7 +1563,7 @@ func TestMetaTable_RenameCollection(t *testing.T) {
).Return(nil, merr.WrapErrCollectionNotFound("error"))
meta := &MetaTable{
dbName2Meta: map[string]*model.Database{
util.DefaultDBName: model.NewDefaultDatabase(),
util.DefaultDBName: model.NewDefaultDatabase(nil),
"db1": model.NewDatabase(2, "db1", pb.DatabaseState_DatabaseCreated, nil),
},
catalog: catalog,
@ -1593,7 +1593,7 @@ func TestMetaTable_RenameCollection(t *testing.T) {
).Return(nil, merr.WrapErrCollectionNotFound("error"))
meta := &MetaTable{
dbName2Meta: map[string]*model.Database{
util.DefaultDBName: model.NewDefaultDatabase(),
util.DefaultDBName: model.NewDefaultDatabase(nil),
"db1": model.NewDatabase(2, "db1", pb.DatabaseState_DatabaseCreated, nil),
},
catalog: catalog,
@ -1630,7 +1630,7 @@ func TestMetaTable_RenameCollection(t *testing.T) {
).Return(nil, merr.WrapErrCollectionNotFound("error"))
meta := &MetaTable{
dbName2Meta: map[string]*model.Database{
util.DefaultDBName: model.NewDefaultDatabase(),
util.DefaultDBName: model.NewDefaultDatabase(nil),
},
catalog: catalog,
names: newNameDb(),
@ -1893,7 +1893,7 @@ func TestMetaTable_EmtpyDatabaseName(t *testing.T) {
mt := &MetaTable{
names: newNameDb(),
dbName2Meta: map[string]*model.Database{
util.DefaultDBName: model.NewDefaultDatabase(),
util.DefaultDBName: model.NewDefaultDatabase(nil),
"db2": model.NewDatabase(2, "db2", pb.DatabaseState_DatabaseCreated, nil),
},
collID2Meta: map[typeutil.UniqueID]*model.Collection{

9
pkg/util/funcutil/func.go
View File

@ -189,6 +189,15 @@ func GetVecFieldIDs(schema *schemapb.CollectionSchema) []int64 {
return vecFieldIDs
}
func String2KeyValuePair(v string) ([]*commonpb.KeyValuePair, error) {
m := make(map[string]string)
err := json.Unmarshal([]byte(v), &m)
if err != nil {
return nil, err
}
return Map2KeyValuePair(m), nil
}
func Map2KeyValuePair(datas map[string]string) []*commonpb.KeyValuePair {
results := make([]*commonpb.KeyValuePair, len(datas))
offset := 0

21
pkg/util/funcutil/func_test.go
View File

@ -916,3 +916,24 @@ func TestChannelConvert(t *testing.T) {
assert.Equal(t, "by-dev-rootcoord-dml_2_1001v0", channel)
})
}
func TestString2KeyValuePair(t *testing.T) {
t.Run("normal", func(t *testing.T) {
kvs, err := String2KeyValuePair("{\"key\": \"value\"}")
assert.NoError(t, err)
assert.Len(t, kvs, 1)
assert.Equal(t, "key", kvs[0].Key)
assert.Equal(t, "value", kvs[0].Value)
})
t.Run("err", func(t *testing.T) {
_, err := String2KeyValuePair("{aa}")
assert.Error(t, err)
})
t.Run("empty", func(t *testing.T) {
kvs, err := String2KeyValuePair("{}")
assert.NoError(t, err)
assert.Len(t, kvs, 0)
})
}

10
pkg/util/paramtable/component_param.go
View File

@ -1136,6 +1136,7 @@ type rootCoordConfig struct {
MaxGeneralCapacity ParamItem `refreshable:"true"`
GracefulStopTimeout ParamItem `refreshable:"true"`
UseLockScheduler ParamItem `refreshable:"true"`
DefaultDBProperties ParamItem `refreshable:"false"`
}
func (p *rootCoordConfig) init(base *BaseTable) {
@ -1219,6 +1220,15 @@ Segments with smaller size than this parameter will not be indexed, and will be
Export: false,
}
p.UseLockScheduler.Init(base.mgr)
p.DefaultDBProperties = ParamItem{
Key: "rootCoord.defaultDBProperties",
Version: "2.4.16",
DefaultValue: "{}",
Doc: "default db properties, should be a json string",
Export: false,
}
p.DefaultDBProperties.Init(base.mgr)
}
// /////////////////////////////////////////////////////////////////////////////

4
pkg/util/paramtable/component_param_test.go
View File

@ -153,6 +153,10 @@ func TestComponentParam(t *testing.T) {
params.Save("rootCoord.gracefulStopTimeout", "100")
assert.Equal(t, 100*time.Second, Params.GracefulStopTimeout.GetAsDuration(time.Second))
assert.Equal(t, "{}", Params.DefaultDBProperties.GetValue())
params.Save("rootCoord.defaultDBProperties", "{\"key\":\"value\"}")
assert.Equal(t, "{\"key\":\"value\"}", Params.DefaultDBProperties.GetValue())
SetCreateTime(time.Now())
SetUpdateTime(time.Now())
})