milvus-io
/
milvus
mirror of https://github.com/milvus-io/milvus.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

fix: can't not get search_cache_budget_gb in create index (#30353) 

issue:https://github.com/milvus-io/milvus/issues/30375
pr: https://github.com/milvus-io/milvus/pull/30119

Signed-off-by: cqy123456 <qianya.cheng@zilliz.com>
pull/30414/head
cqy123456 2024-01-31 15:49:03 +08:00 committed by GitHub
parent 028721db25
commit 3036c19867
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
3 changed files with 40 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
internal/datacoord/index_builder.go
View File

@ -302,7 +302,7 @@ func (ib *indexBuilder) process(buildID UniqueID) bool {
}
if isDiskANNIndex(getIndexType(indexParams)) {
var err error
indexParams, err = indexparams.AppendDiskIndexBuildParams(Params, indexParams)
indexParams, err = indexparams.UpdateDiskIndexBuildParams(Params, indexParams)
if err != nil {
log.Ctx(ib.ctx).Warn("failed to append index build params", zap.Int64("buildID", buildID),
zap.Int64("nodeID", nodeID), zap.Error(err))

47
pkg/util/indexparams/index_params.go
View File

@ -154,11 +154,11 @@ func NewBigDataExtraParamsFromMap(value map[string]string) (*BigDataIndexExtraPa
// FillDiskIndexParams fill ratio params to index param on proxy node
// Which will be used to calculate build and load params
func FillDiskIndexParams(params *paramtable.ComponentParam, indexParams map[string]string) error {
maxDegree := params.CommonCfg.MaxDegree.GetValue()
searchListSize := params.CommonCfg.SearchListSize.GetValue()
pqCodeBudgetGBRatio := params.CommonCfg.PQCodeBudgetGBRatio.GetValue()
buildNumThreadsRatio := params.CommonCfg.BuildNumThreadsRatio.GetValue()
searchCacheBudgetGBRatio := params.CommonCfg.SearchCacheBudgetGBRatio.GetValue()
var maxDegree string
var searchListSize string
var pqCodeBudgetGBRatio string
var buildNumThreadsRatio string
var searchCacheBudgetGBRatio string
if params.AutoIndexConfig.Enable.GetAsBool() {
indexParams := params.AutoIndexConfig.IndexParams.GetAsJSONMap()
@ -177,6 +177,13 @@ func FillDiskIndexParams(params *paramtable.ComponentParam, indexParams map[stri
}
pqCodeBudgetGBRatio = fmt.Sprintf("%f", extraParams.PQCodeBudgetGBRatio)
buildNumThreadsRatio = fmt.Sprintf("%f", extraParams.BuildNumThreadsRatio)
searchCacheBudgetGBRatio = fmt.Sprintf("%f", extraParams.SearchCacheBudgetGBRatio)
} else {
maxDegree = params.CommonCfg.MaxDegree.GetValue()
searchListSize = params.CommonCfg.SearchListSize.GetValue()
pqCodeBudgetGBRatio = params.CommonCfg.PQCodeBudgetGBRatio.GetValue()
buildNumThreadsRatio = params.CommonCfg.BuildNumThreadsRatio.GetValue()
searchCacheBudgetGBRatio = params.CommonCfg.SearchCacheBudgetGBRatio.GetValue()
}
indexParams[MaxDegreeKey] = maxDegree
@ -197,12 +204,9 @@ func GetIndexParams(indexParams []*commonpb.KeyValuePair, key string) string {
return ""
}
// AppendDiskIndexBuildParams append index params for `buildIndex` (params not exist in `CreateIndex`)
func AppendDiskIndexBuildParams(params *paramtable.ComponentParam, indexParams []*commonpb.KeyValuePair) ([]*commonpb.KeyValuePair, error) {
// UpdateDiskIndexBuildParams update index params for `buildIndex` (override search cache size in `CreateIndex`)
func UpdateDiskIndexBuildParams(params *paramtable.ComponentParam, indexParams []*commonpb.KeyValuePair) ([]*commonpb.KeyValuePair, error) {
existedVal := GetIndexParams(indexParams, SearchCacheBudgetRatioKey)
if len(existedVal) > 0 {
return indexParams, nil
}
var searchCacheBudgetGBRatio string
if params.AutoIndexConfig.Enable.GetAsBool() {
@ -219,14 +223,33 @@ func AppendDiskIndexBuildParams(params *paramtable.ComponentParam, indexParams [
searchCacheBudgetGBRatio = fmt.Sprintf("%f", paramVal)
}
if len(searchCacheBudgetGBRatio) > 0 {
// append when not exist
if len(existedVal) == 0 {
indexParams = append(indexParams,
&commonpb.KeyValuePair{
Key: SearchCacheBudgetRatioKey,
Value: searchCacheBudgetGBRatio,
})
return indexParams, nil
}
return indexParams, nil
// override when exist
updatedParams := make([]*commonpb.KeyValuePair, 0, len(indexParams))
for _, param := range indexParams {
if param.Key == SearchCacheBudgetRatioKey {
updatedParams = append(updatedParams,
&commonpb.KeyValuePair{
Key: SearchCacheBudgetRatioKey,
Value: searchCacheBudgetGBRatio,
})
} else {
updatedParams = append(updatedParams,
&commonpb.KeyValuePair{
Key: param.Key,
Value: param.Value,
})
}
}
return updatedParams, nil
}
// SetDiskIndexBuildParams set index build params with ratio params on indexNode

8
pkg/util/indexparams/index_params_test.go
View File

@ -149,7 +149,7 @@ func TestDiskIndexParams(t *testing.T) {
Value: "4.0",
})
indexParams, err := AppendDiskIndexBuildParams(&params, indexParams)
indexParams, err := UpdateDiskIndexBuildParams(&params, indexParams)
assert.NoError(t, err)
assert.True(t, len(indexParams) == 4)
@ -191,7 +191,7 @@ func TestDiskIndexParams(t *testing.T) {
Value: "4.0",
})
autoParams, err = AppendDiskIndexBuildParams(&params, autoParams)
autoParams, err = UpdateDiskIndexBuildParams(&params, autoParams)
assert.NoError(t, err)
assert.True(t, len(autoParams) == 4)
@ -208,7 +208,7 @@ func TestDiskIndexParams(t *testing.T) {
}
`
params.Save(params.AutoIndexConfig.ExtraParams.Key, newJSONStr)
autoParams, err = AppendDiskIndexBuildParams(&params, autoParams)
autoParams, err = UpdateDiskIndexBuildParams(&params, autoParams)
assert.NoError(t, err)
assert.True(t, len(autoParams) == 4)
@ -216,7 +216,7 @@ func TestDiskIndexParams(t *testing.T) {
val = GetIndexParams(autoParams, SearchCacheBudgetRatioKey)
iVal, iErr = strconv.ParseFloat(val, 64)
assert.NoError(t, iErr)
assert.Equal(t, 0.225, iVal)
assert.Equal(t, 0.325, iVal)
})
t.Run("set disk index build params", func(t *testing.T) {