diff --git a/internal/datacoord/index_builder.go b/internal/datacoord/index_builder.go index c2561a3d32..97aae40a01 100644 --- a/internal/datacoord/index_builder.go +++ b/internal/datacoord/index_builder.go @@ -302,7 +302,7 @@ func (ib *indexBuilder) process(buildID UniqueID) bool { } if isDiskANNIndex(getIndexType(indexParams)) { var err error - indexParams, err = indexparams.AppendDiskIndexBuildParams(Params, indexParams) + indexParams, err = indexparams.UpdateDiskIndexBuildParams(Params, indexParams) if err != nil { log.Ctx(ib.ctx).Warn("failed to append index build params", zap.Int64("buildID", buildID), zap.Int64("nodeID", nodeID), zap.Error(err)) diff --git a/pkg/util/indexparams/index_params.go b/pkg/util/indexparams/index_params.go index a07d67b669..ecdcd185ba 100644 --- a/pkg/util/indexparams/index_params.go +++ b/pkg/util/indexparams/index_params.go @@ -154,11 +154,11 @@ func NewBigDataExtraParamsFromMap(value map[string]string) (*BigDataIndexExtraPa // FillDiskIndexParams fill ratio params to index param on proxy node // Which will be used to calculate build and load params func FillDiskIndexParams(params *paramtable.ComponentParam, indexParams map[string]string) error { - maxDegree := params.CommonCfg.MaxDegree.GetValue() - searchListSize := params.CommonCfg.SearchListSize.GetValue() - pqCodeBudgetGBRatio := params.CommonCfg.PQCodeBudgetGBRatio.GetValue() - buildNumThreadsRatio := params.CommonCfg.BuildNumThreadsRatio.GetValue() - searchCacheBudgetGBRatio := params.CommonCfg.SearchCacheBudgetGBRatio.GetValue() + var maxDegree string + var searchListSize string + var pqCodeBudgetGBRatio string + var buildNumThreadsRatio string + var searchCacheBudgetGBRatio string if params.AutoIndexConfig.Enable.GetAsBool() { indexParams := params.AutoIndexConfig.IndexParams.GetAsJSONMap() @@ -177,6 +177,13 @@ func FillDiskIndexParams(params *paramtable.ComponentParam, indexParams map[stri } pqCodeBudgetGBRatio = fmt.Sprintf("%f", extraParams.PQCodeBudgetGBRatio) buildNumThreadsRatio = fmt.Sprintf("%f", extraParams.BuildNumThreadsRatio) + searchCacheBudgetGBRatio = fmt.Sprintf("%f", extraParams.SearchCacheBudgetGBRatio) + } else { + maxDegree = params.CommonCfg.MaxDegree.GetValue() + searchListSize = params.CommonCfg.SearchListSize.GetValue() + pqCodeBudgetGBRatio = params.CommonCfg.PQCodeBudgetGBRatio.GetValue() + buildNumThreadsRatio = params.CommonCfg.BuildNumThreadsRatio.GetValue() + searchCacheBudgetGBRatio = params.CommonCfg.SearchCacheBudgetGBRatio.GetValue() } indexParams[MaxDegreeKey] = maxDegree @@ -197,12 +204,9 @@ func GetIndexParams(indexParams []*commonpb.KeyValuePair, key string) string { return "" } -// AppendDiskIndexBuildParams append index params for `buildIndex` (params not exist in `CreateIndex`) -func AppendDiskIndexBuildParams(params *paramtable.ComponentParam, indexParams []*commonpb.KeyValuePair) ([]*commonpb.KeyValuePair, error) { +// UpdateDiskIndexBuildParams update index params for `buildIndex` (override search cache size in `CreateIndex`) +func UpdateDiskIndexBuildParams(params *paramtable.ComponentParam, indexParams []*commonpb.KeyValuePair) ([]*commonpb.KeyValuePair, error) { existedVal := GetIndexParams(indexParams, SearchCacheBudgetRatioKey) - if len(existedVal) > 0 { - return indexParams, nil - } var searchCacheBudgetGBRatio string if params.AutoIndexConfig.Enable.GetAsBool() { @@ -219,14 +223,33 @@ func AppendDiskIndexBuildParams(params *paramtable.ComponentParam, indexParams [ searchCacheBudgetGBRatio = fmt.Sprintf("%f", paramVal) } - if len(searchCacheBudgetGBRatio) > 0 { + // append when not exist + if len(existedVal) == 0 { indexParams = append(indexParams, &commonpb.KeyValuePair{ Key: SearchCacheBudgetRatioKey, Value: searchCacheBudgetGBRatio, }) + return indexParams, nil } - return indexParams, nil + // override when exist + updatedParams := make([]*commonpb.KeyValuePair, 0, len(indexParams)) + for _, param := range indexParams { + if param.Key == SearchCacheBudgetRatioKey { + updatedParams = append(updatedParams, + &commonpb.KeyValuePair{ + Key: SearchCacheBudgetRatioKey, + Value: searchCacheBudgetGBRatio, + }) + } else { + updatedParams = append(updatedParams, + &commonpb.KeyValuePair{ + Key: param.Key, + Value: param.Value, + }) + } + } + return updatedParams, nil } // SetDiskIndexBuildParams set index build params with ratio params on indexNode diff --git a/pkg/util/indexparams/index_params_test.go b/pkg/util/indexparams/index_params_test.go index ac85303ceb..9b9030ff30 100644 --- a/pkg/util/indexparams/index_params_test.go +++ b/pkg/util/indexparams/index_params_test.go @@ -149,7 +149,7 @@ func TestDiskIndexParams(t *testing.T) { Value: "4.0", }) - indexParams, err := AppendDiskIndexBuildParams(¶ms, indexParams) + indexParams, err := UpdateDiskIndexBuildParams(¶ms, indexParams) assert.NoError(t, err) assert.True(t, len(indexParams) == 4) @@ -191,7 +191,7 @@ func TestDiskIndexParams(t *testing.T) { Value: "4.0", }) - autoParams, err = AppendDiskIndexBuildParams(¶ms, autoParams) + autoParams, err = UpdateDiskIndexBuildParams(¶ms, autoParams) assert.NoError(t, err) assert.True(t, len(autoParams) == 4) @@ -208,7 +208,7 @@ func TestDiskIndexParams(t *testing.T) { } ` params.Save(params.AutoIndexConfig.ExtraParams.Key, newJSONStr) - autoParams, err = AppendDiskIndexBuildParams(¶ms, autoParams) + autoParams, err = UpdateDiskIndexBuildParams(¶ms, autoParams) assert.NoError(t, err) assert.True(t, len(autoParams) == 4) @@ -216,7 +216,7 @@ func TestDiskIndexParams(t *testing.T) { val = GetIndexParams(autoParams, SearchCacheBudgetRatioKey) iVal, iErr = strconv.ParseFloat(val, 64) assert.NoError(t, iErr) - assert.Equal(t, 0.225, iVal) + assert.Equal(t, 0.325, iVal) }) t.Run("set disk index build params", func(t *testing.T) {