milvus-io
/
milvus
mirror of https://github.com/milvus-io/milvus.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Fix spell PQ error (#20725) 

Signed-off-by: xige-16 <xi.ge@zilliz.com>

Signed-off-by: xige-16 <xi.ge@zilliz.com>
pull/20741/head
xige-16 2022-11-21 16:35:11 +08:00 committed by GitHub
parent ec617d4952
commit 81b6703201
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 35 additions and 35 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
configs/milvus.yaml
View File

@ -371,7 +371,7 @@ common:
DiskIndex:
MaxDegree: 56
SearchListSize: 100
PGCodeBudgetGBRatio: 0.125
PQCodeBudgetGBRatio: 0.125
BuildNumThreadsRatio: 1.0
SearchCacheBudgetGBRatio: 0.125
LoadNumThreadRatio: 8.0

14
internal/util/autoindex/bigdata_index.go
View File

@ -22,7 +22,7 @@ import (
)
type BigDataIndexExtraParams struct {
PGCodeBudgetGBRatio float64
PQCodeBudgetGBRatio float64
BuildNumThreadsRatio float64
SearchCacheBudgetGBRatio float64
LoadNumThreadRatio float64
@ -33,7 +33,7 @@ const (
BuildRatioKey = "build_ratio"
PrepareRatioKey = "prepare_ratio"
BeamWidthRatioKey = "beamwidth_ratio"
DefaultPGCodeBudgetGBRatio = 0.125
DefaultPQCodeBudgetGBRatio = 0.125
DefaultBuildNumThreadsRatio = 1.0
DefaultSearchCacheBudgetGBRatio = 0.125
DefaultLoadNumThreadRatio = 8.0
@ -42,7 +42,7 @@ const (
func NewBigDataIndexExtraParams() *BigDataIndexExtraParams {
ret := &BigDataIndexExtraParams{
PGCodeBudgetGBRatio: DefaultPGCodeBudgetGBRatio,
PQCodeBudgetGBRatio: DefaultPQCodeBudgetGBRatio,
BuildNumThreadsRatio: DefaultBuildNumThreadsRatio,
SearchCacheBudgetGBRatio: DefaultSearchCacheBudgetGBRatio,
LoadNumThreadRatio: DefaultLoadNumThreadRatio,
@ -65,7 +65,7 @@ func NewBigDataExtraParamsFromMap(value map[string]string) (*BigDataIndexExtraPa
var err error
buildRatio, ok := value[BuildRatioKey]
if !ok {
ret.PGCodeBudgetGBRatio = DefaultPGCodeBudgetGBRatio
ret.PQCodeBudgetGBRatio = DefaultPQCodeBudgetGBRatio
ret.BuildNumThreadsRatio = DefaultBuildNumThreadsRatio
} else {
valueMap1 := make(map[string]float64)
@ -74,11 +74,11 @@ func NewBigDataExtraParamsFromMap(value map[string]string) (*BigDataIndexExtraPa
return ret, err
}
PGCodeBudgetGBRatio, ok := valueMap1["pg_code_budget_gb"]
PQCodeBudgetGBRatio, ok := valueMap1["pq_code_budget_gb"]
if !ok {
ret.PGCodeBudgetGBRatio = DefaultPGCodeBudgetGBRatio
ret.PQCodeBudgetGBRatio = DefaultPQCodeBudgetGBRatio
} else {
ret.PGCodeBudgetGBRatio = PGCodeBudgetGBRatio
ret.PQCodeBudgetGBRatio = PQCodeBudgetGBRatio
}
BuildNumThreadsRatio, ok := valueMap1["num_threads"]
if !ok {

20
internal/util/autoindex/bigdata_index_test.go
View File

@ -25,13 +25,13 @@ import (
func TestBigDataIndex_parse(t *testing.T) {
t.Run("parse normal", func(t *testing.T) {
mapString := make(map[string]string)
mapString[BuildRatioKey] = "{\"pg_code_budget_gb\": 0.125, \"num_threads\": 1}"
mapString[BuildRatioKey] = "{\"pq_code_budget_gb\": 0.125, \"num_threads\": 1}"
mapString[PrepareRatioKey] = "{\"search_cache_budget_gb\": 0.225, \"num_threads\": 8}"
extraParams, err := NewBigDataExtraParamsFromMap(mapString)
assert.NoError(t, err)
assert.Equal(t, 1.0, extraParams.BuildNumThreadsRatio)
assert.Equal(t, 8.0, extraParams.LoadNumThreadRatio)
assert.Equal(t, 0.125, extraParams.PGCodeBudgetGBRatio)
assert.Equal(t, 0.125, extraParams.PQCodeBudgetGBRatio)
assert.Equal(t, 0.225, extraParams.SearchCacheBudgetGBRatio)
})
@ -42,7 +42,7 @@ func TestBigDataIndex_parse(t *testing.T) {
assert.NoError(t, err)
assert.Equal(t, 1.0, extraParams.BuildNumThreadsRatio)
assert.Equal(t, 8.0, extraParams.LoadNumThreadRatio)
assert.Equal(t, 0.125, extraParams.PGCodeBudgetGBRatio)
assert.Equal(t, 0.125, extraParams.PQCodeBudgetGBRatio)
assert.Equal(t, 0.225, extraParams.SearchCacheBudgetGBRatio)
})
@ -52,7 +52,7 @@ func TestBigDataIndex_parse(t *testing.T) {
assert.NoError(t, err)
assert.Equal(t, 1.0, extraParams.BuildNumThreadsRatio)
assert.Equal(t, 8.0, extraParams.LoadNumThreadRatio)
assert.Equal(t, 0.125, extraParams.PGCodeBudgetGBRatio)
assert.Equal(t, 0.125, extraParams.PQCodeBudgetGBRatio)
assert.Equal(t, 0.125, extraParams.SearchCacheBudgetGBRatio)
})
@ -61,14 +61,14 @@ func TestBigDataIndex_parse(t *testing.T) {
assert.NoError(t, err)
assert.Equal(t, 1.0, extraParams.BuildNumThreadsRatio)
assert.Equal(t, 8.0, extraParams.LoadNumThreadRatio)
assert.Equal(t, 0.125, extraParams.PGCodeBudgetGBRatio)
assert.Equal(t, 0.125, extraParams.PQCodeBudgetGBRatio)
assert.Equal(t, 0.125, extraParams.SearchCacheBudgetGBRatio)
})
t.Run("new from json normal", func(t *testing.T) {
jsonStr := `
{
"build_ratio": "{\"pg_code_budget_gb\": 0.125, \"num_threads\": 1}",
"build_ratio": "{\"pq_code_budget_gb\": 0.125, \"num_threads\": 1}",
"prepare_ratio": "{\"search_cache_budget_gb\": 0.225, \"num_threads\": 8}",
"beamwidth_ratio": "8.0"
}
@ -77,7 +77,7 @@ func TestBigDataIndex_parse(t *testing.T) {
assert.NoError(t, err)
assert.Equal(t, 1.0, extraParams.BuildNumThreadsRatio)
assert.Equal(t, 8.0, extraParams.LoadNumThreadRatio)
assert.Equal(t, 0.125, extraParams.PGCodeBudgetGBRatio)
assert.Equal(t, 0.125, extraParams.PQCodeBudgetGBRatio)
assert.Equal(t, 0.225, extraParams.SearchCacheBudgetGBRatio)
assert.Equal(t, 8.0, extraParams.BeamWidthRatio)
})
@ -85,14 +85,14 @@ func TestBigDataIndex_parse(t *testing.T) {
t.Run("new from json partial", func(t *testing.T) {
jsonStr := `
{
"build_ratio": "{\"pg_code_budget_gb\": 0.125, \"num_threads\": 1}"
"build_ratio": "{\"pq_code_budget_gb\": 0.125, \"num_threads\": 1}"
}
`
extraParams, err := NewBigDataExtraParamsFromJSON(jsonStr)
assert.NoError(t, err)
assert.Equal(t, 1.0, extraParams.BuildNumThreadsRatio)
assert.Equal(t, 8.0, extraParams.LoadNumThreadRatio)
assert.Equal(t, 0.125, extraParams.PGCodeBudgetGBRatio)
assert.Equal(t, 0.125, extraParams.PQCodeBudgetGBRatio)
assert.Equal(t, 0.125, extraParams.SearchCacheBudgetGBRatio)
assert.Equal(t, 4.0, extraParams.BeamWidthRatio)
})
@ -106,7 +106,7 @@ func TestBigDataIndex_parse(t *testing.T) {
assert.NoError(t, err)
assert.Equal(t, 1.0, extraParams.BuildNumThreadsRatio)
assert.Equal(t, 8.0, extraParams.LoadNumThreadRatio)
assert.Equal(t, 0.125, extraParams.PGCodeBudgetGBRatio)
assert.Equal(t, 0.125, extraParams.PQCodeBudgetGBRatio)
assert.Equal(t, 0.125, extraParams.SearchCacheBudgetGBRatio)
assert.Equal(t, 4.0, extraParams.BeamWidthRatio)
})

14
internal/util/indexparams/disk_index_params.go
View File

@ -56,7 +56,7 @@ func getRowDataSizeOfFloatVector(numRows int64, dim int64) int64 {
func FillDiskIndexParams(params *paramtable.ComponentParam, indexParams map[string]string) error {
maxDegree := strconv.FormatInt(params.CommonCfg.MaxDegree, 10)
searchListSize := strconv.FormatInt(params.CommonCfg.SearchListSize, 10)
pgCodeBudgetGBRatio := params.CommonCfg.PGCodeBudgetGBRatio
pqCodeBudgetGBRatio := params.CommonCfg.PQCodeBudgetGBRatio
buildNumThreadsRatio := params.CommonCfg.BuildNumThreadsRatio
searchCacheBudgetGBRatio := params.CommonCfg.SearchCacheBudgetGBRatio
@ -73,7 +73,7 @@ func FillDiskIndexParams(params *paramtable.ComponentParam, indexParams map[stri
if !ok {
return fmt.Errorf("index param search_list_size not exist")
}
pgCodeBudgetGBRatio = params.AutoIndexConfig.BigDataExtraParams.PGCodeBudgetGBRatio
pqCodeBudgetGBRatio = params.AutoIndexConfig.BigDataExtraParams.PQCodeBudgetGBRatio
buildNumThreadsRatio = params.AutoIndexConfig.BigDataExtraParams.BuildNumThreadsRatio
searchCacheBudgetGBRatio = params.AutoIndexConfig.BigDataExtraParams.SearchCacheBudgetGBRatio
loadNumThreadRatio = params.AutoIndexConfig.BigDataExtraParams.LoadNumThreadRatio
@ -82,7 +82,7 @@ func FillDiskIndexParams(params *paramtable.ComponentParam, indexParams map[stri
indexParams[MaxDegreeKey] = maxDegree
indexParams[SearchListSizeKey] = searchListSize
indexParams[PQCodeBudgetRatioKey] = fmt.Sprintf("%f", pgCodeBudgetGBRatio)
indexParams[PQCodeBudgetRatioKey] = fmt.Sprintf("%f", pqCodeBudgetGBRatio)
indexParams[NumBuildThreadRatioKey] = fmt.Sprintf("%f", buildNumThreadsRatio)
indexParams[SearchCacheBudgetRatioKey] = fmt.Sprintf("%f", searchCacheBudgetGBRatio)
indexParams[NumLoadThreadRatioKey] = fmt.Sprintf("%f", loadNumThreadRatio)
@ -104,11 +104,11 @@ func SetDiskIndexBuildParams(indexParams map[string]string, numRows int64) error
return err
}
pgCodeBudgetGBRatioStr, ok := indexParams[PQCodeBudgetRatioKey]
pqCodeBudgetGBRatioStr, ok := indexParams[PQCodeBudgetRatioKey]
if !ok {
return fmt.Errorf("index param pgCodeBudgetGBRatio not exist")
return fmt.Errorf("index param pqCodeBudgetGBRatio not exist")
}
pgCodeBudgetGBRatio, err := strconv.ParseFloat(pgCodeBudgetGBRatioStr, 64)
pqCodeBudgetGBRatio, err := strconv.ParseFloat(pqCodeBudgetGBRatioStr, 64)
if err != nil {
return err
}
@ -122,7 +122,7 @@ func SetDiskIndexBuildParams(indexParams map[string]string, numRows int64) error
}
indexParams[PQCodeBudgetKey] = fmt.Sprintf("%f",
float32(getRowDataSizeOfFloatVector(numRows, dim))*float32(pgCodeBudgetGBRatio)/(1<<30))
float32(getRowDataSizeOfFloatVector(numRows, dim))*float32(pqCodeBudgetGBRatio)/(1<<30))
indexParams[NumBuildThreadKey] = strconv.Itoa(int(float32(hardware.GetCPUNum()) * float32(buildNumThreadsRatio)))
indexParams[BuildDramBudgetKey] = fmt.Sprintf("%f", float32(hardware.GetFreeMemoryCount())/(1<<30))

10
internal/util/indexparams/disk_index_params_test.go
View File

@ -35,9 +35,9 @@ func TestDiskIndexParams(t *testing.T) {
err := FillDiskIndexParams(&params, indexParams)
assert.NoError(t, err)
pgCodeBudgetGBRatio, err := strconv.ParseFloat(indexParams[PQCodeBudgetRatioKey], 64)
pqCodeBudgetGBRatio, err := strconv.ParseFloat(indexParams[PQCodeBudgetRatioKey], 64)
assert.NoError(t, err)
assert.Equal(t, 0.125, pgCodeBudgetGBRatio)
assert.Equal(t, 0.125, pqCodeBudgetGBRatio)
buildNumThreadsRatio, err := strconv.ParseFloat(indexParams[NumBuildThreadRatioKey], 64)
assert.NoError(t, err)
@ -61,7 +61,7 @@ func TestDiskIndexParams(t *testing.T) {
params.AutoIndexConfig.Enable = true
mapString := make(map[string]string)
mapString[autoindex.BuildRatioKey] = "{\"pg_code_budget_gb\": 0.125, \"num_threads\": 1}"
mapString[autoindex.BuildRatioKey] = "{\"pq_code_budget_gb\": 0.125, \"num_threads\": 1}"
mapString[autoindex.PrepareRatioKey] = "{\"search_cache_budget_gb\": 0.225, \"num_threads\": 4}"
extraParams, err := autoindex.NewBigDataExtraParamsFromMap(mapString)
assert.NoError(t, err)
@ -75,9 +75,9 @@ func TestDiskIndexParams(t *testing.T) {
err = FillDiskIndexParams(&params, indexParams)
assert.NoError(t, err)
pgCodeBudgetGBRatio, err := strconv.ParseFloat(indexParams[PQCodeBudgetRatioKey], 64)
pqCodeBudgetGBRatio, err := strconv.ParseFloat(indexParams[PQCodeBudgetRatioKey], 64)
assert.NoError(t, err)
assert.Equal(t, 0.125, pgCodeBudgetGBRatio)
assert.Equal(t, 0.125, pqCodeBudgetGBRatio)
buildNumThreadsRatio, err := strconv.ParseFloat(indexParams[NumBuildThreadRatioKey], 64)
assert.NoError(t, err)

10
internal/util/paramtable/component_param.go
View File

@ -41,7 +41,7 @@ const (
DefaultMaxDegree = 56
DefaultSearchListSize = 100
DefaultPGCodeBudgetGBRatio = 0.125
DefaultPQCodeBudgetGBRatio = 0.125
DefaultBuildNumThreadsRatio = 1.0
DefaultSearchCacheBudgetGBRatio = 0.125
DefaultLoadNumThreadRatio = 8.0
@ -141,7 +141,7 @@ type commonConfig struct {
ThreadCoreCoefficient int64
MaxDegree int64
SearchListSize int64
PGCodeBudgetGBRatio float64
PQCodeBudgetGBRatio float64
BuildNumThreadsRatio float64
SearchCacheBudgetGBRatio float64
LoadNumThreadRatio float64
@ -192,7 +192,7 @@ func (p *commonConfig) init(base *BaseTable) {
p.initIndexSliceSize()
p.initMaxDegree()
p.initSearchListSize()
p.initPGCodeBudgetGBRatio()
p.initPQCodeBudgetGBRatio()
p.initBuildNumThreadsRatio()
p.initSearchCacheBudgetGBRatio()
p.initLoadNumThreadRatio()
@ -402,8 +402,8 @@ func (p *commonConfig) initThreadCoreCoefficient() {
p.ThreadCoreCoefficient = p.Base.ParseInt64WithDefault("common.threadCoreCoefficient", DefaultThreadCoreCoefficient)
}
func (p *commonConfig) initPGCodeBudgetGBRatio() {
p.PGCodeBudgetGBRatio = p.Base.ParseFloatWithDefault("common.DiskIndex.PGCodeBudgetGBRatio", DefaultPGCodeBudgetGBRatio)
func (p *commonConfig) initPQCodeBudgetGBRatio() {
p.PQCodeBudgetGBRatio = p.Base.ParseFloatWithDefault("common.DiskIndex.PQCodeBudgetGBRatio", DefaultPQCodeBudgetGBRatio)
}
func (p *commonConfig) initBuildNumThreadsRatio() {