mirror of https://github.com/milvus-io/milvus.git
enhance: [2.4] autoindex for multi data type (#33867)
issue: #22837 pr: https://github.com/milvus-io/milvus/pull/33868 - opensource autoindex support - metric type check for different data types - autoindex data type for search param Signed-off-by: chasingegg <chao.gao@zilliz.com>pull/33899/head
parent
59144d8246
commit
5fc1370f6f
|
@ -200,9 +200,7 @@ func (cit *createIndexTask) parseIndexParams() error {
|
|||
indexParamsMap[common.MetricTypeKey] = metricType
|
||||
}
|
||||
} else { // behavior change after 2.2.9, adapt autoindex logic here.
|
||||
autoIndexConfig := Params.AutoIndexConfig.IndexParams.GetAsJSONMap()
|
||||
|
||||
useAutoIndex := func() {
|
||||
useAutoIndex := func(autoIndexConfig map[string]string) {
|
||||
fields := make([]zap.Field, 0, len(autoIndexConfig))
|
||||
for k, v := range autoIndexConfig {
|
||||
indexParamsMap[k] = v
|
||||
|
@ -211,13 +209,13 @@ func (cit *createIndexTask) parseIndexParams() error {
|
|||
log.Ctx(cit.ctx).Info("AutoIndex triggered", fields...)
|
||||
}
|
||||
|
||||
handle := func(numberParams int) error {
|
||||
handle := func(numberParams int, autoIndexConfig map[string]string) error {
|
||||
// empty case.
|
||||
if len(indexParamsMap) == numberParams {
|
||||
// though we already know there must be metric type, how to make this safer to avoid crash?
|
||||
metricType := autoIndexConfig[common.MetricTypeKey]
|
||||
cit.newExtraParams = wrapUserIndexParams(metricType)
|
||||
useAutoIndex()
|
||||
useAutoIndex(autoIndexConfig)
|
||||
return nil
|
||||
}
|
||||
|
||||
|
@ -234,7 +232,7 @@ func (cit *createIndexTask) parseIndexParams() error {
|
|||
|
||||
// only metric type is passed.
|
||||
cit.newExtraParams = wrapUserIndexParams(metricType)
|
||||
useAutoIndex()
|
||||
useAutoIndex(autoIndexConfig)
|
||||
// make the users' metric type first class citizen.
|
||||
indexParamsMap[common.MetricTypeKey] = metricType
|
||||
}
|
||||
|
@ -242,12 +240,23 @@ func (cit *createIndexTask) parseIndexParams() error {
|
|||
return nil
|
||||
}
|
||||
|
||||
var config map[string]string
|
||||
if typeutil.IsDenseFloatVectorType(cit.fieldSchema.DataType) {
|
||||
// override float vector index params by autoindex
|
||||
config = Params.AutoIndexConfig.IndexParams.GetAsJSONMap()
|
||||
} else if typeutil.IsSparseFloatVectorType(cit.fieldSchema.DataType) {
|
||||
// override sparse float vector index params by autoindex
|
||||
config = Params.AutoIndexConfig.SparseIndexParams.GetAsJSONMap()
|
||||
} else if typeutil.IsBinaryVectorType(cit.fieldSchema.DataType) {
|
||||
// override binary vector index params by autoindex
|
||||
config = Params.AutoIndexConfig.BinaryIndexParams.GetAsJSONMap()
|
||||
}
|
||||
if !exist {
|
||||
if err := handle(0); err != nil {
|
||||
if err := handle(0, config); err != nil {
|
||||
return err
|
||||
}
|
||||
} else if specifyIndexType == AutoIndexName {
|
||||
if err := handle(1); err != nil {
|
||||
if err := handle(1, config); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
@ -263,10 +272,21 @@ func (cit *createIndexTask) parseIndexParams() error {
|
|||
return err
|
||||
}
|
||||
}
|
||||
if indexType == indexparamcheck.IndexSparseInverted || indexType == indexparamcheck.IndexSparseWand {
|
||||
metricType, metricTypeExist := indexParamsMap[common.MetricTypeKey]
|
||||
if !metricTypeExist || metricType != metric.IP {
|
||||
return fmt.Errorf("only IP is the supported metric type for sparse index")
|
||||
metricType, metricTypeExist := indexParamsMap[common.MetricTypeKey]
|
||||
if !metricTypeExist {
|
||||
return merr.WrapErrParameterInvalid("valid index params", "invalid index params", "metric type not set for vector index")
|
||||
}
|
||||
if typeutil.IsDenseFloatVectorType(cit.fieldSchema.DataType) {
|
||||
if !funcutil.SliceContain(indexparamcheck.FloatVectorMetrics, metricType) {
|
||||
return merr.WrapErrParameterInvalid("valid index params", "invalid index params", "float vector index does not support metric type: "+metricType)
|
||||
}
|
||||
} else if typeutil.IsSparseFloatVectorType(cit.fieldSchema.DataType) {
|
||||
if metricType != metric.IP {
|
||||
return merr.WrapErrParameterInvalid("valid index params", "invalid index params", "only IP is the supported metric type for sparse index")
|
||||
}
|
||||
} else if typeutil.IsBinaryVectorType(cit.fieldSchema.DataType) {
|
||||
if !funcutil.SliceContain(indexparamcheck.BinaryVectorMetrics, metricType) {
|
||||
return merr.WrapErrParameterInvalid("valid index params", "invalid index params", "binary vector index does not support metric type: "+metricType)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
@ -970,8 +970,8 @@ func Test_parseIndexParams_AutoIndex_WithType(t *testing.T) {
|
|||
Params.AutoIndexConfig.Enable.Init(mgr)
|
||||
|
||||
mgr.SetConfig("autoIndex.params.build", `{"M": 30,"efConstruction": 360,"index_type": "HNSW"}`)
|
||||
mgr.SetConfig("autoIndex.params.sparsebuild", `{"drop_ratio_build": 0.2, "index_type": "SPARSE_INVERTED_INDEX"}`)
|
||||
mgr.SetConfig("autoIndex.params.binarybuild", `{"nlist": 1024, "index_type": "BIN_IVF_FLAT"}`)
|
||||
mgr.SetConfig("autoIndex.params.sparse.build", `{"drop_ratio_build": 0.2, "index_type": "SPARSE_INVERTED_INDEX"}`)
|
||||
mgr.SetConfig("autoIndex.params.binary.build", `{"nlist": 1024, "index_type": "BIN_IVF_FLAT"}`)
|
||||
Params.AutoIndexConfig.IndexParams.Init(mgr)
|
||||
Params.AutoIndexConfig.SparseIndexParams.Init(mgr)
|
||||
Params.AutoIndexConfig.BinaryIndexParams.Init(mgr)
|
||||
|
@ -1057,9 +1057,15 @@ func Test_parseIndexParams_AutoIndex(t *testing.T) {
|
|||
mgr := config.NewManager()
|
||||
mgr.SetConfig("autoIndex.enable", "false")
|
||||
mgr.SetConfig("autoIndex.params.build", `{"M": 30,"efConstruction": 360,"index_type": "HNSW", "metric_type": "IP"}`)
|
||||
mgr.SetConfig("autoIndex.params.binary.build", `{"nlist": 1024, "index_type": "BIN_IVF_FLAT", "metric_type": "JACCARD"}`)
|
||||
mgr.SetConfig("autoIndex.params.sparse.build", `{"index_type": "SPARSE_INVERTED_INDEX", "metric_type": "IP"}`)
|
||||
Params.AutoIndexConfig.Enable.Init(mgr)
|
||||
Params.AutoIndexConfig.IndexParams.Init(mgr)
|
||||
Params.AutoIndexConfig.BinaryIndexParams.Init(mgr)
|
||||
Params.AutoIndexConfig.SparseIndexParams.Init(mgr)
|
||||
autoIndexConfig := Params.AutoIndexConfig.IndexParams.GetAsJSONMap()
|
||||
autoIndexConfigBinary := Params.AutoIndexConfig.BinaryIndexParams.GetAsJSONMap()
|
||||
autoIndexConfigSparse := Params.AutoIndexConfig.SparseIndexParams.GetAsJSONMap()
|
||||
fieldSchema := &schemapb.FieldSchema{
|
||||
DataType: schemapb.DataType_FloatVector,
|
||||
TypeParams: []*commonpb.KeyValuePair{
|
||||
|
@ -1067,7 +1073,48 @@ func Test_parseIndexParams_AutoIndex(t *testing.T) {
|
|||
},
|
||||
}
|
||||
|
||||
t.Run("case 1, empty parameters", func(t *testing.T) {
|
||||
fieldSchemaBinary := &schemapb.FieldSchema{
|
||||
DataType: schemapb.DataType_BinaryVector,
|
||||
TypeParams: []*commonpb.KeyValuePair{
|
||||
{Key: common.DimKey, Value: "8"},
|
||||
},
|
||||
}
|
||||
|
||||
fieldSchemaSparse := &schemapb.FieldSchema{
|
||||
DataType: schemapb.DataType_SparseFloatVector,
|
||||
}
|
||||
|
||||
t.Run("case 1, empty parameters binary", func(t *testing.T) {
|
||||
task := &createIndexTask{
|
||||
fieldSchema: fieldSchemaBinary,
|
||||
req: &milvuspb.CreateIndexRequest{
|
||||
ExtraParams: make([]*commonpb.KeyValuePair, 0),
|
||||
},
|
||||
}
|
||||
err := task.parseIndexParams()
|
||||
assert.NoError(t, err)
|
||||
assert.ElementsMatch(t, []*commonpb.KeyValuePair{
|
||||
{Key: common.IndexTypeKey, Value: AutoIndexName},
|
||||
{Key: common.MetricTypeKey, Value: autoIndexConfigBinary[common.MetricTypeKey]},
|
||||
}, task.newExtraParams)
|
||||
})
|
||||
|
||||
t.Run("case 1, empty parameters sparse", func(t *testing.T) {
|
||||
task := &createIndexTask{
|
||||
fieldSchema: fieldSchemaSparse,
|
||||
req: &milvuspb.CreateIndexRequest{
|
||||
ExtraParams: make([]*commonpb.KeyValuePair, 0),
|
||||
},
|
||||
}
|
||||
err := task.parseIndexParams()
|
||||
assert.NoError(t, err)
|
||||
assert.ElementsMatch(t, []*commonpb.KeyValuePair{
|
||||
{Key: common.IndexTypeKey, Value: AutoIndexName},
|
||||
{Key: common.MetricTypeKey, Value: autoIndexConfigSparse[common.MetricTypeKey]},
|
||||
}, task.newExtraParams)
|
||||
})
|
||||
|
||||
t.Run("case 1, empty parameters float vector", func(t *testing.T) {
|
||||
task := &createIndexTask{
|
||||
fieldSchema: fieldSchema,
|
||||
req: &milvuspb.CreateIndexRequest{
|
||||
|
|
|
@ -62,6 +62,7 @@ func OptimizeSearchParams(ctx context.Context, req *querypb.SearchRequest, query
|
|||
common.SearchParamKey: queryInfo.GetSearchParams(),
|
||||
common.SegmentNumKey: estSegmentNum,
|
||||
common.WithFilterKey: withFilter,
|
||||
common.DataTypeKey: plan.GetVectorAnns().GetVectorType(),
|
||||
common.WithOptimizeKey: paramtable.Get().AutoIndexConfig.EnableOptimize.GetAsBool(),
|
||||
common.CollectionKey: req.GetReq().GetCollectionID(),
|
||||
}
|
||||
|
|
|
@ -99,6 +99,7 @@ const (
|
|||
SearchParamKey = "search_param"
|
||||
SegmentNumKey = "segment_num"
|
||||
WithFilterKey = "with_filter"
|
||||
DataTypeKey = "data_type"
|
||||
WithOptimizeKey = "with_optimize"
|
||||
CollectionKey = "collection"
|
||||
|
||||
|
|
|
@ -59,7 +59,7 @@ func (c baseChecker) CheckValidDataType(dType schemapb.DataType) error {
|
|||
return nil
|
||||
}
|
||||
|
||||
func (c baseChecker) SetDefaultMetricTypeIfNotExist(m map[string]string) {}
|
||||
func (c baseChecker) SetDefaultMetricTypeIfNotExist(m map[string]string, dType schemapb.DataType) {}
|
||||
|
||||
func (c baseChecker) StaticCheck(params map[string]string) error {
|
||||
return errors.New("unsupported index type")
|
||||
|
|
|
@ -34,7 +34,7 @@ func (c binaryVectorBaseChecker) CheckValidDataType(dType schemapb.DataType) err
|
|||
return nil
|
||||
}
|
||||
|
||||
func (c binaryVectorBaseChecker) SetDefaultMetricTypeIfNotExist(params map[string]string) {
|
||||
func (c binaryVectorBaseChecker) SetDefaultMetricTypeIfNotExist(params map[string]string, dType schemapb.DataType) {
|
||||
setDefaultIfNotExist(params, common.MetricTypeKey, BinaryVectorDefaultMetricType)
|
||||
}
|
||||
|
||||
|
|
|
@ -46,8 +46,10 @@ const (
|
|||
SparseDropRatioBuild = "drop_ratio_build"
|
||||
)
|
||||
|
||||
// METRICS is a set of all metrics types supported for float vector.
|
||||
var METRICS = []string{metric.L2, metric.IP, metric.COSINE} // const
|
||||
var (
|
||||
FloatVectorMetrics = []string{metric.L2, metric.IP, metric.COSINE} // const
|
||||
BinaryVectorMetrics = []string{metric.HAMMING, metric.JACCARD, metric.SUBSTRUCTURE, metric.SUPERSTRUCTURE} // const
|
||||
)
|
||||
|
||||
// BinIDMapMetrics is a set of all metric types supported for binary vector.
|
||||
var (
|
||||
|
|
|
@ -13,8 +13,8 @@ type floatVectorBaseChecker struct {
|
|||
}
|
||||
|
||||
func (c floatVectorBaseChecker) staticCheck(params map[string]string) error {
|
||||
if !CheckStrByValues(params, Metric, METRICS) {
|
||||
return fmt.Errorf("metric type %s not found or not supported, supported: %v", params[Metric], METRICS)
|
||||
if !CheckStrByValues(params, Metric, FloatVectorMetrics) {
|
||||
return fmt.Errorf("metric type %s not found or not supported, supported: %v", params[Metric], FloatVectorMetrics)
|
||||
}
|
||||
|
||||
return nil
|
||||
|
@ -35,7 +35,7 @@ func (c floatVectorBaseChecker) CheckValidDataType(dType schemapb.DataType) erro
|
|||
return nil
|
||||
}
|
||||
|
||||
func (c floatVectorBaseChecker) SetDefaultMetricTypeIfNotExist(params map[string]string) {
|
||||
func (c floatVectorBaseChecker) SetDefaultMetricTypeIfNotExist(params map[string]string, dType schemapb.DataType) {
|
||||
setDefaultIfNotExist(params, common.MetricTypeKey, FloatVectorDefaultMetricType)
|
||||
}
|
||||
|
||||
|
|
|
@ -4,11 +4,12 @@ import (
|
|||
"fmt"
|
||||
|
||||
"github.com/milvus-io/milvus-proto/go-api/v2/schemapb"
|
||||
"github.com/milvus-io/milvus/pkg/common"
|
||||
"github.com/milvus-io/milvus/pkg/util/typeutil"
|
||||
)
|
||||
|
||||
type hnswChecker struct {
|
||||
floatVectorBaseChecker
|
||||
baseChecker
|
||||
}
|
||||
|
||||
func (c hnswChecker) StaticCheck(params map[string]string) error {
|
||||
|
@ -38,6 +39,16 @@ func (c hnswChecker) CheckValidDataType(dType schemapb.DataType) error {
|
|||
return nil
|
||||
}
|
||||
|
||||
func (c hnswChecker) SetDefaultMetricTypeIfNotExist(params map[string]string, dType schemapb.DataType) {
|
||||
if typeutil.IsDenseFloatVectorType(dType) {
|
||||
setDefaultIfNotExist(params, common.MetricTypeKey, FloatVectorDefaultMetricType)
|
||||
} else if typeutil.IsSparseFloatVectorType(dType) {
|
||||
setDefaultIfNotExist(params, common.MetricTypeKey, SparseFloatVectorDefaultMetricType)
|
||||
} else if typeutil.IsBinaryVectorType(dType) {
|
||||
setDefaultIfNotExist(params, common.MetricTypeKey, BinaryVectorDefaultMetricType)
|
||||
}
|
||||
}
|
||||
|
||||
func newHnswChecker() IndexChecker {
|
||||
return &hnswChecker{}
|
||||
}
|
||||
|
|
|
@ -172,3 +172,42 @@ func Test_hnswChecker_CheckValidDataType(t *testing.T) {
|
|||
}
|
||||
}
|
||||
}
|
||||
|
||||
func Test_hnswChecker_SetDefaultMetricType(t *testing.T) {
|
||||
cases := []struct {
|
||||
dType schemapb.DataType
|
||||
metricType string
|
||||
}{
|
||||
{
|
||||
dType: schemapb.DataType_FloatVector,
|
||||
metricType: metric.IP,
|
||||
},
|
||||
{
|
||||
dType: schemapb.DataType_Float16Vector,
|
||||
metricType: metric.IP,
|
||||
},
|
||||
{
|
||||
dType: schemapb.DataType_BFloat16Vector,
|
||||
metricType: metric.IP,
|
||||
},
|
||||
{
|
||||
dType: schemapb.DataType_SparseFloatVector,
|
||||
metricType: metric.IP,
|
||||
},
|
||||
{
|
||||
dType: schemapb.DataType_BinaryVector,
|
||||
metricType: metric.JACCARD,
|
||||
},
|
||||
}
|
||||
|
||||
c := newHnswChecker()
|
||||
for _, test := range cases {
|
||||
p := map[string]string{
|
||||
DIM: strconv.Itoa(128),
|
||||
HNSWM: strconv.Itoa(16),
|
||||
EFConstruction: strconv.Itoa(200),
|
||||
}
|
||||
c.SetDefaultMetricTypeIfNotExist(p, test.dType)
|
||||
assert.Equal(t, p[Metric], test.metricType)
|
||||
}
|
||||
}
|
||||
|
|
|
@ -23,6 +23,6 @@ import (
|
|||
type IndexChecker interface {
|
||||
CheckTrain(map[string]string) error
|
||||
CheckValidDataType(dType schemapb.DataType) error
|
||||
SetDefaultMetricTypeIfNotExist(map[string]string)
|
||||
SetDefaultMetricTypeIfNotExist(map[string]string, schemapb.DataType)
|
||||
StaticCheck(map[string]string) error
|
||||
}
|
||||
|
|
|
@ -39,7 +39,7 @@ func (c sparseFloatVectorBaseChecker) CheckValidDataType(dType schemapb.DataType
|
|||
return nil
|
||||
}
|
||||
|
||||
func (c sparseFloatVectorBaseChecker) SetDefaultMetricTypeIfNotExist(params map[string]string) {
|
||||
func (c sparseFloatVectorBaseChecker) SetDefaultMetricTypeIfNotExist(params map[string]string, dType schemapb.DataType) {
|
||||
setDefaultIfNotExist(params, common.MetricTypeKey, SparseFloatVectorDefaultMetricType)
|
||||
}
|
||||
|
||||
|
|
|
@ -19,6 +19,7 @@ package paramtable
|
|||
import (
|
||||
"fmt"
|
||||
|
||||
"github.com/milvus-io/milvus-proto/go-api/v2/schemapb"
|
||||
"github.com/milvus-io/milvus/pkg/common"
|
||||
"github.com/milvus-io/milvus/pkg/config"
|
||||
"github.com/milvus-io/milvus/pkg/util/funcutil"
|
||||
|
@ -193,31 +194,36 @@ func (p *autoIndexConfig) init(base *BaseTable) {
|
|||
}
|
||||
|
||||
func (p *autoIndexConfig) panicIfNotValidAndSetDefaultMetricType(mgr *config.Manager) {
|
||||
m := p.IndexParams.GetAsJSONMap()
|
||||
p.panicIfNotValidAndSetDefaultMetricTypeHelper(p.IndexParams.Key, p.IndexParams.GetAsJSONMap(), schemapb.DataType_FloatVector, mgr)
|
||||
p.panicIfNotValidAndSetDefaultMetricTypeHelper(p.BinaryIndexParams.Key, p.BinaryIndexParams.GetAsJSONMap(), schemapb.DataType_BinaryVector, mgr)
|
||||
p.panicIfNotValidAndSetDefaultMetricTypeHelper(p.SparseIndexParams.Key, p.SparseIndexParams.GetAsJSONMap(), schemapb.DataType_SparseFloatVector, mgr)
|
||||
}
|
||||
|
||||
func (p *autoIndexConfig) panicIfNotValidAndSetDefaultMetricTypeHelper(key string, m map[string]string, dtype schemapb.DataType, mgr *config.Manager) {
|
||||
if m == nil {
|
||||
panic("autoIndex.build not invalid, should be json format")
|
||||
panic(fmt.Sprintf("%s invalid, should be json format", key))
|
||||
}
|
||||
|
||||
indexType, ok := m[common.IndexTypeKey]
|
||||
if !ok {
|
||||
panic("autoIndex.build not invalid, index type not found")
|
||||
panic(fmt.Sprintf("%s invalid, index type not found", key))
|
||||
}
|
||||
|
||||
checker, err := indexparamcheck.GetIndexCheckerMgrInstance().GetChecker(indexType)
|
||||
if err != nil {
|
||||
panic(fmt.Sprintf("autoIndex.build not invalid, unsupported index type: %s", indexType))
|
||||
panic(fmt.Sprintf("%s invalid, unsupported index type: %s", key, indexType))
|
||||
}
|
||||
|
||||
checker.SetDefaultMetricTypeIfNotExist(m)
|
||||
checker.SetDefaultMetricTypeIfNotExist(m, dtype)
|
||||
|
||||
if err := checker.StaticCheck(m); err != nil {
|
||||
panic(fmt.Sprintf("autoIndex.build not invalid, parameters not invalid, error: %s", err.Error()))
|
||||
panic(fmt.Sprintf("%s invalid, parameters invalid, error: %s", key, err.Error()))
|
||||
}
|
||||
|
||||
p.reset(m, mgr)
|
||||
p.reset(key, m, mgr)
|
||||
}
|
||||
|
||||
func (p *autoIndexConfig) reset(m map[string]string, mgr *config.Manager) {
|
||||
func (p *autoIndexConfig) reset(key string, m map[string]string, mgr *config.Manager) {
|
||||
j := funcutil.MapToJSON(m)
|
||||
mgr.SetConfig("autoIndex.params.build", string(j))
|
||||
mgr.SetConfig(key, string(j))
|
||||
}
|
||||
|
|
|
@ -23,6 +23,7 @@ import (
|
|||
|
||||
"github.com/stretchr/testify/assert"
|
||||
|
||||
"github.com/milvus-io/milvus-proto/go-api/v2/schemapb"
|
||||
"github.com/milvus-io/milvus/pkg/common"
|
||||
"github.com/milvus-io/milvus/pkg/config"
|
||||
"github.com/milvus-io/milvus/pkg/util/indexparamcheck"
|
||||
|
@ -140,7 +141,7 @@ func Test_autoIndexConfig_panicIfNotValid(t *testing.T) {
|
|||
}
|
||||
p.IndexParams.Init(mgr)
|
||||
assert.Panics(t, func() {
|
||||
p.panicIfNotValidAndSetDefaultMetricType(mgr)
|
||||
p.panicIfNotValidAndSetDefaultMetricTypeHelper(p.IndexParams.Key, p.IndexParams.GetAsJSONMap(), schemapb.DataType_FloatVector, mgr)
|
||||
})
|
||||
})
|
||||
|
||||
|
@ -154,7 +155,7 @@ func Test_autoIndexConfig_panicIfNotValid(t *testing.T) {
|
|||
}
|
||||
p.IndexParams.Init(mgr)
|
||||
assert.Panics(t, func() {
|
||||
p.panicIfNotValidAndSetDefaultMetricType(mgr)
|
||||
p.panicIfNotValidAndSetDefaultMetricTypeHelper(p.IndexParams.Key, p.IndexParams.GetAsJSONMap(), schemapb.DataType_FloatVector, mgr)
|
||||
})
|
||||
})
|
||||
|
||||
|
@ -168,7 +169,7 @@ func Test_autoIndexConfig_panicIfNotValid(t *testing.T) {
|
|||
}
|
||||
p.IndexParams.Init(mgr)
|
||||
assert.Panics(t, func() {
|
||||
p.panicIfNotValidAndSetDefaultMetricType(mgr)
|
||||
p.panicIfNotValidAndSetDefaultMetricTypeHelper(p.IndexParams.Key, p.IndexParams.GetAsJSONMap(), schemapb.DataType_FloatVector, mgr)
|
||||
})
|
||||
})
|
||||
|
||||
|
@ -182,7 +183,58 @@ func Test_autoIndexConfig_panicIfNotValid(t *testing.T) {
|
|||
}
|
||||
p.IndexParams.Init(mgr)
|
||||
assert.NotPanics(t, func() {
|
||||
p.panicIfNotValidAndSetDefaultMetricType(mgr)
|
||||
p.panicIfNotValidAndSetDefaultMetricTypeHelper(p.IndexParams.Key, p.IndexParams.GetAsJSONMap(), schemapb.DataType_FloatVector, mgr)
|
||||
})
|
||||
metricType, exist := p.IndexParams.GetAsJSONMap()[common.MetricTypeKey]
|
||||
assert.True(t, exist)
|
||||
assert.Equal(t, indexparamcheck.FloatVectorDefaultMetricType, metricType)
|
||||
})
|
||||
|
||||
t.Run("normal case, binary vector", func(t *testing.T) {
|
||||
mgr := config.NewManager()
|
||||
mgr.SetConfig("autoIndex.params.binary.build", `{"nlist": 1024, "index_type": "BIN_IVF_FLAT"}`)
|
||||
p := &autoIndexConfig{
|
||||
BinaryIndexParams: ParamItem{
|
||||
Key: "autoIndex.params.binary.build",
|
||||
},
|
||||
}
|
||||
p.BinaryIndexParams.Init(mgr)
|
||||
assert.NotPanics(t, func() {
|
||||
p.panicIfNotValidAndSetDefaultMetricTypeHelper(p.BinaryIndexParams.Key, p.BinaryIndexParams.GetAsJSONMap(), schemapb.DataType_BinaryVector, mgr)
|
||||
})
|
||||
metricType, exist := p.BinaryIndexParams.GetAsJSONMap()[common.MetricTypeKey]
|
||||
assert.True(t, exist)
|
||||
assert.Equal(t, indexparamcheck.BinaryVectorDefaultMetricType, metricType)
|
||||
})
|
||||
|
||||
t.Run("normal case, sparse vector", func(t *testing.T) {
|
||||
mgr := config.NewManager()
|
||||
mgr.SetConfig("autoIndex.params.sparse.build", `{"index_type": "SPARSE_INVERTED_INDEX", "metric_type": "IP"}`)
|
||||
p := &autoIndexConfig{
|
||||
SparseIndexParams: ParamItem{
|
||||
Key: "autoIndex.params.sparse.build",
|
||||
},
|
||||
}
|
||||
p.SparseIndexParams.Init(mgr)
|
||||
assert.NotPanics(t, func() {
|
||||
p.panicIfNotValidAndSetDefaultMetricTypeHelper(p.SparseIndexParams.Key, p.SparseIndexParams.GetAsJSONMap(), schemapb.DataType_SparseFloatVector, mgr)
|
||||
})
|
||||
metricType, exist := p.SparseIndexParams.GetAsJSONMap()[common.MetricTypeKey]
|
||||
assert.True(t, exist)
|
||||
assert.Equal(t, indexparamcheck.SparseFloatVectorDefaultMetricType, metricType)
|
||||
})
|
||||
|
||||
t.Run("normal case, ivf flat", func(t *testing.T) {
|
||||
mgr := config.NewManager()
|
||||
mgr.SetConfig("autoIndex.params.build", `{"nlist": 30, "index_type": "IVF_FLAT"}`)
|
||||
p := &autoIndexConfig{
|
||||
IndexParams: ParamItem{
|
||||
Key: "autoIndex.params.build",
|
||||
},
|
||||
}
|
||||
p.IndexParams.Init(mgr)
|
||||
assert.NotPanics(t, func() {
|
||||
p.panicIfNotValidAndSetDefaultMetricTypeHelper(p.IndexParams.Key, p.IndexParams.GetAsJSONMap(), schemapb.DataType_FloatVector, mgr)
|
||||
})
|
||||
metricType, exist := p.IndexParams.GetAsJSONMap()[common.MetricTypeKey]
|
||||
assert.True(t, exist)
|
||||
|
@ -199,24 +251,7 @@ func Test_autoIndexConfig_panicIfNotValid(t *testing.T) {
|
|||
}
|
||||
p.IndexParams.Init(mgr)
|
||||
assert.NotPanics(t, func() {
|
||||
p.panicIfNotValidAndSetDefaultMetricType(mgr)
|
||||
})
|
||||
metricType, exist := p.IndexParams.GetAsJSONMap()[common.MetricTypeKey]
|
||||
assert.True(t, exist)
|
||||
assert.Equal(t, indexparamcheck.FloatVectorDefaultMetricType, metricType)
|
||||
})
|
||||
|
||||
t.Run("normal case, ivf flat", func(t *testing.T) {
|
||||
mgr := config.NewManager()
|
||||
mgr.SetConfig("autoIndex.params.build", `{"nlist": 30, "index_type": "IVF_FLAT"}`)
|
||||
p := &autoIndexConfig{
|
||||
IndexParams: ParamItem{
|
||||
Key: "autoIndex.params.build",
|
||||
},
|
||||
}
|
||||
p.IndexParams.Init(mgr)
|
||||
assert.NotPanics(t, func() {
|
||||
p.panicIfNotValidAndSetDefaultMetricType(mgr)
|
||||
p.panicIfNotValidAndSetDefaultMetricTypeHelper(p.IndexParams.Key, p.IndexParams.GetAsJSONMap(), schemapb.DataType_FloatVector, mgr)
|
||||
})
|
||||
metricType, exist := p.IndexParams.GetAsJSONMap()[common.MetricTypeKey]
|
||||
assert.True(t, exist)
|
||||
|
@ -233,7 +268,7 @@ func Test_autoIndexConfig_panicIfNotValid(t *testing.T) {
|
|||
}
|
||||
p.IndexParams.Init(mgr)
|
||||
assert.NotPanics(t, func() {
|
||||
p.panicIfNotValidAndSetDefaultMetricType(mgr)
|
||||
p.panicIfNotValidAndSetDefaultMetricTypeHelper(p.IndexParams.Key, p.IndexParams.GetAsJSONMap(), schemapb.DataType_FloatVector, mgr)
|
||||
})
|
||||
metricType, exist := p.IndexParams.GetAsJSONMap()[common.MetricTypeKey]
|
||||
assert.True(t, exist)
|
||||
|
@ -250,7 +285,7 @@ func Test_autoIndexConfig_panicIfNotValid(t *testing.T) {
|
|||
}
|
||||
p.IndexParams.Init(mgr)
|
||||
assert.NotPanics(t, func() {
|
||||
p.panicIfNotValidAndSetDefaultMetricType(mgr)
|
||||
p.panicIfNotValidAndSetDefaultMetricTypeHelper(p.IndexParams.Key, p.IndexParams.GetAsJSONMap(), schemapb.DataType_FloatVector, mgr)
|
||||
})
|
||||
metricType, exist := p.IndexParams.GetAsJSONMap()[common.MetricTypeKey]
|
||||
assert.True(t, exist)
|
||||
|
@ -267,7 +302,7 @@ func Test_autoIndexConfig_panicIfNotValid(t *testing.T) {
|
|||
}
|
||||
p.IndexParams.Init(mgr)
|
||||
assert.NotPanics(t, func() {
|
||||
p.panicIfNotValidAndSetDefaultMetricType(mgr)
|
||||
p.panicIfNotValidAndSetDefaultMetricTypeHelper(p.IndexParams.Key, p.IndexParams.GetAsJSONMap(), schemapb.DataType_FloatVector, mgr)
|
||||
})
|
||||
metricType, exist := p.IndexParams.GetAsJSONMap()[common.MetricTypeKey]
|
||||
assert.True(t, exist)
|
||||
|
|
Loading…
Reference in New Issue