milvus-io
/
milvus
mirror of https://github.com/milvus-io/milvus.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

enhance: remove useless vector index param checker (#37329) 

issue: #34298 
because all vector index config checker has been moved into
vector_index_checker, then the useless checkers can be removed.

Signed-off-by: xianliang.li <xianliang.li@zilliz.com>
pull/37324/head
foxspy 2024-11-01 06:20:21 +08:00 committed by GitHub
parent 3224e58c5b
commit 642a651f60
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
22 changed files with 14 additions and 691 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
internal/util/indexparamcheck/base_checker.go
View File

@ -17,28 +17,14 @@
package indexparamcheck
import (
"fmt"
"math"
"github.com/cockroachdb/errors"
"github.com/milvus-io/milvus-proto/go-api/v2/schemapb"
"github.com/milvus-io/milvus/pkg/util/typeutil"
)
type baseChecker struct{}
func (c baseChecker) CheckTrain(dataType schemapb.DataType, params map[string]string) error {
if typeutil.IsSparseFloatVectorType(dataType) {
if !CheckStrByValues(params, Metric, SparseMetrics) {
return fmt.Errorf("metric type not found or not supported for sparse float vectors, supported: %v", SparseMetrics)
}
} else {
// we do not check dim for sparse
if !CheckIntByRange(params, DIM, 1, math.MaxInt) {
return fmt.Errorf("failed to check vector dimension, should be larger than 0 and smaller than math.MaxInt")
}
}
return nil
}

19
internal/util/indexparamcheck/bin_flat_checker.go
View File

@ -1,19 +0,0 @@
package indexparamcheck
import "github.com/milvus-io/milvus-proto/go-api/v2/schemapb"
type binFlatChecker struct {
binaryVectorBaseChecker
}
func (c binFlatChecker) CheckTrain(dataType schemapb.DataType, params map[string]string) error {
return c.binaryVectorBaseChecker.CheckTrain(dataType, params)
}
func (c binFlatChecker) StaticCheck(dataType schemapb.DataType, params map[string]string) error {
return c.staticCheck(params)
}
func newBinFlatChecker() IndexChecker {
return &binFlatChecker{}
}

35
internal/util/indexparamcheck/bin_ivf_flat_checker.go
View File

@ -1,35 +0,0 @@
package indexparamcheck
import (
"fmt"
"github.com/milvus-io/milvus-proto/go-api/v2/schemapb"
)
type binIVFFlatChecker struct {
binaryVectorBaseChecker
}
func (c binIVFFlatChecker) StaticCheck(dataType schemapb.DataType, params map[string]string) error {
if !CheckStrByValues(params, Metric, BinIvfMetrics) {
return fmt.Errorf("metric type %s not found or not supported, supported: %v", params[Metric], BinIvfMetrics)
}
if !CheckIntByRange(params, NLIST, MinNList, MaxNList) {
return errOutOfRange(NLIST, MinNList, MaxNList)
}
return nil
}
func (c binIVFFlatChecker) CheckTrain(dataType schemapb.DataType, params map[string]string) error {
if err := c.binaryVectorBaseChecker.CheckTrain(dataType, params); err != nil {
return err
}
return c.StaticCheck(schemapb.DataType_BinaryVector, params)
}
func newBinIVFFlatChecker() IndexChecker {
return &binIVFFlatChecker{}
}

43
internal/util/indexparamcheck/binary_vector_base_checker.go
View File

@ -1,43 +0,0 @@
package indexparamcheck
import (
"fmt"
"github.com/milvus-io/milvus-proto/go-api/v2/schemapb"
"github.com/milvus-io/milvus/pkg/common"
)
type binaryVectorBaseChecker struct {
baseChecker
}
func (c binaryVectorBaseChecker) staticCheck(params map[string]string) error {
if !CheckStrByValues(params, Metric, BinIDMapMetrics) {
return fmt.Errorf("metric type %s not found or not supported, supported: %v", params[Metric], BinIDMapMetrics)
}
return nil
}
func (c binaryVectorBaseChecker) CheckTrain(dataType schemapb.DataType, params map[string]string) error {
if err := c.baseChecker.CheckTrain(dataType, params); err != nil {
return err
}
return c.staticCheck(params)
}
func (c binaryVectorBaseChecker) CheckValidDataType(indexType IndexType, field *schemapb.FieldSchema) error {
if field.GetDataType() != schemapb.DataType_BinaryVector {
return fmt.Errorf("binary vector is only supported")
}
return nil
}
func (c binaryVectorBaseChecker) SetDefaultMetricTypeIfNotExist(dType schemapb.DataType, params map[string]string) {
setDefaultIfNotExist(params, common.MetricTypeKey, BinaryVectorDefaultMetricType)
}
func newBinaryVectorBaseChecker() IndexChecker {
return &binaryVectorBaseChecker{}
}

65
internal/util/indexparamcheck/cagra_checker.go
View File

@ -1,65 +0,0 @@
package indexparamcheck
import (
"fmt"
"strconv"
"github.com/milvus-io/milvus-proto/go-api/v2/schemapb"
)
// diskannChecker checks if an diskann index can be built.
type cagraChecker struct {
floatVectorBaseChecker
}
func (c *cagraChecker) CheckTrain(dataType schemapb.DataType, params map[string]string) error {
err := c.baseChecker.CheckTrain(dataType, params)
if err != nil {
return err
}
interDegree := int(0)
graphDegree := int(0)
interDegreeStr, interDegreeExist := params[CagraInterDegree]
if interDegreeExist {
interDegree, err = strconv.Atoi(interDegreeStr)
if err != nil {
return fmt.Errorf("invalid cagra inter degree: %s", interDegreeStr)
}
}
graphDegreeStr, graphDegreeExist := params[CagraGraphDegree]
if graphDegreeExist {
graphDegree, err = strconv.Atoi(graphDegreeStr)
if err != nil {
return fmt.Errorf("invalid cagra graph degree: %s", graphDegreeStr)
}
}
if graphDegreeExist && interDegreeExist && interDegree < graphDegree {
return fmt.Errorf("Graph degree cannot be larger than intermediate graph degree")
}
if !CheckStrByValues(params, Metric, RaftMetrics) {
return fmt.Errorf("metric type not found or not supported, supported: %v", RaftMetrics)
}
setDefaultIfNotExist(params, CagraBuildAlgo, "NN_DESCENT")
if !CheckStrByValues(params, CagraBuildAlgo, CagraBuildAlgoTypes) {
return fmt.Errorf("cagra build algo type not supported, supported: %v", CagraBuildAlgoTypes)
}
setDefaultIfNotExist(params, RaftCacheDatasetOnDevice, "false")
if !CheckStrByValues(params, RaftCacheDatasetOnDevice, []string{"true", "false"}) {
return fmt.Errorf("raft index cache_dataset_on_device param only support true false")
}
return nil
}
func (c cagraChecker) StaticCheck(dataType schemapb.DataType, params map[string]string) error {
return c.staticCheck(params)
}
func newCagraChecker() IndexChecker {
return &cagraChecker{}
}

16
internal/util/indexparamcheck/diskann_checker.go
View File

@ -1,16 +0,0 @@
package indexparamcheck
import "github.com/milvus-io/milvus-proto/go-api/v2/schemapb"
// diskannChecker checks if an diskann index can be built.
type diskannChecker struct {
floatVectorBaseChecker
}
func (c diskannChecker) StaticCheck(dataType schemapb.DataType, params map[string]string) error {
return c.staticCheck(params)
}
func newDiskannChecker() IndexChecker {
return &diskannChecker{}
}

15
internal/util/indexparamcheck/flat_checker.go
View File

@ -1,15 +0,0 @@
package indexparamcheck
import "github.com/milvus-io/milvus-proto/go-api/v2/schemapb"
type flatChecker struct {
floatVectorBaseChecker
}
func (c flatChecker) StaticCheck(dataType schemapb.DataType, m map[string]string) error {
return c.staticCheck(m)
}
func newFlatChecker() IndexChecker {
return &flatChecker{}
}

44
internal/util/indexparamcheck/float_vector_base_checker.go
View File

@ -1,44 +0,0 @@
package indexparamcheck
import (
"fmt"
"github.com/milvus-io/milvus-proto/go-api/v2/schemapb"
"github.com/milvus-io/milvus/pkg/common"
"github.com/milvus-io/milvus/pkg/util/typeutil"
)
type floatVectorBaseChecker struct {
baseChecker
}
func (c floatVectorBaseChecker) staticCheck(params map[string]string) error {
if !CheckStrByValues(params, Metric, FloatVectorMetrics) {
return fmt.Errorf("metric type %s not found or not supported, supported: %v", params[Metric], FloatVectorMetrics)
}
return nil
}
func (c floatVectorBaseChecker) CheckTrain(dataType schemapb.DataType, params map[string]string) error {
if err := c.baseChecker.CheckTrain(dataType, params); err != nil {
return err
}
return c.staticCheck(params)
}
func (c floatVectorBaseChecker) CheckValidDataType(indexType IndexType, field *schemapb.FieldSchema) error {
if !typeutil.IsDenseFloatVectorType(field.GetDataType()) {
return fmt.Errorf("data type should be FloatVector, Float16Vector or BFloat16Vector")
}
return nil
}
func (c floatVectorBaseChecker) SetDefaultMetricTypeIfNotExist(dType schemapb.DataType, params map[string]string) {
setDefaultIfNotExist(params, common.MetricTypeKey, FloatVectorDefaultMetricType)
}
func newFloatVectorBaseChecker() IndexChecker {
return &floatVectorBaseChecker{}
}

54
internal/util/indexparamcheck/hnsw_checker.go
View File

@ -1,54 +0,0 @@
package indexparamcheck
import (
"fmt"
"github.com/milvus-io/milvus-proto/go-api/v2/schemapb"
"github.com/milvus-io/milvus/pkg/common"
"github.com/milvus-io/milvus/pkg/util/typeutil"
)
type hnswChecker struct {
baseChecker
}
func (c hnswChecker) StaticCheck(dataType schemapb.DataType, params map[string]string) error {
if !CheckIntByRange(params, EFConstruction, HNSWMinEfConstruction, HNSWMaxEfConstruction) {
return errOutOfRange(EFConstruction, HNSWMinEfConstruction, HNSWMaxEfConstruction)
}
if !CheckIntByRange(params, HNSWM, HNSWMinM, HNSWMaxM) {
return errOutOfRange(HNSWM, HNSWMinM, HNSWMaxM)
}
if !CheckStrByValues(params, Metric, HnswMetrics) {
return fmt.Errorf("metric type %s not found or not supported, supported: %v", params[Metric], HnswMetrics)
}
return nil
}
func (c hnswChecker) CheckTrain(dataType schemapb.DataType, params map[string]string) error {
if err := c.StaticCheck(dataType, params); err != nil {
return err
}
return c.baseChecker.CheckTrain(dataType, params)
}
func (c hnswChecker) CheckValidDataType(indexType IndexType, field *schemapb.FieldSchema) error {
if !typeutil.IsVectorType(field.GetDataType()) {
return fmt.Errorf("can't build hnsw in not vector type")
}
return nil
}
func (c hnswChecker) SetDefaultMetricTypeIfNotExist(dType schemapb.DataType, params map[string]string) {
if typeutil.IsDenseFloatVectorType(dType) {
setDefaultIfNotExist(params, common.MetricTypeKey, FloatVectorDefaultMetricType)
} else if typeutil.IsSparseFloatVectorType(dType) {
setDefaultIfNotExist(params, common.MetricTypeKey, SparseFloatVectorDefaultMetricType)
} else if typeutil.IsBinaryVectorType(dType) {
setDefaultIfNotExist(params, common.MetricTypeKey, BinaryVectorDefaultMetricType)
}
}
func newHnswChecker() IndexChecker {
return &hnswChecker{}
}

28
internal/util/indexparamcheck/ivf_base_checker.go
View File

@ -1,28 +0,0 @@
package indexparamcheck
import "github.com/milvus-io/milvus-proto/go-api/v2/schemapb"
type ivfBaseChecker struct {
floatVectorBaseChecker
}
func (c ivfBaseChecker) StaticCheck(dataType schemapb.DataType, params map[string]string) error {
if !CheckIntByRange(params, NLIST, MinNList, MaxNList) {
return errOutOfRange(NLIST, MinNList, MaxNList)
}
// skip check number of rows
return c.floatVectorBaseChecker.staticCheck(params)
}
func (c ivfBaseChecker) CheckTrain(dataType schemapb.DataType, params map[string]string) error {
if err := c.StaticCheck(dataType, params); err != nil {
return err
}
return c.floatVectorBaseChecker.CheckTrain(dataType, params)
}
func newIVFBaseChecker() IndexChecker {
return &ivfBaseChecker{}
}

69
internal/util/indexparamcheck/ivf_pq_checker.go
View File

@ -1,69 +0,0 @@
package indexparamcheck
import (
"fmt"
"strconv"
"github.com/milvus-io/milvus-proto/go-api/v2/schemapb"
)
// ivfPQChecker checks if a IVF_PQ index can be built.
type ivfPQChecker struct {
ivfBaseChecker
}
// CheckTrain checks if ivf-pq index can be built with the specific index parameters.
func (c *ivfPQChecker) CheckTrain(dataType schemapb.DataType, params map[string]string) error {
if err := c.ivfBaseChecker.CheckTrain(dataType, params); err != nil {
return err
}
return c.checkPQParams(params)
}
func (c *ivfPQChecker) checkPQParams(params map[string]string) error {
dimStr, dimensionExist := params[DIM]
if !dimensionExist {
return fmt.Errorf("dimension not found")
}
dimension, err := strconv.Atoi(dimStr)
if err != nil { // invalid dimension
return fmt.Errorf("invalid dimension: %s", dimStr)
}
// nbits can be set to default: 8
nbitsStr, nbitsExist := params[NBITS]
if nbitsExist {
nbits, err := strconv.Atoi(nbitsStr)
if err != nil { // invalid nbits
return fmt.Errorf("invalid nbits: %s", nbitsStr)
}
if nbits < 1 || nbits > 64 {
return fmt.Errorf("parameter `nbits` out of range, expect range [1,64], current value: %d", nbits)
}
}
mStr, ok := params[IVFM]
if !ok {
return fmt.Errorf("parameter `m` not found")
}
m, err := strconv.Atoi(mStr)
if err != nil || m == 0 { // invalid m
return fmt.Errorf("invalid `m`: %s", mStr)
}
return c.checkCPUPQParams(dimension, m)
}
func (c *ivfPQChecker) checkCPUPQParams(dimension, m int) error {
if (dimension % m) != 0 {
return fmt.Errorf("dimension must be able to be divided by `m`, dimension: %d, m: %d", dimension, m)
}
return nil
}
func newIVFPQChecker() IndexChecker {
return &ivfPQChecker{}
}

3
internal/util/indexparamcheck/ivf_pq_checker_test.go
View File

@ -142,8 +142,7 @@ func Test_ivfPQChecker_CheckTrain(t *testing.T) {
{p7, false},
}
// c, _ := GetIndexCheckerMgrInstance().GetChecker("IVF_PQ")
c := newIVFPQChecker()
c, _ := GetIndexCheckerMgrInstance().GetChecker("IVF_PQ")
for _, test := range cases {
test.params[common.IndexTypeKey] = "IVF_PQ"
err := c.CheckTrain(schemapb.DataType_FloatVector, test.params)

36
internal/util/indexparamcheck/ivf_sq_checker.go
View File

@ -1,36 +0,0 @@
package indexparamcheck
import (
"fmt"
"github.com/milvus-io/milvus-proto/go-api/v2/schemapb"
)
// ivfSQChecker checks if a IVF_SQ index can be built.
type ivfSQChecker struct {
ivfBaseChecker
}
func (c *ivfSQChecker) checkNBits(params map[string]string) error {
// cgo will set this key to DefaultNBits (8), which is the only value Milvus supports.
_, exist := params[NBITS]
if exist {
// 8 is the only supported nbits.
if !CheckIntByRange(params, NBITS, DefaultNBits, DefaultNBits) {
return fmt.Errorf("nbits can be only set to 8 for IVF_SQ")
}
}
return nil
}
// CheckTrain returns true if the index can be built with the specific index parameters.
func (c *ivfSQChecker) CheckTrain(dataType schemapb.DataType, params map[string]string) error {
if err := c.checkNBits(params); err != nil {
return err
}
return c.ivfBaseChecker.CheckTrain(dataType, params)
}
func newIVFSQChecker() IndexChecker {
return &ivfSQChecker{}
}

26
internal/util/indexparamcheck/raft_brute_force_checker.go
View File

@ -1,26 +0,0 @@
package indexparamcheck
import (
"fmt"
"github.com/milvus-io/milvus-proto/go-api/v2/schemapb"
)
type raftBruteForceChecker struct {
floatVectorBaseChecker
}
// raftBrustForceChecker checks if a Brute_Force index can be built.
func (c raftBruteForceChecker) CheckTrain(dataType schemapb.DataType, params map[string]string) error {
if err := c.floatVectorBaseChecker.CheckTrain(dataType, params); err != nil {
return err
}
if !CheckStrByValues(params, Metric, RaftMetrics) {
return fmt.Errorf("metric type not found or not supported, supported: %v", RaftMetrics)
}
return nil
}
func newRaftBruteForceChecker() IndexChecker {
return &raftBruteForceChecker{}
}

34
internal/util/indexparamcheck/raft_ivf_flat_checker.go
View File

@ -1,34 +0,0 @@
package indexparamcheck
import (
"fmt"
"github.com/milvus-io/milvus-proto/go-api/v2/schemapb"
)
// raftIVFChecker checks if a RAFT_IVF_Flat index can be built.
type raftIVFFlatChecker struct {
ivfBaseChecker
}
// CheckTrain checks if ivf-flat index can be built with the specific index parameters.
func (c *raftIVFFlatChecker) CheckTrain(dataType schemapb.DataType, params map[string]string) error {
if err := c.ivfBaseChecker.CheckTrain(dataType, params); err != nil {
return err
}
if !CheckStrByValues(params, Metric, RaftMetrics) {
return fmt.Errorf("metric type not found or not supported, supported: %v", RaftMetrics)
}
setDefaultIfNotExist(params, RaftCacheDatasetOnDevice, "false")
if !CheckStrByValues(params, RaftCacheDatasetOnDevice, []string{"true", "false"}) {
return fmt.Errorf("raft index cache_dataset_on_device param only support true false")
}
return nil
}
func newRaftIVFFlatChecker() IndexChecker {
return &raftIVFFlatChecker{}
}

74
internal/util/indexparamcheck/raft_ivf_pq_checker.go
View File

@ -1,74 +0,0 @@
package indexparamcheck
import (
"fmt"
"strconv"
"github.com/milvus-io/milvus-proto/go-api/v2/schemapb"
)
// raftIVFPQChecker checks if a RAFT_IVF_PQ index can be built.
type raftIVFPQChecker struct {
ivfBaseChecker
}
// CheckTrain checks if ivf-pq index can be built with the specific index parameters.
func (c *raftIVFPQChecker) CheckTrain(dataType schemapb.DataType, params map[string]string) error {
if err := c.ivfBaseChecker.CheckTrain(dataType, params); err != nil {
return err
}
if !CheckStrByValues(params, Metric, RaftMetrics) {
return fmt.Errorf("metric type not found or not supported, supported: %v", RaftMetrics)
}
return c.checkPQParams(params)
}
func (c *raftIVFPQChecker) checkPQParams(params map[string]string) error {
dimStr, dimensionExist := params[DIM]
if !dimensionExist {
return fmt.Errorf("dimension not found")
}
dimension, err := strconv.Atoi(dimStr)
if err != nil { // invalid dimension
return fmt.Errorf("invalid dimension: %s", dimStr)
}
// nbits can be set to default: 8
nbitsStr, nbitsExist := params[NBITS]
if nbitsExist {
_, err := strconv.Atoi(nbitsStr)
if err != nil { // invalid nbits
return fmt.Errorf("invalid nbits: %s", nbitsStr)
}
}
mStr, ok := params[IVFM]
if !ok {
return fmt.Errorf("parameter `m` not found")
}
m, err := strconv.Atoi(mStr)
if err != nil { // invalid m
return fmt.Errorf("invalid `m`: %s", mStr)
}
// here is the only difference with IVF_PQ
if m == 0 {
return nil
}
if dimension%m != 0 {
return fmt.Errorf("dimension must be able to be divided by `m`, dimension: %d, m: %d", dimension, m)
}
setDefaultIfNotExist(params, RaftCacheDatasetOnDevice, "false")
if !CheckStrByValues(params, RaftCacheDatasetOnDevice, []string{"true", "false"}) {
return fmt.Errorf("raft index cache_dataset_on_device param only support true false")
}
return nil
}
func newRaftIVFPQChecker() IndexChecker {
return &raftIVFPQChecker{}
}

43
internal/util/indexparamcheck/scann_checker.go
View File

@ -1,43 +0,0 @@
package indexparamcheck
import (
"fmt"
"strconv"
"github.com/milvus-io/milvus-proto/go-api/v2/schemapb"
)
// scaNNChecker checks if a SCANN index can be built.
type scaNNChecker struct {
ivfBaseChecker
}
// CheckTrain checks if SCANN index can be built with the specific index parameters.
func (c *scaNNChecker) CheckTrain(dataType schemapb.DataType, params map[string]string) error {
if err := c.ivfBaseChecker.CheckTrain(dataType, params); err != nil {
return err
}
return c.checkScaNNParams(params)
}
func (c *scaNNChecker) checkScaNNParams(params map[string]string) error {
dimStr, dimensionExist := params[DIM]
if !dimensionExist {
return fmt.Errorf("dimension not found")
}
dimension, err := strconv.Atoi(dimStr)
if err != nil { // invalid dimension
return fmt.Errorf("invalid dimension: %s", dimStr)
}
if (dimension % 2) != 0 {
return fmt.Errorf("dimension must be able to be divided by 2, dimension: %d", dimension)
}
return nil
}
func newScaNNChecker() IndexChecker {
return &scaNNChecker{}
}

2
internal/util/indexparamcheck/scann_checker_test.go
View File

@ -159,7 +159,7 @@ func Test_scaNNChecker_CheckValidDataType(t *testing.T) {
},
}
c := newScaNNChecker()
c, _ := GetIndexCheckerMgrInstance().GetChecker("SCANN")
for _, test := range cases {
err := c.CheckValidDataType("SCANN", &schemapb.FieldSchema{DataType: test.dType})
if test.errIsNil {

64
internal/util/indexparamcheck/sparse_float_vector_base_checker.go
View File

@ -1,64 +0,0 @@
package indexparamcheck
import (
"fmt"
"strconv"
"github.com/milvus-io/milvus-proto/go-api/v2/schemapb"
"github.com/milvus-io/milvus/pkg/common"
"github.com/milvus-io/milvus/pkg/util/typeutil"
)
// sparse vector don't check for dim, but baseChecker does, thus not including baseChecker
type sparseFloatVectorBaseChecker struct{}
func (c sparseFloatVectorBaseChecker) StaticCheck(dataType schemapb.DataType, params map[string]string) error {
if !CheckStrByValues(params, Metric, SparseMetrics) {
return fmt.Errorf("metric type not found or not supported, supported: %v", SparseMetrics)
}
return nil
}
func (c sparseFloatVectorBaseChecker) CheckTrain(dataType schemapb.DataType, params map[string]string) error {
dropRatioBuildStr, exist := params[SparseDropRatioBuild]
if exist {
dropRatioBuild, err := strconv.ParseFloat(dropRatioBuildStr, 64)
if err != nil || dropRatioBuild < 0 || dropRatioBuild >= 1 {
return fmt.Errorf("invalid drop_ratio_build: %s, must be in range [0, 1)", dropRatioBuildStr)
}
}
bm25K1Str, exist := params[BM25K1]
if exist {
bm25K1, err := strconv.ParseFloat(bm25K1Str, 64)
if err != nil || bm25K1 < 0 || bm25K1 > 3 {
return fmt.Errorf("invalid bm25_k1: %s, must be in range [0, 3]", bm25K1Str)
}
}
bm25BStr, exist := params[BM25B]
if exist {
bm25B, err := strconv.ParseFloat(bm25BStr, 64)
if err != nil || bm25B < 0 || bm25B > 1 {
return fmt.Errorf("invalid bm25_b: %s, must be in range [0, 1]", bm25BStr)
}
}
return nil
}
func (c sparseFloatVectorBaseChecker) CheckValidDataType(indexType IndexType, field *schemapb.FieldSchema) error {
if !typeutil.IsSparseFloatVectorType(field.GetDataType()) {
return fmt.Errorf("only sparse float vector is supported for the specified index tpye")
}
return nil
}
func (c sparseFloatVectorBaseChecker) SetDefaultMetricTypeIfNotExist(dType schemapb.DataType, params map[string]string) {
setDefaultIfNotExist(params, common.MetricTypeKey, SparseFloatVectorDefaultMetricType)
}
func newSparseFloatVectorBaseChecker() IndexChecker {
return &sparseFloatVectorBaseChecker{}
}

9
internal/util/indexparamcheck/sparse_inverted_index_checker.go
View File

@ -1,9 +0,0 @@
package indexparamcheck
type sparseInvertedIndexChecker struct {
sparseFloatVectorBaseChecker
}
func newSparseInvertedIndexChecker() *sparseInvertedIndexChecker {
return &sparseInvertedIndexChecker{}
}

8
internal/util/indexparamcheck/vector_index_checker.go
View File

@ -10,6 +10,7 @@ import "C"
import (
"fmt"
"math"
"unsafe"
"google.golang.org/protobuf/proto"
@ -90,6 +91,13 @@ func (c vecIndexChecker) CheckTrain(dataType schemapb.DataType, params map[strin
if err := c.StaticCheck(dataType, params); err != nil {
return err
}
if typeutil.IsFixDimVectorType(dataType) {
if !CheckIntByRange(params, DIM, 1, math.MaxInt) {
return fmt.Errorf("failed to check vector dimension, should be larger than 0 and smaller than math.MaxInt")
}
}
return c.baseChecker.CheckTrain(dataType, params)
}

4
pkg/util/typeutil/schema.go
View File

@ -483,6 +483,10 @@ func IsFloatVectorType(dataType schemapb.DataType) bool {
return IsDenseFloatVectorType(dataType) || IsSparseFloatVectorType(dataType)
}
func IsFixDimVectorType(dataType schemapb.DataType) bool {
return IsBinaryVectorType(dataType) || IsDenseFloatVectorType(dataType)
}
// IsVectorType returns true if input is a vector type, otherwise false
func IsVectorType(dataType schemapb.DataType) bool {
return IsBinaryVectorType(dataType) || IsFloatVectorType(dataType)