mirror of https://github.com/milvus-io/milvus.git
fix: fix bitmap supported type (#34350)
#34314 Signed-off-by: luzhang <luzhang@zilliz.com> Co-authored-by: luzhang <luzhang@zilliz.com>pull/34354/head
parent
21fc5f5d46
commit
18f2458385
|
@ -395,7 +395,7 @@ func checkTrain(field *schemapb.FieldSchema, indexParams map[string]string) erro
|
|||
indexParams[IsSparseKey] = "true"
|
||||
}
|
||||
|
||||
if err := checker.CheckValidDataType(field.GetDataType()); err != nil {
|
||||
if err := checker.CheckValidDataType(field); err != nil {
|
||||
log.Info("create index with invalid data type", zap.Error(err), zap.String("data_type", field.GetDataType().String()))
|
||||
return err
|
||||
}
|
||||
|
|
|
@ -13,7 +13,7 @@ func (c *AUTOINDEXChecker) CheckTrain(params map[string]string) error {
|
|||
return nil
|
||||
}
|
||||
|
||||
func (c *AUTOINDEXChecker) CheckValidDataType(dType schemapb.DataType) error {
|
||||
func (c *AUTOINDEXChecker) CheckValidDataType(field *schemapb.FieldSchema) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
|
|
|
@ -55,7 +55,7 @@ func (c baseChecker) CheckTrain(params map[string]string) error {
|
|||
}
|
||||
|
||||
// CheckValidDataType check whether the field data type is supported for the index type
|
||||
func (c baseChecker) CheckValidDataType(dType schemapb.DataType) error {
|
||||
func (c baseChecker) CheckValidDataType(field *schemapb.FieldSchema) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
|
|
|
@ -114,7 +114,8 @@ func Test_baseChecker_CheckValidDataType(t *testing.T) {
|
|||
|
||||
c := newBaseChecker()
|
||||
for _, test := range cases {
|
||||
err := c.CheckValidDataType(test.dType)
|
||||
field_schema := &schemapb.FieldSchema{DataType: test.dType}
|
||||
err := c.CheckValidDataType(field_schema)
|
||||
if test.errIsNil {
|
||||
assert.NoError(t, err)
|
||||
} else {
|
||||
|
|
|
@ -136,7 +136,8 @@ func Test_binFlatChecker_CheckValidDataType(t *testing.T) {
|
|||
|
||||
c := newBinFlatChecker()
|
||||
for _, test := range cases {
|
||||
err := c.CheckValidDataType(test.dType)
|
||||
field_schema := &schemapb.FieldSchema{DataType: test.dType}
|
||||
err := c.CheckValidDataType(field_schema)
|
||||
if test.errIsNil {
|
||||
assert.NoError(t, err)
|
||||
} else {
|
||||
|
|
|
@ -187,7 +187,8 @@ func Test_binIVFFlatChecker_CheckValidDataType(t *testing.T) {
|
|||
|
||||
c := newBinIVFFlatChecker()
|
||||
for _, test := range cases {
|
||||
err := c.CheckValidDataType(test.dType)
|
||||
field_schema := &schemapb.FieldSchema{DataType: test.dType}
|
||||
err := c.CheckValidDataType(field_schema)
|
||||
if test.errIsNil {
|
||||
assert.NoError(t, err)
|
||||
} else {
|
||||
|
|
|
@ -27,8 +27,8 @@ func (c binaryVectorBaseChecker) CheckTrain(params map[string]string) error {
|
|||
return c.staticCheck(params)
|
||||
}
|
||||
|
||||
func (c binaryVectorBaseChecker) CheckValidDataType(dType schemapb.DataType) error {
|
||||
if dType != schemapb.DataType_BinaryVector {
|
||||
func (c binaryVectorBaseChecker) CheckValidDataType(field *schemapb.FieldSchema) error {
|
||||
if field.GetDataType() != schemapb.DataType_BinaryVector {
|
||||
return fmt.Errorf("binary vector is only supported")
|
||||
}
|
||||
return nil
|
||||
|
|
|
@ -69,7 +69,8 @@ func Test_binaryVectorBaseChecker_CheckValidDataType(t *testing.T) {
|
|||
|
||||
c := newBinaryVectorBaseChecker()
|
||||
for _, test := range cases {
|
||||
err := c.CheckValidDataType(test.dType)
|
||||
field_schema := &schemapb.FieldSchema{DataType: test.dType}
|
||||
err := c.CheckValidDataType(field_schema)
|
||||
if test.errIsNil {
|
||||
assert.NoError(t, err)
|
||||
} else {
|
||||
|
|
|
@ -13,12 +13,24 @@ func Test_BitmapIndexChecker(t *testing.T) {
|
|||
|
||||
assert.NoError(t, c.CheckTrain(map[string]string{"bitmap_cardinality_limit": "100"}))
|
||||
|
||||
assert.NoError(t, c.CheckValidDataType(schemapb.DataType_Int64))
|
||||
assert.NoError(t, c.CheckValidDataType(schemapb.DataType_Float))
|
||||
assert.NoError(t, c.CheckValidDataType(schemapb.DataType_String))
|
||||
assert.NoError(t, c.CheckValidDataType(schemapb.DataType_Array))
|
||||
assert.NoError(t, c.CheckValidDataType(&schemapb.FieldSchema{DataType: schemapb.DataType_Bool}))
|
||||
assert.NoError(t, c.CheckValidDataType(&schemapb.FieldSchema{DataType: schemapb.DataType_Int8}))
|
||||
assert.NoError(t, c.CheckValidDataType(&schemapb.FieldSchema{DataType: schemapb.DataType_Int16}))
|
||||
assert.NoError(t, c.CheckValidDataType(&schemapb.FieldSchema{DataType: schemapb.DataType_Int32}))
|
||||
assert.NoError(t, c.CheckValidDataType(&schemapb.FieldSchema{DataType: schemapb.DataType_Int64}))
|
||||
assert.NoError(t, c.CheckValidDataType(&schemapb.FieldSchema{DataType: schemapb.DataType_String}))
|
||||
assert.NoError(t, c.CheckValidDataType(&schemapb.FieldSchema{DataType: schemapb.DataType_Array, ElementType: schemapb.DataType_Bool}))
|
||||
assert.NoError(t, c.CheckValidDataType(&schemapb.FieldSchema{DataType: schemapb.DataType_Array, ElementType: schemapb.DataType_Int8}))
|
||||
assert.NoError(t, c.CheckValidDataType(&schemapb.FieldSchema{DataType: schemapb.DataType_Array, ElementType: schemapb.DataType_Int16}))
|
||||
assert.NoError(t, c.CheckValidDataType(&schemapb.FieldSchema{DataType: schemapb.DataType_Array, ElementType: schemapb.DataType_Int32}))
|
||||
assert.NoError(t, c.CheckValidDataType(&schemapb.FieldSchema{DataType: schemapb.DataType_Array, ElementType: schemapb.DataType_Int64}))
|
||||
assert.NoError(t, c.CheckValidDataType(&schemapb.FieldSchema{DataType: schemapb.DataType_Array, ElementType: schemapb.DataType_String}))
|
||||
|
||||
assert.Error(t, c.CheckValidDataType(schemapb.DataType_JSON))
|
||||
assert.Error(t, c.CheckValidDataType(&schemapb.FieldSchema{DataType: schemapb.DataType_JSON}))
|
||||
assert.Error(t, c.CheckValidDataType(&schemapb.FieldSchema{DataType: schemapb.DataType_Float}))
|
||||
assert.Error(t, c.CheckValidDataType(&schemapb.FieldSchema{DataType: schemapb.DataType_Double}))
|
||||
assert.Error(t, c.CheckValidDataType(&schemapb.FieldSchema{DataType: schemapb.DataType_Array, ElementType: schemapb.DataType_Float}))
|
||||
assert.Error(t, c.CheckValidDataType(&schemapb.FieldSchema{DataType: schemapb.DataType_Array, ElementType: schemapb.DataType_Double}))
|
||||
assert.Error(t, c.CheckTrain(map[string]string{}))
|
||||
assert.Error(t, c.CheckTrain(map[string]string{"bitmap_cardinality_limit": "0"}))
|
||||
}
|
||||
|
|
|
@ -20,9 +20,18 @@ func (c *BITMAPChecker) CheckTrain(params map[string]string) error {
|
|||
return c.scalarIndexChecker.CheckTrain(params)
|
||||
}
|
||||
|
||||
func (c *BITMAPChecker) CheckValidDataType(dType schemapb.DataType) error {
|
||||
if !typeutil.IsArithmetic(dType) && !typeutil.IsStringType(dType) && !typeutil.IsArrayType(dType) {
|
||||
return fmt.Errorf("bitmap index are only supported on numeric, string and array field")
|
||||
func (c *BITMAPChecker) CheckValidDataType(field *schemapb.FieldSchema) error {
|
||||
main_type := field.GetDataType()
|
||||
elem_type := field.GetElementType()
|
||||
if !typeutil.IsBoolType(main_type) && !typeutil.IsIntegerType(main_type) &&
|
||||
!typeutil.IsStringType(main_type) && !typeutil.IsArrayType(main_type) {
|
||||
return fmt.Errorf("bitmap index are only supported on bool, int, string and array field")
|
||||
}
|
||||
if typeutil.IsArrayType(main_type) {
|
||||
if !typeutil.IsBoolType(elem_type) && !typeutil.IsIntegerType(elem_type) &&
|
||||
!typeutil.IsStringType(elem_type) {
|
||||
return fmt.Errorf("bitmap index are only supported on bool, int, string for array field")
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
|
|
@ -144,7 +144,7 @@ func Test_diskannChecker_CheckValidDataType(t *testing.T) {
|
|||
|
||||
c := newDiskannChecker()
|
||||
for _, test := range cases {
|
||||
err := c.CheckValidDataType(test.dType)
|
||||
err := c.CheckValidDataType(&schemapb.FieldSchema{DataType: test.dType})
|
||||
if test.errIsNil {
|
||||
assert.NoError(t, err)
|
||||
} else {
|
||||
|
|
|
@ -28,8 +28,8 @@ func (c floatVectorBaseChecker) CheckTrain(params map[string]string) error {
|
|||
return c.staticCheck(params)
|
||||
}
|
||||
|
||||
func (c floatVectorBaseChecker) CheckValidDataType(dType schemapb.DataType) error {
|
||||
if !typeutil.IsDenseFloatVectorType(dType) {
|
||||
func (c floatVectorBaseChecker) CheckValidDataType(field *schemapb.FieldSchema) error {
|
||||
if !typeutil.IsDenseFloatVectorType(field.GetDataType()) {
|
||||
return fmt.Errorf("data type should be FloatVector, Float16Vector or BFloat16Vector")
|
||||
}
|
||||
return nil
|
||||
|
|
|
@ -69,7 +69,7 @@ func Test_floatVectorBaseChecker_CheckValidDataType(t *testing.T) {
|
|||
|
||||
c := newFloatVectorBaseChecker()
|
||||
for _, test := range cases {
|
||||
err := c.CheckValidDataType(test.dType)
|
||||
err := c.CheckValidDataType(&schemapb.FieldSchema{DataType: test.dType})
|
||||
if test.errIsNil {
|
||||
assert.NoError(t, err)
|
||||
} else {
|
||||
|
|
|
@ -32,9 +32,9 @@ func (c hnswChecker) CheckTrain(params map[string]string) error {
|
|||
return c.baseChecker.CheckTrain(params)
|
||||
}
|
||||
|
||||
func (c hnswChecker) CheckValidDataType(dType schemapb.DataType) error {
|
||||
if !typeutil.IsVectorType(dType) {
|
||||
return fmt.Errorf("can't build hnsw in not vector type.")
|
||||
func (c hnswChecker) CheckValidDataType(field *schemapb.FieldSchema) error {
|
||||
if !typeutil.IsVectorType(field.GetDataType()) {
|
||||
return fmt.Errorf("can't build hnsw in not vector type")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
|
|
@ -164,7 +164,7 @@ func Test_hnswChecker_CheckValidDataType(t *testing.T) {
|
|||
|
||||
c := newHnswChecker()
|
||||
for _, test := range cases {
|
||||
err := c.CheckValidDataType(test.dType)
|
||||
err := c.CheckValidDataType(&schemapb.FieldSchema{DataType: test.dType})
|
||||
if test.errIsNil {
|
||||
assert.NoError(t, err)
|
||||
} else {
|
||||
|
|
|
@ -22,7 +22,7 @@ import (
|
|||
|
||||
type IndexChecker interface {
|
||||
CheckTrain(map[string]string) error
|
||||
CheckValidDataType(dType schemapb.DataType) error
|
||||
CheckValidDataType(field *schemapb.FieldSchema) error
|
||||
SetDefaultMetricTypeIfNotExist(map[string]string, schemapb.DataType)
|
||||
StaticCheck(map[string]string) error
|
||||
}
|
||||
|
|
|
@ -16,7 +16,8 @@ func (c *INVERTEDChecker) CheckTrain(params map[string]string) error {
|
|||
return c.scalarIndexChecker.CheckTrain(params)
|
||||
}
|
||||
|
||||
func (c *INVERTEDChecker) CheckValidDataType(dType schemapb.DataType) error {
|
||||
func (c *INVERTEDChecker) CheckValidDataType(field *schemapb.FieldSchema) error {
|
||||
dType := field.GetDataType()
|
||||
if !typeutil.IsBoolType(dType) && !typeutil.IsArithmetic(dType) && !typeutil.IsStringType(dType) &&
|
||||
!typeutil.IsArrayType(dType) {
|
||||
return fmt.Errorf("INVERTED are not supported on %s field", dType.String())
|
||||
|
|
|
@ -13,13 +13,13 @@ func Test_INVERTEDIndexChecker(t *testing.T) {
|
|||
|
||||
assert.NoError(t, c.CheckTrain(map[string]string{}))
|
||||
|
||||
assert.NoError(t, c.CheckValidDataType(schemapb.DataType_VarChar))
|
||||
assert.NoError(t, c.CheckValidDataType(schemapb.DataType_String))
|
||||
assert.NoError(t, c.CheckValidDataType(schemapb.DataType_Bool))
|
||||
assert.NoError(t, c.CheckValidDataType(schemapb.DataType_Int64))
|
||||
assert.NoError(t, c.CheckValidDataType(schemapb.DataType_Float))
|
||||
assert.NoError(t, c.CheckValidDataType(schemapb.DataType_Array))
|
||||
assert.NoError(t, c.CheckValidDataType(&schemapb.FieldSchema{DataType: schemapb.DataType_VarChar}))
|
||||
assert.NoError(t, c.CheckValidDataType(&schemapb.FieldSchema{DataType: schemapb.DataType_String}))
|
||||
assert.NoError(t, c.CheckValidDataType(&schemapb.FieldSchema{DataType: schemapb.DataType_Bool}))
|
||||
assert.NoError(t, c.CheckValidDataType(&schemapb.FieldSchema{DataType: schemapb.DataType_Int64}))
|
||||
assert.NoError(t, c.CheckValidDataType(&schemapb.FieldSchema{DataType: schemapb.DataType_Float}))
|
||||
assert.NoError(t, c.CheckValidDataType(&schemapb.FieldSchema{DataType: schemapb.DataType_Array}))
|
||||
|
||||
assert.Error(t, c.CheckValidDataType(schemapb.DataType_JSON))
|
||||
assert.Error(t, c.CheckValidDataType(schemapb.DataType_FloatVector))
|
||||
assert.Error(t, c.CheckValidDataType(&schemapb.FieldSchema{DataType: schemapb.DataType_JSON}))
|
||||
assert.Error(t, c.CheckValidDataType(&schemapb.FieldSchema{DataType: schemapb.DataType_FloatVector}))
|
||||
}
|
||||
|
|
|
@ -142,7 +142,7 @@ func Test_ivfBaseChecker_CheckValidDataType(t *testing.T) {
|
|||
|
||||
c := newIVFBaseChecker()
|
||||
for _, test := range cases {
|
||||
err := c.CheckValidDataType(test.dType)
|
||||
err := c.CheckValidDataType(&schemapb.FieldSchema{DataType: test.dType})
|
||||
if test.errIsNil {
|
||||
assert.NoError(t, err)
|
||||
} else {
|
||||
|
|
|
@ -207,7 +207,7 @@ func Test_ivfPQChecker_CheckValidDataType(t *testing.T) {
|
|||
|
||||
c := newIVFPQChecker()
|
||||
for _, test := range cases {
|
||||
err := c.CheckValidDataType(test.dType)
|
||||
err := c.CheckValidDataType(&schemapb.FieldSchema{DataType: test.dType})
|
||||
if test.errIsNil {
|
||||
assert.NoError(t, err)
|
||||
} else {
|
||||
|
|
|
@ -162,7 +162,7 @@ func Test_ivfSQChecker_CheckValidDataType(t *testing.T) {
|
|||
|
||||
c := newIVFSQChecker()
|
||||
for _, test := range cases {
|
||||
err := c.CheckValidDataType(test.dType)
|
||||
err := c.CheckValidDataType(&schemapb.FieldSchema{DataType: test.dType})
|
||||
if test.errIsNil {
|
||||
assert.NoError(t, err)
|
||||
} else {
|
||||
|
|
|
@ -156,7 +156,7 @@ func Test_raftIvfFlatChecker_CheckValidDataType(t *testing.T) {
|
|||
|
||||
c := newRaftIVFFlatChecker()
|
||||
for _, test := range cases {
|
||||
err := c.CheckValidDataType(test.dType)
|
||||
err := c.CheckValidDataType(&schemapb.FieldSchema{DataType: test.dType})
|
||||
if test.errIsNil {
|
||||
assert.NoError(t, err)
|
||||
} else {
|
||||
|
|
|
@ -216,7 +216,7 @@ func Test_raftIVFPQChecker_CheckValidDataType(t *testing.T) {
|
|||
|
||||
c := newRaftIVFPQChecker()
|
||||
for _, test := range cases {
|
||||
err := c.CheckValidDataType(test.dType)
|
||||
err := c.CheckValidDataType(&schemapb.FieldSchema{DataType: test.dType})
|
||||
if test.errIsNil {
|
||||
assert.NoError(t, err)
|
||||
} else {
|
||||
|
|
|
@ -159,7 +159,7 @@ func Test_scaNNChecker_CheckValidDataType(t *testing.T) {
|
|||
|
||||
c := newScaNNChecker()
|
||||
for _, test := range cases {
|
||||
err := c.CheckValidDataType(test.dType)
|
||||
err := c.CheckValidDataType(&schemapb.FieldSchema{DataType: test.dType})
|
||||
if test.errIsNil {
|
||||
assert.NoError(t, err)
|
||||
} else {
|
||||
|
|
|
@ -32,8 +32,8 @@ func (c sparseFloatVectorBaseChecker) CheckTrain(params map[string]string) error
|
|||
return nil
|
||||
}
|
||||
|
||||
func (c sparseFloatVectorBaseChecker) CheckValidDataType(dType schemapb.DataType) error {
|
||||
if !typeutil.IsSparseFloatVectorType(dType) {
|
||||
func (c sparseFloatVectorBaseChecker) CheckValidDataType(field *schemapb.FieldSchema) error {
|
||||
if !typeutil.IsSparseFloatVectorType(field.GetDataType()) {
|
||||
return fmt.Errorf("only sparse float vector is supported for the specified index tpye")
|
||||
}
|
||||
return nil
|
||||
|
|
|
@ -16,8 +16,8 @@ func (c *STLSORTChecker) CheckTrain(params map[string]string) error {
|
|||
return c.scalarIndexChecker.CheckTrain(params)
|
||||
}
|
||||
|
||||
func (c *STLSORTChecker) CheckValidDataType(dType schemapb.DataType) error {
|
||||
if !typeutil.IsArithmetic(dType) {
|
||||
func (c *STLSORTChecker) CheckValidDataType(field *schemapb.FieldSchema) error {
|
||||
if !typeutil.IsArithmetic(field.GetDataType()) {
|
||||
return fmt.Errorf("STL_SORT are only supported on numeric field")
|
||||
}
|
||||
return nil
|
||||
|
|
|
@ -13,10 +13,10 @@ func Test_STLSORTIndexChecker(t *testing.T) {
|
|||
|
||||
assert.NoError(t, c.CheckTrain(map[string]string{}))
|
||||
|
||||
assert.NoError(t, c.CheckValidDataType(schemapb.DataType_Int64))
|
||||
assert.NoError(t, c.CheckValidDataType(schemapb.DataType_Float))
|
||||
assert.NoError(t, c.CheckValidDataType(&schemapb.FieldSchema{DataType: schemapb.DataType_Int64}))
|
||||
assert.NoError(t, c.CheckValidDataType(&schemapb.FieldSchema{DataType: schemapb.DataType_Float}))
|
||||
|
||||
assert.Error(t, c.CheckValidDataType(schemapb.DataType_Bool))
|
||||
assert.Error(t, c.CheckValidDataType(schemapb.DataType_VarChar))
|
||||
assert.Error(t, c.CheckValidDataType(schemapb.DataType_JSON))
|
||||
assert.Error(t, c.CheckValidDataType(&schemapb.FieldSchema{DataType: schemapb.DataType_VarChar}))
|
||||
assert.Error(t, c.CheckValidDataType(&schemapb.FieldSchema{DataType: schemapb.DataType_Bool}))
|
||||
assert.Error(t, c.CheckValidDataType(&schemapb.FieldSchema{DataType: schemapb.DataType_JSON}))
|
||||
}
|
||||
|
|
|
@ -16,8 +16,8 @@ func (c *TRIEChecker) CheckTrain(params map[string]string) error {
|
|||
return c.scalarIndexChecker.CheckTrain(params)
|
||||
}
|
||||
|
||||
func (c *TRIEChecker) CheckValidDataType(dType schemapb.DataType) error {
|
||||
if !typeutil.IsStringType(dType) {
|
||||
func (c *TRIEChecker) CheckValidDataType(field *schemapb.FieldSchema) error {
|
||||
if !typeutil.IsStringType(field.GetDataType()) {
|
||||
return fmt.Errorf("TRIE are only supported on varchar field")
|
||||
}
|
||||
return nil
|
||||
|
|
|
@ -13,11 +13,11 @@ func Test_TrieIndexChecker(t *testing.T) {
|
|||
|
||||
assert.NoError(t, c.CheckTrain(map[string]string{}))
|
||||
|
||||
assert.NoError(t, c.CheckValidDataType(schemapb.DataType_VarChar))
|
||||
assert.NoError(t, c.CheckValidDataType(schemapb.DataType_String))
|
||||
assert.NoError(t, c.CheckValidDataType(&schemapb.FieldSchema{DataType: schemapb.DataType_VarChar}))
|
||||
assert.NoError(t, c.CheckValidDataType(&schemapb.FieldSchema{DataType: schemapb.DataType_String}))
|
||||
|
||||
assert.Error(t, c.CheckValidDataType(schemapb.DataType_Bool))
|
||||
assert.Error(t, c.CheckValidDataType(schemapb.DataType_Int64))
|
||||
assert.Error(t, c.CheckValidDataType(schemapb.DataType_Float))
|
||||
assert.Error(t, c.CheckValidDataType(schemapb.DataType_JSON))
|
||||
assert.Error(t, c.CheckValidDataType(&schemapb.FieldSchema{DataType: schemapb.DataType_Bool}))
|
||||
assert.Error(t, c.CheckValidDataType(&schemapb.FieldSchema{DataType: schemapb.DataType_Int64}))
|
||||
assert.Error(t, c.CheckValidDataType(&schemapb.FieldSchema{DataType: schemapb.DataType_Float}))
|
||||
assert.Error(t, c.CheckValidDataType(&schemapb.FieldSchema{DataType: schemapb.DataType_JSON}))
|
||||
}
|
||||
|
|
Loading…
Reference in New Issue