fix: fix bitmap supported type (#34350)

#34314

Signed-off-by: luzhang <luzhang@zilliz.com>
Co-authored-by: luzhang <luzhang@zilliz.com>
pull/34354/head
zhagnlu 2024-07-05 15:50:10 +08:00 committed by GitHub
parent 21fc5f5d46
commit 18f2458385
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
29 changed files with 84 additions and 58 deletions

View File

@ -395,7 +395,7 @@ func checkTrain(field *schemapb.FieldSchema, indexParams map[string]string) erro
indexParams[IsSparseKey] = "true" indexParams[IsSparseKey] = "true"
} }
if err := checker.CheckValidDataType(field.GetDataType()); err != nil { if err := checker.CheckValidDataType(field); err != nil {
log.Info("create index with invalid data type", zap.Error(err), zap.String("data_type", field.GetDataType().String())) log.Info("create index with invalid data type", zap.Error(err), zap.String("data_type", field.GetDataType().String()))
return err return err
} }

View File

@ -13,7 +13,7 @@ func (c *AUTOINDEXChecker) CheckTrain(params map[string]string) error {
return nil return nil
} }
func (c *AUTOINDEXChecker) CheckValidDataType(dType schemapb.DataType) error { func (c *AUTOINDEXChecker) CheckValidDataType(field *schemapb.FieldSchema) error {
return nil return nil
} }

View File

@ -55,7 +55,7 @@ func (c baseChecker) CheckTrain(params map[string]string) error {
} }
// CheckValidDataType check whether the field data type is supported for the index type // CheckValidDataType check whether the field data type is supported for the index type
func (c baseChecker) CheckValidDataType(dType schemapb.DataType) error { func (c baseChecker) CheckValidDataType(field *schemapb.FieldSchema) error {
return nil return nil
} }

View File

@ -114,7 +114,8 @@ func Test_baseChecker_CheckValidDataType(t *testing.T) {
c := newBaseChecker() c := newBaseChecker()
for _, test := range cases { for _, test := range cases {
err := c.CheckValidDataType(test.dType) field_schema := &schemapb.FieldSchema{DataType: test.dType}
err := c.CheckValidDataType(field_schema)
if test.errIsNil { if test.errIsNil {
assert.NoError(t, err) assert.NoError(t, err)
} else { } else {

View File

@ -136,7 +136,8 @@ func Test_binFlatChecker_CheckValidDataType(t *testing.T) {
c := newBinFlatChecker() c := newBinFlatChecker()
for _, test := range cases { for _, test := range cases {
err := c.CheckValidDataType(test.dType) field_schema := &schemapb.FieldSchema{DataType: test.dType}
err := c.CheckValidDataType(field_schema)
if test.errIsNil { if test.errIsNil {
assert.NoError(t, err) assert.NoError(t, err)
} else { } else {

View File

@ -187,7 +187,8 @@ func Test_binIVFFlatChecker_CheckValidDataType(t *testing.T) {
c := newBinIVFFlatChecker() c := newBinIVFFlatChecker()
for _, test := range cases { for _, test := range cases {
err := c.CheckValidDataType(test.dType) field_schema := &schemapb.FieldSchema{DataType: test.dType}
err := c.CheckValidDataType(field_schema)
if test.errIsNil { if test.errIsNil {
assert.NoError(t, err) assert.NoError(t, err)
} else { } else {

View File

@ -27,8 +27,8 @@ func (c binaryVectorBaseChecker) CheckTrain(params map[string]string) error {
return c.staticCheck(params) return c.staticCheck(params)
} }
func (c binaryVectorBaseChecker) CheckValidDataType(dType schemapb.DataType) error { func (c binaryVectorBaseChecker) CheckValidDataType(field *schemapb.FieldSchema) error {
if dType != schemapb.DataType_BinaryVector { if field.GetDataType() != schemapb.DataType_BinaryVector {
return fmt.Errorf("binary vector is only supported") return fmt.Errorf("binary vector is only supported")
} }
return nil return nil

View File

@ -69,7 +69,8 @@ func Test_binaryVectorBaseChecker_CheckValidDataType(t *testing.T) {
c := newBinaryVectorBaseChecker() c := newBinaryVectorBaseChecker()
for _, test := range cases { for _, test := range cases {
err := c.CheckValidDataType(test.dType) field_schema := &schemapb.FieldSchema{DataType: test.dType}
err := c.CheckValidDataType(field_schema)
if test.errIsNil { if test.errIsNil {
assert.NoError(t, err) assert.NoError(t, err)
} else { } else {

View File

@ -13,12 +13,24 @@ func Test_BitmapIndexChecker(t *testing.T) {
assert.NoError(t, c.CheckTrain(map[string]string{"bitmap_cardinality_limit": "100"})) assert.NoError(t, c.CheckTrain(map[string]string{"bitmap_cardinality_limit": "100"}))
assert.NoError(t, c.CheckValidDataType(schemapb.DataType_Int64)) assert.NoError(t, c.CheckValidDataType(&schemapb.FieldSchema{DataType: schemapb.DataType_Bool}))
assert.NoError(t, c.CheckValidDataType(schemapb.DataType_Float)) assert.NoError(t, c.CheckValidDataType(&schemapb.FieldSchema{DataType: schemapb.DataType_Int8}))
assert.NoError(t, c.CheckValidDataType(schemapb.DataType_String)) assert.NoError(t, c.CheckValidDataType(&schemapb.FieldSchema{DataType: schemapb.DataType_Int16}))
assert.NoError(t, c.CheckValidDataType(schemapb.DataType_Array)) assert.NoError(t, c.CheckValidDataType(&schemapb.FieldSchema{DataType: schemapb.DataType_Int32}))
assert.NoError(t, c.CheckValidDataType(&schemapb.FieldSchema{DataType: schemapb.DataType_Int64}))
assert.NoError(t, c.CheckValidDataType(&schemapb.FieldSchema{DataType: schemapb.DataType_String}))
assert.NoError(t, c.CheckValidDataType(&schemapb.FieldSchema{DataType: schemapb.DataType_Array, ElementType: schemapb.DataType_Bool}))
assert.NoError(t, c.CheckValidDataType(&schemapb.FieldSchema{DataType: schemapb.DataType_Array, ElementType: schemapb.DataType_Int8}))
assert.NoError(t, c.CheckValidDataType(&schemapb.FieldSchema{DataType: schemapb.DataType_Array, ElementType: schemapb.DataType_Int16}))
assert.NoError(t, c.CheckValidDataType(&schemapb.FieldSchema{DataType: schemapb.DataType_Array, ElementType: schemapb.DataType_Int32}))
assert.NoError(t, c.CheckValidDataType(&schemapb.FieldSchema{DataType: schemapb.DataType_Array, ElementType: schemapb.DataType_Int64}))
assert.NoError(t, c.CheckValidDataType(&schemapb.FieldSchema{DataType: schemapb.DataType_Array, ElementType: schemapb.DataType_String}))
assert.Error(t, c.CheckValidDataType(schemapb.DataType_JSON)) assert.Error(t, c.CheckValidDataType(&schemapb.FieldSchema{DataType: schemapb.DataType_JSON}))
assert.Error(t, c.CheckValidDataType(&schemapb.FieldSchema{DataType: schemapb.DataType_Float}))
assert.Error(t, c.CheckValidDataType(&schemapb.FieldSchema{DataType: schemapb.DataType_Double}))
assert.Error(t, c.CheckValidDataType(&schemapb.FieldSchema{DataType: schemapb.DataType_Array, ElementType: schemapb.DataType_Float}))
assert.Error(t, c.CheckValidDataType(&schemapb.FieldSchema{DataType: schemapb.DataType_Array, ElementType: schemapb.DataType_Double}))
assert.Error(t, c.CheckTrain(map[string]string{})) assert.Error(t, c.CheckTrain(map[string]string{}))
assert.Error(t, c.CheckTrain(map[string]string{"bitmap_cardinality_limit": "0"})) assert.Error(t, c.CheckTrain(map[string]string{"bitmap_cardinality_limit": "0"}))
} }

View File

@ -20,9 +20,18 @@ func (c *BITMAPChecker) CheckTrain(params map[string]string) error {
return c.scalarIndexChecker.CheckTrain(params) return c.scalarIndexChecker.CheckTrain(params)
} }
func (c *BITMAPChecker) CheckValidDataType(dType schemapb.DataType) error { func (c *BITMAPChecker) CheckValidDataType(field *schemapb.FieldSchema) error {
if !typeutil.IsArithmetic(dType) && !typeutil.IsStringType(dType) && !typeutil.IsArrayType(dType) { main_type := field.GetDataType()
return fmt.Errorf("bitmap index are only supported on numeric, string and array field") elem_type := field.GetElementType()
if !typeutil.IsBoolType(main_type) && !typeutil.IsIntegerType(main_type) &&
!typeutil.IsStringType(main_type) && !typeutil.IsArrayType(main_type) {
return fmt.Errorf("bitmap index are only supported on bool, int, string and array field")
}
if typeutil.IsArrayType(main_type) {
if !typeutil.IsBoolType(elem_type) && !typeutil.IsIntegerType(elem_type) &&
!typeutil.IsStringType(elem_type) {
return fmt.Errorf("bitmap index are only supported on bool, int, string for array field")
}
} }
return nil return nil
} }

View File

@ -144,7 +144,7 @@ func Test_diskannChecker_CheckValidDataType(t *testing.T) {
c := newDiskannChecker() c := newDiskannChecker()
for _, test := range cases { for _, test := range cases {
err := c.CheckValidDataType(test.dType) err := c.CheckValidDataType(&schemapb.FieldSchema{DataType: test.dType})
if test.errIsNil { if test.errIsNil {
assert.NoError(t, err) assert.NoError(t, err)
} else { } else {

View File

@ -28,8 +28,8 @@ func (c floatVectorBaseChecker) CheckTrain(params map[string]string) error {
return c.staticCheck(params) return c.staticCheck(params)
} }
func (c floatVectorBaseChecker) CheckValidDataType(dType schemapb.DataType) error { func (c floatVectorBaseChecker) CheckValidDataType(field *schemapb.FieldSchema) error {
if !typeutil.IsDenseFloatVectorType(dType) { if !typeutil.IsDenseFloatVectorType(field.GetDataType()) {
return fmt.Errorf("data type should be FloatVector, Float16Vector or BFloat16Vector") return fmt.Errorf("data type should be FloatVector, Float16Vector or BFloat16Vector")
} }
return nil return nil

View File

@ -69,7 +69,7 @@ func Test_floatVectorBaseChecker_CheckValidDataType(t *testing.T) {
c := newFloatVectorBaseChecker() c := newFloatVectorBaseChecker()
for _, test := range cases { for _, test := range cases {
err := c.CheckValidDataType(test.dType) err := c.CheckValidDataType(&schemapb.FieldSchema{DataType: test.dType})
if test.errIsNil { if test.errIsNil {
assert.NoError(t, err) assert.NoError(t, err)
} else { } else {

View File

@ -32,9 +32,9 @@ func (c hnswChecker) CheckTrain(params map[string]string) error {
return c.baseChecker.CheckTrain(params) return c.baseChecker.CheckTrain(params)
} }
func (c hnswChecker) CheckValidDataType(dType schemapb.DataType) error { func (c hnswChecker) CheckValidDataType(field *schemapb.FieldSchema) error {
if !typeutil.IsVectorType(dType) { if !typeutil.IsVectorType(field.GetDataType()) {
return fmt.Errorf("can't build hnsw in not vector type.") return fmt.Errorf("can't build hnsw in not vector type")
} }
return nil return nil
} }

View File

@ -164,7 +164,7 @@ func Test_hnswChecker_CheckValidDataType(t *testing.T) {
c := newHnswChecker() c := newHnswChecker()
for _, test := range cases { for _, test := range cases {
err := c.CheckValidDataType(test.dType) err := c.CheckValidDataType(&schemapb.FieldSchema{DataType: test.dType})
if test.errIsNil { if test.errIsNil {
assert.NoError(t, err) assert.NoError(t, err)
} else { } else {

View File

@ -22,7 +22,7 @@ import (
type IndexChecker interface { type IndexChecker interface {
CheckTrain(map[string]string) error CheckTrain(map[string]string) error
CheckValidDataType(dType schemapb.DataType) error CheckValidDataType(field *schemapb.FieldSchema) error
SetDefaultMetricTypeIfNotExist(map[string]string, schemapb.DataType) SetDefaultMetricTypeIfNotExist(map[string]string, schemapb.DataType)
StaticCheck(map[string]string) error StaticCheck(map[string]string) error
} }

View File

@ -16,7 +16,8 @@ func (c *INVERTEDChecker) CheckTrain(params map[string]string) error {
return c.scalarIndexChecker.CheckTrain(params) return c.scalarIndexChecker.CheckTrain(params)
} }
func (c *INVERTEDChecker) CheckValidDataType(dType schemapb.DataType) error { func (c *INVERTEDChecker) CheckValidDataType(field *schemapb.FieldSchema) error {
dType := field.GetDataType()
if !typeutil.IsBoolType(dType) && !typeutil.IsArithmetic(dType) && !typeutil.IsStringType(dType) && if !typeutil.IsBoolType(dType) && !typeutil.IsArithmetic(dType) && !typeutil.IsStringType(dType) &&
!typeutil.IsArrayType(dType) { !typeutil.IsArrayType(dType) {
return fmt.Errorf("INVERTED are not supported on %s field", dType.String()) return fmt.Errorf("INVERTED are not supported on %s field", dType.String())

View File

@ -13,13 +13,13 @@ func Test_INVERTEDIndexChecker(t *testing.T) {
assert.NoError(t, c.CheckTrain(map[string]string{})) assert.NoError(t, c.CheckTrain(map[string]string{}))
assert.NoError(t, c.CheckValidDataType(schemapb.DataType_VarChar)) assert.NoError(t, c.CheckValidDataType(&schemapb.FieldSchema{DataType: schemapb.DataType_VarChar}))
assert.NoError(t, c.CheckValidDataType(schemapb.DataType_String)) assert.NoError(t, c.CheckValidDataType(&schemapb.FieldSchema{DataType: schemapb.DataType_String}))
assert.NoError(t, c.CheckValidDataType(schemapb.DataType_Bool)) assert.NoError(t, c.CheckValidDataType(&schemapb.FieldSchema{DataType: schemapb.DataType_Bool}))
assert.NoError(t, c.CheckValidDataType(schemapb.DataType_Int64)) assert.NoError(t, c.CheckValidDataType(&schemapb.FieldSchema{DataType: schemapb.DataType_Int64}))
assert.NoError(t, c.CheckValidDataType(schemapb.DataType_Float)) assert.NoError(t, c.CheckValidDataType(&schemapb.FieldSchema{DataType: schemapb.DataType_Float}))
assert.NoError(t, c.CheckValidDataType(schemapb.DataType_Array)) assert.NoError(t, c.CheckValidDataType(&schemapb.FieldSchema{DataType: schemapb.DataType_Array}))
assert.Error(t, c.CheckValidDataType(schemapb.DataType_JSON)) assert.Error(t, c.CheckValidDataType(&schemapb.FieldSchema{DataType: schemapb.DataType_JSON}))
assert.Error(t, c.CheckValidDataType(schemapb.DataType_FloatVector)) assert.Error(t, c.CheckValidDataType(&schemapb.FieldSchema{DataType: schemapb.DataType_FloatVector}))
} }

View File

@ -142,7 +142,7 @@ func Test_ivfBaseChecker_CheckValidDataType(t *testing.T) {
c := newIVFBaseChecker() c := newIVFBaseChecker()
for _, test := range cases { for _, test := range cases {
err := c.CheckValidDataType(test.dType) err := c.CheckValidDataType(&schemapb.FieldSchema{DataType: test.dType})
if test.errIsNil { if test.errIsNil {
assert.NoError(t, err) assert.NoError(t, err)
} else { } else {

View File

@ -207,7 +207,7 @@ func Test_ivfPQChecker_CheckValidDataType(t *testing.T) {
c := newIVFPQChecker() c := newIVFPQChecker()
for _, test := range cases { for _, test := range cases {
err := c.CheckValidDataType(test.dType) err := c.CheckValidDataType(&schemapb.FieldSchema{DataType: test.dType})
if test.errIsNil { if test.errIsNil {
assert.NoError(t, err) assert.NoError(t, err)
} else { } else {

View File

@ -162,7 +162,7 @@ func Test_ivfSQChecker_CheckValidDataType(t *testing.T) {
c := newIVFSQChecker() c := newIVFSQChecker()
for _, test := range cases { for _, test := range cases {
err := c.CheckValidDataType(test.dType) err := c.CheckValidDataType(&schemapb.FieldSchema{DataType: test.dType})
if test.errIsNil { if test.errIsNil {
assert.NoError(t, err) assert.NoError(t, err)
} else { } else {

View File

@ -156,7 +156,7 @@ func Test_raftIvfFlatChecker_CheckValidDataType(t *testing.T) {
c := newRaftIVFFlatChecker() c := newRaftIVFFlatChecker()
for _, test := range cases { for _, test := range cases {
err := c.CheckValidDataType(test.dType) err := c.CheckValidDataType(&schemapb.FieldSchema{DataType: test.dType})
if test.errIsNil { if test.errIsNil {
assert.NoError(t, err) assert.NoError(t, err)
} else { } else {

View File

@ -216,7 +216,7 @@ func Test_raftIVFPQChecker_CheckValidDataType(t *testing.T) {
c := newRaftIVFPQChecker() c := newRaftIVFPQChecker()
for _, test := range cases { for _, test := range cases {
err := c.CheckValidDataType(test.dType) err := c.CheckValidDataType(&schemapb.FieldSchema{DataType: test.dType})
if test.errIsNil { if test.errIsNil {
assert.NoError(t, err) assert.NoError(t, err)
} else { } else {

View File

@ -159,7 +159,7 @@ func Test_scaNNChecker_CheckValidDataType(t *testing.T) {
c := newScaNNChecker() c := newScaNNChecker()
for _, test := range cases { for _, test := range cases {
err := c.CheckValidDataType(test.dType) err := c.CheckValidDataType(&schemapb.FieldSchema{DataType: test.dType})
if test.errIsNil { if test.errIsNil {
assert.NoError(t, err) assert.NoError(t, err)
} else { } else {

View File

@ -32,8 +32,8 @@ func (c sparseFloatVectorBaseChecker) CheckTrain(params map[string]string) error
return nil return nil
} }
func (c sparseFloatVectorBaseChecker) CheckValidDataType(dType schemapb.DataType) error { func (c sparseFloatVectorBaseChecker) CheckValidDataType(field *schemapb.FieldSchema) error {
if !typeutil.IsSparseFloatVectorType(dType) { if !typeutil.IsSparseFloatVectorType(field.GetDataType()) {
return fmt.Errorf("only sparse float vector is supported for the specified index tpye") return fmt.Errorf("only sparse float vector is supported for the specified index tpye")
} }
return nil return nil

View File

@ -16,8 +16,8 @@ func (c *STLSORTChecker) CheckTrain(params map[string]string) error {
return c.scalarIndexChecker.CheckTrain(params) return c.scalarIndexChecker.CheckTrain(params)
} }
func (c *STLSORTChecker) CheckValidDataType(dType schemapb.DataType) error { func (c *STLSORTChecker) CheckValidDataType(field *schemapb.FieldSchema) error {
if !typeutil.IsArithmetic(dType) { if !typeutil.IsArithmetic(field.GetDataType()) {
return fmt.Errorf("STL_SORT are only supported on numeric field") return fmt.Errorf("STL_SORT are only supported on numeric field")
} }
return nil return nil

View File

@ -13,10 +13,10 @@ func Test_STLSORTIndexChecker(t *testing.T) {
assert.NoError(t, c.CheckTrain(map[string]string{})) assert.NoError(t, c.CheckTrain(map[string]string{}))
assert.NoError(t, c.CheckValidDataType(schemapb.DataType_Int64)) assert.NoError(t, c.CheckValidDataType(&schemapb.FieldSchema{DataType: schemapb.DataType_Int64}))
assert.NoError(t, c.CheckValidDataType(schemapb.DataType_Float)) assert.NoError(t, c.CheckValidDataType(&schemapb.FieldSchema{DataType: schemapb.DataType_Float}))
assert.Error(t, c.CheckValidDataType(schemapb.DataType_Bool)) assert.Error(t, c.CheckValidDataType(&schemapb.FieldSchema{DataType: schemapb.DataType_VarChar}))
assert.Error(t, c.CheckValidDataType(schemapb.DataType_VarChar)) assert.Error(t, c.CheckValidDataType(&schemapb.FieldSchema{DataType: schemapb.DataType_Bool}))
assert.Error(t, c.CheckValidDataType(schemapb.DataType_JSON)) assert.Error(t, c.CheckValidDataType(&schemapb.FieldSchema{DataType: schemapb.DataType_JSON}))
} }

View File

@ -16,8 +16,8 @@ func (c *TRIEChecker) CheckTrain(params map[string]string) error {
return c.scalarIndexChecker.CheckTrain(params) return c.scalarIndexChecker.CheckTrain(params)
} }
func (c *TRIEChecker) CheckValidDataType(dType schemapb.DataType) error { func (c *TRIEChecker) CheckValidDataType(field *schemapb.FieldSchema) error {
if !typeutil.IsStringType(dType) { if !typeutil.IsStringType(field.GetDataType()) {
return fmt.Errorf("TRIE are only supported on varchar field") return fmt.Errorf("TRIE are only supported on varchar field")
} }
return nil return nil

View File

@ -13,11 +13,11 @@ func Test_TrieIndexChecker(t *testing.T) {
assert.NoError(t, c.CheckTrain(map[string]string{})) assert.NoError(t, c.CheckTrain(map[string]string{}))
assert.NoError(t, c.CheckValidDataType(schemapb.DataType_VarChar)) assert.NoError(t, c.CheckValidDataType(&schemapb.FieldSchema{DataType: schemapb.DataType_VarChar}))
assert.NoError(t, c.CheckValidDataType(schemapb.DataType_String)) assert.NoError(t, c.CheckValidDataType(&schemapb.FieldSchema{DataType: schemapb.DataType_String}))
assert.Error(t, c.CheckValidDataType(schemapb.DataType_Bool)) assert.Error(t, c.CheckValidDataType(&schemapb.FieldSchema{DataType: schemapb.DataType_Bool}))
assert.Error(t, c.CheckValidDataType(schemapb.DataType_Int64)) assert.Error(t, c.CheckValidDataType(&schemapb.FieldSchema{DataType: schemapb.DataType_Int64}))
assert.Error(t, c.CheckValidDataType(schemapb.DataType_Float)) assert.Error(t, c.CheckValidDataType(&schemapb.FieldSchema{DataType: schemapb.DataType_Float}))
assert.Error(t, c.CheckValidDataType(schemapb.DataType_JSON)) assert.Error(t, c.CheckValidDataType(&schemapb.FieldSchema{DataType: schemapb.DataType_JSON}))
} }