milvus-io
/
milvus
mirror of https://github.com/milvus-io/milvus.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

fix: to fill default value when nullable and default value are both enable (#36030) 

#36003

Signed-off-by: lixinguo <xinguo.li@zilliz.com>
Co-authored-by: lixinguo <xinguo.li@zilliz.com>
pull/36304/head
smellthemoon 2024-09-19 17:35:11 +08:00 committed by GitHub
parent f65261215b
commit e84c4f2a92
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
2 changed files with 347 additions and 28 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

46
internal/proxy/validate_util.go
View File

@ -353,7 +353,12 @@ func (v *validateUtil) fillWithNullValue(field *schemapb.FieldData, fieldSchema
}
case *schemapb.ScalarField_ArrayData:
// Todo: support it
if fieldSchema.GetNullable() {
sd.ArrayData.Data, err = fillWithNullValueImpl(sd.ArrayData.Data, field.GetValidData())
if err != nil {
return err
}
}
case *schemapb.ScalarField_JsonData:
if fieldSchema.GetNullable() {
@ -391,10 +396,6 @@ func (v *validateUtil) fillWithDefaultValue(field *schemapb.FieldData, fieldSche
return err
}
if !fieldSchema.GetNullable() {
field.ValidData = []bool{}
}
case *schemapb.ScalarField_IntData:
if len(field.GetValidData()) != numRows {
msg := fmt.Sprintf("the length of valid_data of field(%s) is wrong", field.GetFieldName())
@ -406,10 +407,6 @@ func (v *validateUtil) fillWithDefaultValue(field *schemapb.FieldData, fieldSche
return err
}
if !fieldSchema.GetNullable() {
field.ValidData = []bool{}
}
case *schemapb.ScalarField_LongData:
if len(field.GetValidData()) != numRows {
msg := fmt.Sprintf("the length of valid_data of field(%s) is wrong", field.GetFieldName())
@ -420,9 +417,6 @@ func (v *validateUtil) fillWithDefaultValue(field *schemapb.FieldData, fieldSche
if err != nil {
return err
}
if !fieldSchema.GetNullable() {
field.ValidData = []bool{}
}
case *schemapb.ScalarField_FloatData:
if len(field.GetValidData()) != numRows {
@ -435,10 +429,6 @@ func (v *validateUtil) fillWithDefaultValue(field *schemapb.FieldData, fieldSche
return err
}
if !fieldSchema.GetNullable() {
field.ValidData = []bool{}
}
case *schemapb.ScalarField_DoubleData:
if len(field.GetValidData()) != numRows {
msg := fmt.Sprintf("the length of valid_data of field(%s) is wrong", field.GetFieldName())
@ -450,10 +440,6 @@ func (v *validateUtil) fillWithDefaultValue(field *schemapb.FieldData, fieldSche
return err
}
if !fieldSchema.GetNullable() {
field.ValidData = []bool{}
}
case *schemapb.ScalarField_StringData:
if len(field.GetValidData()) != numRows {
msg := fmt.Sprintf("the length of valid_data of field(%s) is wrong", field.GetFieldName())
@ -465,10 +451,6 @@ func (v *validateUtil) fillWithDefaultValue(field *schemapb.FieldData, fieldSche
return err
}
if !fieldSchema.GetNullable() {
field.ValidData = []bool{}
}
case *schemapb.ScalarField_ArrayData:
// Todo: support it
log.Error("array type not support default value", zap.String("fieldSchemaName", field.GetFieldName()))
@ -485,10 +467,6 @@ func (v *validateUtil) fillWithDefaultValue(field *schemapb.FieldData, fieldSche
return err
}
if !fieldSchema.GetNullable() {
field.ValidData = []bool{}
}
default:
return merr.WrapErrParameterInvalidMsg(fmt.Sprintf("undefined data type:%s", field.Type.String()))
}
@ -501,6 +479,18 @@ func (v *validateUtil) fillWithDefaultValue(field *schemapb.FieldData, fieldSche
return merr.WrapErrParameterInvalidMsg(fmt.Sprintf("undefined data type:%s", field.Type.String()))
}
if !typeutil.IsVectorType(field.Type) {
if fieldSchema.GetNullable() {
validData := make([]bool, numRows)
for i := range validData {
validData[i] = true
}
field.ValidData = validData
} else {
field.ValidData = []bool{}
}
}
err = checkValidData(field, fieldSchema, numRows)
if err != nil {
return err

329
internal/proxy/validate_util_test.go
View File

@ -3089,6 +3089,53 @@ func Test_validateUtil_fillWithValue(t *testing.T) {
assert.NoError(t, err)
flag := checkfillWithValueData(data[0].GetScalars().GetBoolData().Data, schema.Fields[0].GetDefaultValue().GetBoolData(), 2)
assert.Equal(t, 0, len(data[0].GetValidData()))
assert.True(t, flag)
})
t.Run("bool scalars schema set both nullable==true and default value", func(t *testing.T) {
data := []*schemapb.FieldData{
{
FieldName: "test",
Type: schemapb.DataType_Bool,
Field: &schemapb.FieldData_Scalars{
Scalars: &schemapb.ScalarField{
Data: &schemapb.ScalarField_BoolData{
BoolData: &schemapb.BoolArray{
Data: []bool{},
},
},
},
},
ValidData: []bool{false, false},
},
}
key := true
schema := &schemapb.CollectionSchema{
Fields: []*schemapb.FieldSchema{
{
Name: "test",
DefaultValue: &schemapb.ValueField{
Data: &schemapb.ValueField_BoolData{
BoolData: key,
},
},
Nullable: true,
},
},
}
h, err := typeutil.CreateSchemaHelper(schema)
assert.NoError(t, err)
v := newValidateUtil()
err = v.fillWithValue(data, h, 2)
assert.NoError(t, err)
flag := checkfillWithValueData(data[0].GetScalars().GetBoolData().Data, schema.Fields[0].GetDefaultValue().GetBoolData(), 2)
assert.Equal(t, []bool{true, true}, data[0].GetValidData())
assert.True(t, flag)
})
@ -3475,6 +3522,53 @@ func Test_validateUtil_fillWithValue(t *testing.T) {
assert.NoError(t, err)
flag := checkfillWithValueData(data[0].GetScalars().GetIntData().Data, schema.Fields[0].GetDefaultValue().GetIntData(), 2)
assert.Equal(t, 0, len(data[0].GetValidData()))
assert.True(t, flag)
})
t.Run("int scalars schema set both nullable==true and default value", func(t *testing.T) {
data := []*schemapb.FieldData{
{
FieldName: "test",
Type: schemapb.DataType_Int32,
Field: &schemapb.FieldData_Scalars{
Scalars: &schemapb.ScalarField{
Data: &schemapb.ScalarField_IntData{
IntData: &schemapb.IntArray{
Data: []int32{},
},
},
},
},
ValidData: []bool{false, false},
},
}
schema := &schemapb.CollectionSchema{
Fields: []*schemapb.FieldSchema{
{
Name: "test",
DataType: schemapb.DataType_Int32,
DefaultValue: &schemapb.ValueField{
Data: &schemapb.ValueField_IntData{
IntData: 1,
},
},
Nullable: true,
},
},
}
h, err := typeutil.CreateSchemaHelper(schema)
assert.NoError(t, err)
v := newValidateUtil()
err = v.fillWithValue(data, h, 2)
assert.NoError(t, err)
flag := checkfillWithValueData(data[0].GetScalars().GetIntData().Data, schema.Fields[0].GetDefaultValue().GetIntData(), 2)
assert.Equal(t, []bool{true, true}, data[0].GetValidData())
assert.True(t, flag)
})
@ -3862,6 +3956,52 @@ func Test_validateUtil_fillWithValue(t *testing.T) {
assert.NoError(t, err)
flag := checkfillWithValueData(data[0].GetScalars().GetLongData().Data, schema.Fields[0].GetDefaultValue().GetLongData(), 2)
assert.Equal(t, 0, len(data[0].GetValidData()))
assert.True(t, flag)
})
t.Run("long scalars schema set both nullable==true and default value", func(t *testing.T) {
data := []*schemapb.FieldData{
{
FieldName: "test",
Type: schemapb.DataType_Int64,
Field: &schemapb.FieldData_Scalars{
Scalars: &schemapb.ScalarField{
Data: &schemapb.ScalarField_LongData{
LongData: &schemapb.LongArray{
Data: []int64{},
},
},
},
},
ValidData: []bool{false, false},
},
}
schema := &schemapb.CollectionSchema{
Fields: []*schemapb.FieldSchema{
{
Name: "test",
DataType: schemapb.DataType_Int32,
DefaultValue: &schemapb.ValueField{
Data: &schemapb.ValueField_LongData{
LongData: 1,
},
},
Nullable: true,
},
},
}
h, err := typeutil.CreateSchemaHelper(schema)
assert.NoError(t, err)
v := newValidateUtil()
err = v.fillWithValue(data, h, 2)
assert.NoError(t, err)
flag := checkfillWithValueData(data[0].GetScalars().GetLongData().Data, schema.Fields[0].GetDefaultValue().GetLongData(), 2)
assert.Equal(t, []bool{true, true}, data[0].GetValidData())
assert.True(t, flag)
})
@ -4251,6 +4391,53 @@ func Test_validateUtil_fillWithValue(t *testing.T) {
assert.NoError(t, err)
flag := checkfillWithValueData(data[0].GetScalars().GetFloatData().Data, schema.Fields[0].GetDefaultValue().GetFloatData(), 2)
assert.Equal(t, 0, len(data[0].GetValidData()))
assert.True(t, flag)
})
t.Run("float scalars schema set both nullable==true and default value", func(t *testing.T) {
data := []*schemapb.FieldData{
{
FieldName: "test",
Type: schemapb.DataType_Float,
Field: &schemapb.FieldData_Scalars{
Scalars: &schemapb.ScalarField{
Data: &schemapb.ScalarField_FloatData{
FloatData: &schemapb.FloatArray{
Data: []float32{},
},
},
},
},
ValidData: []bool{false, false},
},
}
schema := &schemapb.CollectionSchema{
Fields: []*schemapb.FieldSchema{
{
Name: "test",
DataType: schemapb.DataType_Float,
DefaultValue: &schemapb.ValueField{
Data: &schemapb.ValueField_FloatData{
FloatData: 1,
},
},
Nullable: true,
},
},
}
h, err := typeutil.CreateSchemaHelper(schema)
assert.NoError(t, err)
v := newValidateUtil()
err = v.fillWithValue(data, h, 2)
assert.NoError(t, err)
flag := checkfillWithValueData(data[0].GetScalars().GetFloatData().Data, schema.Fields[0].GetDefaultValue().GetFloatData(), 2)
assert.Equal(t, []bool{true, true}, data[0].GetValidData())
assert.True(t, flag)
})
@ -4639,6 +4826,53 @@ func Test_validateUtil_fillWithValue(t *testing.T) {
assert.NoError(t, err)
flag := checkfillWithValueData(data[0].GetScalars().GetDoubleData().Data, schema.Fields[0].GetDefaultValue().GetDoubleData(), 2)
assert.Equal(t, 0, len(data[0].GetValidData()))
assert.True(t, flag)
})
t.Run("double scalars schema set both nullable==true and default value", func(t *testing.T) {
data := []*schemapb.FieldData{
{
FieldName: "test",
Type: schemapb.DataType_Double,
Field: &schemapb.FieldData_Scalars{
Scalars: &schemapb.ScalarField{
Data: &schemapb.ScalarField_DoubleData{
DoubleData: &schemapb.DoubleArray{
Data: []float64{},
},
},
},
},
ValidData: []bool{false, false},
},
}
schema := &schemapb.CollectionSchema{
Fields: []*schemapb.FieldSchema{
{
Name: "test",
DataType: schemapb.DataType_Double,
DefaultValue: &schemapb.ValueField{
Data: &schemapb.ValueField_DoubleData{
DoubleData: 1,
},
},
Nullable: true,
},
},
}
h, err := typeutil.CreateSchemaHelper(schema)
assert.NoError(t, err)
v := newValidateUtil()
err = v.fillWithValue(data, h, 2)
assert.NoError(t, err)
flag := checkfillWithValueData(data[0].GetScalars().GetDoubleData().Data, schema.Fields[0].GetDefaultValue().GetDoubleData(), 2)
assert.Equal(t, []bool{true, true}, data[0].GetValidData())
assert.True(t, flag)
})
@ -5027,6 +5261,53 @@ func Test_validateUtil_fillWithValue(t *testing.T) {
assert.NoError(t, err)
flag := checkfillWithValueData(data[0].GetScalars().GetStringData().Data, schema.Fields[0].GetDefaultValue().GetStringData(), 2)
assert.Equal(t, 0, len(data[0].GetValidData()))
assert.True(t, flag)
})
t.Run("string scalars schema set both nullable==true and default value", func(t *testing.T) {
data := []*schemapb.FieldData{
{
FieldName: "test",
Type: schemapb.DataType_VarChar,
Field: &schemapb.FieldData_Scalars{
Scalars: &schemapb.ScalarField{
Data: &schemapb.ScalarField_StringData{
StringData: &schemapb.StringArray{
Data: []string{},
},
},
},
},
ValidData: []bool{false, false},
},
}
schema := &schemapb.CollectionSchema{
Fields: []*schemapb.FieldSchema{
{
Name: "test",
DataType: schemapb.DataType_VarChar,
DefaultValue: &schemapb.ValueField{
Data: &schemapb.ValueField_StringData{
StringData: "b",
},
},
Nullable: true,
},
},
}
h, err := typeutil.CreateSchemaHelper(schema)
assert.NoError(t, err)
v := newValidateUtil()
err = v.fillWithValue(data, h, 2)
assert.NoError(t, err)
flag := checkfillWithValueData(data[0].GetScalars().GetStringData().Data, schema.Fields[0].GetDefaultValue().GetStringData(), 2)
assert.Equal(t, []bool{true, true}, data[0].GetValidData())
assert.True(t, flag)
})
@ -5409,6 +5690,54 @@ func Test_validateUtil_fillWithValue(t *testing.T) {
flag, err := checkJsonfillWithValueData(data[0].GetScalars().GetJsonData().Data, schema.Fields[0].GetDefaultValue().GetBytesData(), 2)
assert.True(t, flag)
assert.Equal(t, 0, len(data[0].GetValidData()))
assert.NoError(t, err)
})
t.Run("json scalars schema set both nullable==true and default value", func(t *testing.T) {
data := []*schemapb.FieldData{
{
FieldName: "test",
Type: schemapb.DataType_JSON,
Field: &schemapb.FieldData_Scalars{
Scalars: &schemapb.ScalarField{
Data: &schemapb.ScalarField_JsonData{
JsonData: &schemapb.JSONArray{
Data: [][]byte{},
},
},
},
},
ValidData: []bool{false, false},
},
}
schema := &schemapb.CollectionSchema{
Fields: []*schemapb.FieldSchema{
{
Name: "test",
DataType: schemapb.DataType_BinaryVector,
DefaultValue: &schemapb.ValueField{
Data: &schemapb.ValueField_BytesData{
BytesData: []byte("{\"Hello\":\"world\"}"),
},
},
Nullable: true,
},
},
}
h, err := typeutil.CreateSchemaHelper(schema)
assert.NoError(t, err)
v := newValidateUtil()
err = v.fillWithValue(data, h, 2)
assert.NoError(t, err)
flag, err := checkJsonfillWithValueData(data[0].GetScalars().GetJsonData().Data, schema.Fields[0].GetDefaultValue().GetBytesData(), 2)
assert.True(t, flag)
assert.Equal(t, []bool{true, true}, data[0].GetValidData())
assert.NoError(t, err)
})