fix: to fill default value when nullable and default value are both enable (#36030)

#36003

Signed-off-by: lixinguo <xinguo.li@zilliz.com>
Co-authored-by: lixinguo <xinguo.li@zilliz.com>

internal/proxy/validate_util.go

@@ -353,7 +353,12 @@ func (v *validateUtil) fillWithNullValue(field *schemapb.FieldData, fieldSchema
 			}
 
 		case *schemapb.ScalarField_ArrayData:
-			// Todo: support it
+			if fieldSchema.GetNullable() {
+				sd.ArrayData.Data, err = fillWithNullValueImpl(sd.ArrayData.Data, field.GetValidData())
+				if err != nil {
+					return err
+				}
+			}
 
 		case *schemapb.ScalarField_JsonData:
 			if fieldSchema.GetNullable() {
@@ -391,10 +396,6 @@ func (v *validateUtil) fillWithDefaultValue(field *schemapb.FieldData, fieldSche
 				return err
 			}
 
-			if !fieldSchema.GetNullable() {
-				field.ValidData = []bool{}
-			}
-
 		case *schemapb.ScalarField_IntData:
 			if len(field.GetValidData()) != numRows {
 				msg := fmt.Sprintf("the length of valid_data of field(%s) is wrong", field.GetFieldName())
@@ -406,10 +407,6 @@ func (v *validateUtil) fillWithDefaultValue(field *schemapb.FieldData, fieldSche
 				return err
 			}
 
-			if !fieldSchema.GetNullable() {
-				field.ValidData = []bool{}
-			}
-
 		case *schemapb.ScalarField_LongData:
 			if len(field.GetValidData()) != numRows {
 				msg := fmt.Sprintf("the length of valid_data of field(%s) is wrong", field.GetFieldName())
@@ -420,9 +417,6 @@ func (v *validateUtil) fillWithDefaultValue(field *schemapb.FieldData, fieldSche
 			if err != nil {
 				return err
 			}
-			if !fieldSchema.GetNullable() {
-				field.ValidData = []bool{}
-			}
 
 		case *schemapb.ScalarField_FloatData:
 			if len(field.GetValidData()) != numRows {
@@ -435,10 +429,6 @@ func (v *validateUtil) fillWithDefaultValue(field *schemapb.FieldData, fieldSche
 				return err
 			}
 
-			if !fieldSchema.GetNullable() {
-				field.ValidData = []bool{}
-			}
-
 		case *schemapb.ScalarField_DoubleData:
 			if len(field.GetValidData()) != numRows {
 				msg := fmt.Sprintf("the length of valid_data of field(%s) is wrong", field.GetFieldName())
@@ -450,10 +440,6 @@ func (v *validateUtil) fillWithDefaultValue(field *schemapb.FieldData, fieldSche
 				return err
 			}
 
-			if !fieldSchema.GetNullable() {
-				field.ValidData = []bool{}
-			}
-
 		case *schemapb.ScalarField_StringData:
 			if len(field.GetValidData()) != numRows {
 				msg := fmt.Sprintf("the length of valid_data of field(%s) is wrong", field.GetFieldName())
@@ -465,10 +451,6 @@ func (v *validateUtil) fillWithDefaultValue(field *schemapb.FieldData, fieldSche
 				return err
 			}
 
-			if !fieldSchema.GetNullable() {
-				field.ValidData = []bool{}
-			}
-
 		case *schemapb.ScalarField_ArrayData:
 			// Todo: support it
 			log.Error("array type not support default value", zap.String("fieldSchemaName", field.GetFieldName()))
@@ -485,10 +467,6 @@ func (v *validateUtil) fillWithDefaultValue(field *schemapb.FieldData, fieldSche
 				return err
 			}
 
-			if !fieldSchema.GetNullable() {
-				field.ValidData = []bool{}
-			}
-
 		default:
 			return merr.WrapErrParameterInvalidMsg(fmt.Sprintf("undefined data type:%s", field.Type.String()))
 		}
@@ -501,6 +479,18 @@ func (v *validateUtil) fillWithDefaultValue(field *schemapb.FieldData, fieldSche
 		return merr.WrapErrParameterInvalidMsg(fmt.Sprintf("undefined data type:%s", field.Type.String()))
 	}
 
+	if !typeutil.IsVectorType(field.Type) {
+		if fieldSchema.GetNullable() {
+			validData := make([]bool, numRows)
+			for i := range validData {
+				validData[i] = true
+			}
+			field.ValidData = validData
+		} else {
+			field.ValidData = []bool{}
+		}
+	}
+
 	err = checkValidData(field, fieldSchema, numRows)
 	if err != nil {
 		return err

internal/proxy/validate_util_test.go

@@ -3089,6 +3089,53 @@ func Test_validateUtil_fillWithValue(t *testing.T) {
 		assert.NoError(t, err)
 
 		flag := checkfillWithValueData(data[0].GetScalars().GetBoolData().Data, schema.Fields[0].GetDefaultValue().GetBoolData(), 2)
+		assert.Equal(t, 0, len(data[0].GetValidData()))
+		assert.True(t, flag)
+	})
+
+	t.Run("bool scalars schema set both nullable==true and default value", func(t *testing.T) {
+		data := []*schemapb.FieldData{
+			{
+				FieldName: "test",
+				Type:      schemapb.DataType_Bool,
+				Field: &schemapb.FieldData_Scalars{
+					Scalars: &schemapb.ScalarField{
+						Data: &schemapb.ScalarField_BoolData{
+							BoolData: &schemapb.BoolArray{
+								Data: []bool{},
+							},
+						},
+					},
+				},
+				ValidData: []bool{false, false},
+			},
+		}
+
+		key := true
+		schema := &schemapb.CollectionSchema{
+			Fields: []*schemapb.FieldSchema{
+				{
+					Name: "test",
+					DefaultValue: &schemapb.ValueField{
+						Data: &schemapb.ValueField_BoolData{
+							BoolData: key,
+						},
+					},
+					Nullable: true,
+				},
+			},
+		}
+		h, err := typeutil.CreateSchemaHelper(schema)
+		assert.NoError(t, err)
+
+		v := newValidateUtil()
+
+		err = v.fillWithValue(data, h, 2)
+
+		assert.NoError(t, err)
+
+		flag := checkfillWithValueData(data[0].GetScalars().GetBoolData().Data, schema.Fields[0].GetDefaultValue().GetBoolData(), 2)
+		assert.Equal(t, []bool{true, true}, data[0].GetValidData())
 		assert.True(t, flag)
 	})
 
@@ -3475,6 +3522,53 @@ func Test_validateUtil_fillWithValue(t *testing.T) {
 		assert.NoError(t, err)
 
 		flag := checkfillWithValueData(data[0].GetScalars().GetIntData().Data, schema.Fields[0].GetDefaultValue().GetIntData(), 2)
+		assert.Equal(t, 0, len(data[0].GetValidData()))
+		assert.True(t, flag)
+	})
+
+	t.Run("int scalars schema set both nullable==true and default value", func(t *testing.T) {
+		data := []*schemapb.FieldData{
+			{
+				FieldName: "test",
+				Type:      schemapb.DataType_Int32,
+				Field: &schemapb.FieldData_Scalars{
+					Scalars: &schemapb.ScalarField{
+						Data: &schemapb.ScalarField_IntData{
+							IntData: &schemapb.IntArray{
+								Data: []int32{},
+							},
+						},
+					},
+				},
+				ValidData: []bool{false, false},
+			},
+		}
+
+		schema := &schemapb.CollectionSchema{
+			Fields: []*schemapb.FieldSchema{
+				{
+					Name:     "test",
+					DataType: schemapb.DataType_Int32,
+					DefaultValue: &schemapb.ValueField{
+						Data: &schemapb.ValueField_IntData{
+							IntData: 1,
+						},
+					},
+					Nullable: true,
+				},
+			},
+		}
+		h, err := typeutil.CreateSchemaHelper(schema)
+		assert.NoError(t, err)
+
+		v := newValidateUtil()
+
+		err = v.fillWithValue(data, h, 2)
+
+		assert.NoError(t, err)
+
+		flag := checkfillWithValueData(data[0].GetScalars().GetIntData().Data, schema.Fields[0].GetDefaultValue().GetIntData(), 2)
+		assert.Equal(t, []bool{true, true}, data[0].GetValidData())
 		assert.True(t, flag)
 	})
 
@@ -3862,6 +3956,52 @@ func Test_validateUtil_fillWithValue(t *testing.T) {
 
 		assert.NoError(t, err)
 		flag := checkfillWithValueData(data[0].GetScalars().GetLongData().Data, schema.Fields[0].GetDefaultValue().GetLongData(), 2)
+		assert.Equal(t, 0, len(data[0].GetValidData()))
+		assert.True(t, flag)
+	})
+
+	t.Run("long scalars schema set both nullable==true and default value", func(t *testing.T) {
+		data := []*schemapb.FieldData{
+			{
+				FieldName: "test",
+				Type:      schemapb.DataType_Int64,
+				Field: &schemapb.FieldData_Scalars{
+					Scalars: &schemapb.ScalarField{
+						Data: &schemapb.ScalarField_LongData{
+							LongData: &schemapb.LongArray{
+								Data: []int64{},
+							},
+						},
+					},
+				},
+				ValidData: []bool{false, false},
+			},
+		}
+
+		schema := &schemapb.CollectionSchema{
+			Fields: []*schemapb.FieldSchema{
+				{
+					Name:     "test",
+					DataType: schemapb.DataType_Int32,
+					DefaultValue: &schemapb.ValueField{
+						Data: &schemapb.ValueField_LongData{
+							LongData: 1,
+						},
+					},
+					Nullable: true,
+				},
+			},
+		}
+		h, err := typeutil.CreateSchemaHelper(schema)
+		assert.NoError(t, err)
+
+		v := newValidateUtil()
+
+		err = v.fillWithValue(data, h, 2)
+
+		assert.NoError(t, err)
+		flag := checkfillWithValueData(data[0].GetScalars().GetLongData().Data, schema.Fields[0].GetDefaultValue().GetLongData(), 2)
+		assert.Equal(t, []bool{true, true}, data[0].GetValidData())
 		assert.True(t, flag)
 	})
 
@@ -4251,6 +4391,53 @@ func Test_validateUtil_fillWithValue(t *testing.T) {
 		assert.NoError(t, err)
 
 		flag := checkfillWithValueData(data[0].GetScalars().GetFloatData().Data, schema.Fields[0].GetDefaultValue().GetFloatData(), 2)
+		assert.Equal(t, 0, len(data[0].GetValidData()))
+		assert.True(t, flag)
+	})
+
+	t.Run("float scalars schema set both nullable==true and default value", func(t *testing.T) {
+		data := []*schemapb.FieldData{
+			{
+				FieldName: "test",
+				Type:      schemapb.DataType_Float,
+				Field: &schemapb.FieldData_Scalars{
+					Scalars: &schemapb.ScalarField{
+						Data: &schemapb.ScalarField_FloatData{
+							FloatData: &schemapb.FloatArray{
+								Data: []float32{},
+							},
+						},
+					},
+				},
+				ValidData: []bool{false, false},
+			},
+		}
+
+		schema := &schemapb.CollectionSchema{
+			Fields: []*schemapb.FieldSchema{
+				{
+					Name:     "test",
+					DataType: schemapb.DataType_Float,
+					DefaultValue: &schemapb.ValueField{
+						Data: &schemapb.ValueField_FloatData{
+							FloatData: 1,
+						},
+					},
+					Nullable: true,
+				},
+			},
+		}
+		h, err := typeutil.CreateSchemaHelper(schema)
+		assert.NoError(t, err)
+
+		v := newValidateUtil()
+
+		err = v.fillWithValue(data, h, 2)
+
+		assert.NoError(t, err)
+
+		flag := checkfillWithValueData(data[0].GetScalars().GetFloatData().Data, schema.Fields[0].GetDefaultValue().GetFloatData(), 2)
+		assert.Equal(t, []bool{true, true}, data[0].GetValidData())
 		assert.True(t, flag)
 	})
 
@@ -4639,6 +4826,53 @@ func Test_validateUtil_fillWithValue(t *testing.T) {
 		assert.NoError(t, err)
 
 		flag := checkfillWithValueData(data[0].GetScalars().GetDoubleData().Data, schema.Fields[0].GetDefaultValue().GetDoubleData(), 2)
+		assert.Equal(t, 0, len(data[0].GetValidData()))
+		assert.True(t, flag)
+	})
+
+	t.Run("double scalars schema set both nullable==true and default value", func(t *testing.T) {
+		data := []*schemapb.FieldData{
+			{
+				FieldName: "test",
+				Type:      schemapb.DataType_Double,
+				Field: &schemapb.FieldData_Scalars{
+					Scalars: &schemapb.ScalarField{
+						Data: &schemapb.ScalarField_DoubleData{
+							DoubleData: &schemapb.DoubleArray{
+								Data: []float64{},
+							},
+						},
+					},
+				},
+				ValidData: []bool{false, false},
+			},
+		}
+
+		schema := &schemapb.CollectionSchema{
+			Fields: []*schemapb.FieldSchema{
+				{
+					Name:     "test",
+					DataType: schemapb.DataType_Double,
+					DefaultValue: &schemapb.ValueField{
+						Data: &schemapb.ValueField_DoubleData{
+							DoubleData: 1,
+						},
+					},
+					Nullable: true,
+				},
+			},
+		}
+		h, err := typeutil.CreateSchemaHelper(schema)
+		assert.NoError(t, err)
+
+		v := newValidateUtil()
+
+		err = v.fillWithValue(data, h, 2)
+
+		assert.NoError(t, err)
+
+		flag := checkfillWithValueData(data[0].GetScalars().GetDoubleData().Data, schema.Fields[0].GetDefaultValue().GetDoubleData(), 2)
+		assert.Equal(t, []bool{true, true}, data[0].GetValidData())
 		assert.True(t, flag)
 	})
 
@@ -5027,6 +5261,53 @@ func Test_validateUtil_fillWithValue(t *testing.T) {
 		assert.NoError(t, err)
 
 		flag := checkfillWithValueData(data[0].GetScalars().GetStringData().Data, schema.Fields[0].GetDefaultValue().GetStringData(), 2)
+		assert.Equal(t, 0, len(data[0].GetValidData()))
+		assert.True(t, flag)
+	})
+
+	t.Run("string scalars schema set both nullable==true and default value", func(t *testing.T) {
+		data := []*schemapb.FieldData{
+			{
+				FieldName: "test",
+				Type:      schemapb.DataType_VarChar,
+				Field: &schemapb.FieldData_Scalars{
+					Scalars: &schemapb.ScalarField{
+						Data: &schemapb.ScalarField_StringData{
+							StringData: &schemapb.StringArray{
+								Data: []string{},
+							},
+						},
+					},
+				},
+				ValidData: []bool{false, false},
+			},
+		}
+
+		schema := &schemapb.CollectionSchema{
+			Fields: []*schemapb.FieldSchema{
+				{
+					Name:     "test",
+					DataType: schemapb.DataType_VarChar,
+					DefaultValue: &schemapb.ValueField{
+						Data: &schemapb.ValueField_StringData{
+							StringData: "b",
+						},
+					},
+					Nullable: true,
+				},
+			},
+		}
+		h, err := typeutil.CreateSchemaHelper(schema)
+		assert.NoError(t, err)
+
+		v := newValidateUtil()
+
+		err = v.fillWithValue(data, h, 2)
+
+		assert.NoError(t, err)
+
+		flag := checkfillWithValueData(data[0].GetScalars().GetStringData().Data, schema.Fields[0].GetDefaultValue().GetStringData(), 2)
+		assert.Equal(t, []bool{true, true}, data[0].GetValidData())
 		assert.True(t, flag)
 	})
 
@@ -5409,6 +5690,54 @@ func Test_validateUtil_fillWithValue(t *testing.T) {
 
 		flag, err := checkJsonfillWithValueData(data[0].GetScalars().GetJsonData().Data, schema.Fields[0].GetDefaultValue().GetBytesData(), 2)
 		assert.True(t, flag)
+		assert.Equal(t, 0, len(data[0].GetValidData()))
+		assert.NoError(t, err)
+	})
+
+	t.Run("json scalars schema set both nullable==true and default value", func(t *testing.T) {
+		data := []*schemapb.FieldData{
+			{
+				FieldName: "test",
+				Type:      schemapb.DataType_JSON,
+				Field: &schemapb.FieldData_Scalars{
+					Scalars: &schemapb.ScalarField{
+						Data: &schemapb.ScalarField_JsonData{
+							JsonData: &schemapb.JSONArray{
+								Data: [][]byte{},
+							},
+						},
+					},
+				},
+				ValidData: []bool{false, false},
+			},
+		}
+
+		schema := &schemapb.CollectionSchema{
+			Fields: []*schemapb.FieldSchema{
+				{
+					Name:     "test",
+					DataType: schemapb.DataType_BinaryVector,
+					DefaultValue: &schemapb.ValueField{
+						Data: &schemapb.ValueField_BytesData{
+							BytesData: []byte("{\"Hello\":\"world\"}"),
+						},
+					},
+					Nullable: true,
+				},
+			},
+		}
+		h, err := typeutil.CreateSchemaHelper(schema)
+		assert.NoError(t, err)
+
+		v := newValidateUtil()
+
+		err = v.fillWithValue(data, h, 2)
+
+		assert.NoError(t, err)
+
+		flag, err := checkJsonfillWithValueData(data[0].GetScalars().GetJsonData().Data, schema.Fields[0].GetDefaultValue().GetBytesData(), 2)
+		assert.True(t, flag)
+		assert.Equal(t, []bool{true, true}, data[0].GetValidData())
 		assert.NoError(t, err)
 	})