milvus-io
/
milvus
mirror of https://github.com/milvus-io/milvus.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

[restapi] Fix insert request lost precision for int64 fields (#20468) 

Signed-off-by: shaoyue.chen <shaoyue.chen@zilliz.com>

Signed-off-by: shaoyue.chen <shaoyue.chen@zilliz.com>
pull/20830/head
shaoyue 2022-11-25 10:43:12 +08:00 committed by GitHub
parent 83d3007744
commit 37725d3470
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 101 additions and 116 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
internal/distributed/proxy/httpserver/handler.go
View File

@ -322,20 +322,11 @@ func (h *Handlers) handleInsert(c *gin.Context) (interface{}, error) {
if err != nil {
return nil, fmt.Errorf("%w: parse body failed: %v", errBadRequest, err)
}
fieldData, err := convertFieldDataArray(wrappedReq.FieldsData)
req, err := wrappedReq.AsInsertRequest()
if err != nil {
return nil, fmt.Errorf("%w: convert field data failed: %v", errBadRequest, err)
return nil, fmt.Errorf("%w: convert body to pb failed: %v", errBadRequest, err)
}
req := milvuspb.InsertRequest{
Base: wrappedReq.Base,
DbName: wrappedReq.DbName,
CollectionName: wrappedReq.CollectionName,
PartitionName: wrappedReq.PartitionName,
FieldsData: fieldData,
HashKeys: wrappedReq.HashKeys,
NumRows: wrappedReq.NumRows,
}
return h.proxy.Insert(c, &req)
return h.proxy.Insert(c, req)
}
func (h *Handlers) handleDelete(c *gin.Context) (interface{}, error) {

15
internal/distributed/proxy/httpserver/handler_test.go
View File

@ -19,6 +19,21 @@ import (
"github.com/stretchr/testify/assert"
)
func Test_WrappedInsertRequest_JSONMarshal_AsInsertRequest(t *testing.T) {
// https://github.com/milvus-io/milvus/issues/20415
insertRaw := []byte(`{
"collection_name": "seller_tag",
"fields_data": [{ "field_name":"kid","type":5,"field":[9999999999999999] },{ "field_name":"seller_id","type":5,"field":[5625300123280813090] },{ "field_name":"vector","type":101,"field":[[0.08090433, 0.19154754, 0.16858263, 0.027101958, 0.07229418, 0.15223257, -0.024227709, 0.13302892, 0.05951315, 0.03572949, -0.015721956, -0.21992287, 0.08134472, 0.18640009, -0.09814235, -0.11117617, 0.10464557, -0.092037976, -0.19489805, -0.069008306, -0.039415136, -0.17841195, 0.076126315, 0.031378396, 0.22680397, 0.045089707, 0.12307317, 0.06711619, 0.15067382, -0.213569, 0.066602595, -0.021743167, -0.2727193, -0.112709574, 0.09504322, 0.02386695, 0.04574049, -0.055642836, -0.16812482, -0.051256, -0.11399734, 0.29519975, 0.109542266, 0.18452083, 0.05543076, -0.064969495, -0.14457555, -0.034600936, 0.045484997, -0.15677887, -0.12983392, 0.20921704, -0.049788076, 0.050687622, -0.23369887, -0.022488454, 0.06089106, 0.14699098, -0.08140416, -0.008949298, -0.14867777, 0.07415456, -0.0027948048, 0.0060837376]] }],
"num_rows": 1
}`)
wrappedInsert := new(WrappedInsertRequest)
err := json.Unmarshal(insertRaw, &wrappedInsert)
assert.NoError(t, err)
insertReq, err := wrappedInsert.AsInsertRequest()
assert.NoError(t, err)
assert.Equal(t, int64(9999999999999999), insertReq.FieldsData[0].GetScalars().GetLongData().Data[0])
}
type mockProxyComponent struct {
// wrap the interface to avoid implement not used func.
// and to let not implemented call panics

131
internal/distributed/proxy/httpserver/wrap_request.go
View File

@ -2,10 +2,10 @@ package httpserver
import (
"encoding/binary"
"encoding/json"
"errors"
"fmt"
"math"
"reflect"
"github.com/golang/protobuf/proto"
"github.com/milvus-io/milvus-proto/go-api/commonpb"
@ -51,11 +51,27 @@ type WrappedInsertRequest struct {
NumRows uint32 `json:"num_rows,omitempty"`
}
func (w *WrappedInsertRequest) AsInsertRequest() (*milvuspb.InsertRequest, error) {
fieldData, err := convertFieldDataArray(w.FieldsData)
if err != nil {
return nil, fmt.Errorf("%w: convert field data failed: %v", errBadRequest, err)
}
return &milvuspb.InsertRequest{
Base: w.Base,
DbName: w.DbName,
CollectionName: w.CollectionName,
PartitionName: w.PartitionName,
FieldsData: fieldData,
HashKeys: w.HashKeys,
NumRows: w.NumRows,
}, nil
}
// FieldData is the field data in RESTful request that can be convertd to schemapb.FieldData
type FieldData struct {
Type schemapb.DataType `json:"type,omitempty"`
FieldName string `json:"field_name,omitempty"`
Field []interface{} `json:"field,omitempty"`
Field json.RawMessage `json:"field,omitempty"` // we use postpone the unmarshal until we know the type
FieldID int64 `json:"field_id,omitempty"`
}
@ -67,19 +83,14 @@ func (f FieldData) AsSchemapb() (*schemapb.FieldData, error) {
FieldName: f.FieldName,
FieldId: f.FieldID,
}
raw := f.Field
switch f.Type {
case schemapb.DataType_Bool:
// its an array in definition, so we only need to check the type of first element
if len(raw) > 0 {
_, ok := raw[0].(bool)
if !ok {
return nil, newTypeError(raw[0])
}
}
data := make([]bool, len(raw))
for i, v := range raw {
data[i] = v.(bool)
data := []bool{}
err := json.Unmarshal(raw, &data)
if err != nil {
return nil, newFieldDataError(f.FieldName, err)
}
ret.Field = &schemapb.FieldData_Scalars{
Scalars: &schemapb.ScalarField{
@ -91,15 +102,10 @@ func (f FieldData) AsSchemapb() (*schemapb.FieldData, error) {
},
}
case schemapb.DataType_VarChar:
if len(raw) > 0 {
_, ok := raw[0].(string)
if !ok {
return nil, newTypeError(raw[0])
}
}
data := make([]string, len(raw))
for i, v := range raw {
data[i] = v.(string)
data := []string{}
err := json.Unmarshal(raw, &data)
if err != nil {
return nil, newFieldDataError(f.FieldName, err)
}
ret.Field = &schemapb.FieldData_Scalars{
Scalars: &schemapb.ScalarField{
@ -111,15 +117,10 @@ func (f FieldData) AsSchemapb() (*schemapb.FieldData, error) {
},
}
case schemapb.DataType_Int8, schemapb.DataType_Int16, schemapb.DataType_Int32:
if len(raw) > 0 {
_, ok := raw[0].(float64)
if !ok {
return nil, newTypeError(raw[0])
}
}
data := make([]int32, len(raw))
for i, v := range raw {
data[i] = int32(v.(float64))
data := []int32{}
err := json.Unmarshal(raw, &data)
if err != nil {
return nil, newFieldDataError(f.FieldName, err)
}
ret.Field = &schemapb.FieldData_Scalars{
Scalars: &schemapb.ScalarField{
@ -131,15 +132,10 @@ func (f FieldData) AsSchemapb() (*schemapb.FieldData, error) {
},
}
case schemapb.DataType_Int64:
if len(raw) > 0 {
_, ok := raw[0].(float64)
if !ok {
return nil, newTypeError(raw[0])
}
}
data := make([]int64, len(raw))
for i, v := range raw {
data[i] = int64(v.(float64))
data := []int64{}
err := json.Unmarshal(raw, &data)
if err != nil {
return nil, newFieldDataError(f.FieldName, err)
}
ret.Field = &schemapb.FieldData_Scalars{
Scalars: &schemapb.ScalarField{
@ -151,15 +147,10 @@ func (f FieldData) AsSchemapb() (*schemapb.FieldData, error) {
},
}
case schemapb.DataType_Float:
if len(raw) > 0 {
_, ok := raw[0].(float64)
if !ok {
return nil, newTypeError(raw[0])
}
}
data := make([]float32, len(raw))
for i, v := range raw {
data[i] = float32(v.(float64))
data := []float32{}
err := json.Unmarshal(raw, &data)
if err != nil {
return nil, newFieldDataError(f.FieldName, err)
}
ret.Field = &schemapb.FieldData_Scalars{
Scalars: &schemapb.ScalarField{
@ -172,15 +163,10 @@ func (f FieldData) AsSchemapb() (*schemapb.FieldData, error) {
}
case schemapb.DataType_Double:
if len(raw) > 0 {
_, ok := raw[0].(float64)
if !ok {
return nil, newTypeError(raw[0])
}
}
data := make([]float64, len(raw))
for i, v := range raw {
data[i] = v.(float64)
data := []float64{}
err := json.Unmarshal(raw, &data)
if err != nil {
return nil, newFieldDataError(f.FieldName, err)
}
ret.Field = &schemapb.FieldData_Scalars{
Scalars: &schemapb.ScalarField{
@ -193,28 +179,25 @@ func (f FieldData) AsSchemapb() (*schemapb.FieldData, error) {
}
case schemapb.DataType_FloatVector:
if len(raw) < 1 {
wrappedData := [][]float32{}
err := json.Unmarshal(raw, &wrappedData)
if err != nil {
return nil, newFieldDataError(f.FieldName, err)
}
if len(wrappedData) < 1 {
return nil, errors.New("at least one row for insert")
}
rawArray0, ok := raw[0].([]interface{})
if !ok {
return nil, newTypeError(raw[0])
}
dim := len(rawArray0)
array0 := wrappedData[0]
dim := len(array0)
if dim < 1 {
return nil, errors.New("dim must >= 1")
}
_, ok = rawArray0[0].(float64)
if !ok {
return nil, newTypeError(rawArray0[0])
}
data := make([]float32, len(raw)*dim)
data := make([]float32, len(wrappedData)*dim)
var i int
for _, rawArray := range raw {
for _, v := range rawArray.([]interface{}) {
data[i] = float32(v.(float64))
for _, dataArray := range wrappedData {
for _, v := range dataArray {
data[i] = v
i++
}
}
@ -234,8 +217,8 @@ func (f FieldData) AsSchemapb() (*schemapb.FieldData, error) {
return &ret, nil
}
func newTypeError(t interface{}) error {
return fmt.Errorf("field type[%s] error", reflect.TypeOf(t).String())
func newFieldDataError(field string, err error) error {
return fmt.Errorf("parse field[%s]: %s", field, err.Error())
}
func convertFieldDataArray(input []*FieldData) ([]*schemapb.FieldData, error) {

56
internal/distributed/proxy/httpserver/wrap_request_test.go
View File

@ -13,7 +13,7 @@ func TestFieldData_AsSchemapb(t *testing.T) {
t.Run("varchar_ok", func(t *testing.T) {
fieldData := FieldData{
Type: schemapb.DataType_VarChar,
Field: []interface{}{"a", "b", "c"},
Field: []byte(`["a", "b", "c"]`),
}
raw, _ := json.Marshal(fieldData)
json.Unmarshal(raw, &fieldData)
@ -23,7 +23,7 @@ func TestFieldData_AsSchemapb(t *testing.T) {
t.Run("varchar_error", func(t *testing.T) {
fieldData := FieldData{
Type: schemapb.DataType_VarChar,
Field: []interface{}{1, 2, 3},
Field: []byte("[1, 2, 3]"),
}
raw, _ := json.Marshal(fieldData)
json.Unmarshal(raw, &fieldData)
@ -33,7 +33,7 @@ func TestFieldData_AsSchemapb(t *testing.T) {
t.Run("bool_ok", func(t *testing.T) {
fieldData := FieldData{
Type: schemapb.DataType_Bool,
Field: []interface{}{true, true, false},
Field: []byte("[true, true, false]"),
}
raw, _ := json.Marshal(fieldData)
json.Unmarshal(raw, &fieldData)
@ -43,7 +43,7 @@ func TestFieldData_AsSchemapb(t *testing.T) {
t.Run("bool_error", func(t *testing.T) {
fieldData := FieldData{
Type: schemapb.DataType_Bool,
Field: []interface{}{1, 2, 3},
Field: []byte("[1, 2, 3]"),
}
raw, _ := json.Marshal(fieldData)
json.Unmarshal(raw, &fieldData)
@ -54,7 +54,7 @@ func TestFieldData_AsSchemapb(t *testing.T) {
t.Run("int8_ok", func(t *testing.T) {
fieldData := FieldData{
Type: schemapb.DataType_Int8,
Field: []interface{}{1, 2, 3},
Field: []byte("[1, 2, 3]"),
}
raw, _ := json.Marshal(fieldData)
json.Unmarshal(raw, &fieldData)
@ -64,7 +64,7 @@ func TestFieldData_AsSchemapb(t *testing.T) {
t.Run("int8_error", func(t *testing.T) {
fieldData := FieldData{
Type: schemapb.DataType_Int8,
Field: []interface{}{"a", "b", "c"},
Field: []byte(`["a", "b", "c"]`),
}
raw, _ := json.Marshal(fieldData)
json.Unmarshal(raw, &fieldData)
@ -74,7 +74,7 @@ func TestFieldData_AsSchemapb(t *testing.T) {
t.Run("int32_ok", func(t *testing.T) {
fieldData := FieldData{
Type: schemapb.DataType_Int32,
Field: []interface{}{1, 2, 3},
Field: []byte("[1, 2, 3]"),
}
raw, _ := json.Marshal(fieldData)
json.Unmarshal(raw, &fieldData)
@ -84,7 +84,7 @@ func TestFieldData_AsSchemapb(t *testing.T) {
t.Run("int32_error", func(t *testing.T) {
fieldData := FieldData{
Type: schemapb.DataType_Int32,
Field: []interface{}{"a", "b", "c"},
Field: []byte(`["a", "b", "c"]`),
}
raw, _ := json.Marshal(fieldData)
json.Unmarshal(raw, &fieldData)
@ -94,7 +94,7 @@ func TestFieldData_AsSchemapb(t *testing.T) {
t.Run("int64_ok", func(t *testing.T) {
fieldData := FieldData{
Type: schemapb.DataType_Int64,
Field: []interface{}{1, 2, 3},
Field: []byte("[1, 2, 3]"),
}
raw, _ := json.Marshal(fieldData)
json.Unmarshal(raw, &fieldData)
@ -104,7 +104,7 @@ func TestFieldData_AsSchemapb(t *testing.T) {
t.Run("int64_error", func(t *testing.T) {
fieldData := FieldData{
Type: schemapb.DataType_Int64,
Field: []interface{}{"a", "b", "c"},
Field: []byte(`["a", "b", "c"]`),
}
raw, _ := json.Marshal(fieldData)
json.Unmarshal(raw, &fieldData)
@ -114,7 +114,7 @@ func TestFieldData_AsSchemapb(t *testing.T) {
t.Run("float_ok", func(t *testing.T) {
fieldData := FieldData{
Type: schemapb.DataType_Float,
Field: []interface{}{1.1, 2.1, 3.1},
Field: []byte(`[1.1, 2.1, 3.1]`),
}
raw, _ := json.Marshal(fieldData)
json.Unmarshal(raw, &fieldData)
@ -124,7 +124,7 @@ func TestFieldData_AsSchemapb(t *testing.T) {
t.Run("float_error", func(t *testing.T) {
fieldData := FieldData{
Type: schemapb.DataType_Float,
Field: []interface{}{"a", "b", "c"},
Field: []byte(`["a", "b", "c"]`),
}
raw, _ := json.Marshal(fieldData)
json.Unmarshal(raw, &fieldData)
@ -134,7 +134,7 @@ func TestFieldData_AsSchemapb(t *testing.T) {
t.Run("double_ok", func(t *testing.T) {
fieldData := FieldData{
Type: schemapb.DataType_Double,
Field: []interface{}{1.1, 2.1, 3.1},
Field: []byte(`[1.1, 2.1, 3.1]`),
}
raw, _ := json.Marshal(fieldData)
json.Unmarshal(raw, &fieldData)
@ -144,7 +144,7 @@ func TestFieldData_AsSchemapb(t *testing.T) {
t.Run("double_error", func(t *testing.T) {
fieldData := FieldData{
Type: schemapb.DataType_Double,
Field: []interface{}{"a", "b", "c"},
Field: []byte(`["a", "b", "c"]`),
}
raw, _ := json.Marshal(fieldData)
json.Unmarshal(raw, &fieldData)
@ -154,7 +154,7 @@ func TestFieldData_AsSchemapb(t *testing.T) {
t.Run("string_not_support", func(t *testing.T) {
fieldData := FieldData{
Type: schemapb.DataType_String,
Field: []interface{}{"a", "b", "c"},
Field: []byte(`["a", "b", "c"]`),
}
raw, _ := json.Marshal(fieldData)
json.Unmarshal(raw, &fieldData)
@ -167,11 +167,11 @@ func TestFieldData_AsSchemapb(t *testing.T) {
t.Run("floatvector_ok", func(t *testing.T) {
fieldData := FieldData{
Type: schemapb.DataType_FloatVector,
Field: []interface{}{
[]float32{1.1, 2.2, 3.1},
[]float32{1.1, 2.2, 3.1},
[]float32{1.1, 2.2, 3.1},
},
Field: []byte(`[
[1.1, 2.2, 3.1],
[1.1, 2.2, 3.1],
[1.1, 2.2, 3.1]
]`),
}
raw, _ := json.Marshal(fieldData)
json.Unmarshal(raw, &fieldData)
@ -181,7 +181,7 @@ func TestFieldData_AsSchemapb(t *testing.T) {
t.Run("floatvector_empty_error", func(t *testing.T) {
fieldData := FieldData{
Type: schemapb.DataType_FloatVector,
Field: []interface{}{},
Field: []byte(""),
}
raw, _ := json.Marshal(fieldData)
json.Unmarshal(raw, &fieldData)
@ -190,10 +190,8 @@ func TestFieldData_AsSchemapb(t *testing.T) {
})
t.Run("floatvector_dim=0_error", func(t *testing.T) {
fieldData := FieldData{
Type: schemapb.DataType_FloatVector,
Field: []interface{}{
[]float32{},
},
Type: schemapb.DataType_FloatVector,
Field: []byte(`[]`),
}
raw, _ := json.Marshal(fieldData)
json.Unmarshal(raw, &fieldData)
@ -202,10 +200,8 @@ func TestFieldData_AsSchemapb(t *testing.T) {
})
t.Run("floatvector_vectorTypeError_error", func(t *testing.T) {
fieldData := FieldData{
Type: schemapb.DataType_FloatVector,
Field: []interface{}{
[]string{"1"},
},
Type: schemapb.DataType_FloatVector,
Field: []byte(`["1"]`),
}
raw, _ := json.Marshal(fieldData)
json.Unmarshal(raw, &fieldData)
@ -215,7 +211,7 @@ func TestFieldData_AsSchemapb(t *testing.T) {
t.Run("floatvector_error", func(t *testing.T) {
fieldData := FieldData{
Type: schemapb.DataType_FloatVector,
Field: []interface{}{"a", "b", "c"},
Field: []byte(`["a", "b", "c"]`),
}
raw, _ := json.Marshal(fieldData)
json.Unmarshal(raw, &fieldData)