milvus-io
/
milvus
mirror of https://github.com/milvus-io/milvus.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

fix: [restful v2] bug list (#30740) 

bug list
1. element data type is needed while create a collection with an array
field #30638
2. spelling mistake about metricsType #30643
3. new collection enable dynamic field / auto id as user defined #30665
4. convert rowCount from string to int #30661
5. try it's best to create a new collection #30652
6. int64 percious #20415
7. insert into collection which has multi vector fields #30674
8. cannot rename a collection to other database #30700
9. update the request body of "indexes/create" #30769
10. got [] while list indexes of a collection which has no index #30722
11. restful need encode password before call
CreateCredential/UpdateCredential #30730
12. some parameters missed the required label #30737
13.define the field to be or not to be a partition key while create a
collection #30797

enhance: support opentelemetry
enhance: support dataType: Float16Vector & BFloat16Vector #22837
enhance: describe collection will show the field is partition key or not
#30789

Signed-off-by: PowderLi <min.li@zilliz.com>
pull/30862/head
PowderLi 2024-02-27 16:48:56 +08:00 committed by GitHub
parent 7f78e9d40d
commit 19911a411f
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
8 changed files with 606 additions and 256 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
internal/distributed/proxy/httpserver/constant.go
View File

@ -87,13 +87,14 @@ const (
HTTPReturnHas = "has"
HTTPReturnFieldName = "name"
HTTPReturnFieldType = "type"
HTTPReturnFieldPrimaryKey = "primaryKey"
HTTPReturnFieldAutoID = "autoId"
HTTPReturnDescription = "description"
HTTPReturnFieldName = "name"
HTTPReturnFieldType = "type"
HTTPReturnFieldPrimaryKey = "primaryKey"
HTTPReturnFieldPartitionKey = "partitionKey"
HTTPReturnFieldAutoID = "autoId"
HTTPReturnDescription = "description"
HTTPReturnIndexMetricsType = "metricType"
HTTPReturnIndexMetricType = "metricType"
HTTPReturnIndexType = "indexType"
HTTPReturnIndexTotalRows = "totalRows"
HTTPReturnIndexPendingRows = "pendingRows"

3
internal/distributed/proxy/httpserver/handler_v1.go
View File

@ -382,7 +382,8 @@ func (h *HandlersV1) getCollectionDetails(c *gin.Context) {
}
vectorField := ""
for _, field := range coll.Schema.Fields {
if field.DataType == schemapb.DataType_BinaryVector || field.DataType == schemapb.DataType_FloatVector {
if field.DataType == schemapb.DataType_BinaryVector || field.DataType == schemapb.DataType_FloatVector ||
field.DataType == schemapb.DataType_Float16Vector || field.DataType == schemapb.DataType_BFloat16Vector {
vectorField = field.Name
break
}

31
internal/distributed/proxy/httpserver/handler_v1_test.go
View File

@ -56,7 +56,7 @@ var DefaultShowCollectionsResp = milvuspb.ShowCollectionsResponse{
var DefaultDescCollectionResp = milvuspb.DescribeCollectionResponse{
CollectionName: DefaultCollectionName,
Schema: generateCollectionSchema(schemapb.DataType_Int64, false),
Schema: generateCollectionSchema(schemapb.DataType_Int64),
ShardsNum: ShardNumDefault,
Status: &StatusSuccess,
}
@ -272,7 +272,7 @@ func TestVectorCollectionsDescribe(t *testing.T) {
name: "get load status fail",
mp: mp2,
exceptCode: http.StatusOK,
expectedBody: "{\"code\":200,\"data\":{\"collectionName\":\"" + DefaultCollectionName + "\",\"description\":\"\",\"enableDynamicField\":true,\"fields\":[{\"autoId\":false,\"description\":\"\",\"name\":\"book_id\",\"primaryKey\":true,\"type\":\"Int64\"},{\"autoId\":false,\"description\":\"\",\"name\":\"word_count\",\"primaryKey\":false,\"type\":\"Int64\"},{\"autoId\":false,\"description\":\"\",\"name\":\"book_intro\",\"primaryKey\":false,\"type\":\"FloatVector(2)\"}],\"indexes\":[{\"fieldName\":\"book_intro\",\"indexName\":\"" + DefaultIndexName + "\",\"metricType\":\"L2\"}],\"load\":\"\",\"shardsNum\":1}}",
expectedBody: "{\"code\":200,\"data\":{\"collectionName\":\"" + DefaultCollectionName + "\",\"description\":\"\",\"enableDynamicField\":true,\"fields\":[{\"autoId\":false,\"description\":\"\",\"name\":\"book_id\",\"partitionKey\":false,\"primaryKey\":true,\"type\":\"Int64\"},{\"autoId\":false,\"description\":\"\",\"name\":\"word_count\",\"partitionKey\":false,\"primaryKey\":false,\"type\":\"Int64\"},{\"autoId\":false,\"description\":\"\",\"name\":\"book_intro\",\"partitionKey\":false,\"primaryKey\":false,\"type\":\"FloatVector(2)\"}],\"indexes\":[{\"fieldName\":\"book_intro\",\"indexName\":\"" + DefaultIndexName + "\",\"metricType\":\"L2\"}],\"load\":\"\",\"shardsNum\":1}}",
})
mp3 := mocks.NewMockProxy(t)
@ -283,7 +283,7 @@ func TestVectorCollectionsDescribe(t *testing.T) {
name: "get indexes fail",
mp: mp3,
exceptCode: http.StatusOK,
expectedBody: "{\"code\":200,\"data\":{\"collectionName\":\"" + DefaultCollectionName + "\",\"description\":\"\",\"enableDynamicField\":true,\"fields\":[{\"autoId\":false,\"description\":\"\",\"name\":\"book_id\",\"primaryKey\":true,\"type\":\"Int64\"},{\"autoId\":false,\"description\":\"\",\"name\":\"word_count\",\"primaryKey\":false,\"type\":\"Int64\"},{\"autoId\":false,\"description\":\"\",\"name\":\"book_intro\",\"primaryKey\":false,\"type\":\"FloatVector(2)\"}],\"indexes\":[],\"load\":\"LoadStateLoaded\",\"shardsNum\":1}}",
expectedBody: "{\"code\":200,\"data\":{\"collectionName\":\"" + DefaultCollectionName + "\",\"description\":\"\",\"enableDynamicField\":true,\"fields\":[{\"autoId\":false,\"description\":\"\",\"name\":\"book_id\",\"partitionKey\":false,\"primaryKey\":true,\"type\":\"Int64\"},{\"autoId\":false,\"description\":\"\",\"name\":\"word_count\",\"partitionKey\":false,\"primaryKey\":false,\"type\":\"Int64\"},{\"autoId\":false,\"description\":\"\",\"name\":\"book_intro\",\"partitionKey\":false,\"primaryKey\":false,\"type\":\"FloatVector(2)\"}],\"indexes\":[],\"load\":\"LoadStateLoaded\",\"shardsNum\":1}}",
})
mp4 := mocks.NewMockProxy(t)
@ -294,7 +294,7 @@ func TestVectorCollectionsDescribe(t *testing.T) {
name: "show collection details success",
mp: mp4,
exceptCode: http.StatusOK,
expectedBody: "{\"code\":200,\"data\":{\"collectionName\":\"" + DefaultCollectionName + "\",\"description\":\"\",\"enableDynamicField\":true,\"fields\":[{\"autoId\":false,\"description\":\"\",\"name\":\"book_id\",\"primaryKey\":true,\"type\":\"Int64\"},{\"autoId\":false,\"description\":\"\",\"name\":\"word_count\",\"primaryKey\":false,\"type\":\"Int64\"},{\"autoId\":false,\"description\":\"\",\"name\":\"book_intro\",\"primaryKey\":false,\"type\":\"FloatVector(2)\"}],\"indexes\":[{\"fieldName\":\"book_intro\",\"indexName\":\"" + DefaultIndexName + "\",\"metricType\":\"L2\"}],\"load\":\"LoadStateLoaded\",\"shardsNum\":1}}",
expectedBody: "{\"code\":200,\"data\":{\"collectionName\":\"" + DefaultCollectionName + "\",\"description\":\"\",\"enableDynamicField\":true,\"fields\":[{\"autoId\":false,\"description\":\"\",\"name\":\"book_id\",\"partitionKey\":false,\"primaryKey\":true,\"type\":\"Int64\"},{\"autoId\":false,\"description\":\"\",\"name\":\"word_count\",\"partitionKey\":false,\"primaryKey\":false,\"type\":\"Int64\"},{\"autoId\":false,\"description\":\"\",\"name\":\"book_intro\",\"partitionKey\":false,\"primaryKey\":false,\"type\":\"FloatVector(2)\"}],\"indexes\":[{\"fieldName\":\"book_intro\",\"indexName\":\"" + DefaultIndexName + "\",\"metricType\":\"L2\"}],\"load\":\"LoadStateLoaded\",\"shardsNum\":1}}",
})
for _, tt := range testCases {
@ -762,10 +762,9 @@ func TestInsertForDataType(t *testing.T) {
paramtable.Init()
paramtable.Get().Save(proxy.Params.HTTPCfg.AcceptTypeAllowInt64.Key, "true")
schemas := map[string]*schemapb.CollectionSchema{
"[success]kinds of data type": newCollectionSchema(generateCollectionSchema(schemapb.DataType_Int64, false)),
"[success]use binary vector": newCollectionSchema(generateCollectionSchema(schemapb.DataType_Int64, true)),
"[success]with dynamic field": withDynamicField(newCollectionSchema(generateCollectionSchema(schemapb.DataType_Int64, false))),
"[success]with array fields": withArrayField(newCollectionSchema(generateCollectionSchema(schemapb.DataType_Int64, false))),
"[success]kinds of data type": newCollectionSchema(generateCollectionSchema(schemapb.DataType_Int64)),
"[success]with dynamic field": withDynamicField(newCollectionSchema(generateCollectionSchema(schemapb.DataType_Int64))),
"[success]with array fields": withArrayField(newCollectionSchema(generateCollectionSchema(schemapb.DataType_Int64))),
}
for name, schema := range schemas {
t.Run(name, func(t *testing.T) {
@ -836,7 +835,7 @@ func TestReturnInt64(t *testing.T) {
}
for _, dataType := range schemas {
t.Run("[insert]httpCfg.allow: false", func(t *testing.T) {
schema := newCollectionSchema(generateCollectionSchema(dataType, false))
schema := newCollectionSchema(generateCollectionSchema(dataType))
mp := mocks.NewMockProxy(t)
mp.EXPECT().DescribeCollection(mock.Anything, mock.Anything).Return(&milvuspb.DescribeCollectionResponse{
CollectionName: DefaultCollectionName,
@ -867,7 +866,7 @@ func TestReturnInt64(t *testing.T) {
for _, dataType := range schemas {
t.Run("[upsert]httpCfg.allow: false", func(t *testing.T) {
schema := newCollectionSchema(generateCollectionSchema(dataType, false))
schema := newCollectionSchema(generateCollectionSchema(dataType))
mp := mocks.NewMockProxy(t)
mp.EXPECT().DescribeCollection(mock.Anything, mock.Anything).Return(&milvuspb.DescribeCollectionResponse{
CollectionName: DefaultCollectionName,
@ -898,7 +897,7 @@ func TestReturnInt64(t *testing.T) {
for _, dataType := range schemas {
t.Run("[insert]httpCfg.allow: false, Accept-Type-Allow-Int64: true", func(t *testing.T) {
schema := newCollectionSchema(generateCollectionSchema(dataType, false))
schema := newCollectionSchema(generateCollectionSchema(dataType))
mp := mocks.NewMockProxy(t)
mp.EXPECT().DescribeCollection(mock.Anything, mock.Anything).Return(&milvuspb.DescribeCollectionResponse{
CollectionName: DefaultCollectionName,
@ -930,7 +929,7 @@ func TestReturnInt64(t *testing.T) {
for _, dataType := range schemas {
t.Run("[upsert]httpCfg.allow: false, Accept-Type-Allow-Int64: true", func(t *testing.T) {
schema := newCollectionSchema(generateCollectionSchema(dataType, false))
schema := newCollectionSchema(generateCollectionSchema(dataType))
mp := mocks.NewMockProxy(t)
mp.EXPECT().DescribeCollection(mock.Anything, mock.Anything).Return(&milvuspb.DescribeCollectionResponse{
CollectionName: DefaultCollectionName,
@ -963,7 +962,7 @@ func TestReturnInt64(t *testing.T) {
paramtable.Get().Save(proxy.Params.HTTPCfg.AcceptTypeAllowInt64.Key, "true")
for _, dataType := range schemas {
t.Run("[insert]httpCfg.allow: true", func(t *testing.T) {
schema := newCollectionSchema(generateCollectionSchema(dataType, false))
schema := newCollectionSchema(generateCollectionSchema(dataType))
mp := mocks.NewMockProxy(t)
mp.EXPECT().DescribeCollection(mock.Anything, mock.Anything).Return(&milvuspb.DescribeCollectionResponse{
CollectionName: DefaultCollectionName,
@ -994,7 +993,7 @@ func TestReturnInt64(t *testing.T) {
for _, dataType := range schemas {
t.Run("[upsert]httpCfg.allow: true", func(t *testing.T) {
schema := newCollectionSchema(generateCollectionSchema(dataType, false))
schema := newCollectionSchema(generateCollectionSchema(dataType))
mp := mocks.NewMockProxy(t)
mp.EXPECT().DescribeCollection(mock.Anything, mock.Anything).Return(&milvuspb.DescribeCollectionResponse{
CollectionName: DefaultCollectionName,
@ -1025,7 +1024,7 @@ func TestReturnInt64(t *testing.T) {
for _, dataType := range schemas {
t.Run("[insert]httpCfg.allow: true, Accept-Type-Allow-Int64: false", func(t *testing.T) {
schema := newCollectionSchema(generateCollectionSchema(dataType, false))
schema := newCollectionSchema(generateCollectionSchema(dataType))
mp := mocks.NewMockProxy(t)
mp.EXPECT().DescribeCollection(mock.Anything, mock.Anything).Return(&milvuspb.DescribeCollectionResponse{
CollectionName: DefaultCollectionName,
@ -1057,7 +1056,7 @@ func TestReturnInt64(t *testing.T) {
for _, dataType := range schemas {
t.Run("[upsert]httpCfg.allow: true, Accept-Type-Allow-Int64: false", func(t *testing.T) {
schema := newCollectionSchema(generateCollectionSchema(dataType, false))
schema := newCollectionSchema(generateCollectionSchema(dataType))
mp := mocks.NewMockProxy(t)
mp.EXPECT().DescribeCollection(mock.Anything, mock.Anything).Return(&milvuspb.DescribeCollectionResponse{
CollectionName: DefaultCollectionName,

147
internal/distributed/proxy/httpserver/handler_v2.go
View File

@ -6,13 +6,15 @@ import (
"io"
"net/http"
"strconv"
"time"
"github.com/cockroachdb/errors"
"github.com/gin-gonic/gin"
"github.com/gin-gonic/gin/binding"
"github.com/go-playground/validator/v10"
"github.com/golang/protobuf/proto"
"github.com/tidwall/gjson"
"go.opentelemetry.io/otel"
"go.opentelemetry.io/otel/trace"
"go.uber.org/zap"
"google.golang.org/grpc"
@ -23,8 +25,10 @@ import (
"github.com/milvus-io/milvus/internal/types"
"github.com/milvus-io/milvus/pkg/common"
"github.com/milvus-io/milvus/pkg/log"
"github.com/milvus-io/milvus/pkg/util/crypto"
"github.com/milvus-io/milvus/pkg/util/merr"
"github.com/milvus-io/milvus/pkg/util/requestutil"
"github.com/milvus-io/milvus/pkg/util/typeutil"
)
type HandlersV2 struct {
@ -137,7 +141,8 @@ func wrapperPost(newReq newReqFunc, v2 handlerFuncV2) gin.HandlerFunc {
return func(c *gin.Context) {
req := newReq()
if err := c.ShouldBindBodyWith(req, binding.JSON); err != nil {
log.Warn("high level restful api, read parameters from request body fail", zap.Any("request", req), zap.Error(err))
log.Warn("high level restful api, read parameters from request body fail", zap.Error(err),
zap.Any("url", c.Request.URL.Path), zap.Any("request", req))
if _, ok := err.(validator.ValidationErrors); ok {
c.AbortWithStatusJSON(http.StatusOK, gin.H{
HTTPReturnCode: merr.Code(merr.ErrMissingRequiredParameters),
@ -156,7 +161,6 @@ func wrapperPost(newReq newReqFunc, v2 handlerFuncV2) gin.HandlerFunc {
}
return
}
log.Debug("high level restful api, read parameters from request body", zap.Any("request", req))
dbName := ""
if getter, ok := req.(requestutil.DBNameGetter); ok {
dbName = getter.GetDbName()
@ -165,10 +169,14 @@ func wrapperPost(newReq newReqFunc, v2 handlerFuncV2) gin.HandlerFunc {
dbName = DefaultDbName
}
username, _ := c.Get(ContextUsername)
ctx := proxy.NewContextWithMetadata(c, username.(string), dbName)
traceID := strconv.FormatInt(time.Now().UnixNano(), 10)
ctx, span := otel.Tracer(typeutil.ProxyRole).Start(context.Background(), c.Request.URL.Path)
defer span.End()
ctx = proxy.NewContextWithMetadata(ctx, username.(string), dbName)
traceID := span.SpanContext().TraceID().String()
ctx = log.WithTraceID(ctx, traceID)
c.Keys["traceID"] = traceID
log.Ctx(ctx).Debug("high level restful api, read parameters from request body, then start to handle.",
zap.Any("url", c.Request.URL.Path), zap.Any("request", req))
v2(ctx, c, req, dbName)
}
}
@ -207,14 +215,17 @@ func wrapperTraceLog(v2 handlerFuncV2) handlerFuncV2 {
}
func wrapperProxy(ctx context.Context, c *gin.Context, req any, checkAuth bool, ignoreErr bool, handler func(reqCtx context.Context, req any) (any, error)) (interface{}, error) {
if baseGetter, ok := req.(BaseGetter); ok {
span := trace.SpanFromContext(ctx)
span.AddEvent(baseGetter.GetBase().GetMsgType().String())
}
if checkAuth {
err := checkAuthorization(ctx, c, req)
if err != nil {
return nil, err
}
}
// todo delete the message
log.Ctx(ctx).Debug("high level restful api, try to do a grpc call", zap.Any("request", req))
log.Ctx(ctx).Debug("high level restful api, try to do a grpc call", zap.Any("grpcRequest", req))
response, err := handler(ctx, req)
if err == nil {
status, ok := requestutil.GetStatusFromResponse(response)
@ -223,7 +234,7 @@ func wrapperProxy(ctx context.Context, c *gin.Context, req any, checkAuth bool,
}
}
if err != nil {
log.Ctx(ctx).Warn("high level restful api, grpc call failed", zap.Error(err), zap.Any("request", req))
log.Ctx(ctx).Warn("high level restful api, grpc call failed", zap.Error(err), zap.Any("grpcRequest", req))
if !ignoreErr {
c.AbortWithStatusJSON(http.StatusOK, gin.H{HTTPReturnCode: merr.Code(err), HTTPReturnMessage: err.Error()})
}
@ -308,7 +319,7 @@ func (h *HandlersV2) getCollectionDetails(ctx context.Context, c *gin.Context, a
primaryField, ok := getPrimaryField(coll.Schema)
autoID := false
if !ok {
log.Ctx(ctx).Warn("high level restful api, get primary field from collection schema fail", zap.Any("collection schema", coll.Schema))
log.Ctx(ctx).Warn("high level restful api, get primary field from collection schema fail", zap.Any("collection schema", coll.Schema), zap.Any("request", anyReq))
} else {
autoID = primaryField.AutoID
}
@ -325,7 +336,8 @@ func (h *HandlersV2) getCollectionDetails(ctx context.Context, c *gin.Context, a
}
vectorField := ""
for _, field := range coll.Schema.Fields {
if field.DataType == schemapb.DataType_BinaryVector || field.DataType == schemapb.DataType_FloatVector {
if field.DataType == schemapb.DataType_BinaryVector || field.DataType == schemapb.DataType_FloatVector ||
field.DataType == schemapb.DataType_Float16Vector || field.DataType == schemapb.DataType_BFloat16Vector {
vectorField = field.Name
break
}
@ -437,7 +449,10 @@ func (h *HandlersV2) renameCollection(ctx context.Context, c *gin.Context, anyRe
DbName: dbName,
OldName: httpReq.CollectionName,
NewName: httpReq.NewCollectionName,
NewDBName: dbName,
NewDBName: httpReq.NewDbName,
}
if req.NewDBName == "" {
req.NewDBName = dbName
}
resp, err := wrapperProxy(ctx, c, req, h.checkAuth, false, func(reqCtx context.Context, req any) (interface{}, error) {
return h.proxy.RenameCollection(reqCtx, req.(*milvuspb.RenameCollectionRequest))
@ -600,7 +615,7 @@ func (h *HandlersV2) insert(ctx context.Context, c *gin.Context, anyReq any, dbN
body, _ := c.Get(gin.BodyBytesKey)
err, httpReq.Data = checkAndSetData(string(body.([]byte)), collSchema)
if err != nil {
log.Ctx(ctx).Warn("high level restful api, fail to deal with insert data", zap.Any("body", body), zap.Error(err))
log.Ctx(ctx).Warn("high level restful api, fail to deal with insert data", zap.Error(err), zap.String("body", string(body.([]byte))))
c.AbortWithStatusJSON(http.StatusOK, gin.H{
HTTPReturnCode: merr.Code(merr.ErrInvalidInsertData),
HTTPReturnMessage: merr.ErrInvalidInsertData.Error() + ", error: " + err.Error(),
@ -849,9 +864,8 @@ func (h *HandlersV2) hybridSearch(ctx context.Context, c *gin.Context, anyReq an
func (h *HandlersV2) createCollection(ctx context.Context, c *gin.Context, anyReq any, dbName string) (interface{}, error) {
httpReq := anyReq.(*CollectionReq)
var schema []byte
vectorFieldNum := 0
valid := true
var err error
fieldNames := map[string]bool{}
if httpReq.Schema.Fields == nil || len(httpReq.Schema.Fields) == 0 {
schema, err = proto.Marshal(&schemapb.CollectionSchema{
Name: httpReq.CollectionName,
@ -883,24 +897,38 @@ func (h *HandlersV2) createCollection(ctx context.Context, c *gin.Context, anyRe
} else {
collSchema := schemapb.CollectionSchema{
Name: httpReq.CollectionName,
AutoID: EnableAutoID,
AutoID: httpReq.Schema.AutoId,
Fields: []*schemapb.FieldSchema{},
EnableDynamicField: EnableDynamic,
EnableDynamicField: httpReq.Schema.EnableDynamicField,
}
allFields := map[string]bool{}
for _, field := range httpReq.Schema.Fields {
dataType := schemapb.DataType(schemapb.DataType_value[field.DataType])
if dataType == schemapb.DataType_BinaryVector || dataType == schemapb.DataType_FloatVector || dataType == schemapb.DataType_Float16Vector {
allFields[field.FieldName] = true
vectorFieldNum++
} else {
allFields[field.FieldName] = false
fieldDataType, ok := schemapb.DataType_value[field.DataType]
if !ok {
log.Ctx(ctx).Warn("field's data type is invalid(case sensitive).", zap.Any("fieldDataType", field.DataType), zap.Any("field", field))
c.AbortWithStatusJSON(http.StatusOK, gin.H{
HTTPReturnCode: merr.Code(merr.ErrParameterInvalid),
HTTPReturnMessage: merr.ErrParameterInvalid.Error() + ", data type " + field.DataType + " is invalid(case sensitive).",
})
return nil, merr.ErrParameterInvalid
}
dataType := schemapb.DataType(fieldDataType)
fieldSchema := schemapb.FieldSchema{
Name: field.FieldName,
IsPrimaryKey: field.IsPrimary,
DataType: dataType,
TypeParams: []*commonpb.KeyValuePair{},
Name: field.FieldName,
IsPrimaryKey: field.IsPrimary,
IsPartitionKey: field.IsPartitionKey,
DataType: dataType,
TypeParams: []*commonpb.KeyValuePair{},
}
if dataType == schemapb.DataType_Array {
if _, ok := schemapb.DataType_value[field.ElementDataType]; !ok {
log.Ctx(ctx).Warn("element's data type is invalid(case sensitive).", zap.Any("elementDataType", field.ElementDataType), zap.Any("field", field))
c.AbortWithStatusJSON(http.StatusOK, gin.H{
HTTPReturnCode: merr.Code(merr.ErrParameterInvalid),
HTTPReturnMessage: merr.ErrParameterInvalid.Error() + ", element data type " + field.ElementDataType + " is invalid(case sensitive).",
})
return nil, merr.ErrParameterInvalid
}
fieldSchema.ElementType = schemapb.DataType(schemapb.DataType_value[field.ElementDataType])
}
if field.IsPrimary {
fieldSchema.AutoID = httpReq.Schema.AutoId
@ -908,29 +936,13 @@ func (h *HandlersV2) createCollection(ctx context.Context, c *gin.Context, anyRe
for key, fieldParam := range field.ElementTypeParams {
fieldSchema.TypeParams = append(fieldSchema.TypeParams, &commonpb.KeyValuePair{Key: key, Value: fieldParam})
}
allFields[field.FieldName] = true
collSchema.Fields = append(collSchema.Fields, &fieldSchema)
}
for _, indexParam := range httpReq.IndexParams {
vectorField, ok := allFields[indexParam.FieldName]
if ok {
if !vectorField {
valid = false // create index for scaler field is not supported
} else {
vectorFieldNum--
}
} else {
c.AbortWithStatusJSON(http.StatusOK, gin.H{
HTTPReturnCode: merr.Code(merr.ErrMissingRequiredParameters),
HTTPReturnMessage: merr.ErrMissingRequiredParameters.Error() + ", error: `" + indexParam.FieldName + "` hasn't defined in schema",
})
return nil, merr.ErrMissingRequiredParameters
}
fieldNames[field.FieldName] = true
}
schema, err = proto.Marshal(&collSchema)
}
if err != nil {
log.Ctx(ctx).Warn("high level restful api, marshal collection schema fail", zap.Any("request", httpReq), zap.Error(err))
log.Ctx(ctx).Warn("high level restful api, marshal collection schema fail", zap.Error(err), zap.Any("request", anyReq))
c.AbortWithStatusJSON(http.StatusOK, gin.H{
HTTPReturnCode: merr.Code(merr.ErrMarshalCollectionSchema),
HTTPReturnMessage: merr.ErrMarshalCollectionSchema.Error() + ", error: " + err.Error(),
@ -950,17 +962,13 @@ func (h *HandlersV2) createCollection(ctx context.Context, c *gin.Context, anyRe
if err != nil {
return resp, err
}
if !valid || vectorFieldNum > 0 {
c.JSON(http.StatusOK, wrapperReturnDefault())
return resp, err
}
if httpReq.Schema.Fields == nil || len(httpReq.Schema.Fields) == 0 {
createIndexReq := &milvuspb.CreateIndexRequest{
DbName: dbName,
CollectionName: httpReq.CollectionName,
FieldName: VectorFieldName,
IndexName: VectorFieldName,
ExtraParams: []*commonpb.KeyValuePair{{Key: common.MetricTypeKey, Value: httpReq.MetricsType}},
ExtraParams: []*commonpb.KeyValuePair{{Key: common.MetricTypeKey, Value: httpReq.MetricType}},
}
statusResponse, err := wrapperProxy(ctx, c, createIndexReq, h.checkAuth, false, func(reqCtx context.Context, req any) (interface{}, error) {
return h.proxy.CreateIndex(ctx, req.(*milvuspb.CreateIndexRequest))
@ -969,13 +977,24 @@ func (h *HandlersV2) createCollection(ctx context.Context, c *gin.Context, anyRe
return statusResponse, err
}
} else {
if len(httpReq.IndexParams) == 0 {
c.JSON(http.StatusOK, wrapperReturnDefault())
return nil, nil
}
for _, indexParam := range httpReq.IndexParams {
if _, ok := fieldNames[indexParam.FieldName]; !ok {
c.AbortWithStatusJSON(http.StatusOK, gin.H{
HTTPReturnCode: merr.Code(merr.ErrMissingRequiredParameters),
HTTPReturnMessage: merr.ErrMissingRequiredParameters.Error() + ", error: `" + indexParam.FieldName + "` hasn't defined in schema",
})
return nil, merr.ErrMissingRequiredParameters
}
createIndexReq := &milvuspb.CreateIndexRequest{
DbName: dbName,
CollectionName: httpReq.CollectionName,
FieldName: indexParam.FieldName,
IndexName: indexParam.IndexName,
ExtraParams: []*commonpb.KeyValuePair{{Key: common.MetricTypeKey, Value: indexParam.MetricsType}},
ExtraParams: []*commonpb.KeyValuePair{{Key: common.MetricTypeKey, Value: indexParam.MetricType}},
}
statusResponse, err := wrapperProxy(ctx, c, createIndexReq, h.checkAuth, false, func(reqCtx context.Context, req any) (interface{}, error) {
return h.proxy.CreateIndex(ctx, req.(*milvuspb.CreateIndexRequest))
@ -1156,7 +1175,7 @@ func (h *HandlersV2) createUser(ctx context.Context, c *gin.Context, anyReq any,
httpReq := anyReq.(*PasswordReq)
req := &milvuspb.CreateCredentialRequest{
Username: httpReq.UserName,
Password: httpReq.Password,
Password: crypto.Base64Encode(httpReq.Password),
}
resp, err := wrapperProxy(ctx, c, req, h.checkAuth, false, func(reqCtx context.Context, req any) (interface{}, error) {
return h.proxy.CreateCredential(reqCtx, req.(*milvuspb.CreateCredentialRequest))
@ -1171,8 +1190,8 @@ func (h *HandlersV2) updateUser(ctx context.Context, c *gin.Context, anyReq any,
httpReq := anyReq.(*NewPasswordReq)
req := &milvuspb.UpdateCredentialRequest{
Username: httpReq.UserName,
OldPassword: httpReq.Password,
NewPassword: httpReq.NewPassword,
OldPassword: crypto.Base64Encode(httpReq.Password),
NewPassword: crypto.Base64Encode(httpReq.NewPassword),
}
resp, err := wrapperProxy(ctx, c, req, h.checkAuth, false, func(reqCtx context.Context, req any) (interface{}, error) {
return h.proxy.UpdateCredential(reqCtx, req.(*milvuspb.UpdateCredentialRequest))
@ -1326,7 +1345,13 @@ func (h *HandlersV2) listIndexes(ctx context.Context, c *gin.Context, anyReq any
CollectionName: collectionGetter.GetCollectionName(),
}
resp, err := wrapperProxy(ctx, c, req, false, false, func(reqCtx context.Context, req any) (any, error) {
return h.proxy.DescribeIndex(reqCtx, req.(*milvuspb.DescribeIndexRequest))
resp, err := h.proxy.DescribeIndex(reqCtx, req.(*milvuspb.DescribeIndexRequest))
if errors.Is(err, merr.ErrIndexNotFound) {
return &milvuspb.DescribeIndexResponse{
IndexDescriptions: []*milvuspb.IndexDescription{},
}, nil
}
return resp, err
})
if err != nil {
return resp, err
@ -1352,11 +1377,11 @@ func (h *HandlersV2) describeIndex(ctx context.Context, c *gin.Context, anyReq a
if err == nil {
indexInfos := [](map[string]any){}
for _, indexDescription := range resp.(*milvuspb.DescribeIndexResponse).IndexDescriptions {
metricsType := ""
metricType := ""
indexType := ""
for _, pair := range indexDescription.Params {
if pair.Key == common.MetricTypeKey {
metricsType = pair.Value
metricType = pair.Value
} else if pair.Key == common.IndexTypeKey {
indexType = pair.Value
}
@ -1365,7 +1390,7 @@ func (h *HandlersV2) describeIndex(ctx context.Context, c *gin.Context, anyReq a
HTTPIndexName: indexDescription.IndexName,
HTTPIndexField: indexDescription.FieldName,
HTTPReturnIndexType: indexType,
HTTPReturnIndexMetricsType: metricsType,
HTTPReturnIndexMetricType: metricType,
HTTPReturnIndexTotalRows: indexDescription.TotalRows,
HTTPReturnIndexPendingRows: indexDescription.PendingIndexRows,
HTTPReturnIndexIndexedRows: indexDescription.IndexedRows,
@ -1388,11 +1413,11 @@ func (h *HandlersV2) createIndex(ctx context.Context, c *gin.Context, anyReq any
FieldName: indexParam.FieldName,
IndexName: indexParam.IndexName,
ExtraParams: []*commonpb.KeyValuePair{
{Key: common.MetricTypeKey, Value: indexParam.MetricsType},
{Key: common.MetricTypeKey, Value: indexParam.MetricType},
},
}
if indexParam.IndexType != "" {
req.ExtraParams = append(req.ExtraParams, &commonpb.KeyValuePair{Key: common.IndexTypeKey, Value: indexParam.IndexType})
for key, value := range indexParam.IndexConfig {
req.ExtraParams = append(req.ExtraParams, &commonpb.KeyValuePair{Key: key, Value: value})
}
resp, err := wrapperProxy(ctx, c, req, false, false, func(reqCtx context.Context, req any) (interface{}, error) {
return h.proxy.CreateIndex(reqCtx, req.(*milvuspb.CreateIndexRequest))

93
internal/distributed/proxy/httpserver/handler_v2_test.go
View File

@ -42,7 +42,15 @@ type requestBodyTestCase struct {
errCode int32
}
type DefaultReq struct{}
type DefaultReq struct {
DbName string `json:"dbName"`
}
func (DefaultReq) GetBase() *commonpb.MsgBase {
return &commonpb.MsgBase{}
}
func (req *DefaultReq) GetDbName() string { return req.DbName }
func TestHTTPWrapper(t *testing.T) {
postTestCases := []requestBodyTestCase{}
@ -84,7 +92,7 @@ func TestHTTPWrapper(t *testing.T) {
errCode: 1801, // ErrIncorrectParameterFormat
})
path = "/wrapper/post/trace"
app.POST(path, wrapperPost(func() any { return &CollectionNameReq{} }, wrapperTraceLog(func(ctx context.Context, c *gin.Context, req any, dbName string) (interface{}, error) {
app.POST(path, wrapperPost(func() any { return &DefaultReq{} }, wrapperTraceLog(func(ctx context.Context, c *gin.Context, req any, dbName string) (interface{}, error) {
return nil, nil
})))
postTestCasesTrace = append(postTestCasesTrace, requestBodyTestCase{
@ -92,7 +100,7 @@ func TestHTTPWrapper(t *testing.T) {
requestBody: []byte(`{"collectionName": "` + DefaultCollectionName + `"}`),
})
path = "/wrapper/post/trace/wrong"
app.POST(path, wrapperPost(func() any { return &CollectionNameReq{} }, wrapperTraceLog(func(ctx context.Context, c *gin.Context, req any, dbName string) (interface{}, error) {
app.POST(path, wrapperPost(func() any { return &DefaultReq{} }, wrapperTraceLog(func(ctx context.Context, c *gin.Context, req any, dbName string) (interface{}, error) {
return nil, merr.ErrCollectionNotFound
})))
postTestCasesTrace = append(postTestCasesTrace, requestBodyTestCase{
@ -100,7 +108,7 @@ func TestHTTPWrapper(t *testing.T) {
requestBody: []byte(`{"collectionName": "` + DefaultCollectionName + `"}`),
})
path = "/wrapper/post/trace/call"
app.POST(path, wrapperPost(func() any { return &CollectionNameReq{} }, wrapperTraceLog(func(ctx context.Context, c *gin.Context, req any, dbName string) (interface{}, error) {
app.POST(path, wrapperPost(func() any { return &DefaultReq{} }, wrapperTraceLog(func(ctx context.Context, c *gin.Context, req any, dbName string) (interface{}, error) {
return wrapperProxy(ctx, c, req, false, false, func(reqctx context.Context, req any) (any, error) {
return nil, nil
})
@ -342,7 +350,7 @@ func TestDatabaseWrapper(t *testing.T) {
ginHandler := gin.Default()
app := ginHandler.Group("", genAuthMiddleWare(false))
path := "/wrapper/database"
app.POST(path, wrapperPost(func() any { return &DatabaseReq{} }, h.wrapperCheckDatabase(func(ctx context.Context, c *gin.Context, req any, dbName string) (interface{}, error) {
app.POST(path, wrapperPost(func() any { return &DefaultReq{} }, h.wrapperCheckDatabase(func(ctx context.Context, c *gin.Context, req any, dbName string) (interface{}, error) {
return nil, nil
})))
postTestCases = append(postTestCases, requestBodyTestCase{
@ -387,7 +395,7 @@ func TestDatabaseWrapper(t *testing.T) {
func TestCreateCollection(t *testing.T) {
postTestCases := []requestBodyTestCase{}
mp := mocks.NewMockProxy(t)
mp.EXPECT().CreateCollection(mock.Anything, mock.Anything).Return(commonSuccessStatus, nil).Times(5)
mp.EXPECT().CreateCollection(mock.Anything, mock.Anything).Return(commonSuccessStatus, nil).Times(7)
mp.EXPECT().CreateIndex(mock.Anything, mock.Anything).Return(commonSuccessStatus, nil).Twice()
mp.EXPECT().LoadCollection(mock.Anything, mock.Anything).Return(commonSuccessStatus, nil).Twice()
mp.EXPECT().CreateIndex(mock.Anything, mock.Anything).Return(commonErrorStatus, nil).Twice()
@ -396,14 +404,14 @@ func TestCreateCollection(t *testing.T) {
path := versionalV2(CollectionCategory, CreateAction)
postTestCases = append(postTestCases, requestBodyTestCase{
path: path,
requestBody: []byte(`{"collectionName": "` + DefaultCollectionName + `", "dimension": 2, "metricsType": "L2"}`),
requestBody: []byte(`{"collectionName": "` + DefaultCollectionName + `", "dimension": 2, "metricType": "L2"}`),
})
postTestCases = append(postTestCases, requestBodyTestCase{
path: path,
requestBody: []byte(`{"collectionName": "` + DefaultCollectionName + `", "schema": {
"fields": [
{"fieldName": "book_id", "dataType": "Int64", "isPrimary": true, "elementTypeParams": {}},
{"fieldName": "word_count", "dataType": "Int64", "elementTypeParams": {}},
{"fieldName": "word_count", "dataType": "Int64", "isPartitionKey": false, "elementTypeParams": {}},
{"fieldName": "book_intro", "dataType": "FloatVector", "elementTypeParams": {"dim": "2"}}
]
}}`),
@ -416,7 +424,41 @@ func TestCreateCollection(t *testing.T) {
{"fieldName": "word_count", "dataType": "Int64", "elementTypeParams": {}},
{"fieldName": "book_intro", "dataType": "FloatVector", "elementTypeParams": {"dim": "2"}}
]
}, "indexParams": [{"fieldName": "book_intro", "indexName": "book_intro_vector", "metricsType": "L2"}]}`),
}, "indexParams": [{"fieldName": "book_intro", "indexName": "book_intro_vector", "metricType": "L2"}]}`),
})
postTestCases = append(postTestCases, requestBodyTestCase{
path: path,
requestBody: []byte(`{"collectionName": "` + DefaultCollectionName + `", "schema": {
"fields": [
{"fieldName": "book_id", "dataType": "int64", "isPrimary": true, "elementTypeParams": {}},
{"fieldName": "word_count", "dataType": "Int64", "elementTypeParams": {}},
{"fieldName": "book_intro", "dataType": "FloatVector", "elementTypeParams": {"dim": "2"}}
]
}}`),
errMsg: "invalid parameter, data type int64 is invalid(case sensitive).",
errCode: 1100, // ErrParameterInvalid
})
postTestCases = append(postTestCases, requestBodyTestCase{
path: path,
requestBody: []byte(`{"collectionName": "` + DefaultCollectionName + `", "schema": {
"fields": [
{"fieldName": "book_id", "dataType": "Int64", "isPrimary": true, "elementTypeParams": {}},
{"fieldName": "word_count", "dataType": "Array", "elementDataType": "Int64", "elementTypeParams": {"max_capacity": "2"}},
{"fieldName": "book_intro", "dataType": "FloatVector", "elementTypeParams": {"dim": "2"}}
]
}}`),
})
postTestCases = append(postTestCases, requestBodyTestCase{
path: path,
requestBody: []byte(`{"collectionName": "` + DefaultCollectionName + `", "schema": {
"fields": [
{"fieldName": "book_id", "dataType": "Int64", "isPrimary": true, "elementTypeParams": {}},
{"fieldName": "word_count", "dataType": "Array", "elementDataType": "int64", "elementTypeParams": {}},
{"fieldName": "book_intro", "dataType": "FloatVector", "elementTypeParams": {"dim": "2"}}
]
}}`),
errMsg: "invalid parameter, element data type int64 is invalid(case sensitive).",
errCode: 1100, // ErrParameterInvalid
})
postTestCases = append(postTestCases, requestBodyTestCase{
path: path,
@ -426,13 +468,13 @@ func TestCreateCollection(t *testing.T) {
{"fieldName": "word_count", "dataType": "Int64", "elementTypeParams": {}},
{"fieldName": "book_intro", "dataType": "FloatVector", "elementTypeParams": {"dim": "2"}}
]
}, "indexParams": [{"fieldName": "book_xxx", "indexName": "book_intro_vector", "metricsType": "L2"}]}`),
}, "indexParams": [{"fieldName": "book_xxx", "indexName": "book_intro_vector", "metricType": "L2"}]}`),
errMsg: "missing required parameters, error: `book_xxx` hasn't defined in schema",
errCode: 1802, // ErrDatabaseNotFound
errCode: 1802,
})
postTestCases = append(postTestCases, requestBodyTestCase{
path: path,
requestBody: []byte(`{"collectionName": "` + DefaultCollectionName + `", "dimension": 2, "metricsType": "L2"}`),
requestBody: []byte(`{"collectionName": "` + DefaultCollectionName + `", "dimension": 2, "metricType": "L2"}`),
errMsg: "",
errCode: 65535,
})
@ -444,13 +486,13 @@ func TestCreateCollection(t *testing.T) {
{"fieldName": "word_count", "dataType": "Int64", "elementTypeParams": {}},
{"fieldName": "book_intro", "dataType": "FloatVector", "elementTypeParams": {"dim": "2"}}
]
}, "indexParams": [{"fieldName": "book_intro", "indexName": "book_intro_vector", "metricsType": "L2"}]}`),
}, "indexParams": [{"fieldName": "book_intro", "indexName": "book_intro_vector", "metricType": "L2"}]}`),
errMsg: "",
errCode: 65535,
})
postTestCases = append(postTestCases, requestBodyTestCase{
path: path,
requestBody: []byte(`{"collectionName": "` + DefaultCollectionName + `", "dimension": 2, "metricsType": "L2"}`),
requestBody: []byte(`{"collectionName": "` + DefaultCollectionName + `", "dimension": 2, "metricType": "L2"}`),
errMsg: "",
errCode: 65535,
})
@ -554,19 +596,26 @@ func TestMethodGet(t *testing.T) {
mp.EXPECT().HasCollection(mock.Anything, mock.Anything).Return(&milvuspb.BoolResponse{Status: commonErrorStatus}, nil).Once()
mp.EXPECT().DescribeCollection(mock.Anything, mock.Anything).Return(&milvuspb.DescribeCollectionResponse{
CollectionName: DefaultCollectionName,
Schema: generateCollectionSchema(schemapb.DataType_Int64, false),
Schema: generateCollectionSchema(schemapb.DataType_Int64),
ShardsNum: ShardNumDefault,
Status: &StatusSuccess,
}, nil).Once()
mp.EXPECT().DescribeCollection(mock.Anything, mock.Anything).Return(&milvuspb.DescribeCollectionResponse{Status: commonErrorStatus}, nil).Once()
mp.EXPECT().GetLoadState(mock.Anything, mock.Anything).Return(&DefaultLoadStateResp, nil).Twice()
mp.EXPECT().DescribeIndex(mock.Anything, mock.Anything).Return(&DefaultDescIndexesReqp, nil).Times(3)
mp.EXPECT().DescribeIndex(mock.Anything, mock.Anything).Return(nil, merr.WrapErrIndexNotFoundForCollection(DefaultCollectionName)).Once()
mp.EXPECT().GetCollectionStatistics(mock.Anything, mock.Anything).Return(&milvuspb.GetCollectionStatisticsResponse{
Status: commonSuccessStatus,
Stats: []*commonpb.KeyValuePair{
{Key: "row_count", Value: "0"},
},
}, nil).Once()
mp.EXPECT().GetCollectionStatistics(mock.Anything, mock.Anything).Return(&milvuspb.GetCollectionStatisticsResponse{
Status: commonSuccessStatus,
Stats: []*commonpb.KeyValuePair{
{Key: "row_count", Value: "abc"},
},
}, nil).Once()
mp.EXPECT().GetLoadingProgress(mock.Anything, mock.Anything).Return(&milvuspb.GetLoadingProgressResponse{
Status: commonSuccessStatus,
Progress: int64(77),
@ -694,6 +743,9 @@ func TestMethodGet(t *testing.T) {
queryTestCases = append(queryTestCases, rawTestCase{
path: versionalV2(CollectionCategory, StatsAction),
})
queryTestCases = append(queryTestCases, rawTestCase{
path: versionalV2(CollectionCategory, StatsAction),
})
queryTestCases = append(queryTestCases, rawTestCase{
path: versionalV2(CollectionCategory, LoadStateAction),
})
@ -724,6 +776,9 @@ func TestMethodGet(t *testing.T) {
queryTestCases = append(queryTestCases, rawTestCase{
path: versionalV2(IndexCategory, DescribeAction),
})
queryTestCases = append(queryTestCases, rawTestCase{
path: versionalV2(IndexCategory, ListAction),
})
queryTestCases = append(queryTestCases, rawTestCase{
path: versionalV2(AliasCategory, ListAction),
})
@ -912,10 +967,10 @@ func TestMethodPost(t *testing.T) {
for _, testcase := range queryTestCases {
t.Run("query", func(t *testing.T) {
bodyReader := bytes.NewReader([]byte(`{` +
`"collectionName": "` + DefaultCollectionName + `", "newCollectionName": "test", "newDbName": "` + DefaultDbName + `",` +
`"collectionName": "` + DefaultCollectionName + `", "newCollectionName": "test", "newDbName": "",` +
`"partitionName": "` + DefaultPartitionName + `", "partitionNames": ["` + DefaultPartitionName + `"],` +
`"schema": {"fields": [{"fieldName": "book_id", "dataType": "int64", "elementTypeParams": {}}, {"fieldName": "book_intro", "dataType": "FloatVector", "elementTypeParams": {"dim": "2"}}]},` +
`"indexParams": [{"indexName": "` + DefaultIndexName + `", "fieldName": "book_intro", "metricsType": "L2", "indexType": "IVF_FLAT"}],` +
`"schema": {"fields": [{"fieldName": "book_id", "dataType": "Int64", "elementTypeParams": {}}, {"fieldName": "book_intro", "dataType": "FloatVector", "elementTypeParams": {"dim": "2"}}]},` +
`"indexParams": [{"indexName": "` + DefaultIndexName + `", "fieldName": "book_intro", "metricType": "L2", "indexConfig": {"nlist": "30", "index_type": "IVF_FLAT"}}],` +
`"userName": "` + util.UserRoot + `", "password": "Milvus", "newPassword": "milvus", "roleName": "` + util.RoleAdmin + `",` +
`"roleName": "` + util.RoleAdmin + `", "objectType": "Global", "objectName": "*", "privilege": "*",` +
`"aliasName": "` + DefaultAliasName + `",` +
@ -944,7 +999,7 @@ func TestDML(t *testing.T) {
mp := mocks.NewMockProxy(t)
mp.EXPECT().DescribeCollection(mock.Anything, mock.Anything).Return(&milvuspb.DescribeCollectionResponse{
CollectionName: DefaultCollectionName,
Schema: generateCollectionSchema(schemapb.DataType_Int64, false),
Schema: generateCollectionSchema(schemapb.DataType_Int64),
ShardsNum: ShardNumDefault,
Status: &StatusSuccess,
}, nil).Times(6)

47
internal/distributed/proxy/httpserver/request_v2.go
View File

@ -2,6 +2,7 @@ package httpserver
import (
"net/http"
"strconv"
"github.com/gin-gonic/gin"
@ -182,11 +183,14 @@ type PartitionsReq struct {
func (req *PartitionsReq) GetDbName() string { return req.DbName }
type UserReq struct {
UserName string `json:"userName"`
UserName string `json:"userName" binding:"required"`
}
func (req *UserReq) GetUserName() string { return req.UserName }
type BaseGetter interface {
GetBase() *commonpb.MsgBase
}
type UserNameGetter interface {
GetUserName() string
}
@ -210,24 +214,23 @@ type TaskIDGetter interface {
}
type PasswordReq struct {
UserName string `json:"userName"`
UserName string `json:"userName" binding:"required"`
Password string `json:"password" binding:"required"`
}
type NewPasswordReq struct {
UserName string `json:"userName"`
Password string `json:"password"`
NewPassword string `json:"newPassword"`
UserName string `json:"userName" binding:"required"`
Password string `json:"password" binding:"required"`
NewPassword string `json:"newPassword" binding:"required"`
}
type UserRoleReq struct {
UserName string `json:"userName"`
RoleName string `json:"roleName"`
UserName string `json:"userName" binding:"required"`
RoleName string `json:"roleName" binding:"required"`
}
type RoleReq struct {
RoleName string `json:"roleName"`
Timeout int32 `json:"timeout"`
RoleName string `json:"roleName" binding:"required"`
}
func (req *RoleReq) GetRoleName() string {
@ -243,11 +246,10 @@ type GrantReq struct {
}
type IndexParam struct {
FieldName string `json:"fieldName" binding:"required"`
IndexName string `json:"indexName" binding:"required"`
MetricsType string `json:"metricsType" binding:"required"`
IndexType string `json:"indexType"`
IndexConfig map[string]interface{} `json:"indexConfig"`
FieldName string `json:"fieldName" binding:"required"`
IndexName string `json:"indexName" binding:"required"`
MetricType string `json:"metricType" binding:"required"`
IndexConfig map[string]string `json:"indexConfig"`
}
type IndexParamReq struct {
@ -261,8 +263,7 @@ func (req *IndexParamReq) GetDbName() string { return req.DbName }
type IndexReq struct {
DbName string `json:"dbName"`
CollectionName string `json:"collectionName" binding:"required"`
IndexName string `json:"indexName"`
Timeout int32 `json:"timeout"`
IndexName string `json:"indexName" binding:"required"`
}
func (req *IndexReq) GetDbName() string { return req.DbName }
@ -277,11 +278,9 @@ func (req *IndexReq) GetIndexName() string {
type FieldSchema struct {
FieldName string `json:"fieldName" binding:"required"`
DataType string `json:"dataType" binding:"required"`
ElementDataType string `json:"elementDataType"`
IsPrimary bool `json:"isPrimary"`
IsPartitionKey bool `json:"isPartitionKey"`
Dim int `json:"dimension"`
MaxLength int `json:"maxLength"`
MaxCapacity int `json:"maxCapacity"`
ElementTypeParams map[string]string `json:"elementTypeParams" binding:"required"`
}
@ -295,7 +294,7 @@ type CollectionReq struct {
DbName string `json:"dbName"`
CollectionName string `json:"collectionName" binding:"required"`
Dimension int32 `json:"dimension"`
MetricsType string `json:"metricsType"`
MetricType string `json:"metricType"`
Schema CollectionSchema `json:"schema"`
IndexParams []IndexParam `json:"indexParams"`
}
@ -339,12 +338,16 @@ func wrapperReturnList(names []string) gin.H {
}
func wrapperReturnRowCount(pairs []*commonpb.KeyValuePair) gin.H {
rowCount := "0"
rowCountValue := "0"
for _, keyValue := range pairs {
if keyValue.Key == "row_count" {
rowCount = keyValue.GetValue()
rowCountValue = keyValue.GetValue()
}
}
rowCount, err := strconv.ParseInt(rowCountValue, 10, 64)
if err != nil {
return gin.H{HTTPReturnCode: http.StatusOK, HTTPReturnData: gin.H{HTTPReturnRowCount: rowCountValue}}
}
return gin.H{HTTPReturnCode: http.StatusOK, HTTPReturnData: gin.H{HTTPReturnRowCount: rowCount}}
}

174
internal/distributed/proxy/httpserver/utils.go
View File

@ -10,7 +10,6 @@ import (
"strconv"
"strings"
"github.com/cockroachdb/errors"
"github.com/gin-gonic/gin"
"github.com/golang/protobuf/proto"
"github.com/spf13/cast"
@ -113,7 +112,7 @@ func convertRange(field *schemapb.FieldSchema, result gjson.Result) (string, err
func checkGetPrimaryKey(coll *schemapb.CollectionSchema, idResult gjson.Result) (string, error) {
primaryField, ok := getPrimaryField(coll)
if !ok {
return "", errors.New("fail to find primary key from collection description")
return "", fmt.Errorf("collection: %s has no primary field", coll.Name)
}
resultStr, err := convertRange(primaryField, idResult)
if err != nil {
@ -128,12 +127,14 @@ func printFields(fields []*schemapb.FieldSchema) []gin.H {
var res []gin.H
for _, field := range fields {
fieldDetail := gin.H{
HTTPReturnFieldName: field.Name,
HTTPReturnFieldPrimaryKey: field.IsPrimaryKey,
HTTPReturnFieldAutoID: field.AutoID,
HTTPReturnDescription: field.Description,
HTTPReturnFieldName: field.Name,
HTTPReturnFieldPrimaryKey: field.IsPrimaryKey,
HTTPReturnFieldPartitionKey: field.IsPartitionKey,
HTTPReturnFieldAutoID: field.AutoID,
HTTPReturnDescription: field.Description,
}
if field.DataType == schemapb.DataType_BinaryVector || field.DataType == schemapb.DataType_FloatVector {
if field.DataType == schemapb.DataType_BinaryVector || field.DataType == schemapb.DataType_FloatVector ||
field.DataType == schemapb.DataType_Float16Vector || field.DataType == schemapb.DataType_BFloat16Vector {
dim, _ := getDim(field)
fieldDetail[HTTPReturnFieldType] = field.DataType.String() + "(" + strconv.FormatInt(dim, 10) + ")"
} else if field.DataType == schemapb.DataType_VarChar {
@ -162,9 +163,9 @@ func printIndexes(indexes []*milvuspb.IndexDescription) []gin.H {
var res []gin.H
for _, index := range indexes {
res = append(res, gin.H{
HTTPIndexName: index.IndexName,
HTTPIndexField: index.FieldName,
HTTPReturnIndexMetricsType: getMetricType(index.Params),
HTTPIndexName: index.IndexName,
HTTPIndexField: index.FieldName,
HTTPReturnIndexMetricType: getMetricType(index.Params),
})
}
return res
@ -187,8 +188,6 @@ func checkAndSetData(body string, collSchema *schemapb.CollectionSchema) (error,
for _, data := range dataResultArray {
reallyData := map[string]interface{}{}
var vectorArray []float32
var binaryArray []byte
if data.Type == gjson.JSON {
for _, field := range collSchema.Fields {
fieldType := field.DataType
@ -205,15 +204,36 @@ func checkAndSetData(body string, collSchema *schemapb.CollectionSchema) (error,
switch fieldType {
case schemapb.DataType_FloatVector:
for _, vector := range gjson.Get(data.Raw, fieldName).Array() {
vectorArray = append(vectorArray, cast.ToFloat32(vector.Num))
var vectorArray []float32
err := json.Unmarshal([]byte(dataString), &vectorArray)
if err != nil {
return merr.WrapErrParameterInvalid(schemapb.DataType_name[int32(fieldType)], dataString, err.Error()), reallyDataArray
}
reallyData[fieldName] = vectorArray
case schemapb.DataType_BinaryVector:
for _, vector := range gjson.Get(data.Raw, fieldName).Array() {
binaryArray = append(binaryArray, cast.ToUint8(vector.Num))
vectorStr := gjson.Get(data.Raw, fieldName).Raw
var vectorArray []byte
err := json.Unmarshal([]byte(vectorStr), &vectorArray)
if err != nil {
return merr.WrapErrParameterInvalid(schemapb.DataType_name[int32(fieldType)], dataString, err.Error()), reallyDataArray
}
reallyData[fieldName] = binaryArray
reallyData[fieldName] = vectorArray
case schemapb.DataType_Float16Vector:
vectorStr := gjson.Get(data.Raw, fieldName).Raw
var vectorArray []byte
err := json.Unmarshal([]byte(vectorStr), &vectorArray)
if err != nil {
return merr.WrapErrParameterInvalid(schemapb.DataType_name[int32(fieldType)], dataString, err.Error()), reallyDataArray
}
reallyData[fieldName] = vectorArray
case schemapb.DataType_BFloat16Vector:
vectorStr := gjson.Get(data.Raw, fieldName).Raw
var vectorArray []byte
err := json.Unmarshal([]byte(vectorStr), &vectorArray)
if err != nil {
return merr.WrapErrParameterInvalid(schemapb.DataType_name[int32(fieldType)], dataString, err.Error()), reallyDataArray
}
reallyData[fieldName] = vectorArray
case schemapb.DataType_Bool:
result, err := cast.ToBoolE(dataString)
if err != nil {
@ -239,7 +259,7 @@ func checkAndSetData(body string, collSchema *schemapb.CollectionSchema) (error,
}
reallyData[fieldName] = result
case schemapb.DataType_Int64:
result, err := cast.ToInt64E(dataString)
result, err := json.Number(dataString).Int64()
if err != nil {
return merr.WrapErrParameterInvalid(schemapb.DataType_name[int32(fieldType)], dataString, err.Error()), reallyDataArray
}
@ -250,7 +270,8 @@ func checkAndSetData(body string, collSchema *schemapb.CollectionSchema) (error,
arr := make([]bool, 0)
err := json.Unmarshal([]byte(dataString), &arr)
if err != nil {
return merr.WrapErrParameterInvalid(schemapb.DataType_name[int32(fieldType)], dataString, err.Error()), reallyDataArray
return merr.WrapErrParameterInvalid(schemapb.DataType_name[int32(fieldType)]+
" of "+schemapb.DataType_name[int32(field.ElementType)], dataString, err.Error()), reallyDataArray
}
reallyData[fieldName] = &schemapb.ScalarField{
Data: &schemapb.ScalarField_BoolData{
@ -263,7 +284,8 @@ func checkAndSetData(body string, collSchema *schemapb.CollectionSchema) (error,
arr := make([]int32, 0)
err := json.Unmarshal([]byte(dataString), &arr)
if err != nil {
return merr.WrapErrParameterInvalid(schemapb.DataType_name[int32(fieldType)], dataString, err.Error()), reallyDataArray
return merr.WrapErrParameterInvalid(schemapb.DataType_name[int32(fieldType)]+
" of "+schemapb.DataType_name[int32(field.ElementType)], dataString, err.Error()), reallyDataArray
}
reallyData[fieldName] = &schemapb.ScalarField{
Data: &schemapb.ScalarField_IntData{
@ -276,7 +298,8 @@ func checkAndSetData(body string, collSchema *schemapb.CollectionSchema) (error,
arr := make([]int32, 0)
err := json.Unmarshal([]byte(dataString), &arr)
if err != nil {
return merr.WrapErrParameterInvalid(schemapb.DataType_name[int32(fieldType)], dataString, err.Error()), reallyDataArray
return merr.WrapErrParameterInvalid(schemapb.DataType_name[int32(fieldType)]+
" of "+schemapb.DataType_name[int32(field.ElementType)], dataString, err.Error()), reallyDataArray
}
reallyData[fieldName] = &schemapb.ScalarField{
Data: &schemapb.ScalarField_IntData{
@ -289,7 +312,8 @@ func checkAndSetData(body string, collSchema *schemapb.CollectionSchema) (error,
arr := make([]int32, 0)
err := json.Unmarshal([]byte(dataString), &arr)
if err != nil {
return merr.WrapErrParameterInvalid(schemapb.DataType_name[int32(fieldType)], dataString, err.Error()), reallyDataArray
return merr.WrapErrParameterInvalid(schemapb.DataType_name[int32(fieldType)]+
" of "+schemapb.DataType_name[int32(field.ElementType)], dataString, err.Error()), reallyDataArray
}
reallyData[fieldName] = &schemapb.ScalarField{
Data: &schemapb.ScalarField_IntData{
@ -300,9 +324,18 @@ func checkAndSetData(body string, collSchema *schemapb.CollectionSchema) (error,
}
case schemapb.DataType_Int64:
arr := make([]int64, 0)
err := json.Unmarshal([]byte(dataString), &arr)
numArr := make([]json.Number, 0)
err := json.Unmarshal([]byte(dataString), &numArr)
if err != nil {
return merr.WrapErrParameterInvalid(schemapb.DataType_name[int32(fieldType)], dataString, err.Error()), reallyDataArray
return merr.WrapErrParameterInvalid(schemapb.DataType_name[int32(fieldType)]+
" of "+schemapb.DataType_name[int32(field.ElementType)], dataString, err.Error()), reallyDataArray
}
for _, num := range numArr {
intVal, err := num.Int64()
if err != nil {
return merr.WrapErrParameterInvalid(schemapb.DataType_name[int32(fieldType)], dataString, err.Error()), reallyDataArray
}
arr = append(arr, intVal)
}
reallyData[fieldName] = &schemapb.ScalarField{
Data: &schemapb.ScalarField_LongData{
@ -315,7 +348,8 @@ func checkAndSetData(body string, collSchema *schemapb.CollectionSchema) (error,
arr := make([]float32, 0)
err := json.Unmarshal([]byte(dataString), &arr)
if err != nil {
return merr.WrapErrParameterInvalid(schemapb.DataType_name[int32(fieldType)], dataString, err.Error()), reallyDataArray
return merr.WrapErrParameterInvalid(schemapb.DataType_name[int32(fieldType)]+
" of "+schemapb.DataType_name[int32(field.ElementType)], dataString, err.Error()), reallyDataArray
}
reallyData[fieldName] = &schemapb.ScalarField{
Data: &schemapb.ScalarField_FloatData{
@ -328,7 +362,8 @@ func checkAndSetData(body string, collSchema *schemapb.CollectionSchema) (error,
arr := make([]float64, 0)
err := json.Unmarshal([]byte(dataString), &arr)
if err != nil {
return merr.WrapErrParameterInvalid(schemapb.DataType_name[int32(fieldType)], dataString, err.Error()), reallyDataArray
return merr.WrapErrParameterInvalid(schemapb.DataType_name[int32(fieldType)]+
" of "+schemapb.DataType_name[int32(field.ElementType)], dataString, err.Error()), reallyDataArray
}
reallyData[fieldName] = &schemapb.ScalarField{
Data: &schemapb.ScalarField_DoubleData{
@ -341,7 +376,8 @@ func checkAndSetData(body string, collSchema *schemapb.CollectionSchema) (error,
arr := make([]string, 0)
err := json.Unmarshal([]byte(dataString), &arr)
if err != nil {
return merr.WrapErrParameterInvalid(schemapb.DataType_name[int32(fieldType)], dataString, err.Error()), reallyDataArray
return merr.WrapErrParameterInvalid(schemapb.DataType_name[int32(fieldType)]+
" of "+schemapb.DataType_name[int32(field.ElementType)], dataString, err.Error()), reallyDataArray
}
reallyData[fieldName] = &schemapb.ScalarField{
Data: &schemapb.ScalarField_StringData{
@ -434,7 +470,8 @@ func convertFloatVectorToArray(vector [][]float32, dim int64) ([]float32, error)
floatArray := make([]float32, 0)
for _, arr := range vector {
if int64(len(arr)) != dim {
return nil, errors.New("vector length diff from dimension")
return nil, fmt.Errorf("[]float32 size %d doesn't equal to vector dimension %d of %s",
len(arr), dim, schemapb.DataType_name[int32(schemapb.DataType_FloatVector)])
}
for i := int64(0); i < dim; i++ {
floatArray = append(floatArray, arr[i])
@ -443,12 +480,21 @@ func convertFloatVectorToArray(vector [][]float32, dim int64) ([]float32, error)
return floatArray, nil
}
func convertBinaryVectorToArray(vector [][]byte, dim int64) ([]byte, error) {
func convertBinaryVectorToArray(vector [][]byte, dim int64, dataType schemapb.DataType) ([]byte, error) {
binaryArray := make([]byte, 0)
bytesLen := dim / 8
var bytesLen int64
switch dataType {
case schemapb.DataType_BinaryVector:
bytesLen = dim / 8
case schemapb.DataType_Float16Vector:
bytesLen = dim * 2
case schemapb.DataType_BFloat16Vector:
bytesLen = dim * 2
}
for _, arr := range vector {
if int64(len(arr)) != bytesLen {
return nil, errors.New("vector length diff from dimension")
return nil, fmt.Errorf("[]byte size %d doesn't equal to vector dimension %d of %s",
len(arr), dim, schemapb.DataType_name[int32(dataType)])
}
for i := int64(0); i < bytesLen; i++ {
binaryArray = append(binaryArray, arr[i])
@ -503,13 +549,13 @@ func convertToIntArray(dataType schemapb.DataType, arr interface{}) []int32 {
func anyToColumns(rows []map[string]interface{}, sch *schemapb.CollectionSchema) ([]*schemapb.FieldData, error) {
rowsLen := len(rows)
if rowsLen == 0 {
return []*schemapb.FieldData{}, errors.New("0 length column")
return []*schemapb.FieldData{}, fmt.Errorf("no row need to be convert to columns")
}
isDynamic := sch.EnableDynamicField
var dim int64
nameColumns := make(map[string]interface{})
nameDims := make(map[string]int64)
fieldData := make(map[string]*schemapb.FieldData)
for _, field := range sch.Fields {
// skip auto id pk field
@ -542,10 +588,20 @@ func anyToColumns(rows []map[string]interface{}, sch *schemapb.CollectionSchema)
data = make([][]byte, 0, rowsLen)
case schemapb.DataType_FloatVector:
data = make([][]float32, 0, rowsLen)
dim, _ = getDim(field)
dim, _ := getDim(field)
nameDims[field.Name] = dim
case schemapb.DataType_BinaryVector:
data = make([][]byte, 0, rowsLen)
dim, _ = getDim(field)
dim, _ := getDim(field)
nameDims[field.Name] = dim
case schemapb.DataType_Float16Vector:
data = make([][]byte, 0, rowsLen)
dim, _ := getDim(field)
nameDims[field.Name] = dim
case schemapb.DataType_BFloat16Vector:
data = make([][]byte, 0, rowsLen)
dim, _ := getDim(field)
nameDims[field.Name] = dim
default:
return nil, fmt.Errorf("the type(%v) of field(%v) is not supported, use other sdk please", field.DataType, field.Name)
}
@ -557,8 +613,8 @@ func anyToColumns(rows []map[string]interface{}, sch *schemapb.CollectionSchema)
IsDynamic: field.IsDynamic,
}
}
if dim == 0 {
return nil, errors.New("cannot find dimension")
if len(nameDims) == 0 {
return nil, fmt.Errorf("collection: %s has no vector field", sch.Name)
}
dynamicCol := make([][]byte, 0, rowsLen)
@ -608,6 +664,10 @@ func anyToColumns(rows []map[string]interface{}, sch *schemapb.CollectionSchema)
nameColumns[field.Name] = append(nameColumns[field.Name].([][]float32), candi.v.Interface().([]float32))
case schemapb.DataType_BinaryVector:
nameColumns[field.Name] = append(nameColumns[field.Name].([][]byte), candi.v.Interface().([]byte))
case schemapb.DataType_Float16Vector:
nameColumns[field.Name] = append(nameColumns[field.Name].([][]byte), candi.v.Interface().([]byte))
case schemapb.DataType_BFloat16Vector:
nameColumns[field.Name] = append(nameColumns[field.Name].([][]byte), candi.v.Interface().([]byte))
default:
return nil, fmt.Errorf("the type(%v) of field(%v) is not supported, use other sdk please", field.DataType, field.Name)
}
@ -742,6 +802,7 @@ func anyToColumns(rows []map[string]interface{}, sch *schemapb.CollectionSchema)
},
}
case schemapb.DataType_FloatVector:
dim := nameDims[name]
arr, err := convertFloatVectorToArray(column.([][]float32), dim)
if err != nil {
return nil, err
@ -757,7 +818,8 @@ func anyToColumns(rows []map[string]interface{}, sch *schemapb.CollectionSchema)
},
}
case schemapb.DataType_BinaryVector:
arr, err := convertBinaryVectorToArray(column.([][]byte), dim)
dim := nameDims[name]
arr, err := convertBinaryVectorToArray(column.([][]byte), dim, colData.Type)
if err != nil {
return nil, err
}
@ -769,6 +831,34 @@ func anyToColumns(rows []map[string]interface{}, sch *schemapb.CollectionSchema)
},
},
}
case schemapb.DataType_Float16Vector:
dim := nameDims[name]
arr, err := convertBinaryVectorToArray(column.([][]byte), dim, colData.Type)
if err != nil {
return nil, err
}
colData.Field = &schemapb.FieldData_Vectors{
Vectors: &schemapb.VectorField{
Dim: dim,
Data: &schemapb.VectorField_Float16Vector{
Float16Vector: arr,
},
},
}
case schemapb.DataType_BFloat16Vector:
dim := nameDims[name]
arr, err := convertBinaryVectorToArray(column.([][]byte), dim, colData.Type)
if err != nil {
return nil, err
}
colData.Field = &schemapb.FieldData_Vectors{
Vectors: &schemapb.VectorField{
Dim: dim,
Data: &schemapb.VectorField_Bfloat16Vector{
Bfloat16Vector: arr,
},
},
}
default:
return nil, fmt.Errorf("the type(%v) of field(%v) is not supported, use other sdk please", colData.Type, name)
}
@ -876,6 +966,10 @@ func buildQueryResp(rowsNum int64, needFields []string, fieldDataList []*schemap
rowsNum = int64(len(fieldDataList[0].GetVectors().GetBinaryVector())*8) / fieldDataList[0].GetVectors().GetDim()
case schemapb.DataType_FloatVector:
rowsNum = int64(len(fieldDataList[0].GetVectors().GetFloatVector().Data)) / fieldDataList[0].GetVectors().GetDim()
case schemapb.DataType_Float16Vector:
rowsNum = int64(len(fieldDataList[0].GetVectors().GetFloat16Vector())/2) / fieldDataList[0].GetVectors().GetDim()
case schemapb.DataType_BFloat16Vector:
rowsNum = int64(len(fieldDataList[0].GetVectors().GetBfloat16Vector())/2) / fieldDataList[0].GetVectors().GetDim()
default:
return nil, fmt.Errorf("the type(%v) of field(%v) is not supported, use other sdk please", fieldDataList[0].Type, fieldDataList[0].FieldName)
}
@ -927,6 +1021,10 @@ func buildQueryResp(rowsNum int64, needFields []string, fieldDataList []*schemap
row[fieldDataList[j].FieldName] = fieldDataList[j].GetVectors().GetBinaryVector()[i*(fieldDataList[j].GetVectors().GetDim()/8) : (i+1)*(fieldDataList[j].GetVectors().GetDim()/8)]
case schemapb.DataType_FloatVector:
row[fieldDataList[j].FieldName] = fieldDataList[j].GetVectors().GetFloatVector().Data[i*fieldDataList[j].GetVectors().GetDim() : (i+1)*fieldDataList[j].GetVectors().GetDim()]
case schemapb.DataType_Float16Vector:
row[fieldDataList[j].FieldName] = fieldDataList[j].GetVectors().GetBinaryVector()[i*(fieldDataList[j].GetVectors().GetDim()*2) : (i+1)*(fieldDataList[j].GetVectors().GetDim()*2)]
case schemapb.DataType_BFloat16Vector:
row[fieldDataList[j].FieldName] = fieldDataList[j].GetVectors().GetBinaryVector()[i*(fieldDataList[j].GetVectors().GetDim()*2) : (i+1)*(fieldDataList[j].GetVectors().GetDim()*2)]
case schemapb.DataType_Array:
row[fieldDataList[j].FieldName] = fieldDataList[j].GetScalars().GetArrayData().Data[i]
case schemapb.DataType_JSON:

354
internal/distributed/proxy/httpserver/utils_test.go
View File

@ -1,6 +1,8 @@
package httpserver
import (
"encoding/json"
"math"
"strconv"
"strings"
"testing"
@ -21,6 +23,8 @@ const (
FieldBookIntro = "book_intro"
)
var DefaultScores = []float32{0.01, 0.04, 0.09}
func generatePrimaryField(datatype schemapb.DataType) schemapb.FieldSchema {
return schemapb.FieldSchema{
FieldID: common.StartOfUserFieldID,
@ -63,42 +67,29 @@ func generateIds(dataType schemapb.DataType, num int) *schemapb.IDs {
return nil
}
func generateVectorFieldSchema(useBinary bool) schemapb.FieldSchema {
if useBinary {
return schemapb.FieldSchema{
FieldID: common.StartOfUserFieldID + 2,
Name: "field-binary",
IsPrimaryKey: false,
Description: "",
DataType: 100,
AutoID: false,
TypeParams: []*commonpb.KeyValuePair{
{
Key: common.DimKey,
Value: "8",
},
},
}
func generateVectorFieldSchema(dataType schemapb.DataType) schemapb.FieldSchema {
dim := "2"
if dataType == schemapb.DataType_BinaryVector {
dim = "8"
}
return schemapb.FieldSchema{
FieldID: common.StartOfUserFieldID + 2,
Name: FieldBookIntro,
FieldID: common.StartOfUserFieldID + int64(dataType),
IsPrimaryKey: false,
Description: "",
DataType: 101,
DataType: dataType,
AutoID: false,
TypeParams: []*commonpb.KeyValuePair{
{
Key: common.DimKey,
Value: "2",
Value: dim,
},
},
}
}
func generateCollectionSchema(datatype schemapb.DataType, useBinary bool) *schemapb.CollectionSchema {
primaryField := generatePrimaryField(datatype)
vectorField := generateVectorFieldSchema(useBinary)
func generateCollectionSchema(primaryDataType schemapb.DataType) *schemapb.CollectionSchema {
primaryField := generatePrimaryField(primaryDataType)
vectorField := generateVectorFieldSchema(schemapb.DataType_FloatVector)
vectorField.Name = FieldBookIntro
return &schemapb.CollectionSchema{
Name: DefaultCollectionName,
Description: "",
@ -142,11 +133,12 @@ func generateIndexes() []*milvuspb.IndexDescription {
}
}
func generateVectorFieldData(useBinary bool) schemapb.FieldData {
if useBinary {
func generateVectorFieldData(vectorType schemapb.DataType) schemapb.FieldData {
switch vectorType {
case schemapb.DataType_BinaryVector:
return schemapb.FieldData{
Type: schemapb.DataType_BinaryVector,
FieldName: "field-binary",
FieldName: FieldBookIntro,
Field: &schemapb.FieldData_Vectors{
Vectors: &schemapb.VectorField{
Dim: 8,
@ -157,6 +149,34 @@ func generateVectorFieldData(useBinary bool) schemapb.FieldData {
},
IsDynamic: false,
}
case schemapb.DataType_Float16Vector:
return schemapb.FieldData{
Type: schemapb.DataType_Float16Vector,
FieldName: FieldBookIntro,
Field: &schemapb.FieldData_Vectors{
Vectors: &schemapb.VectorField{
Dim: 8,
Data: &schemapb.VectorField_Float16Vector{
Float16Vector: []byte{byte(0), byte(0), byte(1), byte(1), byte(2), byte(2)},
},
},
},
IsDynamic: false,
}
case schemapb.DataType_BFloat16Vector:
return schemapb.FieldData{
Type: schemapb.DataType_BFloat16Vector,
FieldName: FieldBookIntro,
Field: &schemapb.FieldData_Vectors{
Vectors: &schemapb.VectorField{
Dim: 8,
Data: &schemapb.VectorField_Bfloat16Vector{
Bfloat16Vector: []byte{byte(0), byte(0), byte(1), byte(1), byte(2), byte(2)},
},
},
},
IsDynamic: false,
}
}
return schemapb.FieldData{
Type: schemapb.DataType_FloatVector,
@ -206,40 +226,68 @@ func generateFieldData() []*schemapb.FieldData {
IsDynamic: false,
}
fieldData3 := generateVectorFieldData(false)
fieldData3 := generateVectorFieldData(schemapb.DataType_FloatVector)
return []*schemapb.FieldData{&fieldData1, &fieldData2, &fieldData3}
}
func generateSearchResult(dataType schemapb.DataType) []map[string]interface{} {
func wrapRequestBody(data []map[string]interface{}) ([]byte, error) {
body := map[string]interface{}{}
body["data"] = data
return json.Marshal(body)
}
func generateRawRows(dataType schemapb.DataType) []map[string]interface{} {
row1 := map[string]interface{}{
DefaultPrimaryFieldName: int64(1),
FieldBookID: int64(1),
FieldWordCount: int64(1000),
FieldBookIntro: []float32{0.1, 0.11},
HTTPReturnDistance: float32(0.01),
FieldBookID: int64(1),
FieldWordCount: int64(1000),
FieldBookIntro: []float32{0.1, 0.11},
}
row2 := map[string]interface{}{
DefaultPrimaryFieldName: int64(2),
FieldBookID: int64(2),
FieldWordCount: int64(2000),
FieldBookIntro: []float32{0.2, 0.22},
HTTPReturnDistance: float32(0.04),
FieldBookID: int64(2),
FieldWordCount: int64(2000),
FieldBookIntro: []float32{0.2, 0.22},
}
row3 := map[string]interface{}{
DefaultPrimaryFieldName: int64(3),
FieldBookID: int64(3),
FieldWordCount: int64(3000),
FieldBookIntro: []float32{0.3, 0.33},
HTTPReturnDistance: float32(0.09),
FieldBookID: int64(3),
FieldWordCount: int64(3000),
FieldBookIntro: []float32{0.3, 0.33},
}
if dataType == schemapb.DataType_String {
row1[DefaultPrimaryFieldName] = "1"
row2[DefaultPrimaryFieldName] = "2"
row3[DefaultPrimaryFieldName] = "3"
row1[FieldBookID] = "1"
row2[FieldBookID] = "2"
row3[FieldBookID] = "3"
}
return []map[string]interface{}{row1, row2, row3}
}
func generateRequestBody(dataType schemapb.DataType) ([]byte, error) {
return wrapRequestBody(generateRawRows(dataType))
}
func generateRequestBodyWithArray(dataType schemapb.DataType) ([]byte, error) {
rows := generateRawRows(dataType)
for _, result := range rows {
result["array-bool"] = "[true]"
result["array-int8"] = "[0]"
result["array-int16"] = "[0]"
result["array-int32"] = "[0]"
result["array-int64"] = "[0]"
result["array-float"] = "[0.0]"
result["array-double"] = "[0.0]"
result["array-varchar"] = "[\"\"]"
}
return wrapRequestBody(rows)
}
func generateSearchResult(dataType schemapb.DataType) []map[string]interface{} {
rows := generateRawRows(dataType)
for i, row := range rows {
row[DefaultPrimaryFieldName] = row[FieldBookID]
row[HTTPReturnDistance] = DefaultScores[i]
}
return rows
}
func generateQueryResult64(withDistance bool) []map[string]interface{} {
row1 := map[string]interface{}{
FieldBookID: float64(1),
@ -265,36 +313,39 @@ func generateQueryResult64(withDistance bool) []map[string]interface{} {
}
func TestPrintCollectionDetails(t *testing.T) {
coll := generateCollectionSchema(schemapb.DataType_Int64, false)
coll := generateCollectionSchema(schemapb.DataType_Int64)
indexes := generateIndexes()
assert.Equal(t, []gin.H{
{
HTTPReturnFieldName: FieldBookID,
HTTPReturnFieldType: "Int64",
HTTPReturnFieldPrimaryKey: true,
HTTPReturnFieldAutoID: false,
HTTPReturnDescription: "",
HTTPReturnFieldName: FieldBookID,
HTTPReturnFieldType: "Int64",
HTTPReturnFieldPartitionKey: false,
HTTPReturnFieldPrimaryKey: true,
HTTPReturnFieldAutoID: false,
HTTPReturnDescription: "",
},
{
HTTPReturnFieldName: FieldWordCount,
HTTPReturnFieldType: "Int64",
HTTPReturnFieldPrimaryKey: false,
HTTPReturnFieldAutoID: false,
HTTPReturnDescription: "",
HTTPReturnFieldName: FieldWordCount,
HTTPReturnFieldType: "Int64",
HTTPReturnFieldPartitionKey: false,
HTTPReturnFieldPrimaryKey: false,
HTTPReturnFieldAutoID: false,
HTTPReturnDescription: "",
},
{
HTTPReturnFieldName: FieldBookIntro,
HTTPReturnFieldType: "FloatVector(2)",
HTTPReturnFieldPrimaryKey: false,
HTTPReturnFieldAutoID: false,
HTTPReturnDescription: "",
HTTPReturnFieldName: FieldBookIntro,
HTTPReturnFieldType: "FloatVector(2)",
HTTPReturnFieldPartitionKey: false,
HTTPReturnFieldPrimaryKey: false,
HTTPReturnFieldAutoID: false,
HTTPReturnDescription: "",
},
}, printFields(coll.Fields))
assert.Equal(t, []gin.H{
{
HTTPIndexName: DefaultIndexName,
HTTPIndexField: FieldBookIntro,
HTTPReturnIndexMetricsType: DefaultMetricType,
HTTPIndexName: DefaultIndexName,
HTTPIndexField: FieldBookIntro,
HTTPReturnIndexMetricType: DefaultMetricType,
},
}, printIndexes(indexes))
assert.Equal(t, DefaultMetricType, getMetricType(indexes[0].Params))
@ -307,17 +358,18 @@ func TestPrintCollectionDetails(t *testing.T) {
}
assert.Equal(t, []gin.H{
{
HTTPReturnFieldName: "field-varchar",
HTTPReturnFieldType: "VarChar(10)",
HTTPReturnFieldPrimaryKey: false,
HTTPReturnFieldAutoID: false,
HTTPReturnDescription: "",
HTTPReturnFieldName: "field-varchar",
HTTPReturnFieldType: "VarChar(10)",
HTTPReturnFieldPartitionKey: false,
HTTPReturnFieldPrimaryKey: false,
HTTPReturnFieldAutoID: false,
HTTPReturnDescription: "",
},
}, printFields(fields))
}
func TestPrimaryField(t *testing.T) {
coll := generateCollectionSchema(schemapb.DataType_Int64, false)
coll := generateCollectionSchema(schemapb.DataType_Int64)
primaryField := generatePrimaryField(schemapb.DataType_Int64)
field, ok := getPrimaryField(coll)
assert.Equal(t, true, ok)
@ -341,7 +393,7 @@ func TestPrimaryField(t *testing.T) {
rangeStr, err = convertRange(&primaryField, idStr)
assert.Equal(t, nil, err)
assert.Equal(t, `"1","2","3"`, rangeStr)
coll2 := generateCollectionSchema(schemapb.DataType_VarChar, false)
coll2 := generateCollectionSchema(schemapb.DataType_VarChar)
filter, err = checkGetPrimaryKey(coll2, idStr)
assert.Equal(t, nil, err)
assert.Equal(t, `book_id in ["1","2","3"]`, filter)
@ -350,7 +402,7 @@ func TestPrimaryField(t *testing.T) {
func TestInsertWithDynamicFields(t *testing.T) {
body := "{\"data\": {\"id\": 0, \"book_id\": 1, \"book_intro\": [0.1, 0.2], \"word_count\": 2, \"classified\": false, \"databaseID\": null}}"
req := InsertReq{}
coll := generateCollectionSchema(schemapb.DataType_Int64, false)
coll := generateCollectionSchema(schemapb.DataType_Int64)
var err error
err, req.Data = checkAndSetData(body, coll)
assert.Equal(t, nil, err)
@ -364,6 +416,27 @@ func TestInsertWithDynamicFields(t *testing.T) {
assert.Equal(t, "{\"classified\":false,\"id\":0}", string(fieldsData[len(fieldsData)-1].GetScalars().GetJsonData().GetData()[0]))
}
func TestInsertWithInt64(t *testing.T) {
arrayFieldName := "array-int64"
body := "{\"data\": {\"book_id\": 9999999999999999, \"book_intro\": [0.1, 0.2], \"word_count\": 2, \"" + arrayFieldName + "\": [9999999999999999]}}"
coll := generateCollectionSchema(schemapb.DataType_Int64)
coll.Fields = append(coll.Fields, &schemapb.FieldSchema{
Name: arrayFieldName,
DataType: schemapb.DataType_Array,
ElementType: schemapb.DataType_Int64,
})
err, data := checkAndSetData(body, coll)
assert.Equal(t, nil, err)
assert.Equal(t, 1, len(data))
assert.Equal(t, int64(9999999999999999), data[0][FieldBookID])
arr, _ := data[0][arrayFieldName].(*schemapb.ScalarField)
assert.Equal(t, int64(9999999999999999), arr.GetLongData().GetData()[0])
body = "{\"data\": {\"book_id\": 9999999999999999, \"book_intro\": [0.1, 0.2], \"word_count\": 2, \"" + arrayFieldName + "\": [9999999999999999.0]}}"
err, _ = checkAndSetData(body, coll)
assert.Error(t, err)
}
func TestSerialize(t *testing.T) {
parameters := []float32{0.11111, 0.22222}
// assert.Equal(t, "\ufffd\ufffd\ufffd=\ufffd\ufffdc\u003e", string(serialize(parameters)))
@ -413,7 +486,7 @@ func compareRow(m1 map[string]interface{}, m2 map[string]interface{}) bool {
return false
}
}
} else if (key == "field-binary") || (key == "field-json") {
} else if key == "field-json" {
arr1 := value.([]byte)
arr2 := m2[key].([]byte)
if len(arr1) != len(arr2) {
@ -424,13 +497,15 @@ func compareRow(m1 map[string]interface{}, m2 map[string]interface{}) bool {
return false
}
}
} else if strings.HasPrefix(key, "array-") {
continue
} else if value != m2[key] {
return false
}
}
for key, value := range m2 {
if (key == FieldBookIntro) || (key == "field-binary") || (key == "field-json") || (key == "field-array") {
if (key == FieldBookIntro) || (key == "field-json") || (key == "field-array") {
continue
} else if strings.HasPrefix(key, "array-") {
continue
@ -457,7 +532,7 @@ func compareRows(row1 []map[string]interface{}, row2 []map[string]interface{}, c
func TestBuildQueryResp(t *testing.T) {
outputFields := []string{FieldBookID, FieldWordCount, "author", "date"}
rows, err := buildQueryResp(int64(0), outputFields, generateFieldData(), generateIds(schemapb.DataType_Int64, 3), []float32{0.01, 0.04, 0.09}, true) // []*schemapb.FieldData{&fieldData1, &fieldData2, &fieldData3}
rows, err := buildQueryResp(int64(0), outputFields, generateFieldData(), generateIds(schemapb.DataType_Int64, 3), DefaultScores, true) // []*schemapb.FieldData{&fieldData1, &fieldData2, &fieldData3}
assert.Equal(t, nil, err)
exceptRows := generateSearchResult(schemapb.DataType_Int64)
assert.Equal(t, true, compareRows(rows, exceptRows, compareRow))
@ -796,10 +871,16 @@ func newFieldData(fieldDatas []*schemapb.FieldData, firstFieldType schemapb.Data
case schemapb.DataType_VarChar:
return []*schemapb.FieldData{&fieldData8}
case schemapb.DataType_BinaryVector:
vectorField := generateVectorFieldData(true)
vectorField := generateVectorFieldData(firstFieldType)
return []*schemapb.FieldData{&vectorField}
case schemapb.DataType_FloatVector:
vectorField := generateVectorFieldData(false)
vectorField := generateVectorFieldData(firstFieldType)
return []*schemapb.FieldData{&vectorField}
case schemapb.DataType_Float16Vector:
vectorField := generateVectorFieldData(firstFieldType)
return []*schemapb.FieldData{&vectorField}
case schemapb.DataType_BFloat16Vector:
vectorField := generateVectorFieldData(firstFieldType)
return []*schemapb.FieldData{&vectorField}
case schemapb.DataType_Array:
return []*schemapb.FieldData{&fieldData10}
@ -827,7 +908,6 @@ func newSearchResult(results []map[string]interface{}) []map[string]interface{}
result["field-double"] = float64(i)
result["field-varchar"] = strconv.Itoa(i)
result["field-string"] = strconv.Itoa(i)
result["field-binary"] = []byte{byte(i)}
result["field-json"] = []byte(`{"XXX": 0}`)
result["field-array"] = []bool{true}
result["array-bool"] = []bool{true}
@ -898,14 +978,14 @@ func newCollectionSchemaWithArray(coll *schemapb.CollectionSchema) *schemapb.Col
fieldSchema8 := schemapb.FieldSchema{
Name: "array-varchar",
DataType: schemapb.DataType_Array,
ElementType: schemapb.DataType_String,
ElementType: schemapb.DataType_VarChar,
}
coll.Fields = append(coll.Fields, &fieldSchema8)
return coll
}
func newSearchResultWithArray(results []map[string]interface{}) []map[string]interface{} {
func newRowsWithArray(results []map[string]interface{}) []map[string]interface{} {
for i, result := range results {
result["array-bool"] = &schemapb.ScalarField{
Data: &schemapb.ScalarField_BoolData{
@ -968,50 +1048,138 @@ func newSearchResultWithArray(results []map[string]interface{}) []map[string]int
return results
}
func TestAnyToColumn(t *testing.T) {
data, err := anyToColumns(newSearchResultWithArray(generateSearchResult(schemapb.DataType_Int64)), newCollectionSchemaWithArray(generateCollectionSchema(schemapb.DataType_Int64, false)))
func TestArray(t *testing.T) {
body, _ := generateRequestBody(schemapb.DataType_Int64)
collectionSchema := generateCollectionSchema(schemapb.DataType_Int64)
err, rows := checkAndSetData(string(body), collectionSchema)
assert.Equal(t, nil, err)
assert.Equal(t, 12, len(data))
assert.Equal(t, true, compareRows(rows, generateRawRows(schemapb.DataType_Int64), compareRow))
data, err := anyToColumns(rows, collectionSchema)
assert.Equal(t, nil, err)
assert.Equal(t, len(collectionSchema.Fields)+1, len(data))
body, _ = generateRequestBodyWithArray(schemapb.DataType_Int64)
collectionSchema = newCollectionSchemaWithArray(generateCollectionSchema(schemapb.DataType_Int64))
err, rows = checkAndSetData(string(body), collectionSchema)
assert.Equal(t, nil, err)
assert.Equal(t, true, compareRows(rows, newRowsWithArray(generateRawRows(schemapb.DataType_Int64)), compareRow))
data, err = anyToColumns(rows, collectionSchema)
assert.Equal(t, nil, err)
assert.Equal(t, len(collectionSchema.Fields)+1, len(data))
}
func TestVector(t *testing.T) {
floatVector := "vector-float"
binaryVector := "vector-binary"
float16Vector := "vector-float16"
bfloat16Vector := "vector-bfloat16"
row1 := map[string]interface{}{
FieldBookID: int64(1),
floatVector: []float32{0.1, 0.11},
binaryVector: []byte{1},
float16Vector: []byte{1, 1, 11, 11},
bfloat16Vector: []byte{1, 1, 11, 11},
}
row2 := map[string]interface{}{
FieldBookID: int64(2),
floatVector: []float32{0.2, 0.22},
binaryVector: []byte{2},
float16Vector: []byte{2, 2, 22, 22},
bfloat16Vector: []byte{2, 2, 22, 22},
}
row3 := map[string]interface{}{
FieldBookID: int64(3),
floatVector: []float32{0.3, 0.33},
binaryVector: []byte{3},
float16Vector: []byte{3, 3, 33, 33},
bfloat16Vector: []byte{3, 3, 33, 33},
}
body, _ := wrapRequestBody([]map[string]interface{}{row1, row2, row3})
primaryField := generatePrimaryField(schemapb.DataType_Int64)
floatVectorField := generateVectorFieldSchema(schemapb.DataType_FloatVector)
floatVectorField.Name = floatVector
binaryVectorField := generateVectorFieldSchema(schemapb.DataType_BinaryVector)
binaryVectorField.Name = binaryVector
float16VectorField := generateVectorFieldSchema(schemapb.DataType_Float16Vector)
float16VectorField.Name = float16Vector
bfloat16VectorField := generateVectorFieldSchema(schemapb.DataType_BFloat16Vector)
bfloat16VectorField.Name = bfloat16Vector
collectionSchema := &schemapb.CollectionSchema{
Name: DefaultCollectionName,
Description: "",
AutoID: false,
Fields: []*schemapb.FieldSchema{
&primaryField, &floatVectorField, &binaryVectorField, &float16VectorField, &bfloat16VectorField,
},
EnableDynamicField: true,
}
err, rows := checkAndSetData(string(body), collectionSchema)
assert.Equal(t, nil, err)
for _, row := range rows {
assert.Equal(t, 1, len(row[binaryVector].([]byte)))
assert.Equal(t, 4, len(row[float16Vector].([]byte)))
assert.Equal(t, 4, len(row[bfloat16Vector].([]byte)))
}
data, err := anyToColumns(rows, collectionSchema)
assert.Equal(t, nil, err)
assert.Equal(t, len(collectionSchema.Fields)+1, len(data))
row1[bfloat16Vector] = []int64{99999999, -99999999}
body, _ = wrapRequestBody([]map[string]interface{}{row1})
err, _ = checkAndSetData(string(body), collectionSchema)
assert.Error(t, err)
row1[float16Vector] = []int64{99999999, -99999999}
body, _ = wrapRequestBody([]map[string]interface{}{row1})
err, _ = checkAndSetData(string(body), collectionSchema)
assert.Error(t, err)
row1[binaryVector] = []int64{99999999, -99999999}
body, _ = wrapRequestBody([]map[string]interface{}{row1})
err, _ = checkAndSetData(string(body), collectionSchema)
assert.Error(t, err)
row1[floatVector] = []float64{math.MaxFloat64, 0}
body, _ = wrapRequestBody([]map[string]interface{}{row1})
err, _ = checkAndSetData(string(body), collectionSchema)
assert.Error(t, err)
}
func TestBuildQueryResps(t *testing.T) {
outputFields := []string{"XXX", "YYY"}
outputFieldsList := [][]string{outputFields, {"$meta"}, {"$meta", FieldBookID, FieldBookIntro, "YYY"}}
for _, theOutputFields := range outputFieldsList {
rows, err := buildQueryResp(int64(0), theOutputFields, newFieldData(generateFieldData(), schemapb.DataType_None), generateIds(schemapb.DataType_Int64, 3), []float32{0.01, 0.04, 0.09}, true)
rows, err := buildQueryResp(int64(0), theOutputFields, newFieldData(generateFieldData(), schemapb.DataType_None), generateIds(schemapb.DataType_Int64, 3), DefaultScores, true)
assert.Equal(t, nil, err)
exceptRows := newSearchResult(generateSearchResult(schemapb.DataType_Int64))
assert.Equal(t, true, compareRows(rows, exceptRows, compareRow))
}
dataTypes := []schemapb.DataType{
schemapb.DataType_FloatVector, schemapb.DataType_BinaryVector,
schemapb.DataType_FloatVector, schemapb.DataType_BinaryVector, schemapb.DataType_Float16Vector, schemapb.DataType_BFloat16Vector,
schemapb.DataType_Bool, schemapb.DataType_Int8, schemapb.DataType_Int16, schemapb.DataType_Int32,
schemapb.DataType_Float, schemapb.DataType_Double,
schemapb.DataType_String, schemapb.DataType_VarChar,
schemapb.DataType_JSON, schemapb.DataType_Array,
}
for _, dateType := range dataTypes {
_, err := buildQueryResp(int64(0), outputFields, newFieldData([]*schemapb.FieldData{}, dateType), generateIds(schemapb.DataType_Int64, 3), []float32{0.01, 0.04, 0.09}, true)
_, err := buildQueryResp(int64(0), outputFields, newFieldData([]*schemapb.FieldData{}, dateType), generateIds(schemapb.DataType_Int64, 3), DefaultScores, true)
assert.Equal(t, nil, err)
}
_, err := buildQueryResp(int64(0), outputFields, newFieldData([]*schemapb.FieldData{}, 1000), generateIds(schemapb.DataType_Int64, 3), []float32{0.01, 0.04, 0.09}, true)
_, err := buildQueryResp(int64(0), outputFields, newFieldData([]*schemapb.FieldData{}, 1000), generateIds(schemapb.DataType_Int64, 3), DefaultScores, true)
assert.Equal(t, "the type(1000) of field(wrong-field-type) is not supported, use other sdk please", err.Error())
res, err := buildQueryResp(int64(0), outputFields, []*schemapb.FieldData{}, generateIds(schemapb.DataType_Int64, 3), []float32{0.01, 0.04, 0.09}, true)
res, err := buildQueryResp(int64(0), outputFields, []*schemapb.FieldData{}, generateIds(schemapb.DataType_Int64, 3), DefaultScores, true)
assert.Equal(t, 3, len(res))
assert.Equal(t, nil, err)
res, err = buildQueryResp(int64(0), outputFields, []*schemapb.FieldData{}, generateIds(schemapb.DataType_Int64, 3), []float32{0.01, 0.04, 0.09}, false)
res, err = buildQueryResp(int64(0), outputFields, []*schemapb.FieldData{}, generateIds(schemapb.DataType_Int64, 3), DefaultScores, false)
assert.Equal(t, 3, len(res))
assert.Equal(t, nil, err)
res, err = buildQueryResp(int64(0), outputFields, []*schemapb.FieldData{}, generateIds(schemapb.DataType_VarChar, 3), []float32{0.01, 0.04, 0.09}, true)
res, err = buildQueryResp(int64(0), outputFields, []*schemapb.FieldData{}, generateIds(schemapb.DataType_VarChar, 3), DefaultScores, true)
assert.Equal(t, 3, len(res))
assert.Equal(t, nil, err)
_, err = buildQueryResp(int64(0), outputFields, generateFieldData(), generateIds(schemapb.DataType_Int64, 3), []float32{0.01, 0.04, 0.09}, false)
_, err = buildQueryResp(int64(0), outputFields, generateFieldData(), generateIds(schemapb.DataType_Int64, 3), DefaultScores, false)
assert.Equal(t, nil, err)
// len(rows) != len(scores), didn't show distance