test: add cases for go milvusclient query (#33884)

issue: #33419

---------

Signed-off-by: ThreadDao <yufen.zong@zilliz.com>
pull/34178/head
ThreadDao 2024-06-26 09:52:04 +08:00 committed by GitHub
parent cc54ab2e47
commit a77e2951af
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
10 changed files with 758 additions and 48 deletions

View File

@ -310,6 +310,25 @@ func FieldDataColumn(fd *schemapb.FieldData, begin, end int) (Column, error) {
vector = append(vector, v)
}
return NewColumnBFloat16Vector(fd.GetFieldName(), dim, vector), nil
case schemapb.DataType_SparseFloatVector:
sparseVectors := fd.GetVectors().GetSparseFloatVector()
if sparseVectors == nil {
return nil, errFieldDataTypeNotMatch
}
data := sparseVectors.Contents
if end < 0 {
end = len(data)
}
data = data[begin:end]
vectors := make([]entity.SparseEmbedding, 0, len(data))
for _, bs := range data {
vector, err := entity.DeserializeSliceSparseEmbedding(bs)
if err != nil {
return nil, err
}
vectors = append(vectors, vector)
}
return NewColumnSparseVectors(fd.GetFieldName(), vectors), nil
default:
return nil, fmt.Errorf("unsupported data type %s", fd.GetType())
}

View File

@ -83,7 +83,7 @@ func (e sliceSparseEmbedding) Swap(i, j int) {
e.values[i], e.values[j] = e.values[j], e.values[i]
}
func deserializeSliceSparceEmbedding(bs []byte) (sliceSparseEmbedding, error) {
func DeserializeSliceSparseEmbedding(bs []byte) (sliceSparseEmbedding, error) {
length := len(bs)
if length%8 != 0 {
return sliceSparseEmbedding{}, errors.New("not valid sparse embedding bytes")

View File

@ -40,7 +40,7 @@ func TestSliceSparseEmbedding(t *testing.T) {
assert.EqualValues(t, length, se.Len())
bs := se.Serialize()
nv, err := deserializeSliceSparceEmbedding(bs)
nv, err := DeserializeSliceSparseEmbedding(bs)
require.NoError(t, err)
for i := 0; i < length; i++ {

View File

@ -2,6 +2,7 @@ package common
import (
"fmt"
"reflect"
"strings"
"testing"
@ -32,7 +33,7 @@ func CheckErr(t *testing.T, actualErr error, expErrNil bool, expErrorMsg ...stri
}
}
if !contains {
t.FailNow()
t.Fatalf("CheckErr failed, actualErr doesn contains any expErrorMsg, please check test cases!")
}
}
}
@ -60,40 +61,64 @@ func EqualColumn(t *testing.T, columnA column.Column, columnB column.Column) {
case entity.FieldTypeVarChar:
require.ElementsMatch(t, columnA.(*column.ColumnVarChar).Data(), columnB.(*column.ColumnVarChar).Data())
case entity.FieldTypeJSON:
log.Debug("columnA", zap.Any("data", columnA.(*column.ColumnJSONBytes).Data()))
log.Debug("columnB", zap.Any("data", columnB.(*column.ColumnJSONBytes).Data()))
require.ElementsMatch(t, columnA.(*column.ColumnJSONBytes).Data(), columnB.(*column.ColumnJSONBytes).Data())
log.Debug("data", zap.String("name", columnA.Name()), zap.Any("type", columnA.Type()), zap.Any("data", columnA.FieldData()))
log.Debug("data", zap.String("name", columnB.Name()), zap.Any("type", columnB.Type()), zap.Any("data", columnB.FieldData()))
require.Equal(t, reflect.TypeOf(columnA), reflect.TypeOf(columnB))
switch columnA.(type) {
case *column.ColumnDynamic:
require.ElementsMatch(t, columnA.(*column.ColumnDynamic).Data(), columnB.(*column.ColumnDynamic).Data())
case *column.ColumnJSONBytes:
require.ElementsMatch(t, columnA.(*column.ColumnJSONBytes).Data(), columnB.(*column.ColumnJSONBytes).Data())
}
case entity.FieldTypeFloatVector:
require.ElementsMatch(t, columnA.(*column.ColumnFloatVector).Data(), columnB.(*column.ColumnFloatVector).Data())
case entity.FieldTypeBinaryVector:
require.ElementsMatch(t, columnA.(*column.ColumnBinaryVector).Data(), columnB.(*column.ColumnBinaryVector).Data())
case entity.FieldTypeFloat16Vector:
require.ElementsMatch(t, columnA.(*column.ColumnFloat16Vector).Data(), columnB.(*column.ColumnFloat16Vector).Data())
case entity.FieldTypeBFloat16Vector:
require.ElementsMatch(t, columnA.(*column.ColumnBFloat16Vector).Data(), columnB.(*column.ColumnBFloat16Vector).Data())
case entity.FieldTypeSparseVector:
require.ElementsMatch(t, columnA.(*column.ColumnSparseFloatVector).Data(), columnB.(*column.ColumnSparseFloatVector).Data())
case entity.FieldTypeArray:
log.Info("TODO support column element type")
EqualArrayColumn(t, columnA, columnB)
default:
log.Info("Support column type is:", zap.Any("FieldType", []entity.FieldType{
entity.FieldTypeBool,
entity.FieldTypeInt8, entity.FieldTypeInt16, entity.FieldTypeInt32, entity.FieldTypeInt64,
entity.FieldTypeFloat, entity.FieldTypeDouble, entity.FieldTypeString, entity.FieldTypeVarChar,
entity.FieldTypeArray, entity.FieldTypeFloatVector, entity.FieldTypeBinaryVector,
entity.FieldTypeInt8, entity.FieldTypeInt16, entity.FieldTypeInt32,
entity.FieldTypeInt64, entity.FieldTypeFloat, entity.FieldTypeDouble, entity.FieldTypeString,
entity.FieldTypeVarChar, entity.FieldTypeArray, entity.FieldTypeFloatVector, entity.FieldTypeBinaryVector,
}))
}
}
// CheckOutputFields check query output fields
func CheckOutputFields(t *testing.T, expFields []string, actualColumns []column.Column) {
actualFields := make([]string, 0)
for _, actualColumn := range actualColumns {
actualFields = append(actualFields, actualColumn.Name())
}
require.ElementsMatchf(t, expFields, actualFields, fmt.Sprintf("Expected search output fields: %v, actual: %v", expFields, actualFields))
}
// CheckSearchResult check search result, check nq, topk, ids, score
func CheckSearchResult(t *testing.T, actualSearchResults []clientv2.ResultSet, expNq int, expTopK int) {
require.Equal(t, len(actualSearchResults), expNq)
require.Len(t, actualSearchResults, expNq)
for _, actualSearchResult := range actualSearchResults {
require.Equal(t, actualSearchResult.ResultCount, expTopK)
// EqualColumn assert field data is equal of two columns
func EqualArrayColumn(t *testing.T, columnA column.Column, columnB column.Column) {
require.Equal(t, columnA.Name(), columnB.Name())
require.IsType(t, columnA.Type(), entity.FieldTypeArray)
require.IsType(t, columnB.Type(), entity.FieldTypeArray)
switch columnA.(type) {
case *column.ColumnBoolArray:
require.ElementsMatch(t, columnA.(*column.ColumnBoolArray).Data(), columnB.(*column.ColumnBoolArray).Data())
case *column.ColumnInt8Array:
require.ElementsMatch(t, columnA.(*column.ColumnInt8Array).Data(), columnB.(*column.ColumnInt8Array).Data())
case *column.ColumnInt16Array:
require.ElementsMatch(t, columnA.(*column.ColumnInt16Array).Data(), columnB.(*column.ColumnInt16Array).Data())
case *column.ColumnInt32Array:
require.ElementsMatch(t, columnA.(*column.ColumnInt32Array).Data(), columnB.(*column.ColumnInt32Array).Data())
case *column.ColumnInt64Array:
require.ElementsMatch(t, columnA.(*column.ColumnInt64Array).Data(), columnB.(*column.ColumnInt64Array).Data())
case *column.ColumnFloatArray:
require.ElementsMatch(t, columnA.(*column.ColumnFloatArray).Data(), columnB.(*column.ColumnFloatArray).Data())
case *column.ColumnDoubleArray:
require.ElementsMatch(t, columnA.(*column.ColumnDoubleArray).Data(), columnB.(*column.ColumnDoubleArray).Data())
case *column.ColumnVarCharArray:
require.ElementsMatch(t, columnA.(*column.ColumnVarCharArray).Data(), columnB.(*column.ColumnVarCharArray).Data())
default:
log.Info("Support array element type is:", zap.Any("FieldType", []entity.FieldType{
entity.FieldTypeBool, entity.FieldTypeInt8, entity.FieldTypeInt16,
entity.FieldTypeInt32, entity.FieldTypeInt64, entity.FieldTypeFloat, entity.FieldTypeDouble, entity.FieldTypeVarChar,
}))
}
}
@ -112,3 +137,40 @@ func CheckInsertResult(t *testing.T, expIds column.Column, insertRes clientv2.In
log.Info("The primary field only support ", zap.Any("type", []entity.FieldType{entity.FieldTypeInt64, entity.FieldTypeVarChar}))
}
}
// CheckOutputFields check query output fields
func CheckOutputFields(t *testing.T, expFields []string, actualColumns []column.Column) {
actualFields := make([]string, 0)
for _, actualColumn := range actualColumns {
actualFields = append(actualFields, actualColumn.Name())
}
log.Debug("CheckOutputFields", zap.Any("expFields", expFields), zap.Any("actualFields", actualFields))
require.ElementsMatchf(t, expFields, actualFields, fmt.Sprintf("Expected search output fields: %v, actual: %v", expFields, actualFields))
}
// CheckSearchResult check search result, check nq, topk, ids, score
func CheckSearchResult(t *testing.T, actualSearchResults []clientv2.ResultSet, expNq int, expTopK int) {
require.Equal(t, len(actualSearchResults), expNq)
require.Len(t, actualSearchResults, expNq)
for _, actualSearchResult := range actualSearchResults {
require.Equal(t, actualSearchResult.ResultCount, expTopK)
}
}
// CheckQueryResult check query result, column name, type and field
func CheckQueryResult(t *testing.T, expColumns []column.Column, actualColumns []column.Column) {
require.Equal(t, len(actualColumns), len(expColumns),
"The len of actual columns %d should greater or equal to the expected columns %d", len(actualColumns), len(expColumns))
for _, expColumn := range expColumns {
exist := false
for _, actualColumn := range actualColumns {
if expColumn.Name() == actualColumn.Name() {
exist = true
EqualColumn(t, expColumn, actualColumn)
}
}
if !exist {
log.Error("CheckQueryResult actualColumns no column", zap.String("name", expColumn.Name()))
}
}
}

View File

@ -5,15 +5,16 @@ go 1.21
toolchain go1.21.10
require (
github.com/milvus-io/milvus/client/v2 v2.0.0-20240612020955-8cb350598c9a
github.com/milvus-io/milvus/client/v2 v2.0.0-20240621033600-e653ad27e2d5
github.com/milvus-io/milvus/pkg v0.0.2-0.20240317152703-17b4938985f3
github.com/quasilyte/go-ruleguard/dsl v0.3.22
github.com/stretchr/testify v1.9.0
github.com/x448/float16 v0.8.4
go.uber.org/zap v1.27.0
google.golang.org/grpc v1.64.0
)
//replace github.com/milvus-io/milvus/client/v2 v2.0.0-20240521081339-017fd7bc25de => ../../../milvus/client
//replace github.com/milvus-io/milvus/client/v2 v2.0.0-20240621033600-e653ad27e2d5 => ../../../milvus/client
require (
github.com/beorn7/perks v1.0.1 // indirect
@ -55,7 +56,7 @@ require (
github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect
github.com/magiconair/properties v1.8.5 // indirect
github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
github.com/milvus-io/milvus-proto/go-api/v2 v2.3.4-0.20240430035521-259ae1d10016 // indirect
github.com/milvus-io/milvus-proto/go-api/v2 v2.4.3 // indirect
github.com/mitchellh/mapstructure v1.4.1 // indirect
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
github.com/modern-go/reflect2 v1.0.2 // indirect
@ -69,7 +70,6 @@ require (
github.com/prometheus/client_model v0.3.0 // indirect
github.com/prometheus/common v0.42.0 // indirect
github.com/prometheus/procfs v0.9.0 // indirect
github.com/quasilyte/go-ruleguard/dsl v0.3.22 // indirect
github.com/rogpeppe/go-internal v1.10.0 // indirect
github.com/samber/lo v1.27.0 // indirect
github.com/shirou/gopsutil/v3 v3.22.9 // indirect

View File

@ -401,12 +401,10 @@ github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfr
github.com/mediocregopher/radix/v3 v3.4.2/go.mod h1:8FL3F6UQRXHXIBSPUs5h0RybMF8i4n7wVopoX3x7Bv8=
github.com/microcosm-cc/bluemonday v1.0.2/go.mod h1:iVP4YcDBq+n/5fb23BhYFvIMq/leAFZyRl6bYmGDlGc=
github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg=
github.com/milvus-io/milvus-proto/go-api/v2 v2.3.4-0.20240430035521-259ae1d10016 h1:8WV4maXLeGEyJCCYIc1DmZ18H+VFAjMrwXJg5iI2nX4=
github.com/milvus-io/milvus-proto/go-api/v2 v2.3.4-0.20240430035521-259ae1d10016/go.mod h1:1OIl0v5PQeNxIJhCvY+K55CBUOYDZevw9g9380u1Wek=
github.com/milvus-io/milvus/client/v2 v2.0.0-20240521081339-017fd7bc25de h1:sZ1K+F+1RMlWzuAFlhF7m68e9+G6UlRDoL7Q4Bmsc08=
github.com/milvus-io/milvus/client/v2 v2.0.0-20240521081339-017fd7bc25de/go.mod h1:FU4rii9fMaMy1CGWgjROyKo5SCaJqcxZeIhdhuuEhrQ=
github.com/milvus-io/milvus/client/v2 v2.0.0-20240612020955-8cb350598c9a h1:iyXy8ee4pDutzO40VA9xem1O1h/o5gI5x86ZVqVzto4=
github.com/milvus-io/milvus/client/v2 v2.0.0-20240612020955-8cb350598c9a/go.mod h1:thfuEkUztRRmQ+qu4hCoO/6uxDJoUVNNx4vHqx9yh5I=
github.com/milvus-io/milvus-proto/go-api/v2 v2.4.3 h1:KUSaWVePVlHMIluAXf2qmNffI1CMlGFLLiP+4iy9014=
github.com/milvus-io/milvus-proto/go-api/v2 v2.4.3/go.mod h1:1OIl0v5PQeNxIJhCvY+K55CBUOYDZevw9g9380u1Wek=
github.com/milvus-io/milvus/client/v2 v2.0.0-20240621033600-e653ad27e2d5 h1:KNE4Smy6HxIpoJHLpds5BI2ZyvhrE5FcLhUcsjxAxAk=
github.com/milvus-io/milvus/client/v2 v2.0.0-20240621033600-e653ad27e2d5/go.mod h1:thfuEkUztRRmQ+qu4hCoO/6uxDJoUVNNx4vHqx9yh5I=
github.com/milvus-io/milvus/pkg v0.0.2-0.20240317152703-17b4938985f3 h1:ZBpRWhBa7FTFxW4YYVv9AUESoW1Xyb3KNXTzTqfkZmw=
github.com/milvus-io/milvus/pkg v0.0.2-0.20240317152703-17b4938985f3/go.mod h1:jQ2BUZny1COsgv1Qbcv8dmbppW+V9J/c4YQZNb3EOm8=
github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc=

View File

@ -200,7 +200,7 @@ func TestCreateArrayCollections(t *testing.T) {
// test create collection with partition key not supported field type
func TestCreateCollectionPartitionKey(t *testing.T) {
ctx := hp.CreateContext(t, time.Second*common.DefaultTimeout)
ctx := hp.CreateContext(t, time.Second*common.DefaultTimeout*2)
mc := createDefaultMilvusClient(ctx, t)
int64Field := entity.NewField().WithName(common.DefaultInt64FieldName).WithDataType(entity.FieldTypeInt64).WithIsPrimaryKey(true)
@ -421,7 +421,7 @@ func TestCreateCollectionWithValidName(t *testing.T) {
func TestCreateCollectionWithInvalidFieldName(t *testing.T) {
t.Parallel()
// connect
ctx := hp.CreateContext(t, time.Second*common.DefaultTimeout)
ctx := hp.CreateContext(t, time.Second*common.DefaultTimeout*2)
mc := createDefaultMilvusClient(ctx, t)
// create collection with invalid field name
@ -436,7 +436,8 @@ func TestCreateCollectionWithInvalidFieldName(t *testing.T) {
common.CheckErr(t, err, false, "field name should not be empty",
"The first character of a field name must be an underscore or letter",
"Field name cannot only contain numbers, letters, and underscores",
"The length of a field name must be less than 255 characters")
"The length of a field name must be less than 255 characters",
"Field name can only contain numbers, letters, and underscores")
}
}
@ -515,7 +516,7 @@ func TestCreateCollectionInvalidFields(t *testing.T) {
// create autoID or not collection with non-int64 and non-varchar field
func TestCreateCollectionInvalidAutoPkField(t *testing.T) {
ctx := hp.CreateContext(t, time.Second*common.DefaultTimeout)
ctx := hp.CreateContext(t, time.Second*common.DefaultTimeout*2)
mc := createDefaultMilvusClient(ctx, t)
t.Parallel()
// create collection with autoID true or not
@ -557,7 +558,7 @@ func TestCreateCollectionDuplicateField(t *testing.T) {
// test create collection with partition key not supported field type
func TestCreateCollectionInvalidPartitionKeyType(t *testing.T) {
ctx := hp.CreateContext(t, time.Second*common.DefaultTimeout)
ctx := hp.CreateContext(t, time.Second*common.DefaultTimeout*2)
mc := createDefaultMilvusClient(ctx, t)
int64Field := entity.NewField().WithName(common.DefaultInt64FieldName).WithDataType(entity.FieldTypeInt64).WithIsPrimaryKey(true)

View File

@ -394,3 +394,22 @@ func GenDynamicColumnData(start int, nb int) []column.Column {
}
return data
}
func MergeColumnsToDynamic(nb int, columns []column.Column, columnName string) *column.ColumnJSONBytes {
values := make([][]byte, 0, nb)
for i := 0; i < nb; i++ {
m := make(map[string]interface{})
for _, c := range columns {
// range guaranteed
m[c.Name()], _ = c.Get(i)
}
bs, err := json.Marshal(&m)
if err != nil {
log.Fatal("MergeColumnsToDynamic failed:", zap.Error(err))
}
values = append(values, bs)
}
jsonColumn := column.NewColumnJSONBytes(columnName, values).WithIsDynamic(true)
return jsonColumn
}

View File

@ -136,16 +136,15 @@ func (chainTask *CollectionPrepare) InsertData(ctx context.Context, t *testing.T
}
fields := ip.Schema.Fields
insertOpt := clientv2.NewColumnBasedInsertOption(ip.Schema.CollectionName)
if ip.Schema.EnableDynamicField {
insertOpt.WithColumns(GenDynamicColumnData(option.start, ip.Nb)...)
}
for _, field := range fields {
if field.IsDynamic {
insertOpt.WithColumns(GenDynamicColumnData(option.start, ip.Nb)...)
} else {
if field.DataType == entity.FieldTypeArray {
option.TWithElementType(field.ElementType)
}
column := GenColumnData(ip.Nb, field.DataType, *option)
insertOpt.WithColumns(column)
if field.DataType == entity.FieldTypeArray {
option.TWithElementType(field.ElementType)
}
column := GenColumnData(ip.Nb, field.DataType, *option)
insertOpt.WithColumns(column)
}
if ip.PartitionName != "" {
insertOpt.WithPartition(ip.PartitionName)

View File

@ -0,0 +1,612 @@
package testcases
import (
"fmt"
"testing"
"time"
"github.com/stretchr/testify/require"
"go.uber.org/zap"
clientv2 "github.com/milvus-io/milvus/client/v2"
"github.com/milvus-io/milvus/client/v2/column"
"github.com/milvus-io/milvus/client/v2/entity"
"github.com/milvus-io/milvus/pkg/log"
"github.com/milvus-io/milvus/tests/go_client/common"
hp "github.com/milvus-io/milvus/tests/go_client/testcases/helper"
)
// test query from default partition
func TestQueryDefault(t *testing.T) {
ctx := hp.CreateContext(t, time.Second*common.DefaultTimeout)
mc := createDefaultMilvusClient(ctx, t)
// create and insert
prepare, schema := hp.CollPrepare.CreateCollection(ctx, t, mc, hp.NewCreateCollectionParams(hp.Int64Vec), hp.TNewFieldsOption(), hp.TNewSchemaOption())
_, insertRes := prepare.InsertData(ctx, t, mc, hp.NewInsertParams(schema, common.DefaultNb), hp.TNewDataOption())
// flush -> index -> load
prepare.FlushData(ctx, t, mc, schema.CollectionName)
prepare.CreateIndex(ctx, t, mc, hp.NewIndexParams(schema))
prepare.Load(ctx, t, mc, hp.NewLoadParams(schema.CollectionName))
// query
expr := fmt.Sprintf("%s < %d", common.DefaultInt64FieldName, 100)
queryRes, err := mc.Query(ctx, clientv2.NewQueryOption(schema.CollectionName).WithFilter(expr))
common.CheckErr(t, err, true)
common.CheckQueryResult(t, queryRes.Fields, []column.Column{insertRes.IDs.Slice(0, 100)})
}
// test query with varchar field filter
func TestQueryVarcharPkDefault(t *testing.T) {
ctx := hp.CreateContext(t, time.Second*common.DefaultTimeout)
mc := createDefaultMilvusClient(ctx, t)
// create and insert
prepare, schema := hp.CollPrepare.CreateCollection(ctx, t, mc, hp.NewCreateCollectionParams(hp.VarcharBinary), hp.TNewFieldsOption(), hp.TNewSchemaOption())
_, insertRes := prepare.InsertData(ctx, t, mc, hp.NewInsertParams(schema, common.DefaultNb), hp.TNewDataOption())
// flush -> index -> load
prepare.CreateIndex(ctx, t, mc, hp.NewIndexParams(schema))
prepare.Load(ctx, t, mc, hp.NewLoadParams(schema.CollectionName))
// query
expr := fmt.Sprintf("%s in ['0', '1', '2', '3', '4']", common.DefaultVarcharFieldName)
queryRes, err := mc.Query(ctx, clientv2.NewQueryOption(schema.CollectionName).WithFilter(expr))
common.CheckErr(t, err, true)
common.CheckQueryResult(t, queryRes.Fields, []column.Column{insertRes.IDs.Slice(0, 5)})
}
// query from not existed collection name and partition name
func TestQueryNotExistName(t *testing.T) {
ctx := hp.CreateContext(t, time.Second*common.DefaultTimeout)
mc := createDefaultMilvusClient(ctx, t)
// query with not existed collection
expr := fmt.Sprintf("%s < %d", common.DefaultInt64FieldName, 100)
_, errCol := mc.Query(ctx, clientv2.NewQueryOption("aaa").WithFilter(expr))
common.CheckErr(t, errCol, false, "can't find collection")
// create -> index -> load
prepare, schema := hp.CollPrepare.CreateCollection(ctx, t, mc, hp.NewCreateCollectionParams(hp.Int64Vec), hp.TNewFieldsOption(), hp.TNewSchemaOption())
prepare.CreateIndex(ctx, t, mc, hp.NewIndexParams(schema))
prepare.Load(ctx, t, mc, hp.NewLoadParams(schema.CollectionName))
// query with not existed partition
_, errPar := mc.Query(ctx, clientv2.NewQueryOption(schema.CollectionName).WithFilter(expr).WithPartitions([]string{"aaa"}))
common.CheckErr(t, errPar, false, "partition name aaa not found")
}
// test query with invalid partition name
func TestQueryInvalidPartitionName(t *testing.T) {
ctx := hp.CreateContext(t, time.Second*common.DefaultTimeout)
mc := createDefaultMilvusClient(ctx, t)
// create collection and partition
_, schema := hp.CollPrepare.CreateCollection(ctx, t, mc, hp.NewCreateCollectionParams(hp.Int64Vec), hp.TNewFieldsOption(), hp.TNewSchemaOption().TWithEnableDynamicField(true))
expr := fmt.Sprintf("%s >= %d", common.DefaultInt64FieldName, 0)
emptyPartitionName := ""
// query from "" partitions, expect to query from default partition
_, err := mc.Query(ctx, clientv2.NewQueryOption(schema.CollectionName).WithFilter(expr).WithPartitions([]string{emptyPartitionName}))
common.CheckErr(t, err, false, "Partition name should not be empty")
}
// test query with empty partition name
func TestQueryPartition(t *testing.T) {
parName := "p1"
ctx := hp.CreateContext(t, time.Second*common.DefaultTimeout)
mc := createDefaultMilvusClient(ctx, t)
// create collection and partition
prepare, schema := hp.CollPrepare.CreateCollection(ctx, t, mc, hp.NewCreateCollectionParams(hp.Int64Vec), hp.TNewFieldsOption(), hp.TNewSchemaOption().TWithEnableDynamicField(true))
err := mc.CreatePartition(ctx, clientv2.NewCreatePartitionOption(schema.CollectionName, parName))
common.CheckErr(t, err, true)
// insert [0, 3000) into default, insert [3000, 6000) into parName
_, i1Res := prepare.InsertData(ctx, t, mc, hp.NewInsertParams(schema, common.DefaultNb), hp.TNewDataOption())
_, i2Res := prepare.InsertData(ctx, t, mc, hp.NewInsertParams(schema, common.DefaultNb).TWithPartitionName(parName), hp.TNewDataOption().TWithStart(common.DefaultNb))
// flush -> index -> load
prepare.FlushData(ctx, t, mc, schema.CollectionName)
prepare.CreateIndex(ctx, t, mc, hp.NewIndexParams(schema))
prepare.Load(ctx, t, mc, hp.NewLoadParams(schema.CollectionName))
expr := fmt.Sprintf("%s >= %d", common.DefaultInt64FieldName, 0)
expColumn := hp.GenColumnData(common.DefaultNb*2, entity.FieldTypeInt64, *hp.TNewDataOption().TWithStart(0))
// query with default params, expect to query from all partitions
queryRes, err := mc.Query(ctx, clientv2.NewQueryOption(schema.CollectionName).WithFilter(expr).WithConsistencyLevel(entity.ClStrong))
common.CheckErr(t, err, true)
common.CheckQueryResult(t, queryRes.Fields, []column.Column{expColumn})
// query with empty partition names
queryRes, err = mc.Query(ctx, clientv2.NewQueryOption(schema.CollectionName).WithFilter(expr).WithPartitions([]string{}).WithConsistencyLevel(entity.ClStrong))
common.CheckErr(t, err, true)
common.CheckQueryResult(t, queryRes.Fields, []column.Column{expColumn})
// query with default partition
queryRes, err = mc.Query(ctx, clientv2.NewQueryOption(schema.CollectionName).WithFilter(expr).WithPartitions([]string{common.DefaultPartition}).WithConsistencyLevel(entity.ClStrong))
common.CheckErr(t, err, true)
common.CheckQueryResult(t, queryRes.Fields, []column.Column{i1Res.IDs})
// query with specify partition
queryRes, err = mc.Query(ctx, clientv2.NewQueryOption(schema.CollectionName).WithFilter(expr).WithPartitions([]string{parName}).WithConsistencyLevel(entity.ClStrong))
common.CheckErr(t, err, true)
common.CheckQueryResult(t, queryRes.Fields, []column.Column{i2Res.IDs})
// query with all partitions
queryRes, err = mc.Query(ctx, clientv2.NewQueryOption(schema.CollectionName).WithFilter(expr).WithPartitions([]string{common.DefaultPartition, parName}).WithConsistencyLevel(entity.ClStrong))
common.CheckErr(t, err, true)
common.CheckQueryResult(t, queryRes.Fields, []column.Column{expColumn})
}
// test query with invalid partition name
func TestQueryWithoutExpr(t *testing.T) {
ctx := hp.CreateContext(t, time.Second*common.DefaultTimeout)
mc := createDefaultMilvusClient(ctx, t)
// create collection and partition
_, schema := hp.CollPrepare.CreateCollection(ctx, t, mc, hp.NewCreateCollectionParams(hp.Int64Vec), hp.TNewFieldsOption(), hp.TNewSchemaOption())
// query without expr
_, err := mc.Query(ctx, clientv2.NewQueryOption(schema.CollectionName))
common.CheckErr(t, err, false, "empty expression should be used with limit")
// query with empty expr
_, err = mc.Query(ctx, clientv2.NewQueryOption(schema.CollectionName).WithFilter(""))
common.CheckErr(t, err, false, "empty expression should be used with limit")
}
// test query empty output fields: []string{}, []string{""}
// test query with not existed field
func TestQueryEmptyOutputFields(t *testing.T) {
t.Parallel()
ctx := hp.CreateContext(t, time.Second*common.DefaultTimeout)
mc := createDefaultMilvusClient(ctx, t)
for _, enableDynamic := range [2]bool{true, false} {
// create -> insert -> flush -> index -> load
prepare, schema := hp.CollPrepare.CreateCollection(ctx, t, mc, hp.NewCreateCollectionParams(hp.Int64Vec), hp.TNewFieldsOption(), hp.TNewSchemaOption().TWithEnableDynamicField(enableDynamic))
prepare.InsertData(ctx, t, mc, hp.NewInsertParams(schema, common.DefaultNb), hp.TNewDataOption())
prepare.FlushData(ctx, t, mc, schema.CollectionName)
prepare.CreateIndex(ctx, t, mc, hp.NewIndexParams(schema))
prepare.Load(ctx, t, mc, hp.NewLoadParams(schema.CollectionName))
expr := fmt.Sprintf("%s < %d", common.DefaultInt64FieldName, 10)
// query with empty output fields []string{}-> output "int64"
queryNilOutputs, err := mc.Query(ctx, clientv2.NewQueryOption(schema.CollectionName).WithConsistencyLevel(entity.ClStrong).WithFilter(expr).WithOutputFields([]string{}))
common.CheckErr(t, err, true)
common.CheckOutputFields(t, []string{common.DefaultInt64FieldName}, queryNilOutputs.Fields)
// query with empty output fields []string{""}-> output "int64" and dynamic field
_, err1 := mc.Query(ctx, clientv2.NewQueryOption(schema.CollectionName).WithConsistencyLevel(entity.ClStrong).WithFilter(expr).WithOutputFields([]string{""}))
if enableDynamic {
common.CheckErr(t, err1, false, "parse output field name failed")
} else {
common.CheckErr(t, err1, false, "not exist")
}
// query with empty not existed field -> output field as dynamic or error
fakeName := "aaa"
res, err2 := mc.Query(ctx, clientv2.NewQueryOption(schema.CollectionName).WithConsistencyLevel(entity.ClStrong).WithFilter(expr).WithOutputFields([]string{fakeName}))
if enableDynamic {
common.CheckErr(t, err2, true)
for _, c := range res.Fields {
log.Debug("data", zap.String("name", c.Name()), zap.Any("type", c.Type()), zap.Any("data", c.FieldData()))
}
common.CheckOutputFields(t, []string{common.DefaultInt64FieldName, fakeName}, res.Fields)
dynamicColumn := hp.MergeColumnsToDynamic(10, hp.GenDynamicColumnData(0, 10), common.DefaultDynamicFieldName)
expColumns := []column.Column{
hp.GenColumnData(10, entity.FieldTypeInt64, *hp.TNewDataOption()),
column.NewColumnDynamic(dynamicColumn, fakeName),
}
common.CheckQueryResult(t, expColumns, res.Fields)
} else {
common.CheckErr(t, err2, false, fmt.Sprintf("%s not exist", fakeName))
}
}
}
// test query output all fields and verify data
func TestOutputAllFieldsColumn(t *testing.T) {
t.Skip("https://github.com/milvus-io/milvus/issues/33848")
ctx := hp.CreateContext(t, time.Second*common.DefaultTimeout)
mc := createDefaultMilvusClient(ctx, t)
// create collection
for _, isDynamic := range [2]bool{true, false} {
prepare, schema := hp.CollPrepare.CreateCollection(ctx, t, mc, hp.NewCreateCollectionParams(hp.AllFields), hp.TNewFieldsOption(), hp.TNewSchemaOption().TWithEnableDynamicField(isDynamic))
prepare.CreateIndex(ctx, t, mc, hp.NewIndexParams(schema))
prepare.Load(ctx, t, mc, hp.NewLoadParams(schema.CollectionName))
// insert
columns := make([]column.Column, 0, len(schema.Fields)+1)
dynamicColumns := hp.GenDynamicColumnData(0, common.DefaultNb)
genDataOpt := hp.TNewDataOption().TWithMaxCapacity(common.TestCapacity)
insertOpt := clientv2.NewColumnBasedInsertOption(schema.CollectionName)
for _, field := range schema.Fields {
if field.DataType == entity.FieldTypeArray {
genDataOpt.TWithElementType(field.ElementType)
}
columns = append(columns, hp.GenColumnData(common.DefaultNb, field.DataType, *genDataOpt.TWithDim(common.DefaultDim)))
}
if isDynamic {
insertOpt.WithColumns(dynamicColumns...)
}
ids, err := mc.Insert(ctx, insertOpt.WithColumns(columns...))
common.CheckErr(t, err, true)
require.Equal(t, int64(common.DefaultNb), ids.InsertCount)
prepare.FlushData(ctx, t, mc, schema.CollectionName)
// query output all fields -> output all fields, includes vector and $meta field
pos := 10
allFieldsName := make([]string, 0, len(schema.Fields))
for _, field := range schema.Fields {
allFieldsName = append(allFieldsName, field.Name)
}
if isDynamic {
allFieldsName = append(allFieldsName, common.DefaultDynamicFieldName)
}
queryResultAll, errQuery := mc.Query(ctx, clientv2.NewQueryOption(schema.CollectionName).WithConsistencyLevel(entity.ClStrong).
WithFilter(fmt.Sprintf("%s < %d", common.DefaultInt64FieldName, pos)).WithOutputFields([]string{"*"}))
common.CheckErr(t, errQuery, true)
common.CheckOutputFields(t, allFieldsName, queryResultAll.Fields)
expColumns := make([]column.Column, 0, len(columns)+1)
for _, _column := range columns {
expColumns = append(expColumns, _column.Slice(0, pos))
}
if isDynamic {
expColumns = append(expColumns, hp.MergeColumnsToDynamic(pos, dynamicColumns, common.DefaultDynamicFieldName))
}
common.CheckQueryResult(t, expColumns, queryResultAll.Fields)
}
}
// test query output all fields
func TestOutputAllFieldsRows(t *testing.T) {
t.Skip("https://github.com/milvus-io/milvus/issues/33459")
ctx := hp.CreateContext(t, time.Second*common.DefaultTimeout)
mc := createDefaultMilvusClient(ctx, t)
// create collection
prepare, schema := hp.CollPrepare.CreateCollection(ctx, t, mc, hp.NewCreateCollectionParams(hp.AllFields), hp.TNewFieldsOption(),
hp.TNewSchemaOption().TWithEnableDynamicField(true))
// prepare and insert data
rows := hp.GenAllFieldsRows(common.DefaultNb, false, *hp.TNewDataOption().TWithMaxCapacity(common.TestCapacity))
ids, err := mc.Insert(ctx, clientv2.NewRowBasedInsertOption(schema.CollectionName, rows...))
common.CheckErr(t, err, true)
require.Equal(t, int64(common.DefaultNb), ids.InsertCount)
prepare.FlushData(ctx, t, mc, schema.CollectionName)
prepare.CreateIndex(ctx, t, mc, hp.NewIndexParams(schema))
prepare.Load(ctx, t, mc, hp.NewLoadParams(schema.CollectionName))
// query output all fields -> output all fields, includes vector and $meta field
allFieldsName := []string{common.DefaultDynamicFieldName}
for _, field := range schema.Fields {
allFieldsName = append(allFieldsName, field.Name)
}
queryResultAll, errQuery := mc.Query(ctx, clientv2.NewQueryOption(schema.CollectionName).WithConsistencyLevel(entity.ClStrong).
WithFilter(fmt.Sprintf("%s < %d", common.DefaultInt64FieldName, 10)).WithOutputFields([]string{"*"}))
common.CheckErr(t, errQuery, true)
common.CheckOutputFields(t, allFieldsName, queryResultAll.Fields)
}
// test query output varchar and binaryVector fields
func TestQueryOutputBinaryAndVarchar(t *testing.T) {
ctx := hp.CreateContext(t, time.Second*common.DefaultTimeout)
mc := createDefaultMilvusClient(ctx, t)
prepare, schema := hp.CollPrepare.CreateCollection(ctx, t, mc, hp.NewCreateCollectionParams(hp.VarcharBinary), hp.TNewFieldsOption(), hp.TNewSchemaOption().TWithEnableDynamicField(true))
prepare.CreateIndex(ctx, t, mc, hp.NewIndexParams(schema))
prepare.Load(ctx, t, mc, hp.NewLoadParams(schema.CollectionName))
// insert
columns := make([]column.Column, 0, len(schema.Fields)+1)
dynamicColumns := hp.GenDynamicColumnData(0, common.DefaultNb)
for _, field := range schema.Fields {
columns = append(columns, hp.GenColumnData(common.DefaultNb, field.DataType, *hp.TNewDataOption().TWithDim(common.DefaultDim)))
}
ids, err := mc.Insert(ctx, clientv2.NewColumnBasedInsertOption(schema.CollectionName, columns...).WithColumns(dynamicColumns...))
common.CheckErr(t, err, true)
require.Equal(t, int64(common.DefaultNb), ids.InsertCount)
prepare.FlushData(ctx, t, mc, schema.CollectionName)
// query output all fields -> output all fields, includes vector and $meta field
expr := fmt.Sprintf("%s in ['0', '1', '2', '3', '4', '5'] ", common.DefaultVarcharFieldName)
allFieldsName := []string{common.DefaultVarcharFieldName, common.DefaultBinaryVecFieldName, common.DefaultDynamicFieldName}
queryResultAll, errQuery := mc.Query(ctx, clientv2.NewQueryOption(schema.CollectionName).WithConsistencyLevel(entity.ClStrong).
WithFilter(expr).WithOutputFields([]string{"*"}))
common.CheckErr(t, errQuery, true)
common.CheckOutputFields(t, allFieldsName, queryResultAll.Fields)
expColumns := []column.Column{hp.MergeColumnsToDynamic(6, dynamicColumns, common.DefaultDynamicFieldName)}
for _, _column := range columns {
expColumns = append(expColumns, _column.Slice(0, 6))
}
common.CheckQueryResult(t, expColumns, queryResultAll.Fields)
}
func TestQueryOutputSparse(t *testing.T) {
t.Skip("https://github.com/milvus-io/milvus-sdk-go/issues/769")
t.Parallel()
ctx := hp.CreateContext(t, time.Second*common.DefaultTimeout)
mc := createDefaultMilvusClient(ctx, t)
prepare, schema := hp.CollPrepare.CreateCollection(ctx, t, mc, hp.NewCreateCollectionParams(hp.Int64VarcharSparseVec), hp.TNewFieldsOption(), hp.TNewSchemaOption())
prepare.CreateIndex(ctx, t, mc, hp.NewIndexParams(schema))
prepare.Load(ctx, t, mc, hp.NewLoadParams(schema.CollectionName))
// insert
columns := make([]column.Column, 0, len(schema.Fields))
for _, field := range schema.Fields {
columns = append(columns, hp.GenColumnData(common.DefaultNb, field.DataType, *hp.TNewDataOption().TWithSparseMaxLen(10)))
}
ids, err := mc.Insert(ctx, clientv2.NewColumnBasedInsertOption(schema.CollectionName, columns...))
common.CheckErr(t, err, true)
require.Equal(t, int64(common.DefaultNb), ids.InsertCount)
prepare.FlushData(ctx, t, mc, schema.CollectionName)
// query output all fields -> output all fields, includes vector and $meta field
expr := fmt.Sprintf("%s < 100 ", common.DefaultInt64FieldName)
expFieldsName := []string{common.DefaultInt64FieldName, common.DefaultVarcharFieldName, common.DefaultSparseVecFieldName}
queryResultAll, errQuery := mc.Query(ctx, clientv2.NewQueryOption(schema.CollectionName).WithConsistencyLevel(entity.ClStrong).WithFilter(expr).WithOutputFields([]string{"*"}))
common.CheckErr(t, errQuery, true)
common.CheckOutputFields(t, expFieldsName, queryResultAll.Fields)
expColumns := make([]column.Column, 0, len(columns))
for _, _column := range columns {
expColumns = append(expColumns, _column.Slice(0, 100))
}
common.CheckQueryResult(t, expColumns, queryResultAll.Fields)
}
// test query different array rows has different element length
func TestQueryArrayDifferentLenBetweenRows(t *testing.T) {
ctx := hp.CreateContext(t, time.Second*common.DefaultTimeout)
mc := createDefaultMilvusClient(ctx, t)
prepare, schema := hp.CollPrepare.CreateCollection(ctx, t, mc, hp.NewCreateCollectionParams(hp.Int64VecAllScalar),
hp.TNewFieldsOption().TWithMaxCapacity(common.TestCapacity*2), hp.TNewSchemaOption())
prepare.CreateIndex(ctx, t, mc, hp.NewIndexParams(schema))
prepare.Load(ctx, t, mc, hp.NewLoadParams(schema.CollectionName))
// insert 2 batch with array capacity 100 and 200
for i := 0; i < 2; i++ {
columns := make([]column.Column, 0, len(schema.Fields))
// each batch has different array capacity
genDataOpt := hp.TNewDataOption().TWithMaxCapacity(common.TestCapacity * (i + 1)).TWithStart(common.DefaultNb * i)
for _, field := range schema.Fields {
if field.DataType == entity.FieldTypeArray {
genDataOpt.TWithElementType(field.ElementType)
}
columns = append(columns, hp.GenColumnData(common.DefaultNb, field.DataType, *genDataOpt))
}
ids, err := mc.Insert(ctx, clientv2.NewColumnBasedInsertOption(schema.CollectionName, columns...))
common.CheckErr(t, err, true)
require.Equal(t, int64(common.DefaultNb), ids.InsertCount)
}
prepare.FlushData(ctx, t, mc, schema.CollectionName)
// query array idx exceeds max capacity, array[200]
expr := fmt.Sprintf("%s[%d] > 0", common.DefaultInt64ArrayField, common.TestCapacity*2)
countRes, err := mc.Query(ctx, clientv2.NewQueryOption(schema.CollectionName).WithConsistencyLevel(entity.ClStrong).WithFilter(expr).WithOutputFields([]string{common.QueryCountFieldName}))
common.CheckErr(t, err, true)
count, _ := countRes.Fields[0].GetAsInt64(0)
require.Equal(t, int64(0), count)
countRes, err = mc.Query(ctx, clientv2.NewQueryOption(schema.CollectionName).WithConsistencyLevel(entity.ClStrong).WithFilter(expr).WithOutputFields([]string{"Count(*)"}))
common.CheckErr(t, err, true)
count, _ = countRes.Fields[0].GetAsInt64(0)
require.Equal(t, int64(0), count)
// query: some rows has element greater than expr index array[100]
expr2 := fmt.Sprintf("%s[%d] > 0", common.DefaultInt64ArrayField, common.TestCapacity)
countRes2, err2 := mc.Query(ctx, clientv2.NewQueryOption(schema.CollectionName).WithConsistencyLevel(entity.ClStrong).WithFilter(expr2).WithOutputFields([]string{common.QueryCountFieldName}))
common.CheckErr(t, err2, true)
count2, _ := countRes2.Fields[0].GetAsInt64(0)
require.Equal(t, int64(common.DefaultNb), count2)
}
// test query with expr and verify output dynamic field data
func TestQueryJsonDynamicExpr(t *testing.T) {
ctx := hp.CreateContext(t, time.Second*common.DefaultTimeout)
mc := createDefaultMilvusClient(ctx, t)
prepare, schema := hp.CollPrepare.CreateCollection(ctx, t, mc, hp.NewCreateCollectionParams(hp.Int64VecJSON),
hp.TNewFieldsOption(), hp.TNewSchemaOption().TWithEnableDynamicField(true))
prepare.InsertData(ctx, t, mc, hp.NewInsertParams(schema, common.DefaultNb), hp.TNewDataOption())
prepare.CreateIndex(ctx, t, mc, hp.NewIndexParams(schema))
prepare.Load(ctx, t, mc, hp.NewLoadParams(schema.CollectionName))
// query with different expr and count
expr := fmt.Sprintf("%s['number'] < 10 || %s < 10", common.DefaultJSONFieldName, common.DefaultDynamicNumberField)
queryRes, err := mc.Query(ctx, clientv2.NewQueryOption(schema.CollectionName).WithFilter(expr).WithConsistencyLevel(entity.ClStrong).
WithOutputFields([]string{common.DefaultJSONFieldName, common.DefaultDynamicFieldName}))
// verify output fields and count, dynamicNumber value
common.CheckErr(t, err, true)
common.CheckOutputFields(t, []string{common.DefaultInt64FieldName, common.DefaultJSONFieldName, common.DefaultDynamicFieldName}, queryRes.Fields)
require.Equal(t, 10, queryRes.ResultCount)
for _, _column := range queryRes.Fields {
if _column.Name() == common.DefaultDynamicNumberField {
var numberData []int64
for i := 0; i < _column.Len(); i++ {
line, _ := _column.GetAsInt64(i)
numberData = append(numberData, line)
}
require.Equal(t, numberData, []int64{0, 1, 2, 3, 4, 5, 6, 7, 8, 9})
}
}
}
// test query with invalid expr
func TestQueryInvalidExpr(t *testing.T) {
ctx := hp.CreateContext(t, time.Second*common.DefaultTimeout)
mc := createDefaultMilvusClient(ctx, t)
prepare, schema := hp.CollPrepare.CreateCollection(ctx, t, mc, hp.NewCreateCollectionParams(hp.Int64VecJSON),
hp.TNewFieldsOption(), hp.TNewSchemaOption().TWithEnableDynamicField(true))
prepare.InsertData(ctx, t, mc, hp.NewInsertParams(schema, 100), hp.TNewDataOption())
prepare.CreateIndex(ctx, t, mc, hp.NewIndexParams(schema))
prepare.Load(ctx, t, mc, hp.NewLoadParams(schema.CollectionName))
for _, _invalidExpr := range common.InvalidExpressions {
_, err := mc.Query(ctx, clientv2.NewQueryOption(schema.CollectionName).WithConsistencyLevel(entity.ClStrong).WithFilter(_invalidExpr.Expr))
common.CheckErr(t, err, _invalidExpr.ErrNil, _invalidExpr.ErrMsg)
}
}
// Test query json and dynamic collection with string expr
func TestQueryCountJsonDynamicExpr(t *testing.T) {
ctx := hp.CreateContext(t, time.Second*common.DefaultTimeout)
mc := createDefaultMilvusClient(ctx, t)
prepare, schema := hp.CollPrepare.CreateCollection(ctx, t, mc, hp.NewCreateCollectionParams(hp.AllFields),
hp.TNewFieldsOption(), hp.TNewSchemaOption().TWithEnableDynamicField(true))
prepare.InsertData(ctx, t, mc, hp.NewInsertParams(schema, common.DefaultNb), hp.TNewDataOption())
prepare.CreateIndex(ctx, t, mc, hp.NewIndexParams(schema))
prepare.Load(ctx, t, mc, hp.NewLoadParams(schema.CollectionName))
// query with different expr and count
type exprCount struct {
expr string
count int64
}
exprCounts := []exprCount{
{expr: "", count: common.DefaultNb},
// pk int64 field expr: < in && ||
{expr: fmt.Sprintf("%s < 1000", common.DefaultInt64FieldName), count: 1000},
{expr: fmt.Sprintf("%s in [0, 1, 2]", common.DefaultInt64FieldName), count: 3},
{expr: fmt.Sprintf("%s >= 1000 && %s < 2000", common.DefaultInt64FieldName, common.DefaultInt64FieldName), count: 1000},
{expr: fmt.Sprintf("%s >= 1000 || %s > 2000", common.DefaultInt64FieldName, common.DefaultInt64FieldName), count: 2000},
{expr: fmt.Sprintf("%s < 1000", common.DefaultFloatFieldName), count: 1000},
// json and dynamic field filter expr: == < in bool/ list/ int
{expr: fmt.Sprintf("%s['number'] == 0", common.DefaultJSONFieldName), count: 0},
{expr: fmt.Sprintf("%s['number'] < 100 and %s['number'] != 0", common.DefaultJSONFieldName, common.DefaultJSONFieldName), count: 50},
{expr: fmt.Sprintf("%s < 100", common.DefaultDynamicNumberField), count: 100},
{expr: "dynamicNumber % 2 == 0", count: 1500},
{expr: fmt.Sprintf("%s['bool'] == true", common.DefaultJSONFieldName), count: 1500 / 2},
{expr: fmt.Sprintf("%s == false", common.DefaultDynamicBoolField), count: 2000},
{expr: fmt.Sprintf("%s in ['1', '2'] ", common.DefaultDynamicStringField), count: 2},
{expr: fmt.Sprintf("%s['string'] in ['1', '2', '5'] ", common.DefaultJSONFieldName), count: 3},
{expr: fmt.Sprintf("%s['list'] == [1, 2] ", common.DefaultJSONFieldName), count: 1},
{expr: fmt.Sprintf("%s['list'] == [0, 1] ", common.DefaultJSONFieldName), count: 0},
{expr: fmt.Sprintf("%s['list'][0] < 10 ", common.DefaultJSONFieldName), count: 5},
{expr: fmt.Sprintf("%s[\"dynamicList\"] != [2, 3]", common.DefaultDynamicFieldName), count: 0},
// json contains
{expr: fmt.Sprintf("json_contains (%s['list'], 2)", common.DefaultJSONFieldName), count: 1},
{expr: fmt.Sprintf("json_contains (%s['number'], 0)", common.DefaultJSONFieldName), count: 0},
{expr: fmt.Sprintf("json_contains_all (%s['list'], [1, 2])", common.DefaultJSONFieldName), count: 1},
{expr: fmt.Sprintf("JSON_CONTAINS_ANY (%s['list'], [1, 3])", common.DefaultJSONFieldName), count: 2},
// string like
{expr: "dynamicString like '1%' ", count: 1111},
// key exist
{expr: fmt.Sprintf("exists %s['list']", common.DefaultJSONFieldName), count: common.DefaultNb / 4},
{expr: "exists a ", count: 0},
{expr: fmt.Sprintf("exists %s ", common.DefaultDynamicListField), count: common.DefaultNb},
{expr: fmt.Sprintf("exists %s ", common.DefaultDynamicStringField), count: common.DefaultNb},
// data type not match and no error
{expr: fmt.Sprintf("%s['number'] == '0' ", common.DefaultJSONFieldName), count: 0},
// json field
{expr: fmt.Sprintf("%s >= 1500", common.DefaultJSONFieldName), count: 1500 / 2}, // json >= 1500
{expr: fmt.Sprintf("%s > 1499.5", common.DefaultJSONFieldName), count: 1500 / 2}, // json >= 1500.0
{expr: fmt.Sprintf("%s like '21%%'", common.DefaultJSONFieldName), count: 100 / 4}, // json like '21%'
{expr: fmt.Sprintf("%s == [1503, 1504]", common.DefaultJSONFieldName), count: 1}, // json == [1,2]
{expr: fmt.Sprintf("%s[0] > 1", common.DefaultJSONFieldName), count: 1500 / 4}, // json[0] > 1
{expr: fmt.Sprintf("%s[0][0] > 1", common.DefaultJSONFieldName), count: 0}, // json == [1,2]
}
for _, _exprCount := range exprCounts {
log.Debug("TestQueryCountJsonDynamicExpr", zap.String("expr", _exprCount.expr))
countRes, _ := mc.Query(ctx, clientv2.NewQueryOption(schema.CollectionName).WithConsistencyLevel(entity.ClStrong).WithFilter(_exprCount.expr).WithOutputFields([]string{common.QueryCountFieldName}))
count, _ := countRes.Fields[0].GetAsInt64(0)
require.Equal(t, _exprCount.count, count)
}
}
// test query with all kinds of array expr
func TestQueryArrayFieldExpr(t *testing.T) {
ctx := hp.CreateContext(t, time.Second*common.DefaultTimeout)
mc := createDefaultMilvusClient(ctx, t)
prepare, schema := hp.CollPrepare.CreateCollection(ctx, t, mc, hp.NewCreateCollectionParams(hp.AllFields),
hp.TNewFieldsOption(), hp.TNewSchemaOption().TWithEnableDynamicField(true))
prepare.InsertData(ctx, t, mc, hp.NewInsertParams(schema, common.DefaultNb), hp.TNewDataOption())
prepare.CreateIndex(ctx, t, mc, hp.NewIndexParams(schema))
prepare.Load(ctx, t, mc, hp.NewLoadParams(schema.CollectionName))
// create collection
capacity := int64(common.TestCapacity)
type exprCount struct {
expr string
count int64
}
exprCounts := []exprCount{
{expr: fmt.Sprintf("%s[0] == false", common.DefaultBoolArrayField), count: common.DefaultNb / 2}, // array[0] ==
{expr: fmt.Sprintf("%s[0] > 0", common.DefaultInt64ArrayField), count: common.DefaultNb - 1}, // array[0] >
{expr: fmt.Sprintf("%s[0] > 0", common.DefaultInt8ArrayField), count: 1524}, // array[0] > int8 range: [-128, 127]
{expr: fmt.Sprintf("json_contains (%s, %d)", common.DefaultInt16ArrayField, capacity), count: capacity}, // json_contains(array, 1)
{expr: fmt.Sprintf("array_contains (%s, %d)", common.DefaultInt16ArrayField, capacity), count: capacity}, // array_contains(array, 1)
{expr: fmt.Sprintf("array_contains (%s, 1)", common.DefaultInt32ArrayField), count: 2}, // array_contains(array, 1)
{expr: fmt.Sprintf("json_contains (%s, 1)", common.DefaultInt32ArrayField), count: 2}, // json_contains(array, 1)
{expr: fmt.Sprintf("array_contains (%s, 1000000)", common.DefaultInt32ArrayField), count: 0}, // array_contains(array, 1)
{expr: fmt.Sprintf("json_contains_all (%s, [90, 91])", common.DefaultInt64ArrayField), count: 91}, // json_contains_all(array, [x])
{expr: fmt.Sprintf("array_contains_all (%s, [1, 2])", common.DefaultInt64ArrayField), count: 2}, // array_contains_all(array, [x])
{expr: fmt.Sprintf("array_contains_any (%s, [0, 100, 10000])", common.DefaultFloatArrayField), count: 101}, // array_contains_any(array, [x])
{expr: fmt.Sprintf("json_contains_any (%s, [0, 100, 10])", common.DefaultFloatArrayField), count: 101}, // json_contains_any (array, [x])
{expr: fmt.Sprintf("%s == [0, 1]", common.DefaultDoubleArrayField), count: 0}, // array ==
{expr: fmt.Sprintf("array_length(%s) == 10", common.DefaultVarcharArrayField), count: 0}, // array_length
{expr: fmt.Sprintf("array_length(%s) == %d", common.DefaultDoubleArrayField, capacity), count: common.DefaultNb}, // array_length
}
for _, _exprCount := range exprCounts {
log.Debug("TestQueryCountJsonDynamicExpr", zap.String("expr", _exprCount.expr))
countRes, _ := mc.Query(ctx, clientv2.NewQueryOption(schema.CollectionName).WithConsistencyLevel(entity.ClStrong).WithFilter(_exprCount.expr).WithOutputFields([]string{common.QueryCountFieldName}))
count, _ := countRes.Fields[0].GetAsInt64(0)
require.Equal(t, _exprCount.count, count)
}
}
// test query output invalid count(*) fields
func TestQueryOutputInvalidOutputFieldCount(t *testing.T) {
type invalidCountStruct struct {
countField string
errMsg string
}
ctx := hp.CreateContext(t, time.Second*common.DefaultTimeout)
mc := createDefaultMilvusClient(ctx, t)
prepare, schema := hp.CollPrepare.CreateCollection(ctx, t, mc, hp.NewCreateCollectionParams(hp.Int64Vec),
hp.TNewFieldsOption(), hp.TNewSchemaOption().TWithEnableDynamicField(false))
prepare.InsertData(ctx, t, mc, hp.NewInsertParams(schema, common.DefaultNb), hp.TNewDataOption())
prepare.CreateIndex(ctx, t, mc, hp.NewIndexParams(schema))
prepare.Load(ctx, t, mc, hp.NewLoadParams(schema.CollectionName))
// invalid expr
invalidOutputFieldCount := []invalidCountStruct{
{countField: "ccount(*)", errMsg: "field ccount(*) not exist"},
{countField: "count[*]", errMsg: "field count[*] not exist"},
{countField: "count", errMsg: "field count not exist"},
{countField: "count(**)", errMsg: "field count(**) not exist"},
}
for _, invalidCount := range invalidOutputFieldCount {
queryExpr := fmt.Sprintf("%s >= 0", common.DefaultInt64FieldName)
// query with empty output fields []string{}-> output "int64"
_, err := mc.Query(ctx, clientv2.NewQueryOption(schema.CollectionName).WithConsistencyLevel(entity.ClStrong).WithFilter(queryExpr).WithOutputFields([]string{invalidCount.countField}))
common.CheckErr(t, err, false, invalidCount.errMsg)
}
}