milvus-io
/
milvus
mirror of https://github.com/milvus-io/milvus.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

test: Optimize sparse vector case (#35661) 

/kind improvement

Signed-off-by: ThreadDao <yufen.zong@zilliz.com>
pull/35699/head
ThreadDao 2024-08-26 14:54:57 +08:00 committed by GitHub
parent 82743c5c50
commit a90133cdf5
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
3 changed files with 36 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
tests/go_client/common/response_checker.go
View File

@ -151,12 +151,12 @@ func CheckOutputFields(t *testing.T, expFields []string, actualColumns []column.
// CheckSearchResult check search result, check nq, topk, ids, score
func CheckSearchResult(t *testing.T, actualSearchResults []client.ResultSet, expNq int, expTopK int) {
require.Equal(t, len(actualSearchResults), expNq)
require.Equalf(t, len(actualSearchResults), expNq, fmt.Sprintf("Expected nq=%d, actual SearchResultsLen=%d", expNq, len(actualSearchResults)))
require.Len(t, actualSearchResults, expNq)
for _, actualSearchResult := range actualSearchResults {
require.Equal(t, actualSearchResult.ResultCount, expTopK)
require.Equal(t, actualSearchResult.IDs.Len(), expTopK)
require.Equal(t, len(actualSearchResult.Scores), expTopK)
require.Equalf(t, actualSearchResult.ResultCount, expTopK, fmt.Sprintf("Expected topK=%d, actual ResultCount=%d", expTopK, actualSearchResult.ResultCount))
require.Equalf(t, actualSearchResult.IDs.Len(), expTopK, fmt.Sprintf("Expected topK=%d, actual IDsLen=%d", expTopK, actualSearchResult.IDs.Len()))
require.Equalf(t, len(actualSearchResult.Scores), expTopK, fmt.Sprintf("Expected topK=%d, actual ScoresLen=%d", expTopK, len(actualSearchResult.Scores)))
}
}

20
tests/go_client/testcases/insert_test.go
View File

@ -740,3 +740,23 @@ func TestInsertAutoIDInvalidRow(t *testing.T) {
common.CheckErr(t, err, false, "missing pk data")
}
}
func TestFlushRate(t *testing.T) {
ctx := hp.CreateContext(t, time.Second*common.DefaultTimeout)
mc := createDefaultMilvusClient(ctx, t)
// create collection
cp := hp.NewCreateCollectionParams(hp.Int64Vec)
_, schema := hp.CollPrepare.CreateCollection(ctx, t, mc, cp, hp.TNewFieldsOption().TWithAutoID(true), hp.TNewSchemaOption())
// insert
columnOpt := hp.TNewDataOption().TWithDim(common.DefaultDim)
vecColumn := hp.GenColumnData(common.DefaultNb, entity.FieldTypeFloatVector, *columnOpt)
insertOpt := client.NewColumnBasedInsertOption(schema.CollectionName).WithColumns(vecColumn)
_, err := mc.Insert(ctx, insertOpt)
common.CheckErr(t, err, true)
_, err = mc.Flush(ctx, client.NewFlushOption(schema.CollectionName))
common.CheckErr(t, err, true)
_, err = mc.Flush(ctx, client.NewFlushOption(schema.CollectionName))
common.CheckErr(t, err, false, "request is rejected by grpc RateLimiter middleware, please retry later: rate limit exceeded[rate=0.1]")
}

19
tests/go_client/testcases/search_test.go
View File

@ -80,6 +80,7 @@ func TestSearchInvalidCollectionPartitionName(t *testing.T) {
// test search empty collection -> return empty
func TestSearchEmptyCollection(t *testing.T) {
t.Skip("https://github.com/milvus-io/milvus/issues/33952")
t.Parallel()
ctx := hp.CreateContext(t, time.Second*common.DefaultTimeout)
mc := createDefaultMilvusClient(ctx, t)
@ -104,13 +105,13 @@ func TestSearchEmptyCollection(t *testing.T) {
resSearch, errSearch := mc.Search(ctx, client.NewSearchOption(schema.CollectionName, common.DefaultLimit, _mNameVec.queryVec).
WithConsistencyLevel(entity.ClStrong).WithANNSField(_mNameVec.fieldName))
common.CheckErr(t, errSearch, true)
t.Log("https://github.com/milvus-io/milvus/issues/33952")
common.CheckSearchResult(t, resSearch, 0, 0)
common.CheckSearchResult(t, resSearch, common.DefaultNq, 0)
}
}
}
func TestSearchEmptySparseCollection(t *testing.T) {
t.Skip("https://github.com/milvus-io/milvus/issues/33952")
ctx := hp.CreateContext(t, time.Second*common.DefaultTimeout)
mc := createDefaultMilvusClient(ctx, t)
@ -124,8 +125,7 @@ func TestSearchEmptySparseCollection(t *testing.T) {
resSearch, errSearch := mc.Search(ctx, client.NewSearchOption(schema.CollectionName, common.DefaultLimit, vectors).
WithConsistencyLevel(entity.ClStrong).WithANNSField(common.DefaultSparseVecFieldName))
common.CheckErr(t, errSearch, true)
t.Log("https://github.com/milvus-io/milvus/issues/33952")
common.CheckSearchResult(t, resSearch, 0, 0)
common.CheckSearchResult(t, resSearch, common.DefaultNq, 0)
}
// test search with partition names []string{}, specify partitions
@ -976,7 +976,8 @@ func TestSearchWithEmptySparseVector(t *testing.T) {
// test search from empty sparse vectors collection
func TestSearchFromEmptySparseVector(t *testing.T) {
t.Parallel()
t.Skip("https://github.com/milvus-io/milvus/issues/33952")
t.Skip("https://github.com/zilliztech/knowhere/issues/774")
idxInverted := index.NewSparseInvertedIndex(entity.IP, 0.1)
ctx := hp.CreateContext(t, time.Second*common.DefaultTimeout*2)
mc := createDefaultMilvusClient(ctx, t)
@ -984,6 +985,8 @@ func TestSearchFromEmptySparseVector(t *testing.T) {
for _, idx := range []index.Index{idxInverted} {
prepare, schema := hp.CollPrepare.CreateCollection(ctx, t, mc, hp.NewCreateCollectionParams(hp.Int64VarcharSparseVec), hp.TNewFieldsOption(), hp.TNewSchemaOption().
TWithEnableDynamicField(true))
prepare.InsertData(ctx, t, mc, hp.NewInsertParams(schema), hp.TNewDataOption().TWithSparseMaxLen(128).TWithStart(common.DefaultNb))
prepare.FlushData(ctx, t, mc, schema.CollectionName)
prepare.CreateIndex(ctx, t, mc, hp.TNewIndexParams(schema).TWithFieldIndex(map[string]index.Index{common.DefaultSparseVecFieldName: idx}))
prepare.Load(ctx, t, mc, hp.NewLoadParams(schema.CollectionName))
@ -1009,10 +1012,12 @@ func TestSearchFromEmptySparseVector(t *testing.T) {
vector1, _ := entity.NewSliceSparseEmbedding([]uint32{}, []float32{})
vector2, _ := entity.NewSliceSparseEmbedding([]uint32{0, 2, 5, 10, 100}, []float32{rand.Float32(), rand.Float32(), rand.Float32(), rand.Float32(), rand.Float32()})
for _, vector := range []entity.Vector{vector1, vector2} {
// search from sparse collection: part normal sparse vectors, part empty sparse
// excepted: The empty vector is not related to any other vector, so it will not be returned，and alsopty obtained as the search vector.
for limit, vector := range map[int]entity.Vector{0: vector1, common.DefaultLimit: vector2} {
searchRes, errSearch1 := mc.Search(ctx, client.NewSearchOption(schema.CollectionName, common.DefaultLimit, []entity.Vector{vector}).WithConsistencyLevel(entity.ClStrong))
common.CheckErr(t, errSearch1, true)
common.CheckSearchResult(t, searchRes, 1, 0)
common.CheckSearchResult(t, searchRes, 1, limit)
}
}
}