milvus-io
/
milvus
mirror of https://github.com/milvus-io/milvus.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Filter NaN when reducing search results (#6806) 

Signed-off-by: dragondriver <jiquan.long@zilliz.com>
pull/6821/head
dragondriver 2021-07-27 10:19:21 +08:00 committed by GitHub
parent 93f0c9d87b
commit 3cc8ee298e
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 10 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
internal/proxy/task.go
View File

@ -1698,6 +1698,10 @@ func reduceSearchResultDataParallel(searchResultData []*schemapb.SearchResultDat
continue
}
distance := searchResultData[q].Scores[idx*topk+loc]
// https://github.com/milvus-io/milvus/issues/6781
if math.IsNaN(float64(distance)) {
continue
}
if distance > maxDistance || (math.Abs(float64(distance-maxDistance)) < math.SmallestNonzeroFloat32 && choice != q) {
choice = q
maxDistance = distance
@ -1710,7 +1714,12 @@ func reduceSearchResultDataParallel(searchResultData []*schemapb.SearchResultDat
choiceOffset := locs[choice]
// check if distance is valid, `invalid` here means very very big,
// in this process, distance here is the smallest, so the rest of distance are all invalid
if searchResultData[choice].Scores[idx*topk+choiceOffset] <= minFloat32 {
// https://github.com/milvus-io/milvus/issues/6781
// tanimoto distance between two binary vectors maybe -inf, so -inf distance shouldn't be filtered,
// otherwise it will cause that the number of hit records is less than needed (topk).
// in the above process, we have already filtered NaN distance.
distance := searchResultData[choice].Scores[idx*topk+choiceOffset]
if distance < minFloat32 {
break
}
curIdx := idx*topk + choiceOffset