milvus-io
/
milvus
mirror of https://github.com/milvus-io/milvus.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Checkout field ids when load segment in query node 

Signed-off-by: xige-16 <xi.ge@zilliz.com>
pull/4973/head^2
xige-16 2021-02-22 10:44:38 +08:00 committed by yefu.chen
parent 7ac1821cf1
commit 861576f77a
5 changed files with 18 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
internal/proxynode/paramtable.go
View File

@ -177,6 +177,7 @@ func (pt *ParamTable) initQueryNodeIDList() []UniqueID {
} }
ret = append(ret, UniqueID(v)) ret = append(ret, UniqueID(v))
} }
pt.QueryNodeIDList = ret
return ret return ret
} }

7
internal/querynode/load_service.go
View File

@ -114,7 +114,12 @@ func (s *loadService) loadSegmentInternal(collectionID UniqueID, partitionID Uni
return err return err
} }
targetFields := s.segLoader.getTargetFields(paths, srcFieldIDs, fieldIDs) //fmt.Println("srcFieldIDs in internal:", srcFieldIDs)
//fmt.Println("dstFieldIDs in internal:", fieldIDs)
targetFields, err := s.segLoader.checkTargetFields(paths, srcFieldIDs, fieldIDs)
if err != nil {
return err
}
err = s.segLoader.loadSegmentFieldsData(segment, targetFields) err = s.segLoader.loadSegmentFieldsData(segment, targetFields)
if err != nil { if err != nil {
return err return err

2
internal/querynode/load_service_test.go
View File

@ -1142,7 +1142,7 @@ func TestSegmentLoad_Search_Vector(t *testing.T) {
paths, srcFieldIDs, err := generateInsertBinLog(collectionID, partitionID, segmentID, keyPrefix) paths, srcFieldIDs, err := generateInsertBinLog(collectionID, partitionID, segmentID, keyPrefix)
assert.NoError(t, err) assert.NoError(t, err)
fieldsMap := node.loadService.segLoader.getTargetFields(paths, srcFieldIDs, fieldIDs) fieldsMap, _ := node.loadService.segLoader.checkTargetFields(paths, srcFieldIDs, fieldIDs)
assert.Equal(t, len(fieldsMap), 2) assert.Equal(t, len(fieldsMap), 2)
segment, err := node.replica.getSegmentByID(segmentID) segment, err := node.replica.getSegmentByID(segmentID)

16
internal/querynode/segment_loader.go
View File

@ -8,6 +8,7 @@ import (
"github.com/zilliztech/milvus-distributed/internal/kv" "github.com/zilliztech/milvus-distributed/internal/kv"
minioKV "github.com/zilliztech/milvus-distributed/internal/kv/minio" minioKV "github.com/zilliztech/milvus-distributed/internal/kv/minio"
"github.com/zilliztech/milvus-distributed/internal/msgstream" "github.com/zilliztech/milvus-distributed/internal/msgstream"
"github.com/zilliztech/milvus-distributed/internal/proto/commonpb"
"github.com/zilliztech/milvus-distributed/internal/proto/datapb" "github.com/zilliztech/milvus-distributed/internal/proto/datapb"
"github.com/zilliztech/milvus-distributed/internal/proto/internalpb2" "github.com/zilliztech/milvus-distributed/internal/proto/internalpb2"
"github.com/zilliztech/milvus-distributed/internal/storage" "github.com/zilliztech/milvus-distributed/internal/storage"
@ -53,11 +54,11 @@ func (loader *segmentLoader) getInsertBinlogPaths(segmentID UniqueID) ([]*intern
} }
pathResponse, err := loader.dataClient.GetInsertBinlogPaths(insertBinlogPathRequest) pathResponse, err := loader.dataClient.GetInsertBinlogPaths(insertBinlogPathRequest)
if err != nil { if err != nil || pathResponse.Status.ErrorCode != commonpb.ErrorCode_SUCCESS {
return nil, nil, err return nil, nil, err
} }
if len(pathResponse.FieldIDs) != len(pathResponse.Paths) { if len(pathResponse.FieldIDs) != len(pathResponse.Paths) || len(pathResponse.FieldIDs) <= 0 {
return nil, nil, errors.New("illegal InsertBinlogPathsResponse") return nil, nil, errors.New("illegal InsertBinlogPathsResponse")
} }
@ -82,7 +83,7 @@ func (loader *segmentLoader) filterOutVectorFields(fieldIDs []int64, vectorField
return targetFields return targetFields
} }
func (loader *segmentLoader) getTargetFields(paths []*internalpb2.StringList, srcFieldIDS []int64, dstFields []int64) map[int64]*internalpb2.StringList { func (loader *segmentLoader) checkTargetFields(paths []*internalpb2.StringList, srcFieldIDs []int64, dstFieldIDs []int64) (map[int64]*internalpb2.StringList, error) {
targetFields := make(map[int64]*internalpb2.StringList) targetFields := make(map[int64]*internalpb2.StringList)
containsFunc := func(s []int64, e int64) bool { containsFunc := func(s []int64, e int64) bool {
@ -94,13 +95,14 @@ func (loader *segmentLoader) getTargetFields(paths []*internalpb2.StringList, sr
return false return false
} }
for i, fieldID := range srcFieldIDS { for i, fieldID := range dstFieldIDs {
if containsFunc(dstFields, fieldID) { if !containsFunc(srcFieldIDs, fieldID) {
targetFields[fieldID] = paths[i] return nil, errors.New("uncompleted fields")
} }
targetFields[fieldID] = paths[i]
} }
return targetFields return targetFields, nil
} }
func (loader *segmentLoader) loadSegmentFieldsData(segment *Segment, targetFields map[int64]*internalpb2.StringList) error { func (loader *segmentLoader) loadSegmentFieldsData(segment *Segment, targetFields map[int64]*internalpb2.StringList) error {

2
tests/python/test_index.py
View File

@ -524,7 +524,6 @@ class TestIndexBase:
connect.drop_index(collection, field_name) connect.drop_index(collection, field_name)
@pytest.mark.skip("r0.3-test")
class TestIndexBinary: class TestIndexBinary:
@pytest.fixture( @pytest.fixture(
scope="function", scope="function",
@ -594,6 +593,7 @@ class TestIndexBinary:
ids = connect.bulk_insert(binary_collection, default_binary_entities, partition_tag=default_tag) ids = connect.bulk_insert(binary_collection, default_binary_entities, partition_tag=default_tag)
connect.create_index(binary_collection, binary_field_name, get_jaccard_index) connect.create_index(binary_collection, binary_field_name, get_jaccard_index)
@pytest.mark.skip("r0.3-test")
@pytest.mark.timeout(BUILD_TIMEOUT) @pytest.mark.timeout(BUILD_TIMEOUT)
def test_create_index_search_with_query_vectors(self, connect, binary_collection, get_jaccard_index, get_nq): def test_create_index_search_with_query_vectors(self, connect, binary_collection, get_jaccard_index, get_nq):
''' '''