Fix load index with empty file list (#26236)

Signed-off-by: yah01 <yah2er0ne@outlook.com>
pull/25877/head
yah01 2023-08-09 18:39:16 +08:00 committed by GitHub
parent a888606423
commit 889424b3f9
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 33 additions and 5 deletions

View File

@ -102,7 +102,9 @@ func (c *IndexChecker) checkReplica(ctx context.Context, collection *meta.Collec
continue
}
for _, info := range infos {
if missingFields.Contain(info.GetFieldID()) && info.GetEnableIndex() {
if missingFields.Contain(info.GetFieldID()) &&
info.GetEnableIndex() &&
len(info.GetIndexFilePaths()) > 0 {
segmentsToUpdate.Insert(segment)
}
}

View File

@ -97,9 +97,10 @@ func (suite *IndexCheckerSuite) TestLoadIndex() {
suite.broker.EXPECT().GetIndexInfo(mock.Anything, int64(1), int64(2)).
Return([]*querypb.FieldIndexInfo{
{
FieldID: 101,
IndexID: 1000,
EnableIndex: true,
FieldID: 101,
IndexID: 1000,
EnableIndex: true,
IndexFilePaths: []string{"index"},
},
}, nil)

View File

@ -470,7 +470,7 @@ func (loader *segmentLoader) loadSegment(ctx context.Context,
if segment.Type() == SegmentTypeSealed {
fieldID2IndexInfo := make(map[int64]*querypb.FieldIndexInfo)
for _, indexInfo := range loadInfo.IndexInfos {
if len(indexInfo.IndexFilePaths) > 0 {
if len(indexInfo.GetIndexFilePaths()) > 0 {
fieldID := indexInfo.FieldID
fieldID2IndexInfo[fieldID] = indexInfo
}
@ -947,6 +947,11 @@ func (loader *segmentLoader) LoadIndex(ctx context.Context, segment *LocalSegmen
func(info *datapb.FieldBinlog) (int64, *datapb.FieldBinlog) { return info.GetFieldID(), info })
for _, info := range loadInfo.GetIndexInfos() {
if len(info.GetIndexFilePaths()) == 0 {
log.Warn("failed to add index for segment, index file list is empty, the segment may be too small")
return merr.WrapErrIndexNotFound("index file list empty")
}
fieldInfo, ok := fieldInfos[info.GetFieldID()]
if !ok {
return merr.WrapErrParameterInvalid("index info with corresponding field info", "missing field info", strconv.FormatInt(fieldInfo.GetFieldID(), 10))

View File

@ -28,6 +28,7 @@ import (
"github.com/milvus-io/milvus/internal/storage"
"github.com/milvus-io/milvus/internal/util/initcore"
"github.com/milvus-io/milvus/pkg/util/funcutil"
"github.com/milvus-io/milvus/pkg/util/merr"
"github.com/milvus-io/milvus/pkg/util/metric"
"github.com/milvus-io/milvus/pkg/util/paramtable"
)
@ -352,6 +353,25 @@ func (suite *SegmentLoaderSuite) TestLoadDeltaLogs() {
}
}
func (suite *SegmentLoaderSuite) TestLoadIndex() {
ctx := context.Background()
segment := &LocalSegment{}
loadInfo := &querypb.SegmentLoadInfo{
SegmentID: 1,
PartitionID: suite.partitionID,
CollectionID: suite.collectionID,
IndexInfos: []*querypb.FieldIndexInfo{
{
IndexFilePaths: []string{},
},
},
}
err := suite.loader.LoadIndex(ctx, segment, loadInfo)
suite.ErrorIs(err, merr.ErrIndexNotFound)
}
func (suite *SegmentLoaderSuite) TestLoadWithMmap() {
key := paramtable.Get().QueryNodeCfg.MmapDirPath.Key
paramtable.Get().Save(key, "/tmp/mmap-test")