milvus-io
/
milvus
mirror of https://github.com/milvus-io/milvus.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

fix: fix mmap enabled check in resource estimation (#31536) (#31574) 

issue: https://github.com/milvus-io/milvus/issues/31569

related pr: https://github.com/milvus-io/milvus/pull/31536

Signed-off-by: sunby <sunbingyi1992@gmail.com>
pull/31641/head^2
Bingyi Sun 2024-03-27 16:37:11 +08:00 committed by GitHub
parent af7da00488
commit 875887244b
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
2 changed files with 16 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
internal/querynodev2/segments/load_index_info.go
View File

@ -70,19 +70,24 @@ func deleteLoadIndexInfo(info *LoadIndexInfo) {
}).Await()
}
func isIndexMmapEnable(indexInfo *querypb.FieldIndexInfo) bool {
enableMmap := common.IsMmapEnabled(indexInfo.IndexParams...)
if !enableMmap {
_, ok := funcutil.KeyValuePair2Map(indexInfo.IndexParams)[common.MmapEnabledKey]
indexType := datacoord.GetIndexType(indexInfo.IndexParams)
indexSupportMmap := indexparamcheck.IsMmapSupported(indexType)
enableMmap = !ok && params.Params.QueryNodeCfg.MmapEnabled.GetAsBool() && indexSupportMmap
}
return enableMmap
}
func (li *LoadIndexInfo) appendLoadIndexInfo(ctx context.Context, indexInfo *querypb.FieldIndexInfo, collectionID int64, partitionID int64, segmentID int64, fieldType schemapb.DataType) error {
fieldID := indexInfo.FieldID
indexPaths := indexInfo.IndexFilePaths
indexParams := funcutil.KeyValuePair2Map(indexInfo.IndexParams)
enableMmap := common.IsMmapEnabled(indexInfo.IndexParams...)
if !enableMmap {
_, ok := indexParams[common.MmapEnabledKey]
indexType := datacoord.GetIndexType(indexInfo.IndexParams)
indexSupportMmap := indexparamcheck.IsMmapSupported(indexType)
enableMmap = !ok && params.Params.QueryNodeCfg.MmapEnabled.GetAsBool() && indexSupportMmap
}
enableMmap := isIndexMmapEnable(indexInfo)
// as Knowhere reports error if encounter a unknown param, we need to delete it
delete(indexParams, common.MmapEnabledKey)

5
internal/querynodev2/segments/segment_loader.go
View File

@ -1456,8 +1456,9 @@ func getResourceUsageEstimateOfSegment(schema *schemapb.CollectionSchema, loadIn
for _, fieldBinlog := range loadInfo.BinlogPaths {
fieldID := fieldBinlog.FieldID
mmapEnabled := common.IsFieldMmapEnabled(schema, fieldID)
var mmapEnabled bool
if fieldIndexInfo, ok := vecFieldID2IndexInfo[fieldID]; ok {
mmapEnabled = isIndexMmapEnable(fieldIndexInfo)
neededMemSize, neededDiskSize, err := getIndexAttrCache().GetIndexResourceUsage(fieldIndexInfo, multiplyFactor.memoryIndexUsageFactor)
if err != nil {
return nil, errors.Wrapf(err, "failed to get index size collection %d, segment %d, indexBuildID %d",
@ -1472,6 +1473,8 @@ func getResourceUsageEstimateOfSegment(schema *schemapb.CollectionSchema, loadIn
segmentDiskSize += neededDiskSize
}
} else {
mmapEnabled = common.IsFieldMmapEnabled(schema, fieldID) ||
(!common.FieldHasMmapKey(schema, fieldID) && params.Params.QueryNodeCfg.MmapEnabled.GetAsBool())
binlogSize := uint64(getBinlogDataSize(fieldBinlog))
if mmapEnabled {
segmentDiskSize += binlogSize