milvus-io
/
milvus
mirror of https://github.com/milvus-io/milvus.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

enhance: skip load bm25 sparse row data (#39078) 

Skip load row data of bm25 sparse field, because it's forbidden to
return.

---------

Signed-off-by: aoiasd <zhicheng.yue@zilliz.com>
pull/39581/head
aoiasd 2025-02-06 18:40:44 +08:00 committed by GitHub
parent 427b6a4c94
commit 74890dabc9
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
2 changed files with 31 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
internal/querynodev2/segments/segment.go
View File

@ -62,6 +62,7 @@ import (
"github.com/milvus-io/milvus/pkg/util/indexparams"
"github.com/milvus-io/milvus/pkg/util/merr"
"github.com/milvus-io/milvus/pkg/util/metautil"
"github.com/milvus-io/milvus/pkg/util/metric"
"github.com/milvus-io/milvus/pkg/util/paramtable"
"github.com/milvus-io/milvus/pkg/util/timerecord"
"github.com/milvus-io/milvus/pkg/util/typeutil"
@ -1067,10 +1068,24 @@ func (s *LocalSegment) innerLoadIndex(ctx context.Context,
return err
}
updateIndexInfoSpan := tr.RecordSpan()
// Skip warnup chunk cache when
// . scalar data
// . index has row data
// . vector was bm25 function output
if !typeutil.IsVectorType(fieldType) || s.HasRawData(indexInfo.GetFieldID()) {
return nil
}
metricType, err := funcutil.GetAttrByKeyFromRepeatedKV(common.MetricTypeKey, indexInfo.IndexParams)
if err != nil {
return fmt.Errorf("metric type not exist in index params")
}
if metricType == metric.BM25 {
return nil
}
// 4.
mmapChunkCache := paramtable.Get().QueryNodeCfg.MmapChunkCache.GetAsBool()
s.WarmupChunkCache(ctx, indexInfo.GetFieldID(), mmapChunkCache)

21
internal/querynodev2/segments/segment_loader.go
View File

@ -57,6 +57,7 @@ import (
"github.com/milvus-io/milvus/pkg/util/funcutil"
"github.com/milvus-io/milvus/pkg/util/hardware"
"github.com/milvus-io/milvus/pkg/util/merr"
"github.com/milvus-io/milvus/pkg/util/metric"
"github.com/milvus-io/milvus/pkg/util/paramtable"
"github.com/milvus-io/milvus/pkg/util/syncutil"
"github.com/milvus-io/milvus/pkg/util/timerecord"
@ -1509,12 +1510,22 @@ func getResourceUsageEstimateOfSegment(schema *schemapb.CollectionSchema, loadIn
if !estimateResult.HasRawData && !isVectorType {
shouldCalculateDataSize = true
}
if !estimateResult.HasRawData && isVectorType {
mmapChunkCache := paramtable.Get().QueryNodeCfg.MmapChunkCache.GetAsBool()
if mmapChunkCache {
segmentDiskSize += binlogSize
} else {
segmentMemorySize += binlogSize
metricType, err := funcutil.GetAttrByKeyFromRepeatedKV(common.MetricTypeKey, fieldIndexInfo.IndexParams)
if err != nil {
return nil, errors.Wrapf(err, "failed to estimate resource usage of index, metric type nout found, collection %d, segment %d, indexBuildID %d",
loadInfo.GetCollectionID(),
loadInfo.GetSegmentID(),
fieldIndexInfo.GetBuildID())
}
if metricType != metric.BM25 {
mmapChunkCache := paramtable.Get().QueryNodeCfg.MmapChunkCache.GetAsBool()
if mmapChunkCache {
segmentDiskSize += binlogSize
} else {
segmentMemorySize += binlogSize
}
}
}
} else {