mirror of https://github.com/milvus-io/milvus.git
enhance: consider the mmap chunck cache config when resource usage estimate (#36814)
- issue: #36530 Signed-off-by: SimFG <bang.fu@zilliz.com>pull/37064/head
parent
3de57ec4fa
commit
903c18ba26
|
@ -173,11 +173,9 @@ IndexFactory::VecIndexLoadResource(
|
|||
index_type, index_version, config);
|
||||
break;
|
||||
default:
|
||||
PanicInfo(
|
||||
milvus::DataTypeInvalid,
|
||||
fmt::format(
|
||||
"invalid data type to estimate index load resource: {}",
|
||||
field_type));
|
||||
LOG_ERROR("invalid data type to estimate index load resource: {}",
|
||||
field_type);
|
||||
return LoadResourceRequest{0, 0, 0, 0, true};
|
||||
}
|
||||
|
||||
LoadResourceRequest request{};
|
||||
|
@ -274,10 +272,10 @@ IndexFactory::ScalarIndexLoadResource(
|
|||
request.max_disk_cost = index_size_gb;
|
||||
request.has_raw_data = false;
|
||||
} else {
|
||||
PanicInfo(milvus::UnexpectedError,
|
||||
fmt::format("invalid index type to estimate scalar index "
|
||||
"load resource: {}",
|
||||
index_type));
|
||||
LOG_ERROR(
|
||||
"invalid index type to estimate scalar index load resource: {}",
|
||||
index_type);
|
||||
return LoadResourceRequest{0, 0, 0, 0, true};
|
||||
}
|
||||
return request;
|
||||
}
|
||||
|
|
|
@ -1434,6 +1434,7 @@ func getResourceUsageEstimateOfSegment(schema *schemapb.CollectionSchema, loadIn
|
|||
return nil, err
|
||||
}
|
||||
binlogSize := uint64(getBinlogDataMemorySize(fieldBinlog))
|
||||
isVectorType := typeutil.IsVectorType(fieldSchema.DataType)
|
||||
shouldCalculateDataSize := false
|
||||
|
||||
if fieldIndexInfo, ok := fieldID2IndexInfo[fieldID]; ok {
|
||||
|
@ -1452,9 +1453,17 @@ func getResourceUsageEstimateOfSegment(schema *schemapb.CollectionSchema, loadIn
|
|||
|
||||
indexMemorySize += estimateResult.MaxMemoryCost
|
||||
segmentDiskSize += estimateResult.MaxDiskCost
|
||||
if !estimateResult.HasRawData {
|
||||
if !estimateResult.HasRawData && !isVectorType {
|
||||
shouldCalculateDataSize = true
|
||||
}
|
||||
if !estimateResult.HasRawData && isVectorType {
|
||||
mmapChunkCache := paramtable.Get().QueryNodeCfg.MmapChunkCache.GetAsBool()
|
||||
if mmapChunkCache {
|
||||
segmentDiskSize += binlogSize
|
||||
} else {
|
||||
segmentMemorySize += binlogSize
|
||||
}
|
||||
}
|
||||
} else {
|
||||
shouldCalculateDataSize = true
|
||||
}
|
||||
|
|
Loading…
Reference in New Issue