enhance: consider the mmap chunck cache config when resource usage estimate (#36814)

- issue: #36530

Signed-off-by: SimFG <bang.fu@zilliz.com>
pull/37064/head
SimFG 2024-10-18 10:17:23 +08:00 committed by GitHub
parent 3de57ec4fa
commit 903c18ba26
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
2 changed files with 17 additions and 10 deletions

View File

@ -173,11 +173,9 @@ IndexFactory::VecIndexLoadResource(
index_type, index_version, config);
break;
default:
PanicInfo(
milvus::DataTypeInvalid,
fmt::format(
"invalid data type to estimate index load resource: {}",
field_type));
LOG_ERROR("invalid data type to estimate index load resource: {}",
field_type);
return LoadResourceRequest{0, 0, 0, 0, true};
}
LoadResourceRequest request{};
@ -274,10 +272,10 @@ IndexFactory::ScalarIndexLoadResource(
request.max_disk_cost = index_size_gb;
request.has_raw_data = false;
} else {
PanicInfo(milvus::UnexpectedError,
fmt::format("invalid index type to estimate scalar index "
"load resource: {}",
index_type));
LOG_ERROR(
"invalid index type to estimate scalar index load resource: {}",
index_type);
return LoadResourceRequest{0, 0, 0, 0, true};
}
return request;
}

View File

@ -1434,6 +1434,7 @@ func getResourceUsageEstimateOfSegment(schema *schemapb.CollectionSchema, loadIn
return nil, err
}
binlogSize := uint64(getBinlogDataMemorySize(fieldBinlog))
isVectorType := typeutil.IsVectorType(fieldSchema.DataType)
shouldCalculateDataSize := false
if fieldIndexInfo, ok := fieldID2IndexInfo[fieldID]; ok {
@ -1452,9 +1453,17 @@ func getResourceUsageEstimateOfSegment(schema *schemapb.CollectionSchema, loadIn
indexMemorySize += estimateResult.MaxMemoryCost
segmentDiskSize += estimateResult.MaxDiskCost
if !estimateResult.HasRawData {
if !estimateResult.HasRawData && !isVectorType {
shouldCalculateDataSize = true
}
if !estimateResult.HasRawData && isVectorType {
mmapChunkCache := paramtable.Get().QueryNodeCfg.MmapChunkCache.GetAsBool()
if mmapChunkCache {
segmentDiskSize += binlogSize
} else {
segmentMemorySize += binlogSize
}
}
} else {
shouldCalculateDataSize = true
}