milvus-io
/
milvus
mirror of https://github.com/milvus-io/milvus.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

enhance: Use mmap.scalarIndex config for text index (#36400) 

issue: https://github.com/milvus-io/milvus/issues/35273

---------

Signed-off-by: bigsheeper <yihao.dai@zilliz.com>
pull/36463/head
yihao.dai 2024-09-24 12:21:13 +08:00 committed by GitHub
parent b3f2d3db6f
commit 8cda48a96a
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
5 changed files with 17 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
internal/core/src/common/type_c.h
View File

@ -100,7 +100,7 @@ typedef struct CMmapConfig {
uint64_t disk_limit;
uint64_t fix_file_size;
bool growing_enable_mmap;
bool enable_mmap;
bool scalar_index_enable_mmap;
} CMmapConfig;
typedef struct CTraceConfig {

2
internal/core/src/segcore/SegmentSealedImpl.cpp
View File

@ -2031,7 +2031,7 @@ SegmentSealedImpl::CreateTextIndex(FieldId field_id) {
const auto& field_meta = schema_->operator[](field_id);
auto& cfg = storage::MmapManager::GetInstance().GetMmapConfig();
std::unique_ptr<index::TextMatchIndex> index;
if (!cfg.GetEnableMmap()) {
if (!cfg.GetScalarIndexEnableMmap()) {
// build text index in ram.
index = std::make_unique<index::TextMatchIndex>(
std::numeric_limits<int64_t>::max(),

13
internal/core/src/storage/Types.h
View File

@ -125,7 +125,7 @@ struct MmapConfig {
uint64_t disk_limit;
uint64_t fix_file_size;
bool growing_enable_mmap;
bool enable_mmap;
bool scalar_index_enable_mmap;
bool
GetEnableGrowingMmap() const {
return growing_enable_mmap;
@ -135,12 +135,12 @@ struct MmapConfig {
this->growing_enable_mmap = flag;
}
bool
GetEnableMmap() const {
return enable_mmap;
GetScalarIndexEnableMmap() const {
return scalar_index_enable_mmap;
}
void
SetEnableMmap(bool flag) {
this->enable_mmap = flag;
SetScalarIndexEnableMmap(bool flag) {
this->scalar_index_enable_mmap = flag;
}
std::string
GetMmapPath() {
@ -154,7 +154,8 @@ struct MmapConfig {
<< ", disk_limit=" << disk_limit / (1024 * 1024) << "MB"
<< ", fix_file_size=" << fix_file_size / (1024 * 1024) << "MB"
<< ", growing_enable_mmap=" << std::boolalpha << growing_enable_mmap
<< ", enable_mmap=" << std::boolalpha << enable_mmap << "]";
<< ", scalar_index_enable_mmap=" << std::boolalpha
<< scalar_index_enable_mmap << "]";
return ss.str();
}
};

3
internal/core/src/storage/storage_c.cpp
View File

@ -95,7 +95,8 @@ InitMmapManager(CMmapConfig c_mmap_config) {
mmap_config.disk_limit = c_mmap_config.disk_limit;
mmap_config.fix_file_size = c_mmap_config.fix_file_size;
mmap_config.growing_enable_mmap = c_mmap_config.growing_enable_mmap;
mmap_config.enable_mmap = c_mmap_config.enable_mmap;
mmap_config.scalar_index_enable_mmap =
c_mmap_config.scalar_index_enable_mmap;
milvus::storage::MmapManager::GetInstance().Init(mmap_config);
return milvus::SuccessCStatus();
} catch (std::exception& e) {

12
internal/util/initcore/init_core.go
View File

@ -181,12 +181,12 @@ func InitMmapManager(params *paramtable.ComponentParam) error {
diskLimit := uint64(float64(params.QueryNodeCfg.MaxMmapDiskPercentageForMmapManager.GetAsUint64()*diskCapacity) * 0.01)
mmapFileSize := params.QueryNodeCfg.FixedFileSizeForMmapManager.GetAsFloat() * 1024 * 1024
mmapConfig := C.CMmapConfig{
cache_read_ahead_policy: cCacheReadAheadPolicy,
mmap_path: cMmapChunkManagerDir,
disk_limit: C.uint64_t(diskLimit),
fix_file_size: C.uint64_t(mmapFileSize),
growing_enable_mmap: C.bool(params.QueryNodeCfg.GrowingMmapEnabled.GetAsBool()),
enable_mmap: C.bool(params.QueryNodeCfg.MmapEnabled.GetAsBool()),
cache_read_ahead_policy: cCacheReadAheadPolicy,
mmap_path: cMmapChunkManagerDir,
disk_limit: C.uint64_t(diskLimit),
fix_file_size: C.uint64_t(mmapFileSize),
growing_enable_mmap: C.bool(params.QueryNodeCfg.GrowingMmapEnabled.GetAsBool()),
scalar_index_enable_mmap: C.bool(params.QueryNodeCfg.MmapScalarIndex.GetAsBool()),
}
status := C.InitMmapManager(mmapConfig)
return HandleCStatus(&status, "InitMmapManager failed")