Influxdata
/
influxdb
mirror of https://github.com/influxdata/influxdb.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

chore: add more info about memory budget at the time of over-file-limit into skipped_compaction for us to see if we shoudl increase the file limit (#5731) 

Co-authored-by: kodiakhq[bot] <49736102+kodiakhq[bot]@users.noreply.github.com>
pull/24376/head
Nga Tran 2022-09-23 09:34:38 -04:00 committed by GitHub
parent 5b5a645ea9
commit da697815ff
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 16 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
compactor/src/lib.rs
View File

@ -153,7 +153,10 @@ async fn compact_candidates_with_memory_budget<C, Fut>(
FilterResult::NothingToCompact => {
debug!(?partition_id, compaction_type, "nothing to compact");
}
FilterResult::OverLimitFileNum { file_num } => {
FilterResult::OverLimitFileNum {
file_num,
budget_bytes,
} => {
// We cannot compact this partition because its first set of overlapped files
// are over the limit of file num
warn!(
@ -161,7 +164,9 @@ async fn compact_candidates_with_memory_budget<C, Fut>(
?table_id,
compaction_type,
file_num,
budget_bytes,
file_num_limit = compactor.config.max_num_compacting_files,
memory_budget_bytes = compactor.config.memory_budget_bytes,
"skipped; over limit of number of files"
);
let reason = format!(

12
compactor/src/parquet_file_filtering.rs
View File

@ -21,6 +21,7 @@ pub(crate) enum FilterResult {
NothingToCompact,
OverLimitFileNum {
file_num: usize,
budget_bytes: u64,
},
OverBudget {
budget_bytes: u64,
@ -146,6 +147,7 @@ fn filter_parquet_files_inner(
// exceed the file limit
return FilterResult::OverLimitFileNum {
file_num: 1 + overlaps.len(),
budget_bytes: estimated_file_bytes,
};
} else {
// Only compact files that are under limit number of files
@ -494,7 +496,10 @@ mod tests {
assert_eq!(
filter_result,
FilterResult::OverLimitFileNum { file_num: 1 }
FilterResult::OverLimitFileNum {
file_num: 1,
budget_bytes: 1176
}
);
}
@ -526,7 +531,10 @@ mod tests {
assert_eq!(
filter_result,
FilterResult::OverLimitFileNum { file_num: 2 }
FilterResult::OverLimitFileNum {
file_num: 2,
budget_bytes: 2 * 1176
}
);
}