Influxdata
/
influxdb
mirror of https://github.com/influxdata/influxdb.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

fix: compaction looping fixes (#8363) 

* fix: selectively merge L1 to L2 when L0s still exist

* fix: avoid grouping files that undo previous splits

* chore: add test case for new fixes

* chore: insta test churn

* chore: lint cleanup
pull/24376/head
Joe-Blount 2023-07-31 08:15:49 -05:00 committed by GitHub
parent aa7a38be55
commit 44e266d000
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 1149 additions and 319 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

58
compactor/src/components/round_info_source/mod.rs
View File

@ -172,7 +172,11 @@ impl RoundInfoSource for LevelBasedRoundInfo {
_partition_info: &PartitionInfo,
files: &[ParquetFile],
) -> Result<RoundInfo, DynError> {
let start_level = get_start_level(files);
let start_level = get_start_level(
files,
self.max_num_files_per_plan,
self.max_total_file_size_per_plan,
);
if self.too_many_small_files_to_compact(files, start_level) {
return Ok(RoundInfo::ManySmallFiles {
@ -187,23 +191,53 @@ impl RoundInfoSource for LevelBasedRoundInfo {
}
}
fn get_start_level(files: &[ParquetFile]) -> CompactionLevel {
// get_start_level decides what level to start compaction from. Often this is the lowest level
// we have ParquetFiles in, but occasionally we decide to compact L1->L2 when L0s still exist.
//
// If we ignore the invariants (where intra-level overlaps are allowed), this would be a math problem
// to optimize write amplification.
//
// However, allowing intra-level overlaps in L0 but not L1/L2 adds extra challenge to compacting L0s to L1.
// This is especially true when there are large quantitites of overlapping L0s and L1s, potentially resulting
// in many split/compact cycles to resolve the overlaps.
//
// Since L1 & L2 only have inter-level overlaps, they can be compacted with just a few splits to align the L1s
// with the L2s. The relative ease of moving data from L1 to L2 provides additional motivation to compact the
// L1s to L2s when a backlog of L0s exist. The easily solvable L1->L2 compaction can give us a clean slate in
// L1, greatly simplifying the remaining L0->L1 compactions.
fn get_start_level(files: &[ParquetFile], max_files: usize, max_bytes: usize) -> CompactionLevel {
// panic if the files are empty
assert!(!files.is_empty());
// Start with initial level
// If there are files in this level, itis the start level
// Otherwise repeat until reaching the final level.
let mut level = CompactionLevel::Initial;
while level != CompactionLevel::Final {
if files.iter().any(|f| f.compaction_level == level) {
return level;
}
let mut l0_cnt: usize = 0;
let mut l0_bytes: usize = 0;
let mut l1_bytes: usize = 0;
level = level.next();
for f in files {
match f.compaction_level {
CompactionLevel::Initial => {
l0_cnt += 1;
l0_bytes += f.file_size_bytes as usize;
}
CompactionLevel::FileNonOverlapped => {
l1_bytes += f.file_size_bytes as usize;
}
_ => {}
}
}
level
if l1_bytes > 3 * max_bytes && (l0_cnt > max_files || l0_bytes > max_bytes) {
// L1 is big enough to pose an overlap challenge compacting from L0, and there is quite a bit more coming from L0.
// The criteria for this early L1->L2 compaction significanly impacts write amplification. The above values optimize
// existing test cases, but may be changed as additional test cases are added.
CompactionLevel::FileNonOverlapped
} else if l0_bytes > 0 {
CompactionLevel::Initial
} else if l1_bytes > 0 {
CompactionLevel::FileNonOverlapped
} else {
CompactionLevel::Final
}
}
fn get_num_overlapped_files(

21
compactor/src/components/split_or_compact/start_level_files_to_split.rs
View File

@ -301,7 +301,26 @@ pub fn merge_small_l0_chains(
for chain in &chains {
let this_chain_bytes = chain.iter().map(|f| f.file_size_bytes as usize).sum();
if prior_chain_bytes > 0 && prior_chain_bytes + this_chain_bytes <= max_compact_size {
// matching max_lo_created_at times indicates that the files were deliberately split. We shouldn't merge
// chains with matching max_lo_created_at times, because that would encourage undoing the previous split,
// which minimally increases write amplification, and may cause unproductive split/compact loops.
let mut matches = 0;
if prior_chain_bytes > 0 {
for f in chain.iter() {
for f2 in &merged_chains[prior_chain_idx as usize] {
if f.max_l0_created_at == f2.max_l0_created_at {
matches += 1;
break;
}
}
}
}
// Merge it if: there a prior chain to merge with, and merging wouldn't make it too big, or undo a previous split
if prior_chain_bytes > 0
&& prior_chain_bytes + this_chain_bytes <= max_compact_size
&& matches == 0
{
// this chain can be added to the prior chain.
merged_chains[prior_chain_idx as usize].append(&mut chain.clone());
prior_chain_bytes += this_chain_bytes;

2
compactor/tests/integration.rs
View File

@ -68,8 +68,8 @@ async fn test_num_files_over_limit() {
assert_levels(
&files,
vec![
(8, CompactionLevel::FileNonOverlapped),
(9, CompactionLevel::FileNonOverlapped),
(10, CompactionLevel::FileNonOverlapped),
],
);
}

365
compactor/tests/layouts/backfill.rs
View File

@ -746,97 +746,85 @@ async fn random_backfill_over_l2s() {
- "Committing partition 1:"
- " Soft Deleting 4 files: L0.76, L0.77, L0.79, L0.80"
- " Creating 8 files"
- "**** Simulation run 15, type=compact(ManySmallFiles). 10 Input Files, 200mb total:"
- "L0 "
- "L0.75[42,356] 1.04us 33mb|-----------L0.75-----------| "
- "L0.86[357,357] 1.04us 0b |L0.86| "
- "L0.87[358,670] 1.04us 33mb |-----------L0.87-----------| "
- "L0.84[671,672] 1.04us 109kb |L0.84| "
- "L0.85[673,986] 1.04us 33mb |-----------L0.85-----------| "
- "L0.78[42,356] 1.05us 33mb|-----------L0.78-----------| "
- "L0.90[357,357] 1.05us 0b |L0.90| "
- "L0.91[358,670] 1.05us 33mb |-----------L0.91-----------| "
- "L0.88[671,672] 1.05us 109kb |L0.88| "
- "L0.89[673,986] 1.05us 33mb |-----------L0.89-----------| "
- "**** 1 Output Files (parquet_file_id not yet assigned), 200mb total:"
- "L0, all files 200mb "
- "L0.?[42,986] 1.05us |------------------------------------------L0.?------------------------------------------|"
- "Committing partition 1:"
- " Soft Deleting 10 files: L0.75, L0.78, L0.84, L0.85, L0.86, L0.87, L0.88, L0.89, L0.90, L0.91"
- " Creating 1 files"
- "**** Simulation run 16, type=split(HighL0OverlapSingleFile)(split_times=[670]). 1 Input Files, 100mb total:"
- "L1, all files 100mb "
- "L1.82[358,672] 1.03us |-----------------------------------------L1.82------------------------------------------|"
- "**** 2 Output Files (parquet_file_id not yet assigned), 100mb total:"
- "L1 "
- "L1.?[358,670] 1.03us 99mb|-----------------------------------------L1.?------------------------------------------| "
- "L1.?[671,672] 1.03us 651kb |L1.?|"
- "**** Simulation run 17, type=split(HighL0OverlapSingleFile)(split_times=[356]). 1 Input Files, 100mb total:"
- "L1, all files 100mb "
- "L1.81[42,357] 1.03us |-----------------------------------------L1.81------------------------------------------|"
- "**** 2 Output Files (parquet_file_id not yet assigned), 100mb total:"
- "L1 "
- "L1.?[42,356] 1.03us 100mb|-----------------------------------------L1.?------------------------------------------| "
- "L1.?[357,357] 1.03us 325kb |L1.?|"
- "**** Simulation run 18, type=split(HighL0OverlapSingleFile)(split_times=[356, 670]). 1 Input Files, 200mb total:"
- "L0, all files 200mb "
- "L0.92[42,986] 1.05us |-----------------------------------------L0.92------------------------------------------|"
- "**** 3 Output Files (parquet_file_id not yet assigned), 200mb total:"
- "L0 "
- "L0.?[42,356] 1.05us 67mb |-----------L0.?------------| "
- "L0.?[357,670] 1.05us 66mb |-----------L0.?------------| "
- "L0.?[671,986] 1.05us 67mb |------------L0.?------------| "
- "Committing partition 1:"
- " Soft Deleting 3 files: L1.81, L1.82, L0.92"
- " Creating 7 files"
- "**** Simulation run 19, type=split(ReduceOverlap)(split_times=[672]). 1 Input Files, 67mb total:"
- "**** Simulation run 15, type=compact(ManySmallFiles). 2 Input Files, 67mb total:"
- "L0, all files 33mb "
- "L0.75[42,356] 1.04us |-----------------------------------------L0.75------------------------------------------|"
- "L0.78[42,356] 1.05us |-----------------------------------------L0.78------------------------------------------|"
- "**** 1 Output Files (parquet_file_id not yet assigned), 67mb total:"
- "L0, all files 67mb "
- "L0.99[671,986] 1.05us |-----------------------------------------L0.99------------------------------------------|"
- "**** 2 Output Files (parquet_file_id not yet assigned), 67mb total:"
- "L0 "
- "L0.?[671,672] 1.05us 218kb|L0.?| "
- "L0.?[673,986] 1.05us 67mb|-----------------------------------------L0.?------------------------------------------| "
- "**** Simulation run 20, type=split(ReduceOverlap)(split_times=[357]). 1 Input Files, 66mb total:"
- "L0, all files 66mb "
- "L0.98[357,670] 1.05us |-----------------------------------------L0.98------------------------------------------|"
- "**** 2 Output Files (parquet_file_id not yet assigned), 66mb total:"
- "L0 "
- "L0.?[357,357] 1.05us 0b |L0.?| "
- "L0.?[358,670] 1.05us 66mb|-----------------------------------------L0.?------------------------------------------| "
- "L0.?[42,356] 1.05us |------------------------------------------L0.?------------------------------------------|"
- "Committing partition 1:"
- " Soft Deleting 2 files: L0.98, L0.99"
- " Creating 4 files"
- "**** Simulation run 21, type=split(CompactAndSplitOutput(FoundSubsetLessThanMaxCompactSize))(split_times=[232]). 4 Input Files, 167mb total:"
- " Soft Deleting 2 files: L0.75, L0.78"
- " Creating 1 files"
- "**** Simulation run 16, type=compact(ManySmallFiles). 2 Input Files, 66mb total:"
- "L0, all files 33mb "
- "L0.87[358,670] 1.04us |-----------------------------------------L0.87------------------------------------------|"
- "L0.91[358,670] 1.05us |-----------------------------------------L0.91------------------------------------------|"
- "**** 1 Output Files (parquet_file_id not yet assigned), 66mb total:"
- "L0, all files 66mb "
- "L0.?[358,670] 1.05us |------------------------------------------L0.?------------------------------------------|"
- "Committing partition 1:"
- " Soft Deleting 2 files: L0.87, L0.91"
- " Creating 1 files"
- "**** Simulation run 17, type=compact(ManySmallFiles). 2 Input Files, 218kb total:"
- "L0, all files 109kb "
- "L0.84[671,672] 1.04us |-----------------------------------------L0.84------------------------------------------|"
- "L0.88[671,672] 1.05us |-----------------------------------------L0.88------------------------------------------|"
- "**** 1 Output Files (parquet_file_id not yet assigned), 218kb total:"
- "L0, all files 218kb "
- "L0.?[671,672] 1.05us |------------------------------------------L0.?------------------------------------------|"
- "Committing partition 1:"
- " Soft Deleting 2 files: L0.84, L0.88"
- " Creating 1 files"
- "**** Simulation run 18, type=compact(ManySmallFiles). 2 Input Files, 67mb total:"
- "L0, all files 33mb "
- "L0.85[673,986] 1.04us |-----------------------------------------L0.85------------------------------------------|"
- "L0.89[673,986] 1.05us |-----------------------------------------L0.89------------------------------------------|"
- "**** 1 Output Files (parquet_file_id not yet assigned), 67mb total:"
- "L0, all files 67mb "
- "L0.?[673,986] 1.05us |------------------------------------------L0.?------------------------------------------|"
- "Committing partition 1:"
- " Soft Deleting 2 files: L0.85, L0.89"
- " Creating 1 files"
- "**** Simulation run 19, type=compact(ManySmallFiles). 2 Input Files, 0b total:"
- "L0, all files 0b "
- "L0.86[357,357] 1.04us |-----------------------------------------L0.86------------------------------------------|"
- "L0.90[357,357] 1.05us |-----------------------------------------L0.90------------------------------------------|"
- "**** 1 Output Files (parquet_file_id not yet assigned), 0b total:"
- "L0, all files 0b "
- "L0.?[357,357] 1.05us |------------------------------------------L0.?------------------------------------------|"
- "Committing partition 1:"
- " Soft Deleting 2 files: L0.86, L0.90"
- " Creating 1 files"
- "**** Simulation run 20, type=split(CompactAndSplitOutput(FoundSubsetLessThanMaxCompactSize))(split_times=[232]). 3 Input Files, 167mb total:"
- "L0 "
- "L0.97[42,356] 1.05us 67mb|-----------------------------------------L0.97-----------------------------------------| "
- "L0.102[357,357] 1.05us 0b |L0.102|"
- "L0.92[42,356] 1.05us 67mb|-----------------------------------------L0.92-----------------------------------------| "
- "L0.96[357,357] 1.05us 0b |L0.96|"
- "L1 "
- "L1.95[42,356] 1.03us 100mb|-----------------------------------------L1.95-----------------------------------------| "
- "L1.96[357,357] 1.03us 325kb |L1.96|"
- "L1.81[42,357] 1.03us 100mb|-----------------------------------------L1.81------------------------------------------|"
- "**** 2 Output Files (parquet_file_id not yet assigned), 167mb total:"
- "L1 "
- "L1.?[42,232] 1.05us 101mb|------------------------L1.?------------------------| "
- "L1.?[233,357] 1.05us 66mb |--------------L1.?---------------| "
- "Committing partition 1:"
- " Soft Deleting 4 files: L1.95, L1.96, L0.97, L0.102"
- " Soft Deleting 3 files: L1.81, L0.92, L0.96"
- " Creating 2 files"
- "**** Simulation run 22, type=split(CompactAndSplitOutput(FoundSubsetLessThanMaxCompactSize))(split_times=[547]). 4 Input Files, 166mb total:"
- "**** Simulation run 21, type=split(CompactAndSplitOutput(FoundSubsetLessThanMaxCompactSize))(split_times=[547]). 3 Input Files, 166mb total:"
- "L0 "
- "L0.103[358,670] 1.05us 66mb|----------------------------------------L0.103-----------------------------------------| "
- "L0.100[671,672] 1.05us 218kb |L0.100|"
- "L0.93[358,670] 1.05us 66mb|-----------------------------------------L0.93-----------------------------------------| "
- "L0.94[671,672] 1.05us 218kb |L0.94|"
- "L1 "
- "L1.93[358,670] 1.03us 99mb|-----------------------------------------L1.93-----------------------------------------| "
- "L1.94[671,672] 1.03us 651kb |L1.94|"
- "L1.82[358,672] 1.03us 100mb|-----------------------------------------L1.82------------------------------------------|"
- "**** 2 Output Files (parquet_file_id not yet assigned), 166mb total:"
- "L1 "
- "L1.?[358,547] 1.05us 100mb|------------------------L1.?------------------------| "
- "L1.?[548,672] 1.05us 66mb |--------------L1.?---------------| "
- "Committing partition 1:"
- " Soft Deleting 4 files: L1.93, L1.94, L0.100, L0.103"
- " Soft Deleting 3 files: L1.82, L0.93, L0.94"
- " Creating 2 files"
- "**** Simulation run 23, type=split(CompactAndSplitOutput(TotalSizeLessThanMaxCompactSize))(split_times=[861]). 2 Input Files, 167mb total:"
- "**** Simulation run 22, type=split(CompactAndSplitOutput(TotalSizeLessThanMaxCompactSize))(split_times=[861]). 2 Input Files, 167mb total:"
- "L0 "
- "L0.101[673,986] 1.05us 67mb|-----------------------------------------L0.101-----------------------------------------|"
- "L0.95[673,986] 1.05us 67mb|-----------------------------------------L0.95------------------------------------------|"
- "L1 "
- "L1.83[673,986] 1.03us 100mb|-----------------------------------------L1.83------------------------------------------|"
- "**** 2 Output Files (parquet_file_id not yet assigned), 167mb total:"
@ -844,60 +832,60 @@ async fn random_backfill_over_l2s() {
- "L1.?[673,861] 1.05us 100mb|------------------------L1.?------------------------| "
- "L1.?[862,986] 1.05us 67mb |--------------L1.?---------------| "
- "Committing partition 1:"
- " Soft Deleting 2 files: L1.83, L0.101"
- " Soft Deleting 2 files: L1.83, L0.95"
- " Creating 2 files"
- "**** Simulation run 24, type=split(ReduceOverlap)(split_times=[399, 499]). 1 Input Files, 100mb total:"
- "**** Simulation run 23, type=split(ReduceOverlap)(split_times=[399, 499]). 1 Input Files, 100mb total:"
- "L1, all files 100mb "
- "L1.106[358,547] 1.05us |-----------------------------------------L1.106-----------------------------------------|"
- "L1.99[358,547] 1.05us |-----------------------------------------L1.99------------------------------------------|"
- "**** 3 Output Files (parquet_file_id not yet assigned), 100mb total:"
- "L1 "
- "L1.?[358,399] 1.05us 22mb|------L1.?-------| "
- "L1.?[400,499] 1.05us 52mb |--------------------L1.?---------------------| "
- "L1.?[500,547] 1.05us 26mb |--------L1.?--------| "
- "**** Simulation run 25, type=split(ReduceOverlap)(split_times=[299]). 1 Input Files, 66mb total:"
- "**** Simulation run 24, type=split(ReduceOverlap)(split_times=[299]). 1 Input Files, 66mb total:"
- "L1, all files 66mb "
- "L1.105[233,357] 1.05us |-----------------------------------------L1.105-----------------------------------------|"
- "L1.98[233,357] 1.05us |-----------------------------------------L1.98------------------------------------------|"
- "**** 2 Output Files (parquet_file_id not yet assigned), 66mb total:"
- "L1 "
- "L1.?[233,299] 1.05us 35mb|--------------------L1.?---------------------| "
- "L1.?[300,357] 1.05us 31mb |-----------------L1.?------------------| "
- "**** Simulation run 26, type=split(ReduceOverlap)(split_times=[99, 199]). 1 Input Files, 101mb total:"
- "**** Simulation run 25, type=split(ReduceOverlap)(split_times=[99, 199]). 1 Input Files, 101mb total:"
- "L1, all files 101mb "
- "L1.104[42,232] 1.05us |-----------------------------------------L1.104-----------------------------------------|"
- "L1.97[42,232] 1.05us |-----------------------------------------L1.97------------------------------------------|"
- "**** 3 Output Files (parquet_file_id not yet assigned), 101mb total:"
- "L1 "
- "L1.?[42,99] 1.05us 30mb |----------L1.?-----------| "
- "L1.?[100,199] 1.05us 52mb |--------------------L1.?--------------------| "
- "L1.?[200,232] 1.05us 18mb |----L1.?-----| "
- "**** Simulation run 27, type=split(ReduceOverlap)(split_times=[599]). 1 Input Files, 66mb total:"
- "**** Simulation run 26, type=split(ReduceOverlap)(split_times=[599]). 1 Input Files, 66mb total:"
- "L1, all files 66mb "
- "L1.107[548,672] 1.05us |-----------------------------------------L1.107-----------------------------------------|"
- "L1.100[548,672] 1.05us |-----------------------------------------L1.100-----------------------------------------|"
- "**** 2 Output Files (parquet_file_id not yet assigned), 66mb total:"
- "L1 "
- "L1.?[548,599] 1.05us 27mb|---------------L1.?----------------| "
- "L1.?[600,672] 1.05us 39mb |-----------------------L1.?-----------------------| "
- "**** Simulation run 28, type=split(ReduceOverlap)(split_times=[899]). 1 Input Files, 67mb total:"
- "**** Simulation run 27, type=split(ReduceOverlap)(split_times=[899]). 1 Input Files, 67mb total:"
- "L1, all files 67mb "
- "L1.109[862,986] 1.05us |-----------------------------------------L1.109-----------------------------------------|"
- "L1.102[862,986] 1.05us |-----------------------------------------L1.102-----------------------------------------|"
- "**** 2 Output Files (parquet_file_id not yet assigned), 67mb total:"
- "L1 "
- "L1.?[862,899] 1.05us 20mb|----------L1.?----------| "
- "L1.?[900,986] 1.05us 47mb |----------------------------L1.?----------------------------| "
- "**** Simulation run 29, type=split(ReduceOverlap)(split_times=[699, 799]). 1 Input Files, 100mb total:"
- "**** Simulation run 28, type=split(ReduceOverlap)(split_times=[699, 799]). 1 Input Files, 100mb total:"
- "L1, all files 100mb "
- "L1.108[673,861] 1.05us |-----------------------------------------L1.108-----------------------------------------|"
- "L1.101[673,861] 1.05us |-----------------------------------------L1.101-----------------------------------------|"
- "**** 3 Output Files (parquet_file_id not yet assigned), 100mb total:"
- "L1 "
- "L1.?[673,699] 1.05us 14mb|---L1.?---| "
- "L1.?[700,799] 1.05us 53mb |--------------------L1.?---------------------| "
- "L1.?[800,861] 1.05us 34mb |-----------L1.?------------| "
- "Committing partition 1:"
- " Soft Deleting 6 files: L1.104, L1.105, L1.106, L1.107, L1.108, L1.109"
- " Soft Deleting 6 files: L1.97, L1.98, L1.99, L1.100, L1.101, L1.102"
- " Creating 15 files"
- "**** Simulation run 30, type=split(CompactAndSplitOutput(FoundSubsetLessThanMaxCompactSize))(split_times=[71, 142]). 4 Input Files, 283mb total:"
- "**** Simulation run 29, type=split(CompactAndSplitOutput(FoundSubsetLessThanMaxCompactSize))(split_times=[71, 142]). 4 Input Files, 283mb total:"
- "L1 "
- "L1.115[42,99] 1.05us 30mb |--------L1.115---------| "
- "L1.116[100,199] 1.05us 52mb |------------------L1.116------------------| "
- "L1.108[42,99] 1.05us 30mb |--------L1.108---------| "
- "L1.109[100,199] 1.05us 52mb |------------------L1.109------------------| "
- "L2 "
- "L2.1[0,99] 99ns 100mb |-------------------L2.1-------------------| "
- "L2.2[100,199] 199ns 100mb |-------------------L2.2-------------------| "
@ -907,13 +895,13 @@ async fn random_backfill_over_l2s() {
- "L2.?[72,142] 1.05us 99mb |------------L2.?-------------| "
- "L2.?[143,199] 1.05us 82mb |---------L2.?----------| "
- "Committing partition 1:"
- " Soft Deleting 4 files: L2.1, L2.2, L1.115, L1.116"
- " Soft Deleting 4 files: L2.1, L2.2, L1.108, L1.109"
- " Creating 3 files"
- "**** Simulation run 31, type=split(CompactAndSplitOutput(FoundSubsetLessThanMaxCompactSize))(split_times=[271, 342]). 5 Input Files, 284mb total:"
- "**** Simulation run 30, type=split(CompactAndSplitOutput(FoundSubsetLessThanMaxCompactSize))(split_times=[271, 342]). 5 Input Files, 284mb total:"
- "L1 "
- "L1.117[200,232] 1.05us 18mb|---L1.117---| "
- "L1.113[233,299] 1.05us 35mb |----------L1.113-----------| "
- "L1.114[300,357] 1.05us 31mb |--------L1.114---------| "
- "L1.110[200,232] 1.05us 18mb|---L1.110---| "
- "L1.106[233,299] 1.05us 35mb |----------L1.106-----------| "
- "L1.107[300,357] 1.05us 31mb |--------L1.107---------| "
- "L2 "
- "L2.3[200,299] 299ns 100mb|-------------------L2.3-------------------| "
- "L2.4[300,399] 399ns 100mb |-------------------L2.4-------------------| "
@ -923,14 +911,14 @@ async fn random_backfill_over_l2s() {
- "L2.?[272,342] 1.05us 100mb |------------L2.?-------------| "
- "L2.?[343,399] 1.05us 83mb |---------L2.?----------| "
- "Committing partition 1:"
- " Soft Deleting 5 files: L2.3, L2.4, L1.113, L1.114, L1.117"
- " Soft Deleting 5 files: L2.3, L2.4, L1.106, L1.107, L1.110"
- " Creating 3 files"
- "**** Simulation run 32, type=split(CompactAndSplitOutput(FoundSubsetLessThanMaxCompactSize))(split_times=[404, 465]). 4 Input Files, 257mb total:"
- "**** Simulation run 31, type=split(CompactAndSplitOutput(FoundSubsetLessThanMaxCompactSize))(split_times=[404, 465]). 4 Input Files, 257mb total:"
- "L1 "
- "L1.110[358,399] 1.05us 22mb |-------L1.110--------| "
- "L1.111[400,499] 1.05us 52mb |------------------------L1.111-------------------------| "
- "L1.103[358,399] 1.05us 22mb |-------L1.103--------| "
- "L1.104[400,499] 1.05us 52mb |------------------------L1.104-------------------------| "
- "L2 "
- "L2.130[343,399] 1.05us 83mb|------------L2.130------------| "
- "L2.123[343,399] 1.05us 83mb|------------L2.123------------| "
- "L2.5[400,499] 499ns 100mb |-------------------------L2.5--------------------------| "
- "**** 3 Output Files (parquet_file_id not yet assigned), 257mb total:"
- "L2 "
@ -938,13 +926,13 @@ async fn random_backfill_over_l2s() {
- "L2.?[405,465] 1.05us 99mb |--------------L2.?--------------| "
- "L2.?[466,499] 1.05us 58mb |------L2.?-------| "
- "Committing partition 1:"
- " Soft Deleting 4 files: L2.5, L1.110, L1.111, L2.130"
- " Soft Deleting 4 files: L2.5, L1.103, L1.104, L2.123"
- " Creating 3 files"
- "**** Simulation run 33, type=split(CompactAndSplitOutput(FoundSubsetLessThanMaxCompactSize))(split_times=[569, 638]). 5 Input Files, 292mb total:"
- "**** Simulation run 32, type=split(CompactAndSplitOutput(FoundSubsetLessThanMaxCompactSize))(split_times=[569, 638]). 5 Input Files, 292mb total:"
- "L1 "
- "L1.112[500,547] 1.05us 26mb|------L1.112-------| "
- "L1.118[548,599] 1.05us 27mb |-------L1.118--------| "
- "L1.119[600,672] 1.05us 39mb |------------L1.119------------| "
- "L1.105[500,547] 1.05us 26mb|------L1.105-------| "
- "L1.111[548,599] 1.05us 27mb |-------L1.111--------| "
- "L1.112[600,672] 1.05us 39mb |------------L1.112------------| "
- "L2 "
- "L2.6[500,599] 599ns 100mb|-------------------L2.6-------------------| "
- "L2.7[600,699] 699ns 100mb |-------------------L2.7-------------------| "
@ -954,14 +942,14 @@ async fn random_backfill_over_l2s() {
- "L2.?[570,638] 1.05us 100mb |------------L2.?------------| "
- "L2.?[639,699] 1.05us 91mb |----------L2.?-----------| "
- "Committing partition 1:"
- " Soft Deleting 5 files: L2.6, L2.7, L1.112, L1.118, L1.119"
- " Soft Deleting 5 files: L2.6, L2.7, L1.105, L1.111, L1.112"
- " Creating 3 files"
- "**** Simulation run 34, type=split(CompactAndSplitOutput(FoundSubsetLessThanMaxCompactSize))(split_times=[702, 765]). 4 Input Files, 258mb total:"
- "**** Simulation run 33, type=split(CompactAndSplitOutput(FoundSubsetLessThanMaxCompactSize))(split_times=[702, 765]). 4 Input Files, 258mb total:"
- "L1 "
- "L1.122[673,699] 1.05us 14mb |---L1.122---| "
- "L1.123[700,799] 1.05us 53mb |-----------------------L1.123------------------------| "
- "L1.115[673,699] 1.05us 14mb |---L1.115---| "
- "L1.116[700,799] 1.05us 53mb |-----------------------L1.116------------------------| "
- "L2 "
- "L2.136[639,699] 1.05us 91mb|------------L2.136-------------| "
- "L2.129[639,699] 1.05us 91mb|------------L2.129-------------| "
- "L2.8[700,799] 799ns 100mb |------------------------L2.8-------------------------| "
- "**** 3 Output Files (parquet_file_id not yet assigned), 258mb total:"
- "L2 "
@ -969,12 +957,12 @@ async fn random_backfill_over_l2s() {
- "L2.?[703,765] 1.05us 100mb |--------------L2.?--------------| "
- "L2.?[766,799] 1.05us 56mb |------L2.?------| "
- "Committing partition 1:"
- " Soft Deleting 4 files: L2.8, L1.122, L1.123, L2.136"
- " Soft Deleting 4 files: L2.8, L1.115, L1.116, L2.129"
- " Creating 3 files"
- "**** Simulation run 35, type=split(CompactAndSplitOutput(FoundSubsetLessThanMaxCompactSize))(split_times=[865]). 3 Input Files, 154mb total:"
- "**** Simulation run 34, type=split(CompactAndSplitOutput(FoundSubsetLessThanMaxCompactSize))(split_times=[865]). 3 Input Files, 154mb total:"
- "L1 "
- "L1.124[800,861] 1.05us 34mb|-----------------------L1.124------------------------| "
- "L1.120[862,899] 1.05us 20mb |------------L1.120-------------| "
- "L1.117[800,861] 1.05us 34mb|-----------------------L1.117------------------------| "
- "L1.113[862,899] 1.05us 20mb |------------L1.113-------------| "
- "L2 "
- "L2.9[800,899] 899ns 100mb|-----------------------------------------L2.9------------------------------------------| "
- "**** 2 Output Files (parquet_file_id not yet assigned), 154mb total:"
@ -982,28 +970,28 @@ async fn random_backfill_over_l2s() {
- "L2.?[800,865] 1.05us 101mb|--------------------------L2.?---------------------------| "
- "L2.?[866,899] 1.05us 53mb |-----------L2.?------------| "
- "Committing partition 1:"
- " Soft Deleting 3 files: L2.9, L1.120, L1.124"
- " Soft Deleting 3 files: L2.9, L1.113, L1.117"
- " Creating 2 files"
- "**** Final Output Files (4.58gb written)"
- "**** Final Output Files (4.06gb written)"
- "L1 "
- "L1.121[900,986] 1.05us 47mb |L1.121| "
- "L1.114[900,986] 1.05us 47mb |L1.114| "
- "L2 "
- "L2.10[900,999] 999ns 100mb |L2.10-| "
- "L2.125[0,71] 1.05us 101mb|L2.125| "
- "L2.126[72,142] 1.05us 99mb |L2.126| "
- "L2.127[143,199] 1.05us 82mb |L2.127| "
- "L2.128[200,271] 1.05us 101mb |L2.128| "
- "L2.129[272,342] 1.05us 100mb |L2.129| "
- "L2.131[343,404] 1.05us 100mb |L2.131| "
- "L2.132[405,465] 1.05us 99mb |L2.132| "
- "L2.133[466,499] 1.05us 58mb |L2.133| "
- "L2.134[500,569] 1.05us 101mb |L2.134| "
- "L2.135[570,638] 1.05us 100mb |L2.135| "
- "L2.137[639,702] 1.05us 101mb |L2.137| "
- "L2.138[703,765] 1.05us 100mb |L2.138| "
- "L2.139[766,799] 1.05us 56mb |L2.139| "
- "L2.140[800,865] 1.05us 101mb |L2.140| "
- "L2.141[866,899] 1.05us 53mb |L2.141| "
- "L2.118[0,71] 1.05us 101mb|L2.118| "
- "L2.119[72,142] 1.05us 99mb |L2.119| "
- "L2.120[143,199] 1.05us 82mb |L2.120| "
- "L2.121[200,271] 1.05us 101mb |L2.121| "
- "L2.122[272,342] 1.05us 100mb |L2.122| "
- "L2.124[343,404] 1.05us 100mb |L2.124| "
- "L2.125[405,465] 1.05us 99mb |L2.125| "
- "L2.126[466,499] 1.05us 58mb |L2.126| "
- "L2.127[500,569] 1.05us 101mb |L2.127| "
- "L2.128[570,638] 1.05us 100mb |L2.128| "
- "L2.130[639,702] 1.05us 101mb |L2.130| "
- "L2.131[703,765] 1.05us 100mb |L2.131| "
- "L2.132[766,799] 1.05us 56mb |L2.132| "
- "L2.133[800,865] 1.05us 101mb |L2.133| "
- "L2.134[866,899] 1.05us 53mb |L2.134| "
"###
);
}
@ -3020,63 +3008,66 @@ async fn actual_case_from_catalog_1() {
- "WARNING: file L0.161[327,333] 336ns 183mb exceeds soft limit 100mb by more than 50%"
- "WARNING: file L0.162[330,338] 340ns 231mb exceeds soft limit 100mb by more than 50%"
- "WARNING: file L0.163[331,338] 341ns 232mb exceeds soft limit 100mb by more than 50%"
- "**** Final Output Files (17.64gb written)"
- "**** Final Output Files (15.47gb written)"
- "L2 "
- "L2.578[134,149] 342ns 202mb |L2.578| "
- "L2.579[150,165] 342ns 218mb |L2.579| "
- "L2.580[166,176] 342ns 186mb |L2.580| "
- "L2.581[177,182] 342ns 150mb |L2.581| "
- "L2.582[183,197] 342ns 267mb |L2.582| "
- "L2.583[198,207] 342ns 157mb |L2.583| "
- "L2.584[208,220] 342ns 147mb |L2.584| "
- "L2.585[221,232] 342ns 270mb |L2.585| "
- "L2.588[233,253] 342ns 286mb |L2.588| "
- "L2.589[254,270] 342ns 289mb |L2.589| "
- "L2.590[271,281] 342ns 225mb |L2.590| "
- "L2.591[282,296] 342ns 234mb |L2.591| "
- "L2.592[297,302] 342ns 232mb |L2.592| "
- "L2.593[303,308] 342ns 244mb |L2.593| "
- "L2.594[309,314] 342ns 282mb |L2.594|"
- "L2.595[315,317] 342ns 214mb |L2.595|"
- "L2.596[318,320] 342ns 222mb |L2.596|"
- "L2.597[321,323] 342ns 146mb |L2.597|"
- "L2.598[324,326] 342ns 254mb |L2.598|"
- "L2.599[327,329] 342ns 197mb |L2.599|"
- "L2.600[330,332] 342ns 228mb |L2.600|"
- "L2.601[333,335] 342ns 199mb |L2.601|"
- "L2.602[336,338] 342ns 280mb |L2.602|"
- "L2.850[1,26] 342ns 101mb |L2.850| "
- "L2.853[69,85] 342ns 104mb |L2.853| "
- "L2.854[86,98] 342ns 107mb |L2.854| "
- "L2.861[27,48] 342ns 103mb |L2.861| "
- "L2.862[49,68] 342ns 98mb |L2.862| "
- "L2.863[99,108] 342ns 102mb |L2.863| "
- "L2.864[109,117] 342ns 91mb |L2.864| "
- "L2.865[118,124] 342ns 91mb |L2.865| "
- "L2.866[125,130] 342ns 107mb |L2.866| "
- "L2.867[131,133] 342ns 64mb |L2.867| "
- "L2.868[339,339] 342ns 25mb |L2.868|"
- "WARNING: file L2.578[134,149] 342ns 202mb exceeds soft limit 100mb by more than 50%"
- "WARNING: file L2.579[150,165] 342ns 218mb exceeds soft limit 100mb by more than 50%"
- "WARNING: file L2.580[166,176] 342ns 186mb exceeds soft limit 100mb by more than 50%"
- "WARNING: file L2.581[177,182] 342ns 150mb exceeds soft limit 100mb by more than 50%"
- "WARNING: file L2.582[183,197] 342ns 267mb exceeds soft limit 100mb by more than 50%"
- "WARNING: file L2.583[198,207] 342ns 157mb exceeds soft limit 100mb by more than 50%"
- "WARNING: file L2.585[221,232] 342ns 270mb exceeds soft limit 100mb by more than 50%"
- "WARNING: file L2.588[233,253] 342ns 286mb exceeds soft limit 100mb by more than 50%"
- "WARNING: file L2.589[254,270] 342ns 289mb exceeds soft limit 100mb by more than 50%"
- "WARNING: file L2.590[271,281] 342ns 225mb exceeds soft limit 100mb by more than 50%"
- "WARNING: file L2.591[282,296] 342ns 234mb exceeds soft limit 100mb by more than 50%"
- "WARNING: file L2.592[297,302] 342ns 232mb exceeds soft limit 100mb by more than 50%"
- "WARNING: file L2.593[303,308] 342ns 244mb exceeds soft limit 100mb by more than 50%"
- "WARNING: file L2.594[309,314] 342ns 282mb exceeds soft limit 100mb by more than 50%"
- "WARNING: file L2.595[315,317] 342ns 214mb exceeds soft limit 100mb by more than 50%"
- "WARNING: file L2.596[318,320] 342ns 222mb exceeds soft limit 100mb by more than 50%"
- "WARNING: file L2.598[324,326] 342ns 254mb exceeds soft limit 100mb by more than 50%"
- "WARNING: file L2.599[327,329] 342ns 197mb exceeds soft limit 100mb by more than 50%"
- "WARNING: file L2.600[330,332] 342ns 228mb exceeds soft limit 100mb by more than 50%"
- "WARNING: file L2.601[333,335] 342ns 199mb exceeds soft limit 100mb by more than 50%"
- "WARNING: file L2.602[336,338] 342ns 280mb exceeds soft limit 100mb by more than 50%"
- "L2.594[150,165] 342ns 218mb |L2.594| "
- "L2.595[166,171] 342ns 118mb |L2.595| "
- "L2.598[183,197] 342ns 267mb |L2.598| "
- "L2.599[198,207] 342ns 157mb |L2.599| "
- "L2.600[208,220] 342ns 147mb |L2.600| "
- "L2.601[221,232] 342ns 270mb |L2.601| "
- "L2.602[233,244] 342ns 147mb |L2.602| "
- "L2.603[245,253] 342ns 139mb |L2.603| "
- "L2.604[271,276] 342ns 117mb |L2.604| "
- "L2.605[277,281] 342ns 109mb |L2.605| "
- "L2.612[254,261] 342ns 105mb |L2.612| "
- "L2.613[262,270] 342ns 184mb |L2.613| "
- "L2.616[309,311] 342ns 101mb |L2.616|"
- "L2.617[312,314] 342ns 181mb |L2.617|"
- "L2.618[315,317] 342ns 214mb |L2.618|"
- "L2.619[318,320] 342ns 222mb |L2.619|"
- "L2.620[321,323] 342ns 146mb |L2.620|"
- "L2.621[324,326] 342ns 254mb |L2.621|"
- "L2.622[327,329] 342ns 197mb |L2.622|"
- "L2.623[330,332] 342ns 228mb |L2.623|"
- "L2.624[333,335] 342ns 199mb |L2.624|"
- "L2.625[336,337] 342ns 156mb |L2.625|"
- "L2.626[338,338] 342ns 124mb |L2.626|"
- "L2.628[1,36] 342ns 103mb |L2.628-| "
- "L2.629[37,71] 342ns 103mb |L2.629-| "
- "L2.630[72,83] 342ns 103mb |L2.630| "
- "L2.638[172,177] 342ns 109mb |L2.638| "
- "L2.639[178,182] 342ns 109mb |L2.639| "
- "L2.640[282,288] 342ns 100mb |L2.640| "
- "L2.643[300,303] 342ns 110mb |L2.643| "
- "L2.646[84,94] 342ns 107mb |L2.646| "
- "L2.647[95,104] 342ns 97mb |L2.647| "
- "L2.648[105,111] 342ns 86mb |L2.648| "
- "L2.649[112,119] 342ns 114mb |L2.649| "
- "L2.650[120,126] 342ns 98mb |L2.650| "
- "L2.651[127,130] 342ns 82mb |L2.651| "
- "L2.652[131,138] 342ns 108mb |L2.652| "
- "L2.653[139,145] 342ns 93mb |L2.653| "
- "L2.654[146,149] 342ns 77mb |L2.654| "
- "L2.655[289,293] 342ns 110mb |L2.655| "
- "L2.656[294,297] 342ns 82mb |L2.656| "
- "L2.657[298,299] 342ns 82mb |L2.657| "
- "L2.658[304,306] 342ns 113mb |L2.658| "
- "L2.659[307,308] 342ns 113mb |L2.659| "
- "L2.660[339,339] 342ns 25mb |L2.660|"
- "WARNING: file L2.594[150,165] 342ns 218mb exceeds soft limit 100mb by more than 50%"
- "WARNING: file L2.598[183,197] 342ns 267mb exceeds soft limit 100mb by more than 50%"
- "WARNING: file L2.599[198,207] 342ns 157mb exceeds soft limit 100mb by more than 50%"
- "WARNING: file L2.601[221,232] 342ns 270mb exceeds soft limit 100mb by more than 50%"
- "WARNING: file L2.613[262,270] 342ns 184mb exceeds soft limit 100mb by more than 50%"
- "WARNING: file L2.617[312,314] 342ns 181mb exceeds soft limit 100mb by more than 50%"
- "WARNING: file L2.618[315,317] 342ns 214mb exceeds soft limit 100mb by more than 50%"
- "WARNING: file L2.619[318,320] 342ns 222mb exceeds soft limit 100mb by more than 50%"
- "WARNING: file L2.621[324,326] 342ns 254mb exceeds soft limit 100mb by more than 50%"
- "WARNING: file L2.622[327,329] 342ns 197mb exceeds soft limit 100mb by more than 50%"
- "WARNING: file L2.623[330,332] 342ns 228mb exceeds soft limit 100mb by more than 50%"
- "WARNING: file L2.624[333,335] 342ns 199mb exceeds soft limit 100mb by more than 50%"
- "WARNING: file L2.625[336,337] 342ns 156mb exceeds soft limit 100mb by more than 50%"
"###
);
}

20
compactor/tests/layouts/many_files.rs
View File

@ -4670,17 +4670,17 @@ async fn l0s_almost_needing_vertical_split() {
- "L0.998[24,100] 1.02us |-----------------------------------------L0.998-----------------------------------------|"
- "L0.999[24,100] 1.02us |-----------------------------------------L0.999-----------------------------------------|"
- "L0.1000[24,100] 1.02us |----------------------------------------L0.1000-----------------------------------------|"
- "**** Final Output Files (6.5gb written)"
- "**** Final Output Files (5.23gb written)"
- "L2 "
- "L2.3141[24,37] 1.02us 108mb|---L2.3141---| "
- "L2.3150[38,49] 1.02us 102mb |--L2.3150--| "
- "L2.3151[50,60] 1.02us 93mb |-L2.3151-| "
- "L2.3152[61,63] 1.02us 37mb |L2.3152| "
- "L2.3153[64,73] 1.02us 101mb |L2.3153-| "
- "L2.3154[74,82] 1.02us 90mb |L2.3154| "
- "L2.3155[83,90] 1.02us 101mb |L2.3155| "
- "L2.3156[91,98] 1.02us 93mb |L2.3156| "
- "L2.3157[99,100] 1.02us 26mb |L2.3157|"
- "L2.3086[24,35] 1.02us 102mb|--L2.3086--| "
- "L2.3095[36,47] 1.02us 105mb |--L2.3095--| "
- "L2.3096[48,58] 1.02us 95mb |-L2.3096-| "
- "L2.3097[59,65] 1.02us 76mb |L2.3097| "
- "L2.3098[66,76] 1.02us 106mb |-L2.3098-| "
- "L2.3099[77,86] 1.02us 96mb |L2.3099-| "
- "L2.3100[87,90] 1.02us 53mb |L2.3100| "
- "L2.3101[91,98] 1.02us 90mb |L2.3101| "
- "L2.3102[99,100] 1.02us 26mb |L2.3102|"
"###
);
}

1002
compactor/tests/layouts/stuck.rs
View File

File diff suppressed because it is too large Load Diff