diff --git a/compactor/src/components/round_info_source/mod.rs b/compactor/src/components/round_info_source/mod.rs index 1d2419df3a..a1e5b96548 100644 --- a/compactor/src/components/round_info_source/mod.rs +++ b/compactor/src/components/round_info_source/mod.rs @@ -172,7 +172,11 @@ impl RoundInfoSource for LevelBasedRoundInfo { _partition_info: &PartitionInfo, files: &[ParquetFile], ) -> Result { - let start_level = get_start_level(files); + let start_level = get_start_level( + files, + self.max_num_files_per_plan, + self.max_total_file_size_per_plan, + ); if self.too_many_small_files_to_compact(files, start_level) { return Ok(RoundInfo::ManySmallFiles { @@ -187,23 +191,53 @@ impl RoundInfoSource for LevelBasedRoundInfo { } } -fn get_start_level(files: &[ParquetFile]) -> CompactionLevel { +// get_start_level decides what level to start compaction from. Often this is the lowest level +// we have ParquetFiles in, but occasionally we decide to compact L1->L2 when L0s still exist. +// +// If we ignore the invariants (where intra-level overlaps are allowed), this would be a math problem +// to optimize write amplification. +// +// However, allowing intra-level overlaps in L0 but not L1/L2 adds extra challenge to compacting L0s to L1. +// This is especially true when there are large quantitites of overlapping L0s and L1s, potentially resulting +// in many split/compact cycles to resolve the overlaps. +// +// Since L1 & L2 only have inter-level overlaps, they can be compacted with just a few splits to align the L1s +// with the L2s. The relative ease of moving data from L1 to L2 provides additional motivation to compact the +// L1s to L2s when a backlog of L0s exist. The easily solvable L1->L2 compaction can give us a clean slate in +// L1, greatly simplifying the remaining L0->L1 compactions. +fn get_start_level(files: &[ParquetFile], max_files: usize, max_bytes: usize) -> CompactionLevel { // panic if the files are empty assert!(!files.is_empty()); - // Start with initial level - // If there are files in this level, itis the start level - // Otherwise repeat until reaching the final level. - let mut level = CompactionLevel::Initial; - while level != CompactionLevel::Final { - if files.iter().any(|f| f.compaction_level == level) { - return level; - } + let mut l0_cnt: usize = 0; + let mut l0_bytes: usize = 0; + let mut l1_bytes: usize = 0; - level = level.next(); + for f in files { + match f.compaction_level { + CompactionLevel::Initial => { + l0_cnt += 1; + l0_bytes += f.file_size_bytes as usize; + } + CompactionLevel::FileNonOverlapped => { + l1_bytes += f.file_size_bytes as usize; + } + _ => {} + } } - level + if l1_bytes > 3 * max_bytes && (l0_cnt > max_files || l0_bytes > max_bytes) { + // L1 is big enough to pose an overlap challenge compacting from L0, and there is quite a bit more coming from L0. + // The criteria for this early L1->L2 compaction significanly impacts write amplification. The above values optimize + // existing test cases, but may be changed as additional test cases are added. + CompactionLevel::FileNonOverlapped + } else if l0_bytes > 0 { + CompactionLevel::Initial + } else if l1_bytes > 0 { + CompactionLevel::FileNonOverlapped + } else { + CompactionLevel::Final + } } fn get_num_overlapped_files( diff --git a/compactor/src/components/split_or_compact/start_level_files_to_split.rs b/compactor/src/components/split_or_compact/start_level_files_to_split.rs index 4a6eaba61e..e7dd794c22 100644 --- a/compactor/src/components/split_or_compact/start_level_files_to_split.rs +++ b/compactor/src/components/split_or_compact/start_level_files_to_split.rs @@ -301,7 +301,26 @@ pub fn merge_small_l0_chains( for chain in &chains { let this_chain_bytes = chain.iter().map(|f| f.file_size_bytes as usize).sum(); - if prior_chain_bytes > 0 && prior_chain_bytes + this_chain_bytes <= max_compact_size { + // matching max_lo_created_at times indicates that the files were deliberately split. We shouldn't merge + // chains with matching max_lo_created_at times, because that would encourage undoing the previous split, + // which minimally increases write amplification, and may cause unproductive split/compact loops. + let mut matches = 0; + if prior_chain_bytes > 0 { + for f in chain.iter() { + for f2 in &merged_chains[prior_chain_idx as usize] { + if f.max_l0_created_at == f2.max_l0_created_at { + matches += 1; + break; + } + } + } + } + + // Merge it if: there a prior chain to merge with, and merging wouldn't make it too big, or undo a previous split + if prior_chain_bytes > 0 + && prior_chain_bytes + this_chain_bytes <= max_compact_size + && matches == 0 + { // this chain can be added to the prior chain. merged_chains[prior_chain_idx as usize].append(&mut chain.clone()); prior_chain_bytes += this_chain_bytes; diff --git a/compactor/tests/integration.rs b/compactor/tests/integration.rs index fbf60bffd2..31ee42ec11 100644 --- a/compactor/tests/integration.rs +++ b/compactor/tests/integration.rs @@ -68,8 +68,8 @@ async fn test_num_files_over_limit() { assert_levels( &files, vec![ - (8, CompactionLevel::FileNonOverlapped), (9, CompactionLevel::FileNonOverlapped), + (10, CompactionLevel::FileNonOverlapped), ], ); } diff --git a/compactor/tests/layouts/backfill.rs b/compactor/tests/layouts/backfill.rs index 708d5d95b1..33302dfc06 100644 --- a/compactor/tests/layouts/backfill.rs +++ b/compactor/tests/layouts/backfill.rs @@ -746,97 +746,85 @@ async fn random_backfill_over_l2s() { - "Committing partition 1:" - " Soft Deleting 4 files: L0.76, L0.77, L0.79, L0.80" - " Creating 8 files" - - "**** Simulation run 15, type=compact(ManySmallFiles). 10 Input Files, 200mb total:" - - "L0 " - - "L0.75[42,356] 1.04us 33mb|-----------L0.75-----------| " - - "L0.86[357,357] 1.04us 0b |L0.86| " - - "L0.87[358,670] 1.04us 33mb |-----------L0.87-----------| " - - "L0.84[671,672] 1.04us 109kb |L0.84| " - - "L0.85[673,986] 1.04us 33mb |-----------L0.85-----------| " - - "L0.78[42,356] 1.05us 33mb|-----------L0.78-----------| " - - "L0.90[357,357] 1.05us 0b |L0.90| " - - "L0.91[358,670] 1.05us 33mb |-----------L0.91-----------| " - - "L0.88[671,672] 1.05us 109kb |L0.88| " - - "L0.89[673,986] 1.05us 33mb |-----------L0.89-----------| " - - "**** 1 Output Files (parquet_file_id not yet assigned), 200mb total:" - - "L0, all files 200mb " - - "L0.?[42,986] 1.05us |------------------------------------------L0.?------------------------------------------|" - - "Committing partition 1:" - - " Soft Deleting 10 files: L0.75, L0.78, L0.84, L0.85, L0.86, L0.87, L0.88, L0.89, L0.90, L0.91" - - " Creating 1 files" - - "**** Simulation run 16, type=split(HighL0OverlapSingleFile)(split_times=[670]). 1 Input Files, 100mb total:" - - "L1, all files 100mb " - - "L1.82[358,672] 1.03us |-----------------------------------------L1.82------------------------------------------|" - - "**** 2 Output Files (parquet_file_id not yet assigned), 100mb total:" - - "L1 " - - "L1.?[358,670] 1.03us 99mb|-----------------------------------------L1.?------------------------------------------| " - - "L1.?[671,672] 1.03us 651kb |L1.?|" - - "**** Simulation run 17, type=split(HighL0OverlapSingleFile)(split_times=[356]). 1 Input Files, 100mb total:" - - "L1, all files 100mb " - - "L1.81[42,357] 1.03us |-----------------------------------------L1.81------------------------------------------|" - - "**** 2 Output Files (parquet_file_id not yet assigned), 100mb total:" - - "L1 " - - "L1.?[42,356] 1.03us 100mb|-----------------------------------------L1.?------------------------------------------| " - - "L1.?[357,357] 1.03us 325kb |L1.?|" - - "**** Simulation run 18, type=split(HighL0OverlapSingleFile)(split_times=[356, 670]). 1 Input Files, 200mb total:" - - "L0, all files 200mb " - - "L0.92[42,986] 1.05us |-----------------------------------------L0.92------------------------------------------|" - - "**** 3 Output Files (parquet_file_id not yet assigned), 200mb total:" - - "L0 " - - "L0.?[42,356] 1.05us 67mb |-----------L0.?------------| " - - "L0.?[357,670] 1.05us 66mb |-----------L0.?------------| " - - "L0.?[671,986] 1.05us 67mb |------------L0.?------------| " - - "Committing partition 1:" - - " Soft Deleting 3 files: L1.81, L1.82, L0.92" - - " Creating 7 files" - - "**** Simulation run 19, type=split(ReduceOverlap)(split_times=[672]). 1 Input Files, 67mb total:" + - "**** Simulation run 15, type=compact(ManySmallFiles). 2 Input Files, 67mb total:" + - "L0, all files 33mb " + - "L0.75[42,356] 1.04us |-----------------------------------------L0.75------------------------------------------|" + - "L0.78[42,356] 1.05us |-----------------------------------------L0.78------------------------------------------|" + - "**** 1 Output Files (parquet_file_id not yet assigned), 67mb total:" - "L0, all files 67mb " - - "L0.99[671,986] 1.05us |-----------------------------------------L0.99------------------------------------------|" - - "**** 2 Output Files (parquet_file_id not yet assigned), 67mb total:" - - "L0 " - - "L0.?[671,672] 1.05us 218kb|L0.?| " - - "L0.?[673,986] 1.05us 67mb|-----------------------------------------L0.?------------------------------------------| " - - "**** Simulation run 20, type=split(ReduceOverlap)(split_times=[357]). 1 Input Files, 66mb total:" - - "L0, all files 66mb " - - "L0.98[357,670] 1.05us |-----------------------------------------L0.98------------------------------------------|" - - "**** 2 Output Files (parquet_file_id not yet assigned), 66mb total:" - - "L0 " - - "L0.?[357,357] 1.05us 0b |L0.?| " - - "L0.?[358,670] 1.05us 66mb|-----------------------------------------L0.?------------------------------------------| " + - "L0.?[42,356] 1.05us |------------------------------------------L0.?------------------------------------------|" - "Committing partition 1:" - - " Soft Deleting 2 files: L0.98, L0.99" - - " Creating 4 files" - - "**** Simulation run 21, type=split(CompactAndSplitOutput(FoundSubsetLessThanMaxCompactSize))(split_times=[232]). 4 Input Files, 167mb total:" + - " Soft Deleting 2 files: L0.75, L0.78" + - " Creating 1 files" + - "**** Simulation run 16, type=compact(ManySmallFiles). 2 Input Files, 66mb total:" + - "L0, all files 33mb " + - "L0.87[358,670] 1.04us |-----------------------------------------L0.87------------------------------------------|" + - "L0.91[358,670] 1.05us |-----------------------------------------L0.91------------------------------------------|" + - "**** 1 Output Files (parquet_file_id not yet assigned), 66mb total:" + - "L0, all files 66mb " + - "L0.?[358,670] 1.05us |------------------------------------------L0.?------------------------------------------|" + - "Committing partition 1:" + - " Soft Deleting 2 files: L0.87, L0.91" + - " Creating 1 files" + - "**** Simulation run 17, type=compact(ManySmallFiles). 2 Input Files, 218kb total:" + - "L0, all files 109kb " + - "L0.84[671,672] 1.04us |-----------------------------------------L0.84------------------------------------------|" + - "L0.88[671,672] 1.05us |-----------------------------------------L0.88------------------------------------------|" + - "**** 1 Output Files (parquet_file_id not yet assigned), 218kb total:" + - "L0, all files 218kb " + - "L0.?[671,672] 1.05us |------------------------------------------L0.?------------------------------------------|" + - "Committing partition 1:" + - " Soft Deleting 2 files: L0.84, L0.88" + - " Creating 1 files" + - "**** Simulation run 18, type=compact(ManySmallFiles). 2 Input Files, 67mb total:" + - "L0, all files 33mb " + - "L0.85[673,986] 1.04us |-----------------------------------------L0.85------------------------------------------|" + - "L0.89[673,986] 1.05us |-----------------------------------------L0.89------------------------------------------|" + - "**** 1 Output Files (parquet_file_id not yet assigned), 67mb total:" + - "L0, all files 67mb " + - "L0.?[673,986] 1.05us |------------------------------------------L0.?------------------------------------------|" + - "Committing partition 1:" + - " Soft Deleting 2 files: L0.85, L0.89" + - " Creating 1 files" + - "**** Simulation run 19, type=compact(ManySmallFiles). 2 Input Files, 0b total:" + - "L0, all files 0b " + - "L0.86[357,357] 1.04us |-----------------------------------------L0.86------------------------------------------|" + - "L0.90[357,357] 1.05us |-----------------------------------------L0.90------------------------------------------|" + - "**** 1 Output Files (parquet_file_id not yet assigned), 0b total:" + - "L0, all files 0b " + - "L0.?[357,357] 1.05us |------------------------------------------L0.?------------------------------------------|" + - "Committing partition 1:" + - " Soft Deleting 2 files: L0.86, L0.90" + - " Creating 1 files" + - "**** Simulation run 20, type=split(CompactAndSplitOutput(FoundSubsetLessThanMaxCompactSize))(split_times=[232]). 3 Input Files, 167mb total:" - "L0 " - - "L0.97[42,356] 1.05us 67mb|-----------------------------------------L0.97-----------------------------------------| " - - "L0.102[357,357] 1.05us 0b |L0.102|" + - "L0.92[42,356] 1.05us 67mb|-----------------------------------------L0.92-----------------------------------------| " + - "L0.96[357,357] 1.05us 0b |L0.96|" - "L1 " - - "L1.95[42,356] 1.03us 100mb|-----------------------------------------L1.95-----------------------------------------| " - - "L1.96[357,357] 1.03us 325kb |L1.96|" + - "L1.81[42,357] 1.03us 100mb|-----------------------------------------L1.81------------------------------------------|" - "**** 2 Output Files (parquet_file_id not yet assigned), 167mb total:" - "L1 " - "L1.?[42,232] 1.05us 101mb|------------------------L1.?------------------------| " - "L1.?[233,357] 1.05us 66mb |--------------L1.?---------------| " - "Committing partition 1:" - - " Soft Deleting 4 files: L1.95, L1.96, L0.97, L0.102" + - " Soft Deleting 3 files: L1.81, L0.92, L0.96" - " Creating 2 files" - - "**** Simulation run 22, type=split(CompactAndSplitOutput(FoundSubsetLessThanMaxCompactSize))(split_times=[547]). 4 Input Files, 166mb total:" + - "**** Simulation run 21, type=split(CompactAndSplitOutput(FoundSubsetLessThanMaxCompactSize))(split_times=[547]). 3 Input Files, 166mb total:" - "L0 " - - "L0.103[358,670] 1.05us 66mb|----------------------------------------L0.103-----------------------------------------| " - - "L0.100[671,672] 1.05us 218kb |L0.100|" + - "L0.93[358,670] 1.05us 66mb|-----------------------------------------L0.93-----------------------------------------| " + - "L0.94[671,672] 1.05us 218kb |L0.94|" - "L1 " - - "L1.93[358,670] 1.03us 99mb|-----------------------------------------L1.93-----------------------------------------| " - - "L1.94[671,672] 1.03us 651kb |L1.94|" + - "L1.82[358,672] 1.03us 100mb|-----------------------------------------L1.82------------------------------------------|" - "**** 2 Output Files (parquet_file_id not yet assigned), 166mb total:" - "L1 " - "L1.?[358,547] 1.05us 100mb|------------------------L1.?------------------------| " - "L1.?[548,672] 1.05us 66mb |--------------L1.?---------------| " - "Committing partition 1:" - - " Soft Deleting 4 files: L1.93, L1.94, L0.100, L0.103" + - " Soft Deleting 3 files: L1.82, L0.93, L0.94" - " Creating 2 files" - - "**** Simulation run 23, type=split(CompactAndSplitOutput(TotalSizeLessThanMaxCompactSize))(split_times=[861]). 2 Input Files, 167mb total:" + - "**** Simulation run 22, type=split(CompactAndSplitOutput(TotalSizeLessThanMaxCompactSize))(split_times=[861]). 2 Input Files, 167mb total:" - "L0 " - - "L0.101[673,986] 1.05us 67mb|-----------------------------------------L0.101-----------------------------------------|" + - "L0.95[673,986] 1.05us 67mb|-----------------------------------------L0.95------------------------------------------|" - "L1 " - "L1.83[673,986] 1.03us 100mb|-----------------------------------------L1.83------------------------------------------|" - "**** 2 Output Files (parquet_file_id not yet assigned), 167mb total:" @@ -844,60 +832,60 @@ async fn random_backfill_over_l2s() { - "L1.?[673,861] 1.05us 100mb|------------------------L1.?------------------------| " - "L1.?[862,986] 1.05us 67mb |--------------L1.?---------------| " - "Committing partition 1:" - - " Soft Deleting 2 files: L1.83, L0.101" + - " Soft Deleting 2 files: L1.83, L0.95" - " Creating 2 files" - - "**** Simulation run 24, type=split(ReduceOverlap)(split_times=[399, 499]). 1 Input Files, 100mb total:" + - "**** Simulation run 23, type=split(ReduceOverlap)(split_times=[399, 499]). 1 Input Files, 100mb total:" - "L1, all files 100mb " - - "L1.106[358,547] 1.05us |-----------------------------------------L1.106-----------------------------------------|" + - "L1.99[358,547] 1.05us |-----------------------------------------L1.99------------------------------------------|" - "**** 3 Output Files (parquet_file_id not yet assigned), 100mb total:" - "L1 " - "L1.?[358,399] 1.05us 22mb|------L1.?-------| " - "L1.?[400,499] 1.05us 52mb |--------------------L1.?---------------------| " - "L1.?[500,547] 1.05us 26mb |--------L1.?--------| " - - "**** Simulation run 25, type=split(ReduceOverlap)(split_times=[299]). 1 Input Files, 66mb total:" + - "**** Simulation run 24, type=split(ReduceOverlap)(split_times=[299]). 1 Input Files, 66mb total:" - "L1, all files 66mb " - - "L1.105[233,357] 1.05us |-----------------------------------------L1.105-----------------------------------------|" + - "L1.98[233,357] 1.05us |-----------------------------------------L1.98------------------------------------------|" - "**** 2 Output Files (parquet_file_id not yet assigned), 66mb total:" - "L1 " - "L1.?[233,299] 1.05us 35mb|--------------------L1.?---------------------| " - "L1.?[300,357] 1.05us 31mb |-----------------L1.?------------------| " - - "**** Simulation run 26, type=split(ReduceOverlap)(split_times=[99, 199]). 1 Input Files, 101mb total:" + - "**** Simulation run 25, type=split(ReduceOverlap)(split_times=[99, 199]). 1 Input Files, 101mb total:" - "L1, all files 101mb " - - "L1.104[42,232] 1.05us |-----------------------------------------L1.104-----------------------------------------|" + - "L1.97[42,232] 1.05us |-----------------------------------------L1.97------------------------------------------|" - "**** 3 Output Files (parquet_file_id not yet assigned), 101mb total:" - "L1 " - "L1.?[42,99] 1.05us 30mb |----------L1.?-----------| " - "L1.?[100,199] 1.05us 52mb |--------------------L1.?--------------------| " - "L1.?[200,232] 1.05us 18mb |----L1.?-----| " - - "**** Simulation run 27, type=split(ReduceOverlap)(split_times=[599]). 1 Input Files, 66mb total:" + - "**** Simulation run 26, type=split(ReduceOverlap)(split_times=[599]). 1 Input Files, 66mb total:" - "L1, all files 66mb " - - "L1.107[548,672] 1.05us |-----------------------------------------L1.107-----------------------------------------|" + - "L1.100[548,672] 1.05us |-----------------------------------------L1.100-----------------------------------------|" - "**** 2 Output Files (parquet_file_id not yet assigned), 66mb total:" - "L1 " - "L1.?[548,599] 1.05us 27mb|---------------L1.?----------------| " - "L1.?[600,672] 1.05us 39mb |-----------------------L1.?-----------------------| " - - "**** Simulation run 28, type=split(ReduceOverlap)(split_times=[899]). 1 Input Files, 67mb total:" + - "**** Simulation run 27, type=split(ReduceOverlap)(split_times=[899]). 1 Input Files, 67mb total:" - "L1, all files 67mb " - - "L1.109[862,986] 1.05us |-----------------------------------------L1.109-----------------------------------------|" + - "L1.102[862,986] 1.05us |-----------------------------------------L1.102-----------------------------------------|" - "**** 2 Output Files (parquet_file_id not yet assigned), 67mb total:" - "L1 " - "L1.?[862,899] 1.05us 20mb|----------L1.?----------| " - "L1.?[900,986] 1.05us 47mb |----------------------------L1.?----------------------------| " - - "**** Simulation run 29, type=split(ReduceOverlap)(split_times=[699, 799]). 1 Input Files, 100mb total:" + - "**** Simulation run 28, type=split(ReduceOverlap)(split_times=[699, 799]). 1 Input Files, 100mb total:" - "L1, all files 100mb " - - "L1.108[673,861] 1.05us |-----------------------------------------L1.108-----------------------------------------|" + - "L1.101[673,861] 1.05us |-----------------------------------------L1.101-----------------------------------------|" - "**** 3 Output Files (parquet_file_id not yet assigned), 100mb total:" - "L1 " - "L1.?[673,699] 1.05us 14mb|---L1.?---| " - "L1.?[700,799] 1.05us 53mb |--------------------L1.?---------------------| " - "L1.?[800,861] 1.05us 34mb |-----------L1.?------------| " - "Committing partition 1:" - - " Soft Deleting 6 files: L1.104, L1.105, L1.106, L1.107, L1.108, L1.109" + - " Soft Deleting 6 files: L1.97, L1.98, L1.99, L1.100, L1.101, L1.102" - " Creating 15 files" - - "**** Simulation run 30, type=split(CompactAndSplitOutput(FoundSubsetLessThanMaxCompactSize))(split_times=[71, 142]). 4 Input Files, 283mb total:" + - "**** Simulation run 29, type=split(CompactAndSplitOutput(FoundSubsetLessThanMaxCompactSize))(split_times=[71, 142]). 4 Input Files, 283mb total:" - "L1 " - - "L1.115[42,99] 1.05us 30mb |--------L1.115---------| " - - "L1.116[100,199] 1.05us 52mb |------------------L1.116------------------| " + - "L1.108[42,99] 1.05us 30mb |--------L1.108---------| " + - "L1.109[100,199] 1.05us 52mb |------------------L1.109------------------| " - "L2 " - "L2.1[0,99] 99ns 100mb |-------------------L2.1-------------------| " - "L2.2[100,199] 199ns 100mb |-------------------L2.2-------------------| " @@ -907,13 +895,13 @@ async fn random_backfill_over_l2s() { - "L2.?[72,142] 1.05us 99mb |------------L2.?-------------| " - "L2.?[143,199] 1.05us 82mb |---------L2.?----------| " - "Committing partition 1:" - - " Soft Deleting 4 files: L2.1, L2.2, L1.115, L1.116" + - " Soft Deleting 4 files: L2.1, L2.2, L1.108, L1.109" - " Creating 3 files" - - "**** Simulation run 31, type=split(CompactAndSplitOutput(FoundSubsetLessThanMaxCompactSize))(split_times=[271, 342]). 5 Input Files, 284mb total:" + - "**** Simulation run 30, type=split(CompactAndSplitOutput(FoundSubsetLessThanMaxCompactSize))(split_times=[271, 342]). 5 Input Files, 284mb total:" - "L1 " - - "L1.117[200,232] 1.05us 18mb|---L1.117---| " - - "L1.113[233,299] 1.05us 35mb |----------L1.113-----------| " - - "L1.114[300,357] 1.05us 31mb |--------L1.114---------| " + - "L1.110[200,232] 1.05us 18mb|---L1.110---| " + - "L1.106[233,299] 1.05us 35mb |----------L1.106-----------| " + - "L1.107[300,357] 1.05us 31mb |--------L1.107---------| " - "L2 " - "L2.3[200,299] 299ns 100mb|-------------------L2.3-------------------| " - "L2.4[300,399] 399ns 100mb |-------------------L2.4-------------------| " @@ -923,14 +911,14 @@ async fn random_backfill_over_l2s() { - "L2.?[272,342] 1.05us 100mb |------------L2.?-------------| " - "L2.?[343,399] 1.05us 83mb |---------L2.?----------| " - "Committing partition 1:" - - " Soft Deleting 5 files: L2.3, L2.4, L1.113, L1.114, L1.117" + - " Soft Deleting 5 files: L2.3, L2.4, L1.106, L1.107, L1.110" - " Creating 3 files" - - "**** Simulation run 32, type=split(CompactAndSplitOutput(FoundSubsetLessThanMaxCompactSize))(split_times=[404, 465]). 4 Input Files, 257mb total:" + - "**** Simulation run 31, type=split(CompactAndSplitOutput(FoundSubsetLessThanMaxCompactSize))(split_times=[404, 465]). 4 Input Files, 257mb total:" - "L1 " - - "L1.110[358,399] 1.05us 22mb |-------L1.110--------| " - - "L1.111[400,499] 1.05us 52mb |------------------------L1.111-------------------------| " + - "L1.103[358,399] 1.05us 22mb |-------L1.103--------| " + - "L1.104[400,499] 1.05us 52mb |------------------------L1.104-------------------------| " - "L2 " - - "L2.130[343,399] 1.05us 83mb|------------L2.130------------| " + - "L2.123[343,399] 1.05us 83mb|------------L2.123------------| " - "L2.5[400,499] 499ns 100mb |-------------------------L2.5--------------------------| " - "**** 3 Output Files (parquet_file_id not yet assigned), 257mb total:" - "L2 " @@ -938,13 +926,13 @@ async fn random_backfill_over_l2s() { - "L2.?[405,465] 1.05us 99mb |--------------L2.?--------------| " - "L2.?[466,499] 1.05us 58mb |------L2.?-------| " - "Committing partition 1:" - - " Soft Deleting 4 files: L2.5, L1.110, L1.111, L2.130" + - " Soft Deleting 4 files: L2.5, L1.103, L1.104, L2.123" - " Creating 3 files" - - "**** Simulation run 33, type=split(CompactAndSplitOutput(FoundSubsetLessThanMaxCompactSize))(split_times=[569, 638]). 5 Input Files, 292mb total:" + - "**** Simulation run 32, type=split(CompactAndSplitOutput(FoundSubsetLessThanMaxCompactSize))(split_times=[569, 638]). 5 Input Files, 292mb total:" - "L1 " - - "L1.112[500,547] 1.05us 26mb|------L1.112-------| " - - "L1.118[548,599] 1.05us 27mb |-------L1.118--------| " - - "L1.119[600,672] 1.05us 39mb |------------L1.119------------| " + - "L1.105[500,547] 1.05us 26mb|------L1.105-------| " + - "L1.111[548,599] 1.05us 27mb |-------L1.111--------| " + - "L1.112[600,672] 1.05us 39mb |------------L1.112------------| " - "L2 " - "L2.6[500,599] 599ns 100mb|-------------------L2.6-------------------| " - "L2.7[600,699] 699ns 100mb |-------------------L2.7-------------------| " @@ -954,14 +942,14 @@ async fn random_backfill_over_l2s() { - "L2.?[570,638] 1.05us 100mb |------------L2.?------------| " - "L2.?[639,699] 1.05us 91mb |----------L2.?-----------| " - "Committing partition 1:" - - " Soft Deleting 5 files: L2.6, L2.7, L1.112, L1.118, L1.119" + - " Soft Deleting 5 files: L2.6, L2.7, L1.105, L1.111, L1.112" - " Creating 3 files" - - "**** Simulation run 34, type=split(CompactAndSplitOutput(FoundSubsetLessThanMaxCompactSize))(split_times=[702, 765]). 4 Input Files, 258mb total:" + - "**** Simulation run 33, type=split(CompactAndSplitOutput(FoundSubsetLessThanMaxCompactSize))(split_times=[702, 765]). 4 Input Files, 258mb total:" - "L1 " - - "L1.122[673,699] 1.05us 14mb |---L1.122---| " - - "L1.123[700,799] 1.05us 53mb |-----------------------L1.123------------------------| " + - "L1.115[673,699] 1.05us 14mb |---L1.115---| " + - "L1.116[700,799] 1.05us 53mb |-----------------------L1.116------------------------| " - "L2 " - - "L2.136[639,699] 1.05us 91mb|------------L2.136-------------| " + - "L2.129[639,699] 1.05us 91mb|------------L2.129-------------| " - "L2.8[700,799] 799ns 100mb |------------------------L2.8-------------------------| " - "**** 3 Output Files (parquet_file_id not yet assigned), 258mb total:" - "L2 " @@ -969,12 +957,12 @@ async fn random_backfill_over_l2s() { - "L2.?[703,765] 1.05us 100mb |--------------L2.?--------------| " - "L2.?[766,799] 1.05us 56mb |------L2.?------| " - "Committing partition 1:" - - " Soft Deleting 4 files: L2.8, L1.122, L1.123, L2.136" + - " Soft Deleting 4 files: L2.8, L1.115, L1.116, L2.129" - " Creating 3 files" - - "**** Simulation run 35, type=split(CompactAndSplitOutput(FoundSubsetLessThanMaxCompactSize))(split_times=[865]). 3 Input Files, 154mb total:" + - "**** Simulation run 34, type=split(CompactAndSplitOutput(FoundSubsetLessThanMaxCompactSize))(split_times=[865]). 3 Input Files, 154mb total:" - "L1 " - - "L1.124[800,861] 1.05us 34mb|-----------------------L1.124------------------------| " - - "L1.120[862,899] 1.05us 20mb |------------L1.120-------------| " + - "L1.117[800,861] 1.05us 34mb|-----------------------L1.117------------------------| " + - "L1.113[862,899] 1.05us 20mb |------------L1.113-------------| " - "L2 " - "L2.9[800,899] 899ns 100mb|-----------------------------------------L2.9------------------------------------------| " - "**** 2 Output Files (parquet_file_id not yet assigned), 154mb total:" @@ -982,28 +970,28 @@ async fn random_backfill_over_l2s() { - "L2.?[800,865] 1.05us 101mb|--------------------------L2.?---------------------------| " - "L2.?[866,899] 1.05us 53mb |-----------L2.?------------| " - "Committing partition 1:" - - " Soft Deleting 3 files: L2.9, L1.120, L1.124" + - " Soft Deleting 3 files: L2.9, L1.113, L1.117" - " Creating 2 files" - - "**** Final Output Files (4.58gb written)" + - "**** Final Output Files (4.06gb written)" - "L1 " - - "L1.121[900,986] 1.05us 47mb |L1.121| " + - "L1.114[900,986] 1.05us 47mb |L1.114| " - "L2 " - "L2.10[900,999] 999ns 100mb |L2.10-| " - - "L2.125[0,71] 1.05us 101mb|L2.125| " - - "L2.126[72,142] 1.05us 99mb |L2.126| " - - "L2.127[143,199] 1.05us 82mb |L2.127| " - - "L2.128[200,271] 1.05us 101mb |L2.128| " - - "L2.129[272,342] 1.05us 100mb |L2.129| " - - "L2.131[343,404] 1.05us 100mb |L2.131| " - - "L2.132[405,465] 1.05us 99mb |L2.132| " - - "L2.133[466,499] 1.05us 58mb |L2.133| " - - "L2.134[500,569] 1.05us 101mb |L2.134| " - - "L2.135[570,638] 1.05us 100mb |L2.135| " - - "L2.137[639,702] 1.05us 101mb |L2.137| " - - "L2.138[703,765] 1.05us 100mb |L2.138| " - - "L2.139[766,799] 1.05us 56mb |L2.139| " - - "L2.140[800,865] 1.05us 101mb |L2.140| " - - "L2.141[866,899] 1.05us 53mb |L2.141| " + - "L2.118[0,71] 1.05us 101mb|L2.118| " + - "L2.119[72,142] 1.05us 99mb |L2.119| " + - "L2.120[143,199] 1.05us 82mb |L2.120| " + - "L2.121[200,271] 1.05us 101mb |L2.121| " + - "L2.122[272,342] 1.05us 100mb |L2.122| " + - "L2.124[343,404] 1.05us 100mb |L2.124| " + - "L2.125[405,465] 1.05us 99mb |L2.125| " + - "L2.126[466,499] 1.05us 58mb |L2.126| " + - "L2.127[500,569] 1.05us 101mb |L2.127| " + - "L2.128[570,638] 1.05us 100mb |L2.128| " + - "L2.130[639,702] 1.05us 101mb |L2.130| " + - "L2.131[703,765] 1.05us 100mb |L2.131| " + - "L2.132[766,799] 1.05us 56mb |L2.132| " + - "L2.133[800,865] 1.05us 101mb |L2.133| " + - "L2.134[866,899] 1.05us 53mb |L2.134| " "### ); } @@ -3020,63 +3008,66 @@ async fn actual_case_from_catalog_1() { - "WARNING: file L0.161[327,333] 336ns 183mb exceeds soft limit 100mb by more than 50%" - "WARNING: file L0.162[330,338] 340ns 231mb exceeds soft limit 100mb by more than 50%" - "WARNING: file L0.163[331,338] 341ns 232mb exceeds soft limit 100mb by more than 50%" - - "**** Final Output Files (17.64gb written)" + - "**** Final Output Files (15.47gb written)" - "L2 " - - "L2.578[134,149] 342ns 202mb |L2.578| " - - "L2.579[150,165] 342ns 218mb |L2.579| " - - "L2.580[166,176] 342ns 186mb |L2.580| " - - "L2.581[177,182] 342ns 150mb |L2.581| " - - "L2.582[183,197] 342ns 267mb |L2.582| " - - "L2.583[198,207] 342ns 157mb |L2.583| " - - "L2.584[208,220] 342ns 147mb |L2.584| " - - "L2.585[221,232] 342ns 270mb |L2.585| " - - "L2.588[233,253] 342ns 286mb |L2.588| " - - "L2.589[254,270] 342ns 289mb |L2.589| " - - "L2.590[271,281] 342ns 225mb |L2.590| " - - "L2.591[282,296] 342ns 234mb |L2.591| " - - "L2.592[297,302] 342ns 232mb |L2.592| " - - "L2.593[303,308] 342ns 244mb |L2.593| " - - "L2.594[309,314] 342ns 282mb |L2.594|" - - "L2.595[315,317] 342ns 214mb |L2.595|" - - "L2.596[318,320] 342ns 222mb |L2.596|" - - "L2.597[321,323] 342ns 146mb |L2.597|" - - "L2.598[324,326] 342ns 254mb |L2.598|" - - "L2.599[327,329] 342ns 197mb |L2.599|" - - "L2.600[330,332] 342ns 228mb |L2.600|" - - "L2.601[333,335] 342ns 199mb |L2.601|" - - "L2.602[336,338] 342ns 280mb |L2.602|" - - "L2.850[1,26] 342ns 101mb |L2.850| " - - "L2.853[69,85] 342ns 104mb |L2.853| " - - "L2.854[86,98] 342ns 107mb |L2.854| " - - "L2.861[27,48] 342ns 103mb |L2.861| " - - "L2.862[49,68] 342ns 98mb |L2.862| " - - "L2.863[99,108] 342ns 102mb |L2.863| " - - "L2.864[109,117] 342ns 91mb |L2.864| " - - "L2.865[118,124] 342ns 91mb |L2.865| " - - "L2.866[125,130] 342ns 107mb |L2.866| " - - "L2.867[131,133] 342ns 64mb |L2.867| " - - "L2.868[339,339] 342ns 25mb |L2.868|" - - "WARNING: file L2.578[134,149] 342ns 202mb exceeds soft limit 100mb by more than 50%" - - "WARNING: file L2.579[150,165] 342ns 218mb exceeds soft limit 100mb by more than 50%" - - "WARNING: file L2.580[166,176] 342ns 186mb exceeds soft limit 100mb by more than 50%" - - "WARNING: file L2.581[177,182] 342ns 150mb exceeds soft limit 100mb by more than 50%" - - "WARNING: file L2.582[183,197] 342ns 267mb exceeds soft limit 100mb by more than 50%" - - "WARNING: file L2.583[198,207] 342ns 157mb exceeds soft limit 100mb by more than 50%" - - "WARNING: file L2.585[221,232] 342ns 270mb exceeds soft limit 100mb by more than 50%" - - "WARNING: file L2.588[233,253] 342ns 286mb exceeds soft limit 100mb by more than 50%" - - "WARNING: file L2.589[254,270] 342ns 289mb exceeds soft limit 100mb by more than 50%" - - "WARNING: file L2.590[271,281] 342ns 225mb exceeds soft limit 100mb by more than 50%" - - "WARNING: file L2.591[282,296] 342ns 234mb exceeds soft limit 100mb by more than 50%" - - "WARNING: file L2.592[297,302] 342ns 232mb exceeds soft limit 100mb by more than 50%" - - "WARNING: file L2.593[303,308] 342ns 244mb exceeds soft limit 100mb by more than 50%" - - "WARNING: file L2.594[309,314] 342ns 282mb exceeds soft limit 100mb by more than 50%" - - "WARNING: file L2.595[315,317] 342ns 214mb exceeds soft limit 100mb by more than 50%" - - "WARNING: file L2.596[318,320] 342ns 222mb exceeds soft limit 100mb by more than 50%" - - "WARNING: file L2.598[324,326] 342ns 254mb exceeds soft limit 100mb by more than 50%" - - "WARNING: file L2.599[327,329] 342ns 197mb exceeds soft limit 100mb by more than 50%" - - "WARNING: file L2.600[330,332] 342ns 228mb exceeds soft limit 100mb by more than 50%" - - "WARNING: file L2.601[333,335] 342ns 199mb exceeds soft limit 100mb by more than 50%" - - "WARNING: file L2.602[336,338] 342ns 280mb exceeds soft limit 100mb by more than 50%" + - "L2.594[150,165] 342ns 218mb |L2.594| " + - "L2.595[166,171] 342ns 118mb |L2.595| " + - "L2.598[183,197] 342ns 267mb |L2.598| " + - "L2.599[198,207] 342ns 157mb |L2.599| " + - "L2.600[208,220] 342ns 147mb |L2.600| " + - "L2.601[221,232] 342ns 270mb |L2.601| " + - "L2.602[233,244] 342ns 147mb |L2.602| " + - "L2.603[245,253] 342ns 139mb |L2.603| " + - "L2.604[271,276] 342ns 117mb |L2.604| " + - "L2.605[277,281] 342ns 109mb |L2.605| " + - "L2.612[254,261] 342ns 105mb |L2.612| " + - "L2.613[262,270] 342ns 184mb |L2.613| " + - "L2.616[309,311] 342ns 101mb |L2.616|" + - "L2.617[312,314] 342ns 181mb |L2.617|" + - "L2.618[315,317] 342ns 214mb |L2.618|" + - "L2.619[318,320] 342ns 222mb |L2.619|" + - "L2.620[321,323] 342ns 146mb |L2.620|" + - "L2.621[324,326] 342ns 254mb |L2.621|" + - "L2.622[327,329] 342ns 197mb |L2.622|" + - "L2.623[330,332] 342ns 228mb |L2.623|" + - "L2.624[333,335] 342ns 199mb |L2.624|" + - "L2.625[336,337] 342ns 156mb |L2.625|" + - "L2.626[338,338] 342ns 124mb |L2.626|" + - "L2.628[1,36] 342ns 103mb |L2.628-| " + - "L2.629[37,71] 342ns 103mb |L2.629-| " + - "L2.630[72,83] 342ns 103mb |L2.630| " + - "L2.638[172,177] 342ns 109mb |L2.638| " + - "L2.639[178,182] 342ns 109mb |L2.639| " + - "L2.640[282,288] 342ns 100mb |L2.640| " + - "L2.643[300,303] 342ns 110mb |L2.643| " + - "L2.646[84,94] 342ns 107mb |L2.646| " + - "L2.647[95,104] 342ns 97mb |L2.647| " + - "L2.648[105,111] 342ns 86mb |L2.648| " + - "L2.649[112,119] 342ns 114mb |L2.649| " + - "L2.650[120,126] 342ns 98mb |L2.650| " + - "L2.651[127,130] 342ns 82mb |L2.651| " + - "L2.652[131,138] 342ns 108mb |L2.652| " + - "L2.653[139,145] 342ns 93mb |L2.653| " + - "L2.654[146,149] 342ns 77mb |L2.654| " + - "L2.655[289,293] 342ns 110mb |L2.655| " + - "L2.656[294,297] 342ns 82mb |L2.656| " + - "L2.657[298,299] 342ns 82mb |L2.657| " + - "L2.658[304,306] 342ns 113mb |L2.658| " + - "L2.659[307,308] 342ns 113mb |L2.659| " + - "L2.660[339,339] 342ns 25mb |L2.660|" + - "WARNING: file L2.594[150,165] 342ns 218mb exceeds soft limit 100mb by more than 50%" + - "WARNING: file L2.598[183,197] 342ns 267mb exceeds soft limit 100mb by more than 50%" + - "WARNING: file L2.599[198,207] 342ns 157mb exceeds soft limit 100mb by more than 50%" + - "WARNING: file L2.601[221,232] 342ns 270mb exceeds soft limit 100mb by more than 50%" + - "WARNING: file L2.613[262,270] 342ns 184mb exceeds soft limit 100mb by more than 50%" + - "WARNING: file L2.617[312,314] 342ns 181mb exceeds soft limit 100mb by more than 50%" + - "WARNING: file L2.618[315,317] 342ns 214mb exceeds soft limit 100mb by more than 50%" + - "WARNING: file L2.619[318,320] 342ns 222mb exceeds soft limit 100mb by more than 50%" + - "WARNING: file L2.621[324,326] 342ns 254mb exceeds soft limit 100mb by more than 50%" + - "WARNING: file L2.622[327,329] 342ns 197mb exceeds soft limit 100mb by more than 50%" + - "WARNING: file L2.623[330,332] 342ns 228mb exceeds soft limit 100mb by more than 50%" + - "WARNING: file L2.624[333,335] 342ns 199mb exceeds soft limit 100mb by more than 50%" + - "WARNING: file L2.625[336,337] 342ns 156mb exceeds soft limit 100mb by more than 50%" "### ); } diff --git a/compactor/tests/layouts/many_files.rs b/compactor/tests/layouts/many_files.rs index 1b78518077..47f724fcd5 100644 --- a/compactor/tests/layouts/many_files.rs +++ b/compactor/tests/layouts/many_files.rs @@ -4670,17 +4670,17 @@ async fn l0s_almost_needing_vertical_split() { - "L0.998[24,100] 1.02us |-----------------------------------------L0.998-----------------------------------------|" - "L0.999[24,100] 1.02us |-----------------------------------------L0.999-----------------------------------------|" - "L0.1000[24,100] 1.02us |----------------------------------------L0.1000-----------------------------------------|" - - "**** Final Output Files (6.5gb written)" + - "**** Final Output Files (5.23gb written)" - "L2 " - - "L2.3141[24,37] 1.02us 108mb|---L2.3141---| " - - "L2.3150[38,49] 1.02us 102mb |--L2.3150--| " - - "L2.3151[50,60] 1.02us 93mb |-L2.3151-| " - - "L2.3152[61,63] 1.02us 37mb |L2.3152| " - - "L2.3153[64,73] 1.02us 101mb |L2.3153-| " - - "L2.3154[74,82] 1.02us 90mb |L2.3154| " - - "L2.3155[83,90] 1.02us 101mb |L2.3155| " - - "L2.3156[91,98] 1.02us 93mb |L2.3156| " - - "L2.3157[99,100] 1.02us 26mb |L2.3157|" + - "L2.3086[24,35] 1.02us 102mb|--L2.3086--| " + - "L2.3095[36,47] 1.02us 105mb |--L2.3095--| " + - "L2.3096[48,58] 1.02us 95mb |-L2.3096-| " + - "L2.3097[59,65] 1.02us 76mb |L2.3097| " + - "L2.3098[66,76] 1.02us 106mb |-L2.3098-| " + - "L2.3099[77,86] 1.02us 96mb |L2.3099-| " + - "L2.3100[87,90] 1.02us 53mb |L2.3100| " + - "L2.3101[91,98] 1.02us 90mb |L2.3101| " + - "L2.3102[99,100] 1.02us 26mb |L2.3102|" "### ); } diff --git a/compactor/tests/layouts/stuck.rs b/compactor/tests/layouts/stuck.rs index ff81c28800..c9e7996c0c 100644 --- a/compactor/tests/layouts/stuck.rs +++ b/compactor/tests/layouts/stuck.rs @@ -1111,81 +1111,78 @@ async fn stuck_l0() { - "L2.59[1686863759000000000,1686867839000000000] 1686928811.43s 96mb |--L2.59--| " - "L2.74[1686867899000000000,1686868319000000000] 1686928811.43s 14mb |L2.74| " - "L2.78[1686868379000000000,1686873599000000000] 1686928118.43s 39mb |---L2.78----| " - - "**** Final Output Files (51.79gb written)" - - "L1 " - - "L1.1998[1686873236374845980,1686873480877117666] 1686936871.55s 48mb |L1.1998|" - - "L1.1999[1686873480877117667,1686873599000000000] 1686936871.55s 23mb |L1.1999|" + - "**** Final Output Files (43.43gb written)" - "L2 " - - "L2.1842[1686841379000000000,1686842394885830950] 1686936871.55s 100mb|L2.1842| " - - "L2.1843[1686842394885830951,1686842786390926609] 1686936871.55s 39mb |L2.1843| " - - "L2.1844[1686842786390926610,1686843316416262506] 1686936871.55s 100mb |L2.1844| " - - "L2.1849[1686843316416262507,1686843684225377382] 1686936871.55s 100mb |L2.1849| " - - "L2.1850[1686843684225377383,1686844052034492257] 1686936871.55s 100mb |L2.1850| " - - "L2.1851[1686844052034492258,1686844193781853218] 1686936871.55s 39mb |L2.1851| " - - "L2.1852[1686844193781853219,1686844848044943955] 1686936871.55s 100mb |L2.1852| " - - "L2.1857[1686844848044943956,1686845161518311556] 1686936871.55s 100mb |L2.1857| " - - "L2.1858[1686845161518311557,1686845474991679156] 1686936871.55s 100mb |L2.1858| " - - "L2.1859[1686845474991679157,1686845579000000000] 1686936871.55s 33mb |L2.1859| " - - "L2.1867[1686845579000000001,1686846586992441776] 1686936871.55s 100mb |L2.1867| " - - "L2.1872[1686846586992441777,1686847213091266628] 1686936871.55s 100mb |L2.1872| " - - "L2.1873[1686847213091266629,1686847594984883551] 1686936871.55s 61mb |L2.1873| " - - "L2.1874[1686847594984883552,1686847967133303901] 1686936871.55s 100mb |L2.1874| " - - "L2.1875[1686847967133303902,1686848339281724250] 1686936871.55s 100mb |L2.1875| " - - "L2.1876[1686848339281724251,1686848602977306099] 1686936871.55s 71mb |L2.1876| " - - "L2.1877[1686848602977306100,1686849027096192900] 1686936871.55s 100mb |L2.1877| " - - "L2.1878[1686849027096192901,1686849451215079700] 1686936871.55s 100mb |L2.1878| " - - "L2.1879[1686849451215079701,1686849779000000000] 1686936871.55s 77mb |L2.1879| " - - "L2.1887[1686849779000000001,1686850474198233689] 1686936871.55s 100mb |L2.1887| " - - "L2.1888[1686850474198233690,1686851169396467377] 1686936871.55s 100mb |L2.1888| " - - "L2.1889[1686851169396467378,1686851644365067698] 1686936871.55s 68mb |L2.1889| " - - "L2.1890[1686851644365067699,1686852420659097557] 1686936871.55s 100mb |L2.1890| " - - "L2.1895[1686852420659097558,1686852801563322990] 1686936871.55s 100mb |L2.1895| " - - "L2.1896[1686852801563322991,1686853182467548422] 1686936871.55s 100mb |L2.1896| " - - "L2.1897[1686853182467548423,1686853509730135395] 1686936871.55s 86mb |L2.1897| " - - "L2.1898[1686853509730135396,1686853969686755621] 1686936871.55s 100mb |L2.1898| " - - "L2.1899[1686853969686755622,1686854429643375846] 1686936871.55s 100mb |L2.1899| " - - "L2.1900[1686854429643375847,1686854819000000000] 1686936871.55s 85mb |L2.1900| " - - "L2.1908[1686854819000000001,1686855825796452886] 1686936871.55s 100mb |L2.1908| " - - "L2.1913[1686855825796452887,1686856447395293366] 1686936871.55s 100mb |L2.1913| " - - "L2.1914[1686856447395293367,1686856832592905771] 1686936871.55s 62mb |L2.1914| " - - "L2.1915[1686856832592905772,1686857205112223577] 1686936871.55s 100mb |L2.1915| " - - "L2.1916[1686857205112223578,1686857577631541382] 1686936871.55s 100mb |L2.1916| " - - "L2.1917[1686857577631541383,1686857839389339451] 1686936871.55s 70mb |L2.1917| " - - "L2.1918[1686857839389339452,1686858253462843732] 1686936871.55s 100mb |L2.1918| " - - "L2.1919[1686858253462843733,1686858667536348012] 1686936871.55s 100mb |L2.1919| " - - "L2.1920[1686858667536348013,1686859019000000000] 1686936871.55s 85mb |L2.1920| " - - "L2.1928[1686859019000000001,1686859728979165803] 1686936871.55s 100mb |L2.1928| " - - "L2.1929[1686859728979165804,1686860438958331605] 1686936871.55s 100mb |L2.1929| " - - "L2.1931[1686860438958331606,1686860836834049919] 1686936871.55s 100mb |L2.1931| " - - "L2.1932[1686860836834049920,1686861148937490634] 1686936871.55s 78mb |L2.1932| " - - "L2.1933[1686861148937490635,1686861914356348643] 1686936871.55s 100mb |L2.1933| " - - "L2.1938[1686861914356348644,1686862293494638783] 1686936871.55s 100mb |L2.1938| " - - "L2.1939[1686862293494638784,1686862672632928922] 1686936871.55s 100mb |L2.1939| " - - "L2.1940[1686862672632928923,1686862679775206651] 1686936871.55s 2mb |L2.1940| " - - "L2.1941[1686862679775206652,1686863077953136510] 1686936871.55s 100mb |L2.1941| " - - "L2.1942[1686863077953136511,1686863476131066368] 1686936871.55s 100mb |L2.1942| " - - "L2.1943[1686863476131066369,1686863699000000000] 1686936871.55s 56mb |L2.1943| " - - "L2.1951[1686863699000000001,1686864349392123893] 1686936871.55s 100mb |L2.1951| " - - "L2.1952[1686864349392123894,1686864999784247785] 1686936871.55s 100mb |L2.1952| " - - "L2.1953[1686864999784247786,1686865607586435465] 1686936871.55s 93mb |L2.1953| " - - "L2.1954[1686865607586435466,1686866298103608885] 1686936871.55s 100mb |L2.1954| " - - "L2.1959[1686866298103608886,1686866809270889026] 1686936871.55s 100mb |L2.1959| " - - "L2.1964[1686866809270889027,1686867189314140837] 1686936871.55s 100mb |L2.1964| " - - "L2.1965[1686867189314140838,1686867569357392647] 1686936871.55s 100mb |L2.1965| " - - "L2.1966[1686867569357392648,1686867839000000000] 1686936871.55s 71mb |L2.1966| " - - "L2.1974[1686867839000000001,1686868754092704252] 1686936871.55s 100mb |L2.1974| " - - "L2.1975[1686868754092704253,1686869669185408503] 1686936871.55s 100mb |L2.1975| " - - "L2.1976[1686869669185408504,1686869927156879962] 1686936871.55s 28mb |L2.1976| " - - "L2.1984[1686869927156879963,1686870583552763572] 1686936871.55s 100mb |L2.1984| " - - "L2.1989[1686870583552763573,1686870997054710512] 1686936871.55s 100mb |L2.1989|" - - "L2.1990[1686870997054710513,1686871410556657451] 1686936871.55s 100mb |L2.1990|" - - "L2.1992[1686871410556657452,1686871702557232840] 1686936871.55s 100mb |L2.1992|" - - "L2.1993[1686871702557232841,1686871994557808228] 1686936871.55s 100mb |L2.1993|" - - "L2.1994[1686871994557808229,1686872015313759923] 1686936871.55s 7mb |L2.1994|" - - "L2.1997[1686873480877117667,1686873599000000000] 1686936871.55s 16mb |L2.1997|" - - "L2.2002[1686872015313759924,1686872503834880725] 1686936871.55s 100mb |L2.2002|" - - "L2.2003[1686872503834880726,1686872992356001526] 1686936871.55s 100mb |L2.2003|" - - "L2.2004[1686872992356001527,1686873480877117666] 1686936871.55s 100mb |L2.2004|" + - "L2.1463[1686841379000000000,1686842332558996896] 1686936871.55s 100mb|L2.1463| " + - "L2.1468[1686842332558996897,1686842963082388201] 1686936871.55s 100mb |L2.1468| " + - "L2.1469[1686842963082388202,1686843593605779505] 1686936871.55s 100mb |L2.1469| " + - "L2.1471[1686843593605779506,1686844269803455042] 1686936871.55s 100mb |L2.1471| " + - "L2.1476[1686844269803455043,1686844592758068181] 1686936871.55s 100mb |L2.1476| " + - "L2.1477[1686844592758068182,1686844915712681319] 1686936871.55s 100mb |L2.1477| " + - "L2.1478[1686844915712681320,1686844946001130578] 1686936871.55s 9mb |L2.1478| " + - "L2.1479[1686844946001130579,1686845261138897644] 1686936871.55s 100mb |L2.1479| " + - "L2.1480[1686845261138897645,1686845576276664709] 1686936871.55s 100mb |L2.1480| " + - "L2.1481[1686845576276664710,1686845579000000000] 1686936871.55s 885kb |L2.1481| " + - "L2.1489[1686845579000000001,1686846612945515506] 1686936871.55s 100mb |L2.1489| " + - "L2.1494[1686846612945515507,1686847302242526939] 1686936871.55s 100mb |L2.1494| " + - "L2.1499[1686847302242526940,1686847769313756192] 1686936871.55s 100mb |L2.1499| " + - "L2.1500[1686847769313756193,1686848236384985444] 1686936871.55s 100mb |L2.1500| " + - "L2.1502[1686848236384985445,1686848816392993760] 1686936871.55s 100mb |L2.1502| " + - "L2.1507[1686848816392993761,1686849165082054031] 1686936871.55s 100mb |L2.1507| " + - "L2.1508[1686849165082054032,1686849513771114301] 1686936871.55s 100mb |L2.1508| " + - "L2.1509[1686849513771114302,1686849779000000000] 1686936871.55s 76mb |L2.1509| " + - "L2.1510[1686849779000000001,1686850288711664442] 1686936871.55s 100mb |L2.1510| " + - "L2.1511[1686850288711664443,1686850559000000000] 1686936871.55s 53mb |L2.1511| " + - "L2.1519[1686850559000000001,1686851147210677461] 1686936871.55s 100mb |L2.1519| " + - "L2.1520[1686851147210677462,1686851735421354921] 1686936871.55s 100mb |L2.1520| " + - "L2.1521[1686851735421354922,1686852240527466641] 1686936871.55s 86mb |L2.1521| " + - "L2.1522[1686852240527466642,1686852812866488092] 1686936871.55s 100mb |L2.1522| " + - "L2.1523[1686852812866488093,1686853385205509542] 1686936871.55s 100mb |L2.1523| " + - "L2.1525[1686853385205509543,1686853965359592641] 1686936871.55s 100mb |L2.1525| " + - "L2.1530[1686853965359592642,1686854382616966390] 1686936871.55s 100mb |L2.1530| " + - "L2.1531[1686854382616966391,1686854799874340138] 1686936871.55s 100mb |L2.1531| " + - "L2.1532[1686854799874340139,1686854819000000000] 1686936871.55s 5mb |L2.1532| " + - "L2.1540[1686854819000000001,1686855555092837650] 1686936871.55s 100mb |L2.1540| " + - "L2.1541[1686855555092837651,1686856291185675299] 1686936871.55s 100mb |L2.1541| " + - "L2.1542[1686856291185675300,1686856502561319445] 1686936871.55s 29mb |L2.1542| " + - "L2.1543[1686856502561319446,1686857135555597834] 1686936871.55s 100mb |L2.1543| " + - "L2.1548[1686857135555597835,1686857409590080289] 1686936871.55s 100mb |L2.1548| " + - "L2.1549[1686857409590080290,1686857683624562743] 1686936871.55s 100mb |L2.1549| " + - "L2.1550[1686857683624562744,1686857768549876222] 1686936871.55s 31mb |L2.1550| " + - "L2.1551[1686857768549876223,1686858217039101175] 1686936871.55s 100mb |L2.1551| " + - "L2.1552[1686858217039101176,1686858665528326127] 1686936871.55s 100mb |L2.1552| " + - "L2.1554[1686858665528326128,1686859092257365665] 1686936871.55s 100mb |L2.1554| " + - "L2.1555[1686859092257365666,1686859499000000000] 1686936871.55s 95mb |L2.1555| " + - "L2.1563[1686859499000000001,1686860075111679382] 1686936871.55s 100mb |L2.1563| " + - "L2.1564[1686860075111679383,1686860651223358763] 1686936871.55s 100mb |L2.1564| " + - "L2.1565[1686860651223358764,1686861094471633615] 1686936871.55s 77mb |L2.1565| " + - "L2.1573[1686861094471633616,1686861597923995018] 1686936871.55s 100mb |L2.1573| " + - "L2.1574[1686861597923995019,1686862101376356420] 1686936871.55s 100mb |L2.1574| " + - "L2.1582[1686862101376356421,1686862532038334079] 1686936871.55s 100mb |L2.1582| " + - "L2.1583[1686862532038334080,1686862962700311737] 1686936871.55s 100mb |L2.1583| " + - "L2.1584[1686862962700311738,1686863391646317185] 1686936871.55s 100mb |L2.1584| " + - "L2.1592[1686863391646317186,1686864106500589152] 1686936871.55s 100mb |L2.1592| " + - "L2.1593[1686864106500589153,1686864821354861118] 1686936871.55s 100mb |L2.1593| " + - "L2.1594[1686864821354861119,1686865116328768791] 1686936871.55s 41mb |L2.1594| " + - "L2.1595[1686865116328768792,1686865695735321041] 1686936871.55s 100mb |L2.1595| " + - "L2.1596[1686865695735321042,1686866275141873290] 1686936871.55s 100mb |L2.1596| " + - "L2.1598[1686866275141873291,1686866862140786963] 1686936871.55s 100mb |L2.1598| " + - "L2.1603[1686866862140786964,1686867205172843469] 1686936871.55s 100mb |L2.1603| " + - "L2.1604[1686867205172843470,1686867548204899974] 1686936871.55s 100mb |L2.1604| " + - "L2.1605[1686867548204899975,1686867839000000000] 1686936871.55s 85mb |L2.1605| " + - "L2.1613[1686867839000000001,1686869156057291877] 1686936871.55s 100mb |L2.1613| " + - "L2.1618[1686869156057291878,1686869793900296627] 1686936871.55s 100mb |L2.1618| " + - "L2.1619[1686869793900296628,1686870431743301376] 1686936871.55s 100mb |L2.1619| " + - "L2.1620[1686870431743301377,1686870473114583753] 1686936871.55s 6mb |L2.1620|" + - "L2.1628[1686870473114583754,1686870911305888930] 1686936871.55s 100mb |L2.1628|" + - "L2.1629[1686870911305888931,1686871349497194106] 1686936871.55s 100mb |L2.1629|" + - "L2.1638[1686871349497194107,1686871921260674819] 1686936871.55s 100mb |L2.1638|" + - "L2.1643[1686871921260674820,1686872342671651604] 1686936871.55s 100mb |L2.1643|" + - "L2.1648[1686872342671651605,1686872646602704048] 1686936871.55s 100mb |L2.1648|" + - "L2.1649[1686872646602704049,1686872950533756491] 1686936871.55s 100mb |L2.1649|" + - "L2.1650[1686872950533756492,1686873064787630789] 1686936871.55s 38mb |L2.1650|" + - "L2.1651[1686873064787630790,1686873383892289532] 1686936871.55s 100mb |L2.1651|" + - "L2.1652[1686873383892289533,1686873599000000000] 1686936871.55s 67mb |L2.1652|" "### ); } @@ -1702,31 +1699,32 @@ async fn stuck_l0_large_l0s() { - "L0.198[197,1970000] 197ns 10b|----------------------------------------L0.198-----------------------------------------| " - "L0.199[198,1980000] 198ns 10b|----------------------------------------L0.199-----------------------------------------| " - "L0.200[199,1990000] 199ns 10b|----------------------------------------L0.200-----------------------------------------| " - - "**** Final Output Files (14.48gb written)" - - "L1 " - - "L1.3770[1752,1811] 199ns 62mb|L1.3770| " + - "**** Final Output Files (11.25gb written)" - "L2 " - - "L2.3711[1812,1990000] 199ns 190mb|----------------------------------------L2.3711----------------------------------------| " - - "L2.3743[1,102] 199ns 100mb|L2.3743| " - - "L2.3744[103,203] 199ns 100mb|L2.3744| " - - "L2.3745[204,305] 199ns 101mb|L2.3745| " - - "L2.3763[1165,1255] 199ns 101mb|L2.3763| " - - "L2.3771[306,407] 199ns 101mb|L2.3771| " - - "L2.3772[408,508] 199ns 100mb|L2.3772| " - - "L2.3773[509,606] 199ns 99mb|L2.3773| " - - "L2.3774[607,711] 199ns 101mb|L2.3774| " - - "L2.3775[712,815] 199ns 100mb|L2.3775| " - - "L2.3776[816,904] 199ns 87mb|L2.3776| " - - "L2.3777[905,1011] 199ns 101mb|L2.3777| " - - "L2.3778[1012,1117] 199ns 100mb|L2.3778| " - - "L2.3779[1118,1164] 199ns 46mb|L2.3779| " - - "L2.3780[1256,1350] 199ns 100mb|L2.3780| " - - "L2.3781[1351,1444] 199ns 99mb|L2.3781| " - - "L2.3782[1445,1480] 199ns 40mb|L2.3782| " - - "L2.3783[1481,1580] 199ns 100mb|L2.3783| " - - "L2.3784[1581,1679] 199ns 99mb|L2.3784| " - - "L2.3785[1680,1751] 199ns 74mb|L2.3785| " - - "WARNING: file L2.3711[1812,1990000] 199ns 190mb exceeds soft limit 100mb by more than 50%" + - "L2.3439[1812,2716] 199ns 190mb|L2.3439| " + - "L2.3523[1,102] 199ns 101mb|L2.3523| " + - "L2.3546[103,204] 199ns 101mb|L2.3546| " + - "L2.3547[205,305] 199ns 100mb|L2.3547| " + - "L2.3548[306,361] 199ns 57mb|L2.3548| " + - "L2.3549[362,463] 199ns 101mb|L2.3549| " + - "L2.3550[464,564] 199ns 100mb|L2.3550| " + - "L2.3551[565,619] 199ns 56mb|L2.3551| " + - "L2.3552[620,720] 199ns 101mb|L2.3552| " + - "L2.3553[721,820] 199ns 100mb|L2.3553| " + - "L2.3554[821,875] 199ns 56mb|L2.3554| " + - "L2.3555[876,975] 199ns 100mb|L2.3555| " + - "L2.3556[976,1074] 199ns 99mb|L2.3556| " + - "L2.3557[1075,1138] 199ns 66mb|L2.3557| " + - "L2.3558[1139,1240] 199ns 101mb|L2.3558| " + - "L2.3559[1241,1341] 199ns 100mb|L2.3559| " + - "L2.3560[1342,1396] 199ns 56mb|L2.3560| " + - "L2.3561[1397,1497] 199ns 101mb|L2.3561| " + - "L2.3562[1498,1597] 199ns 100mb|L2.3562| " + - "L2.3563[1598,1652] 199ns 56mb|L2.3563| " + - "L2.3564[1653,1751] 199ns 101mb|L2.3564| " + - "L2.3565[1752,1811] 199ns 62mb|L2.3565| " + - "L2.3566[2717,1990000] 199ns 2kb|----------------------------------------L2.3566----------------------------------------| " + - "WARNING: file L2.3439[1812,2716] 199ns 190mb exceeds soft limit 100mb by more than 50%" "### ); } @@ -3027,17 +3025,805 @@ async fn split_precent_loop() { - "L1.3[1676005158277000000,1676010156669000000] 1676010160.05s 58mb |L1.3| " - "WARNING: file L0.40[1676020762355000000,1676036230752000000] 1676036233.84s 159mb exceeds soft limit 100mb by more than 50%" - "WARNING: file L0.43[1676039845773000000,1676063836202000000] 1676063839.07s 242mb exceeds soft limit 100mb by more than 50%" - - "**** Final Output Files (4.17gb written)" + - "**** Final Output Files (3.4gb written)" + - "L1 " + - "L1.260[1676045833054395546,1676050409609000000] 1676066475.26s 41mb |L1.260| " - "L2 " - - "L2.304[1676034607207000001,1676066212011000000] 1676066475.26s 286mb |-------------L2.304--------------| " - - "L2.321[1675987200001000000,1675993675383098793] 1676066475.26s 100mb|L2.321| " - - "L2.329[1675993675383098794,1676001131291506471] 1676066475.26s 100mb |L2.329| " - - "L2.330[1676001131291506472,1676008587199914148] 1676066475.26s 100mb |L2.330| " - - "L2.331[1676008587199914149,1676014352965372887] 1676066475.26s 77mb |L2.331| " - - "L2.332[1676014352965372888,1676023648281062782] 1676066475.26s 100mb |-L2.332-| " - - "L2.333[1676023648281062783,1676032943596752676] 1676066475.26s 100mb |-L2.333-| " - - "L2.334[1676032943596752677,1676034607207000000] 1676066475.26s 18mb |L2.334| " - - "WARNING: file L2.304[1676034607207000001,1676066212011000000] 1676066475.26s 286mb exceeds soft limit 100mb by more than 50%" + - "L2.228[1676050409609000001,1676066212011000000] 1676066475.26s 145mb |----L2.228-----| " + - "L2.251[1675987200001000000,1675995209209749739] 1676066475.26s 100mb|L2.251-| " + - "L2.261[1675995209209749740,1676003044683020379] 1676066475.26s 100mb |L2.261| " + - "L2.262[1676003044683020380,1676010880156291018] 1676066475.26s 100mb |L2.262| " + - "L2.263[1676010880156291019,1676018715629412205] 1676066475.26s 100mb |L2.263| " + - "L2.264[1676018715629412206,1676027900853050774] 1676066475.26s 100mb |-L2.264-| " + - "L2.265[1676027900853050775,1676037086076689342] 1676066475.26s 100mb |-L2.265-| " + - "L2.266[1676037086076689343,1676045833054395545] 1676066475.26s 95mb |L2.266-| " + "### + ); +} + +// This is a simplified version of a test generated from actual catalog contents (which was thousands of lines). +// The key attributes are: +// - there are enough bytes of L0 to trigger vertical splitting +// - there are enough L0 files that the individual files are tiny +// - there are lots of L1s that make it a pain to merge down from L0 +// - when the L0s get split, they're split into enough pieces that the algorigthm (pre-fix) would put the L0s back together in a single file. +// The result, prior to the fix motivating this test case, is that the L0s would be vertically split, then regrouped together in a single chain, +// so they get recompacted together, which again prompts the need for vertical splitting, resulting in an unproductive cycle. +#[tokio::test] +async fn very_big_overlapped_backlog() { + test_helpers::maybe_start_logging(); + + let max_files = 20; + let setup = layout_setup_builder() + .await + .with_max_num_files_per_plan(max_files) + .with_max_desired_file_size_bytes(MAX_DESIRED_FILE_SIZE) + .with_partition_timeout(Duration::from_millis(100000)) + .with_suppress_run_output() // remove this to debug + .build() + .await; + + let max_time: i64 = 200000; + let l0_cnt: i64 = 200; + let l0_interval = max_time / l0_cnt; + let l0_size = MAX_DESIRED_FILE_SIZE * 4 / l0_cnt as u64; + let l1_cnt = 100; + let l1_interval = max_time / l1_cnt; + + // Create 100s of overlapping L0s + for i in 0..l0_cnt { + setup + .partition + .create_parquet_file( + parquet_builder() + .with_min_time(i * l0_interval) + .with_max_time((i + 1) * l0_interval) + .with_compaction_level(CompactionLevel::Initial) + .with_max_l0_created_at(Time::from_timestamp_nanos(l1_cnt + i)) + .with_file_size_bytes(l0_size), + ) + .await; + } + + // Create a lot of L1s, on the same time range as the L0s + for i in 0..l1_cnt { + setup + .partition + .create_parquet_file( + parquet_builder() + .with_min_time(i * l1_interval) + .with_max_time((i + 1) * l1_interval - 1) + .with_compaction_level(CompactionLevel::FileNonOverlapped) + .with_max_l0_created_at(Time::from_timestamp_nanos(i)) + .with_file_size_bytes(MAX_DESIRED_FILE_SIZE), + ) + .await; + } + + // Create a lot of L2s, on the same time range as the L0s and L1s + for i in 0..l1_cnt { + setup + .partition + .create_parquet_file( + parquet_builder() + .with_min_time(i * l1_interval) + .with_max_time((i + 1) * l1_interval - 1) + .with_compaction_level(CompactionLevel::Final) + .with_max_l0_created_at(Time::from_timestamp_nanos(i)) + .with_file_size_bytes(MAX_DESIRED_FILE_SIZE), + ) + .await; + } + + insta::assert_yaml_snapshot!( + run_layout_scenario(&setup).await, + @r###" + --- + - "**** Input Files " + - "L0 " + - "L0.1[0,1000] 100ns 2mb |L0.1| " + - "L0.2[1000,2000] 101ns 2mb|L0.2| " + - "L0.3[2000,3000] 102ns 2mb|L0.3| " + - "L0.4[3000,4000] 103ns 2mb |L0.4| " + - "L0.5[4000,5000] 104ns 2mb |L0.5| " + - "L0.6[5000,6000] 105ns 2mb |L0.6| " + - "L0.7[6000,7000] 106ns 2mb |L0.7| " + - "L0.8[7000,8000] 107ns 2mb |L0.8| " + - "L0.9[8000,9000] 108ns 2mb |L0.9| " + - "L0.10[9000,10000] 109ns 2mb |L0.10| " + - "L0.11[10000,11000] 110ns 2mb |L0.11| " + - "L0.12[11000,12000] 111ns 2mb |L0.12| " + - "L0.13[12000,13000] 112ns 2mb |L0.13| " + - "L0.14[13000,14000] 113ns 2mb |L0.14| " + - "L0.15[14000,15000] 114ns 2mb |L0.15| " + - "L0.16[15000,16000] 115ns 2mb |L0.16| " + - "L0.17[16000,17000] 116ns 2mb |L0.17| " + - "L0.18[17000,18000] 117ns 2mb |L0.18| " + - "L0.19[18000,19000] 118ns 2mb |L0.19| " + - "L0.20[19000,20000] 119ns 2mb |L0.20| " + - "L0.21[20000,21000] 120ns 2mb |L0.21| " + - "L0.22[21000,22000] 121ns 2mb |L0.22| " + - "L0.23[22000,23000] 122ns 2mb |L0.23| " + - "L0.24[23000,24000] 123ns 2mb |L0.24| " + - "L0.25[24000,25000] 124ns 2mb |L0.25| " + - "L0.26[25000,26000] 125ns 2mb |L0.26| " + - "L0.27[26000,27000] 126ns 2mb |L0.27| " + - "L0.28[27000,28000] 127ns 2mb |L0.28| " + - "L0.29[28000,29000] 128ns 2mb |L0.29| " + - "L0.30[29000,30000] 129ns 2mb |L0.30| " + - "L0.31[30000,31000] 130ns 2mb |L0.31| " + - "L0.32[31000,32000] 131ns 2mb |L0.32| " + - "L0.33[32000,33000] 132ns 2mb |L0.33| " + - "L0.34[33000,34000] 133ns 2mb |L0.34| " + - "L0.35[34000,35000] 134ns 2mb |L0.35| " + - "L0.36[35000,36000] 135ns 2mb |L0.36| " + - "L0.37[36000,37000] 136ns 2mb |L0.37| " + - "L0.38[37000,38000] 137ns 2mb |L0.38| " + - "L0.39[38000,39000] 138ns 2mb |L0.39| " + - "L0.40[39000,40000] 139ns 2mb |L0.40| " + - "L0.41[40000,41000] 140ns 2mb |L0.41| " + - "L0.42[41000,42000] 141ns 2mb |L0.42| " + - "L0.43[42000,43000] 142ns 2mb |L0.43| " + - "L0.44[43000,44000] 143ns 2mb |L0.44| " + - "L0.45[44000,45000] 144ns 2mb |L0.45| " + - "L0.46[45000,46000] 145ns 2mb |L0.46| " + - "L0.47[46000,47000] 146ns 2mb |L0.47| " + - "L0.48[47000,48000] 147ns 2mb |L0.48| " + - "L0.49[48000,49000] 148ns 2mb |L0.49| " + - "L0.50[49000,50000] 149ns 2mb |L0.50| " + - "L0.51[50000,51000] 150ns 2mb |L0.51| " + - "L0.52[51000,52000] 151ns 2mb |L0.52| " + - "L0.53[52000,53000] 152ns 2mb |L0.53| " + - "L0.54[53000,54000] 153ns 2mb |L0.54| " + - "L0.55[54000,55000] 154ns 2mb |L0.55| " + - "L0.56[55000,56000] 155ns 2mb |L0.56| " + - "L0.57[56000,57000] 156ns 2mb |L0.57| " + - "L0.58[57000,58000] 157ns 2mb |L0.58| " + - "L0.59[58000,59000] 158ns 2mb |L0.59| " + - "L0.60[59000,60000] 159ns 2mb |L0.60| " + - "L0.61[60000,61000] 160ns 2mb |L0.61| " + - "L0.62[61000,62000] 161ns 2mb |L0.62| " + - "L0.63[62000,63000] 162ns 2mb |L0.63| " + - "L0.64[63000,64000] 163ns 2mb |L0.64| " + - "L0.65[64000,65000] 164ns 2mb |L0.65| " + - "L0.66[65000,66000] 165ns 2mb |L0.66| " + - "L0.67[66000,67000] 166ns 2mb |L0.67| " + - "L0.68[67000,68000] 167ns 2mb |L0.68| " + - "L0.69[68000,69000] 168ns 2mb |L0.69| " + - "L0.70[69000,70000] 169ns 2mb |L0.70| " + - "L0.71[70000,71000] 170ns 2mb |L0.71| " + - "L0.72[71000,72000] 171ns 2mb |L0.72| " + - "L0.73[72000,73000] 172ns 2mb |L0.73| " + - "L0.74[73000,74000] 173ns 2mb |L0.74| " + - "L0.75[74000,75000] 174ns 2mb |L0.75| " + - "L0.76[75000,76000] 175ns 2mb |L0.76| " + - "L0.77[76000,77000] 176ns 2mb |L0.77| " + - "L0.78[77000,78000] 177ns 2mb |L0.78| " + - "L0.79[78000,79000] 178ns 2mb |L0.79| " + - "L0.80[79000,80000] 179ns 2mb |L0.80| " + - "L0.81[80000,81000] 180ns 2mb |L0.81| " + - "L0.82[81000,82000] 181ns 2mb |L0.82| " + - "L0.83[82000,83000] 182ns 2mb |L0.83| " + - "L0.84[83000,84000] 183ns 2mb |L0.84| " + - "L0.85[84000,85000] 184ns 2mb |L0.85| " + - "L0.86[85000,86000] 185ns 2mb |L0.86| " + - "L0.87[86000,87000] 186ns 2mb |L0.87| " + - "L0.88[87000,88000] 187ns 2mb |L0.88| " + - "L0.89[88000,89000] 188ns 2mb |L0.89| " + - "L0.90[89000,90000] 189ns 2mb |L0.90| " + - "L0.91[90000,91000] 190ns 2mb |L0.91| " + - "L0.92[91000,92000] 191ns 2mb |L0.92| " + - "L0.93[92000,93000] 192ns 2mb |L0.93| " + - "L0.94[93000,94000] 193ns 2mb |L0.94| " + - "L0.95[94000,95000] 194ns 2mb |L0.95| " + - "L0.96[95000,96000] 195ns 2mb |L0.96| " + - "L0.97[96000,97000] 196ns 2mb |L0.97| " + - "L0.98[97000,98000] 197ns 2mb |L0.98| " + - "L0.99[98000,99000] 198ns 2mb |L0.99| " + - "L0.100[99000,100000] 199ns 2mb |L0.100| " + - "L0.101[100000,101000] 200ns 2mb |L0.101| " + - "L0.102[101000,102000] 201ns 2mb |L0.102| " + - "L0.103[102000,103000] 202ns 2mb |L0.103| " + - "L0.104[103000,104000] 203ns 2mb |L0.104| " + - "L0.105[104000,105000] 204ns 2mb |L0.105| " + - "L0.106[105000,106000] 205ns 2mb |L0.106| " + - "L0.107[106000,107000] 206ns 2mb |L0.107| " + - "L0.108[107000,108000] 207ns 2mb |L0.108| " + - "L0.109[108000,109000] 208ns 2mb |L0.109| " + - "L0.110[109000,110000] 209ns 2mb |L0.110| " + - "L0.111[110000,111000] 210ns 2mb |L0.111| " + - "L0.112[111000,112000] 211ns 2mb |L0.112| " + - "L0.113[112000,113000] 212ns 2mb |L0.113| " + - "L0.114[113000,114000] 213ns 2mb |L0.114| " + - "L0.115[114000,115000] 214ns 2mb |L0.115| " + - "L0.116[115000,116000] 215ns 2mb |L0.116| " + - "L0.117[116000,117000] 216ns 2mb |L0.117| " + - "L0.118[117000,118000] 217ns 2mb |L0.118| " + - "L0.119[118000,119000] 218ns 2mb |L0.119| " + - "L0.120[119000,120000] 219ns 2mb |L0.120| " + - "L0.121[120000,121000] 220ns 2mb |L0.121| " + - "L0.122[121000,122000] 221ns 2mb |L0.122| " + - "L0.123[122000,123000] 222ns 2mb |L0.123| " + - "L0.124[123000,124000] 223ns 2mb |L0.124| " + - "L0.125[124000,125000] 224ns 2mb |L0.125| " + - "L0.126[125000,126000] 225ns 2mb |L0.126| " + - "L0.127[126000,127000] 226ns 2mb |L0.127| " + - "L0.128[127000,128000] 227ns 2mb |L0.128| " + - "L0.129[128000,129000] 228ns 2mb |L0.129| " + - "L0.130[129000,130000] 229ns 2mb |L0.130| " + - "L0.131[130000,131000] 230ns 2mb |L0.131| " + - "L0.132[131000,132000] 231ns 2mb |L0.132| " + - "L0.133[132000,133000] 232ns 2mb |L0.133| " + - "L0.134[133000,134000] 233ns 2mb |L0.134| " + - "L0.135[134000,135000] 234ns 2mb |L0.135| " + - "L0.136[135000,136000] 235ns 2mb |L0.136| " + - "L0.137[136000,137000] 236ns 2mb |L0.137| " + - "L0.138[137000,138000] 237ns 2mb |L0.138| " + - "L0.139[138000,139000] 238ns 2mb |L0.139| " + - "L0.140[139000,140000] 239ns 2mb |L0.140| " + - "L0.141[140000,141000] 240ns 2mb |L0.141| " + - "L0.142[141000,142000] 241ns 2mb |L0.142| " + - "L0.143[142000,143000] 242ns 2mb |L0.143| " + - "L0.144[143000,144000] 243ns 2mb |L0.144| " + - "L0.145[144000,145000] 244ns 2mb |L0.145| " + - "L0.146[145000,146000] 245ns 2mb |L0.146| " + - "L0.147[146000,147000] 246ns 2mb |L0.147| " + - "L0.148[147000,148000] 247ns 2mb |L0.148| " + - "L0.149[148000,149000] 248ns 2mb |L0.149| " + - "L0.150[149000,150000] 249ns 2mb |L0.150| " + - "L0.151[150000,151000] 250ns 2mb |L0.151| " + - "L0.152[151000,152000] 251ns 2mb |L0.152| " + - "L0.153[152000,153000] 252ns 2mb |L0.153| " + - "L0.154[153000,154000] 253ns 2mb |L0.154| " + - "L0.155[154000,155000] 254ns 2mb |L0.155| " + - "L0.156[155000,156000] 255ns 2mb |L0.156| " + - "L0.157[156000,157000] 256ns 2mb |L0.157| " + - "L0.158[157000,158000] 257ns 2mb |L0.158| " + - "L0.159[158000,159000] 258ns 2mb |L0.159| " + - "L0.160[159000,160000] 259ns 2mb |L0.160| " + - "L0.161[160000,161000] 260ns 2mb |L0.161| " + - "L0.162[161000,162000] 261ns 2mb |L0.162| " + - "L0.163[162000,163000] 262ns 2mb |L0.163| " + - "L0.164[163000,164000] 263ns 2mb |L0.164| " + - "L0.165[164000,165000] 264ns 2mb |L0.165| " + - "L0.166[165000,166000] 265ns 2mb |L0.166| " + - "L0.167[166000,167000] 266ns 2mb |L0.167| " + - "L0.168[167000,168000] 267ns 2mb |L0.168| " + - "L0.169[168000,169000] 268ns 2mb |L0.169| " + - "L0.170[169000,170000] 269ns 2mb |L0.170| " + - "L0.171[170000,171000] 270ns 2mb |L0.171| " + - "L0.172[171000,172000] 271ns 2mb |L0.172| " + - "L0.173[172000,173000] 272ns 2mb |L0.173| " + - "L0.174[173000,174000] 273ns 2mb |L0.174| " + - "L0.175[174000,175000] 274ns 2mb |L0.175| " + - "L0.176[175000,176000] 275ns 2mb |L0.176| " + - "L0.177[176000,177000] 276ns 2mb |L0.177| " + - "L0.178[177000,178000] 277ns 2mb |L0.178| " + - "L0.179[178000,179000] 278ns 2mb |L0.179| " + - "L0.180[179000,180000] 279ns 2mb |L0.180| " + - "L0.181[180000,181000] 280ns 2mb |L0.181| " + - "L0.182[181000,182000] 281ns 2mb |L0.182| " + - "L0.183[182000,183000] 282ns 2mb |L0.183| " + - "L0.184[183000,184000] 283ns 2mb |L0.184|" + - "L0.185[184000,185000] 284ns 2mb |L0.185|" + - "L0.186[185000,186000] 285ns 2mb |L0.186|" + - "L0.187[186000,187000] 286ns 2mb |L0.187|" + - "L0.188[187000,188000] 287ns 2mb |L0.188|" + - "L0.189[188000,189000] 288ns 2mb |L0.189|" + - "L0.190[189000,190000] 289ns 2mb |L0.190|" + - "L0.191[190000,191000] 290ns 2mb |L0.191|" + - "L0.192[191000,192000] 291ns 2mb |L0.192|" + - "L0.193[192000,193000] 292ns 2mb |L0.193|" + - "L0.194[193000,194000] 293ns 2mb |L0.194|" + - "L0.195[194000,195000] 294ns 2mb |L0.195|" + - "L0.196[195000,196000] 295ns 2mb |L0.196|" + - "L0.197[196000,197000] 296ns 2mb |L0.197|" + - "L0.198[197000,198000] 297ns 2mb |L0.198|" + - "L0.199[198000,199000] 298ns 2mb |L0.199|" + - "L0.200[199000,200000] 299ns 2mb |L0.200|" + - "L1 " + - "L1.201[0,1999] 0ns 100mb |L1.201| " + - "L1.202[2000,3999] 1ns 100mb|L1.202| " + - "L1.203[4000,5999] 2ns 100mb |L1.203| " + - "L1.204[6000,7999] 3ns 100mb |L1.204| " + - "L1.205[8000,9999] 4ns 100mb |L1.205| " + - "L1.206[10000,11999] 5ns 100mb |L1.206| " + - "L1.207[12000,13999] 6ns 100mb |L1.207| " + - "L1.208[14000,15999] 7ns 100mb |L1.208| " + - "L1.209[16000,17999] 8ns 100mb |L1.209| " + - "L1.210[18000,19999] 9ns 100mb |L1.210| " + - "L1.211[20000,21999] 10ns 100mb |L1.211| " + - "L1.212[22000,23999] 11ns 100mb |L1.212| " + - "L1.213[24000,25999] 12ns 100mb |L1.213| " + - "L1.214[26000,27999] 13ns 100mb |L1.214| " + - "L1.215[28000,29999] 14ns 100mb |L1.215| " + - "L1.216[30000,31999] 15ns 100mb |L1.216| " + - "L1.217[32000,33999] 16ns 100mb |L1.217| " + - "L1.218[34000,35999] 17ns 100mb |L1.218| " + - "L1.219[36000,37999] 18ns 100mb |L1.219| " + - "L1.220[38000,39999] 19ns 100mb |L1.220| " + - "L1.221[40000,41999] 20ns 100mb |L1.221| " + - "L1.222[42000,43999] 21ns 100mb |L1.222| " + - "L1.223[44000,45999] 22ns 100mb |L1.223| " + - "L1.224[46000,47999] 23ns 100mb |L1.224| " + - "L1.225[48000,49999] 24ns 100mb |L1.225| " + - "L1.226[50000,51999] 25ns 100mb |L1.226| " + - "L1.227[52000,53999] 26ns 100mb |L1.227| " + - "L1.228[54000,55999] 27ns 100mb |L1.228| " + - "L1.229[56000,57999] 28ns 100mb |L1.229| " + - "L1.230[58000,59999] 29ns 100mb |L1.230| " + - "L1.231[60000,61999] 30ns 100mb |L1.231| " + - "L1.232[62000,63999] 31ns 100mb |L1.232| " + - "L1.233[64000,65999] 32ns 100mb |L1.233| " + - "L1.234[66000,67999] 33ns 100mb |L1.234| " + - "L1.235[68000,69999] 34ns 100mb |L1.235| " + - "L1.236[70000,71999] 35ns 100mb |L1.236| " + - "L1.237[72000,73999] 36ns 100mb |L1.237| " + - "L1.238[74000,75999] 37ns 100mb |L1.238| " + - "L1.239[76000,77999] 38ns 100mb |L1.239| " + - "L1.240[78000,79999] 39ns 100mb |L1.240| " + - "L1.241[80000,81999] 40ns 100mb |L1.241| " + - "L1.242[82000,83999] 41ns 100mb |L1.242| " + - "L1.243[84000,85999] 42ns 100mb |L1.243| " + - "L1.244[86000,87999] 43ns 100mb |L1.244| " + - "L1.245[88000,89999] 44ns 100mb |L1.245| " + - "L1.246[90000,91999] 45ns 100mb |L1.246| " + - "L1.247[92000,93999] 46ns 100mb |L1.247| " + - "L1.248[94000,95999] 47ns 100mb |L1.248| " + - "L1.249[96000,97999] 48ns 100mb |L1.249| " + - "L1.250[98000,99999] 49ns 100mb |L1.250| " + - "L1.251[100000,101999] 50ns 100mb |L1.251| " + - "L1.252[102000,103999] 51ns 100mb |L1.252| " + - "L1.253[104000,105999] 52ns 100mb |L1.253| " + - "L1.254[106000,107999] 53ns 100mb |L1.254| " + - "L1.255[108000,109999] 54ns 100mb |L1.255| " + - "L1.256[110000,111999] 55ns 100mb |L1.256| " + - "L1.257[112000,113999] 56ns 100mb |L1.257| " + - "L1.258[114000,115999] 57ns 100mb |L1.258| " + - "L1.259[116000,117999] 58ns 100mb |L1.259| " + - "L1.260[118000,119999] 59ns 100mb |L1.260| " + - "L1.261[120000,121999] 60ns 100mb |L1.261| " + - "L1.262[122000,123999] 61ns 100mb |L1.262| " + - "L1.263[124000,125999] 62ns 100mb |L1.263| " + - "L1.264[126000,127999] 63ns 100mb |L1.264| " + - "L1.265[128000,129999] 64ns 100mb |L1.265| " + - "L1.266[130000,131999] 65ns 100mb |L1.266| " + - "L1.267[132000,133999] 66ns 100mb |L1.267| " + - "L1.268[134000,135999] 67ns 100mb |L1.268| " + - "L1.269[136000,137999] 68ns 100mb |L1.269| " + - "L1.270[138000,139999] 69ns 100mb |L1.270| " + - "L1.271[140000,141999] 70ns 100mb |L1.271| " + - "L1.272[142000,143999] 71ns 100mb |L1.272| " + - "L1.273[144000,145999] 72ns 100mb |L1.273| " + - "L1.274[146000,147999] 73ns 100mb |L1.274| " + - "L1.275[148000,149999] 74ns 100mb |L1.275| " + - "L1.276[150000,151999] 75ns 100mb |L1.276| " + - "L1.277[152000,153999] 76ns 100mb |L1.277| " + - "L1.278[154000,155999] 77ns 100mb |L1.278| " + - "L1.279[156000,157999] 78ns 100mb |L1.279| " + - "L1.280[158000,159999] 79ns 100mb |L1.280| " + - "L1.281[160000,161999] 80ns 100mb |L1.281| " + - "L1.282[162000,163999] 81ns 100mb |L1.282| " + - "L1.283[164000,165999] 82ns 100mb |L1.283| " + - "L1.284[166000,167999] 83ns 100mb |L1.284| " + - "L1.285[168000,169999] 84ns 100mb |L1.285| " + - "L1.286[170000,171999] 85ns 100mb |L1.286| " + - "L1.287[172000,173999] 86ns 100mb |L1.287| " + - "L1.288[174000,175999] 87ns 100mb |L1.288| " + - "L1.289[176000,177999] 88ns 100mb |L1.289| " + - "L1.290[178000,179999] 89ns 100mb |L1.290| " + - "L1.291[180000,181999] 90ns 100mb |L1.291| " + - "L1.292[182000,183999] 91ns 100mb |L1.292| " + - "L1.293[184000,185999] 92ns 100mb |L1.293|" + - "L1.294[186000,187999] 93ns 100mb |L1.294|" + - "L1.295[188000,189999] 94ns 100mb |L1.295|" + - "L1.296[190000,191999] 95ns 100mb |L1.296|" + - "L1.297[192000,193999] 96ns 100mb |L1.297|" + - "L1.298[194000,195999] 97ns 100mb |L1.298|" + - "L1.299[196000,197999] 98ns 100mb |L1.299|" + - "L1.300[198000,199999] 99ns 100mb |L1.300|" + - "L2 " + - "L2.301[0,1999] 0ns 100mb |L2.301| " + - "L2.302[2000,3999] 1ns 100mb|L2.302| " + - "L2.303[4000,5999] 2ns 100mb |L2.303| " + - "L2.304[6000,7999] 3ns 100mb |L2.304| " + - "L2.305[8000,9999] 4ns 100mb |L2.305| " + - "L2.306[10000,11999] 5ns 100mb |L2.306| " + - "L2.307[12000,13999] 6ns 100mb |L2.307| " + - "L2.308[14000,15999] 7ns 100mb |L2.308| " + - "L2.309[16000,17999] 8ns 100mb |L2.309| " + - "L2.310[18000,19999] 9ns 100mb |L2.310| " + - "L2.311[20000,21999] 10ns 100mb |L2.311| " + - "L2.312[22000,23999] 11ns 100mb |L2.312| " + - "L2.313[24000,25999] 12ns 100mb |L2.313| " + - "L2.314[26000,27999] 13ns 100mb |L2.314| " + - "L2.315[28000,29999] 14ns 100mb |L2.315| " + - "L2.316[30000,31999] 15ns 100mb |L2.316| " + - "L2.317[32000,33999] 16ns 100mb |L2.317| " + - "L2.318[34000,35999] 17ns 100mb |L2.318| " + - "L2.319[36000,37999] 18ns 100mb |L2.319| " + - "L2.320[38000,39999] 19ns 100mb |L2.320| " + - "L2.321[40000,41999] 20ns 100mb |L2.321| " + - "L2.322[42000,43999] 21ns 100mb |L2.322| " + - "L2.323[44000,45999] 22ns 100mb |L2.323| " + - "L2.324[46000,47999] 23ns 100mb |L2.324| " + - "L2.325[48000,49999] 24ns 100mb |L2.325| " + - "L2.326[50000,51999] 25ns 100mb |L2.326| " + - "L2.327[52000,53999] 26ns 100mb |L2.327| " + - "L2.328[54000,55999] 27ns 100mb |L2.328| " + - "L2.329[56000,57999] 28ns 100mb |L2.329| " + - "L2.330[58000,59999] 29ns 100mb |L2.330| " + - "L2.331[60000,61999] 30ns 100mb |L2.331| " + - "L2.332[62000,63999] 31ns 100mb |L2.332| " + - "L2.333[64000,65999] 32ns 100mb |L2.333| " + - "L2.334[66000,67999] 33ns 100mb |L2.334| " + - "L2.335[68000,69999] 34ns 100mb |L2.335| " + - "L2.336[70000,71999] 35ns 100mb |L2.336| " + - "L2.337[72000,73999] 36ns 100mb |L2.337| " + - "L2.338[74000,75999] 37ns 100mb |L2.338| " + - "L2.339[76000,77999] 38ns 100mb |L2.339| " + - "L2.340[78000,79999] 39ns 100mb |L2.340| " + - "L2.341[80000,81999] 40ns 100mb |L2.341| " + - "L2.342[82000,83999] 41ns 100mb |L2.342| " + - "L2.343[84000,85999] 42ns 100mb |L2.343| " + - "L2.344[86000,87999] 43ns 100mb |L2.344| " + - "L2.345[88000,89999] 44ns 100mb |L2.345| " + - "L2.346[90000,91999] 45ns 100mb |L2.346| " + - "L2.347[92000,93999] 46ns 100mb |L2.347| " + - "L2.348[94000,95999] 47ns 100mb |L2.348| " + - "L2.349[96000,97999] 48ns 100mb |L2.349| " + - "L2.350[98000,99999] 49ns 100mb |L2.350| " + - "L2.351[100000,101999] 50ns 100mb |L2.351| " + - "L2.352[102000,103999] 51ns 100mb |L2.352| " + - "L2.353[104000,105999] 52ns 100mb |L2.353| " + - "L2.354[106000,107999] 53ns 100mb |L2.354| " + - "L2.355[108000,109999] 54ns 100mb |L2.355| " + - "L2.356[110000,111999] 55ns 100mb |L2.356| " + - "L2.357[112000,113999] 56ns 100mb |L2.357| " + - "L2.358[114000,115999] 57ns 100mb |L2.358| " + - "L2.359[116000,117999] 58ns 100mb |L2.359| " + - "L2.360[118000,119999] 59ns 100mb |L2.360| " + - "L2.361[120000,121999] 60ns 100mb |L2.361| " + - "L2.362[122000,123999] 61ns 100mb |L2.362| " + - "L2.363[124000,125999] 62ns 100mb |L2.363| " + - "L2.364[126000,127999] 63ns 100mb |L2.364| " + - "L2.365[128000,129999] 64ns 100mb |L2.365| " + - "L2.366[130000,131999] 65ns 100mb |L2.366| " + - "L2.367[132000,133999] 66ns 100mb |L2.367| " + - "L2.368[134000,135999] 67ns 100mb |L2.368| " + - "L2.369[136000,137999] 68ns 100mb |L2.369| " + - "L2.370[138000,139999] 69ns 100mb |L2.370| " + - "L2.371[140000,141999] 70ns 100mb |L2.371| " + - "L2.372[142000,143999] 71ns 100mb |L2.372| " + - "L2.373[144000,145999] 72ns 100mb |L2.373| " + - "L2.374[146000,147999] 73ns 100mb |L2.374| " + - "L2.375[148000,149999] 74ns 100mb |L2.375| " + - "L2.376[150000,151999] 75ns 100mb |L2.376| " + - "L2.377[152000,153999] 76ns 100mb |L2.377| " + - "L2.378[154000,155999] 77ns 100mb |L2.378| " + - "L2.379[156000,157999] 78ns 100mb |L2.379| " + - "L2.380[158000,159999] 79ns 100mb |L2.380| " + - "L2.381[160000,161999] 80ns 100mb |L2.381| " + - "L2.382[162000,163999] 81ns 100mb |L2.382| " + - "L2.383[164000,165999] 82ns 100mb |L2.383| " + - "L2.384[166000,167999] 83ns 100mb |L2.384| " + - "L2.385[168000,169999] 84ns 100mb |L2.385| " + - "L2.386[170000,171999] 85ns 100mb |L2.386| " + - "L2.387[172000,173999] 86ns 100mb |L2.387| " + - "L2.388[174000,175999] 87ns 100mb |L2.388| " + - "L2.389[176000,177999] 88ns 100mb |L2.389| " + - "L2.390[178000,179999] 89ns 100mb |L2.390| " + - "L2.391[180000,181999] 90ns 100mb |L2.391| " + - "L2.392[182000,183999] 91ns 100mb |L2.392| " + - "L2.393[184000,185999] 92ns 100mb |L2.393|" + - "L2.394[186000,187999] 93ns 100mb |L2.394|" + - "L2.395[188000,189999] 94ns 100mb |L2.395|" + - "L2.396[190000,191999] 95ns 100mb |L2.396|" + - "L2.397[192000,193999] 96ns 100mb |L2.397|" + - "L2.398[194000,195999] 97ns 100mb |L2.398|" + - "L2.399[196000,197999] 98ns 100mb |L2.399|" + - "L2.400[198000,199999] 99ns 100mb |L2.400|" + - "**** Final Output Files (45.56gb written)" + - "L2 " + - "L2.1173[0,983] 104ns 100mb|L2.1173| " + - "L2.1174[984,1966] 104ns 100mb|L2.1174| " + - "L2.1175[1967,1999] 104ns 3mb|L2.1175| " + - "L2.1176[2000,2983] 108ns 100mb|L2.1176| " + - "L2.1177[2984,3966] 108ns 100mb |L2.1177| " + - "L2.1178[3967,3999] 108ns 3mb |L2.1178| " + - "L2.1179[4000,4978] 108ns 100mb |L2.1179| " + - "L2.1180[4979,5956] 108ns 100mb |L2.1180| " + - "L2.1181[5957,5999] 108ns 4mb |L2.1181| " + - "L2.1182[6000,6978] 108ns 100mb |L2.1182| " + - "L2.1183[6979,7956] 108ns 100mb |L2.1183| " + - "L2.1184[7957,7999] 108ns 5mb |L2.1184| " + - "L2.1185[8000,8979] 113ns 100mb |L2.1185| " + - "L2.1186[8980,9958] 113ns 100mb |L2.1186| " + - "L2.1187[9959,9999] 113ns 4mb |L2.1187| " + - "L2.1188[10000,10980] 113ns 100mb |L2.1188| " + - "L2.1189[10981,11960] 113ns 100mb |L2.1189| " + - "L2.1190[11961,11999] 113ns 4mb |L2.1190| " + - "L2.1191[12000,12980] 113ns 100mb |L2.1191| " + - "L2.1192[12981,13960] 113ns 100mb |L2.1192| " + - "L2.1193[13961,13999] 113ns 4mb |L2.1193| " + - "L2.1194[14000,14981] 117ns 100mb |L2.1194| " + - "L2.1195[14982,15962] 117ns 100mb |L2.1195| " + - "L2.1196[15963,15999] 117ns 4mb |L2.1196| " + - "L2.1197[16000,16980] 117ns 100mb |L2.1197| " + - "L2.1198[16981,17960] 117ns 100mb |L2.1198| " + - "L2.1199[17961,17999] 117ns 4mb |L2.1199| " + - "L2.1200[18000,18981] 123ns 100mb |L2.1200| " + - "L2.1201[18982,19962] 123ns 100mb |L2.1201| " + - "L2.1202[19963,19999] 123ns 4mb |L2.1202| " + - "L2.1203[20000,20981] 123ns 100mb |L2.1203| " + - "L2.1204[20982,21962] 123ns 100mb |L2.1204| " + - "L2.1205[21963,21999] 123ns 4mb |L2.1205| " + - "L2.1206[22000,22981] 127ns 100mb |L2.1206| " + - "L2.1207[22982,23962] 127ns 100mb |L2.1207| " + - "L2.1208[23963,23999] 127ns 4mb |L2.1208| " + - "L2.1209[24000,24984] 127ns 100mb |L2.1209| " + - "L2.1210[24985,25968] 127ns 100mb |L2.1210| " + - "L2.1211[25969,25999] 127ns 3mb |L2.1211| " + - "L2.1212[26000,26981] 131ns 100mb |L2.1212| " + - "L2.1213[26982,27962] 131ns 100mb |L2.1213| " + - "L2.1214[27963,27999] 131ns 4mb |L2.1214| " + - "L2.1215[28000,28978] 131ns 100mb |L2.1215| " + - "L2.1216[28979,29956] 131ns 100mb |L2.1216| " + - "L2.1217[29957,29999] 131ns 5mb |L2.1217| " + - "L2.1218[30000,30977] 131ns 100mb |L2.1218| " + - "L2.1219[30978,31954] 131ns 100mb |L2.1219| " + - "L2.1220[31955,31999] 131ns 5mb |L2.1220| " + - "L2.1221[32000,32981] 137ns 100mb |L2.1221| " + - "L2.1222[32982,33962] 137ns 100mb |L2.1222| " + - "L2.1223[33963,33999] 137ns 4mb |L2.1223| " + - "L2.1224[34000,34981] 137ns 100mb |L2.1224| " + - "L2.1225[34982,35962] 137ns 100mb |L2.1225| " + - "L2.1226[35963,35999] 137ns 4mb |L2.1226| " + - "L2.1227[36000,36981] 141ns 100mb |L2.1227| " + - "L2.1228[36982,37962] 141ns 100mb |L2.1228| " + - "L2.1229[37963,37999] 141ns 4mb |L2.1229| " + - "L2.1230[38000,38985] 141ns 100mb |L2.1230| " + - "L2.1231[38986,39970] 141ns 100mb |L2.1231| " + - "L2.1232[39971,39999] 141ns 3mb |L2.1232| " + - "L2.1233[40000,40983] 146ns 100mb |L2.1233| " + - "L2.1234[40984,41966] 146ns 100mb |L2.1234| " + - "L2.1235[41967,41999] 146ns 3mb |L2.1235| " + - "L2.1236[42000,42977] 146ns 100mb |L2.1236| " + - "L2.1237[42978,43954] 146ns 100mb |L2.1237| " + - "L2.1238[43955,43999] 146ns 5mb |L2.1238| " + - "L2.1239[44000,44976] 146ns 100mb |L2.1239| " + - "L2.1240[44977,45952] 146ns 100mb |L2.1240| " + - "L2.1241[45953,45999] 146ns 5mb |L2.1241| " + - "L2.1242[46000,46980] 151ns 100mb |L2.1242| " + - "L2.1243[46981,47960] 151ns 100mb |L2.1243| " + - "L2.1244[47961,47999] 151ns 4mb |L2.1244| " + - "L2.1245[48000,48980] 151ns 100mb |L2.1245| " + - "L2.1246[48981,49960] 151ns 100mb |L2.1246| " + - "L2.1247[49961,49999] 151ns 4mb |L2.1247| " + - "L2.1248[50000,50980] 151ns 100mb |L2.1248| " + - "L2.1249[50981,51960] 151ns 100mb |L2.1249| " + - "L2.1250[51961,51999] 151ns 4mb |L2.1250| " + - "L2.1251[52000,52981] 155ns 100mb |L2.1251| " + - "L2.1252[52982,53962] 155ns 100mb |L2.1252| " + - "L2.1253[53963,53999] 155ns 4mb |L2.1253| " + - "L2.1254[54000,54980] 155ns 100mb |L2.1254| " + - "L2.1255[54981,55960] 155ns 100mb |L2.1255| " + - "L2.1256[55961,55999] 155ns 4mb |L2.1256| " + - "L2.1257[56000,56981] 160ns 100mb |L2.1257| " + - "L2.1258[56982,57962] 160ns 100mb |L2.1258| " + - "L2.1259[57963,57999] 160ns 4mb |L2.1259| " + - "L2.1260[58000,58981] 160ns 100mb |L2.1260| " + - "L2.1261[58982,59962] 160ns 100mb |L2.1261| " + - "L2.1262[59963,59999] 160ns 4mb |L2.1262| " + - "L2.1263[60000,60980] 165ns 100mb |L2.1263| " + - "L2.1264[60981,61960] 165ns 100mb |L2.1264| " + - "L2.1265[61961,61999] 165ns 4mb |L2.1265| " + - "L2.1266[62000,62981] 165ns 100mb |L2.1266| " + - "L2.1267[62982,63962] 165ns 100mb |L2.1267| " + - "L2.1268[63963,63999] 165ns 4mb |L2.1268| " + - "L2.1269[64000,64980] 165ns 100mb |L2.1269| " + - "L2.1270[64981,65960] 165ns 100mb |L2.1270| " + - "L2.1271[65961,65999] 165ns 4mb |L2.1271| " + - "L2.1272[66000,66981] 169ns 100mb |L2.1272| " + - "L2.1273[66982,67962] 169ns 100mb |L2.1273| " + - "L2.1274[67963,67999] 169ns 4mb |L2.1274| " + - "L2.1275[68000,68980] 169ns 100mb |L2.1275| " + - "L2.1276[68981,69960] 169ns 100mb |L2.1276| " + - "L2.1277[69961,69999] 169ns 4mb |L2.1277| " + - "L2.1278[70000,70981] 175ns 100mb |L2.1278| " + - "L2.1279[70982,71962] 175ns 100mb |L2.1279| " + - "L2.1280[71963,71999] 175ns 4mb |L2.1280| " + - "L2.1281[72000,72981] 175ns 100mb |L2.1281| " + - "L2.1282[72982,73962] 175ns 100mb |L2.1282| " + - "L2.1283[73963,73999] 175ns 4mb |L2.1283| " + - "L2.1284[74000,74981] 179ns 100mb |L2.1284| " + - "L2.1285[74982,75962] 179ns 100mb |L2.1285| " + - "L2.1286[75963,75999] 179ns 4mb |L2.1286| " + - "L2.1287[76000,76984] 179ns 100mb |L2.1287| " + - "L2.1288[76985,77968] 179ns 100mb |L2.1288| " + - "L2.1289[77969,77999] 179ns 3mb |L2.1289| " + - "L2.1290[78000,78982] 184ns 100mb |L2.1290| " + - "L2.1291[78983,79964] 184ns 100mb |L2.1291| " + - "L2.1292[79965,79999] 184ns 4mb |L2.1292| " + - "L2.1293[80000,80977] 184ns 100mb |L2.1293| " + - "L2.1294[80978,81954] 184ns 100mb |L2.1294| " + - "L2.1295[81955,81999] 184ns 5mb |L2.1295| " + - "L2.1296[82000,82977] 184ns 100mb |L2.1296| " + - "L2.1297[82978,83954] 184ns 100mb |L2.1297| " + - "L2.1298[83955,83999] 184ns 5mb |L2.1298| " + - "L2.1299[84000,84983] 188ns 100mb |L2.1299| " + - "L2.1300[84984,85966] 188ns 100mb |L2.1300| " + - "L2.1301[85967,85999] 188ns 3mb |L2.1301| " + - "L2.1302[86000,86983] 192ns 100mb |L2.1302| " + - "L2.1303[86984,87966] 192ns 100mb |L2.1303| " + - "L2.1304[87967,87999] 192ns 3mb |L2.1304| " + - "L2.1305[88000,88978] 192ns 100mb |L2.1305| " + - "L2.1306[88979,89956] 192ns 100mb |L2.1306| " + - "L2.1307[89957,89999] 192ns 5mb |L2.1307| " + - "L2.1308[90000,90978] 192ns 100mb |L2.1308| " + - "L2.1309[90979,91956] 192ns 100mb |L2.1309| " + - "L2.1310[91957,91999] 192ns 5mb |L2.1310| " + - "L2.1311[92000,92979] 197ns 100mb |L2.1311| " + - "L2.1312[92980,93958] 197ns 100mb |L2.1312| " + - "L2.1313[93959,93999] 197ns 4mb |L2.1313| " + - "L2.1314[94000,94980] 197ns 100mb |L2.1314| " + - "L2.1315[94981,95960] 197ns 100mb |L2.1315| " + - "L2.1316[95961,95999] 197ns 4mb |L2.1316| " + - "L2.1317[96000,96980] 197ns 100mb |L2.1317| " + - "L2.1318[96981,97960] 197ns 100mb |L2.1318| " + - "L2.1319[97961,97999] 197ns 4mb |L2.1319| " + - "L2.1320[98000,98981] 201ns 100mb |L2.1320| " + - "L2.1321[98982,99962] 201ns 100mb |L2.1321| " + - "L2.1322[99963,99999] 201ns 4mb |L2.1322| " + - "L2.1323[100000,100980] 201ns 100mb |L2.1323| " + - "L2.1324[100981,101960] 201ns 100mb |L2.1324| " + - "L2.1325[101961,101999] 201ns 4mb |L2.1325| " + - "L2.1326[102000,102981] 207ns 100mb |L2.1326| " + - "L2.1327[102982,103962] 207ns 100mb |L2.1327| " + - "L2.1328[103963,103999] 207ns 4mb |L2.1328| " + - "L2.1329[104000,104981] 207ns 100mb |L2.1329| " + - "L2.1330[104982,105962] 207ns 100mb |L2.1330| " + - "L2.1331[105963,105999] 207ns 4mb |L2.1331| " + - "L2.1332[106000,106981] 211ns 100mb |L2.1332| " + - "L2.1333[106982,107962] 211ns 100mb |L2.1333| " + - "L2.1334[107963,107999] 211ns 4mb |L2.1334| " + - "L2.1335[108000,108984] 211ns 100mb |L2.1335| " + - "L2.1336[108985,109968] 211ns 100mb |L2.1336| " + - "L2.1337[109969,109999] 211ns 3mb |L2.1337| " + - "L2.1338[110000,110981] 215ns 100mb |L2.1338| " + - "L2.1339[110982,111962] 215ns 100mb |L2.1339| " + - "L2.1340[111963,111999] 215ns 4mb |L2.1340| " + - "L2.1341[112000,112978] 215ns 100mb |L2.1341| " + - "L2.1342[112979,113956] 215ns 100mb |L2.1342| " + - "L2.1343[113957,113999] 215ns 5mb |L2.1343| " + - "L2.1344[114000,114977] 215ns 100mb |L2.1344| " + - "L2.1345[114978,115954] 215ns 100mb |L2.1345| " + - "L2.1346[115955,115999] 215ns 5mb |L2.1346| " + - "L2.1347[116000,116981] 221ns 100mb |L2.1347| " + - "L2.1348[116982,117962] 221ns 100mb |L2.1348| " + - "L2.1349[117963,117999] 221ns 4mb |L2.1349| " + - "L2.1350[118000,118981] 221ns 100mb |L2.1350| " + - "L2.1351[118982,119962] 221ns 100mb |L2.1351| " + - "L2.1352[119963,119999] 221ns 4mb |L2.1352| " + - "L2.1353[120000,120981] 225ns 100mb |L2.1353| " + - "L2.1354[120982,121962] 225ns 100mb |L2.1354| " + - "L2.1355[121963,121999] 225ns 4mb |L2.1355| " + - "L2.1356[122000,122986] 225ns 100mb |L2.1356| " + - "L2.1357[122987,123972] 225ns 100mb |L2.1357| " + - "L2.1358[123973,123999] 225ns 3mb |L2.1358| " + - "L2.1359[124000,124984] 230ns 100mb |L2.1359| " + - "L2.1360[124985,125968] 230ns 100mb |L2.1360| " + - "L2.1361[125969,125999] 230ns 3mb |L2.1361| " + - "L2.1362[126000,126976] 230ns 100mb |L2.1362| " + - "L2.1363[126977,127952] 230ns 100mb |L2.1363| " + - "L2.1364[127953,127999] 230ns 5mb |L2.1364| " + - "L2.1365[128000,128976] 230ns 100mb |L2.1365| " + - "L2.1366[128977,129952] 230ns 100mb |L2.1366| " + - "L2.1367[129953,129999] 230ns 5mb |L2.1367| " + - "L2.1368[130000,130979] 235ns 100mb |L2.1368| " + - "L2.1369[130980,131958] 235ns 100mb |L2.1369| " + - "L2.1370[131959,131999] 235ns 4mb |L2.1370| " + - "L2.1371[132000,132980] 235ns 100mb |L2.1371| " + - "L2.1372[132981,133960] 235ns 100mb |L2.1372| " + - "L2.1373[133961,133999] 235ns 4mb |L2.1373| " + - "L2.1374[134000,134980] 235ns 100mb |L2.1374| " + - "L2.1375[134981,135960] 235ns 100mb |L2.1375| " + - "L2.1376[135961,135999] 235ns 4mb |L2.1376| " + - "L2.1377[136000,136981] 239ns 100mb |L2.1377| " + - "L2.1378[136982,137962] 239ns 100mb |L2.1378| " + - "L2.1379[137963,137999] 239ns 4mb |L2.1379| " + - "L2.1380[138000,138980] 239ns 100mb |L2.1380| " + - "L2.1381[138981,139960] 239ns 100mb |L2.1381| " + - "L2.1382[139961,139999] 239ns 4mb |L2.1382| " + - "L2.1383[140000,140981] 244ns 100mb |L2.1383| " + - "L2.1384[140982,141962] 244ns 100mb |L2.1384| " + - "L2.1385[141963,141999] 244ns 4mb |L2.1385| " + - "L2.1386[142000,142981] 244ns 100mb |L2.1386| " + - "L2.1387[142982,143962] 244ns 100mb |L2.1387| " + - "L2.1388[143963,143999] 244ns 4mb |L2.1388| " + - "L2.1389[144000,144981] 249ns 100mb |L2.1389| " + - "L2.1390[144982,145962] 249ns 100mb |L2.1390| " + - "L2.1391[145963,145999] 249ns 4mb |L2.1391| " + - "L2.1392[146000,146983] 249ns 100mb |L2.1392| " + - "L2.1393[146984,147966] 249ns 100mb |L2.1393| " + - "L2.1394[147967,147999] 249ns 3mb |L2.1394| " + - "L2.1395[148000,148980] 253ns 100mb |L2.1395| " + - "L2.1396[148981,149960] 253ns 100mb |L2.1396| " + - "L2.1397[149961,149999] 253ns 4mb |L2.1397| " + - "L2.1398[150000,150979] 253ns 100mb |L2.1398| " + - "L2.1399[150980,151958] 253ns 100mb |L2.1399| " + - "L2.1400[151959,151999] 253ns 4mb |L2.1400| " + - "L2.1401[152000,152978] 253ns 100mb |L2.1401| " + - "L2.1402[152979,153956] 253ns 100mb |L2.1402| " + - "L2.1403[153957,153999] 253ns 5mb |L2.1403| " + - "L2.1404[154000,154981] 259ns 100mb |L2.1404| " + - "L2.1405[154982,155962] 259ns 100mb |L2.1405| " + - "L2.1406[155963,155999] 259ns 4mb |L2.1406| " + - "L2.1407[156000,156981] 259ns 100mb |L2.1407| " + - "L2.1408[156982,157962] 259ns 100mb |L2.1408| " + - "L2.1409[157963,157999] 259ns 4mb |L2.1409| " + - "L2.1410[158000,158981] 263ns 100mb |L2.1410| " + - "L2.1411[158982,159962] 263ns 100mb |L2.1411| " + - "L2.1412[159963,159999] 263ns 4mb |L2.1412| " + - "L2.1413[160000,160985] 263ns 100mb |L2.1413| " + - "L2.1414[160986,161970] 263ns 100mb |L2.1414| " + - "L2.1415[161971,161999] 263ns 3mb |L2.1415| " + - "L2.1416[162000,162982] 268ns 100mb |L2.1416| " + - "L2.1417[162983,163964] 268ns 100mb |L2.1417| " + - "L2.1418[163965,163999] 268ns 4mb |L2.1418| " + - "L2.1419[164000,164977] 268ns 100mb |L2.1419| " + - "L2.1420[164978,165954] 268ns 100mb |L2.1420| " + - "L2.1421[165955,165999] 268ns 5mb |L2.1421| " + - "L2.1422[166000,166977] 268ns 100mb |L2.1422| " + - "L2.1423[166978,167954] 268ns 100mb |L2.1423| " + - "L2.1424[167955,167999] 268ns 5mb |L2.1424| " + - "L2.1425[168000,168983] 272ns 100mb |L2.1425| " + - "L2.1426[168984,169966] 272ns 100mb |L2.1426| " + - "L2.1427[169967,169999] 272ns 3mb |L2.1427| " + - "L2.1428[170000,170984] 276ns 100mb |L2.1428| " + - "L2.1429[170985,171968] 276ns 100mb |L2.1429| " + - "L2.1430[171969,171999] 276ns 3mb |L2.1430| " + - "L2.1431[172000,172978] 276ns 100mb |L2.1431| " + - "L2.1432[172979,173956] 276ns 100mb |L2.1432| " + - "L2.1433[173957,173999] 276ns 5mb |L2.1433| " + - "L2.1434[174000,174978] 276ns 100mb |L2.1434| " + - "L2.1435[174979,175956] 276ns 100mb |L2.1435| " + - "L2.1436[175957,175999] 276ns 5mb |L2.1436| " + - "L2.1437[176000,176979] 281ns 100mb |L2.1437| " + - "L2.1438[176980,177958] 281ns 100mb |L2.1438| " + - "L2.1439[177959,177999] 281ns 4mb |L2.1439| " + - "L2.1440[178000,178980] 281ns 100mb |L2.1440| " + - "L2.1441[178981,179960] 281ns 100mb |L2.1441| " + - "L2.1442[179961,179999] 281ns 4mb |L2.1442| " + - "L2.1443[180000,180980] 281ns 100mb |L2.1443|" + - "L2.1444[180981,181960] 281ns 100mb |L2.1444|" + - "L2.1445[181961,181999] 281ns 4mb |L2.1445|" + - "L2.1446[182000,182981] 285ns 100mb |L2.1446|" + - "L2.1447[182982,183962] 285ns 100mb |L2.1447|" + - "L2.1448[183963,183999] 285ns 4mb |L2.1448|" + - "L2.1449[184000,184980] 285ns 100mb |L2.1449|" + - "L2.1450[184981,185960] 285ns 100mb |L2.1450|" + - "L2.1451[185961,185999] 285ns 4mb |L2.1451|" + - "L2.1452[186000,186981] 291ns 100mb |L2.1452|" + - "L2.1453[186982,187962] 291ns 100mb |L2.1453|" + - "L2.1454[187963,187999] 291ns 4mb |L2.1454|" + - "L2.1455[188000,188981] 291ns 100mb |L2.1455|" + - "L2.1456[188982,189962] 291ns 100mb |L2.1456|" + - "L2.1457[189963,189999] 291ns 4mb |L2.1457|" + - "L2.1458[190000,190980] 295ns 100mb |L2.1458|" + - "L2.1459[190981,191960] 295ns 100mb |L2.1459|" + - "L2.1460[191961,191999] 295ns 4mb |L2.1460|" + - "L2.1461[192000,192981] 295ns 100mb |L2.1461|" + - "L2.1462[192982,193962] 295ns 100mb |L2.1462|" + - "L2.1463[193963,193999] 295ns 4mb |L2.1463|" + - "L2.1464[194000,194980] 295ns 100mb |L2.1464|" + - "L2.1465[194981,195960] 295ns 100mb |L2.1465|" + - "L2.1466[195961,195999] 295ns 4mb |L2.1466|" + - "L2.1467[196000,196981] 299ns 100mb |L2.1467|" + - "L2.1468[196982,197962] 299ns 100mb |L2.1468|" + - "L2.1470[197963,198943] 299ns 100mb |L2.1470|" + - "L2.1471[198944,199923] 299ns 100mb |L2.1471|" + - "L2.1472[199924,200000] 299ns 8mb |L2.1472|" "### ); }