From ac9cc243150c252e3b8f4be37a8a545e2dbe5e50 Mon Sep 17 00:00:00 2001 From: Joe-Blount <73478756+Joe-Blount@users.noreply.github.com> Date: Wed, 28 Jun 2023 12:02:21 -0500 Subject: [PATCH] fix: compactor shouldn't leave small L1s in non-overlap leading edge pattern (#8101) * fix: compactor shouldn't leave tiny L1s with non-overlapped leading edge pattern * chore: insta updates for prior commit --- .../files_split/non_overlap_split.rs | 309 +++++++-- compactor/src/components/hardcoded.rs | 2 +- compactor/src/file_group.rs | 5 + compactor/tests/layouts/accumulated_size.rs | 27 +- compactor/tests/layouts/backfill.rs | 58 +- compactor/tests/layouts/common_use_cases.rs | 74 +- compactor/tests/layouts/core.rs | 39 +- compactor/tests/layouts/knobs.rs | 4 +- compactor/tests/layouts/large_files.rs | 2 +- compactor/tests/layouts/many_files.rs | 656 +++++++++--------- compactor_test_utils/src/lib.rs | 63 ++ 11 files changed, 790 insertions(+), 449 deletions(-) diff --git a/compactor/src/components/files_split/non_overlap_split.rs b/compactor/src/components/files_split/non_overlap_split.rs index 09f8609344..9c82b622ac 100644 --- a/compactor/src/components/files_split/non_overlap_split.rs +++ b/compactor/src/components/files_split/non_overlap_split.rs @@ -8,14 +8,22 @@ use super::FilesSplit; #[derive(Debug)] -/// Split files into `[overlapping_files]` and `[non_overlapping_files]` +/// Split files into `[compact_files]` and `[non_overlapping_files]` /// To have better and efficient compaction performance, eligible non-overlapped files /// should not be compacted. -pub struct NonOverlapSplit {} +pub struct NonOverlapSplit { + /// undersized_threshold is the threshold for unnecessarily including & rewriting adjacent + /// small files. This does increase write amplification, so it shouldn't be too high, but + /// it also prevents leaving tiny L1/L2 files that will never be compacted, so it shouldn't + /// be too low. + undersized_threshold: u64, +} impl NonOverlapSplit { - pub fn new() -> Self { - Self {} + pub fn new(undersized_threshold: u64) -> Self { + Self { + undersized_threshold, + } } } @@ -26,10 +34,10 @@ impl Display for NonOverlapSplit { } impl FilesSplit for NonOverlapSplit { - /// Return (`[overlapping_files]`, `[non_overlapping_files]`) of given files - /// such that after combining all `overlapping_files` into a new file, the new file will + /// Return (`[compact_files]`, `[non_overlapping_files]`) of given files + /// such that after combining all `compact_files` into a new file, the new file will /// have no overlap with any file in `non_overlapping_files`. - /// The non_overlapping_files must in the target_level + /// The non_overlapping_files must be in the target_level /// /// Eligible non-overlapping files are files of the target level that do not /// overlap on time range of all files in lower-level. All files in target level @@ -44,7 +52,7 @@ impl FilesSplit for NonOverlapSplit { /// (L1.2, L1.3) must be compacted with L0s to produce the right non-overlapping L1s. /// /// . Output: - /// . overlapping_files: [L0.1, L0.2, L1.2, L1.3] + /// . compact_files: [L0.1, L0.2, L1.2, L1.3] /// . non_overlapping_files: [L1.1, L1.4] /// /// Algorithm: @@ -55,7 +63,12 @@ impl FilesSplit for NonOverlapSplit { /// ==> L0s' time range: |-------L0's time range --------| /// /// L1s |--L1.1--| |--L1.2--| |--L1.3--| |--L1.4--| - /// ==> Only L1.1 and L1.4 are completely outside the time range of L0s + /// ==> Only L1.1 and L1.4 are completely outside the time range of L0s. + /// So L1.1 and L1.4 are usually not included in compact_files. However, if either of L1.1 or L1.4 + /// are small (below undersized_threshold), they will be included in compact_files to avoid leaving + /// tiny L1 files behind. Note that the application of undersized_threshold can only contiguously + /// expand the set of min_time sorted target level files. So if there was a small L1.5 file, we + /// could not skip over a large L1.4 file to include L1.5 in compact_files. /// fn apply( &self, @@ -79,59 +92,89 @@ impl FilesSplit for NonOverlapSplit { return (vec![], target_level_files); }; - // Sort files of target level by min_time - target_level_files.sort_by_key(|f| f.min_time); - - // Convert target_level_files to VecDeque - let mut target_level_files = target_level_files.into_iter().collect::>(); - - // Closure that checks if a file overlaps with any prev_level_files and add it - // to overlapping_files or non_overlapping_files accordingly. Return true if ovelapping + // Split target_level_files into 3 parts, those before, during and after prev_level_files. + // Since target level files during the time range of prev_level_files must be compacted, + // they are the start of compact_files. + let mut before: Vec; + let mut compact_files: Vec; + let mut after: Vec; let mut non_overlapping_files = Vec::with_capacity(num_files); - let mut overlapping_files = Vec::with_capacity(num_files); - let mut check_overlapping_and_add = |file: ParquetFile| -> bool { - let mut overlapping = false; + (before, compact_files, after) = + three_range_split(&mut target_level_files, prev_level_range); + + // Closure that checks if a file is under the size threshold and adds it to compact_files + let mut check_undersize_and_add = |file: ParquetFile| -> bool { + let mut under = false; // Check if file overlaps with (min_time, max_time) - if prev_level_range.contains(&file) { - overlapping = true; - overlapping_files.push(file); + if file.file_size_bytes <= self.undersized_threshold as i64 { + under = true; + compact_files.push(file); } else { non_overlapping_files.push(file); } - overlapping + under }; - // Find non-overlapping files which are only at either end of their min-time sorted target_level_files - // pop_front() until hitting the first overlapping file - while let Some(file) = target_level_files.pop_front() { - if check_overlapping_and_add(file) { - break; - } - } - // pop_back() until hitting the first overlapping file - while let Some(file) = target_level_files.pop_back() { - if check_overlapping_and_add(file) { + // Contiguously add `before` files to the list to compact, so long as they're under the size threshold. + before.sort_by_key(|f| f.min_time); + let mut before = before.into_iter().collect::>(); + while let Some(file) = before.pop_back() { + if !check_undersize_and_add(file) { break; } } - // Add remaining files to overlapping_files - overlapping_files.extend(target_level_files); - overlapping_files.extend(prev_level_files); + // Contiguously add `after` files to the list to compact, so long as they're under the size threshold. + after.sort_by_key(|f| f.min_time); + let mut after = after.into_iter().collect::>(); + while let Some(file) = after.pop_front() { + if !check_undersize_and_add(file) { + break; + } + } - (overlapping_files, non_overlapping_files) + compact_files.extend(prev_level_files); + + // Add remaining files to non_overlapping_files. + non_overlapping_files.extend(before); + non_overlapping_files.extend(after); + + (compact_files, non_overlapping_files) } } +// three_range_split splits the files into 3 vectors: before, during, after the specified time range. +pub fn three_range_split( + files: &mut Vec, + range: FilesTimeRange, +) -> (Vec, Vec, Vec) { + let num_files = files.len(); + let mut before = Vec::with_capacity(num_files); + let mut during = Vec::with_capacity(num_files); + let mut after = Vec::with_capacity(num_files); + + while let Some(file) = files.pop() { + if range.contains(&file) { + during.push(file); + } else if range.before(&file) { + before.push(file); + } else { + after.push(file); + } + } + + (before, during, after) +} + #[cfg(test)] mod tests { use compactor_test_utils::{ create_l1_files, create_overlapped_files, create_overlapped_files_2, - create_overlapped_l0_l1_files, create_overlapped_l1_l2_files, format_files, - format_files_split, + create_overlapped_files_mix_sizes_1, create_overlapped_l0_l1_files, + create_overlapped_l1_l2_files, format_files, format_files_split, }; use super::*; @@ -139,7 +182,7 @@ mod tests { #[test] fn test_display() { assert_eq!( - NonOverlapSplit::new().to_string(), + NonOverlapSplit::new(1024 * 1024).to_string(), "Non-overlapping split for TargetLevel version" ); } @@ -148,7 +191,7 @@ mod tests { #[should_panic] fn test_wrong_target_level() { let files = create_overlapped_files(); - let split = NonOverlapSplit::new(); + let split = NonOverlapSplit::new(1024 * 1024); split.apply(files, CompactionLevel::Initial); } @@ -158,7 +201,7 @@ mod tests { )] fn test_unexpected_compaction_level_2() { let files = create_overlapped_files(); - let split = NonOverlapSplit::new(); + let split = NonOverlapSplit::new(1024 * 1024); // There are L2 files and will panic split.apply(files, CompactionLevel::FileNonOverlapped); } @@ -169,7 +212,7 @@ mod tests { )] fn test_unexpected_compaction_level_0() { let files = create_overlapped_files(); - let split = NonOverlapSplit::new(); + let split = NonOverlapSplit::new(1024 * 1024); // There are L0 files and will panic split.apply(files, CompactionLevel::Final); } @@ -177,7 +220,7 @@ mod tests { #[test] fn test_apply_empty_files() { let files = vec![]; - let split = NonOverlapSplit::new(); + let split = NonOverlapSplit::new(1024 * 1024); let (overlap, non_overlap) = split.apply(files, CompactionLevel::FileNonOverlapped); assert_eq!(overlap.len(), 0); @@ -199,14 +242,14 @@ mod tests { "### ); - let split = NonOverlapSplit::new(); + let split = NonOverlapSplit::new(0); // Lower level is empty -> all files will be in non_overlapping_files let (overlap, non_overlap) = split.apply(files.clone(), CompactionLevel::FileNonOverlapped); assert_eq!(overlap.len(), 0); assert_eq!(non_overlap.len(), 3); - // target level is empty -> all files will be in overlapping_files + // target level is empty -> all files will be in compact_files let (overlap, non_overlap) = split.apply(files, CompactionLevel::Final); assert_eq!(overlap.len(), 3); assert_eq!(non_overlap.len(), 0); @@ -231,7 +274,7 @@ mod tests { "### ); - let split = NonOverlapSplit::new(); + let split = NonOverlapSplit::new(0); let (overlap, non_overlap) = split.apply(files, CompactionLevel::FileNonOverlapped); insta::assert_yaml_snapshot!( format_files_split("overlap", &overlap, "non_overlap", &non_overlap), @@ -274,7 +317,7 @@ mod tests { "### ); - let split = NonOverlapSplit::new(); + let split = NonOverlapSplit::new(0); let (overlap, non_overlap) = split.apply(files, CompactionLevel::Final); insta::assert_yaml_snapshot!( format_files_split("overlap", &overlap, "non_overlap", &non_overlap), @@ -319,7 +362,7 @@ mod tests { "### ); - let split = NonOverlapSplit::new(); + let split = NonOverlapSplit::new(0); let (overlap, non_overlap) = split.apply(files, CompactionLevel::FileNonOverlapped); insta::assert_yaml_snapshot!( format_files_split("overlap", &overlap, "non_overlap", &non_overlap), @@ -339,4 +382,170 @@ mod tests { "### ); } + + #[test] + fn test_undersized_1() { + // Create files with levels and time ranges, where all L1 files are undersized. + // Input: + // |--L0.1--| |--L0.2--| + // |--L1.1--| |--L1.2--| |--L1.3--| |--L1.4--| + // + // Output: (compact, non_overlap) = ( [L0.1, L0.2, L1.1, L1.2, L1.3, L1.4] , [] ) + // Since all the files are below the undersized threshold, they all get compacted, even though some + // don't overlap. + let files = create_overlapped_files_2(10); + insta::assert_yaml_snapshot!( + format_files("initial", &files), + @r###" + --- + - initial + - "L0, all files 10b " + - "L0.2[520,550] 0ns |L0.2| " + - "L0.1[250,350] 0ns |---L0.1---| " + - "L1, all files 10b " + - "L1.13[400,500] 0ns |--L1.13---| " + - "L1.12[200,300] 0ns |--L1.12---| " + - "L1.11[0,100] 0ns |--L1.11---| " + - "L1.14[600,700] 0ns |--L1.14---| " + "### + ); + + let split = NonOverlapSplit::new(11); + let (compact_files, non_overlap) = split.apply(files, CompactionLevel::FileNonOverlapped); + insta::assert_yaml_snapshot!( + format_files_split("compact_files", &compact_files, "non_overlap", &non_overlap), + @r###" + --- + - compact_files + - "L0, all files 10b " + - "L0.2[520,550] 0ns |L0.2| " + - "L0.1[250,350] 0ns |---L0.1---| " + - "L1, all files 10b " + - "L1.12[200,300] 0ns |--L1.12---| " + - "L1.13[400,500] 0ns |--L1.13---| " + - "L1.11[0,100] 0ns |--L1.11---| " + - "L1.14[600,700] 0ns |--L1.14---| " + - non_overlap + "### + ); + } + + #[test] + fn test_undersized_2() { + // Create files with levels and time ranges, where some non-overlapping L1 files are undersized. + // . Input: + // |--L0.1--| |-L0.2-| + // |--L1.1--| |--L1.2--| |--L1.3--| |--L1.4--| |--L1.5--| |--L1.6--| |--L1.7--| |--L1.8--| + // underized: no no yes x x yes no no + // Results: + // L1.4 and L1.5 are compacted regardless of their size because they overlap L0s. + // L1.3 and L1.6 are compacted because they are undersized and contiguous to the overlapping files. + // L1.1, L1.2, L1.7, and L1.8 are not compacted because they are not undersized. + // + let files = create_overlapped_files_mix_sizes_1(10, 20, 30); + insta::assert_yaml_snapshot!( + format_files("initial", &files), + @r###" + --- + - initial + - "L0 " + - "L0.2[820,850] 0ns 10b |L0.2| " + - "L0.1[650,750] 0ns 10b |L0.1| " + - "L1 " + - "L1.13[400,500] 0ns 10b |L1.13| " + - "L1.17[1200,1300] 0ns 30b |L1.17| " + - "L1.12[200,300] 0ns 30b |L1.12| " + - "L1.11[0,100] 0ns 20b |L1.11| " + - "L1.14[600,700] 0ns 20b |L1.14| " + - "L1.15[800,900] 0ns 20b |L1.15| " + - "L1.16[1000,1100] 0ns 10b |L1.16| " + - "L1.18[1400,1500] 0ns 20b |L1.18|" + "### + ); + + let split = NonOverlapSplit::new(11); + let (compact_files, non_overlap) = split.apply(files, CompactionLevel::FileNonOverlapped); + insta::assert_yaml_snapshot!( + format_files_split("compact_files", &compact_files, "non_overlap", &non_overlap), + @r###" + --- + - compact_files + - "L0 " + - "L0.2[820,850] 0ns 10b |L0.2| " + - "L0.1[650,750] 0ns 10b |---L0.1---| " + - "L1 " + - "L1.15[800,900] 0ns 20b |--L1.15---| " + - "L1.14[600,700] 0ns 20b |--L1.14---| " + - "L1.13[400,500] 0ns 10b |--L1.13---| " + - "L1.16[1000,1100] 0ns 10b |--L1.16---| " + - non_overlap + - "L1 " + - "L1.12[200,300] 0ns 30b |L1.12| " + - "L1.17[1200,1300] 0ns 30b |L1.17| " + - "L1.11[0,100] 0ns 20b |L1.11| " + - "L1.18[1400,1500] 0ns 20b |L1.18|" + "### + ); + } + + #[test] + fn test_undersized_3() { + // This case is like undersized 3, except that the outermost L1 files (L1 & L18) are also undersized. But since + // they're separated from the overlapping files by oversized files, they are not compacted. + // Input: + // |--L0.1--| |-L0.2-| + // |--L1.1--| |--L1.2--| |--L1.3--| |--L1.4--| |--L1.5--| |--L1.6--| |--L1.7--| |--L1.8--| + // underized: yes no yes x x yes no yes + // Results: + // L1.4 and L1.5 are compacted regardless of their size because they overlap. + // L1.3 and L1.6 are compacted because they are undersized and contiguous to the overlapping files. + // L1.2 and L1.7 are not compacted because they are not undersized. + // L1.1 and L1.8 are not compacted because even though they are undersized, they are not contiguous to the + // overlapping files. + // + let files = create_overlapped_files_mix_sizes_1(10, 20, 30); + insta::assert_yaml_snapshot!( + format_files("initial", &files), + @r###" + --- + - initial + - "L0 " + - "L0.2[820,850] 0ns 10b |L0.2| " + - "L0.1[650,750] 0ns 10b |L0.1| " + - "L1 " + - "L1.13[400,500] 0ns 10b |L1.13| " + - "L1.17[1200,1300] 0ns 30b |L1.17| " + - "L1.12[200,300] 0ns 30b |L1.12| " + - "L1.11[0,100] 0ns 20b |L1.11| " + - "L1.14[600,700] 0ns 20b |L1.14| " + - "L1.15[800,900] 0ns 20b |L1.15| " + - "L1.16[1000,1100] 0ns 10b |L1.16| " + - "L1.18[1400,1500] 0ns 20b |L1.18|" + "### + ); + + let split = NonOverlapSplit::new(21); + let (compact_files, non_overlap) = split.apply(files, CompactionLevel::FileNonOverlapped); + insta::assert_yaml_snapshot!( + format_files_split("compact_files", &compact_files, "non_overlap", &non_overlap), + @r###" + --- + - compact_files + - "L0 " + - "L0.2[820,850] 0ns 10b |L0.2| " + - "L0.1[650,750] 0ns 10b |---L0.1---| " + - "L1 " + - "L1.15[800,900] 0ns 20b |--L1.15---| " + - "L1.14[600,700] 0ns 20b |--L1.14---| " + - "L1.13[400,500] 0ns 10b |--L1.13---| " + - "L1.16[1000,1100] 0ns 10b |--L1.16---| " + - non_overlap + - "L1 " + - "L1.12[200,300] 0ns 30b |L1.12| " + - "L1.17[1200,1300] 0ns 30b |L1.17| " + - "L1.11[0,100] 0ns 20b |L1.11| " + - "L1.18[1400,1500] 0ns 20b |L1.18|" + "### + ); + } } diff --git a/compactor/src/components/hardcoded.rs b/compactor/src/components/hardcoded.rs index 8dfc067c30..4fbb81c215 100644 --- a/compactor/src/components/hardcoded.rs +++ b/compactor/src/components/hardcoded.rs @@ -376,7 +376,7 @@ fn make_file_classifier(config: &Config) -> Arc { Arc::new(LoggingFileClassifierWrapper::new(Arc::new( SplitBasedFileClassifier::new( TargetLevelSplit::new(), - NonOverlapSplit::new(), + NonOverlapSplit::new(config.max_desired_file_size_bytes / 20), // rewrite non-overlapping files up to 5% of max UpgradeSplit::new(config.max_desired_file_size_bytes), LoggingSplitOrCompactWrapper::new(MetricsSplitOrCompactWrapper::new( SplitCompact::new( diff --git a/compactor/src/file_group.rs b/compactor/src/file_group.rs index 14bfd6e938..a86c48ebbe 100644 --- a/compactor/src/file_group.rs +++ b/compactor/src/file_group.rs @@ -33,6 +33,11 @@ impl FilesTimeRange { pub fn contains(&self, file: &ParquetFile) -> bool { file.min_time <= self.max_time && file.max_time >= self.min_time } + + /// Does `file` fall before the range of self? + pub fn before(&self, file: &ParquetFile) -> bool { + file.max_time < self.min_time + } } /// returns true if the time range of `file` overlaps with any in `files` diff --git a/compactor/tests/layouts/accumulated_size.rs b/compactor/tests/layouts/accumulated_size.rs index 6f30f7da73..19afa5aca7 100644 --- a/compactor/tests/layouts/accumulated_size.rs +++ b/compactor/tests/layouts/accumulated_size.rs @@ -250,21 +250,22 @@ async fn small_l1_plus_nonoverlapping_l0s_single_run() { - "L0.4[31,39] 41ns 10kb |------L0.4------| " - "L1 " - "L1.1[0,9] 11ns 40kb |-------L1.1-------| " - - "**** Simulation run 0, type=compact(TotalSizeLessThanMaxCompactSize). 3 Input Files, 30kb total:" - - "L0, all files 10kb " - - "L0.4[31,39] 41ns |---------L0.4----------| " - - "L0.3[21,29] 31ns |---------L0.3----------| " - - "L0.2[11,19] 21ns |---------L0.2----------| " - - "**** 1 Output Files (parquet_file_id not yet assigned), 30kb total:" - - "L1, all files 30kb " - - "L1.?[11,39] 41ns |------------------------------------------L1.?------------------------------------------|" - - "Committing partition 1:" - - " Soft Deleting 3 files: L0.2, L0.3, L0.4" - - " Creating 1 files" - - "**** Final Output Files (30kb written)" + - "**** Simulation run 0, type=compact(TotalSizeLessThanMaxCompactSize). 4 Input Files, 70kb total:" + - "L0 " + - "L0.4[31,39] 41ns 10kb |------L0.4------| " + - "L0.3[21,29] 31ns 10kb |------L0.3------| " + - "L0.2[11,19] 21ns 10kb |------L0.2------| " - "L1 " - "L1.1[0,9] 11ns 40kb |-------L1.1-------| " - - "L1.5[11,39] 41ns 30kb |-----------------------------L1.5-----------------------------| " + - "**** 1 Output Files (parquet_file_id not yet assigned), 70kb total:" + - "L1, all files 70kb " + - "L1.?[0,39] 41ns |------------------------------------------L1.?------------------------------------------|" + - "Committing partition 1:" + - " Soft Deleting 4 files: L1.1, L0.2, L0.3, L0.4" + - " Creating 1 files" + - "**** Final Output Files (70kb written)" + - "L1, all files 70kb " + - "L1.5[0,39] 41ns |------------------------------------------L1.5------------------------------------------|" "### ); } diff --git a/compactor/tests/layouts/backfill.rs b/compactor/tests/layouts/backfill.rs index 11fac2c66b..eca308e656 100644 --- a/compactor/tests/layouts/backfill.rs +++ b/compactor/tests/layouts/backfill.rs @@ -390,9 +390,9 @@ async fn random_backfill_empty_partition() { - "L0.77[671,714] 1.05us 5mb |L0.77-| " - "L0.85[521,670] 1.05us 16mb|----------L0.85-----------| " - "L1 " - - "L1.83[521,714] 1.04us 75mb|---------------L1.83---------------| " - "L1.89[918,986] 1.04us 34mb |---L1.89---| " - "L1.88[715,917] 1.04us 100mb |----------------L1.88----------------| " + - "L1.83[521,714] 1.04us 75mb|---------------L1.83---------------| " - "**** 3 Output Files (parquet_file_id not yet assigned), 259mb total:" - "L1 " - "L1.?[521,701] 1.05us 100mb|--------------L1.?--------------| " @@ -784,14 +784,7 @@ async fn random_backfill_over_l2s() { - "Committing partition 1:" - " Soft Deleting 2 files: L0.90, L0.91" - " Creating 2 files" - - "**** Simulation run 14, type=split(ReduceOverlap)(split_times=[699]). 1 Input Files, 65mb total:" - - "L1, all files 65mb " - - "L1.94[612,733] 1.05us |-----------------------------------------L1.94-----------------------------------------| " - - "**** 2 Output Files (parquet_file_id not yet assigned), 65mb total:" - - "L1 " - - "L1.?[612,699] 1.05us 47mb|-----------------------------L1.?-----------------------------| " - - "L1.?[700,733] 1.05us 18mb |---------L1.?---------| " - - "**** Simulation run 15, type=split(ReduceOverlap)(split_times=[499, 599]). 1 Input Files, 100mb total:" + - "**** Simulation run 14, type=split(ReduceOverlap)(split_times=[499, 599]). 1 Input Files, 100mb total:" - "L1, all files 100mb " - "L1.93[423,611] 1.05us |-----------------------------------------L1.93------------------------------------------|" - "**** 3 Output Files (parquet_file_id not yet assigned), 100mb total:" @@ -799,7 +792,7 @@ async fn random_backfill_over_l2s() { - "L1.?[423,499] 1.05us 40mb|---------------L1.?---------------| " - "L1.?[500,599] 1.05us 52mb |--------------------L1.?---------------------| " - "L1.?[600,611] 1.05us 7mb |L1.?|" - - "**** Simulation run 16, type=split(ReduceOverlap)(split_times=[299, 399]). 1 Input Files, 100mb total:" + - "**** Simulation run 15, type=split(ReduceOverlap)(split_times=[299, 399]). 1 Input Files, 100mb total:" - "L1, all files 100mb " - "L1.92[233,422] 1.05us |-----------------------------------------L1.92------------------------------------------|" - "**** 3 Output Files (parquet_file_id not yet assigned), 100mb total:" @@ -807,7 +800,7 @@ async fn random_backfill_over_l2s() { - "L1.?[233,299] 1.05us 35mb|------------L1.?-------------| " - "L1.?[300,399] 1.05us 52mb |--------------------L1.?---------------------| " - "L1.?[400,422] 1.05us 13mb |--L1.?--| " - - "**** Simulation run 17, type=split(ReduceOverlap)(split_times=[99, 199]). 1 Input Files, 100mb total:" + - "**** Simulation run 16, type=split(ReduceOverlap)(split_times=[99, 199]). 1 Input Files, 100mb total:" - "L1, all files 100mb " - "L1.86[42,232] 1.05us |-----------------------------------------L1.86------------------------------------------|" - "**** 3 Output Files (parquet_file_id not yet assigned), 100mb total:" @@ -815,6 +808,13 @@ async fn random_backfill_over_l2s() { - "L1.?[42,99] 1.05us 30mb |----------L1.?-----------| " - "L1.?[100,199] 1.05us 52mb |--------------------L1.?--------------------| " - "L1.?[200,232] 1.05us 18mb |----L1.?-----| " + - "**** Simulation run 17, type=split(ReduceOverlap)(split_times=[699]). 1 Input Files, 65mb total:" + - "L1, all files 65mb " + - "L1.94[612,733] 1.05us |-----------------------------------------L1.94-----------------------------------------| " + - "**** 2 Output Files (parquet_file_id not yet assigned), 65mb total:" + - "L1 " + - "L1.?[612,699] 1.05us 47mb|-----------------------------L1.?-----------------------------| " + - "L1.?[700,733] 1.05us 18mb |---------L1.?---------| " - "**** Simulation run 18, type=split(ReduceOverlap)(split_times=[799, 899]). 1 Input Files, 100mb total:" - "L1, all files 100mb " - "L1.95[734,922] 1.05us |-----------------------------------------L1.95------------------------------------------|" @@ -828,8 +828,8 @@ async fn random_backfill_over_l2s() { - " Creating 14 files" - "**** Simulation run 19, type=split(CompactAndSplitOutput(FoundSubsetLessThanMaxCompactSize))(split_times=[71, 142]). 4 Input Files, 282mb total:" - "L1 " - - "L1.105[42,99] 1.05us 30mb |--------L1.105---------| " - - "L1.106[100,199] 1.05us 52mb |------------------L1.106------------------| " + - "L1.103[42,99] 1.05us 30mb |--------L1.103---------| " + - "L1.104[100,199] 1.05us 52mb |------------------L1.104------------------| " - "L2 " - "L2.1[0,99] 99ns 100mb |-------------------L2.1-------------------| " - "L2.2[100,199] 199ns 100mb |-------------------L2.2-------------------| " @@ -839,12 +839,12 @@ async fn random_backfill_over_l2s() { - "L2.?[72,142] 1.05us 99mb |------------L2.?-------------| " - "L2.?[143,199] 1.05us 82mb |---------L2.?----------| " - "Committing partition 1:" - - " Soft Deleting 4 files: L2.1, L2.2, L1.105, L1.106" + - " Soft Deleting 4 files: L2.1, L2.2, L1.103, L1.104" - " Creating 3 files" - "**** Simulation run 20, type=split(CompactAndSplitOutput(FoundSubsetLessThanMaxCompactSize))(split_times=[265]). 3 Input Files, 153mb total:" - "L1 " - - "L1.107[200,232] 1.05us 18mb|----------L1.107-----------| " - - "L1.102[233,299] 1.05us 35mb |-------------------------L1.102--------------------------| " + - "L1.105[200,232] 1.05us 18mb|----------L1.105-----------| " + - "L1.100[233,299] 1.05us 35mb |-------------------------L1.100--------------------------| " - "L2 " - "L2.3[200,299] 299ns 100mb|-----------------------------------------L2.3------------------------------------------| " - "**** 2 Output Files (parquet_file_id not yet assigned), 153mb total:" @@ -852,12 +852,12 @@ async fn random_backfill_over_l2s() { - "L2.?[200,265] 1.05us 100mb|--------------------------L2.?---------------------------| " - "L2.?[266,299] 1.05us 53mb |-----------L2.?------------| " - "Committing partition 1:" - - " Soft Deleting 3 files: L2.3, L1.102, L1.107" + - " Soft Deleting 3 files: L2.3, L1.100, L1.105" - " Creating 2 files" - "**** Simulation run 21, type=split(CompactAndSplitOutput(FoundSubsetLessThanMaxCompactSize))(split_times=[376, 452]). 4 Input Files, 265mb total:" - "L1 " - - "L1.103[300,399] 1.05us 52mb|------------------L1.103------------------| " - - "L1.104[400,422] 1.05us 13mb |L1.104-| " + - "L1.101[300,399] 1.05us 52mb|------------------L1.101------------------| " + - "L1.102[400,422] 1.05us 13mb |L1.102-| " - "L2 " - "L2.4[300,399] 399ns 100mb|-------------------L2.4-------------------| " - "L2.5[400,499] 499ns 100mb |-------------------L2.5-------------------| " @@ -867,17 +867,17 @@ async fn random_backfill_over_l2s() { - "L2.?[377,452] 1.05us 100mb |-------------L2.?--------------| " - "L2.?[453,499] 1.05us 64mb |-------L2.?-------| " - "Committing partition 1:" - - " Soft Deleting 4 files: L2.4, L2.5, L1.103, L1.104" + - " Soft Deleting 4 files: L2.4, L2.5, L1.101, L1.102" - " Creating 3 files" - "**** Simulation run 22, type=split(ReduceOverlap)(split_times=[452]). 1 Input Files, 40mb total:" - "L1, all files 40mb " - - "L1.99[423,499] 1.05us |-----------------------------------------L1.99------------------------------------------|" + - "L1.97[423,499] 1.05us |-----------------------------------------L1.97------------------------------------------|" - "**** 2 Output Files (parquet_file_id not yet assigned), 40mb total:" - "L1 " - "L1.?[423,452] 1.05us 15mb|--------------L1.?--------------| " - "L1.?[453,499] 1.05us 25mb |------------------------L1.?------------------------| " - "Committing partition 1:" - - " Soft Deleting 1 files: L1.99" + - " Soft Deleting 1 files: L1.97" - " Creating 2 files" - "**** Simulation run 23, type=split(CompactAndSplitOutput(FoundSubsetLessThanMaxCompactSize))(split_times=[437, 497]). 4 Input Files, 204mb total:" - "L1 " @@ -896,8 +896,8 @@ async fn random_backfill_over_l2s() { - " Creating 3 files" - "**** Simulation run 24, type=split(CompactAndSplitOutput(FoundSubsetLessThanMaxCompactSize))(split_times=[577, 654]). 4 Input Files, 259mb total:" - "L1 " - - "L1.100[500,599] 1.05us 52mb|------------------L1.100------------------| " - - "L1.101[600,611] 1.05us 7mb |L1.101| " + - "L1.98[500,599] 1.05us 52mb|------------------L1.98-------------------| " + - "L1.99[600,611] 1.05us 7mb |L1.99| " - "L2 " - "L2.6[500,599] 599ns 100mb|-------------------L2.6-------------------| " - "L2.7[600,699] 699ns 100mb |-------------------L2.7-------------------| " @@ -907,17 +907,17 @@ async fn random_backfill_over_l2s() { - "L2.?[578,654] 1.05us 99mb |--------------L2.?--------------| " - "L2.?[655,699] 1.05us 60mb |------L2.?-------| " - "Committing partition 1:" - - " Soft Deleting 4 files: L2.6, L2.7, L1.100, L1.101" + - " Soft Deleting 4 files: L2.6, L2.7, L1.98, L1.99" - " Creating 3 files" - "**** Simulation run 25, type=split(ReduceOverlap)(split_times=[654]). 1 Input Files, 47mb total:" - "L1, all files 47mb " - - "L1.97[612,699] 1.05us |-----------------------------------------L1.97------------------------------------------|" + - "L1.106[612,699] 1.05us |-----------------------------------------L1.106-----------------------------------------|" - "**** 2 Output Files (parquet_file_id not yet assigned), 47mb total:" - "L1 " - "L1.?[612,654] 1.05us 23mb|------------------L1.?-------------------| " - "L1.?[655,699] 1.05us 24mb |-------------------L1.?--------------------| " - "Committing partition 1:" - - " Soft Deleting 1 files: L1.97" + - " Soft Deleting 1 files: L1.106" - " Creating 2 files" - "**** Simulation run 26, type=split(CompactAndSplitOutput(FoundSubsetLessThanMaxCompactSize))(split_times=[637, 696]). 4 Input Files, 206mb total:" - "L1 " @@ -936,7 +936,7 @@ async fn random_backfill_over_l2s() { - " Creating 3 files" - "**** Simulation run 27, type=split(CompactAndSplitOutput(FoundSubsetLessThanMaxCompactSize))(split_times=[765]). 3 Input Files, 153mb total:" - "L1 " - - "L1.98[700,733] 1.05us 18mb|-----------L1.98-----------| " + - "L1.107[700,733] 1.05us 18mb|----------L1.107-----------| " - "L1.108[734,799] 1.05us 35mb |-------------------------L1.108--------------------------| " - "L2 " - "L2.8[700,799] 799ns 100mb|-----------------------------------------L2.8------------------------------------------| " @@ -945,7 +945,7 @@ async fn random_backfill_over_l2s() { - "L2.?[700,765] 1.05us 100mb|--------------------------L2.?---------------------------| " - "L2.?[766,799] 1.05us 53mb |-----------L2.?------------| " - "Committing partition 1:" - - " Soft Deleting 3 files: L2.8, L1.98, L1.108" + - " Soft Deleting 3 files: L2.8, L1.107, L1.108" - " Creating 2 files" - "**** Final Output Files (3.78gb written)" - "L1 " diff --git a/compactor/tests/layouts/common_use_cases.rs b/compactor/tests/layouts/common_use_cases.rs index 53c4beb831..1908df1c48 100644 --- a/compactor/tests/layouts/common_use_cases.rs +++ b/compactor/tests/layouts/common_use_cases.rs @@ -441,9 +441,9 @@ async fn test_keep_ingesting_l0_files_40_percent_overlap_l1_left() { - " Creating 2 files" - "**** Simulation run 84, type=split(CompactAndSplitOutput(TotalSizeLessThanMaxCompactSize))(split_times=[3362]). 6 Input Files, 121mb total:" - "L1 " - - "L1.500[3293,3341] 334ns 24mb |----L1.500-----| " - "L1.493[3244,3292] 329ns 24mb |----L1.493-----| " - "L1.486[3200,3243] 324ns 20mb |----L1.486----| " + - "L1.500[3293,3341] 334ns 24mb |----L1.500-----| " - "L1.508[3392,3404] 339ns 7mb |L1.508|" - "L1.507[3342,3391] 339ns 25mb |-----L1.507-----| " - "L2 " @@ -624,9 +624,9 @@ async fn test_keep_ingesting_l0_files_40_percent_overlap_l1_left() { - " Creating 2 files" - "**** Simulation run 124, type=split(CompactAndSplitOutput(TotalSizeLessThanMaxCompactSize))(split_times=[4962]). 6 Input Files, 121mb total:" - "L1 " - - "L1.740[4893,4941] 494ns 24mb |----L1.740-----| " - "L1.733[4844,4892] 489ns 24mb |----L1.733-----| " - "L1.726[4800,4843] 484ns 20mb |----L1.726----| " + - "L1.740[4893,4941] 494ns 24mb |----L1.740-----| " - "L1.748[4992,5004] 499ns 7mb |L1.748|" - "L1.747[4942,4991] 499ns 25mb |-----L1.747-----| " - "L2 " @@ -820,3 +820,73 @@ async fn test_keep_ingesting_l0_files_40_percent_overlap_output_250mb() { "### ); } + +// The (assumed) most common ingest pattern is a leading edge, overlapping pattern. All data is written approximately +// at the time of the data, but with enough variance to cause overlaps between L0s. Those overlaps mean the first in +// any sequence of L0s overlaps the L1 from the previously compacted L0s. +// This test case covers a slightly different scenario. +// Instead, imagine if the incoming data's timestamps have a courrse granularity (every few seconds, minute, etc). +// This can produce a leading edge non-overlapping pattern in the L0s, which means the L0s won't overlap their prior +// L1. This ingest pattern has a tendency to leave small L1s. If the partition gets enough data, most of them will +// eventually get compacted. But if the partition is small, it may be left with several small L1s that should be +// compacted together. This test case is an incremental simulation of that scenario. +// We can't just create a big backlog of non-overlapping L0s because they'll be compacted all at once and not relfect +// behavior from a compactor that keeps up with the non-overlapping L0s as they arrive. So this test case creates one +// small L1 representing a prior compaction's output, and just enough L0s to trigger a compaction. The purpose of +// this test is to ensure the prior small L1 is not left behind. +#[tokio::test] +async fn test_tiny_ingest_non_overlapping() { + test_helpers::maybe_start_logging(); + let setup = layout_setup_builder().await.build().await; + + // Small L1 simumlating output from prior compaction + setup + .partition + .create_parquet_file( + parquet_builder() + .with_min_time(0) + .with_max_time(5) + .with_file_size_bytes(20000) + .with_max_l0_created_at(Time::from_timestamp_nanos(6)) + .with_compaction_level(CompactionLevel::FileNonOverlapped), + ) + .await; + + // Add two L0 file + setup + .partition + .create_parquet_file( + parquet_builder() + .with_min_time(10) + .with_max_time(19) + .with_file_size_bytes(2000) + .with_max_l0_created_at(Time::from_timestamp_nanos(20)) + .with_compaction_level(CompactionLevel::Initial), + ) + .await; + + setup + .partition + .create_parquet_file( + parquet_builder() + .with_min_time(20) + .with_max_time(29) + .with_file_size_bytes(2000) + .with_max_l0_created_at(Time::from_timestamp_nanos(30)) + .with_compaction_level(CompactionLevel::Initial), + ) + .await; + + run_layout_scenario(&setup).await; + let files = setup.list_by_table_not_to_delete().await; + + insta::assert_yaml_snapshot!( + format_files("final output", &files), + @r###" + --- + - final output + - "L1, all files 23kb " + - "L1.4[0,29] 30ns |------------------------------------------L1.4------------------------------------------|" + "### + ); +} diff --git a/compactor/tests/layouts/core.rs b/compactor/tests/layouts/core.rs index cf6b75588d..ec0c297999 100644 --- a/compactor/tests/layouts/core.rs +++ b/compactor/tests/layouts/core.rs @@ -402,32 +402,33 @@ async fn l1_with_non_overlapping_l0_larger() { - "L1.2[100,149] 2ns 50mb |--L1.2---| " - "L1.3[150,199] 3ns 20mb |--L1.3---| " - "L1.4[200,249] 4ns 3mb |--L1.4---| " - - "**** Simulation run 0, type=compact(TotalSizeLessThanMaxCompactSize). 3 Input Files, 15mb total:" - - "L0, all files 5mb " - - "L0.7[400,450] 13ns |------------L0.7------------|" - - "L0.6[350,400] 12ns |------------L0.6------------| " - - "L0.5[300,350] 11ns |------------L0.5------------| " - - "**** 1 Output Files (parquet_file_id not yet assigned), 15mb total:" - - "L1, all files 15mb " - - "L1.?[300,450] 13ns |------------------------------------------L1.?------------------------------------------|" - - "Committing partition 1:" - - " Soft Deleting 3 files: L0.5, L0.6, L0.7" - - " Creating 1 files" - - "**** Simulation run 1, type=split(CompactAndSplitOutput(TotalSizeLessThanMaxCompactSize))(split_times=[370]). 5 Input Files, 108mb total:" + - "**** Simulation run 0, type=compact(TotalSizeLessThanMaxCompactSize). 4 Input Files, 18mb total:" + - "L0 " + - "L0.7[400,450] 13ns 5mb |------L0.7------|" + - "L0.6[350,400] 12ns 5mb |------L0.6------| " + - "L0.5[300,350] 11ns 5mb |------L0.5------| " + - "L1 " + - "L1.4[200,249] 4ns 3mb |-----L1.4------| " + - "**** 1 Output Files (parquet_file_id not yet assigned), 18mb total:" + - "L1, all files 18mb " + - "L1.?[200,450] 13ns |------------------------------------------L1.?------------------------------------------|" + - "Committing partition 1:" + - " Soft Deleting 4 files: L1.4, L0.5, L0.6, L0.7" + - " Creating 1 files" + - "**** Simulation run 1, type=split(CompactAndSplitOutput(TotalSizeLessThanMaxCompactSize))(split_times=[370]). 4 Input Files, 108mb total:" - "L1 " - - "L1.4[200,249] 4ns 3mb |--L1.4---| " - - "L1.3[150,199] 3ns 20mb |--L1.3---| " - "L1.2[100,149] 2ns 50mb |--L1.2---| " - "L1.1[50,99] 1ns 20mb |--L1.1---| " - - "L1.8[300,450] 13ns 15mb |-------------L1.8--------------| " + - "L1.3[150,199] 3ns 20mb |--L1.3---| " + - "L1.8[200,450] 13ns 18mb |-------------------------L1.8-------------------------| " - "**** 2 Output Files (parquet_file_id not yet assigned), 108mb total:" - "L2 " - "L2.?[50,370] 13ns 86mb |---------------------------------L2.?---------------------------------| " - "L2.?[371,450] 13ns 22mb |-----L2.?------| " - "Committing partition 1:" - - " Soft Deleting 5 files: L1.1, L1.2, L1.3, L1.4, L1.8" + - " Soft Deleting 4 files: L1.1, L1.2, L1.3, L1.8" - " Creating 2 files" - - "**** Final Output Files (123mb written)" + - "**** Final Output Files (126mb written)" - "L2 " - "L2.9[50,370] 13ns 86mb |---------------------------------L2.9---------------------------------| " - "L2.10[371,450] 13ns 22mb |-----L2.10-----| " @@ -646,7 +647,6 @@ async fn many_l1_with_non_overlapping_l0() { - " Creating 1 files" - "**** Simulation run 1, type=split(CompactAndSplitOutput(TotalSizeLessThanMaxCompactSize))(split_times=[530]). 11 Input Files, 88mb total:" - "L1 " - - "L1.10[500,549] 10ns 7mb |L1.10| " - "L1.9[450,499] 9ns 7mb |L1.9-| " - "L1.8[400,449] 8ns 7mb |L1.8-| " - "L1.7[350,399] 7ns 7mb |L1.7-| " @@ -656,6 +656,7 @@ async fn many_l1_with_non_overlapping_l0() { - "L1.3[150,199] 3ns 7mb |L1.3-| " - "L1.2[100,149] 2ns 8mb |L1.2-| " - "L1.1[50,99] 1ns 9mb |L1.1-| " + - "L1.10[500,549] 10ns 7mb |L1.10| " - "L1.14[600,650] 23ns 15mb |L1.14| " - "**** 2 Output Files (parquet_file_id not yet assigned), 88mb total:" - "L2 " @@ -741,8 +742,8 @@ async fn large_l1_with_non_overlapping_l0() { - " Creating 1 files" - "**** Simulation run 1, type=split(CompactAndSplitOutput(TotalSizeLessThanMaxCompactSize))(split_times=[375]). 3 Input Files, 185mb total:" - "L1 " - - "L1.2[100,149] 2ns 80mb |L1.2-| " - "L1.1[50,99] 1ns 90mb |L1.1-| " + - "L1.2[100,149] 2ns 80mb |L1.2-| " - "L1.6[600,650] 23ns 15mb |L1.6-| " - "**** 2 Output Files (parquet_file_id not yet assigned), 185mb total:" - "L2 " diff --git a/compactor/tests/layouts/knobs.rs b/compactor/tests/layouts/knobs.rs index 7a491bfc11..af2acfd548 100644 --- a/compactor/tests/layouts/knobs.rs +++ b/compactor/tests/layouts/knobs.rs @@ -2236,8 +2236,8 @@ async fn all_overlapping_l0_max_input_bytes_per_partition() { - "L0.392[170978,171442] 10ns 21kb |L0.392| " - "L0.427[167315,170977] 10ns 169kb|-L0.427-| " - "L1 " - - "L1.455[167315,191189] 9ns 10mb|----------------------------L1.455-----------------------------| " - "L1.456[191190,200000] 9ns 4mb |--------L1.456--------| " + - "L1.455[167315,191189] 9ns 10mb|----------------------------L1.455-----------------------------| " - "**** 2 Output Files (parquet_file_id not yet assigned), 15mb total:" - "L1 " - "L1.?[167315,188871] 10ns 10mb|--------------------------L1.?---------------------------| " @@ -4178,8 +4178,8 @@ async fn all_overlapping_l0_max_input_bytes_per_partition_small_max_desired_file - "L0.392[170978,171442] 10ns 21kb |L0.392| " - "L0.427[167315,170977] 10ns 169kb|-L0.427-| " - "L1 " - - "L1.455[167315,191189] 9ns 10mb|----------------------------L1.455-----------------------------| " - "L1.456[191190,200000] 9ns 4mb |--------L1.456--------| " + - "L1.455[167315,191189] 9ns 10mb|----------------------------L1.455-----------------------------| " - "**** 2 Output Files (parquet_file_id not yet assigned), 15mb total:" - "L1 " - "L1.?[167315,188871] 10ns 10mb|--------------------------L1.?---------------------------| " diff --git a/compactor/tests/layouts/large_files.rs b/compactor/tests/layouts/large_files.rs index 99908487d6..65934c3bb7 100644 --- a/compactor/tests/layouts/large_files.rs +++ b/compactor/tests/layouts/large_files.rs @@ -397,8 +397,8 @@ async fn two_large_files_total_over_max_compact_size_small_overlap_range() { - "L1 " - "L1.4[668,1000] 9ns 50mb |------------------------------------------L1.4------------------------------------------|" - "L2 " - - "L2.5[800,934] 8ns 101mb |---------------L2.5---------------| " - "L2.6[935,1000] 8ns 50mb |-----L2.6------| " + - "L2.5[800,934] 8ns 101mb |---------------L2.5---------------| " - "**** 2 Output Files (parquet_file_id not yet assigned), 200mb total:" - "L2 " - "L2.?[668,835] 9ns 101mb |-------------------L2.?--------------------| " diff --git a/compactor/tests/layouts/many_files.rs b/compactor/tests/layouts/many_files.rs index a7a68283a0..c083519515 100644 --- a/compactor/tests/layouts/many_files.rs +++ b/compactor/tests/layouts/many_files.rs @@ -398,14 +398,12 @@ async fn many_l1_files() { - " Creating 1 files" - "**** Simulation run 1, type=split(CompactAndSplitOutput(TotalSizeLessThanMaxCompactSize))(split_times=[20]). 20 Input Files, 203mb total:" - "L1 " - - "L1.14[26,27] 14ns 10mb |L1.14| " - - "L1.15[28,29] 15ns 10mb |L1.15| " - - "L1.16[30,31] 16ns 10mb |L1.16| " - - "L1.17[32,33] 17ns 10mb |L1.17| " - - "L1.18[34,35] 18ns 10mb |L1.18| " - - "L1.19[36,37] 19ns 10mb |L1.19|" - "L1.20[38,39] 20ns 10mb |L1.20|" - - "L1.12[22,23] 12ns 10mb |L1.12| " + - "L1.19[36,37] 19ns 10mb |L1.19|" + - "L1.18[34,35] 18ns 10mb |L1.18| " + - "L1.17[32,33] 17ns 10mb |L1.17| " + - "L1.16[30,31] 16ns 10mb |L1.16| " + - "L1.15[28,29] 15ns 10mb |L1.15| " - "L1.11[20,21] 11ns 10mb |L1.11| " - "L1.10[18,19] 10ns 10mb |L1.10| " - "L1.9[16,17] 9ns 10mb |L1.9| " @@ -417,6 +415,8 @@ async fn many_l1_files() { - "L1.3[4,5] 3ns 10mb |L1.3| " - "L1.2[2,3] 2ns 10mb |L1.2| " - "L1.1[0,1] 1ns 10mb |L1.1| " + - "L1.14[26,27] 14ns 10mb |L1.14| " + - "L1.12[22,23] 12ns 10mb |L1.12| " - "L1.24[24,25] 23ns 13mb |L1.24| " - "**** 2 Output Files (parquet_file_id not yet assigned), 203mb total:" - "L2 " @@ -2506,106 +2506,107 @@ async fn many_tiny_l1_files() { - "Committing partition 1:" - " Soft Deleting 200 files: L1.1, L1.2, L1.3, L1.4, L1.5, L1.6, L1.7, L1.8, L1.9, L1.10, L1.11, L1.12, L1.13, L1.14, L1.15, L1.16, L1.17, L1.18, L1.19, L1.20, L1.21, L1.22, L1.23, L1.24, L1.25, L1.26, L1.27, L1.28, L1.29, L1.30, L1.31, L1.32, L1.33, L1.34, L1.35, L1.36, L1.37, L1.38, L1.39, L1.40, L1.41, L1.42, L1.43, L1.44, L1.45, L1.46, L1.47, L1.48, L1.49, L1.50, L1.51, L1.52, L1.53, L1.54, L1.55, L1.56, L1.57, L1.58, L1.59, L1.60, L1.61, L1.62, L1.63, L1.64, L1.65, L1.66, L1.67, L1.68, L1.69, L1.70, L1.71, L1.72, L1.73, L1.74, L1.75, L1.76, L1.77, L1.78, L1.79, L1.80, L1.81, L1.82, L1.83, L1.84, L1.85, L1.86, L1.87, L1.88, L1.89, L1.90, L1.91, L1.92, L1.93, L1.94, L1.95, L1.96, L1.97, L1.98, L1.99, L1.100, L1.101, L1.102, L1.103, L1.104, L1.105, L1.106, L1.107, L1.108, L1.109, L1.110, L1.111, L1.112, L1.113, L1.114, L1.115, L1.116, L1.117, L1.118, L1.119, L1.120, L1.121, L1.122, L1.123, L1.124, L1.125, L1.126, L1.127, L1.128, L1.129, L1.130, L1.131, L1.132, L1.133, L1.134, L1.135, L1.136, L1.137, L1.138, L1.139, L1.140, L1.141, L1.142, L1.143, L1.144, L1.145, L1.146, L1.147, L1.148, L1.149, L1.150, L1.151, L1.152, L1.153, L1.154, L1.155, L1.156, L1.157, L1.158, L1.159, L1.160, L1.161, L1.162, L1.163, L1.164, L1.165, L1.166, L1.167, L1.168, L1.169, L1.170, L1.171, L1.172, L1.173, L1.174, L1.175, L1.176, L1.177, L1.178, L1.179, L1.180, L1.181, L1.182, L1.183, L1.184, L1.185, L1.186, L1.187, L1.188, L1.189, L1.190, L1.191, L1.192, L1.193, L1.194, L1.195, L1.196, L1.197, L1.198, L1.199, L1.200" - " Creating 1 files" - - "**** Simulation run 1, type=compact(TotalSizeLessThanMaxCompactSize). 88 Input Files, 616kb total:" - - "L1, all files 7kb " - - "L1.288[574,575] 288ns |L1.288|" - - "L1.287[572,573] 287ns |L1.287|" - - "L1.286[570,571] 286ns |L1.286|" - - "L1.285[568,569] 285ns |L1.285|" - - "L1.284[566,567] 284ns |L1.284|" - - "L1.283[564,565] 283ns |L1.283|" - - "L1.282[562,563] 282ns |L1.282|" - - "L1.281[560,561] 281ns |L1.281|" - - "L1.280[558,559] 280ns |L1.280| " - - "L1.279[556,557] 279ns |L1.279| " - - "L1.278[554,555] 278ns |L1.278| " - - "L1.277[552,553] 277ns |L1.277| " - - "L1.276[550,551] 276ns |L1.276| " - - "L1.275[548,549] 275ns |L1.275| " - - "L1.274[546,547] 274ns |L1.274| " - - "L1.273[544,545] 273ns |L1.273| " - - "L1.272[542,543] 272ns |L1.272| " - - "L1.271[540,541] 271ns |L1.271| " - - "L1.270[538,539] 270ns |L1.270| " - - "L1.269[536,537] 269ns |L1.269| " - - "L1.268[534,535] 268ns |L1.268| " - - "L1.267[532,533] 267ns |L1.267| " - - "L1.266[530,531] 266ns |L1.266| " - - "L1.265[528,529] 265ns |L1.265| " - - "L1.264[526,527] 264ns |L1.264| " - - "L1.263[524,525] 263ns |L1.263| " - - "L1.262[522,523] 262ns |L1.262| " - - "L1.261[520,521] 261ns |L1.261| " - - "L1.260[518,519] 260ns |L1.260| " - - "L1.259[516,517] 259ns |L1.259| " - - "L1.258[514,515] 258ns |L1.258| " - - "L1.257[512,513] 257ns |L1.257| " - - "L1.256[510,511] 256ns |L1.256| " - - "L1.255[508,509] 255ns |L1.255| " - - "L1.254[506,507] 254ns |L1.254| " - - "L1.253[504,505] 253ns |L1.253| " - - "L1.252[502,503] 252ns |L1.252| " - - "L1.251[500,501] 251ns |L1.251| " - - "L1.250[498,499] 250ns |L1.250| " - - "L1.249[496,497] 249ns |L1.249| " - - "L1.248[494,495] 248ns |L1.248| " - - "L1.247[492,493] 247ns |L1.247| " - - "L1.246[490,491] 246ns |L1.246| " - - "L1.245[488,489] 245ns |L1.245| " - - "L1.244[486,487] 244ns |L1.244| " - - "L1.243[484,485] 243ns |L1.243| " - - "L1.242[482,483] 242ns |L1.242| " - - "L1.241[480,481] 241ns |L1.241| " - - "L1.240[478,479] 240ns |L1.240| " - - "L1.239[476,477] 239ns |L1.239| " - - "L1.238[474,475] 238ns |L1.238| " - - "L1.237[472,473] 237ns |L1.237| " - - "L1.236[470,471] 236ns |L1.236| " - - "L1.235[468,469] 235ns |L1.235| " - - "L1.234[466,467] 234ns |L1.234| " - - "L1.233[464,465] 233ns |L1.233| " - - "L1.232[462,463] 232ns |L1.232| " - - "L1.231[460,461] 231ns |L1.231| " - - "L1.230[458,459] 230ns |L1.230| " - - "L1.229[456,457] 229ns |L1.229| " - - "L1.228[454,455] 228ns |L1.228| " - - "L1.227[452,453] 227ns |L1.227| " - - "L1.226[450,451] 226ns |L1.226| " - - "L1.225[448,449] 225ns |L1.225| " - - "L1.224[446,447] 224ns |L1.224| " - - "L1.223[444,445] 223ns |L1.223| " - - "L1.222[442,443] 222ns |L1.222| " - - "L1.221[440,441] 221ns |L1.221| " - - "L1.220[438,439] 220ns |L1.220| " - - "L1.219[436,437] 219ns |L1.219| " - - "L1.218[434,435] 218ns |L1.218| " - - "L1.217[432,433] 217ns |L1.217| " - - "L1.216[430,431] 216ns |L1.216| " - - "L1.215[428,429] 215ns |L1.215| " - - "L1.214[426,427] 214ns |L1.214| " - - "L1.213[424,425] 213ns |L1.213| " - - "L1.212[422,423] 212ns |L1.212| " - - "L1.211[420,421] 211ns |L1.211| " - - "L1.210[418,419] 210ns |L1.210| " - - "L1.209[416,417] 209ns |L1.209| " - - "L1.208[414,415] 208ns |L1.208| " - - "L1.207[412,413] 207ns |L1.207| " - - "L1.206[410,411] 206ns |L1.206| " - - "L1.205[408,409] 205ns |L1.205| " - - "L1.204[406,407] 204ns |L1.204| " - - "L1.203[404,405] 203ns |L1.203| " - - "L1.202[402,403] 202ns |L1.202| " - - "L1.201[400,401] 201ns |L1.201| " - - "**** 1 Output Files (parquet_file_id not yet assigned), 616kb total:" - - "L2, all files 616kb " - - "L2.?[400,575] 288ns |------------------------------------------L2.?------------------------------------------|" - - "Committing partition 1:" - - " Soft Deleting 88 files: L1.201, L1.202, L1.203, L1.204, L1.205, L1.206, L1.207, L1.208, L1.209, L1.210, L1.211, L1.212, L1.213, L1.214, L1.215, L1.216, L1.217, L1.218, L1.219, L1.220, L1.221, L1.222, L1.223, L1.224, L1.225, L1.226, L1.227, L1.228, L1.229, L1.230, L1.231, L1.232, L1.233, L1.234, L1.235, L1.236, L1.237, L1.238, L1.239, L1.240, L1.241, L1.242, L1.243, L1.244, L1.245, L1.246, L1.247, L1.248, L1.249, L1.250, L1.251, L1.252, L1.253, L1.254, L1.255, L1.256, L1.257, L1.258, L1.259, L1.260, L1.261, L1.262, L1.263, L1.264, L1.265, L1.266, L1.267, L1.268, L1.269, L1.270, L1.271, L1.272, L1.273, L1.274, L1.275, L1.276, L1.277, L1.278, L1.279, L1.280, L1.281, L1.282, L1.283, L1.284, L1.285, L1.286, L1.287, L1.288" - - " Creating 1 files" - - "**** Final Output Files (2mb written)" + - "**** Simulation run 1, type=compact(TotalSizeLessThanMaxCompactSize). 89 Input Files, 2mb total:" + - "L1 " + - "L1.288[574,575] 288ns 7kb |L1.288|" + - "L1.287[572,573] 287ns 7kb |L1.287|" + - "L1.286[570,571] 286ns 7kb |L1.286|" + - "L1.285[568,569] 285ns 7kb |L1.285|" + - "L1.284[566,567] 284ns 7kb |L1.284|" + - "L1.283[564,565] 283ns 7kb |L1.283|" + - "L1.282[562,563] 282ns 7kb |L1.282|" + - "L1.281[560,561] 281ns 7kb |L1.281|" + - "L1.280[558,559] 280ns 7kb |L1.280|" + - "L1.279[556,557] 279ns 7kb |L1.279|" + - "L1.278[554,555] 278ns 7kb |L1.278|" + - "L1.277[552,553] 277ns 7kb |L1.277|" + - "L1.276[550,551] 276ns 7kb |L1.276|" + - "L1.275[548,549] 275ns 7kb |L1.275|" + - "L1.274[546,547] 274ns 7kb |L1.274|" + - "L1.273[544,545] 273ns 7kb |L1.273|" + - "L1.272[542,543] 272ns 7kb |L1.272|" + - "L1.271[540,541] 271ns 7kb |L1.271|" + - "L1.270[538,539] 270ns 7kb |L1.270|" + - "L1.269[536,537] 269ns 7kb |L1.269|" + - "L1.268[534,535] 268ns 7kb |L1.268|" + - "L1.267[532,533] 267ns 7kb |L1.267|" + - "L1.266[530,531] 266ns 7kb |L1.266|" + - "L1.265[528,529] 265ns 7kb |L1.265|" + - "L1.264[526,527] 264ns 7kb |L1.264|" + - "L1.263[524,525] 263ns 7kb |L1.263|" + - "L1.262[522,523] 262ns 7kb |L1.262| " + - "L1.261[520,521] 261ns 7kb |L1.261| " + - "L1.260[518,519] 260ns 7kb |L1.260| " + - "L1.259[516,517] 259ns 7kb |L1.259| " + - "L1.258[514,515] 258ns 7kb |L1.258| " + - "L1.257[512,513] 257ns 7kb |L1.257| " + - "L1.256[510,511] 256ns 7kb |L1.256| " + - "L1.255[508,509] 255ns 7kb |L1.255| " + - "L1.254[506,507] 254ns 7kb |L1.254| " + - "L1.253[504,505] 253ns 7kb |L1.253| " + - "L1.252[502,503] 252ns 7kb |L1.252| " + - "L1.251[500,501] 251ns 7kb |L1.251| " + - "L1.250[498,499] 250ns 7kb |L1.250| " + - "L1.249[496,497] 249ns 7kb |L1.249| " + - "L1.248[494,495] 248ns 7kb |L1.248| " + - "L1.247[492,493] 247ns 7kb |L1.247| " + - "L1.246[490,491] 246ns 7kb |L1.246| " + - "L1.245[488,489] 245ns 7kb |L1.245| " + - "L1.244[486,487] 244ns 7kb |L1.244| " + - "L1.243[484,485] 243ns 7kb |L1.243| " + - "L1.242[482,483] 242ns 7kb |L1.242| " + - "L1.241[480,481] 241ns 7kb |L1.241| " + - "L1.240[478,479] 240ns 7kb |L1.240| " + - "L1.239[476,477] 239ns 7kb |L1.239| " + - "L1.238[474,475] 238ns 7kb |L1.238| " + - "L1.237[472,473] 237ns 7kb |L1.237| " + - "L1.236[470,471] 236ns 7kb |L1.236| " + - "L1.235[468,469] 235ns 7kb |L1.235| " + - "L1.234[466,467] 234ns 7kb |L1.234| " + - "L1.233[464,465] 233ns 7kb |L1.233| " + - "L1.232[462,463] 232ns 7kb |L1.232| " + - "L1.231[460,461] 231ns 7kb |L1.231| " + - "L1.230[458,459] 230ns 7kb |L1.230| " + - "L1.229[456,457] 229ns 7kb |L1.229| " + - "L1.228[454,455] 228ns 7kb |L1.228| " + - "L1.227[452,453] 227ns 7kb |L1.227| " + - "L1.226[450,451] 226ns 7kb |L1.226| " + - "L1.225[448,449] 225ns 7kb |L1.225| " + - "L1.224[446,447] 224ns 7kb |L1.224| " + - "L1.223[444,445] 223ns 7kb |L1.223| " + - "L1.222[442,443] 222ns 7kb |L1.222| " + - "L1.221[440,441] 221ns 7kb |L1.221| " + - "L1.220[438,439] 220ns 7kb |L1.220| " + - "L1.219[436,437] 219ns 7kb |L1.219| " + - "L1.218[434,435] 218ns 7kb |L1.218| " + - "L1.217[432,433] 217ns 7kb |L1.217| " + - "L1.216[430,431] 216ns 7kb |L1.216| " + - "L1.215[428,429] 215ns 7kb |L1.215| " + - "L1.214[426,427] 214ns 7kb |L1.214| " + - "L1.213[424,425] 213ns 7kb |L1.213| " + - "L1.212[422,423] 212ns 7kb |L1.212| " + - "L1.211[420,421] 211ns 7kb |L1.211| " + - "L1.210[418,419] 210ns 7kb |L1.210| " + - "L1.209[416,417] 209ns 7kb |L1.209| " + - "L1.208[414,415] 208ns 7kb |L1.208| " + - "L1.207[412,413] 207ns 7kb |L1.207| " + - "L1.206[410,411] 206ns 7kb |L1.206| " + - "L1.205[408,409] 205ns 7kb |L1.205| " + - "L1.204[406,407] 204ns 7kb |L1.204| " + - "L1.203[404,405] 203ns 7kb |L1.203| " + - "L1.202[402,403] 202ns 7kb |L1.202| " + - "L1.201[400,401] 201ns 7kb |L1.201| " - "L2 " - "L2.289[0,399] 200ns 1mb |---------------------------L2.289---------------------------| " - - "L2.290[400,575] 288ns 616kb |---------L2.290----------| " + - "**** 1 Output Files (parquet_file_id not yet assigned), 2mb total:" + - "L2, all files 2mb " + - "L2.?[0,575] 288ns |------------------------------------------L2.?------------------------------------------|" + - "Committing partition 1:" + - " Soft Deleting 89 files: L1.201, L1.202, L1.203, L1.204, L1.205, L1.206, L1.207, L1.208, L1.209, L1.210, L1.211, L1.212, L1.213, L1.214, L1.215, L1.216, L1.217, L1.218, L1.219, L1.220, L1.221, L1.222, L1.223, L1.224, L1.225, L1.226, L1.227, L1.228, L1.229, L1.230, L1.231, L1.232, L1.233, L1.234, L1.235, L1.236, L1.237, L1.238, L1.239, L1.240, L1.241, L1.242, L1.243, L1.244, L1.245, L1.246, L1.247, L1.248, L1.249, L1.250, L1.251, L1.252, L1.253, L1.254, L1.255, L1.256, L1.257, L1.258, L1.259, L1.260, L1.261, L1.262, L1.263, L1.264, L1.265, L1.266, L1.267, L1.268, L1.269, L1.270, L1.271, L1.272, L1.273, L1.274, L1.275, L1.276, L1.277, L1.278, L1.279, L1.280, L1.281, L1.282, L1.283, L1.284, L1.285, L1.286, L1.287, L1.288, L2.289" + - " Creating 1 files" + - "**** Final Output Files (3mb written)" + - "L2, all files 2mb " + - "L2.290[0,575] 288ns |-----------------------------------------L2.290-----------------------------------------|" "### ); } @@ -3080,26 +3081,26 @@ async fn many_l0_and_overlapped_l1_files() { - "L0 " - "L0.211[0,190] 210ns 1mb |--------------------------------------L0.211---------------------------------------| " - "L1 " - - "L1.191[0,9] 1ns 1mb |L1.191| " - "L1.210[190,199] 20ns 1mb |L1.210|" - - "L1.192[10,19] 2ns 1mb |L1.192| " - - "L1.193[20,29] 3ns 1mb |L1.193| " - - "L1.194[30,39] 4ns 1mb |L1.194| " - - "L1.195[40,49] 5ns 1mb |L1.195| " - - "L1.196[50,59] 6ns 1mb |L1.196| " - - "L1.197[60,69] 7ns 1mb |L1.197| " - - "L1.198[70,79] 8ns 1mb |L1.198| " - - "L1.199[80,89] 9ns 1mb |L1.199| " - - "L1.200[90,99] 10ns 1mb |L1.200| " - - "L1.201[100,109] 11ns 1mb |L1.201| " - - "L1.202[110,119] 12ns 1mb |L1.202| " - - "L1.203[120,129] 13ns 1mb |L1.203| " - - "L1.204[130,139] 14ns 1mb |L1.204| " - - "L1.205[140,149] 15ns 1mb |L1.205| " - - "L1.206[150,159] 16ns 1mb |L1.206| " - - "L1.207[160,169] 17ns 1mb |L1.207| " - - "L1.208[170,179] 18ns 1mb |L1.208| " - "L1.209[180,189] 19ns 1mb |L1.209| " + - "L1.208[170,179] 18ns 1mb |L1.208| " + - "L1.207[160,169] 17ns 1mb |L1.207| " + - "L1.206[150,159] 16ns 1mb |L1.206| " + - "L1.205[140,149] 15ns 1mb |L1.205| " + - "L1.204[130,139] 14ns 1mb |L1.204| " + - "L1.203[120,129] 13ns 1mb |L1.203| " + - "L1.202[110,119] 12ns 1mb |L1.202| " + - "L1.201[100,109] 11ns 1mb |L1.201| " + - "L1.200[90,99] 10ns 1mb |L1.200| " + - "L1.199[80,89] 9ns 1mb |L1.199| " + - "L1.198[70,79] 8ns 1mb |L1.198| " + - "L1.197[60,69] 7ns 1mb |L1.197| " + - "L1.196[50,59] 6ns 1mb |L1.196| " + - "L1.195[40,49] 5ns 1mb |L1.195| " + - "L1.194[30,39] 4ns 1mb |L1.194| " + - "L1.193[20,29] 3ns 1mb |L1.193| " + - "L1.192[10,19] 2ns 1mb |L1.192| " + - "L1.191[0,9] 1ns 1mb |L1.191| " - "**** 2 Output Files (parquet_file_id not yet assigned), 21mb total:" - "L1 " - "L1.?[0,159] 210ns 17mb |--------------------------------L1.?---------------------------------| " @@ -3402,212 +3403,204 @@ async fn not_many_l0_and_overlapped_l1_files() { - "L1.208[2600,2609] 18ns 1mb |L1.208| " - "L1.209[2800,2809] 19ns 1mb |L1.209|" - "L1.210[3000,3009] 20ns 1mb |L1.210|" - - "**** Simulation run 0, type=compact(TotalSizeLessThanMaxCompactSize). 195 Input Files, 6mb total:" + - "**** Simulation run 0, type=split(CompactAndSplitOutput(TotalSizeLessThanMaxCompactSize))(split_times=[2407]). 210 Input Files, 21mb total:" - "L0 " - - "L0.190[189,190] 210ns 7kb |L0.190|" - - "L0.189[188,189] 209ns 7kb |L0.189|" - - "L0.188[187,188] 208ns 7kb |L0.188|" - - "L0.187[186,187] 207ns 7kb |L0.187|" - - "L0.186[185,186] 206ns 7kb |L0.186|" - - "L0.185[184,185] 205ns 7kb |L0.185|" - - "L0.184[183,184] 204ns 7kb |L0.184|" - - "L0.183[182,183] 203ns 7kb |L0.183|" - - "L0.182[181,182] 202ns 7kb |L0.182|" - - "L0.181[180,181] 201ns 7kb |L0.181|" - - "L0.180[179,180] 200ns 7kb |L0.180|" - - "L0.179[178,179] 199ns 7kb |L0.179|" - - "L0.178[177,178] 198ns 7kb |L0.178|" - - "L0.177[176,177] 197ns 7kb |L0.177|" - - "L0.176[175,176] 196ns 7kb |L0.176|" - - "L0.175[174,175] 195ns 7kb |L0.175|" - - "L0.174[173,174] 194ns 7kb |L0.174| " - - "L0.173[172,173] 193ns 7kb |L0.173| " - - "L0.172[171,172] 192ns 7kb |L0.172| " - - "L0.171[170,171] 191ns 7kb |L0.171| " - - "L0.170[169,170] 190ns 7kb |L0.170| " - - "L0.169[168,169] 189ns 7kb |L0.169| " - - "L0.168[167,168] 188ns 7kb |L0.168| " - - "L0.167[166,167] 187ns 7kb |L0.167| " - - "L0.166[165,166] 186ns 7kb |L0.166| " - - "L0.165[164,165] 185ns 7kb |L0.165| " - - "L0.164[163,164] 184ns 7kb |L0.164| " - - "L0.163[162,163] 183ns 7kb |L0.163| " - - "L0.162[161,162] 182ns 7kb |L0.162| " - - "L0.161[160,161] 181ns 7kb |L0.161| " - - "L0.160[159,160] 180ns 7kb |L0.160| " - - "L0.159[158,159] 179ns 7kb |L0.159| " - - "L0.158[157,158] 178ns 7kb |L0.158| " - - "L0.157[156,157] 177ns 7kb |L0.157| " - - "L0.156[155,156] 176ns 7kb |L0.156| " - - "L0.155[154,155] 175ns 7kb |L0.155| " - - "L0.154[153,154] 174ns 7kb |L0.154| " - - "L0.153[152,153] 173ns 7kb |L0.153| " - - "L0.152[151,152] 172ns 7kb |L0.152| " - - "L0.151[150,151] 171ns 7kb |L0.151| " - - "L0.150[149,150] 170ns 7kb |L0.150| " - - "L0.149[148,149] 169ns 7kb |L0.149| " - - "L0.148[147,148] 168ns 7kb |L0.148| " - - "L0.147[146,147] 167ns 7kb |L0.147| " - - "L0.146[145,146] 166ns 7kb |L0.146| " - - "L0.145[144,145] 165ns 7kb |L0.145| " - - "L0.144[143,144] 164ns 7kb |L0.144| " - - "L0.143[142,143] 163ns 7kb |L0.143| " - - "L0.142[141,142] 162ns 7kb |L0.142| " - - "L0.141[140,141] 161ns 7kb |L0.141| " - - "L0.140[139,140] 160ns 7kb |L0.140| " - - "L0.139[138,139] 159ns 7kb |L0.139| " - - "L0.138[137,138] 158ns 7kb |L0.138| " - - "L0.137[136,137] 157ns 7kb |L0.137| " - - "L0.136[135,136] 156ns 7kb |L0.136| " - - "L0.135[134,135] 155ns 7kb |L0.135| " - - "L0.134[133,134] 154ns 7kb |L0.134| " - - "L0.133[132,133] 153ns 7kb |L0.133| " - - "L0.132[131,132] 152ns 7kb |L0.132| " - - "L0.131[130,131] 151ns 7kb |L0.131| " - - "L0.130[129,130] 150ns 7kb |L0.130| " - - "L0.129[128,129] 149ns 7kb |L0.129| " - - "L0.128[127,128] 148ns 7kb |L0.128| " - - "L0.127[126,127] 147ns 7kb |L0.127| " - - "L0.126[125,126] 146ns 7kb |L0.126| " - - "L0.125[124,125] 145ns 7kb |L0.125| " - - "L0.124[123,124] 144ns 7kb |L0.124| " - - "L0.123[122,123] 143ns 7kb |L0.123| " - - "L0.122[121,122] 142ns 7kb |L0.122| " - - "L0.121[120,121] 141ns 7kb |L0.121| " - - "L0.120[119,120] 140ns 7kb |L0.120| " - - "L0.119[118,119] 139ns 7kb |L0.119| " - - "L0.118[117,118] 138ns 7kb |L0.118| " - - "L0.117[116,117] 137ns 7kb |L0.117| " - - "L0.116[115,116] 136ns 7kb |L0.116| " - - "L0.115[114,115] 135ns 7kb |L0.115| " - - "L0.114[113,114] 134ns 7kb |L0.114| " - - "L0.113[112,113] 133ns 7kb |L0.113| " - - "L0.112[111,112] 132ns 7kb |L0.112| " - - "L0.111[110,111] 131ns 7kb |L0.111| " - - "L0.110[109,110] 130ns 7kb |L0.110| " - - "L0.109[108,109] 129ns 7kb |L0.109| " - - "L0.108[107,108] 128ns 7kb |L0.108| " - - "L0.107[106,107] 127ns 7kb |L0.107| " - - "L0.106[105,106] 126ns 7kb |L0.106| " - - "L0.105[104,105] 125ns 7kb |L0.105| " - - "L0.104[103,104] 124ns 7kb |L0.104| " - - "L0.103[102,103] 123ns 7kb |L0.103| " - - "L0.102[101,102] 122ns 7kb |L0.102| " - - "L0.101[100,101] 121ns 7kb |L0.101| " - - "L0.100[99,100] 120ns 7kb |L0.100| " - - "L0.99[98,99] 119ns 7kb |L0.99| " - - "L0.98[97,98] 118ns 7kb |L0.98| " - - "L0.97[96,97] 117ns 7kb |L0.97| " - - "L0.96[95,96] 116ns 7kb |L0.96| " - - "L0.95[94,95] 115ns 7kb |L0.95| " - - "L0.94[93,94] 114ns 7kb |L0.94| " - - "L0.93[92,93] 113ns 7kb |L0.93| " - - "L0.92[91,92] 112ns 7kb |L0.92| " - - "L0.91[90,91] 111ns 7kb |L0.91| " - - "L0.90[89,90] 110ns 7kb |L0.90| " - - "L0.89[88,89] 109ns 7kb |L0.89| " - - "L0.88[87,88] 108ns 7kb |L0.88| " - - "L0.87[86,87] 107ns 7kb |L0.87| " - - "L0.86[85,86] 106ns 7kb |L0.86| " - - "L0.85[84,85] 105ns 7kb |L0.85| " - - "L0.84[83,84] 104ns 7kb |L0.84| " - - "L0.83[82,83] 103ns 7kb |L0.83| " - - "L0.82[81,82] 102ns 7kb |L0.82| " - - "L0.81[80,81] 101ns 7kb |L0.81| " - - "L0.80[79,80] 100ns 7kb |L0.80| " - - "L0.79[78,79] 99ns 7kb |L0.79| " - - "L0.78[77,78] 98ns 7kb |L0.78| " - - "L0.77[76,77] 97ns 7kb |L0.77| " - - "L0.76[75,76] 96ns 7kb |L0.76| " - - "L0.75[74,75] 95ns 7kb |L0.75| " - - "L0.74[73,74] 94ns 7kb |L0.74| " - - "L0.73[72,73] 93ns 7kb |L0.73| " - - "L0.72[71,72] 92ns 7kb |L0.72| " - - "L0.71[70,71] 91ns 7kb |L0.71| " - - "L0.70[69,70] 90ns 7kb |L0.70| " - - "L0.69[68,69] 89ns 7kb |L0.69| " - - "L0.68[67,68] 88ns 7kb |L0.68| " - - "L0.67[66,67] 87ns 7kb |L0.67| " - - "L0.66[65,66] 86ns 7kb |L0.66| " - - "L0.65[64,65] 85ns 7kb |L0.65| " - - "L0.64[63,64] 84ns 7kb |L0.64| " - - "L0.63[62,63] 83ns 7kb |L0.63| " - - "L0.62[61,62] 82ns 7kb |L0.62| " - - "L0.61[60,61] 81ns 7kb |L0.61| " - - "L0.60[59,60] 80ns 7kb |L0.60| " - - "L0.59[58,59] 79ns 7kb |L0.59| " - - "L0.58[57,58] 78ns 7kb |L0.58| " - - "L0.57[56,57] 77ns 7kb |L0.57| " - - "L0.56[55,56] 76ns 7kb |L0.56| " - - "L0.55[54,55] 75ns 7kb |L0.55| " - - "L0.54[53,54] 74ns 7kb |L0.54| " - - "L0.53[52,53] 73ns 7kb |L0.53| " - - "L0.52[51,52] 72ns 7kb |L0.52| " - - "L0.51[50,51] 71ns 7kb |L0.51| " - - "L0.50[49,50] 70ns 7kb |L0.50| " - - "L0.49[48,49] 69ns 7kb |L0.49| " - - "L0.48[47,48] 68ns 7kb |L0.48| " - - "L0.47[46,47] 67ns 7kb |L0.47| " - - "L0.46[45,46] 66ns 7kb |L0.46| " - - "L0.45[44,45] 65ns 7kb |L0.45| " - - "L0.44[43,44] 64ns 7kb |L0.44| " - - "L0.43[42,43] 63ns 7kb |L0.43| " - - "L0.42[41,42] 62ns 7kb |L0.42| " - - "L0.41[40,41] 61ns 7kb |L0.41| " - - "L0.40[39,40] 60ns 7kb |L0.40| " - - "L0.39[38,39] 59ns 7kb |L0.39| " - - "L0.38[37,38] 58ns 7kb |L0.38| " - - "L0.37[36,37] 57ns 7kb |L0.37| " - - "L0.36[35,36] 56ns 7kb |L0.36| " - - "L0.35[34,35] 55ns 7kb |L0.35| " - - "L0.34[33,34] 54ns 7kb |L0.34| " - - "L0.33[32,33] 53ns 7kb |L0.33| " - - "L0.32[31,32] 52ns 7kb |L0.32| " - - "L0.31[30,31] 51ns 7kb |L0.31| " - - "L0.30[29,30] 50ns 7kb |L0.30| " - - "L0.29[28,29] 49ns 7kb |L0.29| " - - "L0.28[27,28] 48ns 7kb |L0.28| " - - "L0.27[26,27] 47ns 7kb |L0.27| " - - "L0.26[25,26] 46ns 7kb |L0.26| " - - "L0.25[24,25] 45ns 7kb |L0.25| " - - "L0.24[23,24] 44ns 7kb |L0.24| " - - "L0.23[22,23] 43ns 7kb |L0.23| " - - "L0.22[21,22] 42ns 7kb |L0.22| " - - "L0.21[20,21] 41ns 7kb |L0.21| " - - "L0.20[19,20] 40ns 7kb |L0.20| " - - "L0.19[18,19] 39ns 7kb |L0.19| " - - "L0.18[17,18] 38ns 7kb |L0.18| " - - "L0.17[16,17] 37ns 7kb |L0.17| " - - "L0.16[15,16] 36ns 7kb |L0.16| " - - "L0.15[14,15] 35ns 7kb |L0.15| " - - "L0.14[13,14] 34ns 7kb |L0.14| " - - "L0.13[12,13] 33ns 7kb |L0.13| " - - "L0.12[11,12] 32ns 7kb |L0.12| " - - "L0.11[10,11] 31ns 7kb |L0.11| " - - "L0.10[9,10] 30ns 7kb |L0.10| " - - "L0.9[8,9] 29ns 7kb |L0.9| " - - "L0.8[7,8] 28ns 7kb |L0.8| " - - "L0.7[6,7] 27ns 7kb |L0.7| " - - "L0.6[5,6] 26ns 7kb |L0.6| " - - "L0.5[4,5] 25ns 7kb |L0.5| " - - "L0.4[3,4] 24ns 7kb |L0.4| " + - "L0.190[189,190] 210ns 7kb |L0.190| " + - "L0.189[188,189] 209ns 7kb |L0.189| " + - "L0.188[187,188] 208ns 7kb |L0.188| " + - "L0.187[186,187] 207ns 7kb |L0.187| " + - "L0.186[185,186] 206ns 7kb |L0.186| " + - "L0.185[184,185] 205ns 7kb |L0.185| " + - "L0.184[183,184] 204ns 7kb |L0.184| " + - "L0.183[182,183] 203ns 7kb |L0.183| " + - "L0.182[181,182] 202ns 7kb |L0.182| " + - "L0.181[180,181] 201ns 7kb |L0.181| " + - "L0.180[179,180] 200ns 7kb |L0.180| " + - "L0.179[178,179] 199ns 7kb |L0.179| " + - "L0.178[177,178] 198ns 7kb |L0.178| " + - "L0.177[176,177] 197ns 7kb |L0.177| " + - "L0.176[175,176] 196ns 7kb |L0.176| " + - "L0.175[174,175] 195ns 7kb |L0.175| " + - "L0.174[173,174] 194ns 7kb |L0.174| " + - "L0.173[172,173] 193ns 7kb |L0.173| " + - "L0.172[171,172] 192ns 7kb |L0.172| " + - "L0.171[170,171] 191ns 7kb |L0.171| " + - "L0.170[169,170] 190ns 7kb |L0.170| " + - "L0.169[168,169] 189ns 7kb |L0.169| " + - "L0.168[167,168] 188ns 7kb |L0.168| " + - "L0.167[166,167] 187ns 7kb |L0.167| " + - "L0.166[165,166] 186ns 7kb |L0.166| " + - "L0.165[164,165] 185ns 7kb |L0.165| " + - "L0.164[163,164] 184ns 7kb |L0.164| " + - "L0.163[162,163] 183ns 7kb |L0.163| " + - "L0.162[161,162] 182ns 7kb |L0.162| " + - "L0.161[160,161] 181ns 7kb |L0.161| " + - "L0.160[159,160] 180ns 7kb |L0.160| " + - "L0.159[158,159] 179ns 7kb |L0.159| " + - "L0.158[157,158] 178ns 7kb |L0.158| " + - "L0.157[156,157] 177ns 7kb |L0.157| " + - "L0.156[155,156] 176ns 7kb |L0.156| " + - "L0.155[154,155] 175ns 7kb |L0.155| " + - "L0.154[153,154] 174ns 7kb |L0.154| " + - "L0.153[152,153] 173ns 7kb |L0.153| " + - "L0.152[151,152] 172ns 7kb |L0.152| " + - "L0.151[150,151] 171ns 7kb |L0.151| " + - "L0.150[149,150] 170ns 7kb |L0.150| " + - "L0.149[148,149] 169ns 7kb |L0.149| " + - "L0.148[147,148] 168ns 7kb |L0.148| " + - "L0.147[146,147] 167ns 7kb |L0.147| " + - "L0.146[145,146] 166ns 7kb |L0.146| " + - "L0.145[144,145] 165ns 7kb |L0.145| " + - "L0.144[143,144] 164ns 7kb |L0.144| " + - "L0.143[142,143] 163ns 7kb |L0.143| " + - "L0.142[141,142] 162ns 7kb |L0.142| " + - "L0.141[140,141] 161ns 7kb |L0.141| " + - "L0.140[139,140] 160ns 7kb |L0.140| " + - "L0.139[138,139] 159ns 7kb |L0.139| " + - "L0.138[137,138] 158ns 7kb |L0.138| " + - "L0.137[136,137] 157ns 7kb |L0.137| " + - "L0.136[135,136] 156ns 7kb |L0.136| " + - "L0.135[134,135] 155ns 7kb |L0.135| " + - "L0.134[133,134] 154ns 7kb |L0.134| " + - "L0.133[132,133] 153ns 7kb |L0.133| " + - "L0.132[131,132] 152ns 7kb |L0.132| " + - "L0.131[130,131] 151ns 7kb |L0.131| " + - "L0.130[129,130] 150ns 7kb |L0.130| " + - "L0.129[128,129] 149ns 7kb |L0.129| " + - "L0.128[127,128] 148ns 7kb |L0.128| " + - "L0.127[126,127] 147ns 7kb |L0.127| " + - "L0.126[125,126] 146ns 7kb |L0.126| " + - "L0.125[124,125] 145ns 7kb |L0.125| " + - "L0.124[123,124] 144ns 7kb |L0.124| " + - "L0.123[122,123] 143ns 7kb |L0.123| " + - "L0.122[121,122] 142ns 7kb |L0.122| " + - "L0.121[120,121] 141ns 7kb |L0.121| " + - "L0.120[119,120] 140ns 7kb |L0.120| " + - "L0.119[118,119] 139ns 7kb |L0.119| " + - "L0.118[117,118] 138ns 7kb |L0.118| " + - "L0.117[116,117] 137ns 7kb |L0.117| " + - "L0.116[115,116] 136ns 7kb |L0.116| " + - "L0.115[114,115] 135ns 7kb |L0.115| " + - "L0.114[113,114] 134ns 7kb |L0.114| " + - "L0.113[112,113] 133ns 7kb |L0.113| " + - "L0.112[111,112] 132ns 7kb |L0.112| " + - "L0.111[110,111] 131ns 7kb |L0.111| " + - "L0.110[109,110] 130ns 7kb |L0.110| " + - "L0.109[108,109] 129ns 7kb |L0.109| " + - "L0.108[107,108] 128ns 7kb |L0.108| " + - "L0.107[106,107] 127ns 7kb |L0.107| " + - "L0.106[105,106] 126ns 7kb |L0.106| " + - "L0.105[104,105] 125ns 7kb |L0.105| " + - "L0.104[103,104] 124ns 7kb |L0.104| " + - "L0.103[102,103] 123ns 7kb |L0.103| " + - "L0.102[101,102] 122ns 7kb |L0.102| " + - "L0.101[100,101] 121ns 7kb |L0.101| " + - "L0.100[99,100] 120ns 7kb |L0.100| " + - "L0.99[98,99] 119ns 7kb |L0.99| " + - "L0.98[97,98] 118ns 7kb |L0.98| " + - "L0.97[96,97] 117ns 7kb |L0.97| " + - "L0.96[95,96] 116ns 7kb |L0.96| " + - "L0.95[94,95] 115ns 7kb |L0.95| " + - "L0.94[93,94] 114ns 7kb |L0.94| " + - "L0.93[92,93] 113ns 7kb |L0.93| " + - "L0.92[91,92] 112ns 7kb |L0.92| " + - "L0.91[90,91] 111ns 7kb |L0.91| " + - "L0.90[89,90] 110ns 7kb |L0.90| " + - "L0.89[88,89] 109ns 7kb |L0.89| " + - "L0.88[87,88] 108ns 7kb |L0.88| " + - "L0.87[86,87] 107ns 7kb |L0.87| " + - "L0.86[85,86] 106ns 7kb |L0.86| " + - "L0.85[84,85] 105ns 7kb |L0.85| " + - "L0.84[83,84] 104ns 7kb |L0.84| " + - "L0.83[82,83] 103ns 7kb |L0.83| " + - "L0.82[81,82] 102ns 7kb |L0.82| " + - "L0.81[80,81] 101ns 7kb |L0.81| " + - "L0.80[79,80] 100ns 7kb |L0.80| " + - "L0.79[78,79] 99ns 7kb |L0.79| " + - "L0.78[77,78] 98ns 7kb |L0.78| " + - "L0.77[76,77] 97ns 7kb |L0.77| " + - "L0.76[75,76] 96ns 7kb |L0.76| " + - "L0.75[74,75] 95ns 7kb |L0.75| " + - "L0.74[73,74] 94ns 7kb |L0.74| " + - "L0.73[72,73] 93ns 7kb |L0.73| " + - "L0.72[71,72] 92ns 7kb |L0.72| " + - "L0.71[70,71] 91ns 7kb |L0.71| " + - "L0.70[69,70] 90ns 7kb |L0.70| " + - "L0.69[68,69] 89ns 7kb |L0.69| " + - "L0.68[67,68] 88ns 7kb |L0.68| " + - "L0.67[66,67] 87ns 7kb |L0.67| " + - "L0.66[65,66] 86ns 7kb |L0.66| " + - "L0.65[64,65] 85ns 7kb |L0.65| " + - "L0.64[63,64] 84ns 7kb |L0.64| " + - "L0.63[62,63] 83ns 7kb |L0.63| " + - "L0.62[61,62] 82ns 7kb |L0.62| " + - "L0.61[60,61] 81ns 7kb |L0.61| " + - "L0.60[59,60] 80ns 7kb |L0.60| " + - "L0.59[58,59] 79ns 7kb |L0.59| " + - "L0.58[57,58] 78ns 7kb |L0.58| " + - "L0.57[56,57] 77ns 7kb |L0.57| " + - "L0.56[55,56] 76ns 7kb |L0.56| " + - "L0.55[54,55] 75ns 7kb |L0.55| " + - "L0.54[53,54] 74ns 7kb |L0.54| " + - "L0.53[52,53] 73ns 7kb |L0.53| " + - "L0.52[51,52] 72ns 7kb |L0.52| " + - "L0.51[50,51] 71ns 7kb |L0.51| " + - "L0.50[49,50] 70ns 7kb |L0.50| " + - "L0.49[48,49] 69ns 7kb |L0.49| " + - "L0.48[47,48] 68ns 7kb |L0.48| " + - "L0.47[46,47] 67ns 7kb |L0.47| " + - "L0.46[45,46] 66ns 7kb |L0.46| " + - "L0.45[44,45] 65ns 7kb |L0.45| " + - "L0.44[43,44] 64ns 7kb |L0.44| " + - "L0.43[42,43] 63ns 7kb |L0.43| " + - "L0.42[41,42] 62ns 7kb |L0.42| " + - "L0.41[40,41] 61ns 7kb |L0.41| " + - "L0.40[39,40] 60ns 7kb |L0.40| " + - "L0.39[38,39] 59ns 7kb |L0.39| " + - "L0.38[37,38] 58ns 7kb |L0.38| " + - "L0.37[36,37] 57ns 7kb |L0.37| " + - "L0.36[35,36] 56ns 7kb |L0.36| " + - "L0.35[34,35] 55ns 7kb |L0.35| " + - "L0.34[33,34] 54ns 7kb |L0.34| " + - "L0.33[32,33] 53ns 7kb |L0.33| " + - "L0.32[31,32] 52ns 7kb |L0.32| " + - "L0.31[30,31] 51ns 7kb |L0.31| " + - "L0.30[29,30] 50ns 7kb |L0.30| " + - "L0.29[28,29] 49ns 7kb |L0.29| " + - "L0.28[27,28] 48ns 7kb |L0.28| " + - "L0.27[26,27] 47ns 7kb |L0.27| " + - "L0.26[25,26] 46ns 7kb |L0.26| " + - "L0.25[24,25] 45ns 7kb |L0.25| " + - "L0.24[23,24] 44ns 7kb |L0.24| " + - "L0.23[22,23] 43ns 7kb |L0.23| " + - "L0.22[21,22] 42ns 7kb |L0.22| " + - "L0.21[20,21] 41ns 7kb |L0.21| " + - "L0.20[19,20] 40ns 7kb |L0.20| " + - "L0.19[18,19] 39ns 7kb |L0.19| " + - "L0.18[17,18] 38ns 7kb |L0.18| " + - "L0.17[16,17] 37ns 7kb |L0.17| " + - "L0.16[15,16] 36ns 7kb |L0.16| " + - "L0.15[14,15] 35ns 7kb |L0.15| " + - "L0.14[13,14] 34ns 7kb |L0.14| " + - "L0.13[12,13] 33ns 7kb |L0.13| " + - "L0.12[11,12] 32ns 7kb |L0.12| " + - "L0.11[10,11] 31ns 7kb |L0.11| " + - "L0.10[9,10] 30ns 7kb |L0.10| " + - "L0.9[8,9] 29ns 7kb |L0.9| " + - "L0.8[7,8] 28ns 7kb |L0.8| " + - "L0.7[6,7] 27ns 7kb |L0.7| " + - "L0.6[5,6] 26ns 7kb |L0.6| " + - "L0.5[4,5] 25ns 7kb |L0.5| " + - "L0.4[3,4] 24ns 7kb |L0.4| " - "L0.3[2,3] 23ns 7kb |L0.3| " - "L0.2[1,2] 22ns 7kb |L0.2| " - "L0.1[0,1] 21ns 7kb |L0.1| " - "L1 " + - "L1.195[40,49] 5ns 1mb |L1.195| " + - "L1.194[30,39] 4ns 1mb |L1.194| " + - "L1.193[20,29] 3ns 1mb |L1.193| " + - "L1.192[10,19] 2ns 1mb |L1.192| " - "L1.191[0,9] 1ns 1mb |L1.191| " - - "L1.195[40,49] 5ns 1mb |L1.195| " - - "L1.192[10,19] 2ns 1mb |L1.192| " - - "L1.193[20,29] 3ns 1mb |L1.193| " - - "L1.194[30,39] 4ns 1mb |L1.194| " - - "**** 1 Output Files (parquet_file_id not yet assigned), 6mb total:" - - "L1, all files 6mb " - - "L1.?[0,190] 210ns |------------------------------------------L1.?------------------------------------------|" - - "Committing partition 1:" - - " Soft Deleting 195 files: L0.1, L0.2, L0.3, L0.4, L0.5, L0.6, L0.7, L0.8, L0.9, L0.10, L0.11, L0.12, L0.13, L0.14, L0.15, L0.16, L0.17, L0.18, L0.19, L0.20, L0.21, L0.22, L0.23, L0.24, L0.25, L0.26, L0.27, L0.28, L0.29, L0.30, L0.31, L0.32, L0.33, L0.34, L0.35, L0.36, L0.37, L0.38, L0.39, L0.40, L0.41, L0.42, L0.43, L0.44, L0.45, L0.46, L0.47, L0.48, L0.49, L0.50, L0.51, L0.52, L0.53, L0.54, L0.55, L0.56, L0.57, L0.58, L0.59, L0.60, L0.61, L0.62, L0.63, L0.64, L0.65, L0.66, L0.67, L0.68, L0.69, L0.70, L0.71, L0.72, L0.73, L0.74, L0.75, L0.76, L0.77, L0.78, L0.79, L0.80, L0.81, L0.82, L0.83, L0.84, L0.85, L0.86, L0.87, L0.88, L0.89, L0.90, L0.91, L0.92, L0.93, L0.94, L0.95, L0.96, L0.97, L0.98, L0.99, L0.100, L0.101, L0.102, L0.103, L0.104, L0.105, L0.106, L0.107, L0.108, L0.109, L0.110, L0.111, L0.112, L0.113, L0.114, L0.115, L0.116, L0.117, L0.118, L0.119, L0.120, L0.121, L0.122, L0.123, L0.124, L0.125, L0.126, L0.127, L0.128, L0.129, L0.130, L0.131, L0.132, L0.133, L0.134, L0.135, L0.136, L0.137, L0.138, L0.139, L0.140, L0.141, L0.142, L0.143, L0.144, L0.145, L0.146, L0.147, L0.148, L0.149, L0.150, L0.151, L0.152, L0.153, L0.154, L0.155, L0.156, L0.157, L0.158, L0.159, L0.160, L0.161, L0.162, L0.163, L0.164, L0.165, L0.166, L0.167, L0.168, L0.169, L0.170, L0.171, L0.172, L0.173, L0.174, L0.175, L0.176, L0.177, L0.178, L0.179, L0.180, L0.181, L0.182, L0.183, L0.184, L0.185, L0.186, L0.187, L0.188, L0.189, L0.190, L1.191, L1.192, L1.193, L1.194, L1.195" - - " Creating 1 files" - - "**** Simulation run 1, type=split(CompactAndSplitOutput(TotalSizeLessThanMaxCompactSize))(split_times=[2407]). 16 Input Files, 21mb total:" - - "L1 " - "L1.196[200,209] 6ns 1mb |L1.196| " - "L1.197[400,409] 7ns 1mb |L1.197| " - "L1.198[600,609] 8ns 1mb |L1.198| " @@ -3623,18 +3616,17 @@ async fn not_many_l0_and_overlapped_l1_files() { - "L1.208[2600,2609] 18ns 1mb |L1.208| " - "L1.209[2800,2809] 19ns 1mb |L1.209|" - "L1.210[3000,3009] 20ns 1mb |L1.210|" - - "L1.211[0,190] 210ns 6mb |L1.211| " - "**** 2 Output Files (parquet_file_id not yet assigned), 21mb total:" - - "L2 " - - "L2.?[0,2407] 210ns 17mb |--------------------------------L2.?---------------------------------| " - - "L2.?[2408,3009] 210ns 4mb |-----L2.?------| " + - "L1 " + - "L1.?[0,2407] 210ns 17mb |--------------------------------L1.?---------------------------------| " + - "L1.?[2408,3009] 210ns 4mb |-----L1.?------| " - "Committing partition 1:" - - " Soft Deleting 16 files: L1.196, L1.197, L1.198, L1.199, L1.200, L1.201, L1.202, L1.203, L1.204, L1.205, L1.206, L1.207, L1.208, L1.209, L1.210, L1.211" + - " Soft Deleting 210 files: L0.1, L0.2, L0.3, L0.4, L0.5, L0.6, L0.7, L0.8, L0.9, L0.10, L0.11, L0.12, L0.13, L0.14, L0.15, L0.16, L0.17, L0.18, L0.19, L0.20, L0.21, L0.22, L0.23, L0.24, L0.25, L0.26, L0.27, L0.28, L0.29, L0.30, L0.31, L0.32, L0.33, L0.34, L0.35, L0.36, L0.37, L0.38, L0.39, L0.40, L0.41, L0.42, L0.43, L0.44, L0.45, L0.46, L0.47, L0.48, L0.49, L0.50, L0.51, L0.52, L0.53, L0.54, L0.55, L0.56, L0.57, L0.58, L0.59, L0.60, L0.61, L0.62, L0.63, L0.64, L0.65, L0.66, L0.67, L0.68, L0.69, L0.70, L0.71, L0.72, L0.73, L0.74, L0.75, L0.76, L0.77, L0.78, L0.79, L0.80, L0.81, L0.82, L0.83, L0.84, L0.85, L0.86, L0.87, L0.88, L0.89, L0.90, L0.91, L0.92, L0.93, L0.94, L0.95, L0.96, L0.97, L0.98, L0.99, L0.100, L0.101, L0.102, L0.103, L0.104, L0.105, L0.106, L0.107, L0.108, L0.109, L0.110, L0.111, L0.112, L0.113, L0.114, L0.115, L0.116, L0.117, L0.118, L0.119, L0.120, L0.121, L0.122, L0.123, L0.124, L0.125, L0.126, L0.127, L0.128, L0.129, L0.130, L0.131, L0.132, L0.133, L0.134, L0.135, L0.136, L0.137, L0.138, L0.139, L0.140, L0.141, L0.142, L0.143, L0.144, L0.145, L0.146, L0.147, L0.148, L0.149, L0.150, L0.151, L0.152, L0.153, L0.154, L0.155, L0.156, L0.157, L0.158, L0.159, L0.160, L0.161, L0.162, L0.163, L0.164, L0.165, L0.166, L0.167, L0.168, L0.169, L0.170, L0.171, L0.172, L0.173, L0.174, L0.175, L0.176, L0.177, L0.178, L0.179, L0.180, L0.181, L0.182, L0.183, L0.184, L0.185, L0.186, L0.187, L0.188, L0.189, L0.190, L1.191, L1.192, L1.193, L1.194, L1.195, L1.196, L1.197, L1.198, L1.199, L1.200, L1.201, L1.202, L1.203, L1.204, L1.205, L1.206, L1.207, L1.208, L1.209, L1.210" - " Creating 2 files" - - "**** Final Output Files (28mb written)" - - "L2 " - - "L2.212[0,2407] 210ns 17mb|-------------------------------L2.212--------------------------------| " - - "L2.213[2408,3009] 210ns 4mb |----L2.213-----| " + - "**** Final Output Files (21mb written)" + - "L1 " + - "L1.211[0,2407] 210ns 17mb|-------------------------------L1.211--------------------------------| " + - "L1.212[2408,3009] 210ns 4mb |----L1.212-----| " "### ); } diff --git a/compactor_test_utils/src/lib.rs b/compactor_test_utils/src/lib.rs index cc994ee5e8..22bad3b2d5 100644 --- a/compactor_test_utils/src/lib.rs +++ b/compactor_test_utils/src/lib.rs @@ -1593,6 +1593,69 @@ pub fn create_overlapped_files_3_mix_size(size: i64) -> Vec { vec![l0_3, l0_2, l0_1, l0_4, l0_5, l0_6, l1_1, l1_2] } +/// This setup will return files with ranges as follows: +/// Input: +/// |--L0.1--| |-L0.2-| +/// |--L1.1--| |--L1.2--| |--L1.3--| |--L1.4--| |--L1.5--| |--L1.6--| |--L1.7--| |--L1.8--| +/// l1 size : med large small med med small large med +pub fn create_overlapped_files_mix_sizes_1(small: i64, med: i64, large: i64) -> Vec { + let l1_1 = ParquetFileBuilder::new(11) + .with_compaction_level(CompactionLevel::FileNonOverlapped) + .with_time_range(0, 100) + .with_file_size_bytes(med) + .build(); + let l1_2 = ParquetFileBuilder::new(12) + .with_compaction_level(CompactionLevel::FileNonOverlapped) + .with_time_range(200, 300) + .with_file_size_bytes(large) + .build(); + let l1_3 = ParquetFileBuilder::new(13) + .with_compaction_level(CompactionLevel::FileNonOverlapped) + .with_time_range(400, 500) + .with_file_size_bytes(small) + .build(); + let l1_4 = ParquetFileBuilder::new(14) + .with_compaction_level(CompactionLevel::FileNonOverlapped) + .with_time_range(600, 700) + .with_file_size_bytes(med) + .build(); + let l1_5 = ParquetFileBuilder::new(15) + .with_compaction_level(CompactionLevel::FileNonOverlapped) + .with_time_range(800, 900) + .with_file_size_bytes(med) + .build(); + let l1_6 = ParquetFileBuilder::new(16) + .with_compaction_level(CompactionLevel::FileNonOverlapped) + .with_time_range(1000, 1100) + .with_file_size_bytes(small) + .build(); + let l1_7 = ParquetFileBuilder::new(17) + .with_compaction_level(CompactionLevel::FileNonOverlapped) + .with_time_range(1200, 1300) + .with_file_size_bytes(large) + .build(); + let l1_8 = ParquetFileBuilder::new(18) + .with_compaction_level(CompactionLevel::FileNonOverlapped) + .with_time_range(1400, 1500) + .with_file_size_bytes(med) + .build(); + + // L0_1 overlaps with L1_4 + let l0_1 = ParquetFileBuilder::new(1) + .with_compaction_level(CompactionLevel::Initial) + .with_time_range(650, 750) + .with_file_size_bytes(small) + .build(); + // L0_2 overlaps with L1_5 + let l0_2 = ParquetFileBuilder::new(2) + .with_compaction_level(CompactionLevel::Initial) + .with_time_range(820, 850) + .with_file_size_bytes(small) + .build(); + + vec![l1_3, l1_7, l1_2, l1_1, l1_4, l1_5, l1_6, l0_2, l0_1, l1_8] +} + #[cfg(test)] mod test { use super::*;