Influxdata
/
influxdb
mirror of https://github.com/influxdata/influxdb.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

fix: compactor shouldn't leave small L1s in non-overlap leading edge pattern (#8101) 

* fix: compactor shouldn't leave tiny L1s with non-overlapped leading edge pattern

* chore: insta updates for prior commit
pull/24376/head
Joe-Blount 2023-06-28 12:02:21 -05:00 committed by GitHub
parent b15c6062a9
commit ac9cc24315
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
11 changed files with 790 additions and 449 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

309
compactor/src/components/files_split/non_overlap_split.rs
View File

@ -8,14 +8,22 @@ use super::FilesSplit;
#[derive(Debug)]
/// Split files into `[overlapping_files]` and `[non_overlapping_files]`
/// Split files into `[compact_files]` and `[non_overlapping_files]`
/// To have better and efficient compaction performance, eligible non-overlapped files
/// should not be compacted.
pub struct NonOverlapSplit {}
pub struct NonOverlapSplit {
/// undersized_threshold is the threshold for unnecessarily including & rewriting adjacent
/// small files. This does increase write amplification, so it shouldn't be too high, but
/// it also prevents leaving tiny L1/L2 files that will never be compacted, so it shouldn't
/// be too low.
undersized_threshold: u64,
}
impl NonOverlapSplit {
pub fn new() -> Self {
Self {}
pub fn new(undersized_threshold: u64) -> Self {
Self {
undersized_threshold,
}
}
}
@ -26,10 +34,10 @@ impl Display for NonOverlapSplit {
}
impl FilesSplit for NonOverlapSplit {
/// Return (`[overlapping_files]`, `[non_overlapping_files]`) of given files
/// such that after combining all `overlapping_files` into a new file, the new file will
/// Return (`[compact_files]`, `[non_overlapping_files]`) of given files
/// such that after combining all `compact_files` into a new file, the new file will
/// have no overlap with any file in `non_overlapping_files`.
/// The non_overlapping_files must in the target_level
/// The non_overlapping_files must be in the target_level
///
/// Eligible non-overlapping files are files of the target level that do not
/// overlap on time range of all files in lower-level. All files in target level
@ -44,7 +52,7 @@ impl FilesSplit for NonOverlapSplit {
/// (L1.2, L1.3) must be compacted with L0s to produce the right non-overlapping L1s.
///
/// . Output:
/// . overlapping_files: [L0.1, L0.2, L1.2, L1.3]
/// . compact_files: [L0.1, L0.2, L1.2, L1.3]
/// . non_overlapping_files: [L1.1, L1.4]
///
/// Algorithm:
@ -55,7 +63,12 @@ impl FilesSplit for NonOverlapSplit {
/// ==> L0s' time range: |-------L0's time range --------|
///
/// L1s |--L1.1--| |--L1.2--| |--L1.3--| |--L1.4--|
/// ==> Only L1.1 and L1.4 are completely outside the time range of L0s
/// ==> Only L1.1 and L1.4 are completely outside the time range of L0s.
/// So L1.1 and L1.4 are usually not included in compact_files. However, if either of L1.1 or L1.4
/// are small (below undersized_threshold), they will be included in compact_files to avoid leaving
/// tiny L1 files behind. Note that the application of undersized_threshold can only contiguously
/// expand the set of min_time sorted target level files. So if there was a small L1.5 file, we
/// could not skip over a large L1.4 file to include L1.5 in compact_files.
///
fn apply(
&self,
@ -79,59 +92,89 @@ impl FilesSplit for NonOverlapSplit {
return (vec![], target_level_files);
};
// Sort files of target level by min_time
target_level_files.sort_by_key(|f| f.min_time);
// Convert target_level_files to VecDeque
let mut target_level_files = target_level_files.into_iter().collect::<VecDeque<_>>();
// Closure that checks if a file overlaps with any prev_level_files and add it
// to overlapping_files or non_overlapping_files accordingly. Return true if ovelapping
// Split target_level_files into 3 parts, those before, during and after prev_level_files.
// Since target level files during the time range of prev_level_files must be compacted,
// they are the start of compact_files.
let mut before: Vec<ParquetFile>;
let mut compact_files: Vec<ParquetFile>;
let mut after: Vec<ParquetFile>;
let mut non_overlapping_files = Vec::with_capacity(num_files);
let mut overlapping_files = Vec::with_capacity(num_files);
let mut check_overlapping_and_add = |file: ParquetFile| -> bool {
let mut overlapping = false;
(before, compact_files, after) =
three_range_split(&mut target_level_files, prev_level_range);
// Closure that checks if a file is under the size threshold and adds it to compact_files
let mut check_undersize_and_add = |file: ParquetFile| -> bool {
let mut under = false;
// Check if file overlaps with (min_time, max_time)
if prev_level_range.contains(&file) {
overlapping = true;
overlapping_files.push(file);
if file.file_size_bytes <= self.undersized_threshold as i64 {
under = true;
compact_files.push(file);
} else {
non_overlapping_files.push(file);
}
overlapping
under
};
// Find non-overlapping files which are only at either end of their min-time sorted target_level_files
// pop_front() until hitting the first overlapping file
while let Some(file) = target_level_files.pop_front() {
if check_overlapping_and_add(file) {
break;
}
}
// pop_back() until hitting the first overlapping file
while let Some(file) = target_level_files.pop_back() {
if check_overlapping_and_add(file) {
// Contiguously add `before` files to the list to compact, so long as they're under the size threshold.
before.sort_by_key(|f| f.min_time);
let mut before = before.into_iter().collect::<VecDeque<_>>();
while let Some(file) = before.pop_back() {
if !check_undersize_and_add(file) {
break;
}
}
// Add remaining files to overlapping_files
overlapping_files.extend(target_level_files);
overlapping_files.extend(prev_level_files);
// Contiguously add `after` files to the list to compact, so long as they're under the size threshold.
after.sort_by_key(|f| f.min_time);
let mut after = after.into_iter().collect::<VecDeque<_>>();
while let Some(file) = after.pop_front() {
if !check_undersize_and_add(file) {
break;
}
}
(overlapping_files, non_overlapping_files)
compact_files.extend(prev_level_files);
// Add remaining files to non_overlapping_files.
non_overlapping_files.extend(before);
non_overlapping_files.extend(after);
(compact_files, non_overlapping_files)
}
}
// three_range_split splits the files into 3 vectors: before, during, after the specified time range.
pub fn three_range_split(
files: &mut Vec<ParquetFile>,
range: FilesTimeRange,
) -> (Vec<ParquetFile>, Vec<ParquetFile>, Vec<ParquetFile>) {
let num_files = files.len();
let mut before = Vec::with_capacity(num_files);
let mut during = Vec::with_capacity(num_files);
let mut after = Vec::with_capacity(num_files);
while let Some(file) = files.pop() {
if range.contains(&file) {
during.push(file);
} else if range.before(&file) {
before.push(file);
} else {
after.push(file);
}
}
(before, during, after)
}
#[cfg(test)]
mod tests {
use compactor_test_utils::{
create_l1_files, create_overlapped_files, create_overlapped_files_2,
create_overlapped_l0_l1_files, create_overlapped_l1_l2_files, format_files,
format_files_split,
create_overlapped_files_mix_sizes_1, create_overlapped_l0_l1_files,
create_overlapped_l1_l2_files, format_files, format_files_split,
};
use super::*;
@ -139,7 +182,7 @@ mod tests {
#[test]
fn test_display() {
assert_eq!(
NonOverlapSplit::new().to_string(),
NonOverlapSplit::new(1024 * 1024).to_string(),
"Non-overlapping split for TargetLevel version"
);
}
@ -148,7 +191,7 @@ mod tests {
#[should_panic]
fn test_wrong_target_level() {
let files = create_overlapped_files();
let split = NonOverlapSplit::new();
let split = NonOverlapSplit::new(1024 * 1024);
split.apply(files, CompactionLevel::Initial);
}
@ -158,7 +201,7 @@ mod tests {
)]
fn test_unexpected_compaction_level_2() {
let files = create_overlapped_files();
let split = NonOverlapSplit::new();
let split = NonOverlapSplit::new(1024 * 1024);
// There are L2 files and will panic
split.apply(files, CompactionLevel::FileNonOverlapped);
}
@ -169,7 +212,7 @@ mod tests {
)]
fn test_unexpected_compaction_level_0() {
let files = create_overlapped_files();
let split = NonOverlapSplit::new();
let split = NonOverlapSplit::new(1024 * 1024);
// There are L0 files and will panic
split.apply(files, CompactionLevel::Final);
}
@ -177,7 +220,7 @@ mod tests {
#[test]
fn test_apply_empty_files() {
let files = vec![];
let split = NonOverlapSplit::new();
let split = NonOverlapSplit::new(1024 * 1024);
let (overlap, non_overlap) = split.apply(files, CompactionLevel::FileNonOverlapped);
assert_eq!(overlap.len(), 0);
@ -199,14 +242,14 @@ mod tests {
"###
);
let split = NonOverlapSplit::new();
let split = NonOverlapSplit::new(0);
// Lower level is empty -> all files will be in non_overlapping_files
let (overlap, non_overlap) = split.apply(files.clone(), CompactionLevel::FileNonOverlapped);
assert_eq!(overlap.len(), 0);
assert_eq!(non_overlap.len(), 3);
// target level is empty -> all files will be in overlapping_files
// target level is empty -> all files will be in compact_files
let (overlap, non_overlap) = split.apply(files, CompactionLevel::Final);
assert_eq!(overlap.len(), 3);
assert_eq!(non_overlap.len(), 0);
@ -231,7 +274,7 @@ mod tests {
"###
);
let split = NonOverlapSplit::new();
let split = NonOverlapSplit::new(0);
let (overlap, non_overlap) = split.apply(files, CompactionLevel::FileNonOverlapped);
insta::assert_yaml_snapshot!(
format_files_split("overlap", &overlap, "non_overlap", &non_overlap),
@ -274,7 +317,7 @@ mod tests {
"###
);
let split = NonOverlapSplit::new();
let split = NonOverlapSplit::new(0);
let (overlap, non_overlap) = split.apply(files, CompactionLevel::Final);
insta::assert_yaml_snapshot!(
format_files_split("overlap", &overlap, "non_overlap", &non_overlap),
@ -319,7 +362,7 @@ mod tests {
"###
);
let split = NonOverlapSplit::new();
let split = NonOverlapSplit::new(0);
let (overlap, non_overlap) = split.apply(files, CompactionLevel::FileNonOverlapped);
insta::assert_yaml_snapshot!(
format_files_split("overlap", &overlap, "non_overlap", &non_overlap),
@ -339,4 +382,170 @@ mod tests {
"###
);
}
#[test]
fn test_undersized_1() {
// Create files with levels and time ranges, where all L1 files are undersized.
// Input:
// |--L0.1--| |--L0.2--|
// |--L1.1--| |--L1.2--| |--L1.3--| |--L1.4--|
//
// Output: (compact, non_overlap) = ( [L0.1, L0.2, L1.1, L1.2, L1.3, L1.4] , [] )
// Since all the files are below the undersized threshold, they all get compacted, even though some
// don't overlap.
let files = create_overlapped_files_2(10);
insta::assert_yaml_snapshot!(
format_files("initial", &files),
@r###"
---
- initial
- "L0, all files 10b "
- "L0.2[520,550] 0ns |L0.2| "
- "L0.1[250,350] 0ns |---L0.1---| "
- "L1, all files 10b "
- "L1.13[400,500] 0ns |--L1.13---| "
- "L1.12[200,300] 0ns |--L1.12---| "
- "L1.11[0,100] 0ns |--L1.11---| "
- "L1.14[600,700] 0ns |--L1.14---| "
"###
);
let split = NonOverlapSplit::new(11);
let (compact_files, non_overlap) = split.apply(files, CompactionLevel::FileNonOverlapped);
insta::assert_yaml_snapshot!(
format_files_split("compact_files", &compact_files, "non_overlap", &non_overlap),
@r###"
---
- compact_files
- "L0, all files 10b "
- "L0.2[520,550] 0ns |L0.2| "
- "L0.1[250,350] 0ns |---L0.1---| "
- "L1, all files 10b "
- "L1.12[200,300] 0ns |--L1.12---| "
- "L1.13[400,500] 0ns |--L1.13---| "
- "L1.11[0,100] 0ns |--L1.11---| "
- "L1.14[600,700] 0ns |--L1.14---| "
- non_overlap
"###
);
}
#[test]
fn test_undersized_2() {
// Create files with levels and time ranges, where some non-overlapping L1 files are undersized.
// . Input:
// |--L0.1--| |-L0.2-|
// |--L1.1--| |--L1.2--| |--L1.3--| |--L1.4--| |--L1.5--| |--L1.6--| |--L1.7--| |--L1.8--|
// underized: no no yes x x yes no no
// Results:
// L1.4 and L1.5 are compacted regardless of their size because they overlap L0s.
// L1.3 and L1.6 are compacted because they are undersized and contiguous to the overlapping files.
// L1.1, L1.2, L1.7, and L1.8 are not compacted because they are not undersized.
//
let files = create_overlapped_files_mix_sizes_1(10, 20, 30);
insta::assert_yaml_snapshot!(
format_files("initial", &files),
@r###"
---
- initial
- "L0 "
- "L0.2[820,850] 0ns 10b |L0.2| "
- "L0.1[650,750] 0ns 10b |L0.1| "
- "L1 "
- "L1.13[400,500] 0ns 10b |L1.13| "
- "L1.17[1200,1300] 0ns 30b |L1.17| "
- "L1.12[200,300] 0ns 30b |L1.12| "
- "L1.11[0,100] 0ns 20b |L1.11| "
- "L1.14[600,700] 0ns 20b |L1.14| "
- "L1.15[800,900] 0ns 20b |L1.15| "
- "L1.16[1000,1100] 0ns 10b |L1.16| "
- "L1.18[1400,1500] 0ns 20b |L1.18|"
"###
);
let split = NonOverlapSplit::new(11);
let (compact_files, non_overlap) = split.apply(files, CompactionLevel::FileNonOverlapped);
insta::assert_yaml_snapshot!(
format_files_split("compact_files", &compact_files, "non_overlap", &non_overlap),
@r###"
---
- compact_files
- "L0 "
- "L0.2[820,850] 0ns 10b |L0.2| "
- "L0.1[650,750] 0ns 10b |---L0.1---| "
- "L1 "
- "L1.15[800,900] 0ns 20b |--L1.15---| "
- "L1.14[600,700] 0ns 20b |--L1.14---| "
- "L1.13[400,500] 0ns 10b |--L1.13---| "
- "L1.16[1000,1100] 0ns 10b |--L1.16---| "
- non_overlap
- "L1 "
- "L1.12[200,300] 0ns 30b |L1.12| "
- "L1.17[1200,1300] 0ns 30b |L1.17| "
- "L1.11[0,100] 0ns 20b |L1.11| "
- "L1.18[1400,1500] 0ns 20b |L1.18|"
"###
);
}
#[test]
fn test_undersized_3() {
// This case is like undersized 3, except that the outermost L1 files (L1 & L18) are also undersized. But since
// they're separated from the overlapping files by oversized files, they are not compacted.
// Input:
// |--L0.1--| |-L0.2-|
// |--L1.1--| |--L1.2--| |--L1.3--| |--L1.4--| |--L1.5--| |--L1.6--| |--L1.7--| |--L1.8--|
// underized: yes no yes x x yes no yes
// Results:
// L1.4 and L1.5 are compacted regardless of their size because they overlap.
// L1.3 and L1.6 are compacted because they are undersized and contiguous to the overlapping files.
// L1.2 and L1.7 are not compacted because they are not undersized.
// L1.1 and L1.8 are not compacted because even though they are undersized, they are not contiguous to the
// overlapping files.
//
let files = create_overlapped_files_mix_sizes_1(10, 20, 30);
insta::assert_yaml_snapshot!(
format_files("initial", &files),
@r###"
---
- initial
- "L0 "
- "L0.2[820,850] 0ns 10b |L0.2| "
- "L0.1[650,750] 0ns 10b |L0.1| "
- "L1 "
- "L1.13[400,500] 0ns 10b |L1.13| "
- "L1.17[1200,1300] 0ns 30b |L1.17| "
- "L1.12[200,300] 0ns 30b |L1.12| "
- "L1.11[0,100] 0ns 20b |L1.11| "
- "L1.14[600,700] 0ns 20b |L1.14| "
- "L1.15[800,900] 0ns 20b |L1.15| "
- "L1.16[1000,1100] 0ns 10b |L1.16| "
- "L1.18[1400,1500] 0ns 20b |L1.18|"
"###
);
let split = NonOverlapSplit::new(21);
let (compact_files, non_overlap) = split.apply(files, CompactionLevel::FileNonOverlapped);
insta::assert_yaml_snapshot!(
format_files_split("compact_files", &compact_files, "non_overlap", &non_overlap),
@r###"
---
- compact_files
- "L0 "
- "L0.2[820,850] 0ns 10b |L0.2| "
- "L0.1[650,750] 0ns 10b |---L0.1---| "
- "L1 "
- "L1.15[800,900] 0ns 20b |--L1.15---| "
- "L1.14[600,700] 0ns 20b |--L1.14---| "
- "L1.13[400,500] 0ns 10b |--L1.13---| "
- "L1.16[1000,1100] 0ns 10b |--L1.16---| "
- non_overlap
- "L1 "
- "L1.12[200,300] 0ns 30b |L1.12| "
- "L1.17[1200,1300] 0ns 30b |L1.17| "
- "L1.11[0,100] 0ns 20b |L1.11| "
- "L1.18[1400,1500] 0ns 20b |L1.18|"
"###
);
}
}

2
compactor/src/components/hardcoded.rs
View File

@ -376,7 +376,7 @@ fn make_file_classifier(config: &Config) -> Arc<dyn FileClassifier> {
Arc::new(LoggingFileClassifierWrapper::new(Arc::new(
SplitBasedFileClassifier::new(
TargetLevelSplit::new(),
NonOverlapSplit::new(),
NonOverlapSplit::new(config.max_desired_file_size_bytes / 20), // rewrite non-overlapping files up to 5% of max
UpgradeSplit::new(config.max_desired_file_size_bytes),
LoggingSplitOrCompactWrapper::new(MetricsSplitOrCompactWrapper::new(
SplitCompact::new(

5
compactor/src/file_group.rs
View File

@ -33,6 +33,11 @@ impl FilesTimeRange {
pub fn contains(&self, file: &ParquetFile) -> bool {
file.min_time <= self.max_time && file.max_time >= self.min_time
}
/// Does `file` fall before the range of self?
pub fn before(&self, file: &ParquetFile) -> bool {
file.max_time < self.min_time
}
}
/// returns true if the time range of `file` overlaps with any in `files`

27
compactor/tests/layouts/accumulated_size.rs
View File

@ -250,21 +250,22 @@ async fn small_l1_plus_nonoverlapping_l0s_single_run() {
- "L0.4[31,39] 41ns 10kb |------L0.4------| "
- "L1 "
- "L1.1[0,9] 11ns 40kb |-------L1.1-------| "
- "**** Simulation run 0, type=compact(TotalSizeLessThanMaxCompactSize). 3 Input Files, 30kb total:"
- "L0, all files 10kb "
- "L0.4[31,39] 41ns |---------L0.4----------| "
- "L0.3[21,29] 31ns |---------L0.3----------| "
- "L0.2[11,19] 21ns |---------L0.2----------| "
- "**** 1 Output Files (parquet_file_id not yet assigned), 30kb total:"
- "L1, all files 30kb "
- "L1.?[11,39] 41ns |------------------------------------------L1.?------------------------------------------|"
- "Committing partition 1:"
- " Soft Deleting 3 files: L0.2, L0.3, L0.4"
- " Creating 1 files"
- "**** Final Output Files (30kb written)"
- "**** Simulation run 0, type=compact(TotalSizeLessThanMaxCompactSize). 4 Input Files, 70kb total:"
- "L0 "
- "L0.4[31,39] 41ns 10kb |------L0.4------| "
- "L0.3[21,29] 31ns 10kb |------L0.3------| "
- "L0.2[11,19] 21ns 10kb |------L0.2------| "
- "L1 "
- "L1.1[0,9] 11ns 40kb |-------L1.1-------| "
- "L1.5[11,39] 41ns 30kb |-----------------------------L1.5-----------------------------| "
- "**** 1 Output Files (parquet_file_id not yet assigned), 70kb total:"
- "L1, all files 70kb "
- "L1.?[0,39] 41ns |------------------------------------------L1.?------------------------------------------|"
- "Committing partition 1:"
- " Soft Deleting 4 files: L1.1, L0.2, L0.3, L0.4"
- " Creating 1 files"
- "**** Final Output Files (70kb written)"
- "L1, all files 70kb "
- "L1.5[0,39] 41ns |------------------------------------------L1.5------------------------------------------|"
"###
);
}

58
compactor/tests/layouts/backfill.rs
View File

@ -390,9 +390,9 @@ async fn random_backfill_empty_partition() {
- "L0.77[671,714] 1.05us 5mb |L0.77-| "
- "L0.85[521,670] 1.05us 16mb|----------L0.85-----------| "
- "L1 "
- "L1.83[521,714] 1.04us 75mb|---------------L1.83---------------| "
- "L1.89[918,986] 1.04us 34mb |---L1.89---| "
- "L1.88[715,917] 1.04us 100mb |----------------L1.88----------------| "
- "L1.83[521,714] 1.04us 75mb|---------------L1.83---------------| "
- "**** 3 Output Files (parquet_file_id not yet assigned), 259mb total:"
- "L1 "
- "L1.?[521,701] 1.05us 100mb|--------------L1.?--------------| "
@ -784,14 +784,7 @@ async fn random_backfill_over_l2s() {
- "Committing partition 1:"
- " Soft Deleting 2 files: L0.90, L0.91"
- " Creating 2 files"
- "**** Simulation run 14, type=split(ReduceOverlap)(split_times=[699]). 1 Input Files, 65mb total:"
- "L1, all files 65mb "
- "L1.94[612,733] 1.05us |-----------------------------------------L1.94-----------------------------------------| "
- "**** 2 Output Files (parquet_file_id not yet assigned), 65mb total:"
- "L1 "
- "L1.?[612,699] 1.05us 47mb|-----------------------------L1.?-----------------------------| "
- "L1.?[700,733] 1.05us 18mb |---------L1.?---------| "
- "**** Simulation run 15, type=split(ReduceOverlap)(split_times=[499, 599]). 1 Input Files, 100mb total:"
- "**** Simulation run 14, type=split(ReduceOverlap)(split_times=[499, 599]). 1 Input Files, 100mb total:"
- "L1, all files 100mb "
- "L1.93[423,611] 1.05us |-----------------------------------------L1.93------------------------------------------|"
- "**** 3 Output Files (parquet_file_id not yet assigned), 100mb total:"
@ -799,7 +792,7 @@ async fn random_backfill_over_l2s() {
- "L1.?[423,499] 1.05us 40mb|---------------L1.?---------------| "
- "L1.?[500,599] 1.05us 52mb |--------------------L1.?---------------------| "
- "L1.?[600,611] 1.05us 7mb |L1.?|"
- "**** Simulation run 16, type=split(ReduceOverlap)(split_times=[299, 399]). 1 Input Files, 100mb total:"
- "**** Simulation run 15, type=split(ReduceOverlap)(split_times=[299, 399]). 1 Input Files, 100mb total:"
- "L1, all files 100mb "
- "L1.92[233,422] 1.05us |-----------------------------------------L1.92------------------------------------------|"
- "**** 3 Output Files (parquet_file_id not yet assigned), 100mb total:"
@ -807,7 +800,7 @@ async fn random_backfill_over_l2s() {
- "L1.?[233,299] 1.05us 35mb|------------L1.?-------------| "
- "L1.?[300,399] 1.05us 52mb |--------------------L1.?---------------------| "
- "L1.?[400,422] 1.05us 13mb |--L1.?--| "
- "**** Simulation run 17, type=split(ReduceOverlap)(split_times=[99, 199]). 1 Input Files, 100mb total:"
- "**** Simulation run 16, type=split(ReduceOverlap)(split_times=[99, 199]). 1 Input Files, 100mb total:"
- "L1, all files 100mb "
- "L1.86[42,232] 1.05us |-----------------------------------------L1.86------------------------------------------|"
- "**** 3 Output Files (parquet_file_id not yet assigned), 100mb total:"
@ -815,6 +808,13 @@ async fn random_backfill_over_l2s() {
- "L1.?[42,99] 1.05us 30mb |----------L1.?-----------| "
- "L1.?[100,199] 1.05us 52mb |--------------------L1.?--------------------| "
- "L1.?[200,232] 1.05us 18mb |----L1.?-----| "
- "**** Simulation run 17, type=split(ReduceOverlap)(split_times=[699]). 1 Input Files, 65mb total:"
- "L1, all files 65mb "
- "L1.94[612,733] 1.05us |-----------------------------------------L1.94-----------------------------------------| "
- "**** 2 Output Files (parquet_file_id not yet assigned), 65mb total:"
- "L1 "
- "L1.?[612,699] 1.05us 47mb|-----------------------------L1.?-----------------------------| "
- "L1.?[700,733] 1.05us 18mb |---------L1.?---------| "
- "**** Simulation run 18, type=split(ReduceOverlap)(split_times=[799, 899]). 1 Input Files, 100mb total:"
- "L1, all files 100mb "
- "L1.95[734,922] 1.05us |-----------------------------------------L1.95------------------------------------------|"
@ -828,8 +828,8 @@ async fn random_backfill_over_l2s() {
- " Creating 14 files"
- "**** Simulation run 19, type=split(CompactAndSplitOutput(FoundSubsetLessThanMaxCompactSize))(split_times=[71, 142]). 4 Input Files, 282mb total:"
- "L1 "
- "L1.105[42,99] 1.05us 30mb |--------L1.105---------| "
- "L1.106[100,199] 1.05us 52mb |------------------L1.106------------------| "
- "L1.103[42,99] 1.05us 30mb |--------L1.103---------| "
- "L1.104[100,199] 1.05us 52mb |------------------L1.104------------------| "
- "L2 "
- "L2.1[0,99] 99ns 100mb |-------------------L2.1-------------------| "
- "L2.2[100,199] 199ns 100mb |-------------------L2.2-------------------| "
@ -839,12 +839,12 @@ async fn random_backfill_over_l2s() {
- "L2.?[72,142] 1.05us 99mb |------------L2.?-------------| "
- "L2.?[143,199] 1.05us 82mb |---------L2.?----------| "
- "Committing partition 1:"
- " Soft Deleting 4 files: L2.1, L2.2, L1.105, L1.106"
- " Soft Deleting 4 files: L2.1, L2.2, L1.103, L1.104"
- " Creating 3 files"
- "**** Simulation run 20, type=split(CompactAndSplitOutput(FoundSubsetLessThanMaxCompactSize))(split_times=[265]). 3 Input Files, 153mb total:"
- "L1 "
- "L1.107[200,232] 1.05us 18mb|----------L1.107-----------| "
- "L1.102[233,299] 1.05us 35mb |-------------------------L1.102--------------------------| "
- "L1.105[200,232] 1.05us 18mb|----------L1.105-----------| "
- "L1.100[233,299] 1.05us 35mb |-------------------------L1.100--------------------------| "
- "L2 "
- "L2.3[200,299] 299ns 100mb|-----------------------------------------L2.3------------------------------------------| "
- "**** 2 Output Files (parquet_file_id not yet assigned), 153mb total:"
@ -852,12 +852,12 @@ async fn random_backfill_over_l2s() {
- "L2.?[200,265] 1.05us 100mb|--------------------------L2.?---------------------------| "
- "L2.?[266,299] 1.05us 53mb |-----------L2.?------------| "
- "Committing partition 1:"
- " Soft Deleting 3 files: L2.3, L1.102, L1.107"
- " Soft Deleting 3 files: L2.3, L1.100, L1.105"
- " Creating 2 files"
- "**** Simulation run 21, type=split(CompactAndSplitOutput(FoundSubsetLessThanMaxCompactSize))(split_times=[376, 452]). 4 Input Files, 265mb total:"
- "L1 "
- "L1.103[300,399] 1.05us 52mb|------------------L1.103------------------| "
- "L1.104[400,422] 1.05us 13mb |L1.104-| "
- "L1.101[300,399] 1.05us 52mb|------------------L1.101------------------| "
- "L1.102[400,422] 1.05us 13mb |L1.102-| "
- "L2 "
- "L2.4[300,399] 399ns 100mb|-------------------L2.4-------------------| "
- "L2.5[400,499] 499ns 100mb |-------------------L2.5-------------------| "
@ -867,17 +867,17 @@ async fn random_backfill_over_l2s() {
- "L2.?[377,452] 1.05us 100mb |-------------L2.?--------------| "
- "L2.?[453,499] 1.05us 64mb |-------L2.?-------| "
- "Committing partition 1:"
- " Soft Deleting 4 files: L2.4, L2.5, L1.103, L1.104"
- " Soft Deleting 4 files: L2.4, L2.5, L1.101, L1.102"
- " Creating 3 files"
- "**** Simulation run 22, type=split(ReduceOverlap)(split_times=[452]). 1 Input Files, 40mb total:"
- "L1, all files 40mb "
- "L1.99[423,499] 1.05us |-----------------------------------------L1.99------------------------------------------|"
- "L1.97[423,499] 1.05us |-----------------------------------------L1.97------------------------------------------|"
- "**** 2 Output Files (parquet_file_id not yet assigned), 40mb total:"
- "L1 "
- "L1.?[423,452] 1.05us 15mb|--------------L1.?--------------| "
- "L1.?[453,499] 1.05us 25mb |------------------------L1.?------------------------| "
- "Committing partition 1:"
- " Soft Deleting 1 files: L1.99"
- " Soft Deleting 1 files: L1.97"
- " Creating 2 files"
- "**** Simulation run 23, type=split(CompactAndSplitOutput(FoundSubsetLessThanMaxCompactSize))(split_times=[437, 497]). 4 Input Files, 204mb total:"
- "L1 "
@ -896,8 +896,8 @@ async fn random_backfill_over_l2s() {
- " Creating 3 files"
- "**** Simulation run 24, type=split(CompactAndSplitOutput(FoundSubsetLessThanMaxCompactSize))(split_times=[577, 654]). 4 Input Files, 259mb total:"
- "L1 "
- "L1.100[500,599] 1.05us 52mb|------------------L1.100------------------| "
- "L1.101[600,611] 1.05us 7mb |L1.101| "
- "L1.98[500,599] 1.05us 52mb|------------------L1.98-------------------| "
- "L1.99[600,611] 1.05us 7mb |L1.99| "
- "L2 "
- "L2.6[500,599] 599ns 100mb|-------------------L2.6-------------------| "
- "L2.7[600,699] 699ns 100mb |-------------------L2.7-------------------| "
@ -907,17 +907,17 @@ async fn random_backfill_over_l2s() {
- "L2.?[578,654] 1.05us 99mb |--------------L2.?--------------| "
- "L2.?[655,699] 1.05us 60mb |------L2.?-------| "
- "Committing partition 1:"
- " Soft Deleting 4 files: L2.6, L2.7, L1.100, L1.101"
- " Soft Deleting 4 files: L2.6, L2.7, L1.98, L1.99"
- " Creating 3 files"
- "**** Simulation run 25, type=split(ReduceOverlap)(split_times=[654]). 1 Input Files, 47mb total:"
- "L1, all files 47mb "
- "L1.97[612,699] 1.05us |-----------------------------------------L1.97------------------------------------------|"
- "L1.106[612,699] 1.05us |-----------------------------------------L1.106-----------------------------------------|"
- "**** 2 Output Files (parquet_file_id not yet assigned), 47mb total:"
- "L1 "
- "L1.?[612,654] 1.05us 23mb|------------------L1.?-------------------| "
- "L1.?[655,699] 1.05us 24mb |-------------------L1.?--------------------| "
- "Committing partition 1:"
- " Soft Deleting 1 files: L1.97"
- " Soft Deleting 1 files: L1.106"
- " Creating 2 files"
- "**** Simulation run 26, type=split(CompactAndSplitOutput(FoundSubsetLessThanMaxCompactSize))(split_times=[637, 696]). 4 Input Files, 206mb total:"
- "L1 "
@ -936,7 +936,7 @@ async fn random_backfill_over_l2s() {
- " Creating 3 files"
- "**** Simulation run 27, type=split(CompactAndSplitOutput(FoundSubsetLessThanMaxCompactSize))(split_times=[765]). 3 Input Files, 153mb total:"
- "L1 "
- "L1.98[700,733] 1.05us 18mb|-----------L1.98-----------| "
- "L1.107[700,733] 1.05us 18mb|----------L1.107-----------| "
- "L1.108[734,799] 1.05us 35mb |-------------------------L1.108--------------------------| "
- "L2 "
- "L2.8[700,799] 799ns 100mb|-----------------------------------------L2.8------------------------------------------| "
@ -945,7 +945,7 @@ async fn random_backfill_over_l2s() {
- "L2.?[700,765] 1.05us 100mb|--------------------------L2.?---------------------------| "
- "L2.?[766,799] 1.05us 53mb |-----------L2.?------------| "
- "Committing partition 1:"
- " Soft Deleting 3 files: L2.8, L1.98, L1.108"
- " Soft Deleting 3 files: L2.8, L1.107, L1.108"
- " Creating 2 files"
- "**** Final Output Files (3.78gb written)"
- "L1 "

74
compactor/tests/layouts/common_use_cases.rs
View File

@ -441,9 +441,9 @@ async fn test_keep_ingesting_l0_files_40_percent_overlap_l1_left() {
- " Creating 2 files"
- "**** Simulation run 84, type=split(CompactAndSplitOutput(TotalSizeLessThanMaxCompactSize))(split_times=[3362]). 6 Input Files, 121mb total:"
- "L1 "
- "L1.500[3293,3341] 334ns 24mb |----L1.500-----| "
- "L1.493[3244,3292] 329ns 24mb |----L1.493-----| "
- "L1.486[3200,3243] 324ns 20mb |----L1.486----| "
- "L1.500[3293,3341] 334ns 24mb |----L1.500-----| "
- "L1.508[3392,3404] 339ns 7mb |L1.508|"
- "L1.507[3342,3391] 339ns 25mb |-----L1.507-----| "
- "L2 "
@ -624,9 +624,9 @@ async fn test_keep_ingesting_l0_files_40_percent_overlap_l1_left() {
- " Creating 2 files"
- "**** Simulation run 124, type=split(CompactAndSplitOutput(TotalSizeLessThanMaxCompactSize))(split_times=[4962]). 6 Input Files, 121mb total:"
- "L1 "
- "L1.740[4893,4941] 494ns 24mb |----L1.740-----| "
- "L1.733[4844,4892] 489ns 24mb |----L1.733-----| "
- "L1.726[4800,4843] 484ns 20mb |----L1.726----| "
- "L1.740[4893,4941] 494ns 24mb |----L1.740-----| "
- "L1.748[4992,5004] 499ns 7mb |L1.748|"
- "L1.747[4942,4991] 499ns 25mb |-----L1.747-----| "
- "L2 "
@ -820,3 +820,73 @@ async fn test_keep_ingesting_l0_files_40_percent_overlap_output_250mb() {
"###
);
}
// The (assumed) most common ingest pattern is a leading edge, overlapping pattern. All data is written approximately
// at the time of the data, but with enough variance to cause overlaps between L0s. Those overlaps mean the first in
// any sequence of L0s overlaps the L1 from the previously compacted L0s.
// This test case covers a slightly different scenario.
// Instead, imagine if the incoming data's timestamps have a courrse granularity (every few seconds, minute, etc).
// This can produce a leading edge non-overlapping pattern in the L0s, which means the L0s won't overlap their prior
// L1. This ingest pattern has a tendency to leave small L1s. If the partition gets enough data, most of them will
// eventually get compacted. But if the partition is small, it may be left with several small L1s that should be
// compacted together. This test case is an incremental simulation of that scenario.
// We can't just create a big backlog of non-overlapping L0s because they'll be compacted all at once and not relfect
// behavior from a compactor that keeps up with the non-overlapping L0s as they arrive. So this test case creates one
// small L1 representing a prior compaction's output, and just enough L0s to trigger a compaction. The purpose of
// this test is to ensure the prior small L1 is not left behind.
#[tokio::test]
async fn test_tiny_ingest_non_overlapping() {
test_helpers::maybe_start_logging();
let setup = layout_setup_builder().await.build().await;
// Small L1 simumlating output from prior compaction
setup
.partition
.create_parquet_file(
parquet_builder()
.with_min_time(0)
.with_max_time(5)
.with_file_size_bytes(20000)
.with_max_l0_created_at(Time::from_timestamp_nanos(6))
.with_compaction_level(CompactionLevel::FileNonOverlapped),
)
.await;
// Add two L0 file
setup
.partition
.create_parquet_file(
parquet_builder()
.with_min_time(10)
.with_max_time(19)
.with_file_size_bytes(2000)
.with_max_l0_created_at(Time::from_timestamp_nanos(20))
.with_compaction_level(CompactionLevel::Initial),
)
.await;
setup
.partition
.create_parquet_file(
parquet_builder()
.with_min_time(20)
.with_max_time(29)
.with_file_size_bytes(2000)
.with_max_l0_created_at(Time::from_timestamp_nanos(30))
.with_compaction_level(CompactionLevel::Initial),
)
.await;
run_layout_scenario(&setup).await;
let files = setup.list_by_table_not_to_delete().await;
insta::assert_yaml_snapshot!(
format_files("final output", &files),
@r###"
---
- final output
- "L1, all files 23kb "
- "L1.4[0,29] 30ns |------------------------------------------L1.4------------------------------------------|"
"###
);
}

39
compactor/tests/layouts/core.rs
View File

@ -402,32 +402,33 @@ async fn l1_with_non_overlapping_l0_larger() {
- "L1.2[100,149] 2ns 50mb |--L1.2---| "
- "L1.3[150,199] 3ns 20mb |--L1.3---| "
- "L1.4[200,249] 4ns 3mb |--L1.4---| "
- "**** Simulation run 0, type=compact(TotalSizeLessThanMaxCompactSize). 3 Input Files, 15mb total:"
- "L0, all files 5mb "
- "L0.7[400,450] 13ns |------------L0.7------------|"
- "L0.6[350,400] 12ns |------------L0.6------------| "
- "L0.5[300,350] 11ns |------------L0.5------------| "
- "**** 1 Output Files (parquet_file_id not yet assigned), 15mb total:"
- "L1, all files 15mb "
- "L1.?[300,450] 13ns |------------------------------------------L1.?------------------------------------------|"
- "Committing partition 1:"
- " Soft Deleting 3 files: L0.5, L0.6, L0.7"
- " Creating 1 files"
- "**** Simulation run 1, type=split(CompactAndSplitOutput(TotalSizeLessThanMaxCompactSize))(split_times=[370]). 5 Input Files, 108mb total:"
- "**** Simulation run 0, type=compact(TotalSizeLessThanMaxCompactSize). 4 Input Files, 18mb total:"
- "L0 "
- "L0.7[400,450] 13ns 5mb |------L0.7------|"
- "L0.6[350,400] 12ns 5mb |------L0.6------| "
- "L0.5[300,350] 11ns 5mb |------L0.5------| "
- "L1 "
- "L1.4[200,249] 4ns 3mb |-----L1.4------| "
- "**** 1 Output Files (parquet_file_id not yet assigned), 18mb total:"
- "L1, all files 18mb "
- "L1.?[200,450] 13ns |------------------------------------------L1.?------------------------------------------|"
- "Committing partition 1:"
- " Soft Deleting 4 files: L1.4, L0.5, L0.6, L0.7"
- " Creating 1 files"
- "**** Simulation run 1, type=split(CompactAndSplitOutput(TotalSizeLessThanMaxCompactSize))(split_times=[370]). 4 Input Files, 108mb total:"
- "L1 "
- "L1.4[200,249] 4ns 3mb |--L1.4---| "
- "L1.3[150,199] 3ns 20mb |--L1.3---| "
- "L1.2[100,149] 2ns 50mb |--L1.2---| "
- "L1.1[50,99] 1ns 20mb |--L1.1---| "
- "L1.8[300,450] 13ns 15mb |-------------L1.8--------------| "
- "L1.3[150,199] 3ns 20mb |--L1.3---| "
- "L1.8[200,450] 13ns 18mb |-------------------------L1.8-------------------------| "
- "**** 2 Output Files (parquet_file_id not yet assigned), 108mb total:"
- "L2 "
- "L2.?[50,370] 13ns 86mb |---------------------------------L2.?---------------------------------| "
- "L2.?[371,450] 13ns 22mb |-----L2.?------| "
- "Committing partition 1:"
- " Soft Deleting 5 files: L1.1, L1.2, L1.3, L1.4, L1.8"
- " Soft Deleting 4 files: L1.1, L1.2, L1.3, L1.8"
- " Creating 2 files"
- "**** Final Output Files (123mb written)"
- "**** Final Output Files (126mb written)"
- "L2 "
- "L2.9[50,370] 13ns 86mb |---------------------------------L2.9---------------------------------| "
- "L2.10[371,450] 13ns 22mb |-----L2.10-----| "
@ -646,7 +647,6 @@ async fn many_l1_with_non_overlapping_l0() {
- " Creating 1 files"
- "**** Simulation run 1, type=split(CompactAndSplitOutput(TotalSizeLessThanMaxCompactSize))(split_times=[530]). 11 Input Files, 88mb total:"
- "L1 "
- "L1.10[500,549] 10ns 7mb |L1.10| "
- "L1.9[450,499] 9ns 7mb |L1.9-| "
- "L1.8[400,449] 8ns 7mb |L1.8-| "
- "L1.7[350,399] 7ns 7mb |L1.7-| "
@ -656,6 +656,7 @@ async fn many_l1_with_non_overlapping_l0() {
- "L1.3[150,199] 3ns 7mb |L1.3-| "
- "L1.2[100,149] 2ns 8mb |L1.2-| "
- "L1.1[50,99] 1ns 9mb |L1.1-| "
- "L1.10[500,549] 10ns 7mb |L1.10| "
- "L1.14[600,650] 23ns 15mb |L1.14| "
- "**** 2 Output Files (parquet_file_id not yet assigned), 88mb total:"
- "L2 "
@ -741,8 +742,8 @@ async fn large_l1_with_non_overlapping_l0() {
- " Creating 1 files"
- "**** Simulation run 1, type=split(CompactAndSplitOutput(TotalSizeLessThanMaxCompactSize))(split_times=[375]). 3 Input Files, 185mb total:"
- "L1 "
- "L1.2[100,149] 2ns 80mb |L1.2-| "
- "L1.1[50,99] 1ns 90mb |L1.1-| "
- "L1.2[100,149] 2ns 80mb |L1.2-| "
- "L1.6[600,650] 23ns 15mb |L1.6-| "
- "**** 2 Output Files (parquet_file_id not yet assigned), 185mb total:"
- "L2 "

4
compactor/tests/layouts/knobs.rs
View File

@ -2236,8 +2236,8 @@ async fn all_overlapping_l0_max_input_bytes_per_partition() {
- "L0.392[170978,171442] 10ns 21kb |L0.392| "
- "L0.427[167315,170977] 10ns 169kb|-L0.427-| "
- "L1 "
- "L1.455[167315,191189] 9ns 10mb|----------------------------L1.455-----------------------------| "
- "L1.456[191190,200000] 9ns 4mb |--------L1.456--------| "
- "L1.455[167315,191189] 9ns 10mb|----------------------------L1.455-----------------------------| "
- "**** 2 Output Files (parquet_file_id not yet assigned), 15mb total:"
- "L1 "
- "L1.?[167315,188871] 10ns 10mb|--------------------------L1.?---------------------------| "
@ -4178,8 +4178,8 @@ async fn all_overlapping_l0_max_input_bytes_per_partition_small_max_desired_file
- "L0.392[170978,171442] 10ns 21kb |L0.392| "
- "L0.427[167315,170977] 10ns 169kb|-L0.427-| "
- "L1 "
- "L1.455[167315,191189] 9ns 10mb|----------------------------L1.455-----------------------------| "
- "L1.456[191190,200000] 9ns 4mb |--------L1.456--------| "
- "L1.455[167315,191189] 9ns 10mb|----------------------------L1.455-----------------------------| "
- "**** 2 Output Files (parquet_file_id not yet assigned), 15mb total:"
- "L1 "
- "L1.?[167315,188871] 10ns 10mb|--------------------------L1.?---------------------------| "

2
compactor/tests/layouts/large_files.rs
View File

@ -397,8 +397,8 @@ async fn two_large_files_total_over_max_compact_size_small_overlap_range() {
- "L1 "
- "L1.4[668,1000] 9ns 50mb |------------------------------------------L1.4------------------------------------------|"
- "L2 "
- "L2.5[800,934] 8ns 101mb |---------------L2.5---------------| "
- "L2.6[935,1000] 8ns 50mb |-----L2.6------| "
- "L2.5[800,934] 8ns 101mb |---------------L2.5---------------| "
- "**** 2 Output Files (parquet_file_id not yet assigned), 200mb total:"
- "L2 "
- "L2.?[668,835] 9ns 101mb |-------------------L2.?--------------------| "

656
compactor/tests/layouts/many_files.rs
View File

@ -398,14 +398,12 @@ async fn many_l1_files() {
- " Creating 1 files"
- "**** Simulation run 1, type=split(CompactAndSplitOutput(TotalSizeLessThanMaxCompactSize))(split_times=[20]). 20 Input Files, 203mb total:"
- "L1 "
- "L1.14[26,27] 14ns 10mb |L1.14| "
- "L1.15[28,29] 15ns 10mb |L1.15| "
- "L1.16[30,31] 16ns 10mb |L1.16| "
- "L1.17[32,33] 17ns 10mb |L1.17| "
- "L1.18[34,35] 18ns 10mb |L1.18| "
- "L1.19[36,37] 19ns 10mb |L1.19|"
- "L1.20[38,39] 20ns 10mb |L1.20|"
- "L1.12[22,23] 12ns 10mb |L1.12| "
- "L1.19[36,37] 19ns 10mb |L1.19|"
- "L1.18[34,35] 18ns 10mb |L1.18| "
- "L1.17[32,33] 17ns 10mb |L1.17| "
- "L1.16[30,31] 16ns 10mb |L1.16| "
- "L1.15[28,29] 15ns 10mb |L1.15| "
- "L1.11[20,21] 11ns 10mb |L1.11| "
- "L1.10[18,19] 10ns 10mb |L1.10| "
- "L1.9[16,17] 9ns 10mb |L1.9| "
@ -417,6 +415,8 @@ async fn many_l1_files() {
- "L1.3[4,5] 3ns 10mb |L1.3| "
- "L1.2[2,3] 2ns 10mb |L1.2| "
- "L1.1[0,1] 1ns 10mb |L1.1| "
- "L1.14[26,27] 14ns 10mb |L1.14| "
- "L1.12[22,23] 12ns 10mb |L1.12| "
- "L1.24[24,25] 23ns 13mb |L1.24| "
- "**** 2 Output Files (parquet_file_id not yet assigned), 203mb total:"
- "L2 "
@ -2506,106 +2506,107 @@ async fn many_tiny_l1_files() {
- "Committing partition 1:"
- " Soft Deleting 200 files: L1.1, L1.2, L1.3, L1.4, L1.5, L1.6, L1.7, L1.8, L1.9, L1.10, L1.11, L1.12, L1.13, L1.14, L1.15, L1.16, L1.17, L1.18, L1.19, L1.20, L1.21, L1.22, L1.23, L1.24, L1.25, L1.26, L1.27, L1.28, L1.29, L1.30, L1.31, L1.32, L1.33, L1.34, L1.35, L1.36, L1.37, L1.38, L1.39, L1.40, L1.41, L1.42, L1.43, L1.44, L1.45, L1.46, L1.47, L1.48, L1.49, L1.50, L1.51, L1.52, L1.53, L1.54, L1.55, L1.56, L1.57, L1.58, L1.59, L1.60, L1.61, L1.62, L1.63, L1.64, L1.65, L1.66, L1.67, L1.68, L1.69, L1.70, L1.71, L1.72, L1.73, L1.74, L1.75, L1.76, L1.77, L1.78, L1.79, L1.80, L1.81, L1.82, L1.83, L1.84, L1.85, L1.86, L1.87, L1.88, L1.89, L1.90, L1.91, L1.92, L1.93, L1.94, L1.95, L1.96, L1.97, L1.98, L1.99, L1.100, L1.101, L1.102, L1.103, L1.104, L1.105, L1.106, L1.107, L1.108, L1.109, L1.110, L1.111, L1.112, L1.113, L1.114, L1.115, L1.116, L1.117, L1.118, L1.119, L1.120, L1.121, L1.122, L1.123, L1.124, L1.125, L1.126, L1.127, L1.128, L1.129, L1.130, L1.131, L1.132, L1.133, L1.134, L1.135, L1.136, L1.137, L1.138, L1.139, L1.140, L1.141, L1.142, L1.143, L1.144, L1.145, L1.146, L1.147, L1.148, L1.149, L1.150, L1.151, L1.152, L1.153, L1.154, L1.155, L1.156, L1.157, L1.158, L1.159, L1.160, L1.161, L1.162, L1.163, L1.164, L1.165, L1.166, L1.167, L1.168, L1.169, L1.170, L1.171, L1.172, L1.173, L1.174, L1.175, L1.176, L1.177, L1.178, L1.179, L1.180, L1.181, L1.182, L1.183, L1.184, L1.185, L1.186, L1.187, L1.188, L1.189, L1.190, L1.191, L1.192, L1.193, L1.194, L1.195, L1.196, L1.197, L1.198, L1.199, L1.200"
- " Creating 1 files"
- "**** Simulation run 1, type=compact(TotalSizeLessThanMaxCompactSize). 88 Input Files, 616kb total:"
- "L1, all files 7kb "
- "L1.288[574,575] 288ns |L1.288|"
- "L1.287[572,573] 287ns |L1.287|"
- "L1.286[570,571] 286ns |L1.286|"
- "L1.285[568,569] 285ns |L1.285|"
- "L1.284[566,567] 284ns |L1.284|"
- "L1.283[564,565] 283ns |L1.283|"
- "L1.282[562,563] 282ns |L1.282|"
- "L1.281[560,561] 281ns |L1.281|"
- "L1.280[558,559] 280ns |L1.280| "
- "L1.279[556,557] 279ns |L1.279| "
- "L1.278[554,555] 278ns |L1.278| "
- "L1.277[552,553] 277ns |L1.277| "
- "L1.276[550,551] 276ns |L1.276| "
- "L1.275[548,549] 275ns |L1.275| "
- "L1.274[546,547] 274ns |L1.274| "
- "L1.273[544,545] 273ns |L1.273| "
- "L1.272[542,543] 272ns |L1.272| "
- "L1.271[540,541] 271ns |L1.271| "
- "L1.270[538,539] 270ns |L1.270| "
- "L1.269[536,537] 269ns |L1.269| "
- "L1.268[534,535] 268ns |L1.268| "
- "L1.267[532,533] 267ns |L1.267| "
- "L1.266[530,531] 266ns |L1.266| "
- "L1.265[528,529] 265ns |L1.265| "
- "L1.264[526,527] 264ns |L1.264| "
- "L1.263[524,525] 263ns |L1.263| "
- "L1.262[522,523] 262ns |L1.262| "
- "L1.261[520,521] 261ns |L1.261| "
- "L1.260[518,519] 260ns |L1.260| "
- "L1.259[516,517] 259ns |L1.259| "
- "L1.258[514,515] 258ns |L1.258| "
- "L1.257[512,513] 257ns |L1.257| "
- "L1.256[510,511] 256ns |L1.256| "
- "L1.255[508,509] 255ns |L1.255| "
- "L1.254[506,507] 254ns |L1.254| "
- "L1.253[504,505] 253ns |L1.253| "
- "L1.252[502,503] 252ns |L1.252| "
- "L1.251[500,501] 251ns |L1.251| "
- "L1.250[498,499] 250ns |L1.250| "
- "L1.249[496,497] 249ns |L1.249| "
- "L1.248[494,495] 248ns |L1.248| "
- "L1.247[492,493] 247ns |L1.247| "
- "L1.246[490,491] 246ns |L1.246| "
- "L1.245[488,489] 245ns |L1.245| "
- "L1.244[486,487] 244ns |L1.244| "
- "L1.243[484,485] 243ns |L1.243| "
- "L1.242[482,483] 242ns |L1.242| "
- "L1.241[480,481] 241ns |L1.241| "
- "L1.240[478,479] 240ns |L1.240| "
- "L1.239[476,477] 239ns |L1.239| "
- "L1.238[474,475] 238ns |L1.238| "
- "L1.237[472,473] 237ns |L1.237| "
- "L1.236[470,471] 236ns |L1.236| "
- "L1.235[468,469] 235ns |L1.235| "
- "L1.234[466,467] 234ns |L1.234| "
- "L1.233[464,465] 233ns |L1.233| "
- "L1.232[462,463] 232ns |L1.232| "
- "L1.231[460,461] 231ns |L1.231| "
- "L1.230[458,459] 230ns |L1.230| "
- "L1.229[456,457] 229ns |L1.229| "
- "L1.228[454,455] 228ns |L1.228| "
- "L1.227[452,453] 227ns |L1.227| "
- "L1.226[450,451] 226ns |L1.226| "
- "L1.225[448,449] 225ns |L1.225| "
- "L1.224[446,447] 224ns |L1.224| "
- "L1.223[444,445] 223ns |L1.223| "
- "L1.222[442,443] 222ns |L1.222| "
- "L1.221[440,441] 221ns |L1.221| "
- "L1.220[438,439] 220ns |L1.220| "
- "L1.219[436,437] 219ns |L1.219| "
- "L1.218[434,435] 218ns |L1.218| "
- "L1.217[432,433] 217ns |L1.217| "
- "L1.216[430,431] 216ns |L1.216| "
- "L1.215[428,429] 215ns |L1.215| "
- "L1.214[426,427] 214ns |L1.214| "
- "L1.213[424,425] 213ns |L1.213| "
- "L1.212[422,423] 212ns |L1.212| "
- "L1.211[420,421] 211ns |L1.211| "
- "L1.210[418,419] 210ns |L1.210| "
- "L1.209[416,417] 209ns |L1.209| "
- "L1.208[414,415] 208ns |L1.208| "
- "L1.207[412,413] 207ns |L1.207| "
- "L1.206[410,411] 206ns |L1.206| "
- "L1.205[408,409] 205ns |L1.205| "
- "L1.204[406,407] 204ns |L1.204| "
- "L1.203[404,405] 203ns |L1.203| "
- "L1.202[402,403] 202ns |L1.202| "
- "L1.201[400,401] 201ns |L1.201| "
- "**** 1 Output Files (parquet_file_id not yet assigned), 616kb total:"
- "L2, all files 616kb "
- "L2.?[400,575] 288ns |------------------------------------------L2.?------------------------------------------|"
- "Committing partition 1:"
- " Soft Deleting 88 files: L1.201, L1.202, L1.203, L1.204, L1.205, L1.206, L1.207, L1.208, L1.209, L1.210, L1.211, L1.212, L1.213, L1.214, L1.215, L1.216, L1.217, L1.218, L1.219, L1.220, L1.221, L1.222, L1.223, L1.224, L1.225, L1.226, L1.227, L1.228, L1.229, L1.230, L1.231, L1.232, L1.233, L1.234, L1.235, L1.236, L1.237, L1.238, L1.239, L1.240, L1.241, L1.242, L1.243, L1.244, L1.245, L1.246, L1.247, L1.248, L1.249, L1.250, L1.251, L1.252, L1.253, L1.254, L1.255, L1.256, L1.257, L1.258, L1.259, L1.260, L1.261, L1.262, L1.263, L1.264, L1.265, L1.266, L1.267, L1.268, L1.269, L1.270, L1.271, L1.272, L1.273, L1.274, L1.275, L1.276, L1.277, L1.278, L1.279, L1.280, L1.281, L1.282, L1.283, L1.284, L1.285, L1.286, L1.287, L1.288"
- " Creating 1 files"
- "**** Final Output Files (2mb written)"
- "**** Simulation run 1, type=compact(TotalSizeLessThanMaxCompactSize). 89 Input Files, 2mb total:"
- "L1 "
- "L1.288[574,575] 288ns 7kb |L1.288|"
- "L1.287[572,573] 287ns 7kb |L1.287|"
- "L1.286[570,571] 286ns 7kb |L1.286|"
- "L1.285[568,569] 285ns 7kb |L1.285|"
- "L1.284[566,567] 284ns 7kb |L1.284|"
- "L1.283[564,565] 283ns 7kb |L1.283|"
- "L1.282[562,563] 282ns 7kb |L1.282|"
- "L1.281[560,561] 281ns 7kb |L1.281|"
- "L1.280[558,559] 280ns 7kb |L1.280|"
- "L1.279[556,557] 279ns 7kb |L1.279|"
- "L1.278[554,555] 278ns 7kb |L1.278|"
- "L1.277[552,553] 277ns 7kb |L1.277|"
- "L1.276[550,551] 276ns 7kb |L1.276|"
- "L1.275[548,549] 275ns 7kb |L1.275|"
- "L1.274[546,547] 274ns 7kb |L1.274|"
- "L1.273[544,545] 273ns 7kb |L1.273|"
- "L1.272[542,543] 272ns 7kb |L1.272|"
- "L1.271[540,541] 271ns 7kb |L1.271|"
- "L1.270[538,539] 270ns 7kb |L1.270|"
- "L1.269[536,537] 269ns 7kb |L1.269|"
- "L1.268[534,535] 268ns 7kb |L1.268|"
- "L1.267[532,533] 267ns 7kb |L1.267|"
- "L1.266[530,531] 266ns 7kb |L1.266|"
- "L1.265[528,529] 265ns 7kb |L1.265|"
- "L1.264[526,527] 264ns 7kb |L1.264|"
- "L1.263[524,525] 263ns 7kb |L1.263|"
- "L1.262[522,523] 262ns 7kb |L1.262| "
- "L1.261[520,521] 261ns 7kb |L1.261| "
- "L1.260[518,519] 260ns 7kb |L1.260| "
- "L1.259[516,517] 259ns 7kb |L1.259| "
- "L1.258[514,515] 258ns 7kb |L1.258| "
- "L1.257[512,513] 257ns 7kb |L1.257| "
- "L1.256[510,511] 256ns 7kb |L1.256| "
- "L1.255[508,509] 255ns 7kb |L1.255| "
- "L1.254[506,507] 254ns 7kb |L1.254| "
- "L1.253[504,505] 253ns 7kb |L1.253| "
- "L1.252[502,503] 252ns 7kb |L1.252| "
- "L1.251[500,501] 251ns 7kb |L1.251| "
- "L1.250[498,499] 250ns 7kb |L1.250| "
- "L1.249[496,497] 249ns 7kb |L1.249| "
- "L1.248[494,495] 248ns 7kb |L1.248| "
- "L1.247[492,493] 247ns 7kb |L1.247| "
- "L1.246[490,491] 246ns 7kb |L1.246| "
- "L1.245[488,489] 245ns 7kb |L1.245| "
- "L1.244[486,487] 244ns 7kb |L1.244| "
- "L1.243[484,485] 243ns 7kb |L1.243| "
- "L1.242[482,483] 242ns 7kb |L1.242| "
- "L1.241[480,481] 241ns 7kb |L1.241| "
- "L1.240[478,479] 240ns 7kb |L1.240| "
- "L1.239[476,477] 239ns 7kb |L1.239| "
- "L1.238[474,475] 238ns 7kb |L1.238| "
- "L1.237[472,473] 237ns 7kb |L1.237| "
- "L1.236[470,471] 236ns 7kb |L1.236| "
- "L1.235[468,469] 235ns 7kb |L1.235| "
- "L1.234[466,467] 234ns 7kb |L1.234| "
- "L1.233[464,465] 233ns 7kb |L1.233| "
- "L1.232[462,463] 232ns 7kb |L1.232| "
- "L1.231[460,461] 231ns 7kb |L1.231| "
- "L1.230[458,459] 230ns 7kb |L1.230| "
- "L1.229[456,457] 229ns 7kb |L1.229| "
- "L1.228[454,455] 228ns 7kb |L1.228| "
- "L1.227[452,453] 227ns 7kb |L1.227| "
- "L1.226[450,451] 226ns 7kb |L1.226| "
- "L1.225[448,449] 225ns 7kb |L1.225| "
- "L1.224[446,447] 224ns 7kb |L1.224| "
- "L1.223[444,445] 223ns 7kb |L1.223| "
- "L1.222[442,443] 222ns 7kb |L1.222| "
- "L1.221[440,441] 221ns 7kb |L1.221| "
- "L1.220[438,439] 220ns 7kb |L1.220| "
- "L1.219[436,437] 219ns 7kb |L1.219| "
- "L1.218[434,435] 218ns 7kb |L1.218| "
- "L1.217[432,433] 217ns 7kb |L1.217| "
- "L1.216[430,431] 216ns 7kb |L1.216| "
- "L1.215[428,429] 215ns 7kb |L1.215| "
- "L1.214[426,427] 214ns 7kb |L1.214| "
- "L1.213[424,425] 213ns 7kb |L1.213| "
- "L1.212[422,423] 212ns 7kb |L1.212| "
- "L1.211[420,421] 211ns 7kb |L1.211| "
- "L1.210[418,419] 210ns 7kb |L1.210| "
- "L1.209[416,417] 209ns 7kb |L1.209| "
- "L1.208[414,415] 208ns 7kb |L1.208| "
- "L1.207[412,413] 207ns 7kb |L1.207| "
- "L1.206[410,411] 206ns 7kb |L1.206| "
- "L1.205[408,409] 205ns 7kb |L1.205| "
- "L1.204[406,407] 204ns 7kb |L1.204| "
- "L1.203[404,405] 203ns 7kb |L1.203| "
- "L1.202[402,403] 202ns 7kb |L1.202| "
- "L1.201[400,401] 201ns 7kb |L1.201| "
- "L2 "
- "L2.289[0,399] 200ns 1mb |---------------------------L2.289---------------------------| "
- "L2.290[400,575] 288ns 616kb |---------L2.290----------| "
- "**** 1 Output Files (parquet_file_id not yet assigned), 2mb total:"
- "L2, all files 2mb "
- "L2.?[0,575] 288ns |------------------------------------------L2.?------------------------------------------|"
- "Committing partition 1:"
- " Soft Deleting 89 files: L1.201, L1.202, L1.203, L1.204, L1.205, L1.206, L1.207, L1.208, L1.209, L1.210, L1.211, L1.212, L1.213, L1.214, L1.215, L1.216, L1.217, L1.218, L1.219, L1.220, L1.221, L1.222, L1.223, L1.224, L1.225, L1.226, L1.227, L1.228, L1.229, L1.230, L1.231, L1.232, L1.233, L1.234, L1.235, L1.236, L1.237, L1.238, L1.239, L1.240, L1.241, L1.242, L1.243, L1.244, L1.245, L1.246, L1.247, L1.248, L1.249, L1.250, L1.251, L1.252, L1.253, L1.254, L1.255, L1.256, L1.257, L1.258, L1.259, L1.260, L1.261, L1.262, L1.263, L1.264, L1.265, L1.266, L1.267, L1.268, L1.269, L1.270, L1.271, L1.272, L1.273, L1.274, L1.275, L1.276, L1.277, L1.278, L1.279, L1.280, L1.281, L1.282, L1.283, L1.284, L1.285, L1.286, L1.287, L1.288, L2.289"
- " Creating 1 files"
- "**** Final Output Files (3mb written)"
- "L2, all files 2mb "
- "L2.290[0,575] 288ns |-----------------------------------------L2.290-----------------------------------------|"
"###
);
}
@ -3080,26 +3081,26 @@ async fn many_l0_and_overlapped_l1_files() {
- "L0 "
- "L0.211[0,190] 210ns 1mb |--------------------------------------L0.211---------------------------------------| "
- "L1 "
- "L1.191[0,9] 1ns 1mb |L1.191| "
- "L1.210[190,199] 20ns 1mb |L1.210|"
- "L1.192[10,19] 2ns 1mb |L1.192| "
- "L1.193[20,29] 3ns 1mb |L1.193| "
- "L1.194[30,39] 4ns 1mb |L1.194| "
- "L1.195[40,49] 5ns 1mb |L1.195| "
- "L1.196[50,59] 6ns 1mb |L1.196| "
- "L1.197[60,69] 7ns 1mb |L1.197| "
- "L1.198[70,79] 8ns 1mb |L1.198| "
- "L1.199[80,89] 9ns 1mb |L1.199| "
- "L1.200[90,99] 10ns 1mb |L1.200| "
- "L1.201[100,109] 11ns 1mb |L1.201| "
- "L1.202[110,119] 12ns 1mb |L1.202| "
- "L1.203[120,129] 13ns 1mb |L1.203| "
- "L1.204[130,139] 14ns 1mb |L1.204| "
- "L1.205[140,149] 15ns 1mb |L1.205| "
- "L1.206[150,159] 16ns 1mb |L1.206| "
- "L1.207[160,169] 17ns 1mb |L1.207| "
- "L1.208[170,179] 18ns 1mb |L1.208| "
- "L1.209[180,189] 19ns 1mb |L1.209| "
- "L1.208[170,179] 18ns 1mb |L1.208| "
- "L1.207[160,169] 17ns 1mb |L1.207| "
- "L1.206[150,159] 16ns 1mb |L1.206| "
- "L1.205[140,149] 15ns 1mb |L1.205| "
- "L1.204[130,139] 14ns 1mb |L1.204| "
- "L1.203[120,129] 13ns 1mb |L1.203| "
- "L1.202[110,119] 12ns 1mb |L1.202| "
- "L1.201[100,109] 11ns 1mb |L1.201| "
- "L1.200[90,99] 10ns 1mb |L1.200| "
- "L1.199[80,89] 9ns 1mb |L1.199| "
- "L1.198[70,79] 8ns 1mb |L1.198| "
- "L1.197[60,69] 7ns 1mb |L1.197| "
- "L1.196[50,59] 6ns 1mb |L1.196| "
- "L1.195[40,49] 5ns 1mb |L1.195| "
- "L1.194[30,39] 4ns 1mb |L1.194| "
- "L1.193[20,29] 3ns 1mb |L1.193| "
- "L1.192[10,19] 2ns 1mb |L1.192| "
- "L1.191[0,9] 1ns 1mb |L1.191| "
- "**** 2 Output Files (parquet_file_id not yet assigned), 21mb total:"
- "L1 "
- "L1.?[0,159] 210ns 17mb |--------------------------------L1.?---------------------------------| "
@ -3402,212 +3403,204 @@ async fn not_many_l0_and_overlapped_l1_files() {
- "L1.208[2600,2609] 18ns 1mb |L1.208| "
- "L1.209[2800,2809] 19ns 1mb |L1.209|"
- "L1.210[3000,3009] 20ns 1mb |L1.210|"
- "**** Simulation run 0, type=compact(TotalSizeLessThanMaxCompactSize). 195 Input Files, 6mb total:"
- "**** Simulation run 0, type=split(CompactAndSplitOutput(TotalSizeLessThanMaxCompactSize))(split_times=[2407]). 210 Input Files, 21mb total:"
- "L0 "
- "L0.190[189,190] 210ns 7kb |L0.190|"
- "L0.189[188,189] 209ns 7kb |L0.189|"
- "L0.188[187,188] 208ns 7kb |L0.188|"
- "L0.187[186,187] 207ns 7kb |L0.187|"
- "L0.186[185,186] 206ns 7kb |L0.186|"
- "L0.185[184,185] 205ns 7kb |L0.185|"
- "L0.184[183,184] 204ns 7kb |L0.184|"
- "L0.183[182,183] 203ns 7kb |L0.183|"
- "L0.182[181,182] 202ns 7kb |L0.182|"
- "L0.181[180,181] 201ns 7kb |L0.181|"
- "L0.180[179,180] 200ns 7kb |L0.180|"
- "L0.179[178,179] 199ns 7kb |L0.179|"
- "L0.178[177,178] 198ns 7kb |L0.178|"
- "L0.177[176,177] 197ns 7kb |L0.177|"
- "L0.176[175,176] 196ns 7kb |L0.176|"
- "L0.175[174,175] 195ns 7kb |L0.175|"
- "L0.174[173,174] 194ns 7kb |L0.174| "
- "L0.173[172,173] 193ns 7kb |L0.173| "
- "L0.172[171,172] 192ns 7kb |L0.172| "
- "L0.171[170,171] 191ns 7kb |L0.171| "
- "L0.170[169,170] 190ns 7kb |L0.170| "
- "L0.169[168,169] 189ns 7kb |L0.169| "
- "L0.168[167,168] 188ns 7kb |L0.168| "
- "L0.167[166,167] 187ns 7kb |L0.167| "
- "L0.166[165,166] 186ns 7kb |L0.166| "
- "L0.165[164,165] 185ns 7kb |L0.165| "
- "L0.164[163,164] 184ns 7kb |L0.164| "
- "L0.163[162,163] 183ns 7kb |L0.163| "
- "L0.162[161,162] 182ns 7kb |L0.162| "
- "L0.161[160,161] 181ns 7kb |L0.161| "
- "L0.160[159,160] 180ns 7kb |L0.160| "
- "L0.159[158,159] 179ns 7kb |L0.159| "
- "L0.158[157,158] 178ns 7kb |L0.158| "
- "L0.157[156,157] 177ns 7kb |L0.157| "
- "L0.156[155,156] 176ns 7kb |L0.156| "
- "L0.155[154,155] 175ns 7kb |L0.155| "
- "L0.154[153,154] 174ns 7kb |L0.154| "
- "L0.153[152,153] 173ns 7kb |L0.153| "
- "L0.152[151,152] 172ns 7kb |L0.152| "
- "L0.151[150,151] 171ns 7kb |L0.151| "
- "L0.150[149,150] 170ns 7kb |L0.150| "
- "L0.149[148,149] 169ns 7kb |L0.149| "
- "L0.148[147,148] 168ns 7kb |L0.148| "
- "L0.147[146,147] 167ns 7kb |L0.147| "
- "L0.146[145,146] 166ns 7kb |L0.146| "
- "L0.145[144,145] 165ns 7kb |L0.145| "
- "L0.144[143,144] 164ns 7kb |L0.144| "
- "L0.143[142,143] 163ns 7kb |L0.143| "
- "L0.142[141,142] 162ns 7kb |L0.142| "
- "L0.141[140,141] 161ns 7kb |L0.141| "
- "L0.140[139,140] 160ns 7kb |L0.140| "
- "L0.139[138,139] 159ns 7kb |L0.139| "
- "L0.138[137,138] 158ns 7kb |L0.138| "
- "L0.137[136,137] 157ns 7kb |L0.137| "
- "L0.136[135,136] 156ns 7kb |L0.136| "
- "L0.135[134,135] 155ns 7kb |L0.135| "
- "L0.134[133,134] 154ns 7kb |L0.134| "
- "L0.133[132,133] 153ns 7kb |L0.133| "
- "L0.132[131,132] 152ns 7kb |L0.132| "
- "L0.131[130,131] 151ns 7kb |L0.131| "
- "L0.130[129,130] 150ns 7kb |L0.130| "
- "L0.129[128,129] 149ns 7kb |L0.129| "
- "L0.128[127,128] 148ns 7kb |L0.128| "
- "L0.127[126,127] 147ns 7kb |L0.127| "
- "L0.126[125,126] 146ns 7kb |L0.126| "
- "L0.125[124,125] 145ns 7kb |L0.125| "
- "L0.124[123,124] 144ns 7kb |L0.124| "
- "L0.123[122,123] 143ns 7kb |L0.123| "
- "L0.122[121,122] 142ns 7kb |L0.122| "
- "L0.121[120,121] 141ns 7kb |L0.121| "
- "L0.120[119,120] 140ns 7kb |L0.120| "
- "L0.119[118,119] 139ns 7kb |L0.119| "
- "L0.118[117,118] 138ns 7kb |L0.118| "
- "L0.117[116,117] 137ns 7kb |L0.117| "
- "L0.116[115,116] 136ns 7kb |L0.116| "
- "L0.115[114,115] 135ns 7kb |L0.115| "
- "L0.114[113,114] 134ns 7kb |L0.114| "
- "L0.113[112,113] 133ns 7kb |L0.113| "
- "L0.112[111,112] 132ns 7kb |L0.112| "
- "L0.111[110,111] 131ns 7kb |L0.111| "
- "L0.110[109,110] 130ns 7kb |L0.110| "
- "L0.109[108,109] 129ns 7kb |L0.109| "
- "L0.108[107,108] 128ns 7kb |L0.108| "
- "L0.107[106,107] 127ns 7kb |L0.107| "
- "L0.106[105,106] 126ns 7kb |L0.106| "
- "L0.105[104,105] 125ns 7kb |L0.105| "
- "L0.104[103,104] 124ns 7kb |L0.104| "
- "L0.103[102,103] 123ns 7kb |L0.103| "
- "L0.102[101,102] 122ns 7kb |L0.102| "
- "L0.101[100,101] 121ns 7kb |L0.101| "
- "L0.100[99,100] 120ns 7kb |L0.100| "
- "L0.99[98,99] 119ns 7kb |L0.99| "
- "L0.98[97,98] 118ns 7kb |L0.98| "
- "L0.97[96,97] 117ns 7kb |L0.97| "
- "L0.96[95,96] 116ns 7kb |L0.96| "
- "L0.95[94,95] 115ns 7kb |L0.95| "
- "L0.94[93,94] 114ns 7kb |L0.94| "
- "L0.93[92,93] 113ns 7kb |L0.93| "
- "L0.92[91,92] 112ns 7kb |L0.92| "
- "L0.91[90,91] 111ns 7kb |L0.91| "
- "L0.90[89,90] 110ns 7kb |L0.90| "
- "L0.89[88,89] 109ns 7kb |L0.89| "
- "L0.88[87,88] 108ns 7kb |L0.88| "
- "L0.87[86,87] 107ns 7kb |L0.87| "
- "L0.86[85,86] 106ns 7kb |L0.86| "
- "L0.85[84,85] 105ns 7kb |L0.85| "
- "L0.84[83,84] 104ns 7kb |L0.84| "
- "L0.83[82,83] 103ns 7kb |L0.83| "
- "L0.82[81,82] 102ns 7kb |L0.82| "
- "L0.81[80,81] 101ns 7kb |L0.81| "
- "L0.80[79,80] 100ns 7kb |L0.80| "
- "L0.79[78,79] 99ns 7kb |L0.79| "
- "L0.78[77,78] 98ns 7kb |L0.78| "
- "L0.77[76,77] 97ns 7kb |L0.77| "
- "L0.76[75,76] 96ns 7kb |L0.76| "
- "L0.75[74,75] 95ns 7kb |L0.75| "
- "L0.74[73,74] 94ns 7kb |L0.74| "
- "L0.73[72,73] 93ns 7kb |L0.73| "
- "L0.72[71,72] 92ns 7kb |L0.72| "
- "L0.71[70,71] 91ns 7kb |L0.71| "
- "L0.70[69,70] 90ns 7kb |L0.70| "
- "L0.69[68,69] 89ns 7kb |L0.69| "
- "L0.68[67,68] 88ns 7kb |L0.68| "
- "L0.67[66,67] 87ns 7kb |L0.67| "
- "L0.66[65,66] 86ns 7kb |L0.66| "
- "L0.65[64,65] 85ns 7kb |L0.65| "
- "L0.64[63,64] 84ns 7kb |L0.64| "
- "L0.63[62,63] 83ns 7kb |L0.63| "
- "L0.62[61,62] 82ns 7kb |L0.62| "
- "L0.61[60,61] 81ns 7kb |L0.61| "
- "L0.60[59,60] 80ns 7kb |L0.60| "
- "L0.59[58,59] 79ns 7kb |L0.59| "
- "L0.58[57,58] 78ns 7kb |L0.58| "
- "L0.57[56,57] 77ns 7kb |L0.57| "
- "L0.56[55,56] 76ns 7kb |L0.56| "
- "L0.55[54,55] 75ns 7kb |L0.55| "
- "L0.54[53,54] 74ns 7kb |L0.54| "
- "L0.53[52,53] 73ns 7kb |L0.53| "
- "L0.52[51,52] 72ns 7kb |L0.52| "
- "L0.51[50,51] 71ns 7kb |L0.51| "
- "L0.50[49,50] 70ns 7kb |L0.50| "
- "L0.49[48,49] 69ns 7kb |L0.49| "
- "L0.48[47,48] 68ns 7kb |L0.48| "
- "L0.47[46,47] 67ns 7kb |L0.47| "
- "L0.46[45,46] 66ns 7kb |L0.46| "
- "L0.45[44,45] 65ns 7kb |L0.45| "
- "L0.44[43,44] 64ns 7kb |L0.44| "
- "L0.43[42,43] 63ns 7kb |L0.43| "
- "L0.42[41,42] 62ns 7kb |L0.42| "
- "L0.41[40,41] 61ns 7kb |L0.41| "
- "L0.40[39,40] 60ns 7kb |L0.40| "
- "L0.39[38,39] 59ns 7kb |L0.39| "
- "L0.38[37,38] 58ns 7kb |L0.38| "
- "L0.37[36,37] 57ns 7kb |L0.37| "
- "L0.36[35,36] 56ns 7kb |L0.36| "
- "L0.35[34,35] 55ns 7kb |L0.35| "
- "L0.34[33,34] 54ns 7kb |L0.34| "
- "L0.33[32,33] 53ns 7kb |L0.33| "
- "L0.32[31,32] 52ns 7kb |L0.32| "
- "L0.31[30,31] 51ns 7kb |L0.31| "
- "L0.30[29,30] 50ns 7kb |L0.30| "
- "L0.29[28,29] 49ns 7kb |L0.29| "
- "L0.28[27,28] 48ns 7kb |L0.28| "
- "L0.27[26,27] 47ns 7kb |L0.27| "
- "L0.26[25,26] 46ns 7kb |L0.26| "
- "L0.25[24,25] 45ns 7kb |L0.25| "
- "L0.24[23,24] 44ns 7kb |L0.24| "
- "L0.23[22,23] 43ns 7kb |L0.23| "
- "L0.22[21,22] 42ns 7kb |L0.22| "
- "L0.21[20,21] 41ns 7kb |L0.21| "
- "L0.20[19,20] 40ns 7kb |L0.20| "
- "L0.19[18,19] 39ns 7kb |L0.19| "
- "L0.18[17,18] 38ns 7kb |L0.18| "
- "L0.17[16,17] 37ns 7kb |L0.17| "
- "L0.16[15,16] 36ns 7kb |L0.16| "
- "L0.15[14,15] 35ns 7kb |L0.15| "
- "L0.14[13,14] 34ns 7kb |L0.14| "
- "L0.13[12,13] 33ns 7kb |L0.13| "
- "L0.12[11,12] 32ns 7kb |L0.12| "
- "L0.11[10,11] 31ns 7kb |L0.11| "
- "L0.10[9,10] 30ns 7kb |L0.10| "
- "L0.9[8,9] 29ns 7kb |L0.9| "
- "L0.8[7,8] 28ns 7kb |L0.8| "
- "L0.7[6,7] 27ns 7kb |L0.7| "
- "L0.6[5,6] 26ns 7kb |L0.6| "
- "L0.5[4,5] 25ns 7kb |L0.5| "
- "L0.4[3,4] 24ns 7kb |L0.4| "
- "L0.190[189,190] 210ns 7kb |L0.190| "
- "L0.189[188,189] 209ns 7kb |L0.189| "
- "L0.188[187,188] 208ns 7kb |L0.188| "
- "L0.187[186,187] 207ns 7kb |L0.187| "
- "L0.186[185,186] 206ns 7kb |L0.186| "
- "L0.185[184,185] 205ns 7kb |L0.185| "
- "L0.184[183,184] 204ns 7kb |L0.184| "
- "L0.183[182,183] 203ns 7kb |L0.183| "
- "L0.182[181,182] 202ns 7kb |L0.182| "
- "L0.181[180,181] 201ns 7kb |L0.181| "
- "L0.180[179,180] 200ns 7kb |L0.180| "
- "L0.179[178,179] 199ns 7kb |L0.179| "
- "L0.178[177,178] 198ns 7kb |L0.178| "
- "L0.177[176,177] 197ns 7kb |L0.177| "
- "L0.176[175,176] 196ns 7kb |L0.176| "
- "L0.175[174,175] 195ns 7kb |L0.175| "
- "L0.174[173,174] 194ns 7kb |L0.174| "
- "L0.173[172,173] 193ns 7kb |L0.173| "
- "L0.172[171,172] 192ns 7kb |L0.172| "
- "L0.171[170,171] 191ns 7kb |L0.171| "
- "L0.170[169,170] 190ns 7kb |L0.170| "
- "L0.169[168,169] 189ns 7kb |L0.169| "
- "L0.168[167,168] 188ns 7kb |L0.168| "
- "L0.167[166,167] 187ns 7kb |L0.167| "
- "L0.166[165,166] 186ns 7kb |L0.166| "
- "L0.165[164,165] 185ns 7kb |L0.165| "
- "L0.164[163,164] 184ns 7kb |L0.164| "
- "L0.163[162,163] 183ns 7kb |L0.163| "
- "L0.162[161,162] 182ns 7kb |L0.162| "
- "L0.161[160,161] 181ns 7kb |L0.161| "
- "L0.160[159,160] 180ns 7kb |L0.160| "
- "L0.159[158,159] 179ns 7kb |L0.159| "
- "L0.158[157,158] 178ns 7kb |L0.158| "
- "L0.157[156,157] 177ns 7kb |L0.157| "
- "L0.156[155,156] 176ns 7kb |L0.156| "
- "L0.155[154,155] 175ns 7kb |L0.155| "
- "L0.154[153,154] 174ns 7kb |L0.154| "
- "L0.153[152,153] 173ns 7kb |L0.153| "
- "L0.152[151,152] 172ns 7kb |L0.152| "
- "L0.151[150,151] 171ns 7kb |L0.151| "
- "L0.150[149,150] 170ns 7kb |L0.150| "
- "L0.149[148,149] 169ns 7kb |L0.149| "
- "L0.148[147,148] 168ns 7kb |L0.148| "
- "L0.147[146,147] 167ns 7kb |L0.147| "
- "L0.146[145,146] 166ns 7kb |L0.146| "
- "L0.145[144,145] 165ns 7kb |L0.145| "
- "L0.144[143,144] 164ns 7kb |L0.144| "
- "L0.143[142,143] 163ns 7kb |L0.143| "
- "L0.142[141,142] 162ns 7kb |L0.142| "
- "L0.141[140,141] 161ns 7kb |L0.141| "
- "L0.140[139,140] 160ns 7kb |L0.140| "
- "L0.139[138,139] 159ns 7kb |L0.139| "
- "L0.138[137,138] 158ns 7kb |L0.138| "
- "L0.137[136,137] 157ns 7kb |L0.137| "
- "L0.136[135,136] 156ns 7kb |L0.136| "
- "L0.135[134,135] 155ns 7kb |L0.135| "
- "L0.134[133,134] 154ns 7kb |L0.134| "
- "L0.133[132,133] 153ns 7kb |L0.133| "
- "L0.132[131,132] 152ns 7kb |L0.132| "
- "L0.131[130,131] 151ns 7kb |L0.131| "
- "L0.130[129,130] 150ns 7kb |L0.130| "
- "L0.129[128,129] 149ns 7kb |L0.129| "
- "L0.128[127,128] 148ns 7kb |L0.128| "
- "L0.127[126,127] 147ns 7kb |L0.127| "
- "L0.126[125,126] 146ns 7kb |L0.126| "
- "L0.125[124,125] 145ns 7kb |L0.125| "
- "L0.124[123,124] 144ns 7kb |L0.124| "
- "L0.123[122,123] 143ns 7kb |L0.123| "
- "L0.122[121,122] 142ns 7kb |L0.122| "
- "L0.121[120,121] 141ns 7kb |L0.121| "
- "L0.120[119,120] 140ns 7kb |L0.120| "
- "L0.119[118,119] 139ns 7kb |L0.119| "
- "L0.118[117,118] 138ns 7kb |L0.118| "
- "L0.117[116,117] 137ns 7kb |L0.117| "
- "L0.116[115,116] 136ns 7kb |L0.116| "
- "L0.115[114,115] 135ns 7kb |L0.115| "
- "L0.114[113,114] 134ns 7kb |L0.114| "
- "L0.113[112,113] 133ns 7kb |L0.113| "
- "L0.112[111,112] 132ns 7kb |L0.112| "
- "L0.111[110,111] 131ns 7kb |L0.111| "
- "L0.110[109,110] 130ns 7kb |L0.110| "
- "L0.109[108,109] 129ns 7kb |L0.109| "
- "L0.108[107,108] 128ns 7kb |L0.108| "
- "L0.107[106,107] 127ns 7kb |L0.107| "
- "L0.106[105,106] 126ns 7kb |L0.106| "
- "L0.105[104,105] 125ns 7kb |L0.105| "
- "L0.104[103,104] 124ns 7kb |L0.104| "
- "L0.103[102,103] 123ns 7kb |L0.103| "
- "L0.102[101,102] 122ns 7kb |L0.102| "
- "L0.101[100,101] 121ns 7kb |L0.101| "
- "L0.100[99,100] 120ns 7kb |L0.100| "
- "L0.99[98,99] 119ns 7kb |L0.99| "
- "L0.98[97,98] 118ns 7kb |L0.98| "
- "L0.97[96,97] 117ns 7kb |L0.97| "
- "L0.96[95,96] 116ns 7kb |L0.96| "
- "L0.95[94,95] 115ns 7kb |L0.95| "
- "L0.94[93,94] 114ns 7kb |L0.94| "
- "L0.93[92,93] 113ns 7kb |L0.93| "
- "L0.92[91,92] 112ns 7kb |L0.92| "
- "L0.91[90,91] 111ns 7kb |L0.91| "
- "L0.90[89,90] 110ns 7kb |L0.90| "
- "L0.89[88,89] 109ns 7kb |L0.89| "
- "L0.88[87,88] 108ns 7kb |L0.88| "
- "L0.87[86,87] 107ns 7kb |L0.87| "
- "L0.86[85,86] 106ns 7kb |L0.86| "
- "L0.85[84,85] 105ns 7kb |L0.85| "
- "L0.84[83,84] 104ns 7kb |L0.84| "
- "L0.83[82,83] 103ns 7kb |L0.83| "
- "L0.82[81,82] 102ns 7kb |L0.82| "
- "L0.81[80,81] 101ns 7kb |L0.81| "
- "L0.80[79,80] 100ns 7kb |L0.80| "
- "L0.79[78,79] 99ns 7kb |L0.79| "
- "L0.78[77,78] 98ns 7kb |L0.78| "
- "L0.77[76,77] 97ns 7kb |L0.77| "
- "L0.76[75,76] 96ns 7kb |L0.76| "
- "L0.75[74,75] 95ns 7kb |L0.75| "
- "L0.74[73,74] 94ns 7kb |L0.74| "
- "L0.73[72,73] 93ns 7kb |L0.73| "
- "L0.72[71,72] 92ns 7kb |L0.72| "
- "L0.71[70,71] 91ns 7kb |L0.71| "
- "L0.70[69,70] 90ns 7kb |L0.70| "
- "L0.69[68,69] 89ns 7kb |L0.69| "
- "L0.68[67,68] 88ns 7kb |L0.68| "
- "L0.67[66,67] 87ns 7kb |L0.67| "
- "L0.66[65,66] 86ns 7kb |L0.66| "
- "L0.65[64,65] 85ns 7kb |L0.65| "
- "L0.64[63,64] 84ns 7kb |L0.64| "
- "L0.63[62,63] 83ns 7kb |L0.63| "
- "L0.62[61,62] 82ns 7kb |L0.62| "
- "L0.61[60,61] 81ns 7kb |L0.61| "
- "L0.60[59,60] 80ns 7kb |L0.60| "
- "L0.59[58,59] 79ns 7kb |L0.59| "
- "L0.58[57,58] 78ns 7kb |L0.58| "
- "L0.57[56,57] 77ns 7kb |L0.57| "
- "L0.56[55,56] 76ns 7kb |L0.56| "
- "L0.55[54,55] 75ns 7kb |L0.55| "
- "L0.54[53,54] 74ns 7kb |L0.54| "
- "L0.53[52,53] 73ns 7kb |L0.53| "
- "L0.52[51,52] 72ns 7kb |L0.52| "
- "L0.51[50,51] 71ns 7kb |L0.51| "
- "L0.50[49,50] 70ns 7kb |L0.50| "
- "L0.49[48,49] 69ns 7kb |L0.49| "
- "L0.48[47,48] 68ns 7kb |L0.48| "
- "L0.47[46,47] 67ns 7kb |L0.47| "
- "L0.46[45,46] 66ns 7kb |L0.46| "
- "L0.45[44,45] 65ns 7kb |L0.45| "
- "L0.44[43,44] 64ns 7kb |L0.44| "
- "L0.43[42,43] 63ns 7kb |L0.43| "
- "L0.42[41,42] 62ns 7kb |L0.42| "
- "L0.41[40,41] 61ns 7kb |L0.41| "
- "L0.40[39,40] 60ns 7kb |L0.40| "
- "L0.39[38,39] 59ns 7kb |L0.39| "
- "L0.38[37,38] 58ns 7kb |L0.38| "
- "L0.37[36,37] 57ns 7kb |L0.37| "
- "L0.36[35,36] 56ns 7kb |L0.36| "
- "L0.35[34,35] 55ns 7kb |L0.35| "
- "L0.34[33,34] 54ns 7kb |L0.34| "
- "L0.33[32,33] 53ns 7kb |L0.33| "
- "L0.32[31,32] 52ns 7kb |L0.32| "
- "L0.31[30,31] 51ns 7kb |L0.31| "
- "L0.30[29,30] 50ns 7kb |L0.30| "
- "L0.29[28,29] 49ns 7kb |L0.29| "
- "L0.28[27,28] 48ns 7kb |L0.28| "
- "L0.27[26,27] 47ns 7kb |L0.27| "
- "L0.26[25,26] 46ns 7kb |L0.26| "
- "L0.25[24,25] 45ns 7kb |L0.25| "
- "L0.24[23,24] 44ns 7kb |L0.24| "
- "L0.23[22,23] 43ns 7kb |L0.23| "
- "L0.22[21,22] 42ns 7kb |L0.22| "
- "L0.21[20,21] 41ns 7kb |L0.21| "
- "L0.20[19,20] 40ns 7kb |L0.20| "
- "L0.19[18,19] 39ns 7kb |L0.19| "
- "L0.18[17,18] 38ns 7kb |L0.18| "
- "L0.17[16,17] 37ns 7kb |L0.17| "
- "L0.16[15,16] 36ns 7kb |L0.16| "
- "L0.15[14,15] 35ns 7kb |L0.15| "
- "L0.14[13,14] 34ns 7kb |L0.14| "
- "L0.13[12,13] 33ns 7kb |L0.13| "
- "L0.12[11,12] 32ns 7kb |L0.12| "
- "L0.11[10,11] 31ns 7kb |L0.11| "
- "L0.10[9,10] 30ns 7kb |L0.10| "
- "L0.9[8,9] 29ns 7kb |L0.9| "
- "L0.8[7,8] 28ns 7kb |L0.8| "
- "L0.7[6,7] 27ns 7kb |L0.7| "
- "L0.6[5,6] 26ns 7kb |L0.6| "
- "L0.5[4,5] 25ns 7kb |L0.5| "
- "L0.4[3,4] 24ns 7kb |L0.4| "
- "L0.3[2,3] 23ns 7kb |L0.3| "
- "L0.2[1,2] 22ns 7kb |L0.2| "
- "L0.1[0,1] 21ns 7kb |L0.1| "
- "L1 "
- "L1.195[40,49] 5ns 1mb |L1.195| "
- "L1.194[30,39] 4ns 1mb |L1.194| "
- "L1.193[20,29] 3ns 1mb |L1.193| "
- "L1.192[10,19] 2ns 1mb |L1.192| "
- "L1.191[0,9] 1ns 1mb |L1.191| "
- "L1.195[40,49] 5ns 1mb |L1.195| "
- "L1.192[10,19] 2ns 1mb |L1.192| "
- "L1.193[20,29] 3ns 1mb |L1.193| "
- "L1.194[30,39] 4ns 1mb |L1.194| "
- "**** 1 Output Files (parquet_file_id not yet assigned), 6mb total:"
- "L1, all files 6mb "
- "L1.?[0,190] 210ns |------------------------------------------L1.?------------------------------------------|"
- "Committing partition 1:"
- " Soft Deleting 195 files: L0.1, L0.2, L0.3, L0.4, L0.5, L0.6, L0.7, L0.8, L0.9, L0.10, L0.11, L0.12, L0.13, L0.14, L0.15, L0.16, L0.17, L0.18, L0.19, L0.20, L0.21, L0.22, L0.23, L0.24, L0.25, L0.26, L0.27, L0.28, L0.29, L0.30, L0.31, L0.32, L0.33, L0.34, L0.35, L0.36, L0.37, L0.38, L0.39, L0.40, L0.41, L0.42, L0.43, L0.44, L0.45, L0.46, L0.47, L0.48, L0.49, L0.50, L0.51, L0.52, L0.53, L0.54, L0.55, L0.56, L0.57, L0.58, L0.59, L0.60, L0.61, L0.62, L0.63, L0.64, L0.65, L0.66, L0.67, L0.68, L0.69, L0.70, L0.71, L0.72, L0.73, L0.74, L0.75, L0.76, L0.77, L0.78, L0.79, L0.80, L0.81, L0.82, L0.83, L0.84, L0.85, L0.86, L0.87, L0.88, L0.89, L0.90, L0.91, L0.92, L0.93, L0.94, L0.95, L0.96, L0.97, L0.98, L0.99, L0.100, L0.101, L0.102, L0.103, L0.104, L0.105, L0.106, L0.107, L0.108, L0.109, L0.110, L0.111, L0.112, L0.113, L0.114, L0.115, L0.116, L0.117, L0.118, L0.119, L0.120, L0.121, L0.122, L0.123, L0.124, L0.125, L0.126, L0.127, L0.128, L0.129, L0.130, L0.131, L0.132, L0.133, L0.134, L0.135, L0.136, L0.137, L0.138, L0.139, L0.140, L0.141, L0.142, L0.143, L0.144, L0.145, L0.146, L0.147, L0.148, L0.149, L0.150, L0.151, L0.152, L0.153, L0.154, L0.155, L0.156, L0.157, L0.158, L0.159, L0.160, L0.161, L0.162, L0.163, L0.164, L0.165, L0.166, L0.167, L0.168, L0.169, L0.170, L0.171, L0.172, L0.173, L0.174, L0.175, L0.176, L0.177, L0.178, L0.179, L0.180, L0.181, L0.182, L0.183, L0.184, L0.185, L0.186, L0.187, L0.188, L0.189, L0.190, L1.191, L1.192, L1.193, L1.194, L1.195"
- " Creating 1 files"
- "**** Simulation run 1, type=split(CompactAndSplitOutput(TotalSizeLessThanMaxCompactSize))(split_times=[2407]). 16 Input Files, 21mb total:"
- "L1 "
- "L1.196[200,209] 6ns 1mb |L1.196| "
- "L1.197[400,409] 7ns 1mb |L1.197| "
- "L1.198[600,609] 8ns 1mb |L1.198| "
@ -3623,18 +3616,17 @@ async fn not_many_l0_and_overlapped_l1_files() {
- "L1.208[2600,2609] 18ns 1mb |L1.208| "
- "L1.209[2800,2809] 19ns 1mb |L1.209|"
- "L1.210[3000,3009] 20ns 1mb |L1.210|"
- "L1.211[0,190] 210ns 6mb |L1.211| "
- "**** 2 Output Files (parquet_file_id not yet assigned), 21mb total:"
- "L2 "
- "L2.?[0,2407] 210ns 17mb |--------------------------------L2.?---------------------------------| "
- "L2.?[2408,3009] 210ns 4mb |-----L2.?------| "
- "L1 "
- "L1.?[0,2407] 210ns 17mb |--------------------------------L1.?---------------------------------| "
- "L1.?[2408,3009] 210ns 4mb |-----L1.?------| "
- "Committing partition 1:"
- " Soft Deleting 16 files: L1.196, L1.197, L1.198, L1.199, L1.200, L1.201, L1.202, L1.203, L1.204, L1.205, L1.206, L1.207, L1.208, L1.209, L1.210, L1.211"
- " Soft Deleting 210 files: L0.1, L0.2, L0.3, L0.4, L0.5, L0.6, L0.7, L0.8, L0.9, L0.10, L0.11, L0.12, L0.13, L0.14, L0.15, L0.16, L0.17, L0.18, L0.19, L0.20, L0.21, L0.22, L0.23, L0.24, L0.25, L0.26, L0.27, L0.28, L0.29, L0.30, L0.31, L0.32, L0.33, L0.34, L0.35, L0.36, L0.37, L0.38, L0.39, L0.40, L0.41, L0.42, L0.43, L0.44, L0.45, L0.46, L0.47, L0.48, L0.49, L0.50, L0.51, L0.52, L0.53, L0.54, L0.55, L0.56, L0.57, L0.58, L0.59, L0.60, L0.61, L0.62, L0.63, L0.64, L0.65, L0.66, L0.67, L0.68, L0.69, L0.70, L0.71, L0.72, L0.73, L0.74, L0.75, L0.76, L0.77, L0.78, L0.79, L0.80, L0.81, L0.82, L0.83, L0.84, L0.85, L0.86, L0.87, L0.88, L0.89, L0.90, L0.91, L0.92, L0.93, L0.94, L0.95, L0.96, L0.97, L0.98, L0.99, L0.100, L0.101, L0.102, L0.103, L0.104, L0.105, L0.106, L0.107, L0.108, L0.109, L0.110, L0.111, L0.112, L0.113, L0.114, L0.115, L0.116, L0.117, L0.118, L0.119, L0.120, L0.121, L0.122, L0.123, L0.124, L0.125, L0.126, L0.127, L0.128, L0.129, L0.130, L0.131, L0.132, L0.133, L0.134, L0.135, L0.136, L0.137, L0.138, L0.139, L0.140, L0.141, L0.142, L0.143, L0.144, L0.145, L0.146, L0.147, L0.148, L0.149, L0.150, L0.151, L0.152, L0.153, L0.154, L0.155, L0.156, L0.157, L0.158, L0.159, L0.160, L0.161, L0.162, L0.163, L0.164, L0.165, L0.166, L0.167, L0.168, L0.169, L0.170, L0.171, L0.172, L0.173, L0.174, L0.175, L0.176, L0.177, L0.178, L0.179, L0.180, L0.181, L0.182, L0.183, L0.184, L0.185, L0.186, L0.187, L0.188, L0.189, L0.190, L1.191, L1.192, L1.193, L1.194, L1.195, L1.196, L1.197, L1.198, L1.199, L1.200, L1.201, L1.202, L1.203, L1.204, L1.205, L1.206, L1.207, L1.208, L1.209, L1.210"
- " Creating 2 files"
- "**** Final Output Files (28mb written)"
- "L2 "
- "L2.212[0,2407] 210ns 17mb|-------------------------------L2.212--------------------------------| "
- "L2.213[2408,3009] 210ns 4mb |----L2.213-----| "
- "**** Final Output Files (21mb written)"
- "L1 "
- "L1.211[0,2407] 210ns 17mb|-------------------------------L1.211--------------------------------| "
- "L1.212[2408,3009] 210ns 4mb |----L1.212-----| "
"###
);
}

63
compactor_test_utils/src/lib.rs
View File

@ -1593,6 +1593,69 @@ pub fn create_overlapped_files_3_mix_size(size: i64) -> Vec<ParquetFile> {
vec![l0_3, l0_2, l0_1, l0_4, l0_5, l0_6, l1_1, l1_2]
}
/// This setup will return files with ranges as follows:
/// Input:
/// |--L0.1--| |-L0.2-|
/// |--L1.1--| |--L1.2--| |--L1.3--| |--L1.4--| |--L1.5--| |--L1.6--| |--L1.7--| |--L1.8--|
/// l1 size : med large small med med small large med
pub fn create_overlapped_files_mix_sizes_1(small: i64, med: i64, large: i64) -> Vec<ParquetFile> {
let l1_1 = ParquetFileBuilder::new(11)
.with_compaction_level(CompactionLevel::FileNonOverlapped)
.with_time_range(0, 100)
.with_file_size_bytes(med)
.build();
let l1_2 = ParquetFileBuilder::new(12)
.with_compaction_level(CompactionLevel::FileNonOverlapped)
.with_time_range(200, 300)
.with_file_size_bytes(large)
.build();
let l1_3 = ParquetFileBuilder::new(13)
.with_compaction_level(CompactionLevel::FileNonOverlapped)
.with_time_range(400, 500)
.with_file_size_bytes(small)
.build();
let l1_4 = ParquetFileBuilder::new(14)
.with_compaction_level(CompactionLevel::FileNonOverlapped)
.with_time_range(600, 700)
.with_file_size_bytes(med)
.build();
let l1_5 = ParquetFileBuilder::new(15)
.with_compaction_level(CompactionLevel::FileNonOverlapped)
.with_time_range(800, 900)
.with_file_size_bytes(med)
.build();
let l1_6 = ParquetFileBuilder::new(16)
.with_compaction_level(CompactionLevel::FileNonOverlapped)
.with_time_range(1000, 1100)
.with_file_size_bytes(small)
.build();
let l1_7 = ParquetFileBuilder::new(17)
.with_compaction_level(CompactionLevel::FileNonOverlapped)
.with_time_range(1200, 1300)
.with_file_size_bytes(large)
.build();
let l1_8 = ParquetFileBuilder::new(18)
.with_compaction_level(CompactionLevel::FileNonOverlapped)
.with_time_range(1400, 1500)
.with_file_size_bytes(med)
.build();
// L0_1 overlaps with L1_4
let l0_1 = ParquetFileBuilder::new(1)
.with_compaction_level(CompactionLevel::Initial)
.with_time_range(650, 750)
.with_file_size_bytes(small)
.build();
// L0_2 overlaps with L1_5
let l0_2 = ParquetFileBuilder::new(2)
.with_compaction_level(CompactionLevel::Initial)
.with_time_range(820, 850)
.with_file_size_bytes(small)
.build();
vec![l1_3, l1_7, l1_2, l1_1, l1_4, l1_5, l1_6, l0_2, l0_1, l1_8]
}
#[cfg(test)]
mod test {
use super::*;