Influxdata
/
influxdb
mirror of https://github.com/influxdata/influxdb.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

feat: Add Compaction Regions (#8559) 

* feat: add CompactRanges RoundInfo type

* chore: insta test updates for adding CompactRange

* feat: simplify/improve ManySmallFiles logic, now that its problem set is simpler

* chore: insta test updates for ManySmallFiles improvement

* chore: upgrade files more aggressively

* chore: insta updates from more aggressive file upgrades

* chore: addressing review comments
pull/24376/head
Joe-Blount 2023-08-28 07:59:12 -05:00 committed by GitHub
parent 8739f30451
commit 1df5948c97
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
18 changed files with 5897 additions and 8128 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
compactor/src/components/divide_initial/multiple_branches.rs
View File

@ -245,6 +245,24 @@ impl DivideInitial for MultipleBranchesDivideInitial {
// RoundSplit already eliminated all the files we don't need to work on.
RoundInfo::VerticalSplit { .. } => (vec![files], more_for_later),
RoundInfo::CompactRanges { ranges, .. } => {
// Each range describes what can be a distinct branch, concurrently compacted.
let mut branches = Vec::with_capacity(ranges.len());
let mut this_branch: Vec<ParquetFile>;
let mut files = files;
for range in &ranges {
(this_branch, files) = files.into_iter().partition(|f2| {
f2.overlaps_time_range(Timestamp::new(range.min), Timestamp::new(range.max))
});
branches.push(this_branch)
}
assert!(files.is_empty(), "all files should map to a range");
(branches, vec![])
}
}
}
}

125
compactor/src/components/file_classifier/split_based.rs
View File

@ -207,11 +207,82 @@ where
files_to_keep,
}
}
RoundInfo::CompactRanges {
max_num_files_to_group,
max_total_file_size_to_group,
..
} => {
let l0_count = files_to_compact
.iter()
.filter(|f| f.compaction_level == CompactionLevel::Initial)
.count();
if l0_count > *max_num_files_to_group {
// Too many L0s, do manySmallFiles within this range.
let (files_to_compact, mut files_to_keep) = files_to_compact
.into_iter()
.partition(|f| f.compaction_level == CompactionLevel::Initial);
let l0_classification = file_classification_for_many_files(
*max_total_file_size_to_group,
*max_num_files_to_group,
files_to_compact,
CompactionLevel::Initial,
);
files_to_keep.extend(l0_classification.files_to_keep);
assert!(!l0_classification.files_to_make_progress_on.is_empty());
FileClassification {
target_level: l0_classification.target_level,
files_to_make_progress_on: l0_classification.files_to_make_progress_on,
files_to_keep,
}
} else {
// There's not too many L0s, so upgrade/split/compact as required to get L0s->L1.
let target_level = CompactionLevel::FileNonOverlapped;
let (files_to_compact, mut files_to_keep) = self
.target_level_split
.apply(files_to_compact, target_level);
// To have efficient compaction performance, we do not need to compact eligible non-overlapped files
// Find eligible non-overlapped files and keep for next round of compaction
let (files_to_compact, non_overlapping_files) =
self.non_overlap_split.apply(files_to_compact, target_level);
files_to_keep.extend(non_overlapping_files);
// To have efficient compaction performance, we only need to upgrade (catalog update only) eligible files
let (files_to_compact, files_to_upgrade) =
self.upgrade_split.apply(files_to_compact, target_level);
// See if we need to split start-level files due to over compaction size limit
let (files_to_split_or_compact, other_files) =
self.split_or_compact
.apply(partition_info, files_to_compact, target_level);
files_to_keep.extend(other_files);
let files_to_make_progress_on = FilesForProgress {
upgrade: files_to_upgrade,
split_or_compact: files_to_split_or_compact,
};
assert!(!files_to_make_progress_on.is_empty());
FileClassification {
target_level,
files_to_make_progress_on,
files_to_keep,
}
}
}
}
}
}
// ManySmallFiles assumes the L0 files are tiny and aims to do L0-> L0 compaction to reduce the number of tiny files.
// With vertical splitting, this only operates on a CompactionRange that's up to the max_compact_size. So while there's
// many files, we know there's not many bytes (in total). Because of this, We can skip anything that's a sizeable portion
// of the max_compact_size, knowing that we can still get the L0 quantity down to max_num_files_to_group.
fn file_classification_for_many_files(
max_total_file_size_to_group: usize,
max_num_files_to_group: usize,
@ -231,27 +302,53 @@ fn file_classification_for_many_files(
let mut files_to_compact = vec![];
let mut files_to_keep: Vec<ParquetFile> = vec![];
// The goal is to compact the small files without repeately rewriting the non-small files (that hurts write amp).
// Assume tiny files separated by non-tiny files, we need to get down to max_num_files_to_group.
// So compute the biggest files we can skip, and still be guaranteed to get down to max_num_files_to_group.
let skip_size = max_total_file_size_to_group * 2 / max_num_files_to_group;
// Enforce max_num_files_to_group
if files.len() > max_num_files_to_group {
let ordered_files = order_files(files, target_level.prev());
ordered_files
.chunks(max_num_files_to_group)
.for_each(|chunk| {
let this_chunk_bytes: usize =
chunk.iter().map(|f| f.file_size_bytes as usize).sum();
if this_chunk_bytes > max_total_file_size_to_group {
// This chunk of files are plenty big and don't fit the ManySmallFiles characteristics.
// If we let ManySmallFiles handle them, it may get stuck with unproductive compactions.
// So set them aside for later (when we're not in ManySmallFiles mode).
files_to_keep.append(chunk.to_vec().as_mut());
} else if files_to_compact.is_empty() {
let mut chunk_bytes: usize = 0;
let mut chunk: Vec<ParquetFile> = Vec::with_capacity(max_num_files_to_group);
for f in ordered_files {
if !files_to_compact.is_empty() {
// We've already got a batch of files to compact, this can wait.
files_to_keep.push(f);
} else if chunk_bytes + f.file_size_bytes as usize > max_total_file_size_to_group
|| chunk.len() + 1 > max_num_files_to_group
|| f.file_size_bytes >= skip_size as i64
{
// This file will not be included in this compaction.
files_to_keep.push(f);
if chunk.len() > 1 {
// Several files; we'll do an L0->L0 comapction on them.
files_to_compact = chunk.to_vec();
} else {
// We've already got a batch of files to compact, these can wait.
chunk = Vec::with_capacity(max_num_files_to_group);
} else if !chunk.is_empty() {
// Just one file, and we don't want to compact it with 'f', so skip it.
files_to_keep.append(chunk.to_vec().as_mut());
chunk = Vec::with_capacity(max_num_files_to_group);
}
});
} else {
// This files goes in our draft chunk to compact
chunk_bytes += f.file_size_bytes as usize;
chunk.push(f);
}
}
if !chunk.is_empty() {
assert!(files_to_compact.is_empty());
if chunk.len() > 1 {
// We need to compact what comes before f
files_to_compact = chunk.to_vec();
} else if !chunk.is_empty() {
files_to_keep.append(chunk.to_vec().as_mut());
}
}
assert!(chunk.is_empty() || chunk.len() > 1);
} else {
files_to_compact = files;
}

150
compactor/src/components/files_split/upgrade_split.rs
View File

@ -34,13 +34,13 @@ impl Display for UpgradeSplit {
impl FilesSplit for UpgradeSplit {
/// Return (`[files_to_compact]`, `[files_to_upgrade]`) of the given files
/// so that `files_to_upgrade` does not overlap with any files in previous level
/// so that `files_to_upgrade` does not overlap with any files in next level
///
/// The files_to_upgrade must in the (target_level - 1)
///
/// Eligible upgradable files are large-enough-file (>= max_desired_file_size) files of the previous level of
/// Eligible upgradable files are large-enough-file (>= max_desired_file_size/2) files of the previous level of
/// the target level that do not overlap on time range with any files in its level and higher-level files.
/// Note: we always have to stick the the invariance that the outout files must not overlap
/// Note: we always have to stick to the invariance that the outout files must not overlap
///
/// Example:
/// |--L0.1--| |--L0.2--| |--L0.3--|
@ -54,7 +54,7 @@ impl FilesSplit for UpgradeSplit {
///
/// Algorithm:
/// The non-overlappings files are files of the (target_level -1) files that:
/// 1. Size >= max_desire_file_size
/// 1. Size >= max_desire_file_size/2
/// 2. Completely outside the time range of all higher level files
/// 3. Not overlap with any files in the same level
/// 4. Not overlap with the time range of the files not meet 3 conditions above
@ -81,7 +81,7 @@ impl FilesSplit for UpgradeSplit {
// Go go over all files of previous level and check if they are NOT eligible to upgrade
// by hit one of this conditions
// . Size < max_desire_file_size
// . Size < max_desire_file_size/2
// . Overlap with time range of target_level_files
// . Overlap with any files in the same level
// Otherwise, they are large and not overlap. Put them in the potential upgradable list
@ -90,7 +90,7 @@ impl FilesSplit for UpgradeSplit {
let mut potential_upgradable_files = Vec::with_capacity(prev_level_files.len());
while let Some(file) = prev_level_files.pop() {
// size is small
if file.file_size_bytes < self.max_desired_file_size_bytes as i64 {
if file.file_size_bytes < self.max_desired_file_size_bytes as i64 / 2 {
files_to_compact.push(file);
} else if let Some(target_time_range) = target_time_range {
// overlap with target_level_files
@ -203,13 +203,13 @@ mod tests {
#[test]
fn test_apply_one_level_overlap_small_l0() {
let files = create_overlapping_l0_files((MAX_SIZE - 1) as i64);
let files = create_overlapping_l0_files((MAX_SIZE / 2 - 1) as i64);
insta::assert_yaml_snapshot!(
format_files("initial", &files),
@r###"
---
- initial
- "L0, all files 99b "
- "L0, all files 49b "
- "L0.2[150,180] 0ns |L0.2| "
- "L0.1[100,200] 0ns |--L0.1---| "
- "L0.3[800,900] 0ns |--L0.3---| "
@ -226,7 +226,7 @@ mod tests {
@r###"
---
- files_to_compact
- "L0, all files 99b "
- "L0, all files 49b "
- "L0.3[800,900] 0ns |--L0.3---| "
- "L0.1[100,200] 0ns |--L0.1---| "
- "L0.2[150,180] 0ns |L0.2| "
@ -274,13 +274,13 @@ mod tests {
#[test]
fn test_apply_one_level_small_l0() {
let files = create_l0_files((MAX_SIZE - 1) as i64);
let files = create_l0_files((MAX_SIZE / 2 - 1) as i64);
insta::assert_yaml_snapshot!(
format_files("initial", &files),
@r###"
---
- initial
- "L0, all files 99b "
- "L0, all files 49b "
- "L0.2[650,750] 0ns |-------L0.2-------| "
- "L0.1[450,620] 0ns |--------------L0.1--------------| "
- "L0.3[800,900] 0ns |-------L0.3-------|"
@ -296,7 +296,7 @@ mod tests {
@r###"
---
- files_to_compact
- "L0, all files 99b "
- "L0, all files 49b "
- "L0.3[800,900] 0ns |-------L0.3-------|"
- "L0.1[450,620] 0ns |--------------L0.1--------------| "
- "L0.2[650,750] 0ns |-------L0.2-------| "
@ -394,7 +394,7 @@ mod tests {
#[test]
fn test_apply_one_level_l1_mix_size() {
let files = create_l1_files_mix_size(MAX_SIZE as i64);
let files = create_l1_files_mix_size((MAX_SIZE / 2) as i64);
// . small files (< size ): L1.1, L1.3
// . Large files (.= size): L1.2, L1.4, L1.5
@ -407,11 +407,11 @@ mod tests {
---
- initial
- "L1 "
- "L1.15[1000,1100] 0ns 200b |-L1.15--| "
- "L1.13[600,700] 0ns 90b |-L1.13--| "
- "L1.12[400,500] 0ns 101b |-L1.12--| "
- "L1.11[250,350] 0ns 99b |-L1.11--| "
- "L1.14[800,900] 0ns 100b |-L1.14--| "
- "L1.15[1000,1100] 0ns 150b |-L1.15--| "
- "L1.13[600,700] 0ns 40b |-L1.13--| "
- "L1.12[400,500] 0ns 51b |-L1.12--| "
- "L1.11[250,350] 0ns 49b |-L1.11--| "
- "L1.14[800,900] 0ns 50b |-L1.14--| "
"###
);
@ -425,30 +425,30 @@ mod tests {
---
- files_to_compact
- "L1 "
- "L1.11[250,350] 0ns 99b |------L1.11-------| "
- "L1.13[600,700] 0ns 90b |------L1.13-------|"
- "L1.12[400,500] 0ns 101b |------L1.12-------| "
- "L1.11[250,350] 0ns 49b |------L1.11-------| "
- "L1.13[600,700] 0ns 40b |------L1.13-------|"
- "L1.12[400,500] 0ns 51b |------L1.12-------| "
- files_to_upgrade
- "L1 "
- "L1.15[1000,1100] 0ns 200b |-----------L1.15------------|"
- "L1.14[800,900] 0ns 100b |-----------L1.14------------| "
- "L1.15[1000,1100] 0ns 150b |-----------L1.15------------|"
- "L1.14[800,900] 0ns 50b |-----------L1.14------------| "
"###
);
}
#[test]
fn test_apply_all_small_target_l1() {
let files = create_overlapped_l0_l1_files((MAX_SIZE - 1) as i64);
let files = create_overlapped_l0_l1_files((MAX_SIZE / 2 - 1) as i64);
insta::assert_yaml_snapshot!(
format_files("initial", &files),
@r###"
---
- initial
- "L0, all files 99b "
- "L0, all files 49b "
- "L0.2[650,750] 180s |---L0.2----| "
- "L0.1[450,620] 120s |--------L0.1---------| "
- "L0.3[800,900] 300s |---L0.3----| "
- "L1, all files 99b "
- "L1, all files 49b "
- "L1.13[600,700] 60s |---L1.13---| "
- "L1.12[400,500] 60s |---L1.12---| "
- "L1.11[250,350] 60s |---L1.11---| "
@ -465,11 +465,11 @@ mod tests {
@r###"
---
- files_to_compact
- "L0, all files 99b "
- "L0, all files 49b "
- "L0.3[800,900] 300s |---L0.3----| "
- "L0.1[450,620] 120s |--------L0.1---------| "
- "L0.2[650,750] 180s |---L0.2----| "
- "L1, all files 99b "
- "L1, all files 49b "
- "L1.13[600,700] 60s |---L1.13---| "
- "L1.12[400,500] 60s |---L1.12---| "
- "L1.11[250,350] 60s |---L1.11---| "
@ -523,7 +523,7 @@ mod tests {
#[test]
fn test_apply_all_small_target_l2() {
let files = create_overlapped_l1_l2_files((MAX_SIZE - 1) as i64);
let files = create_overlapped_l1_l2_files((MAX_SIZE / 2 - 1) as i64);
let split = UpgradeSplit::new(MAX_SIZE);
let (files_to_compact, files_to_upgrade) = split.apply(files, CompactionLevel::Final);
@ -533,11 +533,11 @@ mod tests {
@r###"
---
- files_to_compact
- "L1, all files 99b "
- "L1, all files 49b "
- "L1.11[250,350] 0ns |--L1.11---| "
- "L1.12[400,500] 0ns |--L1.12---| "
- "L1.13[600,700] 0ns |--L1.13---| "
- "L2, all files 99b "
- "L2, all files 49b "
- "L2.21[0,100] 0ns |--L2.21---| "
- "L2.22[200,300] 0ns |--L2.22---| "
- files_to_upgrade
@ -587,7 +587,7 @@ mod tests {
#[test]
fn test_apply_all_small_target_l2_mix_size() {
let files = create_overlapped_l1_l2_files_mix_size(MAX_SIZE as i64);
let files = create_overlapped_l1_l2_files_mix_size((MAX_SIZE / 2) as i64);
// Small files (< size): [L1.3]
// Large files: [L2.1, L2.2, L1.1, L1.2]
// ==> nothing to upgrade
@ -597,12 +597,12 @@ mod tests {
---
- initial
- "L1 "
- "L1.13[600,700] 0ns 99b |--L1.13---| "
- "L1.12[400,500] 0ns 100b |--L1.12---| "
- "L1.11[250,350] 0ns 100b |--L1.11---| "
- "L1.13[600,700] 0ns 49b |--L1.13---| "
- "L1.12[400,500] 0ns 50b |--L1.12---| "
- "L1.11[250,350] 0ns 50b |--L1.11---| "
- "L2 "
- "L2.21[0,100] 0ns 100b |--L2.21---| "
- "L2.22[200,300] 0ns 100b |--L2.22---| "
- "L2.21[0,100] 0ns 50b |--L2.21---| "
- "L2.22[200,300] 0ns 50b |--L2.22---| "
"###
);
@ -615,12 +615,12 @@ mod tests {
---
- files_to_compact
- "L1 "
- "L1.11[250,350] 0ns 100b |--L1.11---| "
- "L1.13[600,700] 0ns 99b |--L1.13---| "
- "L1.12[400,500] 0ns 100b |--L1.12---| "
- "L1.11[250,350] 0ns 50b |--L1.11---| "
- "L1.13[600,700] 0ns 49b |--L1.13---| "
- "L1.12[400,500] 0ns 50b |--L1.12---| "
- "L2 "
- "L2.21[0,100] 0ns 100b |--L2.21---| "
- "L2.22[200,300] 0ns 100b |--L2.22---| "
- "L2.21[0,100] 0ns 50b |--L2.21---| "
- "L2.22[200,300] 0ns 50b |--L2.22---| "
- files_to_upgrade
"###
);
@ -628,7 +628,7 @@ mod tests {
#[test]
fn test_apply_all_small_target_l2_mix_size_2() {
let files = create_overlapped_l1_l2_files_mix_size_2(MAX_SIZE as i64);
let files = create_overlapped_l1_l2_files_mix_size_2((MAX_SIZE / 2) as i64);
// Small files (< size): [L1.2]
// Large files: [L2.1, L2.2, L1.1, L1.3]
// ==> L1.3 is eligible for upgrade
@ -638,12 +638,12 @@ mod tests {
---
- initial
- "L1 "
- "L1.13[600,700] 0ns 100b |--L1.13---| "
- "L1.12[400,500] 0ns 99b |--L1.12---| "
- "L1.11[250,350] 0ns 100b |--L1.11---| "
- "L1.13[600,700] 0ns 50b |--L1.13---| "
- "L1.12[400,500] 0ns 49b |--L1.12---| "
- "L1.11[250,350] 0ns 50b |--L1.11---| "
- "L2 "
- "L2.21[0,100] 0ns 100b |--L2.21---| "
- "L2.22[200,300] 0ns 100b |--L2.22---| "
- "L2.21[0,100] 0ns 50b |--L2.21---| "
- "L2.22[200,300] 0ns 50b |--L2.22---| "
"###
);
@ -655,13 +655,13 @@ mod tests {
---
- files_to_compact
- "L1 "
- "L1.11[250,350] 0ns 100b |-----L1.11------| "
- "L1.12[400,500] 0ns 99b |-----L1.12------|"
- "L1.11[250,350] 0ns 50b |-----L1.11------| "
- "L1.12[400,500] 0ns 49b |-----L1.12------|"
- "L2 "
- "L2.21[0,100] 0ns 100b |-----L2.21------| "
- "L2.22[200,300] 0ns 100b |-----L2.22------| "
- "L2.21[0,100] 0ns 50b |-----L2.21------| "
- "L2.22[200,300] 0ns 50b |-----L2.22------| "
- files_to_upgrade
- "L1, all files 100b "
- "L1, all files 50b "
- "L1.13[600,700] 0ns |-----------------------------------------L1.13------------------------------------------|"
"###
);
@ -711,21 +711,21 @@ mod tests {
#[test]
fn test_apply_all_small_target_l1_2() {
let files = create_overlapped_files_3((MAX_SIZE - 1) as i64);
let files = create_overlapped_files_3((MAX_SIZE / 2 - 1) as i64);
// All small ==> nothing to upgrade
insta::assert_yaml_snapshot!(
format_files("initial", &files),
@r###"
---
- initial
- "L0, all files 99b "
- "L0, all files 49b "
- "L0.3[400,500] 0ns |-L0.3-| "
- "L0.2[200,300] 0ns |-L0.2-| "
- "L0.1[0,100] 0ns |-L0.1-| "
- "L0.4[600,700] 0ns |-L0.4-| "
- "L0.5[800,900] 0ns |-L0.5-| "
- "L0.6[1000,1100] 0ns |-L0.6-| "
- "L1, all files 99b "
- "L1, all files 49b "
- "L1.11[250,350] 0ns |L1.11-| "
- "L1.12[650,750] 0ns |L1.12-| "
"###
@ -740,14 +740,14 @@ mod tests {
@r###"
---
- files_to_compact
- "L0, all files 99b "
- "L0, all files 49b "
- "L0.6[1000,1100] 0ns |-L0.6-| "
- "L0.5[800,900] 0ns |-L0.5-| "
- "L0.4[600,700] 0ns |-L0.4-| "
- "L0.1[0,100] 0ns |-L0.1-| "
- "L0.2[200,300] 0ns |-L0.2-| "
- "L0.3[400,500] 0ns |-L0.3-| "
- "L1, all files 99b "
- "L1, all files 49b "
- "L1.11[250,350] 0ns |L1.11-| "
- "L1.12[650,750] 0ns |L1.12-| "
- files_to_upgrade
@ -806,7 +806,7 @@ mod tests {
#[test]
fn test_apply_mix_size_target_l1_2() {
let files = create_overlapped_files_3_mix_size(MAX_SIZE as i64);
let files = create_overlapped_files_3_mix_size((MAX_SIZE / 2) as i64);
// Small files (< size): L0.6
// Large files: the rest
// ==> only L0.1 is eligible for upgrade
@ -816,15 +816,15 @@ mod tests {
---
- initial
- "L0 "
- "L0.3[400,500] 0ns 100b |-L0.3-| "
- "L0.2[200,300] 0ns 100b |-L0.2-| "
- "L0.1[0,100] 0ns 100b |-L0.1-| "
- "L0.4[600,700] 0ns 100b |-L0.4-| "
- "L0.5[800,900] 0ns 100b |-L0.5-| "
- "L0.6[1000,1100] 0ns 99b |-L0.6-| "
- "L0.3[400,500] 0ns 50b |-L0.3-| "
- "L0.2[200,300] 0ns 50b |-L0.2-| "
- "L0.1[0,100] 0ns 50b |-L0.1-| "
- "L0.4[600,700] 0ns 50b |-L0.4-| "
- "L0.5[800,900] 0ns 50b |-L0.5-| "
- "L0.6[1000,1100] 0ns 49b |-L0.6-| "
- "L1 "
- "L1.11[250,350] 0ns 100b |L1.11-| "
- "L1.12[650,750] 0ns 100b |L1.12-| "
- "L1.11[250,350] 0ns 50b |L1.11-| "
- "L1.12[650,750] 0ns 50b |L1.12-| "
"###
);
@ -838,16 +838,16 @@ mod tests {
---
- files_to_compact
- "L0 "
- "L0.6[1000,1100] 0ns 99b |--L0.6--|"
- "L0.4[600,700] 0ns 100b |--L0.4--| "
- "L0.2[200,300] 0ns 100b |--L0.2--| "
- "L0.3[400,500] 0ns 100b |--L0.3--| "
- "L0.5[800,900] 0ns 100b |--L0.5--| "
- "L0.6[1000,1100] 0ns 49b |--L0.6--|"
- "L0.4[600,700] 0ns 50b |--L0.4--| "
- "L0.2[200,300] 0ns 50b |--L0.2--| "
- "L0.3[400,500] 0ns 50b |--L0.3--| "
- "L0.5[800,900] 0ns 50b |--L0.5--| "
- "L1 "
- "L1.11[250,350] 0ns 100b |-L1.11--| "
- "L1.12[650,750] 0ns 100b |-L1.12--| "
- "L1.11[250,350] 0ns 50b |-L1.11--| "
- "L1.12[650,750] 0ns 50b |-L1.12--| "
- files_to_upgrade
- "L0, all files 100b "
- "L0, all files 50b "
- "L0.1[0,100] 0ns |------------------------------------------L0.1------------------------------------------|"
"###
);

138
compactor/src/components/round_info_source/mod.rs
View File

@ -11,7 +11,7 @@ use crate::components::{
Components,
};
use async_trait::async_trait;
use data_types::{CompactionLevel, ParquetFile, Timestamp};
use data_types::{CompactionLevel, FileRange, ParquetFile, Timestamp};
use itertools::Itertools;
use observability_deps::tracing::debug;
@ -28,6 +28,7 @@ pub trait RoundInfoSource: Debug + Display + Send + Sync {
async fn calculate(
&self,
components: Arc<Components>,
last_round_info: Option<RoundInfo>,
partition_info: &PartitionInfo,
files: Vec<ParquetFile>,
) -> Result<(RoundInfo, Vec<Vec<ParquetFile>>, Vec<ParquetFile>), DynError>;
@ -55,12 +56,13 @@ impl RoundInfoSource for LoggingRoundInfoWrapper {
async fn calculate(
&self,
components: Arc<Components>,
last_round_info: Option<RoundInfo>,
partition_info: &PartitionInfo,
files: Vec<ParquetFile>,
) -> Result<(RoundInfo, Vec<Vec<ParquetFile>>, Vec<ParquetFile>), DynError> {
let res = self
.inner
.calculate(components, partition_info, files)
.calculate(components, last_round_info, partition_info, files)
.await;
if let Ok((round_info, branches, files_later)) = &res {
debug!(round_info_source=%self.inner, %round_info, branches=branches.len(), files_later=files_later.len(), "running round");
@ -90,6 +92,10 @@ impl LevelBasedRoundInfo {
}
/// Returns true if the scenario looks like ManySmallFiles, but we can't group them well into branches.
/// TODO: use this or remove it. For now, keep it in case we need the temporary workaround again.
/// This can be used to identify criteria to trigger a SimulatedLeadingEdge as a temporary workaround
/// for a situation that isn't well handled, when the desire is to postpone optimal handling to a later PR.
#[allow(dead_code)]
pub fn many_ungroupable_files(
&self,
files: &[ParquetFile],
@ -202,19 +208,24 @@ impl LevelBasedRoundInfo {
false
}
/// vertical_split_times evaluates the files to determine if vertical splitting is necessary. If so,
/// a vec of split times is returned. The caller will use those split times in a VerticalSplit RoundInfo.
/// If no split times are returned, that implies veritcal splitting is not appropriate, and the caller
/// will identify another type of RoundInfo for this round of compaction.
pub fn vertical_split_times(
/// vertical_split_handling determines if vertical splitting is necessary, or has already been done.
/// If splitting is necessary, a vec of split times is returned. If a previous split is detected, a
/// vec of CompactionRange is returned to preserve the prior split.
/// The need for more splitting takes precedence over acting on prior splitting. So if a vec of split times
/// is returned, the caller will use those split times in a VerticalSplit RoundInfo for vertical splitting.
/// If only a vec of CompactRanges are returned, the caller will use those to preserve the prior split until
/// all the L0s are compacted to L1.
/// If neither is returned, the caller will identify another type of RoundInfo for this round of compaction.
pub fn vertical_split_handling(
&self,
files: Vec<ParquetFile>,
max_compact_size: usize,
) -> Vec<i64> {
let (start_level_files, target_level_files): (Vec<ParquetFile>, Vec<ParquetFile>) = files
.into_iter()
.filter(|f| f.compaction_level != CompactionLevel::Final)
.partition(|f| f.compaction_level == CompactionLevel::Initial);
) -> (Vec<i64>, Vec<FileRange>) {
let (start_level_files, mut target_level_files): (Vec<ParquetFile>, Vec<ParquetFile>) =
files
.into_iter()
.filter(|f| f.compaction_level != CompactionLevel::Final)
.partition(|f| f.compaction_level == CompactionLevel::Initial);
let len = start_level_files.len();
let mut split_times = Vec::with_capacity(len);
@ -222,8 +233,10 @@ impl LevelBasedRoundInfo {
// Break up the start level files into chains of files that overlap each other.
// Then we'll determine if vertical splitting is needed within each chain.
let chains = split_into_chains(start_level_files);
let chains = merge_small_l0_chains(chains, max_compact_size);
let mut ranges = Vec::with_capacity(chains.len());
for chain in chains {
for chain in &chains {
let chain_cap: usize = chain.iter().map(|f| f.file_size_bytes as usize).sum();
// A single file over max size can just get upgraded to L1, then L2, unless it overlaps other L0s.
@ -233,7 +246,7 @@ impl LevelBasedRoundInfo {
// We can't know the data distribution within each file without reading the file (too expensive), but we can
// still learn a lot about the data distribution accross the set of files by assuming even distribtuion within each
// file and considering the distribution of files within the chain's time range.
let linear_ranges = linear_dist_ranges(&chain, chain_cap, max_compact_size);
let linear_ranges = linear_dist_ranges(chain, chain_cap, max_compact_size);
for range in linear_ranges {
// split at every time range of linear distribution.
@ -284,9 +297,64 @@ impl LevelBasedRoundInfo {
}
}
// If we're not doing vertical splitting, while we've got the chains, lets check for a previous vertical split.
// We'll preserve prior splitting activity by creating CompactionRange for each of the previous splits.
if chains.len() > 1 {
let mut prior_overlapping_max = Timestamp::new(0);
let mut prior_chain_max: i64 = 0;
let mut overlaps: Vec<ParquetFile>;
for chain in &chains {
let mut min = chain.iter().map(|f| f.min_time).min().unwrap();
let max = chain.iter().map(|f| f.max_time).max().unwrap();
if min <= prior_overlapping_max && prior_overlapping_max != Timestamp::new(0) {
// Target level files overlap more than one start level file, and there is a target level file overlapping
// the prior chain of L0s and this one. We'll split the target level file at the pror range/chain max before
// proceeding with compactions.
split_times.push(prior_chain_max)
}
// As we identify overlaps, we'll include some don't quite overlap, but are between the prior chain and this one.
// By including them here, we preserve the opportunity to grow small L1s in a "gappy" leading edge pattern.
// If they're large, they'll be excluded from the L0->L1 compaction, so there's no harm including them.
let search_min =
(prior_overlapping_max + 1).max(Timestamp::new(prior_chain_max + 1));
(overlaps, target_level_files) = target_level_files.into_iter().partition(|f2| {
f2.overlaps_time_range(search_min, max)
&& f2.compaction_level != CompactionLevel::Final
});
let cap: usize = chain
.iter()
.map(|f| f.file_size_bytes as usize)
.sum::<usize>()
+ overlaps
.iter()
.map(|f| f.file_size_bytes as usize)
.sum::<usize>();
if !overlaps.is_empty() {
prior_overlapping_max = overlaps.iter().map(|f| f.max_time).max().unwrap();
let prior_smallest_max = overlaps.iter().map(|f| f.max_time).min().unwrap();
if prior_smallest_max < min {
// Expand the region to include this file, so it can be included (if its small and we'd like to grow it).
min = prior_smallest_max;
}
}
prior_chain_max = max.get();
ranges.push(FileRange {
min: min.get(),
max: max.get(),
cap,
});
}
}
split_times.sort();
split_times.dedup();
split_times
(split_times, ranges)
}
}
@ -297,9 +365,30 @@ impl RoundInfoSource for LevelBasedRoundInfo {
async fn calculate(
&self,
components: Arc<Components>,
last_round_info: Option<RoundInfo>,
_partition_info: &PartitionInfo,
files: Vec<ParquetFile>,
) -> Result<(RoundInfo, Vec<Vec<ParquetFile>>, Vec<ParquetFile>), DynError> {
let mut ranges: Vec<FileRange> = vec![];
if let Some(last_round_info) = last_round_info {
if let Some(last_ranges) = last_round_info.ranges() {
// Last round had L0 CompactRange. If we have unfinished business from that,
// we need to continue with those ranges.
for range in last_ranges {
// If this range still has overapping L0 files, we need to keep it.
for f in &files {
if f.compaction_level == CompactionLevel::Initial
&& f.overlaps_ranges(&vec![range])
{
ranges.push(range);
break;
}
}
}
}
}
// start_level is usually the lowest level we have files in, but occasionally we decide to
// compact L1->L2 when L0s still exist. If this comes back as L1, we'll ignore L0s for this
// round and force an early L1-L2 compaction.
@ -309,14 +398,21 @@ impl RoundInfoSource for LevelBasedRoundInfo {
self.max_total_file_size_per_plan,
);
let round_info = if start_level == CompactionLevel::Initial {
let split_times = self
.vertical_split_times(files.clone().to_vec(), self.max_total_file_size_per_plan);
let round_info = if !ranges.is_empty() {
RoundInfo::CompactRanges {
ranges,
max_num_files_to_group: self.max_num_files_per_plan,
max_total_file_size_to_group: self.max_total_file_size_per_plan,
}
} else if start_level == CompactionLevel::Initial {
let (split_times, ranges) = self
.vertical_split_handling(files.clone().to_vec(), self.max_total_file_size_per_plan);
if !split_times.is_empty() {
RoundInfo::VerticalSplit { split_times }
} else if self.many_ungroupable_files(&files, start_level, self.max_num_files_per_plan)
{
RoundInfo::SimulatedLeadingEdge {
} else if !ranges.is_empty() {
RoundInfo::CompactRanges {
ranges,
max_num_files_to_group: self.max_num_files_per_plan,
max_total_file_size_to_group: self.max_total_file_size_per_plan,
}

18
compactor/src/components/round_split/many_files.rs
View File

@ -67,12 +67,26 @@ impl RoundSplit for ManyFilesRoundSplit {
// We're splitting L0 files at split_times. So any L0 that overlaps a split_time needs processed, and all other files are ignored until later.
let (split_files, rest): (Vec<ParquetFile>, Vec<ParquetFile>) =
files.into_iter().partition(|f| {
f.compaction_level == CompactionLevel::Initial
&& f.needs_split(&split_times)
f.compaction_level != CompactionLevel::Final && f.needs_split(&split_times)
});
assert!(
!split_files.is_empty(),
"if we decided to split, there should be something to split"
);
(split_files, rest)
}
RoundInfo::CompactRanges { ranges, .. } => {
// We're compacting L0 & L1s in the specified ranges. Files outside these ranges are
// ignored until a later round.
let (compact_files, rest): (Vec<ParquetFile>, Vec<ParquetFile>) =
files.into_iter().partition(|f| {
f.compaction_level != CompactionLevel::Final && f.overlaps_ranges(&ranges)
});
(compact_files, rest)
}
}
}
}

4
compactor/src/components/split_or_compact/start_level_files_to_split.rs
View File

@ -335,6 +335,7 @@ pub fn merge_small_l0_chains(
// matching max_lo_created_at times indicates that the files were deliberately split. We shouldn't merge
// chains with matching max_lo_created_at times, because that would encourage undoing the previous split,
// which minimally increases write amplification, and may cause unproductive split/compact loops.
// TODO: this may not be necessary long term (with CompactRanges this might be ok)
let mut matches = 0;
if prior_chain_bytes > 0 {
for f in chain.iter() {
@ -362,6 +363,9 @@ pub fn merge_small_l0_chains(
}
}
// Put it back in the standard order.
merged_chains.sort_by_key(|a| a[0].min_time);
merged_chains
}

3
compactor/src/driver.rs
View File

@ -215,6 +215,7 @@ async fn try_compact_partition(
let mut files = components.partition_files_source.fetch(partition_id).await;
let partition_info = components.partition_info_source.fetch(partition_id).await?;
let transmit_progress_signal = Arc::new(transmit_progress_signal);
let mut last_round_info: Option<RoundInfo> = None;
// loop for each "Round", consider each file in the partition
// for partitions with a lot of compaction work to do, keeping the work divided into multiple rounds,
@ -246,6 +247,7 @@ async fn try_compact_partition(
.round_info_source
.calculate(
Arc::<Components>::clone(&components),
last_round_info,
&partition_info,
files,
)
@ -292,6 +294,7 @@ async fn try_compact_partition(
.await?;
files.extend(branches_output.into_iter().flatten());
last_round_info = Some(round_info);
}
}

38
compactor/src/round_info.rs
View File

@ -2,7 +2,7 @@
use std::fmt::Display;
use data_types::CompactionLevel;
use data_types::{CompactionLevel, FileRange};
/// Information about the current compaction round (see driver.rs for
/// more details about a round)
@ -52,6 +52,17 @@ pub enum RoundInfo {
/// need split.
split_times: Vec<i64>,
},
/// CompactRanges are overlapping chains of L0s are less than max_compact_size, with no L0 or L1 overlaps
/// between ranges.
CompactRanges {
/// Ranges describing distinct chains of L0s to be compacted.
ranges: Vec<FileRange>,
/// max number of files to group in each plan
max_num_files_to_group: usize,
/// max total size limit of files to group in each plan
max_total_file_size_to_group: usize,
},
}
impl Display for RoundInfo {
@ -68,6 +79,7 @@ impl Display for RoundInfo {
max_total_file_size_to_group,
} => write!(f, "SimulatedLeadingEdge: {max_num_files_to_group}, {max_total_file_size_to_group}",),
Self::VerticalSplit { split_times } => write!(f, "VerticalSplit: {split_times:?}"),
Self::CompactRanges { ranges, max_num_files_to_group, max_total_file_size_to_group } => write!(f, "{:?}, {max_num_files_to_group}, {max_total_file_size_to_group}", ranges)
}
}
}
@ -81,6 +93,7 @@ impl RoundInfo {
Self::ManySmallFiles { start_level, .. } => *start_level,
Self::SimulatedLeadingEdge { .. } => CompactionLevel::FileNonOverlapped,
Self::VerticalSplit { .. } => CompactionLevel::Initial,
Self::CompactRanges { .. } => CompactionLevel::Initial,
}
}
@ -107,6 +120,10 @@ impl RoundInfo {
..
} => Some(*max_num_files_to_group),
Self::VerticalSplit { .. } => None,
Self::CompactRanges {
max_num_files_to_group,
..
} => Some(*max_num_files_to_group),
}
}
@ -123,6 +140,25 @@ impl RoundInfo {
..
} => Some(*max_total_file_size_to_group),
Self::VerticalSplit { .. } => None,
Self::CompactRanges {
max_total_file_size_to_group,
..
} => Some(*max_total_file_size_to_group),
}
}
/// return compaction ranges, when available.
/// We could generate ranges from VerticalSplit split times, but that asssumes the splits resulted in
/// no ranges > max_compact_size, which is not guaranteed. Instead, we'll detect the ranges the first
/// time after VerticalSplit, and may decide to resplit subset of the files again if data was more
/// non-linear than expected.
pub fn ranges(&self) -> Option<Vec<FileRange>> {
match self {
Self::TargetLevel { .. } => None,
Self::ManySmallFiles { .. } => None,
Self::SimulatedLeadingEdge { .. } => None,
Self::VerticalSplit { .. } => None,
Self::CompactRanges { ranges, .. } => Some(ranges.clone()),
}
}
}

38
compactor/tests/integration.rs
View File

@ -56,9 +56,9 @@ async fn test_num_files_over_limit() {
setup.run_compact().await;
//
// read files and verify 2 files
// read files and verify 3 files
let files = setup.list_by_table_not_to_delete().await;
assert_eq!(files.len(), 2);
assert_eq!(files.len(), 3);
//
// verify ID and compaction level of the files
@ -68,8 +68,9 @@ async fn test_num_files_over_limit() {
assert_levels(
&files,
vec![
(7, CompactionLevel::FileNonOverlapped),
(8, CompactionLevel::FileNonOverlapped),
(9, CompactionLevel::FileNonOverlapped),
(10, CompactionLevel::FileNonOverlapped),
],
);
}
@ -129,7 +130,7 @@ async fn test_compact_target_level() {
// This is the result of 2-round compaction fomr L0s -> L1s and then L1s -> L2s
// The first round will create two L1 files IDs 7 and 8
// The second round will create tow L2 file IDs 9 and 10
vec![(9, CompactionLevel::Final), (10, CompactionLevel::Final)],
vec![(10, CompactionLevel::Final), (11, CompactionLevel::Final)],
);
assert_max_l0_created_at(
@ -137,8 +138,8 @@ async fn test_compact_target_level() {
// both files have max_l0_created time_5_minutes_future
// which is the max of all L0 input's max_l0_created_at
vec![
(9, times.time_5_minutes_future),
(10, times.time_5_minutes_future),
(11, times.time_5_minutes_future),
],
);
@ -229,13 +230,14 @@ async fn test_compact_large_overlapes() {
---
- initial
- "L2 "
- "L2.6[6000,36000] 300s 3kb|-------L2.6-------| "
- "L2.7[68000,68000] 300s 2kb |L2.7| "
- "L2.8[136000,136000] 300s 3kb |L2.8|"
- "L2.5[136000,136000] 300s 2kb |L2.5|"
- "L2.6[6000,30000] 240s 3kb|-----L2.6-----| "
- "L2.7[36000,36000] 240s 3kb |L2.7| "
- "L2.8[68000,68000] 240s 2kb |L2.8| "
"###
);
assert_eq!(files.len(), 3);
assert_eq!(files.len(), 4);
// order files on their min_time
files.sort_by_key(|f| f.min_time);
@ -253,7 +255,6 @@ async fn test_compact_large_overlapes() {
"| 1500 | WA | | | 1970-01-01T00:00:00.000008Z |",
"| 1601 | | PA | 15 | 1970-01-01T00:00:00.000028Z |",
"| 1601 | | PA | 15 | 1970-01-01T00:00:00.000030Z |",
"| 21 | | OH | 21 | 1970-01-01T00:00:00.000036Z |",
"| 270 | UT | | | 1970-01-01T00:00:00.000025Z |",
"| 70 | UT | | | 1970-01-01T00:00:00.000020Z |",
"| 99 | OR | | | 1970-01-01T00:00:00.000012Z |",
@ -270,7 +271,7 @@ async fn test_compact_large_overlapes() {
"+-----------+------+------+------+-----------------------------+",
"| field_int | tag1 | tag2 | tag3 | time |",
"+-----------+------+------+------+-----------------------------+",
"| 10 | VT | | | 1970-01-01T00:00:00.000068Z |",
"| 21 | | OH | 21 | 1970-01-01T00:00:00.000036Z |",
"+-----------+------+------+------+-----------------------------+",
],
&batches
@ -284,11 +285,24 @@ async fn test_compact_large_overlapes() {
"+-----------+------+------+------+-----------------------------+",
"| field_int | tag1 | tag2 | tag3 | time |",
"+-----------+------+------+------+-----------------------------+",
"| 210 | | OH | 21 | 1970-01-01T00:00:00.000136Z |",
"| 10 | VT | | | 1970-01-01T00:00:00.000068Z |",
"+-----------+------+------+------+-----------------------------+",
],
&batches
);
let file = files[3].clone();
let batches = setup.read_parquet_file(file).await;
assert_batches_sorted_eq!(
&[
"+-----------+------+------+-----------------------------+",
"| field_int | tag2 | tag3 | time |",
"+-----------+------+------+-----------------------------+",
"| 210 | OH | 21 | 1970-01-01T00:00:00.000136Z |",
"+-----------+------+------+-----------------------------+",
],
&batches
);
}
#[tokio::test]

1203
compactor/tests/layouts/backfill.rs
View File

File diff suppressed because it is too large Load Diff

142
compactor/tests/layouts/core.rs
View File

@ -167,21 +167,22 @@ async fn all_non_overlapping_l0() {
- "Committing partition 1:"
- " Soft Deleting 10 files: L0.1, L0.2, L0.3, L0.4, L0.5, L0.6, L0.7, L0.8, L0.9, L0.10"
- " Creating 2 files"
- "**** Simulation run 1, type=split(CompactAndSplitOutput(TotalSizeLessThanMaxCompactSize))(split_times=[720]). 2 Input Files, 100mb total:"
- "L1 "
- "L1.12[721,901] 9ns 20mb |-----L1.12-----| "
- "L1.11[0,720] 9ns 80mb |--------------------------------L1.11--------------------------------| "
- "**** 2 Output Files (parquet_file_id not yet assigned), 100mb total:"
- "**** Simulation run 1, type=split(CompactAndSplitOutput(TotalSizeLessThanMaxCompactSize))(split_times=[865]). 1 Input Files, 20mb total:"
- "L1, all files 20mb "
- "L1.12[721,901] 9ns |-----------------------------------------L1.12------------------------------------------|"
- "**** 2 Output Files (parquet_file_id not yet assigned), 20mb total:"
- "L2 "
- "L2.?[0,720] 9ns 80mb |--------------------------------L2.?---------------------------------| "
- "L2.?[721,901] 9ns 20mb |-----L2.?------| "
- "L2.?[721,865] 9ns 16mb |---------------------------------L2.?---------------------------------| "
- "L2.?[866,901] 9ns 4mb |-----L2.?------| "
- "Committing partition 1:"
- " Soft Deleting 2 files: L1.11, L1.12"
- " Soft Deleting 1 files: L1.12"
- " Upgrading 1 files level to CompactionLevel::L2: L1.11"
- " Creating 2 files"
- "**** Final Output Files (200mb written)"
- "**** Final Output Files (120mb written)"
- "L2 "
- "L2.13[0,720] 9ns 80mb |--------------------------------L2.13--------------------------------| "
- "L2.14[721,901] 9ns 20mb |-----L2.14-----| "
- "L2.11[0,720] 9ns 80mb |--------------------------------L2.11--------------------------------| "
- "L2.13[721,865] 9ns 16mb |---L2.13----| "
- "L2.14[866,901] 9ns 4mb |L2.14|"
"###
);
}
@ -417,10 +418,10 @@ async fn l1_with_non_overlapping_l0_larger() {
- " Creating 1 files"
- "**** Simulation run 1, type=split(CompactAndSplitOutput(TotalSizeLessThanMaxCompactSize))(split_times=[370]). 4 Input Files, 108mb total:"
- "L1 "
- "L1.2[100,149] 2ns 50mb |--L1.2---| "
- "L1.1[50,99] 1ns 20mb |--L1.1---| "
- "L1.3[150,199] 3ns 20mb |--L1.3---| "
- "L1.8[200,450] 13ns 18mb |-------------------------L1.8-------------------------| "
- "L1.2[100,149] 2ns 50mb |--L1.2---| "
- "**** 2 Output Files (parquet_file_id not yet assigned), 108mb total:"
- "L2 "
- "L2.?[50,370] 13ns 86mb |---------------------------------L2.?---------------------------------| "
@ -448,11 +449,11 @@ async fn l1_too_much_with_non_overlapping_l0() {
// enough to upgrade, the total size will be > 256MB and we will
// skip the partition
//
// L1: 90MB, 80MB, 70MB, ..., 70MB
// L1: 45MB, 40MB, 35MB, ..., 35MB
// L0: ..
let mut num_l1_files = 0;
for (i, sz) in [90, 80, 70, 70, 70, 70, 70, 70, 70, 70].iter().enumerate() {
for (i, sz) in [45, 40, 35, 35, 35, 35, 35, 35, 35, 35].iter().enumerate() {
let i = i as i64;
setup
.partition
@ -493,16 +494,16 @@ async fn l1_too_much_with_non_overlapping_l0() {
- "L0.12[600,649] 720s 5mb |L0.12| "
- "L0.13[600,649] 780s 5mb |L0.13| "
- "L1 "
- "L1.1[50,99] 0ns 90mb |L1.1-| "
- "L1.2[100,149] 60s 80mb |L1.2-| "
- "L1.3[150,199] 120s 70mb |L1.3-| "
- "L1.4[200,249] 180s 70mb |L1.4-| "
- "L1.5[250,299] 240s 70mb |L1.5-| "
- "L1.6[300,349] 300s 70mb |L1.6-| "
- "L1.7[350,399] 360s 70mb |L1.7-| "
- "L1.8[400,449] 420s 70mb |L1.8-| "
- "L1.9[450,499] 480s 70mb |L1.9-| "
- "L1.10[500,549] 540s 70mb |L1.10| "
- "L1.1[50,99] 0ns 45mb |L1.1-| "
- "L1.2[100,149] 60s 40mb |L1.2-| "
- "L1.3[150,199] 120s 35mb |L1.3-| "
- "L1.4[200,249] 180s 35mb |L1.4-| "
- "L1.5[250,299] 240s 35mb |L1.5-| "
- "L1.6[300,349] 300s 35mb |L1.6-| "
- "L1.7[350,399] 360s 35mb |L1.7-| "
- "L1.8[400,449] 420s 35mb |L1.8-| "
- "L1.9[450,499] 480s 35mb |L1.9-| "
- "L1.10[500,549] 540s 35mb |L1.10| "
- "**** Simulation run 0, type=compact(TotalSizeLessThanMaxCompactSize). 3 Input Files, 15mb total:"
- "L0, all files 5mb "
- "L0.13[600,649] 780s |-----------------------------------------L0.13------------------------------------------|"
@ -514,58 +515,33 @@ async fn l1_too_much_with_non_overlapping_l0() {
- "Committing partition 1:"
- " Soft Deleting 3 files: L0.11, L0.12, L0.13"
- " Creating 1 files"
- "**** Simulation run 1, type=split(CompactAndSplitOutput(FoundSubsetLessThanMaxCompactSize))(split_times=[113, 176]). 3 Input Files, 240mb total:"
- "**** Simulation run 1, type=split(CompactAndSplitOutput(FoundSubsetLessThanMaxCompactSize))(split_times=[186, 322]). 8 Input Files, 295mb total:"
- "L1 "
- "L1.1[50,99] 0ns 90mb |-----------L1.1------------| "
- "L1.2[100,149] 60s 80mb |-----------L1.2------------| "
- "L1.3[150,199] 120s 70mb |-----------L1.3------------| "
- "**** 3 Output Files (parquet_file_id not yet assigned), 240mb total:"
- "L1.1[50,99] 0ns 45mb |--L1.1---| "
- "L1.2[100,149] 60s 40mb |--L1.2---| "
- "L1.3[150,199] 120s 35mb |--L1.3---| "
- "L1.4[200,249] 180s 35mb |--L1.4---| "
- "L1.5[250,299] 240s 35mb |--L1.5---| "
- "L1.6[300,349] 300s 35mb |--L1.6---| "
- "L1.7[350,399] 360s 35mb |--L1.7---| "
- "L1.8[400,449] 420s 35mb |--L1.8---| "
- "**** 3 Output Files (parquet_file_id not yet assigned), 295mb total:"
- "L2 "
- "L2.?[50,113] 120s 102mb |----------------L2.?----------------| "
- "L2.?[114,176] 120s 101mb |---------------L2.?----------------| "
- "L2.?[177,199] 120s 37mb |---L2.?----| "
- "L2.?[50,186] 420s 101mb |------------L2.?------------| "
- "L2.?[187,322] 420s 100mb |------------L2.?------------| "
- "L2.?[323,449] 420s 94mb |-----------L2.?-----------| "
- "Committing partition 1:"
- " Soft Deleting 3 files: L1.1, L1.2, L1.3"
- " Soft Deleting 8 files: L1.1, L1.2, L1.3, L1.4, L1.5, L1.6, L1.7, L1.8"
- " Creating 3 files"
- "**** Simulation run 2, type=split(CompactAndSplitOutput(FoundSubsetLessThanMaxCompactSize))(split_times=[272, 344]). 4 Input Files, 280mb total:"
- "L1, all files 70mb "
- "L1.4[200,249] 180s |--------L1.4--------| "
- "L1.5[250,299] 240s |--------L1.5--------| "
- "L1.6[300,349] 300s |--------L1.6--------| "
- "L1.7[350,399] 360s |--------L1.7--------| "
- "**** 3 Output Files (parquet_file_id not yet assigned), 280mb total:"
- "L2 "
- "L2.?[200,272] 360s 102mb |-------------L2.?-------------| "
- "L2.?[273,344] 360s 101mb |-------------L2.?-------------| "
- "L2.?[345,399] 360s 77mb |---------L2.?---------| "
- "Committing partition 1:"
- " Soft Deleting 4 files: L1.4, L1.5, L1.6, L1.7"
- " Creating 3 files"
- "**** Simulation run 3, type=split(CompactAndSplitOutput(TotalSizeLessThanMaxCompactSize))(split_times=[511, 622]). 4 Input Files, 225mb total:"
- "**** Final Output Files (310mb written)"
- "L1 "
- "L1.14[600,649] 780s 15mb |-----L1.14-----| "
- "L1.10[500,549] 540s 70mb |-----L1.10-----| "
- "L1.9[450,499] 480s 70mb |-----L1.9------| "
- "L1.8[400,449] 420s 70mb |-----L1.8------| "
- "**** 3 Output Files (parquet_file_id not yet assigned), 225mb total:"
- "L1.9[450,499] 480s 35mb |L1.9-| "
- "L1.10[500,549] 540s 35mb |L1.10| "
- "L1.14[600,649] 780s 15mb |L1.14| "
- "L2 "
- "L2.?[400,511] 780s 101mb |-----------------L2.?-----------------| "
- "L2.?[512,622] 780s 100mb |----------------L2.?-----------------| "
- "L2.?[623,649] 780s 24mb |-L2.?--| "
- "Committing partition 1:"
- " Soft Deleting 4 files: L1.8, L1.9, L1.10, L1.14"
- " Creating 3 files"
- "**** Final Output Files (760mb written)"
- "L2 "
- "L2.15[50,113] 120s 102mb |-L2.15-| "
- "L2.16[114,176] 120s 101mb |-L2.16-| "
- "L2.17[177,199] 120s 37mb |L2.17| "
- "L2.18[200,272] 360s 102mb |-L2.18--| "
- "L2.19[273,344] 360s 101mb |-L2.19--| "
- "L2.20[345,399] 360s 77mb |L2.20-| "
- "L2.21[400,511] 780s 101mb |----L2.21-----| "
- "L2.22[512,622] 780s 100mb |----L2.22-----| "
- "L2.23[623,649] 780s 24mb |L2.23|"
- "L2.15[50,186] 420s 101mb |------L2.15-------| "
- "L2.16[187,322] 420s 100mb |------L2.16-------| "
- "L2.17[323,449] 420s 94mb |-----L2.17------| "
"###
);
}
@ -682,10 +658,10 @@ async fn large_l1_with_non_overlapping_l0() {
// L1 files with total size > 100MB will get compacted after in round 2
// after the L0 files are compacted in round 1
// L1: 90MB, 80MB
// L1: 45MB, 40MB
// L0: ..
for (i, sz) in [90, 80].iter().enumerate() {
for (i, sz) in [45, 40].iter().enumerate() {
let i = i as i64;
setup
.partition
@ -727,8 +703,8 @@ async fn large_l1_with_non_overlapping_l0() {
- "L0.4[600,650] 22ns 5mb |L0.4-| "
- "L0.5[600,650] 23ns 5mb |L0.5-| "
- "L1 "
- "L1.1[50,99] 1ns 90mb |L1.1-| "
- "L1.2[100,149] 2ns 80mb |L1.2-| "
- "L1.1[50,99] 1ns 45mb |L1.1-| "
- "L1.2[100,149] 2ns 40mb |L1.2-| "
- "**** Simulation run 0, type=compact(TotalSizeLessThanMaxCompactSize). 3 Input Files, 15mb total:"
- "L0, all files 5mb "
- "L0.5[600,650] 23ns |------------------------------------------L0.5------------------------------------------|"
@ -740,22 +716,22 @@ async fn large_l1_with_non_overlapping_l0() {
- "Committing partition 1:"
- " Soft Deleting 3 files: L0.3, L0.4, L0.5"
- " Creating 1 files"
- "**** Simulation run 1, type=split(CompactAndSplitOutput(TotalSizeLessThanMaxCompactSize))(split_times=[375]). 3 Input Files, 185mb total:"
- "**** Simulation run 1, type=split(CompactAndSplitOutput(TotalSizeLessThanMaxCompactSize))(split_times=[530]). 3 Input Files, 100mb total:"
- "L1 "
- "L1.1[50,99] 1ns 90mb |L1.1-| "
- "L1.2[100,149] 2ns 80mb |L1.2-| "
- "L1.1[50,99] 1ns 45mb |L1.1-| "
- "L1.2[100,149] 2ns 40mb |L1.2-| "
- "L1.6[600,650] 23ns 15mb |L1.6-| "
- "**** 2 Output Files (parquet_file_id not yet assigned), 185mb total:"
- "**** 2 Output Files (parquet_file_id not yet assigned), 100mb total:"
- "L2 "
- "L2.?[50,375] 23ns 100mb |---------------------L2.?---------------------| "
- "L2.?[376,650] 23ns 85mb |-----------------L2.?------------------| "
- "L2.?[50,530] 23ns 80mb |---------------------------------L2.?---------------------------------| "
- "L2.?[531,650] 23ns 20mb |-----L2.?------| "
- "Committing partition 1:"
- " Soft Deleting 3 files: L1.1, L1.2, L1.6"
- " Creating 2 files"
- "**** Final Output Files (200mb written)"
- "**** Final Output Files (115mb written)"
- "L2 "
- "L2.7[50,375] 23ns 100mb |---------------------L2.7---------------------| "
- "L2.8[376,650] 23ns 85mb |-----------------L2.8------------------| "
- "L2.7[50,530] 23ns 80mb |---------------------------------L2.7---------------------------------| "
- "L2.8[531,650] 23ns 20mb |-----L2.8------| "
"###
);
}

2728
compactor/tests/layouts/knobs.rs
View File

File diff suppressed because it is too large Load Diff

28
compactor/tests/layouts/large_files.rs
View File

@ -645,7 +645,7 @@ async fn two_large_files_total_over_max_compact_size_start_l0() {
- "Committing partition 1:"
- " Soft Deleting 2 files: L0.1, L1.2"
- " Creating 4 files"
- "**** Simulation run 2, type=split(CompactAndSplitOutput(FoundSubsetLessThanMaxCompactSize))(split_times=[334]). 2 Input Files, 200mb total:"
- "**** Simulation run 2, type=split(CompactAndSplitOutput(TotalSizeLessThanMaxCompactSize))(split_times=[334]). 2 Input Files, 200mb total:"
- "L0 "
- "L0.3[0,667] 10ns 100mb |------------------------------------------L0.3------------------------------------------|"
- "L1 "
@ -669,24 +669,24 @@ async fn two_large_files_total_over_max_compact_size_start_l0() {
- "Committing partition 1:"
- " Soft Deleting 2 files: L0.4, L1.6"
- " Creating 2 files"
- "**** Simulation run 4, type=split(CompactAndSplitOutput(TotalSizeLessThanMaxCompactSize))(split_times=[669]). 3 Input Files, 200mb total:"
- "L1 "
- "L1.10[934,1000] 10ns 20mb |L1.10-| "
- "L1.9[668,933] 10ns 80mb |--------------L1.9---------------| "
- "L1.8[335,667] 10ns 100mb |-------------------L1.8-------------------| "
- "**** 2 Output Files (parquet_file_id not yet assigned), 200mb total:"
- "**** Simulation run 4, type=split(CompactAndSplitOutput(TotalSizeLessThanMaxCompactSize))(split_times=[986]). 1 Input Files, 20mb total:"
- "L1, all files 20mb "
- "L1.10[934,1000] 10ns |-----------------------------------------L1.10------------------------------------------|"
- "**** 2 Output Files (parquet_file_id not yet assigned), 20mb total:"
- "L2 "
- "L2.?[335,669] 10ns 100mb |-------------------L2.?--------------------| "
- "L2.?[670,1000] 10ns 99mb |-------------------L2.?-------------------| "
- "L2.?[934,986] 10ns 16mb |--------------------------------L2.?--------------------------------| "
- "L2.?[987,1000] 10ns 4mb |-----L2.?------| "
- "Committing partition 1:"
- " Soft Deleting 3 files: L1.8, L1.9, L1.10"
- " Upgrading 1 files level to CompactionLevel::L2: L1.7"
- " Soft Deleting 1 files: L1.10"
- " Upgrading 3 files level to CompactionLevel::L2: L1.7, L1.8, L1.9"
- " Creating 2 files"
- "**** Final Output Files (800mb written)"
- "**** Final Output Files (620mb written)"
- "L2 "
- "L2.7[0,334] 10ns 100mb |------------L2.7------------| "
- "L2.11[335,669] 10ns 100mb |-----------L2.11------------| "
- "L2.12[670,1000] 10ns 99mb |-----------L2.12-----------| "
- "L2.8[335,667] 10ns 100mb |-----------L2.8------------| "
- "L2.9[668,933] 10ns 80mb |--------L2.9---------| "
- "L2.11[934,986] 10ns 16mb |L2.11|"
- "L2.12[987,1000] 10ns 4mb |L2.12|"
"###
);

661
compactor/tests/layouts/large_overlaps.rs
View File

@ -581,362 +581,333 @@ async fn many_good_size_l0_files() {
- "Committing partition 1:"
- " Soft Deleting 2 files: L0.96, L0.191"
- " Creating 4 files"
- "**** Simulation run 2, type=split(CompactAndSplitOutput(FoundSubsetLessThanMaxCompactSize))(split_times=[50, 100]). 151 Input Files, 300mb total:"
- "**** Simulation run 2, type=split(CompactAndSplitOutput(TotalSizeLessThanMaxCompactSize))(split_times=[50]). 96 Input Files, 191mb total:"
- "L0 "
- "L0.1[0,1] 1ns 2mb |L0.1| "
- "L0.2[1,2] 2ns 2mb |L0.2| "
- "L0.289[95,95] 96ns 1mb |L0.289|"
- "L0.95[94,95] 95ns 2mb |L0.95|"
- "L0.94[93,94] 94ns 2mb |L0.94|"
- "L0.93[92,93] 93ns 2mb |L0.93|"
- "L0.92[91,92] 92ns 2mb |L0.92|"
- "L0.91[90,91] 91ns 2mb |L0.91|"
- "L0.90[89,90] 90ns 2mb |L0.90|"
- "L0.89[88,89] 89ns 2mb |L0.89|"
- "L0.88[87,88] 88ns 2mb |L0.88| "
- "L0.87[86,87] 87ns 2mb |L0.87| "
- "L0.86[85,86] 86ns 2mb |L0.86| "
- "L0.85[84,85] 85ns 2mb |L0.85| "
- "L0.84[83,84] 84ns 2mb |L0.84| "
- "L0.83[82,83] 83ns 2mb |L0.83| "
- "L0.82[81,82] 82ns 2mb |L0.82| "
- "L0.81[80,81] 81ns 2mb |L0.81| "
- "L0.80[79,80] 80ns 2mb |L0.80| "
- "L0.79[78,79] 79ns 2mb |L0.79| "
- "L0.78[77,78] 78ns 2mb |L0.78| "
- "L0.77[76,77] 77ns 2mb |L0.77| "
- "L0.76[75,76] 76ns 2mb |L0.76| "
- "L0.75[74,75] 75ns 2mb |L0.75| "
- "L0.74[73,74] 74ns 2mb |L0.74| "
- "L0.73[72,73] 73ns 2mb |L0.73| "
- "L0.72[71,72] 72ns 2mb |L0.72| "
- "L0.71[70,71] 71ns 2mb |L0.71| "
- "L0.70[69,70] 70ns 2mb |L0.70| "
- "L0.69[68,69] 69ns 2mb |L0.69| "
- "L0.68[67,68] 68ns 2mb |L0.68| "
- "L0.67[66,67] 67ns 2mb |L0.67| "
- "L0.66[65,66] 66ns 2mb |L0.66| "
- "L0.65[64,65] 65ns 2mb |L0.65| "
- "L0.64[63,64] 64ns 2mb |L0.64| "
- "L0.63[62,63] 63ns 2mb |L0.63| "
- "L0.62[61,62] 62ns 2mb |L0.62| "
- "L0.61[60,61] 61ns 2mb |L0.61| "
- "L0.60[59,60] 60ns 2mb |L0.60| "
- "L0.59[58,59] 59ns 2mb |L0.59| "
- "L0.58[57,58] 58ns 2mb |L0.58| "
- "L0.57[56,57] 57ns 2mb |L0.57| "
- "L0.56[55,56] 56ns 2mb |L0.56| "
- "L0.55[54,55] 55ns 2mb |L0.55| "
- "L0.54[53,54] 54ns 2mb |L0.54| "
- "L0.53[52,53] 53ns 2mb |L0.53| "
- "L0.52[51,52] 52ns 2mb |L0.52| "
- "L0.51[50,51] 51ns 2mb |L0.51| "
- "L0.50[49,50] 50ns 2mb |L0.50| "
- "L0.49[48,49] 49ns 2mb |L0.49| "
- "L0.48[47,48] 48ns 2mb |L0.48| "
- "L0.47[46,47] 47ns 2mb |L0.47| "
- "L0.46[45,46] 46ns 2mb |L0.46| "
- "L0.45[44,45] 45ns 2mb |L0.45| "
- "L0.44[43,44] 44ns 2mb |L0.44| "
- "L0.43[42,43] 43ns 2mb |L0.43| "
- "L0.42[41,42] 42ns 2mb |L0.42| "
- "L0.41[40,41] 41ns 2mb |L0.41| "
- "L0.40[39,40] 40ns 2mb |L0.40| "
- "L0.39[38,39] 39ns 2mb |L0.39| "
- "L0.38[37,38] 38ns 2mb |L0.38| "
- "L0.37[36,37] 37ns 2mb |L0.37| "
- "L0.36[35,36] 36ns 2mb |L0.36| "
- "L0.35[34,35] 35ns 2mb |L0.35| "
- "L0.34[33,34] 34ns 2mb |L0.34| "
- "L0.33[32,33] 33ns 2mb |L0.33| "
- "L0.32[31,32] 32ns 2mb |L0.32| "
- "L0.31[30,31] 31ns 2mb |L0.31| "
- "L0.30[29,30] 30ns 2mb |L0.30| "
- "L0.29[28,29] 29ns 2mb |L0.29| "
- "L0.28[27,28] 28ns 2mb |L0.28| "
- "L0.27[26,27] 27ns 2mb |L0.27| "
- "L0.26[25,26] 26ns 2mb |L0.26| "
- "L0.25[24,25] 25ns 2mb |L0.25| "
- "L0.24[23,24] 24ns 2mb |L0.24| "
- "L0.23[22,23] 23ns 2mb |L0.23| "
- "L0.22[21,22] 22ns 2mb |L0.22| "
- "L0.21[20,21] 21ns 2mb |L0.21| "
- "L0.20[19,20] 20ns 2mb |L0.20| "
- "L0.19[18,19] 19ns 2mb |L0.19| "
- "L0.18[17,18] 18ns 2mb |L0.18| "
- "L0.17[16,17] 17ns 2mb |L0.17| "
- "L0.16[15,16] 16ns 2mb |L0.16| "
- "L0.15[14,15] 15ns 2mb |L0.15| "
- "L0.14[13,14] 14ns 2mb |L0.14| "
- "L0.13[12,13] 13ns 2mb |L0.13| "
- "L0.12[11,12] 12ns 2mb |L0.12| "
- "L0.11[10,11] 11ns 2mb |L0.11| "
- "L0.10[9,10] 10ns 2mb |L0.10| "
- "L0.9[8,9] 9ns 2mb |L0.9| "
- "L0.8[7,8] 8ns 2mb |L0.8| "
- "L0.7[6,7] 7ns 2mb |L0.7| "
- "L0.6[5,6] 6ns 2mb |L0.6| "
- "L0.5[4,5] 5ns 2mb |L0.5| "
- "L0.4[3,4] 4ns 2mb |L0.4| "
- "L0.3[2,3] 3ns 2mb |L0.3| "
- "L0.4[3,4] 4ns 2mb |L0.4| "
- "L0.5[4,5] 5ns 2mb |L0.5| "
- "L0.6[5,6] 6ns 2mb |L0.6| "
- "L0.7[6,7] 7ns 2mb |L0.7| "
- "L0.8[7,8] 8ns 2mb |L0.8| "
- "L0.9[8,9] 9ns 2mb |L0.9| "
- "L0.10[9,10] 10ns 2mb |L0.10| "
- "L0.11[10,11] 11ns 2mb |L0.11| "
- "L0.12[11,12] 12ns 2mb |L0.12| "
- "L0.13[12,13] 13ns 2mb |L0.13| "
- "L0.14[13,14] 14ns 2mb |L0.14| "
- "L0.15[14,15] 15ns 2mb |L0.15| "
- "L0.16[15,16] 16ns 2mb |L0.16| "
- "L0.17[16,17] 17ns 2mb |L0.17| "
- "L0.18[17,18] 18ns 2mb |L0.18| "
- "L0.19[18,19] 19ns 2mb |L0.19| "
- "L0.20[19,20] 20ns 2mb |L0.20| "
- "L0.21[20,21] 21ns 2mb |L0.21| "
- "L0.22[21,22] 22ns 2mb |L0.22| "
- "L0.23[22,23] 23ns 2mb |L0.23| "
- "L0.24[23,24] 24ns 2mb |L0.24| "
- "L0.25[24,25] 25ns 2mb |L0.25| "
- "L0.26[25,26] 26ns 2mb |L0.26| "
- "L0.27[26,27] 27ns 2mb |L0.27| "
- "L0.28[27,28] 28ns 2mb |L0.28| "
- "L0.29[28,29] 29ns 2mb |L0.29| "
- "L0.30[29,30] 30ns 2mb |L0.30| "
- "L0.31[30,31] 31ns 2mb |L0.31| "
- "L0.32[31,32] 32ns 2mb |L0.32| "
- "L0.33[32,33] 33ns 2mb |L0.33| "
- "L0.34[33,34] 34ns 2mb |L0.34| "
- "L0.35[34,35] 35ns 2mb |L0.35| "
- "L0.36[35,36] 36ns 2mb |L0.36| "
- "L0.37[36,37] 37ns 2mb |L0.37| "
- "L0.38[37,38] 38ns 2mb |L0.38| "
- "L0.39[38,39] 39ns 2mb |L0.39| "
- "L0.40[39,40] 40ns 2mb |L0.40| "
- "L0.41[40,41] 41ns 2mb |L0.41| "
- "L0.42[41,42] 42ns 2mb |L0.42| "
- "L0.43[42,43] 43ns 2mb |L0.43| "
- "L0.44[43,44] 44ns 2mb |L0.44| "
- "L0.45[44,45] 45ns 2mb |L0.45| "
- "L0.46[45,46] 46ns 2mb |L0.46| "
- "L0.47[46,47] 47ns 2mb |L0.47| "
- "L0.48[47,48] 48ns 2mb |L0.48| "
- "L0.49[48,49] 49ns 2mb |L0.49| "
- "L0.50[49,50] 50ns 2mb |L0.50| "
- "L0.51[50,51] 51ns 2mb |L0.51| "
- "L0.52[51,52] 52ns 2mb |L0.52| "
- "L0.53[52,53] 53ns 2mb |L0.53| "
- "L0.54[53,54] 54ns 2mb |L0.54| "
- "L0.55[54,55] 55ns 2mb |L0.55| "
- "L0.56[55,56] 56ns 2mb |L0.56| "
- "L0.57[56,57] 57ns 2mb |L0.57| "
- "L0.58[57,58] 58ns 2mb |L0.58| "
- "L0.59[58,59] 59ns 2mb |L0.59| "
- "L0.60[59,60] 60ns 2mb |L0.60| "
- "L0.61[60,61] 61ns 2mb |L0.61| "
- "L0.62[61,62] 62ns 2mb |L0.62| "
- "L0.63[62,63] 63ns 2mb |L0.63| "
- "L0.64[63,64] 64ns 2mb |L0.64| "
- "L0.65[64,65] 65ns 2mb |L0.65| "
- "L0.66[65,66] 66ns 2mb |L0.66| "
- "L0.67[66,67] 67ns 2mb |L0.67| "
- "L0.68[67,68] 68ns 2mb |L0.68| "
- "L0.69[68,69] 69ns 2mb |L0.69| "
- "L0.70[69,70] 70ns 2mb |L0.70| "
- "L0.71[70,71] 71ns 2mb |L0.71| "
- "L0.72[71,72] 72ns 2mb |L0.72| "
- "L0.73[72,73] 73ns 2mb |L0.73| "
- "L0.74[73,74] 74ns 2mb |L0.74| "
- "L0.75[74,75] 75ns 2mb |L0.75| "
- "L0.76[75,76] 76ns 2mb |L0.76| "
- "L0.77[76,77] 77ns 2mb |L0.77| "
- "L0.78[77,78] 78ns 2mb |L0.78| "
- "L0.79[78,79] 79ns 2mb |L0.79| "
- "L0.80[79,80] 80ns 2mb |L0.80| "
- "L0.81[80,81] 81ns 2mb |L0.81| "
- "L0.82[81,82] 82ns 2mb |L0.82| "
- "L0.83[82,83] 83ns 2mb |L0.83| "
- "L0.84[83,84] 84ns 2mb |L0.84| "
- "L0.85[84,85] 85ns 2mb |L0.85| "
- "L0.86[85,86] 86ns 2mb |L0.86| "
- "L0.87[86,87] 87ns 2mb |L0.87| "
- "L0.88[87,88] 88ns 2mb |L0.88| "
- "L0.89[88,89] 89ns 2mb |L0.89| "
- "L0.90[89,90] 90ns 2mb |L0.90| "
- "L0.91[90,91] 91ns 2mb |L0.91| "
- "L0.92[91,92] 92ns 2mb |L0.92| "
- "L0.93[92,93] 93ns 2mb |L0.93| "
- "L0.94[93,94] 94ns 2mb |L0.94| "
- "L0.95[94,95] 95ns 2mb |L0.95| "
- "L0.289[95,95] 96ns 1mb |L0.289| "
- "L0.290[96,96] 96ns 1mb |L0.290| "
- "L0.97[96,97] 97ns 2mb |L0.97| "
- "L0.98[97,98] 98ns 2mb |L0.98| "
- "L0.99[98,99] 99ns 2mb |L0.99| "
- "L0.100[99,100] 100ns 2mb |L0.100| "
- "L0.101[100,101] 101ns 2mb |L0.101| "
- "L0.102[101,102] 102ns 2mb |L0.102| "
- "L0.103[102,103] 103ns 2mb |L0.103| "
- "L0.104[103,104] 104ns 2mb |L0.104| "
- "L0.105[104,105] 105ns 2mb |L0.105| "
- "L0.106[105,106] 106ns 2mb |L0.106| "
- "L0.107[106,107] 107ns 2mb |L0.107| "
- "L0.108[107,108] 108ns 2mb |L0.108| "
- "L0.109[108,109] 109ns 2mb |L0.109| "
- "L0.110[109,110] 110ns 2mb |L0.110| "
- "L0.111[110,111] 111ns 2mb |L0.111| "
- "L0.112[111,112] 112ns 2mb |L0.112| "
- "L0.113[112,113] 113ns 2mb |L0.113| "
- "L0.114[113,114] 114ns 2mb |L0.114| "
- "L0.115[114,115] 115ns 2mb |L0.115| "
- "L0.116[115,116] 116ns 2mb |L0.116| "
- "L0.117[116,117] 117ns 2mb |L0.117| "
- "L0.118[117,118] 118ns 2mb |L0.118| "
- "L0.119[118,119] 119ns 2mb |L0.119| "
- "L0.120[119,120] 120ns 2mb |L0.120| "
- "L0.121[120,121] 121ns 2mb |L0.121| "
- "L0.122[121,122] 122ns 2mb |L0.122| "
- "L0.123[122,123] 123ns 2mb |L0.123| "
- "L0.124[123,124] 124ns 2mb |L0.124| "
- "L0.125[124,125] 125ns 2mb |L0.125| "
- "L0.126[125,126] 126ns 2mb |L0.126| "
- "L0.127[126,127] 127ns 2mb |L0.127| "
- "L0.128[127,128] 128ns 2mb |L0.128| "
- "L0.129[128,129] 129ns 2mb |L0.129| "
- "L0.130[129,130] 130ns 2mb |L0.130| "
- "L0.131[130,131] 131ns 2mb |L0.131| "
- "L0.132[131,132] 132ns 2mb |L0.132| "
- "L0.133[132,133] 133ns 2mb |L0.133| "
- "L0.134[133,134] 134ns 2mb |L0.134| "
- "L0.135[134,135] 135ns 2mb |L0.135| "
- "L0.136[135,136] 136ns 2mb |L0.136| "
- "L0.137[136,137] 137ns 2mb |L0.137| "
- "L0.138[137,138] 138ns 2mb |L0.138|"
- "L0.139[138,139] 139ns 2mb |L0.139|"
- "L0.140[139,140] 140ns 2mb |L0.140|"
- "L0.141[140,141] 141ns 2mb |L0.141|"
- "L0.142[141,142] 142ns 2mb |L0.142|"
- "L0.143[142,143] 143ns 2mb |L0.143|"
- "L0.144[143,144] 144ns 2mb |L0.144|"
- "L0.145[144,145] 145ns 2mb |L0.145|"
- "L0.146[145,146] 146ns 2mb |L0.146|"
- "L0.147[146,147] 147ns 2mb |L0.147|"
- "L0.148[147,148] 148ns 2mb |L0.148|"
- "L0.149[148,149] 149ns 2mb |L0.149|"
- "L0.150[149,150] 150ns 2mb |L0.150|"
- "**** 3 Output Files (parquet_file_id not yet assigned), 300mb total:"
- "L0.2[1,2] 2ns 2mb |L0.2| "
- "L0.1[0,1] 1ns 2mb |L0.1| "
- "**** 2 Output Files (parquet_file_id not yet assigned), 191mb total:"
- "L1 "
- "L1.?[0,50] 150ns 101mb |------------L1.?------------| "
- "L1.?[51,100] 150ns 99mb |-----------L1.?------------| "
- "L1.?[101,150] 150ns 99mb |-----------L1.?------------| "
- "L1.?[0,50] 96ns 101mb |--------------------L1.?---------------------| "
- "L1.?[51,95] 96ns 90mb |-----------------L1.?------------------| "
- "Committing partition 1:"
- " Soft Deleting 151 files: L0.1, L0.2, L0.3, L0.4, L0.5, L0.6, L0.7, L0.8, L0.9, L0.10, L0.11, L0.12, L0.13, L0.14, L0.15, L0.16, L0.17, L0.18, L0.19, L0.20, L0.21, L0.22, L0.23, L0.24, L0.25, L0.26, L0.27, L0.28, L0.29, L0.30, L0.31, L0.32, L0.33, L0.34, L0.35, L0.36, L0.37, L0.38, L0.39, L0.40, L0.41, L0.42, L0.43, L0.44, L0.45, L0.46, L0.47, L0.48, L0.49, L0.50, L0.51, L0.52, L0.53, L0.54, L0.55, L0.56, L0.57, L0.58, L0.59, L0.60, L0.61, L0.62, L0.63, L0.64, L0.65, L0.66, L0.67, L0.68, L0.69, L0.70, L0.71, L0.72, L0.73, L0.74, L0.75, L0.76, L0.77, L0.78, L0.79, L0.80, L0.81, L0.82, L0.83, L0.84, L0.85, L0.86, L0.87, L0.88, L0.89, L0.90, L0.91, L0.92, L0.93, L0.94, L0.95, L0.97, L0.98, L0.99, L0.100, L0.101, L0.102, L0.103, L0.104, L0.105, L0.106, L0.107, L0.108, L0.109, L0.110, L0.111, L0.112, L0.113, L0.114, L0.115, L0.116, L0.117, L0.118, L0.119, L0.120, L0.121, L0.122, L0.123, L0.124, L0.125, L0.126, L0.127, L0.128, L0.129, L0.130, L0.131, L0.132, L0.133, L0.134, L0.135, L0.136, L0.137, L0.138, L0.139, L0.140, L0.141, L0.142, L0.143, L0.144, L0.145, L0.146, L0.147, L0.148, L0.149, L0.150, L0.289, L0.290"
- " Creating 3 files"
- "**** Simulation run 3, type=split(CompactAndSplitOutput(FoundSubsetLessThanMaxCompactSize))(split_times=[151, 201]). 102 Input Files, 299mb total:"
- "L0 "
- "L0.151[150,151] 151ns 2mb |L0.151| "
- "L0.152[151,152] 152ns 2mb |L0.152| "
- "L0.153[152,153] 153ns 2mb |L0.153| "
- "L0.154[153,154] 154ns 2mb |L0.154| "
- "L0.155[154,155] 155ns 2mb |L0.155| "
- "L0.156[155,156] 156ns 2mb |L0.156| "
- "L0.157[156,157] 157ns 2mb |L0.157| "
- "L0.158[157,158] 158ns 2mb |L0.158| "
- "L0.159[158,159] 159ns 2mb |L0.159| "
- "L0.160[159,160] 160ns 2mb |L0.160| "
- "L0.161[160,161] 161ns 2mb |L0.161| "
- "L0.162[161,162] 162ns 2mb |L0.162| "
- "L0.163[162,163] 163ns 2mb |L0.163| "
- "L0.164[163,164] 164ns 2mb |L0.164| "
- "L0.165[164,165] 165ns 2mb |L0.165| "
- "L0.166[165,166] 166ns 2mb |L0.166| "
- "L0.167[166,167] 167ns 2mb |L0.167| "
- "L0.168[167,168] 168ns 2mb |L0.168| "
- "L0.169[168,169] 169ns 2mb |L0.169| "
- "L0.170[169,170] 170ns 2mb |L0.170| "
- "L0.171[170,171] 171ns 2mb |L0.171| "
- "L0.172[171,172] 172ns 2mb |L0.172| "
- "L0.173[172,173] 173ns 2mb |L0.173| "
- "L0.174[173,174] 174ns 2mb |L0.174| "
- "L0.175[174,175] 175ns 2mb |L0.175| "
- "L0.176[175,176] 176ns 2mb |L0.176| "
- "L0.177[176,177] 177ns 2mb |L0.177| "
- "L0.178[177,178] 178ns 2mb |L0.178| "
- "L0.179[178,179] 179ns 2mb |L0.179| "
- "L0.180[179,180] 180ns 2mb |L0.180| "
- "L0.181[180,181] 181ns 2mb |L0.181| "
- "L0.182[181,182] 182ns 2mb |L0.182| "
- "L0.183[182,183] 183ns 2mb |L0.183| "
- "L0.184[183,184] 184ns 2mb |L0.184| "
- "L0.185[184,185] 185ns 2mb |L0.185| "
- "L0.186[185,186] 186ns 2mb |L0.186| "
- "L0.187[186,187] 187ns 2mb |L0.187| "
- "L0.188[187,188] 188ns 2mb |L0.188| "
- "L0.189[188,189] 189ns 2mb |L0.189| "
- "L0.190[189,190] 190ns 2mb |L0.190| "
- "L0.291[190,190] 191ns 1mb |L0.291| "
- "L0.292[191,191] 191ns 1mb |L0.292| "
- "L0.192[191,192] 192ns 2mb |L0.192| "
- "L0.193[192,193] 193ns 2mb |L0.193| "
- "L0.194[193,194] 194ns 2mb |L0.194| "
- "L0.195[194,195] 195ns 2mb |L0.195| "
- "L0.196[195,196] 196ns 2mb |L0.196| "
- "L0.197[196,197] 197ns 2mb |L0.197| "
- "L0.198[197,198] 198ns 2mb |L0.198| "
- "L0.199[198,199] 199ns 2mb |L0.199| "
- "L0.200[199,200] 200ns 2mb |L0.200| "
- "L0.201[200,201] 201ns 2mb |L0.201| "
- "L0.202[201,202] 202ns 2mb |L0.202| "
- "L0.203[202,203] 203ns 2mb |L0.203| "
- "L0.204[203,204] 204ns 2mb |L0.204| "
- "L0.205[204,205] 205ns 2mb |L0.205| "
- "L0.206[205,206] 206ns 2mb |L0.206| "
- "L0.207[206,207] 207ns 2mb |L0.207| "
- "L0.208[207,208] 208ns 2mb |L0.208| "
- "L0.209[208,209] 209ns 2mb |L0.209| "
- "L0.210[209,210] 210ns 2mb |L0.210| "
- "L0.211[210,211] 211ns 2mb |L0.211| "
- "L0.212[211,212] 212ns 2mb |L0.212| "
- "L0.213[212,213] 213ns 2mb |L0.213| "
- "L0.214[213,214] 214ns 2mb |L0.214| "
- "L0.215[214,215] 215ns 2mb |L0.215| "
- "L0.216[215,216] 216ns 2mb |L0.216| "
- "L0.217[216,217] 217ns 2mb |L0.217| "
- "L0.218[217,218] 218ns 2mb |L0.218| "
- "L0.219[218,219] 219ns 2mb |L0.219| "
- "L0.220[219,220] 220ns 2mb |L0.220| "
- "L0.221[220,221] 221ns 2mb |L0.221| "
- "L0.222[221,222] 222ns 2mb |L0.222| "
- "L0.223[222,223] 223ns 2mb |L0.223| "
- "L0.224[223,224] 224ns 2mb |L0.224| "
- "L0.225[224,225] 225ns 2mb |L0.225| "
- "L0.226[225,226] 226ns 2mb |L0.226| "
- "L0.227[226,227] 227ns 2mb |L0.227| "
- "L0.228[227,228] 228ns 2mb |L0.228| "
- "L0.229[228,229] 229ns 2mb |L0.229| "
- "L0.230[229,230] 230ns 2mb |L0.230| "
- "L0.231[230,231] 231ns 2mb |L0.231| "
- "L0.232[231,232] 232ns 2mb |L0.232| "
- "L0.233[232,233] 233ns 2mb |L0.233| "
- "L0.234[233,234] 234ns 2mb |L0.234| "
- "L0.235[234,235] 235ns 2mb |L0.235| "
- "L0.236[235,236] 236ns 2mb |L0.236| "
- "L0.237[236,237] 237ns 2mb |L0.237| "
- "L0.238[237,238] 238ns 2mb |L0.238|"
- "L0.239[238,239] 239ns 2mb |L0.239|"
- "L0.240[239,240] 240ns 2mb |L0.240|"
- "L0.241[240,241] 241ns 2mb |L0.241|"
- "L0.242[241,242] 242ns 2mb |L0.242|"
- "L0.243[242,243] 243ns 2mb |L0.243|"
- "L0.244[243,244] 244ns 2mb |L0.244|"
- "L0.245[244,245] 245ns 2mb |L0.245|"
- "L0.246[245,246] 246ns 2mb |L0.246|"
- "L0.247[246,247] 247ns 2mb |L0.247|"
- "L0.248[247,248] 248ns 2mb |L0.248|"
- "L0.249[248,249] 249ns 2mb |L0.249|"
- "L0.250[249,250] 250ns 2mb |L0.250|"
- "L1 "
- "L1.295[101,150] 150ns 99mb|----------L1.295-----------| "
- "**** 3 Output Files (parquet_file_id not yet assigned), 299mb total:"
- "L1 "
- "L1.?[101,151] 250ns 102mb|------------L1.?------------| "
- "L1.?[152,201] 250ns 100mb |-----------L1.?------------| "
- "L1.?[202,250] 250ns 98mb |-----------L1.?-----------| "
- "Committing partition 1:"
- " Soft Deleting 102 files: L0.151, L0.152, L0.153, L0.154, L0.155, L0.156, L0.157, L0.158, L0.159, L0.160, L0.161, L0.162, L0.163, L0.164, L0.165, L0.166, L0.167, L0.168, L0.169, L0.170, L0.171, L0.172, L0.173, L0.174, L0.175, L0.176, L0.177, L0.178, L0.179, L0.180, L0.181, L0.182, L0.183, L0.184, L0.185, L0.186, L0.187, L0.188, L0.189, L0.190, L0.192, L0.193, L0.194, L0.195, L0.196, L0.197, L0.198, L0.199, L0.200, L0.201, L0.202, L0.203, L0.204, L0.205, L0.206, L0.207, L0.208, L0.209, L0.210, L0.211, L0.212, L0.213, L0.214, L0.215, L0.216, L0.217, L0.218, L0.219, L0.220, L0.221, L0.222, L0.223, L0.224, L0.225, L0.226, L0.227, L0.228, L0.229, L0.230, L0.231, L0.232, L0.233, L0.234, L0.235, L0.236, L0.237, L0.238, L0.239, L0.240, L0.241, L0.242, L0.243, L0.244, L0.245, L0.246, L0.247, L0.248, L0.249, L0.250, L0.291, L0.292, L1.295"
- " Creating 3 files"
- "**** Simulation run 4, type=split(CompactAndSplitOutput(TotalSizeLessThanMaxCompactSize))(split_times=[252]). 39 Input Files, 174mb total:"
- "L0 "
- "L0.288[287,288] 288ns 2mb |L0.288|"
- "L0.287[286,287] 287ns 2mb |L0.287|"
- "L0.286[285,286] 286ns 2mb |L0.286|"
- "L0.285[284,285] 285ns 2mb |L0.285|"
- "L0.284[283,284] 284ns 2mb |L0.284|"
- "L0.283[282,283] 283ns 2mb |L0.283|"
- "L0.282[281,282] 282ns 2mb |L0.282|"
- "L0.281[280,281] 281ns 2mb |L0.281| "
- "L0.280[279,280] 280ns 2mb |L0.280| "
- "L0.279[278,279] 279ns 2mb |L0.279| "
- "L0.278[277,278] 278ns 2mb |L0.278| "
- "L0.277[276,277] 277ns 2mb |L0.277| "
- "L0.276[275,276] 276ns 2mb |L0.276| "
- "L0.275[274,275] 275ns 2mb |L0.275| "
- "L0.274[273,274] 274ns 2mb |L0.274| "
- "L0.273[272,273] 273ns 2mb |L0.273| "
- "L0.272[271,272] 272ns 2mb |L0.272| "
- "L0.271[270,271] 271ns 2mb |L0.271| "
- "L0.270[269,270] 270ns 2mb |L0.270| "
- "L0.269[268,269] 269ns 2mb |L0.269| "
- "L0.268[267,268] 268ns 2mb |L0.268| "
- "L0.267[266,267] 267ns 2mb |L0.267| "
- "L0.266[265,266] 266ns 2mb |L0.266| "
- "L0.265[264,265] 265ns 2mb |L0.265| "
- "L0.264[263,264] 264ns 2mb |L0.264| "
- "L0.263[262,263] 263ns 2mb |L0.263| "
- "L0.262[261,262] 262ns 2mb |L0.262| "
- "L0.261[260,261] 261ns 2mb |L0.261| "
- "L0.260[259,260] 260ns 2mb |L0.260| "
- "L0.259[258,259] 259ns 2mb |L0.259| "
- "L0.258[257,258] 258ns 2mb |L0.258| "
- "L0.257[256,257] 257ns 2mb |L0.257| "
- "L0.256[255,256] 256ns 2mb |L0.256| "
- "L0.255[254,255] 255ns 2mb |L0.255| "
- "L0.254[253,254] 254ns 2mb |L0.254| "
- "L0.253[252,253] 253ns 2mb |L0.253| "
- "L0.252[251,252] 252ns 2mb |L0.252| "
- "L0.251[250,251] 251ns 2mb |L0.251| "
- "L1 "
- "L1.298[202,250] 250ns 98mb|---------------------L1.298---------------------| "
- "**** 2 Output Files (parquet_file_id not yet assigned), 174mb total:"
- "L1 "
- "L1.?[202,252] 288ns 102mb|-----------------------L1.?-----------------------| "
- "L1.?[253,288] 288ns 72mb |---------------L1.?---------------| "
- "Committing partition 1:"
- " Soft Deleting 39 files: L0.251, L0.252, L0.253, L0.254, L0.255, L0.256, L0.257, L0.258, L0.259, L0.260, L0.261, L0.262, L0.263, L0.264, L0.265, L0.266, L0.267, L0.268, L0.269, L0.270, L0.271, L0.272, L0.273, L0.274, L0.275, L0.276, L0.277, L0.278, L0.279, L0.280, L0.281, L0.282, L0.283, L0.284, L0.285, L0.286, L0.287, L0.288, L1.298"
- " Soft Deleting 96 files: L0.1, L0.2, L0.3, L0.4, L0.5, L0.6, L0.7, L0.8, L0.9, L0.10, L0.11, L0.12, L0.13, L0.14, L0.15, L0.16, L0.17, L0.18, L0.19, L0.20, L0.21, L0.22, L0.23, L0.24, L0.25, L0.26, L0.27, L0.28, L0.29, L0.30, L0.31, L0.32, L0.33, L0.34, L0.35, L0.36, L0.37, L0.38, L0.39, L0.40, L0.41, L0.42, L0.43, L0.44, L0.45, L0.46, L0.47, L0.48, L0.49, L0.50, L0.51, L0.52, L0.53, L0.54, L0.55, L0.56, L0.57, L0.58, L0.59, L0.60, L0.61, L0.62, L0.63, L0.64, L0.65, L0.66, L0.67, L0.68, L0.69, L0.70, L0.71, L0.72, L0.73, L0.74, L0.75, L0.76, L0.77, L0.78, L0.79, L0.80, L0.81, L0.82, L0.83, L0.84, L0.85, L0.86, L0.87, L0.88, L0.89, L0.90, L0.91, L0.92, L0.93, L0.94, L0.95, L0.289"
- " Creating 2 files"
- "**** Simulation run 5, type=split(CompactAndSplitOutput(FoundSubsetLessThanMaxCompactSize))(split_times=[101]). 2 Input Files, 201mb total:"
- "**** Simulation run 3, type=split(CompactAndSplitOutput(TotalSizeLessThanMaxCompactSize))(split_times=[146]). 96 Input Files, 190mb total:"
- "L0 "
- "L0.291[190,190] 191ns 1mb |L0.291|"
- "L0.290[96,96] 96ns 1mb |L0.290| "
- "L0.190[189,190] 190ns 2mb |L0.190|"
- "L0.189[188,189] 189ns 2mb |L0.189|"
- "L0.188[187,188] 188ns 2mb |L0.188|"
- "L0.187[186,187] 187ns 2mb |L0.187|"
- "L0.186[185,186] 186ns 2mb |L0.186|"
- "L0.185[184,185] 185ns 2mb |L0.185|"
- "L0.184[183,184] 184ns 2mb |L0.184|"
- "L0.183[182,183] 183ns 2mb |L0.183|"
- "L0.182[181,182] 182ns 2mb |L0.182| "
- "L0.181[180,181] 181ns 2mb |L0.181| "
- "L0.180[179,180] 180ns 2mb |L0.180| "
- "L0.179[178,179] 179ns 2mb |L0.179| "
- "L0.178[177,178] 178ns 2mb |L0.178| "
- "L0.177[176,177] 177ns 2mb |L0.177| "
- "L0.176[175,176] 176ns 2mb |L0.176| "
- "L0.175[174,175] 175ns 2mb |L0.175| "
- "L0.174[173,174] 174ns 2mb |L0.174| "
- "L0.173[172,173] 173ns 2mb |L0.173| "
- "L0.172[171,172] 172ns 2mb |L0.172| "
- "L0.171[170,171] 171ns 2mb |L0.171| "
- "L0.170[169,170] 170ns 2mb |L0.170| "
- "L0.169[168,169] 169ns 2mb |L0.169| "
- "L0.168[167,168] 168ns 2mb |L0.168| "
- "L0.167[166,167] 167ns 2mb |L0.167| "
- "L0.166[165,166] 166ns 2mb |L0.166| "
- "L0.165[164,165] 165ns 2mb |L0.165| "
- "L0.164[163,164] 164ns 2mb |L0.164| "
- "L0.163[162,163] 163ns 2mb |L0.163| "
- "L0.162[161,162] 162ns 2mb |L0.162| "
- "L0.161[160,161] 161ns 2mb |L0.161| "
- "L0.160[159,160] 160ns 2mb |L0.160| "
- "L0.159[158,159] 159ns 2mb |L0.159| "
- "L0.158[157,158] 158ns 2mb |L0.158| "
- "L0.157[156,157] 157ns 2mb |L0.157| "
- "L0.156[155,156] 156ns 2mb |L0.156| "
- "L0.155[154,155] 155ns 2mb |L0.155| "
- "L0.154[153,154] 154ns 2mb |L0.154| "
- "L0.153[152,153] 153ns 2mb |L0.153| "
- "L0.152[151,152] 152ns 2mb |L0.152| "
- "L0.151[150,151] 151ns 2mb |L0.151| "
- "L0.150[149,150] 150ns 2mb |L0.150| "
- "L0.149[148,149] 149ns 2mb |L0.149| "
- "L0.148[147,148] 148ns 2mb |L0.148| "
- "L0.147[146,147] 147ns 2mb |L0.147| "
- "L0.146[145,146] 146ns 2mb |L0.146| "
- "L0.145[144,145] 145ns 2mb |L0.145| "
- "L0.144[143,144] 144ns 2mb |L0.144| "
- "L0.143[142,143] 143ns 2mb |L0.143| "
- "L0.142[141,142] 142ns 2mb |L0.142| "
- "L0.141[140,141] 141ns 2mb |L0.141| "
- "L0.140[139,140] 140ns 2mb |L0.140| "
- "L0.139[138,139] 139ns 2mb |L0.139| "
- "L0.138[137,138] 138ns 2mb |L0.138| "
- "L0.137[136,137] 137ns 2mb |L0.137| "
- "L0.136[135,136] 136ns 2mb |L0.136| "
- "L0.135[134,135] 135ns 2mb |L0.135| "
- "L0.134[133,134] 134ns 2mb |L0.134| "
- "L0.133[132,133] 133ns 2mb |L0.133| "
- "L0.132[131,132] 132ns 2mb |L0.132| "
- "L0.131[130,131] 131ns 2mb |L0.131| "
- "L0.130[129,130] 130ns 2mb |L0.130| "
- "L0.129[128,129] 129ns 2mb |L0.129| "
- "L0.128[127,128] 128ns 2mb |L0.128| "
- "L0.127[126,127] 127ns 2mb |L0.127| "
- "L0.126[125,126] 126ns 2mb |L0.126| "
- "L0.125[124,125] 125ns 2mb |L0.125| "
- "L0.124[123,124] 124ns 2mb |L0.124| "
- "L0.123[122,123] 123ns 2mb |L0.123| "
- "L0.122[121,122] 122ns 2mb |L0.122| "
- "L0.121[120,121] 121ns 2mb |L0.121| "
- "L0.120[119,120] 120ns 2mb |L0.120| "
- "L0.119[118,119] 119ns 2mb |L0.119| "
- "L0.118[117,118] 118ns 2mb |L0.118| "
- "L0.117[116,117] 117ns 2mb |L0.117| "
- "L0.116[115,116] 116ns 2mb |L0.116| "
- "L0.115[114,115] 115ns 2mb |L0.115| "
- "L0.114[113,114] 114ns 2mb |L0.114| "
- "L0.113[112,113] 113ns 2mb |L0.113| "
- "L0.112[111,112] 112ns 2mb |L0.112| "
- "L0.111[110,111] 111ns 2mb |L0.111| "
- "L0.110[109,110] 110ns 2mb |L0.110| "
- "L0.109[108,109] 109ns 2mb |L0.109| "
- "L0.108[107,108] 108ns 2mb |L0.108| "
- "L0.107[106,107] 107ns 2mb |L0.107| "
- "L0.106[105,106] 106ns 2mb |L0.106| "
- "L0.105[104,105] 105ns 2mb |L0.105| "
- "L0.104[103,104] 104ns 2mb |L0.104| "
- "L0.103[102,103] 103ns 2mb |L0.103| "
- "L0.102[101,102] 102ns 2mb |L0.102| "
- "L0.101[100,101] 101ns 2mb |L0.101| "
- "L0.100[99,100] 100ns 2mb |L0.100| "
- "L0.99[98,99] 99ns 2mb |L0.99| "
- "L0.98[97,98] 98ns 2mb |L0.98| "
- "L0.97[96,97] 97ns 2mb |L0.97| "
- "**** 2 Output Files (parquet_file_id not yet assigned), 190mb total:"
- "L1 "
- "L1.294[51,100] 150ns 99mb|------------------L1.294------------------| "
- "L1.296[101,151] 250ns 102mb |------------------L1.296-------------------|"
- "**** 2 Output Files (parquet_file_id not yet assigned), 201mb total:"
- "L2 "
- "L2.?[51,101] 250ns 102mb |-------------------L2.?--------------------| "
- "L2.?[102,151] 250ns 100mb |-------------------L2.?-------------------| "
- "L1.?[96,146] 191ns 102mb |--------------------L1.?---------------------| "
- "L1.?[147,190] 191ns 88mb |-----------------L1.?------------------| "
- "Committing partition 1:"
- " Soft Deleting 2 files: L1.294, L1.296"
- " Upgrading 1 files level to CompactionLevel::L2: L1.293"
- " Soft Deleting 96 files: L0.97, L0.98, L0.99, L0.100, L0.101, L0.102, L0.103, L0.104, L0.105, L0.106, L0.107, L0.108, L0.109, L0.110, L0.111, L0.112, L0.113, L0.114, L0.115, L0.116, L0.117, L0.118, L0.119, L0.120, L0.121, L0.122, L0.123, L0.124, L0.125, L0.126, L0.127, L0.128, L0.129, L0.130, L0.131, L0.132, L0.133, L0.134, L0.135, L0.136, L0.137, L0.138, L0.139, L0.140, L0.141, L0.142, L0.143, L0.144, L0.145, L0.146, L0.147, L0.148, L0.149, L0.150, L0.151, L0.152, L0.153, L0.154, L0.155, L0.156, L0.157, L0.158, L0.159, L0.160, L0.161, L0.162, L0.163, L0.164, L0.165, L0.166, L0.167, L0.168, L0.169, L0.170, L0.171, L0.172, L0.173, L0.174, L0.175, L0.176, L0.177, L0.178, L0.179, L0.180, L0.181, L0.182, L0.183, L0.184, L0.185, L0.186, L0.187, L0.188, L0.189, L0.190, L0.290, L0.291"
- " Creating 2 files"
- "**** Simulation run 6, type=split(CompactAndSplitOutput(TotalSizeLessThanMaxCompactSize))(split_times=[202, 252]). 3 Input Files, 274mb total:"
- "**** Simulation run 4, type=split(CompactAndSplitOutput(TotalSizeLessThanMaxCompactSize))(split_times=[241]). 98 Input Files, 195mb total:"
- "L0 "
- "L0.292[191,191] 191ns 1mb|L0.292| "
- "L0.288[287,288] 288ns 2mb |L0.288|"
- "L0.287[286,287] 287ns 2mb |L0.287|"
- "L0.286[285,286] 286ns 2mb |L0.286|"
- "L0.285[284,285] 285ns 2mb |L0.285|"
- "L0.284[283,284] 284ns 2mb |L0.284|"
- "L0.283[282,283] 283ns 2mb |L0.283|"
- "L0.282[281,282] 282ns 2mb |L0.282|"
- "L0.281[280,281] 281ns 2mb |L0.281|"
- "L0.280[279,280] 280ns 2mb |L0.280| "
- "L0.279[278,279] 279ns 2mb |L0.279| "
- "L0.278[277,278] 278ns 2mb |L0.278| "
- "L0.277[276,277] 277ns 2mb |L0.277| "
- "L0.276[275,276] 276ns 2mb |L0.276| "
- "L0.275[274,275] 275ns 2mb |L0.275| "
- "L0.274[273,274] 274ns 2mb |L0.274| "
- "L0.273[272,273] 273ns 2mb |L0.273| "
- "L0.272[271,272] 272ns 2mb |L0.272| "
- "L0.271[270,271] 271ns 2mb |L0.271| "
- "L0.270[269,270] 270ns 2mb |L0.270| "
- "L0.269[268,269] 269ns 2mb |L0.269| "
- "L0.268[267,268] 268ns 2mb |L0.268| "
- "L0.267[266,267] 267ns 2mb |L0.267| "
- "L0.266[265,266] 266ns 2mb |L0.266| "
- "L0.265[264,265] 265ns 2mb |L0.265| "
- "L0.264[263,264] 264ns 2mb |L0.264| "
- "L0.263[262,263] 263ns 2mb |L0.263| "
- "L0.262[261,262] 262ns 2mb |L0.262| "
- "L0.261[260,261] 261ns 2mb |L0.261| "
- "L0.260[259,260] 260ns 2mb |L0.260| "
- "L0.259[258,259] 259ns 2mb |L0.259| "
- "L0.258[257,258] 258ns 2mb |L0.258| "
- "L0.257[256,257] 257ns 2mb |L0.257| "
- "L0.256[255,256] 256ns 2mb |L0.256| "
- "L0.255[254,255] 255ns 2mb |L0.255| "
- "L0.254[253,254] 254ns 2mb |L0.254| "
- "L0.253[252,253] 253ns 2mb |L0.253| "
- "L0.252[251,252] 252ns 2mb |L0.252| "
- "L0.251[250,251] 251ns 2mb |L0.251| "
- "L0.250[249,250] 250ns 2mb |L0.250| "
- "L0.249[248,249] 249ns 2mb |L0.249| "
- "L0.248[247,248] 248ns 2mb |L0.248| "
- "L0.247[246,247] 247ns 2mb |L0.247| "
- "L0.246[245,246] 246ns 2mb |L0.246| "
- "L0.245[244,245] 245ns 2mb |L0.245| "
- "L0.244[243,244] 244ns 2mb |L0.244| "
- "L0.243[242,243] 243ns 2mb |L0.243| "
- "L0.242[241,242] 242ns 2mb |L0.242| "
- "L0.241[240,241] 241ns 2mb |L0.241| "
- "L0.240[239,240] 240ns 2mb |L0.240| "
- "L0.239[238,239] 239ns 2mb |L0.239| "
- "L0.238[237,238] 238ns 2mb |L0.238| "
- "L0.237[236,237] 237ns 2mb |L0.237| "
- "L0.236[235,236] 236ns 2mb |L0.236| "
- "L0.235[234,235] 235ns 2mb |L0.235| "
- "L0.234[233,234] 234ns 2mb |L0.234| "
- "L0.233[232,233] 233ns 2mb |L0.233| "
- "L0.232[231,232] 232ns 2mb |L0.232| "
- "L0.231[230,231] 231ns 2mb |L0.231| "
- "L0.230[229,230] 230ns 2mb |L0.230| "
- "L0.229[228,229] 229ns 2mb |L0.229| "
- "L0.228[227,228] 228ns 2mb |L0.228| "
- "L0.227[226,227] 227ns 2mb |L0.227| "
- "L0.226[225,226] 226ns 2mb |L0.226| "
- "L0.225[224,225] 225ns 2mb |L0.225| "
- "L0.224[223,224] 224ns 2mb |L0.224| "
- "L0.223[222,223] 223ns 2mb |L0.223| "
- "L0.222[221,222] 222ns 2mb |L0.222| "
- "L0.221[220,221] 221ns 2mb |L0.221| "
- "L0.220[219,220] 220ns 2mb |L0.220| "
- "L0.219[218,219] 219ns 2mb |L0.219| "
- "L0.218[217,218] 218ns 2mb |L0.218| "
- "L0.217[216,217] 217ns 2mb |L0.217| "
- "L0.216[215,216] 216ns 2mb |L0.216| "
- "L0.215[214,215] 215ns 2mb |L0.215| "
- "L0.214[213,214] 214ns 2mb |L0.214| "
- "L0.213[212,213] 213ns 2mb |L0.213| "
- "L0.212[211,212] 212ns 2mb |L0.212| "
- "L0.211[210,211] 211ns 2mb |L0.211| "
- "L0.210[209,210] 210ns 2mb |L0.210| "
- "L0.209[208,209] 209ns 2mb |L0.209| "
- "L0.208[207,208] 208ns 2mb |L0.208| "
- "L0.207[206,207] 207ns 2mb |L0.207| "
- "L0.206[205,206] 206ns 2mb |L0.206| "
- "L0.205[204,205] 205ns 2mb |L0.205| "
- "L0.204[203,204] 204ns 2mb |L0.204| "
- "L0.203[202,203] 203ns 2mb |L0.203| "
- "L0.202[201,202] 202ns 2mb |L0.202| "
- "L0.201[200,201] 201ns 2mb |L0.201| "
- "L0.200[199,200] 200ns 2mb |L0.200| "
- "L0.199[198,199] 199ns 2mb |L0.199| "
- "L0.198[197,198] 198ns 2mb |L0.198| "
- "L0.197[196,197] 197ns 2mb |L0.197| "
- "L0.196[195,196] 196ns 2mb |L0.196| "
- "L0.195[194,195] 195ns 2mb |L0.195| "
- "L0.194[193,194] 194ns 2mb |L0.194| "
- "L0.193[192,193] 193ns 2mb|L0.193| "
- "L0.192[191,192] 192ns 2mb|L0.192| "
- "**** 2 Output Files (parquet_file_id not yet assigned), 195mb total:"
- "L1 "
- "L1.300[253,288] 288ns 72mb |-------L1.300--------| "
- "L1.297[152,201] 250ns 100mb|------------L1.297------------| "
- "L1.299[202,252] 288ns 102mb |------------L1.299-------------| "
- "**** 3 Output Files (parquet_file_id not yet assigned), 274mb total:"
- "L2 "
- "L2.?[152,202] 288ns 102mb|-------------L2.?--------------| "
- "L2.?[203,252] 288ns 100mb |-------------L2.?-------------| "
- "L2.?[253,288] 288ns 72mb |--------L2.?---------| "
- "L1.?[191,241] 288ns 101mb|--------------------L1.?--------------------| "
- "L1.?[242,288] 288ns 94mb |------------------L1.?------------------| "
- "Committing partition 1:"
- " Soft Deleting 3 files: L1.297, L1.299, L1.300"
- " Creating 3 files"
- "**** Final Output Files (1.22gb written)"
- " Soft Deleting 98 files: L0.192, L0.193, L0.194, L0.195, L0.196, L0.197, L0.198, L0.199, L0.200, L0.201, L0.202, L0.203, L0.204, L0.205, L0.206, L0.207, L0.208, L0.209, L0.210, L0.211, L0.212, L0.213, L0.214, L0.215, L0.216, L0.217, L0.218, L0.219, L0.220, L0.221, L0.222, L0.223, L0.224, L0.225, L0.226, L0.227, L0.228, L0.229, L0.230, L0.231, L0.232, L0.233, L0.234, L0.235, L0.236, L0.237, L0.238, L0.239, L0.240, L0.241, L0.242, L0.243, L0.244, L0.245, L0.246, L0.247, L0.248, L0.249, L0.250, L0.251, L0.252, L0.253, L0.254, L0.255, L0.256, L0.257, L0.258, L0.259, L0.260, L0.261, L0.262, L0.263, L0.264, L0.265, L0.266, L0.267, L0.268, L0.269, L0.270, L0.271, L0.272, L0.273, L0.274, L0.275, L0.276, L0.277, L0.278, L0.279, L0.280, L0.281, L0.282, L0.283, L0.284, L0.285, L0.286, L0.287, L0.288, L0.292"
- " Creating 2 files"
- "Committing partition 1:"
- " Upgrading 6 files level to CompactionLevel::L2: L1.293, L1.294, L1.295, L1.296, L1.297, L1.298"
- "**** Final Output Files (580mb written)"
- "L2 "
- "L2.293[0,50] 150ns 101mb |---L2.293----| "
- "L2.301[51,101] 250ns 102mb |---L2.301----| "
- "L2.302[102,151] 250ns 100mb |---L2.302----| "
- "L2.303[152,202] 288ns 102mb |---L2.303----| "
- "L2.304[203,252] 288ns 100mb |---L2.304----| "
- "L2.305[253,288] 288ns 72mb |-L2.305-| "
- "L2.293[0,50] 96ns 101mb |---L2.293----| "
- "L2.294[51,95] 96ns 90mb |--L2.294---| "
- "L2.295[96,146] 191ns 102mb |---L2.295----| "
- "L2.296[147,190] 191ns 88mb |--L2.296---| "
- "L2.297[191,241] 288ns 101mb |---L2.297----| "
- "L2.298[242,288] 288ns 94mb |---L2.298---| "
"###
);
}

24
compactor/tests/layouts/many_files.rs
View File

@ -5725,23 +5725,21 @@ async fn l0s_needing_vertical_split() {
- "L0.998[24,100] 1.02us |-----------------------------------------L0.998-----------------------------------------|"
- "L0.999[24,100] 1.02us |-----------------------------------------L0.999-----------------------------------------|"
- "L0.1000[24,100] 1.02us |----------------------------------------L0.1000-----------------------------------------|"
- "**** Final Output Files (4.32gb written)"
- "**** Final Output Files (2.63gb written)"
- "L2 "
- "L2.6077[24,35] 1.02us 112mb|--L2.6077--| "
- "L2.6078[36,46] 1.02us 103mb |-L2.6078-| "
- "L2.6082[74,84] 1.02us 111mb |-L2.6082-| "
- "L2.6083[85,94] 1.02us 101mb |L2.6083-| "
- "L2.6085[47,57] 1.02us 107mb |-L2.6085-| "
- "L2.6086[58,67] 1.02us 97mb |L2.6086-| "
- "L2.6087[68,73] 1.02us 58mb |L2.6087| "
- "L2.6088[95,99] 1.02us 50mb |L2.6088|"
- "L2.6089[100,100] 1.02us 10mb |L2.6089|"
- "L2.6026[24,34] 1.02us 107mb|-L2.6026-| "
- "L2.6034[81,91] 1.02us 107mb |-L2.6034-| "
- "L2.6035[92,100] 1.02us 88mb |L2.6035| "
- "L2.6036[35,45] 1.02us 107mb |-L2.6036-| "
- "L2.6037[46,55] 1.02us 97mb |L2.6037-| "
- "L2.6038[56,63] 1.02us 78mb |L2.6038| "
- "L2.6039[64,74] 1.02us 107mb |-L2.6039-| "
- "L2.6040[75,80] 1.02us 58mb |L2.6040| "
- "**** Breakdown of where bytes were written"
- 2.13gb written by split(CompactAndSplitOutput(FoundSubsetLessThanMaxCompactSize))
- 333mb written by split(CompactAndSplitOutput(TotalSizeLessThanMaxCompactSize))
- 417mb written by split(ReduceOverlap)
- 282mb written by split(CompactAndSplitOutput(FoundSubsetLessThanMaxCompactSize))
- 750mb written by compact(ManySmallFiles)
- 750mb written by split(VerticalSplit)
- 916mb written by split(CompactAndSplitOutput(TotalSizeLessThanMaxCompactSize))
"###
);
}

238
compactor/tests/layouts/single_timestamp.rs
View File

@ -385,234 +385,54 @@ async fn many_medium_files_time_range_1() {
- "Committing partition 1:"
- " Soft Deleting 10 files: L0.11, L0.12, L0.13, L0.14, L0.15, L0.16, L0.17, L0.18, L0.19, L0.20"
- " Creating 20 files"
- "**** Simulation run 12, type=compact(FoundSubsetLessThanMaxCompactSize). 2 Input Files, 165mb total:"
- "**** Simulation run 12, type=compact(TotalSizeLessThanMaxCompactSize). 11 Input Files, 300mb total:"
- "L0 "
- "L0.42[100,100] 20ns 15mb |-----------------------------------------L0.42------------------------------------------|"
- "L0.40[100,100] 19ns 15mb |-----------------------------------------L0.40------------------------------------------|"
- "L0.38[100,100] 18ns 15mb |-----------------------------------------L0.38------------------------------------------|"
- "L0.36[100,100] 17ns 15mb |-----------------------------------------L0.36------------------------------------------|"
- "L0.34[100,100] 16ns 15mb |-----------------------------------------L0.34------------------------------------------|"
- "L0.32[100,100] 15ns 15mb |-----------------------------------------L0.32------------------------------------------|"
- "L0.30[100,100] 14ns 15mb |-----------------------------------------L0.30------------------------------------------|"
- "L0.28[100,100] 13ns 15mb |-----------------------------------------L0.28------------------------------------------|"
- "L0.26[100,100] 12ns 15mb |-----------------------------------------L0.26------------------------------------------|"
- "L0.24[100,100] 11ns 15mb |-----------------------------------------L0.24------------------------------------------|"
- "L1 "
- "L1.22[100,100] 10ns 150mb|-----------------------------------------L1.22------------------------------------------|"
- "**** 1 Output Files (parquet_file_id not yet assigned), 165mb total:"
- "L1, all files 165mb "
- "L1.?[100,100] 11ns |------------------------------------------L1.?------------------------------------------|"
- "Committing partition 1:"
- " Soft Deleting 2 files: L1.22, L0.24"
- " Creating 1 files"
- "**** Simulation run 13, type=compact(FoundSubsetLessThanMaxCompactSize). 2 Input Files, 165mb total:"
- "L0 "
- "L0.25[101,101] 11ns 15mb |-----------------------------------------L0.25------------------------------------------|"
- "L1 "
- "L1.23[101,101] 10ns 150mb|-----------------------------------------L1.23------------------------------------------|"
- "**** 1 Output Files (parquet_file_id not yet assigned), 165mb total:"
- "L1, all files 165mb "
- "L1.?[101,101] 11ns |------------------------------------------L1.?------------------------------------------|"
- "Committing partition 1:"
- " Soft Deleting 2 files: L1.23, L0.25"
- " Creating 1 files"
- "**** Simulation run 14, type=compact(FoundSubsetLessThanMaxCompactSize). 2 Input Files, 180mb total:"
- "L0 "
- "L0.26[100,100] 12ns 15mb |-----------------------------------------L0.26------------------------------------------|"
- "L1 "
- "L1.44[100,100] 11ns 165mb|-----------------------------------------L1.44------------------------------------------|"
- "**** 1 Output Files (parquet_file_id not yet assigned), 180mb total:"
- "L1, all files 180mb "
- "L1.?[100,100] 12ns |------------------------------------------L1.?------------------------------------------|"
- "Committing partition 1:"
- " Soft Deleting 2 files: L0.26, L1.44"
- " Creating 1 files"
- "**** Simulation run 15, type=compact(FoundSubsetLessThanMaxCompactSize). 2 Input Files, 180mb total:"
- "L0 "
- "L0.27[101,101] 12ns 15mb |-----------------------------------------L0.27------------------------------------------|"
- "L1 "
- "L1.45[101,101] 11ns 165mb|-----------------------------------------L1.45------------------------------------------|"
- "**** 1 Output Files (parquet_file_id not yet assigned), 180mb total:"
- "L1, all files 180mb "
- "L1.?[101,101] 12ns |------------------------------------------L1.?------------------------------------------|"
- "Committing partition 1:"
- " Soft Deleting 2 files: L0.27, L1.45"
- " Creating 1 files"
- "**** Simulation run 16, type=compact(FoundSubsetLessThanMaxCompactSize). 2 Input Files, 195mb total:"
- "L0 "
- "L0.28[100,100] 13ns 15mb |-----------------------------------------L0.28------------------------------------------|"
- "L1 "
- "L1.46[100,100] 12ns 180mb|-----------------------------------------L1.46------------------------------------------|"
- "**** 1 Output Files (parquet_file_id not yet assigned), 195mb total:"
- "L1, all files 195mb "
- "L1.?[100,100] 13ns |------------------------------------------L1.?------------------------------------------|"
- "Committing partition 1:"
- " Soft Deleting 2 files: L0.28, L1.46"
- " Creating 1 files"
- "**** Simulation run 17, type=compact(FoundSubsetLessThanMaxCompactSize). 2 Input Files, 195mb total:"
- "L0 "
- "L0.29[101,101] 13ns 15mb |-----------------------------------------L0.29------------------------------------------|"
- "L1 "
- "L1.47[101,101] 12ns 180mb|-----------------------------------------L1.47------------------------------------------|"
- "**** 1 Output Files (parquet_file_id not yet assigned), 195mb total:"
- "L1, all files 195mb "
- "L1.?[101,101] 13ns |------------------------------------------L1.?------------------------------------------|"
- "Committing partition 1:"
- " Soft Deleting 2 files: L0.29, L1.47"
- " Creating 1 files"
- "**** Simulation run 18, type=compact(FoundSubsetLessThanMaxCompactSize). 2 Input Files, 210mb total:"
- "L0 "
- "L0.30[100,100] 14ns 15mb |-----------------------------------------L0.30------------------------------------------|"
- "L1 "
- "L1.48[100,100] 13ns 195mb|-----------------------------------------L1.48------------------------------------------|"
- "**** 1 Output Files (parquet_file_id not yet assigned), 210mb total:"
- "L1, all files 210mb "
- "L1.?[100,100] 14ns |------------------------------------------L1.?------------------------------------------|"
- "Committing partition 1:"
- " Soft Deleting 2 files: L0.30, L1.48"
- " Creating 1 files"
- "**** Simulation run 19, type=compact(FoundSubsetLessThanMaxCompactSize). 2 Input Files, 210mb total:"
- "L0 "
- "L0.31[101,101] 14ns 15mb |-----------------------------------------L0.31------------------------------------------|"
- "L1 "
- "L1.49[101,101] 13ns 195mb|-----------------------------------------L1.49------------------------------------------|"
- "**** 1 Output Files (parquet_file_id not yet assigned), 210mb total:"
- "L1, all files 210mb "
- "L1.?[101,101] 14ns |------------------------------------------L1.?------------------------------------------|"
- "Committing partition 1:"
- " Soft Deleting 2 files: L0.31, L1.49"
- " Creating 1 files"
- "**** Simulation run 20, type=compact(FoundSubsetLessThanMaxCompactSize). 2 Input Files, 225mb total:"
- "L0 "
- "L0.32[100,100] 15ns 15mb |-----------------------------------------L0.32------------------------------------------|"
- "L1 "
- "L1.50[100,100] 14ns 210mb|-----------------------------------------L1.50------------------------------------------|"
- "**** 1 Output Files (parquet_file_id not yet assigned), 225mb total:"
- "L1, all files 225mb "
- "L1.?[100,100] 15ns |------------------------------------------L1.?------------------------------------------|"
- "Committing partition 1:"
- " Soft Deleting 2 files: L0.32, L1.50"
- " Creating 1 files"
- "**** Simulation run 21, type=compact(FoundSubsetLessThanMaxCompactSize). 2 Input Files, 225mb total:"
- "L0 "
- "L0.33[101,101] 15ns 15mb |-----------------------------------------L0.33------------------------------------------|"
- "L1 "
- "L1.51[101,101] 14ns 210mb|-----------------------------------------L1.51------------------------------------------|"
- "**** 1 Output Files (parquet_file_id not yet assigned), 225mb total:"
- "L1, all files 225mb "
- "L1.?[101,101] 15ns |------------------------------------------L1.?------------------------------------------|"
- "Committing partition 1:"
- " Soft Deleting 2 files: L0.33, L1.51"
- " Creating 1 files"
- "**** Simulation run 22, type=compact(FoundSubsetLessThanMaxCompactSize). 2 Input Files, 240mb total:"
- "L0 "
- "L0.34[100,100] 16ns 15mb |-----------------------------------------L0.34------------------------------------------|"
- "L1 "
- "L1.52[100,100] 15ns 225mb|-----------------------------------------L1.52------------------------------------------|"
- "**** 1 Output Files (parquet_file_id not yet assigned), 240mb total:"
- "L1, all files 240mb "
- "L1.?[100,100] 16ns |------------------------------------------L1.?------------------------------------------|"
- "Committing partition 1:"
- " Soft Deleting 2 files: L0.34, L1.52"
- " Creating 1 files"
- "**** Simulation run 23, type=compact(FoundSubsetLessThanMaxCompactSize). 2 Input Files, 240mb total:"
- "L0 "
- "L0.35[101,101] 16ns 15mb |-----------------------------------------L0.35------------------------------------------|"
- "L1 "
- "L1.53[101,101] 15ns 225mb|-----------------------------------------L1.53------------------------------------------|"
- "**** 1 Output Files (parquet_file_id not yet assigned), 240mb total:"
- "L1, all files 240mb "
- "L1.?[101,101] 16ns |------------------------------------------L1.?------------------------------------------|"
- "Committing partition 1:"
- " Soft Deleting 2 files: L0.35, L1.53"
- " Creating 1 files"
- "**** Simulation run 24, type=compact(FoundSubsetLessThanMaxCompactSize). 2 Input Files, 255mb total:"
- "L0 "
- "L0.36[100,100] 17ns 15mb |-----------------------------------------L0.36------------------------------------------|"
- "L1 "
- "L1.54[100,100] 16ns 240mb|-----------------------------------------L1.54------------------------------------------|"
- "**** 1 Output Files (parquet_file_id not yet assigned), 255mb total:"
- "L1, all files 255mb "
- "L1.?[100,100] 17ns |------------------------------------------L1.?------------------------------------------|"
- "Committing partition 1:"
- " Soft Deleting 2 files: L0.36, L1.54"
- " Creating 1 files"
- "**** Simulation run 25, type=compact(FoundSubsetLessThanMaxCompactSize). 2 Input Files, 255mb total:"
- "L0 "
- "L0.37[101,101] 17ns 15mb |-----------------------------------------L0.37------------------------------------------|"
- "L1 "
- "L1.55[101,101] 16ns 240mb|-----------------------------------------L1.55------------------------------------------|"
- "**** 1 Output Files (parquet_file_id not yet assigned), 255mb total:"
- "L1, all files 255mb "
- "L1.?[101,101] 17ns |------------------------------------------L1.?------------------------------------------|"
- "Committing partition 1:"
- " Soft Deleting 2 files: L0.37, L1.55"
- " Creating 1 files"
- "**** Simulation run 26, type=compact(FoundSubsetLessThanMaxCompactSize). 2 Input Files, 270mb total:"
- "L0 "
- "L0.38[100,100] 18ns 15mb |-----------------------------------------L0.38------------------------------------------|"
- "L1 "
- "L1.56[100,100] 17ns 255mb|-----------------------------------------L1.56------------------------------------------|"
- "**** 1 Output Files (parquet_file_id not yet assigned), 270mb total:"
- "L1, all files 270mb "
- "L1.?[100,100] 18ns |------------------------------------------L1.?------------------------------------------|"
- "Committing partition 1:"
- " Soft Deleting 2 files: L0.38, L1.56"
- " Creating 1 files"
- "**** Simulation run 27, type=compact(FoundSubsetLessThanMaxCompactSize). 2 Input Files, 270mb total:"
- "L0 "
- "L0.39[101,101] 18ns 15mb |-----------------------------------------L0.39------------------------------------------|"
- "L1 "
- "L1.57[101,101] 17ns 255mb|-----------------------------------------L1.57------------------------------------------|"
- "**** 1 Output Files (parquet_file_id not yet assigned), 270mb total:"
- "L1, all files 270mb "
- "L1.?[101,101] 18ns |------------------------------------------L1.?------------------------------------------|"
- "Committing partition 1:"
- " Soft Deleting 2 files: L0.39, L1.57"
- " Creating 1 files"
- "**** Simulation run 28, type=compact(FoundSubsetLessThanMaxCompactSize). 2 Input Files, 285mb total:"
- "L0 "
- "L0.40[100,100] 19ns 15mb |-----------------------------------------L0.40------------------------------------------|"
- "L1 "
- "L1.58[100,100] 18ns 270mb|-----------------------------------------L1.58------------------------------------------|"
- "**** 1 Output Files (parquet_file_id not yet assigned), 285mb total:"
- "L1, all files 285mb "
- "L1.?[100,100] 19ns |------------------------------------------L1.?------------------------------------------|"
- "Committing partition 1:"
- " Soft Deleting 2 files: L0.40, L1.58"
- " Creating 1 files"
- "**** Simulation run 29, type=compact(FoundSubsetLessThanMaxCompactSize). 2 Input Files, 285mb total:"
- "L0 "
- "L0.41[101,101] 19ns 15mb |-----------------------------------------L0.41------------------------------------------|"
- "L1 "
- "L1.59[101,101] 18ns 270mb|-----------------------------------------L1.59------------------------------------------|"
- "**** 1 Output Files (parquet_file_id not yet assigned), 285mb total:"
- "L1, all files 285mb "
- "L1.?[101,101] 19ns |------------------------------------------L1.?------------------------------------------|"
- "Committing partition 1:"
- " Soft Deleting 2 files: L0.41, L1.59"
- " Creating 1 files"
- "**** Simulation run 30, type=compact(FoundSubsetLessThanMaxCompactSize). 2 Input Files, 300mb total:"
- "L0 "
- "L0.42[100,100] 20ns 15mb |-----------------------------------------L0.42------------------------------------------|"
- "L1 "
- "L1.60[100,100] 19ns 285mb|-----------------------------------------L1.60------------------------------------------|"
- "**** 1 Output Files (parquet_file_id not yet assigned), 300mb total:"
- "L1, all files 300mb "
- "L1.?[100,100] 20ns |------------------------------------------L1.?------------------------------------------|"
- "Committing partition 1:"
- " Soft Deleting 2 files: L0.42, L1.60"
- " Soft Deleting 11 files: L1.22, L0.24, L0.26, L0.28, L0.30, L0.32, L0.34, L0.36, L0.38, L0.40, L0.42"
- " Creating 1 files"
- "**** Simulation run 31, type=compact(TotalSizeLessThanMaxCompactSize). 2 Input Files, 300mb total:"
- "**** Simulation run 13, type=compact(TotalSizeLessThanMaxCompactSize). 11 Input Files, 300mb total:"
- "L0 "
- "L0.43[101,101] 20ns 15mb |-----------------------------------------L0.43------------------------------------------|"
- "L0.41[101,101] 19ns 15mb |-----------------------------------------L0.41------------------------------------------|"
- "L0.39[101,101] 18ns 15mb |-----------------------------------------L0.39------------------------------------------|"
- "L0.37[101,101] 17ns 15mb |-----------------------------------------L0.37------------------------------------------|"
- "L0.35[101,101] 16ns 15mb |-----------------------------------------L0.35------------------------------------------|"
- "L0.33[101,101] 15ns 15mb |-----------------------------------------L0.33------------------------------------------|"
- "L0.31[101,101] 14ns 15mb |-----------------------------------------L0.31------------------------------------------|"
- "L0.29[101,101] 13ns 15mb |-----------------------------------------L0.29------------------------------------------|"
- "L0.27[101,101] 12ns 15mb |-----------------------------------------L0.27------------------------------------------|"
- "L0.25[101,101] 11ns 15mb |-----------------------------------------L0.25------------------------------------------|"
- "L1 "
- "L1.61[101,101] 19ns 285mb|-----------------------------------------L1.61------------------------------------------|"
- "L1.23[101,101] 10ns 150mb|-----------------------------------------L1.23------------------------------------------|"
- "**** 1 Output Files (parquet_file_id not yet assigned), 300mb total:"
- "L1, all files 300mb "
- "L1.?[101,101] 20ns |------------------------------------------L1.?------------------------------------------|"
- "Committing partition 1:"
- " Soft Deleting 2 files: L0.43, L1.61"
- " Soft Deleting 11 files: L1.23, L0.25, L0.27, L0.29, L0.31, L0.33, L0.35, L0.37, L0.39, L0.41, L0.43"
- " Creating 1 files"
- "Committing partition 1:"
- " Upgrading 2 files level to CompactionLevel::L2: L1.62, L1.63"
- "**** Final Output Files (5.42gb written)"
- " Upgrading 2 files level to CompactionLevel::L2: L1.44, L1.45"
- "**** Final Output Files (1.46gb written)"
- "L2, all files 300mb "
- "L2.62[100,100] 20ns |L2.62| "
- "L2.63[101,101] 20ns |L2.63|"
- "WARNING: file L2.62[100,100] 20ns 300mb exceeds soft limit 100mb by more than 50%"
- "WARNING: file L2.63[101,101] 20ns 300mb exceeds soft limit 100mb by more than 50%"
- "L2.44[100,100] 20ns |L2.44| "
- "L2.45[101,101] 20ns |L2.45|"
- "WARNING: file L2.44[100,100] 20ns 300mb exceeds soft limit 100mb by more than 50%"
- "WARNING: file L2.45[101,101] 20ns 300mb exceeds soft limit 100mb by more than 50%"
"###
);
}

8457
compactor/tests/layouts/stuck.rs
View File

File diff suppressed because it is too large Load Diff

12
data_types/src/lib.rs
View File

@ -627,6 +627,16 @@ impl ParquetFile {
}
false
}
/// Return true if the time range of this file overlaps with any of the given file ranges
pub fn overlaps_ranges(&self, ranges: &Vec<FileRange>) -> bool {
for range in ranges {
if self.min_time.get() <= range.max && self.max_time.get() >= range.min {
return true;
}
}
false
}
}
/// Data for a parquet file to be inserted into the catalog.
@ -1606,7 +1616,7 @@ impl TimestampMinMax {
}
/// FileRange describes a range of files by the min/max time and the sum of their capacities.
#[derive(Clone, Debug, Copy)]
#[derive(Clone, Debug, Copy, PartialEq, Eq)]
pub struct FileRange {
/// The minimum time of any file in the range
pub min: i64,