chore(compactor2): Add tests for remaining file knobs (#7099)
Co-authored-by: kodiakhq[bot] <49736102+kodiakhq[bot]@users.noreply.github.com>pull/24376/head
Andrew Lamb
GitHub
parent
eb488cf55a
commit
6b35af115e
|
|
@ -234,3 +234,204 @@ async fn all_overlapping_l0_split_percentage_and_max_file_size() {
|
|||
"###
|
||||
);
|
||||
}
|
||||
|
||||
/// This test show how the max_num_files_per_plan setting works
|
||||
#[tokio::test]
|
||||
async fn all_overlapping_l0_max_num_files_per_plan() {
|
||||
test_helpers::maybe_start_logging();
|
||||
|
||||
// configure the max num files per plan to 3
|
||||
let setup = layout_setup_builder()
|
||||
.await
|
||||
.with_max_num_files_per_plan(3)
|
||||
.build()
|
||||
.await;
|
||||
|
||||
// create virtual files
|
||||
let setup = all_overlapping_l0_files(setup).await;
|
||||
|
||||
// expect that each plan the compactor runs has at most three input files
|
||||
insta::assert_yaml_snapshot!(
|
||||
run_layout_scenario(&setup).await,
|
||||
@r###"
|
||||
---
|
||||
- "**** Input Files "
|
||||
- "L0, all files 9mb "
|
||||
- "L0.1[100,200] |-------------------------------------L0.1-------------------------------------|"
|
||||
- "L0.2[100,200] |-------------------------------------L0.2-------------------------------------|"
|
||||
- "L0.3[100,200] |-------------------------------------L0.3-------------------------------------|"
|
||||
- "L0.4[100,200] |-------------------------------------L0.4-------------------------------------|"
|
||||
- "L0.5[100,200] |-------------------------------------L0.5-------------------------------------|"
|
||||
- "L0.6[100,200] |-------------------------------------L0.6-------------------------------------|"
|
||||
- "L0.7[100,200] |-------------------------------------L0.7-------------------------------------|"
|
||||
- "L0.8[100,200] |-------------------------------------L0.8-------------------------------------|"
|
||||
- "L0.9[100,200] |-------------------------------------L0.9-------------------------------------|"
|
||||
- "L0.10[100,200] |------------------------------------L0.10-------------------------------------|"
|
||||
- "**** Simulation run 0, type=split(split_times=[180]). 3 Input Files, 27mb total:"
|
||||
- "L0, all files 9mb "
|
||||
- "L0.1[100,200] |-------------------------------------L0.1-------------------------------------|"
|
||||
- "L0.2[100,200] |-------------------------------------L0.2-------------------------------------|"
|
||||
- "L0.3[100,200] |-------------------------------------L0.3-------------------------------------|"
|
||||
- "**** 2 Output Files (parquet_file_id not yet assigned), 27mb total:"
|
||||
- "L0 "
|
||||
- "L0.?[100,180] 21.6mb|-----------------------------L0.?-----------------------------| "
|
||||
- "L0.?[180,200] 5.4mb |-----L0.?-----|"
|
||||
- "Committing partition 1:"
|
||||
- " Soft Deleting 3 files: L0.1, L0.2, L0.3"
|
||||
- " Creating 2 files at level CompactionLevel::L0"
|
||||
- "**** Simulation run 1, type=split(split_times=[180]). 3 Input Files, 27mb total:"
|
||||
- "L0, all files 9mb "
|
||||
- "L0.4[100,200] |-------------------------------------L0.4-------------------------------------|"
|
||||
- "L0.5[100,200] |-------------------------------------L0.5-------------------------------------|"
|
||||
- "L0.6[100,200] |-------------------------------------L0.6-------------------------------------|"
|
||||
- "**** 2 Output Files (parquet_file_id not yet assigned), 27mb total:"
|
||||
- "L0 "
|
||||
- "L0.?[100,180] 21.6mb|-----------------------------L0.?-----------------------------| "
|
||||
- "L0.?[180,200] 5.4mb |-----L0.?-----|"
|
||||
- "Committing partition 1:"
|
||||
- " Soft Deleting 3 files: L0.4, L0.5, L0.6"
|
||||
- " Creating 2 files at level CompactionLevel::L0"
|
||||
- "**** Simulation run 2, type=split(split_times=[180]). 3 Input Files, 27mb total:"
|
||||
- "L0, all files 9mb "
|
||||
- "L0.7[100,200] |-------------------------------------L0.7-------------------------------------|"
|
||||
- "L0.8[100,200] |-------------------------------------L0.8-------------------------------------|"
|
||||
- "L0.9[100,200] |-------------------------------------L0.9-------------------------------------|"
|
||||
- "**** 2 Output Files (parquet_file_id not yet assigned), 27mb total:"
|
||||
- "L0 "
|
||||
- "L0.?[100,180] 21.6mb|-----------------------------L0.?-----------------------------| "
|
||||
- "L0.?[180,200] 5.4mb |-----L0.?-----|"
|
||||
- "Committing partition 1:"
|
||||
- " Soft Deleting 3 files: L0.7, L0.8, L0.9"
|
||||
- " Creating 2 files at level CompactionLevel::L0"
|
||||
- "**** Simulation run 3, type=compact. 1 Input Files, 9mb total:"
|
||||
- "L0, all files 9mb "
|
||||
- "L0.10[100,200] |------------------------------------L0.10-------------------------------------|"
|
||||
- "**** 1 Output Files (parquet_file_id not yet assigned), 9mb total:"
|
||||
- "L0, all files 9mb "
|
||||
- "L0.?[100,200] |-------------------------------------L0.?-------------------------------------|"
|
||||
- "Committing partition 1:"
|
||||
- " Soft Deleting 1 files: L0.10"
|
||||
- " Creating 1 files at level CompactionLevel::L0"
|
||||
- "**** Simulation run 4, type=split(split_times=[180]). 3 Input Files, 48.6mb total:"
|
||||
- "L0 "
|
||||
- "L0.11[100,180] 21.6mb|----------------------------L0.11-----------------------------| "
|
||||
- "L0.12[180,200] 5.4mb |----L0.12-----|"
|
||||
- "L0.13[100,180] 21.6mb|----------------------------L0.13-----------------------------| "
|
||||
- "**** 2 Output Files (parquet_file_id not yet assigned), 48.6mb total:"
|
||||
- "L0 "
|
||||
- "L0.?[100,180] 38.88mb|-----------------------------L0.?-----------------------------| "
|
||||
- "L0.?[180,200] 9.72mb |-----L0.?-----|"
|
||||
- "Committing partition 1:"
|
||||
- " Soft Deleting 3 files: L0.11, L0.12, L0.13"
|
||||
- " Creating 2 files at level CompactionLevel::L0"
|
||||
- "**** Simulation run 5, type=split(split_times=[180]). 3 Input Files, 32.4mb total:"
|
||||
- "L0 "
|
||||
- "L0.14[180,200] 5.4mb |----L0.14-----|"
|
||||
- "L0.15[100,180] 21.6mb|----------------------------L0.15-----------------------------| "
|
||||
- "L0.16[180,200] 5.4mb |----L0.16-----|"
|
||||
- "**** 2 Output Files (parquet_file_id not yet assigned), 32.4mb total:"
|
||||
- "L0 "
|
||||
- "L0.?[100,180] 25.92mb|-----------------------------L0.?-----------------------------| "
|
||||
- "L0.?[180,200] 6.48mb |-----L0.?-----|"
|
||||
- "Committing partition 1:"
|
||||
- " Soft Deleting 3 files: L0.14, L0.15, L0.16"
|
||||
- " Creating 2 files at level CompactionLevel::L0"
|
||||
- "**** Simulation run 6, type=compact. 1 Input Files, 9mb total:"
|
||||
- "L0, all files 9mb "
|
||||
- "L0.17[100,200] |------------------------------------L0.17-------------------------------------|"
|
||||
- "**** 1 Output Files (parquet_file_id not yet assigned), 9mb total:"
|
||||
- "L0, all files 9mb "
|
||||
- "L0.?[100,200] |-------------------------------------L0.?-------------------------------------|"
|
||||
- "Committing partition 1:"
|
||||
- " Soft Deleting 1 files: L0.17"
|
||||
- " Creating 1 files at level CompactionLevel::L0"
|
||||
- "**** Simulation run 7, type=split(split_times=[180]). 3 Input Files, 74.52mb total:"
|
||||
- "L0 "
|
||||
- "L0.18[100,180] 38.88mb|----------------------------L0.18-----------------------------| "
|
||||
- "L0.19[180,200] 9.72mb |----L0.19-----|"
|
||||
- "L0.20[100,180] 25.92mb|----------------------------L0.20-----------------------------| "
|
||||
- "**** 2 Output Files (parquet_file_id not yet assigned), 74.52mb total:"
|
||||
- "L0 "
|
||||
- "L0.?[100,180] 59.62mb|-----------------------------L0.?-----------------------------| "
|
||||
- "L0.?[180,200] 14.9mb |-----L0.?-----|"
|
||||
- "Committing partition 1:"
|
||||
- " Soft Deleting 3 files: L0.18, L0.19, L0.20"
|
||||
- " Creating 2 files at level CompactionLevel::L0"
|
||||
- "**** Simulation run 8, type=compact. 2 Input Files, 15.48mb total:"
|
||||
- "L0 "
|
||||
- "L0.21[180,200] 6.48mb |----L0.21-----|"
|
||||
- "L0.22[100,200] 9mb |------------------------------------L0.22-------------------------------------|"
|
||||
- "**** 1 Output Files (parquet_file_id not yet assigned), 15.48mb total:"
|
||||
- "L0, all files 15.48mb "
|
||||
- "L0.?[100,200] |-------------------------------------L0.?-------------------------------------|"
|
||||
- "Committing partition 1:"
|
||||
- " Soft Deleting 2 files: L0.21, L0.22"
|
||||
- " Creating 1 files at level CompactionLevel::L0"
|
||||
- "**** Simulation run 9, type=split(split_times=[180]). 3 Input Files, 90mb total:"
|
||||
- "L0 "
|
||||
- "L0.25[100,200] 15.48mb|------------------------------------L0.25-------------------------------------|"
|
||||
- "L0.24[180,200] 14.9mb |----L0.24-----|"
|
||||
- "L0.23[100,180] 59.62mb|----------------------------L0.23-----------------------------| "
|
||||
- "**** 2 Output Files (parquet_file_id not yet assigned), 90mb total:"
|
||||
- "L1 "
|
||||
- "L1.?[100,180] 72mb |-----------------------------L1.?-----------------------------| "
|
||||
- "L1.?[180,200] 18mb |-----L1.?-----|"
|
||||
- "Committing partition 1:"
|
||||
- " Soft Deleting 3 files: L0.23, L0.24, L0.25"
|
||||
- " Creating 2 files at level CompactionLevel::L1"
|
||||
- "**** Final Output Files "
|
||||
- "L1 "
|
||||
- "L1.26[100,180] 72mb |----------------------------L1.26-----------------------------| "
|
||||
- "L1.27[180,200] 18mb |----L1.27-----|"
|
||||
"###
|
||||
);
|
||||
}
|
||||
|
||||
/// This test show how the max_input_parquet_bytes_per_partition works
|
||||
#[tokio::test]
|
||||
async fn all_overlapping_l0_max_input_bytes_per_partition() {
|
||||
test_helpers::maybe_start_logging();
|
||||
|
||||
// limit the plan to combining 30 mb at a time
|
||||
let setup = layout_setup_builder()
|
||||
.await
|
||||
.with_max_input_parquet_bytes_per_partition(30 * ONE_MB as usize)
|
||||
.build()
|
||||
.await;
|
||||
|
||||
// create virtual files
|
||||
let setup = all_overlapping_l0_files(setup).await;
|
||||
|
||||
// expect that each plan the compactor runs has at most three input files
|
||||
insta::assert_yaml_snapshot!(
|
||||
run_layout_scenario(&setup).await,
|
||||
@r###"
|
||||
---
|
||||
- "**** Input Files "
|
||||
- "L0, all files 9mb "
|
||||
- "L0.1[100,200] |-------------------------------------L0.1-------------------------------------|"
|
||||
- "L0.2[100,200] |-------------------------------------L0.2-------------------------------------|"
|
||||
- "L0.3[100,200] |-------------------------------------L0.3-------------------------------------|"
|
||||
- "L0.4[100,200] |-------------------------------------L0.4-------------------------------------|"
|
||||
- "L0.5[100,200] |-------------------------------------L0.5-------------------------------------|"
|
||||
- "L0.6[100,200] |-------------------------------------L0.6-------------------------------------|"
|
||||
- "L0.7[100,200] |-------------------------------------L0.7-------------------------------------|"
|
||||
- "L0.8[100,200] |-------------------------------------L0.8-------------------------------------|"
|
||||
- "L0.9[100,200] |-------------------------------------L0.9-------------------------------------|"
|
||||
- "L0.10[100,200] |------------------------------------L0.10-------------------------------------|"
|
||||
- "SKIPPED COMPACTION for PartitionId(1): partition 1 has 94371840 parquet file bytes, limit is 31457280"
|
||||
- "**** Final Output Files "
|
||||
- "L0, all files 9mb "
|
||||
- "L0.1[100,200] |-------------------------------------L0.1-------------------------------------|"
|
||||
- "L0.2[100,200] |-------------------------------------L0.2-------------------------------------|"
|
||||
- "L0.3[100,200] |-------------------------------------L0.3-------------------------------------|"
|
||||
- "L0.4[100,200] |-------------------------------------L0.4-------------------------------------|"
|
||||
- "L0.5[100,200] |-------------------------------------L0.5-------------------------------------|"
|
||||
- "L0.6[100,200] |-------------------------------------L0.6-------------------------------------|"
|
||||
- "L0.7[100,200] |-------------------------------------L0.7-------------------------------------|"
|
||||
- "L0.8[100,200] |-------------------------------------L0.8-------------------------------------|"
|
||||
- "L0.9[100,200] |-------------------------------------L0.9-------------------------------------|"
|
||||
- "L0.10[100,200] |------------------------------------L0.10-------------------------------------|"
|
||||
"###
|
||||
);
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in New Issue