test: reproducer for bug 17221 (#7137)
Nga Tran
GitHub
parent
ed0704ac8d
commit
97bf8d3a89
|
|
@ -3,9 +3,229 @@
|
|||
//! See [crate::layout] module for detailed documentation
|
||||
|
||||
use data_types::CompactionLevel;
|
||||
use iox_time::Time;
|
||||
|
||||
use crate::layouts::{layout_setup_builder, parquet_builder, run_layout_scenario, ONE_MB};
|
||||
|
||||
#[tokio::test]
|
||||
async fn many_l0_files_different_created_order() {
|
||||
test_helpers::maybe_start_logging();
|
||||
|
||||
let setup = layout_setup_builder()
|
||||
.await
|
||||
// compact at most 2 L0 files per plan
|
||||
.with_max_num_files_per_plan(2)
|
||||
.build()
|
||||
.await;
|
||||
|
||||
// L0.1
|
||||
setup
|
||||
.partition
|
||||
.create_parquet_file(
|
||||
parquet_builder()
|
||||
.with_min_time(10)
|
||||
.with_max_time(22)
|
||||
.with_compaction_level(CompactionLevel::Initial)
|
||||
.with_max_l0_created_at(Time::from_timestamp_nanos(1)),
|
||||
)
|
||||
.await;
|
||||
|
||||
// L0.2
|
||||
setup
|
||||
.partition
|
||||
.create_parquet_file(
|
||||
parquet_builder()
|
||||
.with_min_time(30)
|
||||
.with_max_time(42)
|
||||
.with_compaction_level(CompactionLevel::Initial)
|
||||
.with_max_l0_created_at(Time::from_timestamp_nanos(2)),
|
||||
)
|
||||
.await;
|
||||
|
||||
// L0.3
|
||||
setup
|
||||
.partition
|
||||
.create_parquet_file(
|
||||
parquet_builder()
|
||||
.with_min_time(20)
|
||||
.with_max_time(32)
|
||||
.with_compaction_level(CompactionLevel::Initial)
|
||||
.with_max_l0_created_at(Time::from_timestamp_nanos(3)),
|
||||
)
|
||||
.await;
|
||||
|
||||
// L0.4
|
||||
setup
|
||||
.partition
|
||||
.create_parquet_file(
|
||||
parquet_builder()
|
||||
.with_min_time(40)
|
||||
.with_max_time(52)
|
||||
.with_compaction_level(CompactionLevel::Initial)
|
||||
.with_max_l0_created_at(Time::from_timestamp_nanos(4)),
|
||||
)
|
||||
.await;
|
||||
|
||||
// L0 files must be compacted in their max_l0_created_at order to have the right deduplication
|
||||
// L0.1 and L0.2 must be compacted together before they are compacted with L0.3 and L0.4
|
||||
insta::assert_yaml_snapshot!(
|
||||
run_layout_scenario(&setup).await,
|
||||
@r###"
|
||||
---
|
||||
- "**** Input Files "
|
||||
- "L0, all files 2.55kb "
|
||||
- "L0.1[10,22] |--------L0.1--------| "
|
||||
- "L0.2[30,42] |--------L0.2--------| "
|
||||
- "L0.3[20,32] |--------L0.3--------| "
|
||||
- "L0.4[40,52] |--------L0.4--------| "
|
||||
- "**** Simulation run 0, type=compact. 2 Input Files, 5.1kb total:"
|
||||
- "L0, all files 2.55kb "
|
||||
- "L0.1[10,22] |------------L0.1------------| "
|
||||
- "L0.2[30,42] |------------L0.2------------|"
|
||||
- "**** 1 Output Files (parquet_file_id not yet assigned), 5.1kb total:"
|
||||
- "L0, all files 5.1kb "
|
||||
- "L0.?[10,42] |-------------------------------------L0.?-------------------------------------|"
|
||||
- "Committing partition 1:"
|
||||
- " Soft Deleting 2 files: L0.1, L0.2"
|
||||
- " Creating 1 files at level CompactionLevel::L0"
|
||||
- "**** Simulation run 1, type=compact. 2 Input Files, 5.1kb total:"
|
||||
- "L0, all files 2.55kb "
|
||||
- "L0.3[20,32] |------------L0.3------------| "
|
||||
- "L0.4[40,52] |------------L0.4------------|"
|
||||
- "**** 1 Output Files (parquet_file_id not yet assigned), 5.1kb total:"
|
||||
- "L0, all files 5.1kb "
|
||||
- "L0.?[20,52] |-------------------------------------L0.?-------------------------------------|"
|
||||
- "Committing partition 1:"
|
||||
- " Soft Deleting 2 files: L0.3, L0.4"
|
||||
- " Creating 1 files at level CompactionLevel::L0"
|
||||
- "**** Simulation run 2, type=compact. 2 Input Files, 10.2kb total:"
|
||||
- "L0, all files 5.1kb "
|
||||
- "L0.6[20,52] |---------------------------L0.6---------------------------| "
|
||||
- "L0.5[10,42] |---------------------------L0.5---------------------------| "
|
||||
- "**** 1 Output Files (parquet_file_id not yet assigned), 10.2kb total:"
|
||||
- "L1, all files 10.2kb "
|
||||
- "L1.?[10,52] |-------------------------------------L1.?-------------------------------------|"
|
||||
- "Committing partition 1:"
|
||||
- " Soft Deleting 2 files: L0.5, L0.6"
|
||||
- " Creating 1 files at level CompactionLevel::L1"
|
||||
- "**** Final Output Files "
|
||||
- "L1, all files 10.2kb "
|
||||
- "L1.7[10,52] |-------------------------------------L1.7-------------------------------------|"
|
||||
"###
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn many_l1_files_different_created_order() {
|
||||
test_helpers::maybe_start_logging();
|
||||
|
||||
let setup = layout_setup_builder()
|
||||
.await
|
||||
.with_min_num_l1_files_to_compact(2)
|
||||
.with_max_num_files_per_plan(2)
|
||||
.build()
|
||||
.await;
|
||||
|
||||
// L1.1
|
||||
setup
|
||||
.partition
|
||||
.create_parquet_file(
|
||||
parquet_builder()
|
||||
.with_min_time(11)
|
||||
.with_max_time(20)
|
||||
.with_compaction_level(CompactionLevel::FileNonOverlapped)
|
||||
.with_max_l0_created_at(Time::from_timestamp_nanos(1)),
|
||||
)
|
||||
.await;
|
||||
|
||||
// L1.2
|
||||
setup
|
||||
.partition
|
||||
.create_parquet_file(
|
||||
parquet_builder()
|
||||
.with_min_time(31)
|
||||
.with_max_time(40)
|
||||
.with_compaction_level(CompactionLevel::FileNonOverlapped)
|
||||
.with_max_l0_created_at(Time::from_timestamp_nanos(2)),
|
||||
)
|
||||
.await;
|
||||
|
||||
// L1.3
|
||||
setup
|
||||
.partition
|
||||
.create_parquet_file(
|
||||
parquet_builder()
|
||||
.with_min_time(21)
|
||||
.with_max_time(30)
|
||||
.with_compaction_level(CompactionLevel::FileNonOverlapped)
|
||||
.with_max_l0_created_at(Time::from_timestamp_nanos(3)),
|
||||
)
|
||||
.await;
|
||||
|
||||
// L1.4
|
||||
setup
|
||||
.partition
|
||||
.create_parquet_file(
|
||||
parquet_builder()
|
||||
.with_min_time(41)
|
||||
.with_max_time(50)
|
||||
.with_compaction_level(CompactionLevel::FileNonOverlapped)
|
||||
.with_max_l0_created_at(Time::from_timestamp_nanos(4)),
|
||||
)
|
||||
.await;
|
||||
|
||||
// All L1 files are non-overlapped. Copacting them together must kep them non-overlapped.
|
||||
// If we cannot compact them together in one run, they must be split in the order of their min_time (or max_time)
|
||||
// to ensure that the output files are non-overlapped.
|
||||
// BUG: reproducer of https://github.com/influxdata/idpe/issues/17221
|
||||
// L1.1 must be compacted with L1.3 instead of L1.2
|
||||
insta::assert_yaml_snapshot!(
|
||||
run_layout_scenario(&setup).await,
|
||||
@r###"
|
||||
---
|
||||
- "**** Input Files "
|
||||
- "L1, all files 2.55kb "
|
||||
- "L1.1[11,20] |------L1.1------| "
|
||||
- "L1.2[31,40] |------L1.2------| "
|
||||
- "L1.3[21,30] |------L1.3------| "
|
||||
- "L1.4[41,50] |------L1.4------| "
|
||||
- "**** Simulation run 0, type=compact. 2 Input Files, 5.1kb total:"
|
||||
- "L1, all files 2.55kb "
|
||||
- "L1.1[11,20] |---------L1.1---------| "
|
||||
- "L1.2[31,40] |---------L1.2---------| "
|
||||
- "**** 1 Output Files (parquet_file_id not yet assigned), 5.1kb total:"
|
||||
- "L1, all files 5.1kb "
|
||||
- "L1.?[11,40] |-------------------------------------L1.?-------------------------------------|"
|
||||
- "Committing partition 1:"
|
||||
- " Soft Deleting 2 files: L1.1, L1.2"
|
||||
- " Creating 1 files at level CompactionLevel::L1"
|
||||
- "**** Simulation run 1, type=compact. 2 Input Files, 5.1kb total:"
|
||||
- "L1, all files 2.55kb "
|
||||
- "L1.3[21,30] |---------L1.3---------| "
|
||||
- "L1.4[41,50] |---------L1.4---------| "
|
||||
- "**** 1 Output Files (parquet_file_id not yet assigned), 5.1kb total:"
|
||||
- "L1, all files 5.1kb "
|
||||
- "L1.?[21,50] |-------------------------------------L1.?-------------------------------------|"
|
||||
- "Committing partition 1:"
|
||||
- " Soft Deleting 2 files: L1.3, L1.4"
|
||||
- " Creating 1 files at level CompactionLevel::L1"
|
||||
- "**** Simulation run 2, type=compact. 2 Input Files, 10.2kb total:"
|
||||
- "L1, all files 5.1kb "
|
||||
- "L1.6[21,50] |--------------------------L1.6---------------------------| "
|
||||
- "L1.5[11,40] |--------------------------L1.5---------------------------| "
|
||||
- "**** 1 Output Files (parquet_file_id not yet assigned), 10.2kb total:"
|
||||
- "L2, all files 10.2kb "
|
||||
- "L2.?[11,50] |-------------------------------------L2.?-------------------------------------|"
|
||||
- "Committing partition 1:"
|
||||
- " Soft Deleting 2 files: L1.5, L1.6"
|
||||
- " Creating 1 files at level CompactionLevel::L2"
|
||||
- "**** Final Output Files "
|
||||
- "L2, all files 10.2kb "
|
||||
- "L2.7[11,50] |-------------------------------------L2.7-------------------------------------|"
|
||||
"###
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn many_l1_files() {
|
||||
test_helpers::maybe_start_logging();
|
||||
|
|
|
|||
Loading…
Reference in New Issue