Influxdata
/
influxdb
mirror of https://github.com/influxdata/influxdb.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

test(compactor): Improve file size display in compactor tests (#6909) 

* test(compactor): Improve file size display in compactor tests

* fix: try different display

* fix: update

* fix: use b to show size
pull/24376/head
Andrew Lamb 2023-02-08 23:35:12 +01:00 committed by GitHub
parent 18d5924dfd
commit bd2a72a4b6
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 173 additions and 186 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

58
compactor2/src/components/files_split/target_level_non_overlap_split.rs
View File

@ -131,6 +131,7 @@ mod tests {
use crate::test_util::{ use crate::test_util::{
create_l1_files, create_overlapped_files, create_overlapped_files_2, create_l1_files, create_overlapped_files, create_overlapped_files_2,
create_overlapped_l0_l1_files, create_overlapped_l1_l2_files, format_files, create_overlapped_l0_l1_files, create_overlapped_l1_l2_files, format_files,
format_files_split,
}; };
use super::*; use super::*;
@ -191,7 +192,7 @@ mod tests {
@r###" @r###"
--- ---
- initial - initial
- "L1 " - "L1, all files 1b "
- "L1.13[600,700] |-----L1.13-----| " - "L1.13[600,700] |-----L1.13-----| "
- "L1.12[400,500] |-----L1.12-----| " - "L1.12[400,500] |-----L1.12-----| "
- "L1.11[250,350] |-----L1.11-----| " - "L1.11[250,350] |-----L1.11-----| "
@ -219,11 +220,11 @@ mod tests {
@r###" @r###"
--- ---
- initial - initial
- "L0 " - "L0, all files 1b "
- "L0.2[650,750] |---L0.2---| " - "L0.2[650,750] |---L0.2---| "
- "L0.1[450,620] |-------L0.1-------| " - "L0.1[450,620] |-------L0.1-------| "
- "L0.3[800,900] |---L0.3---| " - "L0.3[800,900] |---L0.3---| "
- "L1 " - "L1, all files 1b "
- "L1.13[600,700] |--L1.13---| " - "L1.13[600,700] |--L1.13---| "
- "L1.12[400,500] |--L1.12---| " - "L1.12[400,500] |--L1.12---| "
- "L1.11[250,350] |--L1.11---| " - "L1.11[250,350] |--L1.11---| "
@ -233,26 +234,19 @@ mod tests {
let split = TargetLevelNonOverlapSplit::new(); let split = TargetLevelNonOverlapSplit::new();
let (overlap, non_overlap) = split.apply(files, CompactionLevel::FileNonOverlapped); let (overlap, non_overlap) = split.apply(files, CompactionLevel::FileNonOverlapped);
insta::assert_yaml_snapshot!( insta::assert_yaml_snapshot!(
format_files("overlap", &overlap), format_files_split("overlap", &overlap, "non_overlap", &non_overlap),
@r###" @r###"
--- ---
- overlap - overlap
- "L0 " - "L0, all files 1b "
- "L0.2[650,750] |-----L0.2-----| " - "L0.2[650,750] |-----L0.2-----| "
- "L0.1[450,620] |----------L0.1-----------| " - "L0.1[450,620] |----------L0.1-----------| "
- "L0.3[800,900] |-----L0.3-----|" - "L0.3[800,900] |-----L0.3-----|"
- "L1 " - "L1, all files 1b "
- "L1.12[400,500] |----L1.12-----| " - "L1.12[400,500] |----L1.12-----| "
- "L1.13[600,700] |----L1.13-----| " - "L1.13[600,700] |----L1.13-----| "
"###
);
// verify non-overlapping files
insta::assert_yaml_snapshot!(
format_files("non_overlap", &non_overlap),
@r###"
---
- non_overlap - non_overlap
- "L1 " - "L1, all files 1b "
- "L1.11[250,350] |------------------------------------L1.11-------------------------------------|" - "L1.11[250,350] |------------------------------------L1.11-------------------------------------|"
"### "###
); );
@ -270,11 +264,11 @@ mod tests {
@r###" @r###"
--- ---
- initial - initial
- "L1 " - "L1, all files 1b "
- "L1.13[600,700] |--L1.13--| " - "L1.13[600,700] |--L1.13--| "
- "L1.12[400,500] |--L1.12--| " - "L1.12[400,500] |--L1.12--| "
- "L1.11[250,350] |--L1.11--| " - "L1.11[250,350] |--L1.11--| "
- "L2 " - "L2, all files 1b "
- "L2.21[0,100] |--L2.21--| " - "L2.21[0,100] |--L2.21--| "
- "L2.22[200,300] |--L2.22--| " - "L2.22[200,300] |--L2.22--| "
"### "###
@ -283,24 +277,18 @@ mod tests {
let split = TargetLevelNonOverlapSplit::new(); let split = TargetLevelNonOverlapSplit::new();
let (overlap, non_overlap) = split.apply(files, CompactionLevel::Final); let (overlap, non_overlap) = split.apply(files, CompactionLevel::Final);
insta::assert_yaml_snapshot!( insta::assert_yaml_snapshot!(
format_files("overlap", &overlap), format_files_split("overlap", &overlap, "non_overlap", &non_overlap),
@r###" @r###"
--- ---
- overlap - overlap
- "L1 " - "L1, all files 1b "
- "L1.13[600,700] |----L1.13-----|" - "L1.13[600,700] |----L1.13-----|"
- "L1.12[400,500] |----L1.12-----| " - "L1.12[400,500] |----L1.12-----| "
- "L1.11[250,350] |----L1.11-----| " - "L1.11[250,350] |----L1.11-----| "
- "L2 " - "L2, all files 1b "
- "L2.22[200,300] |----L2.22-----| " - "L2.22[200,300] |----L2.22-----| "
"###
);
insta::assert_yaml_snapshot!(
format_files("non_overlap", &non_overlap),
@r###"
---
- non_overlap - non_overlap
- "L2 " - "L2, all files 1b "
- "L2.21[0,100] |------------------------------------L2.21-------------------------------------|" - "L2.21[0,100] |------------------------------------L2.21-------------------------------------|"
"### "###
); );
@ -320,10 +308,10 @@ mod tests {
@r###" @r###"
--- ---
- initial - initial
- "L0 " - "L0, all files 1b "
- "L0.2[520,550] |L0.2| " - "L0.2[520,550] |L0.2| "
- "L0.1[250,350] |--L0.1---| " - "L0.1[250,350] |--L0.1---| "
- "L1 " - "L1, all files 1b "
- "L1.13[400,500] |--L1.13--| " - "L1.13[400,500] |--L1.13--| "
- "L1.12[200,300] |--L1.12--| " - "L1.12[200,300] |--L1.12--| "
- "L1.11[0,100] |--L1.11--| " - "L1.11[0,100] |--L1.11--| "
@ -334,24 +322,18 @@ mod tests {
let split = TargetLevelNonOverlapSplit::new(); let split = TargetLevelNonOverlapSplit::new();
let (overlap, non_overlap) = split.apply(files, CompactionLevel::FileNonOverlapped); let (overlap, non_overlap) = split.apply(files, CompactionLevel::FileNonOverlapped);
insta::assert_yaml_snapshot!( insta::assert_yaml_snapshot!(
format_files("overlap", &overlap), format_files_split("overlap", &overlap, "non_overlap", &non_overlap),
@r###" @r###"
--- ---
- overlap - overlap
- "L0 " - "L0, all files 1b "
- "L0.2[520,550] |L0.2| " - "L0.2[520,550] |L0.2| "
- "L0.1[250,350] |--------L0.1--------| " - "L0.1[250,350] |--------L0.1--------| "
- "L1 " - "L1, all files 1b "
- "L1.12[200,300] |-------L1.12--------| " - "L1.12[200,300] |-------L1.12--------| "
- "L1.13[400,500] |-------L1.13--------| " - "L1.13[400,500] |-------L1.13--------| "
"###
);
insta::assert_yaml_snapshot!(
format_files("non_overlap", &non_overlap),
@r###"
---
- non_overlap - non_overlap
- "L1 " - "L1, all files 1b "
- "L1.11[0,100] |--L1.11--| " - "L1.11[0,100] |--L1.11--| "
- "L1.14[600,700] |--L1.14--| " - "L1.14[600,700] |--L1.14--| "
"### "###

84
compactor2/src/components/files_split/target_level_target_level_split.rs
View File

@ -68,7 +68,7 @@ mod tests {
@r###" @r###"
--- ---
- initial - initial
- "L0 " - "L0, all files 1b "
- "L0.2[650,750] |-----L0.2------| " - "L0.2[650,750] |-----L0.2------| "
- "L0.1[450,620] |------------L0.1------------| " - "L0.1[450,620] |------------L0.1------------| "
- "L0.3[800,900] |-----L0.3------| " - "L0.3[800,900] |-----L0.3------| "
@ -97,7 +97,7 @@ mod tests {
@r###" @r###"
--- ---
- initial - initial
- "L1 " - "L1, all files 1b "
- "L1.13[600,700] |-----L1.13-----| " - "L1.13[600,700] |-----L1.13-----| "
- "L1.12[400,500] |-----L1.12-----| " - "L1.12[400,500] |-----L1.12-----| "
- "L1.11[250,350] |-----L1.11-----| " - "L1.11[250,350] |-----L1.11-----| "
@ -126,7 +126,7 @@ mod tests {
@r###" @r###"
--- ---
- initial - initial
- "L2 " - "L2, all files 1b "
- "L2.21[0,100] |---------L2.21----------| " - "L2.21[0,100] |---------L2.21----------| "
- "L2.22[200,300] |---------L2.22----------| " - "L2.22[200,300] |---------L2.22----------| "
"### "###
@ -156,16 +156,16 @@ mod tests {
--- ---
- initial - initial
- "L0 " - "L0 "
- "L0.2[650,750]@1 |-L0.2-| " - "L0.2[650,750] 1b |-L0.2-| "
- "L0.1[450,620]@1 |----L0.1-----| " - "L0.1[450,620] 1b |----L0.1-----| "
- "L0.3[800,900]@100 |-L0.3-| " - "L0.3[800,900] 100b |-L0.3-| "
- "L1 " - "L1 "
- "L1.13[600,700]@100 |L1.13-| " - "L1.13[600,700] 100b |L1.13-| "
- "L1.12[400,500]@1 |L1.12-| " - "L1.12[400,500] 1b |L1.12-| "
- "L1.11[250,350]@1 |L1.11-| " - "L1.11[250,350] 1b |L1.11-| "
- "L2 " - "L2 "
- "L2.21[0,100]@1 |L2.21-| " - "L2.21[0,100] 1b |L2.21-| "
- "L2.22[200,300]@1 |L2.22-| " - "L2.22[200,300] 1b |L2.22-| "
"### "###
); );
@ -178,17 +178,17 @@ mod tests {
--- ---
- lower - lower
- "L0 " - "L0 "
- "L0.2[650,750]@1 |-----L0.2------| " - "L0.2[650,750] 1b |-----L0.2------| "
- "L0.1[450,620]@1 |------------L0.1------------| " - "L0.1[450,620] 1b |------------L0.1------------| "
- "L0.3[800,900]@100 |-----L0.3------| " - "L0.3[800,900] 100b |-----L0.3------| "
- higher - higher
- "L1 " - "L1 "
- "L1.13[600,700]@100 |--L1.13--| " - "L1.13[600,700] 100b |--L1.13--| "
- "L1.12[400,500]@1 |--L1.12--| " - "L1.12[400,500] 1b |--L1.12--| "
- "L1.11[250,350]@1 |--L1.11--| " - "L1.11[250,350] 1b |--L1.11--| "
- "L2 " - "L2 "
- "L2.21[0,100]@1 |--L2.21--| " - "L2.21[0,100] 1b |--L2.21--| "
- "L2.22[200,300]@1 |--L2.22--| " - "L2.22[200,300] 1b |--L2.22--| "
"### "###
); );
@ -215,16 +215,16 @@ mod tests {
--- ---
- initial - initial
- "L0 " - "L0 "
- "L0.2[650,750]@1 |-L0.2-| " - "L0.2[650,750] 1b |-L0.2-| "
- "L0.1[450,620]@1 |----L0.1-----| " - "L0.1[450,620] 1b |----L0.1-----| "
- "L0.3[800,900]@100 |-L0.3-| " - "L0.3[800,900] 100b |-L0.3-| "
- "L1 " - "L1 "
- "L1.13[600,700]@100 |L1.13-| " - "L1.13[600,700] 100b |L1.13-| "
- "L1.12[400,500]@1 |L1.12-| " - "L1.12[400,500] 1b |L1.12-| "
- "L1.11[250,350]@1 |L1.11-| " - "L1.11[250,350] 1b |L1.11-| "
- "L2 " - "L2 "
- "L2.21[0,100]@1 |L2.21-| " - "L2.21[0,100] 1b |L2.21-| "
- "L2.22[200,300]@1 |L2.22-| " - "L2.22[200,300] 1b |L2.22-| "
"### "###
); );
@ -237,15 +237,15 @@ mod tests {
--- ---
- lower - lower
- "L0 " - "L0 "
- "L0.2[650,750]@1 |---L0.2---| " - "L0.2[650,750] 1b |---L0.2---| "
- "L0.1[450,620]@1 |-------L0.1-------| " - "L0.1[450,620] 1b |-------L0.1-------| "
- "L0.3[800,900]@100 |---L0.3---| " - "L0.3[800,900] 100b |---L0.3---| "
- "L1 " - "L1 "
- "L1.13[600,700]@100 |--L1.13---| " - "L1.13[600,700] 100b |--L1.13---| "
- "L1.12[400,500]@1 |--L1.12---| " - "L1.12[400,500] 1b |--L1.12---| "
- "L1.11[250,350]@1 |--L1.11---| " - "L1.11[250,350] 1b |--L1.11---| "
- higher - higher
- "L2 " - "L2, all files 1b "
- "L2.21[0,100] |---------L2.21----------| " - "L2.21[0,100] |---------L2.21----------| "
- "L2.22[200,300] |---------L2.22----------| " - "L2.22[200,300] |---------L2.22----------| "
"### "###
@ -274,16 +274,16 @@ mod tests {
--- ---
- initial - initial
- "L0 " - "L0 "
- "L0.2[650,750]@1 |-L0.2-| " - "L0.2[650,750] 1b |-L0.2-| "
- "L0.1[450,620]@1 |----L0.1-----| " - "L0.1[450,620] 1b |----L0.1-----| "
- "L0.3[800,900]@100 |-L0.3-| " - "L0.3[800,900] 100b |-L0.3-| "
- "L1 " - "L1 "
- "L1.13[600,700]@100 |L1.13-| " - "L1.13[600,700] 100b |L1.13-| "
- "L1.12[400,500]@1 |L1.12-| " - "L1.12[400,500] 1b |L1.12-| "
- "L1.11[250,350]@1 |L1.11-| " - "L1.11[250,350] 1b |L1.11-| "
- "L2 " - "L2 "
- "L2.21[0,100]@1 |L2.21-| " - "L2.21[0,100] 1b |L2.21-| "
- "L2.22[200,300]@1 |L2.22-| " - "L2.22[200,300] 1b |L2.22-| "
"### "###
); );

174
compactor2/src/components/files_split/target_level_upgrade_split.rs
View File

@ -209,7 +209,7 @@ mod tests {
@r###" @r###"
--- ---
- initial - initial
- "L0 " - "L0, all files 99b "
- "L0.2[150,180] |L0.2| " - "L0.2[150,180] |L0.2| "
- "L0.1[100,200] |--L0.1--| " - "L0.1[100,200] |--L0.1--| "
- "L0.3[800,900] |--L0.3--|" - "L0.3[800,900] |--L0.3--|"
@ -226,7 +226,7 @@ mod tests {
@r###" @r###"
--- ---
- files_to_compact - files_to_compact
- "L0 " - "L0, all files 99b "
- "L0.3[800,900] |--L0.3--|" - "L0.3[800,900] |--L0.3--|"
- "L0.1[100,200] |--L0.1--| " - "L0.1[100,200] |--L0.1--| "
- "L0.2[150,180] |L0.2| " - "L0.2[150,180] |L0.2| "
@ -243,7 +243,7 @@ mod tests {
@r###" @r###"
--- ---
- initial - initial
- "L0 " - "L0, all files 101b "
- "L0.2[150,180] |L0.2| " - "L0.2[150,180] |L0.2| "
- "L0.1[100,200] |--L0.1--| " - "L0.1[100,200] |--L0.1--| "
- "L0.3[800,900] |--L0.3--|" - "L0.3[800,900] |--L0.3--|"
@ -262,11 +262,11 @@ mod tests {
@r###" @r###"
--- ---
- files_to_compact - files_to_compact
- "L0 " - "L0, all files 101b "
- "L0.1[100,200] |-------------------------------------L0.1-------------------------------------|" - "L0.1[100,200] |-------------------------------------L0.1-------------------------------------|"
- "L0.2[150,180] |---------L0.2---------| " - "L0.2[150,180] |---------L0.2---------| "
- files_to_upgrade - files_to_upgrade
- "L0 " - "L0, all files 101b "
- "L0.3[800,900] |-------------------------------------L0.3-------------------------------------|" - "L0.3[800,900] |-------------------------------------L0.3-------------------------------------|"
"### "###
); );
@ -280,7 +280,7 @@ mod tests {
@r###" @r###"
--- ---
- initial - initial
- "L0 " - "L0, all files 99b "
- "L0.2[650,750] |-----L0.2------| " - "L0.2[650,750] |-----L0.2------| "
- "L0.1[450,620] |------------L0.1------------| " - "L0.1[450,620] |------------L0.1------------| "
- "L0.3[800,900] |-----L0.3------| " - "L0.3[800,900] |-----L0.3------| "
@ -296,7 +296,7 @@ mod tests {
@r###" @r###"
--- ---
- files_to_compact - files_to_compact
- "L0 " - "L0, all files 99b "
- "L0.3[800,900] |-----L0.3------| " - "L0.3[800,900] |-----L0.3------| "
- "L0.1[450,620] |------------L0.1------------| " - "L0.1[450,620] |------------L0.1------------| "
- "L0.2[650,750] |-----L0.2------| " - "L0.2[650,750] |-----L0.2------| "
@ -313,7 +313,7 @@ mod tests {
@r###" @r###"
--- ---
- initial - initial
- "L0 " - "L0, all files 101b "
- "L0.2[650,750] |-----L0.2------| " - "L0.2[650,750] |-----L0.2------| "
- "L0.1[450,620] |------------L0.1------------| " - "L0.1[450,620] |------------L0.1------------| "
- "L0.3[800,900] |-----L0.3------| " - "L0.3[800,900] |-----L0.3------| "
@ -330,7 +330,7 @@ mod tests {
--- ---
- files_to_compact - files_to_compact
- files_to_upgrade - files_to_upgrade
- "L0 " - "L0, all files 101b "
- "L0.2[650,750] |-----L0.2------| " - "L0.2[650,750] |-----L0.2------| "
- "L0.1[450,620] |------------L0.1------------| " - "L0.1[450,620] |------------L0.1------------| "
- "L0.3[800,900] |-----L0.3------| " - "L0.3[800,900] |-----L0.3------| "
@ -346,7 +346,7 @@ mod tests {
@r###" @r###"
--- ---
- initial - initial
- "L1 " - "L1, all files 99b "
- "L1.13[600,700] |-----L1.13-----| " - "L1.13[600,700] |-----L1.13-----| "
- "L1.12[400,500] |-----L1.12-----| " - "L1.12[400,500] |-----L1.12-----| "
- "L1.11[250,350] |-----L1.11-----| " - "L1.11[250,350] |-----L1.11-----| "
@ -362,7 +362,7 @@ mod tests {
@r###" @r###"
--- ---
- files_to_compact - files_to_compact
- "L1 " - "L1, all files 99b "
- "L1.13[600,700] |-----L1.13-----| " - "L1.13[600,700] |-----L1.13-----| "
- "L1.12[400,500] |-----L1.12-----| " - "L1.12[400,500] |-----L1.12-----| "
- "L1.11[250,350] |-----L1.11-----| " - "L1.11[250,350] |-----L1.11-----| "
@ -384,7 +384,7 @@ mod tests {
--- ---
- files_to_compact - files_to_compact
- files_to_upgrade - files_to_upgrade
- "L1 " - "L1, all files 101b "
- "L1.13[600,700] |-----L1.13-----| " - "L1.13[600,700] |-----L1.13-----| "
- "L1.12[400,500] |-----L1.12-----| " - "L1.12[400,500] |-----L1.12-----| "
- "L1.11[250,350] |-----L1.11-----| " - "L1.11[250,350] |-----L1.11-----| "
@ -407,11 +407,11 @@ mod tests {
--- ---
- initial - initial
- "L1 " - "L1 "
- "L1.15[1000,1100]@200 |-L1.15-| " - "L1.15[1000,1100] 200b |-L1.15-| "
- "L1.13[600,700]@90 |-L1.13-| " - "L1.13[600,700] 90b |-L1.13-| "
- "L1.12[400,500]@101 |-L1.12-| " - "L1.12[400,500] 101b |-L1.12-| "
- "L1.11[250,350]@99 |-L1.11-| " - "L1.11[250,350] 99b |-L1.11-| "
- "L1.14[800,900]@100 |-L1.14-| " - "L1.14[800,900] 100b |-L1.14-| "
"### "###
); );
@ -425,13 +425,13 @@ mod tests {
--- ---
- files_to_compact - files_to_compact
- "L1 " - "L1 "
- "L1.11[250,350]@99 |-----L1.11-----| " - "L1.11[250,350] 99b |-----L1.11-----| "
- "L1.13[600,700]@90 |-----L1.13-----| " - "L1.13[600,700] 90b |-----L1.13-----| "
- "L1.12[400,500]@101 |-----L1.12-----| " - "L1.12[400,500] 101b |-----L1.12-----| "
- files_to_upgrade - files_to_upgrade
- "L1 " - "L1 "
- "L1.15[1000,1100]@200 |---------L1.15----------| " - "L1.15[1000,1100] 200b |---------L1.15----------| "
- "L1.14[800,900]@100 |---------L1.14----------| " - "L1.14[800,900] 100b |---------L1.14----------| "
"### "###
); );
} }
@ -444,11 +444,11 @@ mod tests {
@r###" @r###"
--- ---
- initial - initial
- "L0 " - "L0, all files 99b "
- "L0.2[650,750] |---L0.2---| " - "L0.2[650,750] |---L0.2---| "
- "L0.1[450,620] |-------L0.1-------| " - "L0.1[450,620] |-------L0.1-------| "
- "L0.3[800,900] |---L0.3---| " - "L0.3[800,900] |---L0.3---| "
- "L1 " - "L1, all files 99b "
- "L1.13[600,700] |--L1.13---| " - "L1.13[600,700] |--L1.13---| "
- "L1.12[400,500] |--L1.12---| " - "L1.12[400,500] |--L1.12---| "
- "L1.11[250,350] |--L1.11---| " - "L1.11[250,350] |--L1.11---| "
@ -465,11 +465,11 @@ mod tests {
@r###" @r###"
--- ---
- files_to_compact - files_to_compact
- "L0 " - "L0, all files 99b "
- "L0.3[800,900] |---L0.3---| " - "L0.3[800,900] |---L0.3---| "
- "L0.1[450,620] |-------L0.1-------| " - "L0.1[450,620] |-------L0.1-------| "
- "L0.2[650,750] |---L0.2---| " - "L0.2[650,750] |---L0.2---| "
- "L1 " - "L1, all files 99b "
- "L1.13[600,700] |--L1.13---| " - "L1.13[600,700] |--L1.13---| "
- "L1.12[400,500] |--L1.12---| " - "L1.12[400,500] |--L1.12---| "
- "L1.11[250,350] |--L1.11---| " - "L1.11[250,350] |--L1.11---| "
@ -486,11 +486,11 @@ mod tests {
@r###" @r###"
--- ---
- initial - initial
- "L0 " - "L0, all files 100b "
- "L0.2[650,750] |---L0.2---| " - "L0.2[650,750] |---L0.2---| "
- "L0.1[450,620] |-------L0.1-------| " - "L0.1[450,620] |-------L0.1-------| "
- "L0.3[800,900] |---L0.3---| " - "L0.3[800,900] |---L0.3---| "
- "L1 " - "L1, all files 100b "
- "L1.13[600,700] |--L1.13---| " - "L1.13[600,700] |--L1.13---| "
- "L1.12[400,500] |--L1.12---| " - "L1.12[400,500] |--L1.12---| "
- "L1.11[250,350] |--L1.11---| " - "L1.11[250,350] |--L1.11---| "
@ -507,15 +507,15 @@ mod tests {
@r###" @r###"
--- ---
- files_to_compact - files_to_compact
- "L0 " - "L0, all files 100b "
- "L0.1[450,620] |----------L0.1-----------| " - "L0.1[450,620] |----------L0.1-----------| "
- "L0.2[650,750] |-----L0.2-----|" - "L0.2[650,750] |-----L0.2-----|"
- "L1 " - "L1, all files 100b "
- "L1.13[600,700] |----L1.13-----| " - "L1.13[600,700] |----L1.13-----| "
- "L1.12[400,500] |----L1.12-----| " - "L1.12[400,500] |----L1.12-----| "
- "L1.11[250,350] |----L1.11-----| " - "L1.11[250,350] |----L1.11-----| "
- files_to_upgrade - files_to_upgrade
- "L0 " - "L0, all files 100b "
- "L0.3[800,900] |-------------------------------------L0.3-------------------------------------|" - "L0.3[800,900] |-------------------------------------L0.3-------------------------------------|"
"### "###
); );
@ -533,11 +533,11 @@ mod tests {
@r###" @r###"
--- ---
- files_to_compact - files_to_compact
- "L1 " - "L1, all files 99b "
- "L1.11[250,350] |--L1.11--| " - "L1.11[250,350] |--L1.11--| "
- "L1.12[400,500] |--L1.12--| " - "L1.12[400,500] |--L1.12--| "
- "L1.13[600,700] |--L1.13--| " - "L1.13[600,700] |--L1.13--| "
- "L2 " - "L2, all files 99b "
- "L2.21[0,100] |--L2.21--| " - "L2.21[0,100] |--L2.21--| "
- "L2.22[200,300] |--L2.22--| " - "L2.22[200,300] |--L2.22--| "
- files_to_upgrade - files_to_upgrade
@ -553,11 +553,11 @@ mod tests {
@r###" @r###"
--- ---
- initial - initial
- "L1 " - "L1, all files 100b "
- "L1.13[600,700] |--L1.13--| " - "L1.13[600,700] |--L1.13--| "
- "L1.12[400,500] |--L1.12--| " - "L1.12[400,500] |--L1.12--| "
- "L1.11[250,350] |--L1.11--| " - "L1.11[250,350] |--L1.11--| "
- "L2 " - "L2, all files 100b "
- "L2.21[0,100] |--L2.21--| " - "L2.21[0,100] |--L2.21--| "
- "L2.22[200,300] |--L2.22--| " - "L2.22[200,300] |--L2.22--| "
"### "###
@ -572,13 +572,13 @@ mod tests {
@r###" @r###"
--- ---
- files_to_compact - files_to_compact
- "L1 " - "L1, all files 100b "
- "L1.11[250,350] |-------L1.11--------| " - "L1.11[250,350] |-------L1.11--------| "
- "L2 " - "L2, all files 100b "
- "L2.21[0,100] |-------L2.21--------| " - "L2.21[0,100] |-------L2.21--------| "
- "L2.22[200,300] |-------L2.22--------| " - "L2.22[200,300] |-------L2.22--------| "
- files_to_upgrade - files_to_upgrade
- "L1 " - "L1, all files 100b "
- "L1.13[600,700] |---------L1.13----------| " - "L1.13[600,700] |---------L1.13----------| "
- "L1.12[400,500] |---------L1.12----------| " - "L1.12[400,500] |---------L1.12----------| "
"### "###
@ -597,12 +597,12 @@ mod tests {
--- ---
- initial - initial
- "L1 " - "L1 "
- "L1.13[600,700]@99 |--L1.13--| " - "L1.13[600,700] 99b |--L1.13--| "
- "L1.12[400,500]@100 |--L1.12--| " - "L1.12[400,500] 100b |--L1.12--| "
- "L1.11[250,350]@100 |--L1.11--| " - "L1.11[250,350] 100b |--L1.11--| "
- "L2 " - "L2 "
- "L2.21[0,100]@100 |--L2.21--| " - "L2.21[0,100] 100b |--L2.21--| "
- "L2.22[200,300]@100 |--L2.22--| " - "L2.22[200,300] 100b |--L2.22--| "
"### "###
); );
@ -615,12 +615,12 @@ mod tests {
--- ---
- files_to_compact - files_to_compact
- "L1 " - "L1 "
- "L1.11[250,350]@100 |--L1.11--| " - "L1.11[250,350] 100b |--L1.11--| "
- "L1.13[600,700]@99 |--L1.13--| " - "L1.13[600,700] 99b |--L1.13--| "
- "L1.12[400,500]@100 |--L1.12--| " - "L1.12[400,500] 100b |--L1.12--| "
- "L2 " - "L2 "
- "L2.21[0,100]@100 |--L2.21--| " - "L2.21[0,100] 100b |--L2.21--| "
- "L2.22[200,300]@100 |--L2.22--| " - "L2.22[200,300] 100b |--L2.22--| "
- files_to_upgrade - files_to_upgrade
"### "###
); );
@ -638,12 +638,12 @@ mod tests {
--- ---
- initial - initial
- "L1 " - "L1 "
- "L1.13[600,700]@100 |--L1.13--| " - "L1.13[600,700] 100b |--L1.13--| "
- "L1.12[400,500]@99 |--L1.12--| " - "L1.12[400,500] 99b |--L1.12--| "
- "L1.11[250,350]@100 |--L1.11--| " - "L1.11[250,350] 100b |--L1.11--| "
- "L2 " - "L2 "
- "L2.21[0,100]@100 |--L2.21--| " - "L2.21[0,100] 100b |--L2.21--| "
- "L2.22[200,300]@100 |--L2.22--| " - "L2.22[200,300] 100b |--L2.22--| "
"### "###
); );
@ -655,13 +655,13 @@ mod tests {
--- ---
- files_to_compact - files_to_compact
- "L1 " - "L1 "
- "L1.11[250,350]@100 |----L1.11-----| " - "L1.11[250,350] 100b |----L1.11-----| "
- "L1.12[400,500]@99 |----L1.12-----|" - "L1.12[400,500] 99b |----L1.12-----|"
- "L2 " - "L2 "
- "L2.21[0,100]@100 |----L2.21-----| " - "L2.21[0,100] 100b |----L2.21-----| "
- "L2.22[200,300]@100 |----L2.22-----| " - "L2.22[200,300] 100b |----L2.22-----| "
- files_to_upgrade - files_to_upgrade
- "L1 " - "L1, all files 100b "
- "L1.13[600,700] |------------------------------------L1.13-------------------------------------|" - "L1.13[600,700] |------------------------------------L1.13-------------------------------------|"
"### "###
); );
@ -676,10 +676,10 @@ mod tests {
@r###" @r###"
--- ---
- initial - initial
- "L0 " - "L0, all files 100b "
- "L0.2[520,550] |L0.2| " - "L0.2[520,550] |L0.2| "
- "L0.1[250,350] |--L0.1---| " - "L0.1[250,350] |--L0.1---| "
- "L1 " - "L1, all files 100b "
- "L1.13[400,500] |--L1.13--| " - "L1.13[400,500] |--L1.13--| "
- "L1.12[200,300] |--L1.12--| " - "L1.12[200,300] |--L1.12--| "
- "L1.11[0,100] |--L1.11--| " - "L1.11[0,100] |--L1.11--| "
@ -696,10 +696,10 @@ mod tests {
@r###" @r###"
--- ---
- files_to_compact - files_to_compact
- "L0 " - "L0, all files 100b "
- "L0.1[250,350] |--L0.1---| " - "L0.1[250,350] |--L0.1---| "
- "L0.2[520,550] |L0.2| " - "L0.2[520,550] |L0.2| "
- "L1 " - "L1, all files 100b "
- "L1.13[400,500] |--L1.13--| " - "L1.13[400,500] |--L1.13--| "
- "L1.12[200,300] |--L1.12--| " - "L1.12[200,300] |--L1.12--| "
- "L1.11[0,100] |--L1.11--| " - "L1.11[0,100] |--L1.11--| "
@ -718,14 +718,14 @@ mod tests {
@r###" @r###"
--- ---
- initial - initial
- "L0 " - "L0, all files 99b "
- "L0.3[400,500] |L0.3-| " - "L0.3[400,500] |L0.3-| "
- "L0.2[200,300] |L0.2-| " - "L0.2[200,300] |L0.2-| "
- "L0.1[0,100] |L0.1-| " - "L0.1[0,100] |L0.1-| "
- "L0.4[600,700] |L0.4-| " - "L0.4[600,700] |L0.4-| "
- "L0.5[800,900] |L0.5-| " - "L0.5[800,900] |L0.5-| "
- "L0.6[1000,1100] |L0.6-| " - "L0.6[1000,1100] |L0.6-| "
- "L1 " - "L1, all files 99b "
- "L1.11[250,350] |L1.11| " - "L1.11[250,350] |L1.11| "
- "L1.12[650,750] |L1.12| " - "L1.12[650,750] |L1.12| "
"### "###
@ -740,14 +740,14 @@ mod tests {
@r###" @r###"
--- ---
- files_to_compact - files_to_compact
- "L0 " - "L0, all files 99b "
- "L0.6[1000,1100] |L0.6-| " - "L0.6[1000,1100] |L0.6-| "
- "L0.5[800,900] |L0.5-| " - "L0.5[800,900] |L0.5-| "
- "L0.4[600,700] |L0.4-| " - "L0.4[600,700] |L0.4-| "
- "L0.1[0,100] |L0.1-| " - "L0.1[0,100] |L0.1-| "
- "L0.2[200,300] |L0.2-| " - "L0.2[200,300] |L0.2-| "
- "L0.3[400,500] |L0.3-| " - "L0.3[400,500] |L0.3-| "
- "L1 " - "L1, all files 99b "
- "L1.11[250,350] |L1.11| " - "L1.11[250,350] |L1.11| "
- "L1.12[650,750] |L1.12| " - "L1.12[650,750] |L1.12| "
- files_to_upgrade - files_to_upgrade
@ -766,14 +766,14 @@ mod tests {
@r###" @r###"
--- ---
- initial - initial
- "L0 " - "L0, all files 110b "
- "L0.3[400,500] |L0.3-| " - "L0.3[400,500] |L0.3-| "
- "L0.2[200,300] |L0.2-| " - "L0.2[200,300] |L0.2-| "
- "L0.1[0,100] |L0.1-| " - "L0.1[0,100] |L0.1-| "
- "L0.4[600,700] |L0.4-| " - "L0.4[600,700] |L0.4-| "
- "L0.5[800,900] |L0.5-| " - "L0.5[800,900] |L0.5-| "
- "L0.6[1000,1100] |L0.6-| " - "L0.6[1000,1100] |L0.6-| "
- "L1 " - "L1, all files 110b "
- "L1.11[250,350] |L1.11| " - "L1.11[250,350] |L1.11| "
- "L1.12[650,750] |L1.12| " - "L1.12[650,750] |L1.12| "
"### "###
@ -788,15 +788,15 @@ mod tests {
@r###" @r###"
--- ---
- files_to_compact - files_to_compact
- "L0 " - "L0, all files 110b "
- "L0.4[600,700] |----L0.4----| " - "L0.4[600,700] |----L0.4----| "
- "L0.2[200,300] |----L0.2----| " - "L0.2[200,300] |----L0.2----| "
- "L0.3[400,500] |----L0.3----| " - "L0.3[400,500] |----L0.3----| "
- "L1 " - "L1, all files 110b "
- "L1.11[250,350] |---L1.11----| " - "L1.11[250,350] |---L1.11----| "
- "L1.12[650,750] |---L1.12----| " - "L1.12[650,750] |---L1.12----| "
- files_to_upgrade - files_to_upgrade
- "L0 " - "L0, all files 110b "
- "L0.1[0,100] |L0.1-| " - "L0.1[0,100] |L0.1-| "
- "L0.5[800,900] |L0.5-| " - "L0.5[800,900] |L0.5-| "
- "L0.6[1000,1100] |L0.6-| " - "L0.6[1000,1100] |L0.6-| "
@ -816,15 +816,15 @@ mod tests {
--- ---
- initial - initial
- "L0 " - "L0 "
- "L0.3[400,500]@100 |L0.3-| " - "L0.3[400,500] 100b |L0.3-| "
- "L0.2[200,300]@100 |L0.2-| " - "L0.2[200,300] 100b |L0.2-| "
- "L0.1[0,100]@100 |L0.1-| " - "L0.1[0,100] 100b |L0.1-| "
- "L0.4[600,700]@100 |L0.4-| " - "L0.4[600,700] 100b |L0.4-| "
- "L0.5[800,900]@100 |L0.5-| " - "L0.5[800,900] 100b |L0.5-| "
- "L0.6[1000,1100]@99 |L0.6-| " - "L0.6[1000,1100] 99b |L0.6-| "
- "L1 " - "L1 "
- "L1.11[250,350]@100 |L1.11| " - "L1.11[250,350] 100b |L1.11| "
- "L1.12[650,750]@100 |L1.12| " - "L1.12[650,750] 100b |L1.12| "
"### "###
); );
@ -838,16 +838,16 @@ mod tests {
--- ---
- files_to_compact - files_to_compact
- "L0 " - "L0 "
- "L0.6[1000,1100]@99 |-L0.6-| " - "L0.6[1000,1100] 99b |-L0.6-| "
- "L0.4[600,700]@100 |-L0.4-| " - "L0.4[600,700] 100b |-L0.4-| "
- "L0.2[200,300]@100 |-L0.2-| " - "L0.2[200,300] 100b |-L0.2-| "
- "L0.3[400,500]@100 |-L0.3-| " - "L0.3[400,500] 100b |-L0.3-| "
- "L0.5[800,900]@100 |-L0.5-| " - "L0.5[800,900] 100b |-L0.5-| "
- "L1 " - "L1 "
- "L1.11[250,350]@100 |L1.11-| " - "L1.11[250,350] 100b |L1.11-| "
- "L1.12[650,750]@100 |L1.12-| " - "L1.12[650,750] 100b |L1.12-| "
- files_to_upgrade - files_to_upgrade
- "L0 " - "L0, all files 100b "
- "L0.1[0,100] |-------------------------------------L0.1-------------------------------------|" - "L0.1[0,100] |-------------------------------------L0.1-------------------------------------|"
"### "###
); );

39
compactor2/src/test_util/display.rs
View File

@ -94,7 +94,7 @@ fn readable_list_of_files<'a>(
#[derive(Debug, Default)] #[derive(Debug, Default)]
struct ParquetFileFormatter { struct ParquetFileFormatter {
/// should the size of the files be shown (if they are different) /// should the size of the files be shown (if they are different)
show_size: bool, file_size_seen: FileSizeSeen,
/// width in characater /// width in characater
row_heading_chars: usize, row_heading_chars: usize,
/// width, in characters, of the entire min/max timerange /// width, in characters, of the entire min/max timerange
@ -107,9 +107,10 @@ struct ParquetFileFormatter {
max_time: i64, max_time: i64,
} }
#[derive(Debug)] #[derive(Debug, Default)]
/// helper to track if there are multiple file sizes in a set of parquet files /// helper to track if there are multiple file sizes in a set of parquet files
enum FileSizeSeen { enum FileSizeSeen {
#[default]
None, None,
One(i64), One(i64),
Many, Many,
@ -149,15 +150,12 @@ impl ParquetFileFormatter {
file_size_seen.observe(file.file_size_bytes) file_size_seen.observe(file.file_size_bytes)
}); });
// show the size if there are multiple sizes
let show_size = matches!(file_size_seen, FileSizeSeen::Many);
let time_range = max_time - min_time; let time_range = max_time - min_time;
let ns_per_char = (time_range as f64) / (width_chars as f64); let ns_per_char = (time_range as f64) / (width_chars as f64);
Self { Self {
show_size, file_size_seen,
width_chars, width_chars,
ns_per_char, ns_per_char,
min_time, min_time,
@ -179,9 +177,14 @@ impl ParquetFileFormatter {
} }
fn format_level(&self, level: &CompactionLevel) -> String { fn format_level(&self, level: &CompactionLevel) -> String {
let level_heading = display_level(level);
let level_heading = match self.file_size_seen {
FileSizeSeen::One(sz) => format!("{level_heading}, all files {sz}b"),
_ => level_heading.into(),
};
format!( format!(
"{:width$}", "{level_heading:width$}",
display_level(level),
width = self.width_chars + self.row_heading_chars width = self.width_chars + self.row_heading_chars
) )
} }
@ -206,7 +209,9 @@ impl ParquetFileFormatter {
// Get compact display of the file, like 'L0.1' // Get compact display of the file, like 'L0.1'
// add |--- ---| formatting (based on field width) // add |--- ---| formatting (based on field width)
let file_string = format!("|{:-^width$}|", display_file_id(file), width = field_width); let file_string = format!("|{:-^width$}|", display_file_id(file), width = field_width);
let row_heading = display_format(file, self.show_size); // show indvidual file sizes if they are different
let show_size = matches!(self.file_size_seen, FileSizeSeen::Many);
let row_heading = display_format(file, show_size);
// special case "zero" width times // special case "zero" width times
if self.min_time == self.max_time { if self.min_time == self.max_time {
@ -265,7 +270,7 @@ fn display_format(file: &ParquetFile, show_size: bool) -> String {
let max_time = file.max_time.get(); // display as i64 let max_time = file.max_time.get(); // display as i64
let sz = file.file_size_bytes; let sz = file.file_size_bytes;
if show_size { if show_size {
format!("{file_id}[{min_time},{max_time}]@{sz}") format!("{file_id}[{min_time},{max_time}] {sz}b")
} else { } else {
format!("{file_id}[{min_time},{max_time}]") format!("{file_id}[{min_time},{max_time}]")
} }
@ -293,7 +298,7 @@ mod test {
@r###" @r###"
--- ---
- display - display
- "L0 " - "L0, all files 1b "
- "L0.1[0,0] |-------------------------------------L0.1-------------------------------------|" - "L0.1[0,0] |-------------------------------------L0.1-------------------------------------|"
- "L0.2[0,0] |-------------------------------------L0.2-------------------------------------|" - "L0.2[0,0] |-------------------------------------L0.2-------------------------------------|"
"### "###
@ -321,10 +326,10 @@ mod test {
--- ---
- display - display
- "L0 " - "L0 "
- "L0.1[0,0]@1 |-------------------------------------L0.1-------------------------------------|" - "L0.1[0,0] 1b |-------------------------------------L0.1-------------------------------------|"
- "L0.2[0,0]@1 |-------------------------------------L0.2-------------------------------------|" - "L0.2[0,0] 1b |-------------------------------------L0.2-------------------------------------|"
- "L2 " - "L2 "
- "L2.3[0,0]@42 |-------------------------------------L2.3-------------------------------------|" - "L2.3[0,0] 42b |-------------------------------------L2.3-------------------------------------|"
"### "###
); );
} }
@ -354,9 +359,9 @@ mod test {
--- ---
- display - display
- "L0 " - "L0 "
- "L0.1[100,200]@1 |----------L0.1----------| " - "L0.1[100,200] 1b |----------L0.1----------| "
- "L0.2[300,400]@1 |----------L0.2----------| " - "L0.2[300,400] 1b |----------L0.2----------| "
- "L0.11[150,350]@44 |-----------------------L0.11-----------------------| " - "L0.11[150,350] 44b |-----------------------L0.11-----------------------| "
"### "###
); );
} }

4
compactor2/src/tests.rs
View File

@ -514,7 +514,7 @@ async fn test_pr6890() {
@r###" @r###"
--- ---
- input - input
- "L0 " - "L0, all files 1000000b "
- "L0.1[100,200] |-------------------------------------L0.1-------------------------------------|" - "L0.1[100,200] |-------------------------------------L0.1-------------------------------------|"
- "L0.2[100,200] |-------------------------------------L0.2-------------------------------------|" - "L0.2[100,200] |-------------------------------------L0.2-------------------------------------|"
- "L0.3[100,200] |-------------------------------------L0.3-------------------------------------|" - "L0.3[100,200] |-------------------------------------L0.3-------------------------------------|"
@ -537,7 +537,7 @@ async fn test_pr6890() {
@r###" @r###"
--- ---
- input - input
- "L1 " - "L1, all files 1b "
- "L1.11[0,0] |------------------------------------L1.11-------------------------------------|" - "L1.11[0,0] |------------------------------------L1.11-------------------------------------|"
- "L1.12[0,0] |------------------------------------L1.12-------------------------------------|" - "L1.12[0,0] |------------------------------------L1.12-------------------------------------|"
- "L1.13[0,0] |------------------------------------L1.13-------------------------------------|" - "L1.13[0,0] |------------------------------------L1.13-------------------------------------|"