Influxdata
/
influxdb
mirror of https://github.com/influxdata/influxdb.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Merge pull request #8490 from influxdata/jrb_82_split_2ns_files 

feat: allow compactor to split files covering 2ns
pull/24376/head
Joe-Blount 2023-08-15 08:38:00 -05:00 committed by GitHub
parent 964b2f6b97 6d4729db1d
commit 0bfa0a7b38
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 410 additions and 53 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
compactor/src/components/split_or_compact/files_to_compact.rs
View File

@ -205,16 +205,20 @@ pub fn limit_files_to_compact(
.collect::<Vec<_>>();
for f in overlapped_files.clone() {
// When splitting, split time is the last ns included in the 'left' file on the split.
// So if the the split time matches max time of a file, that file does not need split.
let split_times = all_split_times
.iter()
.filter(|t| f.overlaps_time_range(**t, **t))
.filter(|split| split >= &&f.min_time && split < &&f.max_time)
.map(|t| t.get())
.collect::<Vec<_>>();
specific_splits.push(FileToSplit {
file: f.clone(),
split_times,
});
if !split_times.is_empty() {
specific_splits.push(FileToSplit {
file: f.clone(),
split_times,
});
}
}
}
// If the above found some specific splits, we'll go with that. Otherwise, make a general split decision.

62
compactor/src/components/split_or_compact/large_files_to_split.rs
View File

@ -41,22 +41,34 @@ pub fn compute_split_times_for_large_files(
let min_time = file.min_time.get();
let max_time = file.max_time.get();
// TODO: it would be nice to check if these files overlap (e.g. if they're multiple levels), and
// coordinate the split time across all the files, rather than deciding the split time for each file
// as if its the only file under consideration.
// only split files that are larger than max_desired_file_size and have time range at least 2
let max_file_size =
(max_desired_file_size as f64 * (1.0 + PERCENTAGE_OF_SOFT_EXCEEDED)) as u64;
if file_size > max_file_size && file.min_time < file.max_time - 1 {
let file_times = vec![TimestampMinMax {
min: min_time,
max: max_time,
}];
let split_times = V1IRPlanner::compute_split_time(
file_times,
min_time,
max_time,
file_size,
max_desired_file_size,
);
files_to_split.push(FileToSplit { file, split_times });
if file_size > max_file_size && file.min_time < file.max_time {
if file.min_time < file.max_time - 1 {
// The time range of the file is big enough we have choices for split time(s), so compute them.
let file_times = vec![TimestampMinMax {
min: min_time,
max: max_time,
}];
let split_times = V1IRPlanner::compute_split_time(
file_times,
min_time,
max_time,
file_size,
max_desired_file_size,
);
files_to_split.push(FileToSplit { file, split_times });
} else {
// The file covers 2ns. There's nothing to compute, split it the only place possible.
// When splitting, split time is the last ns included in the 'left' file on the split.
// So setting `min` as the split time means `min` goes to the left, and `max` goes to the right.
let split_times = vec![file.min_time.get()];
files_to_split.push(FileToSplit { file, split_times });
}
} else {
files_not_to_split.push(file);
}
@ -379,9 +391,11 @@ mod tests {
let (files_to_split, files_not_to_split) =
compute_split_times_for_large_files(files, max_desired_file_size, max_compact_size);
// The split files should be L1_1 with 2 split times to split the file into 3 smaller files
assert_eq!(files_to_split.len(), 1);
assert_eq!(files_to_split[0].split_times.len(), 2);
// The split files should be L1_1 with 2 split times to split the file into 3 smaller files, and the L1 split at the only
// time possible (since its a 2ns file, there is only one choice)
assert_eq!(files_to_split.len(), 2);
assert_eq!(files_to_split[0].split_times.len(), 1);
assert_eq!(files_to_split[1].split_times.len(), 2);
// See layout of 2 set of files
let files_to_split = files_to_split
@ -395,9 +409,9 @@ mod tests {
- files to split
- "L0, all files 300b "
- "L0.1[400,620] 120s |-----------------------------------------L0.1------------------------------------------| "
- "files not to split:"
- "L1, all files 300b "
- "L1.11[400,401] 60s |-----------------------------------------L1.11------------------------------------------|"
- "L1.11[400,401] 60s |L1.11| "
- "files not to split:"
"###
);
}
@ -443,8 +457,10 @@ mod tests {
let (files_to_split, files_not_to_split) =
compute_split_times_for_large_files(files, max_desired_file_size, max_compact_size);
// The split files should be L1_1 with 2 split times to split the file into 3 smaller files
assert_eq!(files_to_split.len(), 0);
// The split files should be L1_11, split at the only time possible (since its a 2ns file, there is only one choice)
assert_eq!(files_to_split.len(), 1);
assert_eq!(files_to_split[0].split_times.len(), 1);
assert_eq!(files_to_split[0].split_times[0], 400);
// See layout of 2 set of files
let files_to_split = files_to_split
@ -456,11 +472,11 @@ mod tests {
@r###"
---
- files to split
- "files not to split:"
- "L0, all files 300b "
- "L0.1[400,400] 120s |L0.1| "
- "L1, all files 300b "
- "L1.11[400,401] 60s |-----------------------------------------L1.11------------------------------------------|"
- "files not to split:"
- "L0, all files 300b "
- "L0.1[400,400] 120s |------------------------------------------L0.1------------------------------------------|"
"###
);
}

42
compactor/tests/layouts/large_files.rs
View File

@ -1347,13 +1347,43 @@ async fn tiny_time_range() {
- "L2 "
- "L2.2[1,1000] 5ns 52mb |------------------------------------------L2.2------------------------------------------|"
- "WARNING: file L1.1[1,2] 10ns 250mb exceeds soft limit 100mb by more than 50%"
- "SKIPPED COMPACTION for PartitionId(1): partition 1 has overlapped files that exceed max compact size limit 314572800. This may happen if a large amount of data has the same timestamp"
- "**** Final Output Files (0b written)"
- "L1 "
- "L1.1[1,2] 10ns 250mb |L1.1| "
- "**** Simulation run 0, type=split(ReduceLargeFileSize)(split_times=[1]). 1 Input Files, 250mb total:"
- "L1, all files 250mb "
- "L1.1[1,2] 10ns |------------------------------------------L1.1------------------------------------------|"
- "**** 2 Output Files (parquet_file_id not yet assigned), 250mb total:"
- "L1, all files 125mb "
- "L1.?[1,1] 10ns |L1.?| "
- "L1.?[2,2] 10ns |L1.?|"
- "Committing partition 1:"
- " Soft Deleting 1 files: L1.1"
- " Creating 2 files"
- "**** Simulation run 1, type=split(StartLevelOverlapsTooBig)(split_times=[2]). 1 Input Files, 52mb total:"
- "L2, all files 52mb "
- "L2.2[1,1000] 5ns |------------------------------------------L2.2------------------------------------------|"
- "**** 2 Output Files (parquet_file_id not yet assigned), 52mb total:"
- "L2 "
- "L2.2[1,1000] 5ns 52mb |------------------------------------------L2.2------------------------------------------|"
- "WARNING: file L1.1[1,2] 10ns 250mb exceeds soft limit 100mb by more than 50%"
- "L2.?[1,2] 5ns 106kb |L2.?| "
- "L2.?[3,1000] 5ns 52mb |-----------------------------------------L2.?------------------------------------------| "
- "Committing partition 1:"
- " Soft Deleting 1 files: L2.2"
- " Creating 2 files"
- "**** Simulation run 2, type=compact(TotalSizeLessThanMaxCompactSize). 3 Input Files, 250mb total:"
- "L1 "
- "L1.4[2,2] 10ns 125mb |L1.4|"
- "L1.3[1,1] 10ns 125mb |L1.3| "
- "L2 "
- "L2.5[1,2] 5ns 106kb |------------------------------------------L2.5------------------------------------------|"
- "**** 1 Output Files (parquet_file_id not yet assigned), 250mb total:"
- "L2, all files 250mb "
- "L2.?[1,2] 10ns |------------------------------------------L2.?------------------------------------------|"
- "Committing partition 1:"
- " Soft Deleting 3 files: L1.3, L1.4, L2.5"
- " Creating 1 files"
- "**** Final Output Files (552mb written)"
- "L2 "
- "L2.6[3,1000] 5ns 52mb |-----------------------------------------L2.6------------------------------------------| "
- "L2.7[1,2] 10ns 250mb |L2.7| "
- "WARNING: file L2.7[1,2] 10ns 250mb exceeds soft limit 100mb by more than 50%"
"###
);
}

343
compactor/tests/layouts/single_timestamp.rs
View File

@ -125,13 +125,25 @@ async fn two_giant_files_time_range_1() {
- "L0.2[100,101] 2ns |------------------------------------------L0.2------------------------------------------|"
- "WARNING: file L0.1[100,101] 1ns 4.88gb exceeds soft limit 100mb by more than 50%"
- "WARNING: file L0.2[100,101] 2ns 4.88gb exceeds soft limit 100mb by more than 50%"
- "SKIPPED COMPACTION for PartitionId(1): partition 1 has overlapped files that exceed max compact size limit 314572800. This may happen if a large amount of data has the same timestamp"
- "**** Final Output Files (0b written)"
- "**** Simulation run 0, type=split(ReduceLargeFileSize)(split_times=[100]). 1 Input Files, 4.88gb total:"
- "L0, all files 4.88gb "
- "L0.1[100,101] 1ns |------------------------------------------L0.1------------------------------------------|"
- "L0.2[100,101] 2ns |------------------------------------------L0.2------------------------------------------|"
- "WARNING: file L0.1[100,101] 1ns 4.88gb exceeds soft limit 100mb by more than 50%"
- "**** 2 Output Files (parquet_file_id not yet assigned), 4.88gb total:"
- "L0, all files 2.44gb "
- "L0.?[100,100] 1ns |L0.?| "
- "L0.?[101,101] 1ns |L0.?|"
- "Committing partition 1:"
- " Soft Deleting 1 files: L0.1"
- " Creating 2 files"
- "SKIPPED COMPACTION for PartitionId(1): partition 1 has overlapped files that exceed max compact size limit 314572800. This may happen if a large amount of data has the same timestamp"
- "**** Final Output Files (4.88gb written)"
- "L0 "
- "L0.2[100,101] 2ns 4.88gb |------------------------------------------L0.2------------------------------------------|"
- "L0.3[100,100] 1ns 2.44gb |L0.3| "
- "L0.4[101,101] 1ns 2.44gb |L0.4|"
- "WARNING: file L0.2[100,101] 2ns 4.88gb exceeds soft limit 100mb by more than 50%"
- "WARNING: file L0.3[100,100] 1ns 2.44gb exceeds soft limit 100mb by more than 50%"
- "WARNING: file L0.4[101,101] 1ns 2.44gb exceeds soft limit 100mb by more than 50%"
"###
);
}
@ -290,22 +302,317 @@ async fn many_medium_files_time_range_1() {
- "Committing partition 1:"
- " Soft Deleting 10 files: L0.1, L0.2, L0.3, L0.4, L0.5, L0.6, L0.7, L0.8, L0.9, L0.10"
- " Creating 1 files"
- "SKIPPED COMPACTION for PartitionId(1): partition 1 has overlapped files that exceed max compact size limit 314572800. This may happen if a large amount of data has the same timestamp"
- "**** Final Output Files (300mb written)"
- "**** Simulation run 1, type=split(ReduceLargeFileSize)(split_times=[100]). 1 Input Files, 300mb total:"
- "L1, all files 300mb "
- "L1.21[100,101] 10ns |-----------------------------------------L1.21------------------------------------------|"
- "**** 2 Output Files (parquet_file_id not yet assigned), 300mb total:"
- "L1, all files 150mb "
- "L1.?[100,100] 10ns |L1.?| "
- "L1.?[101,101] 10ns |L1.?|"
- "Committing partition 1:"
- " Soft Deleting 1 files: L1.21"
- " Creating 2 files"
- "**** Simulation run 2, type=split(ReduceOverlap)(split_times=[100]). 1 Input Files, 30mb total:"
- "L0, all files 30mb "
- "L0.11[100,101] 11ns |-----------------------------------------L0.11------------------------------------------|"
- "**** 2 Output Files (parquet_file_id not yet assigned), 30mb total:"
- "L0, all files 15mb "
- "L0.?[100,100] 11ns |L0.?| "
- "L0.?[101,101] 11ns |L0.?|"
- "**** Simulation run 3, type=split(ReduceOverlap)(split_times=[100]). 1 Input Files, 30mb total:"
- "L0, all files 30mb "
- "L0.12[100,101] 12ns |-----------------------------------------L0.12------------------------------------------|"
- "**** 2 Output Files (parquet_file_id not yet assigned), 30mb total:"
- "L0, all files 15mb "
- "L0.?[100,100] 12ns |L0.?| "
- "L0.?[101,101] 12ns |L0.?|"
- "**** Simulation run 4, type=split(ReduceOverlap)(split_times=[100]). 1 Input Files, 30mb total:"
- "L0, all files 30mb "
- "L0.13[100,101] 13ns |-----------------------------------------L0.13------------------------------------------|"
- "**** 2 Output Files (parquet_file_id not yet assigned), 30mb total:"
- "L0, all files 15mb "
- "L0.?[100,100] 13ns |L0.?| "
- "L0.?[101,101] 13ns |L0.?|"
- "**** Simulation run 5, type=split(ReduceOverlap)(split_times=[100]). 1 Input Files, 30mb total:"
- "L0, all files 30mb "
- "L0.14[100,101] 14ns |-----------------------------------------L0.14------------------------------------------|"
- "**** 2 Output Files (parquet_file_id not yet assigned), 30mb total:"
- "L0, all files 15mb "
- "L0.?[100,100] 14ns |L0.?| "
- "L0.?[101,101] 14ns |L0.?|"
- "**** Simulation run 6, type=split(ReduceOverlap)(split_times=[100]). 1 Input Files, 30mb total:"
- "L0, all files 30mb "
- "L0.15[100,101] 15ns |-----------------------------------------L0.15------------------------------------------|"
- "**** 2 Output Files (parquet_file_id not yet assigned), 30mb total:"
- "L0, all files 15mb "
- "L0.?[100,100] 15ns |L0.?| "
- "L0.?[101,101] 15ns |L0.?|"
- "**** Simulation run 7, type=split(ReduceOverlap)(split_times=[100]). 1 Input Files, 30mb total:"
- "L0, all files 30mb "
- "L0.16[100,101] 16ns |-----------------------------------------L0.16------------------------------------------|"
- "**** 2 Output Files (parquet_file_id not yet assigned), 30mb total:"
- "L0, all files 15mb "
- "L0.?[100,100] 16ns |L0.?| "
- "L0.?[101,101] 16ns |L0.?|"
- "**** Simulation run 8, type=split(ReduceOverlap)(split_times=[100]). 1 Input Files, 30mb total:"
- "L0, all files 30mb "
- "L0.17[100,101] 17ns |-----------------------------------------L0.17------------------------------------------|"
- "**** 2 Output Files (parquet_file_id not yet assigned), 30mb total:"
- "L0, all files 15mb "
- "L0.?[100,100] 17ns |L0.?| "
- "L0.?[101,101] 17ns |L0.?|"
- "**** Simulation run 9, type=split(ReduceOverlap)(split_times=[100]). 1 Input Files, 30mb total:"
- "L0, all files 30mb "
- "L0.18[100,101] 18ns |-----------------------------------------L0.18------------------------------------------|"
- "**** 2 Output Files (parquet_file_id not yet assigned), 30mb total:"
- "L0, all files 15mb "
- "L0.?[100,100] 18ns |L0.?| "
- "L0.?[101,101] 18ns |L0.?|"
- "**** Simulation run 10, type=split(ReduceOverlap)(split_times=[100]). 1 Input Files, 30mb total:"
- "L0, all files 30mb "
- "L0.19[100,101] 19ns |-----------------------------------------L0.19------------------------------------------|"
- "**** 2 Output Files (parquet_file_id not yet assigned), 30mb total:"
- "L0, all files 15mb "
- "L0.?[100,100] 19ns |L0.?| "
- "L0.?[101,101] 19ns |L0.?|"
- "**** Simulation run 11, type=split(ReduceOverlap)(split_times=[100]). 1 Input Files, 30mb total:"
- "L0, all files 30mb "
- "L0.20[100,101] 20ns |-----------------------------------------L0.20------------------------------------------|"
- "**** 2 Output Files (parquet_file_id not yet assigned), 30mb total:"
- "L0, all files 15mb "
- "L0.?[100,100] 20ns |L0.?| "
- "L0.?[101,101] 20ns |L0.?|"
- "Committing partition 1:"
- " Soft Deleting 10 files: L0.11, L0.12, L0.13, L0.14, L0.15, L0.16, L0.17, L0.18, L0.19, L0.20"
- " Creating 20 files"
- "**** Simulation run 12, type=compact(FoundSubsetLessThanMaxCompactSize). 2 Input Files, 165mb total:"
- "L0 "
- "L0.11[100,101] 11ns 30mb |-----------------------------------------L0.11------------------------------------------|"
- "L0.12[100,101] 12ns 30mb |-----------------------------------------L0.12------------------------------------------|"
- "L0.13[100,101] 13ns 30mb |-----------------------------------------L0.13------------------------------------------|"
- "L0.14[100,101] 14ns 30mb |-----------------------------------------L0.14------------------------------------------|"
- "L0.15[100,101] 15ns 30mb |-----------------------------------------L0.15------------------------------------------|"
- "L0.16[100,101] 16ns 30mb |-----------------------------------------L0.16------------------------------------------|"
- "L0.17[100,101] 17ns 30mb |-----------------------------------------L0.17------------------------------------------|"
- "L0.18[100,101] 18ns 30mb |-----------------------------------------L0.18------------------------------------------|"
- "L0.19[100,101] 19ns 30mb |-----------------------------------------L0.19------------------------------------------|"
- "L0.20[100,101] 20ns 30mb |-----------------------------------------L0.20------------------------------------------|"
- "L0.24[100,100] 11ns 15mb |-----------------------------------------L0.24------------------------------------------|"
- "L1 "
- "L1.21[100,101] 10ns 300mb|-----------------------------------------L1.21------------------------------------------|"
- "WARNING: file L1.21[100,101] 10ns 300mb exceeds soft limit 100mb by more than 50%"
- "L1.22[100,100] 10ns 150mb|-----------------------------------------L1.22------------------------------------------|"
- "**** 1 Output Files (parquet_file_id not yet assigned), 165mb total:"
- "L1, all files 165mb "
- "L1.?[100,100] 11ns |------------------------------------------L1.?------------------------------------------|"
- "Committing partition 1:"
- " Soft Deleting 2 files: L1.22, L0.24"
- " Creating 1 files"
- "**** Simulation run 13, type=compact(FoundSubsetLessThanMaxCompactSize). 2 Input Files, 165mb total:"
- "L0 "
- "L0.25[101,101] 11ns 15mb |-----------------------------------------L0.25------------------------------------------|"
- "L1 "
- "L1.23[101,101] 10ns 150mb|-----------------------------------------L1.23------------------------------------------|"
- "**** 1 Output Files (parquet_file_id not yet assigned), 165mb total:"
- "L1, all files 165mb "
- "L1.?[101,101] 11ns |------------------------------------------L1.?------------------------------------------|"
- "Committing partition 1:"
- " Soft Deleting 2 files: L1.23, L0.25"
- " Creating 1 files"
- "**** Simulation run 14, type=compact(FoundSubsetLessThanMaxCompactSize). 2 Input Files, 180mb total:"
- "L0 "
- "L0.26[100,100] 12ns 15mb |-----------------------------------------L0.26------------------------------------------|"
- "L1 "
- "L1.44[100,100] 11ns 165mb|-----------------------------------------L1.44------------------------------------------|"
- "**** 1 Output Files (parquet_file_id not yet assigned), 180mb total:"
- "L1, all files 180mb "
- "L1.?[100,100] 12ns |------------------------------------------L1.?------------------------------------------|"
- "Committing partition 1:"
- " Soft Deleting 2 files: L0.26, L1.44"
- " Creating 1 files"
- "**** Simulation run 15, type=compact(FoundSubsetLessThanMaxCompactSize). 2 Input Files, 180mb total:"
- "L0 "
- "L0.27[101,101] 12ns 15mb |-----------------------------------------L0.27------------------------------------------|"
- "L1 "
- "L1.45[101,101] 11ns 165mb|-----------------------------------------L1.45------------------------------------------|"
- "**** 1 Output Files (parquet_file_id not yet assigned), 180mb total:"
- "L1, all files 180mb "
- "L1.?[101,101] 12ns |------------------------------------------L1.?------------------------------------------|"
- "Committing partition 1:"
- " Soft Deleting 2 files: L0.27, L1.45"
- " Creating 1 files"
- "**** Simulation run 16, type=compact(FoundSubsetLessThanMaxCompactSize). 2 Input Files, 195mb total:"
- "L0 "
- "L0.28[100,100] 13ns 15mb |-----------------------------------------L0.28------------------------------------------|"
- "L1 "
- "L1.46[100,100] 12ns 180mb|-----------------------------------------L1.46------------------------------------------|"
- "**** 1 Output Files (parquet_file_id not yet assigned), 195mb total:"
- "L1, all files 195mb "
- "L1.?[100,100] 13ns |------------------------------------------L1.?------------------------------------------|"
- "Committing partition 1:"
- " Soft Deleting 2 files: L0.28, L1.46"
- " Creating 1 files"
- "**** Simulation run 17, type=compact(FoundSubsetLessThanMaxCompactSize). 2 Input Files, 195mb total:"
- "L0 "
- "L0.29[101,101] 13ns 15mb |-----------------------------------------L0.29------------------------------------------|"
- "L1 "
- "L1.47[101,101] 12ns 180mb|-----------------------------------------L1.47------------------------------------------|"
- "**** 1 Output Files (parquet_file_id not yet assigned), 195mb total:"
- "L1, all files 195mb "
- "L1.?[101,101] 13ns |------------------------------------------L1.?------------------------------------------|"
- "Committing partition 1:"
- " Soft Deleting 2 files: L0.29, L1.47"
- " Creating 1 files"
- "**** Simulation run 18, type=compact(FoundSubsetLessThanMaxCompactSize). 2 Input Files, 210mb total:"
- "L0 "
- "L0.30[100,100] 14ns 15mb |-----------------------------------------L0.30------------------------------------------|"
- "L1 "
- "L1.48[100,100] 13ns 195mb|-----------------------------------------L1.48------------------------------------------|"
- "**** 1 Output Files (parquet_file_id not yet assigned), 210mb total:"
- "L1, all files 210mb "
- "L1.?[100,100] 14ns |------------------------------------------L1.?------------------------------------------|"
- "Committing partition 1:"
- " Soft Deleting 2 files: L0.30, L1.48"
- " Creating 1 files"
- "**** Simulation run 19, type=compact(FoundSubsetLessThanMaxCompactSize). 2 Input Files, 210mb total:"
- "L0 "
- "L0.31[101,101] 14ns 15mb |-----------------------------------------L0.31------------------------------------------|"
- "L1 "
- "L1.49[101,101] 13ns 195mb|-----------------------------------------L1.49------------------------------------------|"
- "**** 1 Output Files (parquet_file_id not yet assigned), 210mb total:"
- "L1, all files 210mb "
- "L1.?[101,101] 14ns |------------------------------------------L1.?------------------------------------------|"
- "Committing partition 1:"
- " Soft Deleting 2 files: L0.31, L1.49"
- " Creating 1 files"
- "**** Simulation run 20, type=compact(FoundSubsetLessThanMaxCompactSize). 2 Input Files, 225mb total:"
- "L0 "
- "L0.32[100,100] 15ns 15mb |-----------------------------------------L0.32------------------------------------------|"
- "L1 "
- "L1.50[100,100] 14ns 210mb|-----------------------------------------L1.50------------------------------------------|"
- "**** 1 Output Files (parquet_file_id not yet assigned), 225mb total:"
- "L1, all files 225mb "
- "L1.?[100,100] 15ns |------------------------------------------L1.?------------------------------------------|"
- "Committing partition 1:"
- " Soft Deleting 2 files: L0.32, L1.50"
- " Creating 1 files"
- "**** Simulation run 21, type=compact(FoundSubsetLessThanMaxCompactSize). 2 Input Files, 225mb total:"
- "L0 "
- "L0.33[101,101] 15ns 15mb |-----------------------------------------L0.33------------------------------------------|"
- "L1 "
- "L1.51[101,101] 14ns 210mb|-----------------------------------------L1.51------------------------------------------|"
- "**** 1 Output Files (parquet_file_id not yet assigned), 225mb total:"
- "L1, all files 225mb "
- "L1.?[101,101] 15ns |------------------------------------------L1.?------------------------------------------|"
- "Committing partition 1:"
- " Soft Deleting 2 files: L0.33, L1.51"
- " Creating 1 files"
- "**** Simulation run 22, type=compact(FoundSubsetLessThanMaxCompactSize). 2 Input Files, 240mb total:"
- "L0 "
- "L0.34[100,100] 16ns 15mb |-----------------------------------------L0.34------------------------------------------|"
- "L1 "
- "L1.52[100,100] 15ns 225mb|-----------------------------------------L1.52------------------------------------------|"
- "**** 1 Output Files (parquet_file_id not yet assigned), 240mb total:"
- "L1, all files 240mb "
- "L1.?[100,100] 16ns |------------------------------------------L1.?------------------------------------------|"
- "Committing partition 1:"
- " Soft Deleting 2 files: L0.34, L1.52"
- " Creating 1 files"
- "**** Simulation run 23, type=compact(FoundSubsetLessThanMaxCompactSize). 2 Input Files, 240mb total:"
- "L0 "
- "L0.35[101,101] 16ns 15mb |-----------------------------------------L0.35------------------------------------------|"
- "L1 "
- "L1.53[101,101] 15ns 225mb|-----------------------------------------L1.53------------------------------------------|"
- "**** 1 Output Files (parquet_file_id not yet assigned), 240mb total:"
- "L1, all files 240mb "
- "L1.?[101,101] 16ns |------------------------------------------L1.?------------------------------------------|"
- "Committing partition 1:"
- " Soft Deleting 2 files: L0.35, L1.53"
- " Creating 1 files"
- "**** Simulation run 24, type=compact(FoundSubsetLessThanMaxCompactSize). 2 Input Files, 255mb total:"
- "L0 "
- "L0.36[100,100] 17ns 15mb |-----------------------------------------L0.36------------------------------------------|"
- "L1 "
- "L1.54[100,100] 16ns 240mb|-----------------------------------------L1.54------------------------------------------|"
- "**** 1 Output Files (parquet_file_id not yet assigned), 255mb total:"
- "L1, all files 255mb "
- "L1.?[100,100] 17ns |------------------------------------------L1.?------------------------------------------|"
- "Committing partition 1:"
- " Soft Deleting 2 files: L0.36, L1.54"
- " Creating 1 files"
- "**** Simulation run 25, type=compact(FoundSubsetLessThanMaxCompactSize). 2 Input Files, 255mb total:"
- "L0 "
- "L0.37[101,101] 17ns 15mb |-----------------------------------------L0.37------------------------------------------|"
- "L1 "
- "L1.55[101,101] 16ns 240mb|-----------------------------------------L1.55------------------------------------------|"
- "**** 1 Output Files (parquet_file_id not yet assigned), 255mb total:"
- "L1, all files 255mb "
- "L1.?[101,101] 17ns |------------------------------------------L1.?------------------------------------------|"
- "Committing partition 1:"
- " Soft Deleting 2 files: L0.37, L1.55"
- " Creating 1 files"
- "**** Simulation run 26, type=compact(FoundSubsetLessThanMaxCompactSize). 2 Input Files, 270mb total:"
- "L0 "
- "L0.38[100,100] 18ns 15mb |-----------------------------------------L0.38------------------------------------------|"
- "L1 "
- "L1.56[100,100] 17ns 255mb|-----------------------------------------L1.56------------------------------------------|"
- "**** 1 Output Files (parquet_file_id not yet assigned), 270mb total:"
- "L1, all files 270mb "
- "L1.?[100,100] 18ns |------------------------------------------L1.?------------------------------------------|"
- "Committing partition 1:"
- " Soft Deleting 2 files: L0.38, L1.56"
- " Creating 1 files"
- "**** Simulation run 27, type=compact(FoundSubsetLessThanMaxCompactSize). 2 Input Files, 270mb total:"
- "L0 "
- "L0.39[101,101] 18ns 15mb |-----------------------------------------L0.39------------------------------------------|"
- "L1 "
- "L1.57[101,101] 17ns 255mb|-----------------------------------------L1.57------------------------------------------|"
- "**** 1 Output Files (parquet_file_id not yet assigned), 270mb total:"
- "L1, all files 270mb "
- "L1.?[101,101] 18ns |------------------------------------------L1.?------------------------------------------|"
- "Committing partition 1:"
- " Soft Deleting 2 files: L0.39, L1.57"
- " Creating 1 files"
- "**** Simulation run 28, type=compact(FoundSubsetLessThanMaxCompactSize). 2 Input Files, 285mb total:"
- "L0 "
- "L0.40[100,100] 19ns 15mb |-----------------------------------------L0.40------------------------------------------|"
- "L1 "
- "L1.58[100,100] 18ns 270mb|-----------------------------------------L1.58------------------------------------------|"
- "**** 1 Output Files (parquet_file_id not yet assigned), 285mb total:"
- "L1, all files 285mb "
- "L1.?[100,100] 19ns |------------------------------------------L1.?------------------------------------------|"
- "Committing partition 1:"
- " Soft Deleting 2 files: L0.40, L1.58"
- " Creating 1 files"
- "**** Simulation run 29, type=compact(FoundSubsetLessThanMaxCompactSize). 2 Input Files, 285mb total:"
- "L0 "
- "L0.41[101,101] 19ns 15mb |-----------------------------------------L0.41------------------------------------------|"
- "L1 "
- "L1.59[101,101] 18ns 270mb|-----------------------------------------L1.59------------------------------------------|"
- "**** 1 Output Files (parquet_file_id not yet assigned), 285mb total:"
- "L1, all files 285mb "
- "L1.?[101,101] 19ns |------------------------------------------L1.?------------------------------------------|"
- "Committing partition 1:"
- " Soft Deleting 2 files: L0.41, L1.59"
- " Creating 1 files"
- "**** Simulation run 30, type=compact(FoundSubsetLessThanMaxCompactSize). 2 Input Files, 300mb total:"
- "L0 "
- "L0.42[100,100] 20ns 15mb |-----------------------------------------L0.42------------------------------------------|"
- "L1 "
- "L1.60[100,100] 19ns 285mb|-----------------------------------------L1.60------------------------------------------|"
- "**** 1 Output Files (parquet_file_id not yet assigned), 300mb total:"
- "L1, all files 300mb "
- "L1.?[100,100] 20ns |------------------------------------------L1.?------------------------------------------|"
- "Committing partition 1:"
- " Soft Deleting 2 files: L0.42, L1.60"
- " Creating 1 files"
- "**** Simulation run 31, type=compact(TotalSizeLessThanMaxCompactSize). 2 Input Files, 300mb total:"
- "L0 "
- "L0.43[101,101] 20ns 15mb |-----------------------------------------L0.43------------------------------------------|"
- "L1 "
- "L1.61[101,101] 19ns 285mb|-----------------------------------------L1.61------------------------------------------|"
- "**** 1 Output Files (parquet_file_id not yet assigned), 300mb total:"
- "L1, all files 300mb "
- "L1.?[101,101] 20ns |------------------------------------------L1.?------------------------------------------|"
- "Committing partition 1:"
- " Soft Deleting 2 files: L0.43, L1.61"
- " Creating 1 files"
- "Committing partition 1:"
- " Upgrading 2 files level to CompactionLevel::L2: L1.62, L1.63"
- "**** Final Output Files (5.42gb written)"
- "L2, all files 300mb "
- "L2.62[100,100] 20ns |L2.62| "
- "L2.63[101,101] 20ns |L2.63|"
- "WARNING: file L2.62[100,100] 20ns 300mb exceeds soft limit 100mb by more than 50%"
- "WARNING: file L2.63[101,101] 20ns 300mb exceeds soft limit 100mb by more than 50%"
"###
);
}

2
compactor/tests/layouts/stuck.rs
View File

@ -1443,7 +1443,7 @@ async fn stuck_l0_large_l0s() {
.await
.with_max_num_files_per_plan(max_files)
.with_max_desired_file_size_bytes(MAX_DESIRED_FILE_SIZE)
.with_partition_timeout(Duration::from_millis(10000))
.with_partition_timeout(Duration::from_millis(100000))
//.with_suppress_run_output() // remove this to debug
.build()
.await;