fix: compactor stuck looping with unproductive compactions (needs vertical split) (#8039)
* chore: adjust with_max_num_files_per_plan to more common setting This significantly increases write amplification (see change in `written` at the conclusion of the cases) * fix: compactor looping with unproductive compactions * chore: formatting cleanup * chore: fix typo in commentpull/24376/head
Joe-Blount
GitHub
parent
f75736891a
commit
b219b4b003
|
|
@ -49,14 +49,32 @@ impl DivideInitial for MultipleBranchesDivideInitial {
|
|||
.collect::<Vec<_>>();
|
||||
let mut branches = Vec::with_capacity(start_level_files.len());
|
||||
|
||||
let mut chains: Vec<Vec<ParquetFile>>;
|
||||
let mut chains = Vec::with_capacity(start_level_files.len());
|
||||
if start_level == CompactionLevel::Initial {
|
||||
// The splitting & merging of chains in this block is what makes use of the vertical split performed by
|
||||
// high_l0_overlap_split. Without this chain based grouping, compactions would generally undo the work
|
||||
// started by splitting in high_l0_overlap_split.
|
||||
// split start_level_files into chains of overlapping files. This can happen based on min_time/max_time
|
||||
// without regard for max_l0_created_at because there are no overlaps between chains.
|
||||
chains = split_into_chains(start_level_files);
|
||||
// L0 files can be highly overlapping, requiring 'vertical splitting' (see high_l0_overlap_split).
|
||||
// Achieving `vertical splitting` requires we tweak the grouping here for two reasons:
|
||||
// 1) Allow the large highly overlapped groups of L0s to remain in a single branch, so they trigger the split
|
||||
// 2) Prevent the output of a prior split from being grouped together to undo the previous veritacal split.
|
||||
|
||||
// Both of these objectives need to consider the L0s as a chains of overlapping files. The chains are
|
||||
// each a set of L0s that overlap each other, but do not overlap the other chains.
|
||||
// Chains can be created based on min_time/max_time without regard for max_l0_created_at because there
|
||||
// are no overlaps between chains.
|
||||
let initial_chains = split_into_chains(start_level_files);
|
||||
|
||||
// Reason 1) above - keep the large groups of L0s in a single branch to facilitate later splitting.
|
||||
for chain in initial_chains {
|
||||
let this_chain_bytes: usize =
|
||||
chain.iter().map(|f| f.file_size_bytes as usize).sum();
|
||||
if this_chain_bytes > 2 * max_total_file_size_to_group {
|
||||
// This is a very large set of overlapping L0s, its needs vertical splitting, so keep the branch intact
|
||||
// to trigger the split.
|
||||
branches.push(chain);
|
||||
} else {
|
||||
chains.push(chain);
|
||||
}
|
||||
}
|
||||
|
||||
// If the chains are smaller than the max compact size, combine them to get better compaction group sizes.
|
||||
// This combining of chains must happen based on max_l0_created_at (it can only join adjacent chains, when
|
||||
// sorted by max_l0_created_at).
|
||||
|
|
@ -65,6 +83,11 @@ impl DivideInitial for MultipleBranchesDivideInitial {
|
|||
chains = vec![start_level_files];
|
||||
}
|
||||
|
||||
// Reason 2) above - ensure the grouping in branches doesn't undo the vertical splitting.
|
||||
// Assume we start with 30 files (A,B,C,...), that were each split into 3 files (A1, A2, A3, B1, ..). If we create branches
|
||||
// from sorting all files by max_l0_created_at we'd undo the vertical splitting (A1-A3 would get compacted back into one file).
|
||||
// Currently the contents of each chain is more like A1, B1, C1, so by grouping chains together we can preserve the previous
|
||||
// vertical splitting.
|
||||
for chain in chains {
|
||||
let start_level_files = order_files(chain, start_level);
|
||||
|
||||
|
|
@ -183,11 +206,11 @@ mod tests {
|
|||
#[should_panic(
|
||||
expected = "Size of a file 50 is larger than the max size limit to compact. Please adjust the settings"
|
||||
)]
|
||||
fn test_divide_size_limit_too_sall() {
|
||||
fn test_divide_size_limit_too_small() {
|
||||
let round_info = RoundInfo::ManySmallFiles {
|
||||
start_level: CompactionLevel::Initial,
|
||||
max_num_files_to_group: 10,
|
||||
max_total_file_size_to_group: 10,
|
||||
max_total_file_size_to_group: 40,
|
||||
};
|
||||
let divide = MultipleBranchesDivideInitial::new();
|
||||
|
||||
|
|
|
|||
|
|
@ -72,11 +72,12 @@ impl LevelBasedRoundInfo {
|
|||
}
|
||||
|
||||
/// Returns true if number of files of the given start_level and
|
||||
/// their overlapped files in next level is over limit
|
||||
/// their overlapped files in next level is over limit, and if those
|
||||
/// files are sufficiently small.
|
||||
///
|
||||
/// over the limit means that the maximum number of files that a subsequent compaction
|
||||
/// branch may choose to compact in a single plan would exceed `max_num_files_per_plan`
|
||||
pub fn too_many_files_to_compact(
|
||||
pub fn too_many_small_files_to_compact(
|
||||
&self,
|
||||
files: &[ParquetFile],
|
||||
start_level: CompactionLevel,
|
||||
|
|
@ -86,6 +87,10 @@ impl LevelBasedRoundInfo {
|
|||
.filter(|f| f.compaction_level == start_level)
|
||||
.collect::<Vec<_>>();
|
||||
let num_start_level = start_level_files.len();
|
||||
let size_start_level: usize = start_level_files
|
||||
.iter()
|
||||
.map(|f| f.file_size_bytes as usize)
|
||||
.sum();
|
||||
|
||||
let next_level_files = files
|
||||
.iter()
|
||||
|
|
@ -97,6 +102,14 @@ impl LevelBasedRoundInfo {
|
|||
// plan, run a pre-phase to reduce the number of files first
|
||||
let num_overlapped_files = get_num_overlapped_files(start_level_files, next_level_files);
|
||||
if num_start_level + num_overlapped_files > self.max_num_files_per_plan {
|
||||
if size_start_level / num_start_level
|
||||
> self.max_total_file_size_per_plan / self.max_num_files_per_plan
|
||||
{
|
||||
// Average start level file size is more than the average implied by max bytes & files per plan.
|
||||
// Even though there are "many files", this is not "many small files".
|
||||
// There isn't much (perhaps not any) file reduction to be done, so don't try.
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
|
|
@ -113,7 +126,7 @@ impl RoundInfoSource for LevelBasedRoundInfo {
|
|||
) -> Result<RoundInfo, DynError> {
|
||||
let start_level = get_start_level(files);
|
||||
|
||||
if self.too_many_files_to_compact(files, start_level) {
|
||||
if self.too_many_small_files_to_compact(files, start_level) {
|
||||
return Ok(RoundInfo::ManySmallFiles {
|
||||
start_level,
|
||||
max_num_files_to_group: self.max_num_files_per_plan,
|
||||
|
|
@ -200,7 +213,7 @@ mod tests {
|
|||
use crate::components::round_info_source::LevelBasedRoundInfo;
|
||||
|
||||
#[test]
|
||||
fn test_too_many_files_to_compact() {
|
||||
fn test_too_many_small_files_to_compact() {
|
||||
// L0 files
|
||||
let f1 = ParquetFileBuilder::new(1)
|
||||
.with_time_range(0, 100)
|
||||
|
|
@ -229,18 +242,20 @@ mod tests {
|
|||
|
||||
// f1 and f2 are not over limit
|
||||
assert!(!round_info
|
||||
.too_many_files_to_compact(&[f1.clone(), f2.clone()], CompactionLevel::Initial));
|
||||
.too_many_small_files_to_compact(&[f1.clone(), f2.clone()], CompactionLevel::Initial));
|
||||
// f1, f2 and f3 are not over limit
|
||||
assert!(!round_info.too_many_files_to_compact(
|
||||
assert!(!round_info.too_many_small_files_to_compact(
|
||||
&[f1.clone(), f2.clone(), f3.clone()],
|
||||
CompactionLevel::Initial
|
||||
));
|
||||
// f1, f2 and f4 are over limit
|
||||
assert!(round_info.too_many_files_to_compact(
|
||||
assert!(round_info.too_many_small_files_to_compact(
|
||||
&[f1.clone(), f2.clone(), f4.clone()],
|
||||
CompactionLevel::Initial
|
||||
));
|
||||
// f1, f2, f3 and f4 are over limit
|
||||
assert!(round_info.too_many_files_to_compact(&[f1, f2, f3, f4], CompactionLevel::Initial));
|
||||
assert!(
|
||||
round_info.too_many_small_files_to_compact(&[f1, f2, f3, f4], CompactionLevel::Initial)
|
||||
);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load Diff
|
|
@ -564,7 +564,7 @@ async fn many_good_size_l0_files() {
|
|||
- "L0.286[285,286] 286ns |L0.286|"
|
||||
- "L0.287[286,287] 287ns |L0.287|"
|
||||
- "L0.288[287,288] 288ns |L0.288|"
|
||||
- "**** Simulation run 0, type=split(CompactAndSplitOutput(ManySmallFiles))(split_times=[50, 100]). 150 Input Files, 300mb total:"
|
||||
- "**** Simulation run 0, type=split(CompactAndSplitOutput(FoundSubsetLessThanMaxCompactSize))(split_times=[50, 100]). 150 Input Files, 300mb total:"
|
||||
- "L0, all files 2mb "
|
||||
- "L0.1[0,1] 1ns |L0.1| "
|
||||
- "L0.2[1,2] 2ns |L0.2| "
|
||||
|
|
@ -717,218 +717,208 @@ async fn many_good_size_l0_files() {
|
|||
- "L0.149[148,149] 149ns |L0.149|"
|
||||
- "L0.150[149,150] 150ns |L0.150|"
|
||||
- "**** 3 Output Files (parquet_file_id not yet assigned), 300mb total:"
|
||||
- "L0 "
|
||||
- "L0.?[0,50] 150ns 100mb |------------L0.?------------| "
|
||||
- "L0.?[51,100] 150ns 98mb |-----------L0.?------------| "
|
||||
- "L0.?[101,150] 150ns 102mb |-----------L0.?------------| "
|
||||
- "L1 "
|
||||
- "L1.?[0,50] 150ns 100mb |------------L1.?------------| "
|
||||
- "L1.?[51,100] 150ns 98mb |-----------L1.?------------| "
|
||||
- "L1.?[101,150] 150ns 102mb |-----------L1.?------------| "
|
||||
- "Committing partition 1:"
|
||||
- " Soft Deleting 150 files: L0.1, L0.2, L0.3, L0.4, L0.5, L0.6, L0.7, L0.8, L0.9, L0.10, L0.11, L0.12, L0.13, L0.14, L0.15, L0.16, L0.17, L0.18, L0.19, L0.20, L0.21, L0.22, L0.23, L0.24, L0.25, L0.26, L0.27, L0.28, L0.29, L0.30, L0.31, L0.32, L0.33, L0.34, L0.35, L0.36, L0.37, L0.38, L0.39, L0.40, L0.41, L0.42, L0.43, L0.44, L0.45, L0.46, L0.47, L0.48, L0.49, L0.50, L0.51, L0.52, L0.53, L0.54, L0.55, L0.56, L0.57, L0.58, L0.59, L0.60, L0.61, L0.62, L0.63, L0.64, L0.65, L0.66, L0.67, L0.68, L0.69, L0.70, L0.71, L0.72, L0.73, L0.74, L0.75, L0.76, L0.77, L0.78, L0.79, L0.80, L0.81, L0.82, L0.83, L0.84, L0.85, L0.86, L0.87, L0.88, L0.89, L0.90, L0.91, L0.92, L0.93, L0.94, L0.95, L0.96, L0.97, L0.98, L0.99, L0.100, L0.101, L0.102, L0.103, L0.104, L0.105, L0.106, L0.107, L0.108, L0.109, L0.110, L0.111, L0.112, L0.113, L0.114, L0.115, L0.116, L0.117, L0.118, L0.119, L0.120, L0.121, L0.122, L0.123, L0.124, L0.125, L0.126, L0.127, L0.128, L0.129, L0.130, L0.131, L0.132, L0.133, L0.134, L0.135, L0.136, L0.137, L0.138, L0.139, L0.140, L0.141, L0.142, L0.143, L0.144, L0.145, L0.146, L0.147, L0.148, L0.149, L0.150"
|
||||
- " Creating 3 files"
|
||||
- "**** Simulation run 1, type=split(CompactAndSplitOutput(ManySmallFiles))(split_times=[200, 250]). 138 Input Files, 276mb total:"
|
||||
- "L0, all files 2mb "
|
||||
- "L0.151[150,151] 151ns |L0.151| "
|
||||
- "L0.152[151,152] 152ns |L0.152| "
|
||||
- "L0.153[152,153] 153ns |L0.153| "
|
||||
- "L0.154[153,154] 154ns |L0.154| "
|
||||
- "L0.155[154,155] 155ns |L0.155| "
|
||||
- "L0.156[155,156] 156ns |L0.156| "
|
||||
- "L0.157[156,157] 157ns |L0.157| "
|
||||
- "L0.158[157,158] 158ns |L0.158| "
|
||||
- "L0.159[158,159] 159ns |L0.159| "
|
||||
- "L0.160[159,160] 160ns |L0.160| "
|
||||
- "L0.161[160,161] 161ns |L0.161| "
|
||||
- "L0.162[161,162] 162ns |L0.162| "
|
||||
- "L0.163[162,163] 163ns |L0.163| "
|
||||
- "L0.164[163,164] 164ns |L0.164| "
|
||||
- "L0.165[164,165] 165ns |L0.165| "
|
||||
- "L0.166[165,166] 166ns |L0.166| "
|
||||
- "L0.167[166,167] 167ns |L0.167| "
|
||||
- "L0.168[167,168] 168ns |L0.168| "
|
||||
- "L0.169[168,169] 169ns |L0.169| "
|
||||
- "L0.170[169,170] 170ns |L0.170| "
|
||||
- "L0.171[170,171] 171ns |L0.171| "
|
||||
- "L0.172[171,172] 172ns |L0.172| "
|
||||
- "L0.173[172,173] 173ns |L0.173| "
|
||||
- "L0.174[173,174] 174ns |L0.174| "
|
||||
- "L0.175[174,175] 175ns |L0.175| "
|
||||
- "L0.176[175,176] 176ns |L0.176| "
|
||||
- "L0.177[176,177] 177ns |L0.177| "
|
||||
- "L0.178[177,178] 178ns |L0.178| "
|
||||
- "L0.179[178,179] 179ns |L0.179| "
|
||||
- "L0.180[179,180] 180ns |L0.180| "
|
||||
- "L0.181[180,181] 181ns |L0.181| "
|
||||
- "L0.182[181,182] 182ns |L0.182| "
|
||||
- "L0.183[182,183] 183ns |L0.183| "
|
||||
- "L0.184[183,184] 184ns |L0.184| "
|
||||
- "L0.185[184,185] 185ns |L0.185| "
|
||||
- "L0.186[185,186] 186ns |L0.186| "
|
||||
- "L0.187[186,187] 187ns |L0.187| "
|
||||
- "L0.188[187,188] 188ns |L0.188| "
|
||||
- "L0.189[188,189] 189ns |L0.189| "
|
||||
- "L0.190[189,190] 190ns |L0.190| "
|
||||
- "L0.191[190,191] 191ns |L0.191| "
|
||||
- "L0.192[191,192] 192ns |L0.192| "
|
||||
- "L0.193[192,193] 193ns |L0.193| "
|
||||
- "L0.194[193,194] 194ns |L0.194| "
|
||||
- "L0.195[194,195] 195ns |L0.195| "
|
||||
- "L0.196[195,196] 196ns |L0.196| "
|
||||
- "L0.197[196,197] 197ns |L0.197| "
|
||||
- "L0.198[197,198] 198ns |L0.198| "
|
||||
- "L0.199[198,199] 199ns |L0.199| "
|
||||
- "L0.200[199,200] 200ns |L0.200| "
|
||||
- "L0.201[200,201] 201ns |L0.201| "
|
||||
- "L0.202[201,202] 202ns |L0.202| "
|
||||
- "L0.203[202,203] 203ns |L0.203| "
|
||||
- "L0.204[203,204] 204ns |L0.204| "
|
||||
- "L0.205[204,205] 205ns |L0.205| "
|
||||
- "L0.206[205,206] 206ns |L0.206| "
|
||||
- "L0.207[206,207] 207ns |L0.207| "
|
||||
- "L0.208[207,208] 208ns |L0.208| "
|
||||
- "L0.209[208,209] 209ns |L0.209| "
|
||||
- "L0.210[209,210] 210ns |L0.210| "
|
||||
- "L0.211[210,211] 211ns |L0.211| "
|
||||
- "L0.212[211,212] 212ns |L0.212| "
|
||||
- "L0.213[212,213] 213ns |L0.213| "
|
||||
- "L0.214[213,214] 214ns |L0.214| "
|
||||
- "L0.215[214,215] 215ns |L0.215| "
|
||||
- "L0.216[215,216] 216ns |L0.216| "
|
||||
- "L0.217[216,217] 217ns |L0.217| "
|
||||
- "L0.218[217,218] 218ns |L0.218| "
|
||||
- "L0.219[218,219] 219ns |L0.219| "
|
||||
- "L0.220[219,220] 220ns |L0.220| "
|
||||
- "L0.221[220,221] 221ns |L0.221| "
|
||||
- "L0.222[221,222] 222ns |L0.222| "
|
||||
- "L0.223[222,223] 223ns |L0.223| "
|
||||
- "L0.224[223,224] 224ns |L0.224| "
|
||||
- "L0.225[224,225] 225ns |L0.225| "
|
||||
- "L0.226[225,226] 226ns |L0.226| "
|
||||
- "L0.227[226,227] 227ns |L0.227| "
|
||||
- "L0.228[227,228] 228ns |L0.228| "
|
||||
- "L0.229[228,229] 229ns |L0.229| "
|
||||
- "L0.230[229,230] 230ns |L0.230| "
|
||||
- "L0.231[230,231] 231ns |L0.231| "
|
||||
- "L0.232[231,232] 232ns |L0.232| "
|
||||
- "L0.233[232,233] 233ns |L0.233| "
|
||||
- "L0.234[233,234] 234ns |L0.234| "
|
||||
- "L0.235[234,235] 235ns |L0.235| "
|
||||
- "L0.236[235,236] 236ns |L0.236| "
|
||||
- "L0.237[236,237] 237ns |L0.237| "
|
||||
- "L0.238[237,238] 238ns |L0.238| "
|
||||
- "L0.239[238,239] 239ns |L0.239| "
|
||||
- "L0.240[239,240] 240ns |L0.240| "
|
||||
- "L0.241[240,241] 241ns |L0.241| "
|
||||
- "L0.242[241,242] 242ns |L0.242| "
|
||||
- "L0.243[242,243] 243ns |L0.243| "
|
||||
- "L0.244[243,244] 244ns |L0.244| "
|
||||
- "L0.245[244,245] 245ns |L0.245| "
|
||||
- "L0.246[245,246] 246ns |L0.246| "
|
||||
- "L0.247[246,247] 247ns |L0.247| "
|
||||
- "L0.248[247,248] 248ns |L0.248| "
|
||||
- "L0.249[248,249] 249ns |L0.249| "
|
||||
- "L0.250[249,250] 250ns |L0.250| "
|
||||
- "L0.251[250,251] 251ns |L0.251| "
|
||||
- "L0.252[251,252] 252ns |L0.252| "
|
||||
- "L0.253[252,253] 253ns |L0.253| "
|
||||
- "L0.254[253,254] 254ns |L0.254| "
|
||||
- "L0.255[254,255] 255ns |L0.255| "
|
||||
- "L0.256[255,256] 256ns |L0.256| "
|
||||
- "L0.257[256,257] 257ns |L0.257| "
|
||||
- "L0.258[257,258] 258ns |L0.258| "
|
||||
- "L0.259[258,259] 259ns |L0.259| "
|
||||
- "L0.260[259,260] 260ns |L0.260| "
|
||||
- "L0.261[260,261] 261ns |L0.261| "
|
||||
- "L0.262[261,262] 262ns |L0.262| "
|
||||
- "L0.263[262,263] 263ns |L0.263| "
|
||||
- "L0.264[263,264] 264ns |L0.264| "
|
||||
- "L0.265[264,265] 265ns |L0.265| "
|
||||
- "L0.266[265,266] 266ns |L0.266| "
|
||||
- "L0.267[266,267] 267ns |L0.267| "
|
||||
- "L0.268[267,268] 268ns |L0.268| "
|
||||
- "L0.269[268,269] 269ns |L0.269| "
|
||||
- "L0.270[269,270] 270ns |L0.270| "
|
||||
- "L0.271[270,271] 271ns |L0.271| "
|
||||
- "L0.272[271,272] 272ns |L0.272| "
|
||||
- "L0.273[272,273] 273ns |L0.273| "
|
||||
- "L0.274[273,274] 274ns |L0.274| "
|
||||
- "L0.275[274,275] 275ns |L0.275| "
|
||||
- "L0.276[275,276] 276ns |L0.276| "
|
||||
- "L0.277[276,277] 277ns |L0.277|"
|
||||
- "L0.278[277,278] 278ns |L0.278|"
|
||||
- "L0.279[278,279] 279ns |L0.279|"
|
||||
- "L0.280[279,280] 280ns |L0.280|"
|
||||
- "L0.281[280,281] 281ns |L0.281|"
|
||||
- "L0.282[281,282] 282ns |L0.282|"
|
||||
- "L0.283[282,283] 283ns |L0.283|"
|
||||
- "L0.284[283,284] 284ns |L0.284|"
|
||||
- "L0.285[284,285] 285ns |L0.285|"
|
||||
- "L0.286[285,286] 286ns |L0.286|"
|
||||
- "L0.287[286,287] 287ns |L0.287|"
|
||||
- "L0.288[287,288] 288ns |L0.288|"
|
||||
- "**** 3 Output Files (parquet_file_id not yet assigned), 276mb total:"
|
||||
- "**** Simulation run 1, type=split(CompactAndSplitOutput(FoundSubsetLessThanMaxCompactSize))(split_times=[151, 201]). 100 Input Files, 300mb total:"
|
||||
- "L0 "
|
||||
- "L0.?[150,200] 288ns 100mb|-------------L0.?-------------| "
|
||||
- "L0.?[201,250] 288ns 98mb |------------L0.?-------------| "
|
||||
- "L0.?[251,288] 288ns 78mb |---------L0.?---------| "
|
||||
- "Committing partition 1:"
|
||||
- " Soft Deleting 138 files: L0.151, L0.152, L0.153, L0.154, L0.155, L0.156, L0.157, L0.158, L0.159, L0.160, L0.161, L0.162, L0.163, L0.164, L0.165, L0.166, L0.167, L0.168, L0.169, L0.170, L0.171, L0.172, L0.173, L0.174, L0.175, L0.176, L0.177, L0.178, L0.179, L0.180, L0.181, L0.182, L0.183, L0.184, L0.185, L0.186, L0.187, L0.188, L0.189, L0.190, L0.191, L0.192, L0.193, L0.194, L0.195, L0.196, L0.197, L0.198, L0.199, L0.200, L0.201, L0.202, L0.203, L0.204, L0.205, L0.206, L0.207, L0.208, L0.209, L0.210, L0.211, L0.212, L0.213, L0.214, L0.215, L0.216, L0.217, L0.218, L0.219, L0.220, L0.221, L0.222, L0.223, L0.224, L0.225, L0.226, L0.227, L0.228, L0.229, L0.230, L0.231, L0.232, L0.233, L0.234, L0.235, L0.236, L0.237, L0.238, L0.239, L0.240, L0.241, L0.242, L0.243, L0.244, L0.245, L0.246, L0.247, L0.248, L0.249, L0.250, L0.251, L0.252, L0.253, L0.254, L0.255, L0.256, L0.257, L0.258, L0.259, L0.260, L0.261, L0.262, L0.263, L0.264, L0.265, L0.266, L0.267, L0.268, L0.269, L0.270, L0.271, L0.272, L0.273, L0.274, L0.275, L0.276, L0.277, L0.278, L0.279, L0.280, L0.281, L0.282, L0.283, L0.284, L0.285, L0.286, L0.287, L0.288"
|
||||
- " Creating 3 files"
|
||||
- "**** Simulation run 2, type=split(CompactAndSplitOutput(FoundSubsetLessThanMaxCompactSize))(split_times=[101, 151]). 3 Input Files, 300mb total:"
|
||||
- "L0 "
|
||||
- "L0.290[51,100] 150ns 98mb|----------L0.290-----------| "
|
||||
- "L0.291[101,150] 150ns 102mb |----------L0.291-----------| "
|
||||
- "L0.292[150,200] 288ns 100mb |-----------L0.292-----------| "
|
||||
- "L0.151[150,151] 151ns 2mb |L0.151| "
|
||||
- "L0.152[151,152] 152ns 2mb |L0.152| "
|
||||
- "L0.153[152,153] 153ns 2mb |L0.153| "
|
||||
- "L0.154[153,154] 154ns 2mb |L0.154| "
|
||||
- "L0.155[154,155] 155ns 2mb |L0.155| "
|
||||
- "L0.156[155,156] 156ns 2mb |L0.156| "
|
||||
- "L0.157[156,157] 157ns 2mb |L0.157| "
|
||||
- "L0.158[157,158] 158ns 2mb |L0.158| "
|
||||
- "L0.159[158,159] 159ns 2mb |L0.159| "
|
||||
- "L0.160[159,160] 160ns 2mb |L0.160| "
|
||||
- "L0.161[160,161] 161ns 2mb |L0.161| "
|
||||
- "L0.162[161,162] 162ns 2mb |L0.162| "
|
||||
- "L0.163[162,163] 163ns 2mb |L0.163| "
|
||||
- "L0.164[163,164] 164ns 2mb |L0.164| "
|
||||
- "L0.165[164,165] 165ns 2mb |L0.165| "
|
||||
- "L0.166[165,166] 166ns 2mb |L0.166| "
|
||||
- "L0.167[166,167] 167ns 2mb |L0.167| "
|
||||
- "L0.168[167,168] 168ns 2mb |L0.168| "
|
||||
- "L0.169[168,169] 169ns 2mb |L0.169| "
|
||||
- "L0.170[169,170] 170ns 2mb |L0.170| "
|
||||
- "L0.171[170,171] 171ns 2mb |L0.171| "
|
||||
- "L0.172[171,172] 172ns 2mb |L0.172| "
|
||||
- "L0.173[172,173] 173ns 2mb |L0.173| "
|
||||
- "L0.174[173,174] 174ns 2mb |L0.174| "
|
||||
- "L0.175[174,175] 175ns 2mb |L0.175| "
|
||||
- "L0.176[175,176] 176ns 2mb |L0.176| "
|
||||
- "L0.177[176,177] 177ns 2mb |L0.177| "
|
||||
- "L0.178[177,178] 178ns 2mb |L0.178| "
|
||||
- "L0.179[178,179] 179ns 2mb |L0.179| "
|
||||
- "L0.180[179,180] 180ns 2mb |L0.180| "
|
||||
- "L0.181[180,181] 181ns 2mb |L0.181| "
|
||||
- "L0.182[181,182] 182ns 2mb |L0.182| "
|
||||
- "L0.183[182,183] 183ns 2mb |L0.183| "
|
||||
- "L0.184[183,184] 184ns 2mb |L0.184| "
|
||||
- "L0.185[184,185] 185ns 2mb |L0.185| "
|
||||
- "L0.186[185,186] 186ns 2mb |L0.186| "
|
||||
- "L0.187[186,187] 187ns 2mb |L0.187| "
|
||||
- "L0.188[187,188] 188ns 2mb |L0.188| "
|
||||
- "L0.189[188,189] 189ns 2mb |L0.189| "
|
||||
- "L0.190[189,190] 190ns 2mb |L0.190| "
|
||||
- "L0.191[190,191] 191ns 2mb |L0.191| "
|
||||
- "L0.192[191,192] 192ns 2mb |L0.192| "
|
||||
- "L0.193[192,193] 193ns 2mb |L0.193| "
|
||||
- "L0.194[193,194] 194ns 2mb |L0.194| "
|
||||
- "L0.195[194,195] 195ns 2mb |L0.195| "
|
||||
- "L0.196[195,196] 196ns 2mb |L0.196| "
|
||||
- "L0.197[196,197] 197ns 2mb |L0.197| "
|
||||
- "L0.198[197,198] 198ns 2mb |L0.198| "
|
||||
- "L0.199[198,199] 199ns 2mb |L0.199| "
|
||||
- "L0.200[199,200] 200ns 2mb |L0.200| "
|
||||
- "L0.201[200,201] 201ns 2mb |L0.201| "
|
||||
- "L0.202[201,202] 202ns 2mb |L0.202| "
|
||||
- "L0.203[202,203] 203ns 2mb |L0.203| "
|
||||
- "L0.204[203,204] 204ns 2mb |L0.204| "
|
||||
- "L0.205[204,205] 205ns 2mb |L0.205| "
|
||||
- "L0.206[205,206] 206ns 2mb |L0.206| "
|
||||
- "L0.207[206,207] 207ns 2mb |L0.207| "
|
||||
- "L0.208[207,208] 208ns 2mb |L0.208| "
|
||||
- "L0.209[208,209] 209ns 2mb |L0.209| "
|
||||
- "L0.210[209,210] 210ns 2mb |L0.210| "
|
||||
- "L0.211[210,211] 211ns 2mb |L0.211| "
|
||||
- "L0.212[211,212] 212ns 2mb |L0.212| "
|
||||
- "L0.213[212,213] 213ns 2mb |L0.213| "
|
||||
- "L0.214[213,214] 214ns 2mb |L0.214| "
|
||||
- "L0.215[214,215] 215ns 2mb |L0.215| "
|
||||
- "L0.216[215,216] 216ns 2mb |L0.216| "
|
||||
- "L0.217[216,217] 217ns 2mb |L0.217| "
|
||||
- "L0.218[217,218] 218ns 2mb |L0.218| "
|
||||
- "L0.219[218,219] 219ns 2mb |L0.219| "
|
||||
- "L0.220[219,220] 220ns 2mb |L0.220| "
|
||||
- "L0.221[220,221] 221ns 2mb |L0.221| "
|
||||
- "L0.222[221,222] 222ns 2mb |L0.222| "
|
||||
- "L0.223[222,223] 223ns 2mb |L0.223| "
|
||||
- "L0.224[223,224] 224ns 2mb |L0.224| "
|
||||
- "L0.225[224,225] 225ns 2mb |L0.225| "
|
||||
- "L0.226[225,226] 226ns 2mb |L0.226| "
|
||||
- "L0.227[226,227] 227ns 2mb |L0.227| "
|
||||
- "L0.228[227,228] 228ns 2mb |L0.228| "
|
||||
- "L0.229[228,229] 229ns 2mb |L0.229| "
|
||||
- "L0.230[229,230] 230ns 2mb |L0.230| "
|
||||
- "L0.231[230,231] 231ns 2mb |L0.231| "
|
||||
- "L0.232[231,232] 232ns 2mb |L0.232| "
|
||||
- "L0.233[232,233] 233ns 2mb |L0.233| "
|
||||
- "L0.234[233,234] 234ns 2mb |L0.234| "
|
||||
- "L0.235[234,235] 235ns 2mb |L0.235| "
|
||||
- "L0.236[235,236] 236ns 2mb |L0.236| "
|
||||
- "L0.237[236,237] 237ns 2mb |L0.237|"
|
||||
- "L0.238[237,238] 238ns 2mb |L0.238|"
|
||||
- "L0.239[238,239] 239ns 2mb |L0.239|"
|
||||
- "L0.240[239,240] 240ns 2mb |L0.240|"
|
||||
- "L0.241[240,241] 241ns 2mb |L0.241|"
|
||||
- "L0.242[241,242] 242ns 2mb |L0.242|"
|
||||
- "L0.243[242,243] 243ns 2mb |L0.243|"
|
||||
- "L0.244[243,244] 244ns 2mb |L0.244|"
|
||||
- "L0.245[244,245] 245ns 2mb |L0.245|"
|
||||
- "L0.246[245,246] 246ns 2mb |L0.246|"
|
||||
- "L0.247[246,247] 247ns 2mb |L0.247|"
|
||||
- "L0.248[247,248] 248ns 2mb |L0.248|"
|
||||
- "L0.249[248,249] 249ns 2mb |L0.249|"
|
||||
- "L1 "
|
||||
- "L1.291[101,150] 150ns 102mb|----------L1.291-----------| "
|
||||
- "**** 3 Output Files (parquet_file_id not yet assigned), 300mb total:"
|
||||
- "L1 "
|
||||
- "L1.?[51,101] 288ns 101mb |------------L1.?------------| "
|
||||
- "L1.?[102,151] 288ns 99mb |-----------L1.?------------| "
|
||||
- "L1.?[152,200] 288ns 101mb |-----------L1.?-----------| "
|
||||
- "L1.?[101,151] 249ns 101mb|------------L1.?------------| "
|
||||
- "L1.?[152,201] 249ns 99mb |-----------L1.?------------| "
|
||||
- "L1.?[202,249] 249ns 99mb |-----------L1.?-----------| "
|
||||
- "Committing partition 1:"
|
||||
- " Soft Deleting 3 files: L0.290, L0.291, L0.292"
|
||||
- " Upgrading 1 files level to CompactionLevel::L1: L0.289"
|
||||
- " Soft Deleting 100 files: L0.151, L0.152, L0.153, L0.154, L0.155, L0.156, L0.157, L0.158, L0.159, L0.160, L0.161, L0.162, L0.163, L0.164, L0.165, L0.166, L0.167, L0.168, L0.169, L0.170, L0.171, L0.172, L0.173, L0.174, L0.175, L0.176, L0.177, L0.178, L0.179, L0.180, L0.181, L0.182, L0.183, L0.184, L0.185, L0.186, L0.187, L0.188, L0.189, L0.190, L0.191, L0.192, L0.193, L0.194, L0.195, L0.196, L0.197, L0.198, L0.199, L0.200, L0.201, L0.202, L0.203, L0.204, L0.205, L0.206, L0.207, L0.208, L0.209, L0.210, L0.211, L0.212, L0.213, L0.214, L0.215, L0.216, L0.217, L0.218, L0.219, L0.220, L0.221, L0.222, L0.223, L0.224, L0.225, L0.226, L0.227, L0.228, L0.229, L0.230, L0.231, L0.232, L0.233, L0.234, L0.235, L0.236, L0.237, L0.238, L0.239, L0.240, L0.241, L0.242, L0.243, L0.244, L0.245, L0.246, L0.247, L0.248, L0.249, L1.291"
|
||||
- " Creating 3 files"
|
||||
- "**** Simulation run 3, type=split(CompactAndSplitOutput(TotalSizeLessThanMaxCompactSize))(split_times=[251]). 2 Input Files, 176mb total:"
|
||||
- "**** Simulation run 2, type=split(CompactAndSplitOutput(TotalSizeLessThanMaxCompactSize))(split_times=[251]). 40 Input Files, 177mb total:"
|
||||
- "L0 "
|
||||
- "L0.294[251,288] 288ns 78mb |---------------L0.294---------------| "
|
||||
- "L0.293[201,250] 288ns 98mb|---------------------L0.293---------------------| "
|
||||
- "**** 2 Output Files (parquet_file_id not yet assigned), 176mb total:"
|
||||
- "L0.288[287,288] 288ns 2mb |L0.288|"
|
||||
- "L0.287[286,287] 287ns 2mb |L0.287|"
|
||||
- "L0.286[285,286] 286ns 2mb |L0.286|"
|
||||
- "L0.285[284,285] 285ns 2mb |L0.285|"
|
||||
- "L0.284[283,284] 284ns 2mb |L0.284|"
|
||||
- "L0.283[282,283] 283ns 2mb |L0.283|"
|
||||
- "L0.282[281,282] 282ns 2mb |L0.282|"
|
||||
- "L0.281[280,281] 281ns 2mb |L0.281| "
|
||||
- "L0.280[279,280] 280ns 2mb |L0.280| "
|
||||
- "L0.279[278,279] 279ns 2mb |L0.279| "
|
||||
- "L0.278[277,278] 278ns 2mb |L0.278| "
|
||||
- "L0.277[276,277] 277ns 2mb |L0.277| "
|
||||
- "L0.276[275,276] 276ns 2mb |L0.276| "
|
||||
- "L0.275[274,275] 275ns 2mb |L0.275| "
|
||||
- "L0.274[273,274] 274ns 2mb |L0.274| "
|
||||
- "L0.273[272,273] 273ns 2mb |L0.273| "
|
||||
- "L0.272[271,272] 272ns 2mb |L0.272| "
|
||||
- "L0.271[270,271] 271ns 2mb |L0.271| "
|
||||
- "L0.270[269,270] 270ns 2mb |L0.270| "
|
||||
- "L0.269[268,269] 269ns 2mb |L0.269| "
|
||||
- "L0.268[267,268] 268ns 2mb |L0.268| "
|
||||
- "L0.267[266,267] 267ns 2mb |L0.267| "
|
||||
- "L0.266[265,266] 266ns 2mb |L0.266| "
|
||||
- "L0.265[264,265] 265ns 2mb |L0.265| "
|
||||
- "L0.264[263,264] 264ns 2mb |L0.264| "
|
||||
- "L0.263[262,263] 263ns 2mb |L0.263| "
|
||||
- "L0.262[261,262] 262ns 2mb |L0.262| "
|
||||
- "L0.261[260,261] 261ns 2mb |L0.261| "
|
||||
- "L0.260[259,260] 260ns 2mb |L0.260| "
|
||||
- "L0.259[258,259] 259ns 2mb |L0.259| "
|
||||
- "L0.258[257,258] 258ns 2mb |L0.258| "
|
||||
- "L0.257[256,257] 257ns 2mb |L0.257| "
|
||||
- "L0.256[255,256] 256ns 2mb |L0.256| "
|
||||
- "L0.255[254,255] 255ns 2mb |L0.255| "
|
||||
- "L0.254[253,254] 254ns 2mb |L0.254| "
|
||||
- "L0.253[252,253] 253ns 2mb |L0.253| "
|
||||
- "L0.252[251,252] 252ns 2mb |L0.252| "
|
||||
- "L0.251[250,251] 251ns 2mb |L0.251| "
|
||||
- "L0.250[249,250] 250ns 2mb |L0.250| "
|
||||
- "L1 "
|
||||
- "L1.?[201,251] 288ns 101mb|----------------------L1.?-----------------------| "
|
||||
- "L1.?[252,288] 288ns 75mb |---------------L1.?----------------| "
|
||||
- "Committing partition 1:"
|
||||
- " Soft Deleting 2 files: L0.293, L0.294"
|
||||
- " Creating 2 files"
|
||||
- "**** Simulation run 4, type=split(CompactAndSplitOutput(FoundSubsetLessThanMaxCompactSize))(split_times=[152]). 2 Input Files, 199mb total:"
|
||||
- "L1.294[202,249] 249ns 99mb|--------------------L1.294---------------------| "
|
||||
- "**** 2 Output Files (parquet_file_id not yet assigned), 177mb total:"
|
||||
- "L1 "
|
||||
- "L1.296[102,151] 288ns 99mb|------------------L1.296-------------------| "
|
||||
- "L1.297[152,200] 288ns 101mb |------------------L1.297------------------| "
|
||||
- "**** 2 Output Files (parquet_file_id not yet assigned), 199mb total:"
|
||||
- "L2 "
|
||||
- "L2.?[102,152] 288ns 102mb|-------------------L2.?--------------------| "
|
||||
- "L2.?[153,200] 288ns 98mb |------------------L2.?-------------------| "
|
||||
- "L1.?[202,251] 288ns 101mb|----------------------L1.?-----------------------| "
|
||||
- "L1.?[252,288] 288ns 76mb |---------------L1.?----------------| "
|
||||
- "Committing partition 1:"
|
||||
- " Soft Deleting 2 files: L1.296, L1.297"
|
||||
- " Upgrading 2 files level to CompactionLevel::L2: L1.289, L1.295"
|
||||
- " Soft Deleting 40 files: L0.250, L0.251, L0.252, L0.253, L0.254, L0.255, L0.256, L0.257, L0.258, L0.259, L0.260, L0.261, L0.262, L0.263, L0.264, L0.265, L0.266, L0.267, L0.268, L0.269, L0.270, L0.271, L0.272, L0.273, L0.274, L0.275, L0.276, L0.277, L0.278, L0.279, L0.280, L0.281, L0.282, L0.283, L0.284, L0.285, L0.286, L0.287, L0.288, L1.294"
|
||||
- " Creating 2 files"
|
||||
- "**** Simulation run 5, type=split(CompactAndSplitOutput(TotalSizeLessThanMaxCompactSize))(split_times=[280]). 1 Input Files, 75mb total:"
|
||||
- "L1, all files 75mb "
|
||||
- "L1.299[252,288] 288ns |-----------------------------------------L1.299-----------------------------------------|"
|
||||
- "**** 2 Output Files (parquet_file_id not yet assigned), 75mb total:"
|
||||
- "**** Simulation run 3, type=split(CompactAndSplitOutput(FoundSubsetLessThanMaxCompactSize))(split_times=[102, 153]). 3 Input Files, 299mb total:"
|
||||
- "L1 "
|
||||
- "L1.290[51,100] 150ns 98mb|----------L1.290-----------| "
|
||||
- "L1.292[101,151] 249ns 101mb |-----------L1.292-----------| "
|
||||
- "L1.293[152,201] 249ns 99mb |----------L1.293-----------| "
|
||||
- "**** 3 Output Files (parquet_file_id not yet assigned), 299mb total:"
|
||||
- "L2 "
|
||||
- "L2.?[252,280] 288ns 58mb |--------------------------------L2.?--------------------------------| "
|
||||
- "L2.?[51,102] 249ns 102mb |------------L2.?------------| "
|
||||
- "L2.?[103,153] 249ns 100mb |------------L2.?------------| "
|
||||
- "L2.?[154,201] 249ns 98mb |-----------L2.?-----------| "
|
||||
- "Committing partition 1:"
|
||||
- " Soft Deleting 3 files: L1.290, L1.292, L1.293"
|
||||
- " Upgrading 1 files level to CompactionLevel::L2: L1.289"
|
||||
- " Creating 3 files"
|
||||
- "**** Simulation run 4, type=split(CompactAndSplitOutput(TotalSizeLessThanMaxCompactSize))(split_times=[280]). 1 Input Files, 76mb total:"
|
||||
- "L1, all files 76mb "
|
||||
- "L1.296[252,288] 288ns |-----------------------------------------L1.296-----------------------------------------|"
|
||||
- "**** 2 Output Files (parquet_file_id not yet assigned), 76mb total:"
|
||||
- "L2 "
|
||||
- "L2.?[252,280] 288ns 59mb |--------------------------------L2.?--------------------------------| "
|
||||
- "L2.?[281,288] 288ns 17mb |-----L2.?------| "
|
||||
- "Committing partition 1:"
|
||||
- " Soft Deleting 1 files: L1.299"
|
||||
- " Upgrading 1 files level to CompactionLevel::L2: L1.298"
|
||||
- " Soft Deleting 1 files: L1.296"
|
||||
- " Upgrading 1 files level to CompactionLevel::L2: L1.295"
|
||||
- " Creating 2 files"
|
||||
- "**** Final Output Files (1.3gb written)"
|
||||
- "**** Final Output Files (1.13gb written)"
|
||||
- "L2 "
|
||||
- "L2.289[0,50] 150ns 100mb |---L2.289----| "
|
||||
- "L2.295[51,101] 288ns 101mb |---L2.295----| "
|
||||
- "L2.298[201,251] 288ns 101mb |---L2.298----| "
|
||||
- "L2.300[102,152] 288ns 102mb |---L2.300----| "
|
||||
- "L2.301[153,200] 288ns 98mb |---L2.301---| "
|
||||
- "L2.302[252,280] 288ns 58mb |L2.302| "
|
||||
- "L2.303[281,288] 288ns 17mb |L2.303|"
|
||||
- "L2.295[202,251] 288ns 101mb |---L2.295----| "
|
||||
- "L2.297[51,102] 249ns 102mb |---L2.297----| "
|
||||
- "L2.298[103,153] 249ns 100mb |---L2.298----| "
|
||||
- "L2.299[154,201] 249ns 98mb |---L2.299---| "
|
||||
- "L2.300[252,280] 288ns 59mb |L2.300| "
|
||||
- "L2.301[281,288] 288ns 17mb |L2.301|"
|
||||
"###
|
||||
);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -58,6 +58,7 @@ mod large_files;
|
|||
mod large_overlaps;
|
||||
mod many_files;
|
||||
mod single_timestamp;
|
||||
mod stuck;
|
||||
|
||||
use std::{sync::atomic::Ordering, time::Duration};
|
||||
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load Diff
Loading…
Reference in New Issue