diff --git a/compactor/tests/integration.rs b/compactor/tests/integration.rs index 31ee42ec11..fbf60bffd2 100644 --- a/compactor/tests/integration.rs +++ b/compactor/tests/integration.rs @@ -68,8 +68,8 @@ async fn test_num_files_over_limit() { assert_levels( &files, vec![ + (8, CompactionLevel::FileNonOverlapped), (9, CompactionLevel::FileNonOverlapped), - (10, CompactionLevel::FileNonOverlapped), ], ); } diff --git a/compactor/tests/layouts/backfill.rs b/compactor/tests/layouts/backfill.rs index eca308e656..708d5d95b1 100644 --- a/compactor/tests/layouts/backfill.rs +++ b/compactor/tests/layouts/backfill.rs @@ -146,7 +146,7 @@ async fn random_backfill_empty_partition() { - "Committing partition 1:" - " Soft Deleting 1 files: L0.51" - " Creating 1 files" - - "**** Simulation run 1, type=split(CompactAndSplitOutput(ManySmallFiles))(split_times=[514]). 20 Input Files, 200mb total:" + - "**** Simulation run 1, type=compact(ManySmallFiles). 20 Input Files, 200mb total:" - "L0, all files 10mb " - "L0.1[76,932] 1us |-------------------------------------L0.1--------------------------------------| " - "L0.2[42,986] 1us |------------------------------------------L0.2------------------------------------------|" @@ -168,14 +168,13 @@ async fn random_backfill_empty_partition() { - "L0.18[42,986] 1.02us |-----------------------------------------L0.18------------------------------------------|" - "L0.19[173,950] 1.02us |---------------------------------L0.19----------------------------------| " - "L0.20[50,629] 1.02us |------------------------L0.20------------------------| " - - "**** 2 Output Files (parquet_file_id not yet assigned), 200mb total:" - - "L0, all files 100mb " - - "L0.?[42,514] 1.02us |-------------------L0.?--------------------| " - - "L0.?[515,986] 1.02us |-------------------L0.?-------------------| " + - "**** 1 Output Files (parquet_file_id not yet assigned), 200mb total:" + - "L0, all files 200mb " + - "L0.?[42,986] 1.02us |------------------------------------------L0.?------------------------------------------|" - "Committing partition 1:" - " Soft Deleting 20 files: L0.1, L0.2, L0.3, L0.4, L0.5, L0.6, L0.7, L0.8, L0.9, L0.10, L0.11, L0.12, L0.13, L0.14, L0.15, L0.16, L0.17, L0.18, L0.19, L0.20" - - " Creating 2 files" - - "**** Simulation run 2, type=split(CompactAndSplitOutput(ManySmallFiles))(split_times=[514]). 20 Input Files, 200mb total:" + - " Creating 1 files" + - "**** Simulation run 2, type=compact(ManySmallFiles). 20 Input Files, 200mb total:" - "L0, all files 10mb " - "L0.21[76,932] 1.02us |-------------------------------------L0.21-------------------------------------| " - "L0.22[42,986] 1.02us |-----------------------------------------L0.22------------------------------------------|" @@ -197,14 +196,13 @@ async fn random_backfill_empty_partition() { - "L0.38[42,986] 1.04us |-----------------------------------------L0.38------------------------------------------|" - "L0.39[173,950] 1.04us |---------------------------------L0.39----------------------------------| " - "L0.40[50,629] 1.04us |------------------------L0.40------------------------| " - - "**** 2 Output Files (parquet_file_id not yet assigned), 200mb total:" - - "L0, all files 100mb " - - "L0.?[42,514] 1.04us |-------------------L0.?--------------------| " - - "L0.?[515,986] 1.04us |-------------------L0.?-------------------| " + - "**** 1 Output Files (parquet_file_id not yet assigned), 200mb total:" + - "L0, all files 200mb " + - "L0.?[42,986] 1.04us |------------------------------------------L0.?------------------------------------------|" - "Committing partition 1:" - " Soft Deleting 20 files: L0.21, L0.22, L0.23, L0.24, L0.25, L0.26, L0.27, L0.28, L0.29, L0.30, L0.31, L0.32, L0.33, L0.34, L0.35, L0.36, L0.37, L0.38, L0.39, L0.40" - - " Creating 2 files" - - "**** Simulation run 3, type=split(CompactAndSplitOutput(ManySmallFiles))(split_times=[797]). 10 Input Files, 100mb total:" + - " Creating 1 files" + - "**** Simulation run 3, type=compact(ManySmallFiles). 10 Input Files, 100mb total:" - "L0, all files 10mb " - "L0.41[76,932] 1.04us |-------------------------------------L0.41-------------------------------------| " - "L0.42[42,986] 1.04us |-----------------------------------------L0.42------------------------------------------|" @@ -216,224 +214,208 @@ async fn random_backfill_empty_partition() { - "L0.48[50,629] 1.05us |------------------------L0.48------------------------| " - "L0.49[76,932] 1.05us |-------------------------------------L0.49-------------------------------------| " - "L0.50[42,986] 1.05us |-----------------------------------------L0.50------------------------------------------|" - - "**** 2 Output Files (parquet_file_id not yet assigned), 100mb total:" - - "L0 " - - "L0.?[42,797] 1.05us 80mb |--------------------------------L0.?---------------------------------| " - - "L0.?[798,986] 1.05us 20mb |-----L0.?------| " + - "**** 1 Output Files (parquet_file_id not yet assigned), 100mb total:" + - "L0, all files 100mb " + - "L0.?[42,986] 1.05us |------------------------------------------L0.?------------------------------------------|" - "Committing partition 1:" - " Soft Deleting 10 files: L0.41, L0.42, L0.43, L0.44, L0.45, L0.46, L0.47, L0.48, L0.49, L0.50" - - " Creating 2 files" - - "**** Simulation run 4, type=split(HighL0OverlapSingleFile)(split_times=[670]). 1 Input Files, 100mb total:" + - " Creating 1 files" + - "**** Simulation run 4, type=split(HighL0OverlapSingleFile)(split_times=[356, 670]). 1 Input Files, 200mb total:" + - "L0, all files 200mb " + - "L0.53[42,986] 1.02us |-----------------------------------------L0.53------------------------------------------|" + - "**** 3 Output Files (parquet_file_id not yet assigned), 200mb total:" + - "L0 " + - "L0.?[42,356] 1.02us 67mb |-----------L0.?------------| " + - "L0.?[357,670] 1.02us 66mb |-----------L0.?------------| " + - "L0.?[671,986] 1.02us 67mb |------------L0.?------------| " + - "**** Simulation run 5, type=split(HighL0OverlapSingleFile)(split_times=[356, 670]). 1 Input Files, 200mb total:" + - "L0, all files 200mb " + - "L0.54[42,986] 1.04us |-----------------------------------------L0.54------------------------------------------|" + - "**** 3 Output Files (parquet_file_id not yet assigned), 200mb total:" + - "L0 " + - "L0.?[42,356] 1.04us 67mb |-----------L0.?------------| " + - "L0.?[357,670] 1.04us 66mb |-----------L0.?------------| " + - "L0.?[671,986] 1.04us 67mb |------------L0.?------------| " + - "**** Simulation run 6, type=split(HighL0OverlapSingleFile)(split_times=[356, 670]). 1 Input Files, 100mb total:" - "L0, all files 100mb " - - "L0.54[515,986] 1.02us |-----------------------------------------L0.54------------------------------------------|" - - "**** 2 Output Files (parquet_file_id not yet assigned), 100mb total:" + - "L0.55[42,986] 1.05us |-----------------------------------------L0.55------------------------------------------|" + - "**** 3 Output Files (parquet_file_id not yet assigned), 100mb total:" - "L0 " - - "L0.?[515,670] 1.02us 33mb|-----------L0.?------------| " - - "L0.?[671,986] 1.02us 67mb |---------------------------L0.?---------------------------| " - - "**** Simulation run 5, type=split(HighL0OverlapSingleFile)(split_times=[356]). 1 Input Files, 100mb total:" - - "L0, all files 100mb " - - "L0.53[42,514] 1.02us |-----------------------------------------L0.53------------------------------------------|" - - "**** 2 Output Files (parquet_file_id not yet assigned), 100mb total:" - - "L0 " - - "L0.?[42,356] 1.02us 67mb |--------------------------L0.?---------------------------| " - - "L0.?[357,514] 1.02us 33mb |-----------L0.?------------| " - - "**** Simulation run 6, type=split(HighL0OverlapSingleFile)(split_times=[670]). 1 Input Files, 100mb total:" - - "L0, all files 100mb " - - "L0.56[515,986] 1.04us |-----------------------------------------L0.56------------------------------------------|" - - "**** 2 Output Files (parquet_file_id not yet assigned), 100mb total:" - - "L0 " - - "L0.?[515,670] 1.04us 33mb|-----------L0.?------------| " - - "L0.?[671,986] 1.04us 67mb |---------------------------L0.?---------------------------| " - - "**** Simulation run 7, type=split(HighL0OverlapSingleFile)(split_times=[356]). 1 Input Files, 100mb total:" - - "L0, all files 100mb " - - "L0.55[42,514] 1.04us |-----------------------------------------L0.55------------------------------------------|" - - "**** 2 Output Files (parquet_file_id not yet assigned), 100mb total:" - - "L0 " - - "L0.?[42,356] 1.04us 67mb |--------------------------L0.?---------------------------| " - - "L0.?[357,514] 1.04us 33mb |-----------L0.?------------| " - - "**** Simulation run 8, type=split(HighL0OverlapSingleFile)(split_times=[356, 670]). 1 Input Files, 80mb total:" - - "L0, all files 80mb " - - "L0.57[42,797] 1.05us |-----------------------------------------L0.57------------------------------------------|" - - "**** 3 Output Files (parquet_file_id not yet assigned), 80mb total:" - - "L0 " - - "L0.?[42,356] 1.05us 33mb |---------------L0.?----------------| " - - "L0.?[357,670] 1.05us 33mb |---------------L0.?----------------| " - - "L0.?[671,797] 1.05us 14mb |----L0.?-----| " + - "L0.?[42,356] 1.05us 33mb |-----------L0.?------------| " + - "L0.?[357,670] 1.05us 33mb |-----------L0.?------------| " + - "L0.?[671,986] 1.05us 34mb |------------L0.?------------| " - "Committing partition 1:" - - " Soft Deleting 5 files: L0.53, L0.54, L0.55, L0.56, L0.57" - - " Creating 11 files" - - "**** Simulation run 9, type=split(CompactAndSplitOutput(FoundSubsetLessThanMaxCompactSize))(split_times=[357, 714]). 6 Input Files, 277mb total:" + - " Soft Deleting 3 files: L0.53, L0.54, L0.55" + - " Creating 9 files" + - "**** Simulation run 7, type=split(CompactAndSplitOutput(FoundSubsetLessThanMaxCompactSize))(split_times=[357, 714]). 5 Input Files, 277mb total:" - "L0 " - "L0.52[0,1] 999ns 10mb |L0.52| " - - "L0.61[42,356] 1.02us 67mb |----------L0.61-----------| " - - "L0.62[357,514] 1.02us 33mb |---L0.62----| " - - "L0.59[515,670] 1.02us 33mb |---L0.59----| " - - "L0.60[671,986] 1.02us 67mb |----------L0.60-----------| " - - "L0.65[42,356] 1.04us 67mb |----------L0.65-----------| " + - "L0.56[42,356] 1.02us 67mb |----------L0.56-----------| " + - "L0.57[357,670] 1.02us 66mb |----------L0.57-----------| " + - "L0.58[671,986] 1.02us 67mb |----------L0.58-----------| " + - "L0.59[42,356] 1.04us 67mb |----------L0.59-----------| " - "**** 3 Output Files (parquet_file_id not yet assigned), 277mb total:" - "L1 " - "L1.?[0,357] 1.04us 100mb |-------------L1.?-------------| " - "L1.?[358,714] 1.04us 100mb |-------------L1.?-------------| " - "L1.?[715,986] 1.04us 77mb |---------L1.?---------| " - "Committing partition 1:" - - " Soft Deleting 6 files: L0.52, L0.59, L0.60, L0.61, L0.62, L0.65" + - " Soft Deleting 5 files: L0.52, L0.56, L0.57, L0.58, L0.59" - " Creating 3 files" - - "**** Simulation run 10, type=split(ReduceOverlap)(split_times=[714]). 1 Input Files, 67mb total:" + - "**** Simulation run 8, type=split(ReduceOverlap)(split_times=[714]). 1 Input Files, 67mb total:" - "L0, all files 67mb " - - "L0.64[671,986] 1.04us |-----------------------------------------L0.64------------------------------------------|" + - "L0.61[671,986] 1.04us |-----------------------------------------L0.61------------------------------------------|" - "**** 2 Output Files (parquet_file_id not yet assigned), 67mb total:" - "L0 " - "L0.?[671,714] 1.04us 9mb |---L0.?---| " - "L0.?[715,986] 1.04us 58mb |-----------------------------------L0.?------------------------------------| " - - "**** Simulation run 11, type=split(ReduceOverlap)(split_times=[357]). 1 Input Files, 33mb total:" - - "L0, all files 33mb " - - "L0.66[357,514] 1.04us |-----------------------------------------L0.66------------------------------------------|" - - "**** 2 Output Files (parquet_file_id not yet assigned), 33mb total:" + - "**** Simulation run 9, type=split(ReduceOverlap)(split_times=[357]). 1 Input Files, 66mb total:" + - "L0, all files 66mb " + - "L0.60[357,670] 1.04us |-----------------------------------------L0.60------------------------------------------|" + - "**** 2 Output Files (parquet_file_id not yet assigned), 66mb total:" - "L0 " - "L0.?[357,357] 1.04us 0b |L0.?| " - - "L0.?[358,514] 1.04us 33mb|-----------------------------------------L0.?------------------------------------------| " - - "**** Simulation run 12, type=split(ReduceOverlap)(split_times=[714]). 1 Input Files, 14mb total:" - - "L0, all files 14mb " - - "L0.69[671,797] 1.05us |-----------------------------------------L0.69------------------------------------------|" - - "**** 2 Output Files (parquet_file_id not yet assigned), 14mb total:" + - "L0.?[358,670] 1.04us 66mb|-----------------------------------------L0.?------------------------------------------| " + - "**** Simulation run 10, type=split(ReduceOverlap)(split_times=[714]). 1 Input Files, 34mb total:" + - "L0, all files 34mb " + - "L0.64[671,986] 1.05us |-----------------------------------------L0.64------------------------------------------|" + - "**** 2 Output Files (parquet_file_id not yet assigned), 34mb total:" - "L0 " - - "L0.?[671,714] 1.05us 5mb |------------L0.?------------| " - - "L0.?[715,797] 1.05us 9mb |--------------------------L0.?--------------------------| " - - "**** Simulation run 13, type=split(ReduceOverlap)(split_times=[357]). 1 Input Files, 33mb total:" + - "L0.?[671,714] 1.05us 5mb |---L0.?---| " + - "L0.?[715,986] 1.05us 29mb |-----------------------------------L0.?------------------------------------| " + - "**** Simulation run 11, type=split(ReduceOverlap)(split_times=[357]). 1 Input Files, 33mb total:" - "L0, all files 33mb " - - "L0.68[357,670] 1.05us |-----------------------------------------L0.68------------------------------------------|" + - "L0.63[357,670] 1.05us |-----------------------------------------L0.63------------------------------------------|" - "**** 2 Output Files (parquet_file_id not yet assigned), 33mb total:" - "L0 " - "L0.?[357,357] 1.05us 0b |L0.?| " - "L0.?[358,670] 1.05us 33mb|-----------------------------------------L0.?------------------------------------------| " - "Committing partition 1:" - - " Soft Deleting 4 files: L0.64, L0.66, L0.68, L0.69" + - " Soft Deleting 4 files: L0.60, L0.61, L0.63, L0.64" - " Creating 8 files" - - "**** Simulation run 14, type=split(CompactAndSplitOutput(FoundSubsetLessThanMaxCompactSize))(split_times=[260, 520]). 6 Input Files, 276mb total:" + - "**** Simulation run 12, type=split(CompactAndSplitOutput(FoundSubsetLessThanMaxCompactSize))(split_times=[260, 520]). 5 Input Files, 275mb total:" - "L0 " - - "L0.75[357,357] 1.04us 0b |L0.75| " - - "L0.76[358,514] 1.04us 33mb |------L0.76------| " - - "L0.63[515,670] 1.04us 33mb |------L0.63------| " - - "L0.73[671,714] 1.04us 9mb |L0.73|" + - "L0.70[357,357] 1.04us 0b |L0.70| " + - "L0.71[358,670] 1.04us 66mb |----------------L0.71----------------| " + - "L0.68[671,714] 1.04us 9mb |L0.68|" - "L1 " - - "L1.70[0,357] 1.04us 100mb|-------------------L1.70-------------------| " - - "L1.71[358,714] 1.04us 100mb |------------------L1.71-------------------| " - - "**** 3 Output Files (parquet_file_id not yet assigned), 276mb total:" + - "L1.65[0,357] 1.04us 100mb|-------------------L1.65-------------------| " + - "L1.66[358,714] 1.04us 100mb |------------------L1.66-------------------| " + - "**** 3 Output Files (parquet_file_id not yet assigned), 275mb total:" - "L1 " - "L1.?[0,260] 1.04us 100mb |-------------L1.?-------------| " - "L1.?[261,520] 1.04us 100mb |-------------L1.?-------------| " - "L1.?[521,714] 1.04us 75mb |---------L1.?---------| " - "Committing partition 1:" - - " Soft Deleting 6 files: L0.63, L1.70, L1.71, L0.73, L0.75, L0.76" + - " Soft Deleting 5 files: L1.65, L1.66, L0.68, L0.70, L0.71" - " Creating 3 files" - - "**** Simulation run 15, type=split(ReduceOverlap)(split_times=[520]). 1 Input Files, 33mb total:" + - "**** Simulation run 13, type=split(ReduceOverlap)(split_times=[520]). 1 Input Files, 33mb total:" - "L0, all files 33mb " - - "L0.80[358,670] 1.05us |-----------------------------------------L0.80------------------------------------------|" + - "L0.75[358,670] 1.05us |-----------------------------------------L0.75------------------------------------------|" - "**** 2 Output Files (parquet_file_id not yet assigned), 33mb total:" - "L0 " - "L0.?[358,520] 1.05us 17mb|--------------------L0.?--------------------| " - "L0.?[521,670] 1.05us 16mb |------------------L0.?------------------| " - - "**** Simulation run 16, type=split(ReduceOverlap)(split_times=[260]). 1 Input Files, 33mb total:" + - "**** Simulation run 14, type=split(ReduceOverlap)(split_times=[260]). 1 Input Files, 33mb total:" - "L0, all files 33mb " - - "L0.67[42,356] 1.05us |-----------------------------------------L0.67------------------------------------------|" + - "L0.62[42,356] 1.05us |-----------------------------------------L0.62------------------------------------------|" - "**** 2 Output Files (parquet_file_id not yet assigned), 33mb total:" - "L0 " - "L0.?[42,260] 1.05us 23mb |----------------------------L0.?----------------------------| " - "L0.?[261,356] 1.05us 10mb |----------L0.?-----------| " - "Committing partition 1:" - - " Soft Deleting 2 files: L0.67, L0.80" + - " Soft Deleting 2 files: L0.62, L0.75" - " Creating 4 files" - - "**** Simulation run 17, type=split(CompactAndSplitOutput(FoundSubsetLessThanMaxCompactSize))(split_times=[917]). 2 Input Files, 134mb total:" + - "**** Simulation run 15, type=split(CompactAndSplitOutput(FoundSubsetLessThanMaxCompactSize))(split_times=[917]). 2 Input Files, 135mb total:" - "L0 " - - "L0.74[715,986] 1.04us 58mb|-----------------------------------------L0.74------------------------------------------|" + - "L0.69[715,986] 1.04us 58mb|-----------------------------------------L0.69------------------------------------------|" - "L1 " - - "L1.72[715,986] 1.04us 77mb|-----------------------------------------L1.72------------------------------------------|" - - "**** 2 Output Files (parquet_file_id not yet assigned), 134mb total:" + - "L1.67[715,986] 1.04us 77mb|-----------------------------------------L1.67------------------------------------------|" + - "**** 2 Output Files (parquet_file_id not yet assigned), 135mb total:" - "L1 " - "L1.?[715,917] 1.04us 100mb|------------------------------L1.?-------------------------------| " - "L1.?[918,986] 1.04us 34mb |--------L1.?--------| " - "Committing partition 1:" - - " Soft Deleting 2 files: L1.72, L0.74" + - " Soft Deleting 2 files: L1.67, L0.69" - " Creating 2 files" - - "**** Simulation run 18, type=split(ReduceOverlap)(split_times=[917]). 1 Input Files, 20mb total:" - - "L0, all files 20mb " - - "L0.58[798,986] 1.05us |-----------------------------------------L0.58------------------------------------------|" - - "**** 2 Output Files (parquet_file_id not yet assigned), 20mb total:" + - "**** Simulation run 16, type=split(ReduceOverlap)(split_times=[917]). 1 Input Files, 29mb total:" + - "L0, all files 29mb " + - "L0.73[715,986] 1.05us |-----------------------------------------L0.73------------------------------------------|" + - "**** 2 Output Files (parquet_file_id not yet assigned), 29mb total:" - "L0 " - - "L0.?[798,917] 1.05us 13mb|-------------------------L0.?-------------------------| " - - "L0.?[918,986] 1.05us 7mb |-------------L0.?-------------| " + - "L0.?[715,917] 1.05us 22mb|------------------------------L0.?-------------------------------| " + - "L0.?[918,986] 1.05us 7mb |--------L0.?--------| " - "Committing partition 1:" - - " Soft Deleting 1 files: L0.58" + - " Soft Deleting 1 files: L0.73" - " Creating 2 files" - - "**** Simulation run 19, type=split(CompactAndSplitOutput(FoundSubsetLessThanMaxCompactSize))(split_times=[208, 416]). 6 Input Files, 251mb total:" + - "**** Simulation run 17, type=split(CompactAndSplitOutput(FoundSubsetLessThanMaxCompactSize))(split_times=[208, 416]). 6 Input Files, 251mb total:" - "L0 " - - "L0.86[42,260] 1.05us 23mb |---------------L0.86---------------| " - - "L0.87[261,356] 1.05us 10mb |----L0.87-----| " - - "L0.79[357,357] 1.05us 0b |L0.79| " - - "L0.84[358,520] 1.05us 17mb |----------L0.84-----------| " + - "L0.81[42,260] 1.05us 23mb |---------------L0.81---------------| " + - "L0.82[261,356] 1.05us 10mb |----L0.82-----| " + - "L0.74[357,357] 1.05us 0b |L0.74| " + - "L0.79[358,520] 1.05us 17mb |----------L0.79-----------| " - "L1 " - - "L1.81[0,260] 1.04us 100mb|-------------------L1.81-------------------| " - - "L1.82[261,520] 1.04us 100mb |------------------L1.82-------------------| " + - "L1.76[0,260] 1.04us 100mb|-------------------L1.76-------------------| " + - "L1.77[261,520] 1.04us 100mb |------------------L1.77-------------------| " - "**** 3 Output Files (parquet_file_id not yet assigned), 251mb total:" - "L1 " - "L1.?[0,208] 1.05us 100mb |---------------L1.?---------------| " - "L1.?[209,416] 1.05us 100mb |--------------L1.?---------------| " - "L1.?[417,520] 1.05us 51mb |-----L1.?------| " - "Committing partition 1:" - - " Soft Deleting 6 files: L0.79, L1.81, L1.82, L0.84, L0.86, L0.87" + - " Soft Deleting 6 files: L0.74, L1.76, L1.77, L0.79, L0.81, L0.82" - " Creating 3 files" - - "**** Simulation run 20, type=split(CompactAndSplitOutput(TotalSizeLessThanMaxCompactSize))(split_times=[701, 881]). 8 Input Files, 259mb total:" + - "**** Simulation run 18, type=split(CompactAndSplitOutput(TotalSizeLessThanMaxCompactSize))(split_times=[701, 881]). 7 Input Files, 259mb total:" - "L0 " - - "L0.91[918,986] 1.05us 7mb |---L0.91---| " - - "L0.90[798,917] 1.05us 13mb |--------L0.90--------| " - - "L0.78[715,797] 1.05us 9mb |----L0.78----| " - - "L0.77[671,714] 1.05us 5mb |L0.77-| " - - "L0.85[521,670] 1.05us 16mb|----------L0.85-----------| " + - "L0.86[918,986] 1.05us 7mb |---L0.86---| " + - "L0.85[715,917] 1.05us 22mb |----------------L0.85----------------| " + - "L0.72[671,714] 1.05us 5mb |L0.72-| " + - "L0.80[521,670] 1.05us 16mb|----------L0.80-----------| " - "L1 " - - "L1.89[918,986] 1.04us 34mb |---L1.89---| " - - "L1.88[715,917] 1.04us 100mb |----------------L1.88----------------| " - - "L1.83[521,714] 1.04us 75mb|---------------L1.83---------------| " + - "L1.84[918,986] 1.04us 34mb |---L1.84---| " + - "L1.83[715,917] 1.04us 100mb |----------------L1.83----------------| " + - "L1.78[521,714] 1.04us 75mb|---------------L1.78---------------| " - "**** 3 Output Files (parquet_file_id not yet assigned), 259mb total:" - "L1 " - "L1.?[521,701] 1.05us 100mb|--------------L1.?--------------| " - "L1.?[702,881] 1.05us 100mb |--------------L1.?--------------| " - "L1.?[882,986] 1.05us 59mb |-------L1.?-------| " - "Committing partition 1:" - - " Soft Deleting 8 files: L0.77, L0.78, L1.83, L0.85, L1.88, L1.89, L0.90, L0.91" + - " Soft Deleting 7 files: L0.72, L1.78, L0.80, L1.83, L1.84, L0.85, L0.86" - " Creating 3 files" - - "**** Simulation run 21, type=split(CompactAndSplitOutput(FoundSubsetLessThanMaxCompactSize))(split_times=[406, 603]). 3 Input Files, 251mb total:" + - "**** Simulation run 19, type=split(CompactAndSplitOutput(FoundSubsetLessThanMaxCompactSize))(split_times=[406, 603]). 3 Input Files, 251mb total:" - "L1 " - - "L1.93[209,416] 1.05us 100mb|---------------L1.93---------------| " - - "L1.94[417,520] 1.05us 51mb |-----L1.94------| " - - "L1.95[521,701] 1.05us 100mb |------------L1.95-------------| " + - "L1.88[209,416] 1.05us 100mb|---------------L1.88---------------| " + - "L1.89[417,520] 1.05us 51mb |-----L1.89------| " + - "L1.90[521,701] 1.05us 100mb |------------L1.90-------------| " - "**** 3 Output Files (parquet_file_id not yet assigned), 251mb total:" - "L2 " - "L2.?[209,406] 1.05us 100mb|---------------L2.?---------------| " - "L2.?[407,603] 1.05us 100mb |--------------L2.?---------------| " - "L2.?[604,701] 1.05us 50mb |-----L2.?------| " - "Committing partition 1:" - - " Soft Deleting 3 files: L1.93, L1.94, L1.95" - - " Upgrading 1 files level to CompactionLevel::L2: L1.92" + - " Soft Deleting 3 files: L1.88, L1.89, L1.90" + - " Upgrading 1 files level to CompactionLevel::L2: L1.87" - " Creating 3 files" - - "**** Simulation run 22, type=split(CompactAndSplitOutput(TotalSizeLessThanMaxCompactSize))(split_times=[881]). 2 Input Files, 159mb total:" + - "**** Simulation run 20, type=split(CompactAndSplitOutput(TotalSizeLessThanMaxCompactSize))(split_times=[881]). 2 Input Files, 159mb total:" - "L1 " - - "L1.97[882,986] 1.05us 59mb |------------L1.97-------------| " - - "L1.96[702,881] 1.05us 100mb|------------------------L1.96-------------------------| " + - "L1.92[882,986] 1.05us 59mb |------------L1.92-------------| " + - "L1.91[702,881] 1.05us 100mb|------------------------L1.91-------------------------| " - "**** 2 Output Files (parquet_file_id not yet assigned), 159mb total:" - "L2 " - "L2.?[702,881] 1.05us 100mb|-------------------------L2.?-------------------------| " - "L2.?[882,986] 1.05us 59mb |-------------L2.?-------------| " - "Committing partition 1:" - - " Soft Deleting 2 files: L1.96, L1.97" + - " Soft Deleting 2 files: L1.91, L1.92" - " Creating 2 files" - - "**** Final Output Files (2.76gb written)" + - "**** Final Output Files (2.84gb written)" - "L2 " - - "L2.92[0,208] 1.05us 100mb|-----L2.92------| " - - "L2.98[209,406] 1.05us 100mb |-----L2.98-----| " - - "L2.99[407,603] 1.05us 100mb |-----L2.99-----| " - - "L2.100[604,701] 1.05us 50mb |L2.100| " - - "L2.101[702,881] 1.05us 100mb |----L2.101----| " - - "L2.102[882,986] 1.05us 59mb |L2.102-| " + - "L2.87[0,208] 1.05us 100mb|-----L2.87------| " + - "L2.93[209,406] 1.05us 100mb |-----L2.93-----| " + - "L2.94[407,603] 1.05us 100mb |-----L2.94-----| " + - "L2.95[604,701] 1.05us 50mb |L2.95-| " + - "L2.96[702,881] 1.05us 100mb |----L2.96-----| " + - "L2.97[882,986] 1.05us 59mb |-L2.97-| " "### ); } @@ -581,7 +563,7 @@ async fn random_backfill_over_l2s() { - "L2.8[700,799] 799ns 100mb |-L2.8-| " - "L2.9[800,899] 899ns 100mb |-L2.9-| " - "L2.10[900,999] 999ns 100mb |L2.10-| " - - "**** Simulation run 0, type=split(CompactAndSplitOutput(ManySmallFiles))(split_times=[797]). 10 Input Files, 100mb total:" + - "**** Simulation run 0, type=compact(ManySmallFiles). 10 Input Files, 100mb total:" - "L0, all files 10mb " - "L0.11[76,932] 1us |-------------------------------------L0.11-------------------------------------| " - "L0.12[42,986] 1us |-----------------------------------------L0.12------------------------------------------|" @@ -593,14 +575,13 @@ async fn random_backfill_over_l2s() { - "L0.18[50,629] 1.01us |------------------------L0.18------------------------| " - "L0.19[76,932] 1.01us |-------------------------------------L0.19-------------------------------------| " - "L0.20[42,986] 1.01us |-----------------------------------------L0.20------------------------------------------|" - - "**** 2 Output Files (parquet_file_id not yet assigned), 100mb total:" - - "L0 " - - "L0.?[42,797] 1.01us 80mb |--------------------------------L0.?---------------------------------| " - - "L0.?[798,986] 1.01us 20mb |-----L0.?------| " + - "**** 1 Output Files (parquet_file_id not yet assigned), 100mb total:" + - "L0, all files 100mb " + - "L0.?[42,986] 1.01us |------------------------------------------L0.?------------------------------------------|" - "Committing partition 1:" - " Soft Deleting 10 files: L0.11, L0.12, L0.13, L0.14, L0.15, L0.16, L0.17, L0.18, L0.19, L0.20" - - " Creating 2 files" - - "**** Simulation run 1, type=split(CompactAndSplitOutput(ManySmallFiles))(split_times=[797]). 10 Input Files, 100mb total:" + - " Creating 1 files" + - "**** Simulation run 1, type=compact(ManySmallFiles). 10 Input Files, 100mb total:" - "L0, all files 10mb " - "L0.21[173,950] 1.01us |---------------------------------L0.21----------------------------------| " - "L0.22[50,629] 1.01us |------------------------L0.22------------------------| " @@ -612,14 +593,13 @@ async fn random_backfill_over_l2s() { - "L0.28[42,986] 1.02us |-----------------------------------------L0.28------------------------------------------|" - "L0.29[173,950] 1.02us |---------------------------------L0.29----------------------------------| " - "L0.30[50,629] 1.02us |------------------------L0.30------------------------| " - - "**** 2 Output Files (parquet_file_id not yet assigned), 100mb total:" - - "L0 " - - "L0.?[42,797] 1.02us 80mb |--------------------------------L0.?---------------------------------| " - - "L0.?[798,986] 1.02us 20mb |-----L0.?------| " + - "**** 1 Output Files (parquet_file_id not yet assigned), 100mb total:" + - "L0, all files 100mb " + - "L0.?[42,986] 1.02us |------------------------------------------L0.?------------------------------------------|" - "Committing partition 1:" - " Soft Deleting 10 files: L0.21, L0.22, L0.23, L0.24, L0.25, L0.26, L0.27, L0.28, L0.29, L0.30" - - " Creating 2 files" - - "**** Simulation run 2, type=split(CompactAndSplitOutput(ManySmallFiles))(split_times=[797]). 10 Input Files, 100mb total:" + - " Creating 1 files" + - "**** Simulation run 2, type=compact(ManySmallFiles). 10 Input Files, 100mb total:" - "L0, all files 10mb " - "L0.31[76,932] 1.02us |-------------------------------------L0.31-------------------------------------| " - "L0.32[42,986] 1.02us |-----------------------------------------L0.32------------------------------------------|" @@ -631,14 +611,13 @@ async fn random_backfill_over_l2s() { - "L0.38[50,629] 1.03us |------------------------L0.38------------------------| " - "L0.39[76,932] 1.03us |-------------------------------------L0.39-------------------------------------| " - "L0.40[42,986] 1.03us |-----------------------------------------L0.40------------------------------------------|" - - "**** 2 Output Files (parquet_file_id not yet assigned), 100mb total:" - - "L0 " - - "L0.?[42,797] 1.03us 80mb |--------------------------------L0.?---------------------------------| " - - "L0.?[798,986] 1.03us 20mb |-----L0.?------| " + - "**** 1 Output Files (parquet_file_id not yet assigned), 100mb total:" + - "L0, all files 100mb " + - "L0.?[42,986] 1.03us |------------------------------------------L0.?------------------------------------------|" - "Committing partition 1:" - " Soft Deleting 10 files: L0.31, L0.32, L0.33, L0.34, L0.35, L0.36, L0.37, L0.38, L0.39, L0.40" - - " Creating 2 files" - - "**** Simulation run 3, type=split(CompactAndSplitOutput(ManySmallFiles))(split_times=[797]). 10 Input Files, 100mb total:" + - " Creating 1 files" + - "**** Simulation run 3, type=compact(ManySmallFiles). 10 Input Files, 100mb total:" - "L0, all files 10mb " - "L0.41[173,950] 1.03us |---------------------------------L0.41----------------------------------| " - "L0.42[50,629] 1.03us |------------------------L0.42------------------------| " @@ -650,14 +629,13 @@ async fn random_backfill_over_l2s() { - "L0.48[42,986] 1.04us |-----------------------------------------L0.48------------------------------------------|" - "L0.49[173,950] 1.04us |---------------------------------L0.49----------------------------------| " - "L0.50[50,629] 1.04us |------------------------L0.50------------------------| " - - "**** 2 Output Files (parquet_file_id not yet assigned), 100mb total:" - - "L0 " - - "L0.?[42,797] 1.04us 80mb |--------------------------------L0.?---------------------------------| " - - "L0.?[798,986] 1.04us 20mb |-----L0.?------| " + - "**** 1 Output Files (parquet_file_id not yet assigned), 100mb total:" + - "L0, all files 100mb " + - "L0.?[42,986] 1.04us |------------------------------------------L0.?------------------------------------------|" - "Committing partition 1:" - " Soft Deleting 10 files: L0.41, L0.42, L0.43, L0.44, L0.45, L0.46, L0.47, L0.48, L0.49, L0.50" - - " Creating 2 files" - - "**** Simulation run 4, type=split(CompactAndSplitOutput(ManySmallFiles))(split_times=[797]). 10 Input Files, 100mb total:" + - " Creating 1 files" + - "**** Simulation run 4, type=compact(ManySmallFiles). 10 Input Files, 100mb total:" - "L0, all files 10mb " - "L0.51[76,932] 1.04us |-------------------------------------L0.51-------------------------------------| " - "L0.52[42,986] 1.04us |-----------------------------------------L0.52------------------------------------------|" @@ -669,307 +647,363 @@ async fn random_backfill_over_l2s() { - "L0.58[50,629] 1.05us |------------------------L0.58------------------------| " - "L0.59[76,932] 1.05us |-------------------------------------L0.59-------------------------------------| " - "L0.60[42,986] 1.05us |-----------------------------------------L0.60------------------------------------------|" - - "**** 2 Output Files (parquet_file_id not yet assigned), 100mb total:" - - "L0 " - - "L0.?[42,797] 1.05us 80mb |--------------------------------L0.?---------------------------------| " - - "L0.?[798,986] 1.05us 20mb |-----L0.?------| " + - "**** 1 Output Files (parquet_file_id not yet assigned), 100mb total:" + - "L0, all files 100mb " + - "L0.?[42,986] 1.05us |------------------------------------------L0.?------------------------------------------|" - "Committing partition 1:" - " Soft Deleting 10 files: L0.51, L0.52, L0.53, L0.54, L0.55, L0.56, L0.57, L0.58, L0.59, L0.60" - - " Creating 2 files" - - "**** Simulation run 5, type=split(HighL0OverlapSingleFile)(split_times=[293, 544]). 1 Input Files, 80mb total:" - - "L0, all files 80mb " - - "L0.61[42,797] 1.01us |-----------------------------------------L0.61------------------------------------------|" - - "**** 3 Output Files (parquet_file_id not yet assigned), 80mb total:" + - " Creating 1 files" + - "**** Simulation run 5, type=split(HighL0OverlapSingleFile)(split_times=[356, 670]). 1 Input Files, 100mb total:" + - "L0, all files 100mb " + - "L0.61[42,986] 1.01us |-----------------------------------------L0.61------------------------------------------|" + - "**** 3 Output Files (parquet_file_id not yet assigned), 100mb total:" - "L0 " - - "L0.?[42,293] 1.01us 27mb |-----------L0.?------------| " - - "L0.?[294,544] 1.01us 26mb |-----------L0.?------------| " - - "L0.?[545,797] 1.01us 27mb |------------L0.?------------| " - - "**** Simulation run 6, type=split(HighL0OverlapSingleFile)(split_times=[293, 544]). 1 Input Files, 80mb total:" - - "L0, all files 80mb " - - "L0.63[42,797] 1.02us |-----------------------------------------L0.63------------------------------------------|" - - "**** 3 Output Files (parquet_file_id not yet assigned), 80mb total:" + - "L0.?[42,356] 1.01us 33mb |-----------L0.?------------| " + - "L0.?[357,670] 1.01us 33mb |-----------L0.?------------| " + - "L0.?[671,986] 1.01us 34mb |------------L0.?------------| " + - "**** Simulation run 6, type=split(HighL0OverlapSingleFile)(split_times=[356, 670]). 1 Input Files, 100mb total:" + - "L0, all files 100mb " + - "L0.62[42,986] 1.02us |-----------------------------------------L0.62------------------------------------------|" + - "**** 3 Output Files (parquet_file_id not yet assigned), 100mb total:" - "L0 " - - "L0.?[42,293] 1.02us 27mb |-----------L0.?------------| " - - "L0.?[294,544] 1.02us 26mb |-----------L0.?------------| " - - "L0.?[545,797] 1.02us 27mb |------------L0.?------------| " - - "**** Simulation run 7, type=split(HighL0OverlapSingleFile)(split_times=[293, 544]). 1 Input Files, 80mb total:" - - "L0, all files 80mb " - - "L0.65[42,797] 1.03us |-----------------------------------------L0.65------------------------------------------|" - - "**** 3 Output Files (parquet_file_id not yet assigned), 80mb total:" + - "L0.?[42,356] 1.02us 33mb |-----------L0.?------------| " + - "L0.?[357,670] 1.02us 33mb |-----------L0.?------------| " + - "L0.?[671,986] 1.02us 34mb |------------L0.?------------| " + - "**** Simulation run 7, type=split(HighL0OverlapSingleFile)(split_times=[356, 670]). 1 Input Files, 100mb total:" + - "L0, all files 100mb " + - "L0.63[42,986] 1.03us |-----------------------------------------L0.63------------------------------------------|" + - "**** 3 Output Files (parquet_file_id not yet assigned), 100mb total:" - "L0 " - - "L0.?[42,293] 1.03us 27mb |-----------L0.?------------| " - - "L0.?[294,544] 1.03us 26mb |-----------L0.?------------| " - - "L0.?[545,797] 1.03us 27mb |------------L0.?------------| " - - "**** Simulation run 8, type=split(HighL0OverlapSingleFile)(split_times=[293, 544]). 1 Input Files, 80mb total:" - - "L0, all files 80mb " - - "L0.67[42,797] 1.04us |-----------------------------------------L0.67------------------------------------------|" - - "**** 3 Output Files (parquet_file_id not yet assigned), 80mb total:" + - "L0.?[42,356] 1.03us 33mb |-----------L0.?------------| " + - "L0.?[357,670] 1.03us 33mb |-----------L0.?------------| " + - "L0.?[671,986] 1.03us 34mb |------------L0.?------------| " + - "**** Simulation run 8, type=split(HighL0OverlapSingleFile)(split_times=[356, 670]). 1 Input Files, 100mb total:" + - "L0, all files 100mb " + - "L0.64[42,986] 1.04us |-----------------------------------------L0.64------------------------------------------|" + - "**** 3 Output Files (parquet_file_id not yet assigned), 100mb total:" - "L0 " - - "L0.?[42,293] 1.04us 27mb |-----------L0.?------------| " - - "L0.?[294,544] 1.04us 26mb |-----------L0.?------------| " - - "L0.?[545,797] 1.04us 27mb |------------L0.?------------| " - - "**** Simulation run 9, type=split(HighL0OverlapSingleFile)(split_times=[293, 544]). 1 Input Files, 80mb total:" - - "L0, all files 80mb " - - "L0.69[42,797] 1.05us |-----------------------------------------L0.69------------------------------------------|" - - "**** 3 Output Files (parquet_file_id not yet assigned), 80mb total:" + - "L0.?[42,356] 1.04us 33mb |-----------L0.?------------| " + - "L0.?[357,670] 1.04us 33mb |-----------L0.?------------| " + - "L0.?[671,986] 1.04us 34mb |------------L0.?------------| " + - "**** Simulation run 9, type=split(HighL0OverlapSingleFile)(split_times=[356, 670]). 1 Input Files, 100mb total:" + - "L0, all files 100mb " + - "L0.65[42,986] 1.05us |-----------------------------------------L0.65------------------------------------------|" + - "**** 3 Output Files (parquet_file_id not yet assigned), 100mb total:" - "L0 " - - "L0.?[42,293] 1.05us 27mb |-----------L0.?------------| " - - "L0.?[294,544] 1.05us 26mb |-----------L0.?------------| " - - "L0.?[545,797] 1.05us 27mb |------------L0.?------------| " + - "L0.?[42,356] 1.05us 33mb |-----------L0.?------------| " + - "L0.?[357,670] 1.05us 33mb |-----------L0.?------------| " + - "L0.?[671,986] 1.05us 34mb |------------L0.?------------| " - "Committing partition 1:" - - " Soft Deleting 5 files: L0.61, L0.63, L0.65, L0.67, L0.69" + - " Soft Deleting 5 files: L0.61, L0.62, L0.63, L0.64, L0.65" - " Creating 15 files" - - "**** Simulation run 10, type=split(CompactAndSplitOutput(ManySmallFiles))(split_times=[232, 422]). 10 Input Files, 265mb total:" + - "**** Simulation run 10, type=split(CompactAndSplitOutput(FoundSubsetLessThanMaxCompactSize))(split_times=[357, 672]). 9 Input Files, 300mb total:" - "L0 " - - "L0.71[42,293] 1.01us 27mb|-------------------L0.71-------------------| " - - "L0.72[294,544] 1.01us 26mb |------------------L0.72-------------------| " - - "L0.74[42,293] 1.02us 27mb|-------------------L0.74-------------------| " - - "L0.75[294,544] 1.02us 26mb |------------------L0.75-------------------| " - - "L0.77[42,293] 1.03us 27mb|-------------------L0.77-------------------| " - - "L0.78[294,544] 1.03us 26mb |------------------L0.78-------------------| " - - "L0.80[42,293] 1.04us 27mb|-------------------L0.80-------------------| " - - "L0.81[294,544] 1.04us 26mb |------------------L0.81-------------------| " - - "L0.83[42,293] 1.05us 27mb|-------------------L0.83-------------------| " - - "L0.84[294,544] 1.05us 26mb |------------------L0.84-------------------| " - - "**** 3 Output Files (parquet_file_id not yet assigned), 265mb total:" - - "L0 " - - "L0.?[42,232] 1.05us 100mb|--------------L0.?--------------| " - - "L0.?[233,422] 1.05us 100mb |-------------L0.?--------------| " - - "L0.?[423,544] 1.05us 65mb |-------L0.?--------| " - - "Committing partition 1:" - - " Soft Deleting 10 files: L0.71, L0.72, L0.74, L0.75, L0.77, L0.78, L0.80, L0.81, L0.83, L0.84" - - " Creating 3 files" - - "**** Simulation run 11, type=split(CompactAndSplitOutput(ManySmallFiles))(split_times=[733, 921]). 10 Input Files, 235mb total:" - - "L0 " - - "L0.73[545,797] 1.01us 27mb|----------------------L0.73----------------------| " - - "L0.62[798,986] 1.01us 20mb |---------------L0.62----------------| " - - "L0.76[545,797] 1.02us 27mb|----------------------L0.76----------------------| " - - "L0.64[798,986] 1.02us 20mb |---------------L0.64----------------| " - - "L0.79[545,797] 1.03us 27mb|----------------------L0.79----------------------| " - - "L0.66[798,986] 1.03us 20mb |---------------L0.66----------------| " - - "L0.82[545,797] 1.04us 27mb|----------------------L0.82----------------------| " - - "L0.68[798,986] 1.04us 20mb |---------------L0.68----------------| " - - "L0.85[545,797] 1.05us 27mb|----------------------L0.85----------------------| " - - "L0.70[798,986] 1.05us 20mb |---------------L0.70----------------| " - - "**** 3 Output Files (parquet_file_id not yet assigned), 235mb total:" - - "L0 " - - "L0.?[545,733] 1.05us 100mb|----------------L0.?----------------| " - - "L0.?[734,921] 1.05us 99mb |----------------L0.?----------------| " - - "L0.?[922,986] 1.05us 35mb |---L0.?----| " - - "Committing partition 1:" - - " Soft Deleting 10 files: L0.62, L0.64, L0.66, L0.68, L0.70, L0.73, L0.76, L0.79, L0.82, L0.85" - - " Creating 3 files" - - "**** Simulation run 12, type=split(CompactAndSplitOutput(FoundSubsetLessThanMaxCompactSize))(split_times=[422, 611]). 3 Input Files, 265mb total:" - - "L0 " - - "L0.87[233,422] 1.05us 100mb|-------------L0.87--------------| " - - "L0.88[423,544] 1.05us 65mb |-------L0.88-------| " - - "L0.89[545,733] 1.05us 100mb |-------------L0.89-------------| " - - "**** 3 Output Files (parquet_file_id not yet assigned), 265mb total:" + - "L0.66[42,356] 1.01us 33mb|-----------L0.66-----------| " + - "L0.67[357,670] 1.01us 33mb |-----------L0.67-----------| " + - "L0.68[671,986] 1.01us 34mb |-----------L0.68------------| " + - "L0.69[42,356] 1.02us 33mb|-----------L0.69-----------| " + - "L0.70[357,670] 1.02us 33mb |-----------L0.70-----------| " + - "L0.71[671,986] 1.02us 34mb |-----------L0.71------------| " + - "L0.72[42,356] 1.03us 33mb|-----------L0.72-----------| " + - "L0.73[357,670] 1.03us 33mb |-----------L0.73-----------| " + - "L0.74[671,986] 1.03us 34mb |-----------L0.74------------| " + - "**** 3 Output Files (parquet_file_id not yet assigned), 300mb total:" - "L1 " - - "L1.?[233,422] 1.05us 100mb|--------------L1.?--------------| " - - "L1.?[423,611] 1.05us 100mb |-------------L1.?--------------| " - - "L1.?[612,733] 1.05us 65mb |-------L1.?--------| " + - "L1.?[42,357] 1.03us 100mb|------------L1.?------------| " + - "L1.?[358,672] 1.03us 100mb |-----------L1.?------------| " + - "L1.?[673,986] 1.03us 100mb |-----------L1.?------------| " - "Committing partition 1:" - - " Soft Deleting 3 files: L0.87, L0.88, L0.89" - - " Upgrading 1 files level to CompactionLevel::L1: L0.86" + - " Soft Deleting 9 files: L0.66, L0.67, L0.68, L0.69, L0.70, L0.71, L0.72, L0.73, L0.74" - " Creating 3 files" - - "**** Simulation run 13, type=split(CompactAndSplitOutput(TotalSizeLessThanMaxCompactSize))(split_times=[922]). 2 Input Files, 135mb total:" + - "**** Simulation run 11, type=split(ReduceOverlap)(split_times=[672]). 1 Input Files, 34mb total:" + - "L0, all files 34mb " + - "L0.77[671,986] 1.04us |-----------------------------------------L0.77------------------------------------------|" + - "**** 2 Output Files (parquet_file_id not yet assigned), 34mb total:" - "L0 " - - "L0.91[922,986] 1.05us 35mb |-------L0.91--------| " - - "L0.90[734,921] 1.05us 99mb|-----------------------------L0.90------------------------------| " - - "**** 2 Output Files (parquet_file_id not yet assigned), 135mb total:" - - "L1 " - - "L1.?[734,922] 1.05us 100mb|------------------------------L1.?-------------------------------| " - - "L1.?[923,986] 1.05us 34mb |--------L1.?--------| " + - "L0.?[671,672] 1.04us 109kb|L0.?| " + - "L0.?[673,986] 1.04us 33mb|-----------------------------------------L0.?------------------------------------------| " + - "**** Simulation run 12, type=split(ReduceOverlap)(split_times=[357]). 1 Input Files, 33mb total:" + - "L0, all files 33mb " + - "L0.76[357,670] 1.04us |-----------------------------------------L0.76------------------------------------------|" + - "**** 2 Output Files (parquet_file_id not yet assigned), 33mb total:" + - "L0 " + - "L0.?[357,357] 1.04us 0b |L0.?| " + - "L0.?[358,670] 1.04us 33mb|-----------------------------------------L0.?------------------------------------------| " + - "**** Simulation run 13, type=split(ReduceOverlap)(split_times=[672]). 1 Input Files, 34mb total:" + - "L0, all files 34mb " + - "L0.80[671,986] 1.05us |-----------------------------------------L0.80------------------------------------------|" + - "**** 2 Output Files (parquet_file_id not yet assigned), 34mb total:" + - "L0 " + - "L0.?[671,672] 1.05us 109kb|L0.?| " + - "L0.?[673,986] 1.05us 33mb|-----------------------------------------L0.?------------------------------------------| " + - "**** Simulation run 14, type=split(ReduceOverlap)(split_times=[357]). 1 Input Files, 33mb total:" + - "L0, all files 33mb " + - "L0.79[357,670] 1.05us |-----------------------------------------L0.79------------------------------------------|" + - "**** 2 Output Files (parquet_file_id not yet assigned), 33mb total:" + - "L0 " + - "L0.?[357,357] 1.05us 0b |L0.?| " + - "L0.?[358,670] 1.05us 33mb|-----------------------------------------L0.?------------------------------------------| " - "Committing partition 1:" - - " Soft Deleting 2 files: L0.90, L0.91" + - " Soft Deleting 4 files: L0.76, L0.77, L0.79, L0.80" + - " Creating 8 files" + - "**** Simulation run 15, type=compact(ManySmallFiles). 10 Input Files, 200mb total:" + - "L0 " + - "L0.75[42,356] 1.04us 33mb|-----------L0.75-----------| " + - "L0.86[357,357] 1.04us 0b |L0.86| " + - "L0.87[358,670] 1.04us 33mb |-----------L0.87-----------| " + - "L0.84[671,672] 1.04us 109kb |L0.84| " + - "L0.85[673,986] 1.04us 33mb |-----------L0.85-----------| " + - "L0.78[42,356] 1.05us 33mb|-----------L0.78-----------| " + - "L0.90[357,357] 1.05us 0b |L0.90| " + - "L0.91[358,670] 1.05us 33mb |-----------L0.91-----------| " + - "L0.88[671,672] 1.05us 109kb |L0.88| " + - "L0.89[673,986] 1.05us 33mb |-----------L0.89-----------| " + - "**** 1 Output Files (parquet_file_id not yet assigned), 200mb total:" + - "L0, all files 200mb " + - "L0.?[42,986] 1.05us |------------------------------------------L0.?------------------------------------------|" + - "Committing partition 1:" + - " Soft Deleting 10 files: L0.75, L0.78, L0.84, L0.85, L0.86, L0.87, L0.88, L0.89, L0.90, L0.91" + - " Creating 1 files" + - "**** Simulation run 16, type=split(HighL0OverlapSingleFile)(split_times=[670]). 1 Input Files, 100mb total:" + - "L1, all files 100mb " + - "L1.82[358,672] 1.03us |-----------------------------------------L1.82------------------------------------------|" + - "**** 2 Output Files (parquet_file_id not yet assigned), 100mb total:" + - "L1 " + - "L1.?[358,670] 1.03us 99mb|-----------------------------------------L1.?------------------------------------------| " + - "L1.?[671,672] 1.03us 651kb |L1.?|" + - "**** Simulation run 17, type=split(HighL0OverlapSingleFile)(split_times=[356]). 1 Input Files, 100mb total:" + - "L1, all files 100mb " + - "L1.81[42,357] 1.03us |-----------------------------------------L1.81------------------------------------------|" + - "**** 2 Output Files (parquet_file_id not yet assigned), 100mb total:" + - "L1 " + - "L1.?[42,356] 1.03us 100mb|-----------------------------------------L1.?------------------------------------------| " + - "L1.?[357,357] 1.03us 325kb |L1.?|" + - "**** Simulation run 18, type=split(HighL0OverlapSingleFile)(split_times=[356, 670]). 1 Input Files, 200mb total:" + - "L0, all files 200mb " + - "L0.92[42,986] 1.05us |-----------------------------------------L0.92------------------------------------------|" + - "**** 3 Output Files (parquet_file_id not yet assigned), 200mb total:" + - "L0 " + - "L0.?[42,356] 1.05us 67mb |-----------L0.?------------| " + - "L0.?[357,670] 1.05us 66mb |-----------L0.?------------| " + - "L0.?[671,986] 1.05us 67mb |------------L0.?------------| " + - "Committing partition 1:" + - " Soft Deleting 3 files: L1.81, L1.82, L0.92" + - " Creating 7 files" + - "**** Simulation run 19, type=split(ReduceOverlap)(split_times=[672]). 1 Input Files, 67mb total:" + - "L0, all files 67mb " + - "L0.99[671,986] 1.05us |-----------------------------------------L0.99------------------------------------------|" + - "**** 2 Output Files (parquet_file_id not yet assigned), 67mb total:" + - "L0 " + - "L0.?[671,672] 1.05us 218kb|L0.?| " + - "L0.?[673,986] 1.05us 67mb|-----------------------------------------L0.?------------------------------------------| " + - "**** Simulation run 20, type=split(ReduceOverlap)(split_times=[357]). 1 Input Files, 66mb total:" + - "L0, all files 66mb " + - "L0.98[357,670] 1.05us |-----------------------------------------L0.98------------------------------------------|" + - "**** 2 Output Files (parquet_file_id not yet assigned), 66mb total:" + - "L0 " + - "L0.?[357,357] 1.05us 0b |L0.?| " + - "L0.?[358,670] 1.05us 66mb|-----------------------------------------L0.?------------------------------------------| " + - "Committing partition 1:" + - " Soft Deleting 2 files: L0.98, L0.99" + - " Creating 4 files" + - "**** Simulation run 21, type=split(CompactAndSplitOutput(FoundSubsetLessThanMaxCompactSize))(split_times=[232]). 4 Input Files, 167mb total:" + - "L0 " + - "L0.97[42,356] 1.05us 67mb|-----------------------------------------L0.97-----------------------------------------| " + - "L0.102[357,357] 1.05us 0b |L0.102|" + - "L1 " + - "L1.95[42,356] 1.03us 100mb|-----------------------------------------L1.95-----------------------------------------| " + - "L1.96[357,357] 1.03us 325kb |L1.96|" + - "**** 2 Output Files (parquet_file_id not yet assigned), 167mb total:" + - "L1 " + - "L1.?[42,232] 1.05us 101mb|------------------------L1.?------------------------| " + - "L1.?[233,357] 1.05us 66mb |--------------L1.?---------------| " + - "Committing partition 1:" + - " Soft Deleting 4 files: L1.95, L1.96, L0.97, L0.102" - " Creating 2 files" - - "**** Simulation run 14, type=split(ReduceOverlap)(split_times=[499, 599]). 1 Input Files, 100mb total:" + - "**** Simulation run 22, type=split(CompactAndSplitOutput(FoundSubsetLessThanMaxCompactSize))(split_times=[547]). 4 Input Files, 166mb total:" + - "L0 " + - "L0.103[358,670] 1.05us 66mb|----------------------------------------L0.103-----------------------------------------| " + - "L0.100[671,672] 1.05us 218kb |L0.100|" + - "L1 " + - "L1.93[358,670] 1.03us 99mb|-----------------------------------------L1.93-----------------------------------------| " + - "L1.94[671,672] 1.03us 651kb |L1.94|" + - "**** 2 Output Files (parquet_file_id not yet assigned), 166mb total:" + - "L1 " + - "L1.?[358,547] 1.05us 100mb|------------------------L1.?------------------------| " + - "L1.?[548,672] 1.05us 66mb |--------------L1.?---------------| " + - "Committing partition 1:" + - " Soft Deleting 4 files: L1.93, L1.94, L0.100, L0.103" + - " Creating 2 files" + - "**** Simulation run 23, type=split(CompactAndSplitOutput(TotalSizeLessThanMaxCompactSize))(split_times=[861]). 2 Input Files, 167mb total:" + - "L0 " + - "L0.101[673,986] 1.05us 67mb|-----------------------------------------L0.101-----------------------------------------|" + - "L1 " + - "L1.83[673,986] 1.03us 100mb|-----------------------------------------L1.83------------------------------------------|" + - "**** 2 Output Files (parquet_file_id not yet assigned), 167mb total:" + - "L1 " + - "L1.?[673,861] 1.05us 100mb|------------------------L1.?------------------------| " + - "L1.?[862,986] 1.05us 67mb |--------------L1.?---------------| " + - "Committing partition 1:" + - " Soft Deleting 2 files: L1.83, L0.101" + - " Creating 2 files" + - "**** Simulation run 24, type=split(ReduceOverlap)(split_times=[399, 499]). 1 Input Files, 100mb total:" - "L1, all files 100mb " - - "L1.93[423,611] 1.05us |-----------------------------------------L1.93------------------------------------------|" + - "L1.106[358,547] 1.05us |-----------------------------------------L1.106-----------------------------------------|" - "**** 3 Output Files (parquet_file_id not yet assigned), 100mb total:" - "L1 " - - "L1.?[423,499] 1.05us 40mb|---------------L1.?---------------| " - - "L1.?[500,599] 1.05us 52mb |--------------------L1.?---------------------| " - - "L1.?[600,611] 1.05us 7mb |L1.?|" - - "**** Simulation run 15, type=split(ReduceOverlap)(split_times=[299, 399]). 1 Input Files, 100mb total:" - - "L1, all files 100mb " - - "L1.92[233,422] 1.05us |-----------------------------------------L1.92------------------------------------------|" - - "**** 3 Output Files (parquet_file_id not yet assigned), 100mb total:" + - "L1.?[358,399] 1.05us 22mb|------L1.?-------| " + - "L1.?[400,499] 1.05us 52mb |--------------------L1.?---------------------| " + - "L1.?[500,547] 1.05us 26mb |--------L1.?--------| " + - "**** Simulation run 25, type=split(ReduceOverlap)(split_times=[299]). 1 Input Files, 66mb total:" + - "L1, all files 66mb " + - "L1.105[233,357] 1.05us |-----------------------------------------L1.105-----------------------------------------|" + - "**** 2 Output Files (parquet_file_id not yet assigned), 66mb total:" - "L1 " - - "L1.?[233,299] 1.05us 35mb|------------L1.?-------------| " - - "L1.?[300,399] 1.05us 52mb |--------------------L1.?---------------------| " - - "L1.?[400,422] 1.05us 13mb |--L1.?--| " - - "**** Simulation run 16, type=split(ReduceOverlap)(split_times=[99, 199]). 1 Input Files, 100mb total:" - - "L1, all files 100mb " - - "L1.86[42,232] 1.05us |-----------------------------------------L1.86------------------------------------------|" - - "**** 3 Output Files (parquet_file_id not yet assigned), 100mb total:" + - "L1.?[233,299] 1.05us 35mb|--------------------L1.?---------------------| " + - "L1.?[300,357] 1.05us 31mb |-----------------L1.?------------------| " + - "**** Simulation run 26, type=split(ReduceOverlap)(split_times=[99, 199]). 1 Input Files, 101mb total:" + - "L1, all files 101mb " + - "L1.104[42,232] 1.05us |-----------------------------------------L1.104-----------------------------------------|" + - "**** 3 Output Files (parquet_file_id not yet assigned), 101mb total:" - "L1 " - "L1.?[42,99] 1.05us 30mb |----------L1.?-----------| " - "L1.?[100,199] 1.05us 52mb |--------------------L1.?--------------------| " - "L1.?[200,232] 1.05us 18mb |----L1.?-----| " - - "**** Simulation run 17, type=split(ReduceOverlap)(split_times=[699]). 1 Input Files, 65mb total:" - - "L1, all files 65mb " - - "L1.94[612,733] 1.05us |-----------------------------------------L1.94-----------------------------------------| " - - "**** 2 Output Files (parquet_file_id not yet assigned), 65mb total:" + - "**** Simulation run 27, type=split(ReduceOverlap)(split_times=[599]). 1 Input Files, 66mb total:" + - "L1, all files 66mb " + - "L1.107[548,672] 1.05us |-----------------------------------------L1.107-----------------------------------------|" + - "**** 2 Output Files (parquet_file_id not yet assigned), 66mb total:" - "L1 " - - "L1.?[612,699] 1.05us 47mb|-----------------------------L1.?-----------------------------| " - - "L1.?[700,733] 1.05us 18mb |---------L1.?---------| " - - "**** Simulation run 18, type=split(ReduceOverlap)(split_times=[799, 899]). 1 Input Files, 100mb total:" + - "L1.?[548,599] 1.05us 27mb|---------------L1.?----------------| " + - "L1.?[600,672] 1.05us 39mb |-----------------------L1.?-----------------------| " + - "**** Simulation run 28, type=split(ReduceOverlap)(split_times=[899]). 1 Input Files, 67mb total:" + - "L1, all files 67mb " + - "L1.109[862,986] 1.05us |-----------------------------------------L1.109-----------------------------------------|" + - "**** 2 Output Files (parquet_file_id not yet assigned), 67mb total:" + - "L1 " + - "L1.?[862,899] 1.05us 20mb|----------L1.?----------| " + - "L1.?[900,986] 1.05us 47mb |----------------------------L1.?----------------------------| " + - "**** Simulation run 29, type=split(ReduceOverlap)(split_times=[699, 799]). 1 Input Files, 100mb total:" - "L1, all files 100mb " - - "L1.95[734,922] 1.05us |-----------------------------------------L1.95------------------------------------------|" + - "L1.108[673,861] 1.05us |-----------------------------------------L1.108-----------------------------------------|" - "**** 3 Output Files (parquet_file_id not yet assigned), 100mb total:" - "L1 " - - "L1.?[734,799] 1.05us 35mb|------------L1.?-------------| " - - "L1.?[800,899] 1.05us 53mb |--------------------L1.?---------------------| " - - "L1.?[900,922] 1.05us 13mb |--L1.?--| " + - "L1.?[673,699] 1.05us 14mb|---L1.?---| " + - "L1.?[700,799] 1.05us 53mb |--------------------L1.?---------------------| " + - "L1.?[800,861] 1.05us 34mb |-----------L1.?------------| " - "Committing partition 1:" - - " Soft Deleting 5 files: L1.86, L1.92, L1.93, L1.94, L1.95" - - " Creating 14 files" - - "**** Simulation run 19, type=split(CompactAndSplitOutput(FoundSubsetLessThanMaxCompactSize))(split_times=[71, 142]). 4 Input Files, 282mb total:" + - " Soft Deleting 6 files: L1.104, L1.105, L1.106, L1.107, L1.108, L1.109" + - " Creating 15 files" + - "**** Simulation run 30, type=split(CompactAndSplitOutput(FoundSubsetLessThanMaxCompactSize))(split_times=[71, 142]). 4 Input Files, 283mb total:" - "L1 " - - "L1.103[42,99] 1.05us 30mb |--------L1.103---------| " - - "L1.104[100,199] 1.05us 52mb |------------------L1.104------------------| " + - "L1.115[42,99] 1.05us 30mb |--------L1.115---------| " + - "L1.116[100,199] 1.05us 52mb |------------------L1.116------------------| " - "L2 " - "L2.1[0,99] 99ns 100mb |-------------------L2.1-------------------| " - "L2.2[100,199] 199ns 100mb |-------------------L2.2-------------------| " - - "**** 3 Output Files (parquet_file_id not yet assigned), 282mb total:" + - "**** 3 Output Files (parquet_file_id not yet assigned), 283mb total:" - "L2 " - "L2.?[0,71] 1.05us 101mb |-------------L2.?-------------| " - "L2.?[72,142] 1.05us 99mb |------------L2.?-------------| " - "L2.?[143,199] 1.05us 82mb |---------L2.?----------| " - "Committing partition 1:" - - " Soft Deleting 4 files: L2.1, L2.2, L1.103, L1.104" + - " Soft Deleting 4 files: L2.1, L2.2, L1.115, L1.116" - " Creating 3 files" - - "**** Simulation run 20, type=split(CompactAndSplitOutput(FoundSubsetLessThanMaxCompactSize))(split_times=[265]). 3 Input Files, 153mb total:" + - "**** Simulation run 31, type=split(CompactAndSplitOutput(FoundSubsetLessThanMaxCompactSize))(split_times=[271, 342]). 5 Input Files, 284mb total:" - "L1 " - - "L1.105[200,232] 1.05us 18mb|----------L1.105-----------| " - - "L1.100[233,299] 1.05us 35mb |-------------------------L1.100--------------------------| " + - "L1.117[200,232] 1.05us 18mb|---L1.117---| " + - "L1.113[233,299] 1.05us 35mb |----------L1.113-----------| " + - "L1.114[300,357] 1.05us 31mb |--------L1.114---------| " - "L2 " - - "L2.3[200,299] 299ns 100mb|-----------------------------------------L2.3------------------------------------------| " - - "**** 2 Output Files (parquet_file_id not yet assigned), 153mb total:" + - "L2.3[200,299] 299ns 100mb|-------------------L2.3-------------------| " + - "L2.4[300,399] 399ns 100mb |-------------------L2.4-------------------| " + - "**** 3 Output Files (parquet_file_id not yet assigned), 284mb total:" - "L2 " - - "L2.?[200,265] 1.05us 100mb|--------------------------L2.?---------------------------| " - - "L2.?[266,299] 1.05us 53mb |-----------L2.?------------| " + - "L2.?[200,271] 1.05us 101mb|-------------L2.?-------------| " + - "L2.?[272,342] 1.05us 100mb |------------L2.?-------------| " + - "L2.?[343,399] 1.05us 83mb |---------L2.?----------| " - "Committing partition 1:" - - " Soft Deleting 3 files: L2.3, L1.100, L1.105" - - " Creating 2 files" - - "**** Simulation run 21, type=split(CompactAndSplitOutput(FoundSubsetLessThanMaxCompactSize))(split_times=[376, 452]). 4 Input Files, 265mb total:" - - "L1 " - - "L1.101[300,399] 1.05us 52mb|------------------L1.101------------------| " - - "L1.102[400,422] 1.05us 13mb |L1.102-| " - - "L2 " - - "L2.4[300,399] 399ns 100mb|-------------------L2.4-------------------| " - - "L2.5[400,499] 499ns 100mb |-------------------L2.5-------------------| " - - "**** 3 Output Files (parquet_file_id not yet assigned), 265mb total:" - - "L2 " - - "L2.?[300,376] 1.05us 101mb|--------------L2.?--------------| " - - "L2.?[377,452] 1.05us 100mb |-------------L2.?--------------| " - - "L2.?[453,499] 1.05us 64mb |-------L2.?-------| " - - "Committing partition 1:" - - " Soft Deleting 4 files: L2.4, L2.5, L1.101, L1.102" + - " Soft Deleting 5 files: L2.3, L2.4, L1.113, L1.114, L1.117" - " Creating 3 files" - - "**** Simulation run 22, type=split(ReduceOverlap)(split_times=[452]). 1 Input Files, 40mb total:" - - "L1, all files 40mb " - - "L1.97[423,499] 1.05us |-----------------------------------------L1.97------------------------------------------|" - - "**** 2 Output Files (parquet_file_id not yet assigned), 40mb total:" + - "**** Simulation run 32, type=split(CompactAndSplitOutput(FoundSubsetLessThanMaxCompactSize))(split_times=[404, 465]). 4 Input Files, 257mb total:" - "L1 " - - "L1.?[423,452] 1.05us 15mb|--------------L1.?--------------| " - - "L1.?[453,499] 1.05us 25mb |------------------------L1.?------------------------| " - - "Committing partition 1:" - - " Soft Deleting 1 files: L1.97" - - " Creating 2 files" - - "**** Simulation run 23, type=split(CompactAndSplitOutput(FoundSubsetLessThanMaxCompactSize))(split_times=[437, 497]). 4 Input Files, 204mb total:" - - "L1 " - - "L1.119[423,452] 1.05us 15mb |------L1.119-------| " - - "L1.120[453,499] 1.05us 25mb |------------L1.120-------------| " + - "L1.110[358,399] 1.05us 22mb |-------L1.110--------| " + - "L1.111[400,499] 1.05us 52mb |------------------------L1.111-------------------------| " - "L2 " - - "L2.117[377,452] 1.05us 100mb|-----------------------L2.117------------------------| " - - "L2.118[453,499] 1.05us 64mb |------------L2.118-------------| " - - "**** 3 Output Files (parquet_file_id not yet assigned), 204mb total:" + - "L2.130[343,399] 1.05us 83mb|------------L2.130------------| " + - "L2.5[400,499] 499ns 100mb |-------------------------L2.5--------------------------| " + - "**** 3 Output Files (parquet_file_id not yet assigned), 257mb total:" - "L2 " - - "L2.?[377,437] 1.05us 100mb|-------------------L2.?-------------------| " - - "L2.?[438,497] 1.05us 99mb |------------------L2.?-------------------| " - - "L2.?[498,499] 1.05us 5mb |L2.?|" + - "L2.?[343,404] 1.05us 100mb|--------------L2.?---------------| " + - "L2.?[405,465] 1.05us 99mb |--------------L2.?--------------| " + - "L2.?[466,499] 1.05us 58mb |------L2.?-------| " - "Committing partition 1:" - - " Soft Deleting 4 files: L2.117, L2.118, L1.119, L1.120" + - " Soft Deleting 4 files: L2.5, L1.110, L1.111, L2.130" - " Creating 3 files" - - "**** Simulation run 24, type=split(CompactAndSplitOutput(FoundSubsetLessThanMaxCompactSize))(split_times=[577, 654]). 4 Input Files, 259mb total:" + - "**** Simulation run 33, type=split(CompactAndSplitOutput(FoundSubsetLessThanMaxCompactSize))(split_times=[569, 638]). 5 Input Files, 292mb total:" - "L1 " - - "L1.98[500,599] 1.05us 52mb|------------------L1.98-------------------| " - - "L1.99[600,611] 1.05us 7mb |L1.99| " + - "L1.112[500,547] 1.05us 26mb|------L1.112-------| " + - "L1.118[548,599] 1.05us 27mb |-------L1.118--------| " + - "L1.119[600,672] 1.05us 39mb |------------L1.119------------| " - "L2 " - "L2.6[500,599] 599ns 100mb|-------------------L2.6-------------------| " - "L2.7[600,699] 699ns 100mb |-------------------L2.7-------------------| " - - "**** 3 Output Files (parquet_file_id not yet assigned), 259mb total:" + - "**** 3 Output Files (parquet_file_id not yet assigned), 292mb total:" - "L2 " - - "L2.?[500,577] 1.05us 100mb|--------------L2.?--------------| " - - "L2.?[578,654] 1.05us 99mb |--------------L2.?--------------| " - - "L2.?[655,699] 1.05us 60mb |------L2.?-------| " + - "L2.?[500,569] 1.05us 101mb|------------L2.?-------------| " + - "L2.?[570,638] 1.05us 100mb |------------L2.?------------| " + - "L2.?[639,699] 1.05us 91mb |----------L2.?-----------| " - "Committing partition 1:" - - " Soft Deleting 4 files: L2.6, L2.7, L1.98, L1.99" + - " Soft Deleting 5 files: L2.6, L2.7, L1.112, L1.118, L1.119" - " Creating 3 files" - - "**** Simulation run 25, type=split(ReduceOverlap)(split_times=[654]). 1 Input Files, 47mb total:" - - "L1, all files 47mb " - - "L1.106[612,699] 1.05us |-----------------------------------------L1.106-----------------------------------------|" - - "**** 2 Output Files (parquet_file_id not yet assigned), 47mb total:" + - "**** Simulation run 34, type=split(CompactAndSplitOutput(FoundSubsetLessThanMaxCompactSize))(split_times=[702, 765]). 4 Input Files, 258mb total:" - "L1 " - - "L1.?[612,654] 1.05us 23mb|------------------L1.?-------------------| " - - "L1.?[655,699] 1.05us 24mb |-------------------L1.?--------------------| " - - "Committing partition 1:" - - " Soft Deleting 1 files: L1.106" - - " Creating 2 files" - - "**** Simulation run 26, type=split(CompactAndSplitOutput(FoundSubsetLessThanMaxCompactSize))(split_times=[637, 696]). 4 Input Files, 206mb total:" - - "L1 " - - "L1.127[612,654] 1.05us 23mb |-----------L1.127------------| " - - "L1.128[655,699] 1.05us 24mb |------------L1.128------------| " + - "L1.122[673,699] 1.05us 14mb |---L1.122---| " + - "L1.123[700,799] 1.05us 53mb |-----------------------L1.123------------------------| " - "L2 " - - "L2.125[578,654] 1.05us 99mb|------------------------L2.125------------------------| " - - "L2.126[655,699] 1.05us 60mb |------------L2.126------------| " - - "**** 3 Output Files (parquet_file_id not yet assigned), 206mb total:" + - "L2.136[639,699] 1.05us 91mb|------------L2.136-------------| " + - "L2.8[700,799] 799ns 100mb |------------------------L2.8-------------------------| " + - "**** 3 Output Files (parquet_file_id not yet assigned), 258mb total:" - "L2 " - - "L2.?[578,637] 1.05us 100mb|------------------L2.?-------------------| " - - "L2.?[638,696] 1.05us 99mb |------------------L2.?-------------------| " - - "L2.?[697,699] 1.05us 7mb |L2.?|" + - "L2.?[639,702] 1.05us 101mb|--------------L2.?---------------| " + - "L2.?[703,765] 1.05us 100mb |--------------L2.?--------------| " + - "L2.?[766,799] 1.05us 56mb |------L2.?------| " - "Committing partition 1:" - - " Soft Deleting 4 files: L2.125, L2.126, L1.127, L1.128" + - " Soft Deleting 4 files: L2.8, L1.122, L1.123, L2.136" - " Creating 3 files" - - "**** Simulation run 27, type=split(CompactAndSplitOutput(FoundSubsetLessThanMaxCompactSize))(split_times=[765]). 3 Input Files, 153mb total:" + - "**** Simulation run 35, type=split(CompactAndSplitOutput(FoundSubsetLessThanMaxCompactSize))(split_times=[865]). 3 Input Files, 154mb total:" - "L1 " - - "L1.107[700,733] 1.05us 18mb|----------L1.107-----------| " - - "L1.108[734,799] 1.05us 35mb |-------------------------L1.108--------------------------| " + - "L1.124[800,861] 1.05us 34mb|-----------------------L1.124------------------------| " + - "L1.120[862,899] 1.05us 20mb |------------L1.120-------------| " - "L2 " - - "L2.8[700,799] 799ns 100mb|-----------------------------------------L2.8------------------------------------------| " - - "**** 2 Output Files (parquet_file_id not yet assigned), 153mb total:" + - "L2.9[800,899] 899ns 100mb|-----------------------------------------L2.9------------------------------------------| " + - "**** 2 Output Files (parquet_file_id not yet assigned), 154mb total:" - "L2 " - - "L2.?[700,765] 1.05us 100mb|--------------------------L2.?---------------------------| " - - "L2.?[766,799] 1.05us 53mb |-----------L2.?------------| " + - "L2.?[800,865] 1.05us 101mb|--------------------------L2.?---------------------------| " + - "L2.?[866,899] 1.05us 53mb |-----------L2.?------------| " - "Committing partition 1:" - - " Soft Deleting 3 files: L2.8, L1.107, L1.108" + - " Soft Deleting 3 files: L2.9, L1.120, L1.124" - " Creating 2 files" - - "**** Final Output Files (3.78gb written)" + - "**** Final Output Files (4.58gb written)" - "L1 " - - "L1.96[923,986] 1.05us 34mb |L1.96|" - - "L1.109[800,899] 1.05us 53mb |L1.109| " - - "L1.110[900,922] 1.05us 13mb |L1.110| " + - "L1.121[900,986] 1.05us 47mb |L1.121| " - "L2 " - - "L2.9[800,899] 899ns 100mb |-L2.9-| " - "L2.10[900,999] 999ns 100mb |L2.10-| " - - "L2.111[0,71] 1.05us 101mb|L2.111| " - - "L2.112[72,142] 1.05us 99mb |L2.112| " - - "L2.113[143,199] 1.05us 82mb |L2.113| " - - "L2.114[200,265] 1.05us 100mb |L2.114| " - - "L2.115[266,299] 1.05us 53mb |L2.115| " - - "L2.116[300,376] 1.05us 101mb |L2.116| " - - "L2.121[377,437] 1.05us 100mb |L2.121| " - - "L2.122[438,497] 1.05us 99mb |L2.122| " - - "L2.123[498,499] 1.05us 5mb |L2.123| " - - "L2.124[500,577] 1.05us 100mb |L2.124| " - - "L2.129[578,637] 1.05us 100mb |L2.129| " - - "L2.130[638,696] 1.05us 99mb |L2.130| " - - "L2.131[697,699] 1.05us 7mb |L2.131| " - - "L2.132[700,765] 1.05us 100mb |L2.132| " - - "L2.133[766,799] 1.05us 53mb |L2.133| " + - "L2.125[0,71] 1.05us 101mb|L2.125| " + - "L2.126[72,142] 1.05us 99mb |L2.126| " + - "L2.127[143,199] 1.05us 82mb |L2.127| " + - "L2.128[200,271] 1.05us 101mb |L2.128| " + - "L2.129[272,342] 1.05us 100mb |L2.129| " + - "L2.131[343,404] 1.05us 100mb |L2.131| " + - "L2.132[405,465] 1.05us 99mb |L2.132| " + - "L2.133[466,499] 1.05us 58mb |L2.133| " + - "L2.134[500,569] 1.05us 101mb |L2.134| " + - "L2.135[570,638] 1.05us 100mb |L2.135| " + - "L2.137[639,702] 1.05us 101mb |L2.137| " + - "L2.138[703,765] 1.05us 100mb |L2.138| " + - "L2.139[766,799] 1.05us 56mb |L2.139| " + - "L2.140[800,865] 1.05us 101mb |L2.140| " + - "L2.141[866,899] 1.05us 53mb |L2.141| " "### ); } @@ -2986,67 +3020,63 @@ async fn actual_case_from_catalog_1() { - "WARNING: file L0.161[327,333] 336ns 183mb exceeds soft limit 100mb by more than 50%" - "WARNING: file L0.162[330,338] 340ns 231mb exceeds soft limit 100mb by more than 50%" - "WARNING: file L0.163[331,338] 341ns 232mb exceeds soft limit 100mb by more than 50%" - - "**** Final Output Files (23.23gb written)" + - "**** Final Output Files (17.64gb written)" - "L2 " - - "L2.610[282,288] 342ns 100mb |L2.610| " - - "L2.619[314,314] 342ns 113mb |L2.619|" - - "L2.620[315,316] 342ns 107mb |L2.620|" - - "L2.621[317,317] 342ns 107mb |L2.621|" - - "L2.622[318,319] 342ns 111mb |L2.622|" - - "L2.623[320,320] 342ns 111mb |L2.623|" - - "L2.624[321,323] 342ns 146mb |L2.624|" - - "L2.625[324,325] 342ns 127mb |L2.625|" - - "L2.626[326,326] 342ns 127mb |L2.626|" - - "L2.627[327,329] 342ns 197mb |L2.627|" - - "L2.628[330,331] 342ns 114mb |L2.628|" - - "L2.629[332,332] 342ns 114mb |L2.629|" - - "L2.630[333,335] 342ns 199mb |L2.630|" - - "L2.631[336,337] 342ns 140mb |L2.631|" - - "L2.632[338,338] 342ns 140mb |L2.632|" - - "L2.918[1,24] 342ns 102mb |L2.918| " - - "L2.924[83,96] 342ns 107mb |L2.924| " - - "L2.930[143,151] 342ns 108mb |L2.930| " - - "L2.935[182,188] 342ns 119mb |L2.935| " - - "L2.938[205,213] 342ns 107mb |L2.938| " - - "L2.940[227,233] 342ns 114mb |L2.940| " - - "L2.946[261,267] 342ns 117mb |L2.946| " - - "L2.949[289,294] 342ns 124mb |L2.949| " - - "L2.955[311,313] 342ns 133mb |L2.955|" - - "L2.957[25,45] 342ns 105mb |L2.957| " - - "L2.958[46,65] 342ns 100mb |L2.958| " - - "L2.959[66,82] 342ns 94mb |L2.959| " - - "L2.960[97,107] 342ns 107mb |L2.960| " - - "L2.961[108,117] 342ns 96mb |L2.961| " - - "L2.962[118,124] 342ns 85mb |L2.962| " - - "L2.963[125,132] 342ns 115mb |L2.963| " - - "L2.964[133,139] 342ns 99mb |L2.964| " - - "L2.965[140,142] 342ns 66mb |L2.965| " - - "L2.966[152,159] 342ns 108mb |L2.966| " - - "L2.967[160,166] 342ns 92mb |L2.967| " - - "L2.968[167,171] 342ns 92mb |L2.968| " - - "L2.969[172,177] 342ns 121mb |L2.969| " - - "L2.970[178,181] 342ns 97mb |L2.970| " - - "L2.971[189,195] 342ns 113mb |L2.971| " - - "L2.972[196,201] 342ns 94mb |L2.972| " - - "L2.973[202,204] 342ns 75mb |L2.973| " - - "L2.974[214,220] 342ns 108mb |L2.974| " - - "L2.975[221,226] 342ns 108mb |L2.975| " - - "L2.976[234,240] 342ns 104mb |L2.976| " - - "L2.977[241,246] 342ns 86mb |L2.977| " - - "L2.978[247,248] 342ns 52mb |L2.978| " - - "L2.979[249,255] 342ns 102mb |L2.979| " - - "L2.980[256,260] 342ns 85mb |L2.980| " - - "L2.981[268,273] 342ns 111mb |L2.981| " - - "L2.982[274,278] 342ns 89mb |L2.982| " - - "L2.983[279,281] 342ns 89mb |L2.983| " - - "L2.984[295,299] 342ns 130mb |L2.984| " - - "L2.985[300,303] 342ns 98mb |L2.985| " - - "L2.986[304,304] 342ns 65mb |L2.986| " - - "L2.987[305,308] 342ns 137mb |L2.987| " - - "L2.988[309,310] 342ns 92mb |L2.988|" - - "L2.989[339,339] 342ns 25mb |L2.989|" - - "WARNING: file L2.627[327,329] 342ns 197mb exceeds soft limit 100mb by more than 50%" - - "WARNING: file L2.630[333,335] 342ns 199mb exceeds soft limit 100mb by more than 50%" + - "L2.578[134,149] 342ns 202mb |L2.578| " + - "L2.579[150,165] 342ns 218mb |L2.579| " + - "L2.580[166,176] 342ns 186mb |L2.580| " + - "L2.581[177,182] 342ns 150mb |L2.581| " + - "L2.582[183,197] 342ns 267mb |L2.582| " + - "L2.583[198,207] 342ns 157mb |L2.583| " + - "L2.584[208,220] 342ns 147mb |L2.584| " + - "L2.585[221,232] 342ns 270mb |L2.585| " + - "L2.588[233,253] 342ns 286mb |L2.588| " + - "L2.589[254,270] 342ns 289mb |L2.589| " + - "L2.590[271,281] 342ns 225mb |L2.590| " + - "L2.591[282,296] 342ns 234mb |L2.591| " + - "L2.592[297,302] 342ns 232mb |L2.592| " + - "L2.593[303,308] 342ns 244mb |L2.593| " + - "L2.594[309,314] 342ns 282mb |L2.594|" + - "L2.595[315,317] 342ns 214mb |L2.595|" + - "L2.596[318,320] 342ns 222mb |L2.596|" + - "L2.597[321,323] 342ns 146mb |L2.597|" + - "L2.598[324,326] 342ns 254mb |L2.598|" + - "L2.599[327,329] 342ns 197mb |L2.599|" + - "L2.600[330,332] 342ns 228mb |L2.600|" + - "L2.601[333,335] 342ns 199mb |L2.601|" + - "L2.602[336,338] 342ns 280mb |L2.602|" + - "L2.850[1,26] 342ns 101mb |L2.850| " + - "L2.853[69,85] 342ns 104mb |L2.853| " + - "L2.854[86,98] 342ns 107mb |L2.854| " + - "L2.861[27,48] 342ns 103mb |L2.861| " + - "L2.862[49,68] 342ns 98mb |L2.862| " + - "L2.863[99,108] 342ns 102mb |L2.863| " + - "L2.864[109,117] 342ns 91mb |L2.864| " + - "L2.865[118,124] 342ns 91mb |L2.865| " + - "L2.866[125,130] 342ns 107mb |L2.866| " + - "L2.867[131,133] 342ns 64mb |L2.867| " + - "L2.868[339,339] 342ns 25mb |L2.868|" + - "WARNING: file L2.578[134,149] 342ns 202mb exceeds soft limit 100mb by more than 50%" + - "WARNING: file L2.579[150,165] 342ns 218mb exceeds soft limit 100mb by more than 50%" + - "WARNING: file L2.580[166,176] 342ns 186mb exceeds soft limit 100mb by more than 50%" + - "WARNING: file L2.581[177,182] 342ns 150mb exceeds soft limit 100mb by more than 50%" + - "WARNING: file L2.582[183,197] 342ns 267mb exceeds soft limit 100mb by more than 50%" + - "WARNING: file L2.583[198,207] 342ns 157mb exceeds soft limit 100mb by more than 50%" + - "WARNING: file L2.585[221,232] 342ns 270mb exceeds soft limit 100mb by more than 50%" + - "WARNING: file L2.588[233,253] 342ns 286mb exceeds soft limit 100mb by more than 50%" + - "WARNING: file L2.589[254,270] 342ns 289mb exceeds soft limit 100mb by more than 50%" + - "WARNING: file L2.590[271,281] 342ns 225mb exceeds soft limit 100mb by more than 50%" + - "WARNING: file L2.591[282,296] 342ns 234mb exceeds soft limit 100mb by more than 50%" + - "WARNING: file L2.592[297,302] 342ns 232mb exceeds soft limit 100mb by more than 50%" + - "WARNING: file L2.593[303,308] 342ns 244mb exceeds soft limit 100mb by more than 50%" + - "WARNING: file L2.594[309,314] 342ns 282mb exceeds soft limit 100mb by more than 50%" + - "WARNING: file L2.595[315,317] 342ns 214mb exceeds soft limit 100mb by more than 50%" + - "WARNING: file L2.596[318,320] 342ns 222mb exceeds soft limit 100mb by more than 50%" + - "WARNING: file L2.598[324,326] 342ns 254mb exceeds soft limit 100mb by more than 50%" + - "WARNING: file L2.599[327,329] 342ns 197mb exceeds soft limit 100mb by more than 50%" + - "WARNING: file L2.600[330,332] 342ns 228mb exceeds soft limit 100mb by more than 50%" + - "WARNING: file L2.601[333,335] 342ns 199mb exceeds soft limit 100mb by more than 50%" + - "WARNING: file L2.602[336,338] 342ns 280mb exceeds soft limit 100mb by more than 50%" "### ); } diff --git a/compactor/tests/layouts/knobs.rs b/compactor/tests/layouts/knobs.rs index af2acfd548..66b28afd99 100644 --- a/compactor/tests/layouts/knobs.rs +++ b/compactor/tests/layouts/knobs.rs @@ -242,42 +242,39 @@ async fn all_overlapping_l0_max_num_files_per_plan() { - "L0.8[100,200000] 8ns |-----------------------------------------L0.8------------------------------------------| " - "L0.9[100,200000] 9ns |-----------------------------------------L0.9------------------------------------------| " - "L0.10[100,200000] 10ns |-----------------------------------------L0.10-----------------------------------------| " - - "**** Simulation run 0, type=split(CompactAndSplitOutput(ManySmallFiles))(split_times=[160020]). 3 Input Files, 27mb total:" + - "**** Simulation run 0, type=compact(ManySmallFiles). 3 Input Files, 27mb total:" - "L0, all files 9mb " - "L0.1[100,200000] 1ns |-----------------------------------------L0.1------------------------------------------| " - "L0.2[100,200000] 2ns |-----------------------------------------L0.2------------------------------------------| " - "L0.3[100,200000] 3ns |-----------------------------------------L0.3------------------------------------------| " - - "**** 2 Output Files (parquet_file_id not yet assigned), 27mb total:" - - "L0 " - - "L0.?[100,160020] 3ns 22mb|---------------------------------L0.?---------------------------------| " - - "L0.?[160021,200000] 3ns 5mb |-----L0.?------| " + - "**** 1 Output Files (parquet_file_id not yet assigned), 27mb total:" + - "L0, all files 27mb " + - "L0.?[100,200000] 3ns |-----------------------------------------L0.?------------------------------------------| " - "Committing partition 1:" - " Soft Deleting 3 files: L0.1, L0.2, L0.3" - - " Creating 2 files" - - "**** Simulation run 1, type=split(CompactAndSplitOutput(ManySmallFiles))(split_times=[160020]). 3 Input Files, 27mb total:" + - " Creating 1 files" + - "**** Simulation run 1, type=compact(ManySmallFiles). 3 Input Files, 27mb total:" - "L0, all files 9mb " - "L0.4[100,200000] 4ns |-----------------------------------------L0.4------------------------------------------| " - "L0.5[100,200000] 5ns |-----------------------------------------L0.5------------------------------------------| " - "L0.6[100,200000] 6ns |-----------------------------------------L0.6------------------------------------------| " - - "**** 2 Output Files (parquet_file_id not yet assigned), 27mb total:" - - "L0 " - - "L0.?[100,160020] 6ns 22mb|---------------------------------L0.?---------------------------------| " - - "L0.?[160021,200000] 6ns 5mb |-----L0.?------| " + - "**** 1 Output Files (parquet_file_id not yet assigned), 27mb total:" + - "L0, all files 27mb " + - "L0.?[100,200000] 6ns |-----------------------------------------L0.?------------------------------------------| " - "Committing partition 1:" - " Soft Deleting 3 files: L0.4, L0.5, L0.6" - - " Creating 2 files" - - "**** Simulation run 2, type=split(CompactAndSplitOutput(ManySmallFiles))(split_times=[160020]). 3 Input Files, 27mb total:" + - " Creating 1 files" + - "**** Simulation run 2, type=compact(ManySmallFiles). 3 Input Files, 27mb total:" - "L0, all files 9mb " - "L0.7[100,200000] 7ns |-----------------------------------------L0.7------------------------------------------| " - "L0.8[100,200000] 8ns |-----------------------------------------L0.8------------------------------------------| " - "L0.9[100,200000] 9ns |-----------------------------------------L0.9------------------------------------------| " - - "**** 2 Output Files (parquet_file_id not yet assigned), 27mb total:" - - "L0 " - - "L0.?[100,160020] 9ns 22mb|---------------------------------L0.?---------------------------------| " - - "L0.?[160021,200000] 9ns 5mb |-----L0.?------| " + - "**** 1 Output Files (parquet_file_id not yet assigned), 27mb total:" + - "L0, all files 27mb " + - "L0.?[100,200000] 9ns |-----------------------------------------L0.?------------------------------------------| " - "Committing partition 1:" - " Soft Deleting 3 files: L0.7, L0.8, L0.9" - - " Creating 2 files" + - " Creating 1 files" - "**** Simulation run 3, type=compact(ManySmallFiles). 1 Input Files, 9mb total:" - "L0, all files 9mb " - "L0.10[100,200000] 10ns |-----------------------------------------L0.10-----------------------------------------| " @@ -287,77 +284,41 @@ async fn all_overlapping_l0_max_num_files_per_plan() { - "Committing partition 1:" - " Soft Deleting 1 files: L0.10" - " Creating 1 files" - - "**** Simulation run 4, type=split(CompactAndSplitOutput(ManySmallFiles))(split_times=[160020]). 3 Input Files, 49mb total:" - - "L0 " - - "L0.11[100,160020] 3ns 22mb|--------------------------------L0.11---------------------------------| " - - "L0.12[160021,200000] 3ns 5mb |-----L0.12-----| " - - "L0.13[100,160020] 6ns 22mb|--------------------------------L0.13---------------------------------| " - - "**** 2 Output Files (parquet_file_id not yet assigned), 49mb total:" - - "L0 " - - "L0.?[100,160020] 6ns 39mb|---------------------------------L0.?---------------------------------| " - - "L0.?[160021,200000] 6ns 10mb |-----L0.?------| " + - "**** Simulation run 4, type=compact(ManySmallFiles). 3 Input Files, 81mb total:" + - "L0, all files 27mb " + - "L0.11[100,200000] 3ns |-----------------------------------------L0.11-----------------------------------------| " + - "L0.12[100,200000] 6ns |-----------------------------------------L0.12-----------------------------------------| " + - "L0.13[100,200000] 9ns |-----------------------------------------L0.13-----------------------------------------| " + - "**** 1 Output Files (parquet_file_id not yet assigned), 81mb total:" + - "L0, all files 81mb " + - "L0.?[100,200000] 9ns |-----------------------------------------L0.?------------------------------------------| " - "Committing partition 1:" - " Soft Deleting 3 files: L0.11, L0.12, L0.13" - - " Creating 2 files" - - "**** Simulation run 5, type=split(CompactAndSplitOutput(ManySmallFiles))(split_times=[160020]). 3 Input Files, 32mb total:" - - "L0 " - - "L0.14[160021,200000] 6ns 5mb |-----L0.14-----| " - - "L0.15[100,160020] 9ns 22mb|--------------------------------L0.15---------------------------------| " - - "L0.16[160021,200000] 9ns 5mb |-----L0.16-----| " - - "**** 2 Output Files (parquet_file_id not yet assigned), 32mb total:" - - "L0 " - - "L0.?[100,160020] 9ns 26mb|---------------------------------L0.?---------------------------------| " - - "L0.?[160021,200000] 9ns 6mb |-----L0.?------| " - - "Committing partition 1:" - - " Soft Deleting 3 files: L0.14, L0.15, L0.16" - - " Creating 2 files" - - "**** Simulation run 6, type=compact(ManySmallFiles). 1 Input Files, 9mb total:" + - " Creating 1 files" + - "**** Simulation run 5, type=compact(ManySmallFiles). 1 Input Files, 9mb total:" - "L0, all files 9mb " - - "L0.17[100,200000] 10ns |-----------------------------------------L0.17-----------------------------------------| " + - "L0.14[100,200000] 10ns |-----------------------------------------L0.14-----------------------------------------| " - "**** 1 Output Files (parquet_file_id not yet assigned), 9mb total:" - "L0, all files 9mb " - "L0.?[100,200000] 10ns |-----------------------------------------L0.?------------------------------------------| " - "Committing partition 1:" - - " Soft Deleting 1 files: L0.17" + - " Soft Deleting 1 files: L0.14" - " Creating 1 files" - - "**** Simulation run 7, type=split(CompactAndSplitOutput(ManySmallFiles))(split_times=[160020]). 3 Input Files, 75mb total:" + - "**** Simulation run 6, type=split(CompactAndSplitOutput(TotalSizeLessThanMaxCompactSize))(split_times=[160020]). 2 Input Files, 90mb total:" - "L0 " - - "L0.18[100,160020] 6ns 39mb|--------------------------------L0.18---------------------------------| " - - "L0.19[160021,200000] 6ns 10mb |-----L0.19-----| " - - "L0.20[100,160020] 9ns 26mb|--------------------------------L0.20---------------------------------| " - - "**** 2 Output Files (parquet_file_id not yet assigned), 75mb total:" - - "L0 " - - "L0.?[100,160020] 9ns 60mb|---------------------------------L0.?---------------------------------| " - - "L0.?[160021,200000] 9ns 15mb |-----L0.?------| " - - "Committing partition 1:" - - " Soft Deleting 3 files: L0.18, L0.19, L0.20" - - " Creating 2 files" - - "**** Simulation run 8, type=compact(ManySmallFiles). 2 Input Files, 15mb total:" - - "L0 " - - "L0.21[160021,200000] 9ns 6mb |-----L0.21-----| " - - "L0.22[100,200000] 10ns 9mb|-----------------------------------------L0.22-----------------------------------------| " - - "**** 1 Output Files (parquet_file_id not yet assigned), 15mb total:" - - "L0, all files 15mb " - - "L0.?[100,200000] 10ns |-----------------------------------------L0.?------------------------------------------| " - - "Committing partition 1:" - - " Soft Deleting 2 files: L0.21, L0.22" - - " Creating 1 files" - - "**** Simulation run 9, type=split(CompactAndSplitOutput(FoundSubsetLessThanMaxCompactSize))(split_times=[160020]). 3 Input Files, 90mb total:" - - "L0 " - - "L0.23[100,160020] 9ns 60mb|--------------------------------L0.23---------------------------------| " - - "L0.24[160021,200000] 9ns 15mb |-----L0.24-----| " - - "L0.25[100,200000] 10ns 15mb|-----------------------------------------L0.25-----------------------------------------| " + - "L0.16[100,200000] 10ns 9mb|-----------------------------------------L0.16-----------------------------------------| " + - "L0.15[100,200000] 9ns 81mb|-----------------------------------------L0.15-----------------------------------------| " - "**** 2 Output Files (parquet_file_id not yet assigned), 90mb total:" - "L1 " - "L1.?[100,160020] 10ns 72mb|---------------------------------L1.?---------------------------------| " - "L1.?[160021,200000] 10ns 18mb |-----L1.?------| " - "Committing partition 1:" - - " Soft Deleting 3 files: L0.23, L0.24, L0.25" + - " Soft Deleting 2 files: L0.15, L0.16" - " Creating 2 files" - - "**** Final Output Files (360mb written)" + - "**** Final Output Files (270mb written)" - "L1 " - - "L1.26[100,160020] 10ns 72mb|--------------------------------L1.26---------------------------------| " - - "L1.27[160021,200000] 10ns 18mb |-----L1.27-----| " + - "L1.17[100,160020] 10ns 72mb|--------------------------------L1.17---------------------------------| " + - "L1.18[160021,200000] 10ns 18mb |-----L1.18-----| " "### ); } diff --git a/compactor/tests/layouts/many_files.rs b/compactor/tests/layouts/many_files.rs index f2007bd920..1b78518077 100644 --- a/compactor/tests/layouts/many_files.rs +++ b/compactor/tests/layouts/many_files.rs @@ -4670,17 +4670,17 @@ async fn l0s_almost_needing_vertical_split() { - "L0.998[24,100] 1.02us |-----------------------------------------L0.998-----------------------------------------|" - "L0.999[24,100] 1.02us |-----------------------------------------L0.999-----------------------------------------|" - "L0.1000[24,100] 1.02us |----------------------------------------L0.1000-----------------------------------------|" - - "**** Final Output Files (6.75gb written)" + - "**** Final Output Files (6.5gb written)" - "L2 " - - "L2.3191[24,35] 1.02us 105mb|--L2.3191--| " - - "L2.3200[36,47] 1.02us 105mb |--L2.3200--| " - - "L2.3201[48,58] 1.02us 95mb |-L2.3201-| " - - "L2.3202[59,62] 1.02us 48mb |L2.3202| " - - "L2.3203[63,73] 1.02us 103mb |-L2.3203-| " - - "L2.3204[74,83] 1.02us 93mb |L2.3204-| " - - "L2.3205[84,85] 1.02us 31mb |L2.3205| " - - "L2.3206[86,95] 1.02us 110mb |L2.3206-| " - - "L2.3207[96,100] 1.02us 61mb |L2.3207|" + - "L2.3141[24,37] 1.02us 108mb|---L2.3141---| " + - "L2.3150[38,49] 1.02us 102mb |--L2.3150--| " + - "L2.3151[50,60] 1.02us 93mb |-L2.3151-| " + - "L2.3152[61,63] 1.02us 37mb |L2.3152| " + - "L2.3153[64,73] 1.02us 101mb |L2.3153-| " + - "L2.3154[74,82] 1.02us 90mb |L2.3154| " + - "L2.3155[83,90] 1.02us 101mb |L2.3155| " + - "L2.3156[91,98] 1.02us 93mb |L2.3156| " + - "L2.3157[99,100] 1.02us 26mb |L2.3157|" "### ); } diff --git a/compactor/tests/layouts/stuck.rs b/compactor/tests/layouts/stuck.rs index 36b67c655b..120e0237f7 100644 --- a/compactor/tests/layouts/stuck.rs +++ b/compactor/tests/layouts/stuck.rs @@ -1111,79 +1111,81 @@ async fn stuck_l0() { - "L2.59[1686863759000000000,1686867839000000000] 1686928811.43s 96mb |--L2.59--| " - "L2.74[1686867899000000000,1686868319000000000] 1686928811.43s 14mb |L2.74| " - "L2.78[1686868379000000000,1686873599000000000] 1686928118.43s 39mb |---L2.78----| " - - "**** Final Output Files (49.67gb written)" + - "**** Final Output Files (51.79gb written)" - "L1 " - - "L1.1867[1686873206669502294,1686873599000000000] 1686936871.55s 83mb |L1.1867|" + - "L1.1998[1686873236374845980,1686873480877117666] 1686936871.55s 48mb |L1.1998|" + - "L1.1999[1686873480877117667,1686873599000000000] 1686936871.55s 23mb |L1.1999|" - "L2 " - - "L2.1889[1686841379000000000,1686842394885830950] 1686936871.55s 100mb|L2.1889| " - - "L2.1890[1686842394885830951,1686842786390926609] 1686936871.55s 39mb |L2.1890| " - - "L2.1891[1686842786390926610,1686843316416264410] 1686936871.55s 100mb |L2.1891| " - - "L2.1896[1686843316416264411,1686843684225379958] 1686936871.55s 100mb |L2.1896| " - - "L2.1897[1686843684225379959,1686844052034495505] 1686936871.55s 100mb |L2.1897| " - - "L2.1898[1686844052034495506,1686844193781853218] 1686936871.55s 39mb |L2.1898| " - - "L2.1899[1686844193781853219,1686844848044941008] 1686936871.55s 100mb |L2.1899| " - - "L2.1904[1686844848044941009,1686845161518308591] 1686936871.55s 100mb |L2.1904| " - - "L2.1905[1686845161518308592,1686845474991676173] 1686936871.55s 100mb |L2.1905| " - - "L2.1906[1686845474991676174,1686845579000000000] 1686936871.55s 33mb |L2.1906| " - - "L2.1914[1686845579000000001,1686846586992441776] 1686936871.55s 100mb |L2.1914| " - - "L2.1919[1686846586992441777,1686847213091270336] 1686936871.55s 100mb |L2.1919| " - - "L2.1920[1686847213091270337,1686847594984883551] 1686936871.55s 61mb |L2.1920| " - - "L2.1921[1686847594984883552,1686847967133302591] 1686936871.55s 100mb |L2.1921| " - - "L2.1922[1686847967133302592,1686848339281721630] 1686936871.55s 100mb |L2.1922| " - - "L2.1923[1686848339281721631,1686848602977306099] 1686936871.55s 71mb |L2.1923| " - - "L2.1924[1686848602977306100,1686849027096194358] 1686936871.55s 100mb |L2.1924| " - - "L2.1925[1686849027096194359,1686849451215082616] 1686936871.55s 100mb |L2.1925| " - - "L2.1926[1686849451215082617,1686849779000000000] 1686936871.55s 77mb |L2.1926| " - - "L2.1934[1686849779000000001,1686851019138093591] 1686936871.55s 100mb |L2.1934| " - - "L2.1939[1686851019138093592,1686851845896821338] 1686936871.55s 100mb |L2.1939| " - - "L2.1944[1686851845896821339,1686852397069307378] 1686936871.55s 100mb |L2.1944| " - - "L2.1949[1686852397069307379,1686852764517630237] 1686936871.55s 100mb |L2.1949| " - - "L2.1950[1686852764517630238,1686853131965953095] 1686936871.55s 100mb |L2.1950| " - - "L2.1952[1686853131965953096,1686853754631187460] 1686936871.55s 100mb |L2.1952| " - - "L2.1958[1686853754631187461,1686854074019438293] 1686936871.55s 100mb |L2.1958| " - - "L2.1959[1686854074019438294,1686854377296421824] 1686936871.55s 95mb |L2.1959| " - - "L2.1960[1686854377296421825,1686854689009102312] 1686936871.55s 100mb |L2.1960| " - - "L2.1961[1686854689009102313,1686854840368603300] 1686936871.55s 49mb |L2.1961| " - - "L2.1969[1686854840368603301,1686855548468013502] 1686936871.55s 100mb |L2.1969| " - - "L2.1970[1686855548468013503,1686856256567423703] 1686936871.55s 100mb |L2.1970| " - - "L2.1971[1686856256567423704,1686856536982788349] 1686936871.55s 40mb |L2.1971| " - - "L2.1972[1686856536982788350,1686857111714340888] 1686936871.55s 100mb |L2.1972| " - - "L2.1977[1686857111714340889,1686857492337275263] 1686936871.55s 100mb |L2.1977| " - - "L2.1978[1686857492337275264,1686857872960209637] 1686936871.55s 100mb |L2.1978| " - - "L2.1979[1686857872960209638,1686858233596973397] 1686936871.55s 95mb |L2.1979| " - - "L2.1980[1686858233596973398,1686858684102523465] 1686936871.55s 100mb |L2.1980| " - - "L2.1981[1686858684102523466,1686859134608073532] 1686936871.55s 100mb |L2.1981| " - - "L2.1982[1686859134608073533,1686859499000000000] 1686936871.55s 81mb |L2.1982| " - - "L2.1990[1686859499000000001,1686860134536077779] 1686936871.55s 100mb |L2.1990| " - - "L2.1991[1686860134536077780,1686860770072155557] 1686936871.55s 100mb |L2.1991| " - - "L2.1993[1686860770072155558,1686861617453595289] 1686936871.55s 100mb |L2.1993| " - - "L2.1998[1686861617453595290,1686862007484245554] 1686936871.55s 100mb |L2.1998| " - - "L2.1999[1686862007484245555,1686862397514895818] 1686936871.55s 100mb |L2.1999| " - - "L2.2000[1686862397514895819,1686862464835035020] 1686936871.55s 17mb |L2.2000| " - - "L2.2001[1686862464835035021,1686862789833508891] 1686936871.55s 100mb |L2.2001| " - - "L2.2002[1686862789833508892,1686863114831982761] 1686936871.55s 100mb |L2.2002| " - - "L2.2003[1686863114831982762,1686863312216466667] 1686936871.55s 61mb |L2.2003| " - - "L2.2011[1686863312216466668,1686864075006108740] 1686936871.55s 100mb |L2.2011| " - - "L2.2012[1686864075006108741,1686864837795750812] 1686936871.55s 100mb |L2.2012| " - - "L2.2013[1686864837795750813,1686864893143269255] 1686936871.55s 7mb |L2.2013| " - - "L2.2014[1686864893143269256,1686865484385600871] 1686936871.55s 100mb |L2.2014| " - - "L2.2015[1686865484385600872,1686866075627932486] 1686936871.55s 100mb |L2.2015| " - - "L2.2024[1686866075627932487,1686866607164120163] 1686936871.55s 100mb |L2.2024| " - - "L2.2025[1686866607164120164,1686867138700307839] 1686936871.55s 100mb |L2.2025| " - - "L2.2027[1686867138700307840,1686867545868269737] 1686936871.55s 100mb |L2.2027| " - - "L2.2028[1686867545868269738,1686867953036231634] 1686936871.55s 100mb |L2.2028| " - - "L2.2029[1686867953036231635,1686868319000000000] 1686936871.55s 90mb |L2.2029| " - - "L2.2037[1686868319000000001,1686869079291584458] 1686936871.55s 100mb |L2.2037| " - - "L2.2038[1686869079291584459,1686869839583168915] 1686936871.55s 100mb |L2.2038| " - - "L2.2039[1686869839583168916,1686870526488100069] 1686936871.55s 90mb |L2.2039| " - - "L2.2047[1686870526488100070,1686871158741704859] 1686936871.55s 100mb |L2.2047|" - - "L2.2048[1686871158741704860,1686871790995309648] 1686936871.55s 100mb |L2.2048|" - - "L2.2050[1686871790995309649,1686872061589130317] 1686936871.55s 100mb |L2.2050|" - - "L2.2051[1686872061589130318,1686872332182950985] 1686936871.55s 100mb |L2.2051|" - - "L2.2052[1686872332182950986,1686872423248908406] 1686936871.55s 34mb |L2.2052|" - - "L2.2053[1686872423248908407,1686872916648819217] 1686936871.55s 100mb |L2.2053|" - - "L2.2054[1686872916648819218,1686873410048730027] 1686936871.55s 100mb |L2.2054|" - - "L2.2055[1686873410048730028,1686873599000000000] 1686936871.55s 38mb |L2.2055|" + - "L2.1842[1686841379000000000,1686842394885830950] 1686936871.55s 100mb|L2.1842| " + - "L2.1843[1686842394885830951,1686842786390926609] 1686936871.55s 39mb |L2.1843| " + - "L2.1844[1686842786390926610,1686843316416262506] 1686936871.55s 100mb |L2.1844| " + - "L2.1849[1686843316416262507,1686843684225377382] 1686936871.55s 100mb |L2.1849| " + - "L2.1850[1686843684225377383,1686844052034492257] 1686936871.55s 100mb |L2.1850| " + - "L2.1851[1686844052034492258,1686844193781853218] 1686936871.55s 39mb |L2.1851| " + - "L2.1852[1686844193781853219,1686844848044943955] 1686936871.55s 100mb |L2.1852| " + - "L2.1857[1686844848044943956,1686845161518311556] 1686936871.55s 100mb |L2.1857| " + - "L2.1858[1686845161518311557,1686845474991679156] 1686936871.55s 100mb |L2.1858| " + - "L2.1859[1686845474991679157,1686845579000000000] 1686936871.55s 33mb |L2.1859| " + - "L2.1867[1686845579000000001,1686846586992441776] 1686936871.55s 100mb |L2.1867| " + - "L2.1872[1686846586992441777,1686847213091266628] 1686936871.55s 100mb |L2.1872| " + - "L2.1873[1686847213091266629,1686847594984883551] 1686936871.55s 61mb |L2.1873| " + - "L2.1874[1686847594984883552,1686847967133303901] 1686936871.55s 100mb |L2.1874| " + - "L2.1875[1686847967133303902,1686848339281724250] 1686936871.55s 100mb |L2.1875| " + - "L2.1876[1686848339281724251,1686848602977306099] 1686936871.55s 71mb |L2.1876| " + - "L2.1877[1686848602977306100,1686849027096192900] 1686936871.55s 100mb |L2.1877| " + - "L2.1878[1686849027096192901,1686849451215079700] 1686936871.55s 100mb |L2.1878| " + - "L2.1879[1686849451215079701,1686849779000000000] 1686936871.55s 77mb |L2.1879| " + - "L2.1887[1686849779000000001,1686850474198233689] 1686936871.55s 100mb |L2.1887| " + - "L2.1888[1686850474198233690,1686851169396467377] 1686936871.55s 100mb |L2.1888| " + - "L2.1889[1686851169396467378,1686851644365067698] 1686936871.55s 68mb |L2.1889| " + - "L2.1890[1686851644365067699,1686852420659097557] 1686936871.55s 100mb |L2.1890| " + - "L2.1895[1686852420659097558,1686852801563322990] 1686936871.55s 100mb |L2.1895| " + - "L2.1896[1686852801563322991,1686853182467548422] 1686936871.55s 100mb |L2.1896| " + - "L2.1897[1686853182467548423,1686853509730135395] 1686936871.55s 86mb |L2.1897| " + - "L2.1898[1686853509730135396,1686853969686755621] 1686936871.55s 100mb |L2.1898| " + - "L2.1899[1686853969686755622,1686854429643375846] 1686936871.55s 100mb |L2.1899| " + - "L2.1900[1686854429643375847,1686854819000000000] 1686936871.55s 85mb |L2.1900| " + - "L2.1908[1686854819000000001,1686855825796452886] 1686936871.55s 100mb |L2.1908| " + - "L2.1913[1686855825796452887,1686856447395293366] 1686936871.55s 100mb |L2.1913| " + - "L2.1914[1686856447395293367,1686856832592905771] 1686936871.55s 62mb |L2.1914| " + - "L2.1915[1686856832592905772,1686857205112223577] 1686936871.55s 100mb |L2.1915| " + - "L2.1916[1686857205112223578,1686857577631541382] 1686936871.55s 100mb |L2.1916| " + - "L2.1917[1686857577631541383,1686857839389339451] 1686936871.55s 70mb |L2.1917| " + - "L2.1918[1686857839389339452,1686858253462843732] 1686936871.55s 100mb |L2.1918| " + - "L2.1919[1686858253462843733,1686858667536348012] 1686936871.55s 100mb |L2.1919| " + - "L2.1920[1686858667536348013,1686859019000000000] 1686936871.55s 85mb |L2.1920| " + - "L2.1928[1686859019000000001,1686859728979165803] 1686936871.55s 100mb |L2.1928| " + - "L2.1929[1686859728979165804,1686860438958331605] 1686936871.55s 100mb |L2.1929| " + - "L2.1931[1686860438958331606,1686860836834049919] 1686936871.55s 100mb |L2.1931| " + - "L2.1932[1686860836834049920,1686861148937490634] 1686936871.55s 78mb |L2.1932| " + - "L2.1933[1686861148937490635,1686861914356348643] 1686936871.55s 100mb |L2.1933| " + - "L2.1938[1686861914356348644,1686862293494638783] 1686936871.55s 100mb |L2.1938| " + - "L2.1939[1686862293494638784,1686862672632928922] 1686936871.55s 100mb |L2.1939| " + - "L2.1940[1686862672632928923,1686862679775206651] 1686936871.55s 2mb |L2.1940| " + - "L2.1941[1686862679775206652,1686863077953136510] 1686936871.55s 100mb |L2.1941| " + - "L2.1942[1686863077953136511,1686863476131066368] 1686936871.55s 100mb |L2.1942| " + - "L2.1943[1686863476131066369,1686863699000000000] 1686936871.55s 56mb |L2.1943| " + - "L2.1951[1686863699000000001,1686864349392123893] 1686936871.55s 100mb |L2.1951| " + - "L2.1952[1686864349392123894,1686864999784247785] 1686936871.55s 100mb |L2.1952| " + - "L2.1953[1686864999784247786,1686865607586435465] 1686936871.55s 93mb |L2.1953| " + - "L2.1954[1686865607586435466,1686866298103608885] 1686936871.55s 100mb |L2.1954| " + - "L2.1959[1686866298103608886,1686866809270889026] 1686936871.55s 100mb |L2.1959| " + - "L2.1964[1686866809270889027,1686867189314140837] 1686936871.55s 100mb |L2.1964| " + - "L2.1965[1686867189314140838,1686867569357392647] 1686936871.55s 100mb |L2.1965| " + - "L2.1966[1686867569357392648,1686867839000000000] 1686936871.55s 71mb |L2.1966| " + - "L2.1974[1686867839000000001,1686868754092704252] 1686936871.55s 100mb |L2.1974| " + - "L2.1975[1686868754092704253,1686869669185408503] 1686936871.55s 100mb |L2.1975| " + - "L2.1976[1686869669185408504,1686869927156879962] 1686936871.55s 28mb |L2.1976| " + - "L2.1984[1686869927156879963,1686870583552763572] 1686936871.55s 100mb |L2.1984| " + - "L2.1989[1686870583552763573,1686870997054710512] 1686936871.55s 100mb |L2.1989|" + - "L2.1990[1686870997054710513,1686871410556657451] 1686936871.55s 100mb |L2.1990|" + - "L2.1992[1686871410556657452,1686871702557232840] 1686936871.55s 100mb |L2.1992|" + - "L2.1993[1686871702557232841,1686871994557808228] 1686936871.55s 100mb |L2.1993|" + - "L2.1994[1686871994557808229,1686872015313759923] 1686936871.55s 7mb |L2.1994|" + - "L2.1997[1686873480877117667,1686873599000000000] 1686936871.55s 16mb |L2.1997|" + - "L2.2002[1686872015313759924,1686872503834880725] 1686936871.55s 100mb |L2.2002|" + - "L2.2003[1686872503834880726,1686872992356001526] 1686936871.55s 100mb |L2.2003|" + - "L2.2004[1686872992356001527,1686873480877117666] 1686936871.55s 100mb |L2.2004|" "### ); } @@ -1700,33 +1702,31 @@ async fn stuck_l0_large_l0s() { - "L0.198[197,1970000] 197ns 10b|----------------------------------------L0.198-----------------------------------------| " - "L0.199[198,1980000] 198ns 10b|----------------------------------------L0.199-----------------------------------------| " - "L0.200[199,1990000] 199ns 10b|----------------------------------------L0.200-----------------------------------------| " - - "**** Final Output Files (14.37gb written)" + - "**** Final Output Files (14.48gb written)" - "L1 " - - "L1.3823[1047695,1801539] 199ns 72mb |------------L1.3823-------------| " - - "L1.3824[1801540,1990000] 199ns 18mb |L1.3824|" + - "L1.3770[1752,1811] 199ns 62mb|L1.3770| " - "L2 " - - "L2.3801[1,102] 199ns 100mb|L2.3801| " - - "L2.3802[103,203] 199ns 100mb|L2.3802| " - - "L2.3803[204,305] 199ns 100mb|L2.3803| " - - "L2.3820[1553,1652] 199ns 101mb|L2.3820| " - - "L2.3825[306,407] 199ns 101mb|L2.3825| " - - "L2.3826[408,508] 199ns 100mb|L2.3826| " - - "L2.3827[509,513] 199ns 6mb|L2.3827| " - - "L2.3828[514,614] 199ns 101mb|L2.3828| " - - "L2.3829[615,714] 199ns 100mb|L2.3829| " - - "L2.3830[715,719] 199ns 6mb|L2.3830| " - - "L2.3831[720,820] 199ns 101mb|L2.3831| " - - "L2.3832[821,920] 199ns 100mb|L2.3832| " - - "L2.3833[921,1017] 199ns 99mb|L2.3833| " - - "L2.3834[1018,1118] 199ns 100mb|L2.3834| " - - "L2.3835[1119,1218] 199ns 99mb|L2.3835| " - - "L2.3836[1219,1310] 199ns 93mb|L2.3836| " - - "L2.3837[1311,1411] 199ns 100mb|L2.3837| " - - "L2.3838[1412,1511] 199ns 99mb|L2.3838| " - - "L2.3839[1512,1552] 199ns 42mb|L2.3839| " - - "L2.3840[1653,400658] 199ns 100mb|----L2.3840-----| " - - "L2.3841[400659,799663] 199ns 100mb |----L2.3841-----| " - - "L2.3842[799664,1047694] 199ns 62mb |-L2.3842-| " + - "L2.3711[1812,1990000] 199ns 190mb|----------------------------------------L2.3711----------------------------------------| " + - "L2.3743[1,102] 199ns 100mb|L2.3743| " + - "L2.3744[103,203] 199ns 100mb|L2.3744| " + - "L2.3745[204,305] 199ns 101mb|L2.3745| " + - "L2.3763[1165,1255] 199ns 101mb|L2.3763| " + - "L2.3771[306,407] 199ns 101mb|L2.3771| " + - "L2.3772[408,508] 199ns 100mb|L2.3772| " + - "L2.3773[509,606] 199ns 99mb|L2.3773| " + - "L2.3774[607,711] 199ns 101mb|L2.3774| " + - "L2.3775[712,815] 199ns 100mb|L2.3775| " + - "L2.3776[816,904] 199ns 87mb|L2.3776| " + - "L2.3777[905,1011] 199ns 101mb|L2.3777| " + - "L2.3778[1012,1117] 199ns 100mb|L2.3778| " + - "L2.3779[1118,1164] 199ns 46mb|L2.3779| " + - "L2.3780[1256,1350] 199ns 100mb|L2.3780| " + - "L2.3781[1351,1444] 199ns 99mb|L2.3781| " + - "L2.3782[1445,1480] 199ns 40mb|L2.3782| " + - "L2.3783[1481,1580] 199ns 100mb|L2.3783| " + - "L2.3784[1581,1679] 199ns 99mb|L2.3784| " + - "L2.3785[1680,1751] 199ns 74mb|L2.3785| " + - "WARNING: file L2.3711[1812,1990000] 199ns 190mb exceeds soft limit 100mb by more than 50%" "### ); }